Class: Vangrail::StatisticalVerifier

Inherits:
Object
  • Object
show all
Includes:
ArtifactData, EvaluationStatistics, PairedStatistics, StatisticalValidation
Defined in:
lib/vangrail/statistical_verifier.rb

Overview

Recomputes evaluation results only after split and artifact invariants hold.

Constant Summary collapse

SCHEMA = 
'vangrail-statistical-verification-v1'

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(predictions:, threshold:, artifacts:, bootstrap_seed:, bootstrap_replicates:) ⇒ StatisticalVerifier

Returns a new instance of StatisticalVerifier.



22
23
24
25
26
27
28
29
 
# File 'lib/vangrail/statistical_verifier.rb', line 22

def initialize(predictions:, threshold:, artifacts:, bootstrap_seed:,
               bootstrap_replicates:)
  @predictions = stringify(predictions).sort_by { |row| row['case_id'].to_s }
  @threshold = stringify(threshold)
  @artifacts = stringify(artifacts)
  @bootstrap_seed = integer(bootstrap_seed, 'bootstrap seed')
  @bootstrap_replicates = positive_integer(bootstrap_replicates, 'bootstrap replicates')
end

Instance Attribute Details

#artifacts ⇒ Object (readonly)

Returns the value of attribute artifacts.



19
20
21
 
# File 'lib/vangrail/statistical_verifier.rb', line 19

def artifacts
  @artifacts
end

#bootstrap_replicates ⇒ Object (readonly)

Returns the value of attribute bootstrap_replicates.



19
20
21
 
# File 'lib/vangrail/statistical_verifier.rb', line 19

def bootstrap_replicates
  @bootstrap_replicates
end

#bootstrap_seed ⇒ Object (readonly)

Returns the value of attribute bootstrap_seed.



19
20
21
 
# File 'lib/vangrail/statistical_verifier.rb', line 19

def bootstrap_seed
  @bootstrap_seed
end

#predictions ⇒ Object (readonly)

Returns the value of attribute predictions.



19
20
21
 
# File 'lib/vangrail/statistical_verifier.rb', line 19

def predictions
  @predictions
end

#threshold ⇒ Object (readonly)

Returns the value of attribute threshold.



19
20
21
 
# File 'lib/vangrail/statistical_verifier.rb', line 19

def threshold
  @threshold
end

Instance Method Details

#compare(baseline:, candidate:, metrics:) ⇒ Object 



47
48
49
 
# File 'lib/vangrail/statistical_verifier.rb', line 47

def compare(baseline:, candidate:, metrics:)
  paired_comparison(baseline, candidate, metrics)
end

#verify ⇒ Object 



31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
 
# File 'lib/vangrail/statistical_verifier.rb', line 31

def verify
  validate_inputs!
  test_rows = predictions.select { |row| row['role'] == 'test' }
  report = {
    'schema' => SCHEMA,
    'roles' => role_summary,
    'threshold' => threshold_summary,
    'artifacts' => @verified_artifacts,
    'missing_scores' => {
      'policy' => 'worst_case_by_label',
      'count' => test_rows.count { |row| row['status'] != 'ok' || row['score'].nil? },
    },
  }
  report.merge(statistics(test_rows))
end