Class: Vangrail::AdaptiveBenchmarkAdapter

Inherits:
Object
  • Object
show all
Includes:
ArtifactData
Defined in:
lib/vangrail/adaptive_benchmark.rb

Overview

Executes a matrix against a target through a versioned optional runner.

Constant Summary collapse

TARGET_FIELDS = 
%w[id model_id environment_id defense].freeze
RESPONSE_FIELDS = 
%w[
  schema case_id target_id adversary status reason utility_without_attack utility_under_attack
  security_success attack_success score queries model_calls tool_calls duration_seconds trace_sha256
].freeze
SUCCESS_FIELDS = 
%w[
  utility_without_attack utility_under_attack security_success attack_success score
].freeze
BOOLEAN_VALUES = 
[true, false].freeze
DIGEST = 
/\A[0-9a-f]{64}\z/

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(matrix:, runner:) ⇒ AdaptiveBenchmarkAdapter

Returns a new instance of AdaptiveBenchmarkAdapter.

Raises:

  • (ArgumentError) 


57
58
59
60
61
62
63
64
 
# File 'lib/vangrail/adaptive_benchmark.rb', line 57

def initialize(matrix:, runner:)
  raise ArgumentError, 'matrix must be an AdaptiveAttackMatrix' unless matrix.is_a?(AdaptiveAttackMatrix)
  raise ArgumentError, 'runner must respond to run' unless runner.respond_to?(:run)

  @matrix = matrix
  @runner = runner
  freeze
end

Instance Attribute Details

#matrix ⇒ Object (readonly)

Returns the value of attribute matrix.



55
56
57
 
# File 'lib/vangrail/adaptive_benchmark.rb', line 55

def matrix
  @matrix
end

#runner ⇒ Object (readonly)

Returns the value of attribute runner.



55
56
57
 
# File 'lib/vangrail/adaptive_benchmark.rb', line 55

def runner
  @runner
end

Instance Method Details

#run(target:, seed:) ⇒ Object 



66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
 
# File 'lib/vangrail/adaptive_benchmark.rb', line 66

def run(target:, seed:)
  normalized_target = normalize_target(target)
  run_seed = integer(seed, 'benchmark seed')
  cases = matrix.to_h.fetch('cases').map do |attack_case|
    execute_case(attack_case, normalized_target, run_seed)
  end
  cases.sort_by! { |row| row['case_id'] }

  BenchmarkRun.new(
    'schema' => BenchmarkRun::SCHEMA,
    'adapter' => { 'id' => 'adaptive-runner-v1' },
    'benchmark' => { 'name' => 'adaptive-security', 'version' => matrix.version },
    'target' => normalized_target,
    'attack' => {
      'matrix_id' => matrix.id,
      'matrix_version' => matrix.version,
      'matrix_sha256' => matrix.sha256,
    },
    'seed' => run_seed,
    'source' => {
      'schema' => AdaptiveAttackMatrix::SCHEMA,
      'id' => matrix.id,
      'sha256' => matrix.sha256,
    },
    'cases' => cases,
    'status_counts' => status_counts(cases),
    'denominator' => cases.size,
  )
end