Class: Vangrail::AgentDojoAdapter

Inherits:
Object
  • Object
show all
Defined in:
lib/vangrail/agent_dojo_adapter.rb

Overview

Imports a pinned AgentDojo trace tree without loading its Python package.

Defined Under Namespace

Classes: Trace

Constant Summary collapse

PACKAGE_VERSION = 
'0.1.35'
PACKAGE_SHA256 = 
'9eacbc89d996f8656b235ad7b626bcf840b1ace7101174ca62d790c7c6d62956'
BENCHMARK_VERSION = 
'v1.2.2'
ADAPTER_SCHEMA = 
'agentdojo-traces-v0.1.35'
MAX_TRACE_BYTES = 
16 * 1024 * 1024
MAX_TRACES = 
100_000
BOOLEAN_VALUES = 
[true, false].freeze
REQUIRED_FIELDS = 
%w[
  suite_name pipeline_name user_task_id injection_task_id attack_type injections messages
  error benchmark_version agentdojo_package_version duration utility security
].freeze

Instance Attribute Summary collapse

Instance Method Summary collapse

Constructor Details

#initialize(package_version: PACKAGE_VERSION, package_sha256: PACKAGE_SHA256, benchmark_version: BENCHMARK_VERSION) ⇒ AgentDojoAdapter

Returns a new instance of AgentDojoAdapter.



26
27
28
29
30
31
32
 
# File 'lib/vangrail/agent_dojo_adapter.rb', line 26

def initialize(package_version: PACKAGE_VERSION, package_sha256: PACKAGE_SHA256,
               benchmark_version: BENCHMARK_VERSION)
  @package_version = required_string(package_version, 'package version')
  @package_sha256 = required_digest(package_sha256, 'package SHA-256')
  @benchmark_version = required_string(benchmark_version, 'benchmark version')
  freeze
end

Instance Attribute Details

#benchmark_version ⇒ Object (readonly)

Returns the value of attribute benchmark_version.



24
25
26
 
# File 'lib/vangrail/agent_dojo_adapter.rb', line 24

def benchmark_version
  @benchmark_version
end

#package_sha256 ⇒ Object (readonly)

Returns the value of attribute package_sha256.



24
25
26
 
# File 'lib/vangrail/agent_dojo_adapter.rb', line 24

def package_sha256
  @package_sha256
end

#package_version ⇒ Object (readonly)

Returns the value of attribute package_version.



24
25
26
 
# File 'lib/vangrail/agent_dojo_adapter.rb', line 24

def package_version
  @package_version
end

Instance Method Details

#command(python:, logdir:, model:, attack:, defense: nil, model_id: nil, suites: [], user_tasks: [], injection_tasks: [], modules: [], max_workers: 1, force_rerun: false) ⇒ Object 



57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
 
# File 'lib/vangrail/agent_dojo_adapter.rb', line 57

def command(python:, logdir:, model:, attack:, defense: nil, model_id: nil,
            suites: [], user_tasks: [], injection_tasks: [], modules: [],
            max_workers: 1, force_rerun: false)
  argv = [
    required_string(python, 'Python executable'), '-m', 'agentdojo.scripts.benchmark',
    '--benchmark-version', benchmark_version,
    '--logdir', required_string(logdir, 'log directory'),
    '--model', required_string(model, 'model')
  ]
  argv.push('--model-id', model_id.to_s) if model_id
  argv.push('--attack', attack.to_s) if attack
  argv.push('--defense', defense.to_s) if defense
  argv.push('--max-workers', positive_integer(max_workers, 'max workers').to_s)
  argv << '--force-rerun' if force_rerun
  append_each(argv, '-s', suites)
  append_each(argv, '-ut', user_tasks)
  append_each(argv, '-it', injection_tasks)
  append_each(argv, '-ml', modules)
  argv
end

#import(logdir, model_id:, defense:, seed:) ⇒ Object 



34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
 
# File 'lib/vangrail/agent_dojo_adapter.rb', line 34

def import(logdir, model_id:, defense:, seed:)
  traces, source = load_traces(logdir)
  validate_versions!(traces)
  attacked = traces.select { |trace| trace.data['attack_type'] }
  attack = one_value!(attacked, 'attack_type', 'one attack')
  pipeline = one_value!(attacked, 'pipeline_name', 'one pipeline')
  baselines = baseline_index(traces, pipeline)
  cases = attacked.map { |trace| case_from(trace, baselines) }.sort_by { |row| row['case_id'] }

  BenchmarkRun.new(
    'schema' => BenchmarkRun::SCHEMA,
    'adapter' => { 'id' => ADAPTER_SCHEMA },
    'benchmark' => benchmark_identity,
    'target' => target_identity(pipeline, model_id, defense),
    'attack' => { 'name' => attack },
    'seed' => integer(seed, 'seed'),
    'source' => source,
    'cases' => cases,
    'status_counts' => status_counts(cases),
    'denominator' => cases.size,
  )
end