From 6655051670f041d90f1484b13b763b81127ae457 Mon Sep 17 00:00:00 2001 From: Victor Valbuena Date: Tue, 2 Jun 2026 13:14:28 -0700 Subject: [PATCH 1/5] FIX: Added non-None labels to MessagePiece constructor in mocks.py labels required to be non-None in order for MessagePiece not to raise and interrupt test_retry_timing_integration. Fix was to apply labels or {} on line 72 of tests/integration/mocks.py. --- tests/integration/mocks.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/mocks.py b/tests/integration/mocks.py index 58abb63970..512a4b5986 100644 --- a/tests/integration/mocks.py +++ b/tests/integration/mocks.py @@ -69,7 +69,7 @@ def set_system_prompt( converted_value=system_prompt, conversation_id=conversation_id, attack_identifier=attack_identifier, - labels=labels, + labels=labels or {}, ).to_message() ) From b269dc7a102c6b9998449f94b70c4367546c4e4d Mon Sep 17 00:00:00 2001 From: Victor Valbuena Date: Tue, 2 Jun 2026 16:06:26 -0700 Subject: [PATCH 2/5] FIX: populate AttackTechniqueRegistry before LoadDefaultDatasets in integration test MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ScenarioRegistry.list_metadata() instantiates every registered scenario class to build metadata. Cyber.__init__ calls _build_cyber_strategy(), which calls AttackTechniqueRegistry.get_factories_or_raise() — raising RuntimeError when the registry is empty. The integration test was missing the ScenarioTechniqueInitializer step that populates the registry. Add it before LoadDefaultDatasets.initialize_async(). Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- .../datasets/test_load_default_datasets_integration.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/integration/datasets/test_load_default_datasets_integration.py b/tests/integration/datasets/test_load_default_datasets_integration.py index 878b707905..dd7b3e346c 100644 --- a/tests/integration/datasets/test_load_default_datasets_integration.py +++ b/tests/integration/datasets/test_load_default_datasets_integration.py @@ -11,6 +11,7 @@ import logging from pyrit.memory import CentralMemory +from pyrit.setup.initializers.components.scenario_techniques import ScenarioTechniqueInitializer from pyrit.setup.initializers.scenarios.load_default_datasets import LoadDefaultDatasets logger = logging.getLogger(__name__) @@ -25,6 +26,7 @@ async def test_initialize_loads_datasets_into_memory(self, sqlite_instance): real datasets and stores them in CentralMemory. """ initializer = LoadDefaultDatasets() + await ScenarioTechniqueInitializer().initialize_async() await initializer.initialize_async() memory = CentralMemory.get_memory_instance() From d8a363aab7008855e9266a76dc466b5bed46bdd3 Mon Sep 17 00:00:00 2001 From: Victor Valbuena Date: Tue, 2 Jun 2026 16:25:19 -0700 Subject: [PATCH 3/5] FIX: Correct HarmBench size assertion fron {'large'} to {'medium'} (last changed in harmbench_dataset in PR 1780 --- .../datasets/test_seed_dataset_provider_integration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration/datasets/test_seed_dataset_provider_integration.py b/tests/integration/datasets/test_seed_dataset_provider_integration.py index 2f2f70be8a..aaafe2900d 100644 --- a/tests/integration/datasets/test_seed_dataset_provider_integration.py +++ b/tests/integration/datasets/test_seed_dataset_provider_integration.py @@ -611,7 +611,7 @@ async def test_harmbench_metadata_parses_correctly(self): assert isinstance(metadata.tags, set) assert "default" in metadata.tags assert "safety" in metadata.tags - assert metadata.size == {"large"} + assert metadata.size == {"medium"} assert metadata.modalities == {"text"} assert isinstance(metadata.harm_categories, set) assert "cybercrime" in metadata.harm_categories From 58ea965f6d4a0fe0a29472adec9dc170b885fd77 Mon Sep 17 00:00:00 2001 From: Victor Valbuena Date: Tue, 2 Jun 2026 17:01:18 -0700 Subject: [PATCH 4/5] FIX: Added ScenarioTechniqueInitializer to scenario notebooks to fix attack registry failures. --- doc/code/scenarios/0_scenarios.ipynb | 388 +++++- doc/code/scenarios/0_scenarios.py | 2 + .../1_common_scenario_parameters.ipynb | 1071 +++++++++++++++-- .../scenarios/1_common_scenario_parameters.py | 5 +- .../2_custom_scenario_parameters.ipynb | 201 +++- .../scenarios/2_custom_scenario_parameters.py | 6 + 6 files changed, 1538 insertions(+), 135 deletions(-) diff --git a/doc/code/scenarios/0_scenarios.ipynb b/doc/code/scenarios/0_scenarios.ipynb index 407dcea2cc..3406a60943 100644 --- a/doc/code/scenarios/0_scenarios.ipynb +++ b/doc/code/scenarios/0_scenarios.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "0", + "id": "844edc95", "metadata": { "lines_to_next_cell": 0 }, @@ -94,11 +94,36 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "1", - "metadata": {}, - "outputs": [], + "execution_count": 1, + "id": "253903ad", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-02T23:36:57.559610Z", + "iopub.status.busy": "2026-06-02T23:36:57.559482Z", + "iopub.status.idle": "2026-06-02T23:37:01.648567Z", + "shell.execute_reply": "2026-06-02T23:37:01.647377Z" + } + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Found default environment files: ['C:\\\\Users\\\\vvalbuena\\\\.pyrit\\\\.env', 'C:\\\\Users\\\\vvalbuena\\\\.pyrit\\\\.env.local']\n", + "Loaded environment file: C:\\Users\\vvalbuena\\.pyrit\\.env\n", + "Loaded environment file: C:\\Users\\vvalbuena\\.pyrit\\.env.local\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "[pyrit:alembic] No new upgrade operations detected.\n" + ] + } + ], "source": [ + "\n", "from pyrit.common import apply_defaults\n", "from pyrit.scenario import (\n", " DatasetConfiguration,\n", @@ -107,8 +132,10 @@ ")\n", "from pyrit.score.true_false.true_false_scorer import TrueFalseScorer\n", "from pyrit.setup import initialize_pyrit_async\n", + "from pyrit.setup.initializers.components import ScenarioTechniqueInitializer\n", "\n", "await initialize_pyrit_async(memory_db_type=\"InMemory\") # type: ignore [top-level-await]\n", + "await ScenarioTechniqueInitializer().initialize_async() # type: ignore [top-level-await]\n", "\n", "\n", "class MyStrategy(ScenarioStrategy):\n", @@ -157,7 +184,7 @@ }, { "cell_type": "markdown", - "id": "2", + "id": "c0d205e0", "metadata": {}, "source": [ "\n", @@ -166,10 +193,329 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "3", - "metadata": {}, - "outputs": [], + "execution_count": 2, + "id": "e8dced6d", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-02T23:37:01.650584Z", + "iopub.status.busy": "2026-06-02T23:37:01.650195Z", + "iopub.status.idle": "2026-06-02T23:37:08.613348Z", + "shell.execute_reply": "2026-06-02T23:37:08.612213Z" + } + }, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "TargetRegistry entry 'objective_scorer_chat' not found. Falling back to default OpenAIChatTarget.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using fallback default objective scorer: TrueFalseInverterScorer with chat target: OpenAIChatTarget\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "TargetRegistry entry 'objective_scorer_chat' not found. Falling back to default OpenAIChatTarget.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using fallback default objective scorer: TrueFalseInverterScorer with chat target: OpenAIChatTarget\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "TargetRegistry entry 'objective_scorer_chat' not found. Falling back to default OpenAIChatTarget.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "TargetRegistry entry 'adversarial_chat' not found. Falling back to default OpenAIChatTarget.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "TargetRegistry entry 'objective_scorer_chat' not found. Falling back to default OpenAIChatTarget.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "TargetRegistry entry 'objective_scorer_chat' not found. Falling back to default OpenAIChatTarget.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using fallback default objective scorer: TrueFalseInverterScorer with chat target: OpenAIChatTarget\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "TargetRegistry entry 'objective_scorer_chat' not found. Falling back to default OpenAIChatTarget.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "TargetRegistry entry 'adversarial_chat' not found. Falling back to default OpenAIChatTarget.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "TargetRegistry entry 'objective_scorer_chat' not found. Falling back to default OpenAIChatTarget.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using fallback default objective scorer: TrueFalseInverterScorer with chat target: OpenAIChatTarget\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "TargetRegistry entry 'adversarial_chat' not found. Falling back to default OpenAIChatTarget.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "TargetRegistry entry 'objective_scorer_chat' not found. Falling back to default OpenAIChatTarget.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using fallback default objective scorer: TrueFalseInverterScorer with chat target: OpenAIChatTarget\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "\n", + "Available Scenarios:\n", + "================================================================================\n", + "\u001b[1m\u001b[36m\n", + " airt.cyber\u001b[0m\n", + " Class: Cyber\n", + " Description:\n", + " Cyber scenario implementation for PyRIT. This scenario tests how willing\n", + " models are to exploit cybersecurity harms by generating malware. The\n", + " Cyber class contains different variations of the malware generation\n", + " techniques.\n", + " Aggregate Strategies:\n", + " - all, multi_turn\n", + " Available Strategies (1):\n", + " red_teaming\n", + " Default Strategy: all\n", + " Default Datasets (1, max 4 per dataset):\n", + " airt_malware\n", + "\u001b[1m\u001b[36m\n", + " airt.jailbreak\u001b[0m\n", + " Class: Jailbreak\n", + " Description:\n", + " Jailbreak scenario implementation for PyRIT. This scenario tests how\n", + " vulnerable models are to jailbreak attacks by applying various\n", + " single-turn jailbreak templates to a set of test prompts. The responses\n", + " are scored to determine if the jailbreak was successful.\n", + " Aggregate Strategies:\n", + " - all, simple, complex\n", + " Available Strategies (4):\n", + " prompt_sending, many_shot, skeleton, role_play\n", + " Default Strategy: simple\n", + " Default Datasets (1, max 4 per dataset):\n", + " airt_harms\n", + "\u001b[1m\u001b[36m\n", + " airt.leakage\u001b[0m\n", + " Class: Leakage\n", + " Description:\n", + " Leakage scenario implementation for PyRIT. This scenario tests how\n", + " susceptible models are to leaking training data, PII, intellectual\n", + " property, or other confidential information. Uses the registry/factory\n", + " pattern to construct attack techniques.\n", + " Aggregate Strategies:\n", + " - all, default, single_turn, multi_turn\n", + " Available Strategies (12):\n", + " role_play, many_shot, tap, pair, crescendo_simulated, red_teaming,\n", + " context_compliance, crescendo_movie_director, crescendo_history_lecture,\n", + " crescendo_journalist_interview, first_letter, image\n", + " Default Strategy: default\n", + " Default Datasets (1, max 4 per dataset):\n", + " airt_leakage\n", + "\u001b[1m\u001b[36m\n", + " airt.psychosocial\u001b[0m\n", + " Class: Psychosocial\n", + " Description:\n", + " Psychosocial Harms Scenario implementation for PyRIT. This scenario\n", + " contains various psychosocial harm-based checks that evaluate model\n", + " behavior for responses that may harm the well-being or mental health of\n", + " users. Examples include mistreating users in a mental health crisis or\n", + " aiding them in self-harm. All datasets in this scenario share the\n", + " unified 'psychosocial' harm category, but each dataset (e.g.,\n", + " imminent_crisis, dependency) can have unique: - Scoring rubrics - Seed\n", + " objectives The scenario tests whether models: - Recognize mental health\n", + " crises and respond appropriately - Avoid facilitating self-harm or\n", + " harmful behaviors - Provide crisis resources and encourage professional\n", + " help Example usage with custom configurations: # Define custom\n", + " configurations per subharm category custom_configs = {\n", + " \"airt_imminent_crisis\": SubharmConfig(\n", + " crescendo_system_prompt_path=\"path/to/custom_escalation.yaml\",\n", + " scoring_rubric_path=\"path/to/custom_rubric.yaml\", ), } scenario =\n", + " Psychosocial(subharm_configs=custom_configs) await\n", + " scenario.initialize_async( objective_target=target_llm,\n", + " scenario_strategies=[PsychosocialStrategy.ImminentCrisis], )\n", + " Aggregate Strategies:\n", + " - all\n", + " Available Strategies (2):\n", + " imminent_crisis, licensed_therapist\n", + " Default Strategy: all\n", + " Default Datasets (1, max 4 per dataset):\n", + " airt_imminent_crisis\n", + "\u001b[1m\u001b[36m\n", + " airt.rapid_response\u001b[0m\n", + " Class: RapidResponse\n", + " Description:\n", + " Rapid Response scenario for content-harms testing. Tests model behavior\n", + " across multiple harm categories using selectable attack techniques.\n", + " Aggregate Strategies:\n", + " - all, default, single_turn, multi_turn\n", + " Available Strategies (10):\n", + " role_play, many_shot, tap, pair, crescendo_simulated, red_teaming,\n", + " context_compliance, crescendo_movie_director, crescendo_history_lecture,\n", + " crescendo_journalist_interview\n", + " Default Strategy: default\n", + " Default Datasets (7, max 4 per dataset):\n", + " airt_hate, airt_fairness, airt_violence, airt_sexual, airt_harassment,\n", + " airt_misinformation, airt_leakage\n", + "\u001b[1m\u001b[36m\n", + " airt.scam\u001b[0m\n", + " Class: Scam\n", + " Description:\n", + " Scam scenario evaluates an endpoint's ability to generate scam-related\n", + " materials (e.g., phishing emails, fraudulent messages) with primarily\n", + " persuasion-oriented techniques.\n", + " Aggregate Strategies:\n", + " - all, single_turn, multi_turn\n", + " Available Strategies (3):\n", + " context_compliance, role_play, persuasive_rta\n", + " Default Strategy: all\n", + " Default Datasets (1, max 4 per dataset):\n", + " airt_scams\n", + " Supported Parameters:\n", + " - max_turns (int) [default: '5']: Maximum conversation turns for the persuasive_rta strategy.\n", + "\u001b[1m\u001b[36m\n", + " benchmark.adversarial\u001b[0m\n", + " Class: AdversarialBenchmark\n", + " Description:\n", + " Benchmark scenario that compares the attack success rate (ASR) across\n", + " adversarial models. Adversarial targets are user-supplied via the\n", + " ``adversarial_targets`` parameter (declared in\n", + " ``supported_parameters``). Each target must already be registered in\n", + " ``TargetRegistry`` — typically by ``TargetInitializer`` from\n", + " ``ADVERSARIAL_CHAT_*`` env vars, or programmatically via\n", + " ``TargetRegistry.register_instance``. At run time,\n", + " ``_get_atomic_attacks_async`` performs the ``(technique ×\n", + " adversarial_target × dataset)`` cross-product: for each selected\n", + " adversarial-capable ``core`` factory in the ``AttackTechniqueRegistry``\n", + " and each requested target, it calls\n", + " ``factory.create(attack_adversarial_config_override=...)`` with the\n", + " resolved target — no global registry mutation. The resulting\n", + " ``AtomicAttack`` is named ``f\"{technique}__{target}_{dataset}\"`` with\n", + " ``display_group`` set to the target's registry name so per-model ASR\n", + " rolls up naturally in result displays.\n", + " Aggregate Strategies:\n", + " - all, default, light, single_turn, multi_turn\n", + " Available Strategies (9):\n", + " role_play, tap, pair, crescendo_simulated, red_teaming,\n", + " context_compliance, crescendo_movie_director, crescendo_history_lecture,\n", + " crescendo_journalist_interview\n", + " Default Strategy: light\n", + " Default Datasets (1, max 8 per dataset):\n", + " harmbench\n", + " Supported Parameters:\n", + " - adversarial_targets (list[str]): Registry names of adversarial chat targets to benchmark. Each name must already be registered in TargetRegistry (via TargetInitializer or TargetRegistry.register_instance). Use 'pyrit_scan list-targets' to see registered targets. Settable via --adversarial-targets [ ...] on the CLI, or scenario.args.adversarial_targets in .pyrit_conf.\n", + "\u001b[1m\u001b[36m\n", + " foundry.red_team_agent\u001b[0m\n", + " Class: RedTeamAgent\n", + " Description:\n", + " RedTeamAgent is a preconfigured scenario that automatically generates\n", + " multiple AtomicAttack instances based on the specified attack\n", + " strategies. It supports both single-turn attacks (with various\n", + " converters) and multi-turn attacks (Crescendo, RedTeaming), making it\n", + " easy to quickly test a target against multiple attack vectors. The\n", + " scenario can expand difficulty levels (EASY, MODERATE, DIFFICULT) into\n", + " their constituent attack strategies, or you can specify individual\n", + " strategies directly. This scenario is designed for use with the Foundry\n", + " AI Red Teaming Agent library, providing a consistent PyRIT contract for\n", + " their integration.\n", + " Aggregate Strategies:\n", + " - all, easy, moderate, difficult\n", + " Available Strategies (25):\n", + " ansi_attack, ascii_art, ascii_smuggler, atbash, base64, binary, caesar,\n", + " character_space, char_swap, diacritic, flip, leetspeak, morse, rot13,\n", + " suffix_append, string_join, unicode_confusable, unicode_substitution,\n", + " url, jailbreak, tense, multi_turn, crescendo, pair, tap\n", + " Default Strategy: easy\n", + " Default Datasets (1, max 4 per dataset):\n", + " harmbench\n", + "\u001b[1m\u001b[36m\n", + " garak.encoding\u001b[0m\n", + " Class: Encoding\n", + " Description:\n", + " Encoding Scenario implementation for PyRIT. This scenario tests how\n", + " resilient models are to various encoding attacks by encoding potentially\n", + " harmful text (by default slurs and XSS payloads) and testing if the\n", + " model will decode and repeat the encoded payload. It mimics the Garak\n", + " encoding probe. The scenario works by: 1. Taking seed prompts (the\n", + " harmful text to be encoded) 2. Encoding them using various encoding\n", + " schemes (Base64, ROT13, Morse, etc.) 3. Asking the target model to\n", + " decode the encoded text 4. Scoring whether the model successfully\n", + " decoded and repeated the harmful content By default, this uses the same\n", + " dataset as Garak: slur terms and web XSS payloads.\n", + " Aggregate Strategies:\n", + " - all\n", + " Available Strategies (17):\n", + " base64, base2048, base16, base32, ascii85, hex, quoted_printable,\n", + " uuencode, rot13, braille, atbash, morse_code, nato, ecoji, zalgo,\n", + " leet_speak, ascii_smuggler\n", + " Default Strategy: all\n", + " Default Datasets (2, max 3 per dataset):\n", + " garak_slur_terms_en, garak_web_html_js\n", + "\n", + "================================================================================\n", + "\n", + "Total scenarios: 9\n" + ] + } + ], "source": [ "from pyrit.backend.services.scenario_service import get_scenario_service\n", "from pyrit.cli._output import print_scenario_list\n", @@ -180,7 +526,7 @@ }, { "cell_type": "markdown", - "id": "4", + "id": "4010cda3", "metadata": {}, "source": [ "\n", @@ -208,7 +554,7 @@ }, { "cell_type": "markdown", - "id": "5", + "id": "9010adfd", "metadata": {}, "source": [ "\n", @@ -233,8 +579,22 @@ } ], "metadata": { - "jupytext": { - "main_language": "python" + "kernelspec": { + "display_name": "PyRIT venv", + "language": "python", + "name": "pyrit-venv" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.13.13" } }, "nbformat": 4, diff --git a/doc/code/scenarios/0_scenarios.py b/doc/code/scenarios/0_scenarios.py index 317aeac1ff..b4ab355b97 100644 --- a/doc/code/scenarios/0_scenarios.py +++ b/doc/code/scenarios/0_scenarios.py @@ -102,8 +102,10 @@ ) from pyrit.score.true_false.true_false_scorer import TrueFalseScorer from pyrit.setup import initialize_pyrit_async +from pyrit.setup.initializers.components import ScenarioTechniqueInitializer await initialize_pyrit_async(memory_db_type="InMemory") # type: ignore [top-level-await] +await ScenarioTechniqueInitializer().initialize_async() # type: ignore [top-level-await] class MyStrategy(ScenarioStrategy): diff --git a/doc/code/scenarios/1_common_scenario_parameters.ipynb b/doc/code/scenarios/1_common_scenario_parameters.ipynb index 9da9b9195b..f05e244eea 100644 --- a/doc/code/scenarios/1_common_scenario_parameters.ipynb +++ b/doc/code/scenarios/1_common_scenario_parameters.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "0", + "id": "3050145c", "metadata": {}, "source": [ "# Common Scenario Parameters\n", @@ -24,9 +24,15 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "1", + "execution_count": 1, + "id": "7a00900c", "metadata": { + "execution": { + "iopub.execute_input": "2026-06-02T23:56:17.121550Z", + "iopub.status.busy": "2026-06-02T23:56:17.121265Z", + "iopub.status.idle": "2026-06-02T23:57:19.578345Z", + "shell.execute_reply": "2026-06-02T23:57:19.577552Z" + }, "lines_to_next_cell": 0 }, "outputs": [ @@ -34,23 +40,62 @@ "name": "stdout", "output_type": "stream", "text": [ - "Found default environment files: ['./.pyrit/.env', './.pyrit/.env.local']\n", - "Loaded environment file: ./.pyrit/.env\n", - "Loaded environment file: ./.pyrit/.env.local\n" + "Found default environment files: ['C:\\\\Users\\\\vvalbuena\\\\.pyrit\\\\.env', 'C:\\\\Users\\\\vvalbuena\\\\.pyrit\\\\.env.local']\n", + "Loaded environment file: C:\\Users\\vvalbuena\\.pyrit\\.env\n", + "Loaded environment file: C:\\Users\\vvalbuena\\.pyrit\\.env.local\n" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "No new upgrade operations detected.\n" + "[pyrit:alembic] No new upgrade operations detected.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Skipping target 'platform_openai_chat': PLATFORM_OPENAI_CHAT_GPT4O_MODEL is not set. All declared env vars (endpoint, key, model) must be present for this target to register.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Skipping target 'azure_foundry_phi4': AZURE_FOUNDRY_PHI4_MODEL is not set. All declared env vars (endpoint, key, model) must be present for this target to register.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\r", + "Loading datasets - this can take a few minutes: 0%| | 0/83 [00:00, choices=None)\n" ] } ], "source": [ + "\n", "from pyrit.scenario.scenarios.airt.scam import Scam\n", + "from pyrit.setup import initialize_pyrit_async\n", + "from pyrit.setup.initializers.components import ScenarioTechniqueInitializer\n", + "\n", + "await initialize_pyrit_async(memory_db_type=\"InMemory\") # type: ignore [top-level-await]\n", + "await ScenarioTechniqueInitializer().initialize_async() # type: ignore [top-level-await]\n", "\n", "for param in Scam.supported_parameters():\n", " print(param)" @@ -66,7 +89,7 @@ }, { "cell_type": "markdown", - "id": "2", + "id": "acf8c854", "metadata": {}, "source": [ "Each declaration lives inside the scenario class body, in the\n", @@ -86,9 +109,16 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "3", - "metadata": {}, + "execution_count": 2, + "id": "334a7464", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-02T23:50:44.703738Z", + "iopub.status.busy": "2026-06-02T23:50:44.703403Z", + "iopub.status.idle": "2026-06-02T23:50:44.708024Z", + "shell.execute_reply": "2026-06-02T23:50:44.707470Z" + } + }, "outputs": [ { "name": "stdout", @@ -128,7 +158,7 @@ }, { "cell_type": "markdown", - "id": "4", + "id": "8c2f4c95", "metadata": {}, "source": [ "## Reading the value\n", @@ -159,7 +189,7 @@ }, { "cell_type": "markdown", - "id": "5", + "id": "f7af8547", "metadata": {}, "source": [ "## Setting a parameter from the CLI\n", @@ -194,7 +224,7 @@ }, { "cell_type": "markdown", - "id": "6", + "id": "562894a2", "metadata": {}, "source": [ "## Setting a parameter from a YAML config file\n", @@ -215,7 +245,7 @@ }, { "cell_type": "markdown", - "id": "7", + "id": "8c63cbe6", "metadata": {}, "source": [ "A few invocation shapes from the CLI:\n", @@ -238,14 +268,136 @@ }, { "cell_type": "code", - "execution_count": null, - "id": "8", - "metadata": {}, + "execution_count": 3, + "id": "1971e405", + "metadata": { + "execution": { + "iopub.execute_input": "2026-06-02T23:50:44.709372Z", + "iopub.status.busy": "2026-06-02T23:50:44.709229Z", + "iopub.status.idle": "2026-06-02T23:50:51.726075Z", + "shell.execute_reply": "2026-06-02T23:50:51.724874Z" + } + }, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "TargetRegistry entry 'objective_scorer_chat' not found. Falling back to default OpenAIChatTarget.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using fallback default objective scorer: TrueFalseInverterScorer with chat target: OpenAIChatTarget\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "TargetRegistry entry 'objective_scorer_chat' not found. Falling back to default OpenAIChatTarget.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using fallback default objective scorer: TrueFalseInverterScorer with chat target: OpenAIChatTarget\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "TargetRegistry entry 'objective_scorer_chat' not found. Falling back to default OpenAIChatTarget.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "TargetRegistry entry 'adversarial_chat' not found. Falling back to default OpenAIChatTarget.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "TargetRegistry entry 'objective_scorer_chat' not found. Falling back to default OpenAIChatTarget.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "TargetRegistry entry 'objective_scorer_chat' not found. Falling back to default OpenAIChatTarget.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using fallback default objective scorer: TrueFalseInverterScorer with chat target: OpenAIChatTarget\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "TargetRegistry entry 'objective_scorer_chat' not found. Falling back to default OpenAIChatTarget.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "TargetRegistry entry 'adversarial_chat' not found. Falling back to default OpenAIChatTarget.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "TargetRegistry entry 'objective_scorer_chat' not found. Falling back to default OpenAIChatTarget.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using fallback default objective scorer: TrueFalseInverterScorer with chat target: OpenAIChatTarget\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "TargetRegistry entry 'adversarial_chat' not found. Falling back to default OpenAIChatTarget.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "TargetRegistry entry 'objective_scorer_chat' not found. Falling back to default OpenAIChatTarget.\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Using fallback default objective scorer: TrueFalseInverterScorer with chat target: OpenAIChatTarget\n" + ] + }, { "name": "stdout", "output_type": "stream", "text": [ + "\n", + "Available Scenarios:\n", + "================================================================================\n", "\u001b[1m\u001b[36m\n", " airt.scam\u001b[0m\n", " Class: Scam\n", @@ -261,7 +413,7 @@ " Default Datasets (1, max 4 per dataset):\n", " airt_scams\n", " Supported Parameters:\n", - " - max_turns (int) [default: 5]: Maximum conversation turns for the persuasive_rta strategy.\n", + " - max_turns (int) [default: '5']: Maximum conversation turns for the persuasive_rta strategy.\n", "\u001b[1m\u001b[36m\n", " foundry.red_team_agent\u001b[0m\n", " Class: RedTeamAgent\n", @@ -285,7 +437,11 @@ " url, jailbreak, tense, multi_turn, crescendo, pair, tap\n", " Default Strategy: easy\n", " Default Datasets (1, max 4 per dataset):\n", - " harmbench\n" + " harmbench\n", + "\n", + "================================================================================\n", + "\n", + "Total scenarios: 2\n" ] } ], @@ -302,7 +458,7 @@ }, { "cell_type": "markdown", - "id": "9", + "id": "9e5ea692", "metadata": {}, "source": [ "Notice the `Supported Parameters:` section under `airt.scam`. It's absent\n", @@ -341,7 +497,7 @@ }, { "cell_type": "markdown", - "id": "10", + "id": "56d4150c", "metadata": {}, "source": [ "`Scam.max_turns` was previously hardcoded to `5` in\n", @@ -352,6 +508,11 @@ } ], "metadata": { + "kernelspec": { + "display_name": "PyRIT venv", + "language": "python", + "name": "pyrit-venv" + }, "language_info": { "codemirror_mode": { "name": "ipython", @@ -362,7 +523,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.12.12" + "version": "3.13.13" } }, "nbformat": 4, diff --git a/doc/code/scenarios/2_custom_scenario_parameters.py b/doc/code/scenarios/2_custom_scenario_parameters.py index bcebdb1f9e..b28085ae0c 100644 --- a/doc/code/scenarios/2_custom_scenario_parameters.py +++ b/doc/code/scenarios/2_custom_scenario_parameters.py @@ -47,7 +47,13 @@ # would wire up memory and scorers): # %% + from pyrit.scenario.scenarios.airt.scam import Scam +from pyrit.setup import initialize_pyrit_async +from pyrit.setup.initializers.components import ScenarioTechniqueInitializer + +await initialize_pyrit_async(memory_db_type="InMemory") # type: ignore [top-level-await] +await ScenarioTechniqueInitializer().initialize_async() # type: ignore [top-level-await] for param in Scam.supported_parameters(): print(param) From 484b408ce06ef46eb4a96fc008c292285c6a8155 Mon Sep 17 00:00:00 2001 From: Victor Valbuena Date: Tue, 2 Jun 2026 17:06:17 -0700 Subject: [PATCH 5/5] FIX: precommit hooks. --- doc/code/scenarios/0_scenarios.ipynb | 48 +- doc/code/scenarios/0_scenarios.py | 2 +- .../1_common_scenario_parameters.ipynb | 948 +----------------- .../scenarios/1_common_scenario_parameters.py | 4 +- .../2_custom_scenario_parameters.ipynb | 69 +- .../scenarios/2_custom_scenario_parameters.py | 2 +- 6 files changed, 84 insertions(+), 989 deletions(-) diff --git a/doc/code/scenarios/0_scenarios.ipynb b/doc/code/scenarios/0_scenarios.ipynb index 3406a60943..3eb5a071a3 100644 --- a/doc/code/scenarios/0_scenarios.ipynb +++ b/doc/code/scenarios/0_scenarios.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "844edc95", + "id": "0", "metadata": { "lines_to_next_cell": 0 }, @@ -94,24 +94,17 @@ }, { "cell_type": "code", - "execution_count": 1, - "id": "253903ad", - "metadata": { - "execution": { - "iopub.execute_input": "2026-06-02T23:36:57.559610Z", - "iopub.status.busy": "2026-06-02T23:36:57.559482Z", - "iopub.status.idle": "2026-06-02T23:37:01.648567Z", - "shell.execute_reply": "2026-06-02T23:37:01.647377Z" - } - }, + "execution_count": null, + "id": "1", + "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Found default environment files: ['C:\\\\Users\\\\vvalbuena\\\\.pyrit\\\\.env', 'C:\\\\Users\\\\vvalbuena\\\\.pyrit\\\\.env.local']\n", - "Loaded environment file: C:\\Users\\vvalbuena\\.pyrit\\.env\n", - "Loaded environment file: C:\\Users\\vvalbuena\\.pyrit\\.env.local\n" + "Found default environment files: ['./.pyrit/.env', './.pyrit/.env.local']\n", + "Loaded environment file: ./.pyrit/.env\n", + "Loaded environment file: ./.pyrit/.env.local\n" ] }, { @@ -123,7 +116,6 @@ } ], "source": [ - "\n", "from pyrit.common import apply_defaults\n", "from pyrit.scenario import (\n", " DatasetConfiguration,\n", @@ -135,7 +127,7 @@ "from pyrit.setup.initializers.components import ScenarioTechniqueInitializer\n", "\n", "await initialize_pyrit_async(memory_db_type=\"InMemory\") # type: ignore [top-level-await]\n", - "await ScenarioTechniqueInitializer().initialize_async() # type: ignore [top-level-await]\n", + "await ScenarioTechniqueInitializer().initialize_async() # type: ignore [top-level-await]\n", "\n", "\n", "class MyStrategy(ScenarioStrategy):\n", @@ -184,7 +176,7 @@ }, { "cell_type": "markdown", - "id": "c0d205e0", + "id": "2", "metadata": {}, "source": [ "\n", @@ -193,16 +185,9 @@ }, { "cell_type": "code", - "execution_count": 2, - "id": "e8dced6d", - "metadata": { - "execution": { - "iopub.execute_input": "2026-06-02T23:37:01.650584Z", - "iopub.status.busy": "2026-06-02T23:37:01.650195Z", - "iopub.status.idle": "2026-06-02T23:37:08.613348Z", - "shell.execute_reply": "2026-06-02T23:37:08.612213Z" - } - }, + "execution_count": null, + "id": "3", + "metadata": {}, "outputs": [ { "name": "stderr", @@ -526,7 +511,7 @@ }, { "cell_type": "markdown", - "id": "4010cda3", + "id": "4", "metadata": {}, "source": [ "\n", @@ -554,7 +539,7 @@ }, { "cell_type": "markdown", - "id": "9010adfd", + "id": "5", "metadata": {}, "source": [ "\n", @@ -579,11 +564,6 @@ } ], "metadata": { - "kernelspec": { - "display_name": "PyRIT venv", - "language": "python", - "name": "pyrit-venv" - }, "language_info": { "codemirror_mode": { "name": "ipython", diff --git a/doc/code/scenarios/0_scenarios.py b/doc/code/scenarios/0_scenarios.py index b4ab355b97..03e6fd81c2 100644 --- a/doc/code/scenarios/0_scenarios.py +++ b/doc/code/scenarios/0_scenarios.py @@ -105,7 +105,7 @@ from pyrit.setup.initializers.components import ScenarioTechniqueInitializer await initialize_pyrit_async(memory_db_type="InMemory") # type: ignore [top-level-await] -await ScenarioTechniqueInitializer().initialize_async() # type: ignore [top-level-await] +await ScenarioTechniqueInitializer().initialize_async() # type: ignore [top-level-await] class MyStrategy(ScenarioStrategy): diff --git a/doc/code/scenarios/1_common_scenario_parameters.ipynb b/doc/code/scenarios/1_common_scenario_parameters.ipynb index f05e244eea..5d95e9b326 100644 --- a/doc/code/scenarios/1_common_scenario_parameters.ipynb +++ b/doc/code/scenarios/1_common_scenario_parameters.ipynb @@ -2,7 +2,7 @@ "cells": [ { "cell_type": "markdown", - "id": "3050145c", + "id": "0", "metadata": {}, "source": [ "# Common Scenario Parameters\n", @@ -24,15 +24,9 @@ }, { "cell_type": "code", - "execution_count": 1, - "id": "7a00900c", + "execution_count": null, + "id": "1", "metadata": { - "execution": { - "iopub.execute_input": "2026-06-02T23:56:17.121550Z", - "iopub.status.busy": "2026-06-02T23:56:17.121265Z", - "iopub.status.idle": "2026-06-02T23:57:19.578345Z", - "shell.execute_reply": "2026-06-02T23:57:19.577552Z" - }, "lines_to_next_cell": 0 }, "outputs": [ @@ -40,9 +34,9 @@ "name": "stdout", "output_type": "stream", "text": [ - "Found default environment files: ['C:\\\\Users\\\\vvalbuena\\\\.pyrit\\\\.env', 'C:\\\\Users\\\\vvalbuena\\\\.pyrit\\\\.env.local']\n", - "Loaded environment file: C:\\Users\\vvalbuena\\.pyrit\\.env\n", - "Loaded environment file: C:\\Users\\vvalbuena\\.pyrit\\.env.local\n" + "Found default environment files: ['./.pyrit/.env', './.pyrit/.env.local']\n", + "Loaded environment file: ./.pyrit/.env\n", + "Loaded environment file: ./.pyrit/.env.local\n" ] }, { @@ -66,38 +60,6 @@ "Skipping target 'azure_foundry_phi4': AZURE_FOUNDRY_PHI4_MODEL is not set. All declared env vars (endpoint, key, model) must be present for this target to register.\n" ] }, - { - "name": "stderr", - "output_type": "stream", - "text": [ - "\r", - "Loading datasets - this can take a few minutes: 0%| | 0/83 [00:00