{ "environment": { "note": "Every configuration disables d022 semantic classifier so deltas isolate d027 (stylometric discontinuity) and d028 (Smith-Waterman) against the 26-detector regex baseline.", "threshold": 0.7, "block_cutoff_risk_score": 0.5 }, "datasets": { "deepset/prompt-injections": { "sample_count": 116, "attack_count": 60, "benign_count": 56, "configs": { "baseline": { "n": 116, "tp": 1, "tn": 56, "fp": 0, "fn": 59, "precision": 1.0, "recall": 0.0167, "f1": 0.0328, "accuracy": 0.4914, "fpr": 0.0, "elapsed_s": 0.47, "throughput_per_s": 244.3 }, "plus_d028": { "n": 116, "tp": 14, "tn": 56, "fp": 0, "fn": 46, "precision": 1.0, "recall": 0.2333, "f1": 0.3784, "accuracy": 0.6034, "fpr": 0.0, "elapsed_s": 1.32, "throughput_per_s": 88.1 }, "plus_d027": { "n": 116, "tp": 1, "tn": 56, "fp": 0, "fn": 59, "precision": 1.0, "recall": 0.0167, "f1": 0.0328, "accuracy": 0.4914, "fpr": 0.0, "elapsed_s": 0.45, "throughput_per_s": 256.0 }, "plus_d027_d028": { "n": 116, "tp": 14, "tn": 56, "fp": 0, "fn": 46, "precision": 1.0, "recall": 0.2333, "f1": 0.3784, "accuracy": 0.6034, "fpr": 0.0, "elapsed_s": 1.33, "throughput_per_s": 87.4 } }, "control_26_detectors": { "n": 116, "tp": 1, "tn": 56, "fp": 0, "fn": 59, "precision": 1.0, "recall": 0.0167, "f1": 0.0328, "accuracy": 0.4914, "fpr": 0.0, "elapsed_s": 0.47, "throughput_per_s": 244.3 }, "treatment_27_detectors_with_d028": { "n": 116, "tp": 14, "tn": 56, "fp": 0, "fn": 46, "precision": 1.0, "recall": 0.2333, "f1": 0.3784, "accuracy": 0.6034, "fpr": 0.0, "elapsed_s": 1.32, "throughput_per_s": 88.1 }, "delta_treatment_minus_control": { "recall_pp": 21.66, "f1_pp": 34.56, "accuracy_pp": 11.2, "fpr_pp": 0.0, "tp_delta": 13, "fp_delta": 0, "fn_delta": -13 } }, "leolee99/NotInject": { "sample_count": 339, "attack_count": 0, "benign_count": 339, "configs": { "baseline": { "n": 339, "tp": 0, "tn": 336, "fp": 3, "fn": 0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "accuracy": 0.9912, "fpr": 0.0088, "elapsed_s": 1.4, "throughput_per_s": 241.7 }, "plus_d028": { "n": 339, "tp": 0, "tn": 326, "fp": 13, "fn": 0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "accuracy": 0.9617, "fpr": 0.0383, "elapsed_s": 4.19, "throughput_per_s": 80.9 }, "plus_d027": { "n": 339, "tp": 0, "tn": 336, "fp": 3, "fn": 0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "accuracy": 0.9912, "fpr": 0.0088, "elapsed_s": 1.72, "throughput_per_s": 197.0 }, "plus_d027_d028": { "n": 339, "tp": 0, "tn": 326, "fp": 13, "fn": 0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "accuracy": 0.9617, "fpr": 0.0383, "elapsed_s": 4.27, "throughput_per_s": 79.3 } }, "control_26_detectors": { "n": 339, "tp": 0, "tn": 336, "fp": 3, "fn": 0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "accuracy": 0.9912, "fpr": 0.0088, "elapsed_s": 1.4, "throughput_per_s": 241.7 }, "treatment_27_detectors_with_d028": { "n": 339, "tp": 0, "tn": 326, "fp": 13, "fn": 0, "precision": 0.0, "recall": 0.0, "f1": 0.0, "accuracy": 0.9617, "fpr": 0.0383, "elapsed_s": 4.19, "throughput_per_s": 80.9 }, "delta_treatment_minus_control": { "recall_pp": 0.0, "f1_pp": 0.0, "accuracy_pp": -2.95, "fpr_pp": 2.95, "tp_delta": 0, "fp_delta": 10, "fn_delta": 0 } }, "microsoft/llmail-inject-challenge (Phase1, 1000 subset)": { "sample_count": 1000, "attack_count": 1000, "benign_count": 0, "configs": { "baseline": { "n": 1000, "tp": 978, "tn": 0, "fp": 0, "fn": 22, "precision": 1.0, "recall": 0.978, "f1": 0.9889, "accuracy": 0.978, "fpr": 0.0, "elapsed_s": 13.69, "throughput_per_s": 73.1 }, "plus_d028": { "n": 1000, "tp": 980, "tn": 0, "fp": 0, "fn": 20, "precision": 1.0, "recall": 0.98, "f1": 0.9899, "accuracy": 0.98, "fpr": 0.0, "elapsed_s": 253.53, "throughput_per_s": 3.9 }, "plus_d027": { "n": 1000, "tp": 978, "tn": 0, "fp": 0, "fn": 22, "precision": 1.0, "recall": 0.978, "f1": 0.9889, "accuracy": 0.978, "fpr": 0.0, "elapsed_s": 25.87, "throughput_per_s": 38.7 }, "plus_d027_d028": { "n": 1000, "tp": 980, "tn": 0, "fp": 0, "fn": 20, "precision": 1.0, "recall": 0.98, "f1": 0.9899, "accuracy": 0.98, "fpr": 0.0, "elapsed_s": 232.67, "throughput_per_s": 4.3 } }, "control_26_detectors": { "n": 1000, "tp": 978, "tn": 0, "fp": 0, "fn": 22, "precision": 1.0, "recall": 0.978, "f1": 0.9889, "accuracy": 0.978, "fpr": 0.0, "elapsed_s": 13.69, "throughput_per_s": 73.1 }, "treatment_27_detectors_with_d028": { "n": 1000, "tp": 980, "tn": 0, "fp": 0, "fn": 20, "precision": 1.0, "recall": 0.98, "f1": 0.9899, "accuracy": 0.98, "fpr": 0.0, "elapsed_s": 253.53, "throughput_per_s": 3.9 }, "delta_treatment_minus_control": { "recall_pp": 0.2, "f1_pp": 0.1, "accuracy_pp": 0.2, "fpr_pp": 0.0, "tp_delta": 2, "fp_delta": 0, "fn_delta": -2 } }, "ai-safety-institute/AgentHarm": { "sample_count": 352, "attack_count": 176, "benign_count": 176, "configs": { "baseline": { "n": 352, "tp": 44, "tn": 120, "fp": 56, "fn": 132, "precision": 0.44, "recall": 0.25, "f1": 0.3188, "accuracy": 0.4659, "fpr": 0.3182, "elapsed_s": 4.1, "throughput_per_s": 85.9 }, "plus_d028": { "n": 352, "tp": 44, "tn": 120, "fp": 56, "fn": 132, "precision": 0.44, "recall": 0.25, "f1": 0.3188, "accuracy": 0.4659, "fpr": 0.3182, "elapsed_s": 14.65, "throughput_per_s": 24.0 }, "plus_d027": { "n": 352, "tp": 44, "tn": 120, "fp": 56, "fn": 132, "precision": 0.44, "recall": 0.25, "f1": 0.3188, "accuracy": 0.4659, "fpr": 0.3182, "elapsed_s": 2.99, "throughput_per_s": 117.8 }, "plus_d027_d028": { "n": 352, "tp": 44, "tn": 120, "fp": 56, "fn": 132, "precision": 0.44, "recall": 0.25, "f1": 0.3188, "accuracy": 0.4659, "fpr": 0.3182, "elapsed_s": 15.77, "throughput_per_s": 22.3 } }, "control_26_detectors": { "n": 352, "tp": 44, "tn": 120, "fp": 56, "fn": 132, "precision": 0.44, "recall": 0.25, "f1": 0.3188, "accuracy": 0.4659, "fpr": 0.3182, "elapsed_s": 4.1, "throughput_per_s": 85.9 }, "treatment_27_detectors_with_d028": { "n": 352, "tp": 44, "tn": 120, "fp": 56, "fn": 132, "precision": 0.44, "recall": 0.25, "f1": 0.3188, "accuracy": 0.4659, "fpr": 0.3182, "elapsed_s": 14.65, "throughput_per_s": 24.0 }, "delta_treatment_minus_control": { "recall_pp": 0.0, "f1_pp": 0.0, "accuracy_pp": 0.0, "fpr_pp": 0.0, "tp_delta": 0, "fp_delta": 0, "fn_delta": 0 } }, "ethz-spylab/agentdojo (v1.2.1)": { "sample_count": 132, "attack_count": 35, "benign_count": 97, "configs": { "baseline": { "n": 132, "tp": 17, "tn": 86, "fp": 11, "fn": 18, "precision": 0.6071, "recall": 0.4857, "f1": 0.5397, "accuracy": 0.7803, "fpr": 0.1134, "elapsed_s": 0.81, "throughput_per_s": 162.9 }, "plus_d028": { "n": 132, "tp": 18, "tn": 83, "fp": 14, "fn": 17, "precision": 0.5625, "recall": 0.5143, "f1": 0.5373, "accuracy": 0.7652, "fpr": 0.1443, "elapsed_s": 4.04, "throughput_per_s": 32.7 }, "plus_d027": { "n": 132, "tp": 17, "tn": 86, "fp": 11, "fn": 18, "precision": 0.6071, "recall": 0.4857, "f1": 0.5397, "accuracy": 0.7803, "fpr": 0.1134, "elapsed_s": 0.74, "throughput_per_s": 177.6 }, "plus_d027_d028": { "n": 132, "tp": 18, "tn": 83, "fp": 14, "fn": 17, "precision": 0.5625, "recall": 0.5143, "f1": 0.5373, "accuracy": 0.7652, "fpr": 0.1443, "elapsed_s": 4.51, "throughput_per_s": 29.3 } }, "control_26_detectors": { "n": 132, "tp": 17, "tn": 86, "fp": 11, "fn": 18, "precision": 0.6071, "recall": 0.4857, "f1": 0.5397, "accuracy": 0.7803, "fpr": 0.1134, "elapsed_s": 0.81, "throughput_per_s": 162.9 }, "treatment_27_detectors_with_d028": { "n": 132, "tp": 18, "tn": 83, "fp": 14, "fn": 17, "precision": 0.5625, "recall": 0.5143, "f1": 0.5373, "accuracy": 0.7652, "fpr": 0.1443, "elapsed_s": 4.04, "throughput_per_s": 32.7 }, "delta_treatment_minus_control": { "recall_pp": 2.86, "f1_pp": -0.24, "accuracy_pp": -1.51, "fpr_pp": 3.09, "tp_delta": 1, "fp_delta": 3, "fn_delta": -1 } }, "synthetic/indirect-injection": { "sample_count": 80, "attack_count": 30, "benign_count": 50, "configs": { "baseline": { "n": 80, "tp": 24, "tn": 50, "fp": 0, "fn": 6, "precision": 1.0, "recall": 0.8, "f1": 0.8889, "accuracy": 0.925, "fpr": 0.0, "elapsed_s": 1.13, "throughput_per_s": 70.8 }, "plus_d028": { "n": 80, "tp": 30, "tn": 50, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0, "accuracy": 1.0, "fpr": 0.0, "elapsed_s": 9.47, "throughput_per_s": 8.4 }, "plus_d027": { "n": 80, "tp": 30, "tn": 50, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0, "accuracy": 1.0, "fpr": 0.0, "elapsed_s": 1.5, "throughput_per_s": 53.3 }, "plus_d027_d028": { "n": 80, "tp": 30, "tn": 50, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0, "accuracy": 1.0, "fpr": 0.0, "elapsed_s": 9.09, "throughput_per_s": 8.8 } }, "control_26_detectors": { "n": 80, "tp": 24, "tn": 50, "fp": 0, "fn": 6, "precision": 1.0, "recall": 0.8, "f1": 0.8889, "accuracy": 0.925, "fpr": 0.0, "elapsed_s": 1.13, "throughput_per_s": 70.8 }, "treatment_27_detectors_with_d028": { "n": 80, "tp": 30, "tn": 50, "fp": 0, "fn": 0, "precision": 1.0, "recall": 1.0, "f1": 1.0, "accuracy": 1.0, "fpr": 0.0, "elapsed_s": 9.47, "throughput_per_s": 8.4 }, "delta_treatment_minus_control": { "recall_pp": 20.0, "f1_pp": 11.11, "accuracy_pp": 7.5, "fpr_pp": 0.0, "tp_delta": 6, "fp_delta": 0, "fn_delta": -6 } } } }