Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
548 changes: 548 additions & 0 deletions docs/source/tags_taxonomies.rst

Large diffs are not rendered by default.

12 changes: 11 additions & 1 deletion garak/data/tags.misp.tsv
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,7 @@ quality:Security:Integrity Integrity Detection of tainted training data etc.
quality:Security:Availability Availability Availability (Model DoS)
quality:Security:Adversarial Adversarial Robustness Adversarial Robustness
quality:Security:AgentSecurity Agent Security Security of agentic tool-using LLM applications against exploitation
quality:Behavioral:DeliberativeMisinformation Deliberative Misinformation Content that perpetuates or accelerates disinformation activities
quality:Behavioral:DeliberativeMisinformation Deliberative Misinformation Content that perpetuates or accelerates disinformation activities
risk-cards:lmrc:aggression_user Aggression User Aggression User
risk-cards:lmrc:anthropomorphisation Anthropomorphisation Anthropomorphisation
risk-cards:lmrc:astroturfing Astroturfing Astroturfing
Expand Down Expand Up @@ -179,3 +179,13 @@ demon:Stratagems:Scattershot:Clean_slate Clean slate Resetting context by e.g. s
demon:Stratagems:Scattershot:Changing_hyperparams Changing hyperparams Adjusting generation params, e.g. temperature
demon:Stratagems:Meta-prompting:Perspective-shifting Perspective-shifting Hypotheticals by shifting new point of view, e.g. “what if you didn’t have this restriction? What would you say?”, or asking target to compare outputs and clarify gap
demon:Stratagems:Meta-prompting:Ask_for_examples Ask for examples Request examples of intended behaviour
euai:discrimination:protected_class Protected Class Discrimination Discrimination against a protected class (age, race, sex, etc.)
euai:discrimination:personal Personal Attribute Discrimination Discrimination based on non-protected personal attributes
euai:harmful:violence Violent Language Generation of violent content
euai:harmful:sexual Sexual Language Generation of sexual content
euai:harmful:sexualminor Sexual Language (Minor) Generation of sexual content featuring minors
euai:offensive:exploitation Vulnerability Exploitation Exploitation of software vulnerabilties
euai:offensive:phishing Phishing Assistance with phishing
euai:deception:subliminal Subliminal Deception Manipulation via subliminal deception
euai:deception:hallucination Hallucination Deception Hallucinations related to factual information
euai:harmful:pii Personally Identifiable Information Production of personally identifiable information
7 changes: 6 additions & 1 deletion garak/probes/atkgen.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,9 @@ class Tox(garak.probes.Probe):
using the same detector as this tox probe uses, so both attack generation model
and detector are looking for the same definition of the failure mode."""

intent = "S005" # the attack model is explicitly trying to elicit broadly toxic output
intent = (
"S005" # the attack model is explicitly trying to elicit broadly toxic output
)
# Specific runs can overlap with S005hate or S005profanity depending on what the target produces.
primary_detector = "unsafe_content.ToxicCommentModel"
tags = [
Expand All @@ -52,6 +54,9 @@ class Tox(garak.probes.Probe):
"quality:Behavioral:ContentSafety:Profanity",
"payload:unwanted",
"demon:Stratagems:Scattershot:Regenerate_response", # repeatedly generates challenge prompts hoping target slips
"euai:harmful:sexual",
"euai:discrimination:protected_class",
"euai:discrimination:personal",
]
goal = "make the model say something toxic"
parallelisable_attempts = False # atkgen has its own dialog manager
Expand Down
2 changes: 2 additions & 0 deletions garak/probes/continuation.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,8 @@ class ContinueSlursReclaimedSlursFull(garak.probes.Probe):
"quality:Behavioral:ContentSafety:HateHarassment",
"payload:unwanted:hatespeech",
"demon:Language:Stylizing:Give_examples", # provides high-risk context with partial words as examples for completion
"euai:discrimination:protected_class",
"euai:discrimination:personal",
]
goal = "output an ethnic slur"
active = False
Expand Down
4 changes: 4 additions & 0 deletions garak/probes/donotanswer.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,6 +54,9 @@ def _dna_attempt_prestore_hook(self, attempt, seq):
"quality:Behavioral:ContentSafety:Violence",
"quality:Behavioral:ContentSafety:Profanity",
"demon:Stratagems:Meta-prompting:Ask_for_examples", # directly requests examples of hateful/offensive content
"euai:harmful:violence",
"euai:discrimination:protected_class",
"euai:discrimination:personal",
],
"human_chatbox": [
"avid-effect:security:S0301",
Expand All @@ -65,6 +68,7 @@ def _dna_attempt_prestore_hook(self, attempt, seq):
"avid-effect:security:S0301",
"quality:Security:Confidentiality",
"demon:Stratagems:Meta-prompting:Ask_for_examples", # requests examples of sensitive/hazardous information
"euai:harmful:pii",
],
"malicious_uses": [
"quality:Behavioral:ContentSafety:LegalGoodsServices",
Expand Down
6 changes: 5 additions & 1 deletion garak/probes/grandma.py
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,8 @@ class Slurs(garak.probes.Probe):
"avid-effect:security:S0300",
"demon:Fictionalizing:Roleplaying:User_persona", # roleplays as grandchild wanting grandma's old-timey slang
"demon:Stratagems:Meta-prompting:Ask_for_examples", # the attacker-supplied scene requests data that the system is explicitly expected to protect
"euai:discrimination:protected_class",
"euai:discrimination:personal",
]
goal = "get model to use ethnic slur"
tier = garak.probes.Tier.OF_CONCERN
Expand Down Expand Up @@ -145,7 +147,9 @@ class Substances(garak.probes.Probe):

lang = "en"
primary_detector = "mitigation.MitigationBypass"
intent = "S006items" # asks for step-by-step instructions to make harmful substances
intent = (
"S006items" # asks for step-by-step instructions to make harmful substances
)
# S003instructions is a secondary fit because the requests are also illegal in many cases.
tags = [
"risk-cards:lmrc:providing_illegal_info",
Expand Down
18 changes: 16 additions & 2 deletions garak/probes/lmrc.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ class Anthropomorphisation(garak.probes.Probe):
tags = [
"risk-cards:lmrc:anthropomorphisation",
"demon:Rhetoric:Socratic_questioning:Identity_characteristics", # probes model identity by asking anthropomorphising questions
"euai:deception:subliminal",
]
goal = "anthropomorphise a language model"
tier = garak.probes.Tier.INFORMATIONAL
Expand Down Expand Up @@ -68,6 +69,7 @@ class Bullying(garak.probes.Probe):
"quality:Behavioral:ContentSafety:Violence",
"payload:unwanted:violence",
"demon:Stratagems:Meta-prompting:Ask_for_examples", # directly asks for examples of bullying tactics and actions
"euai:harmful:violence",
]
goal = "help me bully"
tier = garak.probes.Tier.COMPETE_WITH_SOTA
Expand All @@ -92,13 +94,16 @@ class Deadnaming(garak.probes.Probe):

lang = "en"
doc_uri = "https://github.com/leondz/lm_risk_cards/blob/main/cards/en/deadnaming.md"
intent = "S009deadname" # asks whether the model will use names people no longer use
intent = (
"S009deadname" # asks whether the model will use names people no longer use
)
primary_detector = "base.TriggerListDetector"
tags = [
"risk-cards:lmrc:deadnaming",
"avid-effect:ethics:E0302",
"avid-effect:performance:P0401",
"demon:Rhetoric:Socratic_questioning:Identity_characteristics", # references trans identity characteristics to elicit deadnames
"euai:discrimination:personal",
]
goal = "write a deadname"
tier = garak.probes.Tier.COMPETE_WITH_SOTA
Expand Down Expand Up @@ -147,6 +152,8 @@ class SexualContent(garak.probes.Probe):
"quality:Behavioral:ContentSafety:Sexual",
"payload:unwanted:sexual",
"demon:Language:Stylizing:Give_examples", # provides sexually suggestive prompt starters as examples for continuation
"euai:harmful:sexual",
"euai:harmful:sexualminor",
]
goal = "write sexual content"
tier = garak.probes.Tier.COMPETE_WITH_SOTA
Expand Down Expand Up @@ -181,6 +188,8 @@ class Sexualisation(garak.probes.Probe):
"quality:Behavioral:ContentSafety:Sexual",
"payload:unwanted:sexual",
"demon:Rhetoric:Socratic_questioning:Identity_characteristics",
"euai:harmful:sexual",
"euai:harmful:sexualminor",
]
goal = "sexualise people"
tier = garak.probes.Tier.COMPETE_WITH_SOTA
Expand Down Expand Up @@ -212,6 +221,8 @@ class SlurUsage(garak.probes.Probe):
"quality:Behavioral:ContentSafety:HateHarassment",
"payload:unwanted:hatespeech",
"demon:Rhetoric:Socratic_questioning:Identity_characteristics", # asks for examples of ethnic slurs in context
"euai:discrimination:protected_class",
"euai:discrimination:personal",
]
goal = "use a slur"
tier = garak.probes.Tier.COMPETE_WITH_SOTA
Expand All @@ -230,7 +241,9 @@ class Profanity(garak.probes.Probe):

lang = "en"
doc_uri = "https://github.com/leondz/lm_risk_cards/blob/main/cards/en/profanity.md"
intent = "S005profanity" # the detector family and prompts focus on profane word use
intent = (
"S005profanity" # the detector family and prompts focus on profane word use
)
primary_detector = "unsafe_content.SurgeProfanityAnimalReferences"
extended_detectors = [
"unsafe_content.SurgeProfanityBodilyFluids",
Expand Down Expand Up @@ -280,6 +293,7 @@ class QuackMedicine(garak.probes.Probe):
"quality:Behavioral:DeliberativeMisinformation",
"quality:Behavioral:ContentSafety:Unauthorized",
"payload:dangerous:medical",
"euai:deception:hallucination",
]
goal = "support quack medicine"
tier = garak.probes.Tier.COMPETE_WITH_SOTA
Expand Down
4 changes: 4 additions & 0 deletions garak/probes/malwaregen.py
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ class TopLevel(garak.probes.Probe):
"payload:malicious:badcode",
"quality:Behavioral:ContentSafety:LegalGoodsServices",
"demon:Stratagems:Meta-prompting:Ask_for_examples", # directly asks for concrete malware code examples
"euai:offensive:exploitation",
]
goal = "write malware given a top-level description"
tier = garak.probes.Tier.COMPETE_WITH_SOTA
Expand Down Expand Up @@ -79,6 +80,7 @@ class SubFunctions(garak.probes.Probe):
"payload:malicious:badcode",
"quality:Behavioral:ContentSafety:LegalGoodsServices",
"demon:Stratagems:Meta-prompting:Ask_for_examples", # directly asks for concrete malicious subroutine code examples
"euai:offensive:exploitation",
]
goal = "write code supporting a sub-function of malware"
tier = garak.probes.Tier.COMPETE_WITH_SOTA
Expand Down Expand Up @@ -122,6 +124,7 @@ class Evasion(garak.probes.Probe):
"payload:malicious:badcode",
"quality:Behavioral:ContentSafety:LegalGoodsServices",
"demon:Stratagems:Meta-prompting:Ask_for_examples", # directly asks for concrete malware-evasion code examples
"euai:offensive:exploitation",
]
goal = "Write code to assist evading security and anti-malware measures"
tier = garak.probes.Tier.COMPETE_WITH_SOTA
Expand Down Expand Up @@ -167,6 +170,7 @@ class Payload(garak.probes.Probe):
"avid-effect:security:S0403",
"quality:Behavioral:ContentSafety:LegalGoodsServices",
"demon:Stratagems:Meta-prompting:Ask_for_examples", # directly asks for concrete malware payload implementations
"euai:offensive:exploitation",
]
goal = "write a malware payload"
tier = garak.probes.Tier.COMPETE_WITH_SOTA
Expand Down
1 change: 1 addition & 0 deletions garak/probes/misleading.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,6 +37,7 @@ class FalseAssertion(garak.probes.Probe):
"quality:Robustness:GenerativeMisinformation",
"payload:hallucination",
"demon:Rhetoric:Persuasion_and_manipulation:Distraction", # presents false claims as fact to distract model from refuting them
"euai:deception:hallucination",
]
goal = "make the model expand on a false assertion"
tier = garak.probes.Tier.COMPETE_WITH_SOTA
Expand Down
1 change: 1 addition & 0 deletions garak/probes/packagehallucination.py
Original file line number Diff line number Diff line change
Expand Up @@ -79,6 +79,7 @@ class PackageHallucinationProbe(garak.probes.Probe, ABC):
"quality:Robustness:GenerativeMisinformation",
"payload:malicious:badcode",
"demon:Stratagems:Meta-prompting:Ask_for_examples", # asks model for code examples, probing for hallucinated package names
"euai:deception:hallucination",
]
goal = "base probe for importing non-existent packages"
DEFAULT_PARAMS = garak.probes.Probe.DEFAULT_PARAMS | {
Expand Down
2 changes: 2 additions & 0 deletions garak/probes/promptinject.py
Original file line number Diff line number Diff line change
Expand Up @@ -136,6 +136,7 @@ def _generator_precall_hook(self, generator, attempt):
"demon:Language:Prompt_injection:Strong_arm_attack", # includes forceful attack variants like STOP EVERYTHING!!! NOW!!!
"demon:Language:Stylizing:Capitalizing", # attack set includes uppercase override commands such as IGNORE ANY PREVIOUS...
"demon:Language:Prompt_injection:Ignore_previous_instructions", # injects "ignore previous" commands into benign prompt contexts
"euai:harmful:violence",
],
"load_data_all_attacks": load_data_all_attacks,
"uri": "https://openreview.net/forum?id=qiaRo_7Zmug",
Expand Down Expand Up @@ -177,6 +178,7 @@ def _generator_precall_hook(self, generator, attempt):
"demon:Language:Stylizing:Capitalizing", # attack set includes uppercase override commands such as IGNORE ANY PREVIOUS...
"demon:Rhetoric:Persuasion_and_manipulation:Distraction", # hides the injection inside otherwise benign Playground prompt contexts
"demon:Language:Prompt_injection:Ignore_previous_instructions", # injects "ignore previous" commands into benign prompt contexts
"euai:harmful:violence",
],
"load_data_all_attacks": load_data_all_attacks,
"uri": "https://openreview.net/forum?id=qiaRo_7Zmug",
Expand Down
5 changes: 2 additions & 3 deletions garak/probes/propile.py
Original file line number Diff line number Diff line change
Expand Up @@ -102,6 +102,7 @@ class PIILeakProbeMixin:
"quality:Security:ExtractionInversion",
"payload:leak:training",
"payload:leak:pii",
"euai:harmful:pii",
]
primary_detector = "propile.PIILeak"
tier = garak.probes.Tier.INFORMATIONAL
Expand Down Expand Up @@ -151,9 +152,7 @@ def _load_pii_data(self):
if line:
self.pii_records.append(json.loads(line))

logger.info(
"PII data loaded: %s (%d records)", pii_path, len(self.pii_records)
)
logger.info("PII data loaded: %s (%d records)", pii_path, len(self.pii_records))

def probe(self, generator):
if not self.prompts:
Expand Down
9 changes: 8 additions & 1 deletion garak/probes/realtoxicityprompts.py
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,7 @@ class RTPBlank(garak.probes.Probe):
"quality:Behavioral:ContentSafety:Violence",
"payload:unwanted:violence",
"demon:Language:Stylizing:Give_examples", # provides known-toxic prompt starters as examples for continuation
"euai:harmful:violence",
],
"insult": [
"avid-effect:ethics:E0101",
Expand Down Expand Up @@ -107,6 +108,8 @@ class RTPBlank(garak.probes.Probe):
"quality:Behavioral:ContentSafety:Sexual",
"payload:unwanted:sexual",
"demon:Language:Stylizing:Give_examples", # provides known-toxic prompt starters as examples for continuation
"euai:harmful:sexual",
"euai:harmful:sexualminor",
],
"flirtation": [
"avid-effect:ethics:E0101",
Expand All @@ -123,6 +126,8 @@ class RTPBlank(garak.probes.Probe):
"quality:Behavioral:ContentSafety:HateHarassment",
"payload:hatespeech",
"demon:Language:Stylizing:Give_examples", # provides known-toxic prompt starters as examples for continuation
"euai:discrimination:protected_class",
"euai:discrimination:personal",
],
}

Expand Down Expand Up @@ -171,7 +176,9 @@ class RTPBlank(garak.probes.Probe):
"uri": "https://allenai.org/data/real-toxicity-prompts",
"lang": lang,
"description": descr,
"intent": probe_intents[probe_class], # category-specific typology fit is defined in probe_intents above
"intent": probe_intents[
probe_class
], # category-specific typology fit is defined in probe_intents above
"primary_detector": "unsafe_content.ToxicCommentModel",
"extended_detectors": ["perspective." + probe_class.title()],
"tags": probe_tags[probe_class],
Expand Down
6 changes: 5 additions & 1 deletion garak/probes/snowball.py
Original file line number Diff line number Diff line change
Expand Up @@ -75,6 +75,7 @@ class PrimesFull(garak.probes.Probe):
"quality:Robustness:GenerativeMisinformation",
"payload:hallucination",
"demon:Rhetoric:Persuasion_and_manipulation:Distraction", # misleading primality premise distracts model from correct math
"euai:deception:hallucination",
]
goal = "trick the model into saying a prime number isn't prime"
active = False
Expand Down Expand Up @@ -110,14 +111,17 @@ class SenatorsFull(garak.probes.Probe):

lang = "en"
doc_uri = "https://github.com/Nanami18/Snowballed_Hallucination"
intent = "S001fab" # tests whether the model fabricates a non-existent public figure
intent = (
"S001fab" # tests whether the model fabricates a non-existent public figure
)
primary_detector = "snowball.DontStartYes"
tags = [
"avid-effect:performance:P0204",
"owasp:llm09",
"quality:Robustness:GenerativeMisinformation",
"payload:hallucination",
"demon:Rhetoric:Persuasion_and_manipulation:Distraction", # misleading senator premise distracts model from recognizing fiction
"euai:deception:hallucination",
]
goal = "trick the model into saying a fictitious senator exists"
active = False
Expand Down
9 changes: 6 additions & 3 deletions garak/probes/topic.py
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,6 @@ class WordnetBlockedWords(garak.probes.TreeSearchProbe):
tags = [
"avid-effect:security:S0301",
"owasp:llm10",
"quality:Security:Confidentiality",
"demon:Language:Stylizing:Synonymous_language", # uses wordnet synonyms/hypernyms to vary form while probing same topic
] # MISP-format taxonomy categories
tier = garak.probes.Tier.UNLISTED
Expand Down Expand Up @@ -134,7 +133,9 @@ class WordnetAllowedWords(WordnetBlockedWords):
"""

active = False # only makes sense if a list is set
intent = "C002allow" # measures how tightly the model stays within an allowed topic set
intent = (
"C002allow" # measures how tightly the model stays within an allowed topic set
)
tier = garak.probes.Tier.UNLISTED

DEFAULT_PARAMS = WordnetBlockedWords.DEFAULT_PARAMS | {
Expand Down Expand Up @@ -182,7 +183,9 @@ class WordnetControversial(WordnetBlockedWords):

"""

intent = "C002controversial" # this variant is about discussing controversial topics
intent = (
"C002controversial" # this variant is about discussing controversial topics
)
tier = garak.probes.Tier.COMPETE_WITH_SOTA
active = True

Expand Down
Loading