isort, -plotlyjs, +chartjs

Author: Shai Dvash

examples.py

@@ -6,7 +6,8 @@
 from fuzzy.fuzzer import Fuzzer
 from fuzzy.handlers.attacks.enums import FuzzerAttackMode
 from fuzzy.handlers.attacks.genetic.handler import GeneticAttackParams
-from fuzzy.handlers.classifiers.disapproval.handler import DisapprovalClassifier
+from fuzzy.handlers.classifiers.disapproval.handler import \
+    DisapprovalClassifier
 from fuzzy.handlers.classifiers.harmful_llm.handler import HarmfulLLMClassifier
 from fuzzy.handlers.classifiers.rating.handler import RatingClassifier
 from fuzzy.llm.providers.enums import LLMProvider

fuzzy/fuzzer.py

@@ -11,10 +11,12 @@
 from fuzzy.consts import DATETIME_FORMAT
 from fuzzy.handlers.attacks.base import attack_handler_fm
 from fuzzy.handlers.attacks.enums import FuzzerAttackMode
-from fuzzy.handlers.attacks.proto import AttackSummary, BaseAttackTechniqueHandlerProto
+from fuzzy.handlers.attacks.proto import (AttackSummary,
+                                          BaseAttackTechniqueHandlerProto)
 from fuzzy.handlers.classifiers.base import BaseClassifier
 from fuzzy.handlers.db.adv_attacks import AdversarialAttacksHandler
-from fuzzy.handlers.db.adv_prompts import AdversarialPromptDTO, AdversarialPromptsHandler
+from fuzzy.handlers.db.adv_prompts import (AdversarialPromptDTO,
+                                           AdversarialPromptsHandler)
 from fuzzy.handlers.db.adv_suffixes import AdversarialSuffixesHandler
 from fuzzy.llm.providers.base import BaseLLMProvider
 from fuzzy.llm.providers.enums import LLMProvider

fuzzy/handlers/attack_manager.py

@@ -14,7 +14,8 @@
 import torch.nn as nn
 import torch.nn.functional as F
 from fastchat.model import get_conversation_template
-from transformers import (AutoModelForCausalLM, AutoTokenizer, GPT2LMHeadModel, GPTJForCausalLM, GPTNeoXForCausalLM,
+from transformers import (AutoModelForCausalLM, AutoTokenizer, GPT2LMHeadModel,
+                          GPTJForCausalLM, GPTNeoXForCausalLM,
                           LlamaForCausalLM)
 
 

fuzzy/handlers/attacks/actor_attack/handler.py

@@ -1,18 +1,21 @@
 import logging
 from typing import Any, Final, Optional, Type
+
 from pydantic import BaseModel, Field
 
-from fuzzy.enums import LLMRole
 from fuzzy.consts import DEFAULT_OPEN_SOURCE_MODEL
-from fuzzy.handlers.attacks.base import (BaseAttackTechniqueHandler, BaseAttackTechniqueHandlerException,
+from fuzzy.enums import LLMRole
+from fuzzy.handlers.attacks.actor_attack.prompts import (
+    ACTORS_GENERATION_PROMPT, BEHAVIOR_EXTRACTION_PROMPT,
+    QUESTIONS_GENERATION_PROMPT)
+from fuzzy.handlers.attacks.actor_attack.utils import generate_model_error
+from fuzzy.handlers.attacks.base import (BaseAttackTechniqueHandler,
+                                         BaseAttackTechniqueHandlerException,
                                          attack_handler_fm)
 from fuzzy.handlers.attacks.enums import FuzzerAttackMode
 from fuzzy.handlers.attacks.models import AttackResultEntry
-from fuzzy.llm.providers.base import BaseLLMProvider, BaseLLMProviderException, BaseLLMMessage
-from fuzzy.handlers.attacks.actor_attack.prompts import (BEHAVIOR_EXTRACTION_PROMPT,
-                                                         ACTORS_GENERATION_PROMPT,
-                                                         QUESTIONS_GENERATION_PROMPT)
-from fuzzy.handlers.attacks.actor_attack.utils import generate_model_error
+from fuzzy.llm.providers.base import (BaseLLMMessage, BaseLLMProvider,
+                                      BaseLLMProviderException)
 
 logger = logging.getLogger(__name__)
 

fuzzy/handlers/attacks/artprompt/handler.py

@@ -7,8 +7,11 @@
 from art import text2art
 from pydantic import BaseModel, Field
 
-from fuzzy.handlers.attacks.artprompt.prompts import JAILBREAK_PROMPT, JAILBREAK_SIMPLE_PROMPT, LEARN_FONT_PROMPT
-from fuzzy.handlers.attacks.base import BaseAttackTechniqueHandler, attack_handler_fm
+from fuzzy.handlers.attacks.artprompt.prompts import (JAILBREAK_PROMPT,
+                                                      JAILBREAK_SIMPLE_PROMPT,
+                                                      LEARN_FONT_PROMPT)
+from fuzzy.handlers.attacks.base import (BaseAttackTechniqueHandler,
+                                         attack_handler_fm)
 from fuzzy.handlers.attacks.enums import FuzzerAttackMode
 from fuzzy.handlers.attacks.models import AttackResultEntry
 from fuzzy.handlers.db.adv_prompts import AdversarialPromptDTO

fuzzy/handlers/attacks/ascii_smuggling/handler.py

@@ -3,7 +3,8 @@
 
 from pydantic import BaseModel, Field
 
-from fuzzy.handlers.attacks.base import BaseAttackTechniqueHandler, attack_handler_fm
+from fuzzy.handlers.attacks.base import (BaseAttackTechniqueHandler,
+                                         attack_handler_fm)
 from fuzzy.handlers.attacks.enums import FuzzerAttackMode
 from fuzzy.handlers.attacks.models import AttackResultEntry
 from fuzzy.llm.providers.base import BaseLLMProvider

fuzzy/handlers/attacks/back_to_the_past/handler.py

@@ -3,7 +3,8 @@
 
 from pydantic import BaseModel, Field
 
-from fuzzy.handlers.attacks.base import (BaseAttackTechniqueHandler, BaseAttackTechniqueHandlerException,
+from fuzzy.handlers.attacks.base import (BaseAttackTechniqueHandler,
+                                         BaseAttackTechniqueHandlerException,
                                          attack_handler_fm)
 from fuzzy.handlers.attacks.enums import FuzzerAttackMode
 from fuzzy.handlers.attacks.models import AttackResultEntry

fuzzy/handlers/attacks/base.py

@@ -12,10 +12,12 @@
 from pydantic_core import PydanticUndefinedType, ValidationError
 from tqdm import tqdm
 
-from fuzzy.handlers.attacks.proto import AttackResultEntry, AttackSummary, BaseAttackTechniqueHandlerProto
+from fuzzy.handlers.attacks.proto import (AttackResultEntry, AttackSummary,
+                                          BaseAttackTechniqueHandlerProto)
 from fuzzy.handlers.classifiers.base import BaseClassifier
 from fuzzy.handlers.db.adv_prompts import AdversarialPromptDTO
-from fuzzy.handlers.response_refinement_handler import RefinementException, ResponseRefinementHandler
+from fuzzy.handlers.response_refinement_handler import (
+    RefinementException, ResponseRefinementHandler)
 from fuzzy.llm.models import BaseLLMProviderResponse
 from fuzzy.llm.providers.base import BaseLLMProvider
 from fuzzy.utils.flavor_manager import FlavorManager

fuzzy/handlers/attacks/bon/handler.py

@@ -3,8 +3,11 @@
 
 from pydantic import BaseModel, Field
 
-from fuzzy.handlers.attacks.base import BaseAttackTechniqueHandler, attack_handler_fm
-from fuzzy.handlers.attacks.bon.utils import character_noising, character_scrambling, random_capitalization
+from fuzzy.handlers.attacks.base import (BaseAttackTechniqueHandler,
+                                         attack_handler_fm)
+from fuzzy.handlers.attacks.bon.utils import (character_noising,
+                                              character_scrambling,
+                                              random_capitalization)
 from fuzzy.handlers.attacks.enums import FuzzerAttackMode
 from fuzzy.handlers.attacks.models import AttackResultEntry
 from fuzzy.llm.providers.base import BaseLLMProvider

fuzzy/handlers/attacks/crescendo/handler.py

@@ -3,16 +3,18 @@
 
 from pydantic import BaseModel, Field
 
-from fuzzy.enums import LLMRole
 from fuzzy.consts import DEFAULT_OPEN_SOURCE_MODEL
-from fuzzy.handlers.attacks.base import (BaseAttackTechniqueHandler, BaseAttackTechniqueHandlerException,
+from fuzzy.enums import LLMRole
+from fuzzy.handlers.attacks.base import (BaseAttackTechniqueHandler,
+                                         BaseAttackTechniqueHandlerException,
                                          attack_handler_fm)
-from fuzzy.handlers.attacks.crescendo.prompts import (QUESTION_GENERATION_PROMPT_TEMPLATE_PREFIX,
-                                                      QUESTION_GENERATION_CONCISE_QUESTIONS,
-                                                      QUESTION_GENERATION_PROMPT_TEMPLATE)
+from fuzzy.handlers.attacks.crescendo.prompts import (
+    QUESTION_GENERATION_CONCISE_QUESTIONS, QUESTION_GENERATION_PROMPT_TEMPLATE,
+    QUESTION_GENERATION_PROMPT_TEMPLATE_PREFIX)
 from fuzzy.handlers.attacks.enums import FuzzerAttackMode
 from fuzzy.handlers.attacks.models import AttackResultEntry
-from fuzzy.llm.providers.base import BaseLLMProvider, BaseLLMProviderException, BaseLLMMessage
+from fuzzy.llm.providers.base import (BaseLLMMessage, BaseLLMProvider,
+                                      BaseLLMProviderException)
 
 logger = logging.getLogger(__name__)
 

fuzzy/handlers/attacks/dan/handler.py

@@ -1,8 +1,10 @@
 import logging
 from typing import Any, Optional, Type
+
 from pydantic import BaseModel, Field
 
-from fuzzy.handlers.attacks.base import BaseAttackTechniqueHandler, attack_handler_fm
+from fuzzy.handlers.attacks.base import (BaseAttackTechniqueHandler,
+                                         attack_handler_fm)
 from fuzzy.handlers.attacks.enums import FuzzerAttackMode
 from fuzzy.handlers.attacks.models import AttackResultEntry
 from fuzzy.llm.providers.base import BaseLLMProvider

fuzzy/handlers/attacks/default/handler.py

@@ -3,7 +3,8 @@
 
 from pydantic import BaseModel
 
-from fuzzy.handlers.attacks.base import BaseAttackTechniqueHandler, attack_handler_fm
+from fuzzy.handlers.attacks.base import (BaseAttackTechniqueHandler,
+                                         attack_handler_fm)
 from fuzzy.handlers.attacks.enums import FuzzerAttackMode
 from fuzzy.handlers.attacks.models import AttackResultEntry
 from fuzzy.llm.providers.base import BaseLLMProvider

fuzzy/handlers/attacks/genetic/handler.py

@@ -1,16 +1,19 @@
 # type: ignore
 import logging
-from typing import Any, Optional, Type, Union
+from typing import Any, Optional, Union
 
 import numpy as np
 import pygad
-from pydantic import BaseModel, Field
+from pydantic import BaseModel
 
-from fuzzy.handlers.attacks.base import BaseAttackTechniqueHandler, attack_handler_fm
+from fuzzy.handlers.attacks.base import (BaseAttackTechniqueHandler,
+                                         attack_handler_fm)
 from fuzzy.handlers.attacks.enums import FuzzerAttackMode
 from fuzzy.handlers.attacks.models import AttackResultEntry
-from fuzzy.handlers.classifiers.cosine_similarity.handler import CosineSimilarityClassifier
-from fuzzy.handlers.classifiers.disapproval.handler import DisapprovalClassifier
+from fuzzy.handlers.classifiers.cosine_similarity.handler import \
+    CosineSimilarityClassifier
+from fuzzy.handlers.classifiers.disapproval.handler import \
+    DisapprovalClassifier
 from fuzzy.handlers.db.adv_prompts import AdversarialPromptDTO
 from fuzzy.handlers.tokenizers.handler import TokensHandler
 from fuzzy.llm.models import BaseLLMProviderResponse

fuzzy/handlers/attacks/gpt_fuzzer/handler.py

@@ -2,15 +2,17 @@
 import random
 import time
 from typing import Any, Final, Optional, Type
+
 from pydantic import BaseModel, Field
 
 from fuzzy.consts import DEFAULT_OPEN_SOURCE_MODEL
-from fuzzy.handlers.attacks.gpt_fuzzer.prompt_templates import (GENERATE_ACTION_PROMPT, CROSSOVER_ACTION_PROMPT,
-                                                                EXPAND_ACTION_PROMPT, SHORTEN_ACTION_PROMPT,
-                                                                REPHRASE_ACTION_PROMPT, ATTACKING_PROMPTS_TEMPLATES)
-from fuzzy.handlers.attacks.base import (BaseAttackTechniqueHandler, BaseAttackTechniqueHandlerException,
+from fuzzy.handlers.attacks.base import (BaseAttackTechniqueHandler,
+                                         BaseAttackTechniqueHandlerException,
                                          attack_handler_fm)
 from fuzzy.handlers.attacks.enums import FuzzerAttackMode
+from fuzzy.handlers.attacks.gpt_fuzzer.prompt_templates import (
+    ATTACKING_PROMPTS_TEMPLATES, CROSSOVER_ACTION_PROMPT, EXPAND_ACTION_PROMPT,
+    GENERATE_ACTION_PROMPT, REPHRASE_ACTION_PROMPT, SHORTEN_ACTION_PROMPT)
 from fuzzy.handlers.attacks.models import AttackResultEntry
 from fuzzy.llm.models import BaseLLMProviderResponse
 from fuzzy.llm.providers.base import BaseLLMProvider

fuzzy/handlers/attacks/hallucinations/handler.py

@@ -3,7 +3,8 @@
 
 from pydantic import BaseModel
 
-from fuzzy.handlers.attacks.base import BaseAttackTechniqueHandler, attack_handler_fm
+from fuzzy.handlers.attacks.base import (BaseAttackTechniqueHandler,
+                                         attack_handler_fm)
 from fuzzy.handlers.attacks.enums import FuzzerAttackMode
 from fuzzy.handlers.attacks.hallucinations.prompt import PROMPT_TEMPLATE
 from fuzzy.handlers.attacks.models import AttackResultEntry

fuzzy/handlers/attacks/manyshot/handler.py

@@ -5,7 +5,8 @@
 import pandas as pd
 from pydantic import BaseModel, Field
 
-from fuzzy.handlers.attacks.base import BaseAttackTechniqueHandler, attack_handler_fm
+from fuzzy.handlers.attacks.base import (BaseAttackTechniqueHandler,
+                                         attack_handler_fm)
 from fuzzy.handlers.attacks.enums import FuzzerAttackMode
 from fuzzy.handlers.attacks.models import AttackResultEntry
 from fuzzy.handlers.db.adv_prompts import AdversarialPromptDTO

fuzzy/handlers/attacks/paraphraser/handler.py

@@ -7,11 +7,15 @@
 
 from fuzzy.consts import DEFAULT_OPEN_SOURCE_MODEL
 from fuzzy.enums import LLMRole
-from fuzzy.handlers.attacks.base import AttackBreakWhen, BaseAttackTechniqueHandler, attack_handler_fm
+from fuzzy.handlers.attacks.base import (AttackBreakWhen,
+                                         BaseAttackTechniqueHandler,
+                                         attack_handler_fm)
 from fuzzy.handlers.attacks.enums import FuzzerAttackMode
 from fuzzy.handlers.attacks.paraphraser.enums import PersuasiveActor
-from fuzzy.handlers.attacks.paraphraser.models import PersuasiveIterationTargetOutput, PersusiveIterationResponse
-from fuzzy.handlers.attacks.paraphraser.prompts import get_attacker_system_prompt
+from fuzzy.handlers.attacks.paraphraser.models import (
+    PersuasiveIterationTargetOutput, PersusiveIterationResponse)
+from fuzzy.handlers.attacks.paraphraser.prompts import \
+    get_attacker_system_prompt
 from fuzzy.handlers.attacks.proto import AttackResultEntry
 from fuzzy.handlers.classifiers.enums import Classifier
 from fuzzy.llm.providers.base import BaseLLMMessage

fuzzy/handlers/attacks/piglatin/handler.py

@@ -3,10 +3,13 @@
 
 from pydantic import BaseModel
 
-from fuzzy.handlers.attacks.base import BaseAttackTechniqueHandler, attack_handler_fm
+from fuzzy.handlers.attacks.base import (BaseAttackTechniqueHandler,
+                                         attack_handler_fm)
 from fuzzy.handlers.attacks.enums import FuzzerAttackMode
 from fuzzy.handlers.attacks.models import AttackResultEntry
-from fuzzy.handlers.attacks.piglatin.utils import english_to_pig_latin, is_pig_latin, pig_latin_to_english
+from fuzzy.handlers.attacks.piglatin.utils import (english_to_pig_latin,
+                                                   is_pig_latin,
+                                                   pig_latin_to_english)
 from fuzzy.llm.providers.base import BaseLLMProvider
 
 logger = logging.getLogger(__name__)

fuzzy/handlers/attacks/please/handler.py

@@ -1,8 +1,10 @@
 import logging
 from typing import Any, Optional, Type
+
 from pydantic import BaseModel, Field
 
-from fuzzy.handlers.attacks.base import BaseAttackTechniqueHandler, attack_handler_fm
+from fuzzy.handlers.attacks.base import (BaseAttackTechniqueHandler,
+                                         attack_handler_fm)
 from fuzzy.handlers.attacks.enums import FuzzerAttackMode
 from fuzzy.handlers.attacks.models import AttackResultEntry
 from fuzzy.llm.providers.base import BaseLLMProvider

fuzzy/handlers/attacks/taxonomy/handler.py

@@ -4,7 +4,8 @@
 
 from pydantic import BaseModel, Field
 
-from fuzzy.handlers.attacks.base import BaseAttackTechniqueHandler, attack_handler_fm
+from fuzzy.handlers.attacks.base import (BaseAttackTechniqueHandler,
+                                         attack_handler_fm)
 from fuzzy.handlers.attacks.enums import FuzzerAttackMode
 from fuzzy.handlers.attacks.models import AttackResultEntry
 from fuzzy.handlers.attacks.taxonomy.prompts import PERSUASION_PROMPT

fuzzy/handlers/attacks/thought_experiment/handler.py

@@ -1,8 +1,10 @@
 import logging
 from typing import Any, Optional, Type
+
 from pydantic import BaseModel, Field
 
-from fuzzy.handlers.attacks.base import BaseAttackTechniqueHandler, attack_handler_fm
+from fuzzy.handlers.attacks.base import (BaseAttackTechniqueHandler,
+                                         attack_handler_fm)
 from fuzzy.handlers.attacks.enums import FuzzerAttackMode
 from fuzzy.handlers.attacks.models import AttackResultEntry
 from fuzzy.llm.providers.base import BaseLLMProvider

fuzzy/handlers/attacks/word_game/handler.py

@@ -6,14 +6,15 @@
 from fuzzy.consts import DEFAULT_OPEN_SOURCE_MODEL
 from fuzzy.enums import LLMRole
 from fuzzy.handlers.attacks.base import (BaseAttackTechniqueHandler,
-                                         attack_handler_fm, BaseAttackTechniqueHandlerException)
+                                         BaseAttackTechniqueHandlerException,
+                                         attack_handler_fm)
 from fuzzy.handlers.attacks.enums import FuzzerAttackMode
 from fuzzy.handlers.attacks.models import AttackResultEntry
-from fuzzy.handlers.attacks.word_game.word_game_prompts import (WORD_IDENTIFICATION_PROMPT,
-                                                                REPHRASING_PROMPT,HINTS_GENERATION_PROMPT,
-                                                                WORD_GAME_PROMPT,WORD_GAME_PLUS_PROMPT)
+from fuzzy.handlers.attacks.word_game.word_game_prompts import (
+    HINTS_GENERATION_PROMPT, REPHRASING_PROMPT, WORD_GAME_PLUS_PROMPT,
+    WORD_GAME_PROMPT, WORD_IDENTIFICATION_PROMPT)
 from fuzzy.llm.models import BaseLLMProviderResponse
-from fuzzy.llm.providers.base import BaseLLMProvider, BaseLLMMessage
+from fuzzy.llm.providers.base import BaseLLMMessage, BaseLLMProvider
 from fuzzy.utils.custom_logging_formatter import CustomFormatter
 
 logger = logging.getLogger(__name__)

fuzzy/handlers/classifiers/__init__.py

@@ -1,13 +1,13 @@
+from .aws_bedrock.handler import AWSBedrockClassifier
+from .aws_guardrails.handler import AWSGuardrailsClassifier
+from .azure_content_safety.handler import AzureContentSafetyClassifier
 from .committee.handler import CommitteeClassifier
 from .cosine_similarity.handler import CosineSimilarityClassifier
-from .obvious_negative.handler import ObviousNegativeClassifier
 from .disapproval.handler import DisapprovalClassifier
-from .llm_response_sentiment.handler import LLMResponseSentimentClassifier
 from .generic_llm.handler import LLMEvaluatorClassifier
 from .harmful_llm.handler import HarmfulLLMClassifier
+from .llm_response_sentiment.handler import LLMResponseSentimentClassifier
+from .obvious_negative.handler import ObviousNegativeClassifier
 from .openai_moderation.handler import OpenAIModerationClassifier
-from .aws_guardrails.handler import AWSGuardrailsClassifier
-from .aws_bedrock.handler import AWSBedrockClassifier
-from .azure_content_safety.handler import AzureContentSafetyClassifier
 from .rating.handler import RatingClassifier
 from .sentiment.handler import HarmfulSentimentClassifier

fuzzy/handlers/classifiers/aws_bedrock/handler.py

@@ -4,7 +4,6 @@
 
 import boto3
 
-from fuzzy.handlers.classifiers.aws_guardrails.handler import AWSGuardrailsClassifier
 from fuzzy.handlers.classifiers.base import BaseClassifier, classifiers_fm
 from fuzzy.handlers.classifiers.enums import Classifier
 

fuzzy/handlers/classifiers/committee/handler.py

@@ -4,7 +4,9 @@
 from typing import Any, Coroutine
 
 from fuzzy.handlers.classifiers.base import BaseClassifier, classifiers_fm
-from fuzzy.handlers.classifiers.committee.prompts import COH_PROMPT, CON_PROMPT, REL_PROMPT
+from fuzzy.handlers.classifiers.committee.prompts import (COH_PROMPT,
+                                                          CON_PROMPT,
+                                                          REL_PROMPT)
 from fuzzy.handlers.classifiers.enums import Classifier
 from fuzzy.handlers.classifiers.harmful_llm.handler import HarmfulLLMClassifier
 from fuzzy.llm.models import BaseLLMProviderResponse

fuzzy/handlers/classifiers/cosine_similarity/handler.py

@@ -4,7 +4,8 @@
 
 from fuzzy.handlers.classifiers.base import BaseClassifier, classifiers_fm
 from fuzzy.handlers.classifiers.enums import Classifier
-from fuzzy.handlers.sentence_similarity_analyzer import CosineSimilarityAnalysis, SentenceSimilarityAnalyzer
+from fuzzy.handlers.sentence_similarity_analyzer import (
+    CosineSimilarityAnalysis, SentenceSimilarityAnalyzer)
 
 logger = logging.getLogger(__name__)
 

fuzzy/handlers/classifiers/llm_response_sentiment/handler.py

@@ -4,7 +4,8 @@
 
 from fuzzy.handlers.classifiers.base import BaseClassifier, classifiers_fm
 from fuzzy.handlers.classifiers.enums import Classifier
-from fuzzy.handlers.text_analyzer import LLMResponseSentimentAnalysis, ZeroShotAnalyzer
+from fuzzy.handlers.text_analyzer import (LLMResponseSentimentAnalysis,
+                                          ZeroShotAnalyzer)
 
 logger = logging.getLogger(__name__)
 

fuzzy/handlers/classifiers/obvious_negative/handler.py

@@ -1,4 +1,5 @@
 from typing import Any
+
 from fuzzy.handlers.classifiers.base import BaseClassifier, classifiers_fm
 from fuzzy.handlers.classifiers.enums import Classifier