Merge pull request #20 from llm-agents-php/feature/output-formatter

Adds an ability to format LLM response into a given format

Author: butschster

src/Agent/AgentExecutor.php

@@ -27,7 +27,7 @@ public function register(AgentInterface $agent): void
 
     public function get(string $key): AgentInterface
     {
-        if (! $this->has($key)) {
+        if (!$this->has($key)) {
             throw new AgentNotFoundException(\sprintf('Agent with key \'%s\' is not registered.', $key));
         }
 

src/Agent/AgentRegistry.php

@@ -6,7 +6,7 @@
 
 use LLM\Agents\LLM\ContextInterface;
 use LLM\Agents\LLM\OptionsInterface;
-use LLM\Agents\LLM\Prompt\Chat\PromptInterface;
+use LLM\Agents\LLM\Prompt\PromptInterface;
 use LLM\Agents\LLM\PromptContextInterface;
 
 /**

src/AgentExecutor/ExecutionInput.php

@@ -10,10 +10,12 @@
 use LLM\Agents\AgentExecutor\ExecutorInterceptorInterface;
 use LLM\Agents\AgentExecutor\InterceptorHandler;
 use LLM\Agents\LLM\AgentPromptGeneratorInterface;
-use LLM\Agents\LLM\Prompt\Chat\PromptInterface;
+use LLM\Agents\LLM\Prompt\PromptInterface;
 
 /**
  * This interceptor is responsible for generating the prompt for the agent.
+ * If the input does not have a prompt, it will generate one using the prompt generator.
+ * After the execution, it will remove temporary messages (which implement LLM\Agents\LLM\Prompt\Chat\TempMessageInterface) from the prompt.
  */
 final readonly class GeneratePromptInterceptor implements ExecutorInterceptorInterface
 {
@@ -36,6 +38,17 @@ public function execute(
             );
         }
 
-        return $next($input);
+        $execution = $next($input);
+
+        // Remove temporary messages from the prompt.
+        $prompt = $execution->prompt;
+        if ($prompt instanceof \LLM\Agents\LLM\Prompt\Chat\PromptInterface) {
+            $prompt = $prompt->withoutTempMessages();
+        }
+
+        return new Execution(
+            result: $execution->result,
+            prompt: $prompt,
+        );
     }
 }

src/AgentExecutor/Interceptor/GeneratePromptInterceptor.php

@@ -10,6 +10,9 @@
 use LLM\Agents\AgentExecutor\ExecutorInterceptorInterface;
 use LLM\Agents\AgentExecutor\InterceptorHandler;
 
+/**
+ * This interceptor is responsible for injecting the agent's configuration options into the execution options.
+ */
 final readonly class InjectOptionsInterceptor implements ExecutorInterceptorInterface
 {
     public function __construct(

src/AgentExecutor/Interceptor/InjectOptionsInterceptor.php

@@ -11,6 +11,9 @@
 use LLM\Agents\LLM\Response\ChatResponse;
 use LLM\Agents\LLM\Response\ToolCalledResponse;
 
+/**
+ * This interceptor is responsible for injecting the LLM response into the prompt history.
+ */
 final class InjectResponseIntoPromptInterceptor implements ExecutorInterceptorInterface
 {
     public function execute(

src/AgentExecutor/Interceptor/InjectResponseIntoPromptInterceptor.php

@@ -16,6 +16,9 @@
 use LLM\Agents\Tool\ToolInterface;
 use LLM\Agents\Tool\ToolRepositoryInterface;
 
+/**
+ * This interceptor is responsible for injecting the tools into the prompt if the agent has linked tools.
+ */
 final readonly class InjectToolsInterceptor implements ExecutorInterceptorInterface
 {
     public function __construct(

src/AgentExecutor/Interceptor/InjectToolsInterceptor.php

@@ -0,0 +1,83 @@
+<?php
+
+declare(strict_types=1);
+
+namespace LLM\Agents\AgentExecutor\Interceptor;
+
+use LLM\Agents\Agent\Execution;
+use LLM\Agents\AgentExecutor\ExecutionInput;
+use LLM\Agents\AgentExecutor\ExecutorInterceptorInterface;
+use LLM\Agents\AgentExecutor\InterceptorHandler;
+use LLM\Agents\LLM\Exception\FormatterException;
+use LLM\Agents\LLM\Output\EnumFormatter;
+use LLM\Agents\LLM\Output\FormatterInterface;
+use LLM\Agents\LLM\Output\JsonSchemaFormatter;
+use LLM\Agents\LLM\Prompt\PromptInterface;
+use LLM\Agents\LLM\Response\ChatResponse;
+
+/**
+ * This interceptor is responsible for formatting the output of the agent based on the provided output formatter.
+ */
+final readonly class OutputFormatterInterceptor implements ExecutorInterceptorInterface
+{
+    public function __construct(
+        private JsonSchemaFormatter $jsonSchemaFormatter,
+    ) {}
+
+    public function execute(ExecutionInput $input, InterceptorHandler $next): Execution
+    {
+        $outputFormatter = $input->options->get('output_formatter');
+
+        if ($outputFormatter === null) {
+            return $next($input);
+        }
+
+        if (!$input->prompt instanceof PromptInterface) {
+            throw new FormatterException('Prompt must implement PromptInterface');
+        }
+
+        if (!$outputFormatter instanceof FormatterInterface) {
+            $outputFormatter = $this->createFormatter($outputFormatter);
+        }
+
+        $input = $input->withPrompt(
+            $input->prompt->withValues(
+                ['output_format_instruction' => $outputFormatter->getInstruction()],
+            ),
+        );
+
+        return $this->formatResponse($next($input), $outputFormatter);
+    }
+
+    private function formatResponse(Execution $execution, FormatterInterface $outputFormatter): Execution
+    {
+        $result = $execution->result;
+
+        if (!$result instanceof ChatResponse) {
+            return $execution;
+        }
+
+        return new Execution(
+            result: new ChatResponse(
+                content: $outputFormatter->format($result->content),
+            ),
+            prompt: $execution->prompt,
+        );
+    }
+
+    /**
+     * @param non-empty-string|class-string $schema
+     */
+    private function createFormatter(string $schema): FormatterInterface
+    {
+        // If the schema is an existing class, check if it is an enum.
+        if (\class_exists($schema)) {
+            $refl = new \ReflectionClass($schema);
+            if ($refl->isEnum()) {
+                return new EnumFormatter($refl->getName());
+            }
+        }
+
+        return $this->jsonSchemaFormatter->withJsonSchema($schema);
+    }
+}

src/AgentExecutor/Interceptor/OutputFormatterInterceptor.php

@@ -10,6 +10,10 @@
 use LLM\Agents\AgentExecutor\InterceptorHandler;
 use LLM\Agents\LLM\Exception\LimitExceededException;
 
+/**
+ * This interceptor is responsible for retrying the execution if the token limit is exceeded.
+ * It will increment the limit by a specified step and retry the execution.
+ */
 final readonly class TokenLimitRetryInterceptor implements ExecutorInterceptorInterface
 {
     public function __construct(

src/AgentExecutor/Interceptor/TokenLimitRetryInterceptor.php

@@ -14,6 +14,13 @@
 use LLM\Agents\LLM\Response\ToolCalledResponse;
 use LLM\Agents\Tool\ToolExecutor;
 
+/**
+ * This interceptor is responsible for calling the tools if LLM asks for it. After calling the tools, it adds the
+ * tools responses to the prompt history and return the result of tools execution to the LLM.
+ *
+ * If the option 'return_tool_result' is set to true, the interceptor will return the tools result instead of adding
+ * it to the prompt.
+ */
 final readonly class ToolExecutorInterceptor implements ExecutorInterceptorInterface
 {
     public function __construct(

src/AgentExecutor/Interceptor/ToolExecutorInterceptor.php

@@ -6,6 +6,9 @@
 
 use LLM\Agents\Agent\Execution;
 
+/**
+ * @internal
+ */
 final readonly class InterceptorHandler
 {
     public function __construct(

src/AgentExecutor/InterceptorHandler.php

@@ -0,0 +1,10 @@
+<?php
+
+declare(strict_types=1);
+
+namespace LLM\Agents\LLM\Exception;
+
+final class FormatterException extends LLMException
+{
+
+}

src/LLM/Exception/FormatterException.php

@@ -0,0 +1,10 @@
+<?php
+
+declare(strict_types=1);
+
+namespace LLM\Agents\LLM\Exception;
+
+final class InvalidArgumentException extends LLMException
+{
+
+}

src/LLM/Exception/InvalidArgumentException.php

@@ -0,0 +1,63 @@
+<?php
+
+declare(strict_types=1);
+
+namespace LLM\Agents\LLM\Output;
+
+use LLM\Agents\LLM\Exception\FormatterException;
+use LLM\Agents\LLM\Exception\InvalidArgumentException;
+
+/**
+ * This formatter is used to validate that the response is one of the provided options.
+ */
+final readonly class EnumFormatter implements FormatterInterface
+{
+    private \ReflectionEnum $enum;
+    private FormatterInterface $formatter;
+
+    /**
+     * @param class-string $enumClass
+     */
+    public function __construct(
+        string $enumClass,
+    ) {
+        // validate class is type of enum of strings and fetch options
+        try {
+            $this->enum = new \ReflectionEnum($enumClass);
+        } catch (\ReflectionException $e) {
+            throw new InvalidArgumentException("Class {$enumClass} is not a valid enum");
+        }
+
+        if (!$this->enum->getBackingType() instanceof \ReflectionNamedType) {
+            throw new InvalidArgumentException("Enum {$enumClass} is not a valid enum of strings");
+        }
+
+        if ($this->enum->getBackingType()->getName() !== 'string') {
+            throw new InvalidArgumentException("Enum {$enumClass} is not a valid enum of strings");
+        }
+
+        // fetching options
+        $this->formatter = new SelectFormatter(
+            ...\array_map(static fn($option) => $option->value, $this->enum->getConstants()),
+        );
+    }
+
+    public function format(string|\Stringable $output): mixed
+    {
+        $value = $this->formatter->format($output);
+
+        // new enum value
+        foreach ($this->enum->getConstants() as $option) {
+            if ($option->value === $value) {
+                return $option;
+            }
+        }
+
+        throw new FormatterException("Invalid enum value {$value}");
+    }
+
+    public function getInstruction(): ?string
+    {
+        return $this->formatter->getInstruction();
+    }
+}