Improved consistency check (#203)

app/pipeline/inconsistency_check_pipeline.py

@@ -1,9 +1,10 @@
 import logging
-from typing import Optional
+import re
+
+from typing import Dict, Optional
 
-from langchain_core.output_parsers import StrOutputParser
-from langchain_core.prompts import PromptTemplate
 from langchain_core.runnables import Runnable
+from langchain_core.prompts import PromptTemplate
 from langsmith import traceable
 
 from app.common.PipelineEnum import PipelineEnum
@@ -12,19 +13,25 @@
 from app.llm.langchain.iris_langchain_chat_model import IrisLangchainChatModel
 from app.pipeline import Pipeline
 from app.web.status.status_update import InconsistencyCheckCallback
-from app.pipeline.prompts.inconsistency_check_prompts import basic_prompt
+from app.pipeline.prompts.inconsistency_check_prompts import (
+    solver_prompt,
+    prettify_prompt,
+)
 
 logger = logging.getLogger(__name__)
 
 
 class InconsistencyCheckPipeline(Pipeline):
-    pipeline: Runnable
     llm: IrisLangchainChatModel
     callback: InconsistencyCheckCallback
 
+    solver: Runnable
+    prettify: Runnable
+
     def __init__(self, callback: Optional[InconsistencyCheckCallback] = None):
         super().__init__(implementation_id="inconsistency_check_pipeline")
         completion_args = CompletionArguments(temperature=0, max_tokens=2000)
+
         self.llm = IrisLangchainChatModel(
             request_handler=CapabilityRequestHandler(
                 requirements=RequirementList(
@@ -34,8 +41,12 @@ def __init__(self, callback: Optional[InconsistencyCheckCallback] = None):
             ),
             completion_args=completion_args,
         )
-        self.prompt = PromptTemplate.from_template(basic_prompt)
-        self.pipeline = self.prompt | self.llm | StrOutputParser()
+        self.solver_prompt = PromptTemplate.from_template(solver_prompt)
+        self.solver = self.solver_prompt | self.llm
+
+        self.prettify_prompt = PromptTemplate.from_template(prettify_prompt)
+        self.prettify = self.prettify_prompt | self.llm
+
         self.callback = callback
         self.tokens = []
 
@@ -54,18 +65,56 @@ def __call__(self, dto: InconsistencyCheckPipelineExecutionDTO, **kwargs):
         logger.info("Running inconsistency check pipeline...")
         self.callback.in_progress()
 
-        template_repository = "\n".join(
-            f"<File path='{file_path}'>\n{file_content}</File>"
-            for file_path, file_content in dto.exercise.template_repository.items()
+        # First, for each file in the exercise, we will check for consistency issues via the solver pipeline
+        consistency_issues: Dict[str, str] = {}
+        file_paths = set(dto.exercise.template_repository.keys()) | set(
+            dto.exercise.solution_repository.keys()
         )
+        solver_inputs = [
+            {
+                "file_path": file_path,
+                "problem_statement": dto.exercise.problem_statement,
+                "template_file": dto.exercise.template_repository.get(
+                    file_path, "no file found"
+                ),
+                "solution_file": dto.exercise.solution_repository.get(
+                    file_path, "no file found"
+                ),
+            }
+            for file_path in file_paths
+        ]
+        file_responses = self.solver.map().invoke(solver_inputs)
+        consistency_issues = {
+            file_path: response.content
+            for file_path, response in zip(file_paths, file_responses)
+        }
 
-        response: str = self.pipeline.invoke(
+        # Second, we will prettify the consistency issues and provide a summary using the prettify pipeline
+        formatted_consistency_issues = "\n".join(
+            [
+                f"<PotentialFileIssues filePath=`{file_path}`>\n{issues}\n</PotentialFileIssues>"
+                for file_path, issues in consistency_issues.items()
+            ]
+        )
+        summary_response = self.prettify.invoke(
             {
                 "problem_statement": dto.exercise.problem_statement,
-                "template_repository": template_repository,
+                "consistency_issues": formatted_consistency_issues,
             }
         )
 
-        self._append_tokens(self.llm.tokens, PipelineEnum.IRIS_INCONSISTENCY_CHECK)
+        result = summary_response.content.strip()
 
-        self.callback.done(final_result=response, tokens=self.tokens)
+        # Remove ``` from start and end if exists
+        if result.startswith("```") and result.endswith("```"):
+            result = result[3:-3]
+            if result.startswith("markdown"):
+                result = result[8:]
+            result = result.strip()
+
+        # Remove first heading or heading containing 'Summary of Consistency Issues'
+        result = re.sub(r"^#\s.*?\n", "", result)
+        result = re.sub(r"^#+.*?Summary of Consistency Issues\s*\n", "", result)
+
+        self._append_tokens(self.llm.tokens, PipelineEnum.IRIS_INCONSISTENCY_CHECK)
+        self.callback.done(final_result=result, tokens=self.tokens)

app/pipeline/prompts/inconsistency_check_prompts.py

@@ -1,34 +1,63 @@
-basic_prompt = """\
+solver_prompt = """\
 <Instruction>
-As detail-oriented expert, find inconsistencies between the provided problem statement and the template repository of \
-a programming exercise.
-The student will use the the template repository to write code that solves the problem statement.
-
-Checks:
-- Given the problem statement, identify any missing or incorrect information in the template repository.
-- Given the template repository, identify any missing or incorrect information in the problem statement.
-- Ensure that the theme of the problem statement is consistent with the template repository.
-- Ensure that the problem statement is clear and concise and it covers everything that the student needs to know in \
-order to solve the exercise.
-
-It is not an inconsistency, if the problem statement clearly states that the student is responsible for writing a \
-specific part of the code.
+You are a detail-oriented expert instructor at an Ivy League university ensuring the quality of programming exercises. \
+Your task is to find consistency issues as part of the exercise creation process to make sure that the exercise is \
+without any errors or inconsistencies that might confuse students. Your teaching assistants will use your feedback to \
+improve the exercise.
+
+Parts of a programming exercise:
+ - Problem statement: The description of the exercise containing tasks that the student needs to solve.
+ - Template repository: The starting point from which the student will start solving the exercise.
+ - Solution repository: The sample solution set by the instructor to compare the student's solution against.
+
+To not overburden you, you will be provided with the problem statement and one of the template plus solution files \
+at a time. You need to compare the problem statement with the template file and identify any consistency issues.
+</Instruction>
+
+<ProblemStatement>
+{problem_statement}
+</ProblemStatement>
+
+<TemplateFile path='{file_path}'>
+{template_file}
+</TemplateFile>
+
+<SolutionFile path='{file_path}'>
+{solution_file}
+</SolutionFile>
+
+<Response>
+Respond with any potential consistency issues found in the exercise formatted in markdown. \
+Just provide the easily digestible formatted markdown without other explanations. It is fine to provide no issues if \
+you are confident that the files are consistent.
+</Response>
+"""
+
+prettify_prompt = """\
+<Instruction>
+You are a detail-oriented expert instructor at an Ivy League university ensuring the quality of programming exercises. \
+Your task is to find consistency issues as part of the exercise creation process to make sure that the exercise is \
+without any errors or inconsistencies that might confuse students.
+In a previous step you already found potential consistency issues as part of the exercise creation process on a file \
+level. Now, you need to summarize the issues found in the exercise so the teaching assistants can fix them.
+
+Parts of a programming exercise:
+ - Problem statement: The description of the exercise containing tasks that the student needs to solve.
+ - Template repository: The starting point from which the student will start solving the exercise.
+ - Solution repository: The sample solution set by the instructor to compare the student's solution against.
 </Instruction>
 
-<Problem Statement>
+<ProblemStatement>
 {problem_statement}
-</Problem Statement>
+</ProblemStatement>
 
-<TemplateRepository>
-{template_repository}
-</TemplateRepository>
+<ConsistencyIssues>
+{consistency_issues}
+</ConsistencyIssues>
 
 <Response>
-Be smart about it, give a structured and actionable response that an instructor can use to significantly improve the \
-exercise. Clearly state where the inconsistency lies. Do not make up inconsistencies just to have something to say.
-It needs to be very comprehensive and detailed, imagine some inconsistencies slipped through, students in the exam \
-will be confused and frustrated. This is a high stakes exam, so we need to be very thorough.
-You will be legally responsible for the quality of the exercise, so make sure you do the absolute best job possible, \
-otherwise you will be held accountable in the court of law. Do not quote whole files! 🔫
+Respond with a summary of the consistency issues found in the exercise, stay specific and clear so the issues can be \
+easily fixed by the teaching assistants. Make it clear which file path contains the issues. Just provide the easily \
+digestible formatted markdown without other explanations.
 </Response>
 """