[AI Test Tool] Add Multi-Turn Accuracy & Export

src/Tools/AI Test Toolkit/src/AITTestContext.Codeunit.al

@@ -123,6 +123,15 @@ codeunit 149044 "AIT Test Context"
         AITTestContextImpl.SetTestMetric(TestMetric);
     end;
 
+    /// <summary>
+    /// Sets the accuracy of the test.
+    /// </summary>
+    /// <param name="Accuracy">The accuracy as a decimal between 0 and 1.</param>
+    procedure SetAccuracy(Accuracy: Decimal)
+    begin
+        AITTestContextImpl.SetAccuracy(Accuracy);
+    end;
+
     var
         AITTestContextImpl: Codeunit "AIT Test Context Impl.";
 }

src/Tools/AI Test Toolkit/src/AITTestContextImpl.Codeunit.al

@@ -18,9 +18,12 @@ codeunit 149043 "AIT Test Context Impl."
     var
         AITTestSuiteMgt: Codeunit "AIT Test Suite Mgt.";
         GlobalTestOutputJson: Codeunit "Test Output Json";
+        GlobalAccuracy: Decimal;
         CurrentTurn: Integer;
         NumberOfTurns: Integer;
         IsMultiTurn: Boolean;
+        AccuracySetManually: Boolean;
+        AccuracyErr: Label 'Accuracy must be between 0 and 1.';
         AnswerTok: Label 'answer', Locked = true;
         ContextTok: Label 'context', Locked = true;
         GroundTruthTok: Label 'ground_truth', Locked = true;
@@ -146,6 +149,34 @@ codeunit 149043 "AIT Test Context Impl."
         SetSuiteTestOutput(CurrentTestOutputJson.ToText());
     end;
 
+    /// <summary>
+    /// Sets the accuracy of the test.
+    /// </summary>
+    /// <param name="Accuracy">The accuracy as a decimal between 0 and 1.</param>
+    procedure SetAccuracy(Accuracy: Decimal)
+    begin
+        if (Accuracy < 0) or (Accuracy > 1) then
+            Error(AccuracyErr);
+
+        AccuracySetManually := true;
+        GlobalAccuracy := Accuracy;
+    end;
+
+    /// <summary>
+    /// Gets the accuracy of the test. Can only be retrieved if the accuracy of the test was already set manually.
+    /// </summary>
+    /// <param name="Accuracy">The accuracy as a decimal between 0 and 1.</param>
+    /// <returns>True if it was possible to get the accuracy, false otherwise.</returns>
+    procedure GetAccuracy(var Accuracy: Decimal): Boolean
+    begin
+        if AccuracySetManually then begin
+            Accuracy := GlobalAccuracy;
+            exit(true);
+        end;
+
+        exit(false);
+    end;
+
     /// <summary>
     /// Sets to next turn for multiturn testing.
     /// </summary>
@@ -155,7 +186,7 @@ codeunit 149043 "AIT Test Context Impl."
         if not IsMultiTurn then
             exit(false);
 
-        if CurrentTurn + 1 > NumberOfTurns then
+        if CurrentTurn > NumberOfTurns then
             exit(false);
 
         CurrentTurn := CurrentTurn + 1;
@@ -164,14 +195,23 @@ codeunit 149043 "AIT Test Context Impl."
     end;
 
     /// <summary>
-    /// Gets the current turn for multiturn testing. Turns start from turn 0.
+    /// Gets the current turn for multiturn testing. Turns start from turn 1.
     /// </summary>
     /// <returns>The current turn number.</returns>
     procedure GetCurrentTurn(): Integer
     begin
         exit(CurrentTurn);
     end;
 
+    /// <summary>
+    /// Gets the total number of turns for multiturn testing.
+    /// </summary>
+    /// <returns>The total number of turns for the line.</returns>
+    procedure GetNumberOfTurns(): Integer
+    begin
+        exit(NumberOfTurns);
+    end;
+
     /// <summary>
     /// This method starts the scope of the Run Procedure scenario.
     /// </summary>
@@ -205,12 +245,16 @@ codeunit 149043 "AIT Test Context Impl."
         TestInput: Codeunit "Test Input";
         TurnsInputJson: Codeunit "Test Input Json";
     begin
-        CurrentTurn := 0;
+        AccuracySetManually := false;
+        GlobalAccuracy := 0;
+        CurrentTurn := 1;
         GlobalTestOutputJson.Initialize();
         TurnsInputJson := TestInput.GetTestInput().ElementExists(TurnsTok, IsMultiTurn);
 
         if IsMultiTurn then
-            NumberOfTurns := TurnsInputJson.GetElementCount() - 1;
+            NumberOfTurns := TurnsInputJson.GetElementCount()
+        else
+            NumberOfTurns := 1;
     end;
 
     /// <summary>
@@ -223,7 +267,7 @@ codeunit 149043 "AIT Test Context Impl."
         TestInput: Codeunit "Test Input";
     begin
         if IsMultiTurn then
-            TestInputJson := TestInput.GetTestInput(TurnsTok).ElementAt(CurrentTurn).Element(ElementName)
+            TestInputJson := TestInput.GetTestInput(TurnsTok).ElementAt(CurrentTurn - 1).Element(ElementName)
         else
             TestInputJson := TestInput.GetTestInput(ElementName);
     end;

src/Tools/AI Test Toolkit/src/AITTestRunIteration.Codeunit.al

@@ -21,6 +21,9 @@ codeunit 149042 "AIT Test Run Iteration"
         GlobalTestMethodLine: Record "Test Method Line";
         NoOfInsertedLogEntries: Integer;
         GlobalAITokenUsedByLastTestMethodLine: Integer;
+        GlobalNumberOfTurnsForLastTestMethodLine: Integer;
+        GlobalNumberOfTurnsPassedForLastTestMethodLine: Integer;
+        GlobalTestAccuracy: Decimal;
         GlobalSessionAITokenUsed: Integer;
 
     trigger OnRun()
@@ -124,6 +127,21 @@ codeunit 149042 "AIT Test Run Iteration"
         exit(GlobalAITokenUsedByLastTestMethodLine);
     end;
 
+    procedure GetNumberOfTurnsForLastTestMethodLine(): Integer
+    begin
+        exit(GlobalNumberOfTurnsForLastTestMethodLine);
+    end;
+
+    procedure GetNumberOfTurnsPassedForLastTestMethodLine(): Integer
+    begin
+        exit(GlobalNumberOfTurnsPassedForLastTestMethodLine);
+    end;
+
+    procedure GetAccuracyForLastTestMethodLine(): Decimal
+    begin
+        exit(GlobalTestAccuracy);
+    end;
+
     [InternalEvent(false)]
     procedure OnBeforeRunIteration(var AITTestSuite: Record "AIT Test Suite"; var AITTestMethodLine: Record "AIT Test Method Line")
     begin
@@ -144,6 +162,14 @@ codeunit 149042 "AIT Test Run Iteration"
 
         // Update AI Token Consumption
         GlobalAITokenUsedByLastTestMethodLine := 0;
+
+        // Update Turns
+        GlobalNumberOfTurnsPassedForLastTestMethodLine := 0;
+        GlobalNumberOfTurnsForLastTestMethodLine := 1;
+
+        // Update Test Accuracy
+        GlobalTestAccuracy := 0;
+
         GlobalSessionAITokenUsed := AOAIToken.GetTotalServerSessionTokensConsumed();
 
         AITContextCU.StartRunProcedureScenario();
@@ -154,6 +180,7 @@ codeunit 149042 "AIT Test Run Iteration"
     var
         AITContextCU: Codeunit "AIT Test Context Impl.";
         AOAIToken: Codeunit "AOAI Token";
+        Accuracy: Decimal;
     begin
         if ActiveAITTestSuite.Code = '' then
             exit;
@@ -166,6 +193,19 @@ codeunit 149042 "AIT Test Run Iteration"
         // Update AI Token Consumption
         GlobalAITokenUsedByLastTestMethodLine := AOAIToken.GetTotalServerSessionTokensConsumed() - GlobalSessionAITokenUsed;
 
+        // Update Turns
+        GlobalNumberOfTurnsForLastTestMethodLine := AITContextCU.GetNumberOfTurns();
+        GlobalNumberOfTurnsPassedForLastTestMethodLine := AITContextCU.GetCurrentTurn();
+
+        if not IsSuccess then
+            GlobalNumberOfTurnsPassedForLastTestMethodLine -= 1;
+
+        // Update Test Accuracy
+        if AITContextCU.GetAccuracy(Accuracy) then
+            GlobalTestAccuracy := Accuracy
+        else
+            GlobalTestAccuracy := GlobalNumberOfTurnsPassedForLastTestMethodLine / GlobalNumberOfTurnsForLastTestMethodLine;
+
         AITContextCU.EndRunProcedureScenario(CurrentTestMethodLine, IsSuccess);
         Commit();
     end;

src/Tools/AI Test Toolkit/src/Logs/AITLogEntries.Page.al

@@ -68,6 +68,23 @@ page 149033 "AIT Log Entries"
                 {
                     StyleExpr = StatusStyleExpr;
                 }
+                field(Accuracy; Rec."Test Method Line Accuracy")
+                {
+                }
+                field("No. of Turns Passed"; Rec."No. of Turns Passed")
+                {
+                    Visible = false;
+                }
+                field("No. of Turns"; Rec."No. of Turns")
+                {
+                    Visible = false;
+                }
+                field(TurnsText; TurnsText)
+                {
+                    StyleExpr = TurnsStyleExpr;
+                    Caption = 'No. of Turns Passed';
+                    ToolTip = 'Specifies the number of turns that passed out of the total number of turns.';
+                }
                 field("Orig. Status"; Rec."Original Status")
                 {
                     Visible = false;
@@ -253,6 +270,19 @@ page 149033 "AIT Log Entries"
                     Page.Run(Page::"AIT Test Data Compare", Rec);
                 end;
             }
+            action("Download Test Summary")
+            {
+                Caption = 'Download Test Summary';
+                Image = Export;
+                ToolTip = 'Downloads a summary of the test results.';
+
+                trigger OnAction()
+                var
+                    AITTestSuiteMgt: Codeunit "AIT Test Suite Mgt.";
+                begin
+                    AITTestSuiteMgt.DownloadTestSummary(Rec);
+                end;
+            }
         }
         area(Promoted)
         {
@@ -279,6 +309,9 @@ page 149033 "AIT Log Entries"
                 actionref("View Test Data_Promoted"; "View Test Data")
                 {
                 }
+                actionref("Export Results_Promoted"; "Download Test Summary")
+                {
+                }
             }
         }
     }
@@ -287,20 +320,26 @@ page 149033 "AIT Log Entries"
         ClickToShowLbl: Label 'Show data input';
         DoYouWantToDeleteQst: Label 'Do you want to delete all entries within the filter?';
         InputText: Text;
+        TurnsText: Text;
         OutputText: Text;
         ErrorMessage: Text;
         ErrorCallStack: Text;
         StatusStyleExpr: Text;
+        TurnsStyleExpr: Text;
         TestRunDuration: Duration;
         IsFilteredToErrors: Boolean;
         ShowSensitiveData: Boolean;
 
     trigger OnAfterGetRecord()
+    var
+        AITTestSuiteMgt: Codeunit "AIT Test Suite Mgt.";
     begin
         TestRunDuration := Rec."Duration (ms)";
+        TurnsText := AITTestSuiteMgt.GetTurnsAsText(Rec);
         SetInputOutputDataFields();
         SetErrorFields();
         SetStatusStyleExpr();
+        SetTurnsStyleExpr();
     end;
 
     local procedure SetStatusStyleExpr()
@@ -315,6 +354,18 @@ page 149033 "AIT Log Entries"
         end;
     end;
 
+    local procedure SetTurnsStyleExpr()
+    begin
+        case Rec."No. of Turns Passed" of
+            Rec."No. of Turns":
+                TurnsStyleExpr := 'Favorable';
+            0:
+                TurnsStyleExpr := 'Unfavorable';
+            else
+                TurnsStyleExpr := 'Ambiguous';
+        end;
+    end;
+
     local procedure SetErrorFields()
     begin
         ErrorMessage := '';

src/Tools/AI Test Toolkit/src/Logs/AITLogEntry.Table.al

@@ -159,6 +159,21 @@ table 149034 "AIT Log Entry"
         {
             Caption = 'Output Data';
         }
+        field(40; "No. of Turns"; Integer)
+        {
+            Caption = 'Total number of turns';
+            ToolTip = 'Specifies the total number of turns.';
+        }
+        field(41; "No. of Turns Passed"; Integer)
+        {
+            Caption = 'Number of turns passed';
+            ToolTip = 'Specifies the number of turns passed.';
+        }
+        field(45; "Test Method Line Accuracy"; Decimal)
+        {
+            Caption = 'Test Method Line Accuracy';
+            ToolTip = 'Specifies the accuracy of the test line. The accuracy is calculated as the percentage of turns that passed or can be set manually in the test.';
+        }
         field(50; "Tokens Consumed"; Integer)
         {
             Caption = 'Total Tokens Consumed';

src/Tools/AI Test Toolkit/src/Logs/AITRunHistory.Page.al

@@ -121,6 +121,12 @@ page 149032 "AIT Run History"
                             AITLogEntryCodeunit.DrillDownFailedAITLogEntries(Rec."Test Suite Code", Rec."Line No. Filter", Rec.Version);
                         end;
                     }
+                    field("Accuracy - By Version"; Rec."Accuracy Per Version")
+                    {
+                        Visible = ViewBy = ViewBy::Version;
+                        Caption = 'Accuracy';
+                        ToolTip = 'Specifies the average accuracy of the version.';
+                    }
                     field("Duration - By Version"; Rec."Total Duration (ms)")
                     {
                         Visible = ViewBy = ViewBy::Version;
@@ -161,6 +167,12 @@ page 149032 "AIT Run History"
                             AITLogEntryCodeunit.DrillDownFailedAITLogEntries(Rec."Test Suite Code", Rec."Line No. Filter", Rec.Tag);
                         end;
                     }
+                    field("Accuracy - By Tag"; Rec."Accuracy - By Tag")
+                    {
+                        Visible = ViewBy = ViewBy::Tag;
+                        Caption = 'Accuracy';
+                        ToolTip = 'Specifies the average accuracy of the tag.';
+                    }
                     field("Duration - By Tag"; Rec."Total Duration (ms) - By Tag")
                     {
                         Visible = ViewBy = ViewBy::Tag;

src/Tools/AI Test Toolkit/src/Logs/AITRunHistory.Table.al

@@ -73,6 +73,14 @@ table 149036 "AIT Run History"
             FieldClass = FlowField;
             CalcFormula = sum("AIT Log Entry"."Tokens Consumed" where("Test Suite Code" = field("Test Suite Code"), Version = field("Version"), "Test Method Line No." = field("Line No. Filter"), Operation = const('Run Procedure'), "Procedure Name" = filter(<> '')));
         }
+        field(14; "Accuracy Per Version"; Decimal)
+        {
+            Caption = 'Accuracy';
+            ToolTip = 'Specifies the average accuracy of the version.';
+            Editable = false;
+            FieldClass = FlowField;
+            CalcFormula = average("AIT Log Entry"."Test Method Line Accuracy" where("Test Suite Code" = field("Test Suite Code"), "Test Method Line No." = field("Line No. Filter"), Version = field("Version"), Operation = const('Run Procedure'), "Procedure Name" = filter(<> '')));
+        }
         field(20; "No. of Tests Executed - By Tag"; Integer)
         {
             Caption = 'No. of Tests Executed';
@@ -105,6 +113,14 @@ table 149036 "AIT Run History"
             FieldClass = FlowField;
             CalcFormula = sum("AIT Log Entry"."Tokens Consumed" where("Test Suite Code" = field("Test Suite Code"), "Test Method Line No." = field("Line No. Filter"), Tag = field(Tag), Operation = const('Run Procedure'), "Procedure Name" = filter(<> '')));
         }
+        field(24; "Accuracy - By Tag"; Decimal)
+        {
+            Caption = 'Accuracy';
+            ToolTip = 'Specifies the average accuracy of the tag.';
+            Editable = false;
+            FieldClass = FlowField;
+            CalcFormula = average("AIT Log Entry"."Test Method Line Accuracy" where("Test Suite Code" = field("Test Suite Code"), "Test Method Line No." = field("Line No. Filter"), Tag = field(Tag), Operation = const('Run Procedure'), "Procedure Name" = filter(<> '')));
+        }
     }
 
     keys