Merge remote-tracking branch 'origin/dev' into issue/OSOE-603

Lombiq.Tests.UI.Samples/Readme.md

@@ -26,6 +26,7 @@ For general details about and on using the Toolbox see the [root Readme](../Read
 - [Database snapshot tests](Tests/DatabaseSnapshotTests.cs)
 - [Basic visual verification tests](Tests/BasicVisualVerificationTests.cs)
 - [Testing in tenants](Tests/TenantTests.cs)
+- [Interactive mode](Tests/InteractiveModeTests.cs)
 
 ## Adding new tutorials
 

Lombiq.Tests.UI.Samples/Tests/InteractiveModeTests.cs

@@ -0,0 +1,76 @@
+using Lombiq.Tests.UI.Attributes;
+using Lombiq.Tests.UI.Extensions;
+using Lombiq.Tests.UI.Services;
+using OpenQA.Selenium;
+using Shouldly;
+using System.Diagnostics.CodeAnalysis;
+using System.Threading.Tasks;
+using Xunit;
+using Xunit.Abstractions;
+
+namespace Lombiq.Tests.UI.Samples.Tests;
+
+// Sometimes you want to debug the test session and assuming direct control would be nice. But you can't just drop a
+// breakpoint in the test, since the Orchard Core webapp and the test are the same process so it would pause both. The
+// `context.SwitchToInteractiveAsync()` extension method opens a new tab with info about the interactive mode and then
+// causes the test thread to wait until you've clicked on the "Continue Test" button in this tab. During that time you
+// can interact with OC as if it was a normal execution.
+// Note: this extension depends on Lombiq.Tests.UI.Shortcuts being enabled in your OC app.
+public class InteractiveModeTests : UITestBase
+{
+    public InteractiveModeTests(ITestOutputHelper testOutputHelper)
+        : base(testOutputHelper)
+    {
+    }
+
+    // If you want to try it out yourself, just remove the "Skip" parameter and run this test.
+    [Theory(Skip = "Use this to test to try out the interactive mode. This is not a real test you can run in CI."), Chrome]
+    [SuppressMessage("Usage", "xUnit1004:Test methods should not be skipped", Justification = "Only a demo.")]
+    public Task SampleTest(Browser browser) =>
+        ExecuteTestAfterSetupAsync(
+            async context =>
+            {
+                await context.SignInDirectlyAsync();
+
+                // One use-case of interactive mode is to look around in the admin dashboard and troubleshoot the
+                // current settings without programmatically navigating there.
+                await context.SignInDirectlyAndGoToDashboardAsync();
+                await context.SwitchToInteractiveAsync();
+
+                // Afterwards if you can still evaluate code as normal so `SignInDirectlyAndGoToDashboardAsync()` can be
+                // inserted anywhere. Bare in mind, that it's safest to use it before code that's already going to
+                // navigate away, like `GetCurrentUserNameAsync()` here. This ensures any manual navigation you did
+                // won't affect the test. If that's not practical, you can also do your thing in a new tab and close it
+                // before continuing the test.
+                (await context.GetCurrentUserNameAsync()).ShouldNotBeNullOrWhiteSpace();
+            },
+            browser);
+
+    // This test checks if interactive mode works by opening it in one thread, and then clicking it away in a different
+    // thread. This ensures that the new tab correctly appears with the clickable "Continue Test" button, and then
+    // disappears once it's clicked.
+    [Theory, Chrome]
+    public Task EnteringInteractiveModeShouldWait(Browser browser) =>
+        ExecuteTestAfterSetupAsync(
+            async context =>
+            {
+                var currentUrl = context.Driver.Url;
+
+                await Task.WhenAll(
+                    context.SwitchToInteractiveAsync(),
+                    Task.Run(async () =>
+                    {
+                        // Ensure that the interactive mode polls for status at least once, so the arbitrary waiting
+                        // actually works in a real testing scenario.
+                        await Task.Delay(1000);
+
+                        await context.ClickReliablyOnAsync(By.ClassName("interactive__continue"));
+                    }));
+
+                // Ensure that the info tab is closed and the control handed back to the last tab.
+                context.Driver.Url.ShouldBe(currentUrl);
+            },
+            browser);
+}
+
+// END OF TRAINING SECTION: Interactive mode.

Lombiq.Tests.UI.Samples/Tests/TenantTests.cs

@@ -63,3 +63,4 @@ await context.CreateAndSwitchToTenantAsync(
 }
 
 // END OF TRAINING SECTION: Testing in tenants.
+// NEXT STATION: Head over to InteractiveModeTests.cs.

Lombiq.Tests.UI.Shortcuts/Controllers/InteractiveModeController.cs

@@ -0,0 +1,31 @@
+using Lombiq.HelpfulLibraries.AspNetCore.Mvc;
+using Lombiq.Tests.UI.Shortcuts.Services;
+using Microsoft.AspNetCore.Authorization;
+using Microsoft.AspNetCore.Mvc;
+
+namespace Lombiq.Tests.UI.Shortcuts.Controllers;
+
+[AllowAnonymous]
+[DevelopmentAndLocalhostOnly]
+public class InteractiveModeController : Controller
+{
+    private readonly IInteractiveModeStatusAccessor _interactiveModeStatusAccessor;
+
+    public InteractiveModeController(IInteractiveModeStatusAccessor interactiveModeStatusAccessor) =>
+        _interactiveModeStatusAccessor = interactiveModeStatusAccessor;
+
+    public IActionResult Index()
+    {
+        _interactiveModeStatusAccessor.Enabled = true;
+        return View();
+    }
+
+    [Route("api/InteractiveMode/IsInteractive")]
+    public IActionResult IsInteractive() => Json(_interactiveModeStatusAccessor.Enabled);
+
+    public IActionResult Continue()
+    {
+        _interactiveModeStatusAccessor.Enabled = false;
+        return Ok();
+    }
+}

Lombiq.Tests.UI.Shortcuts/Services/IInteractiveModeStatusAccessor.cs

@@ -0,0 +1,12 @@
+namespace Lombiq.Tests.UI.Shortcuts.Services;
+
+/// <summary>
+/// A container for the flag indicating if interactive mode is enabled.
+/// </summary>
+public interface IInteractiveModeStatusAccessor
+{
+    /// <summary>
+    /// Gets or sets a value indicating whether interactive mode is enabled.
+    /// </summary>
+    bool Enabled { get; set; }
+}

Lombiq.Tests.UI.Shortcuts/Services/InteractiveModeStatusAccessor.cs

@@ -0,0 +1,6 @@
+namespace Lombiq.Tests.UI.Shortcuts.Services;
+
+public class InteractiveModeStatusAccessor : IInteractiveModeStatusAccessor
+{
+    public bool Enabled { get; set; }
+}

Lombiq.Tests.UI.Shortcuts/Startup.cs

@@ -7,6 +7,10 @@ namespace Lombiq.Tests.UI.Shortcuts;
 
 public class Startup : StartupBase
 {
-    public override void ConfigureServices(IServiceCollection services) =>
+    public override void ConfigureServices(IServiceCollection services)
+    {
+        services.AddSingleton<IInteractiveModeStatusAccessor, InteractiveModeStatusAccessor>();
+
         services.Configure<MvcOptions>((options) => options.Filters.Add(typeof(ApplicationInfoInjectingFilter)));
+    }
 }

Lombiq.Tests.UI.Shortcuts/Views/InteractiveMode/Index.cshtml

@@ -0,0 +1,34 @@
+<style at="Head">
+    .interactive__container {
+        position: fixed;
+        top: 0;
+        left: 0;
+        width: 100vw;
+        height: 100vh;
+        display: flex;
+        align-items: center;
+        justify-content: center;
+        background: white;
+        z-index: 9999999;
+    }
+
+    .interactive {
+        border: 1px solid black;
+        padding: 2rem;
+        width: 30vw;
+        text-align: center;
+    }
+</style>
+
+<div class="interactive__container">
+    <div class="interactive">
+        <h2>@T["Interactive Mode"]</h2>
+        <p>
+            @T["If you see this, your code has called the <code>context.SwitchToInteractiveAsync()</code> method."]
+            @T["This stops the test evaluation and waits until you click the button below."]
+        </p>
+        <a href="@Url.Action(nameof(InteractiveModeController.Continue))" class="interactive__continue btn btn-primary">
+            @T["Continue Test"]
+        </a>
+    </div>
+</div>

Lombiq.Tests.UI.Shortcuts/Views/_ViewImports.cshtml

@@ -0,0 +1,12 @@
+@inherits OrchardCore.DisplayManagement.Razor.RazorPage<TModel>
+
+@addTagHelper *, Microsoft.AspNetCore.Mvc.TagHelpers
+@addTagHelper *, OrchardCore.Contents.TagHelpers
+@addTagHelper *, OrchardCore.DisplayManagement
+@addTagHelper *, OrchardCore.ResourceManagement
+
+@using Lombiq.Tests.UI.Shortcuts.Controllers
+@using Microsoft.AspNetCore.Mvc.Localization
+@using Newtonsoft.Json
+@using Newtonsoft.Json.Serialization
+@using OrchardCore.Mvc.Core.Utilities

Lombiq.Tests.UI/Exceptions/VisualVerificationBaselineImageNotFoundException.cs

@@ -7,14 +7,20 @@ namespace Lombiq.Tests.UI.Exceptions;
 public class VisualVerificationBaselineImageNotFoundException : Exception
 #pragma warning restore CA1032 // Implement standard exception constructors
 {
-    public VisualVerificationBaselineImageNotFoundException(string path, Exception innerException = null)
-        : base(
-            $"Baseline image file not found, thus it was created automatically under the path {path}."
+    public VisualVerificationBaselineImageNotFoundException(
+        string path,
+        int maxRetryCount,
+        Exception innerException = null)
+        : base(GetExceptionMessage(path, maxRetryCount), innerException)
+    {
+    }
+
+    private static string GetExceptionMessage(string path, int maxRetryCount) =>
+        maxRetryCount == 0 ? $"Baseline image file not found, thus it was created automatically under the path {path}."
             + " Please set its \"Build action\" to \"Embedded resource\" if you want to deploy a self-contained"
             + " (like a NuGet package) UI testing assembly. If you run the test again, this newly created verification"
             + " file will be asserted against and the assertion will pass (unless the display of the app changed in the"
-            + " meantime).",
-            innerException)
-    {
-    }
+            + " meantime)."
+        : $"Baseline image file was not found under the path {path} and maxRetryCount is set to "
+            + $"{maxRetryCount.ToTechnicalString()}, so it won't be generated.";
 }

Lombiq.Tests.UI/Extensions/ShortcutsUITestContextExtensions.cs

@@ -434,6 +434,12 @@ public interface IShortcutsApi
         /// </summary>
         [Get("/api/ApplicationInfo")]
         Task<ApplicationInfo> GetApplicationInfoFromApiAsync();
+
+        /// <summary>
+        /// Sends a web request to <see cref="InteractiveModeController.IsInteractive"/> endpoint.
+        /// </summary>
+        [Get("/api/InteractiveMode/IsInteractive")]
+        Task<bool> IsInteractiveModeEnabledAsync();
     }
 
     /// <summary>
@@ -584,6 +590,41 @@ await UsingScopeAsync(
         return eventUrl;
     }
 
+    /// <summary>
+    /// Opens a new tab with the <see cref="InteractiveModeController"/> <see cref="InteractiveModeController.Index"/>
+    /// page. Visiting this page enables the interactive mode flag so it can be awaited with the <see
+    /// cref="WaitInteractiveModeAsync"/> extension method.
+    /// </summary>
+    public static Task EnterInteractiveModeAsync(this UITestContext context)
+    {
+        context.Driver.SwitchTo().NewWindow(WindowType.Tab);
+        context.Driver.SwitchTo().Window(context.Driver.WindowHandles[^1]);
+
+        return context.GoToAsync<InteractiveModeController>(controller => controller.Index());
+    }
+
+    /// <summary>
+    /// Periodically polls the <see cref="IShortcutsApi.IsInteractiveModeEnabledAsync"/> and waits half a second if it's
+    /// <see langword="true"/>.
+    /// </summary>
+    public static async Task WaitInteractiveModeAsync(this UITestContext context)
+    {
+        var client = context.GetApi();
+        while (await client.IsInteractiveModeEnabledAsync())
+        {
+            await Task.Delay(TimeSpan.FromMilliseconds(500));
+        }
+    }
+
+    public static async Task SwitchToInteractiveAsync(this UITestContext context)
+    {
+        await context.EnterInteractiveModeAsync();
+        await context.WaitInteractiveModeAsync();
+
+        context.Driver.Close();
+        context.SwitchToLastWindow();
+    }
+
     private static bool IsAdminTheme(IManifestInfo manifest) =>
         manifest.Tags.Any(tag => tag.EqualsOrdinalIgnoreCase(ManifestConstants.AdminTag));
 

Lombiq.Tests.UI/Extensions/TypedRouteUITestContextExtensions.cs

@@ -1,6 +1,8 @@
 using Lombiq.Tests.UI.Extensions;
 using Lombiq.Tests.UI.Services;
 using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.DependencyInjection;
+using OrchardCore.Admin;
 using System;
 using System.Linq.Expressions;
 using System.Threading.Tasks;
@@ -19,7 +21,7 @@ public static Task GoToAsync<TController>(
         params (string Key, object Value)[] additionalArguments)
         where TController : ControllerBase =>
         context.GoToRelativeUrlAsync(TypedRoute
-            .CreateFromExpression(actionExpression, additionalArguments)
+            .CreateFromExpression(actionExpression, additionalArguments, CreateServiceProvider(context))
             .ToString());
 
     /// <summary>
@@ -32,6 +34,17 @@ public static Task GoToAsync<TController>(
         params (string Key, object Value)[] additionalArguments)
         where TController : ControllerBase =>
         context.GoToRelativeUrlAsync(TypedRoute
-            .CreateFromExpression(actionExpressionAsync.StripResult(), additionalArguments)
+            .CreateFromExpression(actionExpressionAsync.StripResult(), additionalArguments, CreateServiceProvider(context))
             .ToString());
+
+    private static IServiceProvider CreateServiceProvider(UITestContext context)
+    {
+        var services = new ServiceCollection();
+
+        // The AdminOptions.AdminUrlPrefix setter automatically trims out leading or trailing slashes so the format
+        // difference between the two properties doesn't matter.
+        services.Configure<AdminOptions>(options => options.AdminUrlPrefix = context.AdminUrlPrefix);
+
+        return services.BuildServiceProvider();
+    }
 }

Lombiq.Tests.UI/Extensions/VisualVerificationUITestContextExtensions.cs

@@ -389,8 +389,16 @@ private static void AssertVisualVerificationApproved(
 
             if (!File.Exists(approvedContext.BaselineImagePath))
             {
-                context.SaveSuggestedImage(element, approvedContext.BaselineImagePath, approvedContext.BaselineImageFileName);
-                throw new VisualVerificationBaselineImageNotFoundException(approvedContext.BaselineImagePath);
+                if (context.Configuration.MaxRetryCount == 0)
+                {
+                    context.SaveSuggestedImage(
+                        element,
+                        approvedContext.BaselineImagePath,
+                        approvedContext.BaselineImageFileName);
+                }
+
+                throw new VisualVerificationBaselineImageNotFoundException(
+                    approvedContext.BaselineImagePath, context.Configuration.MaxRetryCount);
             }
 
             baselineImage = Image.Load(approvedContext.BaselineImagePath);

Readme.md

@@ -25,8 +25,9 @@ Highlights:
 - Support for [TeamCity test metadata reporting](https://www.jetbrains.com/help/teamcity/reporting-test-metadata.html) so you can see the important details and metrics of a test at a glance in a TeamCity CI/CD server.
 - Visual verification testing: You can make the test fail if the look of the app changes. Demo video [here](https://www.youtube.com/watch?v=a-1zKjxTKkk).
 - If your app uses a camera, a fake video capture source in Chrome is supported. [Here's a demo video of the feature](https://www.youtube.com/watch?v=sGcD0eJ2ytc), and check out the docs [here](Lombiq.Tests.UI/Docs/FakeVideoCaptureSource.md).
+- Interactive mode for debugging the app while the test is paused. [Here's a demo of the feature](Lombiq.Tests.UI.Samples/Tests/InteractiveModeTests.cs).
 
-See a demo video of the project [here](https://www.youtube.com/watch?v=mEUg6-pad-E). Also, see our [Testing Toolbox](https://github.com/Lombiq/Testing-Toolbox) for similar features for lower-level tests.
+See a demo video of the project [here](https://www.youtube.com/watch?v=mEUg6-pad-E), and the Orchard Harvest 2023 conference talk about automated QA in Orchard Core [here](https://youtu.be/CHdhwD2NHBU). Also, see our [Testing Toolbox](https://github.com/Lombiq/Testing-Toolbox) for similar features for lower-level tests.
 
 Looking not just for dynamic testing but also static code analysis? Check out our [.NET Analyzers project](https://github.com/Lombiq/.NET-Analyzers).