Review & fix users e2e tests flakiness (#3378)

* keep artifacts to check where tests are failing

* run only totp test

* try waiting for endpoints after actions

* add profile wait

* try arbitrary waits

* add 2 more waits

* remove two last explicit waits

* reduce waits and add one more after typing totp

* increase again to 1000

* make switch validation dependant on type totp code

* add screenshots in test reports

* add screenshots artifacts in e2e flaky test analysis

* make sure to wait for newly issued code

* custom waits

* wait for totp enrollment

* modify validations order to make test more consistent

* fix rest of totp tests

* modify selector for disable totp feature

* disable python setup and execute all tests

* remove python setup and fix check item menu test

* restore delete artifacts after analysis

* pr changes request

Author: vicenteqa

.github/workflows/e2e-flaky-tests-detection.yaml

@@ -164,10 +164,6 @@ jobs:
         uses: actions/setup-node@v4
         with:
           node-version: ${{ env.NODE_VERSION }}
-      - name: Set up Python
-        uses: actions/setup-python@v5
-        with:
-          python-version: ${{ env.PYTHON_VERSION }}
       - name: Retrieve Cached Dependencies
         uses: actions/cache@v4
         id: mix-cache

test/e2e/cypress/e2e/users.cy.js

@@ -226,15 +226,15 @@ describe('Users', () => {
 
     it('should complete TOTP enrollment properly', () => {
       usersPage.clickAuthenticatorAppSwitch();
-      usersPage.typeUserTotpCode();
+      usersPage.getSecretAndTypeTotpCode();
       usersPage.clickVerifyTotpButton();
       usersPage.authenticatorAppSwitchIsEnabled();
       usersPage.totpEnabledToasterIsDisplayed();
     });
 
     it('should fail to login if TOTP code is not given', () => {
       usersPage.clickAuthenticatorAppSwitch();
-      usersPage.typeUserTotpCode();
+      usersPage.getSecretAndTypeTotpCode();
       usersPage.clickVerifyTotpButton();
       loginPage.loginFailsIfOtpNotProvided(
         usersPage.USER.username,
@@ -244,40 +244,42 @@ describe('Users', () => {
 
     it('should disable TOTP authentication and check login works without TOTP', () => {
       usersPage.clickAuthenticatorAppSwitch();
-      usersPage.typeUserTotpCode();
+      usersPage.getSecretAndTypeTotpCode();
       usersPage.clickVerifyTotpButton();
       usersPage.clickAuthenticatorAppSwitch();
       usersPage.clickDisableTotpButton();
       loginPage.loginShouldSucceed(usersPage.USER.username, usersPage.PASSWORD);
     });
 
-    it('should reconfigure TOTP and validate login cases', () => {
+    it('should configure TOTP and validate login cases, then disable it', () => {
       usersPage.clickAuthenticatorAppSwitch();
-      usersPage.typeUserTotpCode();
-      usersPage.clickVerifyTotpButton();
-      usersPage.clickAuthenticatorAppSwitch();
-      usersPage.clickDisableTotpButton();
-      usersPage.clickAuthenticatorAppSwitch();
-      usersPage.typeUserTotpCode().then((totpSecret) => {
-        usersPage.clickVerifyTotpButton();
-        usersPage.authenticatorAppSwitchIsEnabled();
-        usersPage.clickSignOutButton();
-        loginPage.login(usersPage.USER.username, usersPage.PASSWORD);
-        loginPage.typeInvalidLoginTotpCode();
-        loginPage.clickSubmitLoginButton();
-        loginPage.invalidCredentialsErrorIsDisplayed();
-        loginPage.typeAlreadyUsedTotpCode(totpSecret);
-        loginPage.clickSubmitLoginButton();
-        loginPage.invalidCredentialsErrorIsDisplayed();
-        loginPage.waitForNewTotpCodeAndTypeIt(totpSecret);
-        loginPage.clickSubmitLoginButton();
-        dashboardPage.dashboardPageIsDisplayed();
+      usersPage.getTotpSecret().then((totpSecret) => {
+        usersPage.typeUserTotpCode(totpSecret).then((code) => {
+          usersPage.clickVerifyTotpButton();
+          usersPage.authenticatorAppSwitchIsEnabled();
+          usersPage.clickSignOutButton();
+          loginPage.login(usersPage.USER.username, usersPage.PASSWORD);
+          loginPage.typeInvalidLoginTotpCode();
+          loginPage.clickSubmitLoginButton();
+          loginPage.invalidCredentialsErrorIsDisplayed();
+          loginPage.typeAlreadyUsedTotpCode(code);
+          loginPage.clickSubmitLoginButton();
+          loginPage.invalidCredentialsErrorIsDisplayed();
+          loginPage.typeLoginTotpCode(totpSecret);
+          loginPage.clickSubmitLoginButton();
+          dashboardPage.dashboardPageIsDisplayed();
+          usersPage.visit('/profile');
+          usersPage.clickAuthenticatorAppSwitch();
+          usersPage.clickDisableTotpButton();
+          usersPage.clickAuthenticatorAppSwitch();
+          usersPage.newIssuedTotpSecretIsDifferent();
+        });
       });
     });
 
     it('should be disabled by admin user', () => {
       usersPage.clickAuthenticatorAppSwitch();
-      usersPage.typeUserTotpCode();
+      usersPage.getSecretAndTypeTotpCode();
       usersPage.clickVerifyTotpButton();
       usersPage.clickSignOutButton();
       usersPage.apiLoginAndCreateSession();

test/e2e/cypress/pageObject/base_po.js

@@ -71,13 +71,36 @@ export const clickUserDropdownProfileButton = () => {
   return cy.get(userDropdownProfileButton).click();
 };
 
-export const typeTotpCode = (totpSecret, inputField) => {
-  const { otp } = TOTP.generate(totpSecret);
-  return cy
-    .get(inputField)
-    .clear()
-    .type(otp)
-    .then(() => totpSecret);
+const _getTotpWaitTime = (forceNext) => {
+  const currentTime = Date.now();
+  const totpDuration = 30000;
+  const minimumRemainingTimeToReuse = 10000;
+  const expirationTime = Math.ceil(currentTime / totpDuration) * totpDuration;
+  const remainingTime = Math.floor(expirationTime - currentTime);
+
+  const totpWaitingTime =
+    forceNext === false && remainingTime > minimumRemainingTimeToReuse
+      ? 0
+      : remainingTime;
+
+  return totpWaitingTime;
+};
+
+export const typeNextGeneratedTotpCode = (
+  totpSecret,
+  inputField,
+  forceNext = false
+) => {
+  const timeToWait = _getTotpWaitTime(forceNext);
+
+  return cy.wait(timeToWait).then(() => {
+    const { otp } = TOTP.generate(totpSecret);
+    return cy
+      .get(inputField)
+      .clear()
+      .type(otp)
+      .then(() => otp);
+  });
 };
 
 // UI Validations
@@ -100,17 +123,8 @@ export const validateItemNotPresentInNavigationMenu = (itemName) => {
   });
 };
 
-export const validateItemPresentInNavigationMenu = (navigationMenuItem) => {
-  return cy.get(navigation.navigationItems).then(($elements) => {
-    const itemFound = Array.from($elements).some((element) =>
-      element.innerText.includes(navigationMenuItem)
-    );
-    expect(
-      itemFound,
-      `"${navigationMenuItem}" navigation item should be present`
-    ).to.be.true;
-  });
-};
+export const validateItemPresentInNavigationMenu = (navigationMenuItem) =>
+  cy.get(`a:contains("${navigationMenuItem}")`).should('be.visible');
 
 export const addTagButtonsAreDisabled = () =>
   cy.get(addTagButtons).should('have.class', 'opacity-50');

test/e2e/cypress/pageObject/login_po.js

@@ -61,19 +61,17 @@ export const clickSubmitLoginButton = () => {
   return cy.get(submitLoginButton).click();
 };
 
-export const typeLoginTotpCode = (totpSecret) => {
-  basePage.typeTotpCode(totpSecret, totpCodeInput);
-};
+export const typeLoginTotpCode = (totpSecret) =>
+  basePage.typeNextGeneratedTotpCode(totpSecret, totpCodeInput, true);
 
-export const typeAlreadyUsedTotpCode = (totpSecret) => {
-  return typeLoginTotpCode(totpSecret);
+export const typeAlreadyUsedTotpCode = (code) => {
+  cy.get(totpCodeInput).clear().type(code);
 };
 
 export const typeInvalidLoginTotpCode = () => {
   return cy.get(totpCodeInput).clear().type('invalid');
 };
 
 export const waitForNewTotpCodeAndTypeIt = (totpSecret) => {
-  // eslint-disable-next-line cypress/no-unnecessary-waiting
-  return cy.wait(30000).then(() => typeLoginTotpCode(totpSecret));
+  typeLoginTotpCode(totpSecret);
 };

test/e2e/cypress/pageObject/users_po.js

@@ -35,7 +35,8 @@ const newTotpCodeIssuedMessage = 'div:contains("Your new TOTP secret is:")';
 const totpSecret = `${newTotpCodeIssuedMessage} + div[class*="bold"]`;
 const newTotpCodeInputField = 'input[placeholder="TOTP code"]';
 const verifyTotpButton = 'button:contains("Verify")';
-const confirmDisableTotpButton = 'button:contains("Disable")';
+const confirmDisableTotpButton =
+  'div[id*="headlessui-dialog-panel"] button:contains("Disable")';
 const editUserTotpDropdown = 'button.totp-selection-dropdown';
 const enableUserTotpOption = `${editUserTotpDropdown} + div div:contains("Enabled")`;
 
@@ -153,14 +154,17 @@ export const selectFromTotpDropdown = (choice) => {
   return basePage.selectFromDropdown(editUserTotpDropdown, choice);
 };
 
-const getTotpSecret = () => {
+export const getTotpSecret = () => {
   return cy.get(totpSecret).then((element) => element.text());
 };
 
-export const typeUserTotpCode = () => {
-  return getTotpSecret().then((totpSecret) =>
-    basePage.typeTotpCode(totpSecret, newTotpCodeInputField)
-  );
+export const typeUserTotpCode = (totpSecret) =>
+  basePage.typeNextGeneratedTotpCode(totpSecret, newTotpCodeInputField);
+
+export const getSecretAndTypeTotpCode = () => {
+  getTotpSecret().then((totpSecret) => {
+    typeUserTotpCode(totpSecret);
+  });
 };
 
 export const typeInvalidUserTotpCode = () => {
@@ -359,3 +363,12 @@ export const enableTotpOptionIsDisabled = () => {
     .invoke('attr', 'aria-disabled')
     .should('eq', 'true');
 };
+
+export const newIssuedTotpSecretIsDifferent = (originalTotpSecret) => {
+  getTotpSecret().then((newTotpSecret) => {
+    expect(
+      newTotpSecret === originalTotpSecret,
+      'New issued TOTP secret is different'
+    ).to.be.false;
+  });
+};