fix visual test failures (#364)

* add test for quiz and result pages * add POC for analytics test * update analytics tests and envs * result envs * update analytics only on firefox * add dynamic tests * add dynamic test for quiz page * update quiz test * update test structure * add test for quiz and result pages * add POC for analytics test * update analytics tests and envs * result envs * update analytics only on firefox * add dynamic tests * add dynamic test for quiz page * update quiz test * update test structure * update uar tests to new format * update uar tests * move everything to uar folder * add dependency for js-yaml * update test according to latest comments * update test according to latest comments * update tests according to the latest comments * add uar config * add analytics for uar * update according to feedbacks * update tags * add @cc tag * update according to feedbacks * add dynamic tests for uar * update according to review comments * fix some typos * update according to review comments * merge console log info * add UI screenshots for UAR * move view point to test * update according to feedback * add analytics test for UAR * update validation and uar libs * update libs * update by feedback * update some config and tests * update some config and tests * update test content path to common path * update analytics tests * fix test failures caused by test code * fix test failures * fix test failures * add visual test for CAAS with two pages * update according to feedbacks * update report in config file * add screenshot for milo main live vs uar-integration live * add timestamp js for time stampe recording * add screenshot diff for uar stable and beta * move uar screenshots into visual compare folder * add UI screenshots for DX Quiz * build a function to get screenshots * update locator * udate uar to only run basic tests * update according to feedback * add screenshot for UAR in CC which will go live * update final path for UAR in CC * update the final CC path * update cc uar screenshots to hlx.live vs stage * update test * update screenshot test * update uar analytics to run on STAGE * update tests to run on different envs after go live * Fix test failures for MWPW-143444 * update urls * MWPW-143444: setup automation for cc uar quiz * update for feedbacks * update for feedbacks * fix some validations * update uar automation for Stage testing * fix lint error * fix tests failures caused by timeout * make uar able to run on milo * limited dynamic test to 20 random * delete unuse tests to avoid noises * comment out soft compare to avoid failures * comment out soft compare to avoid failures * remove uar tests from cc folder * remove stage stuff * fix lint and test failures * remove unuse ulrs * get milo stage back for analytics * update id to be unique * update package.json * remove main cc hlx live links * remove cc tags for uar tests * MWPW-146964: unified test run commands * support '-' and '--' * add required for -c and -p * show help if no option provided * MWPW-146964: [Nala] unified test run commands, support run on all projects or some projects * MWPW-146964: [Nala] unified test run commands, support run on all projects or some projects * MWPW-147289: [Nala] Refine screenshot diff tool, first step, cleanup and build new APIs * MWPW-147289: [Nala] Refine screenshot diff tool, rewrite uar screenshots with new way * MWPW-142970: Setup API automation to validate ML results * MWPW-147289: [Nala] Refine screenshot diff tool: add IT s3 support, refine take APIS * MWPW-147289: [Nala] Refine screenshot diff tool, clean take.js and update compare.js and add timestamp * add screenshots for sot * update sot data * update uar-ai screenshot test * update visual failed tests * update visual test from one to two, main vs stage --------- Co-authored-by: Aaron Mauchley <[email protected]> Co-authored-by: xiasun <[email protected]> Co-authored-by: xiasun <[email protected]>
adobecom · Jun 6, 2024 · 82bc448 · 82bc448
1 parent bf61182
commit 82bc448
Show file tree

Hide file tree

Showing 9 changed files with 89 additions and 30 deletions.
diff --git a/data/uar/quiz/ai/quiz-ai-basic.yml b/data/uar/quiz/ai/quiz-ai-basic.yml
@@ -3,7 +3,7 @@
 'How to edit PDF > A student or teacher discount': 'Acrobat Pro'
 'I need to create page design layouts for books, magazines, and brochures > Licenses and business features for teams': InDesign
 'create a simple flyer for a work event > Licenses and business features for teams': Adobe Express
-'How to upload video? > Neither apply': Adobe Express
+'How to upload video > Neither apply': Adobe Express
 'Create a composite of two or more images > Neither apply': We think you'll love
 'Combine images > Neither apply': We think you'll love
-'Photography + Video + PDFs > Take the time to control every detail > Get them sorted and organized > Create, edit, and share on social > Create and export PDFs to Office > A student or teacher discount': We think you'll love
+'Photography + Video > Take the time to control every detail > Get them sorted and organized > Create, edit, and share on social > A student or teacher discount': We think you'll love
diff --git a/features/visual/caas/cards.spec.js b/features/visual/caas/cards.spec.js
@@ -4,29 +4,33 @@ module.exports = {
     {
       tcid: '0',
       name: '@onehalfcollection-visual',
-      stable: '/fragments/caas/test-pages/onehalfcollection',
-      beta: '/fragments/caas/test-pages/onehalfcollection?caasver=beta',
+      path: '/fragments/caas/test-pages/onehalfcollection',
+      stable: '@milo_live',
+      beta: '@milo_stage',
       tags: '@onehalfcollection-visual @visual @caas-screenshots',
     },
     {
       tcid: '1',
       name: '@threefourthscollection-visual',
-      stable: '/fragments/caas/test-pages/threefourthscollection',
-      beta: '/fragments/caas/test-pages/threefourthscollection?caasver=beta',
+      path: '/fragments/caas/test-pages/threefourthscollection',
+      stable: '@milo_live',
+      beta: '@milo_stage',
       tags: '@threefourthscollection-visual @visual @caas-screenshots',
     },
     {
       tcid: '2',
       name: '@halfheightcollection-visual',
-      stable: '/fragments/caas/test-pages/halfheightcollection',
-      beta: '/fragments/caas/test-pages/halfheightcollection?caasver=beta',
+      path: '/fragments/caas/test-pages/halfheightcollection',
+      stable: '@milo_live',
+      beta: '@milo_stage',
       tags: '@halfheightcollection-visual @visual @caas-screenshots',
     },
     {
       tcid: '3',
       name: '@fullwidthcollection-visual',
-      stable: '/fragments/caas/test-pages/textcardcollection',
-      beta: '/fragments/caas/test-pages/textcardcollection?caasver=beta',
+      path: '/fragments/caas/test-pages/textcardcollection',
+      stable: '@milo_live',
+      beta: '@milo_stage',
       tags: '@fullwidthcollection-visual @visual @caas-screenshots',
     },
   ],

diff --git a/features/visual/feds/feds.spec.js b/features/visual/feds/feds.spec.js
@@ -5,13 +5,17 @@ module.exports = {
       tcid: '0',
       name: '@header-visual',
       path: '/libs/feds/drafts/qa/header/feds-header-page',
+      stable: '@milo_live',
+      beta: '@milo_stage',
       tags: '@header-visual @visual @feds-screenshots',
       browserParams: '?georouting=off',
     },
     {
       tcid: '1',
       name: '@footer-visual',
       path: '/libs/feds/drafts/qa/footer/feds-default-footer',
+      stable: '@milo_live',
+      beta: '@milo_stage',
       tags: '@footer-visual @visual @feds-screenshots',
       browserParams: '?georouting=off',
     },

diff --git a/features/visual/milo/milo.spec.js b/features/visual/milo/milo.spec.js
@@ -5,12 +5,16 @@ module.exports = {
       tcid: '0',
       name: '@columns-visual',
       path: '/test/features/blocks/columns',
+      stable: '@milo_live',
+      beta: '@milo_stage',
       tags: '@columns-visual @visual @milo-screenshots',
     },
     {
       tcid: '1',
       name: '@marquee-visual',
       path: '/test/features/blocks/marquee',
+      stable: '@milo_live',
+      beta: '@milo_stage',
       tags: '@marquee-visual @visual @milo-screenshots',
     },
   ],

diff --git a/libs/screenshot/compare.mjs b/libs/screenshot/compare.mjs
@@ -82,7 +82,7 @@ async function main() {
 
       const baseImage = fs.readFileSync(entry.a);
       const currImage = fs.readFileSync(entry.b);
-      result.order = 1;
+      result.order = entry.order;
       result.a = entry.a;
       result.b = entry.b;
       result.urls = entry.urls;

diff --git a/libs/screenshot/take.js b/libs/screenshot/take.js
@@ -65,7 +65,7 @@ async function takeOne(page, url, callback, folderPath, fileName, options = {})
  * @param {string} fileName - The file name of the screenshots
  * @returns {object} The screenshot results
  */
-async function takeTwo(page, urlStable, callbackStable, urlBeta, callbackBeta, folderPath, fileName) {
+async function takeTwo(page, urlStable, callbackStable, urlBeta, callbackBeta, folderPath, fileName, options = {}) {
   const urls = [];
   const result = {};
 
@@ -74,7 +74,12 @@ async function takeTwo(page, urlStable, callbackStable, urlBeta, callbackBeta, f
   urls.push(urlStable);
   if (typeof callbackStable === 'function') { await callbackStable(); }
   const nameStable = `${folderPath}/${fileName}-a.png`;
-  await page.screenshot({ path: nameStable, fullPage: true });
+  options.path = nameStable;
+  if (options.selector) {
+    await page.locator(options.selector).screenshot(options);
+  } else {
+    await page.screenshot(options);
+  }
   result.order = 1;
   result.a = nameStable;
 
@@ -83,7 +88,12 @@ async function takeTwo(page, urlStable, callbackStable, urlBeta, callbackBeta, f
   urls.push(urlBeta);
   if (typeof callbackBeta === 'function') { await callbackBeta(); }
   const nameBeta = `${folderPath}/${fileName}-b.png`;
-  await page.screenshot({ path: nameBeta, fullPage: true });
+  options.path = nameBeta;
+  if (options.selector) {
+    await page.locator(options.selector).screenshot(options);
+  } else {
+    await page.screenshot(options);
+  }
 
   result.b = nameBeta;
   result.urls = urls.join(' | ');

diff --git a/tests/visual/caas/cards.test.js b/tests/visual/caas/cards.test.js
@@ -4,20 +4,26 @@ import { features } from '../../../features/visual/caas/cards.spec.js';
 import { takeTwo } from '../../../libs/screenshot/take.js';
 import { writeResultsToFile } from '../../../libs/screenshot/utils.js';
 
+const envs = require('../../../envs/envs.js');
+
 const folderPath = 'screenshots/caas';
 const results = {};
 
 test.describe('Milo Caas block visual comparison test suite', () => {
   // eslint-disable-next-line no-restricted-syntax
   for (const feature of features) {
     // eslint-disable-next-line no-loop-func
-    test(`${feature.name},${feature.tags}`, async ({ page, baseURL }, testInfo) => {
+    test(`${feature.name},${feature.tags}`, async ({ page }, testInfo) => {
       const name = `${feature.name}-${testInfo.project.name}`;
+      const stableURL = `${envs[feature.stable]}${feature.path}`;
+      console.info(stableURL);
+      const betaURL = `${envs[feature.beta]}${feature.path}`;
+      console.info(betaURL);
       const result = await takeTwo(
         page,
-        baseURL + feature.stable,
+        stableURL,
         async () => { await page.waitForTimeout(3000); },
-        baseURL + feature.beta,
+        betaURL,
         async () => { await page.waitForTimeout(3000); },
         folderPath,
         name,

diff --git a/tests/visual/feds/feds.test.js b/tests/visual/feds/feds.test.js
@@ -1,10 +1,12 @@
 /* eslint-disable import/named */
 import { expect, test } from '@playwright/test';
 import { features } from '../../../features/visual/feds/feds.spec.js';
-import { takeOne } from '../../../libs/screenshot/take.js';
+import { takeTwo } from '../../../libs/screenshot/take.js';
 import { writeResultsToFile } from '../../../libs/screenshot/utils.js';
 import FedsHeader from '../../../selectors/feds/feds.header.page.js';
 
+const envs = require('../../../envs/envs.js');
+
 const folderPath = 'screenshots/feds';
 let Header;
 const results = {};
@@ -14,14 +16,29 @@ test.describe('Feds blocks visual comparison test suite', () => {
     Header = new FedsHeader(page);
   });
 
-  test(`${features[0].name}, ${features[0].tags}`, async ({ page, baseURL }, testInfo) => {
+  test(`${features[0].name}, ${features[0].tags}`, async ({ page }, testInfo) => {
     const name = `${features[0].name}-${testInfo.project.name}`;
-    const result = await takeOne(
+    const stableURL = `${envs[features[0].stable]}${features[0].path}${features[0].browserParams}`;
+    console.info(stableURL);
+    const betaURL = `${envs[features[0].beta]}${features[0].path}${features[0].browserParams}`;
+    console.info(betaURL);
+    const result = await takeTwo(
       page,
-      baseURL + features[0].path + features[0].browserParams,
+      stableURL,
+      async () => {
+        await page.waitForSelector('.feds-footer-privacyLink');
+        if (testInfo.project.name === 'ipad' || testInfo.project.name === 'iphone') {
+          await page.locator('.feds-toggle').click();
+        }
+        await Header.megaMenuToggle.click();
+        await expect(Header.megaMenuContainer).toBeVisible();
+      },
+      betaURL,
       async () => {
-        await page.waitForLoadState('domcontentloaded');
-        await Header.megaMenuToggle.waitFor({ state: 'visible', timeout: 5000 });
+        await page.waitForSelector('.feds-footer-privacyLink');
+        if (testInfo.project.name === 'ipad' || testInfo.project.name === 'iphone') {
+          await page.locator('.feds-toggle').click();
+        }
         await Header.megaMenuToggle.click();
         await expect(Header.megaMenuContainer).toBeVisible();
       },
@@ -33,11 +50,17 @@ test.describe('Feds blocks visual comparison test suite', () => {
     writeResultsToFile(folderPath, testInfo, results);
   });
 
-  test(`${features[1].name}, ${features[1].tags}`, async ({ page, baseURL }, testInfo) => {
+  test(`${features[1].name}, ${features[1].tags}`, async ({ page }, testInfo) => {
     const name = `${features[1].name}-${testInfo.project.name}`;
-    const result = await takeOne(
+    const stableURL = `${envs[features[1].stable]}${features[1].path}${features[1].browserParams}`;
+    console.info(stableURL);
+    const betaURL = `${envs[features[1].beta]}${features[1].path}${features[1].browserParams}`;
+    console.info(betaURL);
+    const result = await takeTwo(
       page,
-      baseURL + features[1].path + features[1].browserParams,
+      stableURL,
+      async () => { await page.waitForSelector('.feds-footer-privacyLink'); },
+      betaURL,
       async () => { await page.waitForSelector('.feds-footer-privacyLink'); },
       folderPath,
       name,

diff --git a/tests/visual/milo/milo.test.js b/tests/visual/milo/milo.test.js
@@ -1,21 +1,29 @@
 /* eslint-disable import/named */
 import { test } from '@playwright/test';
 import { features } from '../../../features/visual/milo/milo.spec.js';
-import { takeOne } from '../../../libs/screenshot/take.js';
+import { takeTwo } from '../../../libs/screenshot/take.js';
 import { writeResultsToFile } from '../../../libs/screenshot/utils.js';
 
+const envs = require('../../../envs/envs.js');
+
 const folderPath = 'screenshots/milo';
 const results = {};
 
 test.describe('Milo blocks visual comparison test suite', () => {
   // eslint-disable-next-line no-restricted-syntax
   for (const feature of features) {
     // eslint-disable-next-line no-loop-func
-    test(`${feature.name},${feature.tags}`, async ({ page, baseURL }, testInfo) => {
+    test(`${feature.name},${feature.tags}`, async ({ page }, testInfo) => {
       const name = `${feature.name}-${testInfo.project.name}`;
-      const result = await takeOne(
+      const stableURL = `${envs[feature.stable]}${feature.path}`;
+      console.info(stableURL);
+      const betaURL = `${envs[feature.beta]}${feature.path}`;
+      console.info(betaURL);
+      const result = await takeTwo(
         page,
-        baseURL + feature.path,
+        stableURL,
+        async () => { await page.waitForSelector('.feds-footer-privacyLink'); },
+        betaURL,
         async () => { await page.waitForSelector('.feds-footer-privacyLink'); },
         folderPath,
         name,