From f70dd1ee1724ad96b50514ebfc79bc46d0da4b13 Mon Sep 17 00:00:00 2001 From: Jacky Sun <67350368+JackySun9@users.noreply.github.com> Date: Thu, 2 May 2024 12:18:46 -0700 Subject: [PATCH] MWPW-147289: [Nala] Refine screenshot diff tool, rewrite uar screenshots in new way (#341) * add test for quiz and result pages * add test for quiz and result pages * add test for quiz and result pages * add POC for analytics test * update analytics tests and envs * result envs * update analytics only on firefox * add dynamic tests * add dynamic test for quiz page * update quiz test * update test structure * add test for quiz and result pages * add POC for analytics test * update analytics tests and envs * result envs * update analytics only on firefox * add dynamic tests * add dynamic test for quiz page * update quiz test * update test structure * update uar tests to new format * update uar tests * move everything to uar folder * add dependency for js-yaml * update test according to latest comments * update test according to latest comments * update tests according to the latest comments * add uar config * add analytics for uar * update according to feedbacks * update tags * add @cc tag * update according to feedbacks * add dynamic tests for uar * update according to review comments * fix some typos * update according to review comments * merge console log info * add UI screenshots for UAR * move view point to test * update according to feedback * add analytics test for UAR * update validation and uar libs * update libs * update by feedback * update some config and tests * update some config and tests * update test content path to common path * update analytics tests * fix test failures caused by test code * fix test failures * fix test failures * add visual test for CAAS with two pages * update according to feedbacks * update report in config file * add screenshot for milo main live vs uar-integration live * add timestamp js for time stampe recording * add screenshot diff for uar stable and beta * move uar screenshots into visual compare folder * add UI screenshots for DX Quiz * build a function to get screenshots * update locator * udate uar to only run basic tests * update according to feedback * add screenshot for UAR in CC which will go live * update final path for UAR in CC * update the final CC path * update cc uar screenshots to hlx.live vs stage * update test * update screenshot test * update uar analytics to run on STAGE * update tests to run on different envs after go live * Fix test failures for MWPW-143444 * update urls * MWPW-143444: setup automation for cc uar quiz * update for feedbacks * update for feedbacks * fix some validations * update uar automation for Stage testing * fix lint error * fix tests failures caused by timeout * make uar able to run on milo * limited dynamic test to 20 random * delete unuse tests to avoid noises * comment out soft compare to avoid failures * comment out soft compare to avoid failures * remove uar tests from cc folder * remove stage stuff * fix lint and test failures * remove unuse ulrs * get milo stage back for analytics * update id to be unique * update package.json * remove main cc hlx live links * remove cc tags for uar tests * MWPW-146964: unified test run commands * support '-' and '--' * add required for -c and -p * show help if no option provided * MWPW-146964: [Nala] unified test run commands, support run on all projects or some projects * MWPW-146964: [Nala] unified test run commands, support run on all projects or some projects * MWPW-147289: [Nala] Refine screenshot diff tool, first step, cleanup and build new APIs * MWPW-147289: [Nala] Refine screenshot diff tool, rewrite uar screenshots with new way --------- Co-authored-by: xiasun Co-authored-by: Aaron Mauchley Co-authored-by: xiasun --- features/uar/quiz.screenshots.spec.js | 12 --- features/visual/milo/milo.spec.js | 2 +- libs/visualutil.js | 6 +- libs/webutil.js | 62 +---------- selectors/uar/quiz.old.page.js | 144 -------------------------- selectors/uar/quiz.page.js | 63 +++++------ tests/uar/quiz.screenshots.test.js | 57 ---------- tests/visual/uar/quiz.test.js | 67 ++++++++++++ 8 files changed, 104 insertions(+), 309 deletions(-) delete mode 100644 features/uar/quiz.screenshots.spec.js delete mode 100644 selectors/uar/quiz.old.page.js delete mode 100644 tests/uar/quiz.screenshots.test.js create mode 100644 tests/visual/uar/quiz.test.js diff --git a/features/uar/quiz.screenshots.spec.js b/features/uar/quiz.screenshots.spec.js deleted file mode 100644 index eef3ca37..00000000 --- a/features/uar/quiz.screenshots.spec.js +++ /dev/null @@ -1,12 +0,0 @@ -module.exports = { - name: 'Quiz Recommender Blocks', - features: [ - { - tcid: '0', - name: '@quiz screenshots', - path: '/drafts/quiz/quiz-2/', - tags: '@uar @uar-quiz-screenshots-milo @uar-quiz-static', - data: 'data/uar/quiz/quiz-basic.yml', - }, - ], -}; diff --git a/features/visual/milo/milo.spec.js b/features/visual/milo/milo.spec.js index c0fe835a..173c48ab 100644 --- a/features/visual/milo/milo.spec.js +++ b/features/visual/milo/milo.spec.js @@ -11,7 +11,7 @@ module.exports = { tcid: '1', name: '@marquee-visual', path: '/test/features/blocks/marquee', - tags: '@marquee-visua @visual @milo-screenshots', + tags: '@marquee-visual @visual @milo-screenshots', }, ], }; diff --git a/libs/visualutil.js b/libs/visualutil.js index cf371ea2..4ad28c48 100644 --- a/libs/visualutil.js +++ b/libs/visualutil.js @@ -5,6 +5,10 @@ import { getComparator } from 'playwright-core/lib/utils'; const fs = require('fs'); +async function take(page, folderPath, fileName) { + await page.screenshot({ path: `${folderPath}/${fileName}`, fullPage: true }); +} + async function takeOne(page, url, callback, folderPath, fileName, isFullPage = true) { const urls = []; const result = {}; @@ -100,4 +104,4 @@ function compareScreenshots(stableArray, betaArray, folderPath) { } } -module.exports = { takeTwoAndCompare, compareScreenshots, takeOne, takeTwo }; +module.exports = { takeTwoAndCompare, compareScreenshots, takeOne, takeTwo, take }; diff --git a/libs/webutil.js b/libs/webutil.js index 421812da..7ba4afa0 100644 --- a/libs/webutil.js +++ b/libs/webutil.js @@ -3,7 +3,6 @@ /* eslint-disable max-len */ // eslint-disable-next-line import/no-import-module-exports import { expect } from '@playwright/test'; -import { getComparator } from 'playwright-core/lib/utils'; const fs = require('fs'); // eslint-disable-next-line import/no-extraneous-dependencies @@ -115,7 +114,7 @@ exports.WebUtil = class WebUtil { return result; } - /** + /** * Verifies that the specified CSS properties of the given locator match the expected values. * @param {Object} locator - The locator to verify CSS properties for. * @param {Object} cssProps - The CSS properties and expected values to verify. @@ -137,7 +136,7 @@ exports.WebUtil = class WebUtil { return result; } - /** + /** * Verifies that the specified CSS properties of the given locator match the expected values. * @param {Object} locator - The locator to verify CSS properties for. * @param {Object} cssProps - The CSS properties and expected values to verify. @@ -328,19 +327,11 @@ exports.WebUtil = class WebUtil { await this.page.unroute('**'); } - async takeScreenshot(folderPath, fileName, width, height) { - if (!fs.existsSync(folderPath)) { - fs.mkdirSync(folderPath, { recursive: true }); - } - await this.page.setViewportSize({ width, height }); - await this.page.screenshot({ path: `${folderPath}/${fileName}`, fullPage: true }); - } - -/** + /** * Generates analytic string for a given project. * @param {string} project - The project identifier, defaulting to 'milo' if not provided. * @returns {string} - A string formatted as 'gnav||nopzn|nopzn'. - */ + */ async getGnavDaalh(project=milo) { return 'gnav'+ '|' + project + '|'+ 'nopzn' + '|' + 'nopzn' ; } @@ -422,47 +413,4 @@ async getBlockDaalh(blockName, counter, pzn = false, pzntext = 'nopzn') { const slicedLastHeaderText = cleanAndSliceText(lastHeaderText); return `${slicedLinkText}-${counter}--${slicedLastHeaderText}`; } - - - async takeScreenshotAndCompare(urlA, callbackA, urlB, callbackB, folderPath, fileName) { - console.info(`[Test Page]: ${urlA}`); - await this.page.goto(urlA); - await callbackA(); - await this.page.screenshot({ path: `${folderPath}/${fileName}-a.png`, fullPage: true }); - const baseImage = fs.readFileSync(`${folderPath}/${fileName}-a.png`); - - console.info(`[Test Page]: ${urlB}`); - await this.page.goto(urlB); - await callbackB(); - await this.page.waitForSelector('.feds-footer-privacyLink'); - await this.page.screenshot({ path: `${folderPath}/${fileName}-b.png`, fullPage: true }); - const currImage = fs.readFileSync(`${folderPath}/${fileName}-b.png`); - - const comparator = getComparator('image/png'); - const diffImage = comparator(baseImage, currImage); - - if (diffImage) { - fs.writeFileSync(`${folderPath}/${fileName}-diff.png`, diffImage.diff); - console.info('Differences found'); - } - } - - static compareScreenshots(stableArray, betaArray, folderPath) { - const comparator = getComparator('image/png'); - for (let i = 0; i < stableArray.length; i += 1) { - if (betaArray[i].slice(-10) === stableArray[i].slice(-10)) { - const stableImage = fs.readFileSync(`${folderPath}/${stableArray[i]}`); - const betaImage = fs.readFileSync(`${folderPath}/${betaArray[i]}`); - const diffImage = comparator(stableImage, betaImage); - - if (diffImage) { - fs.writeFileSync(`${folderPath}/${stableArray[i]}-diff.png`, diffImage.diff); - console.info('Differences found'); - } - } else { - console.info('Screenshots are not matched'); - console.info(`${stableArray[i]} vs ${betaArray[i]}`); - } - } - } -}; \ No newline at end of file +}; diff --git a/selectors/uar/quiz.old.page.js b/selectors/uar/quiz.old.page.js deleted file mode 100644 index 9a60518f..00000000 --- a/selectors/uar/quiz.old.page.js +++ /dev/null @@ -1,144 +0,0 @@ -/* eslint-disable no-await-in-loop */ -/* eslint-disable no-restricted-syntax */ -const { WebUtil } = require('../../libs/webutil.js'); - -export default class QuizOldPage { - constructor(page) { - this.page = page; - this.webUtil = new WebUtil(page); - this.nextButton = page.getByRole('button', { name: 'Next' }); - this.resultButton = page.getByRole('button', { name: 'Get your results' }); - this.uarResult = page.locator('.uar-result h1'); - this.uarResult2 = page.locator('.uar-result b, .uar-quiz-result b'); - } - - /** - * Select answer - * @param {String} answer - */ - async selectAnswer(answer) { - const locator = `//div[text()="${answer}"]/ancestor::div[contains(@class,"quiz-card")]`; - await this.page.locator(locator).click(); - } - - /** - * Click next button - */ - async clickNextButton() { - await this.nextButton.click(); - await this.page.waitForTimeout(500); - } - - /** - * Click get your results button - */ - async clickResultButton() { - await this.resultButton.click(); - } - - /** - * Select each answer and click next button on question page - * @param {string} url - * @param {string} originalAnswer - */ - async clickEachAnswer(url, originalAnswer, keyNumber, isScreenshot = false) { - await this.page.goto(url); - - const answers = originalAnswer.split('>').map((x) => x.trim()); - - for (const answer of answers) { - if (answer.includes('+')) { - const options = answer.split('+').map((x) => x.trim()); - // select more than one answer - for (const option of options) { - await this.selectAnswer(option); - } - } else { - // select one answer - await this.selectAnswer(answer); - } - - if (answers.indexOf(answer) < answers.length - 1) { - if (isScreenshot) { - await this.page.waitForTimeout(500); - - const index = answers.indexOf(answer); - const folderPath = 'screenshots/uar'; - const desktopName = `${keyNumber} - old - desktop - ${index} - ${answer.replace('/', '')}.png`; - const tabletName = `${keyNumber} - old - tablet - ${index} - ${answer.replace('/', '')}.png`; - const mobileName = `${keyNumber} - old - mobile - ${index} - ${answer.replace('/', '')}.png`; - - await this.webUtil.takeScreenshot(folderPath, desktopName, 1920, 1080); - await this.webUtil.takeScreenshot(folderPath, tabletName, 768, 1024); - await this.webUtil.takeScreenshot(folderPath, mobileName, 375, 812); - } - // click next button - await this.clickNextButton(); - } else if (isScreenshot) { - await this.page.waitForTimeout(500); - const index = answers.length - 1; - const folderPath = 'screenshots/uar'; - const desktopName = `${keyNumber} - old - desktop - ${index} - ${answer.replace('/', '')}.png`; - const tabletName = `${keyNumber} - old - tablet - ${index} - ${answer.replace('/', '')}.png`; - const mobileName = `${keyNumber} - old - mobile - ${index} - ${answer.replace('/', '')}.png`; - - await this.webUtil.takeScreenshot(folderPath, desktopName, 1920, 1080); - await this.webUtil.takeScreenshot(folderPath, tabletName, 768, 1024); - await this.webUtil.takeScreenshot(folderPath, mobileName, 375, 812); - } - } - - // click get your results button - await this.clickResultButton(); - } - - /** - * Validate products on result page to match with expect products - * @param {string} name - */ - async checkResultPage(name, originalAnswer, keyNumber, isScreenshot = false) { - const oldProduct = []; - - const actualProduct = await this.uarResult.nth(0); - const text = await actualProduct.innerText(); - - if (text.includes('We think you\'ll love')) { - const actualProduct2 = await this.uarResult2.nth(0); - oldProduct.push(await actualProduct2.innerText()); - - if (name.includes('double') || name.includes('triple')) { - const actualProduct3 = await this.uarResult2.nth(1); - oldProduct.push(await actualProduct3.innerText()); - } - - if (name.includes('triple')) { - const actualProduct4 = await this.uarResult2.nth(2); - oldProduct.push(await actualProduct4.innerText()); - } - } else { - oldProduct.push(text); - } - - if (isScreenshot) { - await this.page.waitForTimeout(1000); - - const folderPath = 'screenshots/uar'; - const desktopName = `${keyNumber} - old - desktop - result.png`; - const tabletName = `${keyNumber} - old - tablet - result.png`; - const mobileName = `${keyNumber} - old - mobile - result.png`; - - await this.webUtil.takeScreenshot(folderPath, desktopName, 1920, 1080); - await this.webUtil.takeScreenshot(folderPath, tabletName, 768, 1024); - await this.webUtil.takeScreenshot(folderPath, mobileName, 375, 812); - } - - oldProduct.forEach((product, index) => { - if (product.includes('Great Deal') || product.includes('Businesses') || product.includes('Students and teachers')) { - oldProduct.splice(index, 1); - } - }); - - console.info(`==========old============\n${oldProduct.sort().join('')}`); - return oldProduct.sort().join(''); - } -} diff --git a/selectors/uar/quiz.page.js b/selectors/uar/quiz.page.js index a955c757..a0414fd4 100644 --- a/selectors/uar/quiz.page.js +++ b/selectors/uar/quiz.page.js @@ -1,8 +1,7 @@ /* eslint-disable no-await-in-loop */ /* eslint-disable no-restricted-syntax */ -import { expect } from '@playwright/test'; - const { WebUtil } = require('../../libs/webutil.js'); +const { take } = require('../../libs/visualutil.js'); export default class Quiz { constructor(page) { @@ -11,7 +10,9 @@ export default class Quiz { this.nextButton = page.getByRole('button', { name: 'Next' }); this.resultButton = page.locator('div.quiz-button-container > button'); this.uarResult = page.locator('.quiz-results h1'); - this.uarResult2 = page.locator('//div[contains(@data-path,"marquee-product")]//strong | //div[contains(@data-path,"check-bullet")]//h1 | //div[contains(@data-path,"express-product")]//h1'); + this.uarResult2 = page.locator('//div[contains(@data-path,"marquee-product")]//strong | ' + + '//div[contains(@data-path,"check-bullet")]//h1 | ' + + '//div[contains(@data-path,"express-product")]//h1'); this.uarResult3 = page.locator('//div[contains(@data-path,"card")]//h3'); this.screenshots = []; } @@ -44,7 +45,7 @@ export default class Quiz { * @param {string} url * @param {string} originalAnswer */ - async clickEachAnswer(url, originalAnswer, keyNumber, version, isScreenshot = false, folderPath = 'screenshots/uar') { + async clickEachAnswer(url, originalAnswer, keyNumber, version, project, folderPath, isScreenshot = false) { await this.page.goto(url); const answers = originalAnswer.split('>').map((x) => x.trim()); @@ -66,12 +67,13 @@ export default class Quiz { await this.page.waitForTimeout(500); const index = answers.indexOf(answer); - await this.takeScreenshotsOnThreeDimensions( + await this.takeScreenshot( keyNumber, version, index, answer, folderPath, + project, ); } @@ -80,12 +82,13 @@ export default class Quiz { } else if (isScreenshot) { await this.page.waitForTimeout(500); const index = answers.length - 1; - await this.takeScreenshotsOnThreeDimensions( + await this.takeScreenshot( keyNumber, version, index, answer, folderPath, + project, ); } } @@ -94,33 +97,33 @@ export default class Quiz { await this.clickResultButton(); } - async takeScreenshotsOnThreeDimensions(keyNumber, version, index, answer, folderPath) { - let desktopName; - let tabletName; - let mobileName; + async takeScreenshot(keyNumber, version, index, answer, folderPath, project) { + let fileName; if (answer === 'result') { - desktopName = `${keyNumber} - ${version} - desktop - result.png`; - tabletName = `${keyNumber} - ${version} - tablet - result.png`; - mobileName = `${keyNumber} - ${version} - mobile - result.png`; + fileName = `${keyNumber} - ${version} - ${project} - result.png`; } else { - desktopName = `${keyNumber} - ${version} - desktop - ${index} - ${answer.replace('/', '')}.png`; - tabletName = `${keyNumber} - ${version} - tablet - ${index} - ${answer.replace('/', '')}.png`; - mobileName = `${keyNumber} - ${version} - mobile - ${index} - ${answer.replace('/', '')}.png`; + fileName = `${keyNumber} - ${version} - ${project} - ${index} - ${answer.replace('/', '')}.png`; } - await this.webUtil.takeScreenshot(folderPath, desktopName, 1920, 1080); - await this.webUtil.takeScreenshot(folderPath, tabletName, 768, 1024); - await this.webUtil.takeScreenshot(folderPath, mobileName, 375, 812); + await take(this.page, folderPath, fileName); - this.screenshots.push(desktopName, tabletName, mobileName); + this.screenshots.push(fileName); } /** * Validate products on result page to match with expect products * @param {string} name */ - async checkResultPage(name, originalAnswer, keyNumber, version, isScreenshot = false, folderPath = 'screenshots/uar') { + async checkResultPage( + name, + originalAnswer, + keyNumber, + version, + project, + folderPath, + isScreenshot = false, + ) { const newProduct = []; let currentUrl = await this.page.url(); @@ -197,31 +200,17 @@ export default class Quiz { if (isScreenshot) { await this.page.waitForTimeout(1000); - await this.takeScreenshotsOnThreeDimensions( + await this.takeScreenshot( keyNumber, version, 0, 'result', folderPath, + project, ); } console.info(`==========new============\n${newProduct.sort().join('')}`); return newProduct.sort().join(''); } - - async checkResultPageDX(value, keyNumber, version, isScreenshot = false, folderPath = 'screenshots/dx') { - expect.soft(await this.page.url()).toContain(value); - if (isScreenshot) { - await this.page.waitForTimeout(1000); - - await this.takeScreenshotsOnThreeDimensions( - keyNumber, - version, - 0, - 'result', - folderPath, - ); - } - } } diff --git a/tests/uar/quiz.screenshots.test.js b/tests/uar/quiz.screenshots.test.js deleted file mode 100644 index d0cfc3b1..00000000 --- a/tests/uar/quiz.screenshots.test.js +++ /dev/null @@ -1,57 +0,0 @@ -/* eslint-disable no-await-in-loop */ -/* eslint-disable no-loop-func */ -/* eslint-disable no-restricted-syntax */ -import { expect, test } from '@playwright/test'; -import Quiz from '../../selectors/uar/quiz.page.js'; - -const QuizSpec = require('../../features/uar/quiz.screenshots.spec.js'); - -const { features } = QuizSpec; -const { WebUtil } = require('../../libs/webutil.js'); -const envs = require('../../envs/envs.js'); - -test.describe('Quiz flow test suite', () => { - // reset timeout because we use this to run all test data - test.setTimeout(10 * 60 * 1000); - for (const feature of features) { - test( - `${feature.name}, ${feature.tags}`, - async ({ page, baseURL }) => { - const quiz = new Quiz(page); - const quizOldPage = new Quiz(page); - const url = `${baseURL}${feature.path}`; - console.info(url); - - // load test data from static files - const testdata = await WebUtil.loadTestData(`${feature.data}`); - - let keyNumber = 0; - - for (const key of Object.keys(testdata)) { - console.log(key); - let oldProduct = ''; - let newProduct = ''; - keyNumber += 1; - - await test.step(`Stable: Select each answer on test page according to ${key}`, async () => { - await quizOldPage.clickEachAnswer(`${envs['@milo_stage']}${feature.path}`, key, keyNumber, 'stable', true); - }); - - await test.step('Stable: Check results on test page', async () => { - oldProduct = await quizOldPage.checkResultPage(testdata[key], key, keyNumber, 'stable', true); - }); - - await test.step(`Beta: Select each answer on test page according to ${key}`, async () => { - await quiz.clickEachAnswer(url, key, keyNumber, 'beta', true); - }); - - await test.step('Beta: Check results on test page', async () => { - newProduct = await quiz.checkResultPage(testdata[key], key, keyNumber, 'beta', true); - }); - - expect.soft(oldProduct).toContain(newProduct); - } - }, - ); - } -}); diff --git a/tests/visual/uar/quiz.test.js b/tests/visual/uar/quiz.test.js new file mode 100644 index 00000000..424091c2 --- /dev/null +++ b/tests/visual/uar/quiz.test.js @@ -0,0 +1,67 @@ +/* eslint-disable no-await-in-loop */ +/* eslint-disable no-loop-func */ +/* eslint-disable no-restricted-syntax */ +import { test } from '@playwright/test'; +import Quiz from '../../../selectors/uar/quiz.page.js'; + +const { features } = require('../../../features/visual/uar/quiz.spec.js'); +const { WebUtil } = require('../../../libs/webutil.js'); +const { compareScreenshots } = require('../../../libs/visualutil.js'); +const envs = require('../../../envs/envs.js'); + +const folderPath = 'screenshots/uar'; + +test.describe('Quiz flow test suite', () => { + // reset timeout because we use this to run all test data + test.setTimeout(10 * 60 * 1000); + for (const feature of features) { + test( + `${feature.name}, ${feature.tags}`, + async ({ page }, testInfo) => { + const stablePage = new Quiz(page); + const betaPage = new Quiz(page); + const stableURL = `${envs[feature.stable]}${feature.path}`; + console.info(stableURL); + const betaURL = `${envs[feature.beta]}${feature.path}`; + console.info(betaURL); + + // load test data from static files + const testdata = await WebUtil.loadTestData(`${feature.data}`); + + let keyNumber = 0; + + for (const key of Object.keys(testdata)) { + console.log(key); + let stableProductScreenshots = []; + let betaProductScreenshots = []; + keyNumber += 1; + const project = testInfo.project.name; + + await test.step(`Stable: Select each answer on test page according to ${key}`, async () => { + await stablePage.clickEachAnswer(stableURL, key, keyNumber, 'stable', project, folderPath, true); + }); + + await test.step('Stable: Check results on test page', async () => { + await stablePage.checkResultPage(testdata[key], key, keyNumber, 'stable', project, folderPath, true); + }); + + stableProductScreenshots = stablePage.screenshots.slice(); + stablePage.screenshots = []; + + await test.step(`Beta: Select each answer on test page according to ${key}`, async () => { + await betaPage.clickEachAnswer(betaURL, key, keyNumber, 'beta', project, folderPath, true); + }); + + await test.step('Beta: Check results on test page', async () => { + await betaPage.checkResultPage(testdata[key], key, keyNumber, 'beta', project, folderPath, true); + }); + + betaProductScreenshots = betaPage.screenshots.slice(); + betaPage.screenshots = []; + + compareScreenshots(stableProductScreenshots, betaProductScreenshots, folderPath); + } + }, + ); + } +});