add tests for graybox-cc and graybox-homepage (#464)

* add tests for graybox-bacom

* just use chrome

* add more waiting time

* add merch card

* add 404 checker for graxybox-bacom

* update gitignore and file name

* add screenshots for graybox-dc

* add 404 checking for graybox dc

* adjust screenshot tests

* update screenshot tests to public s3 and git action to run them

* fix git action

* update git action

* install dependencies

* only able to run hlx.live pages not stage pages

* add tests for graybox cc and homepage

configs/graybox-bacom.config.js

@@ -65,7 +65,7 @@ const config = {
       name: 'bacom-live-chrome',
       use: {
         ...devices['Desktop Chrome'],
-        baseURL: envs['@bacom_graybox'],
+        baseURL: envs['@graybox_bacom'],
       },
     },
     {

configs/graybox-cc.config.js

@@ -0,0 +1,80 @@
+// @ts-check
+const { devices } = require('@playwright/test');
+
+const envs = require('../envs/envs.js');
+
+/**
+ * @see https://playwright.dev/docs/test-configuration
+ * @type {import('@playwright/test').PlaywrightTestConfig}
+ */
+const config = {
+  testDir: '../tests/',
+  testMatch: ['cc/**/*.test.js', 'milo/**/*.test.js', 'graybox/**/*.test.js'],
+  outputDir: '../test-results',
+  /* Maximum time one test can run for. */
+  timeout: 45 * 1000,
+  expect: {
+    /**
+     * Maximum time expect() should wait for the condition to be met.
+     * For example in `await expect(locator).toHaveText();`
+     */
+    timeout: 10 * 1000,
+  },
+  /* Run tests in files in parallel */
+  fullyParallel: true,
+  /* Fail the build on CI if you accidentally left test.only in the source code. */
+  forbidOnly: !!process.env.CI,
+  /* Retry on CI only */
+  retries: process.env.CI ? 2 : 0,
+  /* Opt out of parallel tests on CI. */
+  workers: process.env.CI ? 2 : undefined,
+  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+  reporter: process.env.CI
+    ? [['github'], ['list'], ['../utils/reporters/base-reporter.js'], ['json', { outputFile: '../test-json-results/test-results.json' }]]
+    : [
+      [
+        'html',
+        {
+          outputFolder: 'test-html-results',
+          open: 'never',
+        },
+      ],
+      ['list'],
+      ['../utils/reporters/base-reporter.js'],
+      ['json', { outputFile: '../test-json-results/test-results.json' }],
+    ],
+
+  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  use: {
+    /* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */
+    actionTimeout: 60000,
+    /* Base URL to use in actions like `await page.goto('/')`. */
+    // baseURL: 'http://localhost:3000',
+
+    /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+    trace: 'on-first-retry',
+    baseURL:
+      process.env.BASE_URL
+      || envs['@graybox_dc']
+      || 'https://main--cc--adobecom.hlx.live',
+  },
+
+  /* Configure projects for major browsers */
+  projects: [
+    {
+      name: 'cc-live-chrome',
+      use: {
+        ...devices['Desktop Chrome'],
+        baseURL: envs['@graybox_cc'],
+      },
+    },
+    {
+      name: 'cc-stage-chrome',
+      use: {
+        ...devices['Desktop Chrome'],
+        baseURL: envs['@adobe_stage'],
+      },
+    },
+  ],
+};
+module.exports = config;

configs/graybox-homepage.config.js

@@ -0,0 +1,80 @@
+// @ts-check
+const { devices } = require('@playwright/test');
+
+const envs = require('../envs/envs.js');
+
+/**
+ * @see https://playwright.dev/docs/test-configuration
+ * @type {import('@playwright/test').PlaywrightTestConfig}
+ */
+const config = {
+  testDir: '../tests/',
+  testMatch: ['dc/**/*.test.js', 'milo/**/*.test.js', 'graybox/**/*.test.js'],
+  outputDir: '../test-results',
+  /* Maximum time one test can run for. */
+  timeout: 45 * 1000,
+  expect: {
+    /**
+     * Maximum time expect() should wait for the condition to be met.
+     * For example in `await expect(locator).toHaveText();`
+     */
+    timeout: 10 * 1000,
+  },
+  /* Run tests in files in parallel */
+  fullyParallel: true,
+  /* Fail the build on CI if you accidentally left test.only in the source code. */
+  forbidOnly: !!process.env.CI,
+  /* Retry on CI only */
+  retries: process.env.CI ? 2 : 0,
+  /* Opt out of parallel tests on CI. */
+  workers: process.env.CI ? 2 : undefined,
+  /* Reporter to use. See https://playwright.dev/docs/test-reporters */
+  reporter: process.env.CI
+    ? [['github'], ['list'], ['../utils/reporters/base-reporter.js'], ['json', { outputFile: '../test-json-results/test-results.json' }]]
+    : [
+      [
+        'html',
+        {
+          outputFolder: 'test-html-results',
+          open: 'never',
+        },
+      ],
+      ['list'],
+      ['../utils/reporters/base-reporter.js'],
+      ['json', { outputFile: '../test-json-results/test-results.json' }],
+    ],
+
+  /* Shared settings for all the projects below. See https://playwright.dev/docs/api/class-testoptions. */
+  use: {
+    /* Maximum time each action such as `click()` can take. Defaults to 0 (no limit). */
+    actionTimeout: 60000,
+    /* Base URL to use in actions like `await page.goto('/')`. */
+    // baseURL: 'http://localhost:3000',
+
+    /* Collect trace when retrying the failed test. See https://playwright.dev/docs/trace-viewer */
+    trace: 'on-first-retry',
+    baseURL:
+      process.env.BASE_URL
+      || envs['@graybox_homepage']
+      || 'https://www.stage.adobe.com',
+  },
+
+  /* Configure projects for major browsers */
+  projects: [
+    {
+      name: 'homepage-live-chrome',
+      use: {
+        ...devices['Desktop Chrome'],
+        baseURL: envs['@graybox_homepage'],
+      },
+    },
+    {
+      name: 'homepage-stage-chrome',
+      use: {
+        ...devices['Desktop Chrome'],
+        baseURL: envs['@adobe_stage'],
+      },
+    },
+  ],
+};
+module.exports = config;

data/graybox/sot-cc.yml

@@ -0,0 +1,15 @@
+---
+'CreativeCloud': 'https://www.stage.adobe.com/creativecloud.html'
+'CreativeCloudFeatures': 'https://www.stage.adobe.com/creativecloud/features.html'
+'CreativeCloudPhotography': 'https://www.stage.adobe.com/creativecloud/photography.html'
+'CreativeCloudStudents': 'https://www.stage.adobe.com/creativecloud/buy/students.html'
+'CreativeCloudAllApps': 'https://www.stage.adobe.com/creativecloud/all-apps.html'
+'CreativeCloudPhotoshop': 'https://www.stage.adobe.com/products/photoshop.html'
+'CreativeCloudPhotoshopLightroom': 'https://www.stage.adobe.com/products/photoshop-lightroom.html'
+'CreativeCloudIllustrator': 'https://www.stage.adobe.com/products/illustrator.html'
+'CreativeCloudPremiere': 'https://www.stage.adobe.com/products/premiere.html'
+'CreativeCloudAIOverview': 'https://www.stage.adobe.com/ai/overview.html'
+'CreativeCloudFirefly': 'https://www.stage.adobe.com/products/firefly.html'
+'CreativeCloudPhotographyApps': 'https://www.stage.adobe.com/creativecloud/photography/apps.html'
+'CreativeCloudDesign': 'https://www.stage.adobe.com/creativecloud/design.html'
+'CreativeCloudVideo': 'https://www.stage.adobe.com/creativecloud/video.html'

data/graybox/sot-homepage.yml

@@ -0,0 +1,10 @@
+---
+'AdobeHomepageNoMEP': 'https://www.stage.adobe.com/?mep=off'
+'AdobeHomepage': 'https://www.stage.adobe.com'
+'AdobeHomepageUK': 'https://www.stage.adobe.com/uk/?mep=off&akamaiLocale=uk'
+'AdobeHomepageUKNoMEP': 'https://www.stage.adobe.com/uk/?akamaiLocale=uk'
+'AdobeHomepageDE': 'https://www.stage.adobe.com/de?akamaiLocale=de'
+'AdobeHomepageFR': 'https://www.stage.adobe.com/fr?akamaiLocale=fr'
+'AdobeHomepageES': 'https://www.stage.adobe.com/es?akamaiLocale=es'
+'AdobeHomepageJP': 'https://www.stage.adobe.com/jp?akamaiLocale=jp'
+'AdobeHomepageMENA': 'https://www.stage.adobe.com/mena_ar?akamaiLocale=mena'

envs/envs.js

@@ -27,4 +27,6 @@ module.exports = {
   '@express_live': 'https://main--express--adobecom.hlx.live',
   '@graybox_bacom': 'https://test.business-graybox.adobe.com',
   '@graybox_dc': 'https://test.graybox.adobe.com',
+  '@graybox_homepage': 'https://test.graybox.adobe.com',
+  '@graybox_cc': 'https://test.graybox.adobe.com',
 };

features/graybox/sot.cc.404.spec.js

@@ -0,0 +1,13 @@
+module.exports = {
+  name: 'Graybox CC SOT 404 Check',
+  features: [
+    {
+      tcid: '0',
+      name: '@graybox-cc-404-check',
+      stable: '@adobe_stage',
+      beta: '@graybox_cc',
+      tags: '@graybox-cc-404-check',
+      data: 'data/graybox/sot-cc.yml',
+    },
+  ],
+};

features/graybox/sot.homepage.404.spec.js

@@ -0,0 +1,13 @@
+module.exports = {
+  name: 'Graybox Homepage SOT 404 Check',
+  features: [
+    {
+      tcid: '0',
+      name: '@graybox-homepage-404-check',
+      stable: '@adobe_stage',
+      beta: '@graybox_homepage',
+      tags: '@graybox-homepage-404-check',
+      data: 'data/graybox/sot-homepage.yml',
+    },
+  ],
+};

features/visual/graybox/sot.cc.spec.js

@@ -0,0 +1,13 @@
+module.exports = {
+  name: 'Graybox CC SOT Visual Comparison',
+  features: [
+    {
+      tcid: '0',
+      name: '@graybox-cc-visual',
+      stable: '@adobe_stage',
+      beta: '@graybox_cc',
+      tags: '@graybox-cc-visual @visual @graybox-cc-screenshots',
+      data: 'data/graybox/sot-cc.yml',
+    },
+  ],
+};

features/visual/graybox/sot.homepage.spec.js

@@ -0,0 +1,13 @@
+module.exports = {
+  name: 'Graybox Homepage SOT Visual Comparison',
+  features: [
+    {
+      tcid: '0',
+      name: '@graybox-homepage-visual',
+      stable: '@adobe_stage',
+      beta: '@graybox_homepage  ',
+      tags: '@graybox-homepage-visual @visual @graybox-homepage-screenshots',
+      data: 'data/graybox/sot-homepage.yml',
+    },
+  ],
+};

tests/graybox/sot.bacom.404.test.js

@@ -49,7 +49,9 @@ test.describe('Graybox Bacom SOT 404 Check test suite', () => {
         // Wait for some time to ensure all resources are loaded
         await page.waitForLoadState('networkidle'); // Better way to wait for page load
         // {{ edit_1 }}: Log all links on the page
-        const links = await page.$$eval('a', (anchors) => anchors.map((anchor) => anchor.href));
+        const links = await page.$$eval('a', (anchors) => anchors
+          .map((anchor) => anchor.href)
+          .filter((href) => !href.includes('test.business-graybox')));
         console.log('Links on the page:', links);
         results[betaURL] = {
           four0FourErrors,

tests/graybox/sot.cc.404.test.js

@@ -0,0 +1,70 @@
+/* eslint-disable no-restricted-syntax */
+/* eslint-disable import/named */
+import { test } from '@playwright/test';
+import { features } from '../../features/graybox/sot.cc.404.spec.js';
+
+const fs = require('fs'); // Add this import at the top
+
+const { WebUtil } = require('../../libs/webutil.js');
+
+const results = {};
+let consoleErrors = [];
+let four0FourErrors = [];
+
+test.describe('Graybox CC SOT 404 Check test suite', () => {
+  test.setTimeout(20 * 60 * 1000);
+  test.beforeEach(async ({ page }) => {
+    // Check for 404s
+    page.on('response', (response) => {
+      if (response.status() === 404) {
+        console.log(`Resource not found: ${response.url()}`);
+        four0FourErrors.push(`Resource not found: ${response.url()}`);
+      }
+    });
+
+    // Check for console errors
+    page.on('console', (msg) => {
+      if (msg.type() === 'error') {
+        console.error(`Console error: ${msg.text()}`);
+        consoleErrors.push(`Console error: ${msg.text()}`); // Fix to store console errors in the results object
+      }
+    });
+  });
+
+  for (const feature of features) {
+    // eslint-disable-next-line no-loop-func
+    test(`${feature.name},${feature.tags}`, async ({ page }) => {
+      // load test data from static files
+      const testdata = await WebUtil.loadTestData(`${feature.data}`);
+
+      for (const key of Object.keys(testdata)) {
+        consoleErrors = [];
+        four0FourErrors = [];
+        const stableURL = testdata[key];
+        const betaURL = stableURL.replace('www.stage', 'test.graybox');
+        console.info('Checking for 404s on:', betaURL);
+        // Go to the page you want to check
+        await page.goto(betaURL);
+
+        // Wait for some time to ensure all resources are loaded
+        await page.waitForLoadState('networkidle'); // Better way to wait for page load
+        // {{ edit_1 }}: Log all links on the page
+        const links = await page.$$eval('a', (anchors) => anchors
+          .map((anchor) => anchor.href)
+          .filter((href) => !href.includes('test.graybox')));
+        console.log('Links on the page:', links);
+        results[betaURL] = {
+          four0FourErrors,
+          consoleErrors,
+          links,
+        };
+      }
+    });
+  }
+
+  // {{ edit_3 }}: Write results to JSON file after all tests
+  test.afterAll(async () => {
+    fs.writeFileSync('graybox-cc-404-results.json', JSON.stringify(results, null, 2));
+    console.log('Results saved to graybox-cc-404-results.json');
+  });
+});

tests/graybox/sot.dc.404.test.js

@@ -49,7 +49,9 @@ test.describe('Graybox DC SOT 404 Check test suite', () => {
         // Wait for some time to ensure all resources are loaded
         await page.waitForLoadState('networkidle'); // Better way to wait for page load
         // {{ edit_1 }}: Log all links on the page
-        const links = await page.$$eval('a', (anchors) => anchors.map((anchor) => anchor.href));
+        const links = await page.$$eval('a', (anchors) => anchors
+          .map((anchor) => anchor.href)
+          .filter((href) => !href.includes('test.graybox')));
         console.log('Links on the page:', links);
         results[betaURL] = {
           four0FourErrors,