fix: projected traffic calculation for meta-tags

src/common/opportunity.js

@@ -61,7 +61,7 @@ export async function convertToOpportunity(auditUrl, auditData, context, createO
       return opportunity;
     } else {
       opportunity.setAuditId(auditData.id);
-      if (auditType === Audit.AUDIT_TYPES.CWV) {
+      if (auditType === Audit.AUDIT_TYPES.CWV || auditType === Audit.AUDIT_TYPES.META_TAGS) {
         opportunity.setData({
           ...opportunity.getData(),
           ...props, // kpiDeltas

src/metatags/handler.js

@@ -10,11 +10,13 @@
  * governing permissions and limitations under the License.
  */
 
+import RUMAPIClient from '@adobe/spacecat-shared-rum-api-client';
 import { Audit } from '@adobe/spacecat-shared-data-access';
+import { calculateCPCValue } from '../support/utils.js';
 import { getObjectFromKey, getObjectKeysUsingPrefix } from '../utils/s3-utils.js';
 import SeoChecks from './seo-checks.js';
 import { AuditBuilder } from '../common/audit-builder.js';
-import { noopUrlResolver } from '../common/index.js';
+import { noopUrlResolver, wwwUrlResolver } from '../common/index.js';
 import metatagsAutoSuggest from './metatags-auto-suggest.js';
 import { convertToOpportunity } from '../common/opportunity.js';
 import { getTopPagesForSiteId } from '../canonical/handler.js';
@@ -32,8 +34,14 @@ export async function opportunityAndSuggestions(auditUrl, auditData, context) {
     context,
     createOpportunityData,
     auditType,
+    {
+      projectedTrafficLost: auditData.auditResult.projectedTrafficLost,
+      projectedTrafficValue: auditData.auditResult.projectedTrafficValue,
+    },
   );
   const { log } = context;
+  log.info(`projected traffic data - ${auditData.auditResult.projectedTrafficLost} , ${auditData.auditResult.projectedTrafficValue}`);
+  log.info(`Opportunity data - ${opportunity.getData()}`);
   const { detectedTags } = auditData.auditResult;
   const suggestions = [];
   // Generate suggestions data to be inserted in meta-tags opportunity suggestions
@@ -92,6 +100,80 @@ export async function fetchAndProcessPageObject(s3Client, bucketName, key, prefi
   };
 }
 
+// Extract endpoint from a url, removes trailing slash if present
+function extractEndpoint(url) {
+  const urlObj = new URL(url);
+  return urlObj.pathname.replace(/\/$/, '');
+}
+
+// Preprocess RUM data into a map with endpoint as the key
+function preprocessRumData(rumDataMonthly, rumDataBiMonthly) {
+  const rumDataMapMonthly = new Map();
+  const rumDataMapBiMonthly = new Map();
+  rumDataMonthly.forEach((item) => {
+    const endpoint = extractEndpoint(item.url);
+    rumDataMapMonthly.set(endpoint, item);
+  });
+  rumDataBiMonthly.forEach((item) => {
+    const endpoint = extractEndpoint(item.url);
+    rumDataMapBiMonthly.set(endpoint, item);
+  });
+  return {
+    rumDataMapMonthly,
+    rumDataMapBiMonthly,
+  };
+}
+
+// Get organic traffic for a given endpoint
+function getOrganicTrafficForEndpoint(endpoint, rumDataMapMonthly, rumDataMapBiMonthly, log) {
+  // remove trailing slash from endpoint, if present, and then find in the datamap
+  const target = rumDataMapMonthly.get(endpoint.replace(/\/$/, ''))
+    || rumDataMapBiMonthly.get(endpoint.replace(/\/$/, ''));
+  if (!target) {
+    log.warn(`No rum data found for ${endpoint}.`);
+    return 0;
+  }
+  const trafficSum = target.earned + target.paid;
+  log.info(`Found ${trafficSum} page views for ${endpoint}.`);
+  return trafficSum;
+}
+
+// Calculate the projected traffic lost for a site
+async function calculateProjectedTraffic(context, site, detectedTags, log) {
+  const rumAPIClient = RUMAPIClient.createFrom(context);
+  const options = {
+    domain: wwwUrlResolver(site),
+    interval: 30,
+    granularity: 'DAILY',
+  };
+  const queryResultsMonthly = await rumAPIClient.query('traffic-acquisition', options);
+  const queryResultsBiMonthly = await rumAPIClient.query('traffic-acquisition', {
+    ...options,
+    interval: 60,
+  });
+  const { rumDataMapMonthly, rumDataMapBiMonthly } = preprocessRumData(
+    queryResultsMonthly,
+    queryResultsBiMonthly,
+  );
+  let projectedTraffic = 0;
+  Object.entries(detectedTags).forEach(([endpoint, tags]) => {
+    const organicTraffic = getOrganicTrafficForEndpoint(
+      endpoint,
+      rumDataMapMonthly,
+      rumDataMapBiMonthly,
+      log,
+    );
+    Object.values((tags)).forEach((tagIssueDetails) => {
+      // Multiplying by 1% for missing tags, and 0.5% for other tag issues
+      // For duplicate tags, each page's traffic is multiplied by .5% so
+      // it amounts to 0.5% * number of duplicates.
+      const multiplier = tagIssueDetails.issue.includes('Missing') ? 0.01 : 0.005;
+      projectedTraffic += organicTraffic * multiplier;
+    });
+  });
+  return projectedTraffic;
+}
+
 export async function auditMetaTagsRunner(baseURL, context, site) {
   const { log, s3Client, dataAccess } = context;
   // Get top pages for a site
@@ -119,24 +201,37 @@ export async function auditMetaTagsRunner(baseURL, context, site) {
   if (extractedTagsCount === 0) {
     log.error(`Failed to extract tags from scraped content for bucket ${bucketName} and prefix ${prefix}`);
   }
-  log.info(`Performing SEO checks for ${extractedTagsCount} tags`);
+
   // Perform SEO checks
+  log.info(`Performing SEO checks for ${extractedTagsCount} tags`);
   const seoChecks = new SeoChecks(log);
   for (const [pageUrl, pageTags] of Object.entries(extractedTags)) {
     seoChecks.performChecks(pageUrl, pageTags);
   }
   seoChecks.finalChecks();
+  const detectedTags = seoChecks.getDetectedTags();
+
+  // Calculate projected traffic lost
+  const projectedTrafficLost = await calculateProjectedTraffic(context, site, detectedTags, log);
+  const cpcValue = await calculateCPCValue(context, site.getId());
+  log.info(`Calculated cpc value: ${cpcValue} for site: ${site.getId()}`);
+  const projectedTrafficValue = projectedTrafficLost * cpcValue;
+
+  // Generate AI suggestions for detected tags if auto-suggest enabled for site
   const allTags = {
     detectedTags: seoChecks.getDetectedTags(),
     healthyTags: seoChecks.getFewHealthyTags(),
     extractedTags,
   };
   const updatedDetectedTags = await metatagsAutoSuggest(allTags, context, site);
+
   const auditResult = {
     detectedTags: updatedDetectedTags,
     sourceS3Folder: `${bucketName}/${prefix}`,
-    fullAuditRef: 'na',
+    fullAuditRef: '',
     finalUrl: baseURL,
+    projectedTrafficLost,
+    projectedTrafficValue,
   };
   return {
     auditResult,

src/metatags/opportunity-data-mapper.js

@@ -10,7 +10,7 @@
  * governing permissions and limitations under the License.
  */
 
-export function createOpportunityData() {
+export function createOpportunityData(props = {}) {
   return {
     runbook: 'https://adobe.sharepoint.com/:w:/r/sites/aemsites-engineering/_layouts/15/doc2.aspx?sourcedoc=%7B27CF48AA-5492-435D-B17C-01E38332A5CA%7D&file=Experience_Success_Studio_Metatags_Runbook.docx&action=default&mobileredirect=true',
     origin: 'AUTOMATION',
@@ -26,6 +26,8 @@ export function createOpportunityData() {
       ],
     },
     tags: ['Traffic acquisition'],
-    data: null,
+    data: {
+      ...props,
+    },
   };
 }

src/support/utils.js

@@ -23,6 +23,7 @@ import { ListObjectsV2Command } from '@aws-sdk/client-s3';
 import { getObjectFromKey } from '../utils/s3-utils.js';
 
 URI.preventInvalidHostname = true;
+const DEFAULT_CPC_VALUE = 2.69; // $2.69
 
 // weekly pageview threshold to eliminate urls with lack of samples
 
@@ -211,6 +212,48 @@ export const extractLinksFromHeader = (data, baseUrl, log) => {
   return links;
 };
 
+/**
+ * Fetches the organic traffic data for a site from S3 and calculate the CPC value as per
+ * https://wiki.corp.adobe.com/pages/viewpage.action?spaceKey=AEMSites&title=Success+Studio+Projected+Business+Impact+Metrics#SuccessStudioProjectedBusinessImpactMetrics-IdentifyingCPCvalueforadomain
+ * @param context
+ * @param siteId
+ * @returns {number} CPC value
+ */
+export async function calculateCPCValue(context, siteId) {
+  if (!context?.env?.S3_IMPORTER_BUCKET_NAME) {
+    throw new Error('S3 importer bucket name is required');
+  }
+  if (!context.s3Client) {
+    throw new Error('S3 client is required');
+  }
+  if (!context.log) {
+    throw new Error('Logger is required');
+  }
+  if (!siteId) {
+    throw new Error('SiteId is required');
+  }
+  const { s3Client, log } = context;
+  const bucketName = context.env.S3_IMPORTER_BUCKET_NAME;
+  const key = `metrics/${siteId}/ahrefs/organic-traffic.json`;
+  try {
+    const organicTrafficData = await getObjectFromKey(s3Client, bucketName, key, log);
+    if (!Array.isArray(organicTrafficData) || organicTrafficData.length === 0) {
+      log.warn(`Organic traffic data not available for ${siteId}. Using Default CPC value.`);
+      return DEFAULT_CPC_VALUE;
+    }
+    const lastTraffic = organicTrafficData[organicTrafficData.length - 1];
+    if (!lastTraffic.cost || !lastTraffic.value) {
+      log.warn(`Invalid organic traffic data present for ${siteId} - cost:${lastTraffic.cost} value:${lastTraffic.value}, Using Default CPC value.`);
+      return DEFAULT_CPC_VALUE;
+    }
+    // dividing by 100 for cents to dollar conversion
+    return lastTraffic.cost / lastTraffic.value / 100;
+  } catch (err) {
+    log.error(`Error fetching organic traffic data for site ${siteId}. Using Default CPC value.`, err);
+    return DEFAULT_CPC_VALUE;
+  }
+}
+
 export const getScrapedDataForSiteId = async (site, context) => {
   const { s3Client, env, log } = context;
   const siteId = site.getId();

src/utils/s3-utils.js

@@ -22,10 +22,10 @@ export async function getObjectKeysUsingPrefix(
   let continuationToken = null;
   if (!s3Client || !bucketName || !prefix) {
     log.error(
-      `Invalid input parameters: ensure s3Client, bucketName:${bucketName}, and prefix:${prefix} are provided.`,
+      `Invalid input parameters in getObjectKeysUsingPrefix: ensure s3Client, bucketName:${bucketName}, and prefix:${prefix} are provided.`,
     );
     throw new Error(
-      'Invalid input parameters: ensure s3Client, bucketName, and prefix are provided.',
+      'Invalid input parameters in getObjectKeysUsingPrefix: ensure s3Client, bucketName, and prefix are provided.',
     );
   }
   try {
@@ -41,7 +41,9 @@ export async function getObjectKeysUsingPrefix(
       // eslint-disable-next-line no-await-in-loop
       const data = await s3Client.send(new ListObjectsV2Command(params));
       data?.Contents?.forEach((obj) => {
-        objectKeys.push(obj.Key);
+        if (obj.Key?.endsWith('scrape.json')) {
+          objectKeys.push(obj.Key);
+        }
       });
       continuationToken = data?.NextContinuationToken;
     } while (continuationToken);
@@ -72,7 +74,7 @@ export async function getObjectKeysUsingPrefix(
 export async function getObjectFromKey(s3Client, bucketName, key, log) {
   if (!s3Client || !bucketName || !key) {
     log.error(
-      'Invalid input parameters: ensure s3Client, bucketName, and key are provided.',
+      'Invalid input parameters in getObjectFromKey: ensure s3Client, bucketName, and key are provided.',
     );
     return null;
   }
@@ -93,7 +95,6 @@ export async function getObjectFromKey(s3Client, bucketName, key, log) {
         return null;
       }
     }
-
     // Always return body for non-JSON content types
     return body;
   } catch (err) {