fix: projected traffic calculation for meta-tags

src/metatags/handler.js

@@ -10,11 +10,13 @@
  * governing permissions and limitations under the License.
  */
 
+import RUMAPIClient from '@adobe/spacecat-shared-rum-api-client';
 import { getObjectFromKey, getObjectKeysUsingPrefix } from '../utils/s3-utils.js';
 import SeoChecks from './seo-checks.js';
-import { AuditBuilder } from '../common/audit-builder.js';
-import { noopUrlResolver } from '../common/audit.js';
 import convertToOpportunity from './opportunityHandler.js';
+import { calculateCPCValue, getRUMDomainkey } from '../support/utils.js';
+import { noopUrlResolver, wwwUrlResolver } from '../common/audit.js';
+import { AuditBuilder } from '../common/audit-builder.js';
 
 export async function fetchAndProcessPageObject(s3Client, bucketName, key, prefix, log) {
   const object = await getObjectFromKey(s3Client, bucketName, key, log);
@@ -32,6 +34,85 @@ export async function fetchAndProcessPageObject(s3Client, bucketName, key, prefi
   };
 }
 
+// Extract endpoint from a url, removes trailing slash if present
+function extractEndpoint(url) {
+  const urlObj = new URL(url);
+  return urlObj.pathname.replace(/\/$/, '');
+}
+
+// Preprocess RUM data into a map with endpoint as the key
+function preprocessRumData(rumDataMonthly, rumDataBiMonthly) {
+  const rumDataMapMonthly = new Map();
+  const rumDataMapBiMonthly = new Map();
+  rumDataMonthly.forEach((item) => {
+    const endpoint = extractEndpoint(item.url);
+    rumDataMapMonthly.set(endpoint, item);
+  });
+  rumDataBiMonthly.forEach((item) => {
+    const endpoint = extractEndpoint(item.url);
+    rumDataMapBiMonthly.set(endpoint, item);
+  });
+  return {
+    rumDataMapMonthly,
+    rumDataMapBiMonthly,
+  };
+}
+
+// Get organic traffic for a given endpoint
+function getOrganicTrafficForEndpoint(endpoint, rumDataMapMonthly, rumDataMapBiMonthly, log) {
+  // remove trailing slash from endpoint, if present, and then find in the datamap
+  const target = rumDataMapMonthly.get(endpoint.replace(/\/$/, ''))
+    || rumDataMapBiMonthly.get(endpoint.replace(/\/$/, ''));
+  if (!target) {
+    log.warn(`No rum data found for ${endpoint}.`);
+    return 0;
+  }
+  const trafficSum = target.earned + target.paid;
+  log.info(`Found ${trafficSum} page views for ${endpoint}.`);
+  return trafficSum;
+}
+
+// Calculate the projected traffic lost for a site
+async function calculateProjectedTraffic(context, site, detectedTags, log) {
+  const rumAPIClient = RUMAPIClient.createFrom(context);
+  const domainkey = await getRUMDomainkey(site.getBaseURL(), context);
+  const options = {
+    domain: wwwUrlResolver(site),
+    domainkey,
+    interval: 30,
+    granularity: 'DAILY',
+  };
+  const queryResultsMonthly = await rumAPIClient.query('traffic-acquisition', options);
+  const queryResultsBiMonthly = await rumAPIClient.query('traffic-acquisition', {
+    ...options,
+    interval: 60,
+  });
+  const { rumDataMapMonthly, rumDataMapBiMonthly } = preprocessRumData(
+    queryResultsMonthly,
+    queryResultsBiMonthly,
+  );
+  let projectedTraffic = 0;
+  log.warn(`Detected Tags: ${JSON.stringify(detectedTags)}`);
+  Object.entries(detectedTags).forEach(([endpoint, tags]) => {
+    log.warn(`Checking for endpoint: ${endpoint} !!`);
+    const organicTraffic = getOrganicTrafficForEndpoint(
+      endpoint,
+      rumDataMapMonthly,
+      rumDataMapBiMonthly,
+      log,
+    );
+    log.warn(`traffic for endpoint: ${endpoint} : ${organicTraffic} !!`);
+    Object.values((tags)).forEach((tagIssueDetails) => {
+      // Multiplying by 1% for missing tags, and 0.5% for other tag issues
+      // For duplicate tags, each page's traffic is multiplied by .5% so
+      // it amounts to 0.5% * number of duplicates.
+      const multiplier = tagIssueDetails.issue.includes('Missing') ? 0.01 : 0.005;
+      projectedTraffic += organicTraffic * multiplier;
+    });
+  });
+  return projectedTraffic;
+}
+
 export async function auditMetaTagsRunner(baseURL, context, site) {
   const { log, s3Client } = context;
   // Fetch site's scraped content from S3
@@ -59,14 +140,18 @@ export async function auditMetaTagsRunner(baseURL, context, site) {
   }
   seoChecks.finalChecks();
   const detectedTags = seoChecks.getDetectedTags();
-
+  const projectedTrafficLost = await calculateProjectedTraffic(context, site, detectedTags, log);
+  const cpcValue = await calculateCPCValue(context, site.getId());
+  log.info(`Calculated cpc value: ${cpcValue} for site: ${site.getId()}`);
+  const projectedTrafficValue = projectedTrafficLost * cpcValue;
   const auditResult = {
     detectedTags,
     sourceS3Folder: `${bucketName}/${prefix}`,
-    fullAuditRef: 'na',
+    fullAuditRef: '',
     finalUrl: baseURL,
+    projectedTrafficLost,
+    projectedTrafficValue,
   };
-
   return {
     auditResult,
     fullAuditRef: baseURL,

src/metatags/opportunityHandler.js

@@ -160,6 +160,7 @@ export default async function convertToOpportunity(auditUrl, auditData, context)
     throw new Error(`Failed to fetch opportunities for siteId ${auditData.siteId}: ${e.message}`);
   }
 
+  const { detectedTags, projectedTrafficLost, projectedTrafficValue } = auditData.auditResult;
   try {
     if (!metatagsOppty) {
       const opportunityData = {
@@ -180,19 +181,25 @@ export default async function convertToOpportunity(auditUrl, auditData, context)
           ],
         },
         tags: ['Traffic acquisition'],
+        data: {
+          projectedTrafficLost,
+          projectedTrafficValue,
+        },
       };
       metatagsOppty = await Opportunity.create(opportunityData);
       log.debug('Meta-tags Opportunity created');
     } else {
       metatagsOppty.setAuditId(auditData.siteId);
+      metatagsOppty.setData({
+        projectedTrafficLost,
+        projectedTrafficValue,
+      });
       await metatagsOppty.save();
     }
   } catch (e) {
     log.error(`Creating meta-tags opportunity for siteId ${auditData.siteId} failed with error: ${e.message}`, e);
     throw new Error(`Failed to create meta-tags opportunity for siteId ${auditData.siteId}: ${e.message}`);
   }
-
-  const { detectedTags } = auditData.auditResult;
   const suggestions = [];
   // Generate suggestions data to be inserted in meta-tags opportunity suggestions
   Object.keys(detectedTags).forEach((endpoint) => {

src/support/utils.js

@@ -25,6 +25,7 @@ import { ListObjectsV2Command } from '@aws-sdk/client-s3';
 import { getObjectFromKey } from '../utils/s3-utils.js';
 
 URI.preventInvalidHostname = true;
+const DEFAULT_CPC_VALUE = 1; // $1
 
 // weekly pageview threshold to eliminate urls with lack of samples
 
@@ -234,6 +235,48 @@ export const extractLinksFromHeader = (data, baseUrl, log) => {
   return links;
 };
 
+/**
+ * Fetches the organic traffic data for a site from S3 and calculate the CPC value as per
+ * https://wiki.corp.adobe.com/pages/viewpage.action?spaceKey=AEMSites&title=Success+Studio+Projected+Business+Impact+Metrics#SuccessStudioProjectedBusinessImpactMetrics-IdentifyingCPCvalueforadomain
+ * @param context
+ * @param siteId
+ * @returns {number} CPC value
+ */
+export async function calculateCPCValue(context, siteId) {
+  if (!context?.env?.S3_IMPORTER_BUCKET_NAME) {
+    throw new Error('S3 importer bucket name is required');
+  }
+  if (!context.s3Client) {
+    throw new Error('S3 client is required');
+  }
+  if (!context.log) {
+    throw new Error('Logger is required');
+  }
+  if (!siteId) {
+    throw new Error('SiteId is required');
+  }
+  const { s3Client, log } = context;
+  const bucketName = context.env.S3_IMPORTER_BUCKET_NAME;
+  const key = `metrics/${siteId}/ahrefs/organic-traffic.json`;
+  try {
+    const organicTrafficData = await getObjectFromKey(s3Client, bucketName, key, log);
+    if (!Array.isArray(organicTrafficData) || organicTrafficData.length === 0) {
+      log.warn(`Organic traffic data not available for ${siteId}. Using Default CPC value.`);
+      return DEFAULT_CPC_VALUE;
+    }
+    const lastTraffic = organicTrafficData[organicTrafficData.length - 1];
+    if (!lastTraffic.cost || !lastTraffic.value) {
+      log.warn(`Invalid organic traffic data present for ${siteId} - cost:${lastTraffic.cost} value:${lastTraffic.value}, Using Default CPC value.`);
+      return DEFAULT_CPC_VALUE;
+    }
+    // dividing by 100 for cents to dollar conversion
+    return lastTraffic.cost / lastTraffic.value / 100;
+  } catch (err) {
+    log.error(`Error fetching organic traffic data for site ${siteId}. Using Default CPC value.`, err);
+    return DEFAULT_CPC_VALUE;
+  }
+}
+
 export const getScrapedDataForSiteId = async (site, context) => {
   const { s3Client, env, log } = context;
   const siteId = site.getId();

src/utils/s3-utils.js

@@ -22,10 +22,10 @@ export async function getObjectKeysUsingPrefix(
   let continuationToken = null;
   if (!s3Client || !bucketName || !prefix) {
     log.error(
-      `Invalid input parameters: ensure s3Client, bucketName:${bucketName}, and prefix:${prefix} are provided.`,
+      `Invalid input parameters in getObjectKeysUsingPrefix: ensure s3Client, bucketName:${bucketName}, and prefix:${prefix} are provided.`,
     );
     throw new Error(
-      'Invalid input parameters: ensure s3Client, bucketName, and prefix are provided.',
+      'Invalid input parameters in getObjectKeysUsingPrefix: ensure s3Client, bucketName, and prefix are provided.',
     );
   }
   try {
@@ -41,7 +41,9 @@ export async function getObjectKeysUsingPrefix(
       // eslint-disable-next-line no-await-in-loop
       const data = await s3Client.send(new ListObjectsV2Command(params));
       data?.Contents?.forEach((obj) => {
-        objectKeys.push(obj.Key);
+        if (obj.Key?.endsWith('scrape.json')) {
+          objectKeys.push(obj.Key);
+        }
       });
       continuationToken = data?.NextContinuationToken;
     } while (continuationToken);
@@ -72,7 +74,7 @@ export async function getObjectKeysUsingPrefix(
 export async function getObjectFromKey(s3Client, bucketName, key, log) {
   if (!s3Client || !bucketName || !key) {
     log.error(
-      'Invalid input parameters: ensure s3Client, bucketName, and key are provided.',
+      'Invalid input parameters in getObjectFromKey: ensure s3Client, bucketName, and key are provided.',
     );
     return null;
   }
@@ -93,7 +95,6 @@ export async function getObjectFromKey(s3Client, bucketName, key, log) {
         return null;
       }
     }
-
     // Always return body for non-JSON content types
     return body;
   } catch (err) {