index.ts

import { getAccessToken } from "./shared/auth";
import {
  convertToSiteUrl,
  getPublishMetadata,
  requestIndexing,
  getEmojiForStatus,
  getPageIndexingStatus,
  convertToFilePath,
  checkSiteUrl,
  checkCustomUrls,
} from "./shared/gsc";
import { getSitemapPages } from "./shared/sitemap";
import { Status } from "./shared/types";
import { batch } from "./shared/utils";
import { readFileSync, existsSync, mkdirSync, writeFileSync } from "fs";
import path from "path";

const CACHE_TIMEOUT = 1000 * 60 * 60 * 24 * 14; // 14 days
const SHORT_TIMEOUT = 1000 * 60 * 60; // 1 hour

const indexableStatuses = [
  Status.SubmittedAndIndexed,
  Status.CrawledCurrentlyNotIndexed,
  Status.DiscoveredCurrentlyNotIndexed,
  Status.Forbidden,
  Status.Error,
  Status.RateLimited,
];

const quickFixStatuses = [Status.NotFound, Status.PageWithRedirect];

const shouldRecheck = (status: Status, lastCheckedAt: string) => {
  const timeSinceLastCheck = Date.now() - new Date(lastCheckedAt).getTime();

  if (indexableStatuses.includes(status)) {
    if (status === Status.SubmittedAndIndexed) {
      return timeSinceLastCheck > CACHE_TIMEOUT;
    }
    // For other indexable statuses, check more frequently
    return timeSinceLastCheck > SHORT_TIMEOUT;
  }

  if (quickFixStatuses.includes(status)) {
    // For statuses that might be quickly fixed, use a shorter timeout
    return timeSinceLastCheck > SHORT_TIMEOUT;
  }

  // For any other status, use the standard cache timeout
  return timeSinceLastCheck > CACHE_TIMEOUT;
};

export const QUOTA = {
  rpm: {
    retries: 3,
    waitingTime: 60000, // 1 minute
  },
};

export type IndexOptions = {
  client_email?: string;
  private_key?: string;
  path?: string;
  urls?: string[];
  quota?: {
    rpmRetry?: boolean; // read requests per minute: retry after waiting time
  };
};

/**
 * Indexes the specified domain or site URL.
 * @param input - The domain or site URL to index.
 * @param options - (Optional) Additional options for indexing.
 */
export const index = async (input: string = process.argv[2], options: IndexOptions = {}) => {
  if (!input) {
    console.error("❌ Please provide a domain or site URL as the first argument.");
    console.error("");
    process.exit(1);
  }

  if (!options.client_email) {
    options.client_email = process.env.GIS_CLIENT_EMAIL;
  }
  if (!options.private_key) {
    options.private_key = process.env.GIS_PRIVATE_KEY;
  }
  if (!options.path) {
    options.path = process.env.GIS_PATH;
  }
  if (!options.urls) {
    options.urls = process.env.GIS_URLS ? process.env.GIS_URLS.split(",") : undefined;
  }
  if (!options.quota) {
    options.quota = {
      rpmRetry: process.env.GIS_QUOTA_RPM_RETRY === "true",
    };
  }

  const accessToken = await getAccessToken(options.client_email, options.private_key, options.path);
  let siteUrl = convertToSiteUrl(input);
  console.log(`🔎 Processing site: ${siteUrl}`);
  const cachePath = path.join(".cache", `${convertToFilePath(siteUrl)}.json`);

  if (!accessToken) {
    console.error("❌ Failed to get access token, check your service account credentials.");
    console.error("");
    process.exit(1);
  }

  siteUrl = await checkSiteUrl(accessToken, siteUrl);

  let pages = options.urls || [];
  if (pages.length === 0) {
    console.log(`🔎 Fetching sitemaps and pages...`);
    const [sitemaps, pagesFromSitemaps] = await getSitemapPages(accessToken, siteUrl);

    if (sitemaps.length === 0) {
      console.error("❌ No sitemaps found, add them to Google Search Console and try again.");
      console.error("");
      process.exit(1);
    }

    pages = pagesFromSitemaps;

    console.log(`👉 Found ${pages.length} URLs in ${sitemaps.length} sitemap`);
  } else {
    pages = checkCustomUrls(siteUrl, pages);
    console.log(`👉 Found ${pages.length} URLs in the provided list`);
  }

  const statusPerUrl: Record<string, { status: Status; lastCheckedAt: string }> = existsSync(cachePath)
    ? JSON.parse(readFileSync(cachePath, "utf8"))
    : {};
  const pagesPerStatus: Record<Status, string[]> = {
    [Status.SubmittedAndIndexed]: [],
    [Status.DuplicateWithoutUserSelectedCanonical]: [],
    [Status.CrawledCurrentlyNotIndexed]: [],
    [Status.DiscoveredCurrentlyNotIndexed]: [],
    [Status.PageWithRedirect]: [],
    [Status.URLIsUnknownToGoogle]: [],
    [Status.RateLimited]: [],
    [Status.Forbidden]: [],
    [Status.Error]: [],
    [Status.NotFound]: [],
  };

  const indexableStatuses = [
    Status.SubmittedAndIndexed,
    Status.CrawledCurrentlyNotIndexed,
    Status.DiscoveredCurrentlyNotIndexed,
    Status.URLIsUnknownToGoogle,
    Status.Forbidden,
    Status.Error,
    Status.RateLimited,
    Status.NotFound,
  ];

  const urlsToProcess = pages.filter((url) => {
    const result = statusPerUrl[url];
    return !result || shouldRecheck(result.status, result.lastCheckedAt);
  });

  console.log(`👉 Found ${urlsToProcess.length} URLs that need processing out of ${pages.length} total URLs`);

  await batch(
    async (url) => {
      let result = statusPerUrl[url];
      if (!result || shouldRecheck(result.status, result.lastCheckedAt)) {
        const status = await getPageIndexingStatus(accessToken, siteUrl, url);
        result = { status, lastCheckedAt: new Date().toISOString() };
        statusPerUrl[url] = result;
      }

      pagesPerStatus[result.status] = pagesPerStatus[result.status] ? [...pagesPerStatus[result.status], url] : [url];
    },
    urlsToProcess,
    50,
    (batchIndex, batchCount) => {
      console.log(`📦 Batch ${batchIndex + 1} of ${batchCount} complete`);
    }
  );

  console.log(``);
  console.log(`👍 Done, here's the status of all ${pages.length} pages:`);
  mkdirSync(".cache", { recursive: true });
  writeFileSync(cachePath, JSON.stringify(statusPerUrl, null, 2));

  for (const status of Object.keys(pagesPerStatus)) {
    const pages = pagesPerStatus[status as Status];
    if (pages.length === 0) continue;
    console.log(`• ${getEmojiForStatus(status as Status)} ${status}: ${pages.length} pages`);
  }
  console.log("");

  const indexablePages = Object.entries(pagesPerStatus).flatMap(([status, pages]) =>
    indexableStatuses.includes(status as Status) ? pages : []
  );

  if (indexablePages.length === 0) {
    console.log(`✨ There are no pages that can be indexed. Everything is already indexed!`);
  } else {
    console.log(`✨ Found ${indexablePages.length} pages that can be indexed.`);
    indexablePages.forEach((url) => console.log(`• ${url}`));
  }
  console.log(``);

  for (const url of indexablePages) {
    console.log(`📄 Processing url: ${url}`);
    const status = await getPublishMetadata(accessToken, url, {
      retriesOnRateLimit: options.quota.rpmRetry ? QUOTA.rpm.retries : 0,
    });
    if (status === 404) {
      await requestIndexing(accessToken, url);
      console.log("🚀 Indexing requested successfully. It may take a few days for Google to process it.");
    } else if (status < 400) {
      console.log(`🕛 Indexing already requested previously. It may take a few days for Google to process it.`);
    }
    console.log(``);
  }

  console.log(`👍 All done!`);
  console.log(`💖 Brought to you by https://seogets.com - SEO Analytics.`);
  console.log(``);
};

export * from "./shared";