feat(check-html-links): handle as internal if starts with a provided absolute base url

daKmoR · daKmoR · commit 68e05f4d4ad8 · 2022-08-24T11:33:10.000+02:00
diff --git a/.changeset/fair-falcons-shave.md b/.changeset/fair-falcons-shave.md
@@ -0,0 +1,11 @@
+---
+'@rocket/cli': patch
+---
+
+`rocket lint` can now validate external links.
+
+Example:
+
+```
+rocket lint --validate-externals
+```
diff --git a/.changeset/great-bobcats-destroy.md b/.changeset/great-bobcats-destroy.md
@@ -1,5 +1,15 @@
 ---
-'check-html-links': minor
+'check-html-links': patch
 ---
 
-add external links validation using the flag `--validate-externals`.
+Add external links validation via the flag `--validate-externals`.
+
+You can/should provide an optional `--absolute-base-url` to handle urls starting with it as internal.
+
+```bash
+# check external urls
+npx check-html-links _site --validate-externals
+
+# check external urls but treat links like https://rocket.modern-web.dev/about/ as internal
+npx check-html-links _site --validate-externals --absolute-base-url https://rocket.modern-web.dev
+```
diff --git a/packages/check-html-links/src/CheckHtmlLinksCli.js b/packages/check-html-links/src/CheckHtmlLinksCli.js
@@ -6,7 +6,7 @@ import path from 'path';
 import chalk from 'chalk';
 
 import commandLineArgs from 'command-line-args';
-import { validateFiles } from './validateFolder.js';
+import { prepareFiles, validateFiles } from './validateFolder.js';
 import { formatErrors } from './formatErrors.js';
 import { listFiles } from './listFiles.js';
 
@@ -20,6 +20,7 @@ export class CheckHtmlLinksCli {
       { name: 'root-dir', type: String, defaultOption: true },
       { name: 'continue-on-error', type: Boolean },
       { name: 'validate-externals', type: Boolean },
+      { name: 'absolute-base-url', type: String },
     ];
     const options = commandLineArgs(mainDefinitions, {
       stopAtFirstUnknown: true,
@@ -31,6 +32,7 @@ export class CheckHtmlLinksCli {
       rootDir: options['root-dir'],
       ignoreLinkPatterns: options['ignore-link-pattern'],
       validateExternals: options['validate-externals'],
+      absoluteBaseUrl: options['absolute-base-url'],
     };
   }
 
@@ -45,26 +47,48 @@ export class CheckHtmlLinksCli {
   }
 
   async run() {
-    const { ignoreLinkPatterns, rootDir: userRootDir, validateExternals } = this.options;
+    const {
+      ignoreLinkPatterns,
+      rootDir: userRootDir,
+      validateExternals,
+      absoluteBaseUrl,
+    } = this.options;
     const rootDir = userRootDir ? path.resolve(userRootDir) : process.cwd();
     const performanceStart = process.hrtime();
 
-    console.log('👀 Checking if all internal links work...');
     const files = await listFiles('**/*.html', rootDir);
 
+    console.log('Check HTML Links');
+
     const filesOutput =
       files.length == 0
-        ? '🧐 No files to check. Did you select the correct folder?'
-        : `🔥 Found a total of ${chalk.green.bold(files.length)} files to check!`;
+        ? '  🧐 No files to check. Did you select the correct folder?'
+        : `  📖 Found ${chalk.green.bold(files.length)} files containing`;
     console.log(filesOutput);
 
-    const { errors, numberLinks } = await validateFiles(files, rootDir, {
-      ignoreLinkPatterns,
+    const { numberLinks, checkLocalFiles, checkExternalLinks } = await prepareFiles(
+      files,
+      rootDir,
+      {
+        ignoreLinkPatterns,
+        validateExternals,
+        absoluteBaseUrl,
+      },
+    );
+
+    console.log(`    🔗 ${chalk.green.bold(numberLinks)} internal links`);
+    if (validateExternals) {
+      console.log(`    🌐 ${chalk.green.bold(checkExternalLinks.length)} external links`);
+    }
+
+    console.log('  👀 Checking...');
+
+    const { errors } = await validateFiles({
+      checkLocalFiles,
       validateExternals,
+      checkExternalLinks,
     });
 
-    console.log(`🔗 Found a total of ${chalk.green.bold(numberLinks)} links to validate!\n`);
-
     const performance = process.hrtime(performanceStart);
     /** @type {string[]} */
     let output = [];
@@ -75,15 +99,15 @@ export class CheckHtmlLinksCli {
         referenceCount += error.usage.length;
       }
       output = [
-        `❌ Found ${chalk.red.bold(
+        `  ❌ Found ${chalk.red.bold(
           errors.length.toString(),
         )} missing reference targets (used by ${referenceCount} links) while checking ${
           files.length
         } files:`,
         ...formatErrors(errors)
           .split('\n')
           .map(line => `  ${line}`),
-        `Checking links duration: ${performance[0]}s ${performance[1] / 1000000}ms`,
+        `  🕑 Checking links duration: ${performance[0]}s ${performance[1] / 1000000}ms`,
       ];
       message = output.join('\n');
       if (this.options.printOnError === true) {
@@ -94,7 +118,7 @@ export class CheckHtmlLinksCli {
       }
     } else {
       console.log(
-        `✅ All internal links are valid. (executed in ${performance[0]}s ${
+        `  ✅ All tested links are valid. (executed in ${performance[0]}s ${
           performance[1] / 1000000
         }ms)`,
       );
diff --git a/packages/check-html-links/src/checkLinks.js b/packages/check-html-links/src/checkLinks.js
@@ -22,10 +22,14 @@ const memorizeCheckup = (url, result) => {
  * @param {string} method
  * @returns
  */
-const fetchWrap = async (url, method = 'GET') =>
-  fetch(url, { method })
-    .then(response => response.ok)
-    .catch(() => false);
+const fetchWrap = async (url, method = 'GET') => {
+  return Promise.race([
+    fetch(url, { method })
+      .then(response => response.ok)
+      .catch(() => false),
+    new Promise(resolve => setTimeout(resolve, 10000, false)),
+  ]);
+};
 
 /**
  *
diff --git a/packages/check-html-links/src/formatErrors.js b/packages/check-html-links/src/formatErrors.js
@@ -15,7 +15,7 @@ export function formatErrors(errors, relativeFrom = process.cwd()) {
     const filePath = path.relative(relativeFrom, error.filePath);
     if (error.onlyAnchorMissing === true) {
       output.push(
-        `${number}. missing ${chalk.red.bold(
+        `  ${number}. missing ${chalk.red.bold(
           `id="${error.usage[0].anchor}"`,
         )} in ${chalk.cyanBright(filePath)}`,
       );
@@ -24,7 +24,7 @@ export function formatErrors(errors, relativeFrom = process.cwd()) {
       const title =
         firstAttribute === 'src' || firstAttribute === 'srcset' ? 'file' : 'reference target';
 
-      output.push(`${number}. missing ${title} ${chalk.red.bold(filePath)}`);
+      output.push(`  ${number}. missing ${title} ${chalk.red.bold(filePath)}`);
     }
     const usageLength = error.usage.length;
 
@@ -34,11 +34,11 @@ export function formatErrors(errors, relativeFrom = process.cwd()) {
       const clickAbleLink = chalk.cyanBright(`${usagePath}:${usage.line + 1}:${usage.character}`);
       const attributeStart = chalk.gray(`${usage.attribute}="`);
       const attributeEnd = chalk.gray('"');
-      output.push(`  from ${clickAbleLink} via ${attributeStart}${usage.value}${attributeEnd}`);
+      output.push(`    from ${clickAbleLink} via ${attributeStart}${usage.value}${attributeEnd}`);
     }
     if (usageLength > 3) {
       const more = chalk.red((usageLength - 3).toString());
-      output.push(`  ... ${more} more references to this target`);
+      output.push(`    ... ${more} more references to this target`);
     }
     output.push('');
   }
diff --git a/packages/check-html-links/src/validateFolder.js b/packages/check-html-links/src/validateFolder.js
@@ -223,11 +223,16 @@ function isNonHttpSchema(url) {
  * @param {object} options
  * @param {string} options.htmlFilePath
  * @param {string} options.rootDir
+ * @param {string} options.absoluteBaseUrl
  * @param {function(string): boolean} options.ignoreUsage
  */
-async function resolveLinks(links, { htmlFilePath, rootDir, ignoreUsage }) {
+async function resolveLinks(links, { htmlFilePath, rootDir, ignoreUsage, absoluteBaseUrl }) {
   for (const hrefObj of links) {
-    const { value, anchor } = getValueAndAnchor(hrefObj.value);
+    const { value: rawValue, anchor } = getValueAndAnchor(hrefObj.value);
+
+    const value = rawValue.startsWith(absoluteBaseUrl)
+      ? rawValue.substring(absoluteBaseUrl.length)
+      : rawValue;
 
     const usageObj = {
       attribute: hrefObj.attribute,
@@ -252,8 +257,6 @@ async function resolveLinks(links, { htmlFilePath, rootDir, ignoreUsage }) {
     } else if (valueFile === '' && anchor !== '') {
       addLocalFile(htmlFilePath, anchor, usageObj);
     } else if (value.startsWith('//') || value.startsWith('http')) {
-      // TODO: handle external urls
-      // external url - we do not handle that (yet)
       addExternalLink(htmlFilePath, usageObj);
     } else if (value.startsWith('/')) {
       const filePath = path.join(rootDir, valueFile);
@@ -328,7 +331,7 @@ async function validateExternalLinks(checkExternalLinks) {
  * @param {string} rootDir
  * @param {Options} opts?
  */
-export async function validateFiles(files, rootDir, opts) {
+export async function prepareFiles(files, rootDir, opts) {
   await parserReferences.prepareWasm(saxWasmBuffer);
   await parserIds.prepareWasm(saxWasmBuffer);
 
@@ -350,14 +353,27 @@ export async function validateFiles(files, rootDir, opts) {
   for (const htmlFilePath of files) {
     const { links } = await extractReferences(htmlFilePath);
     numberLinks += links.length;
-    await resolveLinks(links, { htmlFilePath, rootDir, ignoreUsage });
+    await resolveLinks(links, {
+      htmlFilePath,
+      rootDir,
+      ignoreUsage,
+      absoluteBaseUrl: opts?.absoluteBaseUrl,
+    });
   }
+  return { checkLocalFiles, checkExternalLinks, numberLinks };
+}
+
+/**
+ * @param {*} param0
+ * @returns
+ */
+export async function validateFiles({ checkLocalFiles, validateExternals, checkExternalLinks }) {
   await validateLocalFiles(checkLocalFiles);
-  if (opts?.validateExternals) {
+  if (validateExternals) {
     await validateExternalLinks(checkExternalLinks);
   }
 
-  return { errors: errors, numberLinks: numberLinks };
+  return { errors };
 }
 
 /**
@@ -367,6 +383,14 @@ export async function validateFiles(files, rootDir, opts) {
 export async function validateFolder(inRootDir, opts) {
   const rootDir = path.resolve(inRootDir);
   const files = await listFiles('**/*.html', rootDir);
-  const { errors } = await validateFiles(files, rootDir, opts);
+
+  const { checkLocalFiles, checkExternalLinks } = await prepareFiles(files, rootDir, opts);
+
+  const { errors } = await validateFiles({
+    checkLocalFiles,
+    validateExternals: opts?.validateExternals,
+    checkExternalLinks,
+  });
+
   return errors;
 }
diff --git a/packages/check-html-links/test-node/fixtures/internal-own-absolute-base-path/about.html b/packages/check-html-links/test-node/fixtures/internal-own-absolute-base-path/about.html
@@ -0,0 +1 @@
+About
diff --git a/packages/check-html-links/test-node/fixtures/internal-own-absolute-base-path/index.html b/packages/check-html-links/test-node/fixtures/internal-own-absolute-base-path/index.html
@@ -0,0 +1,2 @@
+<a href="about.html">About</a>
+<a href="http://localhost/about.html">About Absolute</a>
diff --git a/packages/check-html-links/test-node/formatErrors.test.js b/packages/check-html-links/test-node/formatErrors.test.js
@@ -20,19 +20,19 @@ describe('formatErrors', () => {
     const result = await executeAndFormat('fixtures/test-case');
     expect(result.trim().split('\n')).to.deep.equal([
       '1. missing id="my-teams" in fixtures/test-case/price/index.html',
-      '  from fixtures/test-case/history/index.html:1:9 via href="/price/#my-teams"',
+      '    from fixtures/test-case/history/index.html:1:9 via href="/price/#my-teams"',
       '',
-      '2. missing file fixtures/test-case/about/images/team.png',
-      '  from fixtures/test-case/about/index.html:3:10 via src="./images/team.png"',
+      '  2. missing file fixtures/test-case/about/images/team.png',
+      '    from fixtures/test-case/about/index.html:3:10 via src="./images/team.png"',
       '',
-      '3. missing reference target fixtures/test-case/aboot',
-      '  from fixtures/test-case/about/index.html:6:11 via href="/aboot"',
-      '  from fixtures/test-case/history/index.html:4:11 via href="/aboot"',
-      '  from fixtures/test-case/index.html:4:11 via href="/aboot"',
-      '  ... 2 more references to this target',
+      '  3. missing reference target fixtures/test-case/aboot',
+      '    from fixtures/test-case/about/index.html:6:11 via href="/aboot"',
+      '    from fixtures/test-case/history/index.html:4:11 via href="/aboot"',
+      '    from fixtures/test-case/index.html:4:11 via href="/aboot"',
+      '    ... 2 more references to this target',
       '',
-      '4. missing reference target fixtures/test-case/prce',
-      '  from fixtures/test-case/index.html:1:9 via href="./prce"',
+      '  4. missing reference target fixtures/test-case/prce',
+      '    from fixtures/test-case/index.html:1:9 via href="./prce"',
     ]);
   });
 });
diff --git a/packages/check-html-links/test-node/validateFolder.test.js b/packages/check-html-links/test-node/validateFolder.test.js
@@ -104,6 +104,36 @@ describe('validateFolder', () => {
     ]);
   });
 
+  it('validates links with own absolute base url as internal', async () => {
+    const { errors, cleanup } = await execute('fixtures/internal-own-absolute-base-path', {
+      validateExternals: true,
+      absoluteBaseUrl: 'http://localhost',
+    });
+    expect(cleanup(errors)).to.deep.equal([]);
+  });
+
+  it('validates all full urls if there is no absoluteBaseUrl provided', async () => {
+    const { errors, cleanup } = await execute('fixtures/internal-own-absolute-base-path', {
+      validateExternals: true,
+    });
+    expect(cleanup(errors)).to.deep.equal([
+      {
+        filePath: 'fixtures/internal-own-absolute-base-path/index.html',
+        onlyAnchorMissing: false,
+        usage: [
+          {
+            anchor: '',
+            attribute: 'href',
+            character: 9,
+            file: 'fixtures/internal-own-absolute-base-path/index.html',
+            line: 1,
+            value: 'http://localhost/about.html',
+          },
+        ],
+      },
+    ]);
+  });
+
   it('groups multiple usage of the same missing file', async () => {
     const { errors, cleanup } = await execute('fixtures/internal-links-to-same-file');
     expect(cleanup(errors)).to.deep.equal([
diff --git a/packages/check-html-links/types/main.d.ts b/packages/check-html-links/types/main.d.ts
@@ -29,10 +29,12 @@ export interface Error {
 export interface Options {
   ignoreLinkPatterns: string[] | null;
   validateExternals: boolean;
+  absoluteBaseUrl: string;
 }
 
 export interface CheckHtmlLinksCliOptions extends Options {
   printOnError: boolean;
   rootDir: string;
   continueOnError: boolean;
+  absoluteBaseUrl: string;
 }
diff --git a/packages/cli/src/RocketLint.js b/packages/cli/src/RocketLint.js
@@ -24,6 +24,7 @@ export class RocketLint {
       .command('lint')
       .option('-i, --input-dir <path>', 'path to where to search for source files')
       .option('-b, --build-html', 'do a quick html only build and then check links')
+      .option('-e, --validate-externals', 'validate external links')
       .action(async options => {
         const { cliOptions, ...lintOptions } = options;
         cli.setOptions({
@@ -77,6 +78,7 @@ export class RocketLint {
       rootDir: folderToCheck,
       printOnError: true,
       continueOnError: false,
+      absoluteBaseUrl: this.cli.options.absoluteBaseUrl,
       ...userCheckHtmlLinksOptions,
     });
 
diff --git a/site/pages/10--docs/30--guides/50--go-live.rocket.md b/site/pages/10--docs/30--guides/50--go-live.rocket.md
@@ -68,6 +68,18 @@ rocket lint --build-html
 
 Note: We can do this as 2-4 generally does not impact links/references (as long as the optimizations scripts do not have related bugs)
 
+### Check external links
+
+`rocket lint` can also check external links.
+This will do a HEAD/GET request to every external link which means if all is green that the website only links to still active websites.
+It however also means that it will do a lot of network requests which will take a while.
+
+Use with care.
+
+```bash
+rocket lint --validate-externals
+```
+
 ## Add a Not Found Page
 
 When a user enters a URL that does not exist, a "famous" 404 Page Not Found error occurs.
diff --git a/site/pages/30--tools/40--check-html-links/10--overview.rocket.md b/site/pages/30--tools/40--check-html-links/10--overview.rocket.md
diff --git a/site/pages/pageTreeData.rocketGenerated.json b/site/pages/pageTreeData.rocketGenerated.json

Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	`+<a href="about.html">About</a>`
	`2`	`+<a href="http://localhost/about.html">About Absolute</a>`
Original file line number	Diff line number	Diff line change
`@@ -29,10 +29,12 @@ export interface Error {`
`29`	`29`	`export interface Options {`
`30`	`30`	`ignoreLinkPatterns: string[] \| null;`
`31`	`31`	`validateExternals: boolean;`
	`32`	`+ absoluteBaseUrl: string;`
`32`	`33`	`}`
`33`	`34`
`34`	`35`	`export interface CheckHtmlLinksCliOptions extends Options {`
`35`	`36`	`printOnError: boolean;`
`36`	`37`	`rootDir: string;`
`37`	`38`	`continueOnError: boolean;`
	`39`	`+ absoluteBaseUrl: string;`
`38`	`40`	`}`