Merge pull request #737 from microlinkhq/profiling

feat: add profiling support
microlinkhq · Jan 12, 2025 · 4a3c5c9 · 4a3c5c9
2 parents 522b1bc + e88d2b1
commit 4a3c5c9
Show file tree

Hide file tree

Showing 88 changed files with 428 additions and 161 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -3,6 +3,16 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+# [5.46.0-beta.0](https://github.com/microlinkhq/metascraper/compare/v5.45.29...v5.46.0-beta.0) (2025-01-10)
+
+### Bug Fixes
+
+* load dependency ([6344788](https://github.com/microlinkhq/metascraper/commit/6344788ddbfc27a03f3ce12b2a842cd438574cc5))
+
+### Features
+
+* add profiling support ([9370e3c](https://github.com/microlinkhq/metascraper/commit/9370e3cdde056e86dcc2d189b3b22dd01a310372))
+
 ## [5.45.29](https://github.com/microlinkhq/metascraper/compare/v5.45.28...v5.45.29) (2025-01-07)
 
 ### Bug Fixes

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
@@ -42,7 +42,7 @@ A set of rules under the same namespace runs in series and only the value return
 You can associate a `test` function with your rule bundle:
 
 ```js
-rules.test = ({ url }) => getVideoInfo(url).service === 'youtube'))
+rules.test = ({ url }) => getVideoInfo(url).service === 'youtube'
 ```
 
 The `test` function will receive the same arguments as a rule. This is useful for skipping all rules that doesn't target a specific URL.
@@ -52,12 +52,31 @@ A good practice is to use a memoize function to prevent unnecessary CPU cycles f
 ```js
 const { memoizeOne } = require('@metascraper/helpers')
 
-const test = memoizeOne(url => getVideoInfo(url).service === 'youtube'))
+const test = memoizeOne(url => getVideoInfo(url).service === 'youtube')
 
 const rules = []
-rules.test ({ url }) => test(url)
+rules.test = ({ url }) => test(url)
 ```
 
+### Defining `pkgName` property
+
+Additionally you can define `pkgName` property associated with your rules:
+
+```js
+const { memoizeOne } = require('@metascraper/helpers')
+
+const rules = []
+rules.pkgName = 'metascraper-module'
+```
+
+This is using for printing debug logs, see debugging section to know how to use it.
+
+## Debugging your Rules
+
+In case you need to see what's happening under the hood, you can set `DEBUG='metascraper*'.
+
+This is useful for verifying rule precedence and detecting slow rules.
+
 ## Testing your Rules
 
 Since the order of the rules is important, testing it is also an important thing in order to be sure more popular rules are executed first over less popular rules.
@@ -74,7 +93,6 @@ const metascraper = require('metascraper')([
   require('metascraper-logo')()
 ])
 
-
 describe('metascraper-logo', () => {
   it('creates an absolute favicon url if the logo is not present', async () => {
     const html = `
@@ -92,8 +110,8 @@ describe('metascraper-logo', () => {
     </body>
     </html>
     `
-    const meta = await metascraper({ html, url }))
-    should(meta.log).be.equal("open graph value")
+    const meta = await metascraper({ html, url })
+    should(meta.log).be.equal('open graph value')
   })
 })
 ```
@@ -129,8 +147,8 @@ const metascraper = require('metascraper')([
 describe('metascraper-logo', () => {
   it('it resolves logo value', async () => {
     const html = fs.readFileSync('index.html', 'utf-8')
-    const meta = await metascraper({ html, url }))
-    should(meta.logo).be.equal("https://metascraper.js.org/static/logo.png")
+    const meta = await metascraper({ html, url })
+    should(meta.logo).be.equal('https://metascraper.js.org/static/logo.png')
   })
 })
 ```
diff --git a/lerna.json b/lerna.json
@@ -2,7 +2,7 @@
   "packages": [
     "packages/*"
   ],
-  "version": "5.45.29",
+  "version": "5.46.0-beta.2",
   "command": {
     "bootstrap": {
       "npmClientArgs": [

diff --git a/packages/metascraper-amazon/CHANGELOG.md b/packages/metascraper-amazon/CHANGELOG.md
@@ -3,6 +3,10 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+# [5.46.0-beta.0](https://github.com/microlinkhq/metascraper/compare/v5.45.29...v5.46.0-beta.0) (2025-01-10)
+
+**Note:** Version bump only for package metascraper-amazon
+
 ## [5.45.28](https://github.com/microlinkhq/metascraper/compare/v5.45.27...v5.45.28) (2025-01-01)
 
 **Note:** Version bump only for package metascraper-amazon

diff --git a/packages/metascraper-amazon/package.json b/packages/metascraper-amazon/package.json
@@ -2,7 +2,7 @@
   "name": "metascraper-amazon",
   "description": "Metascraper integration with Amazon",
   "homepage": "https://github.com/microlinkhq/metascraper/packages/metascraper-amazon",
-  "version": "5.45.28",
+  "version": "5.46.0-beta.2",
   "types": "src/index.d.ts",
   "main": "src/index.js",
   "author": {

diff --git a/packages/metascraper-amazon/src/index.js b/packages/metascraper-amazon/src/index.js
@@ -60,5 +60,7 @@ module.exports = () => {
 
   rules.test = ({ url }) => test(url)
 
+  rules.pkgName = 'metascraper-amazon'
+
   return rules
 }
diff --git a/packages/metascraper-audio/CHANGELOG.md b/packages/metascraper-audio/CHANGELOG.md
@@ -3,6 +3,10 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+# [5.46.0-beta.0](https://github.com/microlinkhq/metascraper/compare/v5.45.29...v5.46.0-beta.0) (2025-01-10)
+
+**Note:** Version bump only for package metascraper-audio
+
 ## [5.45.28](https://github.com/microlinkhq/metascraper/compare/v5.45.27...v5.45.28) (2025-01-01)
 
 **Note:** Version bump only for package metascraper-audio

diff --git a/packages/metascraper-audio/package.json b/packages/metascraper-audio/package.json
@@ -2,7 +2,7 @@
   "name": "metascraper-audio",
   "description": "Get audio property from HTML markup",
   "homepage": "https://github.com/microlinkhq/metascraper/packages/metascraper-audio",
-  "version": "5.45.28",
+  "version": "5.46.0-beta.2",
   "types": "src/index.d.ts",
   "main": "src/index.js",
   "author": {

diff --git a/packages/metascraper-audio/src/index.js b/packages/metascraper-audio/src/index.js
@@ -78,7 +78,7 @@ const _getIframe = (url, $, { src }) =>
   loadIframe(url, $.load(`<iframe src="${src}"></iframe>`))
 
 module.exports = ({ getIframe = _getIframe } = {}) => {
-  return {
+  const rules = {
     audio: audioRules.concat(
       async ({ htmlDom: $, url }) => {
         const srcs = [
@@ -110,4 +110,8 @@ module.exports = ({ getIframe = _getIframe } = {}) => {
       }
     )
   }
+
+  rules.pkgName = 'metascraper-audio'
+
+  return rules
 }
diff --git a/packages/metascraper-author/CHANGELOG.md b/packages/metascraper-author/CHANGELOG.md
@@ -3,6 +3,10 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+# [5.46.0-beta.0](https://github.com/microlinkhq/metascraper/packages/metascraper-author/compare/v5.45.29...v5.46.0-beta.0) (2025-01-10)
+
+**Note:** Version bump only for package metascraper-author
+
 ## [5.45.28](https://github.com/microlinkhq/metascraper/packages/metascraper-author/compare/v5.45.27...v5.45.28) (2025-01-01)
 
 **Note:** Version bump only for package metascraper-author

diff --git a/packages/metascraper-author/package.json b/packages/metascraper-author/package.json
@@ -2,7 +2,7 @@
   "name": "metascraper-author",
   "description": "Get author property from HTML markup",
   "homepage": "https://metascraper.js.org",
-  "version": "5.45.28",
+  "version": "5.46.0-beta.2",
   "types": "src/index.d.ts",
   "main": "src/index.js",
   "author": {

diff --git a/packages/metascraper-author/src/index.js b/packages/metascraper-author/src/index.js
@@ -24,27 +24,33 @@ const strict = rule => $ => {
   return REGEX_STRICT.test(value) && value
 }
 
-module.exports = () => ({
-  author: [
-    toAuthor($jsonld('author.name')),
-    toAuthor($jsonld('brand.name')),
-    toAuthor($ => $('meta[name="author"]').attr('content')),
-    toAuthor($ => $('meta[property="article:author"]').attr('content')),
-    toAuthor($ => $filter($, $('[itemprop*="author" i] [itemprop="name"]'))),
-    toAuthor($ => $filter($, $('[itemprop*="author" i]'))),
-    toAuthor($ => $filter($, $('[rel="author"]'))),
-    strict(toAuthor($ => $filter($, $('a[class*="author" i]')))),
-    strict(toAuthor($ => $filter($, $('[class*="author" i] a')))),
-    strict(toAuthor($ => $filter($, $('a[href*="/author/" i]')))),
-    toAuthor($ => $filter($, $('a[class*="screenname" i]'))),
-    strict(toAuthor($ => $filter($, $('[class*="author" i]')))),
-    strict(
-      toAuthor($ =>
-        $filter($, $('[class*="byline" i]'), el => {
-          const value = $filter.fn(el)
-          return !date(value) && value
-        })
+module.exports = () => {
+  const rules = {
+    author: [
+      toAuthor($jsonld('author.name')),
+      toAuthor($jsonld('brand.name')),
+      toAuthor($ => $('meta[name="author"]').attr('content')),
+      toAuthor($ => $('meta[property="article:author"]').attr('content')),
+      toAuthor($ => $filter($, $('[itemprop*="author" i] [itemprop="name"]'))),
+      toAuthor($ => $filter($, $('[itemprop*="author" i]'))),
+      toAuthor($ => $filter($, $('[rel="author"]'))),
+      strict(toAuthor($ => $filter($, $('a[class*="author" i]')))),
+      strict(toAuthor($ => $filter($, $('[class*="author" i] a')))),
+      strict(toAuthor($ => $filter($, $('a[href*="/author/" i]')))),
+      toAuthor($ => $filter($, $('a[class*="screenname" i]'))),
+      strict(toAuthor($ => $filter($, $('[class*="author" i]')))),
+      strict(
+        toAuthor($ =>
+          $filter($, $('[class*="byline" i]'), el => {
+            const value = $filter.fn(el)
+            return !date(value) && value
+          })
+        )
       )
-    )
-  ]
-})
+    ]
+  }
+
+  rules.pkgName = 'metascraper-author'
+
+  return rules
+}
diff --git a/packages/metascraper-clearbit/CHANGELOG.md b/packages/metascraper-clearbit/CHANGELOG.md
@@ -3,6 +3,10 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+# [5.46.0-beta.0](https://github.com/microlinkhq/metascraper/compare/v5.45.29...v5.46.0-beta.0) (2025-01-10)
+
+**Note:** Version bump only for package metascraper-clearbit
+
 ## [5.45.28](https://github.com/microlinkhq/metascraper/compare/v5.45.27...v5.45.28) (2025-01-01)
 
 **Note:** Version bump only for package metascraper-clearbit

diff --git a/packages/metascraper-clearbit/package.json b/packages/metascraper-clearbit/package.json
@@ -2,7 +2,7 @@
   "name": "metascraper-clearbit",
   "description": "Metascraper integration with Clearbit Logo API",
   "homepage": "https://github.com/microlinkhq/metascraper/packages/metascraper-clearbit",
-  "version": "5.45.28",
+  "version": "5.46.0-beta.2",
   "types": "src/index.d.ts",
   "main": "src/index.js",
   "author": {

diff --git a/packages/metascraper-clearbit/src/index.js b/packages/metascraper-clearbit/src/index.js
@@ -43,8 +43,12 @@ module.exports = opts => {
   const clearbit = createClearbit(opts)
   const getClearbit = composeRule(($, url) => clearbit(parseUrl(url).domain))
 
-  return {
+  const rules = {
     logo: getClearbit({ from: 'logo' }),
     publisher: getClearbit({ from: 'name', to: 'publisher' })
   }
+
+  rules.pkgName = 'metascraper-clearbit'
+
+  return rules
 }
diff --git a/packages/metascraper-date/CHANGELOG.md b/packages/metascraper-date/CHANGELOG.md
@@ -3,6 +3,10 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+# [5.46.0-beta.0](https://github.com/microlinkhq/metascraper/compare/v5.45.29...v5.46.0-beta.0) (2025-01-10)
+
+**Note:** Version bump only for package metascraper-date
+
 ## [5.45.28](https://github.com/microlinkhq/metascraper/compare/v5.45.27...v5.45.28) (2025-01-01)
 
 **Note:** Version bump only for package metascraper-date

diff --git a/packages/metascraper-date/package.json b/packages/metascraper-date/package.json
@@ -2,7 +2,7 @@
   "name": "metascraper-date",
   "description": "Get date property from HTML markup",
   "homepage": "https://github.com/microlinkhq/metascraper/packages/metascraper-date",
-  "version": "5.45.28",
+  "version": "5.46.0-beta.2",
   "types": "src/index.d.ts",
   "main": "src/index.js",
   "author": {

diff --git a/packages/metascraper-date/src/index.js b/packages/metascraper-date/src/index.js
@@ -43,17 +43,19 @@ module.exports = (
     dateModified: false
   }
 ) => {
-  const result = {
+  const rules = {
     date: dateModifiedRules().concat(datePublishedRules(), dateRules())
   }
 
   if (datePublished) {
-    result.datePublished = datePublishedRules()
+    rules.datePublished = datePublishedRules()
   }
 
   if (dateModified) {
-    result.dateModified = dateModifiedRules()
+    rules.dateModified = dateModifiedRules()
   }
 
-  return result
+  rules.pkgName = 'metascraper-date'
+
+  return rules
 }
diff --git a/packages/metascraper-description/CHANGELOG.md b/packages/metascraper-description/CHANGELOG.md
@@ -3,6 +3,10 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+# [5.46.0-beta.0](https://github.com/microlinkhq/metascraper/compare/v5.45.29...v5.46.0-beta.0) (2025-01-10)
+
+**Note:** Version bump only for package metascraper-description
+
 ## [5.45.28](https://github.com/microlinkhq/metascraper/compare/v5.45.27...v5.45.28) (2025-01-01)
 
 **Note:** Version bump only for package metascraper-description

diff --git a/packages/metascraper-description/package.json b/packages/metascraper-description/package.json
@@ -2,7 +2,7 @@
   "name": "metascraper-description",
   "description": "Get description property from HTML markup",
   "homepage": "https://github.com/microlinkhq/metascraper/packages/metascraper-description",
-  "version": "5.45.28",
+  "version": "5.46.0-beta.2",
   "types": "src/index.d.ts",
   "main": "src/index.js",
   "author": {

diff --git a/packages/metascraper-description/src/index.js b/packages/metascraper-description/src/index.js
@@ -5,7 +5,7 @@ const { $jsonld, toRule, description } = require('@metascraper/helpers')
 module.exports = opts => {
   const toDescription = toRule(description, opts)
 
-  return {
+  const rules = {
     description: [
       toDescription($ => $('meta[property="og:description"]').attr('content')),
       toDescription($ => $('meta[name="twitter:description"]').attr('content')),
@@ -18,4 +18,8 @@ module.exports = opts => {
       toDescription($jsonld('description'))
     ]
   }
+
+  rules.pkgName = 'metascraper-description'
+
+  return rules
 }
diff --git a/packages/metascraper-feed/CHANGELOG.md b/packages/metascraper-feed/CHANGELOG.md
@@ -3,6 +3,10 @@
 All notable changes to this project will be documented in this file.
 See [Conventional Commits](https://conventionalcommits.org) for commit guidelines.
 
+# [5.46.0-beta.0](https://github.com/microlinkhq/metascraper/compare/v5.45.29...v5.46.0-beta.0) (2025-01-10)
+
+**Note:** Version bump only for package metascraper-feed
+
 ## [5.45.28](https://github.com/microlinkhq/metascraper/compare/v5.45.27...v5.45.28) (2025-01-01)
 
 **Note:** Version bump only for package metascraper-feed

diff --git a/packages/metascraper-feed/package.json b/packages/metascraper-feed/package.json
@@ -2,7 +2,7 @@
   "name": "metascraper-feed",
   "description": "Get RSS/Atom feed URL from HTML markup",
   "homepage": "https://github.com/microlinkhq/metascraper/packages/metascraper-description",
-  "version": "5.45.28",
+  "version": "5.46.0-beta.2",
   "types": "src/index.d.ts",
   "main": "src/index.js",
   "author": {

diff --git a/packages/metascraper-feed/src/index.js b/packages/metascraper-feed/src/index.js
@@ -5,11 +5,15 @@ const { toRule, url } = require('@metascraper/helpers')
 const toUrl = toRule(url)
 
 module.exports = () => {
-  return {
+  const rules = {
     feed: [
       toUrl($ => $('link[type="application/rss+xml"]').attr('href')),
       toUrl($ => $('link[type="application/feed+json"]').attr('href')),
       toUrl($ => $('link[type="application/atom+xml"]').attr('href'))
     ]
   }
+
+  rules.pkgName = 'metascraper-feed'
+
+  return rules
 }