From 81e0284fba8c3cc55afa22fdac328ca831470679 Mon Sep 17 00:00:00 2001
From: Teppo Kurki <teppo.kurki@iki.fi>
Date: Sat, 8 Feb 2025 09:50:17 +0200
Subject: [PATCH] fix: handle npm search api window

Npm search api windows the results to max 250 results, so keep
fetching repeatedly until we have retrieved all search results.

Fixes #1881.
---
 src/modules.ts | 46 +++++++++++++++++++++++++++++++++++++++-------
 1 file changed, 39 insertions(+), 7 deletions(-)

diff --git a/src/modules.ts b/src/modules.ts
index 21a650c68..53c1aae6c 100644
--- a/src/modules.ts
+++ b/src/modules.ts
@@ -23,7 +23,9 @@ import path from 'path'
 import semver, { SemVer } from 'semver'
 import { Config } from './config/config'
 import { createDebug } from './debug'
+import { nextTick } from 'process'
 const debug = createDebug('signalk:modules')
+const npmDebug = createDebug('signalk:modules:npm')
 
 interface ModuleData {
   module: string
@@ -219,13 +221,12 @@ function findModulesWithKeyword(keyword: string): Promise<NpmModuleData[]> {
       return
     }
 
-    fetch(
-      'http://registry.npmjs.org/-/v1/search?size=250&text=keywords:' + keyword
-    )
-      .then((r) => r.json())
-      .then((parsed) => {
-        const data = parsed.results || parsed.objects || []
-        const result = data.reduce(
+    searchByKeyword(keyword)
+      .then((moduleData) => {
+        npmDebug(
+          `npm search returned ${moduleData.length} modules with keyword ${keyword}`
+        )
+        const result = moduleData.reduce(
           (
             acc: { [packageName: string]: NpmModuleData },
             module: NpmModuleData
@@ -254,6 +255,37 @@ function findModulesWithKeyword(keyword: string): Promise<NpmModuleData[]> {
   })
 }
 
+function searchByKeyword(keyword: string): Promise<NpmModuleData[]> {
+  return new Promise((resolve, reject) => {
+    let fetchedCount = 0
+    let toFetchCount = 1
+    let moduleData: NpmModuleData[] = []
+    const npmFetch = () => {
+      npmDebug(
+        `searching ${keyword} from ${fetchedCount + 1} of ${toFetchCount}`
+      )
+      fetch(
+        `http://registry.npmjs.org/-/v1/search?size=250&from=${
+          fetchedCount > 0 ? fetchedCount : 0
+        }&text=keywords:${keyword}`
+      )
+        .then((r) => r.json())
+        .then((parsed) => {
+          moduleData = moduleData.concat(parsed.objects)
+          fetchedCount += parsed.objects.length
+          toFetchCount = parsed.total
+          if (fetchedCount < toFetchCount) {
+            nextTick(() => npmFetch())
+          } else {
+            resolve(moduleData)
+          }
+        })
+        .catch(reject)
+    }
+    npmFetch()
+  })
+}
+
 function doFetchDistTags() {
   return fetch('http://registry.npmjs.org/-/package/signalk-server/dist-tags')
 }