Add debug and fix rate limit

.github/workflows/test_codeql.yml

@@ -65,3 +65,5 @@ jobs:
     # Run test
     - name: Test
       run: npm run test
+      env:
+        DEBUG: "coreutils:*"

package.json

@@ -14,7 +14,10 @@
     "url": "git+https://github.com/Sirherobrine23/coreUtils.git"
   },
   "keywords": [
-    "http", "got", "jsdom", "proprieties"
+    "http",
+    "got",
+    "jsdom",
+    "proprieties"
   ],
   "bugs": {
     "url": "https://github.com/Sirherobrine23/coreUtils/issues"
@@ -33,6 +36,7 @@
   },
   "devDependencies": {
     "@types/adm-zip": "^0.5.0",
+    "@types/debug": "^4.1.7",
     "@types/jsdom": "^20.0.1",
     "@types/mocha": "^10.0.0",
     "@types/node": "^18.11.9",
@@ -44,6 +48,7 @@
   "dependencies": {
     "@actions/github": "^5.1.1",
     "adm-zip": "^0.5.9",
+    "debug": "^4.3.4",
     "got": "^12.5.3",
     "jsdom": "^20.0.3",
     "tar": "^6.1.12"

src/request/github.test.ts

@@ -0,0 +1,7 @@
+import * as github from "./github";
+
+describe("Github", function(){
+  this.timeout(Infinity);
+  it("Releases", async () => github.GithubRelease("Sirherobrine23", "coreUtils"));
+  it("Tree", async () => github.githubTree("Sirherobrine23", "coreUtils"));
+});

src/request/github.ts

@@ -3,6 +3,8 @@ import { getOctokit } from "@actions/github";
 import { getJSON } from "./simples";
 import stream from "node:stream";
 import fs from "node:fs/promises";
+import debug from "debug";
+const githubRateDebug = debug("coreutils:github:ratelimit");
 export type githubRelease = Awaited<ReturnType<ReturnType<typeof getOctokit>["rest"]["repos"]["listReleases"]>>["data"][number];
 
 export type rateLimit = {
@@ -46,12 +48,15 @@ export type rateLimit = {
   }
 }
 
+export const github_secret = process.env.GITHUB_SECRET||process.env.GITHUB_TOKEN;
 export async function getReateLimit(token?: string) {
+  token = token||github_secret;
   const rate = await getJSON<rateLimit>({
     url: "https://api.github.com/rate_limit",
-    headers: token?{Authorization: `Bearer ${token}`}:{}
+    headers: token?{Authorization: `token ${token}`}:{}
   });
-  if (rate.rate.limit >= rate.rate.used) throw new Error("Github API max requests");
+  githubRateDebug("Limit data: %O", rate);
+  if (rate.rate.remaining === 0) throw new Error("Github API max requests");
   return rate;
 }
 
@@ -66,15 +71,25 @@ export async function GithubRelease(username: string, repo?: string, releaseTag?
   }
   await getReateLimit();
   if (releaseTag) {
-    if (releaseTag.toLowerCase() === "latest") return getJSON<githubRelease>(`https://api.github.com/repos/${fullRepo}/releases/latest`);
-    return getJSON<githubRelease>(`https://api.github.com/repos/${fullRepo}/releases/tags/${releaseTag}`);
+    if (releaseTag.toLowerCase() === "latest") return getJSON<githubRelease>({
+      url: `https://api.github.com/repos/${fullRepo}/releases/latest`,
+      headers: github_secret?{Authorization: `token ${github_secret}`}:{}
+    });
+    return getJSON<githubRelease>({
+      url: `https://api.github.com/repos/${fullRepo}/releases/tags/${releaseTag}`,
+      headers: github_secret?{Authorization: `token ${github_secret}`}:{}
+    });
   }
   const allReleases: githubRelease[] = [];
   let pageIndex = 0
   while (true) {
-    const data = await getJSON<githubRelease[]>(`https://api.github.com/repos/${fullRepo}/releases?per_page=100&page=${pageIndex++}`);
+    const data = await getJSON<githubRelease[]>({
+      url: `https://api.github.com/repos/${fullRepo}/releases?per_page=100&page=${pageIndex++}`,
+      headers: github_secret?{Authorization: `token ${github_secret}`}:{}
+    });
     if (data.length === 0) break;
     allReleases.push(...data);
+    if (data.length < 100) break;
   }
   return allReleases;
 }
@@ -107,7 +122,7 @@ export type releaseOptionsUpload = {
 export async function createRelease(releaseOptions: releaseOptions) {
   if (!releaseOptions) throw new Error("Required release options");
   releaseOptions = {
-    secret: process.env.GITHUB_SECRET||process.env.GITHUB_TOKEN,
+    secret: github_secret,
     prerelease: false,
     createReleaseIfNotExists: true,
     name: releaseOptions?.tagName,
@@ -201,5 +216,8 @@ export async function githubTree(username: string, repo: string, tree: string =
   if (!validate.test(username)) throw new Error("Invalid username");
   if (!validate.test(repo)) throw new Error("Invalid repository name");
   await getReateLimit();
-  return getJSON<githubTree>(`https://api.github.com/repos/${username}/${repo}/git/trees/${tree}?recursive=true`);
+  return getJSON<githubTree>({
+    url: `https://api.github.com/repos/${username}/${repo}/git/trees/${tree}?recursive=true`,
+    headers: github_secret?{Authorization: `token ${github_secret}`}:{}
+  });
 }

src/request/simples.ts

@@ -1,7 +1,10 @@
-import type { Method, Request } from "got";
+import type { Method, Request, RequestError } from "got";
 import { JSDOM } from "jsdom";
 import * as fs from "node:fs";
 import * as stream from "node:stream";
+import debug from "debug";
+const pipeDebug = debug("coreutils:pipe");
+const bufferDebug = debug("coreutils:buffer");
 
 async function getImport<T>(moduleName: string): Promise<T> {return eval(`import("${moduleName}")`);}
 let got: Awaited<ReturnType<typeof gotCjs>>;
@@ -45,15 +48,18 @@ export async function pipeFetch(options: requestOptions & {stream?: fs.WriteStre
     else if (options.body instanceof stream.Writable||options.body instanceof fs.WriteStream) request["body"] = options.body;
     else request["json"] = options.body;
   }
+  pipeDebug("Fetching data with options: %O", {...options, stream: null});
   const gotStream = (await gotCjs()).stream(urlRequest, {
     isStream: true,
     headers: options.headers||{},
     method,
     ...request
   });
 
-  if (!options.stream) return gotStream;
-  else {
+  if (!options.stream) {
+    pipeDebug("without finishing escort");
+    return gotStream;
+  } else {
     await new Promise<void>((done, reject) => {
       gotStream.pipe(options.stream);
       options.stream.on("error", reject);
@@ -63,6 +69,7 @@ export async function pipeFetch(options: requestOptions & {stream?: fs.WriteStre
         return done();
       });
     });
+    pipeDebug("pipe end");
   }
 }
 
@@ -79,12 +86,36 @@ export async function bufferFetch(options: string|requestOptions) {
     else request["json"] = options.body;
   }
 
+  bufferDebug("Fetching data with options: %O", options);
   return (await gotCjs())(urlRequest, {
     responseType: "buffer",
     headers: options.headers||{},
     method,
     ...request
-  }).then(res => ({headers: res.headers, data: Buffer.from(res.body), response: res}));
+  }).then(res => {
+    bufferDebug("end request data");
+    return {
+      headers: res.headers,
+      data: Buffer.from(res.body),
+      response: res
+    };
+  }).catch((err: RequestError) => {
+    const newErrorObject = {
+      code: err?.code,
+      textError: err?.message,
+      data: undefined
+    };
+    if (err?.response?.body) {
+      if (Buffer.isBuffer(err.response.body)) newErrorObject.data = err.response.body.toString("utf8");
+    }
+    if (typeof newErrorObject.data === "string") {
+      try {
+        newErrorObject.data = JSON.parse(newErrorObject.data);
+      } catch {}
+    }
+    bufferDebug("catch error to %s, error object: %O", urlRequest, newErrorObject);
+    throw newErrorObject;
+  });
 }
 
 export async function getJSON<JSONReturn = any>(request: string|requestOptions) {