Skip to content

Commit

Permalink
trace-redirectを削除
Browse files Browse the repository at this point in the history
Co-Authored-By: あわわわとーにゅ <[email protected]>
  • Loading branch information
kakkokari-gtyih and u1-liquid committed Nov 10, 2024
1 parent b628687 commit 533003c
Show file tree
Hide file tree
Showing 4 changed files with 32 additions and 23 deletions.
3 changes: 1 addition & 2 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,6 @@
"html-entities": "2.5.2",
"iconv-lite": "0.6.3",
"jschardet": "3.1.4",
"private-ip": "3.0.2",
"trace-redirect": "1.0.6"
"private-ip": "3.0.2"
}
}
8 changes: 0 additions & 8 deletions pnpm-lock.yaml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

36 changes: 27 additions & 9 deletions src/index.ts
Original file line number Diff line number Diff line change
Expand Up @@ -4,14 +4,13 @@
*/

import { URL } from 'node:url';
import tracer from 'trace-redirect';
import * as Got from 'got';
import { got, type Agents as GotAgents } from 'got';
import type { FastifyInstance } from 'fastify';
import { SummalyResult } from '@/summary.js';
import { SummalyPlugin } from '@/iplugin.js';
export * from '@/iplugin.js';
import general, { GeneralScrapingOptions } from '@/general.js';
import { setAgent } from '@/utils/got.js';
import { DEFAULT_OPERATION_TIMEOUT, DEFAULT_RESPONSE_TIMEOUT, agent, setAgent } from '@/utils/got.js';
import { plugins as builtinPlugins } from '@/plugins/index.js';

export type SummalyOptions = {
Expand All @@ -33,7 +32,7 @@ export type SummalyOptions = {
/**
* Custom HTTP agent
*/
agent?: Got.Agents;
agent?: GotAgents;

/**
* User-Agent for the request
Expand Down Expand Up @@ -85,7 +84,26 @@ export const summaly = async (url: string, options?: SummalyOptions): Promise<Su
if (opts.followRedirects) {
// .catch(() => url)にすればいいけど、jestにtrace-redirectを食わせるのが面倒なのでtry-catch
try {
actualUrl = await tracer(url);
const timeout = opts.responseTimeout ?? DEFAULT_RESPONSE_TIMEOUT;
const operationTimeout = opts.operationTimeout ?? DEFAULT_OPERATION_TIMEOUT;
actualUrl = await got
.head(url, {
timeout: {
lookup: timeout,
connect: timeout,
secureConnect: timeout,
socket: timeout, // read timeout
response: timeout,
send: timeout,
request: operationTimeout, // whole operation timeout
},
agent,
http2: false,
retry: {
limit: 0,
},
})
.then(res => res.url);
} catch (e) {
actualUrl = url;
}
Expand Down Expand Up @@ -121,10 +139,10 @@ export const summaly = async (url: string, options?: SummalyOptions): Promise<Su
// eslint-disable-next-line import/no-default-export
export default function (fastify: FastifyInstance, options: SummalyOptions, done: (err?: Error) => void) {
fastify.get<{
Querystring: {
url?: string;
lang?: string;
};
Querystring: {
url?: string;
lang?: string;
};
}>('/', async (req, reply) => {
const url = req.query.url as string;
// eslint-disable-next-line @typescript-eslint/no-unnecessary-condition
Expand Down
8 changes: 4 additions & 4 deletions src/utils/got.ts
Original file line number Diff line number Diff line change
Expand Up @@ -31,10 +31,10 @@ export type GotOptions = {

const repo = JSON.parse(readFileSync(`${_dirname}/../../package.json`, 'utf8'));

const DEFAULT_RESPONSE_TIMEOUT = 20 * 1000;
const DEFAULT_OPERATION_TIMEOUT = 60 * 1000;
const DEFAULT_MAX_RESPONSE_SIZE = 10 * 1024 * 1024;
const DEFAULT_BOT_UA = `SummalyBot/${repo.version}`;
export const DEFAULT_RESPONSE_TIMEOUT = 20 * 1000;
export const DEFAULT_OPERATION_TIMEOUT = 60 * 1000;
export const DEFAULT_MAX_RESPONSE_SIZE = 10 * 1024 * 1024;
export const DEFAULT_BOT_UA = `SummalyBot/${repo.version}`;

export async function scpaping(
url: string,
Expand Down

0 comments on commit 533003c

Please sign in to comment.