From 38ec0d4a65b5e0b31f18b724ed5bd561801e8add Mon Sep 17 00:00:00 2001 From: Tony Date: Wed, 15 Jan 2025 10:21:38 +0800 Subject: [PATCH] fix(route): juejin post (#18127) * fix(route): juejin * refactor: split metadata and article parsing * fix: clean up * fix: more cleanup --- lib/routes/juejin/books.ts | 12 +- lib/routes/juejin/category.ts | 38 +-- lib/routes/juejin/collection.ts | 20 +- .../juejin/{favorites.ts => collections.ts} | 42 ++-- lib/routes/juejin/column.ts | 31 +-- lib/routes/juejin/dynamic.ts | 16 +- lib/routes/juejin/pins.ts | 11 +- lib/routes/juejin/posts.ts | 31 ++- lib/routes/juejin/tag.ts | 32 +-- lib/routes/juejin/trending.ts | 26 +- lib/routes/juejin/types.ts | 232 ++++++++++++++++++ lib/routes/juejin/utils.ts | 189 ++++++++++---- 12 files changed, 490 insertions(+), 190 deletions(-) rename lib/routes/juejin/{favorites.ts => collections.ts} (50%) create mode 100644 lib/routes/juejin/types.ts diff --git a/lib/routes/juejin/books.ts b/lib/routes/juejin/books.ts index a1c2caae20fa56..8bd1dccbe5ea88 100644 --- a/lib/routes/juejin/books.ts +++ b/lib/routes/juejin/books.ts @@ -1,5 +1,5 @@ import { Route } from '@/types'; -import got from '@/utils/got'; +import ofetch from '@/utils/ofetch'; import { parseDate } from '@/utils/parse-date'; export const route: Route = { @@ -28,14 +28,12 @@ export const route: Route = { }; async function handler() { - const response = await got({ - method: 'post', - url: 'https://api.juejin.cn/booklet_api/v1/booklet/listbycategory', - json: { category_id: '0', cursor: '0', limit: 20 }, + const response = await ofetch('https://api.juejin.cn/booklet_api/v1/booklet/listbycategory', { + method: 'POST', + body: { category_id: '0', cursor: '0', limit: 20 }, }); - const { data } = response.data; - const items = data.map(({ base_info }) => ({ + const items = response.data.map(({ base_info }) => ({ title: base_info.title, link: `https://juejin.cn/book/${base_info.booklet_id}`, description: ` diff --git a/lib/routes/juejin/category.ts b/lib/routes/juejin/category.ts index bf54bbcccb371a..15252a5d74d502 100644 --- a/lib/routes/juejin/category.ts +++ b/lib/routes/juejin/category.ts @@ -1,7 +1,6 @@ import { Route } from '@/types'; -import cache from '@/utils/cache'; -import got from '@/utils/got'; -import util from './utils'; +import ofetch from '@/utils/ofetch'; +import { getCategoryBrief, parseList, ProcessFeed } from './utils'; export const route: Route = { path: '/category/:category', @@ -16,29 +15,33 @@ export const route: Route = { supportPodcast: false, supportScihub: false, }, + radar: [ + { + source: ['juejin.cn/:category'], + }, + ], name: '分类', maintainers: ['DIYgod'], handler, description: `| 后端 | 前端 | Android | iOS | 人工智能 | 开发工具 | 代码人生 | 阅读 | - | ------- | -------- | ------- | --- | -------- | -------- | -------- | ------- | - | backend | frontend | android | ios | ai | freebie | career | article |`, +| ------- | -------- | ------- | --- | -------- | -------- | -------- | ------- | +| backend | frontend | android | ios | ai | freebie | career | article |`, }; async function handler(ctx) { const category = ctx.req.param('category'); - const idResponse = await got({ - method: 'get', - url: 'https://api.juejin.cn/tag_api/v1/query_category_briefs?show_type=0', - }); + const idResponse = await getCategoryBrief(); - const cat = idResponse.data.data.find((item) => item.category_url === category); + const cat = idResponse.find((item) => item.category_url === category); + if (!cat) { + throw new Error('分类不存在'); + } const id = cat.category_id; - const response = await got({ - method: 'post', - url: 'https://api.juejin.cn/recommend_api/v1/article/recommend_cate_feed', - json: { + const response = await ofetch('https://api.juejin.cn/recommend_api/v1/article/recommend_cate_feed', { + method: 'POST', + body: { id_type: 2, sort_type: 300, cate_id: id, @@ -47,11 +50,8 @@ async function handler(ctx) { }, }); - let originalData = []; - if (response.data.data) { - originalData = response.data.data; - } - const resultItems = await util.ProcessFeed(originalData, cache); + const list = parseList(response.data); + const resultItems = await ProcessFeed(list); return { title: `掘金 ${cat.category_name}`, diff --git a/lib/routes/juejin/collection.ts b/lib/routes/juejin/collection.ts index a571afe0c9bf10..bfac8037724470 100644 --- a/lib/routes/juejin/collection.ts +++ b/lib/routes/juejin/collection.ts @@ -1,7 +1,5 @@ import { Route } from '@/types'; -import cache from '@/utils/cache'; -import got from '@/utils/got'; -import util from './utils'; +import { getCollection, parseList, ProcessFeed } from './utils'; export const route: Route = { path: '/collection/:collectionId', @@ -22,27 +20,21 @@ export const route: Route = { }, ], name: '单个收藏夹', - maintainers: ['isQ'], + maintainers: ['yang131323'], handler, }; async function handler(ctx) { const collectionId = ctx.req.param('collectionId'); - const collectPage = await got({ - method: 'get', - url: `https://api.juejin.cn/interact_api/v1/collectionSet/get?tag_id=${collectionId}&cursor=0`, - }); + const collectPage = await getCollection(collectionId); - let items = []; - if (collectPage.data.data && collectPage.data.data.article_list) { - items = collectPage.data.data.article_list.slice(0, 10); - } + const items = parseList(collectPage.article_list); - const result = await util.ProcessFeed(items, cache); + const result = await ProcessFeed(items); return { - title: '掘金 - 单个收藏夹', + title: `${collectPage.detail.tag_name} - ${collectPage.create_user.user_name}的收藏集 - 掘金`, link: `https://juejin.cn/collection/${collectionId}`, description: '掘金，用户单个收藏夹', item: result, diff --git a/lib/routes/juejin/favorites.ts b/lib/routes/juejin/collections.ts similarity index 50% rename from lib/routes/juejin/favorites.ts rename to lib/routes/juejin/collections.ts index 4dc1a6027864a0..d1eb2269c11fb7 100644 --- a/lib/routes/juejin/favorites.ts +++ b/lib/routes/juejin/collections.ts @@ -1,7 +1,7 @@ import { Route } from '@/types'; -import cache from '@/utils/cache'; -import got from '@/utils/got'; -import util from './utils'; +import ofetch from '@/utils/ofetch'; +import { getCollection, parseList, ProcessFeed } from './utils'; +import { Article } from './types'; export const route: Route = { path: '/collections/:userId', @@ -23,37 +23,29 @@ export const route: Route = { }, ], name: '收藏集', - maintainers: ['isQ'], + maintainers: ['yang131323'], handler, }; +// 获取所有收藏夹文章内容 +async function getArticleList(collectionId) { + const collectPage = await getCollection(collectionId); + + return collectPage.article_list; +} + async function handler(ctx) { const userId = ctx.req.param('userId'); - const response = await got({ - method: 'get', - url: `https://api.juejin.cn/interact_api/v1/collectionSet/list?user_id=${userId}&cursor=0&limit=20`, - }); + const response = await ofetch(`https://api.juejin.cn/interact_api/v1/collectionSet/list?user_id=${userId}&cursor=0&limit=20`); // 获取用户所有收藏夹id - const collectionId = response.data.data.map((item) => item.tag_id); - - // 获取所有收藏夹文章内容 - async function getPostId(item) { - const collectPage = await got({ - method: 'get', - url: `https://api.juejin.cn/interact_api/v1/collectionSet/get?tag_id=${item}&cursor=0`, - }); - - return (Array.isArray(collectPage.data.data.article_list) && collectPage.data.data.article_list.slice(0, 10)) || []; - } + const collectionId = response.data.map((item) => item.tag_id); - const temp = await Promise.all(collectionId.map((element) => getPostId(element))); - const posts = []; - for (const item of temp) { - posts.push(...item); - } + const temp = (await Promise.all(collectionId.map((id) => getArticleList(id)))) as Article[][]; + const posts = temp.flat(); + const list = parseList(posts); - const result = await util.ProcessFeed(posts, cache); + const result = await ProcessFeed(list); return { title: '掘金 - 收藏集', diff --git a/lib/routes/juejin/column.ts b/lib/routes/juejin/column.ts index fad5bb71feaf27..2f8a89494ca739 100644 --- a/lib/routes/juejin/column.ts +++ b/lib/routes/juejin/column.ts @@ -1,7 +1,6 @@ import { Route } from '@/types'; -import cache from '@/utils/cache'; -import got from '@/utils/got'; -import util from './utils'; +import ofetch from '@/utils/ofetch'; +import { parseList, ProcessFeed } from './utils'; export const route: Route = { path: '/column/:id', @@ -28,29 +27,25 @@ export const route: Route = { async function handler(ctx) { const id = ctx.req.param('id'); - const detail = await got({ - method: 'get', - url: `https://api.juejin.cn/content_api/v1/column/detail?column_id=${id}`, - }); - const response = await got({ - method: 'post', - url: 'https://api.juejin.cn/content_api/v1/column/articles_cursor', - json: { + const columnDetail = await ofetch(`https://api.juejin.cn/content_api/v1/column/detail?column_id=${id}`); + const response = await ofetch('https://api.juejin.cn/content_api/v1/column/articles_cursor', { + method: 'POST', + body: { column_id: id, - limit: 20, cursor: '0', + limit: 20, sort: 0, }, }); - const { data } = response.data; - const detailData = detail.data.data; - const columnName = detailData && detailData.column_version && detailData.column_version.title; - const resultItems = await util.ProcessFeed(data, cache); + const detailData = columnDetail.data; + const list = parseList(response.data); + const resultItems = await ProcessFeed(list); return { - title: `掘金专栏-${columnName}`, + title: `${detailData.column_version.title} - ${detailData.author.user_name}的专栏 - 掘金`, link: `https://juejin.cn/column/${id}`, - description: `掘金专栏-${columnName}`, + description: detailData.column_version.description, + image: columnDetail.data.column_version.cover, item: resultItems, }; } diff --git a/lib/routes/juejin/dynamic.ts b/lib/routes/juejin/dynamic.ts index 6c7bf26c0ce999..8f997ac99b51e5 100644 --- a/lib/routes/juejin/dynamic.ts +++ b/lib/routes/juejin/dynamic.ts @@ -1,5 +1,5 @@ import { Route } from '@/types'; -import got from '@/utils/got'; +import ofetch from '@/utils/ofetch'; import { parseDate } from '@/utils/parse-date'; export const route: Route = { @@ -28,17 +28,16 @@ export const route: Route = { async function handler(ctx) { const id = ctx.req.param('id'); - const response = await got({ - method: 'get', - url: 'https://api.juejin.cn/user_api/v1/user/dynamic', - searchParams: { + const response = await ofetch('https://api.juejin.cn/user_api/v1/user/dynamic', { + query: { user_id: id, cursor: 0, }, }); - const list = response.data.data.list; + const list = response.data.list; - const username = list[0].user.user_name; + const user = list[0].user; + const username = user.user_name; const items = list.map((e) => { const { target_type, target_data, action, time } = e; // action: 0.发布文章；1.点赞文章；2.发布沸点；3.点赞沸点；4.关注用户 @@ -107,7 +106,8 @@ async function handler(ctx) { return { title: `掘金用户动态-${username}`, link: `https://juejin.cn/user/${id}/`, - description: `掘金用户动态-${username}`, + description: user.description || `掘金用户动态-${username}`, + image: user.avatar_large, item: items, author: username, }; diff --git a/lib/routes/juejin/pins.ts b/lib/routes/juejin/pins.ts index f6267c7d096587..f3c994420ece1a 100644 --- a/lib/routes/juejin/pins.ts +++ b/lib/routes/juejin/pins.ts @@ -1,5 +1,5 @@ import { Route } from '@/types'; -import got from '@/utils/got'; +import ofetch from '@/utils/ofetch'; import { parseDate } from '@/utils/parse-date'; export const route: Route = { @@ -38,7 +38,7 @@ async function handler(ctx) { }; let url = ''; - let json = null; + let json = {}; if (/^\d+$/.test(type)) { url = `https://api.juejin.cn/recommend_api/v1/short_msg/topic`; json = { id_type: 4, sort_type: 500, cursor: '0', limit: 20, topic_id: type }; @@ -47,10 +47,9 @@ async function handler(ctx) { json = { id_type: 4, sort_type: 200, cursor: '0', limit: 20 }; } - const response = await got({ - method: 'post', - url, - json, + const response = await ofetch(url, { + method: 'POST', + body: json, }); const items = response.data.data.map((item) => { diff --git a/lib/routes/juejin/posts.ts b/lib/routes/juejin/posts.ts index 58ceea04d9dcee..a35a4031d4d896 100644 --- a/lib/routes/juejin/posts.ts +++ b/lib/routes/juejin/posts.ts @@ -1,7 +1,7 @@ import { Route } from '@/types'; -import cache from '@/utils/cache'; -import got from '@/utils/got'; -import util from './utils'; +import ofetch from '@/utils/ofetch'; +import { parseList, ProcessFeed } from './utils'; +import { Article, AuthorUserInfo } from './types'; export const route: Route = { path: '/posts/:id', @@ -26,25 +26,32 @@ export const route: Route = { handler, }; +const getUserInfo = (data: AuthorUserInfo) => ({ + username: data.user_name, + description: data.description, + avatar: data.avatar_large, +}); + async function handler(ctx) { const id = ctx.req.param('id'); - const response = await got({ - method: 'post', - url: 'https://api.juejin.cn/content_api/v1/article/query_list', - json: { + const response = await ofetch('https://api.juejin.cn/content_api/v1/article/query_list', { + method: 'POST', + body: { user_id: id, sort_type: 2, }, }); - const { data } = response.data; - const username = data[0] && data[0].author_user_info && data[0].author_user_info.user_name; - const resultItems = await util.ProcessFeed(data, cache); + const data = response.data as Article[]; + const list = parseList(data); + const authorInfo = getUserInfo(data[0].author_user_info); + const resultItems = await ProcessFeed(list); return { - title: `掘金专栏-${username}`, + title: `掘金专栏-${authorInfo.username}`, link: `https://juejin.cn/user/${id}/posts`, - description: `掘金专栏-${username}`, + description: authorInfo.description, + image: authorInfo.avatar, item: resultItems, }; } diff --git a/lib/routes/juejin/tag.ts b/lib/routes/juejin/tag.ts index 934e70b80d6a32..072ce3cb7871a7 100644 --- a/lib/routes/juejin/tag.ts +++ b/lib/routes/juejin/tag.ts @@ -1,7 +1,6 @@ import { Route } from '@/types'; -import cache from '@/utils/cache'; -import got from '@/utils/got'; -import util from './utils'; +import ofetch from '@/utils/ofetch'; +import { getTag, parseList, ProcessFeed } from './utils'; export const route: Route = { path: '/tag/:tag', @@ -29,37 +28,28 @@ export const route: Route = { async function handler(ctx) { const tag = ctx.req.param('tag'); - const idResponse = await got({ - method: 'post', - url: 'https://api.juejin.cn/tag_api/v1/query_tag_detail', - json: { - key_word: tag, - }, - }); + const idResponse = await getTag(tag); - const id = idResponse.data.data.tag_id; + const id = idResponse.tag_id; - const response = await got({ - method: 'post', - url: 'https://api.juejin.cn/recommend_api/v1/article/recommend_tag_feed', - json: { + const response = await ofetch('https://api.juejin.cn/recommend_api/v1/article/recommend_tag_feed', { + method: 'POST', + body: { id_type: 2, cursor: '0', tag_ids: [id], - sort_type: 200, + sort_type: 300, }, }); - let originalData = []; - if (response.data.data) { - originalData = response.data.data.slice(0, 10); - } - const resultItems = await util.ProcessFeed(originalData, cache); + const originalData = parseList(response.data); + const resultItems = await ProcessFeed(originalData); return { title: `掘金 ${tag}`, link: `https://juejin.cn/tag/${encodeURIComponent(tag)}`, description: `掘金 ${tag}`, + image: idResponse.tag.icon, item: resultItems, }; } diff --git a/lib/routes/juejin/trending.ts b/lib/routes/juejin/trending.ts index f06cdbf7fd664f..de465ab8d4b6cf 100644 --- a/lib/routes/juejin/trending.ts +++ b/lib/routes/juejin/trending.ts @@ -1,7 +1,6 @@ import { Route } from '@/types'; -import cache from '@/utils/cache'; -import got from '@/utils/got'; -import util from './utils'; +import ofetch from '@/utils/ofetch'; +import { getCategoryBrief, parseList, ProcessFeed } from './utils'; export const route: Route = { path: '/trending/:category/:type', @@ -46,11 +45,8 @@ async function handler(ctx) { let id = ''; let name = ''; let url = 'recommended'; - const idResponse = await got({ - method: 'get', - url: 'https://api.juejin.cn/tag_api/v1/query_category_briefs', - }); - const cat = idResponse.data.data.find((item) => item.category_url === category); + const idResponse = await getCategoryBrief(); + const cat = idResponse.find((item) => item.category_url === category); if (cat) { id = cat.category_id; name = cat.category_name; @@ -97,18 +93,18 @@ async function handler(ctx) { getJson.cate_id = id; } - const trendingResponse = await got({ - method: 'post', - url: getUrl, - json: getJson, + const trendingResponse = await ofetch(getUrl, { + method: 'POST', + body: getJson, }); - let entrylist = trendingResponse.data.data; + let entrylist = trendingResponse.data; if (category === 'all' || category === 'devops' || category === 'product' || category === 'design') { - entrylist = trendingResponse.data.data.filter((item) => item.item_type === 2).map((item) => item.item_info); + entrylist = trendingResponse.data.filter((item) => item.item_type === 2).map((item) => item.item_info); } + const list = parseList(entrylist); - const resultItems = await util.ProcessFeed(entrylist, cache); + const resultItems = await ProcessFeed(list); return { title, diff --git a/lib/routes/juejin/types.ts b/lib/routes/juejin/types.ts new file mode 100644 index 00000000000000..881eb90bee7218 --- /dev/null +++ b/lib/routes/juejin/types.ts @@ -0,0 +1,232 @@ +interface University { + university_id: string; + name: string; + logo: string; +} + +interface Major { + major_id: string; + parent_id: string; + name: string; +} + +interface UserGrowthInfo { + user_id: number; + jpower: number; + jscore: number; + jpower_level: number; + jscore_level: number; + jscore_title: string; + author_achievement_list: number[]; + vip_level: number; + vip_title: string; + jscore_next_level_score: number; + jscore_this_level_mini_score: number; + vip_score: number; +} + +interface UserPrivInfo { + administrator: number; + builder: number; + favorable_author: number; + book_author: number; + forbidden_words: number; + can_tag_cnt: number; + auto_recommend: number; + signed_author: number; + popular_author: number; + can_add_video: number; +} + +interface ArticleInfo { + article_id: string; + user_id: string; + category_id: string; + tag_ids: number[]; + visible_level: number; + link_url: string; + cover_image: string; + is_gfw: number; + title: string; + brief_content: string; + is_english: number; + is_original: number; + user_index: number; + original_type: number; + original_author: string; + content: string; + ctime: string; + mtime: string; + rtime: string; + draft_id: string; + view_count: number; + collect_count: number; + digg_count: number; + comment_count: number; + hot_index: number; + is_hot: number; + rank_index: number; + status: number; + verify_status: number; + audit_status: number; + mark_content: string; + display_count: number; + is_markdown: number; + app_html_content: string; + version: number; + web_html_content: null; + meta_info: null; + catalog: null; + homepage_top_time: number; + homepage_top_status: number; + content_count: number; + read_time: string; +} + +export interface AuthorUserInfo { + user_id: string; + user_name: string; + company: string; + job_title: string; + avatar_large: string; + level: number; + description: string; + followee_count: number; + follower_count: number; + post_article_count: number; + digg_article_count: number; + got_digg_count: number; + got_view_count: number; + post_shortmsg_count: number; + digg_shortmsg_count: number; + isfollowed: boolean; + favorable_author: number; + power: number; + study_point: number; + university: University; + major: Major; + student_status: number; + select_event_count: number; + select_online_course_count: number; + identity: number; + is_select_annual: boolean; + select_annual_rank: number; + annual_list_type: number; + extraMap: Record; + is_logout: number; + annual_info: any[]; + account_amount: number; + user_growth_info: UserGrowthInfo; + is_vip: boolean; + become_author_days: number; + collection_set_article_count: number; + recommend_article_count_daily: number; + article_collect_count_daily: number; + user_priv_info: UserPrivInfo; +} + +export interface Category { + category_id: string; + category_name: string; + category_url: string; + rank: number; + back_ground: string; + icon: string; + ctime: number; + mtime: number; + show_type: number; + item_type: number; + promote_tag_cap: number; + promote_priority: number; +} + +export interface Tag { + id: number; + tag_id: string; + tag_name: string; + color: string; + icon: string; + back_ground: string; + show_navi: number; + ctime: number; + mtime: number; + id_type: number; + tag_alias: string; + post_article_count: number; + concern_user_count: number; +} + +interface UserInteract { + id: number; + omitempty: number; + user_id: number; + is_digg: boolean; + is_follow: boolean; + is_collect: boolean; + collect_set_count: number; +} + +interface OrgVersion { + version_id: string; + icon: string; + background: string; + name: string; + introduction: string; + weibo_link: string; + github_link: string; + homepage_link: string; + ctime: number; + mtime: number; + org_id: string; + brief_introduction: string; + introduction_preview: string; +} + +interface OrgInfo { + org_type: number; + org_id: string; + online_version_id: number; + latest_version_id: number; + power: number; + ctime: number; + mtime: number; + audit_status: number; + status: number; + org_version: OrgVersion; + follower_count: number; + article_view_count: number; + article_digg_count: number; +} + +interface Org { + is_followed: boolean; + org_info: OrgInfo; +} + +interface Status { + push_status: number; +} + +interface Extra { + extra: string; +} + +export interface Article { + article_id: string; + article_info: ArticleInfo; + author_user_info: AuthorUserInfo; + category: Category; + tags: Tag[]; + user_interact: UserInteract; + org: Org; + req_id: string; + status: Status; + theme_list: any[]; + extra: Extra; +} + +export interface Collection { + article_list: Article[]; + create_user: AuthorUserInfo; + detail: Tag; +} diff --git a/lib/routes/juejin/utils.ts b/lib/routes/juejin/utils.ts index 0c0119743d7fb0..024392c67062dc 100644 --- a/lib/routes/juejin/utils.ts +++ b/lib/routes/juejin/utils.ts @@ -1,55 +1,154 @@ -import got from '@/utils/got'; -import { load } from 'cheerio'; +import ofetch from '@/utils/ofetch'; +import * as cheerio from 'cheerio'; import { parseDate } from '@/utils/parse-date'; -import MarkdownIt from 'markdown-it'; -const md = MarkdownIt({ - html: true, -}); +// import MarkdownIt from 'markdown-it'; +// const md = MarkdownIt({ +// html: true, +// }); +import crypto from 'node:crypto'; +import cache from '@/utils/cache'; +import { Category, Collection, Tag } from './types'; + +const b64tou8a = (str) => Uint8Array.from(Buffer.from(str, 'base64')); +const b64tohex = (str) => Buffer.from(str, 'base64').toString('hex'); +const s256 = (s1: Uint8Array, s2: string) => { + const sha = crypto.createHash('sha256'); + sha.update(s1); + sha.update(s2); + return sha.digest('hex'); +}; + // 加载文章页 -async function loadContent(id) { - const response = await got({ - method: 'post', - url: 'https://api.juejin.cn/content_api/v1/article/detail', - json: { - article_id: id, - }, - }); - let description; - if (response.data.data) { - description = md.render(response.data.data.article_info.mark_content) || response.data.data.article_info.content; +// async function loadContent(id) { +// const response = await ofetch('https://api.juejin.cn/content_api/v1/article/detail', { +// method: 'post', +// body: { +// article_id: id, +// }, +// }); +// let description; +// if (response.data) { +// description = md.render(response.data.article_info.mark_content) || response.data.article_info.content; +// } + +// return { description }; +// } + +const solveWafChallenge = (cs) => { + const c = JSON.parse(Buffer.from(cs, 'base64').toString()); + const prefix = b64tou8a(c.v.a); + const expect = b64tohex(c.v.c); + + for (let i = 0; i < 1_000_000; i++) { + const hash = s256(prefix, i.toString()); + if (hash === expect) { + c.d = Buffer.from(i.toString()).toString('base64'); + break; + } } + return Buffer.from(JSON.stringify(c)).toString('base64'); +}; + +export const getArticle = async (link) => { + let response = await ofetch(link); + let $ = cheerio.load(response); + if ($('script').text().includes('_wafchallengeid')) { + const cs = $('script:contains("_wafchallengeid")') + .text() + .match(/cs="(.*?)",c/)?.[1]; + const cookie = solveWafChallenge(cs); + + response = await ofetch(link, { + headers: { + cookie: `_wafchallengeid=${cookie};`, + }, + }); - return { description }; -} + $ = cheerio.load(response); + } -const loadNews = async (link) => { - const response = await got(link); - const $ = load(response.data); - $('h1.title, .main-box .message').remove(); - return { description: $('.main-box .article').html() }; + return $('.article-viewer').html(); }; -const ProcessFeed = (list, caches) => +// const loadNews = async (link) => { +// const response = await ofetch(link); +// const $ = cheerio.load(response); +// $('h1.title, .main-box .message').remove(); +// return { description: $('.main-box .article').html() }; +// }; + +export const parseList = (data) => + data.map((item) => { + const isArticle = !!item.article_info; + + return { + title: isArticle ? item.article_info.title : item.content_info.title, + description: (isArticle ? item.article_info.brief_content : item.content_info.brief) || '无描述', + pubDate: parseDate(isArticle ? item.article_info.ctime : item.content_info.ctime, 'X'), + author: item.author_user_info.user_name, + link: `https://juejin.cn${isArticle ? `/post/${item.article_id}` : `/news/${item.content_id}`}`, + category: [...new Set([item.category.category_name, ...item.tags.map((tag) => tag.tag_name)])], + }; + }); + +export const ProcessFeed = (list) => Promise.all( - list.map(async (item) => { - const isArticle = !!item.article_info; - const pubDate = parseDate((isArticle ? item.article_info.ctime : item.content_info.ctime) * 1000); - const link = `https://juejin.cn${isArticle ? '/post/' + item.article_id : '/news/' + item.content_id}`; - // 列表上提取到的信息 - const single = { - title: isArticle ? item.article_info.title : item.content_info.title, - description: ((isArticle ? item.article_info.brief_content : item.content_info.brief) || '无描述').replaceAll(/[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F]/g, ''), - pubDate, - author: item.author_user_info.user_name, - link, - }; - - // 使用tryGet方法从缓存获取内容。 - // 当缓存中无法获取到链接内容的时候，则使用load方法加载文章内容。 - const other = await caches.tryGet(link, () => (isArticle ? loadContent(item.article_id) : loadNews(link))); - // 合并解析后的结果集作为该篇文章最终的输出结果 - return { ...single, ...other }; - }) + list.map((item) => + cache.tryGet(item.link, async () => { + item.description = (await getArticle(item.link)) || item.description; + + return item; + }) + ) ); -export default { ProcessFeed }; +// export const ProcessFeed = (list, caches) => +// Promise.all( +// list.map(async (item) => { +// const isArticle = !!item.article_info; +// const pubDate = parseDate((isArticle ? item.article_info.ctime : item.content_info.ctime) * 1000); +// const link = `https://juejin.cn${isArticle ? '/post/' + item.article_id : '/news/' + item.content_id}`; +// // 列表上提取到的信息 +// const single = { +// title: isArticle ? item.article_info.title : item.content_info.title, +// description: ((isArticle ? item.article_info.brief_content : item.content_info.brief) || '无描述').replaceAll(/[\u0000-\u0008\u000B\u000C\u000E-\u001F\u007F]/g, ''), +// pubDate, +// author: item.author_user_info.user_name, +// link, +// }; + +// // 使用tryGet方法从缓存获取内容。 +// // 当缓存中无法获取到链接内容的时候，则使用load方法加载文章内容。 +// const other = await caches.tryGet(link, () => (isArticle ? loadContent(item.article_id) : loadNews(link))); +// // 合并解析后的结果集作为该篇文章最终的输出结果 +// return { ...single, ...other }; +// }) +// ); + +export const getCategoryBrief = () => + cache.tryGet('juejin:categoryBriefs', async () => { + const response = await ofetch('https://api.juejin.cn/tag_api/v1/query_category_briefs'); + return response.data; + }) as Promise; + +export const getCollection = (collectionId) => + cache.tryGet(`juejin:collectionId:${collectionId}`, async () => { + const response = await ofetch('https://api.juejin.cn/interact_api/v1/collectionSet/get', { + query: { + tag_id: collectionId, + cursor: 0, + }, + }); + return response.data; + }) as Promise; + +export const getTag = (tag) => + cache.tryGet(`juejin:tag:${tag}`, async () => { + const response = await ofetch('https://api.juejin.cn/tag_api/v1/query_tag_detail', { + method: 'POST', + body: { + key_word: tag, + }, + }); + return response.data; + }) as Promise<{ tag_id: string; tag: Tag }>;