From c361327581750f5b748171ccf9b518e7195e0ee9 Mon Sep 17 00:00:00 2001 From: Leo Kettmeir Date: Wed, 14 May 2025 15:34:39 +0200 Subject: [PATCH 01/15] init --- Cargo.lock | 8 +- api/Cargo.toml | 7 +- api/src/analysis.rs | 38 +- api/src/api/mod.rs | 33 ++ api/src/api/package.rs | 14 +- api/src/api/scope.rs | 100 ++++- api/src/api/types.rs | 8 - api/src/docs.rs | 25 +- api/src/orama.rs | 32 +- api/src/publish.rs | 4 +- api/src/tarball.rs | 7 +- frontend/components/List.tsx | 8 +- frontend/components/Nav.tsx | 9 +- frontend/deno.json | 2 +- frontend/deno.lock | 415 +++++++++++------- .../@[scope]/(_components)/ScopeNav.tsx | 44 +- .../@[scope]/(_islands)/ScopeSymbolSearch.tsx | 155 +++++++ frontend/routes/@[scope]/index.tsx | 7 + .../routes/package/(_components)/Docs.tsx | 37 +- .../package/(_islands)/BreadcrumbsSticky.tsx | 16 +- .../package/(_islands)/LocalSymbolSearch.tsx | 308 ------------- .../(_islands)/PackageSymbolSearch.tsx | 268 +++++++++++ frontend/routes/package/all_symbols.tsx | 2 + frontend/routes/package/doc/[file].tsx | 2 + frontend/routes/package/doc/[symbol].tsx | 2 + frontend/routes/package/index.tsx | 2 + frontend/routes/package/source.tsx | 18 +- frontend/util.ts | 6 - frontend/utils/api_types.ts | 6 - frontend/utils/data.ts | 6 - frontend/utils/symbolsearch.ts | 161 +++++++ 31 files changed, 1147 insertions(+), 603 deletions(-) create mode 100644 frontend/routes/@[scope]/(_islands)/ScopeSymbolSearch.tsx delete mode 100644 frontend/routes/package/(_islands)/LocalSymbolSearch.tsx create mode 100644 frontend/routes/package/(_islands)/PackageSymbolSearch.tsx create mode 100644 frontend/utils/symbolsearch.ts diff --git a/Cargo.lock b/Cargo.lock index 05283b76a..24f9ad893 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -952,9 +952,7 @@ dependencies = [ [[package]] name = "deno_doc" -version = "0.171.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e2586f93cf7326a45a117affbb76f1330f26cfea62cc13b91b02fce34c1d81d" +version = "0.172.0" dependencies = [ "anyhow", "cfg-if", @@ -1006,9 +1004,9 @@ dependencies = [ [[package]] name = "deno_graph" -version = "0.89.4" +version = "0.90.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5d12e08a843ed1dbef68de3720092e67dcb958eae59ec2f26a459597c2c76be0" +checksum = "63892e3d2eaf61633ecc1046340f862f295e7ad04d75522517e9a2505a158c4e" dependencies = [ "async-trait", "capacity_builder", diff --git a/api/Cargo.toml b/api/Cargo.toml index d567d68e3..a42b96760 100644 --- a/api/Cargo.toml +++ b/api/Cargo.toml @@ -78,9 +78,9 @@ deno_semver = "0.7.1" flate2 = "1" thiserror = "2" async-tar = "0.4.2" -deno_graph = "0.89.4" +deno_graph = "0.90.0" deno_ast = { version = "0.46.5", features = ["view"] } -deno_doc = { version = "0.171.1", features = ["comrak"] } +deno_doc = { path = "../../deno_doc", features = ["comrak"] } deno_error = "0.5.5" comrak = { version = "0.29.0", default-features = false } ammonia = "4.0.0" @@ -117,3 +117,6 @@ lazy_static = "1.5.0" flate2 = "1" deno_semver = "0.7.1" pretty_assertions = "1.4.0" + +[build-dependencies] +deno_doc = { path = "../../deno_doc", features = ["comrak"] } diff --git a/api/src/analysis.rs b/api/src/analysis.rs index 37af87598..ca80567ea 100644 --- a/api/src/analysis.rs +++ b/api/src/analysis.rs @@ -13,6 +13,7 @@ use deno_ast::ModuleSpecifier; use deno_ast::ParsedSource; use deno_ast::SourceRange; use deno_ast::SourceRangedForSpanned; +use deno_doc::html::search::SearchIndexNode; use deno_doc::DocNodeDef; use deno_error::JsErrorBox; use deno_graph::source::load_data_url; @@ -68,7 +69,7 @@ pub struct PackageAnalysisOutput { pub data: PackageAnalysisData, pub module_graph_2: HashMap, pub doc_nodes_json: Bytes, - pub doc_search_json: serde_json::Value, + pub doc_search: Vec, pub dependencies: HashSet<(DependencyKind, PackageReqReference)>, pub npm_tarball: NpmTarball, pub readme_path: Option, @@ -174,7 +175,6 @@ async fn analyze_package_inner( jsr_url_provider: &PassthroughJsrUrlProvider, es_parser: Some(&module_analyzer.analyzer), resolver: Default::default(), - npm_resolver: Default::default(), workspace_fast_check: WorkspaceFastCheckOption::Enabled(&workspace_members), }); @@ -252,8 +252,8 @@ async fn analyze_package_inner( doc_nodes, main_entrypoint, info.rewrite_map, - scope, - name, + scope.clone(), + name.clone(), version, true, None, @@ -266,20 +266,33 @@ async fn analyze_package_inner( bun: None, }, registry_url.to_string(), + Some(format!("{scope}/{name}/")), ); - let search_index = deno_doc::html::generate_search_index(&ctx); - let doc_search_json = if let serde_json::Value::Object(mut obj) = search_index - { - obj.remove("nodes").unwrap() - } else { - unreachable!() - }; + + let doc_nodes = ctx + .doc_nodes + .values() + .flatten() + .map(std::borrow::Cow::Borrowed); + let partitions = + deno_doc::html::partition::partition_nodes_by_name(&ctx, doc_nodes, true); + + let mut doc_search = partitions + .into_iter() + .flat_map(|(name, nodes)| { + deno_doc::html::search::doc_nodes_into_search_index_node( + &ctx, nodes, name, None, + ) + }) + .collect::>(); + + doc_search.sort_by(|a, b| a.file.cmp(&b.file)); Ok(PackageAnalysisOutput { data: PackageAnalysisData { exports, files }, module_graph_2, doc_nodes_json, - doc_search_json, + doc_search, dependencies, npm_tarball, readme_path, @@ -602,7 +615,6 @@ async fn rebuild_npm_tarball_inner( jsr_url_provider: &PassthroughJsrUrlProvider, es_parser: Some(&module_analyzer.analyzer), resolver: None, - npm_resolver: None, workspace_fast_check: WorkspaceFastCheckOption::Enabled(&workspace_members), }); diff --git a/api/src/api/mod.rs b/api/src/api/mod.rs index 20b73573b..5ddf33826 100644 --- a/api/src/api/mod.rs +++ b/api/src/api/mod.rs @@ -59,6 +59,9 @@ pub fn api_router() -> Router { util::json(publishing_task::get_handler), ) .scope("/tickets", tickets_router()) + .get("/ddoc/style.css", util::cache(CacheDuration::ONE_MINUTE, ddoc_style_handler)) + .get("/ddoc/comrak.css", util::cache(CacheDuration::ONE_MINUTE, ddoc_comrak_handler)) + .get("/ddoc/script.js", util::cache(CacheDuration::ONE_MINUTE, ddoc_script_handler)) .get("/.well-known/openapi", openapi_handler) .build() .unwrap() @@ -74,3 +77,33 @@ async fn openapi_handler( .unwrap(); Ok(resp) } + +async fn ddoc_style_handler( + _: hyper::Request, +) -> util::ApiResult> { + let resp = Response::builder() + .header("Content-Type", "text/css") + .body(Body::from(deno_doc::html::STYLESHEET)) + .unwrap(); + Ok(resp) +} + +async fn ddoc_comrak_handler( + _: hyper::Request, +) -> util::ApiResult> { + let resp = Response::builder() + .header("Content-Type", "text/css") + .body(Body::from(deno_doc::html::comrak::COMRAK_STYLESHEET)) + .unwrap(); + Ok(resp) +} + +async fn ddoc_script_handler( + _: hyper::Request, +) -> util::ApiResult> { + let resp = Response::builder() + .header("Content-Type", "application/javascript") + .body(Body::from(deno_doc::html::SCRIPT_JS)) + .unwrap(); + Ok(resp) +} diff --git a/api/src/api/package.rs b/api/src/api/package.rs index af59f9db8..bfd421397 100644 --- a/api/src/api/package.rs +++ b/api/src/api/package.rs @@ -34,7 +34,6 @@ use routerify_query::RequestQueryExt; use serde::Deserialize; use serde::Serialize; use sha2::Digest; -use std::borrow::Cow; use std::io; use std::sync::atomic::AtomicU64; use std::sync::atomic::Ordering; @@ -1248,6 +1247,8 @@ pub async fn get_docs_handler( readme, package.runtime_compat, registry_url, + None, + None, ) .map_err(|e| { error!("failed to generate docs: {}", e); @@ -1257,9 +1258,6 @@ pub async fn get_docs_handler( match docs { GeneratedDocsOutput::Docs(docs) => Ok(ApiPackageVersionDocs::Content { - css: Cow::Borrowed(deno_doc::html::STYLESHEET), - comrak_css: Cow::Borrowed(deno_doc::html::comrak::COMRAK_STYLESHEET), - script: Cow::Borrowed(deno_doc::html::SCRIPT_JS), breadcrumbs: docs.breadcrumbs, toc: docs.toc, main: docs.main, @@ -1335,9 +1333,10 @@ pub async fn get_docs_search_handler( false, package.runtime_compat, registry_url, + None, ); - let search_index = deno_doc::html::generate_search_index(&ctx); + let search_index = deno_doc::html::search::generate_search_index(&ctx); Ok(search_index) } @@ -1407,6 +1406,8 @@ pub async fn get_docs_search_html_handler( None, package.runtime_compat, registry_url, + Some(format!("{}/{}/", scope, package_name)), + None, ) .map_err(|e| { error!("failed to generate docs: {}", e); @@ -1575,9 +1576,6 @@ pub async fn get_source_handler( Ok(ApiPackageVersionSource { version: ApiPackageVersion::from(version), - css: Cow::Borrowed(deno_doc::html::STYLESHEET), - comrak_css: Cow::Borrowed(deno_doc::html::comrak::COMRAK_STYLESHEET), - script: Cow::Borrowed(deno_doc::html::SCRIPT_JS), source, }) } diff --git a/api/src/api/scope.rs b/api/src/api/scope.rs index 9ecba9c22..a9c48e708 100644 --- a/api/src/api/scope.rs +++ b/api/src/api/scope.rs @@ -1,18 +1,19 @@ // Copyright 2024 the JSR authors. All rights reserved. MIT license. -use std::borrow::Cow; -use std::sync::OnceLock; - use crate::api::package::package_router; use crate::emails::EmailArgs; use crate::emails::EmailSender; use crate::iam::ReqIamExt; use crate::RegistryUrl; +use anyhow::Context; use hyper::Body; use hyper::Request; use hyper::Response; use hyper::StatusCode; use routerify::ext::RequestExt; use routerify::Router; +use std::borrow::Cow; +use std::sync::OnceLock; +use tracing::error; use tracing::field; use tracing::instrument; use tracing::Span; @@ -23,10 +24,15 @@ use super::types::*; use crate::auth::lookup_user_by_github_login; use crate::auth::GithubOauth2Client; +use crate::buckets::Buckets; use crate::db::*; +use crate::docs::DocNodesByUrl; +use crate::docs::DocsRequest; +use crate::docs::GeneratedDocsOutput; use crate::util; use crate::util::decode_json; use crate::util::ApiResult; +use crate::util::CacheDuration; use crate::util::RequestIdExt; pub fn scope_router() -> Router { @@ -54,6 +60,13 @@ pub fn scope_router() -> Router { "/:scope/invites/:user_id", util::auth(delete_invite_handler), ) + .get( + "/:scope/search_html", + util::cache( + CacheDuration::ONE_MINUTE, + util::json(get_docs_search_html_handler), + ), + ) .build() .unwrap() } @@ -490,6 +503,87 @@ pub async fn delete_invite_handler( Ok(resp) } +#[instrument( + name = "GET /api/scopes/:scope/search_html", + skip(req), + err, + fields(scope, user_id) +)] +pub async fn get_docs_search_html_handler( + req: Request, +) -> ApiResult { + let scope = req.param_scope()?; + Span::current().record("scope", field::display(&scope)); + + let db = req.data::().unwrap(); + let buckets = req.data::().unwrap(); + + let (_, packages) = db.list_packages_by_scope(&scope, false, 0, 100).await?; + + let registry_url = req.data::().unwrap().0.to_string(); + + let mut outsearch = String::new(); + for (package, _, _) in packages { + let (package, repo, _) = db + .get_package(&scope, &package.name) + .await? + .ok_or(ApiError::PackageNotFound)?; + + let Some(version) = db + .get_latest_unyanked_version_for_package(&scope, &package.name) + .await? + else { + continue; + }; + + let docs_path = + crate::gcs_paths::docs_v1_path(&scope, &package.name, &version.version); + let docs = buckets.docs_bucket.download(docs_path.into()).await?; + let docs = docs.ok_or_else(|| { + error!( + "docs not found for {}/{}/{}", + scope, package.name, version.version + ); + ApiError::InternalServerError + })?; + + let doc_nodes: DocNodesByUrl = + serde_json::from_slice(&docs).context("failed to parse doc nodes")?; + let docs_info = crate::docs::get_docs_info(&version.exports, None); + + let docs = crate::docs::generate_docs_html( + doc_nodes, + docs_info.main_entrypoint, + docs_info.rewrite_map, + DocsRequest::AllSymbols, + scope.clone(), + package.name.clone(), + version.version.clone(), + true, + repo, + None, + package.runtime_compat, + registry_url.clone(), + Some(format!("{}/{}/", scope, package.name)), + Some(format!("@{}/{}/", scope, package.name)), + ) + .map_err(|e| { + error!("failed to generate docs: {}", e); + ApiError::InternalServerError + })? + .unwrap(); + + let search = match docs { + GeneratedDocsOutput::Docs(docs) => docs.main, + GeneratedDocsOutput::Redirect(_) => unreachable!(), + }; + + outsearch.push_str(&search); + } + + Ok(outsearch) +} + #[cfg(test)] pub mod tests { use super::*; diff --git a/api/src/api/types.rs b/api/src/api/types.rs index 4157f973b..133cb6d05 100644 --- a/api/src/api/types.rs +++ b/api/src/api/types.rs @@ -1,6 +1,4 @@ // Copyright 2024 the JSR authors. All rights reserved. MIT license. -use std::borrow::Cow; - use crate::db::*; use crate::ids::PackageName; use crate::ids::PackagePath; @@ -597,9 +595,6 @@ pub enum ApiPackageVersionDocs { #[serde(rename_all = "camelCase")] Content { version: ApiPackageVersion, - css: Cow<'static, str>, - comrak_css: Cow<'static, str>, - script: Cow<'static, str>, breadcrumbs: Option, toc: Option, main: String, @@ -653,9 +648,6 @@ pub enum ApiSource { #[serde(rename_all = "camelCase")] pub struct ApiPackageVersionSource { pub version: ApiPackageVersion, - pub css: Cow<'static, str>, - pub comrak_css: Cow<'static, str>, - pub script: Cow<'static, str>, pub source: ApiSource, } diff --git a/api/src/docs.rs b/api/src/docs.rs index 342b816ea..83cb34d99 100644 --- a/api/src/docs.rs +++ b/api/src/docs.rs @@ -435,6 +435,7 @@ pub fn get_generate_ctx<'a>( has_readme: bool, runtime_compat: RuntimeCompat, registry_url: String, + id_prefix: Option, ) -> GenerateCtx { let package_name = format!("@{scope}/{package}"); let url_rewriter_base = format!("/{package_name}/{version}"); @@ -510,6 +511,7 @@ pub fn get_generate_ctx<'a>( markdown_renderer, markdown_stripper: Rc::new(deno_doc::html::comrak::strip), head_inject: None, + id_prefix, }, None, deno_doc::html::FileMode::Normal, @@ -537,6 +539,8 @@ pub fn generate_docs_html( readme: Option, runtime_compat: RuntimeCompat, registry_url: String, + id_prefix: Option, + all_symbols_section_prefix: Option, ) -> Result, anyhow::Error> { let ctx = get_generate_ctx( doc_nodes_by_url, @@ -550,6 +554,7 @@ pub fn generate_docs_html( readme.is_some(), runtime_compat, registry_url, + id_prefix, ); match req { @@ -570,10 +575,20 @@ pub fn generate_docs_html( partitions_by_kind.into_iter().map(|(path, nodes)| { ( render_ctx.clone(), - deno_doc::html::SectionHeaderCtx::new_for_all_symbols( - &render_ctx, - &path, - ), + { + let mut header = deno_doc::html::SectionHeaderCtx::new_for_all_symbols( + &render_ctx, + &path, + ); + + if let Some(header) = &mut header { + if let Some(all_symbols_section_prefix) = &all_symbols_section_prefix { + header.title = format!("{all_symbols_section_prefix}{}", header.title); + } + } + + header + }, nodes, ) }), @@ -586,7 +601,7 @@ pub fn generate_docs_html( .render( "symbol_content", &deno_doc::html::SymbolContentCtx { - id: String::new(), + id: deno_doc::html::util::Id::empty(), sections, docs: None, }, diff --git a/api/src/orama.rs b/api/src/orama.rs index 99b521e48..055f49e00 100644 --- a/api/src/orama.rs +++ b/api/src/orama.rs @@ -1,14 +1,14 @@ // Copyright 2024 the JSR authors. All rights reserved. MIT license. // Copyright Deno Land Inc. All Rights Reserved. Proprietary and confidential. -use std::sync::Arc; - use crate::api::ApiPackageScore; use crate::db::Package; use crate::db::PackageVersionMeta; use crate::ids::PackageName; use crate::ids::ScopeName; use crate::util::USER_AGENT; +use deno_doc::html::search::SearchIndexNode; +use std::sync::Arc; use tracing::error; use tracing::instrument; use tracing::Instrument; @@ -138,7 +138,7 @@ impl OramaClient { &self, scope_name: &ScopeName, package_name: &PackageName, - search: serde_json::Value, + search: &[SearchIndexNode], ) { let package = format!("{scope_name}/{package_name}"); let body = serde_json::json!({ @@ -172,17 +172,21 @@ impl OramaClient { .instrument(span), ); - let search = if let serde_json::Value::Array(mut array) = search { - for entry in &mut array { - let obj = entry.as_object_mut().unwrap(); - obj.insert("scope".to_string(), scope_name.to_string().into()); - obj.insert("package".to_string(), package_name.to_string().into()); - } - - array - } else { - unreachable!() - }; + let search = search + .iter() + .map(|node| { + serde_json::json!({ + "target_id": node.id, + "name": node.name, + "file": node.file, + "doc": node.doc, + "url": node.url, + "deprecated": node.deprecated, + "scope": scope_name.to_string(), + "package": package_name.to_string(), + }) + }) + .collect::>(); let chunks = { let str_data = serde_json::to_string(&search).unwrap(); diff --git a/api/src/publish.rs b/api/src/publish.rs index 3ccd835c9..634d14163 100644 --- a/api/src/publish.rs +++ b/api/src/publish.rs @@ -206,7 +206,7 @@ async fn process_publishing_task( npm_tarball_info, readme_path, meta, - doc_search_json, + doc_search, } = output; upload_version_manifest( @@ -234,7 +234,7 @@ async fn process_publishing_task( orama_client.upsert_symbols( &publishing_task.package_scope, &publishing_task.package_name, - doc_search_json, + &doc_search, ); } diff --git a/api/src/tarball.rs b/api/src/tarball.rs index 260ca3bc3..61ee7c05f 100644 --- a/api/src/tarball.rs +++ b/api/src/tarball.rs @@ -7,6 +7,7 @@ use std::sync::OnceLock; use async_tar::EntryType; use bytes::Bytes; use deno_ast::MediaType; +use deno_doc::html::search::SearchIndexNode; use deno_graph::ModuleGraphError; use deno_semver::jsr::JsrPackageReqReference; use deno_semver::npm::NpmPackageReqReference; @@ -66,7 +67,7 @@ pub struct ProcessTarballOutput { pub npm_tarball_info: NpmTarballInfo, pub readme_path: Option, pub meta: PackageVersionMeta, - pub doc_search_json: serde_json::Value, + pub doc_search: Vec, } pub struct NpmTarballInfo { @@ -283,7 +284,7 @@ pub async fn process_tarball( data: PackageAnalysisData { exports, files }, module_graph_2, doc_nodes_json, - doc_search_json, + doc_search, dependencies, npm_tarball, readme_path, @@ -460,7 +461,7 @@ pub async fn process_tarball( npm_tarball_info, readme_path, meta, - doc_search_json, + doc_search, }) } diff --git a/frontend/components/List.tsx b/frontend/components/List.tsx index 12bfcbb26..1996ab65e 100644 --- a/frontend/components/List.tsx +++ b/frontend/components/List.tsx @@ -10,15 +10,19 @@ export interface ListDisplayItem { } export function ListDisplay( - { title, pagination, currentUrl, children }: { + { title, pagination, currentUrl, children, id }: { title?: string; pagination?: PaginationData; currentUrl?: URL; children: ListDisplayItem[]; + id?: string; }, ) { return ( -
+
{title && (
diff --git a/frontend/components/Nav.tsx b/frontend/components/Nav.tsx index 34a14da23..29edf8381 100644 --- a/frontend/components/Nav.tsx +++ b/frontend/components/Nav.tsx @@ -4,6 +4,7 @@ import { NavOverflow } from "./NavOverflow.tsx"; export interface NavProps { children?: ComponentChildren; + end?: ComponentChildren; noTopMargin?: boolean; } @@ -13,6 +14,7 @@ export function Nav(props: NavProps) { class={`${ props.noTopMargin ? "" : "mt-3" } border-b border-jsr-cyan-300/30 dark:border-jsr-cyan-600/50 max-w-full flex justify-between overflow-x-auto items-end`} + id="nav-items" >