Add /llms.txt to docs for LLM context (#3194)

Author: jeremygiberson

src/pages/[...llm_slug].ts

@@ -0,0 +1,62 @@
+import { getCollection } from 'astro:content';
+import type { APIRoute } from 'astro';
+
+export const llmsTxtSections = [
+  'start',
+  'concept',
+  'security',
+  'develop',
+  'distribute',
+  'learn',
+  'plugins',
+];
+
+export function groupDocsByPrefix(
+  prefixes: string[],
+  docs: Awaited<ReturnType<typeof getCollection<'docs'>>>
+) {
+  const grouped = new Map<string, typeof docs>();
+  prefixes.forEach((prefix) => {
+    grouped.set(
+      prefix,
+      docs.filter((doc) => doc.slug.startsWith(prefix))
+    );
+  });
+  // sort each group by slug
+  for (const [prefix, items] of grouped) {
+    items.sort((a, b) => a.slug.localeCompare(b.slug));
+  }
+
+  return grouped;
+}
+
+export function toLlmsTxtPath(slug: string): string {
+  return `${slug}/llms.txt`;
+}
+
+export const GET: APIRoute = async ({ params, request }) => {
+  const { llm_slug } = params;
+  const slug = llm_slug?.replace(/\/llms\.txt$/, '');
+  const docs = await getCollection('docs');
+  const doc = docs.find((doc) => doc.slug === slug);
+  if (!doc) {
+    return new Response('Not Found', { status: 404 });
+  }
+
+  const content = `# ${doc.data.title}\n\n${doc.body}\n\n`;
+
+  return new Response(content, {
+    headers: { 'Content-Type': 'text/plain; charset=utf-8' },
+  });
+};
+
+export async function getStaticPaths() {
+  const docs = await getCollection('docs');
+  const docsBySection = groupDocsByPrefix(llmsTxtSections, docs);
+  const paths = Array.from(docsBySection.values())
+    .map((docs) => docs.map((doc) => doc.slug))
+    .flat()
+    .map((slug) => ({ params: { llm_slug: toLlmsTxtPath(slug) } }));
+
+  return [...paths];
+}

src/pages/llms.toc.txt.ts

@@ -0,0 +1,55 @@
+import { getCollection } from 'astro:content';
+import type { APIRoute } from 'astro';
+
+function getHeaderLevel(slug: string): number {
+  return slug.split('/').length + 1;
+}
+function groupDocsByPrefix(docs: Awaited<ReturnType<typeof getCollection<'docs'>>>) {
+  const prefixes = ['start', 'concept', 'security', 'develop', 'distribute', 'learn', 'plugins'];
+
+  const grouped = new Map<string, typeof docs>();
+  prefixes.forEach((prefix) => {
+    grouped.set(
+      prefix,
+      docs.filter((doc) => doc.slug.startsWith(prefix))
+    );
+  });
+
+  return grouped;
+}
+
+const aboutBlurb = `Tauri is a framework for building tiny, fast binaries for all major desktop and mobile platforms. Developers can integrate any frontend framework that compiles to HTML, JavaScript, and CSS for building their user experience while leveraging languages such as Rust, Swift, and Kotlin for backend logic when needed.`;
+
+const organizationBlur = `This index links to documentation that covers everything from getting started to advanced concepts, and distribution of Tauri applications.
+
+The index is organized into key sections:
+- **start**: Information for getting up and running with Tauri, including prerequisites and installation instructions
+- **core concepts**: Topics that you should get more intimately familiar with if you want to get the most out of the framework.
+- **security**: High-level concepts and security features at the core of Tauri’s design and ecosystem that make you, your applications and your users more secure by default
+- **develop**: Topics pertaining to the development of Tauri applications, including how to use the Tauri API, communicating between the frontend and backend, configuration, state management, debugging and more.
+- **distribute**: Information on the tooling you need to distribute your application either to the platform app stores or as platform-specific installers.
+- **learn**: Tutorials intended to provided end-to-end learning experiences to guide you through specific Tauri topics and help you apply knowledge from the guides and reference documentation.
+- **plugins**: Information on the extensibility of Tauri from Built-in Tauri features and functionality to provided plugins and recipes built by the Tauri community
+- **about**: Various information about Tauri from governance, philosophy, and trademark guidelines.
+
+Each section contains links to detailed markdown files that provide comprehensive information about Tauri's features and how to use them effectively.`;
+
+export const GET: APIRoute = async ({ params, request }) => {
+  const docs = await getCollection('docs');
+  const grouped = groupDocsByPrefix(docs);
+  let content = `# Tauri app Full Documentation\n\n${aboutBlurb}\n\n${organizationBlur}\n\n**Table of Contents**\n`;
+  for (const [prefix, items] of grouped) {
+    if (items.length > 0) {
+      content += `\n## ${prefix.charAt(0).toUpperCase() + prefix.slice(1)}\n`;
+      items.forEach((doc) => {
+        const level = getHeaderLevel(doc.slug);
+        const indent = ' '.repeat(level - 2);
+        content += `${indent}- [${doc.data.title}](https://tauri.app/${doc.slug}/)\n`;
+      });
+    }
+  }
+
+  return new Response(content, {
+    headers: { 'Content-Type': 'text/plain; charset=utf-8' },
+  });
+};

src/pages/llms.txt.ts

@@ -0,0 +1,39 @@
+import { getCollection } from 'astro:content';
+import type { APIRoute } from 'astro';
+import { llmsTxtSections, groupDocsByPrefix } from './[...llm_slug]';
+
+const aboutBlurb = `Tauri is a framework for building tiny, fast binaries for all major desktop and mobile platforms. Developers can integrate any frontend framework that compiles to HTML, JavaScript, and CSS for building their user experience while leveraging languages such as Rust, Swift, and Kotlin for backend logic when needed.`;
+
+function getHeaderLevel(slug: string): number {
+  return slug.split('/').length + 1;
+}
+
+export const GET: APIRoute = async ({ params, request }) => {
+  const docs = await getCollection('docs');
+  const docsBySection = groupDocsByPrefix(llmsTxtSections, docs);
+  let content = `# Tauri app Full Documentation\n\n${aboutBlurb}\n\n**Table of Contents**\n`;
+
+  let n = 1;
+  for (const [prefix, items] of docsBySection) {
+    if (items.length > 0) {
+      content += `\n${n++}. ${prefix.charAt(0).toUpperCase() + prefix.slice(1)}\n`;
+      items.forEach((doc) => {
+        const level = getHeaderLevel(doc.slug);
+        const indent = ' '.repeat(level - 2);
+        content += `${indent}- [${doc.data.title}](#${doc.slug})\n`;
+      });
+    }
+  }
+  for (const [prefix, docs] of docsBySection) {
+    content += `\n# ${prefix.charAt(0).toUpperCase() + prefix.slice(1)}\n`;
+    content += docs
+      .map((doc) => {
+        return `# ${doc.data.title}\n\n${doc.body}\n\n`;
+      })
+      .join('');
+  }
+
+  return new Response(content, {
+    headers: { 'Content-Type': 'text/plain; charset=utf-8' },
+  });
+};