feat(search index): add recent changelogs

apps/docs/scripts/fetch-discussion-categories.ts

@@ -0,0 +1,52 @@
+/* eslint-disable turbo/no-undeclared-env-vars */
+import { createAppAuth } from '@octokit/auth-app'
+import { Octokit } from '@octokit/core'
+import dotenv from 'dotenv'
+
+dotenv.config()
+
+async function fetchDiscussionCategories(owner: string, repo: string) {
+  const octokit = new Octokit({
+    authStrategy: createAppAuth,
+    auth: {
+      appId: process.env.SEARCH_GITHUB_APP_ID,
+      installationId: process.env.SEARCH_GITHUB_APP_INSTALLATION_ID,
+      privateKey: process.env.SEARCH_GITHUB_APP_PRIVATE_KEY,
+    },
+  })
+
+  const query = `
+    query fetchDiscussionCategories($owner: String!, $repo: String!) {
+      repository(owner: $owner, name: $repo) {
+        discussionCategories(first: 100) {
+          nodes {
+            id
+            name
+          }
+        }
+      }
+    }
+  `
+
+  const { repository } = await octokit.graphql<{
+    repository: { discussionCategories: { nodes: { id: string; name: string }[] } }
+  }>(query, { owner, repo })
+  return repository.discussionCategories.nodes
+}
+
+async function main() {
+  const owner = 'supabase'
+  const repo = 'supabase'
+
+  try {
+    const categories = await fetchDiscussionCategories(owner, repo)
+    console.log('Discussion Categories:')
+    categories.forEach((category) => {
+      console.log(`ID: ${category.id}, Name: ${category.name}`)
+    })
+  } catch (error) {
+    console.error('Error fetching discussion categories:', error)
+  }
+}
+
+main()

apps/docs/scripts/search/sources/github-discussion.ts

@@ -1,3 +1,4 @@
+/* eslint-disable turbo/no-undeclared-env-vars */
 import { createAppAuth } from '@octokit/auth-app'
 import { Octokit } from '@octokit/core'
 import { paginateGraphql } from '@octokit/plugin-paginate-graphql'
@@ -14,13 +15,24 @@ export type Discussion = {
   title: string
   body: string
   databaseId: number
+  createdAt: string
+  labels: {
+    nodes: {
+      name: string
+    }[]
+  }
+  ragIgnore?: boolean
 }
 
 export type DiscussionsResponse = {
   repository: {
     discussions: {
       totalCount: number
       nodes: Discussion[]
+      pageInfo: {
+        hasNextPage: boolean
+        endCursor: string
+      }
     }
   }
 }
@@ -74,6 +86,87 @@ export async function fetchDiscussions(owner: string, repo: string, categoryId:
   return discussions
 }
 
+export async function fetchDiscussionsSinceDate(
+  owner: string,
+  repo: string,
+  categoryId: string,
+  { batchSize = 100, orderBy, sinceDate }: { batchSize?: number; orderBy?: string; sinceDate: Date }
+) {
+  const octokit = new ExtendedOctokit({
+    authStrategy: createAppAuth,
+    auth: {
+      appId: process.env.SEARCH_GITHUB_APP_ID,
+      installationId: process.env.SEARCH_GITHUB_APP_INSTALLATION_ID,
+      privateKey: process.env.SEARCH_GITHUB_APP_PRIVATE_KEY,
+    },
+  })
+
+  let endCursor: string | undefined
+  let hasNextPage = true
+  let allDiscussions: Discussion[] = []
+
+  while (hasNextPage) {
+    const {
+      repository: {
+        discussions: { nodes: discussions, pageInfo },
+      },
+    } = await octokit.graphql<DiscussionsResponse>(
+      `
+        query troubleshootDiscussions($cursor: String, $owner: String!, $repo: String!, $categoryId: ID!, $batchSize: Int!, $orderBy: DiscussionOrderField!) {
+          repository(owner: $owner, name: $repo) {
+            discussions(first: $batchSize, after: $cursor, categoryId: $categoryId, orderBy: {field: $orderBy, direction: DESC}) {
+              totalCount
+              nodes {
+                id
+                updatedAt
+                url
+                title
+                body
+                databaseId
+                createdAt
+                labels(first: $batchSize) {
+                  nodes {
+                    name
+                  }
+                }
+              }
+              pageInfo {
+                hasNextPage
+                endCursor
+              }
+            }
+          }
+        }
+      `,
+      {
+        owner,
+        repo,
+        categoryId,
+        batchSize,
+        orderBy,
+        cursor: endCursor,
+      }
+    )
+
+    allDiscussions.push(...discussions)
+    hasNextPage = pageInfo.hasNextPage
+    endCursor = pageInfo.endCursor
+
+    if (discussions.length > 0) {
+      const lastDiscussionDate = new Date(discussions[discussions.length - 1].createdAt)
+      if (lastDiscussionDate < sinceDate) {
+        break
+      }
+    }
+  }
+
+  const discussions = allDiscussions.filter(
+    (discussion) => new Date(discussion.createdAt) >= sinceDate
+  )
+
+  return discussions
+}
+
 export class GitHubDiscussionLoader extends BaseLoader {
   type = 'github-discussions' as const
 
@@ -101,7 +194,7 @@ export class GitHubDiscussionSource extends BaseSource {
   }
 
   process() {
-    const { id, title, updatedAt, body, databaseId } = this.discussion
+    const { id, title, updatedAt, body, databaseId, ragIgnore } = this.discussion
 
     const checksum = createHash('sha256').update(updatedAt).digest('base64')
 
@@ -132,6 +225,7 @@ export class GitHubDiscussionSource extends BaseSource {
     return {
       checksum,
       meta,
+      ragIgnore,
       sections,
     }
   }

apps/docs/scripts/search/sources/index.ts

@@ -2,6 +2,7 @@ import {
   GitHubDiscussionLoader,
   type GitHubDiscussionSource,
   fetchDiscussions,
+  fetchDiscussionsSinceDate,
 } from './github-discussion'
 import { MarkdownLoader, type MarkdownSource } from './markdown'
 import { IntegrationLoader, type IntegrationSource, fetchPartners } from './partner-integrations'
@@ -110,6 +111,33 @@ export async function fetchSources() {
     )
   ).map((discussion) => new GitHubDiscussionLoader('supabase/supabase', discussion).load())
 
+  const oneYearAgo = new Date()
+  oneYearAgo.setFullYear(oneYearAgo.getFullYear() - 1)
+
+  const threeMonthsAgo = new Date()
+  threeMonthsAgo.setMonth(threeMonthsAgo.getMonth() - 3)
+
+  const githubChangeLogSources = (
+    await fetchDiscussionsSinceDate(
+      'supabase',
+      'supabase',
+      'DIC_kwDODMpXOc4CAFUr', // 'Changelog' category
+      {
+        orderBy: 'CREATED_AT',
+        sinceDate: oneYearAgo,
+      }
+    )
+  ).map((discussion) => {
+    if (
+      new Date(discussion.createdAt) < threeMonthsAgo &&
+      !discussion.labels.nodes.some((label) => label.name === 'persist-in-search')
+    ) {
+      discussion.ragIgnore = true
+    }
+
+    return new GitHubDiscussionLoader('supabase/supabase', discussion).load()
+  })
+
   const sources: SearchSource[] = (
     await Promise.all([
       openApiReferenceSource,
@@ -121,6 +149,7 @@ export async function fetchSources() {
       ktLibReferenceSource,
       cliReferenceSource,
       ...githubDiscussionSources,
+      ...githubChangeLogSources,
       ...partnerIntegrationSources,
       ...guideSources,
     ])