Skip to content

Commit 2dbea37

Browse files
graceparkgracepark
authored
Removing algolia (github#20633)
1 parent 13caa0b commit 2dbea37

25 files changed

+121
-506
lines changed

‎.env.example

-2
Original file line numberDiff line numberDiff line change
@@ -1,3 +1 @@
1-
ALGOLIA_API_KEY=
2-
ALGOLIA_APPLICATION_ID=
31
ALLOW_TRANSLATION_COMMITS=

‎.github/workflows/browser-test.yml

+4
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,10 @@ jobs:
1919
# Even if if doesn't do anything
2020
- name: Checkout
2121
uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f
22+
with:
23+
lfs: true
24+
- name: Checkout LFS objects
25+
run: git lfs checkout
2226

2327
- name: Setup Node
2428
uses: actions/setup-node@38d90ce44d5275ad62cc48384b3d8a58c500bb5f

‎.github/workflows/dry-run-sync-search-indices.yml

-2
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,5 @@ jobs:
2626
run: npm run build
2727
- name: (Dry run) sync indices
2828
env:
29-
ALGOLIA_APPLICATION_ID: ${{ secrets.ALGOLIA_APPLICATION_ID }}
30-
ALGOLIA_API_KEY: ${{ secrets.ALGOLIA_API_KEY }}
3129
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
3230
run: npm run sync-search-dry-run

‎.github/workflows/sync-search-indices.yml

+67-10
Original file line numberDiff line numberDiff line change
@@ -1,38 +1,95 @@
11
name: Sync search indexes
22

3-
# **What it does**: This updates our search indexes after each deployment.
3+
# **What it does**: This workflow syncs the Lunr search indexes.
4+
# The search indexes are checked into the lib/search/indexes directory.
5+
# Search indexes are checked directly into the `main` branch on both the
6+
# internal and open-source docs repositories. This workflow should be the
7+
# only mechanism that the search indexes are modified. Because of that,
8+
# repo-sync will not sync the search indexes because it should not detect
9+
# a change.
410
# **Why we have it**: We want our search indexes kept up to date.
511
# **Who does it impact**: Anyone using search on docs.
612

13+
# **Testing: To test this workflow, use the workflow_dispatch event and trigger
14+
# the workflow from the action tab. Select the branch with the changes to the
15+
# workflow. Set `fetch-depth: 0` as an input to the checkout action to get all
16+
# branches, including your test branch. Otherwise, you'll only get the main
17+
# branch. For git lfs push and git push commands use the --dry-run switch to
18+
# prevent pushes (e.g., git push --dry-run origin main --no-verify and
19+
# git lfs push --dry-run public-docs-repo).
20+
# The dry-run switch does everything but actually send the updates.
21+
722
on:
823
workflow_dispatch:
9-
push:
10-
branches:
11-
- main
24+
schedule:
25+
- cron: '53 0/4 * * *' # Run every four hours at 53 minutes past the hour
1226

1327
jobs:
14-
updateIndices:
15-
name: Update indices
28+
updateIndexes:
29+
name: Update indexes
1630
if: github.repository == 'github/docs-internal'
1731
runs-on: ubuntu-latest
1832
steps:
33+
# Check out internal docs repository
1934
- name: checkout
2035
uses: actions/checkout@5a4ac9002d0be2fb38bd78e4b4dbde5606d7042f
36+
with:
37+
token: ${{ secrets.DOCS_BOT }}
38+
2139
- name: Setup Node
2240
uses: actions/setup-node@38d90ce44d5275ad62cc48384b3d8a58c500bb5f
2341
with:
2442
node-version: 16.x
2543
cache: npm
44+
2645
- name: Install dependencies
2746
run: npm ci
47+
2848
- name: Run build scripts
2949
run: npm run build
30-
- name: sync indices
50+
51+
- name: Update search indexes
3152
env:
32-
ALGOLIA_APPLICATION_ID: ${{ secrets.ALGOLIA_APPLICATION_ID }}
33-
ALGOLIA_API_KEY: ${{ secrets.ALGOLIA_API_KEY }}
3453
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
35-
run: npm run sync-search
54+
# TODO remove version and language after first successful run to test
55+
run: VERSION=github-ae@latest LANGUAGE=pt npm run sync-search
56+
57+
- name: Update private docs repository search indexes
58+
# Git pre-push hooks push the LFS objects, so if you don't run them and
59+
# don't push the LFS objects manually, the LFS objects won't get
60+
# pushed. That will likely result in the push getting rejected.
61+
# So if you don't use the pre-push hooks or you run with --no-verify
62+
# the LFS objects need to be pushed first.
63+
run: |
64+
echo 'git checkout main'
65+
git checkout main
66+
echo 'git config user.name "GitHub Actions"'
67+
git config user.name "GitHub Actions"
68+
echo 'git config user.email [email protected]'
69+
git config user.email [email protected]
70+
echo 'git commit -am "update search indexes"'
71+
git commit -am "update search indexes"
72+
echo 'git lfs push origin'
73+
git lfs push origin
74+
echo 'git push origin main --no-verify'
75+
git push origin main --no-verify
76+
77+
- name: Update open-source docs repository search indexes
78+
# Git pre-push hooks push the LFS objects, so if you don't run them and
79+
# don't push the LFS objects manually, the LFS objects won't get
80+
# pushed. That will likely result in the push getting rejected.
81+
# So if you don't use the pre-push hooks or you run with --no-verify
82+
# the LFS objects need to be pushed first.
83+
run: |
84+
echo 'git remote add public-docs-repo https://github.com/github/docs.git'
85+
git remote add public-docs-repo https://github.com/github/docs.git
86+
echo 'git lfs push public-docs-repo'
87+
git lfs push public-docs-repo
88+
echo 'git pull public-docs-repo main'
89+
git pull public-docs-repo main
90+
echo 'git push public-docs-repo main --no-verify'
91+
git push public-docs-repo main --no-verify
92+
3693
- name: Send slack notification if workflow run fails
3794
uses: someimportantcompany/github-actions-slack-message@0b470c14b39da4260ed9e3f9a4f1298a74ccdefd
3895
if: failure()

‎.github/workflows/sync-single-english-index.yml

+9-2
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ name: Sync Single English Index
55
# **Who does it impact**:
66

77
on:
8+
workflow_dispatch:
89
pull_request:
910
types:
1011
- labeled
@@ -39,9 +40,15 @@ jobs:
3940
env:
4041
VERSION: ${{ steps.getVersion.outputs.versionToSync }}
4142
LANGUAGE: 'en'
42-
ALGOLIA_APPLICATION_ID: ${{ secrets.ALGOLIA_APPLICATION_ID }}
43-
ALGOLIA_API_KEY: ${{ secrets.ALGOLIA_API_KEY }}
4443
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
4544
run: |
4645
npm run build
4746
npm run sync-search
47+
- name: Check in search index
48+
uses: EndBug/add-and-commit@2bdc0a61a03738a1d1bda24d566ad0dbe3083d87
49+
with:
50+
# The arguments for the `git add` command
51+
add: 'lib/search/indexes'
52+
53+
# The message for the commit
54+
message: 'Updated search index for ${{ steps.getVersion.outputs.versionToSync }}'

‎.github/workflows/triage-unallowed-contributions.yml

+3
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ on:
1616
- 'lib/redirects/**'
1717
- 'lib/rest/**'
1818
- 'lib/webhooks/**'
19+
- 'lib/search/indexes/**'
1920
- 'scripts/**'
2021
- 'translations/**'
2122
- 'package*.json'
@@ -76,6 +77,7 @@ jobs:
7677
- 'lib/redirects/**'
7778
- 'lib/rest/**'
7879
- 'lib/webhooks/**'
80+
- 'lib/search/indexes/**'
7981
- 'scripts/**'
8082
- 'translations/**'
8183
- 'package*.json'
@@ -101,6 +103,7 @@ jobs:
101103
'lib/redirects/**',
102104
'lib/rest/**',
103105
'lib/webhooks/**',
106+
'lib/search/indexes/**'
104107
'scripts/**',
105108
'translations/**',
106109
'package*.json',

‎.gitignore

-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,3 @@
1-
.algolia-cache
21
.search-cache
32
.DS_Store
43
.env

‎components/Search.tsx

+1-1
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,7 @@ export function Search({ isStandalone = false, updateSearchParams = true, childr
4141

4242
// If the user shows up with a query in the URL, go ahead and search for it
4343
useEffect(() => {
44-
if (router.query.query) {
44+
if (updateSearchParams && router.query.query) {
4545
/* await */ fetchSearchResults((router.query.query as string).trim())
4646
}
4747
}, [])

‎components/page-header/Header.tsx

+2-2
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ export const Header = () => {
4848
{/* <!-- GitHub.com homepage and 404 page has a stylized search; Enterprise homepages do not --> */}
4949
{relativePath !== 'index.md' && error !== '404' && (
5050
<div className="d-inline-block ml-4">
51-
<Search />
51+
{router.asPath !== '/graphql/overview/explorer' && <Search />}
5252
</div>
5353
)}
5454
</div>
@@ -112,7 +112,7 @@ export const Header = () => {
112112
{/* <!-- GitHub.com homepage and 404 page has a stylized search; Enterprise homepages do not --> */}
113113
{relativePath !== 'index.md' && error !== '404' && (
114114
<div className="pt-3 border-top">
115-
<Search />
115+
{router.asPath !== '/graphql/overview/explorer' && <Search />}
116116
</div>
117117
)}
118118
</div>

‎contributing/search.md

+9-7
Original file line numberDiff line numberDiff line change
@@ -37,17 +37,19 @@ Using the attribute `topics` in your query will only return results that have th
3737

3838
## Production deploys
3939

40-
A [GitHub Actions workflow](.github/workflows/sync-search-indices.yml) triggered by pushes to the `main` branch syncs the search data. This process generates structured data for all pages on the site, compares that data to what's currently on search, then adds, updates, or removes indices based on the diff of the local and remote data, being careful not to create duplicate records and avoiding any unnecessary (and costly) indexing operations.
40+
A [GitHub Actions workflow](.github/workflows/sync-search-indices.yml) that runs every four hours syncs the search data. This process generates structured data for all pages on the site, compares that data to what's currently on search, then adds, updates, or removes indices based on the diff of the local and remote data, being careful not to create duplicate records and avoiding any unnecessary (and costly) indexing operations.
4141

4242
The Actions workflow progress can be viewed (by GitHub employees) in the [Actions tab](https://github.com/github/docs/actions?query=workflow%3Asearch) of the repo.
4343

44-
Because the workflow runs after a branch is merged to `main`, there is a slight delay for search data updates to appear on the site.
44+
## Manually triggering the search index update workflow
4545

46-
## Manual sync from a checkout
46+
You can manually run the workflow to generate the indexes after you push your changes to `main` to speed up the indexing when needed. To run it manually, click "Run workflow" button in the [Actions tab](https://github.com/github/docs-internal/actions/workflows/sync-search-indices.yml).
4747

48-
It is also possible to manually sync the indices from your local checkout of the repo, before your branch is merged to `main`.
48+
## Generating search indexes for your local checkout
4949

50-
**Prerequisite:** Make sure the environment variables `ALGOLIA_APPLICATION_ID` and `ALGOLIA_API_KEY` are set in your `.env` file. You can find these values on [Algolia](https://www.algolia.com/apps/ZI5KPY1HBE/api-keys/all). _Remove this paragraph when we switch to Lunr._
50+
You can locally generate search indexes, but please do not check them into your local branch because they can get out-of-sync with the `main` branch quickly.
51+
52+
To locally generate the English version of the Dotcom search index locally, run `LANGUAGE=en VERSION=free-pro-team@latest npm run sync-search`. See [Build and sync](#build-and-sync) below for more details. To revert those files run `git checkout lib/search/indexes`.
5153

5254
### Build without sync (dry run)
5355

@@ -75,7 +77,7 @@ VERSION=<PLAN@RELEASE LANGUAGE=<TWO-LETTER CODE> npm run sync-search
7577
```
7678
You can set `VERSION` and `LANGUAGE` individually, too.
7779

78-
Substitute a currently supported version for `<PLAN@RELEASE>` and a currently supported two-letter language code for `<TWO-LETTER-CODE>`.
80+
Substitute a currently supported version for `<PLAN@RELEASE>` and a currently supported two-letter language code for `<TWO-LETTER-CODE>`. Languages and versions are lowercase. The options for version are currently `free-pro-team`, `github-ae`, and `enterprise-server`.
7981

8082
## Label-triggered Actions workflow
8183

@@ -95,7 +97,7 @@ Why do we need this? For our daily shipping needs, it's tolerable that search up
9597

9698
### Actions workflow files
9799

98-
- [`.github/workflows/sync-search-indices.yml`](.github/workflows/sync-search-indices.yml) - Builds and syncs search indices whenever the `main` branch is pushed to (that is, on production deploys).
100+
- [`.github/workflows/sync-search-indices.yml`](.github/workflows/sync-search-indices.yml) - Builds and syncs search indices on the `main` branch every four hours. Search indices are committed directly to the `main` branch on both the `github/docs-internal` and `github/docs` repositories. It can also be run manually. To run it manually, click "Run workflow" button in the [Actions tab](https://github.com/github/docs-internal/actions/workflows/sync-search-indices.yml).
99101
- [`.github/workflows/dry-run-sync-search-indices.yml`](.github/workflows/dry-run-sync-search-indices.yml) - This workflow can be run manually (via `workflow_dispatch`) to do a dry run build of all the indices. Useful for confirming that the indices can build without erroring out.
100102
- [`.github/workflows/sync-single-english-index.yml`](.github/workflows/sync-single-english-index.yml) - This workflow is run when a label in the right format is applied to a PR. See "[Label-triggered Actions workflow](#label-triggered-actions-workflow)" for details.
101103

‎lib/search/algolia-search.js

-31
This file was deleted.

‎middleware/csp.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ export default function csp(req, res, next) {
1212
const csp = {
1313
directives: {
1414
defaultSrc: ["'none'"],
15-
connectSrc: ["'self'", '*.algolia.net', '*.algolianet.com'],
15+
connectSrc: ["'self'"],
1616
fontSrc: ["'self'", 'data:', AZURE_STORAGE_URL],
1717
imgSrc: [
1818
"'self'",

‎middleware/search.js

+6-6
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ import express from 'express'
22
import libLanguages from '../lib/languages.js'
33
import searchVersions from '../lib/search/versions.js'
44
import loadLunrResults from '../lib/search/lunr-search.js'
5-
import loadAlgoliaResults from '../lib/search/algolia-search.js'
65
const languages = new Set(Object.keys(libLanguages))
76
const versions = new Set(Object.values(searchVersions))
87

@@ -24,11 +23,12 @@ router.get('/', async function postSearch(req, res, next) {
2423
}
2524

2625
try {
27-
const results =
28-
process.env.AIRGAP || req.cookies.AIRGAP
29-
? await loadLunrResults({ version, language, query: `${query} ${filters || ''}`, limit })
30-
: await loadAlgoliaResults({ version, language, query, filters, limit })
31-
26+
const results = await loadLunrResults({
27+
version,
28+
language,
29+
query: `${query} ${filters || ''}`,
30+
limit,
31+
})
3232
// Only reply if the headers have not been sent and the request was not aborted...
3333
if (!res.headersSent && !req.aborted) {
3434
return res.status(200).json(results)

0 commit comments

Comments
 (0)