Merge pull request github#20709 from github/repo-sync

repo sync

Author: Octomerger

.github/workflows/sync-search-elasticsearch.yml

@@ -86,8 +86,8 @@ jobs:
         run: |
           mkdir /tmp/records
           npm run sync-search-indices -- \
-            -l ${{ matrix.language }} \
-            -o /tmp/records \
+            --language ${{ matrix.language }} \
+            --out-directory /tmp/records \
             --no-compression --no-lunr-index
 
           ls -lh /tmp/records

.github/workflows/sync-search-pr.yml

@@ -1,6 +1,7 @@
 name: Sync search - PR
 
-# **What it does**: Tries running the sync-search when relevant files change.
+# **What it does**: This does what `sync-sarch-elasticsearch.yml` does but
+#                   with a localhost Elasticsearch and only for English.
 # **Why we have it**: To test that the script works and the popular pages json is valid.
 # **Who does it impact**: Docs engineering.
 
@@ -21,10 +22,26 @@ concurrency:
   group: '${{ github.workflow }} @ ${{ github.event.pull_request.head.label || github.head_ref || github.ref }}'
   cancel-in-progress: true
 
+env:
+  # Yes, it's hardcoded but it makes all the steps look exactly the same
+  # as they do in `sync-search-elasticsearch.yml` where it uses
+  # that `${{ env.ELASTICSEARCH_URL }}`
+  ELASTICSEARCH_URL: http://localhost:9200
+
 jobs:
   lint:
-    runs-on: ubuntu-latest
+    runs-on: ${{ fromJSON('["ubuntu-latest", "ubuntu-20.04-xl"]')[github.repository == 'github/docs-internal'] }}
     steps:
+      - uses: getong/elasticsearch-action@95b501ab0c83dee0aac7c39b7cea3723bef14954
+        with:
+          # # Make sure this matches production and `test.yml`
+          elasticsearch version: '7.11.1'
+          host port: 9200
+          container port: 9200
+          host node port: 9300
+          node port: 9300
+          discovery type: 'single-node'
+
       - name: Check out repo
         uses: actions/checkout@dcd71f646680f2efd8db4afa5ad64fdcba30e748
 
@@ -46,9 +63,57 @@ jobs:
       - name: Build
         run: npm run build
 
-      - name: Run sync-search
+      - name: Start the server in the background
+        env:
+          ENABLE_DEV_LOGGING: false
+        run: |
+          npm run sync-search-server > /tmp/stdout.log 2> /tmp/stderr.log &
+
+          # first sleep to give it a chance to start
+          sleep 6
+          curl --retry-connrefused --retry 4 -I http://localhost:4002/
+
+      - if: ${{ failure() }}
+        name: Debug server outputs on errors
+        run: |
+          echo "____STDOUT____"
+          cat /tmp/stdout.log
+          echo "____STDERR____"
+          cat /tmp/stderr.log
+
+      - name: Scrape records into a temp directory
         env:
-          # Set filtered to only these so it doesn't run for too long.
-          LANGUAGE: en
-          VERSION: free-pro-team@latest
-        run: npm run sync-search
+          # If a reusable, or anything in the `data/*` directory is deleted
+          # you might get a
+          #
+          #   RenderError: Can't find the key 'site.data.reusables...' in the scope
+          #
+          # But that'll get fixed in the next translation pipeline. For now,
+          # let's just accept an empty string instead.
+          THROW_ON_EMPTY: false
+
+        run: |
+          mkdir /tmp/records
+          npm run sync-search-indices -- \
+            --language en \
+            --version dotcom \
+            --out-directory /tmp/records \
+            --no-compression --no-lunr-index
+
+          ls -lh /tmp/records
+
+      - name: Check that Elasticsearch is accessible
+        run: |
+          curl --fail --retry-connrefused --retry 5 -I ${{ env.ELASTICSEARCH_URL }}
+
+      - name: Index into Elasticsearch
+        run: |
+          ./script/search/index-elasticsearch.js \
+            --language en \
+            --version dotcom \
+            --source-directory /tmp/records
+
+      - name: Check created indexes and aliases
+        run: |
+          curl --fail --retry-connrefused --retry 5 ${{ env.ELASTICSEARCH_URL }}/_cat/indices?v
+          curl --fail --retry-connrefused --retry 5 ${{ env.ELASTICSEARCH_URL }}/_cat/indices?v

.github/workflows/test.yml

@@ -55,6 +55,7 @@ jobs:
         if: ${{ matrix.test-group == 'content' }}
         uses: getong/elasticsearch-action@95b501ab0c83dee0aac7c39b7cea3723bef14954
         with:
+          # Make sure this matches production and `sync-search-pr.yml`
           elasticsearch version: '7.11.1'
           host port: 9200
           container port: 9200

lib/hydro.js

@@ -5,7 +5,6 @@ import got from 'got'
 
 import statsd from '../lib/statsd.js'
 import FailBot from '../lib/failbot.js'
-
 const TIME_OUT_TEXT = 'ms has passed since batch creation'
 
 const MOCK_HYDRO_POST =

script/search/sync-search-indices.js

@@ -7,14 +7,26 @@
 //
 // [end-readme]
 
+import assert from 'assert'
 import path from 'path'
 
 import { program, Option } from 'commander'
 
 import { languageKeys } from '../../lib/languages.js'
-import { allVersionKeys } from '../../lib/all-versions.js'
+import { allVersions } from '../../lib/all-versions.js'
 import searchSync from './sync.js'
 
+const shortNames = Object.fromEntries(
+  Object.values(allVersions).map((info) => {
+    const shortName = info.hasNumberedReleases
+      ? info.miscBaseName + info.currentRelease
+      : info.miscBaseName
+    return [shortName, info]
+  })
+)
+
+const allVersionKeys = [...Object.keys(shortNames), ...Object.keys(allVersions)]
+
 const DEFAULT_OUT_DIRECTORY = path.join('lib', 'search', 'indexes')
 
 program
@@ -84,6 +96,25 @@ async function main(opts) {
     }
   }
 
+  // A `--version` or `process.env.VERSION` was specified, we need to convert
+  // it to the long name. I.e. `free-pro-team@latest`. Not `dotcom`.
+  // But it could also have beeb specified as `all` which means that `version`
+  // here ill be `undefined` which is also OK.
+  // const indexVersion = shortNames[version].hasNumberedReleases
+  //   ? shortNames[version].currentRelease
+  //   : shortNames[version].miscBaseName
+
+  let indexVersion
+  if (version && version !== 'all') {
+    // If it has been specified, it needs to be in the "long-form".
+    // I.e. `[email protected]` not `ghes-3.5`.
+    indexVersion = version in shortNames ? shortNames[version].version : version
+  }
+  assert(
+    !indexVersion || indexVersion in allVersions,
+    `version must be undefined or one of ${Object.keys(allVersions)}`
+  )
+
   let dryRun = false
   if ('dryRun' in opts) {
     dryRun = opts.dryRun
@@ -101,7 +132,7 @@ async function main(opts) {
     dryRun,
     language,
     notLanguage,
-    version,
+    version: indexVersion,
     outDirectory,
     compressFiles,
     generateLunrIndex,