Fix robots.txt (#239)

* Fix Robots.txt to be automatically generated for Production
 generation of robots.txt.

Validate on staging site and fix staging yaml code to work correctly with GitHub pages

Update GitHub Action replacing PeaceIris actions with vanilla GitHub actions; align with Hugo's documented deployment process
* Update Readme
Add section on Staging

---------

Signed-off-by: Bill Stumbo <[email protected]>

Author: stumbo

.github/workflows/gh-pages.yml

@@ -25,13 +25,26 @@ on:
   push:
     branches:
       - main  # Set a branch to deploy
+
   schedule:
     - cron: "0 3 * * *"
 
+  workflow_dispatch:
+
+permissions:
+    contents: read
+    pages: write
+    id-token: write
+
+defaults:
+    run:
+        shell: bash
+
 env:
   # ----------------------------------------------------------------------------
   # Specify the deployment environment:  staging or production
-  hugoEnvironment: production
+  HUGO_ENVIRONMENT: production
+  HUGO_VERSION: 0.133.1
 
 jobs:
   # ----------------------------------------------------------------------------
@@ -42,6 +55,7 @@ jobs:
     outputs:
       zoteroVersion: ${{ fromJson(steps.zoteroVersion.outputs.headers).last-modified-version }}
       cacheHit: ${{ steps.cache-zotero-bib.outputs.cache-hit }}
+
     runs-on: ubuntu-latest
     concurrency:
       group:  ${{ github.workflow }}-${{ github.ref }}
@@ -66,15 +80,18 @@ jobs:
             ${{ fromJson(steps.zoteroVersion.outputs.headers).last-modified-version }}
 
   # ----------------------------------------------------------------------------
-  # Deploy the website.  This job is conditional, we will always run it on a
+  # Build the website.  This job is conditional, we will always run it on a
   # push or if on a scheduled run the cache was determined to be out of date.
   #      
-  deploy:
+  build:
     needs: check
     runs-on: ubuntu-latest
     if:  github.event_name == 'push' || needs.check.outputs.cacheHit != 'true'
     steps:
       - uses: actions/checkout@v4
+        with:
+          submodules: recursive
+          fetch-depth: 0
 
       - name: Cache Zotero Bibliography
         id: cache-zotero-bib
@@ -102,13 +119,17 @@ jobs:
             ./update_bibliography.sh
             sudo cp --recursive ${GITHUB_WORKSPACE}/static/data ~/data
           fi 
-        shell: bash
 
-      - name: Setup Hugo
-        uses: peaceiris/actions-hugo@v3
-        with:
-          hugo-version: '0.127.0'
-          extended: true
+      # Install Hugo Extended
+      #    
+      - name: Install Hugo CLI
+        run: |
+          wget -O ${{ runner.temp }}/hugo.deb https://github.com/gohugoio/hugo/releases/download/v${HUGO_VERSION}/hugo_extended_${HUGO_VERSION}_linux-amd64.deb \
+          && sudo dpkg -i ${{ runner.temp }}/hugo.deb 
+
+      - name: Setup Pages
+        id: pages
+        uses: actions/configure-pages@v5
 
       - name: Setup Node
         uses: actions/setup-node@v4
@@ -120,11 +141,25 @@ jobs:
       - run: npm install --verbose
 
       - name: Build
-        run: hugo -e $hugoEnvironment
-
-      - name: Deploy
-        uses: peaceiris/actions-gh-pages@v4
-        if: github.ref == 'refs/heads/main'
+        env:
+          HUGO_CACHEDIR: ${{ runner.temp }}/hugo_cache
+          TZ: America/New York
+        run: hugo --cleanDestinationDir -e $HUGO_ENVIRONMENT
+        
+      - name: Upload artifact
+        uses: actions/upload-pages-artifact@v3
         with:
-          github_token: ${{ secrets.GITHUB_TOKEN }}
-          publish_dir: ./public
+          path: ./public
+
+  deploy:
+    environment:
+      name: github-pages
+      url: ${{ steps.deployment.outputs.page_url }}
+    runs-on: ubuntu-latest
+    needs: build
+
+    steps:
+      - name: Deploy to GitHub Pages
+        id: deployment
+        uses: actions/deploy-pages@v4
+      

README.md

@@ -86,35 +86,117 @@ reflection of the bibliographic material related to Medley and Interlisp.
 
 Building the website is driven by a GitHub workflow.  
 
-The workflow is trigger by one of two events, a `push` to main, representing updates
+The workflow is triggered by one of two events, a `push` to main, representing updates
 to the Interlisp.org website or a scheduled execution of the workflow.  The
 workflow is scheduled to run on a regular basis to ensure the bibliography remains
 consistent with the online Zotero catalog.
 
-The workflow consists of two jobs.  The first job, `check`, uses Zotero's REST
-interface to query for the latest version of the group bibliography.  The call
-made is a `GET` call to `https://api.zotero.org/groups/2914042/items`.  This call
+The GitHub Action workflow can also be initiated from the Action panel within
+the Interlisp.github.io repository.  This option allows manual execution when
+necessary.
+
+The workflow consists of three jobs.  The first job, `check`, uses Zotero's REST
+interface to query for the latest version of the group bibliography.  A `GET` call
+is made to `https://api.zotero.org/groups/2914042/items`.  It
 returns a collection of metadata and information describing the current state of
 items within the catalog.  We are interested in a specific header, `Last-Modified-Version`.
 The value returned with this header is incremented every time the Zotero Interlisp
-catalog is updated.  The value returned is used as a cache-key for a cached
-version of the json file of the bibliography we create.  The first job completes
-by providing the current Zotero version and whether the cache needs to be updated.
+catalog is updated.  We use the value returned as a cache-key for the bibliography.
+If the cache-key matches one in the current GitHub Action cache we use the saved
+bibliography information and save the overhead of building it.
 
-The second job, `deploy`, starts by determining if a deploy needs to occur.  If
-the workflow was initiated by a `push` a deploy will always be done.  However,
-if the workflow was started by a scheduled execution, if the Zotero bibliography
-cache is consistent with the online Zotero catalog, the deploy is skipped.
+The second job, `build`, starts by determining if a build and deploy need to occur.
+If the workflow was initiated by a `push` a deploy will always be done.  However,
+if the workflow was started by a scheduled execution and the Zotero bibliography
+cache is consistent with the online Zotero catalog, the build and deploy are skipped.
 
-A deploy starts by checking out the `Interlisp.github.io` repository.  Then,
-if the cache is valid, the contents are copied into the `data` file within the
+A build starts by checking out the `Interlisp.github.io` repository.  Then,
+if the Zotereo cache is valid, its contents are copied into the `data` file within the
 repository directory structure.  If the cache is invalid, the `update_bibliography.sh`
-shell script is run to download a new copy of the bibliography as a `json` file.
+shell script runs and downloads a new copy of the bibliography as a `json` file.
 Once downloaded the script does some additional processing to complete the
 formatting of the file.
 
-Once this work is completed, Hugo is setup and the website is deployed.
+After this work is completed, Hugo is setup and run to build the website. We use
+Hugo extended to build our site. The version of Hugo currently being used is
+defined by the environment variable, `HUGO_VERSION`.
+
+We run Hugo with two flags:
+
+- `-e $HUGO_ENVIRONMENT` to specify whether we are building a production or staging site.  If the website is being build to deploy to Interlisp.org,it should be built with `HUGO_ENVIRONMENT` set to production.  Deployment to any other site should set the environment flag to staging.
+- `--cleanDestinationDir` clears the destination directory, `./public` on each build.  This will ensure we do not have any unneeded artifacts in our deployment.
+
+The last part of the build activity is to save the created artifact, the information
+in the `./public` directory.  We use the GitHub composition action `upload-pages-artifact`
+for this.  It packages the contents of the directory and stores it in the appropriate
+format for deployment to GitHub pages.
+
+The last job in the tool chain is `deploy`.  This job simply takes the output
+of the build step and formally deploys it to GitHub pages using the GitHub `deploy-pages`
+action.
+
+### Deploying a Staging Site
+
+Successfully deploying a Staging Site requires you to configure your GitHub
+repository to enable GitHub Pages.  The following steps will accomplish this task:
+
+1. Clone the Interlisp.github.io repository into your GitHub site
+2. In GitHub go to the cloned repository, in my case https://github.com/stumbo/InterlispDraft.github.io and select Settings
+3. Under Settings, find Pages and select it
+4. Under **Build and deployment** set Source to Deploy *GitHub Actions*
+
+Once the repository is cloned and GitHub Pages has been setup, you can deploy a
+staging site to validate changes prior to creating a Pull Request to merge your
+changes back into the main site.
+
+When creating a staging site we want to do a couple things to ensure we do not
+interfere with the production site, first we want to disable Google Analytics
+and secondly we want to ensure the site is not crawled and indexed.
+
+#### Setup Your Repository
+
+A best practice for the updating your clone of the repository is to create a branch
+and make the following required changes on the branch you created.
+
+The appropriate settings for this are all enabled by setting the `HUGO_ENVIRONMENT`
+variable in `.github/workflows/gh-pages.yaml` to *staging*.
+
+You also need to set `baseURL` to match the GitHub site you are deploying to
+in the `config/staging/hugo.yaml` file.  The file currently looks like:
+
+```yaml
+baseURL: https://stumbo.github.io/InterlispDraft.github.io/
+
+languageCode: en-us
+
+# title
+#  Insert Staging Environment onto every page to make clear
+#  this is not the production site
+title: 'Staging Environment'
+```
+
+Make sure the `baseURL` reflects the complete path of your repository.  Failure
+to do this will either cause the deployment to fail or URLs within your built
+site may be incorrectly set.  Resulting in 404s or expected resources not found.
+
+With these changes the cloned repository is ready to be deployed to a staging site.
+
+Commit the changes you made and push the new branch to your cloned repository.
+At this point, create a Pull Request to merge the changes you made into your
+repository's main branch.  Complete the operation by merging the pull request.
+
+Once the merge occurs, the GitHub Actions should fire off and your site will be
+built and deployed.
+
+Once you have successfully completed this operation and your staging site is
+deployed and operational you can experiment with adding new content or
+functionality to the Interlisp site.
+
+#### Develop a Feature
 
+To develop new pages or functionality, create a new branch for your work.  once
+you have completed development and testing on your staging site, you can create
+a PR to merge the content into the Interlisp site.
 
 ### Running Hugo and Docsy Locally
 
@@ -233,7 +315,7 @@ that have components specific to `Interlisp.github.io` are as follows:
   - `documentation` - contains the pdf files referenced in the document section of the home page
   - `favicons` - contains `favicon.png` a small icon that browsers can use when referencing the website
   - `Resources` - contains the current `Interlisp-D` logo, used on the home page, and another instance of `favicon.png`
-  - `CNAME` - a oneline text file that provides support for using a [custom domain](https://gohugo.io/hosting-and-deployment/hosting-on-github/#use-a-custom-domain)
+  - `CNAME` - a one line text file that provides support for using a [custom domain](https://gohugo.io/hosting-and-deployment/hosting-on-github/#use-a-custom-domain)
 
 ## Search
 
@@ -256,7 +338,7 @@ layout as being `search`.
 
 ### Updating Search
 
-Modfying the websites that are searched requires updating the Google Custom
+Modifying the websites that are searched requires updating the Google Custom
 Search engine settings.  This is done via logging into Google's Programmable Search
 Engine Dashboard at:  [https://programmablesearchengine.google.com](https://programmablesearchengine.google.com)
 

config/_default/hugo.yaml

@@ -17,7 +17,7 @@ canonifyURLs: false
 # Default to false and set to true for production in config/production/hugo.yaml
 enableRobotsTXT: false
 
-# aasetDir:  Location where Hugo looks for assets
+# assetDir:  Location where Hugo looks for assets
 assetDir: static
 
 # Enable .GitInfo object for each page.  This will give values to .Lastmod etc.

config/_default/markup.yaml

@@ -14,7 +14,7 @@ goldmark:
 # See:  https://gohugo.io/getting-started/configuration-markup/#highlight
 #       for additional information on configuration options.
 #   https://www.docsy.dev/docs/adding-content/lookandfeel/#code-highlighting-with-chroma
-#       contains Docsy specfiic code highlighting information    
+#       contains Docsy specfic code highlighting information    
 highlight:
   # See a complete list of available styles at https://xyproto.github.io/splash/docs/all.html
   #style: solarized-dark

config/_default/params.yaml

@@ -53,7 +53,7 @@ github_branch: main
 github_subdir: 
 github_project_repo: https://github.com/interlisp/medley
 
-# Google custom seach engine configuration
+# Google custom search engine configuration
 #  gcs_engine_id:  search engine 
 gcs_engine_id: 33ef4cbe0703b4f3a
 

config/_default/privacy.yaml

@@ -5,7 +5,7 @@
 # See:  https://gohugo.io/about/hugo-and-gdpr/
 #
 # googleAnalytics:
-#  ananymizIP:  Enable anonymiation of IP addresses
+#  anonymizeIP:  Enable anonymization of IP addresses
 #  disable:  Set to true to disable googleAnalytics
 #  respectDoNotTrack:  Check for Do Not Track in headers
 #  useSessionStorage:  Store session information in storage and

config/staging/hugo.yaml

@@ -1,4 +1,8 @@
-baseURL: https://wasm.interlisp.org
+baseURL: https://stumbo.github.io/InterlispDraft.github.io/
+
 languageCode: en-us
+
+# title
+#  Insert Staging Environment onto every page to make clear
+#  this is not the production site
 title: 'Staging Environment'
-publishDir: stage

layouts/robots.txt

@@ -0,0 +1,4 @@
+# Production robots.txt
+#
+User-agent: *
+Allow: /

static/robots.txt

@@ -1,2 +1,5 @@
+# Placeholder file.  Overwritten in production by setting enableRobotsTXT to true
+# See config/production/hugo.yaml
+#
 User-agent: *
-Allow: /
+Disallow: /