Updating to latest template

Author: gvwilson

.config.yml

@@ -4,7 +4,7 @@
 collections:
   en:
     output: true
-    permalink: /:collection/:title/
+    permalink: /:collection/:title.html
 
 # Specify the use of the "lesson" template for elements of the "en" collection.
 defaults:
@@ -42,5 +42,7 @@ exclude:
 - misc
 - node_modules
 - requirements.txt
+- site.mk
+- site.yml
 - src
 - tex

.gitignore

@@ -9,12 +9,22 @@
 *.pyg
 *.toc
 *~
-.cache
+.DS_Store
+.Rhistory
+.Rproj.user
+.cache/
+.coverage
 .jekyll-metadata
+.pytest_cache
+.~lock.*
 __pycache__
 _minted-*
 _site
 dist
-node_modules
+en_rmd/figures
+htmlcov
+node_modules/
 package-lock.json
+project/results/*.*
+tex/*/*.pdf
 tex/*/all.tex

Makefile

@@ -4,9 +4,16 @@ $(warning Please set 'lang' with 'lang=en' or similar.)
 lang=en
 endif
 
+# Controls
+all : commands
+
 # Pick up project-specific setting for STEM.
+# (Must come after definition of 'all' to avoid confusion about default target.)
 include site.mk
 
+# Overall configuration file.
+CONFIG_YML=_config.yml
+
 # Tools.
 JEKYLL=jekyll
 PANDOC=pandoc
@@ -15,21 +22,20 @@ BIBTEX=bibtex
 PYTHON=python
 
 # Language-dependent settings.
-CONFIG_YML=_config.yml
 DIR_MD=_${lang}
+DIR_RMD=${lang}_rmd
+RMD_SRC=$(wildcard ${DIR_RMD}/*.Rmd)
+RMD_DST=$(patsubst ${DIR_RMD}/%.Rmd,${DIR_MD}/%.md,${RMD_SRC})
 PAGES_MD=$(wildcard ${DIR_MD}/*.md)
 BIB_MD=${DIR_MD}/bib.md
 TOC_JSON=_data/${lang}_toc.json
 DIR_HTML=_site/${lang}
-PAGES_HTML=${DIR_HTML}/index.html $(patsubst ${DIR_MD}/%.md,${DIR_HTML}/%/index.html,$(filter-out ${DIR_MD}/index.md,${PAGES_MD}))
+PAGES_HTML=$(patsubst ${DIR_MD}/%.md,${DIR_HTML}/%.html,${PAGES_MD})
 DIR_TEX=tex/${lang}
 BIB_TEX=${DIR_TEX}/book.bib
 ALL_TEX=${DIR_TEX}/all.tex
 BOOK_PDF=${DIR_TEX}/${STEM}.pdf
 
-# Controls
-all : commands
-
 ## commands       : show all commands.
 commands :
 	@grep -h -E '^##' ${MAKEFILE_LIST} | sed -e 's/## //g'
@@ -54,14 +60,18 @@ config : ${CONFIG_YML}
 ## toc            : regenerate the table of contents JSON file.
 toc : ${TOC_JSON}
 
+## rmarkdown      : rebuild Markdown source from R Markdown.
+rmarkdown : ${RMD_DST}
+
 # ----------------------------------------
 
 # Regenerate PDF once 'all.tex' has been created.
-${BOOK_PDF} : ${ALL_TEX} tex/settings.tex ${DIR_TEX}/book.tex ${BIB_TEX}
+${BOOK_PDF} : ${ALL_TEX} tex/settings.tex ${DIR_TEX}/book.tex ${DIR_TEX}/site.tex ${BIB_TEX}
 	cd ${DIR_TEX} \
 	&& ${LATEX} --shell-escape -jobname=${STEM} book \
 	&& ${BIBTEX} ${STEM} \
 	&& ${LATEX} --shell-escape -jobname=${STEM} book \
+	&& ${LATEX} --shell-escape -jobname=${STEM} book \
 	&& ${LATEX} --shell-escape -jobname=${STEM} book
 
 # Create the unified LaTeX file (separate target to simplify testing).
@@ -83,6 +93,14 @@ test-pandoc:
 	| ${PYTHON} bin/transform.py --pre ${lang} _includes \
 	| ${PANDOC} --wrap=preserve -f html -t latex -o -
 
+# Build Markdown from R Markdown.
+${DIR_MD}/%.md : ${DIR_RMD}/%.Rmd
+	@bin/build.R $< $@
+	@if [ -d ${DIR_RMD}/figures/$$(basename $< .Rmd) ]; \
+	then \
+	  cp ${DIR_RMD}/figures/$$(basename $< .Rmd)/* ./figures/$$(basename $< .Rmd); \
+	fi
+
 # Create all the HTML pages once the Markdown files are up to date.
 ${PAGES_HTML} : ${PAGES_MD} ${BIB_MD} ${CONFIG_YML} ${TOC_JSON}
 	${JEKYLL} build
@@ -176,22 +194,28 @@ spelling :
 undone :
 	@grep -l 'undone: true' _en/*.md | cat
 
+## words          : count words
+words :
+	@for i in _en/*.md; do printf '%5d %s\n' $$(bin/uncode.py $$i | wc -w) $$i; done
+
 ## ----------------------------------------
 
 ## clean          : clean up junk files.
 clean :
-	@rm -r -f _config.yml _site dist
+	@rm -r -f _site dist
 	@find . -name '*~' -delete
 	@find . -name __pycache__ -prune -exec rm -r "{}" \;
 	@find . -name '_minted-*' -prune -exec rm -r "{}" \;
-	@rm -r -f tex/*/all.tex tex/*/*.aux tex/*/*.bbl tex/*/*.blg tex/*/*.log tex/*/*.out tex/*/*.toc
+	@rm -f tex/*/all.tex tex/*/*.{aux,bbl,blg,lof,log,lot,out,toc}
 	@find . -name .DS_Store -prune -exec rm -r "{}" \;
 
 ## settings       : show macro values.
 settings :
 	@echo "CONFIG_YML=${CONFIG_YML}"
 	@echo "JEKYLL=${JEKYLL}"
 	@echo "DIR_MD=${DIR_MD}"
+	@echo "RMD_SRC=${RMD_SRC}"
+	@echo "RMD_DST=${RMD_DST}"
 	@echo "PAGES_MD=${PAGES_MD}"
 	@echo "BIB_MD=${BIB_MD}"
 	@echo "DIR_HTML=${DIR_HTML}"

_config.yml

@@ -54,7 +54,7 @@ datasource: "https://figshare.com/articles/Portal_Project_Teaching_Database/1314
 collections:
   en:
     output: true
-    permalink: /:collection/:title/
+    permalink: /:collection/:title.html
 
 # Specify the use of the "lesson" template for elements of the "en" collection.
 defaults:
@@ -92,5 +92,7 @@ exclude:
 - misc
 - node_modules
 - requirements.txt
+- site.mk
+- site.yml
 - src
 - tex

_en/bib.md

@@ -16,7 +16,7 @@ title: "Bibliography"
 
 **Meek2017**{:#b:Meek2017}: Elijah Meeks: *D3.js in Action*. Manning, 2017, <https://www.manning.com/books/d3js-in-action-second-edition>. *A comprehensive guide to the D3 visualization framework.*
 
-**Mori2012**{:#b:Mori2012}: Andrew Morin and Jennifer Urban AND Piotr Sliz: "A Quick Guide to Software Licensing for the Scientist-Programmer". *PLoS Computational Biology*, 8(7), Jul 2012, <https://doi.org/10.1371/journal.pcbi.1002598>. *A short introduction to software licensing for non-specialists.*
+**Mori2012**{:#b:Mori2012}: Andrew Morin, Jennifer Urban, and Piotr Sliz: "A Quick Guide to Software Licensing for the Scientist-Programmer". *PLoS Computational Biology*, 8(7), Jul 2012, <https://doi.org/10.1371/journal.pcbi.1002598>. *A short introduction to software licensing for non-specialists.*
 
 **Wils2018**{:#b:Wils2018}: Greg Wilson: *Teaching Tech Together*. Lulu.com, 2018, <http://teachtogether.tech>. *How to create and deliver lessons that work and build a teaching community around them.*
 

_en/intro.md

@@ -81,7 +81,7 @@ since it isolates projects from one another.
 ## Who We Are {#s:intro-contributors}
 
 <!-- == noindent -->
-<img src="../../figures/hodges-toby.png" alt="Toby Hodges" width="100" />
+<img src="../figures/hodges-toby.png" alt="Toby Hodges" width="100" />
 
 <!-- == noindent -->
 **[Toby Hodges][hodges-toby]** is a bioinformatician turned community
@@ -93,7 +93,7 @@ to thank his wife for her support and patience while he swore about how annoying
 JavaScript is to debug.
 
 <!-- == noindent -->
-<img src="../../figures/wilson-greg.png" alt="Greg Wilson" width="100" />
+<img src="../figures/wilson-greg.png" alt="Greg Wilson" width="100" />
 
 <!-- == noindent -->
 **[Greg Wilson][wilson-greg]** has worked for 35 years in both industry and

_includes/figure.html

@@ -1 +1,2 @@
-<figure id="{{include.id}}"> <img src="{{include.src}}" /> <figcaption>{{include.caption}}</figcaption> </figure>
+{%- assign figure_path = '/figures/' | append: page.slug | append: '/' | append: include.src -%}
+<figure id="{{include.id}}"> <img src="{{figure_path|relative_url}}" /> <figcaption>{{include.caption}}</figcaption> </figure>

_includes/toc-bib.html

@@ -4,6 +4,6 @@
 
 <p>
   {% if page.slug == "bib" %}<strong>{% endif %}
-    <a href="{{'/'|append: include.language|append: '/bib/'|relative_url}}" class="toc">Bibliography</a>
+    <a href="{{'/'|append: include.language|append: '/bib.html'|relative_url}}" class="toc">Bibliography</a>
   {% if page.slug == "bib" %}</strong>{% endif %}
 </p>

_includes/toc-section.html

@@ -16,7 +16,7 @@
       {% comment %}If there's a match, add a link to the generated ToC.{% endcomment %}
       {%- if lesson == chap.slug -%}
         <li>{% if page.slug == chap.slug %}<strong>{% endif %}
-          <a href="{{'/'|append: include.language|append: '/'|append: chap.slug|append: '/'|relative_url}}" class="toc">
+          <a href="{{'/'|append: include.language|append: '/'|append: chap.slug|append: '.html'|relative_url}}" class="toc">
             {% if chap.undone %}<em>{% endif %}{{chap.title}}{% if chap.undone %} &#9734;</em>{% endif %}
           </a>
           {% if page.slug == chap.slug %}</strong>{% endif %}

assets/site.css

@@ -138,6 +138,10 @@ figcaption {
     font-weight: bold;
 }
 
+figure img {
+    max-width: 90%;
+}
+
 /* Mark quotations. */
 blockquote {
     border-radius: 10px;

assets/site.js

@@ -23,8 +23,7 @@ const fixTables = () => {
 
 // Fix glossary reference URLs.
 const fixGlossRefs = () => {
-  const pageIsRoot = document.currentScript.getAttribute('ROOT') != ''
-  const bibStem = pageIsRoot ? './gloss/' : '../gloss/'
+  const bibStem = './gloss.html'
   Array.from(document.querySelectorAll('a'))
     .filter(e => e.getAttribute('href').startsWith('#g:'))
     .forEach(e => {
@@ -35,8 +34,7 @@ const fixGlossRefs = () => {
 
 // Convert bibliography citation links.
 const fixBibRefs = () => {
-  const pageIsRoot = document.currentScript.getAttribute('ROOT') != ''
-  const bibStem = pageIsRoot ? './bib/#b:' : '../bib/#b:'
+  const bibStem = './bib.html#b:'
   Array.from(document.querySelectorAll('a'))
     .filter(e => e.getAttribute('href') == '#BIB')
     .forEach(e => {
@@ -52,7 +50,6 @@ const fixBibRefs = () => {
 
 // Fix cross-reference links.
 const fixCrossRefs = () => {
-  const prefix = document.currentScript.getAttribute('ROOT') != '' ? '.' : '..'
   const crossref = JSON.parse(document.currentScript.getAttribute('CROSSREF'))
   Array.from(document.querySelectorAll('a'))
     .filter(e => {
@@ -62,7 +59,7 @@ const fixCrossRefs = () => {
     .forEach(e => {
       const key = e.textContent
       const entry = crossref[key]
-      const link = prefix + '/' + entry.slug + '/#' + key
+      const link = './' + entry.slug + '.html#' + key
       const text = entry.text + ' ' + entry.value
       e.setAttribute('href', link)
       e.innerHTML = text
@@ -71,7 +68,6 @@ const fixCrossRefs = () => {
 
 // Fix figure captions.
 const fixFigureCaptions = () => {
-  const prefix = document.currentScript.getAttribute('ROOT') != '' ? '.' : '..'
   const crossref = JSON.parse(document.currentScript.getAttribute('CROSSREF'))
   Array.from(document.querySelectorAll('figure'))
     .forEach(e => {
@@ -84,7 +80,6 @@ const fixFigureCaptions = () => {
 
 // Fix table captions.
 const fixTableCaptions = () => {
-  const prefix = document.currentScript.getAttribute('ROOT') != '' ? '.' : '..'
   const crossref = JSON.parse(document.currentScript.getAttribute('CROSSREF'))
   Array.from(document.querySelectorAll('table'))
     .filter(e => e.hasAttribute('title'))

bin/build.R

@@ -0,0 +1,19 @@
+#!/usr/bin/env Rscript
+
+library(stringr)
+library(purrr)
+library(knitr)
+
+USAGE = "Knit an RMarkdown files to create a Markdown file.
+
+Usage: build.R source dest"
+
+args <- commandArgs(trailingOnly = TRUE)
+if (((length(args) == 1) && (args[1] %in% c("-h", "--help"))) || (length(args) != 2)) {
+  message(USAGE)
+  quit(status = 0)
+}
+src = normalizePath(args[1])
+dst = normalizePath(args[2])
+setwd(dirname(src))
+knit(src, output = dst)

bin/check.py

@@ -130,6 +130,7 @@ def check_figures(language):
     '''
     def _ignore(filename):
         return filename.startswith('.') or \
+            filename.endswith('.gif') or \
             filename.endswith('.odg') or \
             filename.endswith('.pdf') or \
             filename.endswith('.xml')
@@ -139,8 +140,12 @@ def _redundant(filename, defined):
             filename.replace('.png', '.svg') in defined
 
     content = get_all_docs(language)
-    used = _match_lines(content, r'{%\s+include\s+figure.html[^%]+src=".+/figures/([^"]+)"')
-    defined = {f for f in os.listdir(FIGURE_DIR) if not _ignore(f)}
+    by_doc = _match_lines_by_doc(content, r'{%\s+include\s+figure.html[^%]+src="([^"]+)"')
+    used = set()
+    for slug in by_doc:
+        used |= {os.path.join(FIGURE_DIR, slug, filename) for filename in by_doc[slug]}
+    used |= _match_lines(content, r'^!\[.+\]\(\.\./(.+)\)')
+    defined = {f for f in glob.glob(os.path.join(FIGURE_DIR, '**/*.*')) if not _ignore(f)}
     defined -= {f for f in defined if _redundant(f, defined)}
     report('Figures', 'unused', defined - used)
     report('Figures', 'missing', used - defined)
@@ -224,7 +229,8 @@ def _unprefix(filename):
         return filename[prefix_len:]
 
     content = get_all_docs(language, remove_code_blocks=False)
-    referenced = match_body(content, r'{:\s+title="([^"]+)\s*"}')
+    referenced = match_body(content, r'{:\s+title="([^"]+)\s*"[^}]*}') | \
+        match_body(content, r'<!--\s+used="([^"]+)"\s+-->')
     actual = {_unprefix(filename)
               for filename in glob.iglob('{}/**/*.*'.format(SOURCE_DIR), recursive=True)
               if not _ignore_file(filename)}
@@ -278,15 +284,29 @@ def _match_lines(content, pattern, splitter=None):
     '''
     Find all matches in all lines, splitting and flattening if asked to do so.
     '''
-    pat = re.compile(pattern)
+    by_doc = _match_lines_by_doc(content, pattern, splitter=splitter)
     result = set()
+    for doc in by_doc:
+        result |= by_doc[doc]
+    return result
+
+
+def _match_lines_by_doc(content, pattern, splitter=None):
+    '''
+    Find all matches in all lines, splitting and flattening if asked to do so.
+    Return a dictionary of doc:match_set.
+    '''
+    pat = re.compile(pattern)
+    result = {}
     for (slug, filename, body, lines) in content:
+        temp = set()
         for line in lines:
-            result |= set(pat.findall(line))
-    if splitter is not None:
-        result = {individual
-                  for group in result
-                  for individual in group.split(splitter)}
+            temp |= set(pat.findall(line))
+        if splitter is not None:
+            temp = {individual
+                    for group in temp
+                    for individual in group.split(splitter)}
+        result[slug] = temp
     return result
 
 

bin/get_body.py

@@ -35,7 +35,7 @@ def get_all(source_dir):
 def make_filenames(source_dir, slugs):
     '''Turn slugs into filenames.'''
 
-    return [os.path.join(source_dir, s, 'index.html') for s in slugs]
+    return [os.path.join(source_dir, '{}.html'.format(s)) for s in slugs]
 
 
 #-------------------------------------------------------------------------------