|
| 1 | +#!/bin/bash |
| 2 | +# |
| 3 | +# check-requirements.sh checks all requirements files for each top-level |
| 4 | +# convert*.py script. |
| 5 | +# |
| 6 | +# WARNING: This is quite IO intensive, because a fresh venv is set up for every |
| 7 | +# python script. As of 2023-12-22, this writes ~2.7GB of data. An adequately |
| 8 | +# sized tmpfs /tmp or ramdisk is recommended if running this frequently. |
| 9 | +# |
| 10 | +# usage: ./check-requirements.sh [<working_dir>] |
| 11 | +# ./check-requirements.sh 'nocleanup' [<working_dir>] |
| 12 | +# |
| 13 | +# where: |
| 14 | +# - <working_dir> is a directory that can be used as the base for |
| 15 | +# setting up the venvs. Defaults to `/tmp`. |
| 16 | +# - 'nocleanup' as the first argument will disable automatic cleanup |
| 17 | +# of the files created by this script. |
| 18 | +# |
| 19 | +# requires: |
| 20 | +# - bash >= 3.2.57 |
| 21 | +# - shellcheck |
| 22 | +# |
| 23 | +# For each script, it creates a fresh venv, `pip install -r` the |
| 24 | +# requirements, and finally executes the python script with no arguments to |
| 25 | +# check for a `ModuleNotFoundError`. |
| 26 | +# |
| 27 | + |
| 28 | +log() { |
| 29 | + local level="$1"; shift |
| 30 | + local format="$1"; shift |
| 31 | + # shellcheck disable=SC2059 |
| 32 | + >&2 printf "$level: $format\n" "$@" |
| 33 | +} |
| 34 | + |
| 35 | +info() { |
| 36 | + log 'INFO' "$@" |
| 37 | +} |
| 38 | + |
| 39 | +fatal() { |
| 40 | + log 'FATAL' "$@" |
| 41 | + exit 1 |
| 42 | +} |
| 43 | + |
| 44 | +cleanup() { |
| 45 | + if [[ -n ${workdir+x} && -d $workdir && -w $workdir ]]; then |
| 46 | + info "Removing $workdir" |
| 47 | + ( |
| 48 | + count=0 |
| 49 | + rm -rfv "$workdir" | while read -r; do |
| 50 | + if (( count++ > 750 )); then |
| 51 | + printf '.' |
| 52 | + count=0 |
| 53 | + fi |
| 54 | + done |
| 55 | + printf '\n' |
| 56 | + )& |
| 57 | + wait $! |
| 58 | + info "Removed '$workdir'" |
| 59 | + fi |
| 60 | +} |
| 61 | + |
| 62 | +abort() { |
| 63 | + cleanup |
| 64 | + exit 1 |
| 65 | +} |
| 66 | + |
| 67 | +if [[ $1 == nocleanup ]]; then |
| 68 | + shift # discard nocleanup arg |
| 69 | +else |
| 70 | + trap abort SIGINT SIGTERM SIGQUIT SIGABRT |
| 71 | + trap cleanup EXIT |
| 72 | +fi |
| 73 | + |
| 74 | +set -eu -o pipefail |
| 75 | +this="$(realpath "$0")" |
| 76 | +readonly this |
| 77 | +cd "$(dirname "$this")" |
| 78 | + |
| 79 | +shellcheck "$this" |
| 80 | + |
| 81 | +workdir= |
| 82 | +if [[ -n ${1+x} ]]; then |
| 83 | + arg_dir="$(realpath "$1")" |
| 84 | + if [[ ! ( -d $arg_dir && -w $arg_dir ) ]]; then |
| 85 | + fatal "$arg_dir is not a valid directory" |
| 86 | + fi |
| 87 | + workdir="$(mktemp -d "$arg_dir/check-requirements.XXXX")" |
| 88 | +else |
| 89 | + workdir="$(mktemp -d "/tmp/check-requirements.XXXX")" |
| 90 | +fi |
| 91 | +readonly workdir |
| 92 | + |
| 93 | +info "Working directory: $workdir" |
| 94 | + |
| 95 | +assert_arg_count() { |
| 96 | + local argcount="$1"; shift |
| 97 | + if (( $# != argcount )); then |
| 98 | + fatal "${FUNCNAME[1]}: incorrect number of args" |
| 99 | + fi |
| 100 | +} |
| 101 | + |
| 102 | +check_requirements() { |
| 103 | + assert_arg_count 2 "$@" |
| 104 | + local venv="$1" |
| 105 | + local reqs="$2" |
| 106 | + |
| 107 | + info "$reqs: beginning check" |
| 108 | + ( |
| 109 | + # shellcheck source=/dev/null |
| 110 | + source "$venv/bin/activate" |
| 111 | + pip --disable-pip-version-check install -q -r "$reqs" |
| 112 | + ) |
| 113 | + info "$reqs: OK" |
| 114 | +} |
| 115 | + |
| 116 | +check_convert_script() { |
| 117 | + assert_arg_count 1 "$@" |
| 118 | + local py="$1" |
| 119 | + local pyname="${py%.py}" |
| 120 | + |
| 121 | + info "$py: beginning check" |
| 122 | + |
| 123 | + local reqs="requirements-$pyname.txt" |
| 124 | + if [[ ! -r "$reqs" ]]; then |
| 125 | + fatal "$py missing requirements. Expected: $reqs" |
| 126 | + fi |
| 127 | + |
| 128 | + local venv="$workdir/$pyname-venv" |
| 129 | + python3 -m venv "$venv" |
| 130 | + |
| 131 | + check_requirements "$venv" "$reqs" |
| 132 | + set +e |
| 133 | + ( |
| 134 | + # shellcheck source=/dev/null |
| 135 | + source "$venv/bin/activate" |
| 136 | + py_err="$workdir/$pyname.out" |
| 137 | + python "$py" 2> "$py_err" |
| 138 | + >&2 cat "$py_err" |
| 139 | + grep -e 'ModuleNotFoundError' "$py_err" |
| 140 | + ) |
| 141 | + set -e |
| 142 | + # shellcheck disable=SC2181 |
| 143 | + (( $? )) && fatal "$py: some imports not declared in $reqs" |
| 144 | + info "$py: imports OK" |
| 145 | +} |
| 146 | + |
| 147 | +# Check requirements.txt |
| 148 | +all_venv="$workdir/all-venv" |
| 149 | +python3 -m venv "$all_venv" |
| 150 | +check_requirements "$all_venv" 'requirements.txt' |
| 151 | + |
| 152 | +check_convert_script 'convert.py' |
| 153 | +for py in convert-*.py; do |
| 154 | + check_convert_script "$py" |
| 155 | +done |
| 156 | + |
| 157 | +info "Done! No issues found." |
0 commit comments