Merge remote-tracking branch 'pigen/master' into MSW

.dockerignore

@@ -1,3 +1,5 @@
 output/
 work/
 deploy/
+apt-cacher-ng/
+.git/objects/*

.gitignore

@@ -3,5 +3,7 @@ work/*
 config
 postrun.sh
 SKIP
+SKIP_IMAGES
 .pc
 *-pc
+apt-cacher-ng/

Dockerfile

@@ -1,10 +1,12 @@
-FROM debian:jessie
+FROM debian:stretch
+
+ENV DEBIAN_FRONTEND noninteractive
 
 RUN apt-get -y update && \
     apt-get -y install \
         git vim parted \
         quilt realpath qemu-user-static debootstrap zerofree pxz zip dosfstools \
-        bsdtar libcap2-bin rsync grep \
+        bsdtar libcap2-bin rsync grep udev xz-utils curl xxd \
     && rm -rf /var/lib/apt/lists/*
 
 COPY . /pi-gen/

README.md

@@ -1,12 +1,24 @@
 # pi-gen
+
 _Tool used to create the raspberrypi.org Raspbian images_
 
-### TODO
-1. Documentation
 
 ## Dependencies
 
-`quilt parted realpath qemu-user-static debootstrap zerofree pxz zip dosfstools bsdtar libcap2-bin grep rsync`
+pi-gen runs on Debian based operating systems. Currently it is only supported on
+either Debian Stretch or Ubuntu Xenial and is known to have issues building on
+earlier releases of these systems.
+
+To install the required dependencies for pi-gen you should run:
+
+```bash
+apt-get install quilt parted realpath qemu-user-static debootstrap zerofree pxz zip \
+dosfstools bsdtar libcap2-bin grep rsync xz-utils
+```
+
+The file `depends` contains a list of tools needed.  The format of this
+package is `<tool>[:<debian-package>]`.
+
 
 ## Config
 
@@ -29,29 +41,117 @@ The following environment variables are supported:
    will not be included in the image, making it safe to use an `apt-cacher` or
    similar package for development.
 
+   If you have Docker installed, you can set up a local apt caching proxy to
+   like speed up subsequent builds like this:
+
+       docker-compose up -d
+       echo 'APT_PROXY=http://172.17.0.1:3142' >> config
+
+ * `BASE_DIR`  (Default: location of `build.sh`)
+
+   **CAUTION**: Currently, changing this value will probably break build.sh
+
+   Top-level directory for `pi-gen`.  Contains stage directories, build
+   scripts, and by default both work and deployment directories.
+
+ * `WORK_DIR`  (Default: `"$BASE_DIR/work"`)
+
+   Directory in which `pi-gen` builds the target system.  This value can be
+   changed if you have a suitably large, fast storage location for stages to
+   be built and cached.  Note, `WORK_DIR` stores a complete copy of the target
+   system for each build stage, amounting to tens of gigabytes in the case of
+   Raspbian.
+
+   **CAUTION**: If your working directory is on an NTFS partition you probably won't be able to build. Make sure this is a proper Linux filesystem.
+
+ * `DEPLOY_DIR`  (Default: `"$BASE_DIR/deploy"`)
+
+   Output directory for target system images and NOOBS bundles.
+
+ * `USE_QEMU` (Default: `"0"`)
+
+   This enable the Qemu mode and set filesystem and image suffix if set to 1.
+
+
 A simple example for building Raspbian:
 
 ```bash
 IMG_NAME='Raspbian'
 ```
 
+
+## How the build process works
+
+The following process is followed to build images:
+
+ * Loop through all of the stage directories in alphanumeric order
+
+ * Move on to the next directory if this stage directory contains a file called
+   "SKIP"
+
+ * Run the script ```prerun.sh``` which is generally just used to copy the build
+   directory between stages.
+
+ * In each stage directory loop through each subdirectory and then run each of the
+   install scripts it contains, again in alphanumeric order. These need to be named
+   with a two digit padded number at the beginning.
+   There are a number of different files and directories which can be used to
+   control different parts of the build process:
+
+     - **00-run.sh** - A unix shell script. Needs to be made executable for it to run
+
+     - **00-run-chroot.sh** - A unix shell script which will be run in the chroot
+       of the image build directory. Needs to be made executable for it to run.
+
+     - **00-debconf** - Contents of this file are passed to debconf-set-selections
+       to configure things like locale, etc.
+
+     - **00-packages** - A list of packages to install. Can have more than one, space
+       separated, per line.
+
+     - **00-packages-nr** - As 00-packages, except these will be installed using
+       the ```--no-install-recommends -y``` parameters to apt-get
+
+     - **00-patches** - A directory containing patch files to be applied
+
+  * If the stage directory contains files called "EXPORT_NOOBS" or "EXPORT_IMAGE" then
+    add this stage to a list of images to generate
+
+  * Generate the images for any stages that have specified them
+
+It is recommended to examine build.sh for finer details.
+
+
 ## Docker Build
 
 ```bash
 vi config         # Edit your config file. See above.
 ./build-docker.sh
 ```
+
 If everything goes well, your finished image will be in the `deploy/` folder.
-You can then remove the build container with `docker rm pigen_work`
+You can then remove the build container with `docker rm -v pigen_work`
 
 If something breaks along the line, you can edit the corresponding scripts, and
 continue:
 
-```
+```bash
 CONTINUE=1 ./build-docker.sh
 ```
 
-There is a possibility that even when running from a docker container, the installation of `qemu-user-static` will silently fail when building the image because `binfmt-support` _must be enabled on the underlying kernel_. An easy fix is to ensure `binfmt-support` is installed on the host machine before starting the `./build-docker.sh` script (or using your own docker build solution).
+After successful build, the build container is by default removed. This may be undesired when making incremental changes to a customized build. To prevent the build script from remove the container add
+
+```bash
+PRESERVE_CONTAINER=1 ./build-docker.sh
+```
+
+There is a possibility that even when running from a docker container, the
+installation of `qemu-user-static` will silently fail when building the image
+because `binfmt-support` _must be enabled on the underlying kernel_. An easy
+fix is to ensure `binfmt-support` is installed on the host machine before
+starting the `./build-docker.sh` script (or using your own docker build
+solution).
+
 
 ## Stage Anatomy
 
@@ -86,32 +186,62 @@ maintenance and allows for more easy customization.
    standard console hardware permission groups.
 
    There are a few tools that may not make a whole lot of sense here for
-   development purposes on a minimal system such as basic python and lua
+   development purposes on a minimal system such as basic Python and Lua
    packages as well as the `build-essential` package.  They are lumped right
    in with more essential packages presently, though they need not be with
    pi-gen.  These are understandable for Raspbian's target audience, but if
-   you were looking for something between truly minimal and Raspbian-lite,
+   you were looking for something between truly minimal and Raspbian-Lite,
    here's where you start trimming.
 
  - **Stage 3** - desktop system.  Here's where you get the full desktop system
    with X11 and LXDE, web browsers, git for development, Raspbian custom UI
    enhancements, etc.  This is a base desktop system, with some development
    tools installed.
 
- - **Stage 4** - complete Raspbian system.  More development tools, an email
-   client, learning tools like Scratch, specialized packages like sonic-pi and
-   wolfram-engine, system documentation, office productivity, etc.  This is
-   the stage that installs all of the things that make Raspbian friendly to
-   new users.
+ - **Stage 4** - Raspbian system meant to fit on a 4GB card.  More development
+   tools, an email client, learning tools like Scratch, specialized packages
+   like sonic-pi, system documentation, office productivity, etc.  This is the
+   stage that installs all of the things that make Raspbian friendly to new
+   users.
+
+ - **Stage 5** - The official Raspbian Desktop image. Right now only adds
+   Mathematica.
 
 ### Stage specification
-If you wish to build up to a specified stage (such as building up to stage 2 for a lite system), place an empty file named `SKIP` in each of the `./stage` directories you wish not to include.
 
-Then remove the `EXPORT*` files from `./stage4` (if building up to stage 2) or from `./stage2` (if building a minimal system).
+If you wish to build up to a specified stage (such as building up to stage 2
+for a lite system), place an empty file named `SKIP` in each of the `./stage`
+directories you wish not to include.
 
-```
+Then remove the `EXPORT*` files from `./stage4` (if building up to stage 2) or
+from `./stage2` (if building a minimal system).
+
+```bash
 # Example for building a lite system
-$ touch ./stage3/SKIP ./stage4/SKIP
-$ rm stage4/EXPORT*
+echo "IMG_NAME='Raspbian'" > config
+touch ./stage3/SKIP ./stage4/SKIP ./stage5/SKIP
+rm stage4/EXPORT* stage5/EXPORT*
+sudo ./build.sh  # or ./build-docker.sh
 ```
-If you wish to build further configurations upon (for example) the lite system, you can also delete the contents of `./stage3` and `./stage4` and replace with your own contents in the same format.
+
+If you wish to build further configurations upon (for example) the lite
+system, you can also delete the contents of `./stage3` and `./stage4` and
+replace with your own contents in the same format.
+
+
+## Skipping stages to speed up development
+
+If you're working on a specific stage the recommended development process is as
+follows:
+
+ * Add a file called SKIP_IMAGES into the directories containing EXPORT_* files
+   (currently stage2, stage4 and stage5)
+ * Add SKIP files to the stages you don't want to build. For example, if you're
+   basing your image on the lite image you would add these to stages 3, 4 and 5.
+ * Run build.sh to build all stages
+ * Add SKIP files to the earlier successfully built stages
+ * Modify the last stage
+ * Rebuild just the last stage using ```sudo CLEAN=1 ./build.sh```
+ * Once you're happy with the image you can remove the SKIP_IMAGES files and
+   export your image to test
+

build-docker.sh

@@ -1,4 +1,5 @@
 #!/bin/bash -e
+
 DOCKER="docker"
 set +e
 $DOCKER ps >/dev/null 2>&1
@@ -12,26 +13,28 @@ if ! $DOCKER ps >/dev/null; then
 fi
 set -e
 
-config_mount=
+config_file=()
 if [ -f config ]; then
-	config_mount="-v $(pwd)/config:/pi-gen/config:ro"
+	config_file=("--env-file" "$(pwd)/config")
 	source config
 fi
 
 CONTAINER_NAME=${CONTAINER_NAME:-pigen_work}
 CONTINUE=${CONTINUE:-0}
+PRESERVE_CONTAINER=${PRESERVE_CONTAINER:-0}
 
 if [ "$*" != "" ] || [ -z "${IMG_NAME}" ]; then
 	if [ -z "${IMG_NAME}" ]; then
-		echo "IMG_NAME not set in 'build'" 1>&2
+		echo "IMG_NAME not set in 'config'" 1>&2
 		echo 1>&2
 	fi
 	cat >&2 <<EOF
 Usage:
     build-docker.sh [options]
 Optional environment arguments: ( =<default> )
     CONTAINER_NAME=pigen_work  set a name for the build container
-    CONTINUE=0                  continue from a previously started container
+    CONTINUE=1                 continue from a previously started container
+    PRESERVE_CONTAINER=1       keep build container even on successful build
 EOF
 	exit 1
 fi
@@ -45,29 +48,39 @@ fi
 if [ "$CONTAINER_EXISTS" != "" ] && [ "$CONTINUE" != "1" ]; then
 	echo "Container $CONTAINER_NAME already exists and you did not specify CONTINUE=1. Aborting."
 	echo "You can delete the existing container like this:"
-	echo "  docker rm $CONTAINER_NAME"
+	echo "  $DOCKER rm -v $CONTAINER_NAME"
 	exit 1
 fi
 
 $DOCKER build -t pi-gen .
 if [ "$CONTAINER_EXISTS" != "" ]; then
-	trap "echo 'got CTRL+C... please wait 5s';docker stop -t 5 ${CONTAINER_NAME}_cont" SIGINT SIGTERM
+	trap "echo 'got CTRL+C... please wait 5s'; $DOCKER stop -t 5 ${CONTAINER_NAME}_cont" SIGINT SIGTERM
 	time $DOCKER run --rm --privileged \
 		--volumes-from="${CONTAINER_NAME}" --name "${CONTAINER_NAME}_cont" \
+		-e IMG_NAME=${IMG_NAME}\
 		pi-gen \
 		bash -e -o pipefail -c "dpkg-reconfigure qemu-user-static &&
 	cd /pi-gen; ./build.sh;
 	rsync -av work/*/build.log deploy/" &
-	wait
+	wait "$!"
 else
-	trap "echo 'got CTRL+C... please wait 5s'; docker stop -t 5 ${CONTAINER_NAME}" SIGINT SIGTERM
-	$DOCKER run --name "${CONTAINER_NAME}" --privileged \
-		-v $(pwd)/deploy:/pi-gen/deploy \
-		${config_mount} \
+	trap "echo 'got CTRL+C... please wait 5s'; $DOCKER stop -t 5 ${CONTAINER_NAME}" SIGINT SIGTERM
+	time $DOCKER run --name "${CONTAINER_NAME}" --privileged \
+		-e IMG_NAME=${IMG_NAME}\
+		"${config_file[@]}" \
 		pi-gen \
 		bash -e -o pipefail -c "dpkg-reconfigure qemu-user-static &&
 	cd /pi-gen; ./build.sh &&
 	rsync -av work/*/build.log deploy/" &
-	wait
+	wait "$!"
+fi
+echo "copying results from deploy/"
+$DOCKER cp "${CONTAINER_NAME}":/pi-gen/deploy .
+ls -lah deploy
+
+# cleanup
+if [ "$PRESERVE_CONTAINER" != "1" ]; then
+	$DOCKER rm -v $CONTAINER_NAME
 fi
+
 echo "Done! Your image(s) should be in deploy/"

build.sh

@@ -85,8 +85,10 @@ run_stage(){
 	unmount ${WORK_DIR}/${STAGE}
 	STAGE_WORK_DIR=${WORK_DIR}/${STAGE}
 	ROOTFS_DIR=${STAGE_WORK_DIR}/rootfs
-	if [ -f ${STAGE_DIR}/EXPORT_IMAGE ]; then
-		EXPORT_DIRS="${EXPORT_DIRS} ${STAGE_DIR}"
+	if [ ! -f SKIP_IMAGES ]; then
+		if [ -f ${STAGE_DIR}/EXPORT_IMAGE ]; then
+			EXPORT_DIRS="${EXPORT_DIRS} ${STAGE_DIR}"
+		fi
 	fi
 	if [ ! -f SKIP ]; then
 		if [ "${CLEAN}" = "1" ]; then
@@ -119,6 +121,7 @@ if [ "$(id -u)" != "0" ]; then
 	exit 1
 fi
 
+
 if [ -f config ]; then
 	source config
 fi
@@ -128,7 +131,8 @@ if [ -z "${IMG_NAME}" ]; then
 	exit 1
 fi
 
-export IMG_DATE=${IMG_DATE:-"$(date -u +%Y-%m-%d)"}
+export USE_QEMU=${USE_QEMU:-0}
+export IMG_DATE=${IMG_DATE:-"$(date +%Y-%m-%d)"}
 
 export BASE_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
 export SCRIPT_DIR="${BASE_DIR}/scripts"
@@ -177,10 +181,12 @@ for EXPORT_DIR in ${EXPORT_DIRS}; do
 	source "${EXPORT_DIR}/EXPORT_IMAGE"
 	EXPORT_ROOTFS_DIR=${WORK_DIR}/$(basename ${EXPORT_DIR})/rootfs
 	run_stage
-	if [ -e ${EXPORT_DIR}/EXPORT_NOOBS ]; then
-		source ${EXPORT_DIR}/EXPORT_NOOBS
-		STAGE_DIR=${BASE_DIR}/export-noobs
-		run_stage
+	if [ "${USE_QEMU}" != "1" ]; then
+		if [ -e ${EXPORT_DIR}/EXPORT_NOOBS ]; then
+			source ${EXPORT_DIR}/EXPORT_NOOBS
+			STAGE_DIR=${BASE_DIR}/export-noobs
+			run_stage
+		fi
 	fi
 done
 

depends

@@ -9,4 +9,7 @@ mkdosfs:dosfstools
 capsh:libcap2-bin
 bsdtar
 grep
-rsync
+rsync
+xz:xz-utils
+curl
+xxd