build_deb

build_deb.sh

#!/usr/bin/env bash

# Toggle for debug echoing
# set -x

# Turn some bugs into errors
set -o errexit -o pipefail -o noclobber -o nounset

# Shellcheck is a utility which checks bash scripts for common errors and weak
# points in the language. If you need to suppress an error, put a suppression
# comment before the offending line. E.g. `# shellcheck disable=SC2086`.
shellcheck "$(dirname "$0")/"*

##############################################################################
# Function definitions
##############################################################################

usage(){
    cat << USAGE
Usage: $(basename "$0") [-a <arch>] -B <builddir> [-n <buildnumber>]
           [-m <mirror>] [--sdk <package>]  [-s <suite>]

Options:
    -a, --arch      CPU architecture to build for (i386, amd64, armhf, arm64)
    -B, --builddir  working directory and artifact output
    -n, --buildnumber   buildnumber to be concatenated to the release version
    -c, --clean     destroy and reprovision the cached build environment
    -h, --help      print this usage message
    -m, --mirror    mirror to download packages from
    -S, --sdk       path to a .deb build-time dependency. can be specified multiple times.
    -s, --suite     OS release to build for (stretch, buster, xenial, bionic)

USAGE
}

# Given a filepath argument, attempt to umount that path.
tryumount(){
    upath="$1"
    if [ ! -d "$upath" ]; then
        echo "Folder \"$upath\" doesn't exist, therefore it isn't mounted."
        return 0
    fi
    upath="$(realpath "$upath")"
    echo "Attempting to umount \"$upath\"..."
    declare -i attempts
    attempts=$((0))

    # Keep attempting to umount as long as this verbatim path appears in the mount list
    while mount | awk '{print $3}' | grep -qe "^$upath$"; do
        echo "+ umount -R \"$upath\""
        umount -R "$upath"
        attempts=$((attempts+1))
        if (( attempts > 10 )); then
            echo "Ten failed umount attempts; exiting..."
            return 1
        fi
    done
}

##############################################################################
# Parse options
##############################################################################

# Ensure that we have a version of getopt recent enough to understand --longoptions.
! getopt --test > /dev/null
if [[ ${PIPESTATUS[0]} -ne 4 ]]; then
    echo "getopt too old: \`getopt --test\` failed to parse in this environment." >&2
    exit 1
fi

OPTIONS=a:B:chn:m:iS:s:
LONGOPTS=arch:,builddir:,clean,help,buildnumber:,mirror:,sdk:,suite:

# -use ! and PIPESTATUS to get exit code with errexit set
# -temporarily store output to be able to check for errors
# -activate quoting/enhanced mode (e.g. by writing out “--options”)
# -pass arguments only via -- "$@" to separate them correctly
! PARSED=$(getopt --options=$OPTIONS --longoptions=$LONGOPTS --name "$0" -- "$@")
if [[ ${PIPESTATUS[0]} -ne 0 ]]; then
    # getopt has complained about wrong arguments to stdout.
    exit 2
fi

# Read getopt’s output this way to handle the quoting right.
eval set -- "$PARSED"

# Set default option values.
ARCH=amd64
BUILDDIR=""
BUILDNUMBER=""
CLEAN=""
MIRROR=""
SDK=()
SUITE=xenial

# Parse options.
while true; do
    case "$1" in
        -a|--arch) ARCH="$2"; shift 2;;
        -B|--builddir) BUILDDIR="$(realpath "$2")"; shift 2;;
        -n|--buildnumber) BUILDNUMBER="$2"; shift 2;;
        -c|--clean) CLEAN="true"; shift 1;;
        -h|--help) usage; exit 0;;
        -m|--mirror) MIRROR="$2"; shift 2;;
        -S|--sdk) SDK+=("$2"); shift 2;;
        -s|--suite) SUITE="$2"; shift 2;;
        --) shift; break;;
        *) echo "ERROR: Unknown option $1" >&2; usage; exit 1;;
    esac
done

# Ensure elevated permissions.
if [[ $(/usr/bin/id -u) -ne 0 ]]; then
    echo "ERROR: Rerun this script using elevated permissions." >&2
    exit 1
fi

# Ensure mandatory option.
if [[ -z "$BUILDDIR" ]]; then
    echo "ERROR: Missing --builddir option" >&2
    usage
    exit 1
fi

# If the mirror is undefined then use defaults.
if [[ -z $MIRROR ]]; then
    case "$SUITE" in
        stretch|buster)
            MIRROR=http://cdn-fastly.deb.debian.org/debian
            ;;
        xenial|bionic)
            case "$ARCH" in
                i386|amd64)
                    MIRROR=http://archive.ubuntu.com/ubuntu
                    ;;
                armhf|arm64)
                    MIRROR=http://ports.ubuntu.com/ubuntu-ports
                    ;;
                *)
                    echo "ERROR: Architecture $ARCH has not been tested." >&2
                    exit 1
                    ;;
            esac
            ;;
        *)
            echo "ERROR: Suite $SUITE has not been tested." >&2
            exit 1
            ;;
    esac
fi

# Xenial's debootstrap is too old to know about buster.
# If there is no buster script, just use sid's, since it's identical anyway.
if [ -f /usr/share/deboostrap/scripts/buster ]; then
    ln -s /usr/share/deboostrap/scripts/sid /usr/share/deboostrap/scripts/buster
fi

echo "This script: [$0]"
# Usually the script is sitting within the source repo.
pushd "$(dirname "$0")/../../"
SRC_ROOT="$(realpath "$PWD")"
popd
echo "Project source folder is: [$SRC_ROOT]"

# Disallow build folders inside the source tree to prevent accidentally including
# build artifacts in the packages.
if [[ "$BUILDDIR" == "$SRC_ROOT"* ]]; then
    echo "ERROR: Choose a builddir outside the source tree." >&2
    exit 1
fi

##############################################################################
# Provision build environment
##############################################################################
#
# In order that builds should be clean and predictable, provision a pristine
# build environment using debootstrap. This ensures we only build and link
# against the intended code and libraries.
#
# Since the build environment will initially be the same each time, save time
# and bandwidth time by caching it under /var/cache/buildchroot/ and mount it
# read-only using overlayfs. Writes are redirected to folders under $BUILDDIR.
#

PRISTINE_BUILDCHROOT=/var/cache/buildchroot/$SUITE-$ARCH

if [[ ! -z $CLEAN ]]; then
    echo "Deleting cached build environment $PRISTINE_BUILDCHROOT"
    rm -rf "${PRISTINE_BUILDCHROOT:?}"
fi

if [[ -d $PRISTINE_BUILDCHROOT ]]; then
    echo "Build environment already exists! Using cache..."
else
    echo "Build environment does not exist yet. Creating..."
    dpkg --add-architecture "$ARCH"
    # use true to ignore apt-get failures in pipelines
    apt-get update || true
    mkdir -p "$PRISTINE_BUILDCHROOT"
    qemu-debootstrap --arch "$ARCH" "$SUITE" "$PRISTINE_BUILDCHROOT" "$MIRROR"

    # Initial setup beyond what debootstrap does.
    # We want to cache this too, because installing build-essential is slow.
    cp "$SRC_ROOT/scripts/debian/chroot_initial_setup.sh" "$PRISTINE_BUILDCHROOT"
    chroot "$PRISTINE_BUILDCHROOT" /bin/bash -c "su - -c '/chroot_initial_setup.sh'"
fi

BUILDCHROOT="$BUILDDIR/merged-$SUITE-$ARCH"
UPPERDIR="$BUILDDIR/upper-$SUITE-$ARCH"
WORKDIR="$BUILDDIR/work-$SUITE-$ARCH"

# Debian package builds do not support incremental builds.
# Ensure we start with a clean (although possibly cached) build environment.
tryumount "$BUILDCHROOT/dev/"
tryumount "$BUILDCHROOT/sys/"
tryumount "$BUILDCHROOT/proc/"
tryumount "$BUILDCHROOT"
rm -rf "${BUILDDIR:?}"/*"-$SUITE-$ARCH"

# Do our work in an overlay on top of the read-only pristine environment.
mkdir -p "$BUILDCHROOT"
mkdir -p "$UPPERDIR"
mkdir -p "$WORKDIR"
mount -t overlay overlay -o lowerdir="$PRISTINE_BUILDCHROOT",upperdir="$UPPERDIR",workdir="$WORKDIR" "$BUILDCHROOT"

##############################################################################
# Create source package
##############################################################################
#
# Debian binary packages are derived from source packages, which obey certain
# rules. After we have wrapped our code & manifests into a source package, we
# should not edit them during the build process or dpkg will complain that the
# files have changed.
#

# Make last-minute code edits before saving code into the source package.
# shellcheck disable=SC1090
source "$(dirname "$0")/prepackage.sh" "$BUILDNUMBER"

# Parse name and version from the changelog
pushd "$SRC_ROOT"
PKG_NAME=$(dpkg-parsechangelog -S Source)
PKG_VERSION=$(dpkg-parsechangelog -S Version | sed -rne 's,([^-\+]+)(\+dfsg)?.*,\1,p')
PKG_FULLNAME="${PKG_NAME}_${PKG_VERSION}"
echo "Package name: $PKG_NAME"
echo "Package version: $PKG_VERSION"
echo "Full package name: $PKG_FULLNAME"
popd

# Debian wants the source folder name to match the package name and version when we build the source package.
# Unfortunately the tools don't respect symbolic links and want to place files in the true parent folder.
# To prevent clutter, make a temporary mount location and wrap the source package there.
# This prevents race conditions when building multiple architectures at once: each invocation wraps the
# source package in a separate folder, instead of thrashing each other in "$SRC_ROOT/.."
SRC_TMP=$(mktemp -d)
SRC_SRC="$SRC_TMP/$PKG_FULLNAME"
mkdir -p "$SRC_SRC"
mount --bind "$SRC_ROOT" "$SRC_SRC"
pushd "$SRC_SRC"

# Let dpkg do it's thing.
dpkg-source --before-build .
fakeroot debian/rules clean
echo Creating tarball...
tar caf "../${PKG_FULLNAME}.orig.tar.gz" --exclude "./debian" --exclude "./.git" --exclude "./buildout" .
echo Creating source package...
dpkg-source --build .

# Copy the three components of the source package into the build environment:
# description, upstream source, debian-specific
mkdir -p "$BUILDCHROOT/home/builder/"
cp "$SRC_TMP/$PKG_FULLNAME"*.{dsc,orig.tar.gz,debian.tar.xz} "$BUILDCHROOT/home/builder/"

# Cleanup the temp folder we used for source packaging
popd
umount -lR "$SRC_SRC"
rm -rf "$SRC_TMP"

##############################################################################
# Build binary package
##############################################################################
#
# Copy the source package and its dependencies into the build environment, then build it.
#
# All dependencies should be declared in debian/control.
# Any package available in the official repos will be installed automatically.
# If a package is not available from the official repos, use the --sdk argument to
#

cp "$SRC_ROOT/scripts/debian/chroot_helper.sh" "$BUILDCHROOT"

# Copy over non-repo dependencies
if [[ ${#SDK[@]} -eq 0 ]]; then
    echo "No non-repo dependencies to copy."
else
    mkdir -p "$BUILDCHROOT/deps"
    echo "Copying ${#SDK[@]} non-repo dependencies into build environment..."
    for sdk in "${SDK[@]}"; do
        echo "Copying dependency '$sdk'"

        # Use unquoted ls to resolve wildcards in paths.
        # shellcheck disable=SC2086
        sdk_realpath="$(ls -- $sdk)"
        echo "Real path is '$sdk_realpath'"

        cp "$sdk_realpath" "$BUILDCHROOT/deps"
    done
    echo "Non-repo dependencies have been copied:"
    ls "$BUILDCHROOT/deps"
fi

# Make proc, sys, and dev visible to the build environment (needed for many tasks).
mount -t proc /proc "$BUILDCHROOT/proc/"
mount --rbind --make-rslave /sys "$BUILDCHROOT/sys/"
mount --rbind --make-rslave /dev "$BUILDCHROOT/dev/"

# Run the remaining steps inside the build environment.
chroot "$BUILDCHROOT" /bin/bash -c "su - -c '/chroot_helper.sh $BUILDNUMBER'"

# Copy binary packages out of chroot environment.
ARTIFACTDIR="$BUILDDIR/out/$SUITE/"
mkdir -p "$ARTIFACTDIR"
cp "${BUILDCHROOT}/home/builder/"*.deb "$ARTIFACTDIR"

tryumount "$BUILDCHROOT/dev/"
tryumount "$BUILDCHROOT/sys/"
tryumount "$BUILDCHROOT/proc/"
tryumount "$BUILDCHROOT"

chroot_helper.sh

#!/usr/bin/env bash

# Toggle for debug echoing
set -x

# Turn some bugs into errors.
set -o errexit -o pipefail -o noclobber -o nounset

# If a glob doesn't match, expand to nothing
shopt -s nullglob

# Extract source package and cd into the folder.
pushd /home/builder || exit 1
dpkg-source -x ./*.dsc
pushd ./*/ || exit 1

# Install non-repo build dependencies.
for debfile in /deps/*.deb; do
    dpkg -i "$debfile"
done
# Install any missing dependencies of those packages.
apt-get install -y -f

# Install repo build dependencies.
mk-build-deps -i -t "apt-get -y"

# Clang
apt-get install -y clang
pushd /usr/bin
echo "Forcibly replacing C compilers with Clang..."
for filename in /usr/bin/*gcc{,-[0-9]*}; do
    rm "$filename"
    ln -s clang "$filename"
done
echo "Forcibly replacing C++ compilers with Clang++..."
for filename in /usr/bin/{g++,*gnu-cpp,*gnu-g++}{,-[0-9]*}; do
    rm "$filename"
    ln -s clang++ "$filename"
done
popd

# Build binary package.
if [ -z "${1:-}" ]; then
  debuild -uc -us --source-option=--include-binaries
else
  debuild --set-envvar BUILDNUMBER="$1" -uc -us --source-option=--include-binaries
fi

chroot_initial_setup.sh

#!/usr/bin/env bash

# Toggle for debug echoing
set -x

# Turn some bugs into errors.
set -o errexit -o pipefail -o nounset

# First install and configure the language and timezone.
apt-get update
apt-get install -y locales tzdata

# Uncomment the en_US locale.
sed -i -e 's/# en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen
echo 'LANG="en_US.UTF-8"' > /etc/default/locale

# Regenerate locale.
dpkg-reconfigure --frontend=noninteractive locales
update-locale LANG=en_US.UTF-8

# Configure timezone.
echo "America/Los_Angeles" > /etc/timezone
dpkg-reconfigure -f noninteractive tzdata

# If Ubuntu, enable all repos so we can install build dependencies.
if grep -q Ubuntu "/etc/issue.net"; then
    apt-get install -y software-properties-common lsb-release
    case $(uname -m) in
        *86*) repo=http://archive.ubuntu.com/ubuntu ;;
        *)    repo=http://ports.ubuntu.com/ubuntu-ports ;;
    esac
    suite=$(lsb_release -sc)
    add-apt-repository "deb $repo $suite main universe restricted multiverse"
    add-apt-repository "deb $repo ${suite}-updates main universe restricted multiverse"
    add-apt-repository "deb $repo ${suite}-backports main universe restricted multiverse"
    add-apt-repository "deb $repo ${suite}-security main universe restricted multiverse"
    apt-get update

    # Ensure we get the latest package updates. Specifically gcc-5-base version 5.4.0 for Xenial.
    if [[ "$suite" == "xenial" ]]; then
        # Upgrade hangs on the console-setup package unless we provide this answer ahead of time.
        echo "console-setup   console-setup/charmap47 select  UTF-8" > encoding.conf
        debconf-set-selections encoding.conf
        rm encoding.conf
        apt-get upgrade -y
    fi

    apt-get install -y equivs
fi

# The 'buildd' flavor of debootstrap should already have picked these up, but just in case...
apt-get install -y build-essential devscripts

install_debian_prereqs.sh

#!/usr/bin/env bash

# ensure elevated permissions
if [[ $(/usr/bin/id -u) -ne 0 ]]; then
    echo "Rerun this script using elevated permissions." >&2
    exit 1
fi

apt-get update
apt-get install -y build-essential debhelper debianutils devscripts \
                   binfmt-support qemu qemu-user-static debootstrap \
                   shellcheck

prepackage.sh

#!/usr/bin/env bash

# Edit debian/changelog version to match {RELEASE_VERSION}-1
sed -i -r "1 s/\\([a-zA-Z0-9.-]+\\)/($RELEASE_VERSION-1)/" "$SRC_ROOT/debian/changelog"