smartbotic/deploy/zeus/backup-smartbotic-db.sh

#!/bin/sh
# Back up the SmartBotic database running in Docker on zeus.
#
# The database is stopped for the copy. A live copy of a WAL-backed store can be
# torn, and a backup nobody can restore is worse than no backup because it is
# believed in. The stop is seconds; the container comes back whatever happens,
# including if this script dies partway.
#
# The encryption key lives inside the data directory. It is copied with
# everything else and must never be separated from it - data without the key is
# unreadable, and so is a backup without it.
set -eu

CONTAINER=smartbotic-db
DATA=/data/smartbotic-db/data
DEST=/data/backups/smartbotic-db
KEEP=14
STAMP=$(date +%Y%m%d-%H%M%S)
ARCHIVE="$DEST/smartbotic-db-$STAMP.tar.zst"

mkdir -p "$DEST"

was_running=0
if [ "$(docker inspect -f '{{.State.Running}}' "$CONTAINER" 2>/dev/null)" = "true" ]; then
    was_running=1
fi

restart_if_needed() {
    if [ "$was_running" = "1" ]; then
        docker start "$CONTAINER" >/dev/null 2>&1 || true
    fi
}
trap restart_if_needed EXIT INT TERM

if [ "$was_running" = "1" ]; then
    docker stop "$CONTAINER" >/dev/null
fi

tar -C "$DATA" -cf - . | zstd -q -3 -o "$ARCHIVE"

# Started again before the verification, so the database is down only for the
# copy itself rather than for the reading back as well.
restart_if_needed
trap - EXIT INT TERM

# A backup is not a backup until it has been read back. This checks the archive
# is intact and that the encryption key is inside it - the one file whose
# absence would not be noticed until a restore was already needed.
if ! zstd -qt "$ARCHIVE"; then
    echo "BACKUP FAILED: $ARCHIVE does not decompress" >&2
    rm -f "$ARCHIVE"
    exit 1
fi
if ! zstd -qdc "$ARCHIVE" | tar -t | grep -q '^\./storage\.key$'; then
    echo "BACKUP FAILED: $ARCHIVE has no storage.key - the data would be unreadable" >&2
    rm -f "$ARCHIVE"
    exit 1
fi

# Only prune once a good one exists, so a run of failures cannot age out the
# last working copy.
ls -1t "$DEST"/smartbotic-db-*.tar.zst 2>/dev/null | tail -n +$((KEEP + 1)) | while read -r old; do
    rm -f "$old"
done

echo "$(date -Is) ok $ARCHIVE $(du -h "$ARCHIVE" | cut -f1) ($(ls -1 "$DEST"/smartbotic-db-*.tar.zst | wc -l) kept)"