rezervovator/deploy/rezervator.sh

#!/usr/bin/env bash
#
# Management of a Rezervovator stack with one container per customer.
# See README.md in this directory.

set -euo pipefail

DEPLOY_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
ENV_FILE="$DEPLOY_DIR/.env"
CUSTOMERS_DIR="$DEPLOY_DIR/customers"
BACKUPS_DIR="$DEPLOY_DIR/backups"

usage() {
    cat <<USAGE
Usage: $(basename "$0") <command> [arguments]

  init                         Create .env; then start the shared services and check the setup
  add <name> <domain> [from]   Create a customer: database, config, volume, container.
                               [from] is the sender address of e-mails, default noreply@<domain>
  import <name> <domain> <database> [data-dir] [from]
                               Create a customer from an existing installation: copies its
                               database and the files from its data directory. The source is
                               only read.
  list                         List customers and the state of their containers
  upgrade <name>...|--all      Recreate containers from REZERVATOR_IMAGE, one by one.
                               Backs up first, rolls back and stops when one fails.
  backup <name>...|--all       Dump the database and the data volume to backups/<name>/
  restore <name> <timestamp>   Restore a customer from backups/<name>/<timestamp>-*
  remove <name> [--purge]      Stop and remove the container. With --purge also the database,
                               the data volume and the config (asks for confirmation).

<name> is lowercase letters, digits and dashes, starting with a letter.
USAGE
}

die() {
    echo "Error: $*" >&2
    exit 1
}

info() {
    echo "==> $*"
}

random_string() {
    # $1 = number of bytes, output is twice as many hex characters
    openssl rand -hex "$1"
}

load_env() {
    [ -f "$ENV_FILE" ] || die "$ENV_FILE not found, run '$(basename "$0") init' first"
    set -a
    # shellcheck disable=SC1090
    . "$ENV_FILE"
    set +a

    PREFIX="${REZERVATOR_PREFIX:-rezervator}"
    IMAGE="${REZERVATOR_IMAGE:-rezervator:latest}"
    TLS="${REZERVATOR_TLS:-true}"
    PROXY="${REZERVATOR_PROXY:-traefik}"
    POSTGRES="${REZERVATOR_POSTGRES:-container}"
    PG_CONTAINER="$PREFIX-postgres"
    PG_PORT="${PG_PORT:-5432}"
    NETWORK="$PREFIX"
    CUSTOMER_MAX_CONNECTIONS="${CUSTOMER_MAX_CONNECTIONS:-5}"
    BASE_PORT="${BASE_PORT:-3100}"

    case "$PROXY" in traefik|nginx) ;; *) die "REZERVATOR_PROXY must be traefik or nginx" ;; esac
    case "$POSTGRES" in container|host) ;; *) die "REZERVATOR_POSTGRES must be container or host" ;; esac

    if [ "$POSTGRES" = "host" ]; then
        # Address of the Docker host as the containers see it
        DB_HOST="${PG_HOST:-host.docker.internal}"
        # How to run psql and pg_dump as the PostgreSQL superuser; may be empty
        PG_ADMIN="${PG_ADMIN-sudo -u postgres}"
    else
        DB_HOST="$PG_CONTAINER"
    fi
}

compose() {
    docker compose --project-directory "$DEPLOY_DIR" --env-file "$ENV_FILE" "$@"
}

# Starts the shared services that run as containers in this setup, if any
compose_up() {
    local profiles=()
    if [ "$PROXY" = "traefik" ]; then profiles+=(--profile traefik); fi
    if [ "$POSTGRES" = "container" ]; then profiles+=(--profile postgres); fi

    docker network inspect "$NETWORK" >/dev/null 2>&1 || docker network create "$NETWORK" >/dev/null
    if [ "${#profiles[@]}" -gt 0 ]; then
        compose "${profiles[@]}" up --detach
    fi
}

check_name() {
    [[ "$1" =~ ^[a-z][a-z0-9-]{0,30}$ ]] || die "Invalid customer name '$1'"
}

container_name() { echo "$PREFIX-app-$1"; }
volume_name() { echo "$PREFIX-data-$1"; }
# Database and role have the same name
db_name() { echo "rz_${1//-/_}"; }

customer_exists() {
    [ -f "$CUSTOMERS_DIR/$1/customer.env" ]
}

require_customer() {
    check_name "$1"
    customer_exists "$1" || die "Unknown customer '$1'"
}

all_customers() {
    [ -d "$CUSTOMERS_DIR" ] || return 0
    for dir in "$CUSTOMERS_DIR"/*/; do
        if [ -f "${dir}customer.env" ]; then basename "$dir"; fi
    done
    return 0
}

# Reads a value from the customer's customer.env. $1 = name, $2 = variable
customer_value() {
    sed -n "s/^$2=//p" "$CUSTOMERS_DIR/$1/customer.env" | head -n 1
}

# psql as the PostgreSQL superuser, on the host or in the container
psql_admin() {
    if [ "$POSTGRES" = "host" ]; then
        # shellcheck disable=SC2086
        $PG_ADMIN psql -v ON_ERROR_STOP=1 -q "$@"
    else
        docker exec -i "$PG_CONTAINER" psql -U postgres -v ON_ERROR_STOP=1 -q "$@"
    fi
}

pg_dump_admin() {
    if [ "$POSTGRES" = "host" ]; then
        # shellcheck disable=SC2086
        $PG_ADMIN pg_dump "$@"
    else
        docker exec "$PG_CONTAINER" pg_dump -U postgres "$@"
    fi
}

wait_for_postgres() {
    if [ "$POSTGRES" = "host" ]; then
        psql_admin -tAc "SELECT 1" >/dev/null 2>&1 \
            || die "Cannot run psql as the PostgreSQL superuser (PG_ADMIN='$PG_ADMIN')"
        return 0
    fi
    for _ in $(seq 1 60); do
        # Over TCP: while the image initializes a new cluster, its temporary server listens on the socket only
        if docker exec "$PG_CONTAINER" pg_isready -h 127.0.0.1 -U postgres -q 2>/dev/null; then
            return 0
        fi
        sleep 1
    done
    die "PostgreSQL in container $PG_CONTAINER is not ready"
}

# With PostgreSQL on the host, the containers must be able to reach it over the Docker network
check_db_reachable() {
    [ "$POSTGRES" = "host" ] || return 0
    docker run --rm --network "$NETWORK" --add-host "host.docker.internal:host-gateway" \
        --entrypoint bash "$IMAGE" -c "exec 3<>/dev/tcp/$DB_HOST/$PG_PORT" 2>/dev/null \
        || die "Containers cannot reach PostgreSQL at $DB_HOST:$PG_PORT. Check listen_addresses, pg_hba.conf and the firewall, see README.md."
}

# First free port for a customer container behind nginx
next_port() {
    local port="$BASE_PORT" name used
    for name in $(all_customers); do
        used="$(customer_value "$name" PORT)"
        if [ -n "$used" ] && [ "$used" -ge "$port" ]; then
            port=$((used + 1))
        fi
    done
    echo "$port"
}

# Writes the nginx server block of a customer to customers/<name>/nginx.conf
write_nginx_conf() {
    local name="$1" domain port
    domain="$(customer_value "$name" DOMAIN)"
    port="$(customer_value "$name" PORT)"
    cat > "$CUSTOMERS_DIR/$name/nginx.conf" <<NGINX
# Rezervovator, customer '$name'. Generated by rezervator.sh.
# Install it into nginx and add TLS, e.g.:
#   sudo cp $CUSTOMERS_DIR/$name/nginx.conf /etc/nginx/conf.d/$PREFIX-$name.conf
#   sudo nginx -t && sudo systemctl reload nginx
#   sudo certbot --nginx -d $domain
server {
    listen 80;
    listen [::]:80;
    server_name $domain;

    # Banner images are uploaded through the application
    client_max_body_size 20m;

    location / {
        proxy_pass http://127.0.0.1:$port;
        proxy_set_header Host \$host;
        proxy_set_header X-Real-IP \$remote_addr;
        proxy_set_header X-Forwarded-For \$proxy_add_x_forwarded_for;
        proxy_set_header X-Forwarded-Proto \$scheme;
    }
}
NGINX
}

# Starts the container of a customer. $1 = name, $2 = image, $3 = initial admin password (optional)
run_container() {
    local name="$1" image="$2" admin_password="${3:-}"
    local container router domain port
    container="$(container_name "$name")"
    router="$PREFIX-$name"
    domain="$(customer_value "$name" DOMAIN)"
    port="$(customer_value "$name" PORT)"

    local args=(
        --detach
        --name "$container"
        --restart unless-stopped
        --network "$NETWORK"
        --volume "$CUSTOMERS_DIR/$name/config.toml:/app/config/config.toml:ro"
        --volume "$(volume_name "$name"):/app/data"
        --label "rezervator.stack=$PREFIX"
        --label "rezervator.customer=$name"
        --label "rezervator.domain=$domain"
    )

    if [ "$POSTGRES" = "host" ]; then
        args+=(--add-host "host.docker.internal:host-gateway")
    fi

    if [ "$PROXY" = "nginx" ]; then
        [ -n "$port" ] || die "Customer '$name' has no PORT in customer.env"
        # Only the nginx on this host may reach the container
        args+=(--publish "127.0.0.1:$port:3000")
    elif [ "$TLS" = "true" ]; then
        args+=(
            --label "traefik.enable=true"
            --label "traefik.http.services.$router.loadbalancer.server.port=3000"
            --label "traefik.http.routers.$router.rule=Host(\`$domain\`)"
            --label "traefik.http.routers.$router.entrypoints=websecure"
            --label "traefik.http.routers.$router.tls.certresolver=le"
            --label "traefik.http.routers.$router-http.rule=Host(\`$domain\`)"
            --label "traefik.http.routers.$router-http.entrypoints=web"
            --label "traefik.http.routers.$router-http.middlewares=$router-https"
            --label "traefik.http.middlewares.$router-https.redirectscheme.scheme=https"
        )
    else
        args+=(
            --label "traefik.enable=true"
            --label "traefik.http.services.$router.loadbalancer.server.port=3000"
            --label "traefik.http.routers.$router.rule=Host(\`$domain\`)"
            --label "traefik.http.routers.$router.entrypoints=web"
        )
    fi

    if [ -n "$admin_password" ]; then
        args+=(--env "REZERVATOR_ADMIN_PASSWORD=$admin_password")
    fi

    docker run "${args[@]}" "$image" >/dev/null
}

# Waits until the container reports healthy. Returns 1 when it stops or stays unhealthy.
wait_healthy() {
    local container="$1" status
    for _ in $(seq 1 90); do
        status="$(docker inspect --format '{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container" 2>/dev/null || echo "missing")"
        case "$status" in
            "running healthy")
                # Traefik picks a new container up a moment after it turns healthy
                if [ "$PROXY" = "traefik" ]; then sleep 3; fi
                return 0 ;;
            # Image without a health check: running for a while is the best we can tell
            "running none") sleep 5
                [ "$(docker inspect --format '{{.State.Status}}' "$container" 2>/dev/null)" = "running" ] && return 0
                return 1 ;;
            "running starting") ;;
            "running unhealthy") return 1 ;;
            running*) ;;
            *) return 1 ;;
        esac
        sleep 2
    done
    return 1
}

cmd_init() {
    command -v docker >/dev/null || die "docker not found"
    command -v openssl >/dev/null || die "openssl not found"

    if [ ! -f "$ENV_FILE" ]; then
        sed "s/^POSTGRES_PASSWORD=.*/POSTGRES_PASSWORD=$(random_string 24)/" "$DEPLOY_DIR/.env.example" > "$ENV_FILE"
        chmod 600 "$ENV_FILE"
        info "Created $ENV_FILE. Review it (proxy, PostgreSQL, image) and run init again."
        exit 0
    fi

    load_env
    if [ "$PROXY" = "traefik" ] && [ "$TLS" = "true" ] && [ -z "${ACME_EMAIL:-}" ]; then
        die "REZERVATOR_TLS=true needs ACME_EMAIL in $ENV_FILE"
    fi

    mkdir -p "$CUSTOMERS_DIR" "$BACKUPS_DIR"
    chmod 700 "$CUSTOMERS_DIR" "$BACKUPS_DIR"
    compose_up
    wait_for_postgres
    if docker image inspect "$IMAGE" >/dev/null 2>&1; then
        check_db_reachable
    else
        info "Image $IMAGE is not on this host yet, build or pull it before adding customers"
    fi
    info "Ready (proxy: $PROXY, PostgreSQL: $POSTGRES)"
}

# Validates the input and creates everything a customer needs except the running container:
# database and role, config, data volume. $1 = name, $2 = domain, $3 = sender address
prepare_customer() {
    local name="$1" domain="$2" mail_from="$3"
    check_name "$name"
    [[ "$domain" =~ ^[a-z0-9]([a-z0-9.-]*[a-z0-9])?$ ]] || die "Invalid domain '$domain'"
    [[ "$mail_from" =~ ^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+$ ]] || die "Invalid sender address '$mail_from'"
    if customer_exists "$name"; then die "Customer '$name' already exists"; fi
    docker image inspect "$IMAGE" >/dev/null 2>&1 || die "Image $IMAGE not found, build or pull it first"
    wait_for_postgres
    check_db_reachable

    local db db_password dir
    db="$(db_name "$name")"
    db_password="$(random_string 24)"
    dir="$CUSTOMERS_DIR/$name"

    if [ "$(psql_admin -tAc "SELECT 1 FROM pg_database WHERE datname = '$db'")" = "1" ]; then
        die "Database $db already exists"
    fi
    if [ "$(psql_admin -tAc "SELECT 1 FROM pg_roles WHERE rolname = '$db'")" = "1" ]; then
        die "Database role $db already exists"
    fi

    info "Creating database $db"
    psql_admin <<SQL
CREATE ROLE "$db" LOGIN PASSWORD '$db_password';
CREATE DATABASE "$db" OWNER "$db";
REVOKE ALL ON DATABASE "$db" FROM PUBLIC;
SQL

    info "Writing config to $dir"
    mkdir -p "$dir"
    chmod 700 "$dir"
    sed -e "s|@SESSION_KEY@|$(random_string 48)|" \
        -e "s|@DB_USER@|$db|" \
        -e "s|@DB_PASSWORD@|$db_password|" \
        -e "s|@DB_NAME@|$db|" \
        -e "s|@DB_HOST@|$DB_HOST|" \
        -e "s|@DB_PORT@|$PG_PORT|" \
        -e "s|@MAX_CONNECTIONS@|$CUSTOMER_MAX_CONNECTIONS|" \
        -e "s|@MAIL_FROM@|$mail_from|" \
        "$DEPLOY_DIR/config.template.toml" > "$dir/config.toml"
    # The directory is closed to other users; the application in the container must be able to read the file
    chmod 644 "$dir/config.toml"

    if [ "$PROXY" = "nginx" ]; then
        # Computed before customer.env exists, so the new customer is not counted
        local port
        port="$(next_port)"
        printf 'DOMAIN=%s\nPORT=%s\n' "$domain" "$port" > "$dir/customer.env"
        write_nginx_conf "$name"
    else
        printf 'DOMAIN=%s\n' "$domain" > "$dir/customer.env"
    fi

    docker volume create "$(volume_name "$name")" >/dev/null
}

# Starts a prepared customer and waits for it. $1 = name, $2 = initial admin password (optional)
start_new_customer() {
    local name="$1" admin_password="${2:-}"
    info "Starting container $(container_name "$name")"
    run_container "$name" "$IMAGE" "$admin_password"

    if ! wait_healthy "$(container_name "$name")"; then
        docker logs --tail 20 "$(container_name "$name")" >&2 || true
        die "Container $(container_name "$name") did not become healthy. Fix the problem and run 'remove $name --purge' before creating it again."
    fi
}

print_address() {
    local name="$1" domain
    domain="$(customer_value "$name" DOMAIN)"
    echo
    if [ "$PROXY" = "nginx" ]; then
        echo "Customer '$name' is running on 127.0.0.1:$(customer_value "$name" PORT)."
        echo "To publish it at $domain, install $CUSTOMERS_DIR/$name/nginx.conf"
        echo "into nginx; the commands are at the top of that file."
    elif [ "$TLS" = "true" ]; then
        echo "Customer '$name' is running at https://$domain"
    else
        echo "Customer '$name' is running at http://$domain"
    fi
}

cmd_add() {
    [ $# -ge 2 ] || die "Usage: add <name> <domain> [from]"
    local name="$1" domain="$2" mail_from="${3:-noreply@$2}" admin_password
    load_env
    prepare_customer "$name" "$domain" "$mail_from"
    admin_password="$(random_string 9)"
    start_new_customer "$name" "$admin_password"

    print_address "$name"
    cat <<MSG

  Admin login:    admin
  Admin password: $admin_password

The password is shown only now. Hand it over and have it changed after the first login.
MSG
}

cmd_import() {
    [ $# -ge 3 ] || die "Usage: import <name> <domain> <database> [data-dir] [from]"
    local name="$1" domain="$2" source_db="$3" data_dir="${4:-}" mail_from="${5:-noreply@$2}"
    local dir timestamp
    load_env
    [[ "$source_db" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]] || die "Invalid database name '$source_db'"
    if [ -n "$data_dir" ] && [ ! -d "$data_dir" ]; then
        die "Data directory '$data_dir' not found"
    fi
    wait_for_postgres
    [ "$(psql_admin -tAc "SELECT 1 FROM pg_database WHERE datname = '$source_db'")" = "1" ] \
        || die "Source database '$source_db' not found"

    prepare_customer "$name" "$domain" "$mail_from"

    # The copy goes through the same files as a backup, so it can be restored again later
    dir="$BACKUPS_DIR/$name"
    timestamp="$(date -u +%Y%m%dT%H%M%SZ)"
    mkdir -p "$dir"
    chmod 700 "$dir"
    info "Copying database $source_db"
    pg_dump_admin --no-owner --no-privileges "$source_db" | gzip > "$dir/$timestamp-db.sql.gz"
    if [ -n "$data_dir" ]; then
        info "Copying files from $data_dir"
        # Old installations kept the config file with passwords next to the uploads
        tar czf "$dir/$timestamp-data.tar.gz" --exclude='*.toml' -C "$data_dir" .
    else
        tar czf "$dir/$timestamp-data.tar.gz" --files-from /dev/null
    fi
    restore_customer "$name" "$timestamp"

    # The accounts come with the database, so no initial admin password
    start_new_customer "$name"
    print_address "$name"
    echo
    echo "Accounts and passwords were taken over from database $source_db."
    if docker logs "$(container_name "$name")" 2>&1 | grep -q "still uses the default password"; then
        echo "WARNING: the account 'admin' still has the default password 'admin'. Change it now."
    fi
    echo "The source database and files were not changed. Stop the old installation before"
    echo "switching the domain over, otherwise bookings made there in the meantime are lost."
}

cmd_list() {
    load_env
    printf '%-20s %-34s %-7s %-18s %s\n' "CUSTOMER" "DOMAIN" "PORT" "STATE" "IMAGE"
    local name domain port state image
    for name in $(all_customers); do
        domain="$(customer_value "$name" DOMAIN)"
        port="$(customer_value "$name" PORT)"
        state="$(docker inspect --format '{{.State.Status}}{{if .State.Health}}/{{.State.Health.Status}}{{end}}' "$(container_name "$name")" 2>/dev/null || echo "no container")"
        image="$(docker inspect --format '{{.Image}}' "$(container_name "$name")" 2>/dev/null || true)"
        if [ -n "$image" ]; then
            # Tag of the image when it still has one, otherwise the short id
            image="$(docker image inspect --format '{{if .RepoTags}}{{index .RepoTags 0}}{{else}}{{slice .Id 7 19}}{{end}}' "$image" 2>/dev/null || echo "${image:7:12}")"
        else
            image="-"
        fi
        printf '%-20s %-34s %-7s %-18s %s\n' "$name" "$domain" "${port:--}" "$state" "$image"
    done
}

# Dumps the database and the data volume. $1 = name. Prints the timestamp of the backup.
backup_customer() {
    local name="$1" db dir timestamp
    db="$(db_name "$name")"
    dir="$BACKUPS_DIR/$name"
    timestamp="$(date -u +%Y%m%dT%H%M%SZ)"
    mkdir -p "$dir"
    chmod 700 "$dir"

    pg_dump_admin --no-owner --no-privileges "$db" | gzip > "$dir/$timestamp-db.sql.gz"
    # Any image with tar would do; the application image is certainly present
    docker run --rm --volume "$(volume_name "$name"):/data:ro" --entrypoint tar "$IMAGE" \
        czf - -C /data . > "$dir/$timestamp-data.tar.gz"

    # An empty or truncated dump must not pass for a backup
    gzip -t "$dir/$timestamp-db.sql.gz" && gzip -t "$dir/$timestamp-data.tar.gz"
    [ "$(gzip -dc "$dir/$timestamp-db.sql.gz" | wc -c)" -gt 0 ] || die "Database dump of '$name' is empty"

    echo "$timestamp"
}

cmd_backup() {
    [ $# -ge 1 ] || die "Usage: backup <name>...|--all"
    load_env
    wait_for_postgres
    local names=("$@") name timestamp
    if [ "$1" = "--all" ]; then
        # shellcheck disable=SC2207
        names=($(all_customers))
    fi
    [ "${#names[@]}" -gt 0 ] || { info "No customers"; return 0; }
    for name in "${names[@]}"; do
        require_customer "$name"
        timestamp="$(backup_customer "$name")"
        info "Backup of '$name': $BACKUPS_DIR/$name/$timestamp-db.sql.gz, $timestamp-data.tar.gz"
    done
}

# Replaces the database and the data volume by a backup. The container must be stopped.
restore_customer() {
    local name="$1" timestamp="$2" db dir
    db="$(db_name "$name")"
    dir="$BACKUPS_DIR/$name"

    psql_admin <<SQL
DROP DATABASE IF EXISTS "$db" WITH (FORCE);
CREATE DATABASE "$db" OWNER "$db";
REVOKE ALL ON DATABASE "$db" FROM PUBLIC;
SQL
    # Loaded by the superuser acting as the customer's role, so that role owns everything
    { echo "SET ROLE \"$db\";"; gzip -dc "$dir/$timestamp-db.sql.gz"; } | psql_admin -d "$db" >/dev/null

    docker run --rm -i --volume "$(volume_name "$name"):/data" --entrypoint sh "$IMAGE" \
        -c 'find /data -mindepth 1 -delete && tar xzf - -C /data' < "$dir/$timestamp-data.tar.gz"
}

cmd_restore() {
    [ $# -eq 2 ] || die "Usage: restore <name> <timestamp>"
    local name="$1" timestamp="$2" container image
    load_env
    require_customer "$name"
    [ -f "$BACKUPS_DIR/$name/$timestamp-db.sql.gz" ] && [ -f "$BACKUPS_DIR/$name/$timestamp-data.tar.gz" ] \
        || die "Backup $timestamp of '$name' not found in $BACKUPS_DIR/$name"
    wait_for_postgres

    container="$(container_name "$name")"
    # Keep the image the customer runs now; the configured one may be newer than the backup
    image="$(docker inspect --format '{{.Image}}' "$container" 2>/dev/null || echo "$IMAGE")"

    info "Stopping $container"
    docker rm --force "$container" >/dev/null 2>&1 || true
    info "Restoring '$name' from backup $timestamp"
    restore_customer "$name" "$timestamp"
    run_container "$name" "$image"
    wait_healthy "$container" || die "Container $container is not healthy after the restore, see 'docker logs $container'"
    info "Customer '$name' restored and running"
}

cmd_upgrade() {
    [ $# -ge 1 ] || die "Usage: upgrade <name>...|--all"
    load_env
    docker image inspect "$IMAGE" >/dev/null 2>&1 || die "Image $IMAGE not found, build or pull it first"
    wait_for_postgres

    local names=("$@") name container old_image new_image timestamp
    if [ "$1" = "--all" ]; then
        # shellcheck disable=SC2207
        names=($(all_customers))
    fi
    [ "${#names[@]}" -gt 0 ] || { info "No customers"; return 0; }
    new_image="$(docker image inspect --format '{{.Id}}' "$IMAGE")"

    for name in "${names[@]}"; do
        require_customer "$name"
        container="$(container_name "$name")"
        old_image="$(docker inspect --format '{{.Image}}' "$container" 2>/dev/null || true)"

        if [ "$old_image" = "$new_image" ]; then
            info "'$name' already runs $IMAGE, skipping"
            continue
        fi

        info "Upgrading '$name'"
        timestamp="$(backup_customer "$name")"
        docker rm --force "$container" >/dev/null 2>&1 || true
        run_container "$name" "$IMAGE"

        if wait_healthy "$container"; then
            info "'$name' upgraded (backup before the upgrade: $timestamp)"
            continue
        fi

        echo "Upgrade of '$name' failed, last log lines of the new container:" >&2
        docker logs --tail 20 "$container" >&2 || true
        docker rm --force "$container" >/dev/null 2>&1 || true

        if [ -z "$old_image" ]; then
            die "'$name' had no container before, nothing to roll back to. Backup: $timestamp"
        fi

        # Migrations of the new version may have changed the database, so the old version
        # gets the database from before the upgrade
        info "Rolling '$name' back to the previous image and to backup $timestamp"
        restore_customer "$name" "$timestamp"
        run_container "$name" "$old_image"
        if wait_healthy "$container"; then
            die "'$name' was rolled back. Upgrade stopped, the remaining customers were not touched."
        fi
        die "'$name' is DOWN: the rollback did not become healthy either. Backup: $timestamp"
    done
}

cmd_remove() {
    [ $# -ge 1 ] || die "Usage: remove <name> [--purge]"
    local name="$1" purge="${2:-}" db answer
    load_env
    require_customer "$name"

    info "Removing container $(container_name "$name")"
    docker rm --force "$(container_name "$name")" >/dev/null 2>&1 || true

    if [ "$purge" != "--purge" ]; then
        info "Database, data volume and config of '$name' were kept. Start it again with 'upgrade $name'."
        return 0
    fi

    read -r -p "This deletes the database, uploaded files and config of '$name'. Type the name to confirm: " answer
    [ "$answer" = "$name" ] || die "Not confirmed, nothing else was deleted"

    wait_for_postgres
    db="$(db_name "$name")"
    psql_admin <<SQL
DROP DATABASE IF EXISTS "$db" WITH (FORCE);
DROP ROLE IF EXISTS "$db";
SQL
    docker volume rm "$(volume_name "$name")" >/dev/null 2>&1 || true
    rm -rf "${CUSTOMERS_DIR:?}/$name"
    info "Customer '$name' deleted. Backups in $BACKUPS_DIR/$name were kept."
    if [ "$PROXY" = "nginx" ]; then
        info "Remove its server block from nginx as well."
    fi
}

main() {
    [ $# -ge 1 ] || { usage; exit 1; }
    local command="$1"
    shift
    case "$command" in
        init) cmd_init "$@" ;;
        add) cmd_add "$@" ;;
        import) cmd_import "$@" ;;
        list) cmd_list "$@" ;;
        upgrade) cmd_upgrade "$@" ;;
        backup) cmd_backup "$@" ;;
        restore) cmd_restore "$@" ;;
        remove) cmd_remove "$@" ;;
        -h|--help|help) usage ;;
        *) usage; exit 1 ;;
    esac
}

main "$@"