#!/usr/bin/env bash # # Management of a Rezervovator stack with one container per customer. # See README.md in this directory. set -euo pipefail DEPLOY_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ENV_FILE="$DEPLOY_DIR/.env" CUSTOMERS_DIR="$DEPLOY_DIR/customers" BACKUPS_DIR="$DEPLOY_DIR/backups" usage() { cat < [arguments] init Create .env and start the proxy and PostgreSQL add [from] Create a customer: database, config, volume, container. [from] is the sender address of e-mails, default noreply@ list List customers and the state of their containers upgrade ...|--all Recreate containers from REZERVATOR_IMAGE, one by one. Backs up first, rolls back and stops when one fails. backup ...|--all Dump the database and the data volume to backups// restore Restore a customer from backups//-* remove [--purge] Stop and remove the container. With --purge also the database, the data volume and the config (asks for confirmation). is lowercase letters, digits and dashes, starting with a letter. EOF } die() { echo "Error: $*" >&2 exit 1 } info() { echo "==> $*" } random_string() { # $1 = number of bytes, output is twice as many hex characters openssl rand -hex "$1" } load_env() { [ -f "$ENV_FILE" ] || die "$ENV_FILE not found, run '$(basename "$0") init' first" set -a # shellcheck disable=SC1090 . "$ENV_FILE" set +a PREFIX="${REZERVATOR_PREFIX:-rezervator}" IMAGE="${REZERVATOR_IMAGE:-rezervator:latest}" TLS="${REZERVATOR_TLS:-true}" PG_CONTAINER="$PREFIX-postgres" NETWORK="$PREFIX" CUSTOMER_MAX_CONNECTIONS="${CUSTOMER_MAX_CONNECTIONS:-5}" } compose() { docker compose --project-directory "$DEPLOY_DIR" --env-file "$ENV_FILE" "$@" } check_name() { [[ "$1" =~ ^[a-z][a-z0-9-]{0,30}$ ]] || die "Invalid customer name '$1'" } container_name() { echo "$PREFIX-app-$1"; } volume_name() { echo "$PREFIX-data-$1"; } # Database and role have the same name db_name() { echo "rz_${1//-/_}"; } customer_exists() { [ -f "$CUSTOMERS_DIR/$1/customer.env" ] } require_customer() { check_name "$1" customer_exists "$1" || die "Unknown customer '$1'" } all_customers() { [ -d "$CUSTOMERS_DIR" ] || return 0 for dir in "$CUSTOMERS_DIR"/*/; do [ -f "${dir}customer.env" ] && basename "$dir" done return 0 } psql_admin() { docker exec -i "$PG_CONTAINER" psql -U postgres -v ON_ERROR_STOP=1 -q "$@" } wait_for_postgres() { for _ in $(seq 1 60); do # Over TCP: while the image initializes a new cluster, its temporary server listens on the socket only if docker exec "$PG_CONTAINER" pg_isready -h 127.0.0.1 -U postgres -q 2>/dev/null; then return 0 fi sleep 1 done die "PostgreSQL in container $PG_CONTAINER is not ready" } # Starts the container of a customer. $1 = name, $2 = image, $3 = initial admin password (optional) run_container() { local name="$1" image="$2" admin_password="${3:-}" local container router domain container="$(container_name "$name")" router="$PREFIX-$name" # shellcheck disable=SC1090 domain="$(. "$CUSTOMERS_DIR/$name/customer.env" && echo "$DOMAIN")" local args=( --detach --name "$container" --restart unless-stopped --network "$NETWORK" --volume "$CUSTOMERS_DIR/$name/config.toml:/app/config/config.toml:ro" --volume "$(volume_name "$name"):/app/data" --label "rezervator.stack=$PREFIX" --label "rezervator.customer=$name" --label "rezervator.domain=$domain" --label "traefik.enable=true" --label "traefik.http.services.$router.loadbalancer.server.port=3000" ) if [ "$TLS" = "true" ]; then args+=( --label "traefik.http.routers.$router.rule=Host(\`$domain\`)" --label "traefik.http.routers.$router.entrypoints=websecure" --label "traefik.http.routers.$router.tls.certresolver=le" --label "traefik.http.routers.$router-http.rule=Host(\`$domain\`)" --label "traefik.http.routers.$router-http.entrypoints=web" --label "traefik.http.routers.$router-http.middlewares=$router-https" --label "traefik.http.middlewares.$router-https.redirectscheme.scheme=https" ) else args+=( --label "traefik.http.routers.$router.rule=Host(\`$domain\`)" --label "traefik.http.routers.$router.entrypoints=web" ) fi if [ -n "$admin_password" ]; then args+=(--env "REZERVATOR_ADMIN_PASSWORD=$admin_password") fi docker run "${args[@]}" "$image" >/dev/null } # Waits until the container reports healthy. Returns 1 when it stops or stays unhealthy. wait_healthy() { local container="$1" status for _ in $(seq 1 90); do status="$(docker inspect --format '{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container" 2>/dev/null || echo "missing")" case "$status" in # The proxy picks a new container up a moment after it turns healthy "running healthy") sleep 3; return 0 ;; # Image without a health check: running for a while is the best we can tell "running none") sleep 5 [ "$(docker inspect --format '{{.State.Status}}' "$container" 2>/dev/null)" = "running" ] && return 0 return 1 ;; "running starting") ;; "running unhealthy") return 1 ;; running*) ;; *) return 1 ;; esac sleep 2 done return 1 } cmd_init() { command -v docker >/dev/null || die "docker not found" command -v openssl >/dev/null || die "openssl not found" if [ ! -f "$ENV_FILE" ]; then sed "s/^POSTGRES_PASSWORD=.*/POSTGRES_PASSWORD=$(random_string 24)/" "$DEPLOY_DIR/.env.example" > "$ENV_FILE" chmod 600 "$ENV_FILE" info "Created $ENV_FILE with a generated PostgreSQL password. Review it (image, TLS, ACME_EMAIL) and run init again." exit 0 fi load_env if [ "$TLS" = "true" ] && [ -z "${ACME_EMAIL:-}" ]; then die "REZERVATOR_TLS=true needs ACME_EMAIL in $ENV_FILE" fi mkdir -p "$CUSTOMERS_DIR" "$BACKUPS_DIR" chmod 700 "$CUSTOMERS_DIR" "$BACKUPS_DIR" compose up --detach wait_for_postgres info "Proxy and PostgreSQL are running" } cmd_add() { [ $# -ge 2 ] || die "Usage: add [from]" local name="$1" domain="$2" mail_from="${3:-noreply@$2}" load_env check_name "$name" [[ "$domain" =~ ^[a-z0-9]([a-z0-9.-]*[a-z0-9])?$ ]] || die "Invalid domain '$domain'" [[ "$mail_from" =~ ^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+$ ]] || die "Invalid sender address '$mail_from'" customer_exists "$name" && die "Customer '$name' already exists" docker image inspect "$IMAGE" >/dev/null 2>&1 || die "Image $IMAGE not found, build or pull it first" wait_for_postgres local db db_password admin_password dir db="$(db_name "$name")" db_password="$(random_string 24)" admin_password="$(random_string 9)" dir="$CUSTOMERS_DIR/$name" if [ "$(psql_admin -tAc "SELECT 1 FROM pg_database WHERE datname = '$db'")" = "1" ]; then die "Database $db already exists" fi info "Creating database $db" psql_admin < "$dir/config.toml" # The application in the container runs as another user and must be able to read it chmod 644 "$dir/config.toml" printf 'DOMAIN=%s\n' "$domain" > "$dir/customer.env" info "Starting container $(container_name "$name")" docker volume create "$(volume_name "$name")" >/dev/null run_container "$name" "$IMAGE" "$admin_password" if ! wait_healthy "$(container_name "$name")"; then docker logs --tail 20 "$(container_name "$name")" >&2 || true die "Container $(container_name "$name") did not become healthy. Fix the problem and run 'remove $name --purge' before adding it again." fi local scheme="http" [ "$TLS" = "true" ] && scheme="https" cat </dev/null || echo "no container")" image="$(docker inspect --format '{{.Image}}' "$(container_name "$name")" 2>/dev/null || true)" if [ -n "$image" ]; then # Tag of the image when it still has one, otherwise the short id image="$(docker image inspect --format '{{if .RepoTags}}{{index .RepoTags 0}}{{else}}{{slice .Id 7 19}}{{end}}' "$image" 2>/dev/null || echo "${image:7:12}")" else image="-" fi printf '%-20s %-34s %-12s %s\n' "$name" "$domain" "$state" "$image" done } # Dumps the database and the data volume. $1 = name. Prints the timestamp of the backup. backup_customer() { local name="$1" db dir timestamp db="$(db_name "$name")" dir="$BACKUPS_DIR/$name" timestamp="$(date -u +%Y%m%dT%H%M%SZ)" mkdir -p "$dir" chmod 700 "$dir" docker exec "$PG_CONTAINER" pg_dump -U postgres --no-owner --no-privileges "$db" | gzip > "$dir/$timestamp-db.sql.gz" # Any image with tar would do; the application image is certainly present docker run --rm --volume "$(volume_name "$name"):/data:ro" --entrypoint tar "$IMAGE" \ czf - -C /data . > "$dir/$timestamp-data.tar.gz" # An empty or truncated dump must not pass for a backup gzip -t "$dir/$timestamp-db.sql.gz" && gzip -t "$dir/$timestamp-data.tar.gz" [ "$(gzip -dc "$dir/$timestamp-db.sql.gz" | wc -c)" -gt 0 ] || die "Database dump of '$name' is empty" echo "$timestamp" } cmd_backup() { [ $# -ge 1 ] || die "Usage: backup ...|--all" load_env wait_for_postgres local names=("$@") name timestamp if [ "$1" = "--all" ]; then # shellcheck disable=SC2207 names=($(all_customers)) fi [ "${#names[@]}" -gt 0 ] || { info "No customers"; return 0; } for name in "${names[@]}"; do require_customer "$name" timestamp="$(backup_customer "$name")" info "Backup of '$name': $BACKUPS_DIR/$name/$timestamp-db.sql.gz, $timestamp-data.tar.gz" done } # Replaces the database and the data volume by a backup. The container must be stopped. restore_customer() { local name="$1" timestamp="$2" db dir db="$(db_name "$name")" dir="$BACKUPS_DIR/$name" psql_admin </dev/null docker run --rm -i --volume "$(volume_name "$name"):/data" --entrypoint sh "$IMAGE" \ -c 'find /data -mindepth 1 -delete && tar xzf - -C /data' < "$dir/$timestamp-data.tar.gz" } cmd_restore() { [ $# -eq 2 ] || die "Usage: restore " local name="$1" timestamp="$2" container image load_env require_customer "$name" [ -f "$BACKUPS_DIR/$name/$timestamp-db.sql.gz" ] && [ -f "$BACKUPS_DIR/$name/$timestamp-data.tar.gz" ] \ || die "Backup $timestamp of '$name' not found in $BACKUPS_DIR/$name" wait_for_postgres container="$(container_name "$name")" # Keep the image the customer runs now; the configured one may be newer than the backup image="$(docker inspect --format '{{.Image}}' "$container" 2>/dev/null || echo "$IMAGE")" info "Stopping $container" docker rm --force "$container" >/dev/null 2>&1 || true info "Restoring '$name' from backup $timestamp" restore_customer "$name" "$timestamp" run_container "$name" "$image" wait_healthy "$container" || die "Container $container is not healthy after the restore, see 'docker logs $container'" info "Customer '$name' restored and running" } cmd_upgrade() { [ $# -ge 1 ] || die "Usage: upgrade ...|--all" load_env docker image inspect "$IMAGE" >/dev/null 2>&1 || die "Image $IMAGE not found, build or pull it first" wait_for_postgres local names=("$@") name container old_image new_image timestamp if [ "$1" = "--all" ]; then # shellcheck disable=SC2207 names=($(all_customers)) fi [ "${#names[@]}" -gt 0 ] || { info "No customers"; return 0; } new_image="$(docker image inspect --format '{{.Id}}' "$IMAGE")" for name in "${names[@]}"; do require_customer "$name" container="$(container_name "$name")" old_image="$(docker inspect --format '{{.Image}}' "$container" 2>/dev/null || true)" if [ "$old_image" = "$new_image" ]; then info "'$name' already runs $IMAGE, skipping" continue fi info "Upgrading '$name'" timestamp="$(backup_customer "$name")" docker rm --force "$container" >/dev/null 2>&1 || true run_container "$name" "$IMAGE" if wait_healthy "$container"; then info "'$name' upgraded (backup before the upgrade: $timestamp)" continue fi echo "Upgrade of '$name' failed, last log lines of the new container:" >&2 docker logs --tail 20 "$container" >&2 || true docker rm --force "$container" >/dev/null 2>&1 || true if [ -z "$old_image" ]; then die "'$name' had no container before, nothing to roll back to. Backup: $timestamp" fi # Migrations of the new version may have changed the database, so the old version # gets the database from before the upgrade info "Rolling '$name' back to the previous image and to backup $timestamp" restore_customer "$name" "$timestamp" run_container "$name" "$old_image" if wait_healthy "$container"; then die "'$name' was rolled back. Upgrade stopped, the remaining customers were not touched." fi die "'$name' is DOWN: the rollback did not become healthy either. Backup: $timestamp" done } cmd_remove() { [ $# -ge 1 ] || die "Usage: remove [--purge]" local name="$1" purge="${2:-}" db answer load_env require_customer "$name" info "Removing container $(container_name "$name")" docker rm --force "$(container_name "$name")" >/dev/null 2>&1 || true if [ "$purge" != "--purge" ]; then info "Database, data volume and config of '$name' were kept. Start it again with 'upgrade $name'." return 0 fi read -r -p "This deletes the database, uploaded files and config of '$name'. Type the name to confirm: " answer [ "$answer" = "$name" ] || die "Not confirmed, nothing else was deleted" wait_for_postgres db="$(db_name "$name")" psql_admin </dev/null 2>&1 || true rm -rf "${CUSTOMERS_DIR:?}/$name" info "Customer '$name' deleted. Backups in $BACKUPS_DIR/$name were kept." } main() { [ $# -ge 1 ] || { usage; exit 1; } local command="$1" shift case "$command" in init) cmd_init "$@" ;; add) cmd_add "$@" ;; list) cmd_list "$@" ;; upgrade) cmd_upgrade "$@" ;; backup) cmd_backup "$@" ;; restore) cmd_restore "$@" ;; remove) cmd_remove "$@" ;; -h|--help|help) usage ;; *) usage; exit 1 ;; esac } main "$@"