459 lines
16 KiB
Bash
Executable File
459 lines
16 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
#
|
|
# Management of a Rezervovator stack with one container per customer.
|
|
# See README.md in this directory.
|
|
|
|
set -euo pipefail
|
|
|
|
DEPLOY_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
ENV_FILE="$DEPLOY_DIR/.env"
|
|
CUSTOMERS_DIR="$DEPLOY_DIR/customers"
|
|
BACKUPS_DIR="$DEPLOY_DIR/backups"
|
|
|
|
usage() {
|
|
cat <<EOF
|
|
Usage: $(basename "$0") <command> [arguments]
|
|
|
|
init Create .env and start the proxy and PostgreSQL
|
|
add <name> <domain> [from] Create a customer: database, config, volume, container.
|
|
[from] is the sender address of e-mails, default noreply@<domain>
|
|
list List customers and the state of their containers
|
|
upgrade <name>...|--all Recreate containers from REZERVATOR_IMAGE, one by one.
|
|
Backs up first, rolls back and stops when one fails.
|
|
backup <name>...|--all Dump the database and the data volume to backups/<name>/
|
|
restore <name> <timestamp> Restore a customer from backups/<name>/<timestamp>-*
|
|
remove <name> [--purge] Stop and remove the container. With --purge also the database,
|
|
the data volume and the config (asks for confirmation).
|
|
|
|
<name> is lowercase letters, digits and dashes, starting with a letter.
|
|
EOF
|
|
}
|
|
|
|
die() {
|
|
echo "Error: $*" >&2
|
|
exit 1
|
|
}
|
|
|
|
info() {
|
|
echo "==> $*"
|
|
}
|
|
|
|
random_string() {
|
|
# $1 = number of bytes, output is twice as many hex characters
|
|
openssl rand -hex "$1"
|
|
}
|
|
|
|
load_env() {
|
|
[ -f "$ENV_FILE" ] || die "$ENV_FILE not found, run '$(basename "$0") init' first"
|
|
set -a
|
|
# shellcheck disable=SC1090
|
|
. "$ENV_FILE"
|
|
set +a
|
|
|
|
PREFIX="${REZERVATOR_PREFIX:-rezervator}"
|
|
IMAGE="${REZERVATOR_IMAGE:-rezervator:latest}"
|
|
TLS="${REZERVATOR_TLS:-true}"
|
|
PG_CONTAINER="$PREFIX-postgres"
|
|
NETWORK="$PREFIX"
|
|
CUSTOMER_MAX_CONNECTIONS="${CUSTOMER_MAX_CONNECTIONS:-5}"
|
|
}
|
|
|
|
compose() {
|
|
docker compose --project-directory "$DEPLOY_DIR" --env-file "$ENV_FILE" "$@"
|
|
}
|
|
|
|
check_name() {
|
|
[[ "$1" =~ ^[a-z][a-z0-9-]{0,30}$ ]] || die "Invalid customer name '$1'"
|
|
}
|
|
|
|
container_name() { echo "$PREFIX-app-$1"; }
|
|
volume_name() { echo "$PREFIX-data-$1"; }
|
|
# Database and role have the same name
|
|
db_name() { echo "rz_${1//-/_}"; }
|
|
|
|
customer_exists() {
|
|
[ -f "$CUSTOMERS_DIR/$1/customer.env" ]
|
|
}
|
|
|
|
require_customer() {
|
|
check_name "$1"
|
|
customer_exists "$1" || die "Unknown customer '$1'"
|
|
}
|
|
|
|
all_customers() {
|
|
[ -d "$CUSTOMERS_DIR" ] || return 0
|
|
for dir in "$CUSTOMERS_DIR"/*/; do
|
|
[ -f "${dir}customer.env" ] && basename "$dir"
|
|
done
|
|
return 0
|
|
}
|
|
|
|
psql_admin() {
|
|
docker exec -i "$PG_CONTAINER" psql -U postgres -v ON_ERROR_STOP=1 -q "$@"
|
|
}
|
|
|
|
wait_for_postgres() {
|
|
for _ in $(seq 1 60); do
|
|
# Over TCP: while the image initializes a new cluster, its temporary server listens on the socket only
|
|
if docker exec "$PG_CONTAINER" pg_isready -h 127.0.0.1 -U postgres -q 2>/dev/null; then
|
|
return 0
|
|
fi
|
|
sleep 1
|
|
done
|
|
die "PostgreSQL in container $PG_CONTAINER is not ready"
|
|
}
|
|
|
|
# Starts the container of a customer. $1 = name, $2 = image, $3 = initial admin password (optional)
|
|
run_container() {
|
|
local name="$1" image="$2" admin_password="${3:-}"
|
|
local container router domain
|
|
container="$(container_name "$name")"
|
|
router="$PREFIX-$name"
|
|
# shellcheck disable=SC1090
|
|
domain="$(. "$CUSTOMERS_DIR/$name/customer.env" && echo "$DOMAIN")"
|
|
|
|
local args=(
|
|
--detach
|
|
--name "$container"
|
|
--restart unless-stopped
|
|
--network "$NETWORK"
|
|
--volume "$CUSTOMERS_DIR/$name/config.toml:/app/config/config.toml:ro"
|
|
--volume "$(volume_name "$name"):/app/data"
|
|
--label "rezervator.stack=$PREFIX"
|
|
--label "rezervator.customer=$name"
|
|
--label "rezervator.domain=$domain"
|
|
--label "traefik.enable=true"
|
|
--label "traefik.http.services.$router.loadbalancer.server.port=3000"
|
|
)
|
|
|
|
if [ "$TLS" = "true" ]; then
|
|
args+=(
|
|
--label "traefik.http.routers.$router.rule=Host(\`$domain\`)"
|
|
--label "traefik.http.routers.$router.entrypoints=websecure"
|
|
--label "traefik.http.routers.$router.tls.certresolver=le"
|
|
--label "traefik.http.routers.$router-http.rule=Host(\`$domain\`)"
|
|
--label "traefik.http.routers.$router-http.entrypoints=web"
|
|
--label "traefik.http.routers.$router-http.middlewares=$router-https"
|
|
--label "traefik.http.middlewares.$router-https.redirectscheme.scheme=https"
|
|
)
|
|
else
|
|
args+=(
|
|
--label "traefik.http.routers.$router.rule=Host(\`$domain\`)"
|
|
--label "traefik.http.routers.$router.entrypoints=web"
|
|
)
|
|
fi
|
|
|
|
if [ -n "$admin_password" ]; then
|
|
args+=(--env "REZERVATOR_ADMIN_PASSWORD=$admin_password")
|
|
fi
|
|
|
|
docker run "${args[@]}" "$image" >/dev/null
|
|
}
|
|
|
|
# Waits until the container reports healthy. Returns 1 when it stops or stays unhealthy.
|
|
wait_healthy() {
|
|
local container="$1" status
|
|
for _ in $(seq 1 90); do
|
|
status="$(docker inspect --format '{{.State.Status}} {{if .State.Health}}{{.State.Health.Status}}{{else}}none{{end}}' "$container" 2>/dev/null || echo "missing")"
|
|
case "$status" in
|
|
# The proxy picks a new container up a moment after it turns healthy
|
|
"running healthy") sleep 3; return 0 ;;
|
|
# Image without a health check: running for a while is the best we can tell
|
|
"running none") sleep 5
|
|
[ "$(docker inspect --format '{{.State.Status}}' "$container" 2>/dev/null)" = "running" ] && return 0
|
|
return 1 ;;
|
|
"running starting") ;;
|
|
"running unhealthy") return 1 ;;
|
|
running*) ;;
|
|
*) return 1 ;;
|
|
esac
|
|
sleep 2
|
|
done
|
|
return 1
|
|
}
|
|
|
|
cmd_init() {
|
|
command -v docker >/dev/null || die "docker not found"
|
|
command -v openssl >/dev/null || die "openssl not found"
|
|
|
|
if [ ! -f "$ENV_FILE" ]; then
|
|
sed "s/^POSTGRES_PASSWORD=.*/POSTGRES_PASSWORD=$(random_string 24)/" "$DEPLOY_DIR/.env.example" > "$ENV_FILE"
|
|
chmod 600 "$ENV_FILE"
|
|
info "Created $ENV_FILE with a generated PostgreSQL password. Review it (image, TLS, ACME_EMAIL) and run init again."
|
|
exit 0
|
|
fi
|
|
|
|
load_env
|
|
if [ "$TLS" = "true" ] && [ -z "${ACME_EMAIL:-}" ]; then
|
|
die "REZERVATOR_TLS=true needs ACME_EMAIL in $ENV_FILE"
|
|
fi
|
|
|
|
mkdir -p "$CUSTOMERS_DIR" "$BACKUPS_DIR"
|
|
chmod 700 "$CUSTOMERS_DIR" "$BACKUPS_DIR"
|
|
compose up --detach
|
|
wait_for_postgres
|
|
info "Proxy and PostgreSQL are running"
|
|
}
|
|
|
|
cmd_add() {
|
|
[ $# -ge 2 ] || die "Usage: add <name> <domain> [from]"
|
|
local name="$1" domain="$2" mail_from="${3:-noreply@$2}"
|
|
load_env
|
|
check_name "$name"
|
|
[[ "$domain" =~ ^[a-z0-9]([a-z0-9.-]*[a-z0-9])?$ ]] || die "Invalid domain '$domain'"
|
|
[[ "$mail_from" =~ ^[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+$ ]] || die "Invalid sender address '$mail_from'"
|
|
customer_exists "$name" && die "Customer '$name' already exists"
|
|
docker image inspect "$IMAGE" >/dev/null 2>&1 || die "Image $IMAGE not found, build or pull it first"
|
|
wait_for_postgres
|
|
|
|
local db db_password admin_password dir
|
|
db="$(db_name "$name")"
|
|
db_password="$(random_string 24)"
|
|
admin_password="$(random_string 9)"
|
|
dir="$CUSTOMERS_DIR/$name"
|
|
|
|
if [ "$(psql_admin -tAc "SELECT 1 FROM pg_database WHERE datname = '$db'")" = "1" ]; then
|
|
die "Database $db already exists"
|
|
fi
|
|
|
|
info "Creating database $db"
|
|
psql_admin <<EOF
|
|
CREATE ROLE "$db" LOGIN PASSWORD '$db_password';
|
|
CREATE DATABASE "$db" OWNER "$db";
|
|
REVOKE ALL ON DATABASE "$db" FROM PUBLIC;
|
|
EOF
|
|
|
|
info "Writing config to $dir"
|
|
mkdir -p "$dir"
|
|
chmod 700 "$dir"
|
|
sed -e "s|@SESSION_KEY@|$(random_string 48)|" \
|
|
-e "s|@DB_USER@|$db|" \
|
|
-e "s|@DB_PASSWORD@|$db_password|" \
|
|
-e "s|@DB_NAME@|$db|" \
|
|
-e "s|@DB_HOST@|$PG_CONTAINER|" \
|
|
-e "s|@MAX_CONNECTIONS@|$CUSTOMER_MAX_CONNECTIONS|" \
|
|
-e "s|@MAIL_FROM@|$mail_from|" \
|
|
"$DEPLOY_DIR/config.template.toml" > "$dir/config.toml"
|
|
# The application in the container runs as another user and must be able to read it
|
|
chmod 644 "$dir/config.toml"
|
|
printf 'DOMAIN=%s\n' "$domain" > "$dir/customer.env"
|
|
|
|
info "Starting container $(container_name "$name")"
|
|
docker volume create "$(volume_name "$name")" >/dev/null
|
|
run_container "$name" "$IMAGE" "$admin_password"
|
|
|
|
if ! wait_healthy "$(container_name "$name")"; then
|
|
docker logs --tail 20 "$(container_name "$name")" >&2 || true
|
|
die "Container $(container_name "$name") did not become healthy. Fix the problem and run 'remove $name --purge' before adding it again."
|
|
fi
|
|
|
|
local scheme="http"
|
|
[ "$TLS" = "true" ] && scheme="https"
|
|
cat <<EOF
|
|
|
|
Customer '$name' is running at $scheme://$domain
|
|
|
|
Admin login: admin
|
|
Admin password: $admin_password
|
|
|
|
The password is shown only now. Hand it over and have it changed after the first login.
|
|
EOF
|
|
}
|
|
|
|
cmd_list() {
|
|
load_env
|
|
printf '%-20s %-34s %-12s %s\n' "CUSTOMER" "DOMAIN" "STATE" "IMAGE"
|
|
local name domain state image
|
|
for name in $(all_customers); do
|
|
# shellcheck disable=SC1090
|
|
domain="$(. "$CUSTOMERS_DIR/$name/customer.env" && echo "$DOMAIN")"
|
|
state="$(docker inspect --format '{{.State.Status}}{{if .State.Health}}/{{.State.Health.Status}}{{end}}' "$(container_name "$name")" 2>/dev/null || echo "no container")"
|
|
image="$(docker inspect --format '{{.Image}}' "$(container_name "$name")" 2>/dev/null || true)"
|
|
if [ -n "$image" ]; then
|
|
# Tag of the image when it still has one, otherwise the short id
|
|
image="$(docker image inspect --format '{{if .RepoTags}}{{index .RepoTags 0}}{{else}}{{slice .Id 7 19}}{{end}}' "$image" 2>/dev/null || echo "${image:7:12}")"
|
|
else
|
|
image="-"
|
|
fi
|
|
printf '%-20s %-34s %-12s %s\n' "$name" "$domain" "$state" "$image"
|
|
done
|
|
}
|
|
|
|
# Dumps the database and the data volume. $1 = name. Prints the timestamp of the backup.
|
|
backup_customer() {
|
|
local name="$1" db dir timestamp
|
|
db="$(db_name "$name")"
|
|
dir="$BACKUPS_DIR/$name"
|
|
timestamp="$(date -u +%Y%m%dT%H%M%SZ)"
|
|
mkdir -p "$dir"
|
|
chmod 700 "$dir"
|
|
|
|
docker exec "$PG_CONTAINER" pg_dump -U postgres --no-owner --no-privileges "$db" | gzip > "$dir/$timestamp-db.sql.gz"
|
|
# Any image with tar would do; the application image is certainly present
|
|
docker run --rm --volume "$(volume_name "$name"):/data:ro" --entrypoint tar "$IMAGE" \
|
|
czf - -C /data . > "$dir/$timestamp-data.tar.gz"
|
|
|
|
# An empty or truncated dump must not pass for a backup
|
|
gzip -t "$dir/$timestamp-db.sql.gz" && gzip -t "$dir/$timestamp-data.tar.gz"
|
|
[ "$(gzip -dc "$dir/$timestamp-db.sql.gz" | wc -c)" -gt 0 ] || die "Database dump of '$name' is empty"
|
|
|
|
echo "$timestamp"
|
|
}
|
|
|
|
cmd_backup() {
|
|
[ $# -ge 1 ] || die "Usage: backup <name>...|--all"
|
|
load_env
|
|
wait_for_postgres
|
|
local names=("$@") name timestamp
|
|
if [ "$1" = "--all" ]; then
|
|
# shellcheck disable=SC2207
|
|
names=($(all_customers))
|
|
fi
|
|
[ "${#names[@]}" -gt 0 ] || { info "No customers"; return 0; }
|
|
for name in "${names[@]}"; do
|
|
require_customer "$name"
|
|
timestamp="$(backup_customer "$name")"
|
|
info "Backup of '$name': $BACKUPS_DIR/$name/$timestamp-db.sql.gz, $timestamp-data.tar.gz"
|
|
done
|
|
}
|
|
|
|
# Replaces the database and the data volume by a backup. The container must be stopped.
|
|
restore_customer() {
|
|
local name="$1" timestamp="$2" db dir
|
|
db="$(db_name "$name")"
|
|
dir="$BACKUPS_DIR/$name"
|
|
|
|
psql_admin <<EOF
|
|
DROP DATABASE IF EXISTS "$db" WITH (FORCE);
|
|
CREATE DATABASE "$db" OWNER "$db";
|
|
REVOKE ALL ON DATABASE "$db" FROM PUBLIC;
|
|
EOF
|
|
gzip -dc "$dir/$timestamp-db.sql.gz" | docker exec -i "$PG_CONTAINER" psql -U "$db" -d "$db" -v ON_ERROR_STOP=1 -q >/dev/null
|
|
|
|
docker run --rm -i --volume "$(volume_name "$name"):/data" --entrypoint sh "$IMAGE" \
|
|
-c 'find /data -mindepth 1 -delete && tar xzf - -C /data' < "$dir/$timestamp-data.tar.gz"
|
|
}
|
|
|
|
cmd_restore() {
|
|
[ $# -eq 2 ] || die "Usage: restore <name> <timestamp>"
|
|
local name="$1" timestamp="$2" container image
|
|
load_env
|
|
require_customer "$name"
|
|
[ -f "$BACKUPS_DIR/$name/$timestamp-db.sql.gz" ] && [ -f "$BACKUPS_DIR/$name/$timestamp-data.tar.gz" ] \
|
|
|| die "Backup $timestamp of '$name' not found in $BACKUPS_DIR/$name"
|
|
wait_for_postgres
|
|
|
|
container="$(container_name "$name")"
|
|
# Keep the image the customer runs now; the configured one may be newer than the backup
|
|
image="$(docker inspect --format '{{.Image}}' "$container" 2>/dev/null || echo "$IMAGE")"
|
|
|
|
info "Stopping $container"
|
|
docker rm --force "$container" >/dev/null 2>&1 || true
|
|
info "Restoring '$name' from backup $timestamp"
|
|
restore_customer "$name" "$timestamp"
|
|
run_container "$name" "$image"
|
|
wait_healthy "$container" || die "Container $container is not healthy after the restore, see 'docker logs $container'"
|
|
info "Customer '$name' restored and running"
|
|
}
|
|
|
|
cmd_upgrade() {
|
|
[ $# -ge 1 ] || die "Usage: upgrade <name>...|--all"
|
|
load_env
|
|
docker image inspect "$IMAGE" >/dev/null 2>&1 || die "Image $IMAGE not found, build or pull it first"
|
|
wait_for_postgres
|
|
|
|
local names=("$@") name container old_image new_image timestamp
|
|
if [ "$1" = "--all" ]; then
|
|
# shellcheck disable=SC2207
|
|
names=($(all_customers))
|
|
fi
|
|
[ "${#names[@]}" -gt 0 ] || { info "No customers"; return 0; }
|
|
new_image="$(docker image inspect --format '{{.Id}}' "$IMAGE")"
|
|
|
|
for name in "${names[@]}"; do
|
|
require_customer "$name"
|
|
container="$(container_name "$name")"
|
|
old_image="$(docker inspect --format '{{.Image}}' "$container" 2>/dev/null || true)"
|
|
|
|
if [ "$old_image" = "$new_image" ]; then
|
|
info "'$name' already runs $IMAGE, skipping"
|
|
continue
|
|
fi
|
|
|
|
info "Upgrading '$name'"
|
|
timestamp="$(backup_customer "$name")"
|
|
docker rm --force "$container" >/dev/null 2>&1 || true
|
|
run_container "$name" "$IMAGE"
|
|
|
|
if wait_healthy "$container"; then
|
|
info "'$name' upgraded (backup before the upgrade: $timestamp)"
|
|
continue
|
|
fi
|
|
|
|
echo "Upgrade of '$name' failed, last log lines of the new container:" >&2
|
|
docker logs --tail 20 "$container" >&2 || true
|
|
docker rm --force "$container" >/dev/null 2>&1 || true
|
|
|
|
if [ -z "$old_image" ]; then
|
|
die "'$name' had no container before, nothing to roll back to. Backup: $timestamp"
|
|
fi
|
|
|
|
# Migrations of the new version may have changed the database, so the old version
|
|
# gets the database from before the upgrade
|
|
info "Rolling '$name' back to the previous image and to backup $timestamp"
|
|
restore_customer "$name" "$timestamp"
|
|
run_container "$name" "$old_image"
|
|
if wait_healthy "$container"; then
|
|
die "'$name' was rolled back. Upgrade stopped, the remaining customers were not touched."
|
|
fi
|
|
die "'$name' is DOWN: the rollback did not become healthy either. Backup: $timestamp"
|
|
done
|
|
}
|
|
|
|
cmd_remove() {
|
|
[ $# -ge 1 ] || die "Usage: remove <name> [--purge]"
|
|
local name="$1" purge="${2:-}" db answer
|
|
load_env
|
|
require_customer "$name"
|
|
|
|
info "Removing container $(container_name "$name")"
|
|
docker rm --force "$(container_name "$name")" >/dev/null 2>&1 || true
|
|
|
|
if [ "$purge" != "--purge" ]; then
|
|
info "Database, data volume and config of '$name' were kept. Start it again with 'upgrade $name'."
|
|
return 0
|
|
fi
|
|
|
|
read -r -p "This deletes the database, uploaded files and config of '$name'. Type the name to confirm: " answer
|
|
[ "$answer" = "$name" ] || die "Not confirmed, nothing else was deleted"
|
|
|
|
wait_for_postgres
|
|
db="$(db_name "$name")"
|
|
psql_admin <<EOF
|
|
DROP DATABASE IF EXISTS "$db" WITH (FORCE);
|
|
DROP ROLE IF EXISTS "$db";
|
|
EOF
|
|
docker volume rm "$(volume_name "$name")" >/dev/null 2>&1 || true
|
|
rm -rf "${CUSTOMERS_DIR:?}/$name"
|
|
info "Customer '$name' deleted. Backups in $BACKUPS_DIR/$name were kept."
|
|
}
|
|
|
|
main() {
|
|
[ $# -ge 1 ] || { usage; exit 1; }
|
|
local command="$1"
|
|
shift
|
|
case "$command" in
|
|
init) cmd_init "$@" ;;
|
|
add) cmd_add "$@" ;;
|
|
list) cmd_list "$@" ;;
|
|
upgrade) cmd_upgrade "$@" ;;
|
|
backup) cmd_backup "$@" ;;
|
|
restore) cmd_restore "$@" ;;
|
|
remove) cmd_remove "$@" ;;
|
|
-h|--help|help) usage ;;
|
|
*) usage; exit 1 ;;
|
|
esac
|
|
}
|
|
|
|
main "$@"
|