From b596b23cc452133246722c88fd9d625d12906b3d Mon Sep 17 00:00:00 2001 From: Renovate Bot Date: Sun, 22 Feb 2026 23:07:47 +0000 Subject: [PATCH 1/5] Add renovate.json --- renovate.json | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 renovate.json diff --git a/renovate.json b/renovate.json new file mode 100644 index 0000000..01d2b71 --- /dev/null +++ b/renovate.json @@ -0,0 +1,6 @@ +{ + "$schema": "https://docs.renovatebot.com/renovate-schema.json", + "extends": [ + "local>buildplan/renovate-config" + ] +} From e93f3da1cf3b3d4e5262e881b476f6a761958e9b Mon Sep 17 00:00:00 2001 From: buildplan Date: Fri, 27 Mar 2026 16:54:29 +0000 Subject: [PATCH 2/5] feat: time-based retention and exclude options --- restic-backup.conf | 17 ++++++++++++++++- restic-backup.sh | 27 ++++++++++++++++++++++++--- 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/restic-backup.conf b/restic-backup.conf index 99af6d4..65fa605 100644 --- a/restic-backup.conf +++ b/restic-backup.conf @@ -40,13 +40,21 @@ PACK_SIZE="64" ONE_FILE_SYSTEM=true # --- Retention Policy --- -# How many snapshots to keep +# Standard count-based retention KEEP_LAST="10" KEEP_DAILY="7" KEEP_WEEKLY="4" KEEP_MONTHLY="12" KEEP_YEARLY="3" +# Time-based retention (e.g., "30d", "1m", "1y"). +# These guarantee coverage for a specific time window. Leave blank to disable. +KEEP_WITHIN="" +KEEP_WITHIN_DAILY="7d" # Keeps daily snapshots for the last 7 days +KEEP_WITHIN_WEEKLY="1m" # Keeps weekly snapshots for the last month +KEEP_WITHIN_MONTHLY="1y" # Keeps monthly snapshots for the last year +KEEP_WITHIN_YEARLY="" + # --- Performance --- # Use nice and ionice for lower priority LOW_PRIORITY=true @@ -128,6 +136,13 @@ PRUNE_AFTER_FORGET=true # AUTO_FIX_PERMS=false # --- Exclusions --- +# Automatically exclude folders containing a CACHEDIR.TAG file (true/false) +EXCLUDE_CACHES=true + +# Exclude folders containing any of these specific files. +# Use Bash array syntax for multiple files: (".nobackup" ".ignore_restic") +EXCLUDE_IF_PRESENT=(".nobackup") + # File containing exclude patterns (one per line) EXCLUDE_FILE="/etc/restic-excludes.txt" diff --git a/restic-backup.sh b/restic-backup.sh index 9743e24..72253dd 100644 --- a/restic-backup.sh +++ b/restic-backup.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash # ================================================================= -# Restic Backup Script v0.43 - 2026.02.02 +# Restic Backup Script v0.44 - 2026.03.27 # ================================================================= export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:$PATH @@ -9,7 +9,7 @@ set -euo pipefail umask 077 # --- Script Constants --- -SCRIPT_VERSION="0.43" +SCRIPT_VERSION="0.44" SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd) PROG_NAME=$(basename "$0"); readonly PROG_NAME CONFIG_FILE="${SCRIPT_DIR}/restic-backup.conf" @@ -35,7 +35,6 @@ else C_CYAN='' fi -# --- Ensure running as root --- display_help() { local readme_url="https://github.com/buildplan/restic-backup-script/blob/main/README.md" @@ -74,6 +73,11 @@ display_help() { printf " ${C_GREEN}%-20s${C_RESET} %s\n" "--recovery-kit" "Generate a self-contained recovery script (with embedded password)." printf " ${C_GREEN}%-20s${C_RESET} %s\n" "--uninstall-scheduler" "Remove an automated schedule." echo + echo -e "${C_BOLD}${C_YELLOW}CONFIG FEATURES:${C_RESET} (Managed in ${CONFIG_FILE})" + echo -e " ${C_CYAN}Smart Exclusions:${C_RESET} Auto-skips directories with CACHEDIR.TAG or custom files (e.g., .nobackup)." + echo -e " ${C_CYAN}Time Retention:${C_RESET} Keep-within policies (e.g., 30d, 1y) for resilient snapshot coverage." + echo -e " ${C_CYAN}Resource Limits:${C_RESET} Control CPU usage, SFTP connections, and upload bandwidth." + echo echo -e "${C_BOLD}${C_YELLOW}QUICK EXAMPLES:${C_RESET}" echo -e " Run a backup now: ${C_GREEN}sudo $PROG_NAME${C_RESET}" echo -e " Verbose diff summary: ${C_GREEN}sudo $PROG_NAME --verbose --diff${C_RESET}" @@ -444,6 +448,16 @@ build_backup_command() { [ -n "${COMPRESSION:-}" ] && cmd+=(--compression "$COMPRESSION") [ -n "${PACK_SIZE:-}" ] && cmd+=(--pack-size "$PACK_SIZE") [ "${ONE_FILE_SYSTEM:-false}" = "true" ] && cmd+=(--one-file-system) + if [ "${EXCLUDE_CACHES:-false}" = "true" ]; then + cmd+=(--exclude-caches) + fi + if declare -p EXCLUDE_IF_PRESENT 2>/dev/null | grep -q "declare -a"; then + for f in "${EXCLUDE_IF_PRESENT[@]}"; do + cmd+=(--exclude-if-present "$f") + done + elif [ -n "${EXCLUDE_IF_PRESENT:-}" ]; then + cmd+=(--exclude-if-present "$EXCLUDE_IF_PRESENT") + fi [ -n "${EXCLUDE_FILE:-}" ] && [ -f "$EXCLUDE_FILE" ] && cmd+=(--exclude-file "$EXCLUDE_FILE") [ -n "${EXCLUDE_TEMP_FILE:-}" ] && cmd+=(--exclude-file "$EXCLUDE_TEMP_FILE") cmd+=("${BACKUP_SOURCES[@]}") @@ -1383,11 +1397,18 @@ run_forget() { read -ra v_flags <<< "$(get_verbosity_flags)" forget_cmd+=("${v_flags[@]}") forget_cmd+=(forget) + # Count-based retention [ -n "${KEEP_LAST:-}" ] && forget_cmd+=(--keep-last "$KEEP_LAST") [ -n "${KEEP_DAILY:-}" ] && forget_cmd+=(--keep-daily "$KEEP_DAILY") [ -n "${KEEP_WEEKLY:-}" ] && forget_cmd+=(--keep-weekly "$KEEP_WEEKLY") [ -n "${KEEP_MONTHLY:-}" ] && forget_cmd+=(--keep-monthly "$KEEP_MONTHLY") [ -n "${KEEP_YEARLY:-}" ] && forget_cmd+=(--keep-yearly "$KEEP_YEARLY") + # Time-based retention + [ -n "${KEEP_WITHIN:-}" ] && forget_cmd+=(--keep-within "$KEEP_WITHIN") + [ -n "${KEEP_WITHIN_DAILY:-}" ] && forget_cmd+=(--keep-within-daily "$KEEP_WITHIN_DAILY") + [ -n "${KEEP_WITHIN_WEEKLY:-}" ] && forget_cmd+=(--keep-within-weekly "$KEEP_WITHIN_WEEKLY") + [ -n "${KEEP_WITHIN_MONTHLY:-}" ] && forget_cmd+=(--keep-within-monthly "$KEEP_WITHIN_MONTHLY") + [ -n "${KEEP_WITHIN_YEARLY:-}" ] && forget_cmd+=(--keep-within-yearly "$KEEP_WITHIN_YEARLY") [ "${PRUNE_AFTER_FORGET:-true}" = "true" ] && forget_cmd+=(--prune) if run_with_priority "${forget_cmd[@]}" 2>&1 | tee -a "$LOG_FILE"; then log_message "Retention policy applied successfully" From e5bdfd07857b11536b6691d1c744f47372ebdea1 Mon Sep 17 00:00:00 2001 From: buildplan Date: Fri, 27 Mar 2026 17:01:50 +0000 Subject: [PATCH 3/5] chose: checkout action version bump --- .github/workflows/script-checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/script-checks.yml b/.github/workflows/script-checks.yml index f31145e..23ab7ab 100644 --- a/.github/workflows/script-checks.yml +++ b/.github/workflows/script-checks.yml @@ -22,7 +22,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout repository - uses: actions/checkout@v4 + uses: actions/checkout@v6 - name: Run ShellCheck uses: ludeeus/action-shellcheck@2.0.0 From 273e66032e8ad6b9b373e72ccdf39f55d62c0636 Mon Sep 17 00:00:00 2001 From: buildplan Date: Fri, 27 Mar 2026 17:19:22 +0000 Subject: [PATCH 4/5] chore: db-dump backup scripts --- how-to/db-dump.md | 138 +++++++++++++++++++++++++++++++++++++++ how-to/docker-db-dump.md | 91 ++++++++++++++++++++++++++ 2 files changed, 229 insertions(+) create mode 100644 how-to/db-dump.md create mode 100644 how-to/docker-db-dump.md diff --git a/how-to/db-dump.md b/how-to/db-dump.md new file mode 100644 index 0000000..7815313 --- /dev/null +++ b/how-to/db-dump.md @@ -0,0 +1,138 @@ +# db-backup with helper script + +Because `restic` reads files block-by-block, if a database engine updates a table file *while* `restic` is halfway through reading it, the resulting snapshot will contain a fractured, unbootable database. + +To solve this on Linux VMs is to use a **Pre-Backup Dump Script**. This script securely exports the databases to static `.sql` or `.sql.gz` files, rotates old local copies to save space, and then hands the baton over to your `restic-backup.sh` script. + +Here is an outline and a production-grade template for building this. + +## 1. The Strategy + +* **Secure Authentication:** Never hardcode database passwords in the script. Use native credential files (like `~/.my.cnf` for MySQL/MariaDB or `~/.pgpass` for PostgreSQL) secured with `600` permissions. +* **Compression:** Pipe the output directly into a compressor like `zstd` or `gzip`. This reduces local disk I/O and saves local storage. +* **Local Retention:** The script must clean up after itself (e.g., deleting local dumps older than 2 days). We rely on `restic` and your Hetzner box for long-term retention; the local files are just temporary staging. +* **Fail-Fast Execution:** If the database dump fails, the script should exit with an error so we don't back up corrupted or empty zero-byte files. + +--- + +## 2. The Pre-Backup Script (`db-dump.sh`) + +Create this file (e.g., at `/usr/local/bin/db-dump.sh`) and make it executable (`chmod +x /usr/local/bin/db-dump.sh`). + +```bash +#!/usr/bin/env bash + +# Database Pre-Backup Hook +# Application consistency by dumping databases to static files. + +set -euo pipefail +umask 077 + +# --- Configuration --- +DUMP_DIR="/var/backups/db_dumps" +RETENTION_DAYS="2" +DATE_STAMP=$(date +%Y%m%d_%H%M%S) + +# Determine what databases exist on this VM +HAS_MYSQL=$(command -v mysqldump || true) +HAS_POSTGRES=$(command -v pg_dumpall || true) + +echo "--- Starting Database Dumps: ${DATE_STAMP} ---" + +# Ensure dump directory exists securely +mkdir -p "$DUMP_DIR" +chmod 700 "$DUMP_DIR" + +# --- 1. MySQL / MariaDB Dump --- +if [ -n "$HAS_MYSQL" ] && systemctl is-active --quiet mysql 2>/dev/null || systemctl is-active --quiet mariadb 2>/dev/null; then + echo "Dumping MySQL/MariaDB databases..." + + # Note: This assumes you have created a /root/.my.cnf file with credentials. + # --single-transaction is CRITICAL for InnoDB tables to ensure consistency without locking the whole database. + MYSQL_FILE="${DUMP_DIR}/mysql_all_${DATE_STAMP}.sql.gz" + + if mysqldump --defaults-extra-file=/root/.my.cnf --all-databases --single-transaction --quick --events --routines | gzip -9 > "$MYSQL_FILE"; then + echo "✅ MySQL dump successful: $MYSQL_FILE" + else + echo "❌ MySQL dump failed!" >&2 + exit 1 + fi +fi + +# --- 2. PostgreSQL Dump --- +if [ -n "$HAS_POSTGRES" ] && systemctl is-active --quiet postgresql 2>/dev/null; then + echo "Dumping PostgreSQL databases..." + + PG_FILE="${DUMP_DIR}/postgres_all_${DATE_STAMP}.sql.gz" + + # Run as the postgres user to avoid needing passwords for local socket connections + if su - postgres -c "pg_dumpall -c" | gzip -9 > "$PG_FILE"; then + echo "✅ PostgreSQL dump successful: $PG_FILE" + else + echo "❌ PostgreSQL dump failed!" >&2 + rm -f "$PG_FILE" # Remove partial file + exit 1 + fi +fi + +# --- 3. Local Cleanup --- +echo "Cleaning up local dumps older than $RETENTION_DAYS days..." +find "$DUMP_DIR" -type f -name "*.sql.gz" -mtime +${RETENTION_DAYS} -delete + +echo "--- Database Dumps Completed Successfully ---" +exit 0 +``` + +## 3. Setting Up Authentication Files + +If you are using MySQL/MariaDB, you must create the credentials file so the script runs unattended. + +Create `/root/.my.cnf`: + +```ini +[mysqldump] +user=root +password=your_secure_database_password +``` + +Secure it immediately: + +```bash +chmod 600 /root/.my.cnf +``` + +## 4. Tying It All Together + +Now that the databases are safely turning into static files, you need to update your `restic` setup to back them up and orchestrate the two scripts. + +**Step A: Update `restic-backup.conf`** +Add the dump directory to your `BACKUP_SOURCES` in your configuration file: + +```bash +BACKUP_SOURCES=("/home/user_files" "/var/backups/db_dumps") +``` + +### Step B: Update your Scheduler + +If you used your script's `--install-scheduler` to set up a **cron job**, edit the crontab (`/etc/cron.d/restic-backup`) to chain the scripts together using `&&`. This ensures `restic` *only* runs if the database dump succeeds: + +```cron +# Run db dump, and IF successful, run restic backup +00 03 * * * root /usr/local/bin/db-dump.sh && /path/to/restic-backup.sh >> "/var/log/restic-backup.log" 2>&1 +``` + +If you used **systemd timers**, you can utilize `ExecStartPre` to run the dump before `restic` starts. Edit `/etc/systemd/system/restic-backup.service`: + +```ini +[Service] +Type=oneshot +EnvironmentFile=/path/to/restic-backup.conf +# ADD THIS LINE: +ExecStartPre=/usr/local/bin/db-dump.sh +# Keep your existing ExecStart +ExecStart=/path/to/restic-backup.sh +User=root +Group=root +``` + +Then run `systemctl daemon-reload`. diff --git a/how-to/docker-db-dump.md b/how-to/docker-db-dump.md new file mode 100644 index 0000000..8640c4b --- /dev/null +++ b/how-to/docker-db-dump.md @@ -0,0 +1,91 @@ +# docker database-bump for backup + +Using `docker exec`, execute the native dump commands inside the container's isolated environment, and pipe that output directly out to the host's filesystem for compression and eventual backup by `restic`. + +## 1. The Docker Pre-Backup Script (`db-dump-docker.sh`) + +Create this file (e.g., at `/usr/local/bin/db-dump-docker.sh`) and make it executable (`chmod +x /usr/local/bin/db-dump-docker.sh`). + +```bash +#!/usr/bin/env bash + +# Docker Database Pre-Backup Hook + +set -euo pipefail +umask 077 + +# --- Configuration --- +DUMP_DIR="/var/backups/db_dumps" +RETENTION_DAYS="2" +DATE_STAMP=$(date +%Y%m%d_%H%M%S) + +# Define your exact Docker container names here +MYSQL_CONTAINERS=("production-mysql" "staging-mariadb") +POSTGRES_CONTAINERS=("production-postgres" "gitea-db") + +echo "--- Starting Docker Database Dumps: ${DATE_STAMP} ---" + +# Ensure dump directory exists securely +mkdir -p "$DUMP_DIR" +chmod 700 "$DUMP_DIR" + +# --- 1. MySQL / MariaDB Dumps --- +for container in "${MYSQL_CONTAINERS[@]}"; do + # Check if the container is actually running + if [ "$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)" == "true" ]; then + echo "Dumping MySQL container: $container..." + MYSQL_FILE="${DUMP_DIR}/mysql_${container}_${DATE_STAMP}.sql.gz" + + # execute 'sh -c' inside the container to leverage its existing environment variables + # (like MYSQL_ROOT_PASSWORD) so we don't have to hardcode passwords in this script. + if docker exec "$container" sh -c 'mysqldump -uroot -p"${MYSQL_ROOT_PASSWORD:-$MARIADB_ROOT_PASSWORD}" --all-databases --single-transaction --quick --events --routines' | gzip -9 > "$MYSQL_FILE"; then + echo "✅ $container dump successful." + else + echo "❌ $container dump failed!" >&2 + rm -f "$MYSQL_FILE" + exit 1 + fi + else + echo "⚠️ Skipping $container (container is not running)." + fi +done + +# --- 2. PostgreSQL Dumps --- +for container in "${POSTGRES_CONTAINERS[@]}"; do + if [ "$(docker inspect -f '{{.State.Running}}' "$container" 2>/dev/null)" == "true" ]; then + echo "Dumping PostgreSQL container: $container..." + PG_FILE="${DUMP_DIR}/postgres_${container}_${DATE_STAMP}.sql.gz" + + # Postgres usually allows the 'postgres' user to run pg_dumpall locally without a password prompt. + if docker exec "$container" pg_dumpall -c -U postgres | gzip -9 > "$PG_FILE"; then + echo "✅ $container dump successful." + else + echo "❌ $container dump failed!" >&2 + rm -f "$PG_FILE" + exit 1 + fi + else + echo "⚠️ Skipping $container (container is not running)." + fi +done + +# --- 3. Local Cleanup --- +echo "Cleaning up local dumps older than $RETENTION_DAYS days..." +find "$DUMP_DIR" -type f -name "*.sql.gz" -mtime +${RETENTION_DAYS} -delete + +echo "--- Docker Database Dumps Completed Successfully ---" +exit 0 +``` + +## 2. Key Differences in the Docker Approach + +* **No Host Binaries Required:** The `docker exec` command uses the `mysqldump` and `pg_dumpall` binaries that are already baked into the official database container images. You don't need to install database clients on your host VM. +* **Smart Container Checking:** The script uses `docker inspect -f '{{.State.Running}}'` to verify the container is actively running before attempting a dump. If a container is stopped for maintenance, the script skips it instead of crashing the entire backup chain. +* **Credential Handling (MySQL):** Passing passwords securely to Docker containers can be tricky. This script uses a clever trick: `sh -c 'mysqldump ... -p"${MYSQL_ROOT_PASSWORD}"'`. By passing the command as a string to the container's shell, it evaluates the environment variable *inside* the container. This means you don't need a `.my.cnf` file on the host, and the password won't show up in your host's process list. +* **Credential Handling (Postgres):** PostgreSQL containers default to using `peer` or `trust` authentication for local unix socket connections. By specifying `-U postgres` inside the `docker exec` command, it usually bypasses the need for a password entirely. + +## 3. Application Consistency for Docker Volumes + +If you are backing up standard files alongside these databases (e.g., `/var/lib/docker/volumes/my_app_data`), you still use your `restic-backup.sh` wrapper just as before. + +Just make sure to **exclude the raw database volumes** (like `/var/lib/docker/volumes/mysql_data`) from your `restic-backup.conf` `BACKUP_SOURCES`. You only want `restic` to grab the `.sql.gz` files generated by the pre-backup script, not the live, constantly changing database blocks. From 4937478d4d4ab411eff393960e164df14a73ffab Mon Sep 17 00:00:00 2001 From: buildplan Date: Fri, 27 Mar 2026 17:23:05 +0000 Subject: [PATCH 5/5] checksum v0.44 --- restic-backup.sh.sha256 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/restic-backup.sh.sha256 b/restic-backup.sh.sha256 index 2e53876..fa879e3 100644 --- a/restic-backup.sh.sha256 +++ b/restic-backup.sh.sha256 @@ -1 +1 @@ -260732b0a22a5ed4bde396e09bf803481c72202d9600bf0609e5d6a362951c67 restic-backup.sh +d545db8df2f0f3d59b4e59d4cc5f59a2e44adc74f64e188c88540b683811ef57 restic-backup.sh