mirror of
https://ghproxy.net/https://github.com/CaptainCore/captaincore.git
synced 2026-07-28 13:07:02 +08:00
314 lines
No EOL
13 KiB
Bash
Executable file
314 lines
No EOL
13 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
|
|
#
|
|
# Backups one or more sites.
|
|
#
|
|
# `captaincore backup generate <site>`
|
|
#
|
|
# [--skip-remote]
|
|
# Skips saving to Restic backup repo
|
|
#
|
|
# [--skip-db]
|
|
# Skips database backup
|
|
#
|
|
# [--skip-if-recent=<timeframe>]
|
|
# Skip if backup generated or last checked within the specified timeframe (e.g., "24h", "7d").
|
|
#
|
|
|
|
if [ ${#@} -ne 1 ]; then
|
|
echo -e "${COLOR_RED}Error:${COLOR_NORMAL} Requires a <site>"
|
|
exit
|
|
fi
|
|
|
|
while read config; do
|
|
if [[ "$config" == "Error:"* ]]; then
|
|
continue
|
|
fi
|
|
declare "$config"
|
|
done <<< "$(captaincore config fetch --captain-id=$CAPTAIN_ID)"
|
|
|
|
site=$1
|
|
root_path="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"; root_path=${root_path%app*}
|
|
|
|
run_command() {
|
|
|
|
if [[ $SKIP_REMOTE == true ]]; then
|
|
echo "Skipping remote"
|
|
fi
|
|
|
|
# Extract environment
|
|
if [[ "$site" == *"-staging"* ]]; then
|
|
environment=staging
|
|
fi
|
|
|
|
if [[ "$site" == *"-production"* ]]; then
|
|
environment=production
|
|
fi
|
|
|
|
if [[ "$site" != *"-"* ]]; then
|
|
environment=production
|
|
fi
|
|
|
|
# Load site configs
|
|
IFS=$'\n'$'\r'; for line in $(captaincore site get $site --bash --captain-id=$CAPTAIN_ID); do declare "$line"; done; unset IFS
|
|
|
|
# Site found, start the backup
|
|
if [[ $domain == "" ]]; then
|
|
echo "Error: $site missing domain. Skipping backup."
|
|
exit
|
|
fi
|
|
|
|
# --------------------------------------------------
|
|
# Locking Mechanism
|
|
# Prevent concurrent backups on the same environment
|
|
# --------------------------------------------------
|
|
|
|
# Ensure the directory exists
|
|
mkdir -p "$path/${site}_${site_id}/${environment}"
|
|
lock_file="$path/${site}_${site_id}/${environment}/backup.lock"
|
|
|
|
# Check if lock file exists
|
|
if [ -f "$lock_file" ]; then
|
|
local locked_pid
|
|
locked_pid=$(cat "$lock_file")
|
|
|
|
# Check if the process ID in the lock file is actually running
|
|
if [ -n "$locked_pid" ] && ps -p "$locked_pid" > /dev/null 2>&1; then
|
|
echo "Skipping $site ($environment) - Backup currently in process (PID: $locked_pid)"
|
|
exit 0
|
|
else
|
|
# Process is dead, lock is stale. Clean up silently and proceed.
|
|
rm -f "$lock_file"
|
|
fi
|
|
fi
|
|
|
|
# Create the lock file with the current Process ID
|
|
echo $$ > "$lock_file"
|
|
|
|
# Ensure lock file is removed on script exit (Success, Error, or Interrupt)
|
|
cleanup_lock() {
|
|
rm -f "$lock_file"
|
|
}
|
|
trap cleanup_lock EXIT
|
|
trap 'exit 130' INT TERM
|
|
|
|
# --------------------------------------------------
|
|
# Auto-Resume / Skip Logic
|
|
# Check if we processed this recently based on --skip-if-recent flag
|
|
# --------------------------------------------------
|
|
if [[ -n "$SKIP_IF_RECENT" ]]; then
|
|
backup_list_file="$path/${site}_${site_id}/${environment}/backups/list.json"
|
|
should_skip=$(php ${CAPTAINCORE_PATH}/lib/local-scripts/check-last-run.php "$backup_list_file" "$SKIP_IF_RECENT")
|
|
|
|
if [[ "$should_skip" == "true" ]]; then
|
|
echo "Skipping $site ($environment) - Backup exists/checked within last $SKIP_IF_RECENT."
|
|
exit 0
|
|
fi
|
|
fi
|
|
# --------------------------------------------------
|
|
|
|
# Append trailing slash if home_directory exist
|
|
if [ "$home_directory" != "" ]; then
|
|
home_directory="${home_directory}/"
|
|
fi
|
|
|
|
# Define config file
|
|
rclone_config_file="$path/${site}_${site_id}/rclone.conf"
|
|
vault_file="$path/${site}_${site_id}/$environment/vault.txt"
|
|
if [[ ! -f "$rclone_config_file" || ! -f "$vault_file" ]]; then
|
|
captaincore site key-generate $site --captain-id=$CAPTAIN_ID
|
|
fi
|
|
|
|
# Detect broken vault.txt (missing B2 bucket on line 1)
|
|
if [[ -f "$vault_file" ]] && [[ -z "$(sed -n '1p' "$vault_file")" ]]; then
|
|
echo "$(date +'%Y-%m-%d %H:%M') Broken vault.txt for $site ($environment). Regenerating..."
|
|
rm -f "$vault_file"
|
|
captaincore site key-generate $site --captain-id=$CAPTAIN_ID
|
|
fi
|
|
|
|
# Lookup rclone
|
|
remote_check=$( rclone config show $environment --config="$rclone_config_file" )
|
|
remote_backup_check=$( rclone config show backup --config="$rclone_config_file" )
|
|
|
|
if [[ $remote_check == *"Couldn't find type of fs"* || $remote_backup_check == *"Couldn't find type of fs"* ]]; then
|
|
echo "$(date +'%Y-%m-%d %H:%M') Generating rclone configs for $site"
|
|
captaincore site key-generate $site --captain-id=$CAPTAIN_ID
|
|
fi
|
|
|
|
# Captures FTP errors in $ftp_output and file listing to log file
|
|
echo "$(date +'%Y-%m-%d %H:%M') Connecting to $site ($environment)..."
|
|
ftp_output=$( { timeout 30 rclone lsd ${environment}:$home_directory --config="$rclone_config_file" --contimeout=15s --timeout=30s ; } 2>&1 )
|
|
|
|
# If connection failed, regenerate rclone config and retry once
|
|
if [[ $? -ne 0 ]]; then
|
|
echo "$(date +'%Y-%m-%d %H:%M') Connection failed for $site ($environment). Regenerating configs..."
|
|
captaincore site key-generate $site --captain-id=$CAPTAIN_ID
|
|
ftp_output=$( { timeout 30 rclone lsd ${environment}:$home_directory --config="$rclone_config_file" --contimeout=15s --timeout=30s ; } 2>&1 )
|
|
|
|
# If connection still fails after rclone config regeneration, try refreshing SSH credentials from provider
|
|
if [[ $? -ne 0 ]]; then
|
|
echo "$(date +'%Y-%m-%d %H:%M') Connection still failing. Attempting SSH credential refresh from provider..."
|
|
if captaincore site ssh-refresh ${site}-${environment} --captain-id=$CAPTAIN_ID; then
|
|
echo "$(date +'%Y-%m-%d %H:%M') Credentials updated. Retrying connection..."
|
|
# Re-read site configs since credentials changed
|
|
IFS=$'\n'$'\r'; for line in $(captaincore site get $site --bash --captain-id=$CAPTAIN_ID); do declare "$line"; done; unset IFS
|
|
ftp_output=$( { timeout 30 rclone lsd ${environment}:$home_directory --config="$rclone_config_file" --contimeout=15s --timeout=30s ; } 2>&1 )
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
ftp_search_for_wordpress=$( echo "$ftp_output" | perl -wnE'say for /wp-admin/g' )
|
|
|
|
# Handle FTP errors
|
|
if [[ $ftp_search_for_wordpress != "wp-admin"* ]]; then
|
|
wordpress_not_found=true
|
|
fi
|
|
|
|
if [[ "$backup_mode" == "local" ]]; then
|
|
|
|
# Incremental backup locally with rclone
|
|
echo "$(date +'%Y-%m-%d %H:%M') Begin incremental backup ${site}-${environment} to local"
|
|
|
|
if [[ $SKIP_DB != true ]] && [[ $wordpress_not_found != true ]]; then
|
|
|
|
# Database backup
|
|
captaincore ssh ${site}-${environment} --script="db-backup" --captain-id=$CAPTAIN_ID
|
|
|
|
if [[ "$provider" == "wpengine" ]]; then
|
|
rclone sync ${environment}:_wpeprivate/database-backup.sql $path/${site}_${site_id}/${environment}/backup/ --config="$rclone_config_file"
|
|
fi
|
|
|
|
if [[ "$provider" == "kinsta" ]]; then
|
|
rclone sync ${environment}:private/database-backup.sql $path/${site}_${site_id}/${environment}/backup/ --config="$rclone_config_file"
|
|
fi
|
|
|
|
if [[ "$provider" == "rocketdotnet" ]]; then
|
|
rclone sync ${environment}:tmp/database-backup.sql $path/${site}_${site_id}/${environment}/backup/ --config="$rclone_config_file"
|
|
fi
|
|
|
|
fi
|
|
|
|
mkdir -p $path/${site}_${site_id}/${environment}/backup/
|
|
|
|
# Backup site locally
|
|
if [[ "$wp_content" != "wp-content" ]]; then
|
|
echo "$(date +'%Y-%m-%d %H:%M') Backing up ${site}-${environment} alternative wp-content location ($wp_content)"
|
|
rclone sync ${environment}:$home_directory $path/${site}_${site_id}/${environment}/backup/ --exclude-from="${CAPTAINCORE_PATH}/lib/excludes" --filter="+ $wp_content/**" --filter="- wp-content/**" --filter="- content/**" --config="$rclone_config_file"
|
|
else
|
|
rclone sync ${environment}:$home_directory $path/${site}_${site_id}/${environment}/backup/ --exclude-from="${CAPTAINCORE_PATH}/lib/excludes" --filter="- content/**" --config="$rclone_config_file"
|
|
fi
|
|
|
|
# Incremental backup upload to Restic
|
|
if [[ $SKIP_REMOTE != true ]]; then
|
|
echo "$(date +'%Y-%m-%d %H:%M') Storing $site to backup archive"
|
|
if [[ $( restic snapshots --repo rclone:$rclone_backup/${site}_${site_id}/${environment}/restic-repo --password-file="${CAPTAINCORE_PATH}/data/restic.key" ) == "" ]]; then
|
|
echo "Generating restic repo for $site"
|
|
restic init --quiet --repo rclone:$rclone_backup/${site}_${site_id}/${environment}/restic-repo --password-file="${CAPTAINCORE_PATH}/data/restic.key"
|
|
# Back up the repo key locally
|
|
captaincore backup key-backup ${site}-${environment} --captain-id=$CAPTAIN_ID 2>/dev/null || true
|
|
fi
|
|
cd $path/${site}_${site_id}/${environment}/backup/
|
|
restic backup . --quiet --repo rclone:$rclone_backup/${site}_${site_id}/${environment}/restic-repo --exclude-file="${CAPTAINCORE_PATH}/lib/restic-excludes" --password-file="${CAPTAINCORE_PATH}/data/restic.key"
|
|
captaincore backup list-generate ${site}-${environment} --captain-id=$CAPTAIN_ID > /dev/null
|
|
captaincore backup get-generate ${site}-${environment} --captain-id=$CAPTAIN_ID > /dev/null
|
|
captaincore backup verify ${site}-${environment} --captain-id=$CAPTAIN_ID
|
|
fi
|
|
|
|
captaincore usage-update ${site}-${environment} --captain-id=$CAPTAIN_ID
|
|
|
|
|
|
# --------------------------------------------------
|
|
# Touch list.json to update its modification timestamp
|
|
# This marks the time this site was last processed.
|
|
# --------------------------------------------------
|
|
backup_list_file="$path/${site}_${site_id}/${environment}/backups/list.json"
|
|
if [ -f "$backup_list_file" ]; then
|
|
touch "$backup_list_file"
|
|
fi
|
|
# --------------------------------------------------
|
|
|
|
exit
|
|
|
|
fi
|
|
|
|
# --- 1. Generate a unique, random name for the temporary payload file ---
|
|
do_script_path="${root_path}lib/remote-scripts/vault"
|
|
random_token=$(head /dev/urandom | LC_ALL=C tr -dc 'a-zA-Z0-9' | head -c 8)
|
|
payload_file="$path_tmp/do_vault_payload-${CAPTAIN_ID}-${site}-${environment}_${random_token}.sh"
|
|
echo "📝 Creating temporary payload file at: $payload_file"
|
|
|
|
# --- 2. Read secrets and write 'export' commands to the payload file ---
|
|
# This block reads each line from your secrets file and builds the export commands.
|
|
{
|
|
echo "export B2_BUCKET='$(sed -n 1p "$vault_file")'"
|
|
echo "export B2_PATH='$(sed -n 2p "$vault_file")'"
|
|
echo "export B2_ACCOUNT_ID='$(sed -n 3p "$vault_file")'"
|
|
echo "export B2_ACCOUNT_KEY='$(sed -n 4p "$vault_file")'"
|
|
echo "export RESTIC_PASSWORD='$(sed -n 5p "$vault_file")'"
|
|
echo "export EMAIL_NOTIFY=$captaincore_admin_email"
|
|
} > "$payload_file"
|
|
|
|
# --- 3. Append the main _do script to the payload file ---
|
|
tail -n +2 "$do_script_path" >> "$payload_file"
|
|
|
|
# Error handling in case the secrets file is not found
|
|
if [ $? -ne 0 ]; then
|
|
echo "❌ Error: Failed to create export commands. Check vault_file path: '$vault_file'" >&2
|
|
rm -f "$payload_file"
|
|
exit 1
|
|
fi
|
|
|
|
# Test SSH connectivity before vault create
|
|
ssh_test=$( captaincore ssh ${site}-${environment} --command="echo ok" --captain-id=$CAPTAIN_ID 2>&1 )
|
|
if [[ "$ssh_test" != *"ok"* ]]; then
|
|
echo "$(date +'%Y-%m-%d %H:%M') SSH test failed for $site. Attempting credential refresh..."
|
|
if captaincore site ssh-refresh ${site}-${environment} --captain-id=$CAPTAIN_ID; then
|
|
# Re-read site configs since credentials changed
|
|
IFS=$'\n'$'\r'; for line in $(captaincore site get $site --bash --captain-id=$CAPTAIN_ID); do declare "$line"; done; unset IFS
|
|
fi
|
|
fi
|
|
|
|
ssh_string=$( captaincore ssh ${site}-${environment} --debug --command="bash -s -- vault create" --captain-id=$CAPTAIN_ID )
|
|
|
|
# --- Execute the command and capture output for piggybacked snapshot data ---
|
|
echo "🚀 Executing remote vault command..."
|
|
vault_output_file="$path_tmp/vault_output-${CAPTAIN_ID}-${site}-${environment}_${random_token}.log"
|
|
cat $payload_file | eval $ssh_string 2>&1 | tee "$vault_output_file"
|
|
|
|
# --- 5. Clean up the local temporary file ---
|
|
echo "🔥 Cleaning up temporary payload file..."
|
|
rm "$payload_file"
|
|
|
|
# Extract piggybacked snapshot JSON from vault create output
|
|
snapshots_json=$(sed -n '/---CAPTAINCORE_SNAPSHOTS_BEGIN---/,/---CAPTAINCORE_SNAPSHOTS_END---/{//d;p}' "$vault_output_file")
|
|
rm -f "$vault_output_file"
|
|
|
|
if [[ -n "$snapshots_json" ]] && [[ "$snapshots_json" =~ ^\[ ]]; then
|
|
# Save to temp file for Go commands
|
|
snapshots_file="$path_tmp/snapshots-${CAPTAIN_ID}-${site}-${environment}_${random_token}.json"
|
|
echo "$snapshots_json" > "$snapshots_file"
|
|
|
|
# Use piggybacked data instead of running restic locally (avoids cache buildup)
|
|
captaincore backup list-generate ${site}-${environment} --from-file="$snapshots_file" --captain-id=$CAPTAIN_ID > /dev/null
|
|
captaincore backup verify ${site}-${environment} --from-file="$snapshots_file" --captain-id=$CAPTAIN_ID
|
|
|
|
rm -f "$snapshots_file"
|
|
else
|
|
# Fallback to traditional method if piggybacked data not found
|
|
captaincore backup list-generate ${site}-${environment} --captain-id=$CAPTAIN_ID > /dev/null
|
|
captaincore backup verify ${site}-${environment} --captain-id=$CAPTAIN_ID
|
|
fi
|
|
|
|
captaincore backup get-generate ${site}-${environment} --captain-id=$CAPTAIN_ID > /dev/null
|
|
|
|
captaincore usage-update ${site}-${environment} --captain-id=$CAPTAIN_ID
|
|
|
|
# Explicitly touch the list file for direct mode
|
|
backup_list_file="$path/${site}_${site_id}/${environment}/backups/list.json"
|
|
if [ -f "$backup_list_file" ]; then
|
|
touch "$backup_list_file"
|
|
fi
|
|
|
|
}
|
|
|
|
run_command |
