diff --git a/postgres-backup-s3/backup.sh b/postgres-backup-s3/backup.sh index ce7ba9e..350b9cb 100644 --- a/postgres-backup-s3/backup.sh +++ b/postgres-backup-s3/backup.sh @@ -78,16 +78,26 @@ echo "Starting backup at $(date -Iseconds)" if [ "${POSTGRES_BACKUP_ALL}" == "true" ]; then echo "Creating dump of all databases from ${POSTGRES_HOST}..." - - pg_dumpall -h $POSTGRES_HOST -p $POSTGRES_PORT -U $POSTGRES_USER | gzip | gpg -c --batch --passphrase $GPG_PASSWORD -o ${DUMP_PATH}dump.sql.gz.gpg - echo "Uploading dump to $S3_BUCKET" - cat ${DUMP_PATH}dump.sql.gz.gp | aws $AWS_ARGS s3 cp - "s3://${S3_BUCKET}${S3_PREFIX}all_$(date +"%Y-%m-%dT%H:%M:%SZ").sql.gz.gpg" || exit 2 + BACKUP_URI="s3://${S3_BUCKET}${S3_PREFIX}all_$(date +"%Y-%m-%dT%H:%M:%SZ").sql.gz.gpg" + IN_PROGRESS_URI="${BACKUP_URI}.in-progress" + if ! pg_dumpall -h "$POSTGRES_HOST" -p "$POSTGRES_PORT" -U "$POSTGRES_USER" \ + | gzip \ + | gpg -c --batch --pinentry-mode loopback --passphrase "$GPG_PASSWORD" --output - \ + | aws $AWS_ARGS s3 cp - "$IN_PROGRESS_URI"; then + echo "SQL backup failed; removing any partial upload" + aws $AWS_ARGS s3 rm "$IN_PROGRESS_URI" || true + exit 2 + fi - echo "SQL backup uploaded successfully" + if ! aws $AWS_ARGS s3 mv "$IN_PROGRESS_URI" "$BACKUP_URI"; then + echo "SQL backup promotion failed; removing the in-progress upload" + aws $AWS_ARGS s3 rm "$IN_PROGRESS_URI" || true + exit 2 + fi - rm -rf ${DUMP_PATH}dump.sql.gz + echo "SQL backup uploaded successfully" else OIFS="$IFS" IFS=',' @@ -96,16 +106,26 @@ else IFS="$OIFS" echo "Creating dump of ${DB} database from ${POSTGRES_HOST}..." - - pg_dump $POSTGRES_HOST_OPTS $DB | gzip | gpg -c --batch --passphrase $GPG_PASSWORD -o ${DUMP_PATH}dump.sql.gz.gpg - echo "Uploading dump to $S3_BUCKET" - cat ${DUMP_PATH}dump.sql.gz.gpg | aws $AWS_ARGS s3 cp - "s3://${S3_BUCKET}${S3_PREFIX}${DB}_$(date +"%Y-%m-%dT%H:%M:%SZ").sql.gz.gpg" || exit 2 + BACKUP_URI="s3://${S3_BUCKET}${S3_PREFIX}${DB}_$(date +"%Y-%m-%dT%H:%M:%SZ").sql.gz.gpg" + IN_PROGRESS_URI="${BACKUP_URI}.in-progress" + if ! pg_dump $POSTGRES_HOST_OPTS "$DB" \ + | gzip \ + | gpg -c --batch --pinentry-mode loopback --passphrase "$GPG_PASSWORD" --output - \ + | aws $AWS_ARGS s3 cp - "$IN_PROGRESS_URI"; then + echo "SQL backup failed; removing any partial upload" + aws $AWS_ARGS s3 rm "$IN_PROGRESS_URI" || true + exit 2 + fi + + if ! aws $AWS_ARGS s3 mv "$IN_PROGRESS_URI" "$BACKUP_URI"; then + echo "SQL backup promotion failed; removing the in-progress upload" + aws $AWS_ARGS s3 rm "$IN_PROGRESS_URI" || true + exit 2 + fi echo "SQL backup uploaded successfully" - - rm -rf ${DUMP_PATH}dump.sql.gz done fi diff --git a/s3-lastmodified-checker/run.sh b/s3-lastmodified-checker/run.sh index 2be8475..1ceb12d 100755 --- a/s3-lastmodified-checker/run.sh +++ b/s3-lastmodified-checker/run.sh @@ -32,7 +32,7 @@ export AWS_DEFAULT_REGION=$S3_REGION # AWS CLI command to list objects in the S3 bucket sorted by modification time -latest_file=$(aws $AWS_ARGS s3api list-objects-v2 --bucket $S3_BUCKET --prefix $S3_PREFIX --query 'sort_by(Contents, &LastModified)[-1].Key' --output text) +latest_file=$(aws $AWS_ARGS s3api list-objects-v2 --bucket $S3_BUCKET --prefix $S3_PREFIX --query 'sort_by(Contents[?ends_with(Key, `.in-progress`) == `false`], &LastModified)[-1].Key' --output text) echo $latest_file