diff --git a/README.txt b/README.txt new file mode 100644 index 0000000..915c2b5 --- /dev/null +++ b/README.txt @@ -0,0 +1,5 @@ +README.txt + +This is Rod's Incremental Backup System + +To install, run sudo ./install.sh diff --git a/dist/etc/cron.daily/ribs_backups b/dist/etc/cron.daily/ribs_backups new file mode 100755 index 0000000..b7a96d8 --- /dev/null +++ b/dist/etc/cron.daily/ribs_backups @@ -0,0 +1,8 @@ +#!/bin/bash + +log_path="/var/log/ribs" +log_fn="ribs_$(date +%Y%m%d%H%M%S).log" + +/usr/local/bin/ribs > "$log_path"/"$log_fn" 2>&1 + + diff --git a/dist/etc/ribs/conf-available/ribs.conf.sample b/dist/etc/ribs/conf-available/ribs.conf.sample new file mode 100644 index 0000000..66ea6c3 --- /dev/null +++ b/dist/etc/ribs/conf-available/ribs.conf.sample @@ -0,0 +1,116 @@ +# ribs.conf.sample +# Configuration file for Rod's Incremental Backup System +# +# A configuration file should be created in either the system +# config directory (/etc/ribs/conf-available) or the user's +# config directory (~/.ribs/conf-available). The filename must +# have the .conf suffix. Use the ribs command to enable or disable +# the config file with the -e|--enable or -d|--disable options. +# Run ribs -h for help. + +[General] +# General Options +# +# At least source_dir, backup_root, day_limit, and storage_type +# are required. +# +# Source directory - The directory where files to be backed up are +# located. This must be an absolute, not a relative path. If the +# trailing slash is included, the contents of the directory will be +# backed up. If not, the directory itself will be backed up. +# NOTE: Special characters must be escaped with backslashes! +# +source_dir=/var/www/ +# +# Backup root directory - The directory on the backup storage where the +# backups are stored. For offline storage, this is the mount point of +# the backup drive filesystem. For remote storage, this is the path on +# the remote server. Must be writable to user running ribs. This must +# be an absolute, not a relative path. Directories named as the date +# of the backups will be created here containing the backed up files. +# NOTE: Special characters must be escaped with backslashes! +# +backup_root=/mnt/incremental_backup_dir +# +# Backup subdirectory - A subdirectory of backup_root in which to store +# the backup date directories. If this is not set, the date +# directories will be created directly in backup_root. +# NOTE: Special characters must be escaped with backslashes! +# +#backup_subdir=/main +# +# Exclude list - List of file patterns to exclude, separated by +# commas. Anything in the source directory that matches a pattern in +# this list will not be backed up. If you don't want to exclude +# anything from the backup, comment this option out with a '#'. +# NOTE: Special characters must be escaped with backslashes unless +# they represent a wildcard in a pattern! +# +#exclude_list=DatabaseBackups-*, somehugedir, secretstuff +# +# Day limit - The number of days of backups to keep. When the backup is +# complete, directories with dates older than this limit will be +# deleted. +# +day_limit=90 +# +# Storage type - Where the backups are to be stored. Only one of the +# three options should be specified. Set the option you want, then +# refer to the appropriate section below for that specific +# configuration. Possible values are: +# local - Back up to a local directory. Can also be anything already +# mounted on the local filesystem. +# offline - Back up to offline storage that must be mounted. Can be +# anything that you can mount and unmount, including a filesystem +# on a local block device or a remote NFS or CIFS/SMB share. +# remote - Back up to a remote system directly via rsync. If you use +# this, you must define remote_user_host below and setup pubkey +# authentication vi ssh for the remote user and host. +# +storage_type=remote +# + + +[local] +# Options for local storage type +# +# There are currently no specific settings for the local storage type. +# + + +[offline] +# Options for offline storage type +# +# Options in this section are required if storage_type above is set +# to offline. Otherwise, they are ignored. +# +# Mount parameters - Storage filesystem mount parameters. Should be what +# you'd type on the command line between the word "mount" and the +# mountpoint. See the manual page for the mount command (man 8 mount) +# for the available options. +# +mount_parameters=-t ext4 -U 433a2e61-25aa-4da5-a7c7-4d730a5793e7 +#mount_parameters="-t nfs nfsserver:/srv/ribs-backups" +#mount_parameters="/dev/sdc1" +# +# Remount read-only - Set this to True if storage should be remounted in +# read-only mode when not being used for backups, False if not. +# +remount_ro=True +# + +[remote] +# Options for remote storage type +# +# Options in this section are required if storage_type above is set +# to remote. Otherwise, they are ignored. +# +# Remote user and host - The user and host to connect to for backup +# storage. If you want RIBS to be able to run unattended, for example +# as a cron job, you will need to ensure that the user's ssh public +# key is listed in the remote host's authorized_keys file. This is +# usually done with the ssh-copy-id command, which also takes care of +# adding the host's key fingerprint to the local known_hosts file. +# +remote_user_host=user@backupserver +# diff --git a/dist/etc/ribs/conf-available/ribs.config.sample b/dist/etc/ribs/conf-available/ribs.config.sample new file mode 100644 index 0000000..e6ef1b7 --- /dev/null +++ b/dist/etc/ribs/conf-available/ribs.config.sample @@ -0,0 +1,124 @@ +# Configuration file for Rod's Incremental Backup System +# Version 1.3.2 + +# ************************** General Options *************************** +# +# All of the options in the genera options section must be defined. +# +# ******************** +# Dry run - Set to false for normal mode or true for testing. +# +DRY_RUN=true +# +# ******************** +# Source directory - The directory where files to be backed up are +# located. This must be an absolute, not a relative path. If the +# trailing slash is included, the contents of the directory will be +# backed up. If not, the directory itself will be backed up. +# NOTE: Special characters must be escaped with backslashes! +# +SRC="/var/www/" +# +# ******************** +# Backup root directory - The directory on the backup storage where the +# backups are stored. For OFFLINE storage, this is the mount point of +# the backup drive filesystem. For REMOTE storage, this is the path on +# the remote server. Must be writable to user running ribs. This must +# be an absolute, not a relative path. Directories named as the date +# of the backups will be created here containing the backed up files. +# NOTE: Special characters must be escaped with backslashes! +# +BACKUP_ROOT="/mnt/incremental_backup_dir" +# +# ******************** +# Backup subdirectory - A subdirectory of BACKUP_ROOT in which to store +# the backup date directories. To just put the date directories +# directly in BACKUP_ROOT, set this to blank (""). +# NOTE: Special characters must be escaped with backslashes! +# +BACKUP_SUBDIR="" +#BACKUP_SUBDIR="/main" +# +# ******************** +# Exclude list - List of file patterns to exclude, separated by +# newlines. Anything in the source directory that matches a pattern in +# this list will not be backed up. If you don't want to exclude +# anything from the backup, set this to blank (""). +# NOTE: Special characters must be escaped with backslashes unless +# they represent a wildcard in a pattern! +# +EXCLUDE_LIST="" +#EXCLUDE_LIST=" +#DatabaseBackups-* +#somehugedir +#secretstuff +#" +# +# ******************** +# Day limit - The number of days of backups to keep. When the backup is +# complete, directories with dates older than this limit will be +# deleted. +# +DAY_LMT=90 +# +# ******************** +# Storage type - Where the backups are to be stored. Only one of the +# three options should be specified. Set the option you want, then +# refer to the appropriate section below for that specific +# configuration. Possible values are: +# LOCAL - Back up to a local directory. Can also be anything already +# mounted on the local filesystem. +# OFFLINE - Back up to offline storage that must be mounted. Can be +# anything that you can mount and unmount, including a filesystem +# on a local block device or a remote NFS or CIFS/SMB share. +# REMOTE - Back up to a remote system directly via rsync. If you use +# this, you must define REMOTE_USER_HOST below and setup pubkey +# authentication vi ssh for the remote user and host. +# +STORAGE_TYPE="REMOTE" +# +# ********************************************************************** + + +# ******************* Options for LOCAL storage type ******************* +# +# There are no specific settings for the local storage type. +# +# ********************************************************************** + + +# ****************** Options for OFFLINE storage type ****************** +# +# Leave this entire section commented out for LOCAL or REMOTE storage. +# +# Mount parameters - Storage filesystem mount parameters. Should be what +# you'd type on the command line between the word "mount" and the +# mountpoint. See the manual page for the mount command (man 8 mount) +# for the available options. +# +#MOUNT_PARAMETERS="-t nfs nfsserver:/srv/ribs-backups" +#MOUNT_PARAMETERS="-t ext4 -U 433a2e61-25aa-4da5-a7c7-4d730a5793e7" +#MOUNT_PARAMETERS="/dev/sdc1" +# +# Remount read-only - Set this to true if storage should be remounted in +# read-only mode when not being used for backups. +# +#REMOUNT_RO=true +#REMOUNT_RO=false +# +# ********************************************************************** + +# ****************** Options for REMOTE storage type ******************* +# +# Leave this entire section commented out for LOCAL or OFFLINE storage. +# +# Remote user and host - The user and host to connect to for backup +# storage. If you want RIBS to be able to run unattended, for example +# as a cron job, you will need to ensure that the user's ssh public +# key is listed in the remote host's authorized_keys file. This is +# usually done with the ssh-copy-id command, which also takes care of +# adding the host's key fingerprint to the local known_hosts file. +# +REMOTE_USER_HOST="user@backupserver" +# +# ********************************************************************** diff --git a/dist/usr/local/bin/ribs b/dist/usr/local/bin/ribs new file mode 100755 index 0000000..d2b4535 --- /dev/null +++ b/dist/usr/local/bin/ribs @@ -0,0 +1,485 @@ +#!/bin/bash +# Rod's Incremental Backup System + +# Define constants + +RIBS_VERSION="1.3.2" +USER_CONF_DIR="$HOME/.ribs/conf-enabled" +SYSTEM_CONF_DIR="/etc/ribs/conf-enabled" + +# Define functions + +function find_configs() { + # Search for config files in the default places and build a list + # Takes one optional argument, the name of a config file. + # Defines the variable conf_files, a list of conf files + + conf_files=() + # First check user's conf directory, then the system conf directory + if [[ $1 != "" ]] + then + conf_files+=("$1") + elif ls -1 "$USER_CONF_DIR"/* >/dev/null 2>&1 + then + for conf_file in "$USER_CONF_DIR"/* + do + conf_files+=("$conf_file") + done + elif ls -1 "$SYSTEM_CONF_DIR"/* >/dev/null 2>&1 + then + for conf_file in "$SYSTEM_CONF_DIR"/* + do + conf_files+=("$conf_file") + done + else + echo "Couldn't find any config files in" + echo "$USER_CONF_DIR or $SYSTEM_CONF_DIR," + echo "and one was not specified on the command line." + echo "Please refer to the file /etc/ribs/conf-available/ribs.conf.sample" + echo "and create one." + exit 1 + fi +} + +function read_config() { + # Read the configuration file + + # Make sure the provided config file is readable. + if [[ ! -r $CONF_FILE ]] + then + echo -n "Reading configuration file $CONF_FILE" + echo -e "\e[31mFAILED\e[0m" + echo "The config file $CONF_FILE does not exist or is not readable." + echo "Aborting." + exit 1 + fi + echo -n "Reading configuration file... " + if ! source "$CONF_FILE" + then + echo -e "\e[31mFAILED\e[0m" + echo "An error was encountered while reading the configuration file" + echo "$CONF_FILE." + echo "Aborting." + exit 1 + else + echo -e "\e[32mOK\e[0m" + fi + + # Add slash to the beginning of $BACKUP_SUBDIR if it's not there + if [[ "$(echo "$BACKUP_SUBDIR" | cut -c1)" != "/" ]] + then + BACKUP_SUBDIR="/$BACKUP_SUBDIR" + fi + # Remove slash from the end of $BACKUP_SUBDIR if it's there + if [[ "$(echo "$BACKUP_SUBDIR" | rev | cut -c1)" = "/" ]] + then + numchars=$(echo "$BACKUP_SUBDIR" | wc -m);let "numchars-=2" + BACKUP_SUBDIR=$(echo "$BACKUP_SUBDIR" | cut -c-$numchars) + fi +} + +function generate_excludes() { + # Generate exclude array + declare -a excludes + if [[ $EXCLUDE_LIST != "" ]] + then + for pattern in $EXCLUDE_LIST + do + echo "excluding pattern $pattern" + excludes+=' --exclude="$pattern" ' + done + fi +} + +function mount_storage() { + # Takes one argument, rw, ro, or u + mount_mode=$1 + if [[ $mount_mode = "ro" ]] + then + mode_text="read-only" + elif [[ $mount_mode = "rw" ]] + then + mode_text="read-write" + elif [[ $mount_mode = "u" ]] + then + mode_text="unmounted" + else + echo "Program error: invalid option \"$mode_text\" passed to mount_storage()" + exit 1 + fi + + # Mount the backup drive + # First, check to see if backup drive partition is mounted + if ! mount|grep -q "$BACKUP_ROOT" + then + remount_option="remount," + else + remount_option="" + fi + + if [[ $mount_mode = "u" ]] + then + # Unmount drive + echo -n "Unmounting backup storage... " + if ! umount "$BACKUP_ROOT" + then + echo -e "\e[31mFAILED\e[0m" + echo "An error was encountered while attempting to unmount $BACKUP_ROOT." + echo "Aborting." + exit 1 + else + echo -e "\e[32mOK\e[0m" + fi + else + # Mount the drive in the desired mode + echo -n "Mounting backup storage $mode_text... " + if [[ $MOUNT_PARAMETERS != "" ]] + then + if ! mount "$MOUNT_PARAMETERS" "$BACKUP_ROOT" -o"$remount_option""$mount_mode" + then + echo -e "\e[31mFAILED\e[0m" + echo "An error was encountered while attempting to mount the backup storage" + echo "on $BACKUP_ROOT in $mode_text mode. Check MOUNT_PARAMETERS in your" + echo "configuration file." + echo "Aborting." + exit 1 + else + echo -e "\e[32mOK\e[0m" + fi + else + echo -e "\e[31mFAILED\e[0m" + echo "Unable to mount backup storage." + echo "No MOUNT_PARAMETERS were provided in config file." + echo "Aborting." + exit 1 + fi + fi +} + +function list_backups() { + # Generate a list of backups based on storage type + if [[ $STORAGE_TYPE = "REMOTE" ]] + then + ssh $REMOTE_USER_HOST cd $BACKUP_ROOT$BACKUP_SUBDIR && ls -1d ????-??-??* + else + cd $BACKUP_ROOT$BACKUP_SUBDIR && ls -1d ????-??-??* + fi +} + +function delete_backup() { + # Delete a backup. Takes one argument, the date directory to delete + echo -n "Removing old backup $1... " + remove_fail=false + if [[ $STORAGE_TYPE = "REMOTE" ]] + then + if ! ssh $REMOTE_USER_HOST rm -r $BACKUP_ROOT$BACKUP_SUBDIR/$1 + then + remove_fail=true + fi + else + if ! rm -r $BACKUP_ROOT$BACKUP_SUBDIR/$1 + then + remove_fail=true + fi + fi + if $remove_fail + then + echo -e "\e[31mFAILED\e[0m" + else + echo -e "\e[32mOK\e[0m" + fi +} + +function run_backup() { + CONF_FILE="$1" + + echo "Run started with configuration file:" + echo "$CONF_FILE" + date "+%a %b %d %H:%M:%S %Z %Y" + echo "" + + # Read config file + read_config + + # Generate exclude array + generate_excludes + + # Mount offline storage if required + if [[ $STORAGE_TYPE == "OFFLINE" ]] + then + mount_storage rw + fi + + # Today's date + current_datetime=$(date +%Y-%m-%dT%H:%m:%S) + echo "Today's date: $current_datetime" + + # Previous backup's date + if $list_cmd >/dev/null 2>&1 + then + previous_datetime=$($list_cmd | tail -n1) + else + # there are no other date directories + previous_datetime="" + fi + if [[ $previous_datetime == "" ]] + then + echo "No previous backup date." + fi + + # set target and link destination directories and day to remove + if [[ $STORAGE_TYPE == "REMOTE" ]] + then + TRG="$REMOTE_USER_HOST:$BACKUP_ROOT$BACKUP_SUBDIR/$current_datetime/" + LNK="$BACKUP_ROOT$BACKUP_SUBDIR/$previous_datetime/" + if ssh "$REMOTE_USER_HOST" test -e "$LNK" + then + link_dest_exists=true + else + link_dest_exists=false + fi + else + TRG="$BACKUP_ROOT$BACKUP_SUBDIR/$current_datetime/" + LNK="$BACKUP_ROOT$BACKUP_SUBDIR/$previous_datetime" + if [[ -e "$LNK" ]] + then + link_dest_exists=true + else + link_dest_exists=false + fi + fi + + # rsync options + for element in $excludes + do + echo "excludes array contains $element" + done + if ! $link_dest_exists + then + OPT="-s -ah --delete ${excludes[@]}" + else + OPT="-s -ah --delete --link-dest=$LNK ${excludes[@]}" + fi + if $DRY_RUN + then + OPT="$OPT -n" + fi + + if [[ $STORAGE_TYPE != "REMOTE" ]] + then + # -------- Run some tests ----- + testtimestamp=$(date +%Y%m%d%H%M%S%N) + test_filename="RIBS${testtimestamp}" + test_linkname="RIBS${testtimestamp}hardlink" + + # Make sure the backup root is writeable + echo -n "Testing backup location for writeability... " + if ! touch "$BACKUP_ROOT"/"$test_filename" >/dev/null 2>&1 + then + echo -e "\e[31mFAILED\e[0m" + echo "Aborting." + if [[ $STORAGE_TYPE = "OFFLINE" ]] + then + if $REMOUNT_RO + then + mount_storage ro + else + mount_storage u + fi + fi + exit 1 + else + echo -e "\e[32mOK\e[0m" + fi + + # Make sure the backup root supports hard links + echo -n "Testing backup location for hard link support... " + if ! ln "$BACKUP_ROOT"/"$test_filename" "$BACKUP_ROOT"/"$test_linkname" >/dev/null 2>&1 + then + echo -e "\e[31mFAILED\e[0m" + echo "Aborting." + rm -f "$BACKUP_ROOT"/"$test_filename" + if [[ $STORAGE_TYPE == "OFFLINE" ]] + then + if $REMOUNT_RO + then + mount_storage ro + else + mount_storage u + fi + fi + exit 1 + else + echo -e "\e[32mOK\e[0m" + fi + + rm -f "$BACKUP_ROOT"/"$test_filename" + rm -f "$BACKUP_ROOT"/"$test_linkname" + fi + + # -------- Start backup ------- + # Create backup subdir if it's not there + if [[ $STORAGE_TYPE == "REMOTE" ]] + then + if ! $DRY_RUN && [[ $BACKUP_SUBDIR != "" ]] && ! ssh "$REMOTE_USER_HOST" test -d "$BACKUP_ROOT$BACKUP_SUBDIR" + then + ssh "$REMOTE_USER_HOST" mkdir -p "$BACKUP_ROOT""$BACKUP_SUBDIR" + fi + else + if ! $DRY_RUN && [[ $BACKUP_SUBDIR != "" ]] && ! test -d "$BACKUP_ROOT$BACKUP_SUBDIR" + then + mkdir -p "$BACKUP_ROOT""$BACKUP_SUBDIR" + fi + fi + + echo "Starting incremental backup." + if $DRY_RUN + then + echo "This will be a dry run. Nothing will actually be backed up." + sleep 10 + fi + # Execute the backup + echo -n "running rsync backup with command rsync $OPT $SRC $TRG... " + if ! bash -c "rsync $OPT $SRC $TRG" + then + echo -e "\e[31mFAILED\e[0m" + echo "An error was encountered while attempting to backup. No old" + echo "backups will be removed." + if [[ $STORAGE_TYPE == "OFFLINE" ]] + then + if $REMOUNT_RO + then + mount_storage ro + else + mount_storage u + fi + fi + exit 1 + else + echo -e "\e[32mOK\e[0m" + fi + + # Delete the specified backups, if it exists + limit_date=$(date -d "$current_datetime $DAY_LMT days ago") + limit_date_sec=$(date -d "$limit_date" +%s) + + if $DRY_RUN + then + echo "This is a dry run, so no old backups will be removed." + for date_dir in $(list_backups) + do + date_dir_sec=$(date -d "$date_dir" +%s) + if [[ $date_dir_sec < $limit_date_sec ]] + then + echo "We would be removing $date_dir" + fi + done + else + for date_dir in $(list_backups) + do + date_dir_sec=$(date -d "$date_dir" +%s) + if [[ $date_dir_sec < $limit_date_sec ]] + then + delete_backup $date_dir + fi + done + fi + + # Clean up + echo -n "Syncing disks... " + if ! sync + then + echo -e "\e[31mFAILED\e[0m" + else + echo -e "\e[32mOK\e[0m" + fi + + # Update capacity file if using offline storage + if [[ $STORAGE_TYPE == "OFFLINE" ]] + then + echo -n "Updating capacity file... " + if ! df -h |grep "$BACKUP_ROOT" > "$BACKUP_ROOT"/capacity.txt + then + echo -e "\e[31mFAILED\e[0m" + else + echo -e "\e[32mOK\e[0m" + fi + fi + + # Unmount or remount offline storage as required + if [[ $STORAGE_TYPE == "OFFLINE" ]] + then + if $REMOUNT_RO + then + echo -n "Remounting backup storage read-only... " + if ! mount_storage ro + then + echo -e "\e[31mFAILED\e[0m" + else + echo -e "\e[32mOK\e[0m" + fi + else + echo -n "Unmounting backup storage... " + if ! mount_storage u + then + echo -e "\e[31mFAILED\e[0m" + else + echo -e "\e[32mOK\e[0m" + fi + fi + fi + echo "Backup complete." + echo "" + + echo "Run ended with configuration file:" + echo "$CONF_FILE" + date "+%a %b %d %H:%M:%S %Z %Y" + echo "" +} + +function show_help() { + echo "Usage: $0 [-h|--help] [config_file]" + echo "" + echo "Run an incremental backup using a config file specified." + echo "If no config file is specified, search for and read all config" + echo "files in the user's config directory ($USER_CONF_DIR)." + echo "If none are found there, search for and read all config files in" + echo "the system config directory ($SYSTEM_CONF_DIR)." + echo "" + echo "Please refer to the sample config file, " + echo "/etc/ribs/conf-available/ribs.conf.sample for an explanation" + echo "of parameters." + echo "" + echo " Options:" + echo "" + echo " -h|--help: Display this help message." + echo "" +} + + +function main() { + # Begin execution + echo "RIBS $RIBS_VERSION" + echo "Rod's Incremental Backup System" + echo "" + + case "$arg" in + -h|--help) + show_help + exit 0 + ;; + *) + find_configs "$arg" + ;; + esac + + for each_conf in "${conf_files[@]}" + do + run_backup "$each_conf" + done +} + +main "$@" + +exit 0 + diff --git a/dist/usr/local/bin/ribs.py b/dist/usr/local/bin/ribs.py new file mode 100755 index 0000000..8eb5d0a --- /dev/null +++ b/dist/usr/local/bin/ribs.py @@ -0,0 +1,658 @@ +#!/usr/bin/env python3 + +""" + Program: Rod's Incremental Backup System (ribs.py) + Author: Rod Wright + Date: 03/10/2025 +""" + +import subprocess +import sys +import os +import shutil +import glob +from datetime import datetime, timedelta +import textwrap +import argparse +import configparser +from pathlib import Path + +# App version +APP_VERSION="1.3.2" + +# File paths +SYS_EN_CONF_DIR="/etc/ribs/conf-enabled/" +SYS_AV_CONF_DIR="/etc/ribs/conf-available/" +USER_EN_CONF_DIR=str(Path.home())+"/.ribs/conf-enabled/" +USER_AV_CONF_DIR=str(Path.home())+"/.ribs/conf-available/" + + +# Determine if I have superuser privileges +if os.geteuid()==0: + superuser=True +else: + superuser=False + +# ****** BEGIN function definitions ****** + + +def main(): + """ + main function + """ + print("") + # Parse arguments and take required actions + parser=argparse.ArgumentParser(description='Rod\'s Incremental Backup System, version '+APP_VERSION) + group=parser.add_mutually_exclusive_group() + + parser.add_argument( + '-s', '--simulate', + help='simulate operations to be performed', + action="store_true" + ) + + group.add_argument( + '-l', '--list-configs', + help='list available and enabled config files', + action="store_true" + ) + + group.add_argument( + '-e', '--enable', + metavar='CONF_FILE', + help='enable a config file' + ) + + group.add_argument('-d', '--disable', + metavar='CONF_FILE', + help='disable a config file' + ) + + group.add_argument( + "conf_file", + help='Run RIBS using the specified config file. \ + If none specified, run all enabled config files, looking in \'~/.ribs/conf-enabled\' \ + for standard users, or in \'/etc/ribs/conf-enabled\' for superusers.', + nargs='?' + ) + + args=parser.parse_args() + + if args.simulate: + simulate=True + else: + simulate=False + + if args.enable: + if args.enable!=None: + enable_config(args.enable,simulate) + elif args.disable: + if args.disable!=None: + disable_config(args.disable,simulate) + elif args.list_configs: + if args.list_configs!=None: + list_configs() + else: + run_backup(args.conf_file,simulate) + + +def run_rsync(source, destination, options=None, excludes=None): + """ + Runs rsync with the given source, destination, options, and excludes. + + Args: + source (str): The source path. + destination (str): The destination path. + options (list, optional): A list of rsync options. Defaults to None. + excludes (list, optional): A list of patterns to exclude. Defaults to None. + + + Returns: + int: The return code of the rsync command. + """ + command = ["rsync"] + if options: + command.extend(options) + if excludes: + for pattern in excludes: + command.extend(['--exclude='+pattern]) + command.extend([source, destination]) + #print("complete rsync command is:",command) + process = subprocess.run(command, capture_output=True, text=True) + + if '-v' in options or '--verbose' in options: + print(process.stdout) + + if process.returncode != 0: + print(f"Error running rsync: {process.stderr}") + return process.returncode + + +def mount_offline(mountpoint,mountparams,mode): + """ + Handle mounting and unmounting of offline storage. + mountpoint is backup_root. + mountparams are the mount parameters from config file. + mode is ro, rw, or u. + """ + pass + + +def list_configs(): + """ + Show a list of config files. + """ + if superuser: + ena_conf_dir=SYS_EN_CONF_DIR + avail_conf_dir=SYS_AV_CONF_DIR + enabled_configs=find_configs('system','enabled') + available_configs=find_configs('system','available') + else: + ena_conf_dir=USER_EN_CONF_DIR + avail_conf_dir=USER_AV_CONF_DIR + enabled_configs=find_configs('user','enabled') + available_configs=find_configs('user','available') + print("Available config files:\n") + if len(available_configs)==0: + infostring="No available config files found. Refer to the sample config file "+SYS_AV_CONF_DIR+"ribs.conf.sample to create one. Note that config files must have the .conf extension to be recognized." + infostringlist=textwrap.wrap(infostring,width=80,break_on_hyphens=False) + for line in infostringlist: + print(line) + else: + for config in available_configs: + print(Path(config).name) + print("\n") + print("Enabled config files:\n") + if len(enabled_configs)==0: + infostring="No enabled config files found. Refer to the list of available config files above and enable one using ribs -e. Type ribs -h for help." + infostringlist=textwrap.wrap(infostring,width=80,break_on_hyphens=False) + for line in infostringlist: + print(line) + else: + for config in enabled_configs: + print(Path(config).name) + + + + +def enable_config(configfile,simulate=True): + """ + Enable the specified config file + """ + if simulate: + print("All actions will be simulated") + if superuser: + conftype='system' + availconfdir=SYS_AV_CONF_DIR + enaconfdir=SYS_EN_CONF_DIR + else: + conftype='user' + availconfdir=USER_AV_CONF_DIR + enaconfdir=USER_EN_CONF_DIR + + # Look for configs to enable + configs=find_configs(conftype,'available') + if len(configs)>0: + for config in configs: + if Path(config).name == configfile: + link=Path(enaconfdir+configfile) + target=Path(availconfdir+configfile) + if simulate: + print("Simulating enabling",Path(configfile).name) + else: + print("Enabling the user config:",configfile) + link.symlink_to(target) + + +def disable_config(configfile,simulate=True): + """ + Disable the specified config file + """ + if simulate: + print("All actions will be simulated") + if superuser: + conftype='system' + enaconfdir=SYS_EN_CONF_DIR + else: + conftype='user' + enaconfdir=USER_EN_CONF_DIR + + # Look for configs to disable + configs=find_configs(conftype,'enabled') + if len(configs)>0: + for config in configs: + if Path(config).name == configfile: + link=Path(enaconfdir+configfile) + if simulate: + print("Simulating disabling",Path(configfile).name) + else: + print("Disabling the user config:",configfile) + link.unlink(link) + + +def find_configs(conftype,confstate): + """ + Find and produce a list of all enabled config files + """ + if confstate=="enabled": + if conftype=="user": + dirpath=USER_EN_CONF_DIR + elif conftype=="system": + dirpath=SYS_EN_CONF_DIR + elif confstate=="available": + if conftype=="user": + dirpath=USER_AV_CONF_DIR + elif conftype=="system": + dirpath=SYS_AV_CONF_DIR + pattern="*.conf" + configs = [] + matching_files=glob.glob(os.path.join(dirpath,pattern)) + if matching_files: + for filepath in matching_files: + configs.append(filepath) + + return(configs) + + +def parse_config_file(conf_file): + """ + Read the specified config file and return a dictionary of the parameters + """ + config_errors=[] + config=configparser.ConfigParser(allow_no_value=True) + config.read(conf_file) + if config.has_section('General'): + # Pull the general options + if config.has_option('General','source_dir'): + source_dir=config['General']['source_dir'] + else: + config_errors.append('source_dir option missing from config file') + if config.has_option('General','backup_root'): + backup_root=config['General']['backup_root'] + else: + config_errors.append('backup_root option missing from config file') + if config.has_option('General','backup_subdir'): + backup_subdir=config['General']['backup_subdir'] + else: + backup_subdir=None + if config.has_option('General','exclude_list'): + exclude_list=config['General']['exclude_list'].split(',') + exclude_list=[pattern.strip() for pattern in exclude_list] + else: + exclude_list=None + if config.has_option('General','day_limit'): + day_limit=config['General']['day_limit'] + else: + config_errors.append('day_limit option missing from config file') + if config.has_option('General','storage_type'): + storage_type=config['General']['storage_type'] + else: + config_errors.append('storage_type option missing from config file') + else: + config_errors.append('General section missing from config file') + + if len(config_errors)>0: + print("The following configuration file errors were found") + for err_msg in config_errors: + print(err_msg) + print("These are fatal errors and ribs cannot continue.") + print("Please check the config file:",conf_file) + sys.exit(1) + config_errors=[] + + if storage_type=="local": + if config.has_section('local'): + # Reserved for future use + pass + else: + pass + elif storage_type=="offline": + if config.has_section('offline'): + if config.has_option('offline','mount_parameters'): + mount_parameters=config['offline']['mount_parameters'] + else: + config_errors.append('mount_parameters option missing from config file') + if config.has_option('offline','remount_ro'): + remount_ro=config['offline']['remount_ro'] + else: + remount_ro=False + else: + config_errors.append('offline section missing from config file') + elif storage_type=="remote": + if config.has_section('remote'): + if config.has_option('remote','remote_user_host'): + remote_user_host=config['remote']['remote_user_host'] + else: + config_errors.append('remote_user_host option missing from config file') + else: + config_errors.append('remote section missing from config file') + else: + config_errors.append('invalid storage_type. Must be local, offline, or remote.') + + if len(config_errors)>0: + print("The following configuration file errors were found") + for err_msg in config_errors: + print(err_msg) + print("These are fatal errors and ribs cannot continue.") + print("Please check the config file:",conf_file) + sys.exit(1) + + # QC option formatting + config_errors=[] + if not source_dir.startswith('/'): + config_errors.append('source_dir must be an absolute path, but doesn\'t start with a \'/\'') + if not backup_root.startswith('/'): + config_errors.append('backup_root must be an absolute path, but doesn\'t start with a \'/\'') + if day_limit=='0': + config_errors.append('day_limit cannot be 0') + if storage_type=='remote' and not "@" in remote_user_host: + config_errors.append('remote_user_host must be in the format\'user@host\'') + + if len(config_errors)>0: + print("The following configuration file errors were found") + for err_msg in config_errors: + print(err_msg) + print("These are fatal errors and ribs cannot continue.") + print("Please check the config file:",conf_file) + sys.exit(1) + + # Clean up variables + if backup_root.endswith('/'): + backup_root=backup_root.rstrip('/') + if backup_subdir: + if backup_subdir.startswith('/'): + backup_subdir=backup_subdir.lstrip('/') + if backup_subdir.endswith('/'): + backup_subdir=backup_subdir.rstrip('/') + + # Return a parameter:option dictionary + if storage_type=="local": + return { + 'storage_type':storage_type, + 'source_dir':source_dir, + 'backup_root':backup_root, + 'backup_subdir':backup_subdir, + 'exclude_list':exclude_list, + 'day_limit':day_limit + } + elif storage_type=="offline": + return { + 'storage_type':storage_type, + 'source_dir':source_dir, + 'backup_root':backup_root, + 'backup_subdir':backup_subdir, + 'exclude_list':exclude_list, + 'day_limit':day_limit, + 'mount_parameters':mount_parameters, + 'remount_ro':remount_ro + } + elif storage_type=="remote": + return { + 'storage_type':storage_type, + 'source_dir':source_dir, + 'backup_root':backup_root, + 'backup_subdir':backup_subdir, + 'exclude_list':exclude_list, + 'day_limit':day_limit, + 'remote_user_host':remote_user_host + } + else: + # Not sure how we got here, but if we did, something's wrong + return False + + +def run_backup(configfile=None,simulate=True): + """ + Run backup using the specified configfile, or all enabled config + files if none specified. + """ + if simulate: + print("All actions will be simulated") + if configfile==None: + # Find and run all enabled configs + print("Finding enabled config files...") + userconfigs=find_configs("user","enabled") + systemconfigs=find_configs("system","enabled") + if len(userconfigs)>0: + configs=userconfigs + print("Found user config files:") + for config in configs: + print(config) + elif len(systemconfigs)>0: + print("No user config files found.") + configs=systemconfigs + print("Found system config files:") + for config in configs: + print(config) + else: + print("No enabled config files found") + else: + # Run only the specified config + configs=[configfile] + + for config in configs: + print("Running backup from the config file:",config) + + # Parse config file and create variables + conf_parameters=parse_config_file(config) + + source_dir=Path(conf_parameters.get('source_dir')) + backup_root=Path(conf_parameters.get('backup_root')) + backup_subdir=conf_parameters.get('backup_subdir') + if backup_subdir: + backup_subdir=Path(conf_parameters.get('backup_subdir')) + exclude_list=conf_parameters.get('exclude_list') + day_limit=conf_parameters.get('day_limit') + storage_type=conf_parameters.get('storage_type') + mount_parameters=conf_parameters.get('mount_parameters') + remount_ro=conf_parameters.get('remount_ro') + remote_user_host=conf_parameters.get('remote_user_host') + + # Generate exclude list + if exclude_list != None: + for exclude_pattern in exclude_list: + print("excluding",exclude_pattern) + + # If storage_type is offline, mount offline storage read-write + if storage_type=='offline': + try: + if not backup_root.is_dir(): + raise Exception("Error: The mount point "+backup_root+" does not exist.") + except Exception as errmsg: + print("") + print(errmsg) + print("This is a fatal error and ribs cannot continue.") + print("Please check the config file:",config,", or create the mountpoint.") + sys.exit(1) + else: + mount_offline(backup_root,mount_parameters,'rw') + + # Make sure backup_root exists, is writable, and supports hard links + if storage_type=='local' or storage_type=='offline': + testdir=backup_root.joinpath(Path('testdir')) + testfile=testdir.joinpath(Path('testfile')) + testlink=testdir.joinpath(Path('testlink')) + + # Test for existance + if storage_type=='local': + print("Testing for existance of",backup_root,"... ",end="") + try: + if not backup_root.is_dir(): + raise Exception(backup_root+" is not a directory.") + except Exception as errmsg: + print("FAILED") + print("") + print(errmsg) + print("") + print("This is a fatal error and ribs cannot continue.") + print("Please check the config file:",config) + sys.exit(1) + else: + print("OK") + + # Test for writability + if simulate: + print("Skipping all write testing since we're only simulating.") + else: + print("Testing for writability of",backup_root,"... ",end="") + try: + testdir.mkdir(exist_ok=True) + testfile.touch(exist_ok=True) + testfile.unlink(missing_ok=True) + shutil.rmtree(testdir) + except Exception as errmsg: + print("FAILED") + print("") + print(errmsg) + print("") + print("This is a fatal error and ribs cannot continue.") + print("Please check the config file:",config) + if storage_type=='offline': + mount_offline(backup_root,mountparams,'u') + sys.exit(1) + else: + print("OK") + + # Test for hard link support + print("Testing for hard link support on",backup_root,"... ",end="") + try: + testdir.mkdir(exist_ok=True) + testfile.touch(exist_ok=True) + testlink.hardlink_to(testfile) + testlink.unlink(missing_ok=True) + testfile.unlink(missing_ok=True) + shutil.rmtree(testdir) + except Exception as errmsg: + print("FAILED") + print("") + print(errmsg) + print("") + print("This is a fatal error and ribs cannot continue.") + print("Please check the config file:",config) + if storage_type=='offline': + mount_offline(backup_root,mountparams,'u') + sys.exit(1) + else: + print("OK") + + # Determine today's date and generate current datetime directory name + datetimedir_format='%Y-%m-%d_%H:%M:%S' + now=datetime.now() + current_dirname=now.strftime(datetimedir_format) + this_dirname=Path(current_dirname) + + # Find previous backup's directory, if present, to use as link destination + dirname_pattern='????-??-??*' + prev_backups = [] + + if storage_type=='remote': + # get remote listing + pass + else: + # get local or offline listing + if backup_subdir: + print("The backup subdirectory is",backup_subdir) + backups_location=backup_root.joinpath(backup_subdir) + else: + backups_location=backup_root + matches=backups_location.glob(dirname_pattern) + #matching_dirs=glob.glob(os.path.join(backup_root+backup_subdir+"/",dirname_pattern)) + matching_dirs=[match for match in matches if match.is_dir()] + if matching_dirs: + for dirpath in matching_dirs: + prev_backups.append(dirpath) + + if len(prev_backups)==0: + print("No previous backups were found") + has_previous=False + else: + has_previous=True + prev_backups.sort() + last_dirname=prev_backups[-1] + + last_backup=Path(last_dirname) + print("The last backup was in:",last_backup) + + this_backup=backups_location.joinpath(this_dirname) + print("This backup will be in:",this_backup) + + + # Set rsync target and link destination directories + if storage_type=='remote': + target_dir=remote_user_host+":"+this_backup + link_dir=last_backup+'/' + else: + target_dir=this_backup + if has_previous: + link_dir=last_backup + else: + link_dir=None + + # Generate rsync options + rsync_options=[] + #rsync_options.append('--verbose') + rsync_options.append('--secluded-args') + rsync_options.append('--archive') + rsync_options.append('--human-readable') + rsync_options.append('--delete') + if has_previous: + rsync_options.append('--link-dest='+str(link_dir)) + if simulate: + rsync_options.append('-n') + + # Create backup_subdir if necessary + if simulate: + print("Unable to create backup subdirectory since we're just simulating.") + else: + if backup_subdir: + backups_location.mkdir(exist_ok=True) + + # Run the rsync command + try: + print("Running rsync... ",end="") + run_rsync(str(source_dir), str(this_backup)+'/', options=rsync_options, excludes=exclude_list) + except Exception as errmsg: + print("FAILED") + print("") + print(errmsg) + else: + print("OK") + + # Delete backup directories older than day_limit + try: + if simulate: + print("Simulating ",end="") + print("Deleting backups that are more than",day_limit,"days old.") + if has_previous: + would_delete=False + for old_backup in prev_backups: + backup_date=datetime.strptime(os.path.basename(old_backup),datetimedir_format) + if now-backup_date>timedelta(days=int(day_limit)): + would_delete=True + if simulate: + print("Simulating deleting",os.path.basename(old_backup)) + else: + print("Deleting",os.path.basename(old_backup)) + shutil.rmtree(old_backup) + if not would_delete: + print("No backups are more than",day_limit,"days old, so not deleting.") + else: + print("There are no previous backups to delete") + except Exception as errmsg: + print("") + print(errmsg) + + + + # Synchronize disks + + # If STORAGE_TYPE is OFFLINE, update the capacity file + + # If STORAGE_TYPE is OFFLINE, unmount, and remount read-only if required + + + +# ****** END function definitions ****** + +# Call main function +main() diff --git a/etc/ribs/exclude_list b/etc/ribs/exclude_list deleted file mode 100644 index 94e5999..0000000 --- a/etc/ribs/exclude_list +++ /dev/null @@ -1,7 +0,0 @@ -/archive/ -/backups/rescuebru-backups/ -/old_VirtualBox_files/ -/RescueBRU/ -/VirtualBox_VMs/ -/VMwareBackups/ -/VMwareDatastore/ diff --git a/etc/ribs/ribs.conf b/etc/ribs/ribs.conf deleted file mode 100644 index c139a40..0000000 --- a/etc/ribs/ribs.conf +++ /dev/null @@ -1,28 +0,0 @@ -#!/bin/bash -# Configuration file for ribs.sh - -# Dry run option. Set to 0 for normal mode or 1 for testing. -DRY_RUN=0 - -# Remount option. Set to 0 to not remount or 1 to remount read-only. -REMOUNT=1 - -# Mount point of backup drive -BACKUP_ROOT="/mnt/incremental_backup_drive" - -# Backup device -BACKUP_DEV="/dev/sde" -BACKUP_PART=$BACKUP_DEV"1" - -# Backup device serial number (as reported by smartctl -a) -BACKUP_DEV_SN=W4Z1490W - -# File containing patterns to exclude -EXC_FILE="/etc/ribs/exclude_list" - -#Number of days of backups to keep -DAY_LMT=60 - -# source directory -SRC="/mnt/raid/" - diff --git a/install.sh b/install.sh new file mode 100755 index 0000000..8cf747d --- /dev/null +++ b/install.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +# RIBS installer + +rsync -av --chown=root:root dist/ / +chmod 777 /var/log/ribs + diff --git a/usr/local/bin/ribs.sh b/usr/local/bin/ribs.sh deleted file mode 100644 index 97a8fd2..0000000 --- a/usr/local/bin/ribs.sh +++ /dev/null @@ -1,188 +0,0 @@ -#!/bin/bash - -# Full path of configuration file -CONF_FILE="/etc/incremental_backup/incremental_backup.conf" - -echo "----------------- RIBS ----------------" -echo "--- Rod's Incremental Backup System ---" -echo "" - -# Read the configuration file -echo -n "Reading configuration file... " -source $CONF_FILE -if [ $? -ne 0 ]; then - echo -e "\e[31mFAILED\e[0m" - echo "Aborting." - exit 1 -else - echo -e "\e[32mOK\e[0m" -fi - -# Today's date -DAY0=`date -I` -echo "Today's date: $DAY0" - -# Previous backup's date -DAY1=`ls $BACKUP_ROOT | grep -v lost+found | tail -n1` -if [ "$DAY1" == "" ]; then - DAY1=`date -I -d "yesterday"` - echo "No previous backup date. Using yesterday's date." -elif [ "$DAY1" == "$DAY0" ]; then - DAY1=`ls $BACKUP_ROOT | grep -v lost+found | tail -n1 | head -n1` - echo "Previous backup date: $DAY1" -else - echo "Previous backup date: $DAY1" -fi - -# target directory -TRG="$BACKUP_ROOT/$DAY0" - -# link destination directory: -LNK="$BACKUP_ROOT/$DAY1" - -# rsync options -OPT="-ah --delete --link-dest=$LNK --exclude-from=$EXC_FILE" -if [ "$DRY_RUN" -eq 1 ]; then - OPT="$OPT -n" -fi - - -echo -n "Run started " -date -echo "" -# -------- Start backup ------- -echo "Starting incremental backup." -if [ "$DRY_RUN" -eq 1 ]; then - echo "This will be a dry run. Nothing will actually be backed up." - sleep 10 -fi -# Check to see if the backup drive is connected -echo -n "Checking if device connected... " -if [ ! -b "$BACKUP_DEV" ]; then - echo -e "\e[31mFAILED\e[0m" - echo "The backup device is not present." - echo "Check to make sure it's connected and powered on and that the" - echo "device name in $CONF_FILE is correct." - echo "Aborting." - exit 1 -else - echo -e "\e[32mOK\e[0m" -fi -# Check that the backup drive serial number is correct -echo -n "Checking device serial number... " -dev_sn=`smartctl -a $BACKUP_DEV|grep "Serial Number:"|cut -d: -f2| sed 's/[[:space:]]//g'` -if [ "$dev_sn" != "$BACKUP_DEV_SN" ]; then - echo -e "\e[31mFAILED\e[0m" - echo "The backup device that is attached does not have the correct" - echo "serial number. Make sure the correct device is connected and" - echo "check that the serial number in $CONF_FILE" - echo "matches the serial number of the device as reported by" - echo "smartctl -a $BACKUP_DEV" - echo "Aborting." - exit 1 -else - echo -e "\e[32mOK\e[0m" -fi -# Mount the backup drive -# First, check to see that drive is mounted -mount|grep -q $BACKUP_PART -if [ $? -ne 0 ]; then - echo -n "Backup device partition not mounted. Mounting read-only... " - mount $BACKUP_PART $BACKUP_ROOT -o ro - if [ $? -ne 0 ]; then - echo -e "\e[31mFAILED\e[0m" - echo "An error was encountered while attempting to mount" - echo "$BACKUP_PART on $BACKUP_ROOT" - echo "Aborting." - exit 1 - else - echo -e "\e[32mOK\e[0m" - fi -fi -# Next, if it's mounted ro, remount it rw -mount|grep $BACKUP_PART |grep -q "(rw)" -if [ $? -ne 0 ]; then - echo "Backup device partition mounted read-only." - echo -n " Remounting read-write... " - mount $BACKUP_PART $BACKUP_ROOT -o remount,rw - if [ $? -ne 0 ]; then - echo -e "\e[31mFAILED\e[0m" - echo "An error was encountered while attempting to mount" - echo "$BACKUP_PART on $BACKUP_ROOT in read-write mode." - echo "Aborting." - exit 1 - else - echo -e "\e[32mOK\e[0m" - fi -else - echo "Backup device partition already mounted read-write. Continuing" -fi - -# Execute the backup -echo -n "running rsync backup... " -rsync $OPT $SRC $TRG -if [ $? -ne 0 ]; then - echo -e "\e[31mFAILED\e[0m" - echo "An error was encountered while attempting to backup. No old" - echo "backups will be removed." - if [ "$REMOUNT" -eq 1 ]; then - mount $BACKUP_PART $BACKUP_ROOT -o remount,ro - else - umount $BACKUP_ROOT - fi - exit 1 -else - echo -e "\e[32mOK\e[0m" -fi - - -# Date of $DAY_LMT -DAY_TO_REMOVE=`date -I -d "$DAY_LMT days ago"` - -# Delete the specified backup, if it exists -if [ "$DRY_RUN" -eq 1 ]; then - echo "This is a dry run, so no old backups will be removed." -else - if [ -d $BACKUP_ROOT/$DAY_TO_REMOVE ]; then - echo -n "Removing oldest backup... " - rm -r $BACKUP_ROOT/$DAY_TO_REMOVE - if [ $? -ne 0 ];then - echo -e "\e[31mFAILED\e[0m" - else - echo -e "\e[32mOK\e[0m" - fi - fi -fi - -# Unmount the backup drive and exit -echo -n "Syncing disks... " -sync -if [ $? -ne 0 ];then - echo -e "\e[31mFAILED\e[0m" -else - echo -e "\e[32mOK\e[0m" -fi -if [ "$REMOUNT" -eq 1 ]; then - echo -n "Remounting backup device read-only... " - mount $BACKUP_PART $BACKUP_ROOT -o remount,ro - if [ $? -ne 0 ]; then - echo -e "\e[31mFAILED\e[0m" - else - echo -e "\e[32mOK\e[0m" - fi -else - echo -n "Unmounting backup device... " - umount $BACKUP_PART - if [ $? -ne 0 ]; then - echo -e "\e[31mFAILED\e[0m" - else - echo -e "\e[32mOK\e[0m" - fi -fi -echo "Backup complete." -echo "" -echo -n "Run ended " -date -echo "" - -exit 0