#!/usr/bin/env python3 """ Program: Rod's Incremental Backup System (ribs) Author: Rod Wright Date: 03/19/2025 Copyright (C) 2025 Rod Wright SPDX-License-Identifier: GPL-2.0 """ import subprocess import sys import os import shutil import glob from datetime import datetime, timedelta import textwrap import argparse import configparser from pathlib import Path import re # App version APP_VERSION="2.0" # File paths SYS_CONF=Path('/etc/ribs') USER_CONF=Path.home().joinpath(Path('.ribs')) AVAILABLE=Path('conf-available') ENABLED=Path('conf-enabled') CONF_TEMPLATE=Path('ribs.conf.sample') SYS_AVAILABLE=SYS_CONF.joinpath(AVAILABLE) SYS_ENABLED=SYS_CONF.joinpath(ENABLED) USER_AVAILABLE=USER_CONF.joinpath(AVAILABLE) USER_ENABLED=USER_CONF.joinpath(ENABLED) # Determine if I have superuser privileges if os.geteuid()==0: superuser=True else: superuser=False # ****** BEGIN function definitions ****** def main(): """ Main function Args: No arguments Returns: Nothing """ print(f"Rod's Incremental Backup System, version {APP_VERSION}") print("") if not superuser: if not USER_CONF.exists(): print("Creating user conf directory") USER_CONF.mkdir() USER_AVAILABLE.mkdir() USER_ENABLED.mkdir() user_conf_template=USER_AVAILABLE.joinpath(CONF_TEMPLATE) sys_conf_template=SYS_AVAILABLE.joinpath(CONF_TEMPLATE) if user_conf_template.is_file(): user_conf_template.unlink() if not user_conf_template.is_symlink(): user_conf_template.symlink_to(sys_conf_template) print("") cmdline=parse_cmdline() # Set flags global simulate simulate=cmdline['simulate'] # Dispatch operation match cmdline['operation']: case 'config': config_menu() case 'list': config_list() case 'enable'|'disable': config_action(cmdline['operation'],cmdline['conf_file_arg']) case 'run': run_backup(cmdline['conf_file_arg']) case _: print("Invalid operation") def parse_cmdline(): """ Parses command line arguments Args: No arguments Returns: dict(cmdline): { simulate: True | False, operation: 'run' | 'config' | 'enable' | 'disable' | None, conf_file_arg: 'conf_file'|None } """ cmdline={} parser=argparse.ArgumentParser(description='Rod\'s Incremental Backup System, version '+APP_VERSION) group=parser.add_mutually_exclusive_group() parser.add_argument( '-s', '--simulate', help='simulate operations to be performed', action="store_true" ) group.add_argument( '-c', '--config', help='create, remove, modify, enable, or disable config files', action="store_true" ) group.add_argument( '-l', '--list-configs', help='list all config files', action="store_true" ) group.add_argument( '-e', '--enable-config', metavar='CONF_FILE', help='enable a config file', ) group.add_argument( '-d', '--disable-config', metavar='CONF_FILE', help='disable a config file', ) group.add_argument( "conf_file", help="Run RIBS using the specified config file. If none specified, run all enabled config files, looking in '~/.ribs/conf-enabled' for standard users, or in '/etc/ribs/conf-enabled' for superusers.", nargs='?' ) args=parser.parse_args() # Flags if args.simulate: cmdline['simulate']=True else: cmdline['simulate']=False # Operations if args.config: cmdline['operation']='config' cmdline['conf_file_arg']=None elif args.list_configs: cmdline['operation']='list' elif args.enable_config: cmdline['operation']='enable' cmdline['conf_file_arg']=args.enable_config elif args.disable_config: cmdline['operation']='disable' cmdline['conf_file_arg']=args.disable_config else: cmdline['operation']='run' cmdline['conf_file_arg']=args.conf_file return cmdline def run_rsync(source, destination, options=None, excludes=None): """ Runs rsync with the given source, destination, options, and excludes. Args: source (str): The source path. destination (str): The destination path. options (list, optional): A list of rsync options. Defaults to None. excludes (list, optional): A list of patterns to exclude. Defaults to None. Returns: int: The return code of the rsync command. """ command = ["rsync"] if options: command.extend(options) if simulate: command.extend(['--dry-run']) if excludes: for pattern in excludes: command.extend(['--exclude='+pattern]) command.extend([source, destination]) #print(command) process = subprocess.run(command, capture_output=True, text=True) if '-v' in options or '--verbose' in options: print(process.stdout) if process.returncode != 0: print(f"Error running rsync: {process.stderr}") return process.returncode def mount_storage(mountpoint,operation,mountparams=None): """ Handle mounting and unmounting of offline storage. Args: mountpoint (str or path object): The directory on the filesystem where something will be mounted. operation (str): The mount operation. one of: 'ro': mount read-only 'rw': mount read-write 'u': unmount mountparams (str, optional): The mount parameters. This would be a string containing the arguments to the mount(8) command, up to but not including the mount point. Returns: int: The return code of the command """ # Check to see if mountpoint exists try: if not mountpoint.is_dir(): raise Exception(f"Error: The mount point {mountpoint} does not exist or is not a directory. This is a fatal error and ribs cannot continue with this config file. Please check the config file: {config}, or create the mountpoint.") except Exception as errmsg: print("") print(errmsg) return 1 # Check to see if something is already mounted there if mountpoint.is_mount(): mounted=True # Check to see if it is writeable try: writefile=mountpoint.joinpath(Path('writetest.tmp')) writefile.touch(exist_ok=True) writefile.unlink(missing_ok=True) except Exception: # May be mounted read-only. writeable=False else: writeable=True else: mounted=False # Build the command if operation=='u': mountcmd=['umount'] mountcmd.extend([str(mountpoint)]) if not mounted: # Nothing to do mountcmd.clear() elif operation=='ro': mountcmd=['mount'] if mounted: # Already mounted if not writeable: # Already read-only mountcmd.clear() else: # We are mounted read-write. Attempt to remount read-only. if mountparams: mountcmd.extend([mountparams]) mountcmd.extend([str(mountpoint)]) mountcmd.extend(['-oremount,ro']) else: # Not yet mounted if mountparams: mountcmd.extend([mountparams]) mountcmd.extend([str(mountpoint)]) mountcmd.extend(['-oro']) elif operation=='rw': mountcmd=['mount'] if mounted: # Already mounted if writeable: # Already writeable mountcmd.clear() else: # We are mounted read-only. Attempt to remount read-write. if mountparams: mountcmd.extend([mountparams]) mountcmd.extend([str(mountpoint)]) mountcmd.extend(['-oremount,rw']) else: # Not yet mounted if mountparams: mountcmd.extend([mountparams]) mountcmd.extend([str(mountpoint)]) mountcmd.extend(['-orw']) else: print("Unknown operation passed to mount_storage") return 1 if len(mountcmd)>0: mountproc=subprocess.run(mountcmd, capture_output=True, text=True) if mountproc.returncode != 0: print(f"Error occurred while attempting to mount or unmount: {mountproc.stderr}") return mountproc.returncode else: return 0 def config_menu(): """ Show a menu of operations to perform on config files. Args: none Returns: nothing """ while True: print("Select an action",end="") if simulate: print("(simulated):\n") else: print(":\n") print("1. [Llist config files") print("2. [C]reate a config file") print("3. [R]emove an available config file") print("4. [M]odify an available config file") print("5. [E]nable an available config file") print("6. [D]isable an enabled config file") #print("H. Config file help") print("Q. Quit") selection = input("\nEnter your selection: ") match selection: case '1'|'l'|'L': config_list() input("\nPress Enter to continue...") print("") case '2'|'c'|'C': config_create() input("\nPress Enter to continue...") print("") case '3'|'r'|'R': config_action('remove') input("\nPress Enter to continue...") print("") case '4'|'m'|'M': config_action('modify') input("\nPress Enter to continue...") print("") case '5'|'e'|'E': config_action('enable') input("\nPress Enter to continue...") print("") case '6'|'d'|'D': config_action('disable') input("\nPress Enter to continue...") print("") # case 'h' | 'H': # config_help() # input("\nPress Enter to continue...") # print("") case 'q' | 'Q': print("") return 0 case _: print("Invalid selection. Please try again.") print("") def config_list(): """ Find and print a list of config files. Args: No arguments Returns: Nothing """ if superuser: conftype='system' availconfdir=SYS_AVAILABLE enaconfdir=SYS_ENABLED else: conftype='user' availconfdir=USER_AVAILABLE enaconfdir=USER_ENABLED configs=find_configs(conftype) print(f"{conftype} config files:\n") if len(configs)==0: wrap_message(f"No available {conftype} config files found. Refer to the sample config file {SYS_AVAILABLE}/ribs.conf.sample to create one. Note that config files must have the .conf extension to be recognized.") else: for config in configs: if Path(enaconfdir/config).is_symlink(): enabled_ind='enabled ------>' else: enabled_ind='available ---->' print(f"{enabled_ind} {config}") print("") def config_action(action,configfile=None): """ Take some action on a config file. Args: action (str): The action to take. one of: 'enable'|'disable'|'modify'|'remove' configfile (str, optional): The filename of a configuration file Returns: int: 0 on success or 1 on failure """ if simulate: print("\nAll actions will be simulated") if superuser: conftype='system' availconfdir=SYS_AVAILABLE enaconfdir=SYS_ENABLED else: conftype='user' availconfdir=USER_AVAILABLE enaconfdir=USER_ENABLED action_file=None match action: case 'disable': configs=find_configs(conftype,confstatus='enabled') case 'enable': aconfigs=find_configs(conftype) econfigs=find_configs(conftype,confstatus='enabled') configs=[conf for conf in aconfigs if conf not in econfigs] case 'modify'|'remove': configs=find_configs(conftype) if len(configs)==0: print("No config files were found.") return 1 if configfile: configfile=Path(configfile) if configfile in configs: # We're good. That's what we'll use. action_file=configfile else: # The config file supplied doesn't exist print(f"The config file {configfile} does not exist.") return 1 else: action_file=select_config(configs,action=action) match action: case 'enable': if simulate: print("Simulating enabling",Path(action_file).name) else: print("Enabling config:",Path(action_file).name) link=Path(enaconfdir/action_file) target=Path(availconfdir/action_file) try: link.symlink_to(target) except FileExistsError: print("That config file is already enabled.") return 1 case 'disable': if simulate: print("Simulating disabling",Path(action_file).name) else: print("Disabling config:",Path(action_file).name) link=Path(enaconfdir/action_file) try: link.unlink() except FileNotFoundError: print("That config file is already disabled.") return 1 case 'modify': if simulate: print("Simulating modifying",Path(action_file).name) else: print("Modifying config:",action_file) filepath_to_mod=Path(availconfdir/action_file) editor = os.environ.get('EDITOR') if editor: subprocess.run([editor, filepath_to_mod]) else: try: result=subprocess.run(['xdg-open', filepath_to_mod], stderr = subprocess.DEVNULL) if result.returncode!=0: raise ValueError except (ValueError, FileNotFoundError): try: subprocess.run(['vi', filepath_to_mod]) except FileNotFoundError: try: subprocess.run(['nano', filepath_to_mod]) except FileNotFoundError: print("No default editor found. Please set the $EDITOR environment variable or install a common text editor.") return 1 case 'remove': if simulate: print("Simulating removing",Path(action_file).name) else: print("Removing config:",action_file) ena_to_remove=Path(enaconfdir/action_file) avail_to_remove=Path(availconfdir/action_file) try: if ena_to_remove.is_symlink(): ena_to_remove.unlink() avail_to_remove.unlink() except FileNotFoundError: print("That config file has already been removed.") return 1 case _: print("Invalid action passed to config_action()") return 1 return 0 def config_create(): """ Create a config file. Args: none Returns: Nothing """ if simulate: print("\nAll actions will be simulated") if superuser: conftype='system' availconfdir=SYS_AVAILABLE else: conftype='user' availconfdir=USER_AVAILABLE # Prompt for a filename that doesn't already exist print("\nCreating a new config file using ribs.conf.sample as the template.\n") print("Please enter a filename. The file must not already exist") print("and the filename extension must be '.conf'.") print("Enter c to cancel.") fnvalid=False while not fnvalid: newfn=input("\nfilename: ") existing_confs=find_configs(conftype) if newfn=='c' or newfn=='C': return 0 elif Path(newfn) in existing_confs: print("That file already exists.") print("Try again or enter c to cancel.") elif not newfn.endswith('.conf'): print("The filename must end with '.conf'.") print("Try again or enter c to cancel.") else: fnvalid=True # Copy ribs.conf.sample to a new file with the provided filename and edit. if simulate: print("Simulating creating ",newfn) else: try: shutil.copy(availconfdir.joinpath(Path('ribs.conf.sample')), availconfdir.joinpath(Path(newfn))) except Exception as e: print(f"An error occurred: {e}") return 1 print(f"New config file {newfn} created. Opening for editing...") config_action('modify',newfn) print("...done. In order to run this config file, you must enable it.") def find_configs(conftype,confstatus='available'): """ Find config files of the specified type. Args: conftype (str): 'user' | 'system'. confstatus (str, optional): ''available' | 'enabled' Returns: list(configs): A list of names of config files """ if conftype=='user': if confstatus=='enabled': dirpath=USER_ENABLED elif confstatus=='available': dirpath=USER_AVAILABLE elif conftype=='system': if confstatus=='enabled': dirpath=SYS_ENABLED elif confstatus=='available': dirpath=SYS_AVAILABLE pattern="*.conf" configs = [] matching_files=sorted(dirpath.glob(pattern)) if matching_files: for filepath in matching_files: filename=Path(filepath.name) configs.append(filename) return(configs) def select_config(configfiles,action=None): """ Show a menu of config files of the provided list, allow the user to select one, and return the path of the selected one. Args: configfiles (list): a list of config files. Returns: None or path object: A path object of a config file. """ if not configfiles: return None if action: print(f"Select a config file to {action}:\n") else: print(f"Select a config file:\n") for i, item in enumerate(configfiles): print(f"{i + 1}. {item}") print("Q. Quit") while True: try: selection = input("\nEnter your selection: ") if selection=='q' or selection=='Q': return None selection=int(selection) if 1 <= selection <= len(configfiles): return configfiles[selection - 1] else: print("Invalid selection. Please try again.") except ValueError: print("Invalid input. Please enter a number.") def parse_config_file(conf_file): """ Read the specified config file and return the parameters. Args: conf_file (str or path object): The full path of a configuration file. Returns: dict: A dictionary of option:value pairs """ config=configparser.ConfigParser(allow_no_value=True) config.read(conf_file) # Check for the [General] section and its options config_errors=[] if config.has_section('General'): if config.has_option('General','source_dir'): source_dir=config['General']['source_dir'] else: config_errors.append('source_dir option missing from config file') if config.has_option('General','backup_root'): backup_root=config['General']['backup_root'] else: config_errors.append('backup_root option missing from config file') if config.has_option('General','backup_subdir'): backup_subdir=config['General']['backup_subdir'] else: backup_subdir=None if config.has_option('General','exclude_list'): exclude_list=config['General']['exclude_list'].split(',') exclude_list=[pattern.strip() for pattern in exclude_list] else: exclude_list=None if config.has_option('General','day_limit'): day_limit=config['General']['day_limit'] else: config_errors.append('day_limit option missing from config file') if config.has_option('General','storage_type'): storage_type=config['General']['storage_type'] else: config_errors.append('storage_type option missing from config file') else: config_errors.append('General section missing from config file') # Abort on [General] section errors if len(config_errors)>0: print("The following configuration file errors were found") for err_msg in config_errors: print(err_msg) print("These are fatal errors and ribs cannot continue.") print("Please check the config file:",conf_file) return 1 if storage_type=="local": # Check for the [local] section and its options config_errors=[] if config.has_section('local'): # Reserved for future use pass else: pass # Abort on [local] section errors if len(config_errors)>0: print("The following configuration file errors were found") for err_msg in config_errors: print(err_msg) print("These are fatal errors and ribs cannot continue.") print("Please check the config file:",conf_file) return 1 elif storage_type=="offline": # Check for the [offline] section and its options config_errors=[] if config.has_section('offline'): if config.has_option('offline','mount_parameters'): mount_parameters=config['offline']['mount_parameters'] else: mount_parameters=None if config.has_option('offline','remount_ro'): remount_ro=config['offline']['remount_ro'] if remount_ro=="": remount_ro=False else: remount_ro=False else: mount_parameters=None remount_ro=False # Abort on [offline] section errors if len(config_errors)>0: print("The following configuration file errors were found") for err_msg in config_errors: print(err_msg) print("These are fatal errors and ribs cannot continue.") print("Please check the config file:",conf_file) return 1 elif storage_type=="remote": # Check for the [remote] section and its options config_errors=[] if config.has_section('remote'): if config.has_option('remote','remote_user_host'): remote_user_host=config['remote']['remote_user_host'] else: config_errors.append('remote_user_host option missing from config file') else: config_errors.append('remote section missing from config file') # Abort on [remote] section errors if len(config_errors)>0: print("The following configuration file errors were found") for err_msg in config_errors: print(err_msg) print("These are fatal errors and ribs cannot continue.") print("Please check the config file:",conf_file) return 1 else: config_errors.append('invalid storage_type. Must be local, offline, or remote.') if len(config_errors)>0: print("The following configuration file errors were found") for err_msg in config_errors: print(err_msg) print("These are fatal errors and ribs cannot continue.") print("Please check the config file:",conf_file) return 1 # Check and correct option formatting and abort if necessary config_errors=[] if not source_dir.startswith('/'): config_errors.append('source_dir must be an absolute path, but doesn\'t start with a \'/\'') if not backup_root.startswith('/'): config_errors.append('backup_root must be an absolute path, but doesn\'t start with a \'/\'') if day_limit=='0': config_errors.append('day_limit cannot be 0') if storage_type=='remote' and not "@" in remote_user_host: config_errors.append('remote_user_host must be in the format\'user@host\'') # Abort on unacceptable option format errors if len(config_errors)>0: print("The following configuration file errors were found") for err_msg in config_errors: print(err_msg) print("These are fatal errors and ribs cannot continue.") print("Please check the config file:",conf_file) return 1 # Clean up variables if backup_root.endswith('/'): backup_root=backup_root.rstrip('/') if backup_subdir: if backup_subdir.startswith('/'): backup_subdir=backup_subdir.lstrip('/') if backup_subdir.endswith('/'): backup_subdir=backup_subdir.rstrip('/') # Return a parameter:option dictionary if storage_type=="local": return { 'storage_type':storage_type, 'source_dir':source_dir, 'backup_root':backup_root, 'backup_subdir':backup_subdir, 'exclude_list':exclude_list, 'day_limit':day_limit } elif storage_type=="offline": return { 'storage_type':storage_type, 'source_dir':source_dir, 'backup_root':backup_root, 'backup_subdir':backup_subdir, 'exclude_list':exclude_list, 'day_limit':day_limit, 'mount_parameters':mount_parameters, 'remount_ro':remount_ro } elif storage_type=="remote": return { 'storage_type':storage_type, 'source_dir':source_dir, 'backup_root':backup_root, 'backup_subdir':backup_subdir, 'exclude_list':exclude_list, 'day_limit':day_limit, 'remote_user_host':remote_user_host } else: # Not sure how we got here, but if we did, something's wrong return 1 def update_capacity_file(file_location): """ Update the capacity file for offline media. Detects the capacity of the specified file location and writes it to a capacity.txt file in that location. Args: file_location (path object): The full path of the mount point of offline media. Returns: int: The return code of the command """ cap_file=file_location.joinpath(Path('capacity.txt')) # Check to see if the file_location is mounted try: if not file_location.is_mount(): raise Exception(f"Could not find a mounted filesystem at {file_location}. Not creating capacity file.") except Exception as errmsg: print("") print(errmsg) return 1 # Check to see if the file location is writable try: cap_file.touch(exist_ok=True) cap_file.unlink(missing_ok=True) except Exception as errmsg: print("") print(errmsg) print(f"Could not write to the filesystem at {file_location}. Not creating capacity file.") return 1 # Get free space command=['df'] command.extend(['-h']) command.extend([file_location]) proc=subprocess.run(command, capture_output=True, text=True) if proc.returncode != 0: print(f"Error running {command}: {proc.stderr}") return proc.returncode else: capacitytext=proc.stdout # Write capacity info to file try: cap_file.write_text(capacitytext) except Exception as errmsg: print(errmsg) return 1 def run_backup(configfile=None): """ Run backup using the specified configfile, or all enabled config files if none specified. Args: configfile (str or path object, optional): The full path of a config file. Returns: Nothing """ runstart=datetime.now() print(f"Run started {runstart.ctime()}") if simulate: print("\nAll actions will be simulated") if superuser: conftype='system' availconfdir=SYS_AVAILABLE enaconfdir=SYS_ENABLED else: conftype='user' availconfdir=USER_AVAILABLE enaconfdir=USER_ENABLED configs=[] if configfile==None: # Find and run all enabled configs print("Finding enabled config files...") configs=find_configs(conftype,'enabled') if len(configs)>0: print(f"Found {conftype} config files:") for index, config in enumerate(configs): print(config) configs[index]=enaconfdir.joinpath(config) else: print("No enabled config files found") else: # Check to see if the given config file is in the current directory if Path(configfile).is_file(): configs=[Path(configfile)] else: # Given file is not in current directory. Look in conf-enabled print(f"The config file {configfile} is not in the current directory.") print(f"Searching enabled {conftype} config files...") enaconfigs=find_configs(conftype,'enabled') if len(enaconfigs)>0: print(f"Found enabled {conftype} config files...") if Path(configfile) in enaconfigs: print(f"Found {configfile} in enabled {conftype} config files.") configs=[enaconfdir.joinpath(configfile)] else: print("No enabled config files found.") if len(configs)==0: print("Unable to find the config file you specified. Aborting.") return 1 failed_confs=[] for config in configs: print("Running backup from the config file:\n",config) # Parse config file and create variables conf_parameters=parse_config_file(config) source_dir=conf_parameters.get('source_dir') backup_root=Path(conf_parameters.get('backup_root')) backup_subdir=conf_parameters.get('backup_subdir') if backup_subdir: backup_subdir=Path(conf_parameters.get('backup_subdir')) exclude_list=conf_parameters.get('exclude_list') day_limit=conf_parameters.get('day_limit') storage_type=conf_parameters.get('storage_type') mount_parameters=conf_parameters.get('mount_parameters') remount_ro=conf_parameters.get('remount_ro') if remount_ro=='True': remount_ro=True elif remount_ro=='False': remount_ro=False remote_user_host=conf_parameters.get('remote_user_host') # Test for ability to mount filesystems if storage_type=='offline' and not superuser: wrap_message("You have set storage_type to offline in the config file, but you do not have superuser rights, preventing you from mounting filesystems. You can run ribs with sudo or change the storage_type to local and handle the mounting and unmounting outside of ribs. Aborting.") failed_confs.append(config) continue # Test for existance of backup_root if storage_type=='local' or storage_type=='offline': print(f"Testing for existance of {backup_root}... ",end="") try: if not backup_root.is_dir(): raise Exception(f"{backup_root} does not exist or is not a directory. This is a fatal error and ribs cannot continue. Please check the config file: {config}") except Exception as errmsg: print("FAILED") print("") print(errmsg) failed_confs.append(config) continue else: print("OK") elif storage_type=='remote': print(f"Testing for existance of {backup_root}... ",end="",flush=True) try: command=['ssh', remote_user_host,'if [ -d ',str(backup_root),' ]; then echo \"true\"; else echo \"false\";fi'] etest=subprocess.run(command, capture_output=True, text=True) if etest.stdout=='false': raise Exception(f"{backup_root} does not exist or is not a directory. This is a fatal error and ribs cannot continue. Please check the config file: {config}") except Exception as errmsg: print("FAILED") print("") print(errmsg) failed_confs.append(config) continue else: print("OK") if storage_type=='offline': # Mount offline storage read-write try: print(f"Mounting {backup_root}... ",end="",flush=True) mount_result=mount_storage(backup_root,'rw',mountparams=mount_parameters) if mount_result != 0: print("FAILED") raise Exception(f"Failed to mount {backup_root}") else: print("OK") except Exception as errmsg: print("FAILED") print("") print(errmsg) failed_confs.append(config) continue else: print("OK") # Test for writability if simulate: print("Skipping all write testing since we're only simulating.") else: testdir=backup_root.joinpath(Path('testdir')) testfile=testdir.joinpath(Path('testfile')) print(f"Testing for writability of{backup_root}... ",end="",flush=True) try: if storage_type=='local' or storage_type=='offline': testdir.mkdir(exist_ok=True) testfile.touch(exist_ok=True) testfile.unlink(missing_ok=True) shutil.rmtree(testdir) elif storage_type=='remote': command=['ssh', remote_user_host,'mkdir -p ',str(testdir),'&& touch ',str(testfile),'&& rm -rf ',str(testdir)] wtest=subprocess.run(command, capture_output=True, text=True) if wtest.returncode != 0: raise Exception(f"Could not write to the remote directory {backup_root}") except Exception as errmsg: print("FAILED") print("") print(f"{errmsg} \n This is a fatal error and ribs cannot continue. Please check the config file:{config}") if storage_type=='offline': mount_storage(backup_root,'u') failed_confs.append(config) continue else: print("OK") # Test for hard link support testlink=testdir.joinpath(Path('testlink')) print("Testing for hard link support on",backup_root,"... ",end="",flush=True) try: if storage_type=='local' or storage_type=='offline': testdir.mkdir(exist_ok=True) testfile.touch(exist_ok=True) testlink.hardlink_to(testfile) testlink.unlink(missing_ok=True) testfile.unlink(missing_ok=True) shutil.rmtree(testdir) elif storage_type=='remote': command=['ssh', remote_user_host,'mkdir -p ',str(testdir),'&& touch ',str(testfile),'&& ln ',str(testfile),' ',str(testlink),'&& rm -rf ',str(testdir)] ltest=subprocess.run(command, capture_output=True, text=True) if ltest.returncode != 0: raise Exception(f"The remote directory {backup_root} does not appear to support hard links.") except Exception as errmsg: print("FAILED") print("") print(f"{errmsg} \n This is a fatal error and ribs cannot continue. Please check the config file:{config}") if storage_type=='offline': mount_storage(backup_root,'u') failed_confs.append(config) continue else: print("OK") # Generate exclude list if exclude_list != None: for exclude_pattern in exclude_list: print("excluding",exclude_pattern) # Determine today's date and generate current datetime directory name datetimedir_format='%Y-%m-%d_%H:%M:%S' datedir_format='%Y-%m-%d' now=datetime.now() current_dirname=now.strftime(datetimedir_format) this_dirname=Path(current_dirname) # Find previous backup's directory, if present, to use as link destination dirname_pattern='????-??-??*' prev_backups = [] if storage_type=='local' or storage_type=='offline': # get local or offline listing if backup_subdir: backups_location=backup_root.joinpath(backup_subdir) else: backups_location=backup_root matches=backups_location.glob(dirname_pattern) matching_dirs=[match for match in matches if match.is_dir()] if matching_dirs: for dirpath in matching_dirs: prev_backups.append(dirpath) elif storage_type=='remote': # get remote listing if backup_subdir: backups_location=backup_root.joinpath(backup_subdir) else: backups_location=backup_root command=['ssh',remote_user_host,' cd ',str(backups_location),'&& ls -1d ',dirname_pattern] remotelist=subprocess.run(command, capture_output=True, text=True) matching_dirs=remotelist.stdout.splitlines() if matching_dirs: for dirpath in matching_dirs: prev_backups.append(dirpath) if len(prev_backups)==0: print("No previous backups were found") has_previous=False else: has_previous=True prev_backups.sort() last_dirname=prev_backups[-1] last_backup=backups_location.joinpath(last_dirname) print("The last backup was:",last_backup) this_backup=backups_location.joinpath(this_dirname) print("This backup will be:",this_backup) # Set rsync target and link destination directories if storage_type=='local' or storage_type=='offline': target_dir=this_backup if has_previous: link_dir=last_backup else: link_dir=None elif storage_type=='remote': target_dir=f"{remote_user_host}:{this_backup}" if has_previous: link_dir=last_backup else: link_dir=None # Generate rsync options rsync_options=[] #rsync_options.append('--verbose') rsync_options.append('--secluded-args') rsync_options.append('--archive') rsync_options.append('--human-readable') rsync_options.append('--delete') if has_previous: rsync_options.append('--link-dest='+str(link_dir)+'/') # Create backup_subdir if necessary if simulate: print("Unable to create backup subdirectory since we're just simulating.") else: if backup_subdir: print(f"Creating backup subdirectory {backup_subdir} if necessary.") if storage_type=='local' or storage_type=='offline': backups_location.mkdir(exist_ok=True) elif storage_type=='remote': command=['ssh',remote_user_host,' mkdir -p ',str(backups_location)] subcre=subprocess.run(command, capture_output=True, text=True) # Run the rsync command print("Running rsync... ",end="",flush=True) try: if storage_type=='local' or storage_type=='offline': run_rsync(str(source_dir), str(this_backup)+'/', options=rsync_options, excludes=exclude_list) elif storage_type=='remote': run_rsync(str(source_dir), remote_user_host+':'+str(this_backup)+'/', options=rsync_options, excludes=exclude_list) except Exception as errmsg: print("FAILED") print("") print(errmsg) failed_confs.append(config) continue else: print("OK") # Delete backup directories older than day_limit try: if simulate: print(f"Simulating deleting backups that are more than {day_limit} days old.") print(f"Deleting backups that are more than {day_limit} days old.") if has_previous: would_delete=False for old_backup in prev_backups: if re.search(r'\d{4}-\d{2}-\d{2}_\d{2}:\d{2}:\d{2}',str(old_backup)): backup_date=datetime.strptime(os.path.basename(old_backup),datetimedir_format) else: backup_date=datetime.strptime(os.path.basename(old_backup),datedir_format) if now-backup_date>timedelta(days=int(day_limit)): would_delete=True if simulate: print(f"Simulating deleting {os.path.basename(old_backup)}") else: print(f"Deleting {os.path.basename(old_backup)}") if storage_type=='local' or storage_type=='offline': shutil.rmtree(backups_location.joinpath(old_backup)) elif storage_type=='remote': command=['ssh', remote_user_host,'rm -rf ',str(backups_location.joinpath(old_backup))] delbkup=subprocess.run(command, capture_output=True, text=True) if not would_delete: print("No backups are more than",day_limit,"days old, so not deleting.") else: print("There are no previous backups to delete") except Exception as errmsg: print("") print(errmsg) # Synchronize disks if storage_type=='offline': print("Synchronizing disks...",end="") os.sync() print("OK") # If STORAGE_TYPE is OFFLINE, update the capacity file if storage_type=='offline': update_capacity_file(backup_root) # If STORAGE_TYPE is OFFLINE, unmount, and remount read-only if required try: if storage_type=='offline': if remount_ro: print(f"Remounting {backup_root} read-only... ",end="") operation='ro' else: print(f"Unmounting {backup_root}... ",end="") operation='u' mount_result=mount_storage(backup_root,operation,mountparams=mount_parameters) if mount_result != 0: print("FAILED") raise Exception(f"Failed to unmount/remount {backup_root}.") else: print("OK") except Exception as errmsg: print("") print(errmsg) failed_confs.append(config) continue runend=datetime.now() timediff=(runend-runstart).total_seconds() print(f"Run ended: {runend.ctime()}") print(f"Total time: {str(int(timediff //3600))} hours, {str(int((timediff % 3600) // 60))} minutes, {str(int(timediff % 60))} seconds") if len(failed_confs)>0: print("Failures occurred when running the following config files:") for failed_conf in failed_confs: print(f" {failed_conf}") print("") wrap_message(f"The listed backups may have failed completely or partially. Please check the most recent backup to determine the nature of the failure, and check to make sure the settings in the config file are correct.") return 1 def wrap_message(message_text, width=72): """ Display a long message with the necessary line breaks. Args: message_text (str): The text to display. width (int, optional): The max number of columns in each line. Returns: Nothing """ linelist=textwrap.wrap(message_text,width=width,break_on_hyphens=False) for line in linelist: print(line) # ****** END function definitions ****** # Call main function main()