From 9ad8e12522ae34ce26f36e8f3997067b63bcad19 Mon Sep 17 00:00:00 2001 From: Marc Michalsky Date: Mon, 23 Nov 2020 16:58:21 +0100 Subject: [PATCH] implement backup --- Models/Backup.py | 0 Models/Container.py | 91 -------- Models/NextcloudBackupException.py | 8 - Models/__pycache__/Backup.cpython-38.pyc | Bin 152 -> 0 bytes backup.py | 112 ++++++---- models.py | 265 +++++++++++++++++++++++ requirements.txt | 5 +- restore.py | 79 ++++++- utils.py | 12 + 9 files changed, 430 insertions(+), 142 deletions(-) delete mode 100644 Models/Backup.py delete mode 100644 Models/Container.py delete mode 100644 Models/NextcloudBackupException.py delete mode 100644 Models/__pycache__/Backup.cpython-38.pyc create mode 100644 models.py create mode 100644 utils.py diff --git a/Models/Backup.py b/Models/Backup.py deleted file mode 100644 index e69de29..0000000 diff --git a/Models/Container.py b/Models/Container.py deleted file mode 100644 index 87dee5a..0000000 --- a/Models/Container.py +++ /dev/null @@ -1,91 +0,0 @@ -import datetime -import os -import stat -import tarfile -from Models.NextcloudBackupException import NextcloudBackupException - - -class Container: - - def __init__(self, name, password, backup_folder, compose_file_path) -> None: - self.__datetime = datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S") - self.name = name - self.__password = password - self.backup_folder = backup_folder - self.compose_file_path = compose_file_path - self.__dump_file = self.name + '_' + self.__datetime + '.sql' - self.__dump_file_path = os.path.join(self.backup_folder, self.__dump_file) - self.__tar_file = self.name + '_' + self.__datetime + 'tar.gz' - self.__tar_file_path = os.path.join(self.backup_folder, self.__tar_file) - - def __str__(self) -> str: - return F"name: {self.name}\npassword: {self.__password}\ncompose_file_path: {self.compose_file_path}" - - # Create backup folder if it does not yet exist - def create_backup_dir(self): - if not os.path.isfile(self.backup_folder): - try: - os.makedirs(self.backup_folder) - except OSError as e: - raise OSError(e) - return True - else: - return False - - # Dump database - def __dump_db(self) -> bool: - try: - os.system(F"docker exec {self.name} mysqldump --password={self.__password} --all-databases > " - F"{self.__dump_file}") - return os.path.isfile(self.__dump_file) - except Exception as e: - raise Exception(e) - - # Tar config and settings folder within container and copy it into backup folder - def __export_config(self) -> bool: - try: - os.system(F"docker exec {self.name} tar -czf config_settings.tar config settings") - os.system(F"docker cp {self.name}:/var/www/html/config_settings.tar {self.__tar_file_path}") - os.system(F"docker exec {self.name} rm config_settings.tar") - return os.path.isfile(self.__tar_file_path) - except Exception as e: - raise NextcloudBackupException(e) - - # Tar database with config and settings - def __tar_db(self) -> bool: - try: - with tarfile.open(self.__tar_file_path, 'w:gz') as tarball: - tarball.add(self.__dump_file_path, arcname=self.__dump_file) - return tarball.gettarinfo(self.__dump_file).isfile() - except Exception as e: - raise NextcloudBackupException(e) - - # Set secure file permissions - def __set_file_permissions(self) -> bool: - try: - os.chmod(self.__tar_file_path, stat.S_IREAD) - return oct(os.stat(self.__tar_file_path).st_mode)[-3:] == 600 - except Exception as e: - raise NextcloudBackupException(e) - - # Create backup - def create_backup(self) -> dict: - try: - return {"database dump": self.__dump_db(), - "export config": self.__export_config(), - "include db in backup": self.__tar_db(), - "set secure backup file permissions": self.__set_file_permissions()} - except NextcloudBackupException as e: - raise NextcloudBackupException(e) - - @staticmethod - def deserialize_containers(data: dict) -> list: - containers = [] - for name, values in data['nextcloud_containers'].items(): - containers.append(Container( - name, - values['mysql_password'], - values['backup_folder'], - values['compose_file_path']) - ) - return containers diff --git a/Models/NextcloudBackupException.py b/Models/NextcloudBackupException.py deleted file mode 100644 index 8e22ba4..0000000 --- a/Models/NextcloudBackupException.py +++ /dev/null @@ -1,8 +0,0 @@ - -class NextcloudBackupException(Exception): - def __init__(self, message): - self.message = message - - - def __write_to_log(self): - pass diff --git a/Models/__pycache__/Backup.cpython-38.pyc b/Models/__pycache__/Backup.cpython-38.pyc deleted file mode 100644 index 1bc444dd729942a1404fa5a9d1d5770d8f083caa..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 152 zcmWIL<>g`k0*Ohx;z9Id5P=LBfgA@QE@lA|DGb33nv8xc8Hzx{2;!Hmenx(7s(x-_ zQL=tOWpYMhQEos{epYI7NwI!jYDGzMPJU@hd~tG7W&u#dH$Npcr&!-9F*&=mK(C 0: + for func, traceback in log.exceptions.items(): + _print() + _print(F"{Fore.YELLOW}Exception occurred in method: Log.{func}(){Style.RESET_ALL}") + _print(traceback) + _print() + + # Clean up backup folder + if not utils.no_cleanup: + deleted_files = 0 + backup_dir = os.scandir(container.backup_dir) + backup_files = [file for file in backup_dir if + file.is_file() and file.name.startswith(container.name) and file.name.endswith(".tar.gz")] + + while len(backup_files) > container.number_of_backups: + del_file = min(backup_files, key=os.path.getctime) + backup_files.remove(del_file) + os.remove(del_file) + deleted_files += 1 + + if deleted_files == 1: + _print(F"{Fore.YELLOW}Deleted 1 old backup file.{Style.RESET_ALL}") + elif deleted_files >= 1: + _print(F"{Fore.YELLOW}Deleted {deleted_files} old backup files.{Style.RESET_ALL}") + + return backup_status if __name__ == '__main__': - backup(sys.argv) - + backup() diff --git a/models.py b/models.py new file mode 100644 index 0000000..a7cb892 --- /dev/null +++ b/models.py @@ -0,0 +1,265 @@ +import datetime +import os +import stat +import tarfile +import traceback +import shutil +from colorama import Fore, Style +from pathlib import Path +import utils +from utils import _print + + +class Container: + + def __init__(self, name, password, app_container, db_container, backup_dir, number_of_backups) -> None: + self.__datetime = datetime.datetime.now().strftime("%Y-%m-%d_%H%M%S") + self.name = name + self.__password = password + self.app_container = app_container + self.db_container = db_container + self.backup_dir = backup_dir + self.tmp_dir = os.path.join(backup_dir, 'tmp') + self.number_of_backups = number_of_backups + self.__dump_file = self.name + '_' + self.__datetime + '.sql' + self.__dump_file_path = os.path.join(self.tmp_dir, self.__dump_file) + self.__tar_file = self.name + '_' + self.__datetime + '.tar' + self.__tar_file_path = os.path.join(self.tmp_dir, self.__tar_file) + self.tar_gz_file = self.__tar_file + '.gz' + self.tar_gz_file_path = os.path.join(self.backup_dir, self.tar_gz_file) + self.exceptions = {} + self.SUCCESS = F"{Fore.GREEN}success{Style.RESET_ALL}" + self.FAILED = F"{Fore.RED}failed{Style.RESET_ALL}" + + global quiet_mode + + # Create backup dir if it does not yet exist + def __create_backup_dir(self): + if not os.path.isdir(self.backup_dir): + try: + os.makedirs(self.backup_dir) + _print(F"{Fore.GREEN}Backup folder created under: {self.backup_dir}{Style.RESET_ALL}") + return True + except: + _print(F"{Fore.RED}Could not backup tmp folder under {self.backup_dir}{Style.RESET_ALL}") + self.exceptions.update({'create_backup_dir': traceback.format_exc()}) + return False + else: + return True + + # Create tmp dir + def __create_tmp_dir(self) -> bool: + if not os.path.isdir(self.tmp_dir): + try: + os.mkdir(self.tmp_dir) + return os.path.isdir(self.tmp_dir) + except: + _print(F"{Fore.RED}Could not create tmp folder{Style.RESET_ALL}") + self.exceptions.update({'create_tmp_dir': traceback.format_exc()}) + return False + else: + if self.__delete_tmp_dir(): + return self.__create_tmp_dir() + + # Delete tmp dir + def __delete_tmp_dir(self) -> bool: + try: + shutil.rmtree(self.tmp_dir) + return not os.path.isdir(self.tmp_dir) + except: + _print(F"{Fore.RED}Could not delete old tmp folder{Style.RESET_ALL}") + self.exceptions.update({'delete_tmp_dir': traceback.format_exc()}) + return False + + # Dump database + def __dump_db(self) -> bool: + try: + os.system( + F"docker exec {self.db_container} mysqldump --password={self.__password} --all-databases > {self.__dump_file_path}") + status = os.path.isfile(self.__dump_file_path) + _print(F"Dump Nextcloud database: {self.SUCCESS if status else self.FAILED}") + return status + except: + _print(F"Dump Nextcloud database: {self.FAILED}") + self.exceptions.update({'__dump_db': traceback.format_exc()}) + return False + + # Import database + def __import_db(self) -> bool: + try: + os.system( + F"docker exec {self.db_container} mysql -u root --password={self.__password} < {self.__dump_file_path}") + status = True # TODO: Implement test if database import was successful + _print(F"Import Nextcloud database: {self.SUCCESS if status else self.FAILED}") + return status + except: + _print(F"Import Nextcloud database: {self.FAILED}") + self.exceptions.update({'__import_db': traceback.format_exc()}) + return False + + # Tar config folder within container and copy it into backup folder + def __export_config(self) -> bool: + try: + os.system(F"docker exec {self.app_container} tar -cf config.tar config/") + os.system(F"docker cp {self.app_container}:/var/www/html/config.tar {self.__tar_file_path}") + os.system(F"docker exec {self.app_container} rm config.tar") + if os.path.isfile(self.__tar_file_path): + with tarfile.open(self.__tar_file_path, 'r') as tarball: + tarball.extractall(self.tmp_dir) + status = os.path.isdir(os.path.join(self.tmp_dir, "config")) + _print(F"Export Nextcloud configuration: {self.SUCCESS if status else self.FAILED}") + return status + except: + _print(F"Export Nextcloud configuration: {self.FAILED}") + self.exceptions.update({'__export_config': traceback.format_exc()}) + return False + + # Tar config folder within container and copy it into backup folder + def __import_config(self) -> bool: + try: + if os.path.isdir(os.path.join(self.tmp_dir, "config")): + with tarfile.open(self.__tar_file_path, 'w') as tarball: + tarball.add(os.path.join(self.tmp_dir, "config"), arcname="/config") + os.system(F"docker cp {self.__tar_file_path} {self.app_container}:/var/www/html/config.tar") + os.system(F"docker exec {self.app_container} rm -r config") + os.system(F"docker exec {self.app_container} tar -xf config.tar") + status = True # TODO: implement a test if export into docker container was successful + _print(F"Import Nextcloud configuration: {self.SUCCESS if status else self.FAILED}") + return status + except: + _print(F"Import Nextcloud configuration: {self.FAILED}") + self.exceptions.update({'__import_config': traceback.format_exc()}) + return False + + # Tar database with config and settings + def __tar_backup(self) -> bool: + try: + with tarfile.open(self.tar_gz_file_path, 'w:gz') as tarball: + tarball.add(self.__dump_file_path, arcname=self.__dump_file) + tarball.add(os.path.join(self.tmp_dir, "config"), arcname="/config") + status = True # TODO: Implement a test to confirm that files where added to tar file + _print(F"Zip backup: {self.SUCCESS if status else self.FAILED}") + return status + except: + _print(F"Zip backup: {self.FAILED}") + self.exceptions.update({'__tar_db': traceback.format_exc()}) + return False + + # Untar backup + def __untar_backup(self, backup_file_path) -> bool: + try: + with tarfile.open(backup_file_path, 'r:gz') as tarball: + tarball.extractall(self.tmp_dir) + status = os.path.isdir(os.path.join(self.tmp_dir, "config")) and os.path.isfile(self.__dump_file_path) + _print(F"Unzip backup: {self.SUCCESS if status else self.FAILED}") + return status + except: + _print(F"Unzip backup: {self.FAILED}") + self.exceptions.update({'__untar_db': traceback.format_exc()}) + return False + + # Set secure file permissions + def __set_file_permissions(self) -> bool: + try: + os.chmod(self.tar_gz_file_path, stat.S_IREAD) + status = oct(os.stat(self.tar_gz_file_path).st_mode)[-3:] == '400' + _print(F"Set secure file permissions: {self.SUCCESS if True else self.FAILED}") + return status + except: + _print(F"Set secure file permissions: {self.FAILED}") + self.exceptions.update({'__set_file_permissions': traceback.format_exc()}) + return False + + # Create backup and return file size in MB or False if it failed + def create_backup(self): + + if self.__create_backup_dir() and self.__create_tmp_dir(): + try: + step_status = [ + self.__dump_db(), + self.__export_config(), + self.__tar_backup(), + self.__set_file_permissions(), + self.__delete_tmp_dir() + ] + for step in step_status: + if not step: + return False + return round(Path(self.tar_gz_file_path).stat().st_size / 1000000, 2) + except: + self.exceptions.update({'backup': traceback.format_exc()}) + return False + else: + _print(F"{Fore.RED}Backup aborted.{Style.RESET_ALL}") + + def restore_backup(self, backup_file_path): + + if self.__create_tmp_dir(): + try: + step_status = [ + self.__untar_backup(backup_file_path), + #self.__import_config(), + #self.__import_db(), + #self.__delete_tmp_dir() + ] + for step in step_status: + if not step: + return False + return round(Path(self.tar_gz_file_path).stat().st_size / 1000000, 2) + except: + self.exceptions.update({'restore': traceback.format_exc()}) + return False + + @staticmethod + def instantiate_containers(data: dict) -> dict: + containers = {} + for name, values in data['nextcloud_containers'].items(): + containers.update({name: Container( + name, + values['mysql_root_password'], + values['app_container'], + values['db_container'], + values['backup_dir'], + values['number_of_backups']) + }) + return containers + + +class Log: + + def __init__(self, log_dir): + self.log_dir = log_dir + self.__log_file = 'nextcloud_docker_scripts.log' + self.__log_file_path = os.path.join(self.log_dir, self.__log_file) + self.exceptions = {} + + # Create log entry + def log(self, message): + dt = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + entry = dt + " ; " + message + "\n" + if self.__check_log_dir(): + try: + with open(self.__log_file_path, "a+") as log_file: + log_file.writelines(entry) + return True + except: + _print(F"{Fore.RED}Could not write to log file: {self.__log_file}{Style.RESET_ALL}") + self.exceptions.update({'log': traceback.format_exc()}) + return False + + # Try to create log directory if it does not yet exists + def __check_log_dir(self) -> bool: + if not os.path.isdir(self.log_dir): + try: + os.makedirs(self.log_dir) + return True + except: + _print(F"{Fore.RED}Could not create log directory under: {self.log_dir}{Style.RESET_ALL}") + self.exceptions.update({'get_log_file': traceback.format_exc()}) + return False + else: + return True + + +if __name__ == '__main__': + pass diff --git a/requirements.txt b/requirements.txt index bd7afdc..bf87813 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,2 +1,3 @@ -yaml -colorama \ No newline at end of file +pyyaml +colorama +simple_term_menu \ No newline at end of file diff --git a/restore.py b/restore.py index 5f7ce86..09eb955 100644 --- a/restore.py +++ b/restore.py @@ -1 +1,78 @@ -#!/usr/bin/env python3 \ No newline at end of file +#!/usr/bin/env python3 +import datetime +import os +import sys +from pathlib import Path +import yaml +from colorama import Fore, Style +import utils +from utils import _print +from models import Container +from models import Log + +from simple_term_menu import TerminalMenu + + +# terminal_menu = TerminalMenu(["entry 1", "entry 2", "entry 3"]) +# choice_index = terminal_menu.show() +# https://stackoverflow.com/posts/61265356/revisions + +def restore(): + restore_status = True + + # Fetch arguments + utils.all_containers = "--all" in sys.argv + utils.quiet_mode = "--quiet" in sys.argv + utils.no_log = "--nolog" in sys.argv + + # Load settings + settings = Path(__file__).parent / "settings.yaml" + with open(settings) as file: + # Load settings + settings_list = yaml.full_load(file) + log = Log(settings_list['log']['log_dir']) + containers = Container.instantiate_containers(settings_list) + + # If any container names where passed as parameters, do only restore them + containers_wanted = {name: container for name, container in containers.items() if name in sys.argv} + if containers_wanted: + containers = containers_wanted + # If no container was chosen ask for it + elif not utils.all_containers: + containers_to_choose_from = [container.name for container in containers.values()] + terminal_menu = TerminalMenu(containers_to_choose_from, title="For which Nextcloud instance do you want " + "to restore a backup?") + choice_index = terminal_menu.show() + containers = {containers_to_choose_from[choice_index]: containers.get(containers_to_choose_from[choice_index])} + + container: Container + for container in containers.values(): + + # Start backup restore + _print("----------------------------------------------") + _print(F"Restore backup for {container.name}") + + backup_dir = os.scandir(container.backup_dir) + backup_files = {file.name: file for file in backup_dir if + file.is_file() and file.name.startswith(container.name) and file.name.endswith(".tar.gz")} + if len(backup_files) < 1: + _print(F"{Fore.YELLOW}No backups found for {container.name}{Style.RESET_ALL}") + break + backup_files_to_choose_from = [file.name for file in backup_files.values()] + backup_files_to_choose_from.sort(reverse=True) + _print() + terminal_menu = TerminalMenu(backup_files_to_choose_from, title="Which backup do you want to restore?") + choice_index = terminal_menu.show() + backup_file = backup_files.get(backup_files_to_choose_from[choice_index]) + print(backup_file.path) + + confirm = input(F"Are you sure that you want to restore {backup_files_to_choose_from[choice_index]}? " + F"(Type: yes)\n").lower() == "yes" + if confirm: + result = container.restore_backup(backup_file.path) + else: + break + + +if __name__ == '__main__': + restore() diff --git a/utils.py b/utils.py new file mode 100644 index 0000000..626797d --- /dev/null +++ b/utils.py @@ -0,0 +1,12 @@ +quiet_mode = False +no_log = False +no_cleanup = False +all_containers = False + + +def _print(text=None): + if not quiet_mode: + if not text is None: + print(text) + else: + print()