implement backup

Models/Container.py

@@ -1,91 +0,0 @@
-import datetime
-import os
-import stat
-import tarfile
-from Models.NextcloudBackupException import NextcloudBackupException
-
-
-class Container:
-
-    def __init__(self, name, password, backup_folder, compose_file_path) -> None:
-        self.__datetime = datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S")
-        self.name = name
-        self.__password = password
-        self.backup_folder = backup_folder
-        self.compose_file_path = compose_file_path
-        self.__dump_file = self.name + '_' + self.__datetime + '.sql'
-        self.__dump_file_path = os.path.join(self.backup_folder, self.__dump_file)
-        self.__tar_file = self.name + '_' + self.__datetime + 'tar.gz'
-        self.__tar_file_path = os.path.join(self.backup_folder, self.__tar_file)
-
-    def __str__(self) -> str:
-        return F"name: {self.name}\npassword: {self.__password}\ncompose_file_path: {self.compose_file_path}"
-
-    # Create backup folder if it does not yet exist
-    def create_backup_dir(self):
-        if not os.path.isfile(self.backup_folder):
-            try:
-                os.makedirs(self.backup_folder)
-            except OSError as e:
-                raise OSError(e)
-            return True
-        else:
-            return False
-
-    # Dump database
-    def __dump_db(self) -> bool:
-        try:
-            os.system(F"docker exec {self.name} mysqldump --password={self.__password} --all-databases > "
-                      F"{self.__dump_file}")
-            return os.path.isfile(self.__dump_file)
-        except Exception as e:
-            raise Exception(e)
-
-    # Tar config and settings folder within container and copy it into backup folder
-    def __export_config(self) -> bool:
-        try:
-            os.system(F"docker exec {self.name} tar -czf config_settings.tar config settings")
-            os.system(F"docker cp {self.name}:/var/www/html/config_settings.tar {self.__tar_file_path}")
-            os.system(F"docker exec {self.name} rm config_settings.tar")
-            return os.path.isfile(self.__tar_file_path)
-        except Exception as e:
-            raise NextcloudBackupException(e)
-
-    # Tar database with config and settings
-    def __tar_db(self) -> bool:
-        try:
-            with tarfile.open(self.__tar_file_path, 'w:gz') as tarball:
-                tarball.add(self.__dump_file_path, arcname=self.__dump_file)
-            return tarball.gettarinfo(self.__dump_file).isfile()
-        except Exception as e:
-            raise NextcloudBackupException(e)
-
-    # Set secure file permissions
-    def __set_file_permissions(self) -> bool:
-        try:
-            os.chmod(self.__tar_file_path, stat.S_IREAD)
-            return oct(os.stat(self.__tar_file_path).st_mode)[-3:] == 600
-        except Exception as e:
-            raise NextcloudBackupException(e)
-
-    # Create backup
-    def create_backup(self) -> dict:
-        try:
-            return {"database dump": self.__dump_db(),
-                    "export config": self.__export_config(),
-                    "include db in backup": self.__tar_db(),
-                    "set secure backup file permissions": self.__set_file_permissions()}
-        except NextcloudBackupException as e:
-            raise NextcloudBackupException(e)
-
-    @staticmethod
-    def deserialize_containers(data: dict) -> list:
-        containers = []
-        for name, values in data['nextcloud_containers'].items():
-            containers.append(Container(
-                name,
-                values['mysql_password'],
-                values['backup_folder'],
-                values['compose_file_path'])
-            )
-        return containers

Models/NextcloudBackupException.py

@@ -1,8 +0,0 @@
-
-class NextcloudBackupException(Exception):
-    def __init__(self, message):
-        self.message = message
-
-
-    def __write_to_log(self):
-        pass

backup.py

@@ -3,59 +3,91 @@
 import datetime
 import os
 import sys
+from pathlib import Path
 import yaml
 from colorama import Fore, Style
-import Models.Backup
+import utils
-from Models.Container import Container
+from utils import _print
-from Models.NextcloudBackupException import NextcloudBackupException
+from models import Container
+from models import Log
 
 
-def backup(container_names=None):
+def backup():
-    global backup_successfull
+    backup_status = True
-    with open(r"./settings.yaml") as file:
+
+    # Fetch arguments
+    utils.all_containers = "--all" in sys.argv
+    utils.quiet_mode = "--quiet" in sys.argv
+    utils.no_log = "--nolog" in sys.argv
+    utils.no_cleanup = "--nocleanup" in sys.argv
+
+    # Load settings
+    settings = Path(__file__).parent / "settings.yaml"
+    with open(settings) as file:
         # Load settings
         settings_list = yaml.full_load(file)
-        log_file = settings_list['paths']['log_file']
+        log = Log(settings_list['log']['log_dir'])
-        containers = Container.deserialize_containers(settings_list)
+        containers = Container.instantiate_containers(settings_list)
 
-    if type(container_names) is list:
+    # If any container names where passed as parameters, do only backup them
-        containers_tmp = []
+    containers_wanted = {name: container for name, container in containers.items() if name in sys.argv}
-        for container_name in container_names:
+    if containers_wanted:
-            if container_name is str and container_name in containers:
+        containers = containers_wanted
-                containers_tmp.append(containers[container_name])
-            else:
-                print(F"{Fore.RED}Cannot find configuration for {container_name} in settings.yaml{Style.RESET_ALL}")
-        containers = containers_tmp
 
     # Loop through Nextcloud container instances
     container: Container
-    for container in containers:
+    for container in containers.values():
-        # Create backup folder if it does not yet exist
-        if container.create_backup_dir():
-            try:
-                print(F"{Fore.GREEN}Backup folder created under: {container.backup_folder}{Style.RESET_ALL}")
-            except OSError as e:
-                sys.exit(F"{Fore.RED}Could not create backup folder: {e.strerror}{Style.RESET_ALL}")
 
-        print(F"Starting backup for {container.name}:")
+        # Start backup
-        try:
+        _print("----------------------------------------------")
-            backup_successfull = True
+        _print(F"Start backup for {container.name} at {datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
-            for key, status in container.create_backup():
+        result = container.create_backup()
-                if status is True:
+        if result:
-                    print(F"{Fore.GREEN}{key}: success{Style.RESET_ALL}")
+            _print(F"{Fore.GREEN}Backup for {container.name} successfully created under "
-                else:
+                   F"{container.tar_gz_file_path} [{result} MB]{Style.RESET_ALL}")
-                    print(F"{Fore.RED}{key}: failed{Style.RESET_ALL}")
-                    backup_successfull = False
-        except NextcloudBackupException as e:
-            print(F"{Fore.RED}Backup for {container.name} failed!/n{e.message}{Style.RESET_ALL}")
-        print("---------------------------------")
-        if backup_successfull is True:
-            print(F"{Fore.GREEN}Backup for {container.name} was successful{Style.RESET_ALL}")
         else:
-            print(F"{Fore.RED}Backup for {container.name} failed{Style.RESET_ALL}")
+            _print(F"{Fore.RED}Backup for {container.name} failed{Style.RESET_ALL}")
+            for func, traceback in container.exceptions.items():
+                _print()
+                _print(F"{Fore.YELLOW}Exception occurred in method: Container.{func}(){Style.RESET_ALL}")
+                _print(traceback)
+                _print()
+                backup_status = False
 
+        # Log backup
+        if not utils.no_log:
+            if settings_list['log']['logging']:
+                if backup_status:
+                    log.log(F"Created a backup ; {container.name} ; {container.tar_gz_file_path} ; {result} MB")
+                else:
+                    log.log(F"Backup for {container.name} failed")
+                    if len(log.exceptions) > 0:
+                        for func, traceback in log.exceptions.items():
+                            _print()
+                            _print(F"{Fore.YELLOW}Exception occurred in method: Log.{func}(){Style.RESET_ALL}")
+                            _print(traceback)
+                            _print()
+
+        # Clean up backup folder
+        if not utils.no_cleanup:
+            deleted_files = 0
+            backup_dir = os.scandir(container.backup_dir)
+            backup_files = [file for file in backup_dir if
+                            file.is_file() and file.name.startswith(container.name) and file.name.endswith(".tar.gz")]
+
+            while len(backup_files) > container.number_of_backups:
+                del_file = min(backup_files, key=os.path.getctime)
+                backup_files.remove(del_file)
+                os.remove(del_file)
+                deleted_files += 1
+
+            if deleted_files == 1:
+                _print(F"{Fore.YELLOW}Deleted 1 old backup file.{Style.RESET_ALL}")
+            elif deleted_files >= 1:
+                _print(F"{Fore.YELLOW}Deleted {deleted_files} old backup files.{Style.RESET_ALL}")
+
+    return backup_status
 
 
 if __name__ == '__main__':
-    backup(sys.argv)
+    backup()
-

models.py

@@ -0,0 +1,265 @@
+import datetime
+import os
+import stat
+import tarfile
+import traceback
+import shutil
+from colorama import Fore, Style
+from pathlib import Path
+import utils
+from utils import _print
+
+
+class Container:
+
+    def __init__(self, name, password, app_container, db_container, backup_dir, number_of_backups) -> None:
+        self.__datetime = datetime.datetime.now().strftime("%Y-%m-%d_%H%M%S")
+        self.name = name
+        self.__password = password
+        self.app_container = app_container
+        self.db_container = db_container
+        self.backup_dir = backup_dir
+        self.tmp_dir = os.path.join(backup_dir, 'tmp')
+        self.number_of_backups = number_of_backups
+        self.__dump_file = self.name + '_' + self.__datetime + '.sql'
+        self.__dump_file_path = os.path.join(self.tmp_dir, self.__dump_file)
+        self.__tar_file = self.name + '_' + self.__datetime + '.tar'
+        self.__tar_file_path = os.path.join(self.tmp_dir, self.__tar_file)
+        self.tar_gz_file = self.__tar_file + '.gz'
+        self.tar_gz_file_path = os.path.join(self.backup_dir, self.tar_gz_file)
+        self.exceptions = {}
+        self.SUCCESS = F"{Fore.GREEN}success{Style.RESET_ALL}"
+        self.FAILED = F"{Fore.RED}failed{Style.RESET_ALL}"
+
+        global quiet_mode
+
+    # Create backup dir if it does not yet exist
+    def __create_backup_dir(self):
+        if not os.path.isdir(self.backup_dir):
+            try:
+                os.makedirs(self.backup_dir)
+                _print(F"{Fore.GREEN}Backup folder created under: {self.backup_dir}{Style.RESET_ALL}")
+                return True
+            except:
+                _print(F"{Fore.RED}Could not backup tmp folder under {self.backup_dir}{Style.RESET_ALL}")
+                self.exceptions.update({'create_backup_dir': traceback.format_exc()})
+                return False
+        else:
+            return True
+
+    # Create tmp dir
+    def __create_tmp_dir(self) -> bool:
+        if not os.path.isdir(self.tmp_dir):
+            try:
+                os.mkdir(self.tmp_dir)
+                return os.path.isdir(self.tmp_dir)
+            except:
+                _print(F"{Fore.RED}Could not create tmp folder{Style.RESET_ALL}")
+                self.exceptions.update({'create_tmp_dir': traceback.format_exc()})
+                return False
+        else:
+            if self.__delete_tmp_dir():
+                return self.__create_tmp_dir()
+
+    # Delete tmp dir
+    def __delete_tmp_dir(self) -> bool:
+        try:
+            shutil.rmtree(self.tmp_dir)
+            return not os.path.isdir(self.tmp_dir)
+        except:
+            _print(F"{Fore.RED}Could not delete old tmp folder{Style.RESET_ALL}")
+            self.exceptions.update({'delete_tmp_dir': traceback.format_exc()})
+            return False
+
+    # Dump database
+    def __dump_db(self) -> bool:
+        try:
+            os.system(
+                F"docker exec {self.db_container} mysqldump --password={self.__password} --all-databases > {self.__dump_file_path}")
+            status = os.path.isfile(self.__dump_file_path)
+            _print(F"Dump Nextcloud database: {self.SUCCESS if status else self.FAILED}")
+            return status
+        except:
+            _print(F"Dump Nextcloud database: {self.FAILED}")
+            self.exceptions.update({'__dump_db': traceback.format_exc()})
+            return False
+
+    # Import database
+    def __import_db(self) -> bool:
+        try:
+            os.system(
+                F"docker exec {self.db_container} mysql -u root --password={self.__password} < {self.__dump_file_path}")
+            status = True  # TODO: Implement test if database import was successful
+            _print(F"Import Nextcloud database: {self.SUCCESS if status else self.FAILED}")
+            return status
+        except:
+            _print(F"Import Nextcloud database: {self.FAILED}")
+            self.exceptions.update({'__import_db': traceback.format_exc()})
+            return False
+
+    # Tar config folder within container and copy it into backup folder
+    def __export_config(self) -> bool:
+        try:
+            os.system(F"docker exec {self.app_container} tar -cf config.tar config/")
+            os.system(F"docker cp {self.app_container}:/var/www/html/config.tar {self.__tar_file_path}")
+            os.system(F"docker exec {self.app_container} rm config.tar")
+            if os.path.isfile(self.__tar_file_path):
+                with tarfile.open(self.__tar_file_path, 'r') as tarball:
+                    tarball.extractall(self.tmp_dir)
+                status = os.path.isdir(os.path.join(self.tmp_dir, "config"))
+                _print(F"Export Nextcloud configuration: {self.SUCCESS if status else self.FAILED}")
+                return status
+        except:
+            _print(F"Export Nextcloud configuration: {self.FAILED}")
+            self.exceptions.update({'__export_config': traceback.format_exc()})
+            return False
+
+    # Tar config folder within container and copy it into backup folder
+    def __import_config(self) -> bool:
+        try:
+            if os.path.isdir(os.path.join(self.tmp_dir, "config")):
+                with tarfile.open(self.__tar_file_path, 'w') as tarball:
+                    tarball.add(os.path.join(self.tmp_dir, "config"), arcname="/config")
+            os.system(F"docker cp {self.__tar_file_path} {self.app_container}:/var/www/html/config.tar")
+            os.system(F"docker exec {self.app_container} rm -r config")
+            os.system(F"docker exec {self.app_container} tar -xf config.tar")
+            status = True  # TODO: implement a test if export into docker container was successful
+            _print(F"Import Nextcloud configuration: {self.SUCCESS if status else self.FAILED}")
+            return status
+        except:
+            _print(F"Import Nextcloud configuration: {self.FAILED}")
+            self.exceptions.update({'__import_config': traceback.format_exc()})
+            return False
+
+    # Tar database with config and settings
+    def __tar_backup(self) -> bool:
+        try:
+            with tarfile.open(self.tar_gz_file_path, 'w:gz') as tarball:
+                tarball.add(self.__dump_file_path, arcname=self.__dump_file)
+                tarball.add(os.path.join(self.tmp_dir, "config"), arcname="/config")
+            status = True  # TODO: Implement a test to confirm that files where added to tar file
+            _print(F"Zip backup: {self.SUCCESS if status else self.FAILED}")
+            return status
+        except:
+            _print(F"Zip backup: {self.FAILED}")
+            self.exceptions.update({'__tar_db': traceback.format_exc()})
+            return False
+
+    # Untar backup
+    def __untar_backup(self, backup_file_path) -> bool:
+        try:
+            with tarfile.open(backup_file_path, 'r:gz') as tarball:
+                tarball.extractall(self.tmp_dir)
+            status = os.path.isdir(os.path.join(self.tmp_dir, "config")) and os.path.isfile(self.__dump_file_path)
+            _print(F"Unzip backup: {self.SUCCESS if status else self.FAILED}")
+            return status
+        except:
+            _print(F"Unzip backup: {self.FAILED}")
+            self.exceptions.update({'__untar_db': traceback.format_exc()})
+            return False
+
+    # Set secure file permissions
+    def __set_file_permissions(self) -> bool:
+        try:
+            os.chmod(self.tar_gz_file_path, stat.S_IREAD)
+            status = oct(os.stat(self.tar_gz_file_path).st_mode)[-3:] == '400'
+            _print(F"Set secure file permissions: {self.SUCCESS if True else self.FAILED}")
+            return status
+        except:
+            _print(F"Set secure file permissions: {self.FAILED}")
+            self.exceptions.update({'__set_file_permissions': traceback.format_exc()})
+            return False
+
+    # Create backup and return file size in MB or False if it failed
+    def create_backup(self):
+
+        if self.__create_backup_dir() and self.__create_tmp_dir():
+            try:
+                step_status = [
+                    self.__dump_db(),
+                    self.__export_config(),
+                    self.__tar_backup(),
+                    self.__set_file_permissions(),
+                    self.__delete_tmp_dir()
+                ]
+                for step in step_status:
+                    if not step:
+                        return False
+                return round(Path(self.tar_gz_file_path).stat().st_size / 1000000, 2)
+            except:
+                self.exceptions.update({'backup': traceback.format_exc()})
+                return False
+        else:
+            _print(F"{Fore.RED}Backup aborted.{Style.RESET_ALL}")
+
+    def restore_backup(self, backup_file_path):
+
+        if self.__create_tmp_dir():
+            try:
+                step_status = [
+                    self.__untar_backup(backup_file_path),
+                    #self.__import_config(),
+                    #self.__import_db(),
+                    #self.__delete_tmp_dir()
+                ]
+                for step in step_status:
+                    if not step:
+                        return False
+                return round(Path(self.tar_gz_file_path).stat().st_size / 1000000, 2)
+            except:
+                self.exceptions.update({'restore': traceback.format_exc()})
+                return False
+
+    @staticmethod
+    def instantiate_containers(data: dict) -> dict:
+        containers = {}
+        for name, values in data['nextcloud_containers'].items():
+            containers.update({name: Container(
+                name,
+                values['mysql_root_password'],
+                values['app_container'],
+                values['db_container'],
+                values['backup_dir'],
+                values['number_of_backups'])
+            })
+        return containers
+
+
+class Log:
+
+    def __init__(self, log_dir):
+        self.log_dir = log_dir
+        self.__log_file = 'nextcloud_docker_scripts.log'
+        self.__log_file_path = os.path.join(self.log_dir, self.__log_file)
+        self.exceptions = {}
+
+    # Create log entry
+    def log(self, message):
+        dt = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
+        entry = dt + "  ;  " + message + "\n"
+        if self.__check_log_dir():
+            try:
+                with open(self.__log_file_path, "a+") as log_file:
+                    log_file.writelines(entry)
+                return True
+            except:
+                _print(F"{Fore.RED}Could not write to log file: {self.__log_file}{Style.RESET_ALL}")
+                self.exceptions.update({'log': traceback.format_exc()})
+                return False
+
+    # Try to create log directory if it does not yet exists
+    def __check_log_dir(self) -> bool:
+        if not os.path.isdir(self.log_dir):
+            try:
+                os.makedirs(self.log_dir)
+                return True
+            except:
+                _print(F"{Fore.RED}Could not create log directory under: {self.log_dir}{Style.RESET_ALL}")
+                self.exceptions.update({'get_log_file': traceback.format_exc()})
+                return False
+        else:
+            return True
+
+
+if __name__ == '__main__':
+    pass

requirements.txt

@@ -1,2 +1,3 @@
-yaml
+pyyaml
 colorama
+simple_term_menu

restore.py

@@ -1 +1,78 @@
 #!/usr/bin/env python3
+import datetime
+import os
+import sys
+from pathlib import Path
+import yaml
+from colorama import Fore, Style
+import utils
+from utils import _print
+from models import Container
+from models import Log
+
+from simple_term_menu import TerminalMenu
+
+
+# terminal_menu = TerminalMenu(["entry 1", "entry 2", "entry 3"])
+# choice_index = terminal_menu.show()
+# https://stackoverflow.com/posts/61265356/revisions
+
+def restore():
+    restore_status = True
+
+    # Fetch arguments
+    utils.all_containers = "--all" in sys.argv
+    utils.quiet_mode = "--quiet" in sys.argv
+    utils.no_log = "--nolog" in sys.argv
+
+    # Load settings
+    settings = Path(__file__).parent / "settings.yaml"
+    with open(settings) as file:
+        # Load settings
+        settings_list = yaml.full_load(file)
+        log = Log(settings_list['log']['log_dir'])
+        containers = Container.instantiate_containers(settings_list)
+
+    # If any container names where passed as parameters, do only restore them
+    containers_wanted = {name: container for name, container in containers.items() if name in sys.argv}
+    if containers_wanted:
+        containers = containers_wanted
+    # If no container was chosen ask for it
+    elif not utils.all_containers:
+        containers_to_choose_from = [container.name for container in containers.values()]
+        terminal_menu = TerminalMenu(containers_to_choose_from, title="For which Nextcloud instance do you want "
+                                                                      "to restore a backup?")
+        choice_index = terminal_menu.show()
+        containers = {containers_to_choose_from[choice_index]: containers.get(containers_to_choose_from[choice_index])}
+
+    container: Container
+    for container in containers.values():
+
+        # Start backup restore
+        _print("----------------------------------------------")
+        _print(F"Restore backup for {container.name}")
+
+        backup_dir = os.scandir(container.backup_dir)
+        backup_files = {file.name: file for file in backup_dir if
+                        file.is_file() and file.name.startswith(container.name) and file.name.endswith(".tar.gz")}
+        if len(backup_files) < 1:
+            _print(F"{Fore.YELLOW}No backups found for {container.name}{Style.RESET_ALL}")
+            break
+        backup_files_to_choose_from = [file.name for file in backup_files.values()]
+        backup_files_to_choose_from.sort(reverse=True)
+        _print()
+        terminal_menu = TerminalMenu(backup_files_to_choose_from, title="Which backup do you want to restore?")
+        choice_index = terminal_menu.show()
+        backup_file = backup_files.get(backup_files_to_choose_from[choice_index])
+        print(backup_file.path)
+
+        confirm = input(F"Are you sure that you want to restore {backup_files_to_choose_from[choice_index]}? "
+                        F"(Type: yes)\n").lower() == "yes"
+        if confirm:
+            result = container.restore_backup(backup_file.path)
+        else:
+            break
+
+
+if __name__ == '__main__':
+    restore()

utils.py

@@ -0,0 +1,12 @@
+quiet_mode = False
+no_log = False
+no_cleanup = False
+all_containers = False
+
+
+def _print(text=None):
+    if not quiet_mode:
+        if not text is None:
+            print(text)
+        else:
+            print()