implement backup

This commit is contained in:
Marc Koch 2020-11-23 16:58:21 +01:00
parent 7535298a74
commit 9ad8e12522
Signed by: marc
GPG Key ID: AC2D4E00990A6767
9 changed files with 430 additions and 142 deletions

View File

View File

@ -1,91 +0,0 @@
import datetime
import os
import stat
import tarfile
from Models.NextcloudBackupException import NextcloudBackupException
class Container:
def __init__(self, name, password, backup_folder, compose_file_path) -> None:
self.__datetime = datetime.datetime.now().strftime("%Y-%m-%d_%H:%M:%S")
self.name = name
self.__password = password
self.backup_folder = backup_folder
self.compose_file_path = compose_file_path
self.__dump_file = self.name + '_' + self.__datetime + '.sql'
self.__dump_file_path = os.path.join(self.backup_folder, self.__dump_file)
self.__tar_file = self.name + '_' + self.__datetime + 'tar.gz'
self.__tar_file_path = os.path.join(self.backup_folder, self.__tar_file)
def __str__(self) -> str:
return F"name: {self.name}\npassword: {self.__password}\ncompose_file_path: {self.compose_file_path}"
# Create backup folder if it does not yet exist
def create_backup_dir(self):
if not os.path.isfile(self.backup_folder):
try:
os.makedirs(self.backup_folder)
except OSError as e:
raise OSError(e)
return True
else:
return False
# Dump database
def __dump_db(self) -> bool:
try:
os.system(F"docker exec {self.name} mysqldump --password={self.__password} --all-databases > "
F"{self.__dump_file}")
return os.path.isfile(self.__dump_file)
except Exception as e:
raise Exception(e)
# Tar config and settings folder within container and copy it into backup folder
def __export_config(self) -> bool:
try:
os.system(F"docker exec {self.name} tar -czf config_settings.tar config settings")
os.system(F"docker cp {self.name}:/var/www/html/config_settings.tar {self.__tar_file_path}")
os.system(F"docker exec {self.name} rm config_settings.tar")
return os.path.isfile(self.__tar_file_path)
except Exception as e:
raise NextcloudBackupException(e)
# Tar database with config and settings
def __tar_db(self) -> bool:
try:
with tarfile.open(self.__tar_file_path, 'w:gz') as tarball:
tarball.add(self.__dump_file_path, arcname=self.__dump_file)
return tarball.gettarinfo(self.__dump_file).isfile()
except Exception as e:
raise NextcloudBackupException(e)
# Set secure file permissions
def __set_file_permissions(self) -> bool:
try:
os.chmod(self.__tar_file_path, stat.S_IREAD)
return oct(os.stat(self.__tar_file_path).st_mode)[-3:] == 600
except Exception as e:
raise NextcloudBackupException(e)
# Create backup
def create_backup(self) -> dict:
try:
return {"database dump": self.__dump_db(),
"export config": self.__export_config(),
"include db in backup": self.__tar_db(),
"set secure backup file permissions": self.__set_file_permissions()}
except NextcloudBackupException as e:
raise NextcloudBackupException(e)
@staticmethod
def deserialize_containers(data: dict) -> list:
containers = []
for name, values in data['nextcloud_containers'].items():
containers.append(Container(
name,
values['mysql_password'],
values['backup_folder'],
values['compose_file_path'])
)
return containers

View File

@ -1,8 +0,0 @@
class NextcloudBackupException(Exception):
def __init__(self, message):
self.message = message
def __write_to_log(self):
pass

112
backup.py
View File

@ -3,59 +3,91 @@
import datetime
import os
import sys
from pathlib import Path
import yaml
from colorama import Fore, Style
import Models.Backup
from Models.Container import Container
from Models.NextcloudBackupException import NextcloudBackupException
import utils
from utils import _print
from models import Container
from models import Log
def backup(container_names=None):
global backup_successfull
with open(r"./settings.yaml") as file:
def backup():
backup_status = True
# Fetch arguments
utils.all_containers = "--all" in sys.argv
utils.quiet_mode = "--quiet" in sys.argv
utils.no_log = "--nolog" in sys.argv
utils.no_cleanup = "--nocleanup" in sys.argv
# Load settings
settings = Path(__file__).parent / "settings.yaml"
with open(settings) as file:
# Load settings
settings_list = yaml.full_load(file)
log_file = settings_list['paths']['log_file']
containers = Container.deserialize_containers(settings_list)
log = Log(settings_list['log']['log_dir'])
containers = Container.instantiate_containers(settings_list)
if type(container_names) is list:
containers_tmp = []
for container_name in container_names:
if container_name is str and container_name in containers:
containers_tmp.append(containers[container_name])
else:
print(F"{Fore.RED}Cannot find configuration for {container_name} in settings.yaml{Style.RESET_ALL}")
containers = containers_tmp
# If any container names where passed as parameters, do only backup them
containers_wanted = {name: container for name, container in containers.items() if name in sys.argv}
if containers_wanted:
containers = containers_wanted
# Loop through Nextcloud container instances
container: Container
for container in containers:
# Create backup folder if it does not yet exist
if container.create_backup_dir():
try:
print(F"{Fore.GREEN}Backup folder created under: {container.backup_folder}{Style.RESET_ALL}")
except OSError as e:
sys.exit(F"{Fore.RED}Could not create backup folder: {e.strerror}{Style.RESET_ALL}")
for container in containers.values():
print(F"Starting backup for {container.name}:")
try:
backup_successfull = True
for key, status in container.create_backup():
if status is True:
print(F"{Fore.GREEN}{key}: success{Style.RESET_ALL}")
# Start backup
_print("----------------------------------------------")
_print(F"Start backup for {container.name} at {datetime.datetime.now().strftime('%Y-%m-%d %H:%M:%S')}")
result = container.create_backup()
if result:
_print(F"{Fore.GREEN}Backup for {container.name} successfully created under "
F"{container.tar_gz_file_path} [{result} MB]{Style.RESET_ALL}")
else:
print(F"{Fore.RED}{key}: failed{Style.RESET_ALL}")
backup_successfull = False
except NextcloudBackupException as e:
print(F"{Fore.RED}Backup for {container.name} failed!/n{e.message}{Style.RESET_ALL}")
print("---------------------------------")
if backup_successfull is True:
print(F"{Fore.GREEN}Backup for {container.name} was successful{Style.RESET_ALL}")
else:
print(F"{Fore.RED}Backup for {container.name} failed{Style.RESET_ALL}")
_print(F"{Fore.RED}Backup for {container.name} failed{Style.RESET_ALL}")
for func, traceback in container.exceptions.items():
_print()
_print(F"{Fore.YELLOW}Exception occurred in method: Container.{func}(){Style.RESET_ALL}")
_print(traceback)
_print()
backup_status = False
# Log backup
if not utils.no_log:
if settings_list['log']['logging']:
if backup_status:
log.log(F"Created a backup ; {container.name} ; {container.tar_gz_file_path} ; {result} MB")
else:
log.log(F"Backup for {container.name} failed")
if len(log.exceptions) > 0:
for func, traceback in log.exceptions.items():
_print()
_print(F"{Fore.YELLOW}Exception occurred in method: Log.{func}(){Style.RESET_ALL}")
_print(traceback)
_print()
# Clean up backup folder
if not utils.no_cleanup:
deleted_files = 0
backup_dir = os.scandir(container.backup_dir)
backup_files = [file for file in backup_dir if
file.is_file() and file.name.startswith(container.name) and file.name.endswith(".tar.gz")]
while len(backup_files) > container.number_of_backups:
del_file = min(backup_files, key=os.path.getctime)
backup_files.remove(del_file)
os.remove(del_file)
deleted_files += 1
if deleted_files == 1:
_print(F"{Fore.YELLOW}Deleted 1 old backup file.{Style.RESET_ALL}")
elif deleted_files >= 1:
_print(F"{Fore.YELLOW}Deleted {deleted_files} old backup files.{Style.RESET_ALL}")
return backup_status
if __name__ == '__main__':
backup(sys.argv)
backup()

265
models.py Normal file
View File

@ -0,0 +1,265 @@
import datetime
import os
import stat
import tarfile
import traceback
import shutil
from colorama import Fore, Style
from pathlib import Path
import utils
from utils import _print
class Container:
def __init__(self, name, password, app_container, db_container, backup_dir, number_of_backups) -> None:
self.__datetime = datetime.datetime.now().strftime("%Y-%m-%d_%H%M%S")
self.name = name
self.__password = password
self.app_container = app_container
self.db_container = db_container
self.backup_dir = backup_dir
self.tmp_dir = os.path.join(backup_dir, 'tmp')
self.number_of_backups = number_of_backups
self.__dump_file = self.name + '_' + self.__datetime + '.sql'
self.__dump_file_path = os.path.join(self.tmp_dir, self.__dump_file)
self.__tar_file = self.name + '_' + self.__datetime + '.tar'
self.__tar_file_path = os.path.join(self.tmp_dir, self.__tar_file)
self.tar_gz_file = self.__tar_file + '.gz'
self.tar_gz_file_path = os.path.join(self.backup_dir, self.tar_gz_file)
self.exceptions = {}
self.SUCCESS = F"{Fore.GREEN}success{Style.RESET_ALL}"
self.FAILED = F"{Fore.RED}failed{Style.RESET_ALL}"
global quiet_mode
# Create backup dir if it does not yet exist
def __create_backup_dir(self):
if not os.path.isdir(self.backup_dir):
try:
os.makedirs(self.backup_dir)
_print(F"{Fore.GREEN}Backup folder created under: {self.backup_dir}{Style.RESET_ALL}")
return True
except:
_print(F"{Fore.RED}Could not backup tmp folder under {self.backup_dir}{Style.RESET_ALL}")
self.exceptions.update({'create_backup_dir': traceback.format_exc()})
return False
else:
return True
# Create tmp dir
def __create_tmp_dir(self) -> bool:
if not os.path.isdir(self.tmp_dir):
try:
os.mkdir(self.tmp_dir)
return os.path.isdir(self.tmp_dir)
except:
_print(F"{Fore.RED}Could not create tmp folder{Style.RESET_ALL}")
self.exceptions.update({'create_tmp_dir': traceback.format_exc()})
return False
else:
if self.__delete_tmp_dir():
return self.__create_tmp_dir()
# Delete tmp dir
def __delete_tmp_dir(self) -> bool:
try:
shutil.rmtree(self.tmp_dir)
return not os.path.isdir(self.tmp_dir)
except:
_print(F"{Fore.RED}Could not delete old tmp folder{Style.RESET_ALL}")
self.exceptions.update({'delete_tmp_dir': traceback.format_exc()})
return False
# Dump database
def __dump_db(self) -> bool:
try:
os.system(
F"docker exec {self.db_container} mysqldump --password={self.__password} --all-databases > {self.__dump_file_path}")
status = os.path.isfile(self.__dump_file_path)
_print(F"Dump Nextcloud database: {self.SUCCESS if status else self.FAILED}")
return status
except:
_print(F"Dump Nextcloud database: {self.FAILED}")
self.exceptions.update({'__dump_db': traceback.format_exc()})
return False
# Import database
def __import_db(self) -> bool:
try:
os.system(
F"docker exec {self.db_container} mysql -u root --password={self.__password} < {self.__dump_file_path}")
status = True # TODO: Implement test if database import was successful
_print(F"Import Nextcloud database: {self.SUCCESS if status else self.FAILED}")
return status
except:
_print(F"Import Nextcloud database: {self.FAILED}")
self.exceptions.update({'__import_db': traceback.format_exc()})
return False
# Tar config folder within container and copy it into backup folder
def __export_config(self) -> bool:
try:
os.system(F"docker exec {self.app_container} tar -cf config.tar config/")
os.system(F"docker cp {self.app_container}:/var/www/html/config.tar {self.__tar_file_path}")
os.system(F"docker exec {self.app_container} rm config.tar")
if os.path.isfile(self.__tar_file_path):
with tarfile.open(self.__tar_file_path, 'r') as tarball:
tarball.extractall(self.tmp_dir)
status = os.path.isdir(os.path.join(self.tmp_dir, "config"))
_print(F"Export Nextcloud configuration: {self.SUCCESS if status else self.FAILED}")
return status
except:
_print(F"Export Nextcloud configuration: {self.FAILED}")
self.exceptions.update({'__export_config': traceback.format_exc()})
return False
# Tar config folder within container and copy it into backup folder
def __import_config(self) -> bool:
try:
if os.path.isdir(os.path.join(self.tmp_dir, "config")):
with tarfile.open(self.__tar_file_path, 'w') as tarball:
tarball.add(os.path.join(self.tmp_dir, "config"), arcname="/config")
os.system(F"docker cp {self.__tar_file_path} {self.app_container}:/var/www/html/config.tar")
os.system(F"docker exec {self.app_container} rm -r config")
os.system(F"docker exec {self.app_container} tar -xf config.tar")
status = True # TODO: implement a test if export into docker container was successful
_print(F"Import Nextcloud configuration: {self.SUCCESS if status else self.FAILED}")
return status
except:
_print(F"Import Nextcloud configuration: {self.FAILED}")
self.exceptions.update({'__import_config': traceback.format_exc()})
return False
# Tar database with config and settings
def __tar_backup(self) -> bool:
try:
with tarfile.open(self.tar_gz_file_path, 'w:gz') as tarball:
tarball.add(self.__dump_file_path, arcname=self.__dump_file)
tarball.add(os.path.join(self.tmp_dir, "config"), arcname="/config")
status = True # TODO: Implement a test to confirm that files where added to tar file
_print(F"Zip backup: {self.SUCCESS if status else self.FAILED}")
return status
except:
_print(F"Zip backup: {self.FAILED}")
self.exceptions.update({'__tar_db': traceback.format_exc()})
return False
# Untar backup
def __untar_backup(self, backup_file_path) -> bool:
try:
with tarfile.open(backup_file_path, 'r:gz') as tarball:
tarball.extractall(self.tmp_dir)
status = os.path.isdir(os.path.join(self.tmp_dir, "config")) and os.path.isfile(self.__dump_file_path)
_print(F"Unzip backup: {self.SUCCESS if status else self.FAILED}")
return status
except:
_print(F"Unzip backup: {self.FAILED}")
self.exceptions.update({'__untar_db': traceback.format_exc()})
return False
# Set secure file permissions
def __set_file_permissions(self) -> bool:
try:
os.chmod(self.tar_gz_file_path, stat.S_IREAD)
status = oct(os.stat(self.tar_gz_file_path).st_mode)[-3:] == '400'
_print(F"Set secure file permissions: {self.SUCCESS if True else self.FAILED}")
return status
except:
_print(F"Set secure file permissions: {self.FAILED}")
self.exceptions.update({'__set_file_permissions': traceback.format_exc()})
return False
# Create backup and return file size in MB or False if it failed
def create_backup(self):
if self.__create_backup_dir() and self.__create_tmp_dir():
try:
step_status = [
self.__dump_db(),
self.__export_config(),
self.__tar_backup(),
self.__set_file_permissions(),
self.__delete_tmp_dir()
]
for step in step_status:
if not step:
return False
return round(Path(self.tar_gz_file_path).stat().st_size / 1000000, 2)
except:
self.exceptions.update({'backup': traceback.format_exc()})
return False
else:
_print(F"{Fore.RED}Backup aborted.{Style.RESET_ALL}")
def restore_backup(self, backup_file_path):
if self.__create_tmp_dir():
try:
step_status = [
self.__untar_backup(backup_file_path),
#self.__import_config(),
#self.__import_db(),
#self.__delete_tmp_dir()
]
for step in step_status:
if not step:
return False
return round(Path(self.tar_gz_file_path).stat().st_size / 1000000, 2)
except:
self.exceptions.update({'restore': traceback.format_exc()})
return False
@staticmethod
def instantiate_containers(data: dict) -> dict:
containers = {}
for name, values in data['nextcloud_containers'].items():
containers.update({name: Container(
name,
values['mysql_root_password'],
values['app_container'],
values['db_container'],
values['backup_dir'],
values['number_of_backups'])
})
return containers
class Log:
def __init__(self, log_dir):
self.log_dir = log_dir
self.__log_file = 'nextcloud_docker_scripts.log'
self.__log_file_path = os.path.join(self.log_dir, self.__log_file)
self.exceptions = {}
# Create log entry
def log(self, message):
dt = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S")
entry = dt + " ; " + message + "\n"
if self.__check_log_dir():
try:
with open(self.__log_file_path, "a+") as log_file:
log_file.writelines(entry)
return True
except:
_print(F"{Fore.RED}Could not write to log file: {self.__log_file}{Style.RESET_ALL}")
self.exceptions.update({'log': traceback.format_exc()})
return False
# Try to create log directory if it does not yet exists
def __check_log_dir(self) -> bool:
if not os.path.isdir(self.log_dir):
try:
os.makedirs(self.log_dir)
return True
except:
_print(F"{Fore.RED}Could not create log directory under: {self.log_dir}{Style.RESET_ALL}")
self.exceptions.update({'get_log_file': traceback.format_exc()})
return False
else:
return True
if __name__ == '__main__':
pass

View File

@ -1,2 +1,3 @@
yaml
pyyaml
colorama
simple_term_menu

View File

@ -1 +1,78 @@
#!/usr/bin/env python3
import datetime
import os
import sys
from pathlib import Path
import yaml
from colorama import Fore, Style
import utils
from utils import _print
from models import Container
from models import Log
from simple_term_menu import TerminalMenu
# terminal_menu = TerminalMenu(["entry 1", "entry 2", "entry 3"])
# choice_index = terminal_menu.show()
# https://stackoverflow.com/posts/61265356/revisions
def restore():
restore_status = True
# Fetch arguments
utils.all_containers = "--all" in sys.argv
utils.quiet_mode = "--quiet" in sys.argv
utils.no_log = "--nolog" in sys.argv
# Load settings
settings = Path(__file__).parent / "settings.yaml"
with open(settings) as file:
# Load settings
settings_list = yaml.full_load(file)
log = Log(settings_list['log']['log_dir'])
containers = Container.instantiate_containers(settings_list)
# If any container names where passed as parameters, do only restore them
containers_wanted = {name: container for name, container in containers.items() if name in sys.argv}
if containers_wanted:
containers = containers_wanted
# If no container was chosen ask for it
elif not utils.all_containers:
containers_to_choose_from = [container.name for container in containers.values()]
terminal_menu = TerminalMenu(containers_to_choose_from, title="For which Nextcloud instance do you want "
"to restore a backup?")
choice_index = terminal_menu.show()
containers = {containers_to_choose_from[choice_index]: containers.get(containers_to_choose_from[choice_index])}
container: Container
for container in containers.values():
# Start backup restore
_print("----------------------------------------------")
_print(F"Restore backup for {container.name}")
backup_dir = os.scandir(container.backup_dir)
backup_files = {file.name: file for file in backup_dir if
file.is_file() and file.name.startswith(container.name) and file.name.endswith(".tar.gz")}
if len(backup_files) < 1:
_print(F"{Fore.YELLOW}No backups found for {container.name}{Style.RESET_ALL}")
break
backup_files_to_choose_from = [file.name for file in backup_files.values()]
backup_files_to_choose_from.sort(reverse=True)
_print()
terminal_menu = TerminalMenu(backup_files_to_choose_from, title="Which backup do you want to restore?")
choice_index = terminal_menu.show()
backup_file = backup_files.get(backup_files_to_choose_from[choice_index])
print(backup_file.path)
confirm = input(F"Are you sure that you want to restore {backup_files_to_choose_from[choice_index]}? "
F"(Type: yes)\n").lower() == "yes"
if confirm:
result = container.restore_backup(backup_file.path)
else:
break
if __name__ == '__main__':
restore()

12
utils.py Normal file
View File

@ -0,0 +1,12 @@
quiet_mode = False
no_log = False
no_cleanup = False
all_containers = False
def _print(text=None):
if not quiet_mode:
if not text is None:
print(text)
else:
print()