✨ use os.path.join() instead of string concatenation to join paths
This change should make the program compatible with other operating systems.
This commit is contained in:
parent
dfa721e7c0
commit
70e5d574c5
20
main.py
20
main.py
|
|
@ -9,14 +9,16 @@ from models import TazDownloader, TazConfiguration
|
||||||
from exceptions import TazConfigurationError, TazDownloadError, TazDownloadFormatException
|
from exceptions import TazConfigurationError, TazDownloadError, TazDownloadFormatException
|
||||||
|
|
||||||
# Get directory
|
# Get directory
|
||||||
dir_path = os.path.dirname(os.path.realpath(__file__)) + '/'
|
dir_path = os.path.dirname(os.path.realpath(__file__))
|
||||||
|
|
||||||
|
|
||||||
def main(config: dict):
|
def main(config: dict):
|
||||||
|
|
||||||
# Get german date for tomorrow
|
# Get german date for tomorrow
|
||||||
tomorrow = (datetime.now(pytz.timezone('Europe/Berlin')) + timedelta(1)).strftime('%Y_%m_%d')
|
tomorrow = (datetime.now(pytz.timezone('Europe/Berlin')) + timedelta(1)).strftime('%Y_%m_%d')
|
||||||
|
|
||||||
|
# Define tmp/ folder
|
||||||
|
tmp_folder = os.path.join(dir_path, 'tmp')
|
||||||
|
|
||||||
# Set log level
|
# Set log level
|
||||||
try:
|
try:
|
||||||
logging.getLogger().setLevel(config['log_level'].upper())
|
logging.getLogger().setLevel(config['log_level'].upper())
|
||||||
|
|
@ -25,7 +27,7 @@ def main(config: dict):
|
||||||
|
|
||||||
# Read download history from csv file
|
# Read download history from csv file
|
||||||
try:
|
try:
|
||||||
df = pd.read_csv(dir_path + 'download_history.csv', header=0)
|
df = pd.read_csv(os.path.join(dir_path, 'download_history.csv'), header=0)
|
||||||
except FileNotFoundError:
|
except FileNotFoundError:
|
||||||
# In case, there isn't yet a csv file, create data frame with headers
|
# In case, there isn't yet a csv file, create data frame with headers
|
||||||
df = pd.DataFrame(
|
df = pd.DataFrame(
|
||||||
|
|
@ -69,7 +71,7 @@ def main(config: dict):
|
||||||
newspaper_downloaded = []
|
newspaper_downloaded = []
|
||||||
for n in newspaper_to_download:
|
for n in newspaper_to_download:
|
||||||
try:
|
try:
|
||||||
if taz_dl.download_newspaper(n):
|
if taz_dl.download_newspaper(n, tmp_folder):
|
||||||
newspaper_downloaded.append(n)
|
newspaper_downloaded.append(n)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logging.error(f"Could not download {n}\n{e}", exc_info=True)
|
logging.error(f"Could not download {n}\n{e}", exc_info=True)
|
||||||
|
|
@ -85,17 +87,19 @@ def main(config: dict):
|
||||||
)
|
)
|
||||||
df = df.append(df_tmp, ignore_index=True)
|
df = df.append(df_tmp, ignore_index=True)
|
||||||
df.sort_values(by='file', ascending=False, inplace=True)
|
df.sort_values(by='file', ascending=False, inplace=True)
|
||||||
df.to_csv(dir_path + 'download_history.csv', index=False)
|
df.to_csv(os.path.join(dir_path, 'download_history.csv'), index=False)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logging.error(f"Could not update download_history.csv\n{e}", exc_info=True)
|
logging.error(f"Could not update download_history.csv\n{e}", exc_info=True)
|
||||||
|
|
||||||
# Move downloaded file to download folder
|
# Move downloaded file to download folder
|
||||||
if os.path.isdir(config['download_folder']):
|
if os.path.isdir(config['download_folder']):
|
||||||
download_folder = \
|
download_folder = \
|
||||||
config['download_folder'] if config['download_folder'].endswith('/') else config['download_folder'] + "/"
|
config['download_folder'] \
|
||||||
|
if config['download_folder'].endswith(os.path.sep) \
|
||||||
|
else config['download_folder'] + os.path.sep
|
||||||
for n in newspaper_downloaded:
|
for n in newspaper_downloaded:
|
||||||
try:
|
try:
|
||||||
shutil.move(dir_path + 'tmp/' + n, download_folder)
|
shutil.move(os.path.join(tmp_folder, n), download_folder)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
logging.error(f"Could not move file to download folder \"{download_folder}\"\n{e}", exc_info=True)
|
logging.error(f"Could not move file to download folder \"{download_folder}\"\n{e}", exc_info=True)
|
||||||
|
|
||||||
|
|
@ -104,7 +108,7 @@ if __name__ == '__main__':
|
||||||
|
|
||||||
# Set up logging
|
# Set up logging
|
||||||
logging.basicConfig(
|
logging.basicConfig(
|
||||||
filename=dir_path + 'tazPlease.log',
|
filename=os.path.join(dir_path, 'tazPlease.log'),
|
||||||
level=logging.ERROR,
|
level=logging.ERROR,
|
||||||
format='%(asctime)s - %(message)s'
|
format='%(asctime)s - %(message)s'
|
||||||
)
|
)
|
||||||
|
|
|
||||||
16
models.py
16
models.py
|
|
@ -7,7 +7,7 @@ from bs4 import BeautifulSoup
|
||||||
from envyaml import EnvYAML
|
from envyaml import EnvYAML
|
||||||
import argparse
|
import argparse
|
||||||
|
|
||||||
dir_path = os.path.dirname(os.path.realpath(__file__)) + '/'
|
dir_path = os.path.dirname(os.path.realpath(__file__))
|
||||||
|
|
||||||
|
|
||||||
class TazConfiguration:
|
class TazConfiguration:
|
||||||
|
|
@ -43,7 +43,7 @@ class TazConfiguration:
|
||||||
def _load_config(self):
|
def _load_config(self):
|
||||||
# Try to load config.yaml
|
# Try to load config.yaml
|
||||||
try:
|
try:
|
||||||
conf_yaml = EnvYAML(dir_path + 'config.yaml', dir_path + '.env')
|
conf_yaml = EnvYAML(os.path.join(dir_path, 'config.yaml'), os.path.join(dir_path, '.env'))
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
raise Exception(f"Something went wrong when reading config.yaml.\n{e}")
|
raise Exception(f"Something went wrong when reading config.yaml.\n{e}")
|
||||||
|
|
||||||
|
|
@ -141,17 +141,17 @@ class TazDownloader:
|
||||||
except HTTPError as http_e:
|
except HTTPError as http_e:
|
||||||
raise TazDownloadError(f"Could not scrape available newspaper editions:\n{http_e}")
|
raise TazDownloadError(f"Could not scrape available newspaper editions:\n{http_e}")
|
||||||
|
|
||||||
def download_newspaper(self, taz: str, download_folder: str = dir_path + 'tmp/'):
|
def download_newspaper(self, taz: str, download_folder: str = os.path.join(dir_path, 'tmp')):
|
||||||
"""
|
"""
|
||||||
Downloads a newspaper from dl.taz.de and stores it in tmp/
|
Downloads a newspaper from dl.taz.de and stores it in tmp folder
|
||||||
"""
|
"""
|
||||||
|
|
||||||
# Check if folder exists
|
# Check if folder exists
|
||||||
try:
|
try:
|
||||||
if not os.path.isdir(dir_path):
|
if not os.path.isdir(download_folder):
|
||||||
os.mkdirs(dir_path)
|
os.makedirs(download_folder)
|
||||||
except Exception as e:
|
except Exception as e:
|
||||||
raise TazDownloadError(f"Could find or create \"{dir_path}\":\n{e}")
|
raise TazDownloadError(f"Could find or create \"{download_folder}\":\n{e}")
|
||||||
|
|
||||||
# download taz
|
# download taz
|
||||||
try:
|
try:
|
||||||
|
|
@ -167,7 +167,7 @@ class TazDownloader:
|
||||||
}
|
}
|
||||||
) as r:
|
) as r:
|
||||||
# write response to file
|
# write response to file
|
||||||
with open(download_folder + taz, "wb") as f:
|
with open(os.path.join(download_folder, taz), "wb") as f:
|
||||||
for chunk in r.iter_content(chunk_size=8192):
|
for chunk in r.iter_content(chunk_size=8192):
|
||||||
f.write(chunk)
|
f.write(chunk)
|
||||||
return True
|
return True
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue