diff options
Diffstat (limited to 'raphodo/backupfile.py')
| -rwxr-xr-x | raphodo/backupfile.py | 287 |
1 files changed, 287 insertions, 0 deletions
diff --git a/raphodo/backupfile.py b/raphodo/backupfile.py new file mode 100755 index 0000000..34d269f --- /dev/null +++ b/raphodo/backupfile.py @@ -0,0 +1,287 @@ +#!/usr/bin/env python3 + +# Copyright (C) 2011-2017 Damon Lynch <damonlynch@gmail.com> + +# This file is part of Rapid Photo Downloader. +# +# Rapid Photo Downloader is free software: you can redistribute it and/or +# modify it under the terms of the GNU General Public License as published by +# the Free Software Foundation, either version 3 of the License, or +# (at your option) any later version. +# +# Rapid Photo Downloader is distributed in the hope that it will be useful, +# but WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +# GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License +# along with Rapid Photo Downloader. If not, +# see <http://www.gnu.org/licenses/>. + +__author__ = 'Damon Lynch' +__copyright__ = "Copyright 2011-2017, Damon Lynch" + +import pickle +import os +import errno +import hashlib +from datetime import datetime +import shutil +import logging +import locale +# Use the default locale as defined by the LANG variable +locale.setlocale(locale.LC_ALL, '') +from typing import Optional, Tuple +from gettext import gettext as _ + +from raphodo.interprocess import (BackupFileData, BackupResults, BackupArguments, + WorkerInPublishPullPipeline) +from raphodo.copyfiles import FileCopy +from raphodo.constants import (FileType, DownloadStatus, BackupStatus) +from raphodo.rpdfile import RPDFile +from raphodo.cache import FdoCacheNormal, FdoCacheLarge + +from raphodo.copyfiles import copy_file_metadata +from raphodo.problemnotification import ( + BackingUpProblems, BackupSubfolderCreationProblem, make_href, BackupOverwrittenProblem, + BackupAlreadyExistsProblem, FileWriteProblem +) +from raphodo.storage import get_uri + + +class BackupFilesWorker(WorkerInPublishPullPipeline, FileCopy): + def __init__(self): + self.problems = BackingUpProblems() + super().__init__('BackupFiles') + + def update_progress(self, amount_downloaded, total): + self.amount_downloaded = amount_downloaded + chunk_downloaded = amount_downloaded - self.bytes_downloaded + if (chunk_downloaded > self.batch_size_bytes) or (amount_downloaded == total): + self.bytes_downloaded = amount_downloaded + self.content= pickle.dumps(BackupResults( + scan_id=self.scan_id, + device_id=self.device_id, + total_downloaded=self.total_downloaded + amount_downloaded, + chunk_downloaded=chunk_downloaded), + pickle.HIGHEST_PROTOCOL) + self.send_message_to_sink() + + # if amount_downloaded == total: + # self.bytes_downloaded = 0 + + def backup_associate_file(self, dest_dir: str, full_file_name: str) -> None: + """ + Backs up small files like XMP or THM files + """ + + base_name = os.path.basename(full_file_name) + full_dest_name = os.path.join(dest_dir, base_name) + + try: + logging.debug("Backing up additional file %s...", full_dest_name) + shutil.copyfile(full_file_name, full_dest_name) + logging.debug("...backing up additional file %s succeeded", full_dest_name) + except Exception as e: + logging.error("Backup of %s failed", full_file_name) + logging.error(str(e)) + uri = get_uri(full_file_name=full_dest_name) + self.problems.append(FileWriteProblem(name=base_name, uri=uri, exception=e)) + else: + # ignore any metadata copying errors + copy_file_metadata(full_file_name, full_dest_name) + + def do_backup(self, data: BackupFileData) -> None: + rpd_file = data.rpd_file + backup_succeeded = False + self.scan_id = rpd_file.scan_id + self.verify_file = data.verify_file + + mdata_exceptions = None + + if not (data.move_succeeded and data.do_backup): + backup_full_file_name = '' + else: + self.total_reached = False + + if data.path_suffix is None: + dest_base_dir = self.path + else: + dest_base_dir = os.path.join(self.path, data.path_suffix) + + dest_dir = os.path.join(dest_base_dir, rpd_file.download_subfolder) + backup_full_file_name = os.path.join(dest_dir, rpd_file.download_name) + + if not os.path.isdir(dest_dir): + # create the subfolders on the backup path + try: + logging.debug("Creating subfolder %s on backup device %s...", + dest_dir, self.device_name) + os.makedirs(dest_dir) + logging.debug("...backup subfolder created") + except (OSError, PermissionError, FileNotFoundError) as inst: + # There is a minuscule chance directory may have been + # created by another process between the time it + # takes to query and the time it takes to create a + # new directory. Ignore that error. + if inst.errno != errno.EEXIST: + logging.error("Failed to create backup subfolder: %s", + rpd_file.download_path) + logging.error(inst) + + self.problems.append( + BackupSubfolderCreationProblem( + folder=make_href( + name=rpd_file.download_subfolder, uri=get_uri(path=dest_dir) + ), + exception=inst + ) + ) + + backup_already_exists = os.path.exists(backup_full_file_name) + + if backup_already_exists: + try: + modification_time = os.path.getmtime(backup_full_file_name) + dt = datetime.fromtimestamp(modification_time) + date = dt.strftime("%x") + time = dt.strftime("%X") + except Exception: + logging.error("Could not determine the file modification time of %s", + backup_full_file_name) + date = time = '' + + source = rpd_file.get_souce_href() + device = make_href(name=rpd_file.device_display_name, uri=rpd_file.device_uri) + + if data.backup_duplicate_overwrite: + self.problems.append(BackupOverwrittenProblem( + file_type_capitalized=rpd_file.title_capitalized, + file_type=rpd_file.title, + name=rpd_file.download_name, + uri=get_uri(full_file_name=backup_full_file_name), + source=source, + device=device, + date=date, + time=time + )) + msg = "Overwriting backup file %s" % backup_full_file_name + else: + self.problems.append(BackupAlreadyExistsProblem( + file_type_capitalized=rpd_file.title_capitalized, + file_type=rpd_file.title, + name=rpd_file.download_name, + uri=get_uri(full_file_name=backup_full_file_name), + source=source, + device=device, + date=date, + time=time + )) + msg = "Skipping backup of file %s because it already exists" % \ + backup_full_file_name + logging.warning(msg) + + if not backup_already_exists or data.backup_duplicate_overwrite: + logging.debug("Backing up file %s on device %s...", + data.download_count, self.device_name) + source = rpd_file.download_full_file_name + destination = backup_full_file_name + backup_succeeded = self.copy_from_filesystem(source, destination, rpd_file) + if backup_succeeded and self.verify_file: + md5 = hashlib.md5(open(backup_full_file_name).read()).hexdigest() + if md5 != rpd_file.md5: + pass + if backup_succeeded: + logging.debug("...backing up file %s on device %s succeeded", + data.download_count, self.device_name) + + if backup_succeeded: + mdata_exceptions = copy_file_metadata( + rpd_file.download_full_file_name, backup_full_file_name + ) + if not backup_succeeded: + if rpd_file.status == DownloadStatus.download_failed: + rpd_file.status = DownloadStatus.download_and_backup_failed + else: + rpd_file.status = DownloadStatus.backup_problem + else: + # backup any THM, audio or XMP files + if rpd_file.download_thm_full_name: + self.backup_associate_file(dest_dir, rpd_file.download_thm_full_name) + if rpd_file.download_audio_full_name: + self.backup_associate_file(dest_dir, rpd_file.download_audio_full_name) + if rpd_file.download_xmp_full_name: + self.backup_associate_file(dest_dir, rpd_file.download_xmp_full_name) + if rpd_file.download_log_full_name: + self.backup_associate_file(dest_dir, rpd_file.download_log_full_name) + + self.total_downloaded += rpd_file.size + bytes_not_downloaded = rpd_file.size - self.amount_downloaded + if bytes_not_downloaded and data.do_backup: + self.content = pickle.dumps( + BackupResults( + scan_id=self.scan_id, device_id=self.device_id, + total_downloaded=self.total_downloaded, chunk_downloaded=bytes_not_downloaded + ), + pickle.HIGHEST_PROTOCOL + ) + self.send_message_to_sink() + + self.content = pickle.dumps( + BackupResults( + scan_id=self.scan_id, device_id=self.device_id, backup_succeeded=backup_succeeded, + do_backup=data.do_backup, rpd_file=rpd_file, + backup_full_file_name=backup_full_file_name, mdata_exceptions=mdata_exceptions + ), + pickle.HIGHEST_PROTOCOL + ) + self.send_message_to_sink() + + def reset_problems(self) -> None: + self.problems = BackingUpProblems( + name=self.device_name, uri=self.uri + ) + + def send_problems(self) -> None: + if self.problems: + self.content = pickle.dumps( + BackupResults( + scan_id=self.scan_id, device_id=self.device_id, problems=self.problems + ), + pickle.HIGHEST_PROTOCOL + ) + self.send_message_to_sink() + self.reset_problems() + + def cleanup_pre_stop(self): + self.send_problems() + + def do_work(self): + + backup_arguments = pickle.loads(self.content) + self.path = backup_arguments.path + self.device_name = backup_arguments.device_name + self.uri = get_uri(path=self.path) + self.fdo_cache_normal = FdoCacheNormal() + self.fdo_cache_large = FdoCacheLarge() + + while True: + worker_id, directive, content = self.receiver.recv_multipart() + self.device_id = int(worker_id) + + self.check_for_command(directive, content) + + data = pickle.loads(content) # type: BackupFileData + if data.message == BackupStatus.backup_started: + self.reset_problems() + elif data.message == BackupStatus.backup_completed: + self.send_problems() + else: + self.amount_downloaded = 0 + self.init_copy_progress() + + self.do_backup(data=data) + + +if __name__ == "__main__": + backup = BackupFilesWorker()
\ No newline at end of file |