diff options
Diffstat (limited to 'rapid/subfolderfile.py')
| -rw-r--r-- | rapid/subfolderfile.py | 323 |
1 files changed, 267 insertions, 56 deletions
diff --git a/rapid/subfolderfile.py b/rapid/subfolderfile.py index 53118c2..c55b6d8 100644 --- a/rapid/subfolderfile.py +++ b/rapid/subfolderfile.py @@ -23,7 +23,7 @@ Generates names for files and folders. Runs a daemon process. """ -import os +import os, datetime, collections import gio import multiprocessing @@ -40,17 +40,66 @@ import config from gettext import gettext as _ +class SyncRawJpeg: + def __init__(self): + self.photos = {} + + def add_download(self, name, extension, date_time, sub_seconds, sequence_number_used): + if name not in self.photos: + self.photos[name] = ([extension], date_time, sub_seconds, sequence_number_used) + else: + if extension not in self.photos[name][0]: + self.photos[name][0].append(extension) + + + def matching_pair(self, name, extension, date_time, sub_seconds): + """Checks to see if the image matches an image that has already been downloaded. + Image name (minus extension), exif date time, and exif subseconds are checked. + + Returns -1 and a sequence number if the name, extension, and exif values match (i.e. it has already been downloaded) + Returns 0 and a sequence number if name and exif values match, but the extension is different (i.e. a matching RAW + JPG image) + Returns -99 and a sequence number of None if photos detected with the same filenames, but taken at different times + Returns 1 and a sequence number of None if no match""" + + if name in self.photos: + if self.photos[name][1] == date_time and self.photos[name][2] == sub_seconds: + if extension in self.photos[name][0]: + return (-1, self.photos[name][3]) + else: + return (0, self.photos[name][3]) + else: + return (-99, None) + return (1, None) + + def ext_exif_date_time(self, name): + """Returns first extension, exif date time and subseconds data for the already downloaded photo""" + return (self.photos[name][0][0], self.photos[name][1], self.photos[name][2]) + +def time_subseconds_human_readable(date, subseconds): + return _("%(hour)s:%(minute)s:%(second)s:%(subsecond)s") % \ + {'hour':date.strftime("%H"), + 'minute':date.strftime("%M"), + 'second':date.strftime("%S"), + 'subsecond': subseconds} +def load_metadata(rpd_file): + """ + Loads the metadata for the file. Returns True if operation succeeded, false + otherwise + """ + if rpd_file.metadata is None: + if not rpd_file.load_metadata(): + # Error in reading metadata + rpd_file.add_problem(None, pn.CANNOT_DOWNLOAD_BAD_METADATA, {'filetype': rpd_file.title_capitalized}) + return False + return True + def _generate_name(generator, rpd_file): do_generation = True if rpd_file.file_type == rpdfile.FILE_TYPE_PHOTO: - if rpd_file.metadata is None: - if not rpd_file.load_metadata(): - # Error in reading metadata - rpd_file.add_problem(None, pn.CANNOT_DOWNLOAD_BAD_METADATA, {'filetype': rpd_file.title_capitalized}) - do_generation = False + do_generation = load_metadata(rpd_file) else: if rpd_file.metadata is None: rpd_file.load_metadata() @@ -103,12 +152,85 @@ class SubfolderFile(multiprocessing.Process): logger.debug("Start of day is set to %s", self.day_start.value) - - - def progress_callback_no_update(self, amount_downloaded, total): pass + def file_exists(self, rpd_file, identifier=None): + """ + Notify user that the download file already exists + """ + # get information on when the existing file was last modified + try: + modification_time = os.path.getmtime(rpd_file.download_full_file_name) + dt = datetime.datetime.fromtimestamp(modification_time) + date = dt.strftime("%x") + time = dt.strftime("%X") + except: + logger.warning("Could not determine the file modification time of %s", + rpd_file.download_full_file_name) + date = time = '' + + if not identifier: + rpd_file.add_problem(None, pn.FILE_ALREADY_EXISTS_NO_DOWNLOAD, + {'filetype':rpd_file.title_capitalized}) + rpd_file.add_extra_detail(pn.EXISTING_FILE, + {'filetype': rpd_file.title, + 'date': date, 'time': time}) + rpd_file.status = config.STATUS_DOWNLOAD_FAILED + rpd_file.error_extra_detail = pn.extra_detail_definitions[pn.EXISTING_FILE] % \ + {'date':date, 'time':time, 'filetype': rpd_file.title} + else: + rpd_file.add_problem(None, pn.UNIQUE_IDENTIFIER_ADDED, + {'filetype':rpd_file.title_capitalized}) + rpd_file.add_extra_detail(pn.UNIQUE_IDENTIFIER, + {'identifier': identifier, + 'filetype': rpd_file.title, + 'date': date, 'time': time}) + rpd_file.status = config.STATUS_DOWNLOADED_WITH_WARNING + rpd_file.error_extra_detail = pn.extra_detail_definitions[pn.UNIQUE_IDENTIFIER] % \ + {'identifier': identifier, 'filetype': rpd_file.title, + 'date': date, 'time': time} + rpd_file.error_title = rpd_file.problem.get_title() + rpd_file.error_msg = _("Source: %(source)s\nDestination: %(destination)s") \ + % {'source': rpd_file.full_file_name, + 'destination': rpd_file.download_full_file_name} + return rpd_file + + def download_failure_file_error(self, rpd_file, inst): + """ + Handle cases where file failed to download + """ + rpd_file.add_problem(None, pn.DOWNLOAD_COPYING_ERROR, {'filetype': rpd_file.title}) + rpd_file.add_extra_detail(pn.DOWNLOAD_COPYING_ERROR_DETAIL, inst) + rpd_file.status = config.STATUS_DOWNLOAD_FAILED + logger.error("Failed to create file %s: %s", rpd_file.download_full_file_name, inst) + + rpd_file.error_title = rpd_file.problem.get_title() + rpd_file.error_msg = _("%(problem)s\nFile: %(file)s") % \ + {'problem': rpd_file.problem.get_problems(), + 'file': rpd_file.full_file_name} + + return rpd_file + + def same_name_different_exif(self, sync_photo_name, rpd_file): + """Notify the user that a file was already downloaded with the same name, but the exif information was different""" + i1_ext, i1_date_time, i1_subseconds = self.sync_raw_jpeg.ext_exif_date_time(sync_photo_name) + detail = {'image1': "%s%s" % (sync_photo_name, i1_ext), + 'image1_date': i1_date_time.strftime("%x"), + 'image1_time': time_subseconds_human_readable(i1_date_time, i1_subseconds), + 'image2': rpd_file.name, + 'image2_date': rpd_file.metadata.date_time().strftime("%x"), + 'image2_time': time_subseconds_human_readable( + rpd_file.metadata.date_time(), + rpd_file.metadata.sub_seconds())} + rpd_file.add_problem(None, pn.SAME_FILE_DIFFERENT_EXIF, detail) + + rpd_file.error_title = _('Photos detected with the same filenames, but taken at different times') + rpd_file.error_msg = pn.problem_definitions[pn.SAME_FILE_DIFFERENT_EXIF][1] % detail + rpd_file.status = config.STATUS_DOWNLOADED_WITH_WARNING + return rpd_file + + def run(self): """ Get subfolder and name. @@ -118,6 +240,8 @@ class SubfolderFile(multiprocessing.Process): """ i = 0 download_count = 0 + + duplicate_files = {} # Track downloads today, using a class whose purpose is to @@ -133,6 +257,8 @@ class SubfolderFile(multiprocessing.Process): self.sequences = gn.Sequences(self.downloads_today_tracker, self.stored_sequence_no.value) + self.sync_raw_jpeg = SyncRawJpeg() + while True: logger.debug("Finished %s. Getting next task.", download_count) @@ -148,42 +274,79 @@ class SubfolderFile(multiprocessing.Process): if download_succeeded: temp_file = gio.File(rpd_file.temp_full_file_name) - # Generate subfolder name and new file name - generation_succeeded = True - rpd_file = generate_subfolder(rpd_file) - if rpd_file.download_subfolder: + synchronize_raw_jpg_failed = False + if not (rpd_file.synchronize_raw_jpg and + rpd_file.file_type == rpdfile.FILE_TYPE_PHOTO): + synchronize_raw_jpg = False + sequence_to_use = None + else: + synchronize_raw_jpg = True + sync_photo_name, sync_photo_ext = os.path.splitext(rpd_file.name) + if not load_metadata(rpd_file): + synchronize_raw_jpg_failed = True + else: + j, sequence_to_use = self.sync_raw_jpeg.matching_pair( + name=sync_photo_name, extension=sync_photo_ext, + date_time=rpd_file.metadata.date_time(), + sub_seconds=rpd_file.metadata.sub_seconds()) + if j == -1: + # this exact file has already been downloaded (same extension, same filename, and same exif date time subsecond info) + if (rpd_file.download_conflict_resolution <> + config.ADD_UNIQUE_IDENTIFIER): + rpd_file.add_problem(None, pn.FILE_ALREADY_DOWNLOADED, {'filetype': rpd_file.title_capitalized}) + rpd_file.error_title = _('Photo has already been downloaded') + rpd_file.error_msg = _("Source: %(source)s") % {'source': rpd_file.full_file_name} + rpd_file.status = config.STATUS_DOWNLOAD_FAILED + synchronize_raw_jpg_failed = True + else: + self.sequences.set_matched_sequence_value(sequence_to_use) + if j == -99: + rpd_file = self.same_name_different_exif(sync_photo_name, rpd_file) - if self.refresh_downloads_today.value: - # overwrite downloads today value tracked here, - # as user has modified their preferences - self.downloads_today_tracker.set_raw_downloads_today_from_int(self.downloads_today.value) - self.downloads_today_tracker.set_raw_downloads_today_date(self.downloads_today_date.value) - self.downloads_today_tracker.day_start = self.day_start.value - self.refresh_downloads_today.value = False + if synchronize_raw_jpg_failed: + generation_succeeded = False + else: + # Generate subfolder name and new file name + generation_succeeded = True + rpd_file = generate_subfolder(rpd_file) + if rpd_file.download_subfolder: - # update whatever the stored value is - self.sequences.stored_sequence_no = self.stored_sequence_no.value - rpd_file.sequences = self.sequences - - # generate the file name - rpd_file = generate_name(rpd_file) - - if rpd_file.has_problem(): - rpd_file.status = config.STATUS_DOWNLOADED_WITH_WARNING + if self.refresh_downloads_today.value: + # overwrite downloads today value tracked here, + # as user has modified their preferences + self.downloads_today_tracker.set_raw_downloads_today_from_int(self.downloads_today.value) + self.downloads_today_tracker.set_raw_downloads_today_date(self.downloads_today_date.value) + self.downloads_today_tracker.day_start = self.day_start.value + self.refresh_downloads_today.value = False + + # update whatever the stored value is + self.sequences.stored_sequence_no = self.stored_sequence_no.value + rpd_file.sequences = self.sequences + + # generate the file name + rpd_file = generate_name(rpd_file) + + if rpd_file.has_problem(): + rpd_file.status = config.STATUS_DOWNLOADED_WITH_WARNING - # Check for any errors - if not rpd_file.download_subfolder or not rpd_file.download_name: - if not rpd_file.download_subfolder and not rpd_file.download_name: - area = _("subfolder and filename") - elif not rpd_file.download_name: - area = _("filename") - else: - area = _("subfolder") - rpd_file.add_problem(None, pn.ERROR_IN_NAME_GENERATION, {'filetype': rpd_file.title_capitalized, 'area': area}) - rpd_file.add_extra_detail(pn.NO_DATA_TO_NAME, {'filetype': area}) - generation_succeeded = False - rpd_file.status = config.STATUS_DOWNLOAD_FAILED - # FIXME: log error + # Check for any errors + if not rpd_file.download_subfolder or not rpd_file.download_name: + if not rpd_file.download_subfolder and not rpd_file.download_name: + area = _("subfolder and filename") + elif not rpd_file.download_name: + area = _("filename") + else: + area = _("subfolder") + rpd_file.add_problem(None, pn.ERROR_IN_NAME_GENERATION, {'filetype': rpd_file.title_capitalized, 'area': area}) + rpd_file.add_extra_detail(pn.NO_DATA_TO_NAME, {'filetype': area}) + generation_succeeded = False + rpd_file.status = config.STATUS_DOWNLOAD_FAILED + + rpd_file.error_title = rpd_file.problem.get_title() + rpd_file.error_msg = _("%(problem)s\nFile: %(file)s") % \ + {'problem': rpd_file.problem.get_problems(), + 'file': rpd_file.full_file_name} + if generation_succeeded: rpd_file.download_path = os.path.join(rpd_file.download_folder, rpd_file.download_subfolder) @@ -205,36 +368,83 @@ class SubfolderFile(multiprocessing.Process): # between the time it takes to query and the time it takes # to create a new directory. Ignore such errors. if inst.code <> gio.ERROR_EXISTS: - logger.error("Failed to create directory: %s", rpd_file.download_path) + logger.error("Failed to create download subfolder: %s", rpd_file.download_path) logger.error(inst) + rpd_file.error_title = _("Failed to create download subfolder") + rpd_file.error_msg = _("Path: %s") % rpd_file.download_path # Move temp file to subfolder download_file = gio.File(rpd_file.download_full_file_name) + add_unique_identifier = False try: temp_file.move(download_file, self.progress_callback_no_update, cancellable=None) move_succeeded = True if rpd_file.status <> config.STATUS_DOWNLOADED_WITH_WARNING: rpd_file.status = config.STATUS_DOWNLOADED except gio.Error, inst: - rpd_file.add_problem(None, pn.DOWNLOAD_COPYING_ERROR, {'filetype': rpd_file.title}) - rpd_file.add_extra_detail(pn.DOWNLOAD_COPYING_ERROR_DETAIL, inst) - rpd_file.status = config.STATUS_DOWNLOAD_FAILED - logger.error("Failed to create file %s: %s", rpd_file.download_full_file_name, inst) + if inst.code == gio.ERROR_EXISTS: + if (rpd_file.download_conflict_resolution == + config.ADD_UNIQUE_IDENTIFIER): + add_unique_identifier = True + else: + rpd_file = self.file_exists(rpd_file) + else: + rpd_file = self.download_failure_file_error(rpd_file, inst) + + if add_unique_identifier: + name = os.path.splitext(rpd_file.download_name) + full_name = rpd_file.download_full_file_name + suffix_already_used = True + while suffix_already_used: + duplicate_files[full_name] = duplicate_files.get( + full_name, 0) + 1 + identifier = '_%s' % duplicate_files[full_name] + rpd_file.download_name = name[0] + identifier + name[1] + rpd_file.download_full_file_name = os.path.join( + rpd_file.download_path, + rpd_file.download_name) + download_file = gio.File( + rpd_file.download_full_file_name) + + try: + temp_file.move(download_file, self.progress_callback_no_update, cancellable=None) + move_succeeded = True + suffix_already_used = False + rpd_file = self.file_exists(rpd_file, identifier) + logger.error("%s: %s - %s", rpd_file.full_file_name, + rpd_file.problem.get_title(), + rpd_file.problem.get_problems()) + except gio.Error, inst: + if inst.code <> gio.ERROR_EXISTS: + rpd_file = self.download_failure_file_error(rpd_file, inst) + + logger.debug("Finish processing file: %s", download_count) if move_succeeded: - if self.uses_session_sequece_no.value or self.uses_sequence_letter.value: - self.sequences.increment( - self.uses_session_sequece_no.value, - self.uses_sequence_letter.value) - if self.uses_stored_sequence_no.value: - self.stored_sequence_no.value += 1 - self.downloads_today_tracker.increment_downloads_today() - self.downloads_today.value = self.downloads_today_tracker.get_raw_downloads_today() - self.downloads_today_date.value = self.downloads_today_tracker.get_raw_downloads_today_date() + if synchronize_raw_jpg: + if sequence_to_use is None: + sequence = self.sequences.create_matched_sequences() + else: + sequence = sequence_to_use + self.sync_raw_jpeg.add_download(name=sync_photo_name, + extension=sync_photo_ext, + date_time=rpd_file.metadata.date_time(), + sub_seconds=rpd_file.metadata.sub_seconds(), + sequence_number_used=sequence) + if sequence_to_use is None: + if self.uses_session_sequece_no.value or self.uses_sequence_letter.value: + self.sequences.increment( + self.uses_session_sequece_no.value, + self.uses_sequence_letter.value) + if self.uses_stored_sequence_no.value: + self.stored_sequence_no.value += 1 + self.downloads_today_tracker.increment_downloads_today() + self.downloads_today.value = self.downloads_today_tracker.get_raw_downloads_today() + self.downloads_today_date.value = self.downloads_today_tracker.get_raw_downloads_today_date() if not move_succeeded: logger.error("%s: %s - %s", rpd_file.full_file_name, @@ -251,6 +461,7 @@ class SubfolderFile(multiprocessing.Process): rpd_file.metadata = None #purge metadata, as it cannot be pickled + rpd_file.sequences = None self.results_pipe.send((move_succeeded, rpd_file,)) i += 1 |