Merge commit 'upstream/0.4.0_alpha4' into experimental

1 files changed, 267 insertions, 56 deletions

diff --git a/rapid/subfolderfile.py b/rapid/subfolderfile.py
index 53118c2..c55b6d8 100644
--- a/rapid/subfolderfile.py
+++ b/rapid/subfolderfile.py
@@ -23,7 +23,7 @@ Generates names for files and folders.
 Runs a daemon process.
 """
 
-import os
+import os, datetime, collections
 
 import gio
 import multiprocessing
@@ -40,17 +40,66 @@ import config
 
 from gettext import gettext as _
 
+class SyncRawJpeg:
+    def __init__(self):
+        self.photos = {}
+        
+    def add_download(self, name, extension, date_time, sub_seconds, sequence_number_used):
+        if name not in self.photos:
+            self.photos[name] = ([extension], date_time, sub_seconds, sequence_number_used)
+        else:
+            if extension not in self.photos[name][0]:
+                self.photos[name][0].append(extension)
+
+        
+    def matching_pair(self, name, extension, date_time, sub_seconds):
+        """Checks to see if the image matches an image that has already been downloaded.
+        Image name (minus extension), exif date time, and exif subseconds are checked.
+        
+        Returns -1 and a sequence number if the name, extension, and exif values match (i.e. it has already been downloaded)
+        Returns 0 and a sequence number if name and exif values match, but the extension is different (i.e. a matching RAW + JPG image)
+        Returns -99 and a sequence number of None if photos detected with the same filenames, but taken at different times
+        Returns 1 and a sequence number of None if no match"""
+        
+        if name in self.photos:
+            if self.photos[name][1] == date_time and self.photos[name][2] == sub_seconds:
+                if extension in self.photos[name][0]:
+                    return (-1, self.photos[name][3])
+                else:
+                    return (0, self.photos[name][3])
+            else:
+                return (-99, None)
+        return (1, None)
+        
+    def ext_exif_date_time(self, name):
+        """Returns first extension, exif date time and subseconds data for the already downloaded photo"""
+        return (self.photos[name][0][0], self.photos[name][1], self.photos[name][2])
+        
+def time_subseconds_human_readable(date, subseconds):
+    return _("%(hour)s:%(minute)s:%(second)s:%(subsecond)s") % \
+            {'hour':date.strftime("%H"),
+             'minute':date.strftime("%M"), 
+             'second':date.strftime("%S"),
+             'subsecond': subseconds}        
 
+def load_metadata(rpd_file):
+    """
+    Loads the metadata for the file. Returns True if operation succeeded, false
+    otherwise
+    """
+    if rpd_file.metadata is None:        
+        if not rpd_file.load_metadata():
+            # Error in reading metadata
+            rpd_file.add_problem(None, pn.CANNOT_DOWNLOAD_BAD_METADATA, {'filetype': rpd_file.title_capitalized})
+            return False
+    return True
+        
 
 def _generate_name(generator, rpd_file):
     
     do_generation = True
     if rpd_file.file_type == rpdfile.FILE_TYPE_PHOTO:
-        if rpd_file.metadata is None:        
-            if not rpd_file.load_metadata():
-                # Error in reading metadata
-                rpd_file.add_problem(None, pn.CANNOT_DOWNLOAD_BAD_METADATA, {'filetype': rpd_file.title_capitalized})
-                do_generation = False
+        do_generation = load_metadata(rpd_file)
     else:
         if rpd_file.metadata is None:
             rpd_file.load_metadata()
@@ -103,12 +152,85 @@ class SubfolderFile(multiprocessing.Process):
         
         logger.debug("Start of day is set to %s", self.day_start.value)
         
-
-
-        
     def progress_callback_no_update(self, amount_downloaded, total):
         pass
         
+    def file_exists(self, rpd_file, identifier=None):
+        """
+        Notify user that the download file already exists
+        """
+        # get information on when the existing file was last modified
+        try:
+            modification_time = os.path.getmtime(rpd_file.download_full_file_name)
+            dt = datetime.datetime.fromtimestamp(modification_time)
+            date = dt.strftime("%x")
+            time = dt.strftime("%X")
+        except:
+            logger.warning("Could not determine the file modification time of %s", 
+                                rpd_file.download_full_file_name)
+            date = time = ''
+            
+        if not identifier:
+            rpd_file.add_problem(None, pn.FILE_ALREADY_EXISTS_NO_DOWNLOAD, 
+                                {'filetype':rpd_file.title_capitalized})
+            rpd_file.add_extra_detail(pn.EXISTING_FILE, 
+                                {'filetype': rpd_file.title, 
+                                'date': date, 'time': time})
+            rpd_file.status = config.STATUS_DOWNLOAD_FAILED
+            rpd_file.error_extra_detail = pn.extra_detail_definitions[pn.EXISTING_FILE] % \
+                  {'date':date, 'time':time, 'filetype': rpd_file.title}
+        else:
+            rpd_file.add_problem(None, pn.UNIQUE_IDENTIFIER_ADDED, 
+                                {'filetype':rpd_file.title_capitalized})
+            rpd_file.add_extra_detail(pn.UNIQUE_IDENTIFIER, 
+                                {'identifier': identifier, 
+                                'filetype': rpd_file.title,
+                                'date': date, 'time': time})
+            rpd_file.status = config.STATUS_DOWNLOADED_WITH_WARNING
+            rpd_file.error_extra_detail = pn.extra_detail_definitions[pn.UNIQUE_IDENTIFIER] % \
+                   {'identifier': identifier, 'filetype': rpd_file.title,
+                    'date': date, 'time': time}
+        rpd_file.error_title = rpd_file.problem.get_title()
+        rpd_file.error_msg = _("Source: %(source)s\nDestination: %(destination)s") \
+                % {'source': rpd_file.full_file_name, 
+                   'destination': rpd_file.download_full_file_name}
+        return rpd_file
+        
+    def download_failure_file_error(self, rpd_file, inst):
+        """
+        Handle cases where file failed to download
+        """
+        rpd_file.add_problem(None, pn.DOWNLOAD_COPYING_ERROR, {'filetype': rpd_file.title})
+        rpd_file.add_extra_detail(pn.DOWNLOAD_COPYING_ERROR_DETAIL, inst)
+        rpd_file.status = config.STATUS_DOWNLOAD_FAILED
+        logger.error("Failed to create file %s: %s", rpd_file.download_full_file_name, inst)
+        
+        rpd_file.error_title = rpd_file.problem.get_title()
+        rpd_file.error_msg = _("%(problem)s\nFile: %(file)s") % \
+                              {'problem': rpd_file.problem.get_problems(),
+                               'file': rpd_file.full_file_name}
+        
+        return rpd_file
+        
+    def same_name_different_exif(self, sync_photo_name, rpd_file):
+        """Notify the user that a file was already downloaded with the same name, but the exif information was different"""
+        i1_ext, i1_date_time, i1_subseconds = self.sync_raw_jpeg.ext_exif_date_time(sync_photo_name)
+        detail = {'image1': "%s%s" % (sync_photo_name, i1_ext), 
+            'image1_date': i1_date_time.strftime("%x"),
+            'image1_time': time_subseconds_human_readable(i1_date_time, i1_subseconds), 
+            'image2':      rpd_file.name, 
+            'image2_date': rpd_file.metadata.date_time().strftime("%x"),
+            'image2_time': time_subseconds_human_readable(
+                                rpd_file.metadata.date_time(), 
+                                rpd_file.metadata.sub_seconds())}
+        rpd_file.add_problem(None, pn.SAME_FILE_DIFFERENT_EXIF, detail)
+
+        rpd_file.error_title = _('Photos detected with the same filenames, but taken at different times')
+        rpd_file.error_msg = pn.problem_definitions[pn.SAME_FILE_DIFFERENT_EXIF][1] % detail
+        rpd_file.status = config.STATUS_DOWNLOADED_WITH_WARNING
+        return rpd_file
+        
+        
     def run(self):
         """
         Get subfolder and name.
@@ -118,6 +240,8 @@ class SubfolderFile(multiprocessing.Process):
         """
         i = 0
         download_count = 0
+        
+        duplicate_files = {}
 
 
         # Track downloads today, using a class whose purpose is to 
@@ -133,6 +257,8 @@ class SubfolderFile(multiprocessing.Process):
         self.sequences = gn.Sequences(self.downloads_today_tracker, 
                                       self.stored_sequence_no.value)
                                       
+        self.sync_raw_jpeg = SyncRawJpeg()
+                                      
 
         while True:
             logger.debug("Finished %s. Getting next task.", download_count)
@@ -148,42 +274,79 @@ class SubfolderFile(multiprocessing.Process):
             if download_succeeded:
                 temp_file = gio.File(rpd_file.temp_full_file_name)
 
-                # Generate subfolder name and new file name                
-                generation_succeeded = True
-                rpd_file = generate_subfolder(rpd_file)
-                if rpd_file.download_subfolder:
+                synchronize_raw_jpg_failed = False
+                if not (rpd_file.synchronize_raw_jpg and
+                    rpd_file.file_type == rpdfile.FILE_TYPE_PHOTO):
+                    synchronize_raw_jpg = False
+                    sequence_to_use = None
+                else:
+                    synchronize_raw_jpg = True
+                    sync_photo_name, sync_photo_ext = os.path.splitext(rpd_file.name)
+                    if not load_metadata(rpd_file):
+                        synchronize_raw_jpg_failed = True
+                    else:
+                        j, sequence_to_use = self.sync_raw_jpeg.matching_pair(
+                                name=sync_photo_name, extension=sync_photo_ext, 
+                                date_time=rpd_file.metadata.date_time(), 
+                                sub_seconds=rpd_file.metadata.sub_seconds())
+                        if j == -1:
+                            # this exact file has already been downloaded (same extension, same filename, and same exif date time subsecond info)
+                            if (rpd_file.download_conflict_resolution <>
+                                    config.ADD_UNIQUE_IDENTIFIER):
+                                rpd_file.add_problem(None, pn.FILE_ALREADY_DOWNLOADED, {'filetype': rpd_file.title_capitalized})
+                                rpd_file.error_title = _('Photo has already been downloaded')
+                                rpd_file.error_msg = _("Source: %(source)s") % {'source': rpd_file.full_file_name}
+                                rpd_file.status = config.STATUS_DOWNLOAD_FAILED
+                                synchronize_raw_jpg_failed = True
+                        else:
+                            self.sequences.set_matched_sequence_value(sequence_to_use)
+                            if j == -99:
+                                rpd_file = self.same_name_different_exif(sync_photo_name, rpd_file)
                     
-                    if self.refresh_downloads_today.value:
-                        # overwrite downloads today value tracked here,
-                        # as user has modified their preferences
-                        self.downloads_today_tracker.set_raw_downloads_today_from_int(self.downloads_today.value)
-                        self.downloads_today_tracker.set_raw_downloads_today_date(self.downloads_today_date.value)
-                        self.downloads_today_tracker.day_start = self.day_start.value
-                        self.refresh_downloads_today.value = False
+                if synchronize_raw_jpg_failed:
+                    generation_succeeded = False
+                else:
+                    # Generate subfolder name and new file name
+                    generation_succeeded = True
+                    rpd_file = generate_subfolder(rpd_file)
+                    if rpd_file.download_subfolder:
                         
-                    # update whatever the stored value is
-                    self.sequences.stored_sequence_no = self.stored_sequence_no.value
-                    rpd_file.sequences = self.sequences
-                    
-                    # generate the file name
-                    rpd_file = generate_name(rpd_file)
-                    
-                    if rpd_file.has_problem():
-                        rpd_file.status = config.STATUS_DOWNLOADED_WITH_WARNING
+                        if self.refresh_downloads_today.value:
+                            # overwrite downloads today value tracked here,
+                            # as user has modified their preferences
+                            self.downloads_today_tracker.set_raw_downloads_today_from_int(self.downloads_today.value)
+                            self.downloads_today_tracker.set_raw_downloads_today_date(self.downloads_today_date.value)
+                            self.downloads_today_tracker.day_start = self.day_start.value
+                            self.refresh_downloads_today.value = False
+                            
+                        # update whatever the stored value is
+                        self.sequences.stored_sequence_no = self.stored_sequence_no.value
+                        rpd_file.sequences = self.sequences
+                        
+                        # generate the file name
+                        rpd_file = generate_name(rpd_file)
+                        
+                        if rpd_file.has_problem():
+                            rpd_file.status = config.STATUS_DOWNLOADED_WITH_WARNING
 
-                # Check for any errors
-                if not rpd_file.download_subfolder or not rpd_file.download_name:
-                    if not rpd_file.download_subfolder and not rpd_file.download_name:
-                        area = _("subfolder and filename")
-                    elif not rpd_file.download_name:
-                        area = _("filename")
-                    else:
-                        area = _("subfolder")
-                    rpd_file.add_problem(None, pn.ERROR_IN_NAME_GENERATION, {'filetype': rpd_file.title_capitalized, 'area': area})
-                    rpd_file.add_extra_detail(pn.NO_DATA_TO_NAME, {'filetype': area})
-                    generation_succeeded = False
-                    rpd_file.status = config.STATUS_DOWNLOAD_FAILED
-                    # FIXME: log error
+                    # Check for any errors
+                    if not rpd_file.download_subfolder or not rpd_file.download_name:
+                        if not rpd_file.download_subfolder and not rpd_file.download_name:
+                            area = _("subfolder and filename")
+                        elif not rpd_file.download_name:
+                            area = _("filename")
+                        else:
+                            area = _("subfolder")
+                        rpd_file.add_problem(None, pn.ERROR_IN_NAME_GENERATION, {'filetype': rpd_file.title_capitalized, 'area': area})
+                        rpd_file.add_extra_detail(pn.NO_DATA_TO_NAME, {'filetype': area})
+                        generation_succeeded = False
+                        rpd_file.status = config.STATUS_DOWNLOAD_FAILED
+                        
+                        rpd_file.error_title = rpd_file.problem.get_title()
+                        rpd_file.error_msg = _("%(problem)s\nFile: %(file)s") % \
+                                     {'problem': rpd_file.problem.get_problems(),
+                                      'file': rpd_file.full_file_name}
+                    
                     
                 if generation_succeeded:
                     rpd_file.download_path = os.path.join(rpd_file.download_folder, rpd_file.download_subfolder)
@@ -205,36 +368,83 @@ class SubfolderFile(multiprocessing.Process):
                             # between the time it takes to query and the time it takes
                             # to create a new directory. Ignore such errors.
                             if inst.code <> gio.ERROR_EXISTS:
-                                logger.error("Failed to create directory: %s", rpd_file.download_path)
+                                logger.error("Failed to create download subfolder: %s", rpd_file.download_path)
                                 logger.error(inst)
+                                rpd_file.error_title = _("Failed to create download subfolder")
+                                rpd_file.error_msg = _("Path: %s") % rpd_file.download_path
                     
                     # Move temp file to subfolder
 
                     download_file = gio.File(rpd_file.download_full_file_name)
                     
+                    add_unique_identifier = False
                     try:
                         temp_file.move(download_file, self.progress_callback_no_update, cancellable=None)
                         move_succeeded = True
                         if rpd_file.status <> config.STATUS_DOWNLOADED_WITH_WARNING:
                             rpd_file.status = config.STATUS_DOWNLOADED
                     except gio.Error, inst:
-                        rpd_file.add_problem(None, pn.DOWNLOAD_COPYING_ERROR, {'filetype': rpd_file.title})
-                        rpd_file.add_extra_detail(pn.DOWNLOAD_COPYING_ERROR_DETAIL, inst)
-                        rpd_file.status = config.STATUS_DOWNLOAD_FAILED
-                        logger.error("Failed to create file %s: %s", rpd_file.download_full_file_name, inst)
+                        if inst.code == gio.ERROR_EXISTS:
+                            if (rpd_file.download_conflict_resolution == 
+                                config.ADD_UNIQUE_IDENTIFIER):
+                                add_unique_identifier = True
+                            else:
+                                rpd_file = self.file_exists(rpd_file)
+                        else:
+                            rpd_file = self.download_failure_file_error(rpd_file, inst)
+                    
+                    if add_unique_identifier:
+                        name = os.path.splitext(rpd_file.download_name)
+                        full_name = rpd_file.download_full_file_name
+                        suffix_already_used = True
+                        while suffix_already_used:
+                            duplicate_files[full_name] = duplicate_files.get(
+                                                              full_name, 0) + 1
+                            identifier = '_%s' % duplicate_files[full_name]
+                            rpd_file.download_name = name[0] + identifier + name[1]
+                            rpd_file.download_full_file_name = os.path.join(
+                                                    rpd_file.download_path,
+                                                    rpd_file.download_name)
+                            download_file = gio.File(
+                                            rpd_file.download_full_file_name)
+                            
+                            try:
+                                temp_file.move(download_file, self.progress_callback_no_update, cancellable=None)
+                                move_succeeded = True
+                                suffix_already_used = False
+                                rpd_file = self.file_exists(rpd_file, identifier)
+                                logger.error("%s: %s - %s", rpd_file.full_file_name, 
+                                    rpd_file.problem.get_title(), 
+                                    rpd_file.problem.get_problems())
+                            except gio.Error, inst:
+                                if inst.code <> gio.ERROR_EXISTS:
+                                    rpd_file = self.download_failure_file_error(rpd_file, inst)
+                            
+                        
                         
                     logger.debug("Finish processing file: %s", download_count)                    
                 
                 if move_succeeded:
-                    if self.uses_session_sequece_no.value or self.uses_sequence_letter.value:
-                        self.sequences.increment(
-                                        self.uses_session_sequece_no.value, 
-                                        self.uses_sequence_letter.value)
-                    if self.uses_stored_sequence_no.value:
-                        self.stored_sequence_no.value += 1
-                    self.downloads_today_tracker.increment_downloads_today()
-                    self.downloads_today.value = self.downloads_today_tracker.get_raw_downloads_today()
-                    self.downloads_today_date.value = self.downloads_today_tracker.get_raw_downloads_today_date()
+                    if synchronize_raw_jpg:
+                        if sequence_to_use is None:
+                            sequence = self.sequences.create_matched_sequences()
+                        else:
+                            sequence = sequence_to_use
+                        self.sync_raw_jpeg.add_download(name=sync_photo_name,
+                                extension=sync_photo_ext,
+                                date_time=rpd_file.metadata.date_time(),
+                                sub_seconds=rpd_file.metadata.sub_seconds(),
+                                sequence_number_used=sequence)
+                    if sequence_to_use is None:
+                        if self.uses_session_sequece_no.value or self.uses_sequence_letter.value:
+                            self.sequences.increment(
+                                            self.uses_session_sequece_no.value, 
+                                            self.uses_sequence_letter.value)
+                        if self.uses_stored_sequence_no.value:
+                            self.stored_sequence_no.value += 1
+                        self.downloads_today_tracker.increment_downloads_today()
+                        self.downloads_today.value = self.downloads_today_tracker.get_raw_downloads_today()
+                        self.downloads_today_date.value = self.downloads_today_tracker.get_raw_downloads_today_date()
                 
                 if not move_succeeded:
                     logger.error("%s: %s - %s", rpd_file.full_file_name, 
@@ -251,6 +461,7 @@ class SubfolderFile(multiprocessing.Process):
             
             
             rpd_file.metadata = None #purge metadata, as it cannot be pickled
+            rpd_file.sequences = None
             self.results_pipe.send((move_succeeded, rpd_file,))
             
             i += 1