Merge tag 'upstream/0.4.10'

Upstream version 0.4.10

1 files changed, 117 insertions, 44 deletions

diff --git a/rapid/copyfiles.py b/rapid/copyfiles.py
index 5bc243c..c63e60f 100644
--- a/rapid/copyfiles.py
+++ b/rapid/copyfiles.py
@@ -1,7 +1,7 @@
 #!/usr/bin/python
 # -*- coding: latin1 -*-
 
-### Copyright (C) 2011-2012 Damon Lynch <damonlynch@gmail.com>
+### Copyright (C) 2011-2014 Damon Lynch <damonlynch@gmail.com>
 
 ### This program is free software; you can redistribute it and/or modify
 ### it under the terms of the GNU General Public License as published by
@@ -24,8 +24,6 @@ import os
 import random
 import string
 
-import gio
-
 import logging
 logger = multiprocessing.get_logger()
 
@@ -34,16 +32,51 @@ import rpdfile
 import problemnotification as pn
 import config
 import thumbnail as tn
-
+import io
+import shutil
+import stat
+import hashlib
 
 from gettext import gettext as _
 
+def copy_file_metadata(src, dst, logger=None):
+    """Copy all stat info (mode bits, atime, mtime, flags) from src to dst.
+
+    Adapated from python's shutil.copystat.
+
+    Necessary because with some NTFS file systems, there can be problems
+    with setting filesystem metadata like permissions and modification time"""
+
+    st = os.stat(src)
+    mode = stat.S_IMODE(st.st_mode)
+    try:
+        os.utime(dst, (st.st_atime, st.st_mtime))
+    except OSError as inst:
+        if logger:
+            logger.warning("Couldn't adjust file modification time when copying %s. %s: %s", src, inst.errno, inst.strerror)
+    try:
+        os.chmod(dst, mode)
+    except OSError as inst:
+        if logger:
+            logger.warning("Couldn't adjust file permissions when copying %s. %s: %s", src, inst.errno, inst.strerror)
+
+    if hasattr(os, 'chflags') and hasattr(st, 'st_flags'):
+        try:
+            os.chflags(dst, st.st_flags)
+        except OSError as inst:
+            for err in 'EOPNOTSUPP', 'ENOTSUP':
+                if hasattr(errno, err) and inst.errno == getattr(errno, err):
+                    break
+            else:
+                raise
+
 
 class CopyFiles(multiprocessing.Process):
     """
     Copies files from source to temporary directory, giving them a random name
     """
     def __init__(self, photo_download_folder, video_download_folder, files,
+                 verify_file,
                  modify_files_during_download, modify_pipe,
                  scan_pid,
                  batch_size_MB, results_pipe, terminate_queue,
@@ -52,15 +85,18 @@ class CopyFiles(multiprocessing.Process):
         self.results_pipe = results_pipe
         self.terminate_queue = terminate_queue
         self.batch_size_bytes = batch_size_MB * 1048576 # * 1024 * 1024
+        self.io_buffer = 1048576
         self.photo_download_folder = photo_download_folder
         self.video_download_folder = video_download_folder
         self.files = files
+        self.verify_file = verify_file
         self.modify_files_during_download = modify_files_during_download
         self.modify_pipe = modify_pipe
         self.scan_pid = scan_pid
         self.no_files= len(self.files)
         self.run_event = run_event
 
+
     def check_termination_request(self):
         """
         Check to see this process has not been requested to immediately terminate
@@ -74,29 +110,13 @@ class CopyFiles(multiprocessing.Process):
 
 
     def update_progress(self, amount_downloaded, total):
-        # first check if process is being terminated
-        if not self.terminate_queue.empty():
-            # it is - cancel the current copy
-            self.cancel_copy.cancel()
-        else:
-            if not self.total_reached:
-                chunk_downloaded = amount_downloaded - self.bytes_downloaded
-                if (chunk_downloaded > self.batch_size_bytes) or (amount_downloaded == total):
-                    self.bytes_downloaded = amount_downloaded
-                    if amount_downloaded == total:
-                        # this function is called a couple of times when total is reached
-                        self.total_reached = True
-
-                    self.results_pipe.send((rpdmp.CONN_PARTIAL, (rpdmp.MSG_BYTES, (self.scan_pid, self.total_downloaded + amount_downloaded, chunk_downloaded))))
-                    if amount_downloaded == total:
-                        self.bytes_downloaded = 0
-
-    def progress_callback(self, amount_downloaded, total):
-        self.update_progress(amount_downloaded, total)
 
-    def thm_progress_callback(self, amount_downloaded, total):
-        # we don't care about tracking download progress for tiny THM files!
-        pass
+        chunk_downloaded = amount_downloaded - self.bytes_downloaded
+        if (chunk_downloaded > self.batch_size_bytes) or (amount_downloaded == total):
+            self.bytes_downloaded = amount_downloaded
+            self.results_pipe.send((rpdmp.CONN_PARTIAL, (rpdmp.MSG_BYTES, (self.scan_pid, self.total_downloaded + amount_downloaded, chunk_downloaded))))
+            if amount_downloaded == total:
+                self.bytes_downloaded = 0
 
 
     def run(self):
@@ -108,8 +128,6 @@ class CopyFiles(multiprocessing.Process):
         self.bytes_downloaded = 0
         self.total_downloaded = 0
 
-        self.cancel_copy = gio.Cancellable()
-
         self.create_temp_dirs()
 
         # Send the location of both temporary directories, so they can be
@@ -126,7 +144,6 @@ class CopyFiles(multiprocessing.Process):
 
             for i in range(self.no_files):
                 rpd_file = self.files[i]
-                self.total_reached = False
 
                 # pause if instructed by the caller
                 self.run_event.wait()
@@ -134,8 +151,6 @@ class CopyFiles(multiprocessing.Process):
                 if self.check_termination_request():
                     return None
 
-                source = gio.File(path=rpd_file.full_file_name)
-
                 #generate temporary name 5 digits long, no extension
                 temp_name = ''.join(random.choice(filename_characters) for i in xrange(5))
 
@@ -143,19 +158,42 @@ class CopyFiles(multiprocessing.Process):
                                     self._get_dest_dir(rpd_file.file_type),
                                     temp_name)
                 rpd_file.temp_full_file_name = temp_full_file_name
-                dest = gio.File(path=temp_full_file_name)
 
                 copy_succeeded = False
+
+                src_bytes = ''
+
                 try:
-                    source.copy(dest, self.progress_callback, cancellable=self.cancel_copy)
+                    dest = io.open(temp_full_file_name, 'wb', self.io_buffer)
+                    src = io.open(rpd_file.full_file_name, 'rb', self.io_buffer)
+                    total = rpd_file.size
+                    amount_downloaded = 0
+                    while True:
+                        # first check if process is being terminated
+                        if self.check_termination_request():
+                            logger.debug("Closing partially written temporary file")
+                            dest.close()
+                            src.close()
+                            return None
+                        else:
+                            chunk = src.read(self.io_buffer)
+                            if chunk:
+                                dest.write(chunk)
+                                src_bytes += chunk
+                                amount_downloaded += len(chunk)
+                                self.update_progress(amount_downloaded, total)
+                            else:
+                                break
+                    dest.close()
+                    src.close()
                     copy_succeeded = True
-                except gio.Error, inst:
+                except (IOError, OSError) as inst:
                     rpd_file.add_problem(None,
                         pn.DOWNLOAD_COPYING_ERROR_W_NO,
                         {'filetype': rpd_file.title})
                     rpd_file.add_extra_detail(
                         pn.DOWNLOAD_COPYING_ERROR_W_NO_DETAIL,
-                        {'errorno': inst.code, 'strerror': inst.message})
+                        {'errorno': inst.errno, 'strerror': inst.strerror})
 
                     rpd_file.status = config.STATUS_DOWNLOAD_FAILED
 
@@ -167,38 +205,68 @@ class CopyFiles(multiprocessing.Process):
                     logger.error("Failed to download file: %s", rpd_file.full_file_name)
                     logger.error(inst)
                     self.update_progress(rpd_file.size, rpd_file.size)
+                except:
+                    rpd_file.add_problem(None,
+                        pn.DOWNLOAD_COPYING_ERROR,
+                        {'filetype': rpd_file.title})
+                    rpd_file.add_extra_detail(
+                        pn.DOWNLOAD_COPYING_ERROR_DETAIL,
+                        _("An unknown error occurred"))
+
+                    rpd_file.status = config.STATUS_DOWNLOAD_FAILED
+
+                    rpd_file.error_title = rpd_file.problem.get_title()
+                    rpd_file.error_msg = _("%(problem)s\nFile: %(file)s") % \
+                                  {'problem': rpd_file.problem.get_problems(),
+                                   'file': rpd_file.full_file_name}
+
+                    logger.error("Failed to download file: %s", rpd_file.full_file_name)
+                    self.update_progress(rpd_file.size, rpd_file.size)
 
                 # increment this amount regardless of whether the copy actually
                 # succeeded or not. It's neccessary to keep the user informed.
                 self.total_downloaded += rpd_file.size
 
+                try:
+                    copy_file_metadata(rpd_file.full_file_name, temp_full_file_name, logger)
+                except:
+                    logger.error("Unknown error updating filesystem metadata when copying %s", rpd_file.full_file_name)
+
                 # copy THM (video thumbnail file) if there is one
                 if copy_succeeded and rpd_file.thm_full_name:
-                    source = gio.File(path=rpd_file.thm_full_name)
                     # reuse video's file name
                     temp_thm_full_name = temp_full_file_name + '__rpd__thm'
-                    dest = gio.File(path=temp_thm_full_name)
                     try:
-                        source.copy(dest, self.thm_progress_callback, cancellable=self.cancel_copy)
+                        shutil.copyfile(rpd_file.thm_full_name, temp_thm_full_name)
                         rpd_file.temp_thm_full_name = temp_thm_full_name
                         logger.debug("Copied video THM file %s", rpd_file.temp_thm_full_name)
-                    except gio.Error, inst:
+                    except (IOError, OSError) as inst:
                         logger.error("Failed to download video THM file: %s", rpd_file.thm_full_name)
+                        logger.error("%s: %s", inst.errno, inst.strerror)
+                    try:
+                        copy_file_metadata(rpd_file.thm_full_name, temp_thm_full_name. logger)
+                    except:
+                        logger.error("Unknown error updating filesystem metadata when copying %s", rpd_file.thm_full_name)
+
                 else:
                     temp_thm_full_name = None
 
                 #copy audio file if there is one
                 if copy_succeeded and rpd_file.audio_file_full_name:
-                    source = gio.File(path=rpd_file.audio_file_full_name)
                     # reuse photo's file name
                     temp_audio_full_name = temp_full_file_name + '__rpd__audio'
-                    dest = gio.File(path=temp_audio_full_name)
                     try:
-                        source.copy(dest, self.thm_progress_callback, cancellable=self.cancel_copy)
+                        shutil.copyfile(rpd_file.audio_file_full_name, temp_audio_full_name)
                         rpd_file.temp_audio_full_name = temp_audio_full_name
                         logger.debug("Copied audio file %s", rpd_file.temp_audio_full_name)
-                    except gio.Error, inst:
+                    except (IOError, OSError) as inst:
                         logger.error("Failed to download audio file: %s", rpd_file.audio_file_full_name)
+                        logger.error("%s: %s", inst.errno, inst.strerror)
+                    try:
+                        copy_file_metadata(rpd_file.audio_file_full_name, temp_audio_full_name, logger)
+                    except:
+                        logger.error("Unknown error updating filesystem metadata when copying %s", rpd_file.audio_file_full_name)
+
 
 
                 if copy_succeeded and rpd_file.generate_thumbnail:
@@ -211,6 +279,9 @@ class CopyFiles(multiprocessing.Process):
                     thumbnail = None
                     thumbnail_icon = None
 
+                if copy_succeeded and self.verify_file:
+                    rpd_file.md5 = hashlib.md5(src_bytes).hexdigest()
+
                 if rpd_file.metadata is not None:
                     rpd_file.metadata = None
 
@@ -254,8 +325,10 @@ class CopyFiles(multiprocessing.Process):
         self.photo_temp_dir = self.video_temp_dir = None
         if self.photo_download_folder is not None:
             self.photo_temp_dir = self._create_temp_dir(self.photo_download_folder)
+            logger.debug("Photo temporary directory: %s", self.photo_temp_dir)
         if self.video_download_folder is not None:
-            self.video_temp_dir = self._create_temp_dir(self.photo_download_folder)
+            self.video_temp_dir = self._create_temp_dir(self.video_download_folder)
+            logger.debug("Video temporary directory: %s", self.video_temp_dir)