summaryrefslogtreecommitdiff
path: root/rapid/copyfiles.py
diff options
context:
space:
mode:
authorJulien Valroff <julien@kirya.net>2014-03-08 09:07:10 +0100
committerJulien Valroff <julien@kirya.net>2014-03-08 09:07:10 +0100
commite57608218ea2d7fa960b20f2aedf78094eeb79a5 (patch)
tree0fbe665e73c0ce5e1694de65f2cb2d48cd118134 /rapid/copyfiles.py
parente4926afd94329126adacce416fa8cf9daa32673b (diff)
Imported Upstream version 0.4.10upstream/0.4.10
Diffstat (limited to 'rapid/copyfiles.py')
-rw-r--r--rapid/copyfiles.py161
1 files changed, 117 insertions, 44 deletions
diff --git a/rapid/copyfiles.py b/rapid/copyfiles.py
index 5bc243c..c63e60f 100644
--- a/rapid/copyfiles.py
+++ b/rapid/copyfiles.py
@@ -1,7 +1,7 @@
#!/usr/bin/python
# -*- coding: latin1 -*-
-### Copyright (C) 2011-2012 Damon Lynch <damonlynch@gmail.com>
+### Copyright (C) 2011-2014 Damon Lynch <damonlynch@gmail.com>
### This program is free software; you can redistribute it and/or modify
### it under the terms of the GNU General Public License as published by
@@ -24,8 +24,6 @@ import os
import random
import string
-import gio
-
import logging
logger = multiprocessing.get_logger()
@@ -34,16 +32,51 @@ import rpdfile
import problemnotification as pn
import config
import thumbnail as tn
-
+import io
+import shutil
+import stat
+import hashlib
from gettext import gettext as _
+def copy_file_metadata(src, dst, logger=None):
+ """Copy all stat info (mode bits, atime, mtime, flags) from src to dst.
+
+ Adapated from python's shutil.copystat.
+
+ Necessary because with some NTFS file systems, there can be problems
+ with setting filesystem metadata like permissions and modification time"""
+
+ st = os.stat(src)
+ mode = stat.S_IMODE(st.st_mode)
+ try:
+ os.utime(dst, (st.st_atime, st.st_mtime))
+ except OSError as inst:
+ if logger:
+ logger.warning("Couldn't adjust file modification time when copying %s. %s: %s", src, inst.errno, inst.strerror)
+ try:
+ os.chmod(dst, mode)
+ except OSError as inst:
+ if logger:
+ logger.warning("Couldn't adjust file permissions when copying %s. %s: %s", src, inst.errno, inst.strerror)
+
+ if hasattr(os, 'chflags') and hasattr(st, 'st_flags'):
+ try:
+ os.chflags(dst, st.st_flags)
+ except OSError as inst:
+ for err in 'EOPNOTSUPP', 'ENOTSUP':
+ if hasattr(errno, err) and inst.errno == getattr(errno, err):
+ break
+ else:
+ raise
+
class CopyFiles(multiprocessing.Process):
"""
Copies files from source to temporary directory, giving them a random name
"""
def __init__(self, photo_download_folder, video_download_folder, files,
+ verify_file,
modify_files_during_download, modify_pipe,
scan_pid,
batch_size_MB, results_pipe, terminate_queue,
@@ -52,15 +85,18 @@ class CopyFiles(multiprocessing.Process):
self.results_pipe = results_pipe
self.terminate_queue = terminate_queue
self.batch_size_bytes = batch_size_MB * 1048576 # * 1024 * 1024
+ self.io_buffer = 1048576
self.photo_download_folder = photo_download_folder
self.video_download_folder = video_download_folder
self.files = files
+ self.verify_file = verify_file
self.modify_files_during_download = modify_files_during_download
self.modify_pipe = modify_pipe
self.scan_pid = scan_pid
self.no_files= len(self.files)
self.run_event = run_event
+
def check_termination_request(self):
"""
Check to see this process has not been requested to immediately terminate
@@ -74,29 +110,13 @@ class CopyFiles(multiprocessing.Process):
def update_progress(self, amount_downloaded, total):
- # first check if process is being terminated
- if not self.terminate_queue.empty():
- # it is - cancel the current copy
- self.cancel_copy.cancel()
- else:
- if not self.total_reached:
- chunk_downloaded = amount_downloaded - self.bytes_downloaded
- if (chunk_downloaded > self.batch_size_bytes) or (amount_downloaded == total):
- self.bytes_downloaded = amount_downloaded
- if amount_downloaded == total:
- # this function is called a couple of times when total is reached
- self.total_reached = True
-
- self.results_pipe.send((rpdmp.CONN_PARTIAL, (rpdmp.MSG_BYTES, (self.scan_pid, self.total_downloaded + amount_downloaded, chunk_downloaded))))
- if amount_downloaded == total:
- self.bytes_downloaded = 0
-
- def progress_callback(self, amount_downloaded, total):
- self.update_progress(amount_downloaded, total)
- def thm_progress_callback(self, amount_downloaded, total):
- # we don't care about tracking download progress for tiny THM files!
- pass
+ chunk_downloaded = amount_downloaded - self.bytes_downloaded
+ if (chunk_downloaded > self.batch_size_bytes) or (amount_downloaded == total):
+ self.bytes_downloaded = amount_downloaded
+ self.results_pipe.send((rpdmp.CONN_PARTIAL, (rpdmp.MSG_BYTES, (self.scan_pid, self.total_downloaded + amount_downloaded, chunk_downloaded))))
+ if amount_downloaded == total:
+ self.bytes_downloaded = 0
def run(self):
@@ -108,8 +128,6 @@ class CopyFiles(multiprocessing.Process):
self.bytes_downloaded = 0
self.total_downloaded = 0
- self.cancel_copy = gio.Cancellable()
-
self.create_temp_dirs()
# Send the location of both temporary directories, so they can be
@@ -126,7 +144,6 @@ class CopyFiles(multiprocessing.Process):
for i in range(self.no_files):
rpd_file = self.files[i]
- self.total_reached = False
# pause if instructed by the caller
self.run_event.wait()
@@ -134,8 +151,6 @@ class CopyFiles(multiprocessing.Process):
if self.check_termination_request():
return None
- source = gio.File(path=rpd_file.full_file_name)
-
#generate temporary name 5 digits long, no extension
temp_name = ''.join(random.choice(filename_characters) for i in xrange(5))
@@ -143,19 +158,42 @@ class CopyFiles(multiprocessing.Process):
self._get_dest_dir(rpd_file.file_type),
temp_name)
rpd_file.temp_full_file_name = temp_full_file_name
- dest = gio.File(path=temp_full_file_name)
copy_succeeded = False
+
+ src_bytes = ''
+
try:
- source.copy(dest, self.progress_callback, cancellable=self.cancel_copy)
+ dest = io.open(temp_full_file_name, 'wb', self.io_buffer)
+ src = io.open(rpd_file.full_file_name, 'rb', self.io_buffer)
+ total = rpd_file.size
+ amount_downloaded = 0
+ while True:
+ # first check if process is being terminated
+ if self.check_termination_request():
+ logger.debug("Closing partially written temporary file")
+ dest.close()
+ src.close()
+ return None
+ else:
+ chunk = src.read(self.io_buffer)
+ if chunk:
+ dest.write(chunk)
+ src_bytes += chunk
+ amount_downloaded += len(chunk)
+ self.update_progress(amount_downloaded, total)
+ else:
+ break
+ dest.close()
+ src.close()
copy_succeeded = True
- except gio.Error, inst:
+ except (IOError, OSError) as inst:
rpd_file.add_problem(None,
pn.DOWNLOAD_COPYING_ERROR_W_NO,
{'filetype': rpd_file.title})
rpd_file.add_extra_detail(
pn.DOWNLOAD_COPYING_ERROR_W_NO_DETAIL,
- {'errorno': inst.code, 'strerror': inst.message})
+ {'errorno': inst.errno, 'strerror': inst.strerror})
rpd_file.status = config.STATUS_DOWNLOAD_FAILED
@@ -167,38 +205,68 @@ class CopyFiles(multiprocessing.Process):
logger.error("Failed to download file: %s", rpd_file.full_file_name)
logger.error(inst)
self.update_progress(rpd_file.size, rpd_file.size)
+ except:
+ rpd_file.add_problem(None,
+ pn.DOWNLOAD_COPYING_ERROR,
+ {'filetype': rpd_file.title})
+ rpd_file.add_extra_detail(
+ pn.DOWNLOAD_COPYING_ERROR_DETAIL,
+ _("An unknown error occurred"))
+
+ rpd_file.status = config.STATUS_DOWNLOAD_FAILED
+
+ rpd_file.error_title = rpd_file.problem.get_title()
+ rpd_file.error_msg = _("%(problem)s\nFile: %(file)s") % \
+ {'problem': rpd_file.problem.get_problems(),
+ 'file': rpd_file.full_file_name}
+
+ logger.error("Failed to download file: %s", rpd_file.full_file_name)
+ self.update_progress(rpd_file.size, rpd_file.size)
# increment this amount regardless of whether the copy actually
# succeeded or not. It's neccessary to keep the user informed.
self.total_downloaded += rpd_file.size
+ try:
+ copy_file_metadata(rpd_file.full_file_name, temp_full_file_name, logger)
+ except:
+ logger.error("Unknown error updating filesystem metadata when copying %s", rpd_file.full_file_name)
+
# copy THM (video thumbnail file) if there is one
if copy_succeeded and rpd_file.thm_full_name:
- source = gio.File(path=rpd_file.thm_full_name)
# reuse video's file name
temp_thm_full_name = temp_full_file_name + '__rpd__thm'
- dest = gio.File(path=temp_thm_full_name)
try:
- source.copy(dest, self.thm_progress_callback, cancellable=self.cancel_copy)
+ shutil.copyfile(rpd_file.thm_full_name, temp_thm_full_name)
rpd_file.temp_thm_full_name = temp_thm_full_name
logger.debug("Copied video THM file %s", rpd_file.temp_thm_full_name)
- except gio.Error, inst:
+ except (IOError, OSError) as inst:
logger.error("Failed to download video THM file: %s", rpd_file.thm_full_name)
+ logger.error("%s: %s", inst.errno, inst.strerror)
+ try:
+ copy_file_metadata(rpd_file.thm_full_name, temp_thm_full_name. logger)
+ except:
+ logger.error("Unknown error updating filesystem metadata when copying %s", rpd_file.thm_full_name)
+
else:
temp_thm_full_name = None
#copy audio file if there is one
if copy_succeeded and rpd_file.audio_file_full_name:
- source = gio.File(path=rpd_file.audio_file_full_name)
# reuse photo's file name
temp_audio_full_name = temp_full_file_name + '__rpd__audio'
- dest = gio.File(path=temp_audio_full_name)
try:
- source.copy(dest, self.thm_progress_callback, cancellable=self.cancel_copy)
+ shutil.copyfile(rpd_file.audio_file_full_name, temp_audio_full_name)
rpd_file.temp_audio_full_name = temp_audio_full_name
logger.debug("Copied audio file %s", rpd_file.temp_audio_full_name)
- except gio.Error, inst:
+ except (IOError, OSError) as inst:
logger.error("Failed to download audio file: %s", rpd_file.audio_file_full_name)
+ logger.error("%s: %s", inst.errno, inst.strerror)
+ try:
+ copy_file_metadata(rpd_file.audio_file_full_name, temp_audio_full_name, logger)
+ except:
+ logger.error("Unknown error updating filesystem metadata when copying %s", rpd_file.audio_file_full_name)
+
if copy_succeeded and rpd_file.generate_thumbnail:
@@ -211,6 +279,9 @@ class CopyFiles(multiprocessing.Process):
thumbnail = None
thumbnail_icon = None
+ if copy_succeeded and self.verify_file:
+ rpd_file.md5 = hashlib.md5(src_bytes).hexdigest()
+
if rpd_file.metadata is not None:
rpd_file.metadata = None
@@ -254,8 +325,10 @@ class CopyFiles(multiprocessing.Process):
self.photo_temp_dir = self.video_temp_dir = None
if self.photo_download_folder is not None:
self.photo_temp_dir = self._create_temp_dir(self.photo_download_folder)
+ logger.debug("Photo temporary directory: %s", self.photo_temp_dir)
if self.video_download_folder is not None:
- self.video_temp_dir = self._create_temp_dir(self.photo_download_folder)
+ self.video_temp_dir = self._create_temp_dir(self.video_download_folder)
+ logger.debug("Video temporary directory: %s", self.video_temp_dir)