diff options
Diffstat (limited to 'rapid/downloadtracker.py')
| -rw-r--r-- | rapid/downloadtracker.py | 61 |
1 files changed, 45 insertions, 16 deletions
diff --git a/rapid/downloadtracker.py b/rapid/downloadtracker.py index 309da71..75c9c9d 100644 --- a/rapid/downloadtracker.py +++ b/rapid/downloadtracker.py @@ -19,7 +19,8 @@ import time from rpdfile import FILE_TYPE_PHOTO, FILE_TYPE_VIDEO -from config import STATUS_DOWNLOAD_FAILED, STATUS_DOWNLOADED_WITH_WARNING +from config import STATUS_DOWNLOAD_FAILED, STATUS_DOWNLOADED_WITH_WARNING, \ + STATUS_DOWNLOAD_AND_BACKUP_FAILED, STATUS_BACKUP_PROBLEM from gettext import gettext as _ @@ -28,10 +29,17 @@ class DownloadTracker: Track file downloads - their size, number, and any problems """ def __init__(self): + self.file_types_present_by_scan_pid = dict() + self._refresh_values() + + def _refresh_values(self): + """ these values are reset when a download is completed""" self.size_of_download_in_bytes_by_scan_pid = dict() - self.total_bytes_copied_in_bytes_by_scan_pid = dict() + self.raw_size_of_download_in_bytes_by_scan_pid = dict() + self.total_bytes_copied_by_scan_pid = dict() + self.total_bytes_backed_up_by_scan_pid = dict() self.no_files_in_download_by_scan_pid = dict() - self.file_types_present_by_scan_pid = dict() + # 'Download count' tracks the index of the file being downloaded # into the list of files that need to be downloaded -- much like # a counter in a for loop, e.g. 'for i in list', where i is the counter @@ -50,11 +58,16 @@ class DownloadTracker: self.total_video_failures = 0 self.total_warnings = 0 self.total_bytes_to_download = 0 + self.backups_performed_by_unique_id = dict() + + def set_no_backup_devices(self, no_backup_devices): + self.no_backup_devices = no_backup_devices def init_stats(self, scan_pid, bytes, no_files): self.no_files_in_download_by_scan_pid[scan_pid] = no_files self.rename_chunk[scan_pid] = bytes / 10 / no_files self.size_of_download_in_bytes_by_scan_pid[scan_pid] = bytes + self.rename_chunk[scan_pid] * no_files + self.raw_size_of_download_in_bytes_by_scan_pid[scan_pid] = bytes self.total_bytes_to_download += self.size_of_download_in_bytes_by_scan_pid[scan_pid] self.files_downloaded[scan_pid] = 0 self.photos_downloaded[scan_pid] = 0 @@ -62,6 +75,7 @@ class DownloadTracker: self.photo_failures[scan_pid] = 0 self.video_failures[scan_pid] = 0 self.warnings[scan_pid] = 0 + self.total_bytes_backed_up_by_scan_pid[scan_pid] = 0 def get_no_files_in_download(self, scan_pid): return self.no_files_in_download_by_scan_pid[scan_pid] @@ -81,11 +95,19 @@ class DownloadTracker: def get_no_warnings(self, scan_pid): return self.warnings.get(scan_pid, 0) + + def file_backed_up(self, unique_id): + self.backups_performed_by_unique_id[unique_id] = \ + self.backups_performed_by_unique_id.get(unique_id, 0) + 1 + + def all_files_backed_up(self, unique_id): + v = self.backups_performed_by_unique_id[unique_id] == self.no_backup_devices + return v def file_downloaded_increment(self, scan_pid, file_type, status): self.files_downloaded[scan_pid] += 1 - if status <> STATUS_DOWNLOAD_FAILED: + if status <> STATUS_DOWNLOAD_FAILED and status <> STATUS_DOWNLOAD_AND_BACKUP_FAILED: if file_type == FILE_TYPE_PHOTO: self.photos_downloaded[scan_pid] += 1 self.total_photos_downloaded += 1 @@ -93,7 +115,7 @@ class DownloadTracker: self.videos_downloaded[scan_pid] += 1 self.total_videos_downloaded += 1 - if status == STATUS_DOWNLOADED_WITH_WARNING: + if status == STATUS_DOWNLOADED_WITH_WARNING or status == STATUS_BACKUP_PROBLEM: self.warnings[scan_pid] += 1 self.total_warnings += 1 else: @@ -112,16 +134,19 @@ class DownloadTracker: """ # three components: copy (download), rename, and backup - percent_complete = ((float( - self.total_bytes_copied_in_bytes_by_scan_pid[scan_pid]) + percent_complete = (((float( + self.total_bytes_copied_by_scan_pid[scan_pid]) + self.rename_chunk[scan_pid] * self.files_downloaded[scan_pid]) - / self.size_of_download_in_bytes_by_scan_pid[scan_pid]) * 100 + + self.total_bytes_backed_up_by_scan_pid[scan_pid]) + / (self.size_of_download_in_bytes_by_scan_pid[scan_pid] + + self.raw_size_of_download_in_bytes_by_scan_pid[scan_pid] * + self.no_backup_devices)) * 100 return percent_complete def get_overall_percent_complete(self): total = 0 - for scan_pid in self.total_bytes_copied_in_bytes_by_scan_pid: - total += (self.total_bytes_copied_in_bytes_by_scan_pid[scan_pid] + + for scan_pid in self.total_bytes_copied_by_scan_pid: + total += (self.total_bytes_copied_by_scan_pid[scan_pid] + (self.rename_chunk[scan_pid] * self.files_downloaded[scan_pid])) @@ -129,7 +154,10 @@ class DownloadTracker: return percent_complete def set_total_bytes_copied(self, scan_pid, total_bytes): - self.total_bytes_copied_in_bytes_by_scan_pid[scan_pid] = total_bytes + self.total_bytes_copied_by_scan_pid[scan_pid] = total_bytes + + def increment_bytes_backed_up(self, scan_pid, chunk_downloaded): + self.total_bytes_backed_up_by_scan_pid[scan_pid] += chunk_downloaded def set_download_count_for_file(self, unique_id, download_count): self.download_count_for_file_by_unique_id[unique_id] = download_count @@ -152,12 +180,13 @@ class DownloadTracker: else return False """ return (self.total_warnings == 0 and - self.photo_failures == 0 and - self.video_failures == 0) + self.total_photo_failures == 0 and + self.total_video_failures == 0) def purge(self, scan_pid): del self.no_files_in_download_by_scan_pid[scan_pid] del self.size_of_download_in_bytes_by_scan_pid[scan_pid] + del self.raw_size_of_download_in_bytes_by_scan_pid[scan_pid] del self.photos_downloaded[scan_pid] del self.videos_downloaded[scan_pid] del self.files_downloaded[scan_pid] @@ -166,7 +195,7 @@ class DownloadTracker: del self.warnings[scan_pid] def purge_all(self): - self.__init__() + self._refresh_values() @@ -241,9 +270,9 @@ class TimeRemaining: t.time_mark = time.time() self.times[scan_pid] = t - def update(self, scan_pid, total_size): + def update(self, scan_pid, bytes_downloaded): if scan_pid in self.times: - self.times[scan_pid].downloaded = total_size + self.times[scan_pid].downloaded += bytes_downloaded now = time.time() tm = self.times[scan_pid].time_mark amt_time = now - tm |