From 19712e5025e3cf6a33fccd0738f04e018d55025f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Frings-F=C3=BCrst?= Date: Fri, 12 Jul 2019 17:48:12 +0200 Subject: New upstream version 3.0.5 --- engine/SCons/CacheDir.py | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) (limited to 'engine/SCons/CacheDir.py') diff --git a/engine/SCons/CacheDir.py b/engine/SCons/CacheDir.py index b15d715..81c74f0 100644 --- a/engine/SCons/CacheDir.py +++ b/engine/SCons/CacheDir.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2001 - 2017 The SCons Foundation +# Copyright (c) 2001 - 2019 The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -21,12 +21,13 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # -__revision__ = "src/engine/SCons/CacheDir.py 74b2c53bc42290e911b334a6b44f187da698a668 2017/11/14 13:16:53 bdbaddog" +__revision__ = "src/engine/SCons/CacheDir.py a56bbd8c09fb219ab8a9673330ffcd55279219d0 2019-03-26 23:16:31 bdeegan" __doc__ = """ CacheDir support """ +import hashlib import json import os import stat @@ -55,6 +56,10 @@ def CacheRetrieveFunc(target, source, env): fs.symlink(fs.readlink(cachefile), t.get_internal_path()) else: env.copy_from_cache(cachefile, t.get_internal_path()) + try: + os.utime(cachefile, None) + except OSError: + pass st = fs.stat(cachefile) fs.chmod(t.get_internal_path(), stat.S_IMODE(st[stat.ST_MODE]) | stat.S_IWRITE) return 0 @@ -134,12 +139,6 @@ warned = dict() class CacheDir(object): def __init__(self, path): - try: - import hashlib - except ImportError: - msg = "No hashlib or MD5 module available, CacheDir() not supported" - SCons.Warnings.warn(SCons.Warnings.NoMD5ModuleWarning, msg) - path = None self.path = path self.current_cache_debug = None self.debugFP = None @@ -222,7 +221,9 @@ class CacheDir(object): return None, None sig = node.get_cachedir_bsig() + subdir = sig[:self.config['prefix_len']].upper() + dir = os.path.join(self.path, subdir) return dir, os.path.join(dir, sig) -- cgit v1.2.3 From 9c04223086bf606eaac6dc2848af80518210d823 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Frings-F=C3=BCrst?= Date: Tue, 23 Jul 2019 13:30:08 +0200 Subject: New upstream version 3.1.0 --- engine/SCons/CacheDir.py | 118 ++++++++++++++++++++++++++++++++++++++--------- 1 file changed, 97 insertions(+), 21 deletions(-) (limited to 'engine/SCons/CacheDir.py') diff --git a/engine/SCons/CacheDir.py b/engine/SCons/CacheDir.py index 81c74f0..c9e8ea7 100644 --- a/engine/SCons/CacheDir.py +++ b/engine/SCons/CacheDir.py @@ -21,7 +21,7 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # -__revision__ = "src/engine/SCons/CacheDir.py a56bbd8c09fb219ab8a9673330ffcd55279219d0 2019-03-26 23:16:31 bdeegan" +__revision__ = "src/engine/SCons/CacheDir.py e724ae812eb96f4858a132f5b8c769724744faf6 2019-07-21 00:04:47 bdeegan" __doc__ = """ CacheDir support @@ -35,6 +35,7 @@ import sys import SCons.Action import SCons.Warnings +from SCons.Util import PY3 cache_enabled = True cache_debug = False @@ -46,10 +47,12 @@ def CacheRetrieveFunc(target, source, env): t = target[0] fs = t.fs cd = env.get_CacheDir() + cd.requests += 1 cachedir, cachefile = cd.cachepath(t) if not fs.exists(cachefile): cd.CacheDebug('CacheRetrieve(%s): %s not in cache\n', t, cachefile) return 1 + cd.hits += 1 cd.CacheDebug('CacheRetrieve(%s): retrieving from %s\n', t, cachefile) if SCons.Action.execute_actions: if fs.islink(cachefile): @@ -111,7 +114,7 @@ def CachePushFunc(target, source, env): # has beaten us creating the directory. if not fs.isdir(cachedir): msg = errfmt % (str(target), cachefile) - raise SCons.Errors.EnvironmentError(msg) + raise SCons.Errors.SConsEnvironmentError(msg) try: if fs.islink(t.get_internal_path()): @@ -139,34 +142,97 @@ warned = dict() class CacheDir(object): def __init__(self, path): + """ + Initialize a CacheDir object. + + The cache configuration is stored in the object. It + is read from the config file in the supplied path if + one exists, if not the config file is created and + the default config is written, as well as saved in the object. + """ + self.requests = 0 + self.hits = 0 self.path = path self.current_cache_debug = None self.debugFP = None self.config = dict() if path is None: return - # See if there's a config file in the cache directory. If there is, - # use it. If there isn't, and the directory exists and isn't empty, - # produce a warning. If the directory doesn't exist or is empty, - # write a config file. + + if PY3: + self._readconfig3(path) + else: + self._readconfig2(path) + + + def _readconfig3(self, path): + """ + Python3 version of reading the cache config. + + If directory or config file do not exist, create. Take advantage + of Py3 capability in os.makedirs() and in file open(): just try + the operation and handle failure appropriately. + + Omit the check for old cache format, assume that's old enough + there will be none of those left to worry about. + + :param path: path to the cache directory + """ + config_file = os.path.join(path, 'config') + try: + os.makedirs(path, exist_ok=True) + except FileExistsError: + pass + except OSError: + msg = "Failed to create cache directory " + path + raise SCons.Errors.EnvironmentError(msg) + + try: + with open(config_file, 'x') as config: + self.config['prefix_len'] = 2 + try: + json.dump(self.config, config) + except Exception: + msg = "Failed to write cache configuration for " + path + raise SCons.Errors.EnvironmentError(msg) + except FileExistsError: + try: + with open(config_file) as config: + self.config = json.load(config) + except ValueError: + msg = "Failed to read cache configuration for " + path + raise SCons.Errors.EnvironmentError(msg) + + + def _readconfig2(self, path): + """ + Python2 version of reading cache config. + + See if there is a config file in the cache directory. If there is, + use it. If there isn't, and the directory exists and isn't empty, + produce a warning. If the directory does not exist or is empty, + write a config file. + + :param path: path to the cache directory + """ config_file = os.path.join(path, 'config') if not os.path.exists(config_file): - # A note: There is a race hazard here, if two processes start and + # A note: There is a race hazard here if two processes start and # attempt to create the cache directory at the same time. However, - # python doesn't really give you the option to do exclusive file - # creation (it doesn't even give you the option to error on opening - # an existing file for writing...). The ordering of events here - # as an attempt to alleviate this, on the basis that it's a pretty - # unlikely occurence (it'd require two builds with a brand new cache + # Python 2.x does not give you the option to do exclusive file + # creation (not even the option to error on opening an existing + # file for writing...). The ordering of events here is an attempt + # to alleviate this, on the basis that it's a pretty unlikely + # occurrence (would require two builds with a brand new cache # directory) - if os.path.isdir(path) and len(os.listdir(path)) != 0: + if os.path.isdir(path) and any(f != "config" for f in os.listdir(path)): self.config['prefix_len'] = 1 # When building the project I was testing this on, the warning # was output over 20 times. That seems excessive global warned if self.path not in warned: msg = "Please upgrade your cache by running " +\ - " scons-configure-cache.py " + self.path + "scons-configure-cache.py " + self.path SCons.Warnings.warn(SCons.Warnings.CacheVersionWarning, msg) warned[self.path] = True else: @@ -177,24 +243,24 @@ class CacheDir(object): # If someone else is trying to create the directory at # the same time as me, bad things will happen msg = "Failed to create cache directory " + path - raise SCons.Errors.EnvironmentError(msg) - + raise SCons.Errors.SConsEnvironmentError(msg) + self.config['prefix_len'] = 2 if not os.path.exists(config_file): try: with open(config_file, 'w') as config: json.dump(self.config, config) - except: + except Exception: msg = "Failed to write cache configuration for " + path - raise SCons.Errors.EnvironmentError(msg) + raise SCons.Errors.SConsEnvironmentError(msg) else: try: with open(config_file) as config: self.config = json.load(config) except ValueError: msg = "Failed to read cache configuration for " + path - raise SCons.Errors.EnvironmentError(msg) - + raise SCons.Errors.SConsEnvironmentError(msg) + def CacheDebug(self, fmt, target, cachefile): if cache_debug != self.current_cache_debug: @@ -207,9 +273,19 @@ class CacheDir(object): self.current_cache_debug = cache_debug if self.debugFP: self.debugFP.write(fmt % (target, os.path.split(cachefile)[1])) + self.debugFP.write("requests: %d, hits: %d, misses: %d, hit rate: %.2f%%\n" % + (self.requests, self.hits, self.misses, self.hit_ratio)) + + @property + def hit_ratio(self): + return (100.0 * self.hits / self.requests if self.requests > 0 else 100) + + @property + def misses(self): + return self.requests - self.hits def is_enabled(self): - return cache_enabled and not self.path is None + return cache_enabled and self.path is not None def is_readonly(self): return cache_readonly -- cgit v1.2.3 From 3af57a8e6a18986c41351da2447363ede8565402 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Frings-F=C3=BCrst?= Date: Sat, 10 Aug 2019 08:40:22 +0200 Subject: New upstream version 3.1.1 --- engine/SCons/CacheDir.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'engine/SCons/CacheDir.py') diff --git a/engine/SCons/CacheDir.py b/engine/SCons/CacheDir.py index c9e8ea7..a1133ae 100644 --- a/engine/SCons/CacheDir.py +++ b/engine/SCons/CacheDir.py @@ -21,7 +21,7 @@ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # -__revision__ = "src/engine/SCons/CacheDir.py e724ae812eb96f4858a132f5b8c769724744faf6 2019-07-21 00:04:47 bdeegan" +__revision__ = "src/engine/SCons/CacheDir.py 72ae09dc35ac2626f8ff711d8c4b30b6138e08e3 2019-08-08 14:50:06 bdeegan" __doc__ = """ CacheDir support @@ -33,6 +33,7 @@ import os import stat import sys +import SCons import SCons.Action import SCons.Warnings from SCons.Util import PY3 @@ -185,7 +186,7 @@ class CacheDir(object): pass except OSError: msg = "Failed to create cache directory " + path - raise SCons.Errors.EnvironmentError(msg) + raise SCons.Errors.SConsEnvironmentError(msg) try: with open(config_file, 'x') as config: @@ -194,14 +195,14 @@ class CacheDir(object): json.dump(self.config, config) except Exception: msg = "Failed to write cache configuration for " + path - raise SCons.Errors.EnvironmentError(msg) + raise SCons.Errors.SConsEnvironmentError(msg) except FileExistsError: try: with open(config_file) as config: self.config = json.load(config) except ValueError: msg = "Failed to read cache configuration for " + path - raise SCons.Errors.EnvironmentError(msg) + raise SCons.Errors.SConsEnvironmentError(msg) def _readconfig2(self, path): -- cgit v1.2.3