From 9c04223086bf606eaac6dc2848af80518210d823 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Frings-F=C3=BCrst?= Date: Tue, 23 Jul 2019 13:30:08 +0200 Subject: New upstream version 3.1.0 --- engine/SCons/Node/Alias.py | 2 +- engine/SCons/Node/FS.py | 146 ++++++++++++++++++++++++++---------------- engine/SCons/Node/Python.py | 2 +- engine/SCons/Node/__init__.py | 98 +++++++++++++++------------- 4 files changed, 147 insertions(+), 101 deletions(-) (limited to 'engine/SCons/Node') diff --git a/engine/SCons/Node/Alias.py b/engine/SCons/Node/Alias.py index 8fbeef7..a9defb6 100644 --- a/engine/SCons/Node/Alias.py +++ b/engine/SCons/Node/Alias.py @@ -30,7 +30,7 @@ This creates a hash of global Aliases (dummy targets). # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # -__revision__ = "src/engine/SCons/Node/Alias.py a56bbd8c09fb219ab8a9673330ffcd55279219d0 2019-03-26 23:16:31 bdeegan" +__revision__ = "src/engine/SCons/Node/Alias.py e724ae812eb96f4858a132f5b8c769724744faf6 2019-07-21 00:04:47 bdeegan" import collections diff --git a/engine/SCons/Node/FS.py b/engine/SCons/Node/FS.py index be9f7fd..21ccecc 100644 --- a/engine/SCons/Node/FS.py +++ b/engine/SCons/Node/FS.py @@ -33,7 +33,7 @@ that can be used by scripts or modules looking for the canonical default. # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. from __future__ import print_function -__revision__ = "src/engine/SCons/Node/FS.py a56bbd8c09fb219ab8a9673330ffcd55279219d0 2019-03-26 23:16:31 bdeegan" +__revision__ = "src/engine/SCons/Node/FS.py e724ae812eb96f4858a132f5b8c769724744faf6 2019-07-21 00:04:47 bdeegan" import fnmatch import os @@ -57,7 +57,6 @@ import SCons.Util import SCons.Warnings from SCons.Debug import Trace -from . import DeciderNeedsNode print_duplicate = 0 @@ -282,7 +281,7 @@ def set_duplicate(duplicate): 'copy' : _copy_func } - if not duplicate in Valid_Duplicates: + if duplicate not in Valid_Duplicates: raise SCons.Errors.InternalError("The argument of set_duplicate " "should be in Valid_Duplicates") global Link_Funcs @@ -480,7 +479,7 @@ class EntryProxy(SCons.Util.Proxy): return SCons.Subst.SpecialAttrWrapper(r, entry.name + "_posix") def __get_windows_path(self): - """Return the path with \ as the path separator, + r"""Return the path with \ as the path separator, regardless of platform.""" if OS_SEP == '\\': return self @@ -531,7 +530,7 @@ class EntryProxy(SCons.Util.Proxy): except KeyError: try: attr = SCons.Util.Proxy.__getattr__(self, name) - except AttributeError as e: + except AttributeError: # Raise our own AttributeError subclass with an # overridden __str__() method that identifies the # name of the entry that caused the exception. @@ -699,13 +698,13 @@ class Base(SCons.Node.Node): @SCons.Memoize.CountMethodCall def stat(self): - try: + try: return self._memo['stat'] - except KeyError: + except KeyError: pass - try: + try: result = self.fs.stat(self.get_abspath()) - except os.error: + except os.error: result = None self._memo['stat'] = result @@ -719,16 +718,16 @@ class Base(SCons.Node.Node): def getmtime(self): st = self.stat() - if st: + if st: return st[stat.ST_MTIME] - else: + else: return None def getsize(self): st = self.stat() - if st: + if st: return st[stat.ST_SIZE] - else: + else: return None def isdir(self): @@ -1418,7 +1417,7 @@ class FS(LocalFS): self.Top.addRepository(d) def PyPackageDir(self, modulename): - """Locate the directory of a given python module name + r"""Locate the directory of a given python module name For example scons might resolve to Windows: C:\Python27\Lib\site-packages\scons-2.5.1 @@ -1689,7 +1688,7 @@ class Dir(Base): return result def addRepository(self, dir): - if dir != self and not dir in self.repositories: + if dir != self and dir not in self.repositories: self.repositories.append(dir) dir._tpath = '.' self.__clearRepositoryCache() @@ -1729,7 +1728,7 @@ class Dir(Base): if self is other: result = '.' - elif not other in self._path_elements: + elif other not in self._path_elements: try: other_dir = other.get_dir() except AttributeError: @@ -2261,7 +2260,7 @@ class RootDir(Dir): this directory. """ - __slots__ = ['_lookupDict'] + __slots__ = ('_lookupDict', ) def __init__(self, drive, fs): if SCons.Debug.track_instances: logInstanceCreation(self, 'Node.FS.RootDir') @@ -2467,7 +2466,7 @@ class FileNodeInfo(SCons.Node.NodeInfoBase): """ state = getattr(self, '__dict__', {}).copy() for obj in type(self).mro(): - for name in getattr(obj,'__slots__',()): + for name in getattr(obj, '__slots__', ()): if hasattr(self, name): state[name] = getattr(self, name) @@ -2511,7 +2510,7 @@ class FileBuildInfo(SCons.Node.BuildInfoBase): or count of any of these could yield writing wrong csig, and then false positive rebuilds """ - __slots__ = ('dependency_map') + __slots__ = ['dependency_map', ] current_version_id = 2 def __setattr__(self, key, value): @@ -3283,14 +3282,14 @@ class File(Base): self._memo['changed'] = has_changed return has_changed - def changed_content(self, target, prev_ni): + def changed_content(self, target, prev_ni, repo_node=None): cur_csig = self.get_csig() try: return cur_csig != prev_ni.csig except AttributeError: return 1 - def changed_state(self, target, prev_ni): + def changed_state(self, target, prev_ni, repo_node=None): return self.state != SCons.Node.up_to_date @@ -3317,13 +3316,26 @@ class File(Base): len(binfo.bimplicitsigs)) == 0: return {} - - # store this info so we can avoid regenerating it. - binfo.dependency_map = { str(child):signature for child, signature in zip(chain(binfo.bsources, binfo.bdepends, binfo.bimplicit), + binfo.dependency_map = { child:signature for child, signature in zip(chain(binfo.bsources, binfo.bdepends, binfo.bimplicit), chain(binfo.bsourcesigs, binfo.bdependsigs, binfo.bimplicitsigs))} return binfo.dependency_map + # @profile + def _add_strings_to_dependency_map(self, dmap): + """ + In the case comparing node objects isn't sufficient, we'll add the strings for the nodes to the dependency map + :return: + """ + + first_string = str(next(iter(dmap))) + + # print("DMAP:%s"%id(dmap)) + if first_string not in dmap: + string_dict = {str(child): signature for child, signature in dmap.items()} + dmap.update(string_dict) + return dmap + def _get_previous_signatures(self, dmap): """ Return a list of corresponding csigs from previous @@ -3342,37 +3354,62 @@ class File(Base): if len(dmap) == 0: if MD5_TIMESTAMP_DEBUG: print("Nothing dmap shortcutting") return None + elif MD5_TIMESTAMP_DEBUG: print("len(dmap):%d"%len(dmap)) + - if MD5_TIMESTAMP_DEBUG: print("len(dmap):%d"%len(dmap)) - # First try the simple name for node - c_str = str(self) - if MD5_TIMESTAMP_DEBUG: print("Checking :%s"%c_str) - df = dmap.get(c_str, None) + # First try retrieving via Node + if MD5_TIMESTAMP_DEBUG: print("Checking if self is in map:%s id:%s type:%s"%(str(self), id(self), type(self))) + df = dmap.get(self, False) if df: return df - + + # Now check if self's repository file is in map. + rf = self.rfile() + if MD5_TIMESTAMP_DEBUG: print("Checking if self.rfile is in map:%s id:%s type:%s"%(str(rf), id(rf), type(rf))) + rfm = dmap.get(rf, False) + if rfm: + return rfm + + # get default string for node and then also string swapping os.altsep for os.sep (/ for \) + c_strs = [str(self)] + if os.altsep: - c_str = c_str.replace(os.sep, os.altsep) - df = dmap.get(c_str, None) - if MD5_TIMESTAMP_DEBUG: print("-->%s"%df) + c_strs.append(c_strs[0].replace(os.sep, os.altsep)) + + # In some cases the dependency_maps' keys are already strings check. + # Check if either string is now in dmap. + for s in c_strs: + if MD5_TIMESTAMP_DEBUG: print("Checking if str(self) is in map :%s" % s) + df = dmap.get(s, False) + if df: + return df + + # Strings don't exist in map, add them and try again + # If there are no strings in this dmap, then add them. + # This may not be necessary, we could walk the nodes in the dmap and check each string + # rather than adding ALL the strings to dmap. In theory that would be n/2 vs 2n str() calls on node + # if not dmap.has_strings: + dmap = self._add_strings_to_dependency_map(dmap) + + # In some cases the dependency_maps' keys are already strings check. + # Check if either string is now in dmap. + for s in c_strs: + if MD5_TIMESTAMP_DEBUG: print("Checking if str(self) is in map (now with strings) :%s" % s) + df = dmap.get(s, False) if df: return df + # Lastly use nodes get_path() to generate string and see if that's in dmap if not df: try: # this should yield a path which matches what's in the sconsign c_str = self.get_path() - df = dmap.get(c_str, None) - if MD5_TIMESTAMP_DEBUG: print("-->%s"%df) - if df: - return df - if os.altsep: c_str = c_str.replace(os.sep, os.altsep) - df = dmap.get(c_str, None) - if MD5_TIMESTAMP_DEBUG: print("-->%s"%df) - if df: - return df + + if MD5_TIMESTAMP_DEBUG: print("Checking if self.get_path is in map (now with strings) :%s" % s) + + df = dmap.get(c_str, None) except AttributeError as e: raise FileBuildInfoFileToCsigMappingError("No mapping from file name to content signature for :%s"%c_str) @@ -3387,21 +3424,17 @@ class File(Base): file and just copy the prev_ni provided. If the prev_ni is wrong. It will propagate it. See: https://github.com/SCons/scons/issues/2980 - + Args: self - dependency target - target prev_ni - The NodeInfo object loaded from previous builds .sconsign - node - Node instance. This is the only changed* function which requires - node to function. So if we detect that it's not passed. - we throw DeciderNeedsNode, and caller should handle this and pass node. + node - Node instance. Check this node for file existence/timestamp + if specified. - Returns: + Returns: Boolean - Indicates if node(File) has changed. """ - if node is None: - # We need required node argument to get BuildInfo to function - raise DeciderNeedsNode(self.changed_timestamp_then_content) # Now get sconsign name -> csig map and then get proper prev_ni if possible bi = node.get_stored_info().binfo @@ -3433,7 +3466,6 @@ class File(Base): print("Mismatch self.changed_timestamp_match(%s, prev_ni) old:%s new:%s"%(str(target), old, new)) new_prev_ni = self._get_previous_signatures(dependency_map) - if not new: try: # NOTE: We're modifying the current node's csig in a query. @@ -3443,13 +3475,13 @@ class File(Base): return False return self.changed_content(target, new_prev_ni) - def changed_timestamp_newer(self, target, prev_ni): + def changed_timestamp_newer(self, target, prev_ni, repo_node=None): try: return self.get_timestamp() > target.get_timestamp() except AttributeError: return 1 - def changed_timestamp_match(self, target, prev_ni): + def changed_timestamp_match(self, target, prev_ni, repo_node=None): """ Return True if the timestamps don't match or if there is no previous timestamp :param target: @@ -3462,14 +3494,18 @@ class File(Base): return 1 def is_up_to_date(self): + """Check for whether the Node is current + In all cases self is the target we're checking to see if it's up to date + """ + T = 0 if T: Trace('is_up_to_date(%s):' % self) if not self.exists(): if T: Trace(' not self.exists():') - # The file doesn't exist locally... + # The file (always a target) doesn't exist locally... r = self.rfile() if r != self: - # ...but there is one in a Repository... + # ...but there is one (always a target) in a Repository... if not self.changed(r): if T: Trace(' changed(%s):' % r) # ...and it's even up-to-date... diff --git a/engine/SCons/Node/Python.py b/engine/SCons/Node/Python.py index d235de8..d338051 100644 --- a/engine/SCons/Node/Python.py +++ b/engine/SCons/Node/Python.py @@ -27,7 +27,7 @@ Python nodes. # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. # -__revision__ = "src/engine/SCons/Node/Python.py a56bbd8c09fb219ab8a9673330ffcd55279219d0 2019-03-26 23:16:31 bdeegan" +__revision__ = "src/engine/SCons/Node/Python.py e724ae812eb96f4858a132f5b8c769724744faf6 2019-07-21 00:04:47 bdeegan" import SCons.Node diff --git a/engine/SCons/Node/__init__.py b/engine/SCons/Node/__init__.py index 889e7a4..5455bd6 100644 --- a/engine/SCons/Node/__init__.py +++ b/engine/SCons/Node/__init__.py @@ -43,13 +43,18 @@ from __future__ import print_function # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -__revision__ = "src/engine/SCons/Node/__init__.py a56bbd8c09fb219ab8a9673330ffcd55279219d0 2019-03-26 23:16:31 bdeegan" +__revision__ = "src/engine/SCons/Node/__init__.py e724ae812eb96f4858a132f5b8c769724744faf6 2019-07-21 00:04:47 bdeegan" import os import collections import copy from itertools import chain +try: + from itertools import zip_longest +except ImportError: + from itertools import izip_longest as zip_longest + import SCons.Debug from SCons.Debug import logInstanceCreation import SCons.Executor @@ -103,9 +108,9 @@ implicit_deps_changed = 0 # A variable that can be set to an interface-specific function be called # to annotate a Node with information about its creation. -def do_nothing(node): pass +def do_nothing_node(node): pass -Annotate = do_nothing +Annotate = do_nothing_node # Gets set to 'True' if we're running in interactive mode. Is # currently used to release parts of a target's info during @@ -248,25 +253,10 @@ _target_from_source_map = {0 : target_from_source_none, # used by it. # - -class DeciderNeedsNode(Exception): - """ - Indicate that the decider needs the node as well as the target and the dependency. - Normally the node and the target are the same, but in the case of repository - They may be different. Also the NodeInfo is retrieved from the node - """ - def __init__(self, call_this_decider): - """ - :param call_this_decider: to return the decider to call directly since deciders - are called through several levels of indirection - """ - self.decider = call_this_decider - - # # First, the single decider functions # -def changed_since_last_build_node(node, target, prev_ni): +def changed_since_last_build_node(node, target, prev_ni, repo_node=None): """ Must be overridden in a specific subclass to return True if this @@ -287,7 +277,7 @@ def changed_since_last_build_node(node, target, prev_ni): raise NotImplementedError -def changed_since_last_build_alias(node, target, prev_ni): +def changed_since_last_build_alias(node, target, prev_ni, repo_node=None): cur_csig = node.get_csig() try: return cur_csig != prev_ni.csig @@ -295,24 +285,24 @@ def changed_since_last_build_alias(node, target, prev_ni): return 1 -def changed_since_last_build_entry(node, target, prev_ni): +def changed_since_last_build_entry(node, target, prev_ni, repo_node=None): node.disambiguate() - return _decider_map[node.changed_since_last_build](node, target, prev_ni) + return _decider_map[node.changed_since_last_build](node, target, prev_ni, repo_node) -def changed_since_last_build_state_changed(node, target, prev_ni): +def changed_since_last_build_state_changed(node, target, prev_ni, repo_node=None): return node.state != SCons.Node.up_to_date -def decide_source(node, target, prev_ni): - return target.get_build_env().decide_source(node, target, prev_ni) +def decide_source(node, target, prev_ni, repo_node=None): + return target.get_build_env().decide_source(node, target, prev_ni, repo_node) -def decide_target(node, target, prev_ni): - return target.get_build_env().decide_target(node, target, prev_ni) +def decide_target(node, target, prev_ni, repo_node=None): + return target.get_build_env().decide_target(node, target, prev_ni, repo_node) -def changed_since_last_build_python(node, target, prev_ni): +def changed_since_last_build_python(node, target, prev_ni, repo_node=None): cur_csig = node.get_csig() try: return cur_csig != prev_ni.csig @@ -529,6 +519,7 @@ class Node(object, with_metaclass(NoSlotsPyPy)): __slots__ = ['sources', 'sources_set', + 'target_peers', '_specific_sources', 'depends', 'depends_set', @@ -784,6 +775,25 @@ class Node(object, with_metaclass(NoSlotsPyPy)): for parent in self.waiting_parents: parent.implicit = None + # Handle issue where builder emits more than one target and + # the source file for the builder is generated. + # in that case only the first target was getting it's .implicit + # cleared when the source file is built (second scan). + # leaving only partial implicits from scan before source file is generated + # typically the compiler only. Then scanned files are appended + # This is persisted to sconsign and rebuild causes false rebuilds + # because the ordering of the implicit list then changes to what it + # should have been. + # This is at least the following bugs + # https://github.com/SCons/scons/issues/2811 + # https://jira.mongodb.org/browse/SERVER-33111 + try: + for peer in parent.target_peers: + peer.implicit = None + except AttributeError: + pass + + self.clear() if self.pseudo: @@ -1160,7 +1170,7 @@ class Node(object, with_metaclass(NoSlotsPyPy)): binfo.bactsig = SCons.Util.MD5signature(executor.get_contents()) if self._specific_sources: - sources = [s for s in self.sources if not s in ignore_set] + sources = [s for s in self.sources if s not in ignore_set] else: sources = executor.get_unignored_sources(self, self.ignore) @@ -1480,17 +1490,11 @@ class Node(object, with_metaclass(NoSlotsPyPy)): result = True for child, prev_ni in zip(children, then): - try: - if _decider_map[child.changed_since_last_build](child, self, prev_ni): - if t: Trace(': %s changed' % child) - result = True - except DeciderNeedsNode as e: - if e.decider(self, prev_ni, node=node): - if t: Trace(': %s changed' % child) - result = True + if _decider_map[child.changed_since_last_build](child, self, prev_ni, node): + if t: Trace(': %s changed' % child) + result = True if self.has_builder(): - import SCons.Util contents = self.get_executor().get_contents() newsig = SCons.Util.MD5signature(contents) if bi.bactsig != newsig: @@ -1647,14 +1651,14 @@ class Node(object, with_metaclass(NoSlotsPyPy)): lines = [] - removed = [x for x in old_bkids if not x in new_bkids] + removed = [x for x in old_bkids if x not in new_bkids] if removed: removed = [stringify(r) for r in removed] fmt = "`%s' is no longer a dependency\n" lines.extend([fmt % s for s in removed]) for k in new_bkids: - if not k in old_bkids: + if k not in old_bkids: lines.append("`%s' is a new dependency\n" % stringify(k)) else: try: @@ -1666,9 +1670,16 @@ class Node(object, with_metaclass(NoSlotsPyPy)): lines.append("`%s' changed\n" % stringify(k)) if len(lines) == 0 and old_bkids != new_bkids: - lines.append("the dependency order changed:\n" + - "%sold: %s\n" % (' '*15, list(map(stringify, old_bkids))) + - "%snew: %s\n" % (' '*15, list(map(stringify, new_bkids)))) + lines.append("the dependency order changed:\n") + lines.append("->Sources\n") + for (o,n) in zip_longest(old.bsources, new.bsources, fillvalue=None): + lines.append("Old:%s\tNew:%s\n"%(o,n)) + lines.append("->Depends\n") + for (o,n) in zip_longest(old.bdepends, new.bdepends, fillvalue=None): + lines.append("Old:%s\tNew:%s\n"%(o,n)) + lines.append("->Implicit\n") + for (o,n) in zip_longest(old.bimplicit, new.bimplicit, fillvalue=None): + lines.append("Old:%s\tNew:%s\n"%(o,n)) if len(lines) == 0: def fmt_with_title(title, strlines): @@ -1711,7 +1722,6 @@ class Walker(object): This is depth-first, children are visited before the parent. The Walker object can be initialized with any node, and returns the next node on the descent with each get_next() call. - 'kids_func' is an optional function that will be called to get the children of a node instead of calling 'children'. 'cycle_func' is an optional function that will be called when a cycle is detected. -- cgit v1.2.3