summaryrefslogtreecommitdiff
path: root/engine/SCons/Node
diff options
context:
space:
mode:
Diffstat (limited to 'engine/SCons/Node')
-rw-r--r--engine/SCons/Node/Alias.py2
-rw-r--r--engine/SCons/Node/FS.py146
-rw-r--r--engine/SCons/Node/Python.py2
-rw-r--r--engine/SCons/Node/__init__.py98
4 files changed, 147 insertions, 101 deletions
diff --git a/engine/SCons/Node/Alias.py b/engine/SCons/Node/Alias.py
index 8fbeef7..a9defb6 100644
--- a/engine/SCons/Node/Alias.py
+++ b/engine/SCons/Node/Alias.py
@@ -30,7 +30,7 @@ This creates a hash of global Aliases (dummy targets).
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#
-__revision__ = "src/engine/SCons/Node/Alias.py a56bbd8c09fb219ab8a9673330ffcd55279219d0 2019-03-26 23:16:31 bdeegan"
+__revision__ = "src/engine/SCons/Node/Alias.py e724ae812eb96f4858a132f5b8c769724744faf6 2019-07-21 00:04:47 bdeegan"
import collections
diff --git a/engine/SCons/Node/FS.py b/engine/SCons/Node/FS.py
index be9f7fd..21ccecc 100644
--- a/engine/SCons/Node/FS.py
+++ b/engine/SCons/Node/FS.py
@@ -33,7 +33,7 @@ that can be used by scripts or modules looking for the canonical default.
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
from __future__ import print_function
-__revision__ = "src/engine/SCons/Node/FS.py a56bbd8c09fb219ab8a9673330ffcd55279219d0 2019-03-26 23:16:31 bdeegan"
+__revision__ = "src/engine/SCons/Node/FS.py e724ae812eb96f4858a132f5b8c769724744faf6 2019-07-21 00:04:47 bdeegan"
import fnmatch
import os
@@ -57,7 +57,6 @@ import SCons.Util
import SCons.Warnings
from SCons.Debug import Trace
-from . import DeciderNeedsNode
print_duplicate = 0
@@ -282,7 +281,7 @@ def set_duplicate(duplicate):
'copy' : _copy_func
}
- if not duplicate in Valid_Duplicates:
+ if duplicate not in Valid_Duplicates:
raise SCons.Errors.InternalError("The argument of set_duplicate "
"should be in Valid_Duplicates")
global Link_Funcs
@@ -480,7 +479,7 @@ class EntryProxy(SCons.Util.Proxy):
return SCons.Subst.SpecialAttrWrapper(r, entry.name + "_posix")
def __get_windows_path(self):
- """Return the path with \ as the path separator,
+ r"""Return the path with \ as the path separator,
regardless of platform."""
if OS_SEP == '\\':
return self
@@ -531,7 +530,7 @@ class EntryProxy(SCons.Util.Proxy):
except KeyError:
try:
attr = SCons.Util.Proxy.__getattr__(self, name)
- except AttributeError as e:
+ except AttributeError:
# Raise our own AttributeError subclass with an
# overridden __str__() method that identifies the
# name of the entry that caused the exception.
@@ -699,13 +698,13 @@ class Base(SCons.Node.Node):
@SCons.Memoize.CountMethodCall
def stat(self):
- try:
+ try:
return self._memo['stat']
- except KeyError:
+ except KeyError:
pass
- try:
+ try:
result = self.fs.stat(self.get_abspath())
- except os.error:
+ except os.error:
result = None
self._memo['stat'] = result
@@ -719,16 +718,16 @@ class Base(SCons.Node.Node):
def getmtime(self):
st = self.stat()
- if st:
+ if st:
return st[stat.ST_MTIME]
- else:
+ else:
return None
def getsize(self):
st = self.stat()
- if st:
+ if st:
return st[stat.ST_SIZE]
- else:
+ else:
return None
def isdir(self):
@@ -1418,7 +1417,7 @@ class FS(LocalFS):
self.Top.addRepository(d)
def PyPackageDir(self, modulename):
- """Locate the directory of a given python module name
+ r"""Locate the directory of a given python module name
For example scons might resolve to
Windows: C:\Python27\Lib\site-packages\scons-2.5.1
@@ -1689,7 +1688,7 @@ class Dir(Base):
return result
def addRepository(self, dir):
- if dir != self and not dir in self.repositories:
+ if dir != self and dir not in self.repositories:
self.repositories.append(dir)
dir._tpath = '.'
self.__clearRepositoryCache()
@@ -1729,7 +1728,7 @@ class Dir(Base):
if self is other:
result = '.'
- elif not other in self._path_elements:
+ elif other not in self._path_elements:
try:
other_dir = other.get_dir()
except AttributeError:
@@ -2261,7 +2260,7 @@ class RootDir(Dir):
this directory.
"""
- __slots__ = ['_lookupDict']
+ __slots__ = ('_lookupDict', )
def __init__(self, drive, fs):
if SCons.Debug.track_instances: logInstanceCreation(self, 'Node.FS.RootDir')
@@ -2467,7 +2466,7 @@ class FileNodeInfo(SCons.Node.NodeInfoBase):
"""
state = getattr(self, '__dict__', {}).copy()
for obj in type(self).mro():
- for name in getattr(obj,'__slots__',()):
+ for name in getattr(obj, '__slots__', ()):
if hasattr(self, name):
state[name] = getattr(self, name)
@@ -2511,7 +2510,7 @@ class FileBuildInfo(SCons.Node.BuildInfoBase):
or count of any of these could yield writing wrong csig, and then false positive
rebuilds
"""
- __slots__ = ('dependency_map')
+ __slots__ = ['dependency_map', ]
current_version_id = 2
def __setattr__(self, key, value):
@@ -3283,14 +3282,14 @@ class File(Base):
self._memo['changed'] = has_changed
return has_changed
- def changed_content(self, target, prev_ni):
+ def changed_content(self, target, prev_ni, repo_node=None):
cur_csig = self.get_csig()
try:
return cur_csig != prev_ni.csig
except AttributeError:
return 1
- def changed_state(self, target, prev_ni):
+ def changed_state(self, target, prev_ni, repo_node=None):
return self.state != SCons.Node.up_to_date
@@ -3317,13 +3316,26 @@ class File(Base):
len(binfo.bimplicitsigs)) == 0:
return {}
-
- # store this info so we can avoid regenerating it.
- binfo.dependency_map = { str(child):signature for child, signature in zip(chain(binfo.bsources, binfo.bdepends, binfo.bimplicit),
+ binfo.dependency_map = { child:signature for child, signature in zip(chain(binfo.bsources, binfo.bdepends, binfo.bimplicit),
chain(binfo.bsourcesigs, binfo.bdependsigs, binfo.bimplicitsigs))}
return binfo.dependency_map
+ # @profile
+ def _add_strings_to_dependency_map(self, dmap):
+ """
+ In the case comparing node objects isn't sufficient, we'll add the strings for the nodes to the dependency map
+ :return:
+ """
+
+ first_string = str(next(iter(dmap)))
+
+ # print("DMAP:%s"%id(dmap))
+ if first_string not in dmap:
+ string_dict = {str(child): signature for child, signature in dmap.items()}
+ dmap.update(string_dict)
+ return dmap
+
def _get_previous_signatures(self, dmap):
"""
Return a list of corresponding csigs from previous
@@ -3342,37 +3354,62 @@ class File(Base):
if len(dmap) == 0:
if MD5_TIMESTAMP_DEBUG: print("Nothing dmap shortcutting")
return None
+ elif MD5_TIMESTAMP_DEBUG: print("len(dmap):%d"%len(dmap))
+
- if MD5_TIMESTAMP_DEBUG: print("len(dmap):%d"%len(dmap))
- # First try the simple name for node
- c_str = str(self)
- if MD5_TIMESTAMP_DEBUG: print("Checking :%s"%c_str)
- df = dmap.get(c_str, None)
+ # First try retrieving via Node
+ if MD5_TIMESTAMP_DEBUG: print("Checking if self is in map:%s id:%s type:%s"%(str(self), id(self), type(self)))
+ df = dmap.get(self, False)
if df:
return df
-
+
+ # Now check if self's repository file is in map.
+ rf = self.rfile()
+ if MD5_TIMESTAMP_DEBUG: print("Checking if self.rfile is in map:%s id:%s type:%s"%(str(rf), id(rf), type(rf)))
+ rfm = dmap.get(rf, False)
+ if rfm:
+ return rfm
+
+ # get default string for node and then also string swapping os.altsep for os.sep (/ for \)
+ c_strs = [str(self)]
+
if os.altsep:
- c_str = c_str.replace(os.sep, os.altsep)
- df = dmap.get(c_str, None)
- if MD5_TIMESTAMP_DEBUG: print("-->%s"%df)
+ c_strs.append(c_strs[0].replace(os.sep, os.altsep))
+
+ # In some cases the dependency_maps' keys are already strings check.
+ # Check if either string is now in dmap.
+ for s in c_strs:
+ if MD5_TIMESTAMP_DEBUG: print("Checking if str(self) is in map :%s" % s)
+ df = dmap.get(s, False)
+ if df:
+ return df
+
+ # Strings don't exist in map, add them and try again
+ # If there are no strings in this dmap, then add them.
+ # This may not be necessary, we could walk the nodes in the dmap and check each string
+ # rather than adding ALL the strings to dmap. In theory that would be n/2 vs 2n str() calls on node
+ # if not dmap.has_strings:
+ dmap = self._add_strings_to_dependency_map(dmap)
+
+ # In some cases the dependency_maps' keys are already strings check.
+ # Check if either string is now in dmap.
+ for s in c_strs:
+ if MD5_TIMESTAMP_DEBUG: print("Checking if str(self) is in map (now with strings) :%s" % s)
+ df = dmap.get(s, False)
if df:
return df
+ # Lastly use nodes get_path() to generate string and see if that's in dmap
if not df:
try:
# this should yield a path which matches what's in the sconsign
c_str = self.get_path()
- df = dmap.get(c_str, None)
- if MD5_TIMESTAMP_DEBUG: print("-->%s"%df)
- if df:
- return df
-
if os.altsep:
c_str = c_str.replace(os.sep, os.altsep)
- df = dmap.get(c_str, None)
- if MD5_TIMESTAMP_DEBUG: print("-->%s"%df)
- if df:
- return df
+
+ if MD5_TIMESTAMP_DEBUG: print("Checking if self.get_path is in map (now with strings) :%s" % s)
+
+ df = dmap.get(c_str, None)
except AttributeError as e:
raise FileBuildInfoFileToCsigMappingError("No mapping from file name to content signature for :%s"%c_str)
@@ -3387,21 +3424,17 @@ class File(Base):
file and just copy the prev_ni provided. If the prev_ni
is wrong. It will propagate it.
See: https://github.com/SCons/scons/issues/2980
-
+
Args:
self - dependency
target - target
prev_ni - The NodeInfo object loaded from previous builds .sconsign
- node - Node instance. This is the only changed* function which requires
- node to function. So if we detect that it's not passed.
- we throw DeciderNeedsNode, and caller should handle this and pass node.
+ node - Node instance. Check this node for file existence/timestamp
+ if specified.
- Returns:
+ Returns:
Boolean - Indicates if node(File) has changed.
"""
- if node is None:
- # We need required node argument to get BuildInfo to function
- raise DeciderNeedsNode(self.changed_timestamp_then_content)
# Now get sconsign name -> csig map and then get proper prev_ni if possible
bi = node.get_stored_info().binfo
@@ -3433,7 +3466,6 @@ class File(Base):
print("Mismatch self.changed_timestamp_match(%s, prev_ni) old:%s new:%s"%(str(target), old, new))
new_prev_ni = self._get_previous_signatures(dependency_map)
-
if not new:
try:
# NOTE: We're modifying the current node's csig in a query.
@@ -3443,13 +3475,13 @@ class File(Base):
return False
return self.changed_content(target, new_prev_ni)
- def changed_timestamp_newer(self, target, prev_ni):
+ def changed_timestamp_newer(self, target, prev_ni, repo_node=None):
try:
return self.get_timestamp() > target.get_timestamp()
except AttributeError:
return 1
- def changed_timestamp_match(self, target, prev_ni):
+ def changed_timestamp_match(self, target, prev_ni, repo_node=None):
"""
Return True if the timestamps don't match or if there is no previous timestamp
:param target:
@@ -3462,14 +3494,18 @@ class File(Base):
return 1
def is_up_to_date(self):
+ """Check for whether the Node is current
+ In all cases self is the target we're checking to see if it's up to date
+ """
+
T = 0
if T: Trace('is_up_to_date(%s):' % self)
if not self.exists():
if T: Trace(' not self.exists():')
- # The file doesn't exist locally...
+ # The file (always a target) doesn't exist locally...
r = self.rfile()
if r != self:
- # ...but there is one in a Repository...
+ # ...but there is one (always a target) in a Repository...
if not self.changed(r):
if T: Trace(' changed(%s):' % r)
# ...and it's even up-to-date...
diff --git a/engine/SCons/Node/Python.py b/engine/SCons/Node/Python.py
index d235de8..d338051 100644
--- a/engine/SCons/Node/Python.py
+++ b/engine/SCons/Node/Python.py
@@ -27,7 +27,7 @@ Python nodes.
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
#
-__revision__ = "src/engine/SCons/Node/Python.py a56bbd8c09fb219ab8a9673330ffcd55279219d0 2019-03-26 23:16:31 bdeegan"
+__revision__ = "src/engine/SCons/Node/Python.py e724ae812eb96f4858a132f5b8c769724744faf6 2019-07-21 00:04:47 bdeegan"
import SCons.Node
diff --git a/engine/SCons/Node/__init__.py b/engine/SCons/Node/__init__.py
index 889e7a4..5455bd6 100644
--- a/engine/SCons/Node/__init__.py
+++ b/engine/SCons/Node/__init__.py
@@ -43,13 +43,18 @@ from __future__ import print_function
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
-__revision__ = "src/engine/SCons/Node/__init__.py a56bbd8c09fb219ab8a9673330ffcd55279219d0 2019-03-26 23:16:31 bdeegan"
+__revision__ = "src/engine/SCons/Node/__init__.py e724ae812eb96f4858a132f5b8c769724744faf6 2019-07-21 00:04:47 bdeegan"
import os
import collections
import copy
from itertools import chain
+try:
+ from itertools import zip_longest
+except ImportError:
+ from itertools import izip_longest as zip_longest
+
import SCons.Debug
from SCons.Debug import logInstanceCreation
import SCons.Executor
@@ -103,9 +108,9 @@ implicit_deps_changed = 0
# A variable that can be set to an interface-specific function be called
# to annotate a Node with information about its creation.
-def do_nothing(node): pass
+def do_nothing_node(node): pass
-Annotate = do_nothing
+Annotate = do_nothing_node
# Gets set to 'True' if we're running in interactive mode. Is
# currently used to release parts of a target's info during
@@ -248,25 +253,10 @@ _target_from_source_map = {0 : target_from_source_none,
# used by it.
#
-
-class DeciderNeedsNode(Exception):
- """
- Indicate that the decider needs the node as well as the target and the dependency.
- Normally the node and the target are the same, but in the case of repository
- They may be different. Also the NodeInfo is retrieved from the node
- """
- def __init__(self, call_this_decider):
- """
- :param call_this_decider: to return the decider to call directly since deciders
- are called through several levels of indirection
- """
- self.decider = call_this_decider
-
-
#
# First, the single decider functions
#
-def changed_since_last_build_node(node, target, prev_ni):
+def changed_since_last_build_node(node, target, prev_ni, repo_node=None):
"""
Must be overridden in a specific subclass to return True if this
@@ -287,7 +277,7 @@ def changed_since_last_build_node(node, target, prev_ni):
raise NotImplementedError
-def changed_since_last_build_alias(node, target, prev_ni):
+def changed_since_last_build_alias(node, target, prev_ni, repo_node=None):
cur_csig = node.get_csig()
try:
return cur_csig != prev_ni.csig
@@ -295,24 +285,24 @@ def changed_since_last_build_alias(node, target, prev_ni):
return 1
-def changed_since_last_build_entry(node, target, prev_ni):
+def changed_since_last_build_entry(node, target, prev_ni, repo_node=None):
node.disambiguate()
- return _decider_map[node.changed_since_last_build](node, target, prev_ni)
+ return _decider_map[node.changed_since_last_build](node, target, prev_ni, repo_node)
-def changed_since_last_build_state_changed(node, target, prev_ni):
+def changed_since_last_build_state_changed(node, target, prev_ni, repo_node=None):
return node.state != SCons.Node.up_to_date
-def decide_source(node, target, prev_ni):
- return target.get_build_env().decide_source(node, target, prev_ni)
+def decide_source(node, target, prev_ni, repo_node=None):
+ return target.get_build_env().decide_source(node, target, prev_ni, repo_node)
-def decide_target(node, target, prev_ni):
- return target.get_build_env().decide_target(node, target, prev_ni)
+def decide_target(node, target, prev_ni, repo_node=None):
+ return target.get_build_env().decide_target(node, target, prev_ni, repo_node)
-def changed_since_last_build_python(node, target, prev_ni):
+def changed_since_last_build_python(node, target, prev_ni, repo_node=None):
cur_csig = node.get_csig()
try:
return cur_csig != prev_ni.csig
@@ -529,6 +519,7 @@ class Node(object, with_metaclass(NoSlotsPyPy)):
__slots__ = ['sources',
'sources_set',
+ 'target_peers',
'_specific_sources',
'depends',
'depends_set',
@@ -784,6 +775,25 @@ class Node(object, with_metaclass(NoSlotsPyPy)):
for parent in self.waiting_parents:
parent.implicit = None
+ # Handle issue where builder emits more than one target and
+ # the source file for the builder is generated.
+ # in that case only the first target was getting it's .implicit
+ # cleared when the source file is built (second scan).
+ # leaving only partial implicits from scan before source file is generated
+ # typically the compiler only. Then scanned files are appended
+ # This is persisted to sconsign and rebuild causes false rebuilds
+ # because the ordering of the implicit list then changes to what it
+ # should have been.
+ # This is at least the following bugs
+ # https://github.com/SCons/scons/issues/2811
+ # https://jira.mongodb.org/browse/SERVER-33111
+ try:
+ for peer in parent.target_peers:
+ peer.implicit = None
+ except AttributeError:
+ pass
+
+
self.clear()
if self.pseudo:
@@ -1160,7 +1170,7 @@ class Node(object, with_metaclass(NoSlotsPyPy)):
binfo.bactsig = SCons.Util.MD5signature(executor.get_contents())
if self._specific_sources:
- sources = [s for s in self.sources if not s in ignore_set]
+ sources = [s for s in self.sources if s not in ignore_set]
else:
sources = executor.get_unignored_sources(self, self.ignore)
@@ -1480,17 +1490,11 @@ class Node(object, with_metaclass(NoSlotsPyPy)):
result = True
for child, prev_ni in zip(children, then):
- try:
- if _decider_map[child.changed_since_last_build](child, self, prev_ni):
- if t: Trace(': %s changed' % child)
- result = True
- except DeciderNeedsNode as e:
- if e.decider(self, prev_ni, node=node):
- if t: Trace(': %s changed' % child)
- result = True
+ if _decider_map[child.changed_since_last_build](child, self, prev_ni, node):
+ if t: Trace(': %s changed' % child)
+ result = True
if self.has_builder():
- import SCons.Util
contents = self.get_executor().get_contents()
newsig = SCons.Util.MD5signature(contents)
if bi.bactsig != newsig:
@@ -1647,14 +1651,14 @@ class Node(object, with_metaclass(NoSlotsPyPy)):
lines = []
- removed = [x for x in old_bkids if not x in new_bkids]
+ removed = [x for x in old_bkids if x not in new_bkids]
if removed:
removed = [stringify(r) for r in removed]
fmt = "`%s' is no longer a dependency\n"
lines.extend([fmt % s for s in removed])
for k in new_bkids:
- if not k in old_bkids:
+ if k not in old_bkids:
lines.append("`%s' is a new dependency\n" % stringify(k))
else:
try:
@@ -1666,9 +1670,16 @@ class Node(object, with_metaclass(NoSlotsPyPy)):
lines.append("`%s' changed\n" % stringify(k))
if len(lines) == 0 and old_bkids != new_bkids:
- lines.append("the dependency order changed:\n" +
- "%sold: %s\n" % (' '*15, list(map(stringify, old_bkids))) +
- "%snew: %s\n" % (' '*15, list(map(stringify, new_bkids))))
+ lines.append("the dependency order changed:\n")
+ lines.append("->Sources\n")
+ for (o,n) in zip_longest(old.bsources, new.bsources, fillvalue=None):
+ lines.append("Old:%s\tNew:%s\n"%(o,n))
+ lines.append("->Depends\n")
+ for (o,n) in zip_longest(old.bdepends, new.bdepends, fillvalue=None):
+ lines.append("Old:%s\tNew:%s\n"%(o,n))
+ lines.append("->Implicit\n")
+ for (o,n) in zip_longest(old.bimplicit, new.bimplicit, fillvalue=None):
+ lines.append("Old:%s\tNew:%s\n"%(o,n))
if len(lines) == 0:
def fmt_with_title(title, strlines):
@@ -1711,7 +1722,6 @@ class Walker(object):
This is depth-first, children are visited before the parent.
The Walker object can be initialized with any node, and
returns the next node on the descent with each get_next() call.
- 'kids_func' is an optional function that will be called to
get the children of a node instead of calling 'children'.
'cycle_func' is an optional function that will be called
when a cycle is detected.
generated by cgit v1.2.3 (git 2.25.1) at 2024-05-28 06:16:06 +0000