Merge branch 'release/debian/3.1.0+repack-1'debian/3.1.0+repack-1

1 files changed, 119 insertions, 101 deletions

diff --git a/bin/SConsDoc.py b/bin/SConsDoc.py
index cfb4e54..e44a8db 100644
--- a/bin/SConsDoc.py
+++ b/bin/SConsDoc.py
@@ -112,7 +112,6 @@ Tool example:
     </tool>
 """
 
-import imp
 import os.path
 import re
 import sys
@@ -156,8 +155,8 @@ if not has_etree:
                 except ImportError:
                     raise ImportError("Failed to import ElementTree from any known place")
 
-re_entity = re.compile("\&([^;]+);")
-re_entity_header = re.compile("<!DOCTYPE\s+sconsdoc\s+[^\]]+\]>")
+re_entity = re.compile(r"\&([^;]+);")
+re_entity_header = re.compile(r"<!DOCTYPE\s+sconsdoc\s+[^\]]+\]>")
 
 # Namespace for the SCons Docbook XSD
 dbxsd="http://www.scons.org/dbxsd/v1.0"
@@ -168,7 +167,7 @@ xsi = "http://www.w3.org/2001/XMLSchema-instance"
 
 # Header comment with copyright
 copyright_comment = """
-Copyright (c) 2001 - 2017 The SCons Foundation
+Copyright (c) 2001 - 2019 The SCons Foundation
 
 This file is processed by the bin/SConsDoc.py module.
 See its __doc__ string for a discussion of the format.
@@ -179,22 +178,21 @@ def isSConsXml(fpath):
         contains the default target namespace definition.
     """
     try:
-        f = open(fpath,'r')
-        content = f.read()
-        f.close()
+        with open(fpath,'r') as f:
+            content = f.read()
         if content.find('xmlns="%s"' % dbxsd) >= 0:
             return True
     except:
         pass
-    
-    return False 
+
+    return False
 
 def remove_entities(content):
     # Cut out entity inclusions
     content = re_entity_header.sub("", content, re.M)
     # Cut out entities themselves
     content = re_entity.sub(lambda match: match.group(1), content)
-    
+
     return content
 
 default_xsd = os.path.join('doc','xsd','scons.xsd')
@@ -202,11 +200,11 @@ default_xsd = os.path.join('doc','xsd','scons.xsd')
 ARG = "dbscons"
 
 class Libxml2ValidityHandler:
-    
+
     def __init__(self):
         self.errors = []
         self.warnings = []
-        
+
     def error(self, msg, data):
         if data != ARG:
             raise Exception("Error handler did not receive correct argument")
@@ -222,14 +220,14 @@ class DoctypeEntity:
     def __init__(self, name_, uri_):
         self.name = name_
         self.uri = uri_
-        
+
     def getEntityString(self):
         txt = """    <!ENTITY %(perc)s %(name)s SYSTEM "%(uri)s">
     %(perc)s%(name)s;
 """ % {'perc' : perc, 'name' : self.name, 'uri' : self.uri}
 
         return txt
-        
+
 class DoctypeDeclaration:
     def __init__(self, name_=None):
         self.name = name_
@@ -242,23 +240,23 @@ class DoctypeDeclaration:
             self.addEntity("functions-mod", "functions.mod")
             self.addEntity("tools-mod", "tools.mod")
             self.addEntity("variables-mod", "variables.mod")
-        
+
     def addEntity(self, name, uri):
         self.entries.append(DoctypeEntity(name, uri))
-        
+
     def createDoctype(self):
         content = '<!DOCTYPE %s [\n' % self.name
         for e in self.entries:
             content += e.getEntityString()
         content += ']>\n'
-        
+
         return content
 
 if not has_libxml2:
     class TreeFactory:
         def __init__(self):
             pass
-        
+
         def newNode(self, tag):
             return etree.Element(tag)
 
@@ -269,22 +267,22 @@ if not has_libxml2:
                 return etree.Element(tag, nsmap=NSMAP)
 
             return etree.Element(tag)
-        
+
         def copyNode(self, node):
             return copy.deepcopy(node)
-        
+
         def appendNode(self, parent, child):
             parent.append(child)
 
         def hasAttribute(self, node, att):
             return att in node.attrib
-        
+
         def getAttribute(self, node, att):
             return node.attrib[att]
-        
+
         def setAttribute(self, node, att, value):
             node.attrib[att] = value
-            
+
         def getText(self, root):
             return root.text
 
@@ -293,30 +291,27 @@ if not has_libxml2:
 
         def writeGenTree(self, root, fp):
             dt = DoctypeDeclaration()
-            fp.write(etree.tostring(root, xml_declaration=True, 
-                                    encoding="UTF-8", pretty_print=True, 
+            fp.write(etree.tostring(root, xml_declaration=True,
+                                    encoding="UTF-8", pretty_print=True,
                                     doctype=dt.createDoctype()))
 
         def writeTree(self, root, fpath):
-            fp = open(fpath, 'w')
-            fp.write(etree.tostring(root, xml_declaration=True, 
-                                    encoding="UTF-8", pretty_print=True))            
-            fp.close()
+            with open(fpath, 'w') as fp:
+                fp.write(etree.tostring(root, xml_declaration=True,
+                                        encoding="UTF-8", pretty_print=True))
 
         def prettyPrintFile(self, fpath):
-            fin = open(fpath,'r')
-            tree = etree.parse(fin)
-            pretty_content = etree.tostring(tree, pretty_print=True)
-            fin.close()
-    
-            fout = open(fpath,'w')
-            fout.write(pretty_content)
-            fout.close()
+            with open(fpath,'r') as fin:
+                tree = etree.parse(fin)
+                pretty_content = etree.tostring(tree, pretty_print=True)
+
+            with open(fpath,'w') as fout:
+                fout.write(pretty_content)
 
         def decorateWithHeader(self, root):
             root.attrib["{"+xsi+"}schemaLocation"] = "%s %s/scons.xsd" % (dbxsd, dbxsd)
             return root
-            
+
         def newXmlTree(self, root):
             """ Return a XML file tree with the correct namespaces set,
                 the element root as top entry and the given header comment.
@@ -325,7 +320,7 @@ if not has_libxml2:
                      'xsi' : xsi}
             t = etree.Element(root, nsmap=NSMAP)
             return self.decorateWithHeader(t)
-        
+
         def validateXml(self, fpath, xmlschema_context):
             # Use lxml
             xmlschema = etree.XMLSchema(xmlschema_context)
@@ -362,21 +357,21 @@ if not has_libxml2:
                 current XML toolkit.
             """
             return [root]
-        
-else:        
+
+else:
     class TreeFactory:
         def __init__(self):
             pass
-        
+
         def newNode(self, tag):
             return libxml2.newNode(tag)
 
         def newEtreeNode(self, tag, init_ns=False):
             return etree.Element(tag)
-        
+
         def copyNode(self, node):
             return node.copyNode(1)
-        
+
         def appendNode(self, parent, child):
             if hasattr(parent, 'addChild'):
                 parent.addChild(child)
@@ -387,7 +382,7 @@ else:
             if hasattr(node, 'hasProp'):
                 return node.hasProp(att)
             return att in node.attrib
-        
+
         def getAttribute(self, node, att):
             if hasattr(node, 'prop'):
                 return node.prop(att)
@@ -398,7 +393,7 @@ else:
                 node.setProp(att, value)
             else:
                 node.attrib[att] = value
-                
+
         def getText(self, root):
             if hasattr(root, 'getContent'):
                 return root.getContent()
@@ -426,20 +421,18 @@ else:
             doc.freeDoc()
 
         def writeTree(self, root, fpath):
-            fp = open(fpath, 'w')
-            doc = libxml2.newDoc('1.0')
-            doc.setRootElement(root)
-            fp.write(doc.serialize("UTF-8", 1))
-            doc.freeDoc()
-            fp.close()
+            with open(fpath, 'w') as fp:
+                doc = libxml2.newDoc('1.0')
+                doc.setRootElement(root)
+                fp.write(doc.serialize("UTF-8", 1))
+                doc.freeDoc()
 
         def prettyPrintFile(self, fpath):
             # Read file and resolve entities
             doc = libxml2.readFile(fpath, None, libxml2d.XML_PARSE_NOENT)
-            fp = open(fpath, 'w')
-            # Prettyprint
-            fp.write(doc.serialize("UTF-8", 1))
-            fp.close()
+            with open(fpath, 'w') as fp:
+                # Prettyprint
+                fp.write(doc.serialize("UTF-8", 1))
             # Cleanup
             doc.freeDoc()
 
@@ -448,9 +441,9 @@ else:
             ns = root.newNs(dbxsd, None)
             xi = root.newNs(xsi, 'xsi')
             root.setNs(ns)  #put this node in the target namespace
-    
+
             root.setNsProp(xi, 'schemaLocation', "%s %s/scons.xsd" % (dbxsd, dbxsd))
-        
+
             return root
 
         def newXmlTree(self, root):
@@ -461,6 +454,8 @@ else:
             return self.decorateWithHeader(t)
 
         def validateXml(self, fpath, xmlschema_context):
+            retval = True
+
             # Create validation context
             validation_context = xmlschema_context.schemaNewValidCtxt()
             # Set error/warning handlers
@@ -470,17 +465,19 @@ else:
             doc = libxml2.readFile(fpath, None, libxml2.XML_PARSE_NOENT)
             doc.xincludeProcessFlags(libxml2.XML_PARSE_NOENT)
             err = validation_context.schemaValidateDoc(doc)
-            # Cleanup
-            doc.freeDoc()
-            del validation_context
-        
+
             if err or eh.errors:
                 for e in eh.errors:
                     print(e.rstrip("\n"))
+                # import pdb; pdb.set_trace()
                 print("%s fails to validate" % fpath)
-                return False
-                
-            return True
+                retval = False
+
+            # Cleanup
+            doc.freeDoc()
+            del validation_context
+
+            return retval
 
         def findAll(self, root, tag, ns=None, xpath_context=None, nsmap=None):
             if hasattr(root, 'xpathEval') and xpath_context:
@@ -503,7 +500,7 @@ else:
                 expression = "./%s/node()" % tag
                 if ns:
                     expression = "./%s:%s/node()" % (ns, tag)
-                
+
                 return xpath_context.xpathEval(expression)
             else:
                 expression = "./{%s}%s/node()" % (nsmap[ns], tag)
@@ -533,7 +530,7 @@ else:
             if child.tail:
                 tail = libxml2.newText(child.tail)
                 elements.append(tail)
-                
+
             return elements
 
         def convertElementTree(self, root):
@@ -559,7 +556,7 @@ else:
             if root.tail:
                 tail = libxml2.newText(root.tail)
                 elements.append(tail)
-                
+
             return elements
 
 tf = TreeFactory()
@@ -600,7 +597,7 @@ class SConsDocTree:
             # Register namespaces
             for key, val in self.nsmap.items():
                 self.xpath_context.xpathRegisterNs(key, val)
-            
+
     def __del__(self):
         if self.doc is not None:
             self.doc.freeDoc()
@@ -619,7 +616,7 @@ def validate_all_xml(dpaths, xsdfile=default_xsd):
         ctxt = libxml2.schemaNewParserCtxt(xsdfile)
         xmlschema_context = ctxt.schemaParse()
         del ctxt
-    
+
     fpaths = []
     for dp in dpaths:
         if dp.endswith('.xml') and isSConsXml(dp):
@@ -632,13 +629,13 @@ def validate_all_xml(dpaths, xsdfile=default_xsd):
                         fp = os.path.join(path, f)
                         if isSConsXml(fp):
                             fpaths.append(fp)
-                
+
     fails = []
     for idx, fp in enumerate(fpaths):
         fpath = os.path.join(path, fp)
         print("%.2f%s (%d/%d) %s" % (float(idx+1)*100.0/float(len(fpaths)),
                                      perc, idx+1, len(fpaths),fp))
-                                              
+
         if not tf.validateXml(fp, xmlschema_context):
             fails.append(fp)
             continue
@@ -649,7 +646,7 @@ def validate_all_xml(dpaths, xsdfile=default_xsd):
 
     if fails:
         return False
-    
+
     return True
 
 class Item(object):
@@ -730,10 +727,10 @@ class SConsDocHandler(object):
             uses.extend(self.parseItems(u, xpath_context, nsmap))
         for s in tf.findAll(domelem, "sets", dbxid, xpath_context, nsmap):
             sets.extend(self.parseItems(s, xpath_context, nsmap))
-        
+
         return sorted(uses), sorted(sets)
 
-    def parseInstance(self, domelem, map, Class, 
+    def parseInstance(self, domelem, map, Class,
                         xpath_context, nsmap, include_entities=True):
         name = 'unknown'
         if tf.hasAttribute(domelem, 'name'):
@@ -757,24 +754,24 @@ class SConsDocHandler(object):
                     instance.arguments = []
                 instance.arguments.append(tf.copyNode(a))
 
-    def parseDomtree(self, root, xpath_context=None, nsmap=None, include_entities=True):    
+    def parseDomtree(self, root, xpath_context=None, nsmap=None, include_entities=True):
         # Process Builders
         for b in tf.findAll(root, "builder", dbxid, xpath_context, nsmap):
-            self.parseInstance(b, self.builders, Builder, 
+            self.parseInstance(b, self.builders, Builder,
                                xpath_context, nsmap, include_entities)
         # Process Functions
         for f in tf.findAll(root, "scons_function", dbxid, xpath_context, nsmap):
-            self.parseInstance(f, self.functions, Function, 
+            self.parseInstance(f, self.functions, Function,
                                xpath_context, nsmap, include_entities)
         # Process Tools
         for t in tf.findAll(root, "tool", dbxid, xpath_context, nsmap):
-            self.parseInstance(t, self.tools, Tool, 
+            self.parseInstance(t, self.tools, Tool,
                                xpath_context, nsmap, include_entities)
         # Process CVars
         for c in tf.findAll(root, "cvar", dbxid, xpath_context, nsmap):
-            self.parseInstance(c, self.cvars, ConstructionVariable, 
+            self.parseInstance(c, self.cvars, ConstructionVariable,
                                xpath_context, nsmap, include_entities)
-        
+
     def parseContent(self, content, include_entities=True):
         """ Parses the given content as XML file. This method
             is used when we generate the basic lists of entities
@@ -795,27 +792,48 @@ class SConsDocHandler(object):
         t.parseXmlFile(fpath)
         # Parse it
         self.parseDomtree(t.root, t.xpath_context, t.nsmap)
-        
+
 # lifted from Ka-Ping Yee's way cool pydoc module.
-def importfile(path):
-    """Import a Python source file or compiled file given its path."""
-    magic = imp.get_magic()
-    file = open(path, 'r')
-    if file.read(len(magic)) == magic:
-        kind = imp.PY_COMPILED
-    else:
-        kind = imp.PY_SOURCE
-    file.close()
-    filename = os.path.basename(path)
-    name, ext = os.path.splitext(filename)
-    file = open(path, 'r')
-    try:
-        module = imp.load_module(name, file, path, (ext, 'r', kind))
-    except ImportError as e:
-        sys.stderr.write("Could not import %s: %s\n" % (path, e))
-        return None
-    file.close()
-    return module
+if sys.version_info[0] == 2:
+    def importfile(path):
+        """Import a Python source file or compiled file given its path."""
+        import imp
+        magic = imp.get_magic()
+        with open(path, 'r') as ifp:
+            if ifp.read(len(magic)) == magic:
+                kind = imp.PY_COMPILED
+            else:
+                kind = imp.PY_SOURCE
+        filename = os.path.basename(path)
+        name, ext = os.path.splitext(filename)
+        with open(path, 'r') as ifp:
+            try:
+                module = imp.load_module(name, ifp, path, (ext, 'r', kind))
+            except ImportError as e:
+                sys.stderr.write("Could not import %s: %s\n" % (path, e))
+                return None
+        return module
+
+else:  # PY3 version, from newer pydoc
+    def importfile(path):
+        """Import a Python source file or compiled file given its path."""
+        import importlib
+        from pydoc import ErrorDuringImport
+        magic = importlib.util.MAGIC_NUMBER
+        with open(path, 'rb') as ifp:
+            is_bytecode = magic == ifp.read(len(magic))
+        filename = os.path.basename(path)
+        name, ext = os.path.splitext(filename)
+        if is_bytecode:
+            loader = importlib._bootstrap_external.SourcelessFileLoader(name, path)
+        else:
+            loader = importlib._bootstrap_external.SourceFileLoader(name, path)
+        # XXX We probably don't need to pass in the loader here.
+        spec = importlib.util.spec_from_file_location(name, path, loader=loader)
+        try:
+            return importlib._bootstrap._load(spec)
+        except:
+            raise ErrorDuringImport(path, sys.exc_info())
 
 # Local Variables:
 # tab-width:4