diff options
Diffstat (limited to 'QMTest')
| -rw-r--r-- | QMTest/README.txt | 4 | ||||
| -rw-r--r-- | QMTest/SConscript | 6 | ||||
| -rw-r--r-- | QMTest/TestCmd.py | 481 | ||||
| -rw-r--r-- | QMTest/TestCmdTests.py | 266 | ||||
| -rw-r--r-- | QMTest/TestCommon.py | 84 | ||||
| -rw-r--r-- | QMTest/TestCommonTests.py | 13 | ||||
| -rw-r--r-- | QMTest/TestRuntest.py | 4 | ||||
| -rw-r--r-- | QMTest/TestSCons.py | 363 | ||||
| -rw-r--r-- | QMTest/TestSConsMSVS.py | 26 | ||||
| -rw-r--r-- | QMTest/TestSCons_time.py | 21 | ||||
| -rw-r--r-- | QMTest/TestSConsign.py | 7 | ||||
| -rw-r--r-- | QMTest/scons_tdb.py | 20 |
12 files changed, 799 insertions, 496 deletions
diff --git a/QMTest/README.txt b/QMTest/README.txt index 4c7f413..c644565 100644 --- a/QMTest/README.txt +++ b/QMTest/README.txt @@ -54,5 +54,5 @@ the pieces here are local to SCons. from this infrastructure, in no small part because we're not really using it as originally envisioned. -Copyright (c) 2001 - 2016 The SCons Foundation -QMTest/README.txt rel_2.5.1:3735:9dc6cee5c168 2016/11/03 14:02:02 bdbaddog +Copyright (c) 2001 - 2017 The SCons Foundation +QMTest/README.txt rel_3.0.0:4395:8972f6a2f699 2017/09/18 12:59:24 bdbaddog diff --git a/QMTest/SConscript b/QMTest/SConscript index e4c9108..4107862 100644 --- a/QMTest/SConscript +++ b/QMTest/SConscript @@ -3,7 +3,7 @@ # # -# Copyright (c) 2001 - 2016 The SCons Foundation +# Copyright (c) 2001 - 2017 The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -44,12 +44,12 @@ files = [ def copy(target, source, env): t = str(target[0]) s = str(source[0]) - c = open(s, 'rb').read() + c = open(s, 'r').read() # Note: We construct the __ VERSION __ substitution string at # run-time so it doesn't get replaced when this file gets copied # into the tree for packaging. c = c.replace('__' + 'VERSION' + '__', env['VERSION']) - open(t, 'wb').write(c) + open(t, 'w').write(c) for file in files: # Guarantee that real copies of these files always exist in diff --git a/QMTest/TestCmd.py b/QMTest/TestCmd.py index cd559b7..0aab9a8 100644 --- a/QMTest/TestCmd.py +++ b/QMTest/TestCmd.py @@ -285,7 +285,7 @@ version. # PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS" BASIS, # AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. -from __future__ import division +from __future__ import division, print_function __author__ = "Steven Knight <knight at baldmt dot com>" __revision__ = "TestCmd.py 1.3.D001 2010/06/03 12:58:27 knight" @@ -306,8 +306,15 @@ import time import traceback import types + +IS_PY3 = sys.version_info[0] == 3 +IS_WINDOWS = sys.platform == 'win32' + + class null(object): pass + + _Null = null() try: @@ -328,12 +335,28 @@ __all__ = [ 'match_re_dotall', 'python', '_python_', - 'TestCmd' + 'TestCmd', + 'to_bytes', + 'to_str', ] + def is_List(e): return isinstance(e, (list, UserList)) + +def to_bytes(s): + if isinstance(s, bytes) or bytes is str: + return s + return bytes(s, 'utf-8') + + +def to_str(s): + if bytes is str or is_String(s): + return s + return str(s, 'utf-8') + + try: eval('unicode') except NameError: @@ -351,12 +374,13 @@ else: re_space = re.compile('\s') + def _caller(tblist, skip): string = "" arr = [] for file, line, name, text in tblist: if file[-10:] == "TestCmd.py": - break + break arr = [(file, line, name, text)] + arr atfrom = "at" for file, line, name, text in arr[skip:]: @@ -368,7 +392,8 @@ def _caller(tblist, skip): atfrom = "\tfrom" return string -def fail_test(self = None, condition = 1, function = None, skip = 0, message=None): + +def fail_test(self=None, condition=1, function=None, skip=0, message=None): """Cause the test to fail. By default, the fail_test() method reports that the test FAILED @@ -392,14 +417,15 @@ def fail_test(self = None, condition = 1, function = None, skip = 0, message=Non at = _caller(traceback.extract_stack(), skip) if message: - msg = "\t%s\n"%message + msg = "\t%s\n" % message else: msg = "" sys.stderr.write("FAILED test" + of + desc + sep + at + msg) sys.exit(1) -def no_result(self = None, condition = 1, function = None, skip = 0): + +def no_result(self=None, condition=1, function=None, skip=0): """Causes a test to exit with no valid result. By default, the no_result() method reports NO RESULT for the test @@ -426,7 +452,8 @@ def no_result(self = None, condition = 1, function = None, skip = 0): sys.exit(2) -def pass_test(self = None, condition = 1, function = None): + +def pass_test(self=None, condition=1, function=None): """Causes a test to pass. By default, the pass_test() method reports PASSED for the test @@ -440,13 +467,18 @@ def pass_test(self = None, condition = 1, function = None): sys.stderr.write("PASSED\n") sys.exit(0) -def match_exact(lines = None, matches = None): + +def match_exact(lines=None, matches=None, newline=os.sep): """ """ + + if isinstance(lines, bytes) or bytes is str: + newline = to_bytes(newline) + if not is_List(lines): - lines = lines.split("\n") + lines = lines.split(newline) if not is_List(matches): - matches = matches.split("\n") + matches = matches.split(newline) if len(lines) != len(matches): return for i in range(len(lines)): @@ -454,7 +486,8 @@ def match_exact(lines = None, matches = None): return return 1 -def match_caseinsensitive(lines = None, matches = None): + +def match_caseinsensitive(lines=None, matches=None): """ """ if not is_List(lines): @@ -468,7 +501,8 @@ def match_caseinsensitive(lines = None, matches = None): return return 1 -def match_re(lines = None, res = None): + +def match_re(lines=None, res=None): """ """ if not is_List(lines): @@ -477,21 +511,23 @@ def match_re(lines = None, res = None): if not is_List(res): res = res.split("\n") if len(lines) != len(res): - print "match_re: expected %d lines, found %d"%(len(res), len(lines)) + print("match_re: expected %d lines, found %d" % (len(res), len(lines))) return for i in range(len(lines)): s = "^" + res[i] + "$" try: expr = re.compile(s) - except re.error, e: + except re.error as e: msg = "Regular expression error in %s: %s" raise re.error(msg % (repr(s), e.args[0])) if not expr.search(lines[i]): - print "match_re: mismatch at line %d:\n search re='%s'\n line='%s'"%(i,s,lines[i]) + print("match_re: mismatch at line %d:\n search re='%s'\n line='%s'" % ( + i, s, lines[i])) return return 1 -def match_re_dotall(lines = None, res = None): + +def match_re_dotall(lines=None, res=None): """ """ if not isinstance(lines, str): @@ -501,11 +537,12 @@ def match_re_dotall(lines = None, res = None): s = "^" + res + "$" try: expr = re.compile(s, re.DOTALL) - except re.error, e: + except re.error as e: msg = "Regular expression error in %s: %s" raise re.error(msg % (repr(s), e.args[0])) return expr.match(lines) + def simple_diff(a, b, fromfile='', tofile='', fromfiledate='', tofiledate='', n=3, lineterm='\n'): """ @@ -513,26 +550,30 @@ def simple_diff(a, b, fromfile='', tofile='', (diff -c) and difflib.unified_diff (diff -u) but which prints output like the simple, unadorned 'diff" command. """ + a = [to_str(q) for q in a] + b = [to_str(q) for q in b] sm = difflib.SequenceMatcher(None, a, b) + def comma(x1, x2): - return x1+1 == x2 and str(x2) or '%s,%s' % (x1+1, x2) + return x1 + 1 == x2 and str(x2) or '%s,%s' % (x1 + 1, x2) result = [] for op, a1, a2, b1, b2 in sm.get_opcodes(): if op == 'delete': result.append("%sd%d" % (comma(a1, a2), b1)) - result.extend([ '< ' + l for l in a[a1:a2] ]) + result.extend(['< ' + l for l in a[a1:a2]]) elif op == 'insert': result.append("%da%s" % (a1, comma(b1, b2))) - result.extend([ '> ' + l for l in b[b1:b2] ]) + result.extend(['> ' + l for l in b[b1:b2]]) elif op == 'replace': result.append("%sc%s" % (comma(a1, a2), comma(b1, b2))) - result.extend([ '< ' + l for l in a[a1:a2] ]) + result.extend(['< ' + l for l in a[a1:a2]]) result.append('---') - result.extend([ '> ' + l for l in b[b1:b2] ]) + result.extend(['> ' + l for l in b[b1:b2]]) return result + def diff_re(a, b, fromfile='', tofile='', - fromfiledate='', tofiledate='', n=3, lineterm='\n'): + fromfiledate='', tofiledate='', n=3, lineterm='\n'): """ A simple "diff" of two sets of lines when the expected lines are regular expressions. This is a really dumb thing that @@ -543,33 +584,34 @@ def diff_re(a, b, fromfile='', tofile='', result = [] diff = len(a) - len(b) if diff < 0: - a = a + ['']*(-diff) + a = a + [''] * (-diff) elif diff > 0: - b = b + ['']*diff + b = b + [''] * diff i = 0 for aline, bline in zip(a, b): s = "^" + aline + "$" try: expr = re.compile(s) - except re.error, e: + except re.error as e: msg = "Regular expression error in %s: %s" raise re.error(msg % (repr(s), e.args[0])) if not expr.search(bline): - result.append("%sc%s" % (i+1, i+1)) + result.append("%sc%s" % (i + 1, i + 1)) result.append('< ' + repr(a[i])) result.append('---') result.append('> ' + repr(b[i])) - i = i+1 + i = i + 1 return result + if os.name == 'posix': def escape(arg): "escape shell special characters" slash = '\\' special = '"$' - arg = arg.replace(slash, slash+slash) + arg = arg.replace(slash, slash + slash) for c in special: - arg = arg.replace(c, slash+c) + arg = arg.replace(c, slash + c) if re_space.search(arg): arg = '"' + arg + '"' return arg @@ -627,14 +669,13 @@ else: st = os.stat(f) except OSError: continue - if stat.S_IMODE(st[stat.ST_MODE]) & 0111: + if stat.S_IMODE(st[stat.ST_MODE]) & 0o111: return f return None default_sleep_seconds = 1 - import subprocess try: @@ -642,6 +683,7 @@ try: except AttributeError: if sys.platform == 'win32': import win32process + def terminate(self): win32process.TerminateProcess(self._handle, 1) else: @@ -651,53 +693,64 @@ except AttributeError: setattr(subprocess.Popen, 'terminate', method) - # From Josiah Carlson, # ASPN : Python Cookbook : Module to allow Asynchronous subprocess use on Windows and Posix platforms # http://aspn.activestate.com/ASPN/Cookbook/Python/Recipe/440554 PIPE = subprocess.PIPE -if subprocess.mswindows: +if sys.platform == 'win32': # and subprocess.mswindows: try: from win32file import ReadFile, WriteFile from win32pipe import PeekNamedPipe except ImportError: # If PyWin32 is not available, try ctypes instead # XXX These replicate _just_enough_ PyWin32 behaviour for our purposes - import ctypes; from ctypes.wintypes import DWORD + import ctypes + from ctypes.wintypes import DWORD + def ReadFile(hFile, bufSize, ol=None): assert ol is None lpBuffer = ctypes.create_string_buffer(bufSize) bytesRead = DWORD() bErr = ctypes.windll.kernel32.ReadFile( - hFile, lpBuffer, bufSize, ctypes.byref(bytesRead), ol) - if not bErr: raise ctypes.WinError() + hFile, lpBuffer, bufSize, ctypes.byref(bytesRead), ol) + if not bErr: + raise ctypes.WinError() return (0, ctypes.string_at(lpBuffer, bytesRead.value)) + def WriteFile(hFile, data, ol=None): assert ol is None bytesWritten = DWORD() bErr = ctypes.windll.kernel32.WriteFile( - hFile, data, len(data), ctypes.byref(bytesWritten), ol) - if not bErr: raise ctypes.WinError() + hFile, data, len(data), ctypes.byref(bytesWritten), ol) + if not bErr: + raise ctypes.WinError() return (0, bytesWritten.value) + def PeekNamedPipe(hPipe, size): assert size == 0 bytesAvail = DWORD() bErr = ctypes.windll.kernel32.PeekNamedPipe( - hPipe, None, size, None, ctypes.byref(bytesAvail), None) - if not bErr: raise ctypes.WinError() + hPipe, None, size, None, ctypes.byref(bytesAvail), None) + if not bErr: + raise ctypes.WinError() return ("", bytesAvail.value, None) import msvcrt else: import select import fcntl - try: fcntl.F_GETFL - except AttributeError: fcntl.F_GETFL = 3 + try: + fcntl.F_GETFL + except AttributeError: + fcntl.F_GETFL = 3 + + try: + fcntl.F_SETFL + except AttributeError: + fcntl.F_SETFL = 4 - try: fcntl.F_SETFL - except AttributeError: fcntl.F_SETFL = 4 class Popen(subprocess.Popen): def recv(self, maxsize=None): @@ -720,8 +773,9 @@ class Popen(subprocess.Popen): getattr(self, which).close() setattr(self, which, None) - if subprocess.mswindows: + if sys.platform == 'win32': # and subprocess.mswindows: def send(self, input): + input = to_bytes(input) if not self.stdin: return None @@ -730,7 +784,7 @@ class Popen(subprocess.Popen): (errCode, written) = WriteFile(x, input) except ValueError: return self._close('stdin') - except (subprocess.pywintypes.error, Exception), why: + except (subprocess.pywintypes.error, Exception) as why: if why.args[0] in (109, errno.ESHUTDOWN): return self._close('stdin') raise @@ -751,12 +805,12 @@ class Popen(subprocess.Popen): (errCode, read) = ReadFile(x, nAvail, None) except ValueError: return self._close(which) - except (subprocess.pywintypes.error, Exception), why: + except (subprocess.pywintypes.error, Exception) as why: if why.args[0] in (109, errno.ESHUTDOWN): return self._close(which) raise - #if self.universal_newlines: + # if self.universal_newlines: # read = self._translate_newlines(read) return read @@ -769,9 +823,10 @@ class Popen(subprocess.Popen): return 0 try: - written = os.write(self.stdin.fileno(), input) - except OSError, why: - if why.args[0] == errno.EPIPE: #broken pipe + written = os.write(self.stdin.fileno(), + bytearray(input, 'utf-8')) + except OSError as why: + if why.args[0] == errno.EPIPE: # broken pipe return self._close('stdin') raise @@ -788,7 +843,7 @@ class Popen(subprocess.Popen): flags = None else: if not conn.closed: - fcntl.fcntl(conn, fcntl.F_SETFL, flags| os.O_NONBLOCK) + fcntl.fcntl(conn, fcntl.F_SETFL, flags | os.O_NONBLOCK) try: if not select.select([conn], [], [], 0)[0]: @@ -798,19 +853,21 @@ class Popen(subprocess.Popen): if not r: return self._close(which) - #if self.universal_newlines: + # if self.universal_newlines: # r = self._translate_newlines(r) return r finally: if not conn.closed and not flags is None: fcntl.fcntl(conn, fcntl.F_SETFL, flags) + disconnect_message = "Other end disconnected!" + def recv_some(p, t=.1, e=1, tr=5, stderr=0): if tr < 1: tr = 1 - x = time.time()+t + x = time.time() + t y = [] r = '' pr = p.recv @@ -826,9 +883,10 @@ def recv_some(p, t=.1, e=1, tr=5, stderr=0): elif r: y.append(r) else: - time.sleep(max((x-time.time())/tr, 0)) + time.sleep(max((x - time.time()) / tr, 0)) return ''.join(y) + def send_all(p, data): while len(data): sent = p.send(data) @@ -836,16 +894,19 @@ def send_all(p, data): raise Exception(disconnect_message) data = memoryview(data)[sent:] + _Cleanup = [] + def _clean(): global _Cleanup - cleanlist = [ c for c in _Cleanup if c ] + cleanlist = [c for c in _Cleanup if c] del _Cleanup[:] cleanlist.reverse() for test in cleanlist: test.cleanup() + atexit.register(_clean) @@ -853,21 +914,21 @@ class TestCmd(object): """Class TestCmd """ - def __init__(self, description = None, - program = None, - interpreter = None, - workdir = None, - subdir = None, - verbose = None, - match = None, - match_stdout = None, - match_stderr = None, - diff = None, - diff_stdout = None, - diff_stderr = None, - combine = 0, - universal_newlines = 1, - timeout = None): + def __init__(self, description=None, + program=None, + interpreter=None, + workdir=None, + subdir=None, + verbose=None, + match=None, + match_stdout=None, + match_stderr=None, + diff=None, + diff_stdout=None, + diff_stderr=None, + combine=0, + universal_newlines=True, + timeout=None): self.external = os.environ.get('SCONS_EXTERNAL_TEST', 0) self._cwd = os.getcwd() self.description_set(description) @@ -875,7 +936,7 @@ class TestCmd(object): self.interpreter_set(interpreter) if verbose is None: try: - verbose = max( 0, int(os.environ.get('TESTCMD_VERBOSE', 0)) ) + verbose = max(0, int(os.environ.get('TESTCMD_VERBOSE', 0))) except ValueError: verbose = 0 self.verbose_set(verbose) @@ -887,7 +948,8 @@ class TestCmd(object): self.set_diff_function(diff, diff_stdout, diff_stderr) self._dirlist = [] self._preserve = {'pass_test': 0, 'fail_test': 0, 'no_result': 0} - if 'PRESERVE' in os.environ and not os.environ['PRESERVE'] is '': + preserve_value = os.environ.get('PRESERVE', False) + if preserve_value not in [0, '0', 'False']: self._preserve['pass_test'] = os.environ['PRESERVE'] self._preserve['fail_test'] = os.environ['PRESERVE'] self._preserve['no_result'] = os.environ['PRESERVE'] @@ -910,7 +972,7 @@ class TestCmd(object): self.condition = 'no_result' self.workdir_set(workdir) self.subdir(subdir) - self.script_srcdir = None + self.fixture_dirs = [] def __del__(self): self.cleanup() @@ -941,7 +1003,7 @@ class TestCmd(object): path = self.canonicalize(path) os.chmod(path, mode) - def cleanup(self, condition = None): + def cleanup(self, condition=None): """Removes any temporary working directories for the specified TestCmd environment. If the environment variable PRESERVE was set when the TestCmd environment was created, temporary working @@ -964,22 +1026,22 @@ class TestCmd(object): condition = self.condition if self._preserve[condition]: for dir in self._dirlist: - print unicode("Preserved directory " + dir + "\n"), + print(u"Preserved directory " + dir + "\n") else: list = self._dirlist[:] list.reverse() for dir in list: self.writable(dir, 1) - shutil.rmtree(dir, ignore_errors = 1) + shutil.rmtree(dir, ignore_errors=1) self._dirlist = [] global _Cleanup if self in _Cleanup: _Cleanup.remove(self) - def command_args(self, program = None, - interpreter = None, - arguments = None): + def command_args(self, program=None, + interpreter=None, + arguments=None): if not self.external: if program: if isinstance(program, str) and not os.path.isabs(program): @@ -1030,10 +1092,10 @@ class TestCmd(object): if diff_function is None: diff_function = self.simple_diff if name is not None: - print self.banner(name) + print(self.banner(name)) args = (a.splitlines(), b.splitlines()) + args for line in diff_function(*args, **kw): - print line + print(line) def diff_stderr(self, a, b, *args, **kw): """Compare actual and expected file contents. @@ -1061,17 +1123,17 @@ class TestCmd(object): unified_diff = staticmethod(difflib.unified_diff) - def fail_test(self, condition = 1, function = None, skip = 0, message = None): + def fail_test(self, condition=1, function=None, skip=0, message=None): """Cause the test to fail. """ if not condition: return self.condition = 'fail_test' - fail_test(self = self, - condition = condition, - function = function, - skip = skip, - message = message) + fail_test(self=self, + condition=condition, + function=function, + skip=skip, + message=message) def interpreter_set(self, interpreter): """Set the program to be used to interpret the program @@ -1133,24 +1195,24 @@ class TestCmd(object): match_re_dotall = staticmethod(match_re_dotall) - def no_result(self, condition = 1, function = None, skip = 0): + def no_result(self, condition=1, function=None, skip=0): """Report that the test could not be run. """ if not condition: return self.condition = 'no_result' - no_result(self = self, - condition = condition, - function = function, - skip = skip) + no_result(self=self, + condition=condition, + function=function, + skip=skip) - def pass_test(self, condition = 1, function = None): + def pass_test(self, condition=1, function=None): """Cause the test to pass. """ if not condition: return self.condition = 'pass_test' - pass_test(self = self, condition = condition, function = function) + pass_test(self=self, condition=condition, function=function) def preserve(self, *conditions): """Arrange for the temporary working directories for the @@ -1172,7 +1234,7 @@ class TestCmd(object): program = os.path.join(self._cwd, program) self.program = program - def read(self, file, mode = 'rb'): + def read(self, file, mode='rb', newline=None): """Reads and returns the contents of the specified file name. The file name may be a list, in which case the elements are concatenated with the os.path.join() method. The file is @@ -1184,7 +1246,10 @@ class TestCmd(object): file = self.canonicalize(file) if mode[0] != 'r': raise ValueError("mode must begin with 'r'") - return open(file, mode).read() + if IS_PY3 and 'b' not in mode: + return open(file, mode, newline=newline).read() + else: + return open(file, mode).read() def rmdir(self, dir): """Removes the specified dir name. @@ -1234,10 +1299,15 @@ class TestCmd(object): assumed to be under the temporary working directory, it gets created automatically, if it does not already exist. """ - if srcdir and self.script_srcdir and not os.path.isabs(srcdir): - spath = os.path.join(self.script_srcdir, srcdir) + + if srcdir and self.fixture_dirs and not os.path.isabs(srcdir): + for dir in self.fixture_dirs: + spath = os.path.join(dir, srcdir) + if os.path.isdir(spath): + break else: spath = srcdir + if dstdir: dstdir = self.canonicalize(dstdir) else: @@ -1248,7 +1318,7 @@ class TestCmd(object): if len(dstlist) > 0 and dstlist[0] == ".": dstlist = dstlist[1:] for idx in range(len(dstlist)): - self.subdir(dstlist[:idx+1]) + self.subdir(dstlist[:idx + 1]) if dstdir and self.workdir: dstdir = os.path.join(self.workdir, dstdir) @@ -1271,13 +1341,15 @@ class TestCmd(object): automatically, if it does not already exist. """ srcpath, srctail = os.path.split(srcfile) - if srcpath: - if self.script_srcdir and not os.path.isabs(srcpath): - spath = os.path.join(self.script_srcdir, srcfile) - else: - spath = srcfile + + if srcpath and (not self.fixture_dirs or os.path.isabs(srcpath)): + spath = srcfile else: - spath = os.path.join(self.script_srcdir, srcfile) + for dir in self.fixture_dirs: + spath = os.path.join(dir, srcfile) + if os.path.isfile(spath): + break + if not dstfile: if srctail: dpath = os.path.join(self.workdir, srctail) @@ -1291,17 +1363,17 @@ class TestCmd(object): if len(dstlist) > 0 and dstlist[0] == ".": dstlist = dstlist[1:] for idx in range(len(dstlist)): - self.subdir(dstlist[:idx+1]) + self.subdir(dstlist[:idx + 1]) dpath = os.path.join(self.workdir, dstfile) shutil.copy(spath, dpath) - def start(self, program = None, - interpreter = None, - arguments = None, - universal_newlines = None, - timeout = _Null, - **kw): + def start(self, program=None, + interpreter=None, + arguments=None, + universal_newlines=None, + timeout=_Null, + **kw): """ Starts a program or script for the test environment. @@ -1310,7 +1382,7 @@ class TestCmd(object): """ cmd = self.command_args(program, interpreter, arguments) if self.verbose: - cmd_string = ' '.join([ self.escape(c) for c in cmd ]) + cmd_string = ' '.join([self.escape(c) for c in cmd]) sys.stderr.write(cmd_string + "\n") if universal_newlines is None: universal_newlines = self.universal_newlines @@ -1334,14 +1406,56 @@ class TestCmd(object): if timeout: self.timer = threading.Timer(float(timeout), self._timeout) self.timer.start() + + if IS_PY3 and sys.platform == 'win32': + # Set this otherwist stdout/stderr pipes default to + # windows default locale cp1252 which will throw exception + # if using non-ascii characters. + # For example test/Install/non-ascii-name.py + os.environ['PYTHONIOENCODING'] = 'utf-8' + + # It seems that all pythons up to py3.6 still set text mode if you set encoding. + # TODO: File enhancement request on python to propagate universal_newlines even + # if encoding is set.hg c p = Popen(cmd, stdin=stdin, stdout=subprocess.PIPE, stderr=stderr_value, - universal_newlines=universal_newlines) + env=os.environ, + universal_newlines=False) + self.process = p return p + @staticmethod + def fix_binary_stream(stream): + """ + Handle stdout/stderr from popen when we specify universal_newlines = False. + This will read from the pipes in binary mode, not decode the output, + and not convert line endings to \n. + We do this because in py3 (3.5) with universal_newlines=True, it will + choose the default system locale to decode the output, and this breaks unicode + output. Specifically breaking test/option--tree.py which outputs a unicode char. + + py 3.6 allows us to pass an encoding param to popen thus not requiring the decode + nor end of line handling, because we propagate universal_newlines as specified. + + TODO: Do we need to pass universal newlines into this function? + """ + + if not stream: + return stream + # TODO: Run full tests on both platforms and see if this fixes failures + # It seems that py3.6 still sets text mode if you set encoding. + elif sys.version_info[0] == 3:# TODO and sys.version_info[1] < 6: + stream = stream.decode('utf-8') + stream = stream.replace('\r\n', '\n') + elif sys.version_info[0] == 2: + stream = stream.replace('\r\n', '\n') + + return stream + + def finish(self, popen=None, **kw): """ Finishes and waits for the process being run under control of @@ -1351,6 +1465,10 @@ class TestCmd(object): if popen is None: popen = self.process stdout, stderr = popen.communicate() + + stdout = self.fix_binary_stream(stdout) + stderr = self.fix_binary_stream(stderr) + if self.timer: self.timer.cancel() self.timer = None @@ -1359,13 +1477,13 @@ class TestCmd(object): self._stdout.append(stdout or '') self._stderr.append(stderr or '') - def run(self, program = None, - interpreter = None, - arguments = None, - chdir = None, - stdin = None, - universal_newlines = None, - timeout = _Null): + def run(self, program=None, + interpreter=None, + arguments=None, + chdir=None, + stdin=None, + universal_newlines=None, + timeout=_Null): """Runs a test of the program or script for the test environment. Standard output and error output are saved for future retrieval via the stdout() and stderr() methods. @@ -1379,6 +1497,9 @@ class TestCmd(object): if not interpreter: interpreter = self.interpreter + if universal_newlines is None: + universal_newlines = self.universal_newlines + if chdir: oldcwd = os.getcwd() if not os.path.isabs(chdir): @@ -1386,14 +1507,18 @@ class TestCmd(object): if self.verbose: sys.stderr.write("chdir(" + chdir + ")\n") os.chdir(chdir) - p = self.start(program = program, - interpreter = interpreter, - arguments = arguments, - universal_newlines = universal_newlines, - timeout = timeout, - stdin = stdin) + p = self.start(program=program, + interpreter=interpreter, + arguments=arguments, + universal_newlines=universal_newlines, + timeout=timeout, + stdin=stdin) if is_List(stdin): stdin = ''.join(stdin) + + if stdin and IS_PY3:# and sys.version_info[1] < 6: + stdin = to_bytes(stdin) + # TODO(sgk): figure out how to re-use the logic in the .finish() # method above. Just calling it from here causes problems with # subclasses that redefine .finish(). We could abstract this @@ -1405,6 +1530,11 @@ class TestCmd(object): self.timer = None self.status = p.returncode self.process = None + + stdout = self.fix_binary_stream(stdout) + stderr = self.fix_binary_stream(stderr) + + self._stdout.append(stdout or '') self._stderr.append(stderr or '') @@ -1424,7 +1554,7 @@ class TestCmd(object): write(err) write('============ END STDERR\n') - def sleep(self, seconds = default_sleep_seconds): + def sleep(self, seconds=default_sleep_seconds): """Sleeps at least the specified number of seconds. If no number is specified, sleeps at least the minimum number of seconds necessary to advance file time stamps on the current @@ -1432,7 +1562,7 @@ class TestCmd(object): """ time.sleep(seconds) - def stderr(self, run = None): + def stderr(self, run=None): """Returns the error output from the specified run number. If there is no specified run number, then returns the error output of the last run. If the run number is less than zero, @@ -1446,7 +1576,7 @@ class TestCmd(object): run = run - 1 return self._stderr[run] - def stdout(self, run = None): + def stdout(self, run=None): """Returns the standard output from the specified run number. If there is no specified run number, then returns the standard output of the last run. If the run number is less than zero, @@ -1494,6 +1624,11 @@ class TestCmd(object): is an absolute path name. The target is *not* assumed to be under the temporary working directory. """ + if sys.platform == 'win32': + # Skip this on windows as we're not enabling it due to + # it requiring user permissions which aren't always present + # and we don't have a good way to detect those permissions yet. + return link = self.canonicalize(link) try: os.symlink(target, link) @@ -1525,7 +1660,7 @@ class TestCmd(object): # Uppercase the drive letter since the case of drive # letters is pretty much random on win32: - drive,rest = os.path.splitdrive(path) + drive, rest = os.path.splitdrive(path) if drive: path = drive.upper() + rest @@ -1605,14 +1740,22 @@ class TestCmd(object): if read: def do_chmod(fname): - try: st = os.stat(fname) - except OSError: pass - else: os.chmod(fname, stat.S_IMODE(st[stat.ST_MODE]|stat.S_IREAD)) + try: + st = os.stat(fname) + except OSError: + pass + else: + os.chmod(fname, stat.S_IMODE( + st[stat.ST_MODE] | stat.S_IREAD)) else: def do_chmod(fname): - try: st = os.stat(fname) - except OSError: pass - else: os.chmod(fname, stat.S_IMODE(st[stat.ST_MODE]&~stat.S_IREAD)) + try: + st = os.stat(fname) + except OSError: + pass + else: + os.chmod(fname, stat.S_IMODE( + st[stat.ST_MODE] & ~stat.S_IREAD)) if os.path.isfile(top): # If it's a file, that's easy, just chmod it. @@ -1646,25 +1789,36 @@ class TestCmd(object): if write: def do_chmod(fname): - try: os.chmod(fname, stat.S_IWRITE) - except OSError: pass + try: + os.chmod(fname, stat.S_IWRITE) + except OSError: + pass else: def do_chmod(fname): - try: os.chmod(fname, stat.S_IREAD) - except OSError: pass + try: + os.chmod(fname, stat.S_IREAD) + except OSError: + pass else: if write: def do_chmod(fname): - try: st = os.stat(fname) - except OSError: pass - else: os.chmod(fname, stat.S_IMODE(st[stat.ST_MODE]|0200)) + try: + st = os.stat(fname) + except OSError: + pass + else: + os.chmod(fname, stat.S_IMODE(st[stat.ST_MODE] | 0o200)) else: def do_chmod(fname): - try: st = os.stat(fname) - except OSError: pass - else: os.chmod(fname, stat.S_IMODE(st[stat.ST_MODE]&~0200)) + try: + st = os.stat(fname) + except OSError: + pass + else: + os.chmod(fname, stat.S_IMODE( + st[stat.ST_MODE] & ~0o200)) if os.path.isfile(top): do_chmod(top) @@ -1687,14 +1841,22 @@ class TestCmd(object): if execute: def do_chmod(fname): - try: st = os.stat(fname) - except OSError: pass - else: os.chmod(fname, stat.S_IMODE(st[stat.ST_MODE]|stat.S_IEXEC)) + try: + st = os.stat(fname) + except OSError: + pass + else: + os.chmod(fname, stat.S_IMODE( + st[stat.ST_MODE] | stat.S_IEXEC)) else: def do_chmod(fname): - try: st = os.stat(fname) - except OSError: pass - else: os.chmod(fname, stat.S_IMODE(st[stat.ST_MODE]&~stat.S_IEXEC)) + try: + st = os.stat(fname) + except OSError: + pass + else: + os.chmod(fname, stat.S_IMODE( + st[stat.ST_MODE] & ~stat.S_IEXEC)) if os.path.isfile(top): # If it's a file, that's easy, just chmod it. @@ -1719,7 +1881,7 @@ class TestCmd(object): do_chmod(os.path.join(dirpath, name)) do_chmod(top) - def write(self, file, content, mode = 'wb'): + def write(self, file, content, mode='wb'): """Writes the specified content text (second argument) to the specified file name (first argument). The file name may be a list, in which case the elements are concatenated with the @@ -1731,7 +1893,12 @@ class TestCmd(object): file = self.canonicalize(file) if mode[0] != 'w': raise ValueError("mode must begin with 'w'") - open(file, mode).write(content) + with open(file, mode) as f: + try: + f.write(content) + except TypeError as e: + # python 3 default strings are not bytes, but unicode + f.write(bytes(content, 'utf-8')) # Local Variables: # tab-width:4 diff --git a/QMTest/TestCmdTests.py b/QMTest/TestCmdTests.py index 1044ed1..b9226fd 100644 --- a/QMTest/TestCmdTests.py +++ b/QMTest/TestCmdTests.py @@ -26,13 +26,14 @@ import os import shutil import signal import stat -import StringIO +from StringIO import StringIO import sys import tempfile import time import types import unittest -import UserList +from UserList import UserList + # Strip the current directory so we get the right TestCmd.py module. sys.path = sys.path[1:] @@ -131,11 +132,11 @@ class TestCmdTestCase(unittest.TestCase): run_env.write(t.scriptout_path, textout) run_env.write(t.scripterr_path, texterr) - os.chmod(t.script_path, 0644) # XXX UNIX-specific - os.chmod(t.scriptx_path, 0755) # XXX UNIX-specific - os.chmod(t.script1_path, 0644) # XXX UNIX-specific - os.chmod(t.scriptout_path, 0644) # XXX UNIX-specific - os.chmod(t.scripterr_path, 0644) # XXX UNIX-specific + os.chmod(t.script_path, 0o644) # XXX UNIX-specific + os.chmod(t.scriptx_path, 0o755) # XXX UNIX-specific + os.chmod(t.script1_path, 0o644) # XXX UNIX-specific + os.chmod(t.scriptout_path, 0o644) # XXX UNIX-specific + os.chmod(t.scripterr_path, 0o644) # XXX UNIX-specific t.orig_cwd = os.getcwd() @@ -220,8 +221,8 @@ class cleanup_TestCase(TestCmdTestCase): test = TestCmd.TestCmd(workdir = '') wdir = test.workdir test.write('file2', "Test file #2\n") - os.chmod(test.workpath('file2'), 0400) - os.chmod(wdir, 0500) + os.chmod(test.workpath('file2'), 0o400) + os.chmod(wdir, 0o500) test.cleanup() assert not os.path.exists(wdir) @@ -244,11 +245,12 @@ class cleanup_TestCase(TestCmdTestCase): def test_atexit(self): """Test cleanup() when atexit is used""" - self.popen_python("""import sys + self.popen_python("""from __future__ import print_function +import sys sys.path = ['%s'] + sys.path import atexit def my_exitfunc(): - print "my_exitfunc()" + print("my_exitfunc()") atexit.register(my_exitfunc) import TestCmd result = TestCmd.TestCmd(workdir = '') @@ -257,10 +259,11 @@ sys.exit(0) def test_exitfunc(self): """Test cleanup() when sys.exitfunc is set""" - self.popen_python("""import sys + self.popen_python("""from __future__ import print_function +import sys sys.path = ['%s'] + sys.path def my_exitfunc(): - print "my_exitfunc()" + print("my_exitfunc()") sys.exitfunc = my_exitfunc import TestCmd result = TestCmd.TestCmd(workdir = '') @@ -286,35 +289,35 @@ class chmod_TestCase(TestCmdTestCase): test.chmod(['sub', 'file2'], stat.S_IWRITE) file1_mode = stat.S_IMODE(os.stat(wdir_file1)[stat.ST_MODE]) - assert file1_mode == 0444, '0%o' % file1_mode + assert file1_mode == 0o444, '0%o' % file1_mode file2_mode = stat.S_IMODE(os.stat(wdir_sub_file2)[stat.ST_MODE]) - assert file2_mode == 0666, '0%o' % file2_mode + assert file2_mode == 0o666, '0%o' % file2_mode test.chmod('file1', stat.S_IWRITE) test.chmod(wdir_sub_file2, stat.S_IREAD) file1_mode = stat.S_IMODE(os.stat(wdir_file1)[stat.ST_MODE]) - assert file1_mode == 0666, '0%o' % file1_mode + assert file1_mode == 0o666, '0%o' % file1_mode file2_mode = stat.S_IMODE(os.stat(wdir_sub_file2)[stat.ST_MODE]) - assert file2_mode == 0444, '0%o' % file2_mode + assert file2_mode == 0o444, '0%o' % file2_mode else: - test.chmod(wdir_file1, 0700) - test.chmod(['sub', 'file2'], 0760) + test.chmod(wdir_file1, 0o700) + test.chmod(['sub', 'file2'], 0o760) file1_mode = stat.S_IMODE(os.stat(wdir_file1)[stat.ST_MODE]) - assert file1_mode == 0700, '0%o' % file1_mode + assert file1_mode == 0o700, '0%o' % file1_mode file2_mode = stat.S_IMODE(os.stat(wdir_sub_file2)[stat.ST_MODE]) - assert file2_mode == 0760, '0%o' % file2_mode + assert file2_mode == 0o760, '0%o' % file2_mode - test.chmod('file1', 0765) - test.chmod(wdir_sub_file2, 0567) + test.chmod('file1', 0o765) + test.chmod(wdir_sub_file2, 0o567) file1_mode = stat.S_IMODE(os.stat(wdir_file1)[stat.ST_MODE]) - assert file1_mode == 0765, '0%o' % file1_mode + assert file1_mode == 0o765, '0%o' % file1_mode file2_mode = stat.S_IMODE(os.stat(wdir_sub_file2)[stat.ST_MODE]) - assert file2_mode == 0567, '0%o' % file2_mode + assert file2_mode == 0o567, '0%o' % file2_mode @@ -593,13 +596,14 @@ sys.exit(0) def test_diff_stderr_not_affecting_diff_stdout(self): """Test diff_stderr() not affecting diff_stdout() behavior""" - self.popen_python(r"""import sys + self.popen_python(r"""from __future__ import print_function +import sys sys.path = ['%s'] + sys.path import TestCmd test = TestCmd.TestCmd(diff_stderr='diff_re') -print "diff_stderr:" +print("diff_stderr:") test.diff_stderr('a\nb.\nc\n', 'a\nbb\nc\n') -print "diff_stdout:" +print("diff_stdout:") test.diff_stdout('a\nb.\nc\n', 'a\nbb\nc\n') sys.exit(0) """ % self.orig_cwd, @@ -699,13 +703,14 @@ sys.exit(0) def test_diff_stdout_not_affecting_diff_stderr(self): """Test diff_stdout() not affecting diff_stderr() behavior""" - self.popen_python(r"""import sys + self.popen_python(r"""from __future__ import print_function +import sys sys.path = ['%s'] + sys.path import TestCmd test = TestCmd.TestCmd(diff_stdout='diff_re') -print "diff_stdout:" +print("diff_stdout:") test.diff_stdout('a\nb.\nc\n', 'a\nbb\nc\n') -print "diff_stderr:" +print("diff_stderr:") test.diff_stderr('a\nb.\nc\n', 'a\nbb\nc\n') sys.exit(0) """ % self.orig_cwd, @@ -1032,14 +1037,14 @@ class match_exact_TestCase(TestCmdTestCase): assert test.match_exact("abcde\n", "abcde\n") assert not test.match_exact(["12345\n", "abcde\n"], ["1[0-9]*5\n", "a.*e\n"]) assert test.match_exact(["12345\n", "abcde\n"], ["12345\n", "abcde\n"]) - assert not test.match_exact(UserList.UserList(["12345\n", "abcde\n"]), + assert not test.match_exact(UserList(["12345\n", "abcde\n"]), ["1[0-9]*5\n", "a.*e\n"]) - assert test.match_exact(UserList.UserList(["12345\n", "abcde\n"]), + assert test.match_exact(UserList(["12345\n", "abcde\n"]), ["12345\n", "abcde\n"]) assert not test.match_exact(["12345\n", "abcde\n"], - UserList.UserList(["1[0-9]*5\n", "a.*e\n"])) + UserList(["1[0-9]*5\n", "a.*e\n"])) assert test.match_exact(["12345\n", "abcde\n"], - UserList.UserList(["12345\n", "abcde\n"])) + UserList(["12345\n", "abcde\n"])) assert not test.match_exact("12345\nabcde\n", "1[0-9]*5\na.*e\n") assert test.match_exact("12345\nabcde\n", "12345\nabcde\n") lines = ["vwxyz\n", "67890\n"] @@ -1098,28 +1103,28 @@ sys.exit(0) ["1.*j\n"]) assert test.match_re_dotall(["12345\n", "abcde\n", "fghij\n"], ["12345\n", "abcde\n", "fghij\n"]) - assert test.match_re_dotall(UserList.UserList(["12345\n", - "abcde\n", - "fghij\n"]), + assert test.match_re_dotall(UserList(["12345\n", + "abcde\n", + "fghij\n"]), ["1[0-9]*5\n", "a.*e\n", "f.*j\n"]) - assert test.match_re_dotall(UserList.UserList(["12345\n", - "abcde\n", - "fghij\n"]), + assert test.match_re_dotall(UserList(["12345\n", + "abcde\n", + "fghij\n"]), ["1.*j\n"]) - assert test.match_re_dotall(UserList.UserList(["12345\n", - "abcde\n", - "fghij\n"]), + assert test.match_re_dotall(UserList(["12345\n", + "abcde\n", + "fghij\n"]), ["12345\n", "abcde\n", "fghij\n"]) assert test.match_re_dotall(["12345\n", "abcde\n", "fghij\n"], - UserList.UserList(["1[0-9]*5\n", - "a.*e\n", - "f.*j\n"])) + UserList(["1[0-9]*5\n", + "a.*e\n", + "f.*j\n"])) assert test.match_re_dotall(["12345\n", "abcde\n", "fghij\n"], - UserList.UserList(["1.*j\n"])) + UserList(["1.*j\n"])) assert test.match_re_dotall(["12345\n", "abcde\n", "fghij\n"], - UserList.UserList(["12345\n", - "abcde\n", - "fghij\n"])) + UserList(["12345\n", + "abcde\n", + "fghij\n"])) assert test.match_re_dotall("12345\nabcde\nfghij\n", "1[0-9]*5\na.*e\nf.*j\n") assert test.match_re_dotall("12345\nabcde\nfghij\n", "1.*j\n") @@ -1176,14 +1181,14 @@ sys.exit(0) assert test.match_re("abcde\n", "abcde\n") assert test.match_re(["12345\n", "abcde\n"], ["1[0-9]*5\n", "a.*e\n"]) assert test.match_re(["12345\n", "abcde\n"], ["12345\n", "abcde\n"]) - assert test.match_re(UserList.UserList(["12345\n", "abcde\n"]), + assert test.match_re(UserList(["12345\n", "abcde\n"]), ["1[0-9]*5\n", "a.*e\n"]) - assert test.match_re(UserList.UserList(["12345\n", "abcde\n"]), + assert test.match_re(UserList(["12345\n", "abcde\n"]), ["12345\n", "abcde\n"]) assert test.match_re(["12345\n", "abcde\n"], - UserList.UserList(["1[0-9]*5\n", "a.*e\n"])) + UserList(["1[0-9]*5\n", "a.*e\n"])) assert test.match_re(["12345\n", "abcde\n"], - UserList.UserList(["12345\n", "abcde\n"])) + UserList(["12345\n", "abcde\n"])) assert test.match_re("12345\nabcde\n", "1[0-9]*5\na.*e\n") assert test.match_re("12345\nabcde\n", "12345\nabcde\n") lines = ["vwxyz\n", "67890\n"] @@ -1463,7 +1468,7 @@ class preserve_TestCase(TestCmdTestCase): def test_preserve(self): """Test preserve()""" def cleanup_test(test, cond=None, stdout=""): - io = StringIO.StringIO() + io = StringIO() save = sys.stdout sys.stdout = io try: @@ -1603,7 +1608,7 @@ class read_TestCase(TestCmdTestCase): _file_matches(wdir_foo_file3, test.read(['foo', 'file3']), "Test\nfile\n#3.\n") _file_matches(wdir_foo_file3, - test.read(UserList.UserList(['foo', 'file3'])), + test.read(UserList(['foo', 'file3'])), "Test\nfile\n#3.\n") _file_matches(wdir_file4, test.read('file4', mode = 'r'), "Test\nfile\n#4.\n") @@ -1861,24 +1866,24 @@ class run_verbose_TestCase(TestCmdTestCase): interpreter = 'python', workdir = '', verbose = 1) - - sys.stdout = StringIO.StringIO() - sys.stderr = StringIO.StringIO() - + + sys.stdout = StringIO() + sys.stderr = StringIO() + test.run(arguments = ['arg1 arg2']) o = sys.stdout.getvalue() assert o == '', o e = sys.stderr.getvalue() expect = 'python "%s" "arg1 arg2"\n' % t.script_path assert expect == e, (expect, e) - + testx = TestCmd.TestCmd(program = t.scriptx, workdir = '', verbose = 1) - - sys.stdout = StringIO.StringIO() - sys.stderr = StringIO.StringIO() - + + sys.stdout = StringIO() + sys.stderr = StringIO() + testx.run(arguments = ['arg1 arg2']) expect = '"%s" "arg1 arg2"\n' % t.scriptx_path o = sys.stdout.getvalue() @@ -1912,10 +1917,10 @@ class run_verbose_TestCase(TestCmdTestCase): interpreter = 'python', workdir = '', verbose = 2) - - sys.stdout = StringIO.StringIO() - sys.stderr = StringIO.StringIO() - + + sys.stdout = StringIO() + sys.stderr = StringIO() + test.run(arguments = ['arg1 arg2']) line_fmt = "script: %s: %s: ['arg1 arg2']\n" @@ -1929,14 +1934,14 @@ class run_verbose_TestCase(TestCmdTestCase): expect = 'python "%s" "arg1 arg2"\n' % t.script_path e = sys.stderr.getvalue() assert e == expect, (e, expect) - + testx = TestCmd.TestCmd(program = t.scriptx, workdir = '', verbose = 2) - - sys.stdout = StringIO.StringIO() - sys.stderr = StringIO.StringIO() - + + sys.stdout = StringIO() + sys.stderr = StringIO() + testx.run(arguments = ['arg1 arg2']) line_fmt = "scriptx.bat: %s: %s: ['arg1 arg2']\n" @@ -1957,10 +1962,10 @@ class run_verbose_TestCase(TestCmdTestCase): interpreter = 'python', workdir = '', verbose = 2) - - sys.stdout = StringIO.StringIO() - sys.stderr = StringIO.StringIO() - + + sys.stdout = StringIO() + sys.stderr = StringIO() + test.run(arguments = ['arg1 arg2']) line_fmt = "scriptout: %s: %s: ['arg1 arg2']\n" @@ -1977,10 +1982,10 @@ class run_verbose_TestCase(TestCmdTestCase): interpreter = 'python', workdir = '', verbose = 3) - - sys.stdout = StringIO.StringIO() - sys.stderr = StringIO.StringIO() - + + sys.stdout = StringIO() + sys.stderr = StringIO() + test.run(arguments = ['arg1 arg2']) line_fmt = "scriptout: %s: %s: ['arg1 arg2']\n" @@ -2001,10 +2006,10 @@ class run_verbose_TestCase(TestCmdTestCase): test = TestCmd.TestCmd(program = t.script, interpreter = 'python', workdir = '') - - sys.stdout = StringIO.StringIO() - sys.stderr = StringIO.StringIO() - + + sys.stdout = StringIO() + sys.stderr = StringIO() + test.run(arguments = ['arg1 arg2']) line_fmt = "script: %s: %s: ['arg1 arg2']\n" @@ -2018,13 +2023,13 @@ class run_verbose_TestCase(TestCmdTestCase): expect = 'python "%s" "arg1 arg2"\n' % t.script_path e = sys.stderr.getvalue() assert e == expect, (e, expect) - + testx = TestCmd.TestCmd(program = t.scriptx, workdir = '') - - sys.stdout = StringIO.StringIO() - sys.stderr = StringIO.StringIO() - + + sys.stdout = StringIO() + sys.stderr = StringIO() + testx.run(arguments = ['arg1 arg2']) line_fmt = "scriptx.bat: %s: %s: ['arg1 arg2']\n" @@ -2047,24 +2052,24 @@ class run_verbose_TestCase(TestCmdTestCase): interpreter = 'python', workdir = '', verbose = 1) - - sys.stdout = StringIO.StringIO() - sys.stderr = StringIO.StringIO() - + + sys.stdout = StringIO() + sys.stderr = StringIO() + test.run(arguments = ['arg1 arg2']) o = sys.stdout.getvalue() assert o == '', o e = sys.stderr.getvalue() expect = 'python "%s" "arg1 arg2"\n' % t.script_path assert expect == e, (expect, e) - + testx = TestCmd.TestCmd(program = t.scriptx, workdir = '', verbose = 1) - - sys.stdout = StringIO.StringIO() - sys.stderr = StringIO.StringIO() - + + sys.stdout = StringIO() + sys.stderr = StringIO() + testx.run(arguments = ['arg1 arg2']) expect = '"%s" "arg1 arg2"\n' % t.scriptx_path o = sys.stdout.getvalue() @@ -2095,18 +2100,19 @@ sys.exit(0) def test_set_diff_function_stdout(self): """Test set_diff_function(): stdout""" - self.popen_python("""import sys + self.popen_python("""from __future__ import print_function +import sys sys.path = ['%s'] + sys.path import TestCmd test = TestCmd.TestCmd() -print "diff:" +print("diff:") test.diff("a\\n", "a\\n") -print "diff_stdout:" +print("diff_stdout:") test.diff_stdout("a\\n", "a\\n") test.set_diff_function(stdout='diff_re') -print "diff:" +print("diff:") test.diff(".\\n", "a\\n") -print "diff_stdout:" +print("diff_stdout:") test.diff_stdout(".\\n", "a\\n") sys.exit(0) """ % self.orig_cwd, @@ -2123,18 +2129,19 @@ diff_stdout: def test_set_diff_function_stderr(self): """Test set_diff_function(): stderr """ - self.popen_python("""import sys + self.popen_python("""from __future__ import print_function +import sys sys.path = ['%s'] + sys.path import TestCmd test = TestCmd.TestCmd() -print "diff:" +print("diff:") test.diff("a\\n", "a\\n") -print "diff_stderr:" +print("diff_stderr:") test.diff_stderr("a\\n", "a\\n") test.set_diff_function(stderr='diff_re') -print "diff:" +print("diff:") test.diff(".\\n", "a\\n") -print "diff_stderr:" +print("diff_stderr:") test.diff_stderr(".\\n", "a\\n") sys.exit(0) """ % self.orig_cwd, @@ -2347,7 +2354,7 @@ while 1: logfp.close() """ % t.recv_out_path t.run_env.write(t.recv_script_path, text) - os.chmod(t.recv_script_path, 0644) # XXX UNIX-specific + os.chmod(t.recv_script_path, 0o644) # XXX UNIX-specific return t def test_start(self): @@ -2695,9 +2702,10 @@ class stdin_TestCase(TestCmdTestCase): def test_stdin(self): """Test stdin()""" run_env = TestCmd.TestCmd(workdir = '') - run_env.write('run', """import fileinput + run_env.write('run', """from __future__ import print_function +import fileinput for line in fileinput.input(): - print 'Y'.join(line[:-1].split('X')) + print('Y'.join(line[:-1].split('X'))) """) run_env.write('input', "X on X this X line X\n") os.chdir(run_env.workdir) @@ -2765,11 +2773,11 @@ class subdir_TestCase(TestCmdTestCase): assert test.subdir('bar') == 1 assert test.subdir(['foo', 'succeed']) == 1 if os.name != "nt": - os.chmod(test.workpath('foo'), 0500) + os.chmod(test.workpath('foo'), 0o500) assert test.subdir(['foo', 'fail']) == 0 assert test.subdir(['sub', 'dir', 'ectory'], 'sub') == 1 assert test.subdir('one', - UserList.UserList(['one', 'two']), + UserList(['one', 'two']), ['one', 'two', 'three']) == 3 assert os.path.isdir(test.workpath('foo')) assert os.path.isdir(test.workpath('bar')) @@ -2962,7 +2970,7 @@ class unlink_TestCase(TestCmdTestCase): test.unlink(['foo', 'file3a']) assert not os.path.exists(wdir_foo_file3a) - test.unlink(UserList.UserList(['foo', 'file3b'])) + test.unlink(UserList(['foo', 'file3b'])) assert not os.path.exists(wdir_foo_file3b) test.unlink([test.workdir, 'foo', 'file4']) @@ -2971,8 +2979,8 @@ class unlink_TestCase(TestCmdTestCase): # Make it so we can't unlink file5. # For UNIX, remove write permission from the dir and the file. # For Windows, open the file. - os.chmod(test.workdir, 0500) - os.chmod(wdir_file5, 0400) + os.chmod(test.workdir, 0o500) + os.chmod(wdir_file5, 0o400) f = open(wdir_file5, 'r') try: @@ -2983,8 +2991,8 @@ class unlink_TestCase(TestCmdTestCase): except: raise finally: - os.chmod(test.workdir, 0700) - os.chmod(wdir_file5, 0600) + os.chmod(test.workdir, 0o700) + os.chmod(wdir_file5, 0o600) f.close() @@ -3208,11 +3216,11 @@ class executable_TestCase(TestCmdTestCase): def make_executable(fname): st = os.stat(fname) - os.chmod(fname, stat.S_IMODE(st[stat.ST_MODE]|0100)) + os.chmod(fname, stat.S_IMODE(st[stat.ST_MODE]|0o100)) def make_non_executable(fname): st = os.stat(fname) - os.chmod(fname, stat.S_IMODE(st[stat.ST_MODE]&~0100)) + os.chmod(fname, stat.S_IMODE(st[stat.ST_MODE]&~0o100)) test.executable(test.workdir, 0) # XXX skip these tests if euid == 0? @@ -3282,7 +3290,7 @@ class write_TestCase(TestCmdTestCase): test.write('file9', "Test file #9.\r\n", mode = 'wb') if os.name != "nt": - os.chmod(test.workdir, 0500) + os.chmod(test.workdir, 0o500) try: test.write('file10', "Test file #10 (should not get created).\n") except IOError: # expect "Permission denied" @@ -3326,14 +3334,16 @@ class variables_TestCase(TestCmdTestCase): 'TestCmd', ] - script = "import TestCmd\n" + \ - '\n'.join([ "print TestCmd.%s\n" % v for v in variables ]) + script = "from __future__ import print_function\n" + \ + "import TestCmd\n" + \ + '\n'.join([ "print(TestCmd.%s\n)" % v for v in variables ]) run_env.run(program=sys.executable, stdin=script) stderr = run_env.stderr() assert stderr == "", stderr - script = "from TestCmd import *\n" + \ - '\n'.join([ "print %s" % v for v in variables ]) + script = "from __future__ import print_function\n" + \ + "from TestCmd import *\n" + \ + '\n'.join([ "print(%s)" % v for v in variables ]) run_env.run(program=sys.executable, stdin=script) stderr = run_env.stderr() assert stderr == "", stderr diff --git a/QMTest/TestCommon.py b/QMTest/TestCommon.py index dc4c97c..a475ddc 100644 --- a/QMTest/TestCommon.py +++ b/QMTest/TestCommon.py @@ -93,6 +93,8 @@ The TestCommon module also provides the following variables # AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, # SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. +from __future__ import print_function + __author__ = "Steven Knight <knight at baldmt dot com>" __revision__ = "TestCommon.py 1.3.D001 2010/06/03 12:58:27 knight" __version__ = "1.3" @@ -258,9 +260,9 @@ class TestCommon(TestCmd): existing, missing = separate_files(files) unwritable = [x for x in existing if not is_writable(x)] if missing: - print "Missing files: `%s'" % "', `".join(missing) + print("Missing files: `%s'" % "', `".join(missing)) if unwritable: - print "Unwritable files: `%s'" % "', `".join(unwritable) + print("Unwritable files: `%s'" % "', `".join(unwritable)) self.fail_test(missing + unwritable) def must_contain(self, file, required, mode = 'rb', find = None): @@ -275,11 +277,11 @@ class TestCommon(TestCmd): return None contains = find(file_contents, required) if not contains: - print "File `%s' does not contain required string." % file - print self.banner('Required string ') - print required - print self.banner('%s contents ' % file) - print file_contents + print("File `%s' does not contain required string." % file) + print(self.banner('Required string ')) + print(required) + print(self.banner('%s contents ' % file)) + print(file_contents) self.fail_test(not contains) def must_contain_all(self, output, input, title=None, find=None): @@ -306,10 +308,10 @@ class TestCommon(TestCmd): if find(output, input) is None: if title is None: title = 'output' - print 'Missing expected input from %s:' % title - print input - print self.banner(title + ' ') - print output + print('Missing expected input from {}:'.format(title)) + print(input) + print(self.banner(title + ' ')) + print(output) self.fail_test() def must_contain_all_lines(self, output, lines, title=None, find=None): @@ -444,7 +446,7 @@ class TestCommon(TestCmd): files = [is_List(x) and os.path.join(*x) or x for x in files] missing = [x for x in files if not os.path.exists(x) and not os.path.islink(x) ] if missing: - print "Missing files: `%s'" % "', `".join(missing) + print("Missing files: `%s'" % "', `".join(missing)) self.fail_test(missing) def must_exist_one_of(self, files): @@ -464,24 +466,24 @@ class TestCommon(TestCmd): if glob.glob(xpath): return missing.append(xpath) - print "Missing one of: `%s'" % "', `".join(missing) + print("Missing one of: `%s'" % "', `".join(missing)) self.fail_test(missing) - def must_match(self, file, expect, mode = 'rb', match=None): + def must_match(self, file, expect, mode = 'rb', match=None, message=None, newline=None): """Matches the contents of the specified file (first argument) against the expected contents (second argument). The expected contents are a list of lines or a string which will be split on newlines. """ - file_contents = self.read(file, mode) + file_contents = self.read(file, mode, newline) if not match: match = self.match try: - self.fail_test(not match(file_contents, expect)) + self.fail_test(not match(to_str(file_contents), to_str(expect)), message=message) except KeyboardInterrupt: raise except: - print "Unexpected contents of `%s'" % file + print("Unexpected contents of `%s'" % file) self.diff(expect, file_contents, 'contents ') raise @@ -497,11 +499,11 @@ class TestCommon(TestCmd): return None contains = find(file_contents, banned) if contains: - print "File `%s' contains banned string." % file - print self.banner('Banned string ') - print banned - print self.banner('%s contents ' % file) - print file_contents + print("File `%s' contains banned string." % file) + print(self.banner('Banned string ')) + print(banned) + print(self.banner('%s contents ' % file)) + print(file_contents) self.fail_test(contains) def must_not_contain_any_line(self, output, lines, title=None, find=None): @@ -548,7 +550,7 @@ class TestCommon(TestCmd): files = [is_List(x) and os.path.join(*x) or x for x in files] existing = [x for x in files if os.path.exists(x) or os.path.islink(x)] if existing: - print "Unexpected files exist: `%s'" % "', `".join(existing) + print("Unexpected files exist: `%s'" % "', `".join(existing)) self.fail_test(existing) def must_not_exist_any_of(self, files): @@ -568,7 +570,7 @@ class TestCommon(TestCmd): if glob.glob(xpath): existing.append(xpath) if existing: - print "Unexpected files exist: `%s'" % "', `".join(existing) + print("Unexpected files exist: `%s'" % "', `".join(existing)) self.fail_test(existing) def must_not_be_writable(self, *files): @@ -580,11 +582,11 @@ class TestCommon(TestCmd): """ files = [is_List(x) and os.path.join(*x) or x for x in files] existing, missing = separate_files(files) - writable = list(filter(is_writable, existing)) + writable = [file for file in existing if is_writable(file)] if missing: - print "Missing files: `%s'" % "', `".join(missing) + print("Missing files: `%s'" % "', `".join(missing)) if writable: - print "Writable files: `%s'" % "', `".join(writable) + print("Writable files: `%s'" % "', `".join(writable)) self.fail_test(missing + writable) def _complete(self, actual_stdout, expected_stdout, @@ -597,23 +599,23 @@ class TestCommon(TestCmd): expect = '' if status != 0: expect = " (expected %s)" % str(status) - print "%s returned %s%s" % (self.program, _status(self), expect) - print self.banner('STDOUT ') - print actual_stdout - print self.banner('STDERR ') - print actual_stderr + print("%s returned %s%s" % (self.program, _status(self), expect)) + print(self.banner('STDOUT ')) + print(actual_stdout) + print(self.banner('STDERR ')) + print(actual_stderr) self.fail_test() if (expected_stdout is not None and not match(actual_stdout, expected_stdout)): self.diff(expected_stdout, actual_stdout, 'STDOUT ') if actual_stderr: - print self.banner('STDERR ') - print actual_stderr + print(self.banner('STDERR ')) + print(actual_stderr) self.fail_test() if (expected_stderr is not None and not match(actual_stderr, expected_stderr)): - print self.banner('STDOUT ') - print actual_stdout + print(self.banner('STDOUT ')) + print(actual_stdout) self.diff(expected_stderr, actual_stderr, 'STDERR ') self.fail_test() @@ -633,15 +635,15 @@ class TestCommon(TestCmd): universal_newlines, **kw) except KeyboardInterrupt: raise - except Exception, e: - print self.banner('STDOUT ') + except Exception as e: + print(self.banner('STDOUT ')) try: - print self.stdout() + print(self.stdout()) except IndexError: pass - print self.banner('STDERR ') + print(self.banner('STDERR ')) try: - print self.stderr() + print(self.stderr()) except IndexError: pass cmd_args = self.command_args(program, interpreter, arguments) diff --git a/QMTest/TestCommonTests.py b/QMTest/TestCommonTests.py index 30b7d6a..7949cb8 100644 --- a/QMTest/TestCommonTests.py +++ b/QMTest/TestCommonTests.py @@ -168,10 +168,11 @@ class __init__TestCase(TestCommonTestCase): os.chdir(run_env.workdir) script = lstrip("""\ + from __future__ import print_function from TestCommon import TestCommon tc = TestCommon(workdir='') import os - print os.getcwd() + print(os.getcwd()) """) run_env.run(program=sys.executable, stdin=script) stdout = run_env.stdout()[:-1] @@ -2285,14 +2286,16 @@ class variables_TestCase(TestCommonTestCase): 'dll_suffix', ] - script = "import TestCommon\n" + \ - '\n'.join([ "print TestCommon.%s\n" % v for v in variables ]) + script = "from __future__ import print_function" + \ + "import TestCommon\n" + \ + '\n'.join([ "print(TestCommon.%s)\n" % v for v in variables ]) run_env.run(program=sys.executable, stdin=script) stderr = run_env.stderr() assert stderr == "", stderr - script = "from TestCommon import *\n" + \ - '\n'.join([ "print %s" % v for v in variables ]) + script = "from __future__ import print_function" + \ + "from TestCommon import *\n" + \ + '\n'.join([ "print(%s)" % v for v in variables ]) run_env.run(program=sys.executable, stdin=script) stderr = run_env.stderr() assert stderr == "", stderr diff --git a/QMTest/TestRuntest.py b/QMTest/TestRuntest.py index 4ba11ea..f2256de 100644 --- a/QMTest/TestRuntest.py +++ b/QMTest/TestRuntest.py @@ -12,9 +12,9 @@ from those classes, as well as any overridden or additional methods or attributes defined in this subclass. """ -# Copyright (c) 2001 - 2016 The SCons Foundation +# Copyright (c) 2001 - 2017 The SCons Foundation -__revision__ = "QMTest/TestRuntest.py rel_2.5.1:3735:9dc6cee5c168 2016/11/03 14:02:02 bdbaddog" +__revision__ = "QMTest/TestRuntest.py rel_3.0.0:4395:8972f6a2f699 2017/09/18 12:59:24 bdbaddog" import os import os.path diff --git a/QMTest/TestSCons.py b/QMTest/TestSCons.py index 1161d50..c0b4ebc 100644 --- a/QMTest/TestSCons.py +++ b/QMTest/TestSCons.py @@ -12,16 +12,17 @@ from those classes, as well as any overridden or additional methods or attributes defined in this subclass. """ -# Copyright (c) 2001 - 2016 The SCons Foundation -from __future__ import division +# Copyright (c) 2001 - 2017 The SCons Foundation +from __future__ import division, print_function -__revision__ = "QMTest/TestSCons.py rel_2.5.1:3735:9dc6cee5c168 2016/11/03 14:02:02 bdbaddog" +__revision__ = "QMTest/TestSCons.py rel_3.0.0:4395:8972f6a2f699 2017/09/18 12:59:24 bdbaddog" import os import re import shutil import sys import time +import subprocess from TestCommon import * from TestCommon import __all__ @@ -34,7 +35,7 @@ from TestCmd import PIPE # here provides some independent verification that what we packaged # conforms to what we expect. -default_version = '2.5.1' +default_version = '3.0.0' python_version_unsupported = (2, 6, 0) python_version_deprecated = (2, 7, 0) @@ -43,21 +44,22 @@ python_version_deprecated = (2, 7, 0) # line must remain "__ VERSION __" (without the spaces) so the built # version in build/QMTest/TestSCons.py contains the actual version # string of the packages that have been built. -SConsVersion = '2.5.1' +SConsVersion = '3.0.0' if SConsVersion == '__' + 'VERSION' + '__': SConsVersion = default_version -__all__.extend([ 'TestSCons', - 'machine', - 'python', - '_exe', - '_obj', - '_shobj', - 'shobj_', - 'lib_', - '_lib', - 'dll_', - '_dll' +__all__.extend([ + 'TestSCons', + 'machine', + 'python', + '_exe', + '_obj', + '_shobj', + 'shobj_', + 'lib_', + '_lib', + 'dll_', + '_dll' ]) machine_map = { @@ -256,7 +258,7 @@ class TestSCons(TestCommon): # TERM can cause test failures due to control chars in prompts etc. os.environ['TERM'] = 'dumb' - self.ignore_python_version = kw.get('ignore_python_version',1) + self.ignore_python_version = kw.get('ignore_python_version', 1) if kw.get('ignore_python_version', -1) != -1: del kw['ignore_python_version'] @@ -268,7 +270,7 @@ class TestSCons(TestCommon): SCons.Node.FS.default_fs = SCons.Node.FS.FS() try: - self.script_srcdir = os.environ['PYTHON_SCRIPT_DIR'] + self.fixture_dirs = (os.environ['FIXTURE_DIRS']).split(os.pathsep) except KeyError: pass @@ -355,7 +357,7 @@ class TestSCons(TestCommon): # raised so as to not mask possibly serious disk or # network issues. continue - if stat.S_IMODE(st[stat.ST_MODE]) & 0111: + if stat.S_IMODE(st[stat.ST_MODE]) & 0o111: return os.path.normpath(f) else: import SCons.Environment @@ -502,9 +504,9 @@ class TestSCons(TestCommon): self.pass_test() else: # test failed; have to do this by hand... - print self.banner('STDOUT ') - print self.stdout() - print self.diff(warning, stderr, 'STDERR ') + print(self.banner('STDOUT ')) + print(self.stdout()) + print(self.diff(warning, stderr, 'STDERR ')) self.fail_test() return warning @@ -562,21 +564,26 @@ class TestSCons(TestCommon): Returns a Python error line for output comparisons. The exec of the traceback line gives us the correct format for - this version of Python. Before 2.5, this yielded: - - File "<string>", line 1, ? - - Python 2.5 changed this to: + this version of Python. File "<string>", line 1, <module> We stick the requested file name and line number in the right places, abstracting out the version difference. """ - exec 'import traceback; x = traceback.format_stack()[-1]' - x = x.lstrip() - x = x.replace('<string>', file) - x = x.replace('line 1,', 'line %s,' % line) + # This routine used to use traceback to get the proper format + # that doesn't work well with py3. And the format of the + # traceback seems to be stable, so let's just format + # an appropriate string + # + #exec('import traceback; x = traceback.format_stack()[-1]') + # import traceback + # x = traceback.format_stack() + # x = # XXX: .lstrip() + # x = x.replace('<string>', file) + # x = x.replace('line 1,', 'line %s,' % line) + # x="\n".join(x) + x='File "%s", line %s, in <module>\n'%(file,line) return x def normalize_ps(self, s): @@ -591,23 +598,33 @@ class TestSCons(TestCommon): return s + @staticmethod + def to_bytes_re_sub(pattern, repl, str, count=0, flags=0): + """ + Wrapper around re.sub to change pattern and repl to bytes to work with + both python 2 & 3 + """ + pattern = to_bytes(pattern) + repl = to_bytes(repl) + return re.sub(pattern, repl, str, count, flags) + def normalize_pdf(self, s): - s = re.sub(r'/(Creation|Mod)Date \(D:[^)]*\)', - r'/\1Date (D:XXXX)', s) - s = re.sub(r'/ID \[<[0-9a-fA-F]*> <[0-9a-fA-F]*>\]', - r'/ID [<XXXX> <XXXX>]', s) - s = re.sub(r'/(BaseFont|FontName) /[A-Z]{6}', - r'/\1 /XXXXXX', s) - s = re.sub(r'/Length \d+ *\n/Filter /FlateDecode\n', - r'/Length XXXX\n/Filter /FlateDecode\n', s) + s = self.to_bytes_re_sub(r'/(Creation|Mod)Date \(D:[^)]*\)', + r'/\1Date (D:XXXX)', s) + s = self.to_bytes_re_sub(r'/ID \[<[0-9a-fA-F]*> <[0-9a-fA-F]*>\]', + r'/ID [<XXXX> <XXXX>]', s) + s = self.to_bytes_re_sub(r'/(BaseFont|FontName) /[A-Z]{6}', + r'/\1 /XXXXXX', s) + s = self.to_bytes_re_sub(r'/Length \d+ *\n/Filter /FlateDecode\n', + r'/Length XXXX\n/Filter /FlateDecode\n', s) try: import zlib except ImportError: pass else: - begin_marker = '/FlateDecode\n>>\nstream\n' - end_marker = 'endstream\nendobj' + begin_marker = to_bytes('/FlateDecode\n>>\nstream\n') + end_marker = to_bytes('endstream\nendobj') encoded = [] b = s.find(begin_marker, 0) @@ -622,16 +639,16 @@ class TestSCons(TestCommon): for b, e in encoded: r.append(s[x:b]) d = zlib.decompress(s[b:e]) - d = re.sub(r'%%CreationDate: [^\n]*\n', - r'%%CreationDate: 1970 Jan 01 00:00:00\n', d) - d = re.sub(r'%DVIPSSource: TeX output \d\d\d\d\.\d\d\.\d\d:\d\d\d\d', - r'%DVIPSSource: TeX output 1970.01.01:0000', d) - d = re.sub(r'/(BaseFont|FontName) /[A-Z]{6}', - r'/\1 /XXXXXX', d) + d = self.to_bytes_re_sub(r'%%CreationDate: [^\n]*\n', + r'%%CreationDate: 1970 Jan 01 00:00:00\n', d) + d = self.to_bytes_re_sub(r'%DVIPSSource: TeX output \d\d\d\d\.\d\d\.\d\d:\d\d\d\d', + r'%DVIPSSource: TeX output 1970.01.01:0000', d) + d = self.to_bytes_re_sub(r'/(BaseFont|FontName) /[A-Z]{6}', + r'/\1 /XXXXXX', d) r.append(d) x = e r.append(s[x:]) - s = ''.join(r) + s = to_bytes('').join(r) return s @@ -642,6 +659,21 @@ class TestSCons(TestCommon): result.extend(sorted(glob.glob(p))) return result + def unlink_sconsignfile(self,name='.sconsign.dblite'): + """ + Delete sconsign file. + Note on python it seems to append .p3 to the file name so we take care of that + Parameters + ---------- + name - expected name of sconsign file + + Returns + ------- + None + """ + if sys.version_info[0] == 3: + name += '.p3' + self.unlink(name) def java_ENV(self, version=None): """ @@ -685,12 +717,20 @@ class TestSCons(TestCommon): """ Return java include paths compiling java jni code """ - import glob import sys + + result = [] + if sys.platform[:6] == 'darwin': + java_home = self.java_where_java_home(version) + jni_path = os.path.join(java_home,'include','jni.h') + if os.path.exists(jni_path): + result.append(os.path.dirname(jni_path)) + if not version: version='' jni_dirs = ['/System/Library/Frameworks/JavaVM.framework/Headers/jni.h', - '/usr/lib/jvm/default-java/include/jni.h'] + '/usr/lib/jvm/default-java/include/jni.h', + '/usr/lib/jvm/java-*-oracle/include/jni.h'] else: jni_dirs = ['/System/Library/Frameworks/JavaVM.framework/Versions/%s*/Headers/jni.h'%version] jni_dirs.extend(['/usr/lib/jvm/java-*-sun-%s*/include/jni.h'%version, @@ -700,7 +740,7 @@ class TestSCons(TestCommon): if not dirs: return None d=os.path.dirname(self.paths(jni_dirs)[0]) - result=[d] + result.append(d) if sys.platform == 'win32': result.append(os.path.join(d,'win32')) @@ -708,16 +748,34 @@ class TestSCons(TestCommon): result.append(os.path.join(d,'linux')) return result - - def java_where_java_home(self,version=None): + def java_where_java_home(self, version=None): if sys.platform[:6] == 'darwin': + # osx 10.11, 10.12 + home_tool = '/usr/libexec/java_home' + java_home = False + if os.path.exists(home_tool): + java_home = subprocess.check_output(home_tool).strip() + java_home = java_home.decode() + if version is None: - home = '/System/Library/Frameworks/JavaVM.framework/Home' + if java_home: + return java_home + else: + homes = ['/System/Library/Frameworks/JavaVM.framework/Home', + # osx 10.10 + '/System/Library/Frameworks/JavaVM.framework/Versions/Current/Home'] + for home in homes: + if os.path.exists(home): + return home + else: - home = '/System/Library/Frameworks/JavaVM.framework/Versions/%s/Home' % version - if not os.path.exists(home): - # This works on OSX 10.10 - home = '/System/Library/Frameworks/JavaVM.framework/Versions/Current/' + if java_home.find('jdk%s'%version) != -1: + return java_home + else: + home = '/System/Library/Frameworks/JavaVM.framework/Versions/%s/Home' % version + if not os.path.exists(home): + # This works on OSX 10.10 + home = '/System/Library/Frameworks/JavaVM.framework/Versions/Current/' else: jar = self.java_where_jar(version) home = os.path.normpath('%s/..'%jar) @@ -823,12 +881,12 @@ output = None impl = 0 opt_string = '' for opt, arg in cmd_opts: - if opt == '-o': output = open(arg, 'wb') + if opt == '-o': output = open(arg, 'w') elif opt == '-i': impl = 1 else: opt_string = opt_string + ' ' + opt output.write("/* mymoc.py%s */\\n" % opt_string) for a in args: - contents = open(a, 'rb').read() + contents = open(a, 'r').read() a = a.replace('\\\\', '\\\\\\\\') subst = r'{ my_qt_symbol( "' + a + '\\\\n" ); }' if impl: @@ -849,7 +907,7 @@ source = None opt_string = '' for arg in sys.argv[1:]: if output_arg: - output = open(arg, 'wb') + output = open(arg, 'w') output_arg = 0 elif impl_arg: impl = arg @@ -863,7 +921,7 @@ for arg in sys.argv[1:]: else: if source: sys.exit(1) - source = open(arg, 'rb') + source = open(arg, 'r') sourceFile = arg output.write("/* myuic.py%s */\\n" % opt_string) if impl: @@ -936,7 +994,7 @@ if ARGUMENTS.get('variant_dir', 0): else: builddir = 'build' VariantDir(builddir, '.', duplicate=dup) - print builddir, dup + print(builddir, dup) sconscript = Dir(builddir).File('SConscript') else: sconscript = File('SConscript') @@ -958,32 +1016,6 @@ SConscript( sconscript ) # to use cygwin compilers on cmd.exe -> uncomment following line #Configure_lib = 'm' - def gccFortranLibs(self): - """Test which gcc Fortran startup libraries are required. - This should probably move into SCons itself, but is kind of hacky. - """ - if sys.platform.find('irix') != -1: - return ['ftn'] - - libs = ['g2c'] - cmd = ['gcc','-v'] - - try: - p = Popen(cmd, stdout=PIPE, stderr=PIPE) - stdout, stderr = p.communicate() - except: - return libs - - m = re.search('(gcc\s+version|gcc-Version)\s+(\d\.\d)', stderr) - if m: - gcc_version = m.group(2) - if re.match('4.[^0]', gcc_version): - libs = ['gfortranbegin'] - elif gcc_version in ('3.1', '4.0'): - libs = ['frtbegin'] + libs - - return libs - def skip_if_not_msvc(self, check_platform=True): """ Check whether we are on a Windows platform and skip the test if not. This check can be omitted by setting @@ -1009,53 +1041,113 @@ SConscript( sconscript ) def checkLogAndStdout(self, checks, results, cached, logfile, sconf_dir, sconstruct, - doCheckLog=1, doCheckStdout=1): + doCheckLog=True, doCheckStdout=True): + """ + Used to verify the expected output from using Configure() + via the contents of one or both of stdout or config.log file. + The checks, results, cached parameters all are zipped together + for use in comparing results. + + TODO: Perhaps a better API makes sense? + + Parameters + ---------- + checks : The Configure checks being run + + results : The expected results for each check + + cached : If the corresponding check is expected to be cached + + logfile : Name of the config log + + sconf_dir : Name of the sconf dir + + sconstruct : SConstruct file name + + doCheckLog : check specified log file, defaults to true + + doCheckStdout : Check stdout, defaults to true + + Returns + ------- + + """ class NoMatch(Exception): def __init__(self, p): self.pos = p def matchPart(log, logfile, lastEnd, NoMatch=NoMatch): + """ + Match part of the logfile + """ m = re.match(log, logfile[lastEnd:]) if not m: raise NoMatch(lastEnd) return m.end() + lastEnd + try: - #print len(os.linesep) - ls = os.linesep - nols = "(" - for i in range(len(ls)): - nols = nols + "(" - for j in range(i): - nols = nols + ls[j] - nols = nols + "[^" + ls[i] + "])" - if i < len(ls)-1: - nols = nols + "|" - nols = nols + ")" + + # Build regexp for a character which is not + # a linesep, and in the case of CR/LF + # build it with both CR and CR/LF + # TODO: Not sure why this is a good idea. A static string + # could do the same since we only have two variations + # to do with? + # ls = os.linesep + # nols = "(" + # for i in range(len(ls)): + # nols = nols + "(" + # for j in range(i): + # nols = nols + ls[j] + # nols = nols + "[^" + ls[i] + "])" + # if i < len(ls)-1: + # nols = nols + "|" + # nols = nols + ")" + # + # Replaced above logic with \n as we're reading the file + # using non-binary read. Python will translate \r\n -> \n + # For us. + ls = '\n' + nols = '([^\n])' lastEnd = 0 - logfile = self.read(self.workpath(logfile)) + + # Read the whole logfile + logfile = self.read(self.workpath(logfile), mode='r') + + # Some debug code to keep around.. + # sys.stderr.write("LOGFILE[%s]:%s"%(type(logfile),logfile)) + if (doCheckLog and - logfile.find( "scons: warning: The stored build " - "information has an unexpected class." ) >= 0): + logfile.find("scons: warning: The stored build information has an unexpected class.") >= 0): self.fail_test() + sconf_dir = sconf_dir sconstruct = sconstruct log = r'file\ \S*%s\,line \d+:' % re.escape(sconstruct) + ls - if doCheckLog: lastEnd = matchPart(log, logfile, lastEnd) + if doCheckLog: + lastEnd = matchPart(log, logfile, lastEnd) + log = "\t" + re.escape("Configure(confdir = %s)" % sconf_dir) + ls - if doCheckLog: lastEnd = matchPart(log, logfile, lastEnd) + if doCheckLog: + lastEnd = matchPart(log, logfile, lastEnd) + rdstr = "" cnt = 0 for check,result,cache_desc in zip(checks, results, cached): log = re.escape("scons: Configure: " + check) + ls - if doCheckLog: lastEnd = matchPart(log, logfile, lastEnd) + + if doCheckLog: + lastEnd = matchPart(log, logfile, lastEnd) + log = "" result_cached = 1 for bld_desc in cache_desc: # each TryXXX for ext, flag in bld_desc: # each file in TryBuild file = os.path.join(sconf_dir,"conftest_%d%s" % (cnt, ext)) if flag == self.NCR: + # NCR = Non Cached Rebuild # rebuild will pass if ext in ['.c', '.cpp']: log=log + re.escape(file + " <-") + ls @@ -1064,6 +1156,7 @@ SConscript( sconscript ) log=log + "(" + nols + "*" + ls +")*?" result_cached = 0 if flag == self.CR: + # CR = cached rebuild (up to date)s # up to date log=log + \ re.escape("scons: Configure: \"%s\" is up to date." @@ -1089,33 +1182,36 @@ SConscript( sconscript ) result = "(cached) " + result rdstr = rdstr + re.escape(check) + re.escape(result) + "\n" log=log + re.escape("scons: Configure: " + result) + ls + ls - if doCheckLog: lastEnd = matchPart(log, logfile, lastEnd) + + if doCheckLog: + lastEnd = matchPart(log, logfile, lastEnd) + log = "" if doCheckLog: lastEnd = matchPart(ls, logfile, lastEnd) if doCheckLog and lastEnd != len(logfile): raise NoMatch(lastEnd) - except NoMatch, m: - print "Cannot match log file against log regexp." - print "log file: " - print "------------------------------------------------------" - print logfile[m.pos:] - print "------------------------------------------------------" - print "log regexp: " - print "------------------------------------------------------" - print log - print "------------------------------------------------------" + except NoMatch as m: + print("Cannot match log file against log regexp.") + print("log file: ") + print("------------------------------------------------------") + print(logfile[m.pos:]) + print("------------------------------------------------------") + print("log regexp: ") + print("------------------------------------------------------") + print(log) + print("------------------------------------------------------") self.fail_test() if doCheckStdout: exp_stdout = self.wrap_stdout(".*", rdstr) if not self.match_re_dotall(self.stdout(), exp_stdout): - print "Unexpected stdout: " - print "-----------------------------------------------------" - print repr(self.stdout()) - print "-----------------------------------------------------" - print repr(exp_stdout) - print "-----------------------------------------------------" + print("Unexpected stdout: ") + print("-----------------------------------------------------") + print(repr(self.stdout())) + print("-----------------------------------------------------") + print(repr(exp_stdout)) + print("-----------------------------------------------------") self.fail_test() def get_python_version(self): @@ -1149,12 +1245,15 @@ except AttributeError: try: import distutils.sysconfig exec_prefix = distutils.sysconfig.EXEC_PREFIX - print distutils.sysconfig.get_python_inc() - print os.path.join(exec_prefix, 'libs') + print(distutils.sysconfig.get_python_inc()) + lib_path = os.path.join(exec_prefix, 'libs') + if not os.path.exists(lib_path): + lib_path = os.path.join(exec_prefix, 'lib') + print(lib_path) except: - print os.path.join(sys.prefix, 'include', py_ver) - print os.path.join(sys.prefix, 'lib', py_ver, 'config') -print py_ver + print(os.path.join(sys.prefix, 'include', py_ver)) + print(os.path.join(sys.prefix, 'lib', py_ver, 'config')) +print(py_ver) """) return [python] + self.stdout().strip().split('\n') @@ -1214,6 +1313,12 @@ print py_ver alt_cpp_suffix = '.C' return alt_cpp_suffix + def platform_has_symlink(self): + if not hasattr(os, 'symlink') or sys.platform == 'win32': + return False + else: + return True + class Stat: def __init__(self, name, units, expression, convert=None): diff --git a/QMTest/TestSConsMSVS.py b/QMTest/TestSConsMSVS.py index 537ba17..30eea76 100644 --- a/QMTest/TestSConsMSVS.py +++ b/QMTest/TestSConsMSVS.py @@ -13,13 +13,15 @@ as well as any overridden or additional methods or attributes defined in this subclass. """ -# Copyright (c) 2001 - 2016 The SCons Foundation +# Copyright (c) 2001 - 2017 The SCons Foundation -__revision__ = "QMTest/TestSConsMSVS.py rel_2.5.1:3735:9dc6cee5c168 2016/11/03 14:02:02 bdbaddog" +__revision__ = "QMTest/TestSConsMSVS.py rel_3.0.0:4395:8972f6a2f699 2017/09/18 12:59:24 bdbaddog" import os import sys import platform +import traceback +from xml.etree import ElementTree from TestSCons import * from TestSCons import __all__ @@ -1037,8 +1039,8 @@ class TestSConsMSVS(TestSCons): input = """\ import SCons import SCons.Tool.MSCommon -print "self.scons_version =", repr(SCons.__%s__) -print "self._msvs_versions =", str(SCons.Tool.MSCommon.query_versions()) +print("self.scons_version =", repr(SCons.__%s__)) +print("self._msvs_versions =", str(SCons.Tool.MSCommon.query_versions())) """ % 'version' self.run(arguments = '-n -q -Q -f -', stdin = input) @@ -1054,7 +1056,7 @@ print "self._msvs_versions =", str(SCons.Tool.MSCommon.query_versions()) enginepath = repr(os.path.join(self._cwd, '..', 'engine')) replace = 'sys.path = [ %s, join(sys' % enginepath - contents = self.read(fname) + contents = self.read(fname, mode='r') contents = contents.replace(orig, replace) self.write(fname, contents) @@ -1149,13 +1151,25 @@ print "self._msvs_versions =", str(SCons.Tool.MSCommon.query_versions()) try: host = _ARCH_TO_CANONICAL[host_platform] - except KeyError, e: + except KeyError as e: # Default to x86 for all other platforms host = 'x86' return host + def validate_msvs_file(self, file): + try: + x = ElementTree.parse(file) + except: + print("--------------------------------------------------------------") + print("--------------------------------------------------------------") + print(traceback.format_exc()) + print("Failed to validate xml in MSVS file: ") + print(file) + print("--------------------------------------------------------------") + print("--------------------------------------------------------------") + self.fail_test() # Local Variables: # tab-width:4 # indent-tabs-mode:nil diff --git a/QMTest/TestSCons_time.py b/QMTest/TestSCons_time.py index e84f7dc..f74906a 100644 --- a/QMTest/TestSCons_time.py +++ b/QMTest/TestSCons_time.py @@ -11,9 +11,9 @@ from those classes, as well as any overridden or additional methods or attributes defined in this subclass. """ -# Copyright (c) 2001 - 2016 The SCons Foundation +# Copyright (c) 2001 - 2017 The SCons Foundation -__revision__ = "QMTest/TestSCons_time.py rel_2.5.1:3735:9dc6cee5c168 2016/11/03 14:02:02 bdbaddog" +__revision__ = "QMTest/TestSCons_time.py rel_3.0.0:4395:8972f6a2f699 2017/09/18 12:59:24 bdbaddog" import os import os.path @@ -28,8 +28,9 @@ __all__.extend([ 'TestSCons_time', ]) SConstruct = """\ +from __future__ import print_function import os -print "SConstruct file directory:", os.getcwd() +print("SConstruct file directory:", os.getcwd()) """ scons_py = """\ @@ -43,12 +44,12 @@ def write_args(fp, args): write_args(sys.stdout, sys.argv) for arg in sys.argv[1:]: if arg[:10] == '--profile=': - profile = open(arg[10:], 'wb') + profile = open(arg[10:], 'w') profile.write('--profile\\n') write_args(profile, sys.argv) break sys.stdout.write('SCONS_LIB_DIR = ' + os.environ['SCONS_LIB_DIR'] + '\\n') -exec(open('SConstruct', 'rU').read()) +exec(open('SConstruct', 'r').read()) """ aegis_py = """\ @@ -225,7 +226,7 @@ class TestSCons_time(TestCommon): def write_fake_aegis_py(self, name): name = self.workpath(name) self.write(name, aegis_py) - os.chmod(name, 0755) + os.chmod(name, 0o755) return name def write_fake_scons_py(self): @@ -235,7 +236,7 @@ class TestSCons_time(TestCommon): def write_fake_svn_py(self, name): name = self.workpath(name) self.write(name, svn_py) - os.chmod(name, 0755) + os.chmod(name, 0o755) return name def write_sample_directory(self, archive, dir, files): @@ -245,7 +246,7 @@ class TestSCons_time(TestCommon): d, f = os.path.split(path) if not os.path.isdir(d): os.makedirs(d) - open(path, 'wb').write(content) + open(path, 'w').write(content) return dir def write_sample_tarfile(self, archive, dir, files): @@ -270,7 +271,7 @@ class TestSCons_time(TestCommon): tar = tarfile.open(archive, mode[suffix]) for name, content in files: path = os.path.join(dir, name) - open(path, 'wb').write(content) + open(path, 'wb').write(bytearray(content,'utf-8')) tarinfo = tar.gettarinfo(path, path) tarinfo.uid = 111 tarinfo.gid = 111 @@ -295,7 +296,7 @@ class TestSCons_time(TestCommon): zip = zipfile.ZipFile(archive, 'w') for name, content in files: path = os.path.join(dir, name) - open(path, 'wb').write(content) + open(path, 'w').write(content) zip.write(path) zip.close() shutil.rmtree(dir) diff --git a/QMTest/TestSConsign.py b/QMTest/TestSConsign.py index 2016afc..df34aa8 100644 --- a/QMTest/TestSConsign.py +++ b/QMTest/TestSConsign.py @@ -1,6 +1,7 @@ -# Copyright (c) 2001 - 2016 The SCons Foundation +# Copyright (c) 2001 - 2017 The SCons Foundation +from __future__ import print_function -__revision__ = "QMTest/TestSConsign.py rel_2.5.1:3735:9dc6cee5c168 2016/11/03 14:02:02 bdbaddog" +__revision__ = "QMTest/TestSConsign.py rel_3.0.0:4395:8972f6a2f699 2017/09/18 12:59:24 bdbaddog" __doc__ = """ TestSConsign.py: a testing framework for the "sconsign" script @@ -68,7 +69,7 @@ class TestSConsign(TestSCons): elif os.path.exists(self.script_path('sconsign')): sconsign = 'sconsign' else: - print "Can find neither 'sconsign.py' nor 'sconsign' scripts." + print("Can find neither 'sconsign.py' nor 'sconsign' scripts.") self.no_result() self.set_sconsign(sconsign) diff --git a/QMTest/scons_tdb.py b/QMTest/scons_tdb.py index 18e6a59..c3b082f 100644 --- a/QMTest/scons_tdb.py +++ b/QMTest/scons_tdb.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -# Copyright (c) 2001 - 2016 The SCons Foundation +# Copyright (c) 2001 - 2017 The SCons Foundation # # Permission is hereby granted, free of charge, to any person obtaining # a copy of this software and associated documentation files (the @@ -20,7 +20,7 @@ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. -from __future__ import division +from __future__ import division, print_function """ QMTest classes to support SCons' testing and Aegis-inspired workflow. @@ -28,7 +28,7 @@ QMTest classes to support SCons' testing and Aegis-inspired workflow. Thanks to Stefan Seefeld for the initial code. """ -__revision__ = "QMTest/scons_tdb.py rel_2.5.1:3735:9dc6cee5c168 2016/11/03 14:02:02 bdbaddog" +__revision__ = "QMTest/scons_tdb.py rel_3.0.0:4395:8972f6a2f699 2017/09/18 12:59:24 bdbaddog" ######################################################################## # Imports @@ -337,14 +337,14 @@ class AegisChangeStream(AegisStream): # We'd like to use the _FormatStatistics() method to do # this, but it's wrapped around the list in Result.outcomes, # so it's simpler to just do it ourselves. - print " %6d tests total\n" % self._num_tests + print(" %6d tests total\n" % self._num_tests) for outcome in AegisTest.aegis_outcomes: if self._outcome_counts[outcome] != 0: - print " %6d (%3.0f%%) tests %s" % ( + print(" %6d (%3.0f%%) tests %s" % ( self._outcome_counts[outcome], self._percent(outcome), outcome - ) + )) class AegisBaselineStream(AegisStream): def WriteResult(self, result): @@ -368,19 +368,19 @@ class AegisBaselineStream(AegisStream): # this, but it's wrapped around the list in Result.outcomes, # so it's simpler to just do it ourselves. if self._outcome_counts[AegisTest.FAIL]: - print " %6d (%3.0f%%) tests as expected" % ( + print(" %6d (%3.0f%%) tests as expected" % ( self._outcome_counts[AegisTest.FAIL], self._percent(AegisTest.FAIL), - ) + )) non_fail_outcomes = list(AegisTest.aegis_outcomes[:]) non_fail_outcomes.remove(AegisTest.FAIL) for outcome in non_fail_outcomes: if self._outcome_counts[outcome] != 0: - print " %6d (%3.0f%%) tests unexpected %s" % ( + print(" %6d (%3.0f%%) tests unexpected %s" % ( self._outcome_counts[outcome], self._percent(outcome), outcome, - ) + )) class AegisBatchStream(FileResultStream): def __init__(self, arguments): |