diff options
Diffstat (limited to 'testing/framework/TestCommon.py')
| -rw-r--r-- | testing/framework/TestCommon.py | 749 |
1 files changed, 749 insertions, 0 deletions
diff --git a/testing/framework/TestCommon.py b/testing/framework/TestCommon.py new file mode 100644 index 0000000..ca4a147 --- /dev/null +++ b/testing/framework/TestCommon.py @@ -0,0 +1,749 @@ +""" +TestCommon.py: a testing framework for commands and scripts + with commonly useful error handling + +The TestCommon module provides a simple, high-level interface for writing +tests of executable commands and scripts, especially commands and scripts +that interact with the file system. All methods throw exceptions and +exit on failure, with useful error messages. This makes a number of +explicit checks unnecessary, making the test scripts themselves simpler +to write and easier to read. + +The TestCommon class is a subclass of the TestCmd class. In essence, +TestCommon is a wrapper that handles common TestCmd error conditions in +useful ways. You can use TestCommon directly, or subclass it for your +program and add additional (or override) methods to tailor it to your +program's specific needs. Alternatively, the TestCommon class serves +as a useful example of how to define your own TestCmd subclass. + +As a subclass of TestCmd, TestCommon provides access to all of the +variables and methods from the TestCmd module. Consequently, you can +use any variable or method documented in the TestCmd module without +having to explicitly import TestCmd. + +A TestCommon environment object is created via the usual invocation: + + import TestCommon + test = TestCommon.TestCommon() + +You can use all of the TestCmd keyword arguments when instantiating a +TestCommon object; see the TestCmd documentation for details. + +Here is an overview of the methods and keyword arguments that are +provided by the TestCommon class: + + test.must_be_writable('file1', ['file2', ...]) + + test.must_contain('file', 'required text\n') + + test.must_contain_all(output, input, ['title', find]) + + test.must_contain_all_lines(output, lines, ['title', find]) + + test.must_contain_any_line(output, lines, ['title', find]) + + test.must_contain_exactly_lines(output, lines, ['title', find]) + + test.must_exist('file1', ['file2', ...]) + + test.must_match('file', "expected contents\n") + + test.must_not_be_writable('file1', ['file2', ...]) + + test.must_not_contain('file', 'banned text\n') + + test.must_not_contain_any_line(output, lines, ['title', find]) + + test.must_not_exist('file1', ['file2', ...]) + + test.run(options = "options to be prepended to arguments", + stdout = "expected standard output from the program", + stderr = "expected error output from the program", + status = expected_status, + match = match_function) + +The TestCommon module also provides the following variables + + TestCommon.python + TestCommon._python_ + TestCommon.exe_suffix + TestCommon.obj_suffix + TestCommon.shobj_prefix + TestCommon.shobj_suffix + TestCommon.lib_prefix + TestCommon.lib_suffix + TestCommon.dll_prefix + TestCommon.dll_suffix + +""" + +# Copyright 2000-2010 Steven Knight +# This module is free software, and you may redistribute it and/or modify +# it under the same terms as Python itself, so long as this copyright message +# and disclaimer are retained in their original form. +# +# IN NO EVENT SHALL THE AUTHOR BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, +# SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES ARISING OUT OF THE USE OF +# THIS CODE, EVEN IF THE AUTHOR HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH +# DAMAGE. +# +# THE AUTHOR SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A +# PARTICULAR PURPOSE. THE CODE PROVIDED HEREUNDER IS ON AN "AS IS" BASIS, +# AND THERE IS NO OBLIGATION WHATSOEVER TO PROVIDE MAINTENANCE, +# SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. + +from __future__ import print_function + +__author__ = "Steven Knight <knight at baldmt dot com>" +__revision__ = "TestCommon.py 1.3.D001 2010/06/03 12:58:27 knight" +__version__ = "1.3" + +import copy +import os +import stat +import sys +import glob + +try: + from collections import UserList +except ImportError: + # no 'collections' module or no UserList in collections + exec('from UserList import UserList') + +from TestCmd import * +from TestCmd import __all__ + +__all__.extend([ 'TestCommon', + 'exe_suffix', + 'obj_suffix', + 'shobj_prefix', + 'shobj_suffix', + 'lib_prefix', + 'lib_suffix', + 'dll_prefix', + 'dll_suffix', + ]) + +# Variables that describe the prefixes and suffixes on this system. +if sys.platform == 'win32': + exe_suffix = '.exe' + obj_suffix = '.obj' + shobj_suffix = '.obj' + shobj_prefix = '' + lib_prefix = '' + lib_suffix = '.lib' + dll_prefix = '' + dll_suffix = '.dll' +elif sys.platform == 'cygwin': + exe_suffix = '.exe' + obj_suffix = '.o' + shobj_suffix = '.os' + shobj_prefix = '' + lib_prefix = 'lib' + lib_suffix = '.a' + dll_prefix = 'cyg' + dll_suffix = '.dll' +elif sys.platform.find('irix') != -1: + exe_suffix = '' + obj_suffix = '.o' + shobj_suffix = '.o' + shobj_prefix = '' + lib_prefix = 'lib' + lib_suffix = '.a' + dll_prefix = 'lib' + dll_suffix = '.so' +elif sys.platform.find('darwin') != -1: + exe_suffix = '' + obj_suffix = '.o' + shobj_suffix = '.os' + shobj_prefix = '' + lib_prefix = 'lib' + lib_suffix = '.a' + dll_prefix = 'lib' + dll_suffix = '.dylib' +elif sys.platform.find('sunos') != -1: + exe_suffix = '' + obj_suffix = '.o' + shobj_suffix = '.o' + shobj_prefix = 'so_' + lib_prefix = 'lib' + lib_suffix = '.a' + dll_prefix = 'lib' + dll_suffix = '.so' +else: + exe_suffix = '' + obj_suffix = '.o' + shobj_suffix = '.os' + shobj_prefix = '' + lib_prefix = 'lib' + lib_suffix = '.a' + dll_prefix = 'lib' + dll_suffix = '.so' + +def is_List(e): + return isinstance(e, (list, UserList)) + +def is_Tuple(e): + return isinstance(e, tuple) + +def is_Sequence(e): + return (not hasattr(e, "strip") and + hasattr(e, "__getitem__") or + hasattr(e, "__iter__")) + +def is_writable(f): + mode = os.stat(f)[stat.ST_MODE] + return mode & stat.S_IWUSR + +def separate_files(flist): + existing = [] + missing = [] + for f in flist: + if os.path.exists(f): + existing.append(f) + else: + missing.append(f) + return existing, missing + +def contains(seq, subseq, find): + # Returns True or False. + if find is None: + return subseq in seq + else: + f = find(seq, subseq) + return f not in (None, -1) and f is not False + +def find_index(seq, subseq, find): + # Returns either an index of the subseq within the seq, or None. + # Accepts a function find(seq, subseq), which returns an integer on success + # and either: None, False, or -1, on failure. + if find is None: + try: + return seq.index(subseq) + except ValueError: + return None + else: + i = find(seq, subseq) + return None if (i in (None, -1) or i is False) else i + + +if os.name == 'posix': + def _failed(self, status = 0): + if self.status is None or status is None: + return None + return _status(self) != status + def _status(self): + return self.status +elif os.name == 'nt': + def _failed(self, status = 0): + return not (self.status is None or status is None) and \ + self.status != status + def _status(self): + return self.status + +class TestCommon(TestCmd): + + # Additional methods from the Perl Test::Cmd::Common module + # that we may wish to add in the future: + # + # $test->subdir('subdir', ...); + # + # $test->copy('src_file', 'dst_file'); + + def __init__(self, **kw): + """Initialize a new TestCommon instance. This involves just + calling the base class initialization, and then changing directory + to the workdir. + """ + TestCmd.__init__(self, **kw) + os.chdir(self.workdir) + + def options_arguments(self, options, arguments): + """Merges the "options" keyword argument with the arguments.""" + if options: + if arguments is None: + return options + + # If not list, then split into lists + # this way we're not losing arguments specified with + # Spaces in quotes. + if isinstance(options, str): + options = options.split() + if isinstance(arguments, str): + arguments = arguments.split() + arguments = options + arguments + + return arguments + + def must_be_writable(self, *files): + """Ensures that the specified file(s) exist and are writable. + An individual file can be specified as a list of directory names, + in which case the pathname will be constructed by concatenating + them. Exits FAILED if any of the files does not exist or is + not writable. + """ + files = [is_List(x) and os.path.join(*x) or x for x in files] + existing, missing = separate_files(files) + unwritable = [x for x in existing if not is_writable(x)] + if missing: + print("Missing files: `%s'" % "', `".join(missing)) + if unwritable: + print("Unwritable files: `%s'" % "', `".join(unwritable)) + self.fail_test(missing + unwritable) + + def must_contain(self, file, required, mode='rb', find=None): + """Ensures specified file contains the required text. + + Args: + file (string): name of file to search in. + required (string): text to search for. For the default + find function, type must match the return type from + reading the file; current implementation will convert. + mode (string): file open mode. + find (func): optional custom search routine. Must take the + form "find(output, line)" non-negative integer on success + and None, False, or -1, on failure. + + Calling test exits FAILED if search result is false + """ + if 'b' in mode: + # Python 3: reading a file in binary mode returns a + # bytes object. We cannot find the index of a different + # (str) type in that, so convert. + required = to_bytes(required) + file_contents = self.read(file, mode) + + if not contains(file_contents, required, find): + print("File `%s' does not contain required string." % file) + print(self.banner('Required string ')) + print(required) + print(self.banner('%s contents ' % file)) + print(file_contents) + self.fail_test() + + def must_contain_all(self, output, input, title=None, find=None): + """Ensures that the specified output string (first argument) + contains all of the specified input as a block (second argument). + + An optional third argument can be used to describe the type + of output being searched, and only shows up in failure output. + + An optional fourth argument can be used to supply a different + function, of the form "find(output, line)", to use when searching + for lines in the output. + """ + if is_List(output): + output = os.newline.join(output) + + if not contains(output, input, find): + if title is None: + title = 'output' + print('Missing expected input from {}:'.format(title)) + print(input) + print(self.banner(title + ' ')) + print(output) + self.fail_test() + + def must_contain_all_lines(self, output, lines, title=None, find=None): + """Ensures that the specified output string (first argument) + contains all of the specified lines (second argument). + + An optional third argument can be used to describe the type + of output being searched, and only shows up in failure output. + + An optional fourth argument can be used to supply a different + function, of the form "find(output, line)", to use when searching + for lines in the output. + """ + missing = [] + if is_List(output): + output = '\n'.join(output) + + for line in lines: + if not contains(output, line, find): + missing.append(line) + + if missing: + if title is None: + title = 'output' + sys.stdout.write("Missing expected lines from %s:\n" % title) + for line in missing: + sys.stdout.write(' ' + repr(line) + '\n') + sys.stdout.write(self.banner(title + ' ') + '\n') + sys.stdout.write(output) + self.fail_test() + + def must_contain_any_line(self, output, lines, title=None, find=None): + """Ensures that the specified output string (first argument) + contains at least one of the specified lines (second argument). + + An optional third argument can be used to describe the type + of output being searched, and only shows up in failure output. + + An optional fourth argument can be used to supply a different + function, of the form "find(output, line)", to use when searching + for lines in the output. + """ + for line in lines: + if contains(output, line, find): + return + + if title is None: + title = 'output' + sys.stdout.write("Missing any expected line from %s:\n" % title) + for line in lines: + sys.stdout.write(' ' + repr(line) + '\n') + sys.stdout.write(self.banner(title + ' ') + '\n') + sys.stdout.write(output) + self.fail_test() + + def must_contain_exactly_lines(self, output, expect, title=None, find=None): + """Ensures that the specified output string (first argument) + contains all of the lines in the expected string (second argument) + with none left over. + + An optional third argument can be used to describe the type + of output being searched, and only shows up in failure output. + + An optional fourth argument can be used to supply a different + function, of the form "find(output, line)", to use when searching + for lines in the output. The function must return the index + of the found line in the output, or None if the line is not found. + """ + out = output.splitlines() + if is_List(expect): + exp = [ e.rstrip('\n') for e in expect ] + else: + exp = expect.splitlines() + if sorted(out) == sorted(exp): + # early out for exact match + return + missing = [] + for line in exp: + i = find_index(out, line, find) + if i is None: + missing.append(line) + else: + out.pop(i) + + if not missing and not out: + # all lines were matched + return + + if title is None: + title = 'output' + if missing: + sys.stdout.write("Missing expected lines from %s:\n" % title) + for line in missing: + sys.stdout.write(' ' + repr(line) + '\n') + sys.stdout.write(self.banner('Missing %s ' % title) + '\n') + if out: + sys.stdout.write("Extra unexpected lines from %s:\n" % title) + for line in out: + sys.stdout.write(' ' + repr(line) + '\n') + sys.stdout.write(self.banner('Extra %s ' % title) + '\n') + sys.stdout.flush() + self.fail_test() + + def must_contain_lines(self, lines, output, title=None, find = None): + # Deprecated; retain for backwards compatibility. + return self.must_contain_all_lines(output, lines, title, find) + + def must_exist(self, *files): + """Ensures that the specified file(s) must exist. An individual + file be specified as a list of directory names, in which case the + pathname will be constructed by concatenating them. Exits FAILED + if any of the files does not exist. + """ + files = [is_List(x) and os.path.join(*x) or x for x in files] + missing = [x for x in files if not os.path.exists(x) and not os.path.islink(x) ] + if missing: + print("Missing files: `%s'" % "', `".join(missing)) + self.fail_test(missing) + + def must_exist_one_of(self, files): + """Ensures that at least one of the specified file(s) exists. + The filenames can be given as a list, where each entry may be + a single path string, or a tuple of folder names and the final + filename that get concatenated. + Supports wildcard names like 'foo-1.2.3-*.rpm'. + Exits FAILED if none of the files exists. + """ + missing = [] + for x in files: + if is_List(x) or is_Tuple(x): + xpath = os.path.join(*x) + else: + xpath = is_Sequence(x) and os.path.join(x) or x + if glob.glob(xpath): + return + missing.append(xpath) + print("Missing one of: `%s'" % "', `".join(missing)) + self.fail_test(missing) + + def must_match(self, file, expect, mode = 'rb', match=None, message=None, newline=None): + """Matches the contents of the specified file (first argument) + against the expected contents (second argument). The expected + contents are a list of lines or a string which will be split + on newlines. + """ + file_contents = self.read(file, mode, newline) + if not match: + match = self.match + try: + self.fail_test(not match(to_str(file_contents), to_str(expect)), message=message) + except KeyboardInterrupt: + raise + except: + print("Unexpected contents of `%s'" % file) + self.diff(expect, file_contents, 'contents ') + raise + + def must_not_contain(self, file, banned, mode = 'rb', find = None): + """Ensures that the specified file doesn't contain the banned text. + """ + file_contents = self.read(file, mode) + + if contains(file_contents, banned, find): + print("File `%s' contains banned string." % file) + print(self.banner('Banned string ')) + print(banned) + print(self.banner('%s contents ' % file)) + print(file_contents) + self.fail_test() + + def must_not_contain_any_line(self, output, lines, title=None, find=None): + """Ensures that the specified output string (first argument) + does not contain any of the specified lines (second argument). + + An optional third argument can be used to describe the type + of output being searched, and only shows up in failure output. + + An optional fourth argument can be used to supply a different + function, of the form "find(output, line)", to use when searching + for lines in the output. + """ + unexpected = [] + for line in lines: + if contains(output, line, find): + unexpected.append(line) + + if unexpected: + if title is None: + title = 'output' + sys.stdout.write("Unexpected lines in %s:\n" % title) + for line in unexpected: + sys.stdout.write(' ' + repr(line) + '\n') + sys.stdout.write(self.banner(title + ' ') + '\n') + sys.stdout.write(output) + self.fail_test() + + def must_not_contain_lines(self, lines, output, title=None, find=None): + return self.must_not_contain_any_line(output, lines, title, find) + + def must_not_exist(self, *files): + """Ensures that the specified file(s) must not exist. + An individual file be specified as a list of directory names, in + which case the pathname will be constructed by concatenating them. + Exits FAILED if any of the files exists. + """ + files = [is_List(x) and os.path.join(*x) or x for x in files] + existing = [x for x in files if os.path.exists(x) or os.path.islink(x)] + if existing: + print("Unexpected files exist: `%s'" % "', `".join(existing)) + self.fail_test(existing) + + def must_not_exist_any_of(self, files): + """Ensures that none of the specified file(s) exists. + The filenames can be given as a list, where each entry may be + a single path string, or a tuple of folder names and the final + filename that get concatenated. + Supports wildcard names like 'foo-1.2.3-*.rpm'. + Exits FAILED if any of the files exists. + """ + existing = [] + for x in files: + if is_List(x) or is_Tuple(x): + xpath = os.path.join(*x) + else: + xpath = is_Sequence(x) and os.path.join(x) or x + if glob.glob(xpath): + existing.append(xpath) + if existing: + print("Unexpected files exist: `%s'" % "', `".join(existing)) + self.fail_test(existing) + + def must_not_be_writable(self, *files): + """Ensures that the specified file(s) exist and are not writable. + An individual file can be specified as a list of directory names, + in which case the pathname will be constructed by concatenating + them. Exits FAILED if any of the files does not exist or is + writable. + """ + files = [is_List(x) and os.path.join(*x) or x for x in files] + existing, missing = separate_files(files) + writable = [file for file in existing if is_writable(file)] + if missing: + print("Missing files: `%s'" % "', `".join(missing)) + if writable: + print("Writable files: `%s'" % "', `".join(writable)) + self.fail_test(missing + writable) + + def _complete(self, actual_stdout, expected_stdout, + actual_stderr, expected_stderr, status, match): + """ + Post-processes running a subcommand, checking for failure + status and displaying output appropriately. + """ + if _failed(self, status): + expect = '' + if status != 0: + expect = " (expected %s)" % str(status) + print("%s returned %s%s" % (self.program, _status(self), expect)) + print(self.banner('STDOUT ')) + print(actual_stdout) + print(self.banner('STDERR ')) + print(actual_stderr) + self.fail_test() + if (expected_stdout is not None + and not match(actual_stdout, expected_stdout)): + self.diff(expected_stdout, actual_stdout, 'STDOUT ') + if actual_stderr: + print(self.banner('STDERR ')) + print(actual_stderr) + self.fail_test() + if (expected_stderr is not None + and not match(actual_stderr, expected_stderr)): + print(self.banner('STDOUT ')) + print(actual_stdout) + self.diff(expected_stderr, actual_stderr, 'STDERR ') + self.fail_test() + + def start(self, program = None, + interpreter = None, + options = None, + arguments = None, + universal_newlines = None, + **kw): + """ + Starts a program or script for the test environment, handling + any exceptions. + """ + arguments = self.options_arguments(options, arguments) + try: + return TestCmd.start(self, program, interpreter, arguments, + universal_newlines, **kw) + except KeyboardInterrupt: + raise + except Exception as e: + print(self.banner('STDOUT ')) + try: + print(self.stdout()) + except IndexError: + pass + print(self.banner('STDERR ')) + try: + print(self.stderr()) + except IndexError: + pass + cmd_args = self.command_args(program, interpreter, arguments) + sys.stderr.write('Exception trying to execute: %s\n' % cmd_args) + raise e + + def finish(self, popen, stdout = None, stderr = '', status = 0, **kw): + """ + Finishes and waits for the process being run under control of + the specified popen argument. Additional arguments are similar + to those of the run() method: + + stdout The expected standard output from + the command. A value of None means + don't test standard output. + + stderr The expected error output from + the command. A value of None means + don't test error output. + + status The expected exit status from the + command. A value of None means don't + test exit status. + """ + TestCmd.finish(self, popen, **kw) + match = kw.get('match', self.match) + self._complete(self.stdout(), stdout, + self.stderr(), stderr, status, match) + + def run(self, options = None, arguments = None, + stdout = None, stderr = '', status = 0, **kw): + """Runs the program under test, checking that the test succeeded. + + The parameters are the same as the base TestCmd.run() method, + with the addition of: + + options Extra options that get prepended to the beginning + of the arguments. + + stdout The expected standard output from + the command. A value of None means + don't test standard output. + + stderr The expected error output from + the command. A value of None means + don't test error output. + + status The expected exit status from the + command. A value of None means don't + test exit status. + + By default, this expects a successful exit (status = 0), does + not test standard output (stdout = None), and expects that error + output is empty (stderr = ""). + """ + kw['arguments'] = self.options_arguments(options, arguments) + try: + match = kw['match'] + del kw['match'] + except KeyError: + match = self.match + TestCmd.run(self, **kw) + self._complete(self.stdout(), stdout, + self.stderr(), stderr, status, match) + + def skip_test(self, message="Skipping test.\n"): + """Skips a test. + + Proper test-skipping behavior is dependent on the external + TESTCOMMON_PASS_SKIPS environment variable. If set, we treat + the skip as a PASS (exit 0), and otherwise treat it as NO RESULT. + In either case, we print the specified message as an indication + that the substance of the test was skipped. + + (This was originally added to support development under Aegis. + Technically, skipping a test is a NO RESULT, but Aegis would + treat that as a test failure and prevent the change from going to + the next step. Since we ddn't want to force anyone using Aegis + to have to install absolutely every tool used by the tests, we + would actually report to Aegis that a skipped test has PASSED + so that the workflow isn't held up.) + """ + if message: + sys.stdout.write(message) + sys.stdout.flush() + pass_skips = os.environ.get('TESTCOMMON_PASS_SKIPS') + if pass_skips in [None, 0, '0']: + # skip=1 means skip this function when showing where this + # result came from. They only care about the line where the + # script called test.skip_test(), not the line number where + # we call test.no_result(). + self.no_result(skip=1) + else: + # We're under the development directory for this change, + # so this is an Aegis invocation; pass the test (exit 0). + self.pass_test() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: |