From 72c578fd4b0b4a5a43e18594339ac4ff26c376dc Mon Sep 17 00:00:00 2001 From: Luca Falavigna Date: Sat, 2 Jan 2010 20:56:27 +0100 Subject: Imported Upstream version 1.2.0.d20091224 --- bin/Command.py | 143 +++++++++ bin/SConsDoc.py | 336 ++++++++++++++++++++ bin/ae-cvs-ci | 204 ++++++++++++ bin/ae-svn-ci | 240 ++++++++++++++ bin/ae2cvs | 579 +++++++++++++++++++++++++++++++++ bin/calibrate.py | 81 +++++ bin/caller-tree.py | 96 ++++++ bin/docdiff | 16 + bin/docrun | 16 + bin/docupdate | 16 + bin/files | 106 +++++++ bin/import-test.py | 104 ++++++ bin/install_python.py | 141 +++++++++ bin/install_scons.py | 216 +++++++++++++ bin/linecount.py | 123 +++++++ bin/makedocs | 18 ++ bin/memlogs.py | 49 +++ bin/memoicmp.py | 78 +++++ bin/objcounts.py | 109 +++++++ bin/restore.sh | 90 ++++++ bin/rsync-sourceforge | 32 ++ bin/scons-cdist | 272 ++++++++++++++++ bin/scons-diff.py | 195 ++++++++++++ bin/scons-proc.py | 283 +++++++++++++++++ bin/scons-review.sh | 24 ++ bin/scons-test.py | 247 +++++++++++++++ bin/scons-unzip.py | 76 +++++ bin/scons_dev_master.py | 215 +++++++++++++ bin/sconsexamples.py | 512 ++++++++++++++++++++++++++++++ bin/sconsoutput.py | 827 ++++++++++++++++++++++++++++++++++++++++++++++++ bin/sfsum | 148 +++++++++ bin/svn-bisect.py | 72 +++++ bin/time-scons.py | 355 +++++++++++++++++++++ bin/timebuild | 65 ++++ bin/xml_export | 225 +++++++++++++ bin/xml_export-LICENSE | 52 +++ bin/xml_export-README | 56 ++++ bin/xmlagenda.py | 93 ++++++ 38 files changed, 6510 insertions(+) create mode 100644 bin/Command.py create mode 100644 bin/SConsDoc.py create mode 100755 bin/ae-cvs-ci create mode 100755 bin/ae-svn-ci create mode 100644 bin/ae2cvs create mode 100644 bin/calibrate.py create mode 100644 bin/caller-tree.py create mode 100644 bin/docdiff create mode 100644 bin/docrun create mode 100644 bin/docupdate create mode 100644 bin/files create mode 100644 bin/import-test.py create mode 100644 bin/install_python.py create mode 100644 bin/install_scons.py create mode 100644 bin/linecount.py create mode 100644 bin/makedocs create mode 100644 bin/memlogs.py create mode 100644 bin/memoicmp.py create mode 100644 bin/objcounts.py create mode 100644 bin/restore.sh create mode 100644 bin/rsync-sourceforge create mode 100644 bin/scons-cdist create mode 100644 bin/scons-diff.py create mode 100644 bin/scons-proc.py create mode 100755 bin/scons-review.sh create mode 100644 bin/scons-test.py create mode 100644 bin/scons-unzip.py create mode 100644 bin/scons_dev_master.py create mode 100644 bin/sconsexamples.py create mode 100644 bin/sconsoutput.py create mode 100644 bin/sfsum create mode 100755 bin/svn-bisect.py create mode 100644 bin/time-scons.py create mode 100644 bin/timebuild create mode 100644 bin/xml_export create mode 100644 bin/xml_export-LICENSE create mode 100644 bin/xml_export-README create mode 100755 bin/xmlagenda.py (limited to 'bin') diff --git a/bin/Command.py b/bin/Command.py new file mode 100644 index 0000000..efaa356 --- /dev/null +++ b/bin/Command.py @@ -0,0 +1,143 @@ +#!/usr/bin/env python +# +# XXX Python script template +# +# XXX Describe what the script does here. +# + +import getopt +import os +import shlex +import sys + +class Usage(Exception): + def __init__(self, msg): + self.msg = msg + +class CommandRunner: + """ + Representation of a command to be executed. + """ + + def __init__(self, dictionary={}): + self.subst_dictionary(dictionary) + + def subst_dictionary(self, dictionary): + self._subst_dictionary = dictionary + + def subst(self, string, dictionary=None): + """ + Substitutes (via the format operator) the values in the specified + dictionary into the specified command. + + The command can be an (action, string) tuple. In all cases, we + perform substitution on strings and don't worry if something isn't + a string. (It's probably a Python function to be executed.) + """ + if dictionary is None: + dictionary = self._subst_dictionary + if dictionary: + try: + string = string % dictionary + except TypeError: + pass + return string + + def do_display(self, string): + if type(string) == type(()): + func = string[0] + args = string[1:] + s = '%s(%s)' % (func.__name__, ', '.join(map(repr, args))) + else: + s = self.subst(string) + if not s.endswith('\n'): + s += '\n' + sys.stdout.write(s) + sys.stdout.flush() + + def do_not_display(self, string): + pass + + def do_execute(self, command): + if type(command) == type(''): + command = self.subst(command) + cmdargs = shlex.split(command) + if cmdargs[0] == 'cd': + command = (os.chdir,) + tuple(cmdargs[1:]) + elif cmdargs[0] == 'mkdir': + command = (os.mkdir,) + tuple(cmdargs[1:]) + if type(command) == type(()): + func = command[0] + args = command[1:] + return func(*args) + else: + return os.system(command) + + def do_not_execute(self, command): + pass + + display = do_display + execute = do_execute + + def run(self, command, display=None): + """ + Runs this command, displaying it first. + + The actual display() and execute() methods we call may be + overridden if we're printing but not executing, or vice versa. + """ + if display is None: + display = command + self.display(display) + return self.execute(command) + +def main(argv=None): + if argv is None: + argv = sys.argv + + short_options = 'hnq' + long_options = ['help', 'no-exec', 'quiet'] + + helpstr = """\ +Usage: script-template.py [-hnq] + + -h, --help Print this help and exit + -n, --no-exec No execute, just print command lines + -q, --quiet Quiet, don't print command lines +""" + + try: + try: + opts, args = getopt.getopt(argv[1:], short_options, long_options) + except getopt.error, msg: + raise Usage(msg) + + for o, a in opts: + if o in ('-h', '--help'): + print helpstr + sys.exit(0) + elif o in ('-n', '--no-exec'): + Command.execute = Command.do_not_execute + elif o in ('-q', '--quiet'): + Command.display = Command.do_not_display + except Usage, err: + sys.stderr.write(err.msg) + sys.stderr.write('use -h to get help') + return 2 + + commands = [ + ] + + for command in [ Command(c) for c in commands ]: + status = command.run(command) + if status: + sys.exit(status) + +if __name__ == "__main__": + sys.exit(main()) + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/bin/SConsDoc.py b/bin/SConsDoc.py new file mode 100644 index 0000000..d3a043b --- /dev/null +++ b/bin/SConsDoc.py @@ -0,0 +1,336 @@ +#!/usr/bin/env python +# +# Module for handling SCons documentation processing. +# + +__doc__ = """ +This module parses home-brew XML files that document various things +in SCons. Right now, it handles Builders, construction variables, +and Tools, but we expect it to get extended in the future. + +In general, you can use any DocBook tag in the input, and this module +just adds processing various home-brew tags to try to make life a +little easier. + +Builder example: + + + + This is the summary description of an SCons Tool. + It will get placed in the man page, + and in the appropriate User's Guide appendix. + The name of any builder may be interpolated + anywhere in the document by specifying the + &b-VARIABLE; + element. It need not be on a line by itself. + + Unlike normal XML, blank lines are significant in these + descriptions and serve to separate paragraphs. + They'll get replaced in DocBook output with appropriate tags + to indicate a new paragraph. + + + print "this is example code, it will be offset and indented" + + + + +Construction variable example: + + + + This is the summary description of a construction variable. + It will get placed in the man page, + and in the appropriate User's Guide appendix. + The name of any construction variable may be interpolated + anywhere in the document by specifying the + &t-VARIABLE; + element. It need not be on a line by itself. + + Unlike normal XML, blank lines are significant in these + descriptions and serve to separate paragraphs. + They'll get replaced in DocBook output with appropriate tags + to indicate a new paragraph. + + + print "this is example code, it will be offset and indented" + + + + +Tool example: + + + + This is the summary description of an SCons Tool. + It will get placed in the man page, + and in the appropriate User's Guide appendix. + The name of any tool may be interpolated + anywhere in the document by specifying the + &t-VARIABLE; + element. It need not be on a line by itself. + + Unlike normal XML, blank lines are significant in these + descriptions and serve to separate paragraphs. + They'll get replaced in DocBook output with appropriate tags + to indicate a new paragraph. + + + print "this is example code, it will be offset and indented" + + + +""" + +import os.path +import imp +import sys +import xml.sax.handler + +class Item: + def __init__(self, name): + self.name = name + self.sort_name = name.lower() + if self.sort_name[0] == '_': + self.sort_name = self.sort_name[1:] + self.summary = [] + self.sets = None + self.uses = None + def cmp_name(self, name): + if name[0] == '_': + name = name[1:] + return name.lower() + def __cmp__(self, other): + return cmp(self.sort_name, other.sort_name) + +class Builder(Item): + pass + +class Tool(Item): + def __init__(self, name): + Item.__init__(self, name) + self.entity = self.name.replace('+', 'X') + +class ConstructionVariable(Item): + pass + +class Chunk: + def __init__(self, tag, body=None): + self.tag = tag + if not body: + body = [] + self.body = body + def __str__(self): + body = ''.join(self.body) + return "<%s>%s\n" % (self.tag, body, self.tag) + def append(self, data): + self.body.append(data) + +class Summary: + def __init__(self): + self.body = [] + self.collect = [] + def append(self, data): + self.collect.append(data) + def end_para(self): + text = ''.join(self.collect) + paras = text.split('\n\n') + if paras == ['\n']: + return + if paras[0] == '': + self.body.append('\n') + paras = paras[1:] + paras[0] = '\n' + paras[0] + if paras[-1] == '': + paras = paras[:-1] + paras[-1] = paras[-1] + '\n' + last = '\n' + else: + last = None + sep = None + for p in paras: + c = Chunk("para", p) + if sep: + self.body.append(sep) + self.body.append(c) + sep = '\n' + if last: + self.body.append(last) + def begin_chunk(self, chunk): + self.end_para() + self.collect = chunk + def end_chunk(self): + self.body.append(self.collect) + self.collect = [] + +class SConsDocHandler(xml.sax.handler.ContentHandler, + xml.sax.handler.ErrorHandler): + def __init__(self): + self._start_dispatch = {} + self._end_dispatch = {} + keys = self.__class__.__dict__.keys() + start_tag_method_names = filter(lambda k: k[:6] == 'start_', keys) + end_tag_method_names = filter(lambda k: k[:4] == 'end_', keys) + for method_name in start_tag_method_names: + tag = method_name[6:] + self._start_dispatch[tag] = getattr(self, method_name) + for method_name in end_tag_method_names: + tag = method_name[4:] + self._end_dispatch[tag] = getattr(self, method_name) + self.stack = [] + self.collect = [] + self.current_object = [] + self.builders = {} + self.tools = {} + self.cvars = {} + + def startElement(self, name, attrs): + try: + start_element_method = self._start_dispatch[name] + except KeyError: + self.characters('<%s>' % name) + else: + start_element_method(attrs) + + def endElement(self, name): + try: + end_element_method = self._end_dispatch[name] + except KeyError: + self.characters('' % name) + else: + end_element_method() + + # + # + def characters(self, chars): + self.collect.append(chars) + + def begin_collecting(self, chunk): + self.collect = chunk + def end_collecting(self): + self.collect = [] + + def begin_chunk(self): + pass + def end_chunk(self): + pass + + # + # + # + + def begin_xxx(self, obj): + self.stack.append(self.current_object) + self.current_object = obj + def end_xxx(self): + self.current_object = self.stack.pop() + + # + # + # + def start_scons_doc(self, attrs): + pass + def end_scons_doc(self): + pass + + def start_builder(self, attrs): + name = attrs.get('name') + try: + builder = self.builders[name] + except KeyError: + builder = Builder(name) + self.builders[name] = builder + self.begin_xxx(builder) + def end_builder(self): + self.end_xxx() + + def start_tool(self, attrs): + name = attrs.get('name') + try: + tool = self.tools[name] + except KeyError: + tool = Tool(name) + self.tools[name] = tool + self.begin_xxx(tool) + def end_tool(self): + self.end_xxx() + + def start_cvar(self, attrs): + name = attrs.get('name') + try: + cvar = self.cvars[name] + except KeyError: + cvar = ConstructionVariable(name) + self.cvars[name] = cvar + self.begin_xxx(cvar) + def end_cvar(self): + self.end_xxx() + + def start_summary(self, attrs): + summary = Summary() + self.current_object.summary = summary + self.begin_xxx(summary) + self.begin_collecting(summary) + def end_summary(self): + self.current_object.end_para() + self.end_xxx() + + def start_example(self, attrs): + example = Chunk("programlisting") + self.current_object.begin_chunk(example) + def end_example(self): + self.current_object.end_chunk() + + def start_uses(self, attrs): + self.begin_collecting([]) + def end_uses(self): + self.current_object.uses = ''.join(self.collect).split() + self.current_object.uses.sort() + self.end_collecting() + + def start_sets(self, attrs): + self.begin_collecting([]) + def end_sets(self): + self.current_object.sets = ''.join(self.collect).split() + self.current_object.sets.sort() + self.end_collecting() + + # Stuff for the ErrorHandler portion. + def error(self, exception): + linenum = exception._linenum - self.preamble_lines + sys.stderr.write('%s:%d:%d: %s (error)\n' % (self.filename, linenum, exception._colnum, ''.join(exception.args))) + + def fatalError(self, exception): + linenum = exception._linenum - self.preamble_lines + sys.stderr.write('%s:%d:%d: %s (fatalError)\n' % (self.filename, linenum, exception._colnum, ''.join(exception.args))) + + def set_file_info(self, filename, preamble_lines): + self.filename = filename + self.preamble_lines = preamble_lines + +# lifted from Ka-Ping Yee's way cool pydoc module. +def importfile(path): + """Import a Python source file or compiled file given its path.""" + magic = imp.get_magic() + file = open(path, 'r') + if file.read(len(magic)) == magic: + kind = imp.PY_COMPILED + else: + kind = imp.PY_SOURCE + file.close() + filename = os.path.basename(path) + name, ext = os.path.splitext(filename) + file = open(path, 'r') + try: + module = imp.load_module(name, file, path, (ext, 'r', kind)) + except ImportError, e: + sys.stderr.write("Could not import %s: %s\n" % (path, e)) + return None + file.close() + return module + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/bin/ae-cvs-ci b/bin/ae-cvs-ci new file mode 100755 index 0000000..47c8073 --- /dev/null +++ b/bin/ae-cvs-ci @@ -0,0 +1,204 @@ +# +# aegis - project change supervisor +# Copyright (C) 2004 Peter Miller; +# All rights reserved. +# +# As a specific exception to the GPL, you are allowed to copy +# this source file into your own project and modify it, without +# releasing your project under the GPL, unless there is some other +# file or condition which would require it. +# +# MANIFEST: shell script to commit changes to CVS +# +# It is assumed that your CVSROOT and CVS_RSH environment variables have +# already been set appropriately. +# +# This script is expected to be run as by integrate_pass_notify_command +# and as such the baseline has already assumed the shape asked for by +# the change. +# +# integrate_pass_notify_command = +# "$bin/ae-cvs-ci $project $change"; +# +# Alternatively, you may wish to tailor this script to the individual +# needs of your project. Make it a source file, e.g. "etc/ae-cvs-ci.sh" +# and then use the following: +# +# integrate_pass_notify_command = +# "$sh ${s etc/ae-cvs-ci} $project $change"; +# + +USAGE="Usage: $0 " + +PRINT="echo" +EXECUTE="eval" + +while getopts "hnq" FLAG +do + case ${FLAG} in + h ) + echo "${USAGE}" + exit 0 + ;; + n ) + EXECUTE=":" + ;; + q ) + PRINT=":" + ;; + * ) + echo "$0: unknown option ${FLAG}" >&2 + exit 1 + ;; + esac +done + +shift `expr ${OPTIND} - 1` + +case $# in +2) + project=$1 + change=$2 + ;; +*) + echo "${USAGE}" 1>&2 + exit 1 + ;; +esac + +here=`pwd` + +AEGIS_PROJECT=$project +export AEGIS_PROJECT +AEGIS_CHANGE=$change +export AEGIS_CHANGE + +module=`echo $project | sed 's|[.].*||'` + +baseline=`aegis -cd -bl` + +if test X${TMPDIR} = X; then TMPDIR=/var/tmp; fi + +TMP=${TMPDIR}/ae-cvs-ci.$$ +mkdir ${TMP} +cd ${TMP} + +PWD=`pwd` +if test X${PWD} != X${TMP}; then + echo "$0: ended up in ${PWD}, not ${TMP}" >&2 + exit 1 +fi + +fail() +{ + set +x + cd $here + rm -rf ${TMP} + echo "FAILED" 1>&2 + exit 1 +} +trap "fail" 1 2 3 15 + +Command() +{ + ${PRINT} "$*" + ${EXECUTE} "$*" +} + +# +# Create a new CVS work area. +# +# Note: this assumes the module is checked-out into a directory of the +# same name. Is there a way to ask CVS where is is going to put a +# modules, so we can always get the "cd" right? +# +${PRINT} cvs co $module +${EXECUTE} cvs co $module > LOG 2>&1 +if test $? -ne 0; then cat LOG; fail; fi +${EXECUTE} cd $module + +# +# Now we need to extract the sources from Aegis and drop them into the +# CVS work area. There are two ways to do this. +# +# The first way is to use the generated tarball. +# This has the advantage that it has the Makefile.in file in it, and +# will work immediately. +# +# The second way is to use aetar, which will give exact sources, and +# omit all derived files. This will *not* include the Makefile.in, +# and so will not be readily compilable. +# +# gunzip < $baseline/export/${project}.tar.gz | tardy -rp ${project} | tar xf - +aetar -send -comp-alg=gzip -o - | tar xzf - + +# +# If any new directories have been created we will need to add them +# to CVS before we can add the new files which we know are in them, +# or they would not have been created. Do this only if the -n option +# isn't used, because if it is, we won't have actually checked out the +# source and we'd erroneously report that all of them need to be added. +# +if test "X${EXECUTE}" != "X:" +then + find . \( -name CVS -o -name Attic \) -prune -o -type d -print | + xargs --max-args=1 | + while read dir + do + if [ ! -d "$dir/CVS" ] + then + Command cvs add "$dir" + fi + done +fi + +# +# Use the Aegis meta-data to perform some CVS commands that CVS can't +# figure out for itself. +# +aegis -l cf -unf | sed 's| -> [0-9][0-9.]*||' | +while read usage action rev filename +do + if test "x$filename" = "x" + then + filename="$rev" + fi + case $action in + create) + Command cvs add $filename + ;; + remove) + Command rm -f $filename + Command cvs remove $filename + ;; + *) + ;; + esac +done + +# +# Extract the brief description. We'd like to do this using aesub +# or something, like so: +# +# message=`aesub '${version} - ${change description}'` +# +# but the expansion of ${change description} has a lame hard-coded max of +# 80 characters, so we have to do this by hand. (This has the slight +# benefit of preserving backslashes in front of any double-quotes in +# the text; that will have to be handled if we go back to using aesub.) +# +description=`aegis -ca -l | sed -n 's/brief_description = "\(.*\)";$/\1/p'` +version=`aesub '${version}'` +message="$version - $description" + +# +# Now commit all the changes. +# +Command cvs -q commit -m \"$message\" + +# +# All done. Clean up and go home. +# +cd $here +rm -rf ${TMP} +exit 0 diff --git a/bin/ae-svn-ci b/bin/ae-svn-ci new file mode 100755 index 0000000..301d890 --- /dev/null +++ b/bin/ae-svn-ci @@ -0,0 +1,240 @@ +# +# aegis - project change supervisor +# Copyright (C) 2004 Peter Miller; +# All rights reserved. +# +# As a specific exception to the GPL, you are allowed to copy +# this source file into your own project and modify it, without +# releasing your project under the GPL, unless there is some other +# file or condition which would require it. +# +# MANIFEST: shell script to commit changes to Subversion +# +# This script is expected to be run by the integrate_pass_notify_command +# and as such the baseline has already assumed the shape asked for by +# the change. +# +# integrate_pass_notify_command = +# "$bin/ae-svn-ci $project $change http://svn.site.com/svn/trunk --username svn_user"; +# +# Alternatively, you may wish to tailor this script to the individual +# needs of your project. Make it a source file, e.g. "etc/ae-svn-ci.sh" +# and then use the following: +# +# integrate_pass_notify_command = +# "$sh ${s etc/ae-svn-ci} $project $change http://svn.site.com/svn/trunk --username svn_user"; +# + +USAGE="Usage: $0 [-hnq] []" + +PRINT="echo" +EXECUTE="eval" + +while getopts "hnq" FLAG +do + case ${FLAG} in + h ) + echo "${USAGE}" + exit 0 + ;; + n ) + EXECUTE=":" + ;; + q ) + PRINT=":" + ;; + * ) + echo "$0: unknown option ${FLAG}" >&2 + exit 1 + ;; + esac +done + +shift `expr ${OPTIND} - 1` + +case $# in +[012]) + echo "${USAGE}" 1>&2 + exit 1 + ;; +*) + project=$1 + change=$2 + svn_url=$3 + shift 3 + svn_co_flags=$* + ;; +esac + +here=`pwd` + +AEGIS_PROJECT=$project +export AEGIS_PROJECT +AEGIS_CHANGE=$change +export AEGIS_CHANGE + +module=`echo $project | sed 's|[.].*||'` + +baseline=`aegis -cd -bl` + +if test X${TMPDIR} = X; then TMPDIR=/var/tmp; fi + +TMP=${TMPDIR}/ae-svn-ci.$$ +mkdir ${TMP} +cd ${TMP} + +PWD=`pwd` +if test X${PWD} != X${TMP}; then + echo "$0: ended up in ${PWD}, not ${TMP}" >&2 + exit 1 +fi + +fail() +{ + set +x + cd $here + rm -rf ${TMP} + echo "FAILED" 1>&2 + exit 1 +} +trap "fail" 1 2 3 15 + +Command() +{ + ${PRINT} "$*" + ${EXECUTE} "$*" +} + +# +# Create a new Subversion work area. +# +# Note: this assumes the module is checked-out into a directory of the +# same name. Is there a way to ask Subversion where it is going to put a +# module, so we can always get the "cd" right? +# +${PRINT} svn co $svn_url $module $svn_co_flags +${EXECUTE} svn co $svn_url $module $svn_co_flags > LOG 2>&1 +if test $? -ne 0; then cat LOG; fail; fi +${EXECUTE} cd $module + +# +# Now we need to extract the sources from Aegis and drop them into the +# Subversion work area. There are two ways to do this. +# +# The first way is to use the generated tarball. +# This has the advantage that it has the Makefile.in file in it, and +# will work immediately. +# +# The second way is to use aetar, which will give exact sources, and +# omit all derived files. This will *not* include the Makefile.in, +# and so will not be readily compilable. +# +# gunzip < $baseline/export/${project}.tar.gz | tardy -rp ${project} | tar xf - +aetar -send -comp-alg=gzip -o - | tar xzf - + +# +# If any new directories have been created we will need to add them +# to Subversion before we can add the new files which we know are in them, +# or they would not have been created. Do this only if the -n option +# isn't used, because if it is, we won't have actually checked out the +# source and we'd erroneously report that all of them need to be added. +# +if test "X${EXECUTE}" != "X:" +then + find . -name .svn -prune -o -type d -print | + xargs --max-args=1 | + while read dir + do + if [ ! -d "$dir/.svn" ] + then + Command svn add -N "$dir" + fi + done +fi + +# +# Use the Aegis meta-data to perform some commands that Subversion can't +# figure out for itself. We use an inline "aer" report script to identify +# when a remove-create pair are actually due to a move. +# +aegis -rpt -nph -f - <<_EOF_ | +auto cs; +cs = project[project_name()].state.branch.change[change_number()]; + +columns({width = 1000;}); + +auto file, moved; +for (file in cs.src) +{ + if (file.move != "") + moved[file.move] = 1; +} + +auto action; +for (file in cs.src) +{ + if (file.action == "remove" && file.move != "") + action = "move"; + else + action = file.action; + /* + * Suppress printing of any files created as the result of a move. + * These are printed as the destination when printing the line for + * the file that was *removed* as a result of the move. + */ + if (action != "create" || ! moved[file.file_name]) + print(sprintf("%s %s \\"%s\\" \\"%s\\"", file.usage, action, file.file_name, file.move)); +} +_EOF_ +while read line +do + eval set -- "$line" + usage="$1" + action="$2" + srcfile="$3" + dstfile="$4" + case $action in + create) + Command svn add $srcfile + ;; + remove) + Command rm -f $srcfile + Command svn remove $srcfile + ;; + move) + Command mv $dstfile $dstfile.move + Command svn move $srcfile $dstfile + Command cp $dstfile.move $dstfile + Command rm -f $dstfile.move + ;; + *) + ;; + esac +done + +# +# Extract the brief description. We'd like to do this using aesub +# or something, like so: +# +# message=`aesub '${version} - ${change description}'` +# +# but the expansion of ${change description} has a lame hard-coded max of +# 80 characters, so we have to do this by hand. (This has the slight +# benefit of preserving backslashes in front of any double-quotes in +# the text; that will have to be handled if we go back to using aesub.) +# +description=`aegis -ca -l | sed -n 's/brief_description = "\(.*\)";$/\1/p'` +version=`aesub '${version}'` +message="$version - $description" + +# +# Now commit all the changes. +# +Command svn commit -m \"$message\" + +# +# All done. Clean up and go home. +# +cd $here +rm -rf ${TMP} +exit 0 diff --git a/bin/ae2cvs b/bin/ae2cvs new file mode 100644 index 0000000..e7cb22b --- /dev/null +++ b/bin/ae2cvs @@ -0,0 +1,579 @@ +#! /usr/bin/env perl + +$revision = "src/ae2cvs.pl 0.04.D001 2005/08/14 15:13:36 knight"; + +$copyright = "Copyright 2001, 2002, 2003, 2004, 2005 Steven Knight."; + +# +# All rights reserved. This program is free software; you can +# redistribute and/or modify under the same terms as Perl itself. +# + +use strict; +use File::Find; +use File::Spec; +use Pod::Usage (); + +use vars qw( @add_list @args @cleanup @copy_list @libraries + @mkdir_list @remove_list + %seen_dir + $ae_copy $aedir $aedist + $cnum $comment $commit $common $copyright + $cvs_command $cvsmod $cvsroot + $delta $description $exec $help $indent $infile + $proj $pwd $quiet $revision + $summary $usedir $usepath ); + +$aedist = 1; +$cvsroot = undef; +$exec = undef; +$indent = ""; + +sub version { + print "ae2cvs: $revision\n"; + print "$copyright\n"; + exit 0; +} + +{ + use Getopt::Long; + + Getopt::Long::Configure('no_ignore_case'); + + my $ret = GetOptions ( + "aedist" => sub { $aedist = 1 }, + "aegis" => sub { $aedist = 0 }, + "change=i" => \$cnum, + "d=s" => \$cvsroot, + "file=s" => \$infile, + "help|?" => \$help, + "library=s" => \@libraries, + "module=s" => \$cvsmod, + "noexecute" => sub { $exec = 0 }, + "project=s" => \$proj, + "quiet" => \$quiet, + "usedir=s" => \$usedir, + "v|version" => \&version, + "x|execute" => sub { $exec++ if ! defined $exec || $exec != 0 }, + "X|EXECUTE" => sub { $exec = 2 if ! defined $exec || $exec != 0 }, + ); + + Pod::Usage::pod2usage(-verbose => 0) if $help || ! $ret; + + $exec = 0 if ! defined $exec; +} + +$cvs_command = $cvsroot ? "cvs -d $cvsroot -Q" : "cvs -Q"; + +# +# Wrap up the $quiet logic in one place. +# +sub printit { + return if $quiet; + my $string = join('', @_); + $string =~ s/^/$indent/msg if $indent; + print $string; +} + +# +# Wrappers for executing various builtin Perl functions in +# accordance with the -n, -q and -x options. +# +sub execute { + my $cmd = shift; + printit "$cmd\n"; + if (! $exec) { + return 1; + } + ! system($cmd); +} + +sub _copy { + my ($source, $dest) = @_; + printit "cp $source $dest\n"; + if ($exec) { + use File::Copy; + copy($source, $dest); + } +} + +sub _chdir { + my $dir = shift; + printit "cd $dir\n"; + if ($exec) { + chdir($dir) || die "ae2cvs: could not chdir($dir): $!"; + } +} + +sub _mkdir { + my $dir = shift; + printit "mkdir $dir\n"; + if ($exec) { + mkdir($dir); + } +} + +# +# Put some input data through an external filter and capture the output. +# +sub filter { + my ($cmd, $input) = @_; + + use FileHandle; + use IPC::Open2; + + my $pid = open2(*READ, *WRITE, $cmd) || die "Cannot exec '$cmd': $!\n"; + print WRITE $input; + close(WRITE); + my $output = join('', ); + close(READ); + return $output; +} + +# +# Parse a change description, in both 'aegis -l cd" and "aedist" formats. +# +# Returns an array containing the project name, the change number +# (if any), the delta number (if any), the SUMMARY, the DESCRIPTION +# and the lines describing the files in the change. +# +sub parse_change { + my $output = shift; + + my ($p, $c, $d, $c_or_d, $sum, $desc, $filesection, @flines); + + # The project name line comes after NAME in "aegis -l cd" format, + # and PROJECT in "aedist" format. In both cases, the project name + # and the change/delta name are separated a comma. + ($p = $output) =~ s/(?:NAME|PROJECT)\n([^\n]*)\n.*/$1/ms; + ($p, $c_or_d) = (split(/,/, $p)); + + # In "aegis -l cd" format, the project name actually comes after + # the string "Project" and is itself enclosed in double quotes. + $p =~ s/Project "([^"]*)"/$1/; + + # The change or delta string was the right-hand side of the comma. + # "aegis -l cd" format spells it "Change 123." or "Delta 123." while + # "aedist" format spells it "change 123." + if ($c_or_d =~ /\s*[Cc]hange (\d+).*/) { $c = $1 }; + if ($c_or_d =~ /\s*[Dd]elta (\d+).*/) { $d = $1 }; + + # The SUMMARY line is always followed the DESCRIPTION section. + # It seems to always be a single line, but we grab everything in + # between just in case. + ($sum = $output) =~ s/.*\nSUMMARY\n//ms; + $sum =~ s/\nDESCRIPTION\n.*//ms; + + # The DESCRIPTION section is followed ARCHITECTURE in "aegis -l cd" + # format and by CAUSE in "aedist" format. Explicitly under it if the + # string is only "none," which means they didn't supply a description. + ($desc = $output) =~ s/.*\nDESCRIPTION\n//ms; + $desc =~ s/\n(ARCHITECTURE|CAUSE)\n.*//ms; + chomp($desc); + if ($desc eq "none" || $desc eq "none\n") { $desc = undef } + + # The FILES section is followed by HISTORY in "aegis -l cd" format. + # It seems to be the last section in "aedist" format, but stripping + # a non-existent HISTORY section doesn't hurt. + ($filesection = $output) =~ s/.*\nFILES\n//ms; + $filesection =~ s/\nHISTORY\n.*//ms; + + @flines = split(/\n/, $filesection); + + ($p, $c, $d, $sum, $desc, \@flines) +} + +# +# +# +$pwd = Cwd::cwd(); + +# +# Fetch the file list either from our aedist input +# or directly from the project itself. +# +my @filelines; +if ($aedist) { + local ($/); + undef $/; + my $infile_redir = ""; + my $contents; + if (! $infile || $infile eq "-") { + $contents = join('', ); + } else { + open(FILE, "<$infile") || die "Cannot open '$infile': $!\n"; + binmode(FILE); + $contents = join('', ); + close(FILE); + if (! File::Spec->file_name_is_absolute($infile)) { + $infile = File::Spec->catfile($pwd, $infile); + } + $infile_redir = " < $infile"; + } + + my $output = filter("aedist -l -unf", $contents); + my ($p, $c, $d, $s, $desc, $fl) = parse_change($output); + + $proj = $p if ! defined $proj; + $summary = $s; + $description = $desc; + @filelines = @$fl; + + if (! $exec) { + printit qq(MYTMP="/tmp/ae2cvs-ae.\$\$"\n), + qq(mkdir \$MYTMP\n), + qq(cd \$MYTMP\n); + printit q(perl -MMIME::Base64 -e 'undef $/; ($c = <>) =~ s/.*\n\n//ms; print decode_base64($c)'), + $infile_redir, + qq( | zcat), + qq( | cpio -i -d --quiet\n); + $aedir = '$MYTMP'; + push(@cleanup, $aedir); + } else { + $aedir = File::Spec->catfile(File::Spec->tmpdir, "ae2cvs-ae.$$"); + _mkdir($aedir); + push(@cleanup, $aedir); + _chdir($aedir); + + use MIME::Base64; + + $contents =~ s/.*\n\n//ms; + $contents = filter("zcat", decode_base64($contents)); + + open(CPIO, "|cpio -i -d --quiet"); + print CPIO $contents; + close(CPIO); + } + + $ae_copy = sub { + foreach my $dest (@_) { + my $source = File::Spec->catfile($aedir, "src", $dest); + execute(qq(cp $source $dest)); + } + } +} else { + $cnum = $ENV{AEGIS_CHANGE} if ! defined $cnum; + $proj = $ENV{AEGIS_PROJECT} if ! defined $proj; + + $common = "-lib " . join(" -lib ", @libraries) if @libraries; + $common = "$common -proj $proj" if $proj; + + my $output = `aegis -l cd $cnum -unf $common`; + my ($p, $c, $d, $s, $desc, $fl) = parse_change($output); + + $delta = $d; + $summary = $s; + $description = $desc; + @filelines = @$fl; + + if (! $delta) { + print STDERR "ae2cvs: No delta number, exiting.\n"; + exit 1; + } + + $ae_copy = sub { + execute(qq(aegis -cp -ind -delta $delta $common @_)); + } +} + +if (! $usedir) { + $usedir = File::Spec->catfile(File::Spec->tmpdir, "ae2cvs.$$"); + _mkdir($usedir); + push(@cleanup, $usedir); +} + +_chdir($usedir); + +$usepath = $usedir; +if (! File::Spec->file_name_is_absolute($usepath)) { + $usepath = File::Spec->catfile($pwd, $usepath); +} + +if (! -d File::Spec->catfile($usedir, "CVS")) { + $cvsmod = (split(/\./, $proj))[0] if ! defined $cvsmod; + + execute(qq($cvs_command co $cvsmod)); + + _chdir($cvsmod); + + $usepath = File::Spec->catfile($usepath, $cvsmod); +} + +# +# Figure out what we have to do to accomplish everything. +# +foreach (@filelines) { + my @arr = split(/\s+/, $_); + my $type = shift @arr; # source / test + my $act = shift @arr; # modify / create + my $file = pop @arr; + + if ($act eq "create" or $act eq "modify") { + # XXX Do we really only need to do this for + # ($act eq "create") files? + my (undef, $dirs, undef) = File::Spec->splitpath($file); + my $absdir = $usepath; + my $reldir; + my $d; + foreach $d (File::Spec->splitdir($dirs)) { + next if ! $d; + $absdir = File::Spec->catdir($absdir, $d); + $reldir = $reldir ? File::Spec->catdir($reldir, $d) : $d; + if (! -d $absdir && ! $seen_dir{$reldir}) { + $seen_dir{$reldir} = 1; + push(@mkdir_list, $reldir); + } + } + + push(@copy_list, $file); + + if ($act eq "create") { + push(@add_list, $file); + } + } elsif ($act eq "remove") { + push(@remove_list, $file); + } else { + print STDERR "Unsure how to '$act' the '$file' file.\n"; + } +} + +# Now go through and mkdir() the directories, +# adding them to the CVS tree as we do. +if (@mkdir_list) { + if (! $exec) { + printit qq(# The following "mkdir" and "cvs -Q add" calls are not\n), + qq(# necessary for any directories that already exist in the\n), + qq(# CVS tree but which aren't present locally.\n); + } + foreach (@mkdir_list) { + if (! $exec) { + printit qq(if test ! -d $_; then\n); + $indent = " "; + } + _mkdir($_); + execute(qq($cvs_command add $_)); + if (! $exec) { + $indent = ""; + printit qq(fi\n); + } + } + if (! $exec) { + printit qq(# End of directory creation.\n); + } +} + +# Copy in any files in the change, before we try to "cvs add" them. +$ae_copy->(@copy_list) if @copy_list; + +if (@add_list) { + execute(qq($cvs_command add @add_list)); +} + +if (@remove_list) { + execute(qq(rm -f @remove_list)); + execute(qq($cvs_command remove @remove_list)); +} + +# Last, commit the whole bunch. +$comment = $summary; +$comment .= "\n" . $description if $description; +$commit = qq($cvs_command commit -m '$comment' .); +if ($exec == 1) { + printit qq(# Execute the following to commit the changes:\n), + qq(# $commit\n); +} else { + execute($commit); +} + +_chdir($pwd); + +# +# Directory cleanup. +# +sub END { + my $dir; + foreach $dir (@cleanup) { + printit "rm -rf $dir\n"; + if ($exec) { + finddepth(sub { + # print STDERR "unlink($_)\n" if (!-d $_); + # print STDERR "rmdir($_)\n" if (-d $_ && $_ ne "."); + unlink($_) if (!-d $_); + rmdir($_) if (-d $_ && $_ ne "."); + 1; + }, $dir); + rmdir($dir) || print STDERR "Could not remove $dir: $!\n"; + } + } +} + +__END__; + +=head1 NAME + +ae2cvs - convert an Aegis change set to CVS commands + +=head1 SYNOPSIS + +ae2cvs [-aedist|-aegis] [-c change] [-d cvs_root] [-f file] [-l lib] + [-m module] [-n] [-p proj] [-q] [-u dir] [-v] [-x] [-X] + + -aedist use aedist format from input (default) + -aegis query aegis repository directly + -c change change number + -d cvs_root CVS root directory + -f file read aedist from file ('-' == stdin) + -l lib Aegis library directory + -m module CVS module + -n no execute + -p proj project name + -q quiet, don't print commands + -u dir use dir for CVS checkin + -v print version string and exit + -x execute the commands, but don't commit; + two or more -x options commit changes + -X execute the commands and commit changes + +=head1 DESCRIPTION + +The C utility can convert an Aegis change into a set of CVS (and +other) commands to make the corresponding change(s) to a carbon-copy CVS +repository. This can be used to keep a front-end CVS repository in sync +with changes made to an Aegis project, either manually or automatically +using the C attribute of the Aegis +project. + +By default, C makes no changes to any software, and only prints +out the necessary commands. These commands can be examined first for +safety, and then fed to any Bourne shell variant (sh, ksh, or bash) to +make the actual CVS changes. + +An option exists to have C execute the commands directly. + +=head1 OPTIONS + +The C utility supports the following options: + +=over 4 + +=item -aedist + +Reads an aedist change set. +By default, the change set is read from standard input, +or a file specified with the C<-f> option. + +=item -aegis + +Reads the change directly from the Aegis repository +by executing the proper C commands. + +=item -c change + +Specify the Aegis change number to be used. +The value of the C environment variable +is used by default. + +=item -d cvsroot + +Specify the CVS root directory to be used. +This option is passed explicitly to each executed C command. +The default behavior is to omit any C<-d> options +and let the executed C commands use the +C environment variable as they normally would. + +=item -f file + +Reads the aedist change set from the specified C, +or from standard input if C is C<'-'>. + +=item -l lib + +Specifies an Aegis library directory to be searched for global states +files and user state files. + +=item -m module + +Specifies the name of the CVS module to be brought up-to-date. +The default is to use the Aegis project name, +minus any branch numbers; +for example, given an Aegis project name of C, +the default CVS module name is C. + +=item -n + +No execute. Commands are printed (including a command for a final +commit of changes), but not executed. This is the default. + +=item -p proj + +Specifies the name of the Aegis project from which this change is taken. +The value of the C environment variable +is used by default. + +=item -q + +Quiet. Commands are not printed. + +=item -u dir + +Use the already checked-out CVS tree that exists at C +for the checkins and commits. +The default is to use a separately-created temporary directory. + +=item -v + +Print the version string and exit. + +=item -x + +Execute the commands to bring the CVS repository up to date, +except for the final commit of the changes. Two or more +C<-x> options will cause the change to be committed. + +=item -X + +Execute the commands to bring the CVS repository up to date, +including the final commit of the changes. + +=back + +=head1 ENVIRONMENT VARIABLES + +=over 4 + +=item AE2CVS_FLAGS + +Specifies any options to be used to initialize +the C utility. +Options on the command line override these values. + +=back + +=head1 AUTHOR + +Steven Knight (knight at baldmt dot com) + +=head1 BUGS + +If errors occur during the execution of the Aegis or CVS commands, and +the -X option is used, a partial change (consisting of those files for +which the command(s) succeeded) will be committed. It would be safer to +generate code to detect the error and print a warning. + +When a file has been deleted in Aegis, the standard whiteout file can +cause a regex failure in this script. It doesn't necessarily happen all +the time, though, so this needs more investigation. + +=head1 TODO + +Add an explicit test for using ae2cvs in the Aegis +integrate_pass_notify_command field to support fully keeping a +repository in sync automatically. + +=head1 COPYRIGHT + +Copyright 2001, 2002, 2003, 2004, 2005 Steven Knight. + +=head1 SEE ALSO + +aegis(1), cvs(1) diff --git a/bin/calibrate.py b/bin/calibrate.py new file mode 100644 index 0000000..c5d45ce --- /dev/null +++ b/bin/calibrate.py @@ -0,0 +1,81 @@ +#!/usr/bin/env python +# +# Copyright (c) 2009 The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import optparse +import os +import re +import subprocess +import sys + +variable_re = re.compile('^VARIABLE: (.*)$', re.M) +elapsed_re = re.compile('^ELAPSED: (.*)$', re.M) + +def main(argv=None): + if argv is None: + argv = sys.argv + + parser = optparse.OptionParser(usage="calibrate.py [-h] [-p PACKAGE], [--min time] [--max time] timings/*/*-run.py") + parser.add_option('--min', type='float', default=9.5, + help="minimum acceptable execution time (default 9.5)") + parser.add_option('--max', type='float', default=10.00, + help="maximum acceptable execution time (default 10.00)") + parser.add_option('-p', '--package', type="string", + help="package type") + opts, args = parser.parse_args(argv[1:]) + + os.environ['TIMESCONS_CALIBRATE'] = '1' + + for arg in args: + if len(args) > 1: + print arg + ':' + + command = [sys.executable, 'runtest.py', '--noqmtest'] + if opts.package: + command.extend(['-p', opts.package]) + command.append(arg) + + run = 1 + good = 0 + while good < 3: + p = subprocess.Popen(command, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT) + output = p.communicate()[0] + vm = variable_re.search(output) + em = elapsed_re.search(output) + elapsed = float(em.group(1)) + print "run %3d: %7.3f: %s" % (run, elapsed, ' '.join(vm.groups())) + if opts.min < elapsed and elapsed < opts.max: + good += 1 + else: + good = 0 + for v in vm.groups(): + var, value = v.split('=', 1) + value = int((int(value) * opts.max) / elapsed) + os.environ[var] = str(value) + run += 1 + + return 0 + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bin/caller-tree.py b/bin/caller-tree.py new file mode 100644 index 0000000..85bb599 --- /dev/null +++ b/bin/caller-tree.py @@ -0,0 +1,96 @@ +#!/usr/bin/env python +# +# Quick script to process the *summary* output from SCons.Debug.caller() +# and print indented calling trees with call counts. +# +# The way to use this is to add something like the following to a function +# for which you want information about who calls it and how many times: +# +# from SCons.Debug import caller +# caller(0, 1, 2, 3, 4, 5) +# +# Each integer represents how many stack frames back SCons will go +# and capture the calling information, so in the above example it will +# capture the calls six levels up the stack in a central dictionary. +# +# At the end of any run where SCons.Debug.caller() is used, SCons will +# print a summary of the calls and counts that looks like the following: +# +# Callers of Node/__init__.py:629(calc_signature): +# 1 Node/__init__.py:683(calc_signature) +# Callers of Node/__init__.py:676(gen_binfo): +# 6 Node/FS.py:2035(current) +# 1 Node/__init__.py:722(get_bsig) +# +# If you cut-and-paste that summary output and feed it to this script +# on standard input, it will figure out how these entries hook up and +# print a calling tree for each one looking something like: +# +# Node/__init__.py:676(gen_binfo) +# Node/FS.py:2035(current) 6 +# Taskmaster.py:253(make_ready_current) 18 +# Script/Main.py:201(make_ready) 18 +# +# Note that you should *not* look at the call-count numbers in the right +# hand column as the actual number of times each line *was called by* +# the function on the next line. Rather, it's the *total* number +# of times each function was found in the call chain for any of the +# calls to SCons.Debug.caller(). If you're looking at more than one +# function at the same time, for example, their counts will intermix. +# So use this to get a *general* idea of who's calling what, not for +# fine-grained performance tuning. + +import sys + +class Entry: + def __init__(self, file_line_func): + self.file_line_func = file_line_func + self.called_by = [] + self.calls = [] + +AllCalls = {} + +def get_call(flf): + try: + e = AllCalls[flf] + except KeyError: + e = AllCalls[flf] = Entry(flf) + return e + +prefix = 'Callers of ' + +c = None +for line in sys.stdin.readlines(): + if line[0] == '#': + pass + elif line[:len(prefix)] == prefix: + c = get_call(line[len(prefix):-2]) + else: + num_calls, flf = line.strip().split() + e = get_call(flf) + c.called_by.append((e, num_calls)) + e.calls.append(c) + +stack = [] + +def print_entry(e, level, calls): + print '%-72s%6s' % ((' '*2*level) + e.file_line_func, calls) + if e in stack: + print (' '*2*(level+1))+'RECURSION' + print + elif e.called_by: + stack.append(e) + for c in e.called_by: + print_entry(c[0], level+1, c[1]) + stack.pop() + else: + print + +for e in [ e for e in AllCalls.values() if not e.calls ]: + print_entry(e, 0, '') + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/bin/docdiff b/bin/docdiff new file mode 100644 index 0000000..c565a04 --- /dev/null +++ b/bin/docdiff @@ -0,0 +1,16 @@ +#!/bin/sh + +if test $# -eq 0; then + for f in doc/user/*.in; do + xml=doc/user/`basename $f .in`.xml + echo $f: + python bin/sconsoutput.py $f | diff $DIFFFLAGS $xml - + done +else + for a in $*; do + f=doc/user/$a.in + xml=doc/user/$a.xml + echo $f: + python bin/sconsoutput.py $f | diff $DIFFFLAGS $xml - + done +fi diff --git a/bin/docrun b/bin/docrun new file mode 100644 index 0000000..5ba4215 --- /dev/null +++ b/bin/docrun @@ -0,0 +1,16 @@ +#!/bin/sh + +if test $# -eq 0; then + for f in doc/user/*.in; do + xml=doc/user/`basename $f .in`.xml + echo $f: + python bin/sconsoutput.py $f + done +else + for a in $*; do + f=doc/user/$a.in + xml=doc/user/$a.xml + echo $f: + python bin/sconsoutput.py $f + done +fi diff --git a/bin/docupdate b/bin/docupdate new file mode 100644 index 0000000..0e1631b --- /dev/null +++ b/bin/docupdate @@ -0,0 +1,16 @@ +#!/bin/sh + +if test $# -eq 0; then + for f in doc/user/*.in; do + xml=doc/user/`basename $f .in`.xml + echo $f: + python bin/sconsoutput.py $f > $xml + done +else + for a in $*; do + f=doc/user/$a.in + xml=doc/user/$a.xml + echo $f: + python bin/sconsoutput.py $f > $xml + done +fi diff --git a/bin/files b/bin/files new file mode 100644 index 0000000..08b1caa --- /dev/null +++ b/bin/files @@ -0,0 +1,106 @@ +./SCons/Action.py +./SCons/Builder.py +./SCons/Conftest.py +./SCons/Debug.py +./SCons/Defaults.py +./SCons/Environment.py +./SCons/Errors.py +./SCons/Executor.py +./SCons/Job.py +./SCons/Node/Alias.py +./SCons/Node/FS.py +./SCons/Node/Python.py +./SCons/Node/__init__.py +./SCons/Options/__init__.py +./SCons/Options/BoolOption.py +./SCons/Options/EnumOption.py +./SCons/Options/ListOption.py +./SCons/Options/PackageOption.py +./SCons/Options/PathOption.py +./SCons/Platform/__init__.py +./SCons/Platform/aix.py +./SCons/Platform/cygwin.py +./SCons/Platform/hpux.py +./SCons/Platform/irix.py +./SCons/Platform/os2.py +./SCons/Platform/posix.py +./SCons/Platform/sunos.py +./SCons/Platform/win32.py +./SCons/Scanner/C.py +./SCons/Scanner/D.py +./SCons/Scanner/Fortran.py +./SCons/Scanner/IDL.py +./SCons/Scanner/Prog.py +./SCons/Scanner/__init__.py +./SCons/Script/SConscript.py +./SCons/Script/__init__.py +./SCons/Sig/MD5.py +./SCons/Sig/TimeStamp.py +./SCons/Sig/__init__.py +./SCons/Taskmaster.py +./SCons/Tool/__init__.py +./SCons/Tool/aixc++.py +./SCons/Tool/aixcc.py +./SCons/Tool/aixf77.py +./SCons/Tool/aixlink.py +./SCons/Tool/ar.py +./SCons/Tool/as.py +./SCons/Tool/bcc32.py +./SCons/Tool/c++.py +./SCons/Tool/cc.py +./SCons/Tool/CVS.py +./SCons/Tool/dmd.py +./SCons/Tool/default.py +./SCons/Tool/dvipdf.py +./SCons/Tool/dvips.py +./SCons/Tool/f77.py +./SCons/Tool/g++.py +./SCons/Tool/g77.py +./SCons/Tool/gas.py +./SCons/Tool/gcc.py +./SCons/Tool/gnulink.py +./SCons/Tool/hpc++.py +./SCons/Tool/hpcc.py +./SCons/Tool/hplink.py +./SCons/Tool/icc.py +./SCons/Tool/icl.py +./SCons/Tool/ifl.py +./SCons/Tool/ilink.py +./SCons/Tool/ilink32.py +./SCons/Tool/jar.py +./SCons/Tool/javac.py +./SCons/Tool/JavaCommon.py +./SCons/Tool/javah.py +./SCons/Tool/latex.py +./SCons/Tool/lex.py +./SCons/Tool/link.py +./SCons/Tool/m4.py +./SCons/Tool/masm.py +./SCons/Tool/midl.py +./SCons/Tool/mingw.py +./SCons/Tool/mslib.py +./SCons/Tool/mslink.py +./SCons/Tool/msvc.py +./SCons/Tool/msvs.py +./SCons/Tool/nasm.py +./SCons/Tool/pdflatex.py +./SCons/Tool/pdftex.py +./SCons/Tool/qt.py +./SCons/Tool/rmic.py +./SCons/Tool/sgiar.py +./SCons/Tool/sgic++.py +./SCons/Tool/sgicc.py +./SCons/Tool/sgilink.py +./SCons/Tool/sunar.py +./SCons/Tool/sunc++.py +./SCons/Tool/suncc.py +./SCons/Tool/sunlink.py +./SCons/Tool/swig.py +./SCons/Tool/tar.py +./SCons/Tool/tex.py +./SCons/Tool/tlib.py +./SCons/Tool/yacc.py +./SCons/Util.py +./SCons/Warnings.py +./SCons/__init__.py +./SCons/exitfuncs.py diff --git a/bin/import-test.py b/bin/import-test.py new file mode 100644 index 0000000..473a7ed --- /dev/null +++ b/bin/import-test.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python +# +# Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 The SCons Foundation +# +# tree2test.py - turn a directory tree into TestSCons code +# +# A quick script for importing directory hierarchies containing test +# cases that people supply (typically in a .zip or .tar.gz file) into a +# TestSCons.py script. No error checking or options yet, it just walks +# the first command-line argument (assumed to be the directory containing +# the test case) and spits out code looking like the following: +# +# test.subdir(['sub1'], +# ['sub1', 'sub2']) +# +# test.write(['sub1', 'file1'], """\ +# contents of file1 +# """) +# +# test.write(['sub1', 'sub2', 'file2'], """\ +# contents of file2 +# """) +# +# There's no massaging of contents, so any files that themselves contain +# """ triple-quotes will need to have their contents edited by hand. +# + +__revision__ = "bin/import-test.py 4577 2009/12/27 19:44:43 scons" + +import os.path +import sys + +directory = sys.argv[1] + +Top = None +TopPath = None + +class Dir: + def __init__(self, path): + self.path = path + self.entries = {} + def call_for_each_entry(self, func): + entries = self.entries + names = entries.keys() + names.sort() + for name in names: + func(name, entries[name]) + +def lookup(dirname): + global Top, TopPath + if not Top: + Top = Dir([]) + TopPath = dirname + os.sep + return Top + dirname = dirname.replace(TopPath, '') + dirs = dirname.split(os.sep) + t = Top + for d in dirs[:-1]: + t = t.entries[d] + node = t.entries[dirs[-1]] = Dir(dirs) + return node + +def make_nodes(arg, dirname, fnames): + dir = lookup(dirname) + for f in fnames: + dir.entries[f] = None + +def collect_dirs(l, dir): + if dir.path: + l.append(dir.path) + def recurse(n, d): + if d: + collect_dirs(l, d) + dir.call_for_each_entry(recurse) + +def print_files(dir): + def print_a_file(n, d): + if not d: + l = dir.path + [n] + sys.stdout.write('\ntest.write(%s, """\\\n' % l) + p = os.path.join(*([directory] + l)) + sys.stdout.write(open(p, 'r').read()) + sys.stdout.write('""")\n') + dir.call_for_each_entry(print_a_file) + + def recurse(n, d): + if d: + print_files(d) + dir.call_for_each_entry(recurse) + +os.path.walk(directory, make_nodes, None) + +subdir_list = [] +collect_dirs(subdir_list, Top) +subdir_list = [ str(l) for l in subdir_list ] +sys.stdout.write('test.subdir(' + ',\n '.join(subdir_list) + ')\n') + +print_files(Top) + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/bin/install_python.py b/bin/install_python.py new file mode 100644 index 0000000..ca1c8b7 --- /dev/null +++ b/bin/install_python.py @@ -0,0 +1,141 @@ +#!/usr/bin/env python +# +# A script for unpacking and installing different historic versions of +# Python in a consistent manner for side-by-side development testing. +# +# This was written for a Linux system (specifically Ubuntu) but should +# be reasonably generic to any POSIX-style system with a /usr/local +# hierarchy. + +import getopt +import os +import shutil +import sys + +from Command import CommandRunner, Usage + +all_versions = [ + #'1.5.2', # no longer available at python.org + '2.0.1', + '2.1.3', + '2.2', + '2.3.7', + '2.4.5', + #'2.5.2', + '2.6', +] + +def main(argv=None): + if argv is None: + argv = sys.argv + + all = False + downloads_dir = 'Downloads' + downloads_url = 'http://www.python.org/ftp/python' + sudo = 'sudo' + prefix = '/usr/local' + + short_options = 'ad:hnp:q' + long_options = ['all', 'help', 'no-exec', 'prefix=', 'quiet'] + + helpstr = """\ +Usage: install_python.py [-ahnq] [-d DIR] [-p PREFIX] [VERSION ...] + + -a, --all Install all SCons versions. + -d DIR, --downloads=DIR Downloads directory. + -h, --help Print this help and exit + -n, --no-exec No execute, just print command lines + -p PREFIX, --prefix=PREFIX Installation prefix. + -q, --quiet Quiet, don't print command lines +""" + + try: + try: + opts, args = getopt.getopt(argv[1:], short_options, long_options) + except getopt.error, msg: + raise Usage(msg) + + for o, a in opts: + if o in ('-a', '--all'): + all = True + elif o in ('-d', '--downloads'): + downloads_dir = a + elif o in ('-h', '--help'): + print helpstr + sys.exit(0) + elif o in ('-n', '--no-exec'): + CommandRunner.execute = CommandRunner.do_not_execute + elif o in ('-p', '--prefix'): + prefix = a + elif o in ('-q', '--quiet'): + CommandRunner.display = CommandRunner.do_not_display + except Usage, err: + sys.stderr.write(str(err.msg) + '\n') + sys.stderr.write('use -h to get help\n') + return 2 + + if all: + if args: + msg = 'install-scons.py: -a and version arguments both specified' + sys.stderr.write(msg) + sys.exit(1) + + args = all_versions + + cmd = CommandRunner() + + for version in args: + python = 'Python-' + version + tar_gz = os.path.join(downloads_dir, python + '.tgz') + tar_gz_url = os.path.join(downloads_url, version, python + '.tgz') + + if (version.startswith('1.5') or + version.startswith('1.6') or + version.startswith('2.0')): + + configureflags = '--with-threads' + + else: + + configureflags = '' + + cmd.subst_dictionary(locals()) + + if not os.path.exists(tar_gz): + if not os.path.exists(downloads_dir): + cmd.run('mkdir %(downloads_dir)s') + cmd.run('wget -O %(tar_gz)s %(tar_gz_url)s') + + cmd.run('tar zxf %(tar_gz)s') + + cmd.run('cd %(python)s') + + if (version.startswith('1.6') or + version.startswith('2.0')): + + def edit_modules_setup_in(): + content = open('Modules/Setup.in', 'r').read() + content = content.replace('\n#zlib', '\nzlib') + open('Modules/Setup.in', 'w').write(content) + + display = 'ed Modules/Setup.in <&1 | tee configure.out') + cmd.run('make 2>&1 | tee make.out') + cmd.run('%(sudo)s make install') + + cmd.run('%(sudo)s rm -f %(prefix)s/bin/{idle,pydoc,python,python-config,smtpd.py}') + + cmd.run('cd ..') + + cmd.run((shutil.rmtree, python), 'rm -rf %(python)s') + +if __name__ == "__main__": + sys.exit(main()) + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/bin/install_scons.py b/bin/install_scons.py new file mode 100644 index 0000000..e4e6aff --- /dev/null +++ b/bin/install_scons.py @@ -0,0 +1,216 @@ +#!/usr/bin/env python +# +# A script for unpacking and installing different historic versions of +# SCons in a consistent manner for side-by-side development testing. +# +# This abstracts the changes we've made to the SCons setup.py scripts in +# different versions so that, no matter what version is specified, it ends +# up installing the necessary script(s) and library into version-specific +# names that won't interfere with other things. +# +# By default, we expect to extract the .tar.gz files from a Downloads +# subdirectory in the current directory. +# +# Note that this script cleans up after itself, removing the extracted +# directory in which we do the build. +# +# This was written for a Linux system (specifically Ubuntu) but should +# be reasonably generic to any POSIX-style system with a /usr/local +# hierarchy. + +import getopt +import os +import shutil +import sys + +from Command import CommandRunner, Usage + +all_versions = [ + '0.01', + '0.02', + '0.03', + '0.04', + '0.05', + '0.06', + '0.07', + '0.08', + '0.09', + '0.10', + '0.11', + '0.12', + '0.13', + '0.14', + '0.90', + '0.91', + '0.92', + '0.93', + '0.94', + #'0.94.1', + '0.95', + #'0.95.1', + '0.96', + '0.96.1', + '0.96.90', + '0.96.91', + '0.96.92', + '0.96.93', + '0.96.94', + '0.96.95', + '0.96.96', + '0.97', + '0.97.0d20070809', + '0.97.0d20070918', + '0.97.0d20071212', + '0.98.0', + '0.98.1', + '0.98.2', + '0.98.3', + '0.98.4', + '0.98.5', + '1.0.0', + '1.0.0.d20080826', + '1.0.1', + '1.0.1.d20080915', + '1.0.1.d20081001', + '1.1.0', + '1.1.0.d20081104', + '1.1.0.d20081125', + '1.1.0.d20081207', + '1.2.0', + '1.2.0.d20090113', + '1.2.0.d20090223', +] + +def main(argv=None): + if argv is None: + argv = sys.argv + + all = False + downloads_dir = 'Downloads' + downloads_url = 'http://downloads.sourceforge.net/scons' + sudo = 'sudo' + prefix = '/usr/local' + python = sys.executable + + short_options = 'ad:hnp:q' + long_options = ['all', 'help', 'no-exec', 'prefix=', 'quiet'] + + helpstr = """\ +Usage: install_scons.py [-ahnq] [-d DIR] [-p PREFIX] [VERSION ...] + + -a, --all Install all SCons versions. + -d DIR, --downloads=DIR Downloads directory. + -h, --help Print this help and exit + -n, --no-exec No execute, just print command lines + -p PREFIX, --prefix=PREFIX Installation prefix. + -q, --quiet Quiet, don't print command lines +""" + + try: + try: + opts, args = getopt.getopt(argv[1:], short_options, long_options) + except getopt.error, msg: + raise Usage(msg) + + for o, a in opts: + if o in ('-a', '--all'): + all = True + elif o in ('-d', '--downloads'): + downloads_dir = a + elif o in ('-h', '--help'): + print helpstr + sys.exit(0) + elif o in ('-n', '--no-exec'): + CommandRunner.execute = CommandRunner.do_not_execute + elif o in ('-p', '--prefix'): + prefix = a + elif o in ('-q', '--quiet'): + CommandRunner.display = CommandRunner.do_not_display + except Usage, err: + sys.stderr.write(str(err.msg) + '\n') + sys.stderr.write('use -h to get help\n') + return 2 + + if all: + if args: + msg = 'install-scons.py: -a and version arguments both specified' + sys.stderr.write(msg) + sys.exit(1) + + args = all_versions + + cmd = CommandRunner() + + for version in args: + scons = 'scons-' + version + tar_gz = os.path.join(downloads_dir, scons + '.tar.gz') + tar_gz_url = os.path.join(downloads_url, scons + '.tar.gz') + + cmd.subst_dictionary(locals()) + + if not os.path.exists(tar_gz): + if not os.path.exists(downloads_dir): + cmd.run('mkdir %(downloads_dir)s') + cmd.run('wget -O %(tar_gz)s %(tar_gz_url)s') + + cmd.run('tar zxf %(tar_gz)s') + + cmd.run('cd %(scons)s') + + if version in ('0.01', '0.02', '0.03', '0.04', '0.05', + '0.06', '0.07', '0.08', '0.09', '0.10'): + + # 0.01 through 0.10 install /usr/local/bin/scons and + # /usr/local/lib/scons. The "scons" script knows how to + # look up the library in a version-specific directory, but + # we have to move both it and the library directory into + # the right version-specific name by hand. + cmd.run('%(python)s setup.py build') + cmd.run('%(sudo)s %(python)s setup.py install --prefix=%(prefix)s') + cmd.run('%(sudo)s mv %(prefix)s/bin/scons %(prefix)s/bin/scons-%(version)s') + cmd.run('%(sudo)s mv %(prefix)s/lib/scons %(prefix)s/lib/scons-%(version)s') + + elif version in ('0.11', '0.12', '0.13', '0.14', '0.90'): + + # 0.11 through 0.90 install /usr/local/bin/scons and + # /usr/local/lib/scons-%(version)s. We just need to move + # the script to a version-specific name. + cmd.run('%(python)s setup.py build') + cmd.run('%(sudo)s %(python)s setup.py install --prefix=%(prefix)s') + cmd.run('%(sudo)s mv %(prefix)s/bin/scons %(prefix)s/bin/scons-%(version)s') + + elif version in ('0.91', '0.92', '0.93', + '0.94', '0.94.1', + '0.95', '0.95.1', + '0.96', '0.96.1', '0.96.90'): + + # 0.91 through 0.96.90 install /usr/local/bin/scons, + # /usr/local/bin/sconsign and /usr/local/lib/scons-%(version)s. + # We need to move both scripts to version-specific names. + cmd.run('%(python)s setup.py build') + cmd.run('%(sudo)s %(python)s setup.py install --prefix=%(prefix)s') + cmd.run('%(sudo)s mv %(prefix)s/bin/scons %(prefix)s/bin/scons-%(version)s') + cmd.run('%(sudo)s mv %(prefix)s/bin/sconsign %(prefix)s/bin/sconsign-%(version)s') + lib_scons = os.path.join(prefix, 'lib', 'scons') + if os.path.isdir(lib_scons): + cmd.run('%(sudo)s mv %(prefix)s/lib/scons %(prefix)s/lib/scons-%(version)s') + + else: + + # Versions from 0.96.91 and later support what we want + # with a --no-scons-script option. + cmd.run('%(python)s setup.py build') + cmd.run('%(sudo)s %(python)s setup.py install --prefix=%(prefix)s --no-scons-script') + + cmd.run('cd ..') + + cmd.run((shutil.rmtree, scons), 'rm -rf %(scons)s') + +if __name__ == "__main__": + sys.exit(main()) + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/bin/linecount.py b/bin/linecount.py new file mode 100644 index 0000000..359f642 --- /dev/null +++ b/bin/linecount.py @@ -0,0 +1,123 @@ +#!/usr/bin/env python +# +# Copyright (c) 2001, 2002, 2003, 2004, 2005, 2006, 2007, 2008, 2009 The SCons Foundation +# +# Count statistics about SCons test and source files. This must be run +# against a fully-populated tree (for example, one that's been freshly +# checked out). +# +# A test file is anything under the src/ directory that begins with +# 'test_' or ends in 'Tests.py', or anything under the test/ directory +# that ends in '.py'. Note that runtest.py script does *not*, by default, +# consider the files that begin with 'test_' to be tests, because they're +# tests of SCons packaging and installation, not functional tests of +# SCons code. +# +# A source file is anything under the src/engine/ or src/script/ +# directories that ends in '.py' but does NOT begin with 'test_' +# or end in 'Tests.py'. +# +# We report the number of tests and sources, the total number of lines +# in each category, the number of non-blank lines, and the number of +# non-comment lines. The last figure (non-comment) lines is the most +# interesting one for most purposes. +# + +__revision__ = "bin/linecount.py 4577 2009/12/27 19:44:43 scons" + +import os.path +import string + +fmt = "%-16s %5s %7s %9s %11s %11s" + +class Collection(object): + def __init__(self, name, files=None, pred=None): + self._name = name + if files is None: + files = [] + self.files = files + if pred is None: + pred = lambda x: True + self.pred = pred + def __call__(self, fname): + return self.pred(fname) + def __len__(self): + return len(self.files) + def extend(self, files): + self.files.extend(files) + def lines(self): + try: + return self._lines + except AttributeError: + self._lines = lines = [] + for file in self.files: + file_lines = open(file).readlines() + lines.extend([s.lstrip() for s in file_lines]) + return lines + def non_blank(self): + return [s for s in self.lines() if s != ''] + def non_comment(self): + return [s for s in self.lines() if s == '' or s[0] != '#'] + def non_blank_non_comment(self): + return [s for s in self.lines() if s != '' and s[0] != '#'] + def printables(self): + return (self._name + ':', + len(self.files), + len(self.lines()), + len(self.non_blank()), + len(self.non_comment()), + len(self.non_blank_non_comment())) + +def is_Tests_py(x): + return x[-8:] == 'Tests.py' +def is_test_(x): + return x[:5] == 'test_' +def is_python(x): + return x[-3:] == '.py' +def is_source(x): + return is_python(x) and not is_Tests_py(x) and not is_test_(x) + +src_Tests_py_tests = Collection('src/ *Tests.py', pred=is_Tests_py) +src_test_tests = Collection('src/ test_*.py', pred=is_test_) +test_tests = Collection('test/ tests', pred=is_python) +sources = Collection('sources', pred=is_source) + +def t(arg, dirname, names): + try: names.remove('.svn') + except ValueError: pass + names = filter(arg, names) + arg.extend(map(lambda n, d=dirname: os.path.join(d, n), names)) + +os.path.walk('src', t, src_Tests_py_tests) +os.path.walk('src', t, src_test_tests) +os.path.walk('test', t, test_tests) +os.path.walk('src/engine', t, sources) +os.path.walk('src/script', t, sources) + +src_tests = Collection('src/ tests', src_Tests_py_tests.files + + src_test_tests.files) +all_tests = Collection('all tests', src_tests.files + test_tests.files) + +def ratio(over, under): + return "%.2f" % (float(len(over)) / float(len(under))) + +print fmt % ('', '', '', '', '', 'non-blank') +print fmt % ('', 'files', 'lines', 'non-blank', 'non-comment', 'non-comment') +print +print fmt % src_Tests_py_tests.printables() +print fmt % src_test_tests.printables() +print +print fmt % src_tests.printables() +print fmt % test_tests.printables() +print +print fmt % all_tests.printables() +print fmt % sources.printables() +print +print fmt % ('ratio:', + ratio(all_tests, sources), + ratio(all_tests.lines(), sources.lines()), + ratio(all_tests.non_blank(), sources.non_blank()), + ratio(all_tests.non_comment(), sources.non_comment()), + ratio(all_tests.non_blank_non_comment(), + sources.non_blank_non_comment()) + ) diff --git a/bin/makedocs b/bin/makedocs new file mode 100644 index 0000000..2278a97 --- /dev/null +++ b/bin/makedocs @@ -0,0 +1,18 @@ +#! /bin/sh + +# This script uses HappyDoc to create the HTML class documentation for +# SCons. It must be run from the src/engine directory. + +base=`basename $PWD` +if [ "$base" != "engine" ]; then + echo "You must run this script from the engine directory." + exit +fi + +DEVDIR=../../doc/developer +if [ ! -d $DEVDIR ]; then + mkdir $DEVDIR +fi + +SRCFILE=../../bin/files +happydoc -d $DEVDIR `cat $SRCFILE` diff --git a/bin/memlogs.py b/bin/memlogs.py new file mode 100644 index 0000000..9d957c9 --- /dev/null +++ b/bin/memlogs.py @@ -0,0 +1,49 @@ +#!/usr/bin/env python +# +# Copyright (c) 2005 The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import getopt +import sys + +filenames = sys.argv[1:] + +if not filenames: + print """Usage: memlogs.py file [...] + +Summarizes the --debug=memory numbers from one or more build logs. +""" + sys.exit(0) + +fmt = "%12s %12s %12s %12s %s" + +print fmt % ("pre-read", "post-read", "pre-build", "post-build", "") + +for fname in sys.argv[1:]: + lines = [l for l in open(fname).readlines() if l[:7] == 'Memory '] + t = tuple([l.split()[-1] for l in lines]) + (fname,) + print fmt % t + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/bin/memoicmp.py b/bin/memoicmp.py new file mode 100644 index 0000000..f45ecb0 --- /dev/null +++ b/bin/memoicmp.py @@ -0,0 +1,78 @@ +#!/usr/bin/env python +# +# A script to compare the --debug=memoizer output found int +# two different files. + +import sys,string + +def memoize_output(fname): + mout = {} + lines=filter(lambda words: + len(words) == 5 and + words[1] == 'hits' and words[3] == 'misses', + map(string.split, open(fname,'r').readlines())) + for line in lines: + mout[line[-1]] = ( int(line[0]), int(line[2]) ) + return mout + + +def memoize_cmp(filea, fileb): + ma = memoize_output(filea) + mb = memoize_output(fileb) + + print 'All output: %s / %s [delta]'%(filea, fileb) + print '----------HITS---------- ---------MISSES---------' + cfmt='%7d/%-7d [%d]' + ma_o = [] + mb_o = [] + mab = [] + for k in ma.keys(): + if k in mb.keys(): + if k not in mab: + mab.append(k) + else: + ma_o.append(k) + for k in mb.keys(): + if k in ma.keys(): + if k not in mab: + mab.append(k) + else: + mb_o.append(k) + + mab.sort() + ma_o.sort() + mb_o.sort() + + for k in mab: + hits = cfmt%(ma[k][0], mb[k][0], mb[k][0]-ma[k][0]) + miss = cfmt%(ma[k][1], mb[k][1], mb[k][1]-ma[k][1]) + print '%-24s %-24s %s'%(hits, miss, k) + + for k in ma_o: + hits = '%7d/ --'%(ma[k][0]) + miss = '%7d/ --'%(ma[k][1]) + print '%-24s %-24s %s'%(hits, miss, k) + + for k in mb_o: + hits = ' -- /%-7d'%(mb[k][0]) + miss = ' -- /%-7d'%(mb[k][1]) + print '%-24s %-24s %s'%(hits, miss, k) + + print '-'*(24+24+1+20) + + +if __name__ == "__main__": + if len(sys.argv) != 3: + print """Usage: %s file1 file2 + +Compares --debug=memomize output from file1 against file2."""%sys.argv[0] + sys.exit(1) + + memoize_cmp(sys.argv[1], sys.argv[2]) + sys.exit(0) + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/bin/objcounts.py b/bin/objcounts.py new file mode 100644 index 0000000..ca814b4 --- /dev/null +++ b/bin/objcounts.py @@ -0,0 +1,109 @@ +#!/usr/bin/env python +# +# Copyright (c) 2005 The SCons Foundation +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + +import re +import sys + +filenames = sys.argv[1:] + +if len(sys.argv) != 3: + print """Usage: objcounts.py file1 file2 + +Compare the --debug=object counts from two build logs. +""" + sys.exit(0) + +def fetch_counts(fname): + contents = open(fname).read() + m = re.search('\nObject counts:(\n\s[^\n]*)*', contents, re.S) + lines = m.group().split('\n') + list = [l.split() for l in lines if re.match('\s+\d', l)] + d = {} + for l in list: + d[l[-1]] = map(int, l[:-1]) + return d + +c1 = fetch_counts(sys.argv[1]) +c2 = fetch_counts(sys.argv[2]) + +common = {} +for k in c1.keys(): + try: + common[k] = (c1[k], c2[k]) + except KeyError: + # Transition: we added the module to the names of a bunch of + # the logged objects. Assume that c1 might be from an older log + # without the modules in the names, and look for an equivalent + # in c2. + if not '.' in k: + s = '.'+k + l = len(s) + for k2 in c2.keys(): + if k2[-l:] == s: + common[k2] = (c1[k], c2[k2]) + del c1[k] + del c2[k2] + break + else: + del c1[k] + del c2[k] + +def diffstr(c1, c2): + try: + d = c2 - c1 + except TypeError: + d = '' + else: + if d: + d = '[%+d]' % d + else: + d = '' + return " %5s/%-5s %-8s" % (c1, c2, d) + +def printline(c1, c2, classname): + print \ + diffstr(c1[2], c2[2]) + \ + diffstr(c1[3], c2[3]) + \ + ' ' + classname + +keys = common.keys() +keys.sort() +for k in keys: + c = common[k] + printline(c[0], c[1], k) + +keys = c1.keys() +keys.sort() +for k in keys: + printline(c1[k], ['--']*4, k) + +keys = c2.keys() +keys.sort() +for k in keys: + printline(['--']*4, c2[k], k) + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/bin/restore.sh b/bin/restore.sh new file mode 100644 index 0000000..a5da9e8 --- /dev/null +++ b/bin/restore.sh @@ -0,0 +1,90 @@ +#!/usr/bin/env sh +# +# Simple hack script to restore __revision__, __COPYRIGHT_, 1.2.0.d20091224 +# and other similar variables to what gets checked in to source. This +# comes in handy when people send in diffs based on the released source. +# + +if test "X$*" = "X"; then + DIRS="src test" +else + DIRS="$*" +fi + +SEPARATOR="================================================================================" + +header() { + arg_space="$1 " + dots=`echo "$arg_space" | sed 's/./\./g'` + echo "$SEPARATOR" | sed "s;$dots;$arg_space;" +} + +for i in `find $DIRS -name '*.py'`; do + header $i + ed $i <&2 + echo "${USAGE}" >&2 + exit 1 + ;; + esac +done + +shift `expr ${OPTIND} - 1` + +if test "X$1" = "X"; then + echo "${USAGE}" >&2 + exit 1 +fi + +if test "X${AEGIS_PROJECT}" = "X"; then + echo "$PROG: No AEGIS_PROJECT set." >&2 + echo "${USAGE}" >&2 + exit 1 +fi + +if test "X$DO" = "X"; then + DO="alrstz" +fi + +cmd() +{ + $PRINT "$*" + $EXECUTE "$*" +} + +CHANGE=$1 + +if test "X${SANITY_CHECK}" = "Xyes"; then + SCM="cvs" + SCMROOT="/home/scons/CVSROOT/scons" + DELTA=`aegis -l -ter cd ${CHANGE} | sed -n 's/.*, Delta \([0-9]*\)\./\1/p'` + if test "x${DELTA}" = "x"; then + echo "${PROG}: Could not find delta for change ${CHANGE}." >&2 + echo "Has this finished integrating? Change ${CHANGE} not distributed." >&2 + exit 1 + fi + PREV_DELTA=`expr ${DELTA} - 1` + COMMAND="scons-scmcheck -D ${PREV_DELTA} -d q -p ${AEGIS_PROJECT} -s ${SCM} ${SCMROOT}" + $PRINT "${COMMAND}" + OUTPUT=`${COMMAND}` + if test "X${OUTPUT}" != "X"; then + echo "${PROG}: ${SCMROOT} is not up to date:" >&2 + echo "${OUTPUT}" >& 2 + echo "Did you skip any changes? Change ${CHANGE} not distributed." >&2 + exit 1 + fi +fi + +if test X$EXECUTE != "X:" -a "X$SSH_AGENT_PID" = "X"; then + eval `ssh-agent` + ssh-add + trap 'eval `ssh-agent -k`; exit' 0 1 2 3 15 +fi + +cd + +BASELINE=`aesub -p ${AEGIS_PROJECT} -c ${CHANGE} '${Project trunk_name}'` + +TMPBLAE="/tmp/${BASELINE}.ae" +TMPCAE="/tmp/${AEGIS_PROJECT}.C${CHANGE}.ae" + +# Original values for SourceForge. +#SFLOGIN="stevenknight" +#SFHOST="scons.sourceforge.net" +#SFDEST="/home/groups/s/sc/scons/htdocs" + +SCONSLOGIN="scons" +SCONSHOST="manam.pair.com" +#SCONSDEST="public_html/production" +SCONSDEST="public_ftp" + +# +# Copy the baseline .ae to the constant location on SourceForge. +# +case "${DO}" in +*a* ) + cmd "aedist -s -bl -p ${AEGIS_PROJECT} > ${TMPBLAE}" + cmd "scp ${TMPBLAE} ${SCONSLOGIN}@${SCONSHOST}:${SCONSDEST}/${BASELINE}.ae" + cmd "rm ${TMPBLAE}" + ;; +esac + +# +# Copy the latest .tar.gz and .zip files to the constant location on +# SourceForge. +# +case "${DO}" in +*z* ) + BUILD_DIST=`aegis -p ${AEGIS_PROJECT} -cd -bl`/build/dist + SCONS_SRC_TAR_GZ=`echo ${AEGIS_PROJECT} | sed 's/scons./scons-src-/'`*.tar.gz + SCONS_SRC_ZIP=`echo ${AEGIS_PROJECT} | sed 's/scons./scons-src-/'`*.zip + cmd "scp ${BUILD_DIST}/${SCONS_SRC_TAR_GZ} ${SCONSLOGIN}@${SCONSHOST}:${SCONSDEST}/scons-src-latest.tar.gz" + cmd "scp ${BUILD_DIST}/${SCONS_SRC_ZIP} ${SCONSLOGIN}@${SCONSHOST}:${SCONSDEST}/scons-src-latest.zip" +esac + +# +# Sync Aegis tree with SourceForge. +# +# Cribbed and modified from Peter Miller's same-named script in +# /home/groups/a/ae/aegis/aegis at SourceForge. +# +# Guide to what this does with rsync: +# +# --rsh=ssh use ssh for the transfer +# -l copy symlinks as symlinks +# -p preserve permissions +# -r recursive +# -t preserve times +# -z compress data +# --stats file transfer statistics +# --exclude exclude files matching the pattern +# --delete delete files that don't exist locally +# --delete-excluded delete files that match the --exclude patterns +# --progress show progress during the transfer +# -v verbose +# +# We no longer use the --stats option. +# +case "${DO}" in +*r* ) + LOCAL=/home/scons/scons + REMOTE=/home/groups/s/sc/scons/scons + cmd "/usr/bin/rsync --rsh='ssh -l stevenknight' \ + -l -p -r -t -z \ + --exclude build \ + --exclude '*,D' \ + --exclude '*.pyc' \ + --exclude aegis.log \ + --exclude '.sconsign*' \ + --delete --delete-excluded \ + --progress -v \ + ${LOCAL}/. scons.sourceforge.net:${REMOTE}/." + ;; +esac + +# +# Sync the CVS tree with the local repository. +# +case "${DO}" in +*l* ) + ( + export CVSROOT=/home/scons/CVSROOT/scons + #cmd "ae2cvs -X -aegis -p ${AEGIS_PROJECT} -c ${CHANGE} -u $HOME/SCons/baldmt.com/scons" + cmd "ae-cvs-ci ${AEGIS_PROJECT} ${CHANGE}" + ) + ;; +esac + +# +# Sync the Subversion tree with Tigris.org. +# +case "${DO}" in +*t* ) + ( + SVN=http://scons.tigris.org/svn/scons + case ${AEGIS_PROJECT} in + scons.0.96 ) + SVN_URL=${SVN}/branches/core + ;; + scons.0.96.513 ) + SVN_URL=${SVN}/branches/sigrefactor + ;; + * ) + echo "$PROG: Don't know SVN branch for '${AEGIS_PROJECT}'" >&2 + exit 1 + ;; + esac + SVN_CO_FLAGS="--username stevenknight" + #cmd "ae2cvs -X -aegis -p ${AEGIS_PROJECT} -c ${CHANGE} -u $HOME/SCons/tigris.org/scons" + cmd "ae-svn-ci ${AEGIS_PROJECT} ${CHANGE} ${SVN_URL} ${SVN_CO_FLAGS}" + ) + ;; +esac + +# +# Sync the CVS tree with SourceForge. +# +case "${DO}" in +*s* ) + ( + export CVS_RSH=ssh + export CVSROOT=:ext:stevenknight@scons.cvs.sourceforge.net:/cvsroot/scons + #cmd "ae2cvs -X -aegis -p ${AEGIS_PROJECT} -c ${CHANGE} -u $HOME/SCons/sourceforge.net/scons" + cmd "ae-cvs-ci ${AEGIS_PROJECT} ${CHANGE}" + ) + ;; +esac + +# +# Send the change .ae to the scons-aedist mailing list +# +# The subject requires editing by hand... +# +#aedist -s -p ${AEGIS_PROJECT} ${CHANGE} > ${TMPCAE} +#aegis -l -p ${AEGIS_PROJECT} -c ${CHANGE} cd | +# pine -attach_and_delete ${TMPCAE} scons-aedist@lists.sourceforge.net diff --git a/bin/scons-diff.py b/bin/scons-diff.py new file mode 100644 index 0000000..11d1bda --- /dev/null +++ b/bin/scons-diff.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python +# +# scons-diff.py - diff-like utility for comparing SCons trees +# +# This supports most common diff options (with some quirks, like you can't +# just say -c and have it use a default value), but canonicalizes the +# various version strings within the file like __revision__, __build__, +# etc. so that you can diff trees without having to ignore changes in +# version lines. +# + +import difflib +import getopt +import os.path +import re +import sys + +Usage = """\ +Usage: scons-diff.py [OPTIONS] dir1 dir2 +Options: + -c NUM, --context=NUM Print NUM lines of copied context. + -h, --help Print this message and exit. + -n Don't canonicalize SCons lines. + -q, --quiet Print only whether files differ. + -r, --recursive Recursively compare found subdirectories. + -s Report when two files are the same. + -u NUM, --unified=NUM Print NUM lines of unified context. +""" + +opts, args = getopt.getopt(sys.argv[1:], + 'c:dhnqrsu:', + ['context=', 'help', 'recursive', 'unified=']) + +diff_type = None +edit_type = None +context = 2 +recursive = False +report_same = False +diff_options = [] + +def diff_line(left, right): + if diff_options: + opts = ' ' + ' '.join(diff_options) + else: + opts = '' + print 'diff%s %s %s' % (opts, left, right) + +for o, a in opts: + if o in ('-c', '-u'): + diff_type = o + context = int(a) + diff_options.append(o) + elif o in ('-h', '--help'): + print Usage + sys.exit(0) + elif o in ('-n'): + diff_options.append(o) + edit_type = o + elif o in ('-q'): + diff_type = o + diff_line = lambda l, r: None + elif o in ('-r', '--recursive'): + recursive = True + diff_options.append(o) + elif o in ('-s'): + report_same = True + +try: + left, right = args +except ValueError: + sys.stderr.write(Usage) + sys.exit(1) + +def quiet_diff(a, b, fromfile='', tofile='', + fromfiledate='', tofiledate='', n=3, lineterm='\n'): + """ + A function with the same calling signature as difflib.context_diff + (diff -c) and difflib.unified_diff (diff -u) but which prints + output like the simple, unadorned 'diff" command. + """ + if a == b: + return [] + else: + return ['Files %s and %s differ\n' % (fromfile, tofile)] + +def simple_diff(a, b, fromfile='', tofile='', + fromfiledate='', tofiledate='', n=3, lineterm='\n'): + """ + A function with the same calling signature as difflib.context_diff + (diff -c) and difflib.unified_diff (diff -u) but which prints + output like the simple, unadorned 'diff" command. + """ + sm = difflib.SequenceMatcher(None, a, b) + def comma(x1, x2): + return x1+1 == x2 and str(x2) or '%s,%s' % (x1+1, x2) + result = [] + for op, a1, a2, b1, b2 in sm.get_opcodes(): + if op == 'delete': + result.append("%sd%d\n" % (comma(a1, a2), b1)) + result.extend(map(lambda l: '< ' + l, a[a1:a2])) + elif op == 'insert': + result.append("%da%s\n" % (a1, comma(b1, b2))) + result.extend(map(lambda l: '> ' + l, b[b1:b2])) + elif op == 'replace': + result.append("%sc%s\n" % (comma(a1, a2), comma(b1, b2))) + result.extend(map(lambda l: '< ' + l, a[a1:a2])) + result.append('---\n') + result.extend(map(lambda l: '> ' + l, b[b1:b2])) + return result + +diff_map = { + '-c' : difflib.context_diff, + '-q' : quiet_diff, + '-u' : difflib.unified_diff, +} + +diff_function = diff_map.get(diff_type, simple_diff) + +baseline_re = re.compile('(# |@REM )/home/\S+/baseline/') +comment_rev_re = re.compile('(# |@REM )(\S+) 0.96.[CD]\d+ \S+ \S+( knight)') +revision_re = re.compile('__revision__ = "[^"]*"') +build_re = re.compile('__build__ = "[^"]*"') +date_re = re.compile('__date__ = "[^"]*"') + +def lines_read(file): + return open(file).readlines() + +def lines_massage(file): + text = open(file).read() + text = baseline_re.sub('\\1', text) + text = comment_rev_re.sub('\\1\\2\\3', text) + text = revision_re.sub('__revision__ = "bin/scons-diff.py"', text) + text = build_re.sub('__build__ = "0.96.92.DXXX"', text) + text = date_re.sub('__date__ = "2006/08/25 02:59:00"', text) + return text.splitlines(1) + +lines_map = { + '-n' : lines_read, +} + +lines_function = lines_map.get(edit_type, lines_massage) + +def do_diff(left, right, diff_subdirs): + if os.path.isfile(left) and os.path.isfile(right): + diff_file(left, right) + elif not os.path.isdir(left): + diff_file(left, os.path.join(right, os.path.split(left)[1])) + elif not os.path.isdir(right): + diff_file(os.path.join(left, os.path.split(right)[1]), right) + elif diff_subdirs: + diff_dir(left, right) + +def diff_file(left, right): + l = lines_function(left) + r = lines_function(right) + d = diff_function(l, r, left, right, context) + try: + text = ''.join(d) + except IndexError: + sys.stderr.write('IndexError diffing %s and %s\n' % (left, right)) + else: + if text: + diff_line(left, right) + print text, + elif report_same: + print 'Files %s and %s are identical' % (left, right) + +def diff_dir(left, right): + llist = os.listdir(left) + rlist = os.listdir(right) + u = {} + for l in llist: + u[l] = 1 + for r in rlist: + u[r] = 1 + clist = [ x for x in u.keys() if x[-4:] != '.pyc' ] + clist.sort() + for x in clist: + if x in llist: + if x in rlist: + do_diff(os.path.join(left, x), + os.path.join(right, x), + recursive) + else: + print 'Only in %s: %s' % (left, x) + else: + print 'Only in %s: %s' % (right, x) + +do_diff(left, right, True) + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/bin/scons-proc.py b/bin/scons-proc.py new file mode 100644 index 0000000..cc3b085 --- /dev/null +++ b/bin/scons-proc.py @@ -0,0 +1,283 @@ +#!/usr/bin/env python +# +# Process a list of Python and/or XML files containing SCons documentation. +# +# This script creates formatted lists of the Builders, Tools or +# construction variables documented in the specified XML files. +# +# Dependening on the options, the lists are output in either +# DocBook-formatted generated XML files containing the summary text +# and/or .mod files contining the ENTITY definitions for each item, +# or in man-page-formatted output. +# +import getopt +import os.path +import re +import string +import StringIO +import sys +import xml.sax + +import SConsDoc + +base_sys_path = [os.getcwd() + '/build/test-tar-gz/lib/scons'] + sys.path + +helpstr = """\ +Usage: scons-proc.py [--man|--xml] + [-b file(s)] [-t file(s)] [-v file(s)] [infile ...] +Options: + -b file(s) dump builder information to the specified file(s) + -t file(s) dump tool information to the specified file(s) + -v file(s) dump variable information to the specified file(s) + --man print info in man page format, each -[btv] argument + is a single file name + --xml (default) print info in SML format, each -[btv] argument + is a pair of comma-separated .gen,.mod file names +""" + +opts, args = getopt.getopt(sys.argv[1:], + "b:ht:v:", + ['builders=', 'help', + 'man', 'xml', 'tools=', 'variables=']) + +buildersfiles = None +output_type = '--xml' +toolsfiles = None +variablesfiles = None + +for o, a in opts: + if o in ['-b', '--builders']: + buildersfiles = a + elif o in ['-h', '--help']: + sys.stdout.write(helpstr) + sys.exit(0) + elif o in ['--man', '--xml']: + output_type = o + elif o in ['-t', '--tools']: + toolsfiles = a + elif o in ['-v', '--variables']: + variablesfiles = a + +h = SConsDoc.SConsDocHandler() +saxparser = xml.sax.make_parser() +saxparser.setContentHandler(h) +saxparser.setErrorHandler(h) + +xml_preamble = """\ + + +""" + +xml_postamble = """\ + +""" + +for f in args: + _, ext = os.path.splitext(f) + if ext == '.py': + dir, _ = os.path.split(f) + if dir: + sys.path = [dir] + base_sys_path + module = SConsDoc.importfile(f) + h.set_file_info(f, len(xml_preamble.split('\n'))) + try: + content = module.__scons_doc__ + except AttributeError: + content = None + else: + del module.__scons_doc__ + else: + h.set_file_info(f, len(xml_preamble.split('\n'))) + content = open(f).read() + if content: + content = content.replace('&', '&') + input = xml_preamble + content + xml_postamble + try: + saxparser.parse(StringIO.StringIO(input)) + except: + sys.stderr.write("error in %s\n" % f) + raise + +Warning = """\ + +""" + +Regular_Entities_Header = """\ + +""" + +Link_Entities_Header = """\ + +""" + +class SCons_XML: + def __init__(self, entries, **kw): + values = entries.values() + values.sort() + self.values = values + for k, v in kw.items(): + setattr(self, k, v) + def fopen(self, name): + if name == '-': + return sys.stdout + return open(name, 'w') + +class SCons_XML_to_XML(SCons_XML): + def write(self, files): + gen, mod = string.split(files, ',') + g.write_gen(gen) + g.write_mod(mod) + def write_gen(self, filename): + if not filename: + return + f = self.fopen(filename) + for v in self.values: + f.write('\n\n' % + (self.prefix, self.idfunc(v.name))) + for term in self.termfunc(v.name): + f.write('<%s>%s\n' % + (self.tag, term, self.tag)) + f.write('\n') + for chunk in v.summary.body: + f.write(str(chunk)) + if v.sets: + s = map(lambda x: '&cv-link-%s;' % x, v.sets) + f.write('\n') + f.write('Sets: ' + ', '.join(s) + '.\n') + f.write('\n') + if v.uses: + u = map(lambda x: '&cv-link-%s;' % x, v.uses) + f.write('\n') + f.write('Uses: ' + ', '.join(u) + '.\n') + f.write('\n') + f.write('\n') + f.write('\n') + def write_mod(self, filename): + if not filename: + return + f = self.fopen(filename) + f.write(Warning) + f.write('\n') + f.write(Regular_Entities_Header % self.description) + f.write('\n') + for v in self.values: + f.write('%s">\n' % + (self.prefix, self.idfunc(v.name), + self.tag, self.entityfunc(v.name), self.tag)) + f.write('\n') + f.write(Warning) + f.write('\n') + f.write(Link_Entities_Header % self.description) + f.write('\n') + for v in self.values: + f.write('<%s>%s\'>\n' % + (self.prefix, self.idfunc(v.name), + self.prefix, self.idfunc(v.name), + self.tag, self.entityfunc(v.name), self.tag)) + f.write('\n') + f.write(Warning) + +class SCons_XML_to_man(SCons_XML): + def mansep(self): + return ['\n'] + def initial_chunks(self, name): + return [name] + def write(self, filename): + if not filename: + return + f = self.fopen(filename) + chunks = [] + for v in self.values: + chunks.extend(self.mansep()) + for n in self.initial_chunks(v.name): + chunks.append('.IP %s\n' % n) + chunks.extend(map(str, v.summary.body)) + + body = ''.join(chunks) + body = string.replace(body, '', '.ES') + body = string.replace(body, '', '.EE') + body = string.replace(body, '\n\n\n', '\n\n') + body = string.replace(body, '\n', '') + body = string.replace(body, '', '\n') + body = string.replace(body, '\n', '') + body = re.sub('\.EE\n\n+(?!\.IP)', '.EE\n.IP\n', body) + body = re.sub('&(scons|SConstruct|SConscript|jar);', r'\\fB\1\\fP', body) + body = string.replace(body, '&Dir;', r'\fBDir\fP') + body = string.replace(body, '⌖', r'\fItarget\fP') + body = string.replace(body, '&source;', r'\fIsource\fP') + body = re.sub('&b(-link)?-([^;]*);', r'\\fB\2\\fP()', body) + body = re.sub('&cv(-link)?-([^;]*);', r'$\2', body) + body = re.sub(r'<(command|envar|filename|literal|option)>([^<]*)', + r'\\fB\2\\fP', body) + body = re.sub(r'<(classname|emphasis|varname)>([^<]*)', + r'\\fI\2\\fP', body) + body = re.compile(r'^\\f([BI])(.*)\\fP\s*$', re.M).sub(r'.\1 \2', body) + body = re.compile(r'^\\f([BI])(.*)\\fP(\S+)', re.M).sub(r'.\1R \2 \3', body) + body = string.replace(body, '<', '<') + body = string.replace(body, '>', '>') + body = re.sub(r'\\([^f])', r'\\\\\1', body) + body = re.compile("^'\\\\\\\\", re.M).sub("'\\\\", body) + body = re.compile(r'^\.([BI]R?) -', re.M).sub(r'.\1 \-', body) + body = re.compile(r'^\.([BI]R?) (\S+)\\\\(\S+)', re.M).sub(r'.\1 "\2\\\\\\\\\2"', body) + body = re.compile(r'\\f([BI])-', re.M).sub(r'\\f\1\-', body) + f.write(body) + +if output_type == '--man': + processor_class = SCons_XML_to_man +elif output_type == '--xml': + processor_class = SCons_XML_to_XML +else: + sys.stderr.write("Unknown output type '%s'\n" % output_type) + sys.exit(1) + +if buildersfiles: + g = processor_class(h.builders, + description = 'builder', + prefix = 'b-', + tag = 'function', + idfunc = lambda x: x, + termfunc = lambda x: [x+'()', 'env.'+x+'()'], + entityfunc = lambda x: x) + + g.mansep = lambda: ['\n', "'\\" + '"'*69 + '\n'] + g.initial_chunks = lambda n: [n+'()', 'env.'+n+'()'] + + g.write(buildersfiles) + +if toolsfiles: + g = processor_class(h.tools, + description = 'tool', + prefix = 't-', + tag = 'literal', + idfunc = lambda x: string.replace(x, '+', 'X'), + termfunc = lambda x: [x], + entityfunc = lambda x: x) + + g.write(toolsfiles) + +if variablesfiles: + g = processor_class(h.cvars, + description = 'construction variable', + prefix = 'cv-', + tag = 'envar', + idfunc = lambda x: x, + termfunc = lambda x: [x], + entityfunc = lambda x: '$'+x) + + g.write(variablesfiles) + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/bin/scons-review.sh b/bin/scons-review.sh new file mode 100755 index 0000000..f126333 --- /dev/null +++ b/bin/scons-review.sh @@ -0,0 +1,24 @@ +#!/bin/sh + +case "$1" in +'') exec svn diff --diff-cmd diff -x -c $* ;; +-m) svn diff --diff-cmd diff -x -c $* | alpine scons-dev ;; +*) echo "Error: unknown option '$1"; exit 1 ;; +esac + +# OLD CODE FOR USE WITH AEGIS +# +#if test $# -ne 1; then +# echo "Usage: scons-review change#" >&2 +# exit 1 +#fi +#if test "X$AEGIS_PROJECT" = "X"; then +# echo "scons-review: AEGIS_PROJECT is not set" >&2 +# exit 1 +#fi +#DIR=`aegis -cd -dd $*` +#if test "X${DIR}" = "X"; then +# echo "scons-review: No Aegis directory for '$*'" >&2 +# exit 1 +#fi +#(cd ${DIR} && find * -name '*,D' | sort | xargs cat) | pine scons-dev diff --git a/bin/scons-test.py b/bin/scons-test.py new file mode 100644 index 0000000..aa03d72 --- /dev/null +++ b/bin/scons-test.py @@ -0,0 +1,247 @@ +#!/usr/bin/env python +# +# A script that takes an scons-src-{version}.zip file, unwraps it in +# a temporary location, and calls runtest.py to execute one or more of +# its tests. +# +# The default is to download the latest scons-src archive from the SCons +# web site, and to execute all of the tests. +# +# With a little more work, this will become the basis of an automated +# testing and reporting system that anyone will be able to use to +# participate in testing SCons on their system and regularly reporting +# back the results. A --xml option is a stab at gathering a lot of +# relevant information about the system, the Python version, etc., +# so that problems on different platforms can be identified sooner. +# + +import getopt +import imp +import os +import os.path +import string +import sys +import tempfile +import time +import urllib +import zipfile + +helpstr = """\ +Usage: scons-test.py [-f zipfile] [-o outdir] [-v] [--xml] [runtest arguments] +Options: + -f FILE Specify input .zip FILE name + -o DIR, --out DIR Change output directory name to DIR + -v, --verbose Print file names when extracting + --xml XML output +""" + +opts, args = getopt.getopt(sys.argv[1:], + "f:o:v", + ['file=', 'out=', 'verbose', 'xml']) + +format = None +outdir = None +printname = lambda x: x +inputfile = 'http://scons.sourceforge.net/scons-src-latest.zip' + +for o, a in opts: + if o == '-f' or o == '--file': + inputfile = a + elif o == '-o' or o == '--out': + outdir = a + elif o == '-v' or o == '--verbose': + def printname(x): + print x + elif o == '--xml': + format = o + +startdir = os.getcwd() + +tempfile.template = 'scons-test.' +tempdir = tempfile.mktemp() + +if not os.path.exists(tempdir): + os.mkdir(tempdir) + def cleanup(tempdir=tempdir): + import shutil + os.chdir(startdir) + shutil.rmtree(tempdir) + sys.exitfunc = cleanup + +# Fetch the input file if it happens to be across a network somewhere. +# Ohmigod, does Python make this simple... +inputfile, headers = urllib.urlretrieve(inputfile) + +# Unzip the header file in the output directory. We use our own code +# (lifted from scons-unzip.py) to make the output subdirectory name +# match the basename of the .zip file. +zf = zipfile.ZipFile(inputfile, 'r') + +if outdir is None: + name, _ = os.path.splitext(os.path.basename(inputfile)) + outdir = os.path.join(tempdir, name) + +def outname(n, outdir=outdir): + l = [] + while 1: + n, tail = os.path.split(n) + if not n: + break + l.append(tail) + l.append(outdir) + l.reverse() + return apply(os.path.join, l) + +for name in zf.namelist(): + dest = outname(name) + dir = os.path.dirname(dest) + try: + os.makedirs(dir) + except: + pass + printname(dest) + # if the file exists, then delete it before writing + # to it so that we don't end up trying to write to a symlink: + if os.path.isfile(dest) or os.path.islink(dest): + os.unlink(dest) + if not os.path.isdir(dest): + open(dest, 'w').write(zf.read(name)) + +os.chdir(outdir) + +# Load (by hand) the SCons modules we just unwrapped so we can +# extract their version information. Note that we have to override +# SCons.Script.main() with a do_nothing() function, because loading up +# the 'scons' script will actually try to execute SCons... +src_script = os.path.join(outdir, 'src', 'script') +src_engine = os.path.join(outdir, 'src', 'engine') +src_engine_SCons = os.path.join(src_engine, 'SCons') + +fp, pname, desc = imp.find_module('SCons', [src_engine]) +SCons = imp.load_module('SCons', fp, pname, desc) + +fp, pname, desc = imp.find_module('Script', [src_engine_SCons]) +SCons.Script = imp.load_module('Script', fp, pname, desc) + +def do_nothing(): + pass +SCons.Script.main = do_nothing + +fp, pname, desc = imp.find_module('scons', [src_script]) +scons = imp.load_module('scons', fp, pname, desc) +fp.close() + +# Default is to run all the tests by passing the -a flags to runtest.py. +if not args: + runtest_args = '-a' +else: + runtest_args = string.join(args) + +if format == '--xml': + + print "" + print " " + sys_keys = ['byteorder', 'exec_prefix', 'executable', 'maxint', 'maxunicode', 'platform', 'prefix', 'version', 'version_info'] + for k in sys_keys: + print " <%s>%s" % (k, sys.__dict__[k], k) + print " " + + fmt = '%a %b %d %H:%M:%S %Y' + print " " + + print " %s" % tempdir + + def print_version_info(tag, module): + print " <%s>" % tag + print " %s" % module.__version__ + print " %s" % module.__build__ + print " %s" % module.__buildsys__ + print " %s" % module.__date__ + print " %s" % module.__developer__ + print " " % tag + + print " " + print_version_info("script", scons) + print_version_info("engine", SCons) + print " " + + environ_keys = [ + 'PATH', + 'SCONSFLAGS', + 'SCONS_LIB_DIR', + 'PYTHON_ROOT', + 'QTDIR', + + 'COMSPEC', + 'INTEL_LICENSE_FILE', + 'INCLUDE', + 'LIB', + 'MSDEVDIR', + 'OS', + 'PATHEXT', + 'SystemRoot', + 'TEMP', + 'TMP', + 'USERNAME', + 'VXDOMNTOOLS', + 'WINDIR', + 'XYZZY' + + 'ENV', + 'HOME', + 'LANG', + 'LANGUAGE', + 'LOGNAME', + 'MACHINE', + 'OLDPWD', + 'PWD', + 'OPSYS', + 'SHELL', + 'TMPDIR', + 'USER', + ] + + print " " + #keys = os.environ.keys() + keys = environ_keys + keys.sort() + for key in keys: + value = os.environ.get(key) + if value: + print " " + print " %s" % key + print " %s" % value + print " " + print " " + + command = '"%s" runtest.py -q -o - --xml %s' % (sys.executable, runtest_args) + #print command + os.system(command) + print "" + +else: + + def print_version_info(tag, module): + print "\t%s: v%s.%s, %s, by %s on %s" % (tag, + module.__version__, + module.__build__, + module.__date__, + module.__developer__, + module.__buildsys__) + + print "SCons by Steven Knight et al.:" + print_version_info("script", scons) + print_version_info("engine", SCons) + + command = '"%s" runtest.py %s' % (sys.executable, runtest_args) + #print command + os.system(command) + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/bin/scons-unzip.py b/bin/scons-unzip.py new file mode 100644 index 0000000..28c73f8 --- /dev/null +++ b/bin/scons-unzip.py @@ -0,0 +1,76 @@ +#!/usr/bin/env python +# +# A quick script to unzip a .zip archive and put the files in a +# subdirectory that matches the basename of the .zip file. +# +# This is actually generic functionality, it's not SCons-specific, but +# I'm using this to make it more convenient to manage working on multiple +# changes on Windows, where I don't have access to my Aegis tools. +# + +import getopt +import os.path +import sys +import zipfile + +helpstr = """\ +Usage: scons-unzip.py [-o outdir] zipfile +Options: + -o DIR, --out DIR Change output directory name to DIR + -v, --verbose Print file names when extracting +""" + +opts, args = getopt.getopt(sys.argv[1:], + "o:v", + ['out=', 'verbose']) + +outdir = None +printname = lambda x: x + +for o, a in opts: + if o == '-o' or o == '--out': + outdir = a + elif o == '-v' or o == '--verbose': + def printname(x): + print x + +if len(args) != 1: + sys.stderr.write("scons-unzip.py: \n") + sys.exit(1) + +zf = zipfile.ZipFile(str(args[0]), 'r') + +if outdir is None: + outdir, _ = os.path.splitext(os.path.basename(args[0])) + +def outname(n, outdir=outdir): + l = [] + while 1: + n, tail = os.path.split(n) + if not n: + break + l.append(tail) + l.append(outdir) + l.reverse() + return apply(os.path.join, l) + +for name in zf.namelist(): + dest = outname(name) + dir = os.path.dirname(dest) + try: + os.makedirs(dir) + except: + pass + printname(dest) + # if the file exists, then delete it before writing + # to it so that we don't end up trying to write to a symlink: + if os.path.isfile(dest) or os.path.islink(dest): + os.unlink(dest) + if not os.path.isdir(dest): + open(dest, 'w').write(zf.read(name)) + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/bin/scons_dev_master.py b/bin/scons_dev_master.py new file mode 100644 index 0000000..5e8df41 --- /dev/null +++ b/bin/scons_dev_master.py @@ -0,0 +1,215 @@ +#!/bin/sh +# + +# A script for turning a generic Ubuntu system into a master for +# SCons development. + +import getopt +import sys + +from Command import CommandRunner, Usage + +INITIAL_PACKAGES = [ + 'subversion', +] + +INSTALL_PACKAGES = [ + 'wget', +] + +PYTHON_PACKAGES = [ + 'g++', + 'gcc', + 'make', + 'zlib1g-dev', +] + +BUILDING_PACKAGES = [ + 'docbook', + 'docbook-dsssl', + 'docbook-utils', + 'docbook-xml', + 'groff-base', + 'jade', + 'jadetex', + 'man2html', + 'python-epydoc', + 'rpm', + 'sp', + 'tar', + + # additional packages that Bill Deegan's web page suggests + #'docbook-to-man', + #'docbook-xsl', + #'docbook2x', + #'tetex-bin', + #'tetex-latex', +] + +DOCUMENTATION_PACKAGES = [ + 'docbook-doc', + 'epydoc-doc', + 'gcc-doc', + 'pkg-config', + 'python-doc', + 'sun-java5-doc', + 'sun-java6-doc', + 'swig-doc', + 'texlive-doc', +] + +TESTING_PACKAGES = [ + 'bison', + 'cssc', + 'cvs', + 'flex', + 'g++', + 'gcc', + 'gcj', + 'ghostscript', +# 'libgcj7-dev', + 'm4', + 'openssh-client', + 'openssh-server', + 'python-profiler', + 'python-all-dev', + 'rcs', + 'rpm', + 'sun-java5-jdk', + 'sun-java6-jdk', + 'swig', + 'texlive-base-bin', + 'texlive-latex-base', + 'texlive-latex-extra', + 'zip', +] + +BUILDBOT_PACKAGES = [ + 'buildbot', + 'cron', +] + +default_args = [ + 'upgrade', + 'checkout', + 'building', + 'testing', + 'python-versions', + 'scons-versions', +] + +def main(argv=None): + if argv is None: + argv = sys.argv + + short_options = 'hnqy' + long_options = ['help', 'no-exec', 'password=', 'quiet', 'username=', + 'yes', 'assume-yes'] + + helpstr = """\ +Usage: scons_dev_master.py [-hnqy] [--password PASSWORD] [--username USER] + [ACTIONS ...] + + ACTIONS (in default order): + upgrade Upgrade the system + checkout Check out SCons + building Install packages for building SCons + testing Install packages for testing SCons + scons-versions Install versions of SCons + python-versions Install versions of Python + + ACTIONS (optional): + buildbot Install packages for running BuildBot +""" + + scons_url = 'http://scons.tigris.org/svn/scons/trunk' + sudo = 'sudo' + password = '""' + username = 'guest' + yesflag = '' + + try: + try: + opts, args = getopt.getopt(argv[1:], short_options, long_options) + except getopt.error, msg: + raise Usage(msg) + + for o, a in opts: + if o in ('-h', '--help'): + print helpstr + sys.exit(0) + elif o in ('-n', '--no-exec'): + CommandRunner.execute = CommandRunner.do_not_execute + elif o in ('--password'): + password = a + elif o in ('-q', '--quiet'): + CommandRunner.display = CommandRunner.do_not_display + elif o in ('--username'): + username = a + elif o in ('-y', '--yes', '--assume-yes'): + yesflag = o + except Usage, err: + sys.stderr.write(str(err.msg) + '\n') + sys.stderr.write('use -h to get help\n') + return 2 + + if not args: + args = default_args + + initial_packages = ' '.join(INITIAL_PACKAGES) + install_packages = ' '.join(INSTALL_PACKAGES) + building_packages = ' '.join(BUILDING_PACKAGES) + testing_packages = ' '.join(TESTING_PACKAGES) + buildbot_packages = ' '.join(BUILDBOT_PACKAGES) + python_packages = ' '.join(PYTHON_PACKAGES) + + cmd = CommandRunner(locals()) + + for arg in args: + if arg == 'upgrade': + cmd.run('%(sudo)s apt-get %(yesflag)s upgrade') + elif arg == 'checkout': + cmd.run('%(sudo)s apt-get %(yesflag)s install %(initial_packages)s') + cmd.run('svn co --username guest --password "" %(scons_url)s') + elif arg == 'building': + cmd.run('%(sudo)s apt-get %(yesflag)s install %(building_packages)s') + elif arg == 'testing': + cmd.run('%(sudo)s apt-get %(yesflag)s install %(testing_packages)s') + elif arg == 'buildbot': + cmd.run('%(sudo)s apt-get %(yesflag)s install %(buildbot_packages)s') + elif arg == 'python-versions': + if install_packages: + cmd.run('%(sudo)s apt-get %(yesflag)s install %(install_packages)s') + install_packages = None + cmd.run('%(sudo)s apt-get %(yesflag)s install %(python_packages)s') + try: + import install_python + except ImportError: + msg = 'Could not import install_python; skipping python-versions.\n' + sys.stderr.write(msg) + else: + install_python.main(['install_python.py', '-a']) + elif arg == 'scons-versions': + if install_packages: + cmd.run('%(sudo)s apt-get %(yesflag)s install %(install_packages)s') + install_packages = None + try: + import install_scons + except ImportError: + msg = 'Could not import install_scons; skipping scons-versions.\n' + sys.stderr.write(msg) + else: + install_scons.main(['install_scons.py', '-a']) + else: + msg = '%s: unknown argument %s\n' + sys.stderr.write(msg % (argv[0], repr(arg))) + sys.exit(1) + +if __name__ == "__main__": + sys.exit(main()) + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/bin/sconsexamples.py b/bin/sconsexamples.py new file mode 100644 index 0000000..0a409bc --- /dev/null +++ b/bin/sconsexamples.py @@ -0,0 +1,512 @@ +#!/usr/bin/env python2 +# +# scons_examples.py - an SGML preprocessor for capturing SCons output +# and inserting into examples in our DocBook +# documentation +# + +# This script looks for some SGML tags that describe SCons example +# configurations and commands to execute in those configurations, and +# uses TestCmd.py to execute the commands and insert the output into +# the output SGML. This way, we can run a script and update all of +# our example output without having to do a lot of laborious by-hand +# checking. +# +# An "SCons example" looks like this, and essentially describes a set of +# input files (program source files as well as SConscript files): +# +# +# +# env = Environment() +# env.Program('foo') +# +# +# int main() { printf("foo.c\n"); } +# +# +# +# The contents within the tag will get written +# into a temporary directory whenever example output needs to be +# generated. By default, the contents are not inserted into text +# directly, unless you set the "printme" attribute on one or more files, +# in which case they will get inserted within a tag. +# This makes it easy to define the example at the appropriate +# point in the text where you intend to show the SConstruct file. +# +# Note that you should usually give the a "name" +# attribute so that you can refer to the example configuration later to +# run SCons and generate output. +# +# If you just want to show a file's contents without worry about running +# SCons, there's a shorter tag: +# +# +# env = Environment() +# env.Program('foo') +# +# +# This is essentially equivalent to , +# but it's more straightforward. +# +# SCons output is generated from the following sort of tag: +# +# +# scons -Q foo +# scons -Q foo +# +# +# You tell it which example to use with the "example" attribute, and +# then give it a list of tags to execute. You can also supply +# an "os" tag, which specifies the type of operating system this example +# is intended to show; if you omit this, default value is "posix". +# +# The generated SGML will show the command line (with the appropriate +# command-line prompt for the operating system), execute the command in +# a temporary directory with the example files, capture the standard +# output from SCons, and insert it into the text as appropriate. +# Error output gets passed through to your error output so you +# can see if there are any problems executing the command. +# + +import os +import os.path +import re +import sgmllib +import string +import sys + +sys.path.append(os.path.join(os.getcwd(), 'etc')) +sys.path.append(os.path.join(os.getcwd(), 'build', 'etc')) + +scons_py = os.path.join('bootstrap', 'src', 'script', 'scons.py') +if not os.path.exists(scons_py): + scons_py = os.path.join('src', 'script', 'scons.py') + +scons_lib_dir = os.path.join(os.getcwd(), 'bootstrap', 'src', 'engine') +if not os.path.exists(scons_lib_dir): + scons_lib_dir = os.path.join(os.getcwd(), 'src', 'engine') + +import TestCmd + +# The regular expression that identifies entity references in the +# standard sgmllib omits the underscore from the legal characters. +# Override it with our own regular expression that adds underscore. +sgmllib.entityref = re.compile('&([a-zA-Z][-_.a-zA-Z0-9]*)[^-_a-zA-Z0-9]') + +class DataCollector: + """Generic class for collecting data between a start tag and end + tag. We subclass for various types of tags we care about.""" + def __init__(self): + self.data = "" + def afunc(self, data): + self.data = self.data + data + +class Example(DataCollector): + """An SCons example. This is essentially a list of files that + will get written to a temporary directory to collect output + from one or more SCons runs.""" + def __init__(self): + DataCollector.__init__(self) + self.files = [] + self.dirs = [] + +class File(DataCollector): + """A file, that will get written out to a temporary directory + for one or more SCons runs.""" + def __init__(self, name): + DataCollector.__init__(self) + self.name = name + +class Directory(DataCollector): + """A directory, that will get created in a temporary directory + for one or more SCons runs.""" + def __init__(self, name): + DataCollector.__init__(self) + self.name = name + +class Output(DataCollector): + """Where the command output goes. This is essentially + a list of commands that will get executed.""" + def __init__(self): + DataCollector.__init__(self) + self.commandlist = [] + +class Command(DataCollector): + """A tag for where the command output goes. This is essentially + a list of commands that will get executed.""" + pass + +Prompt = { + 'posix' : '% ', + 'win32' : 'C:\\>' +} + +# Magick SCons hackery. +# +# So that our examples can still use the default SConstruct file, we +# actually feed the following into SCons via stdin and then have it +# SConscript() the SConstruct file. This stdin wrapper creates a set +# of ToolSurrogates for the tools for the appropriate platform. These +# Surrogates print output like the real tools and behave like them +# without actually having to be on the right platform or have the right +# tool installed. +# +# The upshot: We transparently change the world out from under the +# top-level SConstruct file in an example just so we can get the +# command output. + +Stdin = """\ +import string +import SCons.Defaults + +platform = '%s' + +class Curry: + def __init__(self, fun, *args, **kwargs): + self.fun = fun + self.pending = args[:] + self.kwargs = kwargs.copy() + + def __call__(self, *args, **kwargs): + if kwargs and self.kwargs: + kw = self.kwargs.copy() + kw.update(kwargs) + else: + kw = kwargs or self.kwargs + + return apply(self.fun, self.pending + args, kw) + +def Str(target, source, env, cmd=""): + result = [] + for cmd in env.subst_list(cmd, target=target, source=source): + result.append(string.join(map(str, cmd))) + return string.join(result, '\\n') + +class ToolSurrogate: + def __init__(self, tool, variable, func): + self.tool = tool + self.variable = variable + self.func = func + def __call__(self, env): + t = Tool(self.tool) + t.generate(env) + orig = env[self.variable] + env[self.variable] = Action(self.func, strfunction=Curry(Str, cmd=orig)) + +def Null(target, source, env): + pass + +def Cat(target, source, env): + target = str(target[0]) + f = open(target, "wb") + for src in map(str, source): + f.write(open(src, "rb").read()) + f.close() + +ToolList = { + 'posix' : [('cc', 'CCCOM', Cat), + ('link', 'LINKCOM', Cat), + ('tar', 'TARCOM', Null), + ('zip', 'ZIPCOM', Null)], + 'win32' : [('msvc', 'CCCOM', Cat), + ('mslink', 'LINKCOM', Cat)] +} + +tools = map(lambda t: apply(ToolSurrogate, t), ToolList[platform]) + +SCons.Defaults.ConstructionEnvironment.update({ + 'PLATFORM' : platform, + 'TOOLS' : tools, +}) + +SConscript('SConstruct') +""" + +class MySGML(sgmllib.SGMLParser): + """A subclass of the standard Python 2.2 sgmllib SGML parser. + + Note that this doesn't work with the 1.5.2 sgmllib module, because + that didn't have the ability to work with ENTITY declarations. + """ + def __init__(self): + sgmllib.SGMLParser.__init__(self) + self.examples = {} + self.afunclist = [] + + def handle_data(self, data): + try: + f = self.afunclist[-1] + except IndexError: + sys.stdout.write(data) + else: + f(data) + + def handle_comment(self, data): + sys.stdout.write('') + + def handle_decl(self, data): + sys.stdout.write('') + + def unknown_starttag(self, tag, attrs): + try: + f = self.example.afunc + except AttributeError: + f = sys.stdout.write + if not attrs: + f('<' + tag + '>') + else: + f('<' + tag) + for name, value in attrs: + f(' ' + name + '=' + '"' + value + '"') + f('>') + + def unknown_endtag(self, tag): + sys.stdout.write('') + + def unknown_entityref(self, ref): + sys.stdout.write('&' + ref + ';') + + def unknown_charref(self, ref): + sys.stdout.write('&#' + ref + ';') + + def start_scons_example(self, attrs): + t = filter(lambda t: t[0] == 'name', attrs) + if t: + name = t[0][1] + try: + e = self.examples[name] + except KeyError: + e = self.examples[name] = Example() + else: + e = Example() + for name, value in attrs: + setattr(e, name, value) + self.e = e + self.afunclist.append(e.afunc) + + def end_scons_example(self): + e = self.e + files = filter(lambda f: f.printme, e.files) + if files: + sys.stdout.write('') + for f in files: + if f.printme: + i = len(f.data) - 1 + while f.data[i] == ' ': + i = i - 1 + output = string.replace(f.data[:i+1], '__ROOT__', '') + sys.stdout.write(output) + if e.data and e.data[0] == '\n': + e.data = e.data[1:] + sys.stdout.write(e.data + '') + delattr(self, 'e') + self.afunclist = self.afunclist[:-1] + + def start_file(self, attrs): + try: + e = self.e + except AttributeError: + self.error(" tag outside of ") + t = filter(lambda t: t[0] == 'name', attrs) + if not t: + self.error("no name attribute found") + try: + e.prefix + except AttributeError: + e.prefix = e.data + e.data = "" + f = File(t[0][1]) + f.printme = None + for name, value in attrs: + setattr(f, name, value) + e.files.append(f) + self.afunclist.append(f.afunc) + + def end_file(self): + self.e.data = "" + self.afunclist = self.afunclist[:-1] + + def start_directory(self, attrs): + try: + e = self.e + except AttributeError: + self.error(" tag outside of ") + t = filter(lambda t: t[0] == 'name', attrs) + if not t: + self.error("no name attribute found") + try: + e.prefix + except AttributeError: + e.prefix = e.data + e.data = "" + d = Directory(t[0][1]) + for name, value in attrs: + setattr(d, name, value) + e.dirs.append(d) + self.afunclist.append(d.afunc) + + def end_directory(self): + self.e.data = "" + self.afunclist = self.afunclist[:-1] + + def start_scons_example_file(self, attrs): + t = filter(lambda t: t[0] == 'example', attrs) + if not t: + self.error("no example attribute found") + exname = t[0][1] + try: + e = self.examples[exname] + except KeyError: + self.error("unknown example name '%s'" % exname) + fattrs = filter(lambda t: t[0] == 'name', attrs) + if not fattrs: + self.error("no name attribute found") + fname = fattrs[0][1] + f = filter(lambda f, fname=fname: f.name == fname, e.files) + if not f: + self.error("example '%s' does not have a file named '%s'" % (exname, fname)) + self.f = f[0] + + def end_scons_example_file(self): + f = self.f + sys.stdout.write('') + i = len(f.data) - 1 + while f.data[i] == ' ': + i = i - 1 + sys.stdout.write(f.data[:i+1] + '') + delattr(self, 'f') + + def start_scons_output(self, attrs): + t = filter(lambda t: t[0] == 'example', attrs) + if not t: + self.error("no example attribute found") + exname = t[0][1] + try: + e = self.examples[exname] + except KeyError: + self.error("unknown example name '%s'" % exname) + # Default values for an example. + o = Output() + o.os = 'posix' + o.e = e + # Locally-set. + for name, value in attrs: + setattr(o, name, value) + self.o = o + self.afunclist.append(o.afunc) + + def end_scons_output(self): + o = self.o + e = o.e + t = TestCmd.TestCmd(workdir='', combine=1) + t.subdir('ROOT', 'WORK') + for d in e.dirs: + dir = t.workpath('WORK', d.name) + if not os.path.exists(dir): + os.makedirs(dir) + for f in e.files: + i = 0 + while f.data[i] == '\n': + i = i + 1 + lines = string.split(f.data[i:], '\n') + i = 0 + while lines[0][i] == ' ': + i = i + 1 + lines = map(lambda l, i=i: l[i:], lines) + path = string.replace(f.name, '__ROOT__', t.workpath('ROOT')) + dir, name = os.path.split(f.name) + if dir: + dir = t.workpath('WORK', dir) + if not os.path.exists(dir): + os.makedirs(dir) + content = string.join(lines, '\n') + content = string.replace(content, + '__ROOT__', + t.workpath('ROOT')) + t.write(t.workpath('WORK', f.name), content) + i = len(o.prefix) + while o.prefix[i-1] != '\n': + i = i - 1 + sys.stdout.write('' + o.prefix[:i]) + p = o.prefix[i:] + for c in o.commandlist: + sys.stdout.write(p + Prompt[o.os]) + d = string.replace(c.data, '__ROOT__', '') + sys.stdout.write('' + d + '\n') + e = string.replace(c.data, '__ROOT__', t.workpath('ROOT')) + args = string.split(e)[1:] + os.environ['SCONS_LIB_DIR'] = scons_lib_dir + t.run(interpreter = sys.executable, + program = scons_py, + arguments = '-f - ' + string.join(args), + chdir = t.workpath('WORK'), + stdin = Stdin % o.os) + out = string.replace(t.stdout(), t.workpath('ROOT'), '') + if out: + lines = string.split(out, '\n') + if lines: + while lines[-1] == '': + lines = lines[:-1] + for l in lines: + sys.stdout.write(p + l + '\n') + #err = t.stderr() + #if err: + # sys.stderr.write(err) + if o.data[0] == '\n': + o.data = o.data[1:] + sys.stdout.write(o.data + '') + delattr(self, 'o') + self.afunclist = self.afunclist[:-1] + + def start_command(self, attrs): + try: + o = self.o + except AttributeError: + self.error(" tag outside of ") + try: + o.prefix + except AttributeError: + o.prefix = o.data + o.data = "" + c = Command() + o.commandlist.append(c) + self.afunclist.append(c.afunc) + + def end_command(self): + self.o.data = "" + self.afunclist = self.afunclist[:-1] + + def start_sconstruct(self, attrs): + sys.stdout.write('') + + def end_sconstruct(self): + sys.stdout.write('') + +try: + file = sys.argv[1] +except IndexError: + file = '-' + +if file == '-': + f = sys.stdin +else: + try: + f = open(file, 'r') + except IOError, msg: + print file, ":", msg + sys.exit(1) + +data = f.read() +if f is not sys.stdin: + f.close() + +x = MySGML() +for c in data: + x.feed(c) +x.close() + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/bin/sconsoutput.py b/bin/sconsoutput.py new file mode 100644 index 0000000..f99ec5a --- /dev/null +++ b/bin/sconsoutput.py @@ -0,0 +1,827 @@ +#!/usr/bin/env python2 + +# +# Copyright (c) 2003 Steven Knight +# +# Permission is hereby granted, free of charge, to any person obtaining +# a copy of this software and associated documentation files (the +# "Software"), to deal in the Software without restriction, including +# without limitation the rights to use, copy, modify, merge, publish, +# distribute, sublicense, and/or sell copies of the Software, and to +# permit persons to whom the Software is furnished to do so, subject to +# the following conditions: +# +# The above copyright notice and this permission notice shall be included +# in all copies or substantial portions of the Software. +# +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY +# KIND, EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE +# WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. +# + +__revision__ = "/home/scons/sconsoutput/branch.0/baseline/src/sconsoutput.py 0.4.D001 2004/11/27 18:44:37 knight" + +# +# sconsoutput.py - an SGML preprocessor for capturing SCons output +# and inserting it into examples in our DocBook +# documentation +# +# This script looks for some SGML tags that describe SCons example +# configurations and commands to execute in those configurations, and +# uses TestCmd.py to execute the commands and insert the output from +# those commands into the SGML that we output. This way, we can run a +# script and update all of our example documentation output without +# a lot of laborious by-hand checking. +# +# An "SCons example" looks like this, and essentially describes a set of +# input files (program source files as well as SConscript files): +# +# +# +# env = Environment() +# env.Program('foo') +# +# +# int main() { printf("foo.c\n"); } +# +# +# +# The contents within the tag will get written +# into a temporary directory whenever example output needs to be +# generated. By default, the contents are not inserted into text +# directly, unless you set the "printme" attribute on one or more files, +# in which case they will get inserted within a tag. +# This makes it easy to define the example at the appropriate +# point in the text where you intend to show the SConstruct file. +# +# Note that you should usually give the a "name" +# attribute so that you can refer to the example configuration later to +# run SCons and generate output. +# +# If you just want to show a file's contents without worry about running +# SCons, there's a shorter tag: +# +# +# env = Environment() +# env.Program('foo') +# +# +# This is essentially equivalent to , +# but it's more straightforward. +# +# SCons output is generated from the following sort of tag: +# +# +# scons -Q foo +# scons -Q foo +# +# +# You tell it which example to use with the "example" attribute, and then +# give it a list of tags to execute. You can also +# supply an "os" tag, which specifies the type of operating system this +# example is intended to show; if you omit this, default value is "posix". +# +# The generated SGML will show the command line (with the appropriate +# command-line prompt for the operating system), execute the command in +# a temporary directory with the example files, capture the standard +# output from SCons, and insert it into the text as appropriate. +# Error output gets passed through to your error output so you +# can see if there are any problems executing the command. +# + +import os +import os.path +import re +import sgmllib +import string +import sys +import time + +sys.path.append(os.path.join(os.getcwd(), 'QMTest')) +sys.path.append(os.path.join(os.getcwd(), 'build', 'QMTest')) + +scons_py = os.path.join('bootstrap', 'src', 'script', 'scons.py') +if not os.path.exists(scons_py): + scons_py = os.path.join('src', 'script', 'scons.py') + +scons_lib_dir = os.path.join(os.getcwd(), 'bootstrap', 'src', 'engine') +if not os.path.exists(scons_lib_dir): + scons_lib_dir = os.path.join(os.getcwd(), 'src', 'engine') + +os.environ['SCONS_LIB_DIR'] = scons_lib_dir + +import TestCmd + +# The regular expression that identifies entity references in the +# standard sgmllib omits the underscore from the legal characters. +# Override it with our own regular expression that adds underscore. +sgmllib.entityref = re.compile('&([a-zA-Z][-_.a-zA-Z0-9]*)[^-_a-zA-Z0-9]') + +# Classes for collecting different types of data we're interested in. +class DataCollector: + """Generic class for collecting data between a start tag and end + tag. We subclass for various types of tags we care about.""" + def __init__(self): + self.data = "" + def afunc(self, data): + self.data = self.data + data + +class Example(DataCollector): + """An SCons example. This is essentially a list of files that + will get written to a temporary directory to collect output + from one or more SCons runs.""" + def __init__(self): + DataCollector.__init__(self) + self.files = [] + self.dirs = [] + +class File(DataCollector): + """A file, that will get written out to a temporary directory + for one or more SCons runs.""" + def __init__(self, name): + DataCollector.__init__(self) + self.name = name + +class Directory(DataCollector): + """A directory, that will get created in a temporary directory + for one or more SCons runs.""" + def __init__(self, name): + DataCollector.__init__(self) + self.name = name + +class Output(DataCollector): + """Where the command output goes. This is essentially + a list of commands that will get executed.""" + def __init__(self): + DataCollector.__init__(self) + self.commandlist = [] + +class Command(DataCollector): + """A tag for where the command output goes. This is essentially + a list of commands that will get executed.""" + def __init__(self): + DataCollector.__init__(self) + self.output = None + +Prompt = { + 'posix' : '% ', + 'win32' : 'C:\\>' +} + +# The magick SCons hackery that makes this work. +# +# So that our examples can still use the default SConstruct file, we +# actually feed the following into SCons via stdin and then have it +# SConscript() the SConstruct file. This stdin wrapper creates a set +# of ToolSurrogates for the tools for the appropriate platform. These +# Surrogates print output like the real tools and behave like them +# without actually having to be on the right platform or have the right +# tool installed. +# +# The upshot: The wrapper transparently changes the world out from +# under the top-level SConstruct file in an example just so we can get +# the command output. + +Stdin = """\ +import os +import re +import string +import SCons.Action +import SCons.Defaults +import SCons.Node.FS + +platform = '%(osname)s' + +Sep = { + 'posix' : '/', + 'win32' : '\\\\', +}[platform] + + +# Slip our own __str__() method into the EntryProxy class used to expand +# $TARGET{S} and $SOURCE{S} to translate the path-name separators from +# what's appropriate for the system we're running on to what's appropriate +# for the example system. +orig = SCons.Node.FS.EntryProxy +class MyEntryProxy(orig): + def __str__(self): + return string.replace(str(self._Proxy__subject), os.sep, Sep) +SCons.Node.FS.EntryProxy = MyEntryProxy + +# Slip our own RDirs() method into the Node.FS.File class so that the +# expansions of $_{CPPINC,F77INC,LIBDIR}FLAGS will have the path-name +# separators translated from what's appropriate for the system we're +# running on to what's appropriate for the example system. +orig_RDirs = SCons.Node.FS.File.RDirs +def my_RDirs(self, pathlist, orig_RDirs=orig_RDirs): + return map(lambda x: string.replace(str(x), os.sep, Sep), + orig_RDirs(self, pathlist)) +SCons.Node.FS.File.RDirs = my_RDirs + +class Curry: + def __init__(self, fun, *args, **kwargs): + self.fun = fun + self.pending = args[:] + self.kwargs = kwargs.copy() + + def __call__(self, *args, **kwargs): + if kwargs and self.kwargs: + kw = self.kwargs.copy() + kw.update(kwargs) + else: + kw = kwargs or self.kwargs + + return apply(self.fun, self.pending + args, kw) + +def Str(target, source, env, cmd=""): + result = [] + for cmd in env.subst_list(cmd, target=target, source=source): + result.append(string.join(map(str, cmd))) + return string.join(result, '\\n') + +class ToolSurrogate: + def __init__(self, tool, variable, func, varlist): + self.tool = tool + if not type(variable) is type([]): + variable = [variable] + self.variable = variable + self.func = func + self.varlist = varlist + def __call__(self, env): + t = Tool(self.tool) + t.generate(env) + for v in self.variable: + orig = env[v] + try: + strfunction = orig.strfunction + except AttributeError: + strfunction = Curry(Str, cmd=orig) + # Don't call Action() through its global function name, because + # that leads to infinite recursion in trying to initialize the + # Default Environment. + env[v] = SCons.Action.Action(self.func, + strfunction=strfunction, + varlist=self.varlist) + def __repr__(self): + # This is for the benefit of printing the 'TOOLS' + # variable through env.Dump(). + return repr(self.tool) + +def Null(target, source, env): + pass + +def Cat(target, source, env): + target = str(target[0]) + f = open(target, "wb") + for src in map(str, source): + f.write(open(src, "rb").read()) + f.close() + +def CCCom(target, source, env): + target = str(target[0]) + fp = open(target, "wb") + def process(source_file, fp=fp): + for line in open(source_file, "rb").readlines(): + m = re.match(r'#include\s[<"]([^<"]+)[>"]', line) + if m: + include = m.group(1) + for d in [str(env.Dir('$CPPPATH')), '.']: + f = os.path.join(d, include) + if os.path.exists(f): + process(f) + break + elif line[:11] != "STRIP CCCOM": + fp.write(line) + for src in map(str, source): + process(src) + fp.write('debug = ' + ARGUMENTS.get('debug', '0') + '\\n') + fp.close() + +public_class_re = re.compile('^public class (\S+)', re.MULTILINE) + +def JavaCCom(target, source, env): + # This is a fake Java compiler that just looks for + # public class FooBar + # lines in the source file(s) and spits those out + # to .class files named after the class. + tlist = map(str, target) + not_copied = {} + for t in tlist: + not_copied[t] = 1 + for src in map(str, source): + contents = open(src, "rb").read() + classes = public_class_re.findall(contents) + for c in classes: + for t in filter(lambda x: string.find(x, c) != -1, tlist): + open(t, "wb").write(contents) + del not_copied[t] + for t in not_copied.keys(): + open(t, "wb").write("\\n") + +def JavaHCom(target, source, env): + tlist = map(str, target) + slist = map(str, source) + for t, s in zip(tlist, slist): + open(t, "wb").write(open(s, "rb").read()) + +def find_class_files(arg, dirname, names): + class_files = filter(lambda n: n[-6:] == '.class', names) + paths = map(lambda n, d=dirname: os.path.join(d, n), class_files) + arg.extend(paths) + +def JarCom(target, source, env): + target = str(target[0]) + class_files = [] + for src in map(str, source): + os.path.walk(src, find_class_files, class_files) + f = open(target, "wb") + for cf in class_files: + f.write(open(cf, "rb").read()) + f.close() + +# XXX Adding COLOR, COLORS and PACKAGE to the 'cc' varlist(s) by hand +# here is bogus. It's for the benefit of doc/user/command-line.in, which +# uses examples that want to rebuild based on changes to these variables. +# It would be better to figure out a way to do it based on the content of +# the generated command-line, or else find a way to let the example markup +# language in doc/user/command-line.in tell this script what variables to +# add, but that's more difficult than I want to figure out how to do right +# now, so let's just use the simple brute force approach for the moment. + +ToolList = { + 'posix' : [('cc', ['CCCOM', 'SHCCCOM'], CCCom, ['CCFLAGS', 'CPPDEFINES', 'COLOR', 'COLORS', 'PACKAGE']), + ('link', ['LINKCOM', 'SHLINKCOM'], Cat, []), + ('ar', ['ARCOM', 'RANLIBCOM'], Cat, []), + ('tar', 'TARCOM', Null, []), + ('zip', 'ZIPCOM', Null, []), + ('BitKeeper', 'BITKEEPERCOM', Cat, []), + ('CVS', 'CVSCOM', Cat, []), + ('RCS', 'RCS_COCOM', Cat, []), + ('SCCS', 'SCCSCOM', Cat, []), + ('javac', 'JAVACCOM', JavaCCom, []), + ('javah', 'JAVAHCOM', JavaHCom, []), + ('jar', 'JARCOM', JarCom, []), + ('rmic', 'RMICCOM', Cat, []), + ], + 'win32' : [('msvc', ['CCCOM', 'SHCCCOM', 'RCCOM'], CCCom, ['CCFLAGS', 'CPPDEFINES', 'COLOR', 'COLORS', 'PACKAGE']), + ('mslink', ['LINKCOM', 'SHLINKCOM'], Cat, []), + ('mslib', 'ARCOM', Cat, []), + ('tar', 'TARCOM', Null, []), + ('zip', 'ZIPCOM', Null, []), + ('BitKeeper', 'BITKEEPERCOM', Cat, []), + ('CVS', 'CVSCOM', Cat, []), + ('RCS', 'RCS_COCOM', Cat, []), + ('SCCS', 'SCCSCOM', Cat, []), + ('javac', 'JAVACCOM', JavaCCom, []), + ('javah', 'JAVAHCOM', JavaHCom, []), + ('jar', 'JARCOM', JarCom, []), + ('rmic', 'RMICCOM', Cat, []), + ], +} + +toollist = ToolList[platform] +filter_tools = string.split('%(tools)s') +if filter_tools: + toollist = filter(lambda x, ft=filter_tools: x[0] in ft, toollist) + +toollist = map(lambda t: apply(ToolSurrogate, t), toollist) + +toollist.append('install') + +def surrogate_spawn(sh, escape, cmd, args, env): + pass + +def surrogate_pspawn(sh, escape, cmd, args, env, stdout, stderr): + pass + +SCons.Defaults.ConstructionEnvironment.update({ + 'PLATFORM' : platform, + 'TOOLS' : toollist, + 'SPAWN' : surrogate_spawn, + 'PSPAWN' : surrogate_pspawn, +}) + +SConscript('SConstruct') +""" + +# "Commands" that we will execute in our examples. +def command_scons(args, c, test, dict): + save_vals = {} + delete_keys = [] + try: + ce = c.environment + except AttributeError: + pass + else: + for arg in string.split(c.environment): + key, val = string.split(arg, '=') + try: + save_vals[key] = os.environ[key] + except KeyError: + delete_keys.append(key) + os.environ[key] = val + test.run(interpreter = sys.executable, + program = scons_py, + arguments = '-f - ' + string.join(args), + chdir = test.workpath('WORK'), + stdin = Stdin % dict) + os.environ.update(save_vals) + for key in delete_keys: + del(os.environ[key]) + out = test.stdout() + out = string.replace(out, test.workpath('ROOT'), '') + out = string.replace(out, test.workpath('WORK/SConstruct'), + '/home/my/project/SConstruct') + lines = string.split(out, '\n') + if lines: + while lines[-1] == '': + lines = lines[:-1] + #err = test.stderr() + #if err: + # sys.stderr.write(err) + return lines + +def command_touch(args, c, test, dict): + if args[0] == '-t': + t = int(time.mktime(time.strptime(args[1], '%Y%m%d%H%M'))) + times = (t, t) + args = args[2:] + else: + time.sleep(1) + times = None + for file in args: + if not os.path.isabs(file): + file = os.path.join(test.workpath('WORK'), file) + if not os.path.exists(file): + open(file, 'wb') + os.utime(file, times) + return [] + +def command_edit(args, c, test, dict): + try: + add_string = c.edit[:] + except AttributeError: + add_string = 'void edit(void) { ; }\n' + if add_string[-1] != '\n': + add_string = add_string + '\n' + for file in args: + if not os.path.isabs(file): + file = os.path.join(test.workpath('WORK'), file) + contents = open(file, 'rb').read() + open(file, 'wb').write(contents + add_string) + return [] + +def command_ls(args, c, test, dict): + def ls(a): + files = os.listdir(a) + files = filter(lambda x: x[0] != '.', files) + files.sort() + return [string.join(files, ' ')] + if args: + l = [] + for a in args: + l.extend(ls(test.workpath('WORK', a))) + return l + else: + return ls(test.workpath('WORK')) + +CommandDict = { + 'scons' : command_scons, + 'touch' : command_touch, + 'edit' : command_edit, + 'ls' : command_ls, +} + +def ExecuteCommand(args, c, t, dict): + try: + func = CommandDict[args[0]] + except KeyError: + func = lambda args, c, t, dict: [] + return func(args[1:], c, t, dict) + +class MySGML(sgmllib.SGMLParser): + """A subclass of the standard Python 2.2 sgmllib SGML parser. + + This extends the standard sgmllib parser to recognize, and do cool + stuff with, the added tags that describe our SCons examples, + commands, and other stuff. + + Note that this doesn't work with the 1.5.2 sgmllib module, because + that didn't have the ability to work with ENTITY declarations. + """ + def __init__(self): + sgmllib.SGMLParser.__init__(self) + self.examples = {} + self.afunclist = [] + + # The first set of methods here essentially implement pass-through + # handling of most of the stuff in an SGML file. We're really + # only concerned with the tags specific to SCons example processing, + # the methods for which get defined below. + + def handle_data(self, data): + try: + f = self.afunclist[-1] + except IndexError: + sys.stdout.write(data) + else: + f(data) + + def handle_comment(self, data): + sys.stdout.write('') + + def handle_decl(self, data): + sys.stdout.write('') + + def unknown_starttag(self, tag, attrs): + try: + f = self.example.afunc + except AttributeError: + f = sys.stdout.write + if not attrs: + f('<' + tag + '>') + else: + f('<' + tag) + for name, value in attrs: + f(' ' + name + '=' + '"' + value + '"') + f('>') + + def unknown_endtag(self, tag): + sys.stdout.write('') + + def unknown_entityref(self, ref): + sys.stdout.write('&' + ref + ';') + + def unknown_charref(self, ref): + sys.stdout.write('&#' + ref + ';') + + # Here is where the heavy lifting begins. The following methods + # handle the begin-end tags of our SCons examples. + + def start_scons_example(self, attrs): + t = filter(lambda t: t[0] == 'name', attrs) + if t: + name = t[0][1] + try: + e = self.examples[name] + except KeyError: + e = self.examples[name] = Example() + else: + e = Example() + for name, value in attrs: + setattr(e, name, value) + self.e = e + self.afunclist.append(e.afunc) + + def end_scons_example(self): + e = self.e + files = filter(lambda f: f.printme, e.files) + if files: + sys.stdout.write('') + for f in files: + if f.printme: + i = len(f.data) - 1 + while f.data[i] == ' ': + i = i - 1 + output = string.replace(f.data[:i+1], '__ROOT__', '') + output = string.replace(output, '<', '<') + output = string.replace(output, '>', '>') + sys.stdout.write(output) + if e.data and e.data[0] == '\n': + e.data = e.data[1:] + sys.stdout.write(e.data + '') + delattr(self, 'e') + self.afunclist = self.afunclist[:-1] + + def start_file(self, attrs): + try: + e = self.e + except AttributeError: + self.error(" tag outside of ") + t = filter(lambda t: t[0] == 'name', attrs) + if not t: + self.error("no name attribute found") + try: + e.prefix + except AttributeError: + e.prefix = e.data + e.data = "" + f = File(t[0][1]) + f.printme = None + for name, value in attrs: + setattr(f, name, value) + e.files.append(f) + self.afunclist.append(f.afunc) + + def end_file(self): + self.e.data = "" + self.afunclist = self.afunclist[:-1] + + def start_directory(self, attrs): + try: + e = self.e + except AttributeError: + self.error(" tag outside of ") + t = filter(lambda t: t[0] == 'name', attrs) + if not t: + self.error("no name attribute found") + try: + e.prefix + except AttributeError: + e.prefix = e.data + e.data = "" + d = Directory(t[0][1]) + for name, value in attrs: + setattr(d, name, value) + e.dirs.append(d) + self.afunclist.append(d.afunc) + + def end_directory(self): + self.e.data = "" + self.afunclist = self.afunclist[:-1] + + def start_scons_example_file(self, attrs): + t = filter(lambda t: t[0] == 'example', attrs) + if not t: + self.error("no example attribute found") + exname = t[0][1] + try: + e = self.examples[exname] + except KeyError: + self.error("unknown example name '%s'" % exname) + fattrs = filter(lambda t: t[0] == 'name', attrs) + if not fattrs: + self.error("no name attribute found") + fname = fattrs[0][1] + f = filter(lambda f, fname=fname: f.name == fname, e.files) + if not f: + self.error("example '%s' does not have a file named '%s'" % (exname, fname)) + self.f = f[0] + + def end_scons_example_file(self): + f = self.f + sys.stdout.write('') + sys.stdout.write(f.data + '') + delattr(self, 'f') + + def start_scons_output(self, attrs): + t = filter(lambda t: t[0] == 'example', attrs) + if not t: + self.error("no example attribute found") + exname = t[0][1] + try: + e = self.examples[exname] + except KeyError: + self.error("unknown example name '%s'" % exname) + # Default values for an example. + o = Output() + o.preserve = None + o.os = 'posix' + o.tools = '' + o.e = e + # Locally-set. + for name, value in attrs: + setattr(o, name, value) + self.o = o + self.afunclist.append(o.afunc) + + def end_scons_output(self): + # The real raison d'etre for this script, this is where we + # actually execute SCons to fetch the output. + o = self.o + e = o.e + t = TestCmd.TestCmd(workdir='', combine=1) + if o.preserve: + t.preserve() + t.subdir('ROOT', 'WORK') + t.rootpath = string.replace(t.workpath('ROOT'), '\\', '\\\\') + + for d in e.dirs: + dir = t.workpath('WORK', d.name) + if not os.path.exists(dir): + os.makedirs(dir) + + for f in e.files: + i = 0 + while f.data[i] == '\n': + i = i + 1 + lines = string.split(f.data[i:], '\n') + i = 0 + while lines[0][i] == ' ': + i = i + 1 + lines = map(lambda l, i=i: l[i:], lines) + path = string.replace(f.name, '__ROOT__', t.rootpath) + if not os.path.isabs(path): + path = t.workpath('WORK', path) + dir, name = os.path.split(path) + if dir and not os.path.exists(dir): + os.makedirs(dir) + content = string.join(lines, '\n') + content = string.replace(content, '__ROOT__', t.rootpath) + path = t.workpath('WORK', path) + t.write(path, content) + if hasattr(f, 'chmod'): + os.chmod(path, int(f.chmod, 0)) + + i = len(o.prefix) + while o.prefix[i-1] != '\n': + i = i - 1 + + sys.stdout.write('' + o.prefix[:i]) + p = o.prefix[i:] + + for c in o.commandlist: + sys.stdout.write(p + Prompt[o.os]) + d = string.replace(c.data, '__ROOT__', '') + sys.stdout.write('' + d + '\n') + + e = string.replace(c.data, '__ROOT__', t.workpath('ROOT')) + args = string.split(e) + lines = ExecuteCommand(args, c, t, {'osname':o.os, 'tools':o.tools}) + content = None + if c.output: + content = c.output + elif lines: + content = string.join(lines, '\n' + p) + if content: + content = re.sub(' at 0x[0-9a-fA-F]*\>', ' at 0x700000>', content) + content = string.replace(content, '<', '<') + content = string.replace(content, '>', '>') + sys.stdout.write(p + content + '\n') + + if o.data[0] == '\n': + o.data = o.data[1:] + sys.stdout.write(o.data + '') + delattr(self, 'o') + self.afunclist = self.afunclist[:-1] + + def start_scons_output_command(self, attrs): + try: + o = self.o + except AttributeError: + self.error(" tag outside of ") + try: + o.prefix + except AttributeError: + o.prefix = o.data + o.data = "" + c = Command() + for name, value in attrs: + setattr(c, name, value) + o.commandlist.append(c) + self.afunclist.append(c.afunc) + + def end_scons_output_command(self): + self.o.data = "" + self.afunclist = self.afunclist[:-1] + + def start_sconstruct(self, attrs): + f = File('') + self.f = f + self.afunclist.append(f.afunc) + + def end_sconstruct(self): + f = self.f + sys.stdout.write('') + output = string.replace(f.data, '__ROOT__', '') + sys.stdout.write(output + '') + delattr(self, 'f') + self.afunclist = self.afunclist[:-1] + +# The main portion of the program itself, short and simple. +try: + file = sys.argv[1] +except IndexError: + file = '-' + +if file == '-': + f = sys.stdin +else: + try: + f = open(file, 'r') + except IOError, msg: + print file, ":", msg + sys.exit(1) + +data = f.read() +if f is not sys.stdin: + f.close() + +if data.startswith(' tag in the output, which contains a list of items + that have tags in them. Right now, these just overwrite + each other in the Arifact object we create. + + We also don't pay attention to any attributes of a tag other + than the "name" attribute. We'll need to extend this class if we + ever want to pay attention to those attributes. + """ + def __init__(self): + self.artifact = None + + def startElement(self, name, attrs): + self.text = "" + if name == 'artifact': + self.artifact = Artifact() + elif not self.artifact is None and name == 'field': + self.fname = attrs.get('name', None) + + def characters(self, ch): + if not self.artifact is None: + self.text = self.text + ch + + def endElement(self, name): + global Artifacts + if name == 'artifact': + type = self.artifact.artifact_type + try: + list = Artifacts[type] + except KeyError: + Artifacts[type] = list = [] + list.append(self.artifact) + self.artifact = None + elif not self.artifact is None and name == 'field': + setattr(self.artifact, self.fname, self.text) + +if __name__ == '__main__': + # Create a parser. + parser = xml.sax.make_parser() + # Tell the parser we are not interested in XML namespaces. + parser.setFeature(xml.sax.handler.feature_namespaces, 0) + + # Instantiate our handler and tell the parser to use it. + parser.setContentHandler(ClassifyArtifacts()) + + # Parse the input. + parser.parse(sys.argv[1]) + + # Hard-coded search for 'Open' bugs. This should be easily + # generalized once we figure out other things for this script to do. + bugs = filter(lambda x: x.status == 'Open', Artifacts['Bugs']) + + print Artifacts.keys() + + print "%d open bugs" % len(bugs) + + # Sort them into a separate list for each assignee. + Assigned = {} + for bug in bugs: + a = bug.assigned_to + try: + list = Assigned[a] + except KeyError: + Assigned[a] = list = [] + list.append(bug) + + for a in SFName.keys(): + try: + b = Assigned[SFName[a]] + except KeyError: + pass + else: + print " %s" % a + b.sort(lambda x, y: cmp(x.artifact_id, y.artifact_id)) + for bug in b: + print " %-6s %s" % (bug.artifact_id, bug.summary) diff --git a/bin/svn-bisect.py b/bin/svn-bisect.py new file mode 100755 index 0000000..e9ebcf8 --- /dev/null +++ b/bin/svn-bisect.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python +# -*- Python -*- + +import sys +from math import log, ceil +from optparse import OptionParser +import subprocess + +# crack the command line +parser = OptionParser( + usage="%prog lower-revision upper-revision test_script [arg1 ...]", + description="Binary search for a bug in a SVN checkout") +(options,script_args) = parser.parse_args() + +# make sure we have sufficient parameters +if len(script_args) < 1: + parser.error("Need a lower revision") +elif len(script_args) < 2: + parser.error("Need an upper revision") +elif len(script_args) < 3: + parser.error("Need a script to run") + +# extract our starting values +lower = int(script_args[0]) +upper = int(script_args[1]) +script = script_args[2:] + +# print an error message and quit +def error(s): + print >>sys.stderr, "******", s, "******" + sys.exit(1) + +# update to the specified version and run test +def testfail(revision): + "Return true if test fails" + print "Updating to revision", revision + if subprocess.call(["svn","up","-qr",str(revision)]) != 0: + m = "SVN did not update properly to revision %d" + raise RuntimeError(m % revision) + return subprocess.call(script,shell=False) != 0 + +# confirm that the endpoints are different +print "****** Checking upper bracket", upper +upperfails = testfail(upper) +print "****** Checking lower bracket", lower +lowerfails = testfail(lower) +if upperfails == lowerfails: + error("Upper and lower revisions must bracket the failure") + +# binary search for transition +msg = "****** max %d revisions to test (bug bracketed by [%d,%d])" +while upper-lower > 1: + print msg % (ceil(log(upper-lower,2)), lower, upper) + + mid = int((lower + upper)/2) + midfails = testfail(mid) + if midfails == lowerfails: + lower = mid + lowerfails = midfails + else: + upper = mid + upperfails = midfails + +# show which revision was first to fail +if upperfails != lowerfails: lower = upper +print "The error was caused by revision", lower + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: diff --git a/bin/time-scons.py b/bin/time-scons.py new file mode 100644 index 0000000..78d26e5 --- /dev/null +++ b/bin/time-scons.py @@ -0,0 +1,355 @@ +#!/usr/bin/env python +# +# time-scons.py: a wrapper script for running SCons timings +# +# This script exists to: +# +# 1) Wrap the invocation of runtest.py to run the actual TimeSCons +# timings consistently. It does this specifically by building +# SCons first, so .pyc compilation is not part of the timing. +# +# 2) Provide an interface for running TimeSCons timings against +# earlier revisions, before the whole TimeSCons infrastructure +# was "frozen" to provide consistent timings. This is done +# by updating the specific pieces containing the TimeSCons +# infrastructure to the earliest revision at which those pieces +# were "stable enough." +# +# By encapsulating all the logic in this script, our Buildbot +# infrastructure only needs to call this script, and we should be able +# to change what we need to in this script and have it affect the build +# automatically when the source code is updated, without having to +# restart either master or slave. + +import optparse +import os +import shutil +import subprocess +import sys +import tempfile +import xml.sax.handler + + +SubversionURL = 'http://scons.tigris.org/svn/scons' + + +# This is the baseline revision when the TimeSCons scripts first +# stabilized and collected "real," consistent timings. If we're timing +# a revision prior to this, we'll forcibly update the TimeSCons pieces +# of the tree to this revision to collect consistent timings for earlier +# revisions. +TimeSCons_revision = 4569 + +# The pieces of the TimeSCons infrastructure that are necessary to +# produce consistent timings, even when the rest of the tree is from +# an earlier revision that doesn't have these pieces. +TimeSCons_pieces = ['QMTest', 'timings', 'runtest.py'] + + +class CommandRunner: + """ + Executor class for commands, including "commands" implemented by + Python functions. + """ + verbose = True + active = True + + def __init__(self, dictionary={}): + self.subst_dictionary(dictionary) + + def subst_dictionary(self, dictionary): + self._subst_dictionary = dictionary + + def subst(self, string, dictionary=None): + """ + Substitutes (via the format operator) the values in the specified + dictionary into the specified command. + + The command can be an (action, string) tuple. In all cases, we + perform substitution on strings and don't worry if something isn't + a string. (It's probably a Python function to be executed.) + """ + if dictionary is None: + dictionary = self._subst_dictionary + if dictionary: + try: + string = string % dictionary + except TypeError: + pass + return string + + def display(self, command, stdout=None, stderr=None): + if not self.verbose: + return + if type(command) == type(()): + func = command[0] + args = command[1:] + s = '%s(%s)' % (func.__name__, ', '.join(map(repr, args))) + if type(command) == type([]): + # TODO: quote arguments containing spaces + # TODO: handle meta characters? + s = ' '.join(command) + else: + s = self.subst(command) + if not s.endswith('\n'): + s += '\n' + sys.stdout.write(s) + sys.stdout.flush() + + def execute(self, command, stdout=None, stderr=None): + """ + Executes a single command. + """ + if not self.active: + return 0 + if type(command) == type(''): + command = self.subst(command) + cmdargs = shlex.split(command) + if cmdargs[0] == 'cd': + command = (os.chdir,) + tuple(cmdargs[1:]) + if type(command) == type(()): + func = command[0] + args = command[1:] + return func(*args) + else: + if stdout is sys.stdout: + # Same as passing sys.stdout, except works with python2.4. + subout = None + elif stdout is None: + # Open pipe for anything else so Popen works on python2.4. + subout = subprocess.PIPE + else: + subout = stdout + if stderr is sys.stderr: + # Same as passing sys.stdout, except works with python2.4. + suberr = None + elif stderr is None: + # Merge with stdout if stderr isn't specified. + suberr = subprocess.STDOUT + else: + suberr = stderr + p = subprocess.Popen(command, + shell=(sys.platform == 'win32'), + stdout=subout, + stderr=suberr) + p.wait() + return p.returncode + + def run(self, command, display=None, stdout=None, stderr=None): + """ + Runs a single command, displaying it first. + """ + if display is None: + display = command + self.display(display) + return self.execute(command, stdout, stderr) + + def run_list(self, command_list, **kw): + """ + Runs a list of commands, stopping with the first error. + + Returns the exit status of the first failed command, or 0 on success. + """ + status = 0 + for command in command_list: + s = self.run(command, **kw) + if s and status == 0: + status = s + return 0 + + +def get_svn_revisions(branch, revisions=None): + """ + Fetch the actual SVN revisions for the given branch querying + "svn log." A string specifying a range of revisions can be + supplied to restrict the output to a subset of the entire log. + """ + command = ['svn', 'log', '--xml'] + if revisions: + command.extend(['-r', revisions]) + command.append(branch) + p = subprocess.Popen(command, stdout=subprocess.PIPE) + + class SVNLogHandler(xml.sax.handler.ContentHandler): + def __init__(self): + self.revisions = [] + def startElement(self, name, attributes): + if name == 'logentry': + self.revisions.append(int(attributes['revision'])) + + parser = xml.sax.make_parser() + handler = SVNLogHandler() + parser.setContentHandler(handler) + parser.parse(p.stdout) + return sorted(handler.revisions) + + +def prepare_commands(): + """ + Returns a list of the commands to be executed to prepare the tree + for testing. This involves building SCons, specifically the + build/scons subdirectory where our packaging build is staged, + and then running setup.py to create a local installed copy + with compiled *.pyc files. The build directory gets removed + first. + """ + commands = [] + if os.path.exists('build'): + commands.extend([ + ['mv', 'build', 'build.OLD'], + ['rm', '-rf', 'build.OLD'], + ]) + commands.append([sys.executable, 'bootstrap.py', 'build/scons']) + commands.append([sys.executable, + 'build/scons/setup.py', + 'install', + '--prefix=' + os.path.abspath('build/usr')]) + return commands + +def script_command(script): + """Returns the command to actually invoke the specified timing + script using our "built" scons.""" + return [sys.executable, 'runtest.py', '-x', 'build/usr/bin/scons', script] + +def do_revisions(cr, opts, branch, revisions, scripts): + """ + Time the SCons branch specified scripts through a list of revisions. + + We assume we're in a (temporary) directory in which we can check + out the source for the specified revisions. + """ + stdout = sys.stdout + stderr = sys.stderr + + status = 0 + + if opts.logsdir and not opts.no_exec and len(scripts) > 1: + for script in scripts: + subdir = os.path.basename(os.path.dirname(script)) + logsubdir = os.path.join(opts.origin, opts.logsdir, subdir) + if not os.path.exists(logsubdir): + os.makedirs(logsubdir) + + for this_revision in revisions: + + if opts.logsdir and not opts.no_exec: + log_name = '%s.log' % this_revision + log_file = os.path.join(opts.origin, opts.logsdir, log_name) + stdout = open(log_file, 'w') + stderr = None + + commands = [ + ['svn', 'co', '-q', '-r', str(this_revision), branch, '.'], + ] + + if int(this_revision) < int(TimeSCons_revision): + commands.append(['svn', 'up', '-q', '-r', str(TimeSCons_revision)] + + TimeSCons_pieces) + + commands.extend(prepare_commands()) + + s = cr.run_list(commands, stdout=stdout, stderr=stderr) + if s: + if status == 0: + status = s + continue + + for script in scripts: + if opts.logsdir and not opts.no_exec and len(scripts) > 1: + subdir = os.path.basename(os.path.dirname(script)) + lf = os.path.join(opts.origin, opts.logsdir, subdir, log_name) + out = open(lf, 'w') + err = None + else: + out = stdout + err = stderr + s = cr.run(script_command(script), stdout=out, stderr=err) + if s and status == 0: + status = s + if out not in (sys.stdout, None): + out.close() + out = None + + if int(this_revision) < int(TimeSCons_revision): + # "Revert" the pieces that we previously updated to the + # TimeSCons_revision, so the update to the next revision + # works cleanly. + command = (['svn', 'up', '-q', '-r', str(this_revision)] + + TimeSCons_pieces) + s = cr.run(command, stdout=stdout, stderr=stderr) + if s: + if status == 0: + status = s + continue + + if stdout not in (sys.stdout, None): + stdout.close() + stdout = None + + return status + +Usage = """\ +time-scons.py [-hnq] [-r REVISION ...] [--branch BRANCH] + [--logsdir DIR] [--svn] SCRIPT ...""" + +def main(argv=None): + if argv is None: + argv = sys.argv + + parser = optparse.OptionParser(usage=Usage) + parser.add_option("--branch", metavar="BRANCH", default="trunk", + help="time revision on BRANCH") + parser.add_option("--logsdir", metavar="DIR", default='.', + help="generate separate log files for each revision") + parser.add_option("-n", "--no-exec", action="store_true", + help="no execute, just print the command line") + parser.add_option("-q", "--quiet", action="store_true", + help="quiet, don't print the command line") + parser.add_option("-r", "--revision", metavar="REVISION", + help="time specified revisions") + parser.add_option("--svn", action="store_true", + help="fetch actual revisions for BRANCH") + opts, scripts = parser.parse_args(argv[1:]) + + if not scripts: + sys.stderr.write('No scripts specified.\n') + sys.exit(1) + + CommandRunner.verbose = not opts.quiet + CommandRunner.active = not opts.no_exec + cr = CommandRunner() + + os.environ['TESTSCONS_SCONSFLAGS'] = '' + + branch = SubversionURL + '/' + opts.branch + + if opts.svn: + revisions = get_svn_revisions(branch, opts.revision) + elif opts.revision: + # TODO(sgk): parse this for SVN-style revision strings + revisions = [opts.revision] + else: + revisions = None + + if opts.logsdir and not os.path.exists(opts.logsdir): + os.makedirs(opts.logsdir) + + if revisions: + opts.origin = os.getcwd() + tempdir = tempfile.mkdtemp(prefix='time-scons-') + try: + os.chdir(tempdir) + status = do_revisions(cr, opts, branch, revisions, scripts) + finally: + os.chdir(opts.origin) + shutil.rmtree(tempdir) + else: + commands = prepare_commands() + commands.extend([ script_command(script) for script in scripts ]) + status = cr.run_list(commands, stdout=sys.stdout, stderr=sys.stderr) + + return status + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/bin/timebuild b/bin/timebuild new file mode 100644 index 0000000..d5af983 --- /dev/null +++ b/bin/timebuild @@ -0,0 +1,65 @@ +#!/bin/sh +# +# Profile running SCons to build itself from the current package. +# +# This runs "aegis -build" to build a current scons-src-*.tar.gz +# package, unpacks it in the supplied directory name, and then +# starts a profiled run of an SCons build, followed by another. +# This results in two profiles: +# +# NAME/NAME-0.prof +# profile of a build-everything run +# +# NAME/NAME-1.prof +# profile of an all-up-to-date run +# +# This also copies the build scons-src-*.tar.gz file to the NAME +# subdirectory, and tars up everything under src/ as NAME/src.tar.gz, +# so that repeated runs with different in-progress changes can serve +# as their own crude version control, so you don't lose that exact +# combination of features which performed best. + +if test X$1 = X; then + echo "Must supply name!" >&2 + exit 1 +fi + +VERSION=0.90 + +DIR=$1 + +SRC="scons-src-$VERSION" +SRC_TAR_GZ="${SRC}.tar.gz" +B_D_SRC_TAR_GZ="build/dist/${SRC_TAR_GZ}" + +echo "Building ${B_D_SRC_TAR_GZ}: " `date` +aegis -build ${B_D_SRC_TAR_GZ} + +echo "mkdir ${DIR}: " `date` +mkdir ${DIR} + +echo "cp ${B_D_SRC_TAR_GZ} ${DIR}: " `date` +cp ${B_D_SRC_TAR_GZ} ${DIR} + +echo "tar cf ${DIR}/src.tar.gz: " `date` +tar cf ${DIR}/src.tar.gz src + +cd ${DIR} + +echo "tar zxf ${SRC_TAR_GZ}: " `date` +tar zxf ${SRC_TAR_GZ} + +cd ${SRC} + +SCRIPT="src/script/scons.py" +ARGS="version=$VERSION" + +export SCONS_LIB_DIR=`pwd`/src/engine + +echo "Build run starting: " `date` +python $SCRIPT --profile=../$DIR-0.prof $ARGS > ../$DIR-0.log 2>&1 + +echo "Up-to-date run starting: " `date` +python $SCRIPT --profile=../$DIR-1.prof $ARGS > ../$DIR-1.log 2>&1 + +echo "Finished $DIR at: " `date` diff --git a/bin/xml_export b/bin/xml_export new file mode 100644 index 0000000..bc9ccbd --- /dev/null +++ b/bin/xml_export @@ -0,0 +1,225 @@ +#!/usr/bin/perl -w +# +# xml_export - Retrieve data from the SF.net XML export for project data +# +# Copyright (C) 2002 Open Source Development Network, Inc. ("OSDN") +# +# Permission is hereby granted, free of charge, to any person obtaining a +# copy of this software and associated documentation files (the "Software"), +# to deal in the Software without restriction, including without limitation +# the rights to use, copy, modify, merge, publish, distribute, sublicense, +# and/or sell copies of the Software, and to permit persons to whom the +# Software is furnished to do so, subject to the license details found +# below in the section marked "$LICENSE_TEXT". +# +# SCons: modified the following RCS Id line so it won't expand during +# our checkins. +# +# $_Id: adocman,v 1.51 2002/06/07 18:56:35 moorman Exp _$ +# +# Written by Nate Oostendorp +# and Jacob Moorman +########################################################################### + +use strict; +use Alexandria::Client; +use HTTP::Request::Common; +my $client = new Alexandria::Client; + +util_verifyvariables("groupid"); + +my $res = $ua->simple_request(GET "$config{hosturl}/export/xml_export.php?group_id=$config{groupid}"); + +if (not $res->is_success()) { + die "Failed to connect: ".$res->as_string(); +} +print $res->content; + +########################################################################### + +__END__ +=head1 NAME + +xml_export - Retrieve data for a project via the SF.net XML export facility + +=head1 DESCRIPTION + +B provides a simple mechanism to download data from the +XML data export facility on SourceForge.net. This utility is needed +(in place of a downloader like wget or curl) since authentication by +a project administrator is required to access the XML export facility. + +=head1 SYNOPSIS + +xml_export [options] > output_file + + OPTIONS + --login Login to the SourceForge.net site + --logout Logout of the SourceForge.net site + --groupid=GROUPID Group ID of the project whose data you wish to export + +=head1 ERROR LEVELS + +The following error levels are returned upon exit of this program: + + 0 success + + 1 failure: general (requested DocManager operation failed) + + 2 failure: authentication failure + + 3 failure: must --login before performing this operation + + 4 failure: bad command-line option specified or variable setting problem + + 5 failure: error in accessing/creating a file or directory + + 6 failure: failed to enter requested input before timeout expired + +=head1 AUTHORITATIVE SOURCE + +The original version of B may be found in the materials +provided from the SourceForge.net Site Documentation project (sitedocs) +on the SourceForge.net site. The latest version of this program +may be found in the CVS repository for the sitedocs project on +SourceForge.net. The sitedocs project pages may be accessed at: +http://sourceforge.net/projects/sitedocs + +=head1 SECURITY + +For security-related information for this application, please review +the documentation provided for the adocman utility. + +=head1 EXAMPLES + +The following are examples for using this program to export project +data via the XML data export facility on SourceForge.net. It is presumed +that you have a valid SourceForge.net user account, which is listed as +a project administrator on the project in question. This tool will +only work for project administrators. The group ID for the project +may be derived from the URL for the Admin page for the project, or by +viewing the Project Admin page for the project (look for the text +"Your Group ID is: xxxxxx"). + +To login to the SourceForge.net site via the command-line: + + adocman --username=myusername --password=mypassword --login \ + --groupid=8675309 + +To login to the SourceForge.net site, and be prompted to enter your +password interactively: + + adocman --username=myusername --interactive --login --groupid=8675309 + +To perform an export (after logging-in): + + xml_export --groupid=8675309 > output.xml + +To logout of SourceForge.net: + + adocman --logout + +Additional capabilities (including the use of configuration files to +specify information that would otherwise be provided interactively +or on the command-line) are detailed in the documentation provided for +the adocman utility. + +To obtain output for debugging a problem, perform the same command +as originally tested, but first add the --verbose flag, and determine +whether you are able to solve the issue on your own. If the problem +persists, see the "SUPPORT AND BUGS" section, below. + +=head1 SUPPORT AND BUGS + +This program was written by a member of the SourceForge.net staff +team. This software has been released under an Open Source license, +for the greater benefit of the SourceForge.net developer community. + +The SourceForge.net Site Documentation project is the caretaker of +this software. Issues related to the use of this program, or bugs +found in using this program, may be reported to the SourceForge.net +Site Documentation project using their Support Request Tracker at: +https://sourceforge.net/tracker/?func=add&group_id=52614&atid=467457 + +Any support that is provided for this program is provided as to +further enhance the stability and functionality of this program +for SourceForge.net users. The SourceForge.net Site Documentation +project makes use of this software for its own internal purposes, +in managing the Site Documentation collection for the SourceForge.net +site. + +=head1 AUTHOR + +Nathan Oostendorp and +Jacob Moorman + +=head1 PREREQUISITES + +C, C, C, C, +C + +These prerequisites may be installed in an interactive, but automated +fashion through the use of perl's CPAN module, invoked as: + + perl -MCPAN -e shell; + +=head1 LICENSE + +Copyright (c) 2002 Open Source Development Network, Inc. ("OSDN") + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +1. The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +2. Neither the names of VA Software Corporation, OSDN, SourceForge.net, +the SourceForge.net Site Documentation project, nor the names of its +contributors may be used to endorse or promote products derived from +the Software without specific prior written permission of OSDN. + +3. The name and trademarks of copyright holders may NOT be used in +advertising or publicity pertaining to the Software without specific, +written prior permission. Title to copyright in the Software and +any associated documentation will at all times remain with copyright +holders. + +4. If any files are modified, you must cause the modified files to carry +prominent notices stating that you changed the files and the date of +any change. We recommend that you provide URLs to the location from which +the code is derived. + +5. Altered versions of the Software must be plainly marked as such, and +must not be misrepresented as being the original Software. + +6. The origin of the Software must not be misrepresented; you must not +claim that you wrote the original Software. If you use the Software in a +product, an acknowledgment in the product documentation would be +appreciated but is not required. + +7. The data files supplied as input to, or produced as output from, +the programs of the Software do not automatically fall under the +copyright of the Software, but belong to whomever generated them, and may +be sold commercially, and may be aggregated with the Software. + +8. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE OR DOCUMENTATION. + +This Software consists of contributions made by OSDN and many individuals +on behalf of OSDN. Specific attributions are listed in the accompanying +credits file. + +=head1 HISTORY + +B<2002-12-03> Completed version 0.10 - move to classes, added POD + +=cut diff --git a/bin/xml_export-LICENSE b/bin/xml_export-LICENSE new file mode 100644 index 0000000..3f06fb7 --- /dev/null +++ b/bin/xml_export-LICENSE @@ -0,0 +1,52 @@ +Copyright (c) 2002 Open Source Development Network, Inc. ("OSDN") + +Permission is hereby granted, free of charge, to any person obtaining a +copy of this software and associated documentation files (the "Software"), +to deal in the Software without restriction, including without limitation +the rights to use, copy, modify, merge, publish, distribute, sublicense, +and/or sell copies of the Software, and to permit persons to whom the +Software is furnished to do so, subject to the following conditions: + +1. The above copyright notice and this permission notice shall be included +in all copies or substantial portions of the Software. + +2. Neither the names of VA Software Corporation, OSDN, SourceForge.net, +the SourceForge.net Site Documentation project, nor the names of its +contributors may be used to endorse or promote products derived from +the Software without specific prior written permission of OSDN. + +3. The name and trademarks of copyright holders may NOT be used in +advertising or publicity pertaining to the Software without specific, +written prior permission. Title to copyright in the Software and +any associated documentation will at all times remain with copyright +holders. + +4. If any files are modified, you must cause the modified files to carry +prominent notices stating that you changed the files and the date of +any change. We recommend that you provide URLs to the location from which +the code is derived. + +5. Altered versions of the Software must be plainly marked as such, and +must not be misrepresented as being the original Software. + +6. The origin of the Software must not be misrepresented; you must not +claim that you wrote the original Software. If you use the Software in a +product, an acknowledgment in the product documentation would be +appreciated but is not required. + +7. The data files supplied as input to, or produced as output from, +the programs of the Software do not automatically fall under the +copyright of the Software, but belong to whomever generated them, and may +be sold commercially, and may be aggregated with the Software. + +8. THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. +IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY +CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, +TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE +SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE OR DOCUMENTATION. + +This Software consists of contributions made by OSDN and many individuals +on behalf of OSDN. Specific attributions are listed in the accompanying +credits file. diff --git a/bin/xml_export-README b/bin/xml_export-README new file mode 100644 index 0000000..82e3524 --- /dev/null +++ b/bin/xml_export-README @@ -0,0 +1,56 @@ +This copy of xml_export was snarfed from adocman-0.10 from SourceForge. +We're checking in a copy as a convenience for any future SCons project +administrator who may need to download exported XML data. The original, +unmodified contents of the README file for that release of adocman are +as follows: + + +adocman - Automation tool for SourceForge.net DocManager handling +Copyright (C) 2002 Open Source Development Network, Inc. ("OSDN") + + +File manifest: + +Alexandria perl-based API for performing operations against the + SourceForge.net site, currently including basic Client + operations (i.e. login/logout) and DocManager operations + +adocman The adocman program, providing the means to perform + DocManager operations from the command-line or scripts + (by project developers or admins listed as DocManager Editors) + +xml_export The xml_export program, providing the means to automate + downloads of data from the XML data export facility + on SourceForge.net (by project administrators) + +adocman.html Manual for adocman, including background information, + command-line options detail, etc. + +xml_export.html Manual for xml_export, including basic info about + command-line options. See adocman.html for additional + information. + +LICENSE License terms for adocman + +README This file + +TODO List of ongoing work in improving adocman. NOTE: + Please contact the maintainer before starting any effort + to improve this code. We have significantly modified + the structure and design of this program for the next + release; structure and command-line interface are subject + to change without notice. + +A list of the prerequisites required to execute 'adocman' may be found +at in the PREREQUISITES section of the adocman manual (adocman.html). +Though not listed, a recent installation of 'perl' is also a prerequisite. + +Support for this program may be obtained as per the SUPPORT AND BUGS +section of the adocman.html manual. Any questions or concerns regarding +this software should be escalated as per the SUPPORT AND BUGS section +of the provided manual. + +The authoritative source of this software is: + https://sourceforge.net/projects/sitedocs + + diff --git a/bin/xmlagenda.py b/bin/xmlagenda.py new file mode 100755 index 0000000..3009e4c --- /dev/null +++ b/bin/xmlagenda.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python + +# Download the issues from Issuzilla as XML; this creates a file named +# 'issues.xml'. Run this script to translate 'issues.xml' into a CSV +# file named 'editlist.csv'. Upload the CSV into a Google spreadsheet. + +# In the spreadsheet, select the last column and pick "delete-->column" (it +# was added by the upload to allow for expansion and we don't need it). +# Select all the columns and pick "align-->top" +# Select the ID and votes columns and pick "align-->right" +# Select the priority column and pick "align-->center" +# Select the first row and click on the "bold" button +# Grab the lines between the column headers to adjust the column widths +# Grab the sort bar on the far left (just above the "1" for row one) +# and move it down one row. (Row one becomes a floating header) +# Voila! + +# The team members +# FIXME: These names really should be external to this script +team = 'Bill Greg Steven Gary Ken Brandon Sohail Jim David'.split() +team.sort() + +# The elements to be picked out of the issue +PickList = [ + # sort key -- these are used to sort the entry + 'target_milestone', 'priority', 'votes_desc', 'creation_ts', + # payload -- these are displayed + 'issue_id', 'votes', 'issue_type', 'target_milestone', + 'priority', 'assigned_to', 'short_desc', + ] + +# Conbert a leaf element into its value as a text string +# We assume it's "short enough" that there's only one substring +def Value(element): + v = element.firstChild + if v is None: return '' + return v.nodeValue + +# Parse the XML issues file and produce a DOM for it +import sys +if len(sys.argv) > 1: xml = sys.argv[1] +else: xml = 'issues.xml' +from xml.dom.minidom import parse +xml = parse(xml) + +# Go through the issues in the DOM, pick out the elements we want, +# and put them in our list of issues. +issues = [] +for issuezilla in xml.childNodes: + # The Issuezilla element contains the issues + if issuezilla.nodeType != issuezilla.ELEMENT_NODE: continue + for issue in issuezilla.childNodes: + # The issue elements contain the info for an issue + if issue.nodeType != issue.ELEMENT_NODE: continue + # Accumulate the pieces we want to include + d = {} + for element in issue.childNodes: + if element.nodeName in PickList: + d[element.nodeName] = Value(element) + # convert 'votes' to numeric, ascending and descending + try: + v = int('0' + d['votes']) + except KeyError: + pass + else: + d['votes_desc'] = -v + d['votes'] = v + # Marshal the elements and add them to the list + issues.append([ d[ix] for ix in PickList ]) +issues.sort() + +# Transcribe the issues into comma-separated values. +# FIXME: parameterize the output file name +import csv +writer = csv.writer(open('editlist.csv', 'w')) +# header +writer.writerow(['ID', 'Votes', 'Type/Member', 'Milestone', + 'Pri', 'Owner', 'Summary/Comments']) +for issue in issues: + row = issue[4:] # strip off sort key + #row[0] = """=hyperlink("http://scons.tigris.org/issues/show_bug.cgi?id=%s","%s")""" % (row[0],row[0]) + if row[3] == '-unspecified-': row[3] = 'triage' + writer.writerow(['','','','','','','']) + writer.writerow(row) + writer.writerow(['','','consensus','','','','']) + writer.writerow(['','','','','','','']) + for member in team: writer.writerow(['','',member,'','','','']) + +# Local Variables: +# tab-width:4 +# indent-tabs-mode:nil +# End: +# vim: set expandtab tabstop=4 shiftwidth=4: -- cgit v1.2.3