diff options
Diffstat (limited to 'Test/lit')
| -rw-r--r-- | Test/lit/TODO | 9 | ||||
| -rw-r--r-- | Test/lit/lit.py | 5 | ||||
| -rw-r--r-- | Test/lit/lit/LitConfig.py | 134 | ||||
| -rw-r--r-- | Test/lit/lit/LitFormats.py | 3 | ||||
| -rw-r--r-- | Test/lit/lit/LitTestCase.py | 30 | ||||
| -rw-r--r-- | Test/lit/lit/ProgressBar.py | 280 | ||||
| -rw-r--r-- | Test/lit/lit/ShCommands.py | 85 | ||||
| -rw-r--r-- | Test/lit/lit/ShUtil.py | 353 | ||||
| -rw-r--r-- | Test/lit/lit/TclUtil.py | 322 | ||||
| -rw-r--r-- | Test/lit/lit/Test.py | 79 | ||||
| -rw-r--r-- | Test/lit/lit/TestFormats.py | 232 | ||||
| -rw-r--r-- | Test/lit/lit/TestRunner.py | 607 | ||||
| -rw-r--r-- | Test/lit/lit/TestingConfig.py | 125 | ||||
| -rw-r--r-- | Test/lit/lit/Util.py | 141 | ||||
| -rw-r--r-- | Test/lit/lit/__init__.py | 10 | ||||
| -rw-r--r-- | Test/lit/lit/main.py | 669 | ||||
| -rw-r--r-- | Test/lit/setup.py | 70 |
17 files changed, 3154 insertions, 0 deletions
diff --git a/Test/lit/TODO b/Test/lit/TODO new file mode 100644 index 00000000..6d7f7ea5 --- /dev/null +++ b/Test/lit/TODO @@ -0,0 +1,9 @@ + - Move temp directory name into local test config. + + - Add --show-unsupported, don't show by default? + + - Optionally use multiprocessing. + + - Support valgrind in all configs, and LLVM style valgrind. + + - Support a timeout / ulimit. diff --git a/Test/lit/lit.py b/Test/lit/lit.py new file mode 100644 index 00000000..851063b3 --- /dev/null +++ b/Test/lit/lit.py @@ -0,0 +1,5 @@ +#!/usr/bin/env python + +if __name__=='__main__': + import lit + lit.main() diff --git a/Test/lit/lit/LitConfig.py b/Test/lit/lit/LitConfig.py new file mode 100644 index 00000000..c71c0ccd --- /dev/null +++ b/Test/lit/lit/LitConfig.py @@ -0,0 +1,134 @@ +class LitConfig: + """LitConfig - Configuration data for a 'lit' test runner instance, shared + across all tests. + + The LitConfig object is also used to communicate with client configuration + files, it is always passed in as the global variable 'lit' so that + configuration files can access common functionality and internal components + easily. + """ + + # Provide access to Test module. + import Test + + # Provide access to built-in formats. + import LitFormats as formats + + # Provide access to built-in utility functions. + import Util as util + + def __init__(self, progname, path, quiet, + useValgrind, valgrindLeakCheck, valgrindArgs, + useTclAsSh, + noExecute, ignoreStdErr, debug, isWindows, + params): + # The name of the test runner. + self.progname = progname + # The items to add to the PATH environment variable. + self.path = list(map(str, path)) + self.quiet = bool(quiet) + self.useValgrind = bool(useValgrind) + self.valgrindLeakCheck = bool(valgrindLeakCheck) + self.valgrindUserArgs = list(valgrindArgs) + self.useTclAsSh = bool(useTclAsSh) + self.noExecute = noExecute + self.ignoreStdErr = ignoreStdErr + self.debug = debug + self.isWindows = bool(isWindows) + self.params = dict(params) + self.bashPath = None + + self.numErrors = 0 + self.numWarnings = 0 + + self.valgrindArgs = [] + self.valgrindTriple = "" + if self.useValgrind: + self.valgrindTriple = "-vg" + self.valgrindArgs = ['valgrind', '-q', '--run-libc-freeres=no', + '--tool=memcheck', '--trace-children=yes', + '--error-exitcode=123'] + if self.valgrindLeakCheck: + self.valgrindTriple += "_leak" + self.valgrindArgs.append('--leak-check=full') + else: + # The default is 'summary'. + self.valgrindArgs.append('--leak-check=no') + self.valgrindArgs.extend(self.valgrindUserArgs) + + + def load_config(self, config, path): + """load_config(config, path) - Load a config object from an alternate + path.""" + from TestingConfig import TestingConfig + if self.debug: + self.note('load_config from %r' % path) + return TestingConfig.frompath(path, config.parent, self, + mustExist = True, + config = config) + + def getBashPath(self): + """getBashPath - Get the path to 'bash'""" + import os, Util + + if self.bashPath is not None: + return self.bashPath + + self.bashPath = Util.which('bash', os.pathsep.join(self.path)) + if self.bashPath is None: + # Check some known paths. + for path in ('/bin/bash', '/usr/bin/bash', '/usr/local/bin/bash'): + if os.path.exists(path): + self.bashPath = path + break + + if self.bashPath is None: + self.warning("Unable to find 'bash', running Tcl tests internally.") + self.bashPath = '' + + return self.bashPath + + def getToolsPath(self, dir, paths, tools): + import os, Util + if dir is not None and os.path.isabs(dir) and os.path.isdir(dir): + if not Util.checkToolsPath(dir, tools): + return None + else: + dir = Util.whichTools(tools, paths) + + # bash + self.bashPath = Util.which('bash', dir) + if self.bashPath is None: + self.note("Unable to find 'bash.exe'.") + self.bashPath = '' + + return dir + + def _write_message(self, kind, message): + import inspect, os, sys + + # Get the file/line where this message was generated. + f = inspect.currentframe() + # Step out of _write_message, and then out of wrapper. + f = f.f_back.f_back + file,line,_,_,_ = inspect.getframeinfo(f) + location = '%s:%d' % (os.path.basename(file), line) + + print >>sys.stderr, '%s: %s: %s: %s' % (self.progname, location, + kind, message) + + def note(self, message): + self._write_message('note', message) + + def warning(self, message): + self._write_message('warning', message) + self.numWarnings += 1 + + def error(self, message): + self._write_message('error', message) + self.numErrors += 1 + + def fatal(self, message): + import sys + self._write_message('fatal', message) + sys.exit(2) diff --git a/Test/lit/lit/LitFormats.py b/Test/lit/lit/LitFormats.py new file mode 100644 index 00000000..931d1071 --- /dev/null +++ b/Test/lit/lit/LitFormats.py @@ -0,0 +1,3 @@ +from TestFormats import FileBasedTest +from TestFormats import GoogleTest, ShTest, TclTest +from TestFormats import SyntaxCheckTest, OneCommandPerFileTest diff --git a/Test/lit/lit/LitTestCase.py b/Test/lit/lit/LitTestCase.py new file mode 100644 index 00000000..89511858 --- /dev/null +++ b/Test/lit/lit/LitTestCase.py @@ -0,0 +1,30 @@ +import unittest +import Test + +""" +TestCase adaptor for providing a 'unittest' compatible interface to 'lit' tests. +""" + +class UnresolvedError(RuntimeError): + pass + +class LitTestCase(unittest.TestCase): + def __init__(self, test, lit_config): + unittest.TestCase.__init__(self) + self._test = test + self._lit_config = lit_config + + def id(self): + return self._test.getFullName() + + def shortDescription(self): + return self._test.getFullName() + + def runTest(self): + tr, output = self._test.config.test_format.execute( + self._test, self._lit_config) + + if tr is Test.UNRESOLVED: + raise UnresolvedError(output) + elif tr.isFailure: + self.fail(output) diff --git a/Test/lit/lit/ProgressBar.py b/Test/lit/lit/ProgressBar.py new file mode 100644 index 00000000..5c85a175 --- /dev/null +++ b/Test/lit/lit/ProgressBar.py @@ -0,0 +1,280 @@ +#!/usr/bin/env python + +# Source: http://code.activestate.com/recipes/475116/, with +# modifications by Daniel Dunbar. + +import sys, re, time + +class TerminalController: + """ + A class that can be used to portably generate formatted output to + a terminal. + + `TerminalController` defines a set of instance variables whose + values are initialized to the control sequence necessary to + perform a given action. These can be simply included in normal + output to the terminal: + + >>> term = TerminalController() + >>> print 'This is '+term.GREEN+'green'+term.NORMAL + + Alternatively, the `render()` method can used, which replaces + '${action}' with the string required to perform 'action': + + >>> term = TerminalController() + >>> print term.render('This is ${GREEN}green${NORMAL}') + + If the terminal doesn't support a given action, then the value of + the corresponding instance variable will be set to ''. As a + result, the above code will still work on terminals that do not + support color, except that their output will not be colored. + Also, this means that you can test whether the terminal supports a + given action by simply testing the truth value of the + corresponding instance variable: + + >>> term = TerminalController() + >>> if term.CLEAR_SCREEN: + ... print 'This terminal supports clearning the screen.' + + Finally, if the width and height of the terminal are known, then + they will be stored in the `COLS` and `LINES` attributes. + """ + # Cursor movement: + BOL = '' #: Move the cursor to the beginning of the line + UP = '' #: Move the cursor up one line + DOWN = '' #: Move the cursor down one line + LEFT = '' #: Move the cursor left one char + RIGHT = '' #: Move the cursor right one char + + # Deletion: + CLEAR_SCREEN = '' #: Clear the screen and move to home position + CLEAR_EOL = '' #: Clear to the end of the line. + CLEAR_BOL = '' #: Clear to the beginning of the line. + CLEAR_EOS = '' #: Clear to the end of the screen + + # Output modes: + BOLD = '' #: Turn on bold mode + BLINK = '' #: Turn on blink mode + DIM = '' #: Turn on half-bright mode + REVERSE = '' #: Turn on reverse-video mode + NORMAL = '' #: Turn off all modes + + # Cursor display: + HIDE_CURSOR = '' #: Make the cursor invisible + SHOW_CURSOR = '' #: Make the cursor visible + + # Terminal size: + COLS = None #: Width of the terminal (None for unknown) + LINES = None #: Height of the terminal (None for unknown) + + # Foreground colors: + BLACK = BLUE = GREEN = CYAN = RED = MAGENTA = YELLOW = WHITE = '' + + # Background colors: + BG_BLACK = BG_BLUE = BG_GREEN = BG_CYAN = '' + BG_RED = BG_MAGENTA = BG_YELLOW = BG_WHITE = '' + + _STRING_CAPABILITIES = """ + BOL=cr UP=cuu1 DOWN=cud1 LEFT=cub1 RIGHT=cuf1 + CLEAR_SCREEN=clear CLEAR_EOL=el CLEAR_BOL=el1 CLEAR_EOS=ed BOLD=bold + BLINK=blink DIM=dim REVERSE=rev UNDERLINE=smul NORMAL=sgr0 + HIDE_CURSOR=cinvis SHOW_CURSOR=cnorm""".split() + _COLORS = """BLACK BLUE GREEN CYAN RED MAGENTA YELLOW WHITE""".split() + _ANSICOLORS = "BLACK RED GREEN YELLOW BLUE MAGENTA CYAN WHITE".split() + + def __init__(self, term_stream=sys.stdout): + """ + Create a `TerminalController` and initialize its attributes + with appropriate values for the current terminal. + `term_stream` is the stream that will be used for terminal + output; if this stream is not a tty, then the terminal is + assumed to be a dumb terminal (i.e., have no capabilities). + """ + # Curses isn't available on all platforms + try: import curses + except: return + + # If the stream isn't a tty, then assume it has no capabilities. + if not term_stream.isatty(): return + + # Check the terminal type. If we fail, then assume that the + # terminal has no capabilities. + try: curses.setupterm() + except: return + + # Look up numeric capabilities. + self.COLS = curses.tigetnum('cols') + self.LINES = curses.tigetnum('lines') + self.XN = curses.tigetflag('xenl') + + # Look up string capabilities. + for capability in self._STRING_CAPABILITIES: + (attrib, cap_name) = capability.split('=') + setattr(self, attrib, self._tigetstr(cap_name) or '') + + # Colors + set_fg = self._tigetstr('setf') + if set_fg: + for i,color in zip(range(len(self._COLORS)), self._COLORS): + setattr(self, color, curses.tparm(set_fg, i) or '') + set_fg_ansi = self._tigetstr('setaf') + if set_fg_ansi: + for i,color in zip(range(len(self._ANSICOLORS)), self._ANSICOLORS): + setattr(self, color, curses.tparm(set_fg_ansi, i) or '') + set_bg = self._tigetstr('setb') + if set_bg: + for i,color in zip(range(len(self._COLORS)), self._COLORS): + setattr(self, 'BG_'+color, curses.tparm(set_bg, i) or '') + set_bg_ansi = self._tigetstr('setab') + if set_bg_ansi: + for i,color in zip(range(len(self._ANSICOLORS)), self._ANSICOLORS): + setattr(self, 'BG_'+color, curses.tparm(set_bg_ansi, i) or '') + + def _tigetstr(self, cap_name): + # String capabilities can include "delays" of the form "$<2>". + # For any modern terminal, we should be able to just ignore + # these, so strip them out. + import curses + cap = curses.tigetstr(cap_name) or '' + return re.sub(r'\$<\d+>[/*]?', '', cap) + + def render(self, template): + """ + Replace each $-substitutions in the given template string with + the corresponding terminal control string (if it's defined) or + '' (if it's not). + """ + return re.sub(r'\$\$|\${\w+}', self._render_sub, template) + + def _render_sub(self, match): + s = match.group() + if s == '$$': return s + else: return getattr(self, s[2:-1]) + +####################################################################### +# Example use case: progress bar +####################################################################### + +class SimpleProgressBar: + """ + A simple progress bar which doesn't need any terminal support. + + This prints out a progress bar like: + 'Header: 0 .. 10.. 20.. ...' + """ + + def __init__(self, header): + self.header = header + self.atIndex = None + + def update(self, percent, message): + if self.atIndex is None: + sys.stdout.write(self.header) + self.atIndex = 0 + + next = int(percent*50) + if next == self.atIndex: + return + + for i in range(self.atIndex, next): + idx = i % 5 + if idx == 0: + sys.stdout.write('%-2d' % (i*2)) + elif idx == 1: + pass # Skip second char + elif idx < 4: + sys.stdout.write('.') + else: + sys.stdout.write(' ') + sys.stdout.flush() + self.atIndex = next + + def clear(self): + if self.atIndex is not None: + sys.stdout.write('\n') + sys.stdout.flush() + self.atIndex = None + +class ProgressBar: + """ + A 3-line progress bar, which looks like:: + + Header + 20% [===========----------------------------------] + progress message + + The progress bar is colored, if the terminal supports color + output; and adjusts to the width of the terminal. + """ + BAR = '%s${GREEN}[${BOLD}%s%s${NORMAL}${GREEN}]${NORMAL}%s' + HEADER = '${BOLD}${CYAN}%s${NORMAL}\n\n' + + def __init__(self, term, header, useETA=True): + self.term = term + if not (self.term.CLEAR_EOL and self.term.UP and self.term.BOL): + raise ValueError("Terminal isn't capable enough -- you " + "should use a simpler progress dispaly.") + self.BOL = self.term.BOL # BoL from col#79 + self.XNL = "\n" # Newline from col#79 + if self.term.COLS: + self.width = self.term.COLS + if not self.term.XN: + self.BOL = self.term.UP + self.term.BOL + self.XNL = "" # Cursor must be fed to the next line + else: + self.width = 75 + self.bar = term.render(self.BAR) + self.header = self.term.render(self.HEADER % header.center(self.width)) + self.cleared = 1 #: true if we haven't drawn the bar yet. + self.useETA = useETA + if self.useETA: + self.startTime = time.time() + self.update(0, '') + + def update(self, percent, message): + if self.cleared: + sys.stdout.write(self.header) + self.cleared = 0 + prefix = '%3d%% ' % (percent*100,) + suffix = '' + if self.useETA: + elapsed = time.time() - self.startTime + if percent > .0001 and elapsed > 1: + total = elapsed / percent + eta = int(total - elapsed) + h = eta//3600. + m = (eta//60) % 60 + s = eta % 60 + suffix = ' ETA: %02d:%02d:%02d'%(h,m,s) + barWidth = self.width - len(prefix) - len(suffix) - 2 + n = int(barWidth*percent) + if len(message) < self.width: + message = message + ' '*(self.width - len(message)) + else: + message = '... ' + message[-(self.width-4):] + sys.stdout.write( + self.BOL + self.term.UP + self.term.CLEAR_EOL + + (self.bar % (prefix, '='*n, '-'*(barWidth-n), suffix)) + + self.XNL + + self.term.CLEAR_EOL + message) + if not self.term.XN: + sys.stdout.flush() + + def clear(self): + if not self.cleared: + sys.stdout.write(self.BOL + self.term.CLEAR_EOL + + self.term.UP + self.term.CLEAR_EOL + + self.term.UP + self.term.CLEAR_EOL) + sys.stdout.flush() + self.cleared = 1 + +def test(): + import time + tc = TerminalController() + p = ProgressBar(tc, 'Tests') + for i in range(101): + p.update(i/100., str(i)) + time.sleep(.3) + +if __name__=='__main__': + test() diff --git a/Test/lit/lit/ShCommands.py b/Test/lit/lit/ShCommands.py new file mode 100644 index 00000000..4550437c --- /dev/null +++ b/Test/lit/lit/ShCommands.py @@ -0,0 +1,85 @@ +class Command: + def __init__(self, args, redirects): + self.args = list(args) + self.redirects = list(redirects) + + def __repr__(self): + return 'Command(%r, %r)' % (self.args, self.redirects) + + def __cmp__(self, other): + if not isinstance(other, Command): + return -1 + + return cmp((self.args, self.redirects), + (other.args, other.redirects)) + + def toShell(self, file): + for arg in self.args: + if "'" not in arg: + quoted = "'%s'" % arg + elif '"' not in arg and '$' not in arg: + quoted = '"%s"' % arg + else: + raise NotImplementedError,'Unable to quote %r' % arg + print >>file, quoted, + + # For debugging / validation. + import ShUtil + dequoted = list(ShUtil.ShLexer(quoted).lex()) + if dequoted != [arg]: + raise NotImplementedError,'Unable to quote %r' % arg + + for r in self.redirects: + if len(r[0]) == 1: + print >>file, "%s '%s'" % (r[0][0], r[1]), + else: + print >>file, "%s%s '%s'" % (r[0][1], r[0][0], r[1]), + +class Pipeline: + def __init__(self, commands, negate=False, pipe_err=False): + self.commands = commands + self.negate = negate + self.pipe_err = pipe_err + + def __repr__(self): + return 'Pipeline(%r, %r, %r)' % (self.commands, self.negate, + self.pipe_err) + + def __cmp__(self, other): + if not isinstance(other, Pipeline): + return -1 + + return cmp((self.commands, self.negate, self.pipe_err), + (other.commands, other.negate, self.pipe_err)) + + def toShell(self, file, pipefail=False): + if pipefail != self.pipe_err: + raise ValueError,'Inconsistent "pipefail" attribute!' + if self.negate: + print >>file, '!', + for cmd in self.commands: + cmd.toShell(file) + if cmd is not self.commands[-1]: + print >>file, '|\n ', + +class Seq: + def __init__(self, lhs, op, rhs): + assert op in (';', '&', '||', '&&') + self.op = op + self.lhs = lhs + self.rhs = rhs + + def __repr__(self): + return 'Seq(%r, %r, %r)' % (self.lhs, self.op, self.rhs) + + def __cmp__(self, other): + if not isinstance(other, Seq): + return -1 + + return cmp((self.lhs, self.op, self.rhs), + (other.lhs, other.op, other.rhs)) + + def toShell(self, file, pipefail=False): + self.lhs.toShell(file, pipefail) + print >>file, ' %s\n' % self.op + self.rhs.toShell(file, pipefail) diff --git a/Test/lit/lit/ShUtil.py b/Test/lit/lit/ShUtil.py new file mode 100644 index 00000000..dda622a4 --- /dev/null +++ b/Test/lit/lit/ShUtil.py @@ -0,0 +1,353 @@ +import itertools + +import Util +from ShCommands import Command, Pipeline, Seq + +class ShLexer: + def __init__(self, data, win32Escapes = False): + self.data = data + self.pos = 0 + self.end = len(data) + self.win32Escapes = win32Escapes + + def eat(self): + c = self.data[self.pos] + self.pos += 1 + return c + + def look(self): + return self.data[self.pos] + + def maybe_eat(self, c): + """ + maybe_eat(c) - Consume the character c if it is the next character, + returning True if a character was consumed. """ + if self.data[self.pos] == c: + self.pos += 1 + return True + return False + + def lex_arg_fast(self, c): + # Get the leading whitespace free section. + chunk = self.data[self.pos - 1:].split(None, 1)[0] + + # If it has special characters, the fast path failed. + if ('|' in chunk or '&' in chunk or + '<' in chunk or '>' in chunk or + "'" in chunk or '"' in chunk or + '\\' in chunk): + return None + + self.pos = self.pos - 1 + len(chunk) + return chunk + + def lex_arg_slow(self, c): + if c in "'\"": + str = self.lex_arg_quoted(c) + else: + str = c + while self.pos != self.end: + c = self.look() + if c.isspace() or c in "|&": + break + elif c in '><': + # This is an annoying case; we treat '2>' as a single token so + # we don't have to track whitespace tokens. + + # If the parse string isn't an integer, do the usual thing. + if not str.isdigit(): + break + + # Otherwise, lex the operator and convert to a redirection + # token. + num = int(str) + tok = self.lex_one_token() + assert isinstance(tok, tuple) and len(tok) == 1 + return (tok[0], num) + elif c == '"': + self.eat() + str += self.lex_arg_quoted('"') + elif c == "'": + self.eat() + str += self.lex_arg_quoted("'") + elif not self.win32Escapes and c == '\\': + # Outside of a string, '\\' escapes everything. + self.eat() + if self.pos == self.end: + Util.warning("escape at end of quoted argument in: %r" % + self.data) + return str + str += self.eat() + else: + str += self.eat() + return str + + def lex_arg_quoted(self, delim): + str = '' + while self.pos != self.end: + c = self.eat() + if c == delim: + return str + elif c == '\\' and delim == '"': + # Inside a '"' quoted string, '\\' only escapes the quote + # character and backslash, otherwise it is preserved. + if self.pos == self.end: + Util.warning("escape at end of quoted argument in: %r" % + self.data) + return str + c = self.eat() + if c == '"': # + str += '"' + elif c == '\\': + str += '\\' + else: + str += '\\' + c + else: + str += c + Util.warning("missing quote character in %r" % self.data) + return str + + def lex_arg_checked(self, c): + pos = self.pos + res = self.lex_arg_fast(c) + end = self.pos + + self.pos = pos + reference = self.lex_arg_slow(c) + if res is not None: + if res != reference: + raise ValueError,"Fast path failure: %r != %r" % (res, reference) + if self.pos != end: + raise ValueError,"Fast path failure: %r != %r" % (self.pos, end) + return reference + + def lex_arg(self, c): + return self.lex_arg_fast(c) or self.lex_arg_slow(c) + + def lex_one_token(self): + """ + lex_one_token - Lex a single 'sh' token. """ + + c = self.eat() + if c in ';!': + return (c,) + if c == '|': + if self.maybe_eat('|'): + return ('||',) + return (c,) + if c == '&': + if self.maybe_eat('&'): + return ('&&',) + if self.maybe_eat('>'): + return ('&>',) + return (c,) + if c == '>': + if self.maybe_eat('&'): + return ('>&',) + if self.maybe_eat('>'): + return ('>>',) + return (c,) + if c == '<': + if self.maybe_eat('&'): + return ('<&',) + if self.maybe_eat('>'): + return ('<<',) + return (c,) + + return self.lex_arg(c) + + def lex(self): + while self.pos != self.end: + if self.look().isspace(): + self.eat() + else: + yield self.lex_one_token() + +### + +class ShParser: + def __init__(self, data, win32Escapes = False): + self.data = data + self.tokens = ShLexer(data, win32Escapes = win32Escapes).lex() + + def lex(self): + try: + return self.tokens.next() + except StopIteration: + return None + + def look(self): + next = self.lex() + if next is not None: + self.tokens = itertools.chain([next], self.tokens) + return next + + def parse_command(self): + tok = self.lex() + if not tok: + raise ValueError,"empty command!" + if isinstance(tok, tuple): + raise ValueError,"syntax error near unexpected token %r" % tok[0] + + args = [tok] + redirects = [] + while 1: + tok = self.look() + + # EOF? + if tok is None: + break + + # If this is an argument, just add it to the current command. + if isinstance(tok, str): + args.append(self.lex()) + continue + + # Otherwise see if it is a terminator. + assert isinstance(tok, tuple) + if tok[0] in ('|',';','&','||','&&'): + break + + # Otherwise it must be a redirection. + op = self.lex() + arg = self.lex() + if not arg: + raise ValueError,"syntax error near token %r" % op[0] + redirects.append((op, arg)) + + return Command(args, redirects) + + def parse_pipeline(self): + negate = False + if self.look() == ('!',): + self.lex() + negate = True + + commands = [self.parse_command()] + while self.look() == ('|',): + self.lex() + commands.append(self.parse_command()) + return Pipeline(commands, negate) + + def parse(self): + lhs = self.parse_pipeline() + + while self.look(): + operator = self.lex() + assert isinstance(operator, tuple) and len(operator) == 1 + + if not self.look(): + raise ValueError, "missing argument to operator %r" % operator[0] + + # FIXME: Operator precedence!! + lhs = Seq(lhs, operator[0], self.parse_pipeline()) + + return lhs + +### + +import unittest + +class TestShLexer(unittest.TestCase): + def lex(self, str, *args, **kwargs): + return list(ShLexer(str, *args, **kwargs).lex()) + + def test_basic(self): + self.assertEqual(self.lex('a|b>c&d<e'), + ['a', ('|',), 'b', ('>',), 'c', ('&',), 'd', + ('<',), 'e']) + + def test_redirection_tokens(self): + self.assertEqual(self.lex('a2>c'), + ['a2', ('>',), 'c']) + self.assertEqual(self.lex('a 2>c'), + ['a', ('>',2), 'c']) + + def test_quoting(self): + self.assertEqual(self.lex(""" 'a' """), + ['a']) + self.assertEqual(self.lex(""" "hello\\"world" """), + ['hello"world']) + self.assertEqual(self.lex(""" "hello\\'world" """), + ["hello\\'world"]) + self.assertEqual(self.lex(""" "hello\\\\world" """), + ["hello\\world"]) + self.assertEqual(self.lex(""" he"llo wo"rld """), + ["hello world"]) + self.assertEqual(self.lex(""" a\\ b a\\\\b """), + ["a b", "a\\b"]) + self.assertEqual(self.lex(""" "" "" """), + ["", ""]) + self.assertEqual(self.lex(""" a\\ b """, win32Escapes = True), + ['a\\', 'b']) + +class TestShParse(unittest.TestCase): + def parse(self, str): + return ShParser(str).parse() + + def test_basic(self): + self.assertEqual(self.parse('echo hello'), + Pipeline([Command(['echo', 'hello'], [])], False)) + self.assertEqual(self.parse('echo ""'), + Pipeline([Command(['echo', ''], [])], False)) + self.assertEqual(self.parse("""echo -DFOO='a'"""), + Pipeline([Command(['echo', '-DFOO=a'], [])], False)) + self.assertEqual(self.parse('echo -DFOO="a"'), + Pipeline([Command(['echo', '-DFOO=a'], [])], False)) + + def test_redirection(self): + self.assertEqual(self.parse('echo hello > c'), + Pipeline([Command(['echo', 'hello'], + [((('>'),), 'c')])], False)) + self.assertEqual(self.parse('echo hello > c >> d'), + Pipeline([Command(['echo', 'hello'], [(('>',), 'c'), + (('>>',), 'd')])], False)) + self.assertEqual(self.parse('a 2>&1'), + Pipeline([Command(['a'], [(('>&',2), '1')])], False)) + + def test_pipeline(self): + self.assertEqual(self.parse('a | b'), + Pipeline([Command(['a'], []), + Command(['b'], [])], + False)) + + self.assertEqual(self.parse('a | b | c'), + Pipeline([Command(['a'], []), + Command(['b'], []), + Command(['c'], [])], + False)) + + self.assertEqual(self.parse('! a'), + Pipeline([Command(['a'], [])], + True)) + + def test_list(self): + self.assertEqual(self.parse('a ; b'), + Seq(Pipeline([Command(['a'], [])], False), + ';', + Pipeline([Command(['b'], [])], False))) + + self.assertEqual(self.parse('a & b'), + Seq(Pipeline([Command(['a'], [])], False), + '&', + Pipeline([Command(['b'], [])], False))) + + self.assertEqual(self.parse('a && b'), + Seq(Pipeline([Command(['a'], [])], False), + '&&', + Pipeline([Command(['b'], [])], False))) + + self.assertEqual(self.parse('a || b'), + Seq(Pipeline([Command(['a'], [])], False), + '||', + Pipeline([Command(['b'], [])], False))) + + self.assertEqual(self.parse('a && b || c'), + Seq(Seq(Pipeline([Command(['a'], [])], False), + '&&', + Pipeline([Command(['b'], [])], False)), + '||', + Pipeline([Command(['c'], [])], False))) + +if __name__ == '__main__': + unittest.main() diff --git a/Test/lit/lit/TclUtil.py b/Test/lit/lit/TclUtil.py new file mode 100644 index 00000000..4a3f3450 --- /dev/null +++ b/Test/lit/lit/TclUtil.py @@ -0,0 +1,322 @@ +import itertools + +from ShCommands import Command, Pipeline + +def tcl_preprocess(data): + # Tcl has a preprocessing step to replace escaped newlines. + i = data.find('\\\n') + if i == -1: + return data + + # Replace '\\\n' and subsequent whitespace by a single space. + n = len(data) + str = data[:i] + i += 2 + while i < n and data[i] in ' \t': + i += 1 + return str + ' ' + data[i:] + +class TclLexer: + """TclLexer - Lex a string into "words", following the Tcl syntax.""" + + def __init__(self, data): + self.data = tcl_preprocess(data) + self.pos = 0 + self.end = len(self.data) + + def at_end(self): + return self.pos == self.end + + def eat(self): + c = self.data[self.pos] + self.pos += 1 + return c + + def look(self): + return self.data[self.pos] + + def maybe_eat(self, c): + """ + maybe_eat(c) - Consume the character c if it is the next character, + returning True if a character was consumed. """ + if self.data[self.pos] == c: + self.pos += 1 + return True + return False + + def escape(self, c): + if c == 'a': + return '\x07' + elif c == 'b': + return '\x08' + elif c == 'f': + return '\x0c' + elif c == 'n': + return '\n' + elif c == 'r': + return '\r' + elif c == 't': + return '\t' + elif c == 'v': + return '\x0b' + elif c in 'uxo': + raise ValueError,'Invalid quoted character %r' % c + else: + return c + + def lex_braced(self): + # Lex until whitespace or end of string, the opening brace has already + # been consumed. + + str = '' + while 1: + if self.at_end(): + raise ValueError,"Unterminated '{' quoted word" + + c = self.eat() + if c == '}': + break + elif c == '{': + str += '{' + self.lex_braced() + '}' + elif c == '\\' and self.look() in '{}': + str += self.eat() + else: + str += c + + return str + + def lex_quoted(self): + str = '' + + while 1: + if self.at_end(): + raise ValueError,"Unterminated '\"' quoted word" + + c = self.eat() + if c == '"': + break + elif c == '\\': + if self.at_end(): + raise ValueError,'Missing quoted character' + + str += self.escape(self.eat()) + else: + str += c + + return str + + def lex_unquoted(self, process_all=False): + # Lex until whitespace or end of string. + str = '' + while not self.at_end(): + if not process_all: + if self.look().isspace() or self.look() == ';': + break + + c = self.eat() + if c == '\\': + if self.at_end(): + raise ValueError,'Missing quoted character' + + str += self.escape(self.eat()) + elif c == '[': + raise NotImplementedError, ('Command substitution is ' + 'not supported') + elif c == '$' and not self.at_end() and (self.look().isalpha() or + self.look() == '{'): + raise NotImplementedError, ('Variable substitution is ' + 'not supported') + else: + str += c + + return str + + def lex_one_token(self): + if self.maybe_eat('"'): + return self.lex_quoted() + elif self.maybe_eat('{'): + # Check for argument substitution. + if not self.maybe_eat('*'): + return self.lex_braced() + + if not self.maybe_eat('}'): + return '*' + self.lex_braced() + + if self.at_end() or self.look().isspace(): + return '*' + + raise NotImplementedError, "Argument substitution is unsupported" + else: + return self.lex_unquoted() + + def lex(self): + while not self.at_end(): + c = self.look() + if c in ' \t': + self.eat() + elif c in ';\n': + self.eat() + yield (';',) + else: + yield self.lex_one_token() + +class TclExecCommand: + kRedirectPrefixes1 = ('<', '>') + kRedirectPrefixes2 = ('<@', '<<', '2>', '>&', '>>', '>@') + kRedirectPrefixes3 = ('2>@', '2>>', '>>&', '>&@') + kRedirectPrefixes4 = ('2>@1',) + + def __init__(self, args): + self.args = iter(args) + + def lex(self): + try: + return self.args.next() + except StopIteration: + return None + + def look(self): + next = self.lex() + if next is not None: + self.args = itertools.chain([next], self.args) + return next + + def parse_redirect(self, tok, length): + if len(tok) == length: + arg = self.lex() + if arg is None: + raise ValueError,'Missing argument to %r redirection' % tok + else: + tok,arg = tok[:length],tok[length:] + + if tok[0] == '2': + op = (tok[1:],2) + else: + op = (tok,) + return (op, arg) + + def parse_pipeline(self): + if self.look() is None: + raise ValueError,"Expected at least one argument to exec" + + commands = [Command([],[])] + while 1: + arg = self.lex() + if arg is None: + break + elif arg == '|': + commands.append(Command([],[])) + elif arg == '|&': + # Write this as a redirect of stderr; it must come first because + # stdout may have already been redirected. + commands[-1].redirects.insert(0, (('>&',2),'1')) + commands.append(Command([],[])) + elif arg[:4] in TclExecCommand.kRedirectPrefixes4: + commands[-1].redirects.append(self.parse_redirect(arg, 4)) + elif arg[:3] in TclExecCommand.kRedirectPrefixes3: + commands[-1].redirects.append(self.parse_redirect(arg, 3)) + elif arg[:2] in TclExecCommand.kRedirectPrefixes2: + commands[-1].redirects.append(self.parse_redirect(arg, 2)) + elif arg[:1] in TclExecCommand.kRedirectPrefixes1: + commands[-1].redirects.append(self.parse_redirect(arg, 1)) + else: + commands[-1].args.append(arg) + + return Pipeline(commands, False, pipe_err=True) + + def parse(self): + ignoreStderr = False + keepNewline = False + + # Parse arguments. + while 1: + next = self.look() + if not isinstance(next, str) or next[0] != '-': + break + + if next == '--': + self.lex() + break + elif next == '-ignorestderr': + ignoreStderr = True + elif next == '-keepnewline': + keepNewline = True + else: + raise ValueError,"Invalid exec argument %r" % next + + return (ignoreStderr, keepNewline, self.parse_pipeline()) + +### + +import unittest + +class TestTclLexer(unittest.TestCase): + def lex(self, str, *args, **kwargs): + return list(TclLexer(str, *args, **kwargs).lex()) + + def test_preprocess(self): + self.assertEqual(tcl_preprocess('a b'), 'a b') + self.assertEqual(tcl_preprocess('a\\\nb c'), 'a b c') + + def test_unquoted(self): + self.assertEqual(self.lex('a b c'), + ['a', 'b', 'c']) + self.assertEqual(self.lex(r'a\nb\tc\ '), + ['a\nb\tc ']) + self.assertEqual(self.lex(r'a \\\$b c $\\'), + ['a', r'\$b', 'c', '$\\']) + + def test_braced(self): + self.assertEqual(self.lex('a {b c} {}'), + ['a', 'b c', '']) + self.assertEqual(self.lex(r'a {b {c\n}}'), + ['a', 'b {c\\n}']) + self.assertEqual(self.lex(r'a {b\{}'), + ['a', 'b{']) + self.assertEqual(self.lex(r'{*}'), ['*']) + self.assertEqual(self.lex(r'{*} a'), ['*', 'a']) + self.assertEqual(self.lex(r'{*} a'), ['*', 'a']) + self.assertEqual(self.lex('{a\\\n b}'), + ['a b']) + + def test_quoted(self): + self.assertEqual(self.lex('a "b c"'), + ['a', 'b c']) + + def test_terminators(self): + self.assertEqual(self.lex('a\nb'), + ['a', (';',), 'b']) + self.assertEqual(self.lex('a;b'), + ['a', (';',), 'b']) + self.assertEqual(self.lex('a ; b'), + ['a', (';',), 'b']) + +class TestTclExecCommand(unittest.TestCase): + def parse(self, str): + return TclExecCommand(list(TclLexer(str).lex())).parse() + + def test_basic(self): + self.assertEqual(self.parse('echo hello'), + (False, False, + Pipeline([Command(['echo', 'hello'], [])], + False, True))) + self.assertEqual(self.parse('echo hello | grep hello'), + (False, False, + Pipeline([Command(['echo', 'hello'], []), + Command(['grep', 'hello'], [])], + False, True))) + + def test_redirect(self): + self.assertEqual(self.parse('echo hello > a >b >>c 2> d |& e'), + (False, False, + Pipeline([Command(['echo', 'hello'], + [(('>&',2),'1'), + (('>',),'a'), + (('>',),'b'), + (('>>',),'c'), + (('>',2),'d')]), + Command(['e'], [])], + False, True))) + +if __name__ == '__main__': + unittest.main() diff --git a/Test/lit/lit/Test.py b/Test/lit/lit/Test.py new file mode 100644 index 00000000..db2e0324 --- /dev/null +++ b/Test/lit/lit/Test.py @@ -0,0 +1,79 @@ +import os + +# Test results. + +class TestResult: + def __init__(self, name, isFailure): + self.name = name + self.isFailure = isFailure + +PASS = TestResult('PASS', False) +XFAIL = TestResult('XFAIL', False) +FAIL = TestResult('FAIL', True) +XPASS = TestResult('XPASS', True) +UNRESOLVED = TestResult('UNRESOLVED', True) +UNSUPPORTED = TestResult('UNSUPPORTED', False) + +# Test classes. + +class TestFormat: + """TestFormat - Test information provider.""" + + def __init__(self, name): + self.name = name + +class TestSuite: + """TestSuite - Information on a group of tests. + + A test suite groups together a set of logically related tests. + """ + + def __init__(self, name, source_root, exec_root, config): + self.name = name + self.source_root = source_root + self.exec_root = exec_root + # The test suite configuration. + self.config = config + + def getSourcePath(self, components): + return os.path.join(self.source_root, *components) + + def getExecPath(self, components): + return os.path.join(self.exec_root, *components) + +class Test: + """Test - Information on a single test instance.""" + + def __init__(self, suite, path_in_suite, config): + self.suite = suite + self.path_in_suite = path_in_suite + self.config = config + # The test result code, once complete. + self.result = None + # Any additional output from the test, once complete. + self.output = None + # The wall time to execute this test, if timing and once complete. + self.elapsed = None + # The repeat index of this test, or None. + self.index = None + + def copyWithIndex(self, index): + import copy + res = copy.copy(self) + res.index = index + return res + + def setResult(self, result, output, elapsed): + assert self.result is None, "Test result already set!" + self.result = result + self.output = output + self.elapsed = elapsed + + def getFullName(self): + return self.suite.config.name + ' :: ' + '/'.join(self.path_in_suite) + + def getSourcePath(self): + return self.suite.getSourcePath(self.path_in_suite) + + def getExecPath(self): + return self.suite.getExecPath(self.path_in_suite) diff --git a/Test/lit/lit/TestFormats.py b/Test/lit/lit/TestFormats.py new file mode 100644 index 00000000..d1c0558b --- /dev/null +++ b/Test/lit/lit/TestFormats.py @@ -0,0 +1,232 @@ +import os +import sys + +import Test +import TestRunner +import Util + +kIsWindows = sys.platform in ['win32', 'cygwin'] + +class GoogleTest(object): + def __init__(self, test_sub_dir, test_suffix): + self.test_sub_dir = os.path.normcase(str(test_sub_dir)).split(';') + self.test_suffix = str(test_suffix) + + # On Windows, assume tests will also end in '.exe'. + if kIsWindows: + self.test_suffix += '.exe' + + def getGTestTests(self, path, litConfig, localConfig): + """getGTestTests(path) - [name] + + Return the tests available in gtest executable. + + Args: + path: String path to a gtest executable + litConfig: LitConfig instance + localConfig: TestingConfig instance""" + + try: + lines = Util.capture([path, '--gtest_list_tests'], + env=localConfig.environment) + if kIsWindows: + lines = lines.replace('\r', '') + lines = lines.split('\n') + except: + litConfig.error("unable to discover google-tests in %r" % path) + raise StopIteration + + nested_tests = [] + for ln in lines: + if not ln.strip(): + continue + + prefix = '' + index = 0 + while ln[index*2:index*2+2] == ' ': + index += 1 + while len(nested_tests) > index: + nested_tests.pop() + + ln = ln[index*2:] + if ln.endswith('.'): + nested_tests.append(ln) + else: + yield ''.join(nested_tests) + ln + + def getTestsInDirectory(self, testSuite, path_in_suite, + litConfig, localConfig): + source_path = testSuite.getSourcePath(path_in_suite) + for filename in os.listdir(source_path): + # Check for the one subdirectory (build directory) tests will be in. + if not '.' in self.test_sub_dir: + if not os.path.normcase(filename) in self.test_sub_dir: + continue + + filepath = os.path.join(source_path, filename) + if not os.path.isdir(filepath): + continue + + for subfilename in os.listdir(filepath): + if subfilename.endswith(self.test_suffix): + execpath = os.path.join(filepath, subfilename) + + # Discover the tests in this executable. + for name in self.getGTestTests(execpath, litConfig, + localConfig): + testPath = path_in_suite + (filename, subfilename, name) + yield Test.Test(testSuite, testPath, localConfig) + + def execute(self, test, litConfig): + testPath,testName = os.path.split(test.getSourcePath()) + while not os.path.exists(testPath): + # Handle GTest parametrized and typed tests, whose name includes + # some '/'s. + testPath, namePrefix = os.path.split(testPath) + testName = os.path.join(namePrefix, testName) + + cmd = [testPath, '--gtest_filter=' + testName] + if litConfig.useValgrind: + cmd = litConfig.valgrindArgs + cmd + + out, err, exitCode = TestRunner.executeCommand( + cmd, env=test.config.environment) + + if not exitCode: + return Test.PASS,'' + + return Test.FAIL, out + err + +### + +class FileBasedTest(object): + def getTestsInDirectory(self, testSuite, path_in_suite, + litConfig, localConfig): + source_path = testSuite.getSourcePath(path_in_suite) + for filename in os.listdir(source_path): + # Ignore dot files and excluded tests. + if (filename.startswith('.') or + filename in localConfig.excludes): + continue + + filepath = os.path.join(source_path, filename) + if not os.path.isdir(filepath): + base,ext = os.path.splitext(filename) + if ext in localConfig.suffixes: + yield Test.Test(testSuite, path_in_suite + (filename,), + localConfig) + +class ShTest(FileBasedTest): + def __init__(self, execute_external = False): + self.execute_external = execute_external + + def execute(self, test, litConfig): + return TestRunner.executeShTest(test, litConfig, + self.execute_external) + +class TclTest(FileBasedTest): + def __init__(self, ignoreStdErr=False): + self.ignoreStdErr = ignoreStdErr + + def execute(self, test, litConfig): + litConfig.ignoreStdErr = self.ignoreStdErr + return TestRunner.executeTclTest(test, litConfig) + +### + +import re +import tempfile + +class OneCommandPerFileTest: + # FIXME: Refactor into generic test for running some command on a directory + # of inputs. + + def __init__(self, command, dir, recursive=False, + pattern=".*", useTempInput=False): + if isinstance(command, str): + self.command = [command] + else: + self.command = list(command) + if dir is not None: + dir = str(dir) + self.dir = dir + self.recursive = bool(recursive) + self.pattern = re.compile(pattern) + self.useTempInput = useTempInput + + def getTestsInDirectory(self, testSuite, path_in_suite, + litConfig, localConfig): + dir = self.dir + if dir is None: + dir = testSuite.getSourcePath(path_in_suite) + + for dirname,subdirs,filenames in os.walk(dir): + if not self.recursive: + subdirs[:] = [] + + subdirs[:] = [d for d in subdirs + if (d != '.svn' and + d not in localConfig.excludes)] + + for filename in filenames: + if (filename.startswith('.') or + not self.pattern.match(filename) or + filename in localConfig.excludes): + continue + + path = os.path.join(dirname,filename) + suffix = path[len(dir):] + if suffix.startswith(os.sep): + suffix = suffix[1:] + test = Test.Test(testSuite, + path_in_suite + tuple(suffix.split(os.sep)), + localConfig) + # FIXME: Hack? + test.source_path = path + yield test + + def createTempInput(self, tmp, test): + abstract + + def execute(self, test, litConfig): + if test.config.unsupported: + return (Test.UNSUPPORTED, 'Test is unsupported') + + cmd = list(self.command) + + # If using temp input, create a temporary file and hand it to the + # subclass. + if self.useTempInput: + tmp = tempfile.NamedTemporaryFile(suffix='.cpp') + self.createTempInput(tmp, test) + tmp.flush() + cmd.append(tmp.name) + elif hasattr(test, 'source_path'): + cmd.append(test.source_path) + else: + cmd.append(test.getSourcePath()) + + out, err, exitCode = TestRunner.executeCommand(cmd) + + diags = out + err + if not exitCode and not diags.strip(): + return Test.PASS,'' + + # Try to include some useful information. + report = """Command: %s\n""" % ' '.join(["'%s'" % a + for a in cmd]) + if self.useTempInput: + report += """Temporary File: %s\n""" % tmp.name + report += "--\n%s--\n""" % open(tmp.name).read() + report += """Output:\n--\n%s--""" % diags + + return Test.FAIL, report + +class SyntaxCheckTest(OneCommandPerFileTest): + def __init__(self, compiler, dir, extra_cxx_args=[], *args, **kwargs): + cmd = [compiler, '-x', 'c++', '-fsyntax-only'] + extra_cxx_args + OneCommandPerFileTest.__init__(self, cmd, dir, + useTempInput=1, *args, **kwargs) + + def createTempInput(self, tmp, test): + print >>tmp, '#include "%s"' % test.source_path diff --git a/Test/lit/lit/TestRunner.py b/Test/lit/lit/TestRunner.py new file mode 100644 index 00000000..3004d2c7 --- /dev/null +++ b/Test/lit/lit/TestRunner.py @@ -0,0 +1,607 @@ +import os, signal, subprocess, sys +import StringIO + +import ShUtil +import Test +import Util + +import platform +import tempfile + +import re + +class InternalShellError(Exception): + def __init__(self, command, message): + self.command = command + self.message = message + +kIsWindows = platform.system() == 'Windows' + +# Don't use close_fds on Windows. +kUseCloseFDs = not kIsWindows + +# Use temporary files to replace /dev/null on Windows. +kAvoidDevNull = kIsWindows + +def executeCommand(command, cwd=None, env=None): + p = subprocess.Popen(command, cwd=cwd, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + env=env) + out,err = p.communicate() + exitCode = p.wait() + + # Detect Ctrl-C in subprocess. + if exitCode == -signal.SIGINT: + raise KeyboardInterrupt + + return out, err, exitCode + +def executeShCmd(cmd, cfg, cwd, results): + if isinstance(cmd, ShUtil.Seq): + if cmd.op == ';': + res = executeShCmd(cmd.lhs, cfg, cwd, results) + return executeShCmd(cmd.rhs, cfg, cwd, results) + + if cmd.op == '&': + raise NotImplementedError,"unsupported test command: '&'" + + if cmd.op == '||': + res = executeShCmd(cmd.lhs, cfg, cwd, results) + if res != 0: + res = executeShCmd(cmd.rhs, cfg, cwd, results) + return res + if cmd.op == '&&': + res = executeShCmd(cmd.lhs, cfg, cwd, results) + if res is None: + return res + + if res == 0: + res = executeShCmd(cmd.rhs, cfg, cwd, results) + return res + + raise ValueError,'Unknown shell command: %r' % cmd.op + + assert isinstance(cmd, ShUtil.Pipeline) + procs = [] + input = subprocess.PIPE + stderrTempFiles = [] + opened_files = [] + named_temp_files = [] + # To avoid deadlock, we use a single stderr stream for piped + # output. This is null until we have seen some output using + # stderr. + for i,j in enumerate(cmd.commands): + # Apply the redirections, we use (N,) as a sentinal to indicate stdin, + # stdout, stderr for N equal to 0, 1, or 2 respectively. Redirects to or + # from a file are represented with a list [file, mode, file-object] + # where file-object is initially None. + redirects = [(0,), (1,), (2,)] + for r in j.redirects: + if r[0] == ('>',2): + redirects[2] = [r[1], 'w', None] + elif r[0] == ('>>',2): + redirects[2] = [r[1], 'a', None] + elif r[0] == ('>&',2) and r[1] in '012': + redirects[2] = redirects[int(r[1])] + elif r[0] == ('>&',) or r[0] == ('&>',): + redirects[1] = redirects[2] = [r[1], 'w', None] + elif r[0] == ('>',): + redirects[1] = [r[1], 'w', None] + elif r[0] == ('>>',): + redirects[1] = [r[1], 'a', None] + elif r[0] == ('<',): + redirects[0] = [r[1], 'r', None] + else: + raise NotImplementedError,"Unsupported redirect: %r" % (r,) + + # Map from the final redirections to something subprocess can handle. + final_redirects = [] + for index,r in enumerate(redirects): + if r == (0,): + result = input + elif r == (1,): + if index == 0: + raise NotImplementedError,"Unsupported redirect for stdin" + elif index == 1: + result = subprocess.PIPE + else: + result = subprocess.STDOUT + elif r == (2,): + if index != 2: + raise NotImplementedError,"Unsupported redirect on stdout" + result = subprocess.PIPE + else: + if r[2] is None: + if kAvoidDevNull and r[0] == '/dev/null': + r[2] = tempfile.TemporaryFile(mode=r[1]) + else: + r[2] = open(r[0], r[1]) + # Workaround a Win32 and/or subprocess bug when appending. + # + # FIXME: Actually, this is probably an instance of PR6753. + if r[1] == 'a': + r[2].seek(0, 2) + opened_files.append(r[2]) + result = r[2] + final_redirects.append(result) + + stdin, stdout, stderr = final_redirects + + # If stderr wants to come from stdout, but stdout isn't a pipe, then put + # stderr on a pipe and treat it as stdout. + if (stderr == subprocess.STDOUT and stdout != subprocess.PIPE): + stderr = subprocess.PIPE + stderrIsStdout = True + else: + stderrIsStdout = False + + # Don't allow stderr on a PIPE except for the last + # process, this could deadlock. + # + # FIXME: This is slow, but so is deadlock. + if stderr == subprocess.PIPE and j != cmd.commands[-1]: + stderr = tempfile.TemporaryFile(mode='w+b') + stderrTempFiles.append((i, stderr)) + + # Resolve the executable path ourselves. + args = list(j.args) + args[0] = Util.which(args[0], cfg.environment['PATH']) + if not args[0]: + raise InternalShellError(j, '%r: command not found' % j.args[0]) + + # Replace uses of /dev/null with temporary files. + if kAvoidDevNull: + for i,arg in enumerate(args): + if arg == "/dev/null": + f = tempfile.NamedTemporaryFile(delete=False) + f.close() + named_temp_files.append(f.name) + args[i] = f.name + + procs.append(subprocess.Popen(args, cwd=cwd, + stdin = stdin, + stdout = stdout, + stderr = stderr, + env = cfg.environment, + close_fds = kUseCloseFDs)) + + # Immediately close stdin for any process taking stdin from us. + if stdin == subprocess.PIPE: + procs[-1].stdin.close() + procs[-1].stdin = None + + # Update the current stdin source. + if stdout == subprocess.PIPE: + input = procs[-1].stdout + elif stderrIsStdout: + input = procs[-1].stderr + else: + input = subprocess.PIPE + + # Explicitly close any redirected files. We need to do this now because we + # need to release any handles we may have on the temporary files (important + # on Win32, for example). Since we have already spawned the subprocess, our + # handles have already been transferred so we do not need them anymore. + for f in opened_files: + f.close() + + # FIXME: There is probably still deadlock potential here. Yawn. + procData = [None] * len(procs) + procData[-1] = procs[-1].communicate() + + for i in range(len(procs) - 1): + if procs[i].stdout is not None: + out = procs[i].stdout.read() + else: + out = '' + if procs[i].stderr is not None: + err = procs[i].stderr.read() + else: + err = '' + procData[i] = (out,err) + + # Read stderr out of the temp files. + for i,f in stderrTempFiles: + f.seek(0, 0) + procData[i] = (procData[i][0], f.read()) + + exitCode = None + for i,(out,err) in enumerate(procData): + res = procs[i].wait() + # Detect Ctrl-C in subprocess. + if res == -signal.SIGINT: + raise KeyboardInterrupt + + results.append((cmd.commands[i], out, err, res)) + if cmd.pipe_err: + # Python treats the exit code as a signed char. + if res < 0: + exitCode = min(exitCode, res) + else: + exitCode = max(exitCode, res) + else: + exitCode = res + + # Remove any named temporary files we created. + for f in named_temp_files: + try: + os.remove(f) + except OSError: + pass + + if cmd.negate: + exitCode = not exitCode + + return exitCode + +def executeScriptInternal(test, litConfig, tmpBase, commands, cwd): + ln = ' &&\n'.join(commands) + try: + cmd = ShUtil.ShParser(ln, litConfig.isWindows).parse() + except: + return (Test.FAIL, "shell parser error on: %r" % ln) + + results = [] + try: + exitCode = executeShCmd(cmd, test.config, cwd, results) + except InternalShellError,e: + out = '' + err = e.message + exitCode = 255 + + out = err = '' + for i,(cmd, cmd_out,cmd_err,res) in enumerate(results): + out += 'Command %d: %s\n' % (i, ' '.join('"%s"' % s for s in cmd.args)) + out += 'Command %d Result: %r\n' % (i, res) + out += 'Command %d Output:\n%s\n\n' % (i, cmd_out) + out += 'Command %d Stderr:\n%s\n\n' % (i, cmd_err) + + return out, err, exitCode + +def executeTclScriptInternal(test, litConfig, tmpBase, commands, cwd): + import TclUtil + cmds = [] + for ln in commands: + # Given the unfortunate way LLVM's test are written, the line gets + # backslash substitution done twice. + ln = TclUtil.TclLexer(ln).lex_unquoted(process_all = True) + + try: + tokens = list(TclUtil.TclLexer(ln).lex()) + except: + return (Test.FAIL, "Tcl lexer error on: %r" % ln) + + # Validate there are no control tokens. + for t in tokens: + if not isinstance(t, str): + return (Test.FAIL, + "Invalid test line: %r containing %r" % (ln, t)) + + try: + cmds.append(TclUtil.TclExecCommand(tokens).parse_pipeline()) + except: + return (Test.FAIL, "Tcl 'exec' parse error on: %r" % ln) + + if litConfig.useValgrind: + for pipeline in cmds: + if pipeline.commands: + # Only valgrind the first command in each pipeline, to avoid + # valgrinding things like grep, not, and FileCheck. + cmd = pipeline.commands[0] + cmd.args = litConfig.valgrindArgs + cmd.args + + cmd = cmds[0] + for c in cmds[1:]: + cmd = ShUtil.Seq(cmd, '&&', c) + + # FIXME: This is lame, we shouldn't need bash. See PR5240. + bashPath = litConfig.getBashPath() + if litConfig.useTclAsSh and bashPath: + script = tmpBase + '.script' + + # Write script file + f = open(script,'w') + print >>f, 'set -o pipefail' + cmd.toShell(f, pipefail = True) + f.close() + + if 0: + print >>sys.stdout, cmd + print >>sys.stdout, open(script).read() + print >>sys.stdout + return '', '', 0 + + command = [litConfig.getBashPath(), script] + out,err,exitCode = executeCommand(command, cwd=cwd, + env=test.config.environment) + + return out,err,exitCode + else: + results = [] + try: + exitCode = executeShCmd(cmd, test.config, cwd, results) + except InternalShellError,e: + results.append((e.command, '', e.message + '\n', 255)) + exitCode = 255 + + out = err = '' + + for i,(cmd, cmd_out, cmd_err, res) in enumerate(results): + out += 'Command %d: %s\n' % (i, ' '.join('"%s"' % s for s in cmd.args)) + out += 'Command %d Result: %r\n' % (i, res) + out += 'Command %d Output:\n%s\n\n' % (i, cmd_out) + out += 'Command %d Stderr:\n%s\n\n' % (i, cmd_err) + + return out, err, exitCode + +def executeScript(test, litConfig, tmpBase, commands, cwd): + bashPath = litConfig.getBashPath(); + isWin32CMDEXE = (litConfig.isWindows and not bashPath) + script = tmpBase + '.script' + if isWin32CMDEXE: + script += '.bat' + + # Write script file + f = open(script,'w') + if isWin32CMDEXE: + f.write('\nif %ERRORLEVEL% NEQ 0 EXIT\n'.join(commands)) + else: + f.write(' &&\n'.join(commands)) + f.write('\n') + f.close() + + if isWin32CMDEXE: + command = ['cmd','/c', script] + else: + if bashPath: + command = [bashPath, script] + else: + command = ['/bin/sh', script] + if litConfig.useValgrind: + # FIXME: Running valgrind on sh is overkill. We probably could just + # run on clang with no real loss. + command = litConfig.valgrindArgs + command + + return executeCommand(command, cwd=cwd, env=test.config.environment) + +def isExpectedFail(xfails, xtargets, target_triple): + # Check if any xfail matches this target. + for item in xfails: + if item == '*' or item in target_triple: + break + else: + return False + + # If so, see if it is expected to pass on this target. + # + # FIXME: Rename XTARGET to something that makes sense, like XPASS. + for item in xtargets: + if item == '*' or item in target_triple: + return False + + return True + +def parseIntegratedTestScript(test, normalize_slashes=False, + extra_substitutions=[]): + """parseIntegratedTestScript - Scan an LLVM/Clang style integrated test + script and extract the lines to 'RUN' as well as 'XFAIL' and 'XTARGET' + information. The RUN lines also will have variable substitution performed. + """ + + # Get the temporary location, this is always relative to the test suite + # root, not test source root. + # + # FIXME: This should not be here? + sourcepath = test.getSourcePath() + sourcedir = os.path.dirname(sourcepath) + execpath = test.getExecPath() + execdir,execbase = os.path.split(execpath) + tmpDir = os.path.join(execdir, 'Output') + tmpBase = os.path.join(tmpDir, execbase) + if test.index is not None: + tmpBase += '_%d' % test.index + + # Normalize slashes, if requested. + if normalize_slashes: + sourcepath = sourcepath.replace('\\', '/') + sourcedir = sourcedir.replace('\\', '/') + tmpDir = tmpDir.replace('\\', '/') + tmpBase = tmpBase.replace('\\', '/') + + # We use #_MARKER_# to hide %% while we do the other substitutions. + substitutions = list(extra_substitutions) + substitutions.extend([('%%', '#_MARKER_#')]) + substitutions.extend(test.config.substitutions) + substitutions.extend([('%s', sourcepath), + ('%S', sourcedir), + ('%p', sourcedir), + ('%{pathsep}', os.pathsep), + ('%t', tmpBase + '.tmp'), + ('%T', tmpDir), + # FIXME: Remove this once we kill DejaGNU. + ('%abs_tmp', tmpBase + '.tmp'), + ('#_MARKER_#', '%')]) + + # Collect the test lines from the script. + script = [] + xfails = [] + xtargets = [] + requires = [] + for ln in open(sourcepath): + if 'RUN:' in ln: + # Isolate the command to run. + index = ln.index('RUN:') + ln = ln[index+4:] + + # Trim trailing whitespace. + ln = ln.rstrip() + + # Collapse lines with trailing '\\'. + if script and script[-1][-1] == '\\': + script[-1] = script[-1][:-1] + ln + else: + script.append(ln) + elif 'XFAIL:' in ln: + items = ln[ln.index('XFAIL:') + 6:].split(',') + xfails.extend([s.strip() for s in items]) + elif 'XTARGET:' in ln: + items = ln[ln.index('XTARGET:') + 8:].split(',') + xtargets.extend([s.strip() for s in items]) + elif 'REQUIRES:' in ln: + items = ln[ln.index('REQUIRES:') + 9:].split(',') + requires.extend([s.strip() for s in items]) + elif 'END.' in ln: + # Check for END. lines. + if ln[ln.index('END.'):].strip() == 'END.': + break + + # Apply substitutions to the script. Allow full regular + # expression syntax. Replace each matching occurrence of regular + # expression pattern a with substitution b in line ln. + def processLine(ln): + # Apply substitutions + for a,b in substitutions: + if kIsWindows: + b = b.replace("\\","\\\\") + ln = re.sub(a, b, ln) + + # Strip the trailing newline and any extra whitespace. + return ln.strip() + script = map(processLine, script) + + # Verify the script contains a run line. + if not script: + return (Test.UNRESOLVED, "Test has no run line!") + + # Check for unterminated run lines. + if script[-1][-1] == '\\': + return (Test.UNRESOLVED, "Test has unterminated run lines (with '\\')") + + # Check that we have the required features: + missing_required_features = [f for f in requires + if f not in test.config.available_features] + if missing_required_features: + msg = ', '.join(missing_required_features) + return (Test.UNSUPPORTED, + "Test requires the following features: %s" % msg) + + isXFail = isExpectedFail(xfails, xtargets, test.suite.config.target_triple) + return script,isXFail,tmpBase,execdir + +def formatTestOutput(status, out, err, exitCode, failDueToStderr, script): + output = StringIO.StringIO() + print >>output, "Script:" + print >>output, "--" + print >>output, '\n'.join(script) + print >>output, "--" + print >>output, "Exit Code: %r" % exitCode, + if failDueToStderr: + print >>output, "(but there was output on stderr)" + else: + print >>output + if out: + print >>output, "Command Output (stdout):" + print >>output, "--" + output.write(out) + print >>output, "--" + if err: + print >>output, "Command Output (stderr):" + print >>output, "--" + output.write(err) + print >>output, "--" + return (status, output.getvalue()) + +def executeTclTest(test, litConfig): + if test.config.unsupported: + return (Test.UNSUPPORTED, 'Test is unsupported') + + # Parse the test script, normalizing slashes in substitutions on Windows + # (since otherwise Tcl style lexing will treat them as escapes). + res = parseIntegratedTestScript(test, normalize_slashes=kIsWindows) + if len(res) == 2: + return res + + script, isXFail, tmpBase, execdir = res + + if litConfig.noExecute: + return (Test.PASS, '') + + # Create the output directory if it does not already exist. + Util.mkdir_p(os.path.dirname(tmpBase)) + + res = executeTclScriptInternal(test, litConfig, tmpBase, script, execdir) + if len(res) == 2: + return res + + # Test for failure. In addition to the exit code, Tcl commands are + # considered to fail if there is any standard error output. + out,err,exitCode = res + if isXFail: + ok = exitCode != 0 or err and not litConfig.ignoreStdErr + if ok: + status = Test.XFAIL + else: + status = Test.XPASS + else: + ok = exitCode == 0 and (not err or litConfig.ignoreStdErr) + if ok: + status = Test.PASS + else: + status = Test.FAIL + + if ok: + return (status,'') + + # Set a flag for formatTestOutput so it can explain why the test was + # considered to have failed, despite having an exit code of 0. + failDueToStderr = exitCode == 0 and err and not litConfig.ignoreStdErr + + return formatTestOutput(status, out, err, exitCode, failDueToStderr, script) + +def executeShTest(test, litConfig, useExternalSh, + extra_substitutions=[]): + if test.config.unsupported: + return (Test.UNSUPPORTED, 'Test is unsupported') + + res = parseIntegratedTestScript(test, useExternalSh, extra_substitutions) + if len(res) == 2: + return res + + script, isXFail, tmpBase, execdir = res + + if litConfig.noExecute: + return (Test.PASS, '') + + # Create the output directory if it does not already exist. + Util.mkdir_p(os.path.dirname(tmpBase)) + + if useExternalSh: + res = executeScript(test, litConfig, tmpBase, script, execdir) + else: + res = executeScriptInternal(test, litConfig, tmpBase, script, execdir) + if len(res) == 2: + return res + + out,err,exitCode = res + if isXFail: + ok = exitCode != 0 + if ok: + status = Test.XFAIL + else: + status = Test.XPASS + else: + ok = exitCode == 0 + if ok: + status = Test.PASS + else: + status = Test.FAIL + + if ok: + return (status,'') + + # Sh tests are not considered to fail just from stderr output. + failDueToStderr = False + + return formatTestOutput(status, out, err, exitCode, failDueToStderr, script) diff --git a/Test/lit/lit/TestingConfig.py b/Test/lit/lit/TestingConfig.py new file mode 100644 index 00000000..223120c4 --- /dev/null +++ b/Test/lit/lit/TestingConfig.py @@ -0,0 +1,125 @@ +import os +import sys + +class TestingConfig: + """" + TestingConfig - Information on the tests inside a suite. + """ + + @staticmethod + def frompath(path, parent, litConfig, mustExist, config = None): + if config is None: + # Set the environment based on the command line arguments. + environment = { + 'LIBRARY_PATH' : os.environ.get('LIBRARY_PATH',''), + 'LD_LIBRARY_PATH' : os.environ.get('LD_LIBRARY_PATH',''), + 'PATH' : os.pathsep.join(litConfig.path + + [os.environ.get('PATH','')]), + 'SYSTEMROOT' : os.environ.get('SYSTEMROOT',''), + 'LLVM_DISABLE_CRASH_REPORT' : '1', + } + + if sys.platform == 'win32': + environment.update({ + 'INCLUDE' : os.environ.get('INCLUDE',''), + 'PATHEXT' : os.environ.get('PATHEXT',''), + 'PYTHONUNBUFFERED' : '1', + 'TEMP' : os.environ.get('TEMP',''), + 'TMP' : os.environ.get('TMP',''), + }) + + config = TestingConfig(parent, + name = '<unnamed>', + suffixes = set(), + test_format = None, + environment = environment, + substitutions = [], + unsupported = False, + on_clone = None, + test_exec_root = None, + test_source_root = None, + excludes = [], + available_features = []) + + if os.path.exists(path): + # FIXME: Improve detection and error reporting of errors in the + # config file. + f = open(path) + cfg_globals = dict(globals()) + cfg_globals['config'] = config + cfg_globals['lit'] = litConfig + cfg_globals['__file__'] = path + try: + exec f in cfg_globals + if litConfig.debug: + litConfig.note('... loaded config %r' % path) + except SystemExit,status: + # We allow normal system exit inside a config file to just + # return control without error. + if status.args: + raise + f.close() + else: + if mustExist: + litConfig.fatal('unable to load config from %r ' % path) + elif litConfig.debug: + litConfig.note('... config not found - %r' %path) + + config.finish(litConfig) + return config + + def __init__(self, parent, name, suffixes, test_format, + environment, substitutions, unsupported, on_clone, + test_exec_root, test_source_root, excludes, + available_features): + self.parent = parent + self.name = str(name) + self.suffixes = set(suffixes) + self.test_format = test_format + self.environment = dict(environment) + self.substitutions = list(substitutions) + self.unsupported = unsupported + self.on_clone = on_clone + self.test_exec_root = test_exec_root + self.test_source_root = test_source_root + self.excludes = set(excludes) + self.available_features = set(available_features) + + def clone(self, path): + # FIXME: Chain implementations? + # + # FIXME: Allow extra parameters? + cfg = TestingConfig(self, self.name, self.suffixes, self.test_format, + self.environment, self.substitutions, + self.unsupported, self.on_clone, + self.test_exec_root, self.test_source_root, + self.excludes, self.available_features) + if cfg.on_clone: + cfg.on_clone(self, cfg, path) + return cfg + + def finish(self, litConfig): + """finish() - Finish this config object, after loading is complete.""" + + self.name = str(self.name) + self.suffixes = set(self.suffixes) + self.environment = dict(self.environment) + self.substitutions = list(self.substitutions) + if self.test_exec_root is not None: + # FIXME: This should really only be suite in test suite config + # files. Should we distinguish them? + self.test_exec_root = str(self.test_exec_root) + if self.test_source_root is not None: + # FIXME: This should really only be suite in test suite config + # files. Should we distinguish them? + self.test_source_root = str(self.test_source_root) + self.excludes = set(self.excludes) + + @property + def root(self): + """root attribute - The root configuration for the test suite.""" + if self.parent is None: + return self + else: + return self.parent.root + diff --git a/Test/lit/lit/Util.py b/Test/lit/lit/Util.py new file mode 100644 index 00000000..226e453f --- /dev/null +++ b/Test/lit/lit/Util.py @@ -0,0 +1,141 @@ +import os, sys + +def detectCPUs(): + """ + Detects the number of CPUs on a system. Cribbed from pp. + """ + # Linux, Unix and MacOS: + if hasattr(os, "sysconf"): + if os.sysconf_names.has_key("SC_NPROCESSORS_ONLN"): + # Linux & Unix: + ncpus = os.sysconf("SC_NPROCESSORS_ONLN") + if isinstance(ncpus, int) and ncpus > 0: + return ncpus + else: # OSX: + return int(capture(['sysctl', '-n', 'hw.ncpu'])) + # Windows: + if os.environ.has_key("NUMBER_OF_PROCESSORS"): + ncpus = int(os.environ["NUMBER_OF_PROCESSORS"]) + if ncpus > 0: + return ncpus + return 1 # Default + +def mkdir_p(path): + """mkdir_p(path) - Make the "path" directory, if it does not exist; this + will also make directories for any missing parent directories.""" + import errno + + if not path or os.path.exists(path): + return + + parent = os.path.dirname(path) + if parent != path: + mkdir_p(parent) + + try: + os.mkdir(path) + except OSError,e: + # Ignore EEXIST, which may occur during a race condition. + if e.errno != errno.EEXIST: + raise + +def capture(args, env=None): + import subprocess + """capture(command) - Run the given command (or argv list) in a shell and + return the standard output.""" + p = subprocess.Popen(args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, + env=env) + out,_ = p.communicate() + return out + +def which(command, paths = None): + """which(command, [paths]) - Look up the given command in the paths string + (or the PATH environment variable, if unspecified).""" + + if paths is None: + paths = os.environ.get('PATH','') + + # Check for absolute match first. + if os.path.exists(command): + return command + + # Would be nice if Python had a lib function for this. + if not paths: + paths = os.defpath + + # Get suffixes to search. + # On Cygwin, 'PATHEXT' may exist but it should not be used. + if os.pathsep == ';': + pathext = os.environ.get('PATHEXT', '').split(';') + else: + pathext = [''] + + # Search the paths... + for path in paths.split(os.pathsep): + for ext in pathext: + p = os.path.join(path, command + ext) + if os.path.exists(p): + return p + + return None + +def checkToolsPath(dir, tools): + for tool in tools: + if not os.path.exists(os.path.join(dir, tool)): + return False; + return True; + +def whichTools(tools, paths): + for path in paths.split(os.pathsep): + if checkToolsPath(path, tools): + return path + return None + +def printHistogram(items, title = 'Items'): + import itertools, math + + items.sort(key = lambda (_,v): v) + + maxValue = max([v for _,v in items]) + + # Select first "nice" bar height that produces more than 10 bars. + power = int(math.ceil(math.log(maxValue, 10))) + for inc in itertools.cycle((5, 2, 2.5, 1)): + barH = inc * 10**power + N = int(math.ceil(maxValue / barH)) + if N > 10: + break + elif inc == 1: + power -= 1 + + histo = [set() for i in range(N)] + for name,v in items: + bin = min(int(N * v/maxValue), N-1) + histo[bin].add(name) + + barW = 40 + hr = '-' * (barW + 34) + print '\nSlowest %s:' % title + print hr + for name,value in items[-20:]: + print '%.2fs: %s' % (value, name) + print '\n%s Times:' % title + print hr + pDigits = int(math.ceil(math.log(maxValue, 10))) + pfDigits = max(0, 3-pDigits) + if pfDigits: + pDigits += pfDigits + 1 + cDigits = int(math.ceil(math.log(len(items), 10))) + print "[%s] :: [%s] :: [%s]" % ('Range'.center((pDigits+1)*2 + 3), + 'Percentage'.center(barW), + 'Count'.center(cDigits*2 + 1)) + print hr + for i,row in enumerate(histo): + pct = float(len(row)) / len(items) + w = int(barW * pct) + print "[%*.*fs,%*.*fs)" % (pDigits, pfDigits, i*barH, + pDigits, pfDigits, (i+1)*barH), + print ":: [%s%s] :: [%*d/%*d]" % ('*'*w, ' '*(barW-w), + cDigits, len(row), + cDigits, len(items)) + diff --git a/Test/lit/lit/__init__.py b/Test/lit/lit/__init__.py new file mode 100644 index 00000000..f3fbb1cd --- /dev/null +++ b/Test/lit/lit/__init__.py @@ -0,0 +1,10 @@ +"""'lit' Testing Tool""" + +from main import main + +__author__ = 'Daniel Dunbar' +__email__ = 'daniel@zuster.org' +__versioninfo__ = (0, 2, 0) +__version__ = '.'.join(map(str, __versioninfo__)) + 'dev' + +__all__ = [] diff --git a/Test/lit/lit/main.py b/Test/lit/lit/main.py new file mode 100644 index 00000000..039868da --- /dev/null +++ b/Test/lit/lit/main.py @@ -0,0 +1,669 @@ +#!/usr/bin/env python + +""" +lit - LLVM Integrated Tester. + +See lit.pod for more information. +""" + +import math, os, platform, random, re, sys, time, threading, traceback + +import ProgressBar +import TestRunner +import Util + +from TestingConfig import TestingConfig +import LitConfig +import Test + +# Configuration files to look for when discovering test suites. These can be +# overridden with --config-prefix. +# +# FIXME: Rename to 'config.lit', 'site.lit', and 'local.lit' ? +gConfigName = 'lit.cfg' +gSiteConfigName = 'lit.site.cfg' + +kLocalConfigName = 'lit.local.cfg' + +class TestingProgressDisplay: + def __init__(self, opts, numTests, progressBar=None): + self.opts = opts + self.numTests = numTests + self.current = None + self.lock = threading.Lock() + self.progressBar = progressBar + self.completed = 0 + + def update(self, test): + # Avoid locking overhead in quiet mode + if self.opts.quiet and not test.result.isFailure: + self.completed += 1 + return + + # Output lock. + self.lock.acquire() + try: + self.handleUpdate(test) + finally: + self.lock.release() + + def finish(self): + if self.progressBar: + self.progressBar.clear() + elif self.opts.quiet: + pass + elif self.opts.succinct: + sys.stdout.write('\n') + + def handleUpdate(self, test): + self.completed += 1 + if self.progressBar: + self.progressBar.update(float(self.completed)/self.numTests, + test.getFullName()) + + if self.opts.succinct and not test.result.isFailure: + return + + if self.progressBar: + self.progressBar.clear() + + print '%s: %s (%d of %d)' % (test.result.name, test.getFullName(), + self.completed, self.numTests) + + if test.result.isFailure and self.opts.showOutput: + print "%s TEST '%s' FAILED %s" % ('*'*20, test.getFullName(), + '*'*20) + print test.output + print "*" * 20 + + sys.stdout.flush() + +class TestProvider: + def __init__(self, tests, maxTime): + self.maxTime = maxTime + self.iter = iter(tests) + self.lock = threading.Lock() + self.startTime = time.time() + + def get(self): + # Check if we have run out of time. + if self.maxTime is not None: + if time.time() - self.startTime > self.maxTime: + return None + + # Otherwise take the next test. + self.lock.acquire() + try: + item = self.iter.next() + except StopIteration: + item = None + self.lock.release() + return item + +class Tester(threading.Thread): + def __init__(self, litConfig, provider, display): + threading.Thread.__init__(self) + self.litConfig = litConfig + self.provider = provider + self.display = display + + def run(self): + while 1: + item = self.provider.get() + if item is None: + break + self.runTest(item) + + def runTest(self, test): + result = None + startTime = time.time() + try: + result, output = test.config.test_format.execute(test, + self.litConfig) + except KeyboardInterrupt: + # This is a sad hack. Unfortunately subprocess goes + # bonkers with ctrl-c and we start forking merrily. + print '\nCtrl-C detected, goodbye.' + os.kill(0,9) + except: + if self.litConfig.debug: + raise + result = Test.UNRESOLVED + output = 'Exception during script execution:\n' + output += traceback.format_exc() + output += '\n' + elapsed = time.time() - startTime + + test.setResult(result, output, elapsed) + self.display.update(test) + +def dirContainsTestSuite(path): + cfgpath = os.path.join(path, gSiteConfigName) + if os.path.exists(cfgpath): + return cfgpath + cfgpath = os.path.join(path, gConfigName) + if os.path.exists(cfgpath): + return cfgpath + +def getTestSuite(item, litConfig, cache): + """getTestSuite(item, litConfig, cache) -> (suite, relative_path) + + Find the test suite containing @arg item. + + @retval (None, ...) - Indicates no test suite contains @arg item. + @retval (suite, relative_path) - The suite that @arg item is in, and its + relative path inside that suite. + """ + def search1(path): + # Check for a site config or a lit config. + cfgpath = dirContainsTestSuite(path) + + # If we didn't find a config file, keep looking. + if not cfgpath: + parent,base = os.path.split(path) + if parent == path: + return (None, ()) + + ts, relative = search(parent) + return (ts, relative + (base,)) + + # We found a config file, load it. + if litConfig.debug: + litConfig.note('loading suite config %r' % cfgpath) + + cfg = TestingConfig.frompath(cfgpath, None, litConfig, mustExist = True) + source_root = os.path.realpath(cfg.test_source_root or path) + exec_root = os.path.realpath(cfg.test_exec_root or path) + return Test.TestSuite(cfg.name, source_root, exec_root, cfg), () + + def search(path): + # Check for an already instantiated test suite. + res = cache.get(path) + if res is None: + cache[path] = res = search1(path) + return res + + # Canonicalize the path. + item = os.path.realpath(item) + + # Skip files and virtual components. + components = [] + while not os.path.isdir(item): + parent,base = os.path.split(item) + if parent == item: + return (None, ()) + components.append(base) + item = parent + components.reverse() + + ts, relative = search(item) + return ts, tuple(relative + tuple(components)) + +def getLocalConfig(ts, path_in_suite, litConfig, cache): + def search1(path_in_suite): + # Get the parent config. + if not path_in_suite: + parent = ts.config + else: + parent = search(path_in_suite[:-1]) + + # Load the local configuration. + source_path = ts.getSourcePath(path_in_suite) + cfgpath = os.path.join(source_path, kLocalConfigName) + if litConfig.debug: + litConfig.note('loading local config %r' % cfgpath) + return TestingConfig.frompath(cfgpath, parent, litConfig, + mustExist = False, + config = parent.clone(cfgpath)) + + def search(path_in_suite): + key = (ts, path_in_suite) + res = cache.get(key) + if res is None: + cache[key] = res = search1(path_in_suite) + return res + + return search(path_in_suite) + +def getTests(path, litConfig, testSuiteCache, localConfigCache): + # Find the test suite for this input and its relative path. + ts,path_in_suite = getTestSuite(path, litConfig, testSuiteCache) + if ts is None: + litConfig.warning('unable to find test suite for %r' % path) + return (),() + + if litConfig.debug: + litConfig.note('resolved input %r to %r::%r' % (path, ts.name, + path_in_suite)) + + return ts, getTestsInSuite(ts, path_in_suite, litConfig, + testSuiteCache, localConfigCache) + +def getTestsInSuite(ts, path_in_suite, litConfig, + testSuiteCache, localConfigCache): + # Check that the source path exists (errors here are reported by the + # caller). + source_path = ts.getSourcePath(path_in_suite) + if not os.path.exists(source_path): + return + + # Check if the user named a test directly. + if not os.path.isdir(source_path): + lc = getLocalConfig(ts, path_in_suite[:-1], litConfig, localConfigCache) + yield Test.Test(ts, path_in_suite, lc) + return + + # Otherwise we have a directory to search for tests, start by getting the + # local configuration. + lc = getLocalConfig(ts, path_in_suite, litConfig, localConfigCache) + + # Search for tests. + if lc.test_format is not None: + for res in lc.test_format.getTestsInDirectory(ts, path_in_suite, + litConfig, lc): + yield res + + # Search subdirectories. + for filename in os.listdir(source_path): + # FIXME: This doesn't belong here? + if filename in ('Output', '.svn') or filename in lc.excludes: + continue + + # Ignore non-directories. + file_sourcepath = os.path.join(source_path, filename) + if not os.path.isdir(file_sourcepath): + continue + + # Check for nested test suites, first in the execpath in case there is a + # site configuration and then in the source path. + file_execpath = ts.getExecPath(path_in_suite + (filename,)) + if dirContainsTestSuite(file_execpath): + sub_ts, subiter = getTests(file_execpath, litConfig, + testSuiteCache, localConfigCache) + elif dirContainsTestSuite(file_sourcepath): + sub_ts, subiter = getTests(file_sourcepath, litConfig, + testSuiteCache, localConfigCache) + else: + # Otherwise, continue loading from inside this test suite. + subiter = getTestsInSuite(ts, path_in_suite + (filename,), + litConfig, testSuiteCache, + localConfigCache) + sub_ts = None + + N = 0 + for res in subiter: + N += 1 + yield res + if sub_ts and not N: + litConfig.warning('test suite %r contained no tests' % sub_ts.name) + +def runTests(numThreads, litConfig, provider, display): + # If only using one testing thread, don't use threads at all; this lets us + # profile, among other things. + if numThreads == 1: + t = Tester(litConfig, provider, display) + t.run() + return + + # Otherwise spin up the testing threads and wait for them to finish. + testers = [Tester(litConfig, provider, display) + for i in range(numThreads)] + for t in testers: + t.start() + try: + for t in testers: + t.join() + except KeyboardInterrupt: + sys.exit(2) + +def load_test_suite(inputs): + import unittest + + # Create the global config object. + litConfig = LitConfig.LitConfig(progname = 'lit', + path = [], + quiet = False, + useValgrind = False, + valgrindLeakCheck = False, + valgrindArgs = [], + useTclAsSh = False, + noExecute = False, + ignoreStdErr = False, + debug = False, + isWindows = (platform.system()=='Windows'), + params = {}) + + # Load the tests from the inputs. + tests = [] + testSuiteCache = {} + localConfigCache = {} + for input in inputs: + prev = len(tests) + tests.extend(getTests(input, litConfig, + testSuiteCache, localConfigCache)[1]) + if prev == len(tests): + litConfig.warning('input %r contained no tests' % input) + + # If there were any errors during test discovery, exit now. + if litConfig.numErrors: + print >>sys.stderr, '%d errors, exiting.' % litConfig.numErrors + sys.exit(2) + + # Return a unittest test suite which just runs the tests in order. + def get_test_fn(test): + return unittest.FunctionTestCase( + lambda: test.config.test_format.execute( + test, litConfig), + description = test.getFullName()) + + from LitTestCase import LitTestCase + return unittest.TestSuite([LitTestCase(test, litConfig) for test in tests]) + +def main(builtinParameters = {}): # Bump the GIL check interval, its more important to get any one thread to a + # blocking operation (hopefully exec) than to try and unblock other threads. + # + # FIXME: This is a hack. + import sys + sys.setcheckinterval(1000) + + global options + from optparse import OptionParser, OptionGroup + parser = OptionParser("usage: %prog [options] {file-or-path}") + + parser.add_option("-j", "--threads", dest="numThreads", metavar="N", + help="Number of testing threads", + type=int, action="store", default=None) + parser.add_option("", "--config-prefix", dest="configPrefix", + metavar="NAME", help="Prefix for 'lit' config files", + action="store", default=None) + parser.add_option("", "--param", dest="userParameters", + metavar="NAME=VAL", + help="Add 'NAME' = 'VAL' to the user defined parameters", + type=str, action="append", default=[]) + + group = OptionGroup(parser, "Output Format") + # FIXME: I find these names very confusing, although I like the + # functionality. + group.add_option("-q", "--quiet", dest="quiet", + help="Suppress no error output", + action="store_true", default=False) + group.add_option("-s", "--succinct", dest="succinct", + help="Reduce amount of output", + action="store_true", default=False) + group.add_option("-v", "--verbose", dest="showOutput", + help="Show all test output", + action="store_true", default=False) + group.add_option("", "--no-progress-bar", dest="useProgressBar", + help="Do not use curses based progress bar", + action="store_false", default=True) + parser.add_option_group(group) + + group = OptionGroup(parser, "Test Execution") + group.add_option("", "--path", dest="path", + help="Additional paths to add to testing environment", + action="append", type=str, default=[]) + group.add_option("", "--vg", dest="useValgrind", + help="Run tests under valgrind", + action="store_true", default=False) + group.add_option("", "--vg-leak", dest="valgrindLeakCheck", + help="Check for memory leaks under valgrind", + action="store_true", default=False) + group.add_option("", "--vg-arg", dest="valgrindArgs", metavar="ARG", + help="Specify an extra argument for valgrind", + type=str, action="append", default=[]) + group.add_option("", "--time-tests", dest="timeTests", + help="Track elapsed wall time for each test", + action="store_true", default=False) + group.add_option("", "--no-execute", dest="noExecute", + help="Don't execute any tests (assume PASS)", + action="store_true", default=False) + parser.add_option_group(group) + + group = OptionGroup(parser, "Test Selection") + group.add_option("", "--max-tests", dest="maxTests", metavar="N", + help="Maximum number of tests to run", + action="store", type=int, default=None) + group.add_option("", "--max-time", dest="maxTime", metavar="N", + help="Maximum time to spend testing (in seconds)", + action="store", type=float, default=None) + group.add_option("", "--shuffle", dest="shuffle", + help="Run tests in random order", + action="store_true", default=False) + group.add_option("", "--filter", dest="filter", metavar="EXPRESSION", + help=("Only run tests with paths matching the given " + "regular expression"), + action="store", default=None) + parser.add_option_group(group) + + group = OptionGroup(parser, "Debug and Experimental Options") + group.add_option("", "--debug", dest="debug", + help="Enable debugging (for 'lit' development)", + action="store_true", default=False) + group.add_option("", "--show-suites", dest="showSuites", + help="Show discovered test suites", + action="store_true", default=False) + group.add_option("", "--no-tcl-as-sh", dest="useTclAsSh", + help="Don't run Tcl scripts using 'sh'", + action="store_false", default=True) + group.add_option("", "--repeat", dest="repeatTests", metavar="N", + help="Repeat tests N times (for timing)", + action="store", default=None, type=int) + parser.add_option_group(group) + + (opts, args) = parser.parse_args() + + if not args: + parser.error('No inputs specified') + + if opts.configPrefix is not None: + global gConfigName, gSiteConfigName, kLocalConfigName + gConfigName = '%s.cfg' % opts.configPrefix + gSiteConfigName = '%s.site.cfg' % opts.configPrefix + kLocalConfigName = '%s.local.cfg' % opts.configPrefix + + if opts.numThreads is None: +# Python <2.5 has a race condition causing lit to always fail with numThreads>1 +# http://bugs.python.org/issue1731717 +# I haven't seen this bug occur with 2.5.2 and later, so only enable multiple +# threads by default there. + if sys.hexversion >= 0x2050200: + opts.numThreads = Util.detectCPUs() + else: + opts.numThreads = 1 + + inputs = args + + # Create the user defined parameters. + userParams = dict(builtinParameters) + for entry in opts.userParameters: + if '=' not in entry: + name,val = entry,'' + else: + name,val = entry.split('=', 1) + userParams[name] = val + + # Create the global config object. + litConfig = LitConfig.LitConfig(progname = os.path.basename(sys.argv[0]), + path = opts.path, + quiet = opts.quiet, + useValgrind = opts.useValgrind, + valgrindLeakCheck = opts.valgrindLeakCheck, + valgrindArgs = opts.valgrindArgs, + useTclAsSh = opts.useTclAsSh, + noExecute = opts.noExecute, + ignoreStdErr = False, + debug = opts.debug, + isWindows = (platform.system()=='Windows'), + params = userParams) + + # Expand '@...' form in inputs. + actual_inputs = [] + for input in inputs: + if os.path.exists(input) or not input.startswith('@'): + actual_inputs.append(input) + else: + f = open(input[1:]) + try: + for ln in f: + ln = ln.strip() + if ln: + actual_inputs.append(ln) + finally: + f.close() + + + # Load the tests from the inputs. + tests = [] + testSuiteCache = {} + localConfigCache = {} + for input in actual_inputs: + prev = len(tests) + tests.extend(getTests(input, litConfig, + testSuiteCache, localConfigCache)[1]) + if prev == len(tests): + litConfig.warning('input %r contained no tests' % input) + + # If there were any errors during test discovery, exit now. + if litConfig.numErrors: + print >>sys.stderr, '%d errors, exiting.' % litConfig.numErrors + sys.exit(2) + + if opts.showSuites: + suitesAndTests = dict([(ts,[]) + for ts,_ in testSuiteCache.values() + if ts]) + for t in tests: + suitesAndTests[t.suite].append(t) + + print '-- Test Suites --' + suitesAndTests = suitesAndTests.items() + suitesAndTests.sort(key = lambda (ts,_): ts.name) + for ts,ts_tests in suitesAndTests: + print ' %s - %d tests' %(ts.name, len(ts_tests)) + print ' Source Root: %s' % ts.source_root + print ' Exec Root : %s' % ts.exec_root + + # Select and order the tests. + numTotalTests = len(tests) + + # First, select based on the filter expression if given. + if opts.filter: + try: + rex = re.compile(opts.filter) + except: + parser.error("invalid regular expression for --filter: %r" % ( + opts.filter)) + tests = [t for t in tests + if rex.search(t.getFullName())] + + # Then select the order. + if opts.shuffle: + random.shuffle(tests) + else: + tests.sort(key = lambda t: t.getFullName()) + + # Finally limit the number of tests, if desired. + if opts.maxTests is not None: + tests = tests[:opts.maxTests] + + extra = '' + if len(tests) != numTotalTests: + extra = ' of %d' % numTotalTests + header = '-- Testing: %d%s tests, %d threads --'%(len(tests),extra, + opts.numThreads) + + if opts.repeatTests: + tests = [t.copyWithIndex(i) + for t in tests + for i in range(opts.repeatTests)] + + progressBar = None + if not opts.quiet: + if opts.succinct and opts.useProgressBar: + try: + tc = ProgressBar.TerminalController() + progressBar = ProgressBar.ProgressBar(tc, header) + except ValueError: + print header + progressBar = ProgressBar.SimpleProgressBar('Testing: ') + else: + print header + + # Don't create more threads than tests. + opts.numThreads = min(len(tests), opts.numThreads) + + startTime = time.time() + display = TestingProgressDisplay(opts, len(tests), progressBar) + provider = TestProvider(tests, opts.maxTime) + runTests(opts.numThreads, litConfig, provider, display) + display.finish() + + if not opts.quiet: + print 'Testing Time: %.2fs'%(time.time() - startTime) + + # Update results for any tests which weren't run. + for t in tests: + if t.result is None: + t.setResult(Test.UNRESOLVED, '', 0.0) + + # List test results organized by kind. + hasFailures = False + byCode = {} + for t in tests: + if t.result not in byCode: + byCode[t.result] = [] + byCode[t.result].append(t) + if t.result.isFailure: + hasFailures = True + + # FIXME: Show unresolved and (optionally) unsupported tests. + for title,code in (('Unexpected Passing Tests', Test.XPASS), + ('Failing Tests', Test.FAIL)): + elts = byCode.get(code) + if not elts: + continue + print '*'*20 + print '%s (%d):' % (title, len(elts)) + for t in elts: + print ' %s' % t.getFullName() + print + + if opts.timeTests: + # Collate, in case we repeated tests. + times = {} + for t in tests: + key = t.getFullName() + times[key] = times.get(key, 0.) + t.elapsed + + byTime = list(times.items()) + byTime.sort(key = lambda (name,elapsed): elapsed) + if byTime: + Util.printHistogram(byTime, title='Tests') + + for name,code in (('Expected Passes ', Test.PASS), + ('Expected Failures ', Test.XFAIL), + ('Unsupported Tests ', Test.UNSUPPORTED), + ('Unresolved Tests ', Test.UNRESOLVED), + ('Unexpected Passes ', Test.XPASS), + ('Unexpected Failures', Test.FAIL),): + if opts.quiet and not code.isFailure: + continue + N = len(byCode.get(code,[])) + if N: + print ' %s: %d' % (name,N) + + # If we encountered any additional errors, exit abnormally. + if litConfig.numErrors: + print >>sys.stderr, '\n%d error(s), exiting.' % litConfig.numErrors + sys.exit(2) + + # Warn about warnings. + if litConfig.numWarnings: + print >>sys.stderr, '\n%d warning(s) in tests.' % litConfig.numWarnings + + if hasFailures: + sys.exit(1) + sys.exit(0) + +if __name__=='__main__': + main() diff --git a/Test/lit/setup.py b/Test/lit/setup.py new file mode 100644 index 00000000..a94e6ea8 --- /dev/null +++ b/Test/lit/setup.py @@ -0,0 +1,70 @@ +import lit + +# FIXME: Support distutils? +from setuptools import setup, find_packages +setup( + name = "lit", + version = lit.__version__, + + author = lit.__author__, + author_email = lit.__email__, + url = 'http://llvm.org', + license = 'BSD', + + description = "A Software Testing Tool", + keywords = 'test C++ automatic discovery', + long_description = """\ +*lit* ++++++ + +About +===== + +*lit* is a portable tool for executing LLVM and Clang style test suites, +summarizing their results, and providing indication of failures. *lit* is +designed to be a lightweight testing tool with as simple a user interface as +possible. + + +Features +======== + + * Portable! + * Flexible test discovery. + * Parallel test execution. + * Support for multiple test formats and test suite designs. + + +Documentation +============= + +The official *lit* documentation is in the man page, available online at the LLVM +Command Guide: http://llvm.org/cmds/lit.html. + + +Source +====== + +The *lit* source is available as part of LLVM, in the LLVM SVN repository: +http://llvm.org/svn/llvm-project/llvm/trunk/utils/lit. +""", + + classifiers=[ + 'Development Status :: 3 - Alpha', + 'Environment :: Console', + 'Intended Audience :: Developers', + 'License :: OSI Approved :: University of Illinois/NCSA Open Source License', + 'Natural Language :: English', + 'Operating System :: OS Independent', + 'Programming Language :: Python', + 'Topic :: Software Development :: Testing', + ], + + zip_safe = False, + packages = find_packages(), + entry_points = { + 'console_scripts': [ + 'lit = lit:main', + ], + } +) |