diff options
Diffstat (limited to 'objectivec/DevTools')
| -rwxr-xr-x | objectivec/DevTools/check_version_stamps.sh | 56 | ||||
| -rwxr-xr-x | objectivec/DevTools/generate_descriptors_proto.sh | 36 | ||||
| -rwxr-xr-x | objectivec/DevTools/pddm.py | 687 | ||||
| -rwxr-xr-x | objectivec/DevTools/pddm_tests.py | 515 |
4 files changed, 1294 insertions, 0 deletions
diff --git a/objectivec/DevTools/check_version_stamps.sh b/objectivec/DevTools/check_version_stamps.sh new file mode 100755 index 00000000..5de9ef14 --- /dev/null +++ b/objectivec/DevTools/check_version_stamps.sh @@ -0,0 +1,56 @@ +#!/bin/bash + +# This script checks that the runtime version number constant in the compiler +# source and in the runtime source is the same. +# +# A distro can be made of the protobuf sources with only a subset of the +# languages, so if the compiler depended on the Objective C runtime, those +# builds would break. At the same time, we don't want the runtime source +# depending on the compiler sources; so two copies of the constant are needed. + +set -eu + +readonly ScriptDir=$(dirname "$(echo $0 | sed -e "s,^\([^/]\),$(pwd)/\1,")") +readonly ProtoRootDir="${ScriptDir}/../.." + +die() { + echo "Error: $1" + exit 1 +} + +readonly ConstantName=GOOGLE_PROTOBUF_OBJC_GEN_VERSION + +# Collect version from plugin sources. + +readonly PluginSrc="${ProtoRootDir}/src/google/protobuf/compiler/objectivec/objectivec_file.cc" +readonly PluginVersion=$( \ + cat "${PluginSrc}" \ + | sed -n -e "s:const int32_t ${ConstantName} = \([0-9]*\);:\1:p" +) + +if [[ -z "${PluginVersion}" ]] ; then + die "Failed to fine ${ConstantName} in the plugin source (${PluginSrc})." +fi + +# Collect version from runtime sources. + +readonly RuntimeSrc="${ProtoRootDir}/objectivec/GPBBootstrap.h" +readonly RuntimeVersion=$( \ + cat "${RuntimeSrc}" \ + | sed -n -e "s:#define ${ConstantName} \([0-9]*\):\1:p" +) + +if [[ -z "${RuntimeVersion}" ]] ; then + die "Failed to fine ${ConstantName} in the runtime source (${RuntimeSrc})." +fi + +# Compare them. + +if [[ "${PluginVersion}" != "${RuntimeVersion}" ]] ; then + die "Versions don't match! + Plugin: ${PluginVersion} from ${PluginSrc} + Runtime: ${RuntimeVersion} from ${RuntimeSrc} +" +fi + +# Success diff --git a/objectivec/DevTools/generate_descriptors_proto.sh b/objectivec/DevTools/generate_descriptors_proto.sh new file mode 100755 index 00000000..42502bfe --- /dev/null +++ b/objectivec/DevTools/generate_descriptors_proto.sh @@ -0,0 +1,36 @@ +#!/bin/bash + +# This script will generate the common descriptors needed by the Objective C +# runtime. + +# HINT: Flags passed to generate_descriptor_proto.sh will be passed directly +# to make when building protoc. This is particularly useful for passing +# -j4 to run 4 jobs simultaneously. + +set -eu + +readonly ScriptDir=$(dirname "$(echo $0 | sed -e "s,^\([^/]\),$(pwd)/\1,")") +readonly ProtoRootDir="${ScriptDir}/../.." +readonly ProtoC="${ProtoRootDir}/src/protoc" + +pushd "${ProtoRootDir}" > /dev/null + +# Compiler build fails if config.h hasn't been made yet (even if configure/etc. +# have been run, so get that made first). +make $@ config.h + +# Make sure the compiler is current. +cd src +make $@ protoc + +# These really should only be run when the inputs or compiler are newer than +# the outputs. + +# Needed by the runtime. +./protoc --objc_out="${ProtoRootDir}/objectivec" google/protobuf/descriptor.proto + +# Well known types that the library provides helpers for. +./protoc --objc_out="${ProtoRootDir}/objectivec" google/protobuf/timestamp.proto +./protoc --objc_out="${ProtoRootDir}/objectivec" google/protobuf/duration.proto + +popd > /dev/null diff --git a/objectivec/DevTools/pddm.py b/objectivec/DevTools/pddm.py new file mode 100755 index 00000000..d1b53f5a --- /dev/null +++ b/objectivec/DevTools/pddm.py @@ -0,0 +1,687 @@ +#! /usr/bin/python +# +# Protocol Buffers - Google's data interchange format +# Copyright 2015 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""PDDM - Poor Developers' Debug-able Macros + +A simple markup that can be added in comments of source so they can then be +expanded out into code. Most of this could be done with CPP macros, but then +developers can't really step through them in the debugger, this way they are +expanded to the same code, but you can debug them. + +Any file can be processed, but the syntax is designed around a C based compiler. +Processed lines start with "//%". There are three types of sections you can +create: Text (left alone), Macro Definitions, and Macro Expansions. There is +no order required between definitions and expansions, all definitions are read +before any expansions are processed (thus, if desired, definitions can be put +at the end of the file to keep them out of the way of the code). + +Macro Definitions are started with "//%PDDM-DEFINE Name(args)" and all lines +afterwards that start with "//%" are included in the definition. Multiple +macros can be defined in one block by just using a another "//%PDDM-DEFINE" +line to start the next macro. Optionally, a macro can be ended with +"//%PDDM-DEFINE-END", this can be useful when you want to make it clear that +trailing blank lines are included in the macro. You can also end a definition +with an expansion. + +Macro Expansions are started by single lines containing +"//%PDDM-EXPAND Name(args)" and then with "//%PDDM-EXPAND-END" or another +expansions. All lines in-between are replaced by the result of the expansion. +The first line of the expansion is always a blank like just for readability. + +Expansion itself is pretty simple, one macro can invoke another macro, but +you cannot nest the invoke of a macro in another macro (i.e. - can't do +"foo(bar(a))", but you can define foo(a) and bar(b) where bar invokes foo() +within its expansion. + +When macros are expanded, the arg references can also add "$O" suffix to the +name (i.e. - "NAME$O") to specify an option to be applied. The options are: + + $S - Replace each character in the value with a space. + $l - Lowercase the first letter of the value. + $L - Lowercase the whole value. + $u - Uppercase the first letter of the value. + $U - Uppercase the whole value. + +Within a macro you can use ## to cause things to get joined together after +expansion (i.e. - "a##b" within a macro will become "ab"). + +Example: + + int foo(MyEnum x) { + switch (x) { + //%PDDM-EXPAND case(Enum_Left, 1) + //%PDDM-EXPAND case(Enum_Center, 2) + //%PDDM-EXPAND case(Enum_Right, 3) + //%PDDM-EXPAND-END + } + + //%PDDM-DEFINE case(_A, _B) + //% case _A: + //% return _B; + + A macro ends at the start of the next one, or an optional %PDDM-DEFINE-END + can be used to avoid adding extra blank lines/returns (or make it clear when + it is desired). + + One macro can invoke another by simply using its name NAME(ARGS). You cannot + nest an invoke inside another (i.e. - NAME1(NAME2(ARGS)) isn't supported). + + Within a macro you can use ## to cause things to get joined together after + processing (i.e. - "a##b" within a macro will become "ab"). + + +""" + +import optparse +import os +import re +import sys + + +# Regex for macro definition. +_MACRO_RE = re.compile(r'(?P<name>\w+)\((?P<args>.*?)\)') +# Regex for macro's argument definition. +_MACRO_ARG_NAME_RE = re.compile(r'^\w+$') + +# Line inserted after each EXPAND. +_GENERATED_CODE_LINE = ( + '// This block of code is generated, do not edit it directly.' +) + + +def _MacroRefRe(macro_names): + # Takes in a list of macro names and makes a regex that will match invokes + # of those macros. + return re.compile(r'\b(?P<macro_ref>(?P<name>(%s))\((?P<args>.*?)\))' % + '|'.join(macro_names)) + +def _MacroArgRefRe(macro_arg_names): + # Takes in a list of macro arg names and makes a regex that will match + # uses of those args. + return re.compile(r'\b(?P<name>(%s))(\$(?P<option>.))?\b' % + '|'.join(macro_arg_names)) + + +class PDDMError(Exception): + """Error thrown by pddm.""" + pass + + +class MacroCollection(object): + """Hold a set of macros and can resolve/expand them.""" + + def __init__(self, a_file=None): + """Initializes the collection. + + Args: + a_file: The file like stream to parse. + + Raises: + PDDMError if there are any issues. + """ + self._macros = dict() + if a_file: + self.ParseInput(a_file) + + class MacroDefinition(object): + """Holds a macro definition.""" + + def __init__(self, name, arg_names): + self._name = name + self._args = tuple(arg_names) + self._body = '' + self._needNewLine = False + + def AppendLine(self, line): + if self._needNewLine: + self._body += '\n' + self._body += line + self._needNewLine = not line.endswith('\n') + + @property + def name(self): + return self._name + + @property + def args(self): + return self._args + + @property + def body(self): + return self._body + + def ParseInput(self, a_file): + """Consumes input extracting definitions. + + Args: + a_file: The file like stream to parse. + + Raises: + PDDMError if there are any issues. + """ + input_lines = a_file.read().splitlines() + self.ParseLines(input_lines) + + def ParseLines(self, input_lines): + """Parses list of lines. + + Args: + input_lines: A list of strings of input to parse (no newlines on the + strings). + + Raises: + PDDMError if there are any issues. + """ + current_macro = None + for line in input_lines: + if line.startswith('PDDM-'): + directive = line.split(' ', 1)[0] + if directive == 'PDDM-DEFINE': + name, args = self._ParseDefineLine(line) + if self._macros.get(name): + raise PDDMError('Attempt to redefine macro: "%s"' % line) + current_macro = self.MacroDefinition(name, args) + self._macros[name] = current_macro + continue + if directive == 'PDDM-DEFINE-END': + if not current_macro: + raise PDDMError('Got DEFINE-END directive without an active macro:' + ' "%s"' % line) + current_macro = None + continue + raise PDDMError('Hit a line with an unknown directive: "%s"' % line) + + if current_macro: + current_macro.AppendLine(line) + continue + + # Allow blank lines between macro definitions. + if line.strip() == '': + continue + + raise PDDMError('Hit a line that wasn\'t a directive and no open macro' + ' definition: "%s"' % line) + + def _ParseDefineLine(self, input_line): + assert input_line.startswith('PDDM-DEFINE') + line = input_line[12:].strip() + match = _MACRO_RE.match(line) + # Must match full line + if match is None or match.group(0) != line: + raise PDDMError('Failed to parse macro definition: "%s"' % input_line) + name = match.group('name') + args_str = match.group('args').strip() + args = [] + if args_str: + for part in args_str.split(','): + arg = part.strip() + if arg == '': + raise PDDMError('Empty arg name in macro definition: "%s"' + % input_line) + if not _MACRO_ARG_NAME_RE.match(arg): + raise PDDMError('Invalid arg name "%s" in macro definition: "%s"' + % (arg, input_line)) + if arg in args: + raise PDDMError('Arg name "%s" used more than once in macro' + ' definition: "%s"' % (arg, input_line)) + args.append(arg) + return (name, tuple(args)) + + def Expand(self, macro_ref_str): + """Expands the macro reference. + + Args: + macro_ref_str: String of a macro reference (i.e. foo(a, b)). + + Returns: + The text from the expansion. + + Raises: + PDDMError if there are any issues. + """ + match = _MACRO_RE.match(macro_ref_str) + if match is None or match.group(0) != macro_ref_str: + raise PDDMError('Failed to parse macro reference: "%s"' % macro_ref_str) + if match.group('name') not in self._macros: + raise PDDMError('No macro named "%s".' % match.group('name')) + return self._Expand(match, [], macro_ref_str) + + def _FormatStack(self, macro_ref_stack): + result = '' + for _, macro_ref in reversed(macro_ref_stack): + result += '\n...while expanding "%s".' % macro_ref + return result + + def _Expand(self, macro_ref_match, macro_stack, macro_ref_str=None): + if macro_ref_str is None: + macro_ref_str = macro_ref_match.group('macro_ref') + name = macro_ref_match.group('name') + for prev_name, prev_macro_ref in macro_stack: + if name == prev_name: + raise PDDMError('Found macro recusion, invoking "%s":%s' % + (macro_ref_str, self._FormatStack(macro_stack))) + macro = self._macros[name] + args_str = macro_ref_match.group('args').strip() + args = [] + if args_str or len(macro.args): + args = [x.strip() for x in args_str.split(',')] + if len(args) != len(macro.args): + raise PDDMError('Expected %d args, got: "%s".%s' % + (len(macro.args), macro_ref_str, + self._FormatStack(macro_stack))) + # Replace args usages. + result = self._ReplaceArgValues(macro, args, macro_ref_str, macro_stack) + # Expand any macro invokes. + new_macro_stack = macro_stack + [(name, macro_ref_str)] + while True: + eval_result = self._EvalMacrosRefs(result, new_macro_stack) + # Consume all ## directives to glue things together. + eval_result = eval_result.replace('##', '') + if eval_result == result: + break + result = eval_result + return result + + def _ReplaceArgValues(self, + macro, arg_values, macro_ref_to_report, macro_stack): + if len(arg_values) == 0: + # Nothing to do + return macro.body + assert len(arg_values) == len(macro.args) + args = dict(zip(macro.args, arg_values)) + def _lookupArg(match): + val = args[match.group('name')] + opt = match.group('option') + if opt: + if opt == 'S': # Spaces for the length + return ' ' * len(val) + elif opt == 'l': # Lowercase first character + if val: + return val[0].lower() + val[1:] + else: + return val + elif opt == 'L': # All Lowercase + return val.lower() + elif opt == 'u': # Uppercase first character + if val: + return val[0].upper() + val[1:] + else: + return val + elif opt == 'U': # All Uppercase + return val.upper() + else: + raise PDDMError('Unknown arg option "%s$%s" while expanding "%s".%s' + % (match.group('name'), match.group('option'), + macro_ref_to_report, + self._FormatStack(macro_stack))) + return val + # Let the regex do the work! + macro_arg_ref_re = _MacroArgRefRe(macro.args) + return macro_arg_ref_re.sub(_lookupArg, macro.body) + + def _EvalMacrosRefs(self, text, macro_stack): + macro_ref_re = _MacroRefRe(self._macros.keys()) + def _resolveMacro(match): + return self._Expand(match, macro_stack) + return macro_ref_re.sub(_resolveMacro, text) + + +class SourceFile(object): + """Represents a source file with PDDM directives in it.""" + + def __init__(self, a_file, import_resolver=None): + """Initializes the file reading in the file. + + Args: + a_file: The file to read in. + import_resolver: a function that given a path will return a stream for + the contents. + + Raises: + PDDMError if there are any issues. + """ + self._sections = [] + self._original_content = a_file.read() + self._import_resolver = import_resolver + self._processed_content = None + + class SectionBase(object): + + def __init__(self, first_line_num): + self._lines = [] + self._first_line_num = first_line_num + + def TryAppend(self, line, line_num): + """Try appending a line. + + Args: + line: The line to append. + line_num: The number of the line. + + Returns: + A tuple of (SUCCESS, CAN_ADD_MORE). If SUCCESS if False, the line + wasn't append. If SUCCESS is True, then CAN_ADD_MORE is True/False to + indicate if more lines can be added after this one. + """ + assert False, "sublcass should have overridden" + return (False, False) + + def HitEOF(self): + """Called when the EOF was reached for for a given section.""" + pass + + def BindMacroCollection(self, macro_collection): + """Binds the chunk to a macro collection. + + Args: + macro_collection: The collection to bind too. + """ + pass + + def Append(self, line): + self._lines.append(line) + + @property + def lines(self): + return self._lines + + @property + def num_lines_captured(self): + return len(self._lines) + + @property + def first_line_num(self): + return self._first_line_num + + @property + def first_line(self): + if not self._lines: + return '' + return self._lines[0] + + @property + def text(self): + return '\n'.join(self.lines) + '\n' + + class TextSection(SectionBase): + """Text section that is echoed out as is.""" + + def TryAppend(self, line, line_num): + if line.startswith('//%PDDM'): + return (False, False) + self.Append(line) + return (True, True) + + class ExpansionSection(SectionBase): + """Section that is the result of an macro expansion.""" + + def __init__(self, first_line_num): + SourceFile.SectionBase.__init__(self, first_line_num) + self._macro_collection = None + + def TryAppend(self, line, line_num): + if line.startswith('//%PDDM'): + directive = line.split(' ', 1)[0] + if directive == '//%PDDM-EXPAND': + self.Append(line) + return (True, True) + if directive == '//%PDDM-EXPAND-END': + assert self.num_lines_captured > 0 + return (True, False) + raise PDDMError('Ran into directive ("%s", line %d) while in "%s".' % + (directive, line_num, self.first_line)) + # Eat other lines. + return (True, True) + + def HitEOF(self): + raise PDDMError('Hit the end of the file while in "%s".' % + self.first_line) + + def BindMacroCollection(self, macro_collection): + self._macro_collection = macro_collection + + @property + def lines(self): + captured_lines = SourceFile.SectionBase.lines.fget(self) + directive_len = len('//%PDDM-EXPAND') + result = [] + for line in captured_lines: + result.append(line) + if self._macro_collection: + # Always add a blank line, seems to read better. (If need be, add an + # option to the EXPAND to indicate if this should be done.) + result.extend([_GENERATED_CODE_LINE, '']) + macro = line[directive_len:].strip() + try: + expand_result = self._macro_collection.Expand(macro) + # Since expansions are line oriented, strip trailing whitespace + # from the lines. + lines = [x.rstrip() for x in expand_result.split('\n')] + result.append('\n'.join(lines)) + except PDDMError as e: + raise PDDMError('%s\n...while expanding "%s" from the section' + ' that started:\n Line %d: %s' % + (e.message, macro, + self.first_line_num, self.first_line)) + + # Add the ending marker. + if len(captured_lines) == 1: + result.append('//%%PDDM-EXPAND-END %s' % + captured_lines[0][directive_len:].strip()) + else: + result.append('//%%PDDM-EXPAND-END (%s expansions)' % len(captured_lines)) + + return result + + class DefinitionSection(SectionBase): + """Section containing macro definitions""" + + def TryAppend(self, line, line_num): + if not line.startswith('//%'): + return (False, False) + if line.startswith('//%PDDM'): + directive = line.split(' ', 1)[0] + if directive == "//%PDDM-EXPAND": + return False, False + if directive not in ('//%PDDM-DEFINE', '//%PDDM-DEFINE-END'): + raise PDDMError('Ran into directive ("%s", line %d) while in "%s".' % + (directive, line_num, self.first_line)) + self.Append(line) + return (True, True) + + def BindMacroCollection(self, macro_collection): + if macro_collection: + try: + # Parse the lines after stripping the prefix. + macro_collection.ParseLines([x[3:] for x in self.lines]) + except PDDMError as e: + raise PDDMError('%s\n...while parsing section that started:\n' + ' Line %d: %s' % + (e.message, self.first_line_num, self.first_line)) + + class ImportDefinesSection(SectionBase): + """Section containing an import of PDDM-DEFINES from an external file.""" + + def __init__(self, first_line_num, import_resolver): + SourceFile.SectionBase.__init__(self, first_line_num) + self._import_resolver = import_resolver + + def TryAppend(self, line, line_num): + if not line.startswith('//%PDDM-IMPORT-DEFINES '): + return (False, False) + assert self.num_lines_captured == 0 + self.Append(line) + return (True, False) + + def BindMacroCollection(self, macro_colletion): + if not macro_colletion: + return + if self._import_resolver is None: + raise PDDMError('Got an IMPORT-DEFINES without a resolver (line %d):' + ' "%s".' % (self.first_line_num, self.first_line)) + import_name = self.first_line.split(' ', 1)[1].strip() + imported_file = self._import_resolver(import_name) + if imported_file is None: + raise PDDMError('Resolver failed to find "%s" (line %d):' + ' "%s".' % + (import_name, self.first_line_num, self.first_line)) + try: + imported_src_file = SourceFile(imported_file, self._import_resolver) + imported_src_file._ParseFile() + for section in imported_src_file._sections: + section.BindMacroCollection(macro_colletion) + except PDDMError as e: + raise PDDMError('%s\n...while importing defines:\n' + ' Line %d: %s' % + (e.message, self.first_line_num, self.first_line)) + + def _ParseFile(self): + self._sections = [] + lines = self._original_content.splitlines() + cur_section = None + for line_num, line in enumerate(lines, 1): + if not cur_section: + cur_section = self._MakeSection(line, line_num) + was_added, accept_more = cur_section.TryAppend(line, line_num) + if not was_added: + cur_section = self._MakeSection(line, line_num) + was_added, accept_more = cur_section.TryAppend(line, line_num) + assert was_added + if not accept_more: + cur_section = None + + if cur_section: + cur_section.HitEOF() + + def _MakeSection(self, line, line_num): + if not line.startswith('//%PDDM'): + section = self.TextSection(line_num) + else: + directive = line.split(' ', 1)[0] + if directive == '//%PDDM-EXPAND': + section = self.ExpansionSection(line_num) + elif directive == '//%PDDM-DEFINE': + section = self.DefinitionSection(line_num) + elif directive == '//%PDDM-IMPORT-DEFINES': + section = self.ImportDefinesSection(line_num, self._import_resolver) + else: + raise PDDMError('Unexpected line %d: "%s".' % (line_num, line)) + self._sections.append(section) + return section + + def ProcessContent(self, strip_expansion=False): + """Processes the file contents.""" + self._ParseFile() + if strip_expansion: + # Without a collection the expansions become blank, removing them. + collection = None + else: + collection = MacroCollection() + for section in self._sections: + section.BindMacroCollection(collection) + result = '' + for section in self._sections: + result += section.text + self._processed_content = result + + @property + def original_content(self): + return self._original_content + + @property + def processed_content(self): + return self._processed_content + + +def main(args): + usage = '%prog [OPTIONS] PATH ...' + description = ( + 'Processes PDDM directives in the the given paths and write them back' + ' out.' + ) + parser = optparse.OptionParser(usage=usage, description=description) + parser.add_option('--dry-run', + default=False, action='store_true', + help='Don\'t write back to the file(s), just report if the' + ' contents needs an update and exit with a value of 1.') + parser.add_option('--verbose', + default=False, action='store_true', + help='Reports is a file is already current.') + parser.add_option('--collapse', + default=False, action='store_true', + help='Removes all the generated code.') + opts, extra_args = parser.parse_args(args) + + if not extra_args: + parser.error('Need atleast one file to process') + + result = 0 + for a_path in extra_args: + if not os.path.exists(a_path): + sys.stderr.write('ERROR: File not found: %s\n' % a_path) + return 100 + + def _ImportResolver(name): + # resolve based on the file being read. + a_dir = os.path.dirname(a_path) + import_path = os.path.join(a_dir, name) + if not os.path.exists(import_path): + return None + return open(import_path, 'r') + + with open(a_path, 'r') as f: + src_file = SourceFile(f, _ImportResolver) + + try: + src_file.ProcessContent(strip_expansion=opts.collapse) + except PDDMError as e: + sys.stderr.write('ERROR: %s\n...While processing "%s"\n' % + (e.message, a_path)) + return 101 + + if src_file.processed_content != src_file.original_content: + if not opts.dry_run: + print 'Updating for "%s".' % a_path + with open(a_path, 'w') as f: + f.write(src_file.processed_content) + else: + # Special result to indicate things need updating. + print 'Update needed for "%s".' % a_path + result = 1 + elif opts.verbose: + print 'No update for "%s".' % a_path + + return result + + +if __name__ == '__main__': + sys.exit(main(sys.argv[1:])) diff --git a/objectivec/DevTools/pddm_tests.py b/objectivec/DevTools/pddm_tests.py new file mode 100755 index 00000000..8a73b842 --- /dev/null +++ b/objectivec/DevTools/pddm_tests.py @@ -0,0 +1,515 @@ +#! /usr/bin/python +# +# Protocol Buffers - Google's data interchange format +# Copyright 2015 Google Inc. All rights reserved. +# https://developers.google.com/protocol-buffers/ +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following disclaimer +# in the documentation and/or other materials provided with the +# distribution. +# * Neither the name of Google Inc. nor the names of its +# contributors may be used to endorse or promote products derived from +# this software without specific prior written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +"""Tests for pddm.py.""" + +import io +import unittest + +import pddm + + +class TestParsingMacros(unittest.TestCase): + + def testParseEmpty(self): + f = io.StringIO(u'') + result = pddm.MacroCollection(f) + self.assertEqual(len(result._macros), 0) + + def testParseOne(self): + f = io.StringIO(u"""PDDM-DEFINE foo( ) +body""") + result = pddm.MacroCollection(f) + self.assertEqual(len(result._macros), 1) + macro = result._macros.get('foo') + self.assertIsNotNone(macro) + self.assertEquals(macro.name, 'foo') + self.assertEquals(macro.args, tuple()) + self.assertEquals(macro.body, 'body') + + def testParseGeneral(self): + # Tests multiple defines, spaces in all places, etc. + f = io.StringIO(u""" +PDDM-DEFINE noArgs( ) +body1 +body2 + +PDDM-DEFINE-END + +PDDM-DEFINE oneArg(foo) +body3 +PDDM-DEFINE twoArgs( bar_ , baz ) +body4 +body5""") + result = pddm.MacroCollection(f) + self.assertEqual(len(result._macros), 3) + macro = result._macros.get('noArgs') + self.assertIsNotNone(macro) + self.assertEquals(macro.name, 'noArgs') + self.assertEquals(macro.args, tuple()) + self.assertEquals(macro.body, 'body1\nbody2\n') + macro = result._macros.get('oneArg') + self.assertIsNotNone(macro) + self.assertEquals(macro.name, 'oneArg') + self.assertEquals(macro.args, ('foo',)) + self.assertEquals(macro.body, 'body3') + macro = result._macros.get('twoArgs') + self.assertIsNotNone(macro) + self.assertEquals(macro.name, 'twoArgs') + self.assertEquals(macro.args, ('bar_', 'baz')) + self.assertEquals(macro.body, 'body4\nbody5') + # Add into existing collection + f = io.StringIO(u""" +PDDM-DEFINE another(a,b,c) +body1 +body2""") + result.ParseInput(f) + self.assertEqual(len(result._macros), 4) + macro = result._macros.get('another') + self.assertIsNotNone(macro) + self.assertEquals(macro.name, 'another') + self.assertEquals(macro.args, ('a', 'b', 'c')) + self.assertEquals(macro.body, 'body1\nbody2') + + def testParseDirectiveIssues(self): + test_list = [ + # Unknown directive + (u'PDDM-DEFINE foo()\nbody\nPDDM-DEFINED foo\nbaz', + 'Hit a line with an unknown directive: '), + # End without begin + (u'PDDM-DEFINE foo()\nbody\nPDDM-DEFINE-END\nPDDM-DEFINE-END\n', + 'Got DEFINE-END directive without an active macro: '), + # Line not in macro block + (u'PDDM-DEFINE foo()\nbody\nPDDM-DEFINE-END\nmumble\n', + 'Hit a line that wasn\'t a directive and no open macro definition: '), + # Redefine macro + (u'PDDM-DEFINE foo()\nbody\nPDDM-DEFINE foo(a)\nmumble\n', + 'Attempt to redefine macro: '), + ] + for idx, (input_str, expected_prefix) in enumerate(test_list, 1): + f = io.StringIO(input_str) + try: + result = pddm.MacroCollection(f) + self.fail('Should throw exception, entry %d' % idx) + except pddm.PDDMError as e: + self.assertTrue(e.message.startswith(expected_prefix), + 'Entry %d failed: %r' % (idx, e)) + + def testParseBeginIssues(self): + test_list = [ + # 1. No name + (u'PDDM-DEFINE\nmumble', + 'Failed to parse macro definition: '), + # 2. No name (with spaces) + (u'PDDM-DEFINE \nmumble', + 'Failed to parse macro definition: '), + # 3. No open paren + (u'PDDM-DEFINE foo\nmumble', + 'Failed to parse macro definition: '), + # 4. No close paren + (u'PDDM-DEFINE foo(\nmumble', + 'Failed to parse macro definition: '), + # 5. No close paren (with args) + (u'PDDM-DEFINE foo(a, b\nmumble', + 'Failed to parse macro definition: '), + # 6. No name before args + (u'PDDM-DEFINE (a, b)\nmumble', + 'Failed to parse macro definition: '), + # 7. No name before args + (u'PDDM-DEFINE foo bar(a, b)\nmumble', + 'Failed to parse macro definition: '), + # 8. Empty arg name + (u'PDDM-DEFINE foo(a, ,b)\nmumble', + 'Empty arg name in macro definition: '), + (u'PDDM-DEFINE foo(a,,b)\nmumble', + 'Empty arg name in macro definition: '), + # 10. Duplicate name + (u'PDDM-DEFINE foo(a,b,a,c)\nmumble', + 'Arg name "a" used more than once in macro definition: '), + # 11. Invalid arg name + (u'PDDM-DEFINE foo(a b,c)\nmumble', + 'Invalid arg name "a b" in macro definition: '), + (u'PDDM-DEFINE foo(a.b,c)\nmumble', + 'Invalid arg name "a.b" in macro definition: '), + (u'PDDM-DEFINE foo(a-b,c)\nmumble', + 'Invalid arg name "a-b" in macro definition: '), + (u'PDDM-DEFINE foo(a,b,c.)\nmumble', + 'Invalid arg name "c." in macro definition: '), + # 15. Extra stuff after the name + (u'PDDM-DEFINE foo(a,c) foo\nmumble', + 'Failed to parse macro definition: '), + (u'PDDM-DEFINE foo(a,c) foo)\nmumble', + 'Failed to parse macro definition: '), + ] + for idx, (input_str, expected_prefix) in enumerate(test_list, 1): + f = io.StringIO(input_str) + try: + result = pddm.MacroCollection(f) + self.fail('Should throw exception, entry %d' % idx) + except pddm.PDDMError as e: + self.assertTrue(e.message.startswith(expected_prefix), + 'Entry %d failed: %r' % (idx, e)) + + +class TestExpandingMacros(unittest.TestCase): + + def testExpandBasics(self): + f = io.StringIO(u""" +PDDM-DEFINE noArgs( ) +body1 +body2 + +PDDM-DEFINE-END + +PDDM-DEFINE oneArg(a) +body3 a + +PDDM-DEFINE-END + +PDDM-DEFINE twoArgs(b,c) +body4 b c +body5 +PDDM-DEFINE-END + +""") + mc = pddm.MacroCollection(f) + test_list = [ + (u'noArgs()', + 'body1\nbody2\n'), + (u'oneArg(wee)', + 'body3 wee\n'), + (u'twoArgs(having some, fun)', + 'body4 having some fun\nbody5'), + # One arg, pass empty. + (u'oneArg()', + 'body3 \n'), + # Two args, gets empty in each slot. + (u'twoArgs(, empty)', + 'body4 empty\nbody5'), + (u'twoArgs(empty, )', + 'body4 empty \nbody5'), + (u'twoArgs(, )', + 'body4 \nbody5'), + ] + for idx, (input_str, expected) in enumerate(test_list, 1): + result = mc.Expand(input_str) + self.assertEqual(result, expected, + 'Entry %d --\n Result: %r\n Expected: %r' % + (idx, result, expected)) + + def testExpandArgOptions(self): + f = io.StringIO(u""" +PDDM-DEFINE bar(a) +a-a$S-a$l-a$L-a$u-a$U +PDDM-DEFINE-END +""") + mc = pddm.MacroCollection(f) + + self.assertEqual(mc.Expand('bar(xYz)'), 'xYz- -xYz-xyz-XYz-XYZ') + self.assertEqual(mc.Expand('bar(MnoP)'), 'MnoP- -mnoP-mnop-MnoP-MNOP') + # Test empty + self.assertEqual(mc.Expand('bar()'), '-----') + + def testExpandSimpleMacroErrors(self): + f = io.StringIO(u""" +PDDM-DEFINE foo(a, b) +<a-z> +PDDM-DEFINE baz(a) +a - a$z +""") + mc = pddm.MacroCollection(f) + test_list = [ + # 1. Unknown macro + (u'bar()', + 'No macro named "bar".'), + (u'bar(a)', + 'No macro named "bar".'), + # 3. Arg mismatch + (u'foo()', + 'Expected 2 args, got: "foo()".'), + (u'foo(a b)', + 'Expected 2 args, got: "foo(a b)".'), + (u'foo(a,b,c)', + 'Expected 2 args, got: "foo(a,b,c)".'), + # 6. Unknown option in expansion + (u'baz(mumble)', + 'Unknown arg option "a$z" while expanding "baz(mumble)".'), + ] + for idx, (input_str, expected_err) in enumerate(test_list, 1): + try: + result = mc.Expand(input_str) + self.fail('Should throw exception, entry %d' % idx) + except pddm.PDDMError as e: + self.assertEqual(e.message, expected_err, + 'Entry %d failed: %r' % (idx, e)) + + def testExpandReferences(self): + f = io.StringIO(u""" +PDDM-DEFINE StartIt() +foo(abc, def) +foo(ghi, jkl) +PDDM-DEFINE foo(a, b) +bar(a, int) +bar(b, NSString *) +PDDM-DEFINE bar(n, t) +- (t)n; +- (void)set##n$u##:(t)value; + +""") + mc = pddm.MacroCollection(f) + expected = """- (int)abc; +- (void)setAbc:(int)value; + +- (NSString *)def; +- (void)setDef:(NSString *)value; + +- (int)ghi; +- (void)setGhi:(int)value; + +- (NSString *)jkl; +- (void)setJkl:(NSString *)value; +""" + self.assertEqual(mc.Expand('StartIt()'), expected) + + def testCatchRecursion(self): + f = io.StringIO(u""" +PDDM-DEFINE foo(a, b) +bar(1, a) +bar(2, b) +PDDM-DEFINE bar(x, y) +foo(x, y) +""") + mc = pddm.MacroCollection(f) + try: + result = mc.Expand('foo(A,B)') + self.fail('Should throw exception, entry %d' % idx) + except pddm.PDDMError as e: + self.assertEqual(e.message, + 'Found macro recusion, invoking "foo(1, A)":\n...while expanding "bar(1, A)".\n...while expanding "foo(A,B)".') + + +class TestParsingSource(unittest.TestCase): + + def testBasicParse(self): + test_list = [ + # 1. no directives + (u'a\nb\nc', + (3,) ), + # 2. One define + (u'a\n//%PDDM-DEFINE foo()\n//%body\nc', + (1, 2, 1) ), + # 3. Two defines + (u'a\n//%PDDM-DEFINE foo()\n//%body\n//%PDDM-DEFINE bar()\n//%body2\nc', + (1, 4, 1) ), + # 4. Two defines with ends + (u'a\n//%PDDM-DEFINE foo()\n//%body\n//%PDDM-DEFINE-END\n' + u'//%PDDM-DEFINE bar()\n//%body2\n//%PDDM-DEFINE-END\nc', + (1, 6, 1) ), + # 5. One expand, one define (that runs to end of file) + (u'a\n//%PDDM-EXPAND foo()\nbody\n//%PDDM-EXPAND-END\n' + u'//%PDDM-DEFINE bar()\n//%body2\n', + (1, 1, 2) ), + # 6. One define ended with an expand. + (u'a\nb\n//%PDDM-DEFINE bar()\n//%body2\n' + u'//%PDDM-EXPAND bar()\nbody2\n//%PDDM-EXPAND-END\n', + (2, 2, 1) ), + # 7. Two expands (one end), one define. + (u'a\n//%PDDM-EXPAND foo(1)\nbody\n//%PDDM-EXPAND foo(2)\nbody2\n//%PDDM-EXPAND-END\n' + u'//%PDDM-DEFINE foo()\n//%body2\n', + (1, 2, 2) ), + ] + for idx, (input_str, line_counts) in enumerate(test_list, 1): + f = io.StringIO(input_str) + sf = pddm.SourceFile(f) + sf._ParseFile() + self.assertEqual(len(sf._sections), len(line_counts), + 'Entry %d -- %d != %d' % + (idx, len(sf._sections), len(line_counts))) + for idx2, (sec, expected) in enumerate(zip(sf._sections, line_counts), 1): + self.assertEqual(sec.num_lines_captured, expected, + 'Entry %d, section %d -- %d != %d' % + (idx, idx2, sec.num_lines_captured, expected)) + + def testErrors(self): + test_list = [ + # 1. Directive within expansion + (u'//%PDDM-EXPAND a()\n//%PDDM-BOGUS', + 'Ran into directive ("//%PDDM-BOGUS", line 2) while in "//%PDDM-EXPAND a()".'), + (u'//%PDDM-EXPAND a()\n//%PDDM-DEFINE a()\n//%body\n', + 'Ran into directive ("//%PDDM-DEFINE", line 2) while in "//%PDDM-EXPAND a()".'), + # 3. Expansion ran off end of file + (u'//%PDDM-EXPAND a()\na\nb\n', + 'Hit the end of the file while in "//%PDDM-EXPAND a()".'), + # 4. Directive within define + (u'//%PDDM-DEFINE a()\n//%body\n//%PDDM-BOGUS', + 'Ran into directive ("//%PDDM-BOGUS", line 3) while in "//%PDDM-DEFINE a()".'), + (u'//%PDDM-DEFINE a()\n//%body\n//%PDDM-EXPAND-END a()', + 'Ran into directive ("//%PDDM-EXPAND-END", line 3) while in "//%PDDM-DEFINE a()".'), + # 6. Directives that shouldn't start sections + (u'a\n//%PDDM-DEFINE-END a()\n//a\n', + 'Unexpected line 2: "//%PDDM-DEFINE-END a()".'), + (u'a\n//%PDDM-EXPAND-END a()\n//a\n', + 'Unexpected line 2: "//%PDDM-EXPAND-END a()".'), + (u'//%PDDM-BOGUS\n//a\n', + 'Unexpected line 1: "//%PDDM-BOGUS".'), + ] + for idx, (input_str, expected_err) in enumerate(test_list, 1): + f = io.StringIO(input_str) + try: + pddm.SourceFile(f)._ParseFile() + self.fail('Should throw exception, entry %d' % idx) + except pddm.PDDMError as e: + self.assertEqual(e.message, expected_err, + 'Entry %d failed: %r' % (idx, e)) + +class TestProcessingSource(unittest.TestCase): + + def testBasics(self): + input_str = u""" +//%PDDM-IMPORT-DEFINES ImportFile +foo +//%PDDM-EXPAND mumble(abc) +//%PDDM-EXPAND-END +bar +//%PDDM-EXPAND mumble(def) +//%PDDM-EXPAND mumble(ghi) +//%PDDM-EXPAND-END +baz +//%PDDM-DEFINE mumble(a_) +//%a_: getName(a_) +""" + input_str2 = u""" +//%PDDM-DEFINE getName(x_) +//%do##x_$u##(int x_); + +""" + expected = u""" +//%PDDM-IMPORT-DEFINES ImportFile +foo +//%PDDM-EXPAND mumble(abc) +// This block of code is generated, do not edit it directly. + +abc: doAbc(int abc); +//%PDDM-EXPAND-END mumble(abc) +bar +//%PDDM-EXPAND mumble(def) +// This block of code is generated, do not edit it directly. + +def: doDef(int def); +//%PDDM-EXPAND mumble(ghi) +// This block of code is generated, do not edit it directly. + +ghi: doGhi(int ghi); +//%PDDM-EXPAND-END (2 expansions) +baz +//%PDDM-DEFINE mumble(a_) +//%a_: getName(a_) +""" + expected_stripped = u""" +//%PDDM-IMPORT-DEFINES ImportFile +foo +//%PDDM-EXPAND mumble(abc) +//%PDDM-EXPAND-END mumble(abc) +bar +//%PDDM-EXPAND mumble(def) +//%PDDM-EXPAND mumble(ghi) +//%PDDM-EXPAND-END (2 expansions) +baz +//%PDDM-DEFINE mumble(a_) +//%a_: getName(a_) +""" + def _Resolver(name): + self.assertEqual(name, 'ImportFile') + return io.StringIO(input_str2) + f = io.StringIO(input_str) + sf = pddm.SourceFile(f, _Resolver) + sf.ProcessContent() + self.assertEqual(sf.processed_content, expected) + # Feed it through and nothing should change. + f2 = io.StringIO(sf.processed_content) + sf2 = pddm.SourceFile(f2, _Resolver) + sf2.ProcessContent() + self.assertEqual(sf2.processed_content, expected) + self.assertEqual(sf2.processed_content, sf.processed_content) + # Test stripping (with the original input and expanded version). + f2 = io.StringIO(input_str) + sf2 = pddm.SourceFile(f2) + sf2.ProcessContent(strip_expansion=True) + self.assertEqual(sf2.processed_content, expected_stripped) + f2 = io.StringIO(sf.processed_content) + sf2 = pddm.SourceFile(f2, _Resolver) + sf2.ProcessContent(strip_expansion=True) + self.assertEqual(sf2.processed_content, expected_stripped) + + def testProcessFileWithMacroParseError(self): + input_str = u""" +foo +//%PDDM-DEFINE mumble(a_) +//%body +//%PDDM-DEFINE mumble(x_) +//%body2 + +""" + f = io.StringIO(input_str) + sf = pddm.SourceFile(f) + try: + sf.ProcessContent() + self.fail('Should throw exception, entry %d' % idx) + except pddm.PDDMError as e: + self.assertEqual(e.message, + 'Attempt to redefine macro: "PDDM-DEFINE mumble(x_)"\n' + '...while parsing section that started:\n' + ' Line 3: //%PDDM-DEFINE mumble(a_)') + + def testProcessFileWithExpandError(self): + input_str = u""" +foo +//%PDDM-DEFINE mumble(a_) +//%body +//%PDDM-EXPAND foobar(x_) +//%PDDM-EXPAND-END + +""" + f = io.StringIO(input_str) + sf = pddm.SourceFile(f) + try: + sf.ProcessContent() + self.fail('Should throw exception, entry %d' % idx) + except pddm.PDDMError as e: + self.assertEqual(e.message, + 'No macro named "foobar".\n' + '...while expanding "foobar(x_)" from the section that' + ' started:\n Line 5: //%PDDM-EXPAND foobar(x_)') + + +if __name__ == '__main__': + unittest.main() |