diff options
Diffstat (limited to 'tensorflow/python/framework/docs.py')
| -rw-r--r-- | tensorflow/python/framework/docs.py | 492 |
1 files changed, 492 insertions, 0 deletions
diff --git a/tensorflow/python/framework/docs.py b/tensorflow/python/framework/docs.py new file mode 100644 index 0000000000..68dbb3df72 --- /dev/null +++ b/tensorflow/python/framework/docs.py @@ -0,0 +1,492 @@ +"""Updates generated docs from Python doc comments. + +Both updates the files in the file-system and executes g4 commands to +make sure any changes are ready to be submitted. +""" + +import inspect +import os +import re +import sys + + +_arg_re = re.compile(" *([*]{0,2}[a-zA-Z][a-zA-Z0-9_]*):") +_section_re = re.compile("([A-Z][a-zA-Z ]*):$") +_always_drop_symbol_re = re.compile("_[_a-zA-Z0-9]") +_anchor_re = re.compile(r"^[\w.]+$") +_member_mark = "@@" + + +class Document(object): + """Base class for an automatically generated document.""" + + def write_markdown_to_file(self, f): + """Writes a Markdown-formatted version of this document to file `f`. + + Args: + f: The output file. + """ + raise NotImplementedError("Document.WriteToFile") + + +class Index(Document): + """An automatically generated index for a collection of documents.""" + + def __init__(self, module_to_name, members, filename_to_library_map): + """Creates a new Index. + + Args: + module_to_name: Dictionary mapping modules to short names. + members: Dictionary mapping member name to (fullname, member). + filename_to_library_map: A list of (filename, Library) pairs. The order + corresponds to the order in which the libraries appear in the index. + """ + self._module_to_name = module_to_name + self._members = members + self._filename_to_library_map = filename_to_library_map + + def write_markdown_to_file(self, f): + """Writes this index to file `f`. + + The output is formatted as an unordered list. Each list element + contains the title of the library, followed by a list of symbols + in that library hyperlinked to the corresponding anchor in that + library. + + Args: + f: The output file. + """ + print >>f, "<!-- This file is machine generated: DO NOT EDIT! -->" + print >>f, "" + print >>f, "# TensorFlow Python reference documentation" + print >>f, "" + for filename, library in self._filename_to_library_map: + per_symbol_links = [] + for name in sorted(library.mentioned): + if name in self._members: + fullname, member = self._members[name] + anchor = _get_anchor(self._module_to_name, fullname) + prefix = "class " * inspect.isclass(member) + per_symbol_links.append("[%s%s](%s#%s)" % + (prefix, name, filename, anchor)) + if per_symbol_links: + print >>f, "* <b>[%s](%s)</b>: %s" % (library.title, filename, + ",\n ".join(per_symbol_links)) + print >>f, "" + + # actually include the files right here + print >>f, '<div class="sections-order" style="display: none;">\n<!--' + for filename, _ in self._filename_to_library_map: + print >>f, "<!-- %s -->" % filename + print >>f, "-->\n</div>" + +def collect_members(module_to_name): + """Collect all symbols from a list of modules. + + Args: + module_to_name: Dictionary mapping modules to short names. + + Returns: + Dictionary mapping name to (fullname, member) pairs. + """ + members = {} + for module, module_name in module_to_name.iteritems(): + for name, member in inspect.getmembers(module): + if ((inspect.isfunction(member) or inspect.isclass(member)) and + not _always_drop_symbol_re.match(name)): + fullname = '%s.%s' % (module_name, name) + if name in members: + other_fullname, other_member = members[name] + if member is not other_member: + raise RuntimeError("Short name collision between %s and %s" % + (fullname, other_fullname)) + if len(fullname) == len(other_fullname): + raise RuntimeError("Can't decide whether to use %s or %s for %s: " + "both full names have length %d" % + (fullname, other_fullname, len(fullname))) + if len(fullname) > len(other_fullname): + continue # Use the shorter full name + members[name] = fullname, member + return members + + +def _get_anchor(module_to_name, fullname): + """Turn a full member name into an anchor. + + Args: + module_to_name: Dictionary mapping modules to short names. + fullname: Fully qualified name of symbol. + + Returns: + HTML anchor string. The longest module name prefix of fullname is + removed to make the anchor. + + Raises: + ValueError: If fullname uses characters invalid in an anchor. + """ + if not _anchor_re.match(fullname): + raise ValueError("'%s' is not a valid anchor" % fullname) + anchor = fullname + for module_name in module_to_name.itervalues(): + if fullname.startswith(module_name + "."): + rest = fullname[len(module_name)+1:] + # Use this prefix iff it is longer than any found before + if len(anchor) > len(rest): + anchor = rest + return anchor + + +class Library(Document): + """An automatically generated document for a set of functions and classes.""" + + def __init__(self, + title, + module, + module_to_name, + members, + documented, + exclude_symbols=(), + catch_all=False): + """Creates a new Library. + + Args: + title: A human-readable title for the library. + module: Module to pull high level docstring from (for table of contents, + list of Ops to document, etc.). + module_to_name: Dictionary mapping modules to short names. + members: Dictionary mapping member name to (fullname, member). + documented: Set of documented names to update. + exclude_symbols: A list of specific symbols to exclude. + """ + self._title = title + self._module = module + self._module_to_name = module_to_name + self._members = dict(members) # Copy since we mutate it below + self._exclude_symbols = frozenset(exclude_symbols) + documented.update(exclude_symbols) + self._documented = documented + self._mentioned = set() + + @property + def title(self): + """The human-readable title for this library.""" + return self._title + + @property + def mentioned(self): + """Set of names mentioned in this library.""" + return self._mentioned + + @property + def exclude_symbols(self): + """Set of excluded symbols.""" + return self._exclude_symbols + + def _should_include_member(self, name, member): + """Returns True if this member should be included in the document.""" + # Always exclude symbols matching _always_drop_symbol_re. + if _always_drop_symbol_re.match(name): + return False + # Finally, exclude any specifically-excluded symbols. + if name in self._exclude_symbols: + return False + return True + + def get_imported_modules(self, module): + """Returns the list of modules imported from `module`.""" + for name, member in inspect.getmembers(module): + if inspect.ismodule(member): + yield name, member + + def get_class_members(self, cls_name, cls): + """Returns the list of class members to document in `cls`. + + This function filters the class member to ONLY return those + defined by the class. It drops the inherited ones. + + Args: + cls_name: Qualified name of `cls`. + cls: An inspect object of type 'class'. + + Yields: + name, member tuples. + """ + for name, member in inspect.getmembers(cls): + # Only show methods and properties presently. + if not (inspect.ismethod(member) or isinstance(member, property)): + continue + if ((inspect.ismethod(member) and member.__name__ == "__init__") + or self._should_include_member(name, member)): + yield name, ("%s.%s" % (cls_name, name), member) + + def _generate_signature_for_function(self, func): + """Given a function, returns a string representing its args.""" + args_list = [] + argspec = inspect.getargspec(func) + first_arg_with_default = ( + len(argspec.args or []) - len(argspec.defaults or [])) + for arg in argspec.args[:first_arg_with_default]: + if arg == "self": + # Python documentation typically skips `self` when printing method + # signatures. + continue + args_list.append(arg) + if argspec.defaults: + for arg, default in zip( + argspec.args[first_arg_with_default:], argspec.defaults): + args_list.append("%s=%r" % (arg, default)) + if argspec.varargs: + args_list.append("*" + argspec.varargs) + if argspec.keywords: + args_list.append("**" + argspec.keywords) + return "(" + ", ".join(args_list) + ")" + + def _remove_docstring_indent(self, docstring): + """Remove indenting. + + We follow Python's convention and remove the minimum indent of the lines + after the first, see: + https://www.python.org/dev/peps/pep-0257/#handling-docstring-indentation + preserving relative indentation. + + Args: + docstring: A docstring. + + Returns: + A list of strings, one per line, with the minimum indent stripped. + """ + docstring = docstring or "" + lines = docstring.strip().split("\n") + + min_indent = len(docstring) + for l in lines[1:]: + l = l.rstrip() + if l: + i = 0 + while i < len(l) and l[i] == " ": + i += 1 + if i < min_indent: min_indent = i + for i in range(1, len(lines)): + l = lines[i].rstrip() + if len(l) >= min_indent: + l = l[min_indent:] + lines[i] = l + return lines + + def _print_formatted_docstring(self, docstring, f): + """Formats the given `docstring` as Markdown and prints it to `f`.""" + lines = self._remove_docstring_indent(docstring) + + # Output the lines, identifying "Args" and other section blocks. + i = 0 + + def _at_start_of_section(): + """Returns the header if lines[i] is at start of a docstring section.""" + l = lines[i] + match = _section_re.match(l) + if match and i + 1 < len( + lines) and lines[i + 1].startswith(" "): + return match.group(1) + else: + return None + + while i < len(lines): + l = lines[i] + + section_header = _at_start_of_section() + if section_header: + if i == 0 or lines[i-1]: + print >>f, "" + # Use at least H4 to keep these out of the TOC. + print >>f, "##### " + section_header + ":" + print >>f, "" + i += 1 + outputting_list = False + while i < len(lines): + l = lines[i] + # A new section header terminates the section. + if _at_start_of_section(): + break + match = _arg_re.match(l) + if match: + if not outputting_list: + # We need to start a list. In Markdown, a blank line needs to + # precede a list. + print >>f, "" + outputting_list = True + suffix = l[len(match.group()):].lstrip() + print >>f, "* <b>" + match.group(1) + "</b>: " + suffix + else: + # For lines that don't start with _arg_re, continue the list if it + # has enough indentation. + outputting_list &= l.startswith(" ") + print >>f, l + i += 1 + else: + print >>f, l + i += 1 + + def _print_function(self, f, prefix, fullname, func): + """Prints the given function to `f`.""" + heading = prefix + " " + fullname + if not isinstance(func, property): + heading += self._generate_signature_for_function(func) + heading += " {#%s}" % _get_anchor(self._module_to_name, fullname) + print >>f, heading + print >>f, "" + self._print_formatted_docstring(inspect.getdoc(func), f) + print >>f, "" + + def _write_member_markdown_to_file(self, f, name, member): + """Print `member` to `f`.""" + if inspect.isfunction(member): + print >>f, "- - -" + print >>f, "" + self._print_function(f, "###", name, member) + print >>f, "" + elif inspect.ismethod(member): + print >>f, "- - -" + print >>f, "" + self._print_function(f, "####", name, member) + print >>f, "" + elif isinstance(member, property): + print >>f, "- - -" + print >>f, "" + self._print_function(f, "####", name, member) + elif inspect.isclass(member): + print >>f, "- - -" + print >>f, "" + print >>f, "### class %s {#%s}" % ( + name, _get_anchor(self._module_to_name, name)) + print >>f, "" + self._write_class_markdown_to_file(f, name, member) + print >>f, "" + else: + raise RuntimeError("Member %s has unknown type %s" % (name, type(member))) + + def _write_docstring_markdown_to_file(self, f, docstring, members, imports): + for l in self._remove_docstring_indent(docstring): + if l.startswith(_member_mark): + name = l[len(_member_mark):].strip(" \t") + if name in members: + self._documented.add(name) + self._mentioned.add(name) + self._write_member_markdown_to_file(f, *members[name]) + del members[name] + elif name in imports: + self._write_module_markdown_to_file(f, imports[name]) + else: + raise ValueError("%s: unknown member `%s`" % (self._title, name)) + else: + print >>f, l + + def _write_class_markdown_to_file(self, f, name, cls): + """Write the class doc to 'f'. + + Args: + f: File to write to. + prefix: Prefix for names. + cls: class object. + name: name to use. + """ + # Build the list of class methods to document. + methods = dict(self.get_class_members(name, cls)) + # Used later to check if any methods were called out in the class + # docstring. + num_methods = len(methods) + self._write_docstring_markdown_to_file(f, inspect.getdoc(cls), methods, {}) + + # If some methods were not described, describe them now if they are + # defined by the class itself (not inherited). If NO methods were + # described, describe all methods. + # + # TODO(mdevin): when all methods have been categorized make it an error + # if some methods are not categorized. + any_method_called_out = (len(methods) != num_methods) + if any_method_called_out: + other_methods = {n: m for n, m in methods.iteritems() + if n in cls.__dict__} + if other_methods: + print >>f, "\n#### Other Methods" + else: + other_methods = methods + for name in sorted(other_methods): + self._write_member_markdown_to_file(f, *other_methods[name]) + + def _write_module_markdown_to_file(self, f, module): + imports = dict(self.get_imported_modules(module)) + self._write_docstring_markdown_to_file(f, inspect.getdoc(module), + self._members, imports) + + def write_markdown_to_file(self, f): + """Prints this library to file `f`. + + Args: + f: File to write to. + + Returns: + Dictionary of documented members. + """ + print >>f, "<!-- This file is machine generated: DO NOT EDIT! -->" + print >>f, "" + # TODO(mdevin): Do not insert these. Let the doc writer put them in + # the module docstring explicitly. + print >>f, "#", self._title + print >>f, "[TOC]" + print >>f, "" + if self._module is not None: + self._write_module_markdown_to_file(f, self._module) + + def write_other_members(self, f, catch_all=False): + """Writes the leftover members to `f`. + + Args: + f: File to write to. + catch_all: If true, document all missing symbols from any module. + Otherwise, document missing symbols from just this module. + """ + if catch_all: + names = self._members.iteritems() + else: + names = inspect.getmembers(self._module) + leftovers = [] + for name, _ in names: + if name in self._members and name not in self._documented: + leftovers.append(name) + if leftovers: + print "%s: undocumented members: %d" % (self._title, len(leftovers)) + print >>f, "\n## Other Functions and Classes" + for name in sorted(leftovers): + print " %s" % name + self._documented.add(name) + self._mentioned.add(name) + self._write_member_markdown_to_file(f, *self._members[name]) + + def assert_no_leftovers(self): + """Generate an error if there are leftover members.""" + leftovers = [] + for name in self._members.iterkeys(): + if name in self._members and name not in self._documented: + leftovers.append(name) + if leftovers: + raise RuntimeError("%s: undocumented members: %s" % + (self._title, ", ".join(leftovers))) + + +def write_libraries(dir, libraries): + """Write a list of libraries to disk. + + Args: + dir: Output directory. + libraries: List of (filename, library) pairs. + """ + files = [open(os.path.join(dir, k), "w") for k, _ in libraries] + # Document mentioned symbols for all libraries + for f, (_, v) in zip(files, libraries): + v.write_markdown_to_file(f) + # Document symbols that no library mentioned. We do this after writing + # out all libraries so that earlier libraries know what later libraries + # documented. + for f, (_, v) in zip(files, libraries): + v.write_other_members(f) + f.close() |