Add doc control decorators to the api-generator.

There are two primary goals here.

First: This gives us a starting point to cleanup/remove the "private_map" and
"do_not_descend" map from `generate_lib.py`. Replacing them with local decorators.
The primary reason for these in the first place is to deal with unsealed modules (The docs generator will crawl anything it can reach).

Second: This gives us options for splitting documentation output between "Users" and "Subclass Implementers". There are many "public" functions that you do not need to know about unless you are interested in implementing a sub-class. These only make sense to document on the base class "Layer", "Model", "Metric".
PiperOrigin-RevId: 208701899

6 files changed, 649 insertions, 7 deletions

diff --git a/tensorflow/tools/docs/BUILD b/tensorflow/tools/docs/BUILD
index cc7885ab1b..4f7efe193f 100644
--- a/tensorflow/tools/docs/BUILD
+++ b/tensorflow/tools/docs/BUILD
@@ -34,11 +34,29 @@ py_test(
 )
 
 py_library(
+    name = "doc_controls",
+    srcs = ["doc_controls.py"],
+    srcs_version = "PY2AND3",
+)
+
+py_test(
+    name = "doc_controls_test",
+    size = "small",
+    srcs = ["doc_controls_test.py"],
+    srcs_version = "PY2AND3",
+    deps = [
+        ":doc_controls",
+        "//tensorflow/python:platform_test",
+    ],
+)
+
+py_library(
     name = "parser",
     srcs = ["parser.py"],
     srcs_version = "PY2AND3",
     visibility = ["//visibility:public"],
     deps = [
+        ":doc_controls",
         "//tensorflow/python:platform",
         "//tensorflow/python:util",
         "@astor_archive//:astor",
@@ -68,6 +86,7 @@ py_binary(
     srcs_version = "PY2AND3",
     visibility = ["//visibility:public"],
     deps = [
+        ":doc_controls",
         ":doc_generator_visitor",
         ":parser",
         ":pretty_docs",
diff --git a/tensorflow/tools/docs/doc_controls.py b/tensorflow/tools/docs/doc_controls.py
new file mode 100644
index 0000000000..5e526443cc
--- /dev/null
+++ b/tensorflow/tools/docs/doc_controls.py
@@ -0,0 +1,319 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Documentation control decorators."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+_DO_NOT_DOC = "_tf_docs_do_not_document"
+
+
+def do_not_generate_docs(obj):
+  """A decorator: Do not generate docs for this object.
+
+  For example the following classes:
+
+  ```
+  class Parent(object):
+    def method1(self):
+      pass
+    def method2(self):
+      pass
+
+  class Child(Parent):
+    def method1(self):
+      pass
+    def method2(self):
+      pass
+  ```
+
+  Produce the following api_docs:
+
+  ```
+  /Parent.md
+    # method1
+    # method2
+  /Child.md
+    # method1
+    # method2
+  ```
+
+  This decorator allows you to skip classes or methods:
+
+  ```
+  @do_not_generate_docs
+  class Parent(object):
+    def method1(self):
+      pass
+    def method2(self):
+      pass
+
+  class Child(Parent):
+    @do_not_generate_docs
+    def method1(self):
+      pass
+    def method2(self):
+      pass
+  ```
+
+  This will only produce the following docs:
+
+  ```
+  /Child.md
+    # method2
+  ```
+
+  Note: This is implemented by adding a hidden attribute on the object, so it
+  cannot be used on objects which do not allow new attributes to be added. So
+  this decorator must go *below* `@property`, `@classmethod`,
+  or `@staticmethod`:
+
+  ```
+  class Example(object):
+    @property
+    @do_not_generate_docs
+    def x(self):
+      return self._x
+  ```
+
+  Args:
+    obj: The object to hide from the generated docs.
+
+  Returns:
+    obj
+  """
+  setattr(obj, _DO_NOT_DOC, None)
+  return obj
+
+
+_DO_NOT_DOC_INHERITABLE = "_tf_docs_do_not_doc_inheritable"
+
+
+def do_not_doc_inheritable(obj):
+  """A decorator: Do not generate docs for this method.
+
+  This version of the decorator is "inherited" by subclasses. No docs will be
+  generated for the decorated method in any subclass. Even if the sub-class
+  overrides the method.
+
+  For example, to ensure that `method1` is **never documented** use this
+  decorator on the base-class:
+
+  ```
+  class Parent(object):
+    @do_not_doc_inheritable
+    def method1(self):
+      pass
+    def method2(self):
+      pass
+
+  class Child(Parent):
+    def method1(self):
+      pass
+    def method2(self):
+      pass
+  ```
+  This will produce the following docs:
+
+  ```
+  /Parent.md
+    # method2
+  /Child.md
+    # method2
+  ```
+
+  When generating docs for a class's arributes, the `__mro__` is searched and
+  the attribute will be skipped if this decorator is detected on the attribute
+  on any class in the `__mro__`.
+
+  Note: This is implemented by adding a hidden attribute on the object, so it
+  cannot be used on objects which do not allow new attributes to be added. So
+  this decorator must go *below* `@property`, `@classmethod`,
+  or `@staticmethod`:
+
+  ```
+  class Example(object):
+    @property
+    @do_not_doc_inheritable
+    def x(self):
+      return self._x
+  ```
+
+  Args:
+    obj: The class-attribute to hide from the generated docs.
+
+  Returns:
+    obj
+  """
+  setattr(obj, _DO_NOT_DOC_INHERITABLE, None)
+  return obj
+
+
+_FOR_SUBCLASS_IMPLEMENTERS = "_tf_docs_tools_for_subclass_implementers"
+
+
+def for_subclass_implementers(obj):
+  """A decorator: Only generate docs for this method in the defining class.
+
+  Also group this method's docs with and `@abstractmethod` in the class's docs.
+
+  No docs will generated for this class attribute in sub-classes.
+
+  The canonical use case for this is `tf.keras.layers.Layer.call`: It's a
+  public method, essential for anyone implementing a subclass, but it should
+  never be called directly.
+
+  Works on method, or other class-attributes.
+
+  When generating docs for a class's arributes, the `__mro__` is searched and
+  the attribute will be skipped if this decorator is detected on the attribute
+  on any **parent** class in the `__mro__`.
+
+  For example:
+
+  ```
+  class Parent(object):
+    @for_subclass_implementers
+    def method1(self):
+      pass
+    def method2(self):
+      pass
+
+  class Child1(Parent):
+    def method1(self):
+      pass
+    def method2(self):
+      pass
+
+  class Child2(Parent):
+    def method1(self):
+      pass
+    def method2(self):
+      pass
+  ```
+
+  This will produce the following docs:
+
+  ```
+  /Parent.md
+    # method1
+    # method2
+  /Child1.md
+    # method2
+  /Child2.md
+    # method2
+  ```
+
+  Note: This is implemented by adding a hidden attribute on the object, so it
+  cannot be used on objects which do not allow new attributes to be added. So
+  this decorator must go *below* `@property`, `@classmethod`,
+  or `@staticmethod`:
+
+  ```
+  class Example(object):
+    @property
+    @for_subclass_implementers
+    def x(self):
+      return self._x
+  ```
+
+  Args:
+    obj: The class-attribute to hide from the generated docs.
+
+  Returns:
+    obj
+  """
+  setattr(obj, _FOR_SUBCLASS_IMPLEMENTERS, None)
+  return obj
+
+
+def should_skip(obj):
+  """Returns true if docs generation should be skipped for this object.
+
+  checks for the `do_not_generate_docs` or `do_not_doc_inheritable` decorators.
+
+  Args:
+    obj: The object to document, or skip.
+
+  Returns:
+    True if the object should be skipped
+  """
+  # Unwrap fget if the object is a property
+  if isinstance(obj, property):
+    obj = obj.fget
+
+  return hasattr(obj, _DO_NOT_DOC) or hasattr(obj, _DO_NOT_DOC_INHERITABLE)
+
+
+def should_skip_class_attr(cls, name):
+  """Returns true if docs should be skipped for this class attribute.
+
+  Args:
+    cls: The class the attribute belongs to.
+    name: The name of the attribute.
+
+  Returns:
+    True if the attribute should be skipped.
+  """
+  # Get the object with standard lookup, from the nearest
+  # defining parent.
+  try:
+    obj = getattr(cls, name)
+  except AttributeError:
+    # Avoid error caused by enum metaclasses in python3
+    if name in ("name", "value"):
+      return True
+    raise
+
+  # Unwrap fget if the object is a property
+  if isinstance(obj, property):
+    obj = obj.fget
+
+  # Skip if the object is decorated with `do_not_generate_docs` or
+  # `do_not_doc_inheritable`
+  if should_skip(obj):
+    return True
+
+  # Use __dict__ lookup to get the version defined in *this* class.
+  obj = cls.__dict__.get(name, None)
+  if isinstance(obj, property):
+    obj = obj.fget
+  if obj is not None:
+    # If not none, the object is defined in *this* class.
+    # Do not skip if decorated with `for_subclass_implementers`.
+    if hasattr(obj, _FOR_SUBCLASS_IMPLEMENTERS):
+      return False
+
+  # for each parent class
+  for parent in cls.__mro__[1:]:
+    obj = getattr(parent, name, None)
+
+    if obj is None:
+      continue
+
+    if isinstance(obj, property):
+      obj = obj.fget
+
+    # Skip if the parent's definition is decorated with `do_not_doc_inheritable`
+    # or `for_subclass_implementers`
+    if hasattr(obj, _DO_NOT_DOC_INHERITABLE):
+      return True
+
+    if hasattr(obj, _FOR_SUBCLASS_IMPLEMENTERS):
+      return True
+
+  # No blockng decorators --> don't skip
+  return False
diff --git a/tensorflow/tools/docs/doc_controls_test.py b/tensorflow/tools/docs/doc_controls_test.py
new file mode 100644
index 0000000000..410342fb69
--- /dev/null
+++ b/tensorflow/tools/docs/doc_controls_test.py
@@ -0,0 +1,183 @@
+# Copyright 2018 The TensorFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+"""Tests for documentation control decorators."""
+
+from __future__ import absolute_import
+from __future__ import division
+from __future__ import print_function
+
+from tensorflow.python.platform import googletest
+from tensorflow.tools.docs import doc_controls
+
+
+class DocControlsTest(googletest.TestCase):
+
+  def test_do_not_generate_docs(self):
+
+    @doc_controls.do_not_generate_docs
+    def dummy_function():
+      pass
+
+    self.assertTrue(doc_controls.should_skip(dummy_function))
+
+  def test_do_not_doc_on_method(self):
+    """The simple decorator is not aware of inheritance."""
+
+    class Parent(object):
+
+      @doc_controls.do_not_generate_docs
+      def my_method(self):
+        pass
+
+    class Child(Parent):
+
+      def my_method(self):
+        pass
+
+    class GrandChild(Child):
+      pass
+
+    self.assertTrue(doc_controls.should_skip(Parent.my_method))
+    self.assertFalse(doc_controls.should_skip(Child.my_method))
+    self.assertFalse(doc_controls.should_skip(GrandChild.my_method))
+
+    self.assertTrue(doc_controls.should_skip_class_attr(Parent, 'my_method'))
+    self.assertFalse(doc_controls.should_skip_class_attr(Child, 'my_method'))
+    self.assertFalse(
+        doc_controls.should_skip_class_attr(GrandChild, 'my_method'))
+
+  def test_do_not_doc_inheritable(self):
+
+    class Parent(object):
+
+      @doc_controls.do_not_doc_inheritable
+      def my_method(self):
+        pass
+
+    class Child(Parent):
+
+      def my_method(self):
+        pass
+
+    class GrandChild(Child):
+      pass
+
+    self.assertTrue(doc_controls.should_skip(Parent.my_method))
+    self.assertFalse(doc_controls.should_skip(Child.my_method))
+    self.assertFalse(doc_controls.should_skip(GrandChild.my_method))
+
+    self.assertTrue(doc_controls.should_skip_class_attr(Parent, 'my_method'))
+    self.assertTrue(doc_controls.should_skip_class_attr(Child, 'my_method'))
+    self.assertTrue(
+        doc_controls.should_skip_class_attr(GrandChild, 'my_method'))
+
+  def test_do_not_doc_inheritable_property(self):
+
+    class Parent(object):
+
+      @property
+      @doc_controls.do_not_doc_inheritable
+      def my_method(self):
+        pass
+
+    class Child(Parent):
+
+      @property
+      def my_method(self):
+        pass
+
+    class GrandChild(Child):
+      pass
+
+    self.assertTrue(doc_controls.should_skip(Parent.my_method))
+    self.assertFalse(doc_controls.should_skip(Child.my_method))
+    self.assertFalse(doc_controls.should_skip(GrandChild.my_method))
+
+    self.assertTrue(doc_controls.should_skip_class_attr(Parent, 'my_method'))
+    self.assertTrue(doc_controls.should_skip_class_attr(Child, 'my_method'))
+    self.assertTrue(
+        doc_controls.should_skip_class_attr(GrandChild, 'my_method'))
+
+  def test_do_not_doc_inheritable_staticmethod(self):
+
+    class GrandParent(object):
+
+      def my_method(self):
+        pass
+
+    class Parent(GrandParent):
+
+      @staticmethod
+      @doc_controls.do_not_doc_inheritable
+      def my_method():
+        pass
+
+    class Child(Parent):
+
+      @staticmethod
+      def my_method():
+        pass
+
+    class GrandChild(Child):
+      pass
+
+    self.assertFalse(doc_controls.should_skip(GrandParent.my_method))
+    self.assertTrue(doc_controls.should_skip(Parent.my_method))
+    self.assertFalse(doc_controls.should_skip(Child.my_method))
+    self.assertFalse(doc_controls.should_skip(GrandChild.my_method))
+
+    self.assertFalse(
+        doc_controls.should_skip_class_attr(GrandParent, 'my_method'))
+    self.assertTrue(doc_controls.should_skip_class_attr(Parent, 'my_method'))
+    self.assertTrue(doc_controls.should_skip_class_attr(Child, 'my_method'))
+    self.assertTrue(
+        doc_controls.should_skip_class_attr(GrandChild, 'my_method'))
+
+  def testfor_subclass_implementers(self):
+
+    class GrandParent(object):
+
+      def my_method(self):
+        pass
+
+    class Parent(GrandParent):
+
+      @doc_controls.for_subclass_implementers
+      def my_method(self):
+        pass
+
+    class Child(Parent):
+      pass
+
+    class GrandChild(Child):
+
+      def my_method(self):
+        pass
+
+    class Grand2Child(Child):
+      pass
+
+    self.assertFalse(
+        doc_controls.should_skip_class_attr(GrandParent, 'my_method'))
+    self.assertFalse(doc_controls.should_skip_class_attr(Parent, 'my_method'))
+    self.assertTrue(doc_controls.should_skip_class_attr(Child, 'my_method'))
+    self.assertTrue(
+        doc_controls.should_skip_class_attr(GrandChild, 'my_method'))
+    self.assertTrue(
+        doc_controls.should_skip_class_attr(Grand2Child, 'my_method'))
+
+
+if __name__ == '__main__':
+  googletest.main()
diff --git a/tensorflow/tools/docs/generate_lib.py b/tensorflow/tools/docs/generate_lib.py
index 4bc8cbf4b4..b9c418db1e 100644
--- a/tensorflow/tools/docs/generate_lib.py
+++ b/tensorflow/tools/docs/generate_lib.py
@@ -28,6 +28,7 @@ import six
 from tensorflow.python.util import tf_inspect
 from tensorflow.tools.common import public_api
 from tensorflow.tools.common import traverse
+from tensorflow.tools.docs import doc_controls
 from tensorflow.tools.docs import doc_generator_visitor
 from tensorflow.tools.docs import parser
 from tensorflow.tools.docs import pretty_docs
@@ -110,6 +111,9 @@ def write_docs(output_dir,
             _is_free_function(py_object, full_name, parser_config.index)):
       continue
 
+    if doc_controls.should_skip(py_object):
+      continue
+
     sitepath = os.path.join('api_docs/python',
                             parser.documentation_path(full_name)[:-3])
 
diff --git a/tensorflow/tools/docs/parser.py b/tensorflow/tools/docs/parser.py
index ffb93027ed..801c8bcb4a 100644
--- a/tensorflow/tools/docs/parser.py
+++ b/tensorflow/tools/docs/parser.py
@@ -32,6 +32,7 @@ import six
 from google.protobuf.message import Message as ProtoMessage
 from tensorflow.python.platform import tf_logging as logging
 from tensorflow.python.util import tf_inspect
+from tensorflow.tools.docs import doc_controls
 
 
 # A regular expression capturing a python identifier.
@@ -1175,15 +1176,18 @@ class _ClassPageInfo(object):
 
       # Don't document anything that is defined in object or by protobuf.
       defining_class = _get_defining_class(py_class, short_name)
-      if (defining_class is object or
-          defining_class is type or defining_class is tuple or
-          defining_class is BaseException or defining_class is Exception or
-          # The following condition excludes most protobuf-defined symbols.
-          defining_class and defining_class.__name__ in ['CMessage', 'Message',
-                                                         'MessageMeta']):
+      if defining_class in [object, type, tuple, BaseException, Exception]:
+        continue
+
+      # The following condition excludes most protobuf-defined symbols.
+      if (defining_class and
+          defining_class.__name__ in ['CMessage', 'Message', 'MessageMeta']):
         continue
       # TODO(markdaoust): Add a note in child docs showing the defining class.
 
+      if doc_controls.should_skip_class_attr(py_class, short_name):
+        continue
+
       child_doc = _parse_md_docstring(child, relative_path,
                                       parser_config.reference_resolver)
 
diff --git a/tensorflow/tools/docs/parser_test.py b/tensorflow/tools/docs/parser_test.py
index 274d48ef66..9f6b185e81 100644
--- a/tensorflow/tools/docs/parser_test.py
+++ b/tensorflow/tools/docs/parser_test.py
@@ -24,6 +24,7 @@ import sys
 
 from tensorflow.python.platform import googletest
 from tensorflow.python.util import tf_inspect
+from tensorflow.tools.docs import doc_controls
 from tensorflow.tools.docs import parser
 
 
@@ -37,13 +38,27 @@ def test_function_with_args_kwargs(unused_arg, *unused_args, **unused_kwargs):
   pass
 
 
-class TestClass(object):
+class ParentClass(object):
+
+  @doc_controls.do_not_doc_inheritable
+  def hidden_method(self):
+    pass
+
+
+class TestClass(ParentClass):
   """Docstring for TestClass itself."""
 
   def a_method(self, arg='default'):
     """Docstring for a method."""
     pass
 
+  def hidden_method(self):
+    pass
+
+  @doc_controls.do_not_generate_docs
+  def hidden_method2(self):
+    pass
+
   class ChildClass(object):
     """Docstring for a child class."""
     pass
@@ -175,6 +190,104 @@ class ParserTest(googletest.TestCase):
     # Make sure this file is contained as the definition location.
     self.assertEqual(os.path.relpath(__file__, '/'), page_info.defined_in.path)
 
+  def test_docs_for_class_should_skip(self):
+
+    class Parent(object):
+
+      @doc_controls.do_not_doc_inheritable
+      def a_method(self, arg='default'):
+        pass
+
+    class Child(Parent):
+
+      def a_method(self, arg='default'):
+        pass
+
+    index = {
+        'Child': Child,
+        'Child.a_method': Child.a_method,
+    }
+
+    visitor = DummyVisitor(index=index, duplicate_of={})
+
+    reference_resolver = parser.ReferenceResolver.from_visitor(
+        visitor=visitor, doc_index={}, py_module_names=['tf'])
+
+    tree = {
+        'Child': ['a_method'],
+    }
+
+    parser_config = parser.ParserConfig(
+        reference_resolver=reference_resolver,
+        duplicates={},
+        duplicate_of={},
+        tree=tree,
+        index=index,
+        reverse_index={},
+        guide_index={},
+        base_dir='/')
+
+    page_info = parser.docs_for_object(
+        full_name='Child', py_object=Child, parser_config=parser_config)
+
+    # Make sure the `a_method` is not present
+    self.assertEqual(0, len(page_info.methods))
+
+  def test_docs_for_message_class(self):
+
+    class CMessage(object):
+
+      def hidden(self):
+        pass
+
+    class Message(object):
+
+      def hidden2(self):
+        pass
+
+    class MessageMeta(object):
+
+      def hidden3(self):
+        pass
+
+    class ChildMessage(CMessage, Message, MessageMeta):
+
+      def my_method(self):
+        pass
+
+    index = {
+        'ChildMessage': ChildMessage,
+        'ChildMessage.hidden': ChildMessage.hidden,
+        'ChildMessage.hidden2': ChildMessage.hidden2,
+        'ChildMessage.hidden3': ChildMessage.hidden3,
+        'ChildMessage.my_method': ChildMessage.my_method,
+    }
+
+    visitor = DummyVisitor(index=index, duplicate_of={})
+
+    reference_resolver = parser.ReferenceResolver.from_visitor(
+        visitor=visitor, doc_index={}, py_module_names=['tf'])
+
+    tree = {'ChildMessage': ['hidden', 'hidden2', 'hidden3', 'my_method']}
+
+    parser_config = parser.ParserConfig(
+        reference_resolver=reference_resolver,
+        duplicates={},
+        duplicate_of={},
+        tree=tree,
+        index=index,
+        reverse_index={},
+        guide_index={},
+        base_dir='/')
+
+    page_info = parser.docs_for_object(
+        full_name='ChildMessage',
+        py_object=ChildMessage,
+        parser_config=parser_config)
+
+    self.assertEqual(1, len(page_info.methods))
+    self.assertEqual('my_method', page_info.methods[0].short_name)
+
   def test_docs_for_module(self):
     # Get the current module.
     module = sys.modules[__name__]