aboutsummaryrefslogtreecommitdiffhomepage
path: root/tensorflow/tools
diff options
context:
space:
mode:
Diffstat (limited to 'tensorflow/tools')
-rw-r--r--tensorflow/tools/api/tests/api_compatibility_test.py2
-rw-r--r--tensorflow/tools/common/public_api.py54
-rw-r--r--tensorflow/tools/docs/doc_generator_visitor.py2
-rw-r--r--tensorflow/tools/docs/generate_1_0.py18
-rw-r--r--tensorflow/tools/docs/generate_lib.py40
-rw-r--r--tensorflow/tools/docs/generate_lib_test.py5
6 files changed, 82 insertions, 39 deletions
diff --git a/tensorflow/tools/api/tests/api_compatibility_test.py b/tensorflow/tools/api/tests/api_compatibility_test.py
index acd119c730..1ffa8fc26c 100644
--- a/tensorflow/tools/api/tests/api_compatibility_test.py
+++ b/tensorflow/tools/api/tests/api_compatibility_test.py
@@ -198,7 +198,7 @@ class ApiCompatibilityTest(test.TestCase):
visitor = python_object_to_proto_visitor.PythonObjectToProtoVisitor()
public_api_visitor = public_api.PublicAPIVisitor(visitor)
- public_api_visitor.do_not_descend_map[''].append('contrib')
+ public_api_visitor.do_not_descend_map['tf'].append('contrib')
traverse.traverse(tf, public_api_visitor)
proto_dict = visitor.GetProtos()
diff --git a/tensorflow/tools/common/public_api.py b/tensorflow/tools/common/public_api.py
index cab3b2ff6a..e0acead919 100644
--- a/tensorflow/tools/common/public_api.py
+++ b/tensorflow/tools/common/public_api.py
@@ -36,12 +36,20 @@ class PublicAPIVisitor(object):
visitor: A visitor to call for the public API.
"""
self._visitor = visitor
+ self._root_name = 'tf'
+
+ # Modules/classes we want to suppress entirely.
+ self._private_map = {
+ # Some implementations have this internal module that we shouldn't
+ # expose.
+ 'tf.flags': ['cpp_flags'],
+ }
# Modules/classes we do not want to descend into if we hit them. Usually,
# system modules exposed through platforms for compatibility reasons.
# Each entry maps a module path to a name to ignore in traversal.
self._do_not_descend_map = {
- '': [
+ 'tf': [
'core',
'examples',
'flags', # Don't add flags
@@ -56,18 +64,26 @@ class PublicAPIVisitor(object):
'tensorboard',
],
- # Some implementations have this internal module that we shouldn't
- # expose.
- 'flags': ['cpp_flags'],
-
## Everything below here is legitimate.
# It'll stay, but it's not officially part of the API.
- 'app': ['flags'],
+ 'tf.app': ['flags'],
# Imported for compatibility between py2/3.
- 'test': ['mock'],
+ 'tf.test': ['mock'],
}
@property
+ def private_map(self):
+ """A map from parents to symbols that should not be included at all.
+
+ This map can be edited, but it should not be edited once traversal has
+ begun.
+
+ Returns:
+ The map marking symbols to not include.
+ """
+ return self._private_map
+
+ @property
def do_not_descend_map(self):
"""A map from parents to symbols that should not be descended into.
@@ -79,11 +95,17 @@ class PublicAPIVisitor(object):
"""
return self._do_not_descend_map
- def _isprivate(self, name):
+ def set_root_name(self, root_name):
+ """Override the default root name of 'tf'."""
+ self._root_name = root_name
+
+ def _is_private(self, path, name):
"""Return whether a name is private."""
# TODO(wicke): Find out what names to exclude.
- return (name.startswith('_') and not re.match('__.*__$', name) or
- name in ['__base__', '__class__'])
+ return ((path in self._private_map and
+ name in self._private_map[path]) or
+ (name.startswith('_') and not re.match('__.*__$', name) or
+ name in ['__base__', '__class__']))
def _do_not_descend(self, path, name):
"""Safely queries if a specific fully qualified name should be excluded."""
@@ -95,17 +117,21 @@ class PublicAPIVisitor(object):
# Avoid long waits in cases of pretty unambiguous failure.
if tf_inspect.ismodule(parent) and len(path.split('.')) > 10:
- raise RuntimeError('Modules nested too deep:\n%s\n\nThis is likely a '
- 'problem with an accidental public import.' % path)
+ raise RuntimeError('Modules nested too deep:\n%s.%s\n\nThis is likely a '
+ 'problem with an accidental public import.' %
+ (self._root_name, path))
+
+ # Includes self._root_name
+ full_path = '.'.join([self._root_name, path]) if path else self._root_name
# Remove things that are not visible.
for name, child in list(children):
- if self._isprivate(name):
+ if self._is_private(full_path, name):
children.remove((name, child))
self._visitor(path, parent, children)
# Remove things that are visible, but which should not be descended into.
for name, child in list(children):
- if self._do_not_descend(path, name):
+ if self._do_not_descend(full_path, name):
children.remove((name, child))
diff --git a/tensorflow/tools/docs/doc_generator_visitor.py b/tensorflow/tools/docs/doc_generator_visitor.py
index 8f7b91fa75..259a4694fd 100644
--- a/tensorflow/tools/docs/doc_generator_visitor.py
+++ b/tensorflow/tools/docs/doc_generator_visitor.py
@@ -30,7 +30,7 @@ class DocGeneratorVisitor(object):
"""Make a visitor.
As this visitor is starting its traversal at a module or class, it will not
- be old the name of that object during traversal. `root_name` is the name it
+ be told the name of that object during traversal. `root_name` is the name it
should use for that object, effectively prefixing all names with
"root_name.".
diff --git a/tensorflow/tools/docs/generate_1_0.py b/tensorflow/tools/docs/generate_1_0.py
index ddafcebd11..cdc03fdcac 100644
--- a/tensorflow/tools/docs/generate_1_0.py
+++ b/tensorflow/tools/docs/generate_1_0.py
@@ -47,8 +47,8 @@ if __name__ == '__main__':
doc_generator.set_py_modules([('tf', tf), ('tfdbg', tf_debug)])
doc_generator.set_do_not_descend_map({
- '': ['cli', 'lib', 'wrappers'],
- 'contrib': [
+ 'tf': ['cli', 'lib', 'wrappers'],
+ 'tf.contrib': [
'compiler',
'factorization',
'grid_rnn',
@@ -65,18 +65,18 @@ if __name__ == '__main__':
'training',
'tfprof',
],
- 'contrib.bayesflow': [
+ 'tf.contrib.bayesflow': [
'entropy', 'monte_carlo', 'special_math',
'stochastic_gradient_estimators', 'stochastic_graph',
'stochastic_tensor', 'stochastic_variables', 'variational_inference'
],
- 'contrib.distributions': ['bijector'],
- 'contrib.ffmpeg': ['ffmpeg_ops'],
- 'contrib.graph_editor': [
+ 'tf.contrib.distributions': ['bijector'],
+ 'tf.contrib.ffmpeg': ['ffmpeg_ops'],
+ 'tf.contrib.graph_editor': [
'edit', 'match', 'reroute', 'subgraph', 'transform', 'select', 'util'
],
- 'contrib.layers': ['feature_column', 'summaries'],
- 'contrib.learn': [
+ 'tf.contrib.layers': ['feature_column', 'summaries'],
+ 'tf.contrib.learn': [
'datasets',
'head',
'graph_actions',
@@ -87,7 +87,7 @@ if __name__ == '__main__':
'preprocessing',
'utils',
],
- 'contrib.util': ['loader'],
+ 'tf.contrib.util': ['loader'],
})
sys.exit(doc_generator.build(flags))
diff --git a/tensorflow/tools/docs/generate_lib.py b/tensorflow/tools/docs/generate_lib.py
index 3494c7f8c5..99872e1d84 100644
--- a/tensorflow/tools/docs/generate_lib.py
+++ b/tensorflow/tools/docs/generate_lib.py
@@ -180,11 +180,16 @@ def add_dict_to_dict(add_from, add_to):
# Exclude some libaries in contrib from the documentation altogether.
+def _get_default_private_map():
+ return {}
+
+
+# Exclude members of some libaries.
def _get_default_do_not_descend_map():
- # TODO(wicke): Shrink this list.
+ # TODO(wicke): Shrink this list once the modules get sealed.
return {
- '': ['cli', 'lib', 'wrappers'],
- 'contrib': [
+ 'tf': ['cli', 'lib', 'wrappers'],
+ 'tf.contrib': [
'compiler',
'factorization',
'grid_rnn',
@@ -200,17 +205,17 @@ def _get_default_do_not_descend_map():
'testing',
'tfprof',
],
- 'contrib.bayesflow': [
+ 'tf.contrib.bayesflow': [
'special_math', 'stochastic_gradient_estimators',
'stochastic_variables'
],
- 'contrib.ffmpeg': ['ffmpeg_ops'],
- 'contrib.graph_editor': [
+ 'tf.contrib.ffmpeg': ['ffmpeg_ops'],
+ 'tf.contrib.graph_editor': [
'edit', 'match', 'reroute', 'subgraph', 'transform', 'select', 'util'
],
- 'contrib.keras': ['api', 'python'],
- 'contrib.layers': ['feature_column', 'summaries'],
- 'contrib.learn': [
+ 'tf.contrib.keras': ['api', 'python'],
+ 'tf.contrib.layers': ['feature_column', 'summaries'],
+ 'tf.contrib.learn': [
'datasets',
'head',
'graph_actions',
@@ -221,15 +226,17 @@ def _get_default_do_not_descend_map():
'preprocessing',
'utils',
],
- 'contrib.util': ['loader'],
+ 'tf.contrib.util': ['loader'],
}
-def extract(py_modules, do_not_descend_map):
+def extract(py_modules, private_map, do_not_descend_map):
"""Extract docs from tf namespace and write them to disk."""
# Traverse the first module.
visitor = doc_generator_visitor.DocGeneratorVisitor(py_modules[0][0])
api_visitor = public_api.PublicAPIVisitor(visitor)
+ api_visitor.set_root_name(py_modules[0][0])
+ add_dict_to_dict(private_map, api_visitor.private_map)
add_dict_to_dict(do_not_descend_map, api_visitor.do_not_descend_map)
traverse.traverse(py_modules[0][1], api_visitor)
@@ -237,6 +244,7 @@ def extract(py_modules, do_not_descend_map):
# Traverse all py_modules after the first:
for module_name, module in py_modules[1:]:
visitor.set_root_name(module_name)
+ api_visitor.set_root_name(module_name)
traverse.traverse(module, api_visitor)
return visitor
@@ -409,6 +417,7 @@ class DocGenerator(object):
def __init__(self):
self.argument_parser = argparse.ArgumentParser()
self._py_modules = None
+ self._private_map = _get_default_private_map()
self._do_not_descend_map = _get_default_do_not_descend_map()
self.yaml_toc = True
@@ -439,9 +448,15 @@ class DocGenerator(object):
flags, _ = self.argument_parser.parse_known_args()
return flags
+ def add_to_private_map(self, d):
+ add_dict_to_dict(d, self._private_map)
+
def add_to_do_not_descend_map(self, d):
add_dict_to_dict(d, self._do_not_descend_map)
+ def set_private_map(self, d):
+ self._private_map = d
+
def set_do_not_descend_map(self, d):
self._do_not_descend_map = d
@@ -471,7 +486,8 @@ class DocGenerator(object):
base_dir=base_dir)
def run_extraction(self):
- return extract(self._py_modules, self._do_not_descend_map)
+ return extract(
+ self._py_modules, self._private_map, self._do_not_descend_map)
def build(self, flags):
"""Actually build the docs."""
diff --git a/tensorflow/tools/docs/generate_lib_test.py b/tensorflow/tools/docs/generate_lib_test.py
index 49f7ff597e..6e5deb6a36 100644
--- a/tensorflow/tools/docs/generate_lib_test.py
+++ b/tensorflow/tools/docs/generate_lib_test.py
@@ -58,8 +58,9 @@ class GenerateTest(googletest.TestCase):
py_modules = [('tf', tf), ('tfdbg', tf_debug)]
try:
- generate_lib.extract(
- py_modules, generate_lib._get_default_do_not_descend_map())
+ generate_lib.extract(py_modules,
+ generate_lib._get_default_private_map(),
+ generate_lib._get_default_do_not_descend_map())
except RuntimeError:
print('*****************************************************************')
print('If this test fails, you have most likely introduced an unsealed')
generated by cgit v1.2.3 (git 2.39.1) at 2024-06-26 17:40:39 +0000