diff options
Diffstat (limited to 'tools/closure_linter-2.3.4/closure_linter/closurizednamespacesinfo.py')
| -rwxr-xr-x | tools/closure_linter-2.3.4/closure_linter/closurizednamespacesinfo.py | 498 |
1 files changed, 498 insertions, 0 deletions
diff --git a/tools/closure_linter-2.3.4/closure_linter/closurizednamespacesinfo.py b/tools/closure_linter-2.3.4/closure_linter/closurizednamespacesinfo.py new file mode 100755 index 0000000..065b5cc --- /dev/null +++ b/tools/closure_linter-2.3.4/closure_linter/closurizednamespacesinfo.py @@ -0,0 +1,498 @@ +#!/usr/bin/env python +# +# Copyright 2008 The Closure Linter Authors. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS-IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Logic for computing dependency information for closurized JavaScript files. + +Closurized JavaScript files express dependencies using goog.require and +goog.provide statements. In order for the linter to detect when a statement is +missing or unnecessary, all identifiers in the JavaScript file must first be +processed to determine if they constitute the creation or usage of a dependency. +""" + + + +from closure_linter import javascripttokens +from closure_linter import tokenutil + +# pylint: disable-msg=C6409 +TokenType = javascripttokens.JavaScriptTokenType + +DEFAULT_EXTRA_NAMESPACES = [ + 'goog.testing.asserts', + 'goog.testing.jsunit', +] + +class ClosurizedNamespacesInfo(object): + """Dependency information for closurized JavaScript files. + + Processes token streams for dependency creation or usage and provides logic + for determining if a given require or provide statement is unnecessary or if + there are missing require or provide statements. + """ + + def __init__(self, closurized_namespaces, ignored_extra_namespaces): + """Initializes an instance the ClosurizedNamespacesInfo class. + + Args: + closurized_namespaces: A list of namespace prefixes that should be + processed for dependency information. Non-matching namespaces are + ignored. + ignored_extra_namespaces: A list of namespaces that should not be reported + as extra regardless of whether they are actually used. + """ + self._closurized_namespaces = closurized_namespaces + self._ignored_extra_namespaces = (ignored_extra_namespaces + + DEFAULT_EXTRA_NAMESPACES) + self.Reset() + + def Reset(self): + """Resets the internal state to prepare for processing a new file.""" + + # A list of goog.provide tokens in the order they appeared in the file. + self._provide_tokens = [] + + # A list of goog.require tokens in the order they appeared in the file. + self._require_tokens = [] + + # Namespaces that are already goog.provided. + self._provided_namespaces = [] + + # Namespaces that are already goog.required. + self._required_namespaces = [] + + # Note that created_namespaces and used_namespaces contain both namespaces + # and identifiers because there are many existing cases where a method or + # constant is provided directly instead of its namespace. Ideally, these + # two lists would only have to contain namespaces. + + # A list of tuples where the first element is the namespace of an identifier + # created in the file and the second is the identifier itself. + self._created_namespaces = [] + + # A list of tuples where the first element is the namespace of an identifier + # used in the file and the second is the identifier itself. + self._used_namespaces = [] + + # A list of seemingly-unnecessary namespaces that are goog.required() and + # annotated with @suppress {extraRequire}. + self._suppressed_requires = [] + + # A list of goog.provide tokens which are duplicates. + self._duplicate_provide_tokens = [] + + # A list of goog.require tokens which are duplicates. + self._duplicate_require_tokens = [] + + # Whether this file is in a goog.scope. Someday, we may add support + # for checking scopified namespaces, but for now let's just fail + # in a more reasonable way. + self._scopified_file = False + + # TODO(user): Handle the case where there are 2 different requires + # that can satisfy the same dependency, but only one is necessary. + + def GetProvidedNamespaces(self): + """Returns the namespaces which are already provided by this file. + + Returns: + A list of strings where each string is a 'namespace' corresponding to an + existing goog.provide statement in the file being checked. + """ + return list(self._provided_namespaces) + + def GetRequiredNamespaces(self): + """Returns the namespaces which are already required by this file. + + Returns: + A list of strings where each string is a 'namespace' corresponding to an + existing goog.require statement in the file being checked. + """ + return list(self._required_namespaces) + + def IsExtraProvide(self, token): + """Returns whether the given goog.provide token is unnecessary. + + Args: + token: A goog.provide token. + + Returns: + True if the given token corresponds to an unnecessary goog.provide + statement, otherwise False. + """ + if self._scopified_file: + return False + + namespace = tokenutil.Search(token, TokenType.STRING_TEXT).string + + base_namespace = namespace.split('.', 1)[0] + if base_namespace not in self._closurized_namespaces: + return False + + if token in self._duplicate_provide_tokens: + return True + + # TODO(user): There's probably a faster way to compute this. + for created_namespace, created_identifier in self._created_namespaces: + if namespace == created_namespace or namespace == created_identifier: + return False + + return True + + def IsExtraRequire(self, token): + """Returns whether the given goog.require token is unnecessary. + + Args: + token: A goog.require token. + + Returns: + True if the given token corresponds to an unnecessary goog.require + statement, otherwise False. + """ + if self._scopified_file: + return False + + namespace = tokenutil.Search(token, TokenType.STRING_TEXT).string + + base_namespace = namespace.split('.', 1)[0] + if base_namespace not in self._closurized_namespaces: + return False + + if namespace in self._ignored_extra_namespaces: + return False + + if token in self._duplicate_require_tokens: + return True + + if namespace in self._suppressed_requires: + return False + + # If the namespace contains a component that is initial caps, then that + # must be the last component of the namespace. + parts = namespace.split('.') + if len(parts) > 1 and parts[-2][0].isupper(): + return True + + # TODO(user): There's probably a faster way to compute this. + for used_namespace, used_identifier in self._used_namespaces: + if namespace == used_namespace or namespace == used_identifier: + return False + + return True + + def GetMissingProvides(self): + """Returns the set of missing provided namespaces for the current file. + + Returns: + Returns a set of strings where each string is a namespace that should be + provided by this file, but is not. + """ + if self._scopified_file: + return set() + + missing_provides = set() + for namespace, identifier in self._created_namespaces: + if (not self._IsPrivateIdentifier(identifier) and + namespace not in self._provided_namespaces and + identifier not in self._provided_namespaces and + namespace not in self._required_namespaces): + missing_provides.add(namespace) + + return missing_provides + + def GetMissingRequires(self): + """Returns the set of missing required namespaces for the current file. + + For each non-private identifier used in the file, find either a + goog.require, goog.provide or a created identifier that satisfies it. + goog.require statements can satisfy the identifier by requiring either the + namespace of the identifier or the identifier itself. goog.provide + statements can satisfy the identifier by providing the namespace of the + identifier. A created identifier can only satisfy the used identifier if + it matches it exactly (necessary since things can be defined on a + namespace in more than one file). Note that provided namespaces should be + a subset of created namespaces, but we check both because in some cases we + can't always detect the creation of the namespace. + + Returns: + Returns a set of strings where each string is a namespace that should be + required by this file, but is not. + """ + if self._scopified_file: + return set() + + external_dependencies = set(self._required_namespaces) + + # Assume goog namespace is always available. + external_dependencies.add('goog') + + created_identifiers = set() + for namespace, identifier in self._created_namespaces: + created_identifiers.add(identifier) + + missing_requires = set() + for namespace, identifier in self._used_namespaces: + if (not self._IsPrivateIdentifier(identifier) and + namespace not in external_dependencies and + namespace not in self._provided_namespaces and + identifier not in external_dependencies and + identifier not in created_identifiers): + missing_requires.add(namespace) + + return missing_requires + + def _IsPrivateIdentifier(self, identifier): + """Returns whether the given identifer is private.""" + pieces = identifier.split('.') + for piece in pieces: + if piece.endswith('_'): + return True + return False + + def IsFirstProvide(self, token): + """Returns whether token is the first provide token.""" + return self._provide_tokens and token == self._provide_tokens[0] + + def IsFirstRequire(self, token): + """Returns whether token is the first require token.""" + return self._require_tokens and token == self._require_tokens[0] + + def IsLastProvide(self, token): + """Returns whether token is the last provide token.""" + return self._provide_tokens and token == self._provide_tokens[-1] + + def IsLastRequire(self, token): + """Returns whether token is the last require token.""" + return self._require_tokens and token == self._require_tokens[-1] + + def ProcessToken(self, token, state_tracker): + """Processes the given token for dependency information. + + Args: + token: The token to process. + state_tracker: The JavaScript state tracker. + """ + + # Note that this method is in the critical path for the linter and has been + # optimized for performance in the following ways: + # - Tokens are checked by type first to minimize the number of function + # calls necessary to determine if action needs to be taken for the token. + # - The most common tokens types are checked for first. + # - The number of function calls has been minimized (thus the length of this + # function. + + if token.type == TokenType.IDENTIFIER: + # TODO(user): Consider saving the whole identifier in metadata. + whole_identifier_string = self._GetWholeIdentifierString(token) + if whole_identifier_string is None: + # We only want to process the identifier one time. If the whole string + # identifier is None, that means this token was part of a multi-token + # identifier, but it was not the first token of the identifier. + return + + # In the odd case that a goog.require is encountered inside a function, + # just ignore it (e.g. dynamic loading in test runners). + if token.string == 'goog.require' and not state_tracker.InFunction(): + self._require_tokens.append(token) + namespace = tokenutil.Search(token, TokenType.STRING_TEXT).string + if namespace in self._required_namespaces: + self._duplicate_require_tokens.append(token) + else: + self._required_namespaces.append(namespace) + + # If there is a suppression for the require, add a usage for it so it + # gets treated as a regular goog.require (i.e. still gets sorted). + jsdoc = state_tracker.GetDocComment() + if jsdoc and ('extraRequire' in jsdoc.suppressions): + self._suppressed_requires.append(namespace) + self._AddUsedNamespace(state_tracker, namespace) + + elif token.string == 'goog.provide': + self._provide_tokens.append(token) + namespace = tokenutil.Search(token, TokenType.STRING_TEXT).string + if namespace in self._provided_namespaces: + self._duplicate_provide_tokens.append(token) + else: + self._provided_namespaces.append(namespace) + + # If there is a suppression for the provide, add a creation for it so it + # gets treated as a regular goog.provide (i.e. still gets sorted). + jsdoc = state_tracker.GetDocComment() + if jsdoc and ('extraProvide' in jsdoc.suppressions): + self._AddCreatedNamespace(state_tracker, namespace) + + elif token.string == 'goog.scope': + self._scopified_file = True + + else: + jsdoc = state_tracker.GetDocComment() + if jsdoc and jsdoc.HasFlag('typedef'): + self._AddCreatedNamespace(state_tracker, whole_identifier_string) + else: + self._AddUsedNamespace(state_tracker, whole_identifier_string) + + elif token.type == TokenType.SIMPLE_LVALUE: + identifier = token.values['identifier'] + namespace = self._GetClosurizedNamespace(identifier) + if state_tracker.InFunction(): + self._AddUsedNamespace(state_tracker, identifier) + elif namespace and namespace != 'goog': + self._AddCreatedNamespace(state_tracker, identifier, namespace) + + elif token.type == TokenType.DOC_FLAG: + flag_type = token.attached_object.flag_type + is_interface = state_tracker.GetDocComment().HasFlag('interface') + if flag_type == 'implements' or (flag_type == 'extends' and is_interface): + # Interfaces should be goog.require'd. + doc_start = tokenutil.Search(token, TokenType.DOC_START_BRACE) + interface = tokenutil.Search(doc_start, TokenType.COMMENT) + self._AddUsedNamespace(state_tracker, interface.string) + + + def _GetWholeIdentifierString(self, token): + """Returns the whole identifier string for the given token. + + Checks the tokens after the current one to see if the token is one in a + sequence of tokens which are actually just one identifier (i.e. a line was + wrapped in the middle of an identifier). + + Args: + token: The token to check. + + Returns: + The whole identifier string or None if this token is not the first token + in a multi-token identifier. + """ + result = '' + + # Search backward to determine if this token is the first token of the + # identifier. If it is not the first token, return None to signal that this + # token should be ignored. + prev_token = token.previous + while prev_token: + if (prev_token.IsType(TokenType.IDENTIFIER) or + prev_token.IsType(TokenType.NORMAL) and prev_token.string == '.'): + return None + elif (not prev_token.IsType(TokenType.WHITESPACE) and + not prev_token.IsAnyType(TokenType.COMMENT_TYPES)): + break + prev_token = prev_token.previous + + # Search forward to find other parts of this identifier separated by white + # space. + next_token = token + while next_token: + if (next_token.IsType(TokenType.IDENTIFIER) or + next_token.IsType(TokenType.NORMAL) and next_token.string == '.'): + result += next_token.string + elif (not next_token.IsType(TokenType.WHITESPACE) and + not next_token.IsAnyType(TokenType.COMMENT_TYPES)): + break + next_token = next_token.next + + return result + + def _AddCreatedNamespace(self, state_tracker, identifier, namespace=None): + """Adds the namespace of an identifier to the list of created namespaces. + + If the identifier is annotated with a 'missingProvide' suppression, it is + not added. + + Args: + state_tracker: The JavaScriptStateTracker instance. + identifier: The identifier to add. + namespace: The namespace of the identifier or None if the identifier is + also the namespace. + """ + if not namespace: + namespace = identifier + + jsdoc = state_tracker.GetDocComment() + if jsdoc and 'missingProvide' in jsdoc.suppressions: + return + + self._created_namespaces.append([namespace, identifier]) + + def _AddUsedNamespace(self, state_tracker, identifier): + """Adds the namespace of an identifier to the list of used namespaces. + + If the identifier is annotated with a 'missingRequire' suppression, it is + not added. + + Args: + state_tracker: The JavaScriptStateTracker instance. + identifier: An identifier which has been used. + """ + jsdoc = state_tracker.GetDocComment() + if jsdoc and 'missingRequire' in jsdoc.suppressions: + return + + namespace = self._GetClosurizedNamespace(identifier) + if namespace: + self._used_namespaces.append([namespace, identifier]) + + def _GetClosurizedNamespace(self, identifier): + """Given an identifier, returns the namespace that identifier is from. + + Args: + identifier: The identifier to extract a namespace from. + + Returns: + The namespace the given identifier resides in, or None if one could not + be found. + """ + if identifier.startswith('goog.global'): + # Ignore goog.global, since it is, by definition, global. + return None + + parts = identifier.split('.') + for namespace in self._closurized_namespaces: + if not identifier.startswith(namespace + '.'): + continue + + last_part = parts[-1] + if not last_part: + # TODO(robbyw): Handle this: it's a multi-line identifier. + return None + + # The namespace for a class is the shortest prefix ending in a class + # name, which starts with a capital letter but is not a capitalized word. + # + # We ultimately do not want to allow requiring or providing of inner + # classes/enums. Instead, a file should provide only the top-level class + # and users should require only that. + namespace = [] + for part in parts: + if part == 'prototype' or part.isupper(): + return '.'.join(namespace) + namespace.append(part) + if part[0].isupper(): + return '.'.join(namespace) + + # At this point, we know there's no class or enum, so the namespace is + # just the identifier with the last part removed. With the exception of + # apply, inherits, and call, which should also be stripped. + if parts[-1] in ('apply', 'inherits', 'call'): + parts.pop() + parts.pop() + + # If the last part ends with an underscore, it is a private variable, + # method, or enum. The namespace is whatever is before it. + if parts and parts[-1].endswith('_'): + parts.pop() + + return '.'.join(parts) + + return None |