diff options
author | 2020-11-24 09:51:56 -0800 | |
---|---|---|
committer | 2020-11-24 09:51:56 -0800 | |
commit | ec5491853daed27f2481fc60cf4bdb02ecfddbed (patch) | |
tree | cd4653e0cbc9fe137073bfbe5c0b9e6fa1cc2f7a | |
parent | a43c85a54c7705b6a59a5a44f95b97346ef1196c (diff) |
[infra] Add retry decorator and use it. (#4702)
-rw-r--r-- | infra/build_specified_commit.py | 15 | ||||
-rw-r--r-- | infra/retry.py | 108 |
2 files changed, 112 insertions, 11 deletions
diff --git a/infra/build_specified_commit.py b/infra/build_specified_commit.py index eef671b7..9a0af01e 100644 --- a/infra/build_specified_commit.py +++ b/infra/build_specified_commit.py @@ -28,10 +28,10 @@ import logging import re import shutil import tempfile -import time import helper import repo_manager +import retry import utils BuildData = collections.namedtuple( @@ -39,7 +39,6 @@ BuildData = collections.namedtuple( _GIT_DIR_MARKER = 'gitdir: ' _IMAGE_BUILD_TRIES = 3 -_IMAGE_BUILD_RETRY_SLEEP = 30.0 class BaseBuilderRepo: @@ -144,17 +143,11 @@ def copy_src_from_docker(project_name, host_dir): return src_dir +@retry.wrap(_IMAGE_BUILD_TRIES, 2, + 'infra.build_specified_commit._build_image_with_retries') def _build_image_with_retries(project_name): """Build image with retries.""" - - for _ in range(_IMAGE_BUILD_TRIES): - result = helper.build_image_impl(project_name) - if result: - return result - - time.sleep(_IMAGE_BUILD_RETRY_SLEEP) - - return result + return helper.build_image_impl(project_name) def get_required_post_checkout_steps(dockerfile_path): diff --git a/infra/retry.py b/infra/retry.py new file mode 100644 index 00000000..4205319e --- /dev/null +++ b/infra/retry.py @@ -0,0 +1,108 @@ +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""Retry decorator. Copied from ClusterFuzz source.""" + +import functools +import inspect +import logging +import sys +import time + +# pylint: disable=too-many-arguments,broad-except + + +def sleep(seconds): + """Invoke time.sleep. This is to avoid the flakiness of time.sleep. See: + crbug.com/770375""" + time.sleep(seconds) + + +def get_delay(num_try, delay, backoff): + """Compute backoff delay.""" + return delay * (backoff**(num_try - 1)) + + +def wrap(retries, + delay, + function, + backoff=2, + exception_type=Exception, + retry_on_false=False): + """Retry decorator for a function.""" + + assert delay > 0 + assert backoff >= 1 + assert retries >= 0 + + def decorator(func): + """Decorator for the given function.""" + tries = retries + 1 + is_generator = inspect.isgeneratorfunction(func) + function_with_type = function + if is_generator: + function_with_type += ' (generator)' + + def handle_retry(num_try, exception=None): + """Handle retry.""" + if (exception is None or + isinstance(exception, exception_type)) and num_try < tries: + logging.log('Retrying on %s failed with %s. Retrying again.', + function_with_type, + sys.exc_info()[1]) + sleep(get_delay(num_try, delay, backoff)) + return True + + logging.log_error('Retrying on %s failed with %s. Raise.' % + (function_with_type, sys.exc_info()[1]), + total=tries) + return False + + @functools.wraps(func) + def _wrapper(*args, **kwargs): + """Regular function wrapper.""" + for num_try in range(1, tries + 1): + try: + result = func(*args, **kwargs) + if retry_on_false and not result: + if not handle_retry(num_try): + return result + + continue + return result + except Exception as error: + if not handle_retry(num_try, exception=error): + raise + + @functools.wraps(func) + def _generator_wrapper(*args, **kwargs): + """Generator function wrapper.""" + # This argument is not applicable for generator functions. + assert not retry_on_false + already_yielded_element_count = 0 + for num_try in range(1, tries + 1): + try: + for index, result in enumerate(func(*args, **kwargs)): + if index >= already_yielded_element_count: + yield result + already_yielded_element_count += 1 + break + except Exception as error: + if not handle_retry(num_try, exception=error): + raise + + if is_generator: + return _generator_wrapper + return _wrapper + + return decorator |