From 6466c35737eff21e9b48c3ce2353d42628f4bb77 Mon Sep 17 00:00:00 2001 From: Konstantin Varlamov Date: Tue, 10 Jul 2018 17:45:16 -0400 Subject: C++ migration: add a C++ implementation of `FSTExponentialBackoff` (#1465) This is a pretty close port of `FSTExponentialBackoff`. The changes are pretty minor: * delay is calculated using duration types, not plain numbers, which should be a little more type-safe; * split a piece of code into a ClampDelay function, because it's reasonably close to std::clamp; * rephrased the class-level comment to make it clearer that the first attempt always has delay = 0; * added simple tests (other platforms don't have tests for this). Also make sure that canceling a DelayedOperation is always valid. --- .../Example/Firestore.xcodeproj/project.pbxproj | 8 +- .../src/firebase/firestore/remote/CMakeLists.txt | 2 + .../firestore/remote/exponential_backoff.cc | 93 +++++++++++++++++ .../firestore/remote/exponential_backoff.h | 116 +++++++++++++++++++++ .../core/src/firebase/firestore/util/executor.h | 4 +- .../test/firebase/firestore/remote/CMakeLists.txt | 3 + .../firestore/remote/exponential_backoff_test.cc | 91 ++++++++++++++++ .../test/firebase/firestore/util/executor_test.cc | 24 +++++ 8 files changed, 338 insertions(+), 3 deletions(-) create mode 100644 Firestore/core/src/firebase/firestore/remote/exponential_backoff.cc create mode 100644 Firestore/core/src/firebase/firestore/remote/exponential_backoff.h create mode 100644 Firestore/core/test/firebase/firestore/remote/exponential_backoff_test.cc (limited to 'Firestore') diff --git a/Firestore/Example/Firestore.xcodeproj/project.pbxproj b/Firestore/Example/Firestore.xcodeproj/project.pbxproj index 7fdb444..6729eab 100644 --- a/Firestore/Example/Firestore.xcodeproj/project.pbxproj +++ b/Firestore/Example/Firestore.xcodeproj/project.pbxproj @@ -194,6 +194,7 @@ B65D34A9203C995B0076A5E1 /* FIRTimestampTest.m in Sources */ = {isa = PBXBuildFile; fileRef = B65D34A7203C99090076A5E1 /* FIRTimestampTest.m */; }; B686F2AF2023DDEE0028D6BE /* field_path_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = B686F2AD2023DDB20028D6BE /* field_path_test.cc */; }; B686F2B22025000D0028D6BE /* resource_path_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = B686F2B02024FFD70028D6BE /* resource_path_test.cc */; }; + B6D1B68520E2AB1B00B35856 /* exponential_backoff_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = B6D1B68420E2AB1A00B35856 /* exponential_backoff_test.cc */; }; B6FB467D208E9D3C00554BA2 /* async_queue_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = B6FB467B208E9A8200554BA2 /* async_queue_test.cc */; }; B6FB4684208EA0EC00554BA2 /* async_queue_libdispatch_test.mm in Sources */ = {isa = PBXBuildFile; fileRef = B6FB4680208EA0BE00554BA2 /* async_queue_libdispatch_test.mm */; }; B6FB4685208EA0F000554BA2 /* async_queue_std_test.cc in Sources */ = {isa = PBXBuildFile; fileRef = B6FB4681208EA0BE00554BA2 /* async_queue_std_test.cc */; }; @@ -284,7 +285,7 @@ 132E36BB104830BD806351AC /* FSTLevelDBTransactionTests.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = FSTLevelDBTransactionTests.mm; sourceTree = ""; }; 2A0CF41BA5AED6049B0BEB2C /* type_traits_apple_test.mm */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.objcpp; path = type_traits_apple_test.mm; sourceTree = ""; }; 2B50B3A0DF77100EEE887891 /* Pods_Firestore_Tests_iOS.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Pods_Firestore_Tests_iOS.framework; sourceTree = BUILT_PRODUCTS_DIR; }; - 358C3B5FE573B1D60A4F7592 /* strerror_test.cc */ = {isa = PBXFileReference; includeInIndex = 1; path = strerror_test.cc; sourceTree = ""; }; + 358C3B5FE573B1D60A4F7592 /* strerror_test.cc */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; path = strerror_test.cc; sourceTree = ""; }; 379B34A1536045869826D82A /* Pods_Firestore_Example_iOS_SwiftBuildTest.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Pods_Firestore_Example_iOS_SwiftBuildTest.framework; sourceTree = BUILT_PRODUCTS_DIR; }; 3B843E4A1F3930A400548890 /* remote_store_spec_test.json */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = text.json; path = remote_store_spec_test.json; sourceTree = ""; }; 3C81DE3772628FE297055662 /* Pods-Firestore_Example_iOS.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-Firestore_Example_iOS.debug.xcconfig"; path = "Pods/Target Support Files/Pods-Firestore_Example_iOS/Pods-Firestore_Example_iOS.debug.xcconfig"; sourceTree = ""; }; @@ -487,6 +488,7 @@ B65D34A7203C99090076A5E1 /* FIRTimestampTest.m */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.objc; path = FIRTimestampTest.m; sourceTree = ""; }; B686F2AD2023DDB20028D6BE /* field_path_test.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = field_path_test.cc; sourceTree = ""; }; B686F2B02024FFD70028D6BE /* resource_path_test.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = resource_path_test.cc; sourceTree = ""; }; + B6D1B68420E2AB1A00B35856 /* exponential_backoff_test.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = exponential_backoff_test.cc; sourceTree = ""; }; B6FB467A208E9A8200554BA2 /* async_queue_test.h */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.c.h; path = async_queue_test.h; sourceTree = ""; }; B6FB467B208E9A8200554BA2 /* async_queue_test.cc */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.cpp; path = async_queue_test.cc; sourceTree = ""; }; B6FB4680208EA0BE00554BA2 /* async_queue_libdispatch_test.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = async_queue_libdispatch_test.mm; sourceTree = ""; }; @@ -519,7 +521,7 @@ E592181BFD7C53C305123739 /* Pods-Firestore_Tests_iOS.debug.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-Firestore_Tests_iOS.debug.xcconfig"; path = "Pods/Target Support Files/Pods-Firestore_Tests_iOS/Pods-Firestore_Tests_iOS.debug.xcconfig"; sourceTree = ""; }; ECEBABC7E7B693BE808A1052 /* Pods_Firestore_IntegrationTests_iOS.framework */ = {isa = PBXFileReference; explicitFileType = wrapper.framework; includeInIndex = 0; path = Pods_Firestore_IntegrationTests_iOS.framework; sourceTree = BUILT_PRODUCTS_DIR; }; F354C0FE92645B56A6C6FD44 /* Pods-Firestore_IntegrationTests_iOS.release.xcconfig */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = text.xcconfig; name = "Pods-Firestore_IntegrationTests_iOS.release.xcconfig"; path = "Pods/Target Support Files/Pods-Firestore_IntegrationTests_iOS/Pods-Firestore_IntegrationTests_iOS.release.xcconfig"; sourceTree = ""; }; - F8043813A5D16963EC02B182 /* local_serializer_test.cc */ = {isa = PBXFileReference; includeInIndex = 1; path = local_serializer_test.cc; sourceTree = ""; }; + F8043813A5D16963EC02B182 /* local_serializer_test.cc */ = {isa = PBXFileReference; includeInIndex = 1; lastKnownFileType = sourcecode.cpp.cpp; path = local_serializer_test.cc; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -608,6 +610,7 @@ 546854A720A3681B004BDBD5 /* remote */ = { isa = PBXGroup; children = ( + B6D1B68420E2AB1A00B35856 /* exponential_backoff_test.cc */, 546854A820A36867004BDBD5 /* datastore_test.cc */, 61F72C5520BC48FD001A68CB /* serializer_test.cc */, ); @@ -1747,6 +1750,7 @@ 5492E03F2021401F00B64F25 /* FSTHelpers.mm in Sources */, DE2EF0861F3D0B6E003D0CDC /* FSTImmutableSortedDictionary+Testing.m in Sources */, DE2EF0871F3D0B6E003D0CDC /* FSTImmutableSortedSet+Testing.m in Sources */, + B6D1B68520E2AB1B00B35856 /* exponential_backoff_test.cc in Sources */, 5491BC721FB44593008B3588 /* FSTIntegrationTestCase.mm in Sources */, 5492E0A72021552D00B64F25 /* FSTLevelDBKeyTests.mm in Sources */, 5492E0A82021552D00B64F25 /* FSTLevelDBLocalStoreTests.mm in Sources */, diff --git a/Firestore/core/src/firebase/firestore/remote/CMakeLists.txt b/Firestore/core/src/firebase/firestore/remote/CMakeLists.txt index fc51b37..0ef0b7c 100644 --- a/Firestore/core/src/firebase/firestore/remote/CMakeLists.txt +++ b/Firestore/core/src/firebase/firestore/remote/CMakeLists.txt @@ -17,6 +17,8 @@ cc_library( SOURCES datastore.h datastore.cc + exponential_backoff.h + exponential_backoff.cc serializer.h serializer.cc DEPENDS diff --git a/Firestore/core/src/firebase/firestore/remote/exponential_backoff.cc b/Firestore/core/src/firebase/firestore/remote/exponential_backoff.cc new file mode 100644 index 0000000..a880c45 --- /dev/null +++ b/Firestore/core/src/firebase/firestore/remote/exponential_backoff.cc @@ -0,0 +1,93 @@ +/* + * Copyright 2018 Google + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "Firestore/core/src/firebase/firestore/remote/exponential_backoff.h" + +#include +#include + +#include "Firestore/core/src/firebase/firestore/util/hard_assert.h" +#include "Firestore/core/src/firebase/firestore/util/log.h" + +namespace firebase { +namespace firestore { +namespace remote { + +using firebase::firestore::util::AsyncQueue; +using firebase::firestore::util::TimerId; +namespace chr = std::chrono; + +ExponentialBackoff::ExponentialBackoff(AsyncQueue* queue, + TimerId timer_id, + double backoff_factor, + Milliseconds initial_delay, + Milliseconds max_delay) + : queue_{queue}, + timer_id_{timer_id}, + backoff_factor_{backoff_factor}, + initial_delay_{initial_delay}, + max_delay_{max_delay} { + HARD_ASSERT(queue, "Queue can't be null"); + + HARD_ASSERT(backoff_factor >= 1.0, "Backoff factor must be at least 1"); + + HARD_ASSERT(initial_delay.count() >= 0, "Delays must be non-negative"); + HARD_ASSERT(max_delay.count() >= 0, "Delays must be non-negative"); + HARD_ASSERT(initial_delay <= max_delay, + "Initial delay can't be greater than max delay"); +} + +void ExponentialBackoff::BackoffAndRun(AsyncQueue::Operation&& operation) { + Cancel(); + + // First schedule the block using the current base (which may be 0 and should + // be honored as such). + Milliseconds delay_with_jitter = current_base_ + GetDelayWithJitter(); + if (delay_with_jitter.count() > 0) { + LOG_DEBUG("Backing off for %s milliseconds (base delay: %s milliseconds)", + delay_with_jitter.count(), current_base_.count()); + } + + delayed_operation_ = queue_->EnqueueAfterDelay(delay_with_jitter, timer_id_, + std::move(operation)); + + // Apply backoff factor to determine next delay, but ensure it is within + // bounds. + current_base_ = ClampDelay( + chr::duration_cast(current_base_ * backoff_factor_)); +} + +ExponentialBackoff::Milliseconds ExponentialBackoff::GetDelayWithJitter() { + std::uniform_real_distribution distribution; + double random_double = distribution(secure_random_); + return chr::duration_cast((random_double - 0.5) * + current_base_); +} + +ExponentialBackoff::Milliseconds ExponentialBackoff::ClampDelay( + Milliseconds delay) const { + if (delay < initial_delay_) { + return initial_delay_; + } + if (delay > max_delay_) { + return max_delay_; + } + return delay; +} + +} // namespace remote +} // namespace firestore +} // namespace firebase diff --git a/Firestore/core/src/firebase/firestore/remote/exponential_backoff.h b/Firestore/core/src/firebase/firestore/remote/exponential_backoff.h new file mode 100644 index 0000000..8836e7e --- /dev/null +++ b/Firestore/core/src/firebase/firestore/remote/exponential_backoff.h @@ -0,0 +1,116 @@ +/* + * Copyright 2018 Google + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef FIRESTORE_CORE_SRC_FIREBASE_FIRESTORE_REMOTE_EXPONENTIAL_BACKOFF_H_ +#define FIRESTORE_CORE_SRC_FIREBASE_FIRESTORE_REMOTE_EXPONENTIAL_BACKOFF_H_ + +#include // NOLINT(build/c++11) + +#include "Firestore/core/src/firebase/firestore/util/async_queue.h" +#include "Firestore/core/src/firebase/firestore/util/secure_random.h" + +namespace firebase { +namespace firestore { +namespace remote { + +/** + * + * A helper for running delayed operations following an exponential backoff + * curve between attempts. + * + * The first attempt will be done immediately. After that, each retry will + * have a delay that is made up of a "base" delay which follows the + * exponential backoff curve, and a +/- <=50% "jitter" that is calculated and + * added to the base delay. This prevents clients from accidentally + * synchronizing their delays causing spikes of load to the backend. + * + */ +class ExponentialBackoff { + public: + /** + * @param queue The queue to run operations on. + * @param timer_id The id to use when scheduling backoff operations on the + * queue. + * @param backoff_factor The multiplier to use to determine the extended base + * delay after each attempt. + * @param initial_delay The initial delay (used as the base delay on the first + * retry attempt, that is, the second attempt). Note that jitter will + * still be applied, so the actual delay could be as little as + * `0.5*initial_delay`. + * @param max_delay The maximum base delay after which no further backoff is + * performed. Note that jitter will still be applied, so the actual delay + * could be as much as `1.5*max_delay`. + */ + ExponentialBackoff(util::AsyncQueue* queue, + util::TimerId timer_id, + double backoff_factor, + util::AsyncQueue::Milliseconds initial_delay, + util::AsyncQueue::Milliseconds max_delay); + + /** + * Resets the backoff delay. + * + * The very next `backoffAndRun` will have no delay. If it is called again + * (i.e. due to an error), `initial_delay` (plus jitter) will be used, and + * subsequent ones will increase according to the `backoff_factor`. + */ + void Reset() { + current_base_ = Milliseconds{0}; + } + + /** + * Resets the backoff to the maximum delay (e.g. for use after + * a RESOURCE_EXHAUSTED error). + */ + void ResetToMax() { + current_base_ = max_delay_; + } + + /** + * Waits for `current_base` seconds (which may be zero), increases the delay + * and runs the specified operation. If there was a pending operation waiting + * to be run already, it will be canceled. + */ + void BackoffAndRun(util::AsyncQueue::Operation&& operation); + + /** Cancels any pending backoff operation scheduled via `BackoffAndRun`. */ + void Cancel() { + delayed_operation_.Cancel(); + } + + private: + using Milliseconds = util::AsyncQueue::Milliseconds; + + // Returns a random value in the range [-current_base_/2, current_base_/2]. + Milliseconds GetDelayWithJitter(); + Milliseconds ClampDelay(Milliseconds delay) const; + + util::AsyncQueue* const queue_; + const util::TimerId timer_id_; + util::DelayedOperation delayed_operation_; + + const double backoff_factor_; + Milliseconds current_base_{0}; + const Milliseconds initial_delay_; + const Milliseconds max_delay_; + util::SecureRandom secure_random_; +}; + +} // namespace remote +} // namespace firestore +} // namespace firebase + +#endif // FIRESTORE_CORE_SRC_FIREBASE_FIRESTORE_REMOTE_EXPONENTIAL_BACKOFF_H_ diff --git a/Firestore/core/src/firebase/firestore/util/executor.h b/Firestore/core/src/firebase/firestore/util/executor.h index df8b0b5..ea67b17 100644 --- a/Firestore/core/src/firebase/firestore/util/executor.h +++ b/Firestore/core/src/firebase/firestore/util/executor.h @@ -38,7 +38,9 @@ class DelayedOperation { // If the operation has not been run yet, cancels the operation. Otherwise, // this function is a no-op. void Cancel() { - cancel_func_(); + if (cancel_func_) { + cancel_func_(); + } } // Internal use only. diff --git a/Firestore/core/test/firebase/firestore/remote/CMakeLists.txt b/Firestore/core/test/firebase/firestore/remote/CMakeLists.txt index 1b4142a..d91dc0f 100644 --- a/Firestore/core/test/firebase/firestore/remote/CMakeLists.txt +++ b/Firestore/core/test/firebase/firestore/remote/CMakeLists.txt @@ -16,8 +16,10 @@ cc_test( firebase_firestore_remote_test SOURCES datastore_test.cc + exponential_backoff_test.cc serializer_test.cc DEPENDS + absl_base # NB: Order is important. We need to include the ffp_libprotobuf library # before ff_remote, or else we'll end up with nanopb's headers earlier in # the include path than libprotobuf's, which makes using libprotobuf in the @@ -26,4 +28,5 @@ cc_test( # exists in both the libprotobuf path and the nanopb path. firebase_firestore_protos_libprotobuf firebase_firestore_remote + firebase_firestore_util_executor_std ) diff --git a/Firestore/core/test/firebase/firestore/remote/exponential_backoff_test.cc b/Firestore/core/test/firebase/firestore/remote/exponential_backoff_test.cc new file mode 100644 index 0000000..dbc34ef --- /dev/null +++ b/Firestore/core/test/firebase/firestore/remote/exponential_backoff_test.cc @@ -0,0 +1,91 @@ +/* + * Copyright 2018 Google + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include // NOLINT(build/c++11) + +#include "Firestore/core/src/firebase/firestore/remote/exponential_backoff.h" +#include "Firestore/core/src/firebase/firestore/util/async_queue.h" +#include "Firestore/core/src/firebase/firestore/util/executor_std.h" +#include "Firestore/core/test/firebase/firestore/util/async_tests_util.h" +#include "absl/memory/memory.h" +#include "gtest/gtest.h" + +using firebase::firestore::util::AsyncQueue; +using firebase::firestore::util::TestWithTimeoutMixin; +using firebase::firestore::util::TimerId; +using firebase::firestore::util::internal::ExecutorStd; + +namespace chr = std::chrono; + +namespace firebase { +namespace firestore { +namespace remote { + +class ExponentialBackoffTest : public TestWithTimeoutMixin, + public testing::Test { + public: + ExponentialBackoffTest() + : queue{absl::make_unique()}, + backoff{&queue, timer_id, 1.5, chr::seconds{5}, chr::seconds{30}} { + } + + TimerId timer_id = TimerId::ListenStreamConnectionBackoff; + AsyncQueue queue; + ExponentialBackoff backoff; +}; + +TEST_F(ExponentialBackoffTest, CanScheduleOperations) { + EXPECT_FALSE(queue.IsScheduled(timer_id)); + + queue.EnqueueBlocking([&] { + backoff.BackoffAndRun([&] { signal_finished(); }); + EXPECT_TRUE(queue.IsScheduled(timer_id)); + }); + + EXPECT_TRUE(WaitForTestToFinish()); + EXPECT_FALSE(queue.IsScheduled(timer_id)); +} + +TEST_F(ExponentialBackoffTest, CanCancelOperations) { + std::string str{"untouched"}; + EXPECT_FALSE(queue.IsScheduled(timer_id)); + + queue.EnqueueBlocking([&] { + backoff.BackoffAndRun([&] { str = "Shouldn't be modified"; }); + EXPECT_TRUE(queue.IsScheduled(timer_id)); + backoff.Cancel(); + }); + + EXPECT_FALSE(queue.IsScheduled(timer_id)); + EXPECT_EQ(str, "untouched"); +} + +TEST_F(ExponentialBackoffTest, SequentialCallsToBackoffAndRun) { + queue.EnqueueBlocking([&] { + backoff.BackoffAndRun([] {}); + backoff.BackoffAndRun([] {}); + backoff.BackoffAndRun([&] { signal_finished(); }); + }); + + // The chosen value of initial_delay is large enough that it shouldn't be + // realistically possible for backoff to finish already. + queue.RunScheduledOperationsUntil(timer_id); + EXPECT_TRUE(WaitForTestToFinish()); +} + +} // namespace remote +} // namespace firestore +} // namespace firebase diff --git a/Firestore/core/test/firebase/firestore/util/executor_test.cc b/Firestore/core/test/firebase/firestore/util/executor_test.cc index 99bddce..e983bfe 100644 --- a/Firestore/core/test/firebase/firestore/util/executor_test.cc +++ b/Firestore/core/test/firebase/firestore/util/executor_test.cc @@ -115,6 +115,30 @@ TEST_P(ExecutorTest, DelayedOperationIsValidAfterTheOperationHasRun) { EXPECT_NO_THROW(delayed_operation.Cancel()); } +TEST_P(ExecutorTest, CancelingEmptyDelayedOperationIsValid) { + DelayedOperation delayed_operation; + EXPECT_NO_THROW(delayed_operation.Cancel()); +} + +TEST_P(ExecutorTest, DoubleCancelingDelayedOperationIsValid) { + std::string steps; + + executor->Execute([&] { + DelayedOperation delayed_operation = Schedule( + executor.get(), Executor::Milliseconds(1), [&steps] { steps += '1'; }); + Schedule(executor.get(), Executor::Milliseconds(5), [&] { + steps += '2'; + signal_finished(); + }); + + delayed_operation.Cancel(); + delayed_operation.Cancel(); + }); + + EXPECT_TRUE(WaitForTestToFinish()); + EXPECT_EQ(steps, "2"); +} + TEST_P(ExecutorTest, IsCurrentExecutor) { EXPECT_FALSE(executor->IsCurrentExecutor()); EXPECT_NE(executor->Name(), executor->CurrentExecutorName()); -- cgit v1.2.3