diff options
| author |  Pirate Praveen <praveen@debian.org> | 2023-03-06 20:25:41 +0530 |
|---|---|---|
| committer | Pirate Praveen <praveen@debian.org> | 2023-03-06 20:25:41 +0530 |
| commit | 079dd8737bbaaaeeca3a95c2b858a62d8a620d5a (patch) | |
| tree | 6af54966e17bcfe48ecdb0b5cdf43cc953d5358c | |
| parent | 2bbc47f307f1e24f3f44a108d571bffa5a3faa63 (diff) | |
| parent | f5afcb784c9b1c501c1144b7aab84555881ca871 (diff) | |
Merge tag '20220623.1-1' into bullseye-backports-staging
519 files changed, 32703 insertions, 11120 deletions
diff --git a/CMake/AbseilDll.cmake b/CMake/AbseilDll.cmake index 253c73ff..00cddb84 100644 --- a/CMake/AbseilDll.cmake +++ b/CMake/AbseilDll.cmake @@ -17,8 +17,6 @@ set(ABSL_INTERNAL_DLL_FILES "base/internal/dynamic_annotations.h" "base/internal/endian.h" "base/internal/errno_saver.h" - "base/internal/exponential_biased.cc" - "base/internal/exponential_biased.h" "base/internal/fast_type_id.h" "base/internal/hide_ptr.h" "base/internal/identity.h" @@ -28,8 +26,7 @@ set(ABSL_INTERNAL_DLL_FILES "base/internal/low_level_alloc.h" "base/internal/low_level_scheduling.h" "base/internal/per_thread_tls.h" - "base/internal/periodic_sampler.cc" - "base/internal/periodic_sampler.h" + "base/internal/prefetch.h" "base/internal/pretty_function.h" "base/internal/raw_logging.cc" "base/internal/raw_logging.h" @@ -82,10 +79,9 @@ set(ABSL_INTERNAL_DLL_FILES "container/internal/hashtablez_sampler.cc" "container/internal/hashtablez_sampler.h" "container/internal/hashtablez_sampler_force_weak_definition.cc" - "container/internal/have_sse.h" "container/internal/inlined_vector.h" "container/internal/layout.h" - "container/internal/node_hash_policy.h" + "container/internal/node_slot_policy.h" "container/internal/raw_hash_map.h" "container/internal/raw_hash_set.cc" "container/internal/raw_hash_set.h" @@ -95,7 +91,6 @@ set(ABSL_INTERNAL_DLL_FILES "debugging/failure_signal_handler.cc" "debugging/failure_signal_handler.h" "debugging/leak_check.h" - "debugging/leak_check_disable.cc" "debugging/stacktrace.cc" "debugging/stacktrace.h" "debugging/symbolize.cc" @@ -114,9 +109,11 @@ set(ABSL_INTERNAL_DLL_FILES "debugging/internal/symbolize.h" "debugging/internal/vdso_support.cc" "debugging/internal/vdso_support.h" + "functional/any_invocable.h" "functional/internal/front_binder.h" "functional/bind_front.h" "functional/function_ref.h" + "functional/internal/any_invocable.h" "functional/internal/function_ref.h" "hash/hash.h" "hash/internal/city.h" @@ -124,8 +121,8 @@ set(ABSL_INTERNAL_DLL_FILES "hash/internal/hash.h" "hash/internal/hash.cc" "hash/internal/spy_hash_state.h" - "hash/internal/wyhash.h" - "hash/internal/wyhash.cc" + "hash/internal/low_level_hash.h" + "hash/internal/low_level_hash.cc" "memory/memory.h" "meta/type_traits.h" "numeric/bits.h" @@ -133,6 +130,11 @@ set(ABSL_INTERNAL_DLL_FILES "numeric/int128.h" "numeric/internal/bits.h" "numeric/internal/representation.h" + "profiling/internal/exponential_biased.cc" + "profiling/internal/exponential_biased.h" + "profiling/internal/periodic_sampler.cc" + "profiling/internal/periodic_sampler.h" + "profiling/internal/sample_recorder.h" "random/bernoulli_distribution.h" "random/beta_distribution.h" "random/bit_gen_ref.h" @@ -195,18 +197,44 @@ set(ABSL_INTERNAL_DLL_FILES "strings/charconv.h" "strings/cord.cc" "strings/cord.h" + "strings/cord_analysis.cc" + "strings/cord_analysis.h" + "strings/cord_buffer.cc" + "strings/cord_buffer.h" "strings/escaping.cc" "strings/escaping.h" + "strings/internal/charconv_bigint.cc" + "strings/internal/charconv_bigint.h" + "strings/internal/charconv_parse.cc" + "strings/internal/charconv_parse.h" + "strings/internal/cord_data_edge.h" "strings/internal/cord_internal.cc" "strings/internal/cord_internal.h" + "strings/internal/cord_rep_btree.cc" + "strings/internal/cord_rep_btree.h" + "strings/internal/cord_rep_btree_navigator.cc" + "strings/internal/cord_rep_btree_navigator.h" + "strings/internal/cord_rep_btree_reader.cc" + "strings/internal/cord_rep_btree_reader.h" + "strings/internal/cord_rep_crc.cc" + "strings/internal/cord_rep_crc.h" + "strings/internal/cord_rep_consume.h" + "strings/internal/cord_rep_consume.cc" "strings/internal/cord_rep_flat.h" "strings/internal/cord_rep_ring.cc" "strings/internal/cord_rep_ring.h" "strings/internal/cord_rep_ring_reader.h" - "strings/internal/charconv_bigint.cc" - "strings/internal/charconv_bigint.h" - "strings/internal/charconv_parse.cc" - "strings/internal/charconv_parse.h" + "strings/internal/cordz_functions.cc" + "strings/internal/cordz_functions.h" + "strings/internal/cordz_handle.cc" + "strings/internal/cordz_handle.h" + "strings/internal/cordz_info.cc" + "strings/internal/cordz_info.h" + "strings/internal/cordz_sample_token.cc" + "strings/internal/cordz_sample_token.h" + "strings/internal/cordz_statistics.h" + "strings/internal/cordz_update_scope.h" + "strings/internal/cordz_update_tracker.h" "strings/internal/stl_type_traits.h" "strings/internal/string_constant.h" "strings/match.cc" @@ -321,6 +349,7 @@ set(ABSL_INTERNAL_DLL_FILES "types/internal/span.h" "types/variant.h" "utility/utility.h" + "debugging/leak_check.cc" ) set(ABSL_INTERNAL_DLL_TARGETS @@ -331,7 +360,6 @@ set(ABSL_INTERNAL_DLL_TARGETS "debugging_internal" "demangle_internal" "leak_check" - "leak_check_disable" "stack_consumption" "debugging" "hash" @@ -362,6 +390,7 @@ set(ABSL_INTERNAL_DLL_TARGETS "kernel_timeout_internal" "synchronization" "thread_pool" + "any_invocable" "bind_front" "function_ref" "atomic_hook" @@ -431,13 +460,13 @@ set(ABSL_INTERNAL_DLL_TARGETS "hashtablez_sampler" "hashtable_debug" "hashtable_debug_hooks" - "have_sse" - "node_hash_policy" + "node_slot_policy" "raw_hash_map" "container_common" "raw_hash_set" "layout" "tracked" + "sample_recorder" ) function(absl_internal_dll_contains) diff --git a/CMake/AbseilHelpers.cmake b/CMake/AbseilHelpers.cmake index 54fb8df3..dbb09fe2 100644 --- a/CMake/AbseilHelpers.cmake +++ b/CMake/AbseilHelpers.cmake @@ -40,7 +40,8 @@ endif() # LINKOPTS: List of link options # PUBLIC: Add this so that this library will be exported under absl:: # Also in IDE, target will appear in Abseil folder while non PUBLIC will be in Abseil/internal. -# TESTONLY: When added, this target will only be built if BUILD_TESTING=ON. +# TESTONLY: When added, this target will only be built if both +# BUILD_TESTING=ON and ABSL_BUILD_TESTING=ON. # # Note: # By default, absl_cc_library will always create a library named absl_${NAME}, @@ -82,7 +83,8 @@ function(absl_cc_library) ${ARGN} ) - if(ABSL_CC_LIB_TESTONLY AND NOT BUILD_TESTING) + if(NOT ABSL_CC_LIB_PUBLIC AND ABSL_CC_LIB_TESTONLY AND + NOT (BUILD_TESTING AND ABSL_BUILD_TESTING)) return() endif() @@ -141,7 +143,8 @@ function(absl_cc_library) endif() # Generate a pkg-config file for every library: - if(_build_type STREQUAL "static" OR _build_type STREQUAL "shared") + if((_build_type STREQUAL "static" OR _build_type STREQUAL "shared") + AND ABSL_ENABLE_INSTALL) if(NOT ABSL_CC_LIB_TESTONLY) if(absl_VERSION) set(PC_VERSION "${absl_VERSION}") @@ -167,18 +170,19 @@ function(absl_cc_library) set(PC_CFLAGS "${PC_CFLAGS} ${cflag}") endif() endforeach() + string(REPLACE ";" " " PC_LINKOPTS "${ABSL_CC_LIB_LINKOPTS}") FILE(GENERATE OUTPUT "${CMAKE_BINARY_DIR}/lib/pkgconfig/absl_${_NAME}.pc" CONTENT "\ prefix=${CMAKE_INSTALL_PREFIX}\n\ exec_prefix=\${prefix}\n\ -libdir=\${prefix}/${CMAKE_INSTALL_LIBDIR}\n\ -includedir=\${prefix}/${CMAKE_INSTALL_INCLUDEDIR}\n\ +libdir=${CMAKE_INSTALL_FULL_LIBDIR}\n\ +includedir=${CMAKE_INSTALL_FULL_INCLUDEDIR}\n\ \n\ Name: absl_${_NAME}\n\ Description: Abseil ${_NAME} library\n\ URL: https://abseil.io/\n\ Version: ${PC_VERSION}\n\ Requires:${PC_DEPS}\n\ -Libs: -L\${libdir} $<JOIN:${ABSL_CC_LIB_LINKOPTS}, > $<$<NOT:$<BOOL:${ABSL_CC_LIB_IS_INTERFACE}>>:-labsl_${_NAME}>\n\ +Libs: -L\${libdir} ${PC_LINKOPTS} $<$<NOT:$<BOOL:${ABSL_CC_LIB_IS_INTERFACE}>>:-labsl_${_NAME}>\n\ Cflags: -I\${includedir}${PC_CFLAGS}\n") INSTALL(FILES "${CMAKE_BINARY_DIR}/lib/pkgconfig/absl_${_NAME}.pc" DESTINATION "${CMAKE_INSTALL_LIBDIR}/pkgconfig") @@ -253,9 +257,23 @@ Cflags: -I\${includedir}${PC_CFLAGS}\n") set_property(TARGET ${_NAME} PROPERTY FOLDER ${ABSL_IDE_FOLDER}/internal) endif() - # INTERFACE libraries can't have the CXX_STANDARD property set - set_property(TARGET ${_NAME} PROPERTY CXX_STANDARD ${ABSL_CXX_STANDARD}) - set_property(TARGET ${_NAME} PROPERTY CXX_STANDARD_REQUIRED ON) + if(ABSL_PROPAGATE_CXX_STD) + # Abseil libraries require C++11 as the current minimum standard. + # Top-level application CMake projects should ensure a consistent C++ + # standard for all compiled sources by setting CMAKE_CXX_STANDARD. + target_compile_features(${_NAME} PUBLIC cxx_std_11) + else() + # Note: This is legacy (before CMake 3.8) behavior. Setting the + # target-level CXX_STANDARD property to ABSL_CXX_STANDARD (which is + # initialized by CMAKE_CXX_STANDARD) should have no real effect, since + # that is the default value anyway. + # + # CXX_STANDARD_REQUIRED does guard against the top-level CMake project + # not having enabled CMAKE_CXX_STANDARD_REQUIRED (which prevents + # "decaying" to an older standard if the requested one isn't available). + set_property(TARGET ${_NAME} PROPERTY CXX_STANDARD ${ABSL_CXX_STANDARD}) + set_property(TARGET ${_NAME} PROPERTY CXX_STANDARD_REQUIRED ON) + endif() # When being installed, we lose the absl_ prefix. We want to put it back # to have properly named lib files. This is a no-op when we are not being @@ -263,7 +281,7 @@ Cflags: -I\${includedir}${PC_CFLAGS}\n") if(ABSL_ENABLE_INSTALL) set_target_properties(${_NAME} PROPERTIES OUTPUT_NAME "absl_${_NAME}" - SOVERSION "2103.0.1" + SOVERSION "2206.0.0" ) endif() else() @@ -286,6 +304,16 @@ Cflags: -I\${includedir}${PC_CFLAGS}\n") ${ABSL_DEFAULT_LINKOPTS} ) target_compile_definitions(${_NAME} INTERFACE ${ABSL_CC_LIB_DEFINES}) + + if(ABSL_PROPAGATE_CXX_STD) + # Abseil libraries require C++11 as the current minimum standard. + # Top-level application CMake projects should ensure a consistent C++ + # standard for all compiled sources by setting CMAKE_CXX_STANDARD. + target_compile_features(${_NAME} INTERFACE cxx_std_11) + + # (INTERFACE libraries can't have the CXX_STANDARD property set, so there + # is no legacy behavior else case). + endif() endif() # TODO currently we don't install googletest alongside abseil sources, so @@ -335,11 +363,11 @@ endfunction() # "awesome_test.cc" # DEPS # absl::awesome -# gmock -# gtest_main +# GTest::gmock +# GTest::gtest_main # ) function(absl_cc_test) - if(NOT BUILD_TESTING) + if(NOT (BUILD_TESTING AND ABSL_BUILD_TESTING)) return() endif() @@ -389,8 +417,23 @@ function(absl_cc_test) # Add all Abseil targets to a folder in the IDE for organization. set_property(TARGET ${_NAME} PROPERTY FOLDER ${ABSL_IDE_FOLDER}/test) - set_property(TARGET ${_NAME} PROPERTY CXX_STANDARD ${ABSL_CXX_STANDARD}) - set_property(TARGET ${_NAME} PROPERTY CXX_STANDARD_REQUIRED ON) + if(ABSL_PROPAGATE_CXX_STD) + # Abseil libraries require C++11 as the current minimum standard. + # Top-level application CMake projects should ensure a consistent C++ + # standard for all compiled sources by setting CMAKE_CXX_STANDARD. + target_compile_features(${_NAME} PUBLIC cxx_std_11) + else() + # Note: This is legacy (before CMake 3.8) behavior. Setting the + # target-level CXX_STANDARD property to ABSL_CXX_STANDARD (which is + # initialized by CMAKE_CXX_STANDARD) should have no real effect, since + # that is the default value anyway. + # + # CXX_STANDARD_REQUIRED does guard against the top-level CMake project + # not having enabled CMAKE_CXX_STANDARD_REQUIRED (which prevents + # "decaying" to an older standard if the requested one isn't available). + set_property(TARGET ${_NAME} PROPERTY CXX_STANDARD ${ABSL_CXX_STANDARD}) + set_property(TARGET ${_NAME} PROPERTY CXX_STANDARD_REQUIRED ON) + endif() add_test(NAME ${_NAME} COMMAND ${_NAME}) endfunction() diff --git a/CMake/README.md b/CMake/README.md index 5eee8171..8134615e 100644 --- a/CMake/README.md +++ b/CMake/README.md @@ -20,8 +20,10 @@ googletest framework ### Step-by-Step Instructions 1. If you want to build the Abseil tests, integrate the Abseil dependency -[Google Test](https://github.com/google/googletest) into your CMake project. To disable Abseil tests, you have to pass -`-DBUILD_TESTING=OFF` when configuring your project with CMake. +[Google Test](https://github.com/google/googletest) into your CMake +project. To disable Abseil tests, you have to pass either +`-DBUILD_TESTING=OFF` or `-DABSL_BUILD_TESTING=OFF` when configuring your +project with CMake. 2. Download Abseil and copy it into a subdirectory in your CMake project or add Abseil as a [git submodule](https://git-scm.com/docs/git-submodule) in your @@ -34,15 +36,16 @@ to include Abseil directly in your CMake project. 4. Add the **absl::** target you wish to use to the [`target_link_libraries()`](https://cmake.org/cmake/help/latest/command/target_link_libraries.html) section of your executable or of your library.<br> -Here is a short CMakeLists.txt example of a project file using Abseil. +Here is a short CMakeLists.txt example of an application project using Abseil. ```cmake -cmake_minimum_required(VERSION 3.5) -project(my_project) +cmake_minimum_required(VERSION 3.8.2) +project(my_app_project) # Pick the C++ standard to compile with. # Abseil currently supports C++11, C++14, and C++17. set(CMAKE_CXX_STANDARD 11) +set(CMAKE_CXX_STANDARD_REQUIRED ON) add_subdirectory(abseil-cpp) @@ -50,9 +53,48 @@ add_executable(my_exe source.cpp) target_link_libraries(my_exe absl::base absl::synchronization absl::strings) ``` +Note that if you are developing a library designed for use by other clients, you +should instead leave `CMAKE_CXX_STANDARD` unset (or only set if being built as +the current top-level CMake project) and configure the minimum required C++ +standard at the target level. If you require a later minimum C++ standard than +Abseil does, it's a good idea to also enforce that `CMAKE_CXX_STANDARD` (which +will control Abseil library targets) is set to at least that minimum. For +example: + +```cmake +cmake_minimum_required(VERSION 3.8.2) +project(my_lib_project) + +# Leave C++ standard up to the root application, so set it only if this is the +# current top-level CMake project. +if(CMAKE_SOURCE_DIR STREQUAL my_lib_project_SOURCE_DIR) + set(CMAKE_CXX_STANDARD 17) + set(CMAKE_CXX_STANDARD_REQUIRED ON) +endif() + +add_subdirectory(abseil-cpp) + +add_library(my_lib source.cpp) +target_link_libraries(my_lib absl::base absl::synchronization absl::strings) + +# Enforce that my_lib requires C++17. Important to document for clients that they +# must set CMAKE_CXX_STANDARD to 17 or higher for proper Abseil ABI compatibility +# (since otherwise, Abseil library targets could be compiled with a lower C++ +# standard than my_lib). +target_compile_features(my_lib PUBLIC cxx_std_17) +if(CMAKE_CXX_STANDARD LESS 17) + message(FATAL_ERROR + "my_lib_project requires CMAKE_CXX_STANDARD >= 17 (got: ${CMAKE_CXX_STANDARD})") +endif() +``` + +Then the top-level application project that uses your library is responsible for +setting a consistent `CMAKE_CXX_STANDARD` that is sufficiently high. + ### Running Abseil Tests with CMake -Use the `-DBUILD_TESTING=ON` flag to run Abseil tests. +Use the `-DABSL_BUILD_TESTING=ON` flag to run Abseil tests. Note that +BUILD_TESTING must also be on (the default). You will need to provide Abseil with a Googletest dependency. There are two options for how to do this: @@ -70,7 +112,7 @@ For example, to run just the Abseil tests, you could use this script: cd path/to/abseil-cpp mkdir build cd build -cmake -DBUILD_TESTING=ON -DABSL_USE_GOOGLETEST_HEAD=ON .. +cmake -DABSL_BUILD_TESTING=ON -DABSL_USE_GOOGLETEST_HEAD=ON .. make -j ctest ``` @@ -99,3 +141,48 @@ absl::synchronization absl::time absl::utility ``` + +## Traditional CMake Set-Up + +For larger projects, it may make sense to use the traditional CMake set-up where you build and install projects separately. + +First, you'd need to build and install Google Test: +``` +cmake -S /source/googletest -B /build/googletest -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/installation/dir -DBUILD_GMOCK=ON +cmake --build /build/googletest --target install +``` + +Then you need to configure and build Abseil. Make sure you enable `ABSL_USE_EXTERNAL_GOOGLETEST` and `ABSL_FIND_GOOGLETEST`. You also need to enable `ABSL_ENABLE_INSTALL` so that you can install Abseil itself. +``` +cmake -S /source/abseil-cpp -B /build/abseil-cpp -DCMAKE_PREFIX_PATH=/installation/dir -DCMAKE_INSTALL_PREFIX=/installation/dir -DABSL_ENABLE_INSTALL=ON -DABSL_USE_EXTERNAL_GOOGLETEST=ON -DABSL_FIND_GOOGLETEST=ON +cmake --build /temporary/build/abseil-cpp +``` + +(`CMAKE_PREFIX_PATH` is where you already have Google Test installed; `CMAKE_INSTALL_PREFIX` is where you want to have Abseil installed; they can be different.) + +Run the tests: +``` +ctest --test-dir /temporary/build/abseil-cpp +``` + +And finally install: +``` +cmake --build /temporary/build/abseil-cpp --target install +``` + +# CMake Option Synposis + +## Enable Standard CMake Installation + +`-DABSL_ENABLE_INSTALL=ON` + +## Google Test Options + +`-DABSL_BUILD_TESTING=ON` must be set to enable testing + +- Have Abseil download and build Google Test for you: `-DABSL_USE_EXTERNAL_GOOGLETEST=OFF` (default) + - Download and build latest Google Test: `-DABSL_USE_GOOGLETEST_HEAD=ON` + - Download specific Google Test version (ZIP archive): `-DABSL_GOOGLETEST_DOWNLOAD_URL=https://.../version.zip` + - Use Google Test from specific local directory: `-DABSL_LOCAL_GOOGLETEST_DIR=/path/to/googletest` +- Use Google Test included elsewhere in your project: `-DABSL_USE_EXTERNAL_GOOGLETEST=ON` +- Use standard CMake `find_package(CTest)` to find installed Google Test: `-DABSL_USE_EXTERNAL_GOOGLETEST=ON -DABSL_FIND_GOOGLETEST=ON` diff --git a/CMake/install_test_project/CMakeLists.txt b/CMake/install_test_project/CMakeLists.txt index 06b797e9..b865b2ec 100644 --- a/CMake/install_test_project/CMakeLists.txt +++ b/CMake/install_test_project/CMakeLists.txt @@ -18,10 +18,8 @@ cmake_minimum_required(VERSION 3.5) project(absl_cmake_testing CXX) -set(CMAKE_CXX_STANDARD 11) - add_executable(simple simple.cc) find_package(absl REQUIRED) -target_link_libraries(simple absl::strings) +target_link_libraries(simple absl::strings absl::config) diff --git a/CMake/install_test_project/simple.cc b/CMake/install_test_project/simple.cc index e9e35291..7daa7f09 100644 --- a/CMake/install_test_project/simple.cc +++ b/CMake/install_test_project/simple.cc @@ -14,8 +14,17 @@ // limitations under the License. #include <iostream> +#include "absl/base/config.h" #include "absl/strings/substitute.h" +#if !defined(ABSL_LTS_RELEASE_VERSION) || ABSL_LTS_RELEASE_VERSION != 99998877 +#error ABSL_LTS_RELEASE_VERSION is not set correctly. +#endif + +#if !defined(ABSL_LTS_RELEASE_PATCH_LEVEL) || ABSL_LTS_RELEASE_PATCH_LEVEL != 0 +#error ABSL_LTS_RELEASE_PATCH_LEVEL is not set correctly. +#endif + int main(int argc, char** argv) { for (int i = 0; i < argc; ++i) { std::cout << absl::Substitute("Arg $0: $1\n", i, argv[i]); diff --git a/CMake/install_test_project/test.sh b/CMake/install_test_project/test.sh index a3d39773..cc028bac 100755 --- a/CMake/install_test_project/test.sh +++ b/CMake/install_test_project/test.sh @@ -19,10 +19,9 @@ # Fail on any error. Treat unset variables an error. Print commands as executed. set -euox pipefail -source ci/cmake_common.sh - absl_dir=/abseil-cpp absl_build_dir=/buildfs +googletest_builddir=/googletest_builddir project_dir="${absl_dir}"/CMake/install_test_project project_build_dir=/buildfs/project-build @@ -31,18 +30,35 @@ if [ "${LINK_TYPE:-}" = "DYNAMIC" ]; then build_shared_libs="ON" fi +# Build and install GoogleTest +mkdir "${googletest_builddir}" +pushd "${googletest_builddir}" +curl -L "${ABSL_GOOGLETEST_DOWNLOAD_URL}" --output "${ABSL_GOOGLETEST_COMMIT}".zip +unzip "${ABSL_GOOGLETEST_COMMIT}".zip +pushd "googletest-${ABSL_GOOGLETEST_COMMIT}" +mkdir build +pushd build +cmake -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS="${build_shared_libs}" .. +make -j $(nproc) +make install +ldconfig +popd +popd +popd + # Run the LTS transformations ./create_lts.py 99998877 -# Install Abseil +# Build and install Abseil pushd "${absl_build_dir}" cmake "${absl_dir}" \ - -DABSL_GOOGLETEST_DOWNLOAD_URL="${ABSL_GOOGLETEST_DOWNLOAD_URL}" \ + -DABSL_USE_EXTERNAL_GOOGLETEST=ON \ + -DABSL_FIND_GOOGLETEST=ON \ -DCMAKE_BUILD_TYPE=Release \ - -DBUILD_TESTING=ON \ + -DABSL_BUILD_TESTING=ON \ -DBUILD_SHARED_LIBS="${build_shared_libs}" make -j $(nproc) -ctest -j $(nproc) +ctest -j $(nproc) --output-on-failure make install ldconfig popd diff --git a/CMakeLists.txt b/CMakeLists.txt index 3a73f707..79869ff5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -41,11 +41,12 @@ if (POLICY CMP0077) cmake_policy(SET CMP0077 NEW) endif (POLICY CMP0077) -# Set BUILD_TESTING to OFF by default. -# This must come before the project() and include(CTest) lines. -OPTION(BUILD_TESTING "Build tests" OFF) +# Allow the user to specify the MSVC runtime +if (POLICY CMP0091) + cmake_policy(SET CMP0091 NEW) +endif (POLICY CMP0091) -project(absl LANGUAGES CXX VERSION 20210324) +project(absl LANGUAGES CXX VERSION 20220623) include(CTest) # Output directory is correct by default for most build setups. However, when @@ -62,6 +63,13 @@ else() option(ABSL_ENABLE_INSTALL "Enable install rule" ON) endif() +option(ABSL_PROPAGATE_CXX_STD + "Use CMake C++ standard meta features (e.g. cxx_std_11) that propagate to targets that link to Abseil" + OFF) # TODO: Default to ON for CMake 3.8 and greater. +if((${CMAKE_VERSION} VERSION_GREATER_EQUAL 3.8) AND (NOT ABSL_PROPAGATE_CXX_STD)) + message(WARNING "A future Abseil release will default ABSL_PROPAGATE_CXX_STD to ON for CMake 3.8 and up. We recommend enabling this option to ensure your project still builds correctly.") +endif() + list(APPEND CMAKE_MODULE_PATH ${CMAKE_CURRENT_LIST_DIR}/CMake ${CMAKE_CURRENT_LIST_DIR}/absl/copts @@ -97,8 +105,20 @@ endif() ## pthread find_package(Threads REQUIRED) +include(CMakeDependentOption) + +option(ABSL_BUILD_TESTING + "If ON, Abseil will build all of Abseil's own tests." OFF) + option(ABSL_USE_EXTERNAL_GOOGLETEST - "If ON, Abseil will assume that the targets for GoogleTest are already provided by the including project. This makes sense when Abseil is used with add_subproject." OFF) + "If ON, Abseil will assume that the targets for GoogleTest are already provided by the including project. This makes sense when Abseil is used with add_subdirectory." OFF) + +cmake_dependent_option(ABSL_FIND_GOOGLETEST + "If ON, Abseil will use find_package(GTest) rather than assuming that GoogleTest is already provided by the including project." + ON + "ABSL_USE_EXTERNAL_GOOGLETEST" + OFF) + option(ABSL_USE_GOOGLETEST_HEAD "If ON, abseil will download HEAD from GoogleTest at config time." OFF) @@ -109,15 +129,29 @@ set(ABSL_LOCAL_GOOGLETEST_DIR "/usr/src/googletest" CACHE PATH "If ABSL_USE_GOOGLETEST_HEAD is OFF and ABSL_GOOGLETEST_URL is not set, specifies the directory of a local GoogleTest checkout." ) -if(BUILD_TESTING) +if(BUILD_TESTING AND ABSL_BUILD_TESTING) ## check targets - if (NOT ABSL_USE_EXTERNAL_GOOGLETEST) + if (ABSL_USE_EXTERNAL_GOOGLETEST) + if (ABSL_FIND_GOOGLETEST) + find_package(GTest REQUIRED) + elseif(NOT TARGET GTest::gtest) + if(TARGET gtest) + # When Google Test is included directly rather than through find_package, the aliases are missing. + add_library(GTest::gtest ALIAS gtest) + add_library(GTest::gtest_main ALIAS gtest_main) + add_library(GTest::gmock ALIAS gmock) + add_library(GTest::gmock_main ALIAS gmock_main) + else() + message(FATAL_ERROR "ABSL_USE_EXTERNAL_GOOGLETEST is ON and ABSL_FIND_GOOGLETEST is OFF, which means that the top-level project must build the Google Test project. However, the target gtest was not found.") + endif() + endif() + else() set(absl_gtest_build_dir ${CMAKE_BINARY_DIR}/googletest-build) if(ABSL_USE_GOOGLETEST_HEAD AND ABSL_GOOGLETEST_DOWNLOAD_URL) message(FATAL_ERROR "Do not set both ABSL_USE_GOOGLETEST_HEAD and ABSL_GOOGLETEST_DOWNLOAD_URL") endif() if(ABSL_USE_GOOGLETEST_HEAD) - set(absl_gtest_download_url "https://github.com/google/googletest/archive/master.zip") + set(absl_gtest_download_url "https://github.com/google/googletest/archive/main.zip") elseif(ABSL_GOOGLETEST_DOWNLOAD_URL) set(absl_gtest_download_url ${ABSL_GOOGLETEST_DOWNLOAD_URL}) endif() @@ -129,16 +163,10 @@ if(BUILD_TESTING) include(CMake/Googletest/DownloadGTest.cmake) endif() - check_target(gtest) - check_target(gtest_main) - check_target(gmock) - - list(APPEND ABSL_TEST_COMMON_LIBRARIES - gtest_main - gtest - gmock - ${CMAKE_THREAD_LIBS_INIT} - ) + check_target(GTest::gtest) + check_target(GTest::gtest_main) + check_target(GTest::gmock) + check_target(GTest::gmock_main) endif() add_subdirectory(absl) @@ -27,7 +27,10 @@ compiler, there several ways to do this: file](https://docs.bazel.build/versions/master/guide.html#bazelrc) If you are using CMake as the build system, you'll need to add a line like -`set(CMAKE_CXX_STANDARD 17)` to your top level `CMakeLists.txt` file. See the +`set(CMAKE_CXX_STANDARD 17)` to your top level `CMakeLists.txt` file. If you +are developing a library designed to be used by other clients, you should +instead leave `CMAKE_CXX_STANDARD` unset and configure the minimum C++ standard +required by each of your library targets via `target_compile_features`. See the [CMake build instructions](https://github.com/abseil/abseil-cpp/blob/master/CMake/README.md) for more information. @@ -92,6 +92,9 @@ Abseil contains the following C++ library components: available within C++14 and C++17 versions of the C++ `<type_traits>` library. * [`numeric`](absl/numeric/) <br /> The `numeric` library contains C++11-compatible 128-bit integers. +* [`profiling`](absl/profiling/) + <br /> The `profiling` library contains utility code for profiling C++ + entities. It is currently a private dependency of other Abseil libraries. * [`status`](absl/status/) <br /> The `status` contains abstractions for error handling, specifically `absl::Status` and `absl::StatusOr<T>`. @@ -15,31 +15,47 @@ # workspace(name = "com_google_absl") + load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") # GoogleTest/GoogleMock framework. Used by most unit-tests. http_archive( name = "com_google_googletest", + sha256 = "ce7366fe57eb49928311189cb0e40e0a8bf3d3682fca89af30d884c25e983786", + strip_prefix = "googletest-release-1.12.0", # Keep this URL in sync with ABSL_GOOGLETEST_COMMIT in ci/cmake_common.sh. - urls = ["https://github.com/google/googletest/archive/8567b09290fe402cf01923e2131c5635b8ed851b.zip"], # 2020-06-12T22:24:28Z - strip_prefix = "googletest-8567b09290fe402cf01923e2131c5635b8ed851b", - sha256 = "9a8a166eb6a56c7b3d7b19dc2c946fe4778fd6f21c7a12368ad3b836d8f1be48", + urls = ["https://github.com/google/googletest/archive/refs/tags/release-1.12.0.zip"], +) + +# RE2 (the regular expression library used by GoogleTest) +# Note this must use a commit from the `abseil` branch of the RE2 project. +# https://github.com/google/re2/tree/abseil +http_archive( + name = "com_googlesource_code_re2", + sha256 = "0a890c2aa0bb05b2ce906a15efb520d0f5ad4c7d37b8db959c43772802991887", + strip_prefix = "re2-a427f10b9fb4622dd6d8643032600aa1b50fbd12", + urls = ["https://github.com/google/re2/archive/a427f10b9fb4622dd6d8643032600aa1b50fbd12.zip"], # 2022-06-09 ) # Google benchmark. http_archive( - name = "com_github_google_benchmark", - urls = ["https://github.com/google/benchmark/archive/bf585a2789e30585b4e3ce6baf11ef2750b54677.zip"], # 2020-11-26T11:14:03Z - strip_prefix = "benchmark-bf585a2789e30585b4e3ce6baf11ef2750b54677", - sha256 = "2a778d821997df7d8646c9c59b8edb9a573a6e04c534c01892a40aa524a7b68c", + name = "com_github_google_benchmark", # 2021-09-20T09:19:51Z + sha256 = "62e2f2e6d8a744d67e4bbc212fcfd06647080de4253c97ad5c6749e09faf2cb0", + strip_prefix = "benchmark-0baacde3618ca617da95375e0af13ce1baadea47", + urls = ["https://github.com/google/benchmark/archive/0baacde3618ca617da95375e0af13ce1baadea47.zip"], +) + +# Bazel Skylib. +http_archive( + name = "bazel_skylib", + urls = ["https://github.com/bazelbuild/bazel-skylib/releases/download/1.2.1/bazel-skylib-1.2.1.tar.gz"], + sha256 = "f7be3474d42aae265405a592bb7da8e171919d74c16f082a5457840f06054728", ) -# C++ rules for Bazel. +# Bazel platform rules. http_archive( - name = "rules_cc", - sha256 = "9a446e9dd9c1bb180c86977a8dc1e9e659550ae732ae58bd2e8fd51e15b2c91d", - strip_prefix = "rules_cc-262ebec3c2296296526740db4aefce68c80de7fa", - urls = [ - "https://github.com/bazelbuild/rules_cc/archive/262ebec3c2296296526740db4aefce68c80de7fa.zip", - ], + name = "platforms", + sha256 = "a879ea428c6d56ab0ec18224f976515948822451473a80d06c2e50af0bbe5121", + strip_prefix = "platforms-da5541f26b7de1dc8e04c075c99df5351742a4a2", + urls = ["https://github.com/bazelbuild/platforms/archive/da5541f26b7de1dc8e04c075c99df5351742a4a2.zip"], # 2022-05-27 ) diff --git a/absl/BUILD.bazel b/absl/BUILD.bazel index c9d4a2da..7cccbbba 100644 --- a/absl/BUILD.bazel +++ b/absl/BUILD.bazel @@ -13,6 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. +load("@bazel_skylib//lib:selects.bzl", "selects") + package(default_visibility = ["//visibility:public"]) licenses(["notice"]) @@ -44,14 +46,14 @@ config_setting( config_setting( name = "osx", constraint_values = [ - "@bazel_tools//platforms:osx", + "@platforms//os:osx", ], ) config_setting( name = "ios", constraint_values = [ - "@bazel_tools//platforms:ios", + "@platforms//os:ios", ], ) @@ -64,9 +66,52 @@ config_setting( ) config_setting( - name = "wasm", + name = "cpu_wasm", + values = { + "cpu": "wasm", + }, + visibility = [":__subpackages__"], +) + +config_setting( + name = "cpu_wasm32", values = { "cpu": "wasm32", }, visibility = [":__subpackages__"], ) + +config_setting( + name = "platforms_wasm32", + constraint_values = [ + "@platforms//cpu:wasm32", + ], + visibility = [":__subpackages__"], +) + +config_setting( + name = "platforms_wasm64", + constraint_values = [ + "@platforms//cpu:wasm64", + ], + visibility = [":__subpackages__"], +) + +selects.config_setting_group( + name = "wasm", + match_any = [ + ":cpu_wasm", + ":cpu_wasm32", + ":platforms_wasm32", + ":platforms_wasm64", + ], + visibility = [":__subpackages__"], +) + +config_setting( + name = "fuchsia", + values = { + "cpu": "fuchsia", + }, + visibility = [":__subpackages__"], +) diff --git a/absl/CMakeLists.txt b/absl/CMakeLists.txt index a41e1eeb..b1715846 100644 --- a/absl/CMakeLists.txt +++ b/absl/CMakeLists.txt @@ -25,6 +25,7 @@ add_subdirectory(hash) add_subdirectory(memory) add_subdirectory(meta) add_subdirectory(numeric) +add_subdirectory(profiling) add_subdirectory(random) add_subdirectory(status) add_subdirectory(strings) diff --git a/absl/algorithm/BUILD.bazel b/absl/algorithm/BUILD.bazel index a3002b7d..f6d74714 100644 --- a/absl/algorithm/BUILD.bazel +++ b/absl/algorithm/BUILD.bazel @@ -14,7 +14,6 @@ # limitations under the License. # -load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") load( "//absl:copts/configure_copts.bzl", "ABSL_DEFAULT_COPTS", @@ -44,6 +43,7 @@ cc_test( linkopts = ABSL_DEFAULT_LINKOPTS, deps = [ ":algorithm", + "//absl/base:config", "@com_google_googletest//:gtest_main", ], ) diff --git a/absl/algorithm/CMakeLists.txt b/absl/algorithm/CMakeLists.txt index 56cd0fb8..181b49ca 100644 --- a/absl/algorithm/CMakeLists.txt +++ b/absl/algorithm/CMakeLists.txt @@ -35,7 +35,8 @@ absl_cc_test( ${ABSL_TEST_COPTS} DEPS absl::algorithm - gmock_main + absl::config + GTest::gmock_main ) absl_cc_library( @@ -65,5 +66,5 @@ absl_cc_test( absl::core_headers absl::memory absl::span - gmock_main + GTest::gmock_main ) diff --git a/absl/algorithm/algorithm_test.cc b/absl/algorithm/algorithm_test.cc index 81fccb61..d18df024 100644 --- a/absl/algorithm/algorithm_test.cc +++ b/absl/algorithm/algorithm_test.cc @@ -20,6 +20,7 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" +#include "absl/base/config.h" namespace { @@ -50,7 +51,15 @@ TEST(EqualTest, EmptyRange) { std::vector<int> empty1; std::vector<int> empty2; + // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=105705 +#if ABSL_INTERNAL_HAVE_MIN_GNUC_VERSION(12, 0) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wnonnull" +#endif EXPECT_FALSE(absl::equal(v1.begin(), v1.end(), empty1.begin(), empty1.end())); +#if ABSL_INTERNAL_HAVE_MIN_GNUC_VERSION(12, 0) +#pragma GCC diagnostic pop +#endif EXPECT_FALSE(absl::equal(empty1.begin(), empty1.end(), v1.begin(), v1.end())); EXPECT_TRUE( absl::equal(empty1.begin(), empty1.end(), empty2.begin(), empty2.end())); diff --git a/absl/algorithm/container.h b/absl/algorithm/container.h index 6398438f..26b19529 100644 --- a/absl/algorithm/container.h +++ b/absl/algorithm/container.h @@ -166,7 +166,7 @@ container_algorithm_internal::ContainerDifferenceType<const C> c_distance( // c_all_of() // // Container-based version of the <algorithm> `std::all_of()` function to -// test a condition on all elements within a container. +// test if all elements within a container satisfy a condition. template <typename C, typename Pred> bool c_all_of(const C& c, Pred&& pred) { return std::all_of(container_algorithm_internal::c_begin(c), @@ -905,11 +905,11 @@ void c_sort(C& c) { // Overload of c_sort() for performing a `comp` comparison other than the // default `operator<`. -template <typename C, typename Compare> -void c_sort(C& c, Compare&& comp) { +template <typename C, typename LessThan> +void c_sort(C& c, LessThan&& comp) { std::sort(container_algorithm_internal::c_begin(c), container_algorithm_internal::c_end(c), - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } // c_stable_sort() @@ -925,11 +925,11 @@ void c_stable_sort(C& c) { // Overload of c_stable_sort() for performing a `comp` comparison other than the // default `operator<`. -template <typename C, typename Compare> -void c_stable_sort(C& c, Compare&& comp) { +template <typename C, typename LessThan> +void c_stable_sort(C& c, LessThan&& comp) { std::stable_sort(container_algorithm_internal::c_begin(c), container_algorithm_internal::c_end(c), - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } // c_is_sorted() @@ -944,11 +944,11 @@ bool c_is_sorted(const C& c) { // c_is_sorted() overload for performing a `comp` comparison other than the // default `operator<`. -template <typename C, typename Compare> -bool c_is_sorted(const C& c, Compare&& comp) { +template <typename C, typename LessThan> +bool c_is_sorted(const C& c, LessThan&& comp) { return std::is_sorted(container_algorithm_internal::c_begin(c), container_algorithm_internal::c_end(c), - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } // c_partial_sort() @@ -966,14 +966,14 @@ void c_partial_sort( // Overload of c_partial_sort() for performing a `comp` comparison other than // the default `operator<`. -template <typename RandomAccessContainer, typename Compare> +template <typename RandomAccessContainer, typename LessThan> void c_partial_sort( RandomAccessContainer& sequence, container_algorithm_internal::ContainerIter<RandomAccessContainer> middle, - Compare&& comp) { + LessThan&& comp) { std::partial_sort(container_algorithm_internal::c_begin(sequence), middle, container_algorithm_internal::c_end(sequence), - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } // c_partial_sort_copy() @@ -994,15 +994,15 @@ c_partial_sort_copy(const C& sequence, RandomAccessContainer& result) { // Overload of c_partial_sort_copy() for performing a `comp` comparison other // than the default `operator<`. -template <typename C, typename RandomAccessContainer, typename Compare> +template <typename C, typename RandomAccessContainer, typename LessThan> container_algorithm_internal::ContainerIter<RandomAccessContainer> c_partial_sort_copy(const C& sequence, RandomAccessContainer& result, - Compare&& comp) { + LessThan&& comp) { return std::partial_sort_copy(container_algorithm_internal::c_begin(sequence), container_algorithm_internal::c_end(sequence), container_algorithm_internal::c_begin(result), container_algorithm_internal::c_end(result), - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } // c_is_sorted_until() @@ -1018,12 +1018,12 @@ container_algorithm_internal::ContainerIter<C> c_is_sorted_until(C& c) { // Overload of c_is_sorted_until() for performing a `comp` comparison other than // the default `operator<`. -template <typename C, typename Compare> +template <typename C, typename LessThan> container_algorithm_internal::ContainerIter<C> c_is_sorted_until( - C& c, Compare&& comp) { + C& c, LessThan&& comp) { return std::is_sorted_until(container_algorithm_internal::c_begin(c), container_algorithm_internal::c_end(c), - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } // c_nth_element() @@ -1043,14 +1043,14 @@ void c_nth_element( // Overload of c_nth_element() for performing a `comp` comparison other than // the default `operator<`. -template <typename RandomAccessContainer, typename Compare> +template <typename RandomAccessContainer, typename LessThan> void c_nth_element( RandomAccessContainer& sequence, container_algorithm_internal::ContainerIter<RandomAccessContainer> nth, - Compare&& comp) { + LessThan&& comp) { std::nth_element(container_algorithm_internal::c_begin(sequence), nth, container_algorithm_internal::c_end(sequence), - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } //------------------------------------------------------------------------------ @@ -1072,12 +1072,12 @@ container_algorithm_internal::ContainerIter<Sequence> c_lower_bound( // Overload of c_lower_bound() for performing a `comp` comparison other than // the default `operator<`. -template <typename Sequence, typename T, typename Compare> +template <typename Sequence, typename T, typename LessThan> container_algorithm_internal::ContainerIter<Sequence> c_lower_bound( - Sequence& sequence, T&& value, Compare&& comp) { + Sequence& sequence, T&& value, LessThan&& comp) { return std::lower_bound(container_algorithm_internal::c_begin(sequence), container_algorithm_internal::c_end(sequence), - std::forward<T>(value), std::forward<Compare>(comp)); + std::forward<T>(value), std::forward<LessThan>(comp)); } // c_upper_bound() @@ -1095,12 +1095,12 @@ container_algorithm_internal::ContainerIter<Sequence> c_upper_bound( // Overload of c_upper_bound() for performing a `comp` comparison other than // the default `operator<`. -template <typename Sequence, typename T, typename Compare> +template <typename Sequence, typename T, typename LessThan> container_algorithm_internal::ContainerIter<Sequence> c_upper_bound( - Sequence& sequence, T&& value, Compare&& comp) { + Sequence& sequence, T&& value, LessThan&& comp) { return std::upper_bound(container_algorithm_internal::c_begin(sequence), container_algorithm_internal::c_end(sequence), - std::forward<T>(value), std::forward<Compare>(comp)); + std::forward<T>(value), std::forward<LessThan>(comp)); } // c_equal_range() @@ -1118,12 +1118,12 @@ c_equal_range(Sequence& sequence, T&& value) { // Overload of c_equal_range() for performing a `comp` comparison other than // the default `operator<`. -template <typename Sequence, typename T, typename Compare> +template <typename Sequence, typename T, typename LessThan> container_algorithm_internal::ContainerIterPairType<Sequence, Sequence> -c_equal_range(Sequence& sequence, T&& value, Compare&& comp) { +c_equal_range(Sequence& sequence, T&& value, LessThan&& comp) { return std::equal_range(container_algorithm_internal::c_begin(sequence), container_algorithm_internal::c_end(sequence), - std::forward<T>(value), std::forward<Compare>(comp)); + std::forward<T>(value), std::forward<LessThan>(comp)); } // c_binary_search() @@ -1140,12 +1140,12 @@ bool c_binary_search(Sequence&& sequence, T&& value) { // Overload of c_binary_search() for performing a `comp` comparison other than // the default `operator<`. -template <typename Sequence, typename T, typename Compare> -bool c_binary_search(Sequence&& sequence, T&& value, Compare&& comp) { +template <typename Sequence, typename T, typename LessThan> +bool c_binary_search(Sequence&& sequence, T&& value, LessThan&& comp) { return std::binary_search(container_algorithm_internal::c_begin(sequence), container_algorithm_internal::c_end(sequence), std::forward<T>(value), - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } //------------------------------------------------------------------------------ @@ -1166,14 +1166,14 @@ OutputIterator c_merge(const C1& c1, const C2& c2, OutputIterator result) { // Overload of c_merge() for performing a `comp` comparison other than // the default `operator<`. -template <typename C1, typename C2, typename OutputIterator, typename Compare> +template <typename C1, typename C2, typename OutputIterator, typename LessThan> OutputIterator c_merge(const C1& c1, const C2& c2, OutputIterator result, - Compare&& comp) { + LessThan&& comp) { return std::merge(container_algorithm_internal::c_begin(c1), container_algorithm_internal::c_end(c1), container_algorithm_internal::c_begin(c2), container_algorithm_internal::c_end(c2), result, - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } // c_inplace_merge() @@ -1189,13 +1189,13 @@ void c_inplace_merge(C& c, // Overload of c_inplace_merge() for performing a merge using a `comp` other // than `operator<`. -template <typename C, typename Compare> +template <typename C, typename LessThan> void c_inplace_merge(C& c, container_algorithm_internal::ContainerIter<C> middle, - Compare&& comp) { + LessThan&& comp) { std::inplace_merge(container_algorithm_internal::c_begin(c), middle, container_algorithm_internal::c_end(c), - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } // c_includes() @@ -1213,13 +1213,13 @@ bool c_includes(const C1& c1, const C2& c2) { // Overload of c_includes() for performing a merge using a `comp` other than // `operator<`. -template <typename C1, typename C2, typename Compare> -bool c_includes(const C1& c1, const C2& c2, Compare&& comp) { +template <typename C1, typename C2, typename LessThan> +bool c_includes(const C1& c1, const C2& c2, LessThan&& comp) { return std::includes(container_algorithm_internal::c_begin(c1), container_algorithm_internal::c_end(c1), container_algorithm_internal::c_begin(c2), container_algorithm_internal::c_end(c2), - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } // c_set_union() @@ -1243,7 +1243,7 @@ OutputIterator c_set_union(const C1& c1, const C2& c2, OutputIterator output) { // Overload of c_set_union() for performing a merge using a `comp` other than // `operator<`. -template <typename C1, typename C2, typename OutputIterator, typename Compare, +template <typename C1, typename C2, typename OutputIterator, typename LessThan, typename = typename std::enable_if< !container_algorithm_internal::IsUnorderedContainer<C1>::value, void>::type, @@ -1251,18 +1251,18 @@ template <typename C1, typename C2, typename OutputIterator, typename Compare, !container_algorithm_internal::IsUnorderedContainer<C2>::value, void>::type> OutputIterator c_set_union(const C1& c1, const C2& c2, OutputIterator output, - Compare&& comp) { + LessThan&& comp) { return std::set_union(container_algorithm_internal::c_begin(c1), container_algorithm_internal::c_end(c1), container_algorithm_internal::c_begin(c2), container_algorithm_internal::c_end(c2), output, - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } // c_set_intersection() // // Container-based version of the <algorithm> `std::set_intersection()` function -// to return an iterator containing the intersection of two containers. +// to return an iterator containing the intersection of two sorted containers. template <typename C1, typename C2, typename OutputIterator, typename = typename std::enable_if< !container_algorithm_internal::IsUnorderedContainer<C1>::value, @@ -1272,6 +1272,11 @@ template <typename C1, typename C2, typename OutputIterator, void>::type> OutputIterator c_set_intersection(const C1& c1, const C2& c2, OutputIterator output) { + // In debug builds, ensure that both containers are sorted with respect to the + // default comparator. std::set_intersection requires the containers be sorted + // using operator<. + assert(absl::c_is_sorted(c1)); + assert(absl::c_is_sorted(c2)); return std::set_intersection(container_algorithm_internal::c_begin(c1), container_algorithm_internal::c_end(c1), container_algorithm_internal::c_begin(c2), @@ -1280,7 +1285,7 @@ OutputIterator c_set_intersection(const C1& c1, const C2& c2, // Overload of c_set_intersection() for performing a merge using a `comp` other // than `operator<`. -template <typename C1, typename C2, typename OutputIterator, typename Compare, +template <typename C1, typename C2, typename OutputIterator, typename LessThan, typename = typename std::enable_if< !container_algorithm_internal::IsUnorderedContainer<C1>::value, void>::type, @@ -1288,12 +1293,17 @@ template <typename C1, typename C2, typename OutputIterator, typename Compare, !container_algorithm_internal::IsUnorderedContainer<C2>::value, void>::type> OutputIterator c_set_intersection(const C1& c1, const C2& c2, - OutputIterator output, Compare&& comp) { + OutputIterator output, LessThan&& comp) { + // In debug builds, ensure that both containers are sorted with respect to the + // default comparator. std::set_intersection requires the containers be sorted + // using the same comparator. + assert(absl::c_is_sorted(c1, comp)); + assert(absl::c_is_sorted(c2, comp)); return std::set_intersection(container_algorithm_internal::c_begin(c1), container_algorithm_internal::c_end(c1), container_algorithm_internal::c_begin(c2), container_algorithm_internal::c_end(c2), output, - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } // c_set_difference() @@ -1318,7 +1328,7 @@ OutputIterator c_set_difference(const C1& c1, const C2& c2, // Overload of c_set_difference() for performing a merge using a `comp` other // than `operator<`. -template <typename C1, typename C2, typename OutputIterator, typename Compare, +template <typename C1, typename C2, typename OutputIterator, typename LessThan, typename = typename std::enable_if< !container_algorithm_internal::IsUnorderedContainer<C1>::value, void>::type, @@ -1326,12 +1336,12 @@ template <typename C1, typename C2, typename OutputIterator, typename Compare, !container_algorithm_internal::IsUnorderedContainer<C2>::value, void>::type> OutputIterator c_set_difference(const C1& c1, const C2& c2, - OutputIterator output, Compare&& comp) { + OutputIterator output, LessThan&& comp) { return std::set_difference(container_algorithm_internal::c_begin(c1), container_algorithm_internal::c_end(c1), container_algorithm_internal::c_begin(c2), container_algorithm_internal::c_end(c2), output, - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } // c_set_symmetric_difference() @@ -1357,7 +1367,7 @@ OutputIterator c_set_symmetric_difference(const C1& c1, const C2& c2, // Overload of c_set_symmetric_difference() for performing a merge using a // `comp` other than `operator<`. -template <typename C1, typename C2, typename OutputIterator, typename Compare, +template <typename C1, typename C2, typename OutputIterator, typename LessThan, typename = typename std::enable_if< !container_algorithm_internal::IsUnorderedContainer<C1>::value, void>::type, @@ -1366,13 +1376,13 @@ template <typename C1, typename C2, typename OutputIterator, typename Compare, void>::type> OutputIterator c_set_symmetric_difference(const C1& c1, const C2& c2, OutputIterator output, - Compare&& comp) { + LessThan&& comp) { return std::set_symmetric_difference( container_algorithm_internal::c_begin(c1), container_algorithm_internal::c_end(c1), container_algorithm_internal::c_begin(c2), container_algorithm_internal::c_end(c2), output, - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } //------------------------------------------------------------------------------ @@ -1391,11 +1401,11 @@ void c_push_heap(RandomAccessContainer& sequence) { // Overload of c_push_heap() for performing a push operation on a heap using a // `comp` other than `operator<`. -template <typename RandomAccessContainer, typename Compare> -void c_push_heap(RandomAccessContainer& sequence, Compare&& comp) { +template <typename RandomAccessContainer, typename LessThan> +void c_push_heap(RandomAccessContainer& sequence, LessThan&& comp) { std::push_heap(container_algorithm_internal::c_begin(sequence), container_algorithm_internal::c_end(sequence), - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } // c_pop_heap() @@ -1410,11 +1420,11 @@ void c_pop_heap(RandomAccessContainer& sequence) { // Overload of c_pop_heap() for performing a pop operation on a heap using a // `comp` other than `operator<`. -template <typename RandomAccessContainer, typename Compare> -void c_pop_heap(RandomAccessContainer& sequence, Compare&& comp) { +template <typename RandomAccessContainer, typename LessThan> +void c_pop_heap(RandomAccessContainer& sequence, LessThan&& comp) { std::pop_heap(container_algorithm_internal::c_begin(sequence), container_algorithm_internal::c_end(sequence), - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } // c_make_heap() @@ -1429,11 +1439,11 @@ void c_make_heap(RandomAccessContainer& sequence) { // Overload of c_make_heap() for performing heap comparisons using a // `comp` other than `operator<` -template <typename RandomAccessContainer, typename Compare> -void c_make_heap(RandomAccessContainer& sequence, Compare&& comp) { +template <typename RandomAccessContainer, typename LessThan> +void c_make_heap(RandomAccessContainer& sequence, LessThan&& comp) { std::make_heap(container_algorithm_internal::c_begin(sequence), container_algorithm_internal::c_end(sequence), - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } // c_sort_heap() @@ -1448,11 +1458,11 @@ void c_sort_heap(RandomAccessContainer& sequence) { // Overload of c_sort_heap() for performing heap comparisons using a // `comp` other than `operator<` -template <typename RandomAccessContainer, typename Compare> -void c_sort_heap(RandomAccessContainer& sequence, Compare&& comp) { +template <typename RandomAccessContainer, typename LessThan> +void c_sort_heap(RandomAccessContainer& sequence, LessThan&& comp) { std::sort_heap(container_algorithm_internal::c_begin(sequence), container_algorithm_internal::c_end(sequence), - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } // c_is_heap() @@ -1467,11 +1477,11 @@ bool c_is_heap(const RandomAccessContainer& sequence) { // Overload of c_is_heap() for performing heap comparisons using a // `comp` other than `operator<` -template <typename RandomAccessContainer, typename Compare> -bool c_is_heap(const RandomAccessContainer& sequence, Compare&& comp) { +template <typename RandomAccessContainer, typename LessThan> +bool c_is_heap(const RandomAccessContainer& sequence, LessThan&& comp) { return std::is_heap(container_algorithm_internal::c_begin(sequence), container_algorithm_internal::c_end(sequence), - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } // c_is_heap_until() @@ -1487,12 +1497,12 @@ c_is_heap_until(RandomAccessContainer& sequence) { // Overload of c_is_heap_until() for performing heap comparisons using a // `comp` other than `operator<` -template <typename RandomAccessContainer, typename Compare> +template <typename RandomAccessContainer, typename LessThan> container_algorithm_internal::ContainerIter<RandomAccessContainer> -c_is_heap_until(RandomAccessContainer& sequence, Compare&& comp) { +c_is_heap_until(RandomAccessContainer& sequence, LessThan&& comp) { return std::is_heap_until(container_algorithm_internal::c_begin(sequence), container_algorithm_internal::c_end(sequence), - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } //------------------------------------------------------------------------------ @@ -1513,12 +1523,12 @@ container_algorithm_internal::ContainerIter<Sequence> c_min_element( // Overload of c_min_element() for performing a `comp` comparison other than // `operator<`. -template <typename Sequence, typename Compare> +template <typename Sequence, typename LessThan> container_algorithm_internal::ContainerIter<Sequence> c_min_element( - Sequence& sequence, Compare&& comp) { + Sequence& sequence, LessThan&& comp) { return std::min_element(container_algorithm_internal::c_begin(sequence), container_algorithm_internal::c_end(sequence), - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } // c_max_element() @@ -1535,12 +1545,12 @@ container_algorithm_internal::ContainerIter<Sequence> c_max_element( // Overload of c_max_element() for performing a `comp` comparison other than // `operator<`. -template <typename Sequence, typename Compare> +template <typename Sequence, typename LessThan> container_algorithm_internal::ContainerIter<Sequence> c_max_element( - Sequence& sequence, Compare&& comp) { + Sequence& sequence, LessThan&& comp) { return std::max_element(container_algorithm_internal::c_begin(sequence), container_algorithm_internal::c_end(sequence), - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } // c_minmax_element() @@ -1558,12 +1568,12 @@ c_minmax_element(C& c) { // Overload of c_minmax_element() for performing `comp` comparisons other than // `operator<`. -template <typename C, typename Compare> +template <typename C, typename LessThan> container_algorithm_internal::ContainerIterPairType<C, C> -c_minmax_element(C& c, Compare&& comp) { +c_minmax_element(C& c, LessThan&& comp) { return std::minmax_element(container_algorithm_internal::c_begin(c), container_algorithm_internal::c_end(c), - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } //------------------------------------------------------------------------------ @@ -1588,15 +1598,15 @@ bool c_lexicographical_compare(Sequence1&& sequence1, Sequence2&& sequence2) { // Overload of c_lexicographical_compare() for performing a lexicographical // comparison using a `comp` operator instead of `operator<`. -template <typename Sequence1, typename Sequence2, typename Compare> +template <typename Sequence1, typename Sequence2, typename LessThan> bool c_lexicographical_compare(Sequence1&& sequence1, Sequence2&& sequence2, - Compare&& comp) { + LessThan&& comp) { return std::lexicographical_compare( container_algorithm_internal::c_begin(sequence1), container_algorithm_internal::c_end(sequence1), container_algorithm_internal::c_begin(sequence2), container_algorithm_internal::c_end(sequence2), - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } // c_next_permutation() @@ -1612,11 +1622,11 @@ bool c_next_permutation(C& c) { // Overload of c_next_permutation() for performing a lexicographical // comparison using a `comp` operator instead of `operator<`. -template <typename C, typename Compare> -bool c_next_permutation(C& c, Compare&& comp) { +template <typename C, typename LessThan> +bool c_next_permutation(C& c, LessThan&& comp) { return std::next_permutation(container_algorithm_internal::c_begin(c), container_algorithm_internal::c_end(c), - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } // c_prev_permutation() @@ -1632,11 +1642,11 @@ bool c_prev_permutation(C& c) { // Overload of c_prev_permutation() for performing a lexicographical // comparison using a `comp` operator instead of `operator<`. -template <typename C, typename Compare> -bool c_prev_permutation(C& c, Compare&& comp) { +template <typename C, typename LessThan> +bool c_prev_permutation(C& c, LessThan&& comp) { return std::prev_permutation(container_algorithm_internal::c_begin(c), container_algorithm_internal::c_end(c), - std::forward<Compare>(comp)); + std::forward<LessThan>(comp)); } //------------------------------------------------------------------------------ diff --git a/absl/base/BUILD.bazel b/absl/base/BUILD.bazel index 65ff0dde..bd023ad8 100644 --- a/absl/base/BUILD.bazel +++ b/absl/base/BUILD.bazel @@ -14,7 +14,6 @@ # limitations under the License. # -load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test") load( "//absl:copts/configure_copts.bzl", "ABSL_DEFAULT_COPTS", @@ -76,6 +75,7 @@ cc_library( ":atomic_hook", ":config", ":core_headers", + ":errno_saver", ":log_severity", ], ) @@ -158,7 +158,9 @@ cc_library( "internal/direct_mmap.h", "internal/low_level_alloc.h", ], - copts = ABSL_DEFAULT_COPTS, + copts = ABSL_DEFAULT_COPTS + select({ + "//conditions:default": [], + }), linkopts = select({ "//absl:msvc_compiler": [], "//absl:clang-cl_compiler": [], @@ -433,6 +435,9 @@ cc_test( srcs = ["spinlock_test_common.cc"], copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + tags = [ + "no_test_wasm", + ], deps = [ ":base", ":base_internal", @@ -558,6 +563,7 @@ cc_test( linkopts = ABSL_DEFAULT_LINKOPTS, tags = [ "no_test_ios_x86_64", + "no_test_wasm", ], deps = [ ":malloc_internal", @@ -571,6 +577,9 @@ cc_test( srcs = ["internal/thread_identity_test.cc"], copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + tags = [ + "no_test_wasm", + ], deps = [ ":base", ":core_headers", @@ -594,75 +603,6 @@ cc_test( ) cc_library( - name = "exponential_biased", - srcs = ["internal/exponential_biased.cc"], - hdrs = ["internal/exponential_biased.h"], - linkopts = ABSL_DEFAULT_LINKOPTS, - visibility = [ - "//absl:__subpackages__", - ], - deps = [ - ":config", - ":core_headers", - ], -) - -cc_test( - name = "exponential_biased_test", - size = "small", - srcs = ["internal/exponential_biased_test.cc"], - copts = ABSL_TEST_COPTS, - linkopts = ABSL_DEFAULT_LINKOPTS, - visibility = ["//visibility:private"], - deps = [ - ":exponential_biased", - "//absl/strings", - "@com_google_googletest//:gtest_main", - ], -) - -cc_library( - name = "periodic_sampler", - srcs = ["internal/periodic_sampler.cc"], - hdrs = ["internal/periodic_sampler.h"], - copts = ABSL_DEFAULT_COPTS, - linkopts = ABSL_DEFAULT_LINKOPTS, - deps = [ - ":core_headers", - ":exponential_biased", - ], -) - -cc_test( - name = "periodic_sampler_test", - size = "small", - srcs = ["internal/periodic_sampler_test.cc"], - copts = ABSL_TEST_COPTS, - linkopts = ABSL_DEFAULT_LINKOPTS, - visibility = ["//visibility:private"], - deps = [ - ":core_headers", - ":periodic_sampler", - "@com_google_googletest//:gtest_main", - ], -) - -cc_binary( - name = "periodic_sampler_benchmark", - testonly = 1, - srcs = ["internal/periodic_sampler_benchmark.cc"], - copts = ABSL_TEST_COPTS, - linkopts = ABSL_DEFAULT_LINKOPTS, - tags = ["benchmark"], - visibility = ["//visibility:private"], - deps = [ - ":core_headers", - ":periodic_sampler", - "@com_github_google_benchmark//:benchmark_main", - ], -) - -cc_library( name = "scoped_set_env", testonly = 1, srcs = ["internal/scoped_set_env.cc"], @@ -772,6 +712,31 @@ cc_test( ], ) +cc_library( + name = "prefetch", + hdrs = ["internal/prefetch.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "//absl:__subpackages__", + ], + deps = [ + ":config", + ], +) + +cc_test( + name = "prefetch_test", + size = "small", + srcs = ["internal/prefetch_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":prefetch", + "@com_google_googletest//:gtest_main", + ], +) + cc_test( name = "unique_small_name_test", size = "small", diff --git a/absl/base/CMakeLists.txt b/absl/base/CMakeLists.txt index 981b8cc0..ed55093a 100644 --- a/absl/base/CMakeLists.txt +++ b/absl/base/CMakeLists.txt @@ -16,6 +16,7 @@ find_library(LIBRT rt) +# Internal-only target, do not depend on directly. absl_cc_library( NAME atomic_hook @@ -28,6 +29,7 @@ absl_cc_library( ${ABSL_DEFAULT_COPTS} ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME errno_saver @@ -52,6 +54,7 @@ absl_cc_library( ${ABSL_DEFAULT_COPTS} ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME raw_logging_internal @@ -63,11 +66,13 @@ absl_cc_library( absl::atomic_hook absl::config absl::core_headers + absl::errno_saver absl::log_severity COPTS ${ABSL_DEFAULT_COPTS} ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME spinlock_wait @@ -131,6 +136,7 @@ absl_cc_library( PUBLIC ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME malloc_internal @@ -151,6 +157,7 @@ absl_cc_library( Threads::Threads ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME base_internal @@ -207,6 +214,7 @@ absl_cc_library( PUBLIC ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME throw_delegate @@ -221,6 +229,7 @@ absl_cc_library( absl::raw_logging_internal ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME exception_testing @@ -230,10 +239,11 @@ absl_cc_library( ${ABSL_DEFAULT_COPTS} DEPS absl::config - gtest + GTest::gtest TESTONLY ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME pretty_function @@ -243,6 +253,7 @@ absl_cc_library( ${ABSL_DEFAULT_COPTS} ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME exception_safety_testing @@ -259,7 +270,7 @@ absl_cc_library( absl::meta absl::strings absl::utility - gtest + GTest::gtest TESTONLY ) @@ -273,9 +284,10 @@ absl_cc_test( DEPS absl::exception_safety_testing absl::memory - gtest_main + GTest::gtest_main ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME atomic_hook_test_helper @@ -300,8 +312,8 @@ absl_cc_test( absl::atomic_hook_test_helper absl::atomic_hook absl::core_headers - gmock - gtest_main + GTest::gmock + GTest::gtest_main ) absl_cc_test( @@ -314,7 +326,7 @@ absl_cc_test( DEPS absl::base absl::core_headers - gtest_main + GTest::gtest_main ) absl_cc_test( @@ -327,8 +339,8 @@ absl_cc_test( DEPS absl::errno_saver absl::strerror - gmock - gtest_main + GTest::gmock + GTest::gtest_main ) absl_cc_test( @@ -342,7 +354,7 @@ absl_cc_test( absl::base absl::config absl::throw_delegate - gtest_main + GTest::gtest_main ) absl_cc_test( @@ -357,7 +369,7 @@ absl_cc_test( ${ABSL_TEST_COPTS} DEPS absl::base_internal - gtest_main + GTest::gtest_main ) absl_cc_test( @@ -371,10 +383,11 @@ absl_cc_test( absl::base_internal absl::memory absl::strings - gmock - gtest_main + GTest::gmock + GTest::gtest_main ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME spinlock_test_common @@ -388,7 +401,7 @@ absl_cc_library( absl::base_internal absl::core_headers absl::synchronization - gtest + GTest::gtest TESTONLY ) @@ -406,9 +419,10 @@ absl_cc_test( absl::config absl::core_headers absl::synchronization - gtest_main + GTest::gtest_main ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME endian @@ -435,7 +449,7 @@ absl_cc_test( absl::base absl::config absl::endian - gtest_main + GTest::gtest_main ) absl_cc_test( @@ -448,7 +462,7 @@ absl_cc_test( DEPS absl::config absl::synchronization - gtest_main + GTest::gtest_main ) absl_cc_test( @@ -462,7 +476,7 @@ absl_cc_test( absl::base absl::core_headers absl::synchronization - gtest_main + GTest::gtest_main ) absl_cc_test( @@ -475,7 +489,7 @@ absl_cc_test( DEPS absl::raw_logging_internal absl::strings - gtest_main + GTest::gtest_main ) absl_cc_test( @@ -488,7 +502,7 @@ absl_cc_test( DEPS absl::base absl::synchronization - gtest_main + GTest::gtest_main ) absl_cc_test( @@ -516,63 +530,10 @@ absl_cc_test( absl::core_headers absl::synchronization Threads::Threads - gtest_main -) - -absl_cc_library( - NAME - exponential_biased - SRCS - "internal/exponential_biased.cc" - HDRS - "internal/exponential_biased.h" - COPTS - ${ABSL_DEFAULT_COPTS} - DEPS - absl::config - absl::core_headers -) - -absl_cc_test( - NAME - exponential_biased_test - SRCS - "internal/exponential_biased_test.cc" - COPTS - ${ABSL_TEST_COPTS} - DEPS - absl::exponential_biased - absl::strings - gmock_main -) - -absl_cc_library( - NAME - periodic_sampler - SRCS - "internal/periodic_sampler.cc" - HDRS - "internal/periodic_sampler.h" - COPTS - ${ABSL_DEFAULT_COPTS} - DEPS - absl::core_headers - absl::exponential_biased -) - -absl_cc_test( - NAME - periodic_sampler_test - SRCS - "internal/periodic_sampler_test.cc" - COPTS - ${ABSL_TEST_COPTS} - DEPS - absl::core_headers - absl::periodic_sampler - gmock_main + GTest::gtest_main ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME scoped_set_env @@ -596,7 +557,7 @@ absl_cc_test( ${ABSL_TEST_COPTS} DEPS absl::scoped_set_env - gtest_main + GTest::gtest_main ) absl_cc_test( @@ -620,10 +581,11 @@ absl_cc_test( absl::flags_marshalling absl::log_severity absl::strings - gmock - gtest_main + GTest::gmock + GTest::gtest_main ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME strerror @@ -651,10 +613,11 @@ absl_cc_test( DEPS absl::strerror absl::strings - gmock - gtest_main + GTest::gmock + GTest::gtest_main ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME fast_type_id @@ -677,7 +640,33 @@ absl_cc_test( ${ABSL_TEST_COPTS} DEPS absl::fast_type_id - gtest_main + GTest::gtest_main +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + prefetch + HDRS + "internal/prefetch.h" + COPTS + ${ABSL_DEFAULT_COPTS} + LINKOPTS + ${ABSL_DEFAULT_LINKOPTS} + DEPS + absl::config +) + +absl_cc_test( + NAME + prefetch_test + SRCS + "internal/prefetch_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::prefetch + GTest::gtest_main ) absl_cc_test( @@ -690,5 +679,5 @@ absl_cc_test( DEPS absl::core_headers absl::optional - gtest_main + GTest::gtest_main ) diff --git a/absl/base/attributes.h b/absl/base/attributes.h index cf2cb550..e4e7a3d8 100644 --- a/absl/base/attributes.h +++ b/absl/base/attributes.h @@ -131,14 +131,15 @@ // ABSL_ATTRIBUTE_WEAK // // Tags a function as weak for the purposes of compilation and linking. -// Weak attributes currently do not work properly in LLVM's Windows backend, -// so disable them there. See https://bugs.llvm.org/show_bug.cgi?id=37598 +// Weak attributes did not work properly in LLVM's Windows backend before +// 9.0.0, so disable them there. See https://bugs.llvm.org/show_bug.cgi?id=37598 // for further information. // The MinGW compiler doesn't complain about the weak attribute until the link // step, presumably because Windows doesn't use ELF binaries. -#if (ABSL_HAVE_ATTRIBUTE(weak) || \ - (defined(__GNUC__) && !defined(__clang__))) && \ - !(defined(__llvm__) && defined(_WIN32)) && !defined(__MINGW32__) +#if (ABSL_HAVE_ATTRIBUTE(weak) || \ + (defined(__GNUC__) && !defined(__clang__))) && \ + (!defined(_WIN32) || (defined(__clang__) && __clang_major__ >= 9)) && \ + !defined(__MINGW32__) #undef ABSL_ATTRIBUTE_WEAK #define ABSL_ATTRIBUTE_WEAK __attribute__((weak)) #define ABSL_HAVE_ATTRIBUTE_WEAK 1 @@ -212,6 +213,9 @@ // https://gcc.gnu.org/gcc-4.8/changes.html #if ABSL_HAVE_ATTRIBUTE(no_sanitize_address) #define ABSL_ATTRIBUTE_NO_SANITIZE_ADDRESS __attribute__((no_sanitize_address)) +#elif defined(_MSC_VER) && _MSC_VER >= 1928 +// https://docs.microsoft.com/en-us/cpp/cpp/no-sanitize-address +#define ABSL_ATTRIBUTE_NO_SANITIZE_ADDRESS __declspec(no_sanitize_address) #else #define ABSL_ATTRIBUTE_NO_SANITIZE_ADDRESS #endif @@ -281,10 +285,7 @@ // ABSL_ATTRIBUTE_RETURNS_NONNULL // // Tells the compiler that a particular function never returns a null pointer. -#if ABSL_HAVE_ATTRIBUTE(returns_nonnull) || \ - (defined(__GNUC__) && \ - (__GNUC__ > 5 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 9)) && \ - !defined(__clang__)) +#if ABSL_HAVE_ATTRIBUTE(returns_nonnull) #define ABSL_ATTRIBUTE_RETURNS_NONNULL __attribute__((returns_nonnull)) #else #define ABSL_ATTRIBUTE_RETURNS_NONNULL @@ -314,15 +315,22 @@ __attribute__((section(#name))) __attribute__((noinline)) #endif - // ABSL_ATTRIBUTE_SECTION_VARIABLE // // Tells the compiler/linker to put a given variable into a section and define // `__start_ ## name` and `__stop_ ## name` symbols to bracket the section. // This functionality is supported by GNU linker. #ifndef ABSL_ATTRIBUTE_SECTION_VARIABLE +#ifdef _AIX +// __attribute__((section(#name))) on AIX is achived by using the `.csect` psudo +// op which includes an additional integer as part of its syntax indcating +// alignment. If data fall under different alignments then you might get a +// compilation error indicating a `Section type conflict`. +#define ABSL_ATTRIBUTE_SECTION_VARIABLE(name) +#else #define ABSL_ATTRIBUTE_SECTION_VARIABLE(name) __attribute__((section(#name))) #endif +#endif // ABSL_DECLARE_ATTRIBUTE_SECTION_VARS // @@ -333,8 +341,8 @@ // a no-op on ELF but not on Mach-O. // #ifndef ABSL_DECLARE_ATTRIBUTE_SECTION_VARS -#define ABSL_DECLARE_ATTRIBUTE_SECTION_VARS(name) \ - extern char __start_##name[] ABSL_ATTRIBUTE_WEAK; \ +#define ABSL_DECLARE_ATTRIBUTE_SECTION_VARS(name) \ + extern char __start_##name[] ABSL_ATTRIBUTE_WEAK; \ extern char __stop_##name[] ABSL_ATTRIBUTE_WEAK #endif #ifndef ABSL_DEFINE_ATTRIBUTE_SECTION_VARS @@ -395,6 +403,9 @@ // // Tells the compiler to warn about unused results. // +// For code or headers that are assured to only build with C++17 and up, prefer +// just using the standard `[[nodiscard]]` directly over this macro. +// // When annotating a function, it must appear as the first part of the // declaration or definition. The compiler will warn if the return value from // such a function is unused: @@ -421,9 +432,10 @@ // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=66425 // // Note: past advice was to place the macro after the argument list. -#if ABSL_HAVE_ATTRIBUTE(nodiscard) -#define ABSL_MUST_USE_RESULT [[nodiscard]] -#elif defined(__clang__) && ABSL_HAVE_ATTRIBUTE(warn_unused_result) +// +// TODO(b/176172494): Use ABSL_HAVE_CPP_ATTRIBUTE(nodiscard) when all code is +// compliant with the stricter [[nodiscard]]. +#if defined(__clang__) && ABSL_HAVE_ATTRIBUTE(warn_unused_result) #define ABSL_MUST_USE_RESULT __attribute__((warn_unused_result)) #else #define ABSL_MUST_USE_RESULT @@ -493,7 +505,7 @@ #define ABSL_XRAY_NEVER_INSTRUMENT [[clang::xray_never_instrument]] #if ABSL_HAVE_CPP_ATTRIBUTE(clang::xray_log_args) #define ABSL_XRAY_LOG_ARGS(N) \ - [[clang::xray_always_instrument, clang::xray_log_args(N)]] + [[clang::xray_always_instrument, clang::xray_log_args(N)]] #else #define ABSL_XRAY_LOG_ARGS(N) [[clang::xray_always_instrument]] #endif @@ -524,6 +536,13 @@ // ABSL_ATTRIBUTE_UNUSED // // Prevents the compiler from complaining about variables that appear unused. +// +// For code or headers that are assured to only build with C++17 and up, prefer +// just using the standard '[[maybe_unused]]' directly over this macro. +// +// Due to differences in positioning requirements between the old, compiler +// specific __attribute__ syntax and the now standard [[maybe_unused]], this +// macro does not attempt to take advantage of '[[maybe_unused]]'. #if ABSL_HAVE_ATTRIBUTE(unused) || (defined(__GNUC__) && !defined(__clang__)) #undef ABSL_ATTRIBUTE_UNUSED #define ABSL_ATTRIBUTE_UNUSED __attribute__((__unused__)) @@ -544,13 +563,19 @@ // ABSL_ATTRIBUTE_PACKED // // Instructs the compiler not to use natural alignment for a tagged data -// structure, but instead to reduce its alignment to 1. This attribute can -// either be applied to members of a structure or to a structure in its -// entirety. Applying this attribute (judiciously) to a structure in its -// entirety to optimize the memory footprint of very commonly-used structs is -// fine. Do not apply this attribute to a structure in its entirety if the -// purpose is to control the offsets of the members in the structure. Instead, -// apply this attribute only to structure members that need it. +// structure, but instead to reduce its alignment to 1. +// +// Therefore, DO NOT APPLY THIS ATTRIBUTE TO STRUCTS CONTAINING ATOMICS. Doing +// so can cause atomic variables to be mis-aligned and silently violate +// atomicity on x86. +// +// This attribute can either be applied to members of a structure or to a +// structure in its entirety. Applying this attribute (judiciously) to a +// structure in its entirety to optimize the memory footprint of very +// commonly-used structs is fine. Do not apply this attribute to a structure in +// its entirety if the purpose is to control the offsets of the members in the +// structure. Instead, apply this attribute only to structure members that need +// it. // // When applying ABSL_ATTRIBUTE_PACKED only to specific structure members the // natural alignment of structure members not annotated is preserved. Aligned @@ -595,31 +620,24 @@ // case 42: // ... // -// Notes: when compiled with clang in C++11 mode, the ABSL_FALLTHROUGH_INTENDED -// macro is expanded to the [[clang::fallthrough]] attribute, which is analysed -// when performing switch labels fall-through diagnostic -// (`-Wimplicit-fallthrough`). See clang documentation on language extensions -// for details: +// Notes: When supported, GCC and Clang can issue a warning on switch labels +// with unannotated fallthrough using the warning `-Wimplicit-fallthrough`. See +// clang documentation on language extensions for details: // https://clang.llvm.org/docs/AttributeReference.html#fallthrough-clang-fallthrough // -// When used with unsupported compilers, the ABSL_FALLTHROUGH_INTENDED macro -// has no effect on diagnostics. In any case this macro has no effect on runtime +// When used with unsupported compilers, the ABSL_FALLTHROUGH_INTENDED macro has +// no effect on diagnostics. In any case this macro has no effect on runtime // behavior and performance of code. #ifdef ABSL_FALLTHROUGH_INTENDED #error "ABSL_FALLTHROUGH_INTENDED should not be defined." -#endif - -// TODO(zhangxy): Use c++17 standard [[fallthrough]] macro, when supported. -#if defined(__clang__) && defined(__has_warning) -#if __has_feature(cxx_attributes) && __has_warning("-Wimplicit-fallthrough") +#elif ABSL_HAVE_CPP_ATTRIBUTE(fallthrough) +#define ABSL_FALLTHROUGH_INTENDED [[fallthrough]] +#elif ABSL_HAVE_CPP_ATTRIBUTE(clang::fallthrough) #define ABSL_FALLTHROUGH_INTENDED [[clang::fallthrough]] -#endif -#elif defined(__GNUC__) && __GNUC__ >= 7 +#elif ABSL_HAVE_CPP_ATTRIBUTE(gnu::fallthrough) #define ABSL_FALLTHROUGH_INTENDED [[gnu::fallthrough]] -#endif - -#ifndef ABSL_FALLTHROUGH_INTENDED +#else #define ABSL_FALLTHROUGH_INTENDED \ do { \ } while (0) @@ -631,6 +649,9 @@ // declarations. The macro argument is used as a custom diagnostic message (e.g. // suggestion of a better alternative). // +// For code or headers that are assured to only build with C++14 and up, prefer +// just using the standard `[[deprecated("message")]]` directly over this macro. +// // Examples: // // class ABSL_DEPRECATED("Use Bar instead") Foo {...}; @@ -641,14 +662,17 @@ // ABSL_DEPRECATED("Use DoThat() instead") // void DoThis(); // +// enum FooEnum { +// kBar ABSL_DEPRECATED("Use kBaz instead"), +// }; +// // Every usage of a deprecated entity will trigger a warning when compiled with -// clang's `-Wdeprecated-declarations` option. This option is turned off by -// default, but the warnings will be reported by clang-tidy. -#if defined(__clang__) && defined(__cplusplus) && __cplusplus >= 201103L +// GCC/Clang's `-Wdeprecated-declarations` option. Google's production toolchain +// turns this warning off by default, instead relying on clang-tidy to report +// new uses of deprecated code. +#if ABSL_HAVE_ATTRIBUTE(deprecated) #define ABSL_DEPRECATED(message) __attribute__((deprecated(message))) -#endif - -#ifndef ABSL_DEPRECATED +#else #define ABSL_DEPRECATED(message) #endif @@ -658,9 +682,18 @@ // not compile (on supported platforms) unless the variable has a constant // initializer. This is useful for variables with static and thread storage // duration, because it guarantees that they will not suffer from the so-called -// "static init order fiasco". Prefer to put this attribute on the most visible -// declaration of the variable, if there's more than one, because code that -// accesses the variable can then use the attribute for optimization. +// "static init order fiasco". +// +// This attribute must be placed on the initializing declaration of the +// variable. Some compilers will give a -Wmissing-constinit warning when this +// attribute is placed on some other declaration but missing from the +// initializing declaration. +// +// In some cases (notably with thread_local variables), `ABSL_CONST_INIT` can +// also be used in a non-initializing declaration to tell the compiler that a +// variable is already initialized, reducing overhead that would otherwise be +// incurred by a hidden guard variable. Thus annotating all declarations with +// this attribute is recommended to potentially enhance optimization. // // Example: // @@ -669,14 +702,19 @@ // ABSL_CONST_INIT static MyType my_var; // }; // -// MyType MyClass::my_var = MakeMyType(...); +// ABSL_CONST_INIT MyType MyClass::my_var = MakeMyType(...); +// +// For code or headers that are assured to only build with C++20 and up, prefer +// just using the standard `constinit` keyword directly over this macro. // // Note that this attribute is redundant if the variable is declared constexpr. -#if ABSL_HAVE_CPP_ATTRIBUTE(clang::require_constant_initialization) +#if defined(__cpp_constinit) && __cpp_constinit >= 201907L +#define ABSL_CONST_INIT constinit +#elif ABSL_HAVE_CPP_ATTRIBUTE(clang::require_constant_initialization) #define ABSL_CONST_INIT [[clang::require_constant_initialization]] #else #define ABSL_CONST_INIT -#endif // ABSL_HAVE_CPP_ATTRIBUTE(clang::require_constant_initialization) +#endif // ABSL_ATTRIBUTE_PURE_FUNCTION // @@ -699,4 +737,26 @@ #define ABSL_ATTRIBUTE_PURE_FUNCTION #endif +// ABSL_ATTRIBUTE_LIFETIME_BOUND indicates that a resource owned by a function +// parameter or implicit object parameter is retained by the return value of the +// annotated function (or, for a parameter of a constructor, in the value of the +// constructed object). This attribute causes warnings to be produced if a +// temporary object does not live long enough. +// +// When applied to a reference parameter, the referenced object is assumed to be +// retained by the return value of the function. When applied to a non-reference +// parameter (for example, a pointer or a class type), all temporaries +// referenced by the parameter are assumed to be retained by the return value of +// the function. +// +// See also the upstream documentation: +// https://clang.llvm.org/docs/AttributeReference.html#lifetimebound +#if ABSL_HAVE_CPP_ATTRIBUTE(clang::lifetimebound) +#define ABSL_ATTRIBUTE_LIFETIME_BOUND [[clang::lifetimebound]] +#elif ABSL_HAVE_ATTRIBUTE(lifetimebound) +#define ABSL_ATTRIBUTE_LIFETIME_BOUND __attribute__((lifetimebound)) +#else +#define ABSL_ATTRIBUTE_LIFETIME_BOUND +#endif + #endif // ABSL_BASE_ATTRIBUTES_H_ diff --git a/absl/base/casts.h b/absl/base/casts.h index 83c69126..b99adb06 100644 --- a/absl/base/casts.h +++ b/absl/base/casts.h @@ -29,6 +29,10 @@ #include <type_traits> #include <utility> +#if defined(__cpp_lib_bit_cast) && __cpp_lib_bit_cast >= 201806L +#include <bit> // For std::bit_cast. +#endif // defined(__cpp_lib_bit_cast) && __cpp_lib_bit_cast >= 201806L + #include "absl/base/internal/identity.h" #include "absl/base/macros.h" #include "absl/meta/type_traits.h" @@ -36,19 +40,6 @@ namespace absl { ABSL_NAMESPACE_BEGIN -namespace internal_casts { - -template <class Dest, class Source> -struct is_bitcastable - : std::integral_constant< - bool, - sizeof(Dest) == sizeof(Source) && - type_traits_internal::is_trivially_copyable<Source>::value && - type_traits_internal::is_trivially_copyable<Dest>::value && - std::is_default_constructible<Dest>::value> {}; - -} // namespace internal_casts - // implicit_cast() // // Performs an implicit conversion between types following the language @@ -105,81 +96,83 @@ constexpr To implicit_cast(typename absl::internal::identity_t<To> to) { // bit_cast() // -// Performs a bitwise cast on a type without changing the underlying bit -// representation of that type's value. The two types must be of the same size -// and both types must be trivially copyable. As with most casts, use with -// caution. A `bit_cast()` might be needed when you need to temporarily treat a -// type as some other type, such as in the following cases: +// Creates a value of the new type `Dest` whose representation is the same as +// that of the argument, which is of (deduced) type `Source` (a "bitwise cast"; +// every bit in the value representation of the result is equal to the +// corresponding bit in the object representation of the source). Source and +// destination types must be of the same size, and both types must be trivially +// copyable. // -// * Serialization (casting temporarily to `char *` for those purposes is -// always allowed by the C++ standard) -// * Managing the individual bits of a type within mathematical operations -// that are not normally accessible through that type -// * Casting non-pointer types to pointer types (casting the other way is -// allowed by `reinterpret_cast()` but round-trips cannot occur the other -// way). -// -// Example: +// As with most casts, use with caution. A `bit_cast()` might be needed when you +// need to treat a value as the value of some other type, for example, to access +// the individual bits of an object which are not normally accessible through +// the object's type, such as for working with the binary representation of a +// floating point value: // // float f = 3.14159265358979; -// int i = bit_cast<int32_t>(f); +// int i = bit_cast<int>(f); // // i = 0x40490fdb // -// Casting non-pointer types to pointer types and then dereferencing them -// traditionally produces undefined behavior. +// Reinterpreting and accessing a value directly as a different type (as shown +// below) usually results in undefined behavior. // // Example: // // // WRONG -// float f = 3.14159265358979; // WRONG -// int i = * reinterpret_cast<int*>(&f); // WRONG +// float f = 3.14159265358979; +// int i = reinterpret_cast<int&>(f); // Wrong +// int j = *reinterpret_cast<int*>(&f); // Equally wrong +// int k = *bit_cast<int*>(&f); // Equally wrong // -// The address-casting method produces undefined behavior according to the ISO -// C++ specification section [basic.lval]. Roughly, this section says: if an -// object in memory has one type, and a program accesses it with a different -// type, the result is undefined behavior for most values of "different type". +// Reinterpret-casting results in undefined behavior according to the ISO C++ +// specification, section [basic.lval]. Roughly, this section says: if an object +// in memory has one type, and a program accesses it with a different type, the +// result is undefined behavior for most "different type". +// +// Using bit_cast on a pointer and then dereferencing it is no better than using +// reinterpret_cast. You should only use bit_cast on the value itself. // // Such casting results in type punning: holding an object in memory of one type // and reading its bits back using a different type. A `bit_cast()` avoids this -// issue by implementing its casts using `memcpy()`, which avoids introducing -// this undefined behavior. -// -// NOTE: The requirements here are more strict than the bit_cast of standard -// proposal p0476 due to the need for workarounds and lack of intrinsics. -// Specifically, this implementation also requires `Dest` to be -// default-constructible. -template < - typename Dest, typename Source, - typename std::enable_if<internal_casts::is_bitcastable<Dest, Source>::value, - int>::type = 0> +// issue by copying the object representation to a new value, which avoids +// introducing this undefined behavior (since the original value is never +// accessed in the wrong way). +// +// The requirements of `absl::bit_cast` are more strict than that of +// `std::bit_cast` unless compiler support is available. Specifically, without +// compiler support, this implementation also requires `Dest` to be +// default-constructible. In C++20, `absl::bit_cast` is replaced by +// `std::bit_cast`. +#if defined(__cpp_lib_bit_cast) && __cpp_lib_bit_cast >= 201806L + +using std::bit_cast; + +#else // defined(__cpp_lib_bit_cast) && __cpp_lib_bit_cast >= 201806L + +template <typename Dest, typename Source, + typename std::enable_if< + sizeof(Dest) == sizeof(Source) && + type_traits_internal::is_trivially_copyable<Source>::value && + type_traits_internal::is_trivially_copyable<Dest>::value +#if !ABSL_HAVE_BUILTIN(__builtin_bit_cast) + && std::is_default_constructible<Dest>::value +#endif // !ABSL_HAVE_BUILTIN(__builtin_bit_cast) + , + int>::type = 0> +#if ABSL_HAVE_BUILTIN(__builtin_bit_cast) +inline constexpr Dest bit_cast(const Source& source) { + return __builtin_bit_cast(Dest, source); +} +#else // ABSL_HAVE_BUILTIN(__builtin_bit_cast) inline Dest bit_cast(const Source& source) { Dest dest; memcpy(static_cast<void*>(std::addressof(dest)), static_cast<const void*>(std::addressof(source)), sizeof(dest)); return dest; } +#endif // ABSL_HAVE_BUILTIN(__builtin_bit_cast) -// NOTE: This overload is only picked if the requirements of bit_cast are -// not met. It is therefore UB, but is provided temporarily as previous -// versions of this function template were unchecked. Do not use this in -// new code. -template < - typename Dest, typename Source, - typename std::enable_if< - !internal_casts::is_bitcastable<Dest, Source>::value, - int>::type = 0> -ABSL_DEPRECATED( - "absl::bit_cast type requirements were violated. Update the types " - "being used such that they are the same size and are both " - "TriviallyCopyable.") -inline Dest bit_cast(const Source& source) { - static_assert(sizeof(Dest) == sizeof(Source), - "Source and destination types should have equal sizes."); - - Dest dest; - memcpy(&dest, &source, sizeof(dest)); - return dest; -} +#endif // defined(__cpp_lib_bit_cast) && __cpp_lib_bit_cast >= 201806L ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/base/config.h b/absl/base/config.h index 95449969..705ecea0 100644 --- a/absl/base/config.h +++ b/absl/base/config.h @@ -56,6 +56,25 @@ #include <cstddef> #endif // __cplusplus +// ABSL_INTERNAL_CPLUSPLUS_LANG +// +// MSVC does not set the value of __cplusplus correctly, but instead uses +// _MSVC_LANG as a stand-in. +// https://docs.microsoft.com/en-us/cpp/preprocessor/predefined-macros +// +// However, there are reports that MSVC even sets _MSVC_LANG incorrectly at +// times, for example: +// https://github.com/microsoft/vscode-cpptools/issues/1770 +// https://reviews.llvm.org/D70996 +// +// For this reason, this symbol is considered INTERNAL and code outside of +// Abseil must not use it. +#if defined(_MSVC_LANG) +#define ABSL_INTERNAL_CPLUSPLUS_LANG _MSVC_LANG +#elif defined(__cplusplus) +#define ABSL_INTERNAL_CPLUSPLUS_LANG __cplusplus +#endif + #if defined(__APPLE__) // Included for TARGET_OS_IPHONE, __IPHONE_OS_VERSION_MIN_REQUIRED, // __IPHONE_8_0. @@ -66,6 +85,35 @@ #include "absl/base/options.h" #include "absl/base/policy_checks.h" +// Abseil long-term support (LTS) releases will define +// `ABSL_LTS_RELEASE_VERSION` to the integer representing the date string of the +// LTS release version, and will define `ABSL_LTS_RELEASE_PATCH_LEVEL` to the +// integer representing the patch-level for that release. +// +// For example, for LTS release version "20300401.2", this would give us +// ABSL_LTS_RELEASE_VERSION == 20300401 && ABSL_LTS_RELEASE_PATCH_LEVEL == 2 +// +// These symbols will not be defined in non-LTS code. +// +// Abseil recommends that clients live-at-head. Therefore, if you are using +// these symbols to assert a minimum version requirement, we recommend you do it +// as +// +// #if defined(ABSL_LTS_RELEASE_VERSION) && ABSL_LTS_RELEASE_VERSION < 20300401 +// #error Project foo requires Abseil LTS version >= 20300401 +// #endif +// +// The `defined(ABSL_LTS_RELEASE_VERSION)` part of the check excludes +// live-at-head clients from the minimum version assertion. +// +// See https://abseil.io/about/releases for more information on Abseil release +// management. +// +// LTS releases can be obtained from +// https://github.com/abseil/abseil-cpp/releases. +#define ABSL_LTS_RELEASE_VERSION 20220623 +#define ABSL_LTS_RELEASE_PATCH_LEVEL 1 + // Helper macro to convert a CPP variable to a string literal. #define ABSL_INTERNAL_DO_TOKEN_STR(x) #x #define ABSL_INTERNAL_TOKEN_STR(x) ABSL_INTERNAL_DO_TOKEN_STR(x) @@ -154,24 +202,35 @@ static_assert(ABSL_INTERNAL_INLINE_NAMESPACE_STR[0] != 'h' || #define ABSL_HAVE_BUILTIN(x) 0 #endif -#if defined(__is_identifier) -#define ABSL_INTERNAL_HAS_KEYWORD(x) !(__is_identifier(x)) -#else -#define ABSL_INTERNAL_HAS_KEYWORD(x) 0 -#endif - #ifdef __has_feature #define ABSL_HAVE_FEATURE(f) __has_feature(f) #else #define ABSL_HAVE_FEATURE(f) 0 #endif +// Portable check for GCC minimum version: +// https://gcc.gnu.org/onlinedocs/cpp/Common-Predefined-Macros.html +#if defined(__GNUC__) && defined(__GNUC_MINOR__) +#define ABSL_INTERNAL_HAVE_MIN_GNUC_VERSION(x, y) \ + (__GNUC__ > (x) || __GNUC__ == (x) && __GNUC_MINOR__ >= (y)) +#else +#define ABSL_INTERNAL_HAVE_MIN_GNUC_VERSION(x, y) 0 +#endif + +#if defined(__clang__) && defined(__clang_major__) && defined(__clang_minor__) +#define ABSL_INTERNAL_HAVE_MIN_CLANG_VERSION(x, y) \ + (__clang_major__ > (x) || __clang_major__ == (x) && __clang_minor__ >= (y)) +#else +#define ABSL_INTERNAL_HAVE_MIN_CLANG_VERSION(x, y) 0 +#endif + // ABSL_HAVE_TLS is defined to 1 when __thread should be supported. -// We assume __thread is supported on Linux when compiled with Clang or compiled -// against libstdc++ with _GLIBCXX_HAVE_TLS defined. +// We assume __thread is supported on Linux or Asylo when compiled with Clang or +// compiled against libstdc++ with _GLIBCXX_HAVE_TLS defined. #ifdef ABSL_HAVE_TLS #error ABSL_HAVE_TLS cannot be directly set -#elif defined(__linux__) && (defined(__clang__) || defined(_GLIBCXX_HAVE_TLS)) +#elif (defined(__linux__) || defined(__ASYLO__)) && \ + (defined(__clang__) || defined(_GLIBCXX_HAVE_TLS)) #define ABSL_HAVE_TLS 1 #endif @@ -183,10 +242,9 @@ static_assert(ABSL_INTERNAL_INLINE_NAMESPACE_STR[0] != 'h' || // gcc >= 4.8.1 using libstdc++, and Visual Studio. #ifdef ABSL_HAVE_STD_IS_TRIVIALLY_DESTRUCTIBLE #error ABSL_HAVE_STD_IS_TRIVIALLY_DESTRUCTIBLE cannot be directly set -#elif defined(_LIBCPP_VERSION) || \ - (!defined(__clang__) && defined(__GNUC__) && defined(__GLIBCXX__) && \ - (__GNUC__ > 4 || (__GNUC__ == 4 && __GNUC_MINOR__ >= 8))) || \ - defined(_MSC_VER) +#elif defined(_LIBCPP_VERSION) || defined(_MSC_VER) || \ + (!defined(__clang__) && defined(__GLIBCXX__) && \ + ABSL_INTERNAL_HAVE_MIN_GNUC_VERSION(4, 8)) #define ABSL_HAVE_STD_IS_TRIVIALLY_DESTRUCTIBLE 1 #endif @@ -199,34 +257,22 @@ static_assert(ABSL_INTERNAL_INLINE_NAMESPACE_STR[0] != 'h' || // // Checks whether `std::is_trivially_copy_assignable<T>` is supported. -// Notes: Clang with libc++ supports these features, as does gcc >= 5.1 with -// either libc++ or libstdc++, and Visual Studio (but not NVCC). +// Notes: Clang with libc++ supports these features, as does gcc >= 7.4 with +// libstdc++, or gcc >= 8.2 with libc++, and Visual Studio (but not NVCC). #if defined(ABSL_HAVE_STD_IS_TRIVIALLY_CONSTRUCTIBLE) #error ABSL_HAVE_STD_IS_TRIVIALLY_CONSTRUCTIBLE cannot be directly set #elif defined(ABSL_HAVE_STD_IS_TRIVIALLY_ASSIGNABLE) #error ABSL_HAVE_STD_IS_TRIVIALLY_ASSIGNABLE cannot directly set -#elif (defined(__clang__) && defined(_LIBCPP_VERSION)) || \ - (!defined(__clang__) && defined(__GNUC__) && \ - (__GNUC__ > 7 || (__GNUC__ == 7 && __GNUC_MINOR__ >= 4)) && \ - (defined(_LIBCPP_VERSION) || defined(__GLIBCXX__))) || \ +#elif (defined(__clang__) && defined(_LIBCPP_VERSION)) || \ + (!defined(__clang__) && \ + ((ABSL_INTERNAL_HAVE_MIN_GNUC_VERSION(7, 4) && defined(__GLIBCXX__)) || \ + (ABSL_INTERNAL_HAVE_MIN_GNUC_VERSION(8, 2) && \ + defined(_LIBCPP_VERSION)))) || \ (defined(_MSC_VER) && !defined(__NVCC__)) #define ABSL_HAVE_STD_IS_TRIVIALLY_CONSTRUCTIBLE 1 #define ABSL_HAVE_STD_IS_TRIVIALLY_ASSIGNABLE 1 #endif -// ABSL_HAVE_SOURCE_LOCATION_CURRENT -// -// Indicates whether `absl::SourceLocation::current()` will return useful -// information in some contexts. -#ifndef ABSL_HAVE_SOURCE_LOCATION_CURRENT -#if ABSL_INTERNAL_HAS_KEYWORD(__builtin_LINE) && \ - ABSL_INTERNAL_HAS_KEYWORD(__builtin_FILE) -#define ABSL_HAVE_SOURCE_LOCATION_CURRENT 1 -#elif defined(__GNUC__) && __GNUC__ >= 5 -#define ABSL_HAVE_SOURCE_LOCATION_CURRENT 1 -#endif -#endif - // ABSL_HAVE_THREAD_LOCAL // // Checks whether C++11's `thread_local` storage duration specifier is @@ -319,25 +365,21 @@ static_assert(ABSL_INTERNAL_INLINE_NAMESPACE_STR[0] != 'h' || // For further details, consult the compiler's documentation. #ifdef ABSL_HAVE_EXCEPTIONS #error ABSL_HAVE_EXCEPTIONS cannot be directly set. - -#elif defined(__clang__) - -#if __clang_major__ > 3 || (__clang_major__ == 3 && __clang_minor__ >= 6) +#elif ABSL_INTERNAL_HAVE_MIN_CLANG_VERSION(3, 6) // Clang >= 3.6 #if ABSL_HAVE_FEATURE(cxx_exceptions) #define ABSL_HAVE_EXCEPTIONS 1 #endif // ABSL_HAVE_FEATURE(cxx_exceptions) -#else +#elif defined(__clang__) // Clang < 3.6 // http://releases.llvm.org/3.6.0/tools/clang/docs/ReleaseNotes.html#the-exceptions-macro #if defined(__EXCEPTIONS) && ABSL_HAVE_FEATURE(cxx_exceptions) #define ABSL_HAVE_EXCEPTIONS 1 #endif // defined(__EXCEPTIONS) && ABSL_HAVE_FEATURE(cxx_exceptions) -#endif // __clang_major__ > 3 || (__clang_major__ == 3 && __clang_minor__ >= 6) - // Handle remaining special cases and default to exceptions being supported. -#elif !(defined(__GNUC__) && (__GNUC__ < 5) && !defined(__EXCEPTIONS)) && \ - !(defined(__GNUC__) && (__GNUC__ >= 5) && !defined(__cpp_exceptions)) && \ +#elif !(defined(__GNUC__) && (__GNUC__ < 5) && !defined(__EXCEPTIONS)) && \ + !(ABSL_INTERNAL_HAVE_MIN_GNUC_VERSION(5, 0) && \ + !defined(__cpp_exceptions)) && \ !(defined(_MSC_VER) && !defined(_CPPUNWIND)) #define ABSL_HAVE_EXCEPTIONS 1 #endif @@ -369,10 +411,12 @@ static_assert(ABSL_INTERNAL_INLINE_NAMESPACE_STR[0] != 'h' || // POSIX.1-2001. #ifdef ABSL_HAVE_MMAP #error ABSL_HAVE_MMAP cannot be directly set -#elif defined(__linux__) || defined(__APPLE__) || defined(__FreeBSD__) || \ - defined(__ros__) || defined(__native_client__) || defined(__asmjs__) || \ - defined(__wasm__) || defined(__Fuchsia__) || defined(__sun) || \ - defined(__ASYLO__) || defined(__myriad2__) +#elif defined(__linux__) || defined(__APPLE__) || defined(__FreeBSD__) || \ + defined(_AIX) || defined(__ros__) || defined(__native_client__) || \ + defined(__asmjs__) || defined(__wasm__) || defined(__Fuchsia__) || \ + defined(__sun) || defined(__ASYLO__) || defined(__myriad2__) || \ + defined(__HAIKU__) || defined(__OpenBSD__) || defined(__NetBSD__) || \ + defined(__QNX__) #define ABSL_HAVE_MMAP 1 #endif @@ -383,7 +427,8 @@ static_assert(ABSL_INTERNAL_INLINE_NAMESPACE_STR[0] != 'h' || #ifdef ABSL_HAVE_PTHREAD_GETSCHEDPARAM #error ABSL_HAVE_PTHREAD_GETSCHEDPARAM cannot be directly set #elif defined(__linux__) || defined(__APPLE__) || defined(__FreeBSD__) || \ - defined(__ros__) + defined(_AIX) || defined(__ros__) || defined(__OpenBSD__) || \ + defined(__NetBSD__) #define ABSL_HAVE_PTHREAD_GETSCHEDPARAM 1 #endif @@ -478,22 +523,41 @@ static_assert(ABSL_INTERNAL_INLINE_NAMESPACE_STR[0] != 'h' || #error "absl endian detection needs to be set up for your compiler" #endif -// macOS 10.13 and iOS 10.11 don't let you use <any>, <optional>, or <variant> -// even though the headers exist and are publicly noted to work. See -// https://github.com/abseil/abseil-cpp/issues/207 and +// macOS < 10.13 and iOS < 11 don't let you use <any>, <optional>, or <variant> +// even though the headers exist and are publicly noted to work, because the +// libc++ shared library shipped on the system doesn't have the requisite +// exported symbols. See https://github.com/abseil/abseil-cpp/issues/207 and // https://developer.apple.com/documentation/xcode_release_notes/xcode_10_release_notes +// // libc++ spells out the availability requirements in the file // llvm-project/libcxx/include/__config via the #define // _LIBCPP_AVAILABILITY_BAD_OPTIONAL_ACCESS. -#if defined(__APPLE__) && defined(_LIBCPP_VERSION) && \ - ((defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && \ - __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 101400) || \ - (defined(__ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__) && \ - __ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__ < 120000) || \ - (defined(__ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__) && \ - __ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__ < 50000) || \ - (defined(__ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__) && \ - __ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__ < 120000)) +// +// Unfortunately, Apple initially mis-stated the requirements as macOS < 10.14 +// and iOS < 12 in the libc++ headers. This was corrected by +// https://github.com/llvm/llvm-project/commit/7fb40e1569dd66292b647f4501b85517e9247953 +// which subsequently made it into the XCode 12.5 release. We need to match the +// old (incorrect) conditions when built with old XCode, but can use the +// corrected earlier versions with new XCode. +#if defined(__APPLE__) && defined(_LIBCPP_VERSION) && \ + ((_LIBCPP_VERSION >= 11000 && /* XCode 12.5 or later: */ \ + ((defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && \ + __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 101300) || \ + (defined(__ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__) && \ + __ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__ < 110000) || \ + (defined(__ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__) && \ + __ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__ < 40000) || \ + (defined(__ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__) && \ + __ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__ < 110000))) || \ + (_LIBCPP_VERSION < 11000 && /* Pre-XCode 12.5: */ \ + ((defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__) && \ + __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 101400) || \ + (defined(__ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__) && \ + __ENVIRONMENT_IPHONE_OS_VERSION_MIN_REQUIRED__ < 120000) || \ + (defined(__ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__) && \ + __ENVIRONMENT_WATCH_OS_VERSION_MIN_REQUIRED__ < 50000) || \ + (defined(__ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__) && \ + __ENVIRONMENT_TV_OS_VERSION_MIN_REQUIRED__ < 120000)))) #define ABSL_INTERNAL_APPLE_CXX17_TYPES_UNAVAILABLE 1 #else #define ABSL_INTERNAL_APPLE_CXX17_TYPES_UNAVAILABLE 0 @@ -663,8 +727,6 @@ static_assert(ABSL_INTERNAL_INLINE_NAMESPACE_STR[0] != 'h' || #endif #endif -#undef ABSL_INTERNAL_HAS_KEYWORD - // ABSL_DLL // // When building Abseil as a DLL, this macro expands to `__declspec(dllexport)` @@ -690,12 +752,6 @@ static_assert(ABSL_INTERNAL_INLINE_NAMESPACE_STR[0] != 'h' || // a compiler instrumentation module and a run-time library. #ifdef ABSL_HAVE_MEMORY_SANITIZER #error "ABSL_HAVE_MEMORY_SANITIZER cannot be directly set." -#elif defined(MEMORY_SANITIZER) -// The MEMORY_SANITIZER macro is deprecated but we will continue to honor it -// for now. -#define ABSL_HAVE_MEMORY_SANITIZER 1 -#elif defined(__SANITIZE_MEMORY__) -#define ABSL_HAVE_MEMORY_SANITIZER 1 #elif !defined(__native_client__) && ABSL_HAVE_FEATURE(memory_sanitizer) #define ABSL_HAVE_MEMORY_SANITIZER 1 #endif @@ -705,10 +761,6 @@ static_assert(ABSL_INTERNAL_INLINE_NAMESPACE_STR[0] != 'h' || // ThreadSanitizer (TSan) is a fast data race detector. #ifdef ABSL_HAVE_THREAD_SANITIZER #error "ABSL_HAVE_THREAD_SANITIZER cannot be directly set." -#elif defined(THREAD_SANITIZER) -// The THREAD_SANITIZER macro is deprecated but we will continue to honor it -// for now. -#define ABSL_HAVE_THREAD_SANITIZER 1 #elif defined(__SANITIZE_THREAD__) #define ABSL_HAVE_THREAD_SANITIZER 1 #elif ABSL_HAVE_FEATURE(thread_sanitizer) @@ -720,16 +772,51 @@ static_assert(ABSL_INTERNAL_INLINE_NAMESPACE_STR[0] != 'h' || // AddressSanitizer (ASan) is a fast memory error detector. #ifdef ABSL_HAVE_ADDRESS_SANITIZER #error "ABSL_HAVE_ADDRESS_SANITIZER cannot be directly set." -#elif defined(ADDRESS_SANITIZER) -// The ADDRESS_SANITIZER macro is deprecated but we will continue to honor it -// for now. -#define ABSL_HAVE_ADDRESS_SANITIZER 1 #elif defined(__SANITIZE_ADDRESS__) #define ABSL_HAVE_ADDRESS_SANITIZER 1 #elif ABSL_HAVE_FEATURE(address_sanitizer) #define ABSL_HAVE_ADDRESS_SANITIZER 1 #endif +// ABSL_HAVE_HWADDRESS_SANITIZER +// +// Hardware-Assisted AddressSanitizer (or HWASAN) is even faster than asan +// memory error detector which can use CPU features like ARM TBI, Intel LAM or +// AMD UAI. +#ifdef ABSL_HAVE_HWADDRESS_SANITIZER +#error "ABSL_HAVE_HWADDRESS_SANITIZER cannot be directly set." +#elif defined(__SANITIZE_HWADDRESS__) +#define ABSL_HAVE_HWADDRESS_SANITIZER 1 +#elif ABSL_HAVE_FEATURE(hwaddress_sanitizer) +#define ABSL_HAVE_HWADDRESS_SANITIZER 1 +#endif + +// ABSL_HAVE_LEAK_SANITIZER +// +// LeakSanitizer (or lsan) is a detector of memory leaks. +// https://clang.llvm.org/docs/LeakSanitizer.html +// https://github.com/google/sanitizers/wiki/AddressSanitizerLeakSanitizer +// +// The macro ABSL_HAVE_LEAK_SANITIZER can be used to detect at compile-time +// whether the LeakSanitizer is potentially available. However, just because the +// LeakSanitizer is available does not mean it is active. Use the +// always-available run-time interface in //absl/debugging/leak_check.h for +// interacting with LeakSanitizer. +#ifdef ABSL_HAVE_LEAK_SANITIZER +#error "ABSL_HAVE_LEAK_SANITIZER cannot be directly set." +#elif defined(LEAK_SANITIZER) +// GCC provides no method for detecting the presense of the standalone +// LeakSanitizer (-fsanitize=leak), so GCC users of -fsanitize=leak should also +// use -DLEAK_SANITIZER. +#define ABSL_HAVE_LEAK_SANITIZER 1 +// Clang standalone LeakSanitizer (-fsanitize=leak) +#elif ABSL_HAVE_FEATURE(leak_sanitizer) +#define ABSL_HAVE_LEAK_SANITIZER 1 +#elif defined(ABSL_HAVE_ADDRESS_SANITIZER) +// GCC or Clang using the LeakSanitizer integrated into AddressSanitizer. +#define ABSL_HAVE_LEAK_SANITIZER 1 +#endif + // ABSL_HAVE_CLASS_TEMPLATE_ARGUMENT_DEDUCTION // // Class template argument deduction is a language feature added in C++17. @@ -739,4 +826,88 @@ static_assert(ABSL_INTERNAL_INLINE_NAMESPACE_STR[0] != 'h' || #define ABSL_HAVE_CLASS_TEMPLATE_ARGUMENT_DEDUCTION 1 #endif +// ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL +// +// Prior to C++17, static constexpr variables defined in classes required a +// separate definition outside of the class body, for example: +// +// class Foo { +// static constexpr int kBar = 0; +// }; +// constexpr int Foo::kBar; +// +// In C++17, these variables defined in classes are considered inline variables, +// and the extra declaration is redundant. Since some compilers warn on the +// extra declarations, ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL can be used +// conditionally ignore them: +// +// #ifdef ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL +// constexpr int Foo::kBar; +// #endif +#if defined(ABSL_INTERNAL_CPLUSPLUS_LANG) && \ + ABSL_INTERNAL_CPLUSPLUS_LANG < 201703L +#define ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL 1 +#endif + +// `ABSL_INTERNAL_HAS_RTTI` determines whether abseil is being compiled with +// RTTI support. +#ifdef ABSL_INTERNAL_HAS_RTTI +#error ABSL_INTERNAL_HAS_RTTI cannot be directly set +#elif !defined(__GNUC__) || defined(__GXX_RTTI) +#define ABSL_INTERNAL_HAS_RTTI 1 +#endif // !defined(__GNUC__) || defined(__GXX_RTTI) + +// ABSL_INTERNAL_HAVE_SSE is used for compile-time detection of SSE support. +// See https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html for an overview of +// which architectures support the various x86 instruction sets. +#ifdef ABSL_INTERNAL_HAVE_SSE +#error ABSL_INTERNAL_HAVE_SSE cannot be directly set +#elif defined(__SSE__) +#define ABSL_INTERNAL_HAVE_SSE 1 +#elif defined(_M_X64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 1) +// MSVC only defines _M_IX86_FP for x86 32-bit code, and _M_IX86_FP >= 1 +// indicates that at least SSE was targeted with the /arch:SSE option. +// All x86-64 processors support SSE, so support can be assumed. +// https://docs.microsoft.com/en-us/cpp/preprocessor/predefined-macros +#define ABSL_INTERNAL_HAVE_SSE 1 +#endif + +// ABSL_INTERNAL_HAVE_SSE2 is used for compile-time detection of SSE2 support. +// See https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html for an overview of +// which architectures support the various x86 instruction sets. +#ifdef ABSL_INTERNAL_HAVE_SSE2 +#error ABSL_INTERNAL_HAVE_SSE2 cannot be directly set +#elif defined(__SSE2__) +#define ABSL_INTERNAL_HAVE_SSE2 1 +#elif defined(_M_X64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2) +// MSVC only defines _M_IX86_FP for x86 32-bit code, and _M_IX86_FP >= 2 +// indicates that at least SSE2 was targeted with the /arch:SSE2 option. +// All x86-64 processors support SSE2, so support can be assumed. +// https://docs.microsoft.com/en-us/cpp/preprocessor/predefined-macros +#define ABSL_INTERNAL_HAVE_SSE2 1 +#endif + +// ABSL_INTERNAL_HAVE_SSSE3 is used for compile-time detection of SSSE3 support. +// See https://gcc.gnu.org/onlinedocs/gcc/x86-Options.html for an overview of +// which architectures support the various x86 instruction sets. +// +// MSVC does not have a mode that targets SSSE3 at compile-time. To use SSSE3 +// with MSVC requires either assuming that the code will only every run on CPUs +// that support SSSE3, otherwise __cpuid() can be used to detect support at +// runtime and fallback to a non-SSSE3 implementation when SSSE3 is unsupported +// by the CPU. +#ifdef ABSL_INTERNAL_HAVE_SSSE3 +#error ABSL_INTERNAL_HAVE_SSSE3 cannot be directly set +#elif defined(__SSSE3__) +#define ABSL_INTERNAL_HAVE_SSSE3 1 +#endif + +// ABSL_INTERNAL_HAVE_ARM_NEON is used for compile-time detection of NEON (ARM +// SIMD). +#ifdef ABSL_INTERNAL_HAVE_ARM_NEON +#error ABSL_INTERNAL_HAVE_ARM_NEON cannot be directly set +#elif defined(__ARM_NEON) +#define ABSL_INTERNAL_HAVE_ARM_NEON 1 +#endif + #endif // ABSL_BASE_CONFIG_H_ diff --git a/absl/base/dynamic_annotations.h b/absl/base/dynamic_annotations.h index 880cbf6e..3ea7c156 100644 --- a/absl/base/dynamic_annotations.h +++ b/absl/base/dynamic_annotations.h @@ -433,31 +433,6 @@ ABSL_NAMESPACE_END #endif -#ifdef __cplusplus -#ifdef ABSL_HAVE_THREAD_SANITIZER -ABSL_INTERNAL_BEGIN_EXTERN_C -int RunningOnValgrind(); -double ValgrindSlowdown(); -ABSL_INTERNAL_END_EXTERN_C -#else -namespace absl { -ABSL_NAMESPACE_BEGIN -namespace base_internal { -ABSL_DEPRECATED( - "Don't use this interface. It is misleading and is being deleted.") -ABSL_ATTRIBUTE_ALWAYS_INLINE inline int RunningOnValgrind() { return 0; } -ABSL_DEPRECATED( - "Don't use this interface. It is misleading and is being deleted.") -ABSL_ATTRIBUTE_ALWAYS_INLINE inline double ValgrindSlowdown() { return 1.0; } -} // namespace base_internal -ABSL_NAMESPACE_END -} // namespace absl - -using absl::base_internal::RunningOnValgrind; -using absl::base_internal::ValgrindSlowdown; -#endif -#endif - // ------------------------------------------------------------------------- // Address sanitizer annotations @@ -471,7 +446,7 @@ using absl::base_internal::ValgrindSlowdown; __sanitizer_annotate_contiguous_container(beg, end, old_mid, new_mid) #define ABSL_ADDRESS_SANITIZER_REDZONE(name) \ struct { \ - char x[8] __attribute__((aligned(8))); \ + alignas(8) char x[8]; \ } name #else diff --git a/absl/base/exception_safety_testing_test.cc b/absl/base/exception_safety_testing_test.cc index a59be29e..a87fd6a9 100644 --- a/absl/base/exception_safety_testing_test.cc +++ b/absl/base/exception_safety_testing_test.cc @@ -701,7 +701,10 @@ struct BasicGuaranteeWithExtraContracts : public NonNegative { static constexpr int kExceptionSentinel = 9999; }; + +#ifdef ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL constexpr int BasicGuaranteeWithExtraContracts::kExceptionSentinel; +#endif TEST(ExceptionCheckTest, BasicGuaranteeWithExtraContracts) { auto tester_with_val = diff --git a/absl/base/internal/cycleclock.cc b/absl/base/internal/cycleclock.cc index 0e65005b..902e3f5e 100644 --- a/absl/base/internal/cycleclock.cc +++ b/absl/base/internal/cycleclock.cc @@ -25,6 +25,8 @@ #include <atomic> #include <chrono> // NOLINT(build/c++11) +#include "absl/base/attributes.h" +#include "absl/base/config.h" #include "absl/base/internal/unscaledcycleclock.h" namespace absl { @@ -33,44 +35,20 @@ namespace base_internal { #if ABSL_USE_UNSCALED_CYCLECLOCK -namespace { - -#ifdef NDEBUG -#ifdef ABSL_INTERNAL_UNSCALED_CYCLECLOCK_FREQUENCY_IS_CPU_FREQUENCY -// Not debug mode and the UnscaledCycleClock frequency is the CPU -// frequency. Scale the CycleClock to prevent overflow if someone -// tries to represent the time as cycles since the Unix epoch. -static constexpr int32_t kShift = 1; -#else -// Not debug mode and the UnscaledCycleClock isn't operating at the -// raw CPU frequency. There is no need to do any scaling, so don't -// needlessly sacrifice precision. -static constexpr int32_t kShift = 0; -#endif -#else -// In debug mode use a different shift to discourage depending on a -// particular shift value. -static constexpr int32_t kShift = 2; +#ifdef ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL +constexpr int32_t CycleClock::kShift; +constexpr double CycleClock::kFrequencyScale; #endif -static constexpr double kFrequencyScale = 1.0 / (1 << kShift); -static std::atomic<CycleClockSourceFunc> cycle_clock_source; +ABSL_CONST_INIT std::atomic<CycleClockSourceFunc> + CycleClock::cycle_clock_source_{nullptr}; -CycleClockSourceFunc LoadCycleClockSource() { - // Optimize for the common case (no callback) by first doing a relaxed load; - // this is significantly faster on non-x86 platforms. - if (cycle_clock_source.load(std::memory_order_relaxed) == nullptr) { - return nullptr; - } - // This corresponds to the store(std::memory_order_release) in - // CycleClockSource::Register, and makes sure that any updates made prior to - // registering the callback are visible to this thread before the callback is - // invoked. - return cycle_clock_source.load(std::memory_order_acquire); +void CycleClockSource::Register(CycleClockSourceFunc source) { + // Corresponds to the load(std::memory_order_acquire) in LoadCycleClockSource. + CycleClock::cycle_clock_source_.store(source, std::memory_order_release); } -} // namespace - +#ifdef _WIN32 int64_t CycleClock::Now() { auto fn = LoadCycleClockSource(); if (fn == nullptr) { @@ -78,15 +56,7 @@ int64_t CycleClock::Now() { } return fn() >> kShift; } - -double CycleClock::Frequency() { - return kFrequencyScale * base_internal::UnscaledCycleClock::Frequency(); -} - -void CycleClockSource::Register(CycleClockSourceFunc source) { - // Corresponds to the load(std::memory_order_acquire) in LoadCycleClockSource. - cycle_clock_source.store(source, std::memory_order_release); -} +#endif #else diff --git a/absl/base/internal/cycleclock.h b/absl/base/internal/cycleclock.h index a18b5844..9704e388 100644 --- a/absl/base/internal/cycleclock.h +++ b/absl/base/internal/cycleclock.h @@ -42,14 +42,19 @@ #ifndef ABSL_BASE_INTERNAL_CYCLECLOCK_H_ #define ABSL_BASE_INTERNAL_CYCLECLOCK_H_ +#include <atomic> #include <cstdint> +#include "absl/base/attributes.h" #include "absl/base/config.h" +#include "absl/base/internal/unscaledcycleclock.h" namespace absl { ABSL_NAMESPACE_BEGIN namespace base_internal { +using CycleClockSourceFunc = int64_t (*)(); + // ----------------------------------------------------------------------------- // CycleClock // ----------------------------------------------------------------------------- @@ -68,12 +73,37 @@ class CycleClock { static double Frequency(); private: +#if ABSL_USE_UNSCALED_CYCLECLOCK + static CycleClockSourceFunc LoadCycleClockSource(); + +#ifdef NDEBUG +#ifdef ABSL_INTERNAL_UNSCALED_CYCLECLOCK_FREQUENCY_IS_CPU_FREQUENCY + // Not debug mode and the UnscaledCycleClock frequency is the CPU + // frequency. Scale the CycleClock to prevent overflow if someone + // tries to represent the time as cycles since the Unix epoch. + static constexpr int32_t kShift = 1; +#else + // Not debug mode and the UnscaledCycleClock isn't operating at the + // raw CPU frequency. There is no need to do any scaling, so don't + // needlessly sacrifice precision. + static constexpr int32_t kShift = 0; +#endif +#else // NDEBUG + // In debug mode use a different shift to discourage depending on a + // particular shift value. + static constexpr int32_t kShift = 2; +#endif // NDEBUG + + static constexpr double kFrequencyScale = 1.0 / (1 << kShift); + ABSL_CONST_INIT static std::atomic<CycleClockSourceFunc> cycle_clock_source_; +#endif // ABSL_USE_UNSCALED_CYCLECLOC + CycleClock() = delete; // no instances CycleClock(const CycleClock&) = delete; CycleClock& operator=(const CycleClock&) = delete; -}; -using CycleClockSourceFunc = int64_t (*)(); + friend class CycleClockSource; +}; class CycleClockSource { private: @@ -87,6 +117,41 @@ class CycleClockSource { static void Register(CycleClockSourceFunc source); }; +#if ABSL_USE_UNSCALED_CYCLECLOCK + +inline CycleClockSourceFunc CycleClock::LoadCycleClockSource() { +#if !defined(__x86_64__) + // Optimize for the common case (no callback) by first doing a relaxed load; + // this is significantly faster on non-x86 platforms. + if (cycle_clock_source_.load(std::memory_order_relaxed) == nullptr) { + return nullptr; + } +#endif // !defined(__x86_64__) + + // This corresponds to the store(std::memory_order_release) in + // CycleClockSource::Register, and makes sure that any updates made prior to + // registering the callback are visible to this thread before the callback + // is invoked. + return cycle_clock_source_.load(std::memory_order_acquire); +} + +// Accessing globals in inlined code in Window DLLs is problematic. +#ifndef _WIN32 +inline int64_t CycleClock::Now() { + auto fn = LoadCycleClockSource(); + if (fn == nullptr) { + return base_internal::UnscaledCycleClock::Now() >> kShift; + } + return fn() >> kShift; +} +#endif + +inline double CycleClock::Frequency() { + return kFrequencyScale * base_internal::UnscaledCycleClock::Frequency(); +} + +#endif // ABSL_USE_UNSCALED_CYCLECLOCK + } // namespace base_internal ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/base/internal/direct_mmap.h b/absl/base/internal/direct_mmap.h index 274054cd..e492bb00 100644 --- a/absl/base/internal/direct_mmap.h +++ b/absl/base/internal/direct_mmap.h @@ -20,7 +20,7 @@ #include "absl/base/config.h" -#if ABSL_HAVE_MMAP +#ifdef ABSL_HAVE_MMAP #include <sys/mman.h> @@ -41,13 +41,13 @@ #ifdef __mips__ // Include definitions of the ABI currently in use. -#ifdef __BIONIC__ +#if defined(__BIONIC__) || !defined(__GLIBC__) // Android doesn't have sgidefs.h, but does have asm/sgidefs.h, which has the // definitions we need. #include <asm/sgidefs.h> #else #include <sgidefs.h> -#endif // __BIONIC__ +#endif // __BIONIC__ || !__GLIBC__ #endif // __mips__ // SYS_mmap and SYS_munmap are not defined in Android. diff --git a/absl/base/internal/endian.h b/absl/base/internal/endian.h index dad0e9ae..50747d75 100644 --- a/absl/base/internal/endian.h +++ b/absl/base/internal/endian.h @@ -16,16 +16,9 @@ #ifndef ABSL_BASE_INTERNAL_ENDIAN_H_ #define ABSL_BASE_INTERNAL_ENDIAN_H_ -// The following guarantees declaration of the byte swap functions -#ifdef _MSC_VER -#include <stdlib.h> // NOLINT(build/include) -#elif defined(__FreeBSD__) -#include <sys/endian.h> -#elif defined(__GLIBC__) -#include <byteswap.h> // IWYU pragma: export -#endif - #include <cstdint> +#include <cstdlib> + #include "absl/base/casts.h" #include "absl/base/config.h" #include "absl/base/internal/unaligned_access.h" @@ -34,47 +27,11 @@ namespace absl { ABSL_NAMESPACE_BEGIN -// Use compiler byte-swapping intrinsics if they are available. 32-bit -// and 64-bit versions are available in Clang and GCC as of GCC 4.3.0. -// The 16-bit version is available in Clang and GCC only as of GCC 4.8.0. -// For simplicity, we enable them all only for GCC 4.8.0 or later. -#if defined(__clang__) || \ - (defined(__GNUC__) && \ - ((__GNUC__ == 4 && __GNUC_MINOR__ >= 8) || __GNUC__ >= 5)) inline uint64_t gbswap_64(uint64_t host_int) { +#if ABSL_HAVE_BUILTIN(__builtin_bswap64) || defined(__GNUC__) return __builtin_bswap64(host_int); -} -inline uint32_t gbswap_32(uint32_t host_int) { - return __builtin_bswap32(host_int); -} -inline uint16_t gbswap_16(uint16_t host_int) { - return __builtin_bswap16(host_int); -} - #elif defined(_MSC_VER) -inline uint64_t gbswap_64(uint64_t host_int) { return _byteswap_uint64(host_int); -} -inline uint32_t gbswap_32(uint32_t host_int) { - return _byteswap_ulong(host_int); -} -inline uint16_t gbswap_16(uint16_t host_int) { - return _byteswap_ushort(host_int); -} - -#else -inline uint64_t gbswap_64(uint64_t host_int) { -#if defined(__GNUC__) && defined(__x86_64__) && !defined(__APPLE__) - // Adapted from /usr/include/byteswap.h. Not available on Mac. - if (__builtin_constant_p(host_int)) { - return __bswap_constant_64(host_int); - } else { - uint64_t result; - __asm__("bswap %0" : "=r"(result) : "0"(host_int)); - return result; - } -#elif defined(__GLIBC__) - return bswap_64(host_int); #else return (((host_int & uint64_t{0xFF}) << 56) | ((host_int & uint64_t{0xFF00}) << 40) | @@ -84,12 +41,14 @@ inline uint64_t gbswap_64(uint64_t host_int) { ((host_int & uint64_t{0xFF0000000000}) >> 24) | ((host_int & uint64_t{0xFF000000000000}) >> 40) | ((host_int & uint64_t{0xFF00000000000000}) >> 56)); -#endif // bswap_64 +#endif } inline uint32_t gbswap_32(uint32_t host_int) { -#if defined(__GLIBC__) - return bswap_32(host_int); +#if ABSL_HAVE_BUILTIN(__builtin_bswap32) || defined(__GNUC__) + return __builtin_bswap32(host_int); +#elif defined(_MSC_VER) + return _byteswap_ulong(host_int); #else return (((host_int & uint32_t{0xFF}) << 24) | ((host_int & uint32_t{0xFF00}) << 8) | @@ -99,33 +58,29 @@ inline uint32_t gbswap_32(uint32_t host_int) { } inline uint16_t gbswap_16(uint16_t host_int) { -#if defined(__GLIBC__) - return bswap_16(host_int); +#if ABSL_HAVE_BUILTIN(__builtin_bswap16) || defined(__GNUC__) + return __builtin_bswap16(host_int); +#elif defined(_MSC_VER) + return _byteswap_ushort(host_int); #else return (((host_int & uint16_t{0xFF}) << 8) | ((host_int & uint16_t{0xFF00}) >> 8)); #endif } -#endif // intrinsics available - #ifdef ABSL_IS_LITTLE_ENDIAN -// Definitions for ntohl etc. that don't require us to include -// netinet/in.h. We wrap gbswap_32 and gbswap_16 in functions rather -// than just #defining them because in debug mode, gcc doesn't -// correctly handle the (rather involved) definitions of bswap_32. -// gcc guarantees that inline functions are as fast as macros, so -// this isn't a performance hit. +// Portable definitions for htonl (host-to-network) and friends on little-endian +// architectures. inline uint16_t ghtons(uint16_t x) { return gbswap_16(x); } inline uint32_t ghtonl(uint32_t x) { return gbswap_32(x); } inline uint64_t ghtonll(uint64_t x) { return gbswap_64(x); } #elif defined ABSL_IS_BIG_ENDIAN -// These definitions are simpler on big-endian machines -// These are functions instead of macros to avoid self-assignment warnings -// on calls such as "i = ghtnol(i);". This also provides type checking. +// Portable definitions for htonl (host-to-network) etc on big-endian +// architectures. These definitions are simpler since the host byte order is the +// same as network byte order. inline uint16_t ghtons(uint16_t x) { return x; } inline uint32_t ghtonl(uint32_t x) { return x; } inline uint64_t ghtonll(uint64_t x) { return x; } diff --git a/absl/base/internal/exception_safety_testing.h b/absl/base/internal/exception_safety_testing.h index 6ba89d05..77a5aec6 100644 --- a/absl/base/internal/exception_safety_testing.h +++ b/absl/base/internal/exception_safety_testing.h @@ -536,7 +536,22 @@ class ThrowingValue : private exceptions_internal::TrackedObject { } // Memory management operators - // Args.. allows us to overload regular and placement new in one shot + static void* operator new(size_t s) noexcept( + IsSpecified(TypeSpec::kNoThrowNew)) { + if (!IsSpecified(TypeSpec::kNoThrowNew)) { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION, true); + } + return ::operator new(s); + } + + static void* operator new[](size_t s) noexcept( + IsSpecified(TypeSpec::kNoThrowNew)) { + if (!IsSpecified(TypeSpec::kNoThrowNew)) { + exceptions_internal::MaybeThrow(ABSL_PRETTY_FUNCTION, true); + } + return ::operator new[](s); + } + template <typename... Args> static void* operator new(size_t s, Args&&... args) noexcept( IsSpecified(TypeSpec::kNoThrowNew)) { @@ -557,12 +572,6 @@ class ThrowingValue : private exceptions_internal::TrackedObject { // Abseil doesn't support throwing overloaded operator delete. These are // provided so a throwing operator-new can clean up after itself. - // - // We provide both regular and templated operator delete because if only the - // templated version is provided as we did with operator new, the compiler has - // no way of knowing which overload of operator delete to call. See - // https://en.cppreference.com/w/cpp/memory/new/operator_delete and - // https://en.cppreference.com/w/cpp/language/delete for the gory details. void operator delete(void* p) noexcept { ::operator delete(p); } template <typename... Args> @@ -726,9 +735,8 @@ class ThrowingAllocator : private exceptions_internal::TrackedObject { ThrowingAllocator select_on_container_copy_construction() noexcept( IsSpecified(AllocSpec::kNoThrowAllocate)) { - auto& out = *this; ReadStateAndMaybeThrow(ABSL_PRETTY_FUNCTION); - return out; + return *this; } template <typename U> diff --git a/absl/base/internal/fast_type_id.h b/absl/base/internal/fast_type_id.h index 3db59e83..a547b3a8 100644 --- a/absl/base/internal/fast_type_id.h +++ b/absl/base/internal/fast_type_id.h @@ -28,8 +28,10 @@ struct FastTypeTag { constexpr static char dummy_var = 0; }; +#ifdef ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL template <typename Type> constexpr char FastTypeTag<Type>::dummy_var; +#endif // FastTypeId<Type>() evaluates at compile/link-time to a unique pointer for the // passed-in type. These are meant to be good match for keys into maps or diff --git a/absl/base/internal/invoke.h b/absl/base/internal/invoke.h index 5c71f328..643c2a42 100644 --- a/absl/base/internal/invoke.h +++ b/absl/base/internal/invoke.h @@ -14,6 +14,8 @@ // // absl::base_internal::invoke(f, args...) is an implementation of // INVOKE(f, args...) from section [func.require] of the C++ standard. +// When compiled as C++17 and later versions, it is implemented as an alias of +// std::invoke. // // [func.require] // Define INVOKE (f, t1, t2, ..., tN) as follows: @@ -35,6 +37,26 @@ #ifndef ABSL_BASE_INTERNAL_INVOKE_H_ #define ABSL_BASE_INTERNAL_INVOKE_H_ +#include "absl/base/config.h" + +#if ABSL_INTERNAL_CPLUSPLUS_LANG >= 201703L + +#include <functional> + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { + +using std::invoke; +using std::invoke_result_t; +using std::is_invocable_r; + +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl + +#else // ABSL_INTERNAL_CPLUSPLUS_LANG >= 201703L + #include <algorithm> #include <type_traits> #include <utility> @@ -80,8 +102,18 @@ struct MemFunAndRef : StrippedAccept<MemFunAndRef> { static decltype((std::declval<Obj>().* std::declval<MemFun>())(std::declval<Args>()...)) Invoke(MemFun&& mem_fun, Obj&& obj, Args&&... args) { +// Ignore bogus GCC warnings on this line. +// See https://gcc.gnu.org/bugzilla/show_bug.cgi?id=101436 for similar example. +#if ABSL_INTERNAL_HAVE_MIN_GNUC_VERSION(11, 0) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Warray-bounds" +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#endif return (std::forward<Obj>(obj).* std::forward<MemFun>(mem_fun))(std::forward<Args>(args)...); +#if ABSL_INTERNAL_HAVE_MIN_GNUC_VERSION(11, 0) +#pragma GCC diagnostic pop +#endif } }; @@ -180,8 +212,30 @@ invoke_result_t<F, Args...> invoke(F&& f, Args&&... args) { return Invoker<F, Args...>::type::Invoke(std::forward<F>(f), std::forward<Args>(args)...); } + +template <typename AlwaysVoid, typename, typename, typename...> +struct IsInvocableRImpl : std::false_type {}; + +template <typename R, typename F, typename... Args> +struct IsInvocableRImpl< + absl::void_t<absl::base_internal::invoke_result_t<F, Args...> >, R, F, + Args...> + : std::integral_constant< + bool, + std::is_convertible<absl::base_internal::invoke_result_t<F, Args...>, + R>::value || + std::is_void<R>::value> {}; + +// Type trait whose member `value` is true if invoking `F` with `Args` is valid, +// and either the return type is convertible to `R`, or `R` is void. +// C++11-compatible version of `std::is_invocable_r`. +template <typename R, typename F, typename... Args> +using is_invocable_r = IsInvocableRImpl<void, R, F, Args...>; + } // namespace base_internal ABSL_NAMESPACE_END } // namespace absl +#endif // ABSL_INTERNAL_CPLUSPLUS_LANG >= 201703L + #endif // ABSL_BASE_INTERNAL_INVOKE_H_ diff --git a/absl/base/internal/low_level_alloc_test.cc b/absl/base/internal/low_level_alloc_test.cc index 31abb888..8fdec09e 100644 --- a/absl/base/internal/low_level_alloc_test.cc +++ b/absl/base/internal/low_level_alloc_test.cc @@ -86,7 +86,7 @@ static void Test(bool use_new_arena, bool call_malloc_hook, int n) { AllocMap::iterator it; BlockDesc block_desc; int rnd; - LowLevelAlloc::Arena *arena = 0; + LowLevelAlloc::Arena *arena = nullptr; if (use_new_arena) { int32_t flags = call_malloc_hook ? LowLevelAlloc::kCallMallocHook : 0; arena = LowLevelAlloc::NewArena(flags); @@ -101,11 +101,10 @@ static void Test(bool use_new_arena, bool call_malloc_hook, int n) { case 0: // coin came up heads: add a block using_low_level_alloc = true; block_desc.len = rand() & 0x3fff; - block_desc.ptr = - reinterpret_cast<char *>( - arena == 0 - ? LowLevelAlloc::Alloc(block_desc.len) - : LowLevelAlloc::AllocWithArena(block_desc.len, arena)); + block_desc.ptr = reinterpret_cast<char *>( + arena == nullptr + ? LowLevelAlloc::Alloc(block_desc.len) + : LowLevelAlloc::AllocWithArena(block_desc.len, arena)); using_low_level_alloc = false; RandomizeBlockDesc(&block_desc); rnd = rand(); diff --git a/absl/base/internal/prefetch.h b/absl/base/internal/prefetch.h new file mode 100644 index 00000000..06419283 --- /dev/null +++ b/absl/base/internal/prefetch.h @@ -0,0 +1,138 @@ +// Copyright 2022 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_BASE_INTERNAL_PREFETCH_H_ +#define ABSL_BASE_INTERNAL_PREFETCH_H_ + +#include "absl/base/config.h" + +#ifdef __SSE__ +#include <xmmintrin.h> +#endif + +#if defined(_MSC_VER) && defined(ABSL_INTERNAL_HAVE_SSE) +#include <intrin.h> +#pragma intrinsic(_mm_prefetch) +#endif + +// Compatibility wrappers around __builtin_prefetch, to prefetch data +// for read if supported by the toolchain. + +// Move data into the cache before it is read, or "prefetch" it. +// +// The value of `addr` is the address of the memory to prefetch. If +// the target and compiler support it, data prefetch instructions are +// generated. If the prefetch is done some time before the memory is +// read, it may be in the cache by the time the read occurs. +// +// The function names specify the temporal locality heuristic applied, +// using the names of Intel prefetch instructions: +// +// T0 - high degree of temporal locality; data should be left in as +// many levels of the cache possible +// T1 - moderate degree of temporal locality +// T2 - low degree of temporal locality +// Nta - no temporal locality, data need not be left in the cache +// after the read +// +// Incorrect or gratuitous use of these functions can degrade +// performance, so use them only when representative benchmarks show +// an improvement. +// +// Example usage: +// +// absl::base_internal::PrefetchT0(addr); +// +// Currently, the different prefetch calls behave on some Intel +// architectures as follows: +// +// SNB..SKL SKX +// PrefetchT0() L1/L2/L3 L1/L2 +// PrefetchT1() L2/L3 L2 +// PrefetchT2() L2/L3 L2 +// PrefetchNta() L1/--/L3 L1* +// +// * On SKX PrefetchNta() will bring the line into L1 but will evict +// from L3 cache. This might result in surprising behavior. +// +// SNB = Sandy Bridge, SKL = Skylake, SKX = Skylake Xeon. +// +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace base_internal { + +void PrefetchT0(const void* addr); +void PrefetchT1(const void* addr); +void PrefetchT2(const void* addr); +void PrefetchNta(const void* addr); + +// Implementation details follow. + +#if ABSL_HAVE_BUILTIN(__builtin_prefetch) || defined(__GNUC__) + +#define ABSL_INTERNAL_HAVE_PREFETCH 1 + +// See __builtin_prefetch: +// https://gcc.gnu.org/onlinedocs/gcc/Other-Builtins.html. +// +// These functions speculatively load for read only. This is +// safe for all currently supported platforms. However, prefetch for +// store may have problems depending on the target platform. +// +inline void PrefetchT0(const void* addr) { + // Note: this uses prefetcht0 on Intel. + __builtin_prefetch(addr, 0, 3); +} +inline void PrefetchT1(const void* addr) { + // Note: this uses prefetcht1 on Intel. + __builtin_prefetch(addr, 0, 2); +} +inline void PrefetchT2(const void* addr) { + // Note: this uses prefetcht2 on Intel. + __builtin_prefetch(addr, 0, 1); +} +inline void PrefetchNta(const void* addr) { + // Note: this uses prefetchtnta on Intel. + __builtin_prefetch(addr, 0, 0); +} + +#elif defined(ABSL_INTERNAL_HAVE_SSE) + +#define ABSL_INTERNAL_HAVE_PREFETCH 1 + +inline void PrefetchT0(const void* addr) { + _mm_prefetch(reinterpret_cast<const char*>(addr), _MM_HINT_T0); +} +inline void PrefetchT1(const void* addr) { + _mm_prefetch(reinterpret_cast<const char*>(addr), _MM_HINT_T1); +} +inline void PrefetchT2(const void* addr) { + _mm_prefetch(reinterpret_cast<const char*>(addr), _MM_HINT_T2); +} +inline void PrefetchNta(const void* addr) { + _mm_prefetch(reinterpret_cast<const char*>(addr), _MM_HINT_NTA); +} + +#else +inline void PrefetchT0(const void*) {} +inline void PrefetchT1(const void*) {} +inline void PrefetchT2(const void*) {} +inline void PrefetchNta(const void*) {} +#endif + +} // namespace base_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_BASE_INTERNAL_PREFETCH_H_ diff --git a/absl/base/internal/prefetch_test.cc b/absl/base/internal/prefetch_test.cc new file mode 100644 index 00000000..7c1dae46 --- /dev/null +++ b/absl/base/internal/prefetch_test.cc @@ -0,0 +1,43 @@ +// Copyright 2022 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/base/internal/prefetch.h" + +#include "gtest/gtest.h" + +namespace { + +int number = 42; + +TEST(Prefetch, TemporalLocalityNone) { + absl::base_internal::PrefetchNta(&number); + EXPECT_EQ(number, 42); +} + +TEST(Prefetch, TemporalLocalityLow) { + absl::base_internal::PrefetchT2(&number); + EXPECT_EQ(number, 42); +} + +TEST(Prefetch, TemporalLocalityMedium) { + absl::base_internal::PrefetchT1(&number); + EXPECT_EQ(number, 42); +} + +TEST(Prefetch, TemporalLocalityHigh) { + absl::base_internal::PrefetchT0(&number); + EXPECT_EQ(number, 42); +} + +} // namespace diff --git a/absl/base/internal/raw_logging.cc b/absl/base/internal/raw_logging.cc index 074e026a..54e71a3f 100644 --- a/absl/base/internal/raw_logging.cc +++ b/absl/base/internal/raw_logging.cc @@ -14,15 +14,17 @@ #include "absl/base/internal/raw_logging.h" -#include <stddef.h> #include <cstdarg> +#include <cstddef> #include <cstdio> #include <cstdlib> #include <cstring> +#include <string> #include "absl/base/attributes.h" #include "absl/base/config.h" #include "absl/base/internal/atomic_hook.h" +#include "absl/base/internal/errno_saver.h" #include "absl/base/log_severity.h" // We know how to perform low-level writes to stderr in POSIX and Windows. For @@ -36,8 +38,8 @@ // This preprocessor token is also defined in raw_io.cc. If you need to copy // this, consider moving both to config.h instead. #if defined(__linux__) || defined(__APPLE__) || defined(__FreeBSD__) || \ - defined(__Fuchsia__) || defined(__native_client__) || \ - defined(__EMSCRIPTEN__) || defined(__ASYLO__) + defined(__Fuchsia__) || defined(__native_client__) || \ + defined(__OpenBSD__) || defined(__EMSCRIPTEN__) || defined(__ASYLO__) #include <unistd.h> @@ -50,7 +52,8 @@ // ABSL_HAVE_SYSCALL_WRITE is defined when the platform provides the syscall // syscall(SYS_write, /*int*/ fd, /*char* */ buf, /*size_t*/ len); // for low level operations that want to avoid libc. -#if (defined(__linux__) || defined(__FreeBSD__)) && !defined(__ANDROID__) +#if (defined(__linux__) || defined(__FreeBSD__) || defined(__OpenBSD__)) && \ + !defined(__ANDROID__) #include <sys/syscall.h> #define ABSL_HAVE_SYSCALL_WRITE 1 #define ABSL_LOW_LEVEL_WRITE_SUPPORTED 1 @@ -76,13 +79,6 @@ namespace { // Explicitly `#error` out when not `ABSL_LOW_LEVEL_WRITE_SUPPORTED`, except for // a selected set of platforms for which we expect not to be able to raw log. -ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES - absl::base_internal::AtomicHook<LogPrefixHook> - log_prefix_hook; -ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES - absl::base_internal::AtomicHook<AbortHook> - abort_hook; - #ifdef ABSL_LOW_LEVEL_WRITE_SUPPORTED constexpr char kTruncated[] = " ... (message truncated)\n"; @@ -130,6 +126,18 @@ bool DoRawLog(char** buf, int* size, const char* format, ...) { return true; } +bool DefaultLogFilterAndPrefix(absl::LogSeverity, const char* file, int line, + char** buf, int* buf_size) { + DoRawLog(buf, buf_size, "[%s : %d] RAW: ", file, line); + return true; +} + +ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES +absl::base_internal::AtomicHook<LogFilterAndPrefixHook> + log_filter_and_prefix_hook(DefaultLogFilterAndPrefix); +ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES +absl::base_internal::AtomicHook<AbortHook> abort_hook; + void RawLogVA(absl::LogSeverity severity, const char* file, int line, const char* format, va_list ap) ABSL_PRINTF_ATTRIBUTE(4, 0); void RawLogVA(absl::LogSeverity severity, const char* file, int line, @@ -150,14 +158,7 @@ void RawLogVA(absl::LogSeverity severity, const char* file, int line, } #endif - auto log_prefix_hook_ptr = log_prefix_hook.Load(); - if (log_prefix_hook_ptr) { - enabled = log_prefix_hook_ptr(severity, file, line, &buf, &size); - } else { - if (enabled) { - DoRawLog(&buf, &size, "[%s : %d] RAW: ", file, line); - } - } + enabled = log_filter_and_prefix_hook(severity, file, line, &buf, &size); const char* const prefix_end = buf; #ifdef ABSL_LOW_LEVEL_WRITE_SUPPORTED @@ -168,11 +169,12 @@ void RawLogVA(absl::LogSeverity severity, const char* file, int line, } else { DoRawLog(&buf, &size, "%s", kTruncated); } - SafeWriteToStderr(buffer, strlen(buffer)); + AsyncSignalSafeWriteToStderr(buffer, strlen(buffer)); } #else static_cast<void>(format); static_cast<void>(ap); + static_cast<void>(enabled); #endif // Abort the process after logging a FATAL message, even if the output itself @@ -195,8 +197,11 @@ void DefaultInternalLog(absl::LogSeverity severity, const char* file, int line, } // namespace -void SafeWriteToStderr(const char *s, size_t len) { +void AsyncSignalSafeWriteToStderr(const char* s, size_t len) { + absl::base_internal::ErrnoSaver errno_saver; #if defined(ABSL_HAVE_SYSCALL_WRITE) + // We prefer calling write via `syscall` to minimize the risk of libc doing + // something "helpful". syscall(SYS_write, STDERR_FILENO, s, len); #elif defined(ABSL_HAVE_POSIX_WRITE) write(STDERR_FILENO, s, len); @@ -229,7 +234,9 @@ ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES ABSL_DLL absl::base_internal::AtomicHook<InternalLogFunction> internal_log_function(DefaultInternalLog); -void RegisterLogPrefixHook(LogPrefixHook func) { log_prefix_hook.Store(func); } +void RegisterLogFilterAndPrefixHook(LogFilterAndPrefixHook func) { + log_filter_and_prefix_hook.Store(func); +} void RegisterAbortHook(AbortHook func) { abort_hook.Store(func); } diff --git a/absl/base/internal/raw_logging.h b/absl/base/internal/raw_logging.h index 2bf7aaba..0747c9df 100644 --- a/absl/base/internal/raw_logging.h +++ b/absl/base/internal/raw_logging.h @@ -109,12 +109,9 @@ namespace raw_logging_internal { void RawLog(absl::LogSeverity severity, const char* file, int line, const char* format, ...) ABSL_PRINTF_ATTRIBUTE(4, 5); -// Writes the provided buffer directly to stderr, in a safe, low-level manner. -// -// In POSIX this means calling write(), which is async-signal safe and does -// not malloc. If the platform supports the SYS_write syscall, we invoke that -// directly to side-step any libc interception. -void SafeWriteToStderr(const char *s, size_t len); +// Writes the provided buffer directly to stderr, in a signal-safe, low-level +// manner. +void AsyncSignalSafeWriteToStderr(const char* s, size_t len); // compile-time function to get the "base" filename, that is, the part of // a filename after the last "/" or "\" path separator. The search starts at @@ -148,11 +145,12 @@ bool RawLoggingFullySupported(); // 'severity' is the severity level of the message being written. // 'file' and 'line' are the file and line number where the ABSL_RAW_LOG macro // was located. -// 'buffer' and 'buf_size' are pointers to the buffer and buffer size. If the -// hook writes a prefix, it must increment *buffer and decrement *buf_size +// 'buf' and 'buf_size' are pointers to the buffer and buffer size. If the +// hook writes a prefix, it must increment *buf and decrement *buf_size // accordingly. -using LogPrefixHook = bool (*)(absl::LogSeverity severity, const char* file, - int line, char** buffer, int* buf_size); +using LogFilterAndPrefixHook = bool (*)(absl::LogSeverity severity, + const char* file, int line, char** buf, + int* buf_size); // Function type for a raw_logging customization hook called to abort a process // when a FATAL message is logged. If the provided AbortHook() returns, the @@ -162,7 +160,10 @@ using LogPrefixHook = bool (*)(absl::LogSeverity severity, const char* file, // was located. // The NUL-terminated logged message lives in the buffer between 'buf_start' // and 'buf_end'. 'prefix_end' points to the first non-prefix character of the -// buffer (as written by the LogPrefixHook.) +// buffer (as written by the LogFilterAndPrefixHook.) +// +// The lifetime of the filename and message buffers will not end while the +// process remains alive. using AbortHook = void (*)(const char* file, int line, const char* buf_start, const char* prefix_end, const char* buf_end); @@ -184,7 +185,7 @@ ABSL_INTERNAL_ATOMIC_HOOK_ATTRIBUTES ABSL_DLL extern base_internal::AtomicHook< // // These functions are safe to call at any point during initialization; they do // not block or malloc, and are async-signal safe. -void RegisterLogPrefixHook(LogPrefixHook func); +void RegisterLogFilterAndPrefixHook(LogFilterAndPrefixHook func); void RegisterAbortHook(AbortHook func); void RegisterInternalLogFunction(InternalLogFunction func); diff --git a/absl/base/internal/spinlock.cc b/absl/base/internal/spinlock.cc index 35c0696a..9b5ed6e4 100644 --- a/absl/base/internal/spinlock.cc +++ b/absl/base/internal/spinlock.cc @@ -19,6 +19,7 @@ #include <limits> #include "absl/base/attributes.h" +#include "absl/base/config.h" #include "absl/base/internal/atomic_hook.h" #include "absl/base/internal/cycleclock.h" #include "absl/base/internal/spinlock_wait.h" @@ -66,12 +67,14 @@ void RegisterSpinLockProfiler(void (*fn)(const void *contendedlock, submit_profile_data.Store(fn); } +#ifdef ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL // Static member variable definitions. constexpr uint32_t SpinLock::kSpinLockHeld; constexpr uint32_t SpinLock::kSpinLockCooperative; constexpr uint32_t SpinLock::kSpinLockDisabledScheduling; constexpr uint32_t SpinLock::kSpinLockSleeper; constexpr uint32_t SpinLock::kWaitTimeMask; +#endif // Uncommon constructors. SpinLock::SpinLock(base_internal::SchedulingMode mode) diff --git a/absl/base/internal/spinlock.h b/absl/base/internal/spinlock.h index c73b5e09..6d8d8ddd 100644 --- a/absl/base/internal/spinlock.h +++ b/absl/base/internal/spinlock.h @@ -16,13 +16,15 @@ // Most users requiring mutual exclusion should use Mutex. // SpinLock is provided for use in two situations: -// - for use in code that Mutex itself depends on +// - for use by Abseil internal code that Mutex itself depends on // - for async signal safety (see below) // SpinLock is async signal safe. If a spinlock is used within a signal // handler, all code that acquires the lock must ensure that the signal cannot // arrive while they are holding the lock. Typically, this is done by blocking // the signal. +// +// Threads waiting on a SpinLock may be woken in an arbitrary order. #ifndef ABSL_BASE_INTERNAL_SPINLOCK_H_ #define ABSL_BASE_INTERNAL_SPINLOCK_H_ @@ -118,6 +120,14 @@ class ABSL_LOCKABLE SpinLock { return (lockword_.load(std::memory_order_relaxed) & kSpinLockHeld) != 0; } + // Return immediately if this thread holds the SpinLock exclusively. + // Otherwise, report an error by crashing with a diagnostic. + inline void AssertHeld() const ABSL_ASSERT_EXCLUSIVE_LOCK() { + if (!IsHeld()) { + ABSL_RAW_LOG(FATAL, "thread should hold the lock on SpinLock"); + } + } + protected: // These should not be exported except for testing. diff --git a/absl/base/internal/spinlock_linux.inc b/absl/base/internal/spinlock_linux.inc index 202f7cdf..fe8ba674 100644 --- a/absl/base/internal/spinlock_linux.inc +++ b/absl/base/internal/spinlock_linux.inc @@ -57,13 +57,10 @@ static_assert(sizeof(std::atomic<uint32_t>) == sizeof(int), extern "C" { ABSL_ATTRIBUTE_WEAK void ABSL_INTERNAL_C_SYMBOL(AbslInternalSpinLockDelay)( - std::atomic<uint32_t> *w, uint32_t value, int loop, + std::atomic<uint32_t> *w, uint32_t value, int, absl::base_internal::SchedulingMode) { absl::base_internal::ErrnoSaver errno_saver; - struct timespec tm; - tm.tv_sec = 0; - tm.tv_nsec = absl::base_internal::SpinLockSuggestedDelayNS(loop); - syscall(SYS_futex, w, FUTEX_WAIT | FUTEX_PRIVATE_FLAG, value, &tm); + syscall(SYS_futex, w, FUTEX_WAIT | FUTEX_PRIVATE_FLAG, value, nullptr); } ABSL_ATTRIBUTE_WEAK void ABSL_INTERNAL_C_SYMBOL(AbslInternalSpinLockWake)( diff --git a/absl/base/internal/spinlock_wait.h b/absl/base/internal/spinlock_wait.h index 579bd09f..9a1adcda 100644 --- a/absl/base/internal/spinlock_wait.h +++ b/absl/base/internal/spinlock_wait.h @@ -39,6 +39,8 @@ struct SpinLockWaitTransition { // satisfying 0<=i<n && trans[i].done, atomically make the transition, // then return the old value of *w. Make any other atomic transitions // where !trans[i].done, but continue waiting. +// +// Wakeups for threads blocked on SpinLockWait do not respect priorities. uint32_t SpinLockWait(std::atomic<uint32_t> *w, int n, const SpinLockWaitTransition trans[], SchedulingMode scheduling_mode); diff --git a/absl/base/internal/sysinfo.cc b/absl/base/internal/sysinfo.cc index 4a3b2050..c8366df1 100644 --- a/absl/base/internal/sysinfo.cc +++ b/absl/base/internal/sysinfo.cc @@ -61,9 +61,77 @@ namespace absl { ABSL_NAMESPACE_BEGIN namespace base_internal { +namespace { + +#if defined(_WIN32) + +// Returns number of bits set in `bitMask` +DWORD Win32CountSetBits(ULONG_PTR bitMask) { + for (DWORD bitSetCount = 0; ; ++bitSetCount) { + if (bitMask == 0) return bitSetCount; + bitMask &= bitMask - 1; + } +} + +// Returns the number of logical CPUs using GetLogicalProcessorInformation(), or +// 0 if the number of processors is not available or can not be computed. +// https://docs.microsoft.com/en-us/windows/win32/api/sysinfoapi/nf-sysinfoapi-getlogicalprocessorinformation +int Win32NumCPUs() { +#pragma comment(lib, "kernel32.lib") + using Info = SYSTEM_LOGICAL_PROCESSOR_INFORMATION; + + DWORD info_size = sizeof(Info); + Info* info(static_cast<Info*>(malloc(info_size))); + if (info == nullptr) return 0; + + bool success = GetLogicalProcessorInformation(info, &info_size); + if (!success && GetLastError() == ERROR_INSUFFICIENT_BUFFER) { + free(info); + info = static_cast<Info*>(malloc(info_size)); + if (info == nullptr) return 0; + success = GetLogicalProcessorInformation(info, &info_size); + } + + DWORD logicalProcessorCount = 0; + if (success) { + Info* ptr = info; + DWORD byteOffset = 0; + while (byteOffset + sizeof(Info) <= info_size) { + switch (ptr->Relationship) { + case RelationProcessorCore: + logicalProcessorCount += Win32CountSetBits(ptr->ProcessorMask); + break; + + case RelationNumaNode: + case RelationCache: + case RelationProcessorPackage: + // Ignore other entries + break; + + default: + // Ignore unknown entries + break; + } + byteOffset += sizeof(Info); + ptr++; + } + } + free(info); + return logicalProcessorCount; +} + +#endif + +} // namespace + static int GetNumCPUs() { #if defined(__myriad2__) return 1; +#elif defined(_WIN32) + const unsigned hardware_concurrency = Win32NumCPUs(); + return hardware_concurrency ? hardware_concurrency : 1; +#elif defined(_AIX) + return sysconf(_SC_NPROCESSORS_ONLN); #else // Other possibilities: // - Read /sys/devices/system/cpu/online and use cpumask_parse() diff --git a/absl/base/internal/sysinfo_test.cc b/absl/base/internal/sysinfo_test.cc index 5f9e45f6..f305b6c5 100644 --- a/absl/base/internal/sysinfo_test.cc +++ b/absl/base/internal/sysinfo_test.cc @@ -37,29 +37,6 @@ TEST(SysinfoTest, NumCPUs) { << "NumCPUs() should not have the default value of 0"; } -// Ensure that NominalCPUFrequency returns a reasonable value, or 1.00 on -// platforms where the CPU frequency is not available through sysfs. -// -// POWER is particularly problematic here; some Linux kernels expose the CPU -// frequency, while others do not. Since we can't predict a priori what a given -// machine is going to do, just disable this test on POWER on Linux. -#if !(defined(__linux) && (defined(__ppc64__) || defined(__PPC64__))) -TEST(SysinfoTest, NominalCPUFrequency) { - // Linux only exposes the CPU frequency on certain architectures, and - // Emscripten doesn't expose it at all. -#if defined(__linux__) && \ - (defined(__aarch64__) || defined(__hppa__) || defined(__mips__) || \ - defined(__riscv) || defined(__s390x__)) || \ - defined(__EMSCRIPTEN__) - EXPECT_EQ(NominalCPUFrequency(), 1.0) - << "CPU frequency detection was fixed! Please update unittest."; -#else - EXPECT_GE(NominalCPUFrequency(), 1000.0) - << "NominalCPUFrequency() did not return a reasonable value"; -#endif -} -#endif - TEST(SysinfoTest, GetTID) { EXPECT_EQ(GetTID(), GetTID()); // Basic compile and equality test. #ifdef __native_client__ diff --git a/absl/base/internal/thread_identity.cc b/absl/base/internal/thread_identity.cc index 9950e63a..79853f09 100644 --- a/absl/base/internal/thread_identity.cc +++ b/absl/base/internal/thread_identity.cc @@ -14,7 +14,7 @@ #include "absl/base/internal/thread_identity.h" -#ifndef _WIN32 +#if !defined(_WIN32) || defined(__MINGW32__) #include <pthread.h> #include <signal.h> #endif @@ -56,6 +56,7 @@ void AllocateThreadIdentityKey(ThreadIdentityReclaimerFunction reclaimer) { // *different* instances of this ptr. // Apple platforms have the visibility attribute, but issue a compile warning // that protected visibility is unsupported. +ABSL_CONST_INIT // Must come before __attribute__((visibility("protected"))) #if ABSL_HAVE_ATTRIBUTE(visibility) && !defined(__APPLE__) __attribute__((visibility("protected"))) #endif // ABSL_HAVE_ATTRIBUTE(visibility) && !defined(__APPLE__) diff --git a/absl/base/internal/thread_identity.h b/absl/base/internal/thread_identity.h index 6e25b92f..659694b3 100644 --- a/absl/base/internal/thread_identity.h +++ b/absl/base/internal/thread_identity.h @@ -188,25 +188,25 @@ void ClearCurrentThreadIdentity(); // May be chosen at compile time via: -DABSL_FORCE_THREAD_IDENTITY_MODE=<mode // index> #ifdef ABSL_THREAD_IDENTITY_MODE_USE_POSIX_SETSPECIFIC -#error ABSL_THREAD_IDENTITY_MODE_USE_POSIX_SETSPECIFIC cannot be direcly set +#error ABSL_THREAD_IDENTITY_MODE_USE_POSIX_SETSPECIFIC cannot be directly set #else #define ABSL_THREAD_IDENTITY_MODE_USE_POSIX_SETSPECIFIC 0 #endif #ifdef ABSL_THREAD_IDENTITY_MODE_USE_TLS -#error ABSL_THREAD_IDENTITY_MODE_USE_TLS cannot be direcly set +#error ABSL_THREAD_IDENTITY_MODE_USE_TLS cannot be directly set #else #define ABSL_THREAD_IDENTITY_MODE_USE_TLS 1 #endif #ifdef ABSL_THREAD_IDENTITY_MODE_USE_CPP11 -#error ABSL_THREAD_IDENTITY_MODE_USE_CPP11 cannot be direcly set +#error ABSL_THREAD_IDENTITY_MODE_USE_CPP11 cannot be directly set #else #define ABSL_THREAD_IDENTITY_MODE_USE_CPP11 2 #endif #ifdef ABSL_THREAD_IDENTITY_MODE -#error ABSL_THREAD_IDENTITY_MODE cannot be direcly set +#error ABSL_THREAD_IDENTITY_MODE cannot be directly set #elif defined(ABSL_FORCE_THREAD_IDENTITY_MODE) #define ABSL_THREAD_IDENTITY_MODE ABSL_FORCE_THREAD_IDENTITY_MODE #elif defined(_WIN32) && !defined(__MINGW32__) diff --git a/absl/base/internal/unscaledcycleclock.cc b/absl/base/internal/unscaledcycleclock.cc index 1545288c..b1c396c6 100644 --- a/absl/base/internal/unscaledcycleclock.cc +++ b/absl/base/internal/unscaledcycleclock.cc @@ -24,8 +24,13 @@ #ifdef __GLIBC__ #include <sys/platform/ppc.h> #elif defined(__FreeBSD__) -#include <sys/sysctl.h> +// clang-format off +// This order does actually matter =(. #include <sys/types.h> +#include <sys/sysctl.h> +// clang-format on + +#include "absl/base/call_once.h" #endif #endif @@ -49,12 +54,6 @@ double UnscaledCycleClock::Frequency() { #elif defined(__x86_64__) -int64_t UnscaledCycleClock::Now() { - uint64_t low, high; - __asm__ volatile("rdtsc" : "=a"(low), "=d"(high)); - return (high << 32) | low; -} - double UnscaledCycleClock::Frequency() { return base_internal::NominalCPUFrequency(); } @@ -87,6 +86,10 @@ int64_t UnscaledCycleClock::Now() { double UnscaledCycleClock::Frequency() { #ifdef __GLIBC__ return __ppc_get_timebase_freq(); +#elif defined(_AIX) + // This is the same constant value as returned by + // __ppc_get_timebase_freq(). + return static_cast<double>(512000000); #elif defined(__FreeBSD__) static once_flag init_timebase_frequency_once; static double timebase_frequency = 0.0; @@ -119,6 +122,18 @@ double UnscaledCycleClock::Frequency() { return aarch64_timer_frequency; } +#elif defined(__riscv) + +int64_t UnscaledCycleClock::Now() { + int64_t virtual_timer_value; + asm volatile("rdcycle %0" : "=r"(virtual_timer_value)); + return virtual_timer_value; +} + +double UnscaledCycleClock::Frequency() { + return base_internal::NominalCPUFrequency(); +} + #elif defined(_M_IX86) || defined(_M_X64) #pragma intrinsic(__rdtsc) diff --git a/absl/base/internal/unscaledcycleclock.h b/absl/base/internal/unscaledcycleclock.h index 82f2c87a..2cbeae31 100644 --- a/absl/base/internal/unscaledcycleclock.h +++ b/absl/base/internal/unscaledcycleclock.h @@ -46,8 +46,8 @@ // The following platforms have an implementation of a hardware counter. #if defined(__i386__) || defined(__x86_64__) || defined(__aarch64__) || \ - defined(__powerpc__) || defined(__ppc__) || \ - defined(_M_IX86) || defined(_M_X64) + defined(__powerpc__) || defined(__ppc__) || defined(__riscv) || \ + defined(_M_IX86) || (defined(_M_X64) && !defined(_M_ARM64EC)) #define ABSL_HAVE_UNSCALED_CYCLECLOCK_IMPLEMENTATION 1 #else #define ABSL_HAVE_UNSCALED_CYCLECLOCK_IMPLEMENTATION 0 @@ -59,8 +59,7 @@ // CycleClock that runs at atleast 1 MHz. We've found some Android // ARM64 devices where this is not the case, so we disable it by // default on Android ARM64. -#if defined(__native_client__) || \ - (defined(TARGET_OS_IPHONE) && TARGET_OS_IPHONE) || \ +#if defined(__native_client__) || (defined(__APPLE__)) || \ (defined(__ANDROID__) && defined(__aarch64__)) #define ABSL_USE_UNSCALED_CYCLECLOCK_DEFAULT 0 #else @@ -80,8 +79,8 @@ // This macro can be used to test if UnscaledCycleClock::Frequency() // is NominalCPUFrequency() on a particular platform. -#if (defined(__i386__) || defined(__x86_64__) || \ - defined(_M_IX86) || defined(_M_X64)) +#if (defined(__i386__) || defined(__x86_64__) || defined(__riscv) || \ + defined(_M_IX86) || defined(_M_X64)) #define ABSL_INTERNAL_UNSCALED_CYCLECLOCK_FREQUENCY_IS_CPU_FREQUENCY #endif @@ -115,6 +114,16 @@ class UnscaledCycleClock { friend class base_internal::UnscaledCycleClockWrapperForInitializeFrequency; }; +#if defined(__x86_64__) + +inline int64_t UnscaledCycleClock::Now() { + uint64_t low, high; + __asm__ volatile("rdtsc" : "=a"(low), "=d"(high)); + return (high << 32) | low; +} + +#endif + } // namespace base_internal ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/base/invoke_test.cc b/absl/base/invoke_test.cc index bcdef36c..7be26f64 100644 --- a/absl/base/invoke_test.cc +++ b/absl/base/invoke_test.cc @@ -31,6 +31,14 @@ namespace { int Function(int a, int b) { return a - b; } +void VoidFunction(int& a, int& b) { + a += b; + b = a - b; + a -= b; +} + +int ZeroArgFunction() { return -1937; } + int Sink(std::unique_ptr<int> p) { return *p; } @@ -223,6 +231,100 @@ TEST(InvokeTest, SfinaeFriendly) { EXPECT_THAT(CallMaybeWithArg(Factory), ::testing::Pointee(42)); } +TEST(IsInvocableRTest, CallableExactMatch) { + static_assert( + base_internal::is_invocable_r<int, decltype(Function), int, int>::value, + "Should be true for exact match of types on a free function"); +} + +TEST(IsInvocableRTest, CallableArgumentConversionMatch) { + static_assert( + base_internal::is_invocable_r<int, decltype(Function), char, int>::value, + "Should be true for convertible argument type"); +} + +TEST(IsInvocableRTest, CallableReturnConversionMatch) { + static_assert(base_internal::is_invocable_r<double, decltype(Function), int, + int>::value, + "Should be true for convertible return type"); +} + +TEST(IsInvocableRTest, CallableReturnVoid) { + static_assert(base_internal::is_invocable_r<void, decltype(VoidFunction), + int&, int&>::value, + "Should be true for void expected and actual return types"); + static_assert( + base_internal::is_invocable_r<void, decltype(Function), int, int>::value, + "Should be true for void expected and non-void actual return types"); +} + +TEST(IsInvocableRTest, CallableRefQualifierMismatch) { + static_assert(!base_internal::is_invocable_r<void, decltype(VoidFunction), + int&, const int&>::value, + "Should be false for reference constness mismatch"); + static_assert(!base_internal::is_invocable_r<void, decltype(VoidFunction), + int&&, int&>::value, + "Should be false for reference value category mismatch"); +} + +TEST(IsInvocableRTest, CallableArgumentTypeMismatch) { + static_assert(!base_internal::is_invocable_r<int, decltype(Function), + std::string, int>::value, + "Should be false for argument type mismatch"); +} + +TEST(IsInvocableRTest, CallableReturnTypeMismatch) { + static_assert(!base_internal::is_invocable_r<std::string, decltype(Function), + int, int>::value, + "Should be false for return type mismatch"); +} + +TEST(IsInvocableRTest, CallableTooFewArgs) { + static_assert( + !base_internal::is_invocable_r<int, decltype(Function), int>::value, + "Should be false for too few arguments"); +} + +TEST(IsInvocableRTest, CallableTooManyArgs) { + static_assert(!base_internal::is_invocable_r<int, decltype(Function), int, + int, int>::value, + "Should be false for too many arguments"); +} + +TEST(IsInvocableRTest, MemberFunctionAndReference) { + static_assert(base_internal::is_invocable_r<int, decltype(&Class::Method), + Class&, int, int>::value, + "Should be true for exact match of types on a member function " + "and class reference"); +} + +TEST(IsInvocableRTest, MemberFunctionAndPointer) { + static_assert(base_internal::is_invocable_r<int, decltype(&Class::Method), + Class*, int, int>::value, + "Should be true for exact match of types on a member function " + "and class pointer"); +} + +TEST(IsInvocableRTest, DataMemberAndReference) { + static_assert(base_internal::is_invocable_r<int, decltype(&Class::member), + Class&>::value, + "Should be true for exact match of types on a data member and " + "class reference"); +} + +TEST(IsInvocableRTest, DataMemberAndPointer) { + static_assert(base_internal::is_invocable_r<int, decltype(&Class::member), + Class*>::value, + "Should be true for exact match of types on a data member and " + "class pointer"); +} + +TEST(IsInvocableRTest, CallableZeroArgs) { + static_assert( + base_internal::is_invocable_r<int, decltype(ZeroArgFunction)>::value, + "Should be true for exact match for a zero-arg free function"); +} + } // namespace } // namespace base_internal ABSL_NAMESPACE_END diff --git a/absl/base/log_severity.cc b/absl/base/log_severity.cc index 72312afd..60a8fc1f 100644 --- a/absl/base/log_severity.cc +++ b/absl/base/log_severity.cc @@ -16,6 +16,8 @@ #include <ostream> +#include "absl/base/attributes.h" + namespace absl { ABSL_NAMESPACE_BEGIN @@ -23,5 +25,31 @@ std::ostream& operator<<(std::ostream& os, absl::LogSeverity s) { if (s == absl::NormalizeLogSeverity(s)) return os << absl::LogSeverityName(s); return os << "absl::LogSeverity(" << static_cast<int>(s) << ")"; } + +std::ostream& operator<<(std::ostream& os, absl::LogSeverityAtLeast s) { + switch (s) { + case absl::LogSeverityAtLeast::kInfo: + case absl::LogSeverityAtLeast::kWarning: + case absl::LogSeverityAtLeast::kError: + case absl::LogSeverityAtLeast::kFatal: + return os << ">=" << static_cast<absl::LogSeverity>(s); + case absl::LogSeverityAtLeast::kInfinity: + return os << "INFINITY"; + } + return os; +} + +std::ostream& operator<<(std::ostream& os, absl::LogSeverityAtMost s) { + switch (s) { + case absl::LogSeverityAtMost::kInfo: + case absl::LogSeverityAtMost::kWarning: + case absl::LogSeverityAtMost::kError: + case absl::LogSeverityAtMost::kFatal: + return os << "<=" << static_cast<absl::LogSeverity>(s); + case absl::LogSeverityAtMost::kNegativeInfinity: + return os << "NEGATIVE_INFINITY"; + } + return os; +} ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/base/log_severity.h b/absl/base/log_severity.h index 22364224..8bdca38b 100644 --- a/absl/base/log_severity.h +++ b/absl/base/log_severity.h @@ -115,6 +115,57 @@ constexpr absl::LogSeverity NormalizeLogSeverity(int s) { // unspecified; do not rely on it. std::ostream& operator<<(std::ostream& os, absl::LogSeverity s); +// Enums representing a lower bound for LogSeverity. APIs that only operate on +// messages of at least a certain level (for example, `SetMinLogLevel()`) use +// this type to specify that level. absl::LogSeverityAtLeast::kInfinity is +// a level above all threshold levels and therefore no log message will +// ever meet this threshold. +enum class LogSeverityAtLeast : int { + kInfo = static_cast<int>(absl::LogSeverity::kInfo), + kWarning = static_cast<int>(absl::LogSeverity::kWarning), + kError = static_cast<int>(absl::LogSeverity::kError), + kFatal = static_cast<int>(absl::LogSeverity::kFatal), + kInfinity = 1000, +}; + +std::ostream& operator<<(std::ostream& os, absl::LogSeverityAtLeast s); + +// Enums representing an upper bound for LogSeverity. APIs that only operate on +// messages of at most a certain level (for example, buffer all messages at or +// below a certain level) use this type to specify that level. +// absl::LogSeverityAtMost::kNegativeInfinity is a level below all threshold +// levels and therefore will exclude all log messages. +enum class LogSeverityAtMost : int { + kNegativeInfinity = -1000, + kInfo = static_cast<int>(absl::LogSeverity::kInfo), + kWarning = static_cast<int>(absl::LogSeverity::kWarning), + kError = static_cast<int>(absl::LogSeverity::kError), + kFatal = static_cast<int>(absl::LogSeverity::kFatal), +}; + +std::ostream& operator<<(std::ostream& os, absl::LogSeverityAtMost s); + +#define COMPOP(op1, op2, T) \ + constexpr bool operator op1(absl::T lhs, absl::LogSeverity rhs) { \ + return static_cast<absl::LogSeverity>(lhs) op1 rhs; \ + } \ + constexpr bool operator op2(absl::LogSeverity lhs, absl::T rhs) { \ + return lhs op2 static_cast<absl::LogSeverity>(rhs); \ + } + +// Comparisons between `LogSeverity` and `LogSeverityAtLeast`/ +// `LogSeverityAtMost` are only supported in one direction. +// Valid checks are: +// LogSeverity >= LogSeverityAtLeast +// LogSeverity < LogSeverityAtLeast +// LogSeverity <= LogSeverityAtMost +// LogSeverity > LogSeverityAtMost +COMPOP(>, <, LogSeverityAtLeast) +COMPOP(<=, >=, LogSeverityAtLeast) +COMPOP(<, >, LogSeverityAtMost) +COMPOP(>=, <=, LogSeverityAtMost) +#undef COMPOP + ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/base/log_severity_test.cc b/absl/base/log_severity_test.cc index 2c6872b0..16091a5b 100644 --- a/absl/base/log_severity_test.cc +++ b/absl/base/log_severity_test.cc @@ -35,7 +35,8 @@ using ::testing::IsTrue; using ::testing::TestWithParam; using ::testing::Values; -std::string StreamHelper(absl::LogSeverity value) { +template <typename T> +std::string StreamHelper(T value) { std::ostringstream stream; stream << value; return stream.str(); @@ -52,9 +53,9 @@ TEST(StreamTest, Works) { Eq("absl::LogSeverity(4)")); } -static_assert( - absl::flags_internal::FlagUseOneWordStorage<absl::LogSeverity>::value, - "Flags of type absl::LogSeverity ought to be lock-free."); +static_assert(absl::flags_internal::FlagUseValueAndInitBitStorage< + absl::LogSeverity>::value, + "Flags of type absl::LogSeverity ought to be lock-free."); using ParseFlagFromOutOfRangeIntegerTest = TestWithParam<int64_t>; INSTANTIATE_TEST_SUITE_P( @@ -201,4 +202,44 @@ TEST_P(UnparseFlagToOtherIntegerTest, ReturnsExpectedValueAndRoundTrips) { IsTrue()); EXPECT_THAT(reparsed_value, Eq(to_unparse)); } + +TEST(LogThresholdTest, LogSeverityAtLeastTest) { + EXPECT_LT(absl::LogSeverity::kError, absl::LogSeverityAtLeast::kFatal); + EXPECT_GT(absl::LogSeverityAtLeast::kError, absl::LogSeverity::kInfo); + + EXPECT_LE(absl::LogSeverityAtLeast::kInfo, absl::LogSeverity::kError); + EXPECT_GE(absl::LogSeverity::kError, absl::LogSeverityAtLeast::kInfo); +} + +TEST(LogThresholdTest, LogSeverityAtMostTest) { + EXPECT_GT(absl::LogSeverity::kError, absl::LogSeverityAtMost::kWarning); + EXPECT_LT(absl::LogSeverityAtMost::kError, absl::LogSeverity::kFatal); + + EXPECT_GE(absl::LogSeverityAtMost::kFatal, absl::LogSeverity::kError); + EXPECT_LE(absl::LogSeverity::kWarning, absl::LogSeverityAtMost::kError); +} + +TEST(LogThresholdTest, Extremes) { + EXPECT_LT(absl::LogSeverity::kFatal, absl::LogSeverityAtLeast::kInfinity); + EXPECT_GT(absl::LogSeverity::kInfo, + absl::LogSeverityAtMost::kNegativeInfinity); +} + +TEST(LogThresholdTest, Output) { + EXPECT_THAT(StreamHelper(absl::LogSeverityAtLeast::kInfo), Eq(">=INFO")); + EXPECT_THAT(StreamHelper(absl::LogSeverityAtLeast::kWarning), + Eq(">=WARNING")); + EXPECT_THAT(StreamHelper(absl::LogSeverityAtLeast::kError), Eq(">=ERROR")); + EXPECT_THAT(StreamHelper(absl::LogSeverityAtLeast::kFatal), Eq(">=FATAL")); + EXPECT_THAT(StreamHelper(absl::LogSeverityAtLeast::kInfinity), + Eq("INFINITY")); + + EXPECT_THAT(StreamHelper(absl::LogSeverityAtMost::kInfo), Eq("<=INFO")); + EXPECT_THAT(StreamHelper(absl::LogSeverityAtMost::kWarning), Eq("<=WARNING")); + EXPECT_THAT(StreamHelper(absl::LogSeverityAtMost::kError), Eq("<=ERROR")); + EXPECT_THAT(StreamHelper(absl::LogSeverityAtMost::kFatal), Eq("<=FATAL")); + EXPECT_THAT(StreamHelper(absl::LogSeverityAtMost::kNegativeInfinity), + Eq("NEGATIVE_INFINITY")); +} + } // namespace diff --git a/absl/base/optimization.h b/absl/base/optimization.h index d090be12..db5cc097 100644 --- a/absl/base/optimization.h +++ b/absl/base/optimization.h @@ -181,35 +181,43 @@ #define ABSL_PREDICT_TRUE(x) (x) #endif -// ABSL_INTERNAL_ASSUME(cond) +// ABSL_ASSUME(cond) +// // Informs the compiler that a condition is always true and that it can assume -// it to be true for optimization purposes. The call has undefined behavior if -// the condition is false. +// it to be true for optimization purposes. +// +// WARNING: If the condition is false, the program can produce undefined and +// potentially dangerous behavior. +// // In !NDEBUG mode, the condition is checked with an assert(). -// NOTE: The expression must not have side effects, as it will only be evaluated -// in some compilation modes and not others. +// +// NOTE: The expression must not have side effects, as it may only be evaluated +// in some compilation modes and not others. Some compilers may issue a warning +// if the compiler cannot prove the expression has no side effects. For example, +// the expression should not use a function call since the compiler cannot prove +// that a function call does not have side effects. // // Example: // // int x = ...; -// ABSL_INTERNAL_ASSUME(x >= 0); +// ABSL_ASSUME(x >= 0); // // The compiler can optimize the division to a simple right shift using the // // assumption specified above. // int y = x / 16; // #if !defined(NDEBUG) -#define ABSL_INTERNAL_ASSUME(cond) assert(cond) +#define ABSL_ASSUME(cond) assert(cond) #elif ABSL_HAVE_BUILTIN(__builtin_assume) -#define ABSL_INTERNAL_ASSUME(cond) __builtin_assume(cond) +#define ABSL_ASSUME(cond) __builtin_assume(cond) #elif defined(__GNUC__) || ABSL_HAVE_BUILTIN(__builtin_unreachable) -#define ABSL_INTERNAL_ASSUME(cond) \ +#define ABSL_ASSUME(cond) \ do { \ if (!(cond)) __builtin_unreachable(); \ } while (0) #elif defined(_MSC_VER) -#define ABSL_INTERNAL_ASSUME(cond) __assume(cond) +#define ABSL_ASSUME(cond) __assume(cond) #else -#define ABSL_INTERNAL_ASSUME(cond) \ +#define ABSL_ASSUME(cond) \ do { \ static_cast<void>(false && (cond)); \ } while (0) diff --git a/absl/base/options.h b/absl/base/options.h index eca879af..bc598470 100644 --- a/absl/base/options.h +++ b/absl/base/options.h @@ -206,7 +206,7 @@ // allowed. #define ABSL_OPTION_USE_INLINE_NAMESPACE 1 -#define ABSL_OPTION_INLINE_NAMESPACE_NAME lts_20210324 +#define ABSL_OPTION_INLINE_NAMESPACE_NAME lts_20220623 // ABSL_OPTION_HARDENED // diff --git a/absl/base/thread_annotations.h b/absl/base/thread_annotations.h index 9695f6de..bc8a6203 100644 --- a/absl/base/thread_annotations.h +++ b/absl/base/thread_annotations.h @@ -154,8 +154,8 @@ // ABSL_LOCKS_EXCLUDED() // -// Documents the locks acquired in the body of the function. These locks -// cannot be held when calling this function (as Abseil's `Mutex` locks are +// Documents the locks that cannot be held by callers of this function, as they +// might be acquired by this function (Abseil's `Mutex` locks are // non-reentrant). #if ABSL_HAVE_ATTRIBUTE(locks_excluded) #define ABSL_LOCKS_EXCLUDED(...) __attribute__((locks_excluded(__VA_ARGS__))) diff --git a/absl/cleanup/BUILD.bazel b/absl/cleanup/BUILD.bazel index 5cca898f..2154d9f1 100644 --- a/absl/cleanup/BUILD.bazel +++ b/absl/cleanup/BUILD.bazel @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") load( "//absl:copts/configure_copts.bzl", "ABSL_DEFAULT_COPTS", diff --git a/absl/cleanup/CMakeLists.txt b/absl/cleanup/CMakeLists.txt index a2dd78a8..f5af40b4 100644 --- a/absl/cleanup/CMakeLists.txt +++ b/absl/cleanup/CMakeLists.txt @@ -12,6 +12,7 @@ # See the License for the specific language governing permissions and # limitations under the License. +# Internal-only target, do not depend on directly. absl_cc_library( NAME cleanup_internal @@ -51,5 +52,5 @@ absl_cc_test( absl::cleanup absl::config absl::utility - gmock_main + GTest::gmock_main ) diff --git a/absl/cleanup/cleanup.h b/absl/cleanup/cleanup.h index 61b53d55..960ccd08 100644 --- a/absl/cleanup/cleanup.h +++ b/absl/cleanup/cleanup.h @@ -86,25 +86,25 @@ class ABSL_MUST_USE_RESULT Cleanup final { "Callbacks that return values are not supported."); public: - Cleanup(Callback callback) // NOLINT - : storage_(std::move(callback), /* is_callback_engaged = */ true) {} + Cleanup(Callback callback) : storage_(std::move(callback)) {} // NOLINT Cleanup(Cleanup&& other) = default; void Cancel() && { ABSL_HARDENING_ASSERT(storage_.IsCallbackEngaged()); - storage_.DisengageCallback(); + storage_.DestroyCallback(); } void Invoke() && { ABSL_HARDENING_ASSERT(storage_.IsCallbackEngaged()); - storage_.DisengageCallback(); storage_.InvokeCallback(); + storage_.DestroyCallback(); } ~Cleanup() { if (storage_.IsCallbackEngaged()) { storage_.InvokeCallback(); + storage_.DestroyCallback(); } } diff --git a/absl/cleanup/cleanup_test.cc b/absl/cleanup/cleanup_test.cc index 792595d6..46b88589 100644 --- a/absl/cleanup/cleanup_test.cc +++ b/absl/cleanup/cleanup_test.cc @@ -264,4 +264,48 @@ TYPED_TEST(CleanupTest, Move) { EXPECT_FALSE(called); // Destructor shouldn't invoke the callback } +int DestructionCount = 0; + +struct DestructionCounter { + void operator()() {} + + ~DestructionCounter() { ++DestructionCount; } +}; + +TYPED_TEST(CleanupTest, DestructorDestroys) { + { + auto cleanup = + absl::MakeCleanup(TypeParam::AsCallback(DestructionCounter())); + DestructionCount = 0; + } + + EXPECT_EQ(DestructionCount, 1); // Engaged cleanup destroys +} + +TYPED_TEST(CleanupTest, CancelDestroys) { + { + auto cleanup = + absl::MakeCleanup(TypeParam::AsCallback(DestructionCounter())); + DestructionCount = 0; + + std::move(cleanup).Cancel(); + EXPECT_EQ(DestructionCount, 1); // Cancel destroys + } + + EXPECT_EQ(DestructionCount, 1); // Canceled cleanup does not double destroy +} + +TYPED_TEST(CleanupTest, InvokeDestroys) { + { + auto cleanup = + absl::MakeCleanup(TypeParam::AsCallback(DestructionCounter())); + DestructionCount = 0; + + std::move(cleanup).Invoke(); + EXPECT_EQ(DestructionCount, 1); // Invoke destroys + } + + EXPECT_EQ(DestructionCount, 1); // Invoked cleanup does not double destroy +} + } // namespace diff --git a/absl/cleanup/internal/cleanup.h b/absl/cleanup/internal/cleanup.h index b4c40737..2783fcb7 100644 --- a/absl/cleanup/internal/cleanup.h +++ b/absl/cleanup/internal/cleanup.h @@ -15,10 +15,12 @@ #ifndef ABSL_CLEANUP_INTERNAL_CLEANUP_H_ #define ABSL_CLEANUP_INTERNAL_CLEANUP_H_ +#include <new> #include <type_traits> #include <utility> #include "absl/base/internal/invoke.h" +#include "absl/base/macros.h" #include "absl/base/thread_annotations.h" #include "absl/utility/utility.h" @@ -45,14 +47,22 @@ class Storage { public: Storage() = delete; - Storage(Callback callback, bool is_callback_engaged) - : callback_(std::move(callback)), - is_callback_engaged_(is_callback_engaged) {} + explicit Storage(Callback callback) { + // Placement-new into a character buffer is used for eager destruction when + // the cleanup is invoked or cancelled. To ensure this optimizes well, the + // behavior is implemented locally instead of using an absl::optional. + ::new (GetCallbackBuffer()) Callback(std::move(callback)); + is_callback_engaged_ = true; + } + + Storage(Storage&& other) { + ABSL_HARDENING_ASSERT(other.IsCallbackEngaged()); - Storage(Storage&& other) - : callback_(std::move(other.callback_)), - is_callback_engaged_( - absl::exchange(other.is_callback_engaged_, false)) {} + ::new (GetCallbackBuffer()) Callback(std::move(other.GetCallback())); + is_callback_engaged_ = true; + + other.DestroyCallback(); + } Storage(const Storage& other) = delete; @@ -60,17 +70,26 @@ class Storage { Storage& operator=(const Storage& other) = delete; + void* GetCallbackBuffer() { return static_cast<void*>(+callback_buffer_); } + + Callback& GetCallback() { + return *reinterpret_cast<Callback*>(GetCallbackBuffer()); + } + bool IsCallbackEngaged() const { return is_callback_engaged_; } - void DisengageCallback() { is_callback_engaged_ = false; } + void DestroyCallback() { + is_callback_engaged_ = false; + GetCallback().~Callback(); + } void InvokeCallback() ABSL_NO_THREAD_SAFETY_ANALYSIS { - std::move(callback_)(); + std::move(GetCallback())(); } private: - Callback callback_; bool is_callback_engaged_; + alignas(Callback) char callback_buffer_[sizeof(Callback)]; }; } // namespace cleanup_internal diff --git a/absl/container/BUILD.bazel b/absl/container/BUILD.bazel index f22fdc60..d01d78e5 100644 --- a/absl/container/BUILD.bazel +++ b/absl/container/BUILD.bazel @@ -14,7 +14,6 @@ # limitations under the License. # -load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test") load( "//absl:copts/configure_copts.bzl", "ABSL_DEFAULT_COPTS", @@ -218,11 +217,6 @@ cc_test( ], ) -NOTEST_TAGS_NONMOBILE = [ - "no_test_darwin_x86_64", - "no_test_loonix", -] - NOTEST_TAGS_MOBILE = [ "no_test_android_arm", "no_test_android_arm64", @@ -230,8 +224,6 @@ NOTEST_TAGS_MOBILE = [ "no_test_ios_x86_64", ] -NOTEST_TAGS = NOTEST_TAGS_MOBILE + NOTEST_TAGS_NONMOBILE - cc_library( name = "flat_hash_map", hdrs = ["flat_hash_map.h"], @@ -242,6 +234,7 @@ cc_library( ":hash_function_defaults", ":raw_hash_map", "//absl/algorithm:container", + "//absl/base:core_headers", "//absl/memory", ], ) @@ -251,7 +244,7 @@ cc_test( srcs = ["flat_hash_map_test.cc"], copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, - tags = NOTEST_TAGS_NONMOBILE, + tags = ["no_test_loonix"], deps = [ ":flat_hash_map", ":hash_generator_testing", @@ -285,7 +278,7 @@ cc_test( srcs = ["flat_hash_set_test.cc"], copts = ABSL_TEST_COPTS + ["-DUNORDERED_SET_CXX17"], linkopts = ABSL_DEFAULT_LINKOPTS, - tags = NOTEST_TAGS_NONMOBILE, + tags = ["no_test_loonix"], deps = [ ":flat_hash_set", ":hash_generator_testing", @@ -308,9 +301,10 @@ cc_library( deps = [ ":container_memory", ":hash_function_defaults", - ":node_hash_policy", + ":node_slot_policy", ":raw_hash_map", "//absl/algorithm:container", + "//absl/base:core_headers", "//absl/memory", ], ) @@ -320,7 +314,7 @@ cc_test( srcs = ["node_hash_map_test.cc"], copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, - tags = NOTEST_TAGS_NONMOBILE, + tags = ["no_test_loonix"], deps = [ ":hash_generator_testing", ":node_hash_map", @@ -340,9 +334,10 @@ cc_library( linkopts = ABSL_DEFAULT_LINKOPTS, deps = [ ":hash_function_defaults", - ":node_hash_policy", + ":node_slot_policy", ":raw_hash_set", "//absl/algorithm:container", + "//absl/base:core_headers", "//absl/memory", ], ) @@ -352,7 +347,7 @@ cc_test( srcs = ["node_hash_set_test.cc"], copts = ABSL_TEST_COPTS + ["-DUNORDERED_SET_CXX17"], linkopts = ABSL_DEFAULT_LINKOPTS, - tags = NOTEST_TAGS_NONMOBILE, + tags = ["no_test_loonix"], deps = [ ":node_hash_set", ":unordered_set_constructor_test", @@ -381,7 +376,7 @@ cc_test( srcs = ["internal/container_memory_test.cc"], copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, - tags = NOTEST_TAGS_NONMOBILE, + tags = ["no_test_loonix"], deps = [ ":container_memory", ":test_instance_tracker", @@ -408,7 +403,7 @@ cc_test( srcs = ["internal/hash_function_defaults_test.cc"], copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, - tags = NOTEST_TAGS, + tags = NOTEST_TAGS_MOBILE + ["no_test_loonix"], deps = [ ":hash_function_defaults", "//absl/hash", @@ -507,12 +502,13 @@ cc_library( copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, deps = [ - ":have_sse", "//absl/base", + "//absl/base:config", "//absl/base:core_headers", - "//absl/base:exponential_biased", "//absl/debugging:stacktrace", "//absl/memory", + "//absl/profiling:exponential_biased", + "//absl/profiling:sample_recorder", "//absl/synchronization", "//absl/utility", ], @@ -522,10 +518,14 @@ cc_test( name = "hashtablez_sampler_test", srcs = ["internal/hashtablez_sampler_test.cc"], linkopts = ABSL_DEFAULT_LINKOPTS, + tags = [ + "no_test_wasm", + ], deps = [ ":hashtablez_sampler", - ":have_sse", + "//absl/base:config", "//absl/base:core_headers", + "//absl/profiling:sample_recorder", "//absl/synchronization", "//absl/synchronization:thread_pool", "//absl/time", @@ -534,21 +534,21 @@ cc_test( ) cc_library( - name = "node_hash_policy", - hdrs = ["internal/node_hash_policy.h"], + name = "node_slot_policy", + hdrs = ["internal/node_slot_policy.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, deps = ["//absl/base:config"], ) cc_test( - name = "node_hash_policy_test", - srcs = ["internal/node_hash_policy_test.cc"], + name = "node_slot_policy_test", + srcs = ["internal/node_slot_policy_test.cc"], copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, deps = [ ":hash_policy_traits", - ":node_hash_policy", + ":node_slot_policy", "@com_google_googletest//:gtest_main", ], ) @@ -566,14 +566,6 @@ cc_library( ) cc_library( - name = "have_sse", - hdrs = ["internal/have_sse.h"], - copts = ABSL_DEFAULT_COPTS, - linkopts = ABSL_DEFAULT_LINKOPTS, - visibility = ["//visibility:private"], -) - -cc_library( name = "common", hdrs = ["internal/common.h"], copts = ABSL_DEFAULT_COPTS, @@ -597,11 +589,10 @@ cc_library( ":hash_policy_traits", ":hashtable_debug_hooks", ":hashtablez_sampler", - ":have_sse", - ":layout", "//absl/base:config", "//absl/base:core_headers", "//absl/base:endian", + "//absl/base:prefetch", "//absl/memory", "//absl/meta:type_traits", "//absl/numeric:bits", @@ -614,7 +605,7 @@ cc_test( srcs = ["internal/raw_hash_set_test.cc"], copts = ABSL_TEST_COPTS, linkstatic = 1, - tags = NOTEST_TAGS, + tags = NOTEST_TAGS_MOBILE + ["no_test_loonix"], deps = [ ":container_memory", ":hash_function_defaults", @@ -624,6 +615,7 @@ cc_test( "//absl/base", "//absl/base:config", "//absl/base:core_headers", + "//absl/base:prefetch", "//absl/base:raw_logging_internal", "//absl/strings", "@com_google_googletest//:gtest_main", @@ -704,7 +696,7 @@ cc_test( srcs = ["internal/layout_test.cc"], copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, - tags = NOTEST_TAGS, + tags = NOTEST_TAGS_MOBILE + ["no_test_loonix"], visibility = ["//visibility:private"], deps = [ ":layout", @@ -851,7 +843,7 @@ cc_test( srcs = ["internal/unordered_set_test.cc"], copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, - tags = NOTEST_TAGS_NONMOBILE, + tags = ["no_test_loonix"], deps = [ ":unordered_set_constructor_test", ":unordered_set_lookup_test", @@ -866,7 +858,7 @@ cc_test( srcs = ["internal/unordered_map_test.cc"], copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, - tags = NOTEST_TAGS_NONMOBILE, + tags = ["no_test_loonix"], deps = [ ":unordered_map_constructor_test", ":unordered_map_lookup_test", @@ -876,6 +868,22 @@ cc_test( ], ) +cc_test( + name = "sample_element_size_test", + srcs = ["sample_element_size_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + tags = ["no_test_loonix"], + visibility = ["//visibility:private"], + deps = [ + ":flat_hash_map", + ":flat_hash_set", + ":node_hash_map", + ":node_hash_set", + "@com_google_googletest//:gtest_main", + ], +) + cc_library( name = "btree", srcs = [ @@ -895,6 +903,7 @@ cc_library( ":container_memory", ":layout", "//absl/base:core_headers", + "//absl/base:raw_logging_internal", "//absl/base:throw_delegate", "//absl/memory", "//absl/meta:type_traits", @@ -930,6 +939,10 @@ cc_test( copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, shard_count = 10, + tags = [ + "no_test_ios", + "no_test_wasm", + ], visibility = ["//visibility:private"], deps = [ ":btree", diff --git a/absl/container/CMakeLists.txt b/absl/container/CMakeLists.txt index 2d7d0e65..9b5c59a4 100644 --- a/absl/container/CMakeLists.txt +++ b/absl/container/CMakeLists.txt @@ -35,12 +35,14 @@ absl_cc_library( absl::core_headers absl::layout absl::memory + absl::raw_logging_internal absl::strings absl::throw_delegate absl::type_traits absl::utility ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME btree_test_common @@ -80,9 +82,10 @@ absl_cc_test( absl::strings absl::test_instance_tracker absl::type_traits - gmock_main + GTest::gmock_main ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME compressed_tuple @@ -109,7 +112,7 @@ absl_cc_test( absl::optional absl::test_instance_tracker absl::utility - gmock_main + GTest::gmock_main ) absl_cc_library( @@ -144,7 +147,7 @@ absl_cc_test( absl::exception_testing absl::hash_testing absl::memory - gmock_main + GTest::gmock_main ) absl_cc_test( @@ -158,9 +161,10 @@ absl_cc_test( absl::fixed_array absl::config absl::exception_safety_testing - gmock_main + GTest::gmock_main ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME inlined_vector_internal @@ -193,6 +197,7 @@ absl_cc_library( PUBLIC ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME counting_allocator @@ -222,7 +227,7 @@ absl_cc_test( absl::memory absl::raw_logging_internal absl::strings - gmock_main + GTest::gmock_main ) absl_cc_test( @@ -236,9 +241,10 @@ absl_cc_test( absl::inlined_vector absl::config absl::exception_safety_testing - gmock_main + GTest::gmock_main ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME test_instance_tracker @@ -262,7 +268,7 @@ absl_cc_test( ${ABSL_TEST_COPTS} DEPS absl::test_instance_tracker - gmock_main + GTest::gmock_main ) absl_cc_library( @@ -274,6 +280,7 @@ absl_cc_library( ${ABSL_DEFAULT_COPTS} DEPS absl::container_memory + absl::core_headers absl::hash_function_defaults absl::raw_hash_map absl::algorithm_container @@ -297,7 +304,7 @@ absl_cc_test( absl::unordered_map_modifiers_test absl::any absl::raw_logging_internal - gmock_main + GTest::gmock_main ) absl_cc_library( @@ -335,7 +342,7 @@ absl_cc_test( absl::memory absl::raw_logging_internal absl::strings - gmock_main + GTest::gmock_main ) absl_cc_library( @@ -347,8 +354,9 @@ absl_cc_library( ${ABSL_DEFAULT_COPTS} DEPS absl::container_memory + absl::core_headers absl::hash_function_defaults - absl::node_hash_policy + absl::node_slot_policy absl::raw_hash_map absl::algorithm_container absl::memory @@ -370,7 +378,7 @@ absl_cc_test( absl::unordered_map_lookup_test absl::unordered_map_members_test absl::unordered_map_modifiers_test - gmock_main + GTest::gmock_main ) absl_cc_library( @@ -381,8 +389,9 @@ absl_cc_library( COPTS ${ABSL_DEFAULT_COPTS} DEPS + absl::core_headers absl::hash_function_defaults - absl::node_hash_policy + absl::node_slot_policy absl::raw_hash_set absl::algorithm_container absl::memory @@ -404,9 +413,10 @@ absl_cc_test( absl::unordered_set_lookup_test absl::unordered_set_members_test absl::unordered_set_modifiers_test - gmock_main + GTest::gmock_main ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME container_memory @@ -433,9 +443,10 @@ absl_cc_test( absl::container_memory absl::strings absl::test_instance_tracker - gmock_main + GTest::gmock_main ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME hash_function_defaults @@ -465,9 +476,10 @@ absl_cc_test( absl::hash absl::random_random absl::strings - gmock_main + GTest::gmock_main ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME hash_generator_testing @@ -485,6 +497,7 @@ absl_cc_library( TESTONLY ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME hash_policy_testing @@ -507,9 +520,10 @@ absl_cc_test( ${ABSL_TEST_COPTS} DEPS absl::hash_policy_testing - gmock_main + GTest::gmock_main ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME hash_policy_traits @@ -531,9 +545,10 @@ absl_cc_test( ${ABSL_TEST_COPTS} DEPS absl::hash_policy_traits - gmock_main + GTest::gmock_main ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME hashtablez_sampler @@ -546,8 +561,9 @@ absl_cc_library( ${ABSL_DEFAULT_COPTS} DEPS absl::base + absl::config absl::exponential_biased - absl::have_sse + absl::sample_recorder absl::synchronization ) @@ -559,11 +575,12 @@ absl_cc_test( COPTS ${ABSL_TEST_COPTS} DEPS + absl::config absl::hashtablez_sampler - absl::have_sse - gmock_main + GTest::gmock_main ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME hashtable_debug @@ -575,6 +592,7 @@ absl_cc_library( absl::hashtable_debug_hooks ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME hashtable_debug_hooks @@ -587,20 +605,12 @@ absl_cc_library( PUBLIC ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME - have_sse - HDRS - "internal/have_sse.h" - COPTS - ${ABSL_DEFAULT_COPTS} -) - -absl_cc_library( - NAME - node_hash_policy + node_slot_policy HDRS - "internal/node_hash_policy.h" + "internal/node_slot_policy.h" COPTS ${ABSL_DEFAULT_COPTS} DEPS @@ -610,17 +620,18 @@ absl_cc_library( absl_cc_test( NAME - node_hash_policy_test + node_slot_policy_test SRCS - "internal/node_hash_policy_test.cc" + "internal/node_slot_policy_test.cc" COPTS ${ABSL_TEST_COPTS} DEPS absl::hash_policy_traits - absl::node_hash_policy - gmock_main + absl::node_slot_policy + GTest::gmock_main ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME raw_hash_map @@ -635,6 +646,7 @@ absl_cc_library( PUBLIC ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME container_common @@ -646,6 +658,7 @@ absl_cc_library( absl::type_traits ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME raw_hash_set @@ -665,11 +678,10 @@ absl_cc_library( absl::endian absl::hash_policy_traits absl::hashtable_debug_hooks - absl::have_sse - absl::layout absl::memory absl::meta absl::optional + absl::prefetch absl::utility absl::hashtablez_sampler PUBLIC @@ -691,9 +703,10 @@ absl_cc_test( absl::base absl::config absl::core_headers + absl::prefetch absl::raw_logging_internal absl::strings - gmock_main + GTest::gmock_main ) absl_cc_test( @@ -707,9 +720,10 @@ absl_cc_test( absl::raw_hash_set absl::tracked absl::core_headers - gmock_main + GTest::gmock_main ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME layout @@ -740,9 +754,10 @@ absl_cc_test( absl::core_headers absl::raw_logging_internal absl::span - gmock_main + GTest::gmock_main ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME tracked @@ -755,6 +770,7 @@ absl_cc_library( TESTONLY ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME unordered_map_constructor_test @@ -765,10 +781,11 @@ absl_cc_library( DEPS absl::hash_generator_testing absl::hash_policy_testing - gmock + GTest::gmock TESTONLY ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME unordered_map_lookup_test @@ -779,10 +796,11 @@ absl_cc_library( DEPS absl::hash_generator_testing absl::hash_policy_testing - gmock + GTest::gmock TESTONLY ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME unordered_map_members_test @@ -792,10 +810,11 @@ absl_cc_library( ${ABSL_TEST_COPTS} DEPS absl::type_traits - gmock + GTest::gmock TESTONLY ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME unordered_map_modifiers_test @@ -806,10 +825,11 @@ absl_cc_library( DEPS absl::hash_generator_testing absl::hash_policy_testing - gmock + GTest::gmock TESTONLY ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME unordered_set_constructor_test @@ -820,10 +840,11 @@ absl_cc_library( DEPS absl::hash_generator_testing absl::hash_policy_testing - gmock + GTest::gmock TESTONLY ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME unordered_set_lookup_test @@ -834,10 +855,11 @@ absl_cc_library( DEPS absl::hash_generator_testing absl::hash_policy_testing - gmock + GTest::gmock TESTONLY ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME unordered_set_members_test @@ -847,10 +869,11 @@ absl_cc_library( ${ABSL_TEST_COPTS} DEPS absl::type_traits - gmock + GTest::gmock TESTONLY ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME unordered_set_modifiers_test @@ -861,7 +884,7 @@ absl_cc_library( DEPS absl::hash_generator_testing absl::hash_policy_testing - gmock + GTest::gmock TESTONLY ) @@ -877,7 +900,7 @@ absl_cc_test( absl::unordered_set_lookup_test absl::unordered_set_members_test absl::unordered_set_modifiers_test - gmock_main + GTest::gmock_main ) absl_cc_test( @@ -892,5 +915,20 @@ absl_cc_test( absl::unordered_map_lookup_test absl::unordered_map_members_test absl::unordered_map_modifiers_test - gmock_main + GTest::gmock_main +) + +absl_cc_test( + NAME + sample_element_size_test + SRCS + "sample_element_size_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::flat_hash_map + absl::flat_hash_set + absl::node_hash_map + absl::node_hash_set + GTest::gmock_main ) diff --git a/absl/container/btree_benchmark.cc b/absl/container/btree_benchmark.cc index 65b6790b..0ca497c8 100644 --- a/absl/container/btree_benchmark.cc +++ b/absl/container/btree_benchmark.cc @@ -153,9 +153,9 @@ void BM_FullLookup(benchmark::State& state) { BM_LookupImpl<T>(state, true); } -// Benchmark deletion of values from a container. +// Benchmark erasing values from a container. template <typename T> -void BM_Delete(benchmark::State& state) { +void BM_Erase(benchmark::State& state) { using V = typename remove_pair_const<typename T::value_type>::type; typename KeyOfValue<typename T::key_type, V>::type key_of_value; std::vector<V> values = GenerateValues<V>(kBenchmarkValues); @@ -180,9 +180,9 @@ void BM_Delete(benchmark::State& state) { } } -// Benchmark deletion of multiple values from a container. +// Benchmark erasing multiple values from a container. template <typename T> -void BM_DeleteRange(benchmark::State& state) { +void BM_EraseRange(benchmark::State& state) { using V = typename remove_pair_const<typename T::value_type>::type; typename KeyOfValue<typename T::key_type, V>::type key_of_value; std::vector<V> values = GenerateValues<V>(kBenchmarkValues); @@ -222,6 +222,40 @@ void BM_DeleteRange(benchmark::State& state) { } } +// Predicate that erases every other element. We can't use a lambda because +// C++11 doesn't support generic lambdas. +// TODO(b/207389011): consider adding benchmarks that remove different fractions +// of keys (e.g. 10%, 90%). +struct EraseIfPred { + uint64_t i = 0; + template <typename T> + bool operator()(const T&) { + return ++i % 2; + } +}; + +// Benchmark erasing multiple values from a container with a predicate. +template <typename T> +void BM_EraseIf(benchmark::State& state) { + using V = typename remove_pair_const<typename T::value_type>::type; + std::vector<V> values = GenerateValues<V>(kBenchmarkValues); + + // Removes half of the keys per batch. + const int batch_size = (kBenchmarkValues + 1) / 2; + EraseIfPred pred; + while (state.KeepRunningBatch(batch_size)) { + state.PauseTiming(); + { + T container(values.begin(), values.end()); + state.ResumeTiming(); + erase_if(container, pred); + benchmark::DoNotOptimize(container); + state.PauseTiming(); + } + state.ResumeTiming(); + } +} + // Benchmark steady-state insert (into first half of range) and remove (from // second half of range), treating the container approximately like a queue with // log-time access for all elements. This benchmark does not test the case where @@ -477,14 +511,14 @@ BTREE_TYPES(Time); void BM_##type##_##func(benchmark::State& state) { BM_##func<type>(state); } \ BENCHMARK(BM_##type##_##func) -#define MY_BENCHMARK3(type) \ +#define MY_BENCHMARK3_STL(type) \ MY_BENCHMARK4(type, Insert); \ MY_BENCHMARK4(type, InsertSorted); \ MY_BENCHMARK4(type, InsertSmall); \ MY_BENCHMARK4(type, Lookup); \ MY_BENCHMARK4(type, FullLookup); \ - MY_BENCHMARK4(type, Delete); \ - MY_BENCHMARK4(type, DeleteRange); \ + MY_BENCHMARK4(type, Erase); \ + MY_BENCHMARK4(type, EraseRange); \ MY_BENCHMARK4(type, QueueAddRem); \ MY_BENCHMARK4(type, MixedAddRem); \ MY_BENCHMARK4(type, Fifo); \ @@ -492,9 +526,13 @@ BTREE_TYPES(Time); MY_BENCHMARK4(type, InsertRangeRandom); \ MY_BENCHMARK4(type, InsertRangeSorted) +#define MY_BENCHMARK3(type) \ + MY_BENCHMARK4(type, EraseIf); \ + MY_BENCHMARK3_STL(type) + #define MY_BENCHMARK2_SUPPORTS_MULTI_ONLY(type) \ - MY_BENCHMARK3(stl_##type); \ - MY_BENCHMARK3(stl_unordered_##type); \ + MY_BENCHMARK3_STL(stl_##type); \ + MY_BENCHMARK3_STL(stl_unordered_##type); \ MY_BENCHMARK3(btree_256_##type) #define MY_BENCHMARK2(type) \ @@ -684,12 +722,12 @@ double ContainerInfo(const btree_map<int, BigTypePtr<Size>>& b) { btree_set<BigTypePtr<SIZE>>; \ using btree_256_map_size##SIZE##copies##SIZE##ptr = \ btree_map<int, BigTypePtr<SIZE>>; \ - MY_BENCHMARK3(stl_set_size##SIZE##copies##SIZE##ptr); \ - MY_BENCHMARK3(stl_unordered_set_size##SIZE##copies##SIZE##ptr); \ + MY_BENCHMARK3_STL(stl_set_size##SIZE##copies##SIZE##ptr); \ + MY_BENCHMARK3_STL(stl_unordered_set_size##SIZE##copies##SIZE##ptr); \ MY_BENCHMARK3(flat_hash_set_size##SIZE##copies##SIZE##ptr); \ MY_BENCHMARK3(btree_256_set_size##SIZE##copies##SIZE##ptr); \ - MY_BENCHMARK3(stl_map_size##SIZE##copies##SIZE##ptr); \ - MY_BENCHMARK3(stl_unordered_map_size##SIZE##copies##SIZE##ptr); \ + MY_BENCHMARK3_STL(stl_map_size##SIZE##copies##SIZE##ptr); \ + MY_BENCHMARK3_STL(stl_unordered_map_size##SIZE##copies##SIZE##ptr); \ MY_BENCHMARK3(flat_hash_map_size##SIZE##copies##SIZE##ptr); \ MY_BENCHMARK3(btree_256_map_size##SIZE##copies##SIZE##ptr) diff --git a/absl/container/btree_map.h b/absl/container/btree_map.h index ea49d446..286817f1 100644 --- a/absl/container/btree_map.h +++ b/absl/container/btree_map.h @@ -35,14 +35,17 @@ // // However, these types should not be considered drop-in replacements for // `std::map` and `std::multimap` as there are some API differences, which are -// noted in this header file. +// noted in this header file. The most consequential differences with respect to +// migrating to b-tree from the STL types are listed in the next paragraph. +// Other API differences are minor. // // Importantly, insertions and deletions may invalidate outstanding iterators, // pointers, and references to elements. Such invalidations are typically only // an issue if insertion and deletion operations are interleaved with the use of // more than one iterator, pointer, or reference simultaneously. For this // reason, `insert()` and `erase()` return a valid iterator at the current -// position. +// position. Another important difference is that key-types must be +// copy-constructible. #ifndef ABSL_CONTAINER_BTREE_MAP_H_ #define ABSL_CONTAINER_BTREE_MAP_H_ @@ -53,6 +56,14 @@ namespace absl { ABSL_NAMESPACE_BEGIN +namespace container_internal { + +template <typename Key, typename Data, typename Compare, typename Alloc, + int TargetNodeSize, bool IsMulti> +struct map_params; + +} // namespace container_internal + // absl::btree_map<> // // An `absl::btree_map<K, V>` is an ordered associative container of @@ -74,7 +85,7 @@ class btree_map : public container_internal::btree_map_container< container_internal::btree<container_internal::map_params< Key, Value, Compare, Alloc, /*TargetNodeSize=*/256, - /*Multi=*/false>>> { + /*IsMulti=*/false>>> { using Base = typename btree_map::btree_map_container; public: @@ -366,8 +377,8 @@ class btree_map // Determines whether an element comparing equal to the given `key` exists // within the `btree_map`, returning `true` if so or `false` otherwise. // - // Supports heterogeneous lookup, provided that the map is provided a - // compatible heterogeneous comparator. + // Supports heterogeneous lookup, provided that the map has a compatible + // heterogeneous comparator. using Base::contains; // btree_map::count() @@ -378,8 +389,8 @@ class btree_map // the `btree_map`. Note that this function will return either `1` or `0` // since duplicate elements are not allowed within a `btree_map`. // - // Supports heterogeneous lookup, provided that the map is provided a - // compatible heterogeneous comparator. + // Supports heterogeneous lookup, provided that the map has a compatible + // heterogeneous comparator. using Base::count; // btree_map::equal_range() @@ -395,10 +406,34 @@ class btree_map // // Finds an element with the passed `key` within the `btree_map`. // - // Supports heterogeneous lookup, provided that the map is provided a - // compatible heterogeneous comparator. + // Supports heterogeneous lookup, provided that the map has a compatible + // heterogeneous comparator. using Base::find; + // btree_map::lower_bound() + // + // template <typename K> iterator lower_bound(const K& key): + // template <typename K> const_iterator lower_bound(const K& key) const: + // + // Finds the first element with a key that is not less than `key` within the + // `btree_map`. + // + // Supports heterogeneous lookup, provided that the map has a compatible + // heterogeneous comparator. + using Base::lower_bound; + + // btree_map::upper_bound() + // + // template <typename K> iterator upper_bound(const K& key): + // template <typename K> const_iterator upper_bound(const K& key) const: + // + // Finds the first element with a key that is greater than `key` within the + // `btree_map`. + // + // Supports heterogeneous lookup, provided that the map has a compatible + // heterogeneous comparator. + using Base::upper_bound; + // btree_map::operator[]() // // Returns a reference to the value mapped to the passed key within the @@ -443,15 +478,11 @@ void swap(btree_map<K, V, C, A> &x, btree_map<K, V, C, A> &y) { // absl::erase_if(absl::btree_map<>, Pred) // // Erases all elements that satisfy the predicate pred from the container. +// Returns the number of erased elements. template <typename K, typename V, typename C, typename A, typename Pred> -void erase_if(btree_map<K, V, C, A> &map, Pred pred) { - for (auto it = map.begin(); it != map.end();) { - if (pred(*it)) { - it = map.erase(it); - } else { - ++it; - } - } +typename btree_map<K, V, C, A>::size_type erase_if( + btree_map<K, V, C, A> &map, Pred pred) { + return container_internal::btree_access::erase_if(map, std::move(pred)); } // absl::btree_multimap @@ -476,7 +507,7 @@ class btree_multimap : public container_internal::btree_multimap_container< container_internal::btree<container_internal::map_params< Key, Value, Compare, Alloc, /*TargetNodeSize=*/256, - /*Multi=*/true>>> { + /*IsMulti=*/true>>> { using Base = typename btree_multimap::btree_multimap_container; public: @@ -669,9 +700,8 @@ class btree_multimap // btree_multimap::merge() // - // Extracts elements from a given `source` btree_multimap into this - // `btree_multimap`. If the destination `btree_multimap` already contains an - // element with an equivalent key, that element is not extracted. + // Extracts all elements from a given `source` btree_multimap into this + // `btree_multimap`. using Base::merge; // btree_multimap::swap(btree_multimap& other) @@ -691,8 +721,8 @@ class btree_multimap // Determines whether an element comparing equal to the given `key` exists // within the `btree_multimap`, returning `true` if so or `false` otherwise. // - // Supports heterogeneous lookup, provided that the map is provided a - // compatible heterogeneous comparator. + // Supports heterogeneous lookup, provided that the map has a compatible + // heterogeneous comparator. using Base::contains; // btree_multimap::count() @@ -702,8 +732,8 @@ class btree_multimap // Returns the number of elements comparing equal to the given `key` within // the `btree_multimap`. // - // Supports heterogeneous lookup, provided that the map is provided a - // compatible heterogeneous comparator. + // Supports heterogeneous lookup, provided that the map has a compatible + // heterogeneous comparator. using Base::count; // btree_multimap::equal_range() @@ -720,10 +750,34 @@ class btree_multimap // // Finds an element with the passed `key` within the `btree_multimap`. // - // Supports heterogeneous lookup, provided that the map is provided a - // compatible heterogeneous comparator. + // Supports heterogeneous lookup, provided that the map has a compatible + // heterogeneous comparator. using Base::find; + // btree_multimap::lower_bound() + // + // template <typename K> iterator lower_bound(const K& key): + // template <typename K> const_iterator lower_bound(const K& key) const: + // + // Finds the first element with a key that is not less than `key` within the + // `btree_multimap`. + // + // Supports heterogeneous lookup, provided that the map has a compatible + // heterogeneous comparator. + using Base::lower_bound; + + // btree_multimap::upper_bound() + // + // template <typename K> iterator upper_bound(const K& key): + // template <typename K> const_iterator upper_bound(const K& key) const: + // + // Finds the first element with a key that is greater than `key` within the + // `btree_multimap`. + // + // Supports heterogeneous lookup, provided that the map has a compatible + // heterogeneous comparator. + using Base::upper_bound; + // btree_multimap::get_allocator() // // Returns the allocator function associated with this `btree_multimap`. @@ -751,17 +805,46 @@ void swap(btree_multimap<K, V, C, A> &x, btree_multimap<K, V, C, A> &y) { // absl::erase_if(absl::btree_multimap<>, Pred) // // Erases all elements that satisfy the predicate pred from the container. +// Returns the number of erased elements. template <typename K, typename V, typename C, typename A, typename Pred> -void erase_if(btree_multimap<K, V, C, A> &map, Pred pred) { - for (auto it = map.begin(); it != map.end();) { - if (pred(*it)) { - it = map.erase(it); - } else { - ++it; - } - } +typename btree_multimap<K, V, C, A>::size_type erase_if( + btree_multimap<K, V, C, A> &map, Pred pred) { + return container_internal::btree_access::erase_if(map, std::move(pred)); } +namespace container_internal { + +// A parameters structure for holding the type parameters for a btree_map. +// Compare and Alloc should be nothrow copy-constructible. +template <typename Key, typename Data, typename Compare, typename Alloc, + int TargetNodeSize, bool IsMulti> +struct map_params : common_params<Key, Compare, Alloc, TargetNodeSize, IsMulti, + /*IsMap=*/true, map_slot_policy<Key, Data>> { + using super_type = typename map_params::common_params; + using mapped_type = Data; + // This type allows us to move keys when it is safe to do so. It is safe + // for maps in which value_type and mutable_value_type are layout compatible. + using slot_policy = typename super_type::slot_policy; + using slot_type = typename super_type::slot_type; + using value_type = typename super_type::value_type; + using init_type = typename super_type::init_type; + + template <typename V> + static auto key(const V &value) -> decltype(value.first) { + return value.first; + } + static const Key &key(const slot_type *s) { return slot_policy::key(s); } + static const Key &key(slot_type *s) { return slot_policy::key(s); } + // For use in node handle. + static auto mutable_key(slot_type *s) + -> decltype(slot_policy::mutable_key(s)) { + return slot_policy::mutable_key(s); + } + static mapped_type &value(value_type *value) { return value->second; } +}; + +} // namespace container_internal + ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/container/btree_set.h b/absl/container/btree_set.h index 21ef0a03..695b09f5 100644 --- a/absl/container/btree_set.h +++ b/absl/container/btree_set.h @@ -35,7 +35,9 @@ // // However, these types should not be considered drop-in replacements for // `std::set` and `std::multiset` as there are some API differences, which are -// noted in this header file. +// noted in this header file. The most consequential differences with respect to +// migrating to b-tree from the STL types are listed in the next paragraph. +// Other API differences are minor. // // Importantly, insertions and deletions may invalidate outstanding iterators, // pointers, and references to elements. Such invalidations are typically only @@ -53,6 +55,17 @@ namespace absl { ABSL_NAMESPACE_BEGIN +namespace container_internal { + +template <typename Key> +struct set_slot_policy; + +template <typename Key, typename Compare, typename Alloc, int TargetNodeSize, + bool IsMulti> +struct set_params; + +} // namespace container_internal + // absl::btree_set<> // // An `absl::btree_set<K>` is an ordered associative container of unique key @@ -74,7 +87,7 @@ class btree_set : public container_internal::btree_set_container< container_internal::btree<container_internal::set_params< Key, Compare, Alloc, /*TargetNodeSize=*/256, - /*Multi=*/false>>> { + /*IsMulti=*/false>>> { using Base = typename btree_set::btree_set_container; public: @@ -300,8 +313,8 @@ class btree_set // Determines whether an element comparing equal to the given `key` exists // within the `btree_set`, returning `true` if so or `false` otherwise. // - // Supports heterogeneous lookup, provided that the set is provided a - // compatible heterogeneous comparator. + // Supports heterogeneous lookup, provided that the set has a compatible + // heterogeneous comparator. using Base::contains; // btree_set::count() @@ -312,8 +325,8 @@ class btree_set // the `btree_set`. Note that this function will return either `1` or `0` // since duplicate elements are not allowed within a `btree_set`. // - // Supports heterogeneous lookup, provided that the set is provided a - // compatible heterogeneous comparator. + // Supports heterogeneous lookup, provided that the set has a compatible + // heterogeneous comparator. using Base::count; // btree_set::equal_range() @@ -330,10 +343,32 @@ class btree_set // // Finds an element with the passed `key` within the `btree_set`. // - // Supports heterogeneous lookup, provided that the set is provided a - // compatible heterogeneous comparator. + // Supports heterogeneous lookup, provided that the set has a compatible + // heterogeneous comparator. using Base::find; + // btree_set::lower_bound() + // + // template <typename K> iterator lower_bound(const K& key): + // template <typename K> const_iterator lower_bound(const K& key) const: + // + // Finds the first element that is not less than `key` within the `btree_set`. + // + // Supports heterogeneous lookup, provided that the set has a compatible + // heterogeneous comparator. + using Base::lower_bound; + + // btree_set::upper_bound() + // + // template <typename K> iterator upper_bound(const K& key): + // template <typename K> const_iterator upper_bound(const K& key) const: + // + // Finds the first element that is greater than `key` within the `btree_set`. + // + // Supports heterogeneous lookup, provided that the set has a compatible + // heterogeneous comparator. + using Base::upper_bound; + // btree_set::get_allocator() // // Returns the allocator function associated with this `btree_set`. @@ -363,15 +398,11 @@ void swap(btree_set<K, C, A> &x, btree_set<K, C, A> &y) { // absl::erase_if(absl::btree_set<>, Pred) // // Erases all elements that satisfy the predicate pred from the container. +// Returns the number of erased elements. template <typename K, typename C, typename A, typename Pred> -void erase_if(btree_set<K, C, A> &set, Pred pred) { - for (auto it = set.begin(); it != set.end();) { - if (pred(*it)) { - it = set.erase(it); - } else { - ++it; - } - } +typename btree_set<K, C, A>::size_type erase_if(btree_set<K, C, A> &set, + Pred pred) { + return container_internal::btree_access::erase_if(set, std::move(pred)); } // absl::btree_multiset<> @@ -396,7 +427,7 @@ class btree_multiset : public container_internal::btree_multiset_container< container_internal::btree<container_internal::set_params< Key, Compare, Alloc, /*TargetNodeSize=*/256, - /*Multi=*/true>>> { + /*IsMulti=*/true>>> { using Base = typename btree_multiset::btree_multiset_container; public: @@ -582,9 +613,8 @@ class btree_multiset // btree_multiset::merge() // - // Extracts elements from a given `source` btree_multiset into this - // `btree_multiset`. If the destination `btree_multiset` already contains an - // element with an equivalent key, that element is not extracted. + // Extracts all elements from a given `source` btree_multiset into this + // `btree_multiset`. using Base::merge; // btree_multiset::swap(btree_multiset& other) @@ -604,8 +634,8 @@ class btree_multiset // Determines whether an element comparing equal to the given `key` exists // within the `btree_multiset`, returning `true` if so or `false` otherwise. // - // Supports heterogeneous lookup, provided that the set is provided a - // compatible heterogeneous comparator. + // Supports heterogeneous lookup, provided that the set has a compatible + // heterogeneous comparator. using Base::contains; // btree_multiset::count() @@ -615,8 +645,8 @@ class btree_multiset // Returns the number of elements comparing equal to the given `key` within // the `btree_multiset`. // - // Supports heterogeneous lookup, provided that the set is provided a - // compatible heterogeneous comparator. + // Supports heterogeneous lookup, provided that the set has a compatible + // heterogeneous comparator. using Base::count; // btree_multiset::equal_range() @@ -633,10 +663,34 @@ class btree_multiset // // Finds an element with the passed `key` within the `btree_multiset`. // - // Supports heterogeneous lookup, provided that the set is provided a - // compatible heterogeneous comparator. + // Supports heterogeneous lookup, provided that the set has a compatible + // heterogeneous comparator. using Base::find; + // btree_multiset::lower_bound() + // + // template <typename K> iterator lower_bound(const K& key): + // template <typename K> const_iterator lower_bound(const K& key) const: + // + // Finds the first element that is not less than `key` within the + // `btree_multiset`. + // + // Supports heterogeneous lookup, provided that the set has a compatible + // heterogeneous comparator. + using Base::lower_bound; + + // btree_multiset::upper_bound() + // + // template <typename K> iterator upper_bound(const K& key): + // template <typename K> const_iterator upper_bound(const K& key) const: + // + // Finds the first element that is greater than `key` within the + // `btree_multiset`. + // + // Supports heterogeneous lookup, provided that the set has a compatible + // heterogeneous comparator. + using Base::upper_bound; + // btree_multiset::get_allocator() // // Returns the allocator function associated with this `btree_multiset`. @@ -666,17 +720,73 @@ void swap(btree_multiset<K, C, A> &x, btree_multiset<K, C, A> &y) { // absl::erase_if(absl::btree_multiset<>, Pred) // // Erases all elements that satisfy the predicate pred from the container. +// Returns the number of erased elements. template <typename K, typename C, typename A, typename Pred> -void erase_if(btree_multiset<K, C, A> &set, Pred pred) { - for (auto it = set.begin(); it != set.end();) { - if (pred(*it)) { - it = set.erase(it); - } else { - ++it; - } - } +typename btree_multiset<K, C, A>::size_type erase_if( + btree_multiset<K, C, A> & set, Pred pred) { + return container_internal::btree_access::erase_if(set, std::move(pred)); } +namespace container_internal { + +// This type implements the necessary functions from the +// absl::container_internal::slot_type interface for btree_(multi)set. +template <typename Key> +struct set_slot_policy { + using slot_type = Key; + using value_type = Key; + using mutable_value_type = Key; + + static value_type &element(slot_type *slot) { return *slot; } + static const value_type &element(const slot_type *slot) { return *slot; } + + template <typename Alloc, class... Args> + static void construct(Alloc *alloc, slot_type *slot, Args &&...args) { + absl::allocator_traits<Alloc>::construct(*alloc, slot, + std::forward<Args>(args)...); + } + + template <typename Alloc> + static void construct(Alloc *alloc, slot_type *slot, slot_type *other) { + absl::allocator_traits<Alloc>::construct(*alloc, slot, std::move(*other)); + } + + template <typename Alloc> + static void construct(Alloc *alloc, slot_type *slot, const slot_type *other) { + absl::allocator_traits<Alloc>::construct(*alloc, slot, *other); + } + + template <typename Alloc> + static void destroy(Alloc *alloc, slot_type *slot) { + absl::allocator_traits<Alloc>::destroy(*alloc, slot); + } + + template <typename Alloc> + static void transfer(Alloc *alloc, slot_type *new_slot, slot_type *old_slot) { + construct(alloc, new_slot, old_slot); + destroy(alloc, old_slot); + } +}; + +// A parameters structure for holding the type parameters for a btree_set. +// Compare and Alloc should be nothrow copy-constructible. +template <typename Key, typename Compare, typename Alloc, int TargetNodeSize, + bool IsMulti> +struct set_params : common_params<Key, Compare, Alloc, TargetNodeSize, IsMulti, + /*IsMap=*/false, set_slot_policy<Key>> { + using value_type = Key; + using slot_type = typename set_params::common_params::slot_type; + + template <typename V> + static const V &key(const V &value) { + return value; + } + static const Key &key(const slot_type *slot) { return *slot; } + static const Key &key(slot_type *slot) { return *slot; } +}; + +} // namespace container_internal + ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/container/btree_test.cc b/absl/container/btree_test.cc index 74337df2..f20f3430 100644 --- a/absl/container/btree_test.cc +++ b/absl/container/btree_test.cc @@ -14,10 +14,14 @@ #include "absl/container/btree_test.h" +#include <algorithm> +#include <array> #include <cstdint> +#include <functional> #include <limits> #include <map> #include <memory> +#include <numeric> #include <stdexcept> #include <string> #include <type_traits> @@ -595,7 +599,7 @@ void BtreeTest() { using V = typename remove_pair_const<typename T::value_type>::type; const std::vector<V> random_values = GenerateValuesWithSeed<V>( absl::GetFlag(FLAGS_test_values), 4 * absl::GetFlag(FLAGS_test_values), - testing::GTEST_FLAG(random_seed)); + GTEST_FLAG_GET(random_seed)); unique_checker<T, C> container; @@ -619,7 +623,7 @@ void BtreeMultiTest() { using V = typename remove_pair_const<typename T::value_type>::type; const std::vector<V> random_values = GenerateValuesWithSeed<V>( absl::GetFlag(FLAGS_test_values), 4 * absl::GetFlag(FLAGS_test_values), - testing::GTEST_FLAG(random_seed)); + GTEST_FLAG_GET(random_seed)); multi_checker<T, C> container; @@ -1212,6 +1216,11 @@ class BtreeNodePeer { constexpr static bool UsesLinearNodeSearch() { return btree_node<typename Btree::params_type>::use_linear_search::value; } + + template <typename Btree> + constexpr static bool UsesGenerations() { + return Btree::params_type::kEnableGenerations; + } }; namespace { @@ -1285,7 +1294,7 @@ TEST(Btree, BtreeMapCanHoldMoveOnlyTypes) { std::unique_ptr<std::string> &v = m["A"]; EXPECT_TRUE(v == nullptr); - v.reset(new std::string("X")); + v = absl::make_unique<std::string>("X"); auto iter = m.find("A"); EXPECT_EQ("X", *iter->second); @@ -1344,38 +1353,34 @@ TEST(Btree, InitializerListInsert) { EXPECT_EQ(++it, range.second); } -template <typename Compare, typename K> -void AssertKeyCompareToAdapted() { - using Adapted = typename key_compare_to_adapter<Compare>::type; - static_assert(!std::is_same<Adapted, Compare>::value, - "key_compare_to_adapter should have adapted this comparator."); +template <typename Compare, typename Key> +void AssertKeyCompareStringAdapted() { + using Adapted = typename key_compare_adapter<Compare, Key>::type; static_assert( - std::is_same<absl::weak_ordering, - absl::result_of_t<Adapted(const K &, const K &)>>::value, - "Adapted comparator should be a key-compare-to comparator."); + std::is_same<Adapted, StringBtreeDefaultLess>::value || + std::is_same<Adapted, StringBtreeDefaultGreater>::value, + "key_compare_adapter should have string-adapted this comparator."); } -template <typename Compare, typename K> -void AssertKeyCompareToNotAdapted() { - using Unadapted = typename key_compare_to_adapter<Compare>::type; - static_assert( - std::is_same<Unadapted, Compare>::value, - "key_compare_to_adapter shouldn't have adapted this comparator."); +template <typename Compare, typename Key> +void AssertKeyCompareNotStringAdapted() { + using Adapted = typename key_compare_adapter<Compare, Key>::type; static_assert( - std::is_same<bool, - absl::result_of_t<Unadapted(const K &, const K &)>>::value, - "Un-adapted comparator should return bool."); + !std::is_same<Adapted, StringBtreeDefaultLess>::value && + !std::is_same<Adapted, StringBtreeDefaultGreater>::value, + "key_compare_adapter shouldn't have string-adapted this comparator."); } -TEST(Btree, KeyCompareToAdapter) { - AssertKeyCompareToAdapted<std::less<std::string>, std::string>(); - AssertKeyCompareToAdapted<std::greater<std::string>, std::string>(); - AssertKeyCompareToAdapted<std::less<absl::string_view>, absl::string_view>(); - AssertKeyCompareToAdapted<std::greater<absl::string_view>, - absl::string_view>(); - AssertKeyCompareToAdapted<std::less<absl::Cord>, absl::Cord>(); - AssertKeyCompareToAdapted<std::greater<absl::Cord>, absl::Cord>(); - AssertKeyCompareToNotAdapted<std::less<int>, int>(); - AssertKeyCompareToNotAdapted<std::greater<int>, int>(); +TEST(Btree, KeyCompareAdapter) { + AssertKeyCompareStringAdapted<std::less<std::string>, std::string>(); + AssertKeyCompareStringAdapted<std::greater<std::string>, std::string>(); + AssertKeyCompareStringAdapted<std::less<absl::string_view>, + absl::string_view>(); + AssertKeyCompareStringAdapted<std::greater<absl::string_view>, + absl::string_view>(); + AssertKeyCompareStringAdapted<std::less<absl::Cord>, absl::Cord>(); + AssertKeyCompareStringAdapted<std::greater<absl::Cord>, absl::Cord>(); + AssertKeyCompareNotStringAdapted<std::less<int>, int>(); + AssertKeyCompareNotStringAdapted<std::greater<int>, int>(); } TEST(Btree, RValueInsert) { @@ -1425,11 +1430,19 @@ TEST(Btree, RValueInsert) { EXPECT_EQ(tracker.swaps(), 0); } -// A btree set with a specific number of values per node. +template <typename Cmp> +struct CheckedCompareOptedOutCmp : Cmp, BtreeTestOnlyCheckedCompareOptOutBase { + using Cmp::Cmp; + CheckedCompareOptedOutCmp() {} + CheckedCompareOptedOutCmp(Cmp cmp) : Cmp(std::move(cmp)) {} // NOLINT +}; + +// A btree set with a specific number of values per node. Opt out of +// checked_compare so that we can expect exact numbers of comparisons. template <typename Key, int TargetValuesPerNode, typename Cmp = std::less<Key>> class SizedBtreeSet : public btree_set_container<btree< - set_params<Key, Cmp, std::allocator<Key>, + set_params<Key, CheckedCompareOptedOutCmp<Cmp>, std::allocator<Key>, BtreeNodePeer::GetTargetNodeSize<Key>(TargetValuesPerNode), /*Multi=*/false>>> { using Base = typename SizedBtreeSet::btree_set_container; @@ -1473,8 +1486,10 @@ TEST(Btree, MovesComparisonsCopiesSwapsTracking) { EXPECT_EQ(BtreeNodePeer::GetNumSlotsPerNode<decltype(set61)>(), 61); EXPECT_EQ(BtreeNodePeer::GetNumSlotsPerNode<decltype(set100)>(), 100); if (sizeof(void *) == 8) { - EXPECT_EQ(BtreeNodePeer::GetNumSlotsPerNode<absl::btree_set<int32_t>>(), - BtreeNodePeer::GetNumSlotsPerNode<decltype(set61)>()); + EXPECT_EQ( + BtreeNodePeer::GetNumSlotsPerNode<absl::btree_set<int32_t>>(), + // When we have generations, there is one fewer slot. + BtreeNodePeer::UsesGenerations<absl::btree_set<int32_t>>() ? 60 : 61); } // Test key insertion/deletion in random order. @@ -1528,8 +1543,10 @@ TEST(Btree, MovesComparisonsCopiesSwapsTrackingThreeWayCompare) { EXPECT_EQ(BtreeNodePeer::GetNumSlotsPerNode<decltype(set61)>(), 61); EXPECT_EQ(BtreeNodePeer::GetNumSlotsPerNode<decltype(set100)>(), 100); if (sizeof(void *) == 8) { - EXPECT_EQ(BtreeNodePeer::GetNumSlotsPerNode<absl::btree_set<int32_t>>(), - BtreeNodePeer::GetNumSlotsPerNode<decltype(set61)>()); + EXPECT_EQ( + BtreeNodePeer::GetNumSlotsPerNode<absl::btree_set<int32_t>>(), + // When we have generations, there is one fewer slot. + BtreeNodePeer::UsesGenerations<absl::btree_set<int32_t>>() ? 60 : 61); } // Test key insertion/deletion in random order. @@ -1708,10 +1725,25 @@ TEST(Btree, StrSplitCompatible) { EXPECT_EQ(split_set, expected_set); } -// We can't use EXPECT_EQ/etc. to compare absl::weak_ordering because they -// convert literal 0 to int and absl::weak_ordering can only be compared with -// literal 0. Defining this function allows for avoiding ClangTidy warnings. -bool Identity(const bool b) { return b; } +TEST(Btree, KeyComp) { + absl::btree_set<int> s; + EXPECT_TRUE(s.key_comp()(1, 2)); + EXPECT_FALSE(s.key_comp()(2, 2)); + EXPECT_FALSE(s.key_comp()(2, 1)); + + absl::btree_map<int, int> m1; + EXPECT_TRUE(m1.key_comp()(1, 2)); + EXPECT_FALSE(m1.key_comp()(2, 2)); + EXPECT_FALSE(m1.key_comp()(2, 1)); + + // Even though we internally adapt the comparator of `m2` to be three-way and + // heterogeneous, the comparator we expose through key_comp() is the original + // unadapted comparator. + absl::btree_map<std::string, int> m2; + EXPECT_TRUE(m2.key_comp()("a", "b")); + EXPECT_FALSE(m2.key_comp()("b", "b")); + EXPECT_FALSE(m2.key_comp()("b", "a")); +} TEST(Btree, ValueComp) { absl::btree_set<int> s; @@ -1724,13 +1756,29 @@ TEST(Btree, ValueComp) { EXPECT_FALSE(m1.value_comp()(std::make_pair(2, 0), std::make_pair(2, 0))); EXPECT_FALSE(m1.value_comp()(std::make_pair(2, 0), std::make_pair(1, 0))); + // Even though we internally adapt the comparator of `m2` to be three-way and + // heterogeneous, the comparator we expose through value_comp() is based on + // the original unadapted comparator. absl::btree_map<std::string, int> m2; - EXPECT_TRUE(Identity( - m2.value_comp()(std::make_pair("a", 0), std::make_pair("b", 0)) < 0)); - EXPECT_TRUE(Identity( - m2.value_comp()(std::make_pair("b", 0), std::make_pair("b", 0)) == 0)); - EXPECT_TRUE(Identity( - m2.value_comp()(std::make_pair("b", 0), std::make_pair("a", 0)) > 0)); + EXPECT_TRUE(m2.value_comp()(std::make_pair("a", 0), std::make_pair("b", 0))); + EXPECT_FALSE(m2.value_comp()(std::make_pair("b", 0), std::make_pair("b", 0))); + EXPECT_FALSE(m2.value_comp()(std::make_pair("b", 0), std::make_pair("a", 0))); +} + +// Test that we have the protected members from the std::map::value_compare API. +// See https://en.cppreference.com/w/cpp/container/map/value_compare. +TEST(Btree, MapValueCompProtected) { + struct key_compare { + bool operator()(int l, int r) const { return l < r; } + int id; + }; + using value_compare = absl::btree_map<int, int, key_compare>::value_compare; + struct value_comp_child : public value_compare { + explicit value_comp_child(key_compare kc) : value_compare(kc) {} + int GetId() const { return comp.id; } + }; + value_comp_child c(key_compare{10}); + EXPECT_EQ(c.GetId(), 10); } TEST(Btree, DefaultConstruction) { @@ -2282,7 +2330,9 @@ TEST(Btree, TryEmplaceWithHintWorks) { }; using Cmp = decltype(cmp); - absl::btree_map<int, int, Cmp> m(cmp); + // Use a map that is opted out of key_compare being adapted so we can expect + // strict comparison call limits. + absl::btree_map<int, int, CheckedCompareOptedOutCmp<Cmp>> m(cmp); for (int i = 0; i < 128; ++i) { m.emplace(i, i); } @@ -2437,23 +2487,28 @@ TEST(Btree, EraseIf) { // Test that erase_if works with all the container types and supports lambdas. { absl::btree_set<int> s = {1, 3, 5, 6, 100}; - erase_if(s, [](int k) { return k > 3; }); + EXPECT_EQ(erase_if(s, [](int k) { return k > 3; }), 3); EXPECT_THAT(s, ElementsAre(1, 3)); } { absl::btree_multiset<int> s = {1, 3, 3, 5, 6, 6, 100}; - erase_if(s, [](int k) { return k <= 3; }); + EXPECT_EQ(erase_if(s, [](int k) { return k <= 3; }), 3); EXPECT_THAT(s, ElementsAre(5, 6, 6, 100)); } { absl::btree_map<int, int> m = {{1, 1}, {3, 3}, {6, 6}, {100, 100}}; - erase_if(m, [](std::pair<const int, int> kv) { return kv.first > 3; }); + EXPECT_EQ( + erase_if(m, [](std::pair<const int, int> kv) { return kv.first > 3; }), + 2); EXPECT_THAT(m, ElementsAre(Pair(1, 1), Pair(3, 3))); } { absl::btree_multimap<int, int> m = {{1, 1}, {3, 3}, {3, 6}, {6, 6}, {6, 7}, {100, 6}}; - erase_if(m, [](std::pair<const int, int> kv) { return kv.second == 6; }); + EXPECT_EQ( + erase_if(m, + [](std::pair<const int, int> kv) { return kv.second == 6; }), + 3); EXPECT_THAT(m, ElementsAre(Pair(1, 1), Pair(3, 3), Pair(6, 7))); } // Test that erasing all elements from a large set works and test support for @@ -2461,15 +2516,29 @@ TEST(Btree, EraseIf) { { absl::btree_set<int> s; for (int i = 0; i < 1000; ++i) s.insert(2 * i); - erase_if(s, IsEven); + EXPECT_EQ(erase_if(s, IsEven), 1000); EXPECT_THAT(s, IsEmpty()); } // Test that erase_if supports other format of function pointers. { absl::btree_set<int> s = {1, 3, 5, 6, 100}; - erase_if(s, &IsEven); + EXPECT_EQ(erase_if(s, &IsEven), 2); EXPECT_THAT(s, ElementsAre(1, 3, 5)); } + // Test that erase_if invokes the predicate once per element. + { + absl::btree_set<int> s; + for (int i = 0; i < 1000; ++i) s.insert(i); + int pred_calls = 0; + EXPECT_EQ(erase_if(s, + [&pred_calls](int k) { + ++pred_calls; + return k % 2; + }), + 500); + EXPECT_THAT(s, SizeIs(500)); + EXPECT_EQ(pred_calls, 1000); + } } TEST(Btree, InsertOrAssign) { @@ -2893,6 +2962,292 @@ TEST(Btree, AllocMoveConstructor_DifferentAlloc) { EXPECT_EQ(bytes_used2, original_bytes_used); } +bool IntCmp(const int a, const int b) { return a < b; } + +TEST(Btree, SupportsFunctionPtrComparator) { + absl::btree_set<int, decltype(IntCmp) *> set(IntCmp); + set.insert({1, 2, 3}); + EXPECT_THAT(set, ElementsAre(1, 2, 3)); + EXPECT_TRUE(set.key_comp()(1, 2)); + EXPECT_TRUE(set.value_comp()(1, 2)); + + absl::btree_map<int, int, decltype(IntCmp) *> map(&IntCmp); + map[1] = 1; + EXPECT_THAT(map, ElementsAre(Pair(1, 1))); + EXPECT_TRUE(map.key_comp()(1, 2)); + EXPECT_TRUE(map.value_comp()(std::make_pair(1, 1), std::make_pair(2, 2))); +} + +template <typename Compare> +struct TransparentPassThroughComp { + using is_transparent = void; + + // This will fail compilation if we attempt a comparison that Compare does not + // support, and the failure will happen inside the function implementation so + // it can't be avoided by using SFINAE on this comparator. + template <typename T, typename U> + bool operator()(const T &lhs, const U &rhs) const { + return Compare()(lhs, rhs); + } +}; + +TEST(Btree, + SupportsTransparentComparatorThatDoesNotImplementAllVisibleOperators) { + absl::btree_set<MultiKey, TransparentPassThroughComp<MultiKeyComp>> set; + set.insert(MultiKey{1, 2}); + EXPECT_TRUE(set.contains(1)); +} + +TEST(Btree, ConstructImplicitlyWithUnadaptedComparator) { + absl::btree_set<MultiKey, MultiKeyComp> set = {{}, MultiKeyComp{}}; +} + +#ifndef NDEBUG +TEST(Btree, InvalidComparatorsCaught) { + { + struct ZeroAlwaysLessCmp { + bool operator()(int lhs, int rhs) const { + if (lhs == 0) return true; + return lhs < rhs; + } + }; + absl::btree_set<int, ZeroAlwaysLessCmp> set; + EXPECT_DEATH(set.insert({0, 1, 2}), "is_self_equivalent"); + } + { + struct ThreeWayAlwaysLessCmp { + absl::weak_ordering operator()(int, int) const { + return absl::weak_ordering::less; + } + }; + absl::btree_set<int, ThreeWayAlwaysLessCmp> set; + EXPECT_DEATH(set.insert({0, 1, 2}), "is_self_equivalent"); + } + { + struct SumGreaterZeroCmp { + bool operator()(int lhs, int rhs) const { + // First, do equivalence correctly - so we can test later condition. + if (lhs == rhs) return false; + return lhs + rhs > 0; + } + }; + absl::btree_set<int, SumGreaterZeroCmp> set; + // Note: '!' only needs to be escaped when it's the first character. + EXPECT_DEATH(set.insert({0, 1, 2}), + R"regex(\!lhs_comp_rhs \|\| !comp\(\)\(rhs, lhs\))regex"); + } + { + struct ThreeWaySumGreaterZeroCmp { + absl::weak_ordering operator()(int lhs, int rhs) const { + // First, do equivalence correctly - so we can test later condition. + if (lhs == rhs) return absl::weak_ordering::equivalent; + + if (lhs + rhs > 0) return absl::weak_ordering::less; + if (lhs + rhs == 0) return absl::weak_ordering::equivalent; + return absl::weak_ordering::greater; + } + }; + absl::btree_set<int, ThreeWaySumGreaterZeroCmp> set; + EXPECT_DEATH(set.insert({0, 1, 2}), "lhs_comp_rhs < 0 -> rhs_comp_lhs > 0"); + } +} +#endif + +#ifndef _MSC_VER +// This test crashes on MSVC. +TEST(Btree, InvalidIteratorUse) { + if (!BtreeNodePeer::UsesGenerations<absl::btree_set<int>>()) + GTEST_SKIP() << "Generation validation for iterators is disabled."; + + { + absl::btree_set<int> set; + for (int i = 0; i < 10; ++i) set.insert(i); + auto it = set.begin(); + set.erase(it++); + EXPECT_DEATH(set.erase(it++), "invalidated iterator"); + } + { + absl::btree_set<int> set; + for (int i = 0; i < 10; ++i) set.insert(i); + auto it = set.insert(20).first; + set.insert(30); + EXPECT_DEATH(*it, "invalidated iterator"); + } + { + absl::btree_set<int> set; + for (int i = 0; i < 10000; ++i) set.insert(i); + auto it = set.find(5000); + ASSERT_NE(it, set.end()); + set.erase(1); + EXPECT_DEATH(*it, "invalidated iterator"); + } +} +#endif + +class OnlyConstructibleByAllocator { + explicit OnlyConstructibleByAllocator(int i) : i_(i) {} + + public: + OnlyConstructibleByAllocator(const OnlyConstructibleByAllocator &other) + : i_(other.i_) {} + OnlyConstructibleByAllocator &operator=( + const OnlyConstructibleByAllocator &other) { + i_ = other.i_; + return *this; + } + int Get() const { return i_; } + bool operator==(int i) const { return i_ == i; } + + private: + template <typename T> + friend class OnlyConstructibleAllocator; + + int i_; +}; + +template <typename T = OnlyConstructibleByAllocator> +class OnlyConstructibleAllocator : public std::allocator<T> { + public: + OnlyConstructibleAllocator() = default; + template <class U> + explicit OnlyConstructibleAllocator(const OnlyConstructibleAllocator<U> &) {} + + void construct(OnlyConstructibleByAllocator *p, int i) { + new (p) OnlyConstructibleByAllocator(i); + } + template <typename Pair> + void construct(Pair *p, const int i) { + OnlyConstructibleByAllocator only(i); + new (p) Pair(std::move(only), i); + } + + template <class U> + struct rebind { + using other = OnlyConstructibleAllocator<U>; + }; +}; + +struct OnlyConstructibleByAllocatorComp { + using is_transparent = void; + bool operator()(OnlyConstructibleByAllocator a, + OnlyConstructibleByAllocator b) const { + return a.Get() < b.Get(); + } + bool operator()(int a, OnlyConstructibleByAllocator b) const { + return a < b.Get(); + } + bool operator()(OnlyConstructibleByAllocator a, int b) const { + return a.Get() < b; + } +}; + +TEST(Btree, OnlyConstructibleByAllocatorType) { + const std::array<int, 2> arr = {3, 4}; + { + absl::btree_set<OnlyConstructibleByAllocator, + OnlyConstructibleByAllocatorComp, + OnlyConstructibleAllocator<>> + set; + set.emplace(1); + set.emplace_hint(set.end(), 2); + set.insert(arr.begin(), arr.end()); + EXPECT_THAT(set, ElementsAre(1, 2, 3, 4)); + } + { + absl::btree_multiset<OnlyConstructibleByAllocator, + OnlyConstructibleByAllocatorComp, + OnlyConstructibleAllocator<>> + set; + set.emplace(1); + set.emplace_hint(set.end(), 2); + // TODO(ezb): fix insert_multi to allow this to compile. + // set.insert(arr.begin(), arr.end()); + EXPECT_THAT(set, ElementsAre(1, 2)); + } + { + absl::btree_map<OnlyConstructibleByAllocator, int, + OnlyConstructibleByAllocatorComp, + OnlyConstructibleAllocator<>> + map; + map.emplace(1); + map.emplace_hint(map.end(), 2); + map.insert(arr.begin(), arr.end()); + EXPECT_THAT(map, + ElementsAre(Pair(1, 1), Pair(2, 2), Pair(3, 3), Pair(4, 4))); + } + { + absl::btree_multimap<OnlyConstructibleByAllocator, int, + OnlyConstructibleByAllocatorComp, + OnlyConstructibleAllocator<>> + map; + map.emplace(1); + map.emplace_hint(map.end(), 2); + // TODO(ezb): fix insert_multi to allow this to compile. + // map.insert(arr.begin(), arr.end()); + EXPECT_THAT(map, ElementsAre(Pair(1, 1), Pair(2, 2))); + } +} + +class NotAssignable { + public: + explicit NotAssignable(int i) : i_(i) {} + NotAssignable(const NotAssignable &other) : i_(other.i_) {} + NotAssignable &operator=(NotAssignable &&other) = delete; + int Get() const { return i_; } + bool operator==(int i) const { return i_ == i; } + friend bool operator<(NotAssignable a, NotAssignable b) { + return a.i_ < b.i_; + } + + private: + int i_; +}; + +TEST(Btree, NotAssignableType) { + { + absl::btree_set<NotAssignable> set; + set.emplace(1); + set.emplace_hint(set.end(), 2); + set.insert(NotAssignable(3)); + set.insert(set.end(), NotAssignable(4)); + EXPECT_THAT(set, ElementsAre(1, 2, 3, 4)); + set.erase(set.begin()); + EXPECT_THAT(set, ElementsAre(2, 3, 4)); + } + { + absl::btree_multiset<NotAssignable> set; + set.emplace(1); + set.emplace_hint(set.end(), 2); + set.insert(NotAssignable(2)); + set.insert(set.end(), NotAssignable(3)); + EXPECT_THAT(set, ElementsAre(1, 2, 2, 3)); + set.erase(set.begin()); + EXPECT_THAT(set, ElementsAre(2, 2, 3)); + } + { + absl::btree_map<NotAssignable, int> map; + map.emplace(NotAssignable(1), 1); + map.emplace_hint(map.end(), NotAssignable(2), 2); + map.insert({NotAssignable(3), 3}); + map.insert(map.end(), {NotAssignable(4), 4}); + EXPECT_THAT(map, + ElementsAre(Pair(1, 1), Pair(2, 2), Pair(3, 3), Pair(4, 4))); + map.erase(map.begin()); + EXPECT_THAT(map, ElementsAre(Pair(2, 2), Pair(3, 3), Pair(4, 4))); + } + { + absl::btree_multimap<NotAssignable, int> map; + map.emplace(NotAssignable(1), 1); + map.emplace_hint(map.end(), NotAssignable(2), 2); + map.insert({NotAssignable(2), 3}); + map.insert(map.end(), {NotAssignable(3), 3}); + EXPECT_THAT(map, + ElementsAre(Pair(1, 1), Pair(2, 2), Pair(2, 3), Pair(3, 3))); + map.erase(map.begin()); + EXPECT_THAT(map, ElementsAre(Pair(2, 2), Pair(2, 3), Pair(3, 3))); + } +} + } // namespace } // namespace container_internal ABSL_NAMESPACE_END diff --git a/absl/container/fixed_array.h b/absl/container/fixed_array.h index fcb3e545..2aefae3b 100644 --- a/absl/container/fixed_array.h +++ b/absl/container/fixed_array.h @@ -73,11 +73,6 @@ constexpr static auto kFixedArrayUseDefault = static_cast<size_t>(-1); // uninitialized (e.g. int, int[4], double), and others default-constructed. // This matches the behavior of c-style arrays and `std::array`, but not // `std::vector`. -// -// Note that `FixedArray` does not provide a public allocator; if it requires a -// heap allocation, it will do so with global `::operator new[]()` and -// `::operator delete[]()`, even if T provides class-scope overrides for these -// operators. template <typename T, size_t N = kFixedArrayUseDefault, typename A = std::allocator<T>> class FixedArray { @@ -494,12 +489,14 @@ class FixedArray { Storage storage_; }; +#ifdef ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL template <typename T, size_t N, typename A> constexpr size_t FixedArray<T, N, A>::kInlineBytesDefault; template <typename T, size_t N, typename A> constexpr typename FixedArray<T, N, A>::size_type FixedArray<T, N, A>::inline_elements; +#endif template <typename T, size_t N, typename A> void FixedArray<T, N, A>::NonEmptyInlinedStorage::AnnotateConstruct( diff --git a/absl/container/flat_hash_map.h b/absl/container/flat_hash_map.h index 74def0df..e6bdbd9e 100644 --- a/absl/container/flat_hash_map.h +++ b/absl/container/flat_hash_map.h @@ -36,6 +36,7 @@ #include <utility> #include "absl/algorithm/container.h" +#include "absl/base/macros.h" #include "absl/container/internal/container_memory.h" #include "absl/container/internal/hash_function_defaults.h" // IWYU pragma: export #include "absl/container/internal/raw_hash_map.h" // IWYU pragma: export @@ -75,6 +76,10 @@ struct FlatHashMapPolicy; // absl/hash/hash.h for information on extending Abseil hashing to user-defined // types. // +// Using `absl::flat_hash_map` at interface boundaries in dynamically loaded +// libraries (e.g. .dll, .so) is unsupported due to way `absl::Hash` values may +// be randomized across dynamically loaded libraries. +// // NOTE: A `flat_hash_map` stores its value types directly inside its // implementation array to avoid memory indirection. Because a `flat_hash_map` // is designed to move data when rehashed, map values will not retain pointer @@ -356,8 +361,8 @@ class flat_hash_map : public absl::container_internal::raw_hash_map< // `flat_hash_map`. // // iterator try_emplace(const_iterator hint, - // const init_type& k, Args&&... args): - // iterator try_emplace(const_iterator hint, init_type&& k, Args&&... args): + // const key_type& k, Args&&... args): + // iterator try_emplace(const_iterator hint, key_type&& k, Args&&... args): // // Inserts (via copy or move) the element of the specified key into the // `flat_hash_map` using the position of `hint` as a non-binding suggestion @@ -541,10 +546,12 @@ class flat_hash_map : public absl::container_internal::raw_hash_map< // erase_if(flat_hash_map<>, Pred) // // Erases all elements that satisfy the predicate `pred` from the container `c`. +// Returns the number of erased elements. template <typename K, typename V, typename H, typename E, typename A, typename Predicate> -void erase_if(flat_hash_map<K, V, H, E, A>& c, Predicate pred) { - container_internal::EraseIf(pred, &c); +typename flat_hash_map<K, V, H, E, A>::size_type erase_if( + flat_hash_map<K, V, H, E, A>& c, Predicate pred) { + return container_internal::EraseIf(pred, &c); } namespace container_internal { diff --git a/absl/container/flat_hash_map_test.cc b/absl/container/flat_hash_map_test.cc index 89ec60c9..263951f1 100644 --- a/absl/container/flat_hash_map_test.cc +++ b/absl/container/flat_hash_map_test.cc @@ -236,33 +236,36 @@ TEST(FlatHashMap, EraseIf) { // Erase all elements. { flat_hash_map<int, int> s = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}}; - erase_if(s, [](std::pair<const int, int>) { return true; }); + EXPECT_EQ(erase_if(s, [](std::pair<const int, int>) { return true; }), 5); EXPECT_THAT(s, IsEmpty()); } // Erase no elements. { flat_hash_map<int, int> s = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}}; - erase_if(s, [](std::pair<const int, int>) { return false; }); + EXPECT_EQ(erase_if(s, [](std::pair<const int, int>) { return false; }), 0); EXPECT_THAT(s, UnorderedElementsAre(Pair(1, 1), Pair(2, 2), Pair(3, 3), Pair(4, 4), Pair(5, 5))); } // Erase specific elements. { flat_hash_map<int, int> s = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}}; - erase_if(s, - [](std::pair<const int, int> kvp) { return kvp.first % 2 == 1; }); + EXPECT_EQ(erase_if(s, + [](std::pair<const int, int> kvp) { + return kvp.first % 2 == 1; + }), + 3); EXPECT_THAT(s, UnorderedElementsAre(Pair(2, 2), Pair(4, 4))); } // Predicate is function reference. { flat_hash_map<int, int> s = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}}; - erase_if(s, FirstIsEven); + EXPECT_EQ(erase_if(s, FirstIsEven), 2); EXPECT_THAT(s, UnorderedElementsAre(Pair(1, 1), Pair(3, 3), Pair(5, 5))); } // Predicate is function pointer. { flat_hash_map<int, int> s = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}}; - erase_if(s, &FirstIsEven); + EXPECT_EQ(erase_if(s, &FirstIsEven), 2); EXPECT_THAT(s, UnorderedElementsAre(Pair(1, 1), Pair(3, 3), Pair(5, 5))); } } @@ -282,6 +285,32 @@ TEST(FlatHashMap, NodeHandleMutableKeyAccess) { } #endif +TEST(FlatHashMap, Reserve) { + // Verify that if we reserve(size() + n) then we can perform n insertions + // without a rehash, i.e., without invalidating any references. + for (size_t trial = 0; trial < 20; ++trial) { + for (size_t initial = 3; initial < 100; ++initial) { + // Fill in `initial` entries, then erase 2 of them, then reserve space for + // two inserts and check for reference stability while doing the inserts. + flat_hash_map<size_t, size_t> map; + for (size_t i = 0; i < initial; ++i) { + map[i] = i; + } + map.erase(0); + map.erase(1); + map.reserve(map.size() + 2); + size_t& a2 = map[2]; + // In the event of a failure, asan will complain in one of these two + // assignments. + map[initial] = a2; + map[initial + 1] = a2; + // Fail even when not under asan: + size_t& a2new = map[2]; + EXPECT_EQ(&a2, &a2new); + } + } +} + } // namespace } // namespace container_internal ABSL_NAMESPACE_END diff --git a/absl/container/flat_hash_set.h b/absl/container/flat_hash_set.h index 6b89da65..4938c703 100644 --- a/absl/container/flat_hash_set.h +++ b/absl/container/flat_hash_set.h @@ -67,11 +67,15 @@ struct FlatHashSetPolicy; // // By default, `flat_hash_set` uses the `absl::Hash` hashing framework. All // fundamental and Abseil types that support the `absl::Hash` framework have a -// compatible equality operator for comparing insertions into `flat_hash_map`. +// compatible equality operator for comparing insertions into `flat_hash_set`. // If your type is not yet supported by the `absl::Hash` framework, see // absl/hash/hash.h for information on extending Abseil hashing to user-defined // types. // +// Using `absl::flat_hash_set` at interface boundaries in dynamically loaded +// libraries (e.g. .dll, .so) is unsupported due to way `absl::Hash` values may +// be randomized across dynamically loaded libraries. +// // NOTE: A `flat_hash_set` stores its keys directly inside its implementation // array to avoid memory indirection. Because a `flat_hash_set` is designed to // move data when rehashed, set keys will not retain pointer stability. If you @@ -106,7 +110,7 @@ class flat_hash_set public: // Constructors and Assignment Operators // - // A flat_hash_set supports the same overload set as `std::unordered_map` + // A flat_hash_set supports the same overload set as `std::unordered_set` // for construction and assignment: // // * Default constructor @@ -173,7 +177,7 @@ class flat_hash_set // available within the `flat_hash_set`. // // NOTE: this member function is particular to `absl::flat_hash_set` and is - // not provided in the `std::unordered_map` API. + // not provided in the `std::unordered_set` API. using Base::capacity; // flat_hash_set::empty() @@ -332,7 +336,7 @@ class flat_hash_set // flat_hash_set::swap(flat_hash_set& other) // // Exchanges the contents of this `flat_hash_set` with those of the `other` - // flat hash map, avoiding invocation of any move, copy, or swap operations on + // flat hash set, avoiding invocation of any move, copy, or swap operations on // individual elements. // // All iterators and references on the `flat_hash_set` remain valid, excepting @@ -340,7 +344,7 @@ class flat_hash_set // // `swap()` requires that the flat hash set's hashing and key equivalence // functions be Swappable, and are exchaged using unqualified calls to - // non-member `swap()`. If the map's allocator has + // non-member `swap()`. If the set's allocator has // `std::allocator_traits<allocator_type>::propagate_on_container_swap::value` // set to `true`, the allocators are also exchanged using an unqualified call // to non-member `swap()`; otherwise, the allocators are not swapped. @@ -395,14 +399,14 @@ class flat_hash_set // flat_hash_set::bucket_count() // // Returns the number of "buckets" within the `flat_hash_set`. Note that - // because a flat hash map contains all elements within its internal storage, + // because a flat hash set contains all elements within its internal storage, // this value simply equals the current capacity of the `flat_hash_set`. using Base::bucket_count; // flat_hash_set::load_factor() // // Returns the current load factor of the `flat_hash_set` (the average number - // of slots occupied with a value within the hash map). + // of slots occupied with a value within the hash set). using Base::load_factor; // flat_hash_set::max_load_factor() @@ -443,9 +447,11 @@ class flat_hash_set // erase_if(flat_hash_set<>, Pred) // // Erases all elements that satisfy the predicate `pred` from the container `c`. +// Returns the number of erased elements. template <typename T, typename H, typename E, typename A, typename Predicate> -void erase_if(flat_hash_set<T, H, E, A>& c, Predicate pred) { - container_internal::EraseIf(pred, &c); +typename flat_hash_set<T, H, E, A>::size_type erase_if( + flat_hash_set<T, H, E, A>& c, Predicate pred) { + return container_internal::EraseIf(pred, &c); } namespace container_internal { diff --git a/absl/container/flat_hash_set_test.cc b/absl/container/flat_hash_set_test.cc index 8f6f9944..b6a72a20 100644 --- a/absl/container/flat_hash_set_test.cc +++ b/absl/container/flat_hash_set_test.cc @@ -143,31 +143,31 @@ TEST(FlatHashSet, EraseIf) { // Erase all elements. { flat_hash_set<int> s = {1, 2, 3, 4, 5}; - erase_if(s, [](int) { return true; }); + EXPECT_EQ(erase_if(s, [](int) { return true; }), 5); EXPECT_THAT(s, IsEmpty()); } // Erase no elements. { flat_hash_set<int> s = {1, 2, 3, 4, 5}; - erase_if(s, [](int) { return false; }); + EXPECT_EQ(erase_if(s, [](int) { return false; }), 0); EXPECT_THAT(s, UnorderedElementsAre(1, 2, 3, 4, 5)); } // Erase specific elements. { flat_hash_set<int> s = {1, 2, 3, 4, 5}; - erase_if(s, [](int k) { return k % 2 == 1; }); + EXPECT_EQ(erase_if(s, [](int k) { return k % 2 == 1; }), 3); EXPECT_THAT(s, UnorderedElementsAre(2, 4)); } // Predicate is function reference. { flat_hash_set<int> s = {1, 2, 3, 4, 5}; - erase_if(s, IsEven); + EXPECT_EQ(erase_if(s, IsEven), 2); EXPECT_THAT(s, UnorderedElementsAre(1, 3, 5)); } // Predicate is function pointer. { flat_hash_set<int> s = {1, 2, 3, 4, 5}; - erase_if(s, &IsEven); + EXPECT_EQ(erase_if(s, &IsEven), 2); EXPECT_THAT(s, UnorderedElementsAre(1, 3, 5)); } } diff --git a/absl/container/inlined_vector.h b/absl/container/inlined_vector.h index 7c182342..bc1c4a77 100644 --- a/absl/container/inlined_vector.h +++ b/absl/container/inlined_vector.h @@ -36,7 +36,6 @@ #define ABSL_CONTAINER_INLINED_VECTOR_H_ #include <algorithm> -#include <cassert> #include <cstddef> #include <cstdlib> #include <cstring> @@ -72,37 +71,43 @@ class InlinedVector { using Storage = inlined_vector_internal::Storage<T, N, A>; - using AllocatorTraits = typename Storage::AllocatorTraits; - using RValueReference = typename Storage::RValueReference; - using MoveIterator = typename Storage::MoveIterator; - using IsMemcpyOk = typename Storage::IsMemcpyOk; + template <typename TheA> + using AllocatorTraits = inlined_vector_internal::AllocatorTraits<TheA>; + template <typename TheA> + using MoveIterator = inlined_vector_internal::MoveIterator<TheA>; + template <typename TheA> + using IsMemcpyOk = inlined_vector_internal::IsMemcpyOk<TheA>; - template <typename Iterator> + template <typename TheA, typename Iterator> using IteratorValueAdapter = - typename Storage::template IteratorValueAdapter<Iterator>; - using CopyValueAdapter = typename Storage::CopyValueAdapter; - using DefaultValueAdapter = typename Storage::DefaultValueAdapter; + inlined_vector_internal::IteratorValueAdapter<TheA, Iterator>; + template <typename TheA> + using CopyValueAdapter = inlined_vector_internal::CopyValueAdapter<TheA>; + template <typename TheA> + using DefaultValueAdapter = + inlined_vector_internal::DefaultValueAdapter<TheA>; template <typename Iterator> using EnableIfAtLeastForwardIterator = absl::enable_if_t< - inlined_vector_internal::IsAtLeastForwardIterator<Iterator>::value>; + inlined_vector_internal::IsAtLeastForwardIterator<Iterator>::value, int>; template <typename Iterator> using DisableIfAtLeastForwardIterator = absl::enable_if_t< - !inlined_vector_internal::IsAtLeastForwardIterator<Iterator>::value>; + !inlined_vector_internal::IsAtLeastForwardIterator<Iterator>::value, int>; public: - using allocator_type = typename Storage::allocator_type; - using value_type = typename Storage::value_type; - using pointer = typename Storage::pointer; - using const_pointer = typename Storage::const_pointer; - using size_type = typename Storage::size_type; - using difference_type = typename Storage::difference_type; - using reference = typename Storage::reference; - using const_reference = typename Storage::const_reference; - using iterator = typename Storage::iterator; - using const_iterator = typename Storage::const_iterator; - using reverse_iterator = typename Storage::reverse_iterator; - using const_reverse_iterator = typename Storage::const_reverse_iterator; + using allocator_type = A; + using value_type = inlined_vector_internal::ValueType<A>; + using pointer = inlined_vector_internal::Pointer<A>; + using const_pointer = inlined_vector_internal::ConstPointer<A>; + using size_type = inlined_vector_internal::SizeType<A>; + using difference_type = inlined_vector_internal::DifferenceType<A>; + using reference = inlined_vector_internal::Reference<A>; + using const_reference = inlined_vector_internal::ConstReference<A>; + using iterator = inlined_vector_internal::Iterator<A>; + using const_iterator = inlined_vector_internal::ConstIterator<A>; + using reverse_iterator = inlined_vector_internal::ReverseIterator<A>; + using const_reverse_iterator = + inlined_vector_internal::ConstReverseIterator<A>; // --------------------------------------------------------------------------- // InlinedVector Constructors and Destructor @@ -111,28 +116,28 @@ class InlinedVector { // Creates an empty inlined vector with a value-initialized allocator. InlinedVector() noexcept(noexcept(allocator_type())) : storage_() {} - // Creates an empty inlined vector with a copy of `alloc`. - explicit InlinedVector(const allocator_type& alloc) noexcept - : storage_(alloc) {} + // Creates an empty inlined vector with a copy of `allocator`. + explicit InlinedVector(const allocator_type& allocator) noexcept + : storage_(allocator) {} // Creates an inlined vector with `n` copies of `value_type()`. explicit InlinedVector(size_type n, - const allocator_type& alloc = allocator_type()) - : storage_(alloc) { - storage_.Initialize(DefaultValueAdapter(), n); + const allocator_type& allocator = allocator_type()) + : storage_(allocator) { + storage_.Initialize(DefaultValueAdapter<A>(), n); } // Creates an inlined vector with `n` copies of `v`. InlinedVector(size_type n, const_reference v, - const allocator_type& alloc = allocator_type()) - : storage_(alloc) { - storage_.Initialize(CopyValueAdapter(v), n); + const allocator_type& allocator = allocator_type()) + : storage_(allocator) { + storage_.Initialize(CopyValueAdapter<A>(std::addressof(v)), n); } // Creates an inlined vector with copies of the elements of `list`. InlinedVector(std::initializer_list<value_type> list, - const allocator_type& alloc = allocator_type()) - : InlinedVector(list.begin(), list.end(), alloc) {} + const allocator_type& allocator = allocator_type()) + : InlinedVector(list.begin(), list.end(), allocator) {} // Creates an inlined vector with elements constructed from the provided // forward iterator range [`first`, `last`). @@ -141,35 +146,36 @@ class InlinedVector { // this constructor with two integral arguments and a call to the above // `InlinedVector(size_type, const_reference)` constructor. template <typename ForwardIterator, - EnableIfAtLeastForwardIterator<ForwardIterator>* = nullptr> + EnableIfAtLeastForwardIterator<ForwardIterator> = 0> InlinedVector(ForwardIterator first, ForwardIterator last, - const allocator_type& alloc = allocator_type()) - : storage_(alloc) { - storage_.Initialize(IteratorValueAdapter<ForwardIterator>(first), - std::distance(first, last)); + const allocator_type& allocator = allocator_type()) + : storage_(allocator) { + storage_.Initialize(IteratorValueAdapter<A, ForwardIterator>(first), + static_cast<size_t>(std::distance(first, last))); } // Creates an inlined vector with elements constructed from the provided input // iterator range [`first`, `last`). template <typename InputIterator, - DisableIfAtLeastForwardIterator<InputIterator>* = nullptr> + DisableIfAtLeastForwardIterator<InputIterator> = 0> InlinedVector(InputIterator first, InputIterator last, - const allocator_type& alloc = allocator_type()) - : storage_(alloc) { + const allocator_type& allocator = allocator_type()) + : storage_(allocator) { std::copy(first, last, std::back_inserter(*this)); } // Creates an inlined vector by copying the contents of `other` using // `other`'s allocator. InlinedVector(const InlinedVector& other) - : InlinedVector(other, *other.storage_.GetAllocPtr()) {} + : InlinedVector(other, other.storage_.GetAllocator()) {} - // Creates an inlined vector by copying the contents of `other` using `alloc`. - InlinedVector(const InlinedVector& other, const allocator_type& alloc) - : storage_(alloc) { + // Creates an inlined vector by copying the contents of `other` using the + // provided `allocator`. + InlinedVector(const InlinedVector& other, const allocator_type& allocator) + : storage_(allocator) { if (other.empty()) { // Empty; nothing to do. - } else if (IsMemcpyOk::value && !other.storage_.GetIsAllocated()) { + } else if (IsMemcpyOk<A>::value && !other.storage_.GetIsAllocated()) { // Memcpy-able and do not need allocation. storage_.MemcpyFrom(other.storage_); } else { @@ -194,23 +200,23 @@ class InlinedVector { InlinedVector(InlinedVector&& other) noexcept( absl::allocator_is_nothrow<allocator_type>::value || std::is_nothrow_move_constructible<value_type>::value) - : storage_(*other.storage_.GetAllocPtr()) { - if (IsMemcpyOk::value) { + : storage_(other.storage_.GetAllocator()) { + if (IsMemcpyOk<A>::value) { storage_.MemcpyFrom(other.storage_); other.storage_.SetInlinedSize(0); } else if (other.storage_.GetIsAllocated()) { - storage_.SetAllocatedData(other.storage_.GetAllocatedData(), - other.storage_.GetAllocatedCapacity()); + storage_.SetAllocation({other.storage_.GetAllocatedData(), + other.storage_.GetAllocatedCapacity()}); storage_.SetAllocatedSize(other.storage_.GetSize()); other.storage_.SetInlinedSize(0); } else { - IteratorValueAdapter<MoveIterator> other_values( - MoveIterator(other.storage_.GetInlinedData())); + IteratorValueAdapter<A, MoveIterator<A>> other_values( + MoveIterator<A>(other.storage_.GetInlinedData())); - inlined_vector_internal::ConstructElements( - storage_.GetAllocPtr(), storage_.GetInlinedData(), &other_values, + inlined_vector_internal::ConstructElements<A>( + storage_.GetAllocator(), storage_.GetInlinedData(), other_values, other.storage_.GetSize()); storage_.SetInlinedSize(other.storage_.GetSize()); @@ -218,30 +224,32 @@ class InlinedVector { } // Creates an inlined vector by moving in the contents of `other` with a copy - // of `alloc`. + // of `allocator`. // - // NOTE: if `other`'s allocator is not equal to `alloc`, even if `other` + // NOTE: if `other`'s allocator is not equal to `allocator`, even if `other` // contains allocated memory, this move constructor will still allocate. Since // allocation is performed, this constructor can only be `noexcept` if the // specified allocator is also `noexcept`. - InlinedVector(InlinedVector&& other, const allocator_type& alloc) noexcept( - absl::allocator_is_nothrow<allocator_type>::value) - : storage_(alloc) { - if (IsMemcpyOk::value) { + InlinedVector( + InlinedVector&& other, + const allocator_type& + allocator) noexcept(absl::allocator_is_nothrow<allocator_type>::value) + : storage_(allocator) { + if (IsMemcpyOk<A>::value) { storage_.MemcpyFrom(other.storage_); other.storage_.SetInlinedSize(0); - } else if ((*storage_.GetAllocPtr() == *other.storage_.GetAllocPtr()) && + } else if ((storage_.GetAllocator() == other.storage_.GetAllocator()) && other.storage_.GetIsAllocated()) { - storage_.SetAllocatedData(other.storage_.GetAllocatedData(), - other.storage_.GetAllocatedCapacity()); + storage_.SetAllocation({other.storage_.GetAllocatedData(), + other.storage_.GetAllocatedCapacity()}); storage_.SetAllocatedSize(other.storage_.GetSize()); other.storage_.SetInlinedSize(0); } else { - storage_.Initialize( - IteratorValueAdapter<MoveIterator>(MoveIterator(other.data())), - other.size()); + storage_.Initialize(IteratorValueAdapter<A, MoveIterator<A>>( + MoveIterator<A>(other.data())), + other.size()); } } @@ -442,7 +450,7 @@ class InlinedVector { // `InlinedVector::get_allocator()` // // Returns a copy of the inlined vector's allocator. - allocator_type get_allocator() const { return *storage_.GetAllocPtr(); } + allocator_type get_allocator() const { return storage_.GetAllocator(); } // --------------------------------------------------------------------------- // InlinedVector Member Mutators @@ -476,16 +484,16 @@ class InlinedVector { // unspecified state. InlinedVector& operator=(InlinedVector&& other) { if (ABSL_PREDICT_TRUE(this != std::addressof(other))) { - if (IsMemcpyOk::value || other.storage_.GetIsAllocated()) { - inlined_vector_internal::DestroyElements(storage_.GetAllocPtr(), data(), - size()); + if (IsMemcpyOk<A>::value || other.storage_.GetIsAllocated()) { + inlined_vector_internal::DestroyAdapter<A>::DestroyElements( + storage_.GetAllocator(), data(), size()); storage_.DeallocateIfAllocated(); storage_.MemcpyFrom(other.storage_); other.storage_.SetInlinedSize(0); } else { - storage_.Assign(IteratorValueAdapter<MoveIterator>( - MoveIterator(other.storage_.GetInlinedData())), + storage_.Assign(IteratorValueAdapter<A, MoveIterator<A>>( + MoveIterator<A>(other.storage_.GetInlinedData())), other.size()); } } @@ -497,7 +505,7 @@ class InlinedVector { // // Replaces the contents of the inlined vector with `n` copies of `v`. void assign(size_type n, const_reference v) { - storage_.Assign(CopyValueAdapter(v), n); + storage_.Assign(CopyValueAdapter<A>(std::addressof(v)), n); } // Overload of `InlinedVector::assign(...)` that replaces the contents of the @@ -511,10 +519,10 @@ class InlinedVector { // // NOTE: this overload is for iterators that are "forward" category or better. template <typename ForwardIterator, - EnableIfAtLeastForwardIterator<ForwardIterator>* = nullptr> + EnableIfAtLeastForwardIterator<ForwardIterator> = 0> void assign(ForwardIterator first, ForwardIterator last) { - storage_.Assign(IteratorValueAdapter<ForwardIterator>(first), - std::distance(first, last)); + storage_.Assign(IteratorValueAdapter<A, ForwardIterator>(first), + static_cast<size_t>(std::distance(first, last))); } // Overload of `InlinedVector::assign(...)` to replace the contents of the @@ -522,7 +530,7 @@ class InlinedVector { // // NOTE: this overload is for iterators that are "input" category. template <typename InputIterator, - DisableIfAtLeastForwardIterator<InputIterator>* = nullptr> + DisableIfAtLeastForwardIterator<InputIterator> = 0> void assign(InputIterator first, InputIterator last) { size_type i = 0; for (; i < size() && first != last; ++i, static_cast<void>(++first)) { @@ -541,7 +549,7 @@ class InlinedVector { // is larger than `size()`, new elements are value-initialized. void resize(size_type n) { ABSL_HARDENING_ASSERT(n <= max_size()); - storage_.Resize(DefaultValueAdapter(), n); + storage_.Resize(DefaultValueAdapter<A>(), n); } // Overload of `InlinedVector::resize(...)` that resizes the inlined vector to @@ -551,7 +559,7 @@ class InlinedVector { // is larger than `size()`, new elements are copied-constructed from `v`. void resize(size_type n, const_reference v) { ABSL_HARDENING_ASSERT(n <= max_size()); - storage_.Resize(CopyValueAdapter(v), n); + storage_.Resize(CopyValueAdapter<A>(std::addressof(v)), n); } // `InlinedVector::insert(...)` @@ -564,7 +572,7 @@ class InlinedVector { // Overload of `InlinedVector::insert(...)` that inserts `v` at `pos` using // move semantics, returning an `iterator` to the newly inserted element. - iterator insert(const_iterator pos, RValueReference v) { + iterator insert(const_iterator pos, value_type&& v) { return emplace(pos, std::move(v)); } @@ -577,7 +585,20 @@ class InlinedVector { if (ABSL_PREDICT_TRUE(n != 0)) { value_type dealias = v; - return storage_.Insert(pos, CopyValueAdapter(dealias), n); + // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=102329#c2 + // It appears that GCC thinks that since `pos` is a const pointer and may + // point to uninitialized memory at this point, a warning should be + // issued. But `pos` is actually only used to compute an array index to + // write to. +#if !defined(__clang__) && defined(__GNUC__) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#endif + return storage_.Insert(pos, CopyValueAdapter<A>(std::addressof(dealias)), + n); +#if !defined(__clang__) && defined(__GNUC__) +#pragma GCC diagnostic pop +#endif } else { return const_cast<iterator>(pos); } @@ -596,14 +617,15 @@ class InlinedVector { // // NOTE: this overload is for iterators that are "forward" category or better. template <typename ForwardIterator, - EnableIfAtLeastForwardIterator<ForwardIterator>* = nullptr> + EnableIfAtLeastForwardIterator<ForwardIterator> = 0> iterator insert(const_iterator pos, ForwardIterator first, ForwardIterator last) { ABSL_HARDENING_ASSERT(pos >= begin()); ABSL_HARDENING_ASSERT(pos <= end()); if (ABSL_PREDICT_TRUE(first != last)) { - return storage_.Insert(pos, IteratorValueAdapter<ForwardIterator>(first), + return storage_.Insert(pos, + IteratorValueAdapter<A, ForwardIterator>(first), std::distance(first, last)); } else { return const_cast<iterator>(pos); @@ -616,7 +638,7 @@ class InlinedVector { // // NOTE: this overload is for iterators that are "input" category. template <typename InputIterator, - DisableIfAtLeastForwardIterator<InputIterator>* = nullptr> + DisableIfAtLeastForwardIterator<InputIterator> = 0> iterator insert(const_iterator pos, InputIterator first, InputIterator last) { ABSL_HARDENING_ASSERT(pos >= begin()); ABSL_HARDENING_ASSERT(pos <= end()); @@ -640,8 +662,8 @@ class InlinedVector { value_type dealias(std::forward<Args>(args)...); return storage_.Insert(pos, - IteratorValueAdapter<MoveIterator>( - MoveIterator(std::addressof(dealias))), + IteratorValueAdapter<A, MoveIterator<A>>( + MoveIterator<A>(std::addressof(dealias))), 1); } @@ -661,7 +683,7 @@ class InlinedVector { // Overload of `InlinedVector::push_back(...)` for inserting `v` at `end()` // using move semantics. - void push_back(RValueReference v) { + void push_back(value_type&& v) { static_cast<void>(emplace_back(std::move(v))); } @@ -671,7 +693,7 @@ class InlinedVector { void pop_back() noexcept { ABSL_HARDENING_ASSERT(!empty()); - AllocatorTraits::destroy(*storage_.GetAllocPtr(), data() + (size() - 1)); + AllocatorTraits<A>::destroy(storage_.GetAllocator(), data() + (size() - 1)); storage_.SubtractSize(1); } @@ -710,8 +732,8 @@ class InlinedVector { // Destroys all elements in the inlined vector, setting the size to `0` and // deallocating any held memory. void clear() noexcept { - inlined_vector_internal::DestroyElements(storage_.GetAllocPtr(), data(), - size()); + inlined_vector_internal::DestroyAdapter<A>::DestroyElements( + storage_.GetAllocator(), data(), size()); storage_.DeallocateIfAllocated(); storage_.SetInlinedSize(0); @@ -724,15 +746,12 @@ class InlinedVector { // `InlinedVector::shrink_to_fit()` // - // Reduces memory usage by freeing unused memory. After being called, calls to - // `capacity()` will be equal to `max(N, size())`. - // - // If `size() <= N` and the inlined vector contains allocated memory, the - // elements will all be moved to the inlined space and the allocated memory - // will be deallocated. + // Attempts to reduce memory usage by moving elements to (or keeping elements + // in) the smallest available buffer sufficient for containing `size()` + // elements. // - // If `size() > N` and `size() < capacity()`, the elements will be moved to a - // smaller allocation. + // If `size()` is sufficiently small, the elements will be moved into (or kept + // in) the inlined space. void shrink_to_fit() { if (storage_.GetIsAllocated()) { storage_.ShrinkToFit(); diff --git a/absl/container/inlined_vector_test.cc b/absl/container/inlined_vector_test.cc index 98aff334..4c1ba04a 100644 --- a/absl/container/inlined_vector_test.cc +++ b/absl/container/inlined_vector_test.cc @@ -1545,17 +1545,18 @@ TYPED_TEST_P(InstanceTest, InitializerListAssign) { } } -REGISTER_TYPED_TEST_CASE_P(InstanceTest, Swap, CountConstructorsDestructors, - CountConstructorsDestructorsOnCopyConstruction, - CountConstructorsDestructorsOnMoveConstruction, - CountConstructorsDestructorsOnAssignment, - CountConstructorsDestructorsOnMoveAssignment, - CountElemAssignInlineBacking, RangedConstructor, - RangedAssign, InitializerListAssign); +REGISTER_TYPED_TEST_SUITE_P(InstanceTest, Swap, CountConstructorsDestructors, + CountConstructorsDestructorsOnCopyConstruction, + CountConstructorsDestructorsOnMoveConstruction, + CountConstructorsDestructorsOnAssignment, + CountConstructorsDestructorsOnMoveAssignment, + CountElemAssignInlineBacking, RangedConstructor, + RangedAssign, InitializerListAssign); using InstanceTypes = ::testing::Types<CopyableOnlyInstance, CopyableMovableInstance>; -INSTANTIATE_TYPED_TEST_CASE_P(InstanceTestOnTypes, InstanceTest, InstanceTypes); +INSTANTIATE_TYPED_TEST_SUITE_P(InstanceTestOnTypes, InstanceTest, + InstanceTypes); TEST(DynamicVec, DynamicVecCompiles) { DynamicVec v; diff --git a/absl/container/internal/btree.h b/absl/container/internal/btree.h index 0bb38366..01f4e749 100644 --- a/absl/container/internal/btree.h +++ b/absl/container/internal/btree.h @@ -58,6 +58,7 @@ #include <type_traits> #include <utility> +#include "absl/base/internal/raw_logging.h" #include "absl/base/macros.h" #include "absl/container/internal/common.h" #include "absl/container/internal/compressed_tuple.h" @@ -74,12 +75,24 @@ namespace absl { ABSL_NAMESPACE_BEGIN namespace container_internal { +#ifdef ABSL_BTREE_ENABLE_GENERATIONS +#error ABSL_BTREE_ENABLE_GENERATIONS cannot be directly set +#elif defined(ABSL_HAVE_ADDRESS_SANITIZER) || \ + defined(ABSL_HAVE_MEMORY_SANITIZER) +// When compiled in sanitizer mode, we add generation integers to the nodes and +// iterators. When iterators are used, we validate that the container has not +// been mutated since the iterator was constructed. +#define ABSL_BTREE_ENABLE_GENERATIONS +#endif + +template <typename Compare, typename T, typename U> +using compare_result_t = absl::result_of_t<const Compare(const T &, const U &)>; + // A helper class that indicates if the Compare parameter is a key-compare-to // comparator. template <typename Compare, typename T> using btree_is_key_compare_to = - std::is_convertible<absl::result_of_t<Compare(const T &, const T &)>, - absl::weak_ordering>; + std::is_convertible<compare_result_t<Compare, T, T>, absl::weak_ordering>; struct StringBtreeDefaultLess { using is_transparent = void; @@ -88,7 +101,12 @@ struct StringBtreeDefaultLess { // Compatibility constructor. StringBtreeDefaultLess(std::less<std::string>) {} // NOLINT - StringBtreeDefaultLess(std::less<string_view>) {} // NOLINT + StringBtreeDefaultLess(std::less<absl::string_view>) {} // NOLINT + + // Allow converting to std::less for use in key_comp()/value_comp(). + explicit operator std::less<std::string>() const { return {}; } + explicit operator std::less<absl::string_view>() const { return {}; } + explicit operator std::less<absl::Cord>() const { return {}; } absl::weak_ordering operator()(absl::string_view lhs, absl::string_view rhs) const { @@ -115,7 +133,12 @@ struct StringBtreeDefaultGreater { StringBtreeDefaultGreater() = default; StringBtreeDefaultGreater(std::greater<std::string>) {} // NOLINT - StringBtreeDefaultGreater(std::greater<string_view>) {} // NOLINT + StringBtreeDefaultGreater(std::greater<absl::string_view>) {} // NOLINT + + // Allow converting to std::greater for use in key_comp()/value_comp(). + explicit operator std::greater<std::string>() const { return {}; } + explicit operator std::greater<absl::string_view>() const { return {}; } + explicit operator std::greater<absl::Cord>() const { return {}; } absl::weak_ordering operator()(absl::string_view lhs, absl::string_view rhs) const { @@ -136,49 +159,140 @@ struct StringBtreeDefaultGreater { } }; -// A helper class to convert a boolean comparison into a three-way "compare-to" -// comparison that returns an `absl::weak_ordering`. This helper -// class is specialized for less<std::string>, greater<std::string>, -// less<string_view>, greater<string_view>, less<absl::Cord>, and -// greater<absl::Cord>. -// -// key_compare_to_adapter is provided so that btree users -// automatically get the more efficient compare-to code when using common -// Abseil string types with common comparison functors. -// These string-like specializations also turn on heterogeneous lookup by -// default. +// See below comments for checked_compare. +template <typename Compare, bool is_class = std::is_class<Compare>::value> +struct checked_compare_base : Compare { + using Compare::Compare; + explicit checked_compare_base(Compare c) : Compare(std::move(c)) {} + const Compare &comp() const { return *this; } +}; template <typename Compare> -struct key_compare_to_adapter { - using type = Compare; +struct checked_compare_base<Compare, false> { + explicit checked_compare_base(Compare c) : compare(std::move(c)) {} + const Compare &comp() const { return compare; } + Compare compare; +}; + +// A mechanism for opting out of checked_compare for use only in btree_test.cc. +struct BtreeTestOnlyCheckedCompareOptOutBase {}; + +// A helper class to adapt the specified comparator for two use cases: +// (1) When using common Abseil string types with common comparison functors, +// convert a boolean comparison into a three-way comparison that returns an +// `absl::weak_ordering`. This helper class is specialized for +// less<std::string>, greater<std::string>, less<string_view>, +// greater<string_view>, less<absl::Cord>, and greater<absl::Cord>. +// (2) Adapt the comparator to diagnose cases of non-strict-weak-ordering (see +// https://en.cppreference.com/w/cpp/named_req/Compare) in debug mode. Whenever +// a comparison is made, we will make assertions to verify that the comparator +// is valid. +template <typename Compare, typename Key> +struct key_compare_adapter { + // Inherit from checked_compare_base to support function pointers and also + // keep empty-base-optimization (EBO) support for classes. + // Note: we can't use CompressedTuple here because that would interfere + // with the EBO for `btree::rightmost_`. `btree::rightmost_` is itself a + // CompressedTuple and nested `CompressedTuple`s don't support EBO. + // TODO(b/214288561): use CompressedTuple instead once it supports EBO for + // nested `CompressedTuple`s. + struct checked_compare : checked_compare_base<Compare> { + private: + using Base = typename checked_compare::checked_compare_base; + using Base::comp; + + // If possible, returns whether `t` is equivalent to itself. We can only do + // this for `Key`s because we can't be sure that it's safe to call + // `comp()(k, k)` otherwise. Even if SFINAE allows it, there could be a + // compilation failure inside the implementation of the comparison operator. + bool is_self_equivalent(const Key &k) const { + // Note: this works for both boolean and three-way comparators. + return comp()(k, k) == 0; + } + // If we can't compare `t` with itself, returns true unconditionally. + template <typename T> + bool is_self_equivalent(const T &) const { + return true; + } + + public: + using Base::Base; + checked_compare(Compare comp) : Base(std::move(comp)) {} // NOLINT + + // Allow converting to Compare for use in key_comp()/value_comp(). + explicit operator Compare() const { return comp(); } + + template <typename T, typename U, + absl::enable_if_t< + std::is_same<bool, compare_result_t<Compare, T, U>>::value, + int> = 0> + bool operator()(const T &lhs, const U &rhs) const { + // NOTE: if any of these assertions fail, then the comparator does not + // establish a strict-weak-ordering (see + // https://en.cppreference.com/w/cpp/named_req/Compare). + assert(is_self_equivalent(lhs)); + assert(is_self_equivalent(rhs)); + const bool lhs_comp_rhs = comp()(lhs, rhs); + assert(!lhs_comp_rhs || !comp()(rhs, lhs)); + return lhs_comp_rhs; + } + + template < + typename T, typename U, + absl::enable_if_t<std::is_convertible<compare_result_t<Compare, T, U>, + absl::weak_ordering>::value, + int> = 0> + absl::weak_ordering operator()(const T &lhs, const U &rhs) const { + // NOTE: if any of these assertions fail, then the comparator does not + // establish a strict-weak-ordering (see + // https://en.cppreference.com/w/cpp/named_req/Compare). + assert(is_self_equivalent(lhs)); + assert(is_self_equivalent(rhs)); + const absl::weak_ordering lhs_comp_rhs = comp()(lhs, rhs); +#ifndef NDEBUG + const absl::weak_ordering rhs_comp_lhs = comp()(rhs, lhs); + if (lhs_comp_rhs > 0) { + assert(rhs_comp_lhs < 0 && "lhs_comp_rhs > 0 -> rhs_comp_lhs < 0"); + } else if (lhs_comp_rhs == 0) { + assert(rhs_comp_lhs == 0 && "lhs_comp_rhs == 0 -> rhs_comp_lhs == 0"); + } else { + assert(rhs_comp_lhs > 0 && "lhs_comp_rhs < 0 -> rhs_comp_lhs > 0"); + } +#endif + return lhs_comp_rhs; + } + }; + using type = absl::conditional_t< + std::is_base_of<BtreeTestOnlyCheckedCompareOptOutBase, Compare>::value, + Compare, checked_compare>; }; template <> -struct key_compare_to_adapter<std::less<std::string>> { +struct key_compare_adapter<std::less<std::string>, std::string> { using type = StringBtreeDefaultLess; }; template <> -struct key_compare_to_adapter<std::greater<std::string>> { +struct key_compare_adapter<std::greater<std::string>, std::string> { using type = StringBtreeDefaultGreater; }; template <> -struct key_compare_to_adapter<std::less<absl::string_view>> { +struct key_compare_adapter<std::less<absl::string_view>, absl::string_view> { using type = StringBtreeDefaultLess; }; template <> -struct key_compare_to_adapter<std::greater<absl::string_view>> { +struct key_compare_adapter<std::greater<absl::string_view>, absl::string_view> { using type = StringBtreeDefaultGreater; }; template <> -struct key_compare_to_adapter<std::less<absl::Cord>> { +struct key_compare_adapter<std::less<absl::Cord>, absl::Cord> { using type = StringBtreeDefaultLess; }; template <> -struct key_compare_to_adapter<std::greater<absl::Cord>> { +struct key_compare_adapter<std::greater<absl::Cord>, absl::Cord> { using type = StringBtreeDefaultGreater; }; @@ -214,19 +328,70 @@ struct prefers_linear_node_search< T, absl::void_t<typename T::absl_btree_prefer_linear_node_search>> : T::absl_btree_prefer_linear_node_search {}; +template <typename Compare, typename Key> +constexpr bool compare_has_valid_result_type() { + using compare_result_type = compare_result_t<Compare, Key, Key>; + return std::is_same<compare_result_type, bool>::value || + std::is_convertible<compare_result_type, absl::weak_ordering>::value; +} + +template <typename original_key_compare, typename value_type> +class map_value_compare { + template <typename Params> + friend class btree; + + // Note: this `protected` is part of the API of std::map::value_compare. See + // https://en.cppreference.com/w/cpp/container/map/value_compare. + protected: + explicit map_value_compare(original_key_compare c) : comp(std::move(c)) {} + + original_key_compare comp; // NOLINT + + public: + auto operator()(const value_type &lhs, const value_type &rhs) const + -> decltype(comp(lhs.first, rhs.first)) { + return comp(lhs.first, rhs.first); + } +}; + template <typename Key, typename Compare, typename Alloc, int TargetNodeSize, - bool Multi, typename SlotPolicy> + bool IsMulti, bool IsMap, typename SlotPolicy> struct common_params { + using original_key_compare = Compare; + // If Compare is a common comparator for a string-like type, then we adapt it // to use heterogeneous lookup and to be a key-compare-to comparator. - using key_compare = typename key_compare_to_adapter<Compare>::type; + // We also adapt the comparator to diagnose invalid comparators in debug mode. + // We disable this when `Compare` is invalid in a way that will cause + // adaptation to fail (having invalid return type) so that we can give a + // better compilation failure in static_assert_validation. If we don't do + // this, then there will be cascading compilation failures that are confusing + // for users. + using key_compare = + absl::conditional_t<!compare_has_valid_result_type<Compare, Key>(), + Compare, + typename key_compare_adapter<Compare, Key>::type>; + + static constexpr bool kIsKeyCompareStringAdapted = + std::is_same<key_compare, StringBtreeDefaultLess>::value || + std::is_same<key_compare, StringBtreeDefaultGreater>::value; + static constexpr bool kIsKeyCompareTransparent = + IsTransparent<original_key_compare>::value || + kIsKeyCompareStringAdapted; + static constexpr bool kEnableGenerations = +#ifdef ABSL_BTREE_ENABLE_GENERATIONS + true; +#else + false; +#endif + // A type which indicates if we have a key-compare-to functor or a plain old // key-compare functor. using is_key_compare_to = btree_is_key_compare_to<key_compare, Key>; using allocator_type = Alloc; using key_type = Key; - using size_type = std::make_signed<size_t>::type; + using size_type = size_t; using difference_type = ptrdiff_t; using slot_policy = SlotPolicy; @@ -238,6 +403,12 @@ struct common_params { using reference = value_type &; using const_reference = const value_type &; + using value_compare = + absl::conditional_t<IsMap, + map_value_compare<original_key_compare, value_type>, + original_key_compare>; + using is_map_container = std::integral_constant<bool, IsMap>; + // For the given lookup key type, returns whether we can have multiple // equivalent keys in the btree. If this is a multi-container, then we can. // Otherwise, we can have multiple equivalent keys only if all of the @@ -248,27 +419,25 @@ struct common_params { // that we know has the same equivalence classes for all lookup types. template <typename LookupKey> constexpr static bool can_have_multiple_equivalent_keys() { - return Multi || - (IsTransparent<key_compare>::value && - !std::is_same<LookupKey, Key>::value && - !std::is_same<key_compare, StringBtreeDefaultLess>::value && - !std::is_same<key_compare, StringBtreeDefaultGreater>::value); + return IsMulti || (IsTransparent<key_compare>::value && + !std::is_same<LookupKey, Key>::value && + !kIsKeyCompareStringAdapted); } enum { kTargetNodeSize = TargetNodeSize, - // Upper bound for the available space for values. This is largest for leaf + // Upper bound for the available space for slots. This is largest for leaf // nodes, which have overhead of at least a pointer + 4 bytes (for storing // 3 field_types and an enum). - kNodeValueSpace = + kNodeSlotSpace = TargetNodeSize - /*minimum overhead=*/(sizeof(void *) + 4), }; - // This is an integral type large enough to hold as many - // ValueSize-values as will fit a node of TargetNodeSize bytes. + // This is an integral type large enough to hold as many slots as will fit a + // node of TargetNodeSize bytes. using node_count_type = - absl::conditional_t<(kNodeValueSpace / sizeof(value_type) > + absl::conditional_t<(kNodeSlotSpace / sizeof(slot_type) > (std::numeric_limits<uint8_t>::max)()), uint16_t, uint8_t>; // NOLINT @@ -291,116 +460,10 @@ struct common_params { slot_policy::destroy(alloc, slot); } static void transfer(Alloc *alloc, slot_type *new_slot, slot_type *old_slot) { - construct(alloc, new_slot, old_slot); - destroy(alloc, old_slot); - } - static void swap(Alloc *alloc, slot_type *a, slot_type *b) { - slot_policy::swap(alloc, a, b); - } - static void move(Alloc *alloc, slot_type *src, slot_type *dest) { - slot_policy::move(alloc, src, dest); - } -}; - -// A parameters structure for holding the type parameters for a btree_map. -// Compare and Alloc should be nothrow copy-constructible. -template <typename Key, typename Data, typename Compare, typename Alloc, - int TargetNodeSize, bool Multi> -struct map_params : common_params<Key, Compare, Alloc, TargetNodeSize, Multi, - map_slot_policy<Key, Data>> { - using super_type = typename map_params::common_params; - using mapped_type = Data; - // This type allows us to move keys when it is safe to do so. It is safe - // for maps in which value_type and mutable_value_type are layout compatible. - using slot_policy = typename super_type::slot_policy; - using slot_type = typename super_type::slot_type; - using value_type = typename super_type::value_type; - using init_type = typename super_type::init_type; - - using key_compare = typename super_type::key_compare; - // Inherit from key_compare for empty base class optimization. - struct value_compare : private key_compare { - value_compare() = default; - explicit value_compare(const key_compare &cmp) : key_compare(cmp) {} - - template <typename T, typename U> - auto operator()(const T &left, const U &right) const - -> decltype(std::declval<key_compare>()(left.first, right.first)) { - return key_compare::operator()(left.first, right.first); - } - }; - using is_map_container = std::true_type; - - template <typename V> - static auto key(const V &value) -> decltype(value.first) { - return value.first; - } - static const Key &key(const slot_type *s) { return slot_policy::key(s); } - static const Key &key(slot_type *s) { return slot_policy::key(s); } - // For use in node handle. - static auto mutable_key(slot_type *s) - -> decltype(slot_policy::mutable_key(s)) { - return slot_policy::mutable_key(s); - } - static mapped_type &value(value_type *value) { return value->second; } -}; - -// This type implements the necessary functions from the -// absl::container_internal::slot_type interface. -template <typename Key> -struct set_slot_policy { - using slot_type = Key; - using value_type = Key; - using mutable_value_type = Key; - - static value_type &element(slot_type *slot) { return *slot; } - static const value_type &element(const slot_type *slot) { return *slot; } - - template <typename Alloc, class... Args> - static void construct(Alloc *alloc, slot_type *slot, Args &&... args) { - absl::allocator_traits<Alloc>::construct(*alloc, slot, - std::forward<Args>(args)...); - } - - template <typename Alloc> - static void construct(Alloc *alloc, slot_type *slot, slot_type *other) { - absl::allocator_traits<Alloc>::construct(*alloc, slot, std::move(*other)); - } - - template <typename Alloc> - static void destroy(Alloc *alloc, slot_type *slot) { - absl::allocator_traits<Alloc>::destroy(*alloc, slot); - } - - template <typename Alloc> - static void swap(Alloc * /*alloc*/, slot_type *a, slot_type *b) { - using std::swap; - swap(*a, *b); - } - - template <typename Alloc> - static void move(Alloc * /*alloc*/, slot_type *src, slot_type *dest) { - *dest = std::move(*src); + slot_policy::transfer(alloc, new_slot, old_slot); } }; -// A parameters structure for holding the type parameters for a btree_set. -// Compare and Alloc should be nothrow copy-constructible. -template <typename Key, typename Compare, typename Alloc, int TargetNodeSize, - bool Multi> -struct set_params : common_params<Key, Compare, Alloc, TargetNodeSize, Multi, - set_slot_policy<Key>> { - using value_type = Key; - using slot_type = typename set_params::common_params::slot_type; - using value_compare = typename set_params::common_params::key_compare; - using is_map_container = std::false_type; - - template <typename V> - static const V &key(const V &value) { return value; } - static const Key &key(const slot_type *slot) { return *slot; } - static const Key &key(slot_type *slot) { return *slot; } -}; - // An adapter class that converts a lower-bound compare into an upper-bound // compare. Note: there is no need to make a version of this adapter specialized // for key-compare-to functors because the upper-bound (the first value greater @@ -435,8 +498,8 @@ struct SearchResult { template <typename V> struct SearchResult<V, false> { SearchResult() {} - explicit SearchResult(V value) : value(value) {} - SearchResult(V value, MatchKind /*match*/) : value(value) {} + explicit SearchResult(V v) : value(v) {} + SearchResult(V v, MatchKind /*match*/) : value(v) {} V value; @@ -453,6 +516,7 @@ class btree_node { using field_type = typename Params::node_count_type; using allocator_type = typename Params::allocator_type; using slot_type = typename Params::slot_type; + using original_key_compare = typename Params::original_key_compare; public: using params_type = Params; @@ -474,21 +538,28 @@ class btree_node { // - Otherwise, choose binary. // TODO(ezb): Might make sense to add condition(s) based on node-size. using use_linear_search = std::integral_constant< - bool, - has_linear_node_search_preference<key_compare>::value - ? prefers_linear_node_search<key_compare>::value - : has_linear_node_search_preference<key_type>::value + bool, has_linear_node_search_preference<original_key_compare>::value + ? prefers_linear_node_search<original_key_compare>::value + : has_linear_node_search_preference<key_type>::value ? prefers_linear_node_search<key_type>::value : std::is_arithmetic<key_type>::value && - (std::is_same<std::less<key_type>, key_compare>::value || + (std::is_same<std::less<key_type>, + original_key_compare>::value || std::is_same<std::greater<key_type>, - key_compare>::value)>; + original_key_compare>::value)>; - // This class is organized by gtl::Layout as if it had the following - // structure: + // This class is organized by absl::container_internal::Layout as if it had + // the following structure: // // A pointer to the node's parent. // btree_node *parent; // + // // When ABSL_BTREE_ENABLE_GENERATIONS is defined, we also have a + // // generation integer in order to check that when iterators are + // // used, they haven't been invalidated already. Only the generation on + // // the root is used, but we have one on each node because whether a node + // // is root or not can change. + // uint32_t generation; + // // // The position of the node in the node's parent. // field_type position; // // The index of the first populated value in `values`. @@ -535,23 +606,27 @@ class btree_node { btree_node() = default; private: - using layout_type = absl::container_internal::Layout<btree_node *, field_type, - slot_type, btree_node *>; + using layout_type = + absl::container_internal::Layout<btree_node *, uint32_t, field_type, + slot_type, btree_node *>; constexpr static size_type SizeWithNSlots(size_type n) { - return layout_type(/*parent*/ 1, - /*position, start, finish, max_count*/ 4, - /*slots*/ n, - /*children*/ 0) + return layout_type( + /*parent*/ 1, + /*generation*/ params_type::kEnableGenerations ? 1 : 0, + /*position, start, finish, max_count*/ 4, + /*slots*/ n, + /*children*/ 0) .AllocSize(); } - // A lower bound for the overhead of fields other than values in a leaf node. + // A lower bound for the overhead of fields other than slots in a leaf node. constexpr static size_type MinimumOverhead() { - return SizeWithNSlots(1) - sizeof(value_type); + return SizeWithNSlots(1) - sizeof(slot_type); } // Compute how many values we can fit onto a leaf node taking into account // padding. - constexpr static size_type NodeTargetSlots(const int begin, const int end) { + constexpr static size_type NodeTargetSlots(const size_type begin, + const size_type end) { return begin == end ? begin : SizeWithNSlots((begin + end) / 2 + 1) > params_type::kTargetNodeSize @@ -580,16 +655,20 @@ class btree_node { // Leaves can have less than kNodeSlots values. constexpr static layout_type LeafLayout(const int slot_count = kNodeSlots) { - return layout_type(/*parent*/ 1, - /*position, start, finish, max_count*/ 4, - /*slots*/ slot_count, - /*children*/ 0); + return layout_type( + /*parent*/ 1, + /*generation*/ params_type::kEnableGenerations ? 1 : 0, + /*position, start, finish, max_count*/ 4, + /*slots*/ slot_count, + /*children*/ 0); } constexpr static layout_type InternalLayout() { - return layout_type(/*parent*/ 1, - /*position, start, finish, max_count*/ 4, - /*slots*/ kNodeSlots, - /*children*/ kNodeSlots + 1); + return layout_type( + /*parent*/ 1, + /*generation*/ params_type::kEnableGenerations ? 1 : 0, + /*position, start, finish, max_count*/ 4, + /*slots*/ kNodeSlots, + /*children*/ kNodeSlots + 1); } constexpr static size_type LeafSize(const int slot_count = kNodeSlots) { return LeafLayout(slot_count).AllocSize(); @@ -603,44 +682,47 @@ class btree_node { template <size_type N> inline typename layout_type::template ElementType<N> *GetField() { // We assert that we don't read from values that aren't there. - assert(N < 3 || !leaf()); + assert(N < 4 || is_internal()); return InternalLayout().template Pointer<N>(reinterpret_cast<char *>(this)); } template <size_type N> inline const typename layout_type::template ElementType<N> *GetField() const { - assert(N < 3 || !leaf()); + assert(N < 4 || is_internal()); return InternalLayout().template Pointer<N>( reinterpret_cast<const char *>(this)); } void set_parent(btree_node *p) { *GetField<0>() = p; } - field_type &mutable_finish() { return GetField<1>()[2]; } - slot_type *slot(int i) { return &GetField<2>()[i]; } + field_type &mutable_finish() { return GetField<2>()[2]; } + slot_type *slot(int i) { return &GetField<3>()[i]; } slot_type *start_slot() { return slot(start()); } slot_type *finish_slot() { return slot(finish()); } - const slot_type *slot(int i) const { return &GetField<2>()[i]; } - void set_position(field_type v) { GetField<1>()[0] = v; } - void set_start(field_type v) { GetField<1>()[1] = v; } - void set_finish(field_type v) { GetField<1>()[2] = v; } + const slot_type *slot(int i) const { return &GetField<3>()[i]; } + void set_position(field_type v) { GetField<2>()[0] = v; } + void set_start(field_type v) { GetField<2>()[1] = v; } + void set_finish(field_type v) { GetField<2>()[2] = v; } // This method is only called by the node init methods. - void set_max_count(field_type v) { GetField<1>()[3] = v; } + void set_max_count(field_type v) { GetField<2>()[3] = v; } public: // Whether this is a leaf node or not. This value doesn't change after the // node is created. - bool leaf() const { return GetField<1>()[3] != kInternalNodeMaxCount; } + bool is_leaf() const { return GetField<2>()[3] != kInternalNodeMaxCount; } + // Whether this is an internal node or not. This value doesn't change after + // the node is created. + bool is_internal() const { return !is_leaf(); } // Getter for the position of this node in its parent. - field_type position() const { return GetField<1>()[0]; } + field_type position() const { return GetField<2>()[0]; } // Getter for the offset of the first value in the `values` array. field_type start() const { - // TODO(ezb): when floating storage is implemented, return GetField<1>()[1]; - assert(GetField<1>()[1] == 0); + // TODO(ezb): when floating storage is implemented, return GetField<2>()[1]; + assert(GetField<2>()[1] == 0); return 0; } // Getter for the offset after the last value in the `values` array. - field_type finish() const { return GetField<1>()[2]; } + field_type finish() const { return GetField<2>()[2]; } // Getters for the number of values stored in this node. field_type count() const { @@ -650,7 +732,7 @@ class btree_node { field_type max_count() const { // Internal nodes have max_count==kInternalNodeMaxCount. // Leaf nodes have max_count in [1, kNodeSlots]. - const field_type max_count = GetField<1>()[3]; + const field_type max_count = GetField<2>()[3]; return max_count == field_type{kInternalNodeMaxCount} ? field_type{kNodeSlots} : max_count; @@ -661,21 +743,44 @@ class btree_node { // Getter for whether the node is the root of the tree. The parent of the // root of the tree is the leftmost node in the tree which is guaranteed to // be a leaf. - bool is_root() const { return parent()->leaf(); } + bool is_root() const { return parent()->is_leaf(); } void make_root() { assert(parent()->is_root()); + set_generation(parent()->generation()); set_parent(parent()->parent()); } + // Gets the root node's generation integer, which is the one used by the tree. + uint32_t *get_root_generation() const { + assert(params_type::kEnableGenerations); + const btree_node *curr = this; + for (; !curr->is_root(); curr = curr->parent()) continue; + return const_cast<uint32_t *>(&curr->GetField<1>()[0]); + } + + // Returns the generation for iterator validation. + uint32_t generation() const { + return params_type::kEnableGenerations ? *get_root_generation() : 0; + } + // Updates generation. Should only be called on a root node or during node + // initialization. + void set_generation(uint32_t generation) { + if (params_type::kEnableGenerations) GetField<1>()[0] = generation; + } + // Updates the generation. We do this whenever the node is mutated. + void next_generation() { + if (params_type::kEnableGenerations) ++*get_root_generation(); + } + // Getters for the key/value at position i in the node. const key_type &key(int i) const { return params_type::key(slot(i)); } reference value(int i) { return params_type::element(slot(i)); } const_reference value(int i) const { return params_type::element(slot(i)); } // Getters/setter for the child at position i in the node. - btree_node *child(int i) const { return GetField<3>()[i]; } + btree_node *child(int i) const { return GetField<4>()[i]; } btree_node *start_child() const { return child(start()); } - btree_node *&mutable_child(int i) { return GetField<3>()[i]; } + btree_node *&mutable_child(int i) { return GetField<4>()[i]; } void clear_child(int i) { absl::container_internal::SanitizerPoisonObject(&mutable_child(i)); } @@ -832,7 +937,8 @@ class btree_node { void merge(btree_node *src, allocator_type *alloc); // Node allocation/deletion routines. - void init_leaf(btree_node *parent, int max_count) { + void init_leaf(int max_count, btree_node *parent) { + set_generation(0); set_parent(parent); set_position(0); set_start(0); @@ -842,7 +948,7 @@ class btree_node { start_slot(), max_count * sizeof(slot_type)); } void init_internal(btree_node *parent) { - init_leaf(parent, kNodeSlots); + init_leaf(kNodeSlots, parent); // Set `max_count` to a sentinel value to indicate that this node is // internal. set_max_count(kInternalNodeMaxCount); @@ -861,15 +967,18 @@ class btree_node { private: template <typename... Args> void value_init(const field_type i, allocator_type *alloc, Args &&... args) { + next_generation(); absl::container_internal::SanitizerUnpoisonObject(slot(i)); params_type::construct(alloc, slot(i), std::forward<Args>(args)...); } void value_destroy(const field_type i, allocator_type *alloc) { + next_generation(); params_type::destroy(alloc, slot(i)); absl::container_internal::SanitizerPoisonObject(slot(i)); } void value_destroy_n(const field_type i, const field_type n, allocator_type *alloc) { + next_generation(); for (slot_type *s = slot(i), *end = slot(i + n); s != end; ++s) { params_type::destroy(alloc, s); absl::container_internal::SanitizerPoisonObject(s); @@ -885,6 +994,7 @@ class btree_node { // Transfers value from slot `src_i` in `src_node` to slot `dest_i` in `this`. void transfer(const size_type dest_i, const size_type src_i, btree_node *src_node, allocator_type *alloc) { + next_generation(); transfer(slot(dest_i), src_node->slot(src_i), alloc); } @@ -893,6 +1003,7 @@ class btree_node { void transfer_n(const size_type n, const size_type dest_i, const size_type src_i, btree_node *src_node, allocator_type *alloc) { + next_generation(); for (slot_type *src = src_node->slot(src_i), *end = src + n, *dest = slot(dest_i); src != end; ++src, ++dest) { @@ -905,6 +1016,7 @@ class btree_node { void transfer_n_backward(const size_type n, const size_type dest_i, const size_type src_i, btree_node *src_node, allocator_type *alloc) { + next_generation(); for (slot_type *src = src_node->slot(src_i + n - 1), *end = src - n, *dest = slot(dest_i + n - 1); src != end; --src, --dest) { @@ -915,13 +1027,13 @@ class btree_node { template <typename P> friend class btree; template <typename N, typename R, typename P> - friend struct btree_iterator; + friend class btree_iterator; friend class BtreeNodePeer; + friend struct btree_access; }; template <typename Node, typename Reference, typename Pointer> -struct btree_iterator { - private: +class btree_iterator { using key_type = typename Node::key_type; using size_type = typename Node::size_type; using params_type = typename Node::params_type; @@ -949,9 +1061,15 @@ struct btree_iterator { using reference = Reference; using iterator_category = std::bidirectional_iterator_tag; - btree_iterator() : node(nullptr), position(-1) {} - explicit btree_iterator(Node *n) : node(n), position(n->start()) {} - btree_iterator(Node *n, int p) : node(n), position(p) {} + btree_iterator() : btree_iterator(nullptr, -1) {} + explicit btree_iterator(Node *n) : btree_iterator(n, n->start()) {} + btree_iterator(Node *n, int p) : node_(n), position_(p) { +#ifdef ABSL_BTREE_ENABLE_GENERATIONS + // Use `~uint32_t{}` as a sentinel value for iterator generations so it + // doesn't match the initial value for the actual generation. + generation_ = n != nullptr ? n->generation() : ~uint32_t{}; +#endif + } // NOTE: this SFINAE allows for implicit conversions from iterator to // const_iterator, but it specifically avoids hiding the copy constructor so @@ -962,58 +1080,32 @@ struct btree_iterator { std::is_same<btree_iterator, const_iterator>::value, int> = 0> btree_iterator(const btree_iterator<N, R, P> other) // NOLINT - : node(other.node), position(other.position) {} - - private: - // This SFINAE allows explicit conversions from const_iterator to - // iterator, but also avoids hiding the copy constructor. - // NOTE: the const_cast is safe because this constructor is only called by - // non-const methods and the container owns the nodes. - template <typename N, typename R, typename P, - absl::enable_if_t< - std::is_same<btree_iterator<N, R, P>, const_iterator>::value && - std::is_same<btree_iterator, iterator>::value, - int> = 0> - explicit btree_iterator(const btree_iterator<N, R, P> other) - : node(const_cast<node_type *>(other.node)), position(other.position) {} - - // Increment/decrement the iterator. - void increment() { - if (node->leaf() && ++position < node->finish()) { - return; - } - increment_slow(); - } - void increment_slow(); - - void decrement() { - if (node->leaf() && --position >= node->start()) { - return; - } - decrement_slow(); + : node_(other.node_), position_(other.position_) { +#ifdef ABSL_BTREE_ENABLE_GENERATIONS + generation_ = other.generation_; +#endif } - void decrement_slow(); - public: bool operator==(const iterator &other) const { - return node == other.node && position == other.position; + return node_ == other.node_ && position_ == other.position_; } bool operator==(const const_iterator &other) const { - return node == other.node && position == other.position; + return node_ == other.node_ && position_ == other.position_; } bool operator!=(const iterator &other) const { - return node != other.node || position != other.position; + return node_ != other.node_ || position_ != other.position_; } bool operator!=(const const_iterator &other) const { - return node != other.node || position != other.position; + return node_ != other.node_ || position_ != other.position_; } // Accessors for the key/value the iterator is pointing at. reference operator*() const { - ABSL_HARDENING_ASSERT(node != nullptr); - ABSL_HARDENING_ASSERT(node->start() <= position); - ABSL_HARDENING_ASSERT(node->finish() > position); - return node->value(position); + ABSL_HARDENING_ASSERT(node_ != nullptr); + ABSL_HARDENING_ASSERT(node_->start() <= position_); + ABSL_HARDENING_ASSERT(node_->finish() > position_); + assert_valid_generation(); + return node_->value(position_); } pointer operator->() const { return &operator*(); } @@ -1051,23 +1143,84 @@ struct btree_iterator { friend class btree_multiset_container; template <typename TreeType, typename CheckerType> friend class base_checker; + friend struct btree_access; - const key_type &key() const { return node->key(position); } - slot_type *slot() { return node->slot(position); } + // This SFINAE allows explicit conversions from const_iterator to + // iterator, but also avoids hiding the copy constructor. + // NOTE: the const_cast is safe because this constructor is only called by + // non-const methods and the container owns the nodes. + template <typename N, typename R, typename P, + absl::enable_if_t< + std::is_same<btree_iterator<N, R, P>, const_iterator>::value && + std::is_same<btree_iterator, iterator>::value, + int> = 0> + explicit btree_iterator(const btree_iterator<N, R, P> other) + : node_(const_cast<node_type *>(other.node_)), + position_(other.position_) { +#ifdef ABSL_BTREE_ENABLE_GENERATIONS + generation_ = other.generation_; +#endif + } + + // Increment/decrement the iterator. + void increment() { + assert_valid_generation(); + if (node_->is_leaf() && ++position_ < node_->finish()) { + return; + } + increment_slow(); + } + void increment_slow(); + + void decrement() { + assert_valid_generation(); + if (node_->is_leaf() && --position_ >= node_->start()) { + return; + } + decrement_slow(); + } + void decrement_slow(); + + // Updates the generation. For use internally right before we return an + // iterator to the user. + void update_generation() { +#ifdef ABSL_BTREE_ENABLE_GENERATIONS + if (node_ != nullptr) generation_ = node_->generation(); +#endif + } + + const key_type &key() const { return node_->key(position_); } + decltype(std::declval<Node *>()->slot(0)) slot() { + return node_->slot(position_); + } + + void assert_valid_generation() const { +#ifdef ABSL_BTREE_ENABLE_GENERATIONS + if (node_ != nullptr && node_->generation() != generation_) { + ABSL_INTERNAL_LOG( + FATAL, + "Attempting to use an invalidated iterator. The corresponding b-tree " + "container has been mutated since this iterator was constructed."); + } +#endif + } // The node in the tree the iterator is pointing at. - Node *node; + Node *node_; // The position within the node of the tree the iterator is pointing at. // NOTE: this is an int rather than a field_type because iterators can point // to invalid positions (such as -1) in certain circumstances. - int position; + int position_; +#ifdef ABSL_BTREE_ENABLE_GENERATIONS + // Used to check that the iterator hasn't been invalidated. + uint32_t generation_; +#endif }; template <typename Params> class btree { using node_type = btree_node<Params>; using is_key_compare_to = typename Params::is_key_compare_to; - using init_type = typename Params::init_type; using field_type = typename node_type::field_type; // We use a static empty node for the root/leftmost/rightmost of empty btrees @@ -1075,6 +1228,9 @@ class btree { struct alignas(node_type::Alignment()) EmptyNodeType : node_type { using field_type = typename node_type::field_type; node_type *parent; +#ifdef ABSL_BTREE_ENABLE_GENERATIONS + uint32_t generation = 0; +#endif field_type position = 0; field_type start = 0; field_type finish = 0; @@ -1129,6 +1285,7 @@ class btree { using size_type = typename Params::size_type; using difference_type = typename Params::difference_type; using key_compare = typename Params::key_compare; + using original_key_compare = typename Params::original_key_compare; using value_compare = typename Params::value_compare; using allocator_type = typename Params::allocator_type; using reference = typename Params::reference; @@ -1147,14 +1304,6 @@ class btree { using slot_type = typename Params::slot_type; private: - // For use in copy_or_move_values_in_order. - const value_type &maybe_move_from_iterator(const_iterator it) { return *it; } - value_type &&maybe_move_from_iterator(iterator it) { - // This is a destructive operation on the other container so it's safe for - // us to const_cast and move from the keys here even if it's a set. - return std::move(const_cast<value_type &>(*it)); - } - // Copies or moves (depending on the template parameter) the values in // other into this btree in their order in other. This btree must be empty // before this method is called. This method is used in copy construction, @@ -1167,7 +1316,7 @@ class btree { public: btree(const key_compare &comp, const allocator_type &alloc) - : root_(comp, alloc, EmptyNode()), rightmost_(EmptyNode()), size_(0) {} + : root_(EmptyNode()), rightmost_(comp, alloc, EmptyNode()), size_(0) {} btree(const btree &other) : btree(other, other.allocator()) {} btree(const btree &other, const allocator_type &alloc) @@ -1175,10 +1324,10 @@ class btree { copy_or_move_values_in_order(other); } btree(btree &&other) noexcept - : root_(std::move(other.root_)), - rightmost_(absl::exchange(other.rightmost_, EmptyNode())), + : root_(absl::exchange(other.root_, EmptyNode())), + rightmost_(std::move(other.rightmost_)), size_(absl::exchange(other.size_, 0)) { - other.mutable_root() = EmptyNode(); + other.mutable_rightmost() = EmptyNode(); } btree(btree &&other, const allocator_type &alloc) : btree(other.key_comp(), alloc) { @@ -1203,9 +1352,9 @@ class btree { iterator begin() { return iterator(leftmost()); } const_iterator begin() const { return const_iterator(leftmost()); } - iterator end() { return iterator(rightmost_, rightmost_->finish()); } + iterator end() { return iterator(rightmost(), rightmost()->finish()); } const_iterator end() const { - return const_iterator(rightmost_, rightmost_->finish()); + return const_iterator(rightmost(), rightmost()->finish()); } reverse_iterator rbegin() { return reverse_iterator(end()); } const_reverse_iterator rbegin() const { @@ -1331,14 +1480,16 @@ class btree { void swap(btree &other); const key_compare &key_comp() const noexcept { - return root_.template get<0>(); + return rightmost_.template get<0>(); } template <typename K1, typename K2> bool compare_keys(const K1 &a, const K2 &b) const { return compare_internal::compare_result_as_less_than(key_comp()(a, b)); } - value_compare value_comp() const { return value_compare(key_comp()); } + value_compare value_comp() const { + return value_compare(original_key_compare(key_comp())); + } // Verifies the structure of the btree. void verify() const; @@ -1376,6 +1527,7 @@ class btree { } // The total number of bytes used by the btree. + // TODO(b/169338300): update to support node_btree_*. size_type bytes_used() const { node_stats stats = internal_stats(root()); if (stats.leaf_nodes == 1 && stats.internal_nodes == 0) { @@ -1419,11 +1571,20 @@ class btree { allocator_type get_allocator() const { return allocator(); } private: + friend struct btree_access; + // Internal accessor routines. - node_type *root() { return root_.template get<2>(); } - const node_type *root() const { return root_.template get<2>(); } - node_type *&mutable_root() noexcept { return root_.template get<2>(); } - key_compare *mutable_key_comp() noexcept { return &root_.template get<0>(); } + node_type *root() { return root_; } + const node_type *root() const { return root_; } + node_type *&mutable_root() noexcept { return root_; } + node_type *rightmost() { return rightmost_.template get<2>(); } + const node_type *rightmost() const { return rightmost_.template get<2>(); } + node_type *&mutable_rightmost() noexcept { + return rightmost_.template get<2>(); + } + key_compare *mutable_key_comp() noexcept { + return &rightmost_.template get<0>(); + } // The leftmost node is stored as the parent of the root node. node_type *leftmost() { return root()->parent(); } @@ -1431,10 +1592,10 @@ class btree { // Allocator routines. allocator_type *mutable_allocator() noexcept { - return &root_.template get<1>(); + return &rightmost_.template get<1>(); } const allocator_type &allocator() const noexcept { - return root_.template get<1>(); + return rightmost_.template get<1>(); } // Allocates a correctly aligned node of at least size bytes using the @@ -1453,12 +1614,12 @@ class btree { } node_type *new_leaf_node(node_type *parent) { node_type *n = allocate(node_type::LeafSize()); - n->init_leaf(parent, kNodeSlots); + n->init_leaf(kNodeSlots, parent); return n; } node_type *new_leaf_root_node(const int max_count) { node_type *n = allocate(node_type::LeafSize(max_count)); - n->init_leaf(/*parent=*/n, max_count); + n->init_leaf(max_count, /*parent=*/n); return n; } @@ -1482,10 +1643,10 @@ class btree { void try_shrink(); iterator internal_end(iterator iter) { - return iter.node != nullptr ? iter : end(); + return iter.node_ != nullptr ? iter : end(); } const_iterator internal_end(const_iterator iter) const { - return iter.node != nullptr ? iter : end(); + return iter.node_ != nullptr ? iter : end(); } // Emplaces a value into the btree immediately before iter. Requires that @@ -1495,9 +1656,8 @@ class btree { // Returns an iterator pointing to the first value >= the value "iter" is // pointing at. Note that "iter" might be pointing to an invalid location such - // as iter.position == iter.node->finish(). This routine simply moves iter up - // in the tree to a valid location. - // Requires: iter.node is non-null. + // as iter.position_ == iter.node_->finish(). This routine simply moves iter + // up in the tree to a valid location. Requires: iter.node_ is non-null. template <typename IterType> static IterType internal_last(IterType iter); @@ -1533,7 +1693,7 @@ class btree { if (node == nullptr || (node == root() && empty())) { return node_stats(0, 0); } - if (node->leaf()) { + if (node->is_leaf()) { return node_stats(1, 0); } node_stats res(0, 1); @@ -1543,15 +1703,14 @@ class btree { return res; } - // We use compressed tuple in order to save space because key_compare and - // allocator_type are usually empty. - absl::container_internal::CompressedTuple<key_compare, allocator_type, - node_type *> - root_; + node_type *root_; // A pointer to the rightmost node. Note that the leftmost node is stored as - // the root's parent. - node_type *rightmost_; + // the root's parent. We use compressed tuple in order to save space because + // key_compare and allocator_type are usually empty. + absl::container_internal::CompressedTuple<key_compare, allocator_type, + node_type *> + rightmost_; // Number of values. size_type size_; @@ -1575,8 +1734,8 @@ inline void btree_node<P>::emplace_value(const size_type i, value_init(i, alloc, std::forward<Args>(args)...); set_finish(finish() + 1); - if (!leaf() && finish() > i + 1) { - for (int j = finish(); j > i + 1; --j) { + if (is_internal() && finish() > i + 1) { + for (field_type j = finish(); j > i + 1; --j) { set_child(j, child(j - 1)); } clear_child(i + 1); @@ -1593,7 +1752,7 @@ inline void btree_node<P>::remove_values(const field_type i, const field_type src_i = i + to_erase; transfer_n(orig_finish - src_i, i, src_i, this, alloc); - if (!leaf()) { + if (is_internal()) { // Delete all children between begin and end. for (int j = 0; j < to_erase; ++j) { clear_and_delete(child(i + j + 1), alloc); @@ -1630,7 +1789,7 @@ void btree_node<P>::rebalance_right_to_left(const int to_move, right->transfer_n(right->count() - to_move, right->start(), right->start() + to_move, right, alloc); - if (!leaf()) { + if (is_internal()) { // Move the child pointers from the right to the left node. for (int i = 0; i < to_move; ++i) { init_child(finish() + i + 1, right->child(i)); @@ -1677,7 +1836,7 @@ void btree_node<P>::rebalance_left_to_right(const int to_move, // 4) Move the new delimiting value to the parent from the left node. parent()->transfer(position(), finish() - to_move, this, alloc); - if (!leaf()) { + if (is_internal()) { // Move the child pointers from the left to the right node. for (int i = right->finish(); i >= right->start(); --i) { right->init_child(i + to_move, right->child(i)); @@ -1723,7 +1882,7 @@ void btree_node<P>::split(const int insert_position, btree_node *dest, value_destroy(finish(), alloc); parent()->init_child(position() + 1, dest); - if (!leaf()) { + if (is_internal()) { for (int i = dest->start(), j = finish() + 1; i <= dest->finish(); ++i, ++j) { assert(child(j) != nullptr); @@ -1744,7 +1903,7 @@ void btree_node<P>::merge(btree_node *src, allocator_type *alloc) { // Move the values from the right to the left node. transfer_n(src->count(), finish() + 1, src->start(), src, alloc); - if (!leaf()) { + if (is_internal()) { // Move the child pointers from the right to the left node. for (int i = src->start(), j = finish() + 1; i <= src->finish(); ++i, ++j) { init_child(j, src->child(i)); @@ -1762,7 +1921,7 @@ void btree_node<P>::merge(btree_node *src, allocator_type *alloc) { template <typename P> void btree_node<P>::clear_and_delete(btree_node *node, allocator_type *alloc) { - if (node->leaf()) { + if (node->is_leaf()) { node->value_destroy_n(node->start(), node->count(), alloc); deallocate(LeafSize(node->max_count()), node, alloc); return; @@ -1776,7 +1935,15 @@ void btree_node<P>::clear_and_delete(btree_node *node, allocator_type *alloc) { btree_node *delete_root_parent = node->parent(); // Navigate to the leftmost leaf under node, and then delete upwards. - while (!node->leaf()) node = node->start_child(); + while (node->is_internal()) node = node->start_child(); +#ifdef ABSL_BTREE_ENABLE_GENERATIONS + // When generations are enabled, we delete the leftmost leaf last in case it's + // the parent of the root and we need to check whether it's a leaf before we + // can update the root's generation. + // TODO(ezb): if we change btree_node::is_root to check a bool inside the node + // instead of checking whether the parent is a leaf, we can remove this logic. + btree_node *leftmost_leaf = node; +#endif // Use `int` because `pos` needs to be able to hold `kNodeSlots+1`, which // isn't guaranteed to be a valid `field_type`. int pos = node->position(); @@ -1786,14 +1953,17 @@ void btree_node<P>::clear_and_delete(btree_node *node, allocator_type *alloc) { assert(pos <= parent->finish()); do { node = parent->child(pos); - if (!node->leaf()) { + if (node->is_internal()) { // Navigate to the leftmost leaf under node. - while (!node->leaf()) node = node->start_child(); + while (node->is_internal()) node = node->start_child(); pos = node->position(); parent = node->parent(); } node->value_destroy_n(node->start(), node->count(), alloc); - deallocate(LeafSize(node->max_count()), node, alloc); +#ifdef ABSL_BTREE_ENABLE_GENERATIONS + if (leftmost_leaf != node) +#endif + deallocate(LeafSize(node->max_count()), node, alloc); ++pos; } while (pos <= parent->finish()); @@ -1805,7 +1975,12 @@ void btree_node<P>::clear_and_delete(btree_node *node, allocator_type *alloc) { parent = node->parent(); node->value_destroy_n(node->start(), node->count(), alloc); deallocate(InternalSize(), node, alloc); - if (parent == delete_root_parent) return; + if (parent == delete_root_parent) { +#ifdef ABSL_BTREE_ENABLE_GENERATIONS + deallocate(LeafSize(leftmost_leaf->max_count()), leftmost_leaf, alloc); +#endif + return; + } ++pos; } while (pos > parent->finish()); } @@ -1815,49 +1990,49 @@ void btree_node<P>::clear_and_delete(btree_node *node, allocator_type *alloc) { // btree_iterator methods template <typename N, typename R, typename P> void btree_iterator<N, R, P>::increment_slow() { - if (node->leaf()) { - assert(position >= node->finish()); + if (node_->is_leaf()) { + assert(position_ >= node_->finish()); btree_iterator save(*this); - while (position == node->finish() && !node->is_root()) { - assert(node->parent()->child(node->position()) == node); - position = node->position(); - node = node->parent(); + while (position_ == node_->finish() && !node_->is_root()) { + assert(node_->parent()->child(node_->position()) == node_); + position_ = node_->position(); + node_ = node_->parent(); } // TODO(ezb): assert we aren't incrementing end() instead of handling. - if (position == node->finish()) { + if (position_ == node_->finish()) { *this = save; } } else { - assert(position < node->finish()); - node = node->child(position + 1); - while (!node->leaf()) { - node = node->start_child(); + assert(position_ < node_->finish()); + node_ = node_->child(position_ + 1); + while (node_->is_internal()) { + node_ = node_->start_child(); } - position = node->start(); + position_ = node_->start(); } } template <typename N, typename R, typename P> void btree_iterator<N, R, P>::decrement_slow() { - if (node->leaf()) { - assert(position <= -1); + if (node_->is_leaf()) { + assert(position_ <= -1); btree_iterator save(*this); - while (position < node->start() && !node->is_root()) { - assert(node->parent()->child(node->position()) == node); - position = node->position() - 1; - node = node->parent(); + while (position_ < node_->start() && !node_->is_root()) { + assert(node_->parent()->child(node_->position()) == node_); + position_ = node_->position() - 1; + node_ = node_->parent(); } // TODO(ezb): assert we aren't decrementing begin() instead of handling. - if (position < node->start()) { + if (position_ < node_->start()) { *this = save; } } else { - assert(position >= node->start()); - node = node->child(position); - while (!node->leaf()) { - node = node->child(node->finish()); + assert(position_ >= node_->start()); + node_ = node_->child(position_); + while (node_->is_internal()) { + node_ = node_->child(node_->finish()); } - position = node->finish() - 1; + position_ = node_->finish() - 1; } } @@ -1875,12 +2050,12 @@ void btree<P>::copy_or_move_values_in_order(Btree &other) { // values is the same order we'll store them in. auto iter = other.begin(); if (iter == other.end()) return; - insert_multi(maybe_move_from_iterator(iter)); + insert_multi(iter.slot()); ++iter; for (; iter != other.end(); ++iter) { // If the btree is not empty, we can just insert the new value at the end // of the tree. - internal_emplace(end(), maybe_move_from_iterator(iter)); + internal_emplace(end(), iter.slot()); } } @@ -1900,15 +2075,12 @@ constexpr bool btree<P>::static_assert_validation() { "target node size too large"); // Verify that key_compare returns an absl::{weak,strong}_ordering or bool. - using compare_result_type = - absl::result_of_t<key_compare(key_type, key_type)>; static_assert( - std::is_same<compare_result_type, bool>::value || - std::is_convertible<compare_result_type, absl::weak_ordering>::value, + compare_has_valid_result_type<key_compare, key_type>(), "key comparison function must return absl::{weak,strong}_ordering or " "bool."); - // Test the assumption made in setting kNodeValueSpace. + // Test the assumption made in setting kNodeSlotSpace. static_assert(node_type::MinimumOverhead() >= sizeof(void *) + 4, "node space assumption incorrect"); @@ -1962,7 +2134,7 @@ template <typename K, typename... Args> auto btree<P>::insert_unique(const K &key, Args &&... args) -> std::pair<iterator, bool> { if (empty()) { - mutable_root() = rightmost_ = new_leaf_root_node(1); + mutable_root() = mutable_rightmost() = new_leaf_root_node(1); } SearchResult<iterator, is_key_compare_to::value> res = internal_locate(key); @@ -1975,7 +2147,7 @@ auto btree<P>::insert_unique(const K &key, Args &&... args) } } else { iterator last = internal_last(iter); - if (last.node && !compare_keys(key, last.key())) { + if (last.node_ && !compare_keys(key, last.key())) { // The key already exists in the tree, do nothing. return {last, false}; } @@ -2020,8 +2192,11 @@ template <typename P> template <typename InputIterator> void btree<P>::insert_iterator_unique(InputIterator b, InputIterator e, char) { for (; b != e; ++b) { - init_type value(*b); - insert_hint_unique(end(), params_type::key(value), std::move(value)); + // Use a node handle to manage a temp slot. + auto node_handle = + CommonAccess::Construct<node_handle_type>(get_allocator(), *b); + slot_type *slot = CommonAccess::GetSlot(node_handle); + insert_hint_unique(end(), params_type::key(slot), slot); } } @@ -2029,11 +2204,11 @@ template <typename P> template <typename ValueType> auto btree<P>::insert_multi(const key_type &key, ValueType &&v) -> iterator { if (empty()) { - mutable_root() = rightmost_ = new_leaf_root_node(1); + mutable_root() = mutable_rightmost() = new_leaf_root_node(1); } iterator iter = internal_upper_bound(key); - if (iter.node == nullptr) { + if (iter.node_ == nullptr) { iter = end(); } return internal_emplace(iter, std::forward<ValueType>(v)); @@ -2093,15 +2268,15 @@ auto btree<P>::operator=(btree &&other) noexcept -> btree & { using std::swap; if (absl::allocator_traits< allocator_type>::propagate_on_container_copy_assignment::value) { - // Note: `root_` also contains the allocator and the key comparator. swap(root_, other.root_); + // Note: `rightmost_` also contains the allocator and the key comparator. swap(rightmost_, other.rightmost_); swap(size_, other.size_); } else { if (allocator() == other.allocator()) { swap(mutable_root(), other.mutable_root()); swap(*mutable_key_comp(), *other.mutable_key_comp()); - swap(rightmost_, other.rightmost_); + swap(mutable_rightmost(), other.mutable_rightmost()); swap(size_, other.size_); } else { // We aren't allowed to propagate the allocator and the allocator is @@ -2119,22 +2294,29 @@ auto btree<P>::operator=(btree &&other) noexcept -> btree & { template <typename P> auto btree<P>::erase(iterator iter) -> iterator { - bool internal_delete = false; - if (!iter.node->leaf()) { - // Deletion of a value on an internal node. First, move the largest value - // from our left child here, then delete that position (in remove_values() - // below). We can get to the largest value from our left child by - // decrementing iter. + iter.node_->value_destroy(iter.position_, mutable_allocator()); + iter.update_generation(); + + const bool internal_delete = iter.node_->is_internal(); + if (internal_delete) { + // Deletion of a value on an internal node. First, transfer the largest + // value from our left child here, then erase/rebalance from that position. + // We can get to the largest value from our left child by decrementing iter. iterator internal_iter(iter); --iter; - assert(iter.node->leaf()); - params_type::move(mutable_allocator(), iter.node->slot(iter.position), - internal_iter.node->slot(internal_iter.position)); - internal_delete = true; - } - - // Delete the key from the leaf. - iter.node->remove_values(iter.position, /*to_erase=*/1, mutable_allocator()); + assert(iter.node_->is_leaf()); + internal_iter.node_->transfer(internal_iter.position_, iter.position_, + iter.node_, mutable_allocator()); + } else { + // Shift values after erased position in leaf. In the internal case, we + // don't need to do this because the leaf position is the end of the node. + const field_type transfer_from = iter.position_ + 1; + const field_type num_to_transfer = iter.node_->finish() - transfer_from; + iter.node_->transfer_n(num_to_transfer, iter.position_, transfer_from, + iter.node_, mutable_allocator()); + } + // Update node finish and container size. + iter.node_->set_finish(iter.node_->finish() - 1); --size_; // We want to return the next value after the one we just erased. If we @@ -2142,7 +2324,7 @@ auto btree<P>::erase(iterator iter) -> iterator { // value is ++(++iter). If we erased from a leaf node (internal_delete == // false) then the next value is ++iter. Note that ++iter may point to an // internal node and the value in the internal node may move to a leaf node - // (iter.node) when rebalancing is performed at the leaf level. + // (iter.node_) when rebalancing is performed at the leaf level. iterator res = rebalance_after_delete(iter); @@ -2159,14 +2341,14 @@ auto btree<P>::rebalance_after_delete(iterator iter) -> iterator { iterator res(iter); bool first_iteration = true; for (;;) { - if (iter.node == root()) { + if (iter.node_ == root()) { try_shrink(); if (empty()) { return end(); } break; } - if (iter.node->count() >= kMinNodeValues) { + if (iter.node_->count() >= kMinNodeValues) { break; } bool merged = try_merge_or_rebalance(&iter); @@ -2179,14 +2361,15 @@ auto btree<P>::rebalance_after_delete(iterator iter) -> iterator { if (!merged) { break; } - iter.position = iter.node->position(); - iter.node = iter.node->parent(); + iter.position_ = iter.node_->position(); + iter.node_ = iter.node_->parent(); } + res.update_generation(); // Adjust our return value. If we're pointing at the end of a node, advance // the iterator. - if (res.position == res.node->finish()) { - res.position = res.node->finish() - 1; + if (res.position_ == res.node_->finish()) { + res.position_ = res.node_->finish() - 1; ++res; } @@ -2203,33 +2386,36 @@ auto btree<P>::erase_range(iterator begin, iterator end) return {0, begin}; } - if (count == size_) { + if (static_cast<size_type>(count) == size_) { clear(); return {count, this->end()}; } - if (begin.node == end.node) { - assert(end.position > begin.position); - begin.node->remove_values(begin.position, end.position - begin.position, - mutable_allocator()); + if (begin.node_ == end.node_) { + assert(end.position_ > begin.position_); + begin.node_->remove_values(begin.position_, end.position_ - begin.position_, + mutable_allocator()); size_ -= count; return {count, rebalance_after_delete(begin)}; } const size_type target_size = size_ - count; while (size_ > target_size) { - if (begin.node->leaf()) { + if (begin.node_->is_leaf()) { const size_type remaining_to_erase = size_ - target_size; - const size_type remaining_in_node = begin.node->finish() - begin.position; + const size_type remaining_in_node = + begin.node_->finish() - begin.position_; const size_type to_erase = (std::min)(remaining_to_erase, remaining_in_node); - begin.node->remove_values(begin.position, to_erase, mutable_allocator()); + begin.node_->remove_values(begin.position_, to_erase, + mutable_allocator()); size_ -= to_erase; begin = rebalance_after_delete(begin); } else { begin = erase(begin); } } + begin.update_generation(); return {count, begin}; } @@ -2238,8 +2424,7 @@ void btree<P>::clear() { if (!empty()) { node_type::clear_and_delete(root(), mutable_allocator()); } - mutable_root() = EmptyNode(); - rightmost_ = EmptyNode(); + mutable_root() = mutable_rightmost() = EmptyNode(); size_ = 0; } @@ -2248,15 +2433,15 @@ void btree<P>::swap(btree &other) { using std::swap; if (absl::allocator_traits< allocator_type>::propagate_on_container_swap::value) { - // Note: `root_` also contains the allocator and the key comparator. - swap(root_, other.root_); + // Note: `rightmost_` also contains the allocator and the key comparator. + swap(rightmost_, other.rightmost_); } else { // It's undefined behavior if the allocators are unequal here. assert(allocator() == other.allocator()); - swap(mutable_root(), other.mutable_root()); + swap(mutable_rightmost(), other.mutable_rightmost()); swap(*mutable_key_comp(), *other.mutable_key_comp()); } - swap(rightmost_, other.rightmost_); + swap(mutable_root(), other.mutable_root()); swap(size_, other.size_); } @@ -2264,18 +2449,18 @@ template <typename P> void btree<P>::verify() const { assert(root() != nullptr); assert(leftmost() != nullptr); - assert(rightmost_ != nullptr); + assert(rightmost() != nullptr); assert(empty() || size() == internal_verify(root(), nullptr, nullptr)); - assert(leftmost() == (++const_iterator(root(), -1)).node); - assert(rightmost_ == (--const_iterator(root(), root()->finish())).node); - assert(leftmost()->leaf()); - assert(rightmost_->leaf()); + assert(leftmost() == (++const_iterator(root(), -1)).node_); + assert(rightmost() == (--const_iterator(root(), root()->finish())).node_); + assert(leftmost()->is_leaf()); + assert(rightmost()->is_leaf()); } template <typename P> void btree<P>::rebalance_or_split(iterator *iter) { - node_type *&node = iter->node; - int &insert_position = iter->position; + node_type *&node = iter->node_; + int &insert_position = iter->position_; assert(node->count() == node->max_count()); assert(kNodeSlots == node->max_count()); @@ -2350,19 +2535,20 @@ void btree<P>::rebalance_or_split(iterator *iter) { // Create a new root node and set the current root node as the child of the // new root. parent = new_internal_node(parent); + parent->set_generation(root()->generation()); parent->init_child(parent->start(), root()); mutable_root() = parent; // If the former root was a leaf node, then it's now the rightmost node. - assert(!parent->start_child()->leaf() || - parent->start_child() == rightmost_); + assert(parent->start_child()->is_internal() || + parent->start_child() == rightmost()); } // Split the node. node_type *split_node; - if (node->leaf()) { + if (node->is_leaf()) { split_node = new_leaf_node(parent); node->split(insert_position, split_node, mutable_allocator()); - if (rightmost_ == node) rightmost_ = split_node; + if (rightmost() == node) mutable_rightmost() = split_node; } else { split_node = new_internal_node(parent); node->split(insert_position, split_node, mutable_allocator()); @@ -2377,55 +2563,56 @@ void btree<P>::rebalance_or_split(iterator *iter) { template <typename P> void btree<P>::merge_nodes(node_type *left, node_type *right) { left->merge(right, mutable_allocator()); - if (rightmost_ == right) rightmost_ = left; + if (rightmost() == right) mutable_rightmost() = left; } template <typename P> bool btree<P>::try_merge_or_rebalance(iterator *iter) { - node_type *parent = iter->node->parent(); - if (iter->node->position() > parent->start()) { + node_type *parent = iter->node_->parent(); + if (iter->node_->position() > parent->start()) { // Try merging with our left sibling. - node_type *left = parent->child(iter->node->position() - 1); + node_type *left = parent->child(iter->node_->position() - 1); assert(left->max_count() == kNodeSlots); - if (1U + left->count() + iter->node->count() <= kNodeSlots) { - iter->position += 1 + left->count(); - merge_nodes(left, iter->node); - iter->node = left; + if (1U + left->count() + iter->node_->count() <= kNodeSlots) { + iter->position_ += 1 + left->count(); + merge_nodes(left, iter->node_); + iter->node_ = left; return true; } } - if (iter->node->position() < parent->finish()) { + if (iter->node_->position() < parent->finish()) { // Try merging with our right sibling. - node_type *right = parent->child(iter->node->position() + 1); + node_type *right = parent->child(iter->node_->position() + 1); assert(right->max_count() == kNodeSlots); - if (1U + iter->node->count() + right->count() <= kNodeSlots) { - merge_nodes(iter->node, right); + if (1U + iter->node_->count() + right->count() <= kNodeSlots) { + merge_nodes(iter->node_, right); return true; } // Try rebalancing with our right sibling. We don't perform rebalancing if - // we deleted the first element from iter->node and the node is not + // we deleted the first element from iter->node_ and the node is not // empty. This is a small optimization for the common pattern of deleting // from the front of the tree. if (right->count() > kMinNodeValues && - (iter->node->count() == 0 || iter->position > iter->node->start())) { - int to_move = (right->count() - iter->node->count()) / 2; + (iter->node_->count() == 0 || iter->position_ > iter->node_->start())) { + int to_move = (right->count() - iter->node_->count()) / 2; to_move = (std::min)(to_move, right->count() - 1); - iter->node->rebalance_right_to_left(to_move, right, mutable_allocator()); + iter->node_->rebalance_right_to_left(to_move, right, mutable_allocator()); return false; } } - if (iter->node->position() > parent->start()) { + if (iter->node_->position() > parent->start()) { // Try rebalancing with our left sibling. We don't perform rebalancing if - // we deleted the last element from iter->node and the node is not + // we deleted the last element from iter->node_ and the node is not // empty. This is a small optimization for the common pattern of deleting // from the back of the tree. - node_type *left = parent->child(iter->node->position() - 1); + node_type *left = parent->child(iter->node_->position() - 1); if (left->count() > kMinNodeValues && - (iter->node->count() == 0 || iter->position < iter->node->finish())) { - int to_move = (left->count() - iter->node->count()) / 2; + (iter->node_->count() == 0 || + iter->position_ < iter->node_->finish())) { + int to_move = (left->count() - iter->node_->count()) / 2; to_move = (std::min)(to_move, left->count() - 1); - left->rebalance_left_to_right(to_move, iter->node, mutable_allocator()); - iter->position += to_move; + left->rebalance_left_to_right(to_move, iter->node_, mutable_allocator()); + iter->position_ += to_move; return false; } } @@ -2439,9 +2626,9 @@ void btree<P>::try_shrink() { return; } // Deleted the last item on the root node, shrink the height of the tree. - if (orig_root->leaf()) { + if (orig_root->is_leaf()) { assert(size() == 0); - mutable_root() = rightmost_ = EmptyNode(); + mutable_root() = mutable_rightmost() = EmptyNode(); } else { node_type *child = orig_root->start_child(); child->make_root(); @@ -2453,15 +2640,16 @@ void btree<P>::try_shrink() { template <typename P> template <typename IterType> inline IterType btree<P>::internal_last(IterType iter) { - assert(iter.node != nullptr); - while (iter.position == iter.node->finish()) { - iter.position = iter.node->position(); - iter.node = iter.node->parent(); - if (iter.node->leaf()) { - iter.node = nullptr; + assert(iter.node_ != nullptr); + while (iter.position_ == iter.node_->finish()) { + iter.position_ = iter.node_->position(); + iter.node_ = iter.node_->parent(); + if (iter.node_->is_leaf()) { + iter.node_ = nullptr; break; } } + iter.update_generation(); return iter; } @@ -2469,37 +2657,39 @@ template <typename P> template <typename... Args> inline auto btree<P>::internal_emplace(iterator iter, Args &&... args) -> iterator { - if (!iter.node->leaf()) { + if (iter.node_->is_internal()) { // We can't insert on an internal node. Instead, we'll insert after the // previous value which is guaranteed to be on a leaf node. --iter; - ++iter.position; + ++iter.position_; } - const field_type max_count = iter.node->max_count(); + const field_type max_count = iter.node_->max_count(); allocator_type *alloc = mutable_allocator(); - if (iter.node->count() == max_count) { + if (iter.node_->count() == max_count) { // Make room in the leaf for the new item. if (max_count < kNodeSlots) { // Insertion into the root where the root is smaller than the full node // size. Simply grow the size of the root node. - assert(iter.node == root()); - iter.node = + assert(iter.node_ == root()); + iter.node_ = new_leaf_root_node((std::min<int>)(kNodeSlots, 2 * max_count)); // Transfer the values from the old root to the new root. node_type *old_root = root(); - node_type *new_root = iter.node; + node_type *new_root = iter.node_; new_root->transfer_n(old_root->count(), new_root->start(), old_root->start(), old_root, alloc); new_root->set_finish(old_root->finish()); old_root->set_finish(old_root->start()); + new_root->set_generation(old_root->generation()); node_type::clear_and_delete(old_root, alloc); - mutable_root() = rightmost_ = new_root; + mutable_root() = mutable_rightmost() = new_root; } else { rebalance_or_split(&iter); } } - iter.node->emplace_value(iter.position, alloc, std::forward<Args>(args)...); + iter.node_->emplace_value(iter.position_, alloc, std::forward<Args>(args)...); ++size_; + iter.update_generation(); return iter; } @@ -2510,8 +2700,8 @@ inline auto btree<P>::internal_locate(const K &key) const iterator iter(const_cast<node_type *>(root())); for (;;) { SearchResult<int, is_key_compare_to::value> res = - iter.node->lower_bound(key, key_comp()); - iter.position = res.value; + iter.node_->lower_bound(key, key_comp()); + iter.position_ = res.value; if (res.IsEq()) { return {iter, MatchKind::kEq}; } @@ -2519,10 +2709,10 @@ inline auto btree<P>::internal_locate(const K &key) const // down the tree if the keys are equal, but determining equality would // require doing an extra comparison on each node on the way down, and we // will need to go all the way to the leaf node in the expected case. - if (iter.node->leaf()) { + if (iter.node_->is_leaf()) { break; } - iter.node = iter.node->child(iter.position); + iter.node_ = iter.node_->child(iter.position_); } // Note: in the non-key-compare-to case, the key may actually be equivalent // here (and the MatchKind::kNe is ignored). @@ -2542,13 +2732,13 @@ auto btree<P>::internal_lower_bound(const K &key) const SearchResult<int, is_key_compare_to::value> res; bool seen_eq = false; for (;;) { - res = iter.node->lower_bound(key, key_comp()); - iter.position = res.value; - if (iter.node->leaf()) { + res = iter.node_->lower_bound(key, key_comp()); + iter.position_ = res.value; + if (iter.node_->is_leaf()) { break; } seen_eq = seen_eq || res.IsEq(); - iter.node = iter.node->child(iter.position); + iter.node_ = iter.node_->child(iter.position_); } if (res.IsEq()) return {iter, MatchKind::kEq}; return {internal_last(iter), seen_eq ? MatchKind::kEq : MatchKind::kNe}; @@ -2559,11 +2749,11 @@ template <typename K> auto btree<P>::internal_upper_bound(const K &key) const -> iterator { iterator iter(const_cast<node_type *>(root())); for (;;) { - iter.position = iter.node->upper_bound(key, key_comp()); - if (iter.node->leaf()) { + iter.position_ = iter.node_->upper_bound(key, key_comp()); + if (iter.node_->is_leaf()) { break; } - iter.node = iter.node->child(iter.position); + iter.node_ = iter.node_->child(iter.position_); } return internal_last(iter); } @@ -2578,7 +2768,7 @@ auto btree<P>::internal_find(const K &key) const -> iterator { } } else { const iterator iter = internal_last(res.value); - if (iter.node != nullptr && !compare_keys(key, iter.key())) { + if (iter.node_ != nullptr && !compare_keys(key, iter.key())) { return iter; } } @@ -2600,7 +2790,7 @@ int btree<P>::internal_verify(const node_type *node, const key_type *lo, assert(!compare_keys(node->key(i), node->key(i - 1))); } int count = node->count(); - if (!node->leaf()) { + if (node->is_internal()) { for (int i = node->start(); i <= node->finish(); ++i) { assert(node->child(i) != nullptr); assert(node->child(i)->parent() == node); @@ -2613,6 +2803,50 @@ int btree<P>::internal_verify(const node_type *node, const key_type *lo, return count; } +struct btree_access { + template <typename BtreeContainer, typename Pred> + static auto erase_if(BtreeContainer &container, Pred pred) + -> typename BtreeContainer::size_type { + const auto initial_size = container.size(); + auto &tree = container.tree_; + auto *alloc = tree.mutable_allocator(); + for (auto it = container.begin(); it != container.end();) { + if (!pred(*it)) { + ++it; + continue; + } + auto *node = it.node_; + if (node->is_internal()) { + // Handle internal nodes normally. + it = container.erase(it); + continue; + } + // If this is a leaf node, then we do all the erases from this node + // at once before doing rebalancing. + + // The current position to transfer slots to. + int to_pos = it.position_; + node->value_destroy(it.position_, alloc); + while (++it.position_ < node->finish()) { + it.update_generation(); + if (pred(*it)) { + node->value_destroy(it.position_, alloc); + } else { + node->transfer(node->slot(to_pos++), node->slot(it.position_), alloc); + } + } + const int num_deleted = node->finish() - to_pos; + tree.size_ -= num_deleted; + node->set_finish(to_pos); + it.position_ = to_pos; + it = tree.rebalance_after_delete(it); + } + return initial_size - container.size(); + } +}; + +#undef ABSL_BTREE_ENABLE_GENERATIONS + } // namespace container_internal ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/container/internal/btree_container.h b/absl/container/internal/btree_container.h index 03be708e..fc2f740a 100644 --- a/absl/container/internal/btree_container.h +++ b/absl/container/internal/btree_container.h @@ -20,6 +20,7 @@ #include <iterator> #include <utility> +#include "absl/base/attributes.h" #include "absl/base/internal/throw_delegate.h" #include "absl/container/internal/btree.h" // IWYU pragma: export #include "absl/container/internal/common.h" @@ -43,15 +44,15 @@ class btree_container { // transparent case. template <class K> using key_arg = - typename KeyArg<IsTransparent<typename Tree::key_compare>::value>:: - template type<K, typename Tree::key_type>; + typename KeyArg<params_type::kIsKeyCompareTransparent>::template type< + K, typename Tree::key_type>; public: using key_type = typename Tree::key_type; using value_type = typename Tree::value_type; using size_type = typename Tree::size_type; using difference_type = typename Tree::difference_type; - using key_compare = typename Tree::key_compare; + using key_compare = typename Tree::original_key_compare; using value_compare = typename Tree::value_compare; using allocator_type = typename Tree::allocator_type; using reference = typename Tree::reference; @@ -165,9 +166,10 @@ class btree_container { // Extract routines. node_type extract(iterator position) { - // Use Move instead of Transfer, because the rebalancing code expects to - // have a valid object to scribble metadata bits on top of. - auto node = CommonAccess::Move<node_type>(get_allocator(), position.slot()); + // Use Construct instead of Transfer because the rebalancing code will + // destroy the slot later. + auto node = + CommonAccess::Construct<node_type>(get_allocator(), position.slot()); erase(position); return node; } @@ -176,7 +178,7 @@ class btree_container { } // Utility routines. - void clear() { tree_.clear(); } + ABSL_ATTRIBUTE_REINITIALIZES void clear() { tree_.clear(); } void swap(btree_container &other) { tree_.swap(other.tree_); } void verify() const { tree_.verify(); } @@ -214,7 +216,7 @@ class btree_container { allocator_type get_allocator() const { return tree_.get_allocator(); } // The key comparator used by the btree. - key_compare key_comp() const { return tree_.key_comp(); } + key_compare key_comp() const { return key_compare(tree_.key_comp()); } value_compare value_comp() const { return tree_.value_comp(); } // Support absl::Hash. @@ -227,6 +229,7 @@ class btree_container { } protected: + friend struct btree_access; Tree tree_; }; @@ -247,7 +250,7 @@ class btree_set_container : public btree_container<Tree> { using key_type = typename Tree::key_type; using value_type = typename Tree::value_type; using size_type = typename Tree::size_type; - using key_compare = typename Tree::key_compare; + using key_compare = typename Tree::original_key_compare; using allocator_type = typename Tree::allocator_type; using iterator = typename Tree::iterator; using const_iterator = typename Tree::const_iterator; @@ -289,8 +292,11 @@ class btree_set_container : public btree_container<Tree> { } template <typename... Args> std::pair<iterator, bool> emplace(Args &&... args) { - init_type v(std::forward<Args>(args)...); - return this->tree_.insert_unique(params_type::key(v), std::move(v)); + // Use a node handle to manage a temp slot. + auto node = CommonAccess::Construct<node_type>(this->get_allocator(), + std::forward<Args>(args)...); + auto *slot = CommonAccess::GetSlot(node); + return this->tree_.insert_unique(params_type::key(slot), slot); } iterator insert(const_iterator hint, const value_type &v) { return this->tree_ @@ -304,9 +310,12 @@ class btree_set_container : public btree_container<Tree> { } template <typename... Args> iterator emplace_hint(const_iterator hint, Args &&... args) { - init_type v(std::forward<Args>(args)...); + // Use a node handle to manage a temp slot. + auto node = CommonAccess::Construct<node_type>(this->get_allocator(), + std::forward<Args>(args)...); + auto *slot = CommonAccess::GetSlot(node); return this->tree_ - .insert_hint_unique(iterator(hint), params_type::key(v), std::move(v)) + .insert_hint_unique(iterator(hint), params_type::key(slot), slot) .first; } template <typename InputIterator> @@ -398,7 +407,7 @@ class btree_map_container : public btree_set_container<Tree> { using key_type = typename Tree::key_type; using mapped_type = typename params_type::mapped_type; using value_type = typename Tree::value_type; - using key_compare = typename Tree::key_compare; + using key_compare = typename Tree::original_key_compare; using allocator_type = typename Tree::allocator_type; using iterator = typename Tree::iterator; using const_iterator = typename Tree::const_iterator; @@ -535,6 +544,7 @@ class btree_multiset_container : public btree_container<Tree> { using params_type = typename Tree::params_type; using init_type = typename params_type::init_type; using is_key_compare_to = typename params_type::is_key_compare_to; + friend class BtreeNodePeer; template <class K> using key_arg = typename super_type::template key_arg<K>; @@ -543,7 +553,7 @@ class btree_multiset_container : public btree_container<Tree> { using key_type = typename Tree::key_type; using value_type = typename Tree::value_type; using size_type = typename Tree::size_type; - using key_compare = typename Tree::key_compare; + using key_compare = typename Tree::original_key_compare; using allocator_type = typename Tree::allocator_type; using iterator = typename Tree::iterator; using const_iterator = typename Tree::const_iterator; @@ -595,12 +605,18 @@ class btree_multiset_container : public btree_container<Tree> { } template <typename... Args> iterator emplace(Args &&... args) { - return this->tree_.insert_multi(init_type(std::forward<Args>(args)...)); + // Use a node handle to manage a temp slot. + auto node = CommonAccess::Construct<node_type>(this->get_allocator(), + std::forward<Args>(args)...); + return this->tree_.insert_multi(CommonAccess::GetSlot(node)); } template <typename... Args> iterator emplace_hint(const_iterator hint, Args &&... args) { - return this->tree_.insert_hint_multi( - iterator(hint), init_type(std::forward<Args>(args)...)); + // Use a node handle to manage a temp slot. + auto node = CommonAccess::Construct<node_type>(this->get_allocator(), + std::forward<Args>(args)...); + return this->tree_.insert_hint_multi(iterator(hint), + CommonAccess::GetSlot(node)); } iterator insert(node_type &&node) { if (!node) return this->end(); @@ -666,6 +682,7 @@ template <typename Tree> class btree_multimap_container : public btree_multiset_container<Tree> { using super_type = btree_multiset_container<Tree>; using params_type = typename Tree::params_type; + friend class BtreeNodePeer; public: using mapped_type = typename params_type::mapped_type; diff --git a/absl/container/internal/common.h b/absl/container/internal/common.h index 030e9d4a..416d9aa3 100644 --- a/absl/container/internal/common.h +++ b/absl/container/internal/common.h @@ -84,10 +84,11 @@ class node_handle_base { PolicyTraits::transfer(alloc(), slot(), s); } - struct move_tag_t {}; - node_handle_base(move_tag_t, const allocator_type& a, slot_type* s) + struct construct_tag_t {}; + template <typename... Args> + node_handle_base(construct_tag_t, const allocator_type& a, Args&&... args) : alloc_(a) { - PolicyTraits::construct(alloc(), slot(), s); + PolicyTraits::construct(alloc(), slot(), std::forward<Args>(args)...); } void destroy() { @@ -186,8 +187,8 @@ struct CommonAccess { } template <typename T, typename... Args> - static T Move(Args&&... args) { - return T(typename T::move_tag_t{}, std::forward<Args>(args)...); + static T Construct(Args&&... args) { + return T(typename T::construct_tag_t{}, std::forward<Args>(args)...); } }; diff --git a/absl/container/internal/compressed_tuple_test.cc b/absl/container/internal/compressed_tuple_test.cc index 62a7483e..74111f97 100644 --- a/absl/container/internal/compressed_tuple_test.cc +++ b/absl/container/internal/compressed_tuple_test.cc @@ -403,6 +403,16 @@ TEST(CompressedTupleTest, EmptyFinalClass) { } #endif +// TODO(b/214288561): enable this test. +TEST(CompressedTupleTest, DISABLED_NestedEbo) { + struct Empty1 {}; + struct Empty2 {}; + CompressedTuple<Empty1, CompressedTuple<Empty2>, int> x; + CompressedTuple<Empty1, Empty2, int> y; + // Currently fails with sizeof(x) == 8, sizeof(y) == 4. + EXPECT_EQ(sizeof(x), sizeof(y)); +} + } // namespace } // namespace container_internal ABSL_NAMESPACE_END diff --git a/absl/container/internal/container_memory.h b/absl/container/internal/container_memory.h index e67529ec..00e9f6d7 100644 --- a/absl/container/internal/container_memory.h +++ b/absl/container/internal/container_memory.h @@ -174,7 +174,7 @@ decltype(std::declval<F>()(std::declval<T>())) WithConstructed( // // 2. auto a = PairArgs(args...); // std::pair<F, S> p(std::piecewise_construct, -// std::move(p.first), std::move(p.second)); +// std::move(a.first), std::move(a.second)); inline std::pair<std::tuple<>, std::tuple<>> PairArgs() { return {}; } template <class F, class S> std::pair<std::tuple<F&&>, std::tuple<S&&>> PairArgs(F&& f, S&& s) { @@ -402,6 +402,15 @@ struct map_slot_policy { } } + // Construct this slot by copying from another slot. + template <class Allocator> + static void construct(Allocator* alloc, slot_type* slot, + const slot_type* other) { + emplace(slot); + absl::allocator_traits<Allocator>::construct(*alloc, &slot->value, + other->value); + } + template <class Allocator> static void destroy(Allocator* alloc, slot_type* slot) { if (kMutableKeys::value) { @@ -424,33 +433,6 @@ struct map_slot_policy { } destroy(alloc, old_slot); } - - template <class Allocator> - static void swap(Allocator* alloc, slot_type* a, slot_type* b) { - if (kMutableKeys::value) { - using std::swap; - swap(a->mutable_value, b->mutable_value); - } else { - value_type tmp = std::move(a->value); - absl::allocator_traits<Allocator>::destroy(*alloc, &a->value); - absl::allocator_traits<Allocator>::construct(*alloc, &a->value, - std::move(b->value)); - absl::allocator_traits<Allocator>::destroy(*alloc, &b->value); - absl::allocator_traits<Allocator>::construct(*alloc, &b->value, - std::move(tmp)); - } - } - - template <class Allocator> - static void move(Allocator* alloc, slot_type* src, slot_type* dest) { - if (kMutableKeys::value) { - dest->mutable_value = std::move(src->mutable_value); - } else { - absl::allocator_traits<Allocator>::destroy(*alloc, &dest->value); - absl::allocator_traits<Allocator>::construct(*alloc, &dest->value, - std::move(src->value)); - } - } }; } // namespace container_internal diff --git a/absl/container/internal/counting_allocator.h b/absl/container/internal/counting_allocator.h index 927cf082..66068a5a 100644 --- a/absl/container/internal/counting_allocator.h +++ b/absl/container/internal/counting_allocator.h @@ -80,7 +80,15 @@ class CountingAllocator { template <typename U> void destroy(U* p) { Allocator allocator; + // Ignore GCC warning bug. +#if ABSL_INTERNAL_HAVE_MIN_GNUC_VERSION(12, 0) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wuse-after-free" +#endif AllocatorTraits::destroy(allocator, p); +#if ABSL_INTERNAL_HAVE_MIN_GNUC_VERSION(12, 0) +#pragma GCC diagnostic pop +#endif if (instance_count_ != nullptr) { *instance_count_ -= 1; } diff --git a/absl/container/internal/hash_function_defaults.h b/absl/container/internal/hash_function_defaults.h index 0683422a..250e662c 100644 --- a/absl/container/internal/hash_function_defaults.h +++ b/absl/container/internal/hash_function_defaults.h @@ -78,24 +78,26 @@ struct StringHash { } }; +struct StringEq { + using is_transparent = void; + bool operator()(absl::string_view lhs, absl::string_view rhs) const { + return lhs == rhs; + } + bool operator()(const absl::Cord& lhs, const absl::Cord& rhs) const { + return lhs == rhs; + } + bool operator()(const absl::Cord& lhs, absl::string_view rhs) const { + return lhs == rhs; + } + bool operator()(absl::string_view lhs, const absl::Cord& rhs) const { + return lhs == rhs; + } +}; + // Supports heterogeneous lookup for string-like elements. struct StringHashEq { using Hash = StringHash; - struct Eq { - using is_transparent = void; - bool operator()(absl::string_view lhs, absl::string_view rhs) const { - return lhs == rhs; - } - bool operator()(const absl::Cord& lhs, const absl::Cord& rhs) const { - return lhs == rhs; - } - bool operator()(const absl::Cord& lhs, absl::string_view rhs) const { - return lhs == rhs; - } - bool operator()(absl::string_view lhs, const absl::Cord& rhs) const { - return lhs == rhs; - } - }; + using Eq = StringEq; }; template <> diff --git a/absl/container/internal/hash_function_defaults_test.cc b/absl/container/internal/hash_function_defaults_test.cc index 59576b8e..9f0a4c72 100644 --- a/absl/container/internal/hash_function_defaults_test.cc +++ b/absl/container/internal/hash_function_defaults_test.cc @@ -310,7 +310,7 @@ struct StringLikeTest : public ::testing::Test { hash_default_hash<typename T::first_type> hash; }; -TYPED_TEST_CASE_P(StringLikeTest); +TYPED_TEST_SUITE_P(StringLikeTest); TYPED_TEST_P(StringLikeTest, Eq) { EXPECT_TRUE(this->eq(this->a1, this->b1)); diff --git a/absl/container/internal/hash_generator_testing.h b/absl/container/internal/hash_generator_testing.h index 6869fe45..f1f555a5 100644 --- a/absl/container/internal/hash_generator_testing.h +++ b/absl/container/internal/hash_generator_testing.h @@ -21,11 +21,13 @@ #include <stdint.h> #include <algorithm> +#include <cassert> #include <iosfwd> #include <random> #include <tuple> #include <type_traits> #include <utility> +#include <vector> #include "absl/container/internal/hash_policy_testing.h" #include "absl/memory/memory.h" @@ -153,6 +155,25 @@ using GeneratedType = decltype( typename Container::value_type, typename Container::key_type>::type>&>()()); +// Naive wrapper that performs a linear search of previous values. +// Beware this is O(SQR), which is reasonable for smaller kMaxValues. +template <class T, size_t kMaxValues = 64, class E = void> +struct UniqueGenerator { + Generator<T, E> gen; + std::vector<T> values; + + T operator()() { + assert(values.size() < kMaxValues); + for (;;) { + T value = gen(); + if (std::find(values.begin(), values.end(), value) == values.end()) { + values.push_back(value); + return value; + } + } + } +}; + } // namespace hash_internal } // namespace container_internal ABSL_NAMESPACE_END diff --git a/absl/container/internal/hashtablez_sampler.cc b/absl/container/internal/hashtablez_sampler.cc index 5a29bed7..efc1be58 100644 --- a/absl/container/internal/hashtablez_sampler.cc +++ b/absl/container/internal/hashtablez_sampler.cc @@ -21,49 +21,55 @@ #include <limits> #include "absl/base/attributes.h" -#include "absl/base/internal/exponential_biased.h" -#include "absl/container/internal/have_sse.h" +#include "absl/base/config.h" #include "absl/debugging/stacktrace.h" #include "absl/memory/memory.h" +#include "absl/profiling/internal/exponential_biased.h" +#include "absl/profiling/internal/sample_recorder.h" #include "absl/synchronization/mutex.h" +#include "absl/utility/utility.h" namespace absl { ABSL_NAMESPACE_BEGIN namespace container_internal { + +#ifdef ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL constexpr int HashtablezInfo::kMaxStackDepth; +#endif namespace { ABSL_CONST_INIT std::atomic<bool> g_hashtablez_enabled{ false }; ABSL_CONST_INIT std::atomic<int32_t> g_hashtablez_sample_parameter{1 << 10}; -ABSL_CONST_INIT std::atomic<int32_t> g_hashtablez_max_samples{1 << 20}; +std::atomic<HashtablezConfigListener> g_hashtablez_config_listener{nullptr}; #if defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE) -ABSL_PER_THREAD_TLS_KEYWORD absl::base_internal::ExponentialBiased +ABSL_PER_THREAD_TLS_KEYWORD absl::profiling_internal::ExponentialBiased g_exponential_biased_generator; #endif +void TriggerHashtablezConfigListener() { + auto* listener = g_hashtablez_config_listener.load(std::memory_order_acquire); + if (listener != nullptr) listener(); +} + } // namespace #if defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE) -ABSL_PER_THREAD_TLS_KEYWORD int64_t global_next_sample = 0; +ABSL_PER_THREAD_TLS_KEYWORD SamplingState global_next_sample = {0, 0}; #endif // defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE) -HashtablezSampler& HashtablezSampler::Global() { +HashtablezSampler& GlobalHashtablezSampler() { static auto* sampler = new HashtablezSampler(); return *sampler; } -HashtablezSampler::DisposeCallback HashtablezSampler::SetDisposeCallback( - DisposeCallback f) { - return dispose_.exchange(f, std::memory_order_relaxed); -} - -HashtablezInfo::HashtablezInfo() { PrepareForSampling(); } +HashtablezInfo::HashtablezInfo() = default; HashtablezInfo::~HashtablezInfo() = default; -void HashtablezInfo::PrepareForSampling() { +void HashtablezInfo::PrepareForSampling(int64_t stride, + size_t inline_element_size_value) { capacity.store(0, std::memory_order_relaxed); size.store(0, std::memory_order_relaxed); num_erases.store(0, std::memory_order_relaxed); @@ -73,100 +79,16 @@ void HashtablezInfo::PrepareForSampling() { hashes_bitwise_or.store(0, std::memory_order_relaxed); hashes_bitwise_and.store(~size_t{}, std::memory_order_relaxed); hashes_bitwise_xor.store(0, std::memory_order_relaxed); + max_reserve.store(0, std::memory_order_relaxed); create_time = absl::Now(); + weight = stride; // The inliner makes hardcoded skip_count difficult (especially when combined // with LTO). We use the ability to exclude stacks by regex when encoding // instead. depth = absl::GetStackTrace(stack, HashtablezInfo::kMaxStackDepth, /* skip_count= */ 0); - dead = nullptr; -} - -HashtablezSampler::HashtablezSampler() - : dropped_samples_(0), size_estimate_(0), all_(nullptr), dispose_(nullptr) { - absl::MutexLock l(&graveyard_.init_mu); - graveyard_.dead = &graveyard_; -} - -HashtablezSampler::~HashtablezSampler() { - HashtablezInfo* s = all_.load(std::memory_order_acquire); - while (s != nullptr) { - HashtablezInfo* next = s->next; - delete s; - s = next; - } -} - -void HashtablezSampler::PushNew(HashtablezInfo* sample) { - sample->next = all_.load(std::memory_order_relaxed); - while (!all_.compare_exchange_weak(sample->next, sample, - std::memory_order_release, - std::memory_order_relaxed)) { - } -} - -void HashtablezSampler::PushDead(HashtablezInfo* sample) { - if (auto* dispose = dispose_.load(std::memory_order_relaxed)) { - dispose(*sample); - } - - absl::MutexLock graveyard_lock(&graveyard_.init_mu); - absl::MutexLock sample_lock(&sample->init_mu); - sample->dead = graveyard_.dead; - graveyard_.dead = sample; -} - -HashtablezInfo* HashtablezSampler::PopDead() { - absl::MutexLock graveyard_lock(&graveyard_.init_mu); - - // The list is circular, so eventually it collapses down to - // graveyard_.dead == &graveyard_ - // when it is empty. - HashtablezInfo* sample = graveyard_.dead; - if (sample == &graveyard_) return nullptr; - - absl::MutexLock sample_lock(&sample->init_mu); - graveyard_.dead = sample->dead; - sample->PrepareForSampling(); - return sample; -} - -HashtablezInfo* HashtablezSampler::Register() { - int64_t size = size_estimate_.fetch_add(1, std::memory_order_relaxed); - if (size > g_hashtablez_max_samples.load(std::memory_order_relaxed)) { - size_estimate_.fetch_sub(1, std::memory_order_relaxed); - dropped_samples_.fetch_add(1, std::memory_order_relaxed); - return nullptr; - } - - HashtablezInfo* sample = PopDead(); - if (sample == nullptr) { - // Resurrection failed. Hire a new warlock. - sample = new HashtablezInfo(); - PushNew(sample); - } - - return sample; -} - -void HashtablezSampler::Unregister(HashtablezInfo* sample) { - PushDead(sample); - size_estimate_.fetch_sub(1, std::memory_order_relaxed); -} - -int64_t HashtablezSampler::Iterate( - const std::function<void(const HashtablezInfo& stack)>& f) { - HashtablezInfo* s = all_.load(std::memory_order_acquire); - while (s != nullptr) { - absl::MutexLock l(&s->init_mu); - if (s->dead == nullptr) { - f(*s); - } - s = s->next; - } - - return dropped_samples_.load(std::memory_order_relaxed); + inline_element_size = inline_element_size_value; } static bool ShouldForceSampling() { @@ -189,21 +111,32 @@ static bool ShouldForceSampling() { return state == kForce; } -HashtablezInfo* SampleSlow(int64_t* next_sample) { +HashtablezInfo* SampleSlow(SamplingState& next_sample, + size_t inline_element_size) { if (ABSL_PREDICT_FALSE(ShouldForceSampling())) { - *next_sample = 1; - return HashtablezSampler::Global().Register(); + next_sample.next_sample = 1; + const int64_t old_stride = exchange(next_sample.sample_stride, 1); + HashtablezInfo* result = + GlobalHashtablezSampler().Register(old_stride, inline_element_size); + return result; } #if !defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE) - *next_sample = std::numeric_limits<int64_t>::max(); + next_sample = { + std::numeric_limits<int64_t>::max(), + std::numeric_limits<int64_t>::max(), + }; return nullptr; #else - bool first = *next_sample < 0; - *next_sample = g_exponential_biased_generator.GetStride( + bool first = next_sample.next_sample < 0; + + const int64_t next_stride = g_exponential_biased_generator.GetStride( g_hashtablez_sample_parameter.load(std::memory_order_relaxed)); + + next_sample.next_sample = next_stride; + const int64_t old_stride = exchange(next_sample.sample_stride, next_stride); // Small values of interval are equivalent to just sampling next time. - ABSL_ASSERT(*next_sample >= 1); + ABSL_ASSERT(next_stride >= 1); // g_hashtablez_enabled can be dynamically flipped, we need to set a threshold // low enough that we will start sampling in a reasonable time, so we just use @@ -213,16 +146,16 @@ HashtablezInfo* SampleSlow(int64_t* next_sample) { // We will only be negative on our first count, so we should just retry in // that case. if (first) { - if (ABSL_PREDICT_TRUE(--*next_sample > 0)) return nullptr; - return SampleSlow(next_sample); + if (ABSL_PREDICT_TRUE(--next_sample.next_sample > 0)) return nullptr; + return SampleSlow(next_sample, inline_element_size); } - return HashtablezSampler::Global().Register(); + return GlobalHashtablezSampler().Register(old_stride, inline_element_size); #endif } void UnsampleSlow(HashtablezInfo* info) { - HashtablezSampler::Global().Unregister(info); + GlobalHashtablezSampler().Unregister(info); } void RecordInsertSlow(HashtablezInfo* info, size_t hash, @@ -230,7 +163,7 @@ void RecordInsertSlow(HashtablezInfo* info, size_t hash, // SwissTables probe in groups of 16, so scale this to count items probes and // not offset from desired. size_t probe_length = distance_from_desired; -#if ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSE2 +#ifdef ABSL_INTERNAL_HAVE_SSE2 probe_length /= 16; #else probe_length /= 8; @@ -247,11 +180,33 @@ void RecordInsertSlow(HashtablezInfo* info, size_t hash, info->size.fetch_add(1, std::memory_order_relaxed); } +void SetHashtablezConfigListener(HashtablezConfigListener l) { + g_hashtablez_config_listener.store(l, std::memory_order_release); +} + +bool IsHashtablezEnabled() { + return g_hashtablez_enabled.load(std::memory_order_acquire); +} + void SetHashtablezEnabled(bool enabled) { + SetHashtablezEnabledInternal(enabled); + TriggerHashtablezConfigListener(); +} + +void SetHashtablezEnabledInternal(bool enabled) { g_hashtablez_enabled.store(enabled, std::memory_order_release); } +int32_t GetHashtablezSampleParameter() { + return g_hashtablez_sample_parameter.load(std::memory_order_acquire); +} + void SetHashtablezSampleParameter(int32_t rate) { + SetHashtablezSampleParameterInternal(rate); + TriggerHashtablezConfigListener(); +} + +void SetHashtablezSampleParameterInternal(int32_t rate) { if (rate > 0) { g_hashtablez_sample_parameter.store(rate, std::memory_order_release); } else { @@ -260,9 +215,18 @@ void SetHashtablezSampleParameter(int32_t rate) { } } +int32_t GetHashtablezMaxSamples() { + return GlobalHashtablezSampler().GetMaxSamples(); +} + void SetHashtablezMaxSamples(int32_t max) { + SetHashtablezMaxSamplesInternal(max); + TriggerHashtablezConfigListener(); +} + +void SetHashtablezMaxSamplesInternal(int32_t max) { if (max > 0) { - g_hashtablez_max_samples.store(max, std::memory_order_release); + GlobalHashtablezSampler().SetMaxSamples(max); } else { ABSL_RAW_LOG(ERROR, "Invalid hashtablez max samples: %lld", static_cast<long long>(max)); // NOLINT(runtime/int) diff --git a/absl/container/internal/hashtablez_sampler.h b/absl/container/internal/hashtablez_sampler.h index 85685f72..d4016d8a 100644 --- a/absl/container/internal/hashtablez_sampler.h +++ b/absl/container/internal/hashtablez_sampler.h @@ -44,9 +44,10 @@ #include <memory> #include <vector> +#include "absl/base/config.h" #include "absl/base/internal/per_thread_tls.h" #include "absl/base/optimization.h" -#include "absl/container/internal/have_sse.h" +#include "absl/profiling/internal/sample_recorder.h" #include "absl/synchronization/mutex.h" #include "absl/utility/utility.h" @@ -57,7 +58,7 @@ namespace container_internal { // Stores information about a sampled hashtable. All mutations to this *must* // be made through `Record*` functions below. All reads from this *must* only // occur in the callback to `HashtablezSampler::Iterate`. -struct HashtablezInfo { +struct HashtablezInfo : public profiling_internal::Sample<HashtablezInfo> { // Constructs the object but does not fill in any fields. HashtablezInfo(); ~HashtablezInfo(); @@ -66,7 +67,8 @@ struct HashtablezInfo { // Puts the object into a clean state, fills in the logically `const` members, // blocking for any readers that are currently sampling the object. - void PrepareForSampling() ABSL_EXCLUSIVE_LOCKS_REQUIRED(init_mu); + void PrepareForSampling(int64_t stride, size_t inline_element_size_value) + ABSL_EXCLUSIVE_LOCKS_REQUIRED(init_mu); // These fields are mutated by the various Record* APIs and need to be // thread-safe. @@ -79,28 +81,22 @@ struct HashtablezInfo { std::atomic<size_t> hashes_bitwise_or; std::atomic<size_t> hashes_bitwise_and; std::atomic<size_t> hashes_bitwise_xor; - - // `HashtablezSampler` maintains intrusive linked lists for all samples. See - // comments on `HashtablezSampler::all_` for details on these. `init_mu` - // guards the ability to restore the sample to a pristine state. This - // prevents races with sampling and resurrecting an object. - absl::Mutex init_mu; - HashtablezInfo* next; - HashtablezInfo* dead ABSL_GUARDED_BY(init_mu); + std::atomic<size_t> max_reserve; // All of the fields below are set by `PrepareForSampling`, they must not be // mutated in `Record*` functions. They are logically `const` in that sense. - // These are guarded by init_mu, but that is not externalized to clients, who - // can only read them during `HashtablezSampler::Iterate` which will hold the - // lock. + // These are guarded by init_mu, but that is not externalized to clients, + // which can read them only during `SampleRecorder::Iterate` which will hold + // the lock. static constexpr int kMaxStackDepth = 64; absl::Time create_time; int32_t depth; void* stack[kMaxStackDepth]; + size_t inline_element_size; // How big is the slot? }; inline void RecordRehashSlow(HashtablezInfo* info, size_t total_probe_length) { -#if ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSE2 +#ifdef ABSL_INTERNAL_HAVE_SSE2 total_probe_length /= 16; #else total_probe_length /= 8; @@ -114,6 +110,18 @@ inline void RecordRehashSlow(HashtablezInfo* info, size_t total_probe_length) { std::memory_order_relaxed); } +inline void RecordReservationSlow(HashtablezInfo* info, + size_t target_capacity) { + info->max_reserve.store( + (std::max)(info->max_reserve.load(std::memory_order_relaxed), + target_capacity), + std::memory_order_relaxed); +} + +inline void RecordClearedReservationSlow(HashtablezInfo* info) { + info->max_reserve.store(0, std::memory_order_relaxed); +} + inline void RecordStorageChangedSlow(HashtablezInfo* info, size_t size, size_t capacity) { info->size.store(size, std::memory_order_relaxed); @@ -137,7 +145,15 @@ inline void RecordEraseSlow(HashtablezInfo* info) { std::memory_order_relaxed); } -HashtablezInfo* SampleSlow(int64_t* next_sample); +struct SamplingState { + int64_t next_sample; + // When we make a sampling decision, we record that distance so we can weight + // each sample. + int64_t sample_stride; +}; + +HashtablezInfo* SampleSlow(SamplingState& next_sample, + size_t inline_element_size); void UnsampleSlow(HashtablezInfo* info); #if defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE) @@ -177,6 +193,16 @@ class HashtablezInfoHandle { RecordRehashSlow(info_, total_probe_length); } + inline void RecordReservation(size_t target_capacity) { + if (ABSL_PREDICT_TRUE(info_ == nullptr)) return; + RecordReservationSlow(info_, target_capacity); + } + + inline void RecordClearedReservation() { + if (ABSL_PREDICT_TRUE(info_ == nullptr)) return; + RecordClearedReservationSlow(info_); + } + inline void RecordInsert(size_t hash, size_t distance_from_desired) { if (ABSL_PREDICT_TRUE(info_ == nullptr)) return; RecordInsertSlow(info_, hash, distance_from_desired); @@ -206,6 +232,8 @@ class HashtablezInfoHandle { inline void RecordStorageChanged(size_t /*size*/, size_t /*capacity*/) {} inline void RecordRehash(size_t /*total_probe_length*/) {} + inline void RecordReservation(size_t /*target_capacity*/) {} + inline void RecordClearedReservation() {} inline void RecordInsert(size_t /*hash*/, size_t /*distance_from_desired*/) {} inline void RecordErase() {} @@ -215,98 +243,47 @@ class HashtablezInfoHandle { #endif // defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE) #if defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE) -extern ABSL_PER_THREAD_TLS_KEYWORD int64_t global_next_sample; +extern ABSL_PER_THREAD_TLS_KEYWORD SamplingState global_next_sample; #endif // defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE) // Returns an RAII sampling handle that manages registration and unregistation // with the global sampler. -inline HashtablezInfoHandle Sample() { +inline HashtablezInfoHandle Sample( + size_t inline_element_size ABSL_ATTRIBUTE_UNUSED) { #if defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE) - if (ABSL_PREDICT_TRUE(--global_next_sample > 0)) { + if (ABSL_PREDICT_TRUE(--global_next_sample.next_sample > 0)) { return HashtablezInfoHandle(nullptr); } - return HashtablezInfoHandle(SampleSlow(&global_next_sample)); + return HashtablezInfoHandle( + SampleSlow(global_next_sample, inline_element_size)); #else return HashtablezInfoHandle(nullptr); #endif // !ABSL_PER_THREAD_TLS } -// Holds samples and their associated stack traces with a soft limit of -// `SetHashtablezMaxSamples()`. -// -// Thread safe. -class HashtablezSampler { - public: - // Returns a global Sampler. - static HashtablezSampler& Global(); - - HashtablezSampler(); - ~HashtablezSampler(); +using HashtablezSampler = + ::absl::profiling_internal::SampleRecorder<HashtablezInfo>; - // Registers for sampling. Returns an opaque registration info. - HashtablezInfo* Register(); +// Returns a global Sampler. +HashtablezSampler& GlobalHashtablezSampler(); - // Unregisters the sample. - void Unregister(HashtablezInfo* sample); - - // The dispose callback will be called on all samples the moment they are - // being unregistered. Only affects samples that are unregistered after the - // callback has been set. - // Returns the previous callback. - using DisposeCallback = void (*)(const HashtablezInfo&); - DisposeCallback SetDisposeCallback(DisposeCallback f); - - // Iterates over all the registered `StackInfo`s. Returning the number of - // samples that have been dropped. - int64_t Iterate(const std::function<void(const HashtablezInfo& stack)>& f); - - private: - void PushNew(HashtablezInfo* sample); - void PushDead(HashtablezInfo* sample); - HashtablezInfo* PopDead(); - - std::atomic<size_t> dropped_samples_; - std::atomic<size_t> size_estimate_; - - // Intrusive lock free linked lists for tracking samples. - // - // `all_` records all samples (they are never removed from this list) and is - // terminated with a `nullptr`. - // - // `graveyard_.dead` is a circular linked list. When it is empty, - // `graveyard_.dead == &graveyard`. The list is circular so that - // every item on it (even the last) has a non-null dead pointer. This allows - // `Iterate` to determine if a given sample is live or dead using only - // information on the sample itself. - // - // For example, nodes [A, B, C, D, E] with [A, C, E] alive and [B, D] dead - // looks like this (G is the Graveyard): - // - // +---+ +---+ +---+ +---+ +---+ - // all -->| A |--->| B |--->| C |--->| D |--->| E | - // | | | | | | | | | | - // +---+ | | +->| |-+ | | +->| |-+ | | - // | G | +---+ | +---+ | +---+ | +---+ | +---+ - // | | | | | | - // | | --------+ +--------+ | - // +---+ | - // ^ | - // +--------------------------------------+ - // - std::atomic<HashtablezInfo*> all_; - HashtablezInfo graveyard_; - - std::atomic<DisposeCallback> dispose_; -}; +using HashtablezConfigListener = void (*)(); +void SetHashtablezConfigListener(HashtablezConfigListener l); // Enables or disables sampling for Swiss tables. +bool IsHashtablezEnabled(); void SetHashtablezEnabled(bool enabled); +void SetHashtablezEnabledInternal(bool enabled); // Sets the rate at which Swiss tables will be sampled. +int32_t GetHashtablezSampleParameter(); void SetHashtablezSampleParameter(int32_t rate); +void SetHashtablezSampleParameterInternal(int32_t rate); // Sets a soft max for the number of samples that will be kept. +int32_t GetHashtablezMaxSamples(); void SetHashtablezMaxSamples(int32_t max); +void SetHashtablezMaxSamplesInternal(int32_t max); // Configuration override. // This allows process-wide sampling without depending on order of diff --git a/absl/container/internal/hashtablez_sampler_test.cc b/absl/container/internal/hashtablez_sampler_test.cc index 5f4c83b7..665d518f 100644 --- a/absl/container/internal/hashtablez_sampler_test.cc +++ b/absl/container/internal/hashtablez_sampler_test.cc @@ -21,7 +21,8 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" #include "absl/base/attributes.h" -#include "absl/container/internal/have_sse.h" +#include "absl/base/config.h" +#include "absl/profiling/internal/sample_recorder.h" #include "absl/synchronization/blocking_counter.h" #include "absl/synchronization/internal/thread_pool.h" #include "absl/synchronization/mutex.h" @@ -29,7 +30,7 @@ #include "absl/time/clock.h" #include "absl/time/time.h" -#if ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSE2 +#ifdef ABSL_INTERNAL_HAVE_SSE2 constexpr int kProbeLength = 16; #else constexpr int kProbeLength = 8; @@ -69,7 +70,9 @@ std::vector<size_t> GetSizes(HashtablezSampler* s) { } HashtablezInfo* Register(HashtablezSampler* s, size_t size) { - auto* info = s->Register(); + const int64_t test_stride = 123; + const size_t test_element_size = 17; + auto* info = s->Register(test_stride, test_element_size); assert(info != nullptr); info->size.store(size); return info; @@ -77,9 +80,11 @@ HashtablezInfo* Register(HashtablezSampler* s, size_t size) { TEST(HashtablezInfoTest, PrepareForSampling) { absl::Time test_start = absl::Now(); + const int64_t test_stride = 123; + const size_t test_element_size = 17; HashtablezInfo info; absl::MutexLock l(&info.init_mu); - info.PrepareForSampling(); + info.PrepareForSampling(test_stride, test_element_size); EXPECT_EQ(info.capacity.load(), 0); EXPECT_EQ(info.size.load(), 0); @@ -90,7 +95,10 @@ TEST(HashtablezInfoTest, PrepareForSampling) { EXPECT_EQ(info.hashes_bitwise_or.load(), 0); EXPECT_EQ(info.hashes_bitwise_and.load(), ~size_t{}); EXPECT_EQ(info.hashes_bitwise_xor.load(), 0); + EXPECT_EQ(info.max_reserve.load(), 0); EXPECT_GE(info.create_time, test_start); + EXPECT_EQ(info.weight, test_stride); + EXPECT_EQ(info.inline_element_size, test_element_size); info.capacity.store(1, std::memory_order_relaxed); info.size.store(1, std::memory_order_relaxed); @@ -100,9 +108,10 @@ TEST(HashtablezInfoTest, PrepareForSampling) { info.hashes_bitwise_or.store(1, std::memory_order_relaxed); info.hashes_bitwise_and.store(1, std::memory_order_relaxed); info.hashes_bitwise_xor.store(1, std::memory_order_relaxed); + info.max_reserve.store(1, std::memory_order_relaxed); info.create_time = test_start - absl::Hours(20); - info.PrepareForSampling(); + info.PrepareForSampling(test_stride * 2, test_element_size); EXPECT_EQ(info.capacity.load(), 0); EXPECT_EQ(info.size.load(), 0); EXPECT_EQ(info.num_erases.load(), 0); @@ -112,13 +121,18 @@ TEST(HashtablezInfoTest, PrepareForSampling) { EXPECT_EQ(info.hashes_bitwise_or.load(), 0); EXPECT_EQ(info.hashes_bitwise_and.load(), ~size_t{}); EXPECT_EQ(info.hashes_bitwise_xor.load(), 0); + EXPECT_EQ(info.max_reserve.load(), 0); + EXPECT_EQ(info.weight, 2 * test_stride); + EXPECT_EQ(info.inline_element_size, test_element_size); EXPECT_GE(info.create_time, test_start); } TEST(HashtablezInfoTest, RecordStorageChanged) { HashtablezInfo info; absl::MutexLock l(&info.init_mu); - info.PrepareForSampling(); + const int64_t test_stride = 21; + const size_t test_element_size = 19; + info.PrepareForSampling(test_stride, test_element_size); RecordStorageChangedSlow(&info, 17, 47); EXPECT_EQ(info.size.load(), 17); EXPECT_EQ(info.capacity.load(), 47); @@ -130,7 +144,9 @@ TEST(HashtablezInfoTest, RecordStorageChanged) { TEST(HashtablezInfoTest, RecordInsert) { HashtablezInfo info; absl::MutexLock l(&info.init_mu); - info.PrepareForSampling(); + const int64_t test_stride = 25; + const size_t test_element_size = 23; + info.PrepareForSampling(test_stride, test_element_size); EXPECT_EQ(info.max_probe_length.load(), 0); RecordInsertSlow(&info, 0x0000FF00, 6 * kProbeLength); EXPECT_EQ(info.max_probe_length.load(), 6); @@ -150,9 +166,11 @@ TEST(HashtablezInfoTest, RecordInsert) { } TEST(HashtablezInfoTest, RecordErase) { + const int64_t test_stride = 31; + const size_t test_element_size = 29; HashtablezInfo info; absl::MutexLock l(&info.init_mu); - info.PrepareForSampling(); + info.PrepareForSampling(test_stride, test_element_size); EXPECT_EQ(info.num_erases.load(), 0); EXPECT_EQ(info.size.load(), 0); RecordInsertSlow(&info, 0x0000FF00, 6 * kProbeLength); @@ -160,12 +178,15 @@ TEST(HashtablezInfoTest, RecordErase) { RecordEraseSlow(&info); EXPECT_EQ(info.size.load(), 0); EXPECT_EQ(info.num_erases.load(), 1); + EXPECT_EQ(info.inline_element_size, test_element_size); } TEST(HashtablezInfoTest, RecordRehash) { + const int64_t test_stride = 33; + const size_t test_element_size = 31; HashtablezInfo info; absl::MutexLock l(&info.init_mu); - info.PrepareForSampling(); + info.PrepareForSampling(test_stride, test_element_size); RecordInsertSlow(&info, 0x1, 0); RecordInsertSlow(&info, 0x2, kProbeLength); RecordInsertSlow(&info, 0x4, kProbeLength); @@ -184,43 +205,67 @@ TEST(HashtablezInfoTest, RecordRehash) { EXPECT_EQ(info.total_probe_length.load(), 3); EXPECT_EQ(info.num_erases.load(), 0); EXPECT_EQ(info.num_rehashes.load(), 1); + EXPECT_EQ(info.inline_element_size, test_element_size); +} + +TEST(HashtablezInfoTest, RecordReservation) { + HashtablezInfo info; + absl::MutexLock l(&info.init_mu); + const int64_t test_stride = 35; + const size_t test_element_size = 33; + info.PrepareForSampling(test_stride, test_element_size); + RecordReservationSlow(&info, 3); + EXPECT_EQ(info.max_reserve.load(), 3); + + RecordReservationSlow(&info, 2); + // High watermark does not change + EXPECT_EQ(info.max_reserve.load(), 3); + + RecordReservationSlow(&info, 10); + // High watermark does change + EXPECT_EQ(info.max_reserve.load(), 10); } #if defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE) TEST(HashtablezSamplerTest, SmallSampleParameter) { + const size_t test_element_size = 31; SetHashtablezEnabled(true); SetHashtablezSampleParameter(100); for (int i = 0; i < 1000; ++i) { - int64_t next_sample = 0; - HashtablezInfo* sample = SampleSlow(&next_sample); - EXPECT_GT(next_sample, 0); + SamplingState next_sample = {0, 0}; + HashtablezInfo* sample = SampleSlow(next_sample, test_element_size); + EXPECT_GT(next_sample.next_sample, 0); + EXPECT_EQ(next_sample.next_sample, next_sample.sample_stride); EXPECT_NE(sample, nullptr); UnsampleSlow(sample); } } TEST(HashtablezSamplerTest, LargeSampleParameter) { + const size_t test_element_size = 31; SetHashtablezEnabled(true); SetHashtablezSampleParameter(std::numeric_limits<int32_t>::max()); for (int i = 0; i < 1000; ++i) { - int64_t next_sample = 0; - HashtablezInfo* sample = SampleSlow(&next_sample); - EXPECT_GT(next_sample, 0); + SamplingState next_sample = {0, 0}; + HashtablezInfo* sample = SampleSlow(next_sample, test_element_size); + EXPECT_GT(next_sample.next_sample, 0); + EXPECT_EQ(next_sample.next_sample, next_sample.sample_stride); EXPECT_NE(sample, nullptr); UnsampleSlow(sample); } } TEST(HashtablezSamplerTest, Sample) { + const size_t test_element_size = 31; SetHashtablezEnabled(true); SetHashtablezSampleParameter(100); int64_t num_sampled = 0; int64_t total = 0; double sample_rate = 0.0; for (int i = 0; i < 1000000; ++i) { - HashtablezInfoHandle h = Sample(); + HashtablezInfoHandle h = Sample(test_element_size); ++total; if (HashtablezInfoHandlePeer::IsSampled(h)) { ++num_sampled; @@ -232,14 +277,17 @@ TEST(HashtablezSamplerTest, Sample) { } TEST(HashtablezSamplerTest, Handle) { - auto& sampler = HashtablezSampler::Global(); - HashtablezInfoHandle h(sampler.Register()); + auto& sampler = GlobalHashtablezSampler(); + const int64_t test_stride = 41; + const size_t test_element_size = 39; + HashtablezInfoHandle h(sampler.Register(test_stride, test_element_size)); auto* info = HashtablezInfoHandlePeer::GetInfo(&h); info->hashes_bitwise_and.store(0x12345678, std::memory_order_relaxed); bool found = false; sampler.Iterate([&](const HashtablezInfo& h) { if (&h == info) { + EXPECT_EQ(h.weight, test_stride); EXPECT_EQ(h.hashes_bitwise_and.load(), 0x12345678); found = true; } @@ -305,18 +353,20 @@ TEST(HashtablezSamplerTest, MultiThreaded) { ThreadPool pool(10); for (int i = 0; i < 10; ++i) { - pool.Schedule([&sampler, &stop]() { + const int64_t sampling_stride = 11 + i % 3; + const size_t elt_size = 10 + i % 2; + pool.Schedule([&sampler, &stop, sampling_stride, elt_size]() { std::random_device rd; std::mt19937 gen(rd()); std::vector<HashtablezInfo*> infoz; while (!stop.HasBeenNotified()) { if (infoz.empty()) { - infoz.push_back(sampler.Register()); + infoz.push_back(sampler.Register(sampling_stride, elt_size)); } switch (std::uniform_int_distribution<>(0, 2)(gen)) { case 0: { - infoz.push_back(sampler.Register()); + infoz.push_back(sampler.Register(sampling_stride, elt_size)); break; } case 1: { @@ -325,6 +375,7 @@ TEST(HashtablezSamplerTest, MultiThreaded) { HashtablezInfo* info = infoz[p]; infoz[p] = infoz.back(); infoz.pop_back(); + EXPECT_EQ(info->weight, sampling_stride); sampler.Unregister(info); break; } diff --git a/absl/container/internal/have_sse.h b/absl/container/internal/have_sse.h deleted file mode 100644 index e75e1a16..00000000 --- a/absl/container/internal/have_sse.h +++ /dev/null @@ -1,50 +0,0 @@ -// Copyright 2018 The Abseil Authors. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -// -// Shared config probing for SSE instructions used in Swiss tables. -#ifndef ABSL_CONTAINER_INTERNAL_HAVE_SSE_H_ -#define ABSL_CONTAINER_INTERNAL_HAVE_SSE_H_ - -#ifndef ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSE2 -#if defined(__SSE2__) || \ - (defined(_MSC_VER) && \ - (defined(_M_X64) || (defined(_M_IX86) && _M_IX86_FP >= 2))) -#define ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSE2 1 -#else -#define ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSE2 0 -#endif -#endif - -#ifndef ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSSE3 -#ifdef __SSSE3__ -#define ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSSE3 1 -#else -#define ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSSE3 0 -#endif -#endif - -#if ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSSE3 && \ - !ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSE2 -#error "Bad configuration!" -#endif - -#if ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSE2 -#include <emmintrin.h> -#endif - -#if ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSSE3 -#include <tmmintrin.h> -#endif - -#endif // ABSL_CONTAINER_INTERNAL_HAVE_SSE_H_ diff --git a/absl/container/internal/inlined_vector.h b/absl/container/internal/inlined_vector.h index b8aec45b..54c92a01 100644 --- a/absl/container/internal/inlined_vector.h +++ b/absl/container/internal/inlined_vector.h @@ -21,8 +21,11 @@ #include <iterator> #include <limits> #include <memory> +#include <new> +#include <type_traits> #include <utility> +#include "absl/base/attributes.h" #include "absl/base/macros.h" #include "absl/container/internal/compressed_tuple.h" #include "absl/memory/memory.h" @@ -36,116 +39,145 @@ namespace inlined_vector_internal { // GCC does not deal very well with the below code #if !defined(__clang__) && defined(__GNUC__) #pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#pragma GCC diagnostic ignored "-Warray-bounds" #endif +template <typename A> +using AllocatorTraits = std::allocator_traits<A>; +template <typename A> +using ValueType = typename AllocatorTraits<A>::value_type; +template <typename A> +using SizeType = typename AllocatorTraits<A>::size_type; +template <typename A> +using Pointer = typename AllocatorTraits<A>::pointer; +template <typename A> +using ConstPointer = typename AllocatorTraits<A>::const_pointer; +template <typename A> +using SizeType = typename AllocatorTraits<A>::size_type; +template <typename A> +using DifferenceType = typename AllocatorTraits<A>::difference_type; +template <typename A> +using Reference = ValueType<A>&; +template <typename A> +using ConstReference = const ValueType<A>&; +template <typename A> +using Iterator = Pointer<A>; +template <typename A> +using ConstIterator = ConstPointer<A>; +template <typename A> +using ReverseIterator = typename std::reverse_iterator<Iterator<A>>; +template <typename A> +using ConstReverseIterator = typename std::reverse_iterator<ConstIterator<A>>; +template <typename A> +using MoveIterator = typename std::move_iterator<Iterator<A>>; + template <typename Iterator> using IsAtLeastForwardIterator = std::is_convertible< typename std::iterator_traits<Iterator>::iterator_category, std::forward_iterator_tag>; -template <typename AllocatorType, - typename ValueType = - typename absl::allocator_traits<AllocatorType>::value_type> +template <typename A> using IsMemcpyOk = - absl::conjunction<std::is_same<AllocatorType, std::allocator<ValueType>>, - absl::is_trivially_copy_constructible<ValueType>, - absl::is_trivially_copy_assignable<ValueType>, - absl::is_trivially_destructible<ValueType>>; - -template <typename AllocatorType, typename Pointer, typename SizeType> -void DestroyElements(AllocatorType* alloc_ptr, Pointer destroy_first, - SizeType destroy_size) { - using AllocatorTraits = absl::allocator_traits<AllocatorType>; - - if (destroy_first != nullptr) { - for (auto i = destroy_size; i != 0;) { + absl::conjunction<std::is_same<A, std::allocator<ValueType<A>>>, + absl::is_trivially_copy_constructible<ValueType<A>>, + absl::is_trivially_copy_assignable<ValueType<A>>, + absl::is_trivially_destructible<ValueType<A>>>; + +template <typename T> +struct TypeIdentity { + using type = T; +}; + +// Used for function arguments in template functions to prevent ADL by forcing +// callers to explicitly specify the template parameter. +template <typename T> +using NoTypeDeduction = typename TypeIdentity<T>::type; + +template <typename A, bool IsTriviallyDestructible = + absl::is_trivially_destructible<ValueType<A>>::value> +struct DestroyAdapter; + +template <typename A> +struct DestroyAdapter<A, /* IsTriviallyDestructible */ false> { + static void DestroyElements(A& allocator, Pointer<A> destroy_first, + SizeType<A> destroy_size) { + for (SizeType<A> i = destroy_size; i != 0;) { --i; - AllocatorTraits::destroy(*alloc_ptr, destroy_first + i); + AllocatorTraits<A>::destroy(allocator, destroy_first + i); } + } +}; -#if !defined(NDEBUG) - { - using ValueType = typename AllocatorTraits::value_type; - - // Overwrite unused memory with `0xab` so we can catch uninitialized - // usage. - // - // Cast to `void*` to tell the compiler that we don't care that we might - // be scribbling on a vtable pointer. - void* memory_ptr = destroy_first; - auto memory_size = destroy_size * sizeof(ValueType); - std::memset(memory_ptr, 0xab, memory_size); - } -#endif // !defined(NDEBUG) +template <typename A> +struct DestroyAdapter<A, /* IsTriviallyDestructible */ true> { + static void DestroyElements(A& allocator, Pointer<A> destroy_first, + SizeType<A> destroy_size) { + static_cast<void>(allocator); + static_cast<void>(destroy_first); + static_cast<void>(destroy_size); } -} +}; -// If kUseMemcpy is true, memcpy(dst, src, n); else do nothing. -// Useful to avoid compiler warnings when memcpy() is used for T values -// that are not trivially copyable in non-reachable code. -template <bool kUseMemcpy> -inline void MemcpyIfAllowed(void* dst, const void* src, size_t n); +template <typename A> +struct Allocation { + Pointer<A> data; + SizeType<A> capacity; +}; -// memcpy when allowed. -template <> -inline void MemcpyIfAllowed<true>(void* dst, const void* src, size_t n) { - memcpy(dst, src, n); -} +template <typename A, + bool IsOverAligned = + (alignof(ValueType<A>) > ABSL_INTERNAL_DEFAULT_NEW_ALIGNMENT)> +struct MallocAdapter { + static Allocation<A> Allocate(A& allocator, SizeType<A> requested_capacity) { + return {AllocatorTraits<A>::allocate(allocator, requested_capacity), + requested_capacity}; + } -// Do nothing for types that are not memcpy-able. This function is only -// called from non-reachable branches. -template <> -inline void MemcpyIfAllowed<false>(void*, const void*, size_t) {} + static void Deallocate(A& allocator, Pointer<A> pointer, + SizeType<A> capacity) { + AllocatorTraits<A>::deallocate(allocator, pointer, capacity); + } +}; -template <typename AllocatorType, typename Pointer, typename ValueAdapter, - typename SizeType> -void ConstructElements(AllocatorType* alloc_ptr, Pointer construct_first, - ValueAdapter* values_ptr, SizeType construct_size) { - for (SizeType i = 0; i < construct_size; ++i) { - ABSL_INTERNAL_TRY { - values_ptr->ConstructNext(alloc_ptr, construct_first + i); - } +template <typename A, typename ValueAdapter> +void ConstructElements(NoTypeDeduction<A>& allocator, + Pointer<A> construct_first, ValueAdapter& values, + SizeType<A> construct_size) { + for (SizeType<A> i = 0; i < construct_size; ++i) { + ABSL_INTERNAL_TRY { values.ConstructNext(allocator, construct_first + i); } ABSL_INTERNAL_CATCH_ANY { - inlined_vector_internal::DestroyElements(alloc_ptr, construct_first, i); + DestroyAdapter<A>::DestroyElements(allocator, construct_first, i); ABSL_INTERNAL_RETHROW; } } } -template <typename Pointer, typename ValueAdapter, typename SizeType> -void AssignElements(Pointer assign_first, ValueAdapter* values_ptr, - SizeType assign_size) { - for (SizeType i = 0; i < assign_size; ++i) { - values_ptr->AssignNext(assign_first + i); +template <typename A, typename ValueAdapter> +void AssignElements(Pointer<A> assign_first, ValueAdapter& values, + SizeType<A> assign_size) { + for (SizeType<A> i = 0; i < assign_size; ++i) { + values.AssignNext(assign_first + i); } } -template <typename AllocatorType> +template <typename A> struct StorageView { - using AllocatorTraits = absl::allocator_traits<AllocatorType>; - using Pointer = typename AllocatorTraits::pointer; - using SizeType = typename AllocatorTraits::size_type; - - Pointer data; - SizeType size; - SizeType capacity; + Pointer<A> data; + SizeType<A> size; + SizeType<A> capacity; }; -template <typename AllocatorType, typename Iterator> +template <typename A, typename Iterator> class IteratorValueAdapter { - using AllocatorTraits = absl::allocator_traits<AllocatorType>; - using Pointer = typename AllocatorTraits::pointer; - public: explicit IteratorValueAdapter(const Iterator& it) : it_(it) {} - void ConstructNext(AllocatorType* alloc_ptr, Pointer construct_at) { - AllocatorTraits::construct(*alloc_ptr, construct_at, *it_); + void ConstructNext(A& allocator, Pointer<A> construct_at) { + AllocatorTraits<A>::construct(allocator, construct_at, *it_); ++it_; } - void AssignNext(Pointer assign_at) { + void AssignNext(Pointer<A> assign_at) { *assign_at = *it_; ++it_; } @@ -154,166 +186,123 @@ class IteratorValueAdapter { Iterator it_; }; -template <typename AllocatorType> +template <typename A> class CopyValueAdapter { - using AllocatorTraits = absl::allocator_traits<AllocatorType>; - using ValueType = typename AllocatorTraits::value_type; - using Pointer = typename AllocatorTraits::pointer; - using ConstPointer = typename AllocatorTraits::const_pointer; - public: - explicit CopyValueAdapter(const ValueType& v) : ptr_(std::addressof(v)) {} + explicit CopyValueAdapter(ConstPointer<A> p) : ptr_(p) {} - void ConstructNext(AllocatorType* alloc_ptr, Pointer construct_at) { - AllocatorTraits::construct(*alloc_ptr, construct_at, *ptr_); + void ConstructNext(A& allocator, Pointer<A> construct_at) { + AllocatorTraits<A>::construct(allocator, construct_at, *ptr_); } - void AssignNext(Pointer assign_at) { *assign_at = *ptr_; } + void AssignNext(Pointer<A> assign_at) { *assign_at = *ptr_; } private: - ConstPointer ptr_; + ConstPointer<A> ptr_; }; -template <typename AllocatorType> +template <typename A> class DefaultValueAdapter { - using AllocatorTraits = absl::allocator_traits<AllocatorType>; - using ValueType = typename AllocatorTraits::value_type; - using Pointer = typename AllocatorTraits::pointer; - public: explicit DefaultValueAdapter() {} - void ConstructNext(AllocatorType* alloc_ptr, Pointer construct_at) { - AllocatorTraits::construct(*alloc_ptr, construct_at); + void ConstructNext(A& allocator, Pointer<A> construct_at) { + AllocatorTraits<A>::construct(allocator, construct_at); } - void AssignNext(Pointer assign_at) { *assign_at = ValueType(); } + void AssignNext(Pointer<A> assign_at) { *assign_at = ValueType<A>(); } }; -template <typename AllocatorType> +template <typename A> class AllocationTransaction { - using AllocatorTraits = absl::allocator_traits<AllocatorType>; - using Pointer = typename AllocatorTraits::pointer; - using SizeType = typename AllocatorTraits::size_type; - public: - explicit AllocationTransaction(AllocatorType* alloc_ptr) - : alloc_data_(*alloc_ptr, nullptr) {} + explicit AllocationTransaction(A& allocator) + : allocator_data_(allocator, nullptr), capacity_(0) {} ~AllocationTransaction() { if (DidAllocate()) { - AllocatorTraits::deallocate(GetAllocator(), GetData(), GetCapacity()); + MallocAdapter<A>::Deallocate(GetAllocator(), GetData(), GetCapacity()); } } AllocationTransaction(const AllocationTransaction&) = delete; void operator=(const AllocationTransaction&) = delete; - AllocatorType& GetAllocator() { return alloc_data_.template get<0>(); } - Pointer& GetData() { return alloc_data_.template get<1>(); } - SizeType& GetCapacity() { return capacity_; } + A& GetAllocator() { return allocator_data_.template get<0>(); } + Pointer<A>& GetData() { return allocator_data_.template get<1>(); } + SizeType<A>& GetCapacity() { return capacity_; } bool DidAllocate() { return GetData() != nullptr; } - Pointer Allocate(SizeType capacity) { - GetData() = AllocatorTraits::allocate(GetAllocator(), capacity); - GetCapacity() = capacity; - return GetData(); + + Pointer<A> Allocate(SizeType<A> requested_capacity) { + Allocation<A> result = + MallocAdapter<A>::Allocate(GetAllocator(), requested_capacity); + GetData() = result.data; + GetCapacity() = result.capacity; + return result.data; } + ABSL_MUST_USE_RESULT Allocation<A> Release() && { + Allocation<A> result = {GetData(), GetCapacity()}; + Reset(); + return result; + } + + private: void Reset() { GetData() = nullptr; GetCapacity() = 0; } - private: - container_internal::CompressedTuple<AllocatorType, Pointer> alloc_data_; - SizeType capacity_ = 0; + container_internal::CompressedTuple<A, Pointer<A>> allocator_data_; + SizeType<A> capacity_; }; -template <typename AllocatorType> +template <typename A> class ConstructionTransaction { - using AllocatorTraits = absl::allocator_traits<AllocatorType>; - using Pointer = typename AllocatorTraits::pointer; - using SizeType = typename AllocatorTraits::size_type; - public: - explicit ConstructionTransaction(AllocatorType* alloc_ptr) - : alloc_data_(*alloc_ptr, nullptr) {} + explicit ConstructionTransaction(A& allocator) + : allocator_data_(allocator, nullptr), size_(0) {} ~ConstructionTransaction() { if (DidConstruct()) { - inlined_vector_internal::DestroyElements(std::addressof(GetAllocator()), - GetData(), GetSize()); + DestroyAdapter<A>::DestroyElements(GetAllocator(), GetData(), GetSize()); } } ConstructionTransaction(const ConstructionTransaction&) = delete; void operator=(const ConstructionTransaction&) = delete; - AllocatorType& GetAllocator() { return alloc_data_.template get<0>(); } - Pointer& GetData() { return alloc_data_.template get<1>(); } - SizeType& GetSize() { return size_; } + A& GetAllocator() { return allocator_data_.template get<0>(); } + Pointer<A>& GetData() { return allocator_data_.template get<1>(); } + SizeType<A>& GetSize() { return size_; } bool DidConstruct() { return GetData() != nullptr; } template <typename ValueAdapter> - void Construct(Pointer data, ValueAdapter* values_ptr, SizeType size) { - inlined_vector_internal::ConstructElements(std::addressof(GetAllocator()), - data, values_ptr, size); + void Construct(Pointer<A> data, ValueAdapter& values, SizeType<A> size) { + ConstructElements<A>(GetAllocator(), data, values, size); GetData() = data; GetSize() = size; } - void Commit() { + void Commit() && { GetData() = nullptr; GetSize() = 0; } private: - container_internal::CompressedTuple<AllocatorType, Pointer> alloc_data_; - SizeType size_ = 0; + container_internal::CompressedTuple<A, Pointer<A>> allocator_data_; + SizeType<A> size_; }; template <typename T, size_t N, typename A> class Storage { public: - using AllocatorTraits = absl::allocator_traits<A>; - using allocator_type = typename AllocatorTraits::allocator_type; - using value_type = typename AllocatorTraits::value_type; - using pointer = typename AllocatorTraits::pointer; - using const_pointer = typename AllocatorTraits::const_pointer; - using size_type = typename AllocatorTraits::size_type; - using difference_type = typename AllocatorTraits::difference_type; - - using reference = value_type&; - using const_reference = const value_type&; - using RValueReference = value_type&&; - using iterator = pointer; - using const_iterator = const_pointer; - using reverse_iterator = std::reverse_iterator<iterator>; - using const_reverse_iterator = std::reverse_iterator<const_iterator>; - using MoveIterator = std::move_iterator<iterator>; - using IsMemcpyOk = inlined_vector_internal::IsMemcpyOk<allocator_type>; - - using StorageView = inlined_vector_internal::StorageView<allocator_type>; - - template <typename Iterator> - using IteratorValueAdapter = - inlined_vector_internal::IteratorValueAdapter<allocator_type, Iterator>; - using CopyValueAdapter = - inlined_vector_internal::CopyValueAdapter<allocator_type>; - using DefaultValueAdapter = - inlined_vector_internal::DefaultValueAdapter<allocator_type>; - - using AllocationTransaction = - inlined_vector_internal::AllocationTransaction<allocator_type>; - using ConstructionTransaction = - inlined_vector_internal::ConstructionTransaction<allocator_type>; - - static size_type NextCapacity(size_type current_capacity) { + static SizeType<A> NextCapacity(SizeType<A> current_capacity) { return current_capacity * 2; } - static size_type ComputeCapacity(size_type current_capacity, - size_type requested_capacity) { + static SizeType<A> ComputeCapacity(SizeType<A> current_capacity, + SizeType<A> requested_capacity) { return (std::max)(NextCapacity(current_capacity), requested_capacity); } @@ -321,15 +310,15 @@ class Storage { // Storage Constructors and Destructor // --------------------------------------------------------------------------- - Storage() : metadata_(allocator_type(), /* size and is_allocated */ 0) {} + Storage() : metadata_(A(), /* size and is_allocated */ 0u) {} - explicit Storage(const allocator_type& alloc) - : metadata_(alloc, /* size and is_allocated */ 0) {} + explicit Storage(const A& allocator) + : metadata_(allocator, /* size and is_allocated */ 0u) {} ~Storage() { if (GetSizeAndIsAllocated() == 0) { // Empty and not allocated; nothing to do. - } else if (IsMemcpyOk::value) { + } else if (IsMemcpyOk<A>::value) { // No destructors need to be run; just deallocate if necessary. DeallocateIfAllocated(); } else { @@ -341,52 +330,48 @@ class Storage { // Storage Member Accessors // --------------------------------------------------------------------------- - size_type& GetSizeAndIsAllocated() { return metadata_.template get<1>(); } + SizeType<A>& GetSizeAndIsAllocated() { return metadata_.template get<1>(); } - const size_type& GetSizeAndIsAllocated() const { + const SizeType<A>& GetSizeAndIsAllocated() const { return metadata_.template get<1>(); } - size_type GetSize() const { return GetSizeAndIsAllocated() >> 1; } + SizeType<A> GetSize() const { return GetSizeAndIsAllocated() >> 1; } bool GetIsAllocated() const { return GetSizeAndIsAllocated() & 1; } - pointer GetAllocatedData() { return data_.allocated.allocated_data; } + Pointer<A> GetAllocatedData() { return data_.allocated.allocated_data; } - const_pointer GetAllocatedData() const { + ConstPointer<A> GetAllocatedData() const { return data_.allocated.allocated_data; } - pointer GetInlinedData() { - return reinterpret_cast<pointer>( + Pointer<A> GetInlinedData() { + return reinterpret_cast<Pointer<A>>( std::addressof(data_.inlined.inlined_data[0])); } - const_pointer GetInlinedData() const { - return reinterpret_cast<const_pointer>( + ConstPointer<A> GetInlinedData() const { + return reinterpret_cast<ConstPointer<A>>( std::addressof(data_.inlined.inlined_data[0])); } - size_type GetAllocatedCapacity() const { + SizeType<A> GetAllocatedCapacity() const { return data_.allocated.allocated_capacity; } - size_type GetInlinedCapacity() const { return static_cast<size_type>(N); } + SizeType<A> GetInlinedCapacity() const { return static_cast<SizeType<A>>(N); } - StorageView MakeStorageView() { - return GetIsAllocated() - ? StorageView{GetAllocatedData(), GetSize(), - GetAllocatedCapacity()} - : StorageView{GetInlinedData(), GetSize(), GetInlinedCapacity()}; + StorageView<A> MakeStorageView() { + return GetIsAllocated() ? StorageView<A>{GetAllocatedData(), GetSize(), + GetAllocatedCapacity()} + : StorageView<A>{GetInlinedData(), GetSize(), + GetInlinedCapacity()}; } - allocator_type* GetAllocPtr() { - return std::addressof(metadata_.template get<0>()); - } + A& GetAllocator() { return metadata_.template get<0>(); } - const allocator_type* GetAllocPtr() const { - return std::addressof(metadata_.template get<0>()); - } + const A& GetAllocator() const { return metadata_.template get<0>(); } // --------------------------------------------------------------------------- // Storage Member Mutators @@ -395,74 +380,68 @@ class Storage { ABSL_ATTRIBUTE_NOINLINE void InitFrom(const Storage& other); template <typename ValueAdapter> - void Initialize(ValueAdapter values, size_type new_size); + void Initialize(ValueAdapter values, SizeType<A> new_size); template <typename ValueAdapter> - void Assign(ValueAdapter values, size_type new_size); + void Assign(ValueAdapter values, SizeType<A> new_size); template <typename ValueAdapter> - void Resize(ValueAdapter values, size_type new_size); + void Resize(ValueAdapter values, SizeType<A> new_size); template <typename ValueAdapter> - iterator Insert(const_iterator pos, ValueAdapter values, - size_type insert_count); + Iterator<A> Insert(ConstIterator<A> pos, ValueAdapter values, + SizeType<A> insert_count); template <typename... Args> - reference EmplaceBack(Args&&... args); + Reference<A> EmplaceBack(Args&&... args); - iterator Erase(const_iterator from, const_iterator to); + Iterator<A> Erase(ConstIterator<A> from, ConstIterator<A> to); - void Reserve(size_type requested_capacity); + void Reserve(SizeType<A> requested_capacity); void ShrinkToFit(); void Swap(Storage* other_storage_ptr); void SetIsAllocated() { - GetSizeAndIsAllocated() |= static_cast<size_type>(1); + GetSizeAndIsAllocated() |= static_cast<SizeType<A>>(1); } void UnsetIsAllocated() { - GetSizeAndIsAllocated() &= ((std::numeric_limits<size_type>::max)() - 1); + GetSizeAndIsAllocated() &= ((std::numeric_limits<SizeType<A>>::max)() - 1); } - void SetSize(size_type size) { + void SetSize(SizeType<A> size) { GetSizeAndIsAllocated() = - (size << 1) | static_cast<size_type>(GetIsAllocated()); + (size << 1) | static_cast<SizeType<A>>(GetIsAllocated()); } - void SetAllocatedSize(size_type size) { - GetSizeAndIsAllocated() = (size << 1) | static_cast<size_type>(1); + void SetAllocatedSize(SizeType<A> size) { + GetSizeAndIsAllocated() = (size << 1) | static_cast<SizeType<A>>(1); } - void SetInlinedSize(size_type size) { - GetSizeAndIsAllocated() = size << static_cast<size_type>(1); + void SetInlinedSize(SizeType<A> size) { + GetSizeAndIsAllocated() = size << static_cast<SizeType<A>>(1); } - void AddSize(size_type count) { - GetSizeAndIsAllocated() += count << static_cast<size_type>(1); + void AddSize(SizeType<A> count) { + GetSizeAndIsAllocated() += count << static_cast<SizeType<A>>(1); } - void SubtractSize(size_type count) { - assert(count <= GetSize()); + void SubtractSize(SizeType<A> count) { + ABSL_HARDENING_ASSERT(count <= GetSize()); - GetSizeAndIsAllocated() -= count << static_cast<size_type>(1); + GetSizeAndIsAllocated() -= count << static_cast<SizeType<A>>(1); } - void SetAllocatedData(pointer data, size_type capacity) { - data_.allocated.allocated_data = data; - data_.allocated.allocated_capacity = capacity; - } - - void AcquireAllocatedData(AllocationTransaction* allocation_tx_ptr) { - SetAllocatedData(allocation_tx_ptr->GetData(), - allocation_tx_ptr->GetCapacity()); - - allocation_tx_ptr->Reset(); + void SetAllocation(Allocation<A> allocation) { + data_.allocated.allocated_data = allocation.data; + data_.allocated.allocated_capacity = allocation.capacity; } void MemcpyFrom(const Storage& other_storage) { - assert(IsMemcpyOk::value || other_storage.GetIsAllocated()); + ABSL_HARDENING_ASSERT(IsMemcpyOk<A>::value || + other_storage.GetIsAllocated()); GetSizeAndIsAllocated() = other_storage.GetSizeAndIsAllocated(); data_ = other_storage.data_; @@ -470,24 +449,23 @@ class Storage { void DeallocateIfAllocated() { if (GetIsAllocated()) { - AllocatorTraits::deallocate(*GetAllocPtr(), GetAllocatedData(), - GetAllocatedCapacity()); + MallocAdapter<A>::Deallocate(GetAllocator(), GetAllocatedData(), + GetAllocatedCapacity()); } } private: ABSL_ATTRIBUTE_NOINLINE void DestroyContents(); - using Metadata = - container_internal::CompressedTuple<allocator_type, size_type>; + using Metadata = container_internal::CompressedTuple<A, SizeType<A>>; struct Allocated { - pointer allocated_data; - size_type allocated_capacity; + Pointer<A> allocated_data; + SizeType<A> allocated_capacity; }; struct Inlined { - alignas(value_type) char inlined_data[sizeof(value_type[N])]; + alignas(ValueType<A>) char inlined_data[sizeof(ValueType<A>[N])]; }; union Data { @@ -496,7 +474,7 @@ class Storage { }; template <typename... Args> - ABSL_ATTRIBUTE_NOINLINE reference EmplaceBackSlow(Args&&... args); + ABSL_ATTRIBUTE_NOINLINE Reference<A> EmplaceBackSlow(Args&&... args); Metadata metadata_; Data data_; @@ -504,17 +482,17 @@ class Storage { template <typename T, size_t N, typename A> void Storage<T, N, A>::DestroyContents() { - pointer data = GetIsAllocated() ? GetAllocatedData() : GetInlinedData(); - inlined_vector_internal::DestroyElements(GetAllocPtr(), data, GetSize()); + Pointer<A> data = GetIsAllocated() ? GetAllocatedData() : GetInlinedData(); + DestroyAdapter<A>::DestroyElements(GetAllocator(), data, GetSize()); DeallocateIfAllocated(); } template <typename T, size_t N, typename A> void Storage<T, N, A>::InitFrom(const Storage& other) { - const auto n = other.GetSize(); - assert(n > 0); // Empty sources handled handled in caller. - const_pointer src; - pointer dst; + const SizeType<A> n = other.GetSize(); + ABSL_HARDENING_ASSERT(n > 0); // Empty sources handled handled in caller. + ConstPointer<A> src; + Pointer<A> dst; if (!other.GetIsAllocated()) { dst = GetInlinedData(); src = other.GetInlinedData(); @@ -522,43 +500,48 @@ void Storage<T, N, A>::InitFrom(const Storage& other) { // Because this is only called from the `InlinedVector` constructors, it's // safe to take on the allocation with size `0`. If `ConstructElements(...)` // throws, deallocation will be automatically handled by `~Storage()`. - size_type new_capacity = ComputeCapacity(GetInlinedCapacity(), n); - dst = AllocatorTraits::allocate(*GetAllocPtr(), new_capacity); - SetAllocatedData(dst, new_capacity); + SizeType<A> requested_capacity = ComputeCapacity(GetInlinedCapacity(), n); + Allocation<A> allocation = + MallocAdapter<A>::Allocate(GetAllocator(), requested_capacity); + SetAllocation(allocation); + dst = allocation.data; src = other.GetAllocatedData(); } - if (IsMemcpyOk::value) { - MemcpyIfAllowed<IsMemcpyOk::value>(dst, src, sizeof(dst[0]) * n); + if (IsMemcpyOk<A>::value) { + std::memcpy(reinterpret_cast<char*>(dst), + reinterpret_cast<const char*>(src), n * sizeof(ValueType<A>)); } else { - auto values = IteratorValueAdapter<const_pointer>(src); - inlined_vector_internal::ConstructElements(GetAllocPtr(), dst, &values, n); + auto values = IteratorValueAdapter<A, ConstPointer<A>>(src); + ConstructElements<A>(GetAllocator(), dst, values, n); } GetSizeAndIsAllocated() = other.GetSizeAndIsAllocated(); } template <typename T, size_t N, typename A> template <typename ValueAdapter> -auto Storage<T, N, A>::Initialize(ValueAdapter values, size_type new_size) +auto Storage<T, N, A>::Initialize(ValueAdapter values, SizeType<A> new_size) -> void { // Only callable from constructors! - assert(!GetIsAllocated()); - assert(GetSize() == 0); + ABSL_HARDENING_ASSERT(!GetIsAllocated()); + ABSL_HARDENING_ASSERT(GetSize() == 0); - pointer construct_data; + Pointer<A> construct_data; if (new_size > GetInlinedCapacity()) { // Because this is only called from the `InlinedVector` constructors, it's // safe to take on the allocation with size `0`. If `ConstructElements(...)` // throws, deallocation will be automatically handled by `~Storage()`. - size_type new_capacity = ComputeCapacity(GetInlinedCapacity(), new_size); - construct_data = AllocatorTraits::allocate(*GetAllocPtr(), new_capacity); - SetAllocatedData(construct_data, new_capacity); + SizeType<A> requested_capacity = + ComputeCapacity(GetInlinedCapacity(), new_size); + Allocation<A> allocation = + MallocAdapter<A>::Allocate(GetAllocator(), requested_capacity); + construct_data = allocation.data; + SetAllocation(allocation); SetIsAllocated(); } else { construct_data = GetInlinedData(); } - inlined_vector_internal::ConstructElements(GetAllocPtr(), construct_data, - &values, new_size); + ConstructElements<A>(GetAllocator(), construct_data, values, new_size); // Since the initial size was guaranteed to be `0` and the allocated bit is // already correct for either case, *adding* `new_size` gives us the correct @@ -568,18 +551,20 @@ auto Storage<T, N, A>::Initialize(ValueAdapter values, size_type new_size) template <typename T, size_t N, typename A> template <typename ValueAdapter> -auto Storage<T, N, A>::Assign(ValueAdapter values, size_type new_size) -> void { - StorageView storage_view = MakeStorageView(); +auto Storage<T, N, A>::Assign(ValueAdapter values, SizeType<A> new_size) + -> void { + StorageView<A> storage_view = MakeStorageView(); - AllocationTransaction allocation_tx(GetAllocPtr()); + AllocationTransaction<A> allocation_tx(GetAllocator()); - absl::Span<value_type> assign_loop; - absl::Span<value_type> construct_loop; - absl::Span<value_type> destroy_loop; + absl::Span<ValueType<A>> assign_loop; + absl::Span<ValueType<A>> construct_loop; + absl::Span<ValueType<A>> destroy_loop; if (new_size > storage_view.capacity) { - size_type new_capacity = ComputeCapacity(storage_view.capacity, new_size); - construct_loop = {allocation_tx.Allocate(new_capacity), new_size}; + SizeType<A> requested_capacity = + ComputeCapacity(storage_view.capacity, new_size); + construct_loop = {allocation_tx.Allocate(requested_capacity), new_size}; destroy_loop = {storage_view.data, storage_view.size}; } else if (new_size > storage_view.size) { assign_loop = {storage_view.data, storage_view.size}; @@ -590,18 +575,17 @@ auto Storage<T, N, A>::Assign(ValueAdapter values, size_type new_size) -> void { destroy_loop = {storage_view.data + new_size, storage_view.size - new_size}; } - inlined_vector_internal::AssignElements(assign_loop.data(), &values, - assign_loop.size()); + AssignElements<A>(assign_loop.data(), values, assign_loop.size()); - inlined_vector_internal::ConstructElements( - GetAllocPtr(), construct_loop.data(), &values, construct_loop.size()); + ConstructElements<A>(GetAllocator(), construct_loop.data(), values, + construct_loop.size()); - inlined_vector_internal::DestroyElements(GetAllocPtr(), destroy_loop.data(), - destroy_loop.size()); + DestroyAdapter<A>::DestroyElements(GetAllocator(), destroy_loop.data(), + destroy_loop.size()); if (allocation_tx.DidAllocate()) { DeallocateIfAllocated(); - AcquireAllocatedData(&allocation_tx); + SetAllocation(std::move(allocation_tx).Release()); SetIsAllocated(); } @@ -610,42 +594,42 @@ auto Storage<T, N, A>::Assign(ValueAdapter values, size_type new_size) -> void { template <typename T, size_t N, typename A> template <typename ValueAdapter> -auto Storage<T, N, A>::Resize(ValueAdapter values, size_type new_size) -> void { - StorageView storage_view = MakeStorageView(); - auto* const base = storage_view.data; - const size_type size = storage_view.size; - auto* alloc = GetAllocPtr(); +auto Storage<T, N, A>::Resize(ValueAdapter values, SizeType<A> new_size) + -> void { + StorageView<A> storage_view = MakeStorageView(); + Pointer<A> const base = storage_view.data; + const SizeType<A> size = storage_view.size; + A& alloc = GetAllocator(); if (new_size <= size) { // Destroy extra old elements. - inlined_vector_internal::DestroyElements(alloc, base + new_size, - size - new_size); + DestroyAdapter<A>::DestroyElements(alloc, base + new_size, size - new_size); } else if (new_size <= storage_view.capacity) { // Construct new elements in place. - inlined_vector_internal::ConstructElements(alloc, base + size, &values, - new_size - size); + ConstructElements<A>(alloc, base + size, values, new_size - size); } else { // Steps: // a. Allocate new backing store. // b. Construct new elements in new backing store. - // c. Move existing elements from old backing store to now. + // c. Move existing elements from old backing store to new backing store. // d. Destroy all elements in old backing store. // Use transactional wrappers for the first two steps so we can roll // back if necessary due to exceptions. - AllocationTransaction allocation_tx(alloc); - size_type new_capacity = ComputeCapacity(storage_view.capacity, new_size); - pointer new_data = allocation_tx.Allocate(new_capacity); + AllocationTransaction<A> allocation_tx(alloc); + SizeType<A> requested_capacity = + ComputeCapacity(storage_view.capacity, new_size); + Pointer<A> new_data = allocation_tx.Allocate(requested_capacity); - ConstructionTransaction construction_tx(alloc); - construction_tx.Construct(new_data + size, &values, new_size - size); + ConstructionTransaction<A> construction_tx(alloc); + construction_tx.Construct(new_data + size, values, new_size - size); - IteratorValueAdapter<MoveIterator> move_values((MoveIterator(base))); - inlined_vector_internal::ConstructElements(alloc, new_data, &move_values, - size); + IteratorValueAdapter<A, MoveIterator<A>> move_values( + (MoveIterator<A>(base))); + ConstructElements<A>(alloc, new_data, move_values, size); - inlined_vector_internal::DestroyElements(alloc, base, size); - construction_tx.Commit(); + DestroyAdapter<A>::DestroyElements(alloc, base, size); + std::move(construction_tx).Commit(); DeallocateIfAllocated(); - AcquireAllocatedData(&allocation_tx); + SetAllocation(std::move(allocation_tx).Release()); SetIsAllocated(); } SetSize(new_size); @@ -653,76 +637,77 @@ auto Storage<T, N, A>::Resize(ValueAdapter values, size_type new_size) -> void { template <typename T, size_t N, typename A> template <typename ValueAdapter> -auto Storage<T, N, A>::Insert(const_iterator pos, ValueAdapter values, - size_type insert_count) -> iterator { - StorageView storage_view = MakeStorageView(); +auto Storage<T, N, A>::Insert(ConstIterator<A> pos, ValueAdapter values, + SizeType<A> insert_count) -> Iterator<A> { + StorageView<A> storage_view = MakeStorageView(); - size_type insert_index = - std::distance(const_iterator(storage_view.data), pos); - size_type insert_end_index = insert_index + insert_count; - size_type new_size = storage_view.size + insert_count; + SizeType<A> insert_index = + std::distance(ConstIterator<A>(storage_view.data), pos); + SizeType<A> insert_end_index = insert_index + insert_count; + SizeType<A> new_size = storage_view.size + insert_count; if (new_size > storage_view.capacity) { - AllocationTransaction allocation_tx(GetAllocPtr()); - ConstructionTransaction construction_tx(GetAllocPtr()); - ConstructionTransaction move_construciton_tx(GetAllocPtr()); + AllocationTransaction<A> allocation_tx(GetAllocator()); + ConstructionTransaction<A> construction_tx(GetAllocator()); + ConstructionTransaction<A> move_construction_tx(GetAllocator()); - IteratorValueAdapter<MoveIterator> move_values( - MoveIterator(storage_view.data)); + IteratorValueAdapter<A, MoveIterator<A>> move_values( + MoveIterator<A>(storage_view.data)); - size_type new_capacity = ComputeCapacity(storage_view.capacity, new_size); - pointer new_data = allocation_tx.Allocate(new_capacity); + SizeType<A> requested_capacity = + ComputeCapacity(storage_view.capacity, new_size); + Pointer<A> new_data = allocation_tx.Allocate(requested_capacity); - construction_tx.Construct(new_data + insert_index, &values, insert_count); + construction_tx.Construct(new_data + insert_index, values, insert_count); - move_construciton_tx.Construct(new_data, &move_values, insert_index); + move_construction_tx.Construct(new_data, move_values, insert_index); - inlined_vector_internal::ConstructElements( - GetAllocPtr(), new_data + insert_end_index, &move_values, - storage_view.size - insert_index); + ConstructElements<A>(GetAllocator(), new_data + insert_end_index, + move_values, storage_view.size - insert_index); - inlined_vector_internal::DestroyElements(GetAllocPtr(), storage_view.data, - storage_view.size); + DestroyAdapter<A>::DestroyElements(GetAllocator(), storage_view.data, + storage_view.size); - construction_tx.Commit(); - move_construciton_tx.Commit(); + std::move(construction_tx).Commit(); + std::move(move_construction_tx).Commit(); DeallocateIfAllocated(); - AcquireAllocatedData(&allocation_tx); + SetAllocation(std::move(allocation_tx).Release()); SetAllocatedSize(new_size); - return iterator(new_data + insert_index); + return Iterator<A>(new_data + insert_index); } else { - size_type move_construction_destination_index = + SizeType<A> move_construction_destination_index = (std::max)(insert_end_index, storage_view.size); - ConstructionTransaction move_construction_tx(GetAllocPtr()); + ConstructionTransaction<A> move_construction_tx(GetAllocator()); - IteratorValueAdapter<MoveIterator> move_construction_values( - MoveIterator(storage_view.data + - (move_construction_destination_index - insert_count))); - absl::Span<value_type> move_construction = { + IteratorValueAdapter<A, MoveIterator<A>> move_construction_values( + MoveIterator<A>(storage_view.data + + (move_construction_destination_index - insert_count))); + absl::Span<ValueType<A>> move_construction = { storage_view.data + move_construction_destination_index, new_size - move_construction_destination_index}; - pointer move_assignment_values = storage_view.data + insert_index; - absl::Span<value_type> move_assignment = { + Pointer<A> move_assignment_values = storage_view.data + insert_index; + absl::Span<ValueType<A>> move_assignment = { storage_view.data + insert_end_index, move_construction_destination_index - insert_end_index}; - absl::Span<value_type> insert_assignment = {move_assignment_values, - move_construction.size()}; + absl::Span<ValueType<A>> insert_assignment = {move_assignment_values, + move_construction.size()}; - absl::Span<value_type> insert_construction = { + absl::Span<ValueType<A>> insert_construction = { insert_assignment.data() + insert_assignment.size(), insert_count - insert_assignment.size()}; move_construction_tx.Construct(move_construction.data(), - &move_construction_values, + move_construction_values, move_construction.size()); - for (pointer destination = move_assignment.data() + move_assignment.size(), - last_destination = move_assignment.data(), - source = move_assignment_values + move_assignment.size(); + for (Pointer<A> + destination = move_assignment.data() + move_assignment.size(), + last_destination = move_assignment.data(), + source = move_assignment_values + move_assignment.size(); ;) { --destination; --source; @@ -730,30 +715,29 @@ auto Storage<T, N, A>::Insert(const_iterator pos, ValueAdapter values, *destination = std::move(*source); } - inlined_vector_internal::AssignElements(insert_assignment.data(), &values, - insert_assignment.size()); + AssignElements<A>(insert_assignment.data(), values, + insert_assignment.size()); - inlined_vector_internal::ConstructElements( - GetAllocPtr(), insert_construction.data(), &values, - insert_construction.size()); + ConstructElements<A>(GetAllocator(), insert_construction.data(), values, + insert_construction.size()); - move_construction_tx.Commit(); + std::move(move_construction_tx).Commit(); AddSize(insert_count); - return iterator(storage_view.data + insert_index); + return Iterator<A>(storage_view.data + insert_index); } } template <typename T, size_t N, typename A> template <typename... Args> -auto Storage<T, N, A>::EmplaceBack(Args&&... args) -> reference { - StorageView storage_view = MakeStorageView(); - const auto n = storage_view.size; +auto Storage<T, N, A>::EmplaceBack(Args&&... args) -> Reference<A> { + StorageView<A> storage_view = MakeStorageView(); + const SizeType<A> n = storage_view.size; if (ABSL_PREDICT_TRUE(n != storage_view.capacity)) { // Fast path; new element fits. - pointer last_ptr = storage_view.data + n; - AllocatorTraits::construct(*GetAllocPtr(), last_ptr, - std::forward<Args>(args)...); + Pointer<A> last_ptr = storage_view.data + n; + AllocatorTraits<A>::construct(GetAllocator(), last_ptr, + std::forward<Args>(args)...); AddSize(1); return *last_ptr; } @@ -763,130 +747,132 @@ auto Storage<T, N, A>::EmplaceBack(Args&&... args) -> reference { template <typename T, size_t N, typename A> template <typename... Args> -auto Storage<T, N, A>::EmplaceBackSlow(Args&&... args) -> reference { - StorageView storage_view = MakeStorageView(); - AllocationTransaction allocation_tx(GetAllocPtr()); - IteratorValueAdapter<MoveIterator> move_values( - MoveIterator(storage_view.data)); - size_type new_capacity = NextCapacity(storage_view.capacity); - pointer construct_data = allocation_tx.Allocate(new_capacity); - pointer last_ptr = construct_data + storage_view.size; +auto Storage<T, N, A>::EmplaceBackSlow(Args&&... args) -> Reference<A> { + StorageView<A> storage_view = MakeStorageView(); + AllocationTransaction<A> allocation_tx(GetAllocator()); + IteratorValueAdapter<A, MoveIterator<A>> move_values( + MoveIterator<A>(storage_view.data)); + SizeType<A> requested_capacity = NextCapacity(storage_view.capacity); + Pointer<A> construct_data = allocation_tx.Allocate(requested_capacity); + Pointer<A> last_ptr = construct_data + storage_view.size; // Construct new element. - AllocatorTraits::construct(*GetAllocPtr(), last_ptr, - std::forward<Args>(args)...); + AllocatorTraits<A>::construct(GetAllocator(), last_ptr, + std::forward<Args>(args)...); // Move elements from old backing store to new backing store. ABSL_INTERNAL_TRY { - inlined_vector_internal::ConstructElements( - GetAllocPtr(), allocation_tx.GetData(), &move_values, - storage_view.size); + ConstructElements<A>(GetAllocator(), allocation_tx.GetData(), move_values, + storage_view.size); } ABSL_INTERNAL_CATCH_ANY { - AllocatorTraits::destroy(*GetAllocPtr(), last_ptr); + AllocatorTraits<A>::destroy(GetAllocator(), last_ptr); ABSL_INTERNAL_RETHROW; } // Destroy elements in old backing store. - inlined_vector_internal::DestroyElements(GetAllocPtr(), storage_view.data, - storage_view.size); + DestroyAdapter<A>::DestroyElements(GetAllocator(), storage_view.data, + storage_view.size); DeallocateIfAllocated(); - AcquireAllocatedData(&allocation_tx); + SetAllocation(std::move(allocation_tx).Release()); SetIsAllocated(); AddSize(1); return *last_ptr; } template <typename T, size_t N, typename A> -auto Storage<T, N, A>::Erase(const_iterator from, const_iterator to) - -> iterator { - StorageView storage_view = MakeStorageView(); +auto Storage<T, N, A>::Erase(ConstIterator<A> from, ConstIterator<A> to) + -> Iterator<A> { + StorageView<A> storage_view = MakeStorageView(); - size_type erase_size = std::distance(from, to); - size_type erase_index = - std::distance(const_iterator(storage_view.data), from); - size_type erase_end_index = erase_index + erase_size; + SizeType<A> erase_size = std::distance(from, to); + SizeType<A> erase_index = + std::distance(ConstIterator<A>(storage_view.data), from); + SizeType<A> erase_end_index = erase_index + erase_size; - IteratorValueAdapter<MoveIterator> move_values( - MoveIterator(storage_view.data + erase_end_index)); + IteratorValueAdapter<A, MoveIterator<A>> move_values( + MoveIterator<A>(storage_view.data + erase_end_index)); - inlined_vector_internal::AssignElements(storage_view.data + erase_index, - &move_values, - storage_view.size - erase_end_index); + AssignElements<A>(storage_view.data + erase_index, move_values, + storage_view.size - erase_end_index); - inlined_vector_internal::DestroyElements( - GetAllocPtr(), storage_view.data + (storage_view.size - erase_size), + DestroyAdapter<A>::DestroyElements( + GetAllocator(), storage_view.data + (storage_view.size - erase_size), erase_size); SubtractSize(erase_size); - return iterator(storage_view.data + erase_index); + return Iterator<A>(storage_view.data + erase_index); } template <typename T, size_t N, typename A> -auto Storage<T, N, A>::Reserve(size_type requested_capacity) -> void { - StorageView storage_view = MakeStorageView(); +auto Storage<T, N, A>::Reserve(SizeType<A> requested_capacity) -> void { + StorageView<A> storage_view = MakeStorageView(); if (ABSL_PREDICT_FALSE(requested_capacity <= storage_view.capacity)) return; - AllocationTransaction allocation_tx(GetAllocPtr()); + AllocationTransaction<A> allocation_tx(GetAllocator()); - IteratorValueAdapter<MoveIterator> move_values( - MoveIterator(storage_view.data)); + IteratorValueAdapter<A, MoveIterator<A>> move_values( + MoveIterator<A>(storage_view.data)); - size_type new_capacity = + SizeType<A> new_requested_capacity = ComputeCapacity(storage_view.capacity, requested_capacity); - pointer new_data = allocation_tx.Allocate(new_capacity); + Pointer<A> new_data = allocation_tx.Allocate(new_requested_capacity); - inlined_vector_internal::ConstructElements(GetAllocPtr(), new_data, - &move_values, storage_view.size); + ConstructElements<A>(GetAllocator(), new_data, move_values, + storage_view.size); - inlined_vector_internal::DestroyElements(GetAllocPtr(), storage_view.data, - storage_view.size); + DestroyAdapter<A>::DestroyElements(GetAllocator(), storage_view.data, + storage_view.size); DeallocateIfAllocated(); - AcquireAllocatedData(&allocation_tx); + SetAllocation(std::move(allocation_tx).Release()); SetIsAllocated(); } template <typename T, size_t N, typename A> auto Storage<T, N, A>::ShrinkToFit() -> void { // May only be called on allocated instances! - assert(GetIsAllocated()); + ABSL_HARDENING_ASSERT(GetIsAllocated()); - StorageView storage_view{GetAllocatedData(), GetSize(), - GetAllocatedCapacity()}; + StorageView<A> storage_view{GetAllocatedData(), GetSize(), + GetAllocatedCapacity()}; if (ABSL_PREDICT_FALSE(storage_view.size == storage_view.capacity)) return; - AllocationTransaction allocation_tx(GetAllocPtr()); + AllocationTransaction<A> allocation_tx(GetAllocator()); - IteratorValueAdapter<MoveIterator> move_values( - MoveIterator(storage_view.data)); + IteratorValueAdapter<A, MoveIterator<A>> move_values( + MoveIterator<A>(storage_view.data)); - pointer construct_data; + Pointer<A> construct_data; if (storage_view.size > GetInlinedCapacity()) { - size_type new_capacity = storage_view.size; - construct_data = allocation_tx.Allocate(new_capacity); + SizeType<A> requested_capacity = storage_view.size; + construct_data = allocation_tx.Allocate(requested_capacity); + if (allocation_tx.GetCapacity() >= storage_view.capacity) { + // Already using the smallest available heap allocation. + return; + } } else { construct_data = GetInlinedData(); } ABSL_INTERNAL_TRY { - inlined_vector_internal::ConstructElements(GetAllocPtr(), construct_data, - &move_values, storage_view.size); + ConstructElements<A>(GetAllocator(), construct_data, move_values, + storage_view.size); } ABSL_INTERNAL_CATCH_ANY { - SetAllocatedData(storage_view.data, storage_view.capacity); + SetAllocation({storage_view.data, storage_view.capacity}); ABSL_INTERNAL_RETHROW; } - inlined_vector_internal::DestroyElements(GetAllocPtr(), storage_view.data, - storage_view.size); + DestroyAdapter<A>::DestroyElements(GetAllocator(), storage_view.data, + storage_view.size); - AllocatorTraits::deallocate(*GetAllocPtr(), storage_view.data, - storage_view.capacity); + MallocAdapter<A>::Deallocate(GetAllocator(), storage_view.data, + storage_view.capacity); if (allocation_tx.DidAllocate()) { - AcquireAllocatedData(&allocation_tx); + SetAllocation(std::move(allocation_tx).Release()); } else { UnsetIsAllocated(); } @@ -895,7 +881,7 @@ auto Storage<T, N, A>::ShrinkToFit() -> void { template <typename T, size_t N, typename A> auto Storage<T, N, A>::Swap(Storage* other_storage_ptr) -> void { using std::swap; - assert(this != other_storage_ptr); + ABSL_HARDENING_ASSERT(this != other_storage_ptr); if (GetIsAllocated() && other_storage_ptr->GetIsAllocated()) { swap(data_.allocated, other_storage_ptr->data_.allocated); @@ -904,20 +890,20 @@ auto Storage<T, N, A>::Swap(Storage* other_storage_ptr) -> void { Storage* large_ptr = other_storage_ptr; if (small_ptr->GetSize() > large_ptr->GetSize()) swap(small_ptr, large_ptr); - for (size_type i = 0; i < small_ptr->GetSize(); ++i) { + for (SizeType<A> i = 0; i < small_ptr->GetSize(); ++i) { swap(small_ptr->GetInlinedData()[i], large_ptr->GetInlinedData()[i]); } - IteratorValueAdapter<MoveIterator> move_values( - MoveIterator(large_ptr->GetInlinedData() + small_ptr->GetSize())); + IteratorValueAdapter<A, MoveIterator<A>> move_values( + MoveIterator<A>(large_ptr->GetInlinedData() + small_ptr->GetSize())); - inlined_vector_internal::ConstructElements( - large_ptr->GetAllocPtr(), - small_ptr->GetInlinedData() + small_ptr->GetSize(), &move_values, - large_ptr->GetSize() - small_ptr->GetSize()); + ConstructElements<A>(large_ptr->GetAllocator(), + small_ptr->GetInlinedData() + small_ptr->GetSize(), + move_values, + large_ptr->GetSize() - small_ptr->GetSize()); - inlined_vector_internal::DestroyElements( - large_ptr->GetAllocPtr(), + DestroyAdapter<A>::DestroyElements( + large_ptr->GetAllocator(), large_ptr->GetInlinedData() + small_ptr->GetSize(), large_ptr->GetSize() - small_ptr->GetSize()); } else { @@ -925,37 +911,37 @@ auto Storage<T, N, A>::Swap(Storage* other_storage_ptr) -> void { Storage* inlined_ptr = other_storage_ptr; if (!allocated_ptr->GetIsAllocated()) swap(allocated_ptr, inlined_ptr); - StorageView allocated_storage_view{allocated_ptr->GetAllocatedData(), - allocated_ptr->GetSize(), - allocated_ptr->GetAllocatedCapacity()}; + StorageView<A> allocated_storage_view{ + allocated_ptr->GetAllocatedData(), allocated_ptr->GetSize(), + allocated_ptr->GetAllocatedCapacity()}; - IteratorValueAdapter<MoveIterator> move_values( - MoveIterator(inlined_ptr->GetInlinedData())); + IteratorValueAdapter<A, MoveIterator<A>> move_values( + MoveIterator<A>(inlined_ptr->GetInlinedData())); ABSL_INTERNAL_TRY { - inlined_vector_internal::ConstructElements( - inlined_ptr->GetAllocPtr(), allocated_ptr->GetInlinedData(), - &move_values, inlined_ptr->GetSize()); + ConstructElements<A>(inlined_ptr->GetAllocator(), + allocated_ptr->GetInlinedData(), move_values, + inlined_ptr->GetSize()); } ABSL_INTERNAL_CATCH_ANY { - allocated_ptr->SetAllocatedData(allocated_storage_view.data, - allocated_storage_view.capacity); + allocated_ptr->SetAllocation(Allocation<A>{ + allocated_storage_view.data, allocated_storage_view.capacity}); ABSL_INTERNAL_RETHROW; } - inlined_vector_internal::DestroyElements(inlined_ptr->GetAllocPtr(), - inlined_ptr->GetInlinedData(), - inlined_ptr->GetSize()); + DestroyAdapter<A>::DestroyElements(inlined_ptr->GetAllocator(), + inlined_ptr->GetInlinedData(), + inlined_ptr->GetSize()); - inlined_ptr->SetAllocatedData(allocated_storage_view.data, - allocated_storage_view.capacity); + inlined_ptr->SetAllocation(Allocation<A>{allocated_storage_view.data, + allocated_storage_view.capacity}); } swap(GetSizeAndIsAllocated(), other_storage_ptr->GetSizeAndIsAllocated()); - swap(*GetAllocPtr(), *other_storage_ptr->GetAllocPtr()); + swap(GetAllocator(), other_storage_ptr->GetAllocator()); } -// End ignore "maybe-uninitialized" +// End ignore "array-bounds" #if !defined(__clang__) && defined(__GNUC__) #pragma GCC diagnostic pop #endif diff --git a/absl/container/internal/layout_test.cc b/absl/container/internal/layout_test.cc index 1d7158ff..54e5d5bb 100644 --- a/absl/container/internal/layout_test.cc +++ b/absl/container/internal/layout_test.cc @@ -1350,7 +1350,13 @@ TEST(Layout, CustomAlignment) { TEST(Layout, OverAligned) { constexpr size_t M = alignof(max_align_t); constexpr Layout<unsigned char, Aligned<unsigned char, 2 * M>> x(1, 3); +#ifdef __GNUC__ + // Using __attribute__ ((aligned ())) instead of alignas to bypass a gcc bug: + // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=89357 + __attribute__((aligned(2 * M))) unsigned char p[x.AllocSize()]; +#else alignas(2 * M) unsigned char p[x.AllocSize()]; +#endif EXPECT_EQ(2 * M + 3, x.AllocSize()); EXPECT_THAT(x.Pointers(p), Tuple(p + 0, p + 2 * M)); } diff --git a/absl/container/internal/node_hash_policy.h b/absl/container/internal/node_slot_policy.h index 4617162f..baba5743 100644 --- a/absl/container/internal/node_hash_policy.h +++ b/absl/container/internal/node_slot_policy.h @@ -30,8 +30,8 @@ // It may also optionally define `value()` and `apply()`. For documentation on // these, see hash_policy_traits.h. -#ifndef ABSL_CONTAINER_INTERNAL_NODE_HASH_POLICY_H_ -#define ABSL_CONTAINER_INTERNAL_NODE_HASH_POLICY_H_ +#ifndef ABSL_CONTAINER_INTERNAL_NODE_SLOT_POLICY_H_ +#define ABSL_CONTAINER_INTERNAL_NODE_SLOT_POLICY_H_ #include <cassert> #include <cstddef> @@ -46,7 +46,7 @@ ABSL_NAMESPACE_BEGIN namespace container_internal { template <class Reference, class Policy> -struct node_hash_policy { +struct node_slot_policy { static_assert(std::is_lvalue_reference<Reference>::value, ""); using slot_type = typename std::remove_cv< @@ -89,4 +89,4 @@ struct node_hash_policy { ABSL_NAMESPACE_END } // namespace absl -#endif // ABSL_CONTAINER_INTERNAL_NODE_HASH_POLICY_H_ +#endif // ABSL_CONTAINER_INTERNAL_NODE_SLOT_POLICY_H_ diff --git a/absl/container/internal/node_hash_policy_test.cc b/absl/container/internal/node_slot_policy_test.cc index 84aabba9..51b7467b 100644 --- a/absl/container/internal/node_hash_policy_test.cc +++ b/absl/container/internal/node_slot_policy_test.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "absl/container/internal/node_hash_policy.h" +#include "absl/container/internal/node_slot_policy.h" #include <memory> @@ -27,7 +27,7 @@ namespace { using ::testing::Pointee; -struct Policy : node_hash_policy<int&, Policy> { +struct Policy : node_slot_policy<int&, Policy> { using key_type = int; using init_type = int; diff --git a/absl/container/internal/raw_hash_map.h b/absl/container/internal/raw_hash_map.h index 0a02757d..c7df2efc 100644 --- a/absl/container/internal/raw_hash_map.h +++ b/absl/container/internal/raw_hash_map.h @@ -51,8 +51,9 @@ class raw_hash_map : public raw_hash_set<Policy, Hash, Eq, Alloc> { using key_arg = typename KeyArgImpl::template type<K, key_type>; static_assert(!std::is_reference<key_type>::value, ""); - // TODO(alkis): remove this assertion and verify that reference mapped_type is - // supported. + + // TODO(b/187807849): Evaluate whether to support reference mapped_type and + // remove this assertion if/when it is supported. static_assert(!std::is_reference<mapped_type>::value, ""); using iterator = typename raw_hash_map::raw_hash_set::iterator; diff --git a/absl/container/internal/raw_hash_set.cc b/absl/container/internal/raw_hash_set.cc index bfef071f..c63a2e02 100644 --- a/absl/container/internal/raw_hash_set.cc +++ b/absl/container/internal/raw_hash_set.cc @@ -23,7 +23,17 @@ namespace absl { ABSL_NAMESPACE_BEGIN namespace container_internal { +// A single block of empty control bytes for tables without any slots allocated. +// This enables removing a branch in the hot path of find(). +alignas(16) ABSL_CONST_INIT ABSL_DLL const ctrl_t kEmptyGroup[16] = { + ctrl_t::kSentinel, ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, + ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, + ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, + ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty, ctrl_t::kEmpty}; + +#ifdef ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL constexpr size_t Group::kWidth; +#endif // Returns "random" seed. inline size_t RandomSeed() { @@ -37,24 +47,24 @@ inline size_t RandomSeed() { return value ^ static_cast<size_t>(reinterpret_cast<uintptr_t>(&counter)); } -bool ShouldInsertBackwards(size_t hash, ctrl_t* ctrl) { +bool ShouldInsertBackwards(size_t hash, const ctrl_t* ctrl) { // To avoid problems with weak hashes and single bit tests, we use % 13. // TODO(kfm,sbenza): revisit after we do unconditional mixing return (H1(hash, ctrl) ^ RandomSeed()) % 13 > 6; } -void ConvertDeletedToEmptyAndFullToDeleted( - ctrl_t* ctrl, size_t capacity) { - assert(ctrl[capacity] == kSentinel); +void ConvertDeletedToEmptyAndFullToDeleted(ctrl_t* ctrl, size_t capacity) { + assert(ctrl[capacity] == ctrl_t::kSentinel); assert(IsValidCapacity(capacity)); - for (ctrl_t* pos = ctrl; pos != ctrl + capacity + 1; pos += Group::kWidth) { + for (ctrl_t* pos = ctrl; pos < ctrl + capacity; pos += Group::kWidth) { Group{pos}.ConvertSpecialToEmptyAndFullToDeleted(pos); } // Copy the cloned ctrl bytes. - std::memcpy(ctrl + capacity + 1, ctrl, Group::kWidth); - ctrl[capacity] = kSentinel; + std::memcpy(ctrl + capacity + 1, ctrl, NumClonedBytes()); + ctrl[capacity] = ctrl_t::kSentinel; } - +// Extern template instantiotion for inline function. +template FindInfo find_first_non_full(const ctrl_t*, size_t, size_t); } // namespace container_internal ABSL_NAMESPACE_END diff --git a/absl/container/internal/raw_hash_set.h b/absl/container/internal/raw_hash_set.h index 8615de8b..ea912f83 100644 --- a/absl/container/internal/raw_hash_set.h +++ b/absl/container/internal/raw_hash_set.h @@ -53,40 +53,121 @@ // // IMPLEMENTATION DETAILS // -// The table stores elements inline in a slot array. In addition to the slot -// array the table maintains some control state per slot. The extra state is one -// byte per slot and stores empty or deleted marks, or alternatively 7 bits from -// the hash of an occupied slot. The table is split into logical groups of -// slots, like so: +// # Table Layout +// +// A raw_hash_set's backing array consists of control bytes followed by slots +// that may or may not contain objects. +// +// The layout of the backing array, for `capacity` slots, is thus, as a +// pseudo-struct: +// +// struct BackingArray { +// // Control bytes for the "real" slots. +// ctrl_t ctrl[capacity]; +// // Always `ctrl_t::kSentinel`. This is used by iterators to find when to +// // stop and serves no other purpose. +// ctrl_t sentinel; +// // A copy of the first `kWidth - 1` elements of `ctrl`. This is used so +// // that if a probe sequence picks a value near the end of `ctrl`, +// // `Group` will have valid control bytes to look at. +// ctrl_t clones[kWidth - 1]; +// // The actual slot data. +// slot_type slots[capacity]; +// }; +// +// The length of this array is computed by `AllocSize()` below. +// +// Control bytes (`ctrl_t`) are bytes (collected into groups of a +// platform-specific size) that define the state of the corresponding slot in +// the slot array. Group manipulation is tightly optimized to be as efficient +// as possible: SSE and friends on x86, clever bit operations on other arches. // // Group 1 Group 2 Group 3 // +---------------+---------------+---------------+ // | | | | | | | | | | | | | | | | | | | | | | | | | // +---------------+---------------+---------------+ // -// On lookup the hash is split into two parts: -// - H2: 7 bits (those stored in the control bytes) -// - H1: the rest of the bits -// The groups are probed using H1. For each group the slots are matched to H2 in -// parallel. Because H2 is 7 bits (128 states) and the number of slots per group -// is low (8 or 16) in almost all cases a match in H2 is also a lookup hit. +// Each control byte is either a special value for empty slots, deleted slots +// (sometimes called *tombstones*), and a special end-of-table marker used by +// iterators, or, if occupied, seven bits (H2) from the hash of the value in the +// corresponding slot. +// +// Storing control bytes in a separate array also has beneficial cache effects, +// since more logical slots will fit into a cache line. +// +// # Hashing +// +// We compute two separate hashes, `H1` and `H2`, from the hash of an object. +// `H1(hash(x))` is an index into `slots`, and essentially the starting point +// for the probe sequence. `H2(hash(x))` is a 7-bit value used to filter out +// objects that cannot possibly be the one we are looking for. +// +// # Table operations. +// +// The key operations are `insert`, `find`, and `erase`. +// +// Since `insert` and `erase` are implemented in terms of `find`, we describe +// `find` first. To `find` a value `x`, we compute `hash(x)`. From +// `H1(hash(x))` and the capacity, we construct a `probe_seq` that visits every +// group of slots in some interesting order. // -// On insert, once the right group is found (as in lookup), its slots are -// filled in order. +// We now walk through these indices. At each index, we select the entire group +// starting with that index and extract potential candidates: occupied slots +// with a control byte equal to `H2(hash(x))`. If we find an empty slot in the +// group, we stop and return an error. Each candidate slot `y` is compared with +// `x`; if `x == y`, we are done and return `&y`; otherwise we contine to the +// next probe index. Tombstones effectively behave like full slots that never +// match the value we're looking for. // -// On erase a slot is cleared. In case the group did not have any empty slots -// before the erase, the erased slot is marked as deleted. +// The `H2` bits ensure when we compare a slot to an object with `==`, we are +// likely to have actually found the object. That is, the chance is low that +// `==` is called and returns `false`. Thus, when we search for an object, we +// are unlikely to call `==` many times. This likelyhood can be analyzed as +// follows (assuming that H2 is a random enough hash function). // -// Groups without empty slots (but maybe with deleted slots) extend the probe -// sequence. The probing algorithm is quadratic. Given N the number of groups, -// the probing function for the i'th probe is: +// Let's assume that there are `k` "wrong" objects that must be examined in a +// probe sequence. For example, when doing a `find` on an object that is in the +// table, `k` is the number of objects between the start of the probe sequence +// and the final found object (not including the final found object). The +// expected number of objects with an H2 match is then `k/128`. Measurements +// and analysis indicate that even at high load factors, `k` is less than 32, +// meaning that the number of "false positive" comparisons we must perform is +// less than 1/8 per `find`. + +// `insert` is implemented in terms of `unchecked_insert`, which inserts a +// value presumed to not be in the table (violating this requirement will cause +// the table to behave erratically). Given `x` and its hash `hash(x)`, to insert +// it, we construct a `probe_seq` once again, and use it to find the first +// group with an unoccupied (empty *or* deleted) slot. We place `x` into the +// first such slot in the group and mark it as full with `x`'s H2. // -// P(0) = H1 % N +// To `insert`, we compose `unchecked_insert` with `find`. We compute `h(x)` and +// perform a `find` to see if it's already present; if it is, we're done. If +// it's not, we may decide the table is getting overcrowded (i.e. the load +// factor is greater than 7/8 for big tables; `is_small()` tables use a max load +// factor of 1); in this case, we allocate a bigger array, `unchecked_insert` +// each element of the table into the new array (we know that no insertion here +// will insert an already-present value), and discard the old backing array. At +// this point, we may `unchecked_insert` the value `x`. // -// P(i) = (P(i - 1) + i) % N +// Below, `unchecked_insert` is partly implemented by `prepare_insert`, which +// presents a viable, initialized slot pointee to the caller. // -// This probing function guarantees that after N probes, all the groups of the -// table will be probed exactly once. +// `erase` is implemented in terms of `erase_at`, which takes an index to a +// slot. Given an offset, we simply create a tombstone and destroy its contents. +// If we can prove that the slot would not appear in a probe sequence, we can +// make the slot as empty, instead. We can prove this by observing that if a +// group has any empty slots, it has never been full (assuming we never create +// an empty slot in a group with no empties, which this heuristic guarantees we +// never do) and find would stop at this group anyways (since it does not probe +// beyond groups with empties). +// +// `erase` is `erase_at` composed with `find`: if we +// have a value `x`, we can perform a `find`, and then `erase_at` the resulting +// slot. +// +// To iterate, we simply traverse the array, skipping empty and deleted slots +// and stopping when we hit a `kSentinel`. #ifndef ABSL_CONTAINER_INTERNAL_RAW_HASH_SET_H_ #define ABSL_CONTAINER_INTERNAL_RAW_HASH_SET_H_ @@ -102,7 +183,9 @@ #include <type_traits> #include <utility> +#include "absl/base/config.h" #include "absl/base/internal/endian.h" +#include "absl/base/internal/prefetch.h" #include "absl/base/optimization.h" #include "absl/base/port.h" #include "absl/container/internal/common.h" @@ -111,13 +194,27 @@ #include "absl/container/internal/hash_policy_traits.h" #include "absl/container/internal/hashtable_debug_hooks.h" #include "absl/container/internal/hashtablez_sampler.h" -#include "absl/container/internal/have_sse.h" -#include "absl/container/internal/layout.h" #include "absl/memory/memory.h" #include "absl/meta/type_traits.h" #include "absl/numeric/bits.h" #include "absl/utility/utility.h" +#ifdef ABSL_INTERNAL_HAVE_SSE2 +#include <emmintrin.h> +#endif + +#ifdef ABSL_INTERNAL_HAVE_SSSE3 +#include <tmmintrin.h> +#endif + +#ifdef _MSC_VER +#include <intrin.h> +#endif + +#ifdef ABSL_INTERNAL_HAVE_ARM_NEON +#include <arm_neon.h> +#endif + namespace absl { ABSL_NAMESPACE_BEGIN namespace container_internal { @@ -132,14 +229,40 @@ template <typename AllocType> void SwapAlloc(AllocType& /*lhs*/, AllocType& /*rhs*/, std::false_type /* propagate_on_container_swap */) {} +// The state for a probe sequence. +// +// Currently, the sequence is a triangular progression of the form +// +// p(i) := Width * (i^2 + i)/2 + hash (mod mask + 1) +// +// The use of `Width` ensures that each probe step does not overlap groups; +// the sequence effectively outputs the addresses of *groups* (although not +// necessarily aligned to any boundary). The `Group` machinery allows us +// to check an entire group with minimal branching. +// +// Wrapping around at `mask + 1` is important, but not for the obvious reason. +// As described above, the first few entries of the control byte array +// are mirrored at the end of the array, which `Group` will find and use +// for selecting candidates. However, when those candidates' slots are +// actually inspected, there are no corresponding slots for the cloned bytes, +// so we need to make sure we've treated those offsets as "wrapping around". +// +// It turns out that this probe sequence visits every group exactly once if the +// number of groups is a power of two, since (i^2+i)/2 is a bijection in +// Z/(2^m). See https://en.wikipedia.org/wiki/Quadratic_probing template <size_t Width> class probe_seq { public: + // Creates a new probe sequence using `hash` as the initial value of the + // sequence and `mask` (usually the capacity of the table) as the mask to + // apply to each value in the progression. probe_seq(size_t hash, size_t mask) { assert(((mask + 1) & mask) == 0 && "not a mask"); mask_ = mask; offset_ = hash & mask_; } + + // The offset within the table, i.e., the value `p(i)` above. size_t offset() const { return offset_; } size_t offset(size_t i) const { return (offset_ + i) & mask_; } @@ -148,7 +271,7 @@ class probe_seq { offset_ += index_; offset_ &= mask_; } - // 0-based probe index. The i-th probe in the probe sequence. + // 0-based probe index, a multiple of `Width`. size_t index() const { return index_; } private: @@ -172,9 +295,9 @@ struct IsDecomposable : std::false_type {}; template <class Policy, class Hash, class Eq, class... Ts> struct IsDecomposable< - absl::void_t<decltype( - Policy::apply(RequireUsableKey<typename Policy::key_type, Hash, Eq>(), - std::declval<Ts>()...))>, + absl::void_t<decltype(Policy::apply( + RequireUsableKey<typename Policy::key_type, Hash, Eq>(), + std::declval<Ts>()...))>, Policy, Hash, Eq, Ts...> : std::true_type {}; // TODO(alkis): Switch to std::is_nothrow_swappable when gcc/clang supports it. @@ -190,57 +313,84 @@ constexpr bool IsNoThrowSwappable(std::false_type /* is_swappable */) { template <typename T> uint32_t TrailingZeros(T x) { - ABSL_INTERNAL_ASSUME(x != 0); - return countr_zero(x); + ABSL_ASSUME(x != 0); + return static_cast<uint32_t>(countr_zero(x)); } -// An abstraction over a bitmask. It provides an easy way to iterate through the -// indexes of the set bits of a bitmask. When Shift=0 (platforms with SSE), -// this is a true bitmask. On non-SSE, platforms the arithematic used to -// emulate the SSE behavior works in bytes (Shift=3) and leaves each bytes as -// either 0x00 or 0x80. +// An abstract bitmask, such as that emitted by a SIMD instruction. // -// For example: -// for (int i : BitMask<uint32_t, 16>(0x5)) -> yields 0, 2 -// for (int i : BitMask<uint64_t, 8, 3>(0x0000000080800000)) -> yields 2, 3 +// Specifically, this type implements a simple bitset whose representation is +// controlled by `SignificantBits` and `Shift`. `SignificantBits` is the number +// of abstract bits in the bitset, while `Shift` is the log-base-two of the +// width of an abstract bit in the representation. +// This mask provides operations for any number of real bits set in an abstract +// bit. To add iteration on top of that, implementation must guarantee no more +// than one real bit is set in an abstract bit. template <class T, int SignificantBits, int Shift = 0> -class BitMask { - static_assert(std::is_unsigned<T>::value, ""); - static_assert(Shift == 0 || Shift == 3, ""); - +class NonIterableBitMask { public: - // These are useful for unit tests (gunit). - using value_type = int; - using iterator = BitMask; - using const_iterator = BitMask; + explicit NonIterableBitMask(T mask) : mask_(mask) {} - explicit BitMask(T mask) : mask_(mask) {} - BitMask& operator++() { - mask_ &= (mask_ - 1); - return *this; - } - explicit operator bool() const { return mask_ != 0; } - int operator*() const { return LowestBitSet(); } + explicit operator bool() const { return this->mask_ != 0; } + + // Returns the index of the lowest *abstract* bit set in `self`. uint32_t LowestBitSet() const { return container_internal::TrailingZeros(mask_) >> Shift; } + + // Returns the index of the highest *abstract* bit set in `self`. uint32_t HighestBitSet() const { return static_cast<uint32_t>((bit_width(mask_) - 1) >> Shift); } - BitMask begin() const { return *this; } - BitMask end() const { return BitMask(0); } - + // Return the number of trailing zero *abstract* bits. uint32_t TrailingZeros() const { return container_internal::TrailingZeros(mask_) >> Shift; } + // Return the number of leading zero *abstract* bits. uint32_t LeadingZeros() const { constexpr int total_significant_bits = SignificantBits << Shift; constexpr int extra_bits = sizeof(T) * 8 - total_significant_bits; - return countl_zero(mask_ << extra_bits) >> Shift; + return static_cast<uint32_t>(countl_zero(mask_ << extra_bits)) >> Shift; } + T mask_; +}; + +// Mask that can be iterable +// +// For example, when `SignificantBits` is 16 and `Shift` is zero, this is just +// an ordinary 16-bit bitset occupying the low 16 bits of `mask`. When +// `SignificantBits` is 8 and `Shift` is 3, abstract bits are represented as +// the bytes `0x00` and `0x80`, and it occupies all 64 bits of the bitmask. +// +// For example: +// for (int i : BitMask<uint32_t, 16>(0b101)) -> yields 0, 2 +// for (int i : BitMask<uint64_t, 8, 3>(0x0000000080800000)) -> yields 2, 3 +template <class T, int SignificantBits, int Shift = 0> +class BitMask : public NonIterableBitMask<T, SignificantBits, Shift> { + using Base = NonIterableBitMask<T, SignificantBits, Shift>; + static_assert(std::is_unsigned<T>::value, ""); + static_assert(Shift == 0 || Shift == 3, ""); + + public: + explicit BitMask(T mask) : Base(mask) {} + // BitMask is an iterator over the indices of its abstract bits. + using value_type = int; + using iterator = BitMask; + using const_iterator = BitMask; + + BitMask& operator++() { + this->mask_ &= (this->mask_ - 1); + return *this; + } + + uint32_t operator*() const { return Base::LowestBitSet(); } + + BitMask begin() const { return *this; } + BitMask end() const { return BitMask(0); } + private: friend bool operator==(const BitMask& a, const BitMask& b) { return a.mask_ == b.mask_; @@ -248,75 +398,127 @@ class BitMask { friend bool operator!=(const BitMask& a, const BitMask& b) { return a.mask_ != b.mask_; } - - T mask_; }; -using ctrl_t = signed char; using h2_t = uint8_t; // The values here are selected for maximum performance. See the static asserts // below for details. -enum Ctrl : ctrl_t { + +// A `ctrl_t` is a single control byte, which can have one of four +// states: empty, deleted, full (which has an associated seven-bit h2_t value) +// and the sentinel. They have the following bit patterns: +// +// empty: 1 0 0 0 0 0 0 0 +// deleted: 1 1 1 1 1 1 1 0 +// full: 0 h h h h h h h // h represents the hash bits. +// sentinel: 1 1 1 1 1 1 1 1 +// +// These values are specifically tuned for SSE-flavored SIMD. +// The static_asserts below detail the source of these choices. +// +// We use an enum class so that when strict aliasing is enabled, the compiler +// knows ctrl_t doesn't alias other types. +enum class ctrl_t : int8_t { kEmpty = -128, // 0b10000000 kDeleted = -2, // 0b11111110 kSentinel = -1, // 0b11111111 }; static_assert( - kEmpty & kDeleted & kSentinel & 0x80, + (static_cast<int8_t>(ctrl_t::kEmpty) & + static_cast<int8_t>(ctrl_t::kDeleted) & + static_cast<int8_t>(ctrl_t::kSentinel) & 0x80) != 0, "Special markers need to have the MSB to make checking for them efficient"); -static_assert(kEmpty < kSentinel && kDeleted < kSentinel, - "kEmpty and kDeleted must be smaller than kSentinel to make the " - "SIMD test of IsEmptyOrDeleted() efficient"); -static_assert(kSentinel == -1, - "kSentinel must be -1 to elide loading it from memory into SIMD " - "registers (pcmpeqd xmm, xmm)"); -static_assert(kEmpty == -128, - "kEmpty must be -128 to make the SIMD check for its " +static_assert( + ctrl_t::kEmpty < ctrl_t::kSentinel && ctrl_t::kDeleted < ctrl_t::kSentinel, + "ctrl_t::kEmpty and ctrl_t::kDeleted must be smaller than " + "ctrl_t::kSentinel to make the SIMD test of IsEmptyOrDeleted() efficient"); +static_assert( + ctrl_t::kSentinel == static_cast<ctrl_t>(-1), + "ctrl_t::kSentinel must be -1 to elide loading it from memory into SIMD " + "registers (pcmpeqd xmm, xmm)"); +static_assert(ctrl_t::kEmpty == static_cast<ctrl_t>(-128), + "ctrl_t::kEmpty must be -128 to make the SIMD check for its " "existence efficient (psignb xmm, xmm)"); -static_assert(~kEmpty & ~kDeleted & kSentinel & 0x7F, - "kEmpty and kDeleted must share an unset bit that is not shared " - "by kSentinel to make the scalar test for MatchEmptyOrDeleted() " - "efficient"); -static_assert(kDeleted == -2, - "kDeleted must be -2 to make the implementation of " +static_assert( + (~static_cast<int8_t>(ctrl_t::kEmpty) & + ~static_cast<int8_t>(ctrl_t::kDeleted) & + static_cast<int8_t>(ctrl_t::kSentinel) & 0x7F) != 0, + "ctrl_t::kEmpty and ctrl_t::kDeleted must share an unset bit that is not " + "shared by ctrl_t::kSentinel to make the scalar test for " + "MaskEmptyOrDeleted() efficient"); +static_assert(ctrl_t::kDeleted == static_cast<ctrl_t>(-2), + "ctrl_t::kDeleted must be -2 to make the implementation of " "ConvertSpecialToEmptyAndFullToDeleted efficient"); -// A single block of empty control bytes for tables without any slots allocated. -// This enables removing a branch in the hot path of find(). +ABSL_DLL extern const ctrl_t kEmptyGroup[16]; + +// Returns a pointer to a control byte group that can be used by empty tables. inline ctrl_t* EmptyGroup() { - alignas(16) static constexpr ctrl_t empty_group[] = { - kSentinel, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, - kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty, kEmpty}; - return const_cast<ctrl_t*>(empty_group); + // Const must be cast away here; no uses of this function will actually write + // to it, because it is only used for empty tables. + return const_cast<ctrl_t*>(kEmptyGroup); } // Mixes a randomly generated per-process seed with `hash` and `ctrl` to // randomize insertion order within groups. -bool ShouldInsertBackwards(size_t hash, ctrl_t* ctrl); +bool ShouldInsertBackwards(size_t hash, const ctrl_t* ctrl); -// Returns a hash seed. +// Returns a per-table, hash salt, which changes on resize. This gets mixed into +// H1 to randomize iteration order per-table. // // The seed consists of the ctrl_ pointer, which adds enough entropy to ensure // non-determinism of iteration order in most cases. -inline size_t HashSeed(const ctrl_t* ctrl) { +inline size_t PerTableSalt(const ctrl_t* ctrl) { // The low bits of the pointer have little or no entropy because of // alignment. We shift the pointer to try to use higher entropy bits. A // good number seems to be 12 bits, because that aligns with page size. return reinterpret_cast<uintptr_t>(ctrl) >> 12; } - +// Extracts the H1 portion of a hash: 57 bits mixed with a per-table salt. inline size_t H1(size_t hash, const ctrl_t* ctrl) { - return (hash >> 7) ^ HashSeed(ctrl); + return (hash >> 7) ^ PerTableSalt(ctrl); } -inline ctrl_t H2(size_t hash) { return hash & 0x7F; } -inline bool IsEmpty(ctrl_t c) { return c == kEmpty; } -inline bool IsFull(ctrl_t c) { return c >= 0; } -inline bool IsDeleted(ctrl_t c) { return c == kDeleted; } -inline bool IsEmptyOrDeleted(ctrl_t c) { return c < kSentinel; } +// Extracts the H2 portion of a hash: the 7 bits not used for H1. +// +// These are used as an occupied control byte. +inline h2_t H2(size_t hash) { return hash & 0x7F; } -#if ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSE2 +// Helpers for checking the state of a control byte. +inline bool IsEmpty(ctrl_t c) { return c == ctrl_t::kEmpty; } +inline bool IsFull(ctrl_t c) { return c >= static_cast<ctrl_t>(0); } +inline bool IsDeleted(ctrl_t c) { return c == ctrl_t::kDeleted; } +inline bool IsEmptyOrDeleted(ctrl_t c) { return c < ctrl_t::kSentinel; } + +#ifdef ABSL_INTERNAL_HAVE_SSE2 +// Quick reference guide for intrinsics used below: +// +// * __m128i: An XMM (128-bit) word. +// +// * _mm_setzero_si128: Returns a zero vector. +// * _mm_set1_epi8: Returns a vector with the same i8 in each lane. +// +// * _mm_subs_epi8: Saturating-subtracts two i8 vectors. +// * _mm_and_si128: Ands two i128s together. +// * _mm_or_si128: Ors two i128s together. +// * _mm_andnot_si128: And-nots two i128s together. +// +// * _mm_cmpeq_epi8: Component-wise compares two i8 vectors for equality, +// filling each lane with 0x00 or 0xff. +// * _mm_cmpgt_epi8: Same as above, but using > rather than ==. +// +// * _mm_loadu_si128: Performs an unaligned load of an i128. +// * _mm_storeu_si128: Performs an unaligned store of an i128. +// +// * _mm_sign_epi8: Retains, negates, or zeroes each i8 lane of the first +// argument if the corresponding lane of the second +// argument is positive, negative, or zero, respectively. +// * _mm_movemask_epi8: Selects the sign bit out of each i8 lane and produces a +// bitmask consisting of those bits. +// * _mm_shuffle_epi8: Selects i8s from the first argument, using the low +// four bits of each i8 lane in the second argument as +// indices. // https://github.com/abseil/abseil-cpp/issues/209 // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=87853 @@ -345,30 +547,32 @@ struct GroupSse2Impl { BitMask<uint32_t, kWidth> Match(h2_t hash) const { auto match = _mm_set1_epi8(hash); return BitMask<uint32_t, kWidth>( - _mm_movemask_epi8(_mm_cmpeq_epi8(match, ctrl))); + static_cast<uint32_t>(_mm_movemask_epi8(_mm_cmpeq_epi8(match, ctrl)))); } // Returns a bitmask representing the positions of empty slots. - BitMask<uint32_t, kWidth> MatchEmpty() const { -#if ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSSE3 - // This only works because kEmpty is -128. - return BitMask<uint32_t, kWidth>( - _mm_movemask_epi8(_mm_sign_epi8(ctrl, ctrl))); + NonIterableBitMask<uint32_t, kWidth> MaskEmpty() const { +#ifdef ABSL_INTERNAL_HAVE_SSSE3 + // This only works because ctrl_t::kEmpty is -128. + return NonIterableBitMask<uint32_t, kWidth>( + static_cast<uint32_t>(_mm_movemask_epi8(_mm_sign_epi8(ctrl, ctrl)))); #else - return Match(static_cast<h2_t>(kEmpty)); + auto match = _mm_set1_epi8(static_cast<h2_t>(ctrl_t::kEmpty)); + return NonIterableBitMask<uint32_t, kWidth>( + static_cast<uint32_t>(_mm_movemask_epi8(_mm_cmpeq_epi8(match, ctrl)))); #endif } // Returns a bitmask representing the positions of empty or deleted slots. - BitMask<uint32_t, kWidth> MatchEmptyOrDeleted() const { - auto special = _mm_set1_epi8(kSentinel); - return BitMask<uint32_t, kWidth>( - _mm_movemask_epi8(_mm_cmpgt_epi8_fixed(special, ctrl))); + NonIterableBitMask<uint32_t, kWidth> MaskEmptyOrDeleted() const { + auto special = _mm_set1_epi8(static_cast<uint8_t>(ctrl_t::kSentinel)); + return NonIterableBitMask<uint32_t, kWidth>(static_cast<uint32_t>( + _mm_movemask_epi8(_mm_cmpgt_epi8_fixed(special, ctrl)))); } // Returns the number of trailing empty or deleted elements in the group. uint32_t CountLeadingEmptyOrDeleted() const { - auto special = _mm_set1_epi8(kSentinel); + auto special = _mm_set1_epi8(static_cast<uint8_t>(ctrl_t::kSentinel)); return TrailingZeros(static_cast<uint32_t>( _mm_movemask_epi8(_mm_cmpgt_epi8_fixed(special, ctrl)) + 1)); } @@ -376,7 +580,7 @@ struct GroupSse2Impl { void ConvertSpecialToEmptyAndFullToDeleted(ctrl_t* dst) const { auto msbs = _mm_set1_epi8(static_cast<char>(-128)); auto x126 = _mm_set1_epi8(126); -#if ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSSE3 +#ifdef ABSL_INTERNAL_HAVE_SSSE3 auto res = _mm_or_si128(_mm_shuffle_epi8(x126, ctrl), msbs); #else auto zero = _mm_setzero_si128(); @@ -390,6 +594,63 @@ struct GroupSse2Impl { }; #endif // ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSE2 +#if defined(ABSL_INTERNAL_HAVE_ARM_NEON) && defined(ABSL_IS_LITTLE_ENDIAN) +struct GroupAArch64Impl { + static constexpr size_t kWidth = 8; + + explicit GroupAArch64Impl(const ctrl_t* pos) { + ctrl = vld1_u8(reinterpret_cast<const uint8_t*>(pos)); + } + + BitMask<uint64_t, kWidth, 3> Match(h2_t hash) const { + uint8x8_t dup = vdup_n_u8(hash); + auto mask = vceq_u8(ctrl, dup); + constexpr uint64_t msbs = 0x8080808080808080ULL; + return BitMask<uint64_t, kWidth, 3>( + vget_lane_u64(vreinterpret_u64_u8(mask), 0) & msbs); + } + + NonIterableBitMask<uint64_t, kWidth, 3> MaskEmpty() const { + uint64_t mask = + vget_lane_u64(vreinterpret_u64_u8( + vceq_s8(vdup_n_s8(static_cast<h2_t>(ctrl_t::kEmpty)), + vreinterpret_s8_u8(ctrl))), + 0); + return NonIterableBitMask<uint64_t, kWidth, 3>(mask); + } + + NonIterableBitMask<uint64_t, kWidth, 3> MaskEmptyOrDeleted() const { + uint64_t mask = + vget_lane_u64(vreinterpret_u64_u8(vcgt_s8( + vdup_n_s8(static_cast<int8_t>(ctrl_t::kSentinel)), + vreinterpret_s8_u8(ctrl))), + 0); + return NonIterableBitMask<uint64_t, kWidth, 3>(mask); + } + + uint32_t CountLeadingEmptyOrDeleted() const { + uint64_t mask = vget_lane_u64(vreinterpret_u64_u8(ctrl), 0); + // ctrl | ~(ctrl >> 7) will have the lowest bit set to zero for kEmpty and + // kDeleted. We lower all other bits and count number of trailing zeros. + // Clang and GCC optimize countr_zero to rbit+clz without any check for 0, + // so we should be fine. + constexpr uint64_t bits = 0x0101010101010101ULL; + return countr_zero((mask | ~(mask >> 7)) & bits) >> 3; + } + + void ConvertSpecialToEmptyAndFullToDeleted(ctrl_t* dst) const { + uint64_t mask = vget_lane_u64(vreinterpret_u64_u8(ctrl), 0); + constexpr uint64_t msbs = 0x8080808080808080ULL; + constexpr uint64_t lsbs = 0x0101010101010101ULL; + auto x = mask & msbs; + auto res = (~x + (x >> 7)) & ~lsbs; + little_endian::Store64(dst, res); + } + + uint8x8_t ctrl; +}; +#endif // ABSL_INTERNAL_HAVE_ARM_NEON && ABSL_IS_LITTLE_ENDIAN + struct GroupPortableImpl { static constexpr size_t kWidth = 8; @@ -403,7 +664,7 @@ struct GroupPortableImpl { // // Caveat: there are false positives but: // - they only occur if there is a real match - // - they never occur on kEmpty, kDeleted, kSentinel + // - they never occur on ctrl_t::kEmpty, ctrl_t::kDeleted, ctrl_t::kSentinel // - they will be handled gracefully by subsequent checks in code // // Example: @@ -416,19 +677,23 @@ struct GroupPortableImpl { return BitMask<uint64_t, kWidth, 3>((x - lsbs) & ~x & msbs); } - BitMask<uint64_t, kWidth, 3> MatchEmpty() const { + NonIterableBitMask<uint64_t, kWidth, 3> MaskEmpty() const { constexpr uint64_t msbs = 0x8080808080808080ULL; - return BitMask<uint64_t, kWidth, 3>((ctrl & (~ctrl << 6)) & msbs); + return NonIterableBitMask<uint64_t, kWidth, 3>((ctrl & (~ctrl << 6)) & + msbs); } - BitMask<uint64_t, kWidth, 3> MatchEmptyOrDeleted() const { + NonIterableBitMask<uint64_t, kWidth, 3> MaskEmptyOrDeleted() const { constexpr uint64_t msbs = 0x8080808080808080ULL; - return BitMask<uint64_t, kWidth, 3>((ctrl & (~ctrl << 7)) & msbs); + return NonIterableBitMask<uint64_t, kWidth, 3>((ctrl & (~ctrl << 7)) & + msbs); } uint32_t CountLeadingEmptyOrDeleted() const { - constexpr uint64_t gaps = 0x00FEFEFEFEFEFEFEULL; - return (TrailingZeros(((~ctrl & (ctrl >> 7)) | gaps) + 1) + 7) >> 3; + // ctrl | ~(ctrl >> 7) will have the lowest bit set to zero for kEmpty and + // kDeleted. We lower all other bits and count number of trailing zeros. + constexpr uint64_t bits = 0x0101010101010101ULL; + return countr_zero((ctrl | ~(ctrl >> 7)) & bits) >> 3; } void ConvertSpecialToEmptyAndFullToDeleted(ctrl_t* dst) const { @@ -442,28 +707,40 @@ struct GroupPortableImpl { uint64_t ctrl; }; -#if ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSE2 +#ifdef ABSL_INTERNAL_HAVE_SSE2 using Group = GroupSse2Impl; +#elif defined(ABSL_INTERNAL_HAVE_ARM_NEON) && defined(ABSL_IS_LITTLE_ENDIAN) +using Group = GroupAArch64Impl; #else using Group = GroupPortableImpl; #endif +// Returns he number of "cloned control bytes". +// +// This is the number of control bytes that are present both at the beginning +// of the control byte array and at the end, such that we can create a +// `Group::kWidth`-width probe window starting from any control byte. +constexpr size_t NumClonedBytes() { return Group::kWidth - 1; } + template <class Policy, class Hash, class Eq, class Alloc> class raw_hash_set; +// Returns whether `n` is a valid capacity (i.e., number of slots). +// +// A valid capacity is a non-zero integer `2^m - 1`. inline bool IsValidCapacity(size_t n) { return ((n + 1) & n) == 0 && n > 0; } +// Applies the following mapping to every byte in the control array: +// * kDeleted -> kEmpty +// * kEmpty -> kEmpty +// * _ -> kDeleted // PRECONDITION: // IsValidCapacity(capacity) -// ctrl[capacity] == kSentinel -// ctrl[i] != kSentinel for all i < capacity -// Applies mapping for every byte in ctrl: -// DELETED -> EMPTY -// EMPTY -> EMPTY -// FULL -> DELETED +// ctrl[capacity] == ctrl_t::kSentinel +// ctrl[i] != ctrl_t::kSentinel for all i < capacity void ConvertDeletedToEmptyAndFullToDeleted(ctrl_t* ctrl, size_t capacity); -// Rounds up the capacity to the next power of 2 minus 1, with a minimum of 1. +// Converts `n` into the next valid capacity, per `IsValidCapacity`. inline size_t NormalizeCapacity(size_t n) { return n ? ~size_t{} >> countl_zero(n) : 1; } @@ -476,8 +753,8 @@ inline size_t NormalizeCapacity(size_t n) { // never need to probe (the whole table fits in one group) so we don't need a // load factor less than 1. -// Given `capacity` of the table, returns the size (i.e. number of full slots) -// at which we should grow the capacity. +// Given `capacity`, applies the load factor; i.e., it returns the maximum +// number of values we should put into the table before a resizing rehash. inline size_t CapacityToGrowth(size_t capacity) { assert(IsValidCapacity(capacity)); // `capacity*7/8` @@ -487,8 +764,12 @@ inline size_t CapacityToGrowth(size_t capacity) { } return capacity - capacity / 8; } -// From desired "growth" to a lowerbound of the necessary capacity. -// Might not be a valid one and requires NormalizeCapacity(). + +// Given `growth`, "unapplies" the load factor to find how large the capacity +// should be to stay within the load factor. +// +// This might not be a valid capacity and `NormalizeCapacity()` should be +// called on this. inline size_t GrowthToLowerboundCapacity(size_t growth) { // `growth*8/7` if (Group::kWidth == 8 && growth == 7) { @@ -498,16 +779,31 @@ inline size_t GrowthToLowerboundCapacity(size_t growth) { return growth + static_cast<size_t>((static_cast<int64_t>(growth) - 1) / 7); } -inline void AssertIsFull(ctrl_t* ctrl) { - ABSL_HARDENING_ASSERT((ctrl != nullptr && IsFull(*ctrl)) && - "Invalid operation on iterator. The element might have " - "been erased, or the table might have rehashed."); +template <class InputIter> +size_t SelectBucketCountForIterRange(InputIter first, InputIter last, + size_t bucket_count) { + if (bucket_count != 0) { + return bucket_count; + } + using InputIterCategory = + typename std::iterator_traits<InputIter>::iterator_category; + if (std::is_base_of<std::random_access_iterator_tag, + InputIterCategory>::value) { + return GrowthToLowerboundCapacity( + static_cast<size_t>(std::distance(first, last))); + } + return 0; } +#define ABSL_INTERNAL_ASSERT_IS_FULL(ctrl, msg) \ + ABSL_HARDENING_ASSERT((ctrl != nullptr && IsFull(*ctrl)) && msg) + inline void AssertIsValid(ctrl_t* ctrl) { - ABSL_HARDENING_ASSERT((ctrl == nullptr || IsFull(*ctrl)) && - "Invalid operation on iterator. The element might have " - "been erased, or the table might have rehashed."); + ABSL_HARDENING_ASSERT( + (ctrl == nullptr || IsFull(*ctrl)) && + "Invalid operation on iterator. The element might have " + "been erased, the table might have rehashed, or this may " + "be an end() iterator."); } struct FindInfo { @@ -515,42 +811,40 @@ struct FindInfo { size_t probe_length; }; -// The representation of the object has two modes: -// - small: For capacities < kWidth-1 -// - large: For the rest. +// Whether a table is "small". A small table fits entirely into a probing +// group, i.e., has a capacity < `Group::kWidth`. // -// Differences: -// - In small mode we are able to use the whole capacity. The extra control -// bytes give us at least one "empty" control byte to stop the iteration. -// This is important to make 1 a valid capacity. +// In small mode we are able to use the whole capacity. The extra control +// bytes give us at least one "empty" control byte to stop the iteration. +// This is important to make 1 a valid capacity. // -// - In small mode only the first `capacity()` control bytes after the -// sentinel are valid. The rest contain dummy kEmpty values that do not -// represent a real slot. This is important to take into account on -// find_first_non_full(), where we never try ShouldInsertBackwards() for -// small tables. +// In small mode only the first `capacity` control bytes after the sentinel +// are valid. The rest contain dummy ctrl_t::kEmpty values that do not +// represent a real slot. This is important to take into account on +// `find_first_non_full()`, where we never try +// `ShouldInsertBackwards()` for small tables. inline bool is_small(size_t capacity) { return capacity < Group::kWidth - 1; } -inline probe_seq<Group::kWidth> probe(ctrl_t* ctrl, size_t hash, +// Begins a probing operation on `ctrl`, using `hash`. +inline probe_seq<Group::kWidth> probe(const ctrl_t* ctrl, size_t hash, size_t capacity) { return probe_seq<Group::kWidth>(H1(hash, ctrl), capacity); } -// Probes the raw_hash_set with the probe sequence for hash and returns the -// pointer to the first empty or deleted slot. -// NOTE: this function must work with tables having both kEmpty and kDelete -// in one group. Such tables appears during drop_deletes_without_resize. +// Probes an array of control bits using a probe sequence derived from `hash`, +// and returns the offset corresponding to the first deleted or empty slot. +// +// Behavior when the entire table is full is undefined. // -// This function is very useful when insertions happen and: -// - the input is already a set -// - there are enough slots -// - the element with the hash is not in the table -inline FindInfo find_first_non_full(ctrl_t* ctrl, size_t hash, +// NOTE: this function must work with tables having both empty and deleted +// slots in the same group. Such tables appear during `erase()`. +template <typename = void> +inline FindInfo find_first_non_full(const ctrl_t* ctrl, size_t hash, size_t capacity) { auto seq = probe(ctrl, hash, capacity); while (true) { Group g{ctrl + seq.offset()}; - auto mask = g.MatchEmptyOrDeleted(); + auto mask = g.MaskEmptyOrDeleted(); if (mask) { #if !defined(NDEBUG) // We want to add entropy even when ASLR is not enabled. @@ -564,10 +858,66 @@ inline FindInfo find_first_non_full(ctrl_t* ctrl, size_t hash, return {seq.offset(mask.LowestBitSet()), seq.index()}; } seq.next(); - assert(seq.index() < capacity && "full table!"); + assert(seq.index() <= capacity && "full table!"); + } +} + +// Extern template for inline function keep possibility of inlining. +// When compiler decided to not inline, no symbols will be added to the +// corresponding translation unit. +extern template FindInfo find_first_non_full(const ctrl_t*, size_t, size_t); + +// Sets `ctrl` to `{kEmpty, kSentinel, ..., kEmpty}`, marking the entire +// array as marked as empty. +inline void ResetCtrl(size_t capacity, ctrl_t* ctrl, const void* slot, + size_t slot_size) { + std::memset(ctrl, static_cast<int8_t>(ctrl_t::kEmpty), + capacity + 1 + NumClonedBytes()); + ctrl[capacity] = ctrl_t::kSentinel; + SanitizerPoisonMemoryRegion(slot, slot_size * capacity); +} + +// Sets `ctrl[i]` to `h`. +// +// Unlike setting it directly, this function will perform bounds checks and +// mirror the value to the cloned tail if necessary. +inline void SetCtrl(size_t i, ctrl_t h, size_t capacity, ctrl_t* ctrl, + const void* slot, size_t slot_size) { + assert(i < capacity); + + auto* slot_i = static_cast<const char*>(slot) + i * slot_size; + if (IsFull(h)) { + SanitizerUnpoisonMemoryRegion(slot_i, slot_size); + } else { + SanitizerPoisonMemoryRegion(slot_i, slot_size); } + + ctrl[i] = h; + ctrl[((i - NumClonedBytes()) & capacity) + (NumClonedBytes() & capacity)] = h; } +// Overload for setting to an occupied `h2_t` rather than a special `ctrl_t`. +inline void SetCtrl(size_t i, h2_t h, size_t capacity, ctrl_t* ctrl, + const void* slot, size_t slot_size) { + SetCtrl(i, static_cast<ctrl_t>(h), capacity, ctrl, slot, slot_size); +} + +// Given the capacity of a table, computes the offset (from the start of the +// backing allocation) at which the slots begin. +inline size_t SlotOffset(size_t capacity, size_t slot_align) { + assert(IsValidCapacity(capacity)); + const size_t num_control_bytes = capacity + 1 + NumClonedBytes(); + return (num_control_bytes + slot_align - 1) & (~slot_align + 1); +} + +// Given the capacity of a table, computes the total size of the backing +// array. +inline size_t AllocSize(size_t capacity, size_t slot_size, size_t slot_align) { + return SlotOffset(capacity, slot_align) + capacity * slot_size; +} + +// A SwissTable. +// // Policy: a policy defines how to perform different operations on // the slots of the hashtable (see hash_policy_traits.h for the full interface // of policy). @@ -624,13 +974,6 @@ class raw_hash_set { auto KeyTypeCanBeHashed(const Hash& h, const key_type& k) -> decltype(h(k)); auto KeyTypeCanBeEq(const Eq& eq, const key_type& k) -> decltype(eq(k, k)); - using Layout = absl::container_internal::Layout<ctrl_t, slot_type>; - - static Layout MakeLayout(size_t capacity) { - assert(IsValidCapacity(capacity)); - return Layout(capacity + Group::kWidth + 1, capacity); - } - using AllocTraits = absl::allocator_traits<allocator_type>; using SlotAlloc = typename absl::allocator_traits< allocator_type>::template rebind_alloc<slot_type>; @@ -689,16 +1032,22 @@ class raw_hash_set { // PRECONDITION: not an end() iterator. reference operator*() const { - AssertIsFull(ctrl_); + ABSL_INTERNAL_ASSERT_IS_FULL(ctrl_, + "operator*() called on invalid iterator."); return PolicyTraits::element(slot_); } // PRECONDITION: not an end() iterator. - pointer operator->() const { return &operator*(); } + pointer operator->() const { + ABSL_INTERNAL_ASSERT_IS_FULL(ctrl_, + "operator-> called on invalid iterator."); + return &operator*(); + } // PRECONDITION: not an end() iterator. iterator& operator++() { - AssertIsFull(ctrl_); + ABSL_INTERNAL_ASSERT_IS_FULL(ctrl_, + "operator++ called on invalid iterator."); ++ctrl_; ++slot_; skip_empty_or_deleted(); @@ -724,16 +1073,20 @@ class raw_hash_set { iterator(ctrl_t* ctrl, slot_type* slot) : ctrl_(ctrl), slot_(slot) { // This assumption helps the compiler know that any non-end iterator is // not equal to any end iterator. - ABSL_INTERNAL_ASSUME(ctrl != nullptr); + ABSL_ASSUME(ctrl != nullptr); } + // Fixes up `ctrl_` to point to a full by advancing it and `slot_` until + // they reach one. + // + // If a sentinel is reached, we null both of them out instead. void skip_empty_or_deleted() { while (IsEmptyOrDeleted(*ctrl_)) { uint32_t shift = Group{ctrl_}.CountLeadingEmptyOrDeleted(); ctrl_ += shift; slot_ += shift; } - if (ABSL_PREDICT_FALSE(*ctrl_ == kSentinel)) ctrl_ = nullptr; + if (ABSL_PREDICT_FALSE(*ctrl_ == ctrl_t::kSentinel)) ctrl_ = nullptr; } ctrl_t* ctrl_ = nullptr; @@ -814,7 +1167,8 @@ class raw_hash_set { raw_hash_set(InputIter first, InputIter last, size_t bucket_count = 0, const hasher& hash = hasher(), const key_equal& eq = key_equal(), const allocator_type& alloc = allocator_type()) - : raw_hash_set(bucket_count, hash, eq, alloc) { + : raw_hash_set(SelectBucketCountForIterRange(first, last, bucket_count), + hash, eq, alloc) { insert(first, last); } @@ -902,7 +1256,8 @@ class raw_hash_set { for (const auto& v : that) { const size_t hash = PolicyTraits::apply(HashElement{hash_ref()}, v); auto target = find_first_non_full(ctrl_, hash, capacity_); - set_ctrl(target.offset, H2(hash)); + SetCtrl(target.offset, H2(hash), capacity_, ctrl_, slots_, + sizeof(slot_type)); emplace_at(target.offset, v); infoz().RecordInsert(hash, target.probe_length); } @@ -998,6 +1353,8 @@ class raw_hash_set { // past that we simply deallocate the array. if (capacity_ > 127) { destroy_slots(); + + infoz().RecordClearedReservation(); } else if (capacity_) { for (size_t i = 0; i != capacity_; ++i) { if (IsFull(ctrl_[i])) { @@ -1005,7 +1362,7 @@ class raw_hash_set { } } size_ = 0; - reset_ctrl(); + ResetCtrl(capacity_, ctrl_, slots_, sizeof(slot_type)); reset_growth_left(); } assert(empty()); @@ -1019,8 +1376,7 @@ class raw_hash_set { // m.insert(std::make_pair("abc", 42)); // TODO(cheshire): A type alias T2 is introduced as a workaround for the nvcc // bug. - template <class T, RequiresInsertable<T> = 0, - class T2 = T, + template <class T, RequiresInsertable<T> = 0, class T2 = T, typename std::enable_if<IsDecomposable<T2>::value, int>::type = 0, T* = nullptr> std::pair<iterator, bool> insert(T&& value) { @@ -1240,7 +1596,8 @@ class raw_hash_set { // This overload is necessary because otherwise erase<K>(const K&) would be // a better match if non-const iterator is passed as an argument. void erase(iterator it) { - AssertIsFull(it.ctrl_); + ABSL_INTERNAL_ASSERT_IS_FULL(it.ctrl_, + "erase() called on invalid iterator."); PolicyTraits::destroy(&alloc_ref(), it.slot_); erase_meta_only(it); } @@ -1274,7 +1631,8 @@ class raw_hash_set { } node_type extract(const_iterator position) { - AssertIsFull(position.inner_.ctrl_); + ABSL_INTERNAL_ASSERT_IS_FULL(position.inner_.ctrl_, + "extract() called on invalid iterator."); auto node = CommonAccess::Transfer<node_type>(alloc_ref(), position.inner_.slot_); erase_meta_only(position); @@ -1311,21 +1669,31 @@ class raw_hash_set { if (n == 0 && size_ == 0) { destroy_slots(); infoz().RecordStorageChanged(0, 0); + infoz().RecordClearedReservation(); return; } + // bitor is a faster way of doing `max` here. We will round up to the next // power-of-2-minus-1, so bitor is good enough. auto m = NormalizeCapacity(n | GrowthToLowerboundCapacity(size())); // n == 0 unconditionally rehashes as per the standard. if (n == 0 || m > capacity_) { resize(m); + + // This is after resize, to ensure that we have completed the allocation + // and have potentially sampled the hashtable. + infoz().RecordReservation(n); } } void reserve(size_t n) { - size_t m = GrowthToLowerboundCapacity(n); - if (m > capacity_) { + if (n > size() + growth_left()) { + size_t m = GrowthToLowerboundCapacity(n); resize(NormalizeCapacity(m)); + + // This is after resize, to ensure that we have completed the allocation + // and have potentially sampled the hashtable. + infoz().RecordReservation(n); } } @@ -1351,11 +1719,13 @@ class raw_hash_set { template <class K = key_type> void prefetch(const key_arg<K>& key) const { (void)key; -#if defined(__GNUC__) + // Avoid probing if we won't be able to prefetch the addresses received. +#ifdef ABSL_INTERNAL_HAVE_PREFETCH + prefetch_heap_block(); auto seq = probe(ctrl_, hash_ref()(key), capacity_); - __builtin_prefetch(static_cast<const void*>(ctrl_ + seq.offset())); - __builtin_prefetch(static_cast<const void*>(slots_ + seq.offset())); -#endif // __GNUC__ + base_internal::PrefetchT0(ctrl_ + seq.offset()); + base_internal::PrefetchT0(slots_ + seq.offset()); +#endif // ABSL_INTERNAL_HAVE_PREFETCH } // The API of find() has two extensions. @@ -1370,19 +1740,20 @@ class raw_hash_set { auto seq = probe(ctrl_, hash, capacity_); while (true) { Group g{ctrl_ + seq.offset()}; - for (int i : g.Match(H2(hash))) { + for (uint32_t i : g.Match(H2(hash))) { if (ABSL_PREDICT_TRUE(PolicyTraits::apply( EqualElement<K>{key, eq_ref()}, PolicyTraits::element(slots_ + seq.offset(i))))) return iterator_at(seq.offset(i)); } - if (ABSL_PREDICT_TRUE(g.MatchEmpty())) return end(); + if (ABSL_PREDICT_TRUE(g.MaskEmpty())) return end(); seq.next(); - assert(seq.index() < capacity_ && "full table!"); + assert(seq.index() <= capacity_ && "full table!"); } } template <class K = key_type> iterator find(const key_arg<K>& key) { + prefetch_heap_block(); return find(key, hash_ref()(key)); } @@ -1392,6 +1763,7 @@ class raw_hash_set { } template <class K = key_type> const_iterator find(const key_arg<K>& key) const { + prefetch_heap_block(); return find(key, hash_ref()(key)); } @@ -1441,6 +1813,14 @@ class raw_hash_set { return !(a == b); } + template <typename H> + friend typename std::enable_if<H::template is_hashable<value_type>::value, + H>::type + AbslHashValue(H h, const raw_hash_set& s) { + return H::combine(H::combine_unordered(std::move(h), s.begin(), s.end()), + s.size()); + } + friend void swap(raw_hash_set& a, raw_hash_set& b) noexcept(noexcept(a.swap(b))) { a.swap(b); @@ -1506,17 +1886,17 @@ class raw_hash_set { slot_type&& slot; }; - // "erases" the object from the container, except that it doesn't actually - // destroy the object. It only updates all the metadata of the class. - // This can be used in conjunction with Policy::transfer to move the object to - // another place. + // Erases, but does not destroy, the value pointed to by `it`. + // + // This merely updates the pertinent control byte. This can be used in + // conjunction with Policy::transfer to move the object to another place. void erase_meta_only(const_iterator it) { assert(IsFull(*it.inner_.ctrl_) && "erasing a dangling iterator"); --size_; - const size_t index = it.inner_.ctrl_ - ctrl_; + const size_t index = static_cast<size_t>(it.inner_.ctrl_ - ctrl_); const size_t index_before = (index - Group::kWidth) & capacity_; - const auto empty_after = Group(it.inner_.ctrl_).MatchEmpty(); - const auto empty_before = Group(ctrl_ + index_before).MatchEmpty(); + const auto empty_after = Group(it.inner_.ctrl_).MaskEmpty(); + const auto empty_before = Group(ctrl_ + index_before).MaskEmpty(); // We count how many consecutive non empties we have to the right and to the // left of `it`. If the sum is >= kWidth then there is at least one probe @@ -1526,11 +1906,17 @@ class raw_hash_set { static_cast<size_t>(empty_after.TrailingZeros() + empty_before.LeadingZeros()) < Group::kWidth; - set_ctrl(index, was_never_full ? kEmpty : kDeleted); + SetCtrl(index, was_never_full ? ctrl_t::kEmpty : ctrl_t::kDeleted, + capacity_, ctrl_, slots_, sizeof(slot_type)); growth_left() += was_never_full; infoz().RecordErase(); } + // Allocates a backing array for `self` and initializes its control bytes. + // This reads `capacity_` and updates all other fields based on the result of + // the allocation. + // + // This does not free the currently held array; `capacity_` must be nonzero. void initialize_slots() { assert(capacity_); // Folks with custom allocators often make unwarranted assumptions about the @@ -1545,19 +1931,24 @@ class raw_hash_set { // bound more carefully. if (std::is_same<SlotAlloc, std::allocator<slot_type>>::value && slots_ == nullptr) { - infoz() = Sample(); + infoz() = Sample(sizeof(slot_type)); } - auto layout = MakeLayout(capacity_); - char* mem = static_cast<char*>( - Allocate<Layout::Alignment()>(&alloc_ref(), layout.AllocSize())); - ctrl_ = reinterpret_cast<ctrl_t*>(layout.template Pointer<0>(mem)); - slots_ = layout.template Pointer<1>(mem); - reset_ctrl(); + char* mem = static_cast<char*>(Allocate<alignof(slot_type)>( + &alloc_ref(), + AllocSize(capacity_, sizeof(slot_type), alignof(slot_type)))); + ctrl_ = reinterpret_cast<ctrl_t*>(mem); + slots_ = reinterpret_cast<slot_type*>( + mem + SlotOffset(capacity_, alignof(slot_type))); + ResetCtrl(capacity_, ctrl_, slots_, sizeof(slot_type)); reset_growth_left(); infoz().RecordStorageChanged(size_, capacity_); } + // Destroys all slots in the backing array, frees the backing array, and + // clears all top-level book-keeping data. + // + // This essentially implements `map = raw_hash_set();`. void destroy_slots() { if (!capacity_) return; for (size_t i = 0; i != capacity_; ++i) { @@ -1565,10 +1956,12 @@ class raw_hash_set { PolicyTraits::destroy(&alloc_ref(), slots_ + i); } } - auto layout = MakeLayout(capacity_); + // Unpoison before returning the memory to the allocator. SanitizerUnpoisonMemoryRegion(slots_, sizeof(slot_type) * capacity_); - Deallocate<Layout::Alignment()>(&alloc_ref(), ctrl_, layout.AllocSize()); + Deallocate<alignof(slot_type)>( + &alloc_ref(), ctrl_, + AllocSize(capacity_, sizeof(slot_type), alignof(slot_type))); ctrl_ = EmptyGroup(); slots_ = nullptr; size_ = 0; @@ -1592,20 +1985,23 @@ class raw_hash_set { auto target = find_first_non_full(ctrl_, hash, capacity_); size_t new_i = target.offset; total_probe_length += target.probe_length; - set_ctrl(new_i, H2(hash)); + SetCtrl(new_i, H2(hash), capacity_, ctrl_, slots_, sizeof(slot_type)); PolicyTraits::transfer(&alloc_ref(), slots_ + new_i, old_slots + i); } } if (old_capacity) { SanitizerUnpoisonMemoryRegion(old_slots, sizeof(slot_type) * old_capacity); - auto layout = MakeLayout(old_capacity); - Deallocate<Layout::Alignment()>(&alloc_ref(), old_ctrl, - layout.AllocSize()); + Deallocate<alignof(slot_type)>( + &alloc_ref(), old_ctrl, + AllocSize(old_capacity, sizeof(slot_type), alignof(slot_type))); } infoz().RecordRehash(total_probe_length); } + // Prunes control bytes to remove as many tombstones as possible. + // + // See the comment on `rehash_and_grow_if_necessary()`. void drop_deletes_without_resize() ABSL_ATTRIBUTE_NOINLINE { assert(IsValidCapacity(capacity_)); assert(!is_small(capacity_)); @@ -1631,35 +2027,35 @@ class raw_hash_set { slot_type* slot = reinterpret_cast<slot_type*>(&raw); for (size_t i = 0; i != capacity_; ++i) { if (!IsDeleted(ctrl_[i])) continue; - size_t hash = PolicyTraits::apply(HashElement{hash_ref()}, - PolicyTraits::element(slots_ + i)); - auto target = find_first_non_full(ctrl_, hash, capacity_); - size_t new_i = target.offset; + const size_t hash = PolicyTraits::apply( + HashElement{hash_ref()}, PolicyTraits::element(slots_ + i)); + const FindInfo target = find_first_non_full(ctrl_, hash, capacity_); + const size_t new_i = target.offset; total_probe_length += target.probe_length; // Verify if the old and new i fall within the same group wrt the hash. // If they do, we don't need to move the object as it falls already in the // best probe we can. - const auto probe_index = [&](size_t pos) { - return ((pos - probe(ctrl_, hash, capacity_).offset()) & capacity_) / - Group::kWidth; + const size_t probe_offset = probe(ctrl_, hash, capacity_).offset(); + const auto probe_index = [probe_offset, this](size_t pos) { + return ((pos - probe_offset) & capacity_) / Group::kWidth; }; // Element doesn't move. if (ABSL_PREDICT_TRUE(probe_index(new_i) == probe_index(i))) { - set_ctrl(i, H2(hash)); + SetCtrl(i, H2(hash), capacity_, ctrl_, slots_, sizeof(slot_type)); continue; } if (IsEmpty(ctrl_[new_i])) { // Transfer element to the empty spot. - // set_ctrl poisons/unpoisons the slots so we have to call it at the + // SetCtrl poisons/unpoisons the slots so we have to call it at the // right time. - set_ctrl(new_i, H2(hash)); + SetCtrl(new_i, H2(hash), capacity_, ctrl_, slots_, sizeof(slot_type)); PolicyTraits::transfer(&alloc_ref(), slots_ + new_i, slots_ + i); - set_ctrl(i, kEmpty); + SetCtrl(i, ctrl_t::kEmpty, capacity_, ctrl_, slots_, sizeof(slot_type)); } else { assert(IsDeleted(ctrl_[new_i])); - set_ctrl(new_i, H2(hash)); + SetCtrl(new_i, H2(hash), capacity_, ctrl_, slots_, sizeof(slot_type)); // Until we are done rehashing, DELETED marks previously FULL slots. // Swap i and new_i elements. PolicyTraits::transfer(&alloc_ref(), slot, slots_ + i); @@ -1672,11 +2068,58 @@ class raw_hash_set { infoz().RecordRehash(total_probe_length); } + // Called whenever the table *might* need to conditionally grow. + // + // This function is an optimization opportunity to perform a rehash even when + // growth is unnecessary, because vacating tombstones is beneficial for + // performance in the long-run. void rehash_and_grow_if_necessary() { if (capacity_ == 0) { resize(1); - } else if (size() <= CapacityToGrowth(capacity()) / 2) { + } else if (capacity_ > Group::kWidth && + // Do these calcuations in 64-bit to avoid overflow. + size() * uint64_t{32} <= capacity_ * uint64_t{25}) { // Squash DELETED without growing if there is enough capacity. + // + // Rehash in place if the current size is <= 25/32 of capacity_. + // Rationale for such a high factor: 1) drop_deletes_without_resize() is + // faster than resize, and 2) it takes quite a bit of work to add + // tombstones. In the worst case, seems to take approximately 4 + // insert/erase pairs to create a single tombstone and so if we are + // rehashing because of tombstones, we can afford to rehash-in-place as + // long as we are reclaiming at least 1/8 the capacity without doing more + // than 2X the work. (Where "work" is defined to be size() for rehashing + // or rehashing in place, and 1 for an insert or erase.) But rehashing in + // place is faster per operation than inserting or even doubling the size + // of the table, so we actually afford to reclaim even less space from a + // resize-in-place. The decision is to rehash in place if we can reclaim + // at about 1/8th of the usable capacity (specifically 3/28 of the + // capacity) which means that the total cost of rehashing will be a small + // fraction of the total work. + // + // Here is output of an experiment using the BM_CacheInSteadyState + // benchmark running the old case (where we rehash-in-place only if we can + // reclaim at least 7/16*capacity_) vs. this code (which rehashes in place + // if we can recover 3/32*capacity_). + // + // Note that although in the worst-case number of rehashes jumped up from + // 15 to 190, but the number of operations per second is almost the same. + // + // Abridged output of running BM_CacheInSteadyState benchmark from + // raw_hash_set_benchmark. N is the number of insert/erase operations. + // + // | OLD (recover >= 7/16 | NEW (recover >= 3/32) + // size | N/s LoadFactor NRehashes | N/s LoadFactor NRehashes + // 448 | 145284 0.44 18 | 140118 0.44 19 + // 493 | 152546 0.24 11 | 151417 0.48 28 + // 538 | 151439 0.26 11 | 151152 0.53 38 + // 583 | 151765 0.28 11 | 150572 0.57 50 + // 628 | 150241 0.31 11 | 150853 0.61 66 + // 672 | 149602 0.33 12 | 150110 0.66 90 + // 717 | 149998 0.35 12 | 149531 0.70 129 + // 762 | 149836 0.37 13 | 148559 0.74 190 + // 807 | 149736 0.39 14 | 151107 0.39 14 + // 852 | 150204 0.42 15 | 151019 0.42 15 drop_deletes_without_resize(); } else { // Otherwise grow the container. @@ -1689,14 +2132,14 @@ class raw_hash_set { auto seq = probe(ctrl_, hash, capacity_); while (true) { Group g{ctrl_ + seq.offset()}; - for (int i : g.Match(H2(hash))) { + for (uint32_t i : g.Match(H2(hash))) { if (ABSL_PREDICT_TRUE(PolicyTraits::element(slots_ + seq.offset(i)) == elem)) return true; } - if (ABSL_PREDICT_TRUE(g.MatchEmpty())) return false; + if (ABSL_PREDICT_TRUE(g.MaskEmpty())) return false; seq.next(); - assert(seq.index() < capacity_ && "full table!"); + assert(seq.index() <= capacity_ && "full table!"); } return false; } @@ -1714,25 +2157,33 @@ class raw_hash_set { } protected: + // Attempts to find `key` in the table; if it isn't found, returns a slot that + // the value can be inserted into, with the control byte already set to + // `key`'s H2. template <class K> std::pair<size_t, bool> find_or_prepare_insert(const K& key) { + prefetch_heap_block(); auto hash = hash_ref()(key); auto seq = probe(ctrl_, hash, capacity_); while (true) { Group g{ctrl_ + seq.offset()}; - for (int i : g.Match(H2(hash))) { + for (uint32_t i : g.Match(H2(hash))) { if (ABSL_PREDICT_TRUE(PolicyTraits::apply( EqualElement<K>{key, eq_ref()}, PolicyTraits::element(slots_ + seq.offset(i))))) return {seq.offset(i), false}; } - if (ABSL_PREDICT_TRUE(g.MatchEmpty())) break; + if (ABSL_PREDICT_TRUE(g.MaskEmpty())) break; seq.next(); - assert(seq.index() < capacity_ && "full table!"); + assert(seq.index() <= capacity_ && "full table!"); } return {prepare_insert(hash), true}; } + // Given the hash of a value not currently in the table, finds the next + // viable slot index to insert it at. + // + // REQUIRES: At least one non-full slot available. size_t prepare_insert(size_t hash) ABSL_ATTRIBUTE_NOINLINE { auto target = find_first_non_full(ctrl_, hash, capacity_); if (ABSL_PREDICT_FALSE(growth_left() == 0 && @@ -1742,7 +2193,8 @@ class raw_hash_set { } ++size_; growth_left() -= IsEmpty(ctrl_[target.offset]); - set_ctrl(target.offset, H2(hash)); + SetCtrl(target.offset, H2(hash), capacity_, ctrl_, slots_, + sizeof(slot_type)); infoz().RecordInsert(hash, target.probe_length); return target.offset; } @@ -1771,35 +2223,29 @@ class raw_hash_set { private: friend struct RawHashSetTestOnlyAccess; - // Reset all ctrl bytes back to kEmpty, except the sentinel. - void reset_ctrl() { - std::memset(ctrl_, kEmpty, capacity_ + Group::kWidth); - ctrl_[capacity_] = kSentinel; - SanitizerPoisonMemoryRegion(slots_, sizeof(slot_type) * capacity_); - } - void reset_growth_left() { growth_left() = CapacityToGrowth(capacity()) - size_; } - // Sets the control byte, and if `i < Group::kWidth`, set the cloned byte at - // the end too. - void set_ctrl(size_t i, ctrl_t h) { - assert(i < capacity_); - - if (IsFull(h)) { - SanitizerUnpoisonObject(slots_ + i); - } else { - SanitizerPoisonObject(slots_ + i); - } + // The number of slots we can still fill without needing to rehash. + // + // This is stored separately due to tombstones: we do not include tombstones + // in the growth capacity, because we'd like to rehash when the table is + // otherwise filled with tombstones: otherwise, probe sequences might get + // unacceptably long without triggering a rehash. Callers can also force a + // rehash via the standard `rehash(0)`, which will recompute this value as a + // side-effect. + // + // See `CapacityToGrowth()`. + size_t& growth_left() { return settings_.template get<0>(); } - ctrl_[i] = h; - ctrl_[((i - Group::kWidth) & capacity_) + 1 + - ((Group::kWidth - 1) & capacity_)] = h; + // Prefetch the heap-allocated memory region to resolve potential TLB misses. + // This is intended to overlap with execution of calculating the hash for a + // key. + void prefetch_heap_block() const { + base_internal::PrefetchT2(ctrl_); } - size_t& growth_left() { return settings_.template get<0>(); } - HashtablezInfoHandle& infoz() { return settings_.template get<1>(); } hasher& hash_ref() { return settings_.template get<2>(); } @@ -1814,26 +2260,41 @@ class raw_hash_set { // TODO(alkis): Investigate removing some of these fields: // - ctrl/slots can be derived from each other // - size can be moved into the slot array - ctrl_t* ctrl_ = EmptyGroup(); // [(capacity + 1) * ctrl_t] - slot_type* slots_ = nullptr; // [capacity * slot_type] - size_t size_ = 0; // number of full slots - size_t capacity_ = 0; // total number of slots + + // The control bytes (and, also, a pointer to the base of the backing array). + // + // This contains `capacity_ + 1 + NumClonedBytes()` entries, even + // when the table is empty (hence EmptyGroup). + ctrl_t* ctrl_ = EmptyGroup(); + // The beginning of the slots, located at `SlotOffset()` bytes after + // `ctrl_`. May be null for empty tables. + slot_type* slots_ = nullptr; + + // The number of filled slots. + size_t size_ = 0; + + // The total number of available slots. + size_t capacity_ = 0; absl::container_internal::CompressedTuple<size_t /* growth_left */, HashtablezInfoHandle, hasher, key_equal, allocator_type> - settings_{0, HashtablezInfoHandle{}, hasher{}, key_equal{}, + settings_{0u, HashtablezInfoHandle{}, hasher{}, key_equal{}, allocator_type{}}; }; // Erases all elements that satisfy the predicate `pred` from the container `c`. template <typename P, typename H, typename E, typename A, typename Predicate> -void EraseIf(Predicate pred, raw_hash_set<P, H, E, A>* c) { +typename raw_hash_set<P, H, E, A>::size_type EraseIf( + Predicate& pred, raw_hash_set<P, H, E, A>* c) { + const auto initial_size = c->size(); for (auto it = c->begin(), last = c->end(); it != last;) { - auto copy_it = it++; - if (pred(*copy_it)) { - c->erase(copy_it); + if (pred(*it)) { + c->erase(it++); + } else { + ++it; } } + return initial_size - c->size(); } namespace hashtable_debug_internal { @@ -1849,7 +2310,7 @@ struct HashtableDebugAccess<Set, absl::void_t<typename Set::raw_hash_set>> { auto seq = probe(set.ctrl_, hash, set.capacity_); while (true) { container_internal::Group g{set.ctrl_ + seq.offset()}; - for (int i : g.Match(container_internal::H2(hash))) { + for (uint32_t i : g.Match(container_internal::H2(hash))) { if (Traits::apply( typename Set::template EqualElement<typename Set::key_type>{ key, set.eq_ref()}, @@ -1857,7 +2318,7 @@ struct HashtableDebugAccess<Set, absl::void_t<typename Set::raw_hash_set>> { return num_probes; ++num_probes; } - if (g.MatchEmpty()) return num_probes; + if (g.MaskEmpty()) return num_probes; seq.next(); ++num_probes; } @@ -1866,8 +2327,7 @@ struct HashtableDebugAccess<Set, absl::void_t<typename Set::raw_hash_set>> { static size_t AllocatedByteSize(const Set& c) { size_t capacity = c.capacity_; if (capacity == 0) return 0; - auto layout = Set::MakeLayout(capacity); - size_t m = layout.AllocSize(); + size_t m = AllocSize(capacity, sizeof(Slot), alignof(Slot)); size_t per_slot = Traits::space_used(static_cast<const Slot*>(nullptr)); if (per_slot != ~size_t{}) { @@ -1885,8 +2345,8 @@ struct HashtableDebugAccess<Set, absl::void_t<typename Set::raw_hash_set>> { static size_t LowerBoundAllocatedByteSize(size_t size) { size_t capacity = GrowthToLowerboundCapacity(size); if (capacity == 0) return 0; - auto layout = Set::MakeLayout(NormalizeCapacity(capacity)); - size_t m = layout.AllocSize(); + size_t m = + AllocSize(NormalizeCapacity(capacity), sizeof(Slot), alignof(Slot)); size_t per_slot = Traits::space_used(static_cast<const Slot*>(nullptr)); if (per_slot != ~size_t{}) { m += per_slot * size; @@ -1900,4 +2360,6 @@ struct HashtableDebugAccess<Set, absl::void_t<typename Set::raw_hash_set>> { ABSL_NAMESPACE_END } // namespace absl +#undef ABSL_INTERNAL_ASSERT_IS_FULL + #endif // ABSL_CONTAINER_INTERNAL_RAW_HASH_SET_H_ diff --git a/absl/container/internal/raw_hash_set_benchmark.cc b/absl/container/internal/raw_hash_set_benchmark.cc index f9be2c5a..47dc9048 100644 --- a/absl/container/internal/raw_hash_set_benchmark.cc +++ b/absl/container/internal/raw_hash_set_benchmark.cc @@ -254,6 +254,23 @@ void BM_CopyAssign(benchmark::State& state) { } BENCHMARK(BM_CopyAssign)->Range(128, 4096); +void BM_RangeCtor(benchmark::State& state) { + std::random_device rd; + std::mt19937 rng(rd()); + std::uniform_int_distribution<uint64_t> dist(0, ~uint64_t{}); + std::vector<int> values; + const size_t desired_size = state.range(0); + while (values.size() < desired_size) { + values.emplace_back(dist(rng)); + } + + for (auto unused : state) { + IntTable t{values.begin(), values.end()}; + benchmark::DoNotOptimize(t); + } +} +BENCHMARK(BM_RangeCtor)->Range(128, 65536); + void BM_NoOpReserveIntTable(benchmark::State& state) { IntTable t; t.reserve(100000); @@ -298,56 +315,79 @@ void BM_ReserveStringTable(benchmark::State& state) { } BENCHMARK(BM_ReserveStringTable)->Range(128, 4096); +// Like std::iota, except that ctrl_t doesn't support operator++. +template <typename CtrlIter> +void Iota(CtrlIter begin, CtrlIter end, int value) { + for (; begin != end; ++begin, ++value) { + *begin = static_cast<ctrl_t>(value); + } +} + void BM_Group_Match(benchmark::State& state) { std::array<ctrl_t, Group::kWidth> group; - std::iota(group.begin(), group.end(), -4); + Iota(group.begin(), group.end(), -4); Group g{group.data()}; h2_t h = 1; for (auto _ : state) { ::benchmark::DoNotOptimize(h); + ::benchmark::DoNotOptimize(g); ::benchmark::DoNotOptimize(g.Match(h)); } } BENCHMARK(BM_Group_Match); -void BM_Group_MatchEmpty(benchmark::State& state) { +void BM_Group_MaskEmpty(benchmark::State& state) { std::array<ctrl_t, Group::kWidth> group; - std::iota(group.begin(), group.end(), -4); + Iota(group.begin(), group.end(), -4); Group g{group.data()}; - for (auto _ : state) ::benchmark::DoNotOptimize(g.MatchEmpty()); + for (auto _ : state) { + ::benchmark::DoNotOptimize(g); + ::benchmark::DoNotOptimize(g.MaskEmpty()); + } } -BENCHMARK(BM_Group_MatchEmpty); +BENCHMARK(BM_Group_MaskEmpty); -void BM_Group_MatchEmptyOrDeleted(benchmark::State& state) { +void BM_Group_MaskEmptyOrDeleted(benchmark::State& state) { std::array<ctrl_t, Group::kWidth> group; - std::iota(group.begin(), group.end(), -4); + Iota(group.begin(), group.end(), -4); Group g{group.data()}; - for (auto _ : state) ::benchmark::DoNotOptimize(g.MatchEmptyOrDeleted()); + for (auto _ : state) { + ::benchmark::DoNotOptimize(g); + ::benchmark::DoNotOptimize(g.MaskEmptyOrDeleted()); + } } -BENCHMARK(BM_Group_MatchEmptyOrDeleted); +BENCHMARK(BM_Group_MaskEmptyOrDeleted); void BM_Group_CountLeadingEmptyOrDeleted(benchmark::State& state) { std::array<ctrl_t, Group::kWidth> group; - std::iota(group.begin(), group.end(), -2); + Iota(group.begin(), group.end(), -2); Group g{group.data()}; - for (auto _ : state) + for (auto _ : state) { + ::benchmark::DoNotOptimize(g); ::benchmark::DoNotOptimize(g.CountLeadingEmptyOrDeleted()); + } } BENCHMARK(BM_Group_CountLeadingEmptyOrDeleted); void BM_Group_MatchFirstEmptyOrDeleted(benchmark::State& state) { std::array<ctrl_t, Group::kWidth> group; - std::iota(group.begin(), group.end(), -2); + Iota(group.begin(), group.end(), -2); Group g{group.data()}; - for (auto _ : state) ::benchmark::DoNotOptimize(*g.MatchEmptyOrDeleted()); + for (auto _ : state) { + ::benchmark::DoNotOptimize(g); + ::benchmark::DoNotOptimize(g.MaskEmptyOrDeleted().LowestBitSet()); + } } BENCHMARK(BM_Group_MatchFirstEmptyOrDeleted); void BM_DropDeletes(benchmark::State& state) { constexpr size_t capacity = (1 << 20) - 1; std::vector<ctrl_t> ctrl(capacity + 1 + Group::kWidth); - ctrl[capacity] = kSentinel; - std::vector<ctrl_t> pattern = {kEmpty, 2, kDeleted, 2, kEmpty, 1, kDeleted}; + ctrl[capacity] = ctrl_t::kSentinel; + std::vector<ctrl_t> pattern = {ctrl_t::kEmpty, static_cast<ctrl_t>(2), + ctrl_t::kDeleted, static_cast<ctrl_t>(2), + ctrl_t::kEmpty, static_cast<ctrl_t>(1), + ctrl_t::kDeleted}; for (size_t i = 0; i != capacity; ++i) { ctrl[i] = pattern[i % pattern.size()]; } @@ -378,6 +418,12 @@ bool CodegenAbslRawHashSetInt64FindNeEnd( return table->find(key) != table->end(); } +auto CodegenAbslRawHashSetInt64Insert(absl::container_internal::IntTable* table, + int64_t key) + -> decltype(table->insert(key)) { + return table->insert(key); +} + bool CodegenAbslRawHashSetInt64Contains( absl::container_internal::IntTable* table, int64_t key) { return table->contains(key); @@ -391,6 +437,7 @@ void CodegenAbslRawHashSetInt64Iterate( int odr = (::benchmark::DoNotOptimize(std::make_tuple( &CodegenAbslRawHashSetInt64Find, &CodegenAbslRawHashSetInt64FindNeEnd, + &CodegenAbslRawHashSetInt64Insert, &CodegenAbslRawHashSetInt64Contains, &CodegenAbslRawHashSetInt64Iterate)), 1); diff --git a/absl/container/internal/raw_hash_set_test.cc b/absl/container/internal/raw_hash_set_test.cc index 7dac65a0..f77ffbc1 100644 --- a/absl/container/internal/raw_hash_set_test.cc +++ b/absl/container/internal/raw_hash_set_test.cc @@ -31,6 +31,7 @@ #include "absl/base/attributes.h" #include "absl/base/config.h" #include "absl/base/internal/cycleclock.h" +#include "absl/base/internal/prefetch.h" #include "absl/base/internal/raw_logging.h" #include "absl/container/internal/container_memory.h" #include "absl/container/internal/hash_function_defaults.h" @@ -58,6 +59,9 @@ using ::testing::Lt; using ::testing::Pair; using ::testing::UnorderedElementsAre; +// Convenience function to static cast to ctrl_t. +ctrl_t CtrlT(int i) { return static_cast<ctrl_t>(i); } + TEST(Util, NormalizeCapacity) { EXPECT_EQ(1, NormalizeCapacity(0)); EXPECT_EQ(1, NormalizeCapacity(1)); @@ -170,15 +174,19 @@ TEST(Group, EmptyGroup) { TEST(Group, Match) { if (Group::kWidth == 16) { - ctrl_t group[] = {kEmpty, 1, kDeleted, 3, kEmpty, 5, kSentinel, 7, - 7, 5, 3, 1, 1, 1, 1, 1}; + ctrl_t group[] = {ctrl_t::kEmpty, CtrlT(1), ctrl_t::kDeleted, CtrlT(3), + ctrl_t::kEmpty, CtrlT(5), ctrl_t::kSentinel, CtrlT(7), + CtrlT(7), CtrlT(5), CtrlT(3), CtrlT(1), + CtrlT(1), CtrlT(1), CtrlT(1), CtrlT(1)}; EXPECT_THAT(Group{group}.Match(0), ElementsAre()); EXPECT_THAT(Group{group}.Match(1), ElementsAre(1, 11, 12, 13, 14, 15)); EXPECT_THAT(Group{group}.Match(3), ElementsAre(3, 10)); EXPECT_THAT(Group{group}.Match(5), ElementsAre(5, 9)); EXPECT_THAT(Group{group}.Match(7), ElementsAre(7, 8)); } else if (Group::kWidth == 8) { - ctrl_t group[] = {kEmpty, 1, 2, kDeleted, 2, 1, kSentinel, 1}; + ctrl_t group[] = {ctrl_t::kEmpty, CtrlT(1), CtrlT(2), + ctrl_t::kDeleted, CtrlT(2), CtrlT(1), + ctrl_t::kSentinel, CtrlT(1)}; EXPECT_THAT(Group{group}.Match(0), ElementsAre()); EXPECT_THAT(Group{group}.Match(1), ElementsAre(1, 5, 7)); EXPECT_THAT(Group{group}.Match(2), ElementsAre(2, 4)); @@ -187,27 +195,39 @@ TEST(Group, Match) { } } -TEST(Group, MatchEmpty) { +TEST(Group, MaskEmpty) { if (Group::kWidth == 16) { - ctrl_t group[] = {kEmpty, 1, kDeleted, 3, kEmpty, 5, kSentinel, 7, - 7, 5, 3, 1, 1, 1, 1, 1}; - EXPECT_THAT(Group{group}.MatchEmpty(), ElementsAre(0, 4)); + ctrl_t group[] = {ctrl_t::kEmpty, CtrlT(1), ctrl_t::kDeleted, CtrlT(3), + ctrl_t::kEmpty, CtrlT(5), ctrl_t::kSentinel, CtrlT(7), + CtrlT(7), CtrlT(5), CtrlT(3), CtrlT(1), + CtrlT(1), CtrlT(1), CtrlT(1), CtrlT(1)}; + EXPECT_THAT(Group{group}.MaskEmpty().LowestBitSet(), 0); + EXPECT_THAT(Group{group}.MaskEmpty().HighestBitSet(), 4); } else if (Group::kWidth == 8) { - ctrl_t group[] = {kEmpty, 1, 2, kDeleted, 2, 1, kSentinel, 1}; - EXPECT_THAT(Group{group}.MatchEmpty(), ElementsAre(0)); + ctrl_t group[] = {ctrl_t::kEmpty, CtrlT(1), CtrlT(2), + ctrl_t::kDeleted, CtrlT(2), CtrlT(1), + ctrl_t::kSentinel, CtrlT(1)}; + EXPECT_THAT(Group{group}.MaskEmpty().LowestBitSet(), 0); + EXPECT_THAT(Group{group}.MaskEmpty().HighestBitSet(), 0); } else { FAIL() << "No test coverage for Group::kWidth==" << Group::kWidth; } } -TEST(Group, MatchEmptyOrDeleted) { +TEST(Group, MaskEmptyOrDeleted) { if (Group::kWidth == 16) { - ctrl_t group[] = {kEmpty, 1, kDeleted, 3, kEmpty, 5, kSentinel, 7, - 7, 5, 3, 1, 1, 1, 1, 1}; - EXPECT_THAT(Group{group}.MatchEmptyOrDeleted(), ElementsAre(0, 2, 4)); + ctrl_t group[] = {ctrl_t::kEmpty, CtrlT(1), ctrl_t::kEmpty, CtrlT(3), + ctrl_t::kDeleted, CtrlT(5), ctrl_t::kSentinel, CtrlT(7), + CtrlT(7), CtrlT(5), CtrlT(3), CtrlT(1), + CtrlT(1), CtrlT(1), CtrlT(1), CtrlT(1)}; + EXPECT_THAT(Group{group}.MaskEmptyOrDeleted().LowestBitSet(), 0); + EXPECT_THAT(Group{group}.MaskEmptyOrDeleted().HighestBitSet(), 4); } else if (Group::kWidth == 8) { - ctrl_t group[] = {kEmpty, 1, 2, kDeleted, 2, 1, kSentinel, 1}; - EXPECT_THAT(Group{group}.MatchEmptyOrDeleted(), ElementsAre(0, 3)); + ctrl_t group[] = {ctrl_t::kEmpty, CtrlT(1), CtrlT(2), + ctrl_t::kDeleted, CtrlT(2), CtrlT(1), + ctrl_t::kSentinel, CtrlT(1)}; + EXPECT_THAT(Group{group}.MaskEmptyOrDeleted().LowestBitSet(), 0); + EXPECT_THAT(Group{group}.MaskEmptyOrDeleted().HighestBitSet(), 3); } else { FAIL() << "No test coverage for Group::kWidth==" << Group::kWidth; } @@ -217,28 +237,32 @@ TEST(Batch, DropDeletes) { constexpr size_t kCapacity = 63; constexpr size_t kGroupWidth = container_internal::Group::kWidth; std::vector<ctrl_t> ctrl(kCapacity + 1 + kGroupWidth); - ctrl[kCapacity] = kSentinel; - std::vector<ctrl_t> pattern = {kEmpty, 2, kDeleted, 2, kEmpty, 1, kDeleted}; + ctrl[kCapacity] = ctrl_t::kSentinel; + std::vector<ctrl_t> pattern = { + ctrl_t::kEmpty, CtrlT(2), ctrl_t::kDeleted, CtrlT(2), + ctrl_t::kEmpty, CtrlT(1), ctrl_t::kDeleted}; for (size_t i = 0; i != kCapacity; ++i) { ctrl[i] = pattern[i % pattern.size()]; if (i < kGroupWidth - 1) ctrl[i + kCapacity + 1] = pattern[i % pattern.size()]; } ConvertDeletedToEmptyAndFullToDeleted(ctrl.data(), kCapacity); - ASSERT_EQ(ctrl[kCapacity], kSentinel); - for (size_t i = 0; i < kCapacity + 1 + kGroupWidth; ++i) { + ASSERT_EQ(ctrl[kCapacity], ctrl_t::kSentinel); + for (size_t i = 0; i < kCapacity + kGroupWidth; ++i) { ctrl_t expected = pattern[i % (kCapacity + 1) % pattern.size()]; - if (i == kCapacity) expected = kSentinel; - if (expected == kDeleted) expected = kEmpty; - if (IsFull(expected)) expected = kDeleted; + if (i == kCapacity) expected = ctrl_t::kSentinel; + if (expected == ctrl_t::kDeleted) expected = ctrl_t::kEmpty; + if (IsFull(expected)) expected = ctrl_t::kDeleted; EXPECT_EQ(ctrl[i], expected) - << i << " " << int{pattern[i % pattern.size()]}; + << i << " " << static_cast<int>(pattern[i % pattern.size()]); } } TEST(Group, CountLeadingEmptyOrDeleted) { - const std::vector<ctrl_t> empty_examples = {kEmpty, kDeleted}; - const std::vector<ctrl_t> full_examples = {0, 1, 2, 3, 5, 9, 127, kSentinel}; + const std::vector<ctrl_t> empty_examples = {ctrl_t::kEmpty, ctrl_t::kDeleted}; + const std::vector<ctrl_t> full_examples = { + CtrlT(0), CtrlT(1), CtrlT(2), CtrlT(3), + CtrlT(5), CtrlT(9), CtrlT(127), ctrl_t::kSentinel}; for (ctrl_t empty : empty_examples) { std::vector<ctrl_t> e(Group::kWidth, empty); @@ -294,6 +318,7 @@ struct ValuePolicy { }; using IntPolicy = ValuePolicy<int64_t>; +using Uint8Policy = ValuePolicy<uint8_t>; class StringPolicy { template <class F, class K, class V, @@ -374,6 +399,13 @@ struct IntTable using Base::Base; }; +struct Uint8Table + : raw_hash_set<Uint8Policy, container_internal::hash_default_hash<uint8_t>, + std::equal_to<uint8_t>, std::allocator<uint8_t>> { + using Base = typename Uint8Table::raw_hash_set; + using Base::Base; +}; + template <typename T> struct CustomAlloc : std::allocator<T> { CustomAlloc() {} @@ -541,6 +573,37 @@ TEST(Table, InsertCollisionAndFindAfterDelete) { EXPECT_TRUE(t.empty()); } +TEST(Table, InsertWithinCapacity) { + IntTable t; + t.reserve(10); + const size_t original_capacity = t.capacity(); + const auto addr = [&](int i) { + return reinterpret_cast<uintptr_t>(&*t.find(i)); + }; + // Inserting an element does not change capacity. + t.insert(0); + EXPECT_THAT(t.capacity(), original_capacity); + const uintptr_t original_addr_0 = addr(0); + // Inserting another element does not rehash. + t.insert(1); + EXPECT_THAT(t.capacity(), original_capacity); + EXPECT_THAT(addr(0), original_addr_0); + // Inserting lots of duplicate elements does not rehash. + for (int i = 0; i < 100; ++i) { + t.insert(i % 10); + } + EXPECT_THAT(t.capacity(), original_capacity); + EXPECT_THAT(addr(0), original_addr_0); + // Inserting a range of duplicate elements does not rehash. + std::vector<int> dup_range; + for (int i = 0; i < 100; ++i) { + dup_range.push_back(i % 10); + } + t.insert(dup_range.begin(), dup_range.end()); + EXPECT_THAT(t.capacity(), original_capacity); + EXPECT_THAT(addr(0), original_addr_0); +} + TEST(Table, LazyEmplace) { StringTable t; bool called = false; @@ -588,28 +651,53 @@ TEST(Table, Contains2) { } int decompose_constructed; +int decompose_copy_constructed; +int decompose_copy_assigned; +int decompose_move_constructed; +int decompose_move_assigned; struct DecomposeType { - DecomposeType(int i) : i(i) { // NOLINT + DecomposeType(int i = 0) : i(i) { // NOLINT ++decompose_constructed; } explicit DecomposeType(const char* d) : DecomposeType(*d) {} + DecomposeType(const DecomposeType& other) : i(other.i) { + ++decompose_copy_constructed; + } + DecomposeType& operator=(const DecomposeType& other) { + ++decompose_copy_assigned; + i = other.i; + return *this; + } + DecomposeType(DecomposeType&& other) : i(other.i) { + ++decompose_move_constructed; + } + DecomposeType& operator=(DecomposeType&& other) { + ++decompose_move_assigned; + i = other.i; + return *this; + } + int i; }; struct DecomposeHash { using is_transparent = void; - size_t operator()(DecomposeType a) const { return a.i; } + size_t operator()(const DecomposeType& a) const { return a.i; } size_t operator()(int a) const { return a; } size_t operator()(const char* a) const { return *a; } }; struct DecomposeEq { using is_transparent = void; - bool operator()(DecomposeType a, DecomposeType b) const { return a.i == b.i; } - bool operator()(DecomposeType a, int b) const { return a.i == b; } - bool operator()(DecomposeType a, const char* b) const { return a.i == *b; } + bool operator()(const DecomposeType& a, const DecomposeType& b) const { + return a.i == b.i; + } + bool operator()(const DecomposeType& a, int b) const { return a.i == b; } + bool operator()(const DecomposeType& a, const char* b) const { + return a.i == *b; + } }; struct DecomposePolicy { @@ -619,9 +707,9 @@ struct DecomposePolicy { template <typename T> static void construct(void*, DecomposeType* slot, T&& v) { - *slot = DecomposeType(std::forward<T>(v)); + ::new (slot) DecomposeType(std::forward<T>(v)); } - static void destroy(void*, DecomposeType*) {} + static void destroy(void*, DecomposeType* slot) { slot->~DecomposeType(); } static DecomposeType& element(slot_type* slot) { return *slot; } template <class F, class T> @@ -636,8 +724,13 @@ void TestDecompose(bool construct_three) { const int one = 1; const char* three_p = "3"; const auto& three = three_p; + const int elem_vector_count = 256; + std::vector<DecomposeType> elem_vector(elem_vector_count, DecomposeType{0}); + std::iota(elem_vector.begin(), elem_vector.end(), 0); - raw_hash_set<DecomposePolicy, Hash, Eq, std::allocator<int>> set1; + using DecomposeSet = + raw_hash_set<DecomposePolicy, Hash, Eq, std::allocator<int>>; + DecomposeSet set1; decompose_constructed = 0; int expected_constructed = 0; @@ -695,20 +788,72 @@ void TestDecompose(bool construct_three) { expected_constructed += construct_three; EXPECT_EQ(expected_constructed, decompose_constructed); } + + decompose_copy_constructed = 0; + decompose_copy_assigned = 0; + decompose_move_constructed = 0; + decompose_move_assigned = 0; + int expected_copy_constructed = 0; + int expected_move_constructed = 0; + { // raw_hash_set(first, last) with random-access iterators + DecomposeSet set2(elem_vector.begin(), elem_vector.end()); + // Expect exactly one copy-constructor call for each element if no + // rehashing is done. + expected_copy_constructed += elem_vector_count; + EXPECT_EQ(expected_copy_constructed, decompose_copy_constructed); + EXPECT_EQ(expected_move_constructed, decompose_move_constructed); + EXPECT_EQ(0, decompose_move_assigned); + EXPECT_EQ(0, decompose_copy_assigned); + } + + { // raw_hash_set(first, last) with forward iterators + std::list<DecomposeType> elem_list(elem_vector.begin(), elem_vector.end()); + expected_copy_constructed = decompose_copy_constructed; + DecomposeSet set2(elem_list.begin(), elem_list.end()); + // Expect exactly N elements copied into set, expect at most 2*N elements + // moving internally for all resizing needed (for a growth factor of 2). + expected_copy_constructed += elem_vector_count; + EXPECT_EQ(expected_copy_constructed, decompose_copy_constructed); + expected_move_constructed += elem_vector_count; + EXPECT_LT(expected_move_constructed, decompose_move_constructed); + expected_move_constructed += elem_vector_count; + EXPECT_GE(expected_move_constructed, decompose_move_constructed); + EXPECT_EQ(0, decompose_move_assigned); + EXPECT_EQ(0, decompose_copy_assigned); + expected_copy_constructed = decompose_copy_constructed; + expected_move_constructed = decompose_move_constructed; + } + + { // insert(first, last) + DecomposeSet set2; + set2.insert(elem_vector.begin(), elem_vector.end()); + // Expect exactly N elements copied into set, expect at most 2*N elements + // moving internally for all resizing needed (for a growth factor of 2). + const int expected_new_elements = elem_vector_count; + const int expected_max_element_moves = 2 * elem_vector_count; + expected_copy_constructed += expected_new_elements; + EXPECT_EQ(expected_copy_constructed, decompose_copy_constructed); + expected_move_constructed += expected_max_element_moves; + EXPECT_GE(expected_move_constructed, decompose_move_constructed); + EXPECT_EQ(0, decompose_move_assigned); + EXPECT_EQ(0, decompose_copy_assigned); + expected_copy_constructed = decompose_copy_constructed; + expected_move_constructed = decompose_move_constructed; + } } TEST(Table, Decompose) { TestDecompose<DecomposeHash, DecomposeEq>(false); struct TransparentHashIntOverload { - size_t operator()(DecomposeType a) const { return a.i; } + size_t operator()(const DecomposeType& a) const { return a.i; } size_t operator()(int a) const { return a; } }; struct TransparentEqIntOverload { - bool operator()(DecomposeType a, DecomposeType b) const { + bool operator()(const DecomposeType& a, const DecomposeType& b) const { return a.i == b.i; } - bool operator()(DecomposeType a, int b) const { return a.i == b; } + bool operator()(const DecomposeType& a, int b) const { return a.i == b; } }; TestDecompose<TransparentHashIntOverload, DecomposeEq>(true); TestDecompose<TransparentHashIntOverload, TransparentEqIntOverload>(true); @@ -750,7 +895,7 @@ TEST(Table, RehashWithNoResize) { const size_t capacity = t.capacity(); // Remove elements from all groups except the first and the last one. - // All elements removed from full groups will be marked as kDeleted. + // All elements removed from full groups will be marked as ctrl_t::kDeleted. const size_t erase_begin = Group::kWidth / 2; const size_t erase_end = (t.size() / Group::kWidth - 1) * Group::kWidth; for (size_t i = erase_begin; i < erase_end; ++i) { @@ -1104,7 +1249,7 @@ ExpectedStats XorSeedExpectedStats() { case 16: if (kRandomizesInserts) { return {0.1, - 1.0, + 2.0, {{0.95, 0.1}}, {{0.95, 0}, {0.99, 1}, {0.999, 8}, {0.9999, 15}}}; } else { @@ -1118,6 +1263,7 @@ ExpectedStats XorSeedExpectedStats() { return {}; } +// TODO(b/80415403): Figure out why this test is so flaky, esp. on MSVC TEST(Table, DISABLED_EnsureNonQuadraticTopNXorSeedByProbeSeqLength) { ProbeStatsPerSize stats; std::vector<size_t> sizes = {Group::kWidth << 5, Group::kWidth << 10}; @@ -1190,17 +1336,17 @@ ExpectedStats LinearTransformExpectedStats() { {{0.95, 0.3}}, {{0.95, 0}, {0.99, 1}, {0.999, 8}, {0.9999, 15}}}; } else { - return {0.15, - 0.5, - {{0.95, 0.3}}, - {{0.95, 0}, {0.99, 3}, {0.999, 15}, {0.9999, 25}}}; + return {0.4, + 0.6, + {{0.95, 0.5}}, + {{0.95, 1}, {0.99, 14}, {0.999, 23}, {0.9999, 26}}}; } case 16: if (kRandomizesInserts) { return {0.1, 0.4, {{0.95, 0.3}}, - {{0.95, 0}, {0.99, 1}, {0.999, 8}, {0.9999, 15}}}; + {{0.95, 1}, {0.99, 2}, {0.999, 9}, {0.9999, 15}}}; } else { return {0.05, 0.2, @@ -1212,6 +1358,7 @@ ExpectedStats LinearTransformExpectedStats() { return {}; } +// TODO(b/80415403): Figure out why this test is so flaky. TEST(Table, DISABLED_EnsureNonQuadraticTopNLinearTransformByProbeSeqLength) { ProbeStatsPerSize stats; std::vector<size_t> sizes = {Group::kWidth << 5, Group::kWidth << 10}; @@ -1888,7 +2035,7 @@ TEST(TableDeathTest, EraseOfEndAsserts) { IntTable t; // Extra simple "regexp" as regexp support is highly varied across platforms. - constexpr char kDeathMsg[] = "Invalid operation on iterator"; + constexpr char kDeathMsg[] = "erase.. called on invalid iterator"; EXPECT_DEATH_IF_SUPPORTED(t.erase(t.end()), kDeathMsg); } @@ -1898,7 +2045,7 @@ TEST(RawHashSamplerTest, Sample) { SetHashtablezEnabled(true); SetHashtablezSampleParameter(100); - auto& sampler = HashtablezSampler::Global(); + auto& sampler = GlobalHashtablezSampler(); size_t start_size = 0; std::unordered_set<const HashtablezInfo*> preexisting_info; start_size += sampler.Iterate([&](const HashtablezInfo& info) { @@ -1909,16 +2056,33 @@ TEST(RawHashSamplerTest, Sample) { std::vector<IntTable> tables; for (int i = 0; i < 1000000; ++i) { tables.emplace_back(); + + const bool do_reserve = (i % 10 > 5); + const bool do_rehash = !do_reserve && (i % 10 > 0); + + if (do_reserve) { + // Don't reserve on all tables. + tables.back().reserve(10 * (i % 10)); + } + tables.back().insert(1); tables.back().insert(i % 5); + + if (do_rehash) { + // Rehash some other tables. + tables.back().rehash(10 * (i % 10)); + } } size_t end_size = 0; std::unordered_map<size_t, int> observed_checksums; + std::unordered_map<ssize_t, int> reservations; end_size += sampler.Iterate([&](const HashtablezInfo& info) { if (preexisting_info.count(&info) == 0) { observed_checksums[info.hashes_bitwise_xor.load( std::memory_order_relaxed)]++; + reservations[info.max_reserve.load(std::memory_order_relaxed)]++; } + EXPECT_EQ(info.inline_element_size, sizeof(int64_t)); ++end_size; }); @@ -1928,6 +2092,15 @@ TEST(RawHashSamplerTest, Sample) { for (const auto& [_, count] : observed_checksums) { EXPECT_NEAR((100 * count) / static_cast<double>(tables.size()), 0.2, 0.05); } + + EXPECT_EQ(reservations.size(), 10); + for (const auto& [reservation, count] : reservations) { + EXPECT_GE(reservation, 0); + EXPECT_LT(reservation, 100); + + EXPECT_NEAR((100 * count) / static_cast<double>(tables.size()), 0.1, 0.05) + << reservation; + } } #endif // ABSL_INTERNAL_HASHTABLEZ_SAMPLE @@ -1936,7 +2109,7 @@ TEST(RawHashSamplerTest, DoNotSampleCustomAllocators) { SetHashtablezEnabled(true); SetHashtablezSampleParameter(100); - auto& sampler = HashtablezSampler::Global(); + auto& sampler = GlobalHashtablezSampler(); size_t start_size = 0; start_size += sampler.Iterate([&](const HashtablezInfo&) { ++start_size; }); @@ -1978,6 +2151,36 @@ TEST(Sanitizer, PoisoningOnErase) { } #endif // ABSL_HAVE_ADDRESS_SANITIZER +TEST(Table, AlignOne) { + // We previously had a bug in which we were copying a control byte over the + // first slot when alignof(value_type) is 1. We test repeated + // insertions/erases and verify that the behavior is correct. + Uint8Table t; + std::unordered_set<uint8_t> verifier; // NOLINT + + // Do repeated insertions/erases from the table. + for (int64_t i = 0; i < 100000; ++i) { + SCOPED_TRACE(i); + const uint8_t u = (i * -i) & 0xFF; + auto it = t.find(u); + auto verifier_it = verifier.find(u); + if (it == t.end()) { + ASSERT_EQ(verifier_it, verifier.end()); + t.insert(u); + verifier.insert(u); + } else { + ASSERT_NE(verifier_it, verifier.end()); + t.erase(it); + verifier.erase(verifier_it); + } + } + + EXPECT_EQ(t.size(), verifier.size()); + for (uint8_t u : t) { + EXPECT_EQ(verifier.count(u), 1); + } +} + } // namespace } // namespace container_internal ABSL_NAMESPACE_END diff --git a/absl/container/internal/unordered_map_constructor_test.h b/absl/container/internal/unordered_map_constructor_test.h index 3f90ad7c..7e84dc25 100644 --- a/absl/container/internal/unordered_map_constructor_test.h +++ b/absl/container/internal/unordered_map_constructor_test.h @@ -179,7 +179,7 @@ TYPED_TEST_P(ConstructorTest, InputIteratorBucketHashEqualAlloc) { A alloc(0); std::vector<T> values; std::generate_n(std::back_inserter(values), 10, - hash_internal::Generator<T>()); + hash_internal::UniqueGenerator<T>()); TypeParam m(values.begin(), values.end(), 123, hasher, equal, alloc); EXPECT_EQ(m.hash_function(), hasher); EXPECT_EQ(m.key_eq(), equal); @@ -198,7 +198,7 @@ void InputIteratorBucketAllocTest(std::true_type) { A alloc(0); std::vector<T> values; std::generate_n(std::back_inserter(values), 10, - hash_internal::Generator<T>()); + hash_internal::UniqueGenerator<T>()); TypeParam m(values.begin(), values.end(), 123, alloc); EXPECT_EQ(m.get_allocator(), alloc); EXPECT_THAT(items(m), ::testing::UnorderedElementsAreArray(values)); @@ -221,7 +221,7 @@ void InputIteratorBucketHashAllocTest(std::true_type) { A alloc(0); std::vector<T> values; std::generate_n(std::back_inserter(values), 10, - hash_internal::Generator<T>()); + hash_internal::UniqueGenerator<T>()); TypeParam m(values.begin(), values.end(), 123, hasher, alloc); EXPECT_EQ(m.hash_function(), hasher); EXPECT_EQ(m.get_allocator(), alloc); @@ -241,8 +241,9 @@ TYPED_TEST_P(ConstructorTest, CopyConstructor) { H hasher; E equal; A alloc(0); + hash_internal::UniqueGenerator<T> gen; TypeParam m(123, hasher, equal, alloc); - for (size_t i = 0; i != 10; ++i) m.insert(hash_internal::Generator<T>()()); + for (size_t i = 0; i != 10; ++i) m.insert(gen()); TypeParam n(m); EXPECT_EQ(m.hash_function(), n.hash_function()); EXPECT_EQ(m.key_eq(), n.key_eq()); @@ -262,8 +263,9 @@ void CopyConstructorAllocTest(std::true_type) { H hasher; E equal; A alloc(0); + hash_internal::UniqueGenerator<T> gen; TypeParam m(123, hasher, equal, alloc); - for (size_t i = 0; i != 10; ++i) m.insert(hash_internal::Generator<T>()()); + for (size_t i = 0; i != 10; ++i) m.insert(gen()); TypeParam n(m, A(11)); EXPECT_EQ(m.hash_function(), n.hash_function()); EXPECT_EQ(m.key_eq(), n.key_eq()); @@ -285,8 +287,9 @@ TYPED_TEST_P(ConstructorTest, MoveConstructor) { H hasher; E equal; A alloc(0); + hash_internal::UniqueGenerator<T> gen; TypeParam m(123, hasher, equal, alloc); - for (size_t i = 0; i != 10; ++i) m.insert(hash_internal::Generator<T>()()); + for (size_t i = 0; i != 10; ++i) m.insert(gen()); TypeParam t(m); TypeParam n(std::move(t)); EXPECT_EQ(m.hash_function(), n.hash_function()); @@ -307,8 +310,9 @@ void MoveConstructorAllocTest(std::true_type) { H hasher; E equal; A alloc(0); + hash_internal::UniqueGenerator<T> gen; TypeParam m(123, hasher, equal, alloc); - for (size_t i = 0; i != 10; ++i) m.insert(hash_internal::Generator<T>()()); + for (size_t i = 0; i != 10; ++i) m.insert(gen()); TypeParam t(m); TypeParam n(std::move(t), A(1)); EXPECT_EQ(m.hash_function(), n.hash_function()); @@ -325,7 +329,7 @@ TYPED_TEST_P(ConstructorTest, MoveConstructorAlloc) { TYPED_TEST_P(ConstructorTest, InitializerListBucketHashEqualAlloc) { using T = hash_internal::GeneratedType<TypeParam>; - hash_internal::Generator<T> gen; + hash_internal::UniqueGenerator<T> gen; std::initializer_list<T> values = {gen(), gen(), gen(), gen(), gen()}; using H = typename TypeParam::hasher; using E = typename TypeParam::key_equal; @@ -348,7 +352,7 @@ template <typename TypeParam> void InitializerListBucketAllocTest(std::true_type) { using T = hash_internal::GeneratedType<TypeParam>; using A = typename TypeParam::allocator_type; - hash_internal::Generator<T> gen; + hash_internal::UniqueGenerator<T> gen; std::initializer_list<T> values = {gen(), gen(), gen(), gen(), gen()}; A alloc(0); TypeParam m(values, 123, alloc); @@ -371,7 +375,7 @@ void InitializerListBucketHashAllocTest(std::true_type) { using A = typename TypeParam::allocator_type; H hasher; A alloc(0); - hash_internal::Generator<T> gen; + hash_internal::UniqueGenerator<T> gen; std::initializer_list<T> values = {gen(), gen(), gen(), gen(), gen()}; TypeParam m(values, 123, hasher, alloc); EXPECT_EQ(m.hash_function(), hasher); @@ -392,7 +396,7 @@ TYPED_TEST_P(ConstructorTest, Assignment) { H hasher; E equal; A alloc(0); - hash_internal::Generator<T> gen; + hash_internal::UniqueGenerator<T> gen; TypeParam m({gen(), gen(), gen()}, 123, hasher, equal, alloc); TypeParam n; n = m; @@ -412,7 +416,7 @@ TYPED_TEST_P(ConstructorTest, MoveAssignment) { H hasher; E equal; A alloc(0); - hash_internal::Generator<T> gen; + hash_internal::UniqueGenerator<T> gen; TypeParam m({gen(), gen(), gen()}, 123, hasher, equal, alloc); TypeParam t(m); TypeParam n; @@ -424,7 +428,7 @@ TYPED_TEST_P(ConstructorTest, MoveAssignment) { TYPED_TEST_P(ConstructorTest, AssignmentFromInitializerList) { using T = hash_internal::GeneratedType<TypeParam>; - hash_internal::Generator<T> gen; + hash_internal::UniqueGenerator<T> gen; std::initializer_list<T> values = {gen(), gen(), gen(), gen(), gen()}; TypeParam m; m = values; @@ -433,7 +437,7 @@ TYPED_TEST_P(ConstructorTest, AssignmentFromInitializerList) { TYPED_TEST_P(ConstructorTest, AssignmentOverwritesExisting) { using T = hash_internal::GeneratedType<TypeParam>; - hash_internal::Generator<T> gen; + hash_internal::UniqueGenerator<T> gen; TypeParam m({gen(), gen(), gen()}); TypeParam n({gen()}); n = m; @@ -442,7 +446,7 @@ TYPED_TEST_P(ConstructorTest, AssignmentOverwritesExisting) { TYPED_TEST_P(ConstructorTest, MoveAssignmentOverwritesExisting) { using T = hash_internal::GeneratedType<TypeParam>; - hash_internal::Generator<T> gen; + hash_internal::UniqueGenerator<T> gen; TypeParam m({gen(), gen(), gen()}); TypeParam t(m); TypeParam n({gen()}); @@ -452,7 +456,7 @@ TYPED_TEST_P(ConstructorTest, MoveAssignmentOverwritesExisting) { TYPED_TEST_P(ConstructorTest, AssignmentFromInitializerListOverwritesExisting) { using T = hash_internal::GeneratedType<TypeParam>; - hash_internal::Generator<T> gen; + hash_internal::UniqueGenerator<T> gen; std::initializer_list<T> values = {gen(), gen(), gen(), gen(), gen()}; TypeParam m; m = values; @@ -461,7 +465,7 @@ TYPED_TEST_P(ConstructorTest, AssignmentFromInitializerListOverwritesExisting) { TYPED_TEST_P(ConstructorTest, AssignmentOnSelf) { using T = hash_internal::GeneratedType<TypeParam>; - hash_internal::Generator<T> gen; + hash_internal::UniqueGenerator<T> gen; std::initializer_list<T> values = {gen(), gen(), gen(), gen(), gen()}; TypeParam m(values); m = *&m; // Avoid -Wself-assign @@ -472,7 +476,7 @@ TYPED_TEST_P(ConstructorTest, AssignmentOnSelf) { // containers in unspecified state (and in practice in causes memory-leak // according to heap-checker!). -REGISTER_TYPED_TEST_CASE_P( +REGISTER_TYPED_TEST_SUITE_P( ConstructorTest, NoArgs, BucketCount, BucketCountHash, BucketCountHashEqual, BucketCountHashEqualAlloc, BucketCountAlloc, BucketCountHashAlloc, Alloc, InputIteratorBucketHashEqualAlloc, InputIteratorBucketAlloc, diff --git a/absl/container/internal/unordered_map_lookup_test.h b/absl/container/internal/unordered_map_lookup_test.h index e76421e5..3713cd9a 100644 --- a/absl/container/internal/unordered_map_lookup_test.h +++ b/absl/container/internal/unordered_map_lookup_test.h @@ -107,8 +107,8 @@ TYPED_TEST_P(LookupTest, EqualRange) { } } -REGISTER_TYPED_TEST_CASE_P(LookupTest, At, OperatorBracket, Count, Find, - EqualRange); +REGISTER_TYPED_TEST_SUITE_P(LookupTest, At, OperatorBracket, Count, Find, + EqualRange); } // namespace container_internal ABSL_NAMESPACE_END diff --git a/absl/container/internal/unordered_map_modifiers_test.h b/absl/container/internal/unordered_map_modifiers_test.h index 8c9ca779..4d9ab30f 100644 --- a/absl/container/internal/unordered_map_modifiers_test.h +++ b/absl/container/internal/unordered_map_modifiers_test.h @@ -81,6 +81,38 @@ TYPED_TEST_P(ModifiersTest, InsertRange) { ASSERT_THAT(items(m), ::testing::UnorderedElementsAreArray(values)); } +TYPED_TEST_P(ModifiersTest, InsertWithinCapacity) { + using T = hash_internal::GeneratedType<TypeParam>; + using V = typename TypeParam::mapped_type; + T val = hash_internal::Generator<T>()(); + TypeParam m; + m.reserve(10); + const size_t original_capacity = m.bucket_count(); + m.insert(val); + EXPECT_EQ(m.bucket_count(), original_capacity); + T val2 = {val.first, hash_internal::Generator<V>()()}; + m.insert(val2); + EXPECT_EQ(m.bucket_count(), original_capacity); +} + +TYPED_TEST_P(ModifiersTest, InsertRangeWithinCapacity) { +#if !defined(__GLIBCXX__) + using T = hash_internal::GeneratedType<TypeParam>; + std::vector<T> base_values; + std::generate_n(std::back_inserter(base_values), 10, + hash_internal::Generator<T>()); + std::vector<T> values; + while (values.size() != 100) { + std::copy_n(base_values.begin(), 10, std::back_inserter(values)); + } + TypeParam m; + m.reserve(10); + const size_t original_capacity = m.bucket_count(); + m.insert(values.begin(), values.end()); + EXPECT_EQ(m.bucket_count(), original_capacity); +#endif +} + TYPED_TEST_P(ModifiersTest, InsertOrAssign) { #ifdef UNORDERED_MAP_CXX17 using std::get; @@ -265,10 +297,12 @@ TYPED_TEST_P(ModifiersTest, Swap) { // TODO(alkis): Write tests for extract. // TODO(alkis): Write tests for merge. -REGISTER_TYPED_TEST_CASE_P(ModifiersTest, Clear, Insert, InsertHint, - InsertRange, InsertOrAssign, InsertOrAssignHint, - Emplace, EmplaceHint, TryEmplace, TryEmplaceHint, - Erase, EraseRange, EraseKey, Swap); +REGISTER_TYPED_TEST_SUITE_P(ModifiersTest, Clear, Insert, InsertHint, + InsertRange, InsertWithinCapacity, + InsertRangeWithinCapacity, InsertOrAssign, + InsertOrAssignHint, Emplace, EmplaceHint, + TryEmplace, TryEmplaceHint, Erase, EraseRange, + EraseKey, Swap); template <typename Type> struct is_unique_ptr : std::false_type {}; diff --git a/absl/container/internal/unordered_set_constructor_test.h b/absl/container/internal/unordered_set_constructor_test.h index 41165b05..af1116e6 100644 --- a/absl/container/internal/unordered_set_constructor_test.h +++ b/absl/container/internal/unordered_set_constructor_test.h @@ -478,7 +478,7 @@ TYPED_TEST_P(ConstructorTest, AssignmentOnSelf) { EXPECT_THAT(keys(m), ::testing::UnorderedElementsAreArray(values)); } -REGISTER_TYPED_TEST_CASE_P( +REGISTER_TYPED_TEST_SUITE_P( ConstructorTest, NoArgs, BucketCount, BucketCountHash, BucketCountHashEqual, BucketCountHashEqualAlloc, BucketCountAlloc, BucketCountHashAlloc, Alloc, InputIteratorBucketHashEqualAlloc, InputIteratorBucketAlloc, diff --git a/absl/container/internal/unordered_set_lookup_test.h b/absl/container/internal/unordered_set_lookup_test.h index 8f2f4b20..b35f766e 100644 --- a/absl/container/internal/unordered_set_lookup_test.h +++ b/absl/container/internal/unordered_set_lookup_test.h @@ -82,7 +82,7 @@ TYPED_TEST_P(LookupTest, EqualRange) { } } -REGISTER_TYPED_TEST_CASE_P(LookupTest, Count, Find, EqualRange); +REGISTER_TYPED_TEST_SUITE_P(LookupTest, Count, Find, EqualRange); } // namespace container_internal ABSL_NAMESPACE_END diff --git a/absl/container/internal/unordered_set_modifiers_test.h b/absl/container/internal/unordered_set_modifiers_test.h index 26be58d9..d8864bb2 100644 --- a/absl/container/internal/unordered_set_modifiers_test.h +++ b/absl/container/internal/unordered_set_modifiers_test.h @@ -74,6 +74,36 @@ TYPED_TEST_P(ModifiersTest, InsertRange) { ASSERT_THAT(keys(m), ::testing::UnorderedElementsAreArray(values)); } +TYPED_TEST_P(ModifiersTest, InsertWithinCapacity) { + using T = hash_internal::GeneratedType<TypeParam>; + T val = hash_internal::Generator<T>()(); + TypeParam m; + m.reserve(10); + const size_t original_capacity = m.bucket_count(); + m.insert(val); + EXPECT_EQ(m.bucket_count(), original_capacity); + m.insert(val); + EXPECT_EQ(m.bucket_count(), original_capacity); +} + +TYPED_TEST_P(ModifiersTest, InsertRangeWithinCapacity) { +#if !defined(__GLIBCXX__) + using T = hash_internal::GeneratedType<TypeParam>; + std::vector<T> base_values; + std::generate_n(std::back_inserter(base_values), 10, + hash_internal::Generator<T>()); + std::vector<T> values; + while (values.size() != 100) { + values.insert(values.end(), base_values.begin(), base_values.end()); + } + TypeParam m; + m.reserve(10); + const size_t original_capacity = m.bucket_count(); + m.insert(values.begin(), values.end()); + EXPECT_EQ(m.bucket_count(), original_capacity); +#endif +} + TYPED_TEST_P(ModifiersTest, Emplace) { using T = hash_internal::GeneratedType<TypeParam>; T val = hash_internal::Generator<T>()(); @@ -179,9 +209,10 @@ TYPED_TEST_P(ModifiersTest, Swap) { // TODO(alkis): Write tests for extract. // TODO(alkis): Write tests for merge. -REGISTER_TYPED_TEST_CASE_P(ModifiersTest, Clear, Insert, InsertHint, - InsertRange, Emplace, EmplaceHint, Erase, EraseRange, - EraseKey, Swap); +REGISTER_TYPED_TEST_SUITE_P(ModifiersTest, Clear, Insert, InsertHint, + InsertRange, InsertWithinCapacity, + InsertRangeWithinCapacity, Emplace, EmplaceHint, + Erase, EraseRange, EraseKey, Swap); } // namespace container_internal ABSL_NAMESPACE_END diff --git a/absl/container/node_hash_map.h b/absl/container/node_hash_map.h index 7a39f628..6868e63a 100644 --- a/absl/container/node_hash_map.h +++ b/absl/container/node_hash_map.h @@ -41,9 +41,10 @@ #include <utility> #include "absl/algorithm/container.h" +#include "absl/base/macros.h" #include "absl/container/internal/container_memory.h" #include "absl/container/internal/hash_function_defaults.h" // IWYU pragma: export -#include "absl/container/internal/node_hash_policy.h" +#include "absl/container/internal/node_slot_policy.h" #include "absl/container/internal/raw_hash_map.h" // IWYU pragma: export #include "absl/memory/memory.h" @@ -77,6 +78,10 @@ class NodeHashMapPolicy; // absl/hash/hash.h for information on extending Abseil hashing to user-defined // types. // +// Using `absl::node_hash_map` at interface boundaries in dynamically loaded +// libraries (e.g. .dll, .so) is unsupported due to way `absl::Hash` values may +// be randomized across dynamically loaded libraries. +// // Example: // // // Create a node hash map of three strings (that map to strings) @@ -347,8 +352,8 @@ class node_hash_map // `node_hash_map`. // // iterator try_emplace(const_iterator hint, - // const init_type& k, Args&&... args): - // iterator try_emplace(const_iterator hint, init_type&& k, Args&&... args): + // const key_type& k, Args&&... args): + // iterator try_emplace(const_iterator hint, key_type&& k, Args&&... args): // // Inserts (via copy or move) the element of the specified key into the // `node_hash_map` using the position of `hint` as a non-binding suggestion @@ -525,17 +530,19 @@ class node_hash_map // erase_if(node_hash_map<>, Pred) // // Erases all elements that satisfy the predicate `pred` from the container `c`. +// Returns the number of erased elements. template <typename K, typename V, typename H, typename E, typename A, typename Predicate> -void erase_if(node_hash_map<K, V, H, E, A>& c, Predicate pred) { - container_internal::EraseIf(pred, &c); +typename node_hash_map<K, V, H, E, A>::size_type erase_if( + node_hash_map<K, V, H, E, A>& c, Predicate pred) { + return container_internal::EraseIf(pred, &c); } namespace container_internal { template <class Key, class Value> class NodeHashMapPolicy - : public absl::container_internal::node_hash_policy< + : public absl::container_internal::node_slot_policy< std::pair<const Key, Value>&, NodeHashMapPolicy<Key, Value>> { using value_type = std::pair<const Key, Value>; diff --git a/absl/container/node_hash_map_test.cc b/absl/container/node_hash_map_test.cc index 8f59a1e4..e941a836 100644 --- a/absl/container/node_hash_map_test.cc +++ b/absl/container/node_hash_map_test.cc @@ -223,33 +223,36 @@ TEST(NodeHashMap, EraseIf) { // Erase all elements. { node_hash_map<int, int> s = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}}; - erase_if(s, [](std::pair<const int, int>) { return true; }); + EXPECT_EQ(erase_if(s, [](std::pair<const int, int>) { return true; }), 5); EXPECT_THAT(s, IsEmpty()); } // Erase no elements. { node_hash_map<int, int> s = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}}; - erase_if(s, [](std::pair<const int, int>) { return false; }); + EXPECT_EQ(erase_if(s, [](std::pair<const int, int>) { return false; }), 0); EXPECT_THAT(s, UnorderedElementsAre(Pair(1, 1), Pair(2, 2), Pair(3, 3), Pair(4, 4), Pair(5, 5))); } // Erase specific elements. { node_hash_map<int, int> s = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}}; - erase_if(s, - [](std::pair<const int, int> kvp) { return kvp.first % 2 == 1; }); + EXPECT_EQ(erase_if(s, + [](std::pair<const int, int> kvp) { + return kvp.first % 2 == 1; + }), + 3); EXPECT_THAT(s, UnorderedElementsAre(Pair(2, 2), Pair(4, 4))); } // Predicate is function reference. { node_hash_map<int, int> s = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}}; - erase_if(s, FirstIsEven); + EXPECT_EQ(erase_if(s, FirstIsEven), 2); EXPECT_THAT(s, UnorderedElementsAre(Pair(1, 1), Pair(3, 3), Pair(5, 5))); } // Predicate is function pointer. { node_hash_map<int, int> s = {{1, 1}, {2, 2}, {3, 3}, {4, 4}, {5, 5}}; - erase_if(s, &FirstIsEven); + EXPECT_EQ(erase_if(s, &FirstIsEven), 2); EXPECT_THAT(s, UnorderedElementsAre(Pair(1, 1), Pair(3, 3), Pair(5, 5))); } } diff --git a/absl/container/node_hash_set.h b/absl/container/node_hash_set.h index 93b15f46..f2cc70c3 100644 --- a/absl/container/node_hash_set.h +++ b/absl/container/node_hash_set.h @@ -38,8 +38,9 @@ #include <type_traits> #include "absl/algorithm/container.h" +#include "absl/base/macros.h" #include "absl/container/internal/hash_function_defaults.h" // IWYU pragma: export -#include "absl/container/internal/node_hash_policy.h" +#include "absl/container/internal/node_slot_policy.h" #include "absl/container/internal/raw_hash_set.h" // IWYU pragma: export #include "absl/memory/memory.h" @@ -73,6 +74,10 @@ struct NodeHashSetPolicy; // absl/hash/hash.h for information on extending Abseil hashing to user-defined // types. // +// Using `absl::node_hash_set` at interface boundaries in dynamically loaded +// libraries (e.g. .dll, .so) is unsupported due to way `absl::Hash` values may +// be randomized across dynamically loaded libraries. +// // Example: // // // Create a node hash set of three strings @@ -433,16 +438,18 @@ class node_hash_set // erase_if(node_hash_set<>, Pred) // // Erases all elements that satisfy the predicate `pred` from the container `c`. +// Returns the number of erased elements. template <typename T, typename H, typename E, typename A, typename Predicate> -void erase_if(node_hash_set<T, H, E, A>& c, Predicate pred) { - container_internal::EraseIf(pred, &c); +typename node_hash_set<T, H, E, A>::size_type erase_if( + node_hash_set<T, H, E, A>& c, Predicate pred) { + return container_internal::EraseIf(pred, &c); } namespace container_internal { template <class T> struct NodeHashSetPolicy - : absl::container_internal::node_hash_policy<T&, NodeHashSetPolicy<T>> { + : absl::container_internal::node_slot_policy<T&, NodeHashSetPolicy<T>> { using key_type = T; using init_type = T; using constant_iterators = std::true_type; diff --git a/absl/container/node_hash_set_test.cc b/absl/container/node_hash_set_test.cc index 7ddad202..98a8dbdd 100644 --- a/absl/container/node_hash_set_test.cc +++ b/absl/container/node_hash_set_test.cc @@ -108,31 +108,31 @@ TEST(NodeHashSet, EraseIf) { // Erase all elements. { node_hash_set<int> s = {1, 2, 3, 4, 5}; - erase_if(s, [](int) { return true; }); + EXPECT_EQ(erase_if(s, [](int) { return true; }), 5); EXPECT_THAT(s, IsEmpty()); } // Erase no elements. { node_hash_set<int> s = {1, 2, 3, 4, 5}; - erase_if(s, [](int) { return false; }); + EXPECT_EQ(erase_if(s, [](int) { return false; }), 0); EXPECT_THAT(s, UnorderedElementsAre(1, 2, 3, 4, 5)); } // Erase specific elements. { node_hash_set<int> s = {1, 2, 3, 4, 5}; - erase_if(s, [](int k) { return k % 2 == 1; }); + EXPECT_EQ(erase_if(s, [](int k) { return k % 2 == 1; }), 3); EXPECT_THAT(s, UnorderedElementsAre(2, 4)); } // Predicate is function reference. { node_hash_set<int> s = {1, 2, 3, 4, 5}; - erase_if(s, IsEven); + EXPECT_EQ(erase_if(s, IsEven), 2); EXPECT_THAT(s, UnorderedElementsAre(1, 3, 5)); } // Predicate is function pointer. { node_hash_set<int> s = {1, 2, 3, 4, 5}; - erase_if(s, &IsEven); + EXPECT_EQ(erase_if(s, &IsEven), 2); EXPECT_THAT(s, UnorderedElementsAre(1, 3, 5)); } } diff --git a/absl/container/sample_element_size_test.cc b/absl/container/sample_element_size_test.cc new file mode 100644 index 00000000..b23626b4 --- /dev/null +++ b/absl/container/sample_element_size_test.cc @@ -0,0 +1,114 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/container/flat_hash_map.h" +#include "absl/container/flat_hash_set.h" +#include "absl/container/node_hash_map.h" +#include "absl/container/node_hash_set.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace container_internal { +namespace { + +#if defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE) +// Create some tables of type `Table`, then look at all the new +// `HashtablezInfo`s to make sure that the `inline_element_size == +// expected_element_size`. The `inline_element_size` is the amount of memory +// allocated for each slot of a hash table, that is `sizeof(slot_type)`. Add +// the new `HashtablezInfo`s to `preexisting_info`. Store all the new tables +// into `tables`. +template <class Table> +void TestInlineElementSize( + HashtablezSampler& sampler, + // clang-tidy gives a false positive on this declaration. This unordered + // set cannot be flat_hash_set, however, since that would introduce a mutex + // deadlock. + std::unordered_set<const HashtablezInfo*>& preexisting_info, // NOLINT + std::vector<Table>& tables, const typename Table::value_type& elt, + size_t expected_element_size) { + for (int i = 0; i < 10; ++i) { + // We create a new table and must store it somewhere so that when we store + // a pointer to the resulting `HashtablezInfo` into `preexisting_info` + // that we aren't storing a dangling pointer. + tables.emplace_back(); + // We must insert an element to get a hashtablez to instantiate. + tables.back().insert(elt); + } + size_t new_count = 0; + sampler.Iterate([&](const HashtablezInfo& info) { + if (preexisting_info.insert(&info).second) { + EXPECT_EQ(info.inline_element_size, expected_element_size); + ++new_count; + } + }); + // Make sure we actually did get a new hashtablez. + EXPECT_GT(new_count, 0); +} + +struct bigstruct { + char a[1000]; + friend bool operator==(const bigstruct& x, const bigstruct& y) { + return memcmp(x.a, y.a, sizeof(x.a)) == 0; + } + template <typename H> + friend H AbslHashValue(H h, const bigstruct& c) { + return H::combine_contiguous(std::move(h), c.a, sizeof(c.a)); + } +}; +#endif + +TEST(FlatHashMap, SampleElementSize) { +#if defined(ABSL_INTERNAL_HASHTABLEZ_SAMPLE) + // Enable sampling even if the prod default is off. + SetHashtablezEnabled(true); + SetHashtablezSampleParameter(1); + + auto& sampler = GlobalHashtablezSampler(); + std::vector<flat_hash_map<int, bigstruct>> flat_map_tables; + std::vector<flat_hash_set<bigstruct>> flat_set_tables; + std::vector<node_hash_map<int, bigstruct>> node_map_tables; + std::vector<node_hash_set<bigstruct>> node_set_tables; + + // It takes thousands of new tables after changing the sampling parameters + // before you actually get some instrumentation. And if you must actually + // put something into those tables. + for (int i = 0; i < 10000; ++i) { + flat_map_tables.emplace_back(); + flat_map_tables.back()[i] = bigstruct{}; + } + + // clang-tidy gives a false positive on this declaration. This unordered set + // cannot be a flat_hash_set, however, since that would introduce a mutex + // deadlock. + std::unordered_set<const HashtablezInfo*> preexisting_info; // NOLINT + sampler.Iterate( + [&](const HashtablezInfo& info) { preexisting_info.insert(&info); }); + TestInlineElementSize(sampler, preexisting_info, flat_map_tables, + {0, bigstruct{}}, sizeof(int) + sizeof(bigstruct)); + TestInlineElementSize(sampler, preexisting_info, node_map_tables, + {0, bigstruct{}}, sizeof(void*)); + TestInlineElementSize(sampler, preexisting_info, flat_set_tables, // + bigstruct{}, sizeof(bigstruct)); + TestInlineElementSize(sampler, preexisting_info, node_set_tables, // + bigstruct{}, sizeof(void*)); +#endif +} + +} // namespace +} // namespace container_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/absl/copts/AbseilConfigureCopts.cmake b/absl/copts/AbseilConfigureCopts.cmake index 942ce90a..73435e99 100644 --- a/absl/copts/AbseilConfigureCopts.cmake +++ b/absl/copts/AbseilConfigureCopts.cmake @@ -1,8 +1,6 @@ # See absl/copts/copts.py and absl/copts/generate_copts.py include(GENERATED_AbseilCopts) -set(ABSL_LSAN_LINKOPTS "") -set(ABSL_HAVE_LSAN OFF) set(ABSL_DEFAULT_LINKOPTS "") if (BUILD_SHARED_LIBS AND MSVC) @@ -12,7 +10,49 @@ else() set(ABSL_BUILD_DLL FALSE) endif() -if(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|amd64|AMD64") +if(APPLE AND CMAKE_CXX_COMPILER_ID MATCHES [[Clang]]) + # Some CMake targets (not known at the moment of processing) could be set to + # compile for multiple architectures as specified by the OSX_ARCHITECTURES + # property, which is target-specific. We should neither inspect nor rely on + # any CMake property or variable to detect an architecture, in particular: + # + # - CMAKE_OSX_ARCHITECTURES + # is just an initial value for OSX_ARCHITECTURES; set too early. + # + # - OSX_ARCHITECTURES + # is a per-target property; targets could be defined later, and their + # properties could be modified any time later. + # + # - CMAKE_SYSTEM_PROCESSOR + # does not reflect multiple architectures at all. + # + # When compiling for multiple architectures, a build system can invoke a + # compiler either + # + # - once: a single command line for multiple architectures (Ninja build) + # - twice: two command lines per each architecture (Xcode build system) + # + # If case of Xcode, it would be possible to set an Xcode-specific attributes + # like XCODE_ATTRIBUTE_OTHER_CPLUSPLUSFLAGS[arch=arm64] or similar. + # + # In both cases, the viable strategy is to pass all arguments at once, allowing + # the compiler to dispatch arch-specific arguments to a designated backend. + set(ABSL_RANDOM_RANDEN_COPTS "") + foreach(_arch IN ITEMS "x86_64" "arm64") + string(TOUPPER "${_arch}" _arch_uppercase) + string(REPLACE "X86_64" "X64" _arch_uppercase ${_arch_uppercase}) + foreach(_flag IN LISTS ABSL_RANDOM_HWAES_${_arch_uppercase}_FLAGS) + list(APPEND ABSL_RANDOM_RANDEN_COPTS "-Xarch_${_arch}" "${_flag}") + endforeach() + endforeach() + # If a compiler happens to deal with an argument for a currently unused + # architecture, it will warn about an unused command line argument. + option(ABSL_RANDOM_RANDEN_COPTS_WARNING OFF + "Warn if one of ABSL_RANDOM_RANDEN_COPTS is unused") + if(ABSL_RANDOM_RANDEN_COPTS AND NOT ABSL_RANDOM_RANDEN_COPTS_WARNING) + list(APPEND ABSL_RANDOM_RANDEN_COPTS "-Wno-unused-command-line-argument") + endif() +elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "x86_64|amd64|AMD64") if (MSVC) set(ABSL_RANDOM_RANDEN_COPTS "${ABSL_RANDOM_HWAES_MSVC_X64_FLAGS}") else() @@ -43,14 +83,6 @@ elseif(CMAKE_CXX_COMPILER_ID MATCHES "Clang") # MATCHES so we get both Clang an else() set(ABSL_DEFAULT_COPTS "${ABSL_LLVM_FLAGS}") set(ABSL_TEST_COPTS "${ABSL_LLVM_FLAGS};${ABSL_LLVM_TEST_FLAGS}") - if(CMAKE_CXX_COMPILER_ID STREQUAL "Clang") - # AppleClang doesn't have lsan - # https://developer.apple.com/documentation/code_diagnostics - if(NOT CMAKE_CXX_COMPILER_VERSION VERSION_LESS 3.5) - set(ABSL_LSAN_LINKOPTS "-fsanitize=leak") - set(ABSL_HAVE_LSAN ON) - endif() - endif() endif() elseif(CMAKE_CXX_COMPILER_ID STREQUAL "MSVC") set(ABSL_DEFAULT_COPTS "${ABSL_MSVC_FLAGS}") diff --git a/absl/copts/GENERATED_AbseilCopts.cmake b/absl/copts/GENERATED_AbseilCopts.cmake index 51742c9b..a4ab1aa2 100644 --- a/absl/copts/GENERATED_AbseilCopts.cmake +++ b/absl/copts/GENERATED_AbseilCopts.cmake @@ -71,12 +71,13 @@ list(APPEND ABSL_LLVM_FLAGS "-Wformat-security" "-Wgnu-redeclared-enum" "-Winfinite-recursion" + "-Winvalid-constexpr" "-Wliteral-conversion" "-Wmissing-declarations" "-Woverlength-strings" "-Wpointer-arith" "-Wself-assign" - "-Wshadow" + "-Wshadow-all" "-Wstring-conversion" "-Wtautological-overlap-compare" "-Wundef" @@ -93,6 +94,7 @@ list(APPEND ABSL_LLVM_FLAGS "-Wno-implicit-int-conversion" "-Wno-shorten-64-to-32" "-Wno-sign-conversion" + "-Wno-unknown-warning-option" "-DNOMINMAX" ) diff --git a/absl/copts/GENERATED_copts.bzl b/absl/copts/GENERATED_copts.bzl index 6707488f..a6efc98e 100644 --- a/absl/copts/GENERATED_copts.bzl +++ b/absl/copts/GENERATED_copts.bzl @@ -72,12 +72,13 @@ ABSL_LLVM_FLAGS = [ "-Wformat-security", "-Wgnu-redeclared-enum", "-Winfinite-recursion", + "-Winvalid-constexpr", "-Wliteral-conversion", "-Wmissing-declarations", "-Woverlength-strings", "-Wpointer-arith", "-Wself-assign", - "-Wshadow", + "-Wshadow-all", "-Wstring-conversion", "-Wtautological-overlap-compare", "-Wundef", @@ -94,6 +95,7 @@ ABSL_LLVM_FLAGS = [ "-Wno-implicit-int-conversion", "-Wno-shorten-64-to-32", "-Wno-sign-conversion", + "-Wno-unknown-warning-option", "-DNOMINMAX", ] diff --git a/absl/copts/configure_copts.bzl b/absl/copts/configure_copts.bzl index 669a9060..40d5849a 100644 --- a/absl/copts/configure_copts.bzl +++ b/absl/copts/configure_copts.bzl @@ -50,6 +50,7 @@ ABSL_RANDOM_RANDEN_COPTS = select({ ":cpu_x64_windows": ABSL_RANDOM_HWAES_MSVC_X64_FLAGS, ":cpu_k8": ABSL_RANDOM_HWAES_X64_FLAGS, ":cpu_ppc": ["-mcrypto"], + ":cpu_aarch64": ABSL_RANDOM_HWAES_ARM64_FLAGS, # Supported by default or unsupported. "//conditions:default": [], @@ -70,6 +71,7 @@ def absl_random_randen_copts_init(): "darwin", "x64_windows_msvc", "x64_windows", + "aarch64", ] for cpu in cpu_configs: native.config_setting( diff --git a/absl/copts/copts.py b/absl/copts/copts.py index cf52981c..0d6c1ec3 100644 --- a/absl/copts/copts.py +++ b/absl/copts/copts.py @@ -87,12 +87,13 @@ COPT_VARS = { "-Wformat-security", "-Wgnu-redeclared-enum", "-Winfinite-recursion", + "-Winvalid-constexpr", "-Wliteral-conversion", "-Wmissing-declarations", "-Woverlength-strings", "-Wpointer-arith", "-Wself-assign", - "-Wshadow", + "-Wshadow-all", "-Wstring-conversion", "-Wtautological-overlap-compare", "-Wundef", @@ -111,6 +112,9 @@ COPT_VARS = { "-Wno-implicit-int-conversion", "-Wno-shorten-64-to-32", "-Wno-sign-conversion", + # Disable warnings on unknown warning flags (when warning flags are + # unknown on older compiler versions) + "-Wno-unknown-warning-option", # Don't define min and max macros (Build on Windows using clang) "-DNOMINMAX", ], diff --git a/absl/copts/generate_copts.py b/absl/copts/generate_copts.py index 0e5dc9fa..34be2fc2 100755 --- a/absl/copts/generate_copts.py +++ b/absl/copts/generate_copts.py @@ -1,4 +1,4 @@ -#!/usr/bin/python +#!/usr/bin/env python3 """Generate Abseil compile compile option configs. Usage: <path_to_absl>/copts/generate_copts.py diff --git a/absl/debugging/BUILD.bazel b/absl/debugging/BUILD.bazel index 5385bcb6..932a8e9f 100644 --- a/absl/debugging/BUILD.bazel +++ b/absl/debugging/BUILD.bazel @@ -14,7 +14,6 @@ # limitations under the License. # -load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") load( "//absl:copts/configure_copts.bzl", "ABSL_DEFAULT_COPTS", @@ -34,8 +33,10 @@ cc_library( "internal/stacktrace_aarch64-inl.inc", "internal/stacktrace_arm-inl.inc", "internal/stacktrace_config.h", + "internal/stacktrace_emscripten-inl.inc", "internal/stacktrace_generic-inl.inc", "internal/stacktrace_powerpc-inl.inc", + "internal/stacktrace_riscv-inl.inc", "internal/stacktrace_unimplemented-inl.inc", "internal/stacktrace_win32-inl.inc", "internal/stacktrace_x86-inl.inc", @@ -57,6 +58,7 @@ cc_library( "symbolize.cc", "symbolize_darwin.inc", "symbolize_elf.inc", + "symbolize_emscripten.inc", "symbolize_unimplemented.inc", "symbolize_win32.inc", ], @@ -141,7 +143,6 @@ cc_library( "//absl/base", "//absl/base:config", "//absl/base:core_headers", - "//absl/base:errno_saver", "//absl/base:raw_logging_internal", ], ) @@ -180,6 +181,7 @@ cc_library( ], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:private"], deps = [ "//absl/base:config", "//absl/base:core_headers", @@ -194,6 +196,8 @@ cc_library( srcs = ["internal/demangle.cc"], hdrs = ["internal/demangle.h"], copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:private"], deps = [ "//absl/base", "//absl/base:config", @@ -221,6 +225,7 @@ cc_library( name = "leak_check", srcs = ["leak_check.cc"], hdrs = ["leak_check.h"], + copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, deps = [ "//absl/base:config", @@ -228,98 +233,33 @@ cc_library( ], ) -# Adding a dependency to leak_check_disable will disable -# sanitizer leak checking (asan/lsan) in a test without -# the need to mess around with build features. -cc_library( - name = "leak_check_disable", - srcs = ["leak_check_disable.cc"], - linkopts = ABSL_DEFAULT_LINKOPTS, - linkstatic = 1, - deps = ["//absl/base:config"], - alwayslink = 1, -) - -# These targets exists for use in tests only, explicitly configuring the -# LEAK_SANITIZER macro. It must be linked with -fsanitize=leak for lsan. -ABSL_LSAN_LINKOPTS = select({ - "//absl:clang_compiler": ["-fsanitize=leak"], - "//conditions:default": [], -}) - -cc_library( - name = "leak_check_api_enabled_for_testing", - testonly = 1, - srcs = ["leak_check.cc"], - hdrs = ["leak_check.h"], - copts = select({ - "//absl:clang_compiler": ["-DLEAK_SANITIZER"], - "//conditions:default": [], - }), - linkopts = ABSL_DEFAULT_LINKOPTS, - visibility = ["//visibility:private"], - deps = [ - "//absl/base:config", - "//absl/base:core_headers", - ], -) - -cc_library( - name = "leak_check_api_disabled_for_testing", - testonly = 1, - srcs = ["leak_check.cc"], - hdrs = ["leak_check.h"], - copts = ["-ULEAK_SANITIZER"], - linkopts = ABSL_DEFAULT_LINKOPTS, - visibility = ["//visibility:private"], - deps = [ - "//absl/base:config", - "//absl/base:core_headers", - ], -) - cc_test( name = "leak_check_test", srcs = ["leak_check_test.cc"], - copts = select({ - "//absl:clang_compiler": ["-DABSL_EXPECT_LEAK_SANITIZER"], - "//conditions:default": [], - }), - linkopts = ABSL_LSAN_LINKOPTS + ABSL_DEFAULT_LINKOPTS, - tags = ["notsan"], - deps = [ - ":leak_check_api_enabled_for_testing", - "//absl/base", - "@com_google_googletest//:gtest_main", - ], -) - -cc_test( - name = "leak_check_no_lsan_test", - srcs = ["leak_check_test.cc"], - copts = ["-UABSL_EXPECT_LEAK_SANITIZER"], + copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, - tags = ["noasan"], + tags = ["notsan"], deps = [ - ":leak_check_api_disabled_for_testing", - "//absl/base", # for raw_logging + ":leak_check", + "//absl/base:config", + "//absl/base:raw_logging_internal", "@com_google_googletest//:gtest_main", ], ) -# Test that leak checking is skipped when lsan is enabled but -# ":leak_check_disable" is linked in. -# -# This test should fail in the absence of a dependency on ":leak_check_disable" -cc_test( - name = "disabled_leak_check_test", +# Binary that leaks memory and expects to fail on exit. This isn't a +# test that expected to pass on its own; it exists to be called by a +# script that checks exit status and output. +# TODO(absl-team): Write a test to run this with a script that +# verifies that it correctly fails. +cc_binary( + name = "leak_check_fail_test_binary", srcs = ["leak_check_fail_test.cc"], - linkopts = ABSL_LSAN_LINKOPTS + ABSL_DEFAULT_LINKOPTS, - tags = ["notsan"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, deps = [ - ":leak_check_api_enabled_for_testing", - ":leak_check_disable", - "//absl/base", + ":leak_check", + "//absl/base:raw_logging_internal", "@com_google_googletest//:gtest_main", ], ) @@ -344,6 +284,7 @@ cc_test( srcs = ["internal/stack_consumption_test.cc"], copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + tags = ["notsan"], deps = [ ":stack_consumption", "//absl/base:core_headers", @@ -351,3 +292,18 @@ cc_test( "@com_google_googletest//:gtest_main", ], ) + +cc_binary( + name = "stacktrace_benchmark", + testonly = 1, + srcs = ["stacktrace_benchmark.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + tags = ["benchmark"], + deps = [ + ":stacktrace", + "//absl/base:config", + "//absl/base:core_headers", + "@com_github_google_benchmark//:benchmark_main", + ], +) diff --git a/absl/debugging/CMakeLists.txt b/absl/debugging/CMakeLists.txt index 074b44cf..d8207d6a 100644 --- a/absl/debugging/CMakeLists.txt +++ b/absl/debugging/CMakeLists.txt @@ -14,6 +14,8 @@ # limitations under the License. # +find_library(EXECINFO_LIBRARY execinfo) + absl_cc_library( NAME stacktrace @@ -22,8 +24,10 @@ absl_cc_library( "internal/stacktrace_aarch64-inl.inc" "internal/stacktrace_arm-inl.inc" "internal/stacktrace_config.h" + "internal/stacktrace_emscripten-inl.inc" "internal/stacktrace_generic-inl.inc" "internal/stacktrace_powerpc-inl.inc" + "internal/stacktrace_riscv-inl.inc" "internal/stacktrace_unimplemented-inl.inc" "internal/stacktrace_win32-inl.inc" "internal/stacktrace_x86-inl.inc" @@ -31,6 +35,8 @@ absl_cc_library( "stacktrace.cc" COPTS ${ABSL_DEFAULT_COPTS} + LINKOPTS + $<$<BOOL:${EXECINFO_LIBRARY}>:${EXECINFO_LIBRARY}> DEPS absl::debugging_internal absl::config @@ -48,6 +54,7 @@ absl_cc_library( "symbolize.cc" "symbolize_darwin.inc" "symbolize_elf.inc" + "symbolize_emscripten.inc" "symbolize_unimplemented.inc" "symbolize_win32.inc" COPTS @@ -87,9 +94,10 @@ absl_cc_test( absl::memory absl::raw_logging_internal absl::strings - gmock + GTest::gmock ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME examine_stack @@ -122,7 +130,6 @@ absl_cc_library( absl::base absl::config absl::core_headers - absl::errno_saver absl::raw_logging_internal PUBLIC ) @@ -141,9 +148,10 @@ absl_cc_test( absl::strings absl::raw_logging_internal Threads::Threads - gmock + GTest::gmock ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME debugging_internal @@ -165,6 +173,7 @@ absl_cc_library( absl::raw_logging_internal ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME demangle_internal @@ -194,7 +203,7 @@ absl_cc_test( absl::core_headers absl::memory absl::raw_logging_internal - gmock_main + GTest::gmock_main ) absl_cc_library( @@ -212,42 +221,6 @@ absl_cc_library( PUBLIC ) -absl_cc_library( - NAME - leak_check_disable - SRCS - "leak_check_disable.cc" - COPTS - ${ABSL_DEFAULT_COPTS} - PUBLIC -) - -absl_cc_library( - NAME - leak_check_api_enabled_for_testing - HDRS - "leak_check.h" - SRCS - "leak_check.cc" - COPTS - ${ABSL_DEFAULT_COPTS} - $<$<BOOL:${ABSL_HAVE_LSAN}>:-DLEAK_SANITIZER> - TESTONLY -) - -absl_cc_library( - NAME - leak_check_api_disabled_for_testing - HDRS - "leak_check.h" - SRCS - "leak_check.cc" - COPTS - ${ABSL_DEFAULT_COPTS} - "-ULEAK_SANITIZER" - TESTONLY -) - absl_cc_test( NAME leak_check_test @@ -255,46 +228,15 @@ absl_cc_test( "leak_check_test.cc" COPTS ${ABSL_TEST_COPTS} - "$<$<BOOL:${ABSL_HAVE_LSAN}>:-DABSL_EXPECT_LEAK_SANITIZER>" - LINKOPTS - "${ABSL_LSAN_LINKOPTS}" - DEPS - absl::leak_check_api_enabled_for_testing - absl::base - gmock_main -) - -absl_cc_test( - NAME - leak_check_no_lsan_test - SRCS - "leak_check_test.cc" - COPTS - ${ABSL_TEST_COPTS} - "-UABSL_EXPECT_LEAK_SANITIZER" - DEPS - absl::leak_check_api_disabled_for_testing - absl::base - gmock_main -) - -absl_cc_test( - NAME - disabled_leak_check_test - SRCS - "leak_check_fail_test.cc" - COPTS - ${ABSL_TEST_COPTS} LINKOPTS - "${ABSL_LSAN_LINKOPTS}" + ${ABSL_DEFAULT_LINKOPTS} DEPS - absl::leak_check_api_enabled_for_testing - absl::leak_check_disable + absl::leak_check absl::base - absl::raw_logging_internal - gmock_main + GTest::gmock_main ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME stack_consumption @@ -322,7 +264,7 @@ absl_cc_test( absl::stack_consumption absl::core_headers absl::raw_logging_internal - gmock_main + GTest::gmock_main ) # component target diff --git a/absl/debugging/failure_signal_handler.cc b/absl/debugging/failure_signal_handler.cc index 3ddebd74..affade3b 100644 --- a/absl/debugging/failure_signal_handler.cc +++ b/absl/debugging/failure_signal_handler.cc @@ -42,7 +42,6 @@ #include <ctime> #include "absl/base/attributes.h" -#include "absl/base/internal/errno_saver.h" #include "absl/base/internal/raw_logging.h" #include "absl/base/internal/sysinfo.h" #include "absl/debugging/internal/examine_stack.h" @@ -52,7 +51,7 @@ #define ABSL_HAVE_SIGACTION // Apple WatchOS and TVOS don't allow sigaltstack #if !(defined(TARGET_OS_WATCH) && TARGET_OS_WATCH) && \ - !(defined(TARGET_OS_TV) && TARGET_OS_TV) + !(defined(TARGET_OS_TV) && TARGET_OS_TV) && !defined(__QNX__) #define ABSL_HAVE_SIGALTSTACK #endif #endif @@ -217,8 +216,7 @@ static void InstallOneFailureHandler(FailureSignalData* data, #endif static void WriteToStderr(const char* data) { - absl::base_internal::ErrnoSaver errno_saver; - absl::raw_logging_internal::SafeWriteToStderr(data, strlen(data)); + absl::raw_logging_internal::AsyncSignalSafeWriteToStderr(data, strlen(data)); } static void WriteSignalMessage(int signo, int cpu, @@ -367,6 +365,7 @@ static void AbslFailureSignalHandler(int signo, siginfo_t*, void* ucontext) { // goes after this point. if (fsh_options.writerfn != nullptr) { WriteFailureInfo(signo, ucontext, my_cpu, fsh_options.writerfn); + fsh_options.writerfn(nullptr); } if (fsh_options.call_previous_handler) { diff --git a/absl/debugging/failure_signal_handler.h b/absl/debugging/failure_signal_handler.h index 0c0f585d..500115c0 100644 --- a/absl/debugging/failure_signal_handler.h +++ b/absl/debugging/failure_signal_handler.h @@ -90,7 +90,7 @@ struct FailureSignalHandlerOptions { // If non-null, indicates a pointer to a callback function that will be called // upon failure, with a string argument containing failure data. This function // may be used as a hook to write failure data to a secondary location, such - // as a log file. This function may also be called with null data, as a hint + // as a log file. This function will also be called with null data, as a hint // to flush any buffered data before the program may be terminated. Consider // flushing any buffered data in all calls to this function. // diff --git a/absl/debugging/internal/address_is_readable.cc b/absl/debugging/internal/address_is_readable.cc index 329c285f..4be6256b 100644 --- a/absl/debugging/internal/address_is_readable.cc +++ b/absl/debugging/internal/address_is_readable.cc @@ -30,16 +30,12 @@ bool AddressIsReadable(const void* /* addr */) { return true; } ABSL_NAMESPACE_END } // namespace absl -#else +#else // __linux__ && !__ANDROID__ -#include <fcntl.h> -#include <sys/syscall.h> +#include <stdint.h> +#include <syscall.h> #include <unistd.h> -#include <atomic> -#include <cerrno> -#include <cstdint> - #include "absl/base/internal/errno_saver.h" #include "absl/base/internal/raw_logging.h" @@ -47,93 +43,54 @@ namespace absl { ABSL_NAMESPACE_BEGIN namespace debugging_internal { -// Pack a pid and two file descriptors into a 64-bit word, -// using 16, 24, and 24 bits for each respectively. -static uint64_t Pack(uint64_t pid, uint64_t read_fd, uint64_t write_fd) { - ABSL_RAW_CHECK((read_fd >> 24) == 0 && (write_fd >> 24) == 0, - "fd out of range"); - return (pid << 48) | ((read_fd & 0xffffff) << 24) | (write_fd & 0xffffff); -} - -// Unpack x into a pid and two file descriptors, where x was created with -// Pack(). -static void Unpack(uint64_t x, int *pid, int *read_fd, int *write_fd) { - *pid = x >> 48; - *read_fd = (x >> 24) & 0xffffff; - *write_fd = x & 0xffffff; -} +// NOTE: be extra careful about adding any interposable function calls here +// (such as open(), read(), etc.). These symbols may be interposed and will get +// invoked in contexts they don't expect. +// +// NOTE: any new system calls here may also require sandbox reconfiguration. +// +bool AddressIsReadable(const void *addr) { + // Align address on 8-byte boundary. On aarch64, checking last + // byte before inaccessible page returned unexpected EFAULT. + const uintptr_t u_addr = reinterpret_cast<uintptr_t>(addr) & ~7; + addr = reinterpret_cast<const void *>(u_addr); -// Return whether the byte at *addr is readable, without faulting. -// Save and restores errno. Returns true on systems where -// unimplemented. -// This is a namespace-scoped variable for correct zero-initialization. -static std::atomic<uint64_t> pid_and_fds; // initially 0, an invalid pid. + // rt_sigprocmask below will succeed for this input. + if (addr == nullptr) return false; -bool AddressIsReadable(const void *addr) { absl::base_internal::ErrnoSaver errno_saver; - // We test whether a byte is readable by using write(). Normally, this would - // be done via a cached file descriptor to /dev/null, but linux fails to - // check whether the byte is readable when the destination is /dev/null, so - // we use a cached pipe. We store the pid of the process that created the - // pipe to handle the case where a process forks, and the child closes all - // the file descriptors and then calls this routine. This is not perfect: - // the child could use the routine, then close all file descriptors and then - // use this routine again. But the likely use of this routine is when - // crashing, to test the validity of pages when dumping the stack. Beware - // that we may leak file descriptors, but we're unlikely to leak many. - int bytes_written; - int current_pid = getpid() & 0xffff; // we use only the low order 16 bits - do { // until we do not get EBADF trying to use file descriptors - int pid; - int read_fd; - int write_fd; - uint64_t local_pid_and_fds = pid_and_fds.load(std::memory_order_acquire); - Unpack(local_pid_and_fds, &pid, &read_fd, &write_fd); - while (current_pid != pid) { - int p[2]; - // new pipe - if (pipe(p) != 0) { - ABSL_RAW_LOG(FATAL, "Failed to create pipe, errno=%d", errno); - } - fcntl(p[0], F_SETFD, FD_CLOEXEC); - fcntl(p[1], F_SETFD, FD_CLOEXEC); - uint64_t new_pid_and_fds = Pack(current_pid, p[0], p[1]); - if (pid_and_fds.compare_exchange_strong( - local_pid_and_fds, new_pid_and_fds, std::memory_order_release, - std::memory_order_relaxed)) { - local_pid_and_fds = new_pid_and_fds; // fds exposed to other threads - } else { // fds not exposed to other threads; we can close them. - close(p[0]); - close(p[1]); - local_pid_and_fds = pid_and_fds.load(std::memory_order_acquire); - } - Unpack(local_pid_and_fds, &pid, &read_fd, &write_fd); - } - errno = 0; - // Use syscall(SYS_write, ...) instead of write() to prevent ASAN - // and other checkers from complaining about accesses to arbitrary - // memory. - do { - bytes_written = syscall(SYS_write, write_fd, addr, 1); - } while (bytes_written == -1 && errno == EINTR); - if (bytes_written == 1) { // remove the byte from the pipe - char c; - while (read(read_fd, &c, 1) == -1 && errno == EINTR) { - } - } - if (errno == EBADF) { // Descriptors invalid. - // If pid_and_fds contains the problematic file descriptors we just used, - // this call will forget them, and the loop will try again. - pid_and_fds.compare_exchange_strong(local_pid_and_fds, 0, - std::memory_order_release, - std::memory_order_relaxed); - } - } while (errno == EBADF); - return bytes_written == 1; + + // Here we probe with some syscall which + // - accepts an 8-byte region of user memory as input + // - tests for EFAULT before other validation + // - has no problematic side-effects + // + // rt_sigprocmask(2) works for this. It copies sizeof(kernel_sigset_t)==8 + // bytes from the address into the kernel memory before any validation. + // + // The call can never succeed, since the `how` parameter is not one of + // SIG_BLOCK, SIG_UNBLOCK, SIG_SETMASK. + // + // This strategy depends on Linux implementation details, + // so we rely on the test to alert us if it stops working. + // + // Some discarded past approaches: + // - msync() doesn't reject PROT_NONE regions + // - write() on /dev/null doesn't return EFAULT + // - write() on a pipe requires creating it and draining the writes + // - connect() works but is problematic for sandboxes and needs a valid + // file descriptor + // + // This can never succeed (invalid first argument to sigprocmask). + ABSL_RAW_CHECK(syscall(SYS_rt_sigprocmask, ~0, addr, nullptr, + /*sizeof(kernel_sigset_t)*/ 8) == -1, + "unexpected success"); + ABSL_RAW_CHECK(errno == EFAULT || errno == EINVAL, "unexpected errno"); + return errno != EFAULT; } } // namespace debugging_internal ABSL_NAMESPACE_END } // namespace absl -#endif +#endif // __linux__ && !__ANDROID__ diff --git a/absl/debugging/internal/demangle.cc b/absl/debugging/internal/demangle.cc index 5cd56320..93ae3279 100644 --- a/absl/debugging/internal/demangle.cc +++ b/absl/debugging/internal/demangle.cc @@ -1617,6 +1617,7 @@ static bool ParseUnresolvedName(State *state) { // ::= <2-ary operator-name> <expression> <expression> // ::= <3-ary operator-name> <expression> <expression> <expression> // ::= cl <expression>+ E +// ::= cp <simple-id> <expression>* E # Clang-specific. // ::= cv <type> <expression> # type (expression) // ::= cv <type> _ <expression>* E # type (expr-list) // ::= st <type> @@ -1639,14 +1640,23 @@ static bool ParseExpression(State *state) { return true; } - // Object/function call expression. ParseState copy = state->parse_state; + + // Object/function call expression. if (ParseTwoCharToken(state, "cl") && OneOrMore(ParseExpression, state) && ParseOneCharToken(state, 'E')) { return true; } state->parse_state = copy; + // Clang-specific "cp <simple-id> <expression>* E" + // https://clang.llvm.org/doxygen/ItaniumMangle_8cpp_source.html#l04338 + if (ParseTwoCharToken(state, "cp") && ParseSimpleId(state) && + ZeroOrMore(ParseExpression, state) && ParseOneCharToken(state, 'E')) { + return true; + } + state->parse_state = copy; + // Function-param expression (level 0). if (ParseTwoCharToken(state, "fp") && Optional(ParseCVQualifiers(state)) && Optional(ParseNumber(state, nullptr)) && ParseOneCharToken(state, '_')) { diff --git a/absl/debugging/internal/elf_mem_image.cc b/absl/debugging/internal/elf_mem_image.cc index 24cc0130..a9d66714 100644 --- a/absl/debugging/internal/elf_mem_image.cc +++ b/absl/debugging/internal/elf_mem_image.cc @@ -22,6 +22,7 @@ #include <string.h> #include <cassert> #include <cstddef> +#include "absl/base/config.h" #include "absl/base/internal/raw_logging.h" // From binutils/include/elf/common.h (this doesn't appear to be documented @@ -43,11 +44,11 @@ namespace debugging_internal { namespace { -#if __WORDSIZE == 32 +#if __SIZEOF_POINTER__ == 4 const int kElfClass = ELFCLASS32; int ElfBind(const ElfW(Sym) *symbol) { return ELF32_ST_BIND(symbol->st_info); } int ElfType(const ElfW(Sym) *symbol) { return ELF32_ST_TYPE(symbol->st_info); } -#elif __WORDSIZE == 64 +#elif __SIZEOF_POINTER__ == 8 const int kElfClass = ELFCLASS64; int ElfBind(const ElfW(Sym) *symbol) { return ELF64_ST_BIND(symbol->st_info); } int ElfType(const ElfW(Sym) *symbol) { return ELF64_ST_TYPE(symbol->st_info); } @@ -175,17 +176,17 @@ void ElfMemImage::Init(const void *base) { } switch (base_as_char[EI_DATA]) { case ELFDATA2LSB: { - if (__LITTLE_ENDIAN != __BYTE_ORDER) { - assert(false); - return; - } +#ifndef ABSL_IS_LITTLE_ENDIAN + assert(false); + return; +#endif break; } case ELFDATA2MSB: { - if (__BIG_ENDIAN != __BYTE_ORDER) { - assert(false); - return; - } +#ifndef ABSL_IS_BIG_ENDIAN + assert(false); + return; +#endif break; } default: { @@ -221,7 +222,7 @@ void ElfMemImage::Init(const void *base) { reinterpret_cast<ElfW(Dyn) *>(dynamic_program_header->p_vaddr + relocation); for (; dynamic_entry->d_tag != DT_NULL; ++dynamic_entry) { - const ElfW(Xword) value = dynamic_entry->d_un.d_val + relocation; + const auto value = dynamic_entry->d_un.d_val + relocation; switch (dynamic_entry->d_tag) { case DT_HASH: hash_ = reinterpret_cast<ElfW(Word) *>(value); @@ -350,7 +351,11 @@ void ElfMemImage::SymbolIterator::Update(int increment) { const ElfW(Versym) *version_symbol = image->GetVersym(index_); ABSL_RAW_CHECK(symbol && version_symbol, ""); const char *const symbol_name = image->GetDynstr(symbol->st_name); +#if defined(__NetBSD__) + const int version_index = version_symbol->vs_vers & VERSYM_VERSION; +#else const ElfW(Versym) version_index = version_symbol[0] & VERSYM_VERSION; +#endif const ElfW(Verdef) *version_definition = nullptr; const char *version_name = ""; if (symbol->st_shndx == SHN_UNDEF) { diff --git a/absl/debugging/internal/elf_mem_image.h b/absl/debugging/internal/elf_mem_image.h index 46bfade3..113071a9 100644 --- a/absl/debugging/internal/elf_mem_image.h +++ b/absl/debugging/internal/elf_mem_image.h @@ -31,8 +31,9 @@ #error ABSL_HAVE_ELF_MEM_IMAGE cannot be directly set #endif -#if defined(__ELF__) && defined(__GLIBC__) && !defined(__native_client__) && \ - !defined(__asmjs__) && !defined(__wasm__) +#if defined(__ELF__) && !defined(__OpenBSD__) && !defined(__QNX__) && \ + !defined(__native_client__) && !defined(__asmjs__) && \ + !defined(__wasm__) && !defined(__HAIKU__) #define ABSL_HAVE_ELF_MEM_IMAGE 1 #endif @@ -40,6 +41,10 @@ #include <link.h> // for ElfW +#if defined(__FreeBSD__) && !defined(ElfW) +#define ElfW(x) __ElfN(x) +#endif + namespace absl { ABSL_NAMESPACE_BEGIN namespace debugging_internal { diff --git a/absl/debugging/internal/examine_stack.cc b/absl/debugging/internal/examine_stack.cc index 589a3ef3..5bdd341e 100644 --- a/absl/debugging/internal/examine_stack.cc +++ b/absl/debugging/internal/examine_stack.cc @@ -20,7 +20,13 @@ #include <unistd.h> #endif -#ifdef __APPLE__ +#include "absl/base/config.h" + +#ifdef ABSL_HAVE_MMAP +#include <sys/mman.h> +#endif + +#if defined(__linux__) || defined(__APPLE__) #include <sys/ucontext.h> #endif @@ -37,10 +43,115 @@ namespace absl { ABSL_NAMESPACE_BEGIN namespace debugging_internal { +namespace { +constexpr int kDefaultDumpStackFramesLimit = 64; +// The %p field width for printf() functions is two characters per byte, +// and two extra for the leading "0x". +constexpr int kPrintfPointerFieldWidth = 2 + 2 * sizeof(void*); + +ABSL_CONST_INIT SymbolizeUrlEmitter debug_stack_trace_hook = nullptr; + +// Async-signal safe mmap allocator. +void* Allocate(size_t num_bytes) { +#ifdef ABSL_HAVE_MMAP + void* p = ::mmap(nullptr, num_bytes, PROT_READ | PROT_WRITE, + MAP_PRIVATE | MAP_ANONYMOUS, -1, 0); + return p == MAP_FAILED ? nullptr : p; +#else + (void)num_bytes; + return nullptr; +#endif // ABSL_HAVE_MMAP +} + +void Deallocate(void* p, size_t size) { +#ifdef ABSL_HAVE_MMAP + ::munmap(p, size); +#else + (void)p; + (void)size; +#endif // ABSL_HAVE_MMAP +} + +// Print a program counter only. +void DumpPC(OutputWriter* writer, void* writer_arg, void* const pc, + const char* const prefix) { + char buf[100]; + snprintf(buf, sizeof(buf), "%s@ %*p\n", prefix, kPrintfPointerFieldWidth, pc); + writer(buf, writer_arg); +} + +// Print a program counter and the corresponding stack frame size. +void DumpPCAndFrameSize(OutputWriter* writer, void* writer_arg, void* const pc, + int framesize, const char* const prefix) { + char buf[100]; + if (framesize <= 0) { + snprintf(buf, sizeof(buf), "%s@ %*p (unknown)\n", prefix, + kPrintfPointerFieldWidth, pc); + } else { + snprintf(buf, sizeof(buf), "%s@ %*p %9d\n", prefix, + kPrintfPointerFieldWidth, pc, framesize); + } + writer(buf, writer_arg); +} + +// Print a program counter and the corresponding symbol. +void DumpPCAndSymbol(OutputWriter* writer, void* writer_arg, void* const pc, + const char* const prefix) { + char tmp[1024]; + const char* symbol = "(unknown)"; + // Symbolizes the previous address of pc because pc may be in the + // next function. The overrun happens when the function ends with + // a call to a function annotated noreturn (e.g. CHECK). + // If symbolization of pc-1 fails, also try pc on the off-chance + // that we crashed on the first instruction of a function (that + // actually happens very often for e.g. __restore_rt). + const uintptr_t prev_pc = reinterpret_cast<uintptr_t>(pc) - 1; + if (absl::Symbolize(reinterpret_cast<const char*>(prev_pc), tmp, + sizeof(tmp)) || + absl::Symbolize(pc, tmp, sizeof(tmp))) { + symbol = tmp; + } + char buf[1024]; + snprintf(buf, sizeof(buf), "%s@ %*p %s\n", prefix, kPrintfPointerFieldWidth, + pc, symbol); + writer(buf, writer_arg); +} + +// Print a program counter, its stack frame size, and its symbol name. +// Note that there is a separate symbolize_pc argument. Return addresses may be +// at the end of the function, and this allows the caller to back up from pc if +// appropriate. +void DumpPCAndFrameSizeAndSymbol(OutputWriter* writer, void* writer_arg, + void* const pc, void* const symbolize_pc, + int framesize, const char* const prefix) { + char tmp[1024]; + const char* symbol = "(unknown)"; + if (absl::Symbolize(symbolize_pc, tmp, sizeof(tmp))) { + symbol = tmp; + } + char buf[1024]; + if (framesize <= 0) { + snprintf(buf, sizeof(buf), "%s@ %*p (unknown) %s\n", prefix, + kPrintfPointerFieldWidth, pc, symbol); + } else { + snprintf(buf, sizeof(buf), "%s@ %*p %9d %s\n", prefix, + kPrintfPointerFieldWidth, pc, framesize, symbol); + } + writer(buf, writer_arg); +} + +} // namespace + +void RegisterDebugStackTraceHook(SymbolizeUrlEmitter hook) { + debug_stack_trace_hook = hook; +} + +SymbolizeUrlEmitter GetDebugStackTraceHook() { return debug_stack_trace_hook; } + // Returns the program counter from signal context, nullptr if // unknown. vuc is a ucontext_t*. We use void* to avoid the use of // ucontext_t on non-POSIX systems. -void* GetProgramCounter(void* vuc) { +void* GetProgramCounter(void* const vuc) { #ifdef __linux__ if (vuc != nullptr) { ucontext_t* context = reinterpret_cast<ucontext_t*>(vuc); @@ -82,6 +193,8 @@ void* GetProgramCounter(void* vuc) { return reinterpret_cast<void*>(context->uc_mcontext.gregs[16]); #elif defined(__e2k__) return reinterpret_cast<void*>(context->uc_mcontext.cr0_hi); +#elif defined(__loongarch__) + return reinterpret_cast<void*>(context->uc_mcontext.__pc); #else #error "Undefined Architecture." #endif @@ -120,59 +233,17 @@ void* GetProgramCounter(void* vuc) { return nullptr; } -// The %p field width for printf() functions is two characters per byte, -// and two extra for the leading "0x". -static constexpr int kPrintfPointerFieldWidth = 2 + 2 * sizeof(void*); - -// Print a program counter, its stack frame size, and its symbol name. -// Note that there is a separate symbolize_pc argument. Return addresses may be -// at the end of the function, and this allows the caller to back up from pc if -// appropriate. -static void DumpPCAndFrameSizeAndSymbol(void (*writerfn)(const char*, void*), - void* writerfn_arg, void* pc, - void* symbolize_pc, int framesize, - const char* const prefix) { - char tmp[1024]; - const char* symbol = "(unknown)"; - if (absl::Symbolize(symbolize_pc, tmp, sizeof(tmp))) { - symbol = tmp; - } - char buf[1024]; - if (framesize <= 0) { - snprintf(buf, sizeof(buf), "%s@ %*p (unknown) %s\n", prefix, - kPrintfPointerFieldWidth, pc, symbol); - } else { - snprintf(buf, sizeof(buf), "%s@ %*p %9d %s\n", prefix, - kPrintfPointerFieldWidth, pc, framesize, symbol); - } - writerfn(buf, writerfn_arg); -} - -// Print a program counter and the corresponding stack frame size. -static void DumpPCAndFrameSize(void (*writerfn)(const char*, void*), - void* writerfn_arg, void* pc, int framesize, - const char* const prefix) { - char buf[100]; - if (framesize <= 0) { - snprintf(buf, sizeof(buf), "%s@ %*p (unknown)\n", prefix, - kPrintfPointerFieldWidth, pc); - } else { - snprintf(buf, sizeof(buf), "%s@ %*p %9d\n", prefix, - kPrintfPointerFieldWidth, pc, framesize); - } - writerfn(buf, writerfn_arg); -} - -void DumpPCAndFrameSizesAndStackTrace( - void* pc, void* const stack[], int frame_sizes[], int depth, - int min_dropped_frames, bool symbolize_stacktrace, - void (*writerfn)(const char*, void*), void* writerfn_arg) { +void DumpPCAndFrameSizesAndStackTrace(void* const pc, void* const stack[], + int frame_sizes[], int depth, + int min_dropped_frames, + bool symbolize_stacktrace, + OutputWriter* writer, void* writer_arg) { if (pc != nullptr) { // We don't know the stack frame size for PC, use 0. if (symbolize_stacktrace) { - DumpPCAndFrameSizeAndSymbol(writerfn, writerfn_arg, pc, pc, 0, "PC: "); + DumpPCAndFrameSizeAndSymbol(writer, writer_arg, pc, pc, 0, "PC: "); } else { - DumpPCAndFrameSize(writerfn, writerfn_arg, pc, 0, "PC: "); + DumpPCAndFrameSize(writer, writer_arg, pc, 0, "PC: "); } } for (int i = 0; i < depth; i++) { @@ -182,20 +253,61 @@ void DumpPCAndFrameSizesAndStackTrace( // call to a function annotated noreturn (e.g. CHECK). Note that we don't // do this for pc above, as the adjustment is only correct for return // addresses. - DumpPCAndFrameSizeAndSymbol(writerfn, writerfn_arg, stack[i], + DumpPCAndFrameSizeAndSymbol(writer, writer_arg, stack[i], reinterpret_cast<char*>(stack[i]) - 1, frame_sizes[i], " "); } else { - DumpPCAndFrameSize(writerfn, writerfn_arg, stack[i], frame_sizes[i], - " "); + DumpPCAndFrameSize(writer, writer_arg, stack[i], frame_sizes[i], " "); } } if (min_dropped_frames > 0) { char buf[100]; snprintf(buf, sizeof(buf), " @ ... and at least %d more frames\n", min_dropped_frames); - writerfn(buf, writerfn_arg); + writer(buf, writer_arg); + } +} + +// Dump current stack trace as directed by writer. +// Make sure this function is not inlined to avoid skipping too many top frames. +ABSL_ATTRIBUTE_NOINLINE +void DumpStackTrace(int min_dropped_frames, int max_num_frames, + bool symbolize_stacktrace, OutputWriter* writer, + void* writer_arg) { + // Print stack trace + void* stack_buf[kDefaultDumpStackFramesLimit]; + void** stack = stack_buf; + int num_stack = kDefaultDumpStackFramesLimit; + int allocated_bytes = 0; + + if (num_stack >= max_num_frames) { + // User requested fewer frames than we already have space for. + num_stack = max_num_frames; + } else { + const size_t needed_bytes = max_num_frames * sizeof(stack[0]); + void* p = Allocate(needed_bytes); + if (p != nullptr) { // We got the space. + num_stack = max_num_frames; + stack = reinterpret_cast<void**>(p); + allocated_bytes = needed_bytes; + } } + + size_t depth = absl::GetStackTrace(stack, num_stack, min_dropped_frames + 1); + for (size_t i = 0; i < depth; i++) { + if (symbolize_stacktrace) { + DumpPCAndSymbol(writer, writer_arg, stack[i], " "); + } else { + DumpPC(writer, writer_arg, stack[i], " "); + } + } + + auto hook = GetDebugStackTraceHook(); + if (hook != nullptr) { + (*hook)(stack, depth, writer, writer_arg); + } + + if (allocated_bytes != 0) Deallocate(stack, allocated_bytes); } } // namespace debugging_internal diff --git a/absl/debugging/internal/examine_stack.h b/absl/debugging/internal/examine_stack.h index 39336913..190af87f 100644 --- a/absl/debugging/internal/examine_stack.h +++ b/absl/debugging/internal/examine_stack.h @@ -23,17 +23,39 @@ namespace absl { ABSL_NAMESPACE_BEGIN namespace debugging_internal { +// Type of function used for printing in stack trace dumping, etc. +// We avoid closures to keep things simple. +typedef void OutputWriter(const char*, void*); + +// RegisterDebugStackTraceHook() allows to register a single routine +// `hook` that is called each time DumpStackTrace() is called. +// `hook` may be called from a signal handler. +typedef void (*SymbolizeUrlEmitter)(void* const stack[], int depth, + OutputWriter* writer, void* writer_arg); + +// Registration of SymbolizeUrlEmitter for use inside of a signal handler. +// This is inherently unsafe and must be signal safe code. +void RegisterDebugStackTraceHook(SymbolizeUrlEmitter hook); +SymbolizeUrlEmitter GetDebugStackTraceHook(); + // Returns the program counter from signal context, or nullptr if // unknown. `vuc` is a ucontext_t*. We use void* to avoid the use of // ucontext_t on non-POSIX systems. -void* GetProgramCounter(void* vuc); +void* GetProgramCounter(void* const vuc); -// Uses `writerfn` to dump the program counter, stack trace, and stack +// Uses `writer` to dump the program counter, stack trace, and stack // frame sizes. -void DumpPCAndFrameSizesAndStackTrace( - void* pc, void* const stack[], int frame_sizes[], int depth, - int min_dropped_frames, bool symbolize_stacktrace, - void (*writerfn)(const char*, void*), void* writerfn_arg); +void DumpPCAndFrameSizesAndStackTrace(void* const pc, void* const stack[], + int frame_sizes[], int depth, + int min_dropped_frames, + bool symbolize_stacktrace, + OutputWriter* writer, void* writer_arg); + +// Dump current stack trace omitting the topmost `min_dropped_frames` stack +// frames. +void DumpStackTrace(int min_dropped_frames, int max_num_frames, + bool symbolize_stacktrace, OutputWriter* writer, + void* writer_arg); } // namespace debugging_internal ABSL_NAMESPACE_END diff --git a/absl/debugging/internal/stack_consumption.cc b/absl/debugging/internal/stack_consumption.cc index e3dd51c3..51348649 100644 --- a/absl/debugging/internal/stack_consumption.cc +++ b/absl/debugging/internal/stack_consumption.cc @@ -43,7 +43,7 @@ namespace { // unspecified. Therefore, instead we hardcode the direction of the // stack on platforms we know about. #if defined(__i386__) || defined(__x86_64__) || defined(__ppc__) || \ - defined(__aarch64__) + defined(__aarch64__) || defined(__riscv) constexpr bool kStackGrowsDown = true; #else #error Need to define kStackGrowsDown diff --git a/absl/debugging/internal/stack_consumption.h b/absl/debugging/internal/stack_consumption.h index 2b5e7151..f41b64c3 100644 --- a/absl/debugging/internal/stack_consumption.h +++ b/absl/debugging/internal/stack_consumption.h @@ -26,7 +26,7 @@ #error ABSL_INTERNAL_HAVE_DEBUGGING_STACK_CONSUMPTION cannot be set directly #elif !defined(__APPLE__) && !defined(_WIN32) && \ (defined(__i386__) || defined(__x86_64__) || defined(__ppc__) || \ - defined(__aarch64__)) + defined(__aarch64__) || defined(__riscv)) #define ABSL_INTERNAL_HAVE_DEBUGGING_STACK_CONSUMPTION 1 namespace absl { diff --git a/absl/debugging/internal/stacktrace_aarch64-inl.inc b/absl/debugging/internal/stacktrace_aarch64-inl.inc index f4859d7c..4f9db9d6 100644 --- a/absl/debugging/internal/stacktrace_aarch64-inl.inc +++ b/absl/debugging/internal/stacktrace_aarch64-inl.inc @@ -176,12 +176,17 @@ static int UnwindImpl(void** result, int* sizes, int max_depth, int skip_count, // Implementation detail: we clamp the max of frames we are willing to // count, so as not to spend too much time in the loop below. const int kMaxUnwind = 200; - int j = 0; - for (; frame_pointer != nullptr && j < kMaxUnwind; j++) { + int num_dropped_frames = 0; + for (int j = 0; frame_pointer != nullptr && j < kMaxUnwind; j++) { + if (skip_count > 0) { + skip_count--; + } else { + num_dropped_frames++; + } frame_pointer = NextStackFrame<!IS_STACK_FRAMES, IS_WITH_CONTEXT>(frame_pointer, ucp); } - *min_dropped_frames = j; + *min_dropped_frames = num_dropped_frames; } return n; } diff --git a/absl/debugging/internal/stacktrace_arm-inl.inc b/absl/debugging/internal/stacktrace_arm-inl.inc index 2a1bf2e8..102a2a12 100644 --- a/absl/debugging/internal/stacktrace_arm-inl.inc +++ b/absl/debugging/internal/stacktrace_arm-inl.inc @@ -112,11 +112,16 @@ static int UnwindImpl(void** result, int* sizes, int max_depth, int skip_count, // Implementation detail: we clamp the max of frames we are willing to // count, so as not to spend too much time in the loop below. const int kMaxUnwind = 200; - int j = 0; - for (; sp != nullptr && j < kMaxUnwind; j++) { + int num_dropped_frames = 0; + for (int j = 0; sp != nullptr && j < kMaxUnwind; j++) { + if (skip_count > 0) { + skip_count--; + } else { + num_dropped_frames++; + } sp = NextStackFrame<!IS_STACK_FRAMES>(sp); } - *min_dropped_frames = j; + *min_dropped_frames = num_dropped_frames; } return n; } diff --git a/absl/debugging/internal/stacktrace_config.h b/absl/debugging/internal/stacktrace_config.h index cca410d4..3929b1b7 100644 --- a/absl/debugging/internal/stacktrace_config.h +++ b/absl/debugging/internal/stacktrace_config.h @@ -35,7 +35,12 @@ // Thread local support required for UnwindImpl. #define ABSL_STACKTRACE_INL_HEADER \ "absl/debugging/internal/stacktrace_generic-inl.inc" -#endif +#endif // defined(ABSL_HAVE_THREAD_LOCAL) + +// Emscripten stacktraces rely on JS. Do not use them in standalone mode. +#elif defined(__EMSCRIPTEN__) && !defined(STANDALONE_WASM) +#define ABSL_STACKTRACE_INL_HEADER \ + "absl/debugging/internal/stacktrace_emscripten-inl.inc" #elif defined(__linux__) && !defined(__ANDROID__) @@ -51,7 +56,7 @@ // Note: When using glibc this may require -funwind-tables to function properly. #define ABSL_STACKTRACE_INL_HEADER \ "absl/debugging/internal/stacktrace_generic-inl.inc" -#endif +#endif // __has_include(<execinfo.h>) #elif defined(__i386__) || defined(__x86_64__) #define ABSL_STACKTRACE_INL_HEADER \ "absl/debugging/internal/stacktrace_x86-inl.inc" @@ -61,15 +66,18 @@ #elif defined(__aarch64__) #define ABSL_STACKTRACE_INL_HEADER \ "absl/debugging/internal/stacktrace_aarch64-inl.inc" +#elif defined(__riscv) +#define ABSL_STACKTRACE_INL_HEADER \ + "absl/debugging/internal/stacktrace_riscv-inl.inc" #elif defined(__has_include) #if __has_include(<execinfo.h>) // Note: When using glibc this may require -funwind-tables to function properly. #define ABSL_STACKTRACE_INL_HEADER \ "absl/debugging/internal/stacktrace_generic-inl.inc" -#endif -#endif +#endif // __has_include(<execinfo.h>) +#endif // defined(__has_include) -#endif +#endif // defined(__linux__) && !defined(__ANDROID__) // Fallback to the empty implementation. #if !defined(ABSL_STACKTRACE_INL_HEADER) diff --git a/absl/debugging/internal/stacktrace_emscripten-inl.inc b/absl/debugging/internal/stacktrace_emscripten-inl.inc new file mode 100644 index 00000000..0f444514 --- /dev/null +++ b/absl/debugging/internal/stacktrace_emscripten-inl.inc @@ -0,0 +1,110 @@ +// Copyright 2017 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Portable implementation - just use glibc +// +// Note: The glibc implementation may cause a call to malloc. +// This can cause a deadlock in HeapProfiler. + +#ifndef ABSL_DEBUGGING_INTERNAL_STACKTRACE_EMSCRIPTEN_INL_H_ +#define ABSL_DEBUGGING_INTERNAL_STACKTRACE_EMSCRIPTEN_INL_H_ + +#include <emscripten.h> + +#include <atomic> +#include <cstring> + +#include "absl/base/attributes.h" +#include "absl/debugging/stacktrace.h" + +extern "C" { +uintptr_t emscripten_stack_snapshot(); +uint32_t emscripten_stack_unwind_buffer(uintptr_t pc, void *buffer, + uint32_t depth); +} + +// Sometimes, we can try to get a stack trace from within a stack +// trace, which can cause a self-deadlock. +// Protect against such reentrant call by failing to get a stack trace. +// +// We use __thread here because the code here is extremely low level -- it is +// called while collecting stack traces from within malloc and mmap, and thus +// can not call anything which might call malloc or mmap itself. +static __thread int recursive = 0; + +// The stack trace function might be invoked very early in the program's +// execution (e.g. from the very first malloc). +// As such, we suppress usage of backtrace during this early stage of execution. +static std::atomic<bool> disable_stacktraces(true); // Disabled until healthy. +// Waiting until static initializers run seems to be late enough. +// This file is included into stacktrace.cc so this will only run once. +ABSL_ATTRIBUTE_UNUSED static int stacktraces_enabler = []() { + // Check if we can even create stacktraces. If not, bail early and leave + // disable_stacktraces set as-is. + // clang-format off + if (!EM_ASM_INT({ return (typeof wasmOffsetConverter !== 'undefined'); })) { + return 0; + } + // clang-format on + disable_stacktraces.store(false, std::memory_order_relaxed); + return 0; +}(); + +template <bool IS_STACK_FRAMES, bool IS_WITH_CONTEXT> +static int UnwindImpl(void **result, int *sizes, int max_depth, int skip_count, + const void *ucp, int *min_dropped_frames) { + if (recursive || disable_stacktraces.load(std::memory_order_relaxed)) { + return 0; + } + ++recursive; + + static_cast<void>(ucp); // Unused. + constexpr int kStackLength = 64; + void *stack[kStackLength]; + + int size; + uintptr_t pc = emscripten_stack_snapshot(); + size = emscripten_stack_unwind_buffer(pc, stack, kStackLength); + + int result_count = size - skip_count; + if (result_count < 0) result_count = 0; + if (result_count > max_depth) result_count = max_depth; + for (int i = 0; i < result_count; i++) result[i] = stack[i + skip_count]; + + if (IS_STACK_FRAMES) { + // No implementation for finding out the stack frame sizes yet. + memset(sizes, 0, sizeof(*sizes) * result_count); + } + if (min_dropped_frames != nullptr) { + if (size - skip_count - max_depth > 0) { + *min_dropped_frames = size - skip_count - max_depth; + } else { + *min_dropped_frames = 0; + } + } + + --recursive; + + return result_count; +} + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace debugging_internal { +bool StackTraceWorksForTest() { return true; } +} // namespace debugging_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_DEBUGGING_INTERNAL_STACKTRACE_EMSCRIPTEN_INL_H_ diff --git a/absl/debugging/internal/stacktrace_powerpc-inl.inc b/absl/debugging/internal/stacktrace_powerpc-inl.inc index cf8c0516..085cef67 100644 --- a/absl/debugging/internal/stacktrace_powerpc-inl.inc +++ b/absl/debugging/internal/stacktrace_powerpc-inl.inc @@ -231,11 +231,16 @@ static int UnwindImpl(void** result, int* sizes, int max_depth, int skip_count, // Implementation detail: we clamp the max of frames we are willing to // count, so as not to spend too much time in the loop below. const int kMaxUnwind = 1000; - int j = 0; - for (; next_sp != nullptr && j < kMaxUnwind; j++) { + int num_dropped_frames = 0; + for (int j = 0; next_sp != nullptr && j < kMaxUnwind; j++) { + if (skip_count > 0) { + skip_count--; + } else { + num_dropped_frames++; + } next_sp = NextStackFrame<!IS_STACK_FRAMES, IS_WITH_CONTEXT>(next_sp, ucp); } - *min_dropped_frames = j; + *min_dropped_frames = num_dropped_frames; } return n; } diff --git a/absl/debugging/internal/stacktrace_riscv-inl.inc b/absl/debugging/internal/stacktrace_riscv-inl.inc new file mode 100644 index 00000000..7123b71b --- /dev/null +++ b/absl/debugging/internal/stacktrace_riscv-inl.inc @@ -0,0 +1,236 @@ +// Copyright 2021 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_DEBUGGING_INTERNAL_STACKTRACE_RISCV_INL_H_ +#define ABSL_DEBUGGING_INTERNAL_STACKTRACE_RISCV_INL_H_ + +// Generate stack trace for riscv + +#include <sys/ucontext.h> + +#include "absl/base/config.h" +#if defined(__linux__) +#include <sys/mman.h> +#include <ucontext.h> +#include <unistd.h> +#endif + +#include <atomic> +#include <cassert> +#include <cstdint> +#include <iostream> + +#include "absl/base/attributes.h" +#include "absl/debugging/internal/address_is_readable.h" +#include "absl/debugging/internal/vdso_support.h" +#include "absl/debugging/stacktrace.h" + +static const uintptr_t kUnknownFrameSize = 0; + +#if defined(__linux__) +// Returns the address of the VDSO __kernel_rt_sigreturn function, if present. +static const unsigned char *GetKernelRtSigreturnAddress() { + constexpr uintptr_t kImpossibleAddress = 0; + ABSL_CONST_INIT static std::atomic<uintptr_t> memoized(kImpossibleAddress); + uintptr_t address = memoized.load(std::memory_order_relaxed); + if (address != kImpossibleAddress) { + return reinterpret_cast<const unsigned char *>(address); + } + + address = reinterpret_cast<uintptr_t>(nullptr); + +#if ABSL_HAVE_VDSO_SUPPORT + absl::debugging_internal::VDSOSupport vdso; + if (vdso.IsPresent()) { + absl::debugging_internal::VDSOSupport::SymbolInfo symbol_info; + // Symbol versioning pulled from arch/riscv/kernel/vdso/vdso.lds at v5.10. + auto lookup = [&](int type) { + return vdso.LookupSymbol("__vdso_rt_sigreturn", "LINUX_4.15", type, + &symbol_info); + }; + if ((!lookup(STT_FUNC) && !lookup(STT_NOTYPE)) || + symbol_info.address == nullptr) { + // Unexpected: VDSO is present, yet the expected symbol is missing or + // null. + assert(false && "VDSO is present, but doesn't have expected symbol"); + } else { + if (reinterpret_cast<uintptr_t>(symbol_info.address) != + kImpossibleAddress) { + address = reinterpret_cast<uintptr_t>(symbol_info.address); + } else { + assert(false && "VDSO returned invalid address"); + } + } + } +#endif + + memoized.store(address, std::memory_order_relaxed); + return reinterpret_cast<const unsigned char *>(address); +} +#endif // __linux__ + +// Compute the size of a stack frame in [low..high). We assume that low < high. +// Return size of kUnknownFrameSize. +template <typename T> +static inline uintptr_t ComputeStackFrameSize(const T *low, const T *high) { + const char *low_char_ptr = reinterpret_cast<const char *>(low); + const char *high_char_ptr = reinterpret_cast<const char *>(high); + return low < high ? high_char_ptr - low_char_ptr : kUnknownFrameSize; +} + +// Given a pointer to a stack frame, locate and return the calling stackframe, +// or return null if no stackframe can be found. Perform sanity checks (the +// strictness of which is controlled by the boolean parameter +// "STRICT_UNWINDING") to reduce the chance that a bad pointer is returned. +template <bool STRICT_UNWINDING, bool WITH_CONTEXT> +ABSL_ATTRIBUTE_NO_SANITIZE_ADDRESS // May read random elements from stack. +ABSL_ATTRIBUTE_NO_SANITIZE_MEMORY // May read random elements from stack. +static void ** NextStackFrame(void **old_frame_pointer, const void *uc) { + // . + // . + // . + // +-> +----------------+ + // | | return address | + // | | previous fp | + // | | ... | + // | +----------------+ <-+ + // | | return address | | + // +---|- previous fp | | + // | ... | | + // $fp ->|----------------+ | + // | return address | | + // | previous fp -|---+ + // $sp ->| ... | + // +----------------+ + void **new_frame_pointer = reinterpret_cast<void **>(old_frame_pointer[-2]); + bool check_frame_size = true; + +#if defined(__linux__) + if (WITH_CONTEXT && uc != nullptr) { + // Check to see if next frame's return address is __kernel_rt_sigreturn. + if (old_frame_pointer[-1] == GetKernelRtSigreturnAddress()) { + const ucontext_t *ucv = static_cast<const ucontext_t *>(uc); + // old_frame_pointer is not suitable for unwinding, look at ucontext to + // discover frame pointer before signal. + // + // RISCV ELF psABI has the frame pointer at x8/fp/s0. + // -- RISCV psABI Table 18.2 + void **const pre_signal_frame_pointer = + reinterpret_cast<void **>(ucv->uc_mcontext.__gregs[8]); + + // Check the alleged frame pointer is actually readable. This is to + // prevent "double fault" in case we hit the first fault due to stack + // corruption. + if (!absl::debugging_internal::AddressIsReadable( + pre_signal_frame_pointer)) + return nullptr; + + // Alleged frame pointer is readable, use it for further unwinding. + new_frame_pointer = pre_signal_frame_pointer; + + // Skip frame size check if we return from a signal. We may be using an + // alterate stack for signals. + check_frame_size = false; + } + } +#endif + + // The RISCV ELF psABI mandates that the stack pointer is always 16-byte + // aligned. + // FIXME(abdulras) this doesn't hold for ILP32E which only mandates a 4-byte + // alignment. + if ((reinterpret_cast<uintptr_t>(new_frame_pointer) & 15) != 0) + return nullptr; + + // Check frame size. In strict mode, we assume frames to be under 100,000 + // bytes. In non-strict mode, we relax the limit to 1MB. + if (check_frame_size) { + const uintptr_t max_size = STRICT_UNWINDING ? 100000 : 1000000; + const uintptr_t frame_size = + ComputeStackFrameSize(old_frame_pointer, new_frame_pointer); + if (frame_size == kUnknownFrameSize || frame_size > max_size) + return nullptr; + } + + return new_frame_pointer; +} + +template <bool IS_STACK_FRAMES, bool IS_WITH_CONTEXT> +ABSL_ATTRIBUTE_NO_SANITIZE_ADDRESS // May read random elements from stack. +ABSL_ATTRIBUTE_NO_SANITIZE_MEMORY // May read random elements from stack. +static int UnwindImpl(void **result, int *sizes, int max_depth, int skip_count, + const void *ucp, int *min_dropped_frames) { + // The `frame_pointer` that is computed here points to the top of the frame. + // The two words preceding the address are the return address and the previous + // frame pointer. +#if defined(__GNUC__) + void **frame_pointer = reinterpret_cast<void **>(__builtin_frame_address(0)); +#else +#error reading stack pointer not yet supported on this platform +#endif + + int n = 0; + void *return_address = nullptr; + while (frame_pointer && n < max_depth) { + return_address = frame_pointer[-1]; + + // The absl::GetStackFrames routine is called when we are in some + // informational context (the failure signal handler for example). Use the + // non-strict unwinding rules to produce a stack trace that is as complete + // as possible (even if it contains a few bogus entries in some rare cases). + void **next_frame_pointer = + NextStackFrame<!IS_STACK_FRAMES, IS_WITH_CONTEXT>(frame_pointer, ucp); + + if (skip_count > 0) { + skip_count--; + } else { + result[n] = return_address; + if (IS_STACK_FRAMES) { + sizes[n] = ComputeStackFrameSize(frame_pointer, next_frame_pointer); + } + n++; + } + + frame_pointer = next_frame_pointer; + } + + if (min_dropped_frames != nullptr) { + // Implementation detail: we clamp the max of frames we are willing to + // count, so as not to spend too much time in the loop below. + const int kMaxUnwind = 200; + int num_dropped_frames = 0; + for (int j = 0; frame_pointer != nullptr && j < kMaxUnwind; j++) { + if (skip_count > 0) { + skip_count--; + } else { + num_dropped_frames++; + } + frame_pointer = + NextStackFrame<!IS_STACK_FRAMES, IS_WITH_CONTEXT>(frame_pointer, ucp); + } + *min_dropped_frames = num_dropped_frames; + } + + return n; +} + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace debugging_internal { +bool StackTraceWorksForTest() { return true; } +} // namespace debugging_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif diff --git a/absl/debugging/internal/stacktrace_x86-inl.inc b/absl/debugging/internal/stacktrace_x86-inl.inc index bc320ff7..1b5d8235 100644 --- a/absl/debugging/internal/stacktrace_x86-inl.inc +++ b/absl/debugging/internal/stacktrace_x86-inl.inc @@ -27,6 +27,7 @@ #include <cassert> #include <cstdint> +#include <limits> #include "absl/base/macros.h" #include "absl/base/port.h" @@ -132,9 +133,8 @@ static uintptr_t GetFP(const void *vuc) { const uintptr_t bp = 0; const uintptr_t sp = 0; #endif - // Sanity-check that the base pointer is valid. It should be as long as - // SHRINK_WRAP_FRAME_POINTER is not set, but it's possible that some code in - // the process is compiled with --copt=-fomit-frame-pointer or + // Sanity-check that the base pointer is valid. It's possible that some + // code in the process is compiled with --copt=-fomit-frame-pointer or // --copt=-momit-leaf-frame-pointer. // // TODO(bcmills): -momit-leaf-frame-pointer is currently the default @@ -159,7 +159,8 @@ static uintptr_t GetFP(const void *vuc) { template <bool STRICT_UNWINDING, bool WITH_CONTEXT> ABSL_ATTRIBUTE_NO_SANITIZE_ADDRESS // May read random elements from stack. ABSL_ATTRIBUTE_NO_SANITIZE_MEMORY // May read random elements from stack. -static void **NextStackFrame(void **old_fp, const void *uc) { +static void **NextStackFrame(void **old_fp, const void *uc, + size_t stack_low, size_t stack_high) { void **new_fp = (void **)*old_fp; #if defined(__linux__) && defined(__i386__) @@ -247,7 +248,7 @@ static void **NextStackFrame(void **old_fp, const void *uc) { // using an alternate signal stack. // // TODO(bcmills): The GetFP call should be completely unnecessary when - // SHRINK_WRAP_FRAME_POINTER is set (because we should be back in the thread's + // ENABLE_COMBINED_UNWINDER is set (because we should be back in the thread's // stack by this point), but it is empirically still needed (e.g. when the // stack includes a call to abort). unw_get_reg returns UNW_EBADREG for some // frames. Figure out why GetValidFrameAddr and/or libunwind isn't doing what @@ -258,6 +259,18 @@ static void **NextStackFrame(void **old_fp, const void *uc) { // at a greater address that the current one. if (new_fp_u <= old_fp_u) return nullptr; if (new_fp_u - old_fp_u > kMaxFrameBytes) return nullptr; + + if (stack_low < old_fp_u && old_fp_u <= stack_high) { + // Old BP was in the expected stack region... + if (!(stack_low < new_fp_u && new_fp_u <= stack_high)) { + // ... but new BP is outside of expected stack region. + // It is most likely bogus. + return nullptr; + } + } else { + // We may be here if we are executing in a co-routine with a + // separate stack. We can't do safety checks in this case. + } } else { if (new_fp == nullptr) return nullptr; // skip AddressIsReadable() below // In the non-strict mode, allow discontiguous stack frames. @@ -297,13 +310,17 @@ static int UnwindImpl(void **result, int *sizes, int max_depth, int skip_count, int n = 0; void **fp = reinterpret_cast<void **>(__builtin_frame_address(0)); + size_t stack_low = getpagesize(); // Assume that the first page is not stack. + size_t stack_high = std::numeric_limits<size_t>::max() - sizeof(void *); + while (fp && n < max_depth) { if (*(fp + 1) == reinterpret_cast<void *>(0)) { // In 64-bit code, we often see a frame that // points to itself and has a return address of 0. break; } - void **next_fp = NextStackFrame<!IS_STACK_FRAMES, IS_WITH_CONTEXT>(fp, ucp); + void **next_fp = NextStackFrame<!IS_STACK_FRAMES, IS_WITH_CONTEXT>( + fp, ucp, stack_low, stack_high); if (skip_count > 0) { skip_count--; } else { @@ -324,11 +341,17 @@ static int UnwindImpl(void **result, int *sizes, int max_depth, int skip_count, // Implementation detail: we clamp the max of frames we are willing to // count, so as not to spend too much time in the loop below. const int kMaxUnwind = 1000; - int j = 0; - for (; fp != nullptr && j < kMaxUnwind; j++) { - fp = NextStackFrame<!IS_STACK_FRAMES, IS_WITH_CONTEXT>(fp, ucp); + int num_dropped_frames = 0; + for (int j = 0; fp != nullptr && j < kMaxUnwind; j++) { + if (skip_count > 0) { + skip_count--; + } else { + num_dropped_frames++; + } + fp = NextStackFrame<!IS_STACK_FRAMES, IS_WITH_CONTEXT>(fp, ucp, stack_low, + stack_high); } - *min_dropped_frames = j; + *min_dropped_frames = num_dropped_frames; } return n; } diff --git a/absl/debugging/internal/symbolize.h b/absl/debugging/internal/symbolize.h index 4f26130f..27d5e652 100644 --- a/absl/debugging/internal/symbolize.h +++ b/absl/debugging/internal/symbolize.h @@ -28,8 +28,8 @@ #ifdef ABSL_INTERNAL_HAVE_ELF_SYMBOLIZE #error ABSL_INTERNAL_HAVE_ELF_SYMBOLIZE cannot be directly set -#elif defined(__ELF__) && defined(__GLIBC__) && !defined(__native_client__) && \ - !defined(__asmjs__) && !defined(__wasm__) +#elif defined(__ELF__) && defined(__GLIBC__) && !defined(__native_client__) \ + && !defined(__asmjs__) && !defined(__wasm__) #define ABSL_INTERNAL_HAVE_ELF_SYMBOLIZE 1 #include <elf.h> @@ -68,6 +68,12 @@ ABSL_NAMESPACE_END #define ABSL_INTERNAL_HAVE_DARWIN_SYMBOLIZE 1 #endif +#ifdef ABSL_INTERNAL_HAVE_EMSCRIPTEN_SYMBOLIZE +#error ABSL_INTERNAL_HAVE_EMSCRIPTEN_SYMBOLIZE cannot be directly set +#elif defined(__EMSCRIPTEN__) +#define ABSL_INTERNAL_HAVE_EMSCRIPTEN_SYMBOLIZE 1 +#endif + namespace absl { ABSL_NAMESPACE_BEGIN namespace debugging_internal { diff --git a/absl/debugging/internal/vdso_support.cc b/absl/debugging/internal/vdso_support.cc index 6be16d90..40eb055f 100644 --- a/absl/debugging/internal/vdso_support.cc +++ b/absl/debugging/internal/vdso_support.cc @@ -20,12 +20,25 @@ #ifdef ABSL_HAVE_VDSO_SUPPORT // defined in vdso_support.h +#if !defined(__has_include) +#define __has_include(header) 0 +#endif + #include <errno.h> #include <fcntl.h> +#if __has_include(<syscall.h>) +#include <syscall.h> +#elif __has_include(<sys/syscall.h>) #include <sys/syscall.h> +#endif #include <unistd.h> -#if __GLIBC_PREREQ(2, 16) // GLIBC-2.16 implements getauxval. +#if !defined(__UCLIBC__) && defined(__GLIBC__) && \ + (__GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 16)) +#define ABSL_HAVE_GETAUXVAL +#endif + +#ifdef ABSL_HAVE_GETAUXVAL #include <sys/auxv.h> #endif @@ -37,6 +50,17 @@ #define AT_SYSINFO_EHDR 33 // for crosstoolv10 #endif +#if defined(__NetBSD__) +using Elf32_auxv_t = Aux32Info; +using Elf64_auxv_t = Aux64Info; +#endif +#if defined(__FreeBSD__) +#if defined(__ELF_WORD_SIZE) && __ELF_WORD_SIZE == 64 +using Elf64_auxv_t = Elf64_Auxinfo; +#endif +using Elf32_auxv_t = Elf32_Auxinfo; +#endif + namespace absl { ABSL_NAMESPACE_BEGIN namespace debugging_internal { @@ -45,7 +69,9 @@ ABSL_CONST_INIT std::atomic<const void *> VDSOSupport::vdso_base_( debugging_internal::ElfMemImage::kInvalidBase); -std::atomic<VDSOSupport::GetCpuFn> VDSOSupport::getcpu_fn_(&InitAndGetCPU); +ABSL_CONST_INIT std::atomic<VDSOSupport::GetCpuFn> VDSOSupport::getcpu_fn_( + &InitAndGetCPU); + VDSOSupport::VDSOSupport() // If vdso_base_ is still set to kInvalidBase, we got here // before VDSOSupport::Init has been called. Call it now. @@ -65,7 +91,7 @@ VDSOSupport::VDSOSupport() // the operation should be idempotent. const void *VDSOSupport::Init() { const auto kInvalidBase = debugging_internal::ElfMemImage::kInvalidBase; -#if __GLIBC_PREREQ(2, 16) +#ifdef ABSL_HAVE_GETAUXVAL if (vdso_base_.load(std::memory_order_relaxed) == kInvalidBase) { errno = 0; const void *const sysinfo_ehdr = @@ -74,7 +100,7 @@ const void *VDSOSupport::Init() { vdso_base_.store(sysinfo_ehdr, std::memory_order_relaxed); } } -#endif // __GLIBC_PREREQ(2, 16) +#endif // ABSL_HAVE_GETAUXVAL if (vdso_base_.load(std::memory_order_relaxed) == kInvalidBase) { int fd = open("/proc/self/auxv", O_RDONLY); if (fd == -1) { @@ -86,8 +112,13 @@ const void *VDSOSupport::Init() { ElfW(auxv_t) aux; while (read(fd, &aux, sizeof(aux)) == sizeof(aux)) { if (aux.a_type == AT_SYSINFO_EHDR) { +#if defined(__NetBSD__) + vdso_base_.store(reinterpret_cast<void *>(aux.a_v), + std::memory_order_relaxed); +#else vdso_base_.store(reinterpret_cast<void *>(aux.a_un.a_val), std::memory_order_relaxed); +#endif break; } } diff --git a/absl/debugging/leak_check.cc b/absl/debugging/leak_check.cc index 764ca0ad..195e82bf 100644 --- a/absl/debugging/leak_check.cc +++ b/absl/debugging/leak_check.cc @@ -11,29 +11,19 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. - +// // Wrappers around lsan_interface functions. -// When lsan is not linked in, these functions are not available, -// therefore Abseil code which depends on these functions is conditioned on the -// definition of LEAK_SANITIZER. -#include "absl/base/attributes.h" -#include "absl/debugging/leak_check.h" +// +// These are always-available run-time functions manipulating the LeakSanitizer, +// even when the lsan_interface (and LeakSanitizer) is not available. When +// LeakSanitizer is not linked in, these functions become no-op stubs. -#ifndef LEAK_SANITIZER +#include "absl/debugging/leak_check.h" -namespace absl { -ABSL_NAMESPACE_BEGIN -bool HaveLeakSanitizer() { return false; } -bool LeakCheckerIsActive() { return false; } -void DoIgnoreLeak(const void*) { } -void RegisterLivePointers(const void*, size_t) { } -void UnRegisterLivePointers(const void*, size_t) { } -LeakCheckDisabler::LeakCheckDisabler() { } -LeakCheckDisabler::~LeakCheckDisabler() { } -ABSL_NAMESPACE_END -} // namespace absl +#include "absl/base/attributes.h" +#include "absl/base/config.h" -#else +#if defined(ABSL_HAVE_LEAK_SANITIZER) #include <sanitizer/lsan_interface.h> @@ -66,4 +56,18 @@ LeakCheckDisabler::~LeakCheckDisabler() { __lsan_enable(); } ABSL_NAMESPACE_END } // namespace absl -#endif // LEAK_SANITIZER +#else // defined(ABSL_HAVE_LEAK_SANITIZER) + +namespace absl { +ABSL_NAMESPACE_BEGIN +bool HaveLeakSanitizer() { return false; } +bool LeakCheckerIsActive() { return false; } +void DoIgnoreLeak(const void*) { } +void RegisterLivePointers(const void*, size_t) { } +void UnRegisterLivePointers(const void*, size_t) { } +LeakCheckDisabler::LeakCheckDisabler() { } +LeakCheckDisabler::~LeakCheckDisabler() { } +ABSL_NAMESPACE_END +} // namespace absl + +#endif // defined(ABSL_HAVE_LEAK_SANITIZER) diff --git a/absl/debugging/leak_check.h b/absl/debugging/leak_check.h index 5fc2b052..eff162f6 100644 --- a/absl/debugging/leak_check.h +++ b/absl/debugging/leak_check.h @@ -24,7 +24,24 @@ // Note: this leak checking API is not yet supported in MSVC. // Leak checking is enabled by default in all ASan builds. // -// See https://github.com/google/sanitizers/wiki/AddressSanitizerLeakSanitizer +// https://clang.llvm.org/docs/LeakSanitizer.html +// https://github.com/google/sanitizers/wiki/AddressSanitizerLeakSanitizer +// +// GCC and Clang both automatically enable LeakSanitizer when AddressSanitizer +// is enabled. To use the mode, simply pass `-fsanitize=address` to both the +// compiler and linker. An example Bazel command could be +// +// $ bazel test --copt=-fsanitize=address --linkopt=-fsanitize=address ... +// +// GCC and Clang auto support a standalone LeakSanitizer mode (a mode which does +// not also use AddressSanitizer). To use the mode, simply pass +// `-fsanitize=leak` to both the compiler and linker. Since GCC does not +// currently provide a way of detecting this mode at compile-time, GCC users +// must also pass -DLEAK_SANIITIZER to the compiler. An example Bazel command +// could be +// +// $ bazel test --copt=-DLEAK_SANITIZER --copt=-fsanitize=leak +// --linkopt=-fsanitize=leak ... // // ----------------------------------------------------------------------------- #ifndef ABSL_DEBUGGING_LEAK_CHECK_H_ diff --git a/absl/debugging/leak_check_test.cc b/absl/debugging/leak_check_test.cc index 9fcfc8e5..6a42e31b 100644 --- a/absl/debugging/leak_check_test.cc +++ b/absl/debugging/leak_check_test.cc @@ -15,27 +15,24 @@ #include <string> #include "gtest/gtest.h" +#include "absl/base/config.h" #include "absl/base/internal/raw_logging.h" #include "absl/debugging/leak_check.h" namespace { -TEST(LeakCheckTest, DetectLeakSanitizer) { -#ifdef ABSL_EXPECT_LEAK_SANITIZER - EXPECT_TRUE(absl::HaveLeakSanitizer()); - EXPECT_TRUE(absl::LeakCheckerIsActive()); -#else - EXPECT_FALSE(absl::HaveLeakSanitizer()); - EXPECT_FALSE(absl::LeakCheckerIsActive()); -#endif -} - TEST(LeakCheckTest, IgnoreLeakSuppressesLeakedMemoryErrors) { + if (!absl::LeakCheckerIsActive()) { + GTEST_SKIP() << "LeakChecker is not active"; + } auto foo = absl::IgnoreLeak(new std::string("some ignored leaked string")); ABSL_RAW_LOG(INFO, "Ignoring leaked string %s", foo->c_str()); } TEST(LeakCheckTest, LeakCheckDisablerIgnoresLeak) { + if (!absl::LeakCheckerIsActive()) { + GTEST_SKIP() << "LeakChecker is not active"; + } absl::LeakCheckDisabler disabler; auto foo = new std::string("some string leaked while checks are disabled"); ABSL_RAW_LOG(INFO, "Ignoring leaked string %s", foo->c_str()); diff --git a/absl/debugging/stacktrace.cc b/absl/debugging/stacktrace.cc index 1f7c7d82..ff8069f8 100644 --- a/absl/debugging/stacktrace.cc +++ b/absl/debugging/stacktrace.cc @@ -49,8 +49,10 @@ # include "absl/debugging/internal/stacktrace_aarch64-inl.inc" # include "absl/debugging/internal/stacktrace_arm-inl.inc" +# include "absl/debugging/internal/stacktrace_emscripten-inl.inc" # include "absl/debugging/internal/stacktrace_generic-inl.inc" # include "absl/debugging/internal/stacktrace_powerpc-inl.inc" +# include "absl/debugging/internal/stacktrace_riscv-inl.inc" # include "absl/debugging/internal/stacktrace_unimplemented-inl.inc" # include "absl/debugging/internal/stacktrace_win32-inl.inc" # include "absl/debugging/internal/stacktrace_x86-inl.inc" diff --git a/absl/debugging/stacktrace_benchmark.cc b/absl/debugging/stacktrace_benchmark.cc new file mode 100644 index 00000000..9360bafe --- /dev/null +++ b/absl/debugging/stacktrace_benchmark.cc @@ -0,0 +1,55 @@ +// Copyright 2022 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/base/attributes.h" +#include "absl/base/config.h" +#include "absl/base/optimization.h" +#include "absl/debugging/stacktrace.h" +#include "benchmark/benchmark.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace { + +static constexpr int kMaxStackDepth = 100; +static constexpr int kCacheSize = (1 << 16); +void* pcs[kMaxStackDepth]; + +ABSL_ATTRIBUTE_NOINLINE void func(benchmark::State& state, int x, int depth) { + if (x <= 0) { + // Touch a significant amount of memory so that the stack is likely to be + // not cached in the L1 cache. + state.PauseTiming(); + int* arr = new int[kCacheSize]; + for (int i = 0; i < kCacheSize; ++i) benchmark::DoNotOptimize(arr[i] = 100); + delete[] arr; + state.ResumeTiming(); + benchmark::DoNotOptimize(absl::GetStackTrace(pcs, depth, 0)); + return; + } + ABSL_BLOCK_TAIL_CALL_OPTIMIZATION(); + func(state, --x, depth); +} + +void BM_GetStackTrace(benchmark::State& state) { + int depth = state.range(0); + for (auto s : state) { + func(state, depth, depth); + } +} + +BENCHMARK(BM_GetStackTrace)->DenseRange(10, kMaxStackDepth, 10); +} // namespace +ABSL_NAMESPACE_END +} // namespace absl diff --git a/absl/debugging/symbolize.cc b/absl/debugging/symbolize.cc index 5e4a25d6..638d3954 100644 --- a/absl/debugging/symbolize.cc +++ b/absl/debugging/symbolize.cc @@ -23,6 +23,11 @@ #endif #endif +// Emscripten symbolization relies on JS. Do not use them in standalone mode. +#if defined(__EMSCRIPTEN__) && !defined(STANDALONE_WASM) +#define ABSL_INTERNAL_HAVE_SYMBOLIZE_WASM +#endif + #if defined(ABSL_INTERNAL_HAVE_ELF_SYMBOLIZE) #include "absl/debugging/symbolize_elf.inc" #elif defined(ABSL_INTERNAL_HAVE_SYMBOLIZE_WIN32) @@ -31,6 +36,8 @@ #include "absl/debugging/symbolize_win32.inc" #elif defined(__APPLE__) #include "absl/debugging/symbolize_darwin.inc" +#elif defined(ABSL_INTERNAL_HAVE_SYMBOLIZE_WASM) +#include "absl/debugging/symbolize_emscripten.inc" #else #include "absl/debugging/symbolize_unimplemented.inc" #endif diff --git a/absl/debugging/symbolize_elf.inc b/absl/debugging/symbolize_elf.inc index f4d5727b..9bfdd915 100644 --- a/absl/debugging/symbolize_elf.inc +++ b/absl/debugging/symbolize_elf.inc @@ -77,6 +77,10 @@ #include "absl/debugging/internal/vdso_support.h" #include "absl/strings/string_view.h" +#if defined(__FreeBSD__) && !defined(ElfW) +#define ElfW(x) __ElfN(x) +#endif + namespace absl { ABSL_NAMESPACE_BEGIN @@ -319,6 +323,7 @@ class Symbolizer { const ptrdiff_t relocation, char *out, int out_size, char *tmp_buf, int tmp_buf_size); + const char *GetUncachedSymbol(const void *pc); enum { SYMBOL_BUF_SIZE = 3072, @@ -701,6 +706,16 @@ static ABSL_ATTRIBUTE_NOINLINE FindSymbolResult FindSymbol( const char *start_address = ComputeOffset(original_start_address, relocation); +#ifdef __arm__ + // ARM functions are always aligned to multiples of two bytes; the + // lowest-order bit in start_address is ignored by the CPU and indicates + // whether the function contains ARM (0) or Thumb (1) code. We don't care + // about what encoding is being used; we just want the real start address + // of the function. + start_address = reinterpret_cast<const char *>( + reinterpret_cast<uintptr_t>(start_address) & ~1); +#endif + if (deref_function_descriptor_pointer && InSection(original_start_address, opd)) { // The opd section is mapped into memory. Just dereference @@ -1131,6 +1146,14 @@ bool Symbolizer::RegisterObjFile(const char *filename, reinterpret_cast<uintptr_t>(old->end_addr), old->filename); } return true; + } else if (old->end_addr == start_addr && + reinterpret_cast<uintptr_t>(old->start_addr) - old->offset == + reinterpret_cast<uintptr_t>(start_addr) - offset && + strcmp(old->filename, filename) == 0) { + // Two contiguous map entries that span a contiguous region of the file, + // perhaps because some part of the file was mlock()ed. Combine them. + old->end_addr = end_addr; + return true; } } ObjFile *obj = impl->addr_map_.Add(); @@ -1319,13 +1342,7 @@ static bool MaybeInitializeObjFile(ObjFile *obj) { // they are called here as well. // To keep stack consumption low, we would like this function to not // get inlined. -const char *Symbolizer::GetSymbol(const void *const pc) { - const char *entry = FindSymbolInCache(pc); - if (entry != nullptr) { - return entry; - } - symbol_buf_[0] = '\0'; - +const char *Symbolizer::GetUncachedSymbol(const void *pc) { ObjFile *const obj = FindObjFile(pc, 1); ptrdiff_t relocation = 0; int fd = -1; @@ -1413,6 +1430,42 @@ const char *Symbolizer::GetSymbol(const void *const pc) { return InsertSymbolInCache(pc, symbol_buf_); } +const char *Symbolizer::GetSymbol(const void *pc) { + const char *entry = FindSymbolInCache(pc); + if (entry != nullptr) { + return entry; + } + symbol_buf_[0] = '\0'; + +#ifdef __hppa__ + { + // In some contexts (e.g., return addresses), PA-RISC uses the lowest two + // bits of the address to indicate the privilege level. Clear those bits + // before trying to symbolize. + const auto pc_bits = reinterpret_cast<uintptr_t>(pc); + const auto address = pc_bits & ~0x3; + entry = GetUncachedSymbol(reinterpret_cast<const void *>(address)); + if (entry != nullptr) { + return entry; + } + + // In some contexts, PA-RISC also uses bit 1 of the address to indicate that + // this is a cross-DSO function pointer. Such function pointers actually + // point to a procedure label, a struct whose first 32-bit (pointer) element + // actually points to the function text. With no symbol found for this + // address so far, try interpreting it as a cross-DSO function pointer and + // see how that goes. + if (pc_bits & 0x2) { + return GetUncachedSymbol(*reinterpret_cast<const void *const *>(address)); + } + + return nullptr; + } +#else + return GetUncachedSymbol(pc); +#endif +} + bool RemoveAllSymbolDecorators(void) { if (!g_decorators_mu.TryLock()) { // Someone else is using decorators. Get out. diff --git a/absl/debugging/symbolize_emscripten.inc b/absl/debugging/symbolize_emscripten.inc new file mode 100644 index 00000000..c226c456 --- /dev/null +++ b/absl/debugging/symbolize_emscripten.inc @@ -0,0 +1,72 @@ +// Copyright 2020 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <cxxabi.h> +#include <emscripten.h> + +#include <algorithm> +#include <cstring> + +#include "absl/base/internal/raw_logging.h" +#include "absl/debugging/internal/demangle.h" +#include "absl/strings/numbers.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" + +extern "C" { +const char* emscripten_pc_get_function(const void* pc); +} + +// clang-format off +EM_JS(bool, HaveOffsetConverter, (), + { return typeof wasmOffsetConverter !== 'undefined'; }); +// clang-format on + +namespace absl { +ABSL_NAMESPACE_BEGIN + +void InitializeSymbolizer(const char*) { + if (!HaveOffsetConverter()) { + ABSL_RAW_LOG(INFO, + "Symbolization unavailable. Rebuild with -sWASM=1 " + "and -sUSE_OFFSET_CONVERTER=1."); + } +} + +bool Symbolize(const void* pc, char* out, int out_size) { + // Check if we have the offset converter necessary for pc_get_function. + // Without it, the program will abort(). + if (!HaveOffsetConverter()) { + return false; + } + const char* func_name = emscripten_pc_get_function(pc); + if (func_name == nullptr) { + return false; + } + + strncpy(out, func_name, out_size); + + if (out[out_size - 1] != '\0') { + // strncpy() does not '\0' terminate when it truncates. + static constexpr char kEllipsis[] = "..."; + int ellipsis_size = std::min<int>(sizeof(kEllipsis) - 1, out_size - 1); + memcpy(out + out_size - ellipsis_size - 1, kEllipsis, ellipsis_size); + out[out_size - 1] = '\0'; + } + + return true; +} + +ABSL_NAMESPACE_END +} // namespace absl diff --git a/absl/debugging/symbolize_test.cc b/absl/debugging/symbolize_test.cc index a2dd4956..3165c6ed 100644 --- a/absl/debugging/symbolize_test.cc +++ b/absl/debugging/symbolize_test.cc @@ -146,8 +146,22 @@ static const char *TrySymbolize(void *pc) { return TrySymbolizeWithLimit(pc, sizeof(try_symbolize_buffer)); } -#if defined(ABSL_INTERNAL_HAVE_ELF_SYMBOLIZE) || \ - defined(ABSL_INTERNAL_HAVE_DARWIN_SYMBOLIZE) +#if defined(ABSL_INTERNAL_HAVE_ELF_SYMBOLIZE) || \ + defined(ABSL_INTERNAL_HAVE_DARWIN_SYMBOLIZE) || \ + defined(ABSL_INTERNAL_HAVE_EMSCRIPTEN_SYMBOLIZE) + +// Test with a return address. +void ABSL_ATTRIBUTE_NOINLINE TestWithReturnAddress() { +#if defined(ABSL_HAVE_ATTRIBUTE_NOINLINE) + void *return_address = __builtin_return_address(0); + const char *symbol = TrySymbolize(return_address); + ABSL_RAW_CHECK(symbol != nullptr, "TestWithReturnAddress failed"); + ABSL_RAW_CHECK(strcmp(symbol, "main") == 0, "TestWithReturnAddress failed"); + std::cout << "TestWithReturnAddress passed" << std::endl; +#endif +} + +#ifndef ABSL_INTERNAL_HAVE_EMSCRIPTEN_SYMBOLIZE TEST(Symbolize, Cached) { // Compilers should give us pointers to them. @@ -378,12 +392,14 @@ TEST(Symbolize, InstallAndRemoveSymbolDecorators) { DummySymbolDecorator, &c_message), 0); - char *address = reinterpret_cast<char *>(1); - EXPECT_STREQ("abc", TrySymbolize(address++)); + // Use addresses 4 and 8 here to ensure that we always use valid addresses + // even on systems that require instructions to be 32-bit aligned. + char *address = reinterpret_cast<char *>(4); + EXPECT_STREQ("abc", TrySymbolize(address)); EXPECT_TRUE(absl::debugging_internal::RemoveSymbolDecorator(ticket_b)); - EXPECT_STREQ("ac", TrySymbolize(address++)); + EXPECT_STREQ("ac", TrySymbolize(address + 4)); // Cleanup: remove all remaining decorators so other stack traces don't // get mystery "ac" decoration. @@ -418,6 +434,7 @@ TEST(Symbolize, ForEachSection) { close(fd); } #endif // !ABSL_INTERNAL_HAVE_DARWIN_SYMBOLIZE +#endif // !ABSL_INTERNAL_HAVE_EMSCRIPTEN_SYMBOLIZE // x86 specific tests. Uses some inline assembler. extern "C" { @@ -466,17 +483,52 @@ void ABSL_ATTRIBUTE_NOINLINE TestWithPCInsideInlineFunction() { } } -// Test with a return address. -void ABSL_ATTRIBUTE_NOINLINE TestWithReturnAddress() { +#if defined(__arm__) && ABSL_HAVE_ATTRIBUTE(target) && \ + ((__ARM_ARCH >= 7) || !defined(__ARM_PCS_VFP)) +// Test that we correctly identify bounds of Thumb functions on ARM. +// +// Thumb functions have the lowest-order bit set in their addresses in the ELF +// symbol table. This requires some extra logic to properly compute function +// bounds. To test this logic, nudge a Thumb function right up against an ARM +// function and try to symbolize the ARM function. +// +// A naive implementation will simply use the Thumb function's entry point as +// written in the symbol table and will therefore treat the Thumb function as +// extending one byte further in the instruction stream than it actually does. +// When asked to symbolize the start of the ARM function, it will identify an +// overlap between the Thumb and ARM functions, and it will return the name of +// the Thumb function. +// +// A correct implementation, on the other hand, will null out the lowest-order +// bit in the Thumb function's entry point. It will correctly compute the end of +// the Thumb function, it will find no overlap between the Thumb and ARM +// functions, and it will return the name of the ARM function. +// +// Unfortunately we cannot perform this test on armv6 or lower systems that use +// the hard float ABI because gcc refuses to compile thumb functions on such +// systems with a "sorry, unimplemented: Thumb-1 hard-float VFP ABI" error. + +__attribute__((target("thumb"))) int ArmThumbOverlapThumb(int x) { + return x * x * x; +} + +__attribute__((target("arm"))) int ArmThumbOverlapArm(int x) { + return x * x * x; +} + +void ABSL_ATTRIBUTE_NOINLINE TestArmThumbOverlap() { #if defined(ABSL_HAVE_ATTRIBUTE_NOINLINE) - void *return_address = __builtin_return_address(0); - const char *symbol = TrySymbolize(return_address); - ABSL_RAW_CHECK(symbol != nullptr, "TestWithReturnAddress failed"); - ABSL_RAW_CHECK(strcmp(symbol, "main") == 0, "TestWithReturnAddress failed"); - std::cout << "TestWithReturnAddress passed" << std::endl; + const char *symbol = TrySymbolize((void *)&ArmThumbOverlapArm); + ABSL_RAW_CHECK(symbol != nullptr, "TestArmThumbOverlap failed"); + ABSL_RAW_CHECK(strcmp("ArmThumbOverlapArm()", symbol) == 0, + "TestArmThumbOverlap failed"); + std::cout << "TestArmThumbOverlap passed" << std::endl; #endif } +#endif // defined(__arm__) && ABSL_HAVE_ATTRIBUTE(target) && ((__ARM_ARCH >= 7) + // || !defined(__ARM_PCS_VFP)) + #elif defined(_WIN32) #if !defined(ABSL_CONSUME_DLL) @@ -519,7 +571,6 @@ TEST(Symbolize, SymbolizeWithDemangling) { #endif // !defined(ABSL_CONSUME_DLL) #else // Symbolizer unimplemented - TEST(Symbolize, Unimplemented) { char buf[64]; EXPECT_FALSE(absl::Symbolize((void *)(&nonstatic_func), buf, sizeof(buf))); @@ -551,6 +602,10 @@ int main(int argc, char **argv) { TestWithPCInsideInlineFunction(); TestWithPCInsideNonInlineFunction(); TestWithReturnAddress(); +#if defined(__arm__) && ABSL_HAVE_ATTRIBUTE(target) && \ + ((__ARM_ARCH >= 7) || !defined(__ARM_PCS_VFP)) + TestArmThumbOverlap(); +#endif #endif return RUN_ALL_TESTS(); diff --git a/absl/flags/BUILD.bazel b/absl/flags/BUILD.bazel index 147249ed..4ca687ee 100644 --- a/absl/flags/BUILD.bazel +++ b/absl/flags/BUILD.bazel @@ -14,7 +14,6 @@ # limitations under the License. # -load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test") load( "//absl:copts/configure_copts.bzl", "ABSL_DEFAULT_COPTS", @@ -101,6 +100,7 @@ cc_library( "//absl/base:log_severity", "//absl/strings", "//absl/strings:str_format", + "//absl/types:optional", ], ) @@ -114,6 +114,9 @@ cc_library( ], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "//visibility:private", + ], deps = [ "//absl/base:config", "//absl/base:fast_type_id", @@ -205,6 +208,7 @@ cc_library( "//absl/base", "//absl/base:config", "//absl/base:core_headers", + "//absl/base:dynamic_annotations", "//absl/memory", "//absl/meta:type_traits", "//absl/strings", @@ -217,6 +221,7 @@ cc_library( name = "flag", srcs = [ "flag.cc", + "internal/flag_msvc.inc", ], hdrs = [ "declare.h", @@ -259,6 +264,7 @@ cc_library( ":reflection", "//absl/base:config", "//absl/base:core_headers", + "//absl/container:flat_hash_map", "//absl/strings", ], ) @@ -320,6 +326,11 @@ cc_test( ], copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + tags = [ + "no_test_android", + "no_test_ios", + "no_test_wasm", + ], deps = [ ":commandlineflag", ":commandlineflag_internal", @@ -356,6 +367,11 @@ cc_test( ], copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + tags = [ + "no_test_android", + "no_test_ios", + "no_test_wasm", + ], deps = [ ":config", ":flag", @@ -417,6 +433,11 @@ cc_test( ], copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + tags = [ + "no_test_android", + "no_test_ios", + "no_test_wasm", + ], deps = [ ":flag", ":parse", @@ -452,6 +473,7 @@ cc_test( ], copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + tags = ["no_test_wasm"], deps = [ ":program_name", "//absl/strings", @@ -467,6 +489,11 @@ cc_test( ], copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + tags = [ + "no_test_android", + "no_test_ios", + "no_test_wasm", + ], deps = [ ":commandlineflag_internal", ":flag", @@ -489,6 +516,7 @@ cc_test( copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, shard_count = 31, + tags = ["no_test_wasm"], deps = [ ":flag_internal", "//absl/base", @@ -523,6 +551,11 @@ cc_test( ], copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + tags = [ + "no_test_android", + "no_test_ios", + "no_test_wasm", + ], deps = [ ":config", ":flag", diff --git a/absl/flags/CMakeLists.txt b/absl/flags/CMakeLists.txt index caac69cf..3e9d5adf 100644 --- a/absl/flags/CMakeLists.txt +++ b/absl/flags/CMakeLists.txt @@ -87,6 +87,7 @@ absl_cc_library( absl::config absl::core_headers absl::log_severity + absl::optional absl::strings absl::str_format ) @@ -105,6 +106,7 @@ absl_cc_library( ${ABSL_DEFAULT_LINKOPTS} DEPS absl::config + absl::dynamic_annotations absl::fast_type_id ) @@ -202,6 +204,7 @@ absl_cc_library( HDRS "declare.h" "flag.h" + "internal/flag_msvc.inc" COPTS ${ABSL_DEFAULT_COPTS} LINKOPTS @@ -239,6 +242,7 @@ absl_cc_library( absl::flags_private_handle_accessor absl::flags_program_name absl::flags_reflection + absl::flat_hash_map absl::strings absl::synchronization ) @@ -309,7 +313,7 @@ absl_cc_test( absl::flags_reflection absl::memory absl::strings - gtest_main + GTest::gtest_main ) absl_cc_test( @@ -321,7 +325,7 @@ absl_cc_test( ${ABSL_TEST_COPTS} DEPS absl::flags_config - gtest_main + GTest::gtest_main ) absl_cc_test( @@ -341,7 +345,7 @@ absl_cc_test( absl::flags_reflection absl::strings absl::time - gtest_main + GTest::gtest_main ) absl_cc_test( @@ -353,7 +357,7 @@ absl_cc_test( ${ABSL_TEST_COPTS} DEPS absl::flags_marshalling - gtest_main + GTest::gtest_main ) absl_cc_test( @@ -372,7 +376,7 @@ absl_cc_test( absl::scoped_set_env absl::span absl::strings - gmock_main + GTest::gmock_main ) absl_cc_test( @@ -384,7 +388,7 @@ absl_cc_test( ${ABSL_TEST_COPTS} DEPS absl::flags_path_util - gtest_main + GTest::gtest_main ) absl_cc_test( @@ -397,7 +401,7 @@ absl_cc_test( DEPS absl::flags_program_name absl::strings - gtest_main + GTest::gtest_main ) absl_cc_test( @@ -414,7 +418,7 @@ absl_cc_test( absl::flags_usage absl::memory absl::strings - gmock_main + GTest::gmock_main ) absl_cc_test( @@ -428,7 +432,7 @@ absl_cc_test( absl::base absl::flags_internal absl::time - gmock_main + GTest::gmock_main ) absl_cc_test( @@ -443,7 +447,7 @@ absl_cc_test( absl::flags_path_util absl::flags_program_name absl::strings - gtest_main + GTest::gtest_main ) absl_cc_test( @@ -462,5 +466,5 @@ absl_cc_test( absl::flags_reflection absl::flags_usage absl::strings - gtest + GTest::gmock ) diff --git a/absl/flags/config.h b/absl/flags/config.h index 5ab1f311..14c4235b 100644 --- a/absl/flags/config.h +++ b/absl/flags/config.h @@ -45,14 +45,6 @@ #define ABSL_FLAGS_STRIP_HELP ABSL_FLAGS_STRIP_NAMES #endif -// ABSL_FLAGS_INTERNAL_HAS_RTTI macro is used for selecting if we can use RTTI -// for flag type identification. -#ifdef ABSL_FLAGS_INTERNAL_HAS_RTTI -#error ABSL_FLAGS_INTERNAL_HAS_RTTI cannot be directly set -#elif !defined(__GNUC__) || defined(__GXX_RTTI) -#define ABSL_FLAGS_INTERNAL_HAS_RTTI 1 -#endif // !defined(__GNUC__) || defined(__GXX_RTTI) - // These macros represent the "source of truth" for the list of supported // built-in types. #define ABSL_FLAGS_INTERNAL_BUILTIN_TYPES(A) \ diff --git a/absl/flags/declare.h b/absl/flags/declare.h index b9794d8b..d1437bb9 100644 --- a/absl/flags/declare.h +++ b/absl/flags/declare.h @@ -60,6 +60,14 @@ ABSL_NAMESPACE_END // The ABSL_DECLARE_FLAG(type, name) macro expands to: // // extern absl::Flag<type> FLAGS_name; -#define ABSL_DECLARE_FLAG(type, name) extern ::absl::Flag<type> FLAGS_##name +#define ABSL_DECLARE_FLAG(type, name) ABSL_DECLARE_FLAG_INTERNAL(type, name) + +// Internal implementation of ABSL_DECLARE_FLAG to allow macro expansion of its +// arguments. Clients must use ABSL_DECLARE_FLAG instead. +#define ABSL_DECLARE_FLAG_INTERNAL(type, name) \ + extern absl::Flag<type> FLAGS_##name; \ + namespace absl /* block flags in namespaces */ {} \ + /* second redeclaration is to allow applying attributes */ \ + extern absl::Flag<type> FLAGS_##name #endif // ABSL_FLAGS_DECLARE_H_ diff --git a/absl/flags/flag.h b/absl/flags/flag.h index f09580b0..b7f94be7 100644 --- a/absl/flags/flag.h +++ b/absl/flags/flag.h @@ -67,105 +67,15 @@ ABSL_NAMESPACE_BEGIN // ABSL_FLAG(int, count, 0, "Count of items to process"); // // No public methods of `absl::Flag<T>` are part of the Abseil Flags API. +// +// For type support of Abseil Flags, see the marshalling.h header file, which +// discusses supported standard types, optional flags, and additional Abseil +// type support. #if !defined(_MSC_VER) || defined(__clang__) template <typename T> using Flag = flags_internal::Flag<T>; #else -// MSVC debug builds do not implement initialization with constexpr constructors -// correctly. To work around this we add a level of indirection, so that the -// class `absl::Flag` contains an `internal::Flag*` (instead of being an alias -// to that class) and dynamically allocates an instance when necessary. We also -// forward all calls to internal::Flag methods via trampoline methods. In this -// setup the `absl::Flag` class does not have constructor and virtual methods, -// all the data members are public and thus MSVC is able to initialize it at -// link time. To deal with multiple threads accessing the flag for the first -// time concurrently we use an atomic boolean indicating if flag object is -// initialized. We also employ the double-checked locking pattern where the -// second level of protection is a global Mutex, so if two threads attempt to -// construct the flag concurrently only one wins. -// This solution is based on a recomendation here: -// https://developercommunity.visualstudio.com/content/problem/336946/class-with-constexpr-constructor-not-using-static.html?childToView=648454#comment-648454 - -namespace flags_internal { -absl::Mutex* GetGlobalConstructionGuard(); -} // namespace flags_internal - -template <typename T> -class Flag { - public: - // No constructor and destructor to ensure this is an aggregate type. - // Visual Studio 2015 still requires the constructor for class to be - // constexpr initializable. -#if _MSC_VER <= 1900 - constexpr Flag(const char* name, const char* filename, - const flags_internal::HelpGenFunc help_gen, - const flags_internal::FlagDfltGenFunc default_value_gen) - : name_(name), - filename_(filename), - help_gen_(help_gen), - default_value_gen_(default_value_gen), - inited_(false), - impl_(nullptr) {} -#endif - - flags_internal::Flag<T>& GetImpl() const { - if (!inited_.load(std::memory_order_acquire)) { - absl::MutexLock l(flags_internal::GetGlobalConstructionGuard()); - - if (inited_.load(std::memory_order_acquire)) { - return *impl_; - } - - impl_ = new flags_internal::Flag<T>( - name_, filename_, - {flags_internal::FlagHelpMsg(help_gen_), - flags_internal::FlagHelpKind::kGenFunc}, - {flags_internal::FlagDefaultSrc(default_value_gen_), - flags_internal::FlagDefaultKind::kGenFunc}); - inited_.store(true, std::memory_order_release); - } - - return *impl_; - } - - // Public methods of `absl::Flag<T>` are NOT part of the Abseil Flags API. - // See https://abseil.io/docs/cpp/guides/flags - bool IsRetired() const { return GetImpl().IsRetired(); } - absl::string_view Name() const { return GetImpl().Name(); } - std::string Help() const { return GetImpl().Help(); } - bool IsModified() const { return GetImpl().IsModified(); } - bool IsSpecifiedOnCommandLine() const { - return GetImpl().IsSpecifiedOnCommandLine(); - } - std::string Filename() const { return GetImpl().Filename(); } - std::string DefaultValue() const { return GetImpl().DefaultValue(); } - std::string CurrentValue() const { return GetImpl().CurrentValue(); } - template <typename U> - inline bool IsOfType() const { - return GetImpl().template IsOfType<U>(); - } - T Get() const { - return flags_internal::FlagImplPeer::InvokeGet<T>(GetImpl()); - } - void Set(const T& v) { - flags_internal::FlagImplPeer::InvokeSet(GetImpl(), v); - } - void InvokeCallback() { GetImpl().InvokeCallback(); } - - const CommandLineFlag& Reflect() const { - return flags_internal::FlagImplPeer::InvokeReflect(GetImpl()); - } - - // The data members are logically private, but they need to be public for - // this to be an aggregate type. - const char* name_; - const char* filename_; - const flags_internal::HelpGenFunc help_gen_; - const flags_internal::FlagDfltGenFunc default_value_gen_; - - mutable std::atomic<bool> inited_; - mutable flags_internal::Flag<T>* impl_; -}; +#include "absl/flags/internal/flag_msvc.inc" #endif // GetFlag() @@ -265,6 +175,8 @@ ABSL_NAMESPACE_END // // ABSL_FLAG(T, name, default_value, help).OnUpdate(callback); // +// `callback` should be convertible to `void (*)()`. +// // After any setting of the flag value, the callback will be called at least // once. A rapid sequence of changes may be merged together into the same // callback. No concurrent calls to the callback will be made for the same @@ -279,7 +191,6 @@ ABSL_NAMESPACE_END // Note: ABSL_FLAG.OnUpdate() does not have a public definition. Hence, this // comment serves as its API documentation. - // ----------------------------------------------------------------------------- // Implementation details below this section // ----------------------------------------------------------------------------- @@ -334,8 +245,8 @@ ABSL_NAMESPACE_END /* default value argument. That keeps temporaries alive */ \ /* long enough for NonConst to work correctly. */ \ static constexpr absl::string_view Value( \ - absl::string_view v = ABSL_FLAG_IMPL_FLAGHELP(txt)) { \ - return v; \ + absl::string_view absl_flag_help = ABSL_FLAG_IMPL_FLAGHELP(txt)) { \ + return absl_flag_help; \ } \ static std::string NonConst() { return std::string(Value()); } \ }; \ @@ -347,8 +258,8 @@ ABSL_NAMESPACE_END #define ABSL_FLAG_IMPL_DECLARE_DEF_VAL_WRAPPER(name, Type, default_value) \ struct AbslFlagDefaultGenFor##name { \ Type value = absl::flags_internal::InitDefaultValue<Type>(default_value); \ - static void Gen(void* p) { \ - new (p) Type(AbslFlagDefaultGenFor##name{}.value); \ + static void Gen(void* absl_flag_default_loc) { \ + new (absl_flag_default_loc) Type(AbslFlagDefaultGenFor##name{}.value); \ } \ }; @@ -358,6 +269,7 @@ ABSL_NAMESPACE_END // global name for FLAGS_no<flag_name> symbol, thus preventing the possibility // of defining two flags with names foo and nofoo. #define ABSL_FLAG_IMPL(Type, name, default_value, help) \ + extern ::absl::Flag<Type> FLAGS_##name; \ namespace absl /* block flags in namespaces */ {} \ ABSL_FLAG_IMPL_DECLARE_DEF_VAL_WRAPPER(name, Type, default_value) \ ABSL_FLAG_IMPL_DECLARE_HELP_WRAPPER(name, help) \ diff --git a/absl/flags/flag_benchmark.cc b/absl/flags/flag_benchmark.cc index 57584f85..fc572d9c 100644 --- a/absl/flags/flag_benchmark.cc +++ b/absl/flags/flag_benchmark.cc @@ -101,7 +101,39 @@ std::string AbslUnparseFlag(const UDT&) { return ""; } A(AbslDuration) \ A(UDT) -#define FLAG_DEF(T) ABSL_FLAG(T, T##_flag, {}, ""); +#define REPLICATE_0(A, T, name, index) A(T, name, index) +#define REPLICATE_1(A, T, name, index) \ + REPLICATE_0(A, T, name, index##0) REPLICATE_0(A, T, name, index##1) +#define REPLICATE_2(A, T, name, index) \ + REPLICATE_1(A, T, name, index##0) REPLICATE_1(A, T, name, index##1) +#define REPLICATE_3(A, T, name, index) \ + REPLICATE_2(A, T, name, index##0) REPLICATE_2(A, T, name, index##1) +#define REPLICATE_4(A, T, name, index) \ + REPLICATE_3(A, T, name, index##0) REPLICATE_3(A, T, name, index##1) +#define REPLICATE_5(A, T, name, index) \ + REPLICATE_4(A, T, name, index##0) REPLICATE_4(A, T, name, index##1) +#define REPLICATE_6(A, T, name, index) \ + REPLICATE_5(A, T, name, index##0) REPLICATE_5(A, T, name, index##1) +#define REPLICATE_7(A, T, name, index) \ + REPLICATE_6(A, T, name, index##0) REPLICATE_6(A, T, name, index##1) +#define REPLICATE_8(A, T, name, index) \ + REPLICATE_7(A, T, name, index##0) REPLICATE_7(A, T, name, index##1) +#define REPLICATE_9(A, T, name, index) \ + REPLICATE_8(A, T, name, index##0) REPLICATE_8(A, T, name, index##1) +#if defined(_MSC_VER) +#define REPLICATE(A, T, name) \ + REPLICATE_7(A, T, name, 0) REPLICATE_7(A, T, name, 1) +#define SINGLE_FLAG(T) FLAGS_##T##_flag_00000000 +#else +#define REPLICATE(A, T, name) \ + REPLICATE_9(A, T, name, 0) REPLICATE_9(A, T, name, 1) +#define SINGLE_FLAG(T) FLAGS_##T##_flag_0000000000 +#endif +#define REPLICATE_ALL(A, T, name) \ + REPLICATE_9(A, T, name, 0) REPLICATE_9(A, T, name, 1) + +#define COUNT(T, name, index) +1 +constexpr size_t kNumFlags = 0 REPLICATE(COUNT, _, _); #if defined(__clang__) && defined(__linux__) // Force the flags used for benchmarks into a separate ELF section. @@ -110,38 +142,87 @@ std::string AbslUnparseFlag(const UDT&) { return ""; } // benchmark results more reproducible across unrelated code changes. #pragma clang section data = ".benchmark_flags" #endif +#define DEFINE_FLAG(T, name, index) ABSL_FLAG(T, name##_##index, {}, ""); +#define FLAG_DEF(T) REPLICATE(DEFINE_FLAG, T, T##_flag); BENCHMARKED_TYPES(FLAG_DEF) #if defined(__clang__) && defined(__linux__) #pragma clang section data = "" #endif // Register thousands of flags to bloat up the size of the registry. // This mimics real life production binaries. -#define DEFINE_FLAG_0(name) ABSL_FLAG(int, name, 0, ""); -#define DEFINE_FLAG_1(name) DEFINE_FLAG_0(name##0) DEFINE_FLAG_0(name##1) -#define DEFINE_FLAG_2(name) DEFINE_FLAG_1(name##0) DEFINE_FLAG_1(name##1) -#define DEFINE_FLAG_3(name) DEFINE_FLAG_2(name##0) DEFINE_FLAG_2(name##1) -#define DEFINE_FLAG_4(name) DEFINE_FLAG_3(name##0) DEFINE_FLAG_3(name##1) -#define DEFINE_FLAG_5(name) DEFINE_FLAG_4(name##0) DEFINE_FLAG_4(name##1) -#define DEFINE_FLAG_6(name) DEFINE_FLAG_5(name##0) DEFINE_FLAG_5(name##1) -#define DEFINE_FLAG_7(name) DEFINE_FLAG_6(name##0) DEFINE_FLAG_6(name##1) -#define DEFINE_FLAG_8(name) DEFINE_FLAG_7(name##0) DEFINE_FLAG_7(name##1) -#define DEFINE_FLAG_9(name) DEFINE_FLAG_8(name##0) DEFINE_FLAG_8(name##1) -#define DEFINE_FLAG_10(name) DEFINE_FLAG_9(name##0) DEFINE_FLAG_9(name##1) -#define DEFINE_FLAG_11(name) DEFINE_FLAG_10(name##0) DEFINE_FLAG_10(name##1) -#define DEFINE_FLAG_12(name) DEFINE_FLAG_11(name##0) DEFINE_FLAG_11(name##1) -DEFINE_FLAG_12(bloat_flag_); +#define BLOAT_FLAG(_unused1, _unused2, index) \ + ABSL_FLAG(int, bloat_flag_##index, 0, ""); +REPLICATE_ALL(BLOAT_FLAG, _, _) namespace { -#define BM_GetFlag(T) \ - void BM_GetFlag_##T(benchmark::State& state) { \ - for (auto _ : state) { \ - benchmark::DoNotOptimize(absl::GetFlag(FLAGS_##T##_flag)); \ - } \ - } \ - BENCHMARK(BM_GetFlag_##T)->ThreadRange(1, 16); +#define FLAG_PTR(T, name, index) &FLAGS_##name##_##index, +#define FLAG_PTR_ARR(T) \ + static constexpr absl::Flag<T>* FlagPtrs_##T[] = { \ + REPLICATE(FLAG_PTR, T, T##_flag)}; +BENCHMARKED_TYPES(FLAG_PTR_ARR) + +#define BM_SingleGetFlag(T) \ + void BM_SingleGetFlag_##T(benchmark::State& state) { \ + for (auto _ : state) { \ + benchmark::DoNotOptimize(absl::GetFlag(SINGLE_FLAG(T))); \ + } \ + } \ + BENCHMARK(BM_SingleGetFlag_##T)->ThreadRange(1, 16); + +BENCHMARKED_TYPES(BM_SingleGetFlag) + +template <typename T> +struct Accumulator { + using type = T; +}; +template <> +struct Accumulator<String> { + using type = size_t; +}; +template <> +struct Accumulator<VectorOfStrings> { + using type = size_t; +}; +template <> +struct Accumulator<OptionalInt> { + using type = bool; +}; +template <> +struct Accumulator<OptionalString> { + using type = bool; +}; +template <> +struct Accumulator<UDT> { + using type = bool; +}; + +template <typename T> +void Accumulate(typename Accumulator<T>::type& a, const T& f) { + a += f; +} +void Accumulate(bool& a, bool f) { a = a || f; } +void Accumulate(size_t& a, const std::string& f) { a += f.size(); } +void Accumulate(size_t& a, const std::vector<std::string>& f) { a += f.size(); } +void Accumulate(bool& a, const OptionalInt& f) { a |= f.has_value(); } +void Accumulate(bool& a, const OptionalString& f) { a |= f.has_value(); } +void Accumulate(bool& a, const UDT& f) { + a |= reinterpret_cast<int64_t>(&f) & 0x1; +} + +#define BM_ManyGetFlag(T) \ + void BM_ManyGetFlag_##T(benchmark::State& state) { \ + Accumulator<T>::type res = {}; \ + while (state.KeepRunningBatch(kNumFlags)) { \ + for (auto* flag_ptr : FlagPtrs_##T) { \ + Accumulate(res, absl::GetFlag(*flag_ptr)); \ + } \ + } \ + benchmark::DoNotOptimize(res); \ + } \ + BENCHMARK(BM_ManyGetFlag_##T)->ThreadRange(1, 8); -BENCHMARKED_TYPES(BM_GetFlag) +BENCHMARKED_TYPES(BM_ManyGetFlag) void BM_ThreadedFindCommandLineFlag(benchmark::State& state) { char dummy[] = "dummy"; @@ -150,17 +231,18 @@ void BM_ThreadedFindCommandLineFlag(benchmark::State& state) { // is finalized. absl::ParseCommandLine(1, argv); - for (auto s : state) { - benchmark::DoNotOptimize( - absl::FindCommandLineFlag("bloat_flag_010101010101")); + while (state.KeepRunningBatch(kNumFlags)) { + for (auto* flag_ptr : FlagPtrs_bool) { + benchmark::DoNotOptimize(absl::FindCommandLineFlag(flag_ptr->Name())); + } } } BENCHMARK(BM_ThreadedFindCommandLineFlag)->ThreadRange(1, 16); } // namespace -#define InvokeGetFlag(T) \ - T AbslInvokeGetFlag##T() { return absl::GetFlag(FLAGS_##T##_flag); } \ +#define InvokeGetFlag(T) \ + T AbslInvokeGetFlag##T() { return absl::GetFlag(SINGLE_FLAG(T)); } \ int odr##T = (benchmark::DoNotOptimize(AbslInvokeGetFlag##T), 1); BENCHMARKED_TYPES(InvokeGetFlag) diff --git a/absl/flags/flag_test.cc b/absl/flags/flag_test.cc index 6912b546..845b4eba 100644 --- a/absl/flags/flag_test.cc +++ b/absl/flags/flag_test.cc @@ -61,6 +61,7 @@ void TestCallback() {} struct UDT { UDT() = default; UDT(const UDT&) = default; + UDT& operator=(const UDT&) = default; }; bool AbslParseFlag(absl::string_view, UDT*, std::string*) { return true; } std::string AbslUnparseFlag(const UDT&) { return ""; } @@ -102,9 +103,9 @@ struct S2 { TEST_F(FlagTest, Traits) { EXPECT_EQ(flags::StorageKind<int>(), - flags::FlagValueStorageKind::kOneWordAtomic); + flags::FlagValueStorageKind::kValueAndInitBit); EXPECT_EQ(flags::StorageKind<bool>(), - flags::FlagValueStorageKind::kOneWordAtomic); + flags::FlagValueStorageKind::kValueAndInitBit); EXPECT_EQ(flags::StorageKind<double>(), flags::FlagValueStorageKind::kOneWordAtomic); EXPECT_EQ(flags::StorageKind<int64_t>(), @@ -723,6 +724,8 @@ ABSL_FLAG(CustomUDT, test_flag_custom_udt, CustomUDT(), "test flag custom UDT"); namespace { TEST_F(FlagTest, TestCustomUDT) { + EXPECT_EQ(flags::StorageKind<CustomUDT>(), + flags::FlagValueStorageKind::kOneWordAtomic); EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_custom_udt), CustomUDT(1, 1)); absl::SetFlag(&FLAGS_test_flag_custom_udt, CustomUDT(2, 3)); EXPECT_EQ(absl::GetFlag(FLAGS_test_flag_custom_udt), CustomUDT(2, 3)); @@ -851,7 +854,9 @@ ABSL_RETIRED_FLAG(bool, old_bool_flag, true, "old descr"); ABSL_RETIRED_FLAG(int, old_int_flag, (int)std::sqrt(10), "old descr"); ABSL_RETIRED_FLAG(std::string, old_str_flag, "", absl::StrCat("old ", "descr")); -bool initializaion_order_fiasco_test = [] { +namespace { + +bool initialization_order_fiasco_test ABSL_ATTRIBUTE_UNUSED = [] { // Iterate over all the flags during static initialization. // This should not trigger ASan's initialization-order-fiasco. auto* handle1 = absl::FindCommandLineFlag("flag_on_separate_file"); @@ -862,8 +867,6 @@ bool initializaion_order_fiasco_test = [] { return true; }(); -namespace { - TEST_F(FlagTest, TestRetiredFlagRegistration) { auto* handle = absl::FindCommandLineFlag("old_bool_flag"); EXPECT_TRUE(handle->IsOfType<bool>()); @@ -943,3 +946,221 @@ TEST_F(FlagTest, TestNonTriviallyCopyableUDT) { } } // namespace + +// -------------------------------------------------------------------- + +namespace { + +enum TestE { A = 1, B = 2, C = 3 }; + +struct EnumWrapper { + EnumWrapper() : e(A) {} + + TestE e; +}; + +bool AbslParseFlag(absl::string_view, EnumWrapper*, std::string*) { + return true; +} +std::string AbslUnparseFlag(const EnumWrapper&) { return ""; } + +} // namespace + +ABSL_FLAG(EnumWrapper, test_enum_wrapper_flag, {}, "help"); + +TEST_F(FlagTest, TesTypeWrappingEnum) { + EnumWrapper value = absl::GetFlag(FLAGS_test_enum_wrapper_flag); + EXPECT_EQ(value.e, A); + + value.e = B; + absl::SetFlag(&FLAGS_test_enum_wrapper_flag, value); + value = absl::GetFlag(FLAGS_test_enum_wrapper_flag); + EXPECT_EQ(value.e, B); +} + +// This is a compile test to ensure macros are expanded within ABSL_FLAG and +// ABSL_DECLARE_FLAG. +#define FLAG_NAME_MACRO(name) prefix_ ## name +ABSL_DECLARE_FLAG(int, FLAG_NAME_MACRO(test_macro_named_flag)); +ABSL_FLAG(int, FLAG_NAME_MACRO(test_macro_named_flag), 0, + "Testing macro expansion within ABSL_FLAG"); + +TEST_F(FlagTest, MacroWithinAbslFlag) { + EXPECT_EQ(absl::GetFlag(FLAGS_prefix_test_macro_named_flag), 0); + absl::SetFlag(&FLAGS_prefix_test_macro_named_flag, 1); + EXPECT_EQ(absl::GetFlag(FLAGS_prefix_test_macro_named_flag), 1); +} + +// -------------------------------------------------------------------- + +#if defined(__GNUC__) && !defined(__clang__) && __GNUC__ <= 5 +#define ABSL_SKIP_OPTIONAL_BOOL_TEST_DUE_TO_GCC_BUG +#endif + +#ifndef ABSL_SKIP_OPTIONAL_BOOL_TEST_DUE_TO_GCC_BUG +ABSL_FLAG(absl::optional<bool>, optional_bool, absl::nullopt, "help"); +#endif +ABSL_FLAG(absl::optional<int>, optional_int, {}, "help"); +ABSL_FLAG(absl::optional<double>, optional_double, 9.3, "help"); +ABSL_FLAG(absl::optional<std::string>, optional_string, absl::nullopt, "help"); +ABSL_FLAG(absl::optional<absl::Duration>, optional_duration, absl::nullopt, + "help"); +ABSL_FLAG(absl::optional<absl::optional<int>>, optional_optional_int, + absl::nullopt, "help"); +#if defined(ABSL_HAVE_STD_OPTIONAL) && !defined(ABSL_USES_STD_OPTIONAL) +ABSL_FLAG(std::optional<int64_t>, std_optional_int64, std::nullopt, "help"); +#endif + +namespace { + +#ifndef ABSL_SKIP_OPTIONAL_BOOL_TEST_DUE_TO_GCC_BUG +TEST_F(FlagTest, TestOptionalBool) { + EXPECT_FALSE(absl::GetFlag(FLAGS_optional_bool).has_value()); + EXPECT_EQ(absl::GetFlag(FLAGS_optional_bool), absl::nullopt); + + absl::SetFlag(&FLAGS_optional_bool, false); + EXPECT_TRUE(absl::GetFlag(FLAGS_optional_bool).has_value()); + EXPECT_EQ(absl::GetFlag(FLAGS_optional_bool), false); + + absl::SetFlag(&FLAGS_optional_bool, true); + EXPECT_TRUE(absl::GetFlag(FLAGS_optional_bool).has_value()); + EXPECT_EQ(absl::GetFlag(FLAGS_optional_bool), true); + + absl::SetFlag(&FLAGS_optional_bool, absl::nullopt); + EXPECT_FALSE(absl::GetFlag(FLAGS_optional_bool).has_value()); + EXPECT_EQ(absl::GetFlag(FLAGS_optional_bool), absl::nullopt); +} + +// -------------------------------------------------------------------- +#endif + +TEST_F(FlagTest, TestOptionalInt) { + EXPECT_FALSE(absl::GetFlag(FLAGS_optional_int).has_value()); + EXPECT_EQ(absl::GetFlag(FLAGS_optional_int), absl::nullopt); + + absl::SetFlag(&FLAGS_optional_int, 0); + EXPECT_TRUE(absl::GetFlag(FLAGS_optional_int).has_value()); + EXPECT_EQ(absl::GetFlag(FLAGS_optional_int), 0); + + absl::SetFlag(&FLAGS_optional_int, 10); + EXPECT_TRUE(absl::GetFlag(FLAGS_optional_int).has_value()); + EXPECT_EQ(absl::GetFlag(FLAGS_optional_int), 10); + + absl::SetFlag(&FLAGS_optional_int, absl::nullopt); + EXPECT_FALSE(absl::GetFlag(FLAGS_optional_int).has_value()); + EXPECT_EQ(absl::GetFlag(FLAGS_optional_int), absl::nullopt); +} + +// -------------------------------------------------------------------- + +TEST_F(FlagTest, TestOptionalDouble) { + EXPECT_TRUE(absl::GetFlag(FLAGS_optional_double).has_value()); + EXPECT_DOUBLE_EQ(*absl::GetFlag(FLAGS_optional_double), 9.3); + + absl::SetFlag(&FLAGS_optional_double, 0.0); + EXPECT_TRUE(absl::GetFlag(FLAGS_optional_double).has_value()); + EXPECT_EQ(absl::GetFlag(FLAGS_optional_double), 0.0); + + absl::SetFlag(&FLAGS_optional_double, 1.234); + EXPECT_TRUE(absl::GetFlag(FLAGS_optional_double).has_value()); + EXPECT_DOUBLE_EQ(*absl::GetFlag(FLAGS_optional_double), 1.234); + + absl::SetFlag(&FLAGS_optional_double, absl::nullopt); + EXPECT_FALSE(absl::GetFlag(FLAGS_optional_double).has_value()); + EXPECT_EQ(absl::GetFlag(FLAGS_optional_double), absl::nullopt); +} + +// -------------------------------------------------------------------- + +TEST_F(FlagTest, TestOptionalString) { + EXPECT_FALSE(absl::GetFlag(FLAGS_optional_string).has_value()); + EXPECT_EQ(absl::GetFlag(FLAGS_optional_string), absl::nullopt); + + // Setting optional string to "" leads to undefined behavior. + + absl::SetFlag(&FLAGS_optional_string, " "); + EXPECT_TRUE(absl::GetFlag(FLAGS_optional_string).has_value()); + EXPECT_EQ(absl::GetFlag(FLAGS_optional_string), " "); + + absl::SetFlag(&FLAGS_optional_string, "QWERTY"); + EXPECT_TRUE(absl::GetFlag(FLAGS_optional_string).has_value()); + EXPECT_EQ(absl::GetFlag(FLAGS_optional_string), "QWERTY"); + + absl::SetFlag(&FLAGS_optional_string, absl::nullopt); + EXPECT_FALSE(absl::GetFlag(FLAGS_optional_string).has_value()); + EXPECT_EQ(absl::GetFlag(FLAGS_optional_string), absl::nullopt); +} + +// -------------------------------------------------------------------- + +TEST_F(FlagTest, TestOptionalDuration) { + EXPECT_FALSE(absl::GetFlag(FLAGS_optional_duration).has_value()); + EXPECT_EQ(absl::GetFlag(FLAGS_optional_duration), absl::nullopt); + + absl::SetFlag(&FLAGS_optional_duration, absl::ZeroDuration()); + EXPECT_TRUE(absl::GetFlag(FLAGS_optional_duration).has_value()); + EXPECT_EQ(absl::GetFlag(FLAGS_optional_duration), absl::Seconds(0)); + + absl::SetFlag(&FLAGS_optional_duration, absl::Hours(3)); + EXPECT_TRUE(absl::GetFlag(FLAGS_optional_duration).has_value()); + EXPECT_EQ(absl::GetFlag(FLAGS_optional_duration), absl::Hours(3)); + + absl::SetFlag(&FLAGS_optional_duration, absl::nullopt); + EXPECT_FALSE(absl::GetFlag(FLAGS_optional_duration).has_value()); + EXPECT_EQ(absl::GetFlag(FLAGS_optional_duration), absl::nullopt); +} + +// -------------------------------------------------------------------- + +TEST_F(FlagTest, TestOptionalOptional) { + EXPECT_FALSE(absl::GetFlag(FLAGS_optional_optional_int).has_value()); + EXPECT_EQ(absl::GetFlag(FLAGS_optional_optional_int), absl::nullopt); + + absl::optional<int> nullint{absl::nullopt}; + + absl::SetFlag(&FLAGS_optional_optional_int, nullint); + EXPECT_TRUE(absl::GetFlag(FLAGS_optional_optional_int).has_value()); + EXPECT_NE(absl::GetFlag(FLAGS_optional_optional_int), nullint); + EXPECT_EQ(absl::GetFlag(FLAGS_optional_optional_int), + absl::optional<absl::optional<int>>{nullint}); + + absl::SetFlag(&FLAGS_optional_optional_int, 0); + EXPECT_TRUE(absl::GetFlag(FLAGS_optional_optional_int).has_value()); + EXPECT_EQ(absl::GetFlag(FLAGS_optional_optional_int), 0); + + absl::SetFlag(&FLAGS_optional_optional_int, absl::optional<int>{0}); + EXPECT_TRUE(absl::GetFlag(FLAGS_optional_optional_int).has_value()); + EXPECT_EQ(absl::GetFlag(FLAGS_optional_optional_int), 0); + EXPECT_EQ(absl::GetFlag(FLAGS_optional_optional_int), absl::optional<int>{0}); + + absl::SetFlag(&FLAGS_optional_optional_int, absl::nullopt); + EXPECT_FALSE(absl::GetFlag(FLAGS_optional_optional_int).has_value()); + EXPECT_EQ(absl::GetFlag(FLAGS_optional_optional_int), absl::nullopt); +} + +// -------------------------------------------------------------------- + +#if defined(ABSL_HAVE_STD_OPTIONAL) && !defined(ABSL_USES_STD_OPTIONAL) + +TEST_F(FlagTest, TestStdOptional) { + EXPECT_FALSE(absl::GetFlag(FLAGS_std_optional_int64).has_value()); + EXPECT_EQ(absl::GetFlag(FLAGS_std_optional_int64), std::nullopt); + + absl::SetFlag(&FLAGS_std_optional_int64, 0); + EXPECT_TRUE(absl::GetFlag(FLAGS_std_optional_int64).has_value()); + EXPECT_EQ(absl::GetFlag(FLAGS_std_optional_int64), 0); + + absl::SetFlag(&FLAGS_std_optional_int64, 0xFFFFFFFFFF16); + EXPECT_TRUE(absl::GetFlag(FLAGS_std_optional_int64).has_value()); + EXPECT_EQ(absl::GetFlag(FLAGS_std_optional_int64), 0xFFFFFFFFFF16); + + absl::SetFlag(&FLAGS_std_optional_int64, std::nullopt); + EXPECT_FALSE(absl::GetFlag(FLAGS_std_optional_int64).has_value()); + EXPECT_EQ(absl::GetFlag(FLAGS_std_optional_int64), std::nullopt); +} + +// -------------------------------------------------------------------- + +#endif + +} // namespace diff --git a/absl/flags/internal/flag.cc b/absl/flags/internal/flag.cc index f83c1fe7..55892d77 100644 --- a/absl/flags/internal/flag.cc +++ b/absl/flags/internal/flag.cc @@ -30,6 +30,7 @@ #include "absl/base/call_once.h" #include "absl/base/casts.h" #include "absl/base/config.h" +#include "absl/base/dynamic_annotations.h" #include "absl/base/optimization.h" #include "absl/flags/config.h" #include "absl/flags/internal/commandlineflag.h" @@ -145,12 +146,7 @@ void FlagImpl::Init() { auto def_kind = static_cast<FlagDefaultKind>(def_kind_); switch (ValueStorageKind()) { - case FlagValueStorageKind::kAlignedBuffer: - // For this storage kind the default_value_ always points to gen_func - // during initialization. - assert(def_kind == FlagDefaultKind::kGenFunc); - (*default_value_.gen_func)(AlignedBufferValue()); - break; + case FlagValueStorageKind::kValueAndInitBit: case FlagValueStorageKind::kOneWordAtomic: { alignas(int64_t) std::array<char, sizeof(int64_t)> buf{}; if (def_kind == FlagDefaultKind::kGenFunc) { @@ -159,6 +155,14 @@ void FlagImpl::Init() { assert(def_kind != FlagDefaultKind::kDynamicValue); std::memcpy(buf.data(), &default_value_, Sizeof(op_)); } + if (ValueStorageKind() == FlagValueStorageKind::kValueAndInitBit) { + // We presume here the memory layout of FlagValueAndInitBit struct. + uint8_t initialized = 1; + std::memcpy(buf.data() + Sizeof(op_), &initialized, + sizeof(initialized)); + } + // Type can contain valid uninitialized bits, e.g. padding. + ABSL_ANNOTATE_MEMORY_IS_INITIALIZED(buf.data(), buf.size()); OneWordValue().store(absl::bit_cast<int64_t>(buf), std::memory_order_release); break; @@ -170,6 +174,12 @@ void FlagImpl::Init() { (*default_value_.gen_func)(AtomicBufferValue()); break; } + case FlagValueStorageKind::kAlignedBuffer: + // For this storage kind the default_value_ always points to gen_func + // during initialization. + assert(def_kind == FlagDefaultKind::kGenFunc); + (*default_value_.gen_func)(AlignedBufferValue()); + break; } seq_lock_.MarkInitialized(); } @@ -198,7 +208,7 @@ void FlagImpl::AssertValidType(FlagFastTypeId rhs_type_id, if (lhs_runtime_type_id == rhs_runtime_type_id) return; -#if defined(ABSL_FLAGS_INTERNAL_HAS_RTTI) +#ifdef ABSL_INTERNAL_HAS_RTTI if (*lhs_runtime_type_id == *rhs_runtime_type_id) return; #endif @@ -226,12 +236,10 @@ std::unique_ptr<void, DynValueDeleter> FlagImpl::MakeInitValue() const { void FlagImpl::StoreValue(const void* src) { switch (ValueStorageKind()) { - case FlagValueStorageKind::kAlignedBuffer: - Copy(op_, src, AlignedBufferValue()); - seq_lock_.IncrementModificationCount(); - break; + case FlagValueStorageKind::kValueAndInitBit: case FlagValueStorageKind::kOneWordAtomic: { - int64_t one_word_val = 0; + // Load the current value to avoid setting 'init' bit manualy. + int64_t one_word_val = OneWordValue().load(std::memory_order_acquire); std::memcpy(&one_word_val, src, Sizeof(op_)); OneWordValue().store(one_word_val, std::memory_order_release); seq_lock_.IncrementModificationCount(); @@ -241,6 +249,10 @@ void FlagImpl::StoreValue(const void* src) { seq_lock_.Write(AtomicBufferValue(), src, Sizeof(op_)); break; } + case FlagValueStorageKind::kAlignedBuffer: + Copy(op_, src, AlignedBufferValue()); + seq_lock_.IncrementModificationCount(); + break; } modified_ = true; InvokeCallback(); @@ -280,10 +292,7 @@ std::string FlagImpl::DefaultValue() const { std::string FlagImpl::CurrentValue() const { auto* guard = DataGuard(); // Make sure flag initialized switch (ValueStorageKind()) { - case FlagValueStorageKind::kAlignedBuffer: { - absl::MutexLock l(guard); - return flags_internal::Unparse(op_, AlignedBufferValue()); - } + case FlagValueStorageKind::kValueAndInitBit: case FlagValueStorageKind::kOneWordAtomic: { const auto one_word_val = absl::bit_cast<std::array<char, sizeof(int64_t)>>( @@ -296,6 +305,10 @@ std::string FlagImpl::CurrentValue() const { ReadSequenceLockedData(cloned.get()); return flags_internal::Unparse(op_, cloned.get()); } + case FlagValueStorageKind::kAlignedBuffer: { + absl::MutexLock l(guard); + return flags_internal::Unparse(op_, AlignedBufferValue()); + } } return ""; @@ -341,11 +354,7 @@ std::unique_ptr<FlagStateInterface> FlagImpl::SaveState() { bool modified = modified_; bool on_command_line = on_command_line_; switch (ValueStorageKind()) { - case FlagValueStorageKind::kAlignedBuffer: { - return absl::make_unique<FlagState>( - *this, flags_internal::Clone(op_, AlignedBufferValue()), modified, - on_command_line, ModificationCount()); - } + case FlagValueStorageKind::kValueAndInitBit: case FlagValueStorageKind::kOneWordAtomic: { return absl::make_unique<FlagState>( *this, OneWordValue().load(std::memory_order_acquire), modified, @@ -361,6 +370,11 @@ std::unique_ptr<FlagStateInterface> FlagImpl::SaveState() { return absl::make_unique<FlagState>(*this, cloned, modified, on_command_line, ModificationCount()); } + case FlagValueStorageKind::kAlignedBuffer: { + return absl::make_unique<FlagState>( + *this, flags_internal::Clone(op_, AlignedBufferValue()), modified, + on_command_line, ModificationCount()); + } } return nullptr; } @@ -372,13 +386,14 @@ bool FlagImpl::RestoreState(const FlagState& flag_state) { } switch (ValueStorageKind()) { - case FlagValueStorageKind::kAlignedBuffer: - case FlagValueStorageKind::kSequenceLocked: - StoreValue(flag_state.value_.heap_allocated); - break; + case FlagValueStorageKind::kValueAndInitBit: case FlagValueStorageKind::kOneWordAtomic: StoreValue(&flag_state.value_.one_word); break; + case FlagValueStorageKind::kSequenceLocked: + case FlagValueStorageKind::kAlignedBuffer: + StoreValue(flag_state.value_.heap_allocated); + break; } modified_ = flag_state.modified_; @@ -407,7 +422,8 @@ std::atomic<uint64_t>* FlagImpl::AtomicBufferValue() const { } std::atomic<int64_t>& FlagImpl::OneWordValue() const { - assert(ValueStorageKind() == FlagValueStorageKind::kOneWordAtomic); + assert(ValueStorageKind() == FlagValueStorageKind::kOneWordAtomic || + ValueStorageKind() == FlagValueStorageKind::kValueAndInitBit); return OffsetValue<FlagOneWordValue>()->value; } @@ -433,11 +449,7 @@ std::unique_ptr<void, DynValueDeleter> FlagImpl::TryParse( void FlagImpl::Read(void* dst) const { auto* guard = DataGuard(); // Make sure flag initialized switch (ValueStorageKind()) { - case FlagValueStorageKind::kAlignedBuffer: { - absl::MutexLock l(guard); - flags_internal::CopyConstruct(op_, AlignedBufferValue(), dst); - break; - } + case FlagValueStorageKind::kValueAndInitBit: case FlagValueStorageKind::kOneWordAtomic: { const int64_t one_word_val = OneWordValue().load(std::memory_order_acquire); @@ -448,9 +460,31 @@ void FlagImpl::Read(void* dst) const { ReadSequenceLockedData(dst); break; } + case FlagValueStorageKind::kAlignedBuffer: { + absl::MutexLock l(guard); + flags_internal::CopyConstruct(op_, AlignedBufferValue(), dst); + break; + } } } +int64_t FlagImpl::ReadOneWord() const { + assert(ValueStorageKind() == FlagValueStorageKind::kOneWordAtomic || + ValueStorageKind() == FlagValueStorageKind::kValueAndInitBit); + auto* guard = DataGuard(); // Make sure flag initialized + (void)guard; + return OneWordValue().load(std::memory_order_acquire); +} + +bool FlagImpl::ReadOneBool() const { + assert(ValueStorageKind() == FlagValueStorageKind::kValueAndInitBit); + auto* guard = DataGuard(); // Make sure flag initialized + (void)guard; + return absl::bit_cast<FlagValueAndInitBit<bool>>( + OneWordValue().load(std::memory_order_acquire)) + .value; +} + void FlagImpl::ReadSequenceLockedData(void* dst) const { int size = Sizeof(op_); // Attempt to read using the sequence lock. diff --git a/absl/flags/internal/flag.h b/absl/flags/internal/flag.h index e6bade0a..6154638c 100644 --- a/absl/flags/internal/flag.h +++ b/absl/flags/internal/flag.h @@ -29,6 +29,7 @@ #include "absl/base/attributes.h" #include "absl/base/call_once.h" +#include "absl/base/casts.h" #include "absl/base/config.h" #include "absl/base/optimization.h" #include "absl/base/thread_annotations.h" @@ -162,7 +163,7 @@ inline ptrdiff_t ValueOffset(FlagOpFn op) { // Returns an address of RTTI's typeid(T). template <typename T> inline const std::type_info* GenRuntimeTypeId() { -#if defined(ABSL_FLAGS_INTERNAL_HAS_RTTI) +#ifdef ABSL_INTERNAL_HAS_RTTI return &typeid(T); #else return nullptr; @@ -289,7 +290,7 @@ constexpr T InitDefaultValue(EmptyBraces) { template <typename ValueT, typename GenT, typename std::enable_if<std::is_integral<ValueT>::value, int>::type = - (GenT{}, 0)> + ((void)GenT{}, 0)> constexpr FlagDefaultArg DefaultArg(int) { return {FlagDefaultSrc(GenT{}.value), FlagDefaultKind::kOneWord}; } @@ -302,7 +303,14 @@ constexpr FlagDefaultArg DefaultArg(char) { /////////////////////////////////////////////////////////////////////////////// // Flag current value auxiliary structs. -constexpr int64_t UninitializedFlagValue() { return 0xababababababababll; } +constexpr int64_t UninitializedFlagValue() { + return static_cast<int64_t>(0xababababababababll); +} + +template <typename T> +using FlagUseValueAndInitBitStorage = std::integral_constant< + bool, absl::type_traits_internal::is_trivially_copyable<T>::value && + std::is_default_constructible<T>::value && (sizeof(T) < 8)>; template <typename T> using FlagUseOneWordStorage = std::integral_constant< @@ -310,43 +318,61 @@ using FlagUseOneWordStorage = std::integral_constant< (sizeof(T) <= 8)>; template <class T> -using FlagShouldUseSequenceLock = std::integral_constant< +using FlagUseSequenceLockStorage = std::integral_constant< bool, absl::type_traits_internal::is_trivially_copyable<T>::value && (sizeof(T) > 8)>; enum class FlagValueStorageKind : uint8_t { - kAlignedBuffer = 0, + kValueAndInitBit = 0, kOneWordAtomic = 1, kSequenceLocked = 2, + kAlignedBuffer = 3, }; template <typename T> static constexpr FlagValueStorageKind StorageKind() { - return FlagUseOneWordStorage<T>::value ? FlagValueStorageKind::kOneWordAtomic - : FlagShouldUseSequenceLock<T>::value + return FlagUseValueAndInitBitStorage<T>::value + ? FlagValueStorageKind::kValueAndInitBit + : FlagUseOneWordStorage<T>::value + ? FlagValueStorageKind::kOneWordAtomic + : FlagUseSequenceLockStorage<T>::value ? FlagValueStorageKind::kSequenceLocked : FlagValueStorageKind::kAlignedBuffer; } struct FlagOneWordValue { - constexpr FlagOneWordValue() : value(UninitializedFlagValue()) {} - + constexpr explicit FlagOneWordValue(int64_t v) : value(v) {} std::atomic<int64_t> value; }; +template <typename T> +struct alignas(8) FlagValueAndInitBit { + T value; + // Use an int instead of a bool to guarantee that a non-zero value has + // a bit set. + uint8_t init; +}; + template <typename T, FlagValueStorageKind Kind = flags_internal::StorageKind<T>()> struct FlagValue; template <typename T> -struct FlagValue<T, FlagValueStorageKind::kAlignedBuffer> { - bool Get(const SequenceLock&, T&) const { return false; } - - alignas(T) char value[sizeof(T)]; +struct FlagValue<T, FlagValueStorageKind::kValueAndInitBit> : FlagOneWordValue { + constexpr FlagValue() : FlagOneWordValue(0) {} + bool Get(const SequenceLock&, T& dst) const { + int64_t storage = value.load(std::memory_order_acquire); + if (ABSL_PREDICT_FALSE(storage == 0)) { + return false; + } + dst = absl::bit_cast<FlagValueAndInitBit<T>>(storage).value; + return true; + } }; template <typename T> struct FlagValue<T, FlagValueStorageKind::kOneWordAtomic> : FlagOneWordValue { + constexpr FlagValue() : FlagOneWordValue(UninitializedFlagValue()) {} bool Get(const SequenceLock&, T& dst) const { int64_t one_word_val = value.load(std::memory_order_acquire); if (ABSL_PREDICT_FALSE(one_word_val == UninitializedFlagValue())) { @@ -370,6 +396,13 @@ struct FlagValue<T, FlagValueStorageKind::kSequenceLocked> { std::atomic<uint64_t>) std::atomic<uint64_t> value_words[kNumWords]; }; +template <typename T> +struct FlagValue<T, FlagValueStorageKind::kAlignedBuffer> { + bool Get(const SequenceLock&, T&) const { return false; } + + alignas(T) char value[sizeof(T)]; +}; + /////////////////////////////////////////////////////////////////////////////// // Flag callback auxiliary structs. @@ -415,7 +448,27 @@ class FlagImpl final : public CommandLineFlag { data_guard_{} {} // Constant access methods + int64_t ReadOneWord() const ABSL_LOCKS_EXCLUDED(*DataGuard()); + bool ReadOneBool() const ABSL_LOCKS_EXCLUDED(*DataGuard()); void Read(void* dst) const override ABSL_LOCKS_EXCLUDED(*DataGuard()); + void Read(bool* value) const ABSL_LOCKS_EXCLUDED(*DataGuard()) { + *value = ReadOneBool(); + } + template <typename T, + absl::enable_if_t<flags_internal::StorageKind<T>() == + FlagValueStorageKind::kOneWordAtomic, + int> = 0> + void Read(T* value) const ABSL_LOCKS_EXCLUDED(*DataGuard()) { + int64_t v = ReadOneWord(); + std::memcpy(value, static_cast<const void*>(&v), sizeof(T)); + } + template <typename T, + typename std::enable_if<flags_internal::StorageKind<T>() == + FlagValueStorageKind::kValueAndInitBit, + int>::type = 0> + void Read(T* value) const ABSL_LOCKS_EXCLUDED(*DataGuard()) { + *value = absl::bit_cast<FlagValueAndInitBit<T>>(ReadOneWord()).value; + } // Mutating access methods void Write(const void* src) ABSL_LOCKS_EXCLUDED(*DataGuard()); @@ -704,8 +757,8 @@ void* FlagOps(FlagOp op, const void* v1, void* v2, void* v3) { case FlagOp::kValueOffset: { // Round sizeof(FlagImp) to a multiple of alignof(FlagValue<T>) to get the // offset of the data. - ptrdiff_t round_to = alignof(FlagValue<T>); - ptrdiff_t offset = + size_t round_to = alignof(FlagValue<T>); + size_t offset = (sizeof(FlagImpl) + round_to - 1) / round_to * round_to; return reinterpret_cast<void*>(offset); } diff --git a/absl/flags/internal/flag_msvc.inc b/absl/flags/internal/flag_msvc.inc new file mode 100644 index 00000000..c31bd27f --- /dev/null +++ b/absl/flags/internal/flag_msvc.inc @@ -0,0 +1,116 @@ +// +// Copyright 2021 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Do not include this file directly. +// Include absl/flags/flag.h instead. + +// MSVC debug builds do not implement initialization with constexpr constructors +// correctly. To work around this we add a level of indirection, so that the +// class `absl::Flag` contains an `internal::Flag*` (instead of being an alias +// to that class) and dynamically allocates an instance when necessary. We also +// forward all calls to internal::Flag methods via trampoline methods. In this +// setup the `absl::Flag` class does not have constructor and virtual methods, +// all the data members are public and thus MSVC is able to initialize it at +// link time. To deal with multiple threads accessing the flag for the first +// time concurrently we use an atomic boolean indicating if flag object is +// initialized. We also employ the double-checked locking pattern where the +// second level of protection is a global Mutex, so if two threads attempt to +// construct the flag concurrently only one wins. +// +// This solution is based on a recomendation here: +// https://developercommunity.visualstudio.com/content/problem/336946/class-with-constexpr-constructor-not-using-static.html?childToView=648454#comment-648454 + +namespace flags_internal { +absl::Mutex* GetGlobalConstructionGuard(); +} // namespace flags_internal + +// Public methods of `absl::Flag<T>` are NOT part of the Abseil Flags API. +// See https://abseil.io/docs/cpp/guides/flags +template <typename T> +class Flag { + public: + // No constructor and destructor to ensure this is an aggregate type. + // Visual Studio 2015 still requires the constructor for class to be + // constexpr initializable. +#if _MSC_VER <= 1900 + constexpr Flag(const char* name, const char* filename, + const flags_internal::HelpGenFunc help_gen, + const flags_internal::FlagDfltGenFunc default_value_gen) + : name_(name), + filename_(filename), + help_gen_(help_gen), + default_value_gen_(default_value_gen), + inited_(false), + impl_(nullptr) {} +#endif + + flags_internal::Flag<T>& GetImpl() const { + if (!inited_.load(std::memory_order_acquire)) { + absl::MutexLock l(flags_internal::GetGlobalConstructionGuard()); + + if (inited_.load(std::memory_order_acquire)) { + return *impl_; + } + + impl_ = new flags_internal::Flag<T>( + name_, filename_, + {flags_internal::FlagHelpMsg(help_gen_), + flags_internal::FlagHelpKind::kGenFunc}, + {flags_internal::FlagDefaultSrc(default_value_gen_), + flags_internal::FlagDefaultKind::kGenFunc}); + inited_.store(true, std::memory_order_release); + } + + return *impl_; + } + + // Public methods of `absl::Flag<T>` are NOT part of the Abseil Flags API. + // See https://abseil.io/docs/cpp/guides/flags + bool IsRetired() const { return GetImpl().IsRetired(); } + absl::string_view Name() const { return GetImpl().Name(); } + std::string Help() const { return GetImpl().Help(); } + bool IsModified() const { return GetImpl().IsModified(); } + bool IsSpecifiedOnCommandLine() const { + return GetImpl().IsSpecifiedOnCommandLine(); + } + std::string Filename() const { return GetImpl().Filename(); } + std::string DefaultValue() const { return GetImpl().DefaultValue(); } + std::string CurrentValue() const { return GetImpl().CurrentValue(); } + template <typename U> + inline bool IsOfType() const { + return GetImpl().template IsOfType<U>(); + } + T Get() const { + return flags_internal::FlagImplPeer::InvokeGet<T>(GetImpl()); + } + void Set(const T& v) { + flags_internal::FlagImplPeer::InvokeSet(GetImpl(), v); + } + void InvokeCallback() { GetImpl().InvokeCallback(); } + + const CommandLineFlag& Reflect() const { + return flags_internal::FlagImplPeer::InvokeReflect(GetImpl()); + } + + // The data members are logically private, but they need to be public for + // this to be an aggregate type. + const char* name_; + const char* filename_; + const flags_internal::HelpGenFunc help_gen_; + const flags_internal::FlagDfltGenFunc default_value_gen_; + + mutable std::atomic<bool> inited_; + mutable flags_internal::Flag<T>* impl_; +}; diff --git a/absl/flags/internal/sequence_lock.h b/absl/flags/internal/sequence_lock.h index 807b2a73..36318ab9 100644 --- a/absl/flags/internal/sequence_lock.h +++ b/absl/flags/internal/sequence_lock.h @@ -49,7 +49,7 @@ inline constexpr size_t AlignUp(size_t x, size_t align) { // The memory reads and writes protected by this lock must use the provided // `TryRead()` and `Write()` functions. These functions behave similarly to // `memcpy()`, with one oddity: the protected data must be an array of -// `std::atomic<int64>`. This is to comply with the C++ standard, which +// `std::atomic<uint64>`. This is to comply with the C++ standard, which // considers data races on non-atomic objects to be undefined behavior. See "Can // Seqlocks Get Along With Programming Language Memory Models?"[1] by Hans J. // Boehm for more details. diff --git a/absl/flags/internal/usage.cc b/absl/flags/internal/usage.cc index a588c7f7..949709e8 100644 --- a/absl/flags/internal/usage.cc +++ b/absl/flags/internal/usage.cc @@ -245,7 +245,7 @@ void FlagsHelpImpl(std::ostream& out, PerFlagFilter filter_cb, << XMLElement("usage", program_usage_message) << '\n'; } - // Map of package name to + // Ordered map of package name to // map of file name to // vector of flags in the file. // This map is used to output matching flags grouped by package and file @@ -273,20 +273,26 @@ void FlagsHelpImpl(std::ostream& out, PerFlagFilter filter_cb, absl::string_view package_separator; // controls blank lines between packages absl::string_view file_separator; // controls blank lines between files - for (const auto& package : matching_flags) { + for (auto& package : matching_flags) { if (format == HelpFormat::kHumanReadable) { out << package_separator; package_separator = "\n\n"; } file_separator = ""; - for (const auto& flags_in_file : package.second) { + for (auto& flags_in_file : package.second) { if (format == HelpFormat::kHumanReadable) { out << file_separator << " Flags from " << flags_in_file.first << ":\n"; file_separator = "\n"; } + std::sort(std::begin(flags_in_file.second), + std::end(flags_in_file.second), + [](const CommandLineFlag* lhs, const CommandLineFlag* rhs) { + return lhs->Name() < rhs->Name(); + }); + for (const auto* flag : flags_in_file.second) { flags_internal::FlagHelp(out, *flag, format); } diff --git a/absl/flags/internal/usage_test.cc b/absl/flags/internal/usage_test.cc index b5c2487d..209a7be9 100644 --- a/absl/flags/internal/usage_test.cc +++ b/absl/flags/internal/usage_test.cc @@ -20,6 +20,7 @@ #include <sstream> #include <string> +#include "gmock/gmock.h" #include "gtest/gtest.h" #include "absl/flags/flag.h" #include "absl/flags/internal/parse.h" @@ -45,9 +46,12 @@ static const char kTestUsageMessage[] = "Custom usage message"; struct UDT { UDT() = default; UDT(const UDT&) = default; + UDT& operator=(const UDT&) = default; }; -bool AbslParseFlag(absl::string_view, UDT*, std::string*) { return true; } -std::string AbslUnparseFlag(const UDT&) { return "UDT{}"; } +static bool AbslParseFlag(absl::string_view, UDT*, std::string*) { + return true; +} +static std::string AbslUnparseFlag(const UDT&) { return "UDT{}"; } ABSL_FLAG(UDT, usage_reporting_test_flag_05, {}, "usage_reporting_test_flag_05 help message"); @@ -102,14 +106,19 @@ class UsageReportingTest : public testing::Test { using UsageReportingDeathTest = UsageReportingTest; TEST_F(UsageReportingDeathTest, TestSetProgramUsageMessage) { +#if !defined(GTEST_HAS_ABSL) || !GTEST_HAS_ABSL + // Check for kTestUsageMessage set in main() below. EXPECT_EQ(absl::ProgramUsageMessage(), kTestUsageMessage); +#else + // Check for part of the usage message set by GoogleTest. + EXPECT_THAT(absl::ProgramUsageMessage(), + ::testing::HasSubstr( + "This program contains tests written using Google Test")); +#endif -#ifndef _WIN32 - // TODO(rogeeff): figure out why this does not work on Windows. EXPECT_DEATH_IF_SUPPORTED( absl::SetProgramUsageMessage("custom usage message"), - ".*SetProgramUsageMessage\\(\\) called twice.*"); -#endif + ::testing::HasSubstr("SetProgramUsageMessage() called twice")); } // -------------------------------------------------------------------- @@ -486,8 +495,10 @@ path. int main(int argc, char* argv[]) { (void)absl::GetFlag(FLAGS_undefok); // Force linking of parse.cc flags::SetProgramInvocationName("usage_test"); +#if !defined(GTEST_HAS_ABSL) || !GTEST_HAS_ABSL + // GoogleTest calls absl::SetProgramUsageMessage() already. absl::SetProgramUsageMessage(kTestUsageMessage); +#endif ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); } diff --git a/absl/flags/marshalling.h b/absl/flags/marshalling.h index 7cbc136d..b1e2ffa5 100644 --- a/absl/flags/marshalling.h +++ b/absl/flags/marshalling.h @@ -33,6 +33,7 @@ // * `double` // * `std::string` // * `std::vector<std::string>` +// * `std::optional<T>` // * `absl::LogSeverity` (provided natively for layering reasons) // // Note that support for integral types is implemented using overloads for @@ -65,6 +66,42 @@ // below.) // // ----------------------------------------------------------------------------- +// Optional Flags +// ----------------------------------------------------------------------------- +// +// The Abseil flags library supports flags of type `std::optional<T>` where +// `T` is a type of one of the supported flags. We refer to this flag type as +// an "optional flag." An optional flag is either "valueless", holding no value +// of type `T` (indicating that the flag has not been set) or a value of type +// `T`. The valueless state in C++ code is represented by a value of +// `std::nullopt` for the optional flag. +// +// Using `std::nullopt` as an optional flag's default value allows you to check +// whether such a flag was ever specified on the command line: +// +// if (absl::GetFlag(FLAGS_foo).has_value()) { +// // flag was set on command line +// } else { +// // flag was not passed on command line +// } +// +// Using an optional flag in this manner avoids common workarounds for +// indicating such an unset flag (such as using sentinal values to indicate this +// state). +// +// An optional flag also allows a developer to pass a flag in an "unset" +// valueless state on the command line, allowing the flag to later be set in +// binary logic. An optional flag's valueless state is indicated by the special +// notation of passing the value as an empty string through the syntax `--flag=` +// or `--flag ""`. +// +// $ binary_with_optional --flag_in_unset_state= +// $ binary_with_optional --flag_in_unset_state "" +// +// Note: as a result of the above syntax requirements, an optional flag cannot +// be set to a `T` of any value which unparses to the empty string. +// +// ----------------------------------------------------------------------------- // Adding Type Support for Abseil Flags // ----------------------------------------------------------------------------- // @@ -162,14 +199,27 @@ #ifndef ABSL_FLAGS_MARSHALLING_H_ #define ABSL_FLAGS_MARSHALLING_H_ +#include "absl/base/config.h" + +#if defined(ABSL_HAVE_STD_OPTIONAL) && !defined(ABSL_USES_STD_OPTIONAL) +#include <optional> +#endif #include <string> #include <vector> -#include "absl/base/config.h" #include "absl/strings/string_view.h" +#include "absl/types/optional.h" namespace absl { ABSL_NAMESPACE_BEGIN + +// Forward declaration to be used inside composable flag parse/unparse +// implementations +template <typename T> +inline bool ParseFlag(absl::string_view input, T* dst, std::string* error); +template <typename T> +inline std::string UnparseFlag(const T& v); + namespace flags_internal { // Overloads of `AbslParseFlag()` and `AbslUnparseFlag()` for fundamental types. @@ -189,6 +239,36 @@ bool AbslParseFlag(absl::string_view, std::string*, std::string*); bool AbslParseFlag(absl::string_view, std::vector<std::string>*, std::string*); template <typename T> +bool AbslParseFlag(absl::string_view text, absl::optional<T>* f, + std::string* err) { + if (text.empty()) { + *f = absl::nullopt; + return true; + } + T value; + if (!absl::ParseFlag(text, &value, err)) return false; + + *f = std::move(value); + return true; +} + +#if defined(ABSL_HAVE_STD_OPTIONAL) && !defined(ABSL_USES_STD_OPTIONAL) +template <typename T> +bool AbslParseFlag(absl::string_view text, std::optional<T>* f, + std::string* err) { + if (text.empty()) { + *f = std::nullopt; + return true; + } + T value; + if (!absl::ParseFlag(text, &value, err)) return false; + + *f = std::move(value); + return true; +} +#endif + +template <typename T> bool InvokeParseFlag(absl::string_view input, T* dst, std::string* err) { // Comment on next line provides a good compiler error message if T // does not have AbslParseFlag(absl::string_view, T*, std::string*). @@ -202,6 +282,18 @@ std::string AbslUnparseFlag(absl::string_view v); std::string AbslUnparseFlag(const std::vector<std::string>&); template <typename T> +std::string AbslUnparseFlag(const absl::optional<T>& f) { + return f.has_value() ? absl::UnparseFlag(*f) : ""; +} + +#if defined(ABSL_HAVE_STD_OPTIONAL) && !defined(ABSL_USES_STD_OPTIONAL) +template <typename T> +std::string AbslUnparseFlag(const std::optional<T>& f) { + return f.has_value() ? absl::UnparseFlag(*f) : ""; +} +#endif + +template <typename T> std::string Unparse(const T& v) { // Comment on next line provides a good compiler error message if T does not // have UnparseFlag. diff --git a/absl/flags/marshalling_test.cc b/absl/flags/marshalling_test.cc index 4a64ce11..7b6d2ad5 100644 --- a/absl/flags/marshalling_test.cc +++ b/absl/flags/marshalling_test.cc @@ -659,6 +659,88 @@ TEST(MarshallingTest, TestVectorOfStringParsing) { // -------------------------------------------------------------------- +TEST(MarshallingTest, TestOptionalBoolParsing) { + std::string err; + absl::optional<bool> value; + + EXPECT_TRUE(absl::ParseFlag("", &value, &err)); + EXPECT_FALSE(value.has_value()); + + EXPECT_TRUE(absl::ParseFlag("true", &value, &err)); + EXPECT_TRUE(value.has_value()); + EXPECT_TRUE(*value); + + EXPECT_TRUE(absl::ParseFlag("false", &value, &err)); + EXPECT_TRUE(value.has_value()); + EXPECT_FALSE(*value); + + EXPECT_FALSE(absl::ParseFlag("nullopt", &value, &err)); +} + +// -------------------------------------------------------------------- + +TEST(MarshallingTest, TestOptionalIntParsing) { + std::string err; + absl::optional<int> value; + + EXPECT_TRUE(absl::ParseFlag("", &value, &err)); + EXPECT_FALSE(value.has_value()); + + EXPECT_TRUE(absl::ParseFlag("10", &value, &err)); + EXPECT_TRUE(value.has_value()); + EXPECT_EQ(*value, 10); + + EXPECT_TRUE(absl::ParseFlag("0x1F", &value, &err)); + EXPECT_TRUE(value.has_value()); + EXPECT_EQ(*value, 31); + + EXPECT_FALSE(absl::ParseFlag("nullopt", &value, &err)); +} + +// -------------------------------------------------------------------- + +TEST(MarshallingTest, TestOptionalDoubleParsing) { + std::string err; + absl::optional<double> value; + + EXPECT_TRUE(absl::ParseFlag("", &value, &err)); + EXPECT_FALSE(value.has_value()); + + EXPECT_TRUE(absl::ParseFlag("1.11", &value, &err)); + EXPECT_TRUE(value.has_value()); + EXPECT_EQ(*value, 1.11); + + EXPECT_TRUE(absl::ParseFlag("-0.12", &value, &err)); + EXPECT_TRUE(value.has_value()); + EXPECT_EQ(*value, -0.12); + + EXPECT_FALSE(absl::ParseFlag("nullopt", &value, &err)); +} + +// -------------------------------------------------------------------- + +TEST(MarshallingTest, TestOptionalStringParsing) { + std::string err; + absl::optional<std::string> value; + + EXPECT_TRUE(absl::ParseFlag("", &value, &err)); + EXPECT_FALSE(value.has_value()); + + EXPECT_TRUE(absl::ParseFlag(" ", &value, &err)); + EXPECT_TRUE(value.has_value()); + EXPECT_EQ(*value, " "); + + EXPECT_TRUE(absl::ParseFlag("aqswde", &value, &err)); + EXPECT_TRUE(value.has_value()); + EXPECT_EQ(*value, "aqswde"); + + EXPECT_TRUE(absl::ParseFlag("nullopt", &value, &err)); + EXPECT_TRUE(value.has_value()); + EXPECT_EQ(*value, "nullopt"); +} + +// -------------------------------------------------------------------- + TEST(MarshallingTest, TestBoolUnparsing) { EXPECT_EQ(absl::UnparseFlag(true), "true"); EXPECT_EQ(absl::UnparseFlag(false), "false"); @@ -808,6 +890,90 @@ TEST(MarshallingTest, TestStringUnparsing) { // -------------------------------------------------------------------- +TEST(MarshallingTest, TestOptionalBoolUnparsing) { + absl::optional<bool> value; + + EXPECT_EQ(absl::UnparseFlag(value), ""); + value = true; + EXPECT_EQ(absl::UnparseFlag(value), "true"); + value = false; + EXPECT_EQ(absl::UnparseFlag(value), "false"); + value = absl::nullopt; + EXPECT_EQ(absl::UnparseFlag(value), ""); +} + +// -------------------------------------------------------------------- + +TEST(MarshallingTest, TestOptionalIntUnparsing) { + absl::optional<int> value; + + EXPECT_EQ(absl::UnparseFlag(value), ""); + value = 0; + EXPECT_EQ(absl::UnparseFlag(value), "0"); + value = -12; + EXPECT_EQ(absl::UnparseFlag(value), "-12"); + value = absl::nullopt; + EXPECT_EQ(absl::UnparseFlag(value), ""); +} + +// -------------------------------------------------------------------- + +TEST(MarshallingTest, TestOptionalDoubleUnparsing) { + absl::optional<double> value; + + EXPECT_EQ(absl::UnparseFlag(value), ""); + value = 1.; + EXPECT_EQ(absl::UnparseFlag(value), "1"); + value = -1.23; + EXPECT_EQ(absl::UnparseFlag(value), "-1.23"); + value = absl::nullopt; + EXPECT_EQ(absl::UnparseFlag(value), ""); +} + +// -------------------------------------------------------------------- + +TEST(MarshallingTest, TestOptionalStringUnparsing) { + absl::optional<std::string> strvalue; + EXPECT_EQ(absl::UnparseFlag(strvalue), ""); + + strvalue = "asdfg"; + EXPECT_EQ(absl::UnparseFlag(strvalue), "asdfg"); + + strvalue = " "; + EXPECT_EQ(absl::UnparseFlag(strvalue), " "); + + strvalue = ""; // It is UB to set an optional string flag to "" + EXPECT_EQ(absl::UnparseFlag(strvalue), ""); +} + +// -------------------------------------------------------------------- + +#if defined(ABSL_HAVE_STD_OPTIONAL) && !defined(ABSL_USES_STD_OPTIONAL) + +TEST(MarshallingTest, TestStdOptionalUnparsing) { + std::optional<std::string> strvalue; + EXPECT_EQ(absl::UnparseFlag(strvalue), ""); + + strvalue = "asdfg"; + EXPECT_EQ(absl::UnparseFlag(strvalue), "asdfg"); + + strvalue = " "; + EXPECT_EQ(absl::UnparseFlag(strvalue), " "); + + strvalue = ""; // It is UB to set an optional string flag to "" + EXPECT_EQ(absl::UnparseFlag(strvalue), ""); + + std::optional<int> intvalue; + EXPECT_EQ(absl::UnparseFlag(intvalue), ""); + + intvalue = 10; + EXPECT_EQ(absl::UnparseFlag(intvalue), "10"); +} + +// -------------------------------------------------------------------- + +#endif + template <typename T> void TestRoundtrip(T v) { T new_v; diff --git a/absl/flags/parse_test.cc b/absl/flags/parse_test.cc index 41bc0bc6..8dc91db2 100644 --- a/absl/flags/parse_test.cc +++ b/absl/flags/parse_test.cc @@ -46,6 +46,7 @@ using absl::base_internal::ScopedSetEnv; struct UDT { UDT() = default; UDT(const UDT&) = default; + UDT& operator=(const UDT&) = default; UDT(int v) : value(v) {} // NOLINT int value; diff --git a/absl/flags/reflection.cc b/absl/flags/reflection.cc index 0c761101..dbce4032 100644 --- a/absl/flags/reflection.cc +++ b/absl/flags/reflection.cc @@ -18,11 +18,11 @@ #include <assert.h> #include <atomic> -#include <map> #include <string> #include "absl/base/config.h" #include "absl/base/thread_annotations.h" +#include "absl/container/flat_hash_map.h" #include "absl/flags/commandlineflag.h" #include "absl/flags/internal/private_handle_accessor.h" #include "absl/flags/internal/registry.h" @@ -68,7 +68,7 @@ class FlagRegistry { friend void FinalizeRegistry(); // The map from name to flag, for FindFlag(). - using FlagMap = std::map<absl::string_view, CommandLineFlag*>; + using FlagMap = absl::flat_hash_map<absl::string_view, CommandLineFlag*>; using FlagIterator = FlagMap::iterator; using FlagConstIterator = FlagMap::const_iterator; FlagMap flags_; @@ -204,6 +204,10 @@ void FinalizeRegistry() { for (const auto& f : registry.flags_) { registry.flat_flags_.push_back(f.second); } + std::sort(std::begin(registry.flat_flags_), std::end(registry.flat_flags_), + [](const CommandLineFlag* lhs, const CommandLineFlag* rhs) { + return lhs->Name() < rhs->Name(); + }); registry.flags_.clear(); registry.finalized_flags_.store(true, std::memory_order_release); } diff --git a/absl/functional/BUILD.bazel b/absl/functional/BUILD.bazel index ebd9b99b..c4fbce98 100644 --- a/absl/functional/BUILD.bazel +++ b/absl/functional/BUILD.bazel @@ -14,7 +14,6 @@ # limitations under the License. # -load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") load( "//absl:copts/configure_copts.bzl", "ABSL_DEFAULT_COPTS", @@ -27,6 +26,40 @@ package(default_visibility = ["//visibility:public"]) licenses(["notice"]) cc_library( + name = "any_invocable", + srcs = ["internal/any_invocable.h"], + hdrs = ["any_invocable.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + "//absl/base:base_internal", + "//absl/base:config", + "//absl/base:core_headers", + "//absl/meta:type_traits", + "//absl/utility", + ], +) + +cc_test( + name = "any_invocable_test", + srcs = [ + "any_invocable_test.cc", + "internal/any_invocable.h", + ], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":any_invocable", + "//absl/base:base_internal", + "//absl/base:config", + "//absl/base:core_headers", + "//absl/meta:type_traits", + "//absl/utility", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( name = "bind_front", srcs = ["internal/front_binder.h"], hdrs = ["bind_front.h"], @@ -60,6 +93,7 @@ cc_library( linkopts = ABSL_DEFAULT_LINKOPTS, deps = [ "//absl/base:base_internal", + "//absl/base:core_headers", "//absl/meta:type_traits", ], ) @@ -78,14 +112,15 @@ cc_test( ) cc_test( - name = "function_ref_benchmark", + name = "function_type_benchmark", srcs = [ - "function_ref_benchmark.cc", + "function_type_benchmark.cc", ], copts = ABSL_TEST_COPTS, tags = ["benchmark"], visibility = ["//visibility:private"], deps = [ + ":any_invocable", ":function_ref", "//absl/base:core_headers", "@com_github_google_benchmark//:benchmark_main", diff --git a/absl/functional/CMakeLists.txt b/absl/functional/CMakeLists.txt index cda914f2..c0f6eaaa 100644 --- a/absl/functional/CMakeLists.txt +++ b/absl/functional/CMakeLists.txt @@ -16,6 +16,42 @@ absl_cc_library( NAME + any_invocable + SRCS + "internal/any_invocable.h" + HDRS + "any_invocable.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base_internal + absl::config + absl::core_headers + absl::type_traits + absl::utility + PUBLIC +) + +absl_cc_test( + NAME + any_invocable_test + SRCS + "any_invocable_test.cc" + "internal/any_invocable.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::any_invocable + absl::base_internal + absl::config + absl::core_headers + absl::type_traits + absl::utility + GTest::gmock_main +) + +absl_cc_library( + NAME bind_front SRCS "internal/front_binder.h" @@ -39,7 +75,7 @@ absl_cc_test( DEPS absl::bind_front absl::memory - gmock_main + GTest::gmock_main ) absl_cc_library( @@ -53,6 +89,7 @@ absl_cc_library( ${ABSL_DEFAULT_COPTS} DEPS absl::base_internal + absl::core_headers absl::meta PUBLIC ) @@ -68,5 +105,5 @@ absl_cc_test( absl::function_ref absl::memory absl::test_instance_tracker - gmock_main + GTest::gmock_main ) diff --git a/absl/functional/any_invocable.h b/absl/functional/any_invocable.h new file mode 100644 index 00000000..0c5faca0 --- /dev/null +++ b/absl/functional/any_invocable.h @@ -0,0 +1,313 @@ +// Copyright 2022 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: any_invocable.h +// ----------------------------------------------------------------------------- +// +// This header file defines an `absl::AnyInvocable` type that assumes ownership +// and wraps an object of an invocable type. (Invocable types adhere to the +// concept specified in https://en.cppreference.com/w/cpp/concepts/invocable.) +// +// In general, prefer `absl::AnyInvocable` when you need a type-erased +// function parameter that needs to take ownership of the type. +// +// NOTE: `absl::AnyInvocable` is similar to the C++23 `std::move_only_function` +// abstraction, but has a slightly different API and is not designed to be a +// drop-in replacement or C++11-compatible backfill of that type. + +#ifndef ABSL_FUNCTIONAL_ANY_INVOCABLE_H_ +#define ABSL_FUNCTIONAL_ANY_INVOCABLE_H_ + +#include <cstddef> +#include <initializer_list> +#include <type_traits> +#include <utility> + +#include "absl/base/config.h" +#include "absl/functional/internal/any_invocable.h" +#include "absl/meta/type_traits.h" +#include "absl/utility/utility.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// absl::AnyInvocable +// +// `absl::AnyInvocable` is a functional wrapper type, like `std::function`, that +// assumes ownership of an invocable object. Unlike `std::function`, an +// `absl::AnyInvocable` is more type-safe and provides the following additional +// benefits: +// +// * Properly adheres to const correctness of the underlying type +// * Is move-only so avoids concurrency problems with copied invocables and +// unnecessary copies in general. +// * Supports reference qualifiers allowing it to perform unique actions (noted +// below). +// +// `absl::AnyInvocable` is a template, and an `absl::AnyInvocable` instantiation +// may wrap any invocable object with a compatible function signature, e.g. +// having arguments and return types convertible to types matching the +// `absl::AnyInvocable` signature, and also matching any stated reference +// qualifiers, as long as that type is moveable. It therefore provides broad +// type erasure for functional objects. +// +// An `absl::AnyInvocable` is typically used as a type-erased function parameter +// for accepting various functional objects: +// +// // Define a function taking an AnyInvocable parameter. +// void my_func(absl::AnyInvocable<int()> f) { +// ... +// }; +// +// // That function can accept any invocable type: +// +// // Accept a function reference. We don't need to move a reference. +// int func1() { return 0; }; +// my_func(func1); +// +// // Accept a lambda. We use std::move here because otherwise my_func would +// // copy the lambda. +// auto lambda = []() { return 0; }; +// my_func(std::move(lambda)); +// +// // Accept a function pointer. We don't need to move a function pointer. +// func2 = &func1; +// my_func(func2); +// +// // Accept an std::function by moving it. Note that the lambda is copyable +// // (satisfying std::function requirements) and moveable (satisfying +// // absl::AnyInvocable requirements). +// std::function<int()> func6 = []() { return 0; }; +// my_func(std::move(func6)); +// +// `AnyInvocable` also properly respects `const` qualifiers, reference +// qualifiers, and the `noexcept` specification (only in C++ 17 and beyond) as +// part of the user-specified function type (e.g. +// `AnyInvocable<void()&& const noexcept>`). These qualifiers will be applied to +// the `AnyInvocable` object's `operator()`, and the underlying invocable must +// be compatible with those qualifiers. +// +// Comparison of const and non-const function types: +// +// // Store a closure inside of `func` with the function type `int()`. +// // Note that we have made `func` itself `const`. +// const AnyInvocable<int()> func = [](){ return 0; }; +// +// func(); // Compile-error: the passed type `int()` isn't `const`. +// +// // Store a closure inside of `const_func` with the function type +// // `int() const`. +// // Note that we have also made `const_func` itself `const`. +// const AnyInvocable<int() const> const_func = [](){ return 0; }; +// +// const_func(); // Fine: `int() const` is `const`. +// +// In the above example, the call `func()` would have compiled if +// `std::function` were used even though the types are not const compatible. +// This is a bug, and using `absl::AnyInvocable` properly detects that bug. +// +// In addition to affecting the signature of `operator()`, the `const` and +// reference qualifiers of the function type also appropriately constrain which +// kinds of invocable objects you are allowed to place into the `AnyInvocable` +// instance. If you specify a function type that is const-qualified, then +// anything that you attempt to put into the `AnyInvocable` must be callable on +// a `const` instance of that type. +// +// Constraint example: +// +// // Fine because the lambda is callable when `const`. +// AnyInvocable<int() const> func = [=](){ return 0; }; +// +// // This is a compile-error because the lambda isn't callable when `const`. +// AnyInvocable<int() const> error = [=]() mutable { return 0; }; +// +// An `&&` qualifier can be used to express that an `absl::AnyInvocable` +// instance should be invoked at most once: +// +// // Invokes `continuation` with the logical result of an operation when +// // that operation completes (common in asynchronous code). +// void CallOnCompletion(AnyInvocable<void(int)&&> continuation) { +// int result_of_foo = foo(); +// +// // `std::move` is required because the `operator()` of `continuation` is +// // rvalue-reference qualified. +// std::move(continuation)(result_of_foo); +// } +// +// Credits to Matt Calabrese (https://github.com/mattcalabrese) for the original +// implementation. +template <class Sig> +class AnyInvocable : private internal_any_invocable::Impl<Sig> { + private: + static_assert( + std::is_function<Sig>::value, + "The template argument of AnyInvocable must be a function type."); + + using Impl = internal_any_invocable::Impl<Sig>; + + public: + // The return type of Sig + using result_type = typename Impl::result_type; + + // Constructors + + // Constructs the `AnyInvocable` in an empty state. + AnyInvocable() noexcept = default; + AnyInvocable(std::nullptr_t) noexcept {} // NOLINT + + // Constructs the `AnyInvocable` from an existing `AnyInvocable` by a move. + // Note that `f` is not guaranteed to be empty after move-construction, + // although it may be. + AnyInvocable(AnyInvocable&& /*f*/) noexcept = default; + + // Constructs an `AnyInvocable` from an invocable object. + // + // Upon construction, `*this` is only empty if `f` is a function pointer or + // member pointer type and is null, or if `f` is an `AnyInvocable` that is + // empty. + template <class F, typename = absl::enable_if_t< + internal_any_invocable::CanConvert<Sig, F>::value>> + AnyInvocable(F&& f) // NOLINT + : Impl(internal_any_invocable::ConversionConstruct(), + std::forward<F>(f)) {} + + // Constructs an `AnyInvocable` that holds an invocable object of type `T`, + // which is constructed in-place from the given arguments. + // + // Example: + // + // AnyInvocable<int(int)> func( + // absl::in_place_type<PossiblyImmovableType>, arg1, arg2); + // + template <class T, class... Args, + typename = absl::enable_if_t< + internal_any_invocable::CanEmplace<Sig, T, Args...>::value>> + explicit AnyInvocable(absl::in_place_type_t<T>, Args&&... args) + : Impl(absl::in_place_type<absl::decay_t<T>>, + std::forward<Args>(args)...) { + static_assert(std::is_same<T, absl::decay_t<T>>::value, + "The explicit template argument of in_place_type is required " + "to be an unqualified object type."); + } + + // Overload of the above constructor to support list-initialization. + template <class T, class U, class... Args, + typename = absl::enable_if_t<internal_any_invocable::CanEmplace< + Sig, T, std::initializer_list<U>&, Args...>::value>> + explicit AnyInvocable(absl::in_place_type_t<T>, + std::initializer_list<U> ilist, Args&&... args) + : Impl(absl::in_place_type<absl::decay_t<T>>, ilist, + std::forward<Args>(args)...) { + static_assert(std::is_same<T, absl::decay_t<T>>::value, + "The explicit template argument of in_place_type is required " + "to be an unqualified object type."); + } + + // Assignment Operators + + // Assigns an `AnyInvocable` through move-assignment. + // Note that `f` is not guaranteed to be empty after move-assignment + // although it may be. + AnyInvocable& operator=(AnyInvocable&& /*f*/) noexcept = default; + + // Assigns an `AnyInvocable` from a nullptr, clearing the `AnyInvocable`. If + // not empty, destroys the target, putting `*this` into an empty state. + AnyInvocable& operator=(std::nullptr_t) noexcept { + this->Clear(); + return *this; + } + + // Assigns an `AnyInvocable` from an existing `AnyInvocable` instance. + // + // Upon assignment, `*this` is only empty if `f` is a function pointer or + // member pointer type and is null, or if `f` is an `AnyInvocable` that is + // empty. + template <class F, typename = absl::enable_if_t< + internal_any_invocable::CanAssign<Sig, F>::value>> + AnyInvocable& operator=(F&& f) { + *this = AnyInvocable(std::forward<F>(f)); + return *this; + } + + // Assigns an `AnyInvocable` from a reference to an invocable object. + // Upon assignment, stores a reference to the invocable object in the + // `AnyInvocable` instance. + template < + class F, + typename = absl::enable_if_t< + internal_any_invocable::CanAssignReferenceWrapper<Sig, F>::value>> + AnyInvocable& operator=(std::reference_wrapper<F> f) noexcept { + *this = AnyInvocable(f); + return *this; + } + + // Destructor + + // If not empty, destroys the target. + ~AnyInvocable() = default; + + // absl::AnyInvocable::swap() + // + // Exchanges the targets of `*this` and `other`. + void swap(AnyInvocable& other) noexcept { std::swap(*this, other); } + + // abl::AnyInvocable::operator bool() + // + // Returns `true` if `*this` is not empty. + explicit operator bool() const noexcept { return this->HasValue(); } + + // Invokes the target object of `*this`. `*this` must not be empty. + // + // Note: The signature of this function call operator is the same as the + // template parameter `Sig`. + using Impl::operator(); + + // Equality operators + + // Returns `true` if `*this` is empty. + friend bool operator==(const AnyInvocable& f, std::nullptr_t) noexcept { + return !f.HasValue(); + } + + // Returns `true` if `*this` is empty. + friend bool operator==(std::nullptr_t, const AnyInvocable& f) noexcept { + return !f.HasValue(); + } + + // Returns `false` if `*this` is empty. + friend bool operator!=(const AnyInvocable& f, std::nullptr_t) noexcept { + return f.HasValue(); + } + + // Returns `false` if `*this` is empty. + friend bool operator!=(std::nullptr_t, const AnyInvocable& f) noexcept { + return f.HasValue(); + } + + // swap() + // + // Exchanges the targets of `f1` and `f2`. + friend void swap(AnyInvocable& f1, AnyInvocable& f2) noexcept { f1.swap(f2); } + + private: + // Friending other instantiations is necessary for conversions. + template <bool /*SigIsNoexcept*/, class /*ReturnType*/, class... /*P*/> + friend class internal_any_invocable::CoreImpl; +}; + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_FUNCTIONAL_ANY_INVOCABLE_H_ diff --git a/absl/functional/any_invocable_test.cc b/absl/functional/any_invocable_test.cc new file mode 100644 index 00000000..fb5e7792 --- /dev/null +++ b/absl/functional/any_invocable_test.cc @@ -0,0 +1,1696 @@ +// Copyright 2022 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/functional/any_invocable.h" + +#include <cstddef> +#include <initializer_list> +#include <numeric> +#include <type_traits> + +#include "gtest/gtest.h" +#include "absl/base/config.h" +#include "absl/meta/type_traits.h" +#include "absl/utility/utility.h" + +static_assert(absl::internal_any_invocable::kStorageSize >= sizeof(void*), + "These tests assume that the small object storage is at least " + "the size of a pointer."); + +namespace { + +// Helper macro used to avoid spelling `noexcept` in language versions older +// than C++17, where it is not part of the type system, in order to avoid +// compilation failures and internal compiler errors. +#if ABSL_INTERNAL_CPLUSPLUS_LANG >= 201703L +#define ABSL_INTERNAL_NOEXCEPT_SPEC(noex) noexcept(noex) +#else +#define ABSL_INTERNAL_NOEXCEPT_SPEC(noex) +#endif + +// A dummy type we use when passing qualifiers to metafunctions +struct _ {}; + +template <class T> +struct Wrapper { + template <class U, + class = absl::enable_if_t<std::is_convertible<U, T>::value>> + Wrapper(U&&); // NOLINT +}; + +// This will cause a recursive trait instantiation if the SFINAE checks are +// not ordered correctly for constructibility. +static_assert(std::is_constructible<Wrapper<absl::AnyInvocable<void()>>, + Wrapper<absl::AnyInvocable<void()>>>::value, + ""); + +// A metafunction that takes the cv and l-value reference qualifiers that were +// associated with a function type (here passed via qualifiers of an object +// type), and . +template <class Qualifiers, class This> +struct QualifiersForThisImpl { + static_assert(std::is_object<This>::value, ""); + using type = + absl::conditional_t<std::is_const<Qualifiers>::value, const This, This>&; +}; + +template <class Qualifiers, class This> +struct QualifiersForThisImpl<Qualifiers&, This> + : QualifiersForThisImpl<Qualifiers, This> {}; + +template <class Qualifiers, class This> +struct QualifiersForThisImpl<Qualifiers&&, This> { + static_assert(std::is_object<This>::value, ""); + using type = + absl::conditional_t<std::is_const<Qualifiers>::value, const This, This>&&; +}; + +template <class Qualifiers, class This> +using QualifiersForThis = + typename QualifiersForThisImpl<Qualifiers, This>::type; + +// A metafunction that takes the cv and l-value reference qualifier of T and +// applies them to U's function type qualifiers. +template <class T, class Fun> +struct GiveQualifiersToFunImpl; + +template <class T, class R, class... P> +struct GiveQualifiersToFunImpl<T, R(P...)> { + using type = + absl::conditional_t<std::is_const<T>::value, R(P...) const, R(P...)>; +}; + +template <class T, class R, class... P> +struct GiveQualifiersToFunImpl<T&, R(P...)> { + using type = + absl::conditional_t<std::is_const<T>::value, R(P...) const&, R(P...)&>; +}; + +template <class T, class R, class... P> +struct GiveQualifiersToFunImpl<T&&, R(P...)> { + using type = + absl::conditional_t<std::is_const<T>::value, R(P...) const&&, R(P...) &&>; +}; + +// If noexcept is a part of the type system, then provide the noexcept forms. +#if defined(__cpp_noexcept_function_type) + +template <class T, class R, class... P> +struct GiveQualifiersToFunImpl<T, R(P...) noexcept> { + using type = absl::conditional_t<std::is_const<T>::value, + R(P...) const noexcept, R(P...) noexcept>; +}; + +template <class T, class R, class... P> +struct GiveQualifiersToFunImpl<T&, R(P...) noexcept> { + using type = + absl::conditional_t<std::is_const<T>::value, R(P...) const & noexcept, + R(P...) & noexcept>; +}; + +template <class T, class R, class... P> +struct GiveQualifiersToFunImpl<T&&, R(P...) noexcept> { + using type = + absl::conditional_t<std::is_const<T>::value, R(P...) const && noexcept, + R(P...) && noexcept>; +}; + +#endif // defined(__cpp_noexcept_function_type) + +template <class T, class Fun> +using GiveQualifiersToFun = typename GiveQualifiersToFunImpl<T, Fun>::type; + +// This is used in template parameters to decide whether or not to use a type +// that fits in the small object optimization storage. +enum class ObjSize { small, large }; + +// A base type that is used with classes as a means to insert an +// appropriately-sized dummy datamember when Size is ObjSize::large so that the +// user's class type is guaranteed to not fit in small object storage. +template <ObjSize Size> +struct TypeErasedPadding; + +template <> +struct TypeErasedPadding<ObjSize::small> {}; + +template <> +struct TypeErasedPadding<ObjSize::large> { + char dummy_data[absl::internal_any_invocable::kStorageSize + 1] = {}; +}; + +struct Int { + Int(int v) noexcept : value(v) {} // NOLINT +#ifndef _MSC_VER + Int(Int&&) noexcept { + // NOTE: Prior to C++17, this not being called requires optimizations to + // take place when performing the top-level invocation. In practice, + // most supported compilers perform this optimization prior to C++17. + std::abort(); + } +#else + Int(Int&& v) noexcept = default; +#endif + operator int() && noexcept { return value; } // NOLINT + + int MemberFunctionAdd(int const& b, int c) noexcept { // NOLINT + return value + b + c; + } + + int value; +}; + +enum class Movable { no, yes, nothrow, trivial }; + +enum class NothrowCall { no, yes }; + +enum class Destructible { nothrow, trivial }; + +enum class ObjAlign : std::size_t { + normal = absl::internal_any_invocable::kAlignment, + large = absl::internal_any_invocable::kAlignment * 2, +}; + +// A function-object template that has knobs for each property that can affect +// how the object is stored in AnyInvocable. +template <Movable Movability, Destructible Destructibility, class Qual, + NothrowCall CallExceptionSpec, ObjSize Size, ObjAlign Alignment> +struct add; + +#define ABSL_INTERNALS_ADD(qual) \ + template <NothrowCall CallExceptionSpec, ObjSize Size, ObjAlign Alignment> \ + struct alignas(static_cast<std::size_t>(Alignment)) \ + add<Movable::trivial, Destructible::trivial, _ qual, CallExceptionSpec, \ + Size, Alignment> : TypeErasedPadding<Size> { \ + explicit add(int state_init) : state(state_init) {} \ + explicit add(std::initializer_list<int> state_init, int tail) \ + : state(std::accumulate(std::begin(state_init), std::end(state_init), \ + 0) + \ + tail) {} \ + add(add&& other) = default; /*NOLINT*/ \ + Int operator()(int a, int b, int c) qual \ + ABSL_INTERNAL_NOEXCEPT_SPEC(CallExceptionSpec == NothrowCall::yes) { \ + return state + a + b + c; \ + } \ + int state; \ + }; \ + \ + template <NothrowCall CallExceptionSpec, ObjSize Size, ObjAlign Alignment> \ + struct alignas(static_cast<std::size_t>(Alignment)) \ + add<Movable::trivial, Destructible::nothrow, _ qual, CallExceptionSpec, \ + Size, Alignment> : TypeErasedPadding<Size> { \ + explicit add(int state_init) : state(state_init) {} \ + explicit add(std::initializer_list<int> state_init, int tail) \ + : state(std::accumulate(std::begin(state_init), std::end(state_init), \ + 0) + \ + tail) {} \ + ~add() noexcept {} \ + add(add&& other) = default; /*NOLINT*/ \ + Int operator()(int a, int b, int c) qual \ + ABSL_INTERNAL_NOEXCEPT_SPEC(CallExceptionSpec == NothrowCall::yes) { \ + return state + a + b + c; \ + } \ + int state; \ + } + +// Explicitly specify an empty argument. +// MSVC (at least up to _MSC_VER 1931, if not beyond) warns that +// ABSL_INTERNALS_ADD() is an undefined zero-arg overload. +#define ABSL_INTERNALS_NOARG +ABSL_INTERNALS_ADD(ABSL_INTERNALS_NOARG); +#undef ABSL_INTERNALS_NOARG + +ABSL_INTERNALS_ADD(const); +ABSL_INTERNALS_ADD(&); +ABSL_INTERNALS_ADD(const&); +ABSL_INTERNALS_ADD(&&); // NOLINT +ABSL_INTERNALS_ADD(const&&); // NOLINT + +#undef ABSL_INTERNALS_ADD + +template <Destructible Destructibility, class Qual, + NothrowCall CallExceptionSpec, ObjSize Size, ObjAlign Alignment> +struct add<Movable::no, Destructibility, Qual, CallExceptionSpec, Size, + Alignment> : private add<Movable::trivial, Destructibility, Qual, + CallExceptionSpec, Size, Alignment> { + using Base = add<Movable::trivial, Destructibility, Qual, CallExceptionSpec, + Size, Alignment>; + + explicit add(int state_init) : Base(state_init) {} + + explicit add(std::initializer_list<int> state_init, int tail) + : Base(state_init, tail) {} + + add(add&&) = delete; + + using Base::operator(); + using Base::state; +}; + +template <Destructible Destructibility, class Qual, + NothrowCall CallExceptionSpec, ObjSize Size, ObjAlign Alignment> +struct add<Movable::yes, Destructibility, Qual, CallExceptionSpec, Size, + Alignment> : private add<Movable::trivial, Destructibility, Qual, + CallExceptionSpec, Size, Alignment> { + using Base = add<Movable::trivial, Destructibility, Qual, CallExceptionSpec, + Size, Alignment>; + + explicit add(int state_init) : Base(state_init) {} + + explicit add(std::initializer_list<int> state_init, int tail) + : Base(state_init, tail) {} + + add(add&& other) noexcept(false) : Base(other.state) {} // NOLINT + + using Base::operator(); + using Base::state; +}; + +template <Destructible Destructibility, class Qual, + NothrowCall CallExceptionSpec, ObjSize Size, ObjAlign Alignment> +struct add<Movable::nothrow, Destructibility, Qual, CallExceptionSpec, Size, + Alignment> : private add<Movable::trivial, Destructibility, Qual, + CallExceptionSpec, Size, Alignment> { + using Base = add<Movable::trivial, Destructibility, Qual, CallExceptionSpec, + Size, Alignment>; + + explicit add(int state_init) : Base(state_init) {} + + explicit add(std::initializer_list<int> state_init, int tail) + : Base(state_init, tail) {} + + add(add&& other) noexcept : Base(other.state) {} + + using Base::operator(); + using Base::state; +}; + +// Actual non-member functions rather than function objects +Int add_function(Int&& a, int b, int c) noexcept { return a.value + b + c; } + +Int mult_function(Int&& a, int b, int c) noexcept { return a.value * b * c; } + +Int square_function(Int const&& a) noexcept { return a.value * a.value; } + +template <class Sig> +using AnyInvocable = absl::AnyInvocable<Sig>; + +// Instantiations of this template contains all of the compile-time parameters +// for a given instantiation of the AnyInvocable test suite. +template <Movable Movability, Destructible Destructibility, class Qual, + NothrowCall CallExceptionSpec, ObjSize Size, ObjAlign Alignment> +struct TestParams { + static constexpr Movable kMovability = Movability; + static constexpr Destructible kDestructibility = Destructibility; + using Qualifiers = Qual; + static constexpr NothrowCall kCallExceptionSpec = CallExceptionSpec; + static constexpr bool kIsNoexcept = kCallExceptionSpec == NothrowCall::yes; + static constexpr bool kIsRvalueQualified = + std::is_rvalue_reference<Qual>::value; + static constexpr ObjSize kSize = Size; + static constexpr ObjAlign kAlignment = Alignment; + + // These types are used when testing with member object pointer Invocables + using UnqualifiedUnaryFunType = int(Int const&&) + ABSL_INTERNAL_NOEXCEPT_SPEC(CallExceptionSpec == NothrowCall::yes); + using UnaryFunType = GiveQualifiersToFun<Qualifiers, UnqualifiedUnaryFunType>; + using MemObjPtrType = int(Int::*); + using UnaryAnyInvType = AnyInvocable<UnaryFunType>; + using UnaryThisParamType = QualifiersForThis<Qualifiers, UnaryAnyInvType>; + + template <class T> + static UnaryThisParamType ToUnaryThisParam(T&& fun) { + return static_cast<UnaryThisParamType>(fun); + } + + // This function type intentionally uses 3 "kinds" of parameter types. + // - A user-defined type + // - A reference type + // - A scalar type + // + // These were chosen because internal forwarding takes place on parameters + // differently depending based on type properties (scalars are forwarded by + // value). + using ResultType = Int; + using AnyInvocableFunTypeNotNoexcept = Int(Int, const int&, int); + using UnqualifiedFunType = + typename std::conditional<kIsNoexcept, Int(Int, const int&, int) noexcept, + Int(Int, const int&, int)>::type; + using FunType = GiveQualifiersToFun<Qualifiers, UnqualifiedFunType>; + using MemFunPtrType = + typename std::conditional<kIsNoexcept, + Int (Int::*)(const int&, int) noexcept, + Int (Int::*)(const int&, int)>::type; + using AnyInvType = AnyInvocable<FunType>; + using AddType = add<kMovability, kDestructibility, Qualifiers, + kCallExceptionSpec, kSize, kAlignment>; + using ThisParamType = QualifiersForThis<Qualifiers, AnyInvType>; + + template <class T> + static ThisParamType ToThisParam(T&& fun) { + return static_cast<ThisParamType>(fun); + } + + // These typedefs are used when testing void return type covariance. + using UnqualifiedVoidFunType = + typename std::conditional<kIsNoexcept, + void(Int, const int&, int) noexcept, + void(Int, const int&, int)>::type; + using VoidFunType = GiveQualifiersToFun<Qualifiers, UnqualifiedVoidFunType>; + using VoidAnyInvType = AnyInvocable<VoidFunType>; + using VoidThisParamType = QualifiersForThis<Qualifiers, VoidAnyInvType>; + + template <class T> + static VoidThisParamType ToVoidThisParam(T&& fun) { + return static_cast<VoidThisParamType>(fun); + } + + using CompatibleAnyInvocableFunType = + absl::conditional_t<std::is_rvalue_reference<Qual>::value, + GiveQualifiersToFun<const _&&, UnqualifiedFunType>, + GiveQualifiersToFun<const _&, UnqualifiedFunType>>; + + using CompatibleAnyInvType = AnyInvocable<CompatibleAnyInvocableFunType>; + + using IncompatibleInvocable = + absl::conditional_t<std::is_rvalue_reference<Qual>::value, + GiveQualifiersToFun<_&, UnqualifiedFunType>(_::*), + GiveQualifiersToFun<_&&, UnqualifiedFunType>(_::*)>; +}; + +// Given a member-pointer type, this metafunction yields the target type of the +// pointer, not including the class-type. It is used to verify that the function +// call operator of AnyInvocable has the proper signature, corresponding to the +// function type that the user provided. +template <class MemberPtrType> +struct MemberTypeOfImpl; + +template <class Class, class T> +struct MemberTypeOfImpl<T(Class::*)> { + using type = T; +}; + +template <class MemberPtrType> +using MemberTypeOf = typename MemberTypeOfImpl<MemberPtrType>::type; + +template <class T, class = void> +struct IsMemberSwappableImpl : std::false_type { + static constexpr bool kIsNothrow = false; +}; + +template <class T> +struct IsMemberSwappableImpl< + T, absl::void_t<decltype(std::declval<T&>().swap(std::declval<T&>()))>> + : std::true_type { + static constexpr bool kIsNothrow = + noexcept(std::declval<T&>().swap(std::declval<T&>())); +}; + +template <class T> +using IsMemberSwappable = IsMemberSwappableImpl<T>; + +template <class T> +using IsNothrowMemberSwappable = + std::integral_constant<bool, IsMemberSwappableImpl<T>::kIsNothrow>; + +template <class T> +class AnyInvTestBasic : public ::testing::Test {}; + +TYPED_TEST_SUITE_P(AnyInvTestBasic); + +TYPED_TEST_P(AnyInvTestBasic, DefaultConstruction) { + using AnyInvType = typename TypeParam::AnyInvType; + + AnyInvType fun; + + EXPECT_FALSE(static_cast<bool>(fun)); + + EXPECT_TRUE(std::is_nothrow_default_constructible<AnyInvType>::value); +} + +TYPED_TEST_P(AnyInvTestBasic, ConstructionNullptr) { + using AnyInvType = typename TypeParam::AnyInvType; + + AnyInvType fun = nullptr; + + EXPECT_FALSE(static_cast<bool>(fun)); + + EXPECT_TRUE( + (std::is_nothrow_constructible<AnyInvType, std::nullptr_t>::value)); +} + +TYPED_TEST_P(AnyInvTestBasic, ConstructionNullFunctionPtr) { + using AnyInvType = typename TypeParam::AnyInvType; + using UnqualifiedFunType = typename TypeParam::UnqualifiedFunType; + + UnqualifiedFunType* const null_fun_ptr = nullptr; + AnyInvType fun = null_fun_ptr; + + EXPECT_FALSE(static_cast<bool>(fun)); +} + +TYPED_TEST_P(AnyInvTestBasic, ConstructionNullMemberFunctionPtr) { + using AnyInvType = typename TypeParam::AnyInvType; + using MemFunPtrType = typename TypeParam::MemFunPtrType; + + const MemFunPtrType null_mem_fun_ptr = nullptr; + AnyInvType fun = null_mem_fun_ptr; + + EXPECT_FALSE(static_cast<bool>(fun)); +} + +TYPED_TEST_P(AnyInvTestBasic, ConstructionNullMemberObjectPtr) { + using UnaryAnyInvType = typename TypeParam::UnaryAnyInvType; + using MemObjPtrType = typename TypeParam::MemObjPtrType; + + const MemObjPtrType null_mem_obj_ptr = nullptr; + UnaryAnyInvType fun = null_mem_obj_ptr; + + EXPECT_FALSE(static_cast<bool>(fun)); +} + +TYPED_TEST_P(AnyInvTestBasic, ConstructionMemberFunctionPtr) { + using AnyInvType = typename TypeParam::AnyInvType; + + AnyInvType fun = &Int::MemberFunctionAdd; + + EXPECT_TRUE(static_cast<bool>(fun)); + EXPECT_EQ(24, TypeParam::ToThisParam(fun)(7, 8, 9).value); +} + +TYPED_TEST_P(AnyInvTestBasic, ConstructionMemberObjectPtr) { + using UnaryAnyInvType = typename TypeParam::UnaryAnyInvType; + + UnaryAnyInvType fun = &Int::value; + + EXPECT_TRUE(static_cast<bool>(fun)); + EXPECT_EQ(13, TypeParam::ToUnaryThisParam(fun)(13)); +} + +TYPED_TEST_P(AnyInvTestBasic, ConstructionFunctionReferenceDecay) { + using AnyInvType = typename TypeParam::AnyInvType; + + AnyInvType fun = add_function; + + EXPECT_TRUE(static_cast<bool>(fun)); + EXPECT_EQ(24, TypeParam::ToThisParam(fun)(7, 8, 9).value); +} + +TYPED_TEST_P(AnyInvTestBasic, ConstructionCompatibleAnyInvocableEmpty) { + using AnyInvType = typename TypeParam::AnyInvType; + using CompatibleAnyInvType = typename TypeParam::CompatibleAnyInvType; + + CompatibleAnyInvType other; + AnyInvType fun = std::move(other); + + EXPECT_FALSE(static_cast<bool>(other)); // NOLINT + EXPECT_EQ(other, nullptr); // NOLINT + EXPECT_EQ(nullptr, other); // NOLINT + + EXPECT_FALSE(static_cast<bool>(fun)); +} + +TYPED_TEST_P(AnyInvTestBasic, ConstructionCompatibleAnyInvocableNonempty) { + using AnyInvType = typename TypeParam::AnyInvType; + using CompatibleAnyInvType = typename TypeParam::CompatibleAnyInvType; + + CompatibleAnyInvType other = &add_function; + AnyInvType fun = std::move(other); + + EXPECT_FALSE(static_cast<bool>(other)); // NOLINT + EXPECT_EQ(other, nullptr); // NOLINT + EXPECT_EQ(nullptr, other); // NOLINT + + EXPECT_TRUE(static_cast<bool>(fun)); + EXPECT_EQ(24, TypeParam::ToThisParam(fun)(7, 8, 9).value); +} + +TYPED_TEST_P(AnyInvTestBasic, ConversionToBool) { + using AnyInvType = typename TypeParam::AnyInvType; + + { + AnyInvType fun; + + // This tests contextually-convertible-to-bool. + EXPECT_FALSE(fun ? true : false); // NOLINT + + // Make sure that the conversion is not implicit. + EXPECT_TRUE( + (std::is_nothrow_constructible<bool, const AnyInvType&>::value)); + EXPECT_FALSE((std::is_convertible<const AnyInvType&, bool>::value)); + } + + { + AnyInvType fun = &add_function; + + // This tests contextually-convertible-to-bool. + EXPECT_TRUE(fun ? true : false); // NOLINT + } +} + +TYPED_TEST_P(AnyInvTestBasic, Invocation) { + using AnyInvType = typename TypeParam::AnyInvType; + + using FunType = typename TypeParam::FunType; + using AnyInvCallType = MemberTypeOf<decltype(&AnyInvType::operator())>; + + // Make sure the function call operator of AnyInvocable always has the + // type that was specified via the template argument. + EXPECT_TRUE((std::is_same<AnyInvCallType, FunType>::value)); + + AnyInvType fun = &add_function; + + EXPECT_EQ(24, TypeParam::ToThisParam(fun)(7, 8, 9).value); +} + +TYPED_TEST_P(AnyInvTestBasic, InPlaceConstruction) { + using AnyInvType = typename TypeParam::AnyInvType; + using AddType = typename TypeParam::AddType; + + AnyInvType fun(absl::in_place_type<AddType>, 5); + + EXPECT_TRUE(static_cast<bool>(fun)); + EXPECT_EQ(29, TypeParam::ToThisParam(fun)(7, 8, 9).value); +} + +TYPED_TEST_P(AnyInvTestBasic, InPlaceConstructionInitializerList) { + using AnyInvType = typename TypeParam::AnyInvType; + using AddType = typename TypeParam::AddType; + + AnyInvType fun(absl::in_place_type<AddType>, {1, 2, 3, 4}, 5); + + EXPECT_TRUE(static_cast<bool>(fun)); + EXPECT_EQ(39, TypeParam::ToThisParam(fun)(7, 8, 9).value); +} + +TYPED_TEST_P(AnyInvTestBasic, InPlaceNullFunPtrConstruction) { + using AnyInvType = typename TypeParam::AnyInvType; + using UnqualifiedFunType = typename TypeParam::UnqualifiedFunType; + + AnyInvType fun(absl::in_place_type<UnqualifiedFunType*>, nullptr); + + // In-place construction does not lead to empty. + EXPECT_TRUE(static_cast<bool>(fun)); +} + +TYPED_TEST_P(AnyInvTestBasic, InPlaceNullFunPtrConstructionValueInit) { + using AnyInvType = typename TypeParam::AnyInvType; + using UnqualifiedFunType = typename TypeParam::UnqualifiedFunType; + + AnyInvType fun(absl::in_place_type<UnqualifiedFunType*>); + + // In-place construction does not lead to empty. + EXPECT_TRUE(static_cast<bool>(fun)); +} + +TYPED_TEST_P(AnyInvTestBasic, InPlaceNullMemFunPtrConstruction) { + using AnyInvType = typename TypeParam::AnyInvType; + using MemFunPtrType = typename TypeParam::MemFunPtrType; + + AnyInvType fun(absl::in_place_type<MemFunPtrType>, nullptr); + + // In-place construction does not lead to empty. + EXPECT_TRUE(static_cast<bool>(fun)); +} + +TYPED_TEST_P(AnyInvTestBasic, InPlaceNullMemFunPtrConstructionValueInit) { + using AnyInvType = typename TypeParam::AnyInvType; + using MemFunPtrType = typename TypeParam::MemFunPtrType; + + AnyInvType fun(absl::in_place_type<MemFunPtrType>); + + // In-place construction does not lead to empty. + EXPECT_TRUE(static_cast<bool>(fun)); +} + +TYPED_TEST_P(AnyInvTestBasic, InPlaceNullMemObjPtrConstruction) { + using UnaryAnyInvType = typename TypeParam::UnaryAnyInvType; + using MemObjPtrType = typename TypeParam::MemObjPtrType; + + UnaryAnyInvType fun(absl::in_place_type<MemObjPtrType>, nullptr); + + // In-place construction does not lead to empty. + EXPECT_TRUE(static_cast<bool>(fun)); +} + +TYPED_TEST_P(AnyInvTestBasic, InPlaceNullMemObjPtrConstructionValueInit) { + using UnaryAnyInvType = typename TypeParam::UnaryAnyInvType; + using MemObjPtrType = typename TypeParam::MemObjPtrType; + + UnaryAnyInvType fun(absl::in_place_type<MemObjPtrType>); + + // In-place construction does not lead to empty. + EXPECT_TRUE(static_cast<bool>(fun)); +} + +TYPED_TEST_P(AnyInvTestBasic, InPlaceVoidCovarianceConstruction) { + using VoidAnyInvType = typename TypeParam::VoidAnyInvType; + using AddType = typename TypeParam::AddType; + + VoidAnyInvType fun(absl::in_place_type<AddType>, 5); + + EXPECT_TRUE(static_cast<bool>(fun)); +} + +TYPED_TEST_P(AnyInvTestBasic, MoveConstructionFromEmpty) { + using AnyInvType = typename TypeParam::AnyInvType; + + AnyInvType source_fun; + AnyInvType fun(std::move(source_fun)); + + EXPECT_FALSE(static_cast<bool>(fun)); + + EXPECT_TRUE(std::is_nothrow_move_constructible<AnyInvType>::value); +} + +TYPED_TEST_P(AnyInvTestBasic, MoveConstructionFromNonEmpty) { + using AnyInvType = typename TypeParam::AnyInvType; + using AddType = typename TypeParam::AddType; + + AnyInvType source_fun(absl::in_place_type<AddType>, 5); + AnyInvType fun(std::move(source_fun)); + + EXPECT_TRUE(static_cast<bool>(fun)); + EXPECT_EQ(29, TypeParam::ToThisParam(fun)(7, 8, 9).value); + + EXPECT_TRUE(std::is_nothrow_move_constructible<AnyInvType>::value); +} + +TYPED_TEST_P(AnyInvTestBasic, ComparisonWithNullptrEmpty) { + using AnyInvType = typename TypeParam::AnyInvType; + + AnyInvType fun; + + EXPECT_TRUE(fun == nullptr); + EXPECT_TRUE(nullptr == fun); + + EXPECT_FALSE(fun != nullptr); + EXPECT_FALSE(nullptr != fun); +} + +TYPED_TEST_P(AnyInvTestBasic, ComparisonWithNullptrNonempty) { + using AnyInvType = typename TypeParam::AnyInvType; + using AddType = typename TypeParam::AddType; + + AnyInvType fun(absl::in_place_type<AddType>, 5); + + EXPECT_FALSE(fun == nullptr); + EXPECT_FALSE(nullptr == fun); + + EXPECT_TRUE(fun != nullptr); + EXPECT_TRUE(nullptr != fun); +} + +TYPED_TEST_P(AnyInvTestBasic, ResultType) { + using AnyInvType = typename TypeParam::AnyInvType; + using ExpectedResultType = typename TypeParam::ResultType; + + EXPECT_TRUE((std::is_same<typename AnyInvType::result_type, + ExpectedResultType>::value)); +} + +template <class T> +class AnyInvTestCombinatoric : public ::testing::Test {}; + +TYPED_TEST_SUITE_P(AnyInvTestCombinatoric); + +TYPED_TEST_P(AnyInvTestCombinatoric, MoveAssignEmptyEmptyLhsRhs) { + using AnyInvType = typename TypeParam::AnyInvType; + + AnyInvType source_fun; + AnyInvType fun; + + fun = std::move(source_fun); + + EXPECT_FALSE(static_cast<bool>(fun)); +} + +TYPED_TEST_P(AnyInvTestCombinatoric, MoveAssignEmptyLhsNonemptyRhs) { + using AnyInvType = typename TypeParam::AnyInvType; + using AddType = typename TypeParam::AddType; + + AnyInvType source_fun(absl::in_place_type<AddType>, 5); + AnyInvType fun; + + fun = std::move(source_fun); + + EXPECT_TRUE(static_cast<bool>(fun)); + EXPECT_EQ(29, TypeParam::ToThisParam(fun)(7, 8, 9).value); +} + +TYPED_TEST_P(AnyInvTestCombinatoric, MoveAssignNonemptyEmptyLhsRhs) { + using AnyInvType = typename TypeParam::AnyInvType; + using AddType = typename TypeParam::AddType; + + AnyInvType source_fun; + AnyInvType fun(absl::in_place_type<AddType>, 5); + + fun = std::move(source_fun); + + EXPECT_FALSE(static_cast<bool>(fun)); +} + +TYPED_TEST_P(AnyInvTestCombinatoric, MoveAssignNonemptyLhsNonemptyRhs) { + using AnyInvType = typename TypeParam::AnyInvType; + using AddType = typename TypeParam::AddType; + + AnyInvType source_fun(absl::in_place_type<AddType>, 5); + AnyInvType fun(absl::in_place_type<AddType>, 20); + + fun = std::move(source_fun); + + EXPECT_TRUE(static_cast<bool>(fun)); + EXPECT_EQ(29, TypeParam::ToThisParam(fun)(7, 8, 9).value); +} + +TYPED_TEST_P(AnyInvTestCombinatoric, SelfMoveAssignEmpty) { + using AnyInvType = typename TypeParam::AnyInvType; + + AnyInvType source_fun; + source_fun = std::move(source_fun); + + // This space intentionally left blank. +} + +TYPED_TEST_P(AnyInvTestCombinatoric, SelfMoveAssignNonempty) { + using AnyInvType = typename TypeParam::AnyInvType; + using AddType = typename TypeParam::AddType; + + AnyInvType source_fun(absl::in_place_type<AddType>, 5); + source_fun = std::move(source_fun); + + // This space intentionally left blank. +} + +TYPED_TEST_P(AnyInvTestCombinatoric, AssignNullptrEmptyLhs) { + using AnyInvType = typename TypeParam::AnyInvType; + + AnyInvType fun; + fun = nullptr; + + EXPECT_FALSE(static_cast<bool>(fun)); +} + +TYPED_TEST_P(AnyInvTestCombinatoric, AssignNullFunctionPtrEmptyLhs) { + using AnyInvType = typename TypeParam::AnyInvType; + using UnqualifiedFunType = typename TypeParam::UnqualifiedFunType; + + UnqualifiedFunType* const null_fun_ptr = nullptr; + AnyInvType fun; + fun = null_fun_ptr; + + EXPECT_FALSE(static_cast<bool>(fun)); +} + +TYPED_TEST_P(AnyInvTestCombinatoric, AssignNullMemberFunctionPtrEmptyLhs) { + using AnyInvType = typename TypeParam::AnyInvType; + using MemFunPtrType = typename TypeParam::MemFunPtrType; + + const MemFunPtrType null_mem_fun_ptr = nullptr; + AnyInvType fun; + fun = null_mem_fun_ptr; + + EXPECT_FALSE(static_cast<bool>(fun)); +} + +TYPED_TEST_P(AnyInvTestCombinatoric, AssignNullMemberObjectPtrEmptyLhs) { + using UnaryAnyInvType = typename TypeParam::UnaryAnyInvType; + using MemObjPtrType = typename TypeParam::MemObjPtrType; + + const MemObjPtrType null_mem_obj_ptr = nullptr; + UnaryAnyInvType fun; + fun = null_mem_obj_ptr; + + EXPECT_FALSE(static_cast<bool>(fun)); +} + +TYPED_TEST_P(AnyInvTestCombinatoric, AssignMemberFunctionPtrEmptyLhs) { + using AnyInvType = typename TypeParam::AnyInvType; + + AnyInvType fun; + fun = &Int::MemberFunctionAdd; + + EXPECT_TRUE(static_cast<bool>(fun)); + EXPECT_EQ(24, TypeParam::ToThisParam(fun)(7, 8, 9).value); +} + +TYPED_TEST_P(AnyInvTestCombinatoric, AssignMemberObjectPtrEmptyLhs) { + using UnaryAnyInvType = typename TypeParam::UnaryAnyInvType; + + UnaryAnyInvType fun; + fun = &Int::value; + + EXPECT_TRUE(static_cast<bool>(fun)); + EXPECT_EQ(13, TypeParam::ToUnaryThisParam(fun)(13)); +} + +TYPED_TEST_P(AnyInvTestCombinatoric, AssignFunctionReferenceDecayEmptyLhs) { + using AnyInvType = typename TypeParam::AnyInvType; + + AnyInvType fun; + fun = add_function; + + EXPECT_TRUE(static_cast<bool>(fun)); + EXPECT_EQ(24, TypeParam::ToThisParam(fun)(7, 8, 9).value); +} + +TYPED_TEST_P(AnyInvTestCombinatoric, + AssignCompatibleAnyInvocableEmptyLhsEmptyRhs) { + using AnyInvType = typename TypeParam::AnyInvType; + using CompatibleAnyInvType = typename TypeParam::CompatibleAnyInvType; + + CompatibleAnyInvType other; + AnyInvType fun; + fun = std::move(other); + + EXPECT_FALSE(static_cast<bool>(other)); // NOLINT + EXPECT_EQ(other, nullptr); // NOLINT + EXPECT_EQ(nullptr, other); // NOLINT + + EXPECT_FALSE(static_cast<bool>(fun)); +} + +TYPED_TEST_P(AnyInvTestCombinatoric, + AssignCompatibleAnyInvocableEmptyLhsNonemptyRhs) { + using AnyInvType = typename TypeParam::AnyInvType; + using CompatibleAnyInvType = typename TypeParam::CompatibleAnyInvType; + + CompatibleAnyInvType other = &add_function; + AnyInvType fun; + fun = std::move(other); + + EXPECT_FALSE(static_cast<bool>(other)); // NOLINT + + EXPECT_TRUE(static_cast<bool>(fun)); + EXPECT_EQ(24, TypeParam::ToThisParam(fun)(7, 8, 9).value); +} + +TYPED_TEST_P(AnyInvTestCombinatoric, AssignNullptrNonemptyLhs) { + using AnyInvType = typename TypeParam::AnyInvType; + + AnyInvType fun = &mult_function; + fun = nullptr; + + EXPECT_FALSE(static_cast<bool>(fun)); +} + +TYPED_TEST_P(AnyInvTestCombinatoric, AssignNullFunctionPtrNonemptyLhs) { + using AnyInvType = typename TypeParam::AnyInvType; + using UnqualifiedFunType = typename TypeParam::UnqualifiedFunType; + + UnqualifiedFunType* const null_fun_ptr = nullptr; + AnyInvType fun = &mult_function; + fun = null_fun_ptr; + + EXPECT_FALSE(static_cast<bool>(fun)); +} + +TYPED_TEST_P(AnyInvTestCombinatoric, AssignNullMemberFunctionPtrNonemptyLhs) { + using AnyInvType = typename TypeParam::AnyInvType; + using MemFunPtrType = typename TypeParam::MemFunPtrType; + + const MemFunPtrType null_mem_fun_ptr = nullptr; + AnyInvType fun = &mult_function; + fun = null_mem_fun_ptr; + + EXPECT_FALSE(static_cast<bool>(fun)); +} + +TYPED_TEST_P(AnyInvTestCombinatoric, AssignNullMemberObjectPtrNonemptyLhs) { + using UnaryAnyInvType = typename TypeParam::UnaryAnyInvType; + using MemObjPtrType = typename TypeParam::MemObjPtrType; + + const MemObjPtrType null_mem_obj_ptr = nullptr; + UnaryAnyInvType fun = &square_function; + fun = null_mem_obj_ptr; + + EXPECT_FALSE(static_cast<bool>(fun)); +} + +TYPED_TEST_P(AnyInvTestCombinatoric, AssignMemberFunctionPtrNonemptyLhs) { + using AnyInvType = typename TypeParam::AnyInvType; + + AnyInvType fun = &mult_function; + fun = &Int::MemberFunctionAdd; + + EXPECT_TRUE(static_cast<bool>(fun)); + EXPECT_EQ(24, TypeParam::ToThisParam(fun)(7, 8, 9).value); +} + +TYPED_TEST_P(AnyInvTestCombinatoric, AssignMemberObjectPtrNonemptyLhs) { + using UnaryAnyInvType = typename TypeParam::UnaryAnyInvType; + + UnaryAnyInvType fun = &square_function; + fun = &Int::value; + + EXPECT_TRUE(static_cast<bool>(fun)); + EXPECT_EQ(13, TypeParam::ToUnaryThisParam(fun)(13)); +} + +TYPED_TEST_P(AnyInvTestCombinatoric, AssignFunctionReferenceDecayNonemptyLhs) { + using AnyInvType = typename TypeParam::AnyInvType; + + AnyInvType fun = &mult_function; + fun = add_function; + + EXPECT_TRUE(static_cast<bool>(fun)); + EXPECT_EQ(24, TypeParam::ToThisParam(fun)(7, 8, 9).value); +} + +TYPED_TEST_P(AnyInvTestCombinatoric, + AssignCompatibleAnyInvocableNonemptyLhsEmptyRhs) { + using AnyInvType = typename TypeParam::AnyInvType; + using CompatibleAnyInvType = typename TypeParam::CompatibleAnyInvType; + + CompatibleAnyInvType other; + AnyInvType fun = &mult_function; + fun = std::move(other); + + EXPECT_FALSE(static_cast<bool>(other)); // NOLINT + EXPECT_EQ(other, nullptr); // NOLINT + EXPECT_EQ(nullptr, other); // NOLINT + + EXPECT_FALSE(static_cast<bool>(fun)); +} + +TYPED_TEST_P(AnyInvTestCombinatoric, + AssignCompatibleAnyInvocableNonemptyLhsNonemptyRhs) { + using AnyInvType = typename TypeParam::AnyInvType; + using CompatibleAnyInvType = typename TypeParam::CompatibleAnyInvType; + + CompatibleAnyInvType other = &add_function; + AnyInvType fun = &mult_function; + fun = std::move(other); + + EXPECT_FALSE(static_cast<bool>(other)); // NOLINT + + EXPECT_TRUE(static_cast<bool>(fun)); + EXPECT_EQ(24, TypeParam::ToThisParam(fun)(7, 8, 9).value); +} + +TYPED_TEST_P(AnyInvTestCombinatoric, SwapEmptyLhsEmptyRhs) { + using AnyInvType = typename TypeParam::AnyInvType; + + // Swap idiom + { + AnyInvType fun; + AnyInvType other; + + using std::swap; + swap(fun, other); + + EXPECT_FALSE(static_cast<bool>(fun)); + EXPECT_FALSE(static_cast<bool>(other)); + + EXPECT_TRUE( + absl::type_traits_internal::IsNothrowSwappable<AnyInvType>::value); + } + + // Member swap + { + AnyInvType fun; + AnyInvType other; + + fun.swap(other); + + EXPECT_FALSE(static_cast<bool>(fun)); + EXPECT_FALSE(static_cast<bool>(other)); + + EXPECT_TRUE(IsNothrowMemberSwappable<AnyInvType>::value); + } +} + +TYPED_TEST_P(AnyInvTestCombinatoric, SwapEmptyLhsNonemptyRhs) { + using AnyInvType = typename TypeParam::AnyInvType; + using AddType = typename TypeParam::AddType; + + // Swap idiom + { + AnyInvType fun; + AnyInvType other(absl::in_place_type<AddType>, 5); + + using std::swap; + swap(fun, other); + + EXPECT_TRUE(static_cast<bool>(fun)); + EXPECT_FALSE(static_cast<bool>(other)); + + EXPECT_EQ(29, TypeParam::ToThisParam(fun)(7, 8, 9).value); + + EXPECT_TRUE( + absl::type_traits_internal::IsNothrowSwappable<AnyInvType>::value); + } + + // Member swap + { + AnyInvType fun; + AnyInvType other(absl::in_place_type<AddType>, 5); + + fun.swap(other); + + EXPECT_TRUE(static_cast<bool>(fun)); + EXPECT_FALSE(static_cast<bool>(other)); + + EXPECT_EQ(29, TypeParam::ToThisParam(fun)(7, 8, 9).value); + + EXPECT_TRUE(IsNothrowMemberSwappable<AnyInvType>::value); + } +} + +TYPED_TEST_P(AnyInvTestCombinatoric, SwapNonemptyLhsEmptyRhs) { + using AnyInvType = typename TypeParam::AnyInvType; + using AddType = typename TypeParam::AddType; + + // Swap idiom + { + AnyInvType fun(absl::in_place_type<AddType>, 5); + AnyInvType other; + + using std::swap; + swap(fun, other); + + EXPECT_FALSE(static_cast<bool>(fun)); + EXPECT_TRUE(static_cast<bool>(other)); + + EXPECT_EQ(29, TypeParam::ToThisParam(other)(7, 8, 9).value); + + EXPECT_TRUE( + absl::type_traits_internal::IsNothrowSwappable<AnyInvType>::value); + } + + // Member swap + { + AnyInvType fun(absl::in_place_type<AddType>, 5); + AnyInvType other; + + fun.swap(other); + + EXPECT_FALSE(static_cast<bool>(fun)); + EXPECT_TRUE(static_cast<bool>(other)); + + EXPECT_EQ(29, TypeParam::ToThisParam(other)(7, 8, 9).value); + + EXPECT_TRUE(IsNothrowMemberSwappable<AnyInvType>::value); + } +} + +TYPED_TEST_P(AnyInvTestCombinatoric, SwapNonemptyLhsNonemptyRhs) { + using AnyInvType = typename TypeParam::AnyInvType; + using AddType = typename TypeParam::AddType; + + // Swap idiom + { + AnyInvType fun(absl::in_place_type<AddType>, 5); + AnyInvType other(absl::in_place_type<AddType>, 6); + + using std::swap; + swap(fun, other); + + EXPECT_TRUE(static_cast<bool>(fun)); + EXPECT_TRUE(static_cast<bool>(other)); + + EXPECT_EQ(30, TypeParam::ToThisParam(fun)(7, 8, 9).value); + EXPECT_EQ(29, TypeParam::ToThisParam(other)(7, 8, 9).value); + + EXPECT_TRUE( + absl::type_traits_internal::IsNothrowSwappable<AnyInvType>::value); + } + + // Member swap + { + AnyInvType fun(absl::in_place_type<AddType>, 5); + AnyInvType other(absl::in_place_type<AddType>, 6); + + fun.swap(other); + + EXPECT_TRUE(static_cast<bool>(fun)); + EXPECT_TRUE(static_cast<bool>(other)); + + EXPECT_EQ(30, TypeParam::ToThisParam(fun)(7, 8, 9).value); + EXPECT_EQ(29, TypeParam::ToThisParam(other)(7, 8, 9).value); + + EXPECT_TRUE(IsNothrowMemberSwappable<AnyInvType>::value); + } +} + +template <class T> +class AnyInvTestMovable : public ::testing::Test {}; + +TYPED_TEST_SUITE_P(AnyInvTestMovable); + +TYPED_TEST_P(AnyInvTestMovable, ConversionConstructionUserDefinedType) { + using AnyInvType = typename TypeParam::AnyInvType; + using AddType = typename TypeParam::AddType; + + AnyInvType fun(AddType(5)); + + EXPECT_TRUE(static_cast<bool>(fun)); + EXPECT_EQ(29, TypeParam::ToThisParam(fun)(7, 8, 9).value); + + EXPECT_TRUE(static_cast<bool>(fun)); + EXPECT_EQ(38, TypeParam::ToThisParam(fun)(10, 11, 12).value); +} + +TYPED_TEST_P(AnyInvTestMovable, ConversionConstructionVoidCovariance) { + using VoidAnyInvType = typename TypeParam::VoidAnyInvType; + using AddType = typename TypeParam::AddType; + + VoidAnyInvType fun(AddType(5)); + + EXPECT_TRUE(static_cast<bool>(fun)); +} + +TYPED_TEST_P(AnyInvTestMovable, ConversionAssignUserDefinedTypeEmptyLhs) { + using AnyInvType = typename TypeParam::AnyInvType; + using AddType = typename TypeParam::AddType; + + AnyInvType fun; + fun = AddType(5); + + EXPECT_TRUE(static_cast<bool>(fun)); + EXPECT_EQ(29, TypeParam::ToThisParam(fun)(7, 8, 9).value); + + EXPECT_TRUE(static_cast<bool>(fun)); + EXPECT_EQ(38, TypeParam::ToThisParam(fun)(10, 11, 12).value); +} + +TYPED_TEST_P(AnyInvTestMovable, ConversionAssignUserDefinedTypeNonemptyLhs) { + using AnyInvType = typename TypeParam::AnyInvType; + using AddType = typename TypeParam::AddType; + + AnyInvType fun = &add_function; + fun = AddType(5); + + EXPECT_TRUE(static_cast<bool>(fun)); + EXPECT_EQ(29, TypeParam::ToThisParam(fun)(7, 8, 9).value); + + EXPECT_TRUE(static_cast<bool>(fun)); + EXPECT_EQ(38, TypeParam::ToThisParam(fun)(10, 11, 12).value); +} + +TYPED_TEST_P(AnyInvTestMovable, ConversionAssignVoidCovariance) { + using VoidAnyInvType = typename TypeParam::VoidAnyInvType; + using AddType = typename TypeParam::AddType; + + VoidAnyInvType fun; + fun = AddType(5); + + EXPECT_TRUE(static_cast<bool>(fun)); +} + +template <class T> +class AnyInvTestNoexceptFalse : public ::testing::Test {}; + +TYPED_TEST_SUITE_P(AnyInvTestNoexceptFalse); + +TYPED_TEST_P(AnyInvTestNoexceptFalse, ConversionConstructionConstraints) { + using AnyInvType = typename TypeParam::AnyInvType; + + EXPECT_TRUE((std::is_constructible< + AnyInvType, + typename TypeParam::AnyInvocableFunTypeNotNoexcept*>::value)); + EXPECT_FALSE(( + std::is_constructible<AnyInvType, + typename TypeParam::IncompatibleInvocable>::value)); +} + +TYPED_TEST_P(AnyInvTestNoexceptFalse, ConversionAssignConstraints) { + using AnyInvType = typename TypeParam::AnyInvType; + + EXPECT_TRUE((std::is_assignable< + AnyInvType&, + typename TypeParam::AnyInvocableFunTypeNotNoexcept*>::value)); + EXPECT_FALSE( + (std::is_assignable<AnyInvType&, + typename TypeParam::IncompatibleInvocable>::value)); +} + +template <class T> +class AnyInvTestNoexceptTrue : public ::testing::Test {}; + +TYPED_TEST_SUITE_P(AnyInvTestNoexceptTrue); + +TYPED_TEST_P(AnyInvTestNoexceptTrue, ConversionConstructionConstraints) { +#if ABSL_INTERNAL_CPLUSPLUS_LANG < 201703L + GTEST_SKIP() << "Noexcept was not part of the type system before C++17."; +#else + using AnyInvType = typename TypeParam::AnyInvType; + +// TODO(b/217761454): Fix this and re-enable for MSVC. +#ifndef _MSC_VER + EXPECT_FALSE((std::is_constructible< + AnyInvType, + typename TypeParam::AnyInvocableFunTypeNotNoexcept*>::value)); +#endif + EXPECT_FALSE(( + std::is_constructible<AnyInvType, + typename TypeParam::IncompatibleInvocable>::value)); +#endif +} + +TYPED_TEST_P(AnyInvTestNoexceptTrue, ConversionAssignConstraints) { +#if ABSL_INTERNAL_CPLUSPLUS_LANG < 201703L + GTEST_SKIP() << "Noexcept was not part of the type system before C++17."; +#else + using AnyInvType = typename TypeParam::AnyInvType; + +// TODO(b/217761454): Fix this and re-enable for MSVC. +#ifndef _MSC_VER + EXPECT_FALSE((std::is_assignable< + AnyInvType&, + typename TypeParam::AnyInvocableFunTypeNotNoexcept*>::value)); +#endif + EXPECT_FALSE( + (std::is_assignable<AnyInvType&, + typename TypeParam::IncompatibleInvocable>::value)); +#endif +} + +template <class T> +class AnyInvTestNonRvalue : public ::testing::Test {}; + +TYPED_TEST_SUITE_P(AnyInvTestNonRvalue); + +TYPED_TEST_P(AnyInvTestNonRvalue, ConversionConstructionReferenceWrapper) { + using AnyInvType = typename TypeParam::AnyInvType; + using AddType = typename TypeParam::AddType; + + AddType add(4); + AnyInvType fun = std::ref(add); + add.state = 5; + + EXPECT_TRUE(static_cast<bool>(fun)); + EXPECT_EQ(29, TypeParam::ToThisParam(fun)(7, 8, 9).value); + + EXPECT_TRUE(static_cast<bool>(fun)); + EXPECT_EQ(38, TypeParam::ToThisParam(fun)(10, 11, 12).value); +} + +TYPED_TEST_P(AnyInvTestNonRvalue, NonMoveableResultType) { +#if ABSL_INTERNAL_CPLUSPLUS_LANG < 201703L + GTEST_SKIP() << "Copy/move elision was not standard before C++17"; +#else + // Define a result type that cannot be copy- or move-constructed. + struct Result { + int x; + + explicit Result(const int x_in) : x(x_in) {} + Result(Result&&) = delete; + }; + + static_assert(!std::is_move_constructible<Result>::value, ""); + static_assert(!std::is_copy_constructible<Result>::value, ""); + + // Assumption check: it should nevertheless be possible to use functors that + // return a Result struct according to the language rules. + const auto return_17 = []() noexcept { return Result(17); }; + EXPECT_EQ(17, return_17().x); + + // Just like plain functors, it should work fine to use an AnyInvocable that + // returns the non-moveable type. + using UnqualifiedFun = + absl::conditional_t<TypeParam::kIsNoexcept, Result() noexcept, Result()>; + + using Fun = + GiveQualifiersToFun<typename TypeParam::Qualifiers, UnqualifiedFun>; + + AnyInvocable<Fun> any_inv(return_17); + EXPECT_EQ(17, any_inv().x); +#endif +} + +TYPED_TEST_P(AnyInvTestNonRvalue, ConversionAssignReferenceWrapperEmptyLhs) { + using AnyInvType = typename TypeParam::AnyInvType; + using AddType = typename TypeParam::AddType; + + AddType add(4); + AnyInvType fun; + fun = std::ref(add); + add.state = 5; + EXPECT_TRUE( + (std::is_nothrow_assignable<AnyInvType&, + std::reference_wrapper<AddType>>::value)); + + EXPECT_TRUE(static_cast<bool>(fun)); + EXPECT_EQ(29, TypeParam::ToThisParam(fun)(7, 8, 9).value); + + EXPECT_TRUE(static_cast<bool>(fun)); + EXPECT_EQ(38, TypeParam::ToThisParam(fun)(10, 11, 12).value); +} + +TYPED_TEST_P(AnyInvTestNonRvalue, ConversionAssignReferenceWrapperNonemptyLhs) { + using AnyInvType = typename TypeParam::AnyInvType; + using AddType = typename TypeParam::AddType; + + AddType add(4); + AnyInvType fun = &mult_function; + fun = std::ref(add); + add.state = 5; + EXPECT_TRUE( + (std::is_nothrow_assignable<AnyInvType&, + std::reference_wrapper<AddType>>::value)); + + EXPECT_TRUE(static_cast<bool>(fun)); + EXPECT_EQ(29, TypeParam::ToThisParam(fun)(7, 8, 9).value); + + EXPECT_TRUE(static_cast<bool>(fun)); + EXPECT_EQ(38, TypeParam::ToThisParam(fun)(10, 11, 12).value); +} + +template <class T> +class AnyInvTestRvalue : public ::testing::Test {}; + +TYPED_TEST_SUITE_P(AnyInvTestRvalue); + +TYPED_TEST_P(AnyInvTestRvalue, ConversionConstructionReferenceWrapper) { + using AnyInvType = typename TypeParam::AnyInvType; + using AddType = typename TypeParam::AddType; + + EXPECT_FALSE(( + std::is_convertible<std::reference_wrapper<AddType>, AnyInvType>::value)); +} + +TYPED_TEST_P(AnyInvTestRvalue, NonMoveableResultType) { +#if ABSL_INTERNAL_CPLUSPLUS_LANG < 201703L + GTEST_SKIP() << "Copy/move elision was not standard before C++17"; +#else + // Define a result type that cannot be copy- or move-constructed. + struct Result { + int x; + + explicit Result(const int x_in) : x(x_in) {} + Result(Result&&) = delete; + }; + + static_assert(!std::is_move_constructible<Result>::value, ""); + static_assert(!std::is_copy_constructible<Result>::value, ""); + + // Assumption check: it should nevertheless be possible to use functors that + // return a Result struct according to the language rules. + const auto return_17 = []() noexcept { return Result(17); }; + EXPECT_EQ(17, return_17().x); + + // Just like plain functors, it should work fine to use an AnyInvocable that + // returns the non-moveable type. + using UnqualifiedFun = + absl::conditional_t<TypeParam::kIsNoexcept, Result() noexcept, Result()>; + + using Fun = + GiveQualifiersToFun<typename TypeParam::Qualifiers, UnqualifiedFun>; + + EXPECT_EQ(17, AnyInvocable<Fun>(return_17)().x); +#endif +} + +TYPED_TEST_P(AnyInvTestRvalue, ConversionAssignReferenceWrapper) { + using AnyInvType = typename TypeParam::AnyInvType; + using AddType = typename TypeParam::AddType; + + EXPECT_FALSE(( + std::is_assignable<AnyInvType&, std::reference_wrapper<AddType>>::value)); +} + +// NOTE: This test suite originally attempted to enumerate all possible +// combinations of type properties but the build-time started getting too large. +// Instead, it is now assumed that certain parameters are orthogonal and so +// some combinations are elided. + +// A metafunction to form a TypeList of all cv and non-rvalue ref combinations, +// coupled with all of the other explicitly specified parameters. +template <Movable Mov, Destructible Dest, NothrowCall CallExceptionSpec, + ObjSize Size, ObjAlign Align> +using NonRvalueQualifiedTestParams = ::testing::Types< // + TestParams<Mov, Dest, _, CallExceptionSpec, Size, Align>, // + TestParams<Mov, Dest, const _, CallExceptionSpec, Size, Align>, // + TestParams<Mov, Dest, _&, CallExceptionSpec, Size, Align>, // + TestParams<Mov, Dest, const _&, CallExceptionSpec, Size, Align>>; + +// A metafunction to form a TypeList of const and non-const rvalue ref +// qualifiers, coupled with all of the other explicitly specified parameters. +template <Movable Mov, Destructible Dest, NothrowCall CallExceptionSpec, + ObjSize Size, ObjAlign Align> +using RvalueQualifiedTestParams = ::testing::Types< + TestParams<Mov, Dest, _&&, CallExceptionSpec, Size, Align>, // + TestParams<Mov, Dest, const _&&, CallExceptionSpec, Size, Align> // + >; + +// All qualifier combinations and a noexcept function type +using TestParameterListNonRvalueQualifiersNothrowCall = + NonRvalueQualifiedTestParams<Movable::trivial, Destructible::trivial, + NothrowCall::yes, ObjSize::small, + ObjAlign::normal>; +using TestParameterListRvalueQualifiersNothrowCall = + RvalueQualifiedTestParams<Movable::trivial, Destructible::trivial, + NothrowCall::yes, ObjSize::small, + ObjAlign::normal>; + +// All qualifier combinations and a non-noexcept function type +using TestParameterListNonRvalueQualifiersCallMayThrow = + NonRvalueQualifiedTestParams<Movable::trivial, Destructible::trivial, + NothrowCall::no, ObjSize::small, + ObjAlign::normal>; +using TestParameterListRvalueQualifiersCallMayThrow = + RvalueQualifiedTestParams<Movable::trivial, Destructible::trivial, + NothrowCall::no, ObjSize::small, + ObjAlign::normal>; + +// Lists of various cases that should lead to remote storage +using TestParameterListRemoteMovable = ::testing::Types< + // "Normal" aligned types that are large and have trivial destructors + TestParams<Movable::trivial, Destructible::trivial, _, NothrowCall::no, + ObjSize::large, ObjAlign::normal>, // + TestParams<Movable::nothrow, Destructible::trivial, _, NothrowCall::no, + ObjSize::large, ObjAlign::normal>, // + TestParams<Movable::yes, Destructible::trivial, _, NothrowCall::no, + ObjSize::small, ObjAlign::normal>, // + TestParams<Movable::yes, Destructible::trivial, _, NothrowCall::no, + ObjSize::large, ObjAlign::normal>, // + + // Same as above but with non-trivial destructors + TestParams<Movable::trivial, Destructible::nothrow, _, NothrowCall::no, + ObjSize::large, ObjAlign::normal>, // + TestParams<Movable::nothrow, Destructible::nothrow, _, NothrowCall::no, + ObjSize::large, ObjAlign::normal>, // + TestParams<Movable::yes, Destructible::nothrow, _, NothrowCall::no, + ObjSize::small, ObjAlign::normal>, // + TestParams<Movable::yes, Destructible::nothrow, _, NothrowCall::no, + ObjSize::large, ObjAlign::normal> // + +// Dynamic memory allocation for over-aligned data was introduced in C++17. +// See https://www.open-std.org/jtc1/sc22/wg21/docs/papers/2016/p0035r4.html +#if ABSL_INTERNAL_CPLUSPLUS_LANG >= 201703L + // Types that must use remote storage because of a large alignment. + , + TestParams<Movable::trivial, Destructible::trivial, _, NothrowCall::no, + ObjSize::small, ObjAlign::large>, // + TestParams<Movable::nothrow, Destructible::trivial, _, NothrowCall::no, + ObjSize::small, ObjAlign::large>, // + TestParams<Movable::trivial, Destructible::nothrow, _, NothrowCall::no, + ObjSize::small, ObjAlign::large>, // + TestParams<Movable::nothrow, Destructible::nothrow, _, NothrowCall::no, + ObjSize::small, ObjAlign::large> // +#endif + >; +using TestParameterListRemoteNonMovable = ::testing::Types< + // "Normal" aligned types that are large and have trivial destructors + TestParams<Movable::no, Destructible::trivial, _, NothrowCall::no, + ObjSize::small, ObjAlign::normal>, // + TestParams<Movable::no, Destructible::trivial, _, NothrowCall::no, + ObjSize::large, ObjAlign::normal>, // + // Same as above but with non-trivial destructors + TestParams<Movable::no, Destructible::nothrow, _, NothrowCall::no, + ObjSize::small, ObjAlign::normal>, // + TestParams<Movable::no, Destructible::nothrow, _, NothrowCall::no, + ObjSize::large, ObjAlign::normal> // + >; + +// Parameters that lead to local storage +using TestParameterListLocal = ::testing::Types< + // Types that meet the requirements and have trivial destructors + TestParams<Movable::trivial, Destructible::trivial, _, NothrowCall::no, + ObjSize::small, ObjAlign::normal>, // + TestParams<Movable::nothrow, Destructible::trivial, _, NothrowCall::no, + ObjSize::small, ObjAlign::normal>, // + + // Same as above but with non-trivial destructors + TestParams<Movable::trivial, Destructible::trivial, _, NothrowCall::no, + ObjSize::small, ObjAlign::normal>, // + TestParams<Movable::nothrow, Destructible::trivial, _, NothrowCall::no, + ObjSize::small, ObjAlign::normal> // + >; + +// All of the tests that are run for every possible combination of types. +REGISTER_TYPED_TEST_SUITE_P( + AnyInvTestBasic, DefaultConstruction, ConstructionNullptr, + ConstructionNullFunctionPtr, ConstructionNullMemberFunctionPtr, + ConstructionNullMemberObjectPtr, ConstructionMemberFunctionPtr, + ConstructionMemberObjectPtr, ConstructionFunctionReferenceDecay, + ConstructionCompatibleAnyInvocableEmpty, + ConstructionCompatibleAnyInvocableNonempty, InPlaceConstruction, + ConversionToBool, Invocation, InPlaceConstructionInitializerList, + InPlaceNullFunPtrConstruction, InPlaceNullFunPtrConstructionValueInit, + InPlaceNullMemFunPtrConstruction, InPlaceNullMemFunPtrConstructionValueInit, + InPlaceNullMemObjPtrConstruction, InPlaceNullMemObjPtrConstructionValueInit, + InPlaceVoidCovarianceConstruction, MoveConstructionFromEmpty, + MoveConstructionFromNonEmpty, ComparisonWithNullptrEmpty, + ComparisonWithNullptrNonempty, ResultType); + +INSTANTIATE_TYPED_TEST_SUITE_P( + NonRvalueCallMayThrow, AnyInvTestBasic, + TestParameterListNonRvalueQualifiersCallMayThrow); +INSTANTIATE_TYPED_TEST_SUITE_P(RvalueCallMayThrow, AnyInvTestBasic, + TestParameterListRvalueQualifiersCallMayThrow); + +INSTANTIATE_TYPED_TEST_SUITE_P(RemoteMovable, AnyInvTestBasic, + TestParameterListRemoteMovable); +INSTANTIATE_TYPED_TEST_SUITE_P(RemoteNonMovable, AnyInvTestBasic, + TestParameterListRemoteNonMovable); + +INSTANTIATE_TYPED_TEST_SUITE_P(Local, AnyInvTestBasic, TestParameterListLocal); + +INSTANTIATE_TYPED_TEST_SUITE_P(NonRvalueCallNothrow, AnyInvTestBasic, + TestParameterListNonRvalueQualifiersNothrowCall); +INSTANTIATE_TYPED_TEST_SUITE_P(CallNothrowRvalue, AnyInvTestBasic, + TestParameterListRvalueQualifiersNothrowCall); + +// Tests for functions that take two operands. +REGISTER_TYPED_TEST_SUITE_P( + AnyInvTestCombinatoric, MoveAssignEmptyEmptyLhsRhs, + MoveAssignEmptyLhsNonemptyRhs, MoveAssignNonemptyEmptyLhsRhs, + MoveAssignNonemptyLhsNonemptyRhs, SelfMoveAssignEmpty, + SelfMoveAssignNonempty, AssignNullptrEmptyLhs, + AssignNullFunctionPtrEmptyLhs, AssignNullMemberFunctionPtrEmptyLhs, + AssignNullMemberObjectPtrEmptyLhs, AssignMemberFunctionPtrEmptyLhs, + AssignMemberObjectPtrEmptyLhs, AssignFunctionReferenceDecayEmptyLhs, + AssignCompatibleAnyInvocableEmptyLhsEmptyRhs, + AssignCompatibleAnyInvocableEmptyLhsNonemptyRhs, AssignNullptrNonemptyLhs, + AssignNullFunctionPtrNonemptyLhs, AssignNullMemberFunctionPtrNonemptyLhs, + AssignNullMemberObjectPtrNonemptyLhs, AssignMemberFunctionPtrNonemptyLhs, + AssignMemberObjectPtrNonemptyLhs, AssignFunctionReferenceDecayNonemptyLhs, + AssignCompatibleAnyInvocableNonemptyLhsEmptyRhs, + AssignCompatibleAnyInvocableNonemptyLhsNonemptyRhs, SwapEmptyLhsEmptyRhs, + SwapEmptyLhsNonemptyRhs, SwapNonemptyLhsEmptyRhs, + SwapNonemptyLhsNonemptyRhs); + +INSTANTIATE_TYPED_TEST_SUITE_P( + NonRvalueCallMayThrow, AnyInvTestCombinatoric, + TestParameterListNonRvalueQualifiersCallMayThrow); +INSTANTIATE_TYPED_TEST_SUITE_P(RvalueCallMayThrow, AnyInvTestCombinatoric, + TestParameterListRvalueQualifiersCallMayThrow); + +INSTANTIATE_TYPED_TEST_SUITE_P(RemoteMovable, AnyInvTestCombinatoric, + TestParameterListRemoteMovable); +INSTANTIATE_TYPED_TEST_SUITE_P(RemoteNonMovable, AnyInvTestCombinatoric, + TestParameterListRemoteNonMovable); + +INSTANTIATE_TYPED_TEST_SUITE_P(Local, AnyInvTestCombinatoric, + TestParameterListLocal); + +INSTANTIATE_TYPED_TEST_SUITE_P(NonRvalueCallNothrow, AnyInvTestCombinatoric, + TestParameterListNonRvalueQualifiersNothrowCall); +INSTANTIATE_TYPED_TEST_SUITE_P(RvalueCallNothrow, AnyInvTestCombinatoric, + TestParameterListRvalueQualifiersNothrowCall); + +REGISTER_TYPED_TEST_SUITE_P(AnyInvTestMovable, + ConversionConstructionUserDefinedType, + ConversionConstructionVoidCovariance, + ConversionAssignUserDefinedTypeEmptyLhs, + ConversionAssignUserDefinedTypeNonemptyLhs, + ConversionAssignVoidCovariance); + +INSTANTIATE_TYPED_TEST_SUITE_P( + NonRvalueCallMayThrow, AnyInvTestMovable, + TestParameterListNonRvalueQualifiersCallMayThrow); +INSTANTIATE_TYPED_TEST_SUITE_P(RvalueCallMayThrow, AnyInvTestMovable, + TestParameterListRvalueQualifiersCallMayThrow); + +INSTANTIATE_TYPED_TEST_SUITE_P(RemoteMovable, AnyInvTestMovable, + TestParameterListRemoteMovable); + +INSTANTIATE_TYPED_TEST_SUITE_P(Local, AnyInvTestMovable, + TestParameterListLocal); + +INSTANTIATE_TYPED_TEST_SUITE_P(NonRvalueCallNothrow, AnyInvTestMovable, + TestParameterListNonRvalueQualifiersNothrowCall); +INSTANTIATE_TYPED_TEST_SUITE_P(RvalueCallNothrow, AnyInvTestMovable, + TestParameterListRvalueQualifiersNothrowCall); + +REGISTER_TYPED_TEST_SUITE_P(AnyInvTestNoexceptFalse, + ConversionConstructionConstraints, + ConversionAssignConstraints); + +INSTANTIATE_TYPED_TEST_SUITE_P( + NonRvalueCallMayThrow, AnyInvTestNoexceptFalse, + TestParameterListNonRvalueQualifiersCallMayThrow); +INSTANTIATE_TYPED_TEST_SUITE_P(RvalueCallMayThrow, AnyInvTestNoexceptFalse, + TestParameterListRvalueQualifiersCallMayThrow); + +INSTANTIATE_TYPED_TEST_SUITE_P(RemoteMovable, AnyInvTestNoexceptFalse, + TestParameterListRemoteMovable); +INSTANTIATE_TYPED_TEST_SUITE_P(RemoteNonMovable, AnyInvTestNoexceptFalse, + TestParameterListRemoteNonMovable); + +INSTANTIATE_TYPED_TEST_SUITE_P(Local, AnyInvTestNoexceptFalse, + TestParameterListLocal); + +REGISTER_TYPED_TEST_SUITE_P(AnyInvTestNoexceptTrue, + ConversionConstructionConstraints, + ConversionAssignConstraints); + +INSTANTIATE_TYPED_TEST_SUITE_P(NonRvalueCallNothrow, AnyInvTestNoexceptTrue, + TestParameterListNonRvalueQualifiersNothrowCall); +INSTANTIATE_TYPED_TEST_SUITE_P(RvalueCallNothrow, AnyInvTestNoexceptTrue, + TestParameterListRvalueQualifiersNothrowCall); + +REGISTER_TYPED_TEST_SUITE_P(AnyInvTestNonRvalue, + ConversionConstructionReferenceWrapper, + NonMoveableResultType, + ConversionAssignReferenceWrapperEmptyLhs, + ConversionAssignReferenceWrapperNonemptyLhs); + +INSTANTIATE_TYPED_TEST_SUITE_P( + NonRvalueCallMayThrow, AnyInvTestNonRvalue, + TestParameterListNonRvalueQualifiersCallMayThrow); + +INSTANTIATE_TYPED_TEST_SUITE_P(RemoteMovable, AnyInvTestNonRvalue, + TestParameterListRemoteMovable); +INSTANTIATE_TYPED_TEST_SUITE_P(RemoteNonMovable, AnyInvTestNonRvalue, + TestParameterListRemoteNonMovable); + +INSTANTIATE_TYPED_TEST_SUITE_P(Local, AnyInvTestNonRvalue, + TestParameterListLocal); + +INSTANTIATE_TYPED_TEST_SUITE_P(NonRvalueCallNothrow, AnyInvTestNonRvalue, + TestParameterListNonRvalueQualifiersNothrowCall); + +REGISTER_TYPED_TEST_SUITE_P(AnyInvTestRvalue, + ConversionConstructionReferenceWrapper, + NonMoveableResultType, + ConversionAssignReferenceWrapper); + +INSTANTIATE_TYPED_TEST_SUITE_P(RvalueCallMayThrow, AnyInvTestRvalue, + TestParameterListRvalueQualifiersCallMayThrow); + +INSTANTIATE_TYPED_TEST_SUITE_P(CallNothrowRvalue, AnyInvTestRvalue, + TestParameterListRvalueQualifiersNothrowCall); + +// Minimal SFINAE testing for platforms where we can't run the tests, but we can +// build binaries for. +static_assert( + std::is_convertible<void (*)(), absl::AnyInvocable<void() &&>>::value, ""); +static_assert(!std::is_convertible<void*, absl::AnyInvocable<void() &&>>::value, + ""); + +#undef ABSL_INTERNAL_NOEXCEPT_SPEC + +} // namespace diff --git a/absl/functional/bind_front.h b/absl/functional/bind_front.h index 5b47970e..f9075bd1 100644 --- a/absl/functional/bind_front.h +++ b/absl/functional/bind_front.h @@ -30,6 +30,10 @@ #ifndef ABSL_FUNCTIONAL_BIND_FRONT_H_ #define ABSL_FUNCTIONAL_BIND_FRONT_H_ +#if defined(__cpp_lib_bind_front) && __cpp_lib_bind_front >= 201907L +#include <functional> // For std::bind_front. +#endif // defined(__cpp_lib_bind_front) && __cpp_lib_bind_front >= 201907L + #include "absl/functional/internal/front_binder.h" #include "absl/utility/utility.h" @@ -46,7 +50,8 @@ ABSL_NAMESPACE_BEGIN // specified. More importantly, it provides more reliable correctness guarantees // than `std::bind()`; while `std::bind()` will silently ignore passing more // parameters than expected, for example, `absl::bind_front()` will report such -// mis-uses as errors. +// mis-uses as errors. In C++20, `absl::bind_front` is replaced by +// `std::bind_front`. // // absl::bind_front(a...) can be seen as storing the results of // std::make_tuple(a...). @@ -170,6 +175,9 @@ ABSL_NAMESPACE_BEGIN // // Doesn't copy "hi". // absl::bind_front(Print, absl::string_view(hi))("Chuk"); // +#if defined(__cpp_lib_bind_front) && __cpp_lib_bind_front >= 201907L +using std::bind_front; +#else // defined(__cpp_lib_bind_front) && __cpp_lib_bind_front >= 201907L template <class F, class... BoundArgs> constexpr functional_internal::bind_front_t<F, BoundArgs...> bind_front( F&& func, BoundArgs&&... args) { @@ -177,6 +185,7 @@ constexpr functional_internal::bind_front_t<F, BoundArgs...> bind_front( absl::in_place, absl::forward<F>(func), absl::forward<BoundArgs>(args)...); } +#endif // defined(__cpp_lib_bind_front) && __cpp_lib_bind_front >= 201907L ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/functional/function_ref.h b/absl/functional/function_ref.h index 6e03ac2e..f9779607 100644 --- a/absl/functional/function_ref.h +++ b/absl/functional/function_ref.h @@ -50,6 +50,7 @@ #include <functional> #include <type_traits> +#include "absl/base/attributes.h" #include "absl/functional/internal/function_ref.h" #include "absl/meta/type_traits.h" @@ -68,7 +69,8 @@ class FunctionRef; // An `absl::FunctionRef` is a lightweight wrapper to any invokable object with // a compatible signature. Generally, an `absl::FunctionRef` should only be used // as an argument type and should be preferred as an argument over a const -// reference to a `std::function`. +// reference to a `std::function`. `absl::FunctionRef` itself does not allocate, +// although the wrapped invokable may. // // Example: // @@ -98,7 +100,8 @@ class FunctionRef<R(Args...)> { public: // Constructs a FunctionRef from any invokable type. template <typename F, typename = EnableIfCompatible<const F&>> - FunctionRef(const F& f) // NOLINT(runtime/explicit) + // NOLINTNEXTLINE(runtime/explicit) + FunctionRef(const F& f ABSL_ATTRIBUTE_LIFETIME_BOUND) : invoker_(&absl::functional_internal::InvokeObject<F, R, Args...>) { absl::functional_internal::AssertNonNull(f); ptr_.obj = &f; @@ -122,6 +125,7 @@ class FunctionRef<R(Args...)> { // To help prevent subtle lifetime bugs, FunctionRef is not assignable. // Typically, it should only be used as an argument type. FunctionRef& operator=(const FunctionRef& rhs) = delete; + FunctionRef(const FunctionRef& rhs) = default; // Call the underlying object. R operator()(Args... args) const { diff --git a/absl/functional/function_ref_test.cc b/absl/functional/function_ref_test.cc index 3aa59745..412027cd 100644 --- a/absl/functional/function_ref_test.cc +++ b/absl/functional/function_ref_test.cc @@ -14,6 +14,7 @@ #include "absl/functional/function_ref.h" +#include <functional> #include <memory> #include "gmock/gmock.h" diff --git a/absl/functional/function_ref_benchmark.cc b/absl/functional/function_type_benchmark.cc index 045305bf..03dc31d8 100644 --- a/absl/functional/function_ref_benchmark.cc +++ b/absl/functional/function_type_benchmark.cc @@ -1,4 +1,4 @@ -// Copyright 2019 The Abseil Authors. +// Copyright 2022 The Abseil Authors. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -12,12 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "absl/functional/function_ref.h" - +#include <functional> #include <memory> +#include <string> #include "benchmark/benchmark.h" #include "absl/base/attributes.h" +#include "absl/functional/any_invocable.h" +#include "absl/functional/function_ref.h" namespace absl { ABSL_NAMESPACE_BEGIN @@ -61,6 +63,12 @@ void BM_TrivialFunctionRef(benchmark::State& state) { } BENCHMARK(BM_TrivialFunctionRef); +void BM_TrivialAnyInvocable(benchmark::State& state) { + ConstructAndCallFunctionBenchmark<AnyInvocable<void()>>(state, + TrivialFunctor{}); +} +BENCHMARK(BM_TrivialAnyInvocable); + void BM_LargeStdFunction(benchmark::State& state) { ConstructAndCallFunctionBenchmark<std::function<void()>>(state, LargeFunctor{}); @@ -72,6 +80,13 @@ void BM_LargeFunctionRef(benchmark::State& state) { } BENCHMARK(BM_LargeFunctionRef); + +void BM_LargeAnyInvocable(benchmark::State& state) { + ConstructAndCallFunctionBenchmark<AnyInvocable<void()>>(state, + LargeFunctor{}); +} +BENCHMARK(BM_LargeAnyInvocable); + void BM_FunPtrStdFunction(benchmark::State& state) { ConstructAndCallFunctionBenchmark<std::function<void()>>(state, FreeFunction); } @@ -82,6 +97,11 @@ void BM_FunPtrFunctionRef(benchmark::State& state) { } BENCHMARK(BM_FunPtrFunctionRef); +void BM_FunPtrAnyInvocable(benchmark::State& state) { + ConstructAndCallFunctionBenchmark<AnyInvocable<void()>>(state, FreeFunction); +} +BENCHMARK(BM_FunPtrAnyInvocable); + // Doesn't include construction or copy overhead in the loop. template <typename Function, typename Callable, typename... Args> void CallFunctionBenchmark(benchmark::State& state, const Callable& c, @@ -113,6 +133,12 @@ void BM_TrivialArgsFunctionRef(benchmark::State& state) { } BENCHMARK(BM_TrivialArgsFunctionRef); +void BM_TrivialArgsAnyInvocable(benchmark::State& state) { + CallFunctionBenchmark<AnyInvocable<void(int, int, int)>>( + state, FunctorWithTrivialArgs{}, 1, 2, 3); +} +BENCHMARK(BM_TrivialArgsAnyInvocable); + struct FunctorWithNonTrivialArgs { void operator()(std::string a, std::string b, std::string c) const { benchmark::DoNotOptimize(&a); @@ -137,6 +163,14 @@ void BM_NonTrivialArgsFunctionRef(benchmark::State& state) { } BENCHMARK(BM_NonTrivialArgsFunctionRef); +void BM_NonTrivialArgsAnyInvocable(benchmark::State& state) { + std::string a, b, c; + CallFunctionBenchmark< + AnyInvocable<void(std::string, std::string, std::string)>>( + state, FunctorWithNonTrivialArgs{}, a, b, c); +} +BENCHMARK(BM_NonTrivialArgsAnyInvocable); + } // namespace ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/functional/internal/any_invocable.h b/absl/functional/internal/any_invocable.h new file mode 100644 index 00000000..f353139c --- /dev/null +++ b/absl/functional/internal/any_invocable.h @@ -0,0 +1,857 @@ +// Copyright 2022 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// Implementation details for `absl::AnyInvocable` + +#ifndef ABSL_FUNCTIONAL_INTERNAL_ANY_INVOCABLE_H_ +#define ABSL_FUNCTIONAL_INTERNAL_ANY_INVOCABLE_H_ + +//////////////////////////////////////////////////////////////////////////////// +// // +// This implementation of the proposed `any_invocable` uses an approach that // +// chooses between local storage and remote storage for the contained target // +// object based on the target object's size, alignment requirements, and // +// whether or not it has a nothrow move constructor. Additional optimizations // +// are performed when the object is a trivially copyable type [basic.types]. // +// // +// There are three datamembers per `AnyInvocable` instance // +// // +// 1) A union containing either // +// - A pointer to the target object referred to via a void*, or // +// - the target object, emplaced into a raw char buffer // +// // +// 2) A function pointer to a "manager" function operation that takes a // +// discriminator and logically branches to either perform a move operation // +// or destroy operation based on that discriminator. // +// // +// 3) A function pointer to an "invoker" function operation that invokes the // +// target object, directly returning the result. // +// // +// When in the logically empty state, the manager function is an empty // +// function and the invoker function is one that would be undefined-behavior // +// to call. // +// // +// An additional optimization is performed when converting from one // +// AnyInvocable to another where only the noexcept specification and/or the // +// cv/ref qualifiers of the function type differ. In these cases, the // +// conversion works by "moving the guts", similar to if they were the same // +// exact type, as opposed to having to perform an additional layer of // +// wrapping through remote storage. // +// // +//////////////////////////////////////////////////////////////////////////////// + +// IWYU pragma: private, include "absl/functional/any_invocable.h" + +#include <cassert> +#include <cstddef> +#include <cstring> +#include <functional> +#include <initializer_list> +#include <memory> +#include <new> +#include <type_traits> +#include <utility> + +#include "absl/base/config.h" +#include "absl/base/internal/invoke.h" +#include "absl/base/macros.h" +#include "absl/meta/type_traits.h" +#include "absl/utility/utility.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// Helper macro used to prevent spelling `noexcept` in language versions older +// than C++17, where it is not part of the type system, in order to avoid +// compilation failures and internal compiler errors. +#if ABSL_INTERNAL_CPLUSPLUS_LANG >= 201703L +#define ABSL_INTERNAL_NOEXCEPT_SPEC(noex) noexcept(noex) +#else +#define ABSL_INTERNAL_NOEXCEPT_SPEC(noex) +#endif + +// Defined in functional/any_invocable.h +template <class Sig> +class AnyInvocable; + +namespace internal_any_invocable { + +// Constants relating to the small-object-storage for AnyInvocable +enum StorageProperty : std::size_t { + kAlignment = alignof(std::max_align_t), // The alignment of the storage + kStorageSize = sizeof(void*) * 2 // The size of the storage +}; + +//////////////////////////////////////////////////////////////////////////////// +// +// A metafunction for checking if a type is an AnyInvocable instantiation. +// This is used during conversion operations. +template <class T> +struct IsAnyInvocable : std::false_type {}; + +template <class Sig> +struct IsAnyInvocable<AnyInvocable<Sig>> : std::true_type {}; +// +//////////////////////////////////////////////////////////////////////////////// + +// A type trait that tells us whether or not a target function type should be +// stored locally in the small object optimization storage +template <class T> +using IsStoredLocally = std::integral_constant< + bool, sizeof(T) <= kStorageSize && alignof(T) <= kAlignment && + kAlignment % alignof(T) == 0 && + std::is_nothrow_move_constructible<T>::value>; + +// An implementation of std::remove_cvref_t of C++20. +template <class T> +using RemoveCVRef = + typename std::remove_cv<typename std::remove_reference<T>::type>::type; + +//////////////////////////////////////////////////////////////////////////////// +// +// An implementation of the C++ standard INVOKE<R> pseudo-macro, operation is +// equivalent to std::invoke except that it forces an implicit conversion to the +// specified return type. If "R" is void, the function is executed and the +// return value is simply ignored. +template <class ReturnType, class F, class... P, + typename = absl::enable_if_t<std::is_void<ReturnType>::value>> +void InvokeR(F&& f, P&&... args) { + absl::base_internal::invoke(std::forward<F>(f), std::forward<P>(args)...); +} + +template <class ReturnType, class F, class... P, + absl::enable_if_t<!std::is_void<ReturnType>::value, int> = 0> +ReturnType InvokeR(F&& f, P&&... args) { + return absl::base_internal::invoke(std::forward<F>(f), + std::forward<P>(args)...); +} + +// +//////////////////////////////////////////////////////////////////////////////// + +//////////////////////////////////////////////////////////////////////////////// +/// +// A metafunction that takes a "T" corresponding to a parameter type of the +// user's specified function type, and yields the parameter type to use for the +// type-erased invoker. In order to prevent observable moves, this must be +// either a reference or, if the type is trivial, the original parameter type +// itself. Since the parameter type may be incomplete at the point that this +// metafunction is used, we can only do this optimization for scalar types +// rather than for any trivial type. +template <typename T> +T ForwardImpl(std::true_type); + +template <typename T> +T&& ForwardImpl(std::false_type); + +// NOTE: We deliberately use an intermediate struct instead of a direct alias, +// as a workaround for b/206991861 on MSVC versions < 1924. +template <class T> +struct ForwardedParameter { + using type = decltype(( + ForwardImpl<T>)(std::integral_constant<bool, + std::is_scalar<T>::value>())); +}; + +template <class T> +using ForwardedParameterType = typename ForwardedParameter<T>::type; +// +//////////////////////////////////////////////////////////////////////////////// + +// A discriminator when calling the "manager" function that describes operation +// type-erased operation should be invoked. +// +// "relocate_from_to" specifies that the manager should perform a move. +// +// "dispose" specifies that the manager should perform a destroy. +enum class FunctionToCall : bool { relocate_from_to, dispose }; + +// The portion of `AnyInvocable` state that contains either a pointer to the +// target object or the object itself in local storage +union TypeErasedState { + struct { + // A pointer to the type-erased object when remotely stored + void* target; + // The size of the object for `RemoteManagerTrivial` + std::size_t size; + } remote; + + // Local-storage for the type-erased object when small and trivial enough + alignas(kAlignment) char storage[kStorageSize]; +}; + +// A typed accessor for the object in `TypeErasedState` storage +template <class T> +T& ObjectInLocalStorage(TypeErasedState* const state) { + // We launder here because the storage may be reused with the same type. +#if ABSL_INTERNAL_CPLUSPLUS_LANG >= 201703L + return *std::launder(reinterpret_cast<T*>(&state->storage)); +#elif ABSL_HAVE_BUILTIN(__builtin_launder) + return *__builtin_launder(reinterpret_cast<T*>(&state->storage)); +#else + + // When `std::launder` or equivalent are not available, we rely on undefined + // behavior, which works as intended on Abseil's officially supported + // platforms as of Q2 2022. +#if !defined(__clang__) && defined(__GNUC__) +#pragma GCC diagnostic ignored "-Wstrict-aliasing" +#pragma GCC diagnostic push +#endif + return *reinterpret_cast<T*>(&state->storage); +#if !defined(__clang__) && defined(__GNUC__) +#pragma GCC diagnostic pop +#endif + +#endif +} + +// The type for functions issuing lifetime-related operations: move and dispose +// A pointer to such a function is contained in each `AnyInvocable` instance. +// NOTE: When specifying `FunctionToCall::`dispose, the same state must be +// passed as both "from" and "to". +using ManagerType = void(FunctionToCall /*operation*/, + TypeErasedState* /*from*/, TypeErasedState* /*to*/) + ABSL_INTERNAL_NOEXCEPT_SPEC(true); + +// The type for functions issuing the actual invocation of the object +// A pointer to such a function is contained in each AnyInvocable instance. +template <bool SigIsNoexcept, class ReturnType, class... P> +using InvokerType = ReturnType(TypeErasedState*, ForwardedParameterType<P>...) + ABSL_INTERNAL_NOEXCEPT_SPEC(SigIsNoexcept); + +// The manager that is used when AnyInvocable is empty +inline void EmptyManager(FunctionToCall /*operation*/, + TypeErasedState* /*from*/, + TypeErasedState* /*to*/) noexcept {} + +// The manager that is used when a target function is in local storage and is +// a trivially copyable type. +inline void LocalManagerTrivial(FunctionToCall /*operation*/, + TypeErasedState* const from, + TypeErasedState* const to) noexcept { + // This single statement without branching handles both possible operations. + // + // For FunctionToCall::dispose, "from" and "to" point to the same state, and + // so this assignment logically would do nothing. + // + // Note: Correctness here relies on http://wg21.link/p0593, which has only + // become standard in C++20, though implementations do not break it in + // practice for earlier versions of C++. + // + // The correct way to do this without that paper is to first placement-new a + // default-constructed T in "to->storage" prior to the memmove, but doing so + // requires a different function to be created for each T that is stored + // locally, which can cause unnecessary bloat and be less cache friendly. + *to = *from; + + // Note: Because the type is trivially copyable, the destructor does not need + // to be called ("trivially copyable" requires a trivial destructor). +} + +// The manager that is used when a target function is in local storage and is +// not a trivially copyable type. +template <class T> +void LocalManagerNontrivial(FunctionToCall operation, + TypeErasedState* const from, + TypeErasedState* const to) noexcept { + static_assert(IsStoredLocally<T>::value, + "Local storage must only be used for supported types."); + static_assert(!std::is_trivially_copyable<T>::value, + "Locally stored types must be trivially copyable."); + + T& from_object = (ObjectInLocalStorage<T>)(from); + + switch (operation) { + case FunctionToCall::relocate_from_to: + // NOTE: Requires that the left-hand operand is already empty. + ::new (static_cast<void*>(&to->storage)) T(std::move(from_object)); + ABSL_FALLTHROUGH_INTENDED; + case FunctionToCall::dispose: + from_object.~T(); // Must not throw. // NOLINT + return; + } + ABSL_INTERNAL_UNREACHABLE; +} + +// The invoker that is used when a target function is in local storage +// Note: QualTRef here is the target function type along with cv and reference +// qualifiers that must be used when calling the function. +template <bool SigIsNoexcept, class ReturnType, class QualTRef, class... P> +ReturnType LocalInvoker( + TypeErasedState* const state, + ForwardedParameterType<P>... args) noexcept(SigIsNoexcept) { + using RawT = RemoveCVRef<QualTRef>; + static_assert( + IsStoredLocally<RawT>::value, + "Target object must be in local storage in order to be invoked from it."); + + auto& f = (ObjectInLocalStorage<RawT>)(state); + return (InvokeR<ReturnType>)(static_cast<QualTRef>(f), + static_cast<ForwardedParameterType<P>>(args)...); +} + +// The manager that is used when a target function is in remote storage and it +// has a trivial destructor +inline void RemoteManagerTrivial(FunctionToCall operation, + TypeErasedState* const from, + TypeErasedState* const to) noexcept { + switch (operation) { + case FunctionToCall::relocate_from_to: + // NOTE: Requires that the left-hand operand is already empty. + to->remote = from->remote; + return; + case FunctionToCall::dispose: +#if defined(__cpp_sized_deallocation) + ::operator delete(from->remote.target, from->remote.size); +#else // __cpp_sized_deallocation + ::operator delete(from->remote.target); +#endif // __cpp_sized_deallocation + return; + } + ABSL_INTERNAL_UNREACHABLE; +} + +// The manager that is used when a target function is in remote storage and the +// destructor of the type is not trivial +template <class T> +void RemoteManagerNontrivial(FunctionToCall operation, + TypeErasedState* const from, + TypeErasedState* const to) noexcept { + static_assert(!IsStoredLocally<T>::value, + "Remote storage must only be used for types that do not " + "qualify for local storage."); + + switch (operation) { + case FunctionToCall::relocate_from_to: + // NOTE: Requires that the left-hand operand is already empty. + to->remote.target = from->remote.target; + return; + case FunctionToCall::dispose: + ::delete static_cast<T*>(from->remote.target); // Must not throw. + return; + } + ABSL_INTERNAL_UNREACHABLE; +} + +// The invoker that is used when a target function is in remote storage +template <bool SigIsNoexcept, class ReturnType, class QualTRef, class... P> +ReturnType RemoteInvoker( + TypeErasedState* const state, + ForwardedParameterType<P>... args) noexcept(SigIsNoexcept) { + using RawT = RemoveCVRef<QualTRef>; + static_assert(!IsStoredLocally<RawT>::value, + "Target object must be in remote storage in order to be " + "invoked from it."); + + auto& f = *static_cast<RawT*>(state->remote.target); + return (InvokeR<ReturnType>)(static_cast<QualTRef>(f), + static_cast<ForwardedParameterType<P>>(args)...); +} + +//////////////////////////////////////////////////////////////////////////////// +// +// A metafunction that checks if a type T is an instantiation of +// absl::in_place_type_t (needed for constructor constraints of AnyInvocable). +template <class T> +struct IsInPlaceType : std::false_type {}; + +template <class T> +struct IsInPlaceType<absl::in_place_type_t<T>> : std::true_type {}; +// +//////////////////////////////////////////////////////////////////////////////// + +// A constructor name-tag used with CoreImpl (below) to request the +// conversion-constructor. QualDecayedTRef is the decayed-type of the object to +// wrap, along with the cv and reference qualifiers that must be applied when +// performing an invocation of the wrapped object. +template <class QualDecayedTRef> +struct TypedConversionConstruct {}; + +// A helper base class for all core operations of AnyInvocable. Most notably, +// this class creates the function call operator and constraint-checkers so that +// the top-level class does not have to be a series of partial specializations. +// +// Note: This definition exists (as opposed to being a declaration) so that if +// the user of the top-level template accidentally passes a template argument +// that is not a function type, they will get a static_assert in AnyInvocable's +// class body rather than an error stating that Impl is not defined. +template <class Sig> +class Impl {}; // Note: This is partially-specialized later. + +// A std::unique_ptr deleter that deletes memory allocated via ::operator new. +#if defined(__cpp_sized_deallocation) +class TrivialDeleter { + public: + explicit TrivialDeleter(std::size_t size) : size_(size) {} + + void operator()(void* target) const { + ::operator delete(target, size_); + } + + private: + std::size_t size_; +}; +#else // __cpp_sized_deallocation +class TrivialDeleter { + public: + explicit TrivialDeleter(std::size_t) {} + + void operator()(void* target) const { ::operator delete(target); } +}; +#endif // __cpp_sized_deallocation + +template <bool SigIsNoexcept, class ReturnType, class... P> +class CoreImpl; + +constexpr bool IsCompatibleConversion(void*, void*) { return false; } +template <bool NoExceptSrc, bool NoExceptDest, class... T> +constexpr bool IsCompatibleConversion(CoreImpl<NoExceptSrc, T...>*, + CoreImpl<NoExceptDest, T...>*) { + return !NoExceptDest || NoExceptSrc; +} + +// A helper base class for all core operations of AnyInvocable that do not +// depend on the cv/ref qualifiers of the function type. +template <bool SigIsNoexcept, class ReturnType, class... P> +class CoreImpl { + public: + using result_type = ReturnType; + + CoreImpl() noexcept : manager_(EmptyManager), invoker_(nullptr) {} + + enum class TargetType : int { + kPointer = 0, + kCompatibleAnyInvocable = 1, + kIncompatibleAnyInvocable = 2, + kOther = 3, + }; + + // Note: QualDecayedTRef here includes the cv-ref qualifiers associated with + // the invocation of the Invocable. The unqualified type is the target object + // type to be stored. + template <class QualDecayedTRef, class F> + explicit CoreImpl(TypedConversionConstruct<QualDecayedTRef>, F&& f) { + using DecayedT = RemoveCVRef<QualDecayedTRef>; + + constexpr TargetType kTargetType = + (std::is_pointer<DecayedT>::value || + std::is_member_pointer<DecayedT>::value) + ? TargetType::kPointer + : IsCompatibleAnyInvocable<DecayedT>::value + ? TargetType::kCompatibleAnyInvocable + : IsAnyInvocable<DecayedT>::value + ? TargetType::kIncompatibleAnyInvocable + : TargetType::kOther; + // NOTE: We only use integers instead of enums as template parameters in + // order to work around a bug on C++14 under MSVC 2017. + // See b/236131881. + Initialize<static_cast<int>(kTargetType), QualDecayedTRef>( + std::forward<F>(f)); + } + + // Note: QualTRef here includes the cv-ref qualifiers associated with the + // invocation of the Invocable. The unqualified type is the target object + // type to be stored. + template <class QualTRef, class... Args> + explicit CoreImpl(absl::in_place_type_t<QualTRef>, Args&&... args) { + InitializeStorage<QualTRef>(std::forward<Args>(args)...); + } + + CoreImpl(CoreImpl&& other) noexcept { + other.manager_(FunctionToCall::relocate_from_to, &other.state_, &state_); + manager_ = other.manager_; + invoker_ = other.invoker_; + other.manager_ = EmptyManager; + other.invoker_ = nullptr; + } + + CoreImpl& operator=(CoreImpl&& other) noexcept { + // Put the left-hand operand in an empty state. + // + // Note: A full reset that leaves us with an object that has its invariants + // intact is necessary in order to handle self-move. This is required by + // types that are used with certain operations of the standard library, such + // as the default definition of std::swap when both operands target the same + // object. + Clear(); + + // Perform the actual move/destory operation on the target function. + other.manager_(FunctionToCall::relocate_from_to, &other.state_, &state_); + manager_ = other.manager_; + invoker_ = other.invoker_; + other.manager_ = EmptyManager; + other.invoker_ = nullptr; + + return *this; + } + + ~CoreImpl() { manager_(FunctionToCall::dispose, &state_, &state_); } + + // Check whether or not the AnyInvocable is in the empty state. + bool HasValue() const { return invoker_ != nullptr; } + + // Effects: Puts the object into its empty state. + void Clear() { + manager_(FunctionToCall::dispose, &state_, &state_); + manager_ = EmptyManager; + invoker_ = nullptr; + } + + template <int target_type, class QualDecayedTRef, class F, + absl::enable_if_t<target_type == 0, int> = 0> + void Initialize(F&& f) { +// This condition handles types that decay into pointers, which includes +// function references. Since function references cannot be null, GCC warns +// against comparing their decayed form with nullptr. +// Since this is template-heavy code, we prefer to disable these warnings +// locally instead of adding yet another overload of this function. +#if !defined(__clang__) && defined(__GNUC__) +#pragma GCC diagnostic ignored "-Wpragmas" +#pragma GCC diagnostic ignored "-Waddress" +#pragma GCC diagnostic ignored "-Wnonnull-compare" +#pragma GCC diagnostic push +#endif + if (static_cast<RemoveCVRef<QualDecayedTRef>>(f) == nullptr) { +#if !defined(__clang__) && defined(__GNUC__) +#pragma GCC diagnostic pop +#endif + manager_ = EmptyManager; + invoker_ = nullptr; + return; + } + InitializeStorage<QualDecayedTRef>(std::forward<F>(f)); + } + + template <int target_type, class QualDecayedTRef, class F, + absl::enable_if_t<target_type == 1, int> = 0> + void Initialize(F&& f) { + // In this case we can "steal the guts" of the other AnyInvocable. + f.manager_(FunctionToCall::relocate_from_to, &f.state_, &state_); + manager_ = f.manager_; + invoker_ = f.invoker_; + + f.manager_ = EmptyManager; + f.invoker_ = nullptr; + } + + template <int target_type, class QualDecayedTRef, class F, + absl::enable_if_t<target_type == 2, int> = 0> + void Initialize(F&& f) { + if (f.HasValue()) { + InitializeStorage<QualDecayedTRef>(std::forward<F>(f)); + } else { + manager_ = EmptyManager; + invoker_ = nullptr; + } + } + + template <int target_type, class QualDecayedTRef, class F, + typename = absl::enable_if_t<target_type == 3>> + void Initialize(F&& f) { + InitializeStorage<QualDecayedTRef>(std::forward<F>(f)); + } + + // Use local (inline) storage for applicable target object types. + template <class QualTRef, class... Args, + typename = absl::enable_if_t< + IsStoredLocally<RemoveCVRef<QualTRef>>::value>> + void InitializeStorage(Args&&... args) { + using RawT = RemoveCVRef<QualTRef>; + ::new (static_cast<void*>(&state_.storage)) + RawT(std::forward<Args>(args)...); + + invoker_ = LocalInvoker<SigIsNoexcept, ReturnType, QualTRef, P...>; + // We can simplify our manager if we know the type is trivially copyable. + InitializeLocalManager<RawT>(); + } + + // Use remote storage for target objects that cannot be stored locally. + template <class QualTRef, class... Args, + absl::enable_if_t<!IsStoredLocally<RemoveCVRef<QualTRef>>::value, + int> = 0> + void InitializeStorage(Args&&... args) { + InitializeRemoteManager<RemoveCVRef<QualTRef>>(std::forward<Args>(args)...); + // This is set after everything else in case an exception is thrown in an + // earlier step of the initialization. + invoker_ = RemoteInvoker<SigIsNoexcept, ReturnType, QualTRef, P...>; + } + + template <class T, + typename = absl::enable_if_t<std::is_trivially_copyable<T>::value>> + void InitializeLocalManager() { + manager_ = LocalManagerTrivial; + } + + template <class T, + absl::enable_if_t<!std::is_trivially_copyable<T>::value, int> = 0> + void InitializeLocalManager() { + manager_ = LocalManagerNontrivial<T>; + } + + template <class T> + using HasTrivialRemoteStorage = + std::integral_constant<bool, std::is_trivially_destructible<T>::value && + alignof(T) <= + ABSL_INTERNAL_DEFAULT_NEW_ALIGNMENT>; + + template <class T, class... Args, + typename = absl::enable_if_t<HasTrivialRemoteStorage<T>::value>> + void InitializeRemoteManager(Args&&... args) { + // unique_ptr is used for exception-safety in case construction throws. + std::unique_ptr<void, TrivialDeleter> uninitialized_target( + ::operator new(sizeof(T)), TrivialDeleter(sizeof(T))); + ::new (uninitialized_target.get()) T(std::forward<Args>(args)...); + state_.remote.target = uninitialized_target.release(); + state_.remote.size = sizeof(T); + manager_ = RemoteManagerTrivial; + } + + template <class T, class... Args, + absl::enable_if_t<!HasTrivialRemoteStorage<T>::value, int> = 0> + void InitializeRemoteManager(Args&&... args) { + state_.remote.target = ::new T(std::forward<Args>(args)...); + manager_ = RemoteManagerNontrivial<T>; + } + + ////////////////////////////////////////////////////////////////////////////// + // + // Type trait to determine if the template argument is an AnyInvocable whose + // function type is compatible enough with ours such that we can + // "move the guts" out of it when moving, rather than having to place a new + // object into remote storage. + + template <typename Other> + struct IsCompatibleAnyInvocable { + static constexpr bool value = false; + }; + + template <typename Sig> + struct IsCompatibleAnyInvocable<AnyInvocable<Sig>> { + static constexpr bool value = + (IsCompatibleConversion)(static_cast< + typename AnyInvocable<Sig>::CoreImpl*>( + nullptr), + static_cast<CoreImpl*>(nullptr)); + }; + + // + ////////////////////////////////////////////////////////////////////////////// + + TypeErasedState state_; + ManagerType* manager_; + InvokerType<SigIsNoexcept, ReturnType, P...>* invoker_; +}; + +// A constructor name-tag used with Impl to request the +// conversion-constructor +struct ConversionConstruct {}; + +//////////////////////////////////////////////////////////////////////////////// +// +// A metafunction that is normally an identity metafunction except that when +// given a std::reference_wrapper<T>, it yields T&. This is necessary because +// currently std::reference_wrapper's operator() is not conditionally noexcept, +// so when checking if such an Invocable is nothrow-invocable, we must pull out +// the underlying type. +template <class T> +struct UnwrapStdReferenceWrapperImpl { + using type = T; +}; + +template <class T> +struct UnwrapStdReferenceWrapperImpl<std::reference_wrapper<T>> { + using type = T&; +}; + +template <class T> +using UnwrapStdReferenceWrapper = + typename UnwrapStdReferenceWrapperImpl<T>::type; +// +//////////////////////////////////////////////////////////////////////////////// + +// An alias that always yields std::true_type (used with constraints) where +// substitution failures happen when forming the template arguments. +template <class... T> +using True = + std::integral_constant<bool, sizeof(absl::void_t<T...>*) != 0>; + +/*SFINAE constraints for the conversion-constructor.*/ +template <class Sig, class F, + class = absl::enable_if_t< + !std::is_same<RemoveCVRef<F>, AnyInvocable<Sig>>::value>> +using CanConvert = + True<absl::enable_if_t<!IsInPlaceType<RemoveCVRef<F>>::value>, + absl::enable_if_t<Impl<Sig>::template CallIsValid<F>::value>, + absl::enable_if_t< + Impl<Sig>::template CallIsNoexceptIfSigIsNoexcept<F>::value>, + absl::enable_if_t<std::is_constructible<absl::decay_t<F>, F>::value>>; + +/*SFINAE constraints for the std::in_place constructors.*/ +template <class Sig, class F, class... Args> +using CanEmplace = True< + absl::enable_if_t<Impl<Sig>::template CallIsValid<F>::value>, + absl::enable_if_t< + Impl<Sig>::template CallIsNoexceptIfSigIsNoexcept<F>::value>, + absl::enable_if_t<std::is_constructible<absl::decay_t<F>, Args...>::value>>; + +/*SFINAE constraints for the conversion-assign operator.*/ +template <class Sig, class F, + class = absl::enable_if_t< + !std::is_same<RemoveCVRef<F>, AnyInvocable<Sig>>::value>> +using CanAssign = + True<absl::enable_if_t<Impl<Sig>::template CallIsValid<F>::value>, + absl::enable_if_t< + Impl<Sig>::template CallIsNoexceptIfSigIsNoexcept<F>::value>, + absl::enable_if_t<std::is_constructible<absl::decay_t<F>, F>::value>>; + +/*SFINAE constraints for the reference-wrapper conversion-assign operator.*/ +template <class Sig, class F> +using CanAssignReferenceWrapper = + True<absl::enable_if_t< + Impl<Sig>::template CallIsValid<std::reference_wrapper<F>>::value>, + absl::enable_if_t<Impl<Sig>::template CallIsNoexceptIfSigIsNoexcept< + std::reference_wrapper<F>>::value>>; + +//////////////////////////////////////////////////////////////////////////////// +// +// The constraint for checking whether or not a call meets the noexcept +// callability requirements. This is a preprocessor macro because specifying it +// this way as opposed to a disjunction/branch can improve the user-side error +// messages and avoids an instantiation of std::is_nothrow_invocable_r in the +// cases where the user did not specify a noexcept function type. +// +#define ABSL_INTERNAL_ANY_INVOCABLE_NOEXCEPT_CONSTRAINT(inv_quals, noex) \ + ABSL_INTERNAL_ANY_INVOCABLE_NOEXCEPT_CONSTRAINT_##noex(inv_quals) + +// The disjunction below is because we can't rely on std::is_nothrow_invocable_r +// to give the right result when ReturnType is non-moveable in toolchains that +// don't treat non-moveable result types correctly. For example this was the +// case in libc++ before commit c3a24882 (2022-05). +#define ABSL_INTERNAL_ANY_INVOCABLE_NOEXCEPT_CONSTRAINT_true(inv_quals) \ + absl::enable_if_t<absl::disjunction< \ + std::is_nothrow_invocable_r< \ + ReturnType, UnwrapStdReferenceWrapper<absl::decay_t<F>> inv_quals, \ + P...>, \ + std::conjunction< \ + std::is_nothrow_invocable< \ + UnwrapStdReferenceWrapper<absl::decay_t<F>> inv_quals, P...>, \ + std::is_same< \ + ReturnType, \ + absl::base_internal::invoke_result_t< \ + UnwrapStdReferenceWrapper<absl::decay_t<F>> inv_quals, \ + P...>>>>::value> + +#define ABSL_INTERNAL_ANY_INVOCABLE_NOEXCEPT_CONSTRAINT_false(inv_quals) +// +//////////////////////////////////////////////////////////////////////////////// + +// A macro to generate partial specializations of Impl with the different +// combinations of supported cv/reference qualifiers and noexcept specifier. +// +// Here, `cv` are the cv-qualifiers if any, `ref` is the ref-qualifier if any, +// inv_quals is the reference type to be used when invoking the target, and +// noex is "true" if the function type is noexcept, or false if it is not. +// +// The CallIsValid condition is more complicated than simply using +// absl::base_internal::is_invocable_r because we can't rely on it to give the +// right result when ReturnType is non-moveable in toolchains that don't treat +// non-moveable result types correctly. For example this was the case in libc++ +// before commit c3a24882 (2022-05). +#define ABSL_INTERNAL_ANY_INVOCABLE_IMPL_(cv, ref, inv_quals, noex) \ + template <class ReturnType, class... P> \ + class Impl<ReturnType(P...) cv ref ABSL_INTERNAL_NOEXCEPT_SPEC(noex)> \ + : public CoreImpl<noex, ReturnType, P...> { \ + public: \ + /*The base class, which contains the datamembers and core operations*/ \ + using Core = CoreImpl<noex, ReturnType, P...>; \ + \ + /*SFINAE constraint to check if F is invocable with the proper signature*/ \ + template <class F> \ + using CallIsValid = True<absl::enable_if_t<absl::disjunction< \ + absl::base_internal::is_invocable_r<ReturnType, \ + absl::decay_t<F> inv_quals, P...>, \ + std::is_same<ReturnType, \ + absl::base_internal::invoke_result_t< \ + absl::decay_t<F> inv_quals, P...>>>::value>>; \ + \ + /*SFINAE constraint to check if F is nothrow-invocable when necessary*/ \ + template <class F> \ + using CallIsNoexceptIfSigIsNoexcept = \ + True<ABSL_INTERNAL_ANY_INVOCABLE_NOEXCEPT_CONSTRAINT(inv_quals, \ + noex)>; \ + \ + /*Put the AnyInvocable into an empty state.*/ \ + Impl() = default; \ + \ + /*The implementation of a conversion-constructor from "f*/ \ + /*This forwards to Core, attaching inv_quals so that the base class*/ \ + /*knows how to properly type-erase the invocation.*/ \ + template <class F> \ + explicit Impl(ConversionConstruct, F&& f) \ + : Core(TypedConversionConstruct< \ + typename std::decay<F>::type inv_quals>(), \ + std::forward<F>(f)) {} \ + \ + /*Forward along the in-place construction parameters.*/ \ + template <class T, class... Args> \ + explicit Impl(absl::in_place_type_t<T>, Args&&... args) \ + : Core(absl::in_place_type<absl::decay_t<T> inv_quals>, \ + std::forward<Args>(args)...) {} \ + \ + /*The actual invocation operation with the proper signature*/ \ + ReturnType operator()(P... args) cv ref noexcept(noex) { \ + assert(this->invoker_ != nullptr); \ + return this->invoker_(const_cast<TypeErasedState*>(&this->state_), \ + static_cast<ForwardedParameterType<P>>(args)...); \ + } \ + } + +// Define the `noexcept(true)` specialization only for C++17 and beyond, when +// `noexcept` is part of the type system. +#if ABSL_INTERNAL_CPLUSPLUS_LANG >= 201703L +// A convenience macro that defines specializations for the noexcept(true) and +// noexcept(false) forms, given the other properties. +#define ABSL_INTERNAL_ANY_INVOCABLE_IMPL(cv, ref, inv_quals) \ + ABSL_INTERNAL_ANY_INVOCABLE_IMPL_(cv, ref, inv_quals, false); \ + ABSL_INTERNAL_ANY_INVOCABLE_IMPL_(cv, ref, inv_quals, true) +#else +#define ABSL_INTERNAL_ANY_INVOCABLE_IMPL(cv, ref, inv_quals) \ + ABSL_INTERNAL_ANY_INVOCABLE_IMPL_(cv, ref, inv_quals, false) +#endif + +// Non-ref-qualified partial specializations +ABSL_INTERNAL_ANY_INVOCABLE_IMPL(, , &); +ABSL_INTERNAL_ANY_INVOCABLE_IMPL(const, , const&); + +// Lvalue-ref-qualified partial specializations +ABSL_INTERNAL_ANY_INVOCABLE_IMPL(, &, &); +ABSL_INTERNAL_ANY_INVOCABLE_IMPL(const, &, const&); + +// Rvalue-ref-qualified partial specializations +ABSL_INTERNAL_ANY_INVOCABLE_IMPL(, &&, &&); +ABSL_INTERNAL_ANY_INVOCABLE_IMPL(const, &&, const&&); + +// Undef the detail-only macros. +#undef ABSL_INTERNAL_ANY_INVOCABLE_IMPL +#undef ABSL_INTERNAL_ANY_INVOCABLE_IMPL_ +#undef ABSL_INTERNAL_ANY_INVOCABLE_NOEXCEPT_CONSTRAINT_false +#undef ABSL_INTERNAL_ANY_INVOCABLE_NOEXCEPT_CONSTRAINT_true +#undef ABSL_INTERNAL_ANY_INVOCABLE_NOEXCEPT_CONSTRAINT +#undef ABSL_INTERNAL_NOEXCEPT_SPEC + +} // namespace internal_any_invocable +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_FUNCTIONAL_INTERNAL_ANY_INVOCABLE_H_ diff --git a/absl/hash/BUILD.bazel b/absl/hash/BUILD.bazel index 4b2c220f..bcc316f9 100644 --- a/absl/hash/BUILD.bazel +++ b/absl/hash/BUILD.bazel @@ -14,7 +14,6 @@ # limitations under the License. # -load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") load( "//absl:copts/configure_copts.bzl", "ABSL_DEFAULT_COPTS", @@ -37,11 +36,12 @@ cc_library( linkopts = ABSL_DEFAULT_LINKOPTS, deps = [ ":city", - ":wyhash", + ":low_level_hash", "//absl/base:config", "//absl/base:core_headers", "//absl/base:endian", "//absl/container:fixed_array", + "//absl/functional:function_ref", "//absl/meta:type_traits", "//absl/numeric:int128", "//absl/strings", @@ -75,7 +75,11 @@ cc_test( ":hash_testing", ":spy_hash_state", "//absl/base:core_headers", + "//absl/container:btree", + "//absl/container:flat_hash_map", "//absl/container:flat_hash_set", + "//absl/container:node_hash_map", + "//absl/container:node_hash_set", "//absl/meta:type_traits", "//absl/numeric:int128", "//absl/strings:cord_test_helpers", @@ -94,6 +98,7 @@ cc_binary( deps = [ ":hash", "//absl/base:core_headers", + "//absl/container:flat_hash_set", "//absl/random", "//absl/strings", "//absl/strings:cord", @@ -143,27 +148,28 @@ cc_test( ) cc_library( - name = "wyhash", - srcs = ["internal/wyhash.cc"], - hdrs = ["internal/wyhash.h"], + name = "low_level_hash", + srcs = ["internal/low_level_hash.cc"], + hdrs = ["internal/low_level_hash.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, visibility = ["//visibility:private"], deps = [ "//absl/base:config", "//absl/base:endian", + "//absl/numeric:bits", "//absl/numeric:int128", ], ) cc_test( - name = "wyhash_test", - srcs = ["internal/wyhash_test.cc"], + name = "low_level_hash_test", + srcs = ["internal/low_level_hash_test.cc"], copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, visibility = ["//visibility:private"], deps = [ - ":wyhash", + ":low_level_hash", "//absl/strings", "@com_google_googletest//:gtest_main", ], diff --git a/absl/hash/CMakeLists.txt b/absl/hash/CMakeLists.txt index b43bfa54..423b74b5 100644 --- a/absl/hash/CMakeLists.txt +++ b/absl/hash/CMakeLists.txt @@ -30,13 +30,14 @@ absl_cc_library( absl::core_headers absl::endian absl::fixed_array + absl::function_ref absl::meta absl::int128 absl::strings absl::optional absl::variant absl::utility - absl::wyhash + absl::low_level_hash PUBLIC ) @@ -52,7 +53,7 @@ absl_cc_library( absl::meta absl::strings absl::variant - gmock + GTest::gmock TESTONLY ) @@ -68,13 +69,18 @@ absl_cc_test( absl::hash absl::hash_testing absl::core_headers + absl::btree + absl::flat_hash_map absl::flat_hash_set + absl::node_hash_map + absl::node_hash_set absl::spy_hash_state absl::meta absl::int128 - gmock_main + GTest::gmock_main ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME spy_hash_state @@ -89,6 +95,7 @@ absl_cc_library( TESTONLY ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME city @@ -113,19 +120,21 @@ absl_cc_test( ${ABSL_TEST_COPTS} DEPS absl::city - gmock_main + GTest::gmock_main ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME - wyhash + low_level_hash HDRS - "internal/wyhash.h" + "internal/low_level_hash.h" SRCS - "internal/wyhash.cc" + "internal/low_level_hash.cc" COPTS ${ABSL_DEFAULT_COPTS} DEPS + absl::bits absl::config absl::endian absl::int128 @@ -133,13 +142,13 @@ absl_cc_library( absl_cc_test( NAME - wyhash_test + low_level_hash_test SRCS - "internal/wyhash_test.cc" + "internal/low_level_hash_test.cc" COPTS ${ABSL_TEST_COPTS} DEPS - absl::wyhash + absl::low_level_hash absl::strings - gmock_main + GTest::gmock_main ) diff --git a/absl/hash/hash.h b/absl/hash/hash.h index 5de132ca..74e2d7c0 100644 --- a/absl/hash/hash.h +++ b/absl/hash/hash.h @@ -26,9 +26,9 @@ // support Abseil hashing without requiring you to define a hashing // algorithm. // * `HashState`, a type-erased class which implements the manipulation of the -// hash state (H) itself, contains member functions `combine()` and -// `combine_contiguous()`, which you can use to contribute to an existing -// hash state when hashing your types. +// hash state (H) itself; contains member functions `combine()`, +// `combine_contiguous()`, and `combine_unordered()`; and which you can use +// to contribute to an existing hash state when hashing your types. // // Unlike `std::hash` or other hashing frameworks, the Abseil hashing framework // provides most of its utility by abstracting away the hash algorithm (and its @@ -40,6 +40,11 @@ // each process. E.g., `absl::Hash<int>{}(9)` in one process and // `absl::Hash<int>{}(9)` in another process are likely to differ. // +// `absl::Hash` may also produce different values from different dynamically +// loaded libraries. For this reason, `absl::Hash` values must never cross +// boundries in dynamically loaded libraries (including when used in types like +// hash containers.) +// // `absl::Hash` is intended to strongly mix input bits with a target of passing // an [Avalanche Test](https://en.wikipedia.org/wiki/Avalanche_effect). // @@ -73,6 +78,10 @@ #ifndef ABSL_HASH_HASH_H_ #define ABSL_HASH_HASH_H_ +#include <tuple> +#include <utility> + +#include "absl/functional/function_ref.h" #include "absl/hash/internal/hash.h" namespace absl { @@ -105,14 +114,27 @@ ABSL_NAMESPACE_BEGIN // * std::string_view (as well as any instance of std::basic_string that // uses char and std::char_traits) // * All the standard sequence containers (provided the elements are hashable) -// * All the standard ordered associative containers (provided the elements are +// * All the standard associative containers (provided the elements are // hashable) // * absl types such as the following: // * absl::string_view -// * absl::InlinedVector -// * absl::FixedArray // * absl::uint128 // * absl::Time, absl::Duration, and absl::TimeZone +// * absl containers (provided the elements are hashable) such as the +// following: +// * absl::flat_hash_set, absl::node_hash_set, absl::btree_set +// * absl::flat_hash_map, absl::node_hash_map, absl::btree_map +// * absl::btree_multiset, absl::btree_multimap +// * absl::InlinedVector +// * absl::FixedArray +// +// When absl::Hash is used to hash an unordered container with a custom hash +// functor, the elements are hashed using default absl::Hash semantics, not +// the custom hash functor. This is consistent with the behavior of +// operator==() on unordered containers, which compares elements pairwise with +// operator==() rather than the custom equality functor. It is usually a +// mistake to use either operator==() or absl::Hash on unordered collections +// that use functors incompatible with operator==() equality. // // Note: the list above is not meant to be exhaustive. Additional type support // may be added, in which case the above list will be updated. @@ -151,7 +173,8 @@ ABSL_NAMESPACE_BEGIN // that are otherwise difficult to extend using `AbslHashValue()`. (See the // `HashState` class below.) // -// The "hash state" concept contains two member functions for mixing hash state: +// The "hash state" concept contains three member functions for mixing hash +// state: // // * `H::combine(state, values...)` // @@ -185,6 +208,15 @@ ABSL_NAMESPACE_BEGIN // (it may perform internal optimizations). If you need this guarantee, use a // loop instead. // +// * `H::combine_unordered(state, begin, end)` +// +// Combines a set of elements denoted by an iterator pair into a hash +// state, returning the updated state. Note that the existing hash +// state is move-only and must be passed by value. +// +// Unlike the other two methods, the hashing is order-independent. +// This can be used to hash unordered collections. +// // ----------------------------------------------------------------------------- // Adding Type Support to `absl::Hash` // ----------------------------------------------------------------------------- @@ -214,6 +246,26 @@ ABSL_NAMESPACE_BEGIN template <typename T> using Hash = absl::hash_internal::Hash<T>; +// HashOf +// +// absl::HashOf() is a helper that generates a hash from the values of its +// arguments. It dispatches to absl::Hash directly, as follows: +// * HashOf(t) == absl::Hash<T>{}(t) +// * HashOf(a, b, c) == HashOf(std::make_tuple(a, b, c)) +// +// HashOf(a1, a2, ...) == HashOf(b1, b2, ...) is guaranteed when +// * The argument lists have pairwise identical C++ types +// * a1 == b1 && a2 == b2 && ... +// +// The requirement that the arguments match in both type and value is critical. +// It means that `a == b` does not necessarily imply `HashOf(a) == HashOf(b)` if +// `a` and `b` have different types. For example, `HashOf(2) != HashOf(2.0)`. +template <int&... ExplicitArgumentBarrier, typename... Types> +size_t HashOf(const Types&... values) { + auto tuple = std::tie(values...); + return absl::Hash<decltype(tuple)>{}(tuple); +} + // HashState // // A type erased version of the hash state concept, for use in user-defined @@ -221,8 +273,9 @@ using Hash = absl::hash_internal::Hash<T>; // classes, virtual functions, etc.). The type erasure adds overhead so it // should be avoided unless necessary. // -// Note: This wrapper will only erase calls to: +// Note: This wrapper will only erase calls to // combine_contiguous(H, const unsigned char*, size_t) +// RunCombineUnordered(H, CombinerF) // // All other calls will be handled internally and will not invoke overloads // provided by the wrapped class. @@ -296,6 +349,8 @@ class HashState : public hash_internal::HashStateBase<HashState> { private: HashState() = default; + friend class HashState::HashStateBase; + template <typename T> static void CombineContiguousImpl(void* p, const unsigned char* first, size_t size) { @@ -307,16 +362,57 @@ class HashState : public hash_internal::HashStateBase<HashState> { void Init(T* state) { state_ = state; combine_contiguous_ = &CombineContiguousImpl<T>; + run_combine_unordered_ = &RunCombineUnorderedImpl<T>; + } + + template <typename HS> + struct CombineUnorderedInvoker { + template <typename T, typename ConsumerT> + void operator()(T inner_state, ConsumerT inner_cb) { + f(HashState::Create(&inner_state), + [&](HashState& inner_erased) { inner_cb(inner_erased.Real<T>()); }); + } + + absl::FunctionRef<void(HS, absl::FunctionRef<void(HS&)>)> f; + }; + + template <typename T> + static HashState RunCombineUnorderedImpl( + HashState state, + absl::FunctionRef<void(HashState, absl::FunctionRef<void(HashState&)>)> + f) { + // Note that this implementation assumes that inner_state and outer_state + // are the same type. This isn't true in the SpyHash case, but SpyHash + // types are move-convertible to each other, so this still works. + T& real_state = state.Real<T>(); + real_state = T::RunCombineUnordered( + std::move(real_state), CombineUnorderedInvoker<HashState>{f}); + return state; + } + + template <typename CombinerT> + static HashState RunCombineUnordered(HashState state, CombinerT combiner) { + auto* run = state.run_combine_unordered_; + return run(std::move(state), std::ref(combiner)); } // Do not erase an already erased state. void Init(HashState* state) { state_ = state->state_; combine_contiguous_ = state->combine_contiguous_; + run_combine_unordered_ = state->run_combine_unordered_; + } + + template <typename T> + T& Real() { + return *static_cast<T*>(state_); } void* state_; void (*combine_contiguous_)(void*, const unsigned char*, size_t); + HashState (*run_combine_unordered_)( + HashState state, + absl::FunctionRef<void(HashState, absl::FunctionRef<void(HashState&)>)>); }; ABSL_NAMESPACE_END diff --git a/absl/hash/hash_benchmark.cc b/absl/hash/hash_benchmark.cc index d498ac29..8712a01c 100644 --- a/absl/hash/hash_benchmark.cc +++ b/absl/hash/hash_benchmark.cc @@ -19,6 +19,7 @@ #include <vector> #include "absl/base/attributes.h" +#include "absl/container/flat_hash_set.h" #include "absl/hash/hash.h" #include "absl/random/random.h" #include "absl/strings/cord.h" @@ -107,6 +108,44 @@ absl::Cord FragmentedCord(size_t size) { return result; } +template <typename T> +std::vector<T> Vector(size_t count) { + std::vector<T> result; + for (size_t v = 0; v < count; ++v) { + result.push_back(v); + } + return result; +} + +// Bogus type that replicates an unorderd_set's bit mixing, but with +// vector-speed iteration. This is intended to measure the overhead of unordered +// hashing without counting the speed of unordered_set iteration. +template <typename T> +struct FastUnorderedSet { + explicit FastUnorderedSet(size_t count) { + for (size_t v = 0; v < count; ++v) { + values.push_back(v); + } + } + std::vector<T> values; + + template <typename H> + friend H AbslHashValue(H h, const FastUnorderedSet& fus) { + return H::combine(H::combine_unordered(std::move(h), fus.values.begin(), + fus.values.end()), + fus.values.size()); + } +}; + +template <typename T> +absl::flat_hash_set<T> FlatHashSet(size_t count) { + absl::flat_hash_set<T> result; + for (size_t v = 0; v < count; ++v) { + result.insert(v); + } + return result; +} + // Generates a benchmark and a codegen method for the provided types. The // codegen method provides a well known entrypoint for dumping assembly. #define MAKE_BENCHMARK(hash, name, ...) \ @@ -145,10 +184,22 @@ MAKE_BENCHMARK(AbslHash, Cord_Flat_200, FlatCord(200)); MAKE_BENCHMARK(AbslHash, Cord_Flat_5000, FlatCord(5000)); MAKE_BENCHMARK(AbslHash, Cord_Fragmented_200, FragmentedCord(200)); MAKE_BENCHMARK(AbslHash, Cord_Fragmented_5000, FragmentedCord(5000)); -MAKE_BENCHMARK(AbslHash, VectorInt64_10, std::vector<int64_t>(10)); -MAKE_BENCHMARK(AbslHash, VectorInt64_100, std::vector<int64_t>(100)); -MAKE_BENCHMARK(AbslHash, VectorDouble_10, std::vector<double>(10, 1.1)); -MAKE_BENCHMARK(AbslHash, VectorDouble_100, std::vector<double>(100, 1.1)); +MAKE_BENCHMARK(AbslHash, VectorInt64_10, Vector<int64_t>(10)); +MAKE_BENCHMARK(AbslHash, VectorInt64_100, Vector<int64_t>(100)); +MAKE_BENCHMARK(AbslHash, VectorInt64_1000, Vector<int64_t>(1000)); +MAKE_BENCHMARK(AbslHash, VectorDouble_10, Vector<double>(10)); +MAKE_BENCHMARK(AbslHash, VectorDouble_100, Vector<double>(100)); +MAKE_BENCHMARK(AbslHash, VectorDouble_1000, Vector<double>(1000)); +MAKE_BENCHMARK(AbslHash, FlatHashSetInt64_10, FlatHashSet<int64_t>(10)); +MAKE_BENCHMARK(AbslHash, FlatHashSetInt64_100, FlatHashSet<int64_t>(100)); +MAKE_BENCHMARK(AbslHash, FlatHashSetInt64_1000, FlatHashSet<int64_t>(1000)); +MAKE_BENCHMARK(AbslHash, FlatHashSetDouble_10, FlatHashSet<double>(10)); +MAKE_BENCHMARK(AbslHash, FlatHashSetDouble_100, FlatHashSet<double>(100)); +MAKE_BENCHMARK(AbslHash, FlatHashSetDouble_1000, FlatHashSet<double>(1000)); +MAKE_BENCHMARK(AbslHash, FastUnorderedSetInt64_1000, + FastUnorderedSet<int64_t>(1000)); +MAKE_BENCHMARK(AbslHash, FastUnorderedSetDouble_1000, + FastUnorderedSet<double>(1000)); MAKE_BENCHMARK(AbslHash, PairStringString_0, std::make_pair(std::string(), std::string())); MAKE_BENCHMARK(AbslHash, PairStringString_10, @@ -180,6 +231,24 @@ MAKE_BENCHMARK(TypeErasedAbslHash, VectorDouble_10, std::vector<double>(10, 1.1)); MAKE_BENCHMARK(TypeErasedAbslHash, VectorDouble_100, std::vector<double>(100, 1.1)); +MAKE_BENCHMARK(TypeErasedAbslHash, VectorDouble_1000, + std::vector<double>(1000, 1.1)); +MAKE_BENCHMARK(TypeErasedAbslHash, FlatHashSetInt64_10, + FlatHashSet<int64_t>(10)); +MAKE_BENCHMARK(TypeErasedAbslHash, FlatHashSetInt64_100, + FlatHashSet<int64_t>(100)); +MAKE_BENCHMARK(TypeErasedAbslHash, FlatHashSetInt64_1000, + FlatHashSet<int64_t>(1000)); +MAKE_BENCHMARK(TypeErasedAbslHash, FlatHashSetDouble_10, + FlatHashSet<double>(10)); +MAKE_BENCHMARK(TypeErasedAbslHash, FlatHashSetDouble_100, + FlatHashSet<double>(100)); +MAKE_BENCHMARK(TypeErasedAbslHash, FlatHashSetDouble_1000, + FlatHashSet<double>(1000)); +MAKE_BENCHMARK(TypeErasedAbslHash, FastUnorderedSetInt64_1000, + FastUnorderedSet<int64_t>(1000)); +MAKE_BENCHMARK(TypeErasedAbslHash, FastUnorderedSetDouble_1000, + FastUnorderedSet<double>(1000)); // The latency benchmark attempts to model the speed of the hash function in // production. When a hash function is used for hashtable lookups it is rarely diff --git a/absl/hash/hash_test.cc b/absl/hash/hash_test.cc index 1d2e6cf0..ffa45e6e 100644 --- a/absl/hash/hash_test.cc +++ b/absl/hash/hash_test.cc @@ -14,12 +14,14 @@ #include "absl/hash/hash.h" +#include <algorithm> #include <array> #include <bitset> #include <cstring> #include <deque> #include <forward_list> #include <functional> +#include <initializer_list> #include <iterator> #include <limits> #include <list> @@ -32,12 +34,18 @@ #include <tuple> #include <type_traits> #include <unordered_map> +#include <unordered_set> #include <utility> #include <vector> #include "gmock/gmock.h" #include "gtest/gtest.h" +#include "absl/container/btree_map.h" +#include "absl/container/btree_set.h" +#include "absl/container/flat_hash_map.h" #include "absl/container/flat_hash_set.h" +#include "absl/container/node_hash_map.h" +#include "absl/container/node_hash_set.h" #include "absl/hash/hash_testing.h" #include "absl/hash/internal/spy_hash_state.h" #include "absl/meta/type_traits.h" @@ -46,6 +54,56 @@ namespace { +// Utility wrapper of T for the purposes of testing the `AbslHash` type erasure +// mechanism. `TypeErasedValue<T>` can be constructed with a `T`, and can +// be compared and hashed. However, all hashing goes through the hashing +// type-erasure framework. +template <typename T> +class TypeErasedValue { + public: + TypeErasedValue() = default; + TypeErasedValue(const TypeErasedValue&) = default; + TypeErasedValue(TypeErasedValue&&) = default; + explicit TypeErasedValue(const T& n) : n_(n) {} + + template <typename H> + friend H AbslHashValue(H hash_state, const TypeErasedValue& v) { + v.HashValue(absl::HashState::Create(&hash_state)); + return hash_state; + } + + void HashValue(absl::HashState state) const { + absl::HashState::combine(std::move(state), n_); + } + + bool operator==(const TypeErasedValue& rhs) const { return n_ == rhs.n_; } + bool operator!=(const TypeErasedValue& rhs) const { return !(*this == rhs); } + + private: + T n_; +}; + +// A TypeErasedValue refinement, for containers. It exposes the wrapped +// `value_type` and is constructible from an initializer list. +template <typename T> +class TypeErasedContainer : public TypeErasedValue<T> { + public: + using value_type = typename T::value_type; + TypeErasedContainer() = default; + TypeErasedContainer(const TypeErasedContainer&) = default; + TypeErasedContainer(TypeErasedContainer&&) = default; + explicit TypeErasedContainer(const T& n) : TypeErasedValue<T>(n) {} + TypeErasedContainer(std::initializer_list<value_type> init_list) + : TypeErasedContainer(T(init_list.begin(), init_list.end())) {} + // one-argument constructor of value type T, to appease older toolchains that + // get confused by one-element initializer lists in some contexts + explicit TypeErasedContainer(const value_type& v) + : TypeErasedContainer(T(&v, &v + 1)) {} +}; + +template <typename T> +using TypeErasedVector = TypeErasedContainer<std::vector<T>>; + using absl::Hash; using absl::hash_internal::SpyHashState; @@ -81,10 +139,10 @@ TYPED_TEST_P(HashValueIntTest, FastPath) { absl::Hash<std::tuple<TypeParam>>{}(std::tuple<TypeParam>(n))); } -REGISTER_TYPED_TEST_CASE_P(HashValueIntTest, BasicUsage, FastPath); +REGISTER_TYPED_TEST_SUITE_P(HashValueIntTest, BasicUsage, FastPath); using IntTypes = testing::Types<unsigned char, char, int, int32_t, int64_t, uint32_t, uint64_t, size_t>; -INSTANTIATE_TYPED_TEST_CASE_P(My, HashValueIntTest, IntTypes); +INSTANTIATE_TYPED_TEST_SUITE_P(My, HashValueIntTest, IntTypes); enum LegacyEnum { kValue1, kValue2, kValue3 }; @@ -127,6 +185,8 @@ TEST(HashValueTest, FloatingPoint) { TEST(HashValueTest, Pointer) { EXPECT_TRUE((is_hashable<int*>::value)); + EXPECT_TRUE((is_hashable<int(*)(char, float)>::value)); + EXPECT_TRUE((is_hashable<void(*)(int, int, ...)>::value)); int i; int* ptr = &i; @@ -166,6 +226,85 @@ TEST(HashValueTest, PointerAlignment) { } } +TEST(HashValueTest, PointerToMember) { + struct Bass { + void q() {} + }; + + struct A : Bass { + virtual ~A() = default; + virtual void vfa() {} + + static auto pq() -> void (A::*)() { return &A::q; } + }; + + struct B : Bass { + virtual ~B() = default; + virtual void vfb() {} + + static auto pq() -> void (B::*)() { return &B::q; } + }; + + struct Foo : A, B { + void f1() {} + void f2() const {} + + int g1() & { return 0; } + int g2() const & { return 0; } + int g3() && { return 0; } + int g4() const && { return 0; } + + int h1() & { return 0; } + int h2() const & { return 0; } + int h3() && { return 0; } + int h4() const && { return 0; } + + int a; + int b; + + const int c = 11; + const int d = 22; + }; + + EXPECT_TRUE((is_hashable<float Foo::*>::value)); + EXPECT_TRUE((is_hashable<double (Foo::*)(int, int)&&>::value)); + + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly( + std::make_tuple(&Foo::a, &Foo::b, static_cast<int Foo::*>(nullptr)))); + + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly( + std::make_tuple(&Foo::c, &Foo::d, static_cast<const int Foo::*>(nullptr), + &Foo::a, &Foo::b))); + + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly(std::make_tuple( + &Foo::f1, static_cast<void (Foo::*)()>(nullptr)))); + + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly(std::make_tuple( + &Foo::f2, static_cast<void (Foo::*)() const>(nullptr)))); + + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly(std::make_tuple( + &Foo::g1, &Foo::h1, static_cast<int (Foo::*)() &>(nullptr)))); + + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly(std::make_tuple( + &Foo::g2, &Foo::h2, static_cast<int (Foo::*)() const &>(nullptr)))); + + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly(std::make_tuple( + &Foo::g3, &Foo::h3, static_cast<int (Foo::*)() &&>(nullptr)))); + + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly(std::make_tuple( + &Foo::g4, &Foo::h4, static_cast<int (Foo::*)() const &&>(nullptr)))); + + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly( + std::make_tuple(static_cast<void (Foo::*)()>(&Foo::vfa), + static_cast<void (Foo::*)()>(&Foo::vfb), + static_cast<void (Foo::*)()>(nullptr)))); + + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly( + std::make_tuple(static_cast<void (Foo::*)()>(Foo::A::pq()), + static_cast<void (Foo::*)()>(Foo::B::pq()), + static_cast<void (Foo::*)()>(nullptr)))); +} + TEST(HashValueTest, PairAndTuple) { EXPECT_TRUE((is_hashable<std::pair<int, int>>::value)); EXPECT_TRUE((is_hashable<std::pair<const int&, const int&>>::value)); @@ -381,6 +520,52 @@ TEST(HashValueTest, StdBitset) { std::bitset<kNumBits>(bit_strings[5].c_str())})); } // namespace +// Dummy type with unordered equality and hashing semantics. This preserves +// input order internally, and is used below to ensure we get test coverage +// for equal sequences with different iteraton orders. +template <typename T> +class UnorderedSequence { + public: + UnorderedSequence() = default; + template <typename TT> + UnorderedSequence(std::initializer_list<TT> l) + : values_(l.begin(), l.end()) {} + template <typename ForwardIterator, + typename std::enable_if<!std::is_integral<ForwardIterator>::value, + bool>::type = true> + UnorderedSequence(ForwardIterator begin, ForwardIterator end) + : values_(begin, end) {} + // one-argument constructor of value type T, to appease older toolchains that + // get confused by one-element initializer lists in some contexts + explicit UnorderedSequence(const T& v) : values_(&v, &v + 1) {} + + using value_type = T; + + size_t size() const { return values_.size(); } + typename std::vector<T>::const_iterator begin() const { + return values_.begin(); + } + typename std::vector<T>::const_iterator end() const { return values_.end(); } + + friend bool operator==(const UnorderedSequence& lhs, + const UnorderedSequence& rhs) { + return lhs.size() == rhs.size() && + std::is_permutation(lhs.begin(), lhs.end(), rhs.begin()); + } + friend bool operator!=(const UnorderedSequence& lhs, + const UnorderedSequence& rhs) { + return !(lhs == rhs); + } + template <typename H> + friend H AbslHashValue(H h, const UnorderedSequence& u) { + return H::combine(H::combine_unordered(std::move(h), u.begin(), u.end()), + u.size()); + } + + private: + std::vector<T> values_; +}; + template <typename T> class HashValueSequenceTest : public testing::Test { }; @@ -389,22 +574,66 @@ TYPED_TEST_SUITE_P(HashValueSequenceTest); TYPED_TEST_P(HashValueSequenceTest, BasicUsage) { EXPECT_TRUE((is_hashable<TypeParam>::value)); - using ValueType = typename TypeParam::value_type; - auto a = static_cast<ValueType>(0); - auto b = static_cast<ValueType>(23); - auto c = static_cast<ValueType>(42); + using IntType = typename TypeParam::value_type; + auto a = static_cast<IntType>(0); + auto b = static_cast<IntType>(23); + auto c = static_cast<IntType>(42); + + std::vector<TypeParam> exemplars = { + TypeParam(), TypeParam(), TypeParam{a, b, c}, + TypeParam{a, c, b}, TypeParam{c, a, b}, TypeParam{a}, + TypeParam{a, a}, TypeParam{a, a, a}, TypeParam{a, a, b}, + TypeParam{a, b, a}, TypeParam{b, a, a}, TypeParam{a, b}, + TypeParam{b, c}}; + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly(exemplars)); +} - EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly( - std::make_tuple(TypeParam(), TypeParam{}, TypeParam{a, b, c}, - TypeParam{a, b}, TypeParam{b, c}))); +REGISTER_TYPED_TEST_SUITE_P(HashValueSequenceTest, BasicUsage); +using IntSequenceTypes = testing::Types< + std::deque<int>, std::forward_list<int>, std::list<int>, std::vector<int>, + std::vector<bool>, TypeErasedContainer<std::vector<int>>, std::set<int>, + std::multiset<int>, UnorderedSequence<int>, + TypeErasedContainer<UnorderedSequence<int>>, std::unordered_set<int>, + std::unordered_multiset<int>, absl::flat_hash_set<int>, + absl::node_hash_set<int>, absl::btree_set<int>>; +INSTANTIATE_TYPED_TEST_SUITE_P(My, HashValueSequenceTest, IntSequenceTypes); + +template <typename T> +class HashValueNestedSequenceTest : public testing::Test {}; +TYPED_TEST_SUITE_P(HashValueNestedSequenceTest); + +TYPED_TEST_P(HashValueNestedSequenceTest, BasicUsage) { + using T = TypeParam; + using V = typename T::value_type; + std::vector<T> exemplars = { + // empty case + T{}, + // sets of empty sets + T{V{}}, T{V{}, V{}}, T{V{}, V{}, V{}}, + // multisets of different values + T{V{1}}, T{V{1, 1}, V{1, 1}}, T{V{1, 1, 1}, V{1, 1, 1}, V{1, 1, 1}}, + // various orderings of same nested sets + T{V{}, V{1, 2}}, T{V{}, V{2, 1}}, T{V{1, 2}, V{}}, T{V{2, 1}, V{}}, + // various orderings of various nested sets, case 2 + T{V{1, 2}, V{3, 4}}, T{V{1, 2}, V{4, 3}}, T{V{1, 3}, V{2, 4}}, + T{V{1, 3}, V{4, 2}}, T{V{1, 4}, V{2, 3}}, T{V{1, 4}, V{3, 2}}, + T{V{2, 3}, V{1, 4}}, T{V{2, 3}, V{4, 1}}, T{V{2, 4}, V{1, 3}}, + T{V{2, 4}, V{3, 1}}, T{V{3, 4}, V{1, 2}}, T{V{3, 4}, V{2, 1}}}; + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly(exemplars)); } -REGISTER_TYPED_TEST_CASE_P(HashValueSequenceTest, BasicUsage); -using IntSequenceTypes = - testing::Types<std::deque<int>, std::forward_list<int>, std::list<int>, - std::vector<int>, std::vector<bool>, std::set<int>, - std::multiset<int>>; -INSTANTIATE_TYPED_TEST_CASE_P(My, HashValueSequenceTest, IntSequenceTypes); +REGISTER_TYPED_TEST_SUITE_P(HashValueNestedSequenceTest, BasicUsage); +template <typename T> +using TypeErasedSet = TypeErasedContainer<UnorderedSequence<T>>; + +using NestedIntSequenceTypes = testing::Types< + std::vector<std::vector<int>>, std::vector<UnorderedSequence<int>>, + std::vector<TypeErasedSet<int>>, UnorderedSequence<std::vector<int>>, + UnorderedSequence<UnorderedSequence<int>>, + UnorderedSequence<TypeErasedSet<int>>, TypeErasedSet<std::vector<int>>, + TypeErasedSet<UnorderedSequence<int>>, TypeErasedSet<TypeErasedSet<int>>>; +INSTANTIATE_TYPED_TEST_SUITE_P(My, HashValueNestedSequenceTest, + NestedIntSequenceTypes); // Private type that only supports AbslHashValue to make sure our chosen hash // implementation is recursive within absl::Hash. @@ -564,23 +793,64 @@ TEST(HashValueTest, Variant) { #endif } -TEST(HashValueTest, Maps) { - EXPECT_TRUE((is_hashable<std::map<int, std::string>>::value)); +template <typename T> +class HashValueAssociativeMapTest : public testing::Test {}; +TYPED_TEST_SUITE_P(HashValueAssociativeMapTest); + +TYPED_TEST_P(HashValueAssociativeMapTest, BasicUsage) { + using M = TypeParam; + using V = typename M::value_type; + std::vector<M> exemplars{M{}, + M{V{0, "foo"}}, + M{V{1, "foo"}}, + M{V{0, "bar"}}, + M{V{1, "bar"}}, + M{V{0, "foo"}, V{42, "bar"}}, + M{V{42, "bar"}, V{0, "foo"}}, + M{V{1, "foo"}, V{42, "bar"}}, + M{V{1, "foo"}, V{43, "bar"}}, + M{V{1, "foo"}, V{43, "baz"}}}; + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly(exemplars)); +} - using M = std::map<int, std::string>; - EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly(std::make_tuple( - M{}, M{{0, "foo"}}, M{{1, "foo"}}, M{{0, "bar"}}, M{{1, "bar"}}, - M{{0, "foo"}, {42, "bar"}}, M{{1, "foo"}, {42, "bar"}}, - M{{1, "foo"}, {43, "bar"}}, M{{1, "foo"}, {43, "baz"}}))); +REGISTER_TYPED_TEST_SUITE_P(HashValueAssociativeMapTest, BasicUsage); +using AssociativeMapTypes = testing::Types< + std::map<int, std::string>, std::unordered_map<int, std::string>, + absl::flat_hash_map<int, std::string>, + absl::node_hash_map<int, std::string>, absl::btree_map<int, std::string>, + UnorderedSequence<std::pair<const int, std::string>>>; +INSTANTIATE_TYPED_TEST_SUITE_P(My, HashValueAssociativeMapTest, + AssociativeMapTypes); - using MM = std::multimap<int, std::string>; - EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly(std::make_tuple( - MM{}, MM{{0, "foo"}}, MM{{1, "foo"}}, MM{{0, "bar"}}, MM{{1, "bar"}}, - MM{{0, "foo"}, {0, "bar"}}, MM{{0, "bar"}, {0, "foo"}}, - MM{{0, "foo"}, {42, "bar"}}, MM{{1, "foo"}, {42, "bar"}}, - MM{{1, "foo"}, {1, "foo"}, {43, "bar"}}, MM{{1, "foo"}, {43, "baz"}}))); +template <typename T> +class HashValueAssociativeMultimapTest : public testing::Test {}; +TYPED_TEST_SUITE_P(HashValueAssociativeMultimapTest); + +TYPED_TEST_P(HashValueAssociativeMultimapTest, BasicUsage) { + using MM = TypeParam; + using V = typename MM::value_type; + std::vector<MM> exemplars{MM{}, + MM{V{0, "foo"}}, + MM{V{1, "foo"}}, + MM{V{0, "bar"}}, + MM{V{1, "bar"}}, + MM{V{0, "foo"}, V{0, "bar"}}, + MM{V{0, "bar"}, V{0, "foo"}}, + MM{V{0, "foo"}, V{42, "bar"}}, + MM{V{1, "foo"}, V{42, "bar"}}, + MM{V{1, "foo"}, V{1, "foo"}, V{43, "bar"}}, + MM{V{1, "foo"}, V{43, "bar"}, V{1, "foo"}}, + MM{V{1, "foo"}, V{43, "baz"}}}; + EXPECT_TRUE(absl::VerifyTypeImplementsAbslHashCorrectly(exemplars)); } +REGISTER_TYPED_TEST_SUITE_P(HashValueAssociativeMultimapTest, BasicUsage); +using AssociativeMultimapTypes = + testing::Types<std::multimap<int, std::string>, + std::unordered_multimap<int, std::string>>; +INSTANTIATE_TYPED_TEST_SUITE_P(My, HashValueAssociativeMultimapTest, + AssociativeMultimapTypes); + TEST(HashValueTest, ReferenceWrapper) { EXPECT_TRUE(is_hashable<std::reference_wrapper<Private>>::value); @@ -818,10 +1088,10 @@ TYPED_TEST_P(HashIntTest, BasicUsage) { Hash<CombineVariadic<TypeParam>>()({})); } -REGISTER_TYPED_TEST_CASE_P(HashIntTest, BasicUsage); +REGISTER_TYPED_TEST_SUITE_P(HashIntTest, BasicUsage); using IntTypes = testing::Types<unsigned char, char, int, int32_t, int64_t, uint32_t, uint64_t, size_t>; -INSTANTIATE_TYPED_TEST_CASE_P(My, HashIntTest, IntTypes); +INSTANTIATE_TYPED_TEST_SUITE_P(My, HashIntTest, IntTypes); struct StructWithPadding { char c; @@ -928,29 +1198,23 @@ TEST(HashTest, SmallValueOn64ByteBoundary) { Hash<IntAndString>()(IntAndString{0, std::string(63, '0')}); } -struct TypeErased { - size_t n; - - template <typename H> - friend H AbslHashValue(H hash_state, const TypeErased& v) { - v.HashValue(absl::HashState::Create(&hash_state)); - return hash_state; - } - - void HashValue(absl::HashState state) const { - absl::HashState::combine(std::move(state), n); - } -}; - TEST(HashTest, TypeErased) { - EXPECT_TRUE((is_hashable<TypeErased>::value)); - EXPECT_TRUE((is_hashable<std::pair<TypeErased, int>>::value)); + EXPECT_TRUE((is_hashable<TypeErasedValue<size_t>>::value)); + EXPECT_TRUE((is_hashable<std::pair<TypeErasedValue<size_t>, int>>::value)); - EXPECT_EQ(SpyHash(TypeErased{7}), SpyHash(size_t{7})); - EXPECT_NE(SpyHash(TypeErased{7}), SpyHash(size_t{13})); + EXPECT_EQ(SpyHash(TypeErasedValue<size_t>(7)), SpyHash(size_t{7})); + EXPECT_NE(SpyHash(TypeErasedValue<size_t>(7)), SpyHash(size_t{13})); - EXPECT_EQ(SpyHash(std::make_pair(TypeErased{7}, 17)), + EXPECT_EQ(SpyHash(std::make_pair(TypeErasedValue<size_t>(7), 17)), SpyHash(std::make_pair(size_t{7}, 17))); + + absl::flat_hash_set<absl::flat_hash_set<int>> ss = {{1, 2}, {3, 4}}; + TypeErasedContainer<absl::flat_hash_set<absl::flat_hash_set<int>>> es = { + absl::flat_hash_set<int>{1, 2}, {3, 4}}; + absl::flat_hash_set<TypeErasedContainer<absl::flat_hash_set<int>>> se = { + {1, 2}, {3, 4}}; + EXPECT_EQ(SpyHash(ss), SpyHash(es)); + EXPECT_EQ(SpyHash(ss), SpyHash(se)); } struct ValueWithBoolConversion { @@ -973,4 +1237,39 @@ TEST(HashTest, DoesNotUseImplicitConversionsToBool) { absl::Hash<ValueWithBoolConversion>()(ValueWithBoolConversion{1})); } +TEST(HashOf, MatchesHashForSingleArgument) { + std::string s = "forty two"; + int i = 42; + double d = 42.0; + std::tuple<int, int> t{4, 2}; + + EXPECT_EQ(absl::HashOf(s), absl::Hash<std::string>{}(s)); + EXPECT_EQ(absl::HashOf(i), absl::Hash<int>{}(i)); + EXPECT_EQ(absl::HashOf(d), absl::Hash<double>{}(d)); + EXPECT_EQ(absl::HashOf(t), (absl::Hash<std::tuple<int, int>>{}(t))); +} + +TEST(HashOf, MatchesHashOfTupleForMultipleArguments) { + std::string hello = "hello"; + std::string world = "world"; + + EXPECT_EQ(absl::HashOf(), absl::HashOf(std::make_tuple())); + EXPECT_EQ(absl::HashOf(hello), absl::HashOf(std::make_tuple(hello))); + EXPECT_EQ(absl::HashOf(hello, world), + absl::HashOf(std::make_tuple(hello, world))); +} + +template <typename T> +std::true_type HashOfExplicitParameter(decltype(absl::HashOf<T>(0))) { + return {}; +} +template <typename T> +std::false_type HashOfExplicitParameter(size_t) { + return {}; +} + +TEST(HashOf, CantPassExplicitTemplateParameters) { + EXPECT_FALSE(HashOfExplicitParameter<int>(0)); +} + } // namespace diff --git a/absl/hash/internal/city_test.cc b/absl/hash/internal/city_test.cc index 251d381d..1bbf02e0 100644 --- a/absl/hash/internal/city_test.cc +++ b/absl/hash/internal/city_test.cc @@ -22,6 +22,7 @@ namespace absl { ABSL_NAMESPACE_BEGIN namespace hash_internal { +namespace { static const uint64_t k0 = 0xc3a5c85c97cb3127ULL; static const uint64_t kSeed0 = 1234567; @@ -590,6 +591,7 @@ TEST(CityHashTest, Unchanging) { TestUnchanging(testdata[i], 0, kDataSize); } +} // namespace } // namespace hash_internal ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/hash/internal/hash.cc b/absl/hash/internal/hash.cc index 1433eb9d..11451e57 100644 --- a/absl/hash/internal/hash.cc +++ b/absl/hash/internal/hash.cc @@ -18,13 +18,12 @@ namespace absl { ABSL_NAMESPACE_BEGIN namespace hash_internal { -uint64_t HashState::CombineLargeContiguousImpl32(uint64_t state, - const unsigned char* first, - size_t len) { +uint64_t MixingHashState::CombineLargeContiguousImpl32( + uint64_t state, const unsigned char* first, size_t len) { while (len >= PiecewiseChunkSize()) { - state = - Mix(state, absl::hash_internal::CityHash32(reinterpret_cast<const char*>(first), - PiecewiseChunkSize())); + state = Mix(state, + hash_internal::CityHash32(reinterpret_cast<const char*>(first), + PiecewiseChunkSize())); len -= PiecewiseChunkSize(); first += PiecewiseChunkSize(); } @@ -33,9 +32,8 @@ uint64_t HashState::CombineLargeContiguousImpl32(uint64_t state, std::integral_constant<int, 4>{}); } -uint64_t HashState::CombineLargeContiguousImpl64(uint64_t state, - const unsigned char* first, - size_t len) { +uint64_t MixingHashState::CombineLargeContiguousImpl64( + uint64_t state, const unsigned char* first, size_t len) { while (len >= PiecewiseChunkSize()) { state = Mix(state, Hash64(first, PiecewiseChunkSize())); len -= PiecewiseChunkSize(); @@ -46,23 +44,24 @@ uint64_t HashState::CombineLargeContiguousImpl64(uint64_t state, std::integral_constant<int, 8>{}); } -ABSL_CONST_INIT const void* const HashState::kSeed = &kSeed; +ABSL_CONST_INIT const void* const MixingHashState::kSeed = &kSeed; -// The salt array used by Wyhash. This array is NOT the mechanism used to make -// absl::Hash non-deterministic between program invocations. See `Seed()` for -// that mechanism. +// The salt array used by LowLevelHash. This array is NOT the mechanism used to +// make absl::Hash non-deterministic between program invocations. See `Seed()` +// for that mechanism. // // Any random values are fine. These values are just digits from the decimal // part of pi. // https://en.wikipedia.org/wiki/Nothing-up-my-sleeve_number -constexpr uint64_t kWyhashSalt[5] = { +constexpr uint64_t kHashSalt[5] = { uint64_t{0x243F6A8885A308D3}, uint64_t{0x13198A2E03707344}, uint64_t{0xA4093822299F31D0}, uint64_t{0x082EFA98EC4E6C89}, uint64_t{0x452821E638D01377}, }; -uint64_t HashState::WyhashImpl(const unsigned char* data, size_t len) { - return Wyhash(data, len, Seed(), kWyhashSalt); +uint64_t MixingHashState::LowLevelHashImpl(const unsigned char* data, + size_t len) { + return LowLevelHash(data, len, Seed(), kHashSalt); } } // namespace hash_internal diff --git a/absl/hash/internal/hash.h b/absl/hash/internal/hash.h index 7fb0af0b..45dfdd46 100644 --- a/absl/hash/internal/hash.h +++ b/absl/hash/internal/hash.h @@ -21,7 +21,9 @@ #include <algorithm> #include <array> +#include <bitset> #include <cmath> +#include <cstddef> #include <cstring> #include <deque> #include <forward_list> @@ -35,6 +37,8 @@ #include <string> #include <tuple> #include <type_traits> +#include <unordered_map> +#include <unordered_set> #include <utility> #include <vector> @@ -42,17 +46,20 @@ #include "absl/base/internal/unaligned_access.h" #include "absl/base/port.h" #include "absl/container/fixed_array.h" -#include "absl/hash/internal/wyhash.h" +#include "absl/hash/internal/city.h" +#include "absl/hash/internal/low_level_hash.h" #include "absl/meta/type_traits.h" #include "absl/numeric/int128.h" #include "absl/strings/string_view.h" #include "absl/types/optional.h" #include "absl/types/variant.h" #include "absl/utility/utility.h" -#include "absl/hash/internal/city.h" namespace absl { ABSL_NAMESPACE_BEGIN + +class HashState; + namespace hash_internal { // Internal detail: Large buffers are hashed in smaller chunks. This function @@ -114,24 +121,66 @@ class PiecewiseCombiner { size_t position_; }; +// is_hashable() +// +// Trait class which returns true if T is hashable by the absl::Hash framework. +// Used for the AbslHashValue implementations for composite types below. +template <typename T> +struct is_hashable; + // HashStateBase // -// A hash state object represents an intermediate state in the computation -// of an unspecified hash algorithm. `HashStateBase` provides a CRTP style -// base class for hash state implementations. Developers adding type support -// for `absl::Hash` should not rely on any parts of the state object other than -// the following member functions: +// An internal implementation detail that contains common implementation details +// for all of the "hash state objects" objects generated by Abseil. This is not +// a public API; users should not create classes that inherit from this. +// +// A hash state object is the template argument `H` passed to `AbslHashValue`. +// It represents an intermediate state in the computation of an unspecified hash +// algorithm. `HashStateBase` provides a CRTP style base class for hash state +// implementations. Developers adding type support for `absl::Hash` should not +// rely on any parts of the state object other than the following member +// functions: // // * HashStateBase::combine() // * HashStateBase::combine_contiguous() +// * HashStateBase::combine_unordered() // -// A derived hash state class of type `H` must provide a static member function +// A derived hash state class of type `H` must provide a public member function // with a signature similar to the following: // // `static H combine_contiguous(H state, const unsigned char*, size_t)`. // +// It must also provide a private template method named RunCombineUnordered. +// +// A "consumer" is a 1-arg functor returning void. Its argument is a reference +// to an inner hash state object, and it may be called multiple times. When +// called, the functor consumes the entropy from the provided state object, +// and resets that object to its empty state. +// +// A "combiner" is a stateless 2-arg functor returning void. Its arguments are +// an inner hash state object and an ElementStateConsumer functor. A combiner +// uses the provided inner hash state object to hash each element of the +// container, passing the inner hash state object to the consumer after hashing +// each element. +// +// Given these definitions, a derived hash state class of type H +// must provide a private template method with a signature similar to the +// following: +// +// `template <typename CombinerT>` +// `static H RunCombineUnordered(H outer_state, CombinerT combiner)` +// +// This function is responsible for constructing the inner state object and +// providing a consumer to the combiner. It uses side effects of the consumer +// and combiner to mix the state of each element in an order-independent manner, +// and uses this to return an updated value of `outer_state`. +// +// This inside-out approach generates efficient object code in the normal case, +// but allows us to use stack storage to implement the absl::HashState type +// erasure mechanism (avoiding heap allocations while hashing). +// // `HashStateBase` will provide a complete implementation for a hash state -// object in terms of this method. +// object in terms of these two methods. // // Example: // @@ -140,6 +189,10 @@ class PiecewiseCombiner { // static H combine_contiguous(H state, const unsigned char*, size_t); // using MyHashState::HashStateBase::combine; // using MyHashState::HashStateBase::combine_contiguous; +// using MyHashState::HashStateBase::combine_unordered; +// private: +// template <typename CombinerT> +// static H RunCombineUnordered(H state, CombinerT combiner); // }; template <typename H> class HashStateBase { @@ -180,7 +233,30 @@ class HashStateBase { template <typename T> static H combine_contiguous(H state, const T* data, size_t size); + template <typename I> + static H combine_unordered(H state, I begin, I end); + using AbslInternalPiecewiseCombiner = PiecewiseCombiner; + + template <typename T> + using is_hashable = absl::hash_internal::is_hashable<T>; + + private: + // Common implementation of the iteration step of a "combiner", as described + // above. + template <typename I> + struct CombineUnorderedCallback { + I begin; + I end; + + template <typename InnerH, typename ElementStateConsumer> + void operator()(InnerH inner_state, ElementStateConsumer cb) { + for (; begin != end; ++begin) { + inner_state = H::combine(std::move(inner_state), *begin); + cb(inner_state); + } + } + }; }; // is_uniquely_represented @@ -345,17 +421,43 @@ H AbslHashValue(H hash_state, std::nullptr_t) { return H::combine(std::move(hash_state), static_cast<void*>(nullptr)); } +// AbslHashValue() for hashing pointers-to-member +template <typename H, typename T, typename C> +H AbslHashValue(H hash_state, T C::* ptr) { + auto salient_ptm_size = [](std::size_t n) -> std::size_t { +#if defined(_MSC_VER) + // Pointers-to-member-function on MSVC consist of one pointer plus 0, 1, 2, + // or 3 ints. In 64-bit mode, they are 8-byte aligned and thus can contain + // padding (namely when they have 1 or 3 ints). The value below is a lower + // bound on the number of salient, non-padding bytes that we use for + // hashing. + if (alignof(T C::*) == alignof(int)) { + // No padding when all subobjects have the same size as the total + // alignment. This happens in 32-bit mode. + return n; + } else { + // Padding for 1 int (size 16) or 3 ints (size 24). + // With 2 ints, the size is 16 with no padding, which we pessimize. + return n == 24 ? 20 : n == 16 ? 12 : n; + } +#else + // On other platforms, we assume that pointers-to-members do not have + // padding. +#ifdef __cpp_lib_has_unique_object_representations + static_assert(std::has_unique_object_representations_v<T C::*>); +#endif // __cpp_lib_has_unique_object_representations + return n; +#endif + }; + return H::combine_contiguous(std::move(hash_state), + reinterpret_cast<unsigned char*>(&ptr), + salient_ptm_size(sizeof ptr)); +} + // ----------------------------------------------------------------------------- // AbslHashValue for Composite Types // ----------------------------------------------------------------------------- -// is_hashable() -// -// Trait class which returns true if T is hashable by the absl::Hash framework. -// Used for the AbslHashValue implementations for composite types below. -template <typename T> -struct is_hashable; - // AbslHashValue() for hashing pairs template <typename H, typename T1, typename T2> typename std::enable_if<is_hashable<T1>::value && is_hashable<T2>::value, @@ -379,7 +481,7 @@ template <typename H, typename... Ts> // This SFINAE gets MSVC confused under some conditions. Let's just disable it // for now. H -#else // _MSC_VER +#else // _MSC_VER typename std::enable_if<absl::conjunction<is_hashable<Ts>...>::value, H>::type #endif // _MSC_VER AbslHashValue(H hash_state, const std::tuple<Ts...>& t) { @@ -489,8 +591,9 @@ typename std::enable_if<is_hashable<T>::value, H>::type AbslHashValue( // AbslHashValue for hashing std::vector // -// Do not use this for vector<bool>. It does not have a .data(), and a fallback -// for std::hash<> is most likely faster. +// Do not use this for vector<bool> on platforms that have a working +// implementation of std::hash. It does not have a .data(), and a fallback for +// std::hash<> is most likely faster. template <typename H, typename T, typename Allocator> typename std::enable_if<is_hashable<T>::value && !std::is_same<T, bool>::value, H>::type @@ -500,6 +603,44 @@ AbslHashValue(H hash_state, const std::vector<T, Allocator>& vector) { vector.size()); } +// AbslHashValue special cases for hashing std::vector<bool> + +#if defined(ABSL_IS_BIG_ENDIAN) && \ + (defined(__GLIBCXX__) || defined(__GLIBCPP__)) + +// std::hash in libstdc++ does not work correctly with vector<bool> on Big +// Endian platforms therefore we need to implement a custom AbslHashValue for +// it. More details on the bug: +// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=102531 +template <typename H, typename T, typename Allocator> +typename std::enable_if<is_hashable<T>::value && std::is_same<T, bool>::value, + H>::type +AbslHashValue(H hash_state, const std::vector<T, Allocator>& vector) { + typename H::AbslInternalPiecewiseCombiner combiner; + for (const auto& i : vector) { + unsigned char c = static_cast<unsigned char>(i); + hash_state = combiner.add_buffer(std::move(hash_state), &c, sizeof(c)); + } + return H::combine(combiner.finalize(std::move(hash_state)), vector.size()); +} +#else +// When not working around the libstdc++ bug above, we still have to contend +// with the fact that std::hash<vector<bool>> is often poor quality, hashing +// directly on the internal words and on no other state. On these platforms, +// vector<bool>{1, 1} and vector<bool>{1, 1, 0} hash to the same value. +// +// Mixing in the size (as we do in our other vector<> implementations) on top +// of the library-provided hash implementation avoids this QOI issue. +template <typename H, typename T, typename Allocator> +typename std::enable_if<is_hashable<T>::value && std::is_same<T, bool>::value, + H>::type +AbslHashValue(H hash_state, const std::vector<T, Allocator>& vector) { + return H::combine(std::move(hash_state), + std::hash<std::vector<T, Allocator>>{}(vector), + vector.size()); +} +#endif + // ----------------------------------------------------------------------------- // AbslHashValue for Ordered Associative Containers // ----------------------------------------------------------------------------- @@ -550,6 +691,55 @@ typename std::enable_if<is_hashable<Key>::value, H>::type AbslHashValue( } // ----------------------------------------------------------------------------- +// AbslHashValue for Unordered Associative Containers +// ----------------------------------------------------------------------------- + +// AbslHashValue for hashing std::unordered_set +template <typename H, typename Key, typename Hash, typename KeyEqual, + typename Alloc> +typename std::enable_if<is_hashable<Key>::value, H>::type AbslHashValue( + H hash_state, const std::unordered_set<Key, Hash, KeyEqual, Alloc>& s) { + return H::combine( + H::combine_unordered(std::move(hash_state), s.begin(), s.end()), + s.size()); +} + +// AbslHashValue for hashing std::unordered_multiset +template <typename H, typename Key, typename Hash, typename KeyEqual, + typename Alloc> +typename std::enable_if<is_hashable<Key>::value, H>::type AbslHashValue( + H hash_state, + const std::unordered_multiset<Key, Hash, KeyEqual, Alloc>& s) { + return H::combine( + H::combine_unordered(std::move(hash_state), s.begin(), s.end()), + s.size()); +} + +// AbslHashValue for hashing std::unordered_set +template <typename H, typename Key, typename T, typename Hash, + typename KeyEqual, typename Alloc> +typename std::enable_if<is_hashable<Key>::value && is_hashable<T>::value, + H>::type +AbslHashValue(H hash_state, + const std::unordered_map<Key, T, Hash, KeyEqual, Alloc>& s) { + return H::combine( + H::combine_unordered(std::move(hash_state), s.begin(), s.end()), + s.size()); +} + +// AbslHashValue for hashing std::unordered_multiset +template <typename H, typename Key, typename T, typename Hash, + typename KeyEqual, typename Alloc> +typename std::enable_if<is_hashable<Key>::value && is_hashable<T>::value, + H>::type +AbslHashValue(H hash_state, + const std::unordered_multimap<Key, T, Hash, KeyEqual, Alloc>& s) { + return H::combine( + H::combine_unordered(std::move(hash_state), s.begin(), s.end()), + s.size()); +} + +// ----------------------------------------------------------------------------- // AbslHashValue for Wrapper Types // ----------------------------------------------------------------------------- @@ -592,9 +782,28 @@ AbslHashValue(H hash_state, const absl::variant<T...>& v) { // AbslHashValue for Other Types // ----------------------------------------------------------------------------- -// AbslHashValue for hashing std::bitset is not defined, for the same reason as -// for vector<bool> (see std::vector above): It does not expose the raw bytes, -// and a fallback to std::hash<> is most likely faster. +// AbslHashValue for hashing std::bitset is not defined on Little Endian +// platforms, for the same reason as for vector<bool> (see std::vector above): +// It does not expose the raw bytes, and a fallback to std::hash<> is most +// likely faster. + +#if defined(ABSL_IS_BIG_ENDIAN) && \ + (defined(__GLIBCXX__) || defined(__GLIBCPP__)) +// AbslHashValue for hashing std::bitset +// +// std::hash in libstdc++ does not work correctly with std::bitset on Big Endian +// platforms therefore we need to implement a custom AbslHashValue for it. More +// details on the bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=102531 +template <typename H, size_t N> +H AbslHashValue(H hash_state, const std::bitset<N>& set) { + typename H::AbslInternalPiecewiseCombiner combiner; + for (int i = 0; i < N; i++) { + unsigned char c = static_cast<unsigned char>(set[i]); + hash_state = combiner.add_buffer(std::move(hash_state), &c, sizeof(c)); + } + return H::combine(combiner.finalize(std::move(hash_state)), N); +} +#endif // ----------------------------------------------------------------------------- @@ -714,8 +923,8 @@ template <typename T> struct is_hashable : std::integral_constant<bool, HashSelect::template Apply<T>::value> {}; -// HashState -class ABSL_DLL HashState : public HashStateBase<HashState> { +// MixingHashState +class ABSL_DLL MixingHashState : public HashStateBase<MixingHashState> { // absl::uint128 is not an alias or a thin wrapper around the intrinsic. // We use the intrinsic when available to improve performance. #ifdef ABSL_HAVE_INTRINSIC_INT128 @@ -734,22 +943,23 @@ class ABSL_DLL HashState : public HashStateBase<HashState> { public: // Move only - HashState(HashState&&) = default; - HashState& operator=(HashState&&) = default; + MixingHashState(MixingHashState&&) = default; + MixingHashState& operator=(MixingHashState&&) = default; - // HashState::combine_contiguous() + // MixingHashState::combine_contiguous() // // Fundamental base case for hash recursion: mixes the given range of bytes // into the hash state. - static HashState combine_contiguous(HashState hash_state, - const unsigned char* first, size_t size) { - return HashState( + static MixingHashState combine_contiguous(MixingHashState hash_state, + const unsigned char* first, + size_t size) { + return MixingHashState( CombineContiguousImpl(hash_state.state_, first, size, std::integral_constant<int, sizeof(size_t)>{})); } - using HashState::HashStateBase::combine_contiguous; + using MixingHashState::HashStateBase::combine_contiguous; - // HashState::hash() + // MixingHashState::hash() // // For performance reasons in non-opt mode, we specialize this for // integral types. @@ -761,24 +971,49 @@ class ABSL_DLL HashState : public HashStateBase<HashState> { return static_cast<size_t>(Mix(Seed(), static_cast<uint64_t>(value))); } - // Overload of HashState::hash() + // Overload of MixingHashState::hash() template <typename T, absl::enable_if_t<!IntegralFastPath<T>::value, int> = 0> static size_t hash(const T& value) { - return static_cast<size_t>(combine(HashState{}, value).state_); + return static_cast<size_t>(combine(MixingHashState{}, value).state_); } private: // Invoked only once for a given argument; that plus the fact that this is // move-only ensures that there is only one non-moved-from object. - HashState() : state_(Seed()) {} + MixingHashState() : state_(Seed()) {} + + friend class MixingHashState::HashStateBase; + + template <typename CombinerT> + static MixingHashState RunCombineUnordered(MixingHashState state, + CombinerT combiner) { + uint64_t unordered_state = 0; + combiner(MixingHashState{}, [&](MixingHashState& inner_state) { + // Add the hash state of the element to the running total, but mix the + // carry bit back into the low bit. This in intended to avoid losing + // entropy to overflow, especially when unordered_multisets contain + // multiple copies of the same value. + auto element_state = inner_state.state_; + unordered_state += element_state; + if (unordered_state < element_state) { + ++unordered_state; + } + inner_state = MixingHashState{}; + }); + return MixingHashState::combine(std::move(state), unordered_state); + } + + // Allow the HashState type-erasure implementation to invoke + // RunCombinedUnordered() directly. + friend class absl::HashState; // Workaround for MSVC bug. // We make the type copyable to fix the calling convention, even though we // never actually copy it. Keep it private to not affect the public API of the // type. - HashState(const HashState&) = default; + MixingHashState(const MixingHashState&) = default; - explicit HashState(uint64_t state) : state_(state) {} + explicit MixingHashState(uint64_t state) : state_(state) {} // Implementation of the base case for combine_contiguous where we actually // mix the bytes into the state. @@ -793,7 +1028,6 @@ class ABSL_DLL HashState : public HashStateBase<HashState> { std::integral_constant<int, 8> /* sizeof_size_t */); - // Slow dispatch path for calls to CombineContiguousImpl with a size argument // larger than PiecewiseChunkSize(). Has the same effect as calling // CombineContiguousImpl() repeatedly with the chunk stride size. @@ -856,6 +1090,8 @@ class ABSL_DLL HashState : public HashStateBase<HashState> { } ABSL_ATTRIBUTE_ALWAYS_INLINE static uint64_t Mix(uint64_t state, uint64_t v) { + // Though the 128-bit product on AArch64 needs two instructions, it is + // still a good balance between speed and hash quality. using MultType = absl::conditional_t<sizeof(size_t) == 4, uint64_t, uint128>; // We do the addition in 64-bit space to make sure the 128-bit @@ -867,16 +1103,16 @@ class ABSL_DLL HashState : public HashStateBase<HashState> { return static_cast<uint64_t>(m ^ (m >> (sizeof(m) * 8 / 2))); } - // An extern to avoid bloat on a direct call to Wyhash() with fixed values for - // both the seed and salt parameters. - static uint64_t WyhashImpl(const unsigned char* data, size_t len); + // An extern to avoid bloat on a direct call to LowLevelHash() with fixed + // values for both the seed and salt parameters. + static uint64_t LowLevelHashImpl(const unsigned char* data, size_t len); ABSL_ATTRIBUTE_ALWAYS_INLINE static uint64_t Hash64(const unsigned char* data, size_t len) { #ifdef ABSL_HAVE_INTRINSIC_INT128 - return WyhashImpl(data, len); + return LowLevelHashImpl(data, len); #else - return absl::hash_internal::CityHash64(reinterpret_cast<const char*>(data), len); + return hash_internal::CityHash64(reinterpret_cast<const char*>(data), len); #endif } @@ -911,8 +1147,8 @@ class ABSL_DLL HashState : public HashStateBase<HashState> { uint64_t state_; }; -// HashState::CombineContiguousImpl() -inline uint64_t HashState::CombineContiguousImpl( +// MixingHashState::CombineContiguousImpl() +inline uint64_t MixingHashState::CombineContiguousImpl( uint64_t state, const unsigned char* first, size_t len, std::integral_constant<int, 4> /* sizeof_size_t */) { // For large values we use CityHash, for small ones we just use a @@ -922,7 +1158,7 @@ inline uint64_t HashState::CombineContiguousImpl( if (ABSL_PREDICT_FALSE(len > PiecewiseChunkSize())) { return CombineLargeContiguousImpl32(state, first, len); } - v = absl::hash_internal::CityHash32(reinterpret_cast<const char*>(first), len); + v = hash_internal::CityHash32(reinterpret_cast<const char*>(first), len); } else if (len >= 4) { v = Read4To8(first, len); } else if (len > 0) { @@ -934,12 +1170,12 @@ inline uint64_t HashState::CombineContiguousImpl( return Mix(state, v); } -// Overload of HashState::CombineContiguousImpl() -inline uint64_t HashState::CombineContiguousImpl( +// Overload of MixingHashState::CombineContiguousImpl() +inline uint64_t MixingHashState::CombineContiguousImpl( uint64_t state, const unsigned char* first, size_t len, std::integral_constant<int, 8> /* sizeof_size_t */) { - // For large values we use Wyhash or CityHash depending on the platform, for - // small ones we just use a multiplicative hash. + // For large values we use LowLevelHash or CityHash depending on the platform, + // for small ones we just use a multiplicative hash. uint64_t v; if (len > 16) { if (ABSL_PREDICT_FALSE(len > PiecewiseChunkSize())) { @@ -976,7 +1212,9 @@ struct PoisonedHash : private AggregateBarrier { template <typename T> struct HashImpl { - size_t operator()(const T& value) const { return HashState::hash(value); } + size_t operator()(const T& value) const { + return MixingHashState::hash(value); + } }; template <typename T> @@ -998,6 +1236,14 @@ H HashStateBase<H>::combine_contiguous(H state, const T* data, size_t size) { return hash_internal::hash_range_or_bytes(std::move(state), data, size); } +// HashStateBase::combine_unordered() +template <typename H> +template <typename I> +H HashStateBase<H>::combine_unordered(H state, I begin, I end) { + return H::RunCombineUnordered(std::move(state), + CombineUnorderedCallback<I>{begin, end}); +} + // HashStateBase::PiecewiseCombiner::add_buffer() template <typename H> H PiecewiseCombiner::add_buffer(H state, const unsigned char* data, diff --git a/absl/hash/internal/wyhash.cc b/absl/hash/internal/low_level_hash.cc index 642bde43..6f9cb9c7 100644 --- a/absl/hash/internal/wyhash.cc +++ b/absl/hash/internal/low_level_hash.cc @@ -12,23 +12,35 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "absl/hash/internal/wyhash.h" +#include "absl/hash/internal/low_level_hash.h" #include "absl/base/internal/unaligned_access.h" +#include "absl/numeric/bits.h" #include "absl/numeric/int128.h" namespace absl { ABSL_NAMESPACE_BEGIN namespace hash_internal { -static uint64_t WyhashMix(uint64_t v0, uint64_t v1) { +static uint64_t Mix(uint64_t v0, uint64_t v1) { +#if !defined(__aarch64__) + // The default bit-mixer uses 64x64->128-bit multiplication. absl::uint128 p = v0; p *= v1; return absl::Uint128Low64(p) ^ absl::Uint128High64(p); +#else + // The default bit-mixer above would perform poorly on some ARM microarchs, + // where calculating a 128-bit product requires a sequence of two + // instructions with a high combined latency and poor throughput. + // Instead, we mix bits using only 64-bit arithmetic, which is faster. + uint64_t p = v0 ^ absl::rotl(v1, 40); + p *= v1 ^ absl::rotl(v0, 39); + return p ^ (p >> 11); +#endif } -uint64_t Wyhash(const void* data, size_t len, uint64_t seed, - const uint64_t salt[]) { +uint64_t LowLevelHash(const void* data, size_t len, uint64_t seed, + const uint64_t salt[]) { const uint8_t* ptr = static_cast<const uint8_t*>(data); uint64_t starting_length = static_cast<uint64_t>(len); uint64_t current_state = seed ^ salt[0]; @@ -49,12 +61,12 @@ uint64_t Wyhash(const void* data, size_t len, uint64_t seed, uint64_t g = absl::base_internal::UnalignedLoad64(ptr + 48); uint64_t h = absl::base_internal::UnalignedLoad64(ptr + 56); - uint64_t cs0 = WyhashMix(a ^ salt[1], b ^ current_state); - uint64_t cs1 = WyhashMix(c ^ salt[2], d ^ current_state); + uint64_t cs0 = Mix(a ^ salt[1], b ^ current_state); + uint64_t cs1 = Mix(c ^ salt[2], d ^ current_state); current_state = (cs0 ^ cs1); - uint64_t ds0 = WyhashMix(e ^ salt[3], f ^ duplicated_state); - uint64_t ds1 = WyhashMix(g ^ salt[4], h ^ duplicated_state); + uint64_t ds0 = Mix(e ^ salt[3], f ^ duplicated_state); + uint64_t ds1 = Mix(g ^ salt[4], h ^ duplicated_state); duplicated_state = (ds0 ^ ds1); ptr += 64; @@ -70,7 +82,7 @@ uint64_t Wyhash(const void* data, size_t len, uint64_t seed, uint64_t a = absl::base_internal::UnalignedLoad64(ptr); uint64_t b = absl::base_internal::UnalignedLoad64(ptr + 8); - current_state = WyhashMix(a ^ salt[1], b ^ current_state); + current_state = Mix(a ^ salt[1], b ^ current_state); ptr += 16; len -= 16; @@ -101,9 +113,9 @@ uint64_t Wyhash(const void* data, size_t len, uint64_t seed, b = 0; } - uint64_t w = WyhashMix(a ^ salt[1], b ^ current_state); + uint64_t w = Mix(a ^ salt[1], b ^ current_state); uint64_t z = salt[1] ^ starting_length; - return WyhashMix(w, z); + return Mix(w, z); } } // namespace hash_internal diff --git a/absl/hash/internal/wyhash.h b/absl/hash/internal/low_level_hash.h index 4aff4e93..439968aa 100644 --- a/absl/hash/internal/wyhash.h +++ b/absl/hash/internal/low_level_hash.h @@ -12,16 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. // -// This file provides the Google-internal implementation of the Wyhash -// algorithm. +// This file provides the Google-internal implementation of LowLevelHash. // -// Wyhash is a fast hash function for hash tables, the fastest we've currently -// (late 2020) found that passes the SMHasher tests. The algorithm relies on -// intrinsic 128-bit multiplication for speed. This is not meant to be secure - -// just fast. +// LowLevelHash is a fast hash function for hash tables, the fastest we've +// currently (late 2020) found that passes the SMHasher tests. The algorithm +// relies on intrinsic 128-bit multiplication for speed. This is not meant to be +// secure - just fast. +// +// It is closely based on a version of wyhash, but does not maintain or +// guarantee future compatibility with it. -#ifndef ABSL_HASH_INTERNAL_WYHASH_H_ -#define ABSL_HASH_INTERNAL_WYHASH_H_ +#ifndef ABSL_HASH_INTERNAL_LOW_LEVEL_HASH_H_ +#define ABSL_HASH_INTERNAL_LOW_LEVEL_HASH_H_ #include <stdint.h> #include <stdlib.h> @@ -36,13 +38,13 @@ namespace hash_internal { // integers are hashed into the result. // // To allow all hashable types (including string_view and Span) to depend on -// this algoritm, we keep the API low-level, with as few dependencies as +// this algorithm, we keep the API low-level, with as few dependencies as // possible. -uint64_t Wyhash(const void* data, size_t len, uint64_t seed, - const uint64_t salt[5]); +uint64_t LowLevelHash(const void* data, size_t len, uint64_t seed, + const uint64_t salt[5]); } // namespace hash_internal ABSL_NAMESPACE_END } // namespace absl -#endif // ABSL_HASH_INTERNAL_WYHASH_H_ +#endif // ABSL_HASH_INTERNAL_LOW_LEVEL_HASH_H_ diff --git a/absl/hash/internal/low_level_hash_test.cc b/absl/hash/internal/low_level_hash_test.cc new file mode 100644 index 00000000..ae930b34 --- /dev/null +++ b/absl/hash/internal/low_level_hash_test.cc @@ -0,0 +1,580 @@ +// Copyright 2020 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/hash/internal/low_level_hash.h" + +#include <cinttypes> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/strings/escaping.h" + +#define UPDATE_GOLDEN 0 + +namespace { + +static const uint64_t kSalt[5] = {0xa0761d6478bd642f, 0xe7037ed1a0b428dbl, + 0x8ebc6af09c88c6e3, 0x589965cc75374cc3l, + 0x1d8e4e27c47d124f}; + +TEST(LowLevelHashTest, VerifyGolden) { + constexpr size_t kNumGoldenOutputs = 134; + static struct { + absl::string_view base64_data; + uint64_t seed; + } cases[] = { + {"", uint64_t{0xec42b7ab404b8acb}}, + {"ICAg", uint64_t{0}}, + {"YWFhYQ==", uint64_t{0}}, + {"AQID", uint64_t{0}}, + {"AQIDBA==", uint64_t{0}}, + {"dGhpcmRfcGFydHl8d3loYXNofDY0", uint64_t{0}}, + {"Zw==", uint64_t{0xeeee074043a3ee0f}}, + {"xmk=", uint64_t{0x857902089c393de}}, + {"c1H/", uint64_t{0x993df040024ca3af}}, + {"SuwpzQ==", uint64_t{0xc4e4c2acea740e96}}, + {"uqvy++M=", uint64_t{0x6a214b3db872d0cf}}, + {"RnzCVPgb", uint64_t{0x44343db6a89dba4d}}, + {"6OeNdlouYw==", uint64_t{0x77b5d6d1ae1dd483}}, + {"M5/JmmYyDbc=", uint64_t{0x89ab8ecb44d221f1}}, + {"MVijWiVdBRdY", uint64_t{0x60244b17577ca81b}}, + {"6V7Uq7LNxpu0VA==", uint64_t{0x59a08dcee0717067}}, + {"EQ6CdEEhPdyHcOk=", uint64_t{0xf5f20db3ade57396}}, + {"PqFB4fxnPgF+l+rc", uint64_t{0xbf8dee0751ad3efb}}, + {"a5aPOFwq7LA7+zKvPA==", uint64_t{0x6b7a06b268d63e30}}, + {"VOwY21wCGv5D+/qqOvs=", uint64_t{0xb8c37f0ae0f54c82}}, + {"KdHmBTx8lHXYvmGJ+Vy7", uint64_t{0x9fcbed0c38e50eef}}, + {"qJkPlbHr8bMF7/cA6aE65Q==", uint64_t{0x2af4bade1d8e3a1d}}, + {"ygvL0EhHZL0fIx6oHHtkxRQ=", uint64_t{0x714e3aa912da2f2c}}, + {"c1rFXkt5YztwZCQRngncqtSs", uint64_t{0xf5ee75e3cbb82c1c}}, + {"8hsQrzszzeNQSEcVXLtvIhm6mw==", uint64_t{0x620e7007321b93b9}}, + {"ffUL4RocfyP4KfikGxO1yk7omDI=", uint64_t{0xc08528cac2e551fc}}, + {"OOB5TT00vF9Od/rLbAWshiErqhpV", uint64_t{0x6a1debf9cc3ad39}}, + {"or5wtXM7BFzTNpSzr+Lw5J5PMhVJ/Q==", uint64_t{0x7e0a3c88111fc226}}, + {"gk6pCHDUsoopVEiaCrzVDhioRKxb844=", uint64_t{0x1301fef15df39edb}}, + {"TNctmwlC5QbEM6/No4R/La3UdkfeMhzs", uint64_t{0x64e181f3d5817ab}}, + {"SsQw9iAjhWz7sgcE9OwLuSC6hsM+BfHs2Q==", uint64_t{0xafafc44961078ecb}}, + {"ZzO3mVCj4xTT2TT3XqDyEKj2BZQBvrS8RHg=", uint64_t{0x4f7bb45549250094}}, + {"+klp5iPQGtppan5MflEls0iEUzqU+zGZkDJX", uint64_t{0xa30061abaa2818c}}, + {"RO6bvOnlJc8I9eniXlNgqtKy0IX6VNg16NRmgg==", + uint64_t{0xd902ee3e44a5705f}}, + {"ZJjZqId1ZXBaij9igClE3nyliU5XWdNRrayGlYA=", uint64_t{0x316d36da516f583}}, + {"7BfkhfGMDGbxfMB8uyL85GbaYQtjr2K8g7RpLzr/", + uint64_t{0x402d83f9f834f616}}, + {"rycWk6wHH7htETQtje9PidS2YzXBx+Qkg2fY7ZYS7A==", + uint64_t{0x9c604164c016b72c}}, + {"RTkC2OUK+J13CdGllsH0H5WqgspsSa6QzRZouqx6pvI=", + uint64_t{0x3f4507e01f9e73ba}}, + {"tKjKmbLCNyrLCM9hycOAXm4DKNpM12oZ7dLTmUx5iwAi", + uint64_t{0xc3fe0d5be8d2c7c7}}, + {"VprUGNH+5NnNRaORxgH/ySrZFQFDL+4VAodhfBNinmn8cg==", + uint64_t{0x531858a40bfa7ea1}}, + {"gc1xZaY+q0nPcUvOOnWnT3bqfmT/geth/f7Dm2e/DemMfk4=", + uint64_t{0x86689478a7a7e8fa}}, + {"Mr35fIxqx1ukPAL0su1yFuzzAU3wABCLZ8+ZUFsXn47UmAph", + uint64_t{0x4ec948b8e7f27288}}, + {"A9G8pw2+m7+rDtWYAdbl8tb2fT7FFo4hLi2vAsa5Y8mKH3CX3g==", + uint64_t{0xce46c7213c10032}}, + {"DFaJGishGwEHDdj9ixbCoaTjz9KS0phLNWHVVdFsM93CvPft3hM=", + uint64_t{0xf63e96ee6f32a8b6}}, + {"7+Ugx+Kr3aRNgYgcUxru62YkTDt5Hqis+2po81hGBkcrJg4N0uuy", + uint64_t{0x1cfe85e65fc5225}}, + {"H2w6O8BUKqu6Tvj2xxaecxEI2wRgIgqnTTG1WwOgDSINR13Nm4d4Vg==", + uint64_t{0x45c474f1cee1d2e8}}, + {"1XBMnIbqD5jy65xTDaf6WtiwtdtQwv1dCVoqpeKj+7cTR1SaMWMyI04=", + uint64_t{0x6e024e14015f329c}}, + {"znZbdXG2TSFrKHEuJc83gPncYpzXGbAebUpP0XxzH0rpe8BaMQ17nDbt", + uint64_t{0x760c40502103ae1c}}, + {"ylu8Atu13j1StlcC1MRMJJXIl7USgDDS22HgVv0WQ8hx/8pNtaiKB17hCQ==", + uint64_t{0x17fd05c3c560c320}}, + {"M6ZVVzsd7vAvbiACSYHioH/440dp4xG2mLlBnxgiqEvI/aIEGpD0Sf4VS0g=", + uint64_t{0x8b34200a6f8e90d9}}, + {"li3oFSXLXI+ubUVGJ4blP6mNinGKLHWkvGruun85AhVn6iuMtocbZPVhqxzn", + uint64_t{0x6be89e50818bdf69}}, + {"kFuQHuUCqBF3Tc3hO4dgdIp223ShaCoog48d5Do5zMqUXOh5XpGK1t5XtxnfGA==", + uint64_t{0xfb389773315b47d8}}, + {"jWmOad0v0QhXVJd1OdGuBZtDYYS8wBVHlvOeTQx9ZZnm8wLEItPMeihj72E0nWY=", + uint64_t{0x4f2512a23f61efee}}, + {"z+DHU52HaOQdW4JrZwDQAebEA6rm13Zg/9lPYA3txt3NjTBqFZlOMvTRnVzRbl23", + uint64_t{0x59ccd92fc16c6fda}}, + {"MmBiGDfYeTayyJa/tVycg+rN7f9mPDFaDc+23j0TlW9094er0ADigsl4QX7V3gG/qw==", + uint64_t{0x25c5a7f5bd330919}}, + {"774RK+9rOL4iFvs1q2qpo/JVc/I39buvNjqEFDtDvyoB0FXxPI2vXqOrk08VPfIHkmU=", + uint64_t{0x51df4174d34c97d7}}, + {"+slatXiQ7/2lK0BkVUI1qzNxOOLP3I1iK6OfHaoxgqT63FpzbElwEXSwdsryq3UlHK0I", + uint64_t{0x80ce6d76f89cb57}}, + {"64mVTbQ47dHjHlOHGS/hjJwr/" + "K2frCNpn87exOqMzNUVYiPKmhCbfS7vBUce5tO6Ec9osQ==", + uint64_t{0x20961c911965f684}}, + {"fIsaG1r530SFrBqaDj1kqE0AJnvvK8MNEZbII2Yw1OK77v0V59xabIh0B5axaz/" + "+a2V5WpA=", + uint64_t{0x4e5b926ec83868e7}}, + {"PGih0zDEOWCYGxuHGDFu9Ivbff/" + "iE7BNUq65tycTR2R76TerrXALRosnzaNYO5fjFhTi+CiS", + uint64_t{0x3927b30b922eecef}}, + {"RnpA/" + "zJnEnnLjmICORByRVb9bCOgxF44p3VMiW10G7PvW7IhwsWajlP9kIwNA9FjAD2GoQHk2Q=" + "=", + uint64_t{0xbd0291284a49b61c}}, + {"qFklMceaTHqJpy2qavJE+EVBiNFOi6OxjOA3LeIcBop1K7w8xQi3TrDk+" + "BrWPRIbfprszSaPfrI=", + uint64_t{0x73a77c575bcc956}}, + {"cLbfUtLl3EcQmITWoTskUR8da/VafRDYF/ylPYwk7/" + "zazk6ssyrzxMN3mmSyvrXR2yDGNZ3WDrTT", + uint64_t{0x766a0e2ade6d09a6}}, + {"s/" + "Jf1+" + "FbsbCpXWPTUSeWyMH6e4CvTFvPE5Fs6Z8hvFITGyr0dtukHzkI84oviVLxhM1xMxrMAy1db" + "w==", + uint64_t{0x2599f4f905115869}}, + {"FvyQ00+j7nmYZVQ8hI1Edxd0AWplhTfWuFGiu34AK5X8u2hLX1bE97sZM0CmeLe+" + "7LgoUT1fJ/axybE=", + uint64_t{0xd8256e5444d21e53}}, + {"L8ncxMaYLBH3g9buPu8hfpWZNlOF7nvWLNv9IozH07uQsIBWSKxoPy8+" + "LW4tTuzC6CIWbRGRRD1sQV/4", + uint64_t{0xf664a91333fb8dfd}}, + {"CDK0meI07yrgV2kQlZZ+" + "wuVqhc2NmzqeLH7bmcA6kchsRWFPeVF5Wqjjaj556ABeUoUr3yBmfU3kWOakkg==", + uint64_t{0x9625b859be372cd1}}, + {"d23/vc5ONh/" + "HkMiq+gYk4gaCNYyuFKwUkvn46t+dfVcKfBTYykr4kdvAPNXGYLjM4u1YkAEFpJP+" + "nX7eOvs=", + uint64_t{0x7b99940782e29898}}, + {"NUR3SRxBkxTSbtQORJpu/GdR6b/h6sSGfsMj/KFd99ahbh+9r7LSgSGmkGVB/" + "mGoT0pnMTQst7Lv2q6QN6Vm", + uint64_t{0x4fe12fa5383b51a8}}, + {"2BOFlcI3Z0RYDtS9T9Ie9yJoXlOdigpPeeT+CRujb/" + "O39Ih5LPC9hP6RQk1kYESGyaLZZi3jtabHs7DiVx/VDg==", + uint64_t{0xe2ccb09ac0f5b4b6}}, + {"FF2HQE1FxEvWBpg6Z9zAMH+Zlqx8S1JD/" + "wIlViL6ZDZY63alMDrxB0GJQahmAtjlm26RGLnjW7jmgQ4Ie3I+014=", + uint64_t{0x7d0a37adbd7b753b}}, + {"tHmO7mqVL/PX11nZrz50Hc+M17Poj5lpnqHkEN+4bpMx/" + "YGbkrGOaYjoQjgmt1X2QyypK7xClFrjeWrCMdlVYtbW", + uint64_t{0xd3ae96ef9f7185f2}}, + {"/WiHi9IQcxRImsudkA/KOTqGe8/" + "gXkhKIHkjddv5S9hi02M049dIK3EUyAEjkjpdGLUs+BN0QzPtZqjIYPOgwsYE9g==", + uint64_t{0x4fb88ea63f79a0d8}}, + {"qds+1ExSnU11L4fTSDz/QE90g4Jh6ioqSh3KDOTOAo2pQGL1k/" + "9CCC7J23YF27dUTzrWsCQA2m4epXoCc3yPHb3xElA=", + uint64_t{0xed564e259bb5ebe9}}, + {"8FVYHx40lSQPTHheh08Oq0/" + "pGm2OlG8BEf8ezvAxHuGGdgCkqpXIueJBF2mQJhTfDy5NncO8ntS7vaKs7sCNdDaNGOEi", + uint64_t{0x3e3256b60c428000}}, + {"4ZoEIrJtstiCkeew3oRzmyJHVt/pAs2pj0HgHFrBPztbQ10NsQ/" + "lM6DM439QVxpznnBSiHMgMQJhER+70l72LqFTO1JiIQ==", + uint64_t{0xfb05bad59ec8705}}, + {"hQPtaYI+wJyxXgwD5n8jGIKFKaFA/" + "P83KqCKZfPthnjwdOFysqEOYwAaZuaaiv4cDyi9TyS8hk5cEbNP/jrI7q6pYGBLbsM=", + uint64_t{0xafdc251dbf97b5f8}}, + {"S4gpMSKzMD7CWPsSfLeYyhSpfWOntyuVZdX1xSBjiGvsspwOZcxNKCRIOqAA0moUfOh3I5+" + "juQV4rsqYElMD/gWfDGpsWZKQ", + uint64_t{0x10ec9c92ddb5dcbc}}, + {"oswxop+" + "bthuDLT4j0PcoSKby4LhF47ZKg8K17xxHf74UsGCzTBbOz0MM8hQEGlyqDT1iUiAYnaPaUp" + "L2mRK0rcIUYA4qLt5uOw==", + uint64_t{0x9a767d5822c7dac4}}, + {"0II/" + "697p+" + "BtLSjxj5989OXI004TogEb94VUnDzOVSgMXie72cuYRvTFNIBgtXlKfkiUjeqVpd4a+" + "n5bxNOD1TGrjQtzKU5r7obo=", + uint64_t{0xee46254080d6e2db}}, + {"E84YZW2qipAlMPmctrg7TKlwLZ68l4L+c0xRDUfyyFrA4MAti0q9sHq3TDFviH0Y+" + "Kq3tEE5srWFA8LM9oomtmvm5PYxoaarWPLc", + uint64_t{0xbbb669588d8bf398}}, + {"x3pa4HIElyZG0Nj7Vdy9IdJIR4izLmypXw5PCmZB5y68QQ4uRaVVi3UthsoJROvbjDJkP2D" + "Q6L/eN8pFeLFzNPKBYzcmuMOb5Ull7w==", + uint64_t{0xdc2afaa529beef44}}, + {"jVDKGYIuWOP/" + "QKLdd2wi8B2VJA8Wh0c8PwrXJVM8FOGM3voPDVPyDJOU6QsBDPseoR8uuKd19OZ/" + "zAvSCB+zlf6upAsBlheUKgCfKww=", + uint64_t{0xf1f67391d45013a8}}, + {"mkquunhmYe1aR2wmUz4vcvLEcKBoe6H+kjUok9VUn2+eTSkWs4oDDtJvNCWtY5efJwg/" + "j4PgjRYWtqnrCkhaqJaEvkkOwVfgMIwF3e+d", + uint64_t{0x16fce2b8c65a3429}}, + {"fRelvKYonTQ+s+rnnvQw+JzGfFoPixtna0vzcSjiDqX5s2Kg2//" + "UGrK+AVCyMUhO98WoB1DDbrsOYSw2QzrcPe0+3ck9sePvb+Q/IRaHbw==", + uint64_t{0xf4b096699f49fe67}}, + {"DUwXFJzagljo44QeJ7/" + "6ZKw4QXV18lhkYT2jglMr8WB3CHUU4vdsytvw6AKv42ZcG6fRkZkq9fpnmXy6xG0aO3WPT1" + "eHuyFirAlkW+zKtwg=", + uint64_t{0xca584c4bc8198682}}, + {"cYmZCrOOBBongNTr7e4nYn52uQUy2mfe48s50JXx2AZ6cRAt/" + "xRHJ5QbEoEJOeOHsJyM4nbzwFm++SlT6gFZZHJpkXJ92JkR86uS/eV1hJUR", + uint64_t{0xed269fc3818b6aad}}, + {"EXeHBDfhwzAKFhsMcH9+2RHwV+mJaN01+9oacF6vgm8mCXRd6jeN9U2oAb0of5c5cO4i+" + "Vb/LlHZSMI490SnHU0bejhSCC2gsC5d2K30ER3iNA==", + uint64_t{0x33f253cbb8fe66a8}}, + {"FzkzRYoNjkxFhZDso94IHRZaJUP61nFYrh5MwDwv9FNoJ5jyNCY/" + "eazPZk+tbmzDyJIGw2h3GxaWZ9bSlsol/vK98SbkMKCQ/wbfrXRLcDzdd/8=", + uint64_t{0xd0b76b2c1523d99c}}, + {"Re4aXISCMlYY/XsX7zkIFR04ta03u4zkL9dVbLXMa/q6hlY/CImVIIYRN3VKP4pnd0AUr/" + "ugkyt36JcstAInb4h9rpAGQ7GMVOgBniiMBZ/MGU7H", + uint64_t{0xfd28f0811a2a237f}}, + {"ueLyMcqJXX+MhO4UApylCN9WlTQ+" + "ltJmItgG7vFUtqs2qNwBMjmAvr5u0sAKd8jpzV0dDPTwchbIeAW5zbtkA2NABJV6hFM48ib" + "4/J3A5mseA3cS8w==", + uint64_t{0x6261fb136482e84}}, + {"6Si7Yi11L+jZMkwaN+GUuzXMrlvEqviEkGOilNq0h8TdQyYKuFXzkYc/" + "q74gP3pVCyiwz9KpVGMM9vfnq36riMHRknkmhQutxLZs5fbmOgEO69HglCU=", + uint64_t{0x458efc750bca7c3a}}, + {"Q6AbOofGuTJOegPh9Clm/" + "9crtUMQqylKrTc1fhfJo1tqvpXxhU4k08kntL1RG7woRnFrVh2UoMrL1kjin+s9CanT+" + "y4hHwLqRranl9FjvxfVKm3yvg68", + uint64_t{0xa7e69ff84e5e7c27}}, + {"ieQEbIPvqY2YfIjHnqfJiO1/MIVRk0RoaG/WWi3kFrfIGiNLCczYoklgaecHMm/" + "1sZ96AjO+a5stQfZbJQwS7Sc1ODABEdJKcTsxeW2hbh9A6CFzpowP1A==", + uint64_t{0x3c59bfd0c29efe9e}}, + {"zQUv8hFB3zh2GGl3KTvCmnfzE+" + "SUgQPVaSVIELFX5H9cE3FuVFGmymkPQZJLAyzC90Cmi8GqYCvPqTuAAB//" + "XTJxy4bCcVArgZG9zJXpjowpNBfr3ngWrSE=", + uint64_t{0x10befacc6afd298d}}, + {"US4hcC1+op5JKGC7eIs8CUgInjKWKlvKQkapulxW262E/" + "B2ye79QxOexf188u2mFwwe3WTISJHRZzS61IwljqAWAWoBAqkUnW8SHmIDwHUP31J0p5sGd" + "P47L", + uint64_t{0x41d5320b0a38efa7}}, + {"9bHUWFna2LNaGF6fQLlkx1Hkt24nrkLE2CmFdWgTQV3FFbUe747SSqYw6ebpTa07MWSpWRP" + "sHesVo2B9tqHbe7eQmqYebPDFnNqrhSdZwFm9arLQVs+7a3Ic6A==", + uint64_t{0x58db1c7450fe17f3}}, + {"Kb3DpHRUPhtyqgs3RuXjzA08jGb59hjKTOeFt1qhoINfYyfTt2buKhD6YVffRCPsgK9SeqZ" + "qRPJSyaqsa0ovyq1WnWW8jI/NhvAkZTVHUrX2pC+cD3OPYT05Dag=", + uint64_t{0x6098c055a335b7a6}}, + {"gzxyMJIPlU+bJBwhFUCHSofZ/" + "319LxqMoqnt3+L6h2U2+ZXJCSsYpE80xmR0Ta77Jq54o92SMH87HV8dGOaCTuAYF+" + "lDL42SY1P316Cl0sZTS2ow3ZqwGbcPNs/1", + uint64_t{0x1bbacec67845a801}}, + {"uR7V0TW+FGVMpsifnaBAQ3IGlr1wx5sKd7TChuqRe6OvUXTlD4hKWy8S+" + "8yyOw8lQabism19vOQxfmocEOW/" + "vzY0pEa87qHrAZy4s9fH2Bltu8vaOIe+agYohhYORQ==", + uint64_t{0xc419cfc7442190}}, + {"1UR5eoo2aCwhacjZHaCh9bkOsITp6QunUxHQ2SfeHv0imHetzt/" + "Z70mhyWZBalv6eAx+YfWKCUib2SHDtz/" + "A2dc3hqUWX5VfAV7FQsghPUAtu6IiRatq4YSLpDvKZBQ=", + uint64_t{0xc95e510d94ba270c}}, + {"opubR7H63BH7OtY+Avd7QyQ25UZ8kLBdFDsBTwZlY6gA/" + "u+x+" + "czC9AaZMgmQrUy15DH7YMGsvdXnviTtI4eVI4aF1H9Rl3NXMKZgwFOsdTfdcZeeHVRzBBKX" + "8jUfh1il", + uint64_t{0xff1ae05c98089c3f}}, + {"DC0kXcSXtfQ9FbSRwirIn5tgPri0sbzHSa78aDZVDUKCMaBGyFU6BmrulywYX8yzvwprdLs" + "oOwTWN2wMjHlPDqrvVHNEjnmufRDblW+nSS+xtKNs3N5xsxXdv6JXDrAB/Q==", + uint64_t{0x90c02b8dceced493}}, + {"BXRBk+3wEP3Lpm1y75wjoz+PgB0AMzLe8tQ1AYU2/" + "oqrQB2YMC6W+9QDbcOfkGbeH+b7IBkt/" + "gwCMw2HaQsRFEsurXtcQ3YwRuPz5XNaw5NAvrNa67Fm7eRzdE1+hWLKtA8=", + uint64_t{0x9f8a76697ab1aa36}}, + {"RRBSvEGYnzR9E45Aps/+WSnpCo/X7gJLO4DRnUqFrJCV/kzWlusLE/" + "6ZU6RoUf2ROwcgEvUiXTGjLs7ts3t9SXnJHxC1KiOzxHdYLMhVvgNd3hVSAXODpKFSkVXND" + "55G2L1W", + uint64_t{0x6ba1bf3d811a531d}}, + {"jeh6Qazxmdi57pa9S3XSnnZFIRrnc6s8QLrah5OX3SB/V2ErSPoEAumavzQPkdKF1/" + "SfvmdL+qgF1C+Yawy562QaFqwVGq7+tW0yxP8FStb56ZRgNI4IOmI30s1Ei7iops9Uuw==", + uint64_t{0x6a418974109c67b4}}, + {"6QO5nnDrY2/" + "wrUXpltlKy2dSBcmK15fOY092CR7KxAjNfaY+" + "aAmtWbbzQk3MjBg03x39afSUN1fkrWACdyQKRaGxgwq6MGNxI6W+8DLWJBHzIXrntrE/" + "ml6fnNXEpxplWJ1vEs4=", + uint64_t{0x8472f1c2b3d230a3}}, + {"0oPxeEHhqhcFuwonNfLd5jF3RNATGZS6NPoS0WklnzyokbTqcl4BeBkMn07+fDQv83j/" + "BpGUwcWO05f3+DYzocfnizpFjLJemFGsls3gxcBYxcbqWYev51tG3lN9EvRE+X9+Pwww", + uint64_t{0x5e06068f884e73a7}}, + {"naSBSjtOKgAOg8XVbR5cHAW3Y+QL4Pb/JO9/" + "oy6L08wvVRZqo0BrssMwhzBP401Um7A4ppAupbQeJFdMrysY34AuSSNvtNUy5VxjNECwiNt" + "gwYHw7yakDUv8WvonctmnoSPKENegQg==", + uint64_t{0x55290b1a8f170f59}}, + {"vPyl8DxVeRe1OpilKb9KNwpGkQRtA94UpAHetNh+" + "95V7nIW38v7PpzhnTWIml5kw3So1Si0TXtIUPIbsu32BNhoH7QwFvLM+" + "JACgSpc5e3RjsL6Qwxxi11npwxRmRUqATDeMUfRAjxg=", + uint64_t{0x5501cfd83dfe706a}}, + {"QC9i2GjdTMuNC1xQJ74ngKfrlA4w3o58FhvNCltdIpuMhHP1YsDA78scQPLbZ3OCUgeQguY" + "f/vw6zAaVKSgwtaykqg5ka/4vhz4hYqWU5ficdXqClHl+zkWEY26slCNYOM5nnDlly8Cj", + uint64_t{0xe43ed13d13a66990}}, + {"7CNIgQhAHX27nxI0HeB5oUTnTdgKpRDYDKwRcXfSFGP1XeT9nQF6WKCMjL1tBV6x7KuJ91G" + "Zz11F4c+8s+MfqEAEpd4FHzamrMNjGcjCyrVtU6y+7HscMVzr7Q/" + "ODLcPEFztFnwjvCjmHw==", + uint64_t{0xdf43bc375cf5283f}}, + {"Qa/hC2RPXhANSospe+gUaPfjdK/yhQvfm4cCV6/pdvCYWPv8p1kMtKOX3h5/" + "8oZ31fsmx4Axphu5qXJokuhZKkBUJueuMpxRyXpwSWz2wELx5glxF7CM0Fn+" + "OevnkhUn5jsPlG2r5jYlVn8=", + uint64_t{0x8112b806d288d7b5}}, + {"kUw/0z4l3a89jTwN5jpG0SHY5km/" + "IVhTjgM5xCiPRLncg40aqWrJ5vcF891AOq5hEpSq0bUCJUMFXgct7kvnys905HjerV7Vs1G" + "y84tgVJ70/2+pAZTsB/PzNOE/G6sOj4+GbTzkQu819OLB", + uint64_t{0xd52a18abb001cb46}}, + {"VDdfSDbO8Tdj3T5W0XM3EI7iHh5xpIutiM6dvcJ/fhe23V/srFEkDy5iZf/" + "VnA9kfi2C79ENnFnbOReeuZW1b3MUXB9lgC6U4pOTuC+" + "jHK3Qnpyiqzj7h3ISJSuo2pob7vY6VHZo6Fn7exEqHg==", + uint64_t{0xe12b76a2433a1236}}, + {"Ldfvy3ORdquM/R2fIkhH/ONi69mcP1AEJ6n/" + "oropwecAsLJzQSgezSY8bEiEs0VnFTBBsW+RtZY6tDj03fnb3amNUOq1b7jbqyQkL9hpl+" + "2Z2J8IaVSeownWl+bQcsR5/xRktIMckC5AtF4YHfU=", + uint64_t{0x175bf7319cf1fa00}}, + {"BrbNpb42+" + "VzZAjJw6QLirXzhweCVRfwlczzZ0VX2xluskwBqyfnGovz5EuX79JJ31VNXa5hTkAyQat3l" + "YKRADTdAdwE5PqM1N7YaMqqsqoAAAeuYVXuk5eWCykYmClNdSspegwgCuT+403JigBzi", + uint64_t{0xd63d57b3f67525ae}}, + {"gB3NGHJJvVcuPyF0ZSvHwnWSIfmaI7La24VMPQVoIIWF7Z74NltPZZpx2f+cocESM+" + "ILzQW9p+BC8x5IWz7N4Str2WLGKMdgmaBfNkEhSHQDU0IJEOnpUt0HmjhFaBlx0/" + "LTmhua+rQ6Wup8ezLwfg==", + uint64_t{0x933faea858832b73}}, + {"hTKHlRxx6Pl4gjG+6ksvvj0CWFicUg3WrPdSJypDpq91LUWRni2KF6+" + "81ZoHBFhEBrCdogKqeK+hy9bLDnx7g6rAFUjtn1+cWzQ2YjiOpz4+" + "ROBB7lnwjyTGWzJD1rXtlso1g2qVH8XJVigC5M9AIxM=", + uint64_t{0x53d061e5f8e7c04f}}, + {"IWQBelSQnhrr0F3BhUpXUIDauhX6f95Qp+A0diFXiUK7irwPG1oqBiqHyK/SH/" + "9S+" + "rln9DlFROAmeFdH0OCJi2tFm4afxYzJTFR4HnR4cG4x12JqHaZLQx6iiu6CE3rtWBVz99oA" + "wCZUOEXIsLU24o2Y", + uint64_t{0xdb4124556dd515e0}}, + {"TKo+l+" + "1dOXdLvIrFqeLaHdm0HZnbcdEgOoLVcGRiCbAMR0j5pIFw8D36tefckAS1RCFOH5IgP8yiF" + "T0Gd0a2hI3+" + "fTKA7iK96NekxWeoeqzJyctc6QsoiyBlkZerRxs5RplrxoeNg29kKDTM0K94mnhD9g==", + uint64_t{0x4fb31a0dd681ee71}}, + {"YU4e7G6EfQYvxCFoCrrT0EFgVLHFfOWRTJQJ5gxM3G2b+" + "1kJf9YPrpsxF6Xr6nYtS8reEEbDoZJYqnlk9lXSkVArm88Cqn6d25VCx3+" + "49MqC0trIlXtb7SXUUhwpJK16T0hJUfPH7s5cMZXc6YmmbFuBNPE=", + uint64_t{0x27cc72eefa138e4c}}, + {"/I/" + "eImMwPo1U6wekNFD1Jxjk9XQVi1D+" + "FPdqcHifYXQuP5aScNQfxMAmaPR2XhuOQhADV5tTVbBKwCDCX4E3jcDNHzCiPvViZF1W27t" + "xaf2BbFQdwKrNCmrtzcluBFYu0XZfc7RU1RmxK/RtnF1qHsq/O4pp", + uint64_t{0x44bc2dfba4bd3ced}}, + {"CJTT9WGcY2XykTdo8KodRIA29qsqY0iHzWZRjKHb9alwyJ7RZAE3V5Juv4MY3MeYEr1EPCC" + "MxO7yFXqT8XA8YTjaMp3bafRt17Pw8JC4iKJ1zN+WWKOESrj+" + "3aluGQqn8z1EzqY4PH7rLG575PYeWsP98BugdA==", + uint64_t{0x242da1e3a439bed8}}, + {"ZlhyQwLhXQyIUEnMH/" + "AEW27vh9xrbNKJxpWGtrEmKhd+nFqAfbeNBQjW0SfG1YI0xQkQMHXjuTt4P/" + "EpZRtA47ibZDVS8TtaxwyBjuIDwqcN09eCtpC+Ls+" + "vWDTLmBeDM3u4hmzz4DQAYsLiZYSJcldg9Q3wszw=", + uint64_t{0xdc559c746e35c139}}, + {"v2KU8y0sCrBghmnm8lzGJlwo6D6ObccAxCf10heoDtYLosk4ztTpLlpSFEyu23MLA1tJkcg" + "Rko04h19QMG0mOw/" + "wc93EXAweriBqXfvdaP85sZABwiKO+6rtS9pacRVpYYhHJeVTQ5NzrvBvi1huxAr+" + "xswhVMfL", + uint64_t{0xd0b0350275b9989}}, + {"QhKlnIS6BuVCTQsnoE67E/" + "yrgogE8EwO7xLaEGei26m0gEU4OksefJgppDh3X0x0Cs78Dr9IHK5b977CmZlrTRmwhlP8p" + "M+UzXPNRNIZuN3ntOum/QhUWP8SGpirheXENWsXMQ/" + "nxtxakyEtrNkKk471Oov9juP8oQ==", + uint64_t{0xb04489e41d17730c}}, + {"/ZRMgnoRt+Uo6fUPr9FqQvKX7syhgVqWu+" + "WUSsiQ68UlN0efSP6Eced5gJZL6tg9gcYJIkhjuQNITU0Q3TjVAnAcobgbJikCn6qZ6pRxK" + "BY4MTiAlfGD3T7R7hwJwx554MAy++Zb/YUFlnCaCJiwQMnowF7aQzwYFCo=", + uint64_t{0x2217285eb4572156}}, + {"NB7tU5fNE8nI+SXGfipc7sRkhnSkUF1krjeo6k+8FITaAtdyz+" + "o7mONgXmGLulBPH9bEwyYhKNVY0L+njNQrZ9YC2aXsFD3PdZsxAFaBT3VXEzh+" + "NGBTjDASNL3mXyS8Yv1iThGfHoY7T4aR0NYGJ+k+pR6f+KrPC96M", + uint64_t{0x12c2e8e68aede73b}}, + {"8T6wrqCtEO6/rwxF6lvMeyuigVOLwPipX/FULvwyu+1wa5sQGav/" + "2FsLHUVn6cGSi0LlFwLewGHPFJDLR0u4t7ZUyM//" + "x6da0sWgOa5hzDqjsVGmjxEHXiaXKW3i4iSZNuxoNbMQkIbVML+" + "DkYu9ND0O2swg4itGeVSzXA==", + uint64_t{0x4d612125bdc4fd00}}, + {"Ntf1bMRdondtMv1CYr3G80iDJ4WSAlKy5H34XdGruQiCrnRGDBa+" + "eUi7vKp4gp3BBcVGl8eYSasVQQjn7MLvb3BjtXx6c/" + "bCL7JtpzQKaDnPr9GWRxpBXVxKREgMM7d8lm35EODv0w+" + "hQLfVSh8OGs7fsBb68nNWPLeeSOo=", + uint64_t{0x81826b553954464e}}, + {"VsSAw72Ro6xks02kaiLuiTEIWBC5bgqr4WDnmP8vglXzAhixk7td926rm9jNimL+" + "kroPSygZ9gl63aF5DCPOACXmsbmhDrAQuUzoh9ZKhWgElLQsrqo1KIjWoZT5b5QfVUXY9lS" + "IBg3U75SqORoTPq7HalxxoIT5diWOcJQi", + uint64_t{0xc2e5d345dc0ddd2d}}, + {"j+loZ+C87+" + "bJxNVebg94gU0mSLeDulcHs84tQT7BZM2rzDSLiCNxUedHr1ZWJ9ejTiBa0dqy2I2ABc++" + "xzOLcv+//YfibtjKtYggC6/3rv0XCc7xu6d/" + "O6xO+XOBhOWAQ+IHJVHf7wZnDxIXB8AUHsnjEISKj7823biqXjyP3g==", + uint64_t{0x3da6830a9e32631e}}, + {"f3LlpcPElMkspNtDq5xXyWU62erEaKn7RWKlo540gR6mZsNpK1czV/" + "sOmqaq8XAQLEn68LKj6/" + "cFkJukxRzCa4OF1a7cCAXYFp9+wZDu0bw4y63qbpjhdCl8GO6Z2lkcXy7KOzbPE01ukg7+" + "gN+7uKpoohgAhIwpAKQXmX5xtd0=", + uint64_t{0xc9ae5c8759b4877a}}, + }; + +#if defined(ABSL_IS_BIG_ENDIAN) + constexpr uint64_t kGolden[kNumGoldenOutputs] = { + 0xe5a40d39ab796423, 0x1766974bf7527d81, 0x5c3bbbe230db17a8, + 0xa6630143a7e6aa6f, 0x17645cb7318b86b, 0x218b175f30ba61f8, + 0xa6564b468248c683, 0xef192f401b116e1c, 0xbe8dc0c54617639d, + 0xe7b01610fc22dbb8, 0x99d9f694404af913, 0xf4eecd37464b45c5, + 0x7d2c653d63596d9b, 0x3f15c8544ec5393a, 0x6b9dc0c1704f796c, + 0xf1ded7a7eae5ed5a, 0x2db2fd7c6dd4641b, 0x151ca2d3d4cd33ab, + 0xa5af5994ac2ccd64, 0x2b2a4ca3191d2fce, 0xf89e68c9364e7c05, + 0x71724c70b799c21, 0x70536fabfd157369, 0xdee92794c3c3082b, + 0xac033a6743d3b3eb, 0xed2956b506cd5151, 0xbd669644755264b6, + 0x6ab1ff5d5f549a63, 0xf6bd551a2e3e04e, 0x7b5a8cef6875ea73, + 0x22bccf4d4db0a91c, 0x4f2bc07754c7c7eb, 0xfb6b8342a86725db, + 0x13a1a0d4c5854da, 0x5f6e44655f7dedac, 0x54a9198dff2bdf85, + 0xdb17e6915d4e4042, 0xa69926cf5c3b89f, 0xf77f031bfd74c096, + 0x1d6f916fdd50ec3c, 0x334ac76013ade393, 0x99370f899111de15, + 0x352457a03ada6de, 0x341974d4f42d854d, 0xda89ab02872aeb5, + 0x6ec2b74e143b10d9, 0x6f284c0b5cd60522, 0xf9670de353438f88, + 0xde920913adf0a2b4, 0xb7a07d7c0c17a8ec, 0x879a69f558ba3a98, + 0x360cf6d802df20f9, 0x53530f8046673738, 0xbd8f5f2bcf35e483, + 0x3f171f047144b983, 0x644d04e820823465, 0x50e44773a20b2702, + 0xe584ed4c05c745dd, 0x9a825c85b95ab6c0, 0xbce2931deb74e775, + 0x10468e9e705c7cfe, 0x12e01de3104141e2, 0x5c11ae2ee3713abd, + 0x6ac5ffb0860319e6, 0xc1e6da1849d30fc9, 0xa0e4d247a458b447, + 0x4530d4615c32b89b, 0x116aa09107a76505, 0xf941339d00d9bb73, + 0x573a0fc1615afb33, 0xa975c81dc868b258, 0x3ab2c5250ab54bda, + 0x37f99f208a3e3b11, 0x4b49b0ff706689d, 0x30bafa0b8f0a87fe, + 0xea6787a65cc20cdd, 0x55861729f1fc3ab8, 0xea38e009c5be9b72, + 0xcb8522cba33c3c66, 0x352e77653fe306f3, 0xe0bb760793bac064, + 0xf66ec59322662956, 0x637aa320455d56f8, 0x46ee546be5824a89, + 0x9e6842421e83d8a4, 0xf98ac2bc96b9fb8c, 0xf2c1002fd9a70b99, + 0x4c2b62b1e39e9405, 0x3248555fa3ade9c4, 0xd4d04c37f6417c21, + 0xf40cd506b1bf5653, 0x6c45d6005c760d2f, 0x61d88a7e61ff0d7e, + 0x131591e8a53cc967, 0xdae85cb9bc29bab6, 0xe98835334905e626, + 0x7cce50a2b66b8754, 0x5b0b3d0c5ac498ae, 0xd35a218c974d1756, + 0xfce436ddc1d003c, 0xd183901de90bb741, 0x9378f8f34974a66, + 0x21f11ae0a0402368, 0xf2fbd7c94ef89cb6, 0xc329c69d0f0d080b, + 0xf2841cba16216a61, 0x47aba97b44916df1, 0x724d4e00a8019fcf, + 0x2df9005c2a728d63, 0xc788892a1a5d7515, 0x9e993a65f9df0480, + 0x76876721ff49f969, 0xbe7a796cfba15bf5, 0xa4c8bd54586f5488, + 0xb390a325275501ab, 0x893f11317427ccf1, 0x92f2bb57da5695b9, + 0x30985b90da88269f, 0x2c690e268e086de8, 0x1c02df6097997196, + 0x1f9778f8bbdf6455, 0x7d57378c7bf8416d, 0xba8582a5f8d84d38, + 0xe8ca43b85050be4e, 0x5048cf6bed8a5d9f, 0xfbc5ba80917d0ea4, + 0x8011026525bf1691, 0x26b8dc6aed9fb50d, 0x191f5bfee77c1fe3, + 0xdd497891465a2cc1, 0x6f1fe8c57a33072e, 0x2c9f4ec078c460c0, + 0x9a725bde8f6a1437, 0x6ce545fa3ef61e4d, + }; +#elif defined(__aarch64__) + constexpr uint64_t kGolden[kNumGoldenOutputs] = { + 0x45c0aadee165dcbe, 0x25ed8587f6f20d06, 0x5f23ae668ce7926d, + 0xfef74d1da0846719, 0x54478408e68cb7d4, 0xee27ddaf88c6fe68, + 0xb7ac7031e81867ca, 0xf1168f818ec6c36d, 0x1dd0b734a83b019a, + 0xd6ae30d4142b54fe, 0xcd860c721ccb80fb, 0x068acf8493794756, + 0xd4ada0be58681307, 0x13ffe0f64ca540ed, 0xffc1d7a3401aec02, + 0xd81c4d865cf95fb9, 0x1dd0793acede62e0, 0xa6722abbca8fe4cf, + 0x5453d3e4111a7e40, 0xf29b3e3204c9dcd2, 0x23be2980e43117f7, + 0x74e2ccbc286f08eb, 0x19ef7c0f9496003a, 0xbfbf1c3e49b27987, + 0x6e6c179eb4a82c70, 0x07f4e184216bc4fc, 0xf17fbc4254927554, + 0xe57696b70a45b1b6, 0x6d3b144631b320e8, 0xccf8729792c75a2d, + 0xe832495b41fa980b, 0x5c96cfdc7b227d34, 0xc4dca234ef4e43f4, + 0x5fc801abf9abe307, 0xe41e3c5076d88f4d, 0x522346200ddec3c3, + 0x72bed1946fd7aaa4, 0x0ac1f84dcc335f96, 0x3af78db5e0a47670, + 0x6100ebf1481f1caf, 0xf5fd10037fc651a3, 0xa01227d8944665f3, + 0x7217681c4bbc9420, 0x4adee538e3eb10d1, 0x35e1761ad96de9a7, + 0x8b370aef9613bfba, 0x824506f749eeaf59, 0x85e805fa04423991, + 0xb61e9c33283c3de7, 0xc79721bbcb039ed6, 0x04e1c19a3a1e6639, + 0x6aaf6346b68dd638, 0x601a4b496be6d0c4, 0x3ece355f91c41787, + 0xd2fc8998448d7888, 0xd7529804f843efa9, 0xabdcc38a288536aa, + 0xdd323e48a9718648, 0x2090279c0030a52a, 0xe2f90faca88a3cd1, + 0x3e0c4e92fc50e4aa, 0xa26d308798e801dd, 0x432eefeedee8c02e, + 0xca4ce494614b77df, 0xbba82911e838066d, 0x4b00821016adee4b, + 0x4cf6e526dfb5a20f, 0x5b8466495142cba2, 0xe28ac1406e88a68c, + 0x8511e5f9d3100999, 0x05acbfe02798890b, 0x74c249c7ce4a8425, + 0xdbe7468d09bc34bc, 0x11079ab10e3b9b58, 0xb7788dec9032035a, + 0xb7e8daa786513f80, 0x34c3288831f46b45, 0x014cce5f0c21ecc6, + 0xc6a8f7b024551a28, 0x49784e902e207fd8, 0x4720d32af0b55158, + 0x8df3ec5de0c1da00, 0xf4db677b2c9e6853, 0xaa419abea78d312d, + 0x181e0f91bd757443, 0xa8c45136fada083b, 0x91303b93f5f0582c, + 0x883b95c6ddc62a08, 0x93186a8875fe952b, 0xd94f533928e957e2, + 0x6ba343003e10c172, 0xc8623b620c715d6a, 0x8ca0c512e180e244, + 0xdc9b74c2536b6216, 0x8eb5fdc61b295d96, 0x2ad83966b37c95ba, + 0xb90bf154ac5edec9, 0x902cf847b326cfb3, 0x7b02d0c0ca7808ca, + 0x492f310d003ea15f, 0x3eb6497a47c95990, 0x5d46b0ced31428b7, + 0x081afa67d1986157, 0x043482ec286b20eb, 0xc103c8f18c1a2a53, + 0xe8e9995a81481e83, 0x6bb3295822bc90b5, 0xeec75297a3fa5672, + 0x591c8440c4857412, 0x74947f455aaf24ad, 0xcf0e571586ec77a9, + 0x0c2553ea8c0400ad, 0x380219118066255f, 0x7595adb88b15ebe2, + 0xb33c00696c64ae23, 0xa143516ddd7c9857, 0x39179af229248d26, + 0x65d387a6f2ee2079, 0x89f8a9b21cd2f195, 0xbfef032d25df92e6, + 0x6b7e18a36c69da71, 0x4b3b15f6c28974e6, 0x032a75917f6c544c, + 0xe3b97ecca6d287cd, 0xa4a563110d3cda81, 0x35e09e8134f4e7f1, + 0xc9419dd03a9a390e, 0x7b86fae9000fd329, 0x1e044f8d54fe74c3, + 0x9c4991d7a47e9666, 0xfb485f3a1df4fdb6, 0xb11519969eeb94ff, + 0x3224ea1c44caeb8d, 0x86570bbd7cc6b80d, + }; +#else + constexpr uint64_t kGolden[kNumGoldenOutputs] = { + 0xe5a40d39ab796423, 0x1766974bf7527d81, 0x5c3bbbe230db17a8, + 0xa6630143a7e6aa6f, 0x8787cb2d04b0c984, 0x33603654ff574ac2, + 0xa6564b468248c683, 0xef192f401b116e1c, 0xbe8dc0c54617639d, + 0x93d7f665b5521c8e, 0x646d70bb42445f28, 0x96a7b1e3cc9bd426, + 0x76020289ab0790c4, 0x39f842e4133b9b44, 0x2b8d7047be4bcaab, + 0x99628abef6716a97, 0x4432e02ba42b2740, 0x74d810efcad7918a, + 0x88c84e986002507f, 0x4f99acf193cf39b9, 0xd90e7a3655891e37, + 0x3bb378b1d4df8fcf, 0xf78e94045c052d47, 0x26da0b2130da6b40, + 0x30b4d426af8c6986, 0x5413b4aaf3baaeae, 0x756ab265370a1597, + 0xdaf5f4b7d09814fb, 0x8f874ae37742b75e, 0x8fecd03956121ce8, + 0x229c292ea7a08285, 0x0bb4bf0692d14bae, 0x207b24ca3bdac1db, + 0x64f6cd6745d3825b, 0xa2b2e1656b58df1e, 0x0d01d30d9ee7a148, + 0x1cb4cd00ab804e3b, 0x4697f2637fd90999, 0x8383a756b5688c07, + 0x695c29cb3696a975, 0xda2e5a5a5e971521, 0x7935d4befa056b2b, + 0x38dd541ca95420fe, 0xcc06c7a4963f967f, 0xbf0f6f66e232fb20, + 0xf7efb32d373fe71a, 0xe2e64634b1c12660, 0x285b8fd1638e306d, + 0x658e8a4e3b714d6c, 0xf391fb968e0eb398, 0x744a9ea0cc144bf2, + 0x12636f2be11012f1, 0x29c57de825948f80, 0x58c6f99ab0d1c021, + 0x13e7b5a7b82fe3bb, 0x10fbc87901e02b63, 0xa24c9184901b748b, + 0xcac4fd4c5080e581, 0xc38bdb7483ba68e1, 0xdb2a8069b2ceaffa, + 0xdf9fe91d0d1c7887, 0xe83f49e96e2e6a08, 0x0c69e61b62ca2b62, + 0xb4a4f3f85f8298fe, 0x167a1b39e1e95f41, 0xf8a2a5649855ee41, + 0x27992565b595c498, 0x3e08cca5b71f9346, 0xad406b10c770a6d2, + 0xd1713ce6e552bcf2, 0x753b287194c73ad3, 0x5ae41a95f600af1c, + 0x4a61163b86a8bb4c, 0x42eeaa79e760c7e4, 0x698df622ef465b0a, + 0x157583111e1a6026, 0xaa1388f078e793e0, 0xf10d68d0f3309360, + 0x2af056184457a3de, 0x6d0058e1590b2489, 0x638f287f68817f12, + 0xc46b71fecefd5467, 0x2c8e94679d964e0a, 0x8612b797ce22503a, + 0x59f929babfba7170, 0x9527556923fb49a0, 0x1039ab644f5e150b, + 0x7816c83f3aa05e6d, 0xf51d2f564518c619, 0x67d494cff03ac004, + 0x2802d636ced1cfbb, 0xf64e20bad771cb12, 0x0b9a6cf84a83e15e, + 0x8da6630319609301, 0x40946a86e2a996f3, 0xcab7f5997953fa76, + 0x39129ca0e04fc465, 0x5238221fd685e1b8, 0x175130c407dbcaab, + 0x02f20e7536c0b0df, 0x2742cb488a04ad56, 0xd6afb593879ff93b, + 0xf50ad64caac0ca7f, 0x2ade95c4261364ae, 0x5c4f3299faacd07a, + 0xfffe3bff0ae5e9bc, 0x1db785c0005166e4, 0xea000d962ad18418, + 0xe42aef38359362d9, 0xc8e95657348a3891, 0xc162eca864f238c6, + 0xbe1fb373e20579ad, 0x628a1d4f40aa6ffd, 0xa87bdb7456340f90, + 0x5960ef3ba982c801, 0x5026586df9a431ec, 0xfe4b8a20fdf0840b, + 0xdcb761867da7072f, 0xc10d4653667275b7, 0x727720deec13110b, + 0x710b009662858dc9, 0xfbf8f7a3ecac1eb7, 0xb6fc4fcd0722e3df, + 0x7cb86dcc55104aac, 0x19e71e9b45c3a51e, 0x51de38573c2bea48, + 0xa73ab6996d6df158, 0x55ef2b8c930817b2, 0xb2850bf5fae87157, + 0xecf3de1acd04651f, 0xcc0a40552559ff32, 0xc385c374f20315b1, + 0xb90208a4c7234183, 0x58aa1ca7a4c075d9, + }; +#endif + +#if UPDATE_GOLDEN + (void)kGolden; // Silence warning. + for (size_t i = 0; i < kNumGoldenOutputs; ++i) { + std::string str; + ASSERT_TRUE(absl::Base64Unescape(cases[i].base64_data, &str)); + uint64_t h = absl::hash_internal::LowLevelHash(str.data(), str.size(), + cases[i].seed, kSalt); + printf("0x%016" PRIx64 ", ", h); + if (i % 3 == 2) { + printf("\n"); + } + } + printf("\n\n\n"); + EXPECT_FALSE(true); +#else + for (size_t i = 0; i < kNumGoldenOutputs; ++i) { + SCOPED_TRACE(::testing::Message() + << "i = " << i << "; input = " << cases[i].base64_data); + std::string str; + ASSERT_TRUE(absl::Base64Unescape(cases[i].base64_data, &str)); + EXPECT_EQ(absl::hash_internal::LowLevelHash(str.data(), str.size(), + cases[i].seed, kSalt), + kGolden[i]); + } +#endif +} + +} // namespace diff --git a/absl/hash/internal/spy_hash_state.h b/absl/hash/internal/spy_hash_state.h index c0831208..09728266 100644 --- a/absl/hash/internal/spy_hash_state.h +++ b/absl/hash/internal/spy_hash_state.h @@ -15,6 +15,7 @@ #ifndef ABSL_HASH_INTERNAL_SPY_HASH_STATE_H_ #define ABSL_HASH_INTERNAL_SPY_HASH_STATE_H_ +#include <algorithm> #include <ostream> #include <string> #include <vector> @@ -167,6 +168,24 @@ class SpyHashStateImpl : public HashStateBase<SpyHashStateImpl<T>> { using SpyHashStateImpl::HashStateBase::combine_contiguous; + template <typename CombinerT> + static SpyHashStateImpl RunCombineUnordered(SpyHashStateImpl state, + CombinerT combiner) { + UnorderedCombinerCallback cb; + + combiner(SpyHashStateImpl<void>{}, std::ref(cb)); + + std::sort(cb.element_hash_representations.begin(), + cb.element_hash_representations.end()); + state.hash_representation_.insert(state.hash_representation_.end(), + cb.element_hash_representations.begin(), + cb.element_hash_representations.end()); + if (cb.error && cb.error->has_value()) { + state.error_ = std::move(cb.error); + } + return state; + } + absl::optional<std::string> error() const { if (moved_from_) { return "Returned a moved-from instance of the hash state object."; @@ -178,6 +197,22 @@ class SpyHashStateImpl : public HashStateBase<SpyHashStateImpl<T>> { template <typename U> friend class SpyHashStateImpl; + struct UnorderedCombinerCallback { + std::vector<std::string> element_hash_representations; + std::shared_ptr<absl::optional<std::string>> error; + + // The inner spy can have a different type. + template <typename U> + void operator()(SpyHashStateImpl<U>& inner) { + element_hash_representations.push_back( + absl::StrJoin(inner.hash_representation_, "")); + if (inner.error_->has_value()) { + error = std::move(inner.error_); + } + inner = SpyHashStateImpl<void>{}; + } + }; + // This is true if SpyHashStateImpl<T> has been passed to a call of // AbslHashValue with the wrong type. This detects that the user called // AbslHashValue directly (because the hash state type does not match). diff --git a/absl/hash/internal/wyhash_test.cc b/absl/hash/internal/wyhash_test.cc deleted file mode 100644 index 9fb06d23..00000000 --- a/absl/hash/internal/wyhash_test.cc +++ /dev/null @@ -1,486 +0,0 @@ -// Copyright 2020 The Abseil Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// https://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -#include "absl/hash/internal/wyhash.h" - -#include "absl/strings/escaping.h" -#include "gmock/gmock.h" -#include "gtest/gtest.h" - -namespace { - -static const uint64_t kCurrentSeed = 0; -static const uint64_t kSalt[5] = {0xa0761d6478bd642f, 0xe7037ed1a0b428dbl, - 0x8ebc6af09c88c6e3, 0x589965cc75374cc3l, - 0x1d8e4e27c47d124f}; - -// Note: We don't account for endianness, so the values here are only correct if -// you're also running on a little endian platform. - -TEST(WyhashTest, EmptyString) { - const std::string s = ""; - EXPECT_EQ( - absl::hash_internal::Wyhash(s.c_str(), s.length(), kCurrentSeed, kSalt), - 4808886099364463827); -} - -TEST(WyhashTest, Spaces) { - const std::string s = " "; - EXPECT_EQ( - absl::hash_internal::Wyhash(s.c_str(), s.length(), kCurrentSeed, kSalt), - 1686201463024549249); -} - -TEST(WyhashTest, RepeatingString) { - const std::string s = "aaaa"; - EXPECT_EQ( - absl::hash_internal::Wyhash(s.c_str(), s.length(), kCurrentSeed, kSalt), - 6646112255271966632); -} - -TEST(WyhashTest, HexString) { - const std::string small = "\x01\x02\x03"; - const std::string med = "\x01\x02\x03\x04"; - - EXPECT_EQ(absl::hash_internal::Wyhash(small.c_str(), small.length(), - kCurrentSeed, kSalt), - 11989428023081740911ULL); - EXPECT_EQ(absl::hash_internal::Wyhash(med.c_str(), med.length(), kCurrentSeed, - kSalt), - 9765997711188871556ULL); -} - -TEST(WyhashTest, Words) { - const std::string s = "third_party|wyhash|64"; - EXPECT_EQ( - absl::hash_internal::Wyhash(s.c_str(), s.length(), kCurrentSeed, kSalt), - 3702018632387611330); -} - -TEST(WyhashTest, LongString) { - const std::string s = - "AbCdEfGhIjKlMnOpQrStUvWxYz0123456789AbCdEfGhIjKlMnOpQrStUvWxYz" - "0123456789AbCdEfGhIjKlMnOpQrStUvWxYz0123456789AbCdEfGhIjKlMnOp" - "QrStUvWxYz0123456789AbCdEfGhIjKlMnOpQrStUvWxYz0123456789AbCdEf" - "GhIjKlMnOpQrStUvWxYz0123456789AbCdEfGhIjKlMnOpQrStUvWxYz012345" - "6789AbCdEfGhIjKlMnOpQrStUvWxYz0123456789"; - - EXPECT_EQ( - absl::hash_internal::Wyhash(s.c_str(), s.length(), kCurrentSeed, kSalt), - 9245411362605796064ULL); -} - -TEST(WyhashTest, BigReference) { - struct ExpectedResult { - absl::string_view base64_data; - uint64_t seed; - uint64_t hash; - } expected_results[] = { - {"", uint64_t{0xec42b7ab404b8acb}, uint64_t{0xe5a40d39ab796423}}, - {"Zw==", uint64_t{0xeeee074043a3ee0f}, uint64_t{0xa6564b468248c683}}, - {"xmk=", uint64_t{0x857902089c393de}, uint64_t{0xef192f401b116e1c}}, - {"c1H/", uint64_t{0x993df040024ca3af}, uint64_t{0xbe8dc0c54617639d}}, - {"SuwpzQ==", uint64_t{0xc4e4c2acea740e96}, uint64_t{0x93d7f665b5521c8e}}, - {"uqvy++M=", uint64_t{0x6a214b3db872d0cf}, uint64_t{0x646d70bb42445f28}}, - {"RnzCVPgb", uint64_t{0x44343db6a89dba4d}, uint64_t{0x96a7b1e3cc9bd426}}, - {"6OeNdlouYw==", uint64_t{0x77b5d6d1ae1dd483}, - uint64_t{0x76020289ab0790c4}}, - {"M5/JmmYyDbc=", uint64_t{0x89ab8ecb44d221f1}, - uint64_t{0x39f842e4133b9b44}}, - {"MVijWiVdBRdY", uint64_t{0x60244b17577ca81b}, - uint64_t{0x2b8d7047be4bcaab}}, - {"6V7Uq7LNxpu0VA==", uint64_t{0x59a08dcee0717067}, - uint64_t{0x99628abef6716a97}}, - {"EQ6CdEEhPdyHcOk=", uint64_t{0xf5f20db3ade57396}, - uint64_t{0x4432e02ba42b2740}}, - {"PqFB4fxnPgF+l+rc", uint64_t{0xbf8dee0751ad3efb}, - uint64_t{0x74d810efcad7918a}}, - {"a5aPOFwq7LA7+zKvPA==", uint64_t{0x6b7a06b268d63e30}, - uint64_t{0x88c84e986002507f}}, - {"VOwY21wCGv5D+/qqOvs=", uint64_t{0xb8c37f0ae0f54c82}, - uint64_t{0x4f99acf193cf39b9}}, - {"KdHmBTx8lHXYvmGJ+Vy7", uint64_t{0x9fcbed0c38e50eef}, - uint64_t{0xd90e7a3655891e37}}, - {"qJkPlbHr8bMF7/cA6aE65Q==", uint64_t{0x2af4bade1d8e3a1d}, - uint64_t{0x3bb378b1d4df8fcf}}, - {"ygvL0EhHZL0fIx6oHHtkxRQ=", uint64_t{0x714e3aa912da2f2c}, - uint64_t{0xf78e94045c052d47}}, - {"c1rFXkt5YztwZCQRngncqtSs", uint64_t{0xf5ee75e3cbb82c1c}, - uint64_t{0x26da0b2130da6b40}}, - {"8hsQrzszzeNQSEcVXLtvIhm6mw==", uint64_t{0x620e7007321b93b9}, - uint64_t{0x30b4d426af8c6986}}, - {"ffUL4RocfyP4KfikGxO1yk7omDI=", uint64_t{0xc08528cac2e551fc}, - uint64_t{0x5413b4aaf3baaeae}}, - {"OOB5TT00vF9Od/rLbAWshiErqhpV", uint64_t{0x6a1debf9cc3ad39}, - uint64_t{0x756ab265370a1597}}, - {"or5wtXM7BFzTNpSzr+Lw5J5PMhVJ/Q==", uint64_t{0x7e0a3c88111fc226}, - uint64_t{0xdaf5f4b7d09814fb}}, - {"gk6pCHDUsoopVEiaCrzVDhioRKxb844=", uint64_t{0x1301fef15df39edb}, - uint64_t{0x8f874ae37742b75e}}, - {"TNctmwlC5QbEM6/No4R/La3UdkfeMhzs", uint64_t{0x64e181f3d5817ab}, - uint64_t{0x8fecd03956121ce8}}, - {"SsQw9iAjhWz7sgcE9OwLuSC6hsM+BfHs2Q==", uint64_t{0xafafc44961078ecb}, - uint64_t{0x229c292ea7a08285}}, - {"ZzO3mVCj4xTT2TT3XqDyEKj2BZQBvrS8RHg=", uint64_t{0x4f7bb45549250094}, - uint64_t{0xbb4bf0692d14bae}}, - {"+klp5iPQGtppan5MflEls0iEUzqU+zGZkDJX", uint64_t{0xa30061abaa2818c}, - uint64_t{0x207b24ca3bdac1db}}, - {"RO6bvOnlJc8I9eniXlNgqtKy0IX6VNg16NRmgg==", uint64_t{0xd902ee3e44a5705f}, - uint64_t{0x64f6cd6745d3825b}}, - {"ZJjZqId1ZXBaij9igClE3nyliU5XWdNRrayGlYA=", uint64_t{0x316d36da516f583}, - uint64_t{0xa2b2e1656b58df1e}}, - {"7BfkhfGMDGbxfMB8uyL85GbaYQtjr2K8g7RpLzr/", uint64_t{0x402d83f9f834f616}, - uint64_t{0xd01d30d9ee7a148}}, - {"rycWk6wHH7htETQtje9PidS2YzXBx+Qkg2fY7ZYS7A==", - uint64_t{0x9c604164c016b72c}, uint64_t{0x1cb4cd00ab804e3b}}, - {"RTkC2OUK+J13CdGllsH0H5WqgspsSa6QzRZouqx6pvI=", - uint64_t{0x3f4507e01f9e73ba}, uint64_t{0x4697f2637fd90999}}, - {"tKjKmbLCNyrLCM9hycOAXm4DKNpM12oZ7dLTmUx5iwAi", - uint64_t{0xc3fe0d5be8d2c7c7}, uint64_t{0x8383a756b5688c07}}, - {"VprUGNH+5NnNRaORxgH/ySrZFQFDL+4VAodhfBNinmn8cg==", - uint64_t{0x531858a40bfa7ea1}, uint64_t{0x695c29cb3696a975}}, - {"gc1xZaY+q0nPcUvOOnWnT3bqfmT/geth/f7Dm2e/DemMfk4=", - uint64_t{0x86689478a7a7e8fa}, uint64_t{0xda2e5a5a5e971521}}, - {"Mr35fIxqx1ukPAL0su1yFuzzAU3wABCLZ8+ZUFsXn47UmAph", - uint64_t{0x4ec948b8e7f27288}, uint64_t{0x7935d4befa056b2b}}, - {"A9G8pw2+m7+rDtWYAdbl8tb2fT7FFo4hLi2vAsa5Y8mKH3CX3g==", - uint64_t{0xce46c7213c10032}, uint64_t{0x38dd541ca95420fe}}, - {"DFaJGishGwEHDdj9ixbCoaTjz9KS0phLNWHVVdFsM93CvPft3hM=", - uint64_t{0xf63e96ee6f32a8b6}, uint64_t{0xcc06c7a4963f967f}}, - {"7+Ugx+Kr3aRNgYgcUxru62YkTDt5Hqis+2po81hGBkcrJg4N0uuy", - uint64_t{0x1cfe85e65fc5225}, uint64_t{0xbf0f6f66e232fb20}}, - {"H2w6O8BUKqu6Tvj2xxaecxEI2wRgIgqnTTG1WwOgDSINR13Nm4d4Vg==", - uint64_t{0x45c474f1cee1d2e8}, uint64_t{0xf7efb32d373fe71a}}, - {"1XBMnIbqD5jy65xTDaf6WtiwtdtQwv1dCVoqpeKj+7cTR1SaMWMyI04=", - uint64_t{0x6e024e14015f329c}, uint64_t{0xe2e64634b1c12660}}, - {"znZbdXG2TSFrKHEuJc83gPncYpzXGbAebUpP0XxzH0rpe8BaMQ17nDbt", - uint64_t{0x760c40502103ae1c}, uint64_t{0x285b8fd1638e306d}}, - {"ylu8Atu13j1StlcC1MRMJJXIl7USgDDS22HgVv0WQ8hx/8pNtaiKB17hCQ==", - uint64_t{0x17fd05c3c560c320}, uint64_t{0x658e8a4e3b714d6c}}, - {"M6ZVVzsd7vAvbiACSYHioH/440dp4xG2mLlBnxgiqEvI/aIEGpD0Sf4VS0g=", - uint64_t{0x8b34200a6f8e90d9}, uint64_t{0xf391fb968e0eb398}}, - {"li3oFSXLXI+ubUVGJ4blP6mNinGKLHWkvGruun85AhVn6iuMtocbZPVhqxzn", - uint64_t{0x6be89e50818bdf69}, uint64_t{0x744a9ea0cc144bf2}}, - {"kFuQHuUCqBF3Tc3hO4dgdIp223ShaCoog48d5Do5zMqUXOh5XpGK1t5XtxnfGA==", - uint64_t{0xfb389773315b47d8}, uint64_t{0x12636f2be11012f1}}, - {"jWmOad0v0QhXVJd1OdGuBZtDYYS8wBVHlvOeTQx9ZZnm8wLEItPMeihj72E0nWY=", - uint64_t{0x4f2512a23f61efee}, uint64_t{0x29c57de825948f80}}, - {"z+DHU52HaOQdW4JrZwDQAebEA6rm13Zg/9lPYA3txt3NjTBqFZlOMvTRnVzRbl23", - uint64_t{0x59ccd92fc16c6fda}, uint64_t{0x58c6f99ab0d1c021}}, - {"MmBiGDfYeTayyJa/tVycg+rN7f9mPDFaDc+23j0TlW9094er0ADigsl4QX7V3gG/qw==", - uint64_t{0x25c5a7f5bd330919}, uint64_t{0x13e7b5a7b82fe3bb}}, - {"774RK+9rOL4iFvs1q2qpo/JVc/I39buvNjqEFDtDvyoB0FXxPI2vXqOrk08VPfIHkmU=", - uint64_t{0x51df4174d34c97d7}, uint64_t{0x10fbc87901e02b63}}, - {"+slatXiQ7/2lK0BkVUI1qzNxOOLP3I1iK6OfHaoxgqT63FpzbElwEXSwdsryq3UlHK0I", - uint64_t{0x80ce6d76f89cb57}, uint64_t{0xa24c9184901b748b}}, - {"64mVTbQ47dHjHlOHGS/hjJwr/" - "K2frCNpn87exOqMzNUVYiPKmhCbfS7vBUce5tO6Ec9osQ==", - uint64_t{0x20961c911965f684}, uint64_t{0xcac4fd4c5080e581}}, - {"fIsaG1r530SFrBqaDj1kqE0AJnvvK8MNEZbII2Yw1OK77v0V59xabIh0B5axaz/" - "+a2V5WpA=", - uint64_t{0x4e5b926ec83868e7}, uint64_t{0xc38bdb7483ba68e1}}, - {"PGih0zDEOWCYGxuHGDFu9Ivbff/" - "iE7BNUq65tycTR2R76TerrXALRosnzaNYO5fjFhTi+CiS", - uint64_t{0x3927b30b922eecef}, uint64_t{0xdb2a8069b2ceaffa}}, - {"RnpA/" - "zJnEnnLjmICORByRVb9bCOgxF44p3VMiW10G7PvW7IhwsWajlP9kIwNA9FjAD2GoQHk2Q=" - "=", - uint64_t{0xbd0291284a49b61c}, uint64_t{0xdf9fe91d0d1c7887}}, - {"qFklMceaTHqJpy2qavJE+EVBiNFOi6OxjOA3LeIcBop1K7w8xQi3TrDk+" - "BrWPRIbfprszSaPfrI=", - uint64_t{0x73a77c575bcc956}, uint64_t{0xe83f49e96e2e6a08}}, - {"cLbfUtLl3EcQmITWoTskUR8da/VafRDYF/ylPYwk7/" - "zazk6ssyrzxMN3mmSyvrXR2yDGNZ3WDrTT", - uint64_t{0x766a0e2ade6d09a6}, uint64_t{0xc69e61b62ca2b62}}, - {"s/" - "Jf1+" - "FbsbCpXWPTUSeWyMH6e4CvTFvPE5Fs6Z8hvFITGyr0dtukHzkI84oviVLxhM1xMxrMAy1db" - "w==", - uint64_t{0x2599f4f905115869}, uint64_t{0xb4a4f3f85f8298fe}}, - {"FvyQ00+j7nmYZVQ8hI1Edxd0AWplhTfWuFGiu34AK5X8u2hLX1bE97sZM0CmeLe+" - "7LgoUT1fJ/axybE=", - uint64_t{0xd8256e5444d21e53}, uint64_t{0x167a1b39e1e95f41}}, - {"L8ncxMaYLBH3g9buPu8hfpWZNlOF7nvWLNv9IozH07uQsIBWSKxoPy8+" - "LW4tTuzC6CIWbRGRRD1sQV/4", - uint64_t{0xf664a91333fb8dfd}, uint64_t{0xf8a2a5649855ee41}}, - {"CDK0meI07yrgV2kQlZZ+" - "wuVqhc2NmzqeLH7bmcA6kchsRWFPeVF5Wqjjaj556ABeUoUr3yBmfU3kWOakkg==", - uint64_t{0x9625b859be372cd1}, uint64_t{0x27992565b595c498}}, - {"d23/vc5ONh/" - "HkMiq+gYk4gaCNYyuFKwUkvn46t+dfVcKfBTYykr4kdvAPNXGYLjM4u1YkAEFpJP+" - "nX7eOvs=", - uint64_t{0x7b99940782e29898}, uint64_t{0x3e08cca5b71f9346}}, - {"NUR3SRxBkxTSbtQORJpu/GdR6b/h6sSGfsMj/KFd99ahbh+9r7LSgSGmkGVB/" - "mGoT0pnMTQst7Lv2q6QN6Vm", - uint64_t{0x4fe12fa5383b51a8}, uint64_t{0xad406b10c770a6d2}}, - {"2BOFlcI3Z0RYDtS9T9Ie9yJoXlOdigpPeeT+CRujb/" - "O39Ih5LPC9hP6RQk1kYESGyaLZZi3jtabHs7DiVx/VDg==", - uint64_t{0xe2ccb09ac0f5b4b6}, uint64_t{0xd1713ce6e552bcf2}}, - {"FF2HQE1FxEvWBpg6Z9zAMH+Zlqx8S1JD/" - "wIlViL6ZDZY63alMDrxB0GJQahmAtjlm26RGLnjW7jmgQ4Ie3I+014=", - uint64_t{0x7d0a37adbd7b753b}, uint64_t{0x753b287194c73ad3}}, - {"tHmO7mqVL/PX11nZrz50Hc+M17Poj5lpnqHkEN+4bpMx/" - "YGbkrGOaYjoQjgmt1X2QyypK7xClFrjeWrCMdlVYtbW", - uint64_t{0xd3ae96ef9f7185f2}, uint64_t{0x5ae41a95f600af1c}}, - {"/WiHi9IQcxRImsudkA/KOTqGe8/" - "gXkhKIHkjddv5S9hi02M049dIK3EUyAEjkjpdGLUs+BN0QzPtZqjIYPOgwsYE9g==", - uint64_t{0x4fb88ea63f79a0d8}, uint64_t{0x4a61163b86a8bb4c}}, - {"qds+1ExSnU11L4fTSDz/QE90g4Jh6ioqSh3KDOTOAo2pQGL1k/" - "9CCC7J23YF27dUTzrWsCQA2m4epXoCc3yPHb3xElA=", - uint64_t{0xed564e259bb5ebe9}, uint64_t{0x42eeaa79e760c7e4}}, - {"8FVYHx40lSQPTHheh08Oq0/" - "pGm2OlG8BEf8ezvAxHuGGdgCkqpXIueJBF2mQJhTfDy5NncO8ntS7vaKs7sCNdDaNGOEi", - uint64_t{0x3e3256b60c428000}, uint64_t{0x698df622ef465b0a}}, - {"4ZoEIrJtstiCkeew3oRzmyJHVt/pAs2pj0HgHFrBPztbQ10NsQ/" - "lM6DM439QVxpznnBSiHMgMQJhER+70l72LqFTO1JiIQ==", - uint64_t{0xfb05bad59ec8705}, uint64_t{0x157583111e1a6026}}, - {"hQPtaYI+wJyxXgwD5n8jGIKFKaFA/" - "P83KqCKZfPthnjwdOFysqEOYwAaZuaaiv4cDyi9TyS8hk5cEbNP/jrI7q6pYGBLbsM=", - uint64_t{0xafdc251dbf97b5f8}, uint64_t{0xaa1388f078e793e0}}, - {"S4gpMSKzMD7CWPsSfLeYyhSpfWOntyuVZdX1xSBjiGvsspwOZcxNKCRIOqAA0moUfOh3I5+" - "juQV4rsqYElMD/gWfDGpsWZKQ", - uint64_t{0x10ec9c92ddb5dcbc}, uint64_t{0xf10d68d0f3309360}}, - {"oswxop+" - "bthuDLT4j0PcoSKby4LhF47ZKg8K17xxHf74UsGCzTBbOz0MM8hQEGlyqDT1iUiAYnaPaUp" - "L2mRK0rcIUYA4qLt5uOw==", - uint64_t{0x9a767d5822c7dac4}, uint64_t{0x2af056184457a3de}}, - {"0II/" - "697p+" - "BtLSjxj5989OXI004TogEb94VUnDzOVSgMXie72cuYRvTFNIBgtXlKfkiUjeqVpd4a+" - "n5bxNOD1TGrjQtzKU5r7obo=", - uint64_t{0xee46254080d6e2db}, uint64_t{0x6d0058e1590b2489}}, - {"E84YZW2qipAlMPmctrg7TKlwLZ68l4L+c0xRDUfyyFrA4MAti0q9sHq3TDFviH0Y+" - "Kq3tEE5srWFA8LM9oomtmvm5PYxoaarWPLc", - uint64_t{0xbbb669588d8bf398}, uint64_t{0x638f287f68817f12}}, - {"x3pa4HIElyZG0Nj7Vdy9IdJIR4izLmypXw5PCmZB5y68QQ4uRaVVi3UthsoJROvbjDJkP2D" - "Q6L/eN8pFeLFzNPKBYzcmuMOb5Ull7w==", - uint64_t{0xdc2afaa529beef44}, uint64_t{0xc46b71fecefd5467}}, - {"jVDKGYIuWOP/" - "QKLdd2wi8B2VJA8Wh0c8PwrXJVM8FOGM3voPDVPyDJOU6QsBDPseoR8uuKd19OZ/" - "zAvSCB+zlf6upAsBlheUKgCfKww=", - uint64_t{0xf1f67391d45013a8}, uint64_t{0x2c8e94679d964e0a}}, - {"mkquunhmYe1aR2wmUz4vcvLEcKBoe6H+kjUok9VUn2+eTSkWs4oDDtJvNCWtY5efJwg/" - "j4PgjRYWtqnrCkhaqJaEvkkOwVfgMIwF3e+d", - uint64_t{0x16fce2b8c65a3429}, uint64_t{0x8612b797ce22503a}}, - {"fRelvKYonTQ+s+rnnvQw+JzGfFoPixtna0vzcSjiDqX5s2Kg2//" - "UGrK+AVCyMUhO98WoB1DDbrsOYSw2QzrcPe0+3ck9sePvb+Q/IRaHbw==", - uint64_t{0xf4b096699f49fe67}, uint64_t{0x59f929babfba7170}}, - {"DUwXFJzagljo44QeJ7/" - "6ZKw4QXV18lhkYT2jglMr8WB3CHUU4vdsytvw6AKv42ZcG6fRkZkq9fpnmXy6xG0aO3WPT1" - "eHuyFirAlkW+zKtwg=", - uint64_t{0xca584c4bc8198682}, uint64_t{0x9527556923fb49a0}}, - {"cYmZCrOOBBongNTr7e4nYn52uQUy2mfe48s50JXx2AZ6cRAt/" - "xRHJ5QbEoEJOeOHsJyM4nbzwFm++SlT6gFZZHJpkXJ92JkR86uS/eV1hJUR", - uint64_t{0xed269fc3818b6aad}, uint64_t{0x1039ab644f5e150b}}, - {"EXeHBDfhwzAKFhsMcH9+2RHwV+mJaN01+9oacF6vgm8mCXRd6jeN9U2oAb0of5c5cO4i+" - "Vb/LlHZSMI490SnHU0bejhSCC2gsC5d2K30ER3iNA==", - uint64_t{0x33f253cbb8fe66a8}, uint64_t{0x7816c83f3aa05e6d}}, - {"FzkzRYoNjkxFhZDso94IHRZaJUP61nFYrh5MwDwv9FNoJ5jyNCY/" - "eazPZk+tbmzDyJIGw2h3GxaWZ9bSlsol/vK98SbkMKCQ/wbfrXRLcDzdd/8=", - uint64_t{0xd0b76b2c1523d99c}, uint64_t{0xf51d2f564518c619}}, - {"Re4aXISCMlYY/XsX7zkIFR04ta03u4zkL9dVbLXMa/q6hlY/CImVIIYRN3VKP4pnd0AUr/" - "ugkyt36JcstAInb4h9rpAGQ7GMVOgBniiMBZ/MGU7H", - uint64_t{0xfd28f0811a2a237f}, uint64_t{0x67d494cff03ac004}}, - {"ueLyMcqJXX+MhO4UApylCN9WlTQ+" - "ltJmItgG7vFUtqs2qNwBMjmAvr5u0sAKd8jpzV0dDPTwchbIeAW5zbtkA2NABJV6hFM48ib" - "4/J3A5mseA3cS8w==", - uint64_t{0x6261fb136482e84}, uint64_t{0x2802d636ced1cfbb}}, - {"6Si7Yi11L+jZMkwaN+GUuzXMrlvEqviEkGOilNq0h8TdQyYKuFXzkYc/" - "q74gP3pVCyiwz9KpVGMM9vfnq36riMHRknkmhQutxLZs5fbmOgEO69HglCU=", - uint64_t{0x458efc750bca7c3a}, uint64_t{0xf64e20bad771cb12}}, - {"Q6AbOofGuTJOegPh9Clm/" - "9crtUMQqylKrTc1fhfJo1tqvpXxhU4k08kntL1RG7woRnFrVh2UoMrL1kjin+s9CanT+" - "y4hHwLqRranl9FjvxfVKm3yvg68", - uint64_t{0xa7e69ff84e5e7c27}, uint64_t{0xb9a6cf84a83e15e}}, - {"ieQEbIPvqY2YfIjHnqfJiO1/MIVRk0RoaG/WWi3kFrfIGiNLCczYoklgaecHMm/" - "1sZ96AjO+a5stQfZbJQwS7Sc1ODABEdJKcTsxeW2hbh9A6CFzpowP1A==", - uint64_t{0x3c59bfd0c29efe9e}, uint64_t{0x8da6630319609301}}, - {"zQUv8hFB3zh2GGl3KTvCmnfzE+" - "SUgQPVaSVIELFX5H9cE3FuVFGmymkPQZJLAyzC90Cmi8GqYCvPqTuAAB//" - "XTJxy4bCcVArgZG9zJXpjowpNBfr3ngWrSE=", - uint64_t{0x10befacc6afd298d}, uint64_t{0x40946a86e2a996f3}}, - {"US4hcC1+op5JKGC7eIs8CUgInjKWKlvKQkapulxW262E/" - "B2ye79QxOexf188u2mFwwe3WTISJHRZzS61IwljqAWAWoBAqkUnW8SHmIDwHUP31J0p5sGd" - "P47L", - uint64_t{0x41d5320b0a38efa7}, uint64_t{0xcab7f5997953fa76}}, - {"9bHUWFna2LNaGF6fQLlkx1Hkt24nrkLE2CmFdWgTQV3FFbUe747SSqYw6ebpTa07MWSpWRP" - "sHesVo2B9tqHbe7eQmqYebPDFnNqrhSdZwFm9arLQVs+7a3Ic6A==", - uint64_t{0x58db1c7450fe17f3}, uint64_t{0x39129ca0e04fc465}}, - {"Kb3DpHRUPhtyqgs3RuXjzA08jGb59hjKTOeFt1qhoINfYyfTt2buKhD6YVffRCPsgK9SeqZ" - "qRPJSyaqsa0ovyq1WnWW8jI/NhvAkZTVHUrX2pC+cD3OPYT05Dag=", - uint64_t{0x6098c055a335b7a6}, uint64_t{0x5238221fd685e1b8}}, - {"gzxyMJIPlU+bJBwhFUCHSofZ/" - "319LxqMoqnt3+L6h2U2+ZXJCSsYpE80xmR0Ta77Jq54o92SMH87HV8dGOaCTuAYF+" - "lDL42SY1P316Cl0sZTS2ow3ZqwGbcPNs/1", - uint64_t{0x1bbacec67845a801}, uint64_t{0x175130c407dbcaab}}, - {"uR7V0TW+FGVMpsifnaBAQ3IGlr1wx5sKd7TChuqRe6OvUXTlD4hKWy8S+" - "8yyOw8lQabism19vOQxfmocEOW/" - "vzY0pEa87qHrAZy4s9fH2Bltu8vaOIe+agYohhYORQ==", - uint64_t{0xc419cfc7442190}, uint64_t{0x2f20e7536c0b0df}}, - {"1UR5eoo2aCwhacjZHaCh9bkOsITp6QunUxHQ2SfeHv0imHetzt/" - "Z70mhyWZBalv6eAx+YfWKCUib2SHDtz/" - "A2dc3hqUWX5VfAV7FQsghPUAtu6IiRatq4YSLpDvKZBQ=", - uint64_t{0xc95e510d94ba270c}, uint64_t{0x2742cb488a04ad56}}, - {"opubR7H63BH7OtY+Avd7QyQ25UZ8kLBdFDsBTwZlY6gA/" - "u+x+" - "czC9AaZMgmQrUy15DH7YMGsvdXnviTtI4eVI4aF1H9Rl3NXMKZgwFOsdTfdcZeeHVRzBBKX" - "8jUfh1il", - uint64_t{0xff1ae05c98089c3f}, uint64_t{0xd6afb593879ff93b}}, - {"DC0kXcSXtfQ9FbSRwirIn5tgPri0sbzHSa78aDZVDUKCMaBGyFU6BmrulywYX8yzvwprdLs" - "oOwTWN2wMjHlPDqrvVHNEjnmufRDblW+nSS+xtKNs3N5xsxXdv6JXDrAB/Q==", - uint64_t{0x90c02b8dceced493}, uint64_t{0xf50ad64caac0ca7f}}, - {"BXRBk+3wEP3Lpm1y75wjoz+PgB0AMzLe8tQ1AYU2/" - "oqrQB2YMC6W+9QDbcOfkGbeH+b7IBkt/" - "gwCMw2HaQsRFEsurXtcQ3YwRuPz5XNaw5NAvrNa67Fm7eRzdE1+hWLKtA8=", - uint64_t{0x9f8a76697ab1aa36}, uint64_t{0x2ade95c4261364ae}}, - {"RRBSvEGYnzR9E45Aps/+WSnpCo/X7gJLO4DRnUqFrJCV/kzWlusLE/" - "6ZU6RoUf2ROwcgEvUiXTGjLs7ts3t9SXnJHxC1KiOzxHdYLMhVvgNd3hVSAXODpKFSkVXND" - "55G2L1W", - uint64_t{0x6ba1bf3d811a531d}, uint64_t{0x5c4f3299faacd07a}}, - {"jeh6Qazxmdi57pa9S3XSnnZFIRrnc6s8QLrah5OX3SB/V2ErSPoEAumavzQPkdKF1/" - "SfvmdL+qgF1C+Yawy562QaFqwVGq7+tW0yxP8FStb56ZRgNI4IOmI30s1Ei7iops9Uuw==", - uint64_t{0x6a418974109c67b4}, uint64_t{0xfffe3bff0ae5e9bc}}, - {"6QO5nnDrY2/" - "wrUXpltlKy2dSBcmK15fOY092CR7KxAjNfaY+" - "aAmtWbbzQk3MjBg03x39afSUN1fkrWACdyQKRaGxgwq6MGNxI6W+8DLWJBHzIXrntrE/" - "ml6fnNXEpxplWJ1vEs4=", - uint64_t{0x8472f1c2b3d230a3}, uint64_t{0x1db785c0005166e4}}, - {"0oPxeEHhqhcFuwonNfLd5jF3RNATGZS6NPoS0WklnzyokbTqcl4BeBkMn07+fDQv83j/" - "BpGUwcWO05f3+DYzocfnizpFjLJemFGsls3gxcBYxcbqWYev51tG3lN9EvRE+X9+Pwww", - uint64_t{0x5e06068f884e73a7}, uint64_t{0xea000d962ad18418}}, - {"naSBSjtOKgAOg8XVbR5cHAW3Y+QL4Pb/JO9/" - "oy6L08wvVRZqo0BrssMwhzBP401Um7A4ppAupbQeJFdMrysY34AuSSNvtNUy5VxjNECwiNt" - "gwYHw7yakDUv8WvonctmnoSPKENegQg==", - uint64_t{0x55290b1a8f170f59}, uint64_t{0xe42aef38359362d9}}, - {"vPyl8DxVeRe1OpilKb9KNwpGkQRtA94UpAHetNh+" - "95V7nIW38v7PpzhnTWIml5kw3So1Si0TXtIUPIbsu32BNhoH7QwFvLM+" - "JACgSpc5e3RjsL6Qwxxi11npwxRmRUqATDeMUfRAjxg=", - uint64_t{0x5501cfd83dfe706a}, uint64_t{0xc8e95657348a3891}}, - {"QC9i2GjdTMuNC1xQJ74ngKfrlA4w3o58FhvNCltdIpuMhHP1YsDA78scQPLbZ3OCUgeQguY" - "f/vw6zAaVKSgwtaykqg5ka/4vhz4hYqWU5ficdXqClHl+zkWEY26slCNYOM5nnDlly8Cj", - uint64_t{0xe43ed13d13a66990}, uint64_t{0xc162eca864f238c6}}, - {"7CNIgQhAHX27nxI0HeB5oUTnTdgKpRDYDKwRcXfSFGP1XeT9nQF6WKCMjL1tBV6x7KuJ91G" - "Zz11F4c+8s+MfqEAEpd4FHzamrMNjGcjCyrVtU6y+7HscMVzr7Q/" - "ODLcPEFztFnwjvCjmHw==", - uint64_t{0xdf43bc375cf5283f}, uint64_t{0xbe1fb373e20579ad}}, - {"Qa/hC2RPXhANSospe+gUaPfjdK/yhQvfm4cCV6/pdvCYWPv8p1kMtKOX3h5/" - "8oZ31fsmx4Axphu5qXJokuhZKkBUJueuMpxRyXpwSWz2wELx5glxF7CM0Fn+" - "OevnkhUn5jsPlG2r5jYlVn8=", - uint64_t{0x8112b806d288d7b5}, uint64_t{0x628a1d4f40aa6ffd}}, - {"kUw/0z4l3a89jTwN5jpG0SHY5km/" - "IVhTjgM5xCiPRLncg40aqWrJ5vcF891AOq5hEpSq0bUCJUMFXgct7kvnys905HjerV7Vs1G" - "y84tgVJ70/2+pAZTsB/PzNOE/G6sOj4+GbTzkQu819OLB", - uint64_t{0xd52a18abb001cb46}, uint64_t{0xa87bdb7456340f90}}, - {"VDdfSDbO8Tdj3T5W0XM3EI7iHh5xpIutiM6dvcJ/fhe23V/srFEkDy5iZf/" - "VnA9kfi2C79ENnFnbOReeuZW1b3MUXB9lgC6U4pOTuC+" - "jHK3Qnpyiqzj7h3ISJSuo2pob7vY6VHZo6Fn7exEqHg==", - uint64_t{0xe12b76a2433a1236}, uint64_t{0x5960ef3ba982c801}}, - {"Ldfvy3ORdquM/R2fIkhH/ONi69mcP1AEJ6n/" - "oropwecAsLJzQSgezSY8bEiEs0VnFTBBsW+RtZY6tDj03fnb3amNUOq1b7jbqyQkL9hpl+" - "2Z2J8IaVSeownWl+bQcsR5/xRktIMckC5AtF4YHfU=", - uint64_t{0x175bf7319cf1fa00}, uint64_t{0x5026586df9a431ec}}, - {"BrbNpb42+" - "VzZAjJw6QLirXzhweCVRfwlczzZ0VX2xluskwBqyfnGovz5EuX79JJ31VNXa5hTkAyQat3l" - "YKRADTdAdwE5PqM1N7YaMqqsqoAAAeuYVXuk5eWCykYmClNdSspegwgCuT+403JigBzi", - uint64_t{0xd63d57b3f67525ae}, uint64_t{0xfe4b8a20fdf0840b}}, - {"gB3NGHJJvVcuPyF0ZSvHwnWSIfmaI7La24VMPQVoIIWF7Z74NltPZZpx2f+cocESM+" - "ILzQW9p+BC8x5IWz7N4Str2WLGKMdgmaBfNkEhSHQDU0IJEOnpUt0HmjhFaBlx0/" - "LTmhua+rQ6Wup8ezLwfg==", - uint64_t{0x933faea858832b73}, uint64_t{0xdcb761867da7072f}}, - {"hTKHlRxx6Pl4gjG+6ksvvj0CWFicUg3WrPdSJypDpq91LUWRni2KF6+" - "81ZoHBFhEBrCdogKqeK+hy9bLDnx7g6rAFUjtn1+cWzQ2YjiOpz4+" - "ROBB7lnwjyTGWzJD1rXtlso1g2qVH8XJVigC5M9AIxM=", - uint64_t{0x53d061e5f8e7c04f}, uint64_t{0xc10d4653667275b7}}, - {"IWQBelSQnhrr0F3BhUpXUIDauhX6f95Qp+A0diFXiUK7irwPG1oqBiqHyK/SH/" - "9S+" - "rln9DlFROAmeFdH0OCJi2tFm4afxYzJTFR4HnR4cG4x12JqHaZLQx6iiu6CE3rtWBVz99oA" - "wCZUOEXIsLU24o2Y", - uint64_t{0xdb4124556dd515e0}, uint64_t{0x727720deec13110b}}, - {"TKo+l+" - "1dOXdLvIrFqeLaHdm0HZnbcdEgOoLVcGRiCbAMR0j5pIFw8D36tefckAS1RCFOH5IgP8yiF" - "T0Gd0a2hI3+" - "fTKA7iK96NekxWeoeqzJyctc6QsoiyBlkZerRxs5RplrxoeNg29kKDTM0K94mnhD9g==", - uint64_t{0x4fb31a0dd681ee71}, uint64_t{0x710b009662858dc9}}, - {"YU4e7G6EfQYvxCFoCrrT0EFgVLHFfOWRTJQJ5gxM3G2b+" - "1kJf9YPrpsxF6Xr6nYtS8reEEbDoZJYqnlk9lXSkVArm88Cqn6d25VCx3+" - "49MqC0trIlXtb7SXUUhwpJK16T0hJUfPH7s5cMZXc6YmmbFuBNPE=", - uint64_t{0x27cc72eefa138e4c}, uint64_t{0xfbf8f7a3ecac1eb7}}, - {"/I/" - "eImMwPo1U6wekNFD1Jxjk9XQVi1D+" - "FPdqcHifYXQuP5aScNQfxMAmaPR2XhuOQhADV5tTVbBKwCDCX4E3jcDNHzCiPvViZF1W27t" - "xaf2BbFQdwKrNCmrtzcluBFYu0XZfc7RU1RmxK/RtnF1qHsq/O4pp", - uint64_t{0x44bc2dfba4bd3ced}, uint64_t{0xb6fc4fcd0722e3df}}, - {"CJTT9WGcY2XykTdo8KodRIA29qsqY0iHzWZRjKHb9alwyJ7RZAE3V5Juv4MY3MeYEr1EPCC" - "MxO7yFXqT8XA8YTjaMp3bafRt17Pw8JC4iKJ1zN+WWKOESrj+" - "3aluGQqn8z1EzqY4PH7rLG575PYeWsP98BugdA==", - uint64_t{0x242da1e3a439bed8}, uint64_t{0x7cb86dcc55104aac}}, - {"ZlhyQwLhXQyIUEnMH/" - "AEW27vh9xrbNKJxpWGtrEmKhd+nFqAfbeNBQjW0SfG1YI0xQkQMHXjuTt4P/" - "EpZRtA47ibZDVS8TtaxwyBjuIDwqcN09eCtpC+Ls+" - "vWDTLmBeDM3u4hmzz4DQAYsLiZYSJcldg9Q3wszw=", - uint64_t{0xdc559c746e35c139}, uint64_t{0x19e71e9b45c3a51e}}, - {"v2KU8y0sCrBghmnm8lzGJlwo6D6ObccAxCf10heoDtYLosk4ztTpLlpSFEyu23MLA1tJkcg" - "Rko04h19QMG0mOw/" - "wc93EXAweriBqXfvdaP85sZABwiKO+6rtS9pacRVpYYhHJeVTQ5NzrvBvi1huxAr+" - "xswhVMfL", - uint64_t{0xd0b0350275b9989}, uint64_t{0x51de38573c2bea48}}, - {"QhKlnIS6BuVCTQsnoE67E/" - "yrgogE8EwO7xLaEGei26m0gEU4OksefJgppDh3X0x0Cs78Dr9IHK5b977CmZlrTRmwhlP8p" - "M+UzXPNRNIZuN3ntOum/QhUWP8SGpirheXENWsXMQ/" - "nxtxakyEtrNkKk471Oov9juP8oQ==", - uint64_t{0xb04489e41d17730c}, uint64_t{0xa73ab6996d6df158}}, - {"/ZRMgnoRt+Uo6fUPr9FqQvKX7syhgVqWu+" - "WUSsiQ68UlN0efSP6Eced5gJZL6tg9gcYJIkhjuQNITU0Q3TjVAnAcobgbJikCn6qZ6pRxK" - "BY4MTiAlfGD3T7R7hwJwx554MAy++Zb/YUFlnCaCJiwQMnowF7aQzwYFCo=", - uint64_t{0x2217285eb4572156}, uint64_t{0x55ef2b8c930817b2}}, - {"NB7tU5fNE8nI+SXGfipc7sRkhnSkUF1krjeo6k+8FITaAtdyz+" - "o7mONgXmGLulBPH9bEwyYhKNVY0L+njNQrZ9YC2aXsFD3PdZsxAFaBT3VXEzh+" - "NGBTjDASNL3mXyS8Yv1iThGfHoY7T4aR0NYGJ+k+pR6f+KrPC96M", - uint64_t{0x12c2e8e68aede73b}, uint64_t{0xb2850bf5fae87157}}, - {"8T6wrqCtEO6/rwxF6lvMeyuigVOLwPipX/FULvwyu+1wa5sQGav/" - "2FsLHUVn6cGSi0LlFwLewGHPFJDLR0u4t7ZUyM//" - "x6da0sWgOa5hzDqjsVGmjxEHXiaXKW3i4iSZNuxoNbMQkIbVML+" - "DkYu9ND0O2swg4itGeVSzXA==", - uint64_t{0x4d612125bdc4fd00}, uint64_t{0xecf3de1acd04651f}}, - {"Ntf1bMRdondtMv1CYr3G80iDJ4WSAlKy5H34XdGruQiCrnRGDBa+" - "eUi7vKp4gp3BBcVGl8eYSasVQQjn7MLvb3BjtXx6c/" - "bCL7JtpzQKaDnPr9GWRxpBXVxKREgMM7d8lm35EODv0w+" - "hQLfVSh8OGs7fsBb68nNWPLeeSOo=", - uint64_t{0x81826b553954464e}, uint64_t{0xcc0a40552559ff32}}, - {"VsSAw72Ro6xks02kaiLuiTEIWBC5bgqr4WDnmP8vglXzAhixk7td926rm9jNimL+" - "kroPSygZ9gl63aF5DCPOACXmsbmhDrAQuUzoh9ZKhWgElLQsrqo1KIjWoZT5b5QfVUXY9lS" - "IBg3U75SqORoTPq7HalxxoIT5diWOcJQi", - uint64_t{0xc2e5d345dc0ddd2d}, uint64_t{0xc385c374f20315b1}}, - {"j+loZ+C87+" - "bJxNVebg94gU0mSLeDulcHs84tQT7BZM2rzDSLiCNxUedHr1ZWJ9ejTiBa0dqy2I2ABc++" - "xzOLcv+//YfibtjKtYggC6/3rv0XCc7xu6d/" - "O6xO+XOBhOWAQ+IHJVHf7wZnDxIXB8AUHsnjEISKj7823biqXjyP3g==", - uint64_t{0x3da6830a9e32631e}, uint64_t{0xb90208a4c7234183}}, - {"f3LlpcPElMkspNtDq5xXyWU62erEaKn7RWKlo540gR6mZsNpK1czV/" - "sOmqaq8XAQLEn68LKj6/" - "cFkJukxRzCa4OF1a7cCAXYFp9+wZDu0bw4y63qbpjhdCl8GO6Z2lkcXy7KOzbPE01ukg7+" - "gN+7uKpoohgAhIwpAKQXmX5xtd0=", - uint64_t{0xc9ae5c8759b4877a}, uint64_t{0x58aa1ca7a4c075d9}}, - }; - - for (const auto& expected_result : expected_results) { - std::string str; - ASSERT_TRUE(absl::Base64Unescape(expected_result.base64_data, &str)); - EXPECT_EQ(absl::hash_internal::Wyhash(str.data(), str.size(), - expected_result.seed, kSalt), - expected_result.hash); - } -} - -} // namespace diff --git a/absl/memory/BUILD.bazel b/absl/memory/BUILD.bazel index d2824a05..389aedf3 100644 --- a/absl/memory/BUILD.bazel +++ b/absl/memory/BUILD.bazel @@ -14,7 +14,6 @@ # limitations under the License. # -load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") load( "//absl:copts/configure_copts.bzl", "ABSL_DEFAULT_COPTS", @@ -30,6 +29,9 @@ cc_library( name = "memory", hdrs = ["memory.h"], copts = ABSL_DEFAULT_COPTS, + defines = select({ + "//conditions:default": [], + }), linkopts = ABSL_DEFAULT_LINKOPTS, deps = [ "//absl/base:core_headers", diff --git a/absl/memory/CMakeLists.txt b/absl/memory/CMakeLists.txt index 78fb7e1b..9d50e1dc 100644 --- a/absl/memory/CMakeLists.txt +++ b/absl/memory/CMakeLists.txt @@ -37,7 +37,7 @@ absl_cc_test( DEPS absl::memory absl::core_headers - gmock_main + GTest::gmock_main ) absl_cc_test( @@ -51,5 +51,5 @@ absl_cc_test( absl::memory absl::config absl::exception_safety_testing - gmock_main + GTest::gmock_main ) diff --git a/absl/memory/memory.h b/absl/memory/memory.h index 2b5ff623..d6332606 100644 --- a/absl/memory/memory.h +++ b/absl/memory/memory.h @@ -420,7 +420,7 @@ struct pointer_traits<T*> { // // A C++11 compatible implementation of C++17's std::allocator_traits. // -#if __cplusplus >= 201703L +#if __cplusplus >= 201703L || (defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) using std::allocator_traits; #else // __cplusplus >= 201703L template <typename Alloc> diff --git a/absl/memory/memory_test.cc b/absl/memory/memory_test.cc index 1990c7ba..5d5719b3 100644 --- a/absl/memory/memory_test.cc +++ b/absl/memory/memory_test.cc @@ -548,22 +548,23 @@ struct MinimalMockAllocator { TEST(AllocatorTraits, FunctionsMinimal) { int trace = 0; int hint; - TestValue x(&trace); + alignas(TestValue) char buffer[sizeof(TestValue)]; + auto* x = reinterpret_cast<TestValue*>(buffer); MinimalMockAllocator mock; using Traits = absl::allocator_traits<MinimalMockAllocator>; - EXPECT_CALL(mock, allocate(7)).WillRepeatedly(Return(&x)); - EXPECT_CALL(mock, deallocate(&x, 7)); + EXPECT_CALL(mock, allocate(7)).WillRepeatedly(Return(x)); + EXPECT_CALL(mock, deallocate(x, 7)); - EXPECT_EQ(&x, Traits::allocate(mock, 7)); + EXPECT_EQ(x, Traits::allocate(mock, 7)); static_cast<void>(Traits::allocate(mock, 7, static_cast<const void*>(&hint))); - EXPECT_EQ(&x, Traits::allocate(mock, 7, static_cast<const void*>(&hint))); - Traits::deallocate(mock, &x, 7); + EXPECT_EQ(x, Traits::allocate(mock, 7, static_cast<const void*>(&hint))); + Traits::deallocate(mock, x, 7); + EXPECT_EQ(0, trace); + Traits::construct(mock, x, &trace); EXPECT_EQ(1, trace); - Traits::construct(mock, &x, &trace); - EXPECT_EQ(2, trace); - Traits::destroy(mock, &x); - EXPECT_EQ(1, trace); + Traits::destroy(mock, x); + EXPECT_EQ(0, trace); EXPECT_EQ(std::numeric_limits<size_t>::max() / sizeof(TestValue), Traits::max_size(mock)); diff --git a/absl/meta/BUILD.bazel b/absl/meta/BUILD.bazel index 5585fcca..fb379251 100644 --- a/absl/meta/BUILD.bazel +++ b/absl/meta/BUILD.bazel @@ -14,7 +14,6 @@ # limitations under the License. # -load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") load( "//absl:copts/configure_copts.bzl", "ABSL_DEFAULT_COPTS", diff --git a/absl/meta/CMakeLists.txt b/absl/meta/CMakeLists.txt index 672ead2f..9de4bd37 100644 --- a/absl/meta/CMakeLists.txt +++ b/absl/meta/CMakeLists.txt @@ -35,7 +35,7 @@ absl_cc_test( ${ABSL_TEST_COPTS} DEPS absl::type_traits - gmock_main + GTest::gmock_main ) # component target diff --git a/absl/meta/type_traits.h b/absl/meta/type_traits.h index d5cb5f3b..d886cb30 100644 --- a/absl/meta/type_traits.h +++ b/absl/meta/type_traits.h @@ -35,7 +35,7 @@ #ifndef ABSL_META_TYPE_TRAITS_H_ #define ABSL_META_TYPE_TRAITS_H_ -#include <stddef.h> +#include <cstddef> #include <functional> #include <type_traits> @@ -47,6 +47,14 @@ #define ABSL_META_INTERNAL_STD_CONSTRUCTION_TRAITS_DONT_CHECK_DESTRUCTION 1 #endif +// Defines the default alignment. `__STDCPP_DEFAULT_NEW_ALIGNMENT__` is a C++17 +// feature. +#if defined(__STDCPP_DEFAULT_NEW_ALIGNMENT__) +#define ABSL_INTERNAL_DEFAULT_NEW_ALIGNMENT __STDCPP_DEFAULT_NEW_ALIGNMENT__ +#else // defined(__STDCPP_DEFAULT_NEW_ALIGNMENT__) +#define ABSL_INTERNAL_DEFAULT_NEW_ALIGNMENT alignof(std::max_align_t) +#endif // defined(__STDCPP_DEFAULT_NEW_ALIGNMENT__) + namespace absl { ABSL_NAMESPACE_BEGIN @@ -499,6 +507,27 @@ struct is_trivially_copy_assignable #endif // ABSL_HAVE_STD_IS_TRIVIALLY_ASSIGNABLE }; +#if defined(__cpp_lib_remove_cvref) && __cpp_lib_remove_cvref >= 201711L +template <typename T> +using remove_cvref = std::remove_cvref<T>; + +template <typename T> +using remove_cvref_t = typename std::remove_cvref<T>::type; +#else +// remove_cvref() +// +// C++11 compatible implementation of std::remove_cvref which was added in +// C++20. +template <typename T> +struct remove_cvref { + using type = + typename std::remove_cv<typename std::remove_reference<T>::type>::type; +}; + +template <typename T> +using remove_cvref_t = typename remove_cvref<T>::type; +#endif + namespace type_traits_internal { // is_trivially_copyable() // @@ -613,7 +642,8 @@ using underlying_type_t = typename std::underlying_type<T>::type; namespace type_traits_internal { -#if __cplusplus >= 201703L +#if (defined(__cpp_lib_is_invocable) && __cpp_lib_is_invocable >= 201703L) || \ + (defined(_MSVC_LANG) && _MSVC_LANG >= 201703L) // std::result_of is deprecated (C++17) or removed (C++20) template<typename> struct result_of; template<typename F, typename... Args> diff --git a/absl/meta/type_traits_test.cc b/absl/meta/type_traits_test.cc index 1aafd0d4..0ef5b665 100644 --- a/absl/meta/type_traits_test.cc +++ b/absl/meta/type_traits_test.cc @@ -942,6 +942,34 @@ TEST(TypeTraitsTest, TestTriviallyCopyable) { absl::type_traits_internal::is_trivially_copyable<Trivial&>::value); } +TEST(TypeTraitsTest, TestRemoveCVRef) { + EXPECT_TRUE( + (std::is_same<typename absl::remove_cvref<int>::type, int>::value)); + EXPECT_TRUE( + (std::is_same<typename absl::remove_cvref<int&>::type, int>::value)); + EXPECT_TRUE( + (std::is_same<typename absl::remove_cvref<int&&>::type, int>::value)); + EXPECT_TRUE(( + std::is_same<typename absl::remove_cvref<const int&>::type, int>::value)); + EXPECT_TRUE( + (std::is_same<typename absl::remove_cvref<int*>::type, int*>::value)); + // Does not remove const in this case. + EXPECT_TRUE((std::is_same<typename absl::remove_cvref<const int*>::type, + const int*>::value)); + EXPECT_TRUE((std::is_same<typename absl::remove_cvref<int[2]>::type, + int[2]>::value)); + EXPECT_TRUE((std::is_same<typename absl::remove_cvref<int(&)[2]>::type, + int[2]>::value)); + EXPECT_TRUE((std::is_same<typename absl::remove_cvref<int(&&)[2]>::type, + int[2]>::value)); + EXPECT_TRUE((std::is_same<typename absl::remove_cvref<const int[2]>::type, + int[2]>::value)); + EXPECT_TRUE((std::is_same<typename absl::remove_cvref<const int(&)[2]>::type, + int[2]>::value)); + EXPECT_TRUE((std::is_same<typename absl::remove_cvref<const int(&&)[2]>::type, + int[2]>::value)); +} + #define ABSL_INTERNAL_EXPECT_ALIAS_EQUIVALENCE(trait_name, ...) \ EXPECT_TRUE((std::is_same<typename std::trait_name<__VA_ARGS__>::type, \ absl::trait_name##_t<__VA_ARGS__>>::value)) diff --git a/absl/numeric/BUILD.bazel b/absl/numeric/BUILD.bazel index ea587bfa..eaa27dfd 100644 --- a/absl/numeric/BUILD.bazel +++ b/absl/numeric/BUILD.bazel @@ -12,7 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") load( "//absl:copts/configure_copts.bzl", "ABSL_DEFAULT_COPTS", @@ -38,6 +37,20 @@ cc_library( ], ) +cc_binary( + name = "bits_benchmark", + testonly = 1, + srcs = ["bits_benchmark.cc"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":bits", + "//absl/base:core_headers", + "//absl/random", + "@com_github_google_benchmark//:benchmark_main", + ], +) + cc_test( name = "bits_test", size = "small", diff --git a/absl/numeric/CMakeLists.txt b/absl/numeric/CMakeLists.txt index 781987dc..26df5cf7 100644 --- a/absl/numeric/CMakeLists.txt +++ b/absl/numeric/CMakeLists.txt @@ -38,7 +38,7 @@ absl_cc_test( absl::bits absl::core_headers absl::random_random - gmock_main + GTest::gmock_main ) absl_cc_library( @@ -73,7 +73,7 @@ absl_cc_test( absl::core_headers absl::hash_testing absl::type_traits - gmock_main + GTest::gmock_main ) # component target diff --git a/absl/numeric/bits.h b/absl/numeric/bits.h index 52013ad4..628cdf50 100644 --- a/absl/numeric/bits.h +++ b/absl/numeric/bits.h @@ -133,7 +133,8 @@ template <class T> ABSL_INTERNAL_CONSTEXPR_CLZ inline typename std::enable_if<std::is_unsigned<T>::value, T>::type bit_width(T x) noexcept { - return std::numeric_limits<T>::digits - countl_zero(x); + return std::numeric_limits<T>::digits - + static_cast<unsigned int>(countl_zero(x)); } // Returns: If x == 0, 0; otherwise the maximal value y such that diff --git a/absl/numeric/bits_benchmark.cc b/absl/numeric/bits_benchmark.cc new file mode 100644 index 00000000..b9759583 --- /dev/null +++ b/absl/numeric/bits_benchmark.cc @@ -0,0 +1,73 @@ +// Copyright 2022 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <cstdint> +#include <vector> + +#include "benchmark/benchmark.h" +#include "absl/base/optimization.h" +#include "absl/numeric/bits.h" +#include "absl/random/random.h" + +namespace absl { +namespace { + +template <typename T> +static void BM_bitwidth(benchmark::State& state) { + const int count = state.range(0); + + absl::BitGen rng; + std::vector<T> values; + values.reserve(count); + for (int i = 0; i < count; ++i) { + values.push_back(absl::Uniform<T>(rng, 0, std::numeric_limits<T>::max())); + } + + while (state.KeepRunningBatch(count)) { + for (int i = 0; i < count; ++i) { + benchmark::DoNotOptimize(values[i]); + } + } +} +BENCHMARK_TEMPLATE(BM_bitwidth, uint8_t)->Range(1, 1 << 20); +BENCHMARK_TEMPLATE(BM_bitwidth, uint16_t)->Range(1, 1 << 20); +BENCHMARK_TEMPLATE(BM_bitwidth, uint32_t)->Range(1, 1 << 20); +BENCHMARK_TEMPLATE(BM_bitwidth, uint64_t)->Range(1, 1 << 20); + +template <typename T> +static void BM_bitwidth_nonzero(benchmark::State& state) { + const int count = state.range(0); + + absl::BitGen rng; + std::vector<T> values; + values.reserve(count); + for (int i = 0; i < count; ++i) { + values.push_back(absl::Uniform<T>(rng, 1, std::numeric_limits<T>::max())); + } + + while (state.KeepRunningBatch(count)) { + for (int i = 0; i < count; ++i) { + const T value = values[i]; + ABSL_ASSUME(value > 0); + benchmark::DoNotOptimize(value); + } + } +} +BENCHMARK_TEMPLATE(BM_bitwidth_nonzero, uint8_t)->Range(1, 1 << 20); +BENCHMARK_TEMPLATE(BM_bitwidth_nonzero, uint16_t)->Range(1, 1 << 20); +BENCHMARK_TEMPLATE(BM_bitwidth_nonzero, uint32_t)->Range(1, 1 << 20); +BENCHMARK_TEMPLATE(BM_bitwidth_nonzero, uint64_t)->Range(1, 1 << 20); + +} // namespace +} // namespace absl diff --git a/absl/numeric/int128.cc b/absl/numeric/int128.cc index 5160df79..8cdcbf05 100644 --- a/absl/numeric/int128.cc +++ b/absl/numeric/int128.cc @@ -42,11 +42,11 @@ namespace { // Returns: 2 inline ABSL_ATTRIBUTE_ALWAYS_INLINE int Fls128(uint128 n) { if (uint64_t hi = Uint128High64(n)) { - ABSL_INTERNAL_ASSUME(hi != 0); + ABSL_ASSUME(hi != 0); return 127 - countl_zero(hi); } const uint64_t low = Uint128Low64(n); - ABSL_INTERNAL_ASSUME(low != 0); + ABSL_ASSUME(low != 0); return 63 - countl_zero(low); } @@ -138,28 +138,21 @@ uint128::uint128(float v) : uint128(MakeUint128FromFloat(v)) {} uint128::uint128(double v) : uint128(MakeUint128FromFloat(v)) {} uint128::uint128(long double v) : uint128(MakeUint128FromFloat(v)) {} +#if !defined(ABSL_HAVE_INTRINSIC_INT128) uint128 operator/(uint128 lhs, uint128 rhs) { -#if defined(ABSL_HAVE_INTRINSIC_INT128) - return static_cast<unsigned __int128>(lhs) / - static_cast<unsigned __int128>(rhs); -#else // ABSL_HAVE_INTRINSIC_INT128 uint128 quotient = 0; uint128 remainder = 0; DivModImpl(lhs, rhs, "ient, &remainder); return quotient; -#endif // ABSL_HAVE_INTRINSIC_INT128 } + uint128 operator%(uint128 lhs, uint128 rhs) { -#if defined(ABSL_HAVE_INTRINSIC_INT128) - return static_cast<unsigned __int128>(lhs) % - static_cast<unsigned __int128>(rhs); -#else // ABSL_HAVE_INTRINSIC_INT128 uint128 quotient = 0; uint128 remainder = 0; DivModImpl(lhs, rhs, "ient, &remainder); return remainder; -#endif // ABSL_HAVE_INTRINSIC_INT128 } +#endif // !defined(ABSL_HAVE_INTRINSIC_INT128) namespace { @@ -339,6 +332,7 @@ std::ostream& operator<<(std::ostream& os, int128 v) { ABSL_NAMESPACE_END } // namespace absl +#ifdef ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL namespace std { constexpr bool numeric_limits<absl::uint128>::is_specialized; constexpr bool numeric_limits<absl::uint128>::is_signed; @@ -388,3 +382,4 @@ constexpr int numeric_limits<absl::int128>::max_exponent10; constexpr bool numeric_limits<absl::int128>::traps; constexpr bool numeric_limits<absl::int128>::tinyness_before; } // namespace std +#endif diff --git a/absl/numeric/int128.h b/absl/numeric/int128.h index 0dd814a8..7a899eec 100644 --- a/absl/numeric/int128.h +++ b/absl/numeric/int128.h @@ -18,6 +18,10 @@ // ----------------------------------------------------------------------------- // // This header file defines 128-bit integer types, `uint128` and `int128`. +// +// TODO(absl-team): This module is inconsistent as many inline `uint128` methods +// are defined in this file, while many inline `int128` methods are defined in +// the `int128_*_intrinsic.inc` files. #ifndef ABSL_NUMERIC_INT128_H_ #define ABSL_NUMERIC_INT128_H_ @@ -40,7 +44,7 @@ // builtin type. We need to make sure not to define operator wchar_t() // alongside operator unsigned short() in these instances. #define ABSL_INTERNAL_WCHAR_T __wchar_t -#if defined(_M_X64) +#if defined(_M_X64) && !defined(_M_ARM64EC) #include <intrin.h> #pragma intrinsic(_umul128) #endif // defined(_M_X64) @@ -582,10 +586,10 @@ inline uint128& uint128::operator=(int128 v) { // Arithmetic operators. -uint128 operator<<(uint128 lhs, int amount); -uint128 operator>>(uint128 lhs, int amount); -uint128 operator+(uint128 lhs, uint128 rhs); -uint128 operator-(uint128 lhs, uint128 rhs); +constexpr uint128 operator<<(uint128 lhs, int amount); +constexpr uint128 operator>>(uint128 lhs, int amount); +constexpr uint128 operator+(uint128 lhs, uint128 rhs); +constexpr uint128 operator-(uint128 lhs, uint128 rhs); uint128 operator*(uint128 lhs, uint128 rhs); uint128 operator/(uint128 lhs, uint128 rhs); uint128 operator%(uint128 lhs, uint128 rhs); @@ -782,16 +786,19 @@ inline uint128::operator long double() const { // Comparison operators. -inline bool operator==(uint128 lhs, uint128 rhs) { +constexpr bool operator==(uint128 lhs, uint128 rhs) { +#if defined(ABSL_HAVE_INTRINSIC_INT128) + return static_cast<unsigned __int128>(lhs) == + static_cast<unsigned __int128>(rhs); +#else return (Uint128Low64(lhs) == Uint128Low64(rhs) && Uint128High64(lhs) == Uint128High64(rhs)); +#endif } -inline bool operator!=(uint128 lhs, uint128 rhs) { - return !(lhs == rhs); -} +constexpr bool operator!=(uint128 lhs, uint128 rhs) { return !(lhs == rhs); } -inline bool operator<(uint128 lhs, uint128 rhs) { +constexpr bool operator<(uint128 lhs, uint128 rhs) { #ifdef ABSL_HAVE_INTRINSIC_INT128 return static_cast<unsigned __int128>(lhs) < static_cast<unsigned __int128>(rhs); @@ -802,118 +809,169 @@ inline bool operator<(uint128 lhs, uint128 rhs) { #endif } -inline bool operator>(uint128 lhs, uint128 rhs) { return rhs < lhs; } +constexpr bool operator>(uint128 lhs, uint128 rhs) { return rhs < lhs; } -inline bool operator<=(uint128 lhs, uint128 rhs) { return !(rhs < lhs); } +constexpr bool operator<=(uint128 lhs, uint128 rhs) { return !(rhs < lhs); } -inline bool operator>=(uint128 lhs, uint128 rhs) { return !(lhs < rhs); } +constexpr bool operator>=(uint128 lhs, uint128 rhs) { return !(lhs < rhs); } // Unary operators. -inline uint128 operator-(uint128 val) { - uint64_t hi = ~Uint128High64(val); - uint64_t lo = ~Uint128Low64(val) + 1; - if (lo == 0) ++hi; // carry - return MakeUint128(hi, lo); +constexpr inline uint128 operator+(uint128 val) { + return val; +} + +constexpr inline int128 operator+(int128 val) { + return val; } -inline bool operator!(uint128 val) { +constexpr uint128 operator-(uint128 val) { +#if defined(ABSL_HAVE_INTRINSIC_INT128) + return -static_cast<unsigned __int128>(val); +#else + return MakeUint128( + ~Uint128High64(val) + static_cast<unsigned long>(Uint128Low64(val) == 0), + ~Uint128Low64(val) + 1); +#endif +} + +constexpr inline bool operator!(uint128 val) { +#if defined(ABSL_HAVE_INTRINSIC_INT128) + return !static_cast<unsigned __int128>(val); +#else return !Uint128High64(val) && !Uint128Low64(val); +#endif } // Logical operators. -inline uint128 operator~(uint128 val) { +constexpr inline uint128 operator~(uint128 val) { +#if defined(ABSL_HAVE_INTRINSIC_INT128) + return ~static_cast<unsigned __int128>(val); +#else return MakeUint128(~Uint128High64(val), ~Uint128Low64(val)); +#endif } -inline uint128 operator|(uint128 lhs, uint128 rhs) { +constexpr inline uint128 operator|(uint128 lhs, uint128 rhs) { +#if defined(ABSL_HAVE_INTRINSIC_INT128) + return static_cast<unsigned __int128>(lhs) | + static_cast<unsigned __int128>(rhs); +#else return MakeUint128(Uint128High64(lhs) | Uint128High64(rhs), - Uint128Low64(lhs) | Uint128Low64(rhs)); + Uint128Low64(lhs) | Uint128Low64(rhs)); +#endif } -inline uint128 operator&(uint128 lhs, uint128 rhs) { +constexpr inline uint128 operator&(uint128 lhs, uint128 rhs) { +#if defined(ABSL_HAVE_INTRINSIC_INT128) + return static_cast<unsigned __int128>(lhs) & + static_cast<unsigned __int128>(rhs); +#else return MakeUint128(Uint128High64(lhs) & Uint128High64(rhs), - Uint128Low64(lhs) & Uint128Low64(rhs)); + Uint128Low64(lhs) & Uint128Low64(rhs)); +#endif } -inline uint128 operator^(uint128 lhs, uint128 rhs) { +constexpr inline uint128 operator^(uint128 lhs, uint128 rhs) { +#if defined(ABSL_HAVE_INTRINSIC_INT128) + return static_cast<unsigned __int128>(lhs) ^ + static_cast<unsigned __int128>(rhs); +#else return MakeUint128(Uint128High64(lhs) ^ Uint128High64(rhs), - Uint128Low64(lhs) ^ Uint128Low64(rhs)); + Uint128Low64(lhs) ^ Uint128Low64(rhs)); +#endif } inline uint128& uint128::operator|=(uint128 other) { - hi_ |= other.hi_; - lo_ |= other.lo_; + *this = *this | other; return *this; } inline uint128& uint128::operator&=(uint128 other) { - hi_ &= other.hi_; - lo_ &= other.lo_; + *this = *this & other; return *this; } inline uint128& uint128::operator^=(uint128 other) { - hi_ ^= other.hi_; - lo_ ^= other.lo_; + *this = *this ^ other; return *this; } // Arithmetic operators. -inline uint128 operator<<(uint128 lhs, int amount) { +constexpr uint128 operator<<(uint128 lhs, int amount) { #ifdef ABSL_HAVE_INTRINSIC_INT128 return static_cast<unsigned __int128>(lhs) << amount; #else // uint64_t shifts of >= 64 are undefined, so we will need some // special-casing. - if (amount < 64) { - if (amount != 0) { - return MakeUint128( - (Uint128High64(lhs) << amount) | (Uint128Low64(lhs) >> (64 - amount)), - Uint128Low64(lhs) << amount); - } - return lhs; - } - return MakeUint128(Uint128Low64(lhs) << (amount - 64), 0); + return amount >= 64 ? MakeUint128(Uint128Low64(lhs) << (amount - 64), 0) + : amount == 0 ? lhs + : MakeUint128((Uint128High64(lhs) << amount) | + (Uint128Low64(lhs) >> (64 - amount)), + Uint128Low64(lhs) << amount); #endif } -inline uint128 operator>>(uint128 lhs, int amount) { +constexpr uint128 operator>>(uint128 lhs, int amount) { #ifdef ABSL_HAVE_INTRINSIC_INT128 return static_cast<unsigned __int128>(lhs) >> amount; #else // uint64_t shifts of >= 64 are undefined, so we will need some // special-casing. - if (amount < 64) { - if (amount != 0) { - return MakeUint128(Uint128High64(lhs) >> amount, - (Uint128Low64(lhs) >> amount) | - (Uint128High64(lhs) << (64 - amount))); - } - return lhs; - } - return MakeUint128(0, Uint128High64(lhs) >> (amount - 64)); + return amount >= 64 ? MakeUint128(0, Uint128High64(lhs) >> (amount - 64)) + : amount == 0 ? lhs + : MakeUint128(Uint128High64(lhs) >> amount, + (Uint128Low64(lhs) >> amount) | + (Uint128High64(lhs) << (64 - amount))); #endif } -inline uint128 operator+(uint128 lhs, uint128 rhs) { - uint128 result = MakeUint128(Uint128High64(lhs) + Uint128High64(rhs), - Uint128Low64(lhs) + Uint128Low64(rhs)); - if (Uint128Low64(result) < Uint128Low64(lhs)) { // check for carry - return MakeUint128(Uint128High64(result) + 1, Uint128Low64(result)); - } - return result; +#if !defined(ABSL_HAVE_INTRINSIC_INT128) +namespace int128_internal { +constexpr uint128 AddResult(uint128 result, uint128 lhs) { + // check for carry + return (Uint128Low64(result) < Uint128Low64(lhs)) + ? MakeUint128(Uint128High64(result) + 1, Uint128Low64(result)) + : result; } +} // namespace int128_internal +#endif -inline uint128 operator-(uint128 lhs, uint128 rhs) { - uint128 result = MakeUint128(Uint128High64(lhs) - Uint128High64(rhs), - Uint128Low64(lhs) - Uint128Low64(rhs)); - if (Uint128Low64(lhs) < Uint128Low64(rhs)) { // check for carry - return MakeUint128(Uint128High64(result) - 1, Uint128Low64(result)); - } - return result; +constexpr uint128 operator+(uint128 lhs, uint128 rhs) { +#if defined(ABSL_HAVE_INTRINSIC_INT128) + return static_cast<unsigned __int128>(lhs) + + static_cast<unsigned __int128>(rhs); +#else + return int128_internal::AddResult( + MakeUint128(Uint128High64(lhs) + Uint128High64(rhs), + Uint128Low64(lhs) + Uint128Low64(rhs)), + lhs); +#endif +} + +#if !defined(ABSL_HAVE_INTRINSIC_INT128) +namespace int128_internal { +constexpr uint128 SubstructResult(uint128 result, uint128 lhs, uint128 rhs) { + // check for carry + return (Uint128Low64(lhs) < Uint128Low64(rhs)) + ? MakeUint128(Uint128High64(result) - 1, Uint128Low64(result)) + : result; +} +} // namespace int128_internal +#endif + +constexpr uint128 operator-(uint128 lhs, uint128 rhs) { +#if defined(ABSL_HAVE_INTRINSIC_INT128) + return static_cast<unsigned __int128>(lhs) - + static_cast<unsigned __int128>(rhs); +#else + return int128_internal::SubstructResult( + MakeUint128(Uint128High64(lhs) - Uint128High64(rhs), + Uint128Low64(lhs) - Uint128Low64(rhs)), + lhs, rhs); +#endif } inline uint128 operator*(uint128 lhs, uint128 rhs) { @@ -922,7 +980,7 @@ inline uint128 operator*(uint128 lhs, uint128 rhs) { // can be used for uint128 storage. return static_cast<unsigned __int128>(lhs) * static_cast<unsigned __int128>(rhs); -#elif defined(_MSC_VER) && defined(_M_X64) +#elif defined(_MSC_VER) && defined(_M_X64) && !defined(_M_ARM64EC) uint64_t carry; uint64_t low = _umul128(Uint128Low64(lhs), Uint128Low64(rhs), &carry); return MakeUint128(Uint128Low64(lhs) * Uint128High64(rhs) + @@ -943,6 +1001,18 @@ inline uint128 operator*(uint128 lhs, uint128 rhs) { #endif // ABSL_HAVE_INTRINSIC128 } +#if defined(ABSL_HAVE_INTRINSIC_INT128) +inline uint128 operator/(uint128 lhs, uint128 rhs) { + return static_cast<unsigned __int128>(lhs) / + static_cast<unsigned __int128>(rhs); +} + +inline uint128 operator%(uint128 lhs, uint128 rhs) { + return static_cast<unsigned __int128>(lhs) % + static_cast<unsigned __int128>(rhs); +} +#endif + // Increment/decrement operators. inline uint128 uint128::operator++(int) { @@ -1000,17 +1070,17 @@ inline int128& int128::operator=(unsigned long long v) { } // Arithmetic operators. - -int128 operator+(int128 lhs, int128 rhs); -int128 operator-(int128 lhs, int128 rhs); +constexpr int128 operator-(int128 v); +constexpr int128 operator+(int128 lhs, int128 rhs); +constexpr int128 operator-(int128 lhs, int128 rhs); int128 operator*(int128 lhs, int128 rhs); int128 operator/(int128 lhs, int128 rhs); int128 operator%(int128 lhs, int128 rhs); -int128 operator|(int128 lhs, int128 rhs); -int128 operator&(int128 lhs, int128 rhs); -int128 operator^(int128 lhs, int128 rhs); -int128 operator<<(int128 lhs, int amount); -int128 operator>>(int128 lhs, int amount); +constexpr int128 operator|(int128 lhs, int128 rhs); +constexpr int128 operator&(int128 lhs, int128 rhs); +constexpr int128 operator^(int128 lhs, int128 rhs); +constexpr int128 operator<<(int128 lhs, int amount); +constexpr int128 operator>>(int128 lhs, int amount); inline int128& int128::operator+=(int128 other) { *this = *this + other; @@ -1062,6 +1132,9 @@ inline int128& int128::operator>>=(int amount) { return *this; } +// Forward declaration for comparison operators. +constexpr bool operator!=(int128 lhs, int128 rhs); + namespace int128_internal { // Casts from unsigned to signed while preserving the underlying binary diff --git a/absl/numeric/int128_have_intrinsic.inc b/absl/numeric/int128_have_intrinsic.inc index d6c76dd3..3945fa29 100644 --- a/absl/numeric/int128_have_intrinsic.inc +++ b/absl/numeric/int128_have_intrinsic.inc @@ -155,7 +155,7 @@ constexpr int128::operator unsigned __int128() const { #if defined(__clang__) && !defined(__ppc64__) inline int128::operator float() const { return static_cast<float>(v_); } -inline int128::operator double () const { return static_cast<double>(v_); } +inline int128::operator double() const { return static_cast<double>(v_); } inline int128::operator long double() const { return static_cast<long double>(v_); @@ -163,8 +163,8 @@ inline int128::operator long double() const { #else // Clang on PowerPC // Forward declaration for conversion operators to floating point types. -int128 operator-(int128 v); -bool operator!=(int128 lhs, int128 rhs); +constexpr int128 operator-(int128 v); +constexpr bool operator!=(int128 lhs, int128 rhs); inline int128::operator float() const { // We must convert the absolute value and then negate as needed, because @@ -199,51 +199,45 @@ inline int128::operator long double() const { // Comparison operators. -inline bool operator==(int128 lhs, int128 rhs) { +constexpr bool operator==(int128 lhs, int128 rhs) { return static_cast<__int128>(lhs) == static_cast<__int128>(rhs); } -inline bool operator!=(int128 lhs, int128 rhs) { +constexpr bool operator!=(int128 lhs, int128 rhs) { return static_cast<__int128>(lhs) != static_cast<__int128>(rhs); } -inline bool operator<(int128 lhs, int128 rhs) { +constexpr bool operator<(int128 lhs, int128 rhs) { return static_cast<__int128>(lhs) < static_cast<__int128>(rhs); } -inline bool operator>(int128 lhs, int128 rhs) { +constexpr bool operator>(int128 lhs, int128 rhs) { return static_cast<__int128>(lhs) > static_cast<__int128>(rhs); } -inline bool operator<=(int128 lhs, int128 rhs) { +constexpr bool operator<=(int128 lhs, int128 rhs) { return static_cast<__int128>(lhs) <= static_cast<__int128>(rhs); } -inline bool operator>=(int128 lhs, int128 rhs) { +constexpr bool operator>=(int128 lhs, int128 rhs) { return static_cast<__int128>(lhs) >= static_cast<__int128>(rhs); } // Unary operators. -inline int128 operator-(int128 v) { - return -static_cast<__int128>(v); -} +constexpr int128 operator-(int128 v) { return -static_cast<__int128>(v); } -inline bool operator!(int128 v) { - return !static_cast<__int128>(v); -} +constexpr bool operator!(int128 v) { return !static_cast<__int128>(v); } -inline int128 operator~(int128 val) { - return ~static_cast<__int128>(val); -} +constexpr int128 operator~(int128 val) { return ~static_cast<__int128>(val); } // Arithmetic operators. -inline int128 operator+(int128 lhs, int128 rhs) { +constexpr int128 operator+(int128 lhs, int128 rhs) { return static_cast<__int128>(lhs) + static_cast<__int128>(rhs); } -inline int128 operator-(int128 lhs, int128 rhs) { +constexpr int128 operator-(int128 lhs, int128 rhs) { return static_cast<__int128>(lhs) - static_cast<__int128>(rhs); } @@ -281,22 +275,22 @@ inline int128& int128::operator--() { return *this; } -inline int128 operator|(int128 lhs, int128 rhs) { +constexpr int128 operator|(int128 lhs, int128 rhs) { return static_cast<__int128>(lhs) | static_cast<__int128>(rhs); } -inline int128 operator&(int128 lhs, int128 rhs) { +constexpr int128 operator&(int128 lhs, int128 rhs) { return static_cast<__int128>(lhs) & static_cast<__int128>(rhs); } -inline int128 operator^(int128 lhs, int128 rhs) { +constexpr int128 operator^(int128 lhs, int128 rhs) { return static_cast<__int128>(lhs) ^ static_cast<__int128>(rhs); } -inline int128 operator<<(int128 lhs, int amount) { +constexpr int128 operator<<(int128 lhs, int amount) { return static_cast<__int128>(lhs) << amount; } -inline int128 operator>>(int128 lhs, int amount) { +constexpr int128 operator>>(int128 lhs, int amount) { return static_cast<__int128>(lhs) >> amount; } diff --git a/absl/numeric/int128_no_intrinsic.inc b/absl/numeric/int128_no_intrinsic.inc index c753771a..8834804c 100644 --- a/absl/numeric/int128_no_intrinsic.inc +++ b/absl/numeric/int128_no_intrinsic.inc @@ -134,10 +134,6 @@ constexpr int128::operator unsigned long long() const { // NOLINT(runtime/int) return static_cast<unsigned long long>(lo_); // NOLINT(runtime/int) } -// Forward declaration for conversion operators to floating point types. -int128 operator-(int128 v); -bool operator!=(int128 lhs, int128 rhs); - inline int128::operator float() const { // We must convert the absolute value and then negate as needed, because // floating point types are typically sign-magnitude. Otherwise, the @@ -169,76 +165,80 @@ inline int128::operator long double() const { // Comparison operators. -inline bool operator==(int128 lhs, int128 rhs) { +constexpr bool operator==(int128 lhs, int128 rhs) { return (Int128Low64(lhs) == Int128Low64(rhs) && Int128High64(lhs) == Int128High64(rhs)); } -inline bool operator!=(int128 lhs, int128 rhs) { - return !(lhs == rhs); -} +constexpr bool operator!=(int128 lhs, int128 rhs) { return !(lhs == rhs); } -inline bool operator<(int128 lhs, int128 rhs) { +constexpr bool operator<(int128 lhs, int128 rhs) { return (Int128High64(lhs) == Int128High64(rhs)) ? (Int128Low64(lhs) < Int128Low64(rhs)) : (Int128High64(lhs) < Int128High64(rhs)); } -inline bool operator>(int128 lhs, int128 rhs) { +constexpr bool operator>(int128 lhs, int128 rhs) { return (Int128High64(lhs) == Int128High64(rhs)) ? (Int128Low64(lhs) > Int128Low64(rhs)) : (Int128High64(lhs) > Int128High64(rhs)); } -inline bool operator<=(int128 lhs, int128 rhs) { - return !(lhs > rhs); -} +constexpr bool operator<=(int128 lhs, int128 rhs) { return !(lhs > rhs); } -inline bool operator>=(int128 lhs, int128 rhs) { - return !(lhs < rhs); -} +constexpr bool operator>=(int128 lhs, int128 rhs) { return !(lhs < rhs); } // Unary operators. -inline int128 operator-(int128 v) { - int64_t hi = ~Int128High64(v); - uint64_t lo = ~Int128Low64(v) + 1; - if (lo == 0) ++hi; // carry - return MakeInt128(hi, lo); +constexpr int128 operator-(int128 v) { + return MakeInt128(~Int128High64(v) + (Int128Low64(v) == 0), + ~Int128Low64(v) + 1); } -inline bool operator!(int128 v) { +constexpr bool operator!(int128 v) { return !Int128Low64(v) && !Int128High64(v); } -inline int128 operator~(int128 val) { +constexpr int128 operator~(int128 val) { return MakeInt128(~Int128High64(val), ~Int128Low64(val)); } // Arithmetic operators. -inline int128 operator+(int128 lhs, int128 rhs) { - int128 result = MakeInt128(Int128High64(lhs) + Int128High64(rhs), - Int128Low64(lhs) + Int128Low64(rhs)); - if (Int128Low64(result) < Int128Low64(lhs)) { // check for carry - return MakeInt128(Int128High64(result) + 1, Int128Low64(result)); - } - return result; +namespace int128_internal { +constexpr int128 SignedAddResult(int128 result, int128 lhs) { + // check for carry + return (Int128Low64(result) < Int128Low64(lhs)) + ? MakeInt128(Int128High64(result) + 1, Int128Low64(result)) + : result; +} +} // namespace int128_internal +constexpr int128 operator+(int128 lhs, int128 rhs) { + return int128_internal::SignedAddResult( + MakeInt128(Int128High64(lhs) + Int128High64(rhs), + Int128Low64(lhs) + Int128Low64(rhs)), + lhs); } -inline int128 operator-(int128 lhs, int128 rhs) { - int128 result = MakeInt128(Int128High64(lhs) - Int128High64(rhs), - Int128Low64(lhs) - Int128Low64(rhs)); - if (Int128Low64(lhs) < Int128Low64(rhs)) { // check for carry - return MakeInt128(Int128High64(result) - 1, Int128Low64(result)); - } - return result; +namespace int128_internal { +constexpr int128 SignedSubstructResult(int128 result, int128 lhs, int128 rhs) { + // check for carry + return (Int128Low64(lhs) < Int128Low64(rhs)) + ? MakeInt128(Int128High64(result) - 1, Int128Low64(result)) + : result; +} +} // namespace int128_internal +constexpr int128 operator-(int128 lhs, int128 rhs) { + return int128_internal::SignedSubstructResult( + MakeInt128(Int128High64(lhs) - Int128High64(rhs), + Int128Low64(lhs) - Int128Low64(rhs)), + lhs, rhs); } inline int128 operator*(int128 lhs, int128 rhs) { - uint128 result = uint128(lhs) * rhs; - return MakeInt128(int128_internal::BitCastToSigned(Uint128High64(result)), - Uint128Low64(result)); + return MakeInt128( + int128_internal::BitCastToSigned(Uint128High64(uint128(lhs) * rhs)), + Uint128Low64(uint128(lhs) * rhs)); } inline int128 int128::operator++(int) { @@ -263,46 +263,49 @@ inline int128& int128::operator--() { return *this; } -inline int128 operator|(int128 lhs, int128 rhs) { +constexpr int128 operator|(int128 lhs, int128 rhs) { return MakeInt128(Int128High64(lhs) | Int128High64(rhs), Int128Low64(lhs) | Int128Low64(rhs)); } -inline int128 operator&(int128 lhs, int128 rhs) { +constexpr int128 operator&(int128 lhs, int128 rhs) { return MakeInt128(Int128High64(lhs) & Int128High64(rhs), Int128Low64(lhs) & Int128Low64(rhs)); } -inline int128 operator^(int128 lhs, int128 rhs) { +constexpr int128 operator^(int128 lhs, int128 rhs) { return MakeInt128(Int128High64(lhs) ^ Int128High64(rhs), Int128Low64(lhs) ^ Int128Low64(rhs)); } -inline int128 operator<<(int128 lhs, int amount) { - // uint64_t shifts of >= 64 are undefined, so we need some special-casing. - if (amount < 64) { - if (amount != 0) { - return MakeInt128( - (Int128High64(lhs) << amount) | - static_cast<int64_t>(Int128Low64(lhs) >> (64 - amount)), - Int128Low64(lhs) << amount); - } - return lhs; - } - return MakeInt128(static_cast<int64_t>(Int128Low64(lhs) << (amount - 64)), 0); -} - -inline int128 operator>>(int128 lhs, int amount) { - // uint64_t shifts of >= 64 are undefined, so we need some special-casing. - if (amount < 64) { - if (amount != 0) { - return MakeInt128( - Int128High64(lhs) >> amount, - (Int128Low64(lhs) >> amount) | - (static_cast<uint64_t>(Int128High64(lhs)) << (64 - amount))); - } - return lhs; - } - return MakeInt128(0, - static_cast<uint64_t>(Int128High64(lhs) >> (amount - 64))); +constexpr int128 operator<<(int128 lhs, int amount) { + // int64_t shifts of >= 64 are undefined, so we need some special-casing. + return amount >= 64 + ? MakeInt128( + static_cast<int64_t>(Int128Low64(lhs) << (amount - 64)), 0) + : amount == 0 + ? lhs + : MakeInt128( + (Int128High64(lhs) << amount) | + static_cast<int64_t>(Int128Low64(lhs) >> (64 - amount)), + Int128Low64(lhs) << amount); +} + +constexpr int128 operator>>(int128 lhs, int amount) { + // int64_t shifts of >= 64 are undefined, so we need some special-casing. + // The (Int128High64(lhs) >> 32) >> 32 "trick" causes the the most significant + // int64 to be inititialized with all zeros or all ones correctly. It takes + // into account whether the number is negative or positive, and whether the + // current architecture does arithmetic or logical right shifts for negative + // numbers. + return amount >= 64 + ? MakeInt128( + (Int128High64(lhs) >> 32) >> 32, + static_cast<uint64_t>(Int128High64(lhs) >> (amount - 64))) + : amount == 0 + ? lhs + : MakeInt128(Int128High64(lhs) >> amount, + (Int128Low64(lhs) >> amount) | + (static_cast<uint64_t>(Int128High64(lhs)) + << (64 - amount))); } diff --git a/absl/numeric/int128_test.cc b/absl/numeric/int128_test.cc index bc86c714..dd9425d7 100644 --- a/absl/numeric/int128_test.cc +++ b/absl/numeric/int128_test.cc @@ -226,6 +226,11 @@ TEST(Uint128, AllTests) { EXPECT_EQ(test >>= 1, one); EXPECT_EQ(test <<= 1, two); + EXPECT_EQ(big, +big); + EXPECT_EQ(two, +two); + EXPECT_EQ(absl::Uint128Max(), +absl::Uint128Max()); + EXPECT_EQ(zero, +zero); + EXPECT_EQ(big, -(-big)); EXPECT_EQ(two, -((-one) - 1)); EXPECT_EQ(absl::Uint128Max(), -one); @@ -234,6 +239,24 @@ TEST(Uint128, AllTests) { EXPECT_EQ(absl::Uint128Max(), absl::kuint128max); } +TEST(Int128, RightShiftOfNegativeNumbers) { + absl::int128 minus_six = -6; + absl::int128 minus_three = -3; + absl::int128 minus_two = -2; + absl::int128 minus_one = -1; + if ((-6 >> 1) == -3) { + // Right shift is arithmetic (sign propagates) + EXPECT_EQ(minus_six >> 1, minus_three); + EXPECT_EQ(minus_six >> 2, minus_two); + EXPECT_EQ(minus_six >> 65, minus_one); + } else { + // Right shift is logical (zeros shifted in at MSB) + EXPECT_EQ(minus_six >> 1, absl::int128(absl::uint128(minus_six) >> 1)); + EXPECT_EQ(minus_six >> 2, absl::int128(absl::uint128(minus_six) >> 2)); + EXPECT_EQ(minus_six >> 65, absl::int128(absl::uint128(minus_six) >> 65)); + } +} + TEST(Uint128, ConversionTests) { EXPECT_TRUE(absl::MakeUint128(1, 0)); @@ -769,6 +792,19 @@ TEST(Int128, ComparisonTest) { } } +TEST(Int128, UnaryPlusTest) { + int64_t values64[] = {0, 1, 12345, 0x4000000000000000, + std::numeric_limits<int64_t>::max()}; + for (int64_t value : values64) { + SCOPED_TRACE(::testing::Message() << "value = " << value); + + EXPECT_EQ(absl::int128(value), +absl::int128(value)); + EXPECT_EQ(absl::int128(-value), +absl::int128(-value)); + EXPECT_EQ(absl::MakeInt128(value, 0), +absl::MakeInt128(value, 0)); + EXPECT_EQ(absl::MakeInt128(-value, 0), +absl::MakeInt128(-value, 0)); + } +} + TEST(Int128, UnaryNegationTest) { int64_t values64[] = {0, 1, 12345, 0x4000000000000000, std::numeric_limits<int64_t>::max()}; diff --git a/absl/profiling/BUILD.bazel b/absl/profiling/BUILD.bazel new file mode 100644 index 00000000..3392c96c --- /dev/null +++ b/absl/profiling/BUILD.bazel @@ -0,0 +1,129 @@ +# Copyright 2021 The Abseil Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load( + "//absl:copts/configure_copts.bzl", + "ABSL_DEFAULT_COPTS", + "ABSL_DEFAULT_LINKOPTS", + "ABSL_TEST_COPTS", +) + +package(default_visibility = ["//visibility:private"]) + +licenses(["notice"]) + +cc_library( + name = "sample_recorder", + hdrs = ["internal/sample_recorder.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "//absl:__subpackages__", + ], + deps = [ + "//absl/base:config", + "//absl/base:core_headers", + "//absl/synchronization", + "//absl/time", + ], +) + +cc_test( + name = "sample_recorder_test", + srcs = ["internal/sample_recorder_test.cc"], + linkopts = ABSL_DEFAULT_LINKOPTS, + tags = [ + "no_test_wasm", + ], + deps = [ + ":sample_recorder", + "//absl/base:core_headers", + "//absl/synchronization", + "//absl/synchronization:thread_pool", + "//absl/time", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "exponential_biased", + srcs = ["internal/exponential_biased.cc"], + hdrs = ["internal/exponential_biased.h"], + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "//absl:__subpackages__", + ], + deps = [ + "//absl/base:config", + "//absl/base:core_headers", + ], +) + +cc_test( + name = "exponential_biased_test", + size = "small", + srcs = ["internal/exponential_biased_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:private"], + deps = [ + ":exponential_biased", + "//absl/strings", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "periodic_sampler", + srcs = ["internal/periodic_sampler.cc"], + hdrs = ["internal/periodic_sampler.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "//absl:__subpackages__", + ], + deps = [ + ":exponential_biased", + "//absl/base:core_headers", + ], +) + +cc_test( + name = "periodic_sampler_test", + size = "small", + srcs = ["internal/periodic_sampler_test.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = ["//visibility:private"], + deps = [ + ":periodic_sampler", + "//absl/base:core_headers", + "@com_google_googletest//:gtest_main", + ], +) + +cc_binary( + name = "periodic_sampler_benchmark", + testonly = 1, + srcs = ["internal/periodic_sampler_benchmark.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + tags = ["benchmark"], + visibility = ["//visibility:private"], + deps = [ + ":periodic_sampler", + "//absl/base:core_headers", + "@com_github_google_benchmark//:benchmark_main", + ], +) diff --git a/absl/profiling/CMakeLists.txt b/absl/profiling/CMakeLists.txt new file mode 100644 index 00000000..9b3a7102 --- /dev/null +++ b/absl/profiling/CMakeLists.txt @@ -0,0 +1,93 @@ +# Copyright 2021 The Abseil Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +absl_cc_library( + NAME + sample_recorder + HDRS + "internal/sample_recorder.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base + absl::synchronization +) + +absl_cc_test( + NAME + sample_recorder_test + SRCS + "internal/sample_recorder_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::sample_recorder + absl::time + GTest::gmock_main +) + +absl_cc_library( + NAME + exponential_biased + SRCS + "internal/exponential_biased.cc" + HDRS + "internal/exponential_biased.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::core_headers +) + +absl_cc_test( + NAME + exponential_biased_test + SRCS + "internal/exponential_biased_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::exponential_biased + absl::strings + GTest::gmock_main +) + +absl_cc_library( + NAME + periodic_sampler + SRCS + "internal/periodic_sampler.cc" + HDRS + "internal/periodic_sampler.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::core_headers + absl::exponential_biased +) + +absl_cc_test( + NAME + periodic_sampler_test + SRCS + "internal/periodic_sampler_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::core_headers + absl::periodic_sampler + GTest::gmock_main +) + diff --git a/absl/base/internal/exponential_biased.cc b/absl/profiling/internal/exponential_biased.cc index 1b30c061..81d9a757 100644 --- a/absl/base/internal/exponential_biased.cc +++ b/absl/profiling/internal/exponential_biased.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "absl/base/internal/exponential_biased.h" +#include "absl/profiling/internal/exponential_biased.h" #include <stdint.h> @@ -26,7 +26,7 @@ namespace absl { ABSL_NAMESPACE_BEGIN -namespace base_internal { +namespace profiling_internal { // The algorithm generates a random number between 0 and 1 and applies the // inverse cumulative distribution function for an exponential. Specifically: @@ -64,7 +64,7 @@ int64_t ExponentialBiased::GetSkipCount(int64_t mean) { // Assume huge values are bias neutral, retain bias for next call. return std::numeric_limits<int64_t>::max() / 2; } - double value = std::round(interval); + double value = std::rint(interval); bias_ = interval - value; return value; } @@ -88,6 +88,6 @@ void ExponentialBiased::Initialize() { initialized_ = true; } -} // namespace base_internal +} // namespace profiling_internal ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/base/internal/exponential_biased.h b/absl/profiling/internal/exponential_biased.h index 94f79a33..d31f7782 100644 --- a/absl/base/internal/exponential_biased.h +++ b/absl/profiling/internal/exponential_biased.h @@ -12,8 +12,8 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef ABSL_BASE_INTERNAL_EXPONENTIAL_BIASED_H_ -#define ABSL_BASE_INTERNAL_EXPONENTIAL_BIASED_H_ +#ifndef ABSL_PROFILING_INTERNAL_EXPONENTIAL_BIASED_H_ +#define ABSL_PROFILING_INTERNAL_EXPONENTIAL_BIASED_H_ #include <stdint.h> @@ -22,7 +22,7 @@ namespace absl { ABSL_NAMESPACE_BEGIN -namespace base_internal { +namespace profiling_internal { // ExponentialBiased provides a small and fast random number generator for a // rounded exponential distribution. This generator manages very little state, @@ -66,7 +66,7 @@ namespace base_internal { // Adjusting with rounding bias is relatively trivial: // // double value = bias_ + exponential_distribution(mean)(); -// double rounded_value = std::round(value); +// double rounded_value = std::rint(value); // bias_ = value - rounded_value; // return rounded_value; // @@ -123,8 +123,8 @@ inline uint64_t ExponentialBiased::NextRandom(uint64_t rnd) { return (prng_mult * rnd + prng_add) & prng_mod_mask; } -} // namespace base_internal +} // namespace profiling_internal ABSL_NAMESPACE_END } // namespace absl -#endif // ABSL_BASE_INTERNAL_EXPONENTIAL_BIASED_H_ +#endif // ABSL_PROFILING_INTERNAL_EXPONENTIAL_BIASED_H_ diff --git a/absl/base/internal/exponential_biased_test.cc b/absl/profiling/internal/exponential_biased_test.cc index 075583ca..6a6c317e 100644 --- a/absl/base/internal/exponential_biased_test.cc +++ b/absl/profiling/internal/exponential_biased_test.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "absl/base/internal/exponential_biased.h" +#include "absl/profiling/internal/exponential_biased.h" #include <stddef.h> @@ -28,7 +28,8 @@ using ::testing::Ge; namespace absl { ABSL_NAMESPACE_BEGIN -namespace base_internal { +namespace profiling_internal { +namespace { MATCHER_P2(IsBetween, a, b, absl::StrCat(std::string(negation ? "isn't" : "is"), " between ", a, @@ -194,6 +195,7 @@ TEST(ExponentialBiasedTest, InitializationModes) { EXPECT_THAT(eb_stack.GetSkipCount(2), Ge(0)); } -} // namespace base_internal +} // namespace +} // namespace profiling_internal ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/base/internal/periodic_sampler.cc b/absl/profiling/internal/periodic_sampler.cc index 520dabba..a738a82c 100644 --- a/absl/base/internal/periodic_sampler.cc +++ b/absl/profiling/internal/periodic_sampler.cc @@ -12,15 +12,15 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "absl/base/internal/periodic_sampler.h" +#include "absl/profiling/internal/periodic_sampler.h" #include <atomic> -#include "absl/base/internal/exponential_biased.h" +#include "absl/profiling/internal/exponential_biased.h" namespace absl { ABSL_NAMESPACE_BEGIN -namespace base_internal { +namespace profiling_internal { int64_t PeriodicSamplerBase::GetExponentialBiased(int period) noexcept { return rng_.GetStride(period); @@ -48,6 +48,6 @@ bool PeriodicSamplerBase::SubtleConfirmSample() noexcept { return true; } -} // namespace base_internal +} // namespace profiling_internal ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/base/internal/periodic_sampler.h b/absl/profiling/internal/periodic_sampler.h index f8a86796..54f0af45 100644 --- a/absl/base/internal/periodic_sampler.h +++ b/absl/profiling/internal/periodic_sampler.h @@ -12,19 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -#ifndef ABSL_BASE_INTERNAL_PERIODIC_SAMPLER_H_ -#define ABSL_BASE_INTERNAL_PERIODIC_SAMPLER_H_ +#ifndef ABSL_PROFILING_INTERNAL_PERIODIC_SAMPLER_H_ +#define ABSL_PROFILING_INTERNAL_PERIODIC_SAMPLER_H_ #include <stdint.h> #include <atomic> -#include "absl/base/internal/exponential_biased.h" #include "absl/base/optimization.h" +#include "absl/profiling/internal/exponential_biased.h" namespace absl { ABSL_NAMESPACE_BEGIN -namespace base_internal { +namespace profiling_internal { // PeriodicSamplerBase provides the basic period sampler implementation. // @@ -149,7 +149,7 @@ class PeriodicSamplerBase { // ICC x64 (OK) : https://gcc.godbolt.org/z/ptTNfD // MSVC x64 (OK) : https://gcc.godbolt.org/z/76j4-5 uint64_t stride_ = 0; - ExponentialBiased rng_; + absl::profiling_internal::ExponentialBiased rng_; }; inline bool PeriodicSamplerBase::SubtleMaybeSample() noexcept { @@ -204,8 +204,8 @@ class PeriodicSampler final : public PeriodicSamplerBase { template <typename Tag, int default_period> std::atomic<int> PeriodicSampler<Tag, default_period>::period_(default_period); -} // namespace base_internal +} // namespace profiling_internal ABSL_NAMESPACE_END } // namespace absl -#endif // ABSL_BASE_INTERNAL_PERIODIC_SAMPLER_H_ +#endif // ABSL_PROFILING_INTERNAL_PERIODIC_SAMPLER_H_ diff --git a/absl/base/internal/periodic_sampler_benchmark.cc b/absl/profiling/internal/periodic_sampler_benchmark.cc index 5ad469ce..8f0e5574 100644 --- a/absl/base/internal/periodic_sampler_benchmark.cc +++ b/absl/profiling/internal/periodic_sampler_benchmark.cc @@ -12,12 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. +#include "absl/profiling/internal/periodic_sampler.h" #include "benchmark/benchmark.h" -#include "absl/base/internal/periodic_sampler.h" namespace absl { ABSL_NAMESPACE_BEGIN -namespace base_internal { +namespace profiling_internal { namespace { template <typename Sampler> @@ -74,6 +74,6 @@ void BM_PeriodicSampler_Disabled(benchmark::State& state) { BENCHMARK(BM_PeriodicSampler_Disabled); } // namespace -} // namespace base_internal +} // namespace profiling_internal ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/base/internal/periodic_sampler_test.cc b/absl/profiling/internal/periodic_sampler_test.cc index 3b301e37..ef986f38 100644 --- a/absl/base/internal/periodic_sampler_test.cc +++ b/absl/profiling/internal/periodic_sampler_test.cc @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -#include "absl/base/internal/periodic_sampler.h" +#include "absl/profiling/internal/periodic_sampler.h" #include <thread> // NOLINT(build/c++11) @@ -23,7 +23,7 @@ namespace absl { ABSL_NAMESPACE_BEGIN -namespace base_internal { +namespace profiling_internal { namespace { using testing::Eq; @@ -172,6 +172,6 @@ TEST(PeriodicSamplerTest, SetGlobalPeriod) { } } // namespace -} // namespace base_internal +} // namespace profiling_internal ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/profiling/internal/sample_recorder.h b/absl/profiling/internal/sample_recorder.h new file mode 100644 index 00000000..5f65983b --- /dev/null +++ b/absl/profiling/internal/sample_recorder.h @@ -0,0 +1,245 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: sample_recorder.h +// ----------------------------------------------------------------------------- +// +// This header file defines a lock-free linked list for recording samples +// collected from a random/stochastic process. +// +// This utility is internal-only. Use at your own risk. + +#ifndef ABSL_PROFILING_INTERNAL_SAMPLE_RECORDER_H_ +#define ABSL_PROFILING_INTERNAL_SAMPLE_RECORDER_H_ + +#include <atomic> +#include <cstddef> +#include <functional> + +#include "absl/base/config.h" +#include "absl/base/thread_annotations.h" +#include "absl/synchronization/mutex.h" +#include "absl/time/time.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace profiling_internal { + +// Sample<T> that has members required for linking samples in the linked list of +// samples maintained by the SampleRecorder. Type T defines the sampled data. +template <typename T> +struct Sample { + // Guards the ability to restore the sample to a pristine state. This + // prevents races with sampling and resurrecting an object. + absl::Mutex init_mu; + T* next = nullptr; + T* dead ABSL_GUARDED_BY(init_mu) = nullptr; + int64_t weight; // How many sampling events were required to sample this one. +}; + +// Holds samples and their associated stack traces with a soft limit of +// `SetHashtablezMaxSamples()`. +// +// Thread safe. +template <typename T> +class SampleRecorder { + public: + SampleRecorder(); + ~SampleRecorder(); + + // Registers for sampling. Returns an opaque registration info. + template <typename... Targs> + T* Register(Targs&&... args); + + // Unregisters the sample. + void Unregister(T* sample); + + // The dispose callback will be called on all samples the moment they are + // being unregistered. Only affects samples that are unregistered after the + // callback has been set. + // Returns the previous callback. + using DisposeCallback = void (*)(const T&); + DisposeCallback SetDisposeCallback(DisposeCallback f); + + // Iterates over all the registered `StackInfo`s. Returning the number of + // samples that have been dropped. + int64_t Iterate(const std::function<void(const T& stack)>& f); + + int32_t GetMaxSamples() const; + void SetMaxSamples(int32_t max); + + private: + void PushNew(T* sample); + void PushDead(T* sample); + template <typename... Targs> + T* PopDead(Targs... args); + + std::atomic<size_t> dropped_samples_; + std::atomic<size_t> size_estimate_; + std::atomic<int32_t> max_samples_{1 << 20}; + + // Intrusive lock free linked lists for tracking samples. + // + // `all_` records all samples (they are never removed from this list) and is + // terminated with a `nullptr`. + // + // `graveyard_.dead` is a circular linked list. When it is empty, + // `graveyard_.dead == &graveyard`. The list is circular so that + // every item on it (even the last) has a non-null dead pointer. This allows + // `Iterate` to determine if a given sample is live or dead using only + // information on the sample itself. + // + // For example, nodes [A, B, C, D, E] with [A, C, E] alive and [B, D] dead + // looks like this (G is the Graveyard): + // + // +---+ +---+ +---+ +---+ +---+ + // all -->| A |--->| B |--->| C |--->| D |--->| E | + // | | | | | | | | | | + // +---+ | | +->| |-+ | | +->| |-+ | | + // | G | +---+ | +---+ | +---+ | +---+ | +---+ + // | | | | | | + // | | --------+ +--------+ | + // +---+ | + // ^ | + // +--------------------------------------+ + // + std::atomic<T*> all_; + T graveyard_; + + std::atomic<DisposeCallback> dispose_; +}; + +template <typename T> +typename SampleRecorder<T>::DisposeCallback +SampleRecorder<T>::SetDisposeCallback(DisposeCallback f) { + return dispose_.exchange(f, std::memory_order_relaxed); +} + +template <typename T> +SampleRecorder<T>::SampleRecorder() + : dropped_samples_(0), size_estimate_(0), all_(nullptr), dispose_(nullptr) { + absl::MutexLock l(&graveyard_.init_mu); + graveyard_.dead = &graveyard_; +} + +template <typename T> +SampleRecorder<T>::~SampleRecorder() { + T* s = all_.load(std::memory_order_acquire); + while (s != nullptr) { + T* next = s->next; + delete s; + s = next; + } +} + +template <typename T> +void SampleRecorder<T>::PushNew(T* sample) { + sample->next = all_.load(std::memory_order_relaxed); + while (!all_.compare_exchange_weak(sample->next, sample, + std::memory_order_release, + std::memory_order_relaxed)) { + } +} + +template <typename T> +void SampleRecorder<T>::PushDead(T* sample) { + if (auto* dispose = dispose_.load(std::memory_order_relaxed)) { + dispose(*sample); + } + + absl::MutexLock graveyard_lock(&graveyard_.init_mu); + absl::MutexLock sample_lock(&sample->init_mu); + sample->dead = graveyard_.dead; + graveyard_.dead = sample; +} + +template <typename T> +template <typename... Targs> +T* SampleRecorder<T>::PopDead(Targs... args) { + absl::MutexLock graveyard_lock(&graveyard_.init_mu); + + // The list is circular, so eventually it collapses down to + // graveyard_.dead == &graveyard_ + // when it is empty. + T* sample = graveyard_.dead; + if (sample == &graveyard_) return nullptr; + + absl::MutexLock sample_lock(&sample->init_mu); + graveyard_.dead = sample->dead; + sample->dead = nullptr; + sample->PrepareForSampling(std::forward<Targs>(args)...); + return sample; +} + +template <typename T> +template <typename... Targs> +T* SampleRecorder<T>::Register(Targs&&... args) { + int64_t size = size_estimate_.fetch_add(1, std::memory_order_relaxed); + if (size > max_samples_.load(std::memory_order_relaxed)) { + size_estimate_.fetch_sub(1, std::memory_order_relaxed); + dropped_samples_.fetch_add(1, std::memory_order_relaxed); + return nullptr; + } + + T* sample = PopDead(args...); + if (sample == nullptr) { + // Resurrection failed. Hire a new warlock. + sample = new T(); + { + absl::MutexLock sample_lock(&sample->init_mu); + sample->PrepareForSampling(std::forward<Targs>(args)...); + } + PushNew(sample); + } + + return sample; +} + +template <typename T> +void SampleRecorder<T>::Unregister(T* sample) { + PushDead(sample); + size_estimate_.fetch_sub(1, std::memory_order_relaxed); +} + +template <typename T> +int64_t SampleRecorder<T>::Iterate( + const std::function<void(const T& stack)>& f) { + T* s = all_.load(std::memory_order_acquire); + while (s != nullptr) { + absl::MutexLock l(&s->init_mu); + if (s->dead == nullptr) { + f(*s); + } + s = s->next; + } + + return dropped_samples_.load(std::memory_order_relaxed); +} + +template <typename T> +void SampleRecorder<T>::SetMaxSamples(int32_t max) { + max_samples_.store(max, std::memory_order_release); +} + +template <typename T> +int32_t SampleRecorder<T>::GetMaxSamples() const { + return max_samples_.load(std::memory_order_acquire); +} + +} // namespace profiling_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_PROFILING_INTERNAL_SAMPLE_RECORDER_H_ diff --git a/absl/profiling/internal/sample_recorder_test.cc b/absl/profiling/internal/sample_recorder_test.cc new file mode 100644 index 00000000..3373329a --- /dev/null +++ b/absl/profiling/internal/sample_recorder_test.cc @@ -0,0 +1,184 @@ +// Copyright 2018 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/profiling/internal/sample_recorder.h" + +#include <atomic> +#include <random> +#include <vector> + +#include "gmock/gmock.h" +#include "absl/base/thread_annotations.h" +#include "absl/synchronization/internal/thread_pool.h" +#include "absl/synchronization/mutex.h" +#include "absl/synchronization/notification.h" +#include "absl/time/time.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace profiling_internal { + +namespace { +using ::absl::synchronization_internal::ThreadPool; +using ::testing::IsEmpty; +using ::testing::UnorderedElementsAre; + +struct Info : public Sample<Info> { + public: + void PrepareForSampling(int64_t w) { weight = w; } + std::atomic<size_t> size; + absl::Time create_time; +}; + +std::vector<size_t> GetSizes(SampleRecorder<Info>* s) { + std::vector<size_t> res; + s->Iterate([&](const Info& info) { + res.push_back(info.size.load(std::memory_order_acquire)); + }); + return res; +} + +std::vector<int64_t> GetWeights(SampleRecorder<Info>* s) { + std::vector<int64_t> res; + s->Iterate([&](const Info& info) { res.push_back(info.weight); }); + return res; +} + +Info* Register(SampleRecorder<Info>* s, int64_t weight, size_t size) { + auto* info = s->Register(weight); + assert(info != nullptr); + info->size.store(size); + return info; +} + +TEST(SampleRecorderTest, Registration) { + SampleRecorder<Info> sampler; + auto* info1 = Register(&sampler, 31, 1); + EXPECT_THAT(GetSizes(&sampler), UnorderedElementsAre(1)); + EXPECT_THAT(GetWeights(&sampler), UnorderedElementsAre(31)); + + auto* info2 = Register(&sampler, 32, 2); + EXPECT_THAT(GetSizes(&sampler), UnorderedElementsAre(1, 2)); + info1->size.store(3); + EXPECT_THAT(GetSizes(&sampler), UnorderedElementsAre(3, 2)); + EXPECT_THAT(GetWeights(&sampler), UnorderedElementsAre(31, 32)); + + sampler.Unregister(info1); + sampler.Unregister(info2); +} + +TEST(SampleRecorderTest, Unregistration) { + SampleRecorder<Info> sampler; + std::vector<Info*> infos; + for (size_t i = 0; i < 3; ++i) { + infos.push_back(Register(&sampler, 33 + i, i)); + } + EXPECT_THAT(GetSizes(&sampler), UnorderedElementsAre(0, 1, 2)); + EXPECT_THAT(GetWeights(&sampler), UnorderedElementsAre(33, 34, 35)); + + sampler.Unregister(infos[1]); + EXPECT_THAT(GetSizes(&sampler), UnorderedElementsAre(0, 2)); + EXPECT_THAT(GetWeights(&sampler), UnorderedElementsAre(33, 35)); + + infos.push_back(Register(&sampler, 36, 3)); + infos.push_back(Register(&sampler, 37, 4)); + EXPECT_THAT(GetSizes(&sampler), UnorderedElementsAre(0, 2, 3, 4)); + EXPECT_THAT(GetWeights(&sampler), UnorderedElementsAre(33, 35, 36, 37)); + sampler.Unregister(infos[3]); + EXPECT_THAT(GetSizes(&sampler), UnorderedElementsAre(0, 2, 4)); + EXPECT_THAT(GetWeights(&sampler), UnorderedElementsAre(33, 35, 37)); + + sampler.Unregister(infos[0]); + sampler.Unregister(infos[2]); + sampler.Unregister(infos[4]); + EXPECT_THAT(GetSizes(&sampler), IsEmpty()); +} + +TEST(SampleRecorderTest, MultiThreaded) { + SampleRecorder<Info> sampler; + Notification stop; + ThreadPool pool(10); + + for (int i = 0; i < 10; ++i) { + pool.Schedule([&sampler, &stop, i]() { + std::random_device rd; + std::mt19937 gen(rd()); + + std::vector<Info*> infoz; + while (!stop.HasBeenNotified()) { + if (infoz.empty()) { + infoz.push_back(sampler.Register(i)); + } + switch (std::uniform_int_distribution<>(0, 2)(gen)) { + case 0: { + infoz.push_back(sampler.Register(i)); + break; + } + case 1: { + size_t p = + std::uniform_int_distribution<>(0, infoz.size() - 1)(gen); + Info* info = infoz[p]; + infoz[p] = infoz.back(); + infoz.pop_back(); + EXPECT_EQ(info->weight, i); + sampler.Unregister(info); + break; + } + case 2: { + absl::Duration oldest = absl::ZeroDuration(); + sampler.Iterate([&](const Info& info) { + oldest = std::max(oldest, absl::Now() - info.create_time); + }); + ASSERT_GE(oldest, absl::ZeroDuration()); + break; + } + } + } + }); + } + // The threads will hammer away. Give it a little bit of time for tsan to + // spot errors. + absl::SleepFor(absl::Seconds(3)); + stop.Notify(); +} + +TEST(SampleRecorderTest, Callback) { + SampleRecorder<Info> sampler; + + auto* info1 = Register(&sampler, 39, 1); + auto* info2 = Register(&sampler, 40, 2); + + static const Info* expected; + + auto callback = [](const Info& info) { + // We can't use `info` outside of this callback because the object will be + // disposed as soon as we return from here. + EXPECT_EQ(&info, expected); + }; + + // Set the callback. + EXPECT_EQ(sampler.SetDisposeCallback(callback), nullptr); + expected = info1; + sampler.Unregister(info1); + + // Unset the callback. + EXPECT_EQ(callback, sampler.SetDisposeCallback(nullptr)); + expected = nullptr; // no more calls. + sampler.Unregister(info2); +} + +} // namespace +} // namespace profiling_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/absl/random/BUILD.bazel b/absl/random/BUILD.bazel index 66ffcbc7..08ecd197 100644 --- a/absl/random/BUILD.bazel +++ b/absl/random/BUILD.bazel @@ -16,7 +16,6 @@ # ABSL random-number generation libraries. -load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test") load( "//absl:copts/configure_copts.bzl", "ABSL_DEFAULT_COPTS", @@ -101,8 +100,7 @@ cc_library( linkopts = ABSL_DEFAULT_LINKOPTS, deps = [ ":seed_gen_exception", - "//absl/container:inlined_vector", - "//absl/random/internal:nonsecure_base", + "//absl/base:config", "//absl/random/internal:pool_urbg", "//absl/random/internal:salted_seed_seq", "//absl/random/internal:seed_material", @@ -129,6 +127,7 @@ cc_library( name = "mock_distributions", testonly = 1, hdrs = ["mock_distributions.h"], + linkopts = ABSL_DEFAULT_LINKOPTS, deps = [ ":distributions", ":mocking_bit_gen", @@ -184,6 +183,9 @@ cc_test( copts = ABSL_TEST_COPTS, flaky = 1, linkopts = ABSL_DEFAULT_LINKOPTS, + tags = [ + "no_test_wasm", + ], deps = [ ":distributions", ":random", @@ -236,6 +238,9 @@ cc_test( ], copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + tags = [ + "no_test_wasm", # Does not converge on WASM. + ], deps = [ ":distributions", ":random", @@ -430,6 +435,9 @@ cc_test( srcs = ["mocking_bit_gen_test.cc"], copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + tags = [ + "no_test_wasm", + ], deps = [ ":bit_gen_ref", ":mock_distributions", @@ -445,6 +453,9 @@ cc_test( srcs = ["mock_distributions_test.cc"], copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + tags = [ + "no_test_wasm", + ], deps = [ ":mock_distributions", ":mocking_bit_gen", @@ -459,6 +470,9 @@ cc_test( srcs = ["examples_test.cc"], copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + tags = [ + "no_test_wasm", + ], deps = [ ":random", "@com_google_googletest//:gtest_main", diff --git a/absl/random/CMakeLists.txt b/absl/random/CMakeLists.txt index 3009a034..d04c7081 100644 --- a/absl/random/CMakeLists.txt +++ b/absl/random/CMakeLists.txt @@ -62,8 +62,8 @@ absl_cc_test( absl::random_random absl::random_internal_sequence_urbg absl::fast_type_id - gmock - gtest_main + GTest::gmock + GTest::gtest_main ) # Internal-only target, do not depend on directly. @@ -119,8 +119,9 @@ absl_cc_library( absl::type_traits absl::utility absl::variant - gmock - gtest + GTest::gmock + GTest::gtest + PUBLIC TESTONLY ) @@ -136,8 +137,8 @@ absl_cc_test( DEPS absl::random_mocking_bit_gen absl::random_random - gmock - gtest_main + GTest::gmock + GTest::gtest_main ) absl_cc_test( @@ -153,8 +154,8 @@ absl_cc_test( absl::random_bit_gen_ref absl::random_mocking_bit_gen absl::random_random - gmock - gtest_main + GTest::gmock + GTest::gtest_main ) absl_cc_library( @@ -222,8 +223,8 @@ absl_cc_library( LINKOPTS ${ABSL_DEFAULT_LINKOPTS} DEPS + absl::config absl::inlined_vector - absl::random_internal_nonsecure_base absl::random_internal_pool_urbg absl::random_internal_salted_seed_seq absl::random_internal_seed_material @@ -245,8 +246,8 @@ absl_cc_test( absl::random_random absl::random_internal_sequence_urbg absl::random_internal_pcg_engine - gmock - gtest_main + GTest::gmock + GTest::gtest_main ) absl_cc_test( @@ -268,8 +269,8 @@ absl_cc_test( absl::raw_logging_internal absl::strings absl::str_format - gmock - gtest_main + GTest::gmock + GTest::gtest_main ) absl_cc_test( @@ -285,8 +286,8 @@ absl_cc_test( absl::random_distributions absl::random_random absl::random_internal_distribution_test_util - gmock - gtest_main + GTest::gmock + GTest::gtest_main ) absl_cc_test( @@ -301,8 +302,8 @@ absl_cc_test( absl::random_distributions absl::random_random absl::raw_logging_internal - gmock - gtest_main + GTest::gmock + GTest::gtest_main ) absl_cc_test( @@ -322,8 +323,8 @@ absl_cc_test( absl::raw_logging_internal absl::strings absl::str_format - gmock - gtest_main + GTest::gmock + GTest::gtest_main ) absl_cc_test( @@ -343,8 +344,8 @@ absl_cc_test( absl::random_random absl::raw_logging_internal absl::strings - gmock - gtest_main + GTest::gmock + GTest::gtest_main ) absl_cc_test( @@ -367,8 +368,8 @@ absl_cc_test( absl::raw_logging_internal absl::strings absl::str_format - gmock - gtest_main + GTest::gmock + GTest::gtest_main ) absl_cc_test( @@ -391,8 +392,8 @@ absl_cc_test( absl::raw_logging_internal absl::strings absl::str_format - gmock - gtest_main + GTest::gmock + GTest::gtest_main ) absl_cc_test( @@ -414,8 +415,8 @@ absl_cc_test( absl::raw_logging_internal absl::strings absl::str_format - gmock - gtest_main + GTest::gmock + GTest::gtest_main ) absl_cc_test( @@ -435,8 +436,8 @@ absl_cc_test( absl::random_random absl::raw_logging_internal absl::strings - gmock - gtest_main + GTest::gmock + GTest::gtest_main ) absl_cc_test( @@ -456,8 +457,8 @@ absl_cc_test( absl::random_internal_sequence_urbg absl::random_random absl::strings - gmock - gtest_main + GTest::gmock + GTest::gtest_main ) absl_cc_test( @@ -477,8 +478,8 @@ absl_cc_test( absl::random_random absl::raw_logging_internal absl::strings - gmock - gtest_main + GTest::gmock + GTest::gtest_main ) absl_cc_test( @@ -492,7 +493,7 @@ absl_cc_test( ${ABSL_DEFAULT_LINKOPTS} DEPS absl::random_random - gtest_main + GTest::gtest_main ) absl_cc_test( @@ -508,8 +509,8 @@ absl_cc_test( absl::random_seed_sequences absl::random_internal_nonsecure_base absl::random_random - gmock - gtest_main + GTest::gmock + GTest::gtest_main ) # Internal-only target, do not depend on directly. @@ -726,7 +727,7 @@ absl_cc_library( ${ABSL_DEFAULT_LINKOPTS} DEPS absl::core_headers - absl::optional + absl::inlined_vector absl::random_internal_pool_urbg absl::random_internal_salted_seed_seq absl::random_internal_seed_material @@ -894,7 +895,7 @@ absl_cc_test( ${ABSL_DEFAULT_LINKOPTS} DEPS absl::random_internal_traits - gtest_main + GTest::gtest_main ) # Internal-only target, do not depend on directly. @@ -911,7 +912,7 @@ absl_cc_test( absl::bits absl::flags absl::random_internal_generate_real - gtest_main + GTest::gtest_main ) # Internal-only target, do not depend on directly. @@ -926,7 +927,7 @@ absl_cc_test( ${ABSL_DEFAULT_LINKOPTS} DEPS absl::random_internal_distribution_test_util - gtest_main + GTest::gtest_main ) # Internal-only target, do not depend on directly. @@ -941,7 +942,7 @@ absl_cc_test( ${ABSL_DEFAULT_LINKOPTS} DEPS absl::random_internal_fastmath - gtest_main + GTest::gtest_main ) # Internal-only target, do not depend on directly. @@ -957,8 +958,8 @@ absl_cc_test( DEPS absl::random_internal_explicit_seed_seq absl::random_seed_sequences - gmock - gtest_main + GTest::gmock + GTest::gtest_main ) # Internal-only target, do not depend on directly. @@ -973,8 +974,8 @@ absl_cc_test( ${ABSL_DEFAULT_LINKOPTS} DEPS absl::random_internal_salted_seed_seq - gmock - gtest_main + GTest::gmock + GTest::gtest_main ) # Internal-only target, do not depend on directly. @@ -990,7 +991,7 @@ absl_cc_test( DEPS absl::core_headers absl::random_internal_distribution_test_util - gtest_main + GTest::gtest_main ) # Internal-only target, do not depend on directly. @@ -1005,7 +1006,7 @@ absl_cc_test( ${ABSL_DEFAULT_LINKOPTS} DEPS absl::random_internal_fast_uniform_bits - gtest_main + GTest::gtest_main ) # Internal-only target, do not depend on directly. @@ -1024,7 +1025,7 @@ absl_cc_test( absl::random_distributions absl::random_seed_sequences absl::strings - gtest_main + GTest::gtest_main ) # Internal-only target, do not depend on directly. @@ -1039,8 +1040,8 @@ absl_cc_test( ${ABSL_DEFAULT_LINKOPTS} DEPS absl::random_internal_seed_material - gmock - gtest_main + GTest::gmock + GTest::gtest_main ) # Internal-only target, do not depend on directly. @@ -1057,7 +1058,7 @@ absl_cc_test( absl::random_internal_pool_urbg absl::span absl::type_traits - gtest_main + GTest::gtest_main ) # Internal-only target, do not depend on directly. @@ -1074,8 +1075,8 @@ absl_cc_test( absl::random_internal_explicit_seed_seq absl::random_internal_pcg_engine absl::time - gmock - gtest_main + GTest::gmock + GTest::gtest_main ) # Internal-only target, do not depend on directly. @@ -1094,8 +1095,8 @@ absl_cc_test( absl::raw_logging_internal absl::strings absl::time - gmock - gtest_main + GTest::gmock + GTest::gtest_main ) # Internal-only target, do not depend on directly. @@ -1111,7 +1112,7 @@ absl_cc_test( DEPS absl::random_internal_randen absl::type_traits - gtest_main + GTest::gtest_main ) # Internal-only target, do not depend on directly. @@ -1127,7 +1128,7 @@ absl_cc_test( DEPS absl::endian absl::random_internal_randen_slow - gtest_main + GTest::gtest_main ) # Internal-only target, do not depend on directly. @@ -1146,8 +1147,8 @@ absl_cc_test( absl::random_internal_randen_hwaes_impl absl::raw_logging_internal absl::str_format - gmock - gtest + GTest::gmock + GTest::gtest ) # Internal-only target, do not depend on directly. @@ -1178,7 +1179,7 @@ absl_cc_test( ${ABSL_DEFAULT_LINKOPTS} DEPS absl::random_internal_uniform_helper - gtest_main + GTest::gtest_main ) # Internal-only target, do not depend on directly. @@ -1193,7 +1194,7 @@ absl_cc_test( ${ABSL_DEFAULT_LINKOPTS} DEPS absl::random_internal_iostream_state_saver - gtest_main + GTest::gtest_main ) # Internal-only target, do not depend on directly. @@ -1210,5 +1211,6 @@ absl_cc_test( absl::random_internal_wide_multiply absl::bits absl::int128 - gtest_main + GTest::gmock + GTest::gtest_main ) diff --git a/absl/random/bernoulli_distribution.h b/absl/random/bernoulli_distribution.h index 25bd0d5c..d81b6ae6 100644 --- a/absl/random/bernoulli_distribution.h +++ b/absl/random/bernoulli_distribution.h @@ -138,16 +138,16 @@ bool bernoulli_distribution::Generate(double p, // 64 bits. // // Second, `c` is constructed by first casting explicitly to a signed - // integer and then converting implicitly to an unsigned integer of the same + // integer and then casting explicitly to an unsigned integer of the same // size. This is done because the hardware conversion instructions produce // signed integers from double; if taken as a uint64_t the conversion would // be wrong for doubles greater than 2^63 (not relevant in this use-case). // If converted directly to an unsigned integer, the compiler would end up // emitting code to handle such large values that are not relevant due to // the known bounds on `c`. To avoid these extra instructions this - // implementation converts first to the signed type and then use the - // implicit conversion to unsigned (which is a no-op). - const uint64_t c = static_cast<int64_t>(p * kP32); + // implementation converts first to the signed type and then convert to + // unsigned (which is a no-op). + const uint64_t c = static_cast<uint64_t>(static_cast<int64_t>(p * kP32)); const uint32_t v = fast_u32(g); // FAST PATH: this path fails with probability 1/2^32. Note that simply // returning v <= c would approximate P very well (up to an absolute error diff --git a/absl/random/beta_distribution_test.cc b/absl/random/beta_distribution_test.cc index 44cdfdd0..c16fbb4f 100644 --- a/absl/random/beta_distribution_test.cc +++ b/absl/random/beta_distribution_test.cc @@ -15,6 +15,7 @@ #include "absl/random/beta_distribution.h" #include <algorithm> +#include <cfloat> #include <cstddef> #include <cstdint> #include <iterator> @@ -44,16 +45,26 @@ namespace { template <typename IntType> class BetaDistributionInterfaceTest : public ::testing::Test {}; -// double-double arithmetic is not supported well by either GCC or Clang; see -// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=99048, -// https://bugs.llvm.org/show_bug.cgi?id=49131, and -// https://bugs.llvm.org/show_bug.cgi?id=49132. Don't bother running these tests -// with double doubles until compiler support is better. -using RealTypes = - std::conditional<absl::numeric_internal::IsDoubleDouble(), - ::testing::Types<float, double>, - ::testing::Types<float, double, long double>>::type; -TYPED_TEST_CASE(BetaDistributionInterfaceTest, RealTypes); +constexpr bool ShouldExerciseLongDoubleTests() { + // long double arithmetic is not supported well by either GCC or Clang on + // most platforms specifically not when implemented in terms of double-double; + // see https://gcc.gnu.org/bugzilla/show_bug.cgi?id=99048, + // https://bugs.llvm.org/show_bug.cgi?id=49131, and + // https://bugs.llvm.org/show_bug.cgi?id=49132. + // So a conservative choice here is to disable long-double tests pretty much + // everywhere except on x64 but only if long double is not implemented as + // double-double. +#if defined(__i686__) && defined(__x86_64__) + return !absl::numeric_internal::IsDoubleDouble(); +#else + return false; +#endif +} + +using RealTypes = std::conditional<ShouldExerciseLongDoubleTests(), + ::testing::Types<float, double, long double>, + ::testing::Types<float, double>>::type; +TYPED_TEST_SUITE(BetaDistributionInterfaceTest, RealTypes); TYPED_TEST(BetaDistributionInterfaceTest, SerializeTest) { // The threshold for whether std::exp(1/a) is finite. @@ -430,13 +441,13 @@ std::string ParamName( return absl::StrReplaceAll(name, {{"+", "_"}, {"-", "_"}, {".", "_"}}); } -INSTANTIATE_TEST_CASE_P( +INSTANTIATE_TEST_SUITE_P( TestSampleStatisticsCombinations, BetaDistributionTest, ::testing::Combine(::testing::Values(0.1, 0.2, 0.9, 1.1, 2.5, 10.0, 123.4), ::testing::Values(0.1, 0.2, 0.9, 1.1, 2.5, 10.0, 123.4)), ParamName); -INSTANTIATE_TEST_CASE_P( +INSTANTIATE_TEST_SUITE_P( TestSampleStatistics_SelectedPairs, BetaDistributionTest, ::testing::Values(std::make_pair(0.5, 1000), std::make_pair(1000, 0.5), std::make_pair(900, 1000), std::make_pair(10000, 20000), @@ -558,6 +569,14 @@ TEST(BetaDistributionTest, StabilityTest) { // dependencies of the distribution change, such as RandU64ToDouble, then this // is also likely to change. TEST(BetaDistributionTest, AlgorithmBounds) { +#if (defined(__i386__) || defined(_M_IX86)) && FLT_EVAL_METHOD != 0 + // We're using an x87-compatible FPU, and intermediate operations are + // performed with 80-bit floats. This produces slightly different results from + // what we expect below. + GTEST_SKIP() + << "Skipping the test because we detected x87 floating-point semantics"; +#endif + { absl::random_internal::sequence_urbg urbg( {0x7fbe76c8b4395800ull, 0x8000000000000000ull}); diff --git a/absl/random/bit_gen_ref.h b/absl/random/bit_gen_ref.h index 9555460f..e475221a 100644 --- a/absl/random/bit_gen_ref.h +++ b/absl/random/bit_gen_ref.h @@ -24,6 +24,10 @@ #ifndef ABSL_RANDOM_BIT_GEN_REF_H_ #define ABSL_RANDOM_BIT_GEN_REF_H_ +#include <limits> +#include <type_traits> +#include <utility> + #include "absl/base/internal/fast_type_id.h" #include "absl/base/macros.h" #include "absl/meta/type_traits.h" diff --git a/absl/random/discrete_distribution_test.cc b/absl/random/discrete_distribution_test.cc index 6d007006..415b14cc 100644 --- a/absl/random/discrete_distribution_test.cc +++ b/absl/random/discrete_distribution_test.cc @@ -99,6 +99,7 @@ TYPED_TEST(DiscreteDistributionTypeTest, Constructor) { } TEST(DiscreteDistributionTest, InitDiscreteDistribution) { + using testing::_; using testing::Pair; { @@ -111,8 +112,8 @@ TEST(DiscreteDistributionTest, InitDiscreteDistribution) { // Each bucket is p=1/3, so bucket 0 will send half it's traffic // to bucket 2, while the rest will retain all of their traffic. EXPECT_THAT(q, testing::ElementsAre(Pair(0.5, 2), // - Pair(1.0, 1), // - Pair(1.0, 2))); + Pair(1.0, _), // + Pair(1.0, _))); } { @@ -135,7 +136,7 @@ TEST(DiscreteDistributionTest, InitDiscreteDistribution) { EXPECT_THAT(q, testing::ElementsAre(Pair(b0, 3), // Pair(b1, 3), // - Pair(1.0, 2), // + Pair(1.0, _), // Pair(b3, 2), // Pair(b1, 3))); } diff --git a/absl/random/distributions.h b/absl/random/distributions.h index 31c79694..37fc3aa7 100644 --- a/absl/random/distributions.h +++ b/absl/random/distributions.h @@ -373,7 +373,7 @@ RealType Gaussian(URBG&& urbg, // NOLINT(runtime/references) template <typename IntType, typename URBG> IntType LogUniform(URBG&& urbg, // NOLINT(runtime/references) IntType lo, IntType hi, IntType base = 2) { - static_assert(std::is_integral<IntType>::value, + static_assert(random_internal::IsIntegral<IntType>::value, "Template-argument 'IntType' must be an integral type, in " "absl::LogUniform<IntType, URBG>(...)"); @@ -403,7 +403,7 @@ IntType LogUniform(URBG&& urbg, // NOLINT(runtime/references) template <typename IntType, typename URBG> IntType Poisson(URBG&& urbg, // NOLINT(runtime/references) double mean = 1.0) { - static_assert(std::is_integral<IntType>::value, + static_assert(random_internal::IsIntegral<IntType>::value, "Template-argument 'IntType' must be an integral type, in " "absl::Poisson<IntType, URBG>(...)"); @@ -435,7 +435,7 @@ template <typename IntType, typename URBG> IntType Zipf(URBG&& urbg, // NOLINT(runtime/references) IntType hi = (std::numeric_limits<IntType>::max)(), double q = 2.0, double v = 1.0) { - static_assert(std::is_integral<IntType>::value, + static_assert(random_internal::IsIntegral<IntType>::value, "Template-argument 'IntType' must be an integral type, in " "absl::Zipf<IntType, URBG>(...)"); diff --git a/absl/random/distributions_test.cc b/absl/random/distributions_test.cc index 5866a072..5321a11c 100644 --- a/absl/random/distributions_test.cc +++ b/absl/random/distributions_test.cc @@ -14,6 +14,7 @@ #include "absl/random/distributions.h" +#include <cfloat> #include <cmath> #include <cstdint> #include <random> @@ -219,11 +220,21 @@ TEST_F(RandomDistributionsTest, UniformNoBounds) { absl::Uniform<uint16_t>(gen); absl::Uniform<uint32_t>(gen); absl::Uniform<uint64_t>(gen); + absl::Uniform<absl::uint128>(gen); } TEST_F(RandomDistributionsTest, UniformNonsenseRanges) { // The ranges used in this test are undefined behavior. // The results are arbitrary and subject to future changes. + +#if (defined(__i386__) || defined(_M_IX86)) && FLT_EVAL_METHOD != 0 + // We're using an x87-compatible FPU, and intermediate operations can be + // performed with 80-bit floats. This produces slightly different results from + // what we expect below. + GTEST_SKIP() + << "Skipping the test because we detected x87 floating-point semantics"; +#endif + absl::InsecureBitGen gen; // <uint> diff --git a/absl/random/exponential_distribution_test.cc b/absl/random/exponential_distribution_test.cc index af11d61c..3c44d9ec 100644 --- a/absl/random/exponential_distribution_test.cc +++ b/absl/random/exponential_distribution_test.cc @@ -15,6 +15,7 @@ #include "absl/random/exponential_distribution.h" #include <algorithm> +#include <cfloat> #include <cmath> #include <cstddef> #include <cstdint> @@ -57,7 +58,7 @@ using RealTypes = std::conditional<absl::numeric_internal::IsDoubleDouble(), ::testing::Types<float, double>, ::testing::Types<float, double, long double>>::type; -TYPED_TEST_CASE(ExponentialDistributionTypedTest, RealTypes); +TYPED_TEST_SUITE(ExponentialDistributionTypedTest, RealTypes); TYPED_TEST(ExponentialDistributionTypedTest, SerializeTest) { using param_type = @@ -342,8 +343,8 @@ std::string ParamName(const ::testing::TestParamInfo<Param>& info) { return absl::StrReplaceAll(name, {{"+", "_"}, {"-", "_"}, {".", "_"}}); } -INSTANTIATE_TEST_CASE_P(All, ExponentialDistributionTests, - ::testing::ValuesIn(GenParams()), ParamName); +INSTANTIATE_TEST_SUITE_P(All, ExponentialDistributionTests, + ::testing::ValuesIn(GenParams()), ParamName); // NOTE: absl::exponential_distribution is not guaranteed to be stable. TEST(ExponentialDistributionTest, StabilityTest) { @@ -384,6 +385,15 @@ TEST(ExponentialDistributionTest, StabilityTest) { TEST(ExponentialDistributionTest, AlgorithmBounds) { // Relies on absl::uniform_real_distribution, so some of these comments // reference that. + +#if (defined(__i386__) || defined(_M_IX86)) && FLT_EVAL_METHOD != 0 + // We're using an x87-compatible FPU, and intermediate operations can be + // performed with 80-bit floats. This produces slightly different results from + // what we expect below. + GTEST_SKIP() + << "Skipping the test because we detected x87 floating-point semantics"; +#endif + absl::exponential_distribution<double> dist; { diff --git a/absl/random/gaussian_distribution_test.cc b/absl/random/gaussian_distribution_test.cc index c0bac2b0..4584ac92 100644 --- a/absl/random/gaussian_distribution_test.cc +++ b/absl/random/gaussian_distribution_test.cc @@ -54,7 +54,7 @@ using RealTypes = std::conditional<absl::numeric_internal::IsDoubleDouble(), ::testing::Types<float, double>, ::testing::Types<float, double, long double>>::type; -TYPED_TEST_CASE(GaussianDistributionInterfaceTest, RealTypes); +TYPED_TEST_SUITE(GaussianDistributionInterfaceTest, RealTypes); TYPED_TEST(GaussianDistributionInterfaceTest, SerializeTest) { using param_type = diff --git a/absl/random/generators_test.cc b/absl/random/generators_test.cc index 41725f13..14fd24e9 100644 --- a/absl/random/generators_test.cc +++ b/absl/random/generators_test.cc @@ -107,6 +107,8 @@ void TestPoisson(URBG* gen) { absl::Poisson<int64_t>(*gen); absl::Poisson<uint64_t>(*gen); absl::Poisson<uint64_t>(URBG()); + absl::Poisson<absl::int128>(*gen); + absl::Poisson<absl::uint128>(*gen); } template <typename URBG> @@ -126,6 +128,8 @@ void TestZipf(URBG* gen) { absl::Zipf<int64_t>(*gen, 1 << 10); absl::Zipf<uint64_t>(*gen, 1 << 10); absl::Zipf<uint64_t>(URBG(), 1 << 10); + absl::Zipf<absl::int128>(*gen, 1 << 10); + absl::Zipf<absl::uint128>(*gen, 1 << 10); } template <typename URBG> @@ -146,6 +150,8 @@ void TestLogNormal(URBG* gen) { absl::LogUniform<int64_t>(*gen, 0, 1 << 10); absl::LogUniform<uint64_t>(*gen, 0, 1 << 10); absl::LogUniform<uint64_t>(URBG(), 0, 1 << 10); + absl::LogUniform<absl::int128>(*gen, 0, 1 << 10); + absl::LogUniform<absl::uint128>(*gen, 0, 1 << 10); } template <typename URBG> diff --git a/absl/random/internal/BUILD.bazel b/absl/random/internal/BUILD.bazel index 612b1505..fd5b6195 100644 --- a/absl/random/internal/BUILD.bazel +++ b/absl/random/internal/BUILD.bazel @@ -14,8 +14,6 @@ # limitations under the License. # -load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test") - # Internal-only implementation classes for Abseil Random load( "//absl:copts/configure_copts.bzl", @@ -37,7 +35,11 @@ cc_library( hdrs = ["traits.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, - deps = ["//absl/base:config"], + deps = [ + "//absl/base:config", + "//absl/numeric:bits", + "//absl/numeric:int128", + ], ) cc_library( @@ -60,6 +62,7 @@ cc_library( copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, deps = [ + ":traits", "//absl/base:config", "//absl/meta:type_traits", ], @@ -82,6 +85,7 @@ cc_library( deps = [ ":fast_uniform_bits", "//absl/base:core_headers", + "//absl/base:dynamic_annotations", "//absl/base:raw_logging_internal", "//absl/strings", "//absl/types:optional", @@ -218,8 +222,8 @@ cc_library( ":salted_seed_seq", ":seed_material", "//absl/base:core_headers", + "//absl/container:inlined_vector", "//absl/meta:type_traits", - "//absl/types:optional", "//absl/types:span", ], ) @@ -296,6 +300,8 @@ cc_library( ":platform", "//absl/base:config", "//absl/base:core_headers", + "//absl/base:endian", + "//absl/numeric:int128", ], ) @@ -336,6 +342,7 @@ cc_library( ":platform", "//absl/base:config", "//absl/base:core_headers", + "//absl/numeric:int128", ], ) @@ -495,6 +502,7 @@ cc_test( cc_library( name = "mock_helpers", hdrs = ["mock_helpers.h"], + linkopts = ABSL_DEFAULT_LINKOPTS, deps = [ "//absl/base:fast_type_id", "//absl/types:optional", @@ -505,6 +513,7 @@ cc_library( name = "mock_overload_set", testonly = 1, hdrs = ["mock_overload_set.h"], + linkopts = ABSL_DEFAULT_LINKOPTS, deps = [ ":mock_helpers", "//absl/random:mocking_bit_gen", @@ -622,7 +631,7 @@ cc_test( name = "randen_hwaes_test", size = "small", srcs = ["randen_hwaes_test.cc"], - copts = ABSL_TEST_COPTS, + copts = ABSL_TEST_COPTS + ABSL_RANDOM_RANDEN_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, tags = ABSL_RANDOM_NONPORTABLE_TAGS, deps = [ @@ -672,6 +681,7 @@ cc_library( ":traits", "//absl/base:config", "//absl/meta:type_traits", + "//absl/numeric:int128", ], ) @@ -685,6 +695,7 @@ cc_test( "benchmark", "no_test_ios_x86_64", "no_test_loonix", # Crashing. + "no_test_wasm", ], deps = [ ":nanobenchmark", @@ -717,7 +728,6 @@ cc_test( cc_test( name = "iostream_state_saver_test", - size = "small", srcs = ["iostream_state_saver_test.cc"], linkopts = ABSL_DEFAULT_LINKOPTS, deps = [ diff --git a/absl/random/internal/chi_square.cc b/absl/random/internal/chi_square.cc index 640d48ce..fbe01732 100644 --- a/absl/random/internal/chi_square.cc +++ b/absl/random/internal/chi_square.cc @@ -125,7 +125,8 @@ double ChiSquareValue(int dof, double p) { const double variance = 2.0 / (9 * dof); // Cannot use this method if the variance is 0. if (variance != 0) { - return std::pow(z * std::sqrt(variance) + mean, 3.0) * dof; + double term = z * std::sqrt(variance) + mean; + return dof * (term * term * term); } } diff --git a/absl/random/internal/distribution_caller.h b/absl/random/internal/distribution_caller.h index fc81b787..0f162a4e 100644 --- a/absl/random/internal/distribution_caller.h +++ b/absl/random/internal/distribution_caller.h @@ -18,6 +18,7 @@ #define ABSL_RANDOM_INTERNAL_DISTRIBUTION_CALLER_H_ #include <utility> +#include <type_traits> #include "absl/base/config.h" #include "absl/base/internal/fast_type_id.h" @@ -32,6 +33,8 @@ namespace random_internal { // to intercept such calls. template <typename URBG> struct DistributionCaller { + static_assert(!std::is_pointer<URBG>::value, + "You must pass a reference, not a pointer."); // SFINAE to detect whether the URBG type includes a member matching // bool InvokeMock(base_internal::FastTypeIdType, void*, void*). // diff --git a/absl/random/internal/explicit_seed_seq.h b/absl/random/internal/explicit_seed_seq.h index e3aa31a1..25f79153 100644 --- a/absl/random/internal/explicit_seed_seq.h +++ b/absl/random/internal/explicit_seed_seq.h @@ -74,7 +74,7 @@ class ExplicitSeedSeq { template <typename OutIterator> void generate(OutIterator begin, OutIterator end) { for (size_t index = 0; begin != end; begin++) { - *begin = state_.empty() ? 0 : little_endian::FromHost32(state_[index++]); + *begin = state_.empty() ? 0 : state_[index++]; if (index >= state_.size()) { index = 0; } diff --git a/absl/random/internal/explicit_seed_seq_test.cc b/absl/random/internal/explicit_seed_seq_test.cc index a55ad739..e36d5fa0 100644 --- a/absl/random/internal/explicit_seed_seq_test.cc +++ b/absl/random/internal/explicit_seed_seq_test.cc @@ -24,6 +24,8 @@ namespace { +using ::absl::random_internal::ExplicitSeedSeq; + template <typename Sseq> bool ConformsToInterface() { // Check that the SeedSequence can be default-constructed. @@ -64,14 +66,14 @@ TEST(SeedSequences, CheckInterfaces) { EXPECT_TRUE(ConformsToInterface<std::seed_seq>()); // Abseil classes - EXPECT_TRUE(ConformsToInterface<absl::random_internal::ExplicitSeedSeq>()); + EXPECT_TRUE(ConformsToInterface<ExplicitSeedSeq>()); } TEST(ExplicitSeedSeq, DefaultConstructorGeneratesZeros) { const size_t kNumBlocks = 128; uint32_t outputs[kNumBlocks]; - absl::random_internal::ExplicitSeedSeq seq; + ExplicitSeedSeq seq; seq.generate(outputs, &outputs[kNumBlocks]); for (uint32_t& seed : outputs) { @@ -87,8 +89,7 @@ TEST(ExplicitSeeqSeq, SeedMaterialIsForwardedIdentically) { for (uint32_t& seed : seed_material) { seed = urandom(); } - absl::random_internal::ExplicitSeedSeq seq(seed_material, - &seed_material[kNumBlocks]); + ExplicitSeedSeq seq(seed_material, &seed_material[kNumBlocks]); // Check that output is same as seed-material provided to constructor. { @@ -133,17 +134,14 @@ TEST(ExplicitSeedSeq, CopyAndMoveConstructors) { for (uint32_t& entry : entropy) { entry = urandom(); } - absl::random_internal::ExplicitSeedSeq seq_from_entropy(std::begin(entropy), - std::end(entropy)); + ExplicitSeedSeq seq_from_entropy(std::begin(entropy), std::end(entropy)); // Copy constructor. { - absl::random_internal::ExplicitSeedSeq seq_copy(seq_from_entropy); + ExplicitSeedSeq seq_copy(seq_from_entropy); EXPECT_EQ(seq_copy.size(), seq_from_entropy.size()); - std::vector<uint32_t> seeds_1; - seeds_1.resize(1000, 0); - std::vector<uint32_t> seeds_2; - seeds_2.resize(1000, 1); + std::vector<uint32_t> seeds_1(1000, 0); + std::vector<uint32_t> seeds_2(1000, 1); seq_from_entropy.generate(seeds_1.begin(), seeds_1.end()); seq_copy.generate(seeds_2.begin(), seeds_2.end()); @@ -155,13 +153,10 @@ TEST(ExplicitSeedSeq, CopyAndMoveConstructors) { for (uint32_t& entry : entropy) { entry = urandom(); } - absl::random_internal::ExplicitSeedSeq another_seq(std::begin(entropy), - std::end(entropy)); + ExplicitSeedSeq another_seq(std::begin(entropy), std::end(entropy)); - std::vector<uint32_t> seeds_1; - seeds_1.resize(1000, 0); - std::vector<uint32_t> seeds_2; - seeds_2.resize(1000, 0); + std::vector<uint32_t> seeds_1(1000, 0); + std::vector<uint32_t> seeds_2(1000, 0); seq_from_entropy.generate(seeds_1.begin(), seeds_1.end()); another_seq.generate(seeds_2.begin(), seeds_2.end()); @@ -170,7 +165,15 @@ TEST(ExplicitSeedSeq, CopyAndMoveConstructors) { EXPECT_THAT(seeds_1, Not(Pointwise(Eq(), seeds_2))); // Apply the assignment-operator. + // GCC 12 has a false-positive -Wstringop-overflow warning here. +#if ABSL_INTERNAL_HAVE_MIN_GNUC_VERSION(12, 0) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wstringop-overflow" +#endif another_seq = seq_from_entropy; +#if ABSL_INTERNAL_HAVE_MIN_GNUC_VERSION(12, 0) +#pragma GCC diagnostic pop +#endif // Re-generate seeds. seq_from_entropy.generate(seeds_1.begin(), seeds_1.end()); @@ -182,15 +185,13 @@ TEST(ExplicitSeedSeq, CopyAndMoveConstructors) { // Move constructor. { // Get seeds from seed-sequence constructed from entropy. - std::vector<uint32_t> seeds_1; - seeds_1.resize(1000, 0); + std::vector<uint32_t> seeds_1(1000, 0); seq_from_entropy.generate(seeds_1.begin(), seeds_1.end()); // Apply move-constructor move the sequence to another instance. absl::random_internal::ExplicitSeedSeq moved_seq( std::move(seq_from_entropy)); - std::vector<uint32_t> seeds_2; - seeds_2.resize(1000, 1); + std::vector<uint32_t> seeds_2(1000, 1); moved_seq.generate(seeds_2.begin(), seeds_2.end()); // Verify that seeds produced by moved-instance are the same as original. EXPECT_THAT(seeds_1, Pointwise(Eq(), seeds_2)); @@ -202,3 +203,35 @@ TEST(ExplicitSeedSeq, CopyAndMoveConstructors) { EXPECT_THAT(seeds_1, Each(Eq(0))); } } + +TEST(ExplicitSeedSeq, StdURBGGoldenTests) { + // Verify that for std::- URBG instances the results are stable across + // platforms (these should have deterministic output). + { + ExplicitSeedSeq seed_sequence{12, 34, 56}; + std::minstd_rand rng(seed_sequence); + + std::minstd_rand::result_type values[4] = {rng(), rng(), rng(), rng()}; + EXPECT_THAT(values, + testing::ElementsAre(579252, 43785881, 464353103, 1501811174)); + } + + { + ExplicitSeedSeq seed_sequence{12, 34, 56}; + std::mt19937 rng(seed_sequence); + + std::mt19937::result_type values[4] = {rng(), rng(), rng(), rng()}; + EXPECT_THAT(values, testing::ElementsAre(138416803, 151130212, 33817739, + 138416803)); + } + + { + ExplicitSeedSeq seed_sequence{12, 34, 56}; + std::mt19937_64 rng(seed_sequence); + + std::mt19937_64::result_type values[4] = {rng(), rng(), rng(), rng()}; + EXPECT_THAT(values, + testing::ElementsAre(19738651785169348, 1464811352364190456, + 18054685302720800, 19738651785169348)); + } +} diff --git a/absl/random/internal/fast_uniform_bits.h b/absl/random/internal/fast_uniform_bits.h index 425aaf7d..f3a5c00f 100644 --- a/absl/random/internal/fast_uniform_bits.h +++ b/absl/random/internal/fast_uniform_bits.h @@ -22,6 +22,7 @@ #include "absl/base/config.h" #include "absl/meta/type_traits.h" +#include "absl/random/internal/traits.h" namespace absl { ABSL_NAMESPACE_BEGIN @@ -98,7 +99,7 @@ class FastUniformBits { result_type operator()(URBG& g); // NOLINT(runtime/references) private: - static_assert(std::is_unsigned<UIntType>::value, + static_assert(IsUnsigned<UIntType>::value, "Class-template FastUniformBits<> must be parameterized using " "an unsigned type."); diff --git a/absl/random/internal/generate_real.h b/absl/random/internal/generate_real.h index 4f62873d..b569450c 100644 --- a/absl/random/internal/generate_real.h +++ b/absl/random/internal/generate_real.h @@ -50,10 +50,10 @@ struct GenerateSignedTag {}; // inputs, otherwise it never returns 0. // // When a value in U(0,1) is required, use: -// Uniform64ToReal<double, PositiveValueT, true>; +// GenerateRealFromBits<double, PositiveValueT, true>; // // When a value in U(-1,1) is required, use: -// Uniform64ToReal<double, SignedValueT, false>; +// GenerateRealFromBits<double, SignedValueT, false>; // // This generates more distinct values than the mathematical equivalent // `U(0, 1) * 2.0 - 1.0`. @@ -127,10 +127,8 @@ inline RealType GenerateRealFromBits(uint64_t bits, int exp_bias = 0) { // Construct the 32-bit or 64-bit IEEE 754 floating-point value from // the individual fields: sign, exp, mantissa(bits). - uint_type val = - (std::is_same<SignedTag, GeneratePositiveTag>::value ? 0u : sign) | - (static_cast<uint_type>(exp) << kExp) | - (static_cast<uint_type>(bits) & kMask); + uint_type val = sign | (static_cast<uint_type>(exp) << kExp) | + (static_cast<uint_type>(bits) & kMask); // bit_cast to the output-type real_type result; diff --git a/absl/random/internal/mock_helpers.h b/absl/random/internal/mock_helpers.h index 9d6ab21e..882b0518 100644 --- a/absl/random/internal/mock_helpers.h +++ b/absl/random/internal/mock_helpers.h @@ -18,6 +18,7 @@ #include <tuple> #include <type_traits> +#include <utility> #include "absl/base/internal/fast_type_id.h" #include "absl/types/optional.h" diff --git a/absl/random/internal/nonsecure_base.h b/absl/random/internal/nonsecure_base.h index 730fa2ea..c7d7fa4b 100644 --- a/absl/random/internal/nonsecure_base.h +++ b/absl/random/internal/nonsecure_base.h @@ -17,28 +17,82 @@ #include <algorithm> #include <cstdint> -#include <iostream> #include <iterator> -#include <random> -#include <string> #include <type_traits> +#include <utility> #include <vector> #include "absl/base/macros.h" +#include "absl/container/inlined_vector.h" #include "absl/meta/type_traits.h" #include "absl/random/internal/pool_urbg.h" #include "absl/random/internal/salted_seed_seq.h" #include "absl/random/internal/seed_material.h" -#include "absl/types/optional.h" #include "absl/types/span.h" namespace absl { ABSL_NAMESPACE_BEGIN namespace random_internal { +// RandenPoolSeedSeq is a custom seed sequence type where generate() fills the +// provided buffer via the RandenPool entropy source. +class RandenPoolSeedSeq { + private: + struct ContiguousTag {}; + struct BufferTag {}; + + // Generate random unsigned values directly into the buffer. + template <typename Contiguous> + void generate_impl(ContiguousTag, Contiguous begin, Contiguous end) { + const size_t n = std::distance(begin, end); + auto* a = &(*begin); + RandenPool<uint8_t>::Fill( + absl::MakeSpan(reinterpret_cast<uint8_t*>(a), sizeof(*a) * n)); + } + + // Construct a buffer of size n and fill it with values, then copy + // those values into the seed iterators. + template <typename RandomAccessIterator> + void generate_impl(BufferTag, RandomAccessIterator begin, + RandomAccessIterator end) { + const size_t n = std::distance(begin, end); + absl::InlinedVector<uint32_t, 8> data(n, 0); + RandenPool<uint32_t>::Fill(absl::MakeSpan(data.begin(), data.end())); + std::copy(std::begin(data), std::end(data), begin); + } + + public: + using result_type = uint32_t; + + size_t size() { return 0; } + + template <typename OutIterator> + void param(OutIterator) const {} + + template <typename RandomAccessIterator> + void generate(RandomAccessIterator begin, RandomAccessIterator end) { + // RandomAccessIterator must be assignable from uint32_t + if (begin != end) { + using U = typename std::iterator_traits<RandomAccessIterator>::value_type; + // ContiguousTag indicates the common case of a known contiguous buffer, + // which allows directly filling the buffer. In C++20, + // std::contiguous_iterator_tag provides a mechanism for testing this + // capability, however until Abseil's support requirements allow us to + // assume C++20, limit checks to a few common cases. + using TagType = absl::conditional_t< + (std::is_pointer<RandomAccessIterator>::value || + std::is_same<RandomAccessIterator, + typename std::vector<U>::iterator>::value), + ContiguousTag, BufferTag>; + + generate_impl(TagType{}, begin, end); + } + } +}; + // Each instance of NonsecureURBGBase<URBG> will be seeded by variates produced // by a thread-unique URBG-instance. -template <typename URBG> +template <typename URBG, typename Seeder = RandenPoolSeedSeq> class NonsecureURBGBase { public: using result_type = typename URBG::result_type; @@ -85,49 +139,6 @@ class NonsecureURBGBase { } private: - // Seeder is a custom seed sequence type where generate() fills the provided - // buffer via the RandenPool entropy source. - struct Seeder { - using result_type = uint32_t; - - size_t size() { return 0; } - - template <typename OutIterator> - void param(OutIterator) const {} - - template <typename RandomAccessIterator> - void generate(RandomAccessIterator begin, RandomAccessIterator end) { - if (begin != end) { - // begin, end must be random access iterators assignable from uint32_t. - generate_impl( - std::integral_constant<bool, sizeof(*begin) == sizeof(uint32_t)>{}, - begin, end); - } - } - - // Commonly, generate is invoked with a pointer to a buffer which - // can be cast to a uint32_t. - template <typename RandomAccessIterator> - void generate_impl(std::integral_constant<bool, true>, - RandomAccessIterator begin, RandomAccessIterator end) { - auto buffer = absl::MakeSpan(begin, end); - auto target = absl::MakeSpan(reinterpret_cast<uint32_t*>(buffer.data()), - buffer.size()); - RandenPool<uint32_t>::Fill(target); - } - - // The non-uint32_t case should be uncommon, and involves an extra copy, - // filling the uint32_t buffer and then mixing into the output. - template <typename RandomAccessIterator> - void generate_impl(std::integral_constant<bool, false>, - RandomAccessIterator begin, RandomAccessIterator end) { - const size_t n = std::distance(begin, end); - absl::InlinedVector<uint32_t, 8> data(n, 0); - RandenPool<uint32_t>::Fill(absl::MakeSpan(data.begin(), data.end())); - std::copy(std::begin(data), std::end(data), begin); - } - }; - static URBG ConstructURBG() { Seeder seeder; return URBG(seeder); diff --git a/absl/random/internal/nonsecure_base_test.cc b/absl/random/internal/nonsecure_base_test.cc index 698027fc..3502243e 100644 --- a/absl/random/internal/nonsecure_base_test.cc +++ b/absl/random/internal/nonsecure_base_test.cc @@ -15,6 +15,7 @@ #include "absl/random/internal/nonsecure_base.h" #include <algorithm> +#include <cstdint> #include <iostream> #include <memory> #include <random> @@ -192,54 +193,35 @@ TEST(NonsecureURBGBase, EqualSeedSequencesYieldEqualVariates) { } } -// This is a PRNG-compatible type specifically designed to test -// that NonsecureURBGBase::Seeder can correctly handle iterators -// to arbitrary non-uint32_t size types. -template <typename T> -struct SeederTestEngine { - using result_type = T; +TEST(RandenPoolSeedSeqTest, SeederWorksForU32) { + absl::random_internal::RandenPoolSeedSeq seeder; - static constexpr result_type(min)() { - return (std::numeric_limits<result_type>::min)(); - } - static constexpr result_type(max)() { - return (std::numeric_limits<result_type>::max)(); - } - - template <class SeedSequence, - typename = typename absl::enable_if_t< - !std::is_same<SeedSequence, SeederTestEngine>::value>> - explicit SeederTestEngine(SeedSequence&& seq) { - seed(seq); - } - - SeederTestEngine(const SeederTestEngine&) = default; - SeederTestEngine& operator=(const SeederTestEngine&) = default; - SeederTestEngine(SeederTestEngine&&) = default; - SeederTestEngine& operator=(SeederTestEngine&&) = default; + uint32_t state[2] = {0, 0}; + seeder.generate(std::begin(state), std::end(state)); + EXPECT_FALSE(state[0] == 0 && state[1] == 0); +} - result_type operator()() { return state[0]; } +TEST(RandenPoolSeedSeqTest, SeederWorksForU64) { + absl::random_internal::RandenPoolSeedSeq seeder; - template <class SeedSequence> - void seed(SeedSequence&& seq) { - std::fill(std::begin(state), std::end(state), T(0)); - seq.generate(std::begin(state), std::end(state)); - } + uint64_t state[2] = {0, 0}; + seeder.generate(std::begin(state), std::end(state)); + EXPECT_FALSE(state[0] == 0 && state[1] == 0); + EXPECT_FALSE((state[0] >> 32) == 0 && (state[1] >> 32) == 0); +} - T state[2]; -}; +TEST(RandenPoolSeedSeqTest, SeederWorksForS32) { + absl::random_internal::RandenPoolSeedSeq seeder; -TEST(NonsecureURBGBase, SeederWorksForU32) { - using U32 = - absl::random_internal::NonsecureURBGBase<SeederTestEngine<uint32_t>>; - U32 x; - EXPECT_NE(0, x()); + int32_t state[2] = {0, 0}; + seeder.generate(std::begin(state), std::end(state)); + EXPECT_FALSE(state[0] == 0 && state[1] == 0); } -TEST(NonsecureURBGBase, SeederWorksForU64) { - using U64 = - absl::random_internal::NonsecureURBGBase<SeederTestEngine<uint64_t>>; +TEST(RandenPoolSeedSeqTest, SeederWorksForVector) { + absl::random_internal::RandenPoolSeedSeq seeder; - U64 x; - EXPECT_NE(0, x()); + std::vector<uint32_t> state(2); + seeder.generate(std::begin(state), std::end(state)); + EXPECT_FALSE(state[0] == 0 && state[1] == 0); } diff --git a/absl/random/internal/pcg_engine.h b/absl/random/internal/pcg_engine.h index 8efaf2e0..4ab44c94 100644 --- a/absl/random/internal/pcg_engine.h +++ b/absl/random/internal/pcg_engine.h @@ -262,7 +262,7 @@ struct pcg_xsl_rr_128_64 { uint64_t rotate = h >> 58u; uint64_t s = Uint128Low64(state) ^ h; #endif - return rotr(s, rotate); + return rotr(s, static_cast<int>(rotate)); } }; diff --git a/absl/random/internal/pool_urbg.cc b/absl/random/internal/pool_urbg.cc index 5bee5307..725100a4 100644 --- a/absl/random/internal/pool_urbg.cc +++ b/absl/random/internal/pool_urbg.cc @@ -194,11 +194,10 @@ RandenPoolEntry* PoolAlignedAlloc() { // Not all the platforms that we build for have std::aligned_alloc, however // since we never free these objects, we can over allocate and munge the // pointers to the correct alignment. - void* memory = std::malloc(sizeof(RandenPoolEntry) + kAlignment); - auto x = reinterpret_cast<intptr_t>(memory); + intptr_t x = reinterpret_cast<intptr_t>( + new char[sizeof(RandenPoolEntry) + kAlignment]); auto y = x % kAlignment; - void* aligned = - (y == 0) ? memory : reinterpret_cast<void*>(x + kAlignment - y); + void* aligned = reinterpret_cast<void*>(y == 0 ? x : (x + kAlignment - y)); return new (aligned) RandenPoolEntry(); } diff --git a/absl/random/internal/randen.h b/absl/random/internal/randen.h index 9a3840b8..9ff4a7a5 100644 --- a/absl/random/internal/randen.h +++ b/absl/random/internal/randen.h @@ -43,10 +43,8 @@ class Randen { // Generate updates the randen sponge. The outer portion of the sponge // (kCapacityBytes .. kStateBytes) may be consumed as PRNG state. - template <typename T, size_t N> - void Generate(T (&state)[N]) const { - static_assert(N * sizeof(T) == kStateBytes, - "Randen::Generate() requires kStateBytes of state"); + // REQUIRES: state points to kStateBytes of state. + inline void Generate(void* state) const { #if ABSL_RANDOM_INTERNAL_AES_DISPATCH // HW AES Dispatch. if (has_crypto_) { @@ -65,13 +63,9 @@ class Randen { // Absorb incorporates additional seed material into the randen sponge. After // absorb returns, Generate must be called before the state may be consumed. - template <typename S, size_t M, typename T, size_t N> - void Absorb(const S (&seed)[M], T (&state)[N]) const { - static_assert(M * sizeof(S) == RandenTraits::kSeedBytes, - "Randen::Absorb() requires kSeedBytes of seed"); - - static_assert(N * sizeof(T) == RandenTraits::kStateBytes, - "Randen::Absorb() requires kStateBytes of state"); + // REQUIRES: seed points to kSeedBytes of seed. + // REQUIRES: state points to kStateBytes of state. + inline void Absorb(const void* seed, void* state) const { #if ABSL_RANDOM_INTERNAL_AES_DISPATCH // HW AES Dispatch. if (has_crypto_) { diff --git a/absl/random/internal/randen_detect.cc b/absl/random/internal/randen_detect.cc index bbe7b965..6dababa3 100644 --- a/absl/random/internal/randen_detect.cc +++ b/absl/random/internal/randen_detect.cc @@ -24,6 +24,11 @@ #include "absl/random/internal/platform.h" +#if !defined(__UCLIBC__) && defined(__GLIBC__) && \ + (__GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 16)) +#define ABSL_HAVE_GETAUXVAL +#endif + #if defined(ABSL_ARCH_X86_64) #define ABSL_INTERNAL_USE_X86_CPUID #elif defined(ABSL_ARCH_PPC) || defined(ABSL_ARCH_ARM) || \ @@ -31,7 +36,7 @@ #if defined(__ANDROID__) #define ABSL_INTERNAL_USE_ANDROID_GETAUXVAL #define ABSL_INTERNAL_USE_GETAUXVAL -#elif defined(__linux__) +#elif defined(__linux__) && defined(ABSL_HAVE_GETAUXVAL) #define ABSL_INTERNAL_USE_LINUX_GETAUXVAL #define ABSL_INTERNAL_USE_GETAUXVAL #endif @@ -40,7 +45,6 @@ #if defined(ABSL_INTERNAL_USE_X86_CPUID) #if defined(_WIN32) || defined(_WIN64) #include <intrin.h> // NOLINT(build/include_order) -#pragma intrinsic(__cpuid) #else // MSVC-equivalent __cpuid intrinsic function. static void __cpuid(int cpu_info[4], int info_type) { diff --git a/absl/random/internal/randen_engine.h b/absl/random/internal/randen_engine.h index 92bb8905..b4708664 100644 --- a/absl/random/internal/randen_engine.h +++ b/absl/random/internal/randen_engine.h @@ -42,7 +42,7 @@ namespace random_internal { // 'Strong' (well-distributed, unpredictable, backtracking-resistant) random // generator, faster in some benchmarks than std::mt19937_64 and pcg64_c32. template <typename T> -class alignas(16) randen_engine { +class alignas(8) randen_engine { public: // C++11 URBG interface: using result_type = T; @@ -58,7 +58,8 @@ class alignas(16) randen_engine { return (std::numeric_limits<result_type>::max)(); } - explicit randen_engine(result_type seed_value = 0) { seed(seed_value); } + randen_engine() : randen_engine(0) {} + explicit randen_engine(result_type seed_value) { seed(seed_value); } template <class SeedSequence, typename = typename absl::enable_if_t< @@ -67,17 +68,27 @@ class alignas(16) randen_engine { seed(seq); } - randen_engine(const randen_engine&) = default; + // alignment requirements dictate custom copy and move constructors. + randen_engine(const randen_engine& other) + : next_(other.next_), impl_(other.impl_) { + std::memcpy(state(), other.state(), kStateSizeT * sizeof(result_type)); + } + randen_engine& operator=(const randen_engine& other) { + next_ = other.next_; + impl_ = other.impl_; + std::memcpy(state(), other.state(), kStateSizeT * sizeof(result_type)); + return *this; + } // Returns random bits from the buffer in units of result_type. result_type operator()() { // Refill the buffer if needed (unlikely). + auto* begin = state(); if (next_ >= kStateSizeT) { next_ = kCapacityT; - impl_.Generate(state_); + impl_.Generate(begin); } - - return little_endian::ToHost(state_[next_++]); + return little_endian::ToHost(begin[next_++]); } template <class SeedSequence> @@ -92,9 +103,10 @@ class alignas(16) randen_engine { void seed(result_type seed_value = 0) { next_ = kStateSizeT; // Zeroes the inner state and fills the outer state with seed_value to - // mimics behaviour of reseed - std::fill(std::begin(state_), std::begin(state_) + kCapacityT, 0); - std::fill(std::begin(state_) + kCapacityT, std::end(state_), seed_value); + // mimic the behaviour of reseed + auto* begin = state(); + std::fill(begin, begin + kCapacityT, 0); + std::fill(begin + kCapacityT, begin + kStateSizeT, seed_value); } // Inserts entropy into (part of) the state. Calling this periodically with @@ -105,7 +117,6 @@ class alignas(16) randen_engine { using sequence_result_type = typename SeedSequence::result_type; static_assert(sizeof(sequence_result_type) == 4, "SeedSequence::result_type must be 32-bit"); - constexpr size_t kBufferSize = Randen::kSeedBytes / sizeof(sequence_result_type); alignas(16) sequence_result_type buffer[kBufferSize]; @@ -119,8 +130,15 @@ class alignas(16) randen_engine { if (entropy_size < kBufferSize) { // ... and only request that many values, or 256-bits, when unspecified. const size_t requested_entropy = (entropy_size == 0) ? 8u : entropy_size; - std::fill(std::begin(buffer) + requested_entropy, std::end(buffer), 0); - seq.generate(std::begin(buffer), std::begin(buffer) + requested_entropy); + std::fill(buffer + requested_entropy, buffer + kBufferSize, 0); + seq.generate(buffer, buffer + requested_entropy); +#ifdef ABSL_IS_BIG_ENDIAN + // Randen expects the seed buffer to be in Little Endian; reverse it on + // Big Endian platforms. + for (sequence_result_type& e : buffer) { + e = absl::little_endian::FromHost(e); + } +#endif // The Randen paper suggests preferentially initializing even-numbered // 128-bit vectors of the randen state (there are 16 such vectors). // The seed data is merged into the state offset by 128-bits, which @@ -139,9 +157,9 @@ class alignas(16) randen_engine { std::swap(buffer[--dst], buffer[--src]); } } else { - seq.generate(std::begin(buffer), std::end(buffer)); + seq.generate(buffer, buffer + kBufferSize); } - impl_.Absorb(buffer, state_); + impl_.Absorb(buffer, state()); // Generate will be called when operator() is called next_ = kStateSizeT; @@ -152,9 +170,10 @@ class alignas(16) randen_engine { count -= step; constexpr uint64_t kRateT = kStateSizeT - kCapacityT; + auto* begin = state(); while (count > 0) { next_ = kCapacityT; - impl_.Generate(state_); + impl_.Generate(*reinterpret_cast<result_type(*)[kStateSizeT]>(begin)); step = std::min<uint64_t>(kRateT, count); count -= step; } @@ -162,9 +181,9 @@ class alignas(16) randen_engine { } bool operator==(const randen_engine& other) const { + const auto* begin = state(); return next_ == other.next_ && - std::equal(std::begin(state_), std::end(state_), - std::begin(other.state_)); + std::equal(begin, begin + kStateSizeT, other.state()); } bool operator!=(const randen_engine& other) const { @@ -178,11 +197,12 @@ class alignas(16) randen_engine { using numeric_type = typename random_internal::stream_format_type<result_type>::type; auto saver = random_internal::make_ostream_state_saver(os); - for (const auto& elem : engine.state_) { + auto* it = engine.state(); + for (auto* end = it + kStateSizeT; it < end; ++it) { // In the case that `elem` is `uint8_t`, it must be cast to something // larger so that it prints as an integer rather than a character. For // simplicity, apply the cast all circumstances. - os << static_cast<numeric_type>(little_endian::FromHost(elem)) + os << static_cast<numeric_type>(little_endian::FromHost(*it)) << os.fill(); } os << engine.next_; @@ -208,7 +228,7 @@ class alignas(16) randen_engine { if (is.fail()) { return is; } - std::memcpy(engine.state_, state, sizeof(engine.state_)); + std::memcpy(engine.state(), state, sizeof(state)); engine.next_ = next; return is; } @@ -219,9 +239,21 @@ class alignas(16) randen_engine { static constexpr size_t kCapacityT = Randen::kCapacityBytes / sizeof(result_type); - // First kCapacityT are `inner', the others are accessible random bits. - alignas(16) result_type state_[kStateSizeT]; - size_t next_; // index within state_ + // Returns the state array pointer, which is aligned to 16 bytes. + // The first kCapacityT are the `inner' sponge; the remainder are available. + result_type* state() { + return reinterpret_cast<result_type*>( + (reinterpret_cast<uintptr_t>(&raw_state_) & 0xf) ? (raw_state_ + 8) + : raw_state_); + } + const result_type* state() const { + return const_cast<randen_engine*>(this)->state(); + } + + // raw state array, manually aligned in state(). This overallocates + // by 8 bytes since C++ does not guarantee extended heap alignment. + alignas(8) char raw_state_[Randen::kStateBytes + 8]; + size_t next_; // index within state() Randen impl_; }; diff --git a/absl/random/internal/randen_hwaes.cc b/absl/random/internal/randen_hwaes.cc index b5a3f90a..fee6677c 100644 --- a/absl/random/internal/randen_hwaes.cc +++ b/absl/random/internal/randen_hwaes.cc @@ -23,49 +23,20 @@ #include <cstring> #include "absl/base/attributes.h" +#include "absl/numeric/int128.h" #include "absl/random/internal/platform.h" #include "absl/random/internal/randen_traits.h" // ABSL_RANDEN_HWAES_IMPL indicates whether this file will contain // a hardware accelerated implementation of randen, or whether it // will contain stubs that exit the process. -#if defined(ABSL_ARCH_X86_64) || defined(ABSL_ARCH_X86_32) -// The platform.h directives are sufficient to indicate whether -// we should build accelerated implementations for x86. -#if (ABSL_HAVE_ACCELERATED_AES || ABSL_RANDOM_INTERNAL_AES_DISPATCH) -#define ABSL_RANDEN_HWAES_IMPL 1 -#endif -#elif defined(ABSL_ARCH_PPC) -// The platform.h directives are sufficient to indicate whether -// we should build accelerated implementations for PPC. -// -// NOTE: This has mostly been tested on 64-bit Power variants, -// and not embedded cpus such as powerpc32-8540 #if ABSL_HAVE_ACCELERATED_AES +// The following plaforms have implemented RandenHwAes. +#if defined(ABSL_ARCH_X86_64) || defined(ABSL_ARCH_X86_32) || \ + defined(ABSL_ARCH_PPC) || defined(ABSL_ARCH_ARM) || \ + defined(ABSL_ARCH_AARCH64) #define ABSL_RANDEN_HWAES_IMPL 1 #endif -#elif defined(ABSL_ARCH_ARM) || defined(ABSL_ARCH_AARCH64) -// ARM is somewhat more complicated. We might support crypto natively... -#if ABSL_HAVE_ACCELERATED_AES || \ - (defined(__ARM_NEON) && defined(__ARM_FEATURE_CRYPTO)) -#define ABSL_RANDEN_HWAES_IMPL 1 - -#elif ABSL_RANDOM_INTERNAL_AES_DISPATCH && !defined(__APPLE__) && \ - (defined(__GNUC__) && __GNUC__ > 4 || __GNUC__ == 4 && __GNUC_MINOR__ > 9) -// ...or, on GCC, we can use an ASM directive to -// instruct the assember to allow crypto instructions. -#define ABSL_RANDEN_HWAES_IMPL 1 -#define ABSL_RANDEN_HWAES_IMPL_CRYPTO_DIRECTIVE 1 -#endif -#else -// HWAES is unsupported by these architectures / platforms: -// __myriad2__ -// __mips__ -// -// Other architectures / platforms are unknown. -// -// See the Abseil documentation on supported macros at: -// https://abseil.io/docs/cpp/platforms/macros #endif #if !defined(ABSL_RANDEN_HWAES_IMPL) @@ -120,11 +91,6 @@ namespace { using absl::random_internal::RandenTraits; -// Randen operates on 128-bit vectors. -struct alignas(16) u64x2 { - uint64_t data[2]; -}; - } // namespace // TARGET_CRYPTO defines a crypto attribute for each architecture. @@ -186,7 +152,7 @@ inline ABSL_TARGET_CRYPTO Vector128 AesRound(const Vector128& state, } // Enables native loads in the round loop by pre-swapping. -inline ABSL_TARGET_CRYPTO void SwapEndian(u64x2* state) { +inline ABSL_TARGET_CRYPTO void SwapEndian(absl::uint128* state) { for (uint32_t block = 0; block < RandenTraits::kFeistelBlocks; ++block) { Vector128Store(ReverseBytes(Vector128Load(state + block)), state + block); } @@ -196,22 +162,6 @@ inline ABSL_TARGET_CRYPTO void SwapEndian(u64x2* state) { #elif defined(ABSL_ARCH_ARM) || defined(ABSL_ARCH_AARCH64) -// This asm directive will cause the file to be compiled with crypto extensions -// whether or not the cpu-architecture supports it. -#if ABSL_RANDEN_HWAES_IMPL_CRYPTO_DIRECTIVE -asm(".arch_extension crypto\n"); - -// Override missing defines. -#if !defined(__ARM_NEON) -#define __ARM_NEON 1 -#endif - -#if !defined(__ARM_FEATURE_CRYPTO) -#define __ARM_FEATURE_CRYPTO 1 -#endif - -#endif - // Rely on the ARM NEON+Crypto advanced simd types, defined in <arm_neon.h>. // uint8x16_t is the user alias for underlying __simd128_uint8_t type. // http://infocenter.arm.com/help/topic/com.arm.doc.ihi0073a/IHI0073A_arm_neon_intrinsics_ref.pdf @@ -261,7 +211,7 @@ inline ABSL_TARGET_CRYPTO void SwapEndian(void*) {} #elif defined(ABSL_ARCH_X86_64) || defined(ABSL_ARCH_X86_32) // On x86 we rely on the aesni instructions -#include <wmmintrin.h> +#include <immintrin.h> namespace { @@ -270,7 +220,7 @@ namespace { class Vector128 { public: // Convert from/to intrinsics. - inline explicit Vector128(const __m128i& Vector128) : data_(Vector128) {} + inline explicit Vector128(const __m128i& v) : data_(v) {} inline __m128i data() const { return data_; } @@ -327,7 +277,7 @@ namespace { // Block shuffles applies a shuffle to the entire state between AES rounds. // Improved odd-even shuffle from "New criterion for diffusion property". -inline ABSL_TARGET_CRYPTO void BlockShuffle(u64x2* state) { +inline ABSL_TARGET_CRYPTO void BlockShuffle(absl::uint128* state) { static_assert(RandenTraits::kFeistelBlocks == 16, "Expecting 16 FeistelBlocks."); @@ -374,8 +324,9 @@ inline ABSL_TARGET_CRYPTO void BlockShuffle(u64x2* state) { // per 16 bytes (vs. 10 for AES-CTR). Computing eight round functions in // parallel hides the 7-cycle AESNI latency on HSW. Note that the Feistel // XORs are 'free' (included in the second AES instruction). -inline ABSL_TARGET_CRYPTO const u64x2* FeistelRound( - u64x2* state, const u64x2* ABSL_RANDOM_INTERNAL_RESTRICT keys) { +inline ABSL_TARGET_CRYPTO const absl::uint128* FeistelRound( + absl::uint128* state, + const absl::uint128* ABSL_RANDOM_INTERNAL_RESTRICT keys) { static_assert(RandenTraits::kFeistelBlocks == 16, "Expecting 16 FeistelBlocks."); @@ -436,7 +387,8 @@ inline ABSL_TARGET_CRYPTO const u64x2* FeistelRound( // 2^64 queries if the round function is a PRF. This is similar to the b=8 case // of Simpira v2, but more efficient than its generic construction for b=16. inline ABSL_TARGET_CRYPTO void Permute( - u64x2* state, const u64x2* ABSL_RANDOM_INTERNAL_RESTRICT keys) { + absl::uint128* state, + const absl::uint128* ABSL_RANDOM_INTERNAL_RESTRICT keys) { // (Successfully unrolled; the first iteration jumps into the second half) #ifdef __clang__ #pragma clang loop unroll_count(2) @@ -473,10 +425,11 @@ void ABSL_TARGET_CRYPTO RandenHwAes::Absorb(const void* seed_void, static_assert(RandenTraits::kStateBytes / sizeof(Vector128) == 16, "Unexpected Randen kStateBlocks"); - auto* state = - reinterpret_cast<u64x2 * ABSL_RANDOM_INTERNAL_RESTRICT>(state_void); + auto* state = reinterpret_cast<absl::uint128 * ABSL_RANDOM_INTERNAL_RESTRICT>( + state_void); const auto* seed = - reinterpret_cast<const u64x2 * ABSL_RANDOM_INTERNAL_RESTRICT>(seed_void); + reinterpret_cast<const absl::uint128 * ABSL_RANDOM_INTERNAL_RESTRICT>( + seed_void); Vector128 b1 = Vector128Load(state + 1); b1 ^= Vector128Load(seed + 0); @@ -545,8 +498,8 @@ void ABSL_TARGET_CRYPTO RandenHwAes::Generate(const void* keys_void, static_assert(RandenTraits::kCapacityBytes == sizeof(Vector128), "Capacity mismatch"); - auto* state = reinterpret_cast<u64x2*>(state_void); - const auto* keys = reinterpret_cast<const u64x2*>(keys_void); + auto* state = reinterpret_cast<absl::uint128*>(state_void); + const auto* keys = reinterpret_cast<const absl::uint128*>(keys_void); const Vector128 prev_inner = Vector128Load(state); diff --git a/absl/random/internal/randen_hwaes_test.cc b/absl/random/internal/randen_hwaes_test.cc index 66ddb43f..2348b55c 100644 --- a/absl/random/internal/randen_hwaes_test.cc +++ b/absl/random/internal/randen_hwaes_test.cc @@ -27,44 +27,39 @@ namespace { using absl::random_internal::RandenHwAes; using absl::random_internal::RandenTraits; -// Local state parameters. -constexpr size_t kSeedBytes = - RandenTraits::kStateBytes - RandenTraits::kCapacityBytes; -constexpr size_t kStateSizeT = RandenTraits::kStateBytes / sizeof(uint64_t); -constexpr size_t kSeedSizeT = kSeedBytes / sizeof(uint32_t); - -struct alignas(16) randen { - uint64_t state[kStateSizeT]; - uint32_t seed[kSeedSizeT]; -}; - TEST(RandenHwAesTest, Default) { EXPECT_TRUE(absl::random_internal::CPUSupportsRandenHwAes()); - constexpr uint64_t kGolden[] = { - 0x6c6534090ee6d3ee, 0x044e2b9b9d5333c6, 0xc3c14f134e433977, - 0xdda9f47cd90410ee, 0x887bf3087fd8ca10, 0xf0b780f545c72912, - 0x15dbb1d37696599f, 0x30ec63baff3c6d59, 0xb29f73606f7f20a6, - 0x02808a316f49a54c, 0x3b8feaf9d5c8e50e, 0x9cbf605e3fd9de8a, - 0xc970ae1a78183bbb, 0xd8b2ffd356301ed5, 0xf4b327fe0fc73c37, - 0xcdfd8d76eb8f9a19, 0xc3a506eb91420c9d, 0xd5af05dd3eff9556, - 0x48db1bb78f83c4a1, 0x7023920e0d6bfe8c, 0x58d3575834956d42, - 0xed1ef4c26b87b840, 0x8eef32a23e0b2df3, 0x497cabf3431154fc, - 0x4e24370570029a8b, 0xd88b5749f090e5ea, 0xc651a582a970692f, - 0x78fcec2cbb6342f5, 0x463cb745612f55db, 0x352ee4ad1816afe3, - 0x026ff374c101da7e, 0x811ef0821c3de851, + constexpr uint8_t kGolden[] = { + 0xee, 0xd3, 0xe6, 0x0e, 0x09, 0x34, 0x65, 0x6c, 0xc6, 0x33, 0x53, 0x9d, + 0x9b, 0x2b, 0x4e, 0x04, 0x77, 0x39, 0x43, 0x4e, 0x13, 0x4f, 0xc1, 0xc3, + 0xee, 0x10, 0x04, 0xd9, 0x7c, 0xf4, 0xa9, 0xdd, 0x10, 0xca, 0xd8, 0x7f, + 0x08, 0xf3, 0x7b, 0x88, 0x12, 0x29, 0xc7, 0x45, 0xf5, 0x80, 0xb7, 0xf0, + 0x9f, 0x59, 0x96, 0x76, 0xd3, 0xb1, 0xdb, 0x15, 0x59, 0x6d, 0x3c, 0xff, + 0xba, 0x63, 0xec, 0x30, 0xa6, 0x20, 0x7f, 0x6f, 0x60, 0x73, 0x9f, 0xb2, + 0x4c, 0xa5, 0x49, 0x6f, 0x31, 0x8a, 0x80, 0x02, 0x0e, 0xe5, 0xc8, 0xd5, + 0xf9, 0xea, 0x8f, 0x3b, 0x8a, 0xde, 0xd9, 0x3f, 0x5e, 0x60, 0xbf, 0x9c, + 0xbb, 0x3b, 0x18, 0x78, 0x1a, 0xae, 0x70, 0xc9, 0xd5, 0x1e, 0x30, 0x56, + 0xd3, 0xff, 0xb2, 0xd8, 0x37, 0x3c, 0xc7, 0x0f, 0xfe, 0x27, 0xb3, 0xf4, + 0x19, 0x9a, 0x8f, 0xeb, 0x76, 0x8d, 0xfd, 0xcd, 0x9d, 0x0c, 0x42, 0x91, + 0xeb, 0x06, 0xa5, 0xc3, 0x56, 0x95, 0xff, 0x3e, 0xdd, 0x05, 0xaf, 0xd5, + 0xa1, 0xc4, 0x83, 0x8f, 0xb7, 0x1b, 0xdb, 0x48, 0x8c, 0xfe, 0x6b, 0x0d, + 0x0e, 0x92, 0x23, 0x70, 0x42, 0x6d, 0x95, 0x34, 0x58, 0x57, 0xd3, 0x58, + 0x40, 0xb8, 0x87, 0x6b, 0xc2, 0xf4, 0x1e, 0xed, 0xf3, 0x2d, 0x0b, 0x3e, + 0xa2, 0x32, 0xef, 0x8e, 0xfc, 0x54, 0x11, 0x43, 0xf3, 0xab, 0x7c, 0x49, + 0x8b, 0x9a, 0x02, 0x70, 0x05, 0x37, 0x24, 0x4e, 0xea, 0xe5, 0x90, 0xf0, + 0x49, 0x57, 0x8b, 0xd8, 0x2f, 0x69, 0x70, 0xa9, 0x82, 0xa5, 0x51, 0xc6, + 0xf5, 0x42, 0x63, 0xbb, 0x2c, 0xec, 0xfc, 0x78, 0xdb, 0x55, 0x2f, 0x61, + 0x45, 0xb7, 0x3c, 0x46, 0xe3, 0xaf, 0x16, 0x18, 0xad, 0xe4, 0x2e, 0x35, + 0x7e, 0xda, 0x01, 0xc1, 0x74, 0xf3, 0x6f, 0x02, 0x51, 0xe8, 0x3d, 0x1c, + 0x82, 0xf0, 0x1e, 0x81, }; - alignas(16) randen d; - memset(d.state, 0, sizeof(d.state)); - RandenHwAes::Generate(RandenHwAes::GetKeys(), d.state); + alignas(16) uint8_t state[RandenTraits::kStateBytes]; + std::memset(state, 0, sizeof(state)); - uint64_t* id = d.state; - for (const auto& elem : kGolden) { - auto a = absl::StrFormat("%#x", elem); - auto b = absl::StrFormat("%#x", *id++); - EXPECT_EQ(a, b); - } + RandenHwAes::Generate(RandenHwAes::GetKeys(), state); + EXPECT_EQ(0, std::memcmp(state, kGolden, sizeof(state))); } } // namespace diff --git a/absl/random/internal/randen_slow.cc b/absl/random/internal/randen_slow.cc index 4e5f3dc1..9bfd2a40 100644 --- a/absl/random/internal/randen_slow.cc +++ b/absl/random/internal/randen_slow.cc @@ -19,6 +19,8 @@ #include <cstring> #include "absl/base/attributes.h" +#include "absl/base/internal/endian.h" +#include "absl/numeric/int128.h" #include "absl/random/internal/platform.h" #include "absl/random/internal/randen_traits.h" @@ -38,192 +40,193 @@ namespace { // AES portions based on rijndael-alg-fst.c, -// https://fastcrypto.org/front/misc/rijndael-alg-fst.c +// https://fastcrypto.org/front/misc/rijndael-alg-fst.c, and modified for +// platform-endianness. // // Implementation of // http://www.csrc.nist.gov/publications/fips/fips197/fips-197.pdf constexpr uint32_t te0[256] = { - 0xc66363a5, 0xf87c7c84, 0xee777799, 0xf67b7b8d, 0xfff2f20d, 0xd66b6bbd, - 0xde6f6fb1, 0x91c5c554, 0x60303050, 0x02010103, 0xce6767a9, 0x562b2b7d, - 0xe7fefe19, 0xb5d7d762, 0x4dababe6, 0xec76769a, 0x8fcaca45, 0x1f82829d, - 0x89c9c940, 0xfa7d7d87, 0xeffafa15, 0xb25959eb, 0x8e4747c9, 0xfbf0f00b, - 0x41adadec, 0xb3d4d467, 0x5fa2a2fd, 0x45afafea, 0x239c9cbf, 0x53a4a4f7, - 0xe4727296, 0x9bc0c05b, 0x75b7b7c2, 0xe1fdfd1c, 0x3d9393ae, 0x4c26266a, - 0x6c36365a, 0x7e3f3f41, 0xf5f7f702, 0x83cccc4f, 0x6834345c, 0x51a5a5f4, - 0xd1e5e534, 0xf9f1f108, 0xe2717193, 0xabd8d873, 0x62313153, 0x2a15153f, - 0x0804040c, 0x95c7c752, 0x46232365, 0x9dc3c35e, 0x30181828, 0x379696a1, - 0x0a05050f, 0x2f9a9ab5, 0x0e070709, 0x24121236, 0x1b80809b, 0xdfe2e23d, - 0xcdebeb26, 0x4e272769, 0x7fb2b2cd, 0xea75759f, 0x1209091b, 0x1d83839e, - 0x582c2c74, 0x341a1a2e, 0x361b1b2d, 0xdc6e6eb2, 0xb45a5aee, 0x5ba0a0fb, - 0xa45252f6, 0x763b3b4d, 0xb7d6d661, 0x7db3b3ce, 0x5229297b, 0xdde3e33e, - 0x5e2f2f71, 0x13848497, 0xa65353f5, 0xb9d1d168, 0x00000000, 0xc1eded2c, - 0x40202060, 0xe3fcfc1f, 0x79b1b1c8, 0xb65b5bed, 0xd46a6abe, 0x8dcbcb46, - 0x67bebed9, 0x7239394b, 0x944a4ade, 0x984c4cd4, 0xb05858e8, 0x85cfcf4a, - 0xbbd0d06b, 0xc5efef2a, 0x4faaaae5, 0xedfbfb16, 0x864343c5, 0x9a4d4dd7, - 0x66333355, 0x11858594, 0x8a4545cf, 0xe9f9f910, 0x04020206, 0xfe7f7f81, - 0xa05050f0, 0x783c3c44, 0x259f9fba, 0x4ba8a8e3, 0xa25151f3, 0x5da3a3fe, - 0x804040c0, 0x058f8f8a, 0x3f9292ad, 0x219d9dbc, 0x70383848, 0xf1f5f504, - 0x63bcbcdf, 0x77b6b6c1, 0xafdada75, 0x42212163, 0x20101030, 0xe5ffff1a, - 0xfdf3f30e, 0xbfd2d26d, 0x81cdcd4c, 0x180c0c14, 0x26131335, 0xc3ecec2f, - 0xbe5f5fe1, 0x359797a2, 0x884444cc, 0x2e171739, 0x93c4c457, 0x55a7a7f2, - 0xfc7e7e82, 0x7a3d3d47, 0xc86464ac, 0xba5d5de7, 0x3219192b, 0xe6737395, - 0xc06060a0, 0x19818198, 0x9e4f4fd1, 0xa3dcdc7f, 0x44222266, 0x542a2a7e, - 0x3b9090ab, 0x0b888883, 0x8c4646ca, 0xc7eeee29, 0x6bb8b8d3, 0x2814143c, - 0xa7dede79, 0xbc5e5ee2, 0x160b0b1d, 0xaddbdb76, 0xdbe0e03b, 0x64323256, - 0x743a3a4e, 0x140a0a1e, 0x924949db, 0x0c06060a, 0x4824246c, 0xb85c5ce4, - 0x9fc2c25d, 0xbdd3d36e, 0x43acacef, 0xc46262a6, 0x399191a8, 0x319595a4, - 0xd3e4e437, 0xf279798b, 0xd5e7e732, 0x8bc8c843, 0x6e373759, 0xda6d6db7, - 0x018d8d8c, 0xb1d5d564, 0x9c4e4ed2, 0x49a9a9e0, 0xd86c6cb4, 0xac5656fa, - 0xf3f4f407, 0xcfeaea25, 0xca6565af, 0xf47a7a8e, 0x47aeaee9, 0x10080818, - 0x6fbabad5, 0xf0787888, 0x4a25256f, 0x5c2e2e72, 0x381c1c24, 0x57a6a6f1, - 0x73b4b4c7, 0x97c6c651, 0xcbe8e823, 0xa1dddd7c, 0xe874749c, 0x3e1f1f21, - 0x964b4bdd, 0x61bdbddc, 0x0d8b8b86, 0x0f8a8a85, 0xe0707090, 0x7c3e3e42, - 0x71b5b5c4, 0xcc6666aa, 0x904848d8, 0x06030305, 0xf7f6f601, 0x1c0e0e12, - 0xc26161a3, 0x6a35355f, 0xae5757f9, 0x69b9b9d0, 0x17868691, 0x99c1c158, - 0x3a1d1d27, 0x279e9eb9, 0xd9e1e138, 0xebf8f813, 0x2b9898b3, 0x22111133, - 0xd26969bb, 0xa9d9d970, 0x078e8e89, 0x339494a7, 0x2d9b9bb6, 0x3c1e1e22, - 0x15878792, 0xc9e9e920, 0x87cece49, 0xaa5555ff, 0x50282878, 0xa5dfdf7a, - 0x038c8c8f, 0x59a1a1f8, 0x09898980, 0x1a0d0d17, 0x65bfbfda, 0xd7e6e631, - 0x844242c6, 0xd06868b8, 0x824141c3, 0x299999b0, 0x5a2d2d77, 0x1e0f0f11, - 0x7bb0b0cb, 0xa85454fc, 0x6dbbbbd6, 0x2c16163a, + 0xa56363c6, 0x847c7cf8, 0x997777ee, 0x8d7b7bf6, 0x0df2f2ff, 0xbd6b6bd6, + 0xb16f6fde, 0x54c5c591, 0x50303060, 0x03010102, 0xa96767ce, 0x7d2b2b56, + 0x19fefee7, 0x62d7d7b5, 0xe6abab4d, 0x9a7676ec, 0x45caca8f, 0x9d82821f, + 0x40c9c989, 0x877d7dfa, 0x15fafaef, 0xeb5959b2, 0xc947478e, 0x0bf0f0fb, + 0xecadad41, 0x67d4d4b3, 0xfda2a25f, 0xeaafaf45, 0xbf9c9c23, 0xf7a4a453, + 0x967272e4, 0x5bc0c09b, 0xc2b7b775, 0x1cfdfde1, 0xae93933d, 0x6a26264c, + 0x5a36366c, 0x413f3f7e, 0x02f7f7f5, 0x4fcccc83, 0x5c343468, 0xf4a5a551, + 0x34e5e5d1, 0x08f1f1f9, 0x937171e2, 0x73d8d8ab, 0x53313162, 0x3f15152a, + 0x0c040408, 0x52c7c795, 0x65232346, 0x5ec3c39d, 0x28181830, 0xa1969637, + 0x0f05050a, 0xb59a9a2f, 0x0907070e, 0x36121224, 0x9b80801b, 0x3de2e2df, + 0x26ebebcd, 0x6927274e, 0xcdb2b27f, 0x9f7575ea, 0x1b090912, 0x9e83831d, + 0x742c2c58, 0x2e1a1a34, 0x2d1b1b36, 0xb26e6edc, 0xee5a5ab4, 0xfba0a05b, + 0xf65252a4, 0x4d3b3b76, 0x61d6d6b7, 0xceb3b37d, 0x7b292952, 0x3ee3e3dd, + 0x712f2f5e, 0x97848413, 0xf55353a6, 0x68d1d1b9, 0x00000000, 0x2cededc1, + 0x60202040, 0x1ffcfce3, 0xc8b1b179, 0xed5b5bb6, 0xbe6a6ad4, 0x46cbcb8d, + 0xd9bebe67, 0x4b393972, 0xde4a4a94, 0xd44c4c98, 0xe85858b0, 0x4acfcf85, + 0x6bd0d0bb, 0x2aefefc5, 0xe5aaaa4f, 0x16fbfbed, 0xc5434386, 0xd74d4d9a, + 0x55333366, 0x94858511, 0xcf45458a, 0x10f9f9e9, 0x06020204, 0x817f7ffe, + 0xf05050a0, 0x443c3c78, 0xba9f9f25, 0xe3a8a84b, 0xf35151a2, 0xfea3a35d, + 0xc0404080, 0x8a8f8f05, 0xad92923f, 0xbc9d9d21, 0x48383870, 0x04f5f5f1, + 0xdfbcbc63, 0xc1b6b677, 0x75dadaaf, 0x63212142, 0x30101020, 0x1affffe5, + 0x0ef3f3fd, 0x6dd2d2bf, 0x4ccdcd81, 0x140c0c18, 0x35131326, 0x2fececc3, + 0xe15f5fbe, 0xa2979735, 0xcc444488, 0x3917172e, 0x57c4c493, 0xf2a7a755, + 0x827e7efc, 0x473d3d7a, 0xac6464c8, 0xe75d5dba, 0x2b191932, 0x957373e6, + 0xa06060c0, 0x98818119, 0xd14f4f9e, 0x7fdcdca3, 0x66222244, 0x7e2a2a54, + 0xab90903b, 0x8388880b, 0xca46468c, 0x29eeeec7, 0xd3b8b86b, 0x3c141428, + 0x79dedea7, 0xe25e5ebc, 0x1d0b0b16, 0x76dbdbad, 0x3be0e0db, 0x56323264, + 0x4e3a3a74, 0x1e0a0a14, 0xdb494992, 0x0a06060c, 0x6c242448, 0xe45c5cb8, + 0x5dc2c29f, 0x6ed3d3bd, 0xefacac43, 0xa66262c4, 0xa8919139, 0xa4959531, + 0x37e4e4d3, 0x8b7979f2, 0x32e7e7d5, 0x43c8c88b, 0x5937376e, 0xb76d6dda, + 0x8c8d8d01, 0x64d5d5b1, 0xd24e4e9c, 0xe0a9a949, 0xb46c6cd8, 0xfa5656ac, + 0x07f4f4f3, 0x25eaeacf, 0xaf6565ca, 0x8e7a7af4, 0xe9aeae47, 0x18080810, + 0xd5baba6f, 0x887878f0, 0x6f25254a, 0x722e2e5c, 0x241c1c38, 0xf1a6a657, + 0xc7b4b473, 0x51c6c697, 0x23e8e8cb, 0x7cdddda1, 0x9c7474e8, 0x211f1f3e, + 0xdd4b4b96, 0xdcbdbd61, 0x868b8b0d, 0x858a8a0f, 0x907070e0, 0x423e3e7c, + 0xc4b5b571, 0xaa6666cc, 0xd8484890, 0x05030306, 0x01f6f6f7, 0x120e0e1c, + 0xa36161c2, 0x5f35356a, 0xf95757ae, 0xd0b9b969, 0x91868617, 0x58c1c199, + 0x271d1d3a, 0xb99e9e27, 0x38e1e1d9, 0x13f8f8eb, 0xb398982b, 0x33111122, + 0xbb6969d2, 0x70d9d9a9, 0x898e8e07, 0xa7949433, 0xb69b9b2d, 0x221e1e3c, + 0x92878715, 0x20e9e9c9, 0x49cece87, 0xff5555aa, 0x78282850, 0x7adfdfa5, + 0x8f8c8c03, 0xf8a1a159, 0x80898909, 0x170d0d1a, 0xdabfbf65, 0x31e6e6d7, + 0xc6424284, 0xb86868d0, 0xc3414182, 0xb0999929, 0x772d2d5a, 0x110f0f1e, + 0xcbb0b07b, 0xfc5454a8, 0xd6bbbb6d, 0x3a16162c, }; constexpr uint32_t te1[256] = { - 0xa5c66363, 0x84f87c7c, 0x99ee7777, 0x8df67b7b, 0x0dfff2f2, 0xbdd66b6b, - 0xb1de6f6f, 0x5491c5c5, 0x50603030, 0x03020101, 0xa9ce6767, 0x7d562b2b, - 0x19e7fefe, 0x62b5d7d7, 0xe64dabab, 0x9aec7676, 0x458fcaca, 0x9d1f8282, - 0x4089c9c9, 0x87fa7d7d, 0x15effafa, 0xebb25959, 0xc98e4747, 0x0bfbf0f0, - 0xec41adad, 0x67b3d4d4, 0xfd5fa2a2, 0xea45afaf, 0xbf239c9c, 0xf753a4a4, - 0x96e47272, 0x5b9bc0c0, 0xc275b7b7, 0x1ce1fdfd, 0xae3d9393, 0x6a4c2626, - 0x5a6c3636, 0x417e3f3f, 0x02f5f7f7, 0x4f83cccc, 0x5c683434, 0xf451a5a5, - 0x34d1e5e5, 0x08f9f1f1, 0x93e27171, 0x73abd8d8, 0x53623131, 0x3f2a1515, - 0x0c080404, 0x5295c7c7, 0x65462323, 0x5e9dc3c3, 0x28301818, 0xa1379696, - 0x0f0a0505, 0xb52f9a9a, 0x090e0707, 0x36241212, 0x9b1b8080, 0x3ddfe2e2, - 0x26cdebeb, 0x694e2727, 0xcd7fb2b2, 0x9fea7575, 0x1b120909, 0x9e1d8383, - 0x74582c2c, 0x2e341a1a, 0x2d361b1b, 0xb2dc6e6e, 0xeeb45a5a, 0xfb5ba0a0, - 0xf6a45252, 0x4d763b3b, 0x61b7d6d6, 0xce7db3b3, 0x7b522929, 0x3edde3e3, - 0x715e2f2f, 0x97138484, 0xf5a65353, 0x68b9d1d1, 0x00000000, 0x2cc1eded, - 0x60402020, 0x1fe3fcfc, 0xc879b1b1, 0xedb65b5b, 0xbed46a6a, 0x468dcbcb, - 0xd967bebe, 0x4b723939, 0xde944a4a, 0xd4984c4c, 0xe8b05858, 0x4a85cfcf, - 0x6bbbd0d0, 0x2ac5efef, 0xe54faaaa, 0x16edfbfb, 0xc5864343, 0xd79a4d4d, - 0x55663333, 0x94118585, 0xcf8a4545, 0x10e9f9f9, 0x06040202, 0x81fe7f7f, - 0xf0a05050, 0x44783c3c, 0xba259f9f, 0xe34ba8a8, 0xf3a25151, 0xfe5da3a3, - 0xc0804040, 0x8a058f8f, 0xad3f9292, 0xbc219d9d, 0x48703838, 0x04f1f5f5, - 0xdf63bcbc, 0xc177b6b6, 0x75afdada, 0x63422121, 0x30201010, 0x1ae5ffff, - 0x0efdf3f3, 0x6dbfd2d2, 0x4c81cdcd, 0x14180c0c, 0x35261313, 0x2fc3ecec, - 0xe1be5f5f, 0xa2359797, 0xcc884444, 0x392e1717, 0x5793c4c4, 0xf255a7a7, - 0x82fc7e7e, 0x477a3d3d, 0xacc86464, 0xe7ba5d5d, 0x2b321919, 0x95e67373, - 0xa0c06060, 0x98198181, 0xd19e4f4f, 0x7fa3dcdc, 0x66442222, 0x7e542a2a, - 0xab3b9090, 0x830b8888, 0xca8c4646, 0x29c7eeee, 0xd36bb8b8, 0x3c281414, - 0x79a7dede, 0xe2bc5e5e, 0x1d160b0b, 0x76addbdb, 0x3bdbe0e0, 0x56643232, - 0x4e743a3a, 0x1e140a0a, 0xdb924949, 0x0a0c0606, 0x6c482424, 0xe4b85c5c, - 0x5d9fc2c2, 0x6ebdd3d3, 0xef43acac, 0xa6c46262, 0xa8399191, 0xa4319595, - 0x37d3e4e4, 0x8bf27979, 0x32d5e7e7, 0x438bc8c8, 0x596e3737, 0xb7da6d6d, - 0x8c018d8d, 0x64b1d5d5, 0xd29c4e4e, 0xe049a9a9, 0xb4d86c6c, 0xfaac5656, - 0x07f3f4f4, 0x25cfeaea, 0xafca6565, 0x8ef47a7a, 0xe947aeae, 0x18100808, - 0xd56fbaba, 0x88f07878, 0x6f4a2525, 0x725c2e2e, 0x24381c1c, 0xf157a6a6, - 0xc773b4b4, 0x5197c6c6, 0x23cbe8e8, 0x7ca1dddd, 0x9ce87474, 0x213e1f1f, - 0xdd964b4b, 0xdc61bdbd, 0x860d8b8b, 0x850f8a8a, 0x90e07070, 0x427c3e3e, - 0xc471b5b5, 0xaacc6666, 0xd8904848, 0x05060303, 0x01f7f6f6, 0x121c0e0e, - 0xa3c26161, 0x5f6a3535, 0xf9ae5757, 0xd069b9b9, 0x91178686, 0x5899c1c1, - 0x273a1d1d, 0xb9279e9e, 0x38d9e1e1, 0x13ebf8f8, 0xb32b9898, 0x33221111, - 0xbbd26969, 0x70a9d9d9, 0x89078e8e, 0xa7339494, 0xb62d9b9b, 0x223c1e1e, - 0x92158787, 0x20c9e9e9, 0x4987cece, 0xffaa5555, 0x78502828, 0x7aa5dfdf, - 0x8f038c8c, 0xf859a1a1, 0x80098989, 0x171a0d0d, 0xda65bfbf, 0x31d7e6e6, - 0xc6844242, 0xb8d06868, 0xc3824141, 0xb0299999, 0x775a2d2d, 0x111e0f0f, - 0xcb7bb0b0, 0xfca85454, 0xd66dbbbb, 0x3a2c1616, + 0x6363c6a5, 0x7c7cf884, 0x7777ee99, 0x7b7bf68d, 0xf2f2ff0d, 0x6b6bd6bd, + 0x6f6fdeb1, 0xc5c59154, 0x30306050, 0x01010203, 0x6767cea9, 0x2b2b567d, + 0xfefee719, 0xd7d7b562, 0xabab4de6, 0x7676ec9a, 0xcaca8f45, 0x82821f9d, + 0xc9c98940, 0x7d7dfa87, 0xfafaef15, 0x5959b2eb, 0x47478ec9, 0xf0f0fb0b, + 0xadad41ec, 0xd4d4b367, 0xa2a25ffd, 0xafaf45ea, 0x9c9c23bf, 0xa4a453f7, + 0x7272e496, 0xc0c09b5b, 0xb7b775c2, 0xfdfde11c, 0x93933dae, 0x26264c6a, + 0x36366c5a, 0x3f3f7e41, 0xf7f7f502, 0xcccc834f, 0x3434685c, 0xa5a551f4, + 0xe5e5d134, 0xf1f1f908, 0x7171e293, 0xd8d8ab73, 0x31316253, 0x15152a3f, + 0x0404080c, 0xc7c79552, 0x23234665, 0xc3c39d5e, 0x18183028, 0x969637a1, + 0x05050a0f, 0x9a9a2fb5, 0x07070e09, 0x12122436, 0x80801b9b, 0xe2e2df3d, + 0xebebcd26, 0x27274e69, 0xb2b27fcd, 0x7575ea9f, 0x0909121b, 0x83831d9e, + 0x2c2c5874, 0x1a1a342e, 0x1b1b362d, 0x6e6edcb2, 0x5a5ab4ee, 0xa0a05bfb, + 0x5252a4f6, 0x3b3b764d, 0xd6d6b761, 0xb3b37dce, 0x2929527b, 0xe3e3dd3e, + 0x2f2f5e71, 0x84841397, 0x5353a6f5, 0xd1d1b968, 0x00000000, 0xededc12c, + 0x20204060, 0xfcfce31f, 0xb1b179c8, 0x5b5bb6ed, 0x6a6ad4be, 0xcbcb8d46, + 0xbebe67d9, 0x3939724b, 0x4a4a94de, 0x4c4c98d4, 0x5858b0e8, 0xcfcf854a, + 0xd0d0bb6b, 0xefefc52a, 0xaaaa4fe5, 0xfbfbed16, 0x434386c5, 0x4d4d9ad7, + 0x33336655, 0x85851194, 0x45458acf, 0xf9f9e910, 0x02020406, 0x7f7ffe81, + 0x5050a0f0, 0x3c3c7844, 0x9f9f25ba, 0xa8a84be3, 0x5151a2f3, 0xa3a35dfe, + 0x404080c0, 0x8f8f058a, 0x92923fad, 0x9d9d21bc, 0x38387048, 0xf5f5f104, + 0xbcbc63df, 0xb6b677c1, 0xdadaaf75, 0x21214263, 0x10102030, 0xffffe51a, + 0xf3f3fd0e, 0xd2d2bf6d, 0xcdcd814c, 0x0c0c1814, 0x13132635, 0xececc32f, + 0x5f5fbee1, 0x979735a2, 0x444488cc, 0x17172e39, 0xc4c49357, 0xa7a755f2, + 0x7e7efc82, 0x3d3d7a47, 0x6464c8ac, 0x5d5dbae7, 0x1919322b, 0x7373e695, + 0x6060c0a0, 0x81811998, 0x4f4f9ed1, 0xdcdca37f, 0x22224466, 0x2a2a547e, + 0x90903bab, 0x88880b83, 0x46468cca, 0xeeeec729, 0xb8b86bd3, 0x1414283c, + 0xdedea779, 0x5e5ebce2, 0x0b0b161d, 0xdbdbad76, 0xe0e0db3b, 0x32326456, + 0x3a3a744e, 0x0a0a141e, 0x494992db, 0x06060c0a, 0x2424486c, 0x5c5cb8e4, + 0xc2c29f5d, 0xd3d3bd6e, 0xacac43ef, 0x6262c4a6, 0x919139a8, 0x959531a4, + 0xe4e4d337, 0x7979f28b, 0xe7e7d532, 0xc8c88b43, 0x37376e59, 0x6d6ddab7, + 0x8d8d018c, 0xd5d5b164, 0x4e4e9cd2, 0xa9a949e0, 0x6c6cd8b4, 0x5656acfa, + 0xf4f4f307, 0xeaeacf25, 0x6565caaf, 0x7a7af48e, 0xaeae47e9, 0x08081018, + 0xbaba6fd5, 0x7878f088, 0x25254a6f, 0x2e2e5c72, 0x1c1c3824, 0xa6a657f1, + 0xb4b473c7, 0xc6c69751, 0xe8e8cb23, 0xdddda17c, 0x7474e89c, 0x1f1f3e21, + 0x4b4b96dd, 0xbdbd61dc, 0x8b8b0d86, 0x8a8a0f85, 0x7070e090, 0x3e3e7c42, + 0xb5b571c4, 0x6666ccaa, 0x484890d8, 0x03030605, 0xf6f6f701, 0x0e0e1c12, + 0x6161c2a3, 0x35356a5f, 0x5757aef9, 0xb9b969d0, 0x86861791, 0xc1c19958, + 0x1d1d3a27, 0x9e9e27b9, 0xe1e1d938, 0xf8f8eb13, 0x98982bb3, 0x11112233, + 0x6969d2bb, 0xd9d9a970, 0x8e8e0789, 0x949433a7, 0x9b9b2db6, 0x1e1e3c22, + 0x87871592, 0xe9e9c920, 0xcece8749, 0x5555aaff, 0x28285078, 0xdfdfa57a, + 0x8c8c038f, 0xa1a159f8, 0x89890980, 0x0d0d1a17, 0xbfbf65da, 0xe6e6d731, + 0x424284c6, 0x6868d0b8, 0x414182c3, 0x999929b0, 0x2d2d5a77, 0x0f0f1e11, + 0xb0b07bcb, 0x5454a8fc, 0xbbbb6dd6, 0x16162c3a, }; constexpr uint32_t te2[256] = { - 0x63a5c663, 0x7c84f87c, 0x7799ee77, 0x7b8df67b, 0xf20dfff2, 0x6bbdd66b, - 0x6fb1de6f, 0xc55491c5, 0x30506030, 0x01030201, 0x67a9ce67, 0x2b7d562b, - 0xfe19e7fe, 0xd762b5d7, 0xabe64dab, 0x769aec76, 0xca458fca, 0x829d1f82, - 0xc94089c9, 0x7d87fa7d, 0xfa15effa, 0x59ebb259, 0x47c98e47, 0xf00bfbf0, - 0xadec41ad, 0xd467b3d4, 0xa2fd5fa2, 0xafea45af, 0x9cbf239c, 0xa4f753a4, - 0x7296e472, 0xc05b9bc0, 0xb7c275b7, 0xfd1ce1fd, 0x93ae3d93, 0x266a4c26, - 0x365a6c36, 0x3f417e3f, 0xf702f5f7, 0xcc4f83cc, 0x345c6834, 0xa5f451a5, - 0xe534d1e5, 0xf108f9f1, 0x7193e271, 0xd873abd8, 0x31536231, 0x153f2a15, - 0x040c0804, 0xc75295c7, 0x23654623, 0xc35e9dc3, 0x18283018, 0x96a13796, - 0x050f0a05, 0x9ab52f9a, 0x07090e07, 0x12362412, 0x809b1b80, 0xe23ddfe2, - 0xeb26cdeb, 0x27694e27, 0xb2cd7fb2, 0x759fea75, 0x091b1209, 0x839e1d83, - 0x2c74582c, 0x1a2e341a, 0x1b2d361b, 0x6eb2dc6e, 0x5aeeb45a, 0xa0fb5ba0, - 0x52f6a452, 0x3b4d763b, 0xd661b7d6, 0xb3ce7db3, 0x297b5229, 0xe33edde3, - 0x2f715e2f, 0x84971384, 0x53f5a653, 0xd168b9d1, 0x00000000, 0xed2cc1ed, - 0x20604020, 0xfc1fe3fc, 0xb1c879b1, 0x5bedb65b, 0x6abed46a, 0xcb468dcb, - 0xbed967be, 0x394b7239, 0x4ade944a, 0x4cd4984c, 0x58e8b058, 0xcf4a85cf, - 0xd06bbbd0, 0xef2ac5ef, 0xaae54faa, 0xfb16edfb, 0x43c58643, 0x4dd79a4d, - 0x33556633, 0x85941185, 0x45cf8a45, 0xf910e9f9, 0x02060402, 0x7f81fe7f, - 0x50f0a050, 0x3c44783c, 0x9fba259f, 0xa8e34ba8, 0x51f3a251, 0xa3fe5da3, - 0x40c08040, 0x8f8a058f, 0x92ad3f92, 0x9dbc219d, 0x38487038, 0xf504f1f5, - 0xbcdf63bc, 0xb6c177b6, 0xda75afda, 0x21634221, 0x10302010, 0xff1ae5ff, - 0xf30efdf3, 0xd26dbfd2, 0xcd4c81cd, 0x0c14180c, 0x13352613, 0xec2fc3ec, - 0x5fe1be5f, 0x97a23597, 0x44cc8844, 0x17392e17, 0xc45793c4, 0xa7f255a7, - 0x7e82fc7e, 0x3d477a3d, 0x64acc864, 0x5de7ba5d, 0x192b3219, 0x7395e673, - 0x60a0c060, 0x81981981, 0x4fd19e4f, 0xdc7fa3dc, 0x22664422, 0x2a7e542a, - 0x90ab3b90, 0x88830b88, 0x46ca8c46, 0xee29c7ee, 0xb8d36bb8, 0x143c2814, - 0xde79a7de, 0x5ee2bc5e, 0x0b1d160b, 0xdb76addb, 0xe03bdbe0, 0x32566432, - 0x3a4e743a, 0x0a1e140a, 0x49db9249, 0x060a0c06, 0x246c4824, 0x5ce4b85c, - 0xc25d9fc2, 0xd36ebdd3, 0xacef43ac, 0x62a6c462, 0x91a83991, 0x95a43195, - 0xe437d3e4, 0x798bf279, 0xe732d5e7, 0xc8438bc8, 0x37596e37, 0x6db7da6d, - 0x8d8c018d, 0xd564b1d5, 0x4ed29c4e, 0xa9e049a9, 0x6cb4d86c, 0x56faac56, - 0xf407f3f4, 0xea25cfea, 0x65afca65, 0x7a8ef47a, 0xaee947ae, 0x08181008, - 0xbad56fba, 0x7888f078, 0x256f4a25, 0x2e725c2e, 0x1c24381c, 0xa6f157a6, - 0xb4c773b4, 0xc65197c6, 0xe823cbe8, 0xdd7ca1dd, 0x749ce874, 0x1f213e1f, - 0x4bdd964b, 0xbddc61bd, 0x8b860d8b, 0x8a850f8a, 0x7090e070, 0x3e427c3e, - 0xb5c471b5, 0x66aacc66, 0x48d89048, 0x03050603, 0xf601f7f6, 0x0e121c0e, - 0x61a3c261, 0x355f6a35, 0x57f9ae57, 0xb9d069b9, 0x86911786, 0xc15899c1, - 0x1d273a1d, 0x9eb9279e, 0xe138d9e1, 0xf813ebf8, 0x98b32b98, 0x11332211, - 0x69bbd269, 0xd970a9d9, 0x8e89078e, 0x94a73394, 0x9bb62d9b, 0x1e223c1e, - 0x87921587, 0xe920c9e9, 0xce4987ce, 0x55ffaa55, 0x28785028, 0xdf7aa5df, - 0x8c8f038c, 0xa1f859a1, 0x89800989, 0x0d171a0d, 0xbfda65bf, 0xe631d7e6, - 0x42c68442, 0x68b8d068, 0x41c38241, 0x99b02999, 0x2d775a2d, 0x0f111e0f, - 0xb0cb7bb0, 0x54fca854, 0xbbd66dbb, 0x163a2c16, + 0x63c6a563, 0x7cf8847c, 0x77ee9977, 0x7bf68d7b, 0xf2ff0df2, 0x6bd6bd6b, + 0x6fdeb16f, 0xc59154c5, 0x30605030, 0x01020301, 0x67cea967, 0x2b567d2b, + 0xfee719fe, 0xd7b562d7, 0xab4de6ab, 0x76ec9a76, 0xca8f45ca, 0x821f9d82, + 0xc98940c9, 0x7dfa877d, 0xfaef15fa, 0x59b2eb59, 0x478ec947, 0xf0fb0bf0, + 0xad41ecad, 0xd4b367d4, 0xa25ffda2, 0xaf45eaaf, 0x9c23bf9c, 0xa453f7a4, + 0x72e49672, 0xc09b5bc0, 0xb775c2b7, 0xfde11cfd, 0x933dae93, 0x264c6a26, + 0x366c5a36, 0x3f7e413f, 0xf7f502f7, 0xcc834fcc, 0x34685c34, 0xa551f4a5, + 0xe5d134e5, 0xf1f908f1, 0x71e29371, 0xd8ab73d8, 0x31625331, 0x152a3f15, + 0x04080c04, 0xc79552c7, 0x23466523, 0xc39d5ec3, 0x18302818, 0x9637a196, + 0x050a0f05, 0x9a2fb59a, 0x070e0907, 0x12243612, 0x801b9b80, 0xe2df3de2, + 0xebcd26eb, 0x274e6927, 0xb27fcdb2, 0x75ea9f75, 0x09121b09, 0x831d9e83, + 0x2c58742c, 0x1a342e1a, 0x1b362d1b, 0x6edcb26e, 0x5ab4ee5a, 0xa05bfba0, + 0x52a4f652, 0x3b764d3b, 0xd6b761d6, 0xb37dceb3, 0x29527b29, 0xe3dd3ee3, + 0x2f5e712f, 0x84139784, 0x53a6f553, 0xd1b968d1, 0x00000000, 0xedc12ced, + 0x20406020, 0xfce31ffc, 0xb179c8b1, 0x5bb6ed5b, 0x6ad4be6a, 0xcb8d46cb, + 0xbe67d9be, 0x39724b39, 0x4a94de4a, 0x4c98d44c, 0x58b0e858, 0xcf854acf, + 0xd0bb6bd0, 0xefc52aef, 0xaa4fe5aa, 0xfbed16fb, 0x4386c543, 0x4d9ad74d, + 0x33665533, 0x85119485, 0x458acf45, 0xf9e910f9, 0x02040602, 0x7ffe817f, + 0x50a0f050, 0x3c78443c, 0x9f25ba9f, 0xa84be3a8, 0x51a2f351, 0xa35dfea3, + 0x4080c040, 0x8f058a8f, 0x923fad92, 0x9d21bc9d, 0x38704838, 0xf5f104f5, + 0xbc63dfbc, 0xb677c1b6, 0xdaaf75da, 0x21426321, 0x10203010, 0xffe51aff, + 0xf3fd0ef3, 0xd2bf6dd2, 0xcd814ccd, 0x0c18140c, 0x13263513, 0xecc32fec, + 0x5fbee15f, 0x9735a297, 0x4488cc44, 0x172e3917, 0xc49357c4, 0xa755f2a7, + 0x7efc827e, 0x3d7a473d, 0x64c8ac64, 0x5dbae75d, 0x19322b19, 0x73e69573, + 0x60c0a060, 0x81199881, 0x4f9ed14f, 0xdca37fdc, 0x22446622, 0x2a547e2a, + 0x903bab90, 0x880b8388, 0x468cca46, 0xeec729ee, 0xb86bd3b8, 0x14283c14, + 0xdea779de, 0x5ebce25e, 0x0b161d0b, 0xdbad76db, 0xe0db3be0, 0x32645632, + 0x3a744e3a, 0x0a141e0a, 0x4992db49, 0x060c0a06, 0x24486c24, 0x5cb8e45c, + 0xc29f5dc2, 0xd3bd6ed3, 0xac43efac, 0x62c4a662, 0x9139a891, 0x9531a495, + 0xe4d337e4, 0x79f28b79, 0xe7d532e7, 0xc88b43c8, 0x376e5937, 0x6ddab76d, + 0x8d018c8d, 0xd5b164d5, 0x4e9cd24e, 0xa949e0a9, 0x6cd8b46c, 0x56acfa56, + 0xf4f307f4, 0xeacf25ea, 0x65caaf65, 0x7af48e7a, 0xae47e9ae, 0x08101808, + 0xba6fd5ba, 0x78f08878, 0x254a6f25, 0x2e5c722e, 0x1c38241c, 0xa657f1a6, + 0xb473c7b4, 0xc69751c6, 0xe8cb23e8, 0xdda17cdd, 0x74e89c74, 0x1f3e211f, + 0x4b96dd4b, 0xbd61dcbd, 0x8b0d868b, 0x8a0f858a, 0x70e09070, 0x3e7c423e, + 0xb571c4b5, 0x66ccaa66, 0x4890d848, 0x03060503, 0xf6f701f6, 0x0e1c120e, + 0x61c2a361, 0x356a5f35, 0x57aef957, 0xb969d0b9, 0x86179186, 0xc19958c1, + 0x1d3a271d, 0x9e27b99e, 0xe1d938e1, 0xf8eb13f8, 0x982bb398, 0x11223311, + 0x69d2bb69, 0xd9a970d9, 0x8e07898e, 0x9433a794, 0x9b2db69b, 0x1e3c221e, + 0x87159287, 0xe9c920e9, 0xce8749ce, 0x55aaff55, 0x28507828, 0xdfa57adf, + 0x8c038f8c, 0xa159f8a1, 0x89098089, 0x0d1a170d, 0xbf65dabf, 0xe6d731e6, + 0x4284c642, 0x68d0b868, 0x4182c341, 0x9929b099, 0x2d5a772d, 0x0f1e110f, + 0xb07bcbb0, 0x54a8fc54, 0xbb6dd6bb, 0x162c3a16, }; constexpr uint32_t te3[256] = { - 0x6363a5c6, 0x7c7c84f8, 0x777799ee, 0x7b7b8df6, 0xf2f20dff, 0x6b6bbdd6, - 0x6f6fb1de, 0xc5c55491, 0x30305060, 0x01010302, 0x6767a9ce, 0x2b2b7d56, - 0xfefe19e7, 0xd7d762b5, 0xababe64d, 0x76769aec, 0xcaca458f, 0x82829d1f, - 0xc9c94089, 0x7d7d87fa, 0xfafa15ef, 0x5959ebb2, 0x4747c98e, 0xf0f00bfb, - 0xadadec41, 0xd4d467b3, 0xa2a2fd5f, 0xafafea45, 0x9c9cbf23, 0xa4a4f753, - 0x727296e4, 0xc0c05b9b, 0xb7b7c275, 0xfdfd1ce1, 0x9393ae3d, 0x26266a4c, - 0x36365a6c, 0x3f3f417e, 0xf7f702f5, 0xcccc4f83, 0x34345c68, 0xa5a5f451, - 0xe5e534d1, 0xf1f108f9, 0x717193e2, 0xd8d873ab, 0x31315362, 0x15153f2a, - 0x04040c08, 0xc7c75295, 0x23236546, 0xc3c35e9d, 0x18182830, 0x9696a137, - 0x05050f0a, 0x9a9ab52f, 0x0707090e, 0x12123624, 0x80809b1b, 0xe2e23ddf, - 0xebeb26cd, 0x2727694e, 0xb2b2cd7f, 0x75759fea, 0x09091b12, 0x83839e1d, - 0x2c2c7458, 0x1a1a2e34, 0x1b1b2d36, 0x6e6eb2dc, 0x5a5aeeb4, 0xa0a0fb5b, - 0x5252f6a4, 0x3b3b4d76, 0xd6d661b7, 0xb3b3ce7d, 0x29297b52, 0xe3e33edd, - 0x2f2f715e, 0x84849713, 0x5353f5a6, 0xd1d168b9, 0x00000000, 0xeded2cc1, - 0x20206040, 0xfcfc1fe3, 0xb1b1c879, 0x5b5bedb6, 0x6a6abed4, 0xcbcb468d, - 0xbebed967, 0x39394b72, 0x4a4ade94, 0x4c4cd498, 0x5858e8b0, 0xcfcf4a85, - 0xd0d06bbb, 0xefef2ac5, 0xaaaae54f, 0xfbfb16ed, 0x4343c586, 0x4d4dd79a, - 0x33335566, 0x85859411, 0x4545cf8a, 0xf9f910e9, 0x02020604, 0x7f7f81fe, - 0x5050f0a0, 0x3c3c4478, 0x9f9fba25, 0xa8a8e34b, 0x5151f3a2, 0xa3a3fe5d, - 0x4040c080, 0x8f8f8a05, 0x9292ad3f, 0x9d9dbc21, 0x38384870, 0xf5f504f1, - 0xbcbcdf63, 0xb6b6c177, 0xdada75af, 0x21216342, 0x10103020, 0xffff1ae5, - 0xf3f30efd, 0xd2d26dbf, 0xcdcd4c81, 0x0c0c1418, 0x13133526, 0xecec2fc3, - 0x5f5fe1be, 0x9797a235, 0x4444cc88, 0x1717392e, 0xc4c45793, 0xa7a7f255, - 0x7e7e82fc, 0x3d3d477a, 0x6464acc8, 0x5d5de7ba, 0x19192b32, 0x737395e6, - 0x6060a0c0, 0x81819819, 0x4f4fd19e, 0xdcdc7fa3, 0x22226644, 0x2a2a7e54, - 0x9090ab3b, 0x8888830b, 0x4646ca8c, 0xeeee29c7, 0xb8b8d36b, 0x14143c28, - 0xdede79a7, 0x5e5ee2bc, 0x0b0b1d16, 0xdbdb76ad, 0xe0e03bdb, 0x32325664, - 0x3a3a4e74, 0x0a0a1e14, 0x4949db92, 0x06060a0c, 0x24246c48, 0x5c5ce4b8, - 0xc2c25d9f, 0xd3d36ebd, 0xacacef43, 0x6262a6c4, 0x9191a839, 0x9595a431, - 0xe4e437d3, 0x79798bf2, 0xe7e732d5, 0xc8c8438b, 0x3737596e, 0x6d6db7da, - 0x8d8d8c01, 0xd5d564b1, 0x4e4ed29c, 0xa9a9e049, 0x6c6cb4d8, 0x5656faac, - 0xf4f407f3, 0xeaea25cf, 0x6565afca, 0x7a7a8ef4, 0xaeaee947, 0x08081810, - 0xbabad56f, 0x787888f0, 0x25256f4a, 0x2e2e725c, 0x1c1c2438, 0xa6a6f157, - 0xb4b4c773, 0xc6c65197, 0xe8e823cb, 0xdddd7ca1, 0x74749ce8, 0x1f1f213e, - 0x4b4bdd96, 0xbdbddc61, 0x8b8b860d, 0x8a8a850f, 0x707090e0, 0x3e3e427c, - 0xb5b5c471, 0x6666aacc, 0x4848d890, 0x03030506, 0xf6f601f7, 0x0e0e121c, - 0x6161a3c2, 0x35355f6a, 0x5757f9ae, 0xb9b9d069, 0x86869117, 0xc1c15899, - 0x1d1d273a, 0x9e9eb927, 0xe1e138d9, 0xf8f813eb, 0x9898b32b, 0x11113322, - 0x6969bbd2, 0xd9d970a9, 0x8e8e8907, 0x9494a733, 0x9b9bb62d, 0x1e1e223c, - 0x87879215, 0xe9e920c9, 0xcece4987, 0x5555ffaa, 0x28287850, 0xdfdf7aa5, - 0x8c8c8f03, 0xa1a1f859, 0x89898009, 0x0d0d171a, 0xbfbfda65, 0xe6e631d7, - 0x4242c684, 0x6868b8d0, 0x4141c382, 0x9999b029, 0x2d2d775a, 0x0f0f111e, - 0xb0b0cb7b, 0x5454fca8, 0xbbbbd66d, 0x16163a2c, + 0xc6a56363, 0xf8847c7c, 0xee997777, 0xf68d7b7b, 0xff0df2f2, 0xd6bd6b6b, + 0xdeb16f6f, 0x9154c5c5, 0x60503030, 0x02030101, 0xcea96767, 0x567d2b2b, + 0xe719fefe, 0xb562d7d7, 0x4de6abab, 0xec9a7676, 0x8f45caca, 0x1f9d8282, + 0x8940c9c9, 0xfa877d7d, 0xef15fafa, 0xb2eb5959, 0x8ec94747, 0xfb0bf0f0, + 0x41ecadad, 0xb367d4d4, 0x5ffda2a2, 0x45eaafaf, 0x23bf9c9c, 0x53f7a4a4, + 0xe4967272, 0x9b5bc0c0, 0x75c2b7b7, 0xe11cfdfd, 0x3dae9393, 0x4c6a2626, + 0x6c5a3636, 0x7e413f3f, 0xf502f7f7, 0x834fcccc, 0x685c3434, 0x51f4a5a5, + 0xd134e5e5, 0xf908f1f1, 0xe2937171, 0xab73d8d8, 0x62533131, 0x2a3f1515, + 0x080c0404, 0x9552c7c7, 0x46652323, 0x9d5ec3c3, 0x30281818, 0x37a19696, + 0x0a0f0505, 0x2fb59a9a, 0x0e090707, 0x24361212, 0x1b9b8080, 0xdf3de2e2, + 0xcd26ebeb, 0x4e692727, 0x7fcdb2b2, 0xea9f7575, 0x121b0909, 0x1d9e8383, + 0x58742c2c, 0x342e1a1a, 0x362d1b1b, 0xdcb26e6e, 0xb4ee5a5a, 0x5bfba0a0, + 0xa4f65252, 0x764d3b3b, 0xb761d6d6, 0x7dceb3b3, 0x527b2929, 0xdd3ee3e3, + 0x5e712f2f, 0x13978484, 0xa6f55353, 0xb968d1d1, 0x00000000, 0xc12ceded, + 0x40602020, 0xe31ffcfc, 0x79c8b1b1, 0xb6ed5b5b, 0xd4be6a6a, 0x8d46cbcb, + 0x67d9bebe, 0x724b3939, 0x94de4a4a, 0x98d44c4c, 0xb0e85858, 0x854acfcf, + 0xbb6bd0d0, 0xc52aefef, 0x4fe5aaaa, 0xed16fbfb, 0x86c54343, 0x9ad74d4d, + 0x66553333, 0x11948585, 0x8acf4545, 0xe910f9f9, 0x04060202, 0xfe817f7f, + 0xa0f05050, 0x78443c3c, 0x25ba9f9f, 0x4be3a8a8, 0xa2f35151, 0x5dfea3a3, + 0x80c04040, 0x058a8f8f, 0x3fad9292, 0x21bc9d9d, 0x70483838, 0xf104f5f5, + 0x63dfbcbc, 0x77c1b6b6, 0xaf75dada, 0x42632121, 0x20301010, 0xe51affff, + 0xfd0ef3f3, 0xbf6dd2d2, 0x814ccdcd, 0x18140c0c, 0x26351313, 0xc32fecec, + 0xbee15f5f, 0x35a29797, 0x88cc4444, 0x2e391717, 0x9357c4c4, 0x55f2a7a7, + 0xfc827e7e, 0x7a473d3d, 0xc8ac6464, 0xbae75d5d, 0x322b1919, 0xe6957373, + 0xc0a06060, 0x19988181, 0x9ed14f4f, 0xa37fdcdc, 0x44662222, 0x547e2a2a, + 0x3bab9090, 0x0b838888, 0x8cca4646, 0xc729eeee, 0x6bd3b8b8, 0x283c1414, + 0xa779dede, 0xbce25e5e, 0x161d0b0b, 0xad76dbdb, 0xdb3be0e0, 0x64563232, + 0x744e3a3a, 0x141e0a0a, 0x92db4949, 0x0c0a0606, 0x486c2424, 0xb8e45c5c, + 0x9f5dc2c2, 0xbd6ed3d3, 0x43efacac, 0xc4a66262, 0x39a89191, 0x31a49595, + 0xd337e4e4, 0xf28b7979, 0xd532e7e7, 0x8b43c8c8, 0x6e593737, 0xdab76d6d, + 0x018c8d8d, 0xb164d5d5, 0x9cd24e4e, 0x49e0a9a9, 0xd8b46c6c, 0xacfa5656, + 0xf307f4f4, 0xcf25eaea, 0xcaaf6565, 0xf48e7a7a, 0x47e9aeae, 0x10180808, + 0x6fd5baba, 0xf0887878, 0x4a6f2525, 0x5c722e2e, 0x38241c1c, 0x57f1a6a6, + 0x73c7b4b4, 0x9751c6c6, 0xcb23e8e8, 0xa17cdddd, 0xe89c7474, 0x3e211f1f, + 0x96dd4b4b, 0x61dcbdbd, 0x0d868b8b, 0x0f858a8a, 0xe0907070, 0x7c423e3e, + 0x71c4b5b5, 0xccaa6666, 0x90d84848, 0x06050303, 0xf701f6f6, 0x1c120e0e, + 0xc2a36161, 0x6a5f3535, 0xaef95757, 0x69d0b9b9, 0x17918686, 0x9958c1c1, + 0x3a271d1d, 0x27b99e9e, 0xd938e1e1, 0xeb13f8f8, 0x2bb39898, 0x22331111, + 0xd2bb6969, 0xa970d9d9, 0x07898e8e, 0x33a79494, 0x2db69b9b, 0x3c221e1e, + 0x15928787, 0xc920e9e9, 0x8749cece, 0xaaff5555, 0x50782828, 0xa57adfdf, + 0x038f8c8c, 0x59f8a1a1, 0x09808989, 0x1a170d0d, 0x65dabfbf, 0xd731e6e6, + 0x84c64242, 0xd0b86868, 0x82c34141, 0x29b09999, 0x5a772d2d, 0x1e110f0f, + 0x7bcbb0b0, 0xa8fc5454, 0x6dd6bbbb, 0x2c3a1616, }; // Software implementation of the Vector128 class, using uint32_t @@ -235,45 +238,13 @@ struct alignas(16) Vector128 { inline ABSL_RANDOM_INTERNAL_ATTRIBUTE_ALWAYS_INLINE Vector128 Vector128Load(const void* from) { Vector128 result; - const uint8_t* src = reinterpret_cast<const uint8_t*>(from); - result.s[0] = static_cast<uint32_t>(src[0]) << 24 | - static_cast<uint32_t>(src[1]) << 16 | - static_cast<uint32_t>(src[2]) << 8 | - static_cast<uint32_t>(src[3]); - result.s[1] = static_cast<uint32_t>(src[4]) << 24 | - static_cast<uint32_t>(src[5]) << 16 | - static_cast<uint32_t>(src[6]) << 8 | - static_cast<uint32_t>(src[7]); - result.s[2] = static_cast<uint32_t>(src[8]) << 24 | - static_cast<uint32_t>(src[9]) << 16 | - static_cast<uint32_t>(src[10]) << 8 | - static_cast<uint32_t>(src[11]); - result.s[3] = static_cast<uint32_t>(src[12]) << 24 | - static_cast<uint32_t>(src[13]) << 16 | - static_cast<uint32_t>(src[14]) << 8 | - static_cast<uint32_t>(src[15]); + std::memcpy(result.s, from, sizeof(Vector128)); return result; } inline ABSL_RANDOM_INTERNAL_ATTRIBUTE_ALWAYS_INLINE void Vector128Store( const Vector128& v, void* to) { - uint8_t* dst = reinterpret_cast<uint8_t*>(to); - dst[0] = static_cast<uint8_t>(v.s[0] >> 24); - dst[1] = static_cast<uint8_t>(v.s[0] >> 16); - dst[2] = static_cast<uint8_t>(v.s[0] >> 8); - dst[3] = static_cast<uint8_t>(v.s[0]); - dst[4] = static_cast<uint8_t>(v.s[1] >> 24); - dst[5] = static_cast<uint8_t>(v.s[1] >> 16); - dst[6] = static_cast<uint8_t>(v.s[1] >> 8); - dst[7] = static_cast<uint8_t>(v.s[1]); - dst[8] = static_cast<uint8_t>(v.s[2] >> 24); - dst[9] = static_cast<uint8_t>(v.s[2] >> 16); - dst[10] = static_cast<uint8_t>(v.s[2] >> 8); - dst[11] = static_cast<uint8_t>(v.s[2]); - dst[12] = static_cast<uint8_t>(v.s[3] >> 24); - dst[13] = static_cast<uint8_t>(v.s[3] >> 16); - dst[14] = static_cast<uint8_t>(v.s[3] >> 8); - dst[15] = static_cast<uint8_t>(v.s[3]); + std::memcpy(to, v.s, sizeof(Vector128)); } // One round of AES. "round_key" is a public constant for breaking the @@ -281,39 +252,57 @@ inline ABSL_RANDOM_INTERNAL_ATTRIBUTE_ALWAYS_INLINE void Vector128Store( inline ABSL_RANDOM_INTERNAL_ATTRIBUTE_ALWAYS_INLINE Vector128 AesRound(const Vector128& state, const Vector128& round_key) { Vector128 result; +#ifdef ABSL_IS_LITTLE_ENDIAN result.s[0] = round_key.s[0] ^ // - te0[uint8_t(state.s[0] >> 24)] ^ // - te1[uint8_t(state.s[1] >> 16)] ^ // - te2[uint8_t(state.s[2] >> 8)] ^ // - te3[uint8_t(state.s[3])]; + te0[uint8_t(state.s[0])] ^ // + te1[uint8_t(state.s[1] >> 8)] ^ // + te2[uint8_t(state.s[2] >> 16)] ^ // + te3[uint8_t(state.s[3] >> 24)]; result.s[1] = round_key.s[1] ^ // - te0[uint8_t(state.s[1] >> 24)] ^ // - te1[uint8_t(state.s[2] >> 16)] ^ // - te2[uint8_t(state.s[3] >> 8)] ^ // - te3[uint8_t(state.s[0])]; + te0[uint8_t(state.s[1])] ^ // + te1[uint8_t(state.s[2] >> 8)] ^ // + te2[uint8_t(state.s[3] >> 16)] ^ // + te3[uint8_t(state.s[0] >> 24)]; result.s[2] = round_key.s[2] ^ // - te0[uint8_t(state.s[2] >> 24)] ^ // - te1[uint8_t(state.s[3] >> 16)] ^ // - te2[uint8_t(state.s[0] >> 8)] ^ // - te3[uint8_t(state.s[1])]; + te0[uint8_t(state.s[2])] ^ // + te1[uint8_t(state.s[3] >> 8)] ^ // + te2[uint8_t(state.s[0] >> 16)] ^ // + te3[uint8_t(state.s[1] >> 24)]; result.s[3] = round_key.s[3] ^ // - te0[uint8_t(state.s[3] >> 24)] ^ // - te1[uint8_t(state.s[0] >> 16)] ^ // - te2[uint8_t(state.s[1] >> 8)] ^ // - te3[uint8_t(state.s[2])]; + te0[uint8_t(state.s[3])] ^ // + te1[uint8_t(state.s[0] >> 8)] ^ // + te2[uint8_t(state.s[1] >> 16)] ^ // + te3[uint8_t(state.s[2] >> 24)]; +#else + result.s[0] = round_key.s[0] ^ // + te0[uint8_t(state.s[0])] ^ // + te1[uint8_t(state.s[3] >> 8)] ^ // + te2[uint8_t(state.s[2] >> 16)] ^ // + te3[uint8_t(state.s[1] >> 24)]; + result.s[1] = round_key.s[1] ^ // + te0[uint8_t(state.s[1])] ^ // + te1[uint8_t(state.s[0] >> 8)] ^ // + te2[uint8_t(state.s[3] >> 16)] ^ // + te3[uint8_t(state.s[2] >> 24)]; + result.s[2] = round_key.s[2] ^ // + te0[uint8_t(state.s[2])] ^ // + te1[uint8_t(state.s[1] >> 8)] ^ // + te2[uint8_t(state.s[0] >> 16)] ^ // + te3[uint8_t(state.s[3] >> 24)]; + result.s[3] = round_key.s[3] ^ // + te0[uint8_t(state.s[3])] ^ // + te1[uint8_t(state.s[2] >> 8)] ^ // + te2[uint8_t(state.s[1] >> 16)] ^ // + te3[uint8_t(state.s[0] >> 24)]; +#endif return result; } using ::absl::random_internal::RandenTraits; -// Randen operates on 128-bit vectors. -struct alignas(16) u64x2 { - uint64_t data[2]; -}; - // The improved Feistel block shuffle function for 16 blocks. inline ABSL_RANDOM_INTERNAL_ATTRIBUTE_ALWAYS_INLINE void BlockShuffle( - u64x2* state) { + absl::uint128* state) { static_assert(RandenTraits::kFeistelBlocks == 16, "Feistel block shuffle only works for 16 blocks."); @@ -323,31 +312,31 @@ inline ABSL_RANDOM_INTERNAL_ATTRIBUTE_ALWAYS_INLINE void BlockShuffle( // The fully unrolled loop without the memcpy improves the speed by about // 30% over the equivalent: #if 0 - u64x2 source[RandenTraits::kFeistelBlocks]; + absl::uint128 source[RandenTraits::kFeistelBlocks]; std::memcpy(source, state, sizeof(source)); for (size_t i = 0; i < RandenTraits::kFeistelBlocks; i++) { - const u64x2 v0 = source[shuffle[i]]; + const absl::uint128 v0 = source[shuffle[i]]; state[i] = v0; } return; #endif - const u64x2 v0 = state[shuffle[0]]; - const u64x2 v1 = state[shuffle[1]]; - const u64x2 v2 = state[shuffle[2]]; - const u64x2 v3 = state[shuffle[3]]; - const u64x2 v4 = state[shuffle[4]]; - const u64x2 v5 = state[shuffle[5]]; - const u64x2 v6 = state[shuffle[6]]; - const u64x2 v7 = state[shuffle[7]]; - const u64x2 w0 = state[shuffle[8]]; - const u64x2 w1 = state[shuffle[9]]; - const u64x2 w2 = state[shuffle[10]]; - const u64x2 w3 = state[shuffle[11]]; - const u64x2 w4 = state[shuffle[12]]; - const u64x2 w5 = state[shuffle[13]]; - const u64x2 w6 = state[shuffle[14]]; - const u64x2 w7 = state[shuffle[15]]; + const absl::uint128 v0 = state[shuffle[0]]; + const absl::uint128 v1 = state[shuffle[1]]; + const absl::uint128 v2 = state[shuffle[2]]; + const absl::uint128 v3 = state[shuffle[3]]; + const absl::uint128 v4 = state[shuffle[4]]; + const absl::uint128 v5 = state[shuffle[5]]; + const absl::uint128 v6 = state[shuffle[6]]; + const absl::uint128 v7 = state[shuffle[7]]; + const absl::uint128 w0 = state[shuffle[8]]; + const absl::uint128 w1 = state[shuffle[9]]; + const absl::uint128 w2 = state[shuffle[10]]; + const absl::uint128 w3 = state[shuffle[11]]; + const absl::uint128 w4 = state[shuffle[12]]; + const absl::uint128 w5 = state[shuffle[13]]; + const absl::uint128 w6 = state[shuffle[14]]; + const absl::uint128 w7 = state[shuffle[15]]; state[0] = v0; state[1] = v1; state[2] = v2; @@ -371,9 +360,9 @@ inline ABSL_RANDOM_INTERNAL_ATTRIBUTE_ALWAYS_INLINE void BlockShuffle( // per 16 bytes (vs. 10 for AES-CTR). Computing eight round functions in // parallel hides the 7-cycle AESNI latency on HSW. Note that the Feistel // XORs are 'free' (included in the second AES instruction). -inline ABSL_RANDOM_INTERNAL_ATTRIBUTE_ALWAYS_INLINE const u64x2* FeistelRound( - u64x2* ABSL_RANDOM_INTERNAL_RESTRICT state, - const u64x2* ABSL_RANDOM_INTERNAL_RESTRICT keys) { +inline ABSL_RANDOM_INTERNAL_ATTRIBUTE_ALWAYS_INLINE const absl::uint128* +FeistelRound(absl::uint128* ABSL_RANDOM_INTERNAL_RESTRICT state, + const absl::uint128* ABSL_RANDOM_INTERNAL_RESTRICT keys) { for (size_t branch = 0; branch < RandenTraits::kFeistelBlocks; branch += 4) { const Vector128 s0 = Vector128Load(state + branch); const Vector128 s1 = Vector128Load(state + branch + 1); @@ -398,13 +387,31 @@ inline ABSL_RANDOM_INTERNAL_ATTRIBUTE_ALWAYS_INLINE const u64x2* FeistelRound( // 2^64 queries if the round function is a PRF. This is similar to the b=8 case // of Simpira v2, but more efficient than its generic construction for b=16. inline ABSL_RANDOM_INTERNAL_ATTRIBUTE_ALWAYS_INLINE void Permute( - u64x2* state, const u64x2* ABSL_RANDOM_INTERNAL_RESTRICT keys) { + absl::uint128* state, + const absl::uint128* ABSL_RANDOM_INTERNAL_RESTRICT keys) { for (size_t round = 0; round < RandenTraits::kFeistelRounds; ++round) { keys = FeistelRound(state, keys); BlockShuffle(state); } } +// Enables native loads in the round loop by pre-swapping. +inline ABSL_RANDOM_INTERNAL_ATTRIBUTE_ALWAYS_INLINE void SwapEndian( + absl::uint128* state) { +#ifdef ABSL_IS_BIG_ENDIAN + for (uint32_t block = 0; block < RandenTraits::kFeistelBlocks; ++block) { + uint64_t new_lo = absl::little_endian::ToHost64( + static_cast<uint64_t>(state[block] >> 64)); + uint64_t new_hi = absl::little_endian::ToHost64( + static_cast<uint64_t>((state[block] << 64) >> 64)); + state[block] = (static_cast<absl::uint128>(new_hi) << 64) | new_lo; + } +#else + // Avoid warning about unused variable. + (void)state; +#endif +} + } // namespace namespace absl { @@ -414,7 +421,11 @@ namespace random_internal { const void* RandenSlow::GetKeys() { // Round keys for one AES per Feistel round and branch. // The canonical implementation uses first digits of Pi. +#ifdef ABSL_IS_LITTLE_ENDIAN return kRandenRoundKeys; +#else + return kRandenRoundKeysBE; +#endif } void RandenSlow::Absorb(const void* seed_void, void* state_void) { @@ -437,19 +448,22 @@ void RandenSlow::Absorb(const void* seed_void, void* state_void) { } void RandenSlow::Generate(const void* keys_void, void* state_void) { - static_assert(RandenTraits::kCapacityBytes == sizeof(u64x2), + static_assert(RandenTraits::kCapacityBytes == sizeof(absl::uint128), "Capacity mismatch"); - auto* state = reinterpret_cast<u64x2*>(state_void); - const auto* keys = reinterpret_cast<const u64x2*>(keys_void); + auto* state = reinterpret_cast<absl::uint128*>(state_void); + const auto* keys = reinterpret_cast<const absl::uint128*>(keys_void); - const u64x2 prev_inner = state[0]; + const absl::uint128 prev_inner = state[0]; + + SwapEndian(state); Permute(state, keys); + SwapEndian(state); + // Ensure backtracking resistance. - state[0].data[0] ^= prev_inner.data[0]; - state[0].data[1] ^= prev_inner.data[1]; + *state ^= prev_inner; } } // namespace random_internal diff --git a/absl/random/internal/randen_slow_test.cc b/absl/random/internal/randen_slow_test.cc index 4861ffa4..ed603958 100644 --- a/absl/random/internal/randen_slow_test.cc +++ b/absl/random/internal/randen_slow_test.cc @@ -25,40 +25,37 @@ namespace { using absl::random_internal::RandenSlow; using absl::random_internal::RandenTraits; -// Local state parameters. -constexpr size_t kSeedBytes = - RandenTraits::kStateBytes - RandenTraits::kCapacityBytes; -constexpr size_t kStateSizeT = RandenTraits::kStateBytes / sizeof(uint64_t); -constexpr size_t kSeedSizeT = kSeedBytes / sizeof(uint32_t); - -struct alignas(16) randen { - uint64_t state[kStateSizeT]; - uint32_t seed[kSeedSizeT]; -}; - TEST(RandenSlowTest, Default) { - constexpr uint64_t kGolden[] = { - 0x6c6534090ee6d3ee, 0x044e2b9b9d5333c6, 0xc3c14f134e433977, - 0xdda9f47cd90410ee, 0x887bf3087fd8ca10, 0xf0b780f545c72912, - 0x15dbb1d37696599f, 0x30ec63baff3c6d59, 0xb29f73606f7f20a6, - 0x02808a316f49a54c, 0x3b8feaf9d5c8e50e, 0x9cbf605e3fd9de8a, - 0xc970ae1a78183bbb, 0xd8b2ffd356301ed5, 0xf4b327fe0fc73c37, - 0xcdfd8d76eb8f9a19, 0xc3a506eb91420c9d, 0xd5af05dd3eff9556, - 0x48db1bb78f83c4a1, 0x7023920e0d6bfe8c, 0x58d3575834956d42, - 0xed1ef4c26b87b840, 0x8eef32a23e0b2df3, 0x497cabf3431154fc, - 0x4e24370570029a8b, 0xd88b5749f090e5ea, 0xc651a582a970692f, - 0x78fcec2cbb6342f5, 0x463cb745612f55db, 0x352ee4ad1816afe3, - 0x026ff374c101da7e, 0x811ef0821c3de851, + constexpr uint8_t kGolden[] = { + 0xee, 0xd3, 0xe6, 0x0e, 0x09, 0x34, 0x65, 0x6c, 0xc6, 0x33, 0x53, 0x9d, + 0x9b, 0x2b, 0x4e, 0x04, 0x77, 0x39, 0x43, 0x4e, 0x13, 0x4f, 0xc1, 0xc3, + 0xee, 0x10, 0x04, 0xd9, 0x7c, 0xf4, 0xa9, 0xdd, 0x10, 0xca, 0xd8, 0x7f, + 0x08, 0xf3, 0x7b, 0x88, 0x12, 0x29, 0xc7, 0x45, 0xf5, 0x80, 0xb7, 0xf0, + 0x9f, 0x59, 0x96, 0x76, 0xd3, 0xb1, 0xdb, 0x15, 0x59, 0x6d, 0x3c, 0xff, + 0xba, 0x63, 0xec, 0x30, 0xa6, 0x20, 0x7f, 0x6f, 0x60, 0x73, 0x9f, 0xb2, + 0x4c, 0xa5, 0x49, 0x6f, 0x31, 0x8a, 0x80, 0x02, 0x0e, 0xe5, 0xc8, 0xd5, + 0xf9, 0xea, 0x8f, 0x3b, 0x8a, 0xde, 0xd9, 0x3f, 0x5e, 0x60, 0xbf, 0x9c, + 0xbb, 0x3b, 0x18, 0x78, 0x1a, 0xae, 0x70, 0xc9, 0xd5, 0x1e, 0x30, 0x56, + 0xd3, 0xff, 0xb2, 0xd8, 0x37, 0x3c, 0xc7, 0x0f, 0xfe, 0x27, 0xb3, 0xf4, + 0x19, 0x9a, 0x8f, 0xeb, 0x76, 0x8d, 0xfd, 0xcd, 0x9d, 0x0c, 0x42, 0x91, + 0xeb, 0x06, 0xa5, 0xc3, 0x56, 0x95, 0xff, 0x3e, 0xdd, 0x05, 0xaf, 0xd5, + 0xa1, 0xc4, 0x83, 0x8f, 0xb7, 0x1b, 0xdb, 0x48, 0x8c, 0xfe, 0x6b, 0x0d, + 0x0e, 0x92, 0x23, 0x70, 0x42, 0x6d, 0x95, 0x34, 0x58, 0x57, 0xd3, 0x58, + 0x40, 0xb8, 0x87, 0x6b, 0xc2, 0xf4, 0x1e, 0xed, 0xf3, 0x2d, 0x0b, 0x3e, + 0xa2, 0x32, 0xef, 0x8e, 0xfc, 0x54, 0x11, 0x43, 0xf3, 0xab, 0x7c, 0x49, + 0x8b, 0x9a, 0x02, 0x70, 0x05, 0x37, 0x24, 0x4e, 0xea, 0xe5, 0x90, 0xf0, + 0x49, 0x57, 0x8b, 0xd8, 0x2f, 0x69, 0x70, 0xa9, 0x82, 0xa5, 0x51, 0xc6, + 0xf5, 0x42, 0x63, 0xbb, 0x2c, 0xec, 0xfc, 0x78, 0xdb, 0x55, 0x2f, 0x61, + 0x45, 0xb7, 0x3c, 0x46, 0xe3, 0xaf, 0x16, 0x18, 0xad, 0xe4, 0x2e, 0x35, + 0x7e, 0xda, 0x01, 0xc1, 0x74, 0xf3, 0x6f, 0x02, 0x51, 0xe8, 0x3d, 0x1c, + 0x82, 0xf0, 0x1e, 0x81, }; - alignas(16) randen d; - std::memset(d.state, 0, sizeof(d.state)); - RandenSlow::Generate(RandenSlow::GetKeys(), d.state); + alignas(16) uint8_t state[RandenTraits::kStateBytes]; + std::memset(state, 0, sizeof(state)); - uint64_t* id = d.state; - for (const auto& elem : kGolden) { - EXPECT_EQ(absl::little_endian::FromHost64(elem), *id++); - } + RandenSlow::Generate(RandenSlow::GetKeys(), state); + EXPECT_EQ(0, std::memcmp(state, kGolden, sizeof(state))); } } // namespace diff --git a/absl/random/internal/randen_test.cc b/absl/random/internal/randen_test.cc index c186fe0d..92773b8d 100644 --- a/absl/random/internal/randen_test.cc +++ b/absl/random/internal/randen_test.cc @@ -23,9 +23,6 @@ namespace { using absl::random_internal::Randen; -// Local state parameters. -constexpr size_t kStateSizeT = Randen::kStateBytes / sizeof(uint64_t); - TEST(RandenTest, CopyAndMove) { static_assert(std::is_copy_constructible<Randen>::value, "Randen must be copy constructible"); @@ -41,30 +38,38 @@ TEST(RandenTest, CopyAndMove) { } TEST(RandenTest, Default) { - constexpr uint64_t kGolden[] = { - 0x6c6534090ee6d3ee, 0x044e2b9b9d5333c6, 0xc3c14f134e433977, - 0xdda9f47cd90410ee, 0x887bf3087fd8ca10, 0xf0b780f545c72912, - 0x15dbb1d37696599f, 0x30ec63baff3c6d59, 0xb29f73606f7f20a6, - 0x02808a316f49a54c, 0x3b8feaf9d5c8e50e, 0x9cbf605e3fd9de8a, - 0xc970ae1a78183bbb, 0xd8b2ffd356301ed5, 0xf4b327fe0fc73c37, - 0xcdfd8d76eb8f9a19, 0xc3a506eb91420c9d, 0xd5af05dd3eff9556, - 0x48db1bb78f83c4a1, 0x7023920e0d6bfe8c, 0x58d3575834956d42, - 0xed1ef4c26b87b840, 0x8eef32a23e0b2df3, 0x497cabf3431154fc, - 0x4e24370570029a8b, 0xd88b5749f090e5ea, 0xc651a582a970692f, - 0x78fcec2cbb6342f5, 0x463cb745612f55db, 0x352ee4ad1816afe3, - 0x026ff374c101da7e, 0x811ef0821c3de851, + constexpr uint8_t kGolden[] = { + 0xee, 0xd3, 0xe6, 0x0e, 0x09, 0x34, 0x65, 0x6c, 0xc6, 0x33, 0x53, 0x9d, + 0x9b, 0x2b, 0x4e, 0x04, 0x77, 0x39, 0x43, 0x4e, 0x13, 0x4f, 0xc1, 0xc3, + 0xee, 0x10, 0x04, 0xd9, 0x7c, 0xf4, 0xa9, 0xdd, 0x10, 0xca, 0xd8, 0x7f, + 0x08, 0xf3, 0x7b, 0x88, 0x12, 0x29, 0xc7, 0x45, 0xf5, 0x80, 0xb7, 0xf0, + 0x9f, 0x59, 0x96, 0x76, 0xd3, 0xb1, 0xdb, 0x15, 0x59, 0x6d, 0x3c, 0xff, + 0xba, 0x63, 0xec, 0x30, 0xa6, 0x20, 0x7f, 0x6f, 0x60, 0x73, 0x9f, 0xb2, + 0x4c, 0xa5, 0x49, 0x6f, 0x31, 0x8a, 0x80, 0x02, 0x0e, 0xe5, 0xc8, 0xd5, + 0xf9, 0xea, 0x8f, 0x3b, 0x8a, 0xde, 0xd9, 0x3f, 0x5e, 0x60, 0xbf, 0x9c, + 0xbb, 0x3b, 0x18, 0x78, 0x1a, 0xae, 0x70, 0xc9, 0xd5, 0x1e, 0x30, 0x56, + 0xd3, 0xff, 0xb2, 0xd8, 0x37, 0x3c, 0xc7, 0x0f, 0xfe, 0x27, 0xb3, 0xf4, + 0x19, 0x9a, 0x8f, 0xeb, 0x76, 0x8d, 0xfd, 0xcd, 0x9d, 0x0c, 0x42, 0x91, + 0xeb, 0x06, 0xa5, 0xc3, 0x56, 0x95, 0xff, 0x3e, 0xdd, 0x05, 0xaf, 0xd5, + 0xa1, 0xc4, 0x83, 0x8f, 0xb7, 0x1b, 0xdb, 0x48, 0x8c, 0xfe, 0x6b, 0x0d, + 0x0e, 0x92, 0x23, 0x70, 0x42, 0x6d, 0x95, 0x34, 0x58, 0x57, 0xd3, 0x58, + 0x40, 0xb8, 0x87, 0x6b, 0xc2, 0xf4, 0x1e, 0xed, 0xf3, 0x2d, 0x0b, 0x3e, + 0xa2, 0x32, 0xef, 0x8e, 0xfc, 0x54, 0x11, 0x43, 0xf3, 0xab, 0x7c, 0x49, + 0x8b, 0x9a, 0x02, 0x70, 0x05, 0x37, 0x24, 0x4e, 0xea, 0xe5, 0x90, 0xf0, + 0x49, 0x57, 0x8b, 0xd8, 0x2f, 0x69, 0x70, 0xa9, 0x82, 0xa5, 0x51, 0xc6, + 0xf5, 0x42, 0x63, 0xbb, 0x2c, 0xec, 0xfc, 0x78, 0xdb, 0x55, 0x2f, 0x61, + 0x45, 0xb7, 0x3c, 0x46, 0xe3, 0xaf, 0x16, 0x18, 0xad, 0xe4, 0x2e, 0x35, + 0x7e, 0xda, 0x01, 0xc1, 0x74, 0xf3, 0x6f, 0x02, 0x51, 0xe8, 0x3d, 0x1c, + 0x82, 0xf0, 0x1e, 0x81, }; - alignas(16) uint64_t state[kStateSizeT]; + alignas(16) uint8_t state[Randen::kStateBytes]; std::memset(state, 0, sizeof(state)); Randen r; r.Generate(state); - auto id = std::begin(state); - for (const auto& elem : kGolden) { - EXPECT_EQ(elem, *id++); - } + EXPECT_EQ(0, std::memcmp(state, kGolden, sizeof(state))); } } // namespace diff --git a/absl/random/internal/salted_seed_seq.h b/absl/random/internal/salted_seed_seq.h index 5953a090..06291865 100644 --- a/absl/random/internal/salted_seed_seq.h +++ b/absl/random/internal/salted_seed_seq.h @@ -22,6 +22,7 @@ #include <memory> #include <type_traits> #include <utility> +#include <vector> #include "absl/container/inlined_vector.h" #include "absl/meta/type_traits.h" @@ -65,15 +66,19 @@ class SaltedSeedSeq { template <typename RandomAccessIterator> void generate(RandomAccessIterator begin, RandomAccessIterator end) { + using U = typename std::iterator_traits<RandomAccessIterator>::value_type; + // The common case is that generate is called with ContiguousIterators // to uint arrays. Such contiguous memory regions may be optimized, // which we detect here. - using tag = absl::conditional_t< - (std::is_pointer<RandomAccessIterator>::value && - std::is_same<absl::decay_t<decltype(*begin)>, uint32_t>::value), + using TagType = absl::conditional_t< + (std::is_same<U, uint32_t>::value && + (std::is_pointer<RandomAccessIterator>::value || + std::is_same<RandomAccessIterator, + typename std::vector<U>::iterator>::value)), ContiguousAndUint32Tag, DefaultTag>; if (begin != end) { - generate_impl(begin, end, tag{}); + generate_impl(TagType{}, begin, end, std::distance(begin, end)); } } @@ -89,8 +94,15 @@ class SaltedSeedSeq { struct DefaultTag {}; // Generate which requires the iterators are contiguous pointers to uint32_t. - void generate_impl(uint32_t* begin, uint32_t* end, ContiguousAndUint32Tag) { - generate_contiguous(absl::MakeSpan(begin, end)); + // Fills the initial seed buffer the underlying SSeq::generate() call, + // then mixes in the salt material. + template <typename Contiguous> + void generate_impl(ContiguousAndUint32Tag, Contiguous begin, Contiguous end, + size_t n) { + seq_->generate(begin, end); + const uint32_t salt = absl::random_internal::GetSaltMaterial().value_or(0); + auto span = absl::Span<uint32_t>(&*begin, n); + MixIntoSeedMaterial(absl::MakeConstSpan(&salt, 1), span); } // The uncommon case for generate is that it is called with iterators over @@ -98,27 +110,13 @@ class SaltedSeedSeq { // case we allocate a temporary 32-bit buffer and then copy-assign back // to the initial inputs. template <typename RandomAccessIterator> - void generate_impl(RandomAccessIterator begin, RandomAccessIterator end, - DefaultTag) { - return generate_and_copy(std::distance(begin, end), begin); - } - - // Fills the initial seed buffer the underlying SSeq::generate() call, - // mixing in the salt material. - void generate_contiguous(absl::Span<uint32_t> buffer) { - seq_->generate(buffer.begin(), buffer.end()); - const uint32_t salt = absl::random_internal::GetSaltMaterial().value_or(0); - MixIntoSeedMaterial(absl::MakeConstSpan(&salt, 1), buffer); - } - - // Allocates a seed buffer of `n` elements, generates the seed, then - // copies the result into the `out` iterator. - template <typename Iterator> - void generate_and_copy(size_t n, Iterator out) { - // Allocate a temporary buffer, generate, and then copy. + void generate_impl(DefaultTag, RandomAccessIterator begin, + RandomAccessIterator, size_t n) { + // Allocates a seed buffer of `n` elements, generates the seed, then + // copies the result into the `out` iterator. absl::InlinedVector<uint32_t, 8> data(n, 0); - generate_contiguous(absl::MakeSpan(data.data(), data.size())); - std::copy(data.begin(), data.end(), out); + generate_impl(ContiguousAndUint32Tag{}, data.begin(), data.end(), n); + std::copy(data.begin(), data.end(), begin); } // Because [rand.req.seedseq] is not required to be copy-constructible, diff --git a/absl/random/internal/seed_material.cc b/absl/random/internal/seed_material.cc index 4d38a574..c03cad85 100644 --- a/absl/random/internal/seed_material.cc +++ b/absl/random/internal/seed_material.cc @@ -28,6 +28,7 @@ #include <cstdlib> #include <cstring> +#include "absl/base/dynamic_annotations.h" #include "absl/base/internal/raw_logging.h" #include "absl/strings/ascii.h" #include "absl/strings/escaping.h" @@ -50,6 +51,18 @@ #endif +#if defined(__GLIBC__) && \ + (__GLIBC__ > 2 || (__GLIBC__ == 2 && __GLIBC_MINOR__ >= 25)) +// glibc >= 2.25 has getentropy() +#define ABSL_RANDOM_USE_GET_ENTROPY 1 +#endif + +#if defined(__EMSCRIPTEN__) +#include <sys/random.h> +// Emscripten has getentropy, but it resides in a different header. +#define ABSL_RANDOM_USE_GET_ENTROPY 1 +#endif + #if defined(ABSL_RANDOM_USE_BCRYPT) #include <bcrypt.h> @@ -122,8 +135,32 @@ bool ReadSeedMaterialFromOSEntropyImpl(absl::Span<uint32_t> values) { #else +#if defined(ABSL_RANDOM_USE_GET_ENTROPY) +// On *nix, use getentropy() if supported. Note that libc may support +// getentropy(), but the kernel may not, in which case this function will return +// false. +bool ReadSeedMaterialFromGetEntropy(absl::Span<uint32_t> values) { + auto buffer = reinterpret_cast<uint8_t*>(values.data()); + size_t buffer_size = sizeof(uint32_t) * values.size(); + while (buffer_size > 0) { + // getentropy() has a maximum permitted length of 256. + size_t to_read = std::min<size_t>(buffer_size, 256); + int result = getentropy(buffer, to_read); + if (result < 0) { + return false; + } + // https://github.com/google/sanitizers/issues/1173 + // MemorySanitizer can't see through getentropy(). + ABSL_ANNOTATE_MEMORY_IS_INITIALIZED(buffer, to_read); + buffer += to_read; + buffer_size -= to_read; + } + return true; +} +#endif // defined(ABSL_RANDOM_GETENTROPY) + // On *nix, read entropy from /dev/urandom. -bool ReadSeedMaterialFromOSEntropyImpl(absl::Span<uint32_t> values) { +bool ReadSeedMaterialFromDevURandom(absl::Span<uint32_t> values) { const char kEntropyFile[] = "/dev/urandom"; auto buffer = reinterpret_cast<uint8_t*>(values.data()); @@ -150,6 +187,17 @@ bool ReadSeedMaterialFromOSEntropyImpl(absl::Span<uint32_t> values) { return success; } +bool ReadSeedMaterialFromOSEntropyImpl(absl::Span<uint32_t> values) { +#if defined(ABSL_RANDOM_USE_GET_ENTROPY) + if (ReadSeedMaterialFromGetEntropy(values)) { + return true; + } +#endif + // Libc may support getentropy, but the kernel may not, so we still have + // to fallback to ReadSeedMaterialFromDevURandom(). + return ReadSeedMaterialFromDevURandom(values); +} + #endif } // namespace diff --git a/absl/random/internal/traits.h b/absl/random/internal/traits.h index 75772bd9..f874a0f7 100644 --- a/absl/random/internal/traits.h +++ b/absl/random/internal/traits.h @@ -20,6 +20,8 @@ #include <type_traits> #include "absl/base/config.h" +#include "absl/numeric/bits.h" +#include "absl/numeric/int128.h" namespace absl { ABSL_NAMESPACE_BEGIN @@ -59,6 +61,31 @@ class is_widening_convertible { rank<A>() <= rank<B>(); }; +template <typename T> +struct IsIntegral : std::is_integral<T> {}; +template <> +struct IsIntegral<absl::int128> : std::true_type {}; +template <> +struct IsIntegral<absl::uint128> : std::true_type {}; + +template <typename T> +struct MakeUnsigned : std::make_unsigned<T> {}; +template <> +struct MakeUnsigned<absl::int128> { + using type = absl::uint128; +}; +template <> +struct MakeUnsigned<absl::uint128> { + using type = absl::uint128; +}; + +template <typename T> +struct IsUnsigned : std::is_unsigned<T> {}; +template <> +struct IsUnsigned<absl::int128> : std::false_type {}; +template <> +struct IsUnsigned<absl::uint128> : std::true_type {}; + // unsigned_bits<N>::type returns the unsigned int type with the indicated // number of bits. template <size_t N> @@ -81,19 +108,40 @@ struct unsigned_bits<64> { using type = uint64_t; }; -#ifdef ABSL_HAVE_INTRINSIC_INT128 template <> struct unsigned_bits<128> { - using type = __uint128_t; + using type = absl::uint128; +}; + +// 256-bit wrapper for wide multiplications. +struct U256 { + uint128 hi; + uint128 lo; +}; +template <> +struct unsigned_bits<256> { + using type = U256; }; -#endif template <typename IntType> struct make_unsigned_bits { - using type = typename unsigned_bits<std::numeric_limits< - typename std::make_unsigned<IntType>::type>::digits>::type; + using type = typename unsigned_bits< + std::numeric_limits<typename MakeUnsigned<IntType>::type>::digits>::type; }; +template <typename T> +int BitWidth(T v) { + // Workaround for bit_width not supporting int128. + // Don't hardcode `64` to make sure this code does not trigger compiler + // warnings in smaller types. + constexpr int half_bits = sizeof(T) * 8 / 2; + if (sizeof(T) == 16 && (v >> half_bits) != 0) { + return bit_width(static_cast<uint64_t>(v >> half_bits)) + half_bits; + } else { + return bit_width(static_cast<uint64_t>(v)); + } +} + } // namespace random_internal ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/random/internal/uniform_helper.h b/absl/random/internal/uniform_helper.h index 1243bc1c..e68b82ee 100644 --- a/absl/random/internal/uniform_helper.h +++ b/absl/random/internal/uniform_helper.h @@ -100,7 +100,7 @@ using uniform_inferred_return_t = template <typename IntType, typename Tag> typename absl::enable_if_t< absl::conjunction< - std::is_integral<IntType>, + IsIntegral<IntType>, absl::disjunction<std::is_same<Tag, IntervalOpenClosedTag>, std::is_same<Tag, IntervalOpenOpenTag>>>::value, IntType> @@ -131,7 +131,7 @@ uniform_lower_bound(Tag, NumType a, NumType) { template <typename IntType, typename Tag> typename absl::enable_if_t< absl::conjunction< - std::is_integral<IntType>, + IsIntegral<IntType>, absl::disjunction<std::is_same<Tag, IntervalClosedOpenTag>, std::is_same<Tag, IntervalOpenOpenTag>>>::value, IntType> @@ -153,7 +153,7 @@ uniform_upper_bound(Tag, FloatType, FloatType b) { template <typename IntType, typename Tag> typename absl::enable_if_t< absl::conjunction< - std::is_integral<IntType>, + IsIntegral<IntType>, absl::disjunction<std::is_same<Tag, IntervalClosedClosedTag>, std::is_same<Tag, IntervalOpenClosedTag>>>::value, IntType> @@ -201,7 +201,7 @@ is_uniform_range_valid(FloatType a, FloatType b) { } template <typename IntType> -absl::enable_if_t<std::is_integral<IntType>::value, bool> +absl::enable_if_t<IsIntegral<IntType>::value, bool> is_uniform_range_valid(IntType a, IntType b) { return a <= b; } @@ -210,7 +210,7 @@ is_uniform_range_valid(IntType a, IntType b) { // or absl::uniform_real_distribution depending on the NumType parameter. template <typename NumType> using UniformDistribution = - typename std::conditional<std::is_integral<NumType>::value, + typename std::conditional<IsIntegral<NumType>::value, absl::uniform_int_distribution<NumType>, absl::uniform_real_distribution<NumType>>::type; diff --git a/absl/random/internal/wide_multiply.h b/absl/random/internal/wide_multiply.h index b6e6c4b6..891e3630 100644 --- a/absl/random/internal/wide_multiply.h +++ b/absl/random/internal/wide_multiply.h @@ -34,43 +34,6 @@ namespace absl { ABSL_NAMESPACE_BEGIN namespace random_internal { -// Helper object to multiply two 64-bit values to a 128-bit value. -// MultiplyU64ToU128 multiplies two 64-bit values to a 128-bit value. -// If an intrinsic is available, it is used, otherwise use native 32-bit -// multiplies to construct the result. -inline absl::uint128 MultiplyU64ToU128(uint64_t a, uint64_t b) { -#if defined(ABSL_HAVE_INTRINSIC_INT128) - return absl::uint128(static_cast<__uint128_t>(a) * b); -#elif defined(ABSL_INTERNAL_USE_UMUL128) - // uint64_t * uint64_t => uint128 multiply using imul intrinsic on MSVC. - uint64_t high = 0; - const uint64_t low = _umul128(a, b, &high); - return absl::MakeUint128(high, low); -#else - // uint128(a) * uint128(b) in emulated mode computes a full 128-bit x 128-bit - // multiply. However there are many cases where that is not necessary, and it - // is only necessary to support a 64-bit x 64-bit = 128-bit multiply. This is - // for those cases. - const uint64_t a00 = static_cast<uint32_t>(a); - const uint64_t a32 = a >> 32; - const uint64_t b00 = static_cast<uint32_t>(b); - const uint64_t b32 = b >> 32; - - const uint64_t c00 = a00 * b00; - const uint64_t c32a = a00 * b32; - const uint64_t c32b = a32 * b00; - const uint64_t c64 = a32 * b32; - - const uint32_t carry = - static_cast<uint32_t>(((c00 >> 32) + static_cast<uint32_t>(c32a) + - static_cast<uint32_t>(c32b)) >> - 32); - - return absl::MakeUint128(c64 + (c32a >> 32) + (c32b >> 32) + carry, - c00 + (c32a << 32) + (c32b << 32)); -#endif -} - // wide_multiply<T> multiplies two N-bit values to a 2N-bit result. template <typename UIntType> struct wide_multiply { @@ -82,27 +45,49 @@ struct wide_multiply { return static_cast<result_type>(a) * b; } - static input_type hi(result_type r) { return r >> kN; } - static input_type lo(result_type r) { return r; } + static input_type hi(result_type r) { + return static_cast<input_type>(r >> kN); + } + static input_type lo(result_type r) { return static_cast<input_type>(r); } static_assert(std::is_unsigned<UIntType>::value, "Class-template wide_multiply<> argument must be unsigned."); }; -#ifndef ABSL_HAVE_INTRINSIC_INT128 +// MultiplyU128ToU256 multiplies two 128-bit values to a 256-bit value. +inline U256 MultiplyU128ToU256(uint128 a, uint128 b) { + const uint128 a00 = static_cast<uint64_t>(a); + const uint128 a64 = a >> 64; + const uint128 b00 = static_cast<uint64_t>(b); + const uint128 b64 = b >> 64; + + const uint128 c00 = a00 * b00; + const uint128 c64a = a00 * b64; + const uint128 c64b = a64 * b00; + const uint128 c128 = a64 * b64; + + const uint64_t carry = + static_cast<uint64_t>(((c00 >> 64) + static_cast<uint64_t>(c64a) + + static_cast<uint64_t>(c64b)) >> + 64); + + return {c128 + (c64a >> 64) + (c64b >> 64) + carry, + c00 + (c64a << 64) + (c64b << 64)}; +} + + template <> -struct wide_multiply<uint64_t> { - using input_type = uint64_t; - using result_type = absl::uint128; +struct wide_multiply<uint128> { + using input_type = uint128; + using result_type = U256; - static result_type multiply(uint64_t a, uint64_t b) { - return MultiplyU64ToU128(a, b); + static result_type multiply(input_type a, input_type b) { + return MultiplyU128ToU256(a, b); } - static uint64_t hi(result_type r) { return absl::Uint128High64(r); } - static uint64_t lo(result_type r) { return absl::Uint128Low64(r); } + static input_type hi(result_type r) { return r.hi; } + static input_type lo(result_type r) { return r.lo; } }; -#endif } // namespace random_internal ABSL_NAMESPACE_END diff --git a/absl/random/internal/wide_multiply_test.cc b/absl/random/internal/wide_multiply_test.cc index e276cb51..f8ee35c0 100644 --- a/absl/random/internal/wide_multiply_test.cc +++ b/absl/random/internal/wide_multiply_test.cc @@ -14,52 +14,106 @@ #include "absl/random/internal/wide_multiply.h" +#include "gmock/gmock.h" #include "gtest/gtest.h" #include "absl/numeric/int128.h" -using absl::random_internal::MultiplyU64ToU128; +using absl::random_internal::MultiplyU128ToU256; +using absl::random_internal::U256; namespace { -TEST(WideMultiplyTest, MultiplyU64ToU128Test) { - constexpr uint64_t k1 = 1; - constexpr uint64_t kMax = ~static_cast<uint64_t>(0); +U256 LeftShift(U256 v, int s) { + if (s == 0) { + return v; + } else if (s < 128) { + return {(v.hi << s) | (v.lo >> (128 - s)), v.lo << s}; + } else { + return {v.lo << (s - 128), 0}; + } +} + +MATCHER_P2(Eq256, hi, lo, "") { return arg.hi == hi && arg.lo == lo; } +MATCHER_P(Eq256, v, "") { return arg.hi == v.hi && arg.lo == v.lo; } + +TEST(WideMultiplyTest, MultiplyU128ToU256Test) { + using absl::uint128; + constexpr uint128 k1 = 1; + constexpr uint128 kMax = ~static_cast<uint128>(0); - EXPECT_EQ(absl::uint128(0), MultiplyU64ToU128(0, 0)); + EXPECT_THAT(MultiplyU128ToU256(0, 0), Eq256(0, 0)); - // Max uint64_t - EXPECT_EQ(MultiplyU64ToU128(kMax, kMax), - absl::MakeUint128(0xfffffffffffffffe, 0x0000000000000001)); - EXPECT_EQ(absl::MakeUint128(0, kMax), MultiplyU64ToU128(kMax, 1)); - EXPECT_EQ(absl::MakeUint128(0, kMax), MultiplyU64ToU128(1, kMax)); + // Max uin128_t + EXPECT_THAT(MultiplyU128ToU256(kMax, kMax), Eq256(kMax << 1, 1)); + EXPECT_THAT(MultiplyU128ToU256(kMax, 1), Eq256(0, kMax)); + EXPECT_THAT(MultiplyU128ToU256(1, kMax), Eq256(0, kMax)); for (int i = 0; i < 64; ++i) { - EXPECT_EQ(absl::MakeUint128(0, kMax) << i, - MultiplyU64ToU128(kMax, k1 << i)); - EXPECT_EQ(absl::MakeUint128(0, kMax) << i, - MultiplyU64ToU128(k1 << i, kMax)); + SCOPED_TRACE(i); + EXPECT_THAT(MultiplyU128ToU256(kMax, k1 << i), + Eq256(LeftShift({0, kMax}, i))); + EXPECT_THAT(MultiplyU128ToU256(k1 << i, kMax), + Eq256(LeftShift({0, kMax}, i))); } // 1-bit x 1-bit. for (int i = 0; i < 64; ++i) { for (int j = 0; j < 64; ++j) { - EXPECT_EQ(absl::MakeUint128(0, 1) << (i + j), - MultiplyU64ToU128(k1 << i, k1 << j)); - EXPECT_EQ(absl::MakeUint128(0, 1) << (i + j), - MultiplyU64ToU128(k1 << i, k1 << j)); + EXPECT_THAT(MultiplyU128ToU256(k1 << i, k1 << j), + Eq256(LeftShift({0, 1}, i + j))); } } // Verified multiplies - EXPECT_EQ(MultiplyU64ToU128(0xffffeeeeddddcccc, 0xbbbbaaaa99998888), - absl::MakeUint128(0xbbbb9e2692c5dddc, 0xc28f7531048d2c60)); - EXPECT_EQ(MultiplyU64ToU128(0x0123456789abcdef, 0xfedcba9876543210), - absl::MakeUint128(0x0121fa00ad77d742, 0x2236d88fe5618cf0)); - EXPECT_EQ(MultiplyU64ToU128(0x0123456789abcdef, 0xfdb97531eca86420), - absl::MakeUint128(0x0120ae99d26725fc, 0xce197f0ecac319e0)); - EXPECT_EQ(MultiplyU64ToU128(0x97a87f4f261ba3f2, 0xfedcba9876543210), - absl::MakeUint128(0x96fbf1a8ae78d0ba, 0x5a6dd4b71f278320)); - EXPECT_EQ(MultiplyU64ToU128(0xfedcba9876543210, 0xfdb97531eca86420), - absl::MakeUint128(0xfc98c6981a413e22, 0x342d0bbf48948200)); + EXPECT_THAT(MultiplyU128ToU256( + absl::MakeUint128(0xc502da0d6ea99fe8, 0xfa3c9141a1f50912), + absl::MakeUint128(0x96bcf1ac37f97bd6, 0x27e2cdeb5fb2299e)), + Eq256(absl::MakeUint128(0x740113d838f96a64, 0x22e8cfa4d71f89ea), + absl::MakeUint128(0x19184a345c62e993, 0x237871b630337b1c))); + EXPECT_THAT(MultiplyU128ToU256( + absl::MakeUint128(0x6f29e670cee07230, 0xc3d8e6c3e4d86759), + absl::MakeUint128(0x3227d29fa6386db1, 0x231682bb1e4b764f)), + Eq256(absl::MakeUint128(0x15c779d9d5d3b07c, 0xd7e6c827f0c81cbe), + absl::MakeUint128(0xf88e3914f7fa287a, 0x15b79975137dea77))); + EXPECT_THAT(MultiplyU128ToU256( + absl::MakeUint128(0xafb77107215646e1, 0x3b844cb1ac5769e7), + absl::MakeUint128(0x1ff7b2d888b62479, 0x92f758ae96fcba0b)), + Eq256(absl::MakeUint128(0x15f13b70181f6985, 0x2adb36bbabce7d02), + absl::MakeUint128(0x6c470d72e13aad04, 0x63fba3f5841762ed))); + EXPECT_THAT(MultiplyU128ToU256( + absl::MakeUint128(0xd85d5558d67ac905, 0xf88c70654dae19b1), + absl::MakeUint128(0x17252c6727db3738, 0x399ff658c511eedc)), + Eq256(absl::MakeUint128(0x138fcdaf8b0421ee, 0x1b465ddf2a0d03f6), + absl::MakeUint128(0x8f573ba68296860f, 0xf327d2738741a21c))); + EXPECT_THAT(MultiplyU128ToU256( + absl::MakeUint128(0x46f0421a37ff6bee, 0xa61df89f09d140b1), + absl::MakeUint128(0x3d712ec9f37ca2e1, 0x9658a2cba47ef4b1)), + Eq256(absl::MakeUint128(0x11069cc48ee7c95d, 0xd35fb1c7aa91c978), + absl::MakeUint128(0xbe2f4a6de874b015, 0xd2f7ac1b76746e61))); + EXPECT_THAT(MultiplyU128ToU256( + absl::MakeUint128(0x730d27c72d58fa49, 0x3ebeda7498f8827c), + absl::MakeUint128(0xa2c959eca9f503af, 0x189c687eb842bbd8)), + Eq256(absl::MakeUint128(0x4928d0ea356ba022, 0x1546d34a2963393), + absl::MakeUint128(0x7481531e1e0a16d1, 0xdd8025015cf6aca0))); + EXPECT_THAT(MultiplyU128ToU256( + absl::MakeUint128(0x6ca41020f856d2f1, 0xb9b0838c04a7f4aa), + absl::MakeUint128(0x9cf41d28a8396f54, 0x1d681695e377ffe6)), + Eq256(absl::MakeUint128(0x429b92934d9be6f1, 0xea182877157c1e7), + absl::MakeUint128(0x7135c23f0a4a475, 0xc1adc366f4a126bc))); + EXPECT_THAT(MultiplyU128ToU256( + absl::MakeUint128(0x57472833797c332, 0x6c79272fdec4687a), + absl::MakeUint128(0xb5f022ea3838e46b, 0x16face2f003e27a6)), + Eq256(absl::MakeUint128(0x3e072e0962b3400, 0x5d9fe8fdc3d0e1f4), + absl::MakeUint128(0x7dc0df47cedafd62, 0xbe6501f1acd2551c))); + EXPECT_THAT(MultiplyU128ToU256( + absl::MakeUint128(0xf0fb4198322eb1c2, 0xfe7f5f31f3885938), + absl::MakeUint128(0xd99012b71bb7aa31, 0xac7a6f9eb190789)), + Eq256(absl::MakeUint128(0xcccc998cf075ca01, 0x642d144322fb873a), + absl::MakeUint128(0xc79dc12b69d91ed4, 0xa83459132ce046f8))); + EXPECT_THAT(MultiplyU128ToU256( + absl::MakeUint128(0xb5c04120848cdb47, 0x8aa62a827bf52635), + absl::MakeUint128(0x8d07a359be2f1380, 0x467bb90d59da0dea)), + Eq256(absl::MakeUint128(0x64205019d139a9ce, 0x99425c5fb6e7a977), + absl::MakeUint128(0xd3e99628a9e5fca7, 0x9c7824cb7279d72))); } } // namespace diff --git a/absl/random/log_uniform_int_distribution.h b/absl/random/log_uniform_int_distribution.h index 43e10116..4afff8f6 100644 --- a/absl/random/log_uniform_int_distribution.h +++ b/absl/random/log_uniform_int_distribution.h @@ -69,10 +69,8 @@ class log_uniform_int_distribution { if (base_ == 2) { // Determine where the first set bit is on range(), giving a log2(range) // value which can be used to construct bounds. - log_range_ = - (std::min)(bit_width(range()), - static_cast<unsigned_type>( - std::numeric_limits<unsigned_type>::digits)); + log_range_ = (std::min)(random_internal::BitWidth(range()), + std::numeric_limits<unsigned_type>::digits); } else { // NOTE: Computing the logN(x) introduces error from 2 sources: // 1. Conversion of int to double loses precision for values >= @@ -83,7 +81,7 @@ class log_uniform_int_distribution { // // Thus a result which should equal K may equal K +/- epsilon, // which can eliminate some values depending on where the bounds fall. - const double inv_log_base = 1.0 / std::log(base_); + const double inv_log_base = 1.0 / std::log(static_cast<double>(base_)); const double log_range = std::log(static_cast<double>(range()) + 0.5); log_range_ = static_cast<int>(std::ceil(inv_log_base * log_range)); } @@ -113,7 +111,7 @@ class log_uniform_int_distribution { unsigned_type range_; // max - min int log_range_; // ceil(logN(range_)) - static_assert(std::is_integral<IntType>::value, + static_assert(random_internal::IsIntegral<IntType>::value, "Class-template absl::log_uniform_int_distribution<> must be " "parameterized using an integral type."); }; @@ -139,7 +137,7 @@ class log_uniform_int_distribution { template <typename URBG> result_type operator()(URBG& g, // NOLINT(runtime/references) const param_type& p) { - return (p.min)() + Generate(g, p); + return static_cast<result_type>((p.min)() + Generate(g, p)); } result_type(min)() const { return (param_.min)(); } @@ -193,8 +191,8 @@ log_uniform_int_distribution<IntType>::Generate( ? (std::numeric_limits<unsigned_type>::max)() : (static_cast<unsigned_type>(1) << e) - 1; } else { - const double r = std::pow(p.base(), d); - const double s = (r * p.base()) - 1.0; + const double r = std::pow(static_cast<double>(p.base()), d); + const double s = (r * static_cast<double>(p.base())) - 1.0; base_e = (r > static_cast<double>((std::numeric_limits<unsigned_type>::max)())) @@ -211,7 +209,8 @@ log_uniform_int_distribution<IntType>::Generate( const unsigned_type hi = (top_e >= p.range()) ? p.range() : top_e; // choose uniformly over [lo, hi] - return absl::uniform_int_distribution<result_type>(lo, hi)(g); + return absl::uniform_int_distribution<result_type>( + static_cast<result_type>(lo), static_cast<result_type>(hi))(g); } template <typename CharT, typename Traits, typename IntType> diff --git a/absl/random/log_uniform_int_distribution_test.cc b/absl/random/log_uniform_int_distribution_test.cc index 5e780d96..0d0fcb95 100644 --- a/absl/random/log_uniform_int_distribution_test.cc +++ b/absl/random/log_uniform_int_distribution_test.cc @@ -42,7 +42,7 @@ class LogUniformIntDistributionTypeTest : public ::testing::Test {}; using IntTypes = ::testing::Types<int8_t, int16_t, int32_t, int64_t, // uint8_t, uint16_t, uint32_t, uint64_t>; -TYPED_TEST_CASE(LogUniformIntDistributionTypeTest, IntTypes); +TYPED_TEST_SUITE(LogUniformIntDistributionTypeTest, IntTypes); TYPED_TEST(LogUniformIntDistributionTypeTest, SerializeTest) { using param_type = diff --git a/absl/random/mocking_bit_gen.h b/absl/random/mocking_bit_gen.h index 7b2b80eb..89fa5a47 100644 --- a/absl/random/mocking_bit_gen.h +++ b/absl/random/mocking_bit_gen.h @@ -87,7 +87,7 @@ class BitGenRef; // // ON_CALL(absl::MockUniform<int>(), Call(bitgen, testing::_, testing::_)) // .WillByDefault([] (int low, int high) { -// return (low + high) / 2; +// return low + (high - low) / 2; // }); // // EXPECT_EQ(absl::Uniform<int>(gen, 0, 10), 5); diff --git a/absl/random/mocking_bit_gen_test.cc b/absl/random/mocking_bit_gen_test.cc index f63b6e42..c713ceaf 100644 --- a/absl/random/mocking_bit_gen_test.cc +++ b/absl/random/mocking_bit_gen_test.cc @@ -15,6 +15,7 @@ // #include "absl/random/mocking_bit_gen.h" +#include <cmath> #include <numeric> #include <random> @@ -328,8 +329,9 @@ TEST(BasicMocking, WillByDefaultWithArgs) { absl::MockingBitGen gen; ON_CALL(absl::MockPoisson<int>(), Call(gen, _)) - .WillByDefault( - [](double lambda) { return static_cast<int>(lambda * 10); }); + .WillByDefault([](double lambda) { + return static_cast<int>(std::rint(lambda * 10)); + }); EXPECT_EQ(absl::Poisson<int>(gen, 1.7), 17); EXPECT_EQ(absl::Poisson<int>(gen, 0.03), 0); } diff --git a/absl/random/poisson_distribution.h b/absl/random/poisson_distribution.h index cb5f5d5d..f4573082 100644 --- a/absl/random/poisson_distribution.h +++ b/absl/random/poisson_distribution.h @@ -26,6 +26,7 @@ #include "absl/random/internal/fastmath.h" #include "absl/random/internal/generate_real.h" #include "absl/random/internal/iostream_state_saver.h" +#include "absl/random/internal/traits.h" namespace absl { ABSL_NAMESPACE_BEGIN @@ -80,7 +81,7 @@ class poisson_distribution { double log_k_; int split_; - static_assert(std::is_integral<IntType>::value, + static_assert(random_internal::IsIntegral<IntType>::value, "Class-template absl::poisson_distribution<> must be " "parameterized using an integral type."); }; @@ -133,7 +134,8 @@ template <typename IntType> poisson_distribution<IntType>::param_type::param_type(double mean) : mean_(mean), split_(0) { assert(mean >= 0); - assert(mean <= (std::numeric_limits<result_type>::max)()); + assert(mean <= + static_cast<double>((std::numeric_limits<result_type>::max)())); // As a defensive measure, avoid large values of the mean. The rejection // algorithm used does not support very large values well. It my be worth // changing algorithms to better deal with these cases. @@ -222,8 +224,9 @@ poisson_distribution<IntType>::operator()( // clang-format on const double lhs = 2.0 * std::log(u) + p.log_k_ + s; if (lhs < rhs) { - return x > (max)() ? (max)() - : static_cast<result_type>(x); // f(x)/k >= u^2 + return x > static_cast<double>((max)()) + ? (max)() + : static_cast<result_type>(x); // f(x)/k >= u^2 } } } diff --git a/absl/random/poisson_distribution_test.cc b/absl/random/poisson_distribution_test.cc index 8baabd11..4f585b9b 100644 --- a/absl/random/poisson_distribution_test.cc +++ b/absl/random/poisson_distribution_test.cc @@ -73,7 +73,7 @@ class PoissonDistributionInterfaceTest : public ::testing::Test {}; using IntTypes = ::testing::Types<int, int8_t, int16_t, int32_t, int64_t, uint8_t, uint16_t, uint32_t, uint64_t>; -TYPED_TEST_CASE(PoissonDistributionInterfaceTest, IntTypes); +TYPED_TEST_SUITE(PoissonDistributionInterfaceTest, IntTypes); TYPED_TEST(PoissonDistributionInterfaceTest, SerializeTest) { using param_type = typename absl::poisson_distribution<TypeParam>::param_type; diff --git a/absl/random/seed_sequences.h b/absl/random/seed_sequences.h index ff1340cc..c3af4b00 100644 --- a/absl/random/seed_sequences.h +++ b/absl/random/seed_sequences.h @@ -28,6 +28,7 @@ #include <iterator> #include <random> +#include "absl/base/config.h" #include "absl/random/internal/salted_seed_seq.h" #include "absl/random/internal/seed_material.h" #include "absl/random/seed_gen_exception.h" diff --git a/absl/random/uniform_int_distribution.h b/absl/random/uniform_int_distribution.h index c1f54cce..fae80252 100644 --- a/absl/random/uniform_int_distribution.h +++ b/absl/random/uniform_int_distribution.h @@ -97,7 +97,7 @@ class uniform_int_distribution { result_type lo_; unsigned_type range_; - static_assert(std::is_integral<result_type>::value, + static_assert(random_internal::IsIntegral<result_type>::value, "Class-template absl::uniform_int_distribution<> must be " "parameterized using an integral type."); }; // param_type @@ -125,7 +125,7 @@ class uniform_int_distribution { template <typename URBG> result_type operator()( URBG& gen, const param_type& param) { // NOLINT(runtime/references) - return param.a() + Generate(gen, param.range()); + return static_cast<result_type>(param.a() + Generate(gen, param.range())); } result_type a() const { return param_.a(); } diff --git a/absl/random/uniform_real_distribution.h b/absl/random/uniform_real_distribution.h index 5ba17b23..19683341 100644 --- a/absl/random/uniform_real_distribution.h +++ b/absl/random/uniform_real_distribution.h @@ -73,12 +73,12 @@ class uniform_real_distribution { : lo_(lo), hi_(hi), range_(hi - lo) { // [rand.dist.uni.real] preconditions 2 & 3 assert(lo <= hi); + // NOTE: For integral types, we can promote the range to an unsigned type, // which gives full width of the range. However for real (fp) types, this // is not possible, so value generation cannot use the full range of the // real type. assert(range_ <= (std::numeric_limits<result_type>::max)()); - assert(std::isfinite(range_)); } result_type a() const { return lo_; } diff --git a/absl/random/uniform_real_distribution_test.cc b/absl/random/uniform_real_distribution_test.cc index 18bcd3bc..07f199d3 100644 --- a/absl/random/uniform_real_distribution_test.cc +++ b/absl/random/uniform_real_distribution_test.cc @@ -14,6 +14,7 @@ #include "absl/random/uniform_real_distribution.h" +#include <cfloat> #include <cmath> #include <cstdint> #include <iterator> @@ -70,61 +71,81 @@ using RealTypes = TYPED_TEST_SUITE(UniformRealDistributionTest, RealTypes); TYPED_TEST(UniformRealDistributionTest, ParamSerializeTest) { - using param_type = - typename absl::uniform_real_distribution<TypeParam>::param_type; +#if (defined(__i386__) || defined(_M_IX86)) && FLT_EVAL_METHOD != 0 + // We're using an x87-compatible FPU, and intermediate operations are + // performed with 80-bit floats. This produces slightly different results from + // what we expect below. + GTEST_SKIP() + << "Skipping the test because we detected x87 floating-point semantics"; +#endif + using DistributionType = absl::uniform_real_distribution<TypeParam>; + using real_type = TypeParam; + using param_type = typename DistributionType::param_type; + + constexpr const real_type kMax = std::numeric_limits<real_type>::max(); + constexpr const real_type kMin = std::numeric_limits<real_type>::min(); + constexpr const real_type kEpsilon = + std::numeric_limits<real_type>::epsilon(); + constexpr const real_type kLowest = + std::numeric_limits<real_type>::lowest(); // -max + + const real_type kDenormMax = std::nextafter(kMin, real_type{0}); + const real_type kOneMinusE = + std::nextafter(real_type{1}, real_type{0}); // 1 - epsilon - constexpr const TypeParam a{1152921504606846976}; + constexpr const real_type kTwo60{1152921504606846976}; // 2^60 constexpr int kCount = 1000; absl::InsecureBitGen gen; for (const auto& param : { param_type(), - param_type(TypeParam(2.0), TypeParam(2.0)), // Same - param_type(TypeParam(-0.1), TypeParam(0.1)), - param_type(TypeParam(0.05), TypeParam(0.12)), - param_type(TypeParam(-0.05), TypeParam(0.13)), - param_type(TypeParam(-0.05), TypeParam(-0.02)), + param_type(real_type{0}, real_type{1}), + param_type(real_type(-0.1), real_type(0.1)), + param_type(real_type(0.05), real_type(0.12)), + param_type(real_type(-0.05), real_type(0.13)), + param_type(real_type(-0.05), real_type(-0.02)), + // range = 0 + param_type(real_type(2.0), real_type(2.0)), // Same // double range = 0 // 2^60 , 2^60 + 2^6 - param_type(a, TypeParam(1152921504606847040)), + param_type(kTwo60, real_type(1152921504606847040)), // 2^60 , 2^60 + 2^7 - param_type(a, TypeParam(1152921504606847104)), + param_type(kTwo60, real_type(1152921504606847104)), // double range = 2^8 // 2^60 , 2^60 + 2^8 - param_type(a, TypeParam(1152921504606847232)), + param_type(kTwo60, real_type(1152921504606847232)), // float range = 0 // 2^60 , 2^60 + 2^36 - param_type(a, TypeParam(1152921573326323712)), + param_type(kTwo60, real_type(1152921573326323712)), // 2^60 , 2^60 + 2^37 - param_type(a, TypeParam(1152921642045800448)), + param_type(kTwo60, real_type(1152921642045800448)), // float range = 2^38 // 2^60 , 2^60 + 2^38 - param_type(a, TypeParam(1152921779484753920)), + param_type(kTwo60, real_type(1152921779484753920)), // Limits - param_type(0, std::numeric_limits<TypeParam>::max()), - param_type(std::numeric_limits<TypeParam>::lowest(), 0), - param_type(0, std::numeric_limits<TypeParam>::epsilon()), - param_type(-std::numeric_limits<TypeParam>::epsilon(), - std::numeric_limits<TypeParam>::epsilon()), - param_type(std::numeric_limits<TypeParam>::epsilon(), - 2 * std::numeric_limits<TypeParam>::epsilon()), + param_type(0, kMax), + param_type(kLowest, 0), + param_type(0, kMin), + param_type(0, kEpsilon), + param_type(-kEpsilon, kEpsilon), + param_type(0, kOneMinusE), + param_type(0, kDenormMax), }) { // Validate parameters. const auto a = param.a(); const auto b = param.b(); - absl::uniform_real_distribution<TypeParam> before(a, b); + DistributionType before(a, b); EXPECT_EQ(before.a(), param.a()); EXPECT_EQ(before.b(), param.b()); { - absl::uniform_real_distribution<TypeParam> via_param(param); + DistributionType via_param(param); EXPECT_EQ(via_param, before); } std::stringstream ss; ss << before; - absl::uniform_real_distribution<TypeParam> after(TypeParam(1.0), - TypeParam(3.1)); + DistributionType after(real_type(1.0), real_type(3.1)); EXPECT_NE(before.a(), after.a()); EXPECT_NE(before.b(), after.b()); @@ -159,7 +180,7 @@ TYPED_TEST(UniformRealDistributionTest, ParamSerializeTest) { } } - if (!std::is_same<TypeParam, long double>::value) { + if (!std::is_same<real_type, long double>::value) { // static_cast<double>(long double) can overflow. std::string msg = absl::StrCat("Range: ", static_cast<double>(sample_min), ", ", static_cast<double>(sample_max)); @@ -173,33 +194,52 @@ TYPED_TEST(UniformRealDistributionTest, ParamSerializeTest) { #pragma warning(disable:4756) // Constant arithmetic overflow. #endif TYPED_TEST(UniformRealDistributionTest, ViolatesPreconditionsDeathTest) { + using DistributionType = absl::uniform_real_distribution<TypeParam>; + using real_type = TypeParam; + #if GTEST_HAS_DEATH_TEST // Hi < Lo - EXPECT_DEBUG_DEATH( - { absl::uniform_real_distribution<TypeParam> dist(10.0, 1.0); }, ""); + EXPECT_DEBUG_DEATH({ DistributionType dist(10.0, 1.0); }, ""); // Hi - Lo > numeric_limits<>::max() EXPECT_DEBUG_DEATH( { - absl::uniform_real_distribution<TypeParam> dist( - std::numeric_limits<TypeParam>::lowest(), - std::numeric_limits<TypeParam>::max()); + DistributionType dist(std::numeric_limits<real_type>::lowest(), + std::numeric_limits<real_type>::max()); + }, + ""); + + // kEpsilon guarantees that max + kEpsilon = inf. + const auto kEpsilon = std::nexttoward( + (std::numeric_limits<real_type>::max() - + std::nexttoward(std::numeric_limits<real_type>::max(), 0.0)) / + 2, + std::numeric_limits<real_type>::max()); + EXPECT_DEBUG_DEATH( + { + DistributionType dist(-kEpsilon, std::numeric_limits<real_type>::max()); + }, + ""); + EXPECT_DEBUG_DEATH( + { + DistributionType dist(std::numeric_limits<real_type>::lowest(), + kEpsilon); }, ""); + #endif // GTEST_HAS_DEATH_TEST #if defined(NDEBUG) // opt-mode, for invalid parameters, will generate a garbage value, // but should not enter an infinite loop. absl::InsecureBitGen gen; { - absl::uniform_real_distribution<TypeParam> dist(10.0, 1.0); + DistributionType dist(10.0, 1.0); auto x = dist(gen); EXPECT_FALSE(std::isnan(x)) << x; } { - absl::uniform_real_distribution<TypeParam> dist( - std::numeric_limits<TypeParam>::lowest(), - std::numeric_limits<TypeParam>::max()); + DistributionType dist(std::numeric_limits<real_type>::lowest(), + std::numeric_limits<real_type>::max()); auto x = dist(gen); // Infinite result. EXPECT_FALSE(std::isfinite(x)) << x; @@ -211,6 +251,8 @@ TYPED_TEST(UniformRealDistributionTest, ViolatesPreconditionsDeathTest) { #endif TYPED_TEST(UniformRealDistributionTest, TestMoments) { + using DistributionType = absl::uniform_real_distribution<TypeParam>; + constexpr int kSize = 1000000; std::vector<double> values(kSize); @@ -219,7 +261,7 @@ TYPED_TEST(UniformRealDistributionTest, TestMoments) { // implementation. absl::random_internal::pcg64_2018_engine rng{0x2B7E151628AED2A6}; - absl::uniform_real_distribution<TypeParam> dist; + DistributionType dist; for (int i = 0; i < kSize; i++) { values[i] = dist(rng); } @@ -233,9 +275,10 @@ TYPED_TEST(UniformRealDistributionTest, TestMoments) { } TYPED_TEST(UniformRealDistributionTest, ChiSquaredTest50) { + using DistributionType = absl::uniform_real_distribution<TypeParam>; + using param_type = typename DistributionType::param_type; + using absl::random_internal::kChiSquared; - using param_type = - typename absl::uniform_real_distribution<TypeParam>::param_type; constexpr size_t kTrials = 100000; constexpr int kBuckets = 50; @@ -260,7 +303,7 @@ TYPED_TEST(UniformRealDistributionTest, ChiSquaredTest50) { const double factor = kBuckets / (max_val - min_val); std::vector<int32_t> counts(kBuckets, 0); - absl::uniform_real_distribution<TypeParam> dist(param); + DistributionType dist(param); for (size_t i = 0; i < kTrials; i++) { auto x = dist(rng); auto bucket = static_cast<size_t>((x - min_val) * factor); @@ -288,8 +331,11 @@ TYPED_TEST(UniformRealDistributionTest, ChiSquaredTest50) { } TYPED_TEST(UniformRealDistributionTest, StabilityTest) { + using DistributionType = absl::uniform_real_distribution<TypeParam>; + using real_type = TypeParam; + // absl::uniform_real_distribution stability relies only on - // random_internal::RandU64ToDouble and random_internal::RandU64ToFloat. + // random_internal::GenerateRealFromBits. absl::random_internal::sequence_urbg urbg( {0x0003eb76f6f7f755ull, 0xFFCEA50FDB2F953Bull, 0xC332DDEFBE6C5AA5ull, 0x6558218568AB9702ull, 0x2AEF7DAD5B6E2F84ull, 0x1521B62829076170ull, @@ -298,9 +344,9 @@ TYPED_TEST(UniformRealDistributionTest, StabilityTest) { std::vector<int> output(12); - absl::uniform_real_distribution<TypeParam> dist; + DistributionType dist; std::generate(std::begin(output), std::end(output), [&] { - return static_cast<int>(TypeParam(1000000) * dist(urbg)); + return static_cast<int>(real_type(1000000) * dist(urbg)); }); EXPECT_THAT( diff --git a/absl/random/zipf_distribution.h b/absl/random/zipf_distribution.h index 22ebc756..03497b1b 100644 --- a/absl/random/zipf_distribution.h +++ b/absl/random/zipf_distribution.h @@ -23,13 +23,14 @@ #include <type_traits> #include "absl/random/internal/iostream_state_saver.h" +#include "absl/random/internal/traits.h" #include "absl/random/uniform_real_distribution.h" namespace absl { ABSL_NAMESPACE_BEGIN // absl::zipf_distribution produces random integer-values in the range [0, k], -// distributed according to the discrete probability function: +// distributed according to the unnormalized discrete probability function: // // P(x) = (v + x) ^ -q // @@ -94,7 +95,7 @@ class zipf_distribution { double hxm_; // h(k + 0.5) double hx0_minus_hxm_; // h(x0) - h(k + 0.5) - static_assert(std::is_integral<IntType>::value, + static_assert(random_internal::IsIntegral<IntType>::value, "Class-template absl::zipf_distribution<> must be " "parameterized using an integral type."); }; @@ -221,7 +222,7 @@ zipf_distribution<IntType>::operator()( const double u = p.hxm_ + v * p.hx0_minus_hxm_; const double x = p.hinv(u); k = rint(x); // std::floor(x + 0.5); - if (k > p.k()) continue; // reject k > max_k + if (k > static_cast<double>(p.k())) continue; // reject k > max_k if (k - x <= p.s_) break; const double h = p.h(k + 0.5); const double r = p.pow_negative_q(p.v_ + k); diff --git a/absl/random/zipf_distribution_test.cc b/absl/random/zipf_distribution_test.cc index f8cf70e0..c8bb89db 100644 --- a/absl/random/zipf_distribution_test.cc +++ b/absl/random/zipf_distribution_test.cc @@ -44,7 +44,7 @@ class ZipfDistributionTypedTest : public ::testing::Test {}; using IntTypes = ::testing::Types<int, int8_t, int16_t, int32_t, int64_t, uint8_t, uint16_t, uint32_t, uint64_t>; -TYPED_TEST_CASE(ZipfDistributionTypedTest, IntTypes); +TYPED_TEST_SUITE(ZipfDistributionTypedTest, IntTypes); TYPED_TEST(ZipfDistributionTypedTest, SerializeTest) { using param_type = typename absl::zipf_distribution<TypeParam>::param_type; diff --git a/absl/status/BUILD.bazel b/absl/status/BUILD.bazel index 189bd73d..ce0ea70c 100644 --- a/absl/status/BUILD.bazel +++ b/absl/status/BUILD.bazel @@ -17,10 +17,10 @@ # It will expand later to have utilities around `Status` like `StatusOr`, # `StatusBuilder` and macros. -load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") load( "//absl:copts/configure_copts.bzl", "ABSL_DEFAULT_COPTS", + "ABSL_DEFAULT_LINKOPTS", "ABSL_TEST_COPTS", ) @@ -40,14 +40,16 @@ cc_library( "status_payload_printer.h", ], copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, deps = [ "//absl/base:atomic_hook", - "//absl/base:config", "//absl/base:core_headers", "//absl/base:raw_logging_internal", + "//absl/base:strerror", "//absl/container:inlined_vector", "//absl/debugging:stacktrace", "//absl/debugging:symbolize", + "//absl/functional:function_ref", "//absl/strings", "//absl/strings:cord", "//absl/strings:str_format", @@ -76,8 +78,10 @@ cc_library( "statusor.h", ], copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, deps = [ ":status", + "//absl/base", "//absl/base:core_headers", "//absl/base:raw_logging_internal", "//absl/meta:type_traits", @@ -96,6 +100,7 @@ cc_test( ":statusor", "//absl/base", "//absl/memory", + "//absl/strings", "//absl/types:any", "//absl/utility", "@com_google_googletest//:gtest_main", diff --git a/absl/status/CMakeLists.txt b/absl/status/CMakeLists.txt index f0d798a3..15db36af 100644 --- a/absl/status/CMakeLists.txt +++ b/absl/status/CMakeLists.txt @@ -28,15 +28,17 @@ absl_cc_library( DEPS absl::atomic_hook absl::config + absl::cord absl::core_headers - absl::raw_logging_internal + absl::function_ref absl::inlined_vector + absl::optional + absl::raw_logging_internal absl::stacktrace - absl::symbolize - absl::strings - absl::cord absl::str_format - absl::optional + absl::strerror + absl::strings + absl::symbolize PUBLIC ) @@ -50,7 +52,7 @@ absl_cc_test( DEPS absl::status absl::strings - gmock_main + GTest::gmock_main ) absl_cc_library( @@ -64,6 +66,7 @@ absl_cc_library( COPTS ${ABSL_DEFAULT_COPTS} DEPS + absl::base absl::status absl::core_headers absl::raw_logging_internal @@ -84,5 +87,5 @@ absl_cc_test( DEPS absl::status absl::statusor - gmock_main + GTest::gmock_main ) diff --git a/absl/status/internal/status_internal.h b/absl/status/internal/status_internal.h index 99a2d964..19a4a7aa 100644 --- a/absl/status/internal/status_internal.h +++ b/absl/status/internal/status_internal.h @@ -15,7 +15,9 @@ #define ABSL_STATUS_INTERNAL_STATUS_INTERNAL_H_ #include <string> +#include <utility> +#include "absl/base/attributes.h" #include "absl/container/inlined_vector.h" #include "absl/strings/cord.h" @@ -25,7 +27,14 @@ namespace absl { ABSL_NAMESPACE_BEGIN // Returned Status objects may not be ignored. Codesearch doesn't handle ifdefs // as part of a class definitions (b/6995610), so we use a forward declaration. +// +// TODO(b/176172494): ABSL_MUST_USE_RESULT should expand to the more strict +// [[nodiscard]]. For now, just use [[nodiscard]] directly when it is available. +#if ABSL_HAVE_CPP_ATTRIBUTE(nodiscard) +class [[nodiscard]] Status; +#else class ABSL_MUST_USE_RESULT Status; +#endif ABSL_NAMESPACE_END } // namespace absl #endif // !SWIG @@ -47,12 +56,12 @@ using Payloads = absl::InlinedVector<Payload, 1>; // Reference-counted representation of Status data. struct StatusRep { - StatusRep(absl::StatusCode code, std::string message, - std::unique_ptr<status_internal::Payloads> payloads) + StatusRep(absl::StatusCode code_arg, absl::string_view message_arg, + std::unique_ptr<status_internal::Payloads> payloads_arg) : ref(int32_t{1}), - code(code), - message(std::move(message)), - payloads(std::move(payloads)) {} + code(code_arg), + message(message_arg), + payloads(std::move(payloads_arg)) {} std::atomic<int32_t> ref; absl::StatusCode code; @@ -61,6 +70,14 @@ struct StatusRep { }; absl::StatusCode MapToLocalCode(int value); + +// Returns a pointer to a newly-allocated string with the given `prefix`, +// suitable for output as an error message in assertion/`CHECK()` failures. +// +// This is an internal implementation detail for Abseil logging. +std::string* MakeCheckFailString(const absl::Status* status, + const char* prefix); + } // namespace status_internal ABSL_NAMESPACE_END diff --git a/absl/status/status.cc b/absl/status/status.cc index 51a0d268..88e8eda9 100644 --- a/absl/status/status.cc +++ b/absl/status/status.cc @@ -13,9 +13,12 @@ // limitations under the License. #include "absl/status/status.h" +#include <errno.h> + #include <cassert> #include "absl/base/internal/raw_logging.h" +#include "absl/base/internal/strerror.h" #include "absl/debugging/stacktrace.h" #include "absl/debugging/symbolize.h" #include "absl/status/status_payload_printer.h" @@ -161,7 +164,7 @@ bool Status::ErasePayload(absl::string_view type_url) { } void Status::ForEachPayload( - const std::function<void(absl::string_view, const absl::Cord&)>& visitor) + absl::FunctionRef<void(absl::string_view, const absl::Cord&)> visitor) const { if (auto* payloads = GetPayloads()) { bool in_reverse = @@ -185,11 +188,16 @@ void Status::ForEachPayload( } const std::string* Status::EmptyString() { - static std::string* empty_string = new std::string(); - return empty_string; + static union EmptyString { + std::string str; + ~EmptyString() {} + } empty = {{}}; + return &empty.str; } +#ifdef ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL constexpr const char Status::kMovedFromString[]; +#endif const std::string* Status::MovedFromString() { static std::string* moved_from_string = new std::string(kMovedFromString); @@ -207,19 +215,10 @@ void Status::UnrefNonInlined(uintptr_t rep) { } } -uintptr_t Status::NewRep( - absl::StatusCode code, absl::string_view msg, - std::unique_ptr<status_internal::Payloads> payloads) { - status_internal::StatusRep* rep = new status_internal::StatusRep( - code, std::string(msg.data(), msg.size()), - std::move(payloads)); - return PointerToRep(rep); -} - Status::Status(absl::StatusCode code, absl::string_view msg) : rep_(CodeToInlinedRep(code)) { if (code != absl::StatusCode::kOk && !msg.empty()) { - rep_ = NewRep(code, msg, nullptr); + rep_ = PointerToRep(new status_internal::StatusRep(code, msg, nullptr)); } } @@ -238,9 +237,9 @@ absl::StatusCode Status::code() const { void Status::PrepareToModify() { ABSL_RAW_CHECK(!ok(), "PrepareToModify shouldn't be called on OK status."); if (IsInlined(rep_)) { - rep_ = - NewRep(static_cast<absl::StatusCode>(raw_code()), absl::string_view(), - nullptr); + rep_ = PointerToRep(new status_internal::StatusRep( + static_cast<absl::StatusCode>(raw_code()), absl::string_view(), + nullptr)); return; } @@ -251,8 +250,9 @@ void Status::PrepareToModify() { if (rep->payloads) { payloads = absl::make_unique<status_internal::Payloads>(*rep->payloads); } - rep_ = NewRep(rep->code, message(), - std::move(payloads)); + status_internal::StatusRep* const new_rep = new status_internal::StatusRep( + rep->code, message(), std::move(payloads)); + rep_ = PointerToRep(new_rep); UnrefNonInlined(rep_i); } } @@ -316,7 +316,7 @@ std::string Status::ToStringSlow(StatusToStringMode mode) const { } std::ostream& operator<<(std::ostream& os, const Status& x) { - os << x.ToString(); + os << x.ToString(StatusToStringMode::kWithEverything); return os; } @@ -448,5 +448,169 @@ bool IsUnknown(const Status& status) { return status.code() == absl::StatusCode::kUnknown; } +StatusCode ErrnoToStatusCode(int error_number) { + switch (error_number) { + case 0: + return StatusCode::kOk; + case EINVAL: // Invalid argument + case ENAMETOOLONG: // Filename too long + case E2BIG: // Argument list too long + case EDESTADDRREQ: // Destination address required + case EDOM: // Mathematics argument out of domain of function + case EFAULT: // Bad address + case EILSEQ: // Illegal byte sequence + case ENOPROTOOPT: // Protocol not available + case ENOSTR: // Not a STREAM + case ENOTSOCK: // Not a socket + case ENOTTY: // Inappropriate I/O control operation + case EPROTOTYPE: // Protocol wrong type for socket + case ESPIPE: // Invalid seek + return StatusCode::kInvalidArgument; + case ETIMEDOUT: // Connection timed out + case ETIME: // Timer expired + return StatusCode::kDeadlineExceeded; + case ENODEV: // No such device + case ENOENT: // No such file or directory +#ifdef ENOMEDIUM + case ENOMEDIUM: // No medium found +#endif + case ENXIO: // No such device or address + case ESRCH: // No such process + return StatusCode::kNotFound; + case EEXIST: // File exists + case EADDRNOTAVAIL: // Address not available + case EALREADY: // Connection already in progress +#ifdef ENOTUNIQ + case ENOTUNIQ: // Name not unique on network +#endif + return StatusCode::kAlreadyExists; + case EPERM: // Operation not permitted + case EACCES: // Permission denied +#ifdef ENOKEY + case ENOKEY: // Required key not available +#endif + case EROFS: // Read only file system + return StatusCode::kPermissionDenied; + case ENOTEMPTY: // Directory not empty + case EISDIR: // Is a directory + case ENOTDIR: // Not a directory + case EADDRINUSE: // Address already in use + case EBADF: // Invalid file descriptor +#ifdef EBADFD + case EBADFD: // File descriptor in bad state +#endif + case EBUSY: // Device or resource busy + case ECHILD: // No child processes + case EISCONN: // Socket is connected +#ifdef EISNAM + case EISNAM: // Is a named type file +#endif +#ifdef ENOTBLK + case ENOTBLK: // Block device required +#endif + case ENOTCONN: // The socket is not connected + case EPIPE: // Broken pipe +#ifdef ESHUTDOWN + case ESHUTDOWN: // Cannot send after transport endpoint shutdown +#endif + case ETXTBSY: // Text file busy +#ifdef EUNATCH + case EUNATCH: // Protocol driver not attached +#endif + return StatusCode::kFailedPrecondition; + case ENOSPC: // No space left on device +#ifdef EDQUOT + case EDQUOT: // Disk quota exceeded +#endif + case EMFILE: // Too many open files + case EMLINK: // Too many links + case ENFILE: // Too many open files in system + case ENOBUFS: // No buffer space available + case ENODATA: // No message is available on the STREAM read queue + case ENOMEM: // Not enough space + case ENOSR: // No STREAM resources +#ifdef EUSERS + case EUSERS: // Too many users +#endif + return StatusCode::kResourceExhausted; +#ifdef ECHRNG + case ECHRNG: // Channel number out of range +#endif + case EFBIG: // File too large + case EOVERFLOW: // Value too large to be stored in data type + case ERANGE: // Result too large + return StatusCode::kOutOfRange; +#ifdef ENOPKG + case ENOPKG: // Package not installed +#endif + case ENOSYS: // Function not implemented + case ENOTSUP: // Operation not supported + case EAFNOSUPPORT: // Address family not supported +#ifdef EPFNOSUPPORT + case EPFNOSUPPORT: // Protocol family not supported +#endif + case EPROTONOSUPPORT: // Protocol not supported +#ifdef ESOCKTNOSUPPORT + case ESOCKTNOSUPPORT: // Socket type not supported +#endif + case EXDEV: // Improper link + return StatusCode::kUnimplemented; + case EAGAIN: // Resource temporarily unavailable +#ifdef ECOMM + case ECOMM: // Communication error on send +#endif + case ECONNREFUSED: // Connection refused + case ECONNABORTED: // Connection aborted + case ECONNRESET: // Connection reset + case EINTR: // Interrupted function call +#ifdef EHOSTDOWN + case EHOSTDOWN: // Host is down +#endif + case EHOSTUNREACH: // Host is unreachable + case ENETDOWN: // Network is down + case ENETRESET: // Connection aborted by network + case ENETUNREACH: // Network unreachable + case ENOLCK: // No locks available + case ENOLINK: // Link has been severed +#ifdef ENONET + case ENONET: // Machine is not on the network +#endif + return StatusCode::kUnavailable; + case EDEADLK: // Resource deadlock avoided +#ifdef ESTALE + case ESTALE: // Stale file handle +#endif + return StatusCode::kAborted; + case ECANCELED: // Operation cancelled + return StatusCode::kCancelled; + default: + return StatusCode::kUnknown; + } +} + +namespace { +std::string MessageForErrnoToStatus(int error_number, + absl::string_view message) { + return absl::StrCat(message, ": ", + absl::base_internal::StrError(error_number)); +} +} // namespace + +Status ErrnoToStatus(int error_number, absl::string_view message) { + return Status(ErrnoToStatusCode(error_number), + MessageForErrnoToStatus(error_number, message)); +} + +namespace status_internal { + +std::string* MakeCheckFailString(const absl::Status* status, + const char* prefix) { + return new std::string( + absl::StrCat(prefix, " (", + status->ToString(StatusToStringMode::kWithEverything), ")")); +} + +} // namespace status_internal + ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/status/status.h b/absl/status/status.h index 61486fee..9292b83a 100644 --- a/absl/status/status.h +++ b/absl/status/status.h @@ -24,11 +24,11 @@ // * A set of helper functions for creating status codes and checking their // values // -// Within Google, `absl::Status` is the primary mechanism for gracefully -// handling errors across API boundaries (and in particular across RPC -// boundaries). Some of these errors may be recoverable, but others may not. -// Most functions that can produce a recoverable error should be designed to -// return an `absl::Status` (or `absl::StatusOr`). +// Within Google, `absl::Status` is the primary mechanism for communicating +// errors in C++, and is used to represent error state in both in-process +// library calls as well as RPC calls. Some of these errors may be recoverable, +// but others may not. Most functions that can produce a recoverable error +// should be designed to return an `absl::Status` (or `absl::StatusOr`). // // Example: // @@ -55,6 +55,7 @@ #include <string> #include "absl/container/inlined_vector.h" +#include "absl/functional/function_ref.h" #include "absl/status/internal/status_internal.h" #include "absl/strings/cord.h" #include "absl/strings/string_view.h" @@ -80,7 +81,7 @@ ABSL_NAMESPACE_BEGIN // `kFailedPrecondition` if both codes apply. Similarly prefer `kNotFound` or // `kAlreadyExists` over `kFailedPrecondition`. // -// Because these errors may travel RPC boundaries, these codes are tied to the +// Because these errors may cross RPC boundaries, these codes are tied to the // `google.rpc.Code` definitions within // https://github.com/googleapis/googleapis/blob/master/google/rpc/code.proto // The string value of these RPC codes is denoted within each enum below. @@ -114,10 +115,10 @@ enum class StatusCode : int { // StatusCode::kInvalidArgument // // kInvalidArgument (gRPC code "INVALID_ARGUMENT") indicates the caller - // specified an invalid argument, such a malformed filename. Note that such - // errors should be narrowly limited to indicate to the invalid nature of the - // arguments themselves. Errors with validly formed arguments that may cause - // errors with the state of the receiving system should be denoted with + // specified an invalid argument, such as a malformed filename. Note that use + // of such errors should be narrowly limited to indicate the invalid nature of + // the arguments themselves. Errors with validly formed arguments that may + // cause errors with the state of the receiving system should be denoted with // `kFailedPrecondition` instead. kInvalidArgument = 3, @@ -137,14 +138,15 @@ enum class StatusCode : int { // // `kNotFound` is useful if a request should be denied for an entire class of // users, such as during a gradual feature rollout or undocumented allow list. - // If, instead, a request should be denied for specific sets of users, such as - // through user-based access control, use `kPermissionDenied` instead. + // If a request should be denied for specific sets of users, such as through + // user-based access control, use `kPermissionDenied` instead. kNotFound = 5, // StatusCode::kAlreadyExists // - // kAlreadyExists (gRPC code "ALREADY_EXISTS") indicates the entity that a - // caller attempted to create (such as file or directory) is already present. + // kAlreadyExists (gRPC code "ALREADY_EXISTS") indicates that the entity a + // caller attempted to create (such as a file or directory) is already + // present. kAlreadyExists = 6, // StatusCode::kPermissionDenied @@ -183,7 +185,7 @@ enum class StatusCode : int { // level (such as when a client-specified test-and-set fails, indicating // the client should restart a read-modify-write sequence). // (c) Use `kFailedPrecondition` if the client should not retry until - // the system state has been explicitly fixed. For example, if an "rmdir" + // the system state has been explicitly fixed. For example, if a "rmdir" // fails because the directory is non-empty, `kFailedPrecondition` // should be returned since the client should not retry unless // the files are deleted from the directory. @@ -283,7 +285,7 @@ std::ostream& operator<<(std::ostream& os, StatusCode code); // absl::StatusToStringMode // // An `absl::StatusToStringMode` is an enumerated type indicating how -// `absl::Status::ToString()` should construct the output string for an non-ok +// `absl::Status::ToString()` should construct the output string for a non-ok // status. enum class StatusToStringMode : int { // ToString will not contain any extra data (such as payloads). It will only @@ -293,6 +295,8 @@ enum class StatusToStringMode : int { kWithPayload = 1 << 0, // ToString will include all the extra data this Status has. kWithEverything = ~kWithNoExtraData, + // Default mode used by ToString. Its exact value might change in the future. + kDefault = kWithPayload, }; // absl::StatusToStringMode is specified as a bitmask type, which means the @@ -343,7 +347,7 @@ inline StatusToStringMode& operator^=(StatusToStringMode& lhs, // API developers should construct their functions to return `absl::OkStatus()` // upon success, or an `absl::StatusCode` upon another type of error (e.g // an `absl::StatusCode::kInvalidArgument` error). The API provides convenience -// functions to constuct each status code. +// functions to construct each status code. // // Example: // @@ -465,8 +469,9 @@ class Status final { // Status::ok() // - // Returns `true` if `this->ok()`. Prefer checking for an OK status using this - // member function. + // Returns `true` if `this->code()` == `absl::StatusCode::kOk`, + // indicating the absence of an error. + // Prefer checking for an OK status using this member function. ABSL_MUST_USE_RESULT bool ok() const; // Status::code() @@ -491,7 +496,7 @@ class Status final { // Returns the error message associated with this error code, if available. // Note that this message rarely describes the error code. It is not unusual // for the error message to be the empty string. As a result, prefer - // `Status::ToString()` for debug logging. + // `operator<<` or `Status::ToString()` for debug logging. absl::string_view message() const; friend bool operator==(const Status&, const Status&); @@ -509,7 +514,7 @@ class Status final { // result, and the payloads to be printed use the status payload printer // mechanism (which is internal). std::string ToString( - StatusToStringMode mode = StatusToStringMode::kWithPayload) const; + StatusToStringMode mode = StatusToStringMode::kDefault) const; // Status::IgnoreError() // @@ -528,7 +533,7 @@ class Status final { //---------------------------------------------------------------------------- // A payload may be attached to a status to provide additional context to an - // error that may not be satisifed by an existing `absl::StatusCode`. + // error that may not be satisfied by an existing `absl::StatusCode`. // Typically, this payload serves one of several purposes: // // * It may provide more fine-grained semantic information about the error @@ -587,7 +592,7 @@ class Status final { // NOTE: Any mutation on the same 'absl::Status' object during visitation is // forbidden and could result in undefined behavior. void ForEachPayload( - const std::function<void(absl::string_view, const absl::Cord&)>& visitor) + absl::FunctionRef<void(absl::string_view, const absl::Cord&)> visitor) const; private: @@ -608,10 +613,6 @@ class Status final { const status_internal::Payloads* GetPayloads() const; status_internal::Payloads* GetPayloads(); - // Takes ownership of payload. - static uintptr_t NewRep( - absl::StatusCode code, absl::string_view msg, - std::unique_ptr<status_internal::Payloads> payload); static bool EqualsSlow(const absl::Status& a, const absl::Status& b); // MSVC 14.0 limitation requires the const. @@ -737,6 +738,19 @@ Status UnavailableError(absl::string_view message); Status UnimplementedError(absl::string_view message); Status UnknownError(absl::string_view message); +// ErrnoToStatusCode() +// +// Returns the StatusCode for `error_number`, which should be an `errno` value. +// See https://en.cppreference.com/w/cpp/error/errno_macros and similar +// references. +absl::StatusCode ErrnoToStatusCode(int error_number); + +// ErrnoToStatus() +// +// Convenience function that creates a `absl::Status` using an `error_number`, +// which should be an `errno` value. +Status ErrnoToStatus(int error_number, absl::string_view message); + //------------------------------------------------------------------------------ // Implementation details follow //------------------------------------------------------------------------------ diff --git a/absl/status/status_test.cc b/absl/status/status_test.cc index 0e1a43ce..89cce7df 100644 --- a/absl/status/status_test.cc +++ b/absl/status/status_test.cc @@ -14,6 +14,8 @@ #include "absl/status/status.h" +#include <errno.h> + #include "gmock/gmock.h" #include "gtest/gtest.h" #include "absl/strings/str_cat.h" @@ -36,7 +38,9 @@ TEST(StatusCode, InsertionOperator) { // its creator, and its classifier. struct ErrorTest { absl::StatusCode code; - using Creator = absl::Status (*)(absl::string_view); + using Creator = absl::Status (*)( + absl::string_view + ); using Classifier = bool (*)(const absl::Status&); Creator creator; Classifier classifier; @@ -78,7 +82,9 @@ TEST(Status, CreateAndClassify) { // expected error code and message. std::string message = absl::StrCat("error code ", test.code, " test message"); - absl::Status status = test.creator(message); + absl::Status status = test.creator( + message + ); EXPECT_EQ(test.code, status.code()); EXPECT_EQ(message, status.message()); @@ -481,4 +487,22 @@ TEST(Status, Swap) { test_swap(no_payload, with_payload); test_swap(with_payload, no_payload); } + +TEST(StatusErrno, ErrnoToStatusCode) { + EXPECT_EQ(absl::ErrnoToStatusCode(0), absl::StatusCode::kOk); + + // Spot-check a few errno values. + EXPECT_EQ(absl::ErrnoToStatusCode(EINVAL), + absl::StatusCode::kInvalidArgument); + EXPECT_EQ(absl::ErrnoToStatusCode(ENOENT), absl::StatusCode::kNotFound); + + // We'll pick a very large number so it hopefully doesn't collide to errno. + EXPECT_EQ(absl::ErrnoToStatusCode(19980927), absl::StatusCode::kUnknown); +} + +TEST(StatusErrno, ErrnoToStatus) { + absl::Status status = absl::ErrnoToStatus(ENOENT, "Cannot open 'path'"); + EXPECT_EQ(status.code(), absl::StatusCode::kNotFound); + EXPECT_EQ(status.message(), "Cannot open 'path': No such file or directory"); +} } // namespace diff --git a/absl/status/statusor.cc b/absl/status/statusor.cc index b954b45e..96642b34 100644 --- a/absl/status/statusor.cc +++ b/absl/status/statusor.cc @@ -16,6 +16,7 @@ #include <cstdlib> #include <utility> +#include "absl/base/call_once.h" #include "absl/base/internal/raw_logging.h" #include "absl/status/status.h" #include "absl/strings/str_cat.h" @@ -26,13 +27,44 @@ ABSL_NAMESPACE_BEGIN BadStatusOrAccess::BadStatusOrAccess(absl::Status status) : status_(std::move(status)) {} -BadStatusOrAccess::~BadStatusOrAccess() = default; +BadStatusOrAccess::BadStatusOrAccess(const BadStatusOrAccess& other) + : status_(other.status_) {} + +BadStatusOrAccess& BadStatusOrAccess::operator=( + const BadStatusOrAccess& other) { + // Ensure assignment is correct regardless of whether this->InitWhat() has + // already been called. + other.InitWhat(); + status_ = other.status_; + what_ = other.what_; + return *this; +} + +BadStatusOrAccess& BadStatusOrAccess::operator=(BadStatusOrAccess&& other) { + // Ensure assignment is correct regardless of whether this->InitWhat() has + // already been called. + other.InitWhat(); + status_ = std::move(other.status_); + what_ = std::move(other.what_); + return *this; +} + +BadStatusOrAccess::BadStatusOrAccess(BadStatusOrAccess&& other) + : status_(std::move(other.status_)) {} + const char* BadStatusOrAccess::what() const noexcept { - return "Bad StatusOr access"; + InitWhat(); + return what_.c_str(); } const absl::Status& BadStatusOrAccess::status() const { return status_; } +void BadStatusOrAccess::InitWhat() const { + absl::call_once(init_what_, [this] { + what_ = absl::StrCat("Bad StatusOr access: ", status_.ToString()); + }); +} + namespace internal_statusor { void Helper::HandleInvalidStatusCtorArg(absl::Status* status) { diff --git a/absl/status/statusor.h b/absl/status/statusor.h index b7c55cc8..a76e7201 100644 --- a/absl/status/statusor.h +++ b/absl/status/statusor.h @@ -44,6 +44,7 @@ #include <utility> #include "absl/base/attributes.h" +#include "absl/base/call_once.h" #include "absl/meta/type_traits.h" #include "absl/status/internal/statusor_internal.h" #include "absl/status/status.h" @@ -72,13 +73,18 @@ ABSL_NAMESPACE_BEGIN class BadStatusOrAccess : public std::exception { public: explicit BadStatusOrAccess(absl::Status status); - ~BadStatusOrAccess() override; + ~BadStatusOrAccess() override = default; + + BadStatusOrAccess(const BadStatusOrAccess& other); + BadStatusOrAccess& operator=(const BadStatusOrAccess& other); + BadStatusOrAccess(BadStatusOrAccess&& other); + BadStatusOrAccess& operator=(BadStatusOrAccess&& other); // BadStatusOrAccess::what() // // Returns the associated explanatory string of the `absl::StatusOr<T>` - // object's error code. This function only returns the string literal "Bad - // StatusOr Access" for cases when evaluating general exceptions. + // object's error code. This function contains information about the failing + // status, but its exact formatting may change and should not be depended on. // // The pointer of this string is guaranteed to be valid until any non-const // function is invoked on the exception object. @@ -91,12 +97,22 @@ class BadStatusOrAccess : public std::exception { const absl::Status& status() const; private: + void InitWhat() const; + absl::Status status_; + mutable absl::once_flag init_what_; + mutable std::string what_; }; // Returned StatusOr objects may not be ignored. template <typename T> +#if ABSL_HAVE_CPP_ATTRIBUTE(nodiscard) +// TODO(b/176172494): ABSL_MUST_USE_RESULT should expand to the more strict +// [[nodiscard]]. For now, just use [[nodiscard]] directly when it is available. +class [[nodiscard]] StatusOr; +#else class ABSL_MUST_USE_RESULT StatusOr; +#endif // ABSL_HAVE_CPP_ATTRIBUTE(nodiscard) // absl::StatusOr<T> // @@ -146,8 +162,8 @@ class ABSL_MUST_USE_RESULT StatusOr; // A `absl::StatusOr<T*>` can be constructed from a null pointer like any other // pointer value, and the result will be that `ok()` returns `true` and // `value()` returns `nullptr`. Checking the value of pointer in an -// `absl::StatusOr<T>` generally requires a bit more care, to ensure both that a -// value is present and that value is not null: +// `absl::StatusOr<T*>` generally requires a bit more care, to ensure both that +// a value is present and that value is not null: // // StatusOr<std::unique_ptr<Foo>> result = FooFactory::MakeNewFoo(arg); // if (!result.ok()) { @@ -419,8 +435,8 @@ class StatusOr : private internal_statusor::StatusOrData<T>, // if `T` can be constructed from a `U`. Can accept move or copy constructors. // // This constructor is explicit if `U` is not convertible to `T`. To avoid - // ambiguity, this constuctor is disabled if `U` is a `StatusOr<J>`, where `J` - // is convertible to `T`. + // ambiguity, this constructor is disabled if `U` is a `StatusOr<J>`, where + // `J` is convertible to `T`. template < typename U = T, absl::enable_if_t< @@ -437,8 +453,7 @@ class StatusOr : private internal_statusor::StatusOrData<T>, T, U&&>>>>>::value, int> = 0> StatusOr(U&& u) // NOLINT - : StatusOr(absl::in_place, std::forward<U>(u)) { - } + : StatusOr(absl::in_place, std::forward<U>(u)) {} template < typename U = T, @@ -457,13 +472,12 @@ class StatusOr : private internal_statusor::StatusOrData<T>, absl::negation<std::is_convertible<U&&, T>>>::value, int> = 0> explicit StatusOr(U&& u) // NOLINT - : StatusOr(absl::in_place, std::forward<U>(u)) { - } + : StatusOr(absl::in_place, std::forward<U>(u)) {} // StatusOr<T>::ok() // // Returns whether or not this `absl::StatusOr<T>` holds a `T` value. This - // member function is analagous to `absl::Status::ok()` and should be used + // member function is analogous to `absl::Status::ok()` and should be used // similarly to check the status of return values. // // Example: @@ -481,7 +495,7 @@ class StatusOr : private internal_statusor::StatusOrData<T>, // Returns a reference to the current `absl::Status` contained within the // `absl::StatusOr<T>`. If `absl::StatusOr<T>` contains a `T`, then this // function returns `absl::OkStatus()`. - const Status& status() const &; + const Status& status() const&; Status status() &&; // StatusOr<T>::value() @@ -510,10 +524,10 @@ class StatusOr : private internal_statusor::StatusOrData<T>, // // The `std::move` on statusor instead of on the whole expression enables // warnings about possible uses of the statusor object after the move. - const T& value() const&; - T& value() &; - const T&& value() const&&; - T&& value() &&; + const T& value() const& ABSL_ATTRIBUTE_LIFETIME_BOUND; + T& value() & ABSL_ATTRIBUTE_LIFETIME_BOUND; + const T&& value() const&& ABSL_ATTRIBUTE_LIFETIME_BOUND; + T&& value() && ABSL_ATTRIBUTE_LIFETIME_BOUND; // StatusOr<T>:: operator*() // @@ -525,10 +539,10 @@ class StatusOr : private internal_statusor::StatusOrData<T>, // `absl::StatusOr<T>`. Alternatively, see the `value()` member function for a // similar API that guarantees crashing or throwing an exception if there is // no current value. - const T& operator*() const&; - T& operator*() &; - const T&& operator*() const&&; - T&& operator*() &&; + const T& operator*() const& ABSL_ATTRIBUTE_LIFETIME_BOUND; + T& operator*() & ABSL_ATTRIBUTE_LIFETIME_BOUND; + const T&& operator*() const&& ABSL_ATTRIBUTE_LIFETIME_BOUND; + T&& operator*() && ABSL_ATTRIBUTE_LIFETIME_BOUND; // StatusOr<T>::operator->() // @@ -537,8 +551,8 @@ class StatusOr : private internal_statusor::StatusOrData<T>, // REQUIRES: `this->ok() == true`, otherwise the behavior is undefined. // // Use `this->ok()` to verify that there is a current value. - const T* operator->() const; - T* operator->(); + const T* operator->() const ABSL_ATTRIBUTE_LIFETIME_BOUND; + T* operator->() ABSL_ATTRIBUTE_LIFETIME_BOUND; // StatusOr<T>::value_or() // @@ -661,7 +675,9 @@ StatusOr<T>::StatusOr(absl::in_place_t, std::initializer_list<U> ilist, : Base(absl::in_place, ilist, std::forward<Args>(args)...) {} template <typename T> -const Status& StatusOr<T>::status() const & { return this->status_; } +const Status& StatusOr<T>::status() const& { + return this->status_; +} template <typename T> Status StatusOr<T>::status() && { return ok() ? OkStatus() : std::move(this->status_); diff --git a/absl/status/statusor_test.cc b/absl/status/statusor_test.cc index c2e8fb7e..7cae90e1 100644 --- a/absl/status/statusor_test.cc +++ b/absl/status/statusor_test.cc @@ -17,6 +17,7 @@ #include <array> #include <initializer_list> #include <memory> +#include <string> #include <type_traits> #include <utility> @@ -25,6 +26,7 @@ #include "absl/base/casts.h" #include "absl/memory/memory.h" #include "absl/status/status.h" +#include "absl/strings/string_view.h" #include "absl/types/any.h" #include "absl/utility/utility.h" @@ -34,6 +36,7 @@ using ::testing::AllOf; using ::testing::AnyWith; using ::testing::ElementsAre; using ::testing::Field; +using ::testing::HasSubstr; using ::testing::Ne; using ::testing::Not; using ::testing::Pointee; @@ -257,9 +260,9 @@ TEST(StatusOr, TestMoveOnlyInitializationFromTemporaryByValueOrDie) { TEST(StatusOr, TestValueOrDieOverloadForConstTemporary) { static_assert( - std::is_same<const int&&, - decltype( - std::declval<const absl::StatusOr<int>&&>().value())>(), + std::is_same< + const int&&, + decltype(std::declval<const absl::StatusOr<int>&&>().value())>(), "value() for const temporaries should return const T&&"); } @@ -303,20 +306,57 @@ TEST(StatusOr, StatusCtorForwards) { EXPECT_NE(status.message(), "Some error"); } +TEST(BadStatusOrAccessTest, CopyConstructionWhatOk) { + absl::Status error = + absl::InternalError("some arbitrary message too big for the sso buffer"); + absl::BadStatusOrAccess e1{error}; + absl::BadStatusOrAccess e2{e1}; + EXPECT_THAT(e1.what(), HasSubstr(error.ToString())); + EXPECT_THAT(e2.what(), HasSubstr(error.ToString())); +} + +TEST(BadStatusOrAccessTest, CopyAssignmentWhatOk) { + absl::Status error = + absl::InternalError("some arbitrary message too big for the sso buffer"); + absl::BadStatusOrAccess e1{error}; + absl::BadStatusOrAccess e2{absl::InternalError("other")}; + e2 = e1; + EXPECT_THAT(e1.what(), HasSubstr(error.ToString())); + EXPECT_THAT(e2.what(), HasSubstr(error.ToString())); +} + +TEST(BadStatusOrAccessTest, MoveConstructionWhatOk) { + absl::Status error = + absl::InternalError("some arbitrary message too big for the sso buffer"); + absl::BadStatusOrAccess e1{error}; + absl::BadStatusOrAccess e2{std::move(e1)}; + EXPECT_THAT(e2.what(), HasSubstr(error.ToString())); +} + +TEST(BadStatusOrAccessTest, MoveAssignmentWhatOk) { + absl::Status error = + absl::InternalError("some arbitrary message too big for the sso buffer"); + absl::BadStatusOrAccess e1{error}; + absl::BadStatusOrAccess e2{absl::InternalError("other")}; + e2 = std::move(e1); + EXPECT_THAT(e2.what(), HasSubstr(error.ToString())); +} + // Define `EXPECT_DEATH_OR_THROW` to test the behavior of `StatusOr::value`, // which either throws `BadStatusOrAccess` or `LOG(FATAL)` based on whether // exceptions are enabled. #ifdef ABSL_HAVE_EXCEPTIONS -#define EXPECT_DEATH_OR_THROW(statement, status_) \ - EXPECT_THROW( \ - { \ - try { \ - statement; \ - } catch (const absl::BadStatusOrAccess& e) { \ - EXPECT_EQ(e.status(), status_); \ - throw; \ - } \ - }, \ +#define EXPECT_DEATH_OR_THROW(statement, status_) \ + EXPECT_THROW( \ + { \ + try { \ + statement; \ + } catch (const absl::BadStatusOrAccess& e) { \ + EXPECT_EQ(e.status(), status_); \ + EXPECT_THAT(e.what(), HasSubstr(e.status().ToString())); \ + throw; \ + } \ + }, \ absl::BadStatusOrAccess); #else // ABSL_HAVE_EXCEPTIONS #define EXPECT_DEATH_OR_THROW(statement, status) \ @@ -412,8 +452,6 @@ TEST(StatusOr, TestStatusCtor) { EXPECT_EQ(thing.status().code(), absl::StatusCode::kCancelled); } - - TEST(StatusOr, TestValueCtor) { const int kI = 4; const absl::StatusOr<int> thing(kI); @@ -1300,8 +1338,6 @@ TEST(StatusOr, TestPointerDefaultCtor) { EXPECT_EQ(thing.status().code(), absl::StatusCode::kUnknown); } - - TEST(StatusOr, TestPointerStatusCtor) { absl::StatusOr<int*> thing(absl::CancelledError()); EXPECT_FALSE(thing.ok()); diff --git a/absl/strings/BUILD.bazel b/absl/strings/BUILD.bazel index 123b5efb..3f51252f 100644 --- a/absl/strings/BUILD.bazel +++ b/absl/strings/BUILD.bazel @@ -13,10 +13,10 @@ # See the License for the specific language governing permissions and # limitations under the License. -load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") load( "//absl:copts/configure_copts.bzl", "ABSL_DEFAULT_COPTS", + "ABSL_DEFAULT_LINKOPTS", "ABSL_TEST_COPTS", ) @@ -66,6 +66,7 @@ cc_library( "substitute.h", ], copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, deps = [ ":internal", "//absl/base", @@ -96,6 +97,7 @@ cc_library( "internal/utf8.h", ], copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, deps = [ "//absl/base:config", "//absl/base:core_headers", @@ -269,15 +271,27 @@ cc_library( name = "cord_internal", srcs = [ "internal/cord_internal.cc", + "internal/cord_rep_btree.cc", + "internal/cord_rep_btree_navigator.cc", + "internal/cord_rep_btree_reader.cc", + "internal/cord_rep_consume.cc", + "internal/cord_rep_crc.cc", "internal/cord_rep_ring.cc", ], hdrs = [ + "internal/cord_data_edge.h", "internal/cord_internal.h", + "internal/cord_rep_btree.h", + "internal/cord_rep_btree_navigator.h", + "internal/cord_rep_btree_reader.h", + "internal/cord_rep_consume.h", + "internal/cord_rep_crc.h", "internal/cord_rep_flat.h", "internal/cord_rep_ring.h", "internal/cord_rep_ring_reader.h", ], copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ "//visibility:private", ], @@ -292,7 +306,111 @@ cc_library( "//absl/container:compressed_tuple", "//absl/container:inlined_vector", "//absl/container:layout", + "//absl/functional:function_ref", "//absl/meta:type_traits", + "//absl/types:span", + ], +) + +cc_test( + name = "cord_data_edge_test", + size = "small", + srcs = ["internal/cord_data_edge_test.cc"], + copts = ABSL_TEST_COPTS, + visibility = ["//visibility:private"], + deps = [ + ":cord_internal", + ":cord_rep_test_util", + ":strings", + "//absl/base:config", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "cord_rep_btree_test", + size = "medium", + srcs = ["internal/cord_rep_btree_test.cc"], + copts = ABSL_TEST_COPTS, + visibility = ["//visibility:private"], + deps = [ + ":cord_internal", + ":cord_rep_test_util", + ":strings", + "//absl/base:config", + "//absl/base:raw_logging_internal", + "//absl/cleanup", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "cord_rep_btree_navigator_test", + size = "medium", + srcs = ["internal/cord_rep_btree_navigator_test.cc"], + copts = ABSL_TEST_COPTS, + visibility = ["//visibility:private"], + deps = [ + ":cord_internal", + ":cord_rep_test_util", + ":strings", + "//absl/base:config", + "//absl/base:raw_logging_internal", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "cord_rep_btree_reader_test", + size = "medium", + srcs = ["internal/cord_rep_btree_reader_test.cc"], + copts = ABSL_TEST_COPTS, + visibility = ["//visibility:private"], + deps = [ + ":cord", + ":cord_internal", + ":cord_rep_test_util", + ":strings", + "//absl/base:config", + "//absl/base:raw_logging_internal", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "cord_rep_crc_test", + size = "small", + srcs = ["internal/cord_rep_crc_test.cc"], + copts = ABSL_TEST_COPTS, + visibility = ["//visibility:private"], + deps = [ + ":cord_internal", + ":cord_rep_test_util", + "//absl/base:config", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "cordz_update_tracker", + hdrs = ["internal/cordz_update_tracker.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "//absl:__subpackages__", + ], + deps = ["//absl/base:config"], +) + +cc_test( + name = "cordz_update_tracker_test", + srcs = ["internal/cordz_update_tracker_test.cc"], + deps = [ + ":cordz_update_tracker", + "//absl/base:config", + "//absl/base:core_headers", + "//absl/synchronization", + "@com_google_googletest//:gtest_main", ], ) @@ -300,17 +418,28 @@ cc_library( name = "cord", srcs = [ "cord.cc", + "cord_analysis.cc", + "cord_analysis.h", + "cord_buffer.cc", ], hdrs = [ "cord.h", + "cord_buffer.h", ], copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, deps = [ ":cord_internal", + ":cordz_functions", + ":cordz_info", + ":cordz_statistics", + ":cordz_update_scope", + ":cordz_update_tracker", ":internal", ":str_format", ":strings", "//absl/base", + "//absl/base:config", "//absl/base:core_headers", "//absl/base:endian", "//absl/base:raw_logging_internal", @@ -318,7 +447,224 @@ cc_library( "//absl/container:inlined_vector", "//absl/functional:function_ref", "//absl/meta:type_traits", + "//absl/numeric:bits", "//absl/types:optional", + "//absl/types:span", + ], +) + +cc_library( + name = "cordz_handle", + srcs = ["internal/cordz_handle.cc"], + hdrs = ["internal/cordz_handle.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "//absl:__subpackages__", + ], + deps = [ + "//absl/base", + "//absl/base:config", + "//absl/base:raw_logging_internal", + "//absl/synchronization", + ], +) + +cc_library( + name = "cordz_info", + srcs = ["internal/cordz_info.cc"], + hdrs = ["internal/cordz_info.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "//absl:__subpackages__", + ], + deps = [ + ":cord_internal", + ":cordz_functions", + ":cordz_handle", + ":cordz_statistics", + ":cordz_update_tracker", + "//absl/base", + "//absl/base:config", + "//absl/base:core_headers", + "//absl/base:raw_logging_internal", + "//absl/container:inlined_vector", + "//absl/debugging:stacktrace", + "//absl/synchronization", + "//absl/types:span", + ], +) + +cc_library( + name = "cordz_update_scope", + hdrs = ["internal/cordz_update_scope.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "//absl:__subpackages__", + ], + deps = [ + ":cord_internal", + ":cordz_info", + ":cordz_update_tracker", + "//absl/base:config", + "//absl/base:core_headers", + ], +) + +cc_test( + name = "cordz_update_scope_test", + srcs = ["internal/cordz_update_scope_test.cc"], + copts = ABSL_DEFAULT_COPTS, + deps = [ + ":cord_internal", + ":cordz_info", + ":cordz_test_helpers", + ":cordz_update_scope", + ":cordz_update_tracker", + "//absl/base:config", + "@com_google_googletest//:gtest_main", + ], +) + +cc_library( + name = "cordz_sample_token", + srcs = ["internal/cordz_sample_token.cc"], + hdrs = ["internal/cordz_sample_token.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "//absl:__subpackages__", + ], + deps = [ + ":cordz_handle", + ":cordz_info", + "//absl/base:config", + ], +) + +cc_library( + name = "cordz_functions", + srcs = ["internal/cordz_functions.cc"], + hdrs = ["internal/cordz_functions.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "//absl:__subpackages__", + ], + deps = [ + "//absl/base:config", + "//absl/base:core_headers", + "//absl/base:raw_logging_internal", + "//absl/profiling:exponential_biased", + ], +) + +cc_library( + name = "cordz_statistics", + hdrs = ["internal/cordz_statistics.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + visibility = [ + "//absl:__subpackages__", + ], + deps = [ + ":cordz_update_tracker", + "//absl/base:config", + ], +) + +cc_test( + name = "cordz_functions_test", + srcs = [ + "internal/cordz_functions_test.cc", + ], + deps = [ + ":cordz_functions", + ":cordz_test_helpers", + "//absl/base:config", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "cordz_handle_test", + srcs = [ + "internal/cordz_handle_test.cc", + ], + deps = [ + ":cordz_handle", + "//absl/base:config", + "//absl/memory", + "//absl/random", + "//absl/random:distributions", + "//absl/synchronization", + "//absl/synchronization:thread_pool", + "//absl/time", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "cordz_info_test", + srcs = [ + "internal/cordz_info_test.cc", + ], + deps = [ + ":cord_internal", + ":cordz_handle", + ":cordz_info", + ":cordz_statistics", + ":cordz_test_helpers", + ":cordz_update_tracker", + ":strings", + "//absl/base:config", + "//absl/debugging:stacktrace", + "//absl/debugging:symbolize", + "//absl/types:span", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "cordz_info_statistics_test", + srcs = [ + "internal/cordz_info_statistics_test.cc", + ], + deps = [ + ":cord", + ":cord_internal", + ":cordz_info", + ":cordz_sample_token", + ":cordz_statistics", + ":cordz_update_scope", + ":cordz_update_tracker", + "//absl/base:config", + "//absl/synchronization", + "//absl/synchronization:thread_pool", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "cordz_sample_token_test", + srcs = [ + "internal/cordz_sample_token_test.cc", + ], + deps = [ + ":cord_internal", + ":cordz_handle", + ":cordz_info", + ":cordz_sample_token", + ":cordz_test_helpers", + "//absl/base:config", + "//absl/memory", + "//absl/random", + "//absl/synchronization", + "//absl/synchronization:thread_pool", + "//absl/time", + "@com_google_googletest//:gtest_main", ], ) @@ -329,8 +675,62 @@ cc_library( "cord_test_helpers.h", ], copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":cord", + ":cord_internal", + ":strings", + "//absl/base:config", + ], +) + +cc_library( + name = "cord_rep_test_util", + testonly = 1, + hdrs = ["internal/cord_rep_test_util.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":cord_internal", + ":strings", + "//absl/base:config", + "//absl/base:raw_logging_internal", + ], +) + +cc_library( + name = "cordz_test_helpers", + testonly = 1, + hdrs = ["cordz_test_helpers.h"], + copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + deps = [ + ":cord", + ":cord_internal", + ":cordz_info", + ":cordz_sample_token", + ":cordz_statistics", + ":cordz_update_tracker", + ":strings", + "//absl/base:config", + "//absl/base:core_headers", + "@com_google_googletest//:gtest", + ], +) + +cc_test( + name = "cord_buffer_test", + size = "small", + srcs = ["cord_buffer_test.cc"], + copts = ABSL_TEST_COPTS, + visibility = ["//visibility:private"], deps = [ ":cord", + ":cord_internal", + ":cord_rep_test_util", + "//absl/base:config", + "//absl/types:span", + "@com_google_googletest//:gtest_main", ], ) @@ -343,6 +743,8 @@ cc_test( deps = [ ":cord", ":cord_test_helpers", + ":cordz_functions", + ":cordz_test_helpers", ":str_format", ":strings", "//absl/base", @@ -351,6 +753,40 @@ cc_test( "//absl/base:endian", "//absl/base:raw_logging_internal", "//absl/container:fixed_array", + "//absl/hash", + "//absl/random", + "@com_google_googletest//:gtest_main", + ], +) + +cc_test( + name = "cordz_test", + size = "medium", + srcs = ["cordz_test.cc"], + copts = ABSL_TEST_COPTS, + tags = [ + "benchmark", + "no_test_android_arm", + "no_test_android_arm64", + "no_test_android_x86", + "no_test_ios_x86_64", + "no_test_loonix", + "no_test_msvc_x64", + ], + visibility = ["//visibility:private"], + deps = [ + ":cord", + ":cord_test_helpers", + ":cordz_functions", + ":cordz_info", + ":cordz_sample_token", + ":cordz_statistics", + ":cordz_test_helpers", + ":cordz_update_tracker", + ":strings", + "//absl/base:config", + "//absl/base:core_headers", + "//absl/base:raw_logging_internal", "@com_google_googletest//:gtest_main", ], ) @@ -434,6 +870,7 @@ cc_test( ":strings", "//absl/base:core_headers", "//absl/base:dynamic_annotations", + "//absl/container:btree", "//absl/container:flat_hash_map", "//absl/container:node_hash_map", "@com_google_googletest//:gtest_main", @@ -675,6 +1112,7 @@ cc_library( "str_format.h", ], copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, deps = [ ":str_format_internal", ], @@ -700,6 +1138,7 @@ cc_library( "internal/str_format/parser.h", ], copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, visibility = ["//visibility:private"], deps = [ ":strings", @@ -712,6 +1151,7 @@ cc_library( "//absl/numeric:representation", "//absl/types:optional", "//absl/types:span", + "//absl/utility", ], ) @@ -787,6 +1227,7 @@ cc_test( deps = [ ":str_format_internal", ":strings", + "//absl/base:core_headers", "//absl/base:raw_logging_internal", "//absl/types:optional", "@com_google_googletest//:gtest_main", @@ -822,6 +1263,7 @@ cc_library( testonly = True, srcs = ["internal/pow10_helper.cc"], hdrs = ["internal/pow10_helper.h"], + linkopts = ABSL_DEFAULT_LINKOPTS, visibility = ["//visibility:private"], deps = ["//absl/base:config"], ) diff --git a/absl/strings/CMakeLists.txt b/absl/strings/CMakeLists.txt index 3b7ae639..bb073301 100644 --- a/absl/strings/CMakeLists.txt +++ b/absl/strings/CMakeLists.txt @@ -68,6 +68,7 @@ absl_cc_library( PUBLIC ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME strings_internal @@ -101,7 +102,7 @@ absl_cc_test( DEPS absl::strings absl::base - gmock_main + GTest::gmock_main ) absl_cc_test( @@ -115,7 +116,7 @@ absl_cc_test( absl::strings absl::core_headers absl::fixed_array - gmock_main + GTest::gmock_main ) absl_cc_test( @@ -128,7 +129,7 @@ absl_cc_test( DEPS absl::strings absl::core_headers - gmock_main + GTest::gmock_main ) absl_cc_test( @@ -142,7 +143,7 @@ absl_cc_test( DEPS absl::strings absl::core_headers - gmock_main + GTest::gmock_main ) absl_cc_test( @@ -156,7 +157,7 @@ absl_cc_test( absl::strings_internal absl::base absl::core_headers - gmock_main + GTest::gmock_main ) absl_cc_test( @@ -169,7 +170,7 @@ absl_cc_test( DEPS absl::strings absl::type_traits - gmock_main + GTest::gmock_main ) absl_cc_test( @@ -184,7 +185,7 @@ absl_cc_test( absl::config absl::core_headers absl::dynamic_annotations - gmock_main + GTest::gmock_main ) absl_cc_test( @@ -197,7 +198,7 @@ absl_cc_test( DEPS absl::strings absl::core_headers - gmock_main + GTest::gmock_main ) absl_cc_test( @@ -209,7 +210,7 @@ absl_cc_test( ${ABSL_TEST_COPTS} DEPS absl::strings - gmock_main + GTest::gmock_main ) absl_cc_test( @@ -221,12 +222,12 @@ absl_cc_test( ${ABSL_TEST_COPTS} DEPS absl::strings - absl::base absl::core_headers absl::dynamic_annotations + absl::btree absl::flat_hash_map absl::node_hash_map - gmock_main + GTest::gmock_main ) absl_cc_test( @@ -238,7 +239,7 @@ absl_cc_test( ${ABSL_TEST_COPTS} DEPS absl::strings_internal - gmock_main + GTest::gmock_main ) absl_cc_test( @@ -253,7 +254,7 @@ absl_cc_test( absl::base absl::core_headers absl::type_traits - gmock_main + GTest::gmock_main ) absl_cc_test( @@ -268,7 +269,7 @@ absl_cc_test( absl::base absl::core_headers absl::memory - gmock_main + GTest::gmock_main ) absl_cc_test( @@ -281,7 +282,7 @@ absl_cc_test( DEPS absl::strings absl::core_headers - gmock_main + GTest::gmock_main ) absl_cc_test( @@ -301,7 +302,7 @@ absl_cc_test( absl::random_random absl::random_distributions absl::strings_internal - gmock_main + GTest::gmock_main ) absl_cc_test( @@ -314,7 +315,7 @@ absl_cc_test( DEPS absl::strings absl::base - gmock_main + GTest::gmock_main ) absl_cc_test( @@ -326,7 +327,7 @@ absl_cc_test( ${ABSL_TEST_COPTS} DEPS absl::strings_internal - gmock_main + GTest::gmock_main ) absl_cc_test( @@ -340,7 +341,7 @@ absl_cc_test( absl::strings absl::str_format absl::pow10_helper - gmock_main + GTest::gmock_main ) absl_cc_test( @@ -355,7 +356,7 @@ absl_cc_test( absl::strings absl::config absl::raw_logging_internal - gmock_main + GTest::gmock_main ) absl_cc_test( @@ -370,7 +371,7 @@ absl_cc_test( DEPS absl::strings absl::config - gmock_main + GTest::gmock_main ) absl_cc_library( @@ -385,6 +386,7 @@ absl_cc_library( PUBLIC ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME str_format_internal @@ -412,6 +414,7 @@ absl_cc_library( absl::core_headers absl::numeric_representation absl::type_traits + absl::utility absl::int128 absl::span ) @@ -428,7 +431,7 @@ absl_cc_test( absl::cord absl::strings absl::core_headers - gmock_main + GTest::gmock_main ) absl_cc_test( @@ -442,7 +445,7 @@ absl_cc_test( absl::str_format absl::str_format_internal absl::strings - gmock_main + GTest::gmock_main ) absl_cc_test( @@ -455,7 +458,7 @@ absl_cc_test( DEPS absl::str_format absl::str_format_internal - gmock_main + GTest::gmock_main ) absl_cc_test( @@ -467,7 +470,7 @@ absl_cc_test( ${ABSL_TEST_COPTS} DEPS absl::str_format_internal - gmock_main + GTest::gmock_main ) absl_cc_test( @@ -479,7 +482,7 @@ absl_cc_test( ${ABSL_TEST_COPTS} DEPS absl::str_format - gmock_main + GTest::gmock_main ) absl_cc_test( @@ -492,9 +495,10 @@ absl_cc_test( DEPS absl::strings absl::str_format_internal + absl::core_headers absl::raw_logging_internal absl::int128 - gmock_main + GTest::gmock_main ) absl_cc_test( @@ -507,7 +511,7 @@ absl_cc_test( DEPS absl::str_format_internal absl::cord - gmock_main + GTest::gmock_main ) absl_cc_test( @@ -520,9 +524,10 @@ absl_cc_test( DEPS absl::str_format_internal absl::core_headers - gmock_main + GTest::gmock_main ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME pow10_helper @@ -547,43 +552,344 @@ absl_cc_test( DEPS absl::pow10_helper absl::str_format - gmock_main + GTest::gmock_main ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME - cord + cord_internal HDRS - "cord.h" - SRCS - "cord.cc" - "internal/cord_internal.cc" + "internal/cord_data_edge.h" "internal/cord_internal.h" + "internal/cord_rep_btree.h" + "internal/cord_rep_btree_navigator.h" + "internal/cord_rep_btree_reader.h" + "internal/cord_rep_crc.h" + "internal/cord_rep_consume.h" + "internal/cord_rep_flat.h" "internal/cord_rep_ring.h" - "internal/cord_rep_ring.cc" "internal/cord_rep_ring_reader.h" - "internal/cord_rep_flat.h" + SRCS + "internal/cord_internal.cc" + "internal/cord_rep_btree.cc" + "internal/cord_rep_btree_navigator.cc" + "internal/cord_rep_btree_reader.cc" + "internal/cord_rep_crc.cc" + "internal/cord_rep_consume.cc" + "internal/cord_rep_ring.cc" COPTS ${ABSL_DEFAULT_COPTS} DEPS - absl::base absl::base_internal absl::compressed_tuple absl::config absl::core_headers absl::endian + absl::inlined_vector + absl::layout + absl::raw_logging_internal + absl::strings + absl::throw_delegate + absl::type_traits +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + cordz_update_tracker + HDRS + "internal/cordz_update_tracker.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config +) + +absl_cc_test( + NAME + cordz_update_tracker_test + SRCS + "internal/cordz_update_tracker_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::config + absl::cordz_update_tracker + absl::core_headers + absl::synchronization + GTest::gmock_main +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + cordz_functions + HDRS + "internal/cordz_functions.h" + SRCS + "internal/cordz_functions.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::core_headers + absl::exponential_biased + absl::raw_logging_internal +) + +absl_cc_test( + NAME + cordz_functions_test + SRCS + "internal/cordz_functions_test.cc" + DEPS + absl::config + absl::cordz_functions + absl::cordz_test_helpers + GTest::gmock_main +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + cordz_statistics + HDRS + "internal/cordz_statistics.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::core_headers + absl::cordz_update_tracker + absl::synchronization +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + cordz_handle + HDRS + "internal/cordz_handle.h" + SRCS + "internal/cordz_handle.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base + absl::config + absl::raw_logging_internal + absl::synchronization +) + +absl_cc_test( + NAME + cordz_handle_test + SRCS + "internal/cordz_handle_test.cc" + DEPS + absl::config + absl::cordz_handle + absl::cordz_test_helpers + absl::memory + absl::random_random + absl::random_distributions + absl::synchronization + absl::time + GTest::gmock_main +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + cordz_info + HDRS + "internal/cordz_info.h" + SRCS + "internal/cordz_info.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base + absl::config + absl::cord_internal + absl::cordz_functions + absl::cordz_handle + absl::cordz_statistics + absl::cordz_update_tracker + absl::core_headers + absl::inlined_vector + absl::span + absl::raw_logging_internal + absl::stacktrace + absl::synchronization +) + +absl_cc_test( + NAME + cordz_info_test + SRCS + "internal/cordz_info_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::config + absl::cord_internal + absl::cordz_test_helpers + absl::cordz_handle + absl::cordz_info + absl::cordz_statistics + absl::cordz_test_helpers + absl::cordz_update_tracker + absl::span + absl::stacktrace + absl::symbolize + GTest::gmock_main +) + +absl_cc_test( + NAME + cordz_info_statistics_test + SRCS + "internal/cordz_info_statistics_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::config + absl::cord + absl::cord_internal + absl::cordz_info + absl::cordz_sample_token + absl::cordz_statistics + absl::cordz_update_scope + absl::cordz_update_tracker + absl::thread_pool + GTest::gmock_main +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + cordz_sample_token + HDRS + "internal/cordz_sample_token.h" + SRCS + "internal/cordz_sample_token.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::cordz_handle + absl::cordz_info +) + +absl_cc_test( + NAME + cordz_sample_token_test + SRCS + "internal/cordz_sample_token_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::config + absl::cord_internal + absl::cordz_handle + absl::cordz_info + absl::cordz_info + absl::cordz_sample_token + absl::cordz_test_helpers + absl::memory + absl::random_random + absl::synchronization + absl::thread_pool + absl::time + GTest::gmock_main +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + cordz_update_scope + HDRS + "internal/cordz_update_scope.h" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::config + absl::cord_internal + absl::cordz_info + absl::cordz_update_tracker + absl::core_headers +) + +absl_cc_test( + NAME + cordz_update_scope_test + SRCS + "internal/cordz_update_scope_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::config + absl::cord_internal + absl::cordz_info + absl::cordz_test_helpers + absl::cordz_update_scope + absl::cordz_update_tracker + absl::core_headers + GTest::gmock_main +) + +absl_cc_library( + NAME + cord + HDRS + "cord.h" + "cord_buffer.h" + SRCS + "cord.cc" + "cord_analysis.cc" + "cord_analysis.h" + "cord_buffer.cc" + COPTS + ${ABSL_DEFAULT_COPTS} + DEPS + absl::base + absl::config + absl::cord_internal + absl::cordz_functions + absl::cordz_info + absl::cordz_update_scope + absl::cordz_update_tracker + absl::core_headers + absl::endian absl::fixed_array absl::function_ref absl::inlined_vector absl::optional absl::raw_logging_internal + absl::span absl::strings - absl::strings_internal - absl::throw_delegate absl::type_traits PUBLIC ) +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + cord_rep_test_util + HDRS + "internal/cord_rep_test_util.h" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::config + absl::cord_internal + absl::raw_logging_internal + absl::strings + TESTONLY +) + absl_cc_library( NAME cord_test_helpers @@ -592,7 +898,31 @@ absl_cc_library( COPTS ${ABSL_TEST_COPTS} DEPS + absl::config + absl::cord + absl::cord_internal + absl::strings + TESTONLY +) + +# Internal-only target, do not depend on directly. +absl_cc_library( + NAME + cordz_test_helpers + HDRS + "cordz_test_helpers.h" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::config absl::cord + absl::cord_internal + absl::cordz_info + absl::cordz_sample_token + absl::cordz_statistics + absl::cordz_update_tracker + absl::core_headers + absl::strings TESTONLY ) @@ -609,28 +939,118 @@ absl_cc_test( absl::strings absl::base absl::config + absl::cord_test_helpers + absl::cordz_test_helpers absl::core_headers absl::endian + absl::hash + absl::random_random absl::raw_logging_internal absl::fixed_array - gmock_main + GTest::gmock_main ) absl_cc_test( NAME - cord_ring_test + cord_data_edge_test SRCS - "cord_ring_test.cc" + "internal/cord_data_edge_test.cc" COPTS ${ABSL_TEST_COPTS} DEPS + absl::base absl::config - absl::cord + absl::cord_internal + absl::cord_rep_test_util + absl::core_headers + absl::strings + GTest::gmock_main +) + +absl_cc_test( + NAME + cord_rep_btree_test + SRCS + "internal/cord_rep_btree_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::base + absl::cleanup + absl::config + absl::cord_internal + absl::cord_rep_test_util + absl::core_headers + absl::raw_logging_internal absl::strings + GTest::gmock_main +) + +absl_cc_test( + NAME + cord_rep_btree_navigator_test + SRCS + "internal/cord_rep_btree_navigator_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS absl::base + absl::config + absl::cord_internal + absl::cord_rep_test_util + absl::core_headers + absl::raw_logging_internal + absl::strings + GTest::gmock_main +) + +absl_cc_test( + NAME + cord_rep_btree_reader_test + SRCS + "internal/cord_rep_btree_reader_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::base + absl::config + absl::cord_internal + absl::cord_rep_test_util absl::core_headers absl::raw_logging_internal - gmock_main + absl::strings + GTest::gmock_main +) + +absl_cc_test( + NAME + cord_rep_crc_test + SRCS + "internal/cord_rep_crc_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::config + absl::cord_internal + absl::cord_rep_test_util + GTest::gmock_main +) + +absl_cc_test( + NAME + cord_ring_test + SRCS + "cord_ring_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::base + absl::config + absl::cord_internal + absl::core_headers + absl::raw_logging_internal + absl::strings + GTest::gmock_main ) absl_cc_test( @@ -641,9 +1061,33 @@ absl_cc_test( COPTS ${ABSL_TEST_COPTS} DEPS - absl::cord + absl::base + absl::cord_internal + absl::core_headers absl::strings + GTest::gmock_main +) + +absl_cc_test( + NAME + cordz_test + SRCS + "cordz_test.cc" + COPTS + ${ABSL_TEST_COPTS} + DEPS + absl::cord + absl::cord_test_helpers + absl::cordz_test_helpers + absl::cordz_functions + absl::cordz_info + absl::cordz_sample_token + absl::cordz_statistics + absl::cordz_update_tracker absl::base + absl::config absl::core_headers - gmock_main + absl::raw_logging_internal + absl::strings + GTest::gmock_main ) diff --git a/absl/strings/ascii.h b/absl/strings/ascii.h index b46bc71f..42eadaea 100644 --- a/absl/strings/ascii.h +++ b/absl/strings/ascii.h @@ -133,7 +133,7 @@ inline bool ascii_isdigit(unsigned char c) { return c >= '0' && c <= '9'; } // ascii_isprint() // -// Determines whether the given character is printable, including whitespace. +// Determines whether the given character is printable, including spaces. inline bool ascii_isprint(unsigned char c) { return c >= 32 && c < 127; } // ascii_isgraph() @@ -197,7 +197,7 @@ ABSL_MUST_USE_RESULT inline std::string AsciiStrToUpper(absl::string_view s) { ABSL_MUST_USE_RESULT inline absl::string_view StripLeadingAsciiWhitespace( absl::string_view str) { auto it = std::find_if_not(str.begin(), str.end(), absl::ascii_isspace); - return str.substr(it - str.begin()); + return str.substr(static_cast<size_t>(it - str.begin())); } // Strips in place whitespace from the beginning of the given string. @@ -211,13 +211,13 @@ inline void StripLeadingAsciiWhitespace(std::string* str) { ABSL_MUST_USE_RESULT inline absl::string_view StripTrailingAsciiWhitespace( absl::string_view str) { auto it = std::find_if_not(str.rbegin(), str.rend(), absl::ascii_isspace); - return str.substr(0, str.rend() - it); + return str.substr(0, static_cast<size_t>(str.rend() - it)); } // Strips in place whitespace from the end of the given string inline void StripTrailingAsciiWhitespace(std::string* str) { auto it = std::find_if_not(str->rbegin(), str->rend(), absl::ascii_isspace); - str->erase(str->rend() - it); + str->erase(static_cast<size_t>(str->rend() - it)); } // Returns absl::string_view with whitespace stripped from both ends of the diff --git a/absl/strings/charconv.cc b/absl/strings/charconv.cc index b8674c28..fefcfc90 100644 --- a/absl/strings/charconv.cc +++ b/absl/strings/charconv.cc @@ -111,7 +111,7 @@ struct FloatTraits<double> { return sign ? -ldexp(mantissa, exponent) : ldexp(mantissa, exponent); #else constexpr uint64_t kMantissaMask = - (uint64_t(1) << (kTargetMantissaBits - 1)) - 1; + (uint64_t{1} << (kTargetMantissaBits - 1)) - 1; uint64_t dbl = static_cast<uint64_t>(sign) << 63; if (mantissa > kMantissaMask) { // Normal value. @@ -151,7 +151,7 @@ struct FloatTraits<float> { return sign ? -ldexpf(mantissa, exponent) : ldexpf(mantissa, exponent); #else constexpr uint32_t kMantissaMask = - (uint32_t(1) << (kTargetMantissaBits - 1)) - 1; + (uint32_t{1} << (kTargetMantissaBits - 1)) - 1; uint32_t flt = static_cast<uint32_t>(sign) << 31; if (mantissa > kMantissaMask) { // Normal value. @@ -499,7 +499,7 @@ bool MustRoundUp(uint64_t guess_mantissa, int guess_exponent, template <typename FloatType> CalculatedFloat CalculatedFloatFromRawValues(uint64_t mantissa, int exponent) { CalculatedFloat result; - if (mantissa == uint64_t(1) << FloatTraits<FloatType>::kTargetMantissaBits) { + if (mantissa == uint64_t{1} << FloatTraits<FloatType>::kTargetMantissaBits) { mantissa >>= 1; exponent += 1; } diff --git a/absl/strings/charconv.h b/absl/strings/charconv.h index e04be32f..7c509812 100644 --- a/absl/strings/charconv.h +++ b/absl/strings/charconv.h @@ -64,8 +64,9 @@ struct from_chars_result { // the result in `value`. // // The matching pattern format is almost the same as that of strtod(), except -// that C locale is not respected, and an initial '+' character in the input -// range will never be matched. +// that (1) C locale is not respected, (2) an initial '+' character in the +// input range will never be matched, and (3) leading whitespaces are not +// ignored. // // If `fmt` is set, it must be one of the enumerator values of the chars_format. // (This is despite the fact that chars_format is a bitmask type.) If set to diff --git a/absl/strings/cord.cc b/absl/strings/cord.cc index 93533757..85a67a08 100644 --- a/absl/strings/cord.cc +++ b/absl/strings/cord.cc @@ -34,10 +34,16 @@ #include "absl/base/port.h" #include "absl/container/fixed_array.h" #include "absl/container/inlined_vector.h" +#include "absl/strings/cord_buffer.h" #include "absl/strings/escaping.h" +#include "absl/strings/internal/cord_data_edge.h" #include "absl/strings/internal/cord_internal.h" +#include "absl/strings/internal/cord_rep_btree.h" +#include "absl/strings/internal/cord_rep_crc.h" #include "absl/strings/internal/cord_rep_flat.h" -#include "absl/strings/internal/cord_rep_ring.h" +#include "absl/strings/internal/cordz_statistics.h" +#include "absl/strings/internal/cordz_update_scope.h" +#include "absl/strings/internal/cordz_update_tracker.h" #include "absl/strings/internal/resize_uninitialized.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" @@ -48,73 +54,19 @@ namespace absl { ABSL_NAMESPACE_BEGIN using ::absl::cord_internal::CordRep; -using ::absl::cord_internal::CordRepConcat; +using ::absl::cord_internal::CordRepBtree; +using ::absl::cord_internal::CordRepCrc; using ::absl::cord_internal::CordRepExternal; using ::absl::cord_internal::CordRepFlat; -using ::absl::cord_internal::CordRepRing; using ::absl::cord_internal::CordRepSubstring; -using ::absl::cord_internal::kMinFlatLength; +using ::absl::cord_internal::CordzUpdateTracker; +using ::absl::cord_internal::InlineData; using ::absl::cord_internal::kMaxFlatLength; - -using ::absl::cord_internal::CONCAT; -using ::absl::cord_internal::EXTERNAL; -using ::absl::cord_internal::FLAT; -using ::absl::cord_internal::RING; -using ::absl::cord_internal::SUBSTRING; +using ::absl::cord_internal::kMinFlatLength; using ::absl::cord_internal::kInlinedVectorSize; using ::absl::cord_internal::kMaxBytesToCopy; -constexpr uint64_t Fibonacci(unsigned char n, uint64_t a = 0, uint64_t b = 1) { - return n == 0 ? a : Fibonacci(n - 1, b, a + b); -} - -static_assert(Fibonacci(63) == 6557470319842, - "Fibonacci values computed incorrectly"); - -// Minimum length required for a given depth tree -- a tree is considered -// balanced if -// length(t) >= min_length[depth(t)] -// The root node depth is allowed to become twice as large to reduce rebalancing -// for larger strings (see IsRootBalanced). -static constexpr uint64_t min_length[] = { - Fibonacci(2), Fibonacci(3), Fibonacci(4), Fibonacci(5), - Fibonacci(6), Fibonacci(7), Fibonacci(8), Fibonacci(9), - Fibonacci(10), Fibonacci(11), Fibonacci(12), Fibonacci(13), - Fibonacci(14), Fibonacci(15), Fibonacci(16), Fibonacci(17), - Fibonacci(18), Fibonacci(19), Fibonacci(20), Fibonacci(21), - Fibonacci(22), Fibonacci(23), Fibonacci(24), Fibonacci(25), - Fibonacci(26), Fibonacci(27), Fibonacci(28), Fibonacci(29), - Fibonacci(30), Fibonacci(31), Fibonacci(32), Fibonacci(33), - Fibonacci(34), Fibonacci(35), Fibonacci(36), Fibonacci(37), - Fibonacci(38), Fibonacci(39), Fibonacci(40), Fibonacci(41), - Fibonacci(42), Fibonacci(43), Fibonacci(44), Fibonacci(45), - Fibonacci(46), Fibonacci(47), - 0xffffffffffffffffull, // Avoid overflow -}; - -static const int kMinLengthSize = ABSL_ARRAYSIZE(min_length); - -static inline bool cord_ring_enabled() { - return cord_internal::cord_ring_buffer_enabled.load( - std::memory_order_relaxed); -} - -static inline bool IsRootBalanced(CordRep* node) { - if (node->tag != CONCAT) { - return true; - } else if (node->concat()->depth() <= 15) { - return true; - } else if (node->concat()->depth() > kMinLengthSize) { - return false; - } else { - // Allow depth to become twice as large as implied by fibonacci rule to - // reduce rebalancing for larger strings. - return (node->length >= min_length[node->concat()->depth() / 2]); - } -} - -static CordRep* Rebalance(CordRep* node); static void DumpNode(CordRep* rep, bool include_data, std::ostream* os, int indent = 0); static bool VerifyNode(CordRep* root, CordRep* start_node, @@ -136,119 +88,32 @@ static inline CordRep* VerifyTree(CordRep* node) { return node; } -// Return the depth of a node -static int Depth(const CordRep* rep) { - if (rep->tag == CONCAT) { - return rep->concat()->depth(); - } else { - return 0; - } -} - -static void SetConcatChildren(CordRepConcat* concat, CordRep* left, - CordRep* right) { - concat->left = left; - concat->right = right; - - concat->length = left->length + right->length; - concat->set_depth(1 + std::max(Depth(left), Depth(right))); -} - -// Create a concatenation of the specified nodes. -// Does not change the refcounts of "left" and "right". -// The returned node has a refcount of 1. -static CordRep* RawConcat(CordRep* left, CordRep* right) { - // Avoid making degenerate concat nodes (one child is empty) - if (left == nullptr) return right; - if (right == nullptr) return left; - if (left->length == 0) { - CordRep::Unref(left); - return right; - } - if (right->length == 0) { - CordRep::Unref(right); - return left; - } - - CordRepConcat* rep = new CordRepConcat(); - rep->tag = CONCAT; - SetConcatChildren(rep, left, right); - - return rep; -} - -static CordRep* Concat(CordRep* left, CordRep* right) { - CordRep* rep = RawConcat(left, right); - if (rep != nullptr && !IsRootBalanced(rep)) { - rep = Rebalance(rep); - } - return VerifyTree(rep); -} - -// Make a balanced tree out of an array of leaf nodes. -static CordRep* MakeBalancedTree(CordRep** reps, size_t n) { - // Make repeated passes over the array, merging adjacent pairs - // until we are left with just a single node. - while (n > 1) { - size_t dst = 0; - for (size_t src = 0; src < n; src += 2) { - if (src + 1 < n) { - reps[dst] = Concat(reps[src], reps[src + 1]); - } else { - reps[dst] = reps[src]; - } - dst++; - } - n = dst; - } - - return reps[0]; -} - static CordRepFlat* CreateFlat(const char* data, size_t length, - size_t alloc_hint) { + size_t alloc_hint) { CordRepFlat* flat = CordRepFlat::New(length + alloc_hint); flat->length = length; memcpy(flat->Data(), data, length); return flat; } -// Creates a new flat or ringbuffer out of the specified array. +// Creates a new flat or Btree out of the specified array. // The returned node has a refcount of 1. -static CordRep* RingNewTree(const char* data, size_t length, - size_t alloc_hint) { +static CordRep* NewBtree(const char* data, size_t length, size_t alloc_hint) { if (length <= kMaxFlatLength) { return CreateFlat(data, length, alloc_hint); } CordRepFlat* flat = CreateFlat(data, kMaxFlatLength, 0); data += kMaxFlatLength; length -= kMaxFlatLength; - size_t extra = (length - 1) / kMaxFlatLength + 1; - auto* root = CordRepRing::Create(flat, extra); - return CordRepRing::Append(root, {data, length}, alloc_hint); + auto* root = CordRepBtree::Create(flat); + return CordRepBtree::Append(root, {data, length}, alloc_hint); } // Create a new tree out of the specified array. // The returned node has a refcount of 1. -static CordRep* NewTree(const char* data, - size_t length, - size_t alloc_hint) { +static CordRep* NewTree(const char* data, size_t length, size_t alloc_hint) { if (length == 0) return nullptr; - if (cord_ring_enabled()) { - return RingNewTree(data, length, alloc_hint); - } - absl::FixedArray<CordRep*> reps((length - 1) / kMaxFlatLength + 1); - size_t n = 0; - do { - const size_t len = std::min(length, kMaxFlatLength); - CordRepFlat* rep = CordRepFlat::New(len + alloc_hint); - rep->length = len; - memcpy(rep->Data(), data, len); - reps[n++] = VerifyTree(rep); - data += len; - length -= len; - } while (length != 0); - return MakeBalancedTree(reps.data(), n); + return NewBtree(data, length, alloc_hint); } namespace cord_internal { @@ -263,32 +128,46 @@ void InitializeCordRepExternal(absl::string_view data, CordRepExternal* rep) { } // namespace cord_internal -static CordRep* NewSubstring(CordRep* child, size_t offset, size_t length) { - // Never create empty substring nodes - if (length == 0) { - CordRep::Unref(child); - return nullptr; - } else { - CordRepSubstring* rep = new CordRepSubstring(); - assert((offset + length) <= child->length); - rep->length = length; - rep->tag = SUBSTRING; - rep->start = offset; - rep->child = child; - return VerifyTree(rep); +// Creates a CordRep from the provided string. If the string is large enough, +// and not wasteful, we move the string into an external cord rep, preserving +// the already allocated string contents. +// Requires the provided string length to be larger than `kMaxInline`. +static CordRep* CordRepFromString(std::string&& src) { + assert(src.length() > cord_internal::kMaxInline); + if ( + // String is short: copy data to avoid external block overhead. + src.size() <= kMaxBytesToCopy || + // String is wasteful: copy data to avoid pinning too much unused memory. + src.size() < src.capacity() / 2 + ) { + return NewTree(src.data(), src.size(), 0); } + + struct StringReleaser { + void operator()(absl::string_view /* data */) {} + std::string data; + }; + const absl::string_view original_data = src; + auto* rep = + static_cast<::absl::cord_internal::CordRepExternalImpl<StringReleaser>*>( + absl::cord_internal::NewExternalRep(original_data, + StringReleaser{std::move(src)})); + // Moving src may have invalidated its data pointer, so adjust it. + rep->base = rep->template get<0>().data.data(); + return rep; } // -------------------------------------------------------------------- // Cord::InlineRep functions +#ifdef ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL constexpr unsigned char Cord::InlineRep::kMaxInline; +#endif -inline void Cord::InlineRep::set_data(const char* data, size_t n, - bool nullify_tail) { +inline void Cord::InlineRep::set_data(const char* data, size_t n) { static_assert(kMaxInline == 15, "set_data is hard-coded for a length of 15"); - cord_internal::SmallMemmove(data_.as_chars(), data, n, nullify_tail); + cord_internal::SmallMemmove<true>(data_.as_chars(), data, n); set_inline_size(n); } @@ -299,20 +178,6 @@ inline char* Cord::InlineRep::set_data(size_t n) { return data_.as_chars(); } -inline CordRep* Cord::InlineRep::force_tree(size_t extra_hint) { - if (data_.is_tree()) { - return data_.as_tree(); - } - - size_t len = inline_size(); - CordRepFlat* result = CordRepFlat::New(len + extra_hint); - result->length = len; - static_assert(kMinFlatLength >= sizeof(data_), ""); - memcpy(result->Data(), data_.as_chars(), sizeof(data_)); - set_tree(result); - return result; -} - inline void Cord::InlineRep::reduce_size(size_t n) { size_t tag = inline_size(); assert(tag <= kMaxInline); @@ -328,31 +193,68 @@ inline void Cord::InlineRep::remove_prefix(size_t n) { reduce_size(n); } -// Returns `rep` converted into a CordRepRing. -// Directly returns `rep` if `rep` is already a CordRepRing. -static CordRepRing* ForceRing(CordRep* rep, size_t extra) { - return (rep->tag == RING) ? rep->ring() : CordRepRing::Create(rep, extra); +// Returns `rep` converted into a CordRepBtree. +// Directly returns `rep` if `rep` is already a CordRepBtree. +static CordRepBtree* ForceBtree(CordRep* rep) { + return rep->IsBtree() + ? rep->btree() + : CordRepBtree::Create(cord_internal::RemoveCrcNode(rep)); +} + +void Cord::InlineRep::AppendTreeToInlined(CordRep* tree, + MethodIdentifier method) { + assert(!is_tree()); + if (!data_.is_empty()) { + CordRepFlat* flat = MakeFlatWithExtraCapacity(0); + tree = CordRepBtree::Append(CordRepBtree::Create(flat), tree); + } + EmplaceTree(tree, method); +} + +void Cord::InlineRep::AppendTreeToTree(CordRep* tree, MethodIdentifier method) { + assert(is_tree()); + const CordzUpdateScope scope(data_.cordz_info(), method); + tree = CordRepBtree::Append(ForceBtree(data_.as_tree()), tree); + SetTree(tree, scope); } -void Cord::InlineRep::AppendTree(CordRep* tree) { - if (tree == nullptr) return; - if (data_.is_empty()) { - set_tree(tree); - } else if (cord_ring_enabled()) { - set_tree(CordRepRing::Append(ForceRing(force_tree(0), 1), tree)); +void Cord::InlineRep::AppendTree(CordRep* tree, MethodIdentifier method) { + assert(tree != nullptr); + assert(tree->length != 0); + assert(!tree->IsCrc()); + if (data_.is_tree()) { + AppendTreeToTree(tree, method); } else { - set_tree(Concat(force_tree(0), tree)); + AppendTreeToInlined(tree, method); } } -void Cord::InlineRep::PrependTree(CordRep* tree) { +void Cord::InlineRep::PrependTreeToInlined(CordRep* tree, + MethodIdentifier method) { + assert(!is_tree()); + if (!data_.is_empty()) { + CordRepFlat* flat = MakeFlatWithExtraCapacity(0); + tree = CordRepBtree::Prepend(CordRepBtree::Create(flat), tree); + } + EmplaceTree(tree, method); +} + +void Cord::InlineRep::PrependTreeToTree(CordRep* tree, + MethodIdentifier method) { + assert(is_tree()); + const CordzUpdateScope scope(data_.cordz_info(), method); + tree = CordRepBtree::Prepend(ForceBtree(data_.as_tree()), tree); + SetTree(tree, scope); +} + +void Cord::InlineRep::PrependTree(CordRep* tree, MethodIdentifier method) { assert(tree != nullptr); - if (data_.is_empty()) { - set_tree(tree); - } else if (cord_ring_enabled()) { - set_tree(CordRepRing::Prepend(ForceRing(force_tree(0), 1), tree)); + assert(tree->length != 0); + assert(!tree->IsCrc()); + if (data_.is_tree()) { + PrependTreeToTree(tree, method); } else { - set_tree(Concat(tree, force_tree(0))); + PrependTreeToInlined(tree, method); } } @@ -362,8 +264,8 @@ void Cord::InlineRep::PrependTree(CordRep* tree) { // written to region and the actual size increase will be written to size. static inline bool PrepareAppendRegion(CordRep* root, char** region, size_t* size, size_t max_length) { - if (root->tag == RING && root->refcount.IsOne()) { - Span<char> span = root->ring()->GetAppendBuffer(max_length); + if (root->IsBtree() && root->refcount.IsOne()) { + Span<char> span = root->btree()->GetAppendBuffer(max_length); if (!span.empty()) { *region = span.data(); *size = span.size(); @@ -371,13 +273,8 @@ static inline bool PrepareAppendRegion(CordRep* root, char** region, } } - // Search down the right-hand path for a non-full FLAT node. CordRep* dst = root; - while (dst->tag == CONCAT && dst->refcount.IsOne()) { - dst = dst->concat()->right; - } - - if (dst->tag < FLAT || !dst->refcount.IsOne()) { + if (!dst->IsFlat() || !dst->refcount.IsOne()) { *region = nullptr; *size = 0; return false; @@ -391,12 +288,7 @@ static inline bool PrepareAppendRegion(CordRep* root, char** region, return false; } - size_t size_increase = std::min(capacity - in_use, max_length); - - // We need to update the length fields for all nodes, including the leaf node. - for (CordRep* rep = root; rep != dst; rep = rep->concat()->right) { - rep->length += size_increase; - } + const size_t size_increase = std::min(capacity - in_use, max_length); dst->length += size_increase; *region = dst->flat()->Data() + in_use; @@ -404,148 +296,56 @@ static inline bool PrepareAppendRegion(CordRep* root, char** region, return true; } -void Cord::InlineRep::GetAppendRegion(char** region, size_t* size, - size_t max_length) { - if (max_length == 0) { - *region = nullptr; - *size = 0; - return; - } - - // Try to fit in the inline buffer if possible. - if (!is_tree()) { - size_t inline_length = inline_size(); - if (max_length <= kMaxInline - inline_length) { - *region = data_.as_chars() + inline_length; - *size = max_length; - set_inline_size(inline_length + max_length); - return; - } - } - - CordRep* root = force_tree(max_length); - - if (PrepareAppendRegion(root, region, size, max_length)) { - return; - } - - // Allocate new node. - CordRepFlat* new_node = - CordRepFlat::New(std::max(static_cast<size_t>(root->length), max_length)); - new_node->length = std::min(new_node->Capacity(), max_length); - *region = new_node->Data(); - *size = new_node->length; - - if (cord_ring_enabled()) { - replace_tree(CordRepRing::Append(ForceRing(root, 1), new_node)); - return; - } - replace_tree(Concat(root, new_node)); -} - -void Cord::InlineRep::GetAppendRegion(char** region, size_t* size) { - const size_t max_length = std::numeric_limits<size_t>::max(); - - // Try to fit in the inline buffer if possible. - if (!data_.is_tree()) { - size_t inline_length = inline_size(); - if (inline_length < kMaxInline) { - *region = data_.as_chars() + inline_length; - *size = kMaxInline - inline_length; - set_inline_size(kMaxInline); - return; - } - } - - CordRep* root = force_tree(max_length); - - if (PrepareAppendRegion(root, region, size, max_length)) { - return; - } - - // Allocate new node. - CordRepFlat* new_node = CordRepFlat::New(root->length); - new_node->length = new_node->Capacity(); - *region = new_node->Data(); - *size = new_node->length; - - if (cord_ring_enabled()) { - replace_tree(CordRepRing::Append(ForceRing(root, 1), new_node)); +void Cord::InlineRep::AssignSlow(const Cord::InlineRep& src) { + assert(&src != this); + assert(is_tree() || src.is_tree()); + auto constexpr method = CordzUpdateTracker::kAssignCord; + if (ABSL_PREDICT_TRUE(!is_tree())) { + EmplaceTree(CordRep::Ref(src.as_tree()), src.data_, method); return; } - replace_tree(Concat(root, new_node)); -} -// If the rep is a leaf, this will increment the value at total_mem_usage and -// will return true. -static bool RepMemoryUsageLeaf(const CordRep* rep, size_t* total_mem_usage) { - if (rep->tag >= FLAT) { - *total_mem_usage += rep->flat()->AllocatedSize(); - return true; - } - if (rep->tag == EXTERNAL) { - *total_mem_usage += sizeof(CordRepConcat) + rep->length; - return true; - } - return false; -} - -void Cord::InlineRep::AssignSlow(const Cord::InlineRep& src) { - ClearSlow(); - - data_ = src.data_; - if (is_tree()) { - data_.set_profiled(false); - CordRep::Ref(tree()); - clear_cordz_info(); + CordRep* tree = as_tree(); + if (CordRep* src_tree = src.tree()) { + // Leave any existing `cordz_info` in place, and let MaybeTrackCord() + // decide if this cord should be (or remains to be) sampled or not. + data_.set_tree(CordRep::Ref(src_tree)); + CordzInfo::MaybeTrackCord(data_, src.data_, method); + } else { + CordzInfo::MaybeUntrackCord(data_.cordz_info()); + data_ = src.data_; } + CordRep::Unref(tree); } -void Cord::InlineRep::ClearSlow() { +void Cord::InlineRep::UnrefTree() { if (is_tree()) { + CordzInfo::MaybeUntrackCord(data_.cordz_info()); CordRep::Unref(tree()); } - ResetToEmpty(); } // -------------------------------------------------------------------- // Constructors and destructors -Cord::Cord(absl::string_view src) { +Cord::Cord(absl::string_view src, MethodIdentifier method) + : contents_(InlineData::kDefaultInit) { const size_t n = src.size(); if (n <= InlineRep::kMaxInline) { - contents_.set_data(src.data(), n, false); + contents_.set_data(src.data(), n); } else { - contents_.set_tree(NewTree(src.data(), n, 0)); + CordRep* rep = NewTree(src.data(), n, 0); + contents_.EmplaceTree(rep, method); } } template <typename T, Cord::EnableIfString<T>> -Cord::Cord(T&& src) { - if ( - // String is short: copy data to avoid external block overhead. - src.size() <= kMaxBytesToCopy || - // String is wasteful: copy data to avoid pinning too much unused memory. - src.size() < src.capacity() / 2 - ) { - if (src.size() <= InlineRep::kMaxInline) { - contents_.set_data(src.data(), src.size(), false); - } else { - contents_.set_tree(NewTree(src.data(), src.size(), 0)); - } +Cord::Cord(T&& src) : contents_(InlineData::kDefaultInit) { + if (src.size() <= InlineRep::kMaxInline) { + contents_.set_data(src.data(), src.size()); } else { - struct StringReleaser { - void operator()(absl::string_view /* data */) {} - std::string data; - }; - const absl::string_view original_data = src; - auto* rep = static_cast< - ::absl::cord_internal::CordRepExternalImpl<StringReleaser>*>( - absl::cord_internal::NewExternalRep( - original_data, StringReleaser{std::forward<T>(src)})); - // Moving src may have invalidated its data pointer, so adjust it. - rep->base = rep->template get<0>().data.data(); - contents_.set_tree(rep); + CordRep* rep = CordRepFromString(std::forward<T>(src)); + contents_.EmplaceTree(rep, CordzUpdateTracker::kConstructorString); } } @@ -554,9 +354,9 @@ template Cord::Cord(std::string&& src); // The destruction code is separate so that the compiler can determine // that it does not need to call the destructor on a moved-from Cord. void Cord::DestroyCordSlow() { - if (CordRep* tree = contents_.tree()) { - CordRep::Unref(VerifyTree(tree)); - } + assert(contents_.is_tree()); + CordzInfo::MaybeUntrackCord(contents_.cordz_info()); + CordRep::Unref(VerifyTree(contents_.as_tree())); } // -------------------------------------------------------------------- @@ -568,109 +368,101 @@ void Cord::Clear() { } } -Cord& Cord::operator=(absl::string_view src) { +Cord& Cord::AssignLargeString(std::string&& src) { + auto constexpr method = CordzUpdateTracker::kAssignString; + assert(src.size() > kMaxBytesToCopy); + CordRep* rep = CordRepFromString(std::move(src)); + if (CordRep* tree = contents_.tree()) { + CordzUpdateScope scope(contents_.cordz_info(), method); + contents_.SetTree(rep, scope); + CordRep::Unref(tree); + } else { + contents_.EmplaceTree(rep, method); + } + return *this; +} +Cord& Cord::operator=(absl::string_view src) { + auto constexpr method = CordzUpdateTracker::kAssignString; const char* data = src.data(); size_t length = src.size(); CordRep* tree = contents_.tree(); if (length <= InlineRep::kMaxInline) { - // Embed into this->contents_ - contents_.set_data(data, length, true); - if (tree) CordRep::Unref(tree); - return *this; - } - if (tree != nullptr && tree->tag >= FLAT && - tree->flat()->Capacity() >= length && - tree->refcount.IsOne()) { - // Copy in place if the existing FLAT node is reusable. - memmove(tree->flat()->Data(), data, length); - tree->length = length; - VerifyTree(tree); + // Embed into this->contents_, which is somewhat subtle: + // - MaybeUntrackCord must be called before Unref(tree). + // - MaybeUntrackCord must be called before set_data() clobbers cordz_info. + // - set_data() must be called before Unref(tree) as it may reference tree. + if (tree != nullptr) CordzInfo::MaybeUntrackCord(contents_.cordz_info()); + contents_.set_data(data, length); + if (tree != nullptr) CordRep::Unref(tree); return *this; } - contents_.set_tree(NewTree(data, length, 0)); - if (tree) CordRep::Unref(tree); - return *this; -} - -template <typename T, Cord::EnableIfString<T>> -Cord& Cord::operator=(T&& src) { - if (src.size() <= kMaxBytesToCopy) { - *this = absl::string_view(src); + if (tree != nullptr) { + CordzUpdateScope scope(contents_.cordz_info(), method); + if (tree->IsFlat() && tree->flat()->Capacity() >= length && + tree->refcount.IsOne()) { + // Copy in place if the existing FLAT node is reusable. + memmove(tree->flat()->Data(), data, length); + tree->length = length; + VerifyTree(tree); + return *this; + } + contents_.SetTree(NewTree(data, length, 0), scope); + CordRep::Unref(tree); } else { - *this = Cord(std::forward<T>(src)); + contents_.EmplaceTree(NewTree(data, length, 0), method); } return *this; } -template Cord& Cord::operator=(std::string&& src); - // TODO(sanjay): Move to Cord::InlineRep section of file. For now, // we keep it here to make diffs easier. -void Cord::InlineRep::AppendArray(const char* src_data, size_t src_size) { - if (src_size == 0) return; // memcpy(_, nullptr, 0) is undefined. +void Cord::InlineRep::AppendArray(absl::string_view src, + MethodIdentifier method) { + if (src.empty()) return; // memcpy(_, nullptr, 0) is undefined. size_t appended = 0; - CordRep* root = nullptr; - if (is_tree()) { - root = data_.as_tree(); + CordRep* rep = tree(); + const CordRep* const root = rep; + CordzUpdateScope scope(root ? cordz_info() : nullptr, method); + if (root != nullptr) { + rep = cord_internal::RemoveCrcNode(rep); char* region; - if (PrepareAppendRegion(root, ®ion, &appended, src_size)) { - memcpy(region, src_data, appended); + if (PrepareAppendRegion(rep, ®ion, &appended, src.size())) { + memcpy(region, src.data(), appended); } } else { // Try to fit in the inline buffer if possible. size_t inline_length = inline_size(); - if (src_size <= kMaxInline - inline_length) { + if (src.size() <= kMaxInline - inline_length) { // Append new data to embedded array - memcpy(data_.as_chars() + inline_length, src_data, src_size); - set_inline_size(inline_length + src_size); + memcpy(data_.as_chars() + inline_length, src.data(), src.size()); + set_inline_size(inline_length + src.size()); return; } - // It is possible that src_data == data_, but when we transition from an - // InlineRep to a tree we need to assign data_ = root via set_tree. To - // avoid corrupting the source data before we copy it, delay calling - // set_tree until after we've copied data. - // We are going from an inline size to beyond inline size. Make the new size - // either double the inlined size, or the added size + 10%. - const size_t size1 = inline_length * 2 + src_size; - const size_t size2 = inline_length + src_size / 10; - root = CordRepFlat::New(std::max<size_t>(size1, size2)); - appended = std::min( - src_size, root->flat()->Capacity() - inline_length); - memcpy(root->flat()->Data(), data_.as_chars(), inline_length); - memcpy(root->flat()->Data() + inline_length, src_data, appended); - root->length = inline_length + appended; - set_tree(root); + // Allocate flat to be a perfect fit on first append exceeding inlined size. + // Subsequent growth will use amortized growth until we reach maximum flat + // size. + rep = CordRepFlat::New(inline_length + src.size()); + appended = std::min(src.size(), rep->flat()->Capacity() - inline_length); + memcpy(rep->flat()->Data(), data_.as_chars(), inline_length); + memcpy(rep->flat()->Data() + inline_length, src.data(), appended); + rep->length = inline_length + appended; } - src_data += appended; - src_size -= appended; - if (src_size == 0) { + src.remove_prefix(appended); + if (src.empty()) { + CommitTree(root, rep, scope, method); return; } - if (cord_ring_enabled()) { - absl::string_view data(src_data, src_size); - root = ForceRing(root, (data.size() - 1) / kMaxFlatLength + 1); - replace_tree(CordRepRing::Append(root->ring(), data)); - return; - } + // TODO(b/192061034): keep legacy 10% growth rate: consider other rates. + rep = ForceBtree(rep); + const size_t min_growth = std::max<size_t>(rep->length / 10, src.size()); + rep = CordRepBtree::Append(rep->btree(), src, min_growth - src.size()); - // Use new block(s) for any remaining bytes that were not handled above. - // Alloc extra memory only if the right child of the root of the new tree is - // going to be a FLAT node, which will permit further inplace appends. - size_t length = src_size; - if (src_size < kMaxFlatLength) { - // The new length is either - // - old size + 10% - // - old_size + src_size - // This will cause a reasonable conservative step-up in size that is still - // large enough to avoid excessive amounts of small fragments being added. - length = std::max<size_t>(root->length / 10, src_size); - } - set_tree(Concat(root, NewTree(src_data, src_size, length - src_size))); + CommitTree(root, rep, scope, method); } inline CordRep* Cord::TakeRep() const& { @@ -685,10 +477,18 @@ inline CordRep* Cord::TakeRep() && { template <typename C> inline void Cord::AppendImpl(C&& src) { + auto constexpr method = CordzUpdateTracker::kAppendCord; if (empty()) { - // In case of an empty destination avoid allocating a new node, do not copy - // data. - *this = std::forward<C>(src); + // Since destination is empty, we can avoid allocating a node, + if (src.contents_.is_tree()) { + // by taking the tree directly + CordRep* rep = + cord_internal::RemoveCrcNode(std::forward<C>(src).TakeRep()); + contents_.EmplaceTree(rep, method); + } else { + // or copying over inline data + contents_.data_ = src.contents_.data_; + } return; } @@ -698,12 +498,12 @@ inline void Cord::AppendImpl(C&& src) { CordRep* src_tree = src.contents_.tree(); if (src_tree == nullptr) { // src has embedded data. - contents_.AppendArray(src.contents_.data(), src_size); + contents_.AppendArray({src.contents_.data(), src_size}, method); return; } - if (src_tree->tag >= FLAT) { + if (src_tree->IsFlat()) { // src tree just has one flat node. - contents_.AppendArray(src_tree->flat()->Data(), src_size); + contents_.AppendArray({src_tree->flat()->Data(), src_size}, method); return; } if (&src == this) { @@ -719,19 +519,65 @@ inline void Cord::AppendImpl(C&& src) { } // Guaranteed to be a tree (kMaxBytesToCopy > kInlinedSize) - contents_.AppendTree(std::forward<C>(src).TakeRep()); + CordRep* rep = cord_internal::RemoveCrcNode(std::forward<C>(src).TakeRep()); + contents_.AppendTree(rep, CordzUpdateTracker::kAppendCord); +} + +static CordRep::ExtractResult ExtractAppendBuffer(CordRep* rep, + size_t min_capacity) { + switch (rep->tag) { + case cord_internal::BTREE: + return CordRepBtree::ExtractAppendBuffer(rep->btree(), min_capacity); + default: + if (rep->IsFlat() && rep->refcount.IsOne() && + rep->flat()->Capacity() - rep->length >= min_capacity) { + return {nullptr, rep}; + } + return {rep, nullptr}; + } } -void Cord::Append(const Cord& src) { AppendImpl(src); } +static CordBuffer CreateAppendBuffer(InlineData& data, size_t capacity) { + // Watch out for overflow, people can ask for size_t::max(). + const size_t size = data.inline_size(); + capacity = (std::min)(std::numeric_limits<size_t>::max() - size, capacity); + CordBuffer buffer = CordBuffer::CreateWithDefaultLimit(size + capacity); + cord_internal::SmallMemmove(buffer.data(), data.as_chars(), size); + buffer.SetLength(size); + data = {}; + return buffer; +} -void Cord::Append(Cord&& src) { AppendImpl(std::move(src)); } +CordBuffer Cord::GetAppendBufferSlowPath(size_t capacity, size_t min_capacity) { + auto constexpr method = CordzUpdateTracker::kGetAppendBuffer; + CordRep* tree = contents_.tree(); + if (tree != nullptr) { + CordzUpdateScope scope(contents_.cordz_info(), method); + CordRep::ExtractResult result = ExtractAppendBuffer(tree, min_capacity); + if (result.extracted != nullptr) { + contents_.SetTreeOrEmpty(result.tree, scope); + return CordBuffer(result.extracted->flat()); + } + return CordBuffer::CreateWithDefaultLimit(capacity); + } + return CreateAppendBuffer(contents_.data_, capacity); +} + +void Cord::Append(const Cord& src) { + AppendImpl(src); +} + +void Cord::Append(Cord&& src) { + AppendImpl(std::move(src)); +} template <typename T, Cord::EnableIfString<T>> void Cord::Append(T&& src) { if (src.size() <= kMaxBytesToCopy) { Append(absl::string_view(src)); } else { - Append(Cord(std::forward<T>(src))); + CordRep* rep = CordRepFromString(std::forward<T>(src)); + contents_.AppendTree(rep, CordzUpdateTracker::kAppendString); } } @@ -741,7 +587,8 @@ void Cord::Prepend(const Cord& src) { CordRep* src_tree = src.contents_.tree(); if (src_tree != nullptr) { CordRep::Ref(src_tree); - contents_.PrependTree(src_tree); + contents_.PrependTree(cord_internal::RemoveCrcNode(src_tree), + CordzUpdateTracker::kPrependCord); return; } @@ -750,7 +597,7 @@ void Cord::Prepend(const Cord& src) { return Prepend(src_contents); } -void Cord::Prepend(absl::string_view src) { +void Cord::PrependArray(absl::string_view src, MethodIdentifier method) { if (src.empty()) return; // memcpy(_, nullptr, 0) is undefined. if (!contents_.is_tree()) { size_t cur_size = contents_.inline_size(); @@ -764,105 +611,49 @@ void Cord::Prepend(absl::string_view src) { return; } } - contents_.PrependTree(NewTree(src.data(), src.size(), 0)); + CordRep* rep = NewTree(src.data(), src.size(), 0); + contents_.PrependTree(rep, method); } -template <typename T, Cord::EnableIfString<T>> -inline void Cord::Prepend(T&& src) { - if (src.size() <= kMaxBytesToCopy) { - Prepend(absl::string_view(src)); +void Cord::AppendPrecise(absl::string_view src, MethodIdentifier method) { + assert(!src.empty()); + assert(src.size() <= cord_internal::kMaxFlatLength); + if (contents_.remaining_inline_capacity() >= src.size()) { + const size_t inline_length = contents_.inline_size(); + memcpy(contents_.data_.as_chars() + inline_length, src.data(), src.size()); + contents_.set_inline_size(inline_length + src.size()); } else { - Prepend(Cord(std::forward<T>(src))); + contents_.AppendTree(CordRepFlat::Create(src), method); } } -template void Cord::Prepend(std::string&& src); - -static CordRep* RemovePrefixFrom(CordRep* node, size_t n) { - if (n >= node->length) return nullptr; - if (n == 0) return CordRep::Ref(node); - absl::InlinedVector<CordRep*, kInlinedVectorSize> rhs_stack; - - while (node->tag == CONCAT) { - assert(n <= node->length); - if (n < node->concat()->left->length) { - // Push right to stack, descend left. - rhs_stack.push_back(node->concat()->right); - node = node->concat()->left; - } else { - // Drop left, descend right. - n -= node->concat()->left->length; - node = node->concat()->right; - } - } - assert(n <= node->length); - - if (n == 0) { - CordRep::Ref(node); +void Cord::PrependPrecise(absl::string_view src, MethodIdentifier method) { + assert(!src.empty()); + assert(src.size() <= cord_internal::kMaxFlatLength); + if (contents_.remaining_inline_capacity() >= src.size()) { + const size_t inline_length = contents_.inline_size(); + char data[InlineRep::kMaxInline + 1] = {0}; + memcpy(data, src.data(), src.size()); + memcpy(data + src.size(), contents_.data(), inline_length); + memcpy(contents_.data_.as_chars(), data, InlineRep::kMaxInline + 1); + contents_.set_inline_size(inline_length + src.size()); } else { - size_t start = n; - size_t len = node->length - n; - if (node->tag == SUBSTRING) { - // Consider in-place update of node, similar to in RemoveSuffixFrom(). - start += node->substring()->start; - node = node->substring()->child; - } - node = NewSubstring(CordRep::Ref(node), start, len); - } - while (!rhs_stack.empty()) { - node = Concat(node, CordRep::Ref(rhs_stack.back())); - rhs_stack.pop_back(); + contents_.PrependTree(CordRepFlat::Create(src), method); } - return node; } -// RemoveSuffixFrom() is very similar to RemovePrefixFrom(), with the -// exception that removing a suffix has an optimization where a node may be -// edited in place iff that node and all its ancestors have a refcount of 1. -static CordRep* RemoveSuffixFrom(CordRep* node, size_t n) { - if (n >= node->length) return nullptr; - if (n == 0) return CordRep::Ref(node); - absl::InlinedVector<CordRep*, kInlinedVectorSize> lhs_stack; - bool inplace_ok = node->refcount.IsOne(); - - while (node->tag == CONCAT) { - assert(n <= node->length); - if (n < node->concat()->right->length) { - // Push left to stack, descend right. - lhs_stack.push_back(node->concat()->left); - node = node->concat()->right; - } else { - // Drop right, descend left. - n -= node->concat()->right->length; - node = node->concat()->left; - } - inplace_ok = inplace_ok && node->refcount.IsOne(); - } - assert(n <= node->length); - - if (n == 0) { - CordRep::Ref(node); - } else if (inplace_ok && node->tag != EXTERNAL) { - // Consider making a new buffer if the current node capacity is much - // larger than the new length. - CordRep::Ref(node); - node->length -= n; +template <typename T, Cord::EnableIfString<T>> +inline void Cord::Prepend(T&& src) { + if (src.size() <= kMaxBytesToCopy) { + Prepend(absl::string_view(src)); } else { - size_t start = 0; - size_t len = node->length - n; - if (node->tag == SUBSTRING) { - start = node->substring()->start; - node = node->substring()->child; - } - node = NewSubstring(CordRep::Ref(node), start, len); + CordRep* rep = CordRepFromString(std::forward<T>(src)); + contents_.PrependTree(rep, CordzUpdateTracker::kPrependString); } - while (!lhs_stack.empty()) { - node = Concat(CordRep::Ref(lhs_stack.back()), node); - lhs_stack.pop_back(); - } - return node; } +template void Cord::Prepend(std::string&& src); + void Cord::RemovePrefix(size_t n) { ABSL_INTERNAL_CHECK(n <= size(), absl::StrCat("Requested prefix size ", n, @@ -870,12 +661,26 @@ void Cord::RemovePrefix(size_t n) { CordRep* tree = contents_.tree(); if (tree == nullptr) { contents_.remove_prefix(n); - } else if (tree->tag == RING) { - contents_.replace_tree(CordRepRing::RemovePrefix(tree->ring(), n)); } else { - CordRep* newrep = RemovePrefixFrom(tree, n); - CordRep::Unref(tree); - contents_.replace_tree(VerifyTree(newrep)); + auto constexpr method = CordzUpdateTracker::kRemovePrefix; + CordzUpdateScope scope(contents_.cordz_info(), method); + tree = cord_internal::RemoveCrcNode(tree); + if (n >= tree->length) { + CordRep::Unref(tree); + tree = nullptr; + } else if (tree->IsBtree()) { + CordRep* old = tree; + tree = tree->btree()->SubTree(n, tree->length - n); + CordRep::Unref(old); + } else if (tree->IsSubstring() && tree->refcount.IsOne()) { + tree->substring()->start += n; + tree->length -= n; + } else { + CordRep* rep = CordRepSubstring::Substring(tree, n, tree->length - n); + CordRep::Unref(tree); + tree = rep; + } + contents_.SetTreeOrEmpty(tree, scope); } } @@ -886,64 +691,25 @@ void Cord::RemoveSuffix(size_t n) { CordRep* tree = contents_.tree(); if (tree == nullptr) { contents_.reduce_size(n); - } else if (tree->tag == RING) { - contents_.replace_tree(CordRepRing::RemoveSuffix(tree->ring(), n)); } else { - CordRep* newrep = RemoveSuffixFrom(tree, n); - CordRep::Unref(tree); - contents_.replace_tree(VerifyTree(newrep)); - } -} - -// Work item for NewSubRange(). -struct SubRange { - SubRange(CordRep* a_node, size_t a_pos, size_t a_n) - : node(a_node), pos(a_pos), n(a_n) {} - CordRep* node; // nullptr means concat last 2 results. - size_t pos; - size_t n; -}; - -static CordRep* NewSubRange(CordRep* node, size_t pos, size_t n) { - absl::InlinedVector<CordRep*, kInlinedVectorSize> results; - absl::InlinedVector<SubRange, kInlinedVectorSize> todo; - todo.push_back(SubRange(node, pos, n)); - do { - const SubRange& sr = todo.back(); - node = sr.node; - pos = sr.pos; - n = sr.n; - todo.pop_back(); - - if (node == nullptr) { - assert(results.size() >= 2); - CordRep* right = results.back(); - results.pop_back(); - CordRep* left = results.back(); - results.pop_back(); - results.push_back(Concat(left, right)); - } else if (pos == 0 && n == node->length) { - results.push_back(CordRep::Ref(node)); - } else if (node->tag != CONCAT) { - if (node->tag == SUBSTRING) { - pos += node->substring()->start; - node = node->substring()->child; - } - results.push_back(NewSubstring(CordRep::Ref(node), pos, n)); - } else if (pos + n <= node->concat()->left->length) { - todo.push_back(SubRange(node->concat()->left, pos, n)); - } else if (pos >= node->concat()->left->length) { - pos -= node->concat()->left->length; - todo.push_back(SubRange(node->concat()->right, pos, n)); + auto constexpr method = CordzUpdateTracker::kRemoveSuffix; + CordzUpdateScope scope(contents_.cordz_info(), method); + tree = cord_internal::RemoveCrcNode(tree); + if (n >= tree->length) { + CordRep::Unref(tree); + tree = nullptr; + } else if (tree->IsBtree()) { + tree = CordRepBtree::RemoveSuffix(tree->btree(), n); + } else if (!tree->IsExternal() && tree->refcount.IsOne()) { + assert(tree->IsFlat() || tree->IsSubstring()); + tree->length -= n; } else { - size_t left_n = node->concat()->left->length - pos; - todo.push_back(SubRange(nullptr, 0, 0)); // Concat() - todo.push_back(SubRange(node->concat()->right, 0, n - left_n)); - todo.push_back(SubRange(node->concat()->left, pos, left_n)); + CordRep* rep = CordRepSubstring::Substring(tree, 0, tree->length - n); + CordRep::Unref(tree); + tree = rep; } - } while (!todo.empty()); - assert(results.size() == 1); - return results[0]; + contents_.SetTreeOrEmpty(tree, scope); + } } Cord Cord::Subcord(size_t pos, size_t new_size) const { @@ -951,17 +717,18 @@ Cord Cord::Subcord(size_t pos, size_t new_size) const { size_t length = size(); if (pos > length) pos = length; if (new_size > length - pos) new_size = length - pos; + if (new_size == 0) return sub_cord; + CordRep* tree = contents_.tree(); if (tree == nullptr) { - // sub_cord is newly constructed, no need to re-zero-out the tail of - // contents_ memory. - sub_cord.contents_.set_data(contents_.data() + pos, new_size, false); - } else if (new_size == 0) { - // We want to return empty subcord, so nothing to do. - } else if (new_size <= InlineRep::kMaxInline) { + sub_cord.contents_.set_data(contents_.data() + pos, new_size); + return sub_cord; + } + + if (new_size <= InlineRep::kMaxInline) { + char* dest = sub_cord.contents_.data_.as_chars(); Cord::ChunkIterator it = chunk_begin(); it.AdvanceBytes(pos); - char* dest = sub_cord.contents_.data_.as_chars(); size_t remaining_size = new_size; while (remaining_size > it->size()) { cord_internal::SmallMemmove(dest, it->data(), it->size()); @@ -971,153 +738,18 @@ Cord Cord::Subcord(size_t pos, size_t new_size) const { } cord_internal::SmallMemmove(dest, it->data(), remaining_size); sub_cord.contents_.set_inline_size(new_size); - } else if (tree->tag == RING) { - tree = CordRepRing::SubRing(CordRep::Ref(tree)->ring(), pos, new_size); - sub_cord.contents_.set_tree(tree); - } else { - sub_cord.contents_.set_tree(NewSubRange(tree, pos, new_size)); - } - return sub_cord; -} - -// -------------------------------------------------------------------- -// Balancing - -class CordForest { - public: - explicit CordForest(size_t length) - : root_length_(length), trees_(kMinLengthSize, nullptr) {} - - void Build(CordRep* cord_root) { - std::vector<CordRep*> pending = {cord_root}; - - while (!pending.empty()) { - CordRep* node = pending.back(); - pending.pop_back(); - CheckNode(node); - if (ABSL_PREDICT_FALSE(node->tag != CONCAT)) { - AddNode(node); - continue; - } - - CordRepConcat* concat_node = node->concat(); - if (concat_node->depth() >= kMinLengthSize || - concat_node->length < min_length[concat_node->depth()]) { - pending.push_back(concat_node->right); - pending.push_back(concat_node->left); - - if (concat_node->refcount.IsOne()) { - concat_node->left = concat_freelist_; - concat_freelist_ = concat_node; - } else { - CordRep::Ref(concat_node->right); - CordRep::Ref(concat_node->left); - CordRep::Unref(concat_node); - } - } else { - AddNode(node); - } - } - } - - CordRep* ConcatNodes() { - CordRep* sum = nullptr; - for (auto* node : trees_) { - if (node == nullptr) continue; - - sum = PrependNode(node, sum); - root_length_ -= node->length; - if (root_length_ == 0) break; - } - ABSL_INTERNAL_CHECK(sum != nullptr, "Failed to locate sum node"); - return VerifyTree(sum); - } - - private: - CordRep* AppendNode(CordRep* node, CordRep* sum) { - return (sum == nullptr) ? node : MakeConcat(sum, node); - } - - CordRep* PrependNode(CordRep* node, CordRep* sum) { - return (sum == nullptr) ? node : MakeConcat(node, sum); - } - - void AddNode(CordRep* node) { - CordRep* sum = nullptr; - - // Collect together everything with which we will merge with node - int i = 0; - for (; node->length > min_length[i + 1]; ++i) { - auto& tree_at_i = trees_[i]; - - if (tree_at_i == nullptr) continue; - sum = PrependNode(tree_at_i, sum); - tree_at_i = nullptr; - } - - sum = AppendNode(node, sum); - - // Insert sum into appropriate place in the forest - for (; sum->length >= min_length[i]; ++i) { - auto& tree_at_i = trees_[i]; - if (tree_at_i == nullptr) continue; - - sum = MakeConcat(tree_at_i, sum); - tree_at_i = nullptr; - } - - // min_length[0] == 1, which means sum->length >= min_length[0] - assert(i > 0); - trees_[i - 1] = sum; - } - - // Make concat node trying to resue existing CordRepConcat nodes we - // already collected in the concat_freelist_. - CordRep* MakeConcat(CordRep* left, CordRep* right) { - if (concat_freelist_ == nullptr) return RawConcat(left, right); - - CordRepConcat* rep = concat_freelist_; - if (concat_freelist_->left == nullptr) { - concat_freelist_ = nullptr; - } else { - concat_freelist_ = concat_freelist_->left->concat(); - } - SetConcatChildren(rep, left, right); - - return rep; + return sub_cord; } - static void CheckNode(CordRep* node) { - ABSL_INTERNAL_CHECK(node->length != 0u, ""); - if (node->tag == CONCAT) { - ABSL_INTERNAL_CHECK(node->concat()->left != nullptr, ""); - ABSL_INTERNAL_CHECK(node->concat()->right != nullptr, ""); - ABSL_INTERNAL_CHECK(node->length == (node->concat()->left->length + - node->concat()->right->length), - ""); - } - } - - size_t root_length_; - - // use an inlined vector instead of a flat array to get bounds checking - absl::InlinedVector<CordRep*, kInlinedVectorSize> trees_; - - // List of concat nodes we can re-use for Cord balancing. - CordRepConcat* concat_freelist_ = nullptr; -}; - -static CordRep* Rebalance(CordRep* node) { - VerifyTree(node); - assert(node->tag == CONCAT); - - if (node->length == 0) { - return nullptr; + tree = cord_internal::SkipCrcNode(tree); + if (tree->IsBtree()) { + tree = tree->btree()->SubTree(pos, new_size); + } else { + tree = CordRepSubstring::Substring(tree, pos, new_size); } - - CordForest forest(node->length); - forest.Build(node); - return forest.ConcatNodes(); + sub_cord.contents_.EmplaceTree(tree, contents_.data_, + CordzUpdateTracker::kSubCord); + return sub_cord; } // -------------------------------------------------------------------- @@ -1159,29 +791,29 @@ bool ComputeCompareResult<bool>(int memcmp_res) { } // namespace -// Helper routine. Locates the first flat chunk of the Cord without -// initializing the iterator. +// Helper routine. Locates the first flat or external chunk of the Cord without +// initializing the iterator, and returns a string_view referencing the data. inline absl::string_view Cord::InlineRep::FindFlatStartPiece() const { if (!is_tree()) { return absl::string_view(data_.as_chars(), data_.inline_size()); } - CordRep* node = tree(); - if (node->tag >= FLAT) { + CordRep* node = cord_internal::SkipCrcNode(tree()); + if (node->IsFlat()) { return absl::string_view(node->flat()->Data(), node->length); } - if (node->tag == EXTERNAL) { + if (node->IsExternal()) { return absl::string_view(node->external()->base, node->length); } - if (node->tag == RING) { - return node->ring()->entry_data(node->ring()->head()); - } - - // Walk down the left branches until we hit a non-CONCAT node. - while (node->tag == CONCAT) { - node = node->concat()->left; + if (node->IsBtree()) { + CordRepBtree* tree = node->btree(); + int height = tree->height(); + while (--height >= 0) { + tree = tree->Edge(CordRepBtree::kFront)->btree(); + } + return tree->Data(tree->begin()); } // Get the child node if we encounter a SUBSTRING. @@ -1189,20 +821,42 @@ inline absl::string_view Cord::InlineRep::FindFlatStartPiece() const { size_t length = node->length; assert(length != 0); - if (node->tag == SUBSTRING) { + if (node->IsSubstring()) { offset = node->substring()->start; node = node->substring()->child; } - if (node->tag >= FLAT) { + if (node->IsFlat()) { return absl::string_view(node->flat()->Data() + offset, length); } - assert((node->tag == EXTERNAL) && "Expect FLAT or EXTERNAL node here"); + assert(node->IsExternal() && "Expect FLAT or EXTERNAL node here"); return absl::string_view(node->external()->base + offset, length); } +void Cord::SetExpectedChecksum(uint32_t crc) { + auto constexpr method = CordzUpdateTracker::kSetExpectedChecksum; + if (empty()) return; + + if (!contents_.is_tree()) { + CordRep* rep = contents_.MakeFlatWithExtraCapacity(0); + rep = CordRepCrc::New(rep, crc); + contents_.EmplaceTree(rep, method); + } else { + const CordzUpdateScope scope(contents_.data_.cordz_info(), method); + CordRep* rep = CordRepCrc::New(contents_.data_.as_tree(), crc); + contents_.SetTree(rep, scope); + } +} + +absl::optional<uint32_t> Cord::ExpectedChecksum() const { + if (!contents_.is_tree() || !contents_.tree()->IsCrc()) { + return absl::nullopt; + } + return contents_.tree()->crc()->crc; +} + inline int Cord::CompareSlowPath(absl::string_view rhs, size_t compared_size, size_t size_to_compare) const { auto advance = [](Cord::ChunkIterator* it, absl::string_view* chunk) { @@ -1378,46 +1032,11 @@ void Cord::CopyToArraySlowPath(char* dst) const { } } -Cord::ChunkIterator& Cord::ChunkIterator::AdvanceStack() { - auto& stack_of_right_children = stack_of_right_children_; - if (stack_of_right_children.empty()) { - assert(!current_chunk_.empty()); // Called on invalid iterator. - // We have reached the end of the Cord. - return *this; - } - - // Process the next node on the stack. - CordRep* node = stack_of_right_children.back(); - stack_of_right_children.pop_back(); - - // Walk down the left branches until we hit a non-CONCAT node. Save the - // right children to the stack for subsequent traversal. - while (node->tag == CONCAT) { - stack_of_right_children.push_back(node->concat()->right); - node = node->concat()->left; - } - - // Get the child node if we encounter a SUBSTRING. - size_t offset = 0; - size_t length = node->length; - if (node->tag == SUBSTRING) { - offset = node->substring()->start; - node = node->substring()->child; - } - - assert(node->tag == EXTERNAL || node->tag >= FLAT); - assert(length != 0); - const char* data = - node->tag == EXTERNAL ? node->external()->base : node->flat()->Data(); - current_chunk_ = absl::string_view(data + offset, length); - current_leaf_ = node; - return *this; -} - Cord Cord::ChunkIterator::AdvanceAndReadBytes(size_t n) { ABSL_HARDENING_ASSERT(bytes_remaining_ >= n && "Attempted to iterate past `end()`"); Cord subcord; + auto constexpr method = CordzUpdateTracker::kCordReader; if (n <= InlineRep::kMaxInline) { // Range to read fits in inline data. Flatten it. @@ -1437,184 +1056,51 @@ Cord Cord::ChunkIterator::AdvanceAndReadBytes(size_t n) { return subcord; } - if (ring_reader_) { + if (btree_reader_) { size_t chunk_size = current_chunk_.size(); if (n <= chunk_size && n <= kMaxBytesToCopy) { - subcord = Cord(current_chunk_.substr(0, n)); - } else { - auto* ring = CordRep::Ref(ring_reader_.ring())->ring(); - size_t offset = ring_reader_.length() - bytes_remaining_; - subcord.contents_.set_tree(CordRepRing::SubRing(ring, offset, n)); - } - if (n < chunk_size) { - bytes_remaining_ -= n; - current_chunk_.remove_prefix(n); + subcord = Cord(current_chunk_.substr(0, n), method); + if (n < chunk_size) { + current_chunk_.remove_prefix(n); + } else { + current_chunk_ = btree_reader_.Next(); + } } else { - AdvanceBytesRing(n); + CordRep* rep; + current_chunk_ = btree_reader_.Read(n, chunk_size, rep); + subcord.contents_.EmplaceTree(rep, method); } + bytes_remaining_ -= n; return subcord; } - auto& stack_of_right_children = stack_of_right_children_; - if (n < current_chunk_.size()) { - // Range to read is a proper subrange of the current chunk. - assert(current_leaf_ != nullptr); - CordRep* subnode = CordRep::Ref(current_leaf_); - const char* data = subnode->tag == EXTERNAL ? subnode->external()->base - : subnode->flat()->Data(); - subnode = NewSubstring(subnode, current_chunk_.data() - data, n); - subcord.contents_.set_tree(VerifyTree(subnode)); - RemoveChunkPrefix(n); - return subcord; - } - - // Range to read begins with a proper subrange of the current chunk. - assert(!current_chunk_.empty()); + // Short circuit if reading the entire data edge. assert(current_leaf_ != nullptr); - CordRep* subnode = CordRep::Ref(current_leaf_); - if (current_chunk_.size() < subnode->length) { - const char* data = subnode->tag == EXTERNAL ? subnode->external()->base - : subnode->flat()->Data(); - subnode = NewSubstring(subnode, current_chunk_.data() - data, - current_chunk_.size()); - } - n -= current_chunk_.size(); - bytes_remaining_ -= current_chunk_.size(); - - // Process the next node(s) on the stack, reading whole subtrees depending on - // their length and how many bytes we are advancing. - CordRep* node = nullptr; - while (!stack_of_right_children.empty()) { - node = stack_of_right_children.back(); - stack_of_right_children.pop_back(); - if (node->length > n) break; - // TODO(qrczak): This might unnecessarily recreate existing concat nodes. - // Avoiding that would need pretty complicated logic (instead of - // current_leaf, keep current_subtree_ which points to the highest node - // such that the current leaf can be found on the path of left children - // starting from current_subtree_; delay creating subnode while node is - // below current_subtree_; find the proper node along the path of left - // children starting from current_subtree_ if this loop exits while staying - // below current_subtree_; etc.; alternatively, push parents instead of - // right children on the stack). - subnode = Concat(subnode, CordRep::Ref(node)); - n -= node->length; - bytes_remaining_ -= node->length; - node = nullptr; - } - - if (node == nullptr) { - // We have reached the end of the Cord. - assert(bytes_remaining_ == 0); - subcord.contents_.set_tree(VerifyTree(subnode)); + if (n == current_leaf_->length) { + bytes_remaining_ = 0; + current_chunk_ = {}; + CordRep* tree = CordRep::Ref(current_leaf_); + subcord.contents_.EmplaceTree(VerifyTree(tree), method); return subcord; } - // Walk down the appropriate branches until we hit a non-CONCAT node. Save the - // right children to the stack for subsequent traversal. - while (node->tag == CONCAT) { - if (node->concat()->left->length > n) { - // Push right, descend left. - stack_of_right_children.push_back(node->concat()->right); - node = node->concat()->left; - } else { - // Read left, descend right. - subnode = Concat(subnode, CordRep::Ref(node->concat()->left)); - n -= node->concat()->left->length; - bytes_remaining_ -= node->concat()->left->length; - node = node->concat()->right; - } - } + // From this point on, we need a partial substring node. + // Get pointer to the underlying flat or external data payload and + // compute data pointer and offset into current flat or external. + CordRep* payload = current_leaf_->IsSubstring() + ? current_leaf_->substring()->child + : current_leaf_; + const char* data = payload->IsExternal() ? payload->external()->base + : payload->flat()->Data(); + const size_t offset = current_chunk_.data() - data; - // Get the child node if we encounter a SUBSTRING. - size_t offset = 0; - size_t length = node->length; - if (node->tag == SUBSTRING) { - offset = node->substring()->start; - node = node->substring()->child; - } - - // Range to read ends with a proper (possibly empty) subrange of the current - // chunk. - assert(node->tag == EXTERNAL || node->tag >= FLAT); - assert(length > n); - if (n > 0) { - subnode = Concat(subnode, NewSubstring(CordRep::Ref(node), offset, n)); - } - const char* data = - node->tag == EXTERNAL ? node->external()->base : node->flat()->Data(); - current_chunk_ = absl::string_view(data + offset + n, length - n); - current_leaf_ = node; + auto* tree = CordRepSubstring::Substring(payload, offset, n); + subcord.contents_.EmplaceTree(VerifyTree(tree), method); bytes_remaining_ -= n; - subcord.contents_.set_tree(VerifyTree(subnode)); + current_chunk_.remove_prefix(n); return subcord; } -void Cord::ChunkIterator::AdvanceBytesSlowPath(size_t n) { - assert(bytes_remaining_ >= n && "Attempted to iterate past `end()`"); - assert(n >= current_chunk_.size()); // This should only be called when - // iterating to a new node. - - n -= current_chunk_.size(); - bytes_remaining_ -= current_chunk_.size(); - - if (stack_of_right_children_.empty()) { - // We have reached the end of the Cord. - assert(bytes_remaining_ == 0); - return; - } - - // Process the next node(s) on the stack, skipping whole subtrees depending on - // their length and how many bytes we are advancing. - CordRep* node = nullptr; - auto& stack_of_right_children = stack_of_right_children_; - while (!stack_of_right_children.empty()) { - node = stack_of_right_children.back(); - stack_of_right_children.pop_back(); - if (node->length > n) break; - n -= node->length; - bytes_remaining_ -= node->length; - node = nullptr; - } - - if (node == nullptr) { - // We have reached the end of the Cord. - assert(bytes_remaining_ == 0); - return; - } - - // Walk down the appropriate branches until we hit a non-CONCAT node. Save the - // right children to the stack for subsequent traversal. - while (node->tag == CONCAT) { - if (node->concat()->left->length > n) { - // Push right, descend left. - stack_of_right_children.push_back(node->concat()->right); - node = node->concat()->left; - } else { - // Skip left, descend right. - n -= node->concat()->left->length; - bytes_remaining_ -= node->concat()->left->length; - node = node->concat()->right; - } - } - - // Get the child node if we encounter a SUBSTRING. - size_t offset = 0; - size_t length = node->length; - if (node->tag == SUBSTRING) { - offset = node->substring()->start; - node = node->substring()->child; - } - - assert(node->tag == EXTERNAL || node->tag >= FLAT); - assert(length > n); - const char* data = - node->tag == EXTERNAL ? node->external()->base : node->flat()->Data(); - current_chunk_ = absl::string_view(data + offset + n, length - n); - current_leaf_ = node; - bytes_remaining_ -= n; -} - char Cord::operator[](size_t i) const { ABSL_HARDENING_ASSERT(i < size()); size_t offset = i; @@ -1622,30 +1108,21 @@ char Cord::operator[](size_t i) const { if (rep == nullptr) { return contents_.data()[i]; } + rep = cord_internal::SkipCrcNode(rep); while (true) { assert(rep != nullptr); assert(offset < rep->length); - if (rep->tag >= FLAT) { + if (rep->IsFlat()) { // Get the "i"th character directly from the flat array. return rep->flat()->Data()[offset]; - } else if (rep->tag == RING) { - return rep->ring()->GetCharacter(offset); - } else if (rep->tag == EXTERNAL) { + } else if (rep->IsBtree()) { + return rep->btree()->GetCharacter(offset); + } else if (rep->IsExternal()) { // Get the "i"th character from the external array. return rep->external()->base[offset]; - } else if (rep->tag == CONCAT) { - // Recursively branch to the side of the concatenation that the "i"th - // character is on. - size_t left_length = rep->concat()->left->length; - if (offset < left_length) { - rep = rep->concat()->left; - } else { - offset -= left_length; - rep = rep->concat()->right; - } } else { // This must be a substring a node, so bypass it to get to the child. - assert(rep->tag == SUBSTRING); + assert(rep->IsSubstring()); offset += rep->substring()->start; rep = rep->substring()->child; } @@ -1653,6 +1130,7 @@ char Cord::operator[](size_t i) const { } absl::string_view Cord::FlattenSlowPath() { + assert(contents_.is_tree()); size_t total_size = size(); CordRep* new_rep; char* new_buffer; @@ -1673,31 +1151,36 @@ absl::string_view Cord::FlattenSlowPath() { s.size()); }); } - if (CordRep* tree = contents_.tree()) { - CordRep::Unref(tree); - } - contents_.set_tree(new_rep); + CordzUpdateScope scope(contents_.cordz_info(), CordzUpdateTracker::kFlatten); + CordRep::Unref(contents_.as_tree()); + contents_.SetTree(new_rep, scope); return absl::string_view(new_buffer, total_size); } /* static */ bool Cord::GetFlatAux(CordRep* rep, absl::string_view* fragment) { assert(rep != nullptr); - if (rep->tag >= FLAT) { + rep = cord_internal::SkipCrcNode(rep); + if (rep->IsFlat()) { *fragment = absl::string_view(rep->flat()->Data(), rep->length); return true; - } else if (rep->tag == EXTERNAL) { + } else if (rep->IsExternal()) { *fragment = absl::string_view(rep->external()->base, rep->length); return true; - } else if (rep->tag == SUBSTRING) { + } else if (rep->IsBtree()) { + return rep->btree()->IsFlat(fragment); + } else if (rep->IsSubstring()) { CordRep* child = rep->substring()->child; - if (child->tag >= FLAT) { + if (child->IsFlat()) { *fragment = absl::string_view( child->flat()->Data() + rep->substring()->start, rep->length); return true; - } else if (child->tag == EXTERNAL) { + } else if (child->IsExternal()) { *fragment = absl::string_view( child->external()->base + rep->substring()->start, rep->length); return true; + } else if (child->IsBtree()) { + return child->btree()->IsFlat(rep->substring()->start, rep->length, + fragment); } } return false; @@ -1706,7 +1189,10 @@ absl::string_view Cord::FlattenSlowPath() { /* static */ void Cord::ForEachChunkAux( absl::cord_internal::CordRep* rep, absl::FunctionRef<void(absl::string_view)> callback) { - if (rep->tag == RING) { + assert(rep != nullptr); + rep = cord_internal::SkipCrcNode(rep); + + if (rep->IsBtree()) { ChunkIterator it(rep), end; while (it != end) { callback(*it); @@ -1715,44 +1201,13 @@ absl::string_view Cord::FlattenSlowPath() { return; } - assert(rep != nullptr); - int stack_pos = 0; - constexpr int stack_max = 128; - // Stack of right branches for tree traversal - absl::cord_internal::CordRep* stack[stack_max]; - absl::cord_internal::CordRep* current_node = rep; - while (true) { - if (current_node->tag == CONCAT) { - if (stack_pos == stack_max) { - // There's no more room on our stack array to add another right branch, - // and the idea is to avoid allocations, so call this function - // recursively to navigate this subtree further. (This is not something - // we expect to happen in practice). - ForEachChunkAux(current_node, callback); - - // Pop the next right branch and iterate. - current_node = stack[--stack_pos]; - continue; - } else { - // Save the right branch for later traversal and continue down the left - // branch. - stack[stack_pos++] = current_node->concat()->right; - current_node = current_node->concat()->left; - continue; - } - } - // This is a leaf node, so invoke our callback. - absl::string_view chunk; - bool success = GetFlatAux(current_node, &chunk); - assert(success); - if (success) { - callback(chunk); - } - if (stack_pos == 0) { - // end of traversal - return; - } - current_node = stack[--stack_pos]; + // This is a leaf node, so invoke our callback. + absl::cord_internal::CordRep* current_node = cord_internal::SkipCrcNode(rep); + absl::string_view chunk; + bool success = GetFlatAux(current_node, &chunk); + assert(success); + if (success) { + callback(chunk); } } @@ -1767,40 +1222,28 @@ static void DumpNode(CordRep* rep, bool include_data, std::ostream* os, *os << " ["; if (include_data) *os << static_cast<void*>(rep); *os << "]"; - *os << " " << (IsRootBalanced(rep) ? 'b' : 'u'); *os << " " << std::setw(indent) << ""; - if (rep->tag == CONCAT) { - *os << "CONCAT depth=" << Depth(rep) << "\n"; + if (rep->IsCrc()) { + *os << "CRC crc=" << rep->crc()->crc << "\n"; indent += kIndentStep; - indents.push_back(indent); - stack.push_back(rep->concat()->right); - rep = rep->concat()->left; - } else if (rep->tag == SUBSTRING) { + rep = rep->crc()->child; + } else if (rep->IsSubstring()) { *os << "SUBSTRING @ " << rep->substring()->start << "\n"; indent += kIndentStep; rep = rep->substring()->child; } else { // Leaf or ring - if (rep->tag == EXTERNAL) { + if (rep->IsExternal()) { *os << "EXTERNAL ["; if (include_data) *os << absl::CEscape(std::string(rep->external()->base, rep->length)); *os << "]\n"; - } else if (rep->tag >= FLAT) { - *os << "FLAT cap=" << rep->flat()->Capacity() - << " ["; + } else if (rep->IsFlat()) { + *os << "FLAT cap=" << rep->flat()->Capacity() << " ["; if (include_data) *os << absl::CEscape(std::string(rep->flat()->Data(), rep->length)); *os << "]\n"; } else { - assert(rep->tag == RING); - auto* ring = rep->ring(); - *os << "RING, entries = " << ring->entries() << "\n"; - CordRepRing::index_type head = ring->head(); - do { - DumpNode(ring->entry_child(head), include_data, os, - indent + kIndentStep); - head = ring->advance(head);; - } while (head != ring->tail()); + CordRepBtree::Dump(rep, /*label=*/ "", include_data, *os); } if (stack.empty()) break; rep = stack.back(); @@ -1820,7 +1263,7 @@ static std::string ReportError(CordRep* root, CordRep* node) { } static bool VerifyNode(CordRep* root, CordRep* start_node, - bool full_validation) { + bool /* full_validation */) { absl::InlinedVector<CordRep*, 2> worklist; worklist.push_back(start_node); do { @@ -1830,100 +1273,33 @@ static bool VerifyNode(CordRep* root, CordRep* start_node, ABSL_INTERNAL_CHECK(node != nullptr, ReportError(root, node)); if (node != root) { ABSL_INTERNAL_CHECK(node->length != 0, ReportError(root, node)); + ABSL_INTERNAL_CHECK(!node->IsCrc(), ReportError(root, node)); } - if (node->tag == CONCAT) { - ABSL_INTERNAL_CHECK(node->concat()->left != nullptr, - ReportError(root, node)); - ABSL_INTERNAL_CHECK(node->concat()->right != nullptr, - ReportError(root, node)); - ABSL_INTERNAL_CHECK((node->length == node->concat()->left->length + - node->concat()->right->length), + if (node->IsFlat()) { + ABSL_INTERNAL_CHECK(node->length <= node->flat()->Capacity(), ReportError(root, node)); - if (full_validation) { - worklist.push_back(node->concat()->right); - worklist.push_back(node->concat()->left); - } - } else if (node->tag >= FLAT) { - ABSL_INTERNAL_CHECK( - node->length <= node->flat()->Capacity(), - ReportError(root, node)); - } else if (node->tag == EXTERNAL) { + } else if (node->IsExternal()) { ABSL_INTERNAL_CHECK(node->external()->base != nullptr, ReportError(root, node)); - } else if (node->tag == SUBSTRING) { + } else if (node->IsSubstring()) { ABSL_INTERNAL_CHECK( node->substring()->start < node->substring()->child->length, ReportError(root, node)); ABSL_INTERNAL_CHECK(node->substring()->start + node->length <= node->substring()->child->length, ReportError(root, node)); + } else if (node->IsCrc()) { + ABSL_INTERNAL_CHECK(node->crc()->child != nullptr, + ReportError(root, node)); + ABSL_INTERNAL_CHECK(node->crc()->length == node->crc()->child->length, + ReportError(root, node)); + worklist.push_back(node->crc()->child); } } while (!worklist.empty()); return true; } -// Traverses the tree and computes the total memory allocated. -/* static */ size_t Cord::MemoryUsageAux(const CordRep* rep) { - size_t total_mem_usage = 0; - - // Allow a quick exit for the common case that the root is a leaf. - if (RepMemoryUsageLeaf(rep, &total_mem_usage)) { - return total_mem_usage; - } - - // Iterate over the tree. cur_node is never a leaf node and leaf nodes will - // never be appended to tree_stack. This reduces overhead from manipulating - // tree_stack. - absl::InlinedVector<const CordRep*, kInlinedVectorSize> tree_stack; - const CordRep* cur_node = rep; - while (true) { - const CordRep* next_node = nullptr; - - if (cur_node->tag == CONCAT) { - total_mem_usage += sizeof(CordRepConcat); - const CordRep* left = cur_node->concat()->left; - if (!RepMemoryUsageLeaf(left, &total_mem_usage)) { - next_node = left; - } - - const CordRep* right = cur_node->concat()->right; - if (!RepMemoryUsageLeaf(right, &total_mem_usage)) { - if (next_node) { - tree_stack.push_back(next_node); - } - next_node = right; - } - } else if (cur_node->tag == RING) { - total_mem_usage += CordRepRing::AllocSize(cur_node->ring()->capacity()); - const CordRepRing* ring = cur_node->ring(); - CordRepRing::index_type pos = ring->head(), tail = ring->tail(); - do { - CordRep* node = ring->entry_child(pos); - assert(node->tag >= FLAT || node->tag == EXTERNAL); - RepMemoryUsageLeaf(node, &total_mem_usage); - } while ((pos = ring->advance(pos)) != tail); - } else { - // Since cur_node is not a leaf or a concat node it must be a substring. - assert(cur_node->tag == SUBSTRING); - total_mem_usage += sizeof(CordRepSubstring); - next_node = cur_node->substring()->child; - if (RepMemoryUsageLeaf(next_node, &total_mem_usage)) { - next_node = nullptr; - } - } - - if (!next_node) { - if (tree_stack.empty()) { - return total_mem_usage; - } - next_node = tree_stack.back(); - tree_stack.pop_back(); - } - cur_node = next_node; - } -} - std::ostream& operator<<(std::ostream& out, const Cord& cord) { for (absl::string_view chunk : cord.Chunks()) { out.write(chunk.data(), chunk.size()); @@ -1941,7 +1317,6 @@ uint8_t CordTestAccess::LengthToTag(size_t s) { ABSL_INTERNAL_CHECK(s <= kMaxFlatLength, absl::StrCat("Invalid length ", s)); return cord_internal::AllocatedSizeToTag(s + cord_internal::kFlatOverhead); } -size_t CordTestAccess::SizeofCordRepConcat() { return sizeof(CordRepConcat); } size_t CordTestAccess::SizeofCordRepExternal() { return sizeof(CordRepExternal); } diff --git a/absl/strings/cord.h b/absl/strings/cord.h index fa9cb913..18d6ab85 100644 --- a/absl/strings/cord.h +++ b/absl/strings/cord.h @@ -70,6 +70,8 @@ #include <string> #include <type_traits> +#include "absl/base/attributes.h" +#include "absl/base/config.h" #include "absl/base/internal/endian.h" #include "absl/base/internal/per_thread_tls.h" #include "absl/base/macros.h" @@ -77,9 +79,19 @@ #include "absl/container/inlined_vector.h" #include "absl/functional/function_ref.h" #include "absl/meta/type_traits.h" +#include "absl/strings/cord_analysis.h" +#include "absl/strings/cord_buffer.h" +#include "absl/strings/internal/cord_data_edge.h" #include "absl/strings/internal/cord_internal.h" +#include "absl/strings/internal/cord_rep_btree.h" +#include "absl/strings/internal/cord_rep_btree_reader.h" +#include "absl/strings/internal/cord_rep_crc.h" #include "absl/strings/internal/cord_rep_ring.h" -#include "absl/strings/internal/cord_rep_ring_reader.h" +#include "absl/strings/internal/cordz_functions.h" +#include "absl/strings/internal/cordz_info.h" +#include "absl/strings/internal/cordz_statistics.h" +#include "absl/strings/internal/cordz_update_scope.h" +#include "absl/strings/internal/cordz_update_tracker.h" #include "absl/strings/internal/resize_uninitialized.h" #include "absl/strings/internal/string_constant.h" #include "absl/strings/string_view.h" @@ -93,6 +105,20 @@ template <typename Releaser> Cord MakeCordFromExternal(absl::string_view, Releaser&&); void CopyCordToString(const Cord& src, std::string* dst); +// Cord memory accounting modes +enum class CordMemoryAccounting { + // Counts the *approximate* number of bytes held in full or in part by this + // Cord (which may not remain the same between invocations). Cords that share + // memory could each be "charged" independently for the same shared memory. + kTotal, + + // Counts the *approximate* number of bytes held in full or in part by this + // Cord weighted by the sharing ratio of that data. For example, if some data + // edge is shared by 4 different Cords, then each cord is attributed 1/4th of + // the total memory usage as a 'fair share' of the total memory usage. + kFairShare, +}; + // Cord // // A Cord is a sequence of characters, designed to be more efficient than a @@ -207,7 +233,7 @@ class Cord { // // Releases the Cord data. Any nodes that share data with other Cords, if // applicable, will have their reference counts reduced by 1. - void Clear(); + ABSL_ATTRIBUTE_REINITIALIZES void Clear(); // Cord::Append() // @@ -219,6 +245,45 @@ class Cord { template <typename T, EnableIfString<T> = 0> void Append(T&& src); + // Appends `buffer` to this cord, unless `buffer` has a zero length in which + // case this method has no effect on this cord instance. + // This method is guaranteed to consume `buffer`. + void Append(CordBuffer buffer); + + // Returns a CordBuffer, re-using potential existing capacity in this cord. + // + // Cord instances may have additional unused capacity in the last (or first) + // nodes of the underlying tree to facilitate amortized growth. This method + // allows applications to explicitly use this spare capacity if available, + // or create a new CordBuffer instance otherwise. + // If this cord has a final non-shared node with at least `min_capacity` + // available, then this method will return that buffer including its data + // contents. I.e.; the returned buffer will have a non-zero length, and + // a capacity of at least `buffer.length + min_capacity`. Otherwise, this + // method will return `CordBuffer::CreateWithDefaultLimit(capacity)`. + // + // Below an example of using GetAppendBuffer. Notice that in this example we + // use `GetAppendBuffer()` only on the first iteration. As we know nothing + // about any initial extra capacity in `cord`, we may be able to use the extra + // capacity. But as we add new buffers with fully utilized contents after that + // we avoid calling `GetAppendBuffer()` on subsequent iterations: while this + // works fine, it results in an unnecessary inspection of cord contents: + // + // void AppendRandomDataToCord(absl::Cord &cord, size_t n) { + // bool first = true; + // while (n > 0) { + // CordBuffer buffer = first ? cord.GetAppendBuffer(n) + // : CordBuffer::CreateWithDefaultLimit(n); + // absl::Span<char> data = buffer.available_up_to(n); + // FillRandomValues(data.data(), data.size()); + // buffer.IncreaseLengthBy(data.size()); + // cord.Append(std::move(buffer)); + // n -= data.size(); + // first = false; + // } + // } + CordBuffer GetAppendBuffer(size_t capacity, size_t min_capacity = 16); + // Cord::Prepend() // // Prepends data to the Cord, which may come from another Cord or other string @@ -228,6 +293,11 @@ class Cord { template <typename T, EnableIfString<T> = 0> void Prepend(T&& src); + // Prepends `buffer` to this cord, unless `buffer` has a zero length in which + // case this method has no effect on this cord instance. + // This method is guaranteed to consume `buffer`. + void Prepend(CordBuffer buffer); + // Cord::RemovePrefix() // // Removes the first `n` bytes of a Cord. @@ -249,9 +319,7 @@ class Cord { // swap() // // Swaps the contents of two Cords. - friend void swap(Cord& x, Cord& y) noexcept { - x.swap(y); - } + friend void swap(Cord& x, Cord& y) noexcept { x.swap(y); } // Cord::size() // @@ -265,11 +333,10 @@ class Cord { // Cord::EstimatedMemoryUsage() // - // Returns the *approximate* number of bytes held in full or in part by this - // Cord (which may not remain the same between invocations). Note that Cords - // that share memory could each be "charged" independently for the same shared - // memory. - size_t EstimatedMemoryUsage() const; + // Returns the *approximate* number of bytes held by this cord. + // See CordMemoryAccounting for more information on the accounting method. + size_t EstimatedMemoryUsage(CordMemoryAccounting accounting_method = + CordMemoryAccounting::kTotal) const; // Cord::Compare() // @@ -319,7 +386,7 @@ class Cord { //---------------------------------------------------------------------------- // // A `Cord::ChunkIterator` allows iteration over the constituent chunks of its - // Cord. Such iteration allows you to perform non-const operatons on the data + // Cord. Such iteration allows you to perform non-const operations on the data // of a Cord without modifying it. // // Generally, you do not instantiate a `Cord::ChunkIterator` directly; @@ -364,14 +431,8 @@ class Cord { private: using CordRep = absl::cord_internal::CordRep; - using CordRepRing = absl::cord_internal::CordRepRing; - using CordRepRingReader = absl::cord_internal::CordRepRingReader; - - // Stack of right children of concat nodes that we have to visit. - // Keep this at the end of the structure to avoid cache-thrashing. - // TODO(jgm): Benchmark to see if there's a more optimal value than 47 for - // the inlined vector size (47 exists for backward compatibility). - using Stack = absl::InlinedVector<absl::cord_internal::CordRep*, 47>; + using CordRepBtree = absl::cord_internal::CordRepBtree; + using CordRepBtreeReader = absl::cord_internal::CordRepBtreeReader; // Constructs a `begin()` iterator from `tree`. `tree` must not be null. explicit ChunkIterator(cord_internal::CordRep* tree); @@ -388,16 +449,9 @@ class Cord { Cord AdvanceAndReadBytes(size_t n); void AdvanceBytes(size_t n); - // Stack specific operator++ - ChunkIterator& AdvanceStack(); - - // Ring buffer specific operator++ - ChunkIterator& AdvanceRing(); - void AdvanceBytesRing(size_t n); - - // Iterates `n` bytes, where `n` is expected to be greater than or equal to - // `current_chunk_.size()`. - void AdvanceBytesSlowPath(size_t n); + // Btree specific operator++ + ChunkIterator& AdvanceBtree(); + void AdvanceBytesBtree(size_t n); // A view into bytes of the current `CordRep`. It may only be a view to a // suffix of bytes if this is being used by `CharIterator`. @@ -409,14 +463,11 @@ class Cord { // The number of bytes left in the `Cord` over which we are iterating. size_t bytes_remaining_ = 0; - // Cord reader for ring buffers. Empty if not traversing a ring buffer. - CordRepRingReader ring_reader_; - - // See 'Stack' alias definition. - Stack stack_of_right_children_; + // Cord reader for cord btrees. Empty if not traversing a btree. + CordRepBtreeReader btree_reader_; }; - // Cord::ChunkIterator::chunk_begin() + // Cord::chunk_begin() // // Returns an iterator to the first chunk of the `Cord`. // @@ -432,7 +483,7 @@ class Cord { // } ChunkIterator chunk_begin() const; - // Cord::ChunkItertator::chunk_end() + // Cord::chunk_end() // // Returns an iterator one increment past the last chunk of the `Cord`. // @@ -442,7 +493,7 @@ class Cord { ChunkIterator chunk_end() const; //---------------------------------------------------------------------------- - // Cord::ChunkIterator::ChunkRange + // Cord::ChunkRange //---------------------------------------------------------------------------- // // `ChunkRange` is a helper class for iterating over the chunks of the `Cord`, @@ -455,6 +506,16 @@ class Cord { // `Cord::chunk_begin()` and `Cord::chunk_end()`. class ChunkRange { public: + // Fulfill minimum c++ container requirements [container.requirements] + // These (partial) container type definitions allow ChunkRange to be used + // in various utilities expecting a subset of [container.requirements]. + // For example, the below enables using `::testing::ElementsAre(...)` + using value_type = absl::string_view; + using reference = value_type&; + using const_reference = const value_type&; + using iterator = ChunkIterator; + using const_iterator = ChunkIterator; + explicit ChunkRange(const Cord* cord) : cord_(cord) {} ChunkIterator begin() const; @@ -466,9 +527,9 @@ class Cord { // Cord::Chunks() // - // Returns a `Cord::ChunkIterator::ChunkRange` for iterating over the chunks - // of a `Cord` with a range-based for-loop. For most iteration tasks on a - // Cord, use `Cord::Chunks()` to retrieve this iterator. + // Returns a `Cord::ChunkRange` for iterating over the chunks of a `Cord` with + // a range-based for-loop. For most iteration tasks on a Cord, use + // `Cord::Chunks()` to retrieve this iterator. // // Example: // @@ -534,7 +595,7 @@ class Cord { ChunkIterator chunk_iterator_; }; - // Cord::CharIterator::AdvanceAndRead() + // Cord::AdvanceAndRead() // // Advances the `Cord::CharIterator` by `n_bytes` and returns the bytes // advanced as a separate `Cord`. `n_bytes` must be less than or equal to the @@ -542,21 +603,21 @@ class Cord { // valid to pass `char_end()` and `0`. static Cord AdvanceAndRead(CharIterator* it, size_t n_bytes); - // Cord::CharIterator::Advance() + // Cord::Advance() // // Advances the `Cord::CharIterator` by `n_bytes`. `n_bytes` must be less than // or equal to the number of bytes remaining within the Cord; otherwise, // behavior is undefined. It is valid to pass `char_end()` and `0`. static void Advance(CharIterator* it, size_t n_bytes); - // Cord::CharIterator::ChunkRemaining() + // Cord::ChunkRemaining() // // Returns the longest contiguous view starting at the iterator's position. // // `it` must be dereferenceable. static absl::string_view ChunkRemaining(const CharIterator& it); - // Cord::CharIterator::char_begin() + // Cord::char_begin() // // Returns an iterator to the first character of the `Cord`. // @@ -565,7 +626,7 @@ class Cord { // a `CharIterator` where range-based for-loops may not be available. CharIterator char_begin() const; - // Cord::CharIterator::char_end() + // Cord::char_end() // // Returns an iterator to one past the last character of the `Cord`. // @@ -574,18 +635,28 @@ class Cord { // a `CharIterator` where range-based for-loops are not useful. CharIterator char_end() const; - // Cord::CharIterator::CharRange + // Cord::CharRange // // `CharRange` is a helper class for iterating over the characters of a // producing an iterator which can be used within a range-based for loop. // Construction of a `CharRange` will return an iterator pointing to the first // character of the Cord. Generally, do not construct a `CharRange` directly; - // instead, prefer to use the `Cord::Chars()` method show below. + // instead, prefer to use the `Cord::Chars()` method shown below. // // Implementation note: `CharRange` is simply a convenience wrapper over // `Cord::char_begin()` and `Cord::char_end()`. class CharRange { public: + // Fulfill minimum c++ container requirements [container.requirements] + // Theses (partial) container type definitions allow CharRange to be used + // in various utilities expecting a subset of [container.requirements]. + // For example, the below enables using `::testing::ElementsAre(...)` + using value_type = char; + using reference = value_type&; + using const_reference = const value_type&; + using iterator = CharIterator; + using const_iterator = CharIterator; + explicit CharRange(const Cord* cord) : cord_(cord) {} CharIterator begin() const; @@ -595,11 +666,11 @@ class Cord { const Cord* cord_; }; - // Cord::CharIterator::Chars() + // Cord::Chars() // - // Returns a `Cord::CharIterator` for iterating over the characters of a - // `Cord` with a range-based for-loop. For most character-based iteration - // tasks on a Cord, use `Cord::Chars()` to retrieve this iterator. + // Returns a `Cord::CharRange` for iterating over the characters of a `Cord` + // with a range-based for-loop. For most character-based iteration tasks on a + // Cord, use `Cord::Chars()` to retrieve this iterator. // // Example: // @@ -646,6 +717,29 @@ class Cord { cord->Append(part); } + // Cord::SetExpectedChecksum() + // + // Stores a checksum value with this non-empty cord instance, for later + // retrieval. + // + // The expected checksum is a number stored out-of-band, alongside the data. + // It is preserved across copies and assignments, but any mutations to a cord + // will cause it to lose its expected checksum. + // + // The expected checksum is not part of a Cord's value, and does not affect + // operations such as equality or hashing. + // + // This field is intended to store a CRC32C checksum for later validation, to + // help support end-to-end checksum workflows. However, the Cord API itself + // does no CRC validation, and assigns no meaning to this number. + // + // This call has no effect if this cord is empty. + void SetExpectedChecksum(uint32_t crc); + + // Returns this cord's expected checksum, if it has one. Otherwise, returns + // nullopt. + absl::optional<uint32_t> ExpectedChecksum() const; + template <typename H> friend H AbslHashValue(H hash_state, const absl::Cord& c) { absl::optional<absl::string_view> maybe_flat = c.TryFlat(); @@ -661,13 +755,28 @@ class Cord { // be used by spelling absl::strings_internal::MakeStringConstant, which is // also an internal API. template <typename T> - explicit constexpr Cord(strings_internal::StringConstant<T>); + // NOLINTNEXTLINE(google-explicit-constructor) + constexpr Cord(strings_internal::StringConstant<T>); private: + using CordRep = absl::cord_internal::CordRep; + using CordRepFlat = absl::cord_internal::CordRepFlat; + using CordzInfo = cord_internal::CordzInfo; + using CordzUpdateScope = cord_internal::CordzUpdateScope; + using CordzUpdateTracker = cord_internal::CordzUpdateTracker; + using InlineData = cord_internal::InlineData; + using MethodIdentifier = CordzUpdateTracker::MethodIdentifier; + + // Creates a cord instance with `method` representing the originating + // public API call causing the cord to be created. + explicit Cord(absl::string_view src, MethodIdentifier method); + friend class CordTestPeer; friend bool operator==(const Cord& lhs, const Cord& rhs); friend bool operator==(const Cord& lhs, absl::string_view rhs); + friend const CordzInfo* GetCordzInfoForTesting(const Cord& cord); + // Calls the provided function once for each cord chunk, in order. Unlike // Chunks(), this API will not allocate memory. void ForEachChunk(absl::FunctionRef<void(absl::string_view)>) const; @@ -687,6 +796,7 @@ class Cord { static_assert(kMaxInline >= sizeof(absl::cord_internal::CordRep*), ""); constexpr InlineRep() : data_() {} + explicit InlineRep(InlineData::DefaultInitType init) : data_(init) {} InlineRep(const InlineRep& src); InlineRep(InlineRep&& src); InlineRep& operator=(const InlineRep& src); @@ -698,29 +808,62 @@ class Cord { bool empty() const; size_t size() const; const char* data() const; // Returns nullptr if holding pointer - void set_data(const char* data, size_t n, - bool nullify_tail); // Discards pointer, if any - char* set_data(size_t n); // Write data to the result + void set_data(const char* data, size_t n); // Discards pointer, if any + char* set_data(size_t n); // Write data to the result // Returns nullptr if holding bytes absl::cord_internal::CordRep* tree() const; absl::cord_internal::CordRep* as_tree() const; - // Discards old pointer, if any - void set_tree(absl::cord_internal::CordRep* rep); - // Replaces a tree with a new root. This is faster than set_tree, but it - // should only be used when it's clear that the old rep was a tree. - void replace_tree(absl::cord_internal::CordRep* rep); + const char* as_chars() const; // Returns non-null iff was holding a pointer absl::cord_internal::CordRep* clear(); // Converts to pointer if necessary. - absl::cord_internal::CordRep* force_tree(size_t extra_hint); - void reduce_size(size_t n); // REQUIRES: holding data + void reduce_size(size_t n); // REQUIRES: holding data void remove_prefix(size_t n); // REQUIRES: holding data - void AppendArray(const char* src_data, size_t src_size); + void AppendArray(absl::string_view src, MethodIdentifier method); absl::string_view FindFlatStartPiece() const; - void AppendTree(absl::cord_internal::CordRep* tree); - void PrependTree(absl::cord_internal::CordRep* tree); - void GetAppendRegion(char** region, size_t* size, size_t max_length); - void GetAppendRegion(char** region, size_t* size); + + // Creates a CordRepFlat instance from the current inlined data with `extra' + // bytes of desired additional capacity. + CordRepFlat* MakeFlatWithExtraCapacity(size_t extra); + + // Sets the tree value for this instance. `rep` must not be null. + // Requires the current instance to hold a tree, and a lock to be held on + // any CordzInfo referenced by this instance. The latter is enforced through + // the CordzUpdateScope argument. If the current instance is sampled, then + // the CordzInfo instance is updated to reference the new `rep` value. + void SetTree(CordRep* rep, const CordzUpdateScope& scope); + + // Identical to SetTree(), except that `rep` is allowed to be null, in + // which case the current instance is reset to an empty value. + void SetTreeOrEmpty(CordRep* rep, const CordzUpdateScope& scope); + + // Sets the tree value for this instance, and randomly samples this cord. + // This function disregards existing contents in `data_`, and should be + // called when a Cord is 'promoted' from an 'uninitialized' or 'inlined' + // value to a non-inlined (tree / ring) value. + void EmplaceTree(CordRep* rep, MethodIdentifier method); + + // Identical to EmplaceTree, except that it copies the parent stack from + // the provided `parent` data if the parent is sampled. + void EmplaceTree(CordRep* rep, const InlineData& parent, + MethodIdentifier method); + + // Commits the change of a newly created, or updated `rep` root value into + // this cord. `old_rep` indicates the old (inlined or tree) value of the + // cord, and determines if the commit invokes SetTree() or EmplaceTree(). + void CommitTree(const CordRep* old_rep, CordRep* rep, + const CordzUpdateScope& scope, MethodIdentifier method); + + void AppendTreeToInlined(CordRep* tree, MethodIdentifier method); + void AppendTreeToTree(CordRep* tree, MethodIdentifier method); + void AppendTree(CordRep* tree, MethodIdentifier method); + void PrependTreeToInlined(CordRep* tree, MethodIdentifier method); + void PrependTreeToTree(CordRep* tree, MethodIdentifier method); + void PrependTree(CordRep* tree, MethodIdentifier method); + + template <bool has_length> + void GetAppendRegion(char** region, size_t* size, size_t length); + bool IsSame(const InlineRep& other) const { return memcmp(&data_, &other.data_, sizeof(data_)) == 0; } @@ -758,6 +901,11 @@ class Cord { // Returns true if the Cord is being profiled by cordz. bool is_profiled() const { return data_.is_tree() && data_.is_profiled(); } + // Returns the available inlined capacity, or 0 if is_tree() == true. + size_t remaining_inline_capacity() const { + return data_.is_tree() ? 0 : kMaxInline - data_.inline_size(); + } + // Returns the profiled CordzInfo, or nullptr if not sampled. absl::cord_internal::CordzInfo* cordz_info() const { return data_.cordz_info(); @@ -776,8 +924,8 @@ class Cord { friend class Cord; void AssignSlow(const InlineRep& src); - // Unrefs the tree, stops profiling, and zeroes the contents - void ClearSlow(); + // Unrefs the tree and stops profiling. + void UnrefTree(); void ResetToEmpty() { data_ = {}; } @@ -788,9 +936,6 @@ class Cord { }; InlineRep contents_; - // Helper for MemoryUsage(). - static size_t MemoryUsageAux(const absl::cord_internal::CordRep* rep); - // Helper for GetFlat() and TryFlat(). static bool GetFlatAux(absl::cord_internal::CordRep* rep, absl::string_view* fragment); @@ -828,6 +973,23 @@ class Cord { template <typename C> void AppendImpl(C&& src); + // Appends / Prepends `src` to this instance, using precise sizing. + // This method does explicitly not attempt to use any spare capacity + // in any pending last added private owned flat. + // Requires `src` to be <= kMaxFlatLength. + void AppendPrecise(absl::string_view src, MethodIdentifier method); + void PrependPrecise(absl::string_view src, MethodIdentifier method); + + CordBuffer GetAppendBufferSlowPath(size_t capacity, size_t min_capacity); + + // Prepends the provided data to this instance. `method` contains the public + // API method for this action which is tracked for Cordz sampling purposes. + void PrependArray(absl::string_view src, MethodIdentifier method); + + // Assigns the value in 'src' to this instance, 'stealing' its contents. + // Requires src.length() > kMaxBytesToCopy. + Cord& AssignLargeString(std::string&& src); + // Helper for AbslHashValue(). template <typename H> H HashFragmented(H hash_state) const { @@ -857,8 +1019,8 @@ namespace cord_internal { // Fast implementation of memmove for up to 15 bytes. This implementation is // safe for overlapping regions. If nullify_tail is true, the destination is // padded with '\0' up to 16 bytes. -inline void SmallMemmove(char* dst, const char* src, size_t n, - bool nullify_tail = false) { +template <bool nullify_tail = false> +inline void SmallMemmove(char* dst, const char* src, size_t n) { if (n >= 8) { assert(n <= 16); uint64_t buf1; @@ -895,22 +1057,16 @@ inline void SmallMemmove(char* dst, const char* src, size_t n, } // Does non-template-specific `CordRepExternal` initialization. -// Expects `data` to be non-empty. +// Requires `data` to be non-empty. void InitializeCordRepExternal(absl::string_view data, CordRepExternal* rep); // Creates a new `CordRep` that owns `data` and `releaser` and returns a pointer -// to it, or `nullptr` if `data` was empty. +// to it. Requires `data` to be non-empty. template <typename Releaser> // NOLINTNEXTLINE - suppress clang-tidy raw pointer return. CordRep* NewExternalRep(absl::string_view data, Releaser&& releaser) { + assert(!data.empty()); using ReleaserType = absl::decay_t<Releaser>; - if (data.empty()) { - // Never create empty external nodes. - InvokeReleaser(Rank0{}, ReleaserType(std::forward<Releaser>(releaser)), - data); - return nullptr; - } - CordRepExternal* rep = new CordRepExternalImpl<ReleaserType>( std::forward<Releaser>(releaser), 0); InitializeCordRepExternal(data, rep); @@ -930,8 +1086,16 @@ inline CordRep* NewExternalRep(absl::string_view data, template <typename Releaser> Cord MakeCordFromExternal(absl::string_view data, Releaser&& releaser) { Cord cord; - cord.contents_.set_tree(::absl::cord_internal::NewExternalRep( - data, std::forward<Releaser>(releaser))); + if (ABSL_PREDICT_TRUE(!data.empty())) { + cord.contents_.EmplaceTree(::absl::cord_internal::NewExternalRep( + data, std::forward<Releaser>(releaser)), + Cord::MethodIdentifier::kMakeCordFromExternal); + } else { + using ReleaserType = absl::decay_t<Releaser>; + cord_internal::InvokeReleaser( + cord_internal::Rank0{}, ReleaserType(std::forward<Releaser>(releaser)), + data); + } return cord; } @@ -939,15 +1103,16 @@ constexpr Cord::InlineRep::InlineRep(cord_internal::InlineData data) : data_(data) {} inline Cord::InlineRep::InlineRep(const Cord::InlineRep& src) - : data_(src.data_) { - if (is_tree()) { - data_.clear_cordz_info(); - absl::cord_internal::CordRep::Ref(as_tree()); + : data_(InlineData::kDefaultInit) { + if (CordRep* tree = src.tree()) { + EmplaceTree(CordRep::Ref(tree), src.data_, + CordzUpdateTracker::kConstructorCord); + } else { + data_ = src.data_; } } -inline Cord::InlineRep::InlineRep(Cord::InlineRep&& src) { - data_ = src.data_; +inline Cord::InlineRep::InlineRep(Cord::InlineRep&& src) : data_(src.data_) { src.ResetToEmpty(); } @@ -966,7 +1131,7 @@ inline Cord::InlineRep& Cord::InlineRep::operator=(const Cord::InlineRep& src) { inline Cord::InlineRep& Cord::InlineRep::operator=( Cord::InlineRep&& src) noexcept { if (is_tree()) { - ClearSlow(); + UnrefTree(); } data_ = src.data_; src.ResetToEmpty(); @@ -984,6 +1149,11 @@ inline const char* Cord::InlineRep::data() const { return is_tree() ? nullptr : data_.as_chars(); } +inline const char* Cord::InlineRep::as_chars() const { + assert(!data_.is_tree()); + return data_.as_chars(); +} + inline absl::cord_internal::CordRep* Cord::InlineRep::as_tree() const { assert(data_.is_tree()); return data_.as_tree(); @@ -1003,31 +1173,62 @@ inline size_t Cord::InlineRep::size() const { return is_tree() ? as_tree()->length : inline_size(); } -inline void Cord::InlineRep::set_tree(absl::cord_internal::CordRep* rep) { - if (rep == nullptr) { - ResetToEmpty(); +inline cord_internal::CordRepFlat* Cord::InlineRep::MakeFlatWithExtraCapacity( + size_t extra) { + static_assert(cord_internal::kMinFlatLength >= sizeof(data_), ""); + size_t len = data_.inline_size(); + auto* result = CordRepFlat::New(len + extra); + result->length = len; + memcpy(result->Data(), data_.as_chars(), sizeof(data_)); + return result; +} + +inline void Cord::InlineRep::EmplaceTree(CordRep* rep, + MethodIdentifier method) { + assert(rep); + data_.make_tree(rep); + CordzInfo::MaybeTrackCord(data_, method); +} + +inline void Cord::InlineRep::EmplaceTree(CordRep* rep, const InlineData& parent, + MethodIdentifier method) { + data_.make_tree(rep); + CordzInfo::MaybeTrackCord(data_, parent, method); +} + +inline void Cord::InlineRep::SetTree(CordRep* rep, + const CordzUpdateScope& scope) { + assert(rep); + assert(data_.is_tree()); + data_.set_tree(rep); + scope.SetCordRep(rep); +} + +inline void Cord::InlineRep::SetTreeOrEmpty(CordRep* rep, + const CordzUpdateScope& scope) { + assert(data_.is_tree()); + if (rep) { + data_.set_tree(rep); } else { - if (data_.is_tree()) { - // `data_` already holds a 'tree' value and an optional cordz_info value. - // Replace the tree value only, leaving the cordz_info value unchanged. - data_.set_tree(rep); - } else { - // `data_` contains inlined data: initialize data_ to tree value `rep`. - data_.make_tree(rep); - } + data_ = {}; } + scope.SetCordRep(rep); } -inline void Cord::InlineRep::replace_tree(absl::cord_internal::CordRep* rep) { - ABSL_ASSERT(is_tree()); - if (ABSL_PREDICT_FALSE(rep == nullptr)) { - set_tree(rep); - return; +inline void Cord::InlineRep::CommitTree(const CordRep* old_rep, CordRep* rep, + const CordzUpdateScope& scope, + MethodIdentifier method) { + if (old_rep) { + SetTree(rep, scope); + } else { + EmplaceTree(rep, method); } - data_.set_tree(rep); } inline absl::cord_internal::CordRep* Cord::InlineRep::clear() { + if (is_tree()) { + CordzInfo::MaybeUntrackCord(cordz_info()); + } absl::cord_internal::CordRep* result = tree(); ResetToEmpty(); return result; @@ -1042,6 +1243,9 @@ inline void Cord::InlineRep::CopyToArray(char* dst) const { constexpr inline Cord::Cord() noexcept {} +inline Cord::Cord(absl::string_view src) + : Cord(src, CordzUpdateTracker::kConstructorString) {} + template <typename T> constexpr Cord::Cord(strings_internal::StringConstant<T>) : contents_(strings_internal::StringConstant<T>::value.size() <= @@ -1057,6 +1261,15 @@ inline Cord& Cord::operator=(const Cord& x) { return *this; } +template <typename T, Cord::EnableIfString<T>> +Cord& Cord::operator=(T&& src) { + if (src.size() <= cord_internal::kMaxBytesToCopy) { + return operator=(absl::string_view(src)); + } else { + return AssignLargeString(std::forward<T>(src)); + } +} + inline Cord::Cord(const Cord& src) : contents_(src.contents_) {} inline Cord::Cord(Cord&& src) noexcept : contents_(std::move(src.contents_)) {} @@ -1071,7 +1284,6 @@ inline Cord& Cord::operator=(Cord&& x) noexcept { } extern template Cord::Cord(std::string&& src); -extern template Cord& Cord::operator=(std::string&& src); inline size_t Cord::size() const { // Length is 1st field in str.rep_ @@ -1080,10 +1292,15 @@ inline size_t Cord::size() const { inline bool Cord::empty() const { return contents_.empty(); } -inline size_t Cord::EstimatedMemoryUsage() const { +inline size_t Cord::EstimatedMemoryUsage( + CordMemoryAccounting accounting_method) const { size_t result = sizeof(Cord); if (const absl::cord_internal::CordRep* rep = contents_.tree()) { - result += MemoryUsageAux(rep); + if (accounting_method == CordMemoryAccounting::kFairShare) { + result += cord_internal::GetEstimatedFairShareMemoryUsage(rep); + } else { + result += cord_internal::GetEstimatedMemoryUsage(rep); + } } return result; } @@ -1114,7 +1331,36 @@ inline absl::string_view Cord::Flatten() { } inline void Cord::Append(absl::string_view src) { - contents_.AppendArray(src.data(), src.size()); + contents_.AppendArray(src, CordzUpdateTracker::kAppendString); +} + +inline void Cord::Prepend(absl::string_view src) { + PrependArray(src, CordzUpdateTracker::kPrependString); +} + +inline void Cord::Append(CordBuffer buffer) { + if (ABSL_PREDICT_FALSE(buffer.length() == 0)) return; + absl::string_view short_value; + if (CordRep* rep = buffer.ConsumeValue(short_value)) { + contents_.AppendTree(rep, CordzUpdateTracker::kAppendCordBuffer); + } else { + AppendPrecise(short_value, CordzUpdateTracker::kAppendCordBuffer); + } +} + +inline void Cord::Prepend(CordBuffer buffer) { + if (ABSL_PREDICT_FALSE(buffer.length() == 0)) return; + absl::string_view short_value; + if (CordRep* rep = buffer.ConsumeValue(short_value)) { + contents_.PrependTree(rep, CordzUpdateTracker::kPrependCordBuffer); + } else { + PrependPrecise(short_value, CordzUpdateTracker::kPrependCordBuffer); + } +} + +inline CordBuffer Cord::GetAppendBuffer(size_t capacity, size_t min_capacity) { + if (empty()) return CordBuffer::CreateWithDefaultLimit(capacity); + return GetAppendBufferSlowPath(capacity, min_capacity); } extern template void Cord::Append(std::string&& src); @@ -1143,44 +1389,44 @@ inline bool Cord::StartsWith(absl::string_view rhs) const { } inline void Cord::ChunkIterator::InitTree(cord_internal::CordRep* tree) { - if (tree->tag == cord_internal::RING) { - current_chunk_ = ring_reader_.Reset(tree->ring()); - return; + tree = cord_internal::SkipCrcNode(tree); + if (tree->tag == cord_internal::BTREE) { + current_chunk_ = btree_reader_.Init(tree->btree()); + } else { + current_leaf_ = tree; + current_chunk_ = cord_internal::EdgeData(tree); } - - stack_of_right_children_.push_back(tree); - operator++(); } -inline Cord::ChunkIterator::ChunkIterator(cord_internal::CordRep* tree) - : bytes_remaining_(tree->length) { +inline Cord::ChunkIterator::ChunkIterator(cord_internal::CordRep* tree) { + bytes_remaining_ = tree->length; InitTree(tree); } -inline Cord::ChunkIterator::ChunkIterator(const Cord* cord) - : bytes_remaining_(cord->size()) { - if (cord->contents_.is_tree()) { - InitTree(cord->contents_.as_tree()); +inline Cord::ChunkIterator::ChunkIterator(const Cord* cord) { + if (CordRep* tree = cord->contents_.tree()) { + bytes_remaining_ = tree->length; + InitTree(tree); } else { - current_chunk_ = - absl::string_view(cord->contents_.data(), bytes_remaining_); + bytes_remaining_ = cord->contents_.inline_size(); + current_chunk_ = {cord->contents_.data(), bytes_remaining_}; } } -inline Cord::ChunkIterator& Cord::ChunkIterator::AdvanceRing() { - current_chunk_ = ring_reader_.Next(); +inline Cord::ChunkIterator& Cord::ChunkIterator::AdvanceBtree() { + current_chunk_ = btree_reader_.Next(); return *this; } -inline void Cord::ChunkIterator::AdvanceBytesRing(size_t n) { +inline void Cord::ChunkIterator::AdvanceBytesBtree(size_t n) { assert(n >= current_chunk_.size()); bytes_remaining_ -= n; if (bytes_remaining_) { if (n == current_chunk_.size()) { - current_chunk_ = ring_reader_.Next(); + current_chunk_ = btree_reader_.Next(); } else { - size_t offset = ring_reader_.length() - bytes_remaining_; - current_chunk_ = ring_reader_.Seek(offset); + size_t offset = btree_reader_.length() - bytes_remaining_; + current_chunk_ = btree_reader_.Seek(offset); } } else { current_chunk_ = {}; @@ -1193,8 +1439,11 @@ inline Cord::ChunkIterator& Cord::ChunkIterator::operator++() { assert(bytes_remaining_ >= current_chunk_.size()); bytes_remaining_ -= current_chunk_.size(); if (bytes_remaining_ > 0) { - return ring_reader_ ? AdvanceRing() : AdvanceStack(); - } else { + if (btree_reader_) { + return AdvanceBtree(); + } else { + assert(!current_chunk_.empty()); // Called on invalid iterator. + } current_chunk_ = {}; } return *this; @@ -1235,7 +1484,11 @@ inline void Cord::ChunkIterator::AdvanceBytes(size_t n) { if (ABSL_PREDICT_TRUE(n < current_chunk_.size())) { RemoveChunkPrefix(n); } else if (n != 0) { - ring_reader_ ? AdvanceBytesRing(n) : AdvanceBytesSlowPath(n); + if (btree_reader_) { + AdvanceBytesBtree(n); + } else { + bytes_remaining_ = 0; + } } } @@ -1326,7 +1579,7 @@ inline void Cord::ForEachChunk( } } -// Nonmember Cord-to-Cord relational operarators. +// Nonmember Cord-to-Cord relational operators. inline bool operator==(const Cord& lhs, const Cord& rhs) { if (lhs.contents_.IsSame(rhs.contents_)) return true; size_t rhs_size = rhs.size(); @@ -1335,12 +1588,8 @@ inline bool operator==(const Cord& lhs, const Cord& rhs) { } inline bool operator!=(const Cord& x, const Cord& y) { return !(x == y); } -inline bool operator<(const Cord& x, const Cord& y) { - return x.Compare(y) < 0; -} -inline bool operator>(const Cord& x, const Cord& y) { - return x.Compare(y) > 0; -} +inline bool operator<(const Cord& x, const Cord& y) { return x.Compare(y) < 0; } +inline bool operator>(const Cord& x, const Cord& y) { return x.Compare(y) > 0; } inline bool operator<=(const Cord& x, const Cord& y) { return x.Compare(y) <= 0; } @@ -1381,7 +1630,6 @@ class CordTestAccess { public: static size_t FlatOverhead(); static size_t MaxFlatLength(); - static size_t SizeofCordRepConcat(); static size_t SizeofCordRepExternal(); static size_t SizeofCordRepSubstring(); static size_t FlatTagToLength(uint8_t tag); diff --git a/absl/strings/cord_analysis.cc b/absl/strings/cord_analysis.cc new file mode 100644 index 00000000..73d3c4e6 --- /dev/null +++ b/absl/strings/cord_analysis.cc @@ -0,0 +1,188 @@ +// Copyright 2021 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/cord_analysis.h" + +#include <cstddef> +#include <cstdint> + +#include "absl/base/attributes.h" +#include "absl/base/config.h" +#include "absl/container/inlined_vector.h" +#include "absl/strings/internal/cord_data_edge.h" +#include "absl/strings/internal/cord_internal.h" +#include "absl/strings/internal/cord_rep_btree.h" +#include "absl/strings/internal/cord_rep_crc.h" +#include "absl/strings/internal/cord_rep_flat.h" +#include "absl/strings/internal/cord_rep_ring.h" +// +#include "absl/base/macros.h" +#include "absl/base/port.h" +#include "absl/functional/function_ref.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { +namespace { + +// Accounting mode for analyzing memory usage. +enum class Mode { kTotal, kFairShare }; + +// CordRepRef holds a `const CordRep*` reference in rep, and depending on mode, +// holds a 'fraction' representing a cumulative inverse refcount weight. +template <Mode mode> +struct CordRepRef { + // Instantiates a CordRepRef instance. + explicit CordRepRef(const CordRep* r) : rep(r) {} + + // Creates a child reference holding the provided child. + // Overloaded to add cumulative reference count for kFairShare. + CordRepRef Child(const CordRep* child) const { return CordRepRef(child); } + + const CordRep* rep; +}; + +// RawUsage holds the computed total number of bytes. +template <Mode mode> +struct RawUsage { + size_t total = 0; + + // Add 'size' to total, ignoring the CordRepRef argument. + void Add(size_t size, CordRepRef<mode>) { total += size; } +}; + +// Returns n / refcount avoiding a div for the common refcount == 1. +template <typename refcount_t> +double MaybeDiv(double d, refcount_t refcount) { + return refcount == 1 ? d : d / refcount; +} + +// Overloaded 'kFairShare' specialization for CordRepRef. This class holds a +// `fraction` value which represents a cumulative inverse refcount weight. +// For example, a top node with a reference count of 2 will have a fraction +// value of 1/2 = 0.5, representing the 'fair share' of memory it references. +// A node below such a node with a reference count of 5 then has a fraction of +// 0.5 / 5 = 0.1 representing the fair share of memory below that node, etc. +template <> +struct CordRepRef<Mode::kFairShare> { + // Creates a CordRepRef with the provided rep and top (parent) fraction. + explicit CordRepRef(const CordRep* r, double frac = 1.0) + : rep(r), fraction(MaybeDiv(frac, r->refcount.Get())) {} + + // Returns a CordRepRef with a fraction of `this->fraction / child.refcount` + CordRepRef Child(const CordRep* child) const { + return CordRepRef(child, fraction); + } + + const CordRep* rep; + double fraction; +}; + +// Overloaded 'kFairShare' specialization for RawUsage +template <> +struct RawUsage<Mode::kFairShare> { + double total = 0; + + // Adds `size` multiplied by `rep.fraction` to the total size. + void Add(size_t size, CordRepRef<Mode::kFairShare> rep) { + total += static_cast<double>(size) * rep.fraction; + } +}; + +// Computes the estimated memory size of the provided data edge. +// External reps are assumed 'heap allocated at their exact size'. +template <Mode mode> +void AnalyzeDataEdge(CordRepRef<mode> rep, RawUsage<mode>& raw_usage) { + assert(IsDataEdge(rep.rep)); + + // Consume all substrings + if (rep.rep->tag == SUBSTRING) { + raw_usage.Add(sizeof(CordRepSubstring), rep); + rep = rep.Child(rep.rep->substring()->child); + } + + // Consume FLAT / EXTERNAL + const size_t size = + rep.rep->tag >= FLAT + ? rep.rep->flat()->AllocatedSize() + : rep.rep->length + sizeof(CordRepExternalImpl<intptr_t>); + raw_usage.Add(size, rep); +} + +// Computes the memory size of the provided Ring tree. +template <Mode mode> +void AnalyzeRing(CordRepRef<mode> rep, RawUsage<mode>& raw_usage) { + const CordRepRing* ring = rep.rep->ring(); + raw_usage.Add(CordRepRing::AllocSize(ring->capacity()), rep); + ring->ForEach([&](CordRepRing::index_type pos) { + AnalyzeDataEdge(rep.Child(ring->entry_child(pos)), raw_usage); + }); +} + +// Computes the memory size of the provided Btree tree. +template <Mode mode> +void AnalyzeBtree(CordRepRef<mode> rep, RawUsage<mode>& raw_usage) { + raw_usage.Add(sizeof(CordRepBtree), rep); + const CordRepBtree* tree = rep.rep->btree(); + if (tree->height() > 0) { + for (CordRep* edge : tree->Edges()) { + AnalyzeBtree(rep.Child(edge), raw_usage); + } + } else { + for (CordRep* edge : tree->Edges()) { + AnalyzeDataEdge(rep.Child(edge), raw_usage); + } + } +} + +template <Mode mode> +size_t GetEstimatedUsage(const CordRep* rep) { + // Zero initialized memory usage totals. + RawUsage<mode> raw_usage; + + // Capture top level node and refcount into a CordRepRef. + CordRepRef<mode> repref(rep); + + // Consume the top level CRC node if present. + if (repref.rep->tag == CRC) { + raw_usage.Add(sizeof(CordRepCrc), repref); + repref = repref.Child(repref.rep->crc()->child); + } + + if (IsDataEdge(repref.rep)) { + AnalyzeDataEdge(repref, raw_usage); + } else if (repref.rep->tag == BTREE) { + AnalyzeBtree(repref, raw_usage); + } else if (repref.rep->tag == RING) { + AnalyzeRing(repref, raw_usage); + } else { + assert(false); + } + + return static_cast<size_t>(raw_usage.total); +} + +} // namespace + +size_t GetEstimatedMemoryUsage(const CordRep* rep) { + return GetEstimatedUsage<Mode::kTotal>(rep); +} + +size_t GetEstimatedFairShareMemoryUsage(const CordRep* rep) { + return GetEstimatedUsage<Mode::kFairShare>(rep); +} + +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/absl/strings/cord_analysis.h b/absl/strings/cord_analysis.h new file mode 100644 index 00000000..7041ad1a --- /dev/null +++ b/absl/strings/cord_analysis.h @@ -0,0 +1,44 @@ +// Copyright 2021 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_STRINGS_CORD_ANALYSIS_H_ +#define ABSL_STRINGS_CORD_ANALYSIS_H_ + +#include <cstddef> +#include <cstdint> + +#include "absl/base/config.h" +#include "absl/strings/internal/cord_internal.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { + +// Returns the *approximate* number of bytes held in full or in part by this +// Cord (which may not remain the same between invocations). Cords that share +// memory could each be "charged" independently for the same shared memory. +size_t GetEstimatedMemoryUsage(const CordRep* rep); + +// Returns the *approximate* number of bytes held in full or in part by this +// CordRep weighted by the sharing ratio of that data. For example, if some data +// edge is shared by 4 different Cords, then each cord is attribute 1/4th of +// the total memory usage as a 'fair share' of the total memory usage. +size_t GetEstimatedFairShareMemoryUsage(const CordRep* rep); + +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl + + +#endif // ABSL_STRINGS_CORD_ANALYSIS_H_ diff --git a/absl/debugging/leak_check_disable.cc b/absl/strings/cord_buffer.cc index 924d6e3d..fad6269c 100644 --- a/absl/debugging/leak_check_disable.cc +++ b/absl/strings/cord_buffer.cc @@ -1,10 +1,10 @@ -// Copyright 2017 The Abseil Authors. +// Copyright 2022 The Abseil Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. // You may obtain a copy of the License at // -// https://www.apache.org/licenses/LICENSE-2.0 +// https://www.apache.org/licenses/LICENSE-2.0 // // Unless required by applicable law or agreed to in writing, software // distributed under the License is distributed on an "AS IS" BASIS, @@ -12,9 +12,19 @@ // See the License for the specific language governing permissions and // limitations under the License. -// Disable LeakSanitizer when this file is linked in. -// This function overrides __lsan_is_turned_off from sanitizer/lsan_interface.h -extern "C" int __lsan_is_turned_off(); -extern "C" int __lsan_is_turned_off() { - return 1; -} +#include "absl/strings/cord_buffer.h" + +#include <cstddef> + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +#ifdef ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL +constexpr size_t CordBuffer::kDefaultLimit; +constexpr size_t CordBuffer::kCustomLimit; +#endif + +ABSL_NAMESPACE_END +} // namespace absl diff --git a/absl/strings/cord_buffer.h b/absl/strings/cord_buffer.h new file mode 100644 index 00000000..56a6ce6f --- /dev/null +++ b/absl/strings/cord_buffer.h @@ -0,0 +1,572 @@ +// Copyright 2021 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ----------------------------------------------------------------------------- +// File: cord_buffer.h +// ----------------------------------------------------------------------------- +// +// This file defines an `absl::CordBuffer` data structure to hold data for +// eventual inclusion within an existing `Cord` data structure. Cord buffers are +// useful for building large Cords that may require custom allocation of its +// associated memory. +// +#ifndef ABSL_STRINGS_CORD_BUFFER_H_ +#define ABSL_STRINGS_CORD_BUFFER_H_ + +#include <algorithm> +#include <cassert> +#include <cstddef> +#include <cstdint> +#include <memory> +#include <utility> + +#include "absl/base/config.h" +#include "absl/base/macros.h" +#include "absl/numeric/bits.h" +#include "absl/strings/internal/cord_internal.h" +#include "absl/strings/internal/cord_rep_flat.h" +#include "absl/types/span.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +class Cord; +class CordBufferTestPeer; + +// CordBuffer +// +// CordBuffer manages memory buffers for purposes such as zero-copy APIs as well +// as applications building cords with large data requiring granular control +// over the allocation and size of cord data. For example, a function creating +// a cord of random data could use a CordBuffer as follows: +// +// absl::Cord CreateRandomCord(size_t length) { +// absl::Cord cord; +// while (length > 0) { +// CordBuffer buffer = CordBuffer::CreateWithDefaultLimit(length); +// absl::Span<char> data = buffer.available_up_to(length); +// FillRandomValues(data.data(), data.size()); +// buffer.IncreaseLengthBy(data.size()); +// cord.Append(std::move(buffer)); +// length -= data.size(); +// } +// return cord; +// } +// +// CordBuffer instances are by default limited to a capacity of `kDefaultLimit` +// bytes. `kDefaultLimit` is currently just under 4KiB, but this default may +// change in the future and/or for specific architectures. The default limit is +// aimed to provide a good trade-off between performance and memory overhead. +// Smaller buffers typically incur more compute cost while larger buffers are +// more CPU efficient but create significant memory overhead because of such +// allocations being less granular. Using larger buffers may also increase the +// risk of memory fragmentation. +// +// Applications create a buffer using one of the `CreateWithDefaultLimit()` or +// `CreateWithCustomLimit()` methods. The returned instance will have a non-zero +// capacity and a zero length. Applications use the `data()` method to set the +// contents of the managed memory, and once done filling the buffer, use the +// `IncreaseLengthBy()` or 'SetLength()' method to specify the length of the +// initialized data before adding the buffer to a Cord. +// +// The `CreateWithCustomLimit()` method is intended for applications needing +// larger buffers than the default memory limit, allowing the allocation of up +// to a capacity of `kCustomLimit` bytes minus some minimum internal overhead. +// The usage of `CreateWithCustomLimit()` should be limited to only those use +// cases where the distribution of the input is relatively well known, and/or +// where the trade-off between the efficiency gains outweigh the risk of memory +// fragmentation. See the documentation for `CreateWithCustomLimit()` for more +// information on using larger custom limits. +// +// The capacity of a `CordBuffer` returned by one of the `Create` methods may +// be larger than the requested capacity due to rounding, alignment and +// granularity of the memory allocator. Applications should use the `capacity` +// method to obtain the effective capacity of the returned instance as +// demonstrated in the provided example above. +// +// CordBuffer is a move-only class. All references into the managed memory are +// invalidated when an instance is moved into either another CordBuffer instance +// or a Cord. Writing to a location obtained by a previous call to `data()` +// after an instance was moved will lead to undefined behavior. +// +// A `moved from` CordBuffer instance will have a valid, but empty state. +// CordBuffer is thread compatible. +class CordBuffer { + public: + // kDefaultLimit + // + // Default capacity limits of allocated CordBuffers. + // See the class comments for more information on allocation limits. + static constexpr size_t kDefaultLimit = cord_internal::kMaxFlatLength; + + // kCustomLimit + // + // Maximum size for CreateWithCustomLimit() allocated buffers. + // Note that the effective capacity may be slightly less + // because of internal overhead of internal cord buffers. + static constexpr size_t kCustomLimit = 64U << 10; + + // Constructors, Destructors and Assignment Operators + + // Creates an empty CordBuffer. + CordBuffer() = default; + + // Destroys this CordBuffer instance and, if not empty, releases any memory + // managed by this instance, invalidating previously returned references. + ~CordBuffer(); + + // CordBuffer is move-only + CordBuffer(CordBuffer&& rhs) noexcept; + CordBuffer& operator=(CordBuffer&&) noexcept; + CordBuffer(const CordBuffer&) = delete; + CordBuffer& operator=(const CordBuffer&) = delete; + + // CordBuffer::MaximumPayload() + // + // Returns the guaranteed maximum payload for a CordBuffer returned by the + // `CreateWithDefaultLimit()` method. While small, each internal buffer inside + // a Cord incurs an overhead to manage the length, type and reference count + // for the buffer managed inside the cord tree. Applications can use this + // method to get approximate number of buffers required for a given byte + // size, etc. + // + // For example: + // const size_t payload = absl::CordBuffer::MaximumPayload(); + // const size_t buffer_count = (total_size + payload - 1) / payload; + // buffers.reserve(buffer_count); + static constexpr size_t MaximumPayload(); + + // Overload to the above `MaximumPayload()` except that it returns the + // maximum payload for a CordBuffer returned by the `CreateWithCustomLimit()` + // method given the provided `block_size`. + static constexpr size_t MaximumPayload(size_t block_size); + + // CordBuffer::CreateWithDefaultLimit() + // + // Creates a CordBuffer instance of the desired `capacity`, capped at the + // default limit `kDefaultLimit`. The returned buffer has a guaranteed + // capacity of at least `min(kDefaultLimit, capacity)`. See the class comments + // for more information on buffer capacities and intended usage. + static CordBuffer CreateWithDefaultLimit(size_t capacity); + + + // CordBuffer::CreateWithCustomLimit() + // + // Creates a CordBuffer instance of the desired `capacity` rounded to an + // appropriate power of 2 size less than, or equal to `block_size`. + // Requires `block_size` to be a power of 2. + // + // If `capacity` is less than or equal to `kDefaultLimit`, then this method + // behaves identical to `CreateWithDefaultLimit`, which means that the caller + // is guaranteed to get a buffer of at least the requested capacity. + // + // If `capacity` is greater than or equal to `block_size`, then this method + // returns a buffer with an `allocated size` of `block_size` bytes. Otherwise, + // this methods returns a buffer with a suitable smaller power of 2 block size + // to satisfy the request. The actual size depends on a number of factors, and + // is typically (but not necessarily) the highest or second highest power of 2 + // value less than or equal to `capacity`. + // + // The 'allocated size' includes a small amount of overhead required for + // internal state, which is currently 13 bytes on 64-bit platforms. For + // example: a buffer created with `block_size` and `capacity' set to 8KiB + // will have an allocated size of 8KiB, and an effective internal `capacity` + // of 8KiB - 13 = 8179 bytes. + // + // To demonstrate this in practice, let's assume we want to read data from + // somewhat larger files using approximately 64KiB buffers: + // + // absl::Cord ReadFromFile(int fd, size_t n) { + // absl::Cord cord; + // while (n > 0) { + // CordBuffer buffer = CordBuffer::CreateWithCustomLimit(64 << 10, n); + // absl::Span<char> data = buffer.available_up_to(n); + // ReadFileDataOrDie(fd, data.data(), data.size()); + // buffer.IncreaseLengthBy(data.size()); + // cord.Append(std::move(buffer)); + // n -= data.size(); + // } + // return cord; + // } + // + // If we'd use this function to read a file of 659KiB, we may get the + // following pattern of allocated cord buffer sizes: + // + // CreateWithCustomLimit(64KiB, 674816) --> ~64KiB (65523) + // CreateWithCustomLimit(64KiB, 674816) --> ~64KiB (65523) + // ... + // CreateWithCustomLimit(64KiB, 19586) --> ~16KiB (16371) + // CreateWithCustomLimit(64KiB, 3215) --> 3215 (at least 3215) + // + // The reason the method returns a 16K buffer instead of a roughly 19K buffer + // is to reduce memory overhead and fragmentation risks. Using carefully + // chosen power of 2 values reduces the entropy of allocated memory sizes. + // + // Additionally, let's assume we'd use the above function on files that are + // generally smaller than 64K. If we'd use 'precise' sized buffers for such + // files, than we'd get a very wide distribution of allocated memory sizes + // rounded to 4K page sizes, and we'd end up with a lot of unused capacity. + // + // In general, application should only use custom sizes if the data they are + // consuming or storing is expected to be many times the chosen block size, + // and be based on objective data and performance metrics. For example, a + // compress function may work faster and consume less CPU when using larger + // buffers. Such an application should pick a size offering a reasonable + // trade-off between expected data size, compute savings with larger buffers, + // and the cost or fragmentation effect of larger buffers. + // Applications must pick a reasonable spot on that curve, and make sure their + // data meets their expectations in size distributions such as "mostly large". + static CordBuffer CreateWithCustomLimit(size_t block_size, size_t capacity); + + // CordBuffer::available() + // + // Returns the span delineating the available capacity in this buffer + // which is defined as `{ data() + length(), capacity() - length() }`. + absl::Span<char> available(); + + // CordBuffer::available_up_to() + // + // Returns the span delineating the available capacity in this buffer limited + // to `size` bytes. This is equivalent to `available().subspan(0, size)`. + absl::Span<char> available_up_to(size_t size); + + // CordBuffer::data() + // + // Returns a non-null reference to the data managed by this instance. + // Applications are allowed to write up to `capacity` bytes of instance data. + // CordBuffer data is uninitialized by default. Reading data from an instance + // that has not yet been initialized will lead to undefined behavior. + char* data(); + const char* data() const; + + // CordBuffer::length() + // + // Returns the length of this instance. The default length of a CordBuffer is + // 0, indicating an 'empty' CordBuffer. Applications must specify the length + // of the data in a CordBuffer before adding it to a Cord. + size_t length() const; + + // CordBuffer::capacity() + // + // Returns the capacity of this instance. All instances have a non-zero + // capacity: default and `moved from` instances have a small internal buffer. + size_t capacity() const; + + // CordBuffer::IncreaseLengthBy() + // + // Increases the length of this buffer by the specified 'n' bytes. + // Applications must make sure all data in this buffer up to the new length + // has been initialized before adding a CordBuffer to a Cord: failure to do so + // will lead to undefined behavior. Requires `length() + n <= capacity()`. + // Typically, applications will use 'available_up_to()` to get a span of the + // desired capacity, and use `span.size()` to increase the length as in: + // absl::Span<char> span = buffer.available_up_to(desired); + // buffer.IncreaseLengthBy(span.size()); + // memcpy(span.data(), src, span.size()); + // etc... + void IncreaseLengthBy(size_t n); + + // CordBuffer::SetLength() + // + // Sets the data length of this instance. Applications must make sure all data + // of the specified length has been initialized before adding a CordBuffer to + // a Cord: failure to do so will lead to undefined behavior. + // Setting the length to a small value or zero does not release any memory + // held by this CordBuffer instance. Requires `length <= capacity()`. + // Applications should preferably use the `IncreaseLengthBy()` method above + // in combination with the 'available()` or `available_up_to()` methods. + void SetLength(size_t length); + + private: + // Make sure we don't accidentally over promise. + static_assert(kCustomLimit <= cord_internal::kMaxLargeFlatSize, ""); + + // Assume the cost of an 'uprounded' allocation to CeilPow2(size) versus + // the cost of allocating at least 1 extra flat <= 4KB: + // - Flat overhead = 13 bytes + // - Btree amortized cost / node =~ 13 bytes + // - 64 byte granularity of tcmalloc at 4K =~ 32 byte average + // CPU cost and efficiency requires we should at least 'save' something by + // splitting, as a poor man's measure, we say the slop needs to be + // at least double the cost offset to make it worth splitting: ~128 bytes. + static constexpr size_t kMaxPageSlop = 128; + + // Overhead for allocation a flat. + static constexpr size_t kOverhead = cord_internal::kFlatOverhead; + + using CordRepFlat = cord_internal::CordRepFlat; + + // `Rep` is the internal data representation of a CordBuffer. The internal + // representation has an internal small size optimization similar to + // std::string (SSO). + struct Rep { + // Inline SSO size of a CordBuffer + static constexpr size_t kInlineCapacity = sizeof(intptr_t) * 2 - 1; + + // Creates a default instance with kInlineCapacity. + Rep() : short_rep{} {} + + // Creates an instance managing an allocated non zero CordRep. + explicit Rep(cord_internal::CordRepFlat* rep) : long_rep{rep} { + assert(rep != nullptr); + } + + // Returns true if this instance manages the SSO internal buffer. + bool is_short() const { + constexpr size_t offset = offsetof(Short, raw_size); + return (reinterpret_cast<const char*>(this)[offset] & 1) != 0; + } + + // Returns the available area of the internal SSO data + absl::Span<char> short_available() { + assert(is_short()); + const size_t length = (short_rep.raw_size >> 1); + return absl::Span<char>(short_rep.data + length, + kInlineCapacity - length); + } + + // Returns the available area of the internal SSO data + absl::Span<char> long_available() { + assert(!is_short()); + const size_t length = long_rep.rep->length; + return absl::Span<char>(long_rep.rep->Data() + length, + long_rep.rep->Capacity() - length); + } + + // Returns the length of the internal SSO data. + size_t short_length() const { + assert(is_short()); + return short_rep.raw_size >> 1; + } + + // Sets the length of the internal SSO data. + // Disregards any previously set CordRep instance. + void set_short_length(size_t length) { + short_rep.raw_size = static_cast<char>((length << 1) + 1); + } + + // Adds `n` to the current short length. + void add_short_length(size_t n) { + assert(is_short()); + short_rep.raw_size += static_cast<char>(n << 1); + } + + // Returns reference to the internal SSO data buffer. + char* data() { + assert(is_short()); + return short_rep.data; + } + const char* data() const { + assert(is_short()); + return short_rep.data; + } + + // Returns a pointer the external CordRep managed by this instance. + cord_internal::CordRepFlat* rep() const { + assert(!is_short()); + return long_rep.rep; + } + + // The internal representation takes advantage of the fact that allocated + // memory is always on an even address, and uses the least significant bit + // of the first or last byte (depending on endianness) as the inline size + // indicator overlapping with the least significant byte of the CordRep*. +#if defined(ABSL_IS_BIG_ENDIAN) + struct Long { + explicit Long(cord_internal::CordRepFlat* rep_arg) : rep(rep_arg) {} + void* padding; + cord_internal::CordRepFlat* rep; + }; + struct Short { + char data[sizeof(Long) - 1]; + char raw_size = 1; + }; +#else + struct Long { + explicit Long(cord_internal::CordRepFlat* rep_arg) : rep(rep_arg) {} + cord_internal::CordRepFlat* rep; + void* padding; + }; + struct Short { + char raw_size = 1; + char data[sizeof(Long) - 1]; + }; +#endif + + union { + Long long_rep; + Short short_rep; + }; + }; + + // Power2 functions + static bool IsPow2(size_t size) { return absl::has_single_bit(size); } + static size_t Log2Floor(size_t size) { return absl::bit_width(size) - 1; } + static size_t Log2Ceil(size_t size) { return absl::bit_width(size - 1); } + + // Implementation of `CreateWithCustomLimit()`. + // This implementation allows for future memory allocation hints to + // be passed down into the CordRepFlat allocation function. + template <typename... AllocationHints> + static CordBuffer CreateWithCustomLimitImpl(size_t block_size, + size_t capacity, + AllocationHints... hints); + + // Consumes the value contained in this instance and resets the instance. + // This method returns a non-null Cordrep* if the current instances manages a + // CordRep*, and resets the instance to an empty SSO instance. If the current + // instance is an SSO instance, then this method returns nullptr and sets + // `short_value` to the inlined data value. In either case, the current + // instance length is reset to zero. + // This method is intended to be used by Cord internal functions only. + cord_internal::CordRep* ConsumeValue(absl::string_view& short_value) { + cord_internal::CordRep* rep = nullptr; + if (rep_.is_short()) { + short_value = absl::string_view(rep_.data(), rep_.short_length()); + } else { + rep = rep_.rep(); + } + rep_.set_short_length(0); + return rep; + } + + // Internal constructor. + explicit CordBuffer(cord_internal::CordRepFlat* rep) : rep_(rep) { + assert(rep != nullptr); + } + + Rep rep_; + + friend class Cord; + friend class CordBufferTestPeer; +}; + +inline constexpr size_t CordBuffer::MaximumPayload() { + return cord_internal::kMaxFlatLength; +} + +inline constexpr size_t CordBuffer::MaximumPayload(size_t block_size) { + // TODO(absl-team): Use std::min when C++11 support is dropped. + return (kCustomLimit < block_size ? kCustomLimit : block_size) - + cord_internal::kFlatOverhead; +} + +inline CordBuffer CordBuffer::CreateWithDefaultLimit(size_t capacity) { + if (capacity > Rep::kInlineCapacity) { + auto* rep = cord_internal::CordRepFlat::New(capacity); + rep->length = 0; + return CordBuffer(rep); + } + return CordBuffer(); +} + +template <typename... AllocationHints> +inline CordBuffer CordBuffer::CreateWithCustomLimitImpl( + size_t block_size, size_t capacity, AllocationHints... hints) { + assert(IsPow2(block_size)); + capacity = (std::min)(capacity, kCustomLimit); + block_size = (std::min)(block_size, kCustomLimit); + if (capacity + kOverhead >= block_size) { + capacity = block_size; + } else if (capacity <= kDefaultLimit) { + capacity = capacity + kOverhead; + } else if (!IsPow2(capacity)) { + // Check if rounded up to next power 2 is a good enough fit + // with limited waste making it an acceptable direct fit. + const size_t rounded_up = size_t{1} << Log2Ceil(capacity); + const size_t slop = rounded_up - capacity; + if (slop >= kOverhead && slop <= kMaxPageSlop + kOverhead) { + capacity = rounded_up; + } else { + // Round down to highest power of 2 <= capacity. + // Consider a more aggressive step down if that may reduce the + // risk of fragmentation where 'people are holding it wrong'. + const size_t rounded_down = size_t{1} << Log2Floor(capacity); + capacity = rounded_down; + } + } + const size_t length = capacity - kOverhead; + auto* rep = CordRepFlat::New(CordRepFlat::Large(), length, hints...); + rep->length = 0; + return CordBuffer(rep); +} + +inline CordBuffer CordBuffer::CreateWithCustomLimit(size_t block_size, + size_t capacity) { + return CreateWithCustomLimitImpl(block_size, capacity); +} + +inline CordBuffer::~CordBuffer() { + if (!rep_.is_short()) { + cord_internal::CordRepFlat::Delete(rep_.rep()); + } +} + +inline CordBuffer::CordBuffer(CordBuffer&& rhs) noexcept : rep_(rhs.rep_) { + rhs.rep_.set_short_length(0); +} + +inline CordBuffer& CordBuffer::operator=(CordBuffer&& rhs) noexcept { + if (!rep_.is_short()) cord_internal::CordRepFlat::Delete(rep_.rep()); + rep_ = rhs.rep_; + rhs.rep_.set_short_length(0); + return *this; +} + +inline absl::Span<char> CordBuffer::available() { + return rep_.is_short() ? rep_.short_available() : rep_.long_available(); +} + +inline absl::Span<char> CordBuffer::available_up_to(size_t size) { + return available().subspan(0, size); +} + +inline char* CordBuffer::data() { + return rep_.is_short() ? rep_.data() : rep_.rep()->Data(); +} + +inline const char* CordBuffer::data() const { + return rep_.is_short() ? rep_.data() : rep_.rep()->Data(); +} + +inline size_t CordBuffer::capacity() const { + return rep_.is_short() ? Rep::kInlineCapacity : rep_.rep()->Capacity(); +} + +inline size_t CordBuffer::length() const { + return rep_.is_short() ? rep_.short_length() : rep_.rep()->length; +} + +inline void CordBuffer::SetLength(size_t length) { + ABSL_HARDENING_ASSERT(length <= capacity()); + if (rep_.is_short()) { + rep_.set_short_length(length); + } else { + rep_.rep()->length = length; + } +} + +inline void CordBuffer::IncreaseLengthBy(size_t n) { + ABSL_HARDENING_ASSERT(n <= capacity() && length() + n <= capacity()); + if (rep_.is_short()) { + rep_.add_short_length(n); + } else { + rep_.rep()->length += n; + } +} + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_CORD_BUFFER_H_ diff --git a/absl/strings/cord_buffer_test.cc b/absl/strings/cord_buffer_test.cc new file mode 100644 index 00000000..5c7437ae --- /dev/null +++ b/absl/strings/cord_buffer_test.cc @@ -0,0 +1,320 @@ +// Copyright 2021 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/cord_buffer.h" + + +#include <algorithm> +#include <climits> +#include <cstring> +#include <string> +#include <utility> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/config.h" +#include "absl/strings/internal/cord_rep_flat.h" +#include "absl/strings/internal/cord_rep_test_util.h" +#include "absl/types/span.h" + +using testing::Eq; +using testing::Ge; +using testing::Le; +using testing::Ne; + +namespace absl { +ABSL_NAMESPACE_BEGIN + +class CordBufferTestPeer { + public: + static cord_internal::CordRep* ConsumeValue(CordBuffer& buffer, + absl::string_view& short_value) { + return buffer.ConsumeValue(short_value); + } +}; + +namespace { + +using ::absl::cordrep_testing::CordToString; + +constexpr size_t kInlinedSize = sizeof(CordBuffer) - 1; +constexpr size_t kDefaultLimit = CordBuffer::kDefaultLimit; +constexpr size_t kCustomLimit = CordBuffer::kCustomLimit; +constexpr size_t kMaxFlatSize = cord_internal::kMaxFlatSize; +constexpr size_t kMaxFlatLength = cord_internal::kMaxFlatLength; +constexpr size_t kFlatOverhead = cord_internal::kFlatOverhead; + +constexpr size_t k8KiB = 8 << 10; +constexpr size_t k16KiB = 16 << 10; +constexpr size_t k64KiB = 64 << 10; +constexpr size_t k1MB = 1 << 20; + +class CordBufferTest : public testing::TestWithParam<size_t> {}; + +INSTANTIATE_TEST_SUITE_P(MediumSize, CordBufferTest, + testing::Values(1, kInlinedSize - 1, kInlinedSize, + kInlinedSize + 1, kDefaultLimit - 1, + kDefaultLimit)); + +TEST_P(CordBufferTest, MaximumPayload) { + EXPECT_THAT(CordBuffer::MaximumPayload(), Eq(kMaxFlatLength)); + EXPECT_THAT(CordBuffer::MaximumPayload(512), Eq(512 - kFlatOverhead)); + EXPECT_THAT(CordBuffer::MaximumPayload(k64KiB), Eq(k64KiB - kFlatOverhead)); + EXPECT_THAT(CordBuffer::MaximumPayload(k1MB), Eq(k64KiB - kFlatOverhead)); +} + +TEST(CordBufferTest, ConstructDefault) { + CordBuffer buffer; + EXPECT_THAT(buffer.capacity(), Eq(sizeof(CordBuffer) - 1)); + EXPECT_THAT(buffer.length(), Eq(0)); + EXPECT_THAT(buffer.data(), Ne(nullptr)); + EXPECT_THAT(buffer.available().data(), Eq(buffer.data())); + EXPECT_THAT(buffer.available().size(), Eq(buffer.capacity())); + memset(buffer.data(), 0xCD, buffer.capacity()); +} + +TEST(CordBufferTest, CreateSsoWithDefaultLimit) { + CordBuffer buffer = CordBuffer::CreateWithDefaultLimit(3); + EXPECT_THAT(buffer.capacity(), Ge(3)); + EXPECT_THAT(buffer.capacity(), Le(sizeof(CordBuffer))); + EXPECT_THAT(buffer.length(), Eq(0)); + memset(buffer.data(), 0xCD, buffer.capacity()); + + memcpy(buffer.data(), "Abc", 3); + buffer.SetLength(3); + EXPECT_THAT(buffer.length(), Eq(3)); + absl::string_view short_value; + EXPECT_THAT(CordBufferTestPeer::ConsumeValue(buffer, short_value), + Eq(nullptr)); + EXPECT_THAT(absl::string_view(buffer.data(), 3), Eq("Abc")); + EXPECT_THAT(short_value, Eq("Abc")); +} + +TEST_P(CordBufferTest, Available) { + const size_t requested = GetParam(); + CordBuffer buffer = CordBuffer::CreateWithDefaultLimit(requested); + EXPECT_THAT(buffer.available().data(), Eq(buffer.data())); + EXPECT_THAT(buffer.available().size(), Eq(buffer.capacity())); + + buffer.SetLength(2); + EXPECT_THAT(buffer.available().data(), Eq(buffer.data() + 2)); + EXPECT_THAT(buffer.available().size(), Eq(buffer.capacity() - 2)); +} + +TEST_P(CordBufferTest, IncreaseLengthBy) { + const size_t requested = GetParam(); + CordBuffer buffer = CordBuffer::CreateWithDefaultLimit(requested); + buffer.IncreaseLengthBy(2); + EXPECT_THAT(buffer.length(), Eq(2)); + buffer.IncreaseLengthBy(5); + EXPECT_THAT(buffer.length(), Eq(7)); +} + +TEST_P(CordBufferTest, AvailableUpTo) { + const size_t requested = GetParam(); + CordBuffer buffer = CordBuffer::CreateWithDefaultLimit(requested); + size_t expected_up_to = std::min<size_t>(3, buffer.capacity()); + EXPECT_THAT(buffer.available_up_to(3).data(), Eq(buffer.data())); + EXPECT_THAT(buffer.available_up_to(3).size(), Eq(expected_up_to)); + + buffer.SetLength(2); + expected_up_to = std::min<size_t>(3, buffer.capacity() - 2); + EXPECT_THAT(buffer.available_up_to(3).data(), Eq(buffer.data() + 2)); + EXPECT_THAT(buffer.available_up_to(3).size(), Eq(expected_up_to)); +} + +// Returns the maximum capacity for a given block_size and requested size. +size_t MaxCapacityFor(size_t block_size, size_t requested) { + requested = (std::min)(requested, cord_internal::kMaxLargeFlatSize); + // Maximum returned size is always capped at block_size - kFlatOverhead. + return block_size - kFlatOverhead; +} + +TEST_P(CordBufferTest, CreateWithDefaultLimit) { + const size_t requested = GetParam(); + CordBuffer buffer = CordBuffer::CreateWithDefaultLimit(requested); + EXPECT_THAT(buffer.capacity(), Ge(requested)); + EXPECT_THAT(buffer.capacity(), Le(MaxCapacityFor(kMaxFlatSize, requested))); + EXPECT_THAT(buffer.length(), Eq(0)); + + memset(buffer.data(), 0xCD, buffer.capacity()); + + std::string data(requested - 1, 'x'); + memcpy(buffer.data(), data.c_str(), requested); + buffer.SetLength(requested); + + EXPECT_THAT(buffer.length(), Eq(requested)); + EXPECT_THAT(absl::string_view(buffer.data()), Eq(data)); +} + +TEST(CordBufferTest, CreateWithDefaultLimitAskingFor2GB) { + constexpr size_t k2GiB = 1U << 31; + CordBuffer buffer = CordBuffer::CreateWithDefaultLimit(k2GiB); + // Expect to never be awarded more than a reasonable memory size, even in + // cases where a (debug) memory allocator may grant us somewhat more memory + // than `kDefaultLimit` which should be no more than `2 * kDefaultLimit` + EXPECT_THAT(buffer.capacity(), Le(2 * CordBuffer::kDefaultLimit)); + EXPECT_THAT(buffer.length(), Eq(0)); + EXPECT_THAT(buffer.data(), Ne(nullptr)); + memset(buffer.data(), 0xCD, buffer.capacity()); +} + +TEST_P(CordBufferTest, MoveConstruct) { + const size_t requested = GetParam(); + CordBuffer from = CordBuffer::CreateWithDefaultLimit(requested); + const size_t capacity = from.capacity(); + memcpy(from.data(), "Abc", 4); + from.SetLength(4); + + CordBuffer to(std::move(from)); + EXPECT_THAT(to.capacity(), Eq(capacity)); + EXPECT_THAT(to.length(), Eq(4)); + EXPECT_THAT(absl::string_view(to.data()), Eq("Abc")); + + EXPECT_THAT(from.length(), Eq(0)); // NOLINT +} + +TEST_P(CordBufferTest, MoveAssign) { + const size_t requested = GetParam(); + CordBuffer from = CordBuffer::CreateWithDefaultLimit(requested); + const size_t capacity = from.capacity(); + memcpy(from.data(), "Abc", 4); + from.SetLength(4); + + CordBuffer to; + to = std::move(from); + EXPECT_THAT(to.capacity(), Eq(capacity)); + EXPECT_THAT(to.length(), Eq(4)); + EXPECT_THAT(absl::string_view(to.data()), Eq("Abc")); + + EXPECT_THAT(from.length(), Eq(0)); // NOLINT +} + +TEST_P(CordBufferTest, ConsumeValue) { + const size_t requested = GetParam(); + CordBuffer buffer = CordBuffer::CreateWithDefaultLimit(requested); + memcpy(buffer.data(), "Abc", 4); + buffer.SetLength(3); + + absl::string_view short_value; + if (cord_internal::CordRep* rep = + CordBufferTestPeer::ConsumeValue(buffer, short_value)) { + EXPECT_THAT(CordToString(rep), Eq("Abc")); + cord_internal::CordRep::Unref(rep); + } else { + EXPECT_THAT(short_value, Eq("Abc")); + } + EXPECT_THAT(buffer.length(), Eq(0)); +} + +TEST_P(CordBufferTest, CreateWithCustomLimitWithinDefaultLimit) { + const size_t requested = GetParam(); + CordBuffer buffer = + CordBuffer::CreateWithCustomLimit(kMaxFlatSize, requested); + EXPECT_THAT(buffer.capacity(), Ge(requested)); + EXPECT_THAT(buffer.capacity(), Le(MaxCapacityFor(kMaxFlatSize, requested))); + EXPECT_THAT(buffer.length(), Eq(0)); + + memset(buffer.data(), 0xCD, buffer.capacity()); + + std::string data(requested - 1, 'x'); + memcpy(buffer.data(), data.c_str(), requested); + buffer.SetLength(requested); + + EXPECT_THAT(buffer.length(), Eq(requested)); + EXPECT_THAT(absl::string_view(buffer.data()), Eq(data)); +} + +TEST(CordLargeBufferTest, CreateAtOrBelowDefaultLimit) { + CordBuffer buffer = CordBuffer::CreateWithCustomLimit(k64KiB, kDefaultLimit); + EXPECT_THAT(buffer.capacity(), Ge(kDefaultLimit)); + EXPECT_THAT(buffer.capacity(), + Le(MaxCapacityFor(kMaxFlatSize, kDefaultLimit))); + + buffer = CordBuffer::CreateWithCustomLimit(k64KiB, 3178); + EXPECT_THAT(buffer.capacity(), Ge(3178)); +} + +TEST(CordLargeBufferTest, CreateWithCustomLimit) { + ASSERT_THAT((kMaxFlatSize & (kMaxFlatSize - 1)) == 0, "Must be power of 2"); + + for (size_t size = kMaxFlatSize; size <= kCustomLimit; size *= 2) { + CordBuffer buffer = CordBuffer::CreateWithCustomLimit(size, size); + size_t expected = size - kFlatOverhead; + ASSERT_THAT(buffer.capacity(), Ge(expected)); + EXPECT_THAT(buffer.capacity(), Le(MaxCapacityFor(size, expected))); + } +} + +TEST(CordLargeBufferTest, CreateWithTooLargeLimit) { + CordBuffer buffer = CordBuffer::CreateWithCustomLimit(k64KiB, k1MB); + ASSERT_THAT(buffer.capacity(), Ge(k64KiB - kFlatOverhead)); + EXPECT_THAT(buffer.capacity(), Le(MaxCapacityFor(k64KiB, k1MB))); +} + +TEST(CordLargeBufferTest, CreateWithHugeValueForOverFlowHardening) { + for (size_t dist_from_max = 0; dist_from_max <= 32; ++dist_from_max) { + size_t capacity = std::numeric_limits<size_t>::max() - dist_from_max; + + CordBuffer buffer = CordBuffer::CreateWithDefaultLimit(capacity); + ASSERT_THAT(buffer.capacity(), Ge(kDefaultLimit)); + EXPECT_THAT(buffer.capacity(), Le(MaxCapacityFor(kMaxFlatSize, capacity))); + + for (size_t limit = kMaxFlatSize; limit <= kCustomLimit; limit *= 2) { + CordBuffer buffer = CordBuffer::CreateWithCustomLimit(limit, capacity); + ASSERT_THAT(buffer.capacity(), Ge(limit - kFlatOverhead)); + EXPECT_THAT(buffer.capacity(), Le(MaxCapacityFor(limit, capacity))); + } + } +} + +TEST(CordLargeBufferTest, CreateWithSmallLimit) { + CordBuffer buffer = CordBuffer::CreateWithCustomLimit(512, 1024); + ASSERT_THAT(buffer.capacity(), Ge(512 - kFlatOverhead)); + EXPECT_THAT(buffer.capacity(), Le(MaxCapacityFor(512, 1024))); + + // Ask for precise block size, should return size - kOverhead + buffer = CordBuffer::CreateWithCustomLimit(512, 512); + ASSERT_THAT(buffer.capacity(), Ge(512 - kFlatOverhead)); + EXPECT_THAT(buffer.capacity(), Le(MaxCapacityFor(512, 512))); + + // Corner case: 511 < block_size, but 511 + kOverhead is above + buffer = CordBuffer::CreateWithCustomLimit(512, 511); + ASSERT_THAT(buffer.capacity(), Ge(512 - kFlatOverhead)); + EXPECT_THAT(buffer.capacity(), Le(MaxCapacityFor(512, 511))); + + // Corner case: 498 + kOverhead < block_size + buffer = CordBuffer::CreateWithCustomLimit(512, 498); + ASSERT_THAT(buffer.capacity(), Ge(512 - kFlatOverhead)); + EXPECT_THAT(buffer.capacity(), Le(MaxCapacityFor(512, 498))); +} + +TEST(CordLargeBufferTest, CreateWasteFull) { + // 15 KiB gets rounded down to next pow2 value. + const size_t requested = (15 << 10); + CordBuffer buffer = CordBuffer::CreateWithCustomLimit(k16KiB, requested); + ASSERT_THAT(buffer.capacity(), Ge(k8KiB - kFlatOverhead)); + EXPECT_THAT(buffer.capacity(), Le(MaxCapacityFor(k8KiB, requested))); +} + +TEST(CordLargeBufferTest, CreateSmallSlop) { + const size_t requested = k16KiB - 2 * kFlatOverhead; + CordBuffer buffer = CordBuffer::CreateWithCustomLimit(k16KiB, requested); + ASSERT_THAT(buffer.capacity(), Ge(k16KiB - kFlatOverhead)); + EXPECT_THAT(buffer.capacity(), Le(MaxCapacityFor(k16KiB, requested))); +} + +} // namespace +ABSL_NAMESPACE_END +} // namespace absl diff --git a/absl/strings/cord_ring_reader_test.cc b/absl/strings/cord_ring_reader_test.cc index 585616f3..8e7183bf 100644 --- a/absl/strings/cord_ring_reader_test.cc +++ b/absl/strings/cord_ring_reader_test.cc @@ -78,6 +78,7 @@ TEST(CordRingReaderTest, Reset) { EXPECT_TRUE(static_cast<bool>(reader)); EXPECT_THAT(reader.ring(), Eq(ring)); EXPECT_THAT(reader.index(), Eq(ring->head())); + EXPECT_THAT(reader.node(), Eq(ring->entry_child(ring->head()))); EXPECT_THAT(reader.length(), Eq(ring->length)); EXPECT_THAT(reader.consumed(), Eq(flats[0].length())); EXPECT_THAT(reader.remaining(), Eq(ring->length - reader.consumed())); @@ -99,11 +100,13 @@ TEST(CordRingReaderTest, Next) { size_t consumed = reader.consumed(); size_t remaining = reader.remaining(); for (int i = 1; i < flats.size(); ++i) { + CordRepRing::index_type index = ring->advance(head, i); consumed += flats[i].length(); remaining -= flats[i].length(); absl::string_view next = reader.Next(); ASSERT_THAT(next, Eq(flats[i])); - ASSERT_THAT(reader.index(), Eq(ring->advance(head, i))); + ASSERT_THAT(reader.index(), Eq(index)); + ASSERT_THAT(reader.node(), Eq(ring->entry_child(index))); ASSERT_THAT(reader.consumed(), Eq(consumed)); ASSERT_THAT(reader.remaining(), Eq(remaining)); } @@ -123,15 +126,17 @@ TEST(CordRingReaderTest, SeekForward) { reader.Reset(ring); size_t consumed = 0; - size_t remaining = ring->length;; + size_t remaining = ring->length; for (int i = 0; i < flats.size(); ++i) { + CordRepRing::index_type index = ring->advance(head, i); size_t offset = consumed; consumed += flats[i].length(); remaining -= flats[i].length(); for (int off = 0; off < flats[i].length(); ++off) { absl::string_view chunk = reader.Seek(offset + off); ASSERT_THAT(chunk, Eq(flats[i].substr(off))); - ASSERT_THAT(reader.index(), Eq(ring->advance(head, i))); + ASSERT_THAT(reader.index(), Eq(index)); + ASSERT_THAT(reader.node(), Eq(ring->entry_child(index))); ASSERT_THAT(reader.consumed(), Eq(consumed)); ASSERT_THAT(reader.remaining(), Eq(remaining)); } @@ -150,11 +155,13 @@ TEST(CordRingReaderTest, SeekBackward) { size_t consumed = ring->length; size_t remaining = 0; for (int i = flats.size() - 1; i >= 0; --i) { + CordRepRing::index_type index = ring->advance(head, i); size_t offset = consumed - flats[i].length(); for (int off = 0; off < flats[i].length(); ++off) { absl::string_view chunk = reader.Seek(offset + off); ASSERT_THAT(chunk, Eq(flats[i].substr(off))); - ASSERT_THAT(reader.index(), Eq(ring->advance(head, i))); + ASSERT_THAT(reader.index(), Eq(index)); + ASSERT_THAT(reader.node(), Eq(ring->entry_child(index))); ASSERT_THAT(reader.consumed(), Eq(consumed)); ASSERT_THAT(reader.remaining(), Eq(remaining)); } diff --git a/absl/strings/cord_ring_test.cc b/absl/strings/cord_ring_test.cc index 7d75e106..f39a0a4f 100644 --- a/absl/strings/cord_ring_test.cc +++ b/absl/strings/cord_ring_test.cc @@ -31,9 +31,6 @@ extern thread_local bool cord_ring; -// TOOD(b/177688959): weird things happened with the original test -#define ASAN_BUG_177688959_FIXED false - namespace absl { ABSL_NAMESPACE_BEGIN namespace { @@ -47,7 +44,6 @@ using ::absl::cord_internal::CordRepFlat; using ::absl::cord_internal::CordRepRing; using ::absl::cord_internal::CordRepSubstring; -using ::absl::cord_internal::CONCAT; using ::absl::cord_internal::EXTERNAL; using ::absl::cord_internal::SUBSTRING; @@ -101,15 +97,22 @@ using TestParams = std::vector<TestParam>; // Matcher validating when mutable copies are required / performed. MATCHER_P2(EqIfPrivate, param, rep, absl::StrCat("Equal 0x", absl::Hex(rep), " if private")) { - return param.refcount_is_one ? arg == rep : arg != rep; + return param.refcount_is_one ? arg == rep : true; } // Matcher validating when mutable copies are required / performed. MATCHER_P2(EqIfPrivateAndCapacity, param, rep, absl::StrCat("Equal 0x", absl::Hex(rep), " if private and capacity")) { - return (param.refcount_is_one && param.with_capacity) ? arg == rep - : arg != rep; + return (param.refcount_is_one && param.with_capacity) ? arg == rep : true; +} + +// Matcher validating a shared ring was re-allocated. Should only be used for +// tests doing exactly one update as subsequent updates could return the +// original (freed and re-used) pointer. +MATCHER_P2(NeIfShared, param, rep, + absl::StrCat("Not equal 0x", absl::Hex(rep), " if shared")) { + return param.refcount_is_one ? true : arg != rep; } MATCHER_P2(EqIfInputPrivate, param, rep, "Equal if input is private") { @@ -219,7 +222,7 @@ CordRepExternal* MakeFakeExternal(size_t length) { std::string s; explicit Rep(size_t len) { this->tag = EXTERNAL; - this->base = this->storage; + this->base = reinterpret_cast<const char*>(this->storage); this->length = len; this->releaser_invoker = [](CordRepExternal* self) { delete static_cast<Rep*>(self); @@ -258,20 +261,10 @@ CordRepSubstring* RemoveSuffix(size_t length, CordRep* rep) { return MakeSubstring(0, rep->length - length, rep); } -CordRepConcat* MakeConcat(CordRep* left, CordRep* right, int depth = 0) { - auto* concat = new CordRepConcat; - concat->tag = CONCAT; - concat->length = left->length + right->length; - concat->left = left; - concat->right = right; - concat->set_depth(depth); - return concat; -} - enum Composition { kMix, kAppend, kPrepend }; Composition RandomComposition() { - RandomEngine rng(testing::GTEST_FLAG(random_seed)); + RandomEngine rng(GTEST_FLAG_GET(random_seed)); return (rng() & 1) ? kMix : ((rng() & 1) ? kAppend : kPrepend); } @@ -292,7 +285,6 @@ constexpr const char* kFox = "The quick brown fox jumps over the lazy dog"; constexpr const char* kFoxFlats[] = {"The ", "quick ", "brown ", "fox ", "jumps ", "over ", "the ", "lazy ", "dog"}; -constexpr const char* kAlphabet = "abcdefghijklmnopqrstuvwxyz"; CordRepRing* FromFlats(Span<const char* const> flats, Composition composition = kAppend) { @@ -340,19 +332,15 @@ std::string TestParamToString(const testing::TestParamInfo<TestParam>& info) { class CordRingTest : public testing::Test { public: ~CordRingTest() override { -#if ASAN_BUG_177688959_FIXED for (CordRep* rep : unrefs_) { CordRep::Unref(rep); } -#endif } template <typename CordRepType> CordRepType* NeedsUnref(CordRepType* rep) { assert(rep); -#if ASAN_BUG_177688959_FIXED unrefs_.push_back(rep); -#endif return rep; } @@ -362,26 +350,16 @@ class CordRingTest : public testing::Test { return NeedsUnref(rep); } - void Unref(CordRep* rep) { -#if !ASAN_BUG_177688959_FIXED - CordRep::Unref(rep); -#endif - } - private: -#if ASAN_BUG_177688959_FIXED std::vector<CordRep*> unrefs_; -#endif }; class CordRingTestWithParam : public testing::TestWithParam<TestParam> { public: ~CordRingTestWithParam() override { -#if ASAN_BUG_177688959_FIXED for (CordRep* rep : unrefs_) { CordRep::Unref(rep); } -#endif } CordRepRing* CreateWithCapacity(CordRep* child, size_t extra_capacity) { @@ -400,9 +378,7 @@ class CordRingTestWithParam : public testing::TestWithParam<TestParam> { template <typename CordRepType> CordRepType* NeedsUnref(CordRepType* rep) { assert(rep); -#if ASAN_BUG_177688959_FIXED unrefs_.push_back(rep); -#endif return rep; } @@ -412,43 +388,23 @@ class CordRingTestWithParam : public testing::TestWithParam<TestParam> { return NeedsUnref(rep); } - void Unref(CordRep* rep) { -#if !ASAN_BUG_177688959_FIXED - CordRep::Unref(rep); -#endif - } - template <typename CordRepType> CordRepType* RefIfShared(CordRepType* rep) { return Shared() ? Ref(rep) : rep; } - void UnrefIfShared(CordRep* rep) { - if (Shared()) Unref(rep); - } - template <typename CordRepType> CordRepType* RefIfInputShared(CordRepType* rep) { return InputShared() ? Ref(rep) : rep; } - void UnrefIfInputShared(CordRep* rep) { - if (InputShared()) Unref(rep); - } - template <typename CordRepType> CordRepType* RefIfInputSharedIndirect(CordRepType* rep) { return InputSharedIndirect() ? Ref(rep) : rep; } - void UnrefIfInputSharedIndirect(CordRep* rep) { - if (InputSharedIndirect()) Unref(rep); - } - private: -#if ASAN_BUG_177688959_FIXED std::vector<CordRep*> unrefs_; -#endif }; class CordRingCreateTest : public CordRingTestWithParam { @@ -520,26 +476,26 @@ class CordRingBuildInputTest : public CordRingTestWithParam { } }; -INSTANTIATE_TEST_CASE_P(WithParam, CordRingSubTest, - testing::ValuesIn(CordRingSubTest::CreateTestParams()), - TestParamToString); +INSTANTIATE_TEST_SUITE_P(WithParam, CordRingSubTest, + testing::ValuesIn(CordRingSubTest::CreateTestParams()), + TestParamToString); -INSTANTIATE_TEST_CASE_P( +INSTANTIATE_TEST_SUITE_P( WithParam, CordRingCreateTest, testing::ValuesIn(CordRingCreateTest::CreateTestParams()), TestParamToString); -INSTANTIATE_TEST_CASE_P( +INSTANTIATE_TEST_SUITE_P( WithParam, CordRingCreateFromTreeTest, testing::ValuesIn(CordRingCreateFromTreeTest::CreateTestParams()), TestParamToString); -INSTANTIATE_TEST_CASE_P( +INSTANTIATE_TEST_SUITE_P( WithParam, CordRingBuildTest, testing::ValuesIn(CordRingBuildTest::CreateTestParams()), TestParamToString); -INSTANTIATE_TEST_CASE_P( +INSTANTIATE_TEST_SUITE_P( WithParam, CordRingBuildInputTest, testing::ValuesIn(CordRingBuildInputTest::CreateTestParams()), TestParamToString); @@ -550,7 +506,6 @@ TEST_P(CordRingCreateTest, CreateFromFlat) { ASSERT_THAT(result, IsValidRingBuffer()); EXPECT_THAT(result->length, Eq(str1.size())); EXPECT_THAT(ToFlats(result), ElementsAre(str1)); - Unref(result); } TEST_P(CordRingCreateTest, CreateFromRing) { @@ -558,9 +513,8 @@ TEST_P(CordRingCreateTest, CreateFromRing) { CordRepRing* result = NeedsUnref(CordRepRing::Create(ring)); ASSERT_THAT(result, IsValidRingBuffer()); EXPECT_THAT(result, EqIfPrivate(GetParam(), ring)); + EXPECT_THAT(result, NeIfShared(GetParam(), ring)); EXPECT_THAT(ToFlats(result), ElementsAreArray(kFoxFlats)); - UnrefIfShared(ring); - Unref(result); } TEST_P(CordRingCreateFromTreeTest, CreateFromSubstringRing) { @@ -570,23 +524,20 @@ TEST_P(CordRingCreateFromTreeTest, CreateFromSubstringRing) { ASSERT_THAT(result, IsValidRingBuffer()); EXPECT_THAT(result, EqIfInputPrivate(GetParam(), ring)); EXPECT_THAT(ToString(result), string_view(kFox).substr(2, 11)); - UnrefIfInputSharedIndirect(ring); - UnrefIfInputShared(sub); - Unref(result); } TEST_F(CordRingTest, CreateWithIllegalExtraCapacity) { - CordRep* flat = NeedsUnref(MakeFlat("Hello world")); #if defined(ABSL_HAVE_EXCEPTIONS) + CordRep* flat = NeedsUnref(MakeFlat("Hello world")); try { CordRepRing::Create(flat, CordRepRing::kMaxCapacity); GTEST_FAIL() << "expected std::length_error exception"; } catch (const std::length_error&) { } #elif defined(GTEST_HAS_DEATH_TEST) + CordRep* flat = NeedsUnref(MakeFlat("Hello world")); EXPECT_DEATH(CordRepRing::Create(flat, CordRepRing::kMaxCapacity), ".*"); #endif - Unref(flat); } TEST_P(CordRingCreateFromTreeTest, CreateFromSubstringOfFlat) { @@ -597,9 +548,6 @@ TEST_P(CordRingCreateFromTreeTest, CreateFromSubstringOfFlat) { ASSERT_THAT(result, IsValidRingBuffer()); EXPECT_THAT(result->length, Eq(20)); EXPECT_THAT(ToFlats(result), ElementsAre(str1.substr(4, 20))); - Unref(result); - UnrefIfInputShared(flat); - UnrefIfInputSharedIndirect(child); } TEST_P(CordRingCreateTest, CreateFromExternal) { @@ -609,8 +557,6 @@ TEST_P(CordRingCreateTest, CreateFromExternal) { ASSERT_THAT(result, IsValidRingBuffer()); EXPECT_THAT(result->length, Eq(str1.size())); EXPECT_THAT(ToFlats(result), ElementsAre(str1)); - Unref(result); - UnrefIfInputShared(child); } TEST_P(CordRingCreateFromTreeTest, CreateFromSubstringOfExternal) { @@ -621,9 +567,6 @@ TEST_P(CordRingCreateFromTreeTest, CreateFromSubstringOfExternal) { ASSERT_THAT(result, IsValidRingBuffer()); EXPECT_THAT(result->length, Eq(24)); EXPECT_THAT(ToFlats(result), ElementsAre(str1.substr(1, 24))); - Unref(result); - UnrefIfInputShared(external); - UnrefIfInputSharedIndirect(child); } TEST_P(CordRingCreateFromTreeTest, CreateFromSubstringOfLargeExternal) { @@ -637,46 +580,6 @@ TEST_P(CordRingCreateFromTreeTest, CreateFromSubstringOfLargeExternal) { ASSERT_THAT(result, IsValidRingBuffer()); EXPECT_THAT(result->length, Eq(str.size())); EXPECT_THAT(ToRawFlats(result), ElementsAre(str)); - Unref(result); - UnrefIfInputShared(external); - UnrefIfInputSharedIndirect(child); -} - -TEST_P(CordRingBuildInputTest, CreateFromConcat) { - CordRep* flats[] = {MakeFlat("abcdefgh"), MakeFlat("ijklm"), - MakeFlat("nopqrstuv"), MakeFlat("wxyz")}; - auto* left = MakeConcat(RefIfInputSharedIndirect(flats[0]), flats[1]); - auto* right = MakeConcat(flats[2], RefIfInputSharedIndirect(flats[3])); - auto* concat = RefIfInputShared(MakeConcat(left, right)); - CordRepRing* result = NeedsUnref(CordRepRing::Create(concat)); - ASSERT_THAT(result, IsValidRingBuffer()); - EXPECT_THAT(result->length, Eq(26)); - EXPECT_THAT(ToString(result), Eq(kAlphabet)); - UnrefIfInputSharedIndirect(flats[0]); - UnrefIfInputSharedIndirect(flats[3]); - UnrefIfInputShared(concat); - Unref(result); -} - -TEST_P(CordRingBuildInputTest, CreateFromSubstringConcat) { - for (size_t off = 0; off < 26; ++off) { - for (size_t len = 1; len < 26 - off; ++len) { - CordRep* flats[] = {MakeFlat("abcdefgh"), MakeFlat("ijklm"), - MakeFlat("nopqrstuv"), MakeFlat("wxyz")}; - auto* left = MakeConcat(RefIfInputSharedIndirect(flats[0]), flats[1]); - auto* right = MakeConcat(flats[2], RefIfInputSharedIndirect(flats[3])); - auto* concat = MakeConcat(left, right); - auto* child = RefIfInputShared(MakeSubstring(off, len, concat)); - CordRepRing* result = NeedsUnref(CordRepRing::Create(child)); - ASSERT_THAT(result, IsValidRingBuffer()); - ASSERT_THAT(result->length, Eq(len)); - ASSERT_THAT(ToString(result), string_view(kAlphabet).substr(off, len)); - UnrefIfInputSharedIndirect(flats[0]); - UnrefIfInputSharedIndirect(flats[3]); - UnrefIfInputShared(child); - Unref(result); - } - } } TEST_P(CordRingCreateTest, Properties) { @@ -689,7 +592,6 @@ TEST_P(CordRingCreateTest, Properties) { EXPECT_THAT(result->capacity(), Le(2 * 120 + 1)); EXPECT_THAT(result->entries(), Eq(1)); EXPECT_THAT(result->begin_pos(), Eq(0)); - Unref(result); } TEST_P(CordRingCreateTest, EntryForNewFlat) { @@ -700,7 +602,6 @@ TEST_P(CordRingCreateTest, EntryForNewFlat) { EXPECT_THAT(result->entry_child(0), Eq(child)); EXPECT_THAT(result->entry_end_pos(0), Eq(str1.length())); EXPECT_THAT(result->entry_data_offset(0), Eq(0)); - Unref(result); } TEST_P(CordRingCreateTest, EntryForNewFlatSubstring) { @@ -712,7 +613,6 @@ TEST_P(CordRingCreateTest, EntryForNewFlatSubstring) { EXPECT_THAT(result->entry_child(0), Eq(child)); EXPECT_THAT(result->entry_end_pos(0), Eq(26)); EXPECT_THAT(result->entry_data_offset(0), Eq(10)); - Unref(result); } TEST_P(CordRingBuildTest, AppendFlat) { @@ -722,10 +622,9 @@ TEST_P(CordRingBuildTest, AppendFlat) { CordRepRing* result = NeedsUnref(CordRepRing::Append(ring, MakeFlat(str2))); ASSERT_THAT(result, IsValidRingBuffer()); EXPECT_THAT(result, EqIfPrivateAndCapacity(GetParam(), ring)); + EXPECT_THAT(result, NeIfShared(GetParam(), ring)); EXPECT_THAT(result->length, Eq(str1.size() + str2.size())); EXPECT_THAT(ToFlats(result), ElementsAre(str1, str2)); - UnrefIfShared(ring); - Unref(result); } TEST_P(CordRingBuildTest, PrependFlat) { @@ -735,10 +634,9 @@ TEST_P(CordRingBuildTest, PrependFlat) { CordRepRing* result = NeedsUnref(CordRepRing::Prepend(ring, MakeFlat(str2))); ASSERT_THAT(result, IsValidRingBuffer()); EXPECT_THAT(result, EqIfPrivateAndCapacity(GetParam(), ring)); + EXPECT_THAT(result, NeIfShared(GetParam(), ring)); EXPECT_THAT(result->length, Eq(str1.size() + str2.size())); EXPECT_THAT(ToFlats(result), ElementsAre(str2, str1)); - UnrefIfShared(ring); - Unref(result); } TEST_P(CordRingBuildTest, AppendString) { @@ -748,10 +646,9 @@ TEST_P(CordRingBuildTest, AppendString) { CordRepRing* result = NeedsUnref(CordRepRing::Append(ring, str2)); ASSERT_THAT(result, IsValidRingBuffer()); EXPECT_THAT(result, EqIfPrivateAndCapacity(GetParam(), ring)); + EXPECT_THAT(result, NeIfShared(GetParam(), ring)); EXPECT_THAT(result->length, Eq(str1.size() + str2.size())); EXPECT_THAT(ToFlats(result), ElementsAre(str1, str2)); - UnrefIfShared(ring); - Unref(result); } TEST_P(CordRingBuildTest, AppendStringHavingExtra) { @@ -762,8 +659,7 @@ TEST_P(CordRingBuildTest, AppendStringHavingExtra) { ASSERT_THAT(result, IsValidRingBuffer()); EXPECT_THAT(result->length, Eq(str1.size() + str2.size())); EXPECT_THAT(result, EqIfPrivate(GetParam(), ring)); - UnrefIfShared(ring); - Unref(result); + EXPECT_THAT(result, NeIfShared(GetParam(), ring)); } TEST_P(CordRingBuildTest, AppendStringHavingPartialExtra) { @@ -785,13 +681,12 @@ TEST_P(CordRingBuildTest, AppendStringHavingPartialExtra) { ASSERT_THAT(result, IsValidRingBuffer()); EXPECT_THAT(result->length, Eq(str1.size() + str2.size())); EXPECT_THAT(result, EqIfPrivateAndCapacity(GetParam(), ring)); + EXPECT_THAT(result, NeIfShared(GetParam(), ring)); if (GetParam().refcount_is_one) { EXPECT_THAT(ToFlats(result), ElementsAre(StrCat(str1, str1a), str2a)); } else { EXPECT_THAT(ToFlats(result), ElementsAre(str1, str2)); } - UnrefIfShared(ring); - Unref(result); } TEST_P(CordRingBuildTest, AppendStringHavingExtraInSubstring) { @@ -802,14 +697,13 @@ TEST_P(CordRingBuildTest, AppendStringHavingExtraInSubstring) { CordRepRing* result = NeedsUnref(CordRepRing::Append(ring, str2)); ASSERT_THAT(result, IsValidRingBuffer()); EXPECT_THAT(result, EqIfPrivate(GetParam(), ring)); + EXPECT_THAT(result, NeIfShared(GetParam(), ring)); EXPECT_THAT(result->length, Eq(4 + str2.size())); if (GetParam().refcount_is_one) { EXPECT_THAT(ToFlats(result), ElementsAre(StrCat("1234", str2))); } else { EXPECT_THAT(ToFlats(result), ElementsAre("1234", str2)); } - UnrefIfShared(ring); - Unref(result); } TEST_P(CordRingBuildTest, AppendStringHavingSharedExtra) { @@ -837,10 +731,9 @@ TEST_P(CordRingBuildTest, AppendStringHavingSharedExtra) { CordRepRing* result = NeedsUnref(CordRepRing::Append(ring, str2)); ASSERT_THAT(result, IsValidRingBuffer()); EXPECT_THAT(result, EqIfPrivateAndCapacity(GetParam(), ring)); + EXPECT_THAT(result, NeIfShared(GetParam(), ring)); EXPECT_THAT(result->length, Eq(4 + str2.size())); EXPECT_THAT(ToFlats(result), ElementsAre("1234", str2)); - UnrefIfShared(ring); - Unref(result); CordRep::Unref(shared_type == 1 ? flat1 : flat); } @@ -857,8 +750,6 @@ TEST_P(CordRingBuildTest, AppendStringWithExtra) { EXPECT_THAT(result->length, Eq(str1.size() + str2.size() + str3.size())); EXPECT_THAT(result, EqIfPrivateAndCapacity(GetParam(), ring)); EXPECT_THAT(ToFlats(result), ElementsAre(str1, StrCat(str2, str3))); - UnrefIfShared(ring); - Unref(result); } TEST_P(CordRingBuildTest, PrependString) { @@ -875,8 +766,6 @@ TEST_P(CordRingBuildTest, PrependString) { } EXPECT_THAT(result->length, Eq(str1.size() + str2.size())); EXPECT_THAT(ToFlats(result), ElementsAre(str2, str1)); - UnrefIfShared(ring); - Unref(result); } TEST_P(CordRingBuildTest, PrependStringHavingExtra) { @@ -887,14 +776,13 @@ TEST_P(CordRingBuildTest, PrependStringHavingExtra) { CordRepRing* result = NeedsUnref(CordRepRing::Prepend(ring, str2)); ASSERT_THAT(result, IsValidRingBuffer()); EXPECT_THAT(result, EqIfPrivate(GetParam(), ring)); + EXPECT_THAT(result, NeIfShared(GetParam(), ring)); EXPECT_THAT(result->length, Eq(4 + str2.size())); if (GetParam().refcount_is_one) { EXPECT_THAT(ToFlats(result), ElementsAre(StrCat(str2, "1234"))); } else { EXPECT_THAT(ToFlats(result), ElementsAre(str2, "1234")); } - UnrefIfShared(ring); - Unref(result); } TEST_P(CordRingBuildTest, PrependStringHavingSharedExtra) { @@ -920,9 +808,8 @@ TEST_P(CordRingBuildTest, PrependStringHavingSharedExtra) { ASSERT_THAT(result, IsValidRingBuffer()); EXPECT_THAT(result->length, Eq(str1a.size() + str2.size())); EXPECT_THAT(result, EqIfPrivateAndCapacity(GetParam(), ring)); + EXPECT_THAT(result, NeIfShared(GetParam(), ring)); EXPECT_THAT(ToFlats(result), ElementsAre(str2, str1a)); - UnrefIfShared(ring); - Unref(result); CordRep::Unref(shared_type == 1 ? flat1 : flat); } } @@ -938,8 +825,6 @@ TEST_P(CordRingBuildTest, PrependStringWithExtra) { EXPECT_THAT(result->length, Eq(str1.size() + str2.size() + str3.size())); EXPECT_THAT(result, EqIfPrivateAndCapacity(GetParam(), ring)); EXPECT_THAT(ToFlats(result), ElementsAre(StrCat(str3, str2), str1)); - UnrefIfShared(ring); - Unref(result); } TEST_P(CordRingBuildTest, AppendPrependStringMix) { @@ -950,12 +835,10 @@ TEST_P(CordRingBuildTest, AppendPrependStringMix) { result = CordRepRing::Prepend(result, flats[4 - i]); result = CordRepRing::Append(result, flats[4 + i]); } - UnrefIfShared(ring); NeedsUnref(result); ASSERT_THAT(result, IsValidRingBuffer()); EXPECT_THAT(result, EqIfPrivateAndCapacity(GetParam(), ring)); EXPECT_THAT(ToString(result), kFox); - Unref(result); } TEST_P(CordRingBuildTest, AppendPrependStringMixWithExtra) { @@ -976,8 +859,6 @@ TEST_P(CordRingBuildTest, AppendPrependStringMixWithExtra) { EXPECT_THAT(ToFlats(result), ElementsAre("The quick brown fox ", "jumps ", "over the lazy dog")); } - UnrefIfShared(ring); - Unref(result); } TEST_P(CordRingBuildTest, AppendPrependStringMixWithPrependedExtra) { @@ -998,8 +879,6 @@ TEST_P(CordRingBuildTest, AppendPrependStringMixWithPrependedExtra) { EXPECT_THAT(ToFlats(result), ElementsAre("The quick brown fox ", "jumps ", "over the lazy dog")); } - UnrefIfShared(ring); - Unref(result); } TEST_P(CordRingSubTest, SubRing) { @@ -1011,16 +890,14 @@ TEST_P(CordRingSubTest, SubRing) { CordRepRing* ring = RefIfShared(FromFlats(flats, composition)); CordRepRing* result = CordRepRing::SubRing(ring, offset, 0); EXPECT_THAT(result, nullptr); - UnrefIfShared(ring); for (size_t len = 1; len < all.size() - offset; ++len) { ring = RefIfShared(FromFlats(flats, composition)); result = NeedsUnref(CordRepRing::SubRing(ring, offset, len)); ASSERT_THAT(result, IsValidRingBuffer()); ASSERT_THAT(result, EqIfPrivate(GetParam(), ring)); + ASSERT_THAT(result, NeIfShared(GetParam(), ring)); ASSERT_THAT(ToString(result), Eq(all.substr(offset, len))); - UnrefIfShared(ring); - Unref(result); } } } @@ -1039,18 +916,16 @@ TEST_P(CordRingSubTest, SubRingFromLargeExternal) { CordRepRing* ring = RefIfShared(FromFlats(flats, composition)); CordRepRing* result = CordRepRing::SubRing(ring, offset, 0); EXPECT_THAT(result, nullptr); - UnrefIfShared(ring); for (size_t len = all.size() - 30; len < all.size() - offset; ++len) { ring = RefIfShared(FromFlats(flats, composition)); result = NeedsUnref(CordRepRing::SubRing(ring, offset, len)); ASSERT_THAT(result, IsValidRingBuffer()); ASSERT_THAT(result, EqIfPrivate(GetParam(), ring)); + ASSERT_THAT(result, NeIfShared(GetParam(), ring)); auto str = ToString(result); ASSERT_THAT(str, SizeIs(len)); ASSERT_THAT(str, Eq(all.substr(offset, len))); - UnrefIfShared(ring); - Unref(result); } } } @@ -1063,16 +938,14 @@ TEST_P(CordRingSubTest, RemovePrefix) { CordRepRing* ring = RefIfShared(FromFlats(flats, composition)); CordRepRing* result = CordRepRing::RemovePrefix(ring, all.size()); EXPECT_THAT(result, nullptr); - UnrefIfShared(ring); for (size_t len = 1; len < all.size(); ++len) { ring = RefIfShared(FromFlats(flats, composition)); result = NeedsUnref(CordRepRing::RemovePrefix(ring, len)); ASSERT_THAT(result, IsValidRingBuffer()); EXPECT_THAT(result, EqIfPrivate(GetParam(), ring)); + ASSERT_THAT(result, NeIfShared(GetParam(), ring)); EXPECT_THAT(ToString(result), Eq(all.substr(len))); - UnrefIfShared(ring); - Unref(result); } } @@ -1087,7 +960,6 @@ TEST_P(CordRingSubTest, RemovePrefixFromLargeExternal) { ElementsAre( not_a_string_view(external1->base, 1 << 20).remove_prefix(1 << 16), not_a_string_view(external2->base, 1 << 20))); - Unref(result); } TEST_P(CordRingSubTest, RemoveSuffix) { @@ -1098,16 +970,14 @@ TEST_P(CordRingSubTest, RemoveSuffix) { CordRepRing* ring = RefIfShared(FromFlats(flats, composition)); CordRepRing* result = CordRepRing::RemoveSuffix(ring, all.size()); EXPECT_THAT(result, nullptr); - UnrefIfShared(ring); for (size_t len = 1; len < all.size(); ++len) { ring = RefIfShared(FromFlats(flats, composition)); result = NeedsUnref(CordRepRing::RemoveSuffix(ring, len)); ASSERT_THAT(result, IsValidRingBuffer()); - EXPECT_THAT(result, EqIfPrivate(GetParam(), ring)); - EXPECT_THAT(ToString(result), Eq(all.substr(0, all.size() - len))); - UnrefIfShared(ring); - Unref(result); + ASSERT_THAT(result, EqIfPrivate(GetParam(), ring)); + ASSERT_THAT(result, NeIfShared(GetParam(), ring)); + ASSERT_THAT(ToString(result), Eq(all.substr(0, all.size() - len))); } } @@ -1120,9 +990,8 @@ TEST_P(CordRingSubTest, AppendRing) { CordRepRing* result = NeedsUnref(CordRepRing::Append(ring, child)); ASSERT_THAT(result, IsValidRingBuffer()); EXPECT_THAT(result, EqIfPrivate(GetParam(), ring)); + EXPECT_THAT(result, NeIfShared(GetParam(), ring)); EXPECT_THAT(ToFlats(result), ElementsAreArray(kFoxFlats)); - UnrefIfShared(ring); - Unref(result); } TEST_P(CordRingBuildInputTest, AppendRingWithFlatOffset) { @@ -1135,11 +1004,9 @@ TEST_P(CordRingBuildInputTest, AppendRingWithFlatOffset) { CordRepRing* result = NeedsUnref(CordRepRing::Append(ring, stripped)); ASSERT_THAT(result, IsValidRingBuffer()); EXPECT_THAT(result, EqIfPrivateAndCapacity(GetParam(), ring)); + EXPECT_THAT(result, NeIfShared(GetParam(), ring)); EXPECT_THAT(ToFlats(result), ElementsAre("Head", "brown ", "fox ", "jumps ", "over ", "the ", "lazy ", "dog")); - UnrefIfInputSharedIndirect(child); - UnrefIfShared(ring); - Unref(result); } TEST_P(CordRingBuildInputTest, AppendRingWithBrokenOffset) { @@ -1152,11 +1019,9 @@ TEST_P(CordRingBuildInputTest, AppendRingWithBrokenOffset) { CordRepRing* result = NeedsUnref(CordRepRing::Append(ring, stripped)); ASSERT_THAT(result, IsValidRingBuffer()); EXPECT_THAT(result, EqIfPrivateAndCapacity(GetParam(), ring)); + EXPECT_THAT(result, NeIfShared(GetParam(), ring)); EXPECT_THAT(ToFlats(result), ElementsAre("Head", "umps ", "over ", "the ", "lazy ", "dog")); - UnrefIfInputSharedIndirect(child); - UnrefIfShared(ring); - Unref(result); } TEST_P(CordRingBuildInputTest, AppendRingWithFlatLength) { @@ -1169,11 +1034,9 @@ TEST_P(CordRingBuildInputTest, AppendRingWithFlatLength) { CordRepRing* result = NeedsUnref(CordRepRing::Append(ring, stripped)); ASSERT_THAT(result, IsValidRingBuffer()); EXPECT_THAT(result, EqIfPrivateAndCapacity(GetParam(), ring)); + EXPECT_THAT(result, NeIfShared(GetParam(), ring)); EXPECT_THAT(ToFlats(result), ElementsAre("Head", "The ", "quick ", "brown ", "fox ", "jumps ", "over ", "the ")); - UnrefIfInputSharedIndirect(child); - UnrefIfShared(ring); - Unref(result); } TEST_P(CordRingBuildTest, AppendRingWithBrokenFlatLength) { @@ -1186,11 +1049,9 @@ TEST_P(CordRingBuildTest, AppendRingWithBrokenFlatLength) { CordRepRing* result = NeedsUnref(CordRepRing::Append(ring, stripped)); ASSERT_THAT(result, IsValidRingBuffer()); EXPECT_THAT(result, EqIfPrivateAndCapacity(GetParam(), ring)); + EXPECT_THAT(result, NeIfShared(GetParam(), ring)); EXPECT_THAT(ToFlats(result), ElementsAre("Head", "The ", "quick ", "brown ", "fox ", "jumps ", "ov")); - UnrefIfInputSharedIndirect(child); - UnrefIfShared(ring); - Unref(result); } TEST_P(CordRingBuildTest, AppendRingMiddlePiece) { @@ -1203,11 +1064,9 @@ TEST_P(CordRingBuildTest, AppendRingMiddlePiece) { CordRepRing* result = NeedsUnref(CordRepRing::Append(ring, stripped)); ASSERT_THAT(result, IsValidRingBuffer()); EXPECT_THAT(result, EqIfPrivateAndCapacity(GetParam(), ring)); + EXPECT_THAT(result, NeIfShared(GetParam(), ring)); EXPECT_THAT(ToFlats(result), ElementsAre("Head", "ck ", "brown ", "fox ", "jum")); - UnrefIfInputSharedIndirect(child); - UnrefIfShared(ring); - Unref(result); } TEST_P(CordRingBuildTest, AppendRingSinglePiece) { @@ -1220,11 +1079,8 @@ TEST_P(CordRingBuildTest, AppendRingSinglePiece) { CordRepRing* result = NeedsUnref(CordRepRing::Append(ring, stripped)); ASSERT_THAT(result, IsValidRingBuffer()); EXPECT_THAT(result, EqIfPrivateAndCapacity(GetParam(), ring)); + EXPECT_THAT(result, NeIfShared(GetParam(), ring)); EXPECT_THAT(ToFlats(result), ElementsAre("Head", "row")); - UnrefIfInputSharedIndirect(child); - UnrefIfInputShared(stripped); - UnrefIfShared(ring); - Unref(result); } TEST_P(CordRingBuildInputTest, AppendRingSinglePieceWithPrefix) { @@ -1241,11 +1097,8 @@ TEST_P(CordRingBuildInputTest, AppendRingSinglePieceWithPrefix) { CordRepRing* result = NeedsUnref(CordRepRing::Append(ring, stripped)); ASSERT_THAT(result, IsValidRingBuffer()); EXPECT_THAT(result, EqIfPrivateAndCapacity(GetParam(), ring)); + EXPECT_THAT(result, NeIfShared(GetParam(), ring)); EXPECT_THAT(ToFlats(result), ElementsAre("Prepend", "Head", "row")); - UnrefIfInputSharedIndirect(child); - UnrefIfInputShared(stripped); - UnrefIfShared(ring); - Unref(result); } TEST_P(CordRingBuildInputTest, PrependRing) { @@ -1258,10 +1111,8 @@ TEST_P(CordRingBuildInputTest, PrependRing) { CordRepRing* result = NeedsUnref(CordRepRing::Prepend(ring, child)); ASSERT_THAT(result, IsValidRingBuffer()); EXPECT_THAT(result, EqIfPrivateAndCapacity(GetParam(), ring)); + EXPECT_THAT(result, NeIfShared(GetParam(), ring)); EXPECT_THAT(ToFlats(result), ElementsAreArray(kFoxFlats)); - UnrefIfInputShared(child); - UnrefIfShared(ring); - Unref(result); } TEST_P(CordRingBuildInputTest, PrependRingWithFlatOffset) { @@ -1274,12 +1125,9 @@ TEST_P(CordRingBuildInputTest, PrependRingWithFlatOffset) { CordRepRing* result = NeedsUnref(CordRepRing::Prepend(ring, stripped)); ASSERT_THAT(result, IsValidRingBuffer()); EXPECT_THAT(result, EqIfPrivateAndCapacity(GetParam(), ring)); + EXPECT_THAT(result, NeIfShared(GetParam(), ring)); EXPECT_THAT(ToFlats(result), ElementsAre("brown ", "fox ", "jumps ", "over ", "the ", "lazy ", "dog", "Tail")); - UnrefIfInputShared(child); - UnrefIfInputSharedIndirect(stripped); - UnrefIfShared(ring); - Unref(result); } TEST_P(CordRingBuildInputTest, PrependRingWithBrokenOffset) { @@ -1291,12 +1139,9 @@ TEST_P(CordRingBuildInputTest, PrependRingWithBrokenOffset) { CordRep* stripped = RefIfInputSharedIndirect(RemovePrefix(21, child)); CordRepRing* result = NeedsUnref(CordRepRing::Prepend(ring, stripped)); EXPECT_THAT(result, EqIfPrivateAndCapacity(GetParam(), ring)); + EXPECT_THAT(result, NeIfShared(GetParam(), ring)); EXPECT_THAT(ToFlats(result), ElementsAre("umps ", "over ", "the ", "lazy ", "dog", "Tail")); - UnrefIfInputShared(child); - UnrefIfInputSharedIndirect(stripped); - UnrefIfShared(ring); - Unref(result); } TEST_P(CordRingBuildInputTest, PrependRingWithFlatLength) { @@ -1309,12 +1154,9 @@ TEST_P(CordRingBuildInputTest, PrependRingWithFlatLength) { CordRepRing* result = NeedsUnref(CordRepRing::Prepend(ring, stripped)); ASSERT_THAT(result, IsValidRingBuffer()); EXPECT_THAT(result, EqIfPrivateAndCapacity(GetParam(), ring)); + EXPECT_THAT(result, NeIfShared(GetParam(), ring)); EXPECT_THAT(ToFlats(result), ElementsAre("The ", "quick ", "brown ", "fox ", "jumps ", "over ", "the ", "Tail")); - UnrefIfShared(ring); - UnrefIfInputShared(child); - UnrefIfInputSharedIndirect(stripped); - Unref(result); } TEST_P(CordRingBuildInputTest, PrependRingWithBrokenFlatLength) { @@ -1327,12 +1169,9 @@ TEST_P(CordRingBuildInputTest, PrependRingWithBrokenFlatLength) { CordRepRing* result = NeedsUnref(CordRepRing::Prepend(ring, stripped)); ASSERT_THAT(result, IsValidRingBuffer()); EXPECT_THAT(result, EqIfPrivateAndCapacity(GetParam(), ring)); + EXPECT_THAT(result, NeIfShared(GetParam(), ring)); EXPECT_THAT(ToFlats(result), ElementsAre("The ", "quick ", "brown ", "fox ", "jumps ", "ov", "Tail")); - UnrefIfInputShared(child); - UnrefIfInputSharedIndirect(stripped); - UnrefIfShared(ring); - Unref(result); } TEST_P(CordRingBuildInputTest, PrependRingMiddlePiece) { @@ -1346,12 +1185,9 @@ TEST_P(CordRingBuildInputTest, PrependRingMiddlePiece) { CordRepRing* result = NeedsUnref(CordRepRing::Prepend(ring, stripped)); ASSERT_THAT(result, IsValidRingBuffer()); EXPECT_THAT(result, EqIfPrivateAndCapacity(GetParam(), ring)); + EXPECT_THAT(result, NeIfShared(GetParam(), ring)); EXPECT_THAT(ToFlats(result), ElementsAre("ck ", "brown ", "fox ", "jum", "Tail")); - UnrefIfInputShared(child); - UnrefIfInputSharedIndirect(stripped); - UnrefIfShared(ring); - Unref(result); } TEST_P(CordRingBuildInputTest, PrependRingSinglePiece) { @@ -1364,11 +1200,8 @@ TEST_P(CordRingBuildInputTest, PrependRingSinglePiece) { CordRepRing* result = NeedsUnref(CordRepRing::Prepend(ring, stripped)); ASSERT_THAT(result, IsValidRingBuffer()); EXPECT_THAT(result, EqIfPrivateAndCapacity(GetParam(), ring)); + EXPECT_THAT(result, NeIfShared(GetParam(), ring)); EXPECT_THAT(ToFlats(result), ElementsAre("row", "Tail")); - UnrefIfInputShared(child); - UnrefIfInputSharedIndirect(stripped); - UnrefIfShared(ring); - Unref(result); } TEST_P(CordRingBuildInputTest, PrependRingSinglePieceWithPrefix) { @@ -1384,11 +1217,8 @@ TEST_P(CordRingBuildInputTest, PrependRingSinglePieceWithPrefix) { CordRepRing* result = NeedsUnref(CordRepRing::Prepend(ring, stripped)); ASSERT_THAT(result, IsValidRingBuffer()); EXPECT_THAT(result, EqIfPrivateAndCapacity(GetParam(), ring)); + EXPECT_THAT(result, NeIfShared(GetParam(), ring)); EXPECT_THAT(ToFlats(result), ElementsAre("row", "Prepend", "Tail")); - UnrefIfInputShared(child); - UnrefIfInputSharedIndirect(stripped); - UnrefIfShared(ring); - Unref(result); } TEST_F(CordRingTest, Find) { @@ -1406,7 +1236,6 @@ TEST_F(CordRingTest, Find) { ASSERT_THAT(found.offset, Lt(data.length())); ASSERT_THAT(data[found.offset], Eq(value[i])); } - Unref(ring); } TEST_F(CordRingTest, FindWithHint) { @@ -1442,7 +1271,6 @@ TEST_F(CordRingTest, FindWithHint) { ++flat_pos; flat_offset += flat.length(); } - Unref(ring); } TEST_F(CordRingTest, FindInLargeRing) { @@ -1464,7 +1292,6 @@ TEST_F(CordRingTest, FindInLargeRing) { ASSERT_THAT(pos.offset, Lt(data.length())); ASSERT_THAT(data[pos.offset], Eq(value[i])); } - Unref(ring); } TEST_F(CordRingTest, FindTail) { @@ -1483,7 +1310,6 @@ TEST_F(CordRingTest, FindTail) { ASSERT_THAT(pos.offset, Lt(data.length())); ASSERT_THAT(data[data.length() - pos.offset - 1], Eq(value[i])); } - Unref(ring); } TEST_F(CordRingTest, FindTailWithHint) { @@ -1510,7 +1336,6 @@ TEST_F(CordRingTest, FindTailWithHint) { ASSERT_THAT(pos.offset, Lt(data.length())); ASSERT_THAT(data[data.length() - pos.offset - 1], Eq(value[i])); } - Unref(ring); } TEST_F(CordRingTest, FindTailInLargeRing) { @@ -1532,7 +1357,6 @@ TEST_F(CordRingTest, FindTailInLargeRing) { ASSERT_THAT(pos.offset, Lt(data.length())); ASSERT_THAT(data[data.length() - pos.offset - 1], Eq(value[i])); } - Unref(ring); } TEST_F(CordRingTest, GetCharacter) { @@ -1544,7 +1368,6 @@ TEST_F(CordRingTest, GetCharacter) { for (int i = 0; i < value.length(); ++i) { ASSERT_THAT(result->GetCharacter(i), Eq(value[i])); } - Unref(result); } TEST_F(CordRingTest, GetCharacterWithSubstring) { @@ -1556,7 +1379,67 @@ TEST_F(CordRingTest, GetCharacterWithSubstring) { for (int i = 0; i < value.length(); ++i) { ASSERT_THAT(result->GetCharacter(i), Eq(value[i])); } - Unref(result); +} + +TEST_F(CordRingTest, IsFlatSingleFlat) { + for (bool external : {false, true}) { + SCOPED_TRACE(external ? "With External" : "With Flat"); + absl::string_view str = "Hello world"; + CordRep* rep = external ? MakeExternal(str) : MakeFlat(str); + CordRepRing* ring = NeedsUnref(CordRepRing::Create(rep)); + + // The ring is a single non-fragmented flat: + absl::string_view fragment; + EXPECT_TRUE(ring->IsFlat(nullptr)); + EXPECT_TRUE(ring->IsFlat(&fragment)); + EXPECT_THAT(fragment, Eq("Hello world")); + fragment = ""; + EXPECT_TRUE(ring->IsFlat(0, 11, nullptr)); + EXPECT_TRUE(ring->IsFlat(0, 11, &fragment)); + EXPECT_THAT(fragment, Eq("Hello world")); + + // Arbitrary ranges must check true as well. + EXPECT_TRUE(ring->IsFlat(1, 4, &fragment)); + EXPECT_THAT(fragment, Eq("ello")); + EXPECT_TRUE(ring->IsFlat(6, 5, &fragment)); + EXPECT_THAT(fragment, Eq("world")); + } +} + +TEST_F(CordRingTest, IsFlatMultiFlat) { + for (bool external : {false, true}) { + SCOPED_TRACE(external ? "With External" : "With Flat"); + absl::string_view str1 = "Hello world"; + absl::string_view str2 = "Halt and catch fire"; + CordRep* rep1 = external ? MakeExternal(str1) : MakeFlat(str1); + CordRep* rep2 = external ? MakeExternal(str2) : MakeFlat(str2); + CordRepRing* ring = CordRepRing::Append(CordRepRing::Create(rep1), rep2); + NeedsUnref(ring); + + // The ring is fragmented, IsFlat() on the entire cord must be false. + EXPECT_FALSE(ring->IsFlat(nullptr)); + absl::string_view fragment = "Don't touch this"; + EXPECT_FALSE(ring->IsFlat(&fragment)); + EXPECT_THAT(fragment, Eq("Don't touch this")); + + // Check for ranges exactly within both flats. + EXPECT_TRUE(ring->IsFlat(0, 11, &fragment)); + EXPECT_THAT(fragment, Eq("Hello world")); + EXPECT_TRUE(ring->IsFlat(11, 19, &fragment)); + EXPECT_THAT(fragment, Eq("Halt and catch fire")); + + // Check for arbitrary partial range inside each flat. + EXPECT_TRUE(ring->IsFlat(1, 4, &fragment)); + EXPECT_THAT(fragment, "ello"); + EXPECT_TRUE(ring->IsFlat(26, 4, &fragment)); + EXPECT_THAT(fragment, "fire"); + + // Check ranges spanning across both flats + fragment = "Don't touch this"; + EXPECT_FALSE(ring->IsFlat(1, 18, &fragment)); + EXPECT_FALSE(ring->IsFlat(10, 2, &fragment)); + EXPECT_THAT(fragment, Eq("Don't touch this")); + } } TEST_F(CordRingTest, Dump) { @@ -1564,7 +1447,6 @@ TEST_F(CordRingTest, Dump) { auto flats = MakeSpan(kFoxFlats); CordRepRing* ring = NeedsUnref(FromFlats(flats, kPrepend)); ss << *ring; - Unref(ring); } } // namespace diff --git a/absl/strings/cord_test.cc b/absl/strings/cord_test.cc index f9982428..0862f69a 100644 --- a/absl/strings/cord_test.cc +++ b/absl/strings/cord_test.cc @@ -34,13 +34,32 @@ #include "absl/base/internal/raw_logging.h" #include "absl/base/macros.h" #include "absl/container/fixed_array.h" +#include "absl/hash/hash.h" +#include "absl/random/random.h" #include "absl/strings/cord_test_helpers.h" +#include "absl/strings/cordz_test_helpers.h" +#include "absl/strings/match.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_format.h" #include "absl/strings/string_view.h" +// convenience local constants +static constexpr auto FLAT = absl::cord_internal::FLAT; +static constexpr auto MAX_FLAT_TAG = absl::cord_internal::MAX_FLAT_TAG; + typedef std::mt19937_64 RandomEngine; +using absl::cord_internal::CordRep; +using absl::cord_internal::CordRepBtree; +using absl::cord_internal::CordRepConcat; +using absl::cord_internal::CordRepCrc; +using absl::cord_internal::CordRepExternal; +using absl::cord_internal::CordRepFlat; +using absl::cord_internal::CordRepSubstring; +using absl::cord_internal::CordzUpdateTracker; +using absl::cord_internal::kFlatOverhead; +using absl::cord_internal::kMaxFlatLength; + static std::string RandomLowercaseString(RandomEngine* rng); static std::string RandomLowercaseString(RandomEngine* rng, size_t length); @@ -183,16 +202,129 @@ class CordTestPeer { } static bool IsTree(const Cord& c) { return c.contents_.is_tree(); } + static CordRep* Tree(const Cord& c) { return c.contents_.tree(); } static cord_internal::CordzInfo* GetCordzInfo(const Cord& c) { return c.contents_.cordz_info(); } + + static Cord MakeSubstring(Cord src, size_t offset, size_t length) { + ABSL_RAW_CHECK(src.contents_.is_tree(), "Can not be inlined"); + ABSL_RAW_CHECK(src.ExpectedChecksum() == absl::nullopt, + "Can not be hardened"); + Cord cord; + auto* tree = cord_internal::SkipCrcNode(src.contents_.tree()); + auto* rep = CordRepSubstring::Create(CordRep::Ref(tree), offset, length); + cord.contents_.EmplaceTree(rep, CordzUpdateTracker::kSubCord); + return cord; + } }; ABSL_NAMESPACE_END } // namespace absl -TEST(Cord, AllFlatSizes) { +// The CordTest fixture runs all tests with and without Cord Btree enabled, +// and with our without expected CRCs being set on the subject Cords. +class CordTest : public testing::TestWithParam<int> { + public: + // Returns true if test is running with btree enabled. + bool UseCrc() const { return GetParam() == 2 || GetParam() == 3; } + void MaybeHarden(absl::Cord& c) { + if (UseCrc()) { + c.SetExpectedChecksum(1); + } + } + absl::Cord MaybeHardened(absl::Cord c) { + MaybeHarden(c); + return c; + } + + // Returns human readable string representation of the test parameter. + static std::string ToString(testing::TestParamInfo<int> param) { + switch (param.param) { + case 0: + return "Btree"; + case 1: + return "BtreeHardened"; + default: + assert(false); + return "???"; + } + } +}; + +INSTANTIATE_TEST_SUITE_P(WithParam, CordTest, testing::Values(0, 1), + CordTest::ToString); + +TEST(CordRepFlat, AllFlatCapacities) { + // Explicitly and redundantly assert built-in min/max limits + static_assert(absl::cord_internal::kFlatOverhead < 32, ""); + static_assert(absl::cord_internal::kMinFlatSize == 32, ""); + static_assert(absl::cord_internal::kMaxLargeFlatSize == 256 << 10, ""); + EXPECT_EQ(absl::cord_internal::TagToAllocatedSize(FLAT), 32); + EXPECT_EQ(absl::cord_internal::TagToAllocatedSize(MAX_FLAT_TAG), 256 << 10); + + // Verify all tags to map perfectly back and forth, and + // that sizes are monotonically increasing. + size_t last_size = 0; + for (int tag = FLAT; tag <= MAX_FLAT_TAG; ++tag) { + size_t size = absl::cord_internal::TagToAllocatedSize(tag); + ASSERT_GT(size, last_size); + ASSERT_EQ(absl::cord_internal::TagToAllocatedSize(tag), size); + last_size = size; + } + + // All flat size from 32 - 512 are 8 byte granularity + for (size_t size = 32; size <= 512; size += 8) { + ASSERT_EQ(absl::cord_internal::RoundUpForTag(size), size); + uint8_t tag = absl::cord_internal::AllocatedSizeToTag(size); + ASSERT_EQ(absl::cord_internal::TagToAllocatedSize(tag), size); + } + + // All flat sizes from 512 - 8192 are 64 byte granularity + for (size_t size = 512; size <= 8192; size += 64) { + ASSERT_EQ(absl::cord_internal::RoundUpForTag(size), size); + uint8_t tag = absl::cord_internal::AllocatedSizeToTag(size); + ASSERT_EQ(absl::cord_internal::TagToAllocatedSize(tag), size); + } + + // All flat sizes from 8KB to 256KB are 4KB granularity + for (size_t size = 8192; size <= 256 * 1024; size += 4 * 1024) { + ASSERT_EQ(absl::cord_internal::RoundUpForTag(size), size); + uint8_t tag = absl::cord_internal::AllocatedSizeToTag(size); + ASSERT_EQ(absl::cord_internal::TagToAllocatedSize(tag), size); + } +} + +TEST(CordRepFlat, MaxFlatSize) { + CordRepFlat* flat = CordRepFlat::New(kMaxFlatLength); + EXPECT_EQ(flat->Capacity(), kMaxFlatLength); + CordRep::Unref(flat); + + flat = CordRepFlat::New(kMaxFlatLength * 4); + EXPECT_EQ(flat->Capacity(), kMaxFlatLength); + CordRep::Unref(flat); +} + +TEST(CordRepFlat, MaxLargeFlatSize) { + const size_t size = 256 * 1024 - kFlatOverhead; + CordRepFlat* flat = CordRepFlat::New(CordRepFlat::Large(), size); + EXPECT_GE(flat->Capacity(), size); + CordRep::Unref(flat); +} + +TEST(CordRepFlat, AllFlatSizes) { + const size_t kMaxSize = 256 * 1024; + for (size_t size = 32; size <= kMaxSize; size *=2) { + const size_t length = size - kFlatOverhead - 1; + CordRepFlat* flat = CordRepFlat::New(CordRepFlat::Large(), length); + EXPECT_GE(flat->Capacity(), length); + memset(flat->Data(), 0xCD, flat->Capacity()); + CordRep::Unref(flat); + } +} + +TEST_P(CordTest, AllFlatSizes) { using absl::strings_internal::CordTestAccess; for (size_t s = 0; s < CordTestAccess::MaxFlatLength(); s++) { @@ -203,6 +335,7 @@ TEST(Cord, AllFlatSizes) { } absl::Cord dst(src); + MaybeHarden(dst); EXPECT_EQ(std::string(dst), src) << s; } } @@ -210,7 +343,7 @@ TEST(Cord, AllFlatSizes) { // We create a Cord at least 128GB in size using the fact that Cords can // internally reference-count; thus the Cord is enormous without actually // consuming very much memory. -TEST(GigabyteCord, FromExternal) { +TEST_P(CordTest, GigabyteCordFromExternal) { const size_t one_gig = 1024U * 1024U * 1024U; size_t max_size = 2 * one_gig; if (sizeof(max_size) > 4) max_size = 128 * one_gig; @@ -227,7 +360,6 @@ TEST(GigabyteCord, FromExternal) { // caused crashes in production. We grow exponentially so that the code will // execute in a reasonable amount of time. absl::Cord c; - ABSL_RAW_LOG(INFO, "Made a Cord with %zu bytes!", c.size()); c.Append(from); while (c.size() < max_size) { c.Append(c); @@ -235,6 +367,7 @@ TEST(GigabyteCord, FromExternal) { c.Append(from); c.Append(from); c.Append(from); + MaybeHarden(c); } for (int i = 0; i < 1024; ++i) { @@ -260,9 +393,11 @@ static absl::Cord MakeExternalCord(int size) { extern bool my_unique_true_boolean; bool my_unique_true_boolean = true; -TEST(Cord, Assignment) { +TEST_P(CordTest, Assignment) { absl::Cord x(absl::string_view("hi there")); absl::Cord y(x); + MaybeHarden(y); + ASSERT_EQ(x.ExpectedChecksum(), absl::nullopt); ASSERT_EQ(std::string(x), "hi there"); ASSERT_EQ(std::string(y), "hi there"); ASSERT_TRUE(x == y); @@ -314,8 +449,9 @@ TEST(Cord, Assignment) { } } -TEST(Cord, StartsEndsWith) { +TEST_P(CordTest, StartsEndsWith) { absl::Cord x(absl::string_view("abcde")); + MaybeHarden(x); absl::Cord empty(""); ASSERT_TRUE(x.StartsWith(absl::Cord("abcde"))); @@ -347,13 +483,14 @@ TEST(Cord, StartsEndsWith) { ASSERT_TRUE(!empty.EndsWith("xyz")); } -TEST(Cord, Subcord) { - RandomEngine rng(testing::GTEST_FLAG(random_seed)); +TEST_P(CordTest, Subcord) { + RandomEngine rng(GTEST_FLAG_GET(random_seed)); const std::string s = RandomLowercaseString(&rng, 1024); absl::Cord a; AppendWithFragments(s, &rng, &a); - ASSERT_EQ(s.size(), a.size()); + MaybeHarden(a); + ASSERT_EQ(s, std::string(a)); // Check subcords of a, from a variety of interesting points. std::set<size_t> positions; @@ -374,6 +511,9 @@ TEST(Cord, Subcord) { ASSERT_EQ(absl::string_view(s).substr(pos, end_pos - pos), std::string(sa)) << a; + if (pos != 0 || end_pos != a.size()) { + ASSERT_EQ(sa.ExpectedChecksum(), absl::nullopt); + } } } @@ -408,15 +548,24 @@ TEST(Cord, Subcord) { EXPECT_TRUE(sa.empty()); } -TEST(Cord, Swap) { +TEST_P(CordTest, Swap) { absl::string_view a("Dexter"); absl::string_view b("Mandark"); absl::Cord x(a); absl::Cord y(b); + MaybeHarden(x); swap(x, y); + if (UseCrc()) { + ASSERT_EQ(x.ExpectedChecksum(), absl::nullopt); + ASSERT_EQ(y.ExpectedChecksum(), 1); + } ASSERT_EQ(x, absl::Cord(b)); ASSERT_EQ(y, absl::Cord(a)); x.swap(y); + if (UseCrc()) { + ASSERT_EQ(x.ExpectedChecksum(), 1); + ASSERT_EQ(y.ExpectedChecksum(), absl::nullopt); + } ASSERT_EQ(x, absl::Cord(a)); ASSERT_EQ(y, absl::Cord(b)); } @@ -440,56 +589,363 @@ static void VerifyCopyToString(const absl::Cord& cord) { } } -TEST(Cord, CopyToString) { - VerifyCopyToString(absl::Cord()); - VerifyCopyToString(absl::Cord("small cord")); - VerifyCopyToString( +TEST_P(CordTest, CopyToString) { + VerifyCopyToString(absl::Cord()); // empty cords cannot carry CRCs + VerifyCopyToString(MaybeHardened(absl::Cord("small cord"))); + VerifyCopyToString(MaybeHardened( absl::MakeFragmentedCord({"fragmented ", "cord ", "to ", "test ", - "copying ", "to ", "a ", "string."})); + "copying ", "to ", "a ", "string."}))); +} + +TEST_P(CordTest, AppendEmptyBuffer) { + absl::Cord cord; + cord.Append(absl::CordBuffer()); + cord.Append(absl::CordBuffer::CreateWithDefaultLimit(2000)); +} + +TEST_P(CordTest, AppendEmptyBufferToFlat) { + absl::Cord cord(std::string(2000, 'x')); + cord.Append(absl::CordBuffer()); + cord.Append(absl::CordBuffer::CreateWithDefaultLimit(2000)); +} + +TEST_P(CordTest, AppendEmptyBufferToTree) { + absl::Cord cord(std::string(2000, 'x')); + cord.Append(std::string(2000, 'y')); + cord.Append(absl::CordBuffer()); + cord.Append(absl::CordBuffer::CreateWithDefaultLimit(2000)); +} + +TEST_P(CordTest, AppendSmallBuffer) { + absl::Cord cord; + absl::CordBuffer buffer = absl::CordBuffer::CreateWithDefaultLimit(3); + ASSERT_THAT(buffer.capacity(), ::testing::Le(15)); + memcpy(buffer.data(), "Abc", 3); + buffer.SetLength(3); + cord.Append(std::move(buffer)); + EXPECT_EQ(buffer.length(), 0); // NOLINT + EXPECT_GT(buffer.capacity(), 0); // NOLINT + + buffer = absl::CordBuffer::CreateWithDefaultLimit(3); + memcpy(buffer.data(), "defgh", 5); + buffer.SetLength(5); + cord.Append(std::move(buffer)); + EXPECT_EQ(buffer.length(), 0); // NOLINT + EXPECT_GT(buffer.capacity(), 0); // NOLINT + + EXPECT_THAT(cord.Chunks(), ::testing::ElementsAre("Abcdefgh")); +} + +TEST_P(CordTest, AppendAndPrependBufferArePrecise) { + // Create a cord large enough to force 40KB flats. + std::string test_data(absl::cord_internal::kMaxFlatLength * 10, 'x'); + absl::Cord cord1(test_data); + absl::Cord cord2(test_data); + const size_t size1 = cord1.EstimatedMemoryUsage(); + const size_t size2 = cord2.EstimatedMemoryUsage(); + + absl::CordBuffer buffer = absl::CordBuffer::CreateWithDefaultLimit(3); + memcpy(buffer.data(), "Abc", 3); + buffer.SetLength(3); + cord1.Append(std::move(buffer)); + + buffer = absl::CordBuffer::CreateWithDefaultLimit(3); + memcpy(buffer.data(), "Abc", 3); + buffer.SetLength(3); + cord2.Prepend(std::move(buffer)); + +#ifndef NDEBUG + // Allow 32 bytes new CordRepFlat, and 128 bytes for 'glue nodes' + constexpr size_t kMaxDelta = 128 + 32; +#else + // Allow 256 bytes extra for 'allocation debug overhead' + constexpr size_t kMaxDelta = 128 + 32 + 256; +#endif + + EXPECT_LE(cord1.EstimatedMemoryUsage() - size1, kMaxDelta); + EXPECT_LE(cord2.EstimatedMemoryUsage() - size2, kMaxDelta); + + EXPECT_EQ(cord1, absl::StrCat(test_data, "Abc")); + EXPECT_EQ(cord2, absl::StrCat("Abc", test_data)); +} + +TEST_P(CordTest, PrependSmallBuffer) { + absl::Cord cord; + absl::CordBuffer buffer = absl::CordBuffer::CreateWithDefaultLimit(3); + ASSERT_THAT(buffer.capacity(), ::testing::Le(15)); + memcpy(buffer.data(), "Abc", 3); + buffer.SetLength(3); + cord.Prepend(std::move(buffer)); + EXPECT_EQ(buffer.length(), 0); // NOLINT + EXPECT_GT(buffer.capacity(), 0); // NOLINT + + buffer = absl::CordBuffer::CreateWithDefaultLimit(3); + memcpy(buffer.data(), "defgh", 5); + buffer.SetLength(5); + cord.Prepend(std::move(buffer)); + EXPECT_EQ(buffer.length(), 0); // NOLINT + EXPECT_GT(buffer.capacity(), 0); // NOLINT + + EXPECT_THAT(cord.Chunks(), ::testing::ElementsAre("defghAbc")); +} + +TEST_P(CordTest, AppendLargeBuffer) { + absl::Cord cord; + + std::string s1(700, '1'); + absl::CordBuffer buffer = absl::CordBuffer::CreateWithDefaultLimit(s1.size()); + memcpy(buffer.data(), s1.data(), s1.size()); + buffer.SetLength(s1.size()); + cord.Append(std::move(buffer)); + EXPECT_EQ(buffer.length(), 0); // NOLINT + EXPECT_GT(buffer.capacity(), 0); // NOLINT + + std::string s2(1000, '2'); + buffer = absl::CordBuffer::CreateWithDefaultLimit(s2.size()); + memcpy(buffer.data(), s2.data(), s2.size()); + buffer.SetLength(s2.size()); + cord.Append(std::move(buffer)); + EXPECT_EQ(buffer.length(), 0); // NOLINT + EXPECT_GT(buffer.capacity(), 0); // NOLINT + + EXPECT_THAT(cord.Chunks(), ::testing::ElementsAre(s1, s2)); +} + +TEST_P(CordTest, PrependLargeBuffer) { + absl::Cord cord; + + std::string s1(700, '1'); + absl::CordBuffer buffer = absl::CordBuffer::CreateWithDefaultLimit(s1.size()); + memcpy(buffer.data(), s1.data(), s1.size()); + buffer.SetLength(s1.size()); + cord.Prepend(std::move(buffer)); + EXPECT_EQ(buffer.length(), 0); // NOLINT + EXPECT_GT(buffer.capacity(), 0); // NOLINT + + std::string s2(1000, '2'); + buffer = absl::CordBuffer::CreateWithDefaultLimit(s2.size()); + memcpy(buffer.data(), s2.data(), s2.size()); + buffer.SetLength(s2.size()); + cord.Prepend(std::move(buffer)); + EXPECT_EQ(buffer.length(), 0); // NOLINT + EXPECT_GT(buffer.capacity(), 0); // NOLINT + + EXPECT_THAT(cord.Chunks(), ::testing::ElementsAre(s2, s1)); } -TEST(TryFlat, Empty) { +TEST_P(CordTest, GetAppendBufferOnEmptyCord) { + absl::Cord cord; + absl::CordBuffer buffer = cord.GetAppendBuffer(1000); + EXPECT_GE(buffer.capacity(), 1000); + EXPECT_EQ(buffer.length(), 0); +} + +TEST_P(CordTest, GetAppendBufferOnInlinedCord) { + static constexpr int kInlinedSize = sizeof(absl::CordBuffer) - 1; + for (int size : {6, kInlinedSize - 3, kInlinedSize - 2, 1000}) { + absl::Cord cord("Abc"); + absl::CordBuffer buffer = cord.GetAppendBuffer(size, 1); + EXPECT_GE(buffer.capacity(), 3 + size); + EXPECT_EQ(buffer.length(), 3); + EXPECT_EQ(absl::string_view(buffer.data(), buffer.length()), "Abc"); + EXPECT_TRUE(cord.empty()); + } +} + +TEST_P(CordTest, GetAppendBufferOnInlinedCordWithCapacityCloseToMax) { + // Cover the use case where we have a non empty inlined cord with some size + // 'n', and ask for something like 'uint64_max - k', assuming internal logic + // could overflow on 'uint64_max - k + size', and return a valid, but + // inefficiently smaller buffer if it would provide is the max allowed size. + for (size_t dist_from_max = 0; dist_from_max <= 4; ++dist_from_max) { + absl::Cord cord("Abc"); + size_t size = std::numeric_limits<size_t>::max() - dist_from_max; + absl::CordBuffer buffer = cord.GetAppendBuffer(size, 1); + EXPECT_EQ(buffer.capacity(), absl::CordBuffer::kDefaultLimit); + EXPECT_EQ(buffer.length(), 3); + EXPECT_EQ(absl::string_view(buffer.data(), buffer.length()), "Abc"); + EXPECT_TRUE(cord.empty()); + } +} + +TEST_P(CordTest, GetAppendBufferOnFlat) { + // Create a cord with a single flat and extra capacity + absl::Cord cord; + absl::CordBuffer buffer = absl::CordBuffer::CreateWithDefaultLimit(500); + buffer.SetLength(3); + memcpy(buffer.data(), "Abc", 3); + cord.Append(std::move(buffer)); + + buffer = cord.GetAppendBuffer(6); + EXPECT_GE(buffer.capacity(), 500); + EXPECT_EQ(buffer.length(), 3); + EXPECT_EQ(absl::string_view(buffer.data(), buffer.length()), "Abc"); + EXPECT_TRUE(cord.empty()); +} + +TEST_P(CordTest, GetAppendBufferOnFlatWithoutMinCapacity) { + // Create a cord with a single flat and extra capacity + absl::Cord cord; + absl::CordBuffer buffer = absl::CordBuffer::CreateWithDefaultLimit(500); + buffer.SetLength(30); + memset(buffer.data(), 'x', 30); + cord.Append(std::move(buffer)); + + buffer = cord.GetAppendBuffer(1000, 900); + EXPECT_GE(buffer.capacity(), 1000); + EXPECT_EQ(buffer.length(), 0); + EXPECT_EQ(cord, std::string(30, 'x')); +} + +TEST_P(CordTest, GetAppendBufferOnTree) { + RandomEngine rng; + for (int num_flats : {2, 3, 100}) { + // Create a cord with `num_flats` flats and extra capacity + absl::Cord cord; + std::string prefix; + std::string last; + for (int i = 0; i < num_flats - 1; ++i) { + prefix += last; + last = RandomLowercaseString(&rng, 10); + absl::CordBuffer buffer = absl::CordBuffer::CreateWithDefaultLimit(500); + buffer.SetLength(10); + memcpy(buffer.data(), last.data(), 10); + cord.Append(std::move(buffer)); + } + absl::CordBuffer buffer = cord.GetAppendBuffer(6); + EXPECT_GE(buffer.capacity(), 500); + EXPECT_EQ(buffer.length(), 10); + EXPECT_EQ(absl::string_view(buffer.data(), buffer.length()), last); + EXPECT_EQ(cord, prefix); + } +} + +TEST_P(CordTest, GetAppendBufferOnTreeWithoutMinCapacity) { + absl::Cord cord; + for (int i = 0; i < 2; ++i) { + absl::CordBuffer buffer = absl::CordBuffer::CreateWithDefaultLimit(500); + buffer.SetLength(3); + memcpy(buffer.data(), i ? "def" : "Abc", 3); + cord.Append(std::move(buffer)); + } + absl::CordBuffer buffer = cord.GetAppendBuffer(1000, 900); + EXPECT_GE(buffer.capacity(), 1000); + EXPECT_EQ(buffer.length(), 0); + EXPECT_EQ(cord, "Abcdef"); +} + +TEST_P(CordTest, GetAppendBufferOnSubstring) { + // Create a large cord with a single flat and some extra capacity + absl::Cord cord; + absl::CordBuffer buffer = absl::CordBuffer::CreateWithDefaultLimit(500); + buffer.SetLength(450); + memset(buffer.data(), 'x', 450); + cord.Append(std::move(buffer)); + cord.RemovePrefix(1); + + // Deny on substring + buffer = cord.GetAppendBuffer(6); + EXPECT_EQ(buffer.length(), 0); + EXPECT_EQ(cord, std::string(449, 'x')); +} + +TEST_P(CordTest, GetAppendBufferOnSharedCord) { + // Create a shared cord with a single flat and extra capacity + absl::Cord cord; + absl::CordBuffer buffer = absl::CordBuffer::CreateWithDefaultLimit(500); + buffer.SetLength(3); + memcpy(buffer.data(), "Abc", 3); + cord.Append(std::move(buffer)); + absl::Cord shared_cord = cord; + + // Deny on flat + buffer = cord.GetAppendBuffer(6); + EXPECT_EQ(buffer.length(), 0); + EXPECT_EQ(cord, "Abc"); + + buffer = absl::CordBuffer::CreateWithDefaultLimit(500); + buffer.SetLength(3); + memcpy(buffer.data(), "def", 3); + cord.Append(std::move(buffer)); + shared_cord = cord; + + // Deny on tree + buffer = cord.GetAppendBuffer(6); + EXPECT_EQ(buffer.length(), 0); + EXPECT_EQ(cord, "Abcdef"); +} + +TEST_P(CordTest, TryFlatEmpty) { absl::Cord c; EXPECT_EQ(c.TryFlat(), ""); } -TEST(TryFlat, Flat) { +TEST_P(CordTest, TryFlatFlat) { absl::Cord c("hello"); + MaybeHarden(c); EXPECT_EQ(c.TryFlat(), "hello"); } -TEST(TryFlat, SubstrInlined) { +TEST_P(CordTest, TryFlatSubstrInlined) { absl::Cord c("hello"); c.RemovePrefix(1); + MaybeHarden(c); EXPECT_EQ(c.TryFlat(), "ello"); } -TEST(TryFlat, SubstrFlat) { +TEST_P(CordTest, TryFlatSubstrFlat) { absl::Cord c("longer than 15 bytes"); - c.RemovePrefix(1); - EXPECT_EQ(c.TryFlat(), "onger than 15 bytes"); + absl::Cord sub = absl::CordTestPeer::MakeSubstring(c, 1, c.size() - 1); + MaybeHarden(sub); + EXPECT_EQ(sub.TryFlat(), "onger than 15 bytes"); } -TEST(TryFlat, Concat) { +TEST_P(CordTest, TryFlatConcat) { absl::Cord c = absl::MakeFragmentedCord({"hel", "lo"}); + MaybeHarden(c); EXPECT_EQ(c.TryFlat(), absl::nullopt); } -TEST(TryFlat, External) { +TEST_P(CordTest, TryFlatExternal) { absl::Cord c = absl::MakeCordFromExternal("hell", [](absl::string_view) {}); + MaybeHarden(c); EXPECT_EQ(c.TryFlat(), "hell"); } -TEST(TryFlat, SubstrExternal) { +TEST_P(CordTest, TryFlatSubstrExternal) { absl::Cord c = absl::MakeCordFromExternal("hell", [](absl::string_view) {}); - c.RemovePrefix(1); - EXPECT_EQ(c.TryFlat(), "ell"); -} - -TEST(TryFlat, SubstrConcat) { - absl::Cord c = absl::MakeFragmentedCord({"hello", " world"}); - c.RemovePrefix(1); - EXPECT_EQ(c.TryFlat(), absl::nullopt); + absl::Cord sub = absl::CordTestPeer::MakeSubstring(c, 1, c.size() - 1); + MaybeHarden(sub); + EXPECT_EQ(sub.TryFlat(), "ell"); +} + +TEST_P(CordTest, TryFlatCommonlyAssumedInvariants) { + // The behavior tested below is not part of the API contract of Cord, but it's + // something we intend to be true in our current implementation. This test + // exists to detect and prevent accidental breakage of the implementation. + absl::string_view fragments[] = {"A fragmented test", + " cord", + " to test subcords", + " of ", + "a", + " cord for", + " each chunk " + "returned by the ", + "iterator"}; + absl::Cord c = absl::MakeFragmentedCord(fragments); + MaybeHarden(c); + int fragment = 0; + int offset = 0; + absl::Cord::CharIterator itc = c.char_begin(); + for (absl::string_view sv : c.Chunks()) { + absl::string_view expected = fragments[fragment]; + absl::Cord subcord1 = c.Subcord(offset, sv.length()); + absl::Cord subcord2 = absl::Cord::AdvanceAndRead(&itc, sv.size()); + EXPECT_EQ(subcord1.TryFlat(), expected); + EXPECT_EQ(subcord2.TryFlat(), expected); + ++fragment; + offset += sv.length(); + } } static bool IsFlat(const absl::Cord& c) { @@ -520,15 +976,17 @@ static void VerifyFlatten(absl::Cord c) { EXPECT_TRUE(IsFlat(c)); } -TEST(Cord, Flatten) { +TEST_P(CordTest, Flatten) { VerifyFlatten(absl::Cord()); - VerifyFlatten(absl::Cord("small cord")); - VerifyFlatten(absl::Cord("larger than small buffer optimization")); - VerifyFlatten(absl::MakeFragmentedCord({"small ", "fragmented ", "cord"})); + VerifyFlatten(MaybeHardened(absl::Cord("small cord"))); + VerifyFlatten( + MaybeHardened(absl::Cord("larger than small buffer optimization"))); + VerifyFlatten(MaybeHardened( + absl::MakeFragmentedCord({"small ", "fragmented ", "cord"}))); // Test with a cord that is longer than the largest flat buffer - RandomEngine rng(testing::GTEST_FLAG(random_seed)); - VerifyFlatten(absl::Cord(RandomLowercaseString(&rng, 8192))); + RandomEngine rng(GTEST_FLAG_GET(random_seed)); + VerifyFlatten(MaybeHardened(absl::Cord(RandomLowercaseString(&rng, 8192)))); } // Test data @@ -574,7 +1032,7 @@ class TestData { }; } // namespace -TEST(Cord, MultipleLengths) { +TEST_P(CordTest, MultipleLengths) { TestData d; for (size_t i = 0; i < d.size(); i++) { std::string a = d.data(i); @@ -582,22 +1040,26 @@ TEST(Cord, MultipleLengths) { { // Construct from Cord absl::Cord tmp(a); absl::Cord x(tmp); + MaybeHarden(x); EXPECT_EQ(a, std::string(x)) << "'" << a << "'"; } { // Construct from absl::string_view absl::Cord x(a); + MaybeHarden(x); EXPECT_EQ(a, std::string(x)) << "'" << a << "'"; } { // Append cord to self absl::Cord self(a); + MaybeHarden(self); self.Append(self); EXPECT_EQ(a + a, std::string(self)) << "'" << a << "' + '" << a << "'"; } { // Prepend cord to self absl::Cord self(a); + MaybeHarden(self); self.Prepend(self); EXPECT_EQ(a + a, std::string(self)) << "'" << a << "' + '" << a << "'"; } @@ -609,12 +1071,14 @@ TEST(Cord, MultipleLengths) { { // CopyFrom Cord absl::Cord x(a); absl::Cord y(b); + MaybeHarden(x); x = y; EXPECT_EQ(b, std::string(x)) << "'" << a << "' + '" << b << "'"; } { // CopyFrom absl::string_view absl::Cord x(a); + MaybeHarden(x); x = b; EXPECT_EQ(b, std::string(x)) << "'" << a << "' + '" << b << "'"; } @@ -622,12 +1086,14 @@ TEST(Cord, MultipleLengths) { { // Cord::Append(Cord) absl::Cord x(a); absl::Cord y(b); + MaybeHarden(x); x.Append(y); EXPECT_EQ(a + b, std::string(x)) << "'" << a << "' + '" << b << "'"; } { // Cord::Append(absl::string_view) absl::Cord x(a); + MaybeHarden(x); x.Append(b); EXPECT_EQ(a + b, std::string(x)) << "'" << a << "' + '" << b << "'"; } @@ -635,12 +1101,14 @@ TEST(Cord, MultipleLengths) { { // Cord::Prepend(Cord) absl::Cord x(a); absl::Cord y(b); + MaybeHarden(x); x.Prepend(y); EXPECT_EQ(b + a, std::string(x)) << "'" << b << "' + '" << a << "'"; } { // Cord::Prepend(absl::string_view) absl::Cord x(a); + MaybeHarden(x); x.Prepend(b); EXPECT_EQ(b + a, std::string(x)) << "'" << b << "' + '" << a << "'"; } @@ -650,25 +1118,29 @@ TEST(Cord, MultipleLengths) { namespace { -TEST(Cord, RemoveSuffixWithExternalOrSubstring) { +TEST_P(CordTest, RemoveSuffixWithExternalOrSubstring) { absl::Cord cord = absl::MakeCordFromExternal( "foo bar baz", [](absl::string_view s) { DoNothing(s, nullptr); }); - EXPECT_EQ("foo bar baz", std::string(cord)); + MaybeHarden(cord); + // This RemoveSuffix() will wrap the EXTERNAL node in a SUBSTRING node. cord.RemoveSuffix(4); EXPECT_EQ("foo bar", std::string(cord)); + MaybeHarden(cord); + // This RemoveSuffix() will adjust the SUBSTRING node in-place. cord.RemoveSuffix(4); EXPECT_EQ("foo", std::string(cord)); } -TEST(Cord, RemoveSuffixMakesZeroLengthNode) { +TEST_P(CordTest, RemoveSuffixMakesZeroLengthNode) { absl::Cord c; c.Append(absl::Cord(std::string(100, 'x'))); absl::Cord other_ref = c; // Prevent inplace appends + MaybeHarden(c); c.Append(absl::Cord(std::string(200, 'y'))); c.RemoveSuffix(200); EXPECT_EQ(std::string(100, 'x'), std::string(c)); @@ -692,24 +1164,27 @@ absl::Cord CordWithZedBlock(size_t size) { } // Establish that ZedBlock does what we think it does. -TEST(CordSpliceTest, ZedBlock) { +TEST_P(CordTest, CordSpliceTestZedBlock) { absl::Cord blob = CordWithZedBlock(10); + MaybeHarden(blob); EXPECT_EQ(10, blob.size()); std::string s; absl::CopyCordToString(blob, &s); EXPECT_EQ("zzzzzzzzzz", s); } -TEST(CordSpliceTest, ZedBlock0) { +TEST_P(CordTest, CordSpliceTestZedBlock0) { absl::Cord blob = CordWithZedBlock(0); + MaybeHarden(blob); EXPECT_EQ(0, blob.size()); std::string s; absl::CopyCordToString(blob, &s); EXPECT_EQ("", s); } -TEST(CordSpliceTest, ZedBlockSuffix1) { +TEST_P(CordTest, CordSpliceTestZedBlockSuffix1) { absl::Cord blob = CordWithZedBlock(10); + MaybeHarden(blob); EXPECT_EQ(10, blob.size()); absl::Cord suffix(blob); suffix.RemovePrefix(9); @@ -720,8 +1195,9 @@ TEST(CordSpliceTest, ZedBlockSuffix1) { } // Remove all of a prefix block -TEST(CordSpliceTest, ZedBlockSuffix0) { +TEST_P(CordTest, CordSpliceTestZedBlockSuffix0) { absl::Cord blob = CordWithZedBlock(10); + MaybeHarden(blob); EXPECT_EQ(10, blob.size()); absl::Cord suffix(blob); suffix.RemovePrefix(10); @@ -752,16 +1228,18 @@ absl::Cord SpliceCord(const absl::Cord& blob, int64_t offset, } // Taking an empty suffix of a block breaks appending. -TEST(CordSpliceTest, RemoveEntireBlock1) { +TEST_P(CordTest, CordSpliceTestRemoveEntireBlock1) { absl::Cord zero = CordWithZedBlock(10); + MaybeHarden(zero); absl::Cord suffix(zero); suffix.RemovePrefix(10); absl::Cord result; result.Append(suffix); } -TEST(CordSpliceTest, RemoveEntireBlock2) { +TEST_P(CordTest, CordSpliceTestRemoveEntireBlock2) { absl::Cord zero = CordWithZedBlock(10); + MaybeHarden(zero); absl::Cord prefix(zero); prefix.RemoveSuffix(10); absl::Cord suffix(zero); @@ -770,16 +1248,22 @@ TEST(CordSpliceTest, RemoveEntireBlock2) { result.Append(suffix); } -TEST(CordSpliceTest, RemoveEntireBlock3) { +TEST_P(CordTest, CordSpliceTestRemoveEntireBlock3) { absl::Cord blob = CordWithZedBlock(10); absl::Cord block = BigCord(10, 'b'); + MaybeHarden(blob); + MaybeHarden(block); blob = SpliceCord(blob, 0, block); } struct CordCompareTestCase { template <typename LHS, typename RHS> - CordCompareTestCase(const LHS& lhs, const RHS& rhs) - : lhs_cord(lhs), rhs_cord(rhs) {} + CordCompareTestCase(const LHS& lhs, const RHS& rhs, bool use_crc) + : lhs_cord(lhs), rhs_cord(rhs) { + if (use_crc) { + lhs_cord.SetExpectedChecksum(1); + } + } absl::Cord lhs_cord; absl::Cord rhs_cord; @@ -801,7 +1285,7 @@ void VerifyComparison(const CordCompareTestCase& test_case) { << "LHS=" << rhs_string << "; RHS=" << lhs_string; } -TEST(Cord, Compare) { +TEST_P(CordTest, Compare) { absl::Cord subcord("aaaaaBBBBBcccccDDDDD"); subcord = subcord.Subcord(3, 10); @@ -816,47 +1300,54 @@ TEST(Cord, Compare) { concat2.Append("cccccccccccDDDDDDDDDDDDDD"); concat2.Append("DD"); + const bool use_crc = UseCrc(); + std::vector<CordCompareTestCase> test_cases = {{ // Inline cords - {"abcdef", "abcdef"}, - {"abcdef", "abcdee"}, - {"abcdef", "abcdeg"}, - {"bbcdef", "abcdef"}, - {"bbcdef", "abcdeg"}, - {"abcdefa", "abcdef"}, - {"abcdef", "abcdefa"}, + {"abcdef", "abcdef", use_crc}, + {"abcdef", "abcdee", use_crc}, + {"abcdef", "abcdeg", use_crc}, + {"bbcdef", "abcdef", use_crc}, + {"bbcdef", "abcdeg", use_crc}, + {"abcdefa", "abcdef", use_crc}, + {"abcdef", "abcdefa", use_crc}, // Small flat cords - {"aaaaaBBBBBcccccDDDDD", "aaaaaBBBBBcccccDDDDD"}, - {"aaaaaBBBBBcccccDDDDD", "aaaaaBBBBBxccccDDDDD"}, - {"aaaaaBBBBBcxcccDDDDD", "aaaaaBBBBBcccccDDDDD"}, - {"aaaaaBBBBBxccccDDDDD", "aaaaaBBBBBcccccDDDDX"}, - {"aaaaaBBBBBcccccDDDDDa", "aaaaaBBBBBcccccDDDDD"}, - {"aaaaaBBBBBcccccDDDDD", "aaaaaBBBBBcccccDDDDDa"}, + {"aaaaaBBBBBcccccDDDDD", "aaaaaBBBBBcccccDDDDD", use_crc}, + {"aaaaaBBBBBcccccDDDDD", "aaaaaBBBBBxccccDDDDD", use_crc}, + {"aaaaaBBBBBcxcccDDDDD", "aaaaaBBBBBcccccDDDDD", use_crc}, + {"aaaaaBBBBBxccccDDDDD", "aaaaaBBBBBcccccDDDDX", use_crc}, + {"aaaaaBBBBBcccccDDDDDa", "aaaaaBBBBBcccccDDDDD", use_crc}, + {"aaaaaBBBBBcccccDDDDD", "aaaaaBBBBBcccccDDDDDa", use_crc}, // Subcords - {subcord, subcord}, - {subcord, "aaBBBBBccc"}, - {subcord, "aaBBBBBccd"}, - {subcord, "aaBBBBBccb"}, - {subcord, "aaBBBBBxcb"}, - {subcord, "aaBBBBBccca"}, - {subcord, "aaBBBBBcc"}, + {subcord, subcord, use_crc}, + {subcord, "aaBBBBBccc", use_crc}, + {subcord, "aaBBBBBccd", use_crc}, + {subcord, "aaBBBBBccb", use_crc}, + {subcord, "aaBBBBBxcb", use_crc}, + {subcord, "aaBBBBBccca", use_crc}, + {subcord, "aaBBBBBcc", use_crc}, // Concats - {concat, concat}, + {concat, concat, use_crc}, {concat, - "aaaaaaaaaaaaaaaaBBBBBBBBBBBBBBBBccccccccccccccccDDDDDDDDDDDDDDDD"}, + "aaaaaaaaaaaaaaaaBBBBBBBBBBBBBBBBccccccccccccccccDDDDDDDDDDDDDDDD", + use_crc}, {concat, - "aaaaaaaaaaaaaaaaBBBBBBBBBBBBBBBBcccccccccccccccxDDDDDDDDDDDDDDDD"}, + "aaaaaaaaaaaaaaaaBBBBBBBBBBBBBBBBcccccccccccccccxDDDDDDDDDDDDDDDD", + use_crc}, {concat, - "aaaaaaaaaaaaaaaaBBBBBBBBBBBBBBBBacccccccccccccccDDDDDDDDDDDDDDDD"}, + "aaaaaaaaaaaaaaaaBBBBBBBBBBBBBBBBacccccccccccccccDDDDDDDDDDDDDDDD", + use_crc}, {concat, - "aaaaaaaaaaaaaaaaBBBBBBBBBBBBBBBBccccccccccccccccDDDDDDDDDDDDDDD"}, + "aaaaaaaaaaaaaaaaBBBBBBBBBBBBBBBBccccccccccccccccDDDDDDDDDDDDDDD", + use_crc}, {concat, - "aaaaaaaaaaaaaaaaBBBBBBBBBBBBBBBBccccccccccccccccDDDDDDDDDDDDDDDDe"}, + "aaaaaaaaaaaaaaaaBBBBBBBBBBBBBBBBccccccccccccccccDDDDDDDDDDDDDDDDe", + use_crc}, - {concat, concat2}, + {concat, concat2, use_crc}, }}; for (const auto& tc : test_cases) { @@ -864,9 +1355,10 @@ TEST(Cord, Compare) { } } -TEST(Cord, CompareAfterAssign) { +TEST_P(CordTest, CompareAfterAssign) { absl::Cord a("aaaaaa1111111"); absl::Cord b("aaaaaa2222222"); + MaybeHarden(a); a = "cccccc"; b = "cccccc"; EXPECT_EQ(a, b); @@ -893,8 +1385,8 @@ static void TestCompare(const absl::Cord& c, const absl::Cord& d, EXPECT_EQ(expected, sign(c.Compare(d))) << c << ", " << d; } -TEST(Compare, ComparisonIsUnsigned) { - RandomEngine rng(testing::GTEST_FLAG(random_seed)); +TEST_P(CordTest, CompareComparisonIsUnsigned) { + RandomEngine rng(GTEST_FLAG_GET(random_seed)); std::uniform_int_distribution<uint32_t> uniform_uint8(0, 255); char x = static_cast<char>(uniform_uint8(rng)); TestCompare( @@ -902,9 +1394,9 @@ TEST(Compare, ComparisonIsUnsigned) { absl::Cord(std::string(GetUniformRandomUpTo(&rng, 100), x ^ 0x80)), &rng); } -TEST(Compare, RandomComparisons) { +TEST_P(CordTest, CompareRandomComparisons) { const int kIters = 5000; - RandomEngine rng(testing::GTEST_FLAG(random_seed)); + RandomEngine rng(GTEST_FLAG_GET(random_seed)); int n = GetUniformRandomUpTo(&rng, 5000); absl::Cord a[] = {MakeExternalCord(n), @@ -925,6 +1417,8 @@ TEST(Compare, RandomComparisons) { d.Append(a[GetUniformRandomUpTo(&rng, ABSL_ARRAYSIZE(a))]); } std::bernoulli_distribution coin_flip(0.5); + MaybeHarden(c); + MaybeHarden(d); TestCompare(coin_flip(rng) ? c : absl::Cord(std::string(c)), coin_flip(rng) ? d : absl::Cord(std::string(d)), &rng); } @@ -960,43 +1454,43 @@ void CompareOperators() { EXPECT_FALSE(b <= a); } -TEST(ComparisonOperators, Cord_Cord) { +TEST_P(CordTest, ComparisonOperators_Cord_Cord) { CompareOperators<absl::Cord, absl::Cord>(); } -TEST(ComparisonOperators, Cord_StringPiece) { +TEST_P(CordTest, ComparisonOperators_Cord_StringPiece) { CompareOperators<absl::Cord, absl::string_view>(); } -TEST(ComparisonOperators, StringPiece_Cord) { +TEST_P(CordTest, ComparisonOperators_StringPiece_Cord) { CompareOperators<absl::string_view, absl::Cord>(); } -TEST(ComparisonOperators, Cord_string) { +TEST_P(CordTest, ComparisonOperators_Cord_string) { CompareOperators<absl::Cord, std::string>(); } -TEST(ComparisonOperators, string_Cord) { +TEST_P(CordTest, ComparisonOperators_string_Cord) { CompareOperators<std::string, absl::Cord>(); } -TEST(ComparisonOperators, stdstring_Cord) { +TEST_P(CordTest, ComparisonOperators_stdstring_Cord) { CompareOperators<std::string, absl::Cord>(); } -TEST(ComparisonOperators, Cord_stdstring) { +TEST_P(CordTest, ComparisonOperators_Cord_stdstring) { CompareOperators<absl::Cord, std::string>(); } -TEST(ComparisonOperators, charstar_Cord) { +TEST_P(CordTest, ComparisonOperators_charstar_Cord) { CompareOperators<const char*, absl::Cord>(); } -TEST(ComparisonOperators, Cord_charstar) { +TEST_P(CordTest, ComparisonOperators_Cord_charstar) { CompareOperators<absl::Cord, const char*>(); } -TEST(ConstructFromExternal, ReleaserInvoked) { +TEST_P(CordTest, ConstructFromExternalReleaserInvoked) { // Empty external memory means the releaser should be called immediately. { bool invoked = false; @@ -1038,8 +1532,8 @@ TEST(ConstructFromExternal, ReleaserInvoked) { } } -TEST(ConstructFromExternal, CompareContents) { - RandomEngine rng(testing::GTEST_FLAG(random_seed)); +TEST_P(CordTest, ConstructFromExternalCompareContents) { + RandomEngine rng(GTEST_FLAG_GET(random_seed)); for (int length = 1; length <= 2048; length *= 2) { std::string data = RandomLowercaseString(&rng, length); @@ -1050,12 +1544,13 @@ TEST(ConstructFromExternal, CompareContents) { EXPECT_EQ(external->size(), sv.size()); delete external; }); + MaybeHarden(cord); EXPECT_EQ(data, cord); } } -TEST(ConstructFromExternal, LargeReleaser) { - RandomEngine rng(testing::GTEST_FLAG(random_seed)); +TEST_P(CordTest, ConstructFromExternalLargeReleaser) { + RandomEngine rng(GTEST_FLAG_GET(random_seed)); constexpr size_t kLength = 256; std::string data = RandomLowercaseString(&rng, kLength); std::array<char, kLength> data_array; @@ -1065,11 +1560,11 @@ TEST(ConstructFromExternal, LargeReleaser) { EXPECT_EQ(data, absl::string_view(data_array.data(), data_array.size())); invoked = true; }; - (void)absl::MakeCordFromExternal(data, releaser); + (void)MaybeHardened(absl::MakeCordFromExternal(data, releaser)); EXPECT_TRUE(invoked); } -TEST(ConstructFromExternal, FunctionPointerReleaser) { +TEST_P(CordTest, ConstructFromExternalFunctionPointerReleaser) { static absl::string_view data("hello world"); static bool invoked; auto* releaser = @@ -1078,15 +1573,15 @@ TEST(ConstructFromExternal, FunctionPointerReleaser) { invoked = true; }); invoked = false; - (void)absl::MakeCordFromExternal(data, releaser); + (void)MaybeHardened(absl::MakeCordFromExternal(data, releaser)); EXPECT_TRUE(invoked); invoked = false; - (void)absl::MakeCordFromExternal(data, *releaser); + (void)MaybeHardened(absl::MakeCordFromExternal(data, *releaser)); EXPECT_TRUE(invoked); } -TEST(ConstructFromExternal, MoveOnlyReleaser) { +TEST_P(CordTest, ConstructFromExternalMoveOnlyReleaser) { struct Releaser { explicit Releaser(bool* invoked) : invoked(invoked) {} Releaser(Releaser&& other) noexcept : invoked(other.invoked) {} @@ -1096,24 +1591,25 @@ TEST(ConstructFromExternal, MoveOnlyReleaser) { }; bool invoked = false; - (void)absl::MakeCordFromExternal("dummy", Releaser(&invoked)); + (void)MaybeHardened(absl::MakeCordFromExternal("dummy", Releaser(&invoked))); EXPECT_TRUE(invoked); } -TEST(ConstructFromExternal, NoArgLambda) { +TEST_P(CordTest, ConstructFromExternalNoArgLambda) { bool invoked = false; - (void)absl::MakeCordFromExternal("dummy", [&invoked]() { invoked = true; }); + (void)MaybeHardened( + absl::MakeCordFromExternal("dummy", [&invoked]() { invoked = true; })); EXPECT_TRUE(invoked); } -TEST(ConstructFromExternal, StringViewArgLambda) { +TEST_P(CordTest, ConstructFromExternalStringViewArgLambda) { bool invoked = false; - (void)absl::MakeCordFromExternal( - "dummy", [&invoked](absl::string_view) { invoked = true; }); + (void)MaybeHardened(absl::MakeCordFromExternal( + "dummy", [&invoked](absl::string_view) { invoked = true; })); EXPECT_TRUE(invoked); } -TEST(ConstructFromExternal, NonTrivialReleaserDestructor) { +TEST_P(CordTest, ConstructFromExternalNonTrivialReleaserDestructor) { struct Releaser { explicit Releaser(bool* destroyed) : destroyed(destroyed) {} ~Releaser() { *destroyed = true; } @@ -1124,57 +1620,94 @@ TEST(ConstructFromExternal, NonTrivialReleaserDestructor) { bool destroyed = false; Releaser releaser(&destroyed); - (void)absl::MakeCordFromExternal("dummy", releaser); + (void)MaybeHardened(absl::MakeCordFromExternal("dummy", releaser)); EXPECT_TRUE(destroyed); } -TEST(ConstructFromExternal, ReferenceQualifierOverloads) { - struct Releaser { - void operator()(absl::string_view) & { *lvalue_invoked = true; } - void operator()(absl::string_view) && { *rvalue_invoked = true; } +TEST_P(CordTest, ConstructFromExternalReferenceQualifierOverloads) { + enum InvokedAs { kMissing, kLValue, kRValue }; + enum CopiedAs { kNone, kMove, kCopy }; + struct Tracker { + CopiedAs copied_as = kNone; + InvokedAs invoked_as = kMissing; + + void Record(InvokedAs rhs) { + ASSERT_EQ(invoked_as, kMissing); + invoked_as = rhs; + } + + void Record(CopiedAs rhs) { + if (copied_as == kNone || rhs == kCopy) copied_as = rhs; + } + } tracker; + + class Releaser { + public: + explicit Releaser(Tracker* tracker) : tr_(tracker) { *tracker = Tracker(); } + Releaser(Releaser&& rhs) : tr_(rhs.tr_) { tr_->Record(kMove); } + Releaser(const Releaser& rhs) : tr_(rhs.tr_) { tr_->Record(kCopy); } + + void operator()(absl::string_view) & { tr_->Record(kLValue); } + void operator()(absl::string_view) && { tr_->Record(kRValue); } - bool* lvalue_invoked; - bool* rvalue_invoked; + private: + Tracker* tr_; }; - bool lvalue_invoked = false; - bool rvalue_invoked = false; - Releaser releaser = {&lvalue_invoked, &rvalue_invoked}; - (void)absl::MakeCordFromExternal("", releaser); - EXPECT_FALSE(lvalue_invoked); - EXPECT_TRUE(rvalue_invoked); - rvalue_invoked = false; + const Releaser releaser1(&tracker); + (void)MaybeHardened(absl::MakeCordFromExternal("", releaser1)); + EXPECT_EQ(tracker.copied_as, kCopy); + EXPECT_EQ(tracker.invoked_as, kRValue); - (void)absl::MakeCordFromExternal("dummy", releaser); - EXPECT_FALSE(lvalue_invoked); - EXPECT_TRUE(rvalue_invoked); - rvalue_invoked = false; + const Releaser releaser2(&tracker); + (void)MaybeHardened(absl::MakeCordFromExternal("", releaser2)); + EXPECT_EQ(tracker.copied_as, kCopy); + EXPECT_EQ(tracker.invoked_as, kRValue); - // NOLINTNEXTLINE: suppress clang-tidy std::move on trivially copyable type. - (void)absl::MakeCordFromExternal("dummy", std::move(releaser)); - EXPECT_FALSE(lvalue_invoked); - EXPECT_TRUE(rvalue_invoked); + Releaser releaser3(&tracker); + (void)MaybeHardened(absl::MakeCordFromExternal("", std::move(releaser3))); + EXPECT_EQ(tracker.copied_as, kMove); + EXPECT_EQ(tracker.invoked_as, kRValue); + + Releaser releaser4(&tracker); + (void)MaybeHardened(absl::MakeCordFromExternal("dummy", releaser4)); + EXPECT_EQ(tracker.copied_as, kCopy); + EXPECT_EQ(tracker.invoked_as, kRValue); + + const Releaser releaser5(&tracker); + (void)MaybeHardened(absl::MakeCordFromExternal("dummy", releaser5)); + EXPECT_EQ(tracker.copied_as, kCopy); + EXPECT_EQ(tracker.invoked_as, kRValue); + + Releaser releaser6(&tracker); + (void)MaybeHardened(absl::MakeCordFromExternal("foo", std::move(releaser6))); + EXPECT_EQ(tracker.copied_as, kMove); + EXPECT_EQ(tracker.invoked_as, kRValue); } -TEST(ExternalMemory, BasicUsage) { +TEST_P(CordTest, ExternalMemoryBasicUsage) { static const char* strings[] = {"", "hello", "there"}; for (const char* str : strings) { absl::Cord dst("(prefix)"); + MaybeHarden(dst); AddExternalMemory(str, &dst); + MaybeHarden(dst); dst.Append("(suffix)"); EXPECT_EQ((std::string("(prefix)") + str + std::string("(suffix)")), std::string(dst)); } } -TEST(ExternalMemory, RemovePrefixSuffix) { +TEST_P(CordTest, ExternalMemoryRemovePrefixSuffix) { // Exhaustively try all sub-strings. absl::Cord cord = MakeComposite(); std::string s = std::string(cord); for (int offset = 0; offset <= s.size(); offset++) { for (int length = 0; length <= s.size() - offset; length++) { absl::Cord result(cord); + MaybeHarden(result); result.RemovePrefix(offset); + MaybeHarden(result); result.RemoveSuffix(result.size() - length); EXPECT_EQ(s.substr(offset, length), std::string(result)) << offset << " " << length; @@ -1182,11 +1715,13 @@ TEST(ExternalMemory, RemovePrefixSuffix) { } } -TEST(ExternalMemory, Get) { +TEST_P(CordTest, ExternalMemoryGet) { absl::Cord cord("hello"); AddExternalMemory(" world!", &cord); + MaybeHarden(cord); AddExternalMemory(" how are ", &cord); cord.Append(" you?"); + MaybeHarden(cord); std::string s = std::string(cord); for (int i = 0; i < s.size(); i++) { EXPECT_EQ(s[i], cord[i]); @@ -1194,58 +1729,138 @@ TEST(ExternalMemory, Get) { } // CordMemoryUsage tests verify the correctness of the EstimatedMemoryUsage() -// These tests take into account that the reported memory usage is approximate -// and non-deterministic. For all tests, We verify that the reported memory -// usage is larger than `size()`, and less than `size() * 1.5` as a cord should -// never reserve more 'extra' capacity than half of its size as it grows. -// Additionally we have some whiteboxed expectations based on our knowledge of -// the layout and size of empty and inlined cords, and flat nodes. +// We use whiteboxed expectations based on our knowledge of the layout and size +// of empty and inlined cords, and flat nodes. + +constexpr auto kFairShare = absl::CordMemoryAccounting::kFairShare; -TEST(CordMemoryUsage, Empty) { - EXPECT_EQ(sizeof(absl::Cord), absl::Cord().EstimatedMemoryUsage()); +// Creates a cord of `n` `c` values, making sure no string stealing occurs. +absl::Cord MakeCord(size_t n, char c) { + const std::string s(n, c); + return absl::Cord(s); } -TEST(CordMemoryUsage, Embedded) { - absl::Cord a("hello"); - EXPECT_EQ(a.EstimatedMemoryUsage(), sizeof(absl::Cord)); +TEST(CordTest, CordMemoryUsageEmpty) { + absl::Cord cord; + EXPECT_EQ(sizeof(absl::Cord), cord.EstimatedMemoryUsage()); + EXPECT_EQ(sizeof(absl::Cord), cord.EstimatedMemoryUsage(kFairShare)); } -TEST(CordMemoryUsage, EmbeddedAppend) { - absl::Cord a("a"); - absl::Cord b("bcd"); - EXPECT_EQ(b.EstimatedMemoryUsage(), sizeof(absl::Cord)); - a.Append(b); +TEST(CordTest, CordMemoryUsageInlined) { + absl::Cord a("hello"); EXPECT_EQ(a.EstimatedMemoryUsage(), sizeof(absl::Cord)); + EXPECT_EQ(a.EstimatedMemoryUsage(kFairShare), sizeof(absl::Cord)); } -TEST(CordMemoryUsage, ExternalMemory) { - static const int kLength = 1000; +TEST(CordTest, CordMemoryUsageExternalMemory) { absl::Cord cord; - AddExternalMemory(std::string(kLength, 'x'), &cord); - EXPECT_GT(cord.EstimatedMemoryUsage(), kLength); - EXPECT_LE(cord.EstimatedMemoryUsage(), kLength * 1.5); -} + AddExternalMemory(std::string(1000, 'x'), &cord); + const size_t expected = + sizeof(absl::Cord) + 1000 + sizeof(CordRepExternal) + sizeof(intptr_t); + EXPECT_EQ(cord.EstimatedMemoryUsage(), expected); + EXPECT_EQ(cord.EstimatedMemoryUsage(kFairShare), expected); +} + +TEST(CordTest, CordMemoryUsageFlat) { + absl::Cord cord = MakeCord(1000, 'a'); + const size_t flat_size = + absl::CordTestPeer::Tree(cord)->flat()->AllocatedSize(); + EXPECT_EQ(cord.EstimatedMemoryUsage(), sizeof(absl::Cord) + flat_size); + EXPECT_EQ(cord.EstimatedMemoryUsage(kFairShare), + sizeof(absl::Cord) + flat_size); +} + +TEST(CordTest, CordMemoryUsageSubStringSharedFlat) { + absl::Cord flat = MakeCord(2000, 'a'); + const size_t flat_size = + absl::CordTestPeer::Tree(flat)->flat()->AllocatedSize(); + absl::Cord cord = flat.Subcord(500, 1000); + EXPECT_EQ(cord.EstimatedMemoryUsage(), + sizeof(absl::Cord) + sizeof(CordRepSubstring) + flat_size); + EXPECT_EQ(cord.EstimatedMemoryUsage(kFairShare), + sizeof(absl::Cord) + sizeof(CordRepSubstring) + flat_size / 2); +} + +TEST(CordTest, CordMemoryUsageFlatShared) { + absl::Cord shared = MakeCord(1000, 'a'); + absl::Cord cord(shared); + const size_t flat_size = + absl::CordTestPeer::Tree(cord)->flat()->AllocatedSize(); + EXPECT_EQ(cord.EstimatedMemoryUsage(), sizeof(absl::Cord) + flat_size); + EXPECT_EQ(cord.EstimatedMemoryUsage(kFairShare), + sizeof(absl::Cord) + flat_size / 2); +} + +TEST(CordTest, CordMemoryUsageFlatHardenedAndShared) { + absl::Cord shared = MakeCord(1000, 'a'); + absl::Cord cord(shared); + const size_t flat_size = + absl::CordTestPeer::Tree(cord)->flat()->AllocatedSize(); + cord.SetExpectedChecksum(1); + EXPECT_EQ(cord.EstimatedMemoryUsage(), + sizeof(absl::Cord) + sizeof(CordRepCrc) + flat_size); + EXPECT_EQ(cord.EstimatedMemoryUsage(kFairShare), + sizeof(absl::Cord) + sizeof(CordRepCrc) + flat_size / 2); + + absl::Cord cord2(cord); + EXPECT_EQ(cord2.EstimatedMemoryUsage(), + sizeof(absl::Cord) + sizeof(CordRepCrc) + flat_size); + EXPECT_EQ(cord2.EstimatedMemoryUsage(kFairShare), + sizeof(absl::Cord) + (sizeof(CordRepCrc) + flat_size / 2) / 2); +} + +TEST(CordTest, CordMemoryUsageBTree) { + absl::Cord cord1; + size_t flats1_size = 0; + absl::Cord flats1[4] = {MakeCord(1000, 'a'), MakeCord(1100, 'a'), + MakeCord(1200, 'a'), MakeCord(1300, 'a')}; + for (absl::Cord flat : flats1) { + flats1_size += absl::CordTestPeer::Tree(flat)->flat()->AllocatedSize(); + cord1.Append(std::move(flat)); + } -TEST(CordMemoryUsage, Flat) { - static const int kLength = 125; - absl::Cord a(std::string(kLength, 'a')); - EXPECT_GT(a.EstimatedMemoryUsage(), kLength); - EXPECT_LE(a.EstimatedMemoryUsage(), kLength * 1.5); -} + // Make sure the created cord is a BTREE tree. Under some builds such as + // windows DLL, we may have ODR like effects on the flag, meaning the DLL + // code will run with the picked up default. + if (!absl::CordTestPeer::Tree(cord1)->IsBtree()) { + ABSL_RAW_LOG(WARNING, "Cord library code not respecting btree flag"); + return; + } -TEST(CordMemoryUsage, AppendFlat) { - using absl::strings_internal::CordTestAccess; - absl::Cord a(std::string(CordTestAccess::MaxFlatLength(), 'a')); - size_t length = a.EstimatedMemoryUsage(); - a.Append(std::string(CordTestAccess::MaxFlatLength(), 'b')); - size_t delta = a.EstimatedMemoryUsage() - length; - EXPECT_GT(delta, CordTestAccess::MaxFlatLength()); - EXPECT_LE(delta, CordTestAccess::MaxFlatLength() * 1.5); + size_t rep1_size = sizeof(CordRepBtree) + flats1_size; + size_t rep1_shared_size = sizeof(CordRepBtree) + flats1_size / 2; + + EXPECT_EQ(cord1.EstimatedMemoryUsage(), sizeof(absl::Cord) + rep1_size); + EXPECT_EQ(cord1.EstimatedMemoryUsage(kFairShare), + sizeof(absl::Cord) + rep1_shared_size); + + absl::Cord cord2; + size_t flats2_size = 0; + absl::Cord flats2[4] = {MakeCord(600, 'a'), MakeCord(700, 'a'), + MakeCord(800, 'a'), MakeCord(900, 'a')}; + for (absl::Cord& flat : flats2) { + flats2_size += absl::CordTestPeer::Tree(flat)->flat()->AllocatedSize(); + cord2.Append(std::move(flat)); + } + size_t rep2_size = sizeof(CordRepBtree) + flats2_size; + + EXPECT_EQ(cord2.EstimatedMemoryUsage(), sizeof(absl::Cord) + rep2_size); + EXPECT_EQ(cord2.EstimatedMemoryUsage(kFairShare), + sizeof(absl::Cord) + rep2_size); + + absl::Cord cord(cord1); + cord.Append(std::move(cord2)); + + EXPECT_EQ(cord.EstimatedMemoryUsage(), + sizeof(absl::Cord) + sizeof(CordRepBtree) + rep1_size + rep2_size); + EXPECT_EQ(cord.EstimatedMemoryUsage(kFairShare), + sizeof(absl::Cord) + sizeof(CordRepBtree) + rep1_shared_size / 2 + + rep2_size); } // Regtest for a change that had to be rolled back because it expanded out // of the InlineRep too soon, which was observable through MemoryUsage(). -TEST(CordMemoryUsage, InlineRep) { +TEST_P(CordTest, CordMemoryUsageInlineRep) { constexpr size_t kMaxInline = 15; // Cord::InlineRep::N const std::string small_string(kMaxInline, 'x'); absl::Cord c1(small_string); @@ -1259,14 +1874,16 @@ TEST(CordMemoryUsage, InlineRep) { } // namespace // Regtest for 7510292 (fix a bug introduced by 7465150) -TEST(Cord, Concat_Append) { +TEST_P(CordTest, Concat_Append) { // Create a rep of type CONCAT absl::Cord s1("foobarbarbarbarbar"); + MaybeHarden(s1); s1.Append("abcdefgabcdefgabcdefgabcdefgabcdefgabcdefgabcdefg"); size_t size = s1.size(); // Create a copy of s1 and append to it. absl::Cord s2 = s1; + MaybeHarden(s2); s2.Append("x"); // 7465150 modifies s1 when it shouldn't. @@ -1274,10 +1891,91 @@ TEST(Cord, Concat_Append) { EXPECT_EQ(s2.size(), size + 1); } -TEST(MakeFragmentedCord, MakeFragmentedCordFromInitializerList) { +TEST_P(CordTest, DiabolicalGrowth) { + // This test exercises a diabolical Append(<one char>) on a cord, making the + // cord shared before each Append call resulting in a terribly fragmented + // resulting cord. + // TODO(b/183983616): Apply some minimum compaction when copying a shared + // source cord into a mutable copy for updates in CordRepRing. + RandomEngine rng(GTEST_FLAG_GET(random_seed)); + const std::string expected = RandomLowercaseString(&rng, 5000); + absl::Cord cord; + for (char c : expected) { + absl::Cord shared(cord); + cord.Append(absl::string_view(&c, 1)); + MaybeHarden(cord); + } + std::string value; + absl::CopyCordToString(cord, &value); + EXPECT_EQ(value, expected); + ABSL_RAW_LOG(INFO, "Diabolical size allocated = %zu", + cord.EstimatedMemoryUsage()); +} + +// The following tests check support for >4GB cords in 64-bit binaries, and +// 2GB-4GB cords in 32-bit binaries. This function returns the large cord size +// that's appropriate for the binary. + +// Construct a huge cord with the specified valid prefix. +static absl::Cord MakeHuge(absl::string_view prefix) { + absl::Cord cord; + if (sizeof(size_t) > 4) { + // In 64-bit binaries, test 64-bit Cord support. + const size_t size = + static_cast<size_t>(std::numeric_limits<uint32_t>::max()) + 314; + cord.Append(absl::MakeCordFromExternal( + absl::string_view(prefix.data(), size), + [](absl::string_view s) { DoNothing(s, nullptr); })); + } else { + // Cords are limited to 32-bit lengths in 32-bit binaries. The following + // tests check for use of "signed int" to represent Cord length/offset. + // However absl::string_view does not allow lengths >= (1u<<31), so we need + // to append in two parts; + const size_t s1 = (1u << 31) - 1; + // For shorter cord, `Append` copies the data rather than allocating a new + // node. The threshold is currently set to 511, so `s2` needs to be bigger + // to not trigger the copy. + const size_t s2 = 600; + cord.Append(absl::MakeCordFromExternal( + absl::string_view(prefix.data(), s1), + [](absl::string_view s) { DoNothing(s, nullptr); })); + cord.Append(absl::MakeCordFromExternal( + absl::string_view("", s2), + [](absl::string_view s) { DoNothing(s, nullptr); })); + } + return cord; +} + +TEST_P(CordTest, HugeCord) { + absl::Cord cord = MakeHuge("huge cord"); + MaybeHarden(cord); + + const size_t acceptable_delta = + 100 + (UseCrc() ? sizeof(absl::cord_internal::CordRepCrc) : 0); + EXPECT_LE(cord.size(), cord.EstimatedMemoryUsage()); + EXPECT_GE(cord.size() + acceptable_delta, cord.EstimatedMemoryUsage()); +} + +// Tests that Append() works ok when handed a self reference +TEST_P(CordTest, AppendSelf) { + // We run the test until data is ~16K + // This guarantees it covers small, medium and large data. + std::string control_data = "Abc"; + absl::Cord data(control_data); + while (control_data.length() < 0x4000) { + MaybeHarden(data); + data.Append(data); + control_data.append(control_data); + ASSERT_EQ(control_data, data); + } +} + +TEST_P(CordTest, MakeFragmentedCordFromInitializerList) { absl::Cord fragmented = absl::MakeFragmentedCord({"A ", "fragmented ", "Cord"}); + MaybeHarden(fragmented); + EXPECT_EQ("A fragmented Cord", fragmented); auto chunk_it = fragmented.chunk_begin(); @@ -1294,10 +1992,12 @@ TEST(MakeFragmentedCord, MakeFragmentedCordFromInitializerList) { ASSERT_TRUE(++chunk_it == fragmented.chunk_end()); } -TEST(MakeFragmentedCord, MakeFragmentedCordFromVector) { +TEST_P(CordTest, MakeFragmentedCordFromVector) { std::vector<absl::string_view> chunks = {"A ", "fragmented ", "Cord"}; absl::Cord fragmented = absl::MakeFragmentedCord(chunks); + MaybeHarden(fragmented); + EXPECT_EQ("A fragmented Cord", fragmented); auto chunk_it = fragmented.chunk_begin(); @@ -1314,7 +2014,7 @@ TEST(MakeFragmentedCord, MakeFragmentedCordFromVector) { ASSERT_TRUE(++chunk_it == fragmented.chunk_end()); } -TEST(CordChunkIterator, Traits) { +TEST_P(CordTest, CordChunkIteratorTraits) { static_assert(std::is_copy_constructible<absl::Cord::ChunkIterator>::value, ""); static_assert(std::is_copy_assignable<absl::Cord::ChunkIterator>::value, ""); @@ -1395,39 +2095,115 @@ static void VerifyChunkIterator(const absl::Cord& cord, EXPECT_TRUE(post_iter == cord.chunk_end()); // NOLINT } -TEST(CordChunkIterator, Operations) { +TEST_P(CordTest, CordChunkIteratorOperations) { absl::Cord empty_cord; VerifyChunkIterator(empty_cord, 0); absl::Cord small_buffer_cord("small cord"); + MaybeHarden(small_buffer_cord); VerifyChunkIterator(small_buffer_cord, 1); absl::Cord flat_node_cord("larger than small buffer optimization"); + MaybeHarden(flat_node_cord); VerifyChunkIterator(flat_node_cord, 1); - VerifyChunkIterator( - absl::MakeFragmentedCord({"a ", "small ", "fragmented ", "cord ", "for ", - "testing ", "chunk ", "iterations."}), - 8); + VerifyChunkIterator(MaybeHardened(absl::MakeFragmentedCord( + {"a ", "small ", "fragmented ", "cord ", "for ", + "testing ", "chunk ", "iterations."})), + 8); absl::Cord reused_nodes_cord(std::string(40, 'c')); reused_nodes_cord.Prepend(absl::Cord(std::string(40, 'b'))); + MaybeHarden(reused_nodes_cord); reused_nodes_cord.Prepend(absl::Cord(std::string(40, 'a'))); size_t expected_chunks = 3; for (int i = 0; i < 8; ++i) { reused_nodes_cord.Prepend(reused_nodes_cord); + MaybeHarden(reused_nodes_cord); expected_chunks *= 2; VerifyChunkIterator(reused_nodes_cord, expected_chunks); } - RandomEngine rng(testing::GTEST_FLAG(random_seed)); + RandomEngine rng(GTEST_FLAG_GET(random_seed)); absl::Cord flat_cord(RandomLowercaseString(&rng, 256)); absl::Cord subcords; for (int i = 0; i < 128; ++i) subcords.Prepend(flat_cord.Subcord(i, 128)); VerifyChunkIterator(subcords, 128); } -TEST(CordCharIterator, Traits) { + +TEST_P(CordTest, AdvanceAndReadOnDataEdge) { + RandomEngine rng(GTEST_FLAG_GET(random_seed)); + const std::string data = RandomLowercaseString(&rng, 2000); + for (bool as_flat : {true, false}) { + SCOPED_TRACE(as_flat ? "Flat" : "External"); + + absl::Cord cord = + as_flat ? absl::Cord(data) + : absl::MakeCordFromExternal(data, [](absl::string_view) {}); + auto it = cord.Chars().begin(); +#if !defined(NDEBUG) || ABSL_OPTION_HARDENED + EXPECT_DEATH_IF_SUPPORTED(cord.AdvanceAndRead(&it, 2001), ".*"); +#endif + + it = cord.Chars().begin(); + absl::Cord frag = cord.AdvanceAndRead(&it, 2000); + EXPECT_EQ(frag, data); + EXPECT_TRUE(it == cord.Chars().end()); + + it = cord.Chars().begin(); + frag = cord.AdvanceAndRead(&it, 200); + EXPECT_EQ(frag, data.substr(0, 200)); + EXPECT_FALSE(it == cord.Chars().end()); + + frag = cord.AdvanceAndRead(&it, 1500); + EXPECT_EQ(frag, data.substr(200, 1500)); + EXPECT_FALSE(it == cord.Chars().end()); + + frag = cord.AdvanceAndRead(&it, 300); + EXPECT_EQ(frag, data.substr(1700, 300)); + EXPECT_TRUE(it == cord.Chars().end()); + } +} + +TEST_P(CordTest, AdvanceAndReadOnSubstringDataEdge) { + RandomEngine rng(GTEST_FLAG_GET(random_seed)); + const std::string data = RandomLowercaseString(&rng, 2500); + for (bool as_flat : {true, false}) { + SCOPED_TRACE(as_flat ? "Flat" : "External"); + + absl::Cord cord = + as_flat ? absl::Cord(data) + : absl::MakeCordFromExternal(data, [](absl::string_view) {}); + cord = cord.Subcord(200, 2000); + const std::string substr = data.substr(200, 2000); + + auto it = cord.Chars().begin(); +#if !defined(NDEBUG) || ABSL_OPTION_HARDENED + EXPECT_DEATH_IF_SUPPORTED(cord.AdvanceAndRead(&it, 2001), ".*"); +#endif + + it = cord.Chars().begin(); + absl::Cord frag = cord.AdvanceAndRead(&it, 2000); + EXPECT_EQ(frag, substr); + EXPECT_TRUE(it == cord.Chars().end()); + + it = cord.Chars().begin(); + frag = cord.AdvanceAndRead(&it, 200); + EXPECT_EQ(frag, substr.substr(0, 200)); + EXPECT_FALSE(it == cord.Chars().end()); + + frag = cord.AdvanceAndRead(&it, 1500); + EXPECT_EQ(frag, substr.substr(200, 1500)); + EXPECT_FALSE(it == cord.Chars().end()); + + frag = cord.AdvanceAndRead(&it, 300); + EXPECT_EQ(frag, substr.substr(1700, 300)); + EXPECT_TRUE(it == cord.Chars().end()); + } +} + +TEST_P(CordTest, CharIteratorTraits) { static_assert(std::is_copy_constructible<absl::Cord::CharIterator>::value, ""); static_assert(std::is_copy_assignable<absl::Cord::CharIterator>::value, ""); @@ -1536,44 +2312,88 @@ static void VerifyCharIterator(const absl::Cord& cord) { } } -TEST(CordCharIterator, Operations) { +TEST_P(CordTest, CharIteratorOperations) { absl::Cord empty_cord; VerifyCharIterator(empty_cord); absl::Cord small_buffer_cord("small cord"); + MaybeHarden(small_buffer_cord); VerifyCharIterator(small_buffer_cord); absl::Cord flat_node_cord("larger than small buffer optimization"); + MaybeHarden(flat_node_cord); VerifyCharIterator(flat_node_cord); - VerifyCharIterator( + VerifyCharIterator(MaybeHardened( absl::MakeFragmentedCord({"a ", "small ", "fragmented ", "cord ", "for ", - "testing ", "character ", "iteration."})); + "testing ", "character ", "iteration."}))); absl::Cord reused_nodes_cord("ghi"); reused_nodes_cord.Prepend(absl::Cord("def")); reused_nodes_cord.Prepend(absl::Cord("abc")); for (int i = 0; i < 4; ++i) { reused_nodes_cord.Prepend(reused_nodes_cord); + MaybeHarden(reused_nodes_cord); VerifyCharIterator(reused_nodes_cord); } - RandomEngine rng(testing::GTEST_FLAG(random_seed)); + RandomEngine rng(GTEST_FLAG_GET(random_seed)); absl::Cord flat_cord(RandomLowercaseString(&rng, 256)); absl::Cord subcords; - for (int i = 0; i < 4; ++i) subcords.Prepend(flat_cord.Subcord(16 * i, 128)); + for (int i = 0; i < 4; ++i) { + subcords.Prepend(flat_cord.Subcord(16 * i, 128)); + MaybeHarden(subcords); + } VerifyCharIterator(subcords); } -TEST(Cord, StreamingOutput) { +TEST_P(CordTest, CharIteratorAdvanceAndRead) { + // Create a Cord holding 6 flats of 2500 bytes each, and then iterate over it + // reading 150, 1500, 2500 and 3000 bytes. This will result in all possible + // partial, full and straddled read combinations including reads below + // kMaxBytesToCopy. b/197776822 surfaced a bug for a specific partial, small + // read 'at end' on Cord which caused a failure on attempting to read past the + // end in CordRepBtreeReader which was not covered by any existing test. + constexpr int kBlocks = 6; + constexpr size_t kBlockSize = 2500; + constexpr size_t kChunkSize1 = 1500; + constexpr size_t kChunkSize2 = 2500; + constexpr size_t kChunkSize3 = 3000; + constexpr size_t kChunkSize4 = 150; + RandomEngine rng; + std::string data = RandomLowercaseString(&rng, kBlocks * kBlockSize); + absl::Cord cord; + for (int i = 0; i < kBlocks; ++i) { + const std::string block = data.substr(i * kBlockSize, kBlockSize); + cord.Append(absl::Cord(block)); + } + + MaybeHarden(cord); + + for (size_t chunk_size : + {kChunkSize1, kChunkSize2, kChunkSize3, kChunkSize4}) { + absl::Cord::CharIterator it = cord.char_begin(); + size_t offset = 0; + while (offset < data.length()) { + const size_t n = std::min<size_t>(data.length() - offset, chunk_size); + absl::Cord chunk = cord.AdvanceAndRead(&it, n); + ASSERT_EQ(chunk.size(), n); + ASSERT_EQ(chunk.Compare(data.substr(offset, n)), 0); + offset += n; + } + } +} + +TEST_P(CordTest, StreamingOutput) { absl::Cord c = absl::MakeFragmentedCord({"A ", "small ", "fragmented ", "Cord", "."}); + MaybeHarden(c); std::stringstream output; output << c; EXPECT_EQ("A small fragmented Cord.", output.str()); } -TEST(Cord, ForEachChunk) { +TEST_P(CordTest, ForEachChunk) { for (int num_elements : {1, 10, 200}) { SCOPED_TRACE(num_elements); std::vector<std::string> cord_chunks; @@ -1581,6 +2401,7 @@ TEST(Cord, ForEachChunk) { cord_chunks.push_back(absl::StrCat("[", i, "]")); } absl::Cord c = absl::MakeFragmentedCord(cord_chunks); + MaybeHarden(c); std::vector<std::string> iterated_chunks; absl::CordTestPeer::ForEachChunk(c, @@ -1591,13 +2412,14 @@ TEST(Cord, ForEachChunk) { } } -TEST(Cord, SmallBufferAssignFromOwnData) { +TEST_P(CordTest, SmallBufferAssignFromOwnData) { constexpr size_t kMaxInline = 15; std::string contents = "small buff cord"; EXPECT_EQ(contents.size(), kMaxInline); for (size_t pos = 0; pos < contents.size(); ++pos) { for (size_t count = contents.size() - pos; count > 0; --count) { absl::Cord c(contents); + MaybeHarden(c); absl::string_view flat = c.Flatten(); c = flat.substr(pos, count); EXPECT_EQ(c, contents.substr(pos, count)) @@ -1606,16 +2428,20 @@ TEST(Cord, SmallBufferAssignFromOwnData) { } } -TEST(Cord, Format) { +TEST_P(CordTest, Format) { absl::Cord c; absl::Format(&c, "There were %04d little %s.", 3, "pigs"); EXPECT_EQ(c, "There were 0003 little pigs."); + MaybeHarden(c); absl::Format(&c, "And %-3llx bad wolf!", 1); + MaybeHarden(c); EXPECT_EQ(c, "There were 0003 little pigs.And 1 bad wolf!"); } -TEST(CordDeathTest, Hardening) { +TEST_P(CordTest, Hardening) { absl::Cord cord("hello"); + MaybeHarden(cord); + // These statement should abort the program in all builds modes. EXPECT_DEATH_IF_SUPPORTED(cord.RemovePrefix(6), ""); EXPECT_DEATH_IF_SUPPORTED(cord.RemoveSuffix(6), ""); @@ -1634,6 +2460,49 @@ TEST(CordDeathTest, Hardening) { EXPECT_DEATH_IF_SUPPORTED(++cord.chunk_end(), ""); } +// This test mimics a specific (and rare) application repeatedly splitting a +// cord, inserting (overwriting) a string value, and composing a new cord from +// the three pieces. This is hostile towards a Btree implementation: A split of +// a node at any level is likely to have the right-most edge of the left split, +// and the left-most edge of the right split shared. For example, splitting a +// leaf node with 6 edges will result likely in a 1-6, 2-5, 3-4, etc. split, +// sharing the 'split node'. When recomposing such nodes, we 'injected' an edge +// in that node. As this happens with some probability on each level of the +// tree, this will quickly grow the tree until it reaches maximum height. +TEST_P(CordTest, BtreeHostileSplitInsertJoin) { + absl::BitGen bitgen; + + // Start with about 1GB of data + std::string data(1 << 10, 'x'); + absl::Cord buffer(data); + absl::Cord cord; + for (int i = 0; i < 1000000; ++i) { + cord.Append(buffer); + } + + for (int j = 0; j < 1000; ++j) { + MaybeHarden(cord); + size_t offset = absl::Uniform(bitgen, 0u, cord.size()); + size_t length = absl::Uniform(bitgen, 100u, data.size()); + if (cord.size() == offset) { + cord.Append(absl::string_view(data.data(), length)); + } else { + absl::Cord suffix; + if (offset + length < cord.size()) { + suffix = cord; + suffix.RemovePrefix(offset + length); + } + if (cord.size() > offset) { + cord.RemoveSuffix(cord.size() - offset); + } + cord.Append(absl::string_view(data.data(), length)); + if (!suffix.empty()) { + cord.Append(suffix); + } + } + } +} + class AfterExitCordTester { public: bool Set(absl::Cord* cord, absl::string_view expected) { @@ -1650,12 +2519,34 @@ class AfterExitCordTester { absl::string_view expected_; }; +// Deliberately prevents the destructor for an absl::Cord from running. The cord +// is accessible via the cord member during the lifetime of the CordLeaker. +// After the CordLeaker is destroyed, pointers to the cord will remain valid +// until the CordLeaker's memory is deallocated. +struct CordLeaker { + union { + absl::Cord cord; + }; + + template <typename Str> + constexpr explicit CordLeaker(const Str& str) : cord(str) {} + + ~CordLeaker() { + // Don't do anything, including running cord's destructor. (cord's + // destructor won't run automatically because cord is hidden inside a + // union.) + } +}; + template <typename Str> void TestConstinitConstructor(Str) { const auto expected = Str::value; // Defined before `cord` to be destroyed after it. static AfterExitCordTester exit_tester; // NOLINT - ABSL_CONST_INIT static absl::Cord cord(Str{}); // NOLINT + ABSL_CONST_INIT static CordLeaker cord_leaker(Str{}); // NOLINT + // cord_leaker is static, so this reference will remain valid through the end + // of program execution. + static absl::Cord& cord = cord_leaker.cord; static bool init_exit_tester = exit_tester.Set(&cord, expected); (void)init_exit_tester; @@ -1707,9 +2598,278 @@ struct LongView { }; -TEST(Cord, ConstinitConstructor) { +TEST_P(CordTest, ConstinitConstructor) { TestConstinitConstructor( absl::strings_internal::MakeStringConstant(ShortView{})); TestConstinitConstructor( absl::strings_internal::MakeStringConstant(LongView{})); } + +namespace { + +// Test helper that generates a populated cord for future manipulation. +// +// By test convention, all generated cords begin with the characters "abcde" at +// the start of the first chunk. +class PopulatedCordFactory { + public: + constexpr PopulatedCordFactory(absl::string_view name, + absl::Cord (*generator)()) + : name_(name), generator_(generator) {} + + absl::string_view Name() const { return name_; } + absl::Cord Generate() const { return generator_(); } + + private: + absl::string_view name_; + absl::Cord (*generator_)(); +}; + +// clang-format off +// This array is constant-initialized in conformant compilers. +PopulatedCordFactory cord_factories[] = { + {"sso", [] { return absl::Cord("abcde"); }}, + {"flat", [] { + // Too large to live in SSO space, but small enough to be a simple FLAT. + absl::Cord flat(absl::StrCat("abcde", std::string(1000, 'x'))); + flat.Flatten(); + return flat; + }}, + {"external", [] { + // A cheat: we are using a string literal as the external storage, so a + // no-op releaser is correct here. + return absl::MakeCordFromExternal("abcde External!", []{}); + }}, + {"external substring", [] { + // A cheat: we are using a string literal as the external storage, so a + // no-op releaser is correct here. + absl::Cord ext = absl::MakeCordFromExternal("-abcde External!", []{}); + return absl::CordTestPeer::MakeSubstring(ext, 1, ext.size() - 1); + }}, + {"substring", [] { + absl::Cord flat(absl::StrCat("-abcde", std::string(1000, 'x'))); + flat.Flatten(); + return flat.Subcord(1, 998); + }}, + {"fragmented", [] { + std::string fragment = absl::StrCat("abcde", std::string(195, 'x')); + std::vector<std::string> fragments(200, fragment); + absl::Cord cord = absl::MakeFragmentedCord(fragments); + assert(cord.size() == 40000); + return cord; + }}, +}; +// clang-format on + +// Test helper that can mutate a cord, and possibly undo the mutation, for +// testing. +class CordMutator { + public: + constexpr CordMutator(absl::string_view name, void (*mutate)(absl::Cord&), + void (*undo)(absl::Cord&) = nullptr) + : name_(name), mutate_(mutate), undo_(undo) {} + + absl::string_view Name() const { return name_; } + void Mutate(absl::Cord& cord) const { mutate_(cord); } + bool CanUndo() const { return undo_ != nullptr; } + void Undo(absl::Cord& cord) const { undo_(cord); } + + private: + absl::string_view name_; + void (*mutate_)(absl::Cord&); + void (*undo_)(absl::Cord&); +}; + +// clang-format off +// This array is constant-initialized in conformant compilers. +CordMutator cord_mutators[] ={ + {"clear", [](absl::Cord& c) { c.Clear(); }}, + {"overwrite", [](absl::Cord& c) { c = "overwritten"; }}, + { + "append string", + [](absl::Cord& c) { c.Append("0123456789"); }, + [](absl::Cord& c) { c.RemoveSuffix(10); } + }, + { + "append cord", + [](absl::Cord& c) { + c.Append(absl::MakeFragmentedCord({"12345", "67890"})); + }, + [](absl::Cord& c) { c.RemoveSuffix(10); } + }, + { + "append checksummed cord", + [](absl::Cord& c) { + absl::Cord to_append = absl::MakeFragmentedCord({"12345", "67890"}); + to_append.SetExpectedChecksum(999); + c.Append(to_append); + }, + [](absl::Cord& c) { c.RemoveSuffix(10); } + }, + { + "append self", + [](absl::Cord& c) { c.Append(c); }, + [](absl::Cord& c) { c.RemoveSuffix(c.size() / 2); } + }, + { + "prepend string", + [](absl::Cord& c) { c.Prepend("9876543210"); }, + [](absl::Cord& c) { c.RemovePrefix(10); } + }, + { + "prepend cord", + [](absl::Cord& c) { + c.Prepend(absl::MakeFragmentedCord({"98765", "43210"})); + }, + [](absl::Cord& c) { c.RemovePrefix(10); } + }, + { + "prepend checksummed cord", + [](absl::Cord& c) { + absl::Cord to_prepend = absl::MakeFragmentedCord({"98765", "43210"}); + to_prepend.SetExpectedChecksum(999); + c.Prepend(to_prepend); + }, + [](absl::Cord& c) { c.RemovePrefix(10); } + }, + { + "prepend self", + [](absl::Cord& c) { c.Prepend(c); }, + [](absl::Cord& c) { c.RemovePrefix(c.size() / 2); } + }, + {"remove prefix", [](absl::Cord& c) { c.RemovePrefix(2); }}, + {"remove suffix", [](absl::Cord& c) { c.RemoveSuffix(2); }}, + {"subcord", [](absl::Cord& c) { c = c.Subcord(1, c.size() - 2); }}, + { + "swap inline", + [](absl::Cord& c) { + absl::Cord other("swap"); + c.swap(other); + } + }, + { + "swap tree", + [](absl::Cord& c) { + absl::Cord other(std::string(10000, 'x')); + c.swap(other); + } + }, +}; +// clang-format on +} // namespace + +TEST_P(CordTest, ExpectedChecksum) { + for (const PopulatedCordFactory& factory : cord_factories) { + SCOPED_TRACE(factory.Name()); + for (bool shared : {false, true}) { + SCOPED_TRACE(shared); + + absl::Cord shared_cord_source = factory.Generate(); + auto make_instance = [=] { + return shared ? shared_cord_source : factory.Generate(); + }; + + const absl::Cord base_value = factory.Generate(); + const std::string base_value_as_string(factory.Generate().Flatten()); + + absl::Cord c1 = make_instance(); + EXPECT_FALSE(c1.ExpectedChecksum().has_value()); + + // Setting an expected checksum works, and retains the cord's bytes + c1.SetExpectedChecksum(12345); + EXPECT_EQ(c1.ExpectedChecksum().value_or(0), 12345); + EXPECT_EQ(c1, base_value); + + // CRC persists through copies, assignments, and moves: + absl::Cord c1_copy_construct = c1; + EXPECT_EQ(c1_copy_construct.ExpectedChecksum().value_or(0), 12345); + + absl::Cord c1_copy_assign; + c1_copy_assign = c1; + EXPECT_EQ(c1_copy_assign.ExpectedChecksum().value_or(0), 12345); + + absl::Cord c1_move(std::move(c1_copy_assign)); + EXPECT_EQ(c1_move.ExpectedChecksum().value_or(0), 12345); + + EXPECT_EQ(c1.ExpectedChecksum().value_or(0), 12345); + + // A CRC Cord compares equal to its non-CRC value. + EXPECT_EQ(c1, make_instance()); + + for (const CordMutator& mutator : cord_mutators) { + SCOPED_TRACE(mutator.Name()); + + // Test that mutating a cord removes its stored checksum + absl::Cord c2 = make_instance(); + c2.SetExpectedChecksum(24680); + + mutator.Mutate(c2); + EXPECT_EQ(c2.ExpectedChecksum(), absl::nullopt); + + if (mutator.CanUndo()) { + // Undoing an operation should not restore the checksum + mutator.Undo(c2); + EXPECT_EQ(c2, base_value); + EXPECT_EQ(c2.ExpectedChecksum(), absl::nullopt); + } + } + + absl::Cord c3 = make_instance(); + c3.SetExpectedChecksum(999); + const absl::Cord& cc3 = c3; + + // Test that all cord reading operations function in the face of an + // expected checksum. + + // Test data precondition + ASSERT_TRUE(cc3.StartsWith("abcde")); + + EXPECT_EQ(cc3.size(), base_value_as_string.size()); + EXPECT_FALSE(cc3.empty()); + EXPECT_EQ(cc3.Compare(base_value), 0); + EXPECT_EQ(cc3.Compare(base_value_as_string), 0); + EXPECT_EQ(cc3.Compare("wxyz"), -1); + EXPECT_EQ(cc3.Compare(absl::Cord("wxyz")), -1); + EXPECT_EQ(cc3.Compare("aaaa"), 1); + EXPECT_EQ(cc3.Compare(absl::Cord("aaaa")), 1); + EXPECT_EQ(absl::Cord("wxyz").Compare(cc3), 1); + EXPECT_EQ(absl::Cord("aaaa").Compare(cc3), -1); + EXPECT_TRUE(cc3.StartsWith("abcd")); + EXPECT_EQ(std::string(cc3), base_value_as_string); + + std::string dest; + absl::CopyCordToString(cc3, &dest); + EXPECT_EQ(dest, base_value_as_string); + + bool first_pass = true; + for (absl::string_view chunk : cc3.Chunks()) { + if (first_pass) { + EXPECT_TRUE(absl::StartsWith(chunk, "abcde")); + } + first_pass = false; + } + first_pass = true; + for (char ch : cc3.Chars()) { + if (first_pass) { + EXPECT_EQ(ch, 'a'); + } + first_pass = false; + } + EXPECT_TRUE(absl::StartsWith(*cc3.chunk_begin(), "abcde")); + EXPECT_EQ(*cc3.char_begin(), 'a'); + + auto char_it = cc3.char_begin(); + absl::Cord::Advance(&char_it, 2); + EXPECT_EQ(absl::Cord::AdvanceAndRead(&char_it, 2), "cd"); + EXPECT_EQ(*char_it, 'e'); + char_it = cc3.char_begin(); + absl::Cord::Advance(&char_it, 2); + EXPECT_TRUE(absl::StartsWith(absl::Cord::ChunkRemaining(char_it), "cde")); + + EXPECT_EQ(cc3[0], 'a'); + EXPECT_EQ(cc3[4], 'e'); + EXPECT_EQ(absl::HashOf(cc3), absl::HashOf(base_value)); + EXPECT_EQ(absl::HashOf(cc3), absl::HashOf(base_value_as_string)); + } + } +} diff --git a/absl/strings/cord_test_helpers.h b/absl/strings/cord_test_helpers.h index f1036e3b..31a1dc89 100644 --- a/absl/strings/cord_test_helpers.h +++ b/absl/strings/cord_test_helpers.h @@ -17,11 +17,73 @@ #ifndef ABSL_STRINGS_CORD_TEST_HELPERS_H_ #define ABSL_STRINGS_CORD_TEST_HELPERS_H_ +#include <cstdint> +#include <iostream> +#include <string> + +#include "absl/base/config.h" #include "absl/strings/cord.h" +#include "absl/strings/internal/cord_internal.h" +#include "absl/strings/string_view.h" namespace absl { ABSL_NAMESPACE_BEGIN +// Cord sizes relevant for testing +enum class TestCordSize { + // An empty value + kEmpty = 0, + + // An inlined string value + kInlined = cord_internal::kMaxInline / 2 + 1, + + // 'Well known' SSO lengths (excluding terminating zero). + // libstdcxx has a maximum SSO of 15, libc++ has a maximum SSO of 22. + kStringSso1 = 15, + kStringSso2 = 22, + + // A string value which is too large to fit in inlined data, but small enough + // such that Cord prefers copying the value if possible, i.e.: not stealing + // std::string inputs, or referencing existing CordReps on Append, etc. + kSmall = cord_internal::kMaxBytesToCopy / 2 + 1, + + // A string value large enough that Cord prefers to reference or steal from + // existing inputs rather than copying contents of the input. + kMedium = cord_internal::kMaxFlatLength / 2 + 1, + + // A string value large enough to cause it to be stored in mutliple flats. + kLarge = cord_internal::kMaxFlatLength * 4 +}; + +// To string helper +inline absl::string_view ToString(TestCordSize size) { + switch (size) { + case TestCordSize::kEmpty: + return "Empty"; + case TestCordSize::kInlined: + return "Inlined"; + case TestCordSize::kSmall: + return "Small"; + case TestCordSize::kStringSso1: + return "StringSso1"; + case TestCordSize::kStringSso2: + return "StringSso2"; + case TestCordSize::kMedium: + return "Medium"; + case TestCordSize::kLarge: + return "Large"; + } + return "???"; +} + +// Returns the length matching the specified size +inline size_t Length(TestCordSize size) { return static_cast<size_t>(size); } + +// Stream output helper +inline std::ostream& operator<<(std::ostream& stream, TestCordSize size) { + return stream << ToString(size); +} + // Creates a multi-segment Cord from an iterable container of strings. The // resulting Cord is guaranteed to have one segment for every string in the // container. This allows code to be unit tested with multi-segment Cord diff --git a/absl/strings/cordz_test.cc b/absl/strings/cordz_test.cc new file mode 100644 index 00000000..2b7d30b0 --- /dev/null +++ b/absl/strings/cordz_test.cc @@ -0,0 +1,466 @@ +// Copyright 2021 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <cstdint> +#include <string> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/config.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/base/macros.h" +#include "absl/strings/cord.h" +#include "absl/strings/cord_test_helpers.h" +#include "absl/strings/cordz_test_helpers.h" +#include "absl/strings/internal/cordz_functions.h" +#include "absl/strings/internal/cordz_info.h" +#include "absl/strings/internal/cordz_sample_token.h" +#include "absl/strings/internal/cordz_statistics.h" +#include "absl/strings/internal/cordz_update_tracker.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" + +#ifdef ABSL_INTERNAL_CORDZ_ENABLED + +using testing::Eq; +using testing::AnyOf; + +namespace absl { +ABSL_NAMESPACE_BEGIN + +using cord_internal::CordzInfo; +using cord_internal::CordzSampleToken; +using cord_internal::CordzStatistics; +using cord_internal::CordzUpdateTracker; +using Method = CordzUpdateTracker::MethodIdentifier; + +// Do not print cord contents, we only care about 'size' perhaps. +// Note that this method must be inside the named namespace. +inline void PrintTo(const Cord& cord, std::ostream* s) { + if (s) *s << "Cord[" << cord.size() << "]"; +} + +namespace { + +auto constexpr kMaxInline = cord_internal::kMaxInline; + +// Returns a string_view value of the specified length +// We do this to avoid 'consuming' large strings in Cord by default. +absl::string_view MakeString(size_t size) { + thread_local std::string str; + str = std::string(size, '.'); + return str; +} + +absl::string_view MakeString(TestCordSize size) { + return MakeString(Length(size)); +} + +// Returns a cord with a sampled method of kAppendString. +absl::Cord MakeAppendStringCord(TestCordSize size) { + CordzSamplingIntervalHelper always(1); + absl::Cord cord; + cord.Append(MakeString(size)); + return cord; +} + +std::string TestParamToString(::testing::TestParamInfo<TestCordSize> size) { + return absl::StrCat("On", ToString(size.param), "Cord"); +} + +class CordzUpdateTest : public testing::TestWithParam<TestCordSize> { + public: + Cord& cord() { return cord_; } + + Method InitialOr(Method method) const { + return (GetParam() > TestCordSize::kInlined) ? Method::kConstructorString + : method; + } + + private: + CordzSamplingIntervalHelper sample_every_{1}; + Cord cord_{MakeString(GetParam())}; +}; + +template <typename T> +std::string ParamToString(::testing::TestParamInfo<T> param) { + return std::string(ToString(param.param)); +} + +INSTANTIATE_TEST_SUITE_P(WithParam, CordzUpdateTest, + testing::Values(TestCordSize::kEmpty, + TestCordSize::kInlined, + TestCordSize::kLarge), + TestParamToString); + +class CordzStringTest : public testing::TestWithParam<TestCordSize> { + private: + CordzSamplingIntervalHelper sample_every_{1}; +}; + +INSTANTIATE_TEST_SUITE_P(WithParam, CordzStringTest, + testing::Values(TestCordSize::kInlined, + TestCordSize::kStringSso1, + TestCordSize::kStringSso2, + TestCordSize::kSmall, + TestCordSize::kLarge), + ParamToString<TestCordSize>); + +TEST(CordzTest, ConstructSmallArray) { + CordzSamplingIntervalHelper sample_every{1}; + Cord cord(MakeString(TestCordSize::kSmall)); + EXPECT_THAT(cord, HasValidCordzInfoOf(Method::kConstructorString)); +} + +TEST(CordzTest, ConstructLargeArray) { + CordzSamplingIntervalHelper sample_every{1}; + Cord cord(MakeString(TestCordSize::kLarge)); + EXPECT_THAT(cord, HasValidCordzInfoOf(Method::kConstructorString)); +} + +TEST_P(CordzStringTest, ConstructString) { + CordzSamplingIntervalHelper sample_every{1}; + Cord cord(std::string(Length(GetParam()), '.')); + if (Length(GetParam()) > kMaxInline) { + EXPECT_THAT(cord, HasValidCordzInfoOf(Method::kConstructorString)); + } +} + +TEST(CordzTest, CopyConstructFromUnsampled) { + CordzSamplingIntervalHelper sample_every{1}; + Cord src = UnsampledCord(MakeString(TestCordSize::kLarge)); + Cord cord(src); + EXPECT_THAT(GetCordzInfoForTesting(cord), Eq(nullptr)); +} + +TEST(CordzTest, CopyConstructFromSampled) { + CordzSamplingIntervalHelper sample_never{99999}; + Cord src = MakeAppendStringCord(TestCordSize::kLarge); + Cord cord(src); + ASSERT_THAT(cord, HasValidCordzInfoOf(Method::kConstructorCord)); + CordzStatistics stats = GetCordzInfoForTesting(cord)->GetCordzStatistics(); + EXPECT_THAT(stats.parent_method, Eq(Method::kAppendString)); + EXPECT_THAT(stats.update_tracker.Value(Method::kAppendString), Eq(1)); +} + +TEST(CordzTest, MoveConstruct) { + CordzSamplingIntervalHelper sample_every{1}; + Cord src(MakeString(TestCordSize::kLarge)); + Cord cord(std::move(src)); + EXPECT_THAT(cord, HasValidCordzInfoOf(Method::kConstructorString)); +} + +TEST_P(CordzUpdateTest, AssignUnsampledCord) { + Cord src = UnsampledCord(MakeString(TestCordSize::kLarge)); + const CordzInfo* info = GetCordzInfoForTesting(cord()); + cord() = src; + EXPECT_THAT(GetCordzInfoForTesting(cord()), Eq(nullptr)); + EXPECT_FALSE(CordzInfoIsListed(info)); +} + +TEST_P(CordzUpdateTest, AssignSampledCord) { + Cord src = MakeAppendStringCord(TestCordSize::kLarge); + cord() = src; + ASSERT_THAT(cord(), HasValidCordzInfoOf(Method::kAssignCord)); + CordzStatistics stats = GetCordzInfoForTesting(cord())->GetCordzStatistics(); + EXPECT_THAT(stats.parent_method, Eq(Method::kAppendString)); + EXPECT_THAT(stats.update_tracker.Value(Method::kAppendString), Eq(1)); + EXPECT_THAT(stats.update_tracker.Value(Method::kConstructorString), Eq(0)); +} + +TEST(CordzUpdateTest, AssignSampledCordToInlined) { + CordzSamplingIntervalHelper sample_never{99999}; + Cord cord; + Cord src = MakeAppendStringCord(TestCordSize::kLarge); + cord = src; + ASSERT_THAT(cord, HasValidCordzInfoOf(Method::kAssignCord)); + CordzStatistics stats = GetCordzInfoForTesting(cord)->GetCordzStatistics(); + EXPECT_THAT(stats.parent_method, Eq(Method::kAppendString)); + EXPECT_THAT(stats.update_tracker.Value(Method::kAppendString), Eq(1)); + EXPECT_THAT(stats.update_tracker.Value(Method::kConstructorString), Eq(0)); +} + +TEST(CordzUpdateTest, AssignSampledCordToUnsampledCord) { + CordzSamplingIntervalHelper sample_never{99999}; + Cord cord = UnsampledCord(MakeString(TestCordSize::kLarge)); + Cord src = MakeAppendStringCord(TestCordSize::kLarge); + cord = src; + ASSERT_THAT(cord, HasValidCordzInfoOf(Method::kAssignCord)); + CordzStatistics stats = GetCordzInfoForTesting(cord)->GetCordzStatistics(); + EXPECT_THAT(stats.parent_method, Eq(Method::kAppendString)); + EXPECT_THAT(stats.update_tracker.Value(Method::kAppendString), Eq(1)); + EXPECT_THAT(stats.update_tracker.Value(Method::kConstructorString), Eq(0)); +} + +TEST(CordzUpdateTest, AssignUnsampledCordToSampledCordWithoutSampling) { + CordzSamplingIntervalHelper sample_never{99999}; + Cord cord = MakeAppendStringCord(TestCordSize::kLarge); + const CordzInfo* info = GetCordzInfoForTesting(cord); + Cord src = UnsampledCord(MakeString(TestCordSize::kLarge)); + cord = src; + EXPECT_THAT(GetCordzInfoForTesting(cord), Eq(nullptr)); + EXPECT_FALSE(CordzInfoIsListed(info)); +} + +TEST(CordzUpdateTest, AssignUnsampledCordToSampledCordWithSampling) { + CordzSamplingIntervalHelper sample_every{1}; + Cord cord = MakeAppendStringCord(TestCordSize::kLarge); + const CordzInfo* info = GetCordzInfoForTesting(cord); + Cord src = UnsampledCord(MakeString(TestCordSize::kLarge)); + cord = src; + EXPECT_THAT(GetCordzInfoForTesting(cord), Eq(nullptr)); + EXPECT_FALSE(CordzInfoIsListed(info)); +} + +TEST(CordzUpdateTest, AssignSampledCordToSampledCord) { + CordzSamplingIntervalHelper sample_every{1}; + Cord src = MakeAppendStringCord(TestCordSize::kLarge); + Cord cord(MakeString(TestCordSize::kLarge)); + cord = src; + ASSERT_THAT(cord, HasValidCordzInfoOf(Method::kAssignCord)); + CordzStatistics stats = GetCordzInfoForTesting(cord)->GetCordzStatistics(); + EXPECT_THAT(stats.parent_method, Eq(Method::kAppendString)); + EXPECT_THAT(stats.update_tracker.Value(Method::kAppendString), Eq(1)); + EXPECT_THAT(stats.update_tracker.Value(Method::kConstructorString), Eq(0)); +} + +TEST(CordzUpdateTest, AssignUnsampledCordToSampledCord) { + CordzSamplingIntervalHelper sample_every{1}; + Cord src = MakeAppendStringCord(TestCordSize::kLarge); + Cord cord(MakeString(TestCordSize::kLarge)); + cord = src; + ASSERT_THAT(cord, HasValidCordzInfoOf(Method::kAssignCord)); + CordzStatistics stats = GetCordzInfoForTesting(cord)->GetCordzStatistics(); + EXPECT_THAT(stats.parent_method, Eq(Method::kAppendString)); + EXPECT_THAT(stats.update_tracker.Value(Method::kAppendString), Eq(1)); + EXPECT_THAT(stats.update_tracker.Value(Method::kConstructorString), Eq(0)); +} + +TEST(CordzTest, AssignInlinedCordToSampledCord) { + CordzSampleToken token; + CordzSamplingIntervalHelper sample_every{1}; + Cord cord(MakeString(TestCordSize::kLarge)); + const CordzInfo* info = GetCordzInfoForTesting(cord); + Cord src = UnsampledCord(MakeString(TestCordSize::kInlined)); + cord = src; + EXPECT_THAT(GetCordzInfoForTesting(cord), Eq(nullptr)); + EXPECT_FALSE(CordzInfoIsListed(info)); +} + +TEST(CordzUpdateTest, MoveAssignCord) { + CordzSamplingIntervalHelper sample_every{1}; + Cord cord; + Cord src(MakeString(TestCordSize::kLarge)); + cord = std::move(src); + EXPECT_THAT(cord, HasValidCordzInfoOf(Method::kConstructorString)); +} + +TEST_P(CordzUpdateTest, AssignLargeArray) { + cord() = MakeString(TestCordSize::kSmall); + EXPECT_THAT(cord(), HasValidCordzInfoOf(Method::kAssignString)); +} + +TEST_P(CordzUpdateTest, AssignSmallArray) { + cord() = MakeString(TestCordSize::kSmall); + EXPECT_THAT(cord(), HasValidCordzInfoOf(Method::kAssignString)); +} + +TEST_P(CordzUpdateTest, AssignInlinedArray) { + cord() = MakeString(TestCordSize::kInlined); + EXPECT_THAT(GetCordzInfoForTesting(cord()), Eq(nullptr)); +} + +TEST_P(CordzStringTest, AssignStringToInlined) { + Cord cord; + cord = std::string(Length(GetParam()), '.'); + if (Length(GetParam()) > kMaxInline) { + EXPECT_THAT(cord, HasValidCordzInfoOf(Method::kAssignString)); + } +} + +TEST_P(CordzStringTest, AssignStringToCord) { + Cord cord(MakeString(TestCordSize::kLarge)); + cord = std::string(Length(GetParam()), '.'); + if (Length(GetParam()) > kMaxInline) { + EXPECT_THAT(cord, HasValidCordzInfoOf(Method::kConstructorString)); + EXPECT_THAT(cord, CordzMethodCountEq(Method::kAssignString, 1)); + } +} + +TEST_P(CordzUpdateTest, AssignInlinedString) { + cord() = std::string(Length(TestCordSize::kInlined), '.'); + EXPECT_THAT(GetCordzInfoForTesting(cord()), Eq(nullptr)); +} + +TEST_P(CordzUpdateTest, AppendCord) { + Cord src = UnsampledCord(MakeString(TestCordSize::kLarge)); + cord().Append(src); + EXPECT_THAT(cord(), HasValidCordzInfoOf(InitialOr(Method::kAppendCord))); +} + +TEST_P(CordzUpdateTest, MoveAppendCord) { + cord().Append(UnsampledCord(MakeString(TestCordSize::kLarge))); + EXPECT_THAT(cord(), HasValidCordzInfoOf(InitialOr(Method::kAppendCord))); +} + +TEST_P(CordzUpdateTest, AppendSmallArray) { + cord().Append(MakeString(TestCordSize::kSmall)); + EXPECT_THAT(cord(), HasValidCordzInfoOf(InitialOr(Method::kAppendString))); +} + +TEST_P(CordzUpdateTest, AppendLargeArray) { + cord().Append(MakeString(TestCordSize::kLarge)); + EXPECT_THAT(cord(), HasValidCordzInfoOf(InitialOr(Method::kAppendString))); +} + +TEST_P(CordzStringTest, AppendStringToEmpty) { + Cord cord; + cord.Append(std::string(Length(GetParam()), '.')); + if (Length(GetParam()) > kMaxInline) { + EXPECT_THAT(cord, HasValidCordzInfoOf(Method::kAppendString)); + } +} + +TEST_P(CordzStringTest, AppendStringToInlined) { + Cord cord(MakeString(TestCordSize::kInlined)); + cord.Append(std::string(Length(GetParam()), '.')); + if (Length(TestCordSize::kInlined) + Length(GetParam()) > kMaxInline) { + EXPECT_THAT(cord, HasValidCordzInfoOf(Method::kAppendString)); + } +} + +TEST_P(CordzStringTest, AppendStringToCord) { + Cord cord(MakeString(TestCordSize::kLarge)); + cord.Append(std::string(Length(GetParam()), '.')); + EXPECT_THAT(cord, HasValidCordzInfoOf(Method::kConstructorString)); + EXPECT_THAT(cord, CordzMethodCountEq(Method::kAppendString, 1)); +} + +TEST(CordzTest, MakeCordFromExternal) { + CordzSamplingIntervalHelper sample_every{1}; + Cord cord = MakeCordFromExternal("Hello world", [](absl::string_view) {}); + EXPECT_THAT(cord, HasValidCordzInfoOf(Method::kMakeCordFromExternal)); +} + +TEST(CordzTest, MakeCordFromEmptyExternal) { + CordzSamplingIntervalHelper sample_every{1}; + Cord cord = MakeCordFromExternal({}, [](absl::string_view) {}); + EXPECT_THAT(GetCordzInfoForTesting(cord), Eq(nullptr)); +} + +TEST_P(CordzUpdateTest, PrependCord) { + Cord src = UnsampledCord(MakeString(TestCordSize::kLarge)); + cord().Prepend(src); + EXPECT_THAT(cord(), HasValidCordzInfoOf(InitialOr(Method::kPrependCord))); +} + +TEST_P(CordzUpdateTest, PrependSmallArray) { + cord().Prepend(MakeString(TestCordSize::kSmall)); + EXPECT_THAT(cord(), HasValidCordzInfoOf(InitialOr(Method::kPrependString))); +} + +TEST_P(CordzUpdateTest, PrependLargeArray) { + cord().Prepend(MakeString(TestCordSize::kLarge)); + EXPECT_THAT(cord(), HasValidCordzInfoOf(InitialOr(Method::kPrependString))); +} + +TEST_P(CordzStringTest, PrependStringToEmpty) { + Cord cord; + cord.Prepend(std::string(Length(GetParam()), '.')); + if (Length(GetParam()) > kMaxInline) { + EXPECT_THAT(cord, HasValidCordzInfoOf(Method::kPrependString)); + } +} + +TEST_P(CordzStringTest, PrependStringToInlined) { + Cord cord(MakeString(TestCordSize::kInlined)); + cord.Prepend(std::string(Length(GetParam()), '.')); + if (Length(TestCordSize::kInlined) + Length(GetParam()) > kMaxInline) { + EXPECT_THAT(cord, HasValidCordzInfoOf(Method::kPrependString)); + } +} + +TEST_P(CordzStringTest, PrependStringToCord) { + Cord cord(MakeString(TestCordSize::kLarge)); + cord.Prepend(std::string(Length(GetParam()), '.')); + EXPECT_THAT(cord, HasValidCordzInfoOf(Method::kConstructorString)); + EXPECT_THAT(cord, CordzMethodCountEq(Method::kPrependString, 1)); +} + +TEST(CordzTest, RemovePrefix) { + CordzSamplingIntervalHelper sample_every(1); + Cord cord(MakeString(TestCordSize::kLarge)); + + // Half the cord + cord.RemovePrefix(cord.size() / 2); + EXPECT_THAT(cord, HasValidCordzInfoOf(Method::kConstructorString)); + EXPECT_THAT(cord, CordzMethodCountEq(Method::kRemovePrefix, 1)); + + // TODO(mvels): RemovePrefix does not reset to inlined, except if empty? + cord.RemovePrefix(cord.size() - kMaxInline); + EXPECT_THAT(cord, HasValidCordzInfoOf(Method::kConstructorString)); + EXPECT_THAT(cord, CordzMethodCountEq(Method::kRemovePrefix, 2)); + + cord.RemovePrefix(cord.size()); + EXPECT_THAT(GetCordzInfoForTesting(cord), Eq(nullptr)); +} + +TEST(CordzTest, RemoveSuffix) { + CordzSamplingIntervalHelper sample_every(1); + Cord cord(MakeString(TestCordSize::kLarge)); + + // Half the cord + cord.RemoveSuffix(cord.size() / 2); + EXPECT_THAT(cord, HasValidCordzInfoOf(Method::kConstructorString)); + EXPECT_THAT(cord, CordzMethodCountEq(Method::kRemoveSuffix, 1)); + + // TODO(mvels): RemoveSuffix does not reset to inlined, except if empty? + cord.RemoveSuffix(cord.size() - kMaxInline); + EXPECT_THAT(cord, HasValidCordzInfoOf(Method::kConstructorString)); + EXPECT_THAT(cord, CordzMethodCountEq(Method::kRemoveSuffix, 2)); + + cord.RemoveSuffix(cord.size()); + EXPECT_THAT(GetCordzInfoForTesting(cord), Eq(nullptr)); +} + +TEST(CordzTest, SubCordFromUnsampledCord) { + CordzSamplingIntervalHelper sample_every{1}; + Cord src = UnsampledCord(MakeString(TestCordSize::kLarge)); + Cord cord = src.Subcord(10, src.size() / 2); + EXPECT_THAT(GetCordzInfoForTesting(cord), Eq(nullptr)); +} + +TEST(CordzTest, SubCordFromSampledCord) { + CordzSamplingIntervalHelper sample_never{99999}; + Cord src = MakeAppendStringCord(TestCordSize::kLarge); + Cord cord = src.Subcord(10, src.size() / 2); + ASSERT_THAT(cord, HasValidCordzInfoOf(Method::kSubCord)); + CordzStatistics stats = GetCordzInfoForTesting(cord)->GetCordzStatistics(); + EXPECT_THAT(stats.parent_method, Eq(Method::kAppendString)); + EXPECT_THAT(stats.update_tracker.Value(Method::kAppendString), Eq(1)); +} + +TEST(CordzTest, SmallSubCord) { + CordzSamplingIntervalHelper sample_never{99999}; + Cord src = MakeAppendStringCord(TestCordSize::kLarge); + Cord cord = src.Subcord(10, kMaxInline + 1); + EXPECT_THAT(cord, HasValidCordzInfoOf(Method::kSubCord)); +} + +} // namespace + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_INTERNAL_CORDZ_ENABLED diff --git a/absl/strings/cordz_test_helpers.h b/absl/strings/cordz_test_helpers.h new file mode 100644 index 00000000..e410eecf --- /dev/null +++ b/absl/strings/cordz_test_helpers.h @@ -0,0 +1,151 @@ +// Copyright 2021 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_STRINGS_CORDZ_TEST_HELPERS_H_ +#define ABSL_STRINGS_CORDZ_TEST_HELPERS_H_ + +#include <utility> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/config.h" +#include "absl/base/macros.h" +#include "absl/strings/cord.h" +#include "absl/strings/internal/cord_internal.h" +#include "absl/strings/internal/cordz_info.h" +#include "absl/strings/internal/cordz_sample_token.h" +#include "absl/strings/internal/cordz_statistics.h" +#include "absl/strings/internal/cordz_update_tracker.h" +#include "absl/strings/str_cat.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN + +// Returns the CordzInfo for the cord, or nullptr if the cord is not sampled. +inline const cord_internal::CordzInfo* GetCordzInfoForTesting( + const Cord& cord) { + if (!cord.contents_.is_tree()) return nullptr; + return cord.contents_.cordz_info(); +} + +// Returns true if the provided cordz_info is in the list of sampled cords. +inline bool CordzInfoIsListed(const cord_internal::CordzInfo* cordz_info, + cord_internal::CordzSampleToken token = {}) { + for (const cord_internal::CordzInfo& info : token) { + if (cordz_info == &info) return true; + } + return false; +} + +// Matcher on Cord that verifies all of: +// - the cord is sampled +// - the CordzInfo of the cord is listed / discoverable. +// - the reported CordzStatistics match the cord's actual properties +// - the cord has an (initial) UpdateTracker count of 1 for `method` +MATCHER_P(HasValidCordzInfoOf, method, "CordzInfo matches cord") { + const cord_internal::CordzInfo* cord_info = GetCordzInfoForTesting(arg); + if (cord_info == nullptr) { + *result_listener << "cord is not sampled"; + return false; + } + if (!CordzInfoIsListed(cord_info)) { + *result_listener << "cord is sampled, but not listed"; + return false; + } + cord_internal::CordzStatistics stat = cord_info->GetCordzStatistics(); + if (stat.size != arg.size()) { + *result_listener << "cordz size " << stat.size + << " does not match cord size " << arg.size(); + return false; + } + if (stat.update_tracker.Value(method) != 1) { + *result_listener << "Expected method count 1 for " << method << ", found " + << stat.update_tracker.Value(method); + return false; + } + return true; +} + +// Matcher on Cord that verifies that the cord is sampled and that the CordzInfo +// update tracker has 'method' with a call count of 'n' +MATCHER_P2(CordzMethodCountEq, method, n, + absl::StrCat("CordzInfo method count equals ", n)) { + const cord_internal::CordzInfo* cord_info = GetCordzInfoForTesting(arg); + if (cord_info == nullptr) { + *result_listener << "cord is not sampled"; + return false; + } + cord_internal::CordzStatistics stat = cord_info->GetCordzStatistics(); + if (stat.update_tracker.Value(method) != n) { + *result_listener << "Expected method count " << n << " for " << method + << ", found " << stat.update_tracker.Value(method); + return false; + } + return true; +} + +// Cordz will only update with a new rate once the previously scheduled event +// has fired. When we disable Cordz, a long delay takes place where we won't +// consider profiling new Cords. CordzSampleIntervalHelper will burn through +// that interval and allow for testing that assumes that the average sampling +// interval is a particular value. +class CordzSamplingIntervalHelper { + public: + explicit CordzSamplingIntervalHelper(int32_t interval) + : orig_mean_interval_(absl::cord_internal::get_cordz_mean_interval()) { + absl::cord_internal::set_cordz_mean_interval(interval); + absl::cord_internal::cordz_set_next_sample_for_testing(interval); + } + + ~CordzSamplingIntervalHelper() { + absl::cord_internal::set_cordz_mean_interval(orig_mean_interval_); + absl::cord_internal::cordz_set_next_sample_for_testing(orig_mean_interval_); + } + + private: + int32_t orig_mean_interval_; +}; + +// Wrapper struct managing a small CordRep `rep` +struct TestCordRep { + cord_internal::CordRepFlat* rep; + + TestCordRep() { + rep = cord_internal::CordRepFlat::New(100); + rep->length = 100; + memset(rep->Data(), 1, 100); + } + ~TestCordRep() { cord_internal::CordRep::Unref(rep); } +}; + +// Wrapper struct managing a small CordRep `rep`, and +// an InlineData `data` initialized with that CordRep. +struct TestCordData { + TestCordRep rep; + cord_internal::InlineData data{rep.rep}; +}; + +// Creates a Cord that is not sampled +template <typename... Args> +Cord UnsampledCord(Args... args) { + CordzSamplingIntervalHelper never(9999); + Cord cord(std::forward<Args>(args)...); + ABSL_ASSERT(GetCordzInfoForTesting(cord) == nullptr); + return cord; +} + +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_CORDZ_TEST_HELPERS_H_ diff --git a/absl/strings/internal/charconv_parse.cc b/absl/strings/internal/charconv_parse.cc index 8b11868c..d29acaf4 100644 --- a/absl/strings/internal/charconv_parse.cc +++ b/absl/strings/internal/charconv_parse.cc @@ -52,7 +52,7 @@ static_assert(std::numeric_limits<double>::digits == 53, "IEEE double fact"); // The lowest valued 19-digit decimal mantissa we can read still contains // sufficient information to reconstruct a binary mantissa. -static_assert(1000000000000000000u > (uint64_t(1) << (53 + 3)), "(b) above"); +static_assert(1000000000000000000u > (uint64_t{1} << (53 + 3)), "(b) above"); // ParseFloat<16> will read the first 15 significant digits of the mantissa. // diff --git a/absl/strings/internal/cord_data_edge.h b/absl/strings/internal/cord_data_edge.h new file mode 100644 index 00000000..e18b33e1 --- /dev/null +++ b/absl/strings/internal/cord_data_edge.h @@ -0,0 +1,63 @@ +// Copyright 2022 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_STRINGS_INTERNAL_CORD_DATA_EDGE_H_ +#define ABSL_STRINGS_INTERNAL_CORD_DATA_EDGE_H_ + +#include <cassert> +#include <cstddef> + +#include "absl/base/config.h" +#include "absl/strings/internal/cord_internal.h" +#include "absl/strings/internal/cord_rep_flat.h" +#include "absl/strings/string_view.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { + +// Returns true if the provided rep is a FLAT, EXTERNAL or a SUBSTRING node +// holding a FLAT or EXTERNAL child rep. Requires `rep != nullptr`. +inline bool IsDataEdge(const CordRep* edge) { + assert(edge != nullptr); + + // The fast path is that `edge` is an EXTERNAL or FLAT node, making the below + // if a single, well predicted branch. We then repeat the FLAT or EXTERNAL + // check in the slow path of the SUBSTRING check to optimize for the hot path. + if (edge->tag == EXTERNAL || edge->tag >= FLAT) return true; + if (edge->tag == SUBSTRING) edge = edge->substring()->child; + return edge->tag == EXTERNAL || edge->tag >= FLAT; +} + +// Returns the `absl::string_view` data reference for the provided data edge. +// Requires 'IsDataEdge(edge) == true`. +inline absl::string_view EdgeData(const CordRep* edge) { + assert(IsDataEdge(edge)); + + size_t offset = 0; + const size_t length = edge->length; + if (edge->IsSubstring()) { + offset = edge->substring()->start; + edge = edge->substring()->child; + } + return edge->tag >= FLAT + ? absl::string_view{edge->flat()->Data() + offset, length} + : absl::string_view{edge->external()->base + offset, length}; +} + +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_INTERNAL_CORD_DATA_EDGE_H_ diff --git a/absl/strings/internal/cord_data_edge_test.cc b/absl/strings/internal/cord_data_edge_test.cc new file mode 100644 index 00000000..8fce3bc6 --- /dev/null +++ b/absl/strings/internal/cord_data_edge_test.cc @@ -0,0 +1,130 @@ +// Copyright 2022 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/cord_data_edge.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/strings/internal/cord_internal.h" +#include "absl/strings/internal/cord_rep_test_util.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { +namespace { + +using ::absl::cordrep_testing::MakeExternal; +using ::absl::cordrep_testing::MakeFlat; +using ::absl::cordrep_testing::MakeSubstring; + +TEST(CordDataEdgeTest, IsDataEdgeOnFlat) { + CordRep* rep = MakeFlat("Lorem ipsum dolor sit amet, consectetur ..."); + EXPECT_TRUE(IsDataEdge(rep)); + CordRep::Unref(rep); +} + +TEST(CordDataEdgeTest, IsDataEdgeOnExternal) { + CordRep* rep = MakeExternal("Lorem ipsum dolor sit amet, consectetur ..."); + EXPECT_TRUE(IsDataEdge(rep)); + CordRep::Unref(rep); +} + +TEST(CordDataEdgeTest, IsDataEdgeOnSubstringOfFlat) { + CordRep* rep = MakeFlat("Lorem ipsum dolor sit amet, consectetur ..."); + CordRep* substr = MakeSubstring(1, 20, rep); + EXPECT_TRUE(IsDataEdge(substr)); + CordRep::Unref(substr); +} + +TEST(CordDataEdgeTest, IsDataEdgeOnSubstringOfExternal) { + CordRep* rep = MakeExternal("Lorem ipsum dolor sit amet, consectetur ..."); + CordRep* substr = MakeSubstring(1, 20, rep); + EXPECT_TRUE(IsDataEdge(substr)); + CordRep::Unref(substr); +} + +TEST(CordDataEdgeTest, IsDataEdgeOnBtree) { + CordRep* rep = MakeFlat("Lorem ipsum dolor sit amet, consectetur ..."); + CordRepBtree* tree = CordRepBtree::New(rep); + EXPECT_FALSE(IsDataEdge(tree)); + CordRep::Unref(tree); +} + +TEST(CordDataEdgeTest, IsDataEdgeOnBadSubstr) { + CordRep* rep = MakeFlat("Lorem ipsum dolor sit amet, consectetur ..."); + CordRep* substr = MakeSubstring(1, 18, MakeSubstring(1, 20, rep)); + EXPECT_FALSE(IsDataEdge(substr)); + CordRep::Unref(substr); +} + +TEST(CordDataEdgeTest, EdgeDataOnFlat) { + absl::string_view value = "Lorem ipsum dolor sit amet, consectetur ..."; + CordRep* rep = MakeFlat(value); + EXPECT_EQ(EdgeData(rep), value); + CordRep::Unref(rep); +} + +TEST(CordDataEdgeTest, EdgeDataOnExternal) { + absl::string_view value = "Lorem ipsum dolor sit amet, consectetur ..."; + CordRep* rep = MakeExternal(value); + EXPECT_EQ(EdgeData(rep), value); + CordRep::Unref(rep); +} + +TEST(CordDataEdgeTest, EdgeDataOnSubstringOfFlat) { + absl::string_view value = "Lorem ipsum dolor sit amet, consectetur ..."; + CordRep* rep = MakeFlat(value); + CordRep* substr = MakeSubstring(1, 20, rep); + EXPECT_EQ(EdgeData(substr), value.substr(1, 20)); + CordRep::Unref(substr); +} + +TEST(CordDataEdgeTest, EdgeDataOnSubstringOfExternal) { + absl::string_view value = "Lorem ipsum dolor sit amet, consectetur ..."; + CordRep* rep = MakeExternal(value); + CordRep* substr = MakeSubstring(1, 20, rep); + EXPECT_EQ(EdgeData(substr), value.substr(1, 20)); + CordRep::Unref(substr); +} + +#if defined(GTEST_HAS_DEATH_TEST) && !defined(NDEBUG) + +TEST(CordDataEdgeTest, IsDataEdgeOnNullPtr) { + EXPECT_DEATH(IsDataEdge(nullptr), ".*"); +} + +TEST(CordDataEdgeTest, EdgeDataOnNullPtr) { + EXPECT_DEATH(EdgeData(nullptr), ".*"); +} + +TEST(CordDataEdgeTest, EdgeDataOnBtree) { + CordRep* rep = MakeFlat("Lorem ipsum dolor sit amet, consectetur ..."); + CordRepBtree* tree = CordRepBtree::New(rep); + EXPECT_DEATH(EdgeData(tree), ".*"); + CordRep::Unref(tree); +} + +TEST(CordDataEdgeTest, EdgeDataOnBadSubstr) { + CordRep* rep = MakeFlat("Lorem ipsum dolor sit amet, consectetur ..."); + CordRep* substr = MakeSubstring(1, 18, MakeSubstring(1, 20, rep)); + EXPECT_DEATH(EdgeData(substr), ".*"); + CordRep::Unref(substr); +} + +#endif // GTEST_HAS_DEATH_TEST && !NDEBUG + +} // namespace +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/absl/strings/internal/cord_internal.cc b/absl/strings/internal/cord_internal.cc index 905ffd0c..b6b06cfa 100644 --- a/absl/strings/internal/cord_internal.cc +++ b/absl/strings/internal/cord_internal.cc @@ -17,9 +17,13 @@ #include <cassert> #include <memory> +#include "absl/base/internal/raw_logging.h" #include "absl/container/inlined_vector.h" +#include "absl/strings/internal/cord_rep_btree.h" +#include "absl/strings/internal/cord_rep_crc.h" #include "absl/strings/internal/cord_rep_flat.h" #include "absl/strings/internal/cord_rep_ring.h" +#include "absl/strings/str_cat.h" namespace absl { ABSL_NAMESPACE_BEGIN @@ -29,51 +33,41 @@ ABSL_CONST_INIT std::atomic<bool> cord_ring_buffer_enabled( kCordEnableRingBufferDefault); ABSL_CONST_INIT std::atomic<bool> shallow_subcords_enabled( kCordShallowSubcordsDefault); +ABSL_CONST_INIT std::atomic<bool> cord_btree_exhaustive_validation(false); + +void LogFatalNodeType(CordRep* rep) { + ABSL_INTERNAL_LOG(FATAL, absl::StrCat("Unexpected node type: ", + static_cast<int>(rep->tag))); +} void CordRep::Destroy(CordRep* rep) { assert(rep != nullptr); - absl::InlinedVector<CordRep*, Constants::kInlinedVectorSize> pending; while (true) { assert(!rep->refcount.IsImmortal()); - if (rep->tag == CONCAT) { - CordRepConcat* rep_concat = rep->concat(); - CordRep* right = rep_concat->right; - if (!right->refcount.Decrement()) { - pending.push_back(right); - } - CordRep* left = rep_concat->left; - delete rep_concat; - rep = nullptr; - if (!left->refcount.Decrement()) { - rep = left; - continue; - } + if (rep->tag == BTREE) { + CordRepBtree::Destroy(rep->btree()); + return; } else if (rep->tag == RING) { CordRepRing::Destroy(rep->ring()); - rep = nullptr; + return; } else if (rep->tag == EXTERNAL) { CordRepExternal::Delete(rep); - rep = nullptr; + return; } else if (rep->tag == SUBSTRING) { CordRepSubstring* rep_substring = rep->substring(); - CordRep* child = rep_substring->child; + rep = rep_substring->child; delete rep_substring; - rep = nullptr; - if (!child->refcount.Decrement()) { - rep = child; - continue; + if (rep->refcount.Decrement()) { + return; } + } else if (rep->tag == CRC) { + CordRepCrc::Destroy(rep->crc()); + return; } else { + assert(rep->IsFlat()); CordRepFlat::Delete(rep); - rep = nullptr; - } - - if (!pending.empty()) { - rep = pending.back(); - pending.pop_back(); - } else { - break; + return; } } } diff --git a/absl/strings/internal/cord_internal.h b/absl/strings/internal/cord_internal.h index a1ba67fe..b50fb79a 100644 --- a/absl/strings/internal/cord_internal.h +++ b/absl/strings/internal/cord_internal.h @@ -21,6 +21,7 @@ #include <cstdint> #include <type_traits> +#include "absl/base/attributes.h" #include "absl/base/config.h" #include "absl/base/internal/endian.h" #include "absl/base/internal/invoke.h" @@ -33,6 +34,19 @@ namespace absl { ABSL_NAMESPACE_BEGIN namespace cord_internal { +// The overhead of a vtable is too much for Cord, so we roll our own subclasses +// using only a single byte to differentiate classes from each other - the "tag" +// byte. Define the subclasses first so we can provide downcasting helper +// functions in the base class. +struct CordRep; +struct CordRepConcat; +struct CordRepExternal; +struct CordRepFlat; +struct CordRepSubstring; +struct CordRepCrc; +class CordRepRing; +class CordRepBtree; + class CordzInfo; // Default feature enable states for cord ring buffers @@ -44,6 +58,12 @@ enum CordFeatureDefaults { extern std::atomic<bool> cord_ring_buffer_enabled; extern std::atomic<bool> shallow_subcords_enabled; +// `cord_btree_exhaustive_validation` can be set to force exhaustive validation +// in debug assertions, and code that calls `IsValid()` explicitly. By default, +// assertions should be relatively cheap and AssertValid() can easily lead to +// O(n^2) complexity as recursive / full tree validation is O(n). +extern std::atomic<bool> cord_btree_exhaustive_validation; + inline void enable_cord_ring_buffer(bool enable) { cord_ring_buffer_enabled.store(enable, std::memory_order_relaxed); } @@ -68,12 +88,16 @@ enum Constants { kMaxBytesToCopy = 511 }; -// Wraps std::atomic for reference counting. -class Refcount { +// Emits a fatal error "Unexpected node type: xyz" and aborts the program. +ABSL_ATTRIBUTE_NORETURN void LogFatalNodeType(CordRep* rep); + +// Compact class for tracking the reference count and state flags for CordRep +// instances. Data is stored in an atomic int32_t for compactness and speed. +class RefcountAndFlags { public: - constexpr Refcount() : count_{kRefIncrement} {} + constexpr RefcountAndFlags() : count_{kRefIncrement} {} struct Immortal {}; - explicit constexpr Refcount(Immortal) : count_(kImmortalTag) {} + explicit constexpr RefcountAndFlags(Immortal) : count_(kImmortalFlag) {} // Increments the reference count. Imposes no memory ordering. inline void Increment() { @@ -86,26 +110,27 @@ class Refcount { // Returns false if there are no references outstanding; true otherwise. // Inserts barriers to ensure that state written before this method returns // false will be visible to a thread that just observed this method returning - // false. + // false. Always returns false when the immortal bit is set. inline bool Decrement() { - int32_t refcount = count_.load(std::memory_order_acquire); - assert(refcount > 0 || refcount & kImmortalTag); + int32_t refcount = count_.load(std::memory_order_acquire) & kRefcountMask; + assert(refcount > 0 || refcount & kImmortalFlag); return refcount != kRefIncrement && - count_.fetch_sub(kRefIncrement, std::memory_order_acq_rel) != - kRefIncrement; + (count_.fetch_sub(kRefIncrement, std::memory_order_acq_rel) & + kRefcountMask) != kRefIncrement; } // Same as Decrement but expect that refcount is greater than 1. inline bool DecrementExpectHighRefcount() { int32_t refcount = - count_.fetch_sub(kRefIncrement, std::memory_order_acq_rel); - assert(refcount > 0 || refcount & kImmortalTag); + count_.fetch_sub(kRefIncrement, std::memory_order_acq_rel) & + kRefcountMask; + assert(refcount > 0 || refcount & kImmortalFlag); return refcount != kRefIncrement; } // Returns the current reference count using acquire semantics. inline int32_t Get() const { - return count_.load(std::memory_order_acquire) >> kImmortalShift; + return count_.load(std::memory_order_acquire) >> kNumFlags; } // Returns whether the atomic integer is 1. @@ -115,83 +140,127 @@ class Refcount { // This call performs the test for a reference count of one, and // performs the memory barrier needed for the owning thread // to act on the object, knowing that it has exclusive access to the - // object. + // object. Always returns false when the immortal bit is set. inline bool IsOne() { - return count_.load(std::memory_order_acquire) == kRefIncrement; + return (count_.load(std::memory_order_acquire) & kRefcountMask) == + kRefIncrement; } bool IsImmortal() const { - return (count_.load(std::memory_order_relaxed) & kImmortalTag) != 0; + return (count_.load(std::memory_order_relaxed) & kImmortalFlag) != 0; } private: - // We reserve the bottom bit to tag a reference count as immortal. - // By making it `1` we ensure that we never reach `0` when adding/subtracting - // `2`, thus it never looks as if it should be destroyed. - // These are used for the StringConstant constructor where we do not increase - // the refcount at construction time (due to constinit requirements) but we - // will still decrease it at destruction time to avoid branching on Unref. - enum { - kImmortalShift = 1, - kRefIncrement = 1 << kImmortalShift, - kImmortalTag = kRefIncrement - 1 + // We reserve the bottom bits for flags. + // kImmortalBit indicates that this entity should never be collected; it is + // used for the StringConstant constructor to avoid collecting immutable + // constant cords. + // kReservedFlag is reserved for future use. + enum Flags { + kNumFlags = 2, + + kImmortalFlag = 0x1, + kReservedFlag = 0x2, + kRefIncrement = (1 << kNumFlags), + + // Bitmask to use when checking refcount by equality. This masks out + // all flags except kImmortalFlag, which is part of the refcount for + // purposes of equality. (A refcount of 0 or 1 does not count as 0 or 1 + // if the immortal bit is set.) + kRefcountMask = ~kReservedFlag, }; std::atomic<int32_t> count_; }; -// The overhead of a vtable is too much for Cord, so we roll our own subclasses -// using only a single byte to differentiate classes from each other - the "tag" -// byte. Define the subclasses first so we can provide downcasting helper -// functions in the base class. - -struct CordRepConcat; -struct CordRepExternal; -struct CordRepFlat; -struct CordRepSubstring; -class CordRepRing; - // Various representations that we allow enum CordRepKind { - CONCAT = 0, - EXTERNAL = 1, - SUBSTRING = 2, - RING = 3, + UNUSED_0 = 0, + SUBSTRING = 1, + CRC = 2, + BTREE = 3, + RING = 4, + EXTERNAL = 5, // We have different tags for different sized flat arrays, - // starting with FLAT, and limited to MAX_FLAT_TAG. The 224 value is based on - // the current 'size to tag' encoding of 8 / 32 bytes. If a new tag is needed - // in the future, then 'FLAT' and 'MAX_FLAT_TAG' should be adjusted as well - // as the Tag <---> Size logic so that FLAT stil represents the minimum flat - // allocation size. (32 bytes as of now). - FLAT = 4, - MAX_FLAT_TAG = 224 + // starting with FLAT, and limited to MAX_FLAT_TAG. The below values map to an + // allocated range of 32 bytes to 256 KB. The current granularity is: + // - 8 byte granularity for flat sizes in [32 - 512] + // - 64 byte granularity for flat sizes in (512 - 8KiB] + // - 4KiB byte granularity for flat sizes in (8KiB, 256 KiB] + // If a new tag is needed in the future, then 'FLAT' and 'MAX_FLAT_TAG' should + // be adjusted as well as the Tag <---> Size mapping logic so that FLAT still + // represents the minimum flat allocation size. (32 bytes as of now). + FLAT = 6, + MAX_FLAT_TAG = 248 }; +// There are various locations where we want to check if some rep is a 'plain' +// data edge, i.e. an external or flat rep. By having FLAT == EXTERNAL + 1, we +// can perform this check in a single branch as 'tag >= EXTERNAL' +// Likewise, we have some locations where we check for 'ring or external/flat', +// so likewise align RING to EXTERNAL. +// Note that we can leave this optimization to the compiler. The compiler will +// DTRT when it sees a condition like `tag == EXTERNAL || tag >= FLAT`. +static_assert(RING == BTREE + 1, "BTREE and RING not consecutive"); +static_assert(EXTERNAL == RING + 1, "BTREE and EXTERNAL not consecutive"); +static_assert(FLAT == EXTERNAL + 1, "EXTERNAL and FLAT not consecutive"); + struct CordRep { + // Result from an `extract edge` operation. Contains the (possibly changed) + // tree node as well as the extracted edge, or {tree, nullptr} if no edge + // could be extracted. + // On success, the returned `tree` value is null if `extracted` was the only + // data edge inside the tree, a data edge if there were only two data edges in + // the tree, or the (possibly new / smaller) remaining tree with the extracted + // data edge removed. + struct ExtractResult { + CordRep* tree; + CordRep* extracted; + }; + CordRep() = default; - constexpr CordRep(Refcount::Immortal immortal, size_t l) + constexpr CordRep(RefcountAndFlags::Immortal immortal, size_t l) : length(l), refcount(immortal), tag(EXTERNAL), storage{} {} // The following three fields have to be less than 32 bytes since // that is the smallest supported flat node size. size_t length; - Refcount refcount; + RefcountAndFlags refcount; // If tag < FLAT, it represents CordRepKind and indicates the type of node. // Otherwise, the node type is CordRepFlat and the tag is the encoded size. uint8_t tag; - char storage[1]; // Starting point for flat array: MUST BE LAST FIELD + + // `storage` provides two main purposes: + // - the starting point for FlatCordRep.Data() [flexible-array-member] + // - 3 bytes of additional storage for use by derived classes. + // The latter is used by CordrepConcat and CordRepBtree. CordRepConcat stores + // a 'depth' value in storage[0], and the (future) CordRepBtree class stores + // `height`, `begin` and `end` in the 3 entries. Otherwise we would need to + // allocate room for these in the derived class, as not all compilers reuse + // padding space from the base class (clang and gcc do, MSVC does not, etc) + uint8_t storage[3]; + + // Returns true if this instance's tag matches the requested type. + constexpr bool IsRing() const { return tag == RING; } + constexpr bool IsSubstring() const { return tag == SUBSTRING; } + constexpr bool IsCrc() const { return tag == CRC; } + constexpr bool IsExternal() const { return tag == EXTERNAL; } + constexpr bool IsFlat() const { return tag >= FLAT; } + constexpr bool IsBtree() const { return tag == BTREE; } inline CordRepRing* ring(); inline const CordRepRing* ring() const; - inline CordRepConcat* concat(); - inline const CordRepConcat* concat() const; inline CordRepSubstring* substring(); inline const CordRepSubstring* substring() const; + inline CordRepCrc* crc(); + inline const CordRepCrc* crc() const; inline CordRepExternal* external(); inline const CordRepExternal* external() const; inline CordRepFlat* flat(); inline const CordRepFlat* flat() const; + inline CordRepBtree* btree(); + inline const CordRepBtree* btree() const; // -------------------------------------------------------------------- // Memory management @@ -208,17 +277,23 @@ struct CordRep { static inline void Unref(CordRep* rep); }; -struct CordRepConcat : public CordRep { - CordRep* left; - CordRep* right; - - uint8_t depth() const { return static_cast<uint8_t>(storage[0]); } - void set_depth(uint8_t depth) { storage[0] = static_cast<char>(depth); } -}; - struct CordRepSubstring : public CordRep { size_t start; // Starting offset of substring in child CordRep* child; + + // Creates a substring on `child`, adopting a reference on `child`. + // Requires `child` to be either a flat or external node, and `pos` and `n` to + // form a non-empty partial sub range of `'child`, i.e.: + // `n > 0 && n < length && n + pos <= length` + static inline CordRepSubstring* Create(CordRep* child, size_t pos, size_t n); + + // Creates a substring of `rep`. Does not adopt a reference on `rep`. + // Requires `IsDataEdge(rep) && n > 0 && pos + n <= rep->length`. + // If `n == rep->length` then this method returns `CordRep::Ref(rep)` + // If `rep` is a substring of a flat or external node, then this method will + // return a new substring of that flat or external node with `pos` adjusted + // with the original `start` position. + static inline CordRep* Substring(CordRep* rep, size_t pos, size_t n); }; // Type for function pointer that will invoke the releaser function and also @@ -231,7 +306,7 @@ using ExternalReleaserInvoker = void (*)(CordRepExternal*); struct CordRepExternal : public CordRep { CordRepExternal() = default; explicit constexpr CordRepExternal(absl::string_view str) - : CordRep(Refcount::Immortal{}, str.size()), + : CordRep(RefcountAndFlags::Immortal{}, str.size()), base(str.data()), releaser_invoker(nullptr) {} @@ -240,7 +315,7 @@ struct CordRepExternal : public CordRep { ExternalReleaserInvoker releaser_invoker; // Deletes (releases) the external rep. - // Requires rep != nullptr and rep->tag == EXTERNAL + // Requires rep != nullptr and rep->IsExternal() static void Delete(CordRep* rep); }; @@ -282,8 +357,49 @@ struct CordRepExternalImpl } }; +inline CordRepSubstring* CordRepSubstring::Create(CordRep* child, size_t pos, + size_t n) { + assert(child != nullptr); + assert(n > 0); + assert(n < child->length); + assert(pos < child->length); + assert(n <= child->length - pos); + + // TODO(b/217376272): Harden internal logic. + // Move to strategical places inside the Cord logic and make this an assert. + if (ABSL_PREDICT_FALSE(!(child->IsExternal() || child->IsFlat()))) { + LogFatalNodeType(child); + } + + CordRepSubstring* rep = new CordRepSubstring(); + rep->length = n; + rep->tag = SUBSTRING; + rep->start = pos; + rep->child = child; + return rep; +} + +inline CordRep* CordRepSubstring::Substring(CordRep* rep, size_t pos, + size_t n) { + assert(rep != nullptr); + assert(n != 0); + assert(pos < rep->length); + assert(n <= rep->length - pos); + if (n == rep->length) return CordRep::Ref(rep); + if (rep->IsSubstring()) { + pos += rep->substring()->start; + rep = rep->substring()->child; + } + CordRepSubstring* substr = new CordRepSubstring(); + substr->length = n; + substr->tag = SUBSTRING; + substr->start = pos; + substr->child = CordRep::Ref(rep); + return substr; +} + inline void CordRepExternal::Delete(CordRep* rep) { - assert(rep != nullptr && rep->tag == EXTERNAL); + assert(rep != nullptr && rep->IsExternal()); auto* rep_external = static_cast<CordRepExternal*>(rep); assert(rep_external->releaser_invoker != nullptr); rep_external->releaser_invoker(rep_external); @@ -295,7 +411,8 @@ struct ConstInitExternalStorage { }; template <typename Str> -CordRepExternal ConstInitExternalStorage<Str>::value(Str::value); +ABSL_CONST_INIT CordRepExternal + ConstInitExternalStorage<Str>::value(Str::value); enum { kMaxInline = 15, @@ -329,18 +446,17 @@ static constexpr cordz_info_t BigEndianByte(unsigned char value) { class InlineData { public: + // DefaultInitType forces the use of the default initialization constructor. + enum DefaultInitType { kDefaultInit }; + // kNullCordzInfo holds the big endian representation of intptr_t(1) // This is the 'null' / initial value of 'cordz_info'. The null value // is specifically big endian 1 as with 64-bit pointers, the last // byte of cordz_info overlaps with the last byte holding the tag. static constexpr cordz_info_t kNullCordzInfo = BigEndianByte(1); - // kFakeCordzInfo holds a 'fake', non-null cordz-info value we use to - // emulate the previous 'kProfiled' tag logic in 'set_profiled' until - // cord code is changed to store cordz_info values in InlineData. - static constexpr cordz_info_t kFakeCordzInfo = BigEndianByte(9); - constexpr InlineData() : as_chars_{0} {} + explicit InlineData(DefaultInitType) {} explicit constexpr InlineData(CordRep* rep) : as_tree_(rep) {} explicit constexpr InlineData(absl::string_view chars) : as_chars_{ @@ -367,13 +483,23 @@ class InlineData { return as_tree_.cordz_info != kNullCordzInfo; } + // Returns true if either of the provided instances hold a cordz_info value. + // This method is more efficient than the equivalent `data1.is_profiled() || + // data2.is_profiled()`. Requires both arguments to hold a tree. + static bool is_either_profiled(const InlineData& data1, + const InlineData& data2) { + assert(data1.is_tree() && data2.is_tree()); + return (data1.as_tree_.cordz_info | data2.as_tree_.cordz_info) != + kNullCordzInfo; + } + // Returns the cordz_info sampling instance for this instance, or nullptr // if the current instance is not sampled and does not have CordzInfo data. // Requires the current instance to hold a tree value. CordzInfo* cordz_info() const { assert(is_tree()); - intptr_t info = - static_cast<intptr_t>(absl::big_endian::ToHost64(as_tree_.cordz_info)); + intptr_t info = static_cast<intptr_t>( + absl::big_endian::ToHost64(static_cast<uint64_t>(as_tree_.cordz_info))); assert(info & 1); return reinterpret_cast<CordzInfo*>(info - 1); } @@ -383,8 +509,9 @@ class InlineData { // Requires the current instance to hold a tree value. void set_cordz_info(CordzInfo* cordz_info) { assert(is_tree()); - intptr_t info = reinterpret_cast<intptr_t>(cordz_info) | 1; - as_tree_.cordz_info = absl::big_endian::FromHost64(info); + uintptr_t info = reinterpret_cast<uintptr_t>(cordz_info) | 1; + as_tree_.cordz_info = + static_cast<cordz_info_t>(absl::big_endian::FromHost64(info)); } // Resets the current cordz_info to null / empty. @@ -454,13 +581,6 @@ class InlineData { tag() = static_cast<char>(size << 1); } - // Sets or unsets the 'is_profiled' state of this instance. - // Requires the current instance to hold a tree value. - void set_profiled(bool profiled) { - assert(is_tree()); - as_tree_.cordz_info = profiled ? kFakeCordzInfo : kNullCordzInfo; - } - private: // See cordz_info_t for forced alignment and size of `cordz_info` details. struct AsTree { @@ -483,7 +603,7 @@ class InlineData { // store the size in the last char of `as_chars_` shifted left + 1. // Else we store it in a tree and store a pointer to that tree in // `as_tree_.rep` and store a tag in `tagged_size`. - union { + union { char as_chars_[kMaxInline + 1]; AsTree as_tree_; }; @@ -491,38 +611,30 @@ class InlineData { static_assert(sizeof(InlineData) == kMaxInline + 1, ""); -inline CordRepConcat* CordRep::concat() { - assert(tag == CONCAT); - return static_cast<CordRepConcat*>(this); -} - -inline const CordRepConcat* CordRep::concat() const { - assert(tag == CONCAT); - return static_cast<const CordRepConcat*>(this); -} - inline CordRepSubstring* CordRep::substring() { - assert(tag == SUBSTRING); + assert(IsSubstring()); return static_cast<CordRepSubstring*>(this); } inline const CordRepSubstring* CordRep::substring() const { - assert(tag == SUBSTRING); + assert(IsSubstring()); return static_cast<const CordRepSubstring*>(this); } inline CordRepExternal* CordRep::external() { - assert(tag == EXTERNAL); + assert(IsExternal()); return static_cast<CordRepExternal*>(this); } inline const CordRepExternal* CordRep::external() const { - assert(tag == EXTERNAL); + assert(IsExternal()); return static_cast<const CordRepExternal*>(this); } inline CordRep* CordRep::Ref(CordRep* rep) { - assert(rep != nullptr); + // ABSL_ASSUME is a workaround for + // https://gcc.gnu.org/bugzilla/show_bug.cgi?id=105585 + ABSL_ASSUME(rep != nullptr); rep->refcount.Increment(); return rep; } diff --git a/absl/strings/internal/cord_rep_btree.cc b/absl/strings/internal/cord_rep_btree.cc new file mode 100644 index 00000000..cacbf3da --- /dev/null +++ b/absl/strings/internal/cord_rep_btree.cc @@ -0,0 +1,1228 @@ +// Copyright 2021 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/cord_rep_btree.h" + +#include <cassert> +#include <cstdint> +#include <iostream> +#include <string> + +#include "absl/base/attributes.h" +#include "absl/base/config.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/strings/internal/cord_data_edge.h" +#include "absl/strings/internal/cord_internal.h" +#include "absl/strings/internal/cord_rep_consume.h" +#include "absl/strings/internal/cord_rep_flat.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { + +#ifdef ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL +constexpr size_t CordRepBtree::kMaxCapacity; +#endif + +namespace { + +using NodeStack = CordRepBtree * [CordRepBtree::kMaxDepth]; +using EdgeType = CordRepBtree::EdgeType; +using OpResult = CordRepBtree::OpResult; +using CopyResult = CordRepBtree::CopyResult; + +constexpr auto kFront = CordRepBtree::kFront; +constexpr auto kBack = CordRepBtree::kBack; + +inline bool exhaustive_validation() { + return cord_btree_exhaustive_validation.load(std::memory_order_relaxed); +} + +// Implementation of the various 'Dump' functions. +// Prints the entire tree structure or 'rep'. External callers should +// not specify 'depth' and leave it to its default (0) value. +// Rep may be a CordRepBtree tree, or a SUBSTRING / EXTERNAL / FLAT node. +void DumpAll(const CordRep* rep, bool include_contents, std::ostream& stream, + int depth = 0) { + // Allow for full height trees + substring -> flat / external nodes. + assert(depth <= CordRepBtree::kMaxDepth + 2); + std::string sharing = const_cast<CordRep*>(rep)->refcount.IsOne() + ? std::string("Private") + : absl::StrCat("Shared(", rep->refcount.Get(), ")"); + std::string sptr = absl::StrCat("0x", absl::Hex(rep)); + + // Dumps the data contents of `rep` if `include_contents` is true. + // Always emits a new line character. + auto maybe_dump_data = [&stream, include_contents](const CordRep* r) { + if (include_contents) { + // Allow for up to 60 wide display of content data, which with some + // indentation and prefix / labels keeps us within roughly 80-100 wide. + constexpr size_t kMaxDataLength = 60; + stream << ", data = \"" + << EdgeData(r).substr(0, kMaxDataLength) + << (r->length > kMaxDataLength ? "\"..." : "\""); + } + stream << '\n'; + }; + + // For each level, we print the 'shared/private' state and the rep pointer, + // indented by two spaces per recursive depth. + stream << std::string(depth * 2, ' ') << sharing << " (" << sptr << ") "; + + if (rep->IsBtree()) { + const CordRepBtree* node = rep->btree(); + std::string label = + node->height() ? absl::StrCat("Node(", node->height(), ")") : "Leaf"; + stream << label << ", len = " << node->length + << ", begin = " << node->begin() << ", end = " << node->end() + << "\n"; + for (CordRep* edge : node->Edges()) { + DumpAll(edge, include_contents, stream, depth + 1); + } + } else if (rep->tag == SUBSTRING) { + const CordRepSubstring* substring = rep->substring(); + stream << "Substring, len = " << rep->length + << ", start = " << substring->start; + maybe_dump_data(rep); + DumpAll(substring->child, include_contents, stream, depth + 1); + } else if (rep->tag >= FLAT) { + stream << "Flat, len = " << rep->length + << ", cap = " << rep->flat()->Capacity(); + maybe_dump_data(rep); + } else if (rep->tag == EXTERNAL) { + stream << "Extn, len = " << rep->length; + maybe_dump_data(rep); + } +} + +// TODO(b/192061034): add 'bytes to copy' logic to avoid large slop on substring +// small data out of large reps, and general efficiency of 'always copy small +// data'. Consider making this a cord rep internal library function. +CordRepSubstring* CreateSubstring(CordRep* rep, size_t offset, size_t n) { + assert(n != 0); + assert(offset + n <= rep->length); + assert(offset != 0 || n != rep->length); + + if (rep->tag == SUBSTRING) { + CordRepSubstring* substring = rep->substring(); + offset += substring->start; + rep = CordRep::Ref(substring->child); + CordRep::Unref(substring); + } + assert(rep->IsExternal() || rep->IsFlat()); + CordRepSubstring* substring = new CordRepSubstring(); + substring->length = n; + substring->tag = SUBSTRING; + substring->start = offset; + substring->child = rep; + return substring; +} + +// TODO(b/192061034): consider making this a cord rep library function. +inline CordRep* MakeSubstring(CordRep* rep, size_t offset, size_t n) { + if (n == rep->length) return rep; + if (n == 0) return CordRep::Unref(rep), nullptr; + return CreateSubstring(rep, offset, n); +} + +// TODO(b/192061034): consider making this a cord rep library function. +inline CordRep* MakeSubstring(CordRep* rep, size_t offset) { + if (offset == 0) return rep; + return CreateSubstring(rep, offset, rep->length - offset); +} + +// Resizes `edge` to the provided `length`. Adopts a reference on `edge`. +// This method directly returns `edge` if `length` equals `edge->length`. +// If `is_mutable` is set to true, this function may return `edge` with +// `edge->length` set to the new length depending on the type and size of +// `edge`. Otherwise, this function returns a new CordRepSubstring value. +// Requires `length > 0 && length <= edge->length`. +CordRep* ResizeEdge(CordRep* edge, size_t length, bool is_mutable) { + assert(length > 0); + assert(length <= edge->length); + assert(IsDataEdge(edge)); + if (length >= edge->length) return edge; + + if (is_mutable && (edge->tag >= FLAT || edge->tag == SUBSTRING)) { + edge->length = length; + return edge; + } + + return CreateSubstring(edge, 0, length); +} + +template <EdgeType edge_type> +inline absl::string_view Consume(absl::string_view s, size_t n) { + return edge_type == kBack ? s.substr(n) : s.substr(0, s.size() - n); +} + +template <EdgeType edge_type> +inline absl::string_view Consume(char* dst, absl::string_view s, size_t n) { + if (edge_type == kBack) { + memcpy(dst, s.data(), n); + return s.substr(n); + } else { + const size_t offset = s.size() - n; + memcpy(dst, s.data() + offset, n); + return s.substr(0, offset); + } +} + +// Known issue / optimization weirdness: the store associated with the +// decrement introduces traffic between cpus (even if the result of that +// traffic does nothing), making this faster than a single call to +// refcount.Decrement() checking the zero refcount condition. +template <typename R, typename Fn> +inline void FastUnref(R* r, Fn&& fn) { + if (r->refcount.IsOne()) { + fn(r); + } else if (!r->refcount.DecrementExpectHighRefcount()) { + fn(r); + } +} + + +void DeleteSubstring(CordRepSubstring* substring) { + CordRep* rep = substring->child; + if (!rep->refcount.Decrement()) { + if (rep->tag >= FLAT) { + CordRepFlat::Delete(rep->flat()); + } else { + assert(rep->tag == EXTERNAL); + CordRepExternal::Delete(rep->external()); + } + } + delete substring; +} + +// Deletes a leaf node data edge. Requires `IsDataEdge(rep)`. +void DeleteLeafEdge(CordRep* rep) { + assert(IsDataEdge(rep)); + if (rep->tag >= FLAT) { + CordRepFlat::Delete(rep->flat()); + } else if (rep->tag == EXTERNAL) { + CordRepExternal::Delete(rep->external()); + } else { + DeleteSubstring(rep->substring()); + } +} + +// StackOperations contains the logic to build a left-most or right-most stack +// (leg) down to the leaf level of a btree, and 'unwind' / 'Finalize' methods to +// propagate node changes up the stack. +template <EdgeType edge_type> +struct StackOperations { + // Returns true if the node at 'depth' is not shared, i.e. has a refcount + // of one and all of its parent nodes have a refcount of one. + inline bool owned(int depth) const { return depth < share_depth; } + + // Returns the node at 'depth'. + inline CordRepBtree* node(int depth) const { return stack[depth]; } + + // Builds a `depth` levels deep stack starting at `tree` recording which nodes + // are private in the form of the 'share depth' where nodes are shared. + inline CordRepBtree* BuildStack(CordRepBtree* tree, int depth) { + assert(depth <= tree->height()); + int current_depth = 0; + while (current_depth < depth && tree->refcount.IsOne()) { + stack[current_depth++] = tree; + tree = tree->Edge(edge_type)->btree(); + } + share_depth = current_depth + (tree->refcount.IsOne() ? 1 : 0); + while (current_depth < depth) { + stack[current_depth++] = tree; + tree = tree->Edge(edge_type)->btree(); + } + return tree; + } + + // Builds a stack with the invariant that all nodes are private owned / not + // shared. This is used in iterative updates where a previous propagation + // guaranteed all nodes are owned / private. + inline void BuildOwnedStack(CordRepBtree* tree, int height) { + assert(height <= CordRepBtree::kMaxHeight); + int depth = 0; + while (depth < height) { + assert(tree->refcount.IsOne()); + stack[depth++] = tree; + tree = tree->Edge(edge_type)->btree(); + } + assert(tree->refcount.IsOne()); + share_depth = depth + 1; + } + + // Processes the final 'top level' result action for the tree. + // See the 'Action' enum for the various action implications. + static inline CordRepBtree* Finalize(CordRepBtree* tree, OpResult result) { + switch (result.action) { + case CordRepBtree::kPopped: + tree = edge_type == kBack ? CordRepBtree::New(tree, result.tree) + : CordRepBtree::New(result.tree, tree); + if (ABSL_PREDICT_FALSE(tree->height() > CordRepBtree::kMaxHeight)) { + tree = CordRepBtree::Rebuild(tree); + ABSL_RAW_CHECK(tree->height() <= CordRepBtree::kMaxHeight, + "Max height exceeded"); + } + return tree; + case CordRepBtree::kCopied: + CordRep::Unref(tree); + ABSL_FALLTHROUGH_INTENDED; + case CordRepBtree::kSelf: + return result.tree; + } + ABSL_INTERNAL_UNREACHABLE; + return result.tree; + } + + // Propagate the action result in 'result' up into all nodes of the stack + // starting at depth 'depth'. 'length' contains the extra length of data that + // was added at the lowest level, and is updated into all nodes of the stack. + // See the 'Action' enum for the various action implications. + // If 'propagate' is true, then any copied node values are updated into the + // stack, which is used for iterative processing on the same stack. + template <bool propagate = false> + inline CordRepBtree* Unwind(CordRepBtree* tree, int depth, size_t length, + OpResult result) { + // TODO(mvels): revisit the below code to check if 3 loops with 3 + // (incremental) conditions is faster than 1 loop with a switch. + // Benchmarking and perf recordings indicate the loop with switch is + // fastest, likely because of indirect jumps on the tight case values and + // dense branches. But it's worth considering 3 loops, as the `action` + // transitions are mono directional. E.g.: + // while (action == kPopped) { + // ... + // } + // while (action == kCopied) { + // ... + // } + // ... + // We also found that an "if () do {}" loop here seems faster, possibly + // because it allows the branch predictor more granular heuristics on + // 'single leaf' (`depth` == 0) and 'single depth' (`depth` == 1) cases + // which appear to be the most common use cases. + if (depth != 0) { + do { + CordRepBtree* node = stack[--depth]; + const bool owned = depth < share_depth; + switch (result.action) { + case CordRepBtree::kPopped: + assert(!propagate); + result = node->AddEdge<edge_type>(owned, result.tree, length); + break; + case CordRepBtree::kCopied: + result = node->SetEdge<edge_type>(owned, result.tree, length); + if (propagate) stack[depth] = result.tree; + break; + case CordRepBtree::kSelf: + node->length += length; + while (depth > 0) { + node = stack[--depth]; + node->length += length; + } + return node; + } + } while (depth > 0); + } + return Finalize(tree, result); + } + + // Invokes `Unwind` with `propagate=true` to update the stack node values. + inline CordRepBtree* Propagate(CordRepBtree* tree, int depth, size_t length, + OpResult result) { + return Unwind</*propagate=*/true>(tree, depth, length, result); + } + + // `share_depth` contains the depth at which the nodes in the stack become + // shared. I.e., if the top most level is shared (i.e.: `!refcount.IsOne()`), + // then `share_depth` is 0. If the 2nd node is shared (and implicitly all + // nodes below that) then `share_depth` is 1, etc. A `share_depth` greater + // than the depth of the stack indicates that none of the nodes in the stack + // are shared. + int share_depth; + + NodeStack stack; +}; + +} // namespace + +void CordRepBtree::Dump(const CordRep* rep, absl::string_view label, + bool include_contents, std::ostream& stream) { + stream << "===================================\n"; + if (!label.empty()) { + stream << label << '\n'; + stream << "-----------------------------------\n"; + } + if (rep) { + DumpAll(rep, include_contents, stream); + } else { + stream << "NULL\n"; + } +} + +void CordRepBtree::Dump(const CordRep* rep, absl::string_view label, + std::ostream& stream) { + Dump(rep, label, false, stream); +} + +void CordRepBtree::Dump(const CordRep* rep, std::ostream& stream) { + Dump(rep, absl::string_view(), false, stream); +} + +template <size_t size> +static void DestroyTree(CordRepBtree* tree) { + for (CordRep* node : tree->Edges()) { + if (node->refcount.Decrement()) continue; + for (CordRep* edge : node->btree()->Edges()) { + if (edge->refcount.Decrement()) continue; + if (size == 1) { + DeleteLeafEdge(edge); + } else { + CordRepBtree::Destroy(edge->btree()); + } + } + CordRepBtree::Delete(node->btree()); + } + CordRepBtree::Delete(tree); +} + +void CordRepBtree::Destroy(CordRepBtree* tree) { + switch (tree->height()) { + case 0: + for (CordRep* edge : tree->Edges()) { + if (!edge->refcount.Decrement()) { + DeleteLeafEdge(edge); + } + } + return CordRepBtree::Delete(tree); + case 1: + return DestroyTree<1>(tree); + default: + return DestroyTree<2>(tree); + } +} + +bool CordRepBtree::IsValid(const CordRepBtree* tree, bool shallow) { +#define NODE_CHECK_VALID(x) \ + if (!(x)) { \ + ABSL_RAW_LOG(ERROR, "CordRepBtree::CheckValid() FAILED: %s", #x); \ + return false; \ + } +#define NODE_CHECK_EQ(x, y) \ + if ((x) != (y)) { \ + ABSL_RAW_LOG(ERROR, \ + "CordRepBtree::CheckValid() FAILED: %s != %s (%s vs %s)", #x, \ + #y, absl::StrCat(x).c_str(), absl::StrCat(y).c_str()); \ + return false; \ + } + + NODE_CHECK_VALID(tree != nullptr); + NODE_CHECK_VALID(tree->IsBtree()); + NODE_CHECK_VALID(tree->height() <= kMaxHeight); + NODE_CHECK_VALID(tree->begin() < tree->capacity()); + NODE_CHECK_VALID(tree->end() <= tree->capacity()); + NODE_CHECK_VALID(tree->begin() <= tree->end()); + size_t child_length = 0; + for (CordRep* edge : tree->Edges()) { + NODE_CHECK_VALID(edge != nullptr); + if (tree->height() > 0) { + NODE_CHECK_VALID(edge->IsBtree()); + NODE_CHECK_VALID(edge->btree()->height() == tree->height() - 1); + } else { + NODE_CHECK_VALID(IsDataEdge(edge)); + } + child_length += edge->length; + } + NODE_CHECK_EQ(child_length, tree->length); + if ((!shallow || exhaustive_validation()) && tree->height() > 0) { + for (CordRep* edge : tree->Edges()) { + if (!IsValid(edge->btree(), shallow)) return false; + } + } + return true; + +#undef NODE_CHECK_VALID +#undef NODE_CHECK_EQ +} + +#ifndef NDEBUG + +CordRepBtree* CordRepBtree::AssertValid(CordRepBtree* tree, bool shallow) { + if (!IsValid(tree, shallow)) { + Dump(tree, "CordRepBtree validation failed:", false, std::cout); + ABSL_RAW_LOG(FATAL, "CordRepBtree::CheckValid() FAILED"); + } + return tree; +} + +const CordRepBtree* CordRepBtree::AssertValid(const CordRepBtree* tree, + bool shallow) { + if (!IsValid(tree, shallow)) { + Dump(tree, "CordRepBtree validation failed:", false, std::cout); + ABSL_RAW_LOG(FATAL, "CordRepBtree::CheckValid() FAILED"); + } + return tree; +} + +#endif // NDEBUG + +template <EdgeType edge_type> +inline OpResult CordRepBtree::AddEdge(bool owned, CordRep* edge, size_t delta) { + if (size() >= kMaxCapacity) return {New(edge), kPopped}; + OpResult result = ToOpResult(owned); + result.tree->Add<edge_type>(edge); + result.tree->length += delta; + return result; +} + +template <EdgeType edge_type> +OpResult CordRepBtree::SetEdge(bool owned, CordRep* edge, size_t delta) { + OpResult result; + const size_t idx = index(edge_type); + if (owned) { + result = {this, kSelf}; + CordRep::Unref(edges_[idx]); + } else { + // Create a copy containing all unchanged edges. Unchanged edges are the + // open interval [begin, back) or [begin + 1, end) depending on `edge_type`. + // We conveniently cover both case using a constexpr `shift` being 0 or 1 + // as `end :== back + 1`. + result = {CopyRaw(), kCopied}; + constexpr int shift = edge_type == kFront ? 1 : 0; + for (CordRep* r : Edges(begin() + shift, back() + shift)) { + CordRep::Ref(r); + } + } + result.tree->edges_[idx] = edge; + result.tree->length += delta; + return result; +} + +template <EdgeType edge_type> +CordRepBtree* CordRepBtree::AddCordRep(CordRepBtree* tree, CordRep* rep) { + const int depth = tree->height(); + const size_t length = rep->length; + StackOperations<edge_type> ops; + CordRepBtree* leaf = ops.BuildStack(tree, depth); + const OpResult result = + leaf->AddEdge<edge_type>(ops.owned(depth), rep, length); + return ops.Unwind(tree, depth, length, result); +} + +template <> +CordRepBtree* CordRepBtree::NewLeaf<kBack>(absl::string_view data, + size_t extra) { + CordRepBtree* leaf = CordRepBtree::New(0); + size_t length = 0; + size_t end = 0; + const size_t cap = leaf->capacity(); + while (!data.empty() && end != cap) { + auto* flat = CordRepFlat::New(data.length() + extra); + flat->length = (std::min)(data.length(), flat->Capacity()); + length += flat->length; + leaf->edges_[end++] = flat; + data = Consume<kBack>(flat->Data(), data, flat->length); + } + leaf->length = length; + leaf->set_end(end); + return leaf; +} + +template <> +CordRepBtree* CordRepBtree::NewLeaf<kFront>(absl::string_view data, + size_t extra) { + CordRepBtree* leaf = CordRepBtree::New(0); + size_t length = 0; + size_t begin = leaf->capacity(); + leaf->set_end(leaf->capacity()); + while (!data.empty() && begin != 0) { + auto* flat = CordRepFlat::New(data.length() + extra); + flat->length = (std::min)(data.length(), flat->Capacity()); + length += flat->length; + leaf->edges_[--begin] = flat; + data = Consume<kFront>(flat->Data(), data, flat->length); + } + leaf->length = length; + leaf->set_begin(begin); + return leaf; +} + +template <> +absl::string_view CordRepBtree::AddData<kBack>(absl::string_view data, + size_t extra) { + assert(!data.empty()); + assert(size() < capacity()); + AlignBegin(); + const size_t cap = capacity(); + do { + CordRepFlat* flat = CordRepFlat::New(data.length() + extra); + const size_t n = (std::min)(data.length(), flat->Capacity()); + flat->length = n; + edges_[fetch_add_end(1)] = flat; + data = Consume<kBack>(flat->Data(), data, n); + } while (!data.empty() && end() != cap); + return data; +} + +template <> +absl::string_view CordRepBtree::AddData<kFront>(absl::string_view data, + size_t extra) { + assert(!data.empty()); + assert(size() < capacity()); + AlignEnd(); + do { + CordRepFlat* flat = CordRepFlat::New(data.length() + extra); + const size_t n = (std::min)(data.length(), flat->Capacity()); + flat->length = n; + edges_[sub_fetch_begin(1)] = flat; + data = Consume<kFront>(flat->Data(), data, n); + } while (!data.empty() && begin() != 0); + return data; +} + +template <EdgeType edge_type> +CordRepBtree* CordRepBtree::AddData(CordRepBtree* tree, absl::string_view data, + size_t extra) { + if (ABSL_PREDICT_FALSE(data.empty())) return tree; + + const size_t original_data_size = data.size(); + int depth = tree->height(); + StackOperations<edge_type> ops; + CordRepBtree* leaf = ops.BuildStack(tree, depth); + + // If there is capacity in the last edge, append as much data + // as possible into this last edge. + if (leaf->size() < leaf->capacity()) { + OpResult result = leaf->ToOpResult(ops.owned(depth)); + data = result.tree->AddData<edge_type>(data, extra); + if (data.empty()) { + result.tree->length += original_data_size; + return ops.Unwind(tree, depth, original_data_size, result); + } + + // We added some data into this leaf, but not all. Propagate the added + // length to the top most node, and rebuild the stack with any newly copied + // or updated nodes. From this point on, the path (leg) from the top most + // node to the right-most node towards the leaf node is privately owned. + size_t delta = original_data_size - data.size(); + assert(delta > 0); + result.tree->length += delta; + tree = ops.Propagate(tree, depth, delta, result); + ops.share_depth = depth + 1; + } + + // We were unable to append all data into the existing right-most leaf node. + // This means all remaining data must be put into (a) new leaf node(s) which + // we append to the tree. To make this efficient, we iteratively build full + // leaf nodes from `data` until the created leaf contains all remaining data. + // We utilize the `Unwind` method to merge the created leaf into the first + // level towards root that has capacity. On each iteration with remaining + // data, we rebuild the stack in the knowledge that right-most nodes are + // privately owned after the first `Unwind` completes. + for (;;) { + OpResult result = {CordRepBtree::NewLeaf<edge_type>(data, extra), kPopped}; + if (result.tree->length == data.size()) { + return ops.Unwind(tree, depth, result.tree->length, result); + } + data = Consume<edge_type>(data, result.tree->length); + tree = ops.Unwind(tree, depth, result.tree->length, result); + depth = tree->height(); + ops.BuildOwnedStack(tree, depth); + } +} + +template <EdgeType edge_type> +CordRepBtree* CordRepBtree::Merge(CordRepBtree* dst, CordRepBtree* src) { + assert(dst->height() >= src->height()); + + // Capture source length as we may consume / destroy `src`. + const size_t length = src->length; + + // We attempt to merge `src` at its corresponding height in `dst`. + const int depth = dst->height() - src->height(); + StackOperations<edge_type> ops; + CordRepBtree* merge_node = ops.BuildStack(dst, depth); + + // If there is enough space in `merge_node` for all edges from `src`, add all + // edges to this node, making a fresh copy as needed if not privately owned. + // If `merge_node` does not have capacity for `src`, we rely on `Unwind` and + // `Finalize` to merge `src` into the first level towards `root` where there + // is capacity for another edge, or create a new top level node. + OpResult result; + if (merge_node->size() + src->size() <= kMaxCapacity) { + result = merge_node->ToOpResult(ops.owned(depth)); + result.tree->Add<edge_type>(src->Edges()); + result.tree->length += src->length; + if (src->refcount.IsOne()) { + Delete(src); + } else { + for (CordRep* edge : src->Edges()) CordRep::Ref(edge); + CordRepBtree::Unref(src); + } + } else { + result = {src, kPopped}; + } + + // Unless we merged at the top level (i.e.: src and dst are equal height), + // unwind the result towards the top level, and finalize the result. + if (depth) { + return ops.Unwind(dst, depth, length, result); + } + return ops.Finalize(dst, result); +} + +CopyResult CordRepBtree::CopySuffix(size_t offset) { + assert(offset < this->length); + + // As long as `offset` starts inside the last edge, we can 'drop' the current + // depth. For the most extreme example: if offset references the last data + // edge in the tree, there is only a single edge / path from the top of the + // tree to that last edge, so we can drop all the nodes except that edge. + // The fast path check for this is `back->length >= length - offset`. + int height = this->height(); + CordRepBtree* node = this; + size_t len = node->length - offset; + CordRep* back = node->Edge(kBack); + while (back->length >= len) { + offset = back->length - len; + if (--height < 0) { + return {MakeSubstring(CordRep::Ref(back), offset), height}; + } + node = back->btree(); + back = node->Edge(kBack); + } + if (offset == 0) return {CordRep::Ref(node), height}; + + // Offset does not point into the last edge, so we span at least two edges. + // Find the index of offset with `IndexBeyond` which provides us the edge + // 'beyond' the offset if offset is not a clean starting point of an edge. + Position pos = node->IndexBeyond(offset); + CordRepBtree* sub = node->CopyToEndFrom(pos.index, len); + const CopyResult result = {sub, height}; + + // `pos.n` contains a non zero value if the offset is not an exact starting + // point of an edge. In this case, `pos.n` contains the 'trailing' amount of + // bytes of the edge preceding that in `pos.index`. We need to iteratively + // adjust the preceding edge with the 'broken' offset until we have a perfect + // start of the edge. + while (pos.n != 0) { + assert(pos.index >= 1); + const size_t begin = pos.index - 1; + sub->set_begin(begin); + CordRep* const edge = node->Edge(begin); + + len = pos.n; + offset = edge->length - len; + + if (--height < 0) { + sub->edges_[begin] = MakeSubstring(CordRep::Ref(edge), offset, len); + return result; + } + + node = edge->btree(); + pos = node->IndexBeyond(offset); + + CordRepBtree* nsub = node->CopyToEndFrom(pos.index, len); + sub->edges_[begin] = nsub; + sub = nsub; + } + sub->set_begin(pos.index); + return result; +} + +CopyResult CordRepBtree::CopyPrefix(size_t n, bool allow_folding) { + assert(n > 0); + assert(n <= this->length); + + // As long as `n` does not exceed the length of the first edge, we can 'drop' + // the current depth. For the most extreme example: if we'd copy a 1 byte + // prefix from a tree, there is only a single edge / path from the top of the + // tree to the single data edge containing this byte, so we can drop all the + // nodes except the data node. + int height = this->height(); + CordRepBtree* node = this; + CordRep* front = node->Edge(kFront); + if (allow_folding) { + while (front->length >= n) { + if (--height < 0) return {MakeSubstring(CordRep::Ref(front), 0, n), -1}; + node = front->btree(); + front = node->Edge(kFront); + } + } + if (node->length == n) return {CordRep::Ref(node), height}; + + // `n` spans at least two nodes, find the end point of the span. + Position pos = node->IndexOf(n); + + // Create a partial copy of the node up to `pos.index`, with a defined length + // of `n`. Any 'partial last edge' is added further below as needed. + CordRepBtree* sub = node->CopyBeginTo(pos.index, n); + const CopyResult result = {sub, height}; + + // `pos.n` contains the 'offset inside the edge for IndexOf(n)'. As long as + // this is not zero, we don't have a 'clean cut', so we need to make a + // (partial) copy of that last edge, and repeat this until pos.n is zero. + while (pos.n != 0) { + size_t end = pos.index; + n = pos.n; + + CordRep* edge = node->Edge(pos.index); + if (--height < 0) { + sub->edges_[end++] = MakeSubstring(CordRep::Ref(edge), 0, n); + sub->set_end(end); + AssertValid(result.edge->btree()); + return result; + } + + node = edge->btree(); + pos = node->IndexOf(n); + CordRepBtree* nsub = node->CopyBeginTo(pos.index, n); + sub->edges_[end++] = nsub; + sub->set_end(end); + sub = nsub; + } + sub->set_end(pos.index); + AssertValid(result.edge->btree()); + return result; +} + +CordRep* CordRepBtree::ExtractFront(CordRepBtree* tree) { + CordRep* front = tree->Edge(tree->begin()); + if (tree->refcount.IsOne()) { + Unref(tree->Edges(tree->begin() + 1, tree->end())); + CordRepBtree::Delete(tree); + } else { + CordRep::Ref(front); + CordRep::Unref(tree); + } + return front; +} + +CordRepBtree* CordRepBtree::ConsumeBeginTo(CordRepBtree* tree, size_t end, + size_t new_length) { + assert(end <= tree->end()); + if (tree->refcount.IsOne()) { + Unref(tree->Edges(end, tree->end())); + tree->set_end(end); + tree->length = new_length; + } else { + CordRepBtree* old = tree; + tree = tree->CopyBeginTo(end, new_length); + CordRep::Unref(old); + } + return tree; +} + +CordRep* CordRepBtree::RemoveSuffix(CordRepBtree* tree, size_t n) { + // Check input and deal with trivial cases 'Remove all/none' + assert(tree != nullptr); + assert(n <= tree->length); + const size_t len = tree->length; + if (ABSL_PREDICT_FALSE(n == 0)) { + return tree; + } + if (ABSL_PREDICT_FALSE(n >= len)) { + CordRepBtree::Unref(tree); + return nullptr; + } + + size_t length = len - n; + int height = tree->height(); + bool is_mutable = tree->refcount.IsOne(); + + // Extract all top nodes which are reduced to size = 1 + Position pos = tree->IndexOfLength(length); + while (pos.index == tree->begin()) { + CordRep* edge = ExtractFront(tree); + is_mutable &= edge->refcount.IsOne(); + if (height-- == 0) return ResizeEdge(edge, length, is_mutable); + tree = edge->btree(); + pos = tree->IndexOfLength(length); + } + + // Repeat the following sequence traversing down the tree: + // - Crop the top node to the 'last remaining edge' adjusting length. + // - Set the length for down edges to the partial length in that last edge. + // - Repeat this until the last edge is 'included in full' + // - If we hit the data edge level, resize and return the last data edge + CordRepBtree* top = tree = ConsumeBeginTo(tree, pos.index + 1, length); + CordRep* edge = tree->Edge(pos.index); + length = pos.n; + while (length != edge->length) { + // ConsumeBeginTo guarantees `tree` is a clean, privately owned copy. + assert(tree->refcount.IsOne()); + const bool edge_is_mutable = edge->refcount.IsOne(); + + if (height-- == 0) { + tree->edges_[pos.index] = ResizeEdge(edge, length, edge_is_mutable); + return AssertValid(top); + } + + if (!edge_is_mutable) { + // We can't 'in place' remove any suffixes down this edge. + // Replace this edge with a prefix copy instead. + tree->edges_[pos.index] = edge->btree()->CopyPrefix(length, false).edge; + CordRep::Unref(edge); + return AssertValid(top); + } + + // Move down one level, rinse repeat. + tree = edge->btree(); + pos = tree->IndexOfLength(length); + tree = ConsumeBeginTo(edge->btree(), pos.index + 1, length); + edge = tree->Edge(pos.index); + length = pos.n; + } + + return AssertValid(top); +} + +CordRep* CordRepBtree::SubTree(size_t offset, size_t n) { + assert(n <= this->length); + assert(offset <= this->length - n); + if (ABSL_PREDICT_FALSE(n == 0)) return nullptr; + + CordRepBtree* node = this; + int height = node->height(); + Position front = node->IndexOf(offset); + CordRep* left = node->edges_[front.index]; + while (front.n + n <= left->length) { + if (--height < 0) return MakeSubstring(CordRep::Ref(left), front.n, n); + node = left->btree(); + front = node->IndexOf(front.n); + left = node->edges_[front.index]; + } + + const Position back = node->IndexBefore(front, n); + CordRep* const right = node->edges_[back.index]; + assert(back.index > front.index); + + // Get partial suffix and prefix entries. + CopyResult prefix; + CopyResult suffix; + if (height > 0) { + // Copy prefix and suffix of the boundary nodes. + prefix = left->btree()->CopySuffix(front.n); + suffix = right->btree()->CopyPrefix(back.n); + + // If there is an edge between the prefix and suffix edges, then the tree + // must remain at its previous (full) height. If we have no edges between + // prefix and suffix edges, then the tree must be as high as either the + // suffix or prefix edges (which are collapsed to their minimum heights). + if (front.index + 1 == back.index) { + height = (std::max)(prefix.height, suffix.height) + 1; + } + + // Raise prefix and suffixes to the new tree height. + for (int h = prefix.height + 1; h < height; ++h) { + prefix.edge = CordRepBtree::New(prefix.edge); + } + for (int h = suffix.height + 1; h < height; ++h) { + suffix.edge = CordRepBtree::New(suffix.edge); + } + } else { + // Leaf node, simply take substrings for prefix and suffix. + prefix = CopyResult{MakeSubstring(CordRep::Ref(left), front.n), -1}; + suffix = CopyResult{MakeSubstring(CordRep::Ref(right), 0, back.n), -1}; + } + + // Compose resulting tree. + CordRepBtree* sub = CordRepBtree::New(height); + size_t end = 0; + sub->edges_[end++] = prefix.edge; + for (CordRep* r : node->Edges(front.index + 1, back.index)) { + sub->edges_[end++] = CordRep::Ref(r); + } + sub->edges_[end++] = suffix.edge; + sub->set_end(end); + sub->length = n; + return AssertValid(sub); +} + +CordRepBtree* CordRepBtree::MergeTrees(CordRepBtree* left, + CordRepBtree* right) { + return left->height() >= right->height() ? Merge<kBack>(left, right) + : Merge<kFront>(right, left); +} + +bool CordRepBtree::IsFlat(absl::string_view* fragment) const { + if (height() == 0 && size() == 1) { + if (fragment) *fragment = Data(begin()); + return true; + } + return false; +} + +bool CordRepBtree::IsFlat(size_t offset, const size_t n, + absl::string_view* fragment) const { + assert(n <= this->length); + assert(offset <= this->length - n); + if (ABSL_PREDICT_FALSE(n == 0)) return false; + int height = this->height(); + const CordRepBtree* node = this; + for (;;) { + const Position front = node->IndexOf(offset); + const CordRep* edge = node->Edge(front.index); + if (edge->length < front.n + n) return false; + if (--height < 0) { + if (fragment) *fragment = EdgeData(edge).substr(front.n, n); + return true; + } + offset = front.n; + node = node->Edge(front.index)->btree(); + } +} + +char CordRepBtree::GetCharacter(size_t offset) const { + assert(offset < length); + const CordRepBtree* node = this; + int height = node->height(); + for (;;) { + Position front = node->IndexOf(offset); + if (--height < 0) return node->Data(front.index)[front.n]; + offset = front.n; + node = node->Edge(front.index)->btree(); + } +} + +Span<char> CordRepBtree::GetAppendBufferSlow(size_t size) { + // The inlined version in `GetAppendBuffer()` deals with all heights <= 3. + assert(height() >= 4); + assert(refcount.IsOne()); + + // Build a stack of nodes we may potentially need to update if we find a + // non-shared FLAT with capacity at the leaf level. + const int depth = height(); + CordRepBtree* node = this; + CordRepBtree* stack[kMaxDepth]; + for (int i = 0; i < depth; ++i) { + node = node->Edge(kBack)->btree(); + if (!node->refcount.IsOne()) return {}; + stack[i] = node; + } + + // Must be a privately owned, mutable flat. + CordRep* const edge = node->Edge(kBack); + if (!edge->refcount.IsOne() || edge->tag < FLAT) return {}; + + // Must have capacity. + const size_t avail = edge->flat()->Capacity() - edge->length; + if (avail == 0) return {}; + + // Build span on remaining capacity. + size_t delta = (std::min)(size, avail); + Span<char> span = {edge->flat()->Data() + edge->length, delta}; + edge->length += delta; + this->length += delta; + for (int i = 0; i < depth; ++i) { + stack[i]->length += delta; + } + return span; +} + +CordRepBtree* CordRepBtree::CreateSlow(CordRep* rep) { + if (rep->IsBtree()) return rep->btree(); + + CordRepBtree* node = nullptr; + auto consume = [&node](CordRep* r, size_t offset, size_t length) { + r = MakeSubstring(r, offset, length); + if (node == nullptr) { + node = New(r); + } else { + node = CordRepBtree::AddCordRep<kBack>(node, r); + } + }; + Consume(rep, consume); + return node; +} + +CordRepBtree* CordRepBtree::AppendSlow(CordRepBtree* tree, CordRep* rep) { + if (ABSL_PREDICT_TRUE(rep->IsBtree())) { + return MergeTrees(tree, rep->btree()); + } + auto consume = [&tree](CordRep* r, size_t offset, size_t length) { + r = MakeSubstring(r, offset, length); + tree = CordRepBtree::AddCordRep<kBack>(tree, r); + }; + Consume(rep, consume); + return tree; +} + +CordRepBtree* CordRepBtree::PrependSlow(CordRepBtree* tree, CordRep* rep) { + if (ABSL_PREDICT_TRUE(rep->IsBtree())) { + return MergeTrees(rep->btree(), tree); + } + auto consume = [&tree](CordRep* r, size_t offset, size_t length) { + r = MakeSubstring(r, offset, length); + tree = CordRepBtree::AddCordRep<kFront>(tree, r); + }; + ReverseConsume(rep, consume); + return tree; +} + +CordRepBtree* CordRepBtree::Append(CordRepBtree* tree, absl::string_view data, + size_t extra) { + return CordRepBtree::AddData<kBack>(tree, data, extra); +} + +CordRepBtree* CordRepBtree::Prepend(CordRepBtree* tree, absl::string_view data, + size_t extra) { + return CordRepBtree::AddData<kFront>(tree, data, extra); +} + +template CordRepBtree* CordRepBtree::AddCordRep<kFront>(CordRepBtree* tree, + CordRep* rep); +template CordRepBtree* CordRepBtree::AddCordRep<kBack>(CordRepBtree* tree, + CordRep* rep); +template CordRepBtree* CordRepBtree::AddData<kFront>(CordRepBtree* tree, + absl::string_view data, + size_t extra); +template CordRepBtree* CordRepBtree::AddData<kBack>(CordRepBtree* tree, + absl::string_view data, + size_t extra); + +void CordRepBtree::Rebuild(CordRepBtree** stack, CordRepBtree* tree, + bool consume) { + bool owned = consume && tree->refcount.IsOne(); + if (tree->height() == 0) { + for (CordRep* edge : tree->Edges()) { + if (!owned) edge = CordRep::Ref(edge); + size_t height = 0; + size_t length = edge->length; + CordRepBtree* node = stack[0]; + OpResult result = node->AddEdge<kBack>(true, edge, length); + while (result.action == CordRepBtree::kPopped) { + stack[height] = result.tree; + if (stack[++height] == nullptr) { + result.action = CordRepBtree::kSelf; + stack[height] = CordRepBtree::New(node, result.tree); + } else { + node = stack[height]; + result = node->AddEdge<kBack>(true, result.tree, length); + } + } + while (stack[++height] != nullptr) { + stack[height]->length += length; + } + } + } else { + for (CordRep* rep : tree->Edges()) { + Rebuild(stack, rep->btree(), owned); + } + } + if (consume) { + if (owned) { + CordRepBtree::Delete(tree); + } else { + CordRepBtree::Unref(tree); + } + } +} + +CordRepBtree* CordRepBtree::Rebuild(CordRepBtree* tree) { + // Set up initial stack with empty leaf node. + CordRepBtree* node = CordRepBtree::New(); + CordRepBtree* stack[CordRepBtree::kMaxDepth + 1] = {node}; + + // Recursively build the tree, consuming the input tree. + Rebuild(stack, tree, /* consume reference */ true); + + // Return top most node + for (CordRepBtree* parent : stack) { + if (parent == nullptr) return node; + node = parent; + } + + // Unreachable + assert(false); + return nullptr; +} + +CordRepBtree::ExtractResult CordRepBtree::ExtractAppendBuffer( + CordRepBtree* tree, size_t extra_capacity) { + int depth = 0; + NodeStack stack; + + // Set up default 'no success' result which is {tree, nullptr}. + ExtractResult result; + result.tree = tree; + result.extracted = nullptr; + + // Dive down the right side of the tree, making sure no edges are shared. + while (tree->height() > 0) { + if (!tree->refcount.IsOne()) return result; + stack[depth++] = tree; + tree = tree->Edge(kBack)->btree(); + } + if (!tree->refcount.IsOne()) return result; + + // Validate we ended on a non shared flat. + CordRep* rep = tree->Edge(kBack); + if (!(rep->IsFlat() && rep->refcount.IsOne())) return result; + + // Verify it has at least the requested extra capacity. + CordRepFlat* flat = rep->flat(); + const size_t length = flat->length; + const size_t avail = flat->Capacity() - flat->length; + if (extra_capacity > avail) return result; + + // Set the extracted flat in the result. + result.extracted = flat; + + // Cascading delete all nodes that become empty. + while (tree->size() == 1) { + CordRepBtree::Delete(tree); + if (--depth < 0) { + // We consumed the entire tree: return nullptr for new tree. + result.tree = nullptr; + return result; + } + rep = tree; + tree = stack[depth]; + } + + // Remove the edge or cascaded up parent node. + tree->set_end(tree->end() - 1); + tree->length -= length; + + // Adjust lengths up the tree. + while (depth > 0) { + tree = stack[--depth]; + tree->length -= length; + } + + // Remove unnecessary top nodes with size = 1. This may iterate all the way + // down to the leaf node in which case we simply return the remaining last + // edge in that node and the extracted flat. + while (tree->size() == 1) { + int height = tree->height(); + rep = tree->Edge(kBack); + Delete(tree); + if (height == 0) { + // We consumed the leaf: return the sole data edge as the new tree. + result.tree = rep; + return result; + } + tree = rep->btree(); + } + + // Done: return the (new) top level node and extracted flat. + result.tree = tree; + return result; +} + +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/absl/strings/internal/cord_rep_btree.h b/absl/strings/internal/cord_rep_btree.h new file mode 100644 index 00000000..2cbc09ec --- /dev/null +++ b/absl/strings/internal/cord_rep_btree.h @@ -0,0 +1,924 @@ +// Copyright 2021 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_STRINGS_INTERNAL_CORD_REP_BTREE_H_ +#define ABSL_STRINGS_INTERNAL_CORD_REP_BTREE_H_ + +#include <cassert> +#include <cstdint> +#include <iosfwd> + +#include "absl/base/config.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/base/optimization.h" +#include "absl/strings/internal/cord_data_edge.h" +#include "absl/strings/internal/cord_internal.h" +#include "absl/strings/internal/cord_rep_flat.h" +#include "absl/strings/string_view.h" +#include "absl/types/span.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { + +class CordRepBtreeNavigator; + +// CordRepBtree is as the name implies a btree implementation of a Cordrep tree. +// Data is stored at the leaf level only, non leaf nodes contain down pointers +// only. Allowed types of data edges are FLAT, EXTERNAL and SUBSTRINGs of FLAT +// or EXTERNAL nodes. The implementation allows for data to be added to either +// end of the tree only, it does not provide any 'insert' logic. This has the +// benefit that we can expect good fill ratios: all nodes except the outer +// 'legs' will have 100% fill ratios for trees built using Append/Prepend +// methods. Merged trees will typically have a fill ratio well above 50% as in a +// similar fashion, one side of the merged tree will typically have a 100% fill +// ratio, and the 'open' end will average 50%. All operations are O(log(n)) or +// better, and the tree never needs balancing. +// +// All methods accepting a CordRep* or CordRepBtree* adopt a reference on that +// input unless explicitly stated otherwise. All functions returning a CordRep* +// or CordRepBtree* instance transfer a reference back to the caller. +// Simplified, callers both 'donate' and 'consume' a reference count on each +// call, simplifying the API. An example of building a tree: +// +// CordRepBtree* tree = CordRepBtree::Create(MakeFlat("Hello")); +// tree = CordRepBtree::Append(tree, MakeFlat("world")); +// +// In the above example, all inputs are consumed, making each call affecting +// `tree` reference count neutral. The returned `tree` value can be different +// from the input if the input is shared with other threads, or if the tree +// grows in height, but callers typically never have to concern themselves with +// that and trust that all methods DTRT at all times. +class CordRepBtree : public CordRep { + public: + // EdgeType identifies `front` and `back` enum values. + // Various implementations in CordRepBtree such as `Add` and `Edge` are + // generic and templated on operating on either of the boundary edges. + // For more information on the possible edges contained in a CordRepBtree + // instance see the documentation for `edges_`. + enum class EdgeType { kFront, kBack }; + + // Convenience constants into `EdgeType` + static constexpr EdgeType kFront = EdgeType::kFront; + static constexpr EdgeType kBack = EdgeType::kBack; + + // Maximum number of edges: based on experiments and performance data, we can + // pick suitable values resulting in optimum cacheline aligned values. The + // preferred values are based on 64-bit systems where we aim to align this + // class onto 64 bytes, i.e.: 6 = 64 bytes, 14 = 128 bytes, etc. + // TODO(b/192061034): experiment with alternative sizes. + static constexpr size_t kMaxCapacity = 6; + + // Reasonable maximum height of the btree. We can expect a fill ratio of at + // least 50%: trees are always expanded at the front or back. Concatenating + // trees will then typically fold at the top most node, where the lower nodes + // are at least at capacity on one side of joined inputs. At a lower fill + // rate of 4 edges per node, we have capacity for ~16 million leaf nodes. + // We will fail / abort if an application ever exceeds this height, which + // should be extremely rare (near impossible) and be an indication of an + // application error: we do not assume it reasonable for any application to + // operate correctly with such monster trees. + // Another compelling reason for the number `12` is that any contextual stack + // required for navigation or insertion requires 12 words and 12 bytes, which + // fits inside 2 cache lines with some room to spare, and is reasonable as a + // local stack variable compared to Cord's current near 400 bytes stack use. + // The maximum `height` value of a node is then `kMaxDepth - 1` as node height + // values start with a value of 0 for leaf nodes. + static constexpr int kMaxDepth = 12; + static constexpr int kMaxHeight = kMaxDepth - 1; + + // `Action` defines the action for unwinding changes done at the btree's leaf + // level that need to be propagated up to the parent node(s). Each operation + // on a node has an effect / action defined as follows: + // - kSelf + // The operation (add / update, etc) was performed directly on the node as + // the node is private to the current thread (i.e.: not shared directly or + // indirectly through a refcount > 1). Changes can be propagated directly to + // all parent nodes as all parent nodes are also then private to the current + // thread. + // - kCopied + // The operation (add / update, etc) was performed on a copy of the original + // node, as the node is (potentially) directly or indirectly shared with + // other threads. Changes need to be propagated into the parent nodes where + // the old down pointer must be unreffed and replaced with this new copy. + // Such changes to parent nodes may themselves require a copy if the parent + // node is also shared. A kCopied action can propagate all the way to the + // top node where we then must unref the `tree` input provided by the + // caller, and return the new copy. + // - kPopped + // The operation (typically add) could not be satisfied due to insufficient + // capacity in the targeted node, and a new 'leg' was created that needs to + // be added into the parent node. For example, adding a FLAT inside a leaf + // node that is at capacity will create a new leaf node containing that + // FLAT, that needs to be 'popped' up the btree. Such 'pop' actions can + // cascade up the tree if parent nodes are also at capacity. A 'Popped' + // action propagating all the way to the top of the tree will result in + // the tree becoming one level higher than the current tree through a final + // `CordRepBtree::New(tree, popped)` call, resulting in a new top node + // referencing the old tree and the new (fully popped upwards) 'leg'. + enum Action { kSelf, kCopied, kPopped }; + + // Result of an operation on a node. See the `Action` enum for details. + struct OpResult { + CordRepBtree* tree; + Action action; + }; + + // Return value of the CopyPrefix and CopySuffix methods which can + // return a node or data edge at any height inside the tree. + // A height of 0 defines the lowest (leaf) node, a height of -1 identifies + // `edge` as being a plain data node: EXTERNAL / FLAT or SUBSTRING thereof. + struct CopyResult { + CordRep* edge; + int height; + }; + + // Logical position inside a node: + // - index: index of the edge. + // - n: size or offset value depending on context. + struct Position { + size_t index; + size_t n; + }; + + // Creates a btree from the given input. Adopts a ref of `rep`. + // If the input `rep` is itself a btree, i.e., `IsBtree()`, then this + // function immediately returns `rep->btree()`. If the input is a valid data + // edge (see IsDataEdge()), then a new leaf node is returned containing `rep` + // as the sole data edge. Else, the input is assumed to be a (legacy) concat + // tree, and the input is consumed and transformed into a btree(). + static CordRepBtree* Create(CordRep* rep); + + // Destroys the provided tree. Should only be called by cord internal API's, + // typically after a ref_count.Decrement() on the last reference count. + static void Destroy(CordRepBtree* tree); + + // Destruction + static void Delete(CordRepBtree* tree) { delete tree; } + + // Use CordRep::Unref() as we overload for absl::Span<CordRep* const>. + using CordRep::Unref; + + // Unrefs all edges in `edges` which are assumed to be 'likely one'. + static void Unref(absl::Span<CordRep* const> edges); + + // Appends / Prepends an existing CordRep instance to this tree. + // The below methods accept three types of input: + // 1) `rep` is a data node (See `IsDataNode` for valid data edges). + // `rep` is appended or prepended to this tree 'as is'. + // 2) `rep` is a BTREE. + // `rep` is merged into `tree` respecting the Append/Prepend order. + // 3) `rep` is some other (legacy) type. + // `rep` is converted in place and added to `tree` + // Requires `tree` and `rep` to be not null. + static CordRepBtree* Append(CordRepBtree* tree, CordRep* rep); + static CordRepBtree* Prepend(CordRepBtree* tree, CordRep* rep); + + // Append/Prepend the data in `data` to this tree. + // The `extra` parameter defines how much extra capacity should be allocated + // for any additional FLAT being allocated. This is an optimization hint from + // the caller. For example, a caller may need to add 2 string_views of data + // "abc" and "defghi" which are not consecutive. The caller can in this case + // invoke `AddData(tree, "abc", 6)`, and any newly added flat is allocated + // where possible with at least 6 bytes of extra capacity beyond `length`. + // This helps avoiding data getting fragmented over multiple flats. + // There is no limit on the size of `data`. If `data` can not be stored inside + // a single flat, then the function will iteratively add flats until all data + // has been consumed and appended or prepended to the tree. + static CordRepBtree* Append(CordRepBtree* tree, string_view data, + size_t extra = 0); + static CordRepBtree* Prepend(CordRepBtree* tree, string_view data, + size_t extra = 0); + + // Returns a new tree, containing `n` bytes of data from this instance + // starting at offset `offset`. Where possible, the returned tree shares + // (re-uses) data edges and nodes with this instance to minimize the + // combined memory footprint of both trees. + // Requires `offset + n <= length`. Returns `nullptr` if `n` is zero. + CordRep* SubTree(size_t offset, size_t n); + + // Removes `n` trailing bytes from `tree`, and returns the resulting tree + // or data edge. Returns `tree` if n is zero, and nullptr if n == length. + // This function is logically identical to: + // result = tree->SubTree(0, tree->length - n); + // Unref(tree); + // return result; + // However, the actual implementation will as much as possible perform 'in + // place' modifications on the tree on all nodes and edges that are mutable. + // For example, in a fully privately owned tree with the last edge being a + // flat of length 12, RemoveSuffix(1) will simply set the length of that data + // edge to 11, and reduce the length of all nodes on the edge path by 1. + static CordRep* RemoveSuffix(CordRepBtree* tree, size_t n); + + // Returns the character at the given offset. + char GetCharacter(size_t offset) const; + + // Returns true if this node holds a single data edge, and if so, sets + // `fragment` to reference the contained data. `fragment` is an optional + // output parameter and allowed to be null. + bool IsFlat(absl::string_view* fragment) const; + + // Returns true if the data of `n` bytes starting at offset `offset` + // is contained in a single data edge, and if so, sets fragment to reference + // the contained data. `fragment` is an optional output parameter and allowed + // to be null. + bool IsFlat(size_t offset, size_t n, absl::string_view* fragment) const; + + // Returns a span (mutable range of bytes) of up to `size` bytes into the + // last FLAT data edge inside this tree under the following conditions: + // - none of the nodes down into the FLAT node are shared. + // - the last data edge in this tree is a non-shared FLAT. + // - the referenced FLAT has additional capacity available. + // If all these conditions are met, a non-empty span is returned, and the + // length of the flat node and involved tree nodes have been increased by + // `span.length()`. The caller is responsible for immediately assigning values + // to all uninitialized data reference by the returned span. + // Requires `this->refcount.IsOne()`: this function forces the caller to do + // this fast path check on the top level node, as this is the most commonly + // shared node of a cord tree. + Span<char> GetAppendBuffer(size_t size); + + // Extracts the right-most data edge from this tree iff: + // - the tree and all internal edges to the right-most node are not shared. + // - the right-most node is a FLAT node and not shared. + // - the right-most node has at least the desired extra capacity. + // + // Returns {tree, nullptr} if any of the above conditions are not met. + // This method effectively removes data from the tree. The intent of this + // method is to allow applications appending small string data to use + // pre-existing capacity, and add the modified rep back to the tree. + // + // Simplified such code would look similar to this: + // void MyTreeBuilder::Append(string_view data) { + // ExtractResult result = CordRepBtree::ExtractAppendBuffer(tree_, 1); + // if (CordRep* rep = result.extracted) { + // size_t available = rep->Capacity() - rep->length; + // size_t n = std::min(data.size(), n); + // memcpy(rep->Data(), data.data(), n); + // rep->length += n; + // data.remove_prefix(n); + // if (!result.tree->IsBtree()) { + // tree_ = CordRepBtree::Create(result.tree); + // } + // tree_ = CordRepBtree::Append(tree_, rep); + // } + // ... + // // Remaining edge in `result.tree`. + // } + static ExtractResult ExtractAppendBuffer(CordRepBtree* tree, + size_t extra_capacity = 1); + + // Returns the `height` of the tree. The height of a tree is limited to + // kMaxHeight. `height` is implemented as an `int` as in some places we + // use negative (-1) values for 'data edges'. + int height() const { return static_cast<int>(storage[0]); } + + // Properties: begin, back, end, front/back boundary indexes. + size_t begin() const { return static_cast<size_t>(storage[1]); } + size_t back() const { return static_cast<size_t>(storage[2]) - 1; } + size_t end() const { return static_cast<size_t>(storage[2]); } + size_t index(EdgeType edge) const { + return edge == kFront ? begin() : back(); + } + + // Properties: size and capacity. + // `capacity` contains the current capacity of this instance, where + // `kMaxCapacity` contains the maximum capacity of a btree node. + // For now, `capacity` and `kMaxCapacity` return the same value, but this may + // change in the future if we see benefit in dynamically sizing 'small' nodes + // to 'large' nodes for large data trees. + size_t size() const { return end() - begin(); } + size_t capacity() const { return kMaxCapacity; } + + // Edge access + inline CordRep* Edge(size_t index) const; + inline CordRep* Edge(EdgeType edge_type) const; + inline absl::Span<CordRep* const> Edges() const; + inline absl::Span<CordRep* const> Edges(size_t begin, size_t end) const; + + // Returns reference to the data edge at `index`. + // Requires this instance to be a leaf node, and `index` to be valid index. + inline absl::string_view Data(size_t index) const; + + // Diagnostics: returns true if `tree` is valid and internally consistent. + // If `shallow` is false, then the provided top level node and all child nodes + // below it are recursively checked. If `shallow` is true, only the provided + // node in `tree` and the cumulative length, type and height of the direct + // child nodes of `tree` are checked. The value of `shallow` is ignored if the + // internal `cord_btree_exhaustive_validation` diagnostics variable is true, + // in which case the performed validations works as if `shallow` were false. + // This function is intended for debugging and testing purposes only. + static bool IsValid(const CordRepBtree* tree, bool shallow = false); + + // Diagnostics: asserts that the provided tree is valid. + // `AssertValid()` performs a shallow validation by default. `shallow` can be + // set to false in which case an exhaustive validation is performed. This + // function is implemented in terms of calling `IsValid()` and asserting the + // return value to be true. See `IsValid()` for more information. + // This function is intended for debugging and testing purposes only. + static CordRepBtree* AssertValid(CordRepBtree* tree, bool shallow = true); + static const CordRepBtree* AssertValid(const CordRepBtree* tree, + bool shallow = true); + + // Diagnostics: dump the contents of this tree to `stream`. + // This function is intended for debugging and testing purposes only. + static void Dump(const CordRep* rep, std::ostream& stream); + static void Dump(const CordRep* rep, absl::string_view label, + std::ostream& stream); + static void Dump(const CordRep* rep, absl::string_view label, + bool include_contents, std::ostream& stream); + + // Adds the edge `edge` to this node if possible. `owned` indicates if the + // current node is potentially shared or not with other threads. Returns: + // - {kSelf, <this>} + // The edge was directly added to this node. + // - {kCopied, <node>} + // The edge was added to a copy of this node. + // - {kPopped, New(edge, height())} + // A new leg with the edge was created as this node has no extra capacity. + template <EdgeType edge_type> + inline OpResult AddEdge(bool owned, CordRep* edge, size_t delta); + + // Replaces the front or back edge with the provided new edge. Returns: + // - {kSelf, <this>} + // The edge was directly set in this node. The old edge is unreffed. + // - {kCopied, <node>} + // A copy of this node was created with the new edge value. + // In both cases, the function adopts a reference on `edge`. + template <EdgeType edge_type> + OpResult SetEdge(bool owned, CordRep* edge, size_t delta); + + // Creates a new empty node at the specified height. + static CordRepBtree* New(int height = 0); + + // Creates a new node containing `rep`, with the height being computed + // automatically based on the type of `rep`. + static CordRepBtree* New(CordRep* rep); + + // Creates a new node containing both `front` and `back` at height + // `front.height() + 1`. Requires `back.height() == front.height()`. + static CordRepBtree* New(CordRepBtree* front, CordRepBtree* back); + + // Creates a fully balanced tree from the provided tree by rebuilding a new + // tree from all data edges in the input. This function is automatically + // invoked internally when the tree exceeds the maximum height. + static CordRepBtree* Rebuild(CordRepBtree* tree); + + private: + CordRepBtree() = default; + ~CordRepBtree() = default; + + // Initializes the main properties `tag`, `begin`, `end`, `height`. + inline void InitInstance(int height, size_t begin = 0, size_t end = 0); + + // Direct property access begin / end + void set_begin(size_t begin) { storage[1] = static_cast<uint8_t>(begin); } + void set_end(size_t end) { storage[2] = static_cast<uint8_t>(end); } + + // Decreases the value of `begin` by `n`, and returns the new value. Notice + // how this returns the new value unlike atomic::fetch_add which returns the + // old value. This is because this is used to prepend edges at 'begin - 1'. + size_t sub_fetch_begin(size_t n) { + storage[1] -= static_cast<uint8_t>(n); + return storage[1]; + } + + // Increases the value of `end` by `n`, and returns the previous value. This + // function is typically used to append edges at 'end'. + size_t fetch_add_end(size_t n) { + const uint8_t current = storage[2]; + storage[2] = static_cast<uint8_t>(current + n); + return current; + } + + // Returns the index of the last edge starting on, or before `offset`, with + // `n` containing the relative offset of `offset` inside that edge. + // Requires `offset` < length. + Position IndexOf(size_t offset) const; + + // Returns the index of the last edge starting before `offset`, with `n` + // containing the relative offset of `offset` inside that edge. + // This function is useful to find the edges for some span of bytes ending at + // `offset` (i.e., `n` bytes). For example: + // + // Position pos = IndexBefore(n) + // edges = Edges(begin(), pos.index) // All full edges (may be empty) + // last = Sub(Edge(pos.index), 0, pos.n) // Last partial edge (may be empty) + // + // Requires 0 < `offset` <= length. + Position IndexBefore(size_t offset) const; + + // Returns the index of the edge ending at (or on) length `length`, and the + // number of bytes inside that edge up to `length`. For example, if we have a + // Node with 2 edges, one of 10 and one of 20 long, then IndexOfLength(27) + // will return {1, 17}, and IndexOfLength(10) will return {0, 10}. + Position IndexOfLength(size_t n) const; + + // Identical to the above function except starting from the position `front`. + // This function is equivalent to `IndexBefore(front.n + offset)`, with + // the difference that this function is optimized to start at `front.index`. + Position IndexBefore(Position front, size_t offset) const; + + // Returns the index of the edge directly beyond the edge containing offset + // `offset`, with `n` containing the distance of that edge from `offset`. + // This function is useful for iteratively finding suffix nodes and remaining + // partial bytes in left-most suffix nodes as for example in CopySuffix. + // Requires `offset` < length. + Position IndexBeyond(size_t offset) const; + + // Creates a new leaf node containing as much data as possible from `data`. + // The data is added either forwards or reversed depending on `edge_type`. + // Callers must check the length of the returned node to determine if all data + // was copied or not. + // See the `Append/Prepend` function for the meaning and purpose of `extra`. + template <EdgeType edge_type> + static CordRepBtree* NewLeaf(absl::string_view data, size_t extra); + + // Creates a raw copy of this Btree node, copying all properties, but + // without adding any references to existing edges. + CordRepBtree* CopyRaw() const; + + // Creates a full copy of this Btree node, adding a reference on all edges. + CordRepBtree* Copy() const; + + // Creates a partial copy of this Btree node, copying all edges up to `end`, + // adding a reference on each copied edge, and sets the length of the newly + // created copy to `new_length`. + CordRepBtree* CopyBeginTo(size_t end, size_t new_length) const; + + // Returns a tree containing the edges [tree->begin(), end) and length + // of `new_length`. This method consumes a reference on the provided + // tree, and logically performs the following operation: + // result = tree->CopyBeginTo(end, new_length); + // CordRep::Unref(tree); + // return result; + static CordRepBtree* ConsumeBeginTo(CordRepBtree* tree, size_t end, + size_t new_length); + + // Creates a partial copy of this Btree node, copying all edges starting at + // `begin`, adding a reference on each copied edge, and sets the length of + // the newly created copy to `new_length`. + CordRepBtree* CopyToEndFrom(size_t begin, size_t new_length) const; + + // Extracts and returns the front edge from the provided tree. + // This method consumes a reference on the provided tree, and logically + // performs the following operation: + // edge = CordRep::Ref(tree->Edge(kFront)); + // CordRep::Unref(tree); + // return edge; + static CordRep* ExtractFront(CordRepBtree* tree); + + // Returns a tree containing the result of appending `right` to `left`. + static CordRepBtree* MergeTrees(CordRepBtree* left, CordRepBtree* right); + + // Fallback functions for `Create()`, `Append()` and `Prepend()` which + // deal with legacy / non conforming input, i.e.: CONCAT trees. + static CordRepBtree* CreateSlow(CordRep* rep); + static CordRepBtree* AppendSlow(CordRepBtree*, CordRep* rep); + static CordRepBtree* PrependSlow(CordRepBtree*, CordRep* rep); + + // Recursively rebuilds `tree` into `stack`. If 'consume` is set to true, the + // function will consume a reference on `tree`. `stack` is a null terminated + // array containing the new tree's state, with the current leaf node at + // stack[0], and parent nodes above that, or null for 'top of tree'. + static void Rebuild(CordRepBtree** stack, CordRepBtree* tree, bool consume); + + // Aligns existing edges to start at index 0, to allow for a new edge to be + // added to the back of the current edges. + inline void AlignBegin(); + + // Aligns existing edges to end at `capacity`, to allow for a new edge to be + // added in front of the current edges. + inline void AlignEnd(); + + // Adds the provided edge to this node. + // Requires this node to have capacity for the edge. Realigns / moves + // existing edges as needed to prepend or append the new edge. + template <EdgeType edge_type> + inline void Add(CordRep* rep); + + // Adds the provided edges to this node. + // Requires this node to have capacity for the edges. Realigns / moves + // existing edges as needed to prepend or append the new edges. + template <EdgeType edge_type> + inline void Add(absl::Span<CordRep* const>); + + // Adds data from `data` to this node until either all data has been consumed, + // or there is no more capacity for additional flat nodes inside this node. + // Requires the current node to be a leaf node, data to be non empty, and the + // current node to have capacity for at least one more data edge. + // Returns any remaining data from `data` that was not added, which is + // depending on the edge type (front / back) either the remaining prefix of + // suffix of the input. + // See the `Append/Prepend` function for the meaning and purpose of `extra`. + template <EdgeType edge_type> + absl::string_view AddData(absl::string_view data, size_t extra); + + // Replace the front or back edge with the provided value. + // Adopts a reference on `edge` and unrefs the old edge. + template <EdgeType edge_type> + inline void SetEdge(CordRep* edge); + + // Returns a partial copy of the current tree containing the first `n` bytes + // of data. `CopyResult` contains both the resulting edge and its height. The + // resulting tree may be less high than the current tree, or even be a single + // matching data edge if `allow_folding` is set to true. + // For example, if `n == 1`, then the result will be the single data edge, and + // height will be set to -1 (one below the owning leaf node). If n == 0, this + // function returns null. Requires `n <= length` + CopyResult CopyPrefix(size_t n, bool allow_folding = true); + + // Returns a partial copy of the current tree containing all data starting + // after `offset`. `CopyResult` contains both the resulting edge and its + // height. The resulting tree may be less high than the current tree, or even + // be a single matching data edge. For example, if `n == length - 1`, then the + // result will be a single data edge, and height will be set to -1 (one below + // the owning leaf node). + // Requires `offset < length` + CopyResult CopySuffix(size_t offset); + + // Returns a OpResult value of {this, kSelf} or {Copy(), kCopied} + // depending on the value of `owned`. + inline OpResult ToOpResult(bool owned); + + // Adds `rep` to the specified tree, returning the modified tree. + template <EdgeType edge_type> + static CordRepBtree* AddCordRep(CordRepBtree* tree, CordRep* rep); + + // Adds `data` to the specified tree, returning the modified tree. + // See the `Append/Prepend` function for the meaning and purpose of `extra`. + template <EdgeType edge_type> + static CordRepBtree* AddData(CordRepBtree* tree, absl::string_view data, + size_t extra = 0); + + // Merges `src` into `dst` with `src` being added either before (kFront) or + // after (kBack) `dst`. Requires the height of `dst` to be greater than or + // equal to the height of `src`. + template <EdgeType edge_type> + static CordRepBtree* Merge(CordRepBtree* dst, CordRepBtree* src); + + // Fallback version of GetAppendBuffer for large trees: GetAppendBuffer() + // implements an inlined version for trees of limited height (3 levels), + // GetAppendBufferSlow implements the logic for large trees. + Span<char> GetAppendBufferSlow(size_t size); + + // `edges_` contains all edges starting from this instance. + // These are explicitly `child` edges only, a cord btree (or any cord tree in + // that respect) does not store `parent` pointers anywhere: multiple trees / + // parents can reference the same shared child edge. The type of these edges + // depends on the height of the node. `Leaf nodes` (height == 0) contain `data + // edges` (external or flat nodes, or sub-strings thereof). All other nodes + // (height > 0) contain pointers to BTREE nodes with a height of `height - 1`. + CordRep* edges_[kMaxCapacity]; + + friend class CordRepBtreeTestPeer; + friend class CordRepBtreeNavigator; +}; + +inline CordRepBtree* CordRep::btree() { + assert(IsBtree()); + return static_cast<CordRepBtree*>(this); +} + +inline const CordRepBtree* CordRep::btree() const { + assert(IsBtree()); + return static_cast<const CordRepBtree*>(this); +} + +inline void CordRepBtree::InitInstance(int height, size_t begin, size_t end) { + tag = BTREE; + storage[0] = static_cast<uint8_t>(height); + storage[1] = static_cast<uint8_t>(begin); + storage[2] = static_cast<uint8_t>(end); +} + +inline CordRep* CordRepBtree::Edge(size_t index) const { + assert(index >= begin()); + assert(index < end()); + return edges_[index]; +} + +inline CordRep* CordRepBtree::Edge(EdgeType edge_type) const { + return edges_[edge_type == kFront ? begin() : back()]; +} + +inline absl::Span<CordRep* const> CordRepBtree::Edges() const { + return {edges_ + begin(), size()}; +} + +inline absl::Span<CordRep* const> CordRepBtree::Edges(size_t begin, + size_t end) const { + assert(begin <= end); + assert(begin >= this->begin()); + assert(end <= this->end()); + return {edges_ + begin, static_cast<size_t>(end - begin)}; +} + +inline absl::string_view CordRepBtree::Data(size_t index) const { + assert(height() == 0); + return EdgeData(Edge(index)); +} + +inline CordRepBtree* CordRepBtree::New(int height) { + CordRepBtree* tree = new CordRepBtree; + tree->length = 0; + tree->InitInstance(height); + return tree; +} + +inline CordRepBtree* CordRepBtree::New(CordRep* rep) { + CordRepBtree* tree = new CordRepBtree; + int height = rep->IsBtree() ? rep->btree()->height() + 1 : 0; + tree->length = rep->length; + tree->InitInstance(height, /*begin=*/0, /*end=*/1); + tree->edges_[0] = rep; + return tree; +} + +inline CordRepBtree* CordRepBtree::New(CordRepBtree* front, + CordRepBtree* back) { + assert(front->height() == back->height()); + CordRepBtree* tree = new CordRepBtree; + tree->length = front->length + back->length; + tree->InitInstance(front->height() + 1, /*begin=*/0, /*end=*/2); + tree->edges_[0] = front; + tree->edges_[1] = back; + return tree; +} + +inline void CordRepBtree::Unref(absl::Span<CordRep* const> edges) { + for (CordRep* edge : edges) { + if (ABSL_PREDICT_FALSE(!edge->refcount.Decrement())) { + CordRep::Destroy(edge); + } + } +} + +inline CordRepBtree* CordRepBtree::CopyRaw() const { + auto* tree = static_cast<CordRepBtree*>(::operator new(sizeof(CordRepBtree))); + memcpy(static_cast<void*>(tree), this, sizeof(CordRepBtree)); + new (&tree->refcount) RefcountAndFlags; + return tree; +} + +inline CordRepBtree* CordRepBtree::Copy() const { + CordRepBtree* tree = CopyRaw(); + for (CordRep* rep : Edges()) CordRep::Ref(rep); + return tree; +} + +inline CordRepBtree* CordRepBtree::CopyToEndFrom(size_t begin, + size_t new_length) const { + assert(begin >= this->begin()); + assert(begin <= this->end()); + CordRepBtree* tree = CopyRaw(); + tree->length = new_length; + tree->set_begin(begin); + for (CordRep* edge : tree->Edges()) CordRep::Ref(edge); + return tree; +} + +inline CordRepBtree* CordRepBtree::CopyBeginTo(size_t end, + size_t new_length) const { + assert(end <= capacity()); + assert(end >= this->begin()); + CordRepBtree* tree = CopyRaw(); + tree->length = new_length; + tree->set_end(end); + for (CordRep* edge : tree->Edges()) CordRep::Ref(edge); + return tree; +} + +inline void CordRepBtree::AlignBegin() { + // The below code itself does not need to be fast as typically we have + // mono-directional append/prepend calls, and `begin` / `end` are typically + // adjusted no more than once. But we want to avoid potential register clobber + // effects, making the compiler emit register save/store/spills, and minimize + // the size of code. + const size_t delta = begin(); + if (ABSL_PREDICT_FALSE(delta != 0)) { + const size_t new_end = end() - delta; + set_begin(0); + set_end(new_end); + // TODO(mvels): we can write this using 2 loads / 2 stores depending on + // total size for the kMaxCapacity = 6 case. I.e., we can branch (switch) on + // size, and then do overlapping load/store of up to 4 pointers (inlined as + // XMM, YMM or ZMM load/store) and up to 2 pointers (XMM / YMM), which is a) + // compact and b) not clobbering any registers. + ABSL_ASSUME(new_end <= kMaxCapacity); +#ifdef __clang__ +#pragma unroll 1 +#endif + for (size_t i = 0; i < new_end; ++i) { + edges_[i] = edges_[i + delta]; + } + } +} + +inline void CordRepBtree::AlignEnd() { + // See comments in `AlignBegin` for motivation on the hand-rolled for loops. + const size_t delta = capacity() - end(); + if (delta != 0) { + const size_t new_begin = begin() + delta; + const size_t new_end = end() + delta; + set_begin(new_begin); + set_end(new_end); + ABSL_ASSUME(new_end <= kMaxCapacity); +#ifdef __clang__ +#pragma unroll 1 +#endif + for (size_t i = new_end - 1; i >= new_begin; --i) { + edges_[i] = edges_[i - delta]; + } + } +} + +template <> +inline void CordRepBtree::Add<CordRepBtree::kBack>(CordRep* rep) { + AlignBegin(); + edges_[fetch_add_end(1)] = rep; +} + +template <> +inline void CordRepBtree::Add<CordRepBtree::kBack>( + absl::Span<CordRep* const> edges) { + AlignBegin(); + size_t new_end = end(); + for (CordRep* edge : edges) edges_[new_end++] = edge; + set_end(new_end); +} + +template <> +inline void CordRepBtree::Add<CordRepBtree::kFront>(CordRep* rep) { + AlignEnd(); + edges_[sub_fetch_begin(1)] = rep; +} + +template <> +inline void CordRepBtree::Add<CordRepBtree::kFront>( + absl::Span<CordRep* const> edges) { + AlignEnd(); + size_t new_begin = begin() - edges.size(); + set_begin(new_begin); + for (CordRep* edge : edges) edges_[new_begin++] = edge; +} + +template <CordRepBtree::EdgeType edge_type> +inline void CordRepBtree::SetEdge(CordRep* edge) { + const int idx = edge_type == kFront ? begin() : back(); + CordRep::Unref(edges_[idx]); + edges_[idx] = edge; +} + +inline CordRepBtree::OpResult CordRepBtree::ToOpResult(bool owned) { + return owned ? OpResult{this, kSelf} : OpResult{Copy(), kCopied}; +} + +inline CordRepBtree::Position CordRepBtree::IndexOf(size_t offset) const { + assert(offset < length); + size_t index = begin(); + while (offset >= edges_[index]->length) offset -= edges_[index++]->length; + return {index, offset}; +} + +inline CordRepBtree::Position CordRepBtree::IndexBefore(size_t offset) const { + assert(offset > 0); + assert(offset <= length); + size_t index = begin(); + while (offset > edges_[index]->length) offset -= edges_[index++]->length; + return {index, offset}; +} + +inline CordRepBtree::Position CordRepBtree::IndexBefore(Position front, + size_t offset) const { + size_t index = front.index; + offset = offset + front.n; + while (offset > edges_[index]->length) offset -= edges_[index++]->length; + return {index, offset}; +} + +inline CordRepBtree::Position CordRepBtree::IndexOfLength(size_t n) const { + assert(n <= length); + size_t index = back(); + size_t strip = length - n; + while (strip >= edges_[index]->length) strip -= edges_[index--]->length; + return {index, edges_[index]->length - strip}; +} + +inline CordRepBtree::Position CordRepBtree::IndexBeyond( + const size_t offset) const { + // We need to find the edge which `starting offset` is beyond (>=)`offset`. + // For this we can't use the `offset -= length` logic of IndexOf. Instead, we + // track the offset of the `current edge` in `off`, which we increase as we + // iterate over the edges until we find the matching edge. + size_t off = 0; + size_t index = begin(); + while (offset > off) off += edges_[index++]->length; + return {index, off - offset}; +} + +inline CordRepBtree* CordRepBtree::Create(CordRep* rep) { + if (IsDataEdge(rep)) return New(rep); + return CreateSlow(rep); +} + +inline Span<char> CordRepBtree::GetAppendBuffer(size_t size) { + assert(refcount.IsOne()); + CordRepBtree* tree = this; + const int height = this->height(); + CordRepBtree* n1 = tree; + CordRepBtree* n2 = tree; + CordRepBtree* n3 = tree; + switch (height) { + case 3: + tree = tree->Edge(kBack)->btree(); + if (!tree->refcount.IsOne()) return {}; + n2 = tree; + ABSL_FALLTHROUGH_INTENDED; + case 2: + tree = tree->Edge(kBack)->btree(); + if (!tree->refcount.IsOne()) return {}; + n1 = tree; + ABSL_FALLTHROUGH_INTENDED; + case 1: + tree = tree->Edge(kBack)->btree(); + if (!tree->refcount.IsOne()) return {}; + ABSL_FALLTHROUGH_INTENDED; + case 0: + CordRep* edge = tree->Edge(kBack); + if (!edge->refcount.IsOne()) return {}; + if (edge->tag < FLAT) return {}; + size_t avail = edge->flat()->Capacity() - edge->length; + if (avail == 0) return {}; + size_t delta = (std::min)(size, avail); + Span<char> span = {edge->flat()->Data() + edge->length, delta}; + edge->length += delta; + switch (height) { + case 3: + n3->length += delta; + ABSL_FALLTHROUGH_INTENDED; + case 2: + n2->length += delta; + ABSL_FALLTHROUGH_INTENDED; + case 1: + n1->length += delta; + ABSL_FALLTHROUGH_INTENDED; + case 0: + tree->length += delta; + return span; + } + break; + } + return GetAppendBufferSlow(size); +} + +extern template CordRepBtree* CordRepBtree::AddCordRep<CordRepBtree::kBack>( + CordRepBtree* tree, CordRep* rep); + +extern template CordRepBtree* CordRepBtree::AddCordRep<CordRepBtree::kFront>( + CordRepBtree* tree, CordRep* rep); + +inline CordRepBtree* CordRepBtree::Append(CordRepBtree* tree, CordRep* rep) { + if (ABSL_PREDICT_TRUE(IsDataEdge(rep))) { + return CordRepBtree::AddCordRep<kBack>(tree, rep); + } + return AppendSlow(tree, rep); +} + +inline CordRepBtree* CordRepBtree::Prepend(CordRepBtree* tree, CordRep* rep) { + if (ABSL_PREDICT_TRUE(IsDataEdge(rep))) { + return CordRepBtree::AddCordRep<kFront>(tree, rep); + } + return PrependSlow(tree, rep); +} + +#ifdef NDEBUG + +inline CordRepBtree* CordRepBtree::AssertValid(CordRepBtree* tree, + bool /* shallow */) { + return tree; +} + +inline const CordRepBtree* CordRepBtree::AssertValid(const CordRepBtree* tree, + bool /* shallow */) { + return tree; +} + +#endif + +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_INTERNAL_CORD_REP_BTREE_H_ diff --git a/absl/strings/internal/cord_rep_btree_navigator.cc b/absl/strings/internal/cord_rep_btree_navigator.cc new file mode 100644 index 00000000..9b896a3d --- /dev/null +++ b/absl/strings/internal/cord_rep_btree_navigator.cc @@ -0,0 +1,187 @@ +// Copyright 2021 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/cord_rep_btree_navigator.h" + +#include <cassert> + +#include "absl/strings/internal/cord_data_edge.h" +#include "absl/strings/internal/cord_internal.h" +#include "absl/strings/internal/cord_rep_btree.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { + +using ReadResult = CordRepBtreeNavigator::ReadResult; + +namespace { + +// Returns a `CordRepSubstring` from `rep` starting at `offset` of size `n`. +// If `rep` is already a `CordRepSubstring` instance, an adjusted instance is +// created based on the old offset and new offset. +// Adopts a reference on `rep`. Rep must be a valid data edge. Returns +// nullptr if `n == 0`, `rep` if `n == rep->length`. +// Requires `offset < rep->length` and `offset + n <= rep->length`. +// TODO(192061034): move to utility library in internal and optimize for small +// substrings of larger reps. +inline CordRep* Substring(CordRep* rep, size_t offset, size_t n) { + assert(n <= rep->length); + assert(offset < rep->length); + assert(offset <= rep->length - n); + assert(IsDataEdge(rep)); + + if (n == 0) return nullptr; + if (n == rep->length) return CordRep::Ref(rep); + + if (rep->tag == SUBSTRING) { + offset += rep->substring()->start; + rep = rep->substring()->child; + } + + assert(rep->IsExternal() || rep->IsFlat()); + CordRepSubstring* substring = new CordRepSubstring(); + substring->length = n; + substring->tag = SUBSTRING; + substring->start = offset; + substring->child = CordRep::Ref(rep); + return substring; +} + +inline CordRep* Substring(CordRep* rep, size_t offset) { + return Substring(rep, offset, rep->length - offset); +} + +} // namespace + +CordRepBtreeNavigator::Position CordRepBtreeNavigator::Skip(size_t n) { + int height = 0; + size_t index = index_[0]; + CordRepBtree* node = node_[0]; + CordRep* edge = node->Edge(index); + + // Overall logic: Find an edge of at least the length we need to skip. + // We consume all edges which are smaller (i.e., must be 100% skipped). + // If we exhausted all edges on the current level, we move one level + // up the tree, and repeat until we either find the edge, or until we hit + // the top of the tree meaning the skip exceeds tree->length. + while (n >= edge->length) { + n -= edge->length; + while (++index == node->end()) { + if (++height > height_) return {nullptr, n}; + node = node_[height]; + index = index_[height]; + } + edge = node->Edge(index); + } + + // If we moved up the tree, descend down to the leaf level, consuming all + // edges that must be skipped. + while (height > 0) { + node = edge->btree(); + index_[height] = index; + node_[--height] = node; + index = node->begin(); + edge = node->Edge(index); + while (n >= edge->length) { + n -= edge->length; + ++index; + assert(index != node->end()); + edge = node->Edge(index); + } + } + index_[0] = index; + return {edge, n}; +} + +ReadResult CordRepBtreeNavigator::Read(size_t edge_offset, size_t n) { + int height = 0; + size_t length = edge_offset + n; + size_t index = index_[0]; + CordRepBtree* node = node_[0]; + CordRep* edge = node->Edge(index); + assert(edge_offset < edge->length); + + if (length < edge->length) { + return {Substring(edge, edge_offset, n), length}; + } + + // Similar to 'Skip', we consume all edges that are inside the 'length' of + // data that needs to be read. If we exhaust the current level, we move one + // level up the tree and repeat until we hit the final edge that must be + // (partially) read. We consume all edges into `subtree`. + CordRepBtree* subtree = CordRepBtree::New(Substring(edge, edge_offset)); + size_t subtree_end = 1; + do { + length -= edge->length; + while (++index == node->end()) { + index_[height] = index; + if (++height > height_) { + subtree->set_end(subtree_end); + if (length == 0) return {subtree, 0}; + CordRep::Unref(subtree); + return {nullptr, length}; + } + if (length != 0) { + subtree->set_end(subtree_end); + subtree = CordRepBtree::New(subtree); + subtree_end = 1; + } + node = node_[height]; + index = index_[height]; + } + edge = node->Edge(index); + if (length >= edge->length) { + subtree->length += edge->length; + subtree->edges_[subtree_end++] = CordRep::Ref(edge); + } + } while (length >= edge->length); + CordRepBtree* tree = subtree; + subtree->length += length; + + // If we moved up the tree, descend down to the leaf level, consuming all + // edges that must be read, adding 'down' nodes to `subtree`. + while (height > 0) { + node = edge->btree(); + index_[height] = index; + node_[--height] = node; + index = node->begin(); + edge = node->Edge(index); + + if (length != 0) { + CordRepBtree* right = CordRepBtree::New(height); + right->length = length; + subtree->edges_[subtree_end++] = right; + subtree->set_end(subtree_end); + subtree = right; + subtree_end = 0; + while (length >= edge->length) { + subtree->edges_[subtree_end++] = CordRep::Ref(edge); + length -= edge->length; + edge = node->Edge(++index); + } + } + } + // Add any (partial) edge still remaining at the leaf level. + if (length != 0) { + subtree->edges_[subtree_end++] = Substring(edge, 0, length); + } + subtree->set_end(subtree_end); + index_[0] = index; + return {tree, length}; +} + +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/absl/strings/internal/cord_rep_btree_navigator.h b/absl/strings/internal/cord_rep_btree_navigator.h new file mode 100644 index 00000000..3d581c87 --- /dev/null +++ b/absl/strings/internal/cord_rep_btree_navigator.h @@ -0,0 +1,267 @@ +// Copyright 2021 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_STRINGS_INTERNAL_CORD_REP_BTREE_NAVIGATOR_H_ +#define ABSL_STRINGS_INTERNAL_CORD_REP_BTREE_NAVIGATOR_H_ + +#include <cassert> +#include <iostream> + +#include "absl/strings/internal/cord_internal.h" +#include "absl/strings/internal/cord_rep_btree.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { + +// CordRepBtreeNavigator is a bi-directional navigator allowing callers to +// navigate all the (leaf) data edges in a CordRepBtree instance. +// +// A CordRepBtreeNavigator instance is by default empty. Callers initialize a +// navigator instance by calling one of `InitFirst()`, `InitLast()` or +// `InitOffset()`, which establishes a current position. Callers can then +// navigate using the `Next`, `Previous`, `Skip` and `Seek` methods. +// +// The navigator instance does not take or adopt a reference on the provided +// `tree` on any of the initialization calls. Callers are responsible for +// guaranteeing the lifecycle of the provided tree. A navigator instance can +// be reset to the empty state by calling `Reset`. +// +// A navigator only keeps positional state on the 'current data edge', it does +// explicitly not keep any 'offset' state. The class does accept and return +// offsets in the `Read()`, `Skip()` and 'Seek()` methods as these would +// otherwise put a big burden on callers. Callers are expected to maintain +// (returned) offset info if they require such granular state. +class CordRepBtreeNavigator { + public: + // The logical position as returned by the Seek() and Skip() functions. + // Returns the current leaf edge for the desired seek or skip position and + // the offset of that position inside that edge. + struct Position { + CordRep* edge; + size_t offset; + }; + + // The read result as returned by the Read() function. + // `tree` contains the resulting tree which is identical to the result + // of calling CordRepBtree::SubTree(...) on the tree being navigated. + // `n` contains the number of bytes used from the last navigated to + // edge of the tree. + struct ReadResult { + CordRep* tree; + size_t n; + }; + + // Returns true if this instance is not empty. + explicit operator bool() const; + + // Returns the tree for this instance or nullptr if empty. + CordRepBtree* btree() const; + + // Returns the data edge of the current position. + // Requires this instance to not be empty. + CordRep* Current() const; + + // Resets this navigator to `tree`, returning the first data edge in the tree. + CordRep* InitFirst(CordRepBtree* tree); + + // Resets this navigator to `tree`, returning the last data edge in the tree. + CordRep* InitLast(CordRepBtree* tree); + + // Resets this navigator to `tree` returning the data edge at position + // `offset` and the relative offset of `offset` into that data edge. + // Returns `Position.edge = nullptr` if the provided offset is greater + // than or equal to the length of the tree, in which case the state of + // the navigator instance remains unchanged. + Position InitOffset(CordRepBtree* tree, size_t offset); + + // Navigates to the next data edge. + // Returns the next data edge or nullptr if there is no next data edge, in + // which case the current position remains unchanged. + CordRep* Next(); + + // Navigates to the previous data edge. + // Returns the previous data edge or nullptr if there is no previous data + // edge, in which case the current position remains unchanged. + CordRep* Previous(); + + // Navigates to the data edge at position `offset`. Returns the navigated to + // data edge in `Position.edge` and the relative offset of `offset` into that + // data edge in `Position.offset`. Returns `Position.edge = nullptr` if the + // provide offset is greater than or equal to the tree's length. + Position Seek(size_t offset); + + // Reads `n` bytes of data starting at offset `edge_offset` of the current + // data edge, and returns the result in `ReadResult.tree`. `ReadResult.n` + // contains the 'bytes used` from the last / current data edge in the tree. + // This allows users that mix regular navigation (using string views) and + // 'read into cord' navigation to keep track of the current state, and which + // bytes have been consumed from a navigator. + // This function returns `ReadResult.tree = nullptr` if the requested length + // exceeds the length of the tree starting at the current data edge. + ReadResult Read(size_t edge_offset, size_t n); + + // Skips `n` bytes forward from the current data edge, returning the navigated + // to data edge in `Position.edge` and `Position.offset` containing the offset + // inside that data edge. Note that the state of the navigator is left + // unchanged if `n` is smaller than the length of the current data edge. + Position Skip(size_t n); + + // Resets this instance to the default / empty state. + void Reset(); + + private: + // Slow path for Next() if Next() reached the end of a leaf node. Backtracks + // up the stack until it finds a node that has a 'next' position available, + // and then does a 'front dive' towards the next leaf node. + CordRep* NextUp(); + + // Slow path for Previous() if Previous() reached the beginning of a leaf + // node. Backtracks up the stack until it finds a node that has a 'previous' + // position available, and then does a 'back dive' towards the previous leaf + // node. + CordRep* PreviousUp(); + + // Generic implementation of InitFirst() and InitLast(). + template <CordRepBtree::EdgeType edge_type> + CordRep* Init(CordRepBtree* tree); + + // `height_` contains the height of the current tree, or -1 if empty. + int height_ = -1; + + // `index_` and `node_` contain the navigation state as the 'path' to the + // current data edge which is at `node_[0]->Edge(index_[0])`. The contents + // of these are undefined until the instance is initialized (`height_ >= 0`). + uint8_t index_[CordRepBtree::kMaxDepth]; + CordRepBtree* node_[CordRepBtree::kMaxDepth]; +}; + +// Returns true if this instance is not empty. +inline CordRepBtreeNavigator::operator bool() const { return height_ >= 0; } + +inline CordRepBtree* CordRepBtreeNavigator::btree() const { + return height_ >= 0 ? node_[height_] : nullptr; +} + +inline CordRep* CordRepBtreeNavigator::Current() const { + assert(height_ >= 0); + return node_[0]->Edge(index_[0]); +} + +inline void CordRepBtreeNavigator::Reset() { height_ = -1; } + +inline CordRep* CordRepBtreeNavigator::InitFirst(CordRepBtree* tree) { + return Init<CordRepBtree::kFront>(tree); +} + +inline CordRep* CordRepBtreeNavigator::InitLast(CordRepBtree* tree) { + return Init<CordRepBtree::kBack>(tree); +} + +template <CordRepBtree::EdgeType edge_type> +inline CordRep* CordRepBtreeNavigator::Init(CordRepBtree* tree) { + assert(tree != nullptr); + assert(tree->size() > 0); + assert(tree->height() <= CordRepBtree::kMaxHeight); + int height = height_ = tree->height(); + size_t index = tree->index(edge_type); + node_[height] = tree; + index_[height] = static_cast<uint8_t>(index); + while (--height >= 0) { + tree = tree->Edge(index)->btree(); + node_[height] = tree; + index = tree->index(edge_type); + index_[height] = static_cast<uint8_t>(index); + } + return node_[0]->Edge(index); +} + +inline CordRepBtreeNavigator::Position CordRepBtreeNavigator::Seek( + size_t offset) { + assert(btree() != nullptr); + int height = height_; + CordRepBtree* edge = node_[height]; + if (ABSL_PREDICT_FALSE(offset >= edge->length)) return {nullptr, 0}; + CordRepBtree::Position index = edge->IndexOf(offset); + index_[height] = static_cast<uint8_t>(index.index); + while (--height >= 0) { + edge = edge->Edge(index.index)->btree(); + node_[height] = edge; + index = edge->IndexOf(index.n); + index_[height] = static_cast<uint8_t>(index.index); + } + return {edge->Edge(index.index), index.n}; +} + +inline CordRepBtreeNavigator::Position CordRepBtreeNavigator::InitOffset( + CordRepBtree* tree, size_t offset) { + assert(tree != nullptr); + assert(tree->height() <= CordRepBtree::kMaxHeight); + if (ABSL_PREDICT_FALSE(offset >= tree->length)) return {nullptr, 0}; + height_ = tree->height(); + node_[height_] = tree; + return Seek(offset); +} + +inline CordRep* CordRepBtreeNavigator::Next() { + CordRepBtree* edge = node_[0]; + return index_[0] == edge->back() ? NextUp() : edge->Edge(++index_[0]); +} + +inline CordRep* CordRepBtreeNavigator::Previous() { + CordRepBtree* edge = node_[0]; + return index_[0] == edge->begin() ? PreviousUp() : edge->Edge(--index_[0]); +} + +inline CordRep* CordRepBtreeNavigator::NextUp() { + assert(index_[0] == node_[0]->back()); + CordRepBtree* edge; + size_t index; + int height = 0; + do { + if (++height > height_) return nullptr; + edge = node_[height]; + index = index_[height] + 1; + } while (index == edge->end()); + index_[height] = static_cast<uint8_t>(index); + do { + node_[--height] = edge = edge->Edge(index)->btree(); + index_[height] = static_cast<uint8_t>(index = edge->begin()); + } while (height > 0); + return edge->Edge(index); +} + +inline CordRep* CordRepBtreeNavigator::PreviousUp() { + assert(index_[0] == node_[0]->begin()); + CordRepBtree* edge; + size_t index; + int height = 0; + do { + if (++height > height_) return nullptr; + edge = node_[height]; + index = index_[height]; + } while (index == edge->begin()); + index_[height] = static_cast<uint8_t>(--index); + do { + node_[--height] = edge = edge->Edge(index)->btree(); + index_[height] = static_cast<uint8_t>(index = edge->back()); + } while (height > 0); + return edge->Edge(index); +} + +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_INTERNAL_CORD_REP_BTREE_NAVIGATOR_H_ diff --git a/absl/strings/internal/cord_rep_btree_navigator_test.cc b/absl/strings/internal/cord_rep_btree_navigator_test.cc new file mode 100644 index 00000000..4f9bd4e5 --- /dev/null +++ b/absl/strings/internal/cord_rep_btree_navigator_test.cc @@ -0,0 +1,346 @@ +// Copyright 2021 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/cord_rep_btree_navigator.h" + +#include <string> +#include <vector> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/config.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/strings/internal/cord_internal.h" +#include "absl/strings/internal/cord_rep_btree.h" +#include "absl/strings/internal/cord_rep_test_util.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { +namespace { + +using ::testing::Eq; +using ::testing::Ne; + +using ::absl::cordrep_testing::CordRepBtreeFromFlats; +using ::absl::cordrep_testing::CordToString; +using ::absl::cordrep_testing::CreateFlatsFromString; +using ::absl::cordrep_testing::CreateRandomString; +using ::absl::cordrep_testing::MakeFlat; +using ::absl::cordrep_testing::MakeSubstring; + +using ReadResult = CordRepBtreeNavigator::ReadResult; +using Position = CordRepBtreeNavigator::Position; + +// CordRepBtreeNavigatorTest is a test fixture which automatically creates a +// tree to test navigation logic on. The parameter `count' defines the number of +// data edges in the test tree. +class CordRepBtreeNavigatorTest : public testing::TestWithParam<int> { + public: + using Flats = std::vector<CordRep*>; + static constexpr size_t kCharsPerFlat = 3; + + CordRepBtreeNavigatorTest() { + data_ = CreateRandomString(count() * kCharsPerFlat); + flats_ = CreateFlatsFromString(data_, kCharsPerFlat); + + // Turn flat 0 or 1 into a substring to cover partial reads on substrings. + if (count() > 1) { + CordRep::Unref(flats_[1]); + flats_[1] = MakeSubstring(kCharsPerFlat, kCharsPerFlat, MakeFlat(data_)); + } else { + CordRep::Unref(flats_[0]); + flats_[0] = MakeSubstring(0, kCharsPerFlat, MakeFlat(data_)); + } + + tree_ = CordRepBtreeFromFlats(flats_); + } + + ~CordRepBtreeNavigatorTest() override { CordRep::Unref(tree_); } + + int count() const { return GetParam(); } + CordRepBtree* tree() { return tree_; } + const std::string& data() const { return data_; } + const std::vector<CordRep*>& flats() const { return flats_; } + + static std::string ToString(testing::TestParamInfo<int> param) { + return absl::StrCat(param.param, "_Flats"); + } + + private: + std::string data_; + Flats flats_; + CordRepBtree* tree_; +}; + +INSTANTIATE_TEST_SUITE_P( + WithParam, CordRepBtreeNavigatorTest, + testing::Values(1, CordRepBtree::kMaxCapacity - 1, + CordRepBtree::kMaxCapacity, + CordRepBtree::kMaxCapacity* CordRepBtree::kMaxCapacity - 1, + CordRepBtree::kMaxCapacity* CordRepBtree::kMaxCapacity, + CordRepBtree::kMaxCapacity* CordRepBtree::kMaxCapacity + 1, + CordRepBtree::kMaxCapacity* CordRepBtree::kMaxCapacity * 2 + + 17), + CordRepBtreeNavigatorTest::ToString); + +TEST(CordRepBtreeNavigatorTest, Uninitialized) { + CordRepBtreeNavigator nav; + EXPECT_FALSE(nav); + EXPECT_THAT(nav.btree(), Eq(nullptr)); +#if defined(GTEST_HAS_DEATH_TEST) && !defined(NDEBUG) + EXPECT_DEATH(nav.Current(), ".*"); +#endif +} + +TEST_P(CordRepBtreeNavigatorTest, InitFirst) { + CordRepBtreeNavigator nav; + CordRep* edge = nav.InitFirst(tree()); + EXPECT_TRUE(nav); + EXPECT_THAT(nav.btree(), Eq(tree())); + EXPECT_THAT(nav.Current(), Eq(flats().front())); + EXPECT_THAT(edge, Eq(flats().front())); +} + +TEST_P(CordRepBtreeNavigatorTest, InitLast) { + CordRepBtreeNavigator nav; + CordRep* edge = nav.InitLast(tree()); + EXPECT_TRUE(nav); + EXPECT_THAT(nav.btree(), Eq(tree())); + EXPECT_THAT(nav.Current(), Eq(flats().back())); + EXPECT_THAT(edge, Eq(flats().back())); +} + +TEST_P(CordRepBtreeNavigatorTest, NextPrev) { + CordRepBtreeNavigator nav; + nav.InitFirst(tree()); + const Flats& flats = this->flats(); + + EXPECT_THAT(nav.Previous(), Eq(nullptr)); + EXPECT_THAT(nav.Current(), Eq(flats.front())); + for (int i = 1; i < flats.size(); ++i) { + ASSERT_THAT(nav.Next(), Eq(flats[i])); + EXPECT_THAT(nav.Current(), Eq(flats[i])); + } + EXPECT_THAT(nav.Next(), Eq(nullptr)); + EXPECT_THAT(nav.Current(), Eq(flats.back())); + for (int i = static_cast<int>(flats.size()) - 2; i >= 0; --i) { + ASSERT_THAT(nav.Previous(), Eq(flats[i])); + EXPECT_THAT(nav.Current(), Eq(flats[i])); + } + EXPECT_THAT(nav.Previous(), Eq(nullptr)); + EXPECT_THAT(nav.Current(), Eq(flats.front())); +} + +TEST_P(CordRepBtreeNavigatorTest, PrevNext) { + CordRepBtreeNavigator nav; + nav.InitLast(tree()); + const Flats& flats = this->flats(); + + EXPECT_THAT(nav.Next(), Eq(nullptr)); + EXPECT_THAT(nav.Current(), Eq(flats.back())); + for (int i = static_cast<int>(flats.size()) - 2; i >= 0; --i) { + ASSERT_THAT(nav.Previous(), Eq(flats[i])); + EXPECT_THAT(nav.Current(), Eq(flats[i])); + } + EXPECT_THAT(nav.Previous(), Eq(nullptr)); + EXPECT_THAT(nav.Current(), Eq(flats.front())); + for (int i = 1; i < flats.size(); ++i) { + ASSERT_THAT(nav.Next(), Eq(flats[i])); + EXPECT_THAT(nav.Current(), Eq(flats[i])); + } + EXPECT_THAT(nav.Next(), Eq(nullptr)); + EXPECT_THAT(nav.Current(), Eq(flats.back())); +} + +TEST(CordRepBtreeNavigatorTest, Reset) { + CordRepBtree* tree = CordRepBtree::Create(MakeFlat("abc")); + CordRepBtreeNavigator nav; + nav.InitFirst(tree); + nav.Reset(); + EXPECT_FALSE(nav); + EXPECT_THAT(nav.btree(), Eq(nullptr)); +#if defined(GTEST_HAS_DEATH_TEST) && !defined(NDEBUG) + EXPECT_DEATH(nav.Current(), ".*"); +#endif + CordRep::Unref(tree); +} + +TEST_P(CordRepBtreeNavigatorTest, Skip) { + int count = this->count(); + const Flats& flats = this->flats(); + CordRepBtreeNavigator nav; + nav.InitFirst(tree()); + + for (int char_offset = 0; char_offset < kCharsPerFlat; ++char_offset) { + Position pos = nav.Skip(char_offset); + EXPECT_THAT(pos.edge, Eq(nav.Current())); + EXPECT_THAT(pos.edge, Eq(flats[0])); + EXPECT_THAT(pos.offset, Eq(char_offset)); + } + + for (int index1 = 0; index1 < count; ++index1) { + for (int index2 = index1; index2 < count; ++index2) { + for (int char_offset = 0; char_offset < kCharsPerFlat; ++char_offset) { + CordRepBtreeNavigator nav; + nav.InitFirst(tree()); + + size_t length1 = index1 * kCharsPerFlat; + Position pos1 = nav.Skip(length1 + char_offset); + ASSERT_THAT(pos1.edge, Eq(flats[index1])); + ASSERT_THAT(pos1.edge, Eq(nav.Current())); + ASSERT_THAT(pos1.offset, Eq(char_offset)); + + size_t length2 = index2 * kCharsPerFlat; + Position pos2 = nav.Skip(length2 - length1 + char_offset); + ASSERT_THAT(pos2.edge, Eq(flats[index2])); + ASSERT_THAT(pos2.edge, Eq(nav.Current())); + ASSERT_THAT(pos2.offset, Eq(char_offset)); + } + } + } +} + +TEST_P(CordRepBtreeNavigatorTest, Seek) { + int count = this->count(); + const Flats& flats = this->flats(); + CordRepBtreeNavigator nav; + nav.InitFirst(tree()); + + for (int char_offset = 0; char_offset < kCharsPerFlat; ++char_offset) { + Position pos = nav.Seek(char_offset); + EXPECT_THAT(pos.edge, Eq(nav.Current())); + EXPECT_THAT(pos.edge, Eq(flats[0])); + EXPECT_THAT(pos.offset, Eq(char_offset)); + } + + for (int index = 0; index < count; ++index) { + for (int char_offset = 0; char_offset < kCharsPerFlat; ++char_offset) { + size_t offset = index * kCharsPerFlat + char_offset; + Position pos1 = nav.Seek(offset); + ASSERT_THAT(pos1.edge, Eq(flats[index])); + ASSERT_THAT(pos1.edge, Eq(nav.Current())); + ASSERT_THAT(pos1.offset, Eq(char_offset)); + } + } +} + +TEST(CordRepBtreeNavigatorTest, InitOffset) { + // Whitebox: InitOffset() is implemented in terms of Seek() which is + // exhaustively tested. Only test it initializes / forwards properly.. + CordRepBtree* tree = CordRepBtree::Create(MakeFlat("abc")); + tree = CordRepBtree::Append(tree, MakeFlat("def")); + CordRepBtreeNavigator nav; + Position pos = nav.InitOffset(tree, 5); + EXPECT_TRUE(nav); + EXPECT_THAT(nav.btree(), Eq(tree)); + EXPECT_THAT(pos.edge, Eq(tree->Edges()[1])); + EXPECT_THAT(pos.edge, Eq(nav.Current())); + EXPECT_THAT(pos.offset, Eq(2)); + CordRep::Unref(tree); +} + +TEST(CordRepBtreeNavigatorTest, InitOffsetAndSeekBeyondLength) { + CordRepBtree* tree1 = CordRepBtree::Create(MakeFlat("abc")); + CordRepBtree* tree2 = CordRepBtree::Create(MakeFlat("def")); + + CordRepBtreeNavigator nav; + nav.InitFirst(tree1); + EXPECT_THAT(nav.Seek(3).edge, Eq(nullptr)); + EXPECT_THAT(nav.Seek(100).edge, Eq(nullptr)); + EXPECT_THAT(nav.btree(), Eq(tree1)); + EXPECT_THAT(nav.Current(), Eq(tree1->Edges().front())); + + EXPECT_THAT(nav.InitOffset(tree2, 3).edge, Eq(nullptr)); + EXPECT_THAT(nav.InitOffset(tree2, 100).edge, Eq(nullptr)); + EXPECT_THAT(nav.btree(), Eq(tree1)); + EXPECT_THAT(nav.Current(), Eq(tree1->Edges().front())); + + CordRep::Unref(tree1); + CordRep::Unref(tree2); +} + +TEST_P(CordRepBtreeNavigatorTest, Read) { + const Flats& flats = this->flats(); + const std::string& data = this->data(); + + for (size_t offset = 0; offset < data.size(); ++offset) { + for (size_t length = 1; length <= data.size() - offset; ++length) { + CordRepBtreeNavigator nav; + nav.InitFirst(tree()); + + // Skip towards edge holding offset + size_t edge_offset = nav.Skip(offset).offset; + + // Read node + ReadResult result = nav.Read(edge_offset, length); + ASSERT_THAT(result.tree, Ne(nullptr)); + EXPECT_THAT(result.tree->length, Eq(length)); + if (result.tree->tag == BTREE) { + ASSERT_TRUE(CordRepBtree::IsValid(result.tree->btree())); + } + + // Verify contents + std::string value = CordToString(result.tree); + EXPECT_THAT(value, Eq(data.substr(offset, length))); + + // Verify 'partial last edge' reads. + size_t partial = (offset + length) % kCharsPerFlat; + ASSERT_THAT(result.n, Eq(partial)); + + // Verify ending position if not EOF + if (offset + length < data.size()) { + size_t index = (offset + length) / kCharsPerFlat; + EXPECT_THAT(nav.Current(), Eq(flats[index])); + } + + CordRep::Unref(result.tree); + } + } +} + +TEST_P(CordRepBtreeNavigatorTest, ReadBeyondLengthOfTree) { + CordRepBtreeNavigator nav; + nav.InitFirst(tree()); + ReadResult result = nav.Read(2, tree()->length); + ASSERT_THAT(result.tree, Eq(nullptr)); +} + +TEST(CordRepBtreeNavigatorTest, NavigateMaximumTreeDepth) { + CordRepFlat* flat1 = MakeFlat("Hello world"); + CordRepFlat* flat2 = MakeFlat("World Hello"); + + CordRepBtree* node = CordRepBtree::Create(flat1); + node = CordRepBtree::Append(node, flat2); + while (node->height() < CordRepBtree::kMaxHeight) { + node = CordRepBtree::New(node); + } + + CordRepBtreeNavigator nav; + CordRep* edge = nav.InitFirst(node); + EXPECT_THAT(edge, Eq(flat1)); + EXPECT_THAT(nav.Next(), Eq(flat2)); + EXPECT_THAT(nav.Next(), Eq(nullptr)); + EXPECT_THAT(nav.Previous(), Eq(flat1)); + EXPECT_THAT(nav.Previous(), Eq(nullptr)); + + CordRep::Unref(node); +} + +} // namespace +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/absl/strings/internal/cord_rep_btree_reader.cc b/absl/strings/internal/cord_rep_btree_reader.cc new file mode 100644 index 00000000..0d0e8601 --- /dev/null +++ b/absl/strings/internal/cord_rep_btree_reader.cc @@ -0,0 +1,69 @@ +// Copyright 2021 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/cord_rep_btree_reader.h" + +#include <cassert> + +#include "absl/base/config.h" +#include "absl/strings/internal/cord_data_edge.h" +#include "absl/strings/internal/cord_internal.h" +#include "absl/strings/internal/cord_rep_btree.h" +#include "absl/strings/internal/cord_rep_btree_navigator.h" +#include "absl/strings/internal/cord_rep_flat.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { + +absl::string_view CordRepBtreeReader::Read(size_t n, size_t chunk_size, + CordRep*& tree) { + assert(chunk_size <= navigator_.Current()->length); + + // If chunk_size is non-zero, we need to start inside last returned edge. + // Else we start reading at the next data edge of the tree. + CordRep* edge = chunk_size ? navigator_.Current() : navigator_.Next(); + const size_t offset = chunk_size ? edge->length - chunk_size : 0; + + // Read the sub tree and verify we got what we wanted. + ReadResult result = navigator_.Read(offset, n); + tree = result.tree; + + // If the data returned in `tree` was covered entirely by `chunk_size`, i.e., + // read from the 'previous' edge, we did not consume any additional data, and + // can directly return the substring into the current data edge as the next + // chunk. We can easily establish from the above code that `navigator_.Next()` + // has not been called as that requires `chunk_size` to be zero. + if (n < chunk_size) return EdgeData(edge).substr(result.n); + + // The amount of data taken from the last edge is `chunk_size` and `result.n` + // contains the offset into the current edge trailing the read data (which can + // be 0). As the call to `navigator_.Read()` could have consumed all remaining + // data, calling `navigator_.Current()` is not safe before checking if we + // already consumed all remaining data. + const size_t consumed_by_read = n - chunk_size - result.n; + if (consumed_by_read >= remaining_) { + remaining_ = 0; + return {}; + } + + // We did not read all data, return remaining data from current edge. + edge = navigator_.Current(); + remaining_ -= consumed_by_read + edge->length; + return EdgeData(edge).substr(result.n); +} + +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/absl/strings/internal/cord_rep_btree_reader.h b/absl/strings/internal/cord_rep_btree_reader.h new file mode 100644 index 00000000..8db8f8dd --- /dev/null +++ b/absl/strings/internal/cord_rep_btree_reader.h @@ -0,0 +1,212 @@ +// Copyright 2021 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_STRINGS_INTERNAL_CORD_REP_BTREE_READER_H_ +#define ABSL_STRINGS_INTERNAL_CORD_REP_BTREE_READER_H_ + +#include <cassert> + +#include "absl/base/config.h" +#include "absl/strings/internal/cord_data_edge.h" +#include "absl/strings/internal/cord_internal.h" +#include "absl/strings/internal/cord_rep_btree.h" +#include "absl/strings/internal/cord_rep_btree_navigator.h" +#include "absl/strings/internal/cord_rep_flat.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { + +// CordRepBtreeReader implements logic to iterate over cord btrees. +// References to the underlying data are returned as absl::string_view values. +// The most typical use case is a forward only iteration over tree data. +// The class also provides `Skip()`, `Seek()` and `Read()` methods similar to +// CordRepBtreeNavigator that allow more advanced navigation. +// +// Example: iterate over all data inside a cord btree: +// +// CordRepBtreeReader reader; +// for (string_view sv = reader.Init(tree); !sv.Empty(); sv = sv.Next()) { +// DoSomethingWithDataIn(sv); +// } +// +// All navigation methods always return the next 'chunk' of data. The class +// assumes that all data is directly 'consumed' by the caller. For example: +// invoking `Skip()` will skip the desired number of bytes, and directly +// read and return the next chunk of data directly after the skipped bytes. +// +// Example: iterate over all data inside a btree skipping the first 100 bytes: +// +// CordRepBtreeReader reader; +// absl::string_view sv = reader.Init(tree); +// if (sv.length() > 100) { +// sv.RemovePrefix(100); +// } else { +// sv = reader.Skip(100 - sv.length()); +// } +// while (!sv.empty()) { +// DoSomethingWithDataIn(sv); +// absl::string_view sv = reader.Next(); +// } +// +// It is important to notice that `remaining` is based on the end position of +// the last data edge returned to the caller, not the cumulative data returned +// to the caller which can be less in cases of skipping or seeking over data. +// +// For example, consider a cord btree with five data edges: "abc", "def", "ghi", +// "jkl" and "mno": +// +// absl::string_view sv; +// CordRepBtreeReader reader; +// +// sv = reader.Init(tree); // sv = "abc", remaining = 12 +// sv = reader.Skip(4); // sv = "hi", remaining = 6 +// sv = reader.Skip(2); // sv = "l", remaining = 3 +// sv = reader.Next(); // sv = "mno", remaining = 0 +// sv = reader.Seek(1); // sv = "bc", remaining = 12 +// +class CordRepBtreeReader { + public: + using ReadResult = CordRepBtreeNavigator::ReadResult; + using Position = CordRepBtreeNavigator::Position; + + // Returns true if this instance is not empty. + explicit operator bool() const { return navigator_.btree() != nullptr; } + + // Returns the tree referenced by this instance or nullptr if empty. + CordRepBtree* btree() const { return navigator_.btree(); } + + // Returns the current data edge inside the referenced btree. + // Requires that the current instance is not empty. + CordRep* node() const { return navigator_.Current(); } + + // Returns the length of the referenced tree. + // Requires that the current instance is not empty. + size_t length() const; + + // Returns the number of remaining bytes available for iteration, which is the + // number of bytes directly following the end of the last chunk returned. + // This value will be zero if we iterated over the last edge in the bound + // tree, in which case any call to Next() or Skip() will return an empty + // string_view reflecting the EOF state. + // Note that a call to `Seek()` resets `remaining` to a value based on the + // end position of the chunk returned by that call. + size_t remaining() const { return remaining_; } + + // Resets this instance to an empty value. + void Reset() { navigator_.Reset(); } + + // Initializes this instance with `tree`. `tree` must not be null. + // Returns a reference to the first data edge of the provided tree. + absl::string_view Init(CordRepBtree* tree); + + // Navigates to and returns the next data edge of the referenced tree. + // Returns an empty string_view if an attempt is made to read beyond the end + // of the tree, i.e.: if `remaining()` is zero indicating an EOF condition. + // Requires that the current instance is not empty. + absl::string_view Next(); + + // Skips the provided amount of bytes and returns a reference to the data + // directly following the skipped bytes. + absl::string_view Skip(size_t skip); + + // Reads `n` bytes into `tree`. + // If `chunk_size` is zero, starts reading at the next data edge. If + // `chunk_size` is non zero, the read starts at the last `chunk_size` bytes of + // the last returned data edge. Effectively, this means that the read starts + // at offset `consumed() - chunk_size`. + // Requires that `chunk_size` is less than or equal to the length of the + // last returned data edge. The purpose of `chunk_size` is to simplify code + // partially consuming a returned chunk and wanting to include the remaining + // bytes in the Read call. For example, the below code will read 1000 bytes of + // data into a cord tree if the first chunk starts with "big:": + // + // CordRepBtreeReader reader; + // absl::string_view sv = reader.Init(tree); + // if (absl::StartsWith(sv, "big:")) { + // CordRepBtree tree; + // sv = reader.Read(1000, sv.size() - 4 /* "big:" */, &tree); + // } + // + // This method will return an empty string view if all remaining data was + // read. If `n` exceeded the amount of remaining data this function will + // return an empty string view and `tree` will be set to nullptr. + // In both cases, `consumed` will be set to `length`. + absl::string_view Read(size_t n, size_t chunk_size, CordRep*& tree); + + // Navigates to the chunk at offset `offset`. + // Returns a reference into the navigated to chunk, adjusted for the relative + // position of `offset` into that chunk. For example, calling `Seek(13)` on a + // cord tree containing 2 chunks of 10 and 20 bytes respectively will return + // a string view into the second chunk starting at offset 3 with a size of 17. + // Returns an empty string view if `offset` is equal to or greater than the + // length of the referenced tree. + absl::string_view Seek(size_t offset); + + private: + size_t remaining_ = 0; + CordRepBtreeNavigator navigator_; +}; + +inline size_t CordRepBtreeReader::length() const { + assert(btree() != nullptr); + return btree()->length; +} + +inline absl::string_view CordRepBtreeReader::Init(CordRepBtree* tree) { + assert(tree != nullptr); + const CordRep* edge = navigator_.InitFirst(tree); + remaining_ = tree->length - edge->length; + return EdgeData(edge); +} + +inline absl::string_view CordRepBtreeReader::Next() { + if (remaining_ == 0) return {}; + const CordRep* edge = navigator_.Next(); + assert(edge != nullptr); + remaining_ -= edge->length; + return EdgeData(edge); +} + +inline absl::string_view CordRepBtreeReader::Skip(size_t skip) { + // As we are always positioned on the last 'consumed' edge, we + // need to skip the current edge as well as `skip`. + const size_t edge_length = navigator_.Current()->length; + CordRepBtreeNavigator::Position pos = navigator_.Skip(skip + edge_length); + if (ABSL_PREDICT_FALSE(pos.edge == nullptr)) { + remaining_ = 0; + return {}; + } + // The combined length of all edges skipped before `pos.edge` is `skip - + // pos.offset`, all of which are 'consumed', as well as the current edge. + remaining_ -= skip - pos.offset + pos.edge->length; + return EdgeData(pos.edge).substr(pos.offset); +} + +inline absl::string_view CordRepBtreeReader::Seek(size_t offset) { + const CordRepBtreeNavigator::Position pos = navigator_.Seek(offset); + if (ABSL_PREDICT_FALSE(pos.edge == nullptr)) { + remaining_ = 0; + return {}; + } + absl::string_view chunk = EdgeData(pos.edge).substr(pos.offset); + remaining_ = length() - offset - chunk.length(); + return chunk; +} + +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_INTERNAL_CORD_REP_BTREE_READER_H_ diff --git a/absl/strings/internal/cord_rep_btree_reader_test.cc b/absl/strings/internal/cord_rep_btree_reader_test.cc new file mode 100644 index 00000000..9b27a81f --- /dev/null +++ b/absl/strings/internal/cord_rep_btree_reader_test.cc @@ -0,0 +1,293 @@ +// Copyright 2021 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/cord_rep_btree_reader.h" + +#include <iostream> +#include <random> +#include <string> +#include <vector> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/config.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/strings/cord.h" +#include "absl/strings/internal/cord_internal.h" +#include "absl/strings/internal/cord_rep_btree.h" +#include "absl/strings/internal/cord_rep_test_util.h" +#include "absl/strings/string_view.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { +namespace { + +using ::testing::Eq; +using ::testing::IsEmpty; +using ::testing::Ne; +using ::testing::Not; + +using ::absl::cordrep_testing::CordRepBtreeFromFlats; +using ::absl::cordrep_testing::MakeFlat; +using ::absl::cordrep_testing::CordToString; +using ::absl::cordrep_testing::CreateFlatsFromString; +using ::absl::cordrep_testing::CreateRandomString; + +using ReadResult = CordRepBtreeReader::ReadResult; + +TEST(CordRepBtreeReaderTest, Next) { + constexpr size_t kChars = 3; + const size_t cap = CordRepBtree::kMaxCapacity; + int counts[] = {1, 2, cap, cap * cap, cap * cap + 1, cap * cap * 2 + 17}; + + for (int count : counts) { + std::string data = CreateRandomString(count * kChars); + std::vector<CordRep*> flats = CreateFlatsFromString(data, kChars); + CordRepBtree* node = CordRepBtreeFromFlats(flats); + + CordRepBtreeReader reader; + size_t remaining = data.length(); + absl::string_view chunk = reader.Init(node); + EXPECT_THAT(chunk, Eq(data.substr(0, chunk.length()))); + + remaining -= chunk.length(); + EXPECT_THAT(reader.remaining(), Eq(remaining)); + + while (remaining > 0) { + const size_t offset = data.length() - remaining; + chunk = reader.Next(); + EXPECT_THAT(chunk, Eq(data.substr(offset, chunk.length()))); + + remaining -= chunk.length(); + EXPECT_THAT(reader.remaining(), Eq(remaining)); + } + + EXPECT_THAT(reader.remaining(), Eq(0)); + + // Verify trying to read beyond EOF returns empty string_view + EXPECT_THAT(reader.Next(), testing::IsEmpty()); + + CordRep::Unref(node); + } +} + +TEST(CordRepBtreeReaderTest, Skip) { + constexpr size_t kChars = 3; + const size_t cap = CordRepBtree::kMaxCapacity; + int counts[] = {1, 2, cap, cap * cap, cap * cap + 1, cap * cap * 2 + 17}; + + for (int count : counts) { + std::string data = CreateRandomString(count * kChars); + std::vector<CordRep*> flats = CreateFlatsFromString(data, kChars); + CordRepBtree* node = CordRepBtreeFromFlats(flats); + + for (size_t skip1 = 0; skip1 < data.length() - kChars; ++skip1) { + for (size_t skip2 = 0; skip2 < data.length() - kChars; ++skip2) { + CordRepBtreeReader reader; + size_t remaining = data.length(); + absl::string_view chunk = reader.Init(node); + remaining -= chunk.length(); + + chunk = reader.Skip(skip1); + size_t offset = data.length() - remaining; + ASSERT_THAT(chunk, Eq(data.substr(offset + skip1, chunk.length()))); + remaining -= chunk.length() + skip1; + ASSERT_THAT(reader.remaining(), Eq(remaining)); + + if (remaining == 0) continue; + + size_t skip = std::min(remaining - 1, skip2); + chunk = reader.Skip(skip); + offset = data.length() - remaining; + ASSERT_THAT(chunk, Eq(data.substr(offset + skip, chunk.length()))); + } + } + + CordRep::Unref(node); + } +} + +TEST(CordRepBtreeReaderTest, SkipBeyondLength) { + CordRepBtree* tree = CordRepBtree::Create(MakeFlat("abc")); + tree = CordRepBtree::Append(tree, MakeFlat("def")); + CordRepBtreeReader reader; + reader.Init(tree); + EXPECT_THAT(reader.Skip(100), IsEmpty()); + EXPECT_THAT(reader.remaining(), Eq(0)); + CordRep::Unref(tree); +} + +TEST(CordRepBtreeReaderTest, Seek) { + constexpr size_t kChars = 3; + const size_t cap = CordRepBtree::kMaxCapacity; + int counts[] = {1, 2, cap, cap * cap, cap * cap + 1, cap * cap * 2 + 17}; + + for (int count : counts) { + std::string data = CreateRandomString(count * kChars); + std::vector<CordRep*> flats = CreateFlatsFromString(data, kChars); + CordRepBtree* node = CordRepBtreeFromFlats(flats); + + for (size_t seek = 0; seek < data.length() - 1; ++seek) { + CordRepBtreeReader reader; + reader.Init(node); + absl::string_view chunk = reader.Seek(seek); + ASSERT_THAT(chunk, Not(IsEmpty())); + ASSERT_THAT(chunk, Eq(data.substr(seek, chunk.length()))); + ASSERT_THAT(reader.remaining(), + Eq(data.length() - seek - chunk.length())); + } + + CordRep::Unref(node); + } +} + +TEST(CordRepBtreeReaderTest, SeekBeyondLength) { + CordRepBtree* tree = CordRepBtree::Create(MakeFlat("abc")); + tree = CordRepBtree::Append(tree, MakeFlat("def")); + CordRepBtreeReader reader; + reader.Init(tree); + EXPECT_THAT(reader.Seek(6), IsEmpty()); + EXPECT_THAT(reader.remaining(), Eq(0)); + EXPECT_THAT(reader.Seek(100), IsEmpty()); + EXPECT_THAT(reader.remaining(), Eq(0)); + CordRep::Unref(tree); +} + +TEST(CordRepBtreeReaderTest, Read) { + std::string data = "abcdefghijklmno"; + std::vector<CordRep*> flats = CreateFlatsFromString(data, 5); + CordRepBtree* node = CordRepBtreeFromFlats(flats); + + CordRep* tree; + CordRepBtreeReader reader; + absl::string_view chunk; + + // Read zero bytes + chunk = reader.Init(node); + chunk = reader.Read(0, chunk.length(), tree); + EXPECT_THAT(tree, Eq(nullptr)); + EXPECT_THAT(chunk, Eq("abcde")); + EXPECT_THAT(reader.remaining(), Eq(10)); + EXPECT_THAT(reader.Next(), Eq("fghij")); + + // Read in full + chunk = reader.Init(node); + chunk = reader.Read(15, chunk.length(), tree); + EXPECT_THAT(tree, Ne(nullptr)); + EXPECT_THAT(CordToString(tree), Eq("abcdefghijklmno")); + EXPECT_THAT(chunk, Eq("")); + EXPECT_THAT(reader.remaining(), Eq(0)); + CordRep::Unref(tree); + + // Read < chunk bytes + chunk = reader.Init(node); + chunk = reader.Read(3, chunk.length(), tree); + ASSERT_THAT(tree, Ne(nullptr)); + EXPECT_THAT(CordToString(tree), Eq("abc")); + EXPECT_THAT(chunk, Eq("de")); + EXPECT_THAT(reader.remaining(), Eq(10)); + EXPECT_THAT(reader.Next(), Eq("fghij")); + CordRep::Unref(tree); + + // Read < chunk bytes at offset + chunk = reader.Init(node); + chunk = reader.Read(2, chunk.length() - 2, tree); + ASSERT_THAT(tree, Ne(nullptr)); + EXPECT_THAT(CordToString(tree), Eq("cd")); + EXPECT_THAT(chunk, Eq("e")); + EXPECT_THAT(reader.remaining(), Eq(10)); + EXPECT_THAT(reader.Next(), Eq("fghij")); + CordRep::Unref(tree); + + // Read from consumed chunk + chunk = reader.Init(node); + chunk = reader.Read(3, 0, tree); + ASSERT_THAT(tree, Ne(nullptr)); + EXPECT_THAT(CordToString(tree), Eq("fgh")); + EXPECT_THAT(chunk, Eq("ij")); + EXPECT_THAT(reader.remaining(), Eq(5)); + EXPECT_THAT(reader.Next(), Eq("klmno")); + CordRep::Unref(tree); + + // Read across chunks + chunk = reader.Init(node); + chunk = reader.Read(12, chunk.length() - 2, tree); + ASSERT_THAT(tree, Ne(nullptr)); + EXPECT_THAT(CordToString(tree), Eq("cdefghijklmn")); + EXPECT_THAT(chunk, Eq("o")); + EXPECT_THAT(reader.remaining(), Eq(0)); + CordRep::Unref(tree); + + // Read across chunks landing on exact edge boundary + chunk = reader.Init(node); + chunk = reader.Read(10 - 2, chunk.length() - 2, tree); + ASSERT_THAT(tree, Ne(nullptr)); + EXPECT_THAT(CordToString(tree), Eq("cdefghij")); + EXPECT_THAT(chunk, Eq("klmno")); + EXPECT_THAT(reader.remaining(), Eq(0)); + CordRep::Unref(tree); + + CordRep::Unref(node); +} + +TEST(CordRepBtreeReaderTest, ReadExhaustive) { + constexpr size_t kChars = 3; + const size_t cap = CordRepBtree::kMaxCapacity; + int counts[] = {1, 2, cap, cap * cap + 1, cap * cap * cap * 2 + 17}; + + for (int count : counts) { + std::string data = CreateRandomString(count * kChars); + std::vector<CordRep*> flats = CreateFlatsFromString(data, kChars); + CordRepBtree* node = CordRepBtreeFromFlats(flats); + + for (size_t read_size : {kChars - 1, kChars, kChars + 7, cap * cap}) { + CordRepBtreeReader reader; + absl::string_view chunk = reader.Init(node); + + // `consumed` tracks the end of last consumed chunk which is the start of + // the next chunk: we always read with `chunk_size = chunk.length()`. + size_t consumed = 0; + size_t remaining = data.length(); + while (remaining > 0) { + CordRep* tree; + size_t n = (std::min)(remaining, read_size); + chunk = reader.Read(n, chunk.length(), tree); + EXPECT_THAT(tree, Ne(nullptr)); + if (tree) { + EXPECT_THAT(CordToString(tree), Eq(data.substr(consumed, n))); + CordRep::Unref(tree); + } + + consumed += n; + remaining -= n; + EXPECT_THAT(reader.remaining(), Eq(remaining - chunk.length())); + + if (remaining > 0) { + ASSERT_FALSE(chunk.empty()); + ASSERT_THAT(chunk, Eq(data.substr(consumed, chunk.length()))); + } else { + ASSERT_TRUE(chunk.empty()) << chunk; + } + } + } + + CordRep::Unref(node); + } +} + +} // namespace +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/absl/strings/internal/cord_rep_btree_test.cc b/absl/strings/internal/cord_rep_btree_test.cc new file mode 100644 index 00000000..51b90db1 --- /dev/null +++ b/absl/strings/internal/cord_rep_btree_test.cc @@ -0,0 +1,1565 @@ +// Copyright 2021 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/cord_rep_btree.h" + +#include <cmath> +#include <deque> +#include <iostream> +#include <string> +#include <vector> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/config.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/cleanup/cleanup.h" +#include "absl/strings/internal/cord_data_edge.h" +#include "absl/strings/internal/cord_internal.h" +#include "absl/strings/internal/cord_rep_test_util.h" +#include "absl/strings/str_cat.h" +#include "absl/strings/string_view.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { + +class CordRepBtreeTestPeer { + public: + static void SetEdge(CordRepBtree* node, size_t idx, CordRep* edge) { + node->edges_[idx] = edge; + } + static void AddEdge(CordRepBtree* node, CordRep* edge) { + node->edges_[node->fetch_add_end(1)] = edge; + } +}; + +namespace { + +using ::absl::cordrep_testing::AutoUnref; +using ::absl::cordrep_testing::CordCollectRepsIf; +using ::absl::cordrep_testing::CordToString; +using ::absl::cordrep_testing::CordVisitReps; +using ::absl::cordrep_testing::CreateFlatsFromString; +using ::absl::cordrep_testing::CreateRandomString; +using ::absl::cordrep_testing::MakeExternal; +using ::absl::cordrep_testing::MakeFlat; +using ::absl::cordrep_testing::MakeSubstring; +using ::testing::_; +using ::testing::AllOf; +using ::testing::AnyOf; +using ::testing::Conditional; +using ::testing::ElementsAre; +using ::testing::ElementsAreArray; +using ::testing::Eq; +using ::testing::HasSubstr; +using ::testing::Le; +using ::testing::Ne; +using ::testing::Not; +using ::testing::SizeIs; +using ::testing::TypedEq; + +MATCHER_P(EqFlatHolding, data, "Equals flat holding data") { + if (arg->tag < FLAT) { + *result_listener << "Expected FLAT, got tag " << static_cast<int>(arg->tag); + return false; + } + std::string actual = CordToString(arg); + if (actual != data) { + *result_listener << "Expected flat holding \"" << data + << "\", got flat holding \"" << actual << "\""; + return false; + } + return true; +} + +MATCHER_P(IsNode, height, absl::StrCat("Is a valid node of height ", height)) { + if (arg == nullptr) { + *result_listener << "Expected NODE, got nullptr"; + return false; + } + if (arg->tag != BTREE) { + *result_listener << "Expected NODE, got " << static_cast<int>(arg->tag); + return false; + } + if (!CordRepBtree::IsValid(arg->btree())) { + CordRepBtree::Dump(arg->btree(), "Expected valid NODE, got:", false, + *result_listener->stream()); + return false; + } + if (arg->btree()->height() != height) { + *result_listener << "Expected NODE of height " << height << ", got " + << arg->btree()->height(); + return false; + } + return true; +} + +MATCHER_P2(IsSubstring, start, length, + absl::StrCat("Is a substring(start = ", start, ", length = ", length, + ")")) { + if (arg == nullptr) { + *result_listener << "Expected substring, got nullptr"; + return false; + } + if (arg->tag != SUBSTRING) { + *result_listener << "Expected SUBSTRING, got " + << static_cast<int>(arg->tag); + return false; + } + const CordRepSubstring* const substr = arg->substring(); + if (substr->start != start || substr->length != length) { + *result_listener << "Expected substring(" << start << ", " << length + << "), got substring(" << substr->start << ", " + << substr->length << ")"; + return false; + } + return true; +} + +MATCHER_P2(EqExtractResult, tree, rep, "Equals ExtractResult") { + if (arg.tree != tree || arg.extracted != rep) { + *result_listener << "Expected {" << static_cast<const void*>(tree) << ", " + << static_cast<const void*>(rep) << "}, got {" << arg.tree + << ", " << arg.extracted << "}"; + return false; + } + return true; +} + +// DataConsumer is a simple helper class used by tests to 'consume' string +// fragments from the provided input in forward or backward direction. +class DataConsumer { + public: + // Starts consumption of `data`. Caller must make sure `data` outlives this + // instance. Consumes data starting at the front if `forward` is true, else + // consumes data from the back. + DataConsumer(absl::string_view data, bool forward) + : data_(data), forward_(forward) {} + + // Return the next `n` bytes from referenced data. + absl::string_view Next(size_t n) { + assert(n <= data_.size() - consumed_); + consumed_ += n; + return data_.substr(forward_ ? consumed_ - n : data_.size() - consumed_, n); + } + + // Returns all data consumed so far. + absl::string_view Consumed() const { + return forward_ ? data_.substr(0, consumed_) + : data_.substr(data_.size() - consumed_); + } + + private: + absl::string_view data_; + size_t consumed_ = 0; + bool forward_; +}; + +// BtreeAdd returns either CordRepBtree::Append or CordRepBtree::Prepend. +CordRepBtree* BtreeAdd(CordRepBtree* node, bool append, + absl::string_view data) { + return append ? CordRepBtree::Append(node, data) + : CordRepBtree::Prepend(node, data); +} + +// Recursively collects all leaf edges from `tree` and appends them to `edges`. +void GetLeafEdges(const CordRepBtree* tree, std::vector<CordRep*>& edges) { + if (tree->height() == 0) { + for (CordRep* edge : tree->Edges()) { + edges.push_back(edge); + } + } else { + for (CordRep* edge : tree->Edges()) { + GetLeafEdges(edge->btree(), edges); + } + } +} + +// Recursively collects and returns all leaf edges from `tree`. +std::vector<CordRep*> GetLeafEdges(const CordRepBtree* tree) { + std::vector<CordRep*> edges; + GetLeafEdges(tree, edges); + return edges; +} + +// Creates a flat containing the hexadecimal value of `i` zero padded +// to at least 4 digits prefixed with "0x", e.g.: "0x04AC". +CordRepFlat* MakeHexFlat(size_t i) { + return MakeFlat(absl::StrCat("0x", absl::Hex(i, absl::kZeroPad4))); +} + +CordRepBtree* MakeLeaf(size_t size = CordRepBtree::kMaxCapacity) { + assert(size <= CordRepBtree::kMaxCapacity); + CordRepBtree* leaf = CordRepBtree::Create(MakeHexFlat(0)); + for (size_t i = 1; i < size; ++i) { + leaf = CordRepBtree::Append(leaf, MakeHexFlat(i)); + } + return leaf; +} + +CordRepBtree* MakeTree(size_t size, bool append = true) { + CordRepBtree* tree = CordRepBtree::Create(MakeHexFlat(0)); + for (size_t i = 1; i < size; ++i) { + tree = append ? CordRepBtree::Append(tree, MakeHexFlat(i)) + : CordRepBtree::Prepend(tree, MakeHexFlat(i)); + } + return tree; +} + +CordRepBtree* CreateTree(absl::Span<CordRep* const> reps) { + auto it = reps.begin(); + CordRepBtree* tree = CordRepBtree::Create(*it); + while (++it != reps.end()) tree = CordRepBtree::Append(tree, *it); + return tree; +} + +CordRepBtree* CreateTree(absl::string_view data, size_t chunk_size) { + return CreateTree(CreateFlatsFromString(data, chunk_size)); +} + +CordRepBtree* CreateTreeReverse(absl::string_view data, size_t chunk_size) { + std::vector<CordRep*> flats = CreateFlatsFromString(data, chunk_size); + auto rit = flats.rbegin(); + CordRepBtree* tree = CordRepBtree::Create(*rit); + while (++rit != flats.rend()) tree = CordRepBtree::Prepend(tree, *rit); + return tree; +} + +class CordRepBtreeTest : public testing::TestWithParam<bool> { + public: + bool shared() const { return GetParam(); } + + static std::string ToString(testing::TestParamInfo<bool> param) { + return param.param ? "Shared" : "Private"; + } +}; + +INSTANTIATE_TEST_SUITE_P(WithParam, CordRepBtreeTest, testing::Bool(), + CordRepBtreeTest::ToString); + +class CordRepBtreeHeightTest : public testing::TestWithParam<int> { + public: + int height() const { return GetParam(); } + + static std::string ToString(testing::TestParamInfo<int> param) { + return absl::StrCat(param.param); + } +}; + +INSTANTIATE_TEST_SUITE_P(WithHeights, CordRepBtreeHeightTest, + testing::Range(0, CordRepBtree::kMaxHeight), + CordRepBtreeHeightTest::ToString); + +using TwoBools = testing::tuple<bool, bool>; + +class CordRepBtreeDualTest : public testing::TestWithParam<TwoBools> { + public: + bool first_shared() const { return std::get<0>(GetParam()); } + bool second_shared() const { return std::get<1>(GetParam()); } + + static std::string ToString(testing::TestParamInfo<TwoBools> param) { + if (std::get<0>(param.param)) { + return std::get<1>(param.param) ? "BothShared" : "FirstShared"; + } + return std::get<1>(param.param) ? "SecondShared" : "Private"; + } +}; + +INSTANTIATE_TEST_SUITE_P(WithParam, CordRepBtreeDualTest, + testing::Combine(testing::Bool(), testing::Bool()), + CordRepBtreeDualTest::ToString); + +TEST(CordRepBtreeTest, SizeIsMultipleOf64) { + // Only enforce for fully 64-bit platforms. + if (sizeof(size_t) == 8 && sizeof(void*) == 8) { + EXPECT_THAT(sizeof(CordRepBtree) % 64, Eq(0)) << "Should be multiple of 64"; + } +} + +TEST(CordRepBtreeTest, NewDestroyEmptyTree) { + auto* tree = CordRepBtree::New(); + EXPECT_THAT(tree->size(), Eq(0)); + EXPECT_THAT(tree->height(), Eq(0)); + EXPECT_THAT(tree->Edges(), ElementsAre()); + CordRepBtree::Destroy(tree); +} + +TEST(CordRepBtreeTest, NewDestroyEmptyTreeAtHeight) { + auto* tree = CordRepBtree::New(3); + EXPECT_THAT(tree->size(), Eq(0)); + EXPECT_THAT(tree->height(), Eq(3)); + EXPECT_THAT(tree->Edges(), ElementsAre()); + CordRepBtree::Destroy(tree); +} + +TEST(CordRepBtreeTest, Btree) { + CordRep* rep = CordRepBtree::New(); + EXPECT_THAT(rep->btree(), Eq(rep)); + EXPECT_THAT(static_cast<const CordRep*>(rep)->btree(), Eq(rep)); + CordRep::Unref(rep); +#if defined(GTEST_HAS_DEATH_TEST) && !defined(NDEBUG) + rep = MakeFlat("Hello world"); + EXPECT_DEATH(rep->btree(), ".*"); + EXPECT_DEATH(static_cast<const CordRep*>(rep)->btree(), ".*"); + CordRep::Unref(rep); +#endif +} + +TEST(CordRepBtreeTest, EdgeData) { + CordRepFlat* flat = MakeFlat("Hello world"); + CordRepExternal* external = MakeExternal("Hello external"); + CordRep* substr1 = MakeSubstring(1, 6, CordRep::Ref(flat)); + CordRep* substr2 = MakeSubstring(1, 6, CordRep::Ref(external)); + CordRep* bad_substr = MakeSubstring(1, 2, CordRep::Ref(substr1)); + + EXPECT_TRUE(IsDataEdge(flat)); + EXPECT_THAT(EdgeData(flat).data(), TypedEq<const void*>(flat->Data())); + EXPECT_THAT(EdgeData(flat), Eq("Hello world")); + + EXPECT_TRUE(IsDataEdge(external)); + EXPECT_THAT(EdgeData(external).data(), TypedEq<const void*>(external->base)); + EXPECT_THAT(EdgeData(external), Eq("Hello external")); + + EXPECT_TRUE(IsDataEdge(substr1)); + EXPECT_THAT(EdgeData(substr1).data(), TypedEq<const void*>(flat->Data() + 1)); + EXPECT_THAT(EdgeData(substr1), Eq("ello w")); + + EXPECT_TRUE(IsDataEdge(substr2)); + EXPECT_THAT(EdgeData(substr2).data(), + TypedEq<const void*>(external->base + 1)); + EXPECT_THAT(EdgeData(substr2), Eq("ello e")); + + EXPECT_FALSE(IsDataEdge(bad_substr)); +#if defined(GTEST_HAS_DEATH_TEST) && !defined(NDEBUG) + EXPECT_DEATH(EdgeData(bad_substr), ".*"); +#endif + + CordRep::Unref(bad_substr); + CordRep::Unref(substr2); + CordRep::Unref(substr1); + CordRep::Unref(external); + CordRep::Unref(flat); +} + +TEST(CordRepBtreeTest, CreateUnrefLeaf) { + auto* flat = MakeFlat("a"); + auto* leaf = CordRepBtree::Create(flat); + EXPECT_THAT(leaf->size(), Eq(1)); + EXPECT_THAT(leaf->height(), Eq(0)); + EXPECT_THAT(leaf->Edges(), ElementsAre(flat)); + CordRepBtree::Unref(leaf); +} + +TEST(CordRepBtreeTest, NewUnrefNode) { + auto* leaf = CordRepBtree::Create(MakeFlat("a")); + CordRepBtree* tree = CordRepBtree::New(leaf); + EXPECT_THAT(tree->size(), Eq(1)); + EXPECT_THAT(tree->height(), Eq(1)); + EXPECT_THAT(tree->Edges(), ElementsAre(leaf)); + CordRepBtree::Unref(tree); +} + +TEST_P(CordRepBtreeTest, AppendToLeafToCapacity) { + AutoUnref refs; + std::vector<CordRep*> flats; + flats.push_back(MakeHexFlat(0)); + auto* leaf = CordRepBtree::Create(flats.back()); + + for (size_t i = 1; i < CordRepBtree::kMaxCapacity; ++i) { + refs.RefIf(shared(), leaf); + flats.push_back(MakeHexFlat(i)); + auto* result = CordRepBtree::Append(leaf, flats.back()); + EXPECT_THAT(result->height(), Eq(0)); + EXPECT_THAT(result, Conditional(shared(), Ne(leaf), Eq(leaf))); + EXPECT_THAT(result->Edges(), ElementsAreArray(flats)); + leaf = result; + } + CordRep::Unref(leaf); +} + +TEST_P(CordRepBtreeTest, PrependToLeafToCapacity) { + AutoUnref refs; + std::deque<CordRep*> flats; + flats.push_front(MakeHexFlat(0)); + auto* leaf = CordRepBtree::Create(flats.front()); + + for (size_t i = 1; i < CordRepBtree::kMaxCapacity; ++i) { + refs.RefIf(shared(), leaf); + flats.push_front(MakeHexFlat(i)); + auto* result = CordRepBtree::Prepend(leaf, flats.front()); + EXPECT_THAT(result->height(), Eq(0)); + EXPECT_THAT(result, Conditional(shared(), Ne(leaf), Eq(leaf))); + EXPECT_THAT(result->Edges(), ElementsAreArray(flats)); + leaf = result; + } + CordRep::Unref(leaf); +} + +// This test specifically aims at code aligning data at either the front or the +// back of the contained `edges[]` array, alternating Append and Prepend will +// move `begin()` and `end()` values as needed for each added value. +TEST_P(CordRepBtreeTest, AppendPrependToLeafToCapacity) { + AutoUnref refs; + std::deque<CordRep*> flats; + flats.push_front(MakeHexFlat(0)); + auto* leaf = CordRepBtree::Create(flats.front()); + + for (size_t i = 1; i < CordRepBtree::kMaxCapacity; ++i) { + refs.RefIf(shared(), leaf); + CordRepBtree* result; + if (i % 2 != 0) { + flats.push_front(MakeHexFlat(i)); + result = CordRepBtree::Prepend(leaf, flats.front()); + } else { + flats.push_back(MakeHexFlat(i)); + result = CordRepBtree::Append(leaf, flats.back()); + } + EXPECT_THAT(result->height(), Eq(0)); + EXPECT_THAT(result, Conditional(shared(), Ne(leaf), Eq(leaf))); + EXPECT_THAT(result->Edges(), ElementsAreArray(flats)); + leaf = result; + } + CordRep::Unref(leaf); +} + +TEST_P(CordRepBtreeTest, AppendToLeafBeyondCapacity) { + AutoUnref refs; + auto* leaf = MakeLeaf(); + refs.RefIf(shared(), leaf); + CordRep* flat = MakeFlat("abc"); + auto* result = CordRepBtree::Append(leaf, flat); + ASSERT_THAT(result, IsNode(1)); + EXPECT_THAT(result, Ne(leaf)); + absl::Span<CordRep* const> edges = result->Edges(); + ASSERT_THAT(edges, ElementsAre(leaf, IsNode(0))); + EXPECT_THAT(edges[1]->btree()->Edges(), ElementsAre(flat)); + CordRep::Unref(result); +} + +TEST_P(CordRepBtreeTest, PrependToLeafBeyondCapacity) { + AutoUnref refs; + auto* leaf = MakeLeaf(); + refs.RefIf(shared(), leaf); + CordRep* flat = MakeFlat("abc"); + auto* result = CordRepBtree::Prepend(leaf, flat); + ASSERT_THAT(result, IsNode(1)); + EXPECT_THAT(result, Ne(leaf)); + absl::Span<CordRep* const> edges = result->Edges(); + ASSERT_THAT(edges, ElementsAre(IsNode(0), leaf)); + EXPECT_THAT(edges[0]->btree()->Edges(), ElementsAre(flat)); + CordRep::Unref(result); +} + +TEST_P(CordRepBtreeTest, AppendToTreeOneDeep) { + constexpr size_t max_cap = CordRepBtree::kMaxCapacity; + AutoUnref refs; + std::vector<CordRep*> flats; + flats.push_back(MakeHexFlat(0)); + CordRepBtree* tree = CordRepBtree::Create(flats.back()); + for (size_t i = 1; i <= max_cap; ++i) { + flats.push_back(MakeHexFlat(i)); + tree = CordRepBtree::Append(tree, flats.back()); + } + ASSERT_THAT(tree, IsNode(1)); + + for (size_t i = max_cap + 1; i < max_cap * max_cap; ++i) { + // Ref top level tree based on param. + // Ref leaf node once every 4 iterations, which should not have an + // observable effect other than that the leaf itself is copied. + refs.RefIf(shared(), tree); + refs.RefIf(i % 4 == 0, tree->Edges().back()); + + flats.push_back(MakeHexFlat(i)); + CordRepBtree* result = CordRepBtree::Append(tree, flats.back()); + ASSERT_THAT(result, IsNode(1)); + ASSERT_THAT(result, Conditional(shared(), Ne(tree), Eq(tree))); + std::vector<CordRep*> edges = GetLeafEdges(result); + ASSERT_THAT(edges, ElementsAreArray(flats)); + tree = result; + } + CordRep::Unref(tree); +} + +TEST_P(CordRepBtreeTest, AppendToTreeTwoDeep) { + constexpr size_t max_cap = CordRepBtree::kMaxCapacity; + AutoUnref refs; + std::vector<CordRep*> flats; + flats.push_back(MakeHexFlat(0)); + CordRepBtree* tree = CordRepBtree::Create(flats.back()); + for (size_t i = 1; i <= max_cap * max_cap; ++i) { + flats.push_back(MakeHexFlat(i)); + tree = CordRepBtree::Append(tree, flats.back()); + } + ASSERT_THAT(tree, IsNode(2)); + for (size_t i = max_cap * max_cap + 1; i < max_cap * max_cap * max_cap; ++i) { + // Ref top level tree based on param. + // Ref child node once every 16 iterations, and leaf node every 4 + // iterrations which which should not have an observable effect other than + // the node and/or the leaf below it being copied. + refs.RefIf(shared(), tree); + refs.RefIf(i % 16 == 0, tree->Edges().back()); + refs.RefIf(i % 4 == 0, tree->Edges().back()->btree()->Edges().back()); + + flats.push_back(MakeHexFlat(i)); + CordRepBtree* result = CordRepBtree::Append(tree, flats.back()); + ASSERT_THAT(result, IsNode(2)); + ASSERT_THAT(result, Conditional(shared(), Ne(tree), Eq(tree))); + std::vector<CordRep*> edges = GetLeafEdges(result); + ASSERT_THAT(edges, ElementsAreArray(flats)); + tree = result; + } + CordRep::Unref(tree); +} + +TEST_P(CordRepBtreeTest, PrependToTreeOneDeep) { + constexpr size_t max_cap = CordRepBtree::kMaxCapacity; + AutoUnref refs; + std::deque<CordRep*> flats; + flats.push_back(MakeHexFlat(0)); + CordRepBtree* tree = CordRepBtree::Create(flats.back()); + for (size_t i = 1; i <= max_cap; ++i) { + flats.push_front(MakeHexFlat(i)); + tree = CordRepBtree::Prepend(tree, flats.front()); + } + ASSERT_THAT(tree, IsNode(1)); + + for (size_t i = max_cap + 1; i < max_cap * max_cap; ++i) { + // Ref top level tree based on param. + // Ref leaf node once every 4 iterations which should not have an observable + // effect other than than the leaf itself is copied. + refs.RefIf(shared(), tree); + refs.RefIf(i % 4 == 0, tree->Edges().back()); + + flats.push_front(MakeHexFlat(i)); + CordRepBtree* result = CordRepBtree::Prepend(tree, flats.front()); + ASSERT_THAT(result, IsNode(1)); + ASSERT_THAT(result, Conditional(shared(), Ne(tree), Eq(tree))); + std::vector<CordRep*> edges = GetLeafEdges(result); + ASSERT_THAT(edges, ElementsAreArray(flats)); + tree = result; + } + CordRep::Unref(tree); +} + +TEST_P(CordRepBtreeTest, PrependToTreeTwoDeep) { + constexpr size_t max_cap = CordRepBtree::kMaxCapacity; + AutoUnref refs; + std::deque<CordRep*> flats; + flats.push_back(MakeHexFlat(0)); + CordRepBtree* tree = CordRepBtree::Create(flats.back()); + for (size_t i = 1; i <= max_cap * max_cap; ++i) { + flats.push_front(MakeHexFlat(i)); + tree = CordRepBtree::Prepend(tree, flats.front()); + } + ASSERT_THAT(tree, IsNode(2)); + for (size_t i = max_cap * max_cap + 1; i < max_cap * max_cap * max_cap; ++i) { + // Ref top level tree based on param. + // Ref child node once every 16 iterations, and leaf node every 4 + // iterrations which which should not have an observable effect other than + // the node and/or the leaf below it being copied. + refs.RefIf(shared(), tree); + refs.RefIf(i % 16 == 0, tree->Edges().back()); + refs.RefIf(i % 4 == 0, tree->Edges().back()->btree()->Edges().back()); + + flats.push_front(MakeHexFlat(i)); + CordRepBtree* result = CordRepBtree::Prepend(tree, flats.front()); + ASSERT_THAT(result, IsNode(2)); + ASSERT_THAT(result, Conditional(shared(), Ne(tree), Eq(tree))); + std::vector<CordRep*> edges = GetLeafEdges(result); + ASSERT_THAT(edges, ElementsAreArray(flats)); + tree = result; + } + CordRep::Unref(tree); +} + +TEST_P(CordRepBtreeDualTest, MergeLeafsNotExceedingCapacity) { + for (bool use_append : {false, true}) { + SCOPED_TRACE(use_append ? "Using Append" : "Using Prepend"); + + AutoUnref refs; + std::vector<CordRep*> flats; + + // Build `left` side leaf appending all contained flats to `flats` + CordRepBtree* left = MakeLeaf(3); + GetLeafEdges(left, flats); + refs.RefIf(first_shared(), left); + + // Build `right` side leaf appending all contained flats to `flats` + CordRepBtree* right = MakeLeaf(2); + GetLeafEdges(right, flats); + refs.RefIf(second_shared(), right); + + CordRepBtree* tree = use_append ? CordRepBtree::Append(left, right) + : CordRepBtree::Prepend(right, left); + EXPECT_THAT(tree, IsNode(0)); + + // `tree` contains all flats originally belonging to `left` and `right`. + EXPECT_THAT(tree->Edges(), ElementsAreArray(flats)); + CordRepBtree::Unref(tree); + } +} + +TEST_P(CordRepBtreeDualTest, MergeLeafsExceedingCapacity) { + for (bool use_append : {false, true}) { + SCOPED_TRACE(use_append ? "Using Append" : "Using Prepend"); + + AutoUnref refs; + + // Build `left` side tree appending all contained flats to `flats` + CordRepBtree* left = MakeLeaf(CordRepBtree::kMaxCapacity - 2); + refs.RefIf(first_shared(), left); + + // Build `right` side tree appending all contained flats to `flats` + CordRepBtree* right = MakeLeaf(CordRepBtree::kMaxCapacity - 1); + refs.RefIf(second_shared(), right); + + CordRepBtree* tree = use_append ? CordRepBtree::Append(left, right) + : CordRepBtree::Prepend(right, left); + EXPECT_THAT(tree, IsNode(1)); + EXPECT_THAT(tree->Edges(), ElementsAre(left, right)); + CordRepBtree::Unref(tree); + } +} + +TEST_P(CordRepBtreeDualTest, MergeEqualHeightTrees) { + for (bool use_append : {false, true}) { + SCOPED_TRACE(use_append ? "Using Append" : "Using Prepend"); + + AutoUnref refs; + std::vector<CordRep*> flats; + + // Build `left` side tree appending all contained flats to `flats` + CordRepBtree* left = MakeTree(CordRepBtree::kMaxCapacity * 3); + GetLeafEdges(left, flats); + refs.RefIf(first_shared(), left); + + // Build `right` side tree appending all contained flats to `flats` + CordRepBtree* right = MakeTree(CordRepBtree::kMaxCapacity * 2); + GetLeafEdges(right, flats); + refs.RefIf(second_shared(), right); + + CordRepBtree* tree = use_append ? CordRepBtree::Append(left, right) + : CordRepBtree::Prepend(right, left); + EXPECT_THAT(tree, IsNode(1)); + EXPECT_THAT(tree->Edges(), SizeIs(5)); + + // `tree` contains all flats originally belonging to `left` and `right`. + EXPECT_THAT(GetLeafEdges(tree), ElementsAreArray(flats)); + CordRepBtree::Unref(tree); + } +} + +TEST_P(CordRepBtreeDualTest, MergeLeafWithTreeNotExceedingLeafCapacity) { + for (bool use_append : {false, true}) { + SCOPED_TRACE(use_append ? "Using Append" : "Using Prepend"); + + AutoUnref refs; + std::vector<CordRep*> flats; + + // Build `left` side tree appending all added flats to `flats` + CordRepBtree* left = MakeTree(CordRepBtree::kMaxCapacity * 2 + 2); + GetLeafEdges(left, flats); + refs.RefIf(first_shared(), left); + + // Build `right` side tree appending all added flats to `flats` + CordRepBtree* right = MakeTree(3); + GetLeafEdges(right, flats); + refs.RefIf(second_shared(), right); + + CordRepBtree* tree = use_append ? CordRepBtree::Append(left, right) + : CordRepBtree::Prepend(right, left); + EXPECT_THAT(tree, IsNode(1)); + EXPECT_THAT(tree->Edges(), SizeIs(3)); + + // `tree` contains all flats originally belonging to `left` and `right`. + EXPECT_THAT(GetLeafEdges(tree), ElementsAreArray(flats)); + CordRepBtree::Unref(tree); + } +} + +TEST_P(CordRepBtreeDualTest, MergeLeafWithTreeExceedingLeafCapacity) { + for (bool use_append : {false, true}) { + SCOPED_TRACE(use_append ? "Using Append" : "Using Prepend"); + + AutoUnref refs; + std::vector<CordRep*> flats; + + // Build `left` side tree appending all added flats to `flats` + CordRepBtree* left = MakeTree(CordRepBtree::kMaxCapacity * 3 - 2); + GetLeafEdges(left, flats); + refs.RefIf(first_shared(), left); + + // Build `right` side tree appending all added flats to `flats` + CordRepBtree* right = MakeTree(3); + GetLeafEdges(right, flats); + refs.RefIf(second_shared(), right); + + CordRepBtree* tree = use_append ? CordRepBtree::Append(left, right) + : CordRepBtree::Prepend(right, left); + EXPECT_THAT(tree, IsNode(1)); + EXPECT_THAT(tree->Edges(), SizeIs(4)); + + // `tree` contains all flats originally belonging to `left` and `right`. + EXPECT_THAT(GetLeafEdges(tree), ElementsAreArray(flats)); + CordRepBtree::Unref(tree); + } +} + +void RefEdgesAt(size_t depth, AutoUnref& refs, CordRepBtree* tree) { + absl::Span<CordRep* const> edges = tree->Edges(); + if (depth == 0) { + refs.Ref(edges.front()); + refs.Ref(edges.back()); + } else { + assert(tree->height() > 0); + RefEdgesAt(depth - 1, refs, edges.front()->btree()); + RefEdgesAt(depth - 1, refs, edges.back()->btree()); + } +} + +TEST(CordRepBtreeTest, MergeFuzzTest) { + constexpr size_t max_cap = CordRepBtree::kMaxCapacity; + std::minstd_rand rnd; + std::uniform_int_distribution<int> coin_flip(0, 1); + std::uniform_int_distribution<int> dice_throw(1, 6); + + auto random_leaf_count = [&]() { + std::uniform_int_distribution<int> dist_height(0, 3); + std::uniform_int_distribution<int> dist_leaf(0, max_cap - 1); + const size_t height = dist_height(rnd); + return (height ? pow(max_cap, height) : 0) + dist_leaf(rnd); + }; + + for (int i = 0; i < 10000; ++i) { + AutoUnref refs; + std::vector<CordRep*> flats; + + CordRepBtree* left = MakeTree(random_leaf_count(), coin_flip(rnd)); + GetLeafEdges(left, flats); + if (dice_throw(rnd) == 1) { + std::uniform_int_distribution<int> dist(0, left->height()); + RefEdgesAt(dist(rnd), refs, left); + } + + CordRepBtree* right = MakeTree(random_leaf_count(), coin_flip(rnd)); + GetLeafEdges(right, flats); + if (dice_throw(rnd) == 1) { + std::uniform_int_distribution<int> dist(0, right->height()); + RefEdgesAt(dist(rnd), refs, right); + } + + CordRepBtree* tree = CordRepBtree::Append(left, right); + EXPECT_THAT(GetLeafEdges(tree), ElementsAreArray(flats)); + CordRepBtree::Unref(tree); + } +} + +TEST_P(CordRepBtreeTest, RemoveSuffix) { + // Create tree of 1, 2 and 3 levels high + constexpr size_t max_cap = CordRepBtree::kMaxCapacity; + for (size_t cap : {max_cap - 1, max_cap * 2, max_cap * max_cap * 2}) { + const std::string data = CreateRandomString(cap * 512); + + { + // Verify RemoveSuffix(<all>) + AutoUnref refs; + CordRepBtree* node = refs.RefIf(shared(), CreateTree(data, 512)); + EXPECT_THAT(CordRepBtree::RemoveSuffix(node, data.length()), Eq(nullptr)); + + // Verify RemoveSuffix(<none>) + node = refs.RefIf(shared(), CreateTree(data, 512)); + EXPECT_THAT(CordRepBtree::RemoveSuffix(node, 0), Eq(node)); + CordRep::Unref(node); + } + + for (int n = 1; n < data.length(); ++n) { + AutoUnref refs; + auto flats = CreateFlatsFromString(data, 512); + CordRepBtree* node = refs.RefIf(shared(), CreateTree(flats)); + CordRep* rep = refs.Add(CordRepBtree::RemoveSuffix(node, n)); + EXPECT_THAT(CordToString(rep), Eq(data.substr(0, data.length() - n))); + + // Collect all flats + auto is_flat = [](CordRep* rep) { return rep->tag >= FLAT; }; + std::vector<CordRep*> edges = CordCollectRepsIf(is_flat, rep); + ASSERT_THAT(edges.size(), Le(flats.size())); + + // Isolate last edge + CordRep* last_edge = edges.back(); + edges.pop_back(); + const size_t last_length = rep->length - edges.size() * 512; + + // All flats except the last edge must be kept or copied 'as is' + int index = 0; + for (CordRep* edge : edges) { + ASSERT_THAT(edge, Eq(flats[index++])); + ASSERT_THAT(edge->length, Eq(512)); + } + + // CordRepBtree may optimize small substrings to avoid waste, so only + // check for flat sharing / updates where the code should always do this. + if (last_length >= 500) { + EXPECT_THAT(last_edge, Eq(flats[index++])); + if (shared()) { + EXPECT_THAT(last_edge->length, Eq(512)); + } else { + EXPECT_TRUE(last_edge->refcount.IsOne()); + EXPECT_THAT(last_edge->length, Eq(last_length)); + } + } + } + } +} + +TEST(CordRepBtreeTest, SubTree) { + // Create tree of at least 2 levels high + constexpr size_t max_cap = CordRepBtree::kMaxCapacity; + const size_t n = max_cap * max_cap * 2; + const std::string data = CreateRandomString(n * 3); + std::vector<CordRep*> flats; + for (absl::string_view s = data; !s.empty(); s.remove_prefix(3)) { + flats.push_back(MakeFlat(s.substr(0, 3))); + } + CordRepBtree* node = CordRepBtree::Create(CordRep::Ref(flats[0])); + for (size_t i = 1; i < flats.size(); ++i) { + node = CordRepBtree::Append(node, CordRep::Ref(flats[i])); + } + + for (int offset = 0; offset < data.length(); ++offset) { + for (int length = 1; length <= data.length() - offset; ++length) { + CordRep* rep = node->SubTree(offset, length); + EXPECT_THAT(CordToString(rep), Eq(data.substr(offset, length))); + CordRep::Unref(rep); + } + } + CordRepBtree::Unref(node); + for (CordRep* rep : flats) { + CordRep::Unref(rep); + } +} + +TEST(CordRepBtreeTest, SubTreeOnExistingSubstring) { + // This test verifies that a SubTree call on a pre-existing (large) substring + // adjusts the existing substring if not shared, and else rewrites the + // existing substring. + AutoUnref refs; + std::string data = CreateRandomString(1000); + CordRepBtree* leaf = CordRepBtree::Create(MakeFlat("abc")); + CordRep* flat = MakeFlat(data); + leaf = CordRepBtree::Append(leaf, flat); + + // Setup tree containing substring. + CordRep* result = leaf->SubTree(0, 3 + 990); + ASSERT_THAT(result->tag, Eq(BTREE)); + CordRep::Unref(leaf); + leaf = result->btree(); + ASSERT_THAT(leaf->Edges(), ElementsAre(_, IsSubstring(0, 990))); + EXPECT_THAT(leaf->Edges()[1]->substring()->child, Eq(flat)); + + // Verify substring of substring. + result = leaf->SubTree(3 + 5, 970); + ASSERT_THAT(result, IsSubstring(5, 970)); + EXPECT_THAT(result->substring()->child, Eq(flat)); + CordRep::Unref(result); + + CordRep::Unref(leaf); +} + +TEST_P(CordRepBtreeTest, AddDataToLeaf) { + const size_t n = CordRepBtree::kMaxCapacity; + const std::string data = CreateRandomString(n * 3); + + for (bool append : {true, false}) { + AutoUnref refs; + DataConsumer consumer(data, append); + SCOPED_TRACE(append ? "Append" : "Prepend"); + + CordRepBtree* leaf = CordRepBtree::Create(MakeFlat(consumer.Next(3))); + for (size_t i = 1; i < n; ++i) { + refs.RefIf(shared(), leaf); + CordRepBtree* result = BtreeAdd(leaf, append, consumer.Next(3)); + EXPECT_THAT(result, Conditional(shared(), Ne(leaf), Eq(leaf))); + EXPECT_THAT(CordToString(result), Eq(consumer.Consumed())); + leaf = result; + } + CordRep::Unref(leaf); + } +} + +TEST_P(CordRepBtreeTest, AppendDataToTree) { + AutoUnref refs; + size_t n = CordRepBtree::kMaxCapacity + CordRepBtree::kMaxCapacity / 2; + std::string data = CreateRandomString(n * 3); + CordRepBtree* tree = refs.RefIf(shared(), CreateTree(data, 3)); + CordRepBtree* leaf0 = tree->Edges()[0]->btree(); + CordRepBtree* leaf1 = tree->Edges()[1]->btree(); + CordRepBtree* result = CordRepBtree::Append(tree, "123456789"); + EXPECT_THAT(result, Conditional(shared(), Ne(tree), Eq(tree))); + EXPECT_THAT(result->Edges(), + ElementsAre(leaf0, Conditional(shared(), Ne(leaf1), Eq(leaf1)))); + EXPECT_THAT(CordToString(result), Eq(data + "123456789")); + CordRep::Unref(result); +} + +TEST_P(CordRepBtreeTest, PrependDataToTree) { + AutoUnref refs; + size_t n = CordRepBtree::kMaxCapacity + CordRepBtree::kMaxCapacity / 2; + std::string data = CreateRandomString(n * 3); + CordRepBtree* tree = refs.RefIf(shared(), CreateTreeReverse(data, 3)); + CordRepBtree* leaf0 = tree->Edges()[0]->btree(); + CordRepBtree* leaf1 = tree->Edges()[1]->btree(); + CordRepBtree* result = CordRepBtree::Prepend(tree, "123456789"); + EXPECT_THAT(result, Conditional(shared(), Ne(tree), Eq(tree))); + EXPECT_THAT(result->Edges(), + ElementsAre(Conditional(shared(), Ne(leaf0), Eq(leaf0)), leaf1)); + EXPECT_THAT(CordToString(result), Eq("123456789" + data)); + CordRep::Unref(result); +} + +TEST_P(CordRepBtreeTest, AddDataToTreeThreeLevelsDeep) { + constexpr size_t max_cap = CordRepBtree::kMaxCapacity; + const size_t n = max_cap * max_cap * max_cap; + const std::string data = CreateRandomString(n * 3); + + for (bool append : {true, false}) { + AutoUnref refs; + DataConsumer consumer(data, append); + SCOPED_TRACE(append ? "Append" : "Prepend"); + + // Fill leaf + CordRepBtree* tree = CordRepBtree::Create(MakeFlat(consumer.Next(3))); + for (size_t i = 1; i < max_cap; ++i) { + tree = BtreeAdd(tree, append, consumer.Next(3)); + } + ASSERT_THAT(CordToString(tree), Eq(consumer.Consumed())); + + // Fill to maximum at one deep + refs.RefIf(shared(), tree); + CordRepBtree* result = BtreeAdd(tree, append, consumer.Next(3)); + ASSERT_THAT(result, IsNode(1)); + ASSERT_THAT(result, Ne(tree)); + ASSERT_THAT(CordToString(result), Eq(consumer.Consumed())); + tree = result; + for (size_t i = max_cap + 1; i < max_cap * max_cap; ++i) { + refs.RefIf(shared(), tree); + result = BtreeAdd(tree, append, consumer.Next(3)); + ASSERT_THAT(result, Conditional(shared(), Ne(tree), Eq(tree))); + ASSERT_THAT(CordToString(result), Eq(consumer.Consumed())); + tree = result; + } + + // Fill to maximum at two deep + refs.RefIf(shared(), tree); + result = BtreeAdd(tree, append, consumer.Next(3)); + ASSERT_THAT(result, IsNode(2)); + ASSERT_THAT(result, Ne(tree)); + ASSERT_THAT(CordToString(result), Eq(consumer.Consumed())); + tree = result; + for (size_t i = max_cap * max_cap + 1; i < max_cap * max_cap * max_cap; + ++i) { + refs.RefIf(shared(), tree); + result = BtreeAdd(tree, append, consumer.Next(3)); + ASSERT_THAT(result, Conditional(shared(), Ne(tree), Eq(tree))); + ASSERT_THAT(CordToString(result), Eq(consumer.Consumed())); + tree = result; + } + + CordRep::Unref(tree); + } +} + +TEST_P(CordRepBtreeTest, AddLargeDataToLeaf) { + const size_t max_cap = CordRepBtree::kMaxCapacity; + const size_t n = max_cap * max_cap * max_cap * 3 + 2; + const std::string data = CreateRandomString(n * kMaxFlatLength); + + for (bool append : {true, false}) { + AutoUnref refs; + SCOPED_TRACE(append ? "Append" : "Prepend"); + + CordRepBtree* leaf = CordRepBtree::Create(MakeFlat("abc")); + refs.RefIf(shared(), leaf); + CordRepBtree* result = BtreeAdd(leaf, append, data); + EXPECT_THAT(CordToString(result), Eq(append ? "abc" + data : data + "abc")); + CordRep::Unref(result); + } +} + +TEST_P(CordRepBtreeTest, CreateFromTreeReturnsTree) { + AutoUnref refs; + CordRepBtree* leaf = CordRepBtree::Create(MakeFlat("Hello world")); + refs.RefIf(shared(), leaf); + CordRepBtree* result = CordRepBtree::Create(leaf); + EXPECT_THAT(result, Eq(leaf)); + CordRep::Unref(result); +} + +TEST(CordRepBtreeTest, GetCharacter) { + size_t n = CordRepBtree::kMaxCapacity * CordRepBtree::kMaxCapacity + 2; + std::string data = CreateRandomString(n * 3); + CordRepBtree* tree = CreateTree(data, 3); + // Add a substring node for good measure. + tree = tree->Append(tree, MakeSubstring(4, 5, MakeFlat("abcdefghijklm"))); + data += "efghi"; + for (size_t i = 0; i < data.length(); ++i) { + ASSERT_THAT(tree->GetCharacter(i), Eq(data[i])); + } + CordRep::Unref(tree); +} + +TEST_P(CordRepBtreeTest, IsFlatSingleFlat) { + CordRepBtree* leaf = CordRepBtree::Create(MakeFlat("Hello world")); + + absl::string_view fragment; + EXPECT_TRUE(leaf->IsFlat(nullptr)); + EXPECT_TRUE(leaf->IsFlat(&fragment)); + EXPECT_THAT(fragment, Eq("Hello world")); + fragment = ""; + EXPECT_TRUE(leaf->IsFlat(0, 11, nullptr)); + EXPECT_TRUE(leaf->IsFlat(0, 11, &fragment)); + EXPECT_THAT(fragment, Eq("Hello world")); + + // Arbitrary ranges must check true as well. + EXPECT_TRUE(leaf->IsFlat(1, 4, &fragment)); + EXPECT_THAT(fragment, Eq("ello")); + EXPECT_TRUE(leaf->IsFlat(6, 5, &fragment)); + EXPECT_THAT(fragment, Eq("world")); + + CordRep::Unref(leaf); +} + +TEST(CordRepBtreeTest, IsFlatMultiFlat) { + size_t n = CordRepBtree::kMaxCapacity * CordRepBtree::kMaxCapacity + 2; + std::string data = CreateRandomString(n * 3); + CordRepBtree* tree = CreateTree(data, 3); + // Add substring nodes for good measure. + tree = tree->Append(tree, MakeSubstring(4, 3, MakeFlat("abcdefghijklm"))); + tree = tree->Append(tree, MakeSubstring(8, 3, MakeFlat("abcdefghijklm"))); + data += "efgijk"; + + EXPECT_FALSE(tree->IsFlat(nullptr)); + absl::string_view fragment = "Can't touch this"; + EXPECT_FALSE(tree->IsFlat(&fragment)); + EXPECT_THAT(fragment, Eq("Can't touch this")); + + for (size_t offset = 0; offset < data.size(); offset += 3) { + EXPECT_TRUE(tree->IsFlat(offset, 3, nullptr)); + EXPECT_TRUE(tree->IsFlat(offset, 3, &fragment)); + EXPECT_THAT(fragment, Eq(data.substr(offset, 3))); + + fragment = "Can't touch this"; + if (offset > 0) { + EXPECT_FALSE(tree->IsFlat(offset - 1, 4, nullptr)); + EXPECT_FALSE(tree->IsFlat(offset - 1, 4, &fragment)); + EXPECT_THAT(fragment, Eq("Can't touch this")); + } + if (offset < data.size() - 4) { + EXPECT_FALSE(tree->IsFlat(offset, 4, nullptr)); + EXPECT_FALSE(tree->IsFlat(offset, 4, &fragment)); + EXPECT_THAT(fragment, Eq("Can't touch this")); + } + } + + CordRep::Unref(tree); +} + +#if defined(GTEST_HAS_DEATH_TEST) && !defined(NDEBUG) + +TEST_P(CordRepBtreeHeightTest, GetAppendBufferNotPrivate) { + CordRepBtree* tree = CordRepBtree::Create(MakeExternal("Foo")); + CordRepBtree::Ref(tree); + EXPECT_DEATH(tree->GetAppendBuffer(1), ".*"); + CordRepBtree::Unref(tree); + CordRepBtree::Unref(tree); +} + +#endif // defined(GTEST_HAS_DEATH_TEST) && !defined(NDEBUG) + +TEST_P(CordRepBtreeHeightTest, GetAppendBufferNotFlat) { + CordRepBtree* tree = CordRepBtree::Create(MakeExternal("Foo")); + for (int i = 1; i <= height(); ++i) { + tree = CordRepBtree::New(tree); + } + EXPECT_THAT(tree->GetAppendBuffer(1), SizeIs(0)); + CordRepBtree::Unref(tree); +} + +TEST_P(CordRepBtreeHeightTest, GetAppendBufferFlatNotPrivate) { + CordRepFlat* flat = MakeFlat("abc"); + CordRepBtree* tree = CordRepBtree::Create(CordRep::Ref(flat)); + for (int i = 1; i <= height(); ++i) { + tree = CordRepBtree::New(tree); + } + EXPECT_THAT(tree->GetAppendBuffer(1), SizeIs(0)); + CordRepBtree::Unref(tree); + CordRep::Unref(flat); +} + +TEST_P(CordRepBtreeHeightTest, GetAppendBufferTreeNotPrivate) { + if (height() == 0) return; + AutoUnref refs; + CordRepFlat* flat = MakeFlat("abc"); + CordRepBtree* tree = CordRepBtree::Create(CordRep::Ref(flat)); + for (int i = 1; i <= height(); ++i) { + if (i == (height() + 1) / 2) refs.Ref(tree); + tree = CordRepBtree::New(tree); + } + EXPECT_THAT(tree->GetAppendBuffer(1), SizeIs(0)); + CordRepBtree::Unref(tree); + CordRep::Unref(flat); +} + +TEST_P(CordRepBtreeHeightTest, GetAppendBufferFlatNoCapacity) { + CordRepFlat* flat = MakeFlat("abc"); + flat->length = flat->Capacity(); + CordRepBtree* tree = CordRepBtree::Create(flat); + for (int i = 1; i <= height(); ++i) { + tree = CordRepBtree::New(tree); + } + EXPECT_THAT(tree->GetAppendBuffer(1), SizeIs(0)); + CordRepBtree::Unref(tree); +} + +TEST_P(CordRepBtreeHeightTest, GetAppendBufferFlatWithCapacity) { + CordRepFlat* flat = MakeFlat("abc"); + CordRepBtree* tree = CordRepBtree::Create(flat); + for (int i = 1; i <= height(); ++i) { + tree = CordRepBtree::New(tree); + } + absl::Span<char> span = tree->GetAppendBuffer(2); + EXPECT_THAT(span, SizeIs(2)); + EXPECT_THAT(span.data(), TypedEq<void*>(flat->Data() + 3)); + EXPECT_THAT(tree->length, Eq(5)); + + size_t avail = flat->Capacity() - 5; + span = tree->GetAppendBuffer(avail + 100); + EXPECT_THAT(span, SizeIs(avail)); + EXPECT_THAT(span.data(), TypedEq<void*>(flat->Data() + 5)); + EXPECT_THAT(tree->length, Eq(5 + avail)); + + CordRepBtree::Unref(tree); +} + +TEST(CordRepBtreeTest, Dump) { + // Handles nullptr + std::stringstream ss; + CordRepBtree::Dump(nullptr, ss); + CordRepBtree::Dump(nullptr, "Once upon a label", ss); + CordRepBtree::Dump(nullptr, "Once upon a label", false, ss); + CordRepBtree::Dump(nullptr, "Once upon a label", true, ss); + + // Cover legal edges + CordRepFlat* flat = MakeFlat("Hello world"); + CordRepExternal* external = MakeExternal("Hello external"); + CordRep* substr_flat = MakeSubstring(1, 6, CordRep::Ref(flat)); + CordRep* substr_external = MakeSubstring(2, 7, CordRep::Ref(external)); + + // Build tree + CordRepBtree* tree = CordRepBtree::Create(flat); + tree = CordRepBtree::Append(tree, external); + tree = CordRepBtree::Append(tree, substr_flat); + tree = CordRepBtree::Append(tree, substr_external); + + // Repeat until we have a tree + while (tree->height() == 0) { + tree = CordRepBtree::Append(tree, CordRep::Ref(flat)); + tree = CordRepBtree::Append(tree, CordRep::Ref(external)); + tree = CordRepBtree::Append(tree, CordRep::Ref(substr_flat)); + tree = CordRepBtree::Append(tree, CordRep::Ref(substr_external)); + } + + for (int api = 0; api <= 3; ++api) { + absl::string_view api_scope; + std::stringstream ss; + switch (api) { + case 0: + api_scope = "Bare"; + CordRepBtree::Dump(tree, ss); + break; + case 1: + api_scope = "Label only"; + CordRepBtree::Dump(tree, "Once upon a label", ss); + break; + case 2: + api_scope = "Label no content"; + CordRepBtree::Dump(tree, "Once upon a label", false, ss); + break; + default: + api_scope = "Label and content"; + CordRepBtree::Dump(tree, "Once upon a label", true, ss); + break; + } + SCOPED_TRACE(api_scope); + std::string str = ss.str(); + + // Contains Node(depth) / Leaf and private / shared indicators + EXPECT_THAT(str, AllOf(HasSubstr("Node(1)"), HasSubstr("Leaf"), + HasSubstr("Private"), HasSubstr("Shared"))); + + // Contains length and start offset of all data edges + EXPECT_THAT(str, AllOf(HasSubstr("len = 11"), HasSubstr("len = 14"), + HasSubstr("len = 6"), HasSubstr("len = 7"), + HasSubstr("start = 1"), HasSubstr("start = 2"))); + + // Contains address of all data edges + EXPECT_THAT( + str, AllOf(HasSubstr(absl::StrCat("0x", absl::Hex(flat))), + HasSubstr(absl::StrCat("0x", absl::Hex(external))), + HasSubstr(absl::StrCat("0x", absl::Hex(substr_flat))), + HasSubstr(absl::StrCat("0x", absl::Hex(substr_external))))); + + if (api != 0) { + // Contains label + EXPECT_THAT(str, HasSubstr("Once upon a label")); + } + + if (api != 3) { + // Does not contain contents + EXPECT_THAT(str, Not(AnyOf((HasSubstr("data = \"Hello world\""), + HasSubstr("data = \"Hello external\""), + HasSubstr("data = \"ello w\""), + HasSubstr("data = \"llo ext\""))))); + } else { + // Contains contents + EXPECT_THAT(str, AllOf((HasSubstr("data = \"Hello world\""), + HasSubstr("data = \"Hello external\""), + HasSubstr("data = \"ello w\""), + HasSubstr("data = \"llo ext\"")))); + } + } + + CordRep::Unref(tree); +} + +TEST(CordRepBtreeTest, IsValid) { + EXPECT_FALSE(CordRepBtree::IsValid(nullptr)); + + CordRepBtree* empty = CordRepBtree::New(0); + EXPECT_TRUE(CordRepBtree::IsValid(empty)); + CordRep::Unref(empty); + + for (bool as_tree : {false, true}) { + CordRepBtree* leaf = CordRepBtree::Create(MakeFlat("abc")); + CordRepBtree* tree = as_tree ? CordRepBtree::New(leaf) : nullptr; + CordRepBtree* check = as_tree ? tree : leaf; + + ASSERT_TRUE(CordRepBtree::IsValid(check)); + leaf->length--; + EXPECT_FALSE(CordRepBtree::IsValid(check)); + leaf->length++; + + ASSERT_TRUE(CordRepBtree::IsValid(check)); + leaf->tag--; + EXPECT_FALSE(CordRepBtree::IsValid(check)); + leaf->tag++; + + // Height + ASSERT_TRUE(CordRepBtree::IsValid(check)); + leaf->storage[0] = static_cast<uint8_t>(CordRepBtree::kMaxHeight + 1); + EXPECT_FALSE(CordRepBtree::IsValid(check)); + leaf->storage[0] = 1; + EXPECT_FALSE(CordRepBtree::IsValid(check)); + leaf->storage[0] = 0; + + // Begin + ASSERT_TRUE(CordRepBtree::IsValid(check)); + const uint8_t begin = leaf->storage[1]; + leaf->storage[1] = static_cast<uint8_t>(CordRepBtree::kMaxCapacity); + EXPECT_FALSE(CordRepBtree::IsValid(check)); + leaf->storage[1] = 2; + EXPECT_FALSE(CordRepBtree::IsValid(check)); + leaf->storage[1] = begin; + + // End + ASSERT_TRUE(CordRepBtree::IsValid(check)); + const uint8_t end = leaf->storage[2]; + leaf->storage[2] = static_cast<uint8_t>(CordRepBtree::kMaxCapacity + 1); + EXPECT_FALSE(CordRepBtree::IsValid(check)); + leaf->storage[2] = end; + + // DataEdge tag and value + ASSERT_TRUE(CordRepBtree::IsValid(check)); + CordRep* const edge = leaf->Edges()[0]; + const uint8_t tag = edge->tag; + CordRepBtreeTestPeer::SetEdge(leaf, begin, nullptr); + EXPECT_FALSE(CordRepBtree::IsValid(check)); + CordRepBtreeTestPeer::SetEdge(leaf, begin, edge); + edge->tag = BTREE; + EXPECT_FALSE(CordRepBtree::IsValid(check)); + edge->tag = tag; + + if (as_tree) { + ASSERT_TRUE(CordRepBtree::IsValid(check)); + leaf->length--; + EXPECT_FALSE(CordRepBtree::IsValid(check)); + leaf->length++; + + // Height + ASSERT_TRUE(CordRepBtree::IsValid(check)); + tree->storage[0] = static_cast<uint8_t>(2); + EXPECT_FALSE(CordRepBtree::IsValid(check)); + tree->storage[0] = 1; + + // Btree edge + ASSERT_TRUE(CordRepBtree::IsValid(check)); + CordRep* const edge = tree->Edges()[0]; + const uint8_t tag = edge->tag; + edge->tag = FLAT; + EXPECT_FALSE(CordRepBtree::IsValid(check)); + edge->tag = tag; + } + + ASSERT_TRUE(CordRepBtree::IsValid(check)); + CordRep::Unref(check); + } +} + +TEST(CordRepBtreeTest, AssertValid) { + CordRepBtree* tree = CordRepBtree::Create(MakeFlat("abc")); + const CordRepBtree* ctree = tree; + EXPECT_THAT(CordRepBtree::AssertValid(tree), Eq(tree)); + EXPECT_THAT(CordRepBtree::AssertValid(ctree), Eq(ctree)); + +#if defined(GTEST_HAS_DEATH_TEST) + CordRepBtree* nulltree = nullptr; + const CordRepBtree* cnulltree = nullptr; + EXPECT_DEBUG_DEATH( + EXPECT_THAT(CordRepBtree::AssertValid(nulltree), Eq(nulltree)), ".*"); + EXPECT_DEBUG_DEATH( + EXPECT_THAT(CordRepBtree::AssertValid(cnulltree), Eq(cnulltree)), ".*"); + + tree->length--; + EXPECT_DEBUG_DEATH(EXPECT_THAT(CordRepBtree::AssertValid(tree), Eq(tree)), + ".*"); + EXPECT_DEBUG_DEATH(EXPECT_THAT(CordRepBtree::AssertValid(ctree), Eq(ctree)), + ".*"); + tree->length++; +#endif + CordRep::Unref(tree); +} + +TEST(CordRepBtreeTest, CheckAssertValidShallowVsDeep) { + // Restore exhaustive validation on any exit. + const bool exhaustive_validation = cord_btree_exhaustive_validation.load(); + auto cleanup = absl::MakeCleanup([exhaustive_validation] { + cord_btree_exhaustive_validation.store(exhaustive_validation); + }); + + // Create a tree of at least 2 levels, and mess with the original flat, which + // should go undetected in shallow mode as the flat is too far away, but + // should be detected in forced non-shallow mode. + CordRep* flat = MakeFlat("abc"); + CordRepBtree* tree = CordRepBtree::Create(flat); + constexpr size_t max_cap = CordRepBtree::kMaxCapacity; + const size_t n = max_cap * max_cap * 2; + for (size_t i = 0; i < n; ++i) { + tree = CordRepBtree::Append(tree, MakeFlat("Hello world")); + } + flat->length = 100; + + cord_btree_exhaustive_validation.store(false); + EXPECT_FALSE(CordRepBtree::IsValid(tree)); + EXPECT_TRUE(CordRepBtree::IsValid(tree, true)); + EXPECT_FALSE(CordRepBtree::IsValid(tree, false)); + CordRepBtree::AssertValid(tree); + CordRepBtree::AssertValid(tree, true); +#if defined(GTEST_HAS_DEATH_TEST) + EXPECT_DEBUG_DEATH(CordRepBtree::AssertValid(tree, false), ".*"); +#endif + + cord_btree_exhaustive_validation.store(true); + EXPECT_FALSE(CordRepBtree::IsValid(tree)); + EXPECT_FALSE(CordRepBtree::IsValid(tree, true)); + EXPECT_FALSE(CordRepBtree::IsValid(tree, false)); +#if defined(GTEST_HAS_DEATH_TEST) + EXPECT_DEBUG_DEATH(CordRepBtree::AssertValid(tree), ".*"); + EXPECT_DEBUG_DEATH(CordRepBtree::AssertValid(tree, true), ".*"); +#endif + + flat->length = 3; + CordRep::Unref(tree); +} + +TEST_P(CordRepBtreeTest, Rebuild) { + for (size_t size : {3, 8, 100, 10000, 1000000}) { + SCOPED_TRACE(absl::StrCat("Rebuild @", size)); + + std::vector<CordRepFlat*> flats; + for (int i = 0; i < size; ++i) { + flats.push_back(CordRepFlat::New(2)); + flats.back()->Data()[0] = 'x'; + flats.back()->length = 1; + } + + // Build the tree into 'right', and each so many 'split_limit' edges, + // combine 'left' + 'right' into a new 'left', and start a new 'right'. + // This guarantees we get a reasonable amount of chaos in the tree. + size_t split_count = 0; + size_t split_limit = 3; + auto it = flats.begin(); + CordRepBtree* left = nullptr; + CordRepBtree* right = CordRepBtree::New(*it); + while (++it != flats.end()) { + if (++split_count >= split_limit) { + split_limit += split_limit / 16; + left = left ? CordRepBtree::Append(left, right) : right; + right = CordRepBtree::New(*it); + } else { + right = CordRepBtree::Append(right, *it); + } + } + + // Finalize tree + left = left ? CordRepBtree::Append(left, right) : right; + + // Rebuild + AutoUnref ref; + left = ref.Add(CordRepBtree::Rebuild(ref.RefIf(shared(), left))); + ASSERT_TRUE(CordRepBtree::IsValid(left)); + + // Verify we have the exact same edges in the exact same order. + bool ok = true; + it = flats.begin(); + CordVisitReps(left, [&](CordRep* edge) { + if (edge->tag < FLAT) return; + ok = ok && (it != flats.end() && *it++ == edge); + }); + EXPECT_TRUE(ok && it == flats.end()) << "Rebuild edges mismatch"; + } +} + +// Convenience helper for CordRepBtree::ExtractAppendBuffer +CordRepBtree::ExtractResult ExtractLast(CordRepBtree* input, size_t cap = 1) { + return CordRepBtree::ExtractAppendBuffer(input, cap); +} + +TEST(CordRepBtreeTest, ExtractAppendBufferLeafSingleFlat) { + CordRep* flat = MakeFlat("Abc"); + CordRepBtree* leaf = CordRepBtree::Create(flat); + EXPECT_THAT(ExtractLast(leaf), EqExtractResult(nullptr, flat)); + CordRep::Unref(flat); +} + +TEST(CordRepBtreeTest, ExtractAppendBufferNodeSingleFlat) { + CordRep* flat = MakeFlat("Abc"); + CordRepBtree* leaf = CordRepBtree::Create(flat); + CordRepBtree* node = CordRepBtree::New(leaf); + EXPECT_THAT(ExtractLast(node), EqExtractResult(nullptr, flat)); + CordRep::Unref(flat); +} + +TEST(CordRepBtreeTest, ExtractAppendBufferLeafTwoFlats) { + std::vector<CordRep*> flats = CreateFlatsFromString("abcdef", 3); + CordRepBtree* leaf = CreateTree(flats); + EXPECT_THAT(ExtractLast(leaf), EqExtractResult(flats[0], flats[1])); + CordRep::Unref(flats[0]); + CordRep::Unref(flats[1]); +} + +TEST(CordRepBtreeTest, ExtractAppendBufferNodeTwoFlats) { + std::vector<CordRep*> flats = CreateFlatsFromString("abcdef", 3); + CordRepBtree* leaf = CreateTree(flats); + CordRepBtree* node = CordRepBtree::New(leaf); + EXPECT_THAT(ExtractLast(node), EqExtractResult(flats[0], flats[1])); + CordRep::Unref(flats[0]); + CordRep::Unref(flats[1]); +} + +TEST(CordRepBtreeTest, ExtractAppendBufferNodeTwoFlatsInTwoLeafs) { + std::vector<CordRep*> flats = CreateFlatsFromString("abcdef", 3); + CordRepBtree* leaf1 = CordRepBtree::Create(flats[0]); + CordRepBtree* leaf2 = CordRepBtree::Create(flats[1]); + CordRepBtree* node = CordRepBtree::New(leaf1, leaf2); + EXPECT_THAT(ExtractLast(node), EqExtractResult(flats[0], flats[1])); + CordRep::Unref(flats[0]); + CordRep::Unref(flats[1]); +} + +TEST(CordRepBtreeTest, ExtractAppendBufferLeafThreeFlats) { + std::vector<CordRep*> flats = CreateFlatsFromString("abcdefghi", 3); + CordRepBtree* leaf = CreateTree(flats); + EXPECT_THAT(ExtractLast(leaf), EqExtractResult(leaf, flats[2])); + CordRep::Unref(flats[2]); + CordRep::Unref(leaf); +} + +TEST(CordRepBtreeTest, ExtractAppendBufferNodeThreeFlatsRightNoFolding) { + CordRep* flat = MakeFlat("Abc"); + std::vector<CordRep*> flats = CreateFlatsFromString("defghi", 3); + CordRepBtree* leaf1 = CordRepBtree::Create(flat); + CordRepBtree* leaf2 = CreateTree(flats); + CordRepBtree* node = CordRepBtree::New(leaf1, leaf2); + EXPECT_THAT(ExtractLast(node), EqExtractResult(node, flats[1])); + EXPECT_THAT(node->Edges(), ElementsAre(leaf1, leaf2)); + EXPECT_THAT(leaf1->Edges(), ElementsAre(flat)); + EXPECT_THAT(leaf2->Edges(), ElementsAre(flats[0])); + CordRep::Unref(node); + CordRep::Unref(flats[1]); +} + +TEST(CordRepBtreeTest, ExtractAppendBufferNodeThreeFlatsRightLeafFolding) { + CordRep* flat = MakeFlat("Abc"); + std::vector<CordRep*> flats = CreateFlatsFromString("defghi", 3); + CordRepBtree* leaf1 = CreateTree(flats); + CordRepBtree* leaf2 = CordRepBtree::Create(flat); + CordRepBtree* node = CordRepBtree::New(leaf1, leaf2); + EXPECT_THAT(ExtractLast(node), EqExtractResult(leaf1, flat)); + EXPECT_THAT(leaf1->Edges(), ElementsAreArray(flats)); + CordRep::Unref(leaf1); + CordRep::Unref(flat); +} + +TEST(CordRepBtreeTest, ExtractAppendBufferNoCapacity) { + std::vector<CordRep*> flats = CreateFlatsFromString("abcdef", 3); + CordRepBtree* leaf = CreateTree(flats); + size_t avail = flats[1]->flat()->Capacity() - flats[1]->length; + EXPECT_THAT(ExtractLast(leaf, avail + 1), EqExtractResult(leaf, nullptr)); + EXPECT_THAT(ExtractLast(leaf, avail), EqExtractResult(flats[0], flats[1])); + CordRep::Unref(flats[0]); + CordRep::Unref(flats[1]); +} + +TEST(CordRepBtreeTest, ExtractAppendBufferNotFlat) { + std::vector<CordRep*> flats = CreateFlatsFromString("abcdef", 3); + auto substr = MakeSubstring(1, 2, flats[1]); + CordRepBtree* leaf = CreateTree({flats[0], substr}); + EXPECT_THAT(ExtractLast(leaf), EqExtractResult(leaf, nullptr)); + CordRep::Unref(leaf); +} + +TEST(CordRepBtreeTest, ExtractAppendBufferShared) { + std::vector<CordRep*> flats = CreateFlatsFromString("abcdef", 3); + CordRepBtree* leaf = CreateTree(flats); + + CordRep::Ref(flats[1]); + EXPECT_THAT(ExtractLast(leaf), EqExtractResult(leaf, nullptr)); + CordRep::Unref(flats[1]); + + CordRep::Ref(leaf); + EXPECT_THAT(ExtractLast(leaf), EqExtractResult(leaf, nullptr)); + CordRep::Unref(leaf); + + CordRepBtree* node = CordRepBtree::New(leaf); + CordRep::Ref(node); + EXPECT_THAT(ExtractLast(node), EqExtractResult(node, nullptr)); + CordRep::Unref(node); + + CordRep::Unref(node); +} + +} // namespace +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/absl/strings/internal/cord_rep_consume.cc b/absl/strings/internal/cord_rep_consume.cc new file mode 100644 index 00000000..20a55797 --- /dev/null +++ b/absl/strings/internal/cord_rep_consume.cc @@ -0,0 +1,62 @@ +// Copyright 2021 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/cord_rep_consume.h" + +#include <array> +#include <utility> + +#include "absl/container/inlined_vector.h" +#include "absl/functional/function_ref.h" +#include "absl/strings/internal/cord_internal.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { + +namespace { + +// Unrefs the provided `substring`, and returns `substring->child` +// Adds or assumes a reference on `substring->child` +CordRep* ClipSubstring(CordRepSubstring* substring) { + CordRep* child = substring->child; + if (substring->refcount.IsOne()) { + delete substring; + } else { + CordRep::Ref(child); + CordRep::Unref(substring); + } + return child; +} + +} // namespace + +void Consume(CordRep* rep, ConsumeFn consume_fn) { + size_t offset = 0; + size_t length = rep->length; + + if (rep->tag == SUBSTRING) { + offset += rep->substring()->start; + rep = ClipSubstring(rep->substring()); + } + consume_fn(rep, offset, length); +} + +void ReverseConsume(CordRep* rep, ConsumeFn consume_fn) { + return Consume(rep, std::move(consume_fn)); +} + +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/absl/strings/internal/cord_rep_consume.h b/absl/strings/internal/cord_rep_consume.h new file mode 100644 index 00000000..d46fca2b --- /dev/null +++ b/absl/strings/internal/cord_rep_consume.h @@ -0,0 +1,50 @@ +// Copyright 2021 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_STRINGS_INTERNAL_CORD_REP_CONSUME_H_ +#define ABSL_STRINGS_INTERNAL_CORD_REP_CONSUME_H_ + +#include <functional> + +#include "absl/functional/function_ref.h" +#include "absl/strings/internal/cord_internal.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { + +// Functor for the Consume() and ReverseConsume() functions: +// void ConsumeFunc(CordRep* rep, size_t offset, size_t length); +// See the Consume() and ReverseConsume() function comments for documentation. +using ConsumeFn = FunctionRef<void(CordRep*, size_t, size_t)>; + +// Consume() and ReverseConsume() consume CONCAT based trees and invoke the +// provided functor with the contained nodes in the proper forward or reverse +// order, which is used to convert CONCAT trees into other tree or cord data. +// All CONCAT and SUBSTRING nodes are processed internally. The 'offset` +// parameter of the functor is non-zero for any nodes below SUBSTRING nodes. +// It's up to the caller to form these back into SUBSTRING nodes or otherwise +// store offset / prefix information. These functions are intended to be used +// only for migration / transitional code where due to factors such as ODR +// violations, we can not 100% guarantee that all code respects 'new format' +// settings and flags, so we need to be able to parse old data on the fly until +// all old code is deprecated / no longer the default format. +void Consume(CordRep* rep, ConsumeFn consume_fn); +void ReverseConsume(CordRep* rep, ConsumeFn consume_fn); + +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_INTERNAL_CORD_REP_CONSUME_H_ diff --git a/absl/strings/internal/cord_rep_crc.cc b/absl/strings/internal/cord_rep_crc.cc new file mode 100644 index 00000000..ee140354 --- /dev/null +++ b/absl/strings/internal/cord_rep_crc.cc @@ -0,0 +1,54 @@ +// Copyright 2021 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/cord_rep_crc.h" + +#include <cassert> +#include <cstdint> + +#include "absl/base/config.h" +#include "absl/strings/internal/cord_internal.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { + +CordRepCrc* CordRepCrc::New(CordRep* child, uint32_t crc) { + assert(child != nullptr); + if (child->IsCrc()) { + if (child->refcount.IsOne()) { + child->crc()->crc = crc; + return child->crc(); + } + CordRep* old = child; + child = old->crc()->child; + CordRep::Ref(child); + CordRep::Unref(old); + } + auto* new_cordrep = new CordRepCrc; + new_cordrep->length = child->length; + new_cordrep->tag = cord_internal::CRC; + new_cordrep->child = child; + new_cordrep->crc = crc; + return new_cordrep; +} + +void CordRepCrc::Destroy(CordRepCrc* node) { + CordRep::Unref(node->child); + delete node; +} + +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/absl/strings/internal/cord_rep_crc.h b/absl/strings/internal/cord_rep_crc.h new file mode 100644 index 00000000..5294b0d1 --- /dev/null +++ b/absl/strings/internal/cord_rep_crc.h @@ -0,0 +1,102 @@ +// Copyright 2021 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_STRINGS_INTERNAL_CORD_REP_CRC_H_ +#define ABSL_STRINGS_INTERNAL_CORD_REP_CRC_H_ + +#include <cassert> +#include <cstdint> + +#include "absl/base/config.h" +#include "absl/base/optimization.h" +#include "absl/strings/internal/cord_internal.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { + +// CordRepCrc is a CordRep node intended only to appear at the top level of a +// cord tree. It associates an "expected CRC" with the contained data, to allow +// for easy passage of checksum data in Cord data flows. +// +// From Cord's perspective, the crc value has no semantics; any validation of +// the contained checksum is the user's responsibility. +struct CordRepCrc : public CordRep { + CordRep* child; + uint32_t crc; + + // Consumes `child` and returns a CordRepCrc prefixed tree containing `child`. + // If the specified `child` is itself a CordRepCrc node, then this method + // either replaces the existing node, or directly updates the crc value in it + // depending on the node being shared or not, i.e.: refcount.IsOne(). + // `child` must not be null. Never returns null. + static CordRepCrc* New(CordRep* child, uint32_t crc); + + // Destroys (deletes) the provided node. `node` must not be null. + static void Destroy(CordRepCrc* node); +}; + +// Consumes `rep` and returns a CordRep* with any outer CordRepCrc wrapper +// removed. This is usually a no-op (returning `rep`), but this will remove and +// unref an outer CordRepCrc node. +inline CordRep* RemoveCrcNode(CordRep* rep) { + assert(rep != nullptr); + if (ABSL_PREDICT_FALSE(rep->IsCrc())) { + CordRep* child = rep->crc()->child; + if (rep->refcount.IsOne()) { + delete rep->crc(); + } else { + CordRep::Ref(child); + CordRep::Unref(rep); + } + return child; + } + return rep; +} + +// Returns `rep` if it is not a CordRepCrc node, or its child if it is. +// Does not consume or create a reference on `rep` or the returned value. +inline CordRep* SkipCrcNode(CordRep* rep) { + assert(rep != nullptr); + if (ABSL_PREDICT_FALSE(rep->IsCrc())) { + return rep->crc()->child; + } else { + return rep; + } +} + +inline const CordRep* SkipCrcNode(const CordRep* rep) { + assert(rep != nullptr); + if (ABSL_PREDICT_FALSE(rep->IsCrc())) { + return rep->crc()->child; + } else { + return rep; + } +} + +inline CordRepCrc* CordRep::crc() { + assert(IsCrc()); + return static_cast<CordRepCrc*>(this); +} + +inline const CordRepCrc* CordRep::crc() const { + assert(IsCrc()); + return static_cast<const CordRepCrc*>(this); +} + +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_INTERNAL_CORD_REP_CRC_H_ diff --git a/absl/strings/internal/cord_rep_crc_test.cc b/absl/strings/internal/cord_rep_crc_test.cc new file mode 100644 index 00000000..80f53485 --- /dev/null +++ b/absl/strings/internal/cord_rep_crc_test.cc @@ -0,0 +1,115 @@ +// Copyright 2021 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/cord_rep_crc.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/config.h" +#include "absl/strings/internal/cord_internal.h" +#include "absl/strings/internal/cord_rep_test_util.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { +namespace { + +using ::absl::cordrep_testing::MakeFlat; +using ::testing::Eq; +using ::testing::Ne; + +#if !defined(NDEBUG) && GTEST_HAS_DEATH_TEST + +TEST(CordRepCrc, NewWithNullPtr) { + EXPECT_DEATH(CordRepCrc::New(nullptr, 0), ""); +} + +TEST(CordRepCrc, RemoveCrcWithNullptr) { + EXPECT_DEATH(RemoveCrcNode(nullptr), ""); +} + +#endif // !NDEBUG && GTEST_HAS_DEATH_TEST + +TEST(CordRepCrc, NewDestroy) { + CordRep* rep = cordrep_testing::MakeFlat("Hello world"); + CordRepCrc* crc = CordRepCrc::New(rep, 12345); + EXPECT_TRUE(crc->refcount.IsOne()); + EXPECT_THAT(crc->child, Eq(rep)); + EXPECT_THAT(crc->crc, Eq(12345)); + EXPECT_TRUE(rep->refcount.IsOne()); + CordRepCrc::Destroy(crc); +} + +TEST(CordRepCrc, NewExistingCrcNotShared) { + CordRep* rep = cordrep_testing::MakeFlat("Hello world"); + CordRepCrc* crc = CordRepCrc::New(rep, 12345); + CordRepCrc* new_crc = CordRepCrc::New(crc, 54321); + EXPECT_THAT(new_crc, Eq(crc)); + EXPECT_TRUE(new_crc->refcount.IsOne()); + EXPECT_THAT(new_crc->child, Eq(rep)); + EXPECT_THAT(new_crc->crc, Eq(54321)); + EXPECT_TRUE(rep->refcount.IsOne()); + CordRepCrc::Destroy(new_crc); +} + +TEST(CordRepCrc, NewExistingCrcShared) { + CordRep* rep = cordrep_testing::MakeFlat("Hello world"); + CordRepCrc* crc = CordRepCrc::New(rep, 12345); + CordRep::Ref(crc); + CordRepCrc* new_crc = CordRepCrc::New(crc, 54321); + + EXPECT_THAT(new_crc, Ne(crc)); + EXPECT_TRUE(new_crc->refcount.IsOne()); + EXPECT_TRUE(crc->refcount.IsOne()); + EXPECT_FALSE(rep->refcount.IsOne()); + EXPECT_THAT(crc->child, Eq(rep)); + EXPECT_THAT(new_crc->child, Eq(rep)); + EXPECT_THAT(crc->crc, Eq(12345)); + EXPECT_THAT(new_crc->crc, Eq(54321)); + + CordRep::Unref(crc); + CordRep::Unref(new_crc); +} + +TEST(CordRepCrc, RemoveCrcNotCrc) { + CordRep* rep = cordrep_testing::MakeFlat("Hello world"); + CordRep* nocrc = RemoveCrcNode(rep); + EXPECT_THAT(nocrc, Eq(rep)); + CordRep::Unref(nocrc); +} + +TEST(CordRepCrc, RemoveCrcNotShared) { + CordRep* rep = cordrep_testing::MakeFlat("Hello world"); + CordRepCrc* crc = CordRepCrc::New(rep, 12345); + CordRep* nocrc = RemoveCrcNode(crc); + EXPECT_THAT(nocrc, Eq(rep)); + EXPECT_TRUE(rep->refcount.IsOne()); + CordRep::Unref(nocrc); +} + +TEST(CordRepCrc, RemoveCrcShared) { + CordRep* rep = cordrep_testing::MakeFlat("Hello world"); + CordRepCrc* crc = CordRepCrc::New(rep, 12345); + CordRep::Ref(crc); + CordRep* nocrc = RemoveCrcNode(crc); + EXPECT_THAT(nocrc, Eq(rep)); + EXPECT_FALSE(rep->refcount.IsOne()); + CordRep::Unref(nocrc); + CordRep::Unref(crc); +} + +} // namespace +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/absl/strings/internal/cord_rep_flat.h b/absl/strings/internal/cord_rep_flat.h index a98aa9df..e3e27fcd 100644 --- a/absl/strings/internal/cord_rep_flat.h +++ b/absl/strings/internal/cord_rep_flat.h @@ -20,6 +20,8 @@ #include <cstdint> #include <memory> +#include "absl/base/config.h" +#include "absl/base/macros.h" #include "absl/strings/internal/cord_internal.h" namespace absl { @@ -42,23 +44,45 @@ static constexpr size_t kMinFlatSize = 32; static constexpr size_t kMaxFlatSize = 4096; static constexpr size_t kMaxFlatLength = kMaxFlatSize - kFlatOverhead; static constexpr size_t kMinFlatLength = kMinFlatSize - kFlatOverhead; +static constexpr size_t kMaxLargeFlatSize = 256 * 1024; +static constexpr size_t kMaxLargeFlatLength = kMaxLargeFlatSize - kFlatOverhead; +// kTagBase should make the Size <--> Tag computation resilient +// against changes to the value of FLAT when we add a new tag.. +static constexpr uint8_t kTagBase = FLAT - 4; + +// Converts the provided rounded size to the corresponding tag constexpr uint8_t AllocatedSizeToTagUnchecked(size_t size) { - return static_cast<uint8_t>((size <= 1024) ? size / 8 - : 128 + size / 32 - 1024 / 32); + return static_cast<uint8_t>(size <= 512 ? kTagBase + size / 8 + : size <= 8192 + ? kTagBase + 512 / 8 + size / 64 - 512 / 64 + : kTagBase + 512 / 8 + ((8192 - 512) / 64) + + size / 4096 - 8192 / 4096); +} + +// Converts the provided tag to the corresponding allocated size +constexpr size_t TagToAllocatedSize(uint8_t tag) { + return (tag <= kTagBase + 512 / 8) ? tag * 8 - kTagBase * 8 + : (tag <= kTagBase + (512 / 8) + ((8192 - 512) / 64)) + ? 512 + tag * 64 - kTagBase * 64 - 512 / 8 * 64 + : 8192 + tag * 4096 - kTagBase * 4096 - + ((512 / 8) + ((8192 - 512) / 64)) * 4096; } -static_assert(kMinFlatSize / 8 >= FLAT, ""); -static_assert(AllocatedSizeToTagUnchecked(kMaxFlatSize) <= MAX_FLAT_TAG, ""); +static_assert(AllocatedSizeToTagUnchecked(kMinFlatSize) == FLAT, ""); +static_assert(AllocatedSizeToTagUnchecked(kMaxLargeFlatSize) == MAX_FLAT_TAG, + ""); -// Helper functions for rounded div, and rounding to exact sizes. -constexpr size_t DivUp(size_t n, size_t m) { return (n + m - 1) / m; } -constexpr size_t RoundUp(size_t n, size_t m) { return DivUp(n, m) * m; } +// RoundUp logically performs `((n + m - 1) / m) * m` to round up to the nearest +// multiple of `m`, optimized for the invariant that `m` is a power of 2. +constexpr size_t RoundUp(size_t n, size_t m) { + return (n + m - 1) & (0 - m); +} // Returns the size to the nearest equal or larger value that can be // expressed exactly as a tag value. inline size_t RoundUpForTag(size_t size) { - return RoundUp(size, (size <= 1024) ? 8 : 32); + return RoundUp(size, (size <= 512) ? 8 : (size <= 8192 ? 64 : 4096)); } // Converts the allocated size to a tag, rounding down if the size @@ -71,26 +95,26 @@ inline uint8_t AllocatedSizeToTag(size_t size) { return tag; } -// Converts the provided tag to the corresponding allocated size -constexpr size_t TagToAllocatedSize(uint8_t tag) { - return (tag <= 128) ? (tag * 8) : (1024 + (tag - 128) * 32); -} - // Converts the provided tag to the corresponding available data length constexpr size_t TagToLength(uint8_t tag) { return TagToAllocatedSize(tag) - kFlatOverhead; } // Enforce that kMaxFlatSize maps to a well-known exact tag value. -static_assert(TagToAllocatedSize(224) == kMaxFlatSize, "Bad tag logic"); +static_assert(TagToAllocatedSize(MAX_FLAT_TAG) == kMaxLargeFlatSize, + "Bad tag logic"); struct CordRepFlat : public CordRep { + // Tag for explicit 'large flat' allocation + struct Large {}; + // Creates a new flat node. - static CordRepFlat* New(size_t len) { + template <size_t max_flat_size, typename... Args> + static CordRepFlat* NewImpl(size_t len, Args... args ABSL_ATTRIBUTE_UNUSED) { if (len <= kMinFlatLength) { len = kMinFlatLength; - } else if (len > kMaxFlatLength) { - len = kMaxFlatLength; + } else if (len > max_flat_size - kFlatOverhead) { + len = max_flat_size - kFlatOverhead; } // Round size up so it matches a size we can exactly express in a tag. @@ -101,6 +125,12 @@ struct CordRepFlat : public CordRep { return rep; } + static CordRepFlat* New(size_t len) { return NewImpl<kMaxFlatSize>(len); } + + static CordRepFlat* New(Large, size_t len) { + return NewImpl<kMaxLargeFlatSize>(len); + } + // Deletes a CordRepFlat instance created previously through a call to New(). // Flat CordReps are allocated and constructed with raw ::operator new and // placement new, and must be destructed and deallocated accordingly. @@ -117,9 +147,20 @@ struct CordRepFlat : public CordRep { #endif } + // Create a CordRepFlat containing `data`, with an optional additional + // extra capacity of up to `extra` bytes. Requires that `data.size()` + // is less than kMaxFlatLength. + static CordRepFlat* Create(absl::string_view data, size_t extra = 0) { + assert(data.size() <= kMaxFlatLength); + CordRepFlat* flat = New(data.size() + (std::min)(extra, kMaxFlatLength)); + memcpy(flat->Data(), data.data(), data.size()); + flat->length = data.size(); + return flat; + } + // Returns a pointer to the data inside this flat rep. - char* Data() { return storage; } - const char* Data() const { return storage; } + char* Data() { return reinterpret_cast<char*>(storage); } + const char* Data() const { return reinterpret_cast<const char*>(storage); } // Returns the maximum capacity (payload size) of this instance. size_t Capacity() const { return TagToLength(tag); } diff --git a/absl/strings/internal/cord_rep_ring.cc b/absl/strings/internal/cord_rep_ring.cc index 4d31d1d9..af2fc768 100644 --- a/absl/strings/internal/cord_rep_ring.cc +++ b/absl/strings/internal/cord_rep_ring.cc @@ -26,21 +26,13 @@ #include "absl/base/macros.h" #include "absl/container/inlined_vector.h" #include "absl/strings/internal/cord_internal.h" +#include "absl/strings/internal/cord_rep_consume.h" #include "absl/strings/internal/cord_rep_flat.h" namespace absl { ABSL_NAMESPACE_BEGIN namespace cord_internal { -// See https://bugs.llvm.org/show_bug.cgi?id=48477 -#ifdef __clang__ -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wshadow" -#if __has_warning("-Wshadow-field") -#pragma clang diagnostic ignored "-Wshadow-field" -#endif -#endif - namespace { using index_type = CordRepRing::index_type; @@ -48,7 +40,7 @@ using index_type = CordRepRing::index_type; enum class Direction { kForward, kReversed }; inline bool IsFlatOrExternal(CordRep* rep) { - return rep->tag >= FLAT || rep->tag == EXTERNAL; + return rep->IsFlat() || rep->IsExternal(); } // Verifies that n + extra <= kMaxCapacity: throws std::length_error otherwise. @@ -58,14 +50,6 @@ inline void CheckCapacity(size_t n, size_t extra) { } } -// Removes a reference from `rep` only. -// Asserts that the refcount after decrement is not zero. -inline bool UnrefNeverOne(CordRep* rep) { - bool result = rep->refcount.Decrement(); - assert(result); - return result; -} - // Creates a flat from the provided string data, allocating up to `extra` // capacity in the returned flat depending on kMaxFlatLength limitations. // Requires `len` to be less or equal to `kMaxFlatLength` @@ -77,40 +61,6 @@ CordRepFlat* CreateFlat(const char* s, size_t n, size_t extra = 0) { // NOLINT return rep; } -// Unrefs the provided `substring`, and returns `substring->child` -// Adds or assumes a reference on `substring->child` -CordRep* ClipSubstring(CordRepSubstring* substring) { - CordRep* child = substring->child; - if (substring->refcount.IsOne()) { - delete substring; - } else { - CordRep::Ref(child); - if (ABSL_PREDICT_FALSE(!substring->refcount.Decrement())) { - UnrefNeverOne(child); - delete substring; - } - } - return child; -} - -// Unrefs the provided `concat`, and returns `{concat->left, concat->right}` -// Adds or assumes a reference on `concat->left` and `concat->right`. -std::pair<CordRep*, CordRep*> ClipConcat(CordRepConcat* concat) { - auto result = std::make_pair(concat->left, concat->right); - if (concat->refcount.IsOne()) { - delete concat; - } else { - CordRep::Ref(result.first); - CordRep::Ref(result.second); - if (ABSL_PREDICT_FALSE(!concat->refcount.Decrement())) { - UnrefNeverOne(result.first); - UnrefNeverOne(result.second); - delete concat; - } - } - return result; -} - // Unrefs the entries in `[head, tail)`. // Requires all entries to be a FLAT or EXTERNAL node. void UnrefEntries(const CordRepRing* rep, index_type head, index_type tail) { @@ -126,79 +76,6 @@ void UnrefEntries(const CordRepRing* rep, index_type head, index_type tail) { }); } -template <typename F> -void Consume(Direction direction, CordRep* rep, F&& fn) { - size_t offset = 0; - size_t length = rep->length; - struct Entry { - CordRep* rep; - size_t offset; - size_t length; - }; - absl::InlinedVector<Entry, 40> stack; - - for (;;) { - if (rep->tag >= FLAT || rep->tag == EXTERNAL || rep->tag == RING) { - fn(rep, offset, length); - if (stack.empty()) return; - - rep = stack.back().rep; - offset = stack.back().offset; - length = stack.back().length; - stack.pop_back(); - } else if (rep->tag == SUBSTRING) { - offset += rep->substring()->start; - rep = ClipSubstring(rep->substring()); - } else if (rep->tag == CONCAT) { - auto res = ClipConcat(rep->concat()); - CordRep* left = res.first; - CordRep* right = res.second; - - if (left->length <= offset) { - // Don't need left node - offset -= left->length; - CordRep::Unref(left); - rep = right; - continue; - } - - size_t length_left = left->length - offset; - if (length_left >= length) { - // Don't need right node - CordRep::Unref(right); - rep = left; - continue; - } - - // Need both nodes - size_t length_right = length - length_left; - if (direction == Direction::kReversed) { - stack.push_back({left, offset, length_left}); - rep = right; - offset = 0; - length = length_right; - } else { - stack.push_back({right, 0, length_right}); - rep = left; - length = length_left; - } - } else { - assert("Valid tag" == nullptr); - return; - } - } -} - -template <typename F> -void Consume(CordRep* rep, F&& fn) { - return Consume(Direction::kForward, rep, std::forward<F>(fn)); -} - -template <typename F> -void RConsume(CordRep* rep, F&& fn) { - return Consume(Direction::kReversed, rep, std::forward<F>(fn)); -} - } // namespace std::ostream& operator<<(std::ostream& s, const CordRepRing& rep) { @@ -252,7 +129,9 @@ class CordRepRing::Filler { index_type pos_; }; -constexpr size_t CordRepRing::kMaxCapacity; // NOLINT: needed for c++11 +#ifdef ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL +constexpr size_t CordRepRing::kMaxCapacity; +#endif bool CordRepRing::IsValid(std::ostream& output) const { if (capacity_ == 0) { @@ -301,7 +180,7 @@ bool CordRepRing::IsValid(std::ostream& output) const { if (offset >= child->length || entry_length > child->length - offset) { output << "entry[" << head << "] has offset " << offset << " and entry length " << entry_length - << " which are outside of the childs length of " << child->length; + << " which are outside of the child's length of " << child->length; return false; } @@ -352,7 +231,7 @@ void CordRepRing::SetCapacityForTesting(size_t capacity) { } void CordRepRing::Delete(CordRepRing* rep) { - assert(rep != nullptr && rep->tag == RING); + assert(rep != nullptr && rep->IsRing()); #if defined(__cpp_sized_deallocation) size_t size = AllocSize(rep->capacity_); rep->~CordRepRing(); @@ -400,10 +279,11 @@ CordRepRing* CordRepRing::Mutable(CordRepRing* rep, size_t extra) { // Get current number of entries, and check for max capacity. size_t entries = rep->entries(); - size_t min_extra = (std::max)(extra, rep->capacity() * 2 - entries); if (!rep->refcount.IsOne()) { - return Copy(rep, rep->head(), rep->tail(), min_extra); + return Copy(rep, rep->head(), rep->tail(), extra); } else if (entries + extra > rep->capacity()) { + const size_t min_grow = rep->capacity() + rep->capacity() / 2; + const size_t min_extra = (std::max)(extra, min_grow - entries); CordRepRing* newrep = CordRepRing::New(entries, min_extra); newrep->Fill<false>(rep, rep->head(), rep->tail()); CordRepRing::Delete(rep); @@ -449,12 +329,12 @@ Span<char> CordRepRing::GetPrependBuffer(size_t size) { } CordRepRing* CordRepRing::CreateFromLeaf(CordRep* child, size_t offset, - size_t length, size_t extra) { + size_t len, size_t extra) { CordRepRing* rep = CordRepRing::New(1, extra); rep->head_ = 0; rep->tail_ = rep->advance(0); - rep->length = length; - rep->entry_end_pos()[0] = length; + rep->length = len; + rep->entry_end_pos()[0] = len; rep->entry_child()[0] = child; rep->entry_data_offset()[0] = static_cast<offset_type>(offset); return Validate(rep); @@ -462,16 +342,16 @@ CordRepRing* CordRepRing::CreateFromLeaf(CordRep* child, size_t offset, CordRepRing* CordRepRing::CreateSlow(CordRep* child, size_t extra) { CordRepRing* rep = nullptr; - Consume(child, [&](CordRep* child, size_t offset, size_t length) { - if (IsFlatOrExternal(child)) { - rep = rep ? AppendLeaf(rep, child, offset, length) - : CreateFromLeaf(child, offset, length, extra); + Consume(child, [&](CordRep* child_arg, size_t offset, size_t len) { + if (IsFlatOrExternal(child_arg)) { + rep = rep ? AppendLeaf(rep, child_arg, offset, len) + : CreateFromLeaf(child_arg, offset, len, extra); } else if (rep) { - rep = AddRing<AddMode::kAppend>(rep, child->ring(), offset, length); - } else if (offset == 0 && child->length == length) { - rep = Mutable(child->ring(), extra); + rep = AddRing<AddMode::kAppend>(rep, child_arg->ring(), offset, len); + } else if (offset == 0 && child_arg->length == len) { + rep = Mutable(child_arg->ring(), extra); } else { - rep = SubRing(child->ring(), offset, length, extra); + rep = SubRing(child_arg->ring(), offset, len, extra); } }); return Validate(rep, nullptr, __LINE__); @@ -482,7 +362,7 @@ CordRepRing* CordRepRing::Create(CordRep* child, size_t extra) { if (IsFlatOrExternal(child)) { return CreateFromLeaf(child, 0, length, extra); } - if (child->tag == RING) { + if (child->IsRing()) { return Mutable(child->ring(), extra); } return CreateSlow(child, extra); @@ -490,18 +370,18 @@ CordRepRing* CordRepRing::Create(CordRep* child, size_t extra) { template <CordRepRing::AddMode mode> CordRepRing* CordRepRing::AddRing(CordRepRing* rep, CordRepRing* ring, - size_t offset, size_t length) { + size_t offset, size_t len) { assert(offset < ring->length); constexpr bool append = mode == AddMode::kAppend; Position head = ring->Find(offset); - Position tail = ring->FindTail(head.index, offset + length); + Position tail = ring->FindTail(head.index, offset + len); const index_type entries = ring->entries(head.index, tail.index); rep = Mutable(rep, entries); // The delta for making ring[head].end_pos into 'len - offset' const pos_type delta_length = - (append ? rep->begin_pos_ + rep->length : rep->begin_pos_ - length) - + (append ? rep->begin_pos_ + rep->length : rep->begin_pos_ - len) - ring->entry_begin_pos(head.index) - head.offset; // Start filling at `tail`, or `entries` before `head` @@ -542,36 +422,36 @@ CordRepRing* CordRepRing::AddRing(CordRepRing* rep, CordRepRing* ring, } // Commit changes - rep->length += length; + rep->length += len; if (append) { rep->tail_ = filler.pos(); } else { rep->head_ = filler.head(); - rep->begin_pos_ -= length; + rep->begin_pos_ -= len; } return Validate(rep); } CordRepRing* CordRepRing::AppendSlow(CordRepRing* rep, CordRep* child) { - Consume(child, [&rep](CordRep* child, size_t offset, size_t length) { - if (child->tag == RING) { - rep = AddRing<AddMode::kAppend>(rep, child->ring(), offset, length); + Consume(child, [&rep](CordRep* child_arg, size_t offset, size_t len) { + if (child_arg->IsRing()) { + rep = AddRing<AddMode::kAppend>(rep, child_arg->ring(), offset, len); } else { - rep = AppendLeaf(rep, child, offset, length); + rep = AppendLeaf(rep, child_arg, offset, len); } }); return rep; } CordRepRing* CordRepRing::AppendLeaf(CordRepRing* rep, CordRep* child, - size_t offset, size_t length) { + size_t offset, size_t len) { rep = Mutable(rep, 1); index_type back = rep->tail_; const pos_type begin_pos = rep->begin_pos_ + rep->length; rep->tail_ = rep->advance(rep->tail_); - rep->length += length; - rep->entry_end_pos()[back] = begin_pos + length; + rep->length += len; + rep->entry_end_pos()[back] = begin_pos + len; rep->entry_child()[back] = child; rep->entry_data_offset()[back] = static_cast<offset_type>(offset); return Validate(rep, nullptr, __LINE__); @@ -582,31 +462,31 @@ CordRepRing* CordRepRing::Append(CordRepRing* rep, CordRep* child) { if (IsFlatOrExternal(child)) { return AppendLeaf(rep, child, 0, length); } - if (child->tag == RING) { + if (child->IsRing()) { return AddRing<AddMode::kAppend>(rep, child->ring(), 0, length); } return AppendSlow(rep, child); } CordRepRing* CordRepRing::PrependSlow(CordRepRing* rep, CordRep* child) { - RConsume(child, [&](CordRep* child, size_t offset, size_t length) { - if (IsFlatOrExternal(child)) { - rep = PrependLeaf(rep, child, offset, length); + ReverseConsume(child, [&](CordRep* child_arg, size_t offset, size_t len) { + if (IsFlatOrExternal(child_arg)) { + rep = PrependLeaf(rep, child_arg, offset, len); } else { - rep = AddRing<AddMode::kPrepend>(rep, child->ring(), offset, length); + rep = AddRing<AddMode::kPrepend>(rep, child_arg->ring(), offset, len); } }); return Validate(rep); } CordRepRing* CordRepRing::PrependLeaf(CordRepRing* rep, CordRep* child, - size_t offset, size_t length) { + size_t offset, size_t len) { rep = Mutable(rep, 1); index_type head = rep->retreat(rep->head_); pos_type end_pos = rep->begin_pos_; rep->head_ = head; - rep->length += length; - rep->begin_pos_ -= length; + rep->length += len; + rep->begin_pos_ -= len; rep->entry_end_pos()[head] = end_pos; rep->entry_child()[head] = child; rep->entry_data_offset()[head] = static_cast<offset_type>(offset); @@ -618,7 +498,7 @@ CordRepRing* CordRepRing::Prepend(CordRepRing* rep, CordRep* child) { if (IsFlatOrExternal(child)) { return PrependLeaf(rep, child, 0, length); } - if (child->tag == RING) { + if (child->IsRing()) { return AddRing<AddMode::kPrepend>(rep, child->ring(), 0, length); } return PrependSlow(rep, child); @@ -786,18 +666,18 @@ char CordRepRing::GetCharacter(size_t offset) const { } CordRepRing* CordRepRing::SubRing(CordRepRing* rep, size_t offset, - size_t length, size_t extra) { + size_t len, size_t extra) { assert(offset <= rep->length); - assert(offset <= rep->length - length); + assert(offset <= rep->length - len); - if (length == 0) { + if (len == 0) { CordRep::Unref(rep); return nullptr; } // Find position of first byte Position head = rep->Find(offset); - Position tail = rep->FindTail(head.index, offset + length); + Position tail = rep->FindTail(head.index, offset + len); const size_t new_entries = rep->entries(head.index, tail.index); if (rep->refcount.IsOne() && extra <= (rep->capacity() - new_entries)) { @@ -814,7 +694,7 @@ CordRepRing* CordRepRing::SubRing(CordRepRing* rep, size_t offset, } // Adjust begin_pos and length - rep->length = length; + rep->length = len; rep->begin_pos_ += offset; // Adjust head and tail blocks @@ -888,10 +768,6 @@ CordRepRing* CordRepRing::RemoveSuffix(CordRepRing* rep, size_t len, return Validate(rep); } -#ifdef __clang__ -#pragma clang diagnostic pop -#endif - } // namespace cord_internal ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/strings/internal/cord_rep_ring.h b/absl/strings/internal/cord_rep_ring.h index c74d3353..2000e21e 100644 --- a/absl/strings/internal/cord_rep_ring.h +++ b/absl/strings/internal/cord_rep_ring.h @@ -30,15 +30,6 @@ namespace absl { ABSL_NAMESPACE_BEGIN namespace cord_internal { -// See https://bugs.llvm.org/show_bug.cgi?id=48477 -#ifdef __clang__ -#pragma clang diagnostic push -#pragma clang diagnostic ignored "-Wshadow" -#if __has_warning("-Wshadow-field") -#pragma clang diagnostic ignored "-Wshadow-field" -#endif -#endif - // All operations modifying a ring buffer are implemented as static methods // requiring a CordRepRing instance with a reference adopted by the method. // @@ -210,23 +201,23 @@ class CordRepRing : public CordRep { // referencing up to `size` capacity directly before the existing data. Span<char> GetPrependBuffer(size_t size); - // Returns a cord ring buffer containing `length` bytes of data starting at + // Returns a cord ring buffer containing `len` bytes of data starting at // `offset`. If the input is not shared, this function will remove all head // and tail child nodes outside of the requested range, and adjust the new // head and tail nodes as required. If the input is shared, this function // returns a new instance sharing some or all of the nodes from the input. - static CordRepRing* SubRing(CordRepRing* r, size_t offset, size_t length, + static CordRepRing* SubRing(CordRepRing* r, size_t offset, size_t len, size_t extra = 0); - // Returns a cord ring buffer with the first `length` bytes removed. + // Returns a cord ring buffer with the first `len` bytes removed. // If the input is not shared, this function will remove all head child nodes // fully inside the first `length` bytes, and adjust the new head as required. // If the input is shared, this function returns a new instance sharing some // or all of the nodes from the input. - static CordRepRing* RemoveSuffix(CordRepRing* r, size_t length, + static CordRepRing* RemoveSuffix(CordRepRing* r, size_t len, size_t extra = 0); - // Returns a cord ring buffer with the last `length` bytes removed. + // Returns a cord ring buffer with the last `len` bytes removed. // If the input is not shared, this function will remove all head child nodes // fully inside the first `length` bytes, and adjust the new head as required. // If the input is shared, this function returns a new instance sharing some @@ -237,6 +228,18 @@ class CordRepRing : public CordRep { // Returns the character at `offset`. Requires that `offset < length`. char GetCharacter(size_t offset) const; + // Returns true if this instance manages a single contiguous buffer, in which + // case the (optional) output parameter `fragment` is set. Otherwise, the + // function returns false, and `fragment` is left unchanged. + bool IsFlat(absl::string_view* fragment) const; + + // Returns true if the data starting at `offset` with length `len` is + // managed by this instance inside a single contiguous buffer, in which case + // the (optional) output parameter `fragment` is set to the contiguous memory + // starting at offset `offset` with length `length`. Otherwise, the function + // returns false, and `fragment` is left unchanged. + bool IsFlat(size_t offset, size_t len, absl::string_view* fragment) const; + // Testing only: set capacity to requested capacity. void SetCapacityForTesting(size_t capacity); @@ -380,8 +383,8 @@ class CordRepRing : public CordRep { // Destroys the provided ring buffer, decrementing the reference count of all // contained child CordReps. The provided 1\`rep` should have a ref count of - // one (pre decrement destroy call observing `refcount.IsOne()`) or zero (post - // decrement destroy call observing `!refcount.Decrement()`). + // one (pre decrement destroy call observing `refcount.IsOne()`) or zero + // (post decrement destroy call observing `!refcount.Decrement()`). static void Destroy(CordRepRing* rep); // Returns a mutable reference to the logical end position array. @@ -461,10 +464,10 @@ class CordRepRing : public CordRep { size_t length, size_t extra); // Appends or prepends (depending on AddMode) the ring buffer in `ring' to - // `rep` starting at `offset` with length `length`. + // `rep` starting at `offset` with length `len`. template <AddMode mode> static CordRepRing* AddRing(CordRepRing* rep, CordRepRing* ring, - size_t offset, size_t length); + size_t offset, size_t len); // Increases the data offset for entry `index` by `n`. void AddDataOffset(index_type index, size_t n); @@ -567,20 +570,35 @@ inline CordRepRing::Position CordRepRing::FindTail(index_type head, // Now that CordRepRing is defined, we can define CordRep's helper casts: inline CordRepRing* CordRep::ring() { - assert(tag == RING); + assert(IsRing()); return static_cast<CordRepRing*>(this); } inline const CordRepRing* CordRep::ring() const { - assert(tag == RING); + assert(IsRing()); return static_cast<const CordRepRing*>(this); } -std::ostream& operator<<(std::ostream& s, const CordRepRing& rep); +inline bool CordRepRing::IsFlat(absl::string_view* fragment) const { + if (entries() == 1) { + if (fragment) *fragment = entry_data(head()); + return true; + } + return false; +} -#ifdef __clang__ -#pragma clang diagnostic pop -#endif +inline bool CordRepRing::IsFlat(size_t offset, size_t len, + absl::string_view* fragment) const { + const Position pos = Find(offset); + const absl::string_view data = entry_data(pos.index); + if (data.length() >= len && data.length() - len >= pos.offset) { + if (fragment) *fragment = data.substr(pos.offset, len); + return true; + } + return false; +} + +std::ostream& operator<<(std::ostream& s, const CordRepRing& rep); } // namespace cord_internal ABSL_NAMESPACE_END diff --git a/absl/strings/internal/cord_rep_ring_reader.h b/absl/strings/internal/cord_rep_ring_reader.h index 396c0e2c..7ceeaa00 100644 --- a/absl/strings/internal/cord_rep_ring_reader.h +++ b/absl/strings/internal/cord_rep_ring_reader.h @@ -40,6 +40,10 @@ class CordRepRingReader { // The returned value is undefined if this instance is empty. CordRepRing::index_type index() const { return index_; } + // Returns the current node inside the ring buffer for this instance. + // The returned value is undefined if this instance is empty. + CordRep* node() const { return ring_->entry_child(index_); } + // Returns the length of the referenced ring buffer. // Requires the current instance to be non empty. size_t length() const { diff --git a/absl/strings/internal/cord_rep_test_util.h b/absl/strings/internal/cord_rep_test_util.h new file mode 100644 index 00000000..18a0a195 --- /dev/null +++ b/absl/strings/internal/cord_rep_test_util.h @@ -0,0 +1,205 @@ +// Copyright 2021 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_STRINGS_INTERNAL_CORD_REP_TEST_UTIL_H_ +#define ABSL_STRINGS_INTERNAL_CORD_REP_TEST_UTIL_H_ + +#include <cassert> +#include <memory> +#include <random> +#include <string> +#include <vector> + +#include "absl/base/config.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/strings/internal/cord_internal.h" +#include "absl/strings/internal/cord_rep_btree.h" +#include "absl/strings/internal/cord_rep_flat.h" +#include "absl/strings/string_view.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cordrep_testing { + +inline cord_internal::CordRepSubstring* MakeSubstring( + size_t start, size_t len, cord_internal::CordRep* rep) { + auto* sub = new cord_internal::CordRepSubstring; + sub->tag = cord_internal::SUBSTRING; + sub->start = start; + sub->length = len <= 0 ? rep->length - start + len : len; + sub->child = rep; + return sub; +} + +inline cord_internal::CordRepFlat* MakeFlat(absl::string_view value) { + assert(value.length() <= cord_internal::kMaxFlatLength); + auto* flat = cord_internal::CordRepFlat::New(value.length()); + flat->length = value.length(); + memcpy(flat->Data(), value.data(), value.length()); + return flat; +} + +// Creates an external node for testing +inline cord_internal::CordRepExternal* MakeExternal(absl::string_view s) { + struct Rep : public cord_internal::CordRepExternal { + std::string s; + explicit Rep(absl::string_view sv) : s(sv) { + this->tag = cord_internal::EXTERNAL; + this->base = s.data(); + this->length = s.length(); + this->releaser_invoker = [](cord_internal::CordRepExternal* self) { + delete static_cast<Rep*>(self); + }; + } + }; + return new Rep(s); +} + +inline std::string CreateRandomString(size_t n) { + absl::string_view data = + "abcdefghijklmnopqrstuvwxyz" + "ABCDEFGHIJKLMNOPQRSTUVWXYZ" + "0123456789~!@#$%^&*()_+=-<>?:\"{}[]|"; + std::minstd_rand rnd; + std::uniform_int_distribution<size_t> dist(0, data.size() - 1); + std::string s(n, ' '); + for (size_t i = 0; i < n; ++i) { + s[i] = data[dist(rnd)]; + } + return s; +} + +// Creates an array of flats from the provided string, chopping +// the provided string up into flats of size `chunk_size` characters +// resulting in roughly `data.size() / chunk_size` total flats. +inline std::vector<cord_internal::CordRep*> CreateFlatsFromString( + absl::string_view data, size_t chunk_size) { + assert(chunk_size > 0); + std::vector<cord_internal::CordRep*> flats; + for (absl::string_view s = data; !s.empty(); s.remove_prefix(chunk_size)) { + flats.push_back(MakeFlat(s.substr(0, chunk_size))); + } + return flats; +} + +inline cord_internal::CordRepBtree* CordRepBtreeFromFlats( + absl::Span<cord_internal::CordRep* const> flats) { + assert(!flats.empty()); + auto* node = cord_internal::CordRepBtree::Create(flats[0]); + for (size_t i = 1; i < flats.size(); ++i) { + node = cord_internal::CordRepBtree::Append(node, flats[i]); + } + return node; +} + +template <typename Fn> +inline void CordVisitReps(cord_internal::CordRep* rep, Fn&& fn) { + fn(rep); + while (rep->tag == cord_internal::SUBSTRING) { + rep = rep->substring()->child; + fn(rep); + } + if (rep->tag == cord_internal::BTREE) { + for (cord_internal::CordRep* edge : rep->btree()->Edges()) { + CordVisitReps(edge, fn); + } + } +} + +template <typename Predicate> +inline std::vector<cord_internal::CordRep*> CordCollectRepsIf( + Predicate&& predicate, cord_internal::CordRep* rep) { + std::vector<cord_internal::CordRep*> reps; + CordVisitReps(rep, [&reps, &predicate](cord_internal::CordRep* rep) { + if (predicate(rep)) reps.push_back(rep); + }); + return reps; +} + +inline std::vector<cord_internal::CordRep*> CordCollectReps( + cord_internal::CordRep* rep) { + std::vector<cord_internal::CordRep*> reps; + auto fn = [&reps](cord_internal::CordRep* rep) { reps.push_back(rep); }; + CordVisitReps(rep, fn); + return reps; +} + +inline void CordToString(cord_internal::CordRep* rep, std::string& s) { + size_t offset = 0; + size_t length = rep->length; + while (rep->tag == cord_internal::SUBSTRING) { + offset += rep->substring()->start; + rep = rep->substring()->child; + } + if (rep->tag == cord_internal::BTREE) { + for (cord_internal::CordRep* edge : rep->btree()->Edges()) { + CordToString(edge, s); + } + } else if (rep->tag >= cord_internal::FLAT) { + s.append(rep->flat()->Data() + offset, length); + } else if (rep->tag == cord_internal::EXTERNAL) { + s.append(rep->external()->base + offset, length); + } else { + ABSL_RAW_LOG(FATAL, "Unsupported tag %d", rep->tag); + } +} + +inline std::string CordToString(cord_internal::CordRep* rep) { + std::string s; + s.reserve(rep->length); + CordToString(rep, s); + return s; +} + +// RAII Helper class to automatically unref reps on destruction. +class AutoUnref { + public: + ~AutoUnref() { + for (CordRep* rep : unrefs_) CordRep::Unref(rep); + } + + // Adds `rep` to the list of reps to be unreffed at destruction. + template <typename CordRepType> + CordRepType* Add(CordRepType* rep) { + unrefs_.push_back(rep); + return rep; + } + + // Increments the reference count of `rep` by one, and adds it to + // the list of reps to be unreffed at destruction. + template <typename CordRepType> + CordRepType* Ref(CordRepType* rep) { + unrefs_.push_back(CordRep::Ref(rep)); + return rep; + } + + // Increments the reference count of `rep` by one if `condition` is true, + // and adds it to the list of reps to be unreffed at destruction. + template <typename CordRepType> + CordRepType* RefIf(bool condition, CordRepType* rep) { + if (condition) unrefs_.push_back(CordRep::Ref(rep)); + return rep; + } + + private: + using CordRep = absl::cord_internal::CordRep; + + std::vector<CordRep*> unrefs_; +}; + +} // namespace cordrep_testing +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_INTERNAL_CORD_REP_TEST_UTIL_H_ diff --git a/absl/strings/internal/cordz_functions.cc b/absl/strings/internal/cordz_functions.cc new file mode 100644 index 00000000..20d314f0 --- /dev/null +++ b/absl/strings/internal/cordz_functions.cc @@ -0,0 +1,96 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/cordz_functions.h" + +#include <atomic> +#include <cmath> +#include <limits> +#include <random> + +#include "absl/base/attributes.h" +#include "absl/base/config.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/profiling/internal/exponential_biased.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { +namespace { + +// The average interval until the next sample. A value of 0 disables profiling +// while a value of 1 will profile all Cords. +std::atomic<int> g_cordz_mean_interval(50000); + +} // namespace + +#ifdef ABSL_INTERNAL_CORDZ_ENABLED + +// Special negative 'not initialized' per thread value for cordz_next_sample. +static constexpr int64_t kInitCordzNextSample = -1; + +ABSL_CONST_INIT thread_local int64_t cordz_next_sample = kInitCordzNextSample; + +// kIntervalIfDisabled is the number of profile-eligible events need to occur +// before the code will confirm that cordz is still disabled. +constexpr int64_t kIntervalIfDisabled = 1 << 16; + +ABSL_ATTRIBUTE_NOINLINE bool cordz_should_profile_slow() { + + thread_local absl::profiling_internal::ExponentialBiased + exponential_biased_generator; + int32_t mean_interval = get_cordz_mean_interval(); + + // Check if we disabled profiling. If so, set the next sample to a "large" + // number to minimize the overhead of the should_profile codepath. + if (mean_interval <= 0) { + cordz_next_sample = kIntervalIfDisabled; + return false; + } + + // Check if we're always sampling. + if (mean_interval == 1) { + cordz_next_sample = 1; + return true; + } + + if (cordz_next_sample <= 0) { + // If first check on current thread, check cordz_should_profile() + // again using the created (initial) stride in cordz_next_sample. + const bool initialized = cordz_next_sample != kInitCordzNextSample; + cordz_next_sample = exponential_biased_generator.GetStride(mean_interval); + return initialized || cordz_should_profile(); + } + + --cordz_next_sample; + return false; +} + +void cordz_set_next_sample_for_testing(int64_t next_sample) { + cordz_next_sample = next_sample; +} + +#endif // ABSL_INTERNAL_CORDZ_ENABLED + +int32_t get_cordz_mean_interval() { + return g_cordz_mean_interval.load(std::memory_order_acquire); +} + +void set_cordz_mean_interval(int32_t mean_interval) { + g_cordz_mean_interval.store(mean_interval, std::memory_order_release); +} + +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/absl/strings/internal/cordz_functions.h b/absl/strings/internal/cordz_functions.h new file mode 100644 index 00000000..c9ba1450 --- /dev/null +++ b/absl/strings/internal/cordz_functions.h @@ -0,0 +1,85 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_STRINGS_CORDZ_FUNCTIONS_H_ +#define ABSL_STRINGS_CORDZ_FUNCTIONS_H_ + +#include <stdint.h> + +#include "absl/base/attributes.h" +#include "absl/base/config.h" +#include "absl/base/optimization.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { + +// Returns the current sample rate. This represents the average interval +// between samples. +int32_t get_cordz_mean_interval(); + +// Sets the sample rate with the average interval between samples. +void set_cordz_mean_interval(int32_t mean_interval); + +// Enable cordz unless any of the following applies: +// - no thread local support +// - MSVC build +// - Android build +// - Apple build +// - DLL build +// Hashtablez is turned off completely in opensource builds. +// MSVC's static atomics are dynamically initialized in debug mode, which breaks +// sampling. +#if defined(ABSL_HAVE_THREAD_LOCAL) && !defined(_MSC_VER) && \ + !defined(ABSL_BUILD_DLL) && !defined(ABSL_CONSUME_DLL) && \ + !defined(__ANDROID__) && !defined(__APPLE__) +#define ABSL_INTERNAL_CORDZ_ENABLED 1 +#endif + +#ifdef ABSL_INTERNAL_CORDZ_ENABLED + +// cordz_next_sample is the number of events until the next sample event. If +// the value is 1 or less, the code will check on the next event if cordz is +// enabled, and if so, will sample the Cord. cordz is only enabled when we can +// use thread locals. +ABSL_CONST_INIT extern thread_local int64_t cordz_next_sample; + +// Determines if the next sample should be profiled. If it is, the value pointed +// at by next_sample will be set with the interval until the next sample. +bool cordz_should_profile_slow(); + +// Returns true if the next cord should be sampled. +inline bool cordz_should_profile() { + if (ABSL_PREDICT_TRUE(cordz_next_sample > 1)) { + cordz_next_sample--; + return false; + } + return cordz_should_profile_slow(); +} + +// Sets the interval until the next sample (for testing only) +void cordz_set_next_sample_for_testing(int64_t next_sample); + +#else // ABSL_INTERNAL_CORDZ_ENABLED + +inline bool cordz_should_profile() { return false; } +inline void cordz_set_next_sample_for_testing(int64_t) {} + +#endif // ABSL_INTERNAL_CORDZ_ENABLED + +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_CORDZ_FUNCTIONS_H_ diff --git a/absl/strings/internal/cordz_functions_test.cc b/absl/strings/internal/cordz_functions_test.cc new file mode 100644 index 00000000..350623c1 --- /dev/null +++ b/absl/strings/internal/cordz_functions_test.cc @@ -0,0 +1,149 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/cordz_functions.h" + +#include <thread> // NOLINT we need real clean new threads + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { +namespace { + +using ::testing::Eq; +using ::testing::Ge; +using ::testing::Le; + +TEST(CordzFunctionsTest, SampleRate) { + int32_t orig_sample_rate = get_cordz_mean_interval(); + int32_t expected_sample_rate = 123; + set_cordz_mean_interval(expected_sample_rate); + EXPECT_THAT(get_cordz_mean_interval(), Eq(expected_sample_rate)); + set_cordz_mean_interval(orig_sample_rate); +} + +// Cordz is disabled when we don't have thread_local. All calls to +// should_profile will return false when cordz is diabled, so we might want to +// avoid those tests. +#ifdef ABSL_INTERNAL_CORDZ_ENABLED + +TEST(CordzFunctionsTest, ShouldProfileDisable) { + int32_t orig_sample_rate = get_cordz_mean_interval(); + + set_cordz_mean_interval(0); + cordz_set_next_sample_for_testing(0); + EXPECT_FALSE(cordz_should_profile()); + // 1 << 16 is from kIntervalIfDisabled in cordz_functions.cc. + EXPECT_THAT(cordz_next_sample, Eq(1 << 16)); + + set_cordz_mean_interval(orig_sample_rate); +} + +TEST(CordzFunctionsTest, ShouldProfileAlways) { + int32_t orig_sample_rate = get_cordz_mean_interval(); + + set_cordz_mean_interval(1); + cordz_set_next_sample_for_testing(1); + EXPECT_TRUE(cordz_should_profile()); + EXPECT_THAT(cordz_next_sample, Le(1)); + + set_cordz_mean_interval(orig_sample_rate); +} + +TEST(CordzFunctionsTest, DoesNotAlwaysSampleFirstCord) { + // Set large enough interval such that the chance of 'tons' of threads + // randomly sampling the first call is infinitely small. + set_cordz_mean_interval(10000); + int tries = 0; + bool sampled = false; + do { + ++tries; + ASSERT_THAT(tries, Le(1000)); + std::thread thread([&sampled] { + sampled = cordz_should_profile(); + }); + thread.join(); + } while (sampled); +} + +TEST(CordzFunctionsTest, ShouldProfileRate) { + static constexpr int kDesiredMeanInterval = 1000; + static constexpr int kSamples = 10000; + int32_t orig_sample_rate = get_cordz_mean_interval(); + + set_cordz_mean_interval(kDesiredMeanInterval); + + int64_t sum_of_intervals = 0; + for (int i = 0; i < kSamples; i++) { + // Setting next_sample to 0 will force cordz_should_profile to generate a + // new value for next_sample each iteration. + cordz_set_next_sample_for_testing(0); + cordz_should_profile(); + sum_of_intervals += cordz_next_sample; + } + + // The sum of independent exponential variables is an Erlang distribution, + // which is a gamma distribution where the shape parameter is equal to the + // number of summands. The distribution used for cordz_should_profile is + // actually floor(Exponential(1/mean)) which introduces bias. However, we can + // apply the squint-really-hard correction factor. That is, when mean is + // large, then if we squint really hard the shape of the distribution between + // N and N+1 looks like a uniform distribution. On average, each value for + // next_sample will be about 0.5 lower than we would expect from an + // exponential distribution. This squint-really-hard correction approach won't + // work when mean is smaller than about 10 but works fine when mean is 1000. + // + // We can use R to calculate a confidence interval. This + // shows how to generate a confidence interval with a false positive rate of + // one in a billion. + // + // $ R -q + // > mean = 1000 + // > kSamples = 10000 + // > errorRate = 1e-9 + // > correction = -kSamples / 2 + // > low = qgamma(errorRate/2, kSamples, 1/mean) + correction + // > high = qgamma(1 - errorRate/2, kSamples, 1/mean) + correction + // > low + // [1] 9396115 + // > high + // [1] 10618100 + EXPECT_THAT(sum_of_intervals, Ge(9396115)); + EXPECT_THAT(sum_of_intervals, Le(10618100)); + + set_cordz_mean_interval(orig_sample_rate); +} + +#else // ABSL_INTERNAL_CORDZ_ENABLED + +TEST(CordzFunctionsTest, ShouldProfileDisabled) { + int32_t orig_sample_rate = get_cordz_mean_interval(); + + set_cordz_mean_interval(1); + cordz_set_next_sample_for_testing(0); + EXPECT_FALSE(cordz_should_profile()); + + set_cordz_mean_interval(orig_sample_rate); +} + +#endif // ABSL_INTERNAL_CORDZ_ENABLED + +} // namespace +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/absl/strings/internal/cordz_handle.cc b/absl/strings/internal/cordz_handle.cc new file mode 100644 index 00000000..a73fefed --- /dev/null +++ b/absl/strings/internal/cordz_handle.cc @@ -0,0 +1,139 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include "absl/strings/internal/cordz_handle.h" + +#include <atomic> + +#include "absl/base/internal/raw_logging.h" // For ABSL_RAW_CHECK +#include "absl/base/internal/spinlock.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { + +using ::absl::base_internal::SpinLockHolder; + +ABSL_CONST_INIT CordzHandle::Queue CordzHandle::global_queue_(absl::kConstInit); + +CordzHandle::CordzHandle(bool is_snapshot) : is_snapshot_(is_snapshot) { + if (is_snapshot) { + SpinLockHolder lock(&queue_->mutex); + CordzHandle* dq_tail = queue_->dq_tail.load(std::memory_order_acquire); + if (dq_tail != nullptr) { + dq_prev_ = dq_tail; + dq_tail->dq_next_ = this; + } + queue_->dq_tail.store(this, std::memory_order_release); + } +} + +CordzHandle::~CordzHandle() { + ODRCheck(); + if (is_snapshot_) { + std::vector<CordzHandle*> to_delete; + { + SpinLockHolder lock(&queue_->mutex); + CordzHandle* next = dq_next_; + if (dq_prev_ == nullptr) { + // We were head of the queue, delete every CordzHandle until we reach + // either the end of the list, or a snapshot handle. + while (next && !next->is_snapshot_) { + to_delete.push_back(next); + next = next->dq_next_; + } + } else { + // Another CordzHandle existed before this one, don't delete anything. + dq_prev_->dq_next_ = next; + } + if (next) { + next->dq_prev_ = dq_prev_; + } else { + queue_->dq_tail.store(dq_prev_, std::memory_order_release); + } + } + for (CordzHandle* handle : to_delete) { + delete handle; + } + } +} + +bool CordzHandle::SafeToDelete() const { + return is_snapshot_ || queue_->IsEmpty(); +} + +void CordzHandle::Delete(CordzHandle* handle) { + assert(handle); + if (handle) { + handle->ODRCheck(); + Queue* const queue = handle->queue_; + if (!handle->SafeToDelete()) { + SpinLockHolder lock(&queue->mutex); + CordzHandle* dq_tail = queue->dq_tail.load(std::memory_order_acquire); + if (dq_tail != nullptr) { + handle->dq_prev_ = dq_tail; + dq_tail->dq_next_ = handle; + queue->dq_tail.store(handle, std::memory_order_release); + return; + } + } + delete handle; + } +} + +std::vector<const CordzHandle*> CordzHandle::DiagnosticsGetDeleteQueue() { + std::vector<const CordzHandle*> handles; + SpinLockHolder lock(&global_queue_.mutex); + CordzHandle* dq_tail = global_queue_.dq_tail.load(std::memory_order_acquire); + for (const CordzHandle* p = dq_tail; p; p = p->dq_prev_) { + handles.push_back(p); + } + return handles; +} + +bool CordzHandle::DiagnosticsHandleIsSafeToInspect( + const CordzHandle* handle) const { + ODRCheck(); + if (!is_snapshot_) return false; + if (handle == nullptr) return true; + if (handle->is_snapshot_) return false; + bool snapshot_found = false; + SpinLockHolder lock(&queue_->mutex); + for (const CordzHandle* p = queue_->dq_tail; p; p = p->dq_prev_) { + if (p == handle) return !snapshot_found; + if (p == this) snapshot_found = true; + } + ABSL_ASSERT(snapshot_found); // Assert that 'this' is in delete queue. + return true; +} + +std::vector<const CordzHandle*> +CordzHandle::DiagnosticsGetSafeToInspectDeletedHandles() { + ODRCheck(); + std::vector<const CordzHandle*> handles; + if (!is_snapshot()) { + return handles; + } + + SpinLockHolder lock(&queue_->mutex); + for (const CordzHandle* p = dq_next_; p != nullptr; p = p->dq_next_) { + if (!p->is_snapshot()) { + handles.push_back(p); + } + } + return handles; +} + +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/absl/strings/internal/cordz_handle.h b/absl/strings/internal/cordz_handle.h new file mode 100644 index 00000000..5df53c78 --- /dev/null +++ b/absl/strings/internal/cordz_handle.h @@ -0,0 +1,131 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_STRINGS_CORDZ_HANDLE_H_ +#define ABSL_STRINGS_CORDZ_HANDLE_H_ + +#include <atomic> +#include <vector> + +#include "absl/base/config.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/base/internal/spinlock.h" +#include "absl/synchronization/mutex.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { + +// This base class allows multiple types of object (CordzInfo and +// CordzSampleToken) to exist simultaneously on the delete queue (pointed to by +// global_dq_tail and traversed using dq_prev_ and dq_next_). The +// delete queue guarantees that once a profiler creates a CordzSampleToken and +// has gained visibility into a CordzInfo object, that CordzInfo object will not +// be deleted prematurely. This allows the profiler to inspect all CordzInfo +// objects that are alive without needing to hold a global lock. +class CordzHandle { + public: + CordzHandle() : CordzHandle(false) {} + + bool is_snapshot() const { return is_snapshot_; } + + // Returns true if this instance is safe to be deleted because it is either a + // snapshot, which is always safe to delete, or not included in the global + // delete queue and thus not included in any snapshot. + // Callers are responsible for making sure this instance can not be newly + // discovered by other threads. For example, CordzInfo instances first de-list + // themselves from the global CordzInfo list before determining if they are + // safe to be deleted directly. + // If SafeToDelete returns false, callers MUST use the Delete() method to + // safely queue CordzHandle instances for deletion. + bool SafeToDelete() const; + + // Deletes the provided instance, or puts it on the delete queue to be deleted + // once there are no more sample tokens (snapshot) instances potentially + // referencing the instance. `handle` should not be null. + static void Delete(CordzHandle* handle); + + // Returns the current entries in the delete queue in LIFO order. + static std::vector<const CordzHandle*> DiagnosticsGetDeleteQueue(); + + // Returns true if the provided handle is nullptr or guarded by this handle. + // Since the CordzSnapshot token is itself a CordzHandle, this method will + // allow tests to check if that token is keeping an arbitrary CordzHandle + // alive. + bool DiagnosticsHandleIsSafeToInspect(const CordzHandle* handle) const; + + // Returns the current entries in the delete queue, in LIFO order, that are + // protected by this. CordzHandle objects are only placed on the delete queue + // after CordzHandle::Delete is called with them as an argument. Only + // CordzHandle objects that are not also CordzSnapshot objects will be + // included in the return vector. For each of the handles in the return + // vector, the earliest that their memory can be freed is when this + // CordzSnapshot object is deleted. + std::vector<const CordzHandle*> DiagnosticsGetSafeToInspectDeletedHandles(); + + protected: + explicit CordzHandle(bool is_snapshot); + virtual ~CordzHandle(); + + private: + // Global queue data. CordzHandle stores a pointer to the global queue + // instance to harden against ODR violations. + struct Queue { + constexpr explicit Queue(absl::ConstInitType) + : mutex(absl::kConstInit, + absl::base_internal::SCHEDULE_COOPERATIVE_AND_KERNEL) {} + + absl::base_internal::SpinLock mutex; + std::atomic<CordzHandle*> dq_tail ABSL_GUARDED_BY(mutex){nullptr}; + + // Returns true if this delete queue is empty. This method does not acquire + // the lock, but does a 'load acquire' observation on the delete queue tail. + // It is used inside Delete() to check for the presence of a delete queue + // without holding the lock. The assumption is that the caller is in the + // state of 'being deleted', and can not be newly discovered by a concurrent + // 'being constructed' snapshot instance. Practically, this means that any + // such discovery (`find`, 'first' or 'next', etc) must have proper 'happens + // before / after' semantics and atomic fences. + bool IsEmpty() const ABSL_NO_THREAD_SAFETY_ANALYSIS { + return dq_tail.load(std::memory_order_acquire) == nullptr; + } + }; + + void ODRCheck() const { +#ifndef NDEBUG + ABSL_RAW_CHECK(queue_ == &global_queue_, "ODR violation in Cord"); +#endif + } + + ABSL_CONST_INIT static Queue global_queue_; + Queue* const queue_ = &global_queue_; + const bool is_snapshot_; + + // dq_prev_ and dq_next_ require the global queue mutex to be held. + // Unfortunately we can't use thread annotations such that the thread safety + // analysis understands that queue_ and global_queue_ are one and the same. + CordzHandle* dq_prev_ = nullptr; + CordzHandle* dq_next_ = nullptr; +}; + +class CordzSnapshot : public CordzHandle { + public: + CordzSnapshot() : CordzHandle(true) {} +}; + +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_CORDZ_HANDLE_H_ diff --git a/absl/strings/internal/cordz_handle_test.cc b/absl/strings/internal/cordz_handle_test.cc new file mode 100644 index 00000000..fd68e06b --- /dev/null +++ b/absl/strings/internal/cordz_handle_test.cc @@ -0,0 +1,265 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +#include "absl/strings/internal/cordz_handle.h" + +#include <random> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/memory/memory.h" +#include "absl/synchronization/internal/thread_pool.h" +#include "absl/synchronization/notification.h" +#include "absl/time/clock.h" +#include "absl/time/time.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { +namespace { + +using ::testing::ElementsAre; +using ::testing::Gt; +using ::testing::IsEmpty; +using ::testing::SizeIs; + +// Local less verbose helper +std::vector<const CordzHandle*> DeleteQueue() { + return CordzHandle::DiagnosticsGetDeleteQueue(); +} + +struct CordzHandleDeleteTracker : public CordzHandle { + bool* deleted; + explicit CordzHandleDeleteTracker(bool* deleted) : deleted(deleted) {} + ~CordzHandleDeleteTracker() override { *deleted = true; } +}; + +TEST(CordzHandleTest, DeleteQueueIsEmpty) { + EXPECT_THAT(DeleteQueue(), SizeIs(0)); +} + +TEST(CordzHandleTest, CordzHandleCreateDelete) { + bool deleted = false; + auto* handle = new CordzHandleDeleteTracker(&deleted); + EXPECT_FALSE(handle->is_snapshot()); + EXPECT_TRUE(handle->SafeToDelete()); + EXPECT_THAT(DeleteQueue(), SizeIs(0)); + + CordzHandle::Delete(handle); + EXPECT_THAT(DeleteQueue(), SizeIs(0)); + EXPECT_TRUE(deleted); +} + +TEST(CordzHandleTest, CordzSnapshotCreateDelete) { + auto* snapshot = new CordzSnapshot(); + EXPECT_TRUE(snapshot->is_snapshot()); + EXPECT_TRUE(snapshot->SafeToDelete()); + EXPECT_THAT(DeleteQueue(), ElementsAre(snapshot)); + delete snapshot; + EXPECT_THAT(DeleteQueue(), SizeIs(0)); +} + +TEST(CordzHandleTest, CordzHandleCreateDeleteWithSnapshot) { + bool deleted = false; + auto* snapshot = new CordzSnapshot(); + auto* handle = new CordzHandleDeleteTracker(&deleted); + EXPECT_FALSE(handle->SafeToDelete()); + + CordzHandle::Delete(handle); + EXPECT_THAT(DeleteQueue(), ElementsAre(handle, snapshot)); + EXPECT_FALSE(deleted); + EXPECT_FALSE(handle->SafeToDelete()); + + delete snapshot; + EXPECT_THAT(DeleteQueue(), SizeIs(0)); + EXPECT_TRUE(deleted); +} + +TEST(CordzHandleTest, MultiSnapshot) { + bool deleted[3] = {false, false, false}; + + CordzSnapshot* snapshot[3]; + CordzHandleDeleteTracker* handle[3]; + for (int i = 0; i < 3; ++i) { + snapshot[i] = new CordzSnapshot(); + handle[i] = new CordzHandleDeleteTracker(&deleted[i]); + CordzHandle::Delete(handle[i]); + } + + EXPECT_THAT(DeleteQueue(), ElementsAre(handle[2], snapshot[2], handle[1], + snapshot[1], handle[0], snapshot[0])); + EXPECT_THAT(deleted, ElementsAre(false, false, false)); + + delete snapshot[1]; + EXPECT_THAT(DeleteQueue(), ElementsAre(handle[2], snapshot[2], handle[1], + handle[0], snapshot[0])); + EXPECT_THAT(deleted, ElementsAre(false, false, false)); + + delete snapshot[0]; + EXPECT_THAT(DeleteQueue(), ElementsAre(handle[2], snapshot[2])); + EXPECT_THAT(deleted, ElementsAre(true, true, false)); + + delete snapshot[2]; + EXPECT_THAT(DeleteQueue(), SizeIs(0)); + EXPECT_THAT(deleted, ElementsAre(true, true, deleted)); +} + +TEST(CordzHandleTest, DiagnosticsHandleIsSafeToInspect) { + CordzSnapshot snapshot1; + EXPECT_TRUE(snapshot1.DiagnosticsHandleIsSafeToInspect(nullptr)); + + auto* handle1 = new CordzHandle(); + EXPECT_TRUE(snapshot1.DiagnosticsHandleIsSafeToInspect(handle1)); + + CordzHandle::Delete(handle1); + EXPECT_TRUE(snapshot1.DiagnosticsHandleIsSafeToInspect(handle1)); + + CordzSnapshot snapshot2; + auto* handle2 = new CordzHandle(); + EXPECT_TRUE(snapshot1.DiagnosticsHandleIsSafeToInspect(handle1)); + EXPECT_TRUE(snapshot1.DiagnosticsHandleIsSafeToInspect(handle2)); + EXPECT_FALSE(snapshot2.DiagnosticsHandleIsSafeToInspect(handle1)); + EXPECT_TRUE(snapshot2.DiagnosticsHandleIsSafeToInspect(handle2)); + + CordzHandle::Delete(handle2); + EXPECT_TRUE(snapshot1.DiagnosticsHandleIsSafeToInspect(handle1)); +} + +TEST(CordzHandleTest, DiagnosticsGetSafeToInspectDeletedHandles) { + EXPECT_THAT(DeleteQueue(), IsEmpty()); + + auto* handle = new CordzHandle(); + auto* snapshot1 = new CordzSnapshot(); + + // snapshot1 should be able to see handle. + EXPECT_THAT(DeleteQueue(), ElementsAre(snapshot1)); + EXPECT_TRUE(snapshot1->DiagnosticsHandleIsSafeToInspect(handle)); + EXPECT_THAT(snapshot1->DiagnosticsGetSafeToInspectDeletedHandles(), + IsEmpty()); + + // This handle will be safe to inspect as long as snapshot1 is alive. However, + // since only snapshot1 can prove that it's alive, it will be hidden from + // snapshot2. + CordzHandle::Delete(handle); + + // This snapshot shouldn't be able to see handle because handle was already + // sent to Delete. + auto* snapshot2 = new CordzSnapshot(); + + // DeleteQueue elements are LIFO order. + EXPECT_THAT(DeleteQueue(), ElementsAre(snapshot2, handle, snapshot1)); + + EXPECT_TRUE(snapshot1->DiagnosticsHandleIsSafeToInspect(handle)); + EXPECT_FALSE(snapshot2->DiagnosticsHandleIsSafeToInspect(handle)); + + EXPECT_THAT(snapshot1->DiagnosticsGetSafeToInspectDeletedHandles(), + ElementsAre(handle)); + EXPECT_THAT(snapshot2->DiagnosticsGetSafeToInspectDeletedHandles(), + IsEmpty()); + + CordzHandle::Delete(snapshot1); + EXPECT_THAT(DeleteQueue(), ElementsAre(snapshot2)); + + CordzHandle::Delete(snapshot2); + EXPECT_THAT(DeleteQueue(), IsEmpty()); +} + +// Create and delete CordzHandle and CordzSnapshot objects in multiple threads +// so that tsan has some time to chew on it and look for memory problems. +TEST(CordzHandleTest, MultiThreaded) { + Notification stop; + static constexpr int kNumThreads = 4; + // Keep the number of handles relatively small so that the test will naturally + // transition to an empty delete queue during the test. If there are, say, 100 + // handles, that will virtually never happen. With 10 handles and around 50k + // iterations in each of 4 threads, the delete queue appears to become empty + // around 200 times. + static constexpr int kNumHandles = 10; + + // Each thread is going to pick a random index and atomically swap its + // CordzHandle with one in handles. This way, each thread can avoid + // manipulating a CordzHandle that might be operated upon in another thread. + std::vector<std::atomic<CordzHandle*>> handles(kNumHandles); + + // global bool which is set when any thread did get some 'safe to inspect' + // handles. On some platforms and OSS tests, we might risk that some pool + // threads are starved, stalled, or just got a few unlikely random 'handle' + // coin tosses, so we satisfy this test with simply observing 'some' thread + // did something meaningful, which should minimize the potential for flakes. + std::atomic<bool> found_safe_to_inspect(false); + + { + absl::synchronization_internal::ThreadPool pool(kNumThreads); + for (int i = 0; i < kNumThreads; ++i) { + pool.Schedule([&stop, &handles, &found_safe_to_inspect]() { + std::minstd_rand gen; + std::uniform_int_distribution<int> dist_type(0, 2); + std::uniform_int_distribution<int> dist_handle(0, kNumHandles - 1); + + while (!stop.HasBeenNotified()) { + CordzHandle* handle; + switch (dist_type(gen)) { + case 0: + handle = new CordzHandle(); + break; + case 1: + handle = new CordzSnapshot(); + break; + default: + handle = nullptr; + break; + } + CordzHandle* old_handle = handles[dist_handle(gen)].exchange(handle); + if (old_handle != nullptr) { + std::vector<const CordzHandle*> safe_to_inspect = + old_handle->DiagnosticsGetSafeToInspectDeletedHandles(); + for (const CordzHandle* handle : safe_to_inspect) { + // We're in a tight loop, so don't generate too many error + // messages. + ASSERT_FALSE(handle->is_snapshot()); + } + if (!safe_to_inspect.empty()) { + found_safe_to_inspect.store(true); + } + CordzHandle::Delete(old_handle); + } + } + + // Have each thread attempt to clean up everything. Some thread will be + // the last to reach this cleanup code, and it will be guaranteed to + // clean up everything because nothing remains to create new handles. + for (auto& h : handles) { + if (CordzHandle* handle = h.exchange(nullptr)) { + CordzHandle::Delete(handle); + } + } + }); + } + + // The threads will hammer away. Give it a little bit of time for tsan to + // spot errors. + absl::SleepFor(absl::Seconds(3)); + stop.Notify(); + } + + // Confirm that the test did *something*. This check will be satisfied as + // long as any thread has deleted a CordzSnapshot object and a non-snapshot + // CordzHandle was deleted after the CordzSnapshot was created. + // See also comments on `found_safe_to_inspect` + EXPECT_TRUE(found_safe_to_inspect.load()); +} + +} // namespace +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/absl/strings/internal/cordz_info.cc b/absl/strings/internal/cordz_info.cc new file mode 100644 index 00000000..dac3fd8b --- /dev/null +++ b/absl/strings/internal/cordz_info.cc @@ -0,0 +1,418 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/cordz_info.h" + +#include "absl/base/config.h" +#include "absl/base/internal/spinlock.h" +#include "absl/container/inlined_vector.h" +#include "absl/debugging/stacktrace.h" +#include "absl/strings/internal/cord_internal.h" +#include "absl/strings/internal/cord_rep_btree.h" +#include "absl/strings/internal/cord_rep_crc.h" +#include "absl/strings/internal/cord_rep_ring.h" +#include "absl/strings/internal/cordz_handle.h" +#include "absl/strings/internal/cordz_statistics.h" +#include "absl/strings/internal/cordz_update_tracker.h" +#include "absl/synchronization/mutex.h" +#include "absl/types/span.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { + +using ::absl::base_internal::SpinLockHolder; + +#ifdef ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL +constexpr int CordzInfo::kMaxStackDepth; +#endif + +ABSL_CONST_INIT CordzInfo::List CordzInfo::global_list_{absl::kConstInit}; + +namespace { + +// CordRepAnalyzer performs the analysis of a cord. +// +// It computes absolute node counts and total memory usage, and an 'estimated +// fair share memory usage` statistic. +// Conceptually, it divides the 'memory usage' at each location in the 'cord +// graph' by the cumulative reference count of that location. The cumulative +// reference count is the factored total of all edges leading into that node. +// +// The top level node is treated specially: we assume the current thread +// (typically called from the CordzHandler) to hold a reference purely to +// perform a safe analysis, and not being part of the application. So we +// substract 1 from the reference count of the top node to compute the +// 'application fair share' excluding the reference of the current thread. +// +// An example of fair sharing, and why we multiply reference counts: +// Assume we have 2 CordReps, both being a Substring referencing a Flat: +// CordSubstring A (refcount = 5) --> child Flat C (refcount = 2) +// CordSubstring B (refcount = 9) --> child Flat C (refcount = 2) +// +// Flat C has 2 incoming edges from the 2 substrings (refcount = 2) and is not +// referenced directly anywhere else. Translated into a 'fair share', we then +// attribute 50% of the memory (memory / refcount = 2) to each incoming edge. +// Rep A has a refcount of 5, so we attribute each incoming edge 1 / 5th of the +// memory cost below it, i.e.: the fair share of Rep A of the memory used by C +// is then 'memory C / (refcount C * refcount A) + (memory A / refcount A)'. +// It is also easy to see how all incoming edges add up to 100%. +class CordRepAnalyzer { + public: + // Creates an analyzer instance binding to `statistics`. + explicit CordRepAnalyzer(CordzStatistics& statistics) + : statistics_(statistics) {} + + // Analyzes the memory statistics and node counts for the provided `rep`, and + // adds the results to `statistics`. Note that node counts and memory sizes + // are not initialized, computed values are added to any existing values. + void AnalyzeCordRep(const CordRep* rep) { + // Process all linear nodes. + // As per the class comments, use refcout - 1 on the top level node, as the + // top level node is assumed to be referenced only for analysis purposes. + size_t refcount = rep->refcount.Get(); + RepRef repref{rep, (refcount > 1) ? refcount - 1 : 1}; + + // Process the top level CRC node, if present. + if (repref.rep->tag == CRC) { + statistics_.node_count++; + statistics_.node_counts.crc++; + memory_usage_.Add(sizeof(CordRepCrc), repref.refcount); + repref = repref.Child(repref.rep->crc()->child); + } + + // Process all top level linear nodes (substrings and flats). + repref = CountLinearReps(repref, memory_usage_); + + if (repref.rep != nullptr) { + if (repref.rep->tag == RING) { + AnalyzeRing(repref); + } else if (repref.rep->tag == BTREE) { + AnalyzeBtree(repref); + } else { + // We should have either a concat, btree, or ring node if not null. + assert(false); + } + } + + // Adds values to output + statistics_.estimated_memory_usage += memory_usage_.total; + statistics_.estimated_fair_share_memory_usage += + static_cast<size_t>(memory_usage_.fair_share); + } + + private: + // RepRef identifies a CordRep* inside the Cord tree with its cumulative + // refcount including itself. For example, a tree consisting of a substring + // with a refcount of 3 and a child flat with a refcount of 4 will have RepRef + // refcounts of 3 and 12 respectively. + struct RepRef { + const CordRep* rep; + size_t refcount; + + // Returns a 'child' RepRef which contains the cumulative reference count of + // this instance multiplied by the child's reference count. + RepRef Child(const CordRep* child) const { + return RepRef{child, refcount * child->refcount.Get()}; + } + }; + + // Memory usage values + struct MemoryUsage { + size_t total = 0; + double fair_share = 0.0; + + // Adds 'size` memory usage to this class, with a cumulative (recursive) + // reference count of `refcount` + void Add(size_t size, size_t refcount) { + total += size; + fair_share += static_cast<double>(size) / refcount; + } + }; + + // Counts a flat of the provide allocated size + void CountFlat(size_t size) { + statistics_.node_count++; + statistics_.node_counts.flat++; + if (size <= 64) { + statistics_.node_counts.flat_64++; + } else if (size <= 128) { + statistics_.node_counts.flat_128++; + } else if (size <= 256) { + statistics_.node_counts.flat_256++; + } else if (size <= 512) { + statistics_.node_counts.flat_512++; + } else if (size <= 1024) { + statistics_.node_counts.flat_1k++; + } + } + + // Processes 'linear' reps (substring, flat, external) not requiring iteration + // or recursion. Returns RefRep{null} if all reps were processed, else returns + // the top-most non-linear concat or ring cordrep. + // Node counts are updated into `statistics_`, memory usage is update into + // `memory_usage`, which typically references `memory_usage_` except for ring + // buffers where we count children unrounded. + RepRef CountLinearReps(RepRef rep, MemoryUsage& memory_usage) { + // Consume all substrings + while (rep.rep->tag == SUBSTRING) { + statistics_.node_count++; + statistics_.node_counts.substring++; + memory_usage.Add(sizeof(CordRepSubstring), rep.refcount); + rep = rep.Child(rep.rep->substring()->child); + } + + // Consume possible FLAT + if (rep.rep->tag >= FLAT) { + size_t size = rep.rep->flat()->AllocatedSize(); + CountFlat(size); + memory_usage.Add(size, rep.refcount); + return RepRef{nullptr, 0}; + } + + // Consume possible external + if (rep.rep->tag == EXTERNAL) { + statistics_.node_count++; + statistics_.node_counts.external++; + size_t size = rep.rep->length + sizeof(CordRepExternalImpl<intptr_t>); + memory_usage.Add(size, rep.refcount); + return RepRef{nullptr, 0}; + } + + return rep; + } + + // Analyzes the provided ring. + void AnalyzeRing(RepRef rep) { + statistics_.node_count++; + statistics_.node_counts.ring++; + const CordRepRing* ring = rep.rep->ring(); + memory_usage_.Add(CordRepRing::AllocSize(ring->capacity()), rep.refcount); + ring->ForEach([&](CordRepRing::index_type pos) { + CountLinearReps(rep.Child(ring->entry_child(pos)), memory_usage_); + }); + } + + // Analyzes the provided btree. + void AnalyzeBtree(RepRef rep) { + statistics_.node_count++; + statistics_.node_counts.btree++; + memory_usage_.Add(sizeof(CordRepBtree), rep.refcount); + const CordRepBtree* tree = rep.rep->btree(); + if (tree->height() > 0) { + for (CordRep* edge : tree->Edges()) { + AnalyzeBtree(rep.Child(edge)); + } + } else { + for (CordRep* edge : tree->Edges()) { + CountLinearReps(rep.Child(edge), memory_usage_); + } + } + } + + CordzStatistics& statistics_; + MemoryUsage memory_usage_; +}; + +} // namespace + +CordzInfo* CordzInfo::Head(const CordzSnapshot& snapshot) { + ABSL_ASSERT(snapshot.is_snapshot()); + + // We can do an 'unsafe' load of 'head', as we are guaranteed that the + // instance it points to is kept alive by the provided CordzSnapshot, so we + // can simply return the current value using an acquire load. + // We do enforce in DEBUG builds that the 'head' value is present in the + // delete queue: ODR violations may lead to 'snapshot' and 'global_list_' + // being in different libraries / modules. + CordzInfo* head = global_list_.head.load(std::memory_order_acquire); + ABSL_ASSERT(snapshot.DiagnosticsHandleIsSafeToInspect(head)); + return head; +} + +CordzInfo* CordzInfo::Next(const CordzSnapshot& snapshot) const { + ABSL_ASSERT(snapshot.is_snapshot()); + + // Similar to the 'Head()' function, we do not need a mutex here. + CordzInfo* next = ci_next_.load(std::memory_order_acquire); + ABSL_ASSERT(snapshot.DiagnosticsHandleIsSafeToInspect(this)); + ABSL_ASSERT(snapshot.DiagnosticsHandleIsSafeToInspect(next)); + return next; +} + +void CordzInfo::TrackCord(InlineData& cord, MethodIdentifier method) { + assert(cord.is_tree()); + assert(!cord.is_profiled()); + CordzInfo* cordz_info = new CordzInfo(cord.as_tree(), nullptr, method); + cord.set_cordz_info(cordz_info); + cordz_info->Track(); +} + +void CordzInfo::TrackCord(InlineData& cord, const InlineData& src, + MethodIdentifier method) { + assert(cord.is_tree()); + assert(src.is_tree()); + + // Unsample current as we the current cord is being replaced with 'src', + // so any method history is no longer relevant. + CordzInfo* cordz_info = cord.cordz_info(); + if (cordz_info != nullptr) cordz_info->Untrack(); + + // Start new cord sample + cordz_info = new CordzInfo(cord.as_tree(), src.cordz_info(), method); + cord.set_cordz_info(cordz_info); + cordz_info->Track(); +} + +void CordzInfo::MaybeTrackCordImpl(InlineData& cord, const InlineData& src, + MethodIdentifier method) { + if (src.is_profiled()) { + TrackCord(cord, src, method); + } else if (cord.is_profiled()) { + cord.cordz_info()->Untrack(); + cord.clear_cordz_info(); + } +} + +CordzInfo::MethodIdentifier CordzInfo::GetParentMethod(const CordzInfo* src) { + if (src == nullptr) return MethodIdentifier::kUnknown; + return src->parent_method_ != MethodIdentifier::kUnknown ? src->parent_method_ + : src->method_; +} + +int CordzInfo::FillParentStack(const CordzInfo* src, void** stack) { + assert(stack); + if (src == nullptr) return 0; + if (src->parent_stack_depth_) { + memcpy(stack, src->parent_stack_, src->parent_stack_depth_ * sizeof(void*)); + return src->parent_stack_depth_; + } + memcpy(stack, src->stack_, src->stack_depth_ * sizeof(void*)); + return src->stack_depth_; +} + +CordzInfo::CordzInfo(CordRep* rep, const CordzInfo* src, + MethodIdentifier method) + : rep_(rep), + stack_depth_(absl::GetStackTrace(stack_, /*max_depth=*/kMaxStackDepth, + /*skip_count=*/1)), + parent_stack_depth_(FillParentStack(src, parent_stack_)), + method_(method), + parent_method_(GetParentMethod(src)), + create_time_(absl::Now()) { + update_tracker_.LossyAdd(method); + if (src) { + // Copy parent counters. + update_tracker_.LossyAdd(src->update_tracker_); + } +} + +CordzInfo::~CordzInfo() { + // `rep_` is potentially kept alive if CordzInfo is included + // in a collection snapshot (which should be rare). + if (ABSL_PREDICT_FALSE(rep_)) { + CordRep::Unref(rep_); + } +} + +void CordzInfo::Track() { + SpinLockHolder l(&list_->mutex); + + CordzInfo* const head = list_->head.load(std::memory_order_acquire); + if (head != nullptr) { + head->ci_prev_.store(this, std::memory_order_release); + } + ci_next_.store(head, std::memory_order_release); + list_->head.store(this, std::memory_order_release); +} + +void CordzInfo::Untrack() { + ODRCheck(); + { + SpinLockHolder l(&list_->mutex); + + CordzInfo* const head = list_->head.load(std::memory_order_acquire); + CordzInfo* const next = ci_next_.load(std::memory_order_acquire); + CordzInfo* const prev = ci_prev_.load(std::memory_order_acquire); + + if (next) { + ABSL_ASSERT(next->ci_prev_.load(std::memory_order_acquire) == this); + next->ci_prev_.store(prev, std::memory_order_release); + } + if (prev) { + ABSL_ASSERT(head != this); + ABSL_ASSERT(prev->ci_next_.load(std::memory_order_acquire) == this); + prev->ci_next_.store(next, std::memory_order_release); + } else { + ABSL_ASSERT(head == this); + list_->head.store(next, std::memory_order_release); + } + } + + // We can no longer be discovered: perform a fast path check if we are not + // listed on any delete queue, so we can directly delete this instance. + if (SafeToDelete()) { + UnsafeSetCordRep(nullptr); + delete this; + return; + } + + // We are likely part of a snapshot, extend the life of the CordRep + { + absl::MutexLock lock(&mutex_); + if (rep_) CordRep::Ref(rep_); + } + CordzHandle::Delete(this); +} + +void CordzInfo::Lock(MethodIdentifier method) + ABSL_EXCLUSIVE_LOCK_FUNCTION(mutex_) { + mutex_.Lock(); + update_tracker_.LossyAdd(method); + assert(rep_); +} + +void CordzInfo::Unlock() ABSL_UNLOCK_FUNCTION(mutex_) { + bool tracked = rep_ != nullptr; + mutex_.Unlock(); + if (!tracked) { + Untrack(); + } +} + +absl::Span<void* const> CordzInfo::GetStack() const { + return absl::MakeConstSpan(stack_, stack_depth_); +} + +absl::Span<void* const> CordzInfo::GetParentStack() const { + return absl::MakeConstSpan(parent_stack_, parent_stack_depth_); +} + +CordzStatistics CordzInfo::GetCordzStatistics() const { + CordzStatistics stats; + stats.method = method_; + stats.parent_method = parent_method_; + stats.update_tracker = update_tracker_; + if (CordRep* rep = RefCordRep()) { + stats.size = rep->length; + CordRepAnalyzer analyzer(stats); + analyzer.AnalyzeCordRep(rep); + CordRep::Unref(rep); + } + return stats; +} + +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/absl/strings/internal/cordz_info.h b/absl/strings/internal/cordz_info.h new file mode 100644 index 00000000..026d5b99 --- /dev/null +++ b/absl/strings/internal/cordz_info.h @@ -0,0 +1,298 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_STRINGS_CORDZ_INFO_H_ +#define ABSL_STRINGS_CORDZ_INFO_H_ + +#include <atomic> +#include <cstdint> +#include <functional> + +#include "absl/base/config.h" +#include "absl/base/internal/raw_logging.h" +#include "absl/base/internal/spinlock.h" +#include "absl/base/thread_annotations.h" +#include "absl/strings/internal/cord_internal.h" +#include "absl/strings/internal/cordz_functions.h" +#include "absl/strings/internal/cordz_handle.h" +#include "absl/strings/internal/cordz_statistics.h" +#include "absl/strings/internal/cordz_update_tracker.h" +#include "absl/synchronization/mutex.h" +#include "absl/types/span.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { + +// CordzInfo tracks a profiled Cord. Each of these objects can be in two places. +// If a Cord is alive, the CordzInfo will be in the global_cordz_infos map, and +// can also be retrieved via the linked list starting with +// global_cordz_infos_head and continued via the cordz_info_next() method. When +// a Cord has reached the end of its lifespan, the CordzInfo object will be +// migrated out of the global_cordz_infos list and the global_cordz_infos_map, +// and will either be deleted or appended to the global_delete_queue. If it is +// placed on the global_delete_queue, the CordzInfo object will be cleaned in +// the destructor of a CordzSampleToken object. +class ABSL_LOCKABLE CordzInfo : public CordzHandle { + public: + using MethodIdentifier = CordzUpdateTracker::MethodIdentifier; + + // TrackCord creates a CordzInfo instance which tracks important metrics of + // a sampled cord, and stores the created CordzInfo instance into `cord'. All + // CordzInfo instances are placed in a global list which is used to discover + // and snapshot all actively tracked cords. Callers are responsible for + // calling UntrackCord() before the tracked Cord instance is deleted, or to + // stop tracking the sampled Cord. Callers are also responsible for guarding + // changes to the 'tree' value of a Cord (InlineData.tree) through the Lock() + // and Unlock() calls. Any change resulting in a new tree value for the cord + // requires a call to SetCordRep() before the old tree has been unreffed + // and/or deleted. `method` identifies the Cord public API method initiating + // the cord to be sampled. + // Requires `cord` to hold a tree, and `cord.cordz_info()` to be null. + static void TrackCord(InlineData& cord, MethodIdentifier method); + + // Identical to TrackCord(), except that this function fills the + // `parent_stack` and `parent_method` properties of the returned CordzInfo + // instance from the provided `src` instance if `src` is sampled. + // This function should be used for sampling 'copy constructed' and 'copy + // assigned' cords. This function allows 'cord` to be already sampled, in + // which case the CordzInfo will be newly created from `src`. + static void TrackCord(InlineData& cord, const InlineData& src, + MethodIdentifier method); + + // Maybe sample the cord identified by 'cord' for method 'method'. + // Uses `cordz_should_profile` to randomly pick cords to be sampled, and if + // so, invokes `TrackCord` to start sampling `cord`. + static void MaybeTrackCord(InlineData& cord, MethodIdentifier method); + + // Maybe sample the cord identified by 'cord' for method 'method'. + // `src` identifies a 'parent' cord which is assigned to `cord`, typically the + // input cord for a copy constructor, or an assign method such as `operator=` + // `cord` will be sampled if (and only if) `src` is sampled. + // If `cord` is currently being sampled and `src` is not being sampled, then + // this function will stop sampling the cord and reset the cord's cordz_info. + // + // Previously this function defined that `cord` will be sampled if either + // `src` is sampled, or if `cord` is randomly picked for sampling. However, + // this can cause issues, as there may be paths where some cord is assigned an + // indirect copy of it's own value. As such a 'string of copies' would then + // remain sampled (`src.is_profiled`), then assigning such a cord back to + // 'itself' creates a cycle where the cord will converge to 'always sampled`. + // + // For example: + // + // Cord x; + // for (...) { + // // Copy ctor --> y.is_profiled := x.is_profiled | random(...) + // Cord y = x; + // ... + // // Assign x = y --> x.is_profiled = y.is_profiled | random(...) + // // ==> x.is_profiled |= random(...) + // // ==> x converges to 'always profiled' + // x = y; + // } + static void MaybeTrackCord(InlineData& cord, const InlineData& src, + MethodIdentifier method); + + // Stops tracking changes for a sampled cord, and deletes the provided info. + // This function must be called before the sampled cord instance is deleted, + // and before the root cordrep of the sampled cord is unreffed. + // This function may extend the lifetime of the cordrep in cases where the + // CordInfo instance is being held by a concurrent collection thread. + void Untrack(); + + // Invokes UntrackCord() on `info` if `info` is not null. + static void MaybeUntrackCord(CordzInfo* info); + + CordzInfo() = delete; + CordzInfo(const CordzInfo&) = delete; + CordzInfo& operator=(const CordzInfo&) = delete; + + // Retrieves the oldest existing CordzInfo. + static CordzInfo* Head(const CordzSnapshot& snapshot) + ABSL_NO_THREAD_SAFETY_ANALYSIS; + + // Retrieves the next oldest existing CordzInfo older than 'this' instance. + CordzInfo* Next(const CordzSnapshot& snapshot) const + ABSL_NO_THREAD_SAFETY_ANALYSIS; + + // Locks this instance for the update identified by `method`. + // Increases the count for `method` in `update_tracker`. + void Lock(MethodIdentifier method) ABSL_EXCLUSIVE_LOCK_FUNCTION(mutex_); + + // Unlocks this instance. If the contained `rep` has been set to null + // indicating the Cord has been cleared or is otherwise no longer sampled, + // then this method will delete this CordzInfo instance. + void Unlock() ABSL_UNLOCK_FUNCTION(mutex_); + + // Asserts that this CordzInfo instance is locked. + void AssertHeld() ABSL_ASSERT_EXCLUSIVE_LOCK(mutex_); + + // Updates the `rep` property of this instance. This methods is invoked by + // Cord logic each time the root node of a sampled Cord changes, and before + // the old root reference count is deleted. This guarantees that collection + // code can always safely take a reference on the tracked cord. + // Requires a lock to be held through the `Lock()` method. + // TODO(b/117940323): annotate with ABSL_EXCLUSIVE_LOCKS_REQUIRED once all + // Cord code is in a state where this can be proven true by the compiler. + void SetCordRep(CordRep* rep); + + // Returns the current `rep` property of this instance with a reference + // added, or null if this instance represents a cord that has since been + // deleted or untracked. + CordRep* RefCordRep() const ABSL_LOCKS_EXCLUDED(mutex_); + + // Returns the current value of `rep_` for testing purposes only. + CordRep* GetCordRepForTesting() const ABSL_NO_THREAD_SAFETY_ANALYSIS { + return rep_; + } + + // Sets the current value of `rep_` for testing purposes only. + void SetCordRepForTesting(CordRep* rep) ABSL_NO_THREAD_SAFETY_ANALYSIS { + rep_ = rep; + } + + // Returns the stack trace for where the cord was first sampled. Cords are + // potentially sampled when they promote from an inlined cord to a tree or + // ring representation, which is not necessarily the location where the cord + // was first created. Some cords are created as inlined cords, and only as + // data is added do they become a non-inlined cord. However, typically the + // location represents reasonably well where the cord is 'created'. + absl::Span<void* const> GetStack() const; + + // Returns the stack trace for a sampled cord's 'parent stack trace'. This + // value may be set if the cord is sampled (promoted) after being created + // from, or being assigned the value of an existing (sampled) cord. + absl::Span<void* const> GetParentStack() const; + + // Retrieves the CordzStatistics associated with this Cord. The statistics + // are only updated when a Cord goes through a mutation, such as an Append + // or RemovePrefix. + CordzStatistics GetCordzStatistics() const; + + private: + using SpinLock = absl::base_internal::SpinLock; + using SpinLockHolder = ::absl::base_internal::SpinLockHolder; + + // Global cordz info list. CordzInfo stores a pointer to the global list + // instance to harden against ODR violations. + struct List { + constexpr explicit List(absl::ConstInitType) + : mutex(absl::kConstInit, + absl::base_internal::SCHEDULE_COOPERATIVE_AND_KERNEL) {} + + SpinLock mutex; + std::atomic<CordzInfo*> head ABSL_GUARDED_BY(mutex){nullptr}; + }; + + static constexpr int kMaxStackDepth = 64; + + explicit CordzInfo(CordRep* rep, const CordzInfo* src, + MethodIdentifier method); + ~CordzInfo() override; + + // Sets `rep_` without holding a lock. + void UnsafeSetCordRep(CordRep* rep) ABSL_NO_THREAD_SAFETY_ANALYSIS; + + void Track(); + + // Returns the parent method from `src`, which is either `parent_method_` or + // `method_` depending on `parent_method_` being kUnknown. + // Returns kUnknown if `src` is null. + static MethodIdentifier GetParentMethod(const CordzInfo* src); + + // Fills the provided stack from `src`, copying either `parent_stack_` or + // `stack_` depending on `parent_stack_` being empty, returning the size of + // the parent stack. + // Returns 0 if `src` is null. + static int FillParentStack(const CordzInfo* src, void** stack); + + void ODRCheck() const { +#ifndef NDEBUG + ABSL_RAW_CHECK(list_ == &global_list_, "ODR violation in Cord"); +#endif + } + + // Non-inlined implementation of `MaybeTrackCord`, which is executed if + // either `src` is sampled or `cord` is sampled, and either untracks or + // tracks `cord` as documented per `MaybeTrackCord`. + static void MaybeTrackCordImpl(InlineData& cord, const InlineData& src, + MethodIdentifier method); + + ABSL_CONST_INIT static List global_list_; + List* const list_ = &global_list_; + + // ci_prev_ and ci_next_ require the global list mutex to be held. + // Unfortunately we can't use thread annotations such that the thread safety + // analysis understands that list_ and global_list_ are one and the same. + std::atomic<CordzInfo*> ci_prev_{nullptr}; + std::atomic<CordzInfo*> ci_next_{nullptr}; + + mutable absl::Mutex mutex_; + CordRep* rep_ ABSL_GUARDED_BY(mutex_); + + void* stack_[kMaxStackDepth]; + void* parent_stack_[kMaxStackDepth]; + const int stack_depth_; + const int parent_stack_depth_; + const MethodIdentifier method_; + const MethodIdentifier parent_method_; + CordzUpdateTracker update_tracker_; + const absl::Time create_time_; +}; + +inline ABSL_ATTRIBUTE_ALWAYS_INLINE void CordzInfo::MaybeTrackCord( + InlineData& cord, MethodIdentifier method) { + if (ABSL_PREDICT_FALSE(cordz_should_profile())) { + TrackCord(cord, method); + } +} + +inline ABSL_ATTRIBUTE_ALWAYS_INLINE void CordzInfo::MaybeTrackCord( + InlineData& cord, const InlineData& src, MethodIdentifier method) { + if (ABSL_PREDICT_FALSE(InlineData::is_either_profiled(cord, src))) { + MaybeTrackCordImpl(cord, src, method); + } +} + +inline ABSL_ATTRIBUTE_ALWAYS_INLINE void CordzInfo::MaybeUntrackCord( + CordzInfo* info) { + if (ABSL_PREDICT_FALSE(info)) { + info->Untrack(); + } +} + +inline void CordzInfo::AssertHeld() ABSL_ASSERT_EXCLUSIVE_LOCK(mutex_) { +#ifndef NDEBUG + mutex_.AssertHeld(); +#endif +} + +inline void CordzInfo::SetCordRep(CordRep* rep) { + AssertHeld(); + rep_ = rep; +} + +inline void CordzInfo::UnsafeSetCordRep(CordRep* rep) { rep_ = rep; } + +inline CordRep* CordzInfo::RefCordRep() const ABSL_LOCKS_EXCLUDED(mutex_) { + MutexLock lock(&mutex_); + return rep_ ? CordRep::Ref(rep_) : nullptr; +} + +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_CORDZ_INFO_H_ diff --git a/absl/strings/internal/cordz_info_statistics_test.cc b/absl/strings/internal/cordz_info_statistics_test.cc new file mode 100644 index 00000000..476c38d2 --- /dev/null +++ b/absl/strings/internal/cordz_info_statistics_test.cc @@ -0,0 +1,555 @@ +// Copyright 2021 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <iostream> +#include <random> +#include <vector> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/config.h" +#include "absl/strings/cord.h" +#include "absl/strings/internal/cord_internal.h" +#include "absl/strings/internal/cord_rep_btree.h" +#include "absl/strings/internal/cord_rep_crc.h" +#include "absl/strings/internal/cord_rep_flat.h" +#include "absl/strings/internal/cord_rep_ring.h" +#include "absl/strings/internal/cordz_info.h" +#include "absl/strings/internal/cordz_sample_token.h" +#include "absl/strings/internal/cordz_statistics.h" +#include "absl/strings/internal/cordz_update_scope.h" +#include "absl/strings/internal/cordz_update_tracker.h" +#include "absl/synchronization/internal/thread_pool.h" +#include "absl/synchronization/notification.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { + +// Do not print statistics contents, the matcher prints them as needed. +inline void PrintTo(const CordzStatistics& stats, std::ostream* s) { + if (s) *s << "CordzStatistics{...}"; +} + +namespace { + +using ::testing::Ge; + +// Creates a flat of the specified allocated size +CordRepFlat* Flat(size_t size) { + // Round up to a tag size, as we are going to poke an exact tag size back into + // the allocated flat. 'size returning allocators' could grant us more than we + // wanted, but we are ok to poke the 'requested' size in the tag, even in the + // presence of sized deletes, so we need to make sure the size rounds + // perfectly to a tag value. + assert(size >= kMinFlatSize); + size = RoundUpForTag(size); + CordRepFlat* flat = CordRepFlat::New(size - kFlatOverhead); + flat->tag = AllocatedSizeToTag(size); + flat->length = size - kFlatOverhead; + return flat; +} + +// Creates an external of the specified length +CordRepExternal* External(int length = 512) { + return static_cast<CordRepExternal*>( + NewExternalRep(absl::string_view("", length), [](absl::string_view) {})); +} + +// Creates a substring on the provided rep of length - 1 +CordRepSubstring* Substring(CordRep* rep) { + auto* substring = new CordRepSubstring; + substring->length = rep->length - 1; + substring->tag = SUBSTRING; + substring->child = rep; + return substring; +} + +// Reference count helper +struct RefHelper { + std::vector<CordRep*> refs; + + ~RefHelper() { + for (CordRep* rep : refs) { + CordRep::Unref(rep); + } + } + + // Invokes CordRep::Unref() on `rep` when this instance is destroyed. + template <typename T> + T* NeedsUnref(T* rep) { + refs.push_back(rep); + return rep; + } + + // Adds `n` reference counts to `rep` which will be unreffed when this + // instance is destroyed. + template <typename T> + T* Ref(T* rep, size_t n = 1) { + while (n--) { + NeedsUnref(CordRep::Ref(rep)); + } + return rep; + } +}; + +// Sizeof helper. Returns the allocated size of `p`, excluding any child +// elements for substring, concat and ring cord reps. +template <typename T> +size_t SizeOf(const T* rep) { + return sizeof(T); +} + +template <> +size_t SizeOf(const CordRepFlat* rep) { + return rep->AllocatedSize(); +} + +template <> +size_t SizeOf(const CordRepExternal* rep) { + // See cord.cc + return sizeof(CordRepExternalImpl<intptr_t>) + rep->length; +} + +template <> +size_t SizeOf(const CordRepRing* rep) { + return CordRepRing::AllocSize(rep->capacity()); +} + +// Computes fair share memory used in a naive 'we dare to recurse' way. +double FairShareImpl(CordRep* rep, size_t ref) { + double self = 0.0, children = 0.0; + ref *= rep->refcount.Get(); + if (rep->tag >= FLAT) { + self = SizeOf(rep->flat()); + } else if (rep->tag == EXTERNAL) { + self = SizeOf(rep->external()); + } else if (rep->tag == SUBSTRING) { + self = SizeOf(rep->substring()); + children = FairShareImpl(rep->substring()->child, ref); + } else if (rep->tag == BTREE) { + self = SizeOf(rep->btree()); + for (CordRep*edge : rep->btree()->Edges()) { + children += FairShareImpl(edge, ref); + } + } else if (rep->tag == RING) { + self = SizeOf(rep->ring()); + rep->ring()->ForEach([&](CordRepRing::index_type i) { + self += FairShareImpl(rep->ring()->entry_child(i), 1); + }); + } else { + assert(false); + } + return self / ref + children; +} + +// Returns the fair share memory size from `ShareFhareImpl()` as a size_t. +size_t FairShare(CordRep* rep, size_t ref = 1) { + return static_cast<size_t>(FairShareImpl(rep, ref)); +} + +// Samples the cord and returns CordzInfo::GetStatistics() +CordzStatistics SampleCord(CordRep* rep) { + InlineData cord(rep); + CordzInfo::TrackCord(cord, CordzUpdateTracker::kUnknown); + CordzStatistics stats = cord.cordz_info()->GetCordzStatistics(); + cord.cordz_info()->Untrack(); + return stats; +} + +MATCHER_P(EqStatistics, stats, "Statistics equal expected values") { + bool ok = true; + +#define STATS_MATCHER_EXPECT_EQ(member) \ + if (stats.member != arg.member) { \ + *result_listener << "\n stats." << #member \ + << ": actual = " << arg.member << ", expected " \ + << stats.member; \ + ok = false; \ + } + + STATS_MATCHER_EXPECT_EQ(size); + STATS_MATCHER_EXPECT_EQ(node_count); + STATS_MATCHER_EXPECT_EQ(node_counts.flat); + STATS_MATCHER_EXPECT_EQ(node_counts.flat_64); + STATS_MATCHER_EXPECT_EQ(node_counts.flat_128); + STATS_MATCHER_EXPECT_EQ(node_counts.flat_256); + STATS_MATCHER_EXPECT_EQ(node_counts.flat_512); + STATS_MATCHER_EXPECT_EQ(node_counts.flat_1k); + STATS_MATCHER_EXPECT_EQ(node_counts.external); + STATS_MATCHER_EXPECT_EQ(node_counts.concat); + STATS_MATCHER_EXPECT_EQ(node_counts.substring); + STATS_MATCHER_EXPECT_EQ(node_counts.ring); + STATS_MATCHER_EXPECT_EQ(node_counts.btree); + STATS_MATCHER_EXPECT_EQ(estimated_memory_usage); + STATS_MATCHER_EXPECT_EQ(estimated_fair_share_memory_usage); + +#undef STATS_MATCHER_EXPECT_EQ + + return ok; +} + +TEST(CordzInfoStatisticsTest, Flat) { + RefHelper ref; + auto* flat = ref.NeedsUnref(Flat(512)); + + CordzStatistics expected; + expected.size = flat->length; + expected.estimated_memory_usage = SizeOf(flat); + expected.estimated_fair_share_memory_usage = expected.estimated_memory_usage; + expected.node_count = 1; + expected.node_counts.flat = 1; + expected.node_counts.flat_512 = 1; + + EXPECT_THAT(SampleCord(flat), EqStatistics(expected)); +} + +TEST(CordzInfoStatisticsTest, SharedFlat) { + RefHelper ref; + auto* flat = ref.Ref(ref.NeedsUnref(Flat(64))); + + CordzStatistics expected; + expected.size = flat->length; + expected.estimated_memory_usage = SizeOf(flat); + expected.estimated_fair_share_memory_usage = SizeOf(flat) / 2; + expected.node_count = 1; + expected.node_counts.flat = 1; + expected.node_counts.flat_64 = 1; + + EXPECT_THAT(SampleCord(flat), EqStatistics(expected)); +} + +TEST(CordzInfoStatisticsTest, External) { + RefHelper ref; + auto* external = ref.NeedsUnref(External()); + + CordzStatistics expected; + expected.size = external->length; + expected.estimated_memory_usage = SizeOf(external); + expected.estimated_fair_share_memory_usage = SizeOf(external); + expected.node_count = 1; + expected.node_counts.external = 1; + + EXPECT_THAT(SampleCord(external), EqStatistics(expected)); +} + +TEST(CordzInfoStatisticsTest, SharedExternal) { + RefHelper ref; + auto* external = ref.Ref(ref.NeedsUnref(External())); + + CordzStatistics expected; + expected.size = external->length; + expected.estimated_memory_usage = SizeOf(external); + expected.estimated_fair_share_memory_usage = SizeOf(external) / 2; + expected.node_count = 1; + expected.node_counts.external = 1; + + EXPECT_THAT(SampleCord(external), EqStatistics(expected)); +} + +TEST(CordzInfoStatisticsTest, Substring) { + RefHelper ref; + auto* flat = Flat(1024); + auto* substring = ref.NeedsUnref(Substring(flat)); + + CordzStatistics expected; + expected.size = substring->length; + expected.estimated_memory_usage = SizeOf(substring) + SizeOf(flat); + expected.estimated_fair_share_memory_usage = expected.estimated_memory_usage; + expected.node_count = 2; + expected.node_counts.flat = 1; + expected.node_counts.flat_1k = 1; + expected.node_counts.substring = 1; + + EXPECT_THAT(SampleCord(substring), EqStatistics(expected)); +} + +TEST(CordzInfoStatisticsTest, SharedSubstring) { + RefHelper ref; + auto* flat = ref.Ref(Flat(511), 2); + auto* substring = ref.Ref(ref.NeedsUnref(Substring(flat))); + + CordzStatistics expected; + expected.size = substring->length; + expected.estimated_memory_usage = SizeOf(flat) + SizeOf(substring); + expected.estimated_fair_share_memory_usage = + SizeOf(substring) / 2 + SizeOf(flat) / 6; + expected.node_count = 2; + expected.node_counts.flat = 1; + expected.node_counts.flat_512 = 1; + expected.node_counts.substring = 1; + + EXPECT_THAT(SampleCord(substring), EqStatistics(expected)); +} + + +TEST(CordzInfoStatisticsTest, Ring) { + RefHelper ref; + auto* flat1 = Flat(240); + auto* flat2 = Flat(2000); + auto* flat3 = Flat(70); + auto* external = External(3000); + CordRepRing* ring = CordRepRing::Create(flat1); + ring = CordRepRing::Append(ring, flat2); + ring = CordRepRing::Append(ring, flat3); + ring = ref.NeedsUnref(CordRepRing::Append(ring, external)); + + CordzStatistics expected; + expected.size = ring->length; + expected.estimated_memory_usage = SizeOf(ring) + SizeOf(flat1) + + SizeOf(flat2) + SizeOf(flat3) + + SizeOf(external); + expected.estimated_fair_share_memory_usage = expected.estimated_memory_usage; + expected.node_count = 5; + expected.node_counts.flat = 3; + expected.node_counts.flat_128 = 1; + expected.node_counts.flat_256 = 1; + expected.node_counts.external = 1; + expected.node_counts.ring = 1; + + EXPECT_THAT(SampleCord(ring), EqStatistics(expected)); +} + +TEST(CordzInfoStatisticsTest, SharedSubstringRing) { + RefHelper ref; + auto* flat1 = ref.Ref(Flat(240)); + auto* flat2 = Flat(200); + auto* flat3 = Flat(70); + auto* external = ref.Ref(External(3000), 5); + CordRepRing* ring = CordRepRing::Create(flat1); + ring = CordRepRing::Append(ring, flat2); + ring = CordRepRing::Append(ring, flat3); + ring = ref.Ref(CordRepRing::Append(ring, external), 4); + auto* substring = ref.Ref(ref.NeedsUnref(Substring(ring))); + + + CordzStatistics expected; + expected.size = substring->length; + expected.estimated_memory_usage = SizeOf(ring) + SizeOf(flat1) + + SizeOf(flat2) + SizeOf(flat3) + + SizeOf(external) + SizeOf(substring); + expected.estimated_fair_share_memory_usage = FairShare(substring); + expected.node_count = 6; + expected.node_counts.flat = 3; + expected.node_counts.flat_128 = 1; + expected.node_counts.flat_256 = 2; + expected.node_counts.external = 1; + expected.node_counts.ring = 1; + expected.node_counts.substring = 1; + + EXPECT_THAT(SampleCord(substring), EqStatistics(expected)); +} + +TEST(CordzInfoStatisticsTest, BtreeLeaf) { + ASSERT_THAT(CordRepBtree::kMaxCapacity, Ge(3)); + RefHelper ref; + auto* flat1 = Flat(2000); + auto* flat2 = Flat(200); + auto* substr = Substring(flat2); + auto* external = External(3000); + + CordRepBtree* tree = CordRepBtree::Create(flat1); + tree = CordRepBtree::Append(tree, substr); + tree = CordRepBtree::Append(tree, external); + size_t flat3_count = CordRepBtree::kMaxCapacity - 3; + size_t flat3_size = 0; + for (size_t i = 0; i < flat3_count; ++i) { + auto* flat3 = Flat(70); + flat3_size += SizeOf(flat3); + tree = CordRepBtree::Append(tree, flat3); + } + ref.NeedsUnref(tree); + + CordzStatistics expected; + expected.size = tree->length; + expected.estimated_memory_usage = SizeOf(tree) + SizeOf(flat1) + + SizeOf(flat2) + SizeOf(substr) + + flat3_size + SizeOf(external); + expected.estimated_fair_share_memory_usage = expected.estimated_memory_usage; + expected.node_count = 1 + 3 + 1 + flat3_count; + expected.node_counts.flat = 2 + flat3_count; + expected.node_counts.flat_128 = flat3_count; + expected.node_counts.flat_256 = 1; + expected.node_counts.external = 1; + expected.node_counts.substring = 1; + expected.node_counts.btree = 1; + + EXPECT_THAT(SampleCord(tree), EqStatistics(expected)); +} + +TEST(CordzInfoStatisticsTest, BtreeNodeShared) { + RefHelper ref; + static constexpr int leaf_count = 3; + const size_t flat3_count = CordRepBtree::kMaxCapacity - 3; + ASSERT_THAT(flat3_count, Ge(0)); + + CordRepBtree* tree = nullptr; + size_t mem_size = 0; + for (int i = 0; i < leaf_count; ++i) { + auto* flat1 = ref.Ref(Flat(2000), 9); + mem_size += SizeOf(flat1); + if (i == 0) { + tree = CordRepBtree::Create(flat1); + } else { + tree = CordRepBtree::Append(tree, flat1); + } + + auto* flat2 = Flat(200); + auto* substr = Substring(flat2); + mem_size += SizeOf(flat2) + SizeOf(substr); + tree = CordRepBtree::Append(tree, substr); + + auto* external = External(30); + mem_size += SizeOf(external); + tree = CordRepBtree::Append(tree, external); + + for (size_t i = 0; i < flat3_count; ++i) { + auto* flat3 = Flat(70); + mem_size += SizeOf(flat3); + tree = CordRepBtree::Append(tree, flat3); + } + + if (i == 0) { + mem_size += SizeOf(tree); + } else { + mem_size += SizeOf(tree->Edges().back()->btree()); + } + } + ref.NeedsUnref(tree); + + // Ref count: 2 for top (add 1), 5 for leaf 0 (add 4). + ref.Ref(tree, 1); + ref.Ref(tree->Edges().front(), 4); + + CordzStatistics expected; + expected.size = tree->length; + expected.estimated_memory_usage = SizeOf(tree) + mem_size; + expected.estimated_fair_share_memory_usage = FairShare(tree); + + expected.node_count = 1 + leaf_count * (1 + 3 + 1 + flat3_count); + expected.node_counts.flat = leaf_count * (2 + flat3_count); + expected.node_counts.flat_128 = leaf_count * flat3_count; + expected.node_counts.flat_256 = leaf_count; + expected.node_counts.external = leaf_count; + expected.node_counts.substring = leaf_count; + expected.node_counts.btree = 1 + leaf_count; + + EXPECT_THAT(SampleCord(tree), EqStatistics(expected)); +} + +TEST(CordzInfoStatisticsTest, Crc) { + RefHelper ref; + auto* left = Flat(1000); + auto* crc = ref.NeedsUnref(CordRepCrc::New(left, 12345)); + + CordzStatistics expected; + expected.size = left->length; + expected.estimated_memory_usage = SizeOf(crc) + SizeOf(left); + expected.estimated_fair_share_memory_usage = expected.estimated_memory_usage; + expected.node_count = 2; + expected.node_counts.flat = 1; + expected.node_counts.flat_1k = 1; + expected.node_counts.crc = 1; + + EXPECT_THAT(SampleCord(crc), EqStatistics(expected)); +} + +TEST(CordzInfoStatisticsTest, ThreadSafety) { + Notification stop; + static constexpr int kNumThreads = 8; + int64_t sampled_node_count = 0; + + { + absl::synchronization_internal::ThreadPool pool(kNumThreads); + + // Run analyzer thread emulating a CordzHandler collection. + pool.Schedule([&]() { + while (!stop.HasBeenNotified()) { + // Run every 10us (about 100K total collections). + absl::SleepFor(absl::Microseconds(10)); + CordzSampleToken token; + for (const CordzInfo& cord_info : token) { + CordzStatistics stats = cord_info.GetCordzStatistics(); + sampled_node_count += stats.node_count; + } + } + }); + + // Run 'application threads' + for (int i = 0; i < kNumThreads; ++i) { + pool.Schedule([&]() { + // Track 0 - 2 cordz infos at a time, providing permutations of 0, 1 + // and 2 CordzHandle and CordzInfo queues being active, with plenty of + // 'empty to non empty' transitions. + InlineData cords[2]; + std::minstd_rand gen; + std::uniform_int_distribution<int> coin_toss(0, 1); + + while (!stop.HasBeenNotified()) { + for (InlineData& cord : cords) { + // 50/50 flip the state of the cord + if (coin_toss(gen) != 0) { + if (cord.is_tree()) { + // 50/50 simulate delete (untrack) or 'edit to empty' + if (coin_toss(gen) != 0) { + CordzInfo::MaybeUntrackCord(cord.cordz_info()); + } else { + CordzUpdateScope scope(cord.cordz_info(), + CordzUpdateTracker::kUnknown); + scope.SetCordRep(nullptr); + } + CordRep::Unref(cord.as_tree()); + cord.set_inline_size(0); + } else { + // Coin toss to 25% ring, 25% btree, and 50% flat. + CordRep* rep = Flat(256); + if (coin_toss(gen) != 0) { + if (coin_toss(gen) != 0) { + rep = CordRepRing::Create(rep); + } else { + rep = CordRepBtree::Create(rep); + } + } + cord.make_tree(rep); + + // 50/50 sample + if (coin_toss(gen) != 0) { + CordzInfo::TrackCord(cord, CordzUpdateTracker::kUnknown); + } + } + } + } + } + for (InlineData& cord : cords) { + if (cord.is_tree()) { + CordzInfo::MaybeUntrackCord(cord.cordz_info()); + CordRep::Unref(cord.as_tree()); + } + } + }); + } + + // Run for 1 second to give memory and thread safety analyzers plenty of + // time to detect any mishaps or undefined behaviors. + absl::SleepFor(absl::Seconds(1)); + stop.Notify(); + } + + std::cout << "Sampled " << sampled_node_count << " nodes\n"; +} + +} // namespace +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/absl/strings/internal/cordz_info_test.cc b/absl/strings/internal/cordz_info_test.cc new file mode 100644 index 00000000..b98343ae --- /dev/null +++ b/absl/strings/internal/cordz_info_test.cc @@ -0,0 +1,341 @@ +// Copyright 2019 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/cordz_info.h" + +#include <vector> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/config.h" +#include "absl/debugging/stacktrace.h" +#include "absl/debugging/symbolize.h" +#include "absl/strings/cordz_test_helpers.h" +#include "absl/strings/internal/cord_rep_flat.h" +#include "absl/strings/internal/cordz_handle.h" +#include "absl/strings/internal/cordz_statistics.h" +#include "absl/strings/internal/cordz_update_tracker.h" +#include "absl/strings/str_cat.h" +#include "absl/types/span.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { +namespace { + +using ::testing::ElementsAre; +using ::testing::Eq; +using ::testing::HasSubstr; +using ::testing::Ne; +using ::testing::SizeIs; + +// Used test values +auto constexpr kUnknownMethod = CordzUpdateTracker::kUnknown; +auto constexpr kTrackCordMethod = CordzUpdateTracker::kConstructorString; +auto constexpr kChildMethod = CordzUpdateTracker::kConstructorCord; +auto constexpr kUpdateMethod = CordzUpdateTracker::kAppendString; + +// Local less verbose helper +std::vector<const CordzHandle*> DeleteQueue() { + return CordzHandle::DiagnosticsGetDeleteQueue(); +} + +std::string FormatStack(absl::Span<void* const> raw_stack) { + static constexpr size_t buf_size = 1 << 14; + std::unique_ptr<char[]> buf(new char[buf_size]); + std::string output; + for (void* stackp : raw_stack) { + if (absl::Symbolize(stackp, buf.get(), buf_size)) { + absl::StrAppend(&output, " ", buf.get(), "\n"); + } + } + return output; +} + +TEST(CordzInfoTest, TrackCord) { + TestCordData data; + CordzInfo::TrackCord(data.data, kTrackCordMethod); + CordzInfo* info = data.data.cordz_info(); + ASSERT_THAT(info, Ne(nullptr)); + EXPECT_FALSE(info->is_snapshot()); + EXPECT_THAT(CordzInfo::Head(CordzSnapshot()), Eq(info)); + EXPECT_THAT(info->GetCordRepForTesting(), Eq(data.rep.rep)); + info->Untrack(); +} + +TEST(CordzInfoTest, MaybeTrackChildCordWithoutSampling) { + CordzSamplingIntervalHelper sample_none(99999); + TestCordData parent, child; + CordzInfo::MaybeTrackCord(child.data, parent.data, kTrackCordMethod); + EXPECT_THAT(child.data.cordz_info(), Eq(nullptr)); +} + +TEST(CordzInfoTest, MaybeTrackChildCordWithSampling) { + CordzSamplingIntervalHelper sample_all(1); + TestCordData parent, child; + CordzInfo::MaybeTrackCord(child.data, parent.data, kTrackCordMethod); + EXPECT_THAT(child.data.cordz_info(), Eq(nullptr)); +} + +TEST(CordzInfoTest, MaybeTrackChildCordWithoutSamplingParentSampled) { + CordzSamplingIntervalHelper sample_none(99999); + TestCordData parent, child; + CordzInfo::TrackCord(parent.data, kTrackCordMethod); + CordzInfo::MaybeTrackCord(child.data, parent.data, kTrackCordMethod); + CordzInfo* parent_info = parent.data.cordz_info(); + CordzInfo* child_info = child.data.cordz_info(); + ASSERT_THAT(child_info, Ne(nullptr)); + EXPECT_THAT(child_info->GetCordRepForTesting(), Eq(child.rep.rep)); + EXPECT_THAT(child_info->GetParentStack(), parent_info->GetStack()); + parent_info->Untrack(); + child_info->Untrack(); +} + +TEST(CordzInfoTest, MaybeTrackChildCordWithoutSamplingChildSampled) { + CordzSamplingIntervalHelper sample_none(99999); + TestCordData parent, child; + CordzInfo::TrackCord(child.data, kTrackCordMethod); + CordzInfo::MaybeTrackCord(child.data, parent.data, kTrackCordMethod); + EXPECT_THAT(child.data.cordz_info(), Eq(nullptr)); +} + +TEST(CordzInfoTest, MaybeTrackChildCordWithSamplingChildSampled) { + CordzSamplingIntervalHelper sample_all(1); + TestCordData parent, child; + CordzInfo::TrackCord(child.data, kTrackCordMethod); + CordzInfo::MaybeTrackCord(child.data, parent.data, kTrackCordMethod); + EXPECT_THAT(child.data.cordz_info(), Eq(nullptr)); +} + +TEST(CordzInfoTest, UntrackCord) { + TestCordData data; + CordzInfo::TrackCord(data.data, kTrackCordMethod); + CordzInfo* info = data.data.cordz_info(); + + info->Untrack(); + EXPECT_THAT(DeleteQueue(), SizeIs(0)); +} + +TEST(CordzInfoTest, UntrackCordWithSnapshot) { + TestCordData data; + CordzInfo::TrackCord(data.data, kTrackCordMethod); + CordzInfo* info = data.data.cordz_info(); + + CordzSnapshot snapshot; + info->Untrack(); + EXPECT_THAT(CordzInfo::Head(CordzSnapshot()), Eq(nullptr)); + EXPECT_THAT(info->GetCordRepForTesting(), Eq(data.rep.rep)); + EXPECT_THAT(DeleteQueue(), ElementsAre(info, &snapshot)); +} + +TEST(CordzInfoTest, SetCordRep) { + TestCordData data; + CordzInfo::TrackCord(data.data, kTrackCordMethod); + CordzInfo* info = data.data.cordz_info(); + + TestCordRep rep; + info->Lock(CordzUpdateTracker::kAppendCord); + info->SetCordRep(rep.rep); + info->Unlock(); + EXPECT_THAT(info->GetCordRepForTesting(), Eq(rep.rep)); + + info->Untrack(); +} + +TEST(CordzInfoTest, SetCordRepNullUntracksCordOnUnlock) { + TestCordData data; + CordzInfo::TrackCord(data.data, kTrackCordMethod); + CordzInfo* info = data.data.cordz_info(); + + info->Lock(CordzUpdateTracker::kAppendString); + info->SetCordRep(nullptr); + EXPECT_THAT(info->GetCordRepForTesting(), Eq(nullptr)); + EXPECT_THAT(CordzInfo::Head(CordzSnapshot()), Eq(info)); + + info->Unlock(); + EXPECT_THAT(CordzInfo::Head(CordzSnapshot()), Eq(nullptr)); +} + +TEST(CordzInfoTest, RefCordRep) { + TestCordData data; + CordzInfo::TrackCord(data.data, kTrackCordMethod); + CordzInfo* info = data.data.cordz_info(); + + size_t refcount = data.rep.rep->refcount.Get(); + EXPECT_THAT(info->RefCordRep(), Eq(data.rep.rep)); + EXPECT_THAT(data.rep.rep->refcount.Get(), Eq(refcount + 1)); + CordRep::Unref(data.rep.rep); + info->Untrack(); +} + +#if GTEST_HAS_DEATH_TEST + +TEST(CordzInfoTest, SetCordRepRequiresMutex) { + TestCordData data; + CordzInfo::TrackCord(data.data, kTrackCordMethod); + CordzInfo* info = data.data.cordz_info(); + TestCordRep rep; + EXPECT_DEBUG_DEATH(info->SetCordRep(rep.rep), ".*"); + info->Untrack(); +} + +#endif // GTEST_HAS_DEATH_TEST + +TEST(CordzInfoTest, TrackUntrackHeadFirstV2) { + CordzSnapshot snapshot; + EXPECT_THAT(CordzInfo::Head(snapshot), Eq(nullptr)); + + TestCordData data; + CordzInfo::TrackCord(data.data, kTrackCordMethod); + CordzInfo* info1 = data.data.cordz_info(); + ASSERT_THAT(CordzInfo::Head(snapshot), Eq(info1)); + EXPECT_THAT(info1->Next(snapshot), Eq(nullptr)); + + TestCordData data2; + CordzInfo::TrackCord(data2.data, kTrackCordMethod); + CordzInfo* info2 = data2.data.cordz_info(); + ASSERT_THAT(CordzInfo::Head(snapshot), Eq(info2)); + EXPECT_THAT(info2->Next(snapshot), Eq(info1)); + EXPECT_THAT(info1->Next(snapshot), Eq(nullptr)); + + info2->Untrack(); + ASSERT_THAT(CordzInfo::Head(snapshot), Eq(info1)); + EXPECT_THAT(info1->Next(snapshot), Eq(nullptr)); + + info1->Untrack(); + ASSERT_THAT(CordzInfo::Head(snapshot), Eq(nullptr)); +} + +TEST(CordzInfoTest, TrackUntrackTailFirstV2) { + CordzSnapshot snapshot; + EXPECT_THAT(CordzInfo::Head(snapshot), Eq(nullptr)); + + TestCordData data; + CordzInfo::TrackCord(data.data, kTrackCordMethod); + CordzInfo* info1 = data.data.cordz_info(); + ASSERT_THAT(CordzInfo::Head(snapshot), Eq(info1)); + EXPECT_THAT(info1->Next(snapshot), Eq(nullptr)); + + TestCordData data2; + CordzInfo::TrackCord(data2.data, kTrackCordMethod); + CordzInfo* info2 = data2.data.cordz_info(); + ASSERT_THAT(CordzInfo::Head(snapshot), Eq(info2)); + EXPECT_THAT(info2->Next(snapshot), Eq(info1)); + EXPECT_THAT(info1->Next(snapshot), Eq(nullptr)); + + info1->Untrack(); + ASSERT_THAT(CordzInfo::Head(snapshot), Eq(info2)); + EXPECT_THAT(info2->Next(snapshot), Eq(nullptr)); + + info2->Untrack(); + ASSERT_THAT(CordzInfo::Head(snapshot), Eq(nullptr)); +} + +TEST(CordzInfoTest, StackV2) { + TestCordData data; + // kMaxStackDepth is intentionally less than 64 (which is the max depth that + // Cordz will record) because if the actual stack depth is over 64 + // (which it is on Apple platforms) then the expected_stack will end up + // catching a few frames at the end that the actual_stack didn't get and + // it will no longer be subset. At the time of this writing 58 is the max + // that will allow this test to pass (with a minimum os version of iOS 9), so + // rounded down to 50 to hopefully not run into this in the future if Apple + // makes small modifications to its testing stack. 50 is sufficient to prove + // that we got a decent stack. + static constexpr int kMaxStackDepth = 50; + CordzInfo::TrackCord(data.data, kTrackCordMethod); + CordzInfo* info = data.data.cordz_info(); + std::vector<void*> local_stack; + local_stack.resize(kMaxStackDepth); + // In some environments we don't get stack traces. For example in Android + // absl::GetStackTrace will return 0 indicating it didn't find any stack. The + // resultant formatted stack will be "", but that still equals the stack + // recorded in CordzInfo, which is also empty. The skip_count is 1 so that the + // line number of the current stack isn't included in the HasSubstr check. + local_stack.resize(absl::GetStackTrace(local_stack.data(), kMaxStackDepth, + /*skip_count=*/1)); + + std::string got_stack = FormatStack(info->GetStack()); + std::string expected_stack = FormatStack(local_stack); + // If TrackCord is inlined, got_stack should match expected_stack. If it isn't + // inlined, got_stack should include an additional frame not present in + // expected_stack. Either way, expected_stack should be a substring of + // got_stack. + EXPECT_THAT(got_stack, HasSubstr(expected_stack)); + + info->Untrack(); +} + +// Local helper functions to get different stacks for child and parent. +CordzInfo* TrackChildCord(InlineData& data, const InlineData& parent) { + CordzInfo::TrackCord(data, parent, kChildMethod); + return data.cordz_info(); +} +CordzInfo* TrackParentCord(InlineData& data) { + CordzInfo::TrackCord(data, kTrackCordMethod); + return data.cordz_info(); +} + +TEST(CordzInfoTest, GetStatistics) { + TestCordData data; + CordzInfo* info = TrackParentCord(data.data); + + CordzStatistics statistics = info->GetCordzStatistics(); + EXPECT_THAT(statistics.size, Eq(data.rep.rep->length)); + EXPECT_THAT(statistics.method, Eq(kTrackCordMethod)); + EXPECT_THAT(statistics.parent_method, Eq(kUnknownMethod)); + EXPECT_THAT(statistics.update_tracker.Value(kTrackCordMethod), Eq(1)); + + info->Untrack(); +} + +TEST(CordzInfoTest, LockCountsMethod) { + TestCordData data; + CordzInfo* info = TrackParentCord(data.data); + + info->Lock(kUpdateMethod); + info->Unlock(); + info->Lock(kUpdateMethod); + info->Unlock(); + + CordzStatistics statistics = info->GetCordzStatistics(); + EXPECT_THAT(statistics.update_tracker.Value(kUpdateMethod), Eq(2)); + + info->Untrack(); +} + +TEST(CordzInfoTest, FromParent) { + TestCordData parent; + TestCordData child; + CordzInfo* info_parent = TrackParentCord(parent.data); + CordzInfo* info_child = TrackChildCord(child.data, parent.data); + + std::string stack = FormatStack(info_parent->GetStack()); + std::string parent_stack = FormatStack(info_child->GetParentStack()); + EXPECT_THAT(stack, Eq(parent_stack)); + + CordzStatistics statistics = info_child->GetCordzStatistics(); + EXPECT_THAT(statistics.size, Eq(child.rep.rep->length)); + EXPECT_THAT(statistics.method, Eq(kChildMethod)); + EXPECT_THAT(statistics.parent_method, Eq(kTrackCordMethod)); + EXPECT_THAT(statistics.update_tracker.Value(kChildMethod), Eq(1)); + + info_parent->Untrack(); + info_child->Untrack(); +} + +} // namespace +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/absl/strings/internal/cordz_sample_token.cc b/absl/strings/internal/cordz_sample_token.cc new file mode 100644 index 00000000..ba1270d8 --- /dev/null +++ b/absl/strings/internal/cordz_sample_token.cc @@ -0,0 +1,64 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/cordz_sample_token.h" + +#include "absl/base/config.h" +#include "absl/strings/internal/cordz_handle.h" +#include "absl/strings/internal/cordz_info.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { + +CordzSampleToken::Iterator& CordzSampleToken::Iterator::operator++() { + if (current_) { + current_ = current_->Next(*token_); + } + return *this; +} + +CordzSampleToken::Iterator CordzSampleToken::Iterator::operator++(int) { + Iterator it(*this); + operator++(); + return it; +} + +bool operator==(const CordzSampleToken::Iterator& lhs, + const CordzSampleToken::Iterator& rhs) { + return lhs.current_ == rhs.current_ && + (lhs.current_ == nullptr || lhs.token_ == rhs.token_); +} + +bool operator!=(const CordzSampleToken::Iterator& lhs, + const CordzSampleToken::Iterator& rhs) { + return !(lhs == rhs); +} + +CordzSampleToken::Iterator::reference CordzSampleToken::Iterator::operator*() + const { + return *current_; +} + +CordzSampleToken::Iterator::pointer CordzSampleToken::Iterator::operator->() + const { + return current_; +} + +CordzSampleToken::Iterator::Iterator(const CordzSampleToken* token) + : token_(token), current_(CordzInfo::Head(*token)) {} + +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/absl/strings/internal/cordz_sample_token.h b/absl/strings/internal/cordz_sample_token.h new file mode 100644 index 00000000..28a1d70c --- /dev/null +++ b/absl/strings/internal/cordz_sample_token.h @@ -0,0 +1,97 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/base/config.h" +#include "absl/strings/internal/cordz_handle.h" +#include "absl/strings/internal/cordz_info.h" + +#ifndef ABSL_STRINGS_CORDZ_SAMPLE_TOKEN_H_ +#define ABSL_STRINGS_CORDZ_SAMPLE_TOKEN_H_ + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { + +// The existence of a CordzSampleToken guarantees that a reader can traverse the +// global_cordz_infos_head linked-list without needing to hold a mutex. When a +// CordzSampleToken exists, all CordzInfo objects that would be destroyed are +// instead appended to a deletion queue. When the CordzSampleToken is destroyed, +// it will also clean up any of these CordzInfo objects. +// +// E.g., ST are CordzSampleToken objects and CH are CordzHandle objects. +// ST1 <- CH1 <- CH2 <- ST2 <- CH3 <- global_delete_queue_tail +// +// This list tracks that CH1 and CH2 were created after ST1, so the thread +// holding ST1 might have a referece to CH1, CH2, ST2, and CH3. However, ST2 was +// created later, so the thread holding the ST2 token cannot have a reference to +// ST1, CH1, or CH2. If ST1 is cleaned up first, that thread will delete ST1, +// CH1, and CH2. If instead ST2 is cleaned up first, that thread will only +// delete ST2. +// +// If ST1 is cleaned up first, the new list will be: +// ST2 <- CH3 <- global_delete_queue_tail +// +// If ST2 is cleaned up first, the new list will be: +// ST1 <- CH1 <- CH2 <- CH3 <- global_delete_queue_tail +// +// All new CordzHandle objects are appended to the list, so if a new thread +// comes along before either ST1 or ST2 are cleaned up, the new list will be: +// ST1 <- CH1 <- CH2 <- ST2 <- CH3 <- ST3 <- global_delete_queue_tail +// +// A thread must hold the global_delete_queue_mu mutex whenever it's altering +// this list. +// +// It is safe for thread that holds a CordzSampleToken to read +// global_cordz_infos at any time since the objects it is able to retrieve will +// not be deleted while the CordzSampleToken exists. +class CordzSampleToken : public CordzSnapshot { + public: + class Iterator { + public: + using iterator_category = std::input_iterator_tag; + using value_type = const CordzInfo&; + using difference_type = ptrdiff_t; + using pointer = const CordzInfo*; + using reference = value_type; + + Iterator() = default; + + Iterator& operator++(); + Iterator operator++(int); + friend bool operator==(const Iterator& lhs, const Iterator& rhs); + friend bool operator!=(const Iterator& lhs, const Iterator& rhs); + reference operator*() const; + pointer operator->() const; + + private: + friend class CordzSampleToken; + explicit Iterator(const CordzSampleToken* token); + + const CordzSampleToken* token_ = nullptr; + pointer current_ = nullptr; + }; + + CordzSampleToken() = default; + CordzSampleToken(const CordzSampleToken&) = delete; + CordzSampleToken& operator=(const CordzSampleToken&) = delete; + + Iterator begin() { return Iterator(this); } + Iterator end() { return Iterator(); } +}; + +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_CORDZ_SAMPLE_TOKEN_H_ diff --git a/absl/strings/internal/cordz_sample_token_test.cc b/absl/strings/internal/cordz_sample_token_test.cc new file mode 100644 index 00000000..6be1770d --- /dev/null +++ b/absl/strings/internal/cordz_sample_token_test.cc @@ -0,0 +1,208 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/cordz_sample_token.h" + +#include <memory> +#include <type_traits> +#include <vector> + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/memory/memory.h" +#include "absl/random/random.h" +#include "absl/strings/cordz_test_helpers.h" +#include "absl/strings/internal/cord_rep_flat.h" +#include "absl/strings/internal/cordz_handle.h" +#include "absl/strings/internal/cordz_info.h" +#include "absl/synchronization/internal/thread_pool.h" +#include "absl/synchronization/notification.h" +#include "absl/time/clock.h" +#include "absl/time/time.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { +namespace { + +using ::testing::ElementsAre; +using ::testing::Eq; +using ::testing::Ne; + +// Used test values +auto constexpr kTrackCordMethod = CordzUpdateTracker::kConstructorString; + +TEST(CordzSampleTokenTest, IteratorTraits) { + static_assert(std::is_copy_constructible<CordzSampleToken::Iterator>::value, + ""); + static_assert(std::is_copy_assignable<CordzSampleToken::Iterator>::value, ""); + static_assert(std::is_move_constructible<CordzSampleToken::Iterator>::value, + ""); + static_assert(std::is_move_assignable<CordzSampleToken::Iterator>::value, ""); + static_assert( + std::is_same< + std::iterator_traits<CordzSampleToken::Iterator>::iterator_category, + std::input_iterator_tag>::value, + ""); + static_assert( + std::is_same<std::iterator_traits<CordzSampleToken::Iterator>::value_type, + const CordzInfo&>::value, + ""); + static_assert( + std::is_same< + std::iterator_traits<CordzSampleToken::Iterator>::difference_type, + ptrdiff_t>::value, + ""); + static_assert( + std::is_same<std::iterator_traits<CordzSampleToken::Iterator>::pointer, + const CordzInfo*>::value, + ""); + static_assert( + std::is_same<std::iterator_traits<CordzSampleToken::Iterator>::reference, + const CordzInfo&>::value, + ""); +} + +TEST(CordzSampleTokenTest, IteratorEmpty) { + CordzSampleToken token; + EXPECT_THAT(token.begin(), Eq(token.end())); +} + +TEST(CordzSampleTokenTest, Iterator) { + TestCordData cord1, cord2, cord3; + CordzInfo::TrackCord(cord1.data, kTrackCordMethod); + CordzInfo* info1 = cord1.data.cordz_info(); + CordzInfo::TrackCord(cord2.data, kTrackCordMethod); + CordzInfo* info2 = cord2.data.cordz_info(); + CordzInfo::TrackCord(cord3.data, kTrackCordMethod); + CordzInfo* info3 = cord3.data.cordz_info(); + + CordzSampleToken token; + std::vector<const CordzInfo*> found; + for (const CordzInfo& cord_info : token) { + found.push_back(&cord_info); + } + + EXPECT_THAT(found, ElementsAre(info3, info2, info1)); + + info1->Untrack(); + info2->Untrack(); + info3->Untrack(); +} + +TEST(CordzSampleTokenTest, IteratorEquality) { + TestCordData cord1; + TestCordData cord2; + TestCordData cord3; + CordzInfo::TrackCord(cord1.data, kTrackCordMethod); + CordzInfo* info1 = cord1.data.cordz_info(); + + CordzSampleToken token1; + // lhs starts with the CordzInfo corresponding to cord1 at the head. + CordzSampleToken::Iterator lhs = token1.begin(); + + CordzInfo::TrackCord(cord2.data, kTrackCordMethod); + CordzInfo* info2 = cord2.data.cordz_info(); + + CordzSampleToken token2; + // rhs starts with the CordzInfo corresponding to cord2 at the head. + CordzSampleToken::Iterator rhs = token2.begin(); + + CordzInfo::TrackCord(cord3.data, kTrackCordMethod); + CordzInfo* info3 = cord3.data.cordz_info(); + + // lhs is on cord1 while rhs is on cord2. + EXPECT_THAT(lhs, Ne(rhs)); + + rhs++; + // lhs and rhs are both on cord1, but they didn't come from the same + // CordzSampleToken. + EXPECT_THAT(lhs, Ne(rhs)); + + lhs++; + rhs++; + // Both lhs and rhs are done, so they are on nullptr. + EXPECT_THAT(lhs, Eq(rhs)); + + info1->Untrack(); + info2->Untrack(); + info3->Untrack(); +} + +TEST(CordzSampleTokenTest, MultiThreaded) { + Notification stop; + static constexpr int kNumThreads = 4; + static constexpr int kNumCords = 3; + static constexpr int kNumTokens = 3; + absl::synchronization_internal::ThreadPool pool(kNumThreads); + + for (int i = 0; i < kNumThreads; ++i) { + pool.Schedule([&stop]() { + absl::BitGen gen; + TestCordData cords[kNumCords]; + std::unique_ptr<CordzSampleToken> tokens[kNumTokens]; + + while (!stop.HasBeenNotified()) { + // Randomly perform one of five actions: + // 1) Untrack + // 2) Track + // 3) Iterate over Cords visible to a token. + // 4) Unsample + // 5) Sample + int index = absl::Uniform(gen, 0, kNumCords); + if (absl::Bernoulli(gen, 0.5)) { + TestCordData& cord = cords[index]; + // Track/untrack. + if (cord.data.is_profiled()) { + // 1) Untrack + cord.data.cordz_info()->Untrack(); + cord.data.clear_cordz_info(); + } else { + // 2) Track + CordzInfo::TrackCord(cord.data, kTrackCordMethod); + } + } else { + std::unique_ptr<CordzSampleToken>& token = tokens[index]; + if (token) { + if (absl::Bernoulli(gen, 0.5)) { + // 3) Iterate over Cords visible to a token. + for (const CordzInfo& info : *token) { + // This is trivial work to allow us to compile the loop. + EXPECT_THAT(info.Next(*token), Ne(&info)); + } + } else { + // 4) Unsample + token = nullptr; + } + } else { + // 5) Sample + token = absl::make_unique<CordzSampleToken>(); + } + } + } + for (TestCordData& cord : cords) { + CordzInfo::MaybeUntrackCord(cord.data.cordz_info()); + } + }); + } + // The threads will hammer away. Give it a little bit of time for tsan to + // spot errors. + absl::SleepFor(absl::Seconds(3)); + stop.Notify(); +} + +} // namespace +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/absl/strings/internal/cordz_statistics.h b/absl/strings/internal/cordz_statistics.h new file mode 100644 index 00000000..57071905 --- /dev/null +++ b/absl/strings/internal/cordz_statistics.h @@ -0,0 +1,88 @@ +// Copyright 2019 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_STRINGS_INTERNAL_CORDZ_STATISTICS_H_ +#define ABSL_STRINGS_INTERNAL_CORDZ_STATISTICS_H_ + +#include <cstdint> + +#include "absl/base/config.h" +#include "absl/strings/internal/cordz_update_tracker.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { + +// CordzStatistics captures some meta information about a Cord's shape. +struct CordzStatistics { + using MethodIdentifier = CordzUpdateTracker::MethodIdentifier; + + // Node counts information + struct NodeCounts { + size_t flat = 0; // #flats + size_t flat_64 = 0; // #flats up to 64 bytes + size_t flat_128 = 0; // #flats up to 128 bytes + size_t flat_256 = 0; // #flats up to 256 bytes + size_t flat_512 = 0; // #flats up to 512 bytes + size_t flat_1k = 0; // #flats up to 1K bytes + size_t external = 0; // #external reps + size_t substring = 0; // #substring reps + size_t concat = 0; // #concat reps + size_t ring = 0; // #ring buffer reps + size_t btree = 0; // #btree reps + size_t crc = 0; // #crc reps + }; + + // The size of the cord in bytes. This matches the result of Cord::size(). + int64_t size = 0; + + // The estimated memory used by the sampled cord. This value matches the + // value as reported by Cord::EstimatedMemoryUsage(). + // A value of 0 implies the property has not been recorded. + int64_t estimated_memory_usage = 0; + + // The effective memory used by the sampled cord, inversely weighted by the + // effective indegree of each allocated node. This is a representation of the + // fair share of memory usage that should be attributed to the sampled cord. + // This value is more useful for cases where one or more nodes are referenced + // by multiple Cord instances, and for cases where a Cord includes the same + // node multiple times (either directly or indirectly). + // A value of 0 implies the property has not been recorded. + int64_t estimated_fair_share_memory_usage = 0; + + // The total number of nodes referenced by this cord. + // For ring buffer Cords, this includes the 'ring buffer' node. + // For btree Cords, this includes all 'CordRepBtree' tree nodes as well as all + // the substring, flat and external nodes referenced by the tree. + // A value of 0 implies the property has not been recorded. + int64_t node_count = 0; + + // Detailed node counts per type + NodeCounts node_counts; + + // The cord method responsible for sampling the cord. + MethodIdentifier method = MethodIdentifier::kUnknown; + + // The cord method responsible for sampling the parent cord if applicable. + MethodIdentifier parent_method = MethodIdentifier::kUnknown; + + // Update tracker tracking invocation count per cord method. + CordzUpdateTracker update_tracker; +}; + +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_INTERNAL_CORDZ_STATISTICS_H_ diff --git a/absl/strings/internal/cordz_update_scope.h b/absl/strings/internal/cordz_update_scope.h new file mode 100644 index 00000000..57ba75de --- /dev/null +++ b/absl/strings/internal/cordz_update_scope.h @@ -0,0 +1,71 @@ +// Copyright 2021 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_STRINGS_INTERNAL_CORDZ_UPDATE_SCOPE_H_ +#define ABSL_STRINGS_INTERNAL_CORDZ_UPDATE_SCOPE_H_ + +#include "absl/base/config.h" +#include "absl/base/optimization.h" +#include "absl/base/thread_annotations.h" +#include "absl/strings/internal/cord_internal.h" +#include "absl/strings/internal/cordz_info.h" +#include "absl/strings/internal/cordz_update_tracker.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { + +// CordzUpdateScope scopes an update to the provided CordzInfo. +// The class invokes `info->Lock(method)` and `info->Unlock()` to guard +// cordrep updates. This class does nothing if `info` is null. +// See also the 'Lock`, `Unlock` and `SetCordRep` methods in `CordzInfo`. +class ABSL_SCOPED_LOCKABLE CordzUpdateScope { + public: + CordzUpdateScope(CordzInfo* info, CordzUpdateTracker::MethodIdentifier method) + ABSL_EXCLUSIVE_LOCK_FUNCTION(info) + : info_(info) { + if (ABSL_PREDICT_FALSE(info_)) { + info->Lock(method); + } + } + + // CordzUpdateScope can not be copied or assigned to. + CordzUpdateScope(CordzUpdateScope&& rhs) = delete; + CordzUpdateScope(const CordzUpdateScope&) = delete; + CordzUpdateScope& operator=(CordzUpdateScope&& rhs) = delete; + CordzUpdateScope& operator=(const CordzUpdateScope&) = delete; + + ~CordzUpdateScope() ABSL_UNLOCK_FUNCTION() { + if (ABSL_PREDICT_FALSE(info_)) { + info_->Unlock(); + } + } + + void SetCordRep(CordRep* rep) const { + if (ABSL_PREDICT_FALSE(info_)) { + info_->SetCordRep(rep); + } + } + + CordzInfo* info() const { return info_; } + + private: + CordzInfo* info_; +}; + +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_INTERNAL_CORDZ_UPDATE_SCOPE_H_ diff --git a/absl/strings/internal/cordz_update_scope_test.cc b/absl/strings/internal/cordz_update_scope_test.cc new file mode 100644 index 00000000..3d08c622 --- /dev/null +++ b/absl/strings/internal/cordz_update_scope_test.cc @@ -0,0 +1,49 @@ +// Copyright 2021 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/cordz_update_scope.h" + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/config.h" +#include "absl/strings/cordz_test_helpers.h" +#include "absl/strings/internal/cord_rep_flat.h" +#include "absl/strings/internal/cordz_info.h" +#include "absl/strings/internal/cordz_update_tracker.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { + +namespace { + +// Used test values +auto constexpr kTrackCordMethod = CordzUpdateTracker::kConstructorString; + +TEST(CordzUpdateScopeTest, ScopeNullptr) { + CordzUpdateScope scope(nullptr, kTrackCordMethod); +} + +TEST(CordzUpdateScopeTest, ScopeSampledCord) { + TestCordData cord; + CordzInfo::TrackCord(cord.data, kTrackCordMethod); + CordzUpdateScope scope(cord.data.cordz_info(), kTrackCordMethod); + cord.data.cordz_info()->SetCordRep(nullptr); +} + +} // namespace +ABSL_NAMESPACE_END +} // namespace cord_internal + +} // namespace absl diff --git a/absl/strings/internal/cordz_update_tracker.h b/absl/strings/internal/cordz_update_tracker.h new file mode 100644 index 00000000..c5170662 --- /dev/null +++ b/absl/strings/internal/cordz_update_tracker.h @@ -0,0 +1,123 @@ +// Copyright 2021 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#ifndef ABSL_STRINGS_INTERNAL_CORDZ_UPDATE_TRACKER_H_ +#define ABSL_STRINGS_INTERNAL_CORDZ_UPDATE_TRACKER_H_ + +#include <atomic> +#include <cstdint> + +#include "absl/base/config.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { + +// CordzUpdateTracker tracks counters for Cord update methods. +// +// The purpose of CordzUpdateTracker is to track the number of calls to methods +// updating Cord data for sampled cords. The class internally uses 'lossy' +// atomic operations: Cord is thread-compatible, so there is no need to +// synchronize updates. However, Cordz collection threads may call 'Value()' at +// any point, so the class needs to provide thread safe access. +// +// This class is thread-safe. But as per above comments, all non-const methods +// should be used single-threaded only: updates are thread-safe but lossy. +class CordzUpdateTracker { + public: + // Tracked update methods. + enum MethodIdentifier { + kUnknown, + kAppendCord, + kAppendCordBuffer, + kAppendExternalMemory, + kAppendString, + kAssignCord, + kAssignString, + kClear, + kConstructorCord, + kConstructorString, + kCordReader, + kFlatten, + kGetAppendBuffer, + kGetAppendRegion, + kMakeCordFromExternal, + kMoveAppendCord, + kMoveAssignCord, + kMovePrependCord, + kPrependCord, + kPrependCordBuffer, + kPrependString, + kRemovePrefix, + kRemoveSuffix, + kSetExpectedChecksum, + kSubCord, + + // kNumMethods defines the number of entries: must be the last entry. + kNumMethods, + }; + + // Constructs a new instance. All counters are zero-initialized. + constexpr CordzUpdateTracker() noexcept : values_{} {} + + // Copy constructs a new instance. + CordzUpdateTracker(const CordzUpdateTracker& rhs) noexcept { *this = rhs; } + + // Assigns the provided value to this instance. + CordzUpdateTracker& operator=(const CordzUpdateTracker& rhs) noexcept { + for (int i = 0; i < kNumMethods; ++i) { + values_[i].store(rhs.values_[i].load(std::memory_order_relaxed), + std::memory_order_relaxed); + } + return *this; + } + + // Returns the value for the specified method. + int64_t Value(MethodIdentifier method) const { + return values_[method].load(std::memory_order_relaxed); + } + + // Increases the value for the specified method by `n` + void LossyAdd(MethodIdentifier method, int64_t n = 1) { + auto& value = values_[method]; + value.store(value.load(std::memory_order_relaxed) + n, + std::memory_order_relaxed); + } + + // Adds all the values from `src` to this instance + void LossyAdd(const CordzUpdateTracker& src) { + for (int i = 0; i < kNumMethods; ++i) { + MethodIdentifier method = static_cast<MethodIdentifier>(i); + if (int64_t value = src.Value(method)) { + LossyAdd(method, value); + } + } + } + + private: + // Until C++20 std::atomic is not constexpr default-constructible, so we need + // a wrapper for this class to be constexpr constructible. + class Counter : public std::atomic<int64_t> { + public: + constexpr Counter() noexcept : std::atomic<int64_t>(0) {} + }; + + Counter values_[kNumMethods]; +}; + +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl + +#endif // ABSL_STRINGS_INTERNAL_CORDZ_UPDATE_TRACKER_H_ diff --git a/absl/strings/internal/cordz_update_tracker_test.cc b/absl/strings/internal/cordz_update_tracker_test.cc new file mode 100644 index 00000000..9b1f7986 --- /dev/null +++ b/absl/strings/internal/cordz_update_tracker_test.cc @@ -0,0 +1,147 @@ +// Copyright 2021 The Abseil Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "absl/strings/internal/cordz_update_tracker.h" + +#include <array> +#include <thread> // NOLINT + +#include "gmock/gmock.h" +#include "gtest/gtest.h" +#include "absl/base/attributes.h" +#include "absl/base/config.h" +#include "absl/synchronization/notification.h" + +namespace absl { +ABSL_NAMESPACE_BEGIN +namespace cord_internal { +namespace { + +using ::testing::AnyOf; +using ::testing::Eq; + +using Method = CordzUpdateTracker::MethodIdentifier; +using Methods = std::array<Method, Method::kNumMethods>; + +// Returns an array of all methods defined in `MethodIdentifier` +Methods AllMethods() { + return Methods{Method::kUnknown, + Method::kAppendCord, + Method::kAppendCordBuffer, + Method::kAppendExternalMemory, + Method::kAppendString, + Method::kAssignCord, + Method::kAssignString, + Method::kClear, + Method::kConstructorCord, + Method::kConstructorString, + Method::kCordReader, + Method::kFlatten, + Method::kGetAppendBuffer, + Method::kGetAppendRegion, + Method::kMakeCordFromExternal, + Method::kMoveAppendCord, + Method::kMoveAssignCord, + Method::kMovePrependCord, + Method::kPrependCord, + Method::kPrependCordBuffer, + Method::kPrependString, + Method::kRemovePrefix, + Method::kRemoveSuffix, + Method::kSetExpectedChecksum, + Method::kSubCord}; +} + +TEST(CordzUpdateTracker, IsConstExprAndInitializesToZero) { + constexpr CordzUpdateTracker tracker; + for (Method method : AllMethods()) { + ASSERT_THAT(tracker.Value(method), Eq(0)); + } +} + +TEST(CordzUpdateTracker, LossyAdd) { + int64_t n = 1; + CordzUpdateTracker tracker; + for (Method method : AllMethods()) { + tracker.LossyAdd(method, n); + EXPECT_THAT(tracker.Value(method), Eq(n)); + n += 2; + } +} + +TEST(CordzUpdateTracker, CopyConstructor) { + int64_t n = 1; + CordzUpdateTracker src; + for (Method method : AllMethods()) { + src.LossyAdd(method, n); + n += 2; + } + + n = 1; + CordzUpdateTracker tracker(src); + for (Method method : AllMethods()) { + EXPECT_THAT(tracker.Value(method), Eq(n)); + n += 2; + } +} + +TEST(CordzUpdateTracker, OperatorAssign) { + int64_t n = 1; + CordzUpdateTracker src; + CordzUpdateTracker tracker; + for (Method method : AllMethods()) { + src.LossyAdd(method, n); + n += 2; + } + + n = 1; + tracker = src; + for (Method method : AllMethods()) { + EXPECT_THAT(tracker.Value(method), Eq(n)); + n += 2; + } +} + +TEST(CordzUpdateTracker, ThreadSanitizedValueCheck) { + absl::Notification done; + CordzUpdateTracker tracker; + + std::thread reader([&done, &tracker] { + while (!done.HasBeenNotified()) { + int n = 1; + for (Method method : AllMethods()) { + EXPECT_THAT(tracker.Value(method), AnyOf(Eq(n), Eq(0))); + n += 2; + } + } + int n = 1; + for (Method method : AllMethods()) { + EXPECT_THAT(tracker.Value(method), Eq(n)); + n += 2; + } + }); + + int64_t n = 1; + for (Method method : AllMethods()) { + tracker.LossyAdd(method, n); + n += 2; + } + done.Notify(); + reader.join(); +} + +} // namespace +} // namespace cord_internal +ABSL_NAMESPACE_END +} // namespace absl diff --git a/absl/strings/internal/escaping.cc b/absl/strings/internal/escaping.cc index c5271286..cfea0961 100644 --- a/absl/strings/internal/escaping.cc +++ b/absl/strings/internal/escaping.cc @@ -21,7 +21,7 @@ namespace absl { ABSL_NAMESPACE_BEGIN namespace strings_internal { -const char kBase64Chars[] = +ABSL_CONST_INIT const char kBase64Chars[] = "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; size_t CalculateBase64EscapedLenInternal(size_t input_len, bool do_padding) { @@ -102,8 +102,8 @@ size_t Base64EscapeInternal(const unsigned char* src, size_t szsrc, char* dest, } } // To save time, we didn't update szdest or szsrc in the loop. So do it now. - szdest = limit_dest - cur_dest; - szsrc = limit_src - cur_src; + szdest = static_cast<size_t>(limit_dest - cur_dest); + szsrc = static_cast<size_t>(limit_src - cur_src); /* now deal with the tail (<=3 bytes) */ switch (szsrc) { @@ -154,7 +154,8 @@ size_t Base64EscapeInternal(const unsigned char* src, size_t szsrc, char* dest, // the loop because the loop above always reads 4 bytes, and the fourth // byte is past the end of the input. if (szdest < 4) return 0; - uint32_t in = (cur_src[0] << 16) + absl::big_endian::Load16(cur_src + 1); + uint32_t in = + (uint32_t{cur_src[0]} << 16) + absl::big_endian::Load16(cur_src + 1); cur_dest[0] = base64[in >> 18]; in &= 0x3FFFF; cur_dest[1] = base64[in >> 12]; @@ -172,7 +173,7 @@ size_t Base64EscapeInternal(const unsigned char* src, size_t szsrc, char* dest, ABSL_RAW_LOG(FATAL, "Logic problem? szsrc = %zu", szsrc); break; } - return (cur_dest - dest); + return static_cast<size_t>(cur_dest - dest); } } // namespace strings_internal diff --git a/absl/strings/internal/ostringstream.cc b/absl/strings/internal/ostringstream.cc index 05324c78..dc6cfe16 100644 --- a/absl/strings/internal/ostringstream.cc +++ b/absl/strings/internal/ostringstream.cc @@ -27,7 +27,7 @@ OStringStream::Buf::int_type OStringStream::overflow(int c) { std::streamsize OStringStream::xsputn(const char* s, std::streamsize n) { assert(s_); - s_->append(s, n); + s_->append(s, static_cast<size_t>(n)); return n; } diff --git a/absl/strings/internal/resize_uninitialized.h b/absl/strings/internal/resize_uninitialized.h index e42628e3..49859dcc 100644 --- a/absl/strings/internal/resize_uninitialized.h +++ b/absl/strings/internal/resize_uninitialized.h @@ -17,6 +17,7 @@ #ifndef ABSL_STRINGS_INTERNAL_RESIZE_UNINITIALIZED_H_ #define ABSL_STRINGS_INTERNAL_RESIZE_UNINITIALIZED_H_ +#include <algorithm> #include <string> #include <type_traits> #include <utility> @@ -28,8 +29,9 @@ namespace absl { ABSL_NAMESPACE_BEGIN namespace strings_internal { -// Is a subclass of true_type or false_type, depending on whether or not -// T has a __resize_default_init member. +// In this type trait, we look for a __resize_default_init member function, and +// we use it if available, otherwise, we use resize. We provide HasMember to +// indicate whether __resize_default_init is present. template <typename string_type, typename = void> struct ResizeUninitializedTraits { using HasMember = std::false_type; @@ -66,6 +68,50 @@ inline void STLStringResizeUninitialized(string_type* s, size_t new_size) { ResizeUninitializedTraits<string_type>::Resize(s, new_size); } +// Used to ensure exponential growth so that the amortized complexity of +// increasing the string size by a small amount is O(1), in contrast to +// O(str->size()) in the case of precise growth. +template <typename string_type> +void STLStringReserveAmortized(string_type* s, size_t new_size) { + const size_t cap = s->capacity(); + if (new_size > cap) { + // Make sure to always grow by at least a factor of 2x. + s->reserve((std::max)(new_size, 2 * cap)); + } +} + +// In this type trait, we look for an __append_default_init member function, and +// we use it if available, otherwise, we use append. +template <typename string_type, typename = void> +struct AppendUninitializedTraits { + static void Append(string_type* s, size_t n) { + s->append(n, typename string_type::value_type()); + } +}; + +template <typename string_type> +struct AppendUninitializedTraits< + string_type, absl::void_t<decltype(std::declval<string_type&>() + .__append_default_init(237))> > { + static void Append(string_type* s, size_t n) { + s->__append_default_init(n); + } +}; + +// Like STLStringResizeUninitialized(str, new_size), except guaranteed to use +// exponential growth so that the amortized complexity of increasing the string +// size by a small amount is O(1), in contrast to O(str->size()) in the case of +// precise growth. +template <typename string_type> +void STLStringResizeUninitializedAmortized(string_type* s, size_t new_size) { + const size_t size = s->size(); + if (new_size > size) { + AppendUninitializedTraits<string_type>::Append(s, new_size - size); + } else { + s->erase(new_size); + } +} + } // namespace strings_internal ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/strings/internal/resize_uninitialized_test.cc b/absl/strings/internal/resize_uninitialized_test.cc index 0f8b3c2a..ad1b9c58 100644 --- a/absl/strings/internal/resize_uninitialized_test.cc +++ b/absl/strings/internal/resize_uninitialized_test.cc @@ -19,64 +19,115 @@ namespace { int resize_call_count = 0; +int append_call_count = 0; // A mock string class whose only purpose is to track how many times its -// resize() method has been called. +// resize()/append() methods have been called. struct resizable_string { + using value_type = char; size_t size() const { return 0; } + size_t capacity() const { return 0; } char& operator[](size_t) { static char c = '\0'; return c; } void resize(size_t) { resize_call_count += 1; } + void append(size_t, value_type) { append_call_count += 1; } + void reserve(size_t) {} + resizable_string& erase(size_t = 0, size_t = 0) { return *this; } }; int resize_default_init_call_count = 0; +int append_default_init_call_count = 0; // A mock string class whose only purpose is to track how many times its -// resize() and __resize_default_init() methods have been called. -struct resize_default_init_string { +// resize()/__resize_default_init()/append()/__append_default_init() methods +// have been called. +struct default_init_string { size_t size() const { return 0; } + size_t capacity() const { return 0; } char& operator[](size_t) { static char c = '\0'; return c; } void resize(size_t) { resize_call_count += 1; } void __resize_default_init(size_t) { resize_default_init_call_count += 1; } + void __append_default_init(size_t) { append_default_init_call_count += 1; } + void reserve(size_t) {} + default_init_string& erase(size_t = 0, size_t = 0) { return *this; } }; TEST(ResizeUninit, WithAndWithout) { resize_call_count = 0; + append_call_count = 0; resize_default_init_call_count = 0; + append_default_init_call_count = 0; { resizable_string rs; EXPECT_EQ(resize_call_count, 0); + EXPECT_EQ(append_call_count, 0); EXPECT_EQ(resize_default_init_call_count, 0); + EXPECT_EQ(append_default_init_call_count, 0); EXPECT_FALSE( absl::strings_internal::STLStringSupportsNontrashingResize(&rs)); EXPECT_EQ(resize_call_count, 0); + EXPECT_EQ(append_call_count, 0); EXPECT_EQ(resize_default_init_call_count, 0); + EXPECT_EQ(append_default_init_call_count, 0); absl::strings_internal::STLStringResizeUninitialized(&rs, 237); EXPECT_EQ(resize_call_count, 1); + EXPECT_EQ(append_call_count, 0); EXPECT_EQ(resize_default_init_call_count, 0); + EXPECT_EQ(append_default_init_call_count, 0); + absl::strings_internal::STLStringResizeUninitializedAmortized(&rs, 1000); + EXPECT_EQ(resize_call_count, 1); + EXPECT_EQ(append_call_count, 1); + EXPECT_EQ(resize_default_init_call_count, 0); + EXPECT_EQ(append_default_init_call_count, 0); } resize_call_count = 0; + append_call_count = 0; resize_default_init_call_count = 0; + append_default_init_call_count = 0; { - resize_default_init_string rus; + default_init_string rus; EXPECT_EQ(resize_call_count, 0); + EXPECT_EQ(append_call_count, 0); EXPECT_EQ(resize_default_init_call_count, 0); + EXPECT_EQ(append_default_init_call_count, 0); EXPECT_TRUE( absl::strings_internal::STLStringSupportsNontrashingResize(&rus)); EXPECT_EQ(resize_call_count, 0); + EXPECT_EQ(append_call_count, 0); EXPECT_EQ(resize_default_init_call_count, 0); + EXPECT_EQ(append_default_init_call_count, 0); absl::strings_internal::STLStringResizeUninitialized(&rus, 237); EXPECT_EQ(resize_call_count, 0); + EXPECT_EQ(append_call_count, 0); + EXPECT_EQ(resize_default_init_call_count, 1); + EXPECT_EQ(append_default_init_call_count, 0); + absl::strings_internal::STLStringResizeUninitializedAmortized(&rus, 1000); + EXPECT_EQ(resize_call_count, 0); + EXPECT_EQ(append_call_count, 0); EXPECT_EQ(resize_default_init_call_count, 1); + EXPECT_EQ(append_default_init_call_count, 1); + } +} + +TEST(ResizeUninit, Amortized) { + std::string str; + size_t prev_cap = str.capacity(); + int cap_increase_count = 0; + for (int i = 0; i < 1000; ++i) { + absl::strings_internal::STLStringResizeUninitializedAmortized(&str, i); + size_t new_cap = str.capacity(); + if (new_cap > prev_cap) ++cap_increase_count; + prev_cap = new_cap; } + EXPECT_LT(cap_increase_count, 50); } } // namespace diff --git a/absl/strings/internal/str_format/arg.cc b/absl/strings/internal/str_format/arg.cc index e28a29b1..02aeeebe 100644 --- a/absl/strings/internal/str_format/arg.cc +++ b/absl/strings/internal/str_format/arg.cc @@ -320,7 +320,7 @@ bool ConvertIntArg(T v, const FormatConversionSpecImpl conv, return ConvertFloatImpl(static_cast<double>(v), conv, sink); default: - ABSL_INTERNAL_ASSUME(false); + ABSL_ASSUME(false); } if (conv.is_basic()) { diff --git a/absl/strings/internal/str_format/arg.h b/absl/strings/internal/str_format/arg.h index 7040c866..b9dda909 100644 --- a/absl/strings/internal/str_format/arg.h +++ b/absl/strings/internal/str_format/arg.h @@ -122,6 +122,14 @@ StringConvertResult FormatConvertImpl(const std::string& v, StringConvertResult FormatConvertImpl(string_view v, FormatConversionSpecImpl conv, FormatSinkImpl* sink); +#if defined(ABSL_HAVE_STD_STRING_VIEW) && !defined(ABSL_USES_STD_STRING_VIEW) +inline StringConvertResult FormatConvertImpl(std::string_view v, + FormatConversionSpecImpl conv, + FormatSinkImpl* sink) { + return FormatConvertImpl(absl::string_view(v.data(), v.size()), conv, sink); +} +#endif // ABSL_HAVE_STD_STRING_VIEW && !ABSL_USES_STD_STRING_VIEW + ArgConvertResult<FormatConversionCharSetUnion( FormatConversionCharSetInternal::s, FormatConversionCharSetInternal::p)> FormatConvertImpl(const char* v, const FormatConversionSpecImpl conv, @@ -136,7 +144,7 @@ StringConvertResult FormatConvertImpl(const AbslCord& value, size_t space_remaining = 0; int width = conv.width(); - if (width >= 0) space_remaining = width; + if (width >= 0) space_remaining = static_cast<size_t>(width); size_t to_write = value.size(); diff --git a/absl/strings/internal/str_format/bind.cc b/absl/strings/internal/str_format/bind.cc index 4e68b90b..c988ba8f 100644 --- a/absl/strings/internal/str_format/bind.cc +++ b/absl/strings/internal/str_format/bind.cc @@ -58,7 +58,7 @@ inline bool ArgContext::Bind(const UnboundConversion* unbound, if (static_cast<size_t>(arg_position - 1) >= pack_.size()) return false; arg = &pack_[arg_position - 1]; // 1-based - if (!unbound->flags.basic) { + if (unbound->flags != Flags::kBasic) { int width = unbound->width.value(); bool force_left = false; if (unbound->width.is_from_arg()) { @@ -84,9 +84,8 @@ inline bool ArgContext::Bind(const UnboundConversion* unbound, FormatConversionSpecImplFriend::SetPrecision(precision, bound); if (force_left) { - Flags flags = unbound->flags; - flags.left = true; - FormatConversionSpecImplFriend::SetFlags(flags, bound); + FormatConversionSpecImplFriend::SetFlags(unbound->flags | Flags::kLeft, + bound); } else { FormatConversionSpecImplFriend::SetFlags(unbound->flags, bound); } diff --git a/absl/strings/internal/str_format/bind.h b/absl/strings/internal/str_format/bind.h index 267cc0ef..80f29654 100644 --- a/absl/strings/internal/str_format/bind.h +++ b/absl/strings/internal/str_format/bind.h @@ -25,6 +25,7 @@ #include "absl/strings/internal/str_format/checker.h" #include "absl/strings/internal/str_format/parser.h" #include "absl/types/span.h" +#include "absl/utility/utility.h" namespace absl { ABSL_NAMESPACE_BEGIN @@ -87,6 +88,36 @@ class FormatSpecTemplate : public MakeDependent<UntypedFormatSpec, Args...>::type { using Base = typename MakeDependent<UntypedFormatSpec, Args...>::type; + template <bool res> + struct ErrorMaker { + constexpr bool operator()(int) const { return res; } + }; + + template <int i, int j> + static constexpr bool CheckArity(ErrorMaker<true> SpecifierCount = {}, + ErrorMaker<i == j> ParametersPassed = {}) { + static_assert(SpecifierCount(i) == ParametersPassed(j), + "Number of arguments passed must match the number of " + "conversion specifiers."); + return true; + } + + template <FormatConversionCharSet specified, FormatConversionCharSet passed, + int arg> + static constexpr bool CheckMatch( + ErrorMaker<Contains(specified, passed)> MismatchedArgumentNumber = {}) { + static_assert(MismatchedArgumentNumber(arg), + "Passed argument must match specified format."); + return true; + } + + template <FormatConversionCharSet... C, size_t... I> + static bool CheckMatches(absl::index_sequence<I...>) { + bool res[] = {true, CheckMatch<Args, C, I + 1>()...}; + (void)res; + return true; + } + public: #ifdef ABSL_INTERNAL_ENABLE_FORMAT_CHECKER @@ -100,7 +131,7 @@ class FormatSpecTemplate // We use the 'unavailable' attribute to give a better compiler error than // just 'method is deleted'. // To avoid checking the format twice, we just check that the format is - // constexpr. If is it valid, then the overload below will kick in. + // constexpr. If it is valid, then the overload below will kick in. // We add the template here to make this overload have lower priority. template <typename = void> FormatSpecTemplate(const char* s) // NOLINT @@ -112,7 +143,8 @@ class FormatSpecTemplate template <typename T = void> FormatSpecTemplate(string_view s) // NOLINT __attribute__((enable_if(str_format_internal::EnsureConstexpr(s), - "constexpr trap"))) { + "constexpr trap"))) + : Base("to avoid noise in the compiler error") { static_assert(sizeof(T*) == 0, "Format specified does not match the arguments passed."); } @@ -133,13 +165,12 @@ class FormatSpecTemplate #endif // ABSL_INTERNAL_ENABLE_FORMAT_CHECKER - template < - FormatConversionCharSet... C, - typename = typename std::enable_if<sizeof...(C) == sizeof...(Args)>::type, - typename = typename std::enable_if<AllOf(Contains(Args, - C)...)>::type> + template <FormatConversionCharSet... C> FormatSpecTemplate(const ExtendedParsedFormat<C...>& pc) // NOLINT - : Base(&pc) {} + : Base(&pc) { + CheckArity<sizeof...(C), sizeof...(Args)>(); + CheckMatches<C...>(absl::make_index_sequence<sizeof...(C)>{}); + } }; class Streamable { diff --git a/absl/strings/internal/str_format/checker.h b/absl/strings/internal/str_format/checker.h index 2a2601ec..4fd19d13 100644 --- a/absl/strings/internal/str_format/checker.h +++ b/absl/strings/internal/str_format/checker.h @@ -22,9 +22,14 @@ // Compile time check support for entry points. #ifndef ABSL_INTERNAL_ENABLE_FORMAT_CHECKER -#if ABSL_HAVE_ATTRIBUTE(enable_if) && !defined(__native_client__) +// We disable format checker under vscode intellisense compilation. +// See https://github.com/microsoft/vscode-cpptools/issues/3683 for +// more details. +#if ABSL_HAVE_ATTRIBUTE(enable_if) && !defined(__native_client__) && \ + !defined(__INTELLISENSE__) #define ABSL_INTERNAL_ENABLE_FORMAT_CHECKER 1 -#endif // ABSL_HAVE_ATTRIBUTE(enable_if) && !defined(__native_client__) +#endif // ABSL_HAVE_ATTRIBUTE(enable_if) && !defined(__native_client__) && + // !defined(__INTELLISENSE__) #endif // ABSL_INTERNAL_ENABLE_FORMAT_CHECKER namespace absl { diff --git a/absl/strings/internal/str_format/convert_test.cc b/absl/strings/internal/str_format/convert_test.cc index 926283cf..300612b7 100644 --- a/absl/strings/internal/str_format/convert_test.cc +++ b/absl/strings/internal/str_format/convert_test.cc @@ -24,6 +24,7 @@ #include "gmock/gmock.h" #include "gtest/gtest.h" +#include "absl/base/attributes.h" #include "absl/base/internal/raw_logging.h" #include "absl/strings/internal/str_format/bind.h" #include "absl/strings/match.h" @@ -124,6 +125,7 @@ void StrAppendV(std::string *dst, const char *format, va_list ap) { delete[] buf; } +void StrAppend(std::string *, const char *, ...) ABSL_PRINTF_ATTRIBUTE(2, 3); void StrAppend(std::string *out, const char *format, ...) { va_list ap; va_start(ap, format); @@ -131,6 +133,7 @@ void StrAppend(std::string *out, const char *format, ...) { va_end(ap); } +std::string StrPrint(const char *, ...) ABSL_PRINTF_ATTRIBUTE(1, 2); std::string StrPrint(const char *format, ...) { va_list ap; va_start(ap, format); @@ -229,6 +232,9 @@ TEST_F(FormatConvertTest, BasicString) { TestStringConvert(static_cast<const char*>("hello")); TestStringConvert(std::string("hello")); TestStringConvert(string_view("hello")); +#if defined(ABSL_HAVE_STD_STRING_VIEW) + TestStringConvert(std::string_view("hello")); +#endif // ABSL_HAVE_STD_STRING_VIEW } TEST_F(FormatConvertTest, NullString) { @@ -452,25 +458,36 @@ TYPED_TEST_P(TypedFormatConvertTest, AllIntsWithFlags) { } TYPED_TEST_P(TypedFormatConvertTest, Char) { + // Pass a bunch of values of type TypeParam to both FormatPack and libc's + // vsnprintf("%c", ...) (wrapped in StrPrint) to make sure we get the same + // value. typedef TypeParam T; using remove_volatile_t = typename std::remove_volatile<T>::type; - static const T kMin = std::numeric_limits<remove_volatile_t>::min(); - static const T kMax = std::numeric_limits<remove_volatile_t>::max(); - T kVals[] = { - remove_volatile_t(1), remove_volatile_t(2), remove_volatile_t(10), - remove_volatile_t(-1), remove_volatile_t(-2), remove_volatile_t(-10), - remove_volatile_t(0), - kMin + remove_volatile_t(1), kMin, - kMax - remove_volatile_t(1), kMax + std::vector<remove_volatile_t> vals = { + remove_volatile_t(1), remove_volatile_t(2), remove_volatile_t(10), // + remove_volatile_t(-1), remove_volatile_t(-2), remove_volatile_t(-10), // + remove_volatile_t(0), }; - for (const T &c : kVals) { + + // We'd like to test values near std::numeric_limits::min() and + // std::numeric_limits::max(), too, but vsnprintf("%c", ...) can't handle + // anything larger than an int. Add in the most extreme values we can without + // exceeding that range. + static const T kMin = + static_cast<remove_volatile_t>(std::numeric_limits<int>::min()); + static const T kMax = + static_cast<remove_volatile_t>(std::numeric_limits<int>::max()); + vals.insert(vals.end(), {kMin + 1, kMin, kMax - 1, kMax}); + + for (const T c : vals) { const FormatArgImpl args[] = {FormatArgImpl(c)}; UntypedFormatSpecImpl format("%c"); - EXPECT_EQ(StrPrint("%c", c), FormatPack(format, absl::MakeSpan(args))); + EXPECT_EQ(StrPrint("%c", static_cast<int>(c)), + FormatPack(format, absl::MakeSpan(args))); } } -REGISTER_TYPED_TEST_CASE_P(TypedFormatConvertTest, AllIntsWithFlags, Char); +REGISTER_TYPED_TEST_SUITE_P(TypedFormatConvertTest, AllIntsWithFlags, Char); typedef ::testing::Types< int, unsigned, volatile int, @@ -479,8 +496,8 @@ typedef ::testing::Types< long long, unsigned long long, signed char, unsigned char, char> AllIntTypes; -INSTANTIATE_TYPED_TEST_CASE_P(TypedFormatConvertTestWithAllIntTypes, - TypedFormatConvertTest, AllIntTypes); +INSTANTIATE_TYPED_TEST_SUITE_P(TypedFormatConvertTestWithAllIntTypes, + TypedFormatConvertTest, AllIntTypes); TEST_F(FormatConvertTest, VectorBool) { // Make sure vector<bool>'s values behave as bools. std::vector<bool> v = {true, false}; diff --git a/absl/strings/internal/str_format/extension.cc b/absl/strings/internal/str_format/extension.cc index bb0d96cf..f93153d5 100644 --- a/absl/strings/internal/str_format/extension.cc +++ b/absl/strings/internal/str_format/extension.cc @@ -23,16 +23,18 @@ namespace absl { ABSL_NAMESPACE_BEGIN namespace str_format_internal { -std::string Flags::ToString() const { +std::string FlagsToString(Flags v) { std::string s; - s.append(left ? "-" : ""); - s.append(show_pos ? "+" : ""); - s.append(sign_col ? " " : ""); - s.append(alt ? "#" : ""); - s.append(zero ? "0" : ""); + s.append(FlagsContains(v, Flags::kLeft) ? "-" : ""); + s.append(FlagsContains(v, Flags::kShowPos) ? "+" : ""); + s.append(FlagsContains(v, Flags::kSignCol) ? " " : ""); + s.append(FlagsContains(v, Flags::kAlt) ? "#" : ""); + s.append(FlagsContains(v, Flags::kZero) ? "0" : ""); return s; } +#ifdef ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL + #define ABSL_INTERNAL_X_VAL(id) \ constexpr absl::FormatConversionChar FormatConversionCharInternal::id; ABSL_INTERNAL_CONVERSION_CHARS_EXPAND_(ABSL_INTERNAL_X_VAL, ) @@ -45,17 +47,14 @@ constexpr absl::FormatConversionChar FormatConversionCharInternal::kNone; ABSL_INTERNAL_CONVERSION_CHARS_EXPAND_(ABSL_INTERNAL_CHAR_SET_CASE, ) #undef ABSL_INTERNAL_CHAR_SET_CASE -// NOLINTNEXTLINE(readability-redundant-declaration) constexpr FormatConversionCharSet FormatConversionCharSetInternal::kStar; -// NOLINTNEXTLINE(readability-redundant-declaration) constexpr FormatConversionCharSet FormatConversionCharSetInternal::kIntegral; -// NOLINTNEXTLINE(readability-redundant-declaration) constexpr FormatConversionCharSet FormatConversionCharSetInternal::kFloating; -// NOLINTNEXTLINE(readability-redundant-declaration) constexpr FormatConversionCharSet FormatConversionCharSetInternal::kNumeric; -// NOLINTNEXTLINE(readability-redundant-declaration) constexpr FormatConversionCharSet FormatConversionCharSetInternal::kPointer; +#endif // ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL + bool FormatSinkImpl::PutPaddedString(string_view value, int width, int precision, bool left) { size_t space_remaining = 0; diff --git a/absl/strings/internal/str_format/extension.h b/absl/strings/internal/str_format/extension.h index a9b9e137..55e8ac88 100644 --- a/absl/strings/internal/str_format/extension.h +++ b/absl/strings/internal/str_format/extension.h @@ -19,6 +19,7 @@ #include <limits.h> #include <cstddef> +#include <cstdint> #include <cstring> #include <ostream> @@ -70,7 +71,7 @@ class FormatSinkImpl { ~FormatSinkImpl() { Flush(); } void Flush() { - raw_.Write(string_view(buf_, pos_ - buf_)); + raw_.Write(string_view(buf_, static_cast<size_t>(pos_ - buf_))); pos_ = buf_; } @@ -120,7 +121,9 @@ class FormatSinkImpl { } private: - size_t Avail() const { return buf_ + sizeof(buf_) - pos_; } + size_t Avail() const { + return static_cast<size_t>(buf_ + sizeof(buf_) - pos_); + } FormatRawSinkImpl raw_; size_t size_ = 0; @@ -128,19 +131,33 @@ class FormatSinkImpl { char buf_[1024]; }; -struct Flags { - bool basic : 1; // fastest conversion: no flags, width, or precision - bool left : 1; // "-" - bool show_pos : 1; // "+" - bool sign_col : 1; // " " - bool alt : 1; // "#" - bool zero : 1; // "0" - std::string ToString() const; - friend std::ostream& operator<<(std::ostream& os, const Flags& v) { - return os << v.ToString(); - } +enum class Flags : uint8_t { + kBasic = 0, + kLeft = 1 << 0, + kShowPos = 1 << 1, + kSignCol = 1 << 2, + kAlt = 1 << 3, + kZero = 1 << 4, + // This is not a real flag. It just exists to turn off kBasic when no other + // flags are set. This is for when width/precision are specified. + kNonBasic = 1 << 5, }; +constexpr Flags operator|(Flags a, Flags b) { + return static_cast<Flags>(static_cast<uint8_t>(a) | static_cast<uint8_t>(b)); +} + +constexpr bool FlagsContains(Flags haystack, Flags needle) { + return (static_cast<uint8_t>(haystack) & static_cast<uint8_t>(needle)) == + static_cast<uint8_t>(needle); +} + +std::string FlagsToString(Flags v); + +inline std::ostream& operator<<(std::ostream& os, Flags v) { + return os << FlagsToString(v); +} + // clang-format off #define ABSL_INTERNAL_CONVERSION_CHARS_EXPAND_(X_VAL, X_SEP) \ /* text */ \ @@ -257,12 +274,16 @@ struct FormatConversionSpecImplFriend; class FormatConversionSpecImpl { public: // Width and precison are not specified, no flags are set. - bool is_basic() const { return flags_.basic; } - bool has_left_flag() const { return flags_.left; } - bool has_show_pos_flag() const { return flags_.show_pos; } - bool has_sign_col_flag() const { return flags_.sign_col; } - bool has_alt_flag() const { return flags_.alt; } - bool has_zero_flag() const { return flags_.zero; } + bool is_basic() const { return flags_ == Flags::kBasic; } + bool has_left_flag() const { return FlagsContains(flags_, Flags::kLeft); } + bool has_show_pos_flag() const { + return FlagsContains(flags_, Flags::kShowPos); + } + bool has_sign_col_flag() const { + return FlagsContains(flags_, Flags::kSignCol); + } + bool has_alt_flag() const { return FlagsContains(flags_, Flags::kAlt); } + bool has_zero_flag() const { return FlagsContains(flags_, Flags::kZero); } FormatConversionChar conversion_char() const { // Keep this field first in the struct . It generates better code when @@ -306,7 +327,7 @@ struct FormatConversionSpecImplFriend final { conv->precision_ = p; } static std::string FlagsToString(const FormatConversionSpecImpl& spec) { - return spec.flags_.ToString(); + return str_format_internal::FlagsToString(spec.flags_); } }; diff --git a/absl/strings/internal/str_format/output.h b/absl/strings/internal/str_format/output.h index 8030dae0..15e751ab 100644 --- a/absl/strings/internal/str_format/output.h +++ b/absl/strings/internal/str_format/output.h @@ -22,6 +22,7 @@ #define ABSL_STRINGS_INTERNAL_STR_FORMAT_OUTPUT_H_ #include <cstdio> +#include <ios> #include <ostream> #include <string> @@ -71,7 +72,7 @@ inline void AbslFormatFlush(std::string* out, string_view s) { out->append(s.data(), s.size()); } inline void AbslFormatFlush(std::ostream* out, string_view s) { - out->write(s.data(), s.size()); + out->write(s.data(), static_cast<std::streamsize>(s.size())); } inline void AbslFormatFlush(FILERawSink* sink, string_view v) { diff --git a/absl/strings/internal/str_format/parser.cc b/absl/strings/internal/str_format/parser.cc index f308d023..2c9c07da 100644 --- a/absl/strings/internal/str_format/parser.cc +++ b/absl/strings/internal/str_format/parser.cc @@ -34,60 +34,67 @@ namespace str_format_internal { using CC = FormatConversionCharInternal; using LM = LengthMod; +// Abbreviations to fit in the table below. +constexpr auto f_sign = Flags::kSignCol; +constexpr auto f_alt = Flags::kAlt; +constexpr auto f_pos = Flags::kShowPos; +constexpr auto f_left = Flags::kLeft; +constexpr auto f_zero = Flags::kZero; + ABSL_CONST_INIT const ConvTag kTags[256] = { - {}, {}, {}, {}, {}, {}, {}, {}, // 00-07 - {}, {}, {}, {}, {}, {}, {}, {}, // 08-0f - {}, {}, {}, {}, {}, {}, {}, {}, // 10-17 - {}, {}, {}, {}, {}, {}, {}, {}, // 18-1f - {}, {}, {}, {}, {}, {}, {}, {}, // 20-27 - {}, {}, {}, {}, {}, {}, {}, {}, // 28-2f - {}, {}, {}, {}, {}, {}, {}, {}, // 30-37 - {}, {}, {}, {}, {}, {}, {}, {}, // 38-3f - {}, CC::A, {}, {}, {}, CC::E, CC::F, CC::G, // @ABCDEFG - {}, {}, {}, {}, LM::L, {}, {}, {}, // HIJKLMNO - {}, {}, {}, {}, {}, {}, {}, {}, // PQRSTUVW - CC::X, {}, {}, {}, {}, {}, {}, {}, // XYZ[\]^_ - {}, CC::a, {}, CC::c, CC::d, CC::e, CC::f, CC::g, // `abcdefg - LM::h, CC::i, LM::j, {}, LM::l, {}, CC::n, CC::o, // hijklmno - CC::p, LM::q, {}, CC::s, LM::t, CC::u, {}, {}, // pqrstuvw - CC::x, {}, LM::z, {}, {}, {}, {}, {}, // xyz{|}! - {}, {}, {}, {}, {}, {}, {}, {}, // 80-87 - {}, {}, {}, {}, {}, {}, {}, {}, // 88-8f - {}, {}, {}, {}, {}, {}, {}, {}, // 90-97 - {}, {}, {}, {}, {}, {}, {}, {}, // 98-9f - {}, {}, {}, {}, {}, {}, {}, {}, // a0-a7 - {}, {}, {}, {}, {}, {}, {}, {}, // a8-af - {}, {}, {}, {}, {}, {}, {}, {}, // b0-b7 - {}, {}, {}, {}, {}, {}, {}, {}, // b8-bf - {}, {}, {}, {}, {}, {}, {}, {}, // c0-c7 - {}, {}, {}, {}, {}, {}, {}, {}, // c8-cf - {}, {}, {}, {}, {}, {}, {}, {}, // d0-d7 - {}, {}, {}, {}, {}, {}, {}, {}, // d8-df - {}, {}, {}, {}, {}, {}, {}, {}, // e0-e7 - {}, {}, {}, {}, {}, {}, {}, {}, // e8-ef - {}, {}, {}, {}, {}, {}, {}, {}, // f0-f7 - {}, {}, {}, {}, {}, {}, {}, {}, // f8-ff + {}, {}, {}, {}, {}, {}, {}, {}, // 00-07 + {}, {}, {}, {}, {}, {}, {}, {}, // 08-0f + {}, {}, {}, {}, {}, {}, {}, {}, // 10-17 + {}, {}, {}, {}, {}, {}, {}, {}, // 18-1f + f_sign, {}, {}, f_alt, {}, {}, {}, {}, // !"#$%&' + {}, {}, {}, f_pos, {}, f_left, {}, {}, // ()*+,-./ + f_zero, {}, {}, {}, {}, {}, {}, {}, // 01234567 + {}, {}, {}, {}, {}, {}, {}, {}, // 89:;<=>? + {}, CC::A, {}, {}, {}, CC::E, CC::F, CC::G, // @ABCDEFG + {}, {}, {}, {}, LM::L, {}, {}, {}, // HIJKLMNO + {}, {}, {}, {}, {}, {}, {}, {}, // PQRSTUVW + CC::X, {}, {}, {}, {}, {}, {}, {}, // XYZ[\]^_ + {}, CC::a, {}, CC::c, CC::d, CC::e, CC::f, CC::g, // `abcdefg + LM::h, CC::i, LM::j, {}, LM::l, {}, CC::n, CC::o, // hijklmno + CC::p, LM::q, {}, CC::s, LM::t, CC::u, {}, {}, // pqrstuvw + CC::x, {}, LM::z, {}, {}, {}, {}, {}, // xyz{|}! + {}, {}, {}, {}, {}, {}, {}, {}, // 80-87 + {}, {}, {}, {}, {}, {}, {}, {}, // 88-8f + {}, {}, {}, {}, {}, {}, {}, {}, // 90-97 + {}, {}, {}, {}, {}, {}, {}, {}, // 98-9f + {}, {}, {}, {}, {}, {}, {}, {}, // a0-a7 + {}, {}, {}, {}, {}, {}, {}, {}, // a8-af + {}, {}, {}, {}, {}, {}, {}, {}, // b0-b7 + {}, {}, {}, {}, {}, {}, {}, {}, // b8-bf + {}, {}, {}, {}, {}, {}, {}, {}, // c0-c7 + {}, {}, {}, {}, {}, {}, {}, {}, // c8-cf + {}, {}, {}, {}, {}, {}, {}, {}, // d0-d7 + {}, {}, {}, {}, {}, {}, {}, {}, // d8-df + {}, {}, {}, {}, {}, {}, {}, {}, // e0-e7 + {}, {}, {}, {}, {}, {}, {}, {}, // e8-ef + {}, {}, {}, {}, {}, {}, {}, {}, // f0-f7 + {}, {}, {}, {}, {}, {}, {}, {}, // f8-ff }; namespace { bool CheckFastPathSetting(const UnboundConversion& conv) { - bool should_be_basic = !conv.flags.left && // - !conv.flags.show_pos && // - !conv.flags.sign_col && // - !conv.flags.alt && // - !conv.flags.zero && // - (conv.width.value() == -1) && - (conv.precision.value() == -1); - if (should_be_basic != conv.flags.basic) { + bool width_precision_needed = + conv.width.value() >= 0 || conv.precision.value() >= 0; + if (width_precision_needed && conv.flags == Flags::kBasic) { fprintf(stderr, "basic=%d left=%d show_pos=%d sign_col=%d alt=%d zero=%d " "width=%d precision=%d\n", - conv.flags.basic, conv.flags.left, conv.flags.show_pos, - conv.flags.sign_col, conv.flags.alt, conv.flags.zero, - conv.width.value(), conv.precision.value()); + conv.flags == Flags::kBasic ? 1 : 0, + FlagsContains(conv.flags, Flags::kLeft) ? 1 : 0, + FlagsContains(conv.flags, Flags::kShowPos) ? 1 : 0, + FlagsContains(conv.flags, Flags::kSignCol) ? 1 : 0, + FlagsContains(conv.flags, Flags::kAlt) ? 1 : 0, + FlagsContains(conv.flags, Flags::kZero) ? 1 : 0, conv.width.value(), + conv.precision.value()); + return false; } - return should_be_basic == conv.flags.basic; + return true; } template <bool is_positional> @@ -131,40 +138,21 @@ const char *ConsumeConversion(const char *pos, const char *const end, ABSL_FORMAT_PARSER_INTERNAL_GET_CHAR(); // We should start with the basic flag on. - assert(conv->flags.basic); + assert(conv->flags == Flags::kBasic); // Any non alpha character makes this conversion not basic. // This includes flags (-+ #0), width (1-9, *) or precision (.). // All conversion characters and length modifiers are alpha characters. if (c < 'A') { - conv->flags.basic = false; - - for (; c <= '0';) { - // FIXME: We might be able to speed this up reusing the lookup table from - // above. It might require changing Flags to be a plain integer where we - // can |= a value. - switch (c) { - case '-': - conv->flags.left = true; - break; - case '+': - conv->flags.show_pos = true; - break; - case ' ': - conv->flags.sign_col = true; - break; - case '#': - conv->flags.alt = true; - break; - case '0': - conv->flags.zero = true; - break; - default: - goto flags_done; + while (c <= '0') { + auto tag = GetTagForChar(c); + if (tag.is_flags()) { + conv->flags = conv->flags | tag.as_flags(); + ABSL_FORMAT_PARSER_INTERNAL_GET_CHAR(); + } else { + break; } - ABSL_FORMAT_PARSER_INTERNAL_GET_CHAR(); } -flags_done: if (c <= '9') { if (c >= '0') { @@ -173,12 +161,12 @@ flags_done: if (ABSL_PREDICT_FALSE(*next_arg != 0)) return nullptr; // Positional conversion. *next_arg = -1; - conv->flags = Flags(); - conv->flags.basic = true; return ConsumeConversion<true>(original_pos, end, conv, next_arg); } + conv->flags = conv->flags | Flags::kNonBasic; conv->width.set_value(maybe_width); } else if (c == '*') { + conv->flags = conv->flags | Flags::kNonBasic; ABSL_FORMAT_PARSER_INTERNAL_GET_CHAR(); if (is_positional) { if (ABSL_PREDICT_FALSE(c < '1' || c > '9')) return nullptr; @@ -192,6 +180,7 @@ flags_done: } if (c == '.') { + conv->flags = conv->flags | Flags::kNonBasic; ABSL_FORMAT_PARSER_INTERNAL_GET_CHAR(); if (std::isdigit(c)) { conv->precision.set_value(parse_digits()); diff --git a/absl/strings/internal/str_format/parser.h b/absl/strings/internal/str_format/parser.h index 6504dd3d..32b91d03 100644 --- a/absl/strings/internal/str_format/parser.h +++ b/absl/strings/internal/str_format/parser.h @@ -41,10 +41,7 @@ std::string LengthModToString(LengthMod v); // The analyzed properties of a single specified conversion. struct UnboundConversion { - UnboundConversion() - : flags() /* This is required to zero all the fields of flags. */ { - flags.basic = true; - } + UnboundConversion() {} class InputValue { public: @@ -79,7 +76,7 @@ struct UnboundConversion { InputValue width; InputValue precision; - Flags flags; + Flags flags = Flags::kBasic; LengthMod length_mod = LengthMod::none; FormatConversionChar conv = FormatConversionCharInternal::kNone; }; @@ -93,32 +90,43 @@ const char* ConsumeUnboundConversion(const char* p, const char* end, UnboundConversion* conv, int* next_arg); // Helper tag class for the table below. -// It allows fast `char -> ConversionChar/LengthMod` checking and +// It allows fast `char -> ConversionChar/LengthMod/Flags` checking and // conversions. class ConvTag { public: constexpr ConvTag(FormatConversionChar conversion_char) // NOLINT - : tag_(static_cast<int8_t>(conversion_char)) {} - // We invert the length modifiers to make them negative so that we can easily - // test for them. + : tag_(static_cast<uint8_t>(conversion_char)) {} constexpr ConvTag(LengthMod length_mod) // NOLINT - : tag_(~static_cast<std::int8_t>(length_mod)) {} - // Everything else is -128, which is negative to make is_conv() simpler. - constexpr ConvTag() : tag_(-128) {} + : tag_(0x80 | static_cast<uint8_t>(length_mod)) {} + constexpr ConvTag(Flags flags) // NOLINT + : tag_(0xc0 | static_cast<uint8_t>(flags)) {} + constexpr ConvTag() : tag_(0xFF) {} + + bool is_conv() const { return (tag_ & 0x80) == 0; } + bool is_length() const { return (tag_ & 0xC0) == 0x80; } + bool is_flags() const { return (tag_ & 0xE0) == 0xC0; } - bool is_conv() const { return tag_ >= 0; } - bool is_length() const { return tag_ < 0 && tag_ != -128; } FormatConversionChar as_conv() const { assert(is_conv()); + assert(!is_length()); + assert(!is_flags()); return static_cast<FormatConversionChar>(tag_); } LengthMod as_length() const { + assert(!is_conv()); assert(is_length()); - return static_cast<LengthMod>(~tag_); + assert(!is_flags()); + return static_cast<LengthMod>(tag_ & 0x3F); + } + Flags as_flags() const { + assert(!is_conv()); + assert(!is_length()); + assert(is_flags()); + return static_cast<Flags>(tag_ & 0x1F); } private: - std::int8_t tag_; + uint8_t tag_; }; extern const ConvTag kTags[256]; @@ -143,7 +151,8 @@ bool ParseFormatString(string_view src, Consumer consumer) { const char* p = src.data(); const char* const end = p + src.size(); while (p != end) { - const char* percent = static_cast<const char*>(memchr(p, '%', end - p)); + const char* percent = + static_cast<const char*>(memchr(p, '%', static_cast<size_t>(end - p))); if (!percent) { // We found the last substring. return consumer.Append(string_view(p, end - p)); @@ -234,7 +243,8 @@ class ParsedFormatBase { string_view text(base, 0); for (const auto& item : items_) { const char* const end = text.data() + text.size(); - text = string_view(end, (base + item.text_end) - end); + text = + string_view(end, static_cast<size_t>((base + item.text_end) - end)); if (item.is_conversion) { if (!consumer.ConvertOne(item.conv, text)) return false; } else { diff --git a/absl/strings/internal/str_format/parser_test.cc b/absl/strings/internal/str_format/parser_test.cc index a5fa1c79..fe0d2963 100644 --- a/absl/strings/internal/str_format/parser_test.cc +++ b/absl/strings/internal/str_format/parser_test.cc @@ -270,15 +270,22 @@ TEST_F(ConsumeUnboundConversionTest, Flags) { for (int k = 0; k < kNumFlags; ++k) if ((i >> k) & 1) fmt += kAllFlags[k]; // flag order shouldn't matter - if (rev == 1) { std::reverse(fmt.begin(), fmt.end()); } + if (rev == 1) { + std::reverse(fmt.begin(), fmt.end()); + } fmt += 'd'; SCOPED_TRACE(fmt); EXPECT_TRUE(Run(fmt.c_str())); - EXPECT_EQ(fmt.find('-') == std::string::npos, !o.flags.left); - EXPECT_EQ(fmt.find('+') == std::string::npos, !o.flags.show_pos); - EXPECT_EQ(fmt.find(' ') == std::string::npos, !o.flags.sign_col); - EXPECT_EQ(fmt.find('#') == std::string::npos, !o.flags.alt); - EXPECT_EQ(fmt.find('0') == std::string::npos, !o.flags.zero); + EXPECT_EQ(fmt.find('-') == std::string::npos, + !FlagsContains(o.flags, Flags::kLeft)); + EXPECT_EQ(fmt.find('+') == std::string::npos, + !FlagsContains(o.flags, Flags::kShowPos)); + EXPECT_EQ(fmt.find(' ') == std::string::npos, + !FlagsContains(o.flags, Flags::kSignCol)); + EXPECT_EQ(fmt.find('#') == std::string::npos, + !FlagsContains(o.flags, Flags::kAlt)); + EXPECT_EQ(fmt.find('0') == std::string::npos, + !FlagsContains(o.flags, Flags::kZero)); } } } @@ -288,14 +295,14 @@ TEST_F(ConsumeUnboundConversionTest, BasicFlag) { for (const char* fmt : {"d", "llx", "G", "1$X"}) { SCOPED_TRACE(fmt); EXPECT_TRUE(Run(fmt)); - EXPECT_TRUE(o.flags.basic); + EXPECT_EQ(o.flags, Flags::kBasic); } // Flag is off for (const char* fmt : {"3d", ".llx", "-G", "1$#X"}) { SCOPED_TRACE(fmt); EXPECT_TRUE(Run(fmt)); - EXPECT_FALSE(o.flags.basic); + EXPECT_NE(o.flags, Flags::kBasic); } } diff --git a/absl/strings/internal/str_join_internal.h b/absl/strings/internal/str_join_internal.h index 31dbf672..d97d5033 100644 --- a/absl/strings/internal/str_join_internal.h +++ b/absl/strings/internal/str_join_internal.h @@ -229,10 +229,11 @@ std::string JoinAlgorithm(Iterator start, Iterator end, absl::string_view s, std::string result; if (start != end) { // Sums size - size_t result_size = start->size(); + auto&& start_value = *start; + size_t result_size = start_value.size(); for (Iterator it = start; ++it != end;) { result_size += s.size(); - result_size += it->size(); + result_size += (*it).size(); } if (result_size > 0) { @@ -240,13 +241,15 @@ std::string JoinAlgorithm(Iterator start, Iterator end, absl::string_view s, // Joins strings char* result_buf = &*result.begin(); - memcpy(result_buf, start->data(), start->size()); - result_buf += start->size(); + + memcpy(result_buf, start_value.data(), start_value.size()); + result_buf += start_value.size(); for (Iterator it = start; ++it != end;) { memcpy(result_buf, s.data(), s.size()); result_buf += s.size(); - memcpy(result_buf, it->data(), it->size()); - result_buf += it->size(); + auto&& value = *it; + memcpy(result_buf, value.data(), value.size()); + result_buf += value.size(); } } } diff --git a/absl/strings/internal/str_split_internal.h b/absl/strings/internal/str_split_internal.h index a2f41c15..e7664216 100644 --- a/absl/strings/internal/str_split_internal.h +++ b/absl/strings/internal/str_split_internal.h @@ -32,7 +32,7 @@ #include <array> #include <initializer_list> #include <iterator> -#include <map> +#include <tuple> #include <type_traits> #include <utility> #include <vector> @@ -64,7 +64,7 @@ class ConvertibleToStringView { ConvertibleToStringView(const std::string& s) // NOLINT(runtime/explicit) : value_(s) {} - // Matches rvalue strings and moves their data to a member. + // Disable conversion from rvalue strings. ConvertibleToStringView(std::string&& s) = delete; ConvertibleToStringView(const std::string&& s) = delete; @@ -182,6 +182,13 @@ template <typename T> struct HasConstIterator<T, absl::void_t<typename T::const_iterator>> : std::true_type {}; +// HasEmplace<T>::value is true iff there exists a method T::emplace(). +template <typename T, typename = void> +struct HasEmplace : std::false_type {}; +template <typename T> +struct HasEmplace<T, absl::void_t<decltype(std::declval<T>().emplace())>> + : std::true_type {}; + // IsInitializerList<T>::value is true iff T is an std::initializer_list. More // details below in Splitter<> where this is used. std::false_type IsInitializerListDispatch(...); // default: No @@ -372,50 +379,43 @@ class Splitter { // value. template <typename Container, typename First, typename Second> struct ConvertToContainer<Container, std::pair<const First, Second>, true> { + using iterator = typename Container::iterator; + Container operator()(const Splitter& splitter) const { Container m; - typename Container::iterator it; + iterator it; bool insert = true; - for (const auto& sp : splitter) { + for (const absl::string_view sv : splitter) { if (insert) { - it = Inserter<Container>::Insert(&m, First(sp), Second()); + it = InsertOrEmplace(&m, sv); } else { - it->second = Second(sp); + it->second = Second(sv); } insert = !insert; } return m; } - // Inserts the key and value into the given map, returning an iterator to - // the inserted item. Specialized for std::map and std::multimap to use - // emplace() and adapt emplace()'s return value. - template <typename Map> - struct Inserter { - using M = Map; - template <typename... Args> - static typename M::iterator Insert(M* m, Args&&... args) { - return m->insert(std::make_pair(std::forward<Args>(args)...)).first; - } - }; - - template <typename... Ts> - struct Inserter<std::map<Ts...>> { - using M = std::map<Ts...>; - template <typename... Args> - static typename M::iterator Insert(M* m, Args&&... args) { - return m->emplace(std::make_pair(std::forward<Args>(args)...)).first; - } - }; - - template <typename... Ts> - struct Inserter<std::multimap<Ts...>> { - using M = std::multimap<Ts...>; - template <typename... Args> - static typename M::iterator Insert(M* m, Args&&... args) { - return m->emplace(std::make_pair(std::forward<Args>(args)...)); - } - }; + // Inserts the key and an empty value into the map, returning an iterator to + // the inserted item. We use emplace() if available, otherwise insert(). + template <typename M> + static absl::enable_if_t<HasEmplace<M>::value, iterator> InsertOrEmplace( + M* m, absl::string_view key) { + // Use piecewise_construct to support old versions of gcc in which pair + // constructor can't otherwise construct string from string_view. + return ToIter(m->emplace(std::piecewise_construct, std::make_tuple(key), + std::tuple<>())); + } + template <typename M> + static absl::enable_if_t<!HasEmplace<M>::value, iterator> InsertOrEmplace( + M* m, absl::string_view key) { + return ToIter(m->insert(std::make_pair(First(key), Second("")))); + } + + static iterator ToIter(std::pair<iterator, bool> pair) { + return pair.first; + } + static iterator ToIter(iterator iter) { return iter; } }; StringType text_; diff --git a/absl/strings/internal/string_constant.h b/absl/strings/internal/string_constant.h index a11336b7..f68b17d7 100644 --- a/absl/strings/internal/string_constant.h +++ b/absl/strings/internal/string_constant.h @@ -35,17 +35,25 @@ namespace strings_internal { // below. template <typename T> struct StringConstant { + private: + static constexpr bool TryConstexprEval(absl::string_view view) { + return view.empty() || 2 * view[0] != 1; + } + + public: static constexpr absl::string_view value = T{}(); constexpr absl::string_view operator()() const { return value; } // Check to be sure `view` points to constant data. // Otherwise, it can't be constant evaluated. - static_assert(value.empty() || 2 * value[0] != 1, + static_assert(TryConstexprEval(value), "The input string_view must point to constant data."); }; +#ifdef ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL template <typename T> -constexpr absl::string_view StringConstant<T>::value; // NOLINT +constexpr absl::string_view StringConstant<T>::value; +#endif // Factory function for `StringConstant` instances. // It supports callables that have a constexpr default constructor and a diff --git a/absl/strings/internal/utf8.cc b/absl/strings/internal/utf8.cc index 8fd8edc1..7ecb93df 100644 --- a/absl/strings/internal/utf8.cc +++ b/absl/strings/internal/utf8.cc @@ -25,25 +25,25 @@ size_t EncodeUTF8Char(char *buffer, char32_t utf8_char) { *buffer = static_cast<char>(utf8_char); return 1; } else if (utf8_char <= 0x7FF) { - buffer[1] = 0x80 | (utf8_char & 0x3F); + buffer[1] = static_cast<char>(0x80 | (utf8_char & 0x3F)); utf8_char >>= 6; - buffer[0] = 0xC0 | utf8_char; + buffer[0] = static_cast<char>(0xC0 | utf8_char); return 2; } else if (utf8_char <= 0xFFFF) { - buffer[2] = 0x80 | (utf8_char & 0x3F); + buffer[2] = static_cast<char>(0x80 | (utf8_char & 0x3F)); utf8_char >>= 6; - buffer[1] = 0x80 | (utf8_char & 0x3F); + buffer[1] = static_cast<char>(0x80 | (utf8_char & 0x3F)); utf8_char >>= 6; - buffer[0] = 0xE0 | utf8_char; + buffer[0] = static_cast<char>(0xE0 | utf8_char); return 3; } else { - buffer[3] = 0x80 | (utf8_char & 0x3F); + buffer[3] = static_cast<char>(0x80 | (utf8_char & 0x3F)); utf8_char >>= 6; - buffer[2] = 0x80 | (utf8_char & 0x3F); + buffer[2] = static_cast<char>(0x80 | (utf8_char & 0x3F)); utf8_char >>= 6; - buffer[1] = 0x80 | (utf8_char & 0x3F); + buffer[1] = static_cast<char>(0x80 | (utf8_char & 0x3F)); utf8_char >>= 6; - buffer[0] = 0xF0 | utf8_char; + buffer[0] = static_cast<char>(0xF0 | utf8_char); return 4; } } diff --git a/absl/strings/numbers.cc b/absl/strings/numbers.cc index 966d94bd..e798fc69 100644 --- a/absl/strings/numbers.cc +++ b/absl/strings/numbers.cc @@ -505,7 +505,7 @@ size_t numbers_internal::SixDigitsToBuffer(double d, char* const buffer) { *out++ = '-'; d = -d; } - if (std::isinf(d)) { + if (d > std::numeric_limits<double>::max()) { strcpy(out, "inf"); // NOLINT(runtime/printf) return out + 3 - buffer; } @@ -757,8 +757,8 @@ struct LookupTables { // // uint128& operator/=(uint128) is not constexpr, so hardcode the resulting // array to avoid a static initializer. -template<> -const uint128 LookupTables<uint128>::kVmaxOverBase[] = { +template <> +ABSL_CONST_INIT const uint128 LookupTables<uint128>::kVmaxOverBase[] = { 0, 0, MakeUint128(9223372036854775807u, 18446744073709551615u), @@ -809,8 +809,8 @@ const uint128 LookupTables<uint128>::kVmaxOverBase[] = { // // int128& operator/=(int128) is not constexpr, so hardcode the resulting array // to avoid a static initializer. -template<> -const int128 LookupTables<int128>::kVmaxOverBase[] = { +template <> +ABSL_CONST_INIT const int128 LookupTables<int128>::kVmaxOverBase[] = { 0, 0, MakeInt128(4611686018427387903, 18446744073709551615u), @@ -862,8 +862,8 @@ const int128 LookupTables<int128>::kVmaxOverBase[] = { // // int128& operator/=(int128) is not constexpr, so hardcode the resulting array // to avoid a static initializer. -template<> -const int128 LookupTables<int128>::kVminOverBase[] = { +template <> +ABSL_CONST_INIT const int128 LookupTables<int128>::kVminOverBase[] = { 0, 0, MakeInt128(-4611686018427387904, 0u), @@ -904,11 +904,11 @@ const int128 LookupTables<int128>::kVminOverBase[] = { }; template <typename IntType> -const IntType LookupTables<IntType>::kVmaxOverBase[] = +ABSL_CONST_INIT const IntType LookupTables<IntType>::kVmaxOverBase[] = X_OVER_BASE_INITIALIZER(std::numeric_limits<IntType>::max()); template <typename IntType> -const IntType LookupTables<IntType>::kVminOverBase[] = +ABSL_CONST_INIT const IntType LookupTables<IntType>::kVminOverBase[] = X_OVER_BASE_INITIALIZER(std::numeric_limits<IntType>::min()); #undef X_OVER_BASE_INITIALIZER diff --git a/absl/strings/numbers.h b/absl/strings/numbers.h index 1780bb44..86c84ed3 100644 --- a/absl/strings/numbers.h +++ b/absl/strings/numbers.h @@ -23,8 +23,12 @@ #ifndef ABSL_STRINGS_NUMBERS_H_ #define ABSL_STRINGS_NUMBERS_H_ -#ifdef __SSE4_2__ -#include <x86intrin.h> +#ifdef __SSSE3__ +#include <tmmintrin.h> +#endif + +#ifdef _MSC_VER +#include <intrin.h> #endif #include <cstddef> @@ -36,14 +40,7 @@ #include <type_traits> #include "absl/base/config.h" -#ifdef __SSE4_2__ -// TODO(jorg): Remove this when we figure out the right way -// to swap bytes on SSE 4.2 that works with the compilers -// we claim to support. Also, add tests for the compiler -// that doesn't support the Intel _bswap64 intrinsic but -// does support all the SSE 4.2 intrinsics #include "absl/base/internal/endian.h" -#endif #include "absl/base/macros.h" #include "absl/base/port.h" #include "absl/numeric/bits.h" @@ -96,6 +93,25 @@ ABSL_MUST_USE_RESULT bool SimpleAtod(absl::string_view str, double* out); // unspecified state. ABSL_MUST_USE_RESULT bool SimpleAtob(absl::string_view str, bool* out); +// SimpleHexAtoi() +// +// Converts a hexadecimal string (optionally followed or preceded by ASCII +// whitespace) to an integer, returning `true` if successful. Only valid base-16 +// hexadecimal integers whose value falls within the range of the integer type +// (optionally preceded by a `+` or `-`) can be converted. A valid hexadecimal +// value may include both upper and lowercase character symbols, and may +// optionally include a leading "0x" (or "0X") number prefix, which is ignored +// by this function. If any errors are encountered, this function returns +// `false`, leaving `out` in an unspecified state. +template <typename int_type> +ABSL_MUST_USE_RESULT bool SimpleHexAtoi(absl::string_view str, int_type* out); + +// Overloads of SimpleHexAtoi() for 128 bit integers. +ABSL_MUST_USE_RESULT inline bool SimpleHexAtoi(absl::string_view str, + absl::int128* out); +ABSL_MUST_USE_RESULT inline bool SimpleHexAtoi(absl::string_view str, + absl::uint128* out); + ABSL_NAMESPACE_END } // namespace absl @@ -162,16 +178,19 @@ char* FastIntToBuffer(int_type i, char* buffer) { // TODO(jorg): This signed-ness check is used because it works correctly // with enums, and it also serves to check that int_type is not a pointer. // If one day something like std::is_signed<enum E> works, switch to it. - if (static_cast<int_type>(1) - 2 < 0) { // Signed - if (sizeof(i) > 32 / 8) { // 33-bit to 64-bit + // These conditions are constexpr bools to suppress MSVC warning C4127. + constexpr bool kIsSigned = static_cast<int_type>(1) - 2 < 0; + constexpr bool kUse64Bit = sizeof(i) > 32 / 8; + if (kIsSigned) { + if (kUse64Bit) { return FastIntToBuffer(static_cast<int64_t>(i), buffer); - } else { // 32-bit or less + } else { return FastIntToBuffer(static_cast<int32_t>(i), buffer); } - } else { // Unsigned - if (sizeof(i) > 32 / 8) { // 33-bit to 64-bit + } else { + if (kUse64Bit) { return FastIntToBuffer(static_cast<uint64_t>(i), buffer); - } else { // 32-bit or less + } else { return FastIntToBuffer(static_cast<uint32_t>(i), buffer); } } @@ -190,22 +209,25 @@ ABSL_MUST_USE_RESULT bool safe_strtoi_base(absl::string_view s, int_type* out, // TODO(jorg): This signed-ness check is used because it works correctly // with enums, and it also serves to check that int_type is not a pointer. // If one day something like std::is_signed<enum E> works, switch to it. - if (static_cast<int_type>(1) - 2 < 0) { // Signed - if (sizeof(*out) == 64 / 8) { // 64-bit + // These conditions are constexpr bools to suppress MSVC warning C4127. + constexpr bool kIsSigned = static_cast<int_type>(1) - 2 < 0; + constexpr bool kUse64Bit = sizeof(*out) == 64 / 8; + if (kIsSigned) { + if (kUse64Bit) { int64_t val; parsed = numbers_internal::safe_strto64_base(s, &val, base); *out = static_cast<int_type>(val); - } else { // 32-bit + } else { int32_t val; parsed = numbers_internal::safe_strto32_base(s, &val, base); *out = static_cast<int_type>(val); } - } else { // Unsigned - if (sizeof(*out) == 64 / 8) { // 64-bit + } else { + if (kUse64Bit) { uint64_t val; parsed = numbers_internal::safe_strtou64_base(s, &val, base); *out = static_cast<int_type>(val); - } else { // 32-bit + } else { uint32_t val; parsed = numbers_internal::safe_strtou32_base(s, &val, base); *out = static_cast<int_type>(val); @@ -221,7 +243,7 @@ ABSL_MUST_USE_RESULT bool safe_strtoi_base(absl::string_view s, int_type* out, // Returns the number of non-pad digits of the output (it can never be zero // since 0 has one digit). inline size_t FastHexToBufferZeroPad16(uint64_t val, char* out) { -#ifdef __SSE4_2__ +#ifdef ABSL_INTERNAL_HAVE_SSSE3 uint64_t be = absl::big_endian::FromHost64(val); const auto kNibbleMask = _mm_set1_epi8(0xf); const auto kHexDigits = _mm_setr_epi8('0', '1', '2', '3', '4', '5', '6', '7', @@ -240,7 +262,7 @@ inline size_t FastHexToBufferZeroPad16(uint64_t val, char* out) { } #endif // | 0x1 so that even 0 has 1 digit. - return 16 - countl_zero(val | 0x1) / 4; + return 16 - static_cast<size_t>(countl_zero(val | 0x1) / 4); } } // namespace numbers_internal @@ -260,6 +282,21 @@ ABSL_MUST_USE_RESULT inline bool SimpleAtoi(absl::string_view str, return numbers_internal::safe_strtou128_base(str, out, 10); } +template <typename int_type> +ABSL_MUST_USE_RESULT bool SimpleHexAtoi(absl::string_view str, int_type* out) { + return numbers_internal::safe_strtoi_base(str, out, 16); +} + +ABSL_MUST_USE_RESULT inline bool SimpleHexAtoi(absl::string_view str, + absl::int128* out) { + return numbers_internal::safe_strto128_base(str, out, 16); +} + +ABSL_MUST_USE_RESULT inline bool SimpleHexAtoi(absl::string_view str, + absl::uint128* out) { + return numbers_internal::safe_strtou128_base(str, out, 16); +} + ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/strings/numbers_test.cc b/absl/strings/numbers_test.cc index f3103106..498c210d 100644 --- a/absl/strings/numbers_test.cc +++ b/absl/strings/numbers_test.cc @@ -47,6 +47,7 @@ namespace { using absl::SimpleAtoi; +using absl::SimpleHexAtoi; using absl::numbers_internal::kSixDigitsToBufferSize; using absl::numbers_internal::safe_strto32_base; using absl::numbers_internal::safe_strto64_base; @@ -468,6 +469,148 @@ TEST(NumbersTest, Atoenum) { VerifySimpleAtoiGood<E_biguint>(E_biguint_max32, E_biguint_max32); } +template <typename int_type, typename in_val_type> +void VerifySimpleHexAtoiGood(in_val_type in_value, int_type exp_value) { + std::string s; + // uint128 can be streamed but not StrCat'd + absl::strings_internal::OStringStream strm(&s); + if (in_value >= 0) { + strm << std::hex << in_value; + } else { + // Inefficient for small integers, but works with all integral types. + strm << "-" << std::hex << -absl::uint128(in_value); + } + int_type x = static_cast<int_type>(~exp_value); + EXPECT_TRUE(SimpleHexAtoi(s, &x)) + << "in_value=" << std::hex << in_value << " s=" << s << " x=" << x; + EXPECT_EQ(exp_value, x); + x = static_cast<int_type>(~exp_value); + EXPECT_TRUE(SimpleHexAtoi( + s.c_str(), &x)); // NOLINT: readability-redundant-string-conversions + EXPECT_EQ(exp_value, x); +} + +template <typename int_type, typename in_val_type> +void VerifySimpleHexAtoiBad(in_val_type in_value) { + std::string s; + // uint128 can be streamed but not StrCat'd + absl::strings_internal::OStringStream strm(&s); + if (in_value >= 0) { + strm << std::hex << in_value; + } else { + // Inefficient for small integers, but works with all integral types. + strm << "-" << std::hex << -absl::uint128(in_value); + } + int_type x; + EXPECT_FALSE(SimpleHexAtoi(s, &x)); + EXPECT_FALSE(SimpleHexAtoi( + s.c_str(), &x)); // NOLINT: readability-redundant-string-conversions +} + +TEST(NumbersTest, HexAtoi) { + // SimpleHexAtoi(absl::string_view, int32_t) + VerifySimpleHexAtoiGood<int32_t>(0, 0); + VerifySimpleHexAtoiGood<int32_t>(0x42, 0x42); + VerifySimpleHexAtoiGood<int32_t>(-0x42, -0x42); + + VerifySimpleHexAtoiGood<int32_t>(std::numeric_limits<int32_t>::min(), + std::numeric_limits<int32_t>::min()); + VerifySimpleHexAtoiGood<int32_t>(std::numeric_limits<int32_t>::max(), + std::numeric_limits<int32_t>::max()); + + // SimpleHexAtoi(absl::string_view, uint32_t) + VerifySimpleHexAtoiGood<uint32_t>(0, 0); + VerifySimpleHexAtoiGood<uint32_t>(0x42, 0x42); + VerifySimpleHexAtoiBad<uint32_t>(-0x42); + + VerifySimpleHexAtoiBad<uint32_t>(std::numeric_limits<int32_t>::min()); + VerifySimpleHexAtoiGood<uint32_t>(std::numeric_limits<int32_t>::max(), + std::numeric_limits<int32_t>::max()); + VerifySimpleHexAtoiGood<uint32_t>(std::numeric_limits<uint32_t>::max(), + std::numeric_limits<uint32_t>::max()); + VerifySimpleHexAtoiBad<uint32_t>(std::numeric_limits<int64_t>::min()); + VerifySimpleHexAtoiBad<uint32_t>(std::numeric_limits<int64_t>::max()); + VerifySimpleHexAtoiBad<uint32_t>(std::numeric_limits<uint64_t>::max()); + + // SimpleHexAtoi(absl::string_view, int64_t) + VerifySimpleHexAtoiGood<int64_t>(0, 0); + VerifySimpleHexAtoiGood<int64_t>(0x42, 0x42); + VerifySimpleHexAtoiGood<int64_t>(-0x42, -0x42); + + VerifySimpleHexAtoiGood<int64_t>(std::numeric_limits<int32_t>::min(), + std::numeric_limits<int32_t>::min()); + VerifySimpleHexAtoiGood<int64_t>(std::numeric_limits<int32_t>::max(), + std::numeric_limits<int32_t>::max()); + VerifySimpleHexAtoiGood<int64_t>(std::numeric_limits<uint32_t>::max(), + std::numeric_limits<uint32_t>::max()); + VerifySimpleHexAtoiGood<int64_t>(std::numeric_limits<int64_t>::min(), + std::numeric_limits<int64_t>::min()); + VerifySimpleHexAtoiGood<int64_t>(std::numeric_limits<int64_t>::max(), + std::numeric_limits<int64_t>::max()); + VerifySimpleHexAtoiBad<int64_t>(std::numeric_limits<uint64_t>::max()); + + // SimpleHexAtoi(absl::string_view, uint64_t) + VerifySimpleHexAtoiGood<uint64_t>(0, 0); + VerifySimpleHexAtoiGood<uint64_t>(0x42, 0x42); + VerifySimpleHexAtoiBad<uint64_t>(-0x42); + + VerifySimpleHexAtoiBad<uint64_t>(std::numeric_limits<int32_t>::min()); + VerifySimpleHexAtoiGood<uint64_t>(std::numeric_limits<int32_t>::max(), + std::numeric_limits<int32_t>::max()); + VerifySimpleHexAtoiGood<uint64_t>(std::numeric_limits<uint32_t>::max(), + std::numeric_limits<uint32_t>::max()); + VerifySimpleHexAtoiBad<uint64_t>(std::numeric_limits<int64_t>::min()); + VerifySimpleHexAtoiGood<uint64_t>(std::numeric_limits<int64_t>::max(), + std::numeric_limits<int64_t>::max()); + VerifySimpleHexAtoiGood<uint64_t>(std::numeric_limits<uint64_t>::max(), + std::numeric_limits<uint64_t>::max()); + + // SimpleHexAtoi(absl::string_view, absl::uint128) + VerifySimpleHexAtoiGood<absl::uint128>(0, 0); + VerifySimpleHexAtoiGood<absl::uint128>(0x42, 0x42); + VerifySimpleHexAtoiBad<absl::uint128>(-0x42); + + VerifySimpleHexAtoiBad<absl::uint128>(std::numeric_limits<int32_t>::min()); + VerifySimpleHexAtoiGood<absl::uint128>(std::numeric_limits<int32_t>::max(), + std::numeric_limits<int32_t>::max()); + VerifySimpleHexAtoiGood<absl::uint128>(std::numeric_limits<uint32_t>::max(), + std::numeric_limits<uint32_t>::max()); + VerifySimpleHexAtoiBad<absl::uint128>(std::numeric_limits<int64_t>::min()); + VerifySimpleHexAtoiGood<absl::uint128>(std::numeric_limits<int64_t>::max(), + std::numeric_limits<int64_t>::max()); + VerifySimpleHexAtoiGood<absl::uint128>(std::numeric_limits<uint64_t>::max(), + std::numeric_limits<uint64_t>::max()); + VerifySimpleHexAtoiGood<absl::uint128>( + std::numeric_limits<absl::uint128>::max(), + std::numeric_limits<absl::uint128>::max()); + + // Some other types + VerifySimpleHexAtoiGood<int>(-0x42, -0x42); + VerifySimpleHexAtoiGood<int32_t>(-0x42, -0x42); + VerifySimpleHexAtoiGood<uint32_t>(0x42, 0x42); + VerifySimpleHexAtoiGood<unsigned int>(0x42, 0x42); + VerifySimpleHexAtoiGood<int64_t>(-0x42, -0x42); + VerifySimpleHexAtoiGood<long>(-0x42, -0x42); // NOLINT: runtime-int + VerifySimpleHexAtoiGood<uint64_t>(0x42, 0x42); + VerifySimpleHexAtoiGood<size_t>(0x42, 0x42); + VerifySimpleHexAtoiGood<std::string::size_type>(0x42, 0x42); + + // Number prefix + int32_t value; + EXPECT_TRUE(safe_strto32_base("0x34234324", &value, 16)); + EXPECT_EQ(0x34234324, value); + + EXPECT_TRUE(safe_strto32_base("0X34234324", &value, 16)); + EXPECT_EQ(0x34234324, value); + + // ASCII whitespace + EXPECT_TRUE(safe_strto32_base(" \t\n 34234324", &value, 16)); + EXPECT_EQ(0x34234324, value); + + EXPECT_TRUE(safe_strto32_base("34234324 \t\n ", &value, 16)); + EXPECT_EQ(0x34234324, value); +} + TEST(stringtest, safe_strto32_base) { int32_t value; EXPECT_TRUE(safe_strto32_base("0x34234324", &value, 16)); diff --git a/absl/strings/str_cat.cc b/absl/strings/str_cat.cc index dd5d25b0..f4a77493 100644 --- a/absl/strings/str_cat.cc +++ b/absl/strings/str_cat.cc @@ -174,7 +174,7 @@ void AppendPieces(std::string* dest, ASSERT_NO_OVERLAP(*dest, piece); total_size += piece.size(); } - strings_internal::STLStringResizeUninitialized(dest, total_size); + strings_internal::STLStringResizeUninitializedAmortized(dest, total_size); char* const begin = &(*dest)[0]; char* out = begin + old_size; @@ -199,7 +199,7 @@ void StrAppend(std::string* dest, const AlphaNum& a, const AlphaNum& b) { ASSERT_NO_OVERLAP(*dest, a); ASSERT_NO_OVERLAP(*dest, b); std::string::size_type old_size = dest->size(); - strings_internal::STLStringResizeUninitialized( + strings_internal::STLStringResizeUninitializedAmortized( dest, old_size + a.size() + b.size()); char* const begin = &(*dest)[0]; char* out = begin + old_size; @@ -214,7 +214,7 @@ void StrAppend(std::string* dest, const AlphaNum& a, const AlphaNum& b, ASSERT_NO_OVERLAP(*dest, b); ASSERT_NO_OVERLAP(*dest, c); std::string::size_type old_size = dest->size(); - strings_internal::STLStringResizeUninitialized( + strings_internal::STLStringResizeUninitializedAmortized( dest, old_size + a.size() + b.size() + c.size()); char* const begin = &(*dest)[0]; char* out = begin + old_size; @@ -231,7 +231,7 @@ void StrAppend(std::string* dest, const AlphaNum& a, const AlphaNum& b, ASSERT_NO_OVERLAP(*dest, c); ASSERT_NO_OVERLAP(*dest, d); std::string::size_type old_size = dest->size(); - strings_internal::STLStringResizeUninitialized( + strings_internal::STLStringResizeUninitializedAmortized( dest, old_size + a.size() + b.size() + c.size() + d.size()); char* const begin = &(*dest)[0]; char* out = begin + old_size; diff --git a/absl/strings/str_cat.h b/absl/strings/str_cat.h index a8a85c73..a94bc5df 100644 --- a/absl/strings/str_cat.h +++ b/absl/strings/str_cat.h @@ -214,23 +214,29 @@ class AlphaNum { // A bool ctor would also convert incoming pointers (bletch). AlphaNum(int x) // NOLINT(runtime/explicit) - : piece_(digits_, - numbers_internal::FastIntToBuffer(x, digits_) - &digits_[0]) {} + : piece_(digits_, static_cast<size_t>( + numbers_internal::FastIntToBuffer(x, digits_) - + &digits_[0])) {} AlphaNum(unsigned int x) // NOLINT(runtime/explicit) - : piece_(digits_, - numbers_internal::FastIntToBuffer(x, digits_) - &digits_[0]) {} + : piece_(digits_, static_cast<size_t>( + numbers_internal::FastIntToBuffer(x, digits_) - + &digits_[0])) {} AlphaNum(long x) // NOLINT(*) - : piece_(digits_, - numbers_internal::FastIntToBuffer(x, digits_) - &digits_[0]) {} + : piece_(digits_, static_cast<size_t>( + numbers_internal::FastIntToBuffer(x, digits_) - + &digits_[0])) {} AlphaNum(unsigned long x) // NOLINT(*) - : piece_(digits_, - numbers_internal::FastIntToBuffer(x, digits_) - &digits_[0]) {} + : piece_(digits_, static_cast<size_t>( + numbers_internal::FastIntToBuffer(x, digits_) - + &digits_[0])) {} AlphaNum(long long x) // NOLINT(*) - : piece_(digits_, - numbers_internal::FastIntToBuffer(x, digits_) - &digits_[0]) {} + : piece_(digits_, static_cast<size_t>( + numbers_internal::FastIntToBuffer(x, digits_) - + &digits_[0])) {} AlphaNum(unsigned long long x) // NOLINT(*) - : piece_(digits_, - numbers_internal::FastIntToBuffer(x, digits_) - &digits_[0]) {} + : piece_(digits_, static_cast<size_t>( + numbers_internal::FastIntToBuffer(x, digits_) - + &digits_[0])) {} AlphaNum(float f) // NOLINT(runtime/explicit) : piece_(digits_, numbers_internal::SixDigitsToBuffer(f, digits_)) {} @@ -245,7 +251,8 @@ class AlphaNum { const strings_internal::AlphaNumBuffer<size>& buf) : piece_(&buf.data[0], buf.size) {} - AlphaNum(const char* c_str) : piece_(c_str) {} // NOLINT(runtime/explicit) + AlphaNum(const char* c_str) // NOLINT(runtime/explicit) + : piece_(NullSafeStringView(c_str)) {} // NOLINT(runtime/explicit) AlphaNum(absl::string_view pc) : piece_(pc) {} // NOLINT(runtime/explicit) template <typename Allocator> diff --git a/absl/strings/str_cat_test.cc b/absl/strings/str_cat_test.cc index f3770dc0..69df2502 100644 --- a/absl/strings/str_cat_test.cc +++ b/absl/strings/str_cat_test.cc @@ -210,6 +210,11 @@ TEST(StrCat, CornerCases) { EXPECT_EQ(result, ""); } +TEST(StrCat, NullConstCharPtr) { + const char* null = nullptr; + EXPECT_EQ(absl::StrCat("mon", null, "key"), "monkey"); +} + // A minimal allocator that uses malloc(). template <typename T> struct Mallocator { diff --git a/absl/strings/str_format.h b/absl/strings/str_format.h index 01465107..4b05c70c 100644 --- a/absl/strings/str_format.h +++ b/absl/strings/str_format.h @@ -536,8 +536,7 @@ using FormatArg = str_format_internal::FormatArgImpl; // The arguments are provided in an `absl::Span<const absl::FormatArg>`. // Each `absl::FormatArg` object binds to a single argument and keeps a // reference to it. The values used to create the `FormatArg` objects must -// outlive this function call. (See `str_format_arg.h` for information on -// the `FormatArg` class.)_ +// outlive this function call. // // Example: // diff --git a/absl/strings/str_format_test.cc b/absl/strings/str_format_test.cc index c60027ad..804e6c22 100644 --- a/absl/strings/str_format_test.cc +++ b/absl/strings/str_format_test.cc @@ -719,6 +719,7 @@ TEST_F(FormatWrapperTest, ParsedFormat) { ABSL_NAMESPACE_END } // namespace absl +namespace { using FormatExtensionTest = ::testing::Test; struct Point { @@ -750,6 +751,7 @@ TEST_F(FormatExtensionTest, AbslFormatConvertExample) { // FormatUntyped will return false for bad character. EXPECT_FALSE(absl::FormatUntyped(&actual, f1, {absl::FormatArg(p)})); } +} // namespace // Some codegen thunks that we can use to easily dump the generated assembly for // different StrFormat calls. diff --git a/absl/strings/str_join.h b/absl/strings/str_join.h index 33534536..ee5ae7ef 100644 --- a/absl/strings/str_join.h +++ b/absl/strings/str_join.h @@ -72,21 +72,15 @@ ABSL_NAMESPACE_BEGIN // functions. You may provide your own Formatter to enable `absl::StrJoin()` to // work with arbitrary types. // -// The following is an example of a custom Formatter that simply uses -// `std::to_string()` to format an integer as a std::string. -// -// struct MyFormatter { -// void operator()(std::string* out, int i) const { -// out->append(std::to_string(i)); -// } -// }; -// -// You would use the above formatter by passing an instance of it as the final -// argument to `absl::StrJoin()`: -// -// std::vector<int> v = {1, 2, 3, 4}; -// std::string s = absl::StrJoin(v, "-", MyFormatter()); -// EXPECT_EQ("1-2-3-4", s); +// The following is an example of a custom Formatter that uses +// `absl::FormatDuration` to join a list of `absl::Duration`s. +// +// std::vector<absl::Duration> v = {absl::Seconds(1), absl::Milliseconds(10)}; +// std::string s = +// absl::StrJoin(v, ", ", [](std::string* out, absl::Duration dur) { +// absl::StrAppend(out, absl::FormatDuration(dur)); +// }); +// EXPECT_EQ("1s, 10ms", s); // // The following standard formatters are provided within this file: // diff --git a/absl/strings/str_join_test.cc b/absl/strings/str_join_test.cc index 2be6256e..c986e863 100644 --- a/absl/strings/str_join_test.cc +++ b/absl/strings/str_join_test.cc @@ -21,6 +21,7 @@ #include <cstdio> #include <functional> #include <initializer_list> +#include <iterator> #include <map> #include <memory> #include <ostream> @@ -33,6 +34,7 @@ #include "absl/memory/memory.h" #include "absl/strings/str_cat.h" #include "absl/strings/str_split.h" +#include "absl/strings/string_view.h" namespace { @@ -471,4 +473,136 @@ TEST(StrJoin, Tuple) { "-", absl::DereferenceFormatter(TestFormatter()))); } +// A minimal value type for `StrJoin` inputs. +// Used to ensure we do not excessively require more a specific type, such as a +// `string_view`. +// +// Anything that can be `data()` and `size()` is OK. +class TestValue { + public: + TestValue(const char* data, size_t size) : data_(data), size_(size) {} + const char* data() const { return data_; } + size_t size() const { return size_; } + + private: + const char* data_; + size_t size_; +}; + +// A minimal C++20 forward iterator, used to test that we do not impose +// excessive requirements on StrJoin inputs. +// +// The 2 main differences between pre-C++20 LegacyForwardIterator and the +// C++20 ForwardIterator are: +// 1. `operator->` is not required in C++20. +// 2. `operator*` result does not need to be an lvalue (a reference). +// +// The `operator->` requirement was removed on page 17 in: +// http://www.open-std.org/jtc1/sc22/wg21/docs/papers/2018/p1037r0.pdf +// +// See the `[iterator.requirements]` section of the C++ standard. +// +// The value type is a template parameter so that we can test the behaviour +// of `StrJoin` specializations, e.g. the NoFormatter specialization for +// `string_view`. +template <typename ValueT> +class TestIterator { + public: + using iterator_category = std::forward_iterator_tag; + using value_type = ValueT; + using pointer = void; + using reference = const value_type&; + using difference_type = int; + + // `data` must outlive the result. + static TestIterator begin(const std::vector<absl::string_view>& data) { + return TestIterator(&data, 0); + } + + static TestIterator end(const std::vector<absl::string_view>& data) { + return TestIterator(nullptr, data.size()); + } + + bool operator==(const TestIterator& other) const { + return pos_ == other.pos_; + } + bool operator!=(const TestIterator& other) const { + return pos_ != other.pos_; + } + + // This deliberately returns a `prvalue`. + // The requirement to return a reference was removed in C++20. + value_type operator*() const { + return ValueT((*data_)[pos_].data(), (*data_)[pos_].size()); + } + + // `operator->()` is deliberately omitted. + // The requirement to provide it was removed in C++20. + + TestIterator& operator++() { + ++pos_; + return *this; + } + + TestIterator operator++(int) { + TestIterator result = *this; + ++(*this); + return result; + } + + TestIterator& operator--() { + --pos_; + return *this; + } + + TestIterator operator--(int) { + TestIterator result = *this; + --(*this); + return result; + } + + private: + TestIterator(const std::vector<absl::string_view>* data, size_t pos) + : data_(data), pos_(pos) {} + + const std::vector<absl::string_view>* data_; + size_t pos_; +}; + +template <typename ValueT> +class TestIteratorRange { + public: + // `data` must be non-null and must outlive the result. + explicit TestIteratorRange(const std::vector<absl::string_view>& data) + : begin_(TestIterator<ValueT>::begin(data)), + end_(TestIterator<ValueT>::end(data)) {} + + const TestIterator<ValueT>& begin() const { return begin_; } + const TestIterator<ValueT>& end() const { return end_; } + + private: + TestIterator<ValueT> begin_; + TestIterator<ValueT> end_; +}; + +TEST(StrJoin, TestIteratorRequirementsNoFormatter) { + const std::vector<absl::string_view> a = {"a", "b", "c"}; + + // When the value type is string-like (`std::string` or `string_view`), + // the NoFormatter template specialization is used internally. + EXPECT_EQ("a-b-c", + absl::StrJoin(TestIteratorRange<absl::string_view>(a), "-")); +} + +TEST(StrJoin, TestIteratorRequirementsCustomFormatter) { + const std::vector<absl::string_view> a = {"a", "b", "c"}; + EXPECT_EQ("a-b-c", + absl::StrJoin(TestIteratorRange<TestValue>(a), "-", + [](std::string* out, const TestValue& value) { + absl::StrAppend( + out, + absl::string_view(value.data(), value.size())); + })); +} + } // namespace diff --git a/absl/strings/str_split.h b/absl/strings/str_split.h index bfbca422..7bbb68a3 100644 --- a/absl/strings/str_split.h +++ b/absl/strings/str_split.h @@ -461,8 +461,7 @@ using EnableSplitIfString = // first two split strings become the `std::pair` `.first` and `.second` // members, respectively. The remaining split substrings are discarded. If there // are less than two split substrings, the empty string is used for the -// corresponding -// `std::pair` member. +// corresponding `std::pair` member. // // Example: // diff --git a/absl/strings/str_split_test.cc b/absl/strings/str_split_test.cc index 7f7c097f..1b4427b8 100644 --- a/absl/strings/str_split_test.cc +++ b/absl/strings/str_split_test.cc @@ -29,6 +29,8 @@ #include "gtest/gtest.h" #include "absl/base/dynamic_annotations.h" #include "absl/base/macros.h" +#include "absl/container/btree_map.h" +#include "absl/container/btree_set.h" #include "absl/container/flat_hash_map.h" #include "absl/container/node_hash_map.h" #include "absl/strings/numbers.h" @@ -405,6 +407,10 @@ TEST(Splitter, ConversionOperator) { TestConversionOperator<std::set<std::string>>(splitter); TestConversionOperator<std::multiset<absl::string_view>>(splitter); TestConversionOperator<std::multiset<std::string>>(splitter); + TestConversionOperator<absl::btree_set<absl::string_view>>(splitter); + TestConversionOperator<absl::btree_set<std::string>>(splitter); + TestConversionOperator<absl::btree_multiset<absl::string_view>>(splitter); + TestConversionOperator<absl::btree_multiset<std::string>>(splitter); TestConversionOperator<std::unordered_set<std::string>>(splitter); // Tests conversion to map-like objects. @@ -421,6 +427,22 @@ TEST(Splitter, ConversionOperator) { TestMapConversionOperator<std::multimap<std::string, absl::string_view>>( splitter); TestMapConversionOperator<std::multimap<std::string, std::string>>(splitter); + TestMapConversionOperator< + absl::btree_map<absl::string_view, absl::string_view>>(splitter); + TestMapConversionOperator<absl::btree_map<absl::string_view, std::string>>( + splitter); + TestMapConversionOperator<absl::btree_map<std::string, absl::string_view>>( + splitter); + TestMapConversionOperator<absl::btree_map<std::string, std::string>>( + splitter); + TestMapConversionOperator< + absl::btree_multimap<absl::string_view, absl::string_view>>(splitter); + TestMapConversionOperator< + absl::btree_multimap<absl::string_view, std::string>>(splitter); + TestMapConversionOperator< + absl::btree_multimap<std::string, absl::string_view>>(splitter); + TestMapConversionOperator<absl::btree_multimap<std::string, std::string>>( + splitter); TestMapConversionOperator<std::unordered_map<std::string, std::string>>( splitter); TestMapConversionOperator< @@ -921,8 +943,14 @@ TEST(Delimiter, ByLength) { } TEST(Split, WorksWithLargeStrings) { +#if defined(ABSL_HAVE_ADDRESS_SANITIZER) || \ + defined(ABSL_HAVE_MEMORY_SANITIZER) || defined(ABSL_HAVE_THREAD_SANITIZER) + constexpr size_t kSize = (uint32_t{1} << 26) + 1; // 64M + 1 byte +#else + constexpr size_t kSize = (uint32_t{1} << 31) + 1; // 2G + 1 byte +#endif if (sizeof(size_t) > 4) { - std::string s((uint32_t{1} << 31) + 1, 'x'); // 2G + 1 byte + std::string s(kSize, 'x'); s.back() = '-'; std::vector<absl::string_view> v = absl::StrSplit(s, '-'); EXPECT_EQ(2, v.size()); diff --git a/absl/strings/string_view.cc b/absl/strings/string_view.cc index c5f5de93..adce3be9 100644 --- a/absl/strings/string_view.cc +++ b/absl/strings/string_view.cc @@ -78,8 +78,8 @@ std::ostream& operator<<(std::ostream& o, string_view piece) { return o; } -string_view::size_type string_view::find(string_view s, size_type pos) const - noexcept { +string_view::size_type string_view::find(string_view s, + size_type pos) const noexcept { if (empty() || pos > length_) { if (empty() && pos == 0 && s.empty()) return 0; return npos; @@ -98,8 +98,8 @@ string_view::size_type string_view::find(char c, size_type pos) const noexcept { return result != nullptr ? result - ptr_ : npos; } -string_view::size_type string_view::rfind(string_view s, size_type pos) const - noexcept { +string_view::size_type string_view::rfind(string_view s, + size_type pos) const noexcept { if (length_ < s.length_) return npos; if (s.empty()) return std::min(length_, pos); const char* last = ptr_ + std::min(length_ - s.length_, pos) + s.length_; @@ -108,8 +108,8 @@ string_view::size_type string_view::rfind(string_view s, size_type pos) const } // Search range is [0..pos] inclusive. If pos == npos, search everything. -string_view::size_type string_view::rfind(char c, size_type pos) const - noexcept { +string_view::size_type string_view::rfind(char c, + size_type pos) const noexcept { // Note: memrchr() is not available on Windows. if (empty()) return npos; for (size_type i = std::min(pos, length_ - 1);; --i) { @@ -121,9 +121,8 @@ string_view::size_type string_view::rfind(char c, size_type pos) const return npos; } -string_view::size_type string_view::find_first_of(string_view s, - size_type pos) const - noexcept { +string_view::size_type string_view::find_first_of( + string_view s, size_type pos) const noexcept { if (empty() || s.empty()) { return npos; } @@ -138,9 +137,8 @@ string_view::size_type string_view::find_first_of(string_view s, return npos; } -string_view::size_type string_view::find_first_not_of(string_view s, - size_type pos) const - noexcept { +string_view::size_type string_view::find_first_not_of( + string_view s, size_type pos) const noexcept { if (empty()) return npos; // Avoid the cost of LookupTable() for a single-character search. if (s.length_ == 1) return find_first_not_of(s.ptr_[0], pos); @@ -153,9 +151,8 @@ string_view::size_type string_view::find_first_not_of(string_view s, return npos; } -string_view::size_type string_view::find_first_not_of(char c, - size_type pos) const - noexcept { +string_view::size_type string_view::find_first_not_of( + char c, size_type pos) const noexcept { if (empty()) return npos; for (; pos < length_; ++pos) { if (ptr_[pos] != c) { @@ -180,9 +177,8 @@ string_view::size_type string_view::find_last_of(string_view s, return npos; } -string_view::size_type string_view::find_last_not_of(string_view s, - size_type pos) const - noexcept { +string_view::size_type string_view::find_last_not_of( + string_view s, size_type pos) const noexcept { if (empty()) return npos; size_type i = std::min(pos, length_ - 1); if (s.empty()) return i; @@ -198,9 +194,8 @@ string_view::size_type string_view::find_last_not_of(string_view s, return npos; } -string_view::size_type string_view::find_last_not_of(char c, - size_type pos) const - noexcept { +string_view::size_type string_view::find_last_not_of( + char c, size_type pos) const noexcept { if (empty()) return npos; size_type i = std::min(pos, length_ - 1); for (;; --i) { @@ -212,22 +207,11 @@ string_view::size_type string_view::find_last_not_of(char c, return npos; } -// MSVC has non-standard behavior that implicitly creates definitions for static -// const members. These implicit definitions conflict with explicit out-of-class -// member definitions that are required by the C++ standard, resulting in -// LNK1169 "multiply defined" errors at link time. __declspec(selectany) asks -// MSVC to choose only one definition for the symbol it decorates. See details -// at https://msdn.microsoft.com/en-us/library/34h23df8(v=vs.100).aspx -#ifdef _MSC_VER -#define ABSL_STRING_VIEW_SELECTANY __declspec(selectany) -#else -#define ABSL_STRING_VIEW_SELECTANY -#endif -ABSL_STRING_VIEW_SELECTANY +#ifdef ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL constexpr string_view::size_type string_view::npos; -ABSL_STRING_VIEW_SELECTANY constexpr string_view::size_type string_view::kMaxSize; +#endif ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/strings/string_view.h b/absl/strings/string_view.h index 5260b5b7..e3239f57 100644 --- a/absl/strings/string_view.h +++ b/absl/strings/string_view.h @@ -36,6 +36,7 @@ #include <limits> #include <string> +#include "absl/base/attributes.h" #include "absl/base/config.h" #include "absl/base/internal/throw_delegate.h" #include "absl/base/macros.h" @@ -54,13 +55,20 @@ ABSL_NAMESPACE_END #else // ABSL_USES_STD_STRING_VIEW -#if ABSL_HAVE_BUILTIN(__builtin_memcmp) || \ - (defined(__GNUC__) && !defined(__clang__)) +#if ABSL_HAVE_BUILTIN(__builtin_memcmp) || \ + (defined(__GNUC__) && !defined(__clang__)) || \ + (defined(_MSC_VER) && _MSC_VER >= 1928) #define ABSL_INTERNAL_STRING_VIEW_MEMCMP __builtin_memcmp #else // ABSL_HAVE_BUILTIN(__builtin_memcmp) #define ABSL_INTERNAL_STRING_VIEW_MEMCMP memcmp #endif // ABSL_HAVE_BUILTIN(__builtin_memcmp) +#if defined(__cplusplus) && __cplusplus >= 201402L +#define ABSL_INTERNAL_STRING_VIEW_CXX14_CONSTEXPR constexpr +#else +#define ABSL_INTERNAL_STRING_VIEW_CXX14_CONSTEXPR +#endif + namespace absl { ABSL_NAMESPACE_BEGIN @@ -180,18 +188,20 @@ class string_view { template <typename Allocator> string_view( // NOLINT(runtime/explicit) - const std::basic_string<char, std::char_traits<char>, Allocator>& - str) noexcept + const std::basic_string<char, std::char_traits<char>, Allocator>& str + ABSL_ATTRIBUTE_LIFETIME_BOUND) noexcept // This is implemented in terms of `string_view(p, n)` so `str.size()` // doesn't need to be reevaluated after `ptr_` is set. - : string_view(str.data(), str.size()) {} + // The length check is also skipped since it is unnecessary and causes + // code bloat. + : string_view(str.data(), str.size(), SkipCheckLengthTag{}) {} // Implicit constructor of a `string_view` from NUL-terminated `str`. When // accepting possibly null strings, use `absl::NullSafeStringView(str)` // instead (see below). + // The length check is skipped since it is unnecessary and causes code bloat. constexpr string_view(const char* str) // NOLINT(runtime/explicit) - : ptr_(str), - length_(str ? CheckLengthInternal(StrlenInternal(str)) : 0) {} + : ptr_(str), length_(str ? StrlenInternal(str) : 0) {} // Implicit constructor of a `string_view` from a `const char*` and length. constexpr string_view(const char* data, size_type len) @@ -264,9 +274,7 @@ class string_view { // string_view::size() // // Returns the number of characters in the `string_view`. - constexpr size_type size() const noexcept { - return length_; - } + constexpr size_type size() const noexcept { return length_; } // string_view::length() // @@ -333,7 +341,7 @@ class string_view { // // Removes the first `n` characters from the `string_view`. Note that the // underlying string is not changed, only the view. - void remove_prefix(size_type n) { + ABSL_INTERNAL_STRING_VIEW_CXX14_CONSTEXPR void remove_prefix(size_type n) { ABSL_HARDENING_ASSERT(n <= length_); ptr_ += n; length_ -= n; @@ -343,7 +351,7 @@ class string_view { // // Removes the last `n` characters from the `string_view`. Note that the // underlying string is not changed, only the view. - void remove_suffix(size_type n) { + ABSL_INTERNAL_STRING_VIEW_CXX14_CONSTEXPR void remove_suffix(size_type n) { ABSL_HARDENING_ASSERT(n <= length_); length_ -= n; } @@ -351,7 +359,7 @@ class string_view { // string_view::swap() // // Swaps this `string_view` with another `string_view`. - void swap(string_view& s) noexcept { + ABSL_INTERNAL_STRING_VIEW_CXX14_CONSTEXPR void swap(string_view& s) noexcept { auto t = *this; *this = s; s = t; @@ -388,7 +396,7 @@ class string_view { // `n`) as another string_view. This function throws `std::out_of_bounds` if // `pos > size`. // Use absl::ClippedSubstr if you need a truncating substr operation. - constexpr string_view substr(size_type pos, size_type n = npos) const { + constexpr string_view substr(size_type pos = 0, size_type n = npos) const { return ABSL_PREDICT_FALSE(pos > length_) ? (base_internal::ThrowStdOutOfRange( "absl::string_view::substr"), @@ -398,12 +406,10 @@ class string_view { // string_view::compare() // - // Performs a lexicographical comparison between the `string_view` and - // another `absl::string_view`, returning -1 if `this` is less than, 0 if - // `this` is equal to, and 1 if `this` is greater than the passed string - // view. Note that in the case of data equality, a further comparison is made - // on the respective sizes of the two `string_view`s to determine which is - // smaller, equal, or greater. + // Performs a lexicographical comparison between this `string_view` and + // another `string_view` `x`, returning a negative value if `*this` is less + // than `x`, 0 if `*this` is equal to `x`, and a positive value if `*this` + // is greater than `x`. constexpr int compare(string_view x) const noexcept { return CompareImpl(length_, x.length_, Min(length_, x.length_) == 0 @@ -414,31 +420,31 @@ class string_view { // Overload of `string_view::compare()` for comparing a substring of the // 'string_view` and another `absl::string_view`. - int compare(size_type pos1, size_type count1, string_view v) const { + constexpr int compare(size_type pos1, size_type count1, string_view v) const { return substr(pos1, count1).compare(v); } // Overload of `string_view::compare()` for comparing a substring of the // `string_view` and a substring of another `absl::string_view`. - int compare(size_type pos1, size_type count1, string_view v, size_type pos2, - size_type count2) const { + constexpr int compare(size_type pos1, size_type count1, string_view v, + size_type pos2, size_type count2) const { return substr(pos1, count1).compare(v.substr(pos2, count2)); } // Overload of `string_view::compare()` for comparing a `string_view` and a - // a different C-style string `s`. - int compare(const char* s) const { return compare(string_view(s)); } + // a different C-style string `s`. + constexpr int compare(const char* s) const { return compare(string_view(s)); } // Overload of `string_view::compare()` for comparing a substring of the // `string_view` and a different string C-style string `s`. - int compare(size_type pos1, size_type count1, const char* s) const { + constexpr int compare(size_type pos1, size_type count1, const char* s) const { return substr(pos1, count1).compare(string_view(s)); } // Overload of `string_view::compare()` for comparing a substring of the // `string_view` and a substring of a different C-style string `s`. - int compare(size_type pos1, size_type count1, const char* s, - size_type count2) const { + constexpr int compare(size_type pos1, size_type count1, const char* s, + size_type count2) const { return substr(pos1, count1).compare(string_view(s, count2)); } @@ -455,48 +461,92 @@ class string_view { // within the `string_view`. size_type find(char c, size_type pos = 0) const noexcept; + // Overload of `string_view::find()` for finding a substring of a different + // C-style string `s` within the `string_view`. + size_type find(const char* s, size_type pos, size_type count) const { + return find(string_view(s, count), pos); + } + + // Overload of `string_view::find()` for finding a different C-style string + // `s` within the `string_view`. + size_type find(const char* s, size_type pos = 0) const { + return find(string_view(s), pos); + } + // string_view::rfind() // // Finds the last occurrence of a substring `s` within the `string_view`, // returning the position of the first character's match, or `npos` if no // match was found. - size_type rfind(string_view s, size_type pos = npos) const - noexcept; + size_type rfind(string_view s, size_type pos = npos) const noexcept; // Overload of `string_view::rfind()` for finding the last given character `c` // within the `string_view`. size_type rfind(char c, size_type pos = npos) const noexcept; + // Overload of `string_view::rfind()` for finding a substring of a different + // C-style string `s` within the `string_view`. + size_type rfind(const char* s, size_type pos, size_type count) const { + return rfind(string_view(s, count), pos); + } + + // Overload of `string_view::rfind()` for finding a different C-style string + // `s` within the `string_view`. + size_type rfind(const char* s, size_type pos = npos) const { + return rfind(string_view(s), pos); + } + // string_view::find_first_of() // // Finds the first occurrence of any of the characters in `s` within the // `string_view`, returning the start position of the match, or `npos` if no // match was found. - size_type find_first_of(string_view s, size_type pos = 0) const - noexcept; + size_type find_first_of(string_view s, size_type pos = 0) const noexcept; // Overload of `string_view::find_first_of()` for finding a character `c` // within the `string_view`. - size_type find_first_of(char c, size_type pos = 0) const - noexcept { + size_type find_first_of(char c, size_type pos = 0) const noexcept { return find(c, pos); } + // Overload of `string_view::find_first_of()` for finding a substring of a + // different C-style string `s` within the `string_view`. + size_type find_first_of(const char* s, size_type pos, + size_type count) const { + return find_first_of(string_view(s, count), pos); + } + + // Overload of `string_view::find_first_of()` for finding a different C-style + // string `s` within the `string_view`. + size_type find_first_of(const char* s, size_type pos = 0) const { + return find_first_of(string_view(s), pos); + } + // string_view::find_last_of() // // Finds the last occurrence of any of the characters in `s` within the // `string_view`, returning the start position of the match, or `npos` if no // match was found. - size_type find_last_of(string_view s, size_type pos = npos) const - noexcept; + size_type find_last_of(string_view s, size_type pos = npos) const noexcept; // Overload of `string_view::find_last_of()` for finding a character `c` // within the `string_view`. - size_type find_last_of(char c, size_type pos = npos) const - noexcept { + size_type find_last_of(char c, size_type pos = npos) const noexcept { return rfind(c, pos); } + // Overload of `string_view::find_last_of()` for finding a substring of a + // different C-style string `s` within the `string_view`. + size_type find_last_of(const char* s, size_type pos, size_type count) const { + return find_last_of(string_view(s, count), pos); + } + + // Overload of `string_view::find_last_of()` for finding a different C-style + // string `s` within the `string_view`. + size_type find_last_of(const char* s, size_type pos = npos) const { + return find_last_of(string_view(s), pos); + } + // string_view::find_first_not_of() // // Finds the first occurrence of any of the characters not in `s` within the @@ -508,20 +558,51 @@ class string_view { // that is not `c` within the `string_view`. size_type find_first_not_of(char c, size_type pos = 0) const noexcept; + // Overload of `string_view::find_first_not_of()` for finding a substring of a + // different C-style string `s` within the `string_view`. + size_type find_first_not_of(const char* s, size_type pos, + size_type count) const { + return find_first_not_of(string_view(s, count), pos); + } + + // Overload of `string_view::find_first_not_of()` for finding a different + // C-style string `s` within the `string_view`. + size_type find_first_not_of(const char* s, size_type pos = 0) const { + return find_first_not_of(string_view(s), pos); + } + // string_view::find_last_not_of() // // Finds the last occurrence of any of the characters not in `s` within the // `string_view`, returning the start position of the last non-match, or // `npos` if no non-match was found. size_type find_last_not_of(string_view s, - size_type pos = npos) const noexcept; + size_type pos = npos) const noexcept; // Overload of `string_view::find_last_not_of()` for finding a character // that is not `c` within the `string_view`. - size_type find_last_not_of(char c, size_type pos = npos) const - noexcept; + size_type find_last_not_of(char c, size_type pos = npos) const noexcept; + + // Overload of `string_view::find_last_not_of()` for finding a substring of a + // different C-style string `s` within the `string_view`. + size_type find_last_not_of(const char* s, size_type pos, + size_type count) const { + return find_last_not_of(string_view(s, count), pos); + } + + // Overload of `string_view::find_last_not_of()` for finding a different + // C-style string `s` within the `string_view`. + size_type find_last_not_of(const char* s, size_type pos = npos) const { + return find_last_not_of(string_view(s), pos); + } private: + // The constructor from std::string delegates to this constructor. + // See the comment on that constructor for the rationale. + struct SkipCheckLengthTag {}; + string_view(const char* data, size_type len, SkipCheckLengthTag) noexcept + : ptr_(data), length_(len) {} + static constexpr size_type kMaxSize = (std::numeric_limits<difference_type>::max)(); @@ -597,6 +678,7 @@ std::ostream& operator<<(std::ostream& o, string_view piece); ABSL_NAMESPACE_END } // namespace absl +#undef ABSL_INTERNAL_STRING_VIEW_CXX14_CONSTEXPR #undef ABSL_INTERNAL_STRING_VIEW_MEMCMP #endif // ABSL_USES_STD_STRING_VIEW diff --git a/absl/strings/string_view_test.cc b/absl/strings/string_view_test.cc index 643af8f8..9d5463a1 100644 --- a/absl/strings/string_view_test.cc +++ b/absl/strings/string_view_test.cc @@ -449,6 +449,24 @@ TEST(StringViewTest, STL2) { EXPECT_EQ(d.find('x', 4), absl::string_view::npos); EXPECT_EQ(e.find('x', 7), absl::string_view::npos); + EXPECT_EQ(a.find(b.data(), 1, 0), 1); + EXPECT_EQ(a.find(c.data(), 9, 0), 9); + EXPECT_EQ(a.find(c.data(), absl::string_view::npos, 0), + absl::string_view::npos); + EXPECT_EQ(b.find(c.data(), absl::string_view::npos, 0), + absl::string_view::npos); + // empty string nonsense + EXPECT_EQ(d.find(b.data(), 4, 0), absl::string_view::npos); + EXPECT_EQ(e.find(b.data(), 7, 0), absl::string_view::npos); + + EXPECT_EQ(a.find(b.data(), 1), absl::string_view::npos); + EXPECT_EQ(a.find(c.data(), 9), 23); + EXPECT_EQ(a.find(c.data(), absl::string_view::npos), absl::string_view::npos); + EXPECT_EQ(b.find(c.data(), absl::string_view::npos), absl::string_view::npos); + // empty string nonsense + EXPECT_EQ(d.find(b.data(), 4), absl::string_view::npos); + EXPECT_EQ(e.find(b.data(), 7), absl::string_view::npos); + EXPECT_EQ(a.rfind(b), 0); EXPECT_EQ(a.rfind(b, 1), 0); EXPECT_EQ(a.rfind(c), 23); @@ -490,6 +508,14 @@ TEST(StringViewTest, STL2) { EXPECT_EQ(e.rfind('o'), absl::string_view::npos); EXPECT_EQ(d.rfind('o', 4), absl::string_view::npos); EXPECT_EQ(e.rfind('o', 7), absl::string_view::npos); + + EXPECT_EQ(a.rfind(b.data(), 1, 0), 1); + EXPECT_EQ(a.rfind(c.data(), 22, 0), 22); + EXPECT_EQ(a.rfind(c.data(), 1, 0), 1); + EXPECT_EQ(a.rfind(c.data(), 0, 0), 0); + EXPECT_EQ(b.rfind(c.data(), 0, 0), 0); + EXPECT_EQ(d.rfind(b.data(), 4, 0), 0); + EXPECT_EQ(e.rfind(b.data(), 7, 0), 0); } // Continued from STL2 @@ -678,6 +704,7 @@ TEST(StringViewTest, STL2Substr) { EXPECT_EQ(a.substr(23, 3), c); EXPECT_EQ(a.substr(23, 99), c); EXPECT_EQ(a.substr(0), a); + EXPECT_EQ(a.substr(), a); EXPECT_EQ(a.substr(3, 2), "de"); // empty string nonsense EXPECT_EQ(d.substr(0, 99), e); @@ -1087,7 +1114,24 @@ TEST(StringViewTest, ConstexprCompiles) { EXPECT_EQ(sp_npos, -1); } -TEST(StringViewTest, ConstexprSubstr) { +constexpr char ConstexprMethodsHelper() { +#if defined(__cplusplus) && __cplusplus >= 201402L + absl::string_view str("123", 3); + str.remove_prefix(1); + str.remove_suffix(1); + absl::string_view bar; + str.swap(bar); + return bar.front(); +#else + return '2'; +#endif +} + +TEST(StringViewTest, ConstexprMethods) { + // remove_prefix, remove_suffix, swap + static_assert(ConstexprMethodsHelper() == '2', ""); + + // substr constexpr absl::string_view foobar("foobar", 6); constexpr absl::string_view foo = foobar.substr(0, 3); constexpr absl::string_view bar = foobar.substr(3); @@ -1145,7 +1189,7 @@ TEST(ComparisonOpsTest, StringCompareNotAmbiguous) { EXPECT_LT("hello", std::string("world")); } -TEST(ComparisonOpsTest, HeterogenousStringViewEquals) { +TEST(ComparisonOpsTest, HeterogeneousStringViewEquals) { EXPECT_EQ(absl::string_view("hello"), std::string("hello")); EXPECT_EQ("hello", absl::string_view("hello")); } diff --git a/absl/strings/strip.h b/absl/strings/strip.h index 111872ca..341e66fc 100644 --- a/absl/strings/strip.h +++ b/absl/strings/strip.h @@ -34,8 +34,9 @@ ABSL_NAMESPACE_BEGIN // ConsumePrefix() // -// Strips the `expected` prefix from the start of the given string, returning -// `true` if the strip operation succeeded or false otherwise. +// Strips the `expected` prefix, if found, from the start of `str`. +// If the operation succeeded, `true` is returned. If not, `false` +// is returned and `str` is not modified. // // Example: // @@ -49,8 +50,9 @@ inline bool ConsumePrefix(absl::string_view* str, absl::string_view expected) { } // ConsumeSuffix() // -// Strips the `expected` suffix from the end of the given string, returning -// `true` if the strip operation succeeded or false otherwise. +// Strips the `expected` suffix, if found, from the end of `str`. +// If the operation succeeded, `true` is returned. If not, `false` +// is returned and `str` is not modified. // // Example: // @@ -65,7 +67,7 @@ inline bool ConsumeSuffix(absl::string_view* str, absl::string_view expected) { // StripPrefix() // -// Returns a view into the input string 'str' with the given 'prefix' removed, +// Returns a view into the input string `str` with the given `prefix` removed, // but leaving the original string intact. If the prefix does not match at the // start of the string, returns the original string instead. ABSL_MUST_USE_RESULT inline absl::string_view StripPrefix( @@ -76,7 +78,7 @@ ABSL_MUST_USE_RESULT inline absl::string_view StripPrefix( // StripSuffix() // -// Returns a view into the input string 'str' with the given 'suffix' removed, +// Returns a view into the input string `str` with the given `suffix` removed, // but leaving the original string intact. If the suffix does not match at the // end of the string, returns the original string instead. ABSL_MUST_USE_RESULT inline absl::string_view StripSuffix( diff --git a/absl/strings/substitute.cc b/absl/strings/substitute.cc index 1f3c7409..8980b198 100644 --- a/absl/strings/substitute.cc +++ b/absl/strings/substitute.cc @@ -75,7 +75,8 @@ void SubstituteAndAppendArray(std::string* output, absl::string_view format, // Build the string. size_t original_size = output->size(); - strings_internal::STLStringResizeUninitialized(output, original_size + size); + strings_internal::STLStringResizeUninitializedAmortized(output, + original_size + size); char* target = &(*output)[original_size]; for (size_t i = 0; i < format.size(); i++) { if (format[i] == '$') { diff --git a/absl/strings/substitute.h b/absl/strings/substitute.h index c6da4dc6..6d2b08ab 100644 --- a/absl/strings/substitute.h +++ b/absl/strings/substitute.h @@ -159,8 +159,8 @@ class Arg { Arg(Hex hex); // NOLINT(runtime/explicit) Arg(Dec dec); // NOLINT(runtime/explicit) - // vector<bool>::reference and const_reference require special help to - // convert to `AlphaNum` because it requires two user defined conversions. + // vector<bool>::reference and const_reference require special help to convert + // to `Arg` because it requires two user defined conversions. template <typename T, absl::enable_if_t< std::is_class<T>::value && @@ -174,6 +174,14 @@ class Arg { // "0x<hex value>". However, in the case of `nullptr`, "NULL" is printed. Arg(const void* value); // NOLINT(runtime/explicit) + // Normal enums are already handled by the integer formatters. + // This overload matches only scoped enums. + template <typename T, + typename = typename std::enable_if< + std::is_enum<T>{} && !std::is_convertible<T, int>{}>::type> + Arg(T value) // NOLINT(google-explicit-constructor) + : Arg(static_cast<typename std::underlying_type<T>::type>(value)) {} + Arg(const Arg&) = delete; Arg& operator=(const Arg&) = delete; @@ -361,43 +369,49 @@ inline void SubstituteAndAppend( // This body of functions catches cases where the number of placeholders // doesn't match the number of data arguments. void SubstituteAndAppend(std::string* output, const char* format) - ABSL_BAD_CALL_IF(substitute_internal::PlaceholderBitmask(format) != 0, - "There were no substitution arguments " - "but this format string has a $[0-9] in it"); + ABSL_BAD_CALL_IF( + substitute_internal::PlaceholderBitmask(format) != 0, + "There were no substitution arguments " + "but this format string either has a $[0-9] in it or contains " + "an unescaped $ character (use $$ instead)"); void SubstituteAndAppend(std::string* output, const char* format, const substitute_internal::Arg& a0) ABSL_BAD_CALL_IF(substitute_internal::PlaceholderBitmask(format) != 1, "There was 1 substitution argument given, but " - "this format string is either missing its $0, or " - "contains one of $1-$9"); + "this format string is missing its $0, contains " + "one of $1-$9, or contains an unescaped $ character (use " + "$$ instead)"); void SubstituteAndAppend(std::string* output, const char* format, const substitute_internal::Arg& a0, const substitute_internal::Arg& a1) - ABSL_BAD_CALL_IF(substitute_internal::PlaceholderBitmask(format) != 3, - "There were 2 substitution arguments given, but " - "this format string is either missing its $0/$1, or " - "contains one of $2-$9"); + ABSL_BAD_CALL_IF( + substitute_internal::PlaceholderBitmask(format) != 3, + "There were 2 substitution arguments given, but this format string is " + "missing its $0/$1, contains one of $2-$9, or contains an " + "unescaped $ character (use $$ instead)"); void SubstituteAndAppend(std::string* output, const char* format, const substitute_internal::Arg& a0, const substitute_internal::Arg& a1, const substitute_internal::Arg& a2) - ABSL_BAD_CALL_IF(substitute_internal::PlaceholderBitmask(format) != 7, - "There were 3 substitution arguments given, but " - "this format string is either missing its $0/$1/$2, or " - "contains one of $3-$9"); + ABSL_BAD_CALL_IF( + substitute_internal::PlaceholderBitmask(format) != 7, + "There were 3 substitution arguments given, but " + "this format string is missing its $0/$1/$2, contains one of " + "$3-$9, or contains an unescaped $ character (use $$ instead)"); void SubstituteAndAppend(std::string* output, const char* format, const substitute_internal::Arg& a0, const substitute_internal::Arg& a1, const substitute_internal::Arg& a2, const substitute_internal::Arg& a3) - ABSL_BAD_CALL_IF(substitute_internal::PlaceholderBitmask(format) != 15, - "There were 4 substitution arguments given, but " - "this format string is either missing its $0-$3, or " - "contains one of $4-$9"); + ABSL_BAD_CALL_IF( + substitute_internal::PlaceholderBitmask(format) != 15, + "There were 4 substitution arguments given, but " + "this format string is missing its $0-$3, contains one of " + "$4-$9, or contains an unescaped $ character (use $$ instead)"); void SubstituteAndAppend(std::string* output, const char* format, const substitute_internal::Arg& a0, @@ -405,10 +419,11 @@ void SubstituteAndAppend(std::string* output, const char* format, const substitute_internal::Arg& a2, const substitute_internal::Arg& a3, const substitute_internal::Arg& a4) - ABSL_BAD_CALL_IF(substitute_internal::PlaceholderBitmask(format) != 31, - "There were 5 substitution arguments given, but " - "this format string is either missing its $0-$4, or " - "contains one of $5-$9"); + ABSL_BAD_CALL_IF( + substitute_internal::PlaceholderBitmask(format) != 31, + "There were 5 substitution arguments given, but " + "this format string is missing its $0-$4, contains one of " + "$5-$9, or contains an unescaped $ character (use $$ instead)"); void SubstituteAndAppend(std::string* output, const char* format, const substitute_internal::Arg& a0, @@ -417,20 +432,22 @@ void SubstituteAndAppend(std::string* output, const char* format, const substitute_internal::Arg& a3, const substitute_internal::Arg& a4, const substitute_internal::Arg& a5) - ABSL_BAD_CALL_IF(substitute_internal::PlaceholderBitmask(format) != 63, - "There were 6 substitution arguments given, but " - "this format string is either missing its $0-$5, or " - "contains one of $6-$9"); + ABSL_BAD_CALL_IF( + substitute_internal::PlaceholderBitmask(format) != 63, + "There were 6 substitution arguments given, but " + "this format string is missing its $0-$5, contains one of " + "$6-$9, or contains an unescaped $ character (use $$ instead)"); void SubstituteAndAppend( std::string* output, const char* format, const substitute_internal::Arg& a0, const substitute_internal::Arg& a1, const substitute_internal::Arg& a2, const substitute_internal::Arg& a3, const substitute_internal::Arg& a4, const substitute_internal::Arg& a5, const substitute_internal::Arg& a6) - ABSL_BAD_CALL_IF(substitute_internal::PlaceholderBitmask(format) != 127, - "There were 7 substitution arguments given, but " - "this format string is either missing its $0-$6, or " - "contains one of $7-$9"); + ABSL_BAD_CALL_IF( + substitute_internal::PlaceholderBitmask(format) != 127, + "There were 7 substitution arguments given, but " + "this format string is missing its $0-$6, contains one of " + "$7-$9, or contains an unescaped $ character (use $$ instead)"); void SubstituteAndAppend( std::string* output, const char* format, const substitute_internal::Arg& a0, @@ -438,10 +455,11 @@ void SubstituteAndAppend( const substitute_internal::Arg& a3, const substitute_internal::Arg& a4, const substitute_internal::Arg& a5, const substitute_internal::Arg& a6, const substitute_internal::Arg& a7) - ABSL_BAD_CALL_IF(substitute_internal::PlaceholderBitmask(format) != 255, - "There were 8 substitution arguments given, but " - "this format string is either missing its $0-$7, or " - "contains one of $8-$9"); + ABSL_BAD_CALL_IF( + substitute_internal::PlaceholderBitmask(format) != 255, + "There were 8 substitution arguments given, but " + "this format string is missing its $0-$7, contains one of " + "$8-$9, or contains an unescaped $ character (use $$ instead)"); void SubstituteAndAppend( std::string* output, const char* format, const substitute_internal::Arg& a0, @@ -452,7 +470,8 @@ void SubstituteAndAppend( ABSL_BAD_CALL_IF( substitute_internal::PlaceholderBitmask(format) != 511, "There were 9 substitution arguments given, but " - "this format string is either missing its $0-$8, or contains a $9"); + "this format string is missing its $0-$8, contains a $9, or " + "contains an unescaped $ character (use $$ instead)"); void SubstituteAndAppend( std::string* output, const char* format, const substitute_internal::Arg& a0, @@ -461,9 +480,11 @@ void SubstituteAndAppend( const substitute_internal::Arg& a5, const substitute_internal::Arg& a6, const substitute_internal::Arg& a7, const substitute_internal::Arg& a8, const substitute_internal::Arg& a9) - ABSL_BAD_CALL_IF(substitute_internal::PlaceholderBitmask(format) != 1023, - "There were 10 substitution arguments given, but this " - "format string doesn't contain all of $0 through $9"); + ABSL_BAD_CALL_IF( + substitute_internal::PlaceholderBitmask(format) != 1023, + "There were 10 substitution arguments given, but this " + "format string either doesn't contain all of $0 through $9 or " + "contains an unescaped $ character (use $$ instead)"); #endif // ABSL_BAD_CALL_IF // Substitute() @@ -589,47 +610,53 @@ ABSL_MUST_USE_RESULT inline std::string Substitute( std::string Substitute(const char* format) ABSL_BAD_CALL_IF(substitute_internal::PlaceholderBitmask(format) != 0, "There were no substitution arguments " - "but this format string has a $[0-9] in it"); + "but this format string either has a $[0-9] in it or " + "contains an unescaped $ character (use $$ instead)"); std::string Substitute(const char* format, const substitute_internal::Arg& a0) - ABSL_BAD_CALL_IF(substitute_internal::PlaceholderBitmask(format) != 1, - "There was 1 substitution argument given, but " - "this format string is either missing its $0, or " - "contains one of $1-$9"); + ABSL_BAD_CALL_IF( + substitute_internal::PlaceholderBitmask(format) != 1, + "There was 1 substitution argument given, but " + "this format string is missing its $0, contains one of $1-$9, " + "or contains an unescaped $ character (use $$ instead)"); std::string Substitute(const char* format, const substitute_internal::Arg& a0, const substitute_internal::Arg& a1) - ABSL_BAD_CALL_IF(substitute_internal::PlaceholderBitmask(format) != 3, - "There were 2 substitution arguments given, but " - "this format string is either missing its $0/$1, or " - "contains one of $2-$9"); + ABSL_BAD_CALL_IF( + substitute_internal::PlaceholderBitmask(format) != 3, + "There were 2 substitution arguments given, but " + "this format string is missing its $0/$1, contains one of " + "$2-$9, or contains an unescaped $ character (use $$ instead)"); std::string Substitute(const char* format, const substitute_internal::Arg& a0, const substitute_internal::Arg& a1, const substitute_internal::Arg& a2) - ABSL_BAD_CALL_IF(substitute_internal::PlaceholderBitmask(format) != 7, - "There were 3 substitution arguments given, but " - "this format string is either missing its $0/$1/$2, or " - "contains one of $3-$9"); + ABSL_BAD_CALL_IF( + substitute_internal::PlaceholderBitmask(format) != 7, + "There were 3 substitution arguments given, but " + "this format string is missing its $0/$1/$2, contains one of " + "$3-$9, or contains an unescaped $ character (use $$ instead)"); std::string Substitute(const char* format, const substitute_internal::Arg& a0, const substitute_internal::Arg& a1, const substitute_internal::Arg& a2, const substitute_internal::Arg& a3) - ABSL_BAD_CALL_IF(substitute_internal::PlaceholderBitmask(format) != 15, - "There were 4 substitution arguments given, but " - "this format string is either missing its $0-$3, or " - "contains one of $4-$9"); + ABSL_BAD_CALL_IF( + substitute_internal::PlaceholderBitmask(format) != 15, + "There were 4 substitution arguments given, but " + "this format string is missing its $0-$3, contains one of " + "$4-$9, or contains an unescaped $ character (use $$ instead)"); std::string Substitute(const char* format, const substitute_internal::Arg& a0, const substitute_internal::Arg& a1, const substitute_internal::Arg& a2, const substitute_internal::Arg& a3, const substitute_internal::Arg& a4) - ABSL_BAD_CALL_IF(substitute_internal::PlaceholderBitmask(format) != 31, - "There were 5 substitution arguments given, but " - "this format string is either missing its $0-$4, or " - "contains one of $5-$9"); + ABSL_BAD_CALL_IF( + substitute_internal::PlaceholderBitmask(format) != 31, + "There were 5 substitution arguments given, but " + "this format string is missing its $0-$4, contains one of " + "$5-$9, or contains an unescaped $ character (use $$ instead)"); std::string Substitute(const char* format, const substitute_internal::Arg& a0, const substitute_internal::Arg& a1, @@ -637,10 +664,11 @@ std::string Substitute(const char* format, const substitute_internal::Arg& a0, const substitute_internal::Arg& a3, const substitute_internal::Arg& a4, const substitute_internal::Arg& a5) - ABSL_BAD_CALL_IF(substitute_internal::PlaceholderBitmask(format) != 63, - "There were 6 substitution arguments given, but " - "this format string is either missing its $0-$5, or " - "contains one of $6-$9"); + ABSL_BAD_CALL_IF( + substitute_internal::PlaceholderBitmask(format) != 63, + "There were 6 substitution arguments given, but " + "this format string is missing its $0-$5, contains one of " + "$6-$9, or contains an unescaped $ character (use $$ instead)"); std::string Substitute(const char* format, const substitute_internal::Arg& a0, const substitute_internal::Arg& a1, @@ -649,10 +677,11 @@ std::string Substitute(const char* format, const substitute_internal::Arg& a0, const substitute_internal::Arg& a4, const substitute_internal::Arg& a5, const substitute_internal::Arg& a6) - ABSL_BAD_CALL_IF(substitute_internal::PlaceholderBitmask(format) != 127, - "There were 7 substitution arguments given, but " - "this format string is either missing its $0-$6, or " - "contains one of $7-$9"); + ABSL_BAD_CALL_IF( + substitute_internal::PlaceholderBitmask(format) != 127, + "There were 7 substitution arguments given, but " + "this format string is missing its $0-$6, contains one of " + "$7-$9, or contains an unescaped $ character (use $$ instead)"); std::string Substitute(const char* format, const substitute_internal::Arg& a0, const substitute_internal::Arg& a1, @@ -662,10 +691,11 @@ std::string Substitute(const char* format, const substitute_internal::Arg& a0, const substitute_internal::Arg& a5, const substitute_internal::Arg& a6, const substitute_internal::Arg& a7) - ABSL_BAD_CALL_IF(substitute_internal::PlaceholderBitmask(format) != 255, - "There were 8 substitution arguments given, but " - "this format string is either missing its $0-$7, or " - "contains one of $8-$9"); + ABSL_BAD_CALL_IF( + substitute_internal::PlaceholderBitmask(format) != 255, + "There were 8 substitution arguments given, but " + "this format string is missing its $0-$7, contains one of " + "$8-$9, or contains an unescaped $ character (use $$ instead)"); std::string Substitute( const char* format, const substitute_internal::Arg& a0, @@ -676,7 +706,8 @@ std::string Substitute( ABSL_BAD_CALL_IF( substitute_internal::PlaceholderBitmask(format) != 511, "There were 9 substitution arguments given, but " - "this format string is either missing its $0-$8, or contains a $9"); + "this format string is missing its $0-$8, contains a $9, or " + "contains an unescaped $ character (use $$ instead)"); std::string Substitute( const char* format, const substitute_internal::Arg& a0, @@ -685,9 +716,11 @@ std::string Substitute( const substitute_internal::Arg& a5, const substitute_internal::Arg& a6, const substitute_internal::Arg& a7, const substitute_internal::Arg& a8, const substitute_internal::Arg& a9) - ABSL_BAD_CALL_IF(substitute_internal::PlaceholderBitmask(format) != 1023, - "There were 10 substitution arguments given, but this " - "format string doesn't contain all of $0 through $9"); + ABSL_BAD_CALL_IF( + substitute_internal::PlaceholderBitmask(format) != 1023, + "There were 10 substitution arguments given, but this " + "format string either doesn't contain all of $0 through $9 or " + "contains an unescaped $ character (use $$ instead)"); #endif // ABSL_BAD_CALL_IF ABSL_NAMESPACE_END diff --git a/absl/strings/substitute_test.cc b/absl/strings/substitute_test.cc index 442c9215..9e6b9403 100644 --- a/absl/strings/substitute_test.cc +++ b/absl/strings/substitute_test.cc @@ -184,6 +184,54 @@ TEST(SubstituteTest, VectorBoolRef) { EXPECT_EQ("Logic be like: true false true false", str); } +TEST(SubstituteTest, Enums) { + enum UnscopedEnum { kEnum0 = 0, kEnum1 = 1 }; + EXPECT_EQ("0 1", absl::Substitute("$0 $1", UnscopedEnum::kEnum0, + UnscopedEnum::kEnum1)); + + enum class ScopedEnum { kEnum0 = 0, kEnum1 = 1 }; + EXPECT_EQ("0 1", + absl::Substitute("$0 $1", ScopedEnum::kEnum0, ScopedEnum::kEnum1)); + + enum class ScopedEnumInt32 : int32_t { kEnum0 = 989, kEnum1 = INT32_MIN }; + EXPECT_EQ("989 -2147483648", + absl::Substitute("$0 $1", ScopedEnumInt32::kEnum0, + ScopedEnumInt32::kEnum1)); + + enum class ScopedEnumUInt32 : uint32_t { kEnum0 = 1, kEnum1 = UINT32_MAX }; + EXPECT_EQ("1 4294967295", absl::Substitute("$0 $1", ScopedEnumUInt32::kEnum0, + ScopedEnumUInt32::kEnum1)); + + enum class ScopedEnumInt64 : int64_t { kEnum0 = -1, kEnum1 = 42949672950 }; + EXPECT_EQ("-1 42949672950", absl::Substitute("$0 $1", ScopedEnumInt64::kEnum0, + ScopedEnumInt64::kEnum1)); + + enum class ScopedEnumUInt64 : uint64_t { kEnum0 = 1, kEnum1 = 42949672950 }; + EXPECT_EQ("1 42949672950", absl::Substitute("$0 $1", ScopedEnumUInt64::kEnum0, + ScopedEnumUInt64::kEnum1)); + + enum class ScopedEnumChar : signed char { kEnum0 = -1, kEnum1 = 1 }; + EXPECT_EQ("-1 1", absl::Substitute("$0 $1", ScopedEnumChar::kEnum0, + ScopedEnumChar::kEnum1)); + + enum class ScopedEnumUChar : unsigned char { + kEnum0 = 0, + kEnum1 = 1, + kEnumMax = 255 + }; + EXPECT_EQ("0 1 255", absl::Substitute("$0 $1 $2", ScopedEnumUChar::kEnum0, + ScopedEnumUChar::kEnum1, + ScopedEnumUChar::kEnumMax)); + + enum class ScopedEnumInt16 : int16_t { kEnum0 = -100, kEnum1 = 10000 }; + EXPECT_EQ("-100 10000", absl::Substitute("$0 $1", ScopedEnumInt16::kEnum0, + ScopedEnumInt16::kEnum1)); + + enum class ScopedEnumUInt16 : uint16_t { kEnum0 = 0, kEnum1 = 10000 }; + EXPECT_EQ("0 10000", absl::Substitute("$0 $1", ScopedEnumUInt16::kEnum0, + ScopedEnumUInt16::kEnum1)); +} + #ifdef GTEST_HAS_DEATH_TEST TEST(SubstituteDeathTest, SubstituteDeath) { diff --git a/absl/synchronization/BUILD.bazel b/absl/synchronization/BUILD.bazel index 5ce16958..64d3b929 100644 --- a/absl/synchronization/BUILD.bazel +++ b/absl/synchronization/BUILD.bazel @@ -14,7 +14,6 @@ # limitations under the License. # -load("@rules_cc//cc:defs.bzl", "cc_binary", "cc_library", "cc_test") load( "//absl:copts/configure_copts.bzl", "ABSL_DEFAULT_COPTS", @@ -35,7 +34,9 @@ cc_library( hdrs = [ "internal/graphcycles.h", ], - copts = ABSL_DEFAULT_COPTS, + copts = ABSL_DEFAULT_COPTS + select({ + "//conditions:default": [], + }), linkopts = ABSL_DEFAULT_LINKOPTS, visibility = [ "//absl:__subpackages__", @@ -96,8 +97,8 @@ cc_library( deps = [ ":graphcycles_internal", ":kernel_timeout_internal", - "//absl/base", "//absl/base:atomic_hook", + "//absl/base", "//absl/base:base_internal", "//absl/base:config", "//absl/base:core_headers", @@ -107,7 +108,9 @@ cc_library( "//absl/debugging:stacktrace", "//absl/debugging:symbolize", "//absl/time", - ], + ] + select({ + "//conditions:default": [], + }), ) cc_test( @@ -116,6 +119,9 @@ cc_test( srcs = ["barrier_test.cc"], copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + tags = [ + "no_test_wasm", + ], deps = [ ":synchronization", "//absl/time", @@ -129,6 +135,9 @@ cc_test( srcs = ["blocking_counter_test.cc"], copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + tags = [ + "no_test_wasm", + ], deps = [ ":synchronization", "//absl/time", @@ -136,6 +145,21 @@ cc_test( ], ) +cc_binary( + name = "blocking_counter_benchmark", + testonly = 1, + srcs = ["blocking_counter_benchmark.cc"], + copts = ABSL_TEST_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, + tags = ["benchmark"], + visibility = ["//visibility:private"], + deps = [ + ":synchronization", + ":thread_pool", + "@com_github_google_benchmark//:benchmark_main", + ], +) + cc_test( name = "graphcycles_test", size = "medium", @@ -264,6 +288,9 @@ cc_test( size = "medium", copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, + tags = [ + "no_test_wasm", + ], deps = [ ":per_thread_sem_test_common", ":synchronization", @@ -280,7 +307,10 @@ cc_test( ], copts = ABSL_TEST_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, - tags = ["no_test_ios_x86_64"], + tags = [ + "no_test_ios_x86_64", + "no_test_wasm", + ], deps = [ ":synchronization", "//absl/base:core_headers", diff --git a/absl/synchronization/CMakeLists.txt b/absl/synchronization/CMakeLists.txt index e633d0bf..9335c264 100644 --- a/absl/synchronization/CMakeLists.txt +++ b/absl/synchronization/CMakeLists.txt @@ -14,6 +14,7 @@ # limitations under the License. # +# Internal-only target, do not depend on directly. absl_cc_library( NAME graphcycles_internal @@ -32,6 +33,7 @@ absl_cc_library( absl::raw_logging_internal ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME kernel_timeout_internal @@ -95,7 +97,7 @@ absl_cc_test( DEPS absl::synchronization absl::time - gmock_main + GTest::gmock_main ) absl_cc_test( @@ -108,7 +110,7 @@ absl_cc_test( DEPS absl::synchronization absl::time - gmock_main + GTest::gmock_main ) absl_cc_test( @@ -122,9 +124,10 @@ absl_cc_test( absl::graphcycles_internal absl::core_headers absl::raw_logging_internal - gmock_main + GTest::gmock_main ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME thread_pool @@ -154,7 +157,7 @@ absl_cc_test( absl::memory absl::raw_logging_internal absl::time - gmock_main + GTest::gmock_main ) absl_cc_test( @@ -167,9 +170,10 @@ absl_cc_test( DEPS absl::synchronization absl::time - gmock_main + GTest::gmock_main ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME per_thread_sem_test_common @@ -183,7 +187,7 @@ absl_cc_library( absl::config absl::strings absl::time - gmock + GTest::gmock TESTONLY ) @@ -199,7 +203,7 @@ absl_cc_test( absl::synchronization absl::strings absl::time - gmock_main + GTest::gmock_main ) absl_cc_test( diff --git a/absl/synchronization/blocking_counter.cc b/absl/synchronization/blocking_counter.cc index 3cea7aed..d2f82da3 100644 --- a/absl/synchronization/blocking_counter.cc +++ b/absl/synchronization/blocking_counter.cc @@ -14,41 +14,51 @@ #include "absl/synchronization/blocking_counter.h" +#include <atomic> + #include "absl/base/internal/raw_logging.h" namespace absl { ABSL_NAMESPACE_BEGIN -// Return whether int *arg is zero. -static bool IsZero(void *arg) { - return 0 == *reinterpret_cast<int *>(arg); +namespace { + +// Return whether int *arg is true. +bool IsDone(void *arg) { return *reinterpret_cast<bool *>(arg); } + +} // namespace + +BlockingCounter::BlockingCounter(int initial_count) + : count_(initial_count), + num_waiting_(0), + done_{initial_count == 0 ? true : false} { + ABSL_RAW_CHECK(initial_count >= 0, "BlockingCounter initial_count negative"); } bool BlockingCounter::DecrementCount() { - MutexLock l(&lock_); - count_--; - if (count_ < 0) { - ABSL_RAW_LOG( - FATAL, - "BlockingCounter::DecrementCount() called too many times. count=%d", - count_); + int count = count_.fetch_sub(1, std::memory_order_acq_rel) - 1; + ABSL_RAW_CHECK(count >= 0, + "BlockingCounter::DecrementCount() called too many times"); + if (count == 0) { + MutexLock l(&lock_); + done_ = true; + return true; } - return count_ == 0; + return false; } void BlockingCounter::Wait() { MutexLock l(&this->lock_); - ABSL_RAW_CHECK(count_ >= 0, "BlockingCounter underflow"); // only one thread may call Wait(). To support more than one thread, // implement a counter num_to_exit, like in the Barrier class. ABSL_RAW_CHECK(num_waiting_ == 0, "multiple threads called Wait()"); num_waiting_++; - this->lock_.Await(Condition(IsZero, &this->count_)); + this->lock_.Await(Condition(IsDone, &this->done_)); - // At this point, We know that all threads executing DecrementCount have - // released the lock, and so will not touch this object again. + // At this point, we know that all threads executing DecrementCount + // will not touch this object again. // Therefore, the thread calling this method is free to delete the object // after we return from this method. } diff --git a/absl/synchronization/blocking_counter.h b/absl/synchronization/blocking_counter.h index 1f53f9f2..1908fdb1 100644 --- a/absl/synchronization/blocking_counter.h +++ b/absl/synchronization/blocking_counter.h @@ -20,6 +20,8 @@ #ifndef ABSL_SYNCHRONIZATION_BLOCKING_COUNTER_H_ #define ABSL_SYNCHRONIZATION_BLOCKING_COUNTER_H_ +#include <atomic> + #include "absl/base/thread_annotations.h" #include "absl/synchronization/mutex.h" @@ -60,8 +62,7 @@ ABSL_NAMESPACE_BEGIN // class BlockingCounter { public: - explicit BlockingCounter(int initial_count) - : count_(initial_count), num_waiting_(0) {} + explicit BlockingCounter(int initial_count); BlockingCounter(const BlockingCounter&) = delete; BlockingCounter& operator=(const BlockingCounter&) = delete; @@ -89,8 +90,9 @@ class BlockingCounter { private: Mutex lock_; - int count_ ABSL_GUARDED_BY(lock_); + std::atomic<int> count_; int num_waiting_ ABSL_GUARDED_BY(lock_); + bool done_ ABSL_GUARDED_BY(lock_); }; ABSL_NAMESPACE_END diff --git a/absl/synchronization/blocking_counter_benchmark.cc b/absl/synchronization/blocking_counter_benchmark.cc new file mode 100644 index 00000000..b504d1a5 --- /dev/null +++ b/absl/synchronization/blocking_counter_benchmark.cc @@ -0,0 +1,83 @@ +// Copyright 2021 The Abseil Authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include <limits> + +#include "absl/synchronization/blocking_counter.h" +#include "absl/synchronization/internal/thread_pool.h" +#include "benchmark/benchmark.h" + +namespace { + +void BM_BlockingCounter_SingleThread(benchmark::State& state) { + for (auto _ : state) { + int iterations = state.range(0); + absl::BlockingCounter counter{iterations}; + for (int i = 0; i < iterations; ++i) { + counter.DecrementCount(); + } + counter.Wait(); + } +} +BENCHMARK(BM_BlockingCounter_SingleThread) + ->ArgName("iterations") + ->Arg(2) + ->Arg(4) + ->Arg(16) + ->Arg(64) + ->Arg(256); + +void BM_BlockingCounter_DecrementCount(benchmark::State& state) { + static absl::BlockingCounter* counter = + new absl::BlockingCounter{std::numeric_limits<int>::max()}; + for (auto _ : state) { + counter->DecrementCount(); + } +} +BENCHMARK(BM_BlockingCounter_DecrementCount) + ->Threads(2) + ->Threads(4) + ->Threads(6) + ->Threads(8) + ->Threads(10) + ->Threads(12) + ->Threads(16) + ->Threads(32) + ->Threads(64) + ->Threads(128); + +void BM_BlockingCounter_Wait(benchmark::State& state) { + int num_threads = state.range(0); + absl::synchronization_internal::ThreadPool pool(num_threads); + for (auto _ : state) { + absl::BlockingCounter counter{num_threads}; + pool.Schedule([num_threads, &counter, &pool]() { + for (int i = 0; i < num_threads; ++i) { + pool.Schedule([&counter]() { counter.DecrementCount(); }); + } + }); + counter.Wait(); + } +} +BENCHMARK(BM_BlockingCounter_Wait) + ->ArgName("threads") + ->Arg(2) + ->Arg(4) + ->Arg(8) + ->Arg(16) + ->Arg(32) + ->Arg(64) + ->Arg(128); + +} // namespace diff --git a/absl/synchronization/blocking_counter_test.cc b/absl/synchronization/blocking_counter_test.cc index 2926224a..06885f57 100644 --- a/absl/synchronization/blocking_counter_test.cc +++ b/absl/synchronization/blocking_counter_test.cc @@ -63,6 +63,18 @@ TEST(BlockingCounterTest, BasicFunctionality) { } } +TEST(BlockingCounterTest, WaitZeroInitialCount) { + BlockingCounter counter(0); + counter.Wait(); +} + +#if GTEST_HAS_DEATH_TEST +TEST(BlockingCounterTest, WaitNegativeInitialCount) { + EXPECT_DEATH(BlockingCounter counter(-1), + "BlockingCounter initial_count negative"); +} +#endif + } // namespace ABSL_NAMESPACE_END } // namespace absl diff --git a/absl/synchronization/internal/create_thread_identity.cc b/absl/synchronization/internal/create_thread_identity.cc index 53a71b34..44e6129b 100644 --- a/absl/synchronization/internal/create_thread_identity.cc +++ b/absl/synchronization/internal/create_thread_identity.cc @@ -38,7 +38,7 @@ ABSL_CONST_INIT static base_internal::ThreadIdentity* thread_identity_freelist; // A per-thread destructor for reclaiming associated ThreadIdentity objects. // Since we must preserve their storage we cache them for re-use. -void ReclaimThreadIdentity(void* v) { +static void ReclaimThreadIdentity(void* v) { base_internal::ThreadIdentity* identity = static_cast<base_internal::ThreadIdentity*>(v); @@ -48,8 +48,6 @@ void ReclaimThreadIdentity(void* v) { base_internal::LowLevelAlloc::Free(identity->per_thread_synch.all_locks); } - PerThreadSem::Destroy(identity); - // We must explicitly clear the current thread's identity: // (a) Subsequent (unrelated) per-thread destructors may require an identity. // We must guarantee a new identity is used in this case (this instructor @@ -71,7 +69,12 @@ static intptr_t RoundUp(intptr_t addr, intptr_t align) { return (addr + align - 1) & ~(align - 1); } -static void ResetThreadIdentity(base_internal::ThreadIdentity* identity) { +void OneTimeInitThreadIdentity(base_internal::ThreadIdentity* identity) { + PerThreadSem::Init(identity); +} + +static void ResetThreadIdentityBetweenReuse( + base_internal::ThreadIdentity* identity) { base_internal::PerThreadSynch* pts = &identity->per_thread_synch; pts->next = nullptr; pts->skip = nullptr; @@ -116,8 +119,9 @@ static base_internal::ThreadIdentity* NewThreadIdentity() { identity = reinterpret_cast<base_internal::ThreadIdentity*>( RoundUp(reinterpret_cast<intptr_t>(allocation), base_internal::PerThreadSynch::kAlignment)); + OneTimeInitThreadIdentity(identity); } - ResetThreadIdentity(identity); + ResetThreadIdentityBetweenReuse(identity); return identity; } @@ -127,7 +131,6 @@ static base_internal::ThreadIdentity* NewThreadIdentity() { // REQUIRES: CurrentThreadIdentity(false) == nullptr base_internal::ThreadIdentity* CreateThreadIdentity() { base_internal::ThreadIdentity* identity = NewThreadIdentity(); - PerThreadSem::Init(identity); // Associate the value with the current thread, and attach our destructor. base_internal::SetCurrentThreadIdentity(identity, ReclaimThreadIdentity); return identity; diff --git a/absl/synchronization/internal/create_thread_identity.h b/absl/synchronization/internal/create_thread_identity.h index e121f683..4cfde091 100644 --- a/absl/synchronization/internal/create_thread_identity.h +++ b/absl/synchronization/internal/create_thread_identity.h @@ -36,10 +36,6 @@ namespace synchronization_internal { // For private use only. base_internal::ThreadIdentity* CreateThreadIdentity(); -// A per-thread destructor for reclaiming associated ThreadIdentity objects. -// For private use only. -void ReclaimThreadIdentity(void* v); - // Returns the ThreadIdentity object representing the calling thread; guaranteed // to be unique for its lifetime. The returned object will remain valid for the // program's lifetime; although it may be re-assigned to a subsequent thread. diff --git a/absl/synchronization/internal/per_thread_sem.cc b/absl/synchronization/internal/per_thread_sem.cc index a6031787..469e8f32 100644 --- a/absl/synchronization/internal/per_thread_sem.cc +++ b/absl/synchronization/internal/per_thread_sem.cc @@ -47,10 +47,6 @@ void PerThreadSem::Init(base_internal::ThreadIdentity *identity) { identity->is_idle.store(false, std::memory_order_relaxed); } -void PerThreadSem::Destroy(base_internal::ThreadIdentity *identity) { - Waiter::GetWaiter(identity)->~Waiter(); -} - void PerThreadSem::Tick(base_internal::ThreadIdentity *identity) { const int ticker = identity->ticker.fetch_add(1, std::memory_order_relaxed) + 1; diff --git a/absl/synchronization/internal/per_thread_sem.h b/absl/synchronization/internal/per_thread_sem.h index 7beae8ef..90a88809 100644 --- a/absl/synchronization/internal/per_thread_sem.h +++ b/absl/synchronization/internal/per_thread_sem.h @@ -66,10 +66,6 @@ class PerThreadSem { // REQUIRES: May only be called by ThreadIdentity. static void Init(base_internal::ThreadIdentity* identity); - // Destroy the PerThreadSem associated with "identity". - // REQUIRES: May only be called by ThreadIdentity. - static void Destroy(base_internal::ThreadIdentity* identity); - // Increments "identity"'s count. static inline void Post(base_internal::ThreadIdentity* identity); @@ -81,8 +77,7 @@ class PerThreadSem { // Permitted callers. friend class PerThreadSemTest; friend class absl::Mutex; - friend absl::base_internal::ThreadIdentity* CreateThreadIdentity(); - friend void ReclaimThreadIdentity(void* v); + friend void OneTimeInitThreadIdentity(absl::base_internal::ThreadIdentity*); }; } // namespace synchronization_internal diff --git a/absl/synchronization/internal/per_thread_sem_test.cc b/absl/synchronization/internal/per_thread_sem_test.cc index 8cf59e64..24a6b548 100644 --- a/absl/synchronization/internal/per_thread_sem_test.cc +++ b/absl/synchronization/internal/per_thread_sem_test.cc @@ -159,7 +159,7 @@ TEST_F(PerThreadSemTest, Timeouts) { const absl::Duration elapsed = absl::Now() - start; // Allow for a slight early return, to account for quality of implementation // issues on various platforms. - const absl::Duration slop = absl::Microseconds(200); + const absl::Duration slop = absl::Milliseconds(1); EXPECT_LE(delay - slop, elapsed) << "Wait returned " << delay - elapsed << " early (with " << slop << " slop), start time was " << start; @@ -174,6 +174,15 @@ TEST_F(PerThreadSemTest, Timeouts) { EXPECT_TRUE(Wait(negative_timeout)); } +TEST_F(PerThreadSemTest, ThreadIdentityReuse) { + // Create a base_internal::ThreadIdentity object and keep reusing it. There + // should be no memory or resource leaks. + for (int i = 0; i < 10000; i++) { + std::thread t([]() { GetOrCreateCurrentThreadIdentity(); }); + t.join(); + } +} + } // namespace } // namespace synchronization_internal diff --git a/absl/synchronization/internal/waiter.cc b/absl/synchronization/internal/waiter.cc index 2123be60..f2051d67 100644 --- a/absl/synchronization/internal/waiter.cc +++ b/absl/synchronization/internal/waiter.cc @@ -71,14 +71,13 @@ Waiter::Waiter() { futex_.store(0, std::memory_order_relaxed); } -Waiter::~Waiter() = default; - bool Waiter::Wait(KernelTimeout t) { // Loop until we can atomically decrement futex from a positive // value, waiting on a futex while we believe it is zero. // Note that, since the thread ticker is just reset, we don't need to check // whether the thread is idle on the very first pass of the loop. bool first_pass = true; + while (true) { int32_t x = futex_.load(std::memory_order_relaxed); while (x != 0) { @@ -90,7 +89,6 @@ bool Waiter::Wait(KernelTimeout t) { return true; // Consumed a wakeup, we are done. } - if (!first_pass) MaybeBecomeIdle(); const int err = Futex::WaitUntil(&futex_, 0, t); if (err != 0) { @@ -161,18 +159,6 @@ Waiter::Waiter() { wakeup_count_ = 0; } -Waiter::~Waiter() { - const int err = pthread_mutex_destroy(&mu_); - if (err != 0) { - ABSL_RAW_LOG(FATAL, "pthread_mutex_destroy failed: %d", err); - } - - const int err2 = pthread_cond_destroy(&cv_); - if (err2 != 0) { - ABSL_RAW_LOG(FATAL, "pthread_cond_destroy failed: %d", err2); - } -} - bool Waiter::Wait(KernelTimeout t) { struct timespec abs_timeout; if (t.has_timeout()) { @@ -240,12 +226,6 @@ Waiter::Waiter() { wakeups_.store(0, std::memory_order_relaxed); } -Waiter::~Waiter() { - if (sem_destroy(&sem_) != 0) { - ABSL_RAW_LOG(FATAL, "sem_destroy failed with errno %d\n", errno); - } -} - bool Waiter::Wait(KernelTimeout t) { struct timespec abs_timeout; if (t.has_timeout()) { @@ -363,11 +343,6 @@ Waiter::Waiter() { wakeup_count_ = 0; } -// SRW locks and condition variables do not need to be explicitly destroyed. -// https://docs.microsoft.com/en-us/windows/win32/api/synchapi/nf-synchapi-initializesrwlock -// https://stackoverflow.com/questions/28975958/why-does-windows-have-no-deleteconditionvariable-function-to-go-together-with -Waiter::~Waiter() = default; - bool Waiter::Wait(KernelTimeout t) { SRWLOCK *mu = WinHelper::GetLock(this); CONDITION_VARIABLE *cv = WinHelper::GetCond(this); diff --git a/absl/synchronization/internal/waiter.h b/absl/synchronization/internal/waiter.h index be3df180..b8adfeb5 100644 --- a/absl/synchronization/internal/waiter.h +++ b/absl/synchronization/internal/waiter.h @@ -71,9 +71,6 @@ class Waiter { Waiter(const Waiter&) = delete; Waiter& operator=(const Waiter&) = delete; - // Destroy any data to track waits. - ~Waiter(); - // Blocks the calling thread until a matching call to `Post()` or // `t` has passed. Returns `true` if woken (`Post()` called), // `false` on timeout. @@ -106,6 +103,12 @@ class Waiter { #endif private: + // The destructor must not be called since Mutex/CondVar + // can use PerThreadSem/Waiter after the thread exits. + // Waiter objects are embedded in ThreadIdentity objects, + // which are reused via a freelist and are never destroyed. + ~Waiter() = delete; + #if ABSL_WAITER_MODE == ABSL_WAITER_MODE_FUTEX // Futexes are defined by specification to be 32-bits. // Thus std::atomic<int32_t> must be just an int32_t with lockfree methods. @@ -136,8 +139,11 @@ class Waiter { // REQUIRES: WinHelper::GetLock(this) must be held. void InternalCondVarPoke(); - // We can't include Windows.h in our headers, so we use aligned charachter + // We can't include Windows.h in our headers, so we use aligned character // buffers to define the storage of SRWLOCK and CONDITION_VARIABLE. + // SRW locks and condition variables do not need to be explicitly destroyed. + // https://docs.microsoft.com/en-us/windows/win32/api/synchapi/nf-synchapi-initializesrwlock + // https://stackoverflow.com/questions/28975958/why-does-windows-have-no-deleteconditionvariable-function-to-go-together-with alignas(void*) unsigned char mu_storage_[sizeof(void*)]; alignas(void*) unsigned char cv_storage_[sizeof(void*)]; int waiter_count_; diff --git a/absl/synchronization/mutex.cc b/absl/synchronization/mutex.cc index 76ad41fe..52e2455d 100644 --- a/absl/synchronization/mutex.cc +++ b/absl/synchronization/mutex.cc @@ -109,7 +109,7 @@ static inline bool EvalConditionAnnotated(const Condition *cond, Mutex *mu, bool locking, bool trylock, bool read_lock); -void RegisterMutexProfiler(void (*fn)(int64_t wait_timestamp)) { +void RegisterMutexProfiler(void (*fn)(int64_t wait_cycles)) { submit_profile_data.Store(fn); } @@ -1744,23 +1744,33 @@ ABSL_XRAY_LOG_ARGS(1) void Mutex::ReaderUnlock() { ABSL_TSAN_MUTEX_POST_UNLOCK(this, __tsan_mutex_read_lock); } -// The zap_desig_waker bitmask is used to clear the designated waker flag in -// the mutex if this thread has blocked, and therefore may be the designated -// waker. -static const intptr_t zap_desig_waker[] = { - ~static_cast<intptr_t>(0), // not blocked - ~static_cast<intptr_t>( - kMuDesig) // blocked; turn off the designated waker bit -}; +// Clears the designated waker flag in the mutex if this thread has blocked, and +// therefore may be the designated waker. +static intptr_t ClearDesignatedWakerMask(int flag) { + assert(flag >= 0); + assert(flag <= 1); + switch (flag) { + case 0: // not blocked + return ~static_cast<intptr_t>(0); + case 1: // blocked; turn off the designated waker bit + return ~static_cast<intptr_t>(kMuDesig); + } + ABSL_INTERNAL_UNREACHABLE; +} -// The ignore_waiting_writers bitmask is used to ignore the existence -// of waiting writers if a reader that has already blocked once -// wakes up. -static const intptr_t ignore_waiting_writers[] = { - ~static_cast<intptr_t>(0), // not blocked - ~static_cast<intptr_t>( - kMuWrWait) // blocked; pretend there are no waiting writers -}; +// Conditionally ignores the existence of waiting writers if a reader that has +// already blocked once wakes up. +static intptr_t IgnoreWaitingWritersMask(int flag) { + assert(flag >= 0); + assert(flag <= 1); + switch (flag) { + case 0: // not blocked + return ~static_cast<intptr_t>(0); + case 1: // blocked; pretend there are no waiting writers + return ~static_cast<intptr_t>(kMuWrWait); + } + ABSL_INTERNAL_UNREACHABLE; +} // Internal version of LockWhen(). See LockSlowWithDeadline() ABSL_ATTRIBUTE_NOINLINE void Mutex::LockSlow(MuHow how, const Condition *cond, @@ -1852,8 +1862,10 @@ bool Mutex::LockSlowWithDeadline(MuHow how, const Condition *cond, bool unlock = false; if ((v & how->fast_need_zero) == 0 && // try fast acquire mu_.compare_exchange_strong( - v, (how->fast_or | (v & zap_desig_waker[flags & kMuHasBlocked])) + - how->fast_add, + v, + (how->fast_or | + (v & ClearDesignatedWakerMask(flags & kMuHasBlocked))) + + how->fast_add, std::memory_order_acquire, std::memory_order_relaxed)) { if (cond == nullptr || EvalConditionAnnotated(cond, this, true, false, how == kShared)) { @@ -1927,9 +1939,10 @@ void Mutex::LockSlowLoop(SynchWaitParams *waitp, int flags) { CheckForMutexCorruption(v, "Lock"); if ((v & waitp->how->slow_need_zero) == 0) { if (mu_.compare_exchange_strong( - v, (waitp->how->fast_or | - (v & zap_desig_waker[flags & kMuHasBlocked])) + - waitp->how->fast_add, + v, + (waitp->how->fast_or | + (v & ClearDesignatedWakerMask(flags & kMuHasBlocked))) + + waitp->how->fast_add, std::memory_order_acquire, std::memory_order_relaxed)) { if (waitp->cond == nullptr || EvalConditionAnnotated(waitp->cond, this, true, false, @@ -1946,8 +1959,9 @@ void Mutex::LockSlowLoop(SynchWaitParams *waitp, int flags) { if ((v & (kMuSpin|kMuWait)) == 0) { // no waiters // This thread tries to become the one and only waiter. PerThreadSynch *new_h = Enqueue(nullptr, waitp, v, flags); - intptr_t nv = (v & zap_desig_waker[flags & kMuHasBlocked] & kMuLow) | - kMuWait; + intptr_t nv = + (v & ClearDesignatedWakerMask(flags & kMuHasBlocked) & kMuLow) | + kMuWait; ABSL_RAW_CHECK(new_h != nullptr, "Enqueue to empty list failed"); if (waitp->how == kExclusive && (v & kMuReader) != 0) { nv |= kMuWrWait; @@ -1961,12 +1975,13 @@ void Mutex::LockSlowLoop(SynchWaitParams *waitp, int flags) { waitp->thread->waitp = nullptr; } } else if ((v & waitp->how->slow_inc_need_zero & - ignore_waiting_writers[flags & kMuHasBlocked]) == 0) { + IgnoreWaitingWritersMask(flags & kMuHasBlocked)) == 0) { // This is a reader that needs to increment the reader count, // but the count is currently held in the last waiter. if (mu_.compare_exchange_strong( - v, (v & zap_desig_waker[flags & kMuHasBlocked]) | kMuSpin | - kMuReader, + v, + (v & ClearDesignatedWakerMask(flags & kMuHasBlocked)) | + kMuSpin | kMuReader, std::memory_order_acquire, std::memory_order_relaxed)) { PerThreadSynch *h = GetPerThreadSynch(v); h->readers += kMuOne; // inc reader count in waiter @@ -1987,8 +2002,9 @@ void Mutex::LockSlowLoop(SynchWaitParams *waitp, int flags) { } } else if ((v & kMuSpin) == 0 && // attempt to queue ourselves mu_.compare_exchange_strong( - v, (v & zap_desig_waker[flags & kMuHasBlocked]) | kMuSpin | - kMuWait, + v, + (v & ClearDesignatedWakerMask(flags & kMuHasBlocked)) | + kMuSpin | kMuWait, std::memory_order_acquire, std::memory_order_relaxed)) { PerThreadSynch *h = GetPerThreadSynch(v); PerThreadSynch *new_h = Enqueue(h, waitp, v, flags); @@ -2315,19 +2331,21 @@ ABSL_ATTRIBUTE_NOINLINE void Mutex::UnlockSlow(SynchWaitParams *waitp) { } // end of for(;;)-loop if (wake_list != kPerThreadSynchNull) { - int64_t enqueue_timestamp = wake_list->waitp->contention_start_cycles; - bool cond_waiter = wake_list->cond_waiter; + int64_t wait_cycles = 0; + int64_t now = base_internal::CycleClock::Now(); do { + // Sample lock contention events only if the waiter was trying to acquire + // the lock, not waiting on a condition variable or Condition. + if (!wake_list->cond_waiter) { + wait_cycles += (now - wake_list->waitp->contention_start_cycles); + wake_list->waitp->contention_start_cycles = now; + } wake_list = Wakeup(wake_list); // wake waiters } while (wake_list != kPerThreadSynchNull); - if (!cond_waiter) { - // Sample lock contention events only if the (first) waiter was trying to - // acquire the lock, not waiting on a condition variable or Condition. - int64_t wait_cycles = - base_internal::CycleClock::Now() - enqueue_timestamp; + if (wait_cycles > 0) { mutex_tracer("slow release", this, wait_cycles); ABSL_TSAN_MUTEX_PRE_DIVERT(this, 0); - submit_profile_data(enqueue_timestamp); + submit_profile_data(wait_cycles); ABSL_TSAN_MUTEX_POST_DIVERT(this, 0); } } @@ -2492,9 +2510,9 @@ void CondVar::Remove(PerThreadSynch *s) { // before calling Mutex::UnlockSlow(), the Mutex code might be re-entered (via // the logging code, or via a Condition function) and might potentially attempt // to block this thread. That would be a problem if the thread were already on -// a the condition variable waiter queue. Thus, we use the waitp->cv_word -// to tell the unlock code to call CondVarEnqueue() to queue the thread on the -// condition variable queue just before the mutex is to be unlocked, and (most +// a condition variable waiter queue. Thus, we use the waitp->cv_word to tell +// the unlock code to call CondVarEnqueue() to queue the thread on the condition +// variable queue just before the mutex is to be unlocked, and (most // importantly) after any call to an external routine that might re-enter the // mutex code. static void CondVarEnqueue(SynchWaitParams *waitp) { @@ -2557,6 +2575,23 @@ bool CondVar::WaitCommon(Mutex *mutex, KernelTimeout t) { while (waitp.thread->state.load(std::memory_order_acquire) == PerThreadSynch::kQueued) { if (!Mutex::DecrementSynchSem(mutex, waitp.thread, t)) { + // DecrementSynchSem returned due to timeout. + // Now we will either (1) remove ourselves from the wait list in Remove + // below, in which case Remove will set thread.state = kAvailable and + // we will not call DecrementSynchSem again; or (2) Signal/SignalAll + // has removed us concurrently and is calling Wakeup, which will set + // thread.state = kAvailable and post to the semaphore. + // It's important to reset the timeout for the case (2) because otherwise + // we can live-lock in this loop since DecrementSynchSem will always + // return immediately due to timeout, but Signal/SignalAll is not + // necessary set thread.state = kAvailable yet (and is not scheduled + // due to thread priorities or other scheduler artifacts). + // Note this could also be resolved if Signal/SignalAll would set + // thread.state = kAvailable while holding the wait list spin lock. + // But this can't be easily done for SignalAll since it grabs the whole + // wait list with a single compare-exchange and does not really grab + // the spin lock. + t = KernelTimeout::Never(); this->Remove(waitp.thread); rc = true; } diff --git a/absl/synchronization/mutex.h b/absl/synchronization/mutex.h index f49e0c83..b69b7089 100644 --- a/absl/synchronization/mutex.h +++ b/absl/synchronization/mutex.h @@ -174,9 +174,12 @@ class ABSL_LOCKABLE Mutex { // Mutex::AssertHeld() // - // Return immediately if this thread holds the `Mutex` exclusively (in write - // mode). Otherwise, may report an error (typically by crashing with a - // diagnostic), or may return immediately. + // Require that the mutex be held exclusively (write mode) by this thread. + // + // If the mutex is not currently held by this thread, this function may report + // an error (typically by crashing with a diagnostic) or it may do nothing. + // This function is intended only as a tool to assist debugging; it doesn't + // guarantee correctness. void AssertHeld() const ABSL_ASSERT_EXCLUSIVE_LOCK(); // --------------------------------------------------------------------------- @@ -236,9 +239,13 @@ class ABSL_LOCKABLE Mutex { // Mutex::AssertReaderHeld() // - // Returns immediately if this thread holds the `Mutex` in at least shared - // mode (read mode). Otherwise, may report an error (typically by - // crashing with a diagnostic), or may return immediately. + // Require that the mutex be held at least in shared mode (read mode) by this + // thread. + // + // If the mutex is not currently held by this thread, this function may report + // an error (typically by crashing with a diagnostic) or it may do nothing. + // This function is intended only as a tool to assist debugging; it doesn't + // guarantee correctness. void AssertReaderHeld() const ABSL_ASSERT_SHARED_LOCK(); // Mutex::WriterLock() @@ -778,9 +785,9 @@ class Condition { // // Usage to wake T is: // mu.Lock(); -// // process data, possibly establishing C -// if (C) { cv->Signal(); } -// mu.Unlock(); +// // process data, possibly establishing C +// if (C) { cv->Signal(); } +// mu.Unlock(); // // If C may be useful to more than one waiter, use `SignalAll()` instead of // `Signal()`. @@ -984,14 +991,15 @@ inline Condition::Condition(const T *object, // Register a hook for profiling support. // // The function pointer registered here will be called whenever a mutex is -// contended. The callback is given the absl/base/cycleclock.h timestamp when -// waiting began. +// contended. The callback is given the cycles for which waiting happened (as +// measured by //absl/base/internal/cycleclock.h, and which may not +// be real "cycle" counts.) // // Calls to this function do not race or block, but there is no ordering // guaranteed between calls to this function and call to the provided hook. // In particular, the previously registered hook may still be called for some // time after this function returns. -void RegisterMutexProfiler(void (*fn)(int64_t wait_timestamp)); +void RegisterMutexProfiler(void (*fn)(int64_t wait_cycles)); // Register a hook for Mutex tracing. // diff --git a/absl/synchronization/mutex_benchmark.cc b/absl/synchronization/mutex_benchmark.cc index e35aed8b..b5d2fbc4 100644 --- a/absl/synchronization/mutex_benchmark.cc +++ b/absl/synchronization/mutex_benchmark.cc @@ -97,7 +97,7 @@ void BM_MutexEnqueue(benchmark::State& state) { // Mutex queueing behavior is modified. const bool multiple_priorities = state.range(0); ScopedThreadMutexPriority priority_setter( - (multiple_priorities && state.thread_index != 0) ? 1 : 0); + (multiple_priorities && state.thread_index() != 0) ? 1 : 0); struct Shared { absl::Mutex mu; @@ -176,7 +176,7 @@ BENCHMARK(BM_MutexEnqueue) template <typename MutexType> void BM_Contended(benchmark::State& state) { - int priority = state.thread_index % state.range(1); + int priority = state.thread_index() % state.range(1); ScopedThreadMutexPriority priority_setter(priority); struct Shared { @@ -196,7 +196,7 @@ void BM_Contended(benchmark::State& state) { // To achieve this amount of local work is multiplied by number of threads // to keep ratio between local work and critical section approximately // equal regardless of number of threads. - DelayNs(100 * state.threads, &local); + DelayNs(100 * state.threads(), &local); RaiiLocker<MutexType> locker(&shared->mu); DelayNs(state.range(0), &shared->data); } diff --git a/absl/synchronization/mutex_test.cc b/absl/synchronization/mutex_test.cc index f8fbf948..99bb0175 100644 --- a/absl/synchronization/mutex_test.cc +++ b/absl/synchronization/mutex_test.cc @@ -26,6 +26,7 @@ #include <random> #include <string> #include <thread> // NOLINT(build/c++11) +#include <type_traits> #include <vector> #include "gtest/gtest.h" @@ -870,33 +871,6 @@ TEST(Mutex, LockedMutexDestructionBug) ABSL_NO_THREAD_SAFETY_ANALYSIS { } } -// -------------------------------------------------------- -// Test for bug with pattern of readers using a condvar. The bug was that if a -// reader went to sleep on a condition variable while one or more other readers -// held the lock, but there were no waiters, the reader count (held in the -// mutex word) would be lost. (This is because Enqueue() had at one time -// always placed the thread on the Mutex queue. Later (CL 4075610), to -// tolerate re-entry into Mutex from a Condition predicate, Enqueue() was -// changed so that it could also place a thread on a condition-variable. This -// introduced the case where Enqueue() returned with an empty queue, and this -// case was handled incorrectly in one place.) - -static void ReaderForReaderOnCondVar(absl::Mutex *mu, absl::CondVar *cv, - int *running) { - std::random_device dev; - std::mt19937 gen(dev()); - std::uniform_int_distribution<int> random_millis(0, 15); - mu->ReaderLock(); - while (*running == 3) { - absl::SleepFor(absl::Milliseconds(random_millis(gen))); - cv->WaitWithTimeout(mu, absl::Milliseconds(random_millis(gen))); - } - mu->ReaderUnlock(); - mu->Lock(); - (*running)--; - mu->Unlock(); -} - struct True { template <class... Args> bool operator()(Args...) const { @@ -945,6 +919,33 @@ TEST(Mutex, FunctorCondition) { } } +// -------------------------------------------------------- +// Test for bug with pattern of readers using a condvar. The bug was that if a +// reader went to sleep on a condition variable while one or more other readers +// held the lock, but there were no waiters, the reader count (held in the +// mutex word) would be lost. (This is because Enqueue() had at one time +// always placed the thread on the Mutex queue. Later (CL 4075610), to +// tolerate re-entry into Mutex from a Condition predicate, Enqueue() was +// changed so that it could also place a thread on a condition-variable. This +// introduced the case where Enqueue() returned with an empty queue, and this +// case was handled incorrectly in one place.) + +static void ReaderForReaderOnCondVar(absl::Mutex *mu, absl::CondVar *cv, + int *running) { + std::random_device dev; + std::mt19937 gen(dev()); + std::uniform_int_distribution<int> random_millis(0, 15); + mu->ReaderLock(); + while (*running == 3) { + absl::SleepFor(absl::Milliseconds(random_millis(gen))); + cv->WaitWithTimeout(mu, absl::Milliseconds(random_millis(gen))); + } + mu->ReaderUnlock(); + mu->Lock(); + (*running)--; + mu->Unlock(); +} + static bool IntIsZero(int *x) { return *x == 0; } // Test for reader waiting condition variable when there are other readers @@ -1703,4 +1704,30 @@ TEST(Mutex, MuTime) { EXPECT_EQ(RunTest(&TestMuTime, threads, iterations, 1), threads * iterations); } +TEST(Mutex, SignalExitedThread) { + // The test may expose a race when Mutex::Unlock signals a thread + // that has already exited. +#if defined(__wasm__) || defined(__asmjs__) + constexpr int kThreads = 1; // OOMs under WASM +#else + constexpr int kThreads = 100; +#endif + std::vector<std::thread> top; + for (unsigned i = 0; i < 2 * std::thread::hardware_concurrency(); i++) { + top.emplace_back([&]() { + for (int i = 0; i < kThreads; i++) { + absl::Mutex mu; + std::thread t([&]() { + mu.Lock(); + mu.Unlock(); + }); + mu.Lock(); + mu.Unlock(); + t.join(); + } + }); + } + for (auto &th : top) th.join(); +} + } // namespace diff --git a/absl/synchronization/notification.h b/absl/synchronization/notification.h index 9a354ca2..4bec2689 100644 --- a/absl/synchronization/notification.h +++ b/absl/synchronization/notification.h @@ -22,7 +22,7 @@ // The `Notification` object maintains a private boolean "notified" state that // transitions to `true` at most once. The `Notification` class provides the // following primary member functions: -// * `HasBeenNotified() `to query its state +// * `HasBeenNotified()` to query its state // * `WaitForNotification*()` to have threads wait until the "notified" state // is `true`. // * `Notify()` to set the notification's "notified" state to `true` and @@ -52,6 +52,7 @@ #include <atomic> +#include "absl/base/attributes.h" #include "absl/base/macros.h" #include "absl/synchronization/mutex.h" #include "absl/time/time.h" @@ -74,7 +75,7 @@ class Notification { // Notification::HasBeenNotified() // // Returns the value of the notification's internal "notified" state. - bool HasBeenNotified() const { + ABSL_MUST_USE_RESULT bool HasBeenNotified() const { return HasBeenNotifiedInternal(&this->notified_yet_); } diff --git a/absl/time/BUILD.bazel b/absl/time/BUILD.bazel index 3e25ca26..aa07df3d 100644 --- a/absl/time/BUILD.bazel +++ b/absl/time/BUILD.bazel @@ -14,7 +14,6 @@ # limitations under the License. # -load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") load( "//absl:copts/configure_copts.bzl", "ABSL_DEFAULT_COPTS", @@ -65,9 +64,7 @@ cc_library( hdrs = ["internal/test_util.h"], copts = ABSL_DEFAULT_COPTS, linkopts = ABSL_DEFAULT_LINKOPTS, - visibility = [ - "//absl/time:__pkg__", - ], + visibility = ["//visibility:private"], deps = [ ":time", "//absl/base:config", diff --git a/absl/time/CMakeLists.txt b/absl/time/CMakeLists.txt index 00bdd499..debab3ba 100644 --- a/absl/time/CMakeLists.txt +++ b/absl/time/CMakeLists.txt @@ -87,6 +87,7 @@ absl_cc_library( $<$<PLATFORM_ID:Darwin>:${CoreFoundation}> ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME time_internal_test_util @@ -102,7 +103,7 @@ absl_cc_library( absl::config absl::raw_logging_internal absl::time_zone - gmock + GTest::gmock TESTONLY ) @@ -124,5 +125,5 @@ absl_cc_test( absl::config absl::core_headers absl::time_zone - gmock_main + GTest::gmock_main ) diff --git a/absl/time/civil_time.cc b/absl/time/civil_time.cc index bdfe9ce0..6a231edb 100644 --- a/absl/time/civil_time.cc +++ b/absl/time/civil_time.cc @@ -38,9 +38,7 @@ std::string FormatYearAnd(string_view fmt, CivilSecond cs) { const CivilSecond ncs(NormalizeYear(cs.year()), cs.month(), cs.day(), cs.hour(), cs.minute(), cs.second()); const TimeZone utc = UTCTimeZone(); - // TODO(absl-team): Avoid conversion of fmt string. - return StrCat(cs.year(), - FormatTime(std::string(fmt), FromCivil(ncs, utc), utc)); + return StrCat(cs.year(), FormatTime(fmt, FromCivil(ncs, utc), utc)); } template <typename CivilT> diff --git a/absl/time/clock.cc b/absl/time/clock.cc index 7b204c4e..a091efdc 100644 --- a/absl/time/clock.cc +++ b/absl/time/clock.cc @@ -196,7 +196,7 @@ struct ABSL_CACHELINE_ALIGNED TimeState { absl::base_internal::SpinLock lock{absl::kConstInit, base_internal::SCHEDULE_KERNEL_ONLY}; }; -ABSL_CONST_INIT static TimeState time_state{}; +ABSL_CONST_INIT static TimeState time_state; // Return the time in ns as told by the kernel interface. Place in *cycleclock // the value of the cycleclock at about the time of the syscall. diff --git a/absl/time/clock_test.cc b/absl/time/clock_test.cc index 4bcfc6bc..bc77dbc2 100644 --- a/absl/time/clock_test.cc +++ b/absl/time/clock_test.cc @@ -18,6 +18,10 @@ #if defined(ABSL_HAVE_ALARM) #include <signal.h> #include <unistd.h> +#ifdef _AIX +// sig_t is not defined in AIX. +typedef void (*sig_t)(int); +#endif #elif defined(__linux__) || defined(__APPLE__) #error all known Linux and Apple targets have alarm #endif diff --git a/absl/time/duration.cc b/absl/time/duration.cc index 4443109a..2bba62da 100644 --- a/absl/time/duration.cc +++ b/absl/time/duration.cc @@ -766,13 +766,14 @@ void AppendNumberUnit(std::string* out, double n, DisplayUnit unit) { // is non-zero. // Unlike Go, we format the zero duration as 0, with no unit. std::string FormatDuration(Duration d) { - const Duration min_duration = Seconds(kint64min); - if (d == min_duration) { + constexpr Duration kMinDuration = Seconds(kint64min); + std::string s; + if (d == kMinDuration) { // Avoid needing to negate kint64min by directly returning what the // following code should produce in that case. - return "-2562047788015215h30m8s"; + s = "-2562047788015215h30m8s"; + return s; } - std::string s; if (d < ZeroDuration()) { s.append("-"); d = -d; diff --git a/absl/time/duration_test.cc b/absl/time/duration_test.cc index fb28fa98..b7abf4ba 100644 --- a/absl/time/duration_test.cc +++ b/absl/time/duration_test.cc @@ -17,6 +17,7 @@ #endif #include <chrono> // NOLINT(build/c++11) +#include <cfloat> #include <cmath> #include <cstdint> #include <ctime> @@ -348,6 +349,11 @@ TEST(Duration, ToChrono) { } TEST(Duration, FactoryOverloads) { +#if defined(ABSL_SKIP_TIME_TESTS_BROKEN_ON_MSVC_OPT) && \ + ABSL_SKIP_TIME_TESTS_BROKEN_ON_MSVC_OPT + GTEST_SKIP(); +#endif + enum E { kOne = 1 }; #define TEST_FACTORY_OVERLOADS(NAME) \ EXPECT_EQ(1, NAME(kOne) / NAME(kOne)); \ @@ -878,6 +884,11 @@ TEST(Duration, RelationalOperators) { } TEST(Duration, Addition) { +#if defined(ABSL_SKIP_TIME_TESTS_BROKEN_ON_MSVC_OPT) && \ + ABSL_SKIP_TIME_TESTS_BROKEN_ON_MSVC_OPT + GTEST_SKIP(); +#endif + #define TEST_ADD_OPS(UNIT) \ do { \ EXPECT_EQ(UNIT(2), UNIT(1) + UNIT(1)); \ @@ -971,6 +982,11 @@ TEST(Duration, Negation) { } TEST(Duration, AbsoluteValue) { +#if defined(ABSL_SKIP_TIME_TESTS_BROKEN_ON_MSVC_OPT) && \ + ABSL_SKIP_TIME_TESTS_BROKEN_ON_MSVC_OPT + GTEST_SKIP(); +#endif + EXPECT_EQ(absl::ZeroDuration(), AbsDuration(absl::ZeroDuration())); EXPECT_EQ(absl::Seconds(1), AbsDuration(absl::Seconds(1))); EXPECT_EQ(absl::Seconds(1), AbsDuration(absl::Seconds(-1))); @@ -988,6 +1004,11 @@ TEST(Duration, AbsoluteValue) { } TEST(Duration, Multiplication) { +#if defined(ABSL_SKIP_TIME_TESTS_BROKEN_ON_MSVC_OPT) && \ + ABSL_SKIP_TIME_TESTS_BROKEN_ON_MSVC_OPT + GTEST_SKIP(); +#endif + #define TEST_MUL_OPS(UNIT) \ do { \ EXPECT_EQ(UNIT(5), UNIT(2) * 2.5); \ @@ -1240,6 +1261,11 @@ TEST(Duration, RoundTripUnits) { } TEST(Duration, TruncConversions) { +#if defined(ABSL_SKIP_TIME_TESTS_BROKEN_ON_MSVC_OPT) && \ + ABSL_SKIP_TIME_TESTS_BROKEN_ON_MSVC_OPT + GTEST_SKIP(); +#endif + // Tests ToTimespec()/DurationFromTimespec() const struct { absl::Duration d; @@ -1320,7 +1346,7 @@ TEST(Duration, SmallConversions) { EXPECT_EQ(absl::ZeroDuration(), absl::Seconds(0)); // TODO(bww): Is the next one OK? - EXPECT_EQ(absl::ZeroDuration(), absl::Seconds(0.124999999e-9)); + EXPECT_EQ(absl::ZeroDuration(), absl::Seconds(std::nextafter(0.125e-9, 0))); EXPECT_EQ(absl::Nanoseconds(1) / 4, absl::Seconds(0.125e-9)); EXPECT_EQ(absl::Nanoseconds(1) / 4, absl::Seconds(0.250e-9)); EXPECT_EQ(absl::Nanoseconds(1) / 2, absl::Seconds(0.375e-9)); @@ -1330,7 +1356,7 @@ TEST(Duration, SmallConversions) { EXPECT_EQ(absl::Nanoseconds(1), absl::Seconds(0.875e-9)); EXPECT_EQ(absl::Nanoseconds(1), absl::Seconds(1.000e-9)); - EXPECT_EQ(absl::ZeroDuration(), absl::Seconds(-0.124999999e-9)); + EXPECT_EQ(absl::ZeroDuration(), absl::Seconds(std::nextafter(-0.125e-9, 0))); EXPECT_EQ(-absl::Nanoseconds(1) / 4, absl::Seconds(-0.125e-9)); EXPECT_EQ(-absl::Nanoseconds(1) / 4, absl::Seconds(-0.250e-9)); EXPECT_EQ(-absl::Nanoseconds(1) / 2, absl::Seconds(-0.375e-9)); @@ -1390,6 +1416,14 @@ void VerifyApproxSameAsMul(double time_as_seconds, int* const misses) { // Seconds(point) returns a duration near point * Seconds(1.0). (They may // not be exactly equal due to fused multiply/add contraction.) TEST(Duration, ToDoubleSecondsCheckEdgeCases) { +#if (defined(__i386__) || defined(_M_IX86)) && FLT_EVAL_METHOD != 0 + // We're using an x87-compatible FPU, and intermediate operations can be + // performed with 80-bit floats. This means the edge cases are different than + // what we expect here, so just skip this test. + GTEST_SKIP() + << "Skipping the test because we detected x87 floating-point semantics"; +#endif + constexpr uint32_t kTicksPerSecond = absl::time_internal::kTicksPerSecond; constexpr auto duration_tick = absl::time_internal::MakeDuration(0, 1u); int misses = 0; @@ -1528,6 +1562,11 @@ TEST(Duration, ConversionSaturation) { } TEST(Duration, FormatDuration) { +#if defined(ABSL_SKIP_TIME_TESTS_BROKEN_ON_MSVC_OPT) && \ + ABSL_SKIP_TIME_TESTS_BROKEN_ON_MSVC_OPT + GTEST_SKIP(); +#endif + // Example from Go's docs. EXPECT_EQ("72h3m0.5s", absl::FormatDuration(absl::Hours(72) + absl::Minutes(3) + @@ -1662,6 +1701,11 @@ TEST(Duration, FormatDuration) { } TEST(Duration, ParseDuration) { +#if defined(ABSL_SKIP_TIME_TESTS_BROKEN_ON_MSVC_OPT) && \ + ABSL_SKIP_TIME_TESTS_BROKEN_ON_MSVC_OPT + GTEST_SKIP(); +#endif + absl::Duration d; // No specified unit. Should only work for zero and infinity. diff --git a/absl/time/internal/cctz/BUILD.bazel b/absl/time/internal/cctz/BUILD.bazel index 45a95292..7304d40d 100644 --- a/absl/time/internal/cctz/BUILD.bazel +++ b/absl/time/internal/cctz/BUILD.bazel @@ -12,8 +12,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") - package(features = ["-parse_headers"]) licenses(["notice"]) @@ -26,14 +24,14 @@ filegroup( config_setting( name = "osx", constraint_values = [ - "@bazel_tools//platforms:osx", + "@platforms//os:osx", ], ) config_setting( name = "ios", constraint_values = [ - "@bazel_tools//platforms:ios", + "@platforms//os:ios", ], ) @@ -87,7 +85,11 @@ cc_library( deps = [ ":civil_time", "//absl/base:config", - ], + ] + select( + { + "//conditions:default": [], + }, + ), ) ### tests @@ -117,6 +119,7 @@ cc_test( "no_test_android_arm", "no_test_android_arm64", "no_test_android_x86", + "no_test_wasm", ], deps = [ ":civil_time", @@ -136,6 +139,7 @@ cc_test( "no_test_android_arm", "no_test_android_arm64", "no_test_android_x86", + "no_test_wasm", ], deps = [ ":civil_time", diff --git a/absl/time/internal/cctz/include/cctz/civil_time_detail.h b/absl/time/internal/cctz/include/cctz/civil_time_detail.h index 8aadde57..a5b084e6 100644 --- a/absl/time/internal/cctz/include/cctz/civil_time_detail.h +++ b/absl/time/internal/cctz/include/cctz/civil_time_detail.h @@ -84,14 +84,13 @@ CONSTEXPR_F bool is_leap_year(year_t y) noexcept { return y % 4 == 0 && (y % 100 != 0 || y % 400 == 0); } CONSTEXPR_F int year_index(year_t y, month_t m) noexcept { - return (static_cast<int>((y + (m > 2)) % 400) + 400) % 400; + const int yi = static_cast<int>((y + (m > 2)) % 400); + return yi < 0 ? yi + 400 : yi; } -CONSTEXPR_F int days_per_century(year_t y, month_t m) noexcept { - const int yi = year_index(y, m); +CONSTEXPR_F int days_per_century(int yi) noexcept { return 36524 + (yi == 0 || yi > 300); } -CONSTEXPR_F int days_per_4years(year_t y, month_t m) noexcept { - const int yi = year_index(y, m); +CONSTEXPR_F int days_per_4years(int yi) noexcept { return 1460 + (yi == 0 || yi > 300 || (yi - 1) % 100 < 96); } CONSTEXPR_F int days_per_year(year_t y, month_t m) noexcept { @@ -133,17 +132,22 @@ CONSTEXPR_F fields n_day(year_t y, month_t m, diff_t d, diff_t cd, hour_t hh, } } if (d > 365) { + int yi = year_index(ey, m); // Index into Gregorian 400 year cycle. for (;;) { - int n = days_per_century(ey, m); + int n = days_per_century(yi); if (d <= n) break; d -= n; ey += 100; + yi += 100; + if (yi >= 400) yi -= 400; } for (;;) { - int n = days_per_4years(ey, m); + int n = days_per_4years(yi); if (d <= n) break; d -= n; ey += 4; + yi += 4; + if (yi >= 400) yi -= 400; } for (;;) { int n = days_per_year(ey, m); diff --git a/absl/time/internal/cctz/include/cctz/time_zone.h b/absl/time/internal/cctz/include/cctz/time_zone.h index 5562a37b..6e382dc6 100644 --- a/absl/time/internal/cctz/include/cctz/time_zone.h +++ b/absl/time/internal/cctz/include/cctz/time_zone.h @@ -22,6 +22,7 @@ #include <chrono> #include <cstdint> +#include <limits> #include <string> #include <utility> @@ -41,20 +42,9 @@ using sys_seconds = seconds; // Deprecated. Use cctz::seconds instead. namespace detail { template <typename D> -inline std::pair<time_point<seconds>, D> split_seconds( - const time_point<D>& tp) { - auto sec = std::chrono::time_point_cast<seconds>(tp); - auto sub = tp - sec; - if (sub.count() < 0) { - sec -= seconds(1); - sub += seconds(1); - } - return {sec, std::chrono::duration_cast<D>(sub)}; -} -inline std::pair<time_point<seconds>, seconds> split_seconds( - const time_point<seconds>& tp) { - return {tp, seconds::zero()}; -} +std::pair<time_point<seconds>, D> split_seconds(const time_point<D>& tp); +std::pair<time_point<seconds>, seconds> split_seconds( + const time_point<seconds>& tp); } // namespace detail // cctz::time_zone is an opaque, small, value-type class representing a @@ -279,6 +269,20 @@ std::string format(const std::string&, const time_point<seconds>&, const femtoseconds&, const time_zone&); bool parse(const std::string&, const std::string&, const time_zone&, time_point<seconds>*, femtoseconds*, std::string* err = nullptr); +template <typename Rep, std::intmax_t Denom> +bool join_seconds( + const time_point<seconds>& sec, const femtoseconds& fs, + time_point<std::chrono::duration<Rep, std::ratio<1, Denom>>>* tpp); +template <typename Rep, std::intmax_t Num> +bool join_seconds( + const time_point<seconds>& sec, const femtoseconds& fs, + time_point<std::chrono::duration<Rep, std::ratio<Num, 1>>>* tpp); +template <typename Rep> +bool join_seconds( + const time_point<seconds>& sec, const femtoseconds& fs, + time_point<std::chrono::duration<Rep, std::ratio<1, 1>>>* tpp); +bool join_seconds(const time_point<seconds>& sec, const femtoseconds&, + time_point<seconds>* tpp); } // namespace detail // Formats the given time_point in the given cctz::time_zone according to @@ -369,15 +373,84 @@ inline bool parse(const std::string& fmt, const std::string& input, const time_zone& tz, time_point<D>* tpp) { time_point<seconds> sec; detail::femtoseconds fs; - const bool b = detail::parse(fmt, input, tz, &sec, &fs); - if (b) { - // TODO: Return false if unrepresentable as a time_point<D>. - *tpp = std::chrono::time_point_cast<D>(sec); - *tpp += std::chrono::duration_cast<D>(fs); + return detail::parse(fmt, input, tz, &sec, &fs) && + detail::join_seconds(sec, fs, tpp); +} + +namespace detail { + +// Split a time_point<D> into a time_point<seconds> and a D subseconds. +// Undefined behavior if time_point<seconds> is not of sufficient range. +// Note that this means it is UB to call cctz::time_zone::lookup(tp) or +// cctz::format(fmt, tp, tz) with a time_point that is outside the range +// of a 64-bit std::time_t. +template <typename D> +std::pair<time_point<seconds>, D> split_seconds(const time_point<D>& tp) { + auto sec = std::chrono::time_point_cast<seconds>(tp); + auto sub = tp - sec; + if (sub.count() < 0) { + sec -= seconds(1); + sub += seconds(1); } - return b; + return {sec, std::chrono::duration_cast<D>(sub)}; +} + +inline std::pair<time_point<seconds>, seconds> split_seconds( + const time_point<seconds>& tp) { + return {tp, seconds::zero()}; } +// Join a time_point<seconds> and femto subseconds into a time_point<D>. +// Floors to the resolution of time_point<D>. Returns false if time_point<D> +// is not of sufficient range. +template <typename Rep, std::intmax_t Denom> +bool join_seconds( + const time_point<seconds>& sec, const femtoseconds& fs, + time_point<std::chrono::duration<Rep, std::ratio<1, Denom>>>* tpp) { + using D = std::chrono::duration<Rep, std::ratio<1, Denom>>; + // TODO(#199): Return false if result unrepresentable as a time_point<D>. + *tpp = std::chrono::time_point_cast<D>(sec); + *tpp += std::chrono::duration_cast<D>(fs); + return true; +} + +template <typename Rep, std::intmax_t Num> +bool join_seconds( + const time_point<seconds>& sec, const femtoseconds&, + time_point<std::chrono::duration<Rep, std::ratio<Num, 1>>>* tpp) { + using D = std::chrono::duration<Rep, std::ratio<Num, 1>>; + auto count = sec.time_since_epoch().count(); + if (count >= 0 || count % Num == 0) { + count /= Num; + } else { + count /= Num; + count -= 1; + } + if (count > (std::numeric_limits<Rep>::max)()) return false; + if (count < (std::numeric_limits<Rep>::min)()) return false; + *tpp = time_point<D>() + D{static_cast<Rep>(count)}; + return true; +} + +template <typename Rep> +bool join_seconds( + const time_point<seconds>& sec, const femtoseconds&, + time_point<std::chrono::duration<Rep, std::ratio<1, 1>>>* tpp) { + using D = std::chrono::duration<Rep, std::ratio<1, 1>>; + auto count = sec.time_since_epoch().count(); + if (count > (std::numeric_limits<Rep>::max)()) return false; + if (count < (std::numeric_limits<Rep>::min)()) return false; + *tpp = time_point<D>() + D{static_cast<Rep>(count)}; + return true; +} + +inline bool join_seconds(const time_point<seconds>& sec, const femtoseconds&, + time_point<seconds>* tpp) { + *tpp = sec; + return true; +} + +} // namespace detail } // namespace cctz } // namespace time_internal ABSL_NAMESPACE_END diff --git a/absl/time/internal/cctz/src/cctz_benchmark.cc b/absl/time/internal/cctz/src/cctz_benchmark.cc index 4e39188f..6770ad6b 100644 --- a/absl/time/internal/cctz/src/cctz_benchmark.cc +++ b/absl/time/internal/cctz/src/cctz_benchmark.cc @@ -648,6 +648,7 @@ const char* const kTimeZoneNames[] = {"Africa/Abidjan", "Pacific/Guam", "Pacific/Honolulu", "Pacific/Johnston", + "Pacific/Kanton", "Pacific/Kiritimati", "Pacific/Kosrae", "Pacific/Kwajalein", diff --git a/absl/time/internal/cctz/src/time_zone_fixed.cc b/absl/time/internal/cctz/src/time_zone_fixed.cc index 303c0244..f2b3294e 100644 --- a/absl/time/internal/cctz/src/time_zone_fixed.cc +++ b/absl/time/internal/cctz/src/time_zone_fixed.cc @@ -53,7 +53,7 @@ int Parse02d(const char* p) { } // namespace bool FixedOffsetFromName(const std::string& name, seconds* offset) { - if (name.compare(0, std::string::npos, "UTC", 3) == 0) { + if (name == "UTC" || name == "UTC0") { *offset = seconds::zero(); return true; } diff --git a/absl/time/internal/cctz/src/time_zone_format_test.cc b/absl/time/internal/cctz/src/time_zone_format_test.cc index a11f93e2..f1f79a20 100644 --- a/absl/time/internal/cctz/src/time_zone_format_test.cc +++ b/absl/time/internal/cctz/src/time_zone_format_test.cc @@ -13,15 +13,20 @@ // limitations under the License. #include <chrono> +#include <cstdint> #include <iomanip> #include <sstream> #include <string> +#include "absl/base/config.h" +#include "absl/time/internal/cctz/include/cctz/time_zone.h" +#if defined(__linux__) +#include <features.h> +#endif + #include "gmock/gmock.h" #include "gtest/gtest.h" -#include "absl/base/config.h" #include "absl/time/internal/cctz/include/cctz/civil_time.h" -#include "absl/time/internal/cctz/include/cctz/time_zone.h" namespace chrono = std::chrono; @@ -182,8 +187,10 @@ TEST(Format, PosixConversions) { TestFormatSpecifier(tp, tz, "%F", "1970-01-01"); TestFormatSpecifier(tp, tz, "%g", "70"); TestFormatSpecifier(tp, tz, "%G", "1970"); +#if defined(__GLIBC__) TestFormatSpecifier(tp, tz, "%k", " 0"); TestFormatSpecifier(tp, tz, "%l", "12"); +#endif TestFormatSpecifier(tp, tz, "%n", "\n"); TestFormatSpecifier(tp, tz, "%R", "00:00"); TestFormatSpecifier(tp, tz, "%t", "\t"); @@ -215,7 +222,9 @@ TEST(Format, LocaleSpecific) { #if defined(__linux__) // SU/C99/TZ extensions TestFormatSpecifier(tp, tz, "%h", "Jan"); // Same as %b +#if defined(__GLIBC__) TestFormatSpecifier(tp, tz, "%P", "am"); +#endif TestFormatSpecifier(tp, tz, "%r", "12:00:00 AM"); // Modified conversion specifiers %E_ @@ -1044,9 +1053,11 @@ TEST(Parse, LocaleSpecific) { EXPECT_TRUE(parse("%h", "Feb", tz, &tp)); EXPECT_EQ(2, convert(tp, tz).month()); // Equivalent to %b +#if defined(__GLIBC__) tp = reset; EXPECT_TRUE(parse("%l %p", "5 PM", tz, &tp)); EXPECT_EQ(17, convert(tp, tz).hour()); +#endif tp = reset; EXPECT_TRUE(parse("%r", "03:44:55 PM", tz, &tp)); @@ -1054,6 +1065,7 @@ TEST(Parse, LocaleSpecific) { EXPECT_EQ(44, convert(tp, tz).minute()); EXPECT_EQ(55, convert(tp, tz).second()); +#if defined(__GLIBC__) tp = reset; EXPECT_TRUE(parse("%Ec", "Tue Nov 19 05:06:07 2013", tz, &tp)); EXPECT_EQ(convert(civil_second(2013, 11, 19, 5, 6, 7), tz), tp); @@ -1125,6 +1137,7 @@ TEST(Parse, LocaleSpecific) { EXPECT_TRUE(parse("%Oy", "04", tz, &tp)); EXPECT_EQ(2004, convert(tp, tz).year()); #endif +#endif } TEST(Parse, ExtendedSeconds) { @@ -1135,7 +1148,7 @@ TEST(Parse, ExtendedSeconds) { // All %E<prec>S cases are treated the same as %E*S on input. auto precisions = {"*", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15"}; - for (const std::string& prec : precisions) { + for (const std::string prec : precisions) { const std::string fmt = "%E" + prec + "S"; SCOPED_TRACE(fmt); time_point<chrono::nanoseconds> tp = unix_epoch; @@ -1217,7 +1230,7 @@ TEST(Parse, ExtendedSubeconds) { // All %E<prec>f cases are treated the same as %E*f on input. auto precisions = {"*", "0", "1", "2", "3", "4", "5", "6", "7", "8", "9", "10", "11", "12", "13", "14", "15"}; - for (const std::string& prec : precisions) { + for (const std::string prec : precisions) { const std::string fmt = "%E" + prec + "f"; SCOPED_TRACE(fmt); time_point<chrono::nanoseconds> tp = unix_epoch - chrono::seconds(1); @@ -1504,7 +1517,7 @@ TEST(Parse, MaxRange) { parse(RFC3339_sec, "292277026596-12-04T14:30:07-01:00", utc, &tp)); EXPECT_EQ(tp, time_point<absl::time_internal::cctz::seconds>::max()); EXPECT_FALSE( - parse(RFC3339_sec, "292277026596-12-04T15:30:07-01:00", utc, &tp)); + parse(RFC3339_sec, "292277026596-12-04T14:30:08-01:00", utc, &tp)); // tests the lower limit using +00:00 offset EXPECT_TRUE( @@ -1525,10 +1538,82 @@ TEST(Parse, MaxRange) { parse(RFC3339_sec, "9223372036854775807-12-31T23:59:59-00:01", utc, &tp)); EXPECT_FALSE(parse(RFC3339_sec, "-9223372036854775808-01-01T00:00:00+00:01", utc, &tp)); +} + +TEST(Parse, TimePointOverflow) { + const time_zone utc = utc_time_zone(); + + using D = chrono::duration<std::int64_t, std::nano>; + time_point<D> tp; + + EXPECT_TRUE( + parse(RFC3339_full, "2262-04-11T23:47:16.8547758079+00:00", utc, &tp)); + EXPECT_EQ(tp, time_point<D>::max()); + EXPECT_EQ("2262-04-11T23:47:16.854775807+00:00", + format(RFC3339_full, tp, utc)); +#if 0 + // TODO(#199): Will fail until cctz::parse() properly detects overflow. + EXPECT_FALSE( + parse(RFC3339_full, "2262-04-11T23:47:16.8547758080+00:00", utc, &tp)); + EXPECT_TRUE( + parse(RFC3339_full, "1677-09-21T00:12:43.1452241920+00:00", utc, &tp)); + EXPECT_EQ(tp, time_point<D>::min()); + EXPECT_EQ("1677-09-21T00:12:43.145224192+00:00", + format(RFC3339_full, tp, utc)); + EXPECT_FALSE( + parse(RFC3339_full, "1677-09-21T00:12:43.1452241919+00:00", utc, &tp)); +#endif + + using DS = chrono::duration<std::int8_t, chrono::seconds::period>; + time_point<DS> stp; + + EXPECT_TRUE(parse(RFC3339_full, "1970-01-01T00:02:07.9+00:00", utc, &stp)); + EXPECT_EQ(stp, time_point<DS>::max()); + EXPECT_EQ("1970-01-01T00:02:07+00:00", format(RFC3339_full, stp, utc)); + EXPECT_FALSE(parse(RFC3339_full, "1970-01-01T00:02:08+00:00", utc, &stp)); + + EXPECT_TRUE(parse(RFC3339_full, "1969-12-31T23:57:52+00:00", utc, &stp)); + EXPECT_EQ(stp, time_point<DS>::min()); + EXPECT_EQ("1969-12-31T23:57:52+00:00", format(RFC3339_full, stp, utc)); + EXPECT_FALSE(parse(RFC3339_full, "1969-12-31T23:57:51.9+00:00", utc, &stp)); + + using DM = chrono::duration<std::int8_t, chrono::minutes::period>; + time_point<DM> mtp; + + EXPECT_TRUE(parse(RFC3339_full, "1970-01-01T02:07:59+00:00", utc, &mtp)); + EXPECT_EQ(mtp, time_point<DM>::max()); + EXPECT_EQ("1970-01-01T02:07:00+00:00", format(RFC3339_full, mtp, utc)); + EXPECT_FALSE(parse(RFC3339_full, "1970-01-01T02:08:00+00:00", utc, &mtp)); + + EXPECT_TRUE(parse(RFC3339_full, "1969-12-31T21:52:00+00:00", utc, &mtp)); + EXPECT_EQ(mtp, time_point<DM>::min()); + EXPECT_EQ("1969-12-31T21:52:00+00:00", format(RFC3339_full, mtp, utc)); + EXPECT_FALSE(parse(RFC3339_full, "1969-12-31T21:51:59+00:00", utc, &mtp)); +} - // TODO: Add tests that parsing times with fractional seconds overflow - // appropriately. This can't be done until cctz::parse() properly detects - // overflow when combining the chrono seconds and femto. +TEST(Parse, TimePointOverflowFloor) { + const time_zone utc = utc_time_zone(); + + using D = chrono::duration<std::int64_t, std::micro>; + time_point<D> tp; + + EXPECT_TRUE( + parse(RFC3339_full, "294247-01-10T04:00:54.7758079+00:00", utc, &tp)); + EXPECT_EQ(tp, time_point<D>::max()); + EXPECT_EQ("294247-01-10T04:00:54.775807+00:00", + format(RFC3339_full, tp, utc)); +#if 0 + // TODO(#199): Will fail until cctz::parse() properly detects overflow. + EXPECT_FALSE( + parse(RFC3339_full, "294247-01-10T04:00:54.7758080+00:00", utc, &tp)); + EXPECT_TRUE( + parse(RFC3339_full, "-290308-12-21T19:59:05.2241920+00:00", utc, &tp)); + EXPECT_EQ(tp, time_point<D>::min()); + EXPECT_EQ("-290308-12-21T19:59:05.224192+00:00", + format(RFC3339_full, tp, utc)); + EXPECT_FALSE( + parse(RFC3339_full, "-290308-12-21T19:59:05.2241919+00:00", utc, &tp)); +#endif } // diff --git a/absl/time/internal/cctz/src/time_zone_if.h b/absl/time/internal/cctz/src/time_zone_if.h index 32c0891c..7d3e42d3 100644 --- a/absl/time/internal/cctz/src/time_zone_if.h +++ b/absl/time/internal/cctz/src/time_zone_if.h @@ -56,7 +56,8 @@ class TimeZoneIf { // Convert between time_point<seconds> and a count of seconds since the // Unix epoch. We assume that the std::chrono::system_clock and the -// Unix clock are second aligned, but not that they share an epoch. +// Unix clock are second aligned, and that the results are representable. +// (That is, that they share an epoch, which is required since C++20.) inline std::int_fast64_t ToUnixSeconds(const time_point<seconds>& tp) { return (tp - std::chrono::time_point_cast<seconds>( std::chrono::system_clock::from_time_t(0))) diff --git a/absl/time/internal/cctz/src/time_zone_info.cc b/absl/time/internal/cctz/src/time_zone_info.cc index 8039353e..4f175d95 100644 --- a/absl/time/internal/cctz/src/time_zone_info.cc +++ b/absl/time/internal/cctz/src/time_zone_info.cc @@ -39,10 +39,12 @@ #include <cstdio> #include <cstdlib> #include <cstring> +#include <fstream> #include <functional> #include <memory> #include <sstream> #include <string> +#include <utility> #include "absl/base/config.h" #include "absl/time/internal/cctz/include/cctz/civil_time.h" @@ -576,14 +578,17 @@ bool TimeZoneInfo::Load(ZoneInfoSource* zip) { namespace { +using FilePtr = std::unique_ptr<FILE, int (*)(FILE*)>; + // fopen(3) adaptor. -inline FILE* FOpen(const char* path, const char* mode) { +inline FilePtr FOpen(const char* path, const char* mode) { #if defined(_MSC_VER) FILE* fp; if (fopen_s(&fp, path, mode) != 0) fp = nullptr; - return fp; + return FilePtr(fp, fclose); #else - return fopen(path, mode); // TODO: Enable the close-on-exec flag. + // TODO: Enable the close-on-exec flag. + return FilePtr(fopen(path, mode), fclose); #endif } @@ -611,11 +616,11 @@ class FileZoneInfoSource : public ZoneInfoSource { protected: explicit FileZoneInfoSource( - FILE* fp, std::size_t len = std::numeric_limits<std::size_t>::max()) - : fp_(fp, fclose), len_(len) {} + FilePtr fp, std::size_t len = std::numeric_limits<std::size_t>::max()) + : fp_(std::move(fp)), len_(len) {} private: - std::unique_ptr<FILE, int (*)(FILE*)> fp_; + FilePtr fp_; std::size_t len_; }; @@ -644,17 +649,9 @@ std::unique_ptr<ZoneInfoSource> FileZoneInfoSource::Open( path.append(name, pos, std::string::npos); // Open the zoneinfo file. - FILE* fp = FOpen(path.c_str(), "rb"); + auto fp = FOpen(path.c_str(), "rb"); if (fp == nullptr) return nullptr; - std::size_t length = 0; - if (fseek(fp, 0, SEEK_END) == 0) { - long offset = ftell(fp); - if (offset >= 0) { - length = static_cast<std::size_t>(offset); - } - rewind(fp); - } - return std::unique_ptr<ZoneInfoSource>(new FileZoneInfoSource(fp, length)); + return std::unique_ptr<ZoneInfoSource>(new FileZoneInfoSource(std::move(fp))); } class AndroidZoneInfoSource : public FileZoneInfoSource { @@ -663,8 +660,9 @@ class AndroidZoneInfoSource : public FileZoneInfoSource { std::string Version() const override { return version_; } private: - explicit AndroidZoneInfoSource(FILE* fp, std::size_t len, const char* vers) - : FileZoneInfoSource(fp, len), version_(vers) {} + explicit AndroidZoneInfoSource(FilePtr fp, std::size_t len, + std::string version) + : FileZoneInfoSource(std::move(fp), len), version_(std::move(version)) {} std::string version_; }; @@ -676,8 +674,8 @@ std::unique_ptr<ZoneInfoSource> AndroidZoneInfoSource::Open( // See Android's libc/tzcode/bionic.cpp for additional information. for (const char* tzdata : {"/data/misc/zoneinfo/current/tzdata", "/system/usr/share/zoneinfo/tzdata"}) { - std::unique_ptr<FILE, int (*)(FILE*)> fp(FOpen(tzdata, "rb"), fclose); - if (fp.get() == nullptr) continue; + auto fp = FOpen(tzdata, "rb"); + if (fp == nullptr) continue; char hbuf[24]; // covers header.zonetab_offset too if (fread(hbuf, 1, sizeof(hbuf), fp.get()) != sizeof(hbuf)) continue; @@ -703,7 +701,7 @@ std::unique_ptr<ZoneInfoSource> AndroidZoneInfoSource::Open( if (strcmp(name.c_str() + pos, ebuf) == 0) { if (fseek(fp.get(), static_cast<long>(start), SEEK_SET) != 0) break; return std::unique_ptr<ZoneInfoSource>(new AndroidZoneInfoSource( - fp.release(), static_cast<std::size_t>(length), vers)); + std::move(fp), static_cast<std::size_t>(length), vers)); } } } @@ -711,6 +709,69 @@ std::unique_ptr<ZoneInfoSource> AndroidZoneInfoSource::Open( return nullptr; } +// A zoneinfo source for use inside Fuchsia components. This attempts to +// read zoneinfo files from one of several known paths in a component's +// incoming namespace. [Config data][1] is preferred, but package-specific +// resources are also supported. +// +// Fuchsia's implementation supports `FileZoneInfoSource::Version()`. +// +// [1]: +// https://fuchsia.dev/fuchsia-src/development/components/data#using_config_data_in_your_component +class FuchsiaZoneInfoSource : public FileZoneInfoSource { + public: + static std::unique_ptr<ZoneInfoSource> Open(const std::string& name); + std::string Version() const override { return version_; } + + private: + explicit FuchsiaZoneInfoSource(FilePtr fp, std::string version) + : FileZoneInfoSource(std::move(fp)), version_(std::move(version)) {} + std::string version_; +}; + +std::unique_ptr<ZoneInfoSource> FuchsiaZoneInfoSource::Open( + const std::string& name) { + // Use of the "file:" prefix is intended for testing purposes only. + const std::size_t pos = (name.compare(0, 5, "file:") == 0) ? 5 : 0; + + // Prefixes where a Fuchsia component might find zoneinfo files, + // in descending order of preference. + const auto kTzdataPrefixes = { + "/config/data/tzdata/", + "/pkg/data/tzdata/", + "/data/tzdata/", + }; + const auto kEmptyPrefix = {""}; + const bool name_absolute = (pos != name.size() && name[pos] == '/'); + const auto prefixes = name_absolute ? kEmptyPrefix : kTzdataPrefixes; + + // Fuchsia builds place zoneinfo files at "<prefix><format><name>". + for (const std::string prefix : prefixes) { + std::string path = prefix; + if (!prefix.empty()) path += "zoneinfo/tzif2/"; // format + path.append(name, pos, std::string::npos); + + auto fp = FOpen(path.c_str(), "rb"); + if (fp == nullptr) continue; + + std::string version; + if (!prefix.empty()) { + // Fuchsia builds place the version in "<prefix>revision.txt". + std::ifstream version_stream(prefix + "revision.txt"); + if (version_stream.is_open()) { + // revision.txt should contain no newlines, but to be + // defensive we read just the first line. + std::getline(version_stream, version); + } + } + + return std::unique_ptr<ZoneInfoSource>( + new FuchsiaZoneInfoSource(std::move(fp), std::move(version))); + } + + return nullptr; +} + } // namespace bool TimeZoneInfo::Load(const std::string& name) { @@ -728,6 +789,7 @@ bool TimeZoneInfo::Load(const std::string& name) { name, [](const std::string& n) -> std::unique_ptr<ZoneInfoSource> { if (auto z = FileZoneInfoSource::Open(n)) return z; if (auto z = AndroidZoneInfoSource::Open(n)) return z; + if (auto z = FuchsiaZoneInfoSource::Open(n)) return z; return nullptr; }); return zip != nullptr && Load(zip.get()); diff --git a/absl/time/internal/cctz/src/time_zone_lookup.cc b/absl/time/internal/cctz/src/time_zone_lookup.cc index efdea64b..898d04c1 100644 --- a/absl/time/internal/cctz/src/time_zone_lookup.cc +++ b/absl/time/internal/cctz/src/time_zone_lookup.cc @@ -28,6 +28,13 @@ #include <vector> #endif +#if defined(__Fuchsia__) +#include <fuchsia/intl/cpp/fidl.h> +#include <lib/async-loop/cpp/loop.h> +#include <lib/sys/cpp/component_context.h> +#include <zircon/types.h> +#endif + #include <cstdlib> #include <cstring> #include <string> @@ -140,6 +147,48 @@ time_zone local_time_zone() { } CFRelease(tz_default); #endif +#if defined(__Fuchsia__) + std::string primary_tz; + [&]() { + // Note: We can't use the synchronous FIDL API here because it doesn't + // allow timeouts; if the FIDL call failed, local_time_zone() would never + // return. + + const zx::duration kTimeout = zx::msec(500); + + // Don't attach to the thread because otherwise the thread's dispatcher + // would be set to null when the loop is destroyed, causing any other FIDL + // code running on the same thread to crash. + async::Loop loop(&kAsyncLoopConfigNeverAttachToThread); + std::unique_ptr<sys::ComponentContext> context = + sys::ComponentContext::Create(); + + fuchsia::intl::PropertyProviderHandle handle; + zx_status_t status = context->svc()->Connect(handle.NewRequest()); + if (status != ZX_OK) { + return; + } + + fuchsia::intl::PropertyProviderPtr intl_provider; + status = intl_provider.Bind(std::move(handle), loop.dispatcher()); + if (status != ZX_OK) { + return; + } + + intl_provider->GetProfile( + [&loop, &primary_tz](fuchsia::intl::Profile profile) { + if (!profile.time_zones().empty()) { + primary_tz = profile.time_zones()[0].id; + } + loop.Quit(); + }); + loop.Run(zx::deadline_after(kTimeout)); + }(); + + if (!primary_tz.empty()) { + zone = primary_tz.c_str(); + } +#endif // Allow ${TZ} to override to default zone. char* tz_env = nullptr; diff --git a/absl/time/internal/cctz/src/time_zone_lookup_test.cc b/absl/time/internal/cctz/src/time_zone_lookup_test.cc index 9a1a8d6e..27a53c5c 100644 --- a/absl/time/internal/cctz/src/time_zone_lookup_test.cc +++ b/absl/time/internal/cctz/src/time_zone_lookup_test.cc @@ -21,10 +21,14 @@ #include <thread> #include <vector> -#include "gtest/gtest.h" #include "absl/base/config.h" -#include "absl/time/internal/cctz/include/cctz/civil_time.h" #include "absl/time/internal/cctz/include/cctz/time_zone.h" +#if defined(__linux__) +#include <features.h> +#endif + +#include "gtest/gtest.h" +#include "absl/time/internal/cctz/include/cctz/civil_time.h" namespace chrono = std::chrono; @@ -579,6 +583,7 @@ const char* const kTimeZoneNames[] = {"Africa/Abidjan", "Pacific/Guam", "Pacific/Honolulu", "Pacific/Johnston", + "Pacific/Kanton", "Pacific/Kiritimati", "Pacific/Kosrae", "Pacific/Kwajalein", @@ -717,6 +722,18 @@ TEST(TimeZones, LoadZonesConcurrently) { } #endif +TEST(TimeZone, UTC) { + const time_zone utc = utc_time_zone(); + + time_zone loaded_utc; + EXPECT_TRUE(load_time_zone("UTC", &loaded_utc)); + EXPECT_EQ(loaded_utc, utc); + + time_zone loaded_utc0; + EXPECT_TRUE(load_time_zone("UTC0", &loaded_utc0)); + EXPECT_EQ(loaded_utc0, utc); +} + TEST(TimeZone, NamedTimeZones) { const time_zone utc = utc_time_zone(); EXPECT_EQ("UTC", utc.name()); @@ -1014,8 +1031,12 @@ TEST(MakeTime, SysSecondsLimits) { #endif const year_t min_tm_year = year_t{std::numeric_limits<int>::min()} + 1900; tp = convert(civil_second(min_tm_year, 1, 1, 0, 0, 0), cut); +#if defined(__Fuchsia__) + // Fuchsia's gmtime_r() fails on extreme negative values (fxbug.dev/78527). +#else EXPECT_EQ("-2147481748-01-01T00:00:00+00:00", format(RFC3339, tp, cut)); #endif +#endif } } @@ -1026,7 +1047,7 @@ TEST(MakeTime, LocalTimeLibC) { // 1) we know how to change the time zone used by localtime()/mktime(), // 2) cctz and localtime()/mktime() will use similar-enough tzdata, and // 3) we have some idea about how mktime() behaves during transitions. -#if defined(__linux__) && !defined(__ANDROID__) +#if defined(__linux__) && defined(__GLIBC__) && !defined(__ANDROID__) const char* const ep = getenv("TZ"); std::string tz_name = (ep != nullptr) ? ep : ""; for (const char* const* np = kTimeZoneNames; *np != nullptr; ++np) { @@ -1165,6 +1186,44 @@ TEST(PrevTransition, AmericaNewYork) { // We have a transition but we don't know which one. } +TEST(NextTransition, Scan) { + for (const char* const* np = kTimeZoneNames; *np != nullptr; ++np) { + time_zone tz; + if (!load_time_zone(*np, &tz)) { + continue; // tolerate kTimeZoneNames/zoneinfo skew + } + SCOPED_TRACE(testing::Message() << "In " << *np); + + auto tp = time_point<absl::time_internal::cctz::seconds>::min(); + time_zone::civil_transition trans; + while (tz.next_transition(tp, &trans)) { + time_zone::civil_lookup from_cl = tz.lookup(trans.from); + EXPECT_NE(from_cl.kind, time_zone::civil_lookup::REPEATED); + time_zone::civil_lookup to_cl = tz.lookup(trans.to); + EXPECT_NE(to_cl.kind, time_zone::civil_lookup::SKIPPED); + + auto trans_tp = to_cl.trans; + time_zone::absolute_lookup trans_al = tz.lookup(trans_tp); + EXPECT_EQ(trans_al.cs, trans.to); + auto pre_trans_tp = trans_tp - absl::time_internal::cctz::seconds(1); + time_zone::absolute_lookup pre_trans_al = tz.lookup(pre_trans_tp); + EXPECT_EQ(pre_trans_al.cs + 1, trans.from); + + auto offset_delta = trans_al.offset - pre_trans_al.offset; + EXPECT_EQ(offset_delta, trans.to - trans.from); + if (offset_delta == 0) { + // This "transition" is only an is_dst or abbr change. + EXPECT_EQ(to_cl.kind, time_zone::civil_lookup::UNIQUE); + if (trans_al.is_dst == pre_trans_al.is_dst) { + EXPECT_STRNE(trans_al.abbr, pre_trans_al.abbr); + } + } + + tp = trans_tp; // continue scan from transition + } + } +} + TEST(TimeZoneEdgeCase, AmericaNewYork) { const time_zone tz = LoadZone("America/New_York"); diff --git a/absl/time/internal/cctz/src/tzfile.h b/absl/time/internal/cctz/src/tzfile.h index 269fa36c..31e85982 100644 --- a/absl/time/internal/cctz/src/tzfile.h +++ b/absl/time/internal/cctz/src/tzfile.h @@ -43,7 +43,7 @@ struct tzhead { char tzh_magic[4]; /* TZ_MAGIC */ - char tzh_version[1]; /* '\0' or '2' or '3' as of 2013 */ + char tzh_version[1]; /* '\0' or '2'-'4' as of 2021 */ char tzh_reserved[15]; /* reserved; must be zero */ char tzh_ttisutcnt[4]; /* coded number of trans. time flags */ char tzh_ttisstdcnt[4]; /* coded number of trans. time flags */ diff --git a/absl/time/internal/cctz/src/zone_info_source.cc b/absl/time/internal/cctz/src/zone_info_source.cc index 72095339..5ab5a59e 100644 --- a/absl/time/internal/cctz/src/zone_info_source.cc +++ b/absl/time/internal/cctz/src/zone_info_source.cc @@ -65,7 +65,7 @@ ZoneInfoSourceFactory zone_info_source_factory __attribute__((weak)) = extern ZoneInfoSourceFactory zone_info_source_factory; extern ZoneInfoSourceFactory default_factory; ZoneInfoSourceFactory default_factory = DefaultFactory; -#if defined(_M_IX86) +#if defined(_M_IX86) || defined(_M_ARM) #pragma comment( \ linker, \ "/alternatename:?zone_info_source_factory@cctz_extension@time_internal@" ABSL_INTERNAL_MANGLED_NS \ @@ -83,8 +83,7 @@ ZoneInfoSourceFactory default_factory = DefaultFactory; "@@U?$default_delete@VZoneInfoSource@cctz@time_internal@" ABSL_INTERNAL_MANGLED_NS \ "@@@std@@@std@@ABV?$basic_string@DU?$char_traits@D@std@@V?$allocator@D@2@@2@@Z@" ABSL_INTERNAL_MANGLED_BACKREFERENCE \ "@@ZA") -#elif defined(_M_IA_64) || defined(_M_AMD64) || defined(_M_ARM) || \ - defined(_M_ARM64) +#elif defined(_M_IA_64) || defined(_M_AMD64) || defined(_M_ARM64) #pragma comment( \ linker, \ "/alternatename:?zone_info_source_factory@cctz_extension@time_internal@" ABSL_INTERNAL_MANGLED_NS \ diff --git a/absl/time/internal/cctz/testdata/README.zoneinfo b/absl/time/internal/cctz/testdata/README.zoneinfo index 95fb4a91..a41c7b88 100644 --- a/absl/time/internal/cctz/testdata/README.zoneinfo +++ b/absl/time/internal/cctz/testdata/README.zoneinfo @@ -13,7 +13,12 @@ New versions can be generated using the following shell script. trap "rm -fr ${DESTDIR}" 0 2 15 ( cd ${DESTDIR} - git clone https://github.com/eggert/tz.git + if [ -n "${USE_GLOBAL_TZ}" ] + then + git clone -b global-tz https://github.com/JodaOrg/global-tz.git tz + else + git clone https://github.com/eggert/tz.git + fi make --directory=tz \ install DESTDIR=${DESTDIR} \ DATAFORM=vanguard \ diff --git a/absl/time/internal/cctz/testdata/version b/absl/time/internal/cctz/testdata/version index 1d590958..ca002de2 100644 --- a/absl/time/internal/cctz/testdata/version +++ b/absl/time/internal/cctz/testdata/version @@ -1 +1 @@ -2021a +2022a diff --git a/absl/time/internal/cctz/testdata/zoneinfo/Africa/Accra b/absl/time/internal/cctz/testdata/zoneinfo/Africa/Accra Binary files differindex c39ae382..8906e88c 100644 --- a/absl/time/internal/cctz/testdata/zoneinfo/Africa/Accra +++ b/absl/time/internal/cctz/testdata/zoneinfo/Africa/Accra diff --git a/absl/time/internal/cctz/testdata/zoneinfo/America/Anguilla b/absl/time/internal/cctz/testdata/zoneinfo/America/Anguilla Binary files differindex f4fe5903..47b4dc34 100644 --- a/absl/time/internal/cctz/testdata/zoneinfo/America/Anguilla +++ b/absl/time/internal/cctz/testdata/zoneinfo/America/Anguilla diff --git a/absl/time/internal/cctz/testdata/zoneinfo/America/Antigua b/absl/time/internal/cctz/testdata/zoneinfo/America/Antigua Binary files differindex f4fe5903..47b4dc34 100644 --- a/absl/time/internal/cctz/testdata/zoneinfo/America/Antigua +++ b/absl/time/internal/cctz/testdata/zoneinfo/America/Antigua diff --git a/absl/time/internal/cctz/testdata/zoneinfo/America/Aruba b/absl/time/internal/cctz/testdata/zoneinfo/America/Aruba Binary files differindex d6ddf7d8..47b4dc34 100644 --- a/absl/time/internal/cctz/testdata/zoneinfo/America/Aruba +++ b/absl/time/internal/cctz/testdata/zoneinfo/America/Aruba diff --git a/absl/time/internal/cctz/testdata/zoneinfo/America/Atikokan b/absl/time/internal/cctz/testdata/zoneinfo/America/Atikokan Binary files differindex c8287152..9154643f 100644 --- a/absl/time/internal/cctz/testdata/zoneinfo/America/Atikokan +++ b/absl/time/internal/cctz/testdata/zoneinfo/America/Atikokan diff --git a/absl/time/internal/cctz/testdata/zoneinfo/America/Barbados b/absl/time/internal/cctz/testdata/zoneinfo/America/Barbados Binary files differindex 9d3afa6a..720c9863 100644 --- a/absl/time/internal/cctz/testdata/zoneinfo/America/Barbados +++ b/absl/time/internal/cctz/testdata/zoneinfo/America/Barbados diff --git a/absl/time/internal/cctz/testdata/zoneinfo/America/Blanc-Sablon b/absl/time/internal/cctz/testdata/zoneinfo/America/Blanc-Sablon Binary files differindex 7096b69a..47b4dc34 100644 --- a/absl/time/internal/cctz/testdata/zoneinfo/America/Blanc-Sablon +++ b/absl/time/internal/cctz/testdata/zoneinfo/America/Blanc-Sablon diff --git a/absl/time/internal/cctz/testdata/zoneinfo/America/Coral_Harbour b/absl/time/internal/cctz/testdata/zoneinfo/America/Coral_Harbour Binary files differindex c8287152..9154643f 100644 --- a/absl/time/internal/cctz/testdata/zoneinfo/America/Coral_Harbour +++ b/absl/time/internal/cctz/testdata/zoneinfo/America/Coral_Harbour diff --git a/absl/time/internal/cctz/testdata/zoneinfo/America/Creston b/absl/time/internal/cctz/testdata/zoneinfo/America/Creston Binary files differindex 9d69a0ab..c2bd2f94 100644 --- a/absl/time/internal/cctz/testdata/zoneinfo/America/Creston +++ b/absl/time/internal/cctz/testdata/zoneinfo/America/Creston diff --git a/absl/time/internal/cctz/testdata/zoneinfo/America/Curacao b/absl/time/internal/cctz/testdata/zoneinfo/America/Curacao Binary files differindex d6ddf7d8..47b4dc34 100644 --- a/absl/time/internal/cctz/testdata/zoneinfo/America/Curacao +++ b/absl/time/internal/cctz/testdata/zoneinfo/America/Curacao diff --git a/absl/time/internal/cctz/testdata/zoneinfo/America/Dominica b/absl/time/internal/cctz/testdata/zoneinfo/America/Dominica Binary files differindex f4fe5903..47b4dc34 100644 --- a/absl/time/internal/cctz/testdata/zoneinfo/America/Dominica +++ b/absl/time/internal/cctz/testdata/zoneinfo/America/Dominica diff --git a/absl/time/internal/cctz/testdata/zoneinfo/America/Grenada b/absl/time/internal/cctz/testdata/zoneinfo/America/Grenada Binary files differindex f4fe5903..47b4dc34 100644 --- a/absl/time/internal/cctz/testdata/zoneinfo/America/Grenada +++ b/absl/time/internal/cctz/testdata/zoneinfo/America/Grenada diff --git a/absl/time/internal/cctz/testdata/zoneinfo/America/Guadeloupe b/absl/time/internal/cctz/testdata/zoneinfo/America/Guadeloupe Binary files differindex f4fe5903..47b4dc34 100644 --- a/absl/time/internal/cctz/testdata/zoneinfo/America/Guadeloupe +++ b/absl/time/internal/cctz/testdata/zoneinfo/America/Guadeloupe diff --git a/absl/time/internal/cctz/testdata/zoneinfo/America/Guyana b/absl/time/internal/cctz/testdata/zoneinfo/America/Guyana Binary files differindex ebd85d0f..bcc66881 100644 --- a/absl/time/internal/cctz/testdata/zoneinfo/America/Guyana +++ b/absl/time/internal/cctz/testdata/zoneinfo/America/Guyana diff --git a/absl/time/internal/cctz/testdata/zoneinfo/America/Kralendijk b/absl/time/internal/cctz/testdata/zoneinfo/America/Kralendijk Binary files differindex d6ddf7d8..47b4dc34 100644 --- a/absl/time/internal/cctz/testdata/zoneinfo/America/Kralendijk +++ b/absl/time/internal/cctz/testdata/zoneinfo/America/Kralendijk diff --git a/absl/time/internal/cctz/testdata/zoneinfo/America/Lower_Princes b/absl/time/internal/cctz/testdata/zoneinfo/America/Lower_Princes Binary files differindex d6ddf7d8..47b4dc34 100644 --- a/absl/time/internal/cctz/testdata/zoneinfo/America/Lower_Princes +++ b/absl/time/internal/cctz/testdata/zoneinfo/America/Lower_Princes diff --git a/absl/time/internal/cctz/testdata/zoneinfo/America/Marigot b/absl/time/internal/cctz/testdata/zoneinfo/America/Marigot Binary files differindex f4fe5903..47b4dc34 100644 --- a/absl/time/internal/cctz/testdata/zoneinfo/America/Marigot +++ b/absl/time/internal/cctz/testdata/zoneinfo/America/Marigot diff --git a/absl/time/internal/cctz/testdata/zoneinfo/America/Montserrat b/absl/time/internal/cctz/testdata/zoneinfo/America/Montserrat Binary files differindex f4fe5903..47b4dc34 100644 --- a/absl/time/internal/cctz/testdata/zoneinfo/America/Montserrat +++ b/absl/time/internal/cctz/testdata/zoneinfo/America/Montserrat diff --git a/absl/time/internal/cctz/testdata/zoneinfo/America/Nassau b/absl/time/internal/cctz/testdata/zoneinfo/America/Nassau Binary files differindex 2ef2aa89..fe6be8ea 100644 --- a/absl/time/internal/cctz/testdata/zoneinfo/America/Nassau +++ b/absl/time/internal/cctz/testdata/zoneinfo/America/Nassau diff --git a/absl/time/internal/cctz/testdata/zoneinfo/America/Port_of_Spain b/absl/time/internal/cctz/testdata/zoneinfo/America/Port_of_Spain Binary files differindex f4fe5903..47b4dc34 100644 --- a/absl/time/internal/cctz/testdata/zoneinfo/America/Port_of_Spain +++ b/absl/time/internal/cctz/testdata/zoneinfo/America/Port_of_Spain diff --git a/absl/time/internal/cctz/testdata/zoneinfo/America/Punta_Arenas b/absl/time/internal/cctz/testdata/zoneinfo/America/Punta_Arenas Binary files differindex 5c9a20b9..c0421040 100644 --- a/absl/time/internal/cctz/testdata/zoneinfo/America/Punta_Arenas +++ b/absl/time/internal/cctz/testdata/zoneinfo/America/Punta_Arenas diff --git a/absl/time/internal/cctz/testdata/zoneinfo/America/Santiago b/absl/time/internal/cctz/testdata/zoneinfo/America/Santiago Binary files differindex 8d603226..cde8dbbf 100644 --- a/absl/time/internal/cctz/testdata/zoneinfo/America/Santiago +++ b/absl/time/internal/cctz/testdata/zoneinfo/America/Santiago diff --git a/absl/time/internal/cctz/testdata/zoneinfo/America/St_Barthelemy b/absl/time/internal/cctz/testdata/zoneinfo/America/St_Barthelemy Binary files differindex f4fe5903..47b4dc34 100644 --- a/absl/time/internal/cctz/testdata/zoneinfo/America/St_Barthelemy +++ b/absl/time/internal/cctz/testdata/zoneinfo/America/St_Barthelemy diff --git a/absl/time/internal/cctz/testdata/zoneinfo/America/St_Kitts b/absl/time/internal/cctz/testdata/zoneinfo/America/St_Kitts Binary files differindex f4fe5903..47b4dc34 100644 --- a/absl/time/internal/cctz/testdata/zoneinfo/America/St_Kitts +++ b/absl/time/internal/cctz/testdata/zoneinfo/America/St_Kitts diff --git a/absl/time/internal/cctz/testdata/zoneinfo/America/St_Lucia b/absl/time/internal/cctz/testdata/zoneinfo/America/St_Lucia Binary files differindex f4fe5903..47b4dc34 100644 --- a/absl/time/internal/cctz/testdata/zoneinfo/America/St_Lucia +++ b/absl/time/internal/cctz/testdata/zoneinfo/America/St_Lucia diff --git a/absl/time/internal/cctz/testdata/zoneinfo/America/St_Thomas b/absl/time/internal/cctz/testdata/zoneinfo/America/St_Thomas Binary files differindex f4fe5903..47b4dc34 100644 --- a/absl/time/internal/cctz/testdata/zoneinfo/America/St_Thomas +++ b/absl/time/internal/cctz/testdata/zoneinfo/America/St_Thomas diff --git a/absl/time/internal/cctz/testdata/zoneinfo/America/St_Vincent b/absl/time/internal/cctz/testdata/zoneinfo/America/St_Vincent Binary files differindex f4fe5903..47b4dc34 100644 --- a/absl/time/internal/cctz/testdata/zoneinfo/America/St_Vincent +++ b/absl/time/internal/cctz/testdata/zoneinfo/America/St_Vincent diff --git a/absl/time/internal/cctz/testdata/zoneinfo/America/Tortola b/absl/time/internal/cctz/testdata/zoneinfo/America/Tortola Binary files differindex f4fe5903..47b4dc34 100644 --- a/absl/time/internal/cctz/testdata/zoneinfo/America/Tortola +++ b/absl/time/internal/cctz/testdata/zoneinfo/America/Tortola diff --git a/absl/time/internal/cctz/testdata/zoneinfo/America/Virgin b/absl/time/internal/cctz/testdata/zoneinfo/America/Virgin Binary files differindex f4fe5903..47b4dc34 100644 --- a/absl/time/internal/cctz/testdata/zoneinfo/America/Virgin +++ b/absl/time/internal/cctz/testdata/zoneinfo/America/Virgin diff --git a/absl/time/internal/cctz/testdata/zoneinfo/Antarctica/DumontDUrville b/absl/time/internal/cctz/testdata/zoneinfo/Antarctica/DumontDUrville Binary files differindex c0cfc85a..5d8fc3a1 100644 --- a/absl/time/internal/cctz/testdata/zoneinfo/Antarctica/DumontDUrville +++ b/absl/time/internal/cctz/testdata/zoneinfo/Antarctica/DumontDUrville diff --git a/absl/time/internal/cctz/testdata/zoneinfo/Antarctica/Syowa b/absl/time/internal/cctz/testdata/zoneinfo/Antarctica/Syowa Binary files differindex 97d80d75..01c47ccb 100644 --- a/absl/time/internal/cctz/testdata/zoneinfo/Antarctica/Syowa +++ b/absl/time/internal/cctz/testdata/zoneinfo/Antarctica/Syowa diff --git a/absl/time/internal/cctz/testdata/zoneinfo/Asia/Amman b/absl/time/internal/cctz/testdata/zoneinfo/Asia/Amman Binary files differindex 1bd09fef..d97d308d 100644 --- a/absl/time/internal/cctz/testdata/zoneinfo/Asia/Amman +++ b/absl/time/internal/cctz/testdata/zoneinfo/Asia/Amman diff --git a/absl/time/internal/cctz/testdata/zoneinfo/Asia/Gaza b/absl/time/internal/cctz/testdata/zoneinfo/Asia/Gaza Binary files differindex 58e9fdf4..effc4df5 100644 --- a/absl/time/internal/cctz/testdata/zoneinfo/Asia/Gaza +++ b/absl/time/internal/cctz/testdata/zoneinfo/Asia/Gaza diff --git a/absl/time/internal/cctz/testdata/zoneinfo/Asia/Hebron b/absl/time/internal/cctz/testdata/zoneinfo/Asia/Hebron Binary files differindex aeda06b5..aa52bd26 100644 --- a/absl/time/internal/cctz/testdata/zoneinfo/Asia/Hebron +++ b/absl/time/internal/cctz/testdata/zoneinfo/Asia/Hebron diff --git a/absl/time/internal/cctz/testdata/zoneinfo/Atlantic/Azores b/absl/time/internal/cctz/testdata/zoneinfo/Atlantic/Azores Binary files differindex b7f75a9c..e6e2616e 100644 --- a/absl/time/internal/cctz/testdata/zoneinfo/Atlantic/Azores +++ b/absl/time/internal/cctz/testdata/zoneinfo/Atlantic/Azores diff --git a/absl/time/internal/cctz/testdata/zoneinfo/Atlantic/Madeira b/absl/time/internal/cctz/testdata/zoneinfo/Atlantic/Madeira Binary files differindex 7c3a49c0..cf965c3f 100644 --- a/absl/time/internal/cctz/testdata/zoneinfo/Atlantic/Madeira +++ b/absl/time/internal/cctz/testdata/zoneinfo/Atlantic/Madeira diff --git a/absl/time/internal/cctz/testdata/zoneinfo/Chile/Continental b/absl/time/internal/cctz/testdata/zoneinfo/Chile/Continental Binary files differindex 8d603226..cde8dbbf 100644 --- a/absl/time/internal/cctz/testdata/zoneinfo/Chile/Continental +++ b/absl/time/internal/cctz/testdata/zoneinfo/Chile/Continental diff --git a/absl/time/internal/cctz/testdata/zoneinfo/Europe/Kiev b/absl/time/internal/cctz/testdata/zoneinfo/Europe/Kiev Binary files differindex 8f83cefb..4e026859 100644 --- a/absl/time/internal/cctz/testdata/zoneinfo/Europe/Kiev +++ b/absl/time/internal/cctz/testdata/zoneinfo/Europe/Kiev diff --git a/absl/time/internal/cctz/testdata/zoneinfo/Europe/Lisbon b/absl/time/internal/cctz/testdata/zoneinfo/Europe/Lisbon Binary files differindex 64841661..f0c70b69 100644 --- a/absl/time/internal/cctz/testdata/zoneinfo/Europe/Lisbon +++ b/absl/time/internal/cctz/testdata/zoneinfo/Europe/Lisbon diff --git a/absl/time/internal/cctz/testdata/zoneinfo/Europe/Simferopol b/absl/time/internal/cctz/testdata/zoneinfo/Europe/Simferopol Binary files differindex 88a6f3bd..40d23c02 100644 --- a/absl/time/internal/cctz/testdata/zoneinfo/Europe/Simferopol +++ b/absl/time/internal/cctz/testdata/zoneinfo/Europe/Simferopol diff --git a/absl/time/internal/cctz/testdata/zoneinfo/Europe/Uzhgorod b/absl/time/internal/cctz/testdata/zoneinfo/Europe/Uzhgorod Binary files differindex a5755685..d4c35914 100644 --- a/absl/time/internal/cctz/testdata/zoneinfo/Europe/Uzhgorod +++ b/absl/time/internal/cctz/testdata/zoneinfo/Europe/Uzhgorod diff --git a/absl/time/internal/cctz/testdata/zoneinfo/Europe/Zaporozhye b/absl/time/internal/cctz/testdata/zoneinfo/Europe/Zaporozhye Binary files differindex 4ea8dae4..71819a5a 100644 --- a/absl/time/internal/cctz/testdata/zoneinfo/Europe/Zaporozhye +++ b/absl/time/internal/cctz/testdata/zoneinfo/Europe/Zaporozhye diff --git a/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Apia b/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Apia Binary files differindex 244af26f..a6b835aa 100644 --- a/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Apia +++ b/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Apia diff --git a/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Enderbury b/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Enderbury Binary files differindex b22ab147..2b6a0608 100644 --- a/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Enderbury +++ b/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Enderbury diff --git a/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Fiji b/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Fiji Binary files differindex e3934e42..8b2dd52b 100644 --- a/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Fiji +++ b/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Fiji diff --git a/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Kanton b/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Kanton Binary files differnew file mode 100644 index 00000000..2b6a0608 --- /dev/null +++ b/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Kanton diff --git a/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Niue b/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Niue Binary files differindex 7b357935..be874e24 100644 --- a/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Niue +++ b/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Niue diff --git a/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Rarotonga b/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Rarotonga Binary files differindex 143a1883..7220bda0 100644 --- a/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Rarotonga +++ b/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Rarotonga diff --git a/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Tongatapu b/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Tongatapu Binary files differindex 54aeb0ff..f28c8401 100644 --- a/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Tongatapu +++ b/absl/time/internal/cctz/testdata/zoneinfo/Pacific/Tongatapu diff --git a/absl/time/internal/cctz/testdata/zoneinfo/Portugal b/absl/time/internal/cctz/testdata/zoneinfo/Portugal Binary files differindex 64841661..f0c70b69 100644 --- a/absl/time/internal/cctz/testdata/zoneinfo/Portugal +++ b/absl/time/internal/cctz/testdata/zoneinfo/Portugal diff --git a/absl/time/internal/cctz/testdata/zoneinfo/zone1970.tab b/absl/time/internal/cctz/testdata/zoneinfo/zone1970.tab index 396e4d38..c614be81 100644 --- a/absl/time/internal/cctz/testdata/zoneinfo/zone1970.tab +++ b/absl/time/internal/cctz/testdata/zoneinfo/zone1970.tab @@ -40,11 +40,9 @@ AL +4120+01950 Europe/Tirane AM +4011+04430 Asia/Yerevan AQ -6617+11031 Antarctica/Casey Casey AQ -6835+07758 Antarctica/Davis Davis -AQ -6640+14001 Antarctica/DumontDUrville Dumont-d'Urville AQ -6736+06253 Antarctica/Mawson Mawson AQ -6448-06406 Antarctica/Palmer Palmer AQ -6734-06808 Antarctica/Rothera Rothera -AQ -690022+0393524 Antarctica/Syowa Syowa AQ -720041+0023206 Antarctica/Troll Troll AQ -7824+10654 Antarctica/Vostok Vostok AR -3436-05827 America/Argentina/Buenos_Aires Buenos Aires (BA, CF) @@ -97,7 +95,6 @@ BR +0249-06040 America/Boa_Vista Roraima BR -0308-06001 America/Manaus Amazonas (east) BR -0640-06952 America/Eirunepe Amazonas (west) BR -0958-06748 America/Rio_Branco Acre -BS +2505-07721 America/Nassau BT +2728+08939 Asia/Thimphu BY +5354+02734 Europe/Minsk BZ +1730-08812 America/Belize @@ -106,13 +103,11 @@ CA +4439-06336 America/Halifax Atlantic - NS (most areas); PE CA +4612-05957 America/Glace_Bay Atlantic - NS (Cape Breton) CA +4606-06447 America/Moncton Atlantic - New Brunswick CA +5320-06025 America/Goose_Bay Atlantic - Labrador (most areas) -CA +5125-05707 America/Blanc-Sablon AST - QC (Lower North Shore) -CA +4339-07923 America/Toronto Eastern - ON, QC (most areas) +CA,BS +4339-07923 America/Toronto Eastern - ON, QC (most areas), Bahamas CA +4901-08816 America/Nipigon Eastern - ON, QC (no DST 1967-73) CA +4823-08915 America/Thunder_Bay Eastern - ON (Thunder Bay) CA +6344-06828 America/Iqaluit Eastern - NU (most east areas) CA +6608-06544 America/Pangnirtung Eastern - NU (Pangnirtung) -CA +484531-0913718 America/Atikokan EST - ON (Atikokan); NU (Coral H) CA +4953-09709 America/Winnipeg Central - ON (west); Manitoba CA +4843-09434 America/Rainy_River Central - ON (Rainy R, Ft Frances) CA +744144-0944945 America/Resolute Central - NU (Resolute) @@ -123,7 +118,6 @@ CA +5333-11328 America/Edmonton Mountain - AB; BC (E); SK (W) CA +690650-1050310 America/Cambridge_Bay Mountain - NU (west) CA +6227-11421 America/Yellowknife Mountain - NT (central) CA +682059-1334300 America/Inuvik Mountain - NT (west) -CA +4906-11631 America/Creston MST - BC (Creston) CA +5946-12014 America/Dawson_Creek MST - BC (Dawson Cr, Ft St John) CA +5848-12242 America/Fort_Nelson MST - BC (Ft Nelson) CA +6043-13503 America/Whitehorse MST - Yukon (east) @@ -131,7 +125,7 @@ CA +6404-13925 America/Dawson MST - Yukon (west) CA +4916-12307 America/Vancouver Pacific - BC (most areas) CC -1210+09655 Indian/Cocos CH,DE,LI +4723+00832 Europe/Zurich Swiss time -CI,BF,GM,GN,ML,MR,SH,SL,SN,TG +0519-00402 Africa/Abidjan +CI,BF,GH,GM,GN,ML,MR,SH,SL,SN,TG +0519-00402 Africa/Abidjan CK -2114-15946 Pacific/Rarotonga CL -3327-07040 America/Santiago Chile (most areas) CL -5309-07055 America/Punta_Arenas Region of Magallanes @@ -142,7 +136,6 @@ CO +0436-07405 America/Bogota CR +0956-08405 America/Costa_Rica CU +2308-08222 America/Havana CV +1455-02331 Atlantic/Cape_Verde -CW,AW,BQ,SX +1211-06900 America/Curacao CX -1025+10543 Indian/Christmas CY +3510+03322 Asia/Nicosia Cyprus (most areas) CY +3507+03357 Asia/Famagusta Northern Cyprus @@ -170,7 +163,6 @@ FR +4852+00220 Europe/Paris GB,GG,IM,JE +513030-0000731 Europe/London GE +4143+04449 Asia/Tbilisi GF +0456-05220 America/Cayenne -GH +0533-00013 Africa/Accra GI +3608-00521 Europe/Gibraltar GL +6411-05144 America/Nuuk Greenland (most areas) GL +7646-01840 America/Danmarkshavn National Park (east coast) @@ -204,7 +196,7 @@ JP +353916+1394441 Asia/Tokyo KE,DJ,ER,ET,KM,MG,SO,TZ,UG,YT -0117+03649 Africa/Nairobi KG +4254+07436 Asia/Bishkek KI +0125+17300 Pacific/Tarawa Gilbert Islands -KI -0308-17105 Pacific/Enderbury Phoenix Islands +KI -0247-17143 Pacific/Kanton Phoenix Islands KI +0152-15720 Pacific/Kiritimati Line Islands KP +3901+12545 Asia/Pyongyang KR +3733+12658 Asia/Seoul @@ -262,19 +254,19 @@ NR -0031+16655 Pacific/Nauru NU -1901-16955 Pacific/Niue NZ,AQ -3652+17446 Pacific/Auckland New Zealand time NZ -4357-17633 Pacific/Chatham Chatham Islands -PA,KY +0858-07932 America/Panama +PA,CA,KY +0858-07932 America/Panama EST - Panama, Cayman, ON (Atikokan), NU (Coral H) PE -1203-07703 America/Lima PF -1732-14934 Pacific/Tahiti Society Islands PF -0900-13930 Pacific/Marquesas Marquesas Islands PF -2308-13457 Pacific/Gambier Gambier Islands -PG -0930+14710 Pacific/Port_Moresby Papua New Guinea (most areas) +PG,AQ -0930+14710 Pacific/Port_Moresby Papua New Guinea (most areas), Dumont d'Urville PG -0613+15534 Pacific/Bougainville Bougainville PH +1435+12100 Asia/Manila PK +2452+06703 Asia/Karachi PL +5215+02100 Europe/Warsaw PM +4703-05620 America/Miquelon PN -2504-13005 Pacific/Pitcairn -PR +182806-0660622 America/Puerto_Rico +PR,AG,CA,AI,AW,BL,BQ,CW,DM,GD,GP,KN,LC,MF,MS,SX,TT,VC,VG,VI +182806-0660622 America/Puerto_Rico AST PS +3130+03428 Asia/Gaza Gaza Strip PS +313200+0350542 Asia/Hebron West Bank PT +3843-00908 Europe/Lisbon Portugal (mainland) @@ -314,12 +306,12 @@ RU +4658+14242 Asia/Sakhalin MSK+08 - Sakhalin Island RU +6728+15343 Asia/Srednekolymsk MSK+08 - Sakha (E); North Kuril Is RU +5301+15839 Asia/Kamchatka MSK+09 - Kamchatka RU +6445+17729 Asia/Anadyr MSK+09 - Bering Sea -SA,KW,YE +2438+04643 Asia/Riyadh +SA,AQ,KW,YE +2438+04643 Asia/Riyadh Arabia, Syowa SB -0932+16012 Pacific/Guadalcanal SC -0440+05528 Indian/Mahe SD +1536+03232 Africa/Khartoum SE +5920+01803 Europe/Stockholm -SG +0117+10351 Asia/Singapore +SG,MY +0117+10351 Asia/Singapore Singapore, peninsular Malaysia SR +0550-05510 America/Paramaribo SS +0451+03137 Africa/Juba ST +0020+00644 Africa/Sao_Tome @@ -334,9 +326,8 @@ TK -0922-17114 Pacific/Fakaofo TL -0833+12535 Asia/Dili TM +3757+05823 Asia/Ashgabat TN +3648+01011 Africa/Tunis -TO -2110-17510 Pacific/Tongatapu +TO -210800-1751200 Pacific/Tongatapu TR +4101+02858 Europe/Istanbul -TT,AG,AI,BL,DM,GD,GP,KN,LC,MF,MS,VC,VG,VI +1039-06131 America/Port_of_Spain TV -0831+17913 Pacific/Funafuti TW +2503+12130 Asia/Taipei UA +5026+03031 Europe/Kiev Ukraine (most areas) @@ -362,7 +353,7 @@ US +465042-1012439 America/North_Dakota/New_Salem Central - ND (Morton rural) US +471551-1014640 America/North_Dakota/Beulah Central - ND (Mercer) US +394421-1045903 America/Denver Mountain (most areas) US +433649-1161209 America/Boise Mountain - ID (south); OR (east) -US +332654-1120424 America/Phoenix MST - Arizona (except Navajo) +US,CA +332654-1120424 America/Phoenix MST - Arizona (except Navajo), Creston BC US +340308-1181434 America/Los_Angeles Pacific US +611305-1495401 America/Anchorage Alaska (most areas) US +581807-1342511 America/Juneau Alaska - Juneau area diff --git a/absl/time/internal/test_util.cc b/absl/time/internal/test_util.cc index 9a485a07..454b33a1 100644 --- a/absl/time/internal/test_util.cc +++ b/absl/time/internal/test_util.cc @@ -17,6 +17,7 @@ #include <algorithm> #include <cstddef> #include <cstring> +#include <memory> #include "absl/base/config.h" #include "absl/base/internal/raw_logging.h" @@ -67,10 +68,6 @@ const struct ZoneInfo { {"Invalid/TimeZone", nullptr, 0}, {"", nullptr, 0}, - // Also allow for loading the local time zone under TZ=US/Pacific. - {"US/Pacific", // - reinterpret_cast<char*>(America_Los_Angeles), America_Los_Angeles_len}, - // Allows use of the local time zone from a system-specific location. #ifdef _MSC_VER {"localtime", // @@ -114,7 +111,10 @@ std::unique_ptr<cctz::ZoneInfoSource> TestFactory( new TestZoneInfoSource(zoneinfo.data, zoneinfo.length)); } } - ABSL_RAW_LOG(FATAL, "Unexpected time zone \"%s\" in test", name.c_str()); + + // The embedded zoneinfo data does not include the zone, so fallback to + // built-in UTC. The tests have been crafted so that this should only + // happen when testing absl::LocalTimeZone() with an unconstrained ${TZ}. return nullptr; } diff --git a/absl/time/internal/zoneinfo.inc b/absl/time/internal/zoneinfo.inc index bfed8299..7d8b3ff2 100644 --- a/absl/time/internal/zoneinfo.inc +++ b/absl/time/internal/zoneinfo.inc @@ -88,157 +88,156 @@ unsigned char America_Los_Angeles[] = { 0x00, 0x01, 0x54, 0x5a, 0x69, 0x66, 0x32, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0xbb, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x14, 0xf8, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x5e, 0x04, - 0x1a, 0xc0, 0xff, 0xff, 0xff, 0xff, 0x9e, 0xa6, 0x48, 0xa0, 0xff, 0xff, - 0xff, 0xff, 0x9f, 0xbb, 0x15, 0x90, 0xff, 0xff, 0xff, 0xff, 0xa0, 0x86, - 0x2a, 0xa0, 0xff, 0xff, 0xff, 0xff, 0xa1, 0x9a, 0xf7, 0x90, 0xff, 0xff, - 0xff, 0xff, 0xcb, 0x89, 0x1a, 0xa0, 0xff, 0xff, 0xff, 0xff, 0xd2, 0x23, - 0xf4, 0x70, 0xff, 0xff, 0xff, 0xff, 0xd2, 0x61, 0x26, 0x10, 0xff, 0xff, - 0xff, 0xff, 0xd6, 0xfe, 0x74, 0x5c, 0xff, 0xff, 0xff, 0xff, 0xd8, 0x80, - 0xad, 0x90, 0xff, 0xff, 0xff, 0xff, 0xda, 0xfe, 0xc3, 0x90, 0xff, 0xff, - 0xff, 0xff, 0xdb, 0xc0, 0x90, 0x10, 0xff, 0xff, 0xff, 0xff, 0xdc, 0xde, - 0xa5, 0x90, 0xff, 0xff, 0xff, 0xff, 0xdd, 0xa9, 0xac, 0x90, 0xff, 0xff, - 0xff, 0xff, 0xde, 0xbe, 0x87, 0x90, 0xff, 0xff, 0xff, 0xff, 0xdf, 0x89, - 0x8e, 0x90, 0xff, 0xff, 0xff, 0xff, 0xe0, 0x9e, 0x69, 0x90, 0xff, 0xff, - 0xff, 0xff, 0xe1, 0x69, 0x70, 0x90, 0xff, 0xff, 0xff, 0xff, 0xe2, 0x7e, - 0x4b, 0x90, 0xff, 0xff, 0xff, 0xff, 0xe3, 0x49, 0x52, 0x90, 0xff, 0xff, - 0xff, 0xff, 0xe4, 0x5e, 0x2d, 0x90, 0xff, 0xff, 0xff, 0xff, 0xe5, 0x29, - 0x34, 0x90, 0xff, 0xff, 0xff, 0xff, 0xe6, 0x47, 0x4a, 0x10, 0xff, 0xff, - 0xff, 0xff, 0xe7, 0x12, 0x51, 0x10, 0xff, 0xff, 0xff, 0xff, 0xe8, 0x27, - 0x2c, 0x10, 0xff, 0xff, 0xff, 0xff, 0xe8, 0xf2, 0x33, 0x10, 0xff, 0xff, - 0xff, 0xff, 0xea, 0x07, 0x0e, 0x10, 0xff, 0xff, 0xff, 0xff, 0xea, 0xd2, - 0x15, 0x10, 0xff, 0xff, 0xff, 0xff, 0xeb, 0xe6, 0xf0, 0x10, 0xff, 0xff, - 0xff, 0xff, 0xec, 0xb1, 0xf7, 0x10, 0xff, 0xff, 0xff, 0xff, 0xed, 0xc6, - 0xd2, 0x10, 0xff, 0xff, 0xff, 0xff, 0xee, 0x91, 0xd9, 0x10, 0xff, 0xff, - 0xff, 0xff, 0xef, 0xaf, 0xee, 0x90, 0xff, 0xff, 0xff, 0xff, 0xf0, 0x71, - 0xbb, 0x10, 0xff, 0xff, 0xff, 0xff, 0xf1, 0x8f, 0xd0, 0x90, 0xff, 0xff, - 0xff, 0xff, 0xf2, 0x7f, 0xc1, 0x90, 0xff, 0xff, 0xff, 0xff, 0xf3, 0x6f, - 0xb2, 0x90, 0xff, 0xff, 0xff, 0xff, 0xf4, 0x5f, 0xa3, 0x90, 0xff, 0xff, - 0xff, 0xff, 0xf5, 0x4f, 0x94, 0x90, 0xff, 0xff, 0xff, 0xff, 0xf6, 0x3f, - 0x85, 0x90, 0xff, 0xff, 0xff, 0xff, 0xf7, 0x2f, 0x76, 0x90, 0xff, 0xff, - 0xff, 0xff, 0xf8, 0x28, 0xa2, 0x10, 0xff, 0xff, 0xff, 0xff, 0xf9, 0x0f, - 0x58, 0x90, 0xff, 0xff, 0xff, 0xff, 0xfa, 0x08, 0x84, 0x10, 0xff, 0xff, - 0xff, 0xff, 0xfa, 0xf8, 0x83, 0x20, 0xff, 0xff, 0xff, 0xff, 0xfb, 0xe8, - 0x66, 0x10, 0xff, 0xff, 0xff, 0xff, 0xfc, 0xd8, 0x65, 0x20, 0xff, 0xff, - 0xff, 0xff, 0xfd, 0xc8, 0x48, 0x10, 0xff, 0xff, 0xff, 0xff, 0xfe, 0xb8, - 0x47, 0x20, 0xff, 0xff, 0xff, 0xff, 0xff, 0xa8, 0x2a, 0x10, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x98, 0x29, 0x20, 0x00, 0x00, 0x00, 0x00, 0x01, 0x88, - 0x0c, 0x10, 0x00, 0x00, 0x00, 0x00, 0x02, 0x78, 0x0b, 0x20, 0x00, 0x00, - 0x00, 0x00, 0x03, 0x71, 0x28, 0x90, 0x00, 0x00, 0x00, 0x00, 0x04, 0x61, - 0x27, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x05, 0x51, 0x0a, 0x90, 0x00, 0x00, - 0x00, 0x00, 0x06, 0x41, 0x09, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x07, 0x30, - 0xec, 0x90, 0x00, 0x00, 0x00, 0x00, 0x07, 0x8d, 0x43, 0xa0, 0x00, 0x00, - 0x00, 0x00, 0x09, 0x10, 0xce, 0x90, 0x00, 0x00, 0x00, 0x00, 0x09, 0xad, - 0xbf, 0x20, 0x00, 0x00, 0x00, 0x00, 0x0a, 0xf0, 0xb0, 0x90, 0x00, 0x00, - 0x00, 0x00, 0x0b, 0xe0, 0xaf, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x0c, 0xd9, - 0xcd, 0x10, 0x00, 0x00, 0x00, 0x00, 0x0d, 0xc0, 0x91, 0xa0, 0x00, 0x00, - 0x00, 0x00, 0x0e, 0xb9, 0xaf, 0x10, 0x00, 0x00, 0x00, 0x00, 0x0f, 0xa9, - 0xae, 0x20, 0x00, 0x00, 0x00, 0x00, 0x10, 0x99, 0x91, 0x10, 0x00, 0x00, - 0x00, 0x00, 0x11, 0x89, 0x90, 0x20, 0x00, 0x00, 0x00, 0x00, 0x12, 0x79, - 0x73, 0x10, 0x00, 0x00, 0x00, 0x00, 0x13, 0x69, 0x72, 0x20, 0x00, 0x00, - 0x00, 0x00, 0x14, 0x59, 0x55, 0x10, 0x00, 0x00, 0x00, 0x00, 0x15, 0x49, - 0x54, 0x20, 0x00, 0x00, 0x00, 0x00, 0x16, 0x39, 0x37, 0x10, 0x00, 0x00, - 0x00, 0x00, 0x17, 0x29, 0x36, 0x20, 0x00, 0x00, 0x00, 0x00, 0x18, 0x22, - 0x53, 0x90, 0x00, 0x00, 0x00, 0x00, 0x19, 0x09, 0x18, 0x20, 0x00, 0x00, - 0x00, 0x00, 0x1a, 0x02, 0x35, 0x90, 0x00, 0x00, 0x00, 0x00, 0x1a, 0xf2, - 0x34, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x1b, 0xe2, 0x17, 0x90, 0x00, 0x00, - 0x00, 0x00, 0x1c, 0xd2, 0x16, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x1d, 0xc1, - 0xf9, 0x90, 0x00, 0x00, 0x00, 0x00, 0x1e, 0xb1, 0xf8, 0xa0, 0x00, 0x00, - 0x00, 0x00, 0x1f, 0xa1, 0xdb, 0x90, 0x00, 0x00, 0x00, 0x00, 0x20, 0x76, - 0x2b, 0x20, 0x00, 0x00, 0x00, 0x00, 0x21, 0x81, 0xbd, 0x90, 0x00, 0x00, - 0x00, 0x00, 0x22, 0x56, 0x0d, 0x20, 0x00, 0x00, 0x00, 0x00, 0x23, 0x6a, - 0xda, 0x10, 0x00, 0x00, 0x00, 0x00, 0x24, 0x35, 0xef, 0x20, 0x00, 0x00, - 0x00, 0x00, 0x25, 0x4a, 0xbc, 0x10, 0x00, 0x00, 0x00, 0x00, 0x26, 0x15, - 0xd1, 0x20, 0x00, 0x00, 0x00, 0x00, 0x27, 0x2a, 0x9e, 0x10, 0x00, 0x00, - 0x00, 0x00, 0x27, 0xfe, 0xed, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x29, 0x0a, - 0x80, 0x10, 0x00, 0x00, 0x00, 0x00, 0x29, 0xde, 0xcf, 0xa0, 0x00, 0x00, - 0x00, 0x00, 0x2a, 0xea, 0x62, 0x10, 0x00, 0x00, 0x00, 0x00, 0x2b, 0xbe, - 0xb1, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x2c, 0xd3, 0x7e, 0x90, 0x00, 0x00, - 0x00, 0x00, 0x2d, 0x9e, 0x93, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x2e, 0xb3, - 0x60, 0x90, 0x00, 0x00, 0x00, 0x00, 0x2f, 0x7e, 0x75, 0xa0, 0x00, 0x00, - 0x00, 0x00, 0x30, 0x93, 0x42, 0x90, 0x00, 0x00, 0x00, 0x00, 0x31, 0x67, - 0x92, 0x20, 0x00, 0x00, 0x00, 0x00, 0x32, 0x73, 0x24, 0x90, 0x00, 0x00, - 0x00, 0x00, 0x33, 0x47, 0x74, 0x20, 0x00, 0x00, 0x00, 0x00, 0x34, 0x53, - 0x06, 0x90, 0x00, 0x00, 0x00, 0x00, 0x35, 0x27, 0x56, 0x20, 0x00, 0x00, - 0x00, 0x00, 0x36, 0x32, 0xe8, 0x90, 0x00, 0x00, 0x00, 0x00, 0x37, 0x07, - 0x38, 0x20, 0x00, 0x00, 0x00, 0x00, 0x38, 0x1c, 0x05, 0x10, 0x00, 0x00, - 0x00, 0x00, 0x38, 0xe7, 0x1a, 0x20, 0x00, 0x00, 0x00, 0x00, 0x39, 0xfb, - 0xe7, 0x10, 0x00, 0x00, 0x00, 0x00, 0x3a, 0xc6, 0xfc, 0x20, 0x00, 0x00, - 0x00, 0x00, 0x3b, 0xdb, 0xc9, 0x10, 0x00, 0x00, 0x00, 0x00, 0x3c, 0xb0, - 0x18, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x3d, 0xbb, 0xab, 0x10, 0x00, 0x00, - 0x00, 0x00, 0x3e, 0x8f, 0xfa, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x3f, 0x9b, - 0x8d, 0x10, 0x00, 0x00, 0x00, 0x00, 0x40, 0x6f, 0xdc, 0xa0, 0x00, 0x00, - 0x00, 0x00, 0x41, 0x84, 0xa9, 0x90, 0x00, 0x00, 0x00, 0x00, 0x42, 0x4f, - 0xbe, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x43, 0x64, 0x8b, 0x90, 0x00, 0x00, - 0x00, 0x00, 0x44, 0x2f, 0xa0, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x45, 0x44, - 0x6d, 0x90, 0x00, 0x00, 0x00, 0x00, 0x45, 0xf3, 0xd3, 0x20, 0x00, 0x00, - 0x00, 0x00, 0x47, 0x2d, 0x8a, 0x10, 0x00, 0x00, 0x00, 0x00, 0x47, 0xd3, - 0xb5, 0x20, 0x00, 0x00, 0x00, 0x00, 0x49, 0x0d, 0x6c, 0x10, 0x00, 0x00, - 0x00, 0x00, 0x49, 0xb3, 0x97, 0x20, 0x00, 0x00, 0x00, 0x00, 0x4a, 0xed, - 0x4e, 0x10, 0x00, 0x00, 0x00, 0x00, 0x4b, 0x9c, 0xb3, 0xa0, 0x00, 0x00, - 0x00, 0x00, 0x4c, 0xd6, 0x6a, 0x90, 0x00, 0x00, 0x00, 0x00, 0x4d, 0x7c, - 0x95, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x4e, 0xb6, 0x4c, 0x90, 0x00, 0x00, - 0x00, 0x00, 0x4f, 0x5c, 0x77, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x50, 0x96, - 0x2e, 0x90, 0x00, 0x00, 0x00, 0x00, 0x51, 0x3c, 0x59, 0xa0, 0x00, 0x00, - 0x00, 0x00, 0x52, 0x76, 0x10, 0x90, 0x00, 0x00, 0x00, 0x00, 0x53, 0x1c, - 0x3b, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x54, 0x55, 0xf2, 0x90, 0x00, 0x00, - 0x00, 0x00, 0x54, 0xfc, 0x1d, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x56, 0x35, - 0xd4, 0x90, 0x00, 0x00, 0x00, 0x00, 0x56, 0xe5, 0x3a, 0x20, 0x00, 0x00, - 0x00, 0x00, 0x58, 0x1e, 0xf1, 0x10, 0x00, 0x00, 0x00, 0x00, 0x58, 0xc5, - 0x1c, 0x20, 0x00, 0x00, 0x00, 0x00, 0x59, 0xfe, 0xd3, 0x10, 0x00, 0x00, - 0x00, 0x00, 0x5a, 0xa4, 0xfe, 0x20, 0x00, 0x00, 0x00, 0x00, 0x5b, 0xde, - 0xb5, 0x10, 0x00, 0x00, 0x00, 0x00, 0x5c, 0x84, 0xe0, 0x20, 0x00, 0x00, - 0x00, 0x00, 0x5d, 0xbe, 0x97, 0x10, 0x00, 0x00, 0x00, 0x00, 0x5e, 0x64, - 0xc2, 0x20, 0x00, 0x00, 0x00, 0x00, 0x5f, 0x9e, 0x79, 0x10, 0x00, 0x00, - 0x00, 0x00, 0x60, 0x4d, 0xde, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x61, 0x87, - 0x95, 0x90, 0x00, 0x00, 0x00, 0x00, 0x62, 0x2d, 0xc0, 0xa0, 0x00, 0x00, - 0x00, 0x00, 0x63, 0x67, 0x77, 0x90, 0x00, 0x00, 0x00, 0x00, 0x64, 0x0d, - 0xa2, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x65, 0x47, 0x59, 0x90, 0x00, 0x00, - 0x00, 0x00, 0x65, 0xed, 0x84, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x67, 0x27, - 0x3b, 0x90, 0x00, 0x00, 0x00, 0x00, 0x67, 0xcd, 0x66, 0xa0, 0x00, 0x00, - 0x00, 0x00, 0x69, 0x07, 0x1d, 0x90, 0x00, 0x00, 0x00, 0x00, 0x69, 0xad, - 0x48, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x6a, 0xe6, 0xff, 0x90, 0x00, 0x00, - 0x00, 0x00, 0x6b, 0x96, 0x65, 0x20, 0x00, 0x00, 0x00, 0x00, 0x6c, 0xd0, - 0x1c, 0x10, 0x00, 0x00, 0x00, 0x00, 0x6d, 0x76, 0x47, 0x20, 0x00, 0x00, - 0x00, 0x00, 0x6e, 0xaf, 0xfe, 0x10, 0x00, 0x00, 0x00, 0x00, 0x6f, 0x56, - 0x29, 0x20, 0x00, 0x00, 0x00, 0x00, 0x70, 0x8f, 0xe0, 0x10, 0x00, 0x00, - 0x00, 0x00, 0x71, 0x36, 0x0b, 0x20, 0x00, 0x00, 0x00, 0x00, 0x72, 0x6f, - 0xc2, 0x10, 0x00, 0x00, 0x00, 0x00, 0x73, 0x15, 0xed, 0x20, 0x00, 0x00, - 0x00, 0x00, 0x74, 0x4f, 0xa4, 0x10, 0x00, 0x00, 0x00, 0x00, 0x74, 0xff, - 0x09, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x76, 0x38, 0xc0, 0x90, 0x00, 0x00, - 0x00, 0x00, 0x76, 0xde, 0xeb, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x78, 0x18, - 0xa2, 0x90, 0x00, 0x00, 0x00, 0x00, 0x78, 0xbe, 0xcd, 0xa0, 0x00, 0x00, - 0x00, 0x00, 0x79, 0xf8, 0x84, 0x90, 0x00, 0x00, 0x00, 0x00, 0x7a, 0x9e, - 0xaf, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x7b, 0xd8, 0x66, 0x90, 0x00, 0x00, - 0x00, 0x00, 0x7c, 0x7e, 0x91, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x7d, 0xb8, - 0x48, 0x90, 0x00, 0x00, 0x00, 0x00, 0x7e, 0x5e, 0x73, 0xa0, 0x00, 0x00, - 0x00, 0x00, 0x7f, 0x98, 0x2a, 0x90, 0x00, 0x02, 0x01, 0x02, 0x01, 0x02, - 0x03, 0x04, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, - 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, - 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, - 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, - 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, - 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, - 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, - 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, - 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, - 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, - 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, - 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, - 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, - 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, - 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, - 0x02, 0xff, 0xff, 0x91, 0x26, 0x00, 0x00, 0xff, 0xff, 0x9d, 0x90, 0x01, - 0x04, 0xff, 0xff, 0x8f, 0x80, 0x00, 0x08, 0xff, 0xff, 0x9d, 0x90, 0x01, - 0x0c, 0xff, 0xff, 0x9d, 0x90, 0x01, 0x10, 0x4c, 0x4d, 0x54, 0x00, 0x50, - 0x44, 0x54, 0x00, 0x50, 0x53, 0x54, 0x00, 0x50, 0x57, 0x54, 0x00, 0x50, - 0x50, 0x54, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, - 0x01, 0x0a, 0x50, 0x53, 0x54, 0x38, 0x50, 0x44, 0x54, 0x2c, 0x4d, 0x33, - 0x2e, 0x32, 0x2e, 0x30, 0x2c, 0x4d, 0x31, 0x31, 0x2e, 0x31, 0x2e, 0x30, - 0x0a + 0x00, 0xba, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x14, 0xff, 0xff, + 0xff, 0xff, 0x5e, 0x04, 0x1a, 0xc0, 0xff, 0xff, 0xff, 0xff, 0x9e, 0xa6, + 0x48, 0xa0, 0xff, 0xff, 0xff, 0xff, 0x9f, 0xbb, 0x15, 0x90, 0xff, 0xff, + 0xff, 0xff, 0xa0, 0x86, 0x2a, 0xa0, 0xff, 0xff, 0xff, 0xff, 0xa1, 0x9a, + 0xf7, 0x90, 0xff, 0xff, 0xff, 0xff, 0xcb, 0x89, 0x1a, 0xa0, 0xff, 0xff, + 0xff, 0xff, 0xd2, 0x23, 0xf4, 0x70, 0xff, 0xff, 0xff, 0xff, 0xd2, 0x61, + 0x26, 0x10, 0xff, 0xff, 0xff, 0xff, 0xd6, 0xfe, 0x74, 0x5c, 0xff, 0xff, + 0xff, 0xff, 0xd8, 0x80, 0xad, 0x90, 0xff, 0xff, 0xff, 0xff, 0xda, 0xfe, + 0xc3, 0x90, 0xff, 0xff, 0xff, 0xff, 0xdb, 0xc0, 0x90, 0x10, 0xff, 0xff, + 0xff, 0xff, 0xdc, 0xde, 0xa5, 0x90, 0xff, 0xff, 0xff, 0xff, 0xdd, 0xa9, + 0xac, 0x90, 0xff, 0xff, 0xff, 0xff, 0xde, 0xbe, 0x87, 0x90, 0xff, 0xff, + 0xff, 0xff, 0xdf, 0x89, 0x8e, 0x90, 0xff, 0xff, 0xff, 0xff, 0xe0, 0x9e, + 0x69, 0x90, 0xff, 0xff, 0xff, 0xff, 0xe1, 0x69, 0x70, 0x90, 0xff, 0xff, + 0xff, 0xff, 0xe2, 0x7e, 0x4b, 0x90, 0xff, 0xff, 0xff, 0xff, 0xe3, 0x49, + 0x52, 0x90, 0xff, 0xff, 0xff, 0xff, 0xe4, 0x5e, 0x2d, 0x90, 0xff, 0xff, + 0xff, 0xff, 0xe5, 0x29, 0x34, 0x90, 0xff, 0xff, 0xff, 0xff, 0xe6, 0x47, + 0x4a, 0x10, 0xff, 0xff, 0xff, 0xff, 0xe7, 0x12, 0x51, 0x10, 0xff, 0xff, + 0xff, 0xff, 0xe8, 0x27, 0x2c, 0x10, 0xff, 0xff, 0xff, 0xff, 0xe8, 0xf2, + 0x33, 0x10, 0xff, 0xff, 0xff, 0xff, 0xea, 0x07, 0x0e, 0x10, 0xff, 0xff, + 0xff, 0xff, 0xea, 0xd2, 0x15, 0x10, 0xff, 0xff, 0xff, 0xff, 0xeb, 0xe6, + 0xf0, 0x10, 0xff, 0xff, 0xff, 0xff, 0xec, 0xb1, 0xf7, 0x10, 0xff, 0xff, + 0xff, 0xff, 0xed, 0xc6, 0xd2, 0x10, 0xff, 0xff, 0xff, 0xff, 0xee, 0x91, + 0xd9, 0x10, 0xff, 0xff, 0xff, 0xff, 0xef, 0xaf, 0xee, 0x90, 0xff, 0xff, + 0xff, 0xff, 0xf0, 0x71, 0xbb, 0x10, 0xff, 0xff, 0xff, 0xff, 0xf1, 0x8f, + 0xd0, 0x90, 0xff, 0xff, 0xff, 0xff, 0xf2, 0x7f, 0xc1, 0x90, 0xff, 0xff, + 0xff, 0xff, 0xf3, 0x6f, 0xb2, 0x90, 0xff, 0xff, 0xff, 0xff, 0xf4, 0x5f, + 0xa3, 0x90, 0xff, 0xff, 0xff, 0xff, 0xf5, 0x4f, 0x94, 0x90, 0xff, 0xff, + 0xff, 0xff, 0xf6, 0x3f, 0x85, 0x90, 0xff, 0xff, 0xff, 0xff, 0xf7, 0x2f, + 0x76, 0x90, 0xff, 0xff, 0xff, 0xff, 0xf8, 0x28, 0xa2, 0x10, 0xff, 0xff, + 0xff, 0xff, 0xf9, 0x0f, 0x58, 0x90, 0xff, 0xff, 0xff, 0xff, 0xfa, 0x08, + 0x84, 0x10, 0xff, 0xff, 0xff, 0xff, 0xfa, 0xf8, 0x83, 0x20, 0xff, 0xff, + 0xff, 0xff, 0xfb, 0xe8, 0x66, 0x10, 0xff, 0xff, 0xff, 0xff, 0xfc, 0xd8, + 0x65, 0x20, 0xff, 0xff, 0xff, 0xff, 0xfd, 0xc8, 0x48, 0x10, 0xff, 0xff, + 0xff, 0xff, 0xfe, 0xb8, 0x47, 0x20, 0xff, 0xff, 0xff, 0xff, 0xff, 0xa8, + 0x2a, 0x10, 0x00, 0x00, 0x00, 0x00, 0x00, 0x98, 0x29, 0x20, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x88, 0x0c, 0x10, 0x00, 0x00, 0x00, 0x00, 0x02, 0x78, + 0x0b, 0x20, 0x00, 0x00, 0x00, 0x00, 0x03, 0x71, 0x28, 0x90, 0x00, 0x00, + 0x00, 0x00, 0x04, 0x61, 0x27, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x05, 0x51, + 0x0a, 0x90, 0x00, 0x00, 0x00, 0x00, 0x06, 0x41, 0x09, 0xa0, 0x00, 0x00, + 0x00, 0x00, 0x07, 0x30, 0xec, 0x90, 0x00, 0x00, 0x00, 0x00, 0x07, 0x8d, + 0x43, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x09, 0x10, 0xce, 0x90, 0x00, 0x00, + 0x00, 0x00, 0x09, 0xad, 0xbf, 0x20, 0x00, 0x00, 0x00, 0x00, 0x0a, 0xf0, + 0xb0, 0x90, 0x00, 0x00, 0x00, 0x00, 0x0b, 0xe0, 0xaf, 0xa0, 0x00, 0x00, + 0x00, 0x00, 0x0c, 0xd9, 0xcd, 0x10, 0x00, 0x00, 0x00, 0x00, 0x0d, 0xc0, + 0x91, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x0e, 0xb9, 0xaf, 0x10, 0x00, 0x00, + 0x00, 0x00, 0x0f, 0xa9, 0xae, 0x20, 0x00, 0x00, 0x00, 0x00, 0x10, 0x99, + 0x91, 0x10, 0x00, 0x00, 0x00, 0x00, 0x11, 0x89, 0x90, 0x20, 0x00, 0x00, + 0x00, 0x00, 0x12, 0x79, 0x73, 0x10, 0x00, 0x00, 0x00, 0x00, 0x13, 0x69, + 0x72, 0x20, 0x00, 0x00, 0x00, 0x00, 0x14, 0x59, 0x55, 0x10, 0x00, 0x00, + 0x00, 0x00, 0x15, 0x49, 0x54, 0x20, 0x00, 0x00, 0x00, 0x00, 0x16, 0x39, + 0x37, 0x10, 0x00, 0x00, 0x00, 0x00, 0x17, 0x29, 0x36, 0x20, 0x00, 0x00, + 0x00, 0x00, 0x18, 0x22, 0x53, 0x90, 0x00, 0x00, 0x00, 0x00, 0x19, 0x09, + 0x18, 0x20, 0x00, 0x00, 0x00, 0x00, 0x1a, 0x02, 0x35, 0x90, 0x00, 0x00, + 0x00, 0x00, 0x1a, 0xf2, 0x34, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x1b, 0xe2, + 0x17, 0x90, 0x00, 0x00, 0x00, 0x00, 0x1c, 0xd2, 0x16, 0xa0, 0x00, 0x00, + 0x00, 0x00, 0x1d, 0xc1, 0xf9, 0x90, 0x00, 0x00, 0x00, 0x00, 0x1e, 0xb1, + 0xf8, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x1f, 0xa1, 0xdb, 0x90, 0x00, 0x00, + 0x00, 0x00, 0x20, 0x76, 0x2b, 0x20, 0x00, 0x00, 0x00, 0x00, 0x21, 0x81, + 0xbd, 0x90, 0x00, 0x00, 0x00, 0x00, 0x22, 0x56, 0x0d, 0x20, 0x00, 0x00, + 0x00, 0x00, 0x23, 0x6a, 0xda, 0x10, 0x00, 0x00, 0x00, 0x00, 0x24, 0x35, + 0xef, 0x20, 0x00, 0x00, 0x00, 0x00, 0x25, 0x4a, 0xbc, 0x10, 0x00, 0x00, + 0x00, 0x00, 0x26, 0x15, 0xd1, 0x20, 0x00, 0x00, 0x00, 0x00, 0x27, 0x2a, + 0x9e, 0x10, 0x00, 0x00, 0x00, 0x00, 0x27, 0xfe, 0xed, 0xa0, 0x00, 0x00, + 0x00, 0x00, 0x29, 0x0a, 0x80, 0x10, 0x00, 0x00, 0x00, 0x00, 0x29, 0xde, + 0xcf, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x2a, 0xea, 0x62, 0x10, 0x00, 0x00, + 0x00, 0x00, 0x2b, 0xbe, 0xb1, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x2c, 0xd3, + 0x7e, 0x90, 0x00, 0x00, 0x00, 0x00, 0x2d, 0x9e, 0x93, 0xa0, 0x00, 0x00, + 0x00, 0x00, 0x2e, 0xb3, 0x60, 0x90, 0x00, 0x00, 0x00, 0x00, 0x2f, 0x7e, + 0x75, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x30, 0x93, 0x42, 0x90, 0x00, 0x00, + 0x00, 0x00, 0x31, 0x67, 0x92, 0x20, 0x00, 0x00, 0x00, 0x00, 0x32, 0x73, + 0x24, 0x90, 0x00, 0x00, 0x00, 0x00, 0x33, 0x47, 0x74, 0x20, 0x00, 0x00, + 0x00, 0x00, 0x34, 0x53, 0x06, 0x90, 0x00, 0x00, 0x00, 0x00, 0x35, 0x27, + 0x56, 0x20, 0x00, 0x00, 0x00, 0x00, 0x36, 0x32, 0xe8, 0x90, 0x00, 0x00, + 0x00, 0x00, 0x37, 0x07, 0x38, 0x20, 0x00, 0x00, 0x00, 0x00, 0x38, 0x1c, + 0x05, 0x10, 0x00, 0x00, 0x00, 0x00, 0x38, 0xe7, 0x1a, 0x20, 0x00, 0x00, + 0x00, 0x00, 0x39, 0xfb, 0xe7, 0x10, 0x00, 0x00, 0x00, 0x00, 0x3a, 0xc6, + 0xfc, 0x20, 0x00, 0x00, 0x00, 0x00, 0x3b, 0xdb, 0xc9, 0x10, 0x00, 0x00, + 0x00, 0x00, 0x3c, 0xb0, 0x18, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x3d, 0xbb, + 0xab, 0x10, 0x00, 0x00, 0x00, 0x00, 0x3e, 0x8f, 0xfa, 0xa0, 0x00, 0x00, + 0x00, 0x00, 0x3f, 0x9b, 0x8d, 0x10, 0x00, 0x00, 0x00, 0x00, 0x40, 0x6f, + 0xdc, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x41, 0x84, 0xa9, 0x90, 0x00, 0x00, + 0x00, 0x00, 0x42, 0x4f, 0xbe, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x43, 0x64, + 0x8b, 0x90, 0x00, 0x00, 0x00, 0x00, 0x44, 0x2f, 0xa0, 0xa0, 0x00, 0x00, + 0x00, 0x00, 0x45, 0x44, 0x6d, 0x90, 0x00, 0x00, 0x00, 0x00, 0x45, 0xf3, + 0xd3, 0x20, 0x00, 0x00, 0x00, 0x00, 0x47, 0x2d, 0x8a, 0x10, 0x00, 0x00, + 0x00, 0x00, 0x47, 0xd3, 0xb5, 0x20, 0x00, 0x00, 0x00, 0x00, 0x49, 0x0d, + 0x6c, 0x10, 0x00, 0x00, 0x00, 0x00, 0x49, 0xb3, 0x97, 0x20, 0x00, 0x00, + 0x00, 0x00, 0x4a, 0xed, 0x4e, 0x10, 0x00, 0x00, 0x00, 0x00, 0x4b, 0x9c, + 0xb3, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x4c, 0xd6, 0x6a, 0x90, 0x00, 0x00, + 0x00, 0x00, 0x4d, 0x7c, 0x95, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x4e, 0xb6, + 0x4c, 0x90, 0x00, 0x00, 0x00, 0x00, 0x4f, 0x5c, 0x77, 0xa0, 0x00, 0x00, + 0x00, 0x00, 0x50, 0x96, 0x2e, 0x90, 0x00, 0x00, 0x00, 0x00, 0x51, 0x3c, + 0x59, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x52, 0x76, 0x10, 0x90, 0x00, 0x00, + 0x00, 0x00, 0x53, 0x1c, 0x3b, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x54, 0x55, + 0xf2, 0x90, 0x00, 0x00, 0x00, 0x00, 0x54, 0xfc, 0x1d, 0xa0, 0x00, 0x00, + 0x00, 0x00, 0x56, 0x35, 0xd4, 0x90, 0x00, 0x00, 0x00, 0x00, 0x56, 0xe5, + 0x3a, 0x20, 0x00, 0x00, 0x00, 0x00, 0x58, 0x1e, 0xf1, 0x10, 0x00, 0x00, + 0x00, 0x00, 0x58, 0xc5, 0x1c, 0x20, 0x00, 0x00, 0x00, 0x00, 0x59, 0xfe, + 0xd3, 0x10, 0x00, 0x00, 0x00, 0x00, 0x5a, 0xa4, 0xfe, 0x20, 0x00, 0x00, + 0x00, 0x00, 0x5b, 0xde, 0xb5, 0x10, 0x00, 0x00, 0x00, 0x00, 0x5c, 0x84, + 0xe0, 0x20, 0x00, 0x00, 0x00, 0x00, 0x5d, 0xbe, 0x97, 0x10, 0x00, 0x00, + 0x00, 0x00, 0x5e, 0x64, 0xc2, 0x20, 0x00, 0x00, 0x00, 0x00, 0x5f, 0x9e, + 0x79, 0x10, 0x00, 0x00, 0x00, 0x00, 0x60, 0x4d, 0xde, 0xa0, 0x00, 0x00, + 0x00, 0x00, 0x61, 0x87, 0x95, 0x90, 0x00, 0x00, 0x00, 0x00, 0x62, 0x2d, + 0xc0, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x63, 0x67, 0x77, 0x90, 0x00, 0x00, + 0x00, 0x00, 0x64, 0x0d, 0xa2, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x65, 0x47, + 0x59, 0x90, 0x00, 0x00, 0x00, 0x00, 0x65, 0xed, 0x84, 0xa0, 0x00, 0x00, + 0x00, 0x00, 0x67, 0x27, 0x3b, 0x90, 0x00, 0x00, 0x00, 0x00, 0x67, 0xcd, + 0x66, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x69, 0x07, 0x1d, 0x90, 0x00, 0x00, + 0x00, 0x00, 0x69, 0xad, 0x48, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x6a, 0xe6, + 0xff, 0x90, 0x00, 0x00, 0x00, 0x00, 0x6b, 0x96, 0x65, 0x20, 0x00, 0x00, + 0x00, 0x00, 0x6c, 0xd0, 0x1c, 0x10, 0x00, 0x00, 0x00, 0x00, 0x6d, 0x76, + 0x47, 0x20, 0x00, 0x00, 0x00, 0x00, 0x6e, 0xaf, 0xfe, 0x10, 0x00, 0x00, + 0x00, 0x00, 0x6f, 0x56, 0x29, 0x20, 0x00, 0x00, 0x00, 0x00, 0x70, 0x8f, + 0xe0, 0x10, 0x00, 0x00, 0x00, 0x00, 0x71, 0x36, 0x0b, 0x20, 0x00, 0x00, + 0x00, 0x00, 0x72, 0x6f, 0xc2, 0x10, 0x00, 0x00, 0x00, 0x00, 0x73, 0x15, + 0xed, 0x20, 0x00, 0x00, 0x00, 0x00, 0x74, 0x4f, 0xa4, 0x10, 0x00, 0x00, + 0x00, 0x00, 0x74, 0xff, 0x09, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x76, 0x38, + 0xc0, 0x90, 0x00, 0x00, 0x00, 0x00, 0x76, 0xde, 0xeb, 0xa0, 0x00, 0x00, + 0x00, 0x00, 0x78, 0x18, 0xa2, 0x90, 0x00, 0x00, 0x00, 0x00, 0x78, 0xbe, + 0xcd, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x79, 0xf8, 0x84, 0x90, 0x00, 0x00, + 0x00, 0x00, 0x7a, 0x9e, 0xaf, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x7b, 0xd8, + 0x66, 0x90, 0x00, 0x00, 0x00, 0x00, 0x7c, 0x7e, 0x91, 0xa0, 0x00, 0x00, + 0x00, 0x00, 0x7d, 0xb8, 0x48, 0x90, 0x00, 0x00, 0x00, 0x00, 0x7e, 0x5e, + 0x73, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x7f, 0x98, 0x2a, 0x90, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x03, 0x04, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0xff, 0xff, 0x91, 0x26, 0x00, 0x00, 0xff, 0xff, + 0x9d, 0x90, 0x01, 0x04, 0xff, 0xff, 0x8f, 0x80, 0x00, 0x08, 0xff, 0xff, + 0x9d, 0x90, 0x01, 0x0c, 0xff, 0xff, 0x9d, 0x90, 0x01, 0x10, 0x4c, 0x4d, + 0x54, 0x00, 0x50, 0x44, 0x54, 0x00, 0x50, 0x53, 0x54, 0x00, 0x50, 0x57, + 0x54, 0x00, 0x50, 0x50, 0x54, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, + 0x00, 0x00, 0x00, 0x01, 0x0a, 0x50, 0x53, 0x54, 0x38, 0x50, 0x44, 0x54, + 0x2c, 0x4d, 0x33, 0x2e, 0x32, 0x2e, 0x30, 0x2c, 0x4d, 0x31, 0x31, 0x2e, + 0x31, 0x2e, 0x30, 0x0a }; -unsigned int America_Los_Angeles_len = 2845; +unsigned int America_Los_Angeles_len = 2836; unsigned char America_New_York[] = { 0x54, 0x5a, 0x69, 0x66, 0x32, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, @@ -349,203 +348,202 @@ unsigned char America_New_York[] = { 0x54, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, 0x54, 0x5a, 0x69, 0x66, 0x32, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, - 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xed, - 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x14, 0xf8, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, 0x5e, 0x03, 0xf0, 0x90, - 0xff, 0xff, 0xff, 0xff, 0x9e, 0xa6, 0x1e, 0x70, 0xff, 0xff, 0xff, 0xff, - 0x9f, 0xba, 0xeb, 0x60, 0xff, 0xff, 0xff, 0xff, 0xa0, 0x86, 0x00, 0x70, - 0xff, 0xff, 0xff, 0xff, 0xa1, 0x9a, 0xcd, 0x60, 0xff, 0xff, 0xff, 0xff, - 0xa2, 0x65, 0xe2, 0x70, 0xff, 0xff, 0xff, 0xff, 0xa3, 0x83, 0xe9, 0xe0, - 0xff, 0xff, 0xff, 0xff, 0xa4, 0x6a, 0xae, 0x70, 0xff, 0xff, 0xff, 0xff, - 0xa5, 0x35, 0xa7, 0x60, 0xff, 0xff, 0xff, 0xff, 0xa6, 0x53, 0xca, 0xf0, - 0xff, 0xff, 0xff, 0xff, 0xa7, 0x15, 0x89, 0x60, 0xff, 0xff, 0xff, 0xff, - 0xa8, 0x33, 0xac, 0xf0, 0xff, 0xff, 0xff, 0xff, 0xa8, 0xfe, 0xa5, 0xe0, - 0xff, 0xff, 0xff, 0xff, 0xaa, 0x13, 0x8e, 0xf0, 0xff, 0xff, 0xff, 0xff, - 0xaa, 0xde, 0x87, 0xe0, 0xff, 0xff, 0xff, 0xff, 0xab, 0xf3, 0x70, 0xf0, - 0xff, 0xff, 0xff, 0xff, 0xac, 0xbe, 0x69, 0xe0, 0xff, 0xff, 0xff, 0xff, - 0xad, 0xd3, 0x52, 0xf0, 0xff, 0xff, 0xff, 0xff, 0xae, 0x9e, 0x4b, 0xe0, - 0xff, 0xff, 0xff, 0xff, 0xaf, 0xb3, 0x34, 0xf0, 0xff, 0xff, 0xff, 0xff, - 0xb0, 0x7e, 0x2d, 0xe0, 0xff, 0xff, 0xff, 0xff, 0xb1, 0x9c, 0x51, 0x70, - 0xff, 0xff, 0xff, 0xff, 0xb2, 0x67, 0x4a, 0x60, 0xff, 0xff, 0xff, 0xff, - 0xb3, 0x7c, 0x33, 0x70, 0xff, 0xff, 0xff, 0xff, 0xb4, 0x47, 0x2c, 0x60, - 0xff, 0xff, 0xff, 0xff, 0xb5, 0x5c, 0x15, 0x70, 0xff, 0xff, 0xff, 0xff, - 0xb6, 0x27, 0x0e, 0x60, 0xff, 0xff, 0xff, 0xff, 0xb7, 0x3b, 0xf7, 0x70, - 0xff, 0xff, 0xff, 0xff, 0xb8, 0x06, 0xf0, 0x60, 0xff, 0xff, 0xff, 0xff, - 0xb9, 0x1b, 0xd9, 0x70, 0xff, 0xff, 0xff, 0xff, 0xb9, 0xe6, 0xd2, 0x60, - 0xff, 0xff, 0xff, 0xff, 0xbb, 0x04, 0xf5, 0xf0, 0xff, 0xff, 0xff, 0xff, - 0xbb, 0xc6, 0xb4, 0x60, 0xff, 0xff, 0xff, 0xff, 0xbc, 0xe4, 0xd7, 0xf0, - 0xff, 0xff, 0xff, 0xff, 0xbd, 0xaf, 0xd0, 0xe0, 0xff, 0xff, 0xff, 0xff, - 0xbe, 0xc4, 0xb9, 0xf0, 0xff, 0xff, 0xff, 0xff, 0xbf, 0x8f, 0xb2, 0xe0, - 0xff, 0xff, 0xff, 0xff, 0xc0, 0xa4, 0x9b, 0xf0, 0xff, 0xff, 0xff, 0xff, - 0xc1, 0x6f, 0x94, 0xe0, 0xff, 0xff, 0xff, 0xff, 0xc2, 0x84, 0x7d, 0xf0, - 0xff, 0xff, 0xff, 0xff, 0xc3, 0x4f, 0x76, 0xe0, 0xff, 0xff, 0xff, 0xff, - 0xc4, 0x64, 0x5f, 0xf0, 0xff, 0xff, 0xff, 0xff, 0xc5, 0x2f, 0x58, 0xe0, - 0xff, 0xff, 0xff, 0xff, 0xc6, 0x4d, 0x7c, 0x70, 0xff, 0xff, 0xff, 0xff, - 0xc7, 0x0f, 0x3a, 0xe0, 0xff, 0xff, 0xff, 0xff, 0xc8, 0x2d, 0x5e, 0x70, - 0xff, 0xff, 0xff, 0xff, 0xc8, 0xf8, 0x57, 0x60, 0xff, 0xff, 0xff, 0xff, - 0xca, 0x0d, 0x40, 0x70, 0xff, 0xff, 0xff, 0xff, 0xca, 0xd8, 0x39, 0x60, - 0xff, 0xff, 0xff, 0xff, 0xcb, 0x88, 0xf0, 0x70, 0xff, 0xff, 0xff, 0xff, - 0xd2, 0x23, 0xf4, 0x70, 0xff, 0xff, 0xff, 0xff, 0xd2, 0x60, 0xfb, 0xe0, - 0xff, 0xff, 0xff, 0xff, 0xd3, 0x75, 0xe4, 0xf0, 0xff, 0xff, 0xff, 0xff, - 0xd4, 0x40, 0xdd, 0xe0, 0xff, 0xff, 0xff, 0xff, 0xd5, 0x55, 0xc6, 0xf0, - 0xff, 0xff, 0xff, 0xff, 0xd6, 0x20, 0xbf, 0xe0, 0xff, 0xff, 0xff, 0xff, - 0xd7, 0x35, 0xa8, 0xf0, 0xff, 0xff, 0xff, 0xff, 0xd8, 0x00, 0xa1, 0xe0, - 0xff, 0xff, 0xff, 0xff, 0xd9, 0x15, 0x8a, 0xf0, 0xff, 0xff, 0xff, 0xff, - 0xd9, 0xe0, 0x83, 0xe0, 0xff, 0xff, 0xff, 0xff, 0xda, 0xfe, 0xa7, 0x70, - 0xff, 0xff, 0xff, 0xff, 0xdb, 0xc0, 0x65, 0xe0, 0xff, 0xff, 0xff, 0xff, - 0xdc, 0xde, 0x89, 0x70, 0xff, 0xff, 0xff, 0xff, 0xdd, 0xa9, 0x82, 0x60, - 0xff, 0xff, 0xff, 0xff, 0xde, 0xbe, 0x6b, 0x70, 0xff, 0xff, 0xff, 0xff, - 0xdf, 0x89, 0x64, 0x60, 0xff, 0xff, 0xff, 0xff, 0xe0, 0x9e, 0x4d, 0x70, - 0xff, 0xff, 0xff, 0xff, 0xe1, 0x69, 0x46, 0x60, 0xff, 0xff, 0xff, 0xff, - 0xe2, 0x7e, 0x2f, 0x70, 0xff, 0xff, 0xff, 0xff, 0xe3, 0x49, 0x28, 0x60, - 0xff, 0xff, 0xff, 0xff, 0xe4, 0x5e, 0x11, 0x70, 0xff, 0xff, 0xff, 0xff, - 0xe5, 0x57, 0x2e, 0xe0, 0xff, 0xff, 0xff, 0xff, 0xe6, 0x47, 0x2d, 0xf0, - 0xff, 0xff, 0xff, 0xff, 0xe7, 0x37, 0x10, 0xe0, 0xff, 0xff, 0xff, 0xff, - 0xe8, 0x27, 0x0f, 0xf0, 0xff, 0xff, 0xff, 0xff, 0xe9, 0x16, 0xf2, 0xe0, - 0xff, 0xff, 0xff, 0xff, 0xea, 0x06, 0xf1, 0xf0, 0xff, 0xff, 0xff, 0xff, - 0xea, 0xf6, 0xd4, 0xe0, 0xff, 0xff, 0xff, 0xff, 0xeb, 0xe6, 0xd3, 0xf0, - 0xff, 0xff, 0xff, 0xff, 0xec, 0xd6, 0xb6, 0xe0, 0xff, 0xff, 0xff, 0xff, - 0xed, 0xc6, 0xb5, 0xf0, 0xff, 0xff, 0xff, 0xff, 0xee, 0xbf, 0xd3, 0x60, - 0xff, 0xff, 0xff, 0xff, 0xef, 0xaf, 0xd2, 0x70, 0xff, 0xff, 0xff, 0xff, - 0xf0, 0x9f, 0xb5, 0x60, 0xff, 0xff, 0xff, 0xff, 0xf1, 0x8f, 0xb4, 0x70, - 0xff, 0xff, 0xff, 0xff, 0xf2, 0x7f, 0x97, 0x60, 0xff, 0xff, 0xff, 0xff, - 0xf3, 0x6f, 0x96, 0x70, 0xff, 0xff, 0xff, 0xff, 0xf4, 0x5f, 0x79, 0x60, - 0xff, 0xff, 0xff, 0xff, 0xf5, 0x4f, 0x78, 0x70, 0xff, 0xff, 0xff, 0xff, - 0xf6, 0x3f, 0x5b, 0x60, 0xff, 0xff, 0xff, 0xff, 0xf7, 0x2f, 0x5a, 0x70, - 0xff, 0xff, 0xff, 0xff, 0xf8, 0x28, 0x77, 0xe0, 0xff, 0xff, 0xff, 0xff, - 0xf9, 0x0f, 0x3c, 0x70, 0xff, 0xff, 0xff, 0xff, 0xfa, 0x08, 0x59, 0xe0, - 0xff, 0xff, 0xff, 0xff, 0xfa, 0xf8, 0x58, 0xf0, 0xff, 0xff, 0xff, 0xff, - 0xfb, 0xe8, 0x3b, 0xe0, 0xff, 0xff, 0xff, 0xff, 0xfc, 0xd8, 0x3a, 0xf0, - 0xff, 0xff, 0xff, 0xff, 0xfd, 0xc8, 0x1d, 0xe0, 0xff, 0xff, 0xff, 0xff, - 0xfe, 0xb8, 0x1c, 0xf0, 0xff, 0xff, 0xff, 0xff, 0xff, 0xa7, 0xff, 0xe0, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x97, 0xfe, 0xf0, 0x00, 0x00, 0x00, 0x00, - 0x01, 0x87, 0xe1, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x02, 0x77, 0xe0, 0xf0, - 0x00, 0x00, 0x00, 0x00, 0x03, 0x70, 0xfe, 0x60, 0x00, 0x00, 0x00, 0x00, - 0x04, 0x60, 0xfd, 0x70, 0x00, 0x00, 0x00, 0x00, 0x05, 0x50, 0xe0, 0x60, - 0x00, 0x00, 0x00, 0x00, 0x06, 0x40, 0xdf, 0x70, 0x00, 0x00, 0x00, 0x00, - 0x07, 0x30, 0xc2, 0x60, 0x00, 0x00, 0x00, 0x00, 0x07, 0x8d, 0x19, 0x70, - 0x00, 0x00, 0x00, 0x00, 0x09, 0x10, 0xa4, 0x60, 0x00, 0x00, 0x00, 0x00, - 0x09, 0xad, 0x94, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x0a, 0xf0, 0x86, 0x60, - 0x00, 0x00, 0x00, 0x00, 0x0b, 0xe0, 0x85, 0x70, 0x00, 0x00, 0x00, 0x00, - 0x0c, 0xd9, 0xa2, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x0d, 0xc0, 0x67, 0x70, - 0x00, 0x00, 0x00, 0x00, 0x0e, 0xb9, 0x84, 0xe0, 0x00, 0x00, 0x00, 0x00, - 0x0f, 0xa9, 0x83, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x10, 0x99, 0x66, 0xe0, - 0x00, 0x00, 0x00, 0x00, 0x11, 0x89, 0x65, 0xf0, 0x00, 0x00, 0x00, 0x00, - 0x12, 0x79, 0x48, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x13, 0x69, 0x47, 0xf0, - 0x00, 0x00, 0x00, 0x00, 0x14, 0x59, 0x2a, 0xe0, 0x00, 0x00, 0x00, 0x00, - 0x15, 0x49, 0x29, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x16, 0x39, 0x0c, 0xe0, - 0x00, 0x00, 0x00, 0x00, 0x17, 0x29, 0x0b, 0xf0, 0x00, 0x00, 0x00, 0x00, - 0x18, 0x22, 0x29, 0x60, 0x00, 0x00, 0x00, 0x00, 0x19, 0x08, 0xed, 0xf0, - 0x00, 0x00, 0x00, 0x00, 0x1a, 0x02, 0x0b, 0x60, 0x00, 0x00, 0x00, 0x00, - 0x1a, 0xf2, 0x0a, 0x70, 0x00, 0x00, 0x00, 0x00, 0x1b, 0xe1, 0xed, 0x60, - 0x00, 0x00, 0x00, 0x00, 0x1c, 0xd1, 0xec, 0x70, 0x00, 0x00, 0x00, 0x00, - 0x1d, 0xc1, 0xcf, 0x60, 0x00, 0x00, 0x00, 0x00, 0x1e, 0xb1, 0xce, 0x70, - 0x00, 0x00, 0x00, 0x00, 0x1f, 0xa1, 0xb1, 0x60, 0x00, 0x00, 0x00, 0x00, - 0x20, 0x76, 0x00, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x21, 0x81, 0x93, 0x60, - 0x00, 0x00, 0x00, 0x00, 0x22, 0x55, 0xe2, 0xf0, 0x00, 0x00, 0x00, 0x00, - 0x23, 0x6a, 0xaf, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x24, 0x35, 0xc4, 0xf0, - 0x00, 0x00, 0x00, 0x00, 0x25, 0x4a, 0x91, 0xe0, 0x00, 0x00, 0x00, 0x00, - 0x26, 0x15, 0xa6, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x27, 0x2a, 0x73, 0xe0, - 0x00, 0x00, 0x00, 0x00, 0x27, 0xfe, 0xc3, 0x70, 0x00, 0x00, 0x00, 0x00, - 0x29, 0x0a, 0x55, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x29, 0xde, 0xa5, 0x70, - 0x00, 0x00, 0x00, 0x00, 0x2a, 0xea, 0x37, 0xe0, 0x00, 0x00, 0x00, 0x00, - 0x2b, 0xbe, 0x87, 0x70, 0x00, 0x00, 0x00, 0x00, 0x2c, 0xd3, 0x54, 0x60, - 0x00, 0x00, 0x00, 0x00, 0x2d, 0x9e, 0x69, 0x70, 0x00, 0x00, 0x00, 0x00, - 0x2e, 0xb3, 0x36, 0x60, 0x00, 0x00, 0x00, 0x00, 0x2f, 0x7e, 0x4b, 0x70, - 0x00, 0x00, 0x00, 0x00, 0x30, 0x93, 0x18, 0x60, 0x00, 0x00, 0x00, 0x00, - 0x31, 0x67, 0x67, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x32, 0x72, 0xfa, 0x60, - 0x00, 0x00, 0x00, 0x00, 0x33, 0x47, 0x49, 0xf0, 0x00, 0x00, 0x00, 0x00, - 0x34, 0x52, 0xdc, 0x60, 0x00, 0x00, 0x00, 0x00, 0x35, 0x27, 0x2b, 0xf0, - 0x00, 0x00, 0x00, 0x00, 0x36, 0x32, 0xbe, 0x60, 0x00, 0x00, 0x00, 0x00, - 0x37, 0x07, 0x0d, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x38, 0x1b, 0xda, 0xe0, - 0x00, 0x00, 0x00, 0x00, 0x38, 0xe6, 0xef, 0xf0, 0x00, 0x00, 0x00, 0x00, - 0x39, 0xfb, 0xbc, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x3a, 0xc6, 0xd1, 0xf0, - 0x00, 0x00, 0x00, 0x00, 0x3b, 0xdb, 0x9e, 0xe0, 0x00, 0x00, 0x00, 0x00, - 0x3c, 0xaf, 0xee, 0x70, 0x00, 0x00, 0x00, 0x00, 0x3d, 0xbb, 0x80, 0xe0, - 0x00, 0x00, 0x00, 0x00, 0x3e, 0x8f, 0xd0, 0x70, 0x00, 0x00, 0x00, 0x00, - 0x3f, 0x9b, 0x62, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x40, 0x6f, 0xb2, 0x70, - 0x00, 0x00, 0x00, 0x00, 0x41, 0x84, 0x7f, 0x60, 0x00, 0x00, 0x00, 0x00, - 0x42, 0x4f, 0x94, 0x70, 0x00, 0x00, 0x00, 0x00, 0x43, 0x64, 0x61, 0x60, - 0x00, 0x00, 0x00, 0x00, 0x44, 0x2f, 0x76, 0x70, 0x00, 0x00, 0x00, 0x00, - 0x45, 0x44, 0x43, 0x60, 0x00, 0x00, 0x00, 0x00, 0x45, 0xf3, 0xa8, 0xf0, - 0x00, 0x00, 0x00, 0x00, 0x47, 0x2d, 0x5f, 0xe0, 0x00, 0x00, 0x00, 0x00, - 0x47, 0xd3, 0x8a, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x49, 0x0d, 0x41, 0xe0, - 0x00, 0x00, 0x00, 0x00, 0x49, 0xb3, 0x6c, 0xf0, 0x00, 0x00, 0x00, 0x00, - 0x4a, 0xed, 0x23, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x4b, 0x9c, 0x89, 0x70, - 0x00, 0x00, 0x00, 0x00, 0x4c, 0xd6, 0x40, 0x60, 0x00, 0x00, 0x00, 0x00, - 0x4d, 0x7c, 0x6b, 0x70, 0x00, 0x00, 0x00, 0x00, 0x4e, 0xb6, 0x22, 0x60, - 0x00, 0x00, 0x00, 0x00, 0x4f, 0x5c, 0x4d, 0x70, 0x00, 0x00, 0x00, 0x00, - 0x50, 0x96, 0x04, 0x60, 0x00, 0x00, 0x00, 0x00, 0x51, 0x3c, 0x2f, 0x70, - 0x00, 0x00, 0x00, 0x00, 0x52, 0x75, 0xe6, 0x60, 0x00, 0x00, 0x00, 0x00, - 0x53, 0x1c, 0x11, 0x70, 0x00, 0x00, 0x00, 0x00, 0x54, 0x55, 0xc8, 0x60, - 0x00, 0x00, 0x00, 0x00, 0x54, 0xfb, 0xf3, 0x70, 0x00, 0x00, 0x00, 0x00, - 0x56, 0x35, 0xaa, 0x60, 0x00, 0x00, 0x00, 0x00, 0x56, 0xe5, 0x0f, 0xf0, - 0x00, 0x00, 0x00, 0x00, 0x58, 0x1e, 0xc6, 0xe0, 0x00, 0x00, 0x00, 0x00, - 0x58, 0xc4, 0xf1, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x59, 0xfe, 0xa8, 0xe0, - 0x00, 0x00, 0x00, 0x00, 0x5a, 0xa4, 0xd3, 0xf0, 0x00, 0x00, 0x00, 0x00, - 0x5b, 0xde, 0x8a, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x5c, 0x84, 0xb5, 0xf0, - 0x00, 0x00, 0x00, 0x00, 0x5d, 0xbe, 0x6c, 0xe0, 0x00, 0x00, 0x00, 0x00, - 0x5e, 0x64, 0x97, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x5f, 0x9e, 0x4e, 0xe0, - 0x00, 0x00, 0x00, 0x00, 0x60, 0x4d, 0xb4, 0x70, 0x00, 0x00, 0x00, 0x00, - 0x61, 0x87, 0x6b, 0x60, 0x00, 0x00, 0x00, 0x00, 0x62, 0x2d, 0x96, 0x70, - 0x00, 0x00, 0x00, 0x00, 0x63, 0x67, 0x4d, 0x60, 0x00, 0x00, 0x00, 0x00, - 0x64, 0x0d, 0x78, 0x70, 0x00, 0x00, 0x00, 0x00, 0x65, 0x47, 0x2f, 0x60, - 0x00, 0x00, 0x00, 0x00, 0x65, 0xed, 0x5a, 0x70, 0x00, 0x00, 0x00, 0x00, - 0x67, 0x27, 0x11, 0x60, 0x00, 0x00, 0x00, 0x00, 0x67, 0xcd, 0x3c, 0x70, - 0x00, 0x00, 0x00, 0x00, 0x69, 0x06, 0xf3, 0x60, 0x00, 0x00, 0x00, 0x00, - 0x69, 0xad, 0x1e, 0x70, 0x00, 0x00, 0x00, 0x00, 0x6a, 0xe6, 0xd5, 0x60, - 0x00, 0x00, 0x00, 0x00, 0x6b, 0x96, 0x3a, 0xf0, 0x00, 0x00, 0x00, 0x00, - 0x6c, 0xcf, 0xf1, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x6d, 0x76, 0x1c, 0xf0, - 0x00, 0x00, 0x00, 0x00, 0x6e, 0xaf, 0xd3, 0xe0, 0x00, 0x00, 0x00, 0x00, - 0x6f, 0x55, 0xfe, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x70, 0x8f, 0xb5, 0xe0, - 0x00, 0x00, 0x00, 0x00, 0x71, 0x35, 0xe0, 0xf0, 0x00, 0x00, 0x00, 0x00, - 0x72, 0x6f, 0x97, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x73, 0x15, 0xc2, 0xf0, - 0x00, 0x00, 0x00, 0x00, 0x74, 0x4f, 0x79, 0xe0, 0x00, 0x00, 0x00, 0x00, - 0x74, 0xfe, 0xdf, 0x70, 0x00, 0x00, 0x00, 0x00, 0x76, 0x38, 0x96, 0x60, - 0x00, 0x00, 0x00, 0x00, 0x76, 0xde, 0xc1, 0x70, 0x00, 0x00, 0x00, 0x00, - 0x78, 0x18, 0x78, 0x60, 0x00, 0x00, 0x00, 0x00, 0x78, 0xbe, 0xa3, 0x70, - 0x00, 0x00, 0x00, 0x00, 0x79, 0xf8, 0x5a, 0x60, 0x00, 0x00, 0x00, 0x00, - 0x7a, 0x9e, 0x85, 0x70, 0x00, 0x00, 0x00, 0x00, 0x7b, 0xd8, 0x3c, 0x60, - 0x00, 0x00, 0x00, 0x00, 0x7c, 0x7e, 0x67, 0x70, 0x00, 0x00, 0x00, 0x00, - 0x7d, 0xb8, 0x1e, 0x60, 0x00, 0x00, 0x00, 0x00, 0x7e, 0x5e, 0x49, 0x70, - 0x00, 0x00, 0x00, 0x00, 0x7f, 0x98, 0x00, 0x60, 0x00, 0x02, 0x01, 0x02, - 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, - 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, - 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, - 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x03, 0x04, - 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, - 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, - 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, - 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, - 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, - 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, - 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, - 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, - 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, - 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, - 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xec, + 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x14, 0xff, 0xff, 0xff, 0xff, + 0x5e, 0x03, 0xf0, 0x90, 0xff, 0xff, 0xff, 0xff, 0x9e, 0xa6, 0x1e, 0x70, + 0xff, 0xff, 0xff, 0xff, 0x9f, 0xba, 0xeb, 0x60, 0xff, 0xff, 0xff, 0xff, + 0xa0, 0x86, 0x00, 0x70, 0xff, 0xff, 0xff, 0xff, 0xa1, 0x9a, 0xcd, 0x60, + 0xff, 0xff, 0xff, 0xff, 0xa2, 0x65, 0xe2, 0x70, 0xff, 0xff, 0xff, 0xff, + 0xa3, 0x83, 0xe9, 0xe0, 0xff, 0xff, 0xff, 0xff, 0xa4, 0x6a, 0xae, 0x70, + 0xff, 0xff, 0xff, 0xff, 0xa5, 0x35, 0xa7, 0x60, 0xff, 0xff, 0xff, 0xff, + 0xa6, 0x53, 0xca, 0xf0, 0xff, 0xff, 0xff, 0xff, 0xa7, 0x15, 0x89, 0x60, + 0xff, 0xff, 0xff, 0xff, 0xa8, 0x33, 0xac, 0xf0, 0xff, 0xff, 0xff, 0xff, + 0xa8, 0xfe, 0xa5, 0xe0, 0xff, 0xff, 0xff, 0xff, 0xaa, 0x13, 0x8e, 0xf0, + 0xff, 0xff, 0xff, 0xff, 0xaa, 0xde, 0x87, 0xe0, 0xff, 0xff, 0xff, 0xff, + 0xab, 0xf3, 0x70, 0xf0, 0xff, 0xff, 0xff, 0xff, 0xac, 0xbe, 0x69, 0xe0, + 0xff, 0xff, 0xff, 0xff, 0xad, 0xd3, 0x52, 0xf0, 0xff, 0xff, 0xff, 0xff, + 0xae, 0x9e, 0x4b, 0xe0, 0xff, 0xff, 0xff, 0xff, 0xaf, 0xb3, 0x34, 0xf0, + 0xff, 0xff, 0xff, 0xff, 0xb0, 0x7e, 0x2d, 0xe0, 0xff, 0xff, 0xff, 0xff, + 0xb1, 0x9c, 0x51, 0x70, 0xff, 0xff, 0xff, 0xff, 0xb2, 0x67, 0x4a, 0x60, + 0xff, 0xff, 0xff, 0xff, 0xb3, 0x7c, 0x33, 0x70, 0xff, 0xff, 0xff, 0xff, + 0xb4, 0x47, 0x2c, 0x60, 0xff, 0xff, 0xff, 0xff, 0xb5, 0x5c, 0x15, 0x70, + 0xff, 0xff, 0xff, 0xff, 0xb6, 0x27, 0x0e, 0x60, 0xff, 0xff, 0xff, 0xff, + 0xb7, 0x3b, 0xf7, 0x70, 0xff, 0xff, 0xff, 0xff, 0xb8, 0x06, 0xf0, 0x60, + 0xff, 0xff, 0xff, 0xff, 0xb9, 0x1b, 0xd9, 0x70, 0xff, 0xff, 0xff, 0xff, + 0xb9, 0xe6, 0xd2, 0x60, 0xff, 0xff, 0xff, 0xff, 0xbb, 0x04, 0xf5, 0xf0, + 0xff, 0xff, 0xff, 0xff, 0xbb, 0xc6, 0xb4, 0x60, 0xff, 0xff, 0xff, 0xff, + 0xbc, 0xe4, 0xd7, 0xf0, 0xff, 0xff, 0xff, 0xff, 0xbd, 0xaf, 0xd0, 0xe0, + 0xff, 0xff, 0xff, 0xff, 0xbe, 0xc4, 0xb9, 0xf0, 0xff, 0xff, 0xff, 0xff, + 0xbf, 0x8f, 0xb2, 0xe0, 0xff, 0xff, 0xff, 0xff, 0xc0, 0xa4, 0x9b, 0xf0, + 0xff, 0xff, 0xff, 0xff, 0xc1, 0x6f, 0x94, 0xe0, 0xff, 0xff, 0xff, 0xff, + 0xc2, 0x84, 0x7d, 0xf0, 0xff, 0xff, 0xff, 0xff, 0xc3, 0x4f, 0x76, 0xe0, + 0xff, 0xff, 0xff, 0xff, 0xc4, 0x64, 0x5f, 0xf0, 0xff, 0xff, 0xff, 0xff, + 0xc5, 0x2f, 0x58, 0xe0, 0xff, 0xff, 0xff, 0xff, 0xc6, 0x4d, 0x7c, 0x70, + 0xff, 0xff, 0xff, 0xff, 0xc7, 0x0f, 0x3a, 0xe0, 0xff, 0xff, 0xff, 0xff, + 0xc8, 0x2d, 0x5e, 0x70, 0xff, 0xff, 0xff, 0xff, 0xc8, 0xf8, 0x57, 0x60, + 0xff, 0xff, 0xff, 0xff, 0xca, 0x0d, 0x40, 0x70, 0xff, 0xff, 0xff, 0xff, + 0xca, 0xd8, 0x39, 0x60, 0xff, 0xff, 0xff, 0xff, 0xcb, 0x88, 0xf0, 0x70, + 0xff, 0xff, 0xff, 0xff, 0xd2, 0x23, 0xf4, 0x70, 0xff, 0xff, 0xff, 0xff, + 0xd2, 0x60, 0xfb, 0xe0, 0xff, 0xff, 0xff, 0xff, 0xd3, 0x75, 0xe4, 0xf0, + 0xff, 0xff, 0xff, 0xff, 0xd4, 0x40, 0xdd, 0xe0, 0xff, 0xff, 0xff, 0xff, + 0xd5, 0x55, 0xc6, 0xf0, 0xff, 0xff, 0xff, 0xff, 0xd6, 0x20, 0xbf, 0xe0, + 0xff, 0xff, 0xff, 0xff, 0xd7, 0x35, 0xa8, 0xf0, 0xff, 0xff, 0xff, 0xff, + 0xd8, 0x00, 0xa1, 0xe0, 0xff, 0xff, 0xff, 0xff, 0xd9, 0x15, 0x8a, 0xf0, + 0xff, 0xff, 0xff, 0xff, 0xd9, 0xe0, 0x83, 0xe0, 0xff, 0xff, 0xff, 0xff, + 0xda, 0xfe, 0xa7, 0x70, 0xff, 0xff, 0xff, 0xff, 0xdb, 0xc0, 0x65, 0xe0, + 0xff, 0xff, 0xff, 0xff, 0xdc, 0xde, 0x89, 0x70, 0xff, 0xff, 0xff, 0xff, + 0xdd, 0xa9, 0x82, 0x60, 0xff, 0xff, 0xff, 0xff, 0xde, 0xbe, 0x6b, 0x70, + 0xff, 0xff, 0xff, 0xff, 0xdf, 0x89, 0x64, 0x60, 0xff, 0xff, 0xff, 0xff, + 0xe0, 0x9e, 0x4d, 0x70, 0xff, 0xff, 0xff, 0xff, 0xe1, 0x69, 0x46, 0x60, + 0xff, 0xff, 0xff, 0xff, 0xe2, 0x7e, 0x2f, 0x70, 0xff, 0xff, 0xff, 0xff, + 0xe3, 0x49, 0x28, 0x60, 0xff, 0xff, 0xff, 0xff, 0xe4, 0x5e, 0x11, 0x70, + 0xff, 0xff, 0xff, 0xff, 0xe5, 0x57, 0x2e, 0xe0, 0xff, 0xff, 0xff, 0xff, + 0xe6, 0x47, 0x2d, 0xf0, 0xff, 0xff, 0xff, 0xff, 0xe7, 0x37, 0x10, 0xe0, + 0xff, 0xff, 0xff, 0xff, 0xe8, 0x27, 0x0f, 0xf0, 0xff, 0xff, 0xff, 0xff, + 0xe9, 0x16, 0xf2, 0xe0, 0xff, 0xff, 0xff, 0xff, 0xea, 0x06, 0xf1, 0xf0, + 0xff, 0xff, 0xff, 0xff, 0xea, 0xf6, 0xd4, 0xe0, 0xff, 0xff, 0xff, 0xff, + 0xeb, 0xe6, 0xd3, 0xf0, 0xff, 0xff, 0xff, 0xff, 0xec, 0xd6, 0xb6, 0xe0, + 0xff, 0xff, 0xff, 0xff, 0xed, 0xc6, 0xb5, 0xf0, 0xff, 0xff, 0xff, 0xff, + 0xee, 0xbf, 0xd3, 0x60, 0xff, 0xff, 0xff, 0xff, 0xef, 0xaf, 0xd2, 0x70, + 0xff, 0xff, 0xff, 0xff, 0xf0, 0x9f, 0xb5, 0x60, 0xff, 0xff, 0xff, 0xff, + 0xf1, 0x8f, 0xb4, 0x70, 0xff, 0xff, 0xff, 0xff, 0xf2, 0x7f, 0x97, 0x60, + 0xff, 0xff, 0xff, 0xff, 0xf3, 0x6f, 0x96, 0x70, 0xff, 0xff, 0xff, 0xff, + 0xf4, 0x5f, 0x79, 0x60, 0xff, 0xff, 0xff, 0xff, 0xf5, 0x4f, 0x78, 0x70, + 0xff, 0xff, 0xff, 0xff, 0xf6, 0x3f, 0x5b, 0x60, 0xff, 0xff, 0xff, 0xff, + 0xf7, 0x2f, 0x5a, 0x70, 0xff, 0xff, 0xff, 0xff, 0xf8, 0x28, 0x77, 0xe0, + 0xff, 0xff, 0xff, 0xff, 0xf9, 0x0f, 0x3c, 0x70, 0xff, 0xff, 0xff, 0xff, + 0xfa, 0x08, 0x59, 0xe0, 0xff, 0xff, 0xff, 0xff, 0xfa, 0xf8, 0x58, 0xf0, + 0xff, 0xff, 0xff, 0xff, 0xfb, 0xe8, 0x3b, 0xe0, 0xff, 0xff, 0xff, 0xff, + 0xfc, 0xd8, 0x3a, 0xf0, 0xff, 0xff, 0xff, 0xff, 0xfd, 0xc8, 0x1d, 0xe0, + 0xff, 0xff, 0xff, 0xff, 0xfe, 0xb8, 0x1c, 0xf0, 0xff, 0xff, 0xff, 0xff, + 0xff, 0xa7, 0xff, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x97, 0xfe, 0xf0, + 0x00, 0x00, 0x00, 0x00, 0x01, 0x87, 0xe1, 0xe0, 0x00, 0x00, 0x00, 0x00, + 0x02, 0x77, 0xe0, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x03, 0x70, 0xfe, 0x60, + 0x00, 0x00, 0x00, 0x00, 0x04, 0x60, 0xfd, 0x70, 0x00, 0x00, 0x00, 0x00, + 0x05, 0x50, 0xe0, 0x60, 0x00, 0x00, 0x00, 0x00, 0x06, 0x40, 0xdf, 0x70, + 0x00, 0x00, 0x00, 0x00, 0x07, 0x30, 0xc2, 0x60, 0x00, 0x00, 0x00, 0x00, + 0x07, 0x8d, 0x19, 0x70, 0x00, 0x00, 0x00, 0x00, 0x09, 0x10, 0xa4, 0x60, + 0x00, 0x00, 0x00, 0x00, 0x09, 0xad, 0x94, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x0a, 0xf0, 0x86, 0x60, 0x00, 0x00, 0x00, 0x00, 0x0b, 0xe0, 0x85, 0x70, + 0x00, 0x00, 0x00, 0x00, 0x0c, 0xd9, 0xa2, 0xe0, 0x00, 0x00, 0x00, 0x00, + 0x0d, 0xc0, 0x67, 0x70, 0x00, 0x00, 0x00, 0x00, 0x0e, 0xb9, 0x84, 0xe0, + 0x00, 0x00, 0x00, 0x00, 0x0f, 0xa9, 0x83, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x10, 0x99, 0x66, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x11, 0x89, 0x65, 0xf0, + 0x00, 0x00, 0x00, 0x00, 0x12, 0x79, 0x48, 0xe0, 0x00, 0x00, 0x00, 0x00, + 0x13, 0x69, 0x47, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x14, 0x59, 0x2a, 0xe0, + 0x00, 0x00, 0x00, 0x00, 0x15, 0x49, 0x29, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x16, 0x39, 0x0c, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x17, 0x29, 0x0b, 0xf0, + 0x00, 0x00, 0x00, 0x00, 0x18, 0x22, 0x29, 0x60, 0x00, 0x00, 0x00, 0x00, + 0x19, 0x08, 0xed, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x1a, 0x02, 0x0b, 0x60, + 0x00, 0x00, 0x00, 0x00, 0x1a, 0xf2, 0x0a, 0x70, 0x00, 0x00, 0x00, 0x00, + 0x1b, 0xe1, 0xed, 0x60, 0x00, 0x00, 0x00, 0x00, 0x1c, 0xd1, 0xec, 0x70, + 0x00, 0x00, 0x00, 0x00, 0x1d, 0xc1, 0xcf, 0x60, 0x00, 0x00, 0x00, 0x00, + 0x1e, 0xb1, 0xce, 0x70, 0x00, 0x00, 0x00, 0x00, 0x1f, 0xa1, 0xb1, 0x60, + 0x00, 0x00, 0x00, 0x00, 0x20, 0x76, 0x00, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x21, 0x81, 0x93, 0x60, 0x00, 0x00, 0x00, 0x00, 0x22, 0x55, 0xe2, 0xf0, + 0x00, 0x00, 0x00, 0x00, 0x23, 0x6a, 0xaf, 0xe0, 0x00, 0x00, 0x00, 0x00, + 0x24, 0x35, 0xc4, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x25, 0x4a, 0x91, 0xe0, + 0x00, 0x00, 0x00, 0x00, 0x26, 0x15, 0xa6, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x27, 0x2a, 0x73, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x27, 0xfe, 0xc3, 0x70, + 0x00, 0x00, 0x00, 0x00, 0x29, 0x0a, 0x55, 0xe0, 0x00, 0x00, 0x00, 0x00, + 0x29, 0xde, 0xa5, 0x70, 0x00, 0x00, 0x00, 0x00, 0x2a, 0xea, 0x37, 0xe0, + 0x00, 0x00, 0x00, 0x00, 0x2b, 0xbe, 0x87, 0x70, 0x00, 0x00, 0x00, 0x00, + 0x2c, 0xd3, 0x54, 0x60, 0x00, 0x00, 0x00, 0x00, 0x2d, 0x9e, 0x69, 0x70, + 0x00, 0x00, 0x00, 0x00, 0x2e, 0xb3, 0x36, 0x60, 0x00, 0x00, 0x00, 0x00, + 0x2f, 0x7e, 0x4b, 0x70, 0x00, 0x00, 0x00, 0x00, 0x30, 0x93, 0x18, 0x60, + 0x00, 0x00, 0x00, 0x00, 0x31, 0x67, 0x67, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x32, 0x72, 0xfa, 0x60, 0x00, 0x00, 0x00, 0x00, 0x33, 0x47, 0x49, 0xf0, + 0x00, 0x00, 0x00, 0x00, 0x34, 0x52, 0xdc, 0x60, 0x00, 0x00, 0x00, 0x00, + 0x35, 0x27, 0x2b, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x36, 0x32, 0xbe, 0x60, + 0x00, 0x00, 0x00, 0x00, 0x37, 0x07, 0x0d, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x38, 0x1b, 0xda, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x38, 0xe6, 0xef, 0xf0, + 0x00, 0x00, 0x00, 0x00, 0x39, 0xfb, 0xbc, 0xe0, 0x00, 0x00, 0x00, 0x00, + 0x3a, 0xc6, 0xd1, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x3b, 0xdb, 0x9e, 0xe0, + 0x00, 0x00, 0x00, 0x00, 0x3c, 0xaf, 0xee, 0x70, 0x00, 0x00, 0x00, 0x00, + 0x3d, 0xbb, 0x80, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x3e, 0x8f, 0xd0, 0x70, + 0x00, 0x00, 0x00, 0x00, 0x3f, 0x9b, 0x62, 0xe0, 0x00, 0x00, 0x00, 0x00, + 0x40, 0x6f, 0xb2, 0x70, 0x00, 0x00, 0x00, 0x00, 0x41, 0x84, 0x7f, 0x60, + 0x00, 0x00, 0x00, 0x00, 0x42, 0x4f, 0x94, 0x70, 0x00, 0x00, 0x00, 0x00, + 0x43, 0x64, 0x61, 0x60, 0x00, 0x00, 0x00, 0x00, 0x44, 0x2f, 0x76, 0x70, + 0x00, 0x00, 0x00, 0x00, 0x45, 0x44, 0x43, 0x60, 0x00, 0x00, 0x00, 0x00, + 0x45, 0xf3, 0xa8, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x47, 0x2d, 0x5f, 0xe0, + 0x00, 0x00, 0x00, 0x00, 0x47, 0xd3, 0x8a, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x49, 0x0d, 0x41, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x49, 0xb3, 0x6c, 0xf0, + 0x00, 0x00, 0x00, 0x00, 0x4a, 0xed, 0x23, 0xe0, 0x00, 0x00, 0x00, 0x00, + 0x4b, 0x9c, 0x89, 0x70, 0x00, 0x00, 0x00, 0x00, 0x4c, 0xd6, 0x40, 0x60, + 0x00, 0x00, 0x00, 0x00, 0x4d, 0x7c, 0x6b, 0x70, 0x00, 0x00, 0x00, 0x00, + 0x4e, 0xb6, 0x22, 0x60, 0x00, 0x00, 0x00, 0x00, 0x4f, 0x5c, 0x4d, 0x70, + 0x00, 0x00, 0x00, 0x00, 0x50, 0x96, 0x04, 0x60, 0x00, 0x00, 0x00, 0x00, + 0x51, 0x3c, 0x2f, 0x70, 0x00, 0x00, 0x00, 0x00, 0x52, 0x75, 0xe6, 0x60, + 0x00, 0x00, 0x00, 0x00, 0x53, 0x1c, 0x11, 0x70, 0x00, 0x00, 0x00, 0x00, + 0x54, 0x55, 0xc8, 0x60, 0x00, 0x00, 0x00, 0x00, 0x54, 0xfb, 0xf3, 0x70, + 0x00, 0x00, 0x00, 0x00, 0x56, 0x35, 0xaa, 0x60, 0x00, 0x00, 0x00, 0x00, + 0x56, 0xe5, 0x0f, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x58, 0x1e, 0xc6, 0xe0, + 0x00, 0x00, 0x00, 0x00, 0x58, 0xc4, 0xf1, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x59, 0xfe, 0xa8, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x5a, 0xa4, 0xd3, 0xf0, + 0x00, 0x00, 0x00, 0x00, 0x5b, 0xde, 0x8a, 0xe0, 0x00, 0x00, 0x00, 0x00, + 0x5c, 0x84, 0xb5, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x5d, 0xbe, 0x6c, 0xe0, + 0x00, 0x00, 0x00, 0x00, 0x5e, 0x64, 0x97, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x5f, 0x9e, 0x4e, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x60, 0x4d, 0xb4, 0x70, + 0x00, 0x00, 0x00, 0x00, 0x61, 0x87, 0x6b, 0x60, 0x00, 0x00, 0x00, 0x00, + 0x62, 0x2d, 0x96, 0x70, 0x00, 0x00, 0x00, 0x00, 0x63, 0x67, 0x4d, 0x60, + 0x00, 0x00, 0x00, 0x00, 0x64, 0x0d, 0x78, 0x70, 0x00, 0x00, 0x00, 0x00, + 0x65, 0x47, 0x2f, 0x60, 0x00, 0x00, 0x00, 0x00, 0x65, 0xed, 0x5a, 0x70, + 0x00, 0x00, 0x00, 0x00, 0x67, 0x27, 0x11, 0x60, 0x00, 0x00, 0x00, 0x00, + 0x67, 0xcd, 0x3c, 0x70, 0x00, 0x00, 0x00, 0x00, 0x69, 0x06, 0xf3, 0x60, + 0x00, 0x00, 0x00, 0x00, 0x69, 0xad, 0x1e, 0x70, 0x00, 0x00, 0x00, 0x00, + 0x6a, 0xe6, 0xd5, 0x60, 0x00, 0x00, 0x00, 0x00, 0x6b, 0x96, 0x3a, 0xf0, + 0x00, 0x00, 0x00, 0x00, 0x6c, 0xcf, 0xf1, 0xe0, 0x00, 0x00, 0x00, 0x00, + 0x6d, 0x76, 0x1c, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x6e, 0xaf, 0xd3, 0xe0, + 0x00, 0x00, 0x00, 0x00, 0x6f, 0x55, 0xfe, 0xf0, 0x00, 0x00, 0x00, 0x00, + 0x70, 0x8f, 0xb5, 0xe0, 0x00, 0x00, 0x00, 0x00, 0x71, 0x35, 0xe0, 0xf0, + 0x00, 0x00, 0x00, 0x00, 0x72, 0x6f, 0x97, 0xe0, 0x00, 0x00, 0x00, 0x00, + 0x73, 0x15, 0xc2, 0xf0, 0x00, 0x00, 0x00, 0x00, 0x74, 0x4f, 0x79, 0xe0, + 0x00, 0x00, 0x00, 0x00, 0x74, 0xfe, 0xdf, 0x70, 0x00, 0x00, 0x00, 0x00, + 0x76, 0x38, 0x96, 0x60, 0x00, 0x00, 0x00, 0x00, 0x76, 0xde, 0xc1, 0x70, + 0x00, 0x00, 0x00, 0x00, 0x78, 0x18, 0x78, 0x60, 0x00, 0x00, 0x00, 0x00, + 0x78, 0xbe, 0xa3, 0x70, 0x00, 0x00, 0x00, 0x00, 0x79, 0xf8, 0x5a, 0x60, + 0x00, 0x00, 0x00, 0x00, 0x7a, 0x9e, 0x85, 0x70, 0x00, 0x00, 0x00, 0x00, + 0x7b, 0xd8, 0x3c, 0x60, 0x00, 0x00, 0x00, 0x00, 0x7c, 0x7e, 0x67, 0x70, + 0x00, 0x00, 0x00, 0x00, 0x7d, 0xb8, 0x1e, 0x60, 0x00, 0x00, 0x00, 0x00, + 0x7e, 0x5e, 0x49, 0x70, 0x00, 0x00, 0x00, 0x00, 0x7f, 0x98, 0x00, 0x60, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, - 0x02, 0x01, 0x02, 0x01, 0x02, 0xff, 0xff, 0xba, 0x9e, 0x00, 0x00, 0xff, - 0xff, 0xc7, 0xc0, 0x01, 0x04, 0xff, 0xff, 0xb9, 0xb0, 0x00, 0x08, 0xff, - 0xff, 0xc7, 0xc0, 0x01, 0x0c, 0xff, 0xff, 0xc7, 0xc0, 0x01, 0x10, 0x4c, - 0x4d, 0x54, 0x00, 0x45, 0x44, 0x54, 0x00, 0x45, 0x53, 0x54, 0x00, 0x45, - 0x57, 0x54, 0x00, 0x45, 0x50, 0x54, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, - 0x00, 0x00, 0x00, 0x00, 0x01, 0x0a, 0x45, 0x53, 0x54, 0x35, 0x45, 0x44, - 0x54, 0x2c, 0x4d, 0x33, 0x2e, 0x32, 0x2e, 0x30, 0x2c, 0x4d, 0x31, 0x31, - 0x2e, 0x31, 0x2e, 0x30, 0x0a + 0x02, 0x03, 0x04, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, + 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0xff, 0xff, 0xba, 0x9e, + 0x00, 0x00, 0xff, 0xff, 0xc7, 0xc0, 0x01, 0x04, 0xff, 0xff, 0xb9, 0xb0, + 0x00, 0x08, 0xff, 0xff, 0xc7, 0xc0, 0x01, 0x0c, 0xff, 0xff, 0xc7, 0xc0, + 0x01, 0x10, 0x4c, 0x4d, 0x54, 0x00, 0x45, 0x44, 0x54, 0x00, 0x45, 0x53, + 0x54, 0x00, 0x45, 0x57, 0x54, 0x00, 0x45, 0x50, 0x54, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x01, 0x00, 0x00, 0x00, 0x00, 0x01, 0x0a, 0x45, 0x53, 0x54, + 0x35, 0x45, 0x44, 0x54, 0x2c, 0x4d, 0x33, 0x2e, 0x32, 0x2e, 0x30, 0x2c, + 0x4d, 0x31, 0x31, 0x2e, 0x31, 0x2e, 0x30, 0x0a }; -unsigned int America_New_York_len = 3545; +unsigned int America_New_York_len = 3536; unsigned char Australia_Sydney[] = { 0x54, 0x5a, 0x69, 0x66, 0x32, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, - 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8e, - 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x0e, 0x80, 0x00, 0x00, 0x00, - 0x9c, 0x4e, 0xa6, 0x9c, 0x9c, 0xbc, 0x20, 0xf0, 0xcb, 0x54, 0xb3, 0x00, - 0xcb, 0xc7, 0x57, 0x70, 0xcc, 0xb7, 0x56, 0x80, 0xcd, 0xa7, 0x39, 0x70, - 0xce, 0xa0, 0x73, 0x00, 0xcf, 0x87, 0x1b, 0x70, 0x03, 0x70, 0x39, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x8e, + 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x0e, 0x80, 0x00, 0x00, 0x00, + 0x9c, 0x4e, 0xc2, 0x80, 0x9c, 0xbc, 0x2f, 0x00, 0xcb, 0x54, 0xb3, 0x00, + 0xcb, 0xc7, 0x65, 0x80, 0xcc, 0xb7, 0x56, 0x80, 0xcd, 0xa7, 0x47, 0x80, + 0xce, 0xa0, 0x73, 0x00, 0xcf, 0x87, 0x29, 0x80, 0x03, 0x70, 0x39, 0x80, 0x04, 0x0d, 0x1c, 0x00, 0x05, 0x50, 0x1b, 0x80, 0x05, 0xf6, 0x38, 0x80, 0x07, 0x2f, 0xfd, 0x80, 0x07, 0xd6, 0x1a, 0x80, 0x09, 0x0f, 0xdf, 0x80, 0x09, 0xb5, 0xfc, 0x80, 0x0a, 0xef, 0xc1, 0x80, 0x0b, 0x9f, 0x19, 0x00, @@ -590,140 +588,137 @@ unsigned char Australia_Sydney[] = { 0x77, 0xe9, 0x8f, 0x00, 0x78, 0xd9, 0x80, 0x00, 0x79, 0xc9, 0x71, 0x00, 0x7a, 0xb9, 0x62, 0x00, 0x7b, 0xb2, 0x8d, 0x80, 0x7c, 0xa2, 0x7e, 0x80, 0x7d, 0x92, 0x6f, 0x80, 0x7e, 0x82, 0x60, 0x80, 0x7f, 0x72, 0x51, 0x80, - 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x03, 0x04, 0x03, - 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, - 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, - 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, - 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, - 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, - 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, - 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, - 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, - 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, - 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, - 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x00, 0x00, + 0x03, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x00, 0x00, 0x8d, 0xc4, 0x00, 0x00, 0x00, 0x00, 0x9a, 0xb0, 0x01, 0x04, 0x00, 0x00, - 0x8c, 0xa0, 0x00, 0x09, 0x00, 0x00, 0x9a, 0xb0, 0x01, 0x04, 0x00, 0x00, - 0x8c, 0xa0, 0x00, 0x09, 0x4c, 0x4d, 0x54, 0x00, 0x41, 0x45, 0x44, 0x54, - 0x00, 0x41, 0x45, 0x53, 0x54, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x54, 0x5a, 0x69, 0x66, 0x32, 0x00, 0x00, 0x00, + 0x8c, 0xa0, 0x00, 0x09, 0x00, 0x00, 0x8c, 0xa0, 0x00, 0x09, 0x4c, 0x4d, + 0x54, 0x00, 0x41, 0x45, 0x44, 0x54, 0x00, 0x41, 0x45, 0x53, 0x54, 0x00, + 0x00, 0x01, 0x01, 0x00, 0x54, 0x5a, 0x69, 0x66, 0x32, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x8f, 0x00, 0x00, 0x00, 0x05, 0x00, 0x00, 0x00, 0x0e, - 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0xff, 0xff, 0xff, 0xff, - 0x73, 0x16, 0x7f, 0x3c, 0xff, 0xff, 0xff, 0xff, 0x9c, 0x4e, 0xa6, 0x9c, - 0xff, 0xff, 0xff, 0xff, 0x9c, 0xbc, 0x20, 0xf0, 0xff, 0xff, 0xff, 0xff, - 0xcb, 0x54, 0xb3, 0x00, 0xff, 0xff, 0xff, 0xff, 0xcb, 0xc7, 0x57, 0x70, - 0xff, 0xff, 0xff, 0xff, 0xcc, 0xb7, 0x56, 0x80, 0xff, 0xff, 0xff, 0xff, - 0xcd, 0xa7, 0x39, 0x70, 0xff, 0xff, 0xff, 0xff, 0xce, 0xa0, 0x73, 0x00, - 0xff, 0xff, 0xff, 0xff, 0xcf, 0x87, 0x1b, 0x70, 0x00, 0x00, 0x00, 0x00, - 0x03, 0x70, 0x39, 0x80, 0x00, 0x00, 0x00, 0x00, 0x04, 0x0d, 0x1c, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x05, 0x50, 0x1b, 0x80, 0x00, 0x00, 0x00, 0x00, - 0x05, 0xf6, 0x38, 0x80, 0x00, 0x00, 0x00, 0x00, 0x07, 0x2f, 0xfd, 0x80, - 0x00, 0x00, 0x00, 0x00, 0x07, 0xd6, 0x1a, 0x80, 0x00, 0x00, 0x00, 0x00, - 0x09, 0x0f, 0xdf, 0x80, 0x00, 0x00, 0x00, 0x00, 0x09, 0xb5, 0xfc, 0x80, - 0x00, 0x00, 0x00, 0x00, 0x0a, 0xef, 0xc1, 0x80, 0x00, 0x00, 0x00, 0x00, - 0x0b, 0x9f, 0x19, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0c, 0xd8, 0xde, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x0d, 0x7e, 0xfb, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x0e, 0xb8, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0f, 0x5e, 0xdd, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x10, 0x98, 0xa2, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x11, 0x3e, 0xbf, 0x00, 0x00, 0x00, 0x00, 0x00, 0x12, 0x78, 0x84, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x13, 0x1e, 0xa1, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x14, 0x58, 0x66, 0x00, 0x00, 0x00, 0x00, 0x00, 0x14, 0xfe, 0x83, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x16, 0x38, 0x48, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x17, 0x0c, 0x89, 0x80, 0x00, 0x00, 0x00, 0x00, 0x18, 0x21, 0x64, 0x80, - 0x00, 0x00, 0x00, 0x00, 0x18, 0xc7, 0x81, 0x80, 0x00, 0x00, 0x00, 0x00, - 0x1a, 0x01, 0x46, 0x80, 0x00, 0x00, 0x00, 0x00, 0x1a, 0xa7, 0x63, 0x80, - 0x00, 0x00, 0x00, 0x00, 0x1b, 0xe1, 0x28, 0x80, 0x00, 0x00, 0x00, 0x00, - 0x1c, 0x87, 0x45, 0x80, 0x00, 0x00, 0x00, 0x00, 0x1d, 0xc1, 0x0a, 0x80, - 0x00, 0x00, 0x00, 0x00, 0x1e, 0x79, 0x9c, 0x80, 0x00, 0x00, 0x00, 0x00, - 0x1f, 0x97, 0xb2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x20, 0x59, 0x7e, 0x80, - 0x00, 0x00, 0x00, 0x00, 0x21, 0x80, 0xce, 0x80, 0x00, 0x00, 0x00, 0x00, - 0x22, 0x42, 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, 0x23, 0x69, 0xeb, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x24, 0x22, 0x7d, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x25, 0x49, 0xcd, 0x00, 0x00, 0x00, 0x00, 0x00, 0x25, 0xef, 0xea, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x27, 0x29, 0xaf, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x27, 0xcf, 0xcc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x29, 0x09, 0x91, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x29, 0xaf, 0xae, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x2a, 0xe9, 0x73, 0x00, 0x00, 0x00, 0x00, 0x00, 0x2b, 0x98, 0xca, 0x80, - 0x00, 0x00, 0x00, 0x00, 0x2c, 0xd2, 0x8f, 0x80, 0x00, 0x00, 0x00, 0x00, - 0x2d, 0x78, 0xac, 0x80, 0x00, 0x00, 0x00, 0x00, 0x2e, 0xb2, 0x71, 0x80, - 0x00, 0x00, 0x00, 0x00, 0x2f, 0x58, 0x8e, 0x80, 0x00, 0x00, 0x00, 0x00, - 0x30, 0x92, 0x53, 0x80, 0x00, 0x00, 0x00, 0x00, 0x31, 0x5d, 0x5a, 0x80, - 0x00, 0x00, 0x00, 0x00, 0x32, 0x72, 0x35, 0x80, 0x00, 0x00, 0x00, 0x00, - 0x33, 0x3d, 0x3c, 0x80, 0x00, 0x00, 0x00, 0x00, 0x34, 0x52, 0x17, 0x80, - 0x00, 0x00, 0x00, 0x00, 0x35, 0x1d, 0x1e, 0x80, 0x00, 0x00, 0x00, 0x00, - 0x36, 0x31, 0xf9, 0x80, 0x00, 0x00, 0x00, 0x00, 0x36, 0xfd, 0x00, 0x80, - 0x00, 0x00, 0x00, 0x00, 0x38, 0x1b, 0x16, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x38, 0xdc, 0xe2, 0x80, 0x00, 0x00, 0x00, 0x00, 0x39, 0xa7, 0xe9, 0x80, - 0x00, 0x00, 0x00, 0x00, 0x3a, 0xbc, 0xc4, 0x80, 0x00, 0x00, 0x00, 0x00, - 0x3b, 0xda, 0xda, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3c, 0xa5, 0xe1, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x3d, 0xba, 0xbc, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x3e, 0x85, 0xc3, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3f, 0x9a, 0x9e, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x40, 0x65, 0xa5, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x41, 0x83, 0xba, 0x80, 0x00, 0x00, 0x00, 0x00, 0x42, 0x45, 0x87, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x43, 0x63, 0x9c, 0x80, 0x00, 0x00, 0x00, 0x00, - 0x44, 0x2e, 0xa3, 0x80, 0x00, 0x00, 0x00, 0x00, 0x45, 0x43, 0x7e, 0x80, - 0x00, 0x00, 0x00, 0x00, 0x46, 0x05, 0x4b, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x47, 0x23, 0x60, 0x80, 0x00, 0x00, 0x00, 0x00, 0x47, 0xf7, 0xa2, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x48, 0xe7, 0x93, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x49, 0xd7, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4a, 0xc7, 0x75, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x4b, 0xb7, 0x66, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x4c, 0xa7, 0x57, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4d, 0x97, 0x48, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x4e, 0x87, 0x39, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x4f, 0x77, 0x2a, 0x00, 0x00, 0x00, 0x00, 0x00, 0x50, 0x70, 0x55, 0x80, - 0x00, 0x00, 0x00, 0x00, 0x51, 0x60, 0x46, 0x80, 0x00, 0x00, 0x00, 0x00, - 0x52, 0x50, 0x37, 0x80, 0x00, 0x00, 0x00, 0x00, 0x53, 0x40, 0x28, 0x80, - 0x00, 0x00, 0x00, 0x00, 0x54, 0x30, 0x19, 0x80, 0x00, 0x00, 0x00, 0x00, - 0x55, 0x20, 0x0a, 0x80, 0x00, 0x00, 0x00, 0x00, 0x56, 0x0f, 0xfb, 0x80, - 0x00, 0x00, 0x00, 0x00, 0x56, 0xff, 0xec, 0x80, 0x00, 0x00, 0x00, 0x00, - 0x57, 0xef, 0xdd, 0x80, 0x00, 0x00, 0x00, 0x00, 0x58, 0xdf, 0xce, 0x80, - 0x00, 0x00, 0x00, 0x00, 0x59, 0xcf, 0xbf, 0x80, 0x00, 0x00, 0x00, 0x00, - 0x5a, 0xbf, 0xb0, 0x80, 0x00, 0x00, 0x00, 0x00, 0x5b, 0xb8, 0xdc, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x5c, 0xa8, 0xcd, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x5d, 0x98, 0xbe, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5e, 0x88, 0xaf, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x5f, 0x78, 0xa0, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x60, 0x68, 0x91, 0x00, 0x00, 0x00, 0x00, 0x00, 0x61, 0x58, 0x82, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x62, 0x48, 0x73, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x63, 0x38, 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, 0x64, 0x28, 0x55, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x65, 0x18, 0x46, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x66, 0x11, 0x71, 0x80, 0x00, 0x00, 0x00, 0x00, 0x67, 0x01, 0x62, 0x80, - 0x00, 0x00, 0x00, 0x00, 0x67, 0xf1, 0x53, 0x80, 0x00, 0x00, 0x00, 0x00, - 0x68, 0xe1, 0x44, 0x80, 0x00, 0x00, 0x00, 0x00, 0x69, 0xd1, 0x35, 0x80, - 0x00, 0x00, 0x00, 0x00, 0x6a, 0xc1, 0x26, 0x80, 0x00, 0x00, 0x00, 0x00, - 0x6b, 0xb1, 0x17, 0x80, 0x00, 0x00, 0x00, 0x00, 0x6c, 0xa1, 0x08, 0x80, - 0x00, 0x00, 0x00, 0x00, 0x6d, 0x90, 0xf9, 0x80, 0x00, 0x00, 0x00, 0x00, - 0x6e, 0x80, 0xea, 0x80, 0x00, 0x00, 0x00, 0x00, 0x6f, 0x70, 0xdb, 0x80, - 0x00, 0x00, 0x00, 0x00, 0x70, 0x6a, 0x07, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x71, 0x59, 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, 0x72, 0x49, 0xe9, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x73, 0x39, 0xda, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x74, 0x29, 0xcb, 0x00, 0x00, 0x00, 0x00, 0x00, 0x75, 0x19, 0xbc, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x76, 0x09, 0xad, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x76, 0xf9, 0x9e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x77, 0xe9, 0x8f, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x78, 0xd9, 0x80, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x79, 0xc9, 0x71, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7a, 0xb9, 0x62, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x7b, 0xb2, 0x8d, 0x80, 0x00, 0x00, 0x00, 0x00, - 0x7c, 0xa2, 0x7e, 0x80, 0x00, 0x00, 0x00, 0x00, 0x7d, 0x92, 0x6f, 0x80, - 0x00, 0x00, 0x00, 0x00, 0x7e, 0x82, 0x60, 0x80, 0x00, 0x00, 0x00, 0x00, - 0x7f, 0x72, 0x51, 0x80, 0x00, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, - 0x01, 0x02, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, - 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, - 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, - 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, - 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, - 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, - 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, - 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, - 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, - 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, - 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, 0x03, 0x04, - 0x03, 0x04, 0x03, 0x00, 0x00, 0x8d, 0xc4, 0x00, 0x00, 0x00, 0x00, 0x9a, - 0xb0, 0x01, 0x04, 0x00, 0x00, 0x8c, 0xa0, 0x00, 0x09, 0x00, 0x00, 0x9a, - 0xb0, 0x01, 0x04, 0x00, 0x00, 0x8c, 0xa0, 0x00, 0x09, 0x4c, 0x4d, 0x54, - 0x00, 0x41, 0x45, 0x44, 0x54, 0x00, 0x41, 0x45, 0x53, 0x54, 0x00, 0x00, - 0x00, 0x00, 0x01, 0x01, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0a, 0x41, 0x45, - 0x53, 0x54, 0x2d, 0x31, 0x30, 0x41, 0x45, 0x44, 0x54, 0x2c, 0x4d, 0x31, - 0x30, 0x2e, 0x31, 0x2e, 0x30, 0x2c, 0x4d, 0x34, 0x2e, 0x31, 0x2e, 0x30, - 0x2f, 0x33, 0x0a + 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x00, + 0x00, 0x00, 0x00, 0x8e, 0x00, 0x00, 0x00, 0x04, 0x00, 0x00, 0x00, 0x0e, + 0xff, 0xff, 0xff, 0xff, 0x73, 0x16, 0x7f, 0x3c, 0xff, 0xff, 0xff, 0xff, + 0x9c, 0x4e, 0xc2, 0x80, 0xff, 0xff, 0xff, 0xff, 0x9c, 0xbc, 0x2f, 0x00, + 0xff, 0xff, 0xff, 0xff, 0xcb, 0x54, 0xb3, 0x00, 0xff, 0xff, 0xff, 0xff, + 0xcb, 0xc7, 0x65, 0x80, 0xff, 0xff, 0xff, 0xff, 0xcc, 0xb7, 0x56, 0x80, + 0xff, 0xff, 0xff, 0xff, 0xcd, 0xa7, 0x47, 0x80, 0xff, 0xff, 0xff, 0xff, + 0xce, 0xa0, 0x73, 0x00, 0xff, 0xff, 0xff, 0xff, 0xcf, 0x87, 0x29, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x03, 0x70, 0x39, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x04, 0x0d, 0x1c, 0x00, 0x00, 0x00, 0x00, 0x00, 0x05, 0x50, 0x1b, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x05, 0xf6, 0x38, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x07, 0x2f, 0xfd, 0x80, 0x00, 0x00, 0x00, 0x00, 0x07, 0xd6, 0x1a, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x09, 0x0f, 0xdf, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x09, 0xb5, 0xfc, 0x80, 0x00, 0x00, 0x00, 0x00, 0x0a, 0xef, 0xc1, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x0b, 0x9f, 0x19, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x0c, 0xd8, 0xde, 0x00, 0x00, 0x00, 0x00, 0x00, 0x0d, 0x7e, 0xfb, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x0e, 0xb8, 0xc0, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x0f, 0x5e, 0xdd, 0x00, 0x00, 0x00, 0x00, 0x00, 0x10, 0x98, 0xa2, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x11, 0x3e, 0xbf, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x12, 0x78, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, 0x13, 0x1e, 0xa1, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x14, 0x58, 0x66, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x14, 0xfe, 0x83, 0x00, 0x00, 0x00, 0x00, 0x00, 0x16, 0x38, 0x48, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x17, 0x0c, 0x89, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x18, 0x21, 0x64, 0x80, 0x00, 0x00, 0x00, 0x00, 0x18, 0xc7, 0x81, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x1a, 0x01, 0x46, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x1a, 0xa7, 0x63, 0x80, 0x00, 0x00, 0x00, 0x00, 0x1b, 0xe1, 0x28, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x1c, 0x87, 0x45, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x1d, 0xc1, 0x0a, 0x80, 0x00, 0x00, 0x00, 0x00, 0x1e, 0x79, 0x9c, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x1f, 0x97, 0xb2, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x20, 0x59, 0x7e, 0x80, 0x00, 0x00, 0x00, 0x00, 0x21, 0x80, 0xce, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x22, 0x42, 0x9b, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x23, 0x69, 0xeb, 0x00, 0x00, 0x00, 0x00, 0x00, 0x24, 0x22, 0x7d, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x25, 0x49, 0xcd, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x25, 0xef, 0xea, 0x00, 0x00, 0x00, 0x00, 0x00, 0x27, 0x29, 0xaf, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x27, 0xcf, 0xcc, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x29, 0x09, 0x91, 0x00, 0x00, 0x00, 0x00, 0x00, 0x29, 0xaf, 0xae, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x2a, 0xe9, 0x73, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x2b, 0x98, 0xca, 0x80, 0x00, 0x00, 0x00, 0x00, 0x2c, 0xd2, 0x8f, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x2d, 0x78, 0xac, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x2e, 0xb2, 0x71, 0x80, 0x00, 0x00, 0x00, 0x00, 0x2f, 0x58, 0x8e, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x30, 0x92, 0x53, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x31, 0x5d, 0x5a, 0x80, 0x00, 0x00, 0x00, 0x00, 0x32, 0x72, 0x35, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x33, 0x3d, 0x3c, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x34, 0x52, 0x17, 0x80, 0x00, 0x00, 0x00, 0x00, 0x35, 0x1d, 0x1e, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x36, 0x31, 0xf9, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x36, 0xfd, 0x00, 0x80, 0x00, 0x00, 0x00, 0x00, 0x38, 0x1b, 0x16, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x38, 0xdc, 0xe2, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x39, 0xa7, 0xe9, 0x80, 0x00, 0x00, 0x00, 0x00, 0x3a, 0xbc, 0xc4, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x3b, 0xda, 0xda, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x3c, 0xa5, 0xe1, 0x00, 0x00, 0x00, 0x00, 0x00, 0x3d, 0xba, 0xbc, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x3e, 0x85, 0xc3, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x3f, 0x9a, 0x9e, 0x00, 0x00, 0x00, 0x00, 0x00, 0x40, 0x65, 0xa5, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x41, 0x83, 0xba, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x42, 0x45, 0x87, 0x00, 0x00, 0x00, 0x00, 0x00, 0x43, 0x63, 0x9c, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x44, 0x2e, 0xa3, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x45, 0x43, 0x7e, 0x80, 0x00, 0x00, 0x00, 0x00, 0x46, 0x05, 0x4b, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x47, 0x23, 0x60, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x47, 0xf7, 0xa2, 0x00, 0x00, 0x00, 0x00, 0x00, 0x48, 0xe7, 0x93, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x49, 0xd7, 0x84, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x4a, 0xc7, 0x75, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4b, 0xb7, 0x66, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x4c, 0xa7, 0x57, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x4d, 0x97, 0x48, 0x00, 0x00, 0x00, 0x00, 0x00, 0x4e, 0x87, 0x39, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x4f, 0x77, 0x2a, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x50, 0x70, 0x55, 0x80, 0x00, 0x00, 0x00, 0x00, 0x51, 0x60, 0x46, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x52, 0x50, 0x37, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x53, 0x40, 0x28, 0x80, 0x00, 0x00, 0x00, 0x00, 0x54, 0x30, 0x19, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x55, 0x20, 0x0a, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x56, 0x0f, 0xfb, 0x80, 0x00, 0x00, 0x00, 0x00, 0x56, 0xff, 0xec, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x57, 0xef, 0xdd, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x58, 0xdf, 0xce, 0x80, 0x00, 0x00, 0x00, 0x00, 0x59, 0xcf, 0xbf, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x5a, 0xbf, 0xb0, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x5b, 0xb8, 0xdc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5c, 0xa8, 0xcd, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x5d, 0x98, 0xbe, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x5e, 0x88, 0xaf, 0x00, 0x00, 0x00, 0x00, 0x00, 0x5f, 0x78, 0xa0, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x60, 0x68, 0x91, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x61, 0x58, 0x82, 0x00, 0x00, 0x00, 0x00, 0x00, 0x62, 0x48, 0x73, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x63, 0x38, 0x64, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x64, 0x28, 0x55, 0x00, 0x00, 0x00, 0x00, 0x00, 0x65, 0x18, 0x46, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x66, 0x11, 0x71, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x67, 0x01, 0x62, 0x80, 0x00, 0x00, 0x00, 0x00, 0x67, 0xf1, 0x53, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x68, 0xe1, 0x44, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x69, 0xd1, 0x35, 0x80, 0x00, 0x00, 0x00, 0x00, 0x6a, 0xc1, 0x26, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x6b, 0xb1, 0x17, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x6c, 0xa1, 0x08, 0x80, 0x00, 0x00, 0x00, 0x00, 0x6d, 0x90, 0xf9, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x6e, 0x80, 0xea, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x6f, 0x70, 0xdb, 0x80, 0x00, 0x00, 0x00, 0x00, 0x70, 0x6a, 0x07, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x71, 0x59, 0xf8, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x72, 0x49, 0xe9, 0x00, 0x00, 0x00, 0x00, 0x00, 0x73, 0x39, 0xda, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x74, 0x29, 0xcb, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x75, 0x19, 0xbc, 0x00, 0x00, 0x00, 0x00, 0x00, 0x76, 0x09, 0xad, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x76, 0xf9, 0x9e, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x77, 0xe9, 0x8f, 0x00, 0x00, 0x00, 0x00, 0x00, 0x78, 0xd9, 0x80, 0x00, + 0x00, 0x00, 0x00, 0x00, 0x79, 0xc9, 0x71, 0x00, 0x00, 0x00, 0x00, 0x00, + 0x7a, 0xb9, 0x62, 0x00, 0x00, 0x00, 0x00, 0x00, 0x7b, 0xb2, 0x8d, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x7c, 0xa2, 0x7e, 0x80, 0x00, 0x00, 0x00, 0x00, + 0x7d, 0x92, 0x6f, 0x80, 0x00, 0x00, 0x00, 0x00, 0x7e, 0x82, 0x60, 0x80, + 0x00, 0x00, 0x00, 0x00, 0x7f, 0x72, 0x51, 0x80, 0x03, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, + 0x02, 0x01, 0x02, 0x01, 0x02, 0x01, 0x00, 0x00, 0x8d, 0xc4, 0x00, 0x00, + 0x00, 0x00, 0x9a, 0xb0, 0x01, 0x04, 0x00, 0x00, 0x8c, 0xa0, 0x00, 0x09, + 0x00, 0x00, 0x8c, 0xa0, 0x00, 0x09, 0x4c, 0x4d, 0x54, 0x00, 0x41, 0x45, + 0x44, 0x54, 0x00, 0x41, 0x45, 0x53, 0x54, 0x00, 0x00, 0x01, 0x01, 0x00, + 0x0a, 0x41, 0x45, 0x53, 0x54, 0x2d, 0x31, 0x30, 0x41, 0x45, 0x44, 0x54, + 0x2c, 0x4d, 0x31, 0x30, 0x2e, 0x31, 0x2e, 0x30, 0x2c, 0x4d, 0x34, 0x2e, + 0x31, 0x2e, 0x30, 0x2f, 0x33, 0x0a }; -unsigned int Australia_Sydney_len = 2223; +unsigned int Australia_Sydney_len = 2190; diff --git a/absl/time/time.h b/absl/time/time.h index 2df68581..bd01867e 100644 --- a/absl/time/time.h +++ b/absl/time/time.h @@ -120,7 +120,7 @@ using EnableIfFloat = // Duration // -// The `absl::Duration` class represents a signed, fixed-length span of time. +// The `absl::Duration` class represents a signed, fixed-length amount of time. // A `Duration` is generated using a unit-specific factory function, or is // the result of subtracting one `absl::Time` from another. Durations behave // like unit-safe integers and they support all the natural integer-like @@ -162,7 +162,7 @@ class Duration { constexpr Duration() : rep_hi_(0), rep_lo_(0) {} // zero-length duration // Copyable. -#if !defined(__clang__) && defined(_MSC_VER) && _MSC_VER < 1910 +#if !defined(__clang__) && defined(_MSC_VER) && _MSC_VER < 1930 // Explicitly defining the constexpr copy constructor avoids an MSVC bug. constexpr Duration(const Duration& d) : rep_hi_(d.rep_hi_), rep_lo_(d.rep_lo_) {} @@ -182,18 +182,29 @@ class Duration { // Overloads that forward to either the int64_t or double overloads above. // Integer operands must be representable as int64_t. - template <typename T> + template <typename T, time_internal::EnableIfIntegral<T> = 0> Duration& operator*=(T r) { int64_t x = r; return *this *= x; } - template <typename T> + + template <typename T, time_internal::EnableIfIntegral<T> = 0> Duration& operator/=(T r) { int64_t x = r; return *this /= x; } - Duration& operator*=(float r) { return *this *= static_cast<double>(r); } - Duration& operator/=(float r) { return *this /= static_cast<double>(r); } + + template <typename T, time_internal::EnableIfFloat<T> = 0> + Duration& operator*=(T r) { + double x = r; + return *this *= x; + } + + template <typename T, time_internal::EnableIfFloat<T> = 0> + Duration& operator/=(T r) { + double x = r; + return *this /= x; + } template <typename H> friend H AbslHashValue(H h, Duration d) { @@ -392,12 +403,30 @@ constexpr Duration InfiniteDuration(); // // absl::Duration a = absl::Seconds(60); // absl::Duration b = absl::Minutes(1); // b == a -constexpr Duration Nanoseconds(int64_t n); -constexpr Duration Microseconds(int64_t n); -constexpr Duration Milliseconds(int64_t n); -constexpr Duration Seconds(int64_t n); -constexpr Duration Minutes(int64_t n); -constexpr Duration Hours(int64_t n); +template <typename T, time_internal::EnableIfIntegral<T> = 0> +constexpr Duration Nanoseconds(T n) { + return time_internal::FromInt64(n, std::nano{}); +} +template <typename T, time_internal::EnableIfIntegral<T> = 0> +constexpr Duration Microseconds(T n) { + return time_internal::FromInt64(n, std::micro{}); +} +template <typename T, time_internal::EnableIfIntegral<T> = 0> +constexpr Duration Milliseconds(T n) { + return time_internal::FromInt64(n, std::milli{}); +} +template <typename T, time_internal::EnableIfIntegral<T> = 0> +constexpr Duration Seconds(T n) { + return time_internal::FromInt64(n, std::ratio<1>{}); +} +template <typename T, time_internal::EnableIfIntegral<T> = 0> +constexpr Duration Minutes(T n) { + return time_internal::FromInt64(n, std::ratio<60>{}); +} +template <typename T, time_internal::EnableIfIntegral<T> = 0> +constexpr Duration Hours(T n) { + return time_internal::FromInt64(n, std::ratio<3600>{}); +} // Factory overloads for constructing `Duration` values from a floating-point // number of the unit indicated by the factory function's name. These functions @@ -451,8 +480,9 @@ Duration Hours(T n) { // ToInt64Hours() // // Helper functions that convert a Duration to an integral count of the -// indicated unit. These functions are shorthand for the `IDivDuration()` -// function above; see its documentation for details about overflow, etc. +// indicated unit. These return the same results as the `IDivDuration()` +// function, though they usually do so more efficiently; see the +// documentation of `IDivDuration()` for details about overflow, etc. // // Example: // @@ -547,10 +577,20 @@ inline std::ostream& operator<<(std::ostream& os, Duration d) { // `ZeroDuration()`. Parses "inf" and "-inf" as +/- `InfiniteDuration()`. bool ParseDuration(absl::string_view dur_string, Duration* d); -// Support for flag values of type Duration. Duration flags must be specified -// in a format that is valid input for absl::ParseDuration(). +// AbslParseFlag() +// +// Parses a command-line flag string representation `text` into a Duration +// value. Duration flags must be specified in a format that is valid input for +// `absl::ParseDuration()`. bool AbslParseFlag(absl::string_view text, Duration* dst, std::string* error); + + +// AbslUnparseFlag() +// +// Unparses a Duration value into a command-line string representation using +// the format specified by `absl::ParseDuration()`. std::string AbslUnparseFlag(Duration d); + ABSL_DEPRECATED("Use AbslParseFlag() instead.") bool ParseFlag(const std::string& text, Duration* dst, std::string* error); ABSL_DEPRECATED("Use AbslUnparseFlag() instead.") @@ -710,23 +750,24 @@ constexpr Time UnixEpoch() { return Time(); } constexpr Time UniversalEpoch() { // 719162 is the number of days from 0001-01-01 to 1970-01-01, // assuming the Gregorian calendar. - return Time(time_internal::MakeDuration(-24 * 719162 * int64_t{3600}, 0U)); + return Time( + time_internal::MakeDuration(-24 * 719162 * int64_t{3600}, uint32_t{0})); } // InfiniteFuture() // // Returns an `absl::Time` that is infinitely far in the future. constexpr Time InfiniteFuture() { - return Time( - time_internal::MakeDuration((std::numeric_limits<int64_t>::max)(), ~0U)); + return Time(time_internal::MakeDuration((std::numeric_limits<int64_t>::max)(), + ~uint32_t{0})); } // InfinitePast() // // Returns an `absl::Time` that is infinitely far in the past. constexpr Time InfinitePast() { - return Time( - time_internal::MakeDuration((std::numeric_limits<int64_t>::min)(), ~0U)); + return Time(time_internal::MakeDuration((std::numeric_limits<int64_t>::min)(), + ~uint32_t{0})); } // FromUnixNanos() @@ -813,8 +854,12 @@ Time FromChrono(const std::chrono::system_clock::time_point& tp); // // tp == std::chrono::system_clock::from_time_t(123); std::chrono::system_clock::time_point ToChronoTime(Time); -// Support for flag values of type Time. Time flags must be specified in a -// format that matches absl::RFC3339_full. For example: +// AbslParseFlag() +// +// Parses the command-line flag string representation `text` into a Time value. +// Time flags must be specified in a format that matches absl::RFC3339_full. +// +// For example: // // --start_time=2016-01-02T03:04:05.678+08:00 // @@ -824,7 +869,13 @@ std::chrono::system_clock::time_point ToChronoTime(Time); // seconds/milliseconds/etc from the Unix epoch, use an absl::Duration flag // and add that duration to absl::UnixEpoch() to get an absl::Time. bool AbslParseFlag(absl::string_view text, Time* t, std::string* error); + +// AbslUnparseFlag() +// +// Unparses a Time value into a command-line string representation using +// the format specified by `absl::ParseTime()`. std::string AbslUnparseFlag(Time t); + ABSL_DEPRECATED("Use AbslParseFlag() instead.") bool ParseFlag(const std::string& text, Time* t, std::string* error); ABSL_DEPRECATED("Use AbslUnparseFlag() instead.") @@ -1352,7 +1403,7 @@ constexpr Duration MakeDuration(int64_t hi, int64_t lo) { inline Duration MakePosDoubleDuration(double n) { const int64_t int_secs = static_cast<int64_t>(n); const uint32_t ticks = static_cast<uint32_t>( - (n - static_cast<double>(int_secs)) * kTicksPerSecond + 0.5); + std::round((n - static_cast<double>(int_secs)) * kTicksPerSecond)); return ticks < kTicksPerSecond ? MakeDuration(int_secs, ticks) : MakeDuration(int_secs + 1, ticks - kTicksPerSecond); @@ -1372,14 +1423,17 @@ constexpr int64_t GetRepHi(Duration d) { return d.rep_hi_; } constexpr uint32_t GetRepLo(Duration d) { return d.rep_lo_; } // Returns true iff d is positive or negative infinity. -constexpr bool IsInfiniteDuration(Duration d) { return GetRepLo(d) == ~0U; } +constexpr bool IsInfiniteDuration(Duration d) { + return GetRepLo(d) == ~uint32_t{0}; +} // Returns an infinite Duration with the opposite sign. // REQUIRES: IsInfiniteDuration(d) constexpr Duration OppositeInfinity(Duration d) { return GetRepHi(d) < 0 - ? MakeDuration((std::numeric_limits<int64_t>::max)(), ~0U) - : MakeDuration((std::numeric_limits<int64_t>::min)(), ~0U); + ? MakeDuration((std::numeric_limits<int64_t>::max)(), ~uint32_t{0}) + : MakeDuration((std::numeric_limits<int64_t>::min)(), + ~uint32_t{0}); } // Returns (-n)-1 (equivalently -(n+1)) without avoidable overflow. @@ -1476,25 +1530,6 @@ T ToChronoDuration(Duration d) { } // namespace time_internal -constexpr Duration Nanoseconds(int64_t n) { - return time_internal::FromInt64(n, std::nano{}); -} -constexpr Duration Microseconds(int64_t n) { - return time_internal::FromInt64(n, std::micro{}); -} -constexpr Duration Milliseconds(int64_t n) { - return time_internal::FromInt64(n, std::milli{}); -} -constexpr Duration Seconds(int64_t n) { - return time_internal::FromInt64(n, std::ratio<1>{}); -} -constexpr Duration Minutes(int64_t n) { - return time_internal::FromInt64(n, std::ratio<60>{}); -} -constexpr Duration Hours(int64_t n) { - return time_internal::FromInt64(n, std::ratio<3600>{}); -} - constexpr bool operator<(Duration lhs, Duration rhs) { return time_internal::GetRepHi(lhs) != time_internal::GetRepHi(rhs) ? time_internal::GetRepHi(lhs) < time_internal::GetRepHi(rhs) @@ -1537,7 +1572,7 @@ constexpr Duration operator-(Duration d) { constexpr Duration InfiniteDuration() { return time_internal::MakeDuration((std::numeric_limits<int64_t>::max)(), - ~0U); + ~uint32_t{0}); } constexpr Duration FromChrono(const std::chrono::nanoseconds& d) { diff --git a/absl/time/time_test.cc b/absl/time/time_test.cc index cde9423f..d235e9ad 100644 --- a/absl/time/time_test.cc +++ b/absl/time/time_test.cc @@ -377,6 +377,11 @@ TEST(Time, FloorConversion) { } TEST(Time, RoundtripConversion) { +#if defined(ABSL_SKIP_TIME_TESTS_BROKEN_ON_MSVC_OPT) && \ + ABSL_SKIP_TIME_TESTS_BROKEN_ON_MSVC_OPT + GTEST_SKIP(); +#endif + #define TEST_CONVERSION_ROUND_TRIP(SOURCE, FROM, TO, MATCHER) \ EXPECT_THAT(TO(FROM(SOURCE)), MATCHER(SOURCE)) @@ -558,6 +563,11 @@ TEST(Time, FromChrono) { } TEST(Time, ToChronoTime) { +#if defined(ABSL_SKIP_TIME_TESTS_BROKEN_ON_MSVC_OPT) && \ + ABSL_SKIP_TIME_TESTS_BROKEN_ON_MSVC_OPT + GTEST_SKIP(); +#endif + EXPECT_EQ(std::chrono::system_clock::from_time_t(-1), absl::ToChronoTime(absl::FromTimeT(-1))); EXPECT_EQ(std::chrono::system_clock::from_time_t(0), diff --git a/absl/types/BUILD.bazel b/absl/types/BUILD.bazel index 83be9360..bb801012 100644 --- a/absl/types/BUILD.bazel +++ b/absl/types/BUILD.bazel @@ -13,7 +13,6 @@ # See the License for the specific language governing permissions and # limitations under the License. -load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") load( "//absl:copts/configure_copts.bzl", "ABSL_DEFAULT_COPTS", @@ -315,6 +314,7 @@ cc_library( name = "compare", hdrs = ["compare.h"], copts = ABSL_DEFAULT_COPTS, + linkopts = ABSL_DEFAULT_LINKOPTS, deps = [ "//absl/base:core_headers", "//absl/meta:type_traits", diff --git a/absl/types/CMakeLists.txt b/absl/types/CMakeLists.txt index c356b211..830953ae 100644 --- a/absl/types/CMakeLists.txt +++ b/absl/types/CMakeLists.txt @@ -43,6 +43,7 @@ absl_cc_library( PUBLIC ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME bad_any_cast_impl @@ -69,7 +70,7 @@ absl_cc_test( absl::exception_testing absl::raw_logging_internal absl::test_instance_tracker - gmock_main + GTest::gmock_main ) absl_cc_test( @@ -85,7 +86,7 @@ absl_cc_test( absl::exception_testing absl::raw_logging_internal absl::test_instance_tracker - gmock_main + GTest::gmock_main ) absl_cc_test( @@ -99,7 +100,7 @@ absl_cc_test( absl::any absl::config absl::exception_safety_testing - gmock_main + GTest::gmock_main ) absl_cc_library( @@ -136,7 +137,7 @@ absl_cc_test( absl::inlined_vector absl::hash_testing absl::strings - gmock_main + GTest::gmock_main ) absl_cc_test( @@ -156,7 +157,7 @@ absl_cc_test( absl::inlined_vector absl::hash_testing absl::strings - gmock_main + GTest::gmock_main ) absl_cc_library( @@ -222,7 +223,7 @@ absl_cc_test( absl::raw_logging_internal absl::strings absl::type_traits - gmock_main + GTest::gmock_main ) absl_cc_test( @@ -236,9 +237,10 @@ absl_cc_test( absl::optional absl::config absl::exception_safety_testing - gmock_main + GTest::gmock_main ) +# Internal-only target, do not depend on directly. absl_cc_library( NAME conformance_testing @@ -258,7 +260,7 @@ absl_cc_library( absl::type_traits absl::strings absl::utility - gmock_main + GTest::gmock_main TESTONLY ) @@ -275,7 +277,7 @@ absl_cc_test( DEPS absl::conformance_testing absl::type_traits - gmock_main + GTest::gmock_main ) absl_cc_test( @@ -288,7 +290,7 @@ absl_cc_test( DEPS absl::conformance_testing absl::type_traits - gmock_main + GTest::gmock_main ) absl_cc_library( @@ -324,7 +326,7 @@ absl_cc_test( absl::memory absl::type_traits absl::strings - gmock_main + GTest::gmock_main ) absl_cc_library( @@ -350,7 +352,7 @@ absl_cc_test( DEPS absl::base absl::compare - gmock_main + GTest::gmock_main ) absl_cc_test( @@ -365,5 +367,5 @@ absl_cc_test( absl::config absl::exception_safety_testing absl::memory - gmock_main + GTest::gmock_main ) diff --git a/absl/types/any.h b/absl/types/any.h index fc5a0746..204da26d 100644 --- a/absl/types/any.h +++ b/absl/types/any.h @@ -81,18 +81,9 @@ ABSL_NAMESPACE_END #include <utility> #include "absl/base/internal/fast_type_id.h" -#include "absl/base/macros.h" #include "absl/meta/type_traits.h" #include "absl/types/bad_any_cast.h" -// NOTE: This macro is an implementation detail that is undefined at the bottom -// of the file. It is not intended for expansion directly from user code. -#ifdef ABSL_ANY_DETAIL_HAS_RTTI -#error ABSL_ANY_DETAIL_HAS_RTTI cannot be directly set -#elif !defined(__GNUC__) || defined(__GXX_RTTI) -#define ABSL_ANY_DETAIL_HAS_RTTI 1 -#endif // !defined(__GNUC__) || defined(__GXX_RTTI) - namespace absl { ABSL_NAMESPACE_BEGIN @@ -348,7 +339,7 @@ class any { // returns `false`. bool has_value() const noexcept { return obj_ != nullptr; } -#if ABSL_ANY_DETAIL_HAS_RTTI +#ifdef ABSL_INTERNAL_HAS_RTTI // Returns: typeid(T) if *this has a contained object of type T, otherwise // typeid(void). const std::type_info& type() const noexcept { @@ -358,7 +349,7 @@ class any { return typeid(void); } -#endif // ABSL_ANY_DETAIL_HAS_RTTI +#endif // ABSL_INTERNAL_HAS_RTTI private: // Tagged type-erased abstraction for holding a cloneable object. @@ -367,9 +358,9 @@ class any { virtual ~ObjInterface() = default; virtual std::unique_ptr<ObjInterface> Clone() const = 0; virtual const void* ObjTypeId() const noexcept = 0; -#if ABSL_ANY_DETAIL_HAS_RTTI +#ifdef ABSL_INTERNAL_HAS_RTTI virtual const std::type_info& Type() const noexcept = 0; -#endif // ABSL_ANY_DETAIL_HAS_RTTI +#endif // ABSL_INTERNAL_HAS_RTTI }; // Hold a value of some queryable type, with an ability to Clone it. @@ -386,9 +377,9 @@ class any { const void* ObjTypeId() const noexcept final { return IdForType<T>(); } -#if ABSL_ANY_DETAIL_HAS_RTTI +#ifdef ABSL_INTERNAL_HAS_RTTI const std::type_info& Type() const noexcept final { return typeid(T); } -#endif // ABSL_ANY_DETAIL_HAS_RTTI +#endif // ABSL_INTERNAL_HAS_RTTI T value; }; @@ -521,8 +512,6 @@ T* any_cast(any* operand) noexcept { ABSL_NAMESPACE_END } // namespace absl -#undef ABSL_ANY_DETAIL_HAS_RTTI - #endif // ABSL_USES_STD_ANY #endif // ABSL_TYPES_ANY_H_ diff --git a/absl/types/any_test.cc b/absl/types/any_test.cc index 70e4ba22..d382b927 100644 --- a/absl/types/any_test.cc +++ b/absl/types/any_test.cc @@ -754,26 +754,23 @@ TEST(AnyTest, FailedCopy) { // Test the guarantees regarding exceptions in emplace. TEST(AnyTest, FailedEmplace) { - { - BadCopyable bad; - absl::any target; - ABSL_ANY_TEST_EXPECT_BAD_COPY(target.emplace<BadCopyable>(bad)); - } + BadCopyable bad; + absl::any target; + ABSL_ANY_TEST_EXPECT_BAD_COPY(target.emplace<BadCopyable>(bad)); +} - { - BadCopyable bad; - absl::any target(absl::in_place_type<int>); - ABSL_ANY_TEST_EXPECT_BAD_COPY(target.emplace<BadCopyable>(bad)); -#if defined(ABSL_USES_STD_ANY) && defined(__GLIBCXX__) - // libstdc++ std::any::emplace() implementation (as of 7.2) has a bug: if an - // exception is thrown, *this contains a value. -#define ABSL_GLIBCXX_ANY_EMPLACE_EXCEPTION_BUG 1 -#endif -#if defined(ABSL_HAVE_EXCEPTIONS) && \ - !defined(ABSL_GLIBCXX_ANY_EMPLACE_EXCEPTION_BUG) - EXPECT_FALSE(target.has_value()); +// GCC and Clang have a bug here. +// Ine some cases, the exception seems to be thrown at the wrong time, and +// target may contain a value. +#ifdef __GNUC__ +TEST(AnyTest, DISABLED_FailedEmplaceInPlace) { +#else +TEST(AnyTest, FailedEmplaceInPlace) { #endif - } + BadCopyable bad; + absl::any target(absl::in_place_type<int>); + ABSL_ANY_TEST_EXPECT_BAD_COPY(target.emplace<BadCopyable>(bad)); + EXPECT_FALSE(target.has_value()); } } // namespace diff --git a/absl/types/bad_optional_access.h b/absl/types/bad_optional_access.h index a500286a..049e72ad 100644 --- a/absl/types/bad_optional_access.h +++ b/absl/types/bad_optional_access.h @@ -67,7 +67,7 @@ class bad_optional_access : public std::exception { namespace optional_internal { // throw delegator -[[noreturn]] void throw_bad_optional_access(); +[[noreturn]] ABSL_DLL void throw_bad_optional_access(); } // namespace optional_internal ABSL_NAMESPACE_END diff --git a/absl/types/bad_variant_access.h b/absl/types/bad_variant_access.h index 095969f9..8ab215e9 100644 --- a/absl/types/bad_variant_access.h +++ b/absl/types/bad_variant_access.h @@ -70,8 +70,8 @@ class bad_variant_access : public std::exception { namespace variant_internal { -[[noreturn]] void ThrowBadVariantAccess(); -[[noreturn]] void Rethrow(); +[[noreturn]] ABSL_DLL void ThrowBadVariantAccess(); +[[noreturn]] ABSL_DLL void Rethrow(); } // namespace variant_internal ABSL_NAMESPACE_END diff --git a/absl/types/internal/conformance_profile.h b/absl/types/internal/conformance_profile.h index cf64ff4f..37b017db 100644 --- a/absl/types/internal/conformance_profile.h +++ b/absl/types/internal/conformance_profile.h @@ -719,6 +719,7 @@ struct SyntacticConformanceProfileOf { type##_support); \ ABSL_INTERNAL_CONFORMANCE_TESTING_DATA_MEMBER_DEF_IMPL(bool, is_##type) +#ifdef ABSL_INTERNAL_NEED_REDUNDANT_CONSTEXPR_DECL ABSL_INTERNAL_CONFORMANCE_TESTING_DATA_MEMBER_DEF(default_constructible); ABSL_INTERNAL_CONFORMANCE_TESTING_DATA_MEMBER_DEF(move_constructible); ABSL_INTERNAL_CONFORMANCE_TESTING_DATA_MEMBER_DEF(copy_constructible); @@ -733,6 +734,7 @@ ABSL_INTERNAL_CONFORMANCE_TESTING_DATA_MEMBER_DEF(greater_equal_comparable); ABSL_INTERNAL_CONFORMANCE_TESTING_DATA_MEMBER_DEF(greater_than_comparable); ABSL_INTERNAL_CONFORMANCE_TESTING_DATA_MEMBER_DEF(swappable); ABSL_INTERNAL_CONFORMANCE_TESTING_DATA_MEMBER_DEF(hashable); +#endif #undef ABSL_INTERNAL_CONFORMANCE_TESTING_DATA_MEMBER_DEF #undef ABSL_INTERNAL_CONFORMANCE_TESTING_DATA_MEMBER_DEF_IMPL diff --git a/absl/types/internal/optional.h b/absl/types/internal/optional.h index 92932b60..6ed0c669 100644 --- a/absl/types/internal/optional.h +++ b/absl/types/internal/optional.h @@ -91,7 +91,15 @@ class optional_data_dtor_base { void destruct() noexcept { if (engaged_) { + // `data_` must be initialized if `engaged_` is true. +#if ABSL_INTERNAL_HAVE_MIN_GNUC_VERSION(12, 0) +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#endif data_.~T(); +#if ABSL_INTERNAL_HAVE_MIN_GNUC_VERSION(12, 0) +#pragma GCC diagnostic pop +#endif engaged_ = false; } } diff --git a/absl/types/internal/variant.h b/absl/types/internal/variant.h index 772008c7..7c402ece 100644 --- a/absl/types/internal/variant.h +++ b/absl/types/internal/variant.h @@ -16,8 +16,8 @@ // separate file to avoid cluttering the top of the API header with // implementation details. -#ifndef ABSL_TYPES_variant_internal_H_ -#define ABSL_TYPES_variant_internal_H_ +#ifndef ABSL_TYPES_VARIANT_INTERNAL_H_ +#define ABSL_TYPES_VARIANT_INTERNAL_H_ #include <cassert> #include <cstddef> @@ -1643,4 +1643,4 @@ ABSL_NAMESPACE_END } // namespace absl #endif // !defined(ABSL_USES_STD_VARIANT) -#endif // ABSL_TYPES_variant_internal_H_ +#endif // ABSL_TYPES_VARIANT_INTERNAL_H_ diff --git a/absl/types/optional.h b/absl/types/optional.h index 61540cfd..134b2aff 100644 --- a/absl/types/optional.h +++ b/absl/types/optional.h @@ -282,15 +282,16 @@ class optional : private optional_internal::optional_data<T>, optional& operator=(optional&& src) = default; // Value assignment operators - template < - typename U = T, - typename = typename std::enable_if<absl::conjunction< - absl::negation< - std::is_same<optional<T>, typename std::decay<U>::type>>, - absl::negation< - absl::conjunction<std::is_scalar<T>, - std::is_same<T, typename std::decay<U>::type>>>, - std::is_constructible<T, U>, std::is_assignable<T&, U>>::value>::type> + template <typename U = T, + int&..., // Workaround an internal compiler error in GCC 5 to 10. + typename = typename std::enable_if<absl::conjunction< + absl::negation< + std::is_same<optional<T>, typename std::decay<U>::type> >, + absl::negation<absl::conjunction< + std::is_scalar<T>, + std::is_same<T, typename std::decay<U>::type> > >, + std::is_constructible<T, U>, + std::is_assignable<T&, U> >::value>::type> optional& operator=(U&& v) { this->assign(std::forward<U>(v)); return *this; @@ -298,13 +299,14 @@ class optional : private optional_internal::optional_data<T>, template < typename U, + int&..., // Workaround an internal compiler error in GCC 5 to 10. typename = typename std::enable_if<absl::conjunction< - absl::negation<std::is_same<T, U>>, + absl::negation<std::is_same<T, U> >, std::is_constructible<T, const U&>, std::is_assignable<T&, const U&>, absl::negation< optional_internal:: is_constructible_convertible_assignable_from_optional< - T, U>>>::value>::type> + T, U> > >::value>::type> optional& operator=(const optional<U>& rhs) { if (rhs) { this->assign(*rhs); @@ -315,13 +317,14 @@ class optional : private optional_internal::optional_data<T>, } template <typename U, + int&..., // Workaround an internal compiler error in GCC 5 to 10. typename = typename std::enable_if<absl::conjunction< - absl::negation<std::is_same<T, U>>, std::is_constructible<T, U>, - std::is_assignable<T&, U>, + absl::negation<std::is_same<T, U> >, + std::is_constructible<T, U>, std::is_assignable<T&, U>, absl::negation< optional_internal:: is_constructible_convertible_assignable_from_optional< - T, U>>>::value>::type> + T, U> > >::value>::type> optional& operator=(optional<U>&& rhs) { if (rhs) { this->assign(std::move(*rhs)); diff --git a/absl/types/optional_test.cc b/absl/types/optional_test.cc index 7ef142cb..9477c150 100644 --- a/absl/types/optional_test.cc +++ b/absl/types/optional_test.cc @@ -27,6 +27,37 @@ #include "absl/meta/type_traits.h" #include "absl/strings/string_view.h" +#if defined(__cplusplus) && __cplusplus >= 202002L +// In C++20, volatile-qualified return types are deprecated. +#define ABSL_VOLATILE_RETURN_TYPES_DEPRECATED 1 +#endif + +// The following types help test an internal compiler error in GCC5 though +// GCC10. The case OptionalTest.InternalCompilerErrorInGcc5ToGcc10 crashes the +// compiler without a workaround. This test case should remain at the beginning +// of the file as the internal compiler error is sensitive to other constructs +// in this file. +template <class T, class...> +using GccIceHelper1 = T; +template <typename T> +struct GccIceHelper2 {}; +template <typename T> +class GccIce { + template <typename U, + typename SecondTemplateArgHasToExistForSomeReason = void, + typename DependentType = void, + typename = std::is_assignable<GccIceHelper1<T, DependentType>&, U>> + GccIce& operator=(GccIceHelper2<U> const&) {} +}; + +TEST(OptionalTest, InternalCompilerErrorInGcc5ToGcc10) { + GccIce<int> instantiate_ice_with_same_type_as_optional; + static_cast<void>(instantiate_ice_with_same_type_as_optional); + absl::optional<int> val1; + absl::optional<int> val2; + val1 = val2; +} + struct Hashable {}; namespace std { @@ -205,6 +236,7 @@ TEST(optionalTest, CopyConstructor) { EXPECT_TRUE(opt42_copy); EXPECT_EQ(42, *opt42_copy); } +#if !defined(ABSL_VOLATILE_RETURN_TYPES_DEPRECATED) { absl::optional<volatile int> empty, opt42 = 42; absl::optional<volatile int> empty_copy(empty); @@ -213,6 +245,7 @@ TEST(optionalTest, CopyConstructor) { EXPECT_TRUE(opt42_copy); EXPECT_EQ(42, *opt42_copy); } +#endif // test copyablility EXPECT_TRUE(std::is_copy_constructible<absl::optional<int>>::value); EXPECT_TRUE(std::is_copy_constructible<absl::optional<Copyable>>::value); @@ -224,18 +257,11 @@ TEST(optionalTest, CopyConstructor) { EXPECT_FALSE( absl::is_trivially_copy_constructible<absl::optional<Copyable>>::value); -#if defined(ABSL_USES_STD_OPTIONAL) && defined(__GLIBCXX__) - // libstdc++ std::optional implementation (as of 7.2) has a bug: when T is - // trivially copyable, optional<T> is not trivially copyable (due to one of - // its base class is unconditionally nontrivial). -#define ABSL_GLIBCXX_OPTIONAL_TRIVIALITY_BUG 1 -#endif -#ifndef ABSL_GLIBCXX_OPTIONAL_TRIVIALITY_BUG EXPECT_TRUE( absl::is_trivially_copy_constructible<absl::optional<int>>::value); EXPECT_TRUE( absl::is_trivially_copy_constructible<absl::optional<const int>>::value); -#ifndef _MSC_VER +#if !defined(_MSC_VER) && !defined(ABSL_VOLATILE_RETURN_TYPES_DEPRECATED) // See defect report "Trivial copy/move constructor for class with volatile // member" at // http://www.open-std.org/jtc1/sc22/wg21/docs/cwg_defects.html#2094 @@ -244,8 +270,7 @@ TEST(optionalTest, CopyConstructor) { // Also a cv-qualified scalar type should be trivially copyable. EXPECT_TRUE(absl::is_trivially_copy_constructible< absl::optional<volatile int>>::value); -#endif // _MSC_VER -#endif // ABSL_GLIBCXX_OPTIONAL_TRIVIALITY_BUG +#endif // !defined(_MSC_VER) && !defined(ABSL_VOLATILE_RETURN_TYPES_DEPRECATED) // constexpr copy constructor for trivially copyable types { @@ -275,17 +300,10 @@ TEST(optionalTest, CopyConstructor) { EXPECT_TRUE(absl::is_trivially_copy_constructible< absl::optional<const TrivialCopyable>>::value); #endif - // When testing with VS 2017 15.3, there seems to be a bug in MSVC - // std::optional when T is volatile-qualified. So skipping this test. - // Bug report: - // https://connect.microsoft.com/VisualStudio/feedback/details/3142534 -#if defined(ABSL_USES_STD_OPTIONAL) && defined(_MSC_VER) && _MSC_VER >= 1911 -#define ABSL_MSVC_OPTIONAL_VOLATILE_COPY_BUG 1 -#endif -#ifndef ABSL_MSVC_OPTIONAL_VOLATILE_COPY_BUG +#if !defined(ABSL_VOLATILE_RETURN_TYPES_DEPRECATED) EXPECT_FALSE(std::is_copy_constructible< absl::optional<volatile TrivialCopyable>>::value); -#endif +#endif // !defined(ABSL_VOLATILE_RETURN_TYPES_DEPRECATED) } } @@ -305,11 +323,9 @@ TEST(optionalTest, MoveConstructor) { EXPECT_FALSE(std::is_move_constructible<absl::optional<NonMovable>>::value); // test noexcept EXPECT_TRUE(std::is_nothrow_move_constructible<absl::optional<int>>::value); -#ifndef ABSL_USES_STD_OPTIONAL EXPECT_EQ( absl::default_allocator_is_nothrow::value, std::is_nothrow_move_constructible<absl::optional<MoveableThrow>>::value); -#endif EXPECT_TRUE(std::is_nothrow_move_constructible< absl::optional<MoveableNoThrow>>::value); } @@ -638,8 +654,7 @@ TEST(optionalTest, CopyAssignment) { EXPECT_TRUE(absl::is_copy_assignable<NonTrivial>::value); EXPECT_FALSE(absl::is_trivially_copy_assignable<NonTrivial>::value); - // std::optional doesn't support volatile nontrivial types. -#ifndef ABSL_USES_STD_OPTIONAL +#if !defined(ABSL_VOLATILE_RETURN_TYPES_DEPRECATED) { StructorListener listener; Listenable::listener = &listener; @@ -658,7 +673,7 @@ TEST(optionalTest, CopyAssignment) { EXPECT_EQ(1, listener.destruct); EXPECT_EQ(1, listener.volatile_copy_assign); } -#endif // ABSL_USES_STD_OPTIONAL +#endif // !defined(ABSL_VOLATILE_RETURN_TYPES_DEPRECATED) } TEST(optionalTest, MoveAssignment) { @@ -681,8 +696,7 @@ TEST(optionalTest, MoveAssignment) { EXPECT_EQ(1, listener.destruct); EXPECT_EQ(1, listener.move_assign); } - // std::optional doesn't support volatile nontrivial types. -#ifndef ABSL_USES_STD_OPTIONAL +#if !defined(ABSL_VOLATILE_RETURN_TYPES_DEPRECATED) { StructorListener listener; Listenable::listener = &listener; @@ -702,7 +716,7 @@ TEST(optionalTest, MoveAssignment) { EXPECT_EQ(1, listener.destruct); EXPECT_EQ(1, listener.volatile_move_assign); } -#endif // ABSL_USES_STD_OPTIONAL +#endif // !defined(ABSL_VOLATILE_RETURN_TYPES_DEPRECATED) EXPECT_FALSE(absl::is_move_assignable<absl::optional<const int>>::value); EXPECT_TRUE(absl::is_move_assignable<absl::optional<Copyable>>::value); EXPECT_TRUE(absl::is_move_assignable<absl::optional<MoveableThrow>>::value); @@ -1038,6 +1052,7 @@ TEST(optionalTest, Value) { #endif EXPECT_EQ("c&&", TypeQuals(OC(absl::in_place, "xvalue_c").value())); +#if !defined(ABSL_VOLATILE_RETURN_TYPES_DEPRECATED) // test on volatile type using OV = absl::optional<volatile int>; OV lvalue_v(absl::in_place, 42); @@ -1045,6 +1060,7 @@ TEST(optionalTest, Value) { EXPECT_EQ(42, OV(42).value()); EXPECT_TRUE((std::is_same<volatile int&, decltype(lvalue_v.value())>::value)); EXPECT_TRUE((std::is_same<volatile int&&, decltype(OV(42).value())>::value)); +#endif // !defined(ABSL_VOLATILE_RETURN_TYPES_DEPRECATED) // test exception throw on value() absl::optional<int> empty; @@ -1087,6 +1103,7 @@ TEST(optionalTest, DerefOperator) { #endif EXPECT_EQ("c&&", TypeQuals(*OC(absl::in_place, "xvalue_c"))); +#if !defined(ABSL_VOLATILE_RETURN_TYPES_DEPRECATED) // test on volatile type using OV = absl::optional<volatile int>; OV lvalue_v(absl::in_place, 42); @@ -1094,6 +1111,7 @@ TEST(optionalTest, DerefOperator) { EXPECT_EQ(42, *OV(42)); EXPECT_TRUE((std::is_same<volatile int&, decltype(*lvalue_v)>::value)); EXPECT_TRUE((std::is_same<volatile int&&, decltype(*OV(42))>::value)); +#endif // !defined(ABSL_VOLATILE_RETURN_TYPES_DEPRECATED) constexpr absl::optional<int> opt1(1); static_assert(*opt1 == 1, ""); @@ -1558,12 +1576,10 @@ TEST(optionalTest, NoExcept) { static_assert( std::is_nothrow_move_constructible<absl::optional<MoveMeNoThrow>>::value, ""); -#ifndef ABSL_USES_STD_OPTIONAL static_assert(absl::default_allocator_is_nothrow::value == std::is_nothrow_move_constructible< absl::optional<MoveMeThrow>>::value, ""); -#endif std::vector<absl::optional<MoveMeNoThrow>> v; for (int i = 0; i < 10; ++i) v.emplace_back(); } diff --git a/absl/types/span.h b/absl/types/span.h index 95fe7926..fdfbd77c 100644 --- a/absl/types/span.h +++ b/absl/types/span.h @@ -40,7 +40,6 @@ // * `absl::Span` has compiler-provided move and copy constructors and // assignment. This is due to them being specified as `constexpr`, but that // implies const in C++11. -// * `absl::Span` has no `element_type` typedef // * A read-only `absl::Span<const T>` can be implicitly constructed from an // initializer list. // * `absl::Span` has no `bytes()`, `size_bytes()`, `as_bytes()`, or @@ -170,6 +169,7 @@ class Span { typename std::enable_if<!std::is_const<T>::value, U>::type; public: + using element_type = T; using value_type = absl::remove_cv_t<T>; using pointer = T*; using const_pointer = const T*; @@ -243,8 +243,8 @@ class Span { // template <typename LazyT = T, typename = EnableIfConstView<LazyT>> - Span( - std::initializer_list<value_type> v) noexcept // NOLINT(runtime/explicit) + Span(std::initializer_list<value_type> v + ABSL_ATTRIBUTE_LIFETIME_BOUND) noexcept // NOLINT(runtime/explicit) : Span(v.begin(), v.size()) {} // Accessors @@ -664,7 +664,8 @@ constexpr Span<T> MakeSpan(T* ptr, size_t size) noexcept { template <int&... ExplicitArgumentBarrier, typename T> Span<T> MakeSpan(T* begin, T* end) noexcept { - return ABSL_HARDENING_ASSERT(begin <= end), Span<T>(begin, end - begin); + return ABSL_HARDENING_ASSERT(begin <= end), + Span<T>(begin, static_cast<size_t>(end - begin)); } template <int&... ExplicitArgumentBarrier, typename C> diff --git a/absl/types/span_test.cc b/absl/types/span_test.cc index 2584339b..13264aae 100644 --- a/absl/types/span_test.cc +++ b/absl/types/span_test.cc @@ -661,6 +661,8 @@ TEST(IntSpan, ExposesContainerTypesAndConsts) { CheckType<absl::Span<int>::const_reverse_iterator>(slice.crend()); testing::StaticAssertTypeEq<int, absl::Span<int>::value_type>(); testing::StaticAssertTypeEq<int, absl::Span<const int>::value_type>(); + testing::StaticAssertTypeEq<int, absl::Span<int>::element_type>(); + testing::StaticAssertTypeEq<const int, absl::Span<const int>::element_type>(); testing::StaticAssertTypeEq<int*, absl::Span<int>::pointer>(); testing::StaticAssertTypeEq<const int*, absl::Span<const int>::pointer>(); testing::StaticAssertTypeEq<int&, absl::Span<int>::reference>(); diff --git a/absl/utility/BUILD.bazel b/absl/utility/BUILD.bazel index 02b2c407..ca4bc0a6 100644 --- a/absl/utility/BUILD.bazel +++ b/absl/utility/BUILD.bazel @@ -14,7 +14,6 @@ # limitations under the License. # -load("@rules_cc//cc:defs.bzl", "cc_library", "cc_test") load( "//absl:copts/configure_copts.bzl", "ABSL_DEFAULT_COPTS", diff --git a/absl/utility/CMakeLists.txt b/absl/utility/CMakeLists.txt index e1edd19a..865b758f 100644 --- a/absl/utility/CMakeLists.txt +++ b/absl/utility/CMakeLists.txt @@ -40,5 +40,5 @@ absl_cc_test( absl::core_headers absl::memory absl::strings - gmock_main + GTest::gmock_main ) diff --git a/ci/cmake_common.sh b/ci/cmake_common.sh index aec8a117..3c08255b 100644 --- a/ci/cmake_common.sh +++ b/ci/cmake_common.sh @@ -14,7 +14,7 @@ # The commit of GoogleTest to be used in the CMake tests in this directory. # Keep this in sync with the commit in the WORKSPACE file. -readonly ABSL_GOOGLETEST_COMMIT="8567b09290fe402cf01923e2131c5635b8ed851b" +readonly ABSL_GOOGLETEST_COMMIT="15460959cbbfa20e66ef0b5ab497367e47fc0a04" # release-1.12.0 # Avoid depending on GitHub by looking for a cached copy of the commit first. if [[ -r "${KOKORO_GFILE_DIR:-}/distdir/${ABSL_GOOGLETEST_COMMIT}.zip" ]]; then diff --git a/ci/cmake_install_test.sh b/ci/cmake_install_test.sh index ffc6b516..97ed8478 100755 --- a/ci/cmake_install_test.sh +++ b/ci/cmake_install_test.sh @@ -36,8 +36,11 @@ for link_type in ${LINK_TYPE}; do --tmpfs=/abseil-cpp:exec \ --workdir=/abseil-cpp \ --cap-add=SYS_PTRACE \ + -e "ABSL_GOOGLETEST_COMMIT=${ABSL_GOOGLETEST_COMMIT}" \ + -e "ABSL_GOOGLETEST_DOWNLOAD_URL=${ABSL_GOOGLETEST_DOWNLOAD_URL}" \ -e "LINK_TYPE=${link_type}" \ --rm \ + ${DOCKER_EXTRA_ARGS:-} \ ${DOCKER_CONTAINER} \ /bin/bash -c "cp -r /abseil-cpp-ro/* . && CMake/install_test_project/test.sh" done diff --git a/ci/linux_clang-latest_libcxx_asan_bazel.sh b/ci/linux_clang-latest_libcxx_asan_bazel.sh index ffbb8327..196e5c1d 100755 --- a/ci/linux_clang-latest_libcxx_asan_bazel.sh +++ b/ci/linux_clang-latest_libcxx_asan_bazel.sh @@ -70,19 +70,21 @@ for std in ${STD}; do --rm \ -e CC="/opt/llvm/clang/bin/clang" \ -e BAZEL_CXXOPTS="-std=${std}:-nostdinc++" \ - -e BAZEL_LINKOPTS="-L/opt/llvm/libcxx/lib:-lc++:-lc++abi:-lm:-Wl,-rpath=/opt/llvm/libcxx/lib" \ - -e CPLUS_INCLUDE_PATH="/opt/llvm/libcxx/include/c++/v1" \ + -e BAZEL_LINKOPTS="-L/opt/llvm/libcxx/lib/x86_64-unknown-linux-gnu:-lc++:-lc++abi:-lm:-Wl,-rpath=/opt/llvm/libcxx/lib/x86_64-unknown-linux-gnu" \ + -e CPLUS_INCLUDE_PATH="/opt/llvm/libcxx/include/x86_64-unknown-linux-gnu/c++/v1:/opt/llvm/libcxx/include/c++/v1" \ ${DOCKER_EXTRA_ARGS:-} \ ${DOCKER_CONTAINER} \ /usr/local/bin/bazel test ... \ --compilation_mode="${compilation_mode}" \ --copt="${exceptions_mode}" \ + --copt="-DGTEST_REMOVE_LEGACY_TEST_CASEAPI_=1" \ --copt="-fsanitize=address" \ --copt="-fsanitize=float-divide-by-zero" \ --copt="-fsanitize=nullability" \ --copt="-fsanitize=undefined" \ --copt="-fno-sanitize-blacklist" \ --copt=-Werror \ + --distdir="/bazel-distdir" \ --keep_going \ --linkopt="-fsanitize=address" \ --linkopt="-fsanitize-link-c++-runtime" \ diff --git a/ci/linux_clang-latest_libcxx_bazel.sh b/ci/linux_clang-latest_libcxx_bazel.sh index f6a2221e..39aecf58 100755 --- a/ci/linux_clang-latest_libcxx_bazel.sh +++ b/ci/linux_clang-latest_libcxx_bazel.sh @@ -71,8 +71,8 @@ for std in ${STD}; do --rm \ -e CC="/opt/llvm/clang/bin/clang" \ -e BAZEL_CXXOPTS="-std=${std}:-nostdinc++" \ - -e BAZEL_LINKOPTS="-L/opt/llvm/libcxx/lib:-lc++:-lc++abi:-lm:-Wl,-rpath=/opt/llvm/libcxx/lib" \ - -e CPLUS_INCLUDE_PATH="/opt/llvm/libcxx/include/c++/v1" \ + -e BAZEL_LINKOPTS="-L/opt/llvm/libcxx/lib/x86_64-unknown-linux-gnu:-lc++:-lc++abi:-lm:-Wl,-rpath=/opt/llvm/libcxx/lib/x86_64-unknown-linux-gnu" \ + -e CPLUS_INCLUDE_PATH="/opt/llvm/libcxx/include/x86_64-unknown-linux-gnu/c++/v1:/opt/llvm/libcxx/include/c++/v1" \ ${DOCKER_EXTRA_ARGS:-} \ ${DOCKER_CONTAINER} \ /bin/sh -c " @@ -83,8 +83,10 @@ for std in ${STD}; do /usr/local/bin/bazel test ... \ --compilation_mode=\"${compilation_mode}\" \ --copt=\"${exceptions_mode}\" \ + --copt=\"-DGTEST_REMOVE_LEGACY_TEST_CASEAPI_=1\" \ --copt=-Werror \ --define=\"absl=1\" \ + --distdir=\"/bazel-distdir\" \ --keep_going \ --show_timestamps \ --test_env=\"GTEST_INSTALL_FAILURE_SIGNAL_HANDLER=1\" \ diff --git a/ci/linux_clang-latest_libcxx_tsan_bazel.sh b/ci/linux_clang-latest_libcxx_tsan_bazel.sh index e70e8214..b0a1abff 100755 --- a/ci/linux_clang-latest_libcxx_tsan_bazel.sh +++ b/ci/linux_clang-latest_libcxx_tsan_bazel.sh @@ -70,17 +70,19 @@ for std in ${STD}; do --rm \ -e CC="/opt/llvm/clang/bin/clang" \ -e BAZEL_CXXOPTS="-std=${std}:-nostdinc++" \ - -e BAZEL_LINKOPTS="-L/opt/llvm/libcxx-tsan/lib:-lc++:-lc++abi:-lm:-Wl,-rpath=/opt/llvm/libcxx-tsan/lib" \ - -e CPLUS_INCLUDE_PATH="/opt/llvm/libcxx-tsan/include/c++/v1" \ + -e BAZEL_LINKOPTS="-L/opt/llvm/libcxx-tsan/lib/x86_64-unknown-linux-gnu:-lc++:-lc++abi:-lm:-Wl,-rpath=/opt/llvm/libcxx-tsan/lib/x86_64-unknown-linux-gnu" \ + -e CPLUS_INCLUDE_PATH="/opt/llvm/libcxx-tsan/include/x86_64-unknown-linux-gnu/c++/v1:/opt/llvm/libcxx-tsan/include/c++/v1" \ ${DOCKER_EXTRA_ARGS:-} \ ${DOCKER_CONTAINER} \ /usr/local/bin/bazel test ... \ --build_tag_filters="-notsan" \ --compilation_mode="${compilation_mode}" \ --copt="${exceptions_mode}" \ + --copt="-DGTEST_REMOVE_LEGACY_TEST_CASEAPI_=1" \ --copt="-fsanitize=thread" \ --copt="-fno-sanitize-blacklist" \ --copt=-Werror \ + --distdir="/bazel-distdir" \ --keep_going \ --linkopt="-fsanitize=thread" \ --show_timestamps \ diff --git a/ci/linux_clang-latest_libstdcxx_bazel.sh b/ci/linux_clang-latest_libstdcxx_bazel.sh index 0986ff40..8550481b 100755 --- a/ci/linux_clang-latest_libstdcxx_bazel.sh +++ b/ci/linux_clang-latest_libstdcxx_bazel.sh @@ -25,7 +25,7 @@ if [[ -z ${ABSEIL_ROOT:-} ]]; then fi if [[ -z ${STD:-} ]]; then - STD="c++11 c++14 c++17 c++20" + STD="c++11 c++14 c++17" fi if [[ -z ${COMPILATION_MODE:-} ]]; then @@ -75,9 +75,11 @@ for std in ${STD}; do /usr/local/bin/bazel test ... \ --compilation_mode="${compilation_mode}" \ --copt="--gcc-toolchain=/usr/local" \ + --copt="-DGTEST_REMOVE_LEGACY_TEST_CASEAPI_=1" \ --copt="${exceptions_mode}" \ --copt=-Werror \ --define="absl=1" \ + --distdir="/bazel-distdir" \ --keep_going \ --linkopt="--gcc-toolchain=/usr/local" \ --show_timestamps \ diff --git a/ci/linux_docker_containers.sh b/ci/linux_docker_containers.sh index 1c29d9a1..f55e153b 100644 --- a/ci/linux_docker_containers.sh +++ b/ci/linux_docker_containers.sh @@ -16,6 +16,6 @@ # Test scripts should source this file to get the identifiers. readonly LINUX_ALPINE_CONTAINER="gcr.io/google.com/absl-177019/alpine:20201026" -readonly LINUX_CLANG_LATEST_CONTAINER="gcr.io/google.com/absl-177019/linux_hybrid-latest:20201008" -readonly LINUX_GCC_LATEST_CONTAINER="gcr.io/google.com/absl-177019/linux_hybrid-latest:20201008" -readonly LINUX_GCC_FLOOR_CONTAINER="gcr.io/google.com/absl-177019/linux_gcc-floor:20201015" +readonly LINUX_CLANG_LATEST_CONTAINER="gcr.io/google.com/absl-177019/linux_hybrid-latest:20220217" +readonly LINUX_GCC_LATEST_CONTAINER="gcr.io/google.com/absl-177019/linux_hybrid-latest:20220217" +readonly LINUX_GCC_FLOOR_CONTAINER="gcr.io/google.com/absl-177019/linux_gcc-floor:20220621" diff --git a/ci/linux_gcc-floor_libstdcxx_bazel.sh b/ci/linux_gcc-floor_libstdcxx_bazel.sh index 224aef81..8b41b5d7 100755 --- a/ci/linux_gcc-floor_libstdcxx_bazel.sh +++ b/ci/linux_gcc-floor_libstdcxx_bazel.sh @@ -75,8 +75,10 @@ for std in ${STD}; do /usr/local/bin/bazel test ... \ --compilation_mode="${compilation_mode}" \ --copt="${exceptions_mode}" \ + --copt="-DGTEST_REMOVE_LEGACY_TEST_CASEAPI_=1" \ --copt=-Werror \ --define="absl=1" \ + --distdir="/bazel-distdir" \ --keep_going \ --show_timestamps \ --test_env="GTEST_INSTALL_FAILURE_SIGNAL_HANDLER=1" \ diff --git a/ci/linux_gcc-latest_libstdcxx_bazel.sh b/ci/linux_gcc-latest_libstdcxx_bazel.sh index 37d89d9f..3f7c71af 100755 --- a/ci/linux_gcc-latest_libstdcxx_bazel.sh +++ b/ci/linux_gcc-latest_libstdcxx_bazel.sh @@ -81,8 +81,10 @@ for std in ${STD}; do /usr/local/bin/bazel test ... \ --compilation_mode=\"${compilation_mode}\" \ --copt=\"${exceptions_mode}\" \ + --copt=\"-DGTEST_REMOVE_LEGACY_TEST_CASEAPI_=1\" \ --copt=-Werror \ --define=\"absl=1\" \ + --distdir=\"/bazel-distdir\" \ --keep_going \ --show_timestamps \ --test_env=\"GTEST_INSTALL_FAILURE_SIGNAL_HANDLER=1\" \ diff --git a/ci/linux_gcc-latest_libstdcxx_cmake.sh b/ci/linux_gcc-latest_libstdcxx_cmake.sh index ab06aa05..eccb3818 100755 --- a/ci/linux_gcc-latest_libstdcxx_cmake.sh +++ b/ci/linux_gcc-latest_libstdcxx_cmake.sh @@ -54,7 +54,7 @@ for std in ${ABSL_CMAKE_CXX_STANDARDS}; do cmake /abseil-cpp \ -DABSL_GOOGLETEST_DOWNLOAD_URL=${ABSL_GOOGLETEST_DOWNLOAD_URL} \ -DBUILD_SHARED_LIBS=${build_shared} \ - -DBUILD_TESTING=ON \ + -DABSL_BUILD_TESTING=ON \ -DCMAKE_BUILD_TYPE=${compilation_mode} \ -DCMAKE_CXX_STANDARD=${std} \ -DCMAKE_MODULE_LINKER_FLAGS=\"-Wl,--no-undefined\" && \ diff --git a/ci/linux_gcc_alpine_cmake.sh b/ci/linux_gcc_alpine_cmake.sh index bce27d29..bf2e1239 100755 --- a/ci/linux_gcc_alpine_cmake.sh +++ b/ci/linux_gcc_alpine_cmake.sh @@ -53,7 +53,7 @@ for std in ${ABSL_CMAKE_CXX_STANDARDS}; do /bin/sh -c " cmake /abseil-cpp \ -DABSL_GOOGLETEST_DOWNLOAD_URL=${ABSL_GOOGLETEST_DOWNLOAD_URL} \ - -DBUILD_TESTING=ON \ + -DABSL_BUILD_TESTING=ON \ -DCMAKE_BUILD_TYPE=${compilation_mode} \ -DCMAKE_CXX_STANDARD=${std} \ -DCMAKE_MODULE_LINKER_FLAGS=\"-Wl,--no-undefined\" && \ diff --git a/ci/macos_xcode_bazel.sh b/ci/macos_xcode_bazel.sh index 738adf94..8d69f1de 100755 --- a/ci/macos_xcode_bazel.sh +++ b/ci/macos_xcode_bazel.sh @@ -24,7 +24,7 @@ if [[ -z ${ABSEIL_ROOT:-} ]]; then fi # If we are running on Kokoro, check for a versioned Bazel binary. -KOKORO_GFILE_BAZEL_BIN="bazel-2.0.0-darwin-x86_64" +KOKORO_GFILE_BAZEL_BIN="bazel-5.1.1-darwin-x86_64" if [[ ${KOKORO_GFILE_DIR:-} ]] && [[ -f ${KOKORO_GFILE_DIR}/${KOKORO_GFILE_BAZEL_BIN} ]]; then BAZEL_BIN="${KOKORO_GFILE_DIR}/${KOKORO_GFILE_BAZEL_BIN}" chmod +x ${BAZEL_BIN} @@ -32,6 +32,13 @@ else BAZEL_BIN="bazel" fi +# Avoid depending on external sites like GitHub by checking --distdir for +# external dependencies first. +# https://docs.bazel.build/versions/master/guide.html#distdir +if [[ ${KOKORO_GFILE_DIR:-} ]] && [[ -d "${KOKORO_GFILE_DIR}/distdir" ]]; then + BAZEL_EXTRA_ARGS="--distdir=${KOKORO_GFILE_DIR}/distdir ${BAZEL_EXTRA_ARGS:-}" +fi + # Print the compiler and Bazel versions. echo "---------------" gcc -v @@ -46,9 +53,11 @@ if [[ -n "${ALTERNATE_OPTIONS:-}" ]]; then fi ${BAZEL_BIN} test ... \ + --copt="-DGTEST_REMOVE_LEGACY_TEST_CASEAPI_=1" \ --copt=-Werror \ --keep_going \ --show_timestamps \ --test_env="TZDIR=${ABSEIL_ROOT}/absl/time/internal/cctz/testdata/zoneinfo" \ --test_output=errors \ - --test_tag_filters=-benchmark + --test_tag_filters=-benchmark \ + ${BAZEL_EXTRA_ARGS:-} diff --git a/ci/macos_xcode_cmake.sh b/ci/macos_xcode_cmake.sh index 2a870cf4..71ea2534 100755 --- a/ci/macos_xcode_cmake.sh +++ b/ci/macos_xcode_cmake.sh @@ -45,7 +45,7 @@ for compilation_mode in ${ABSL_CMAKE_BUILD_TYPES}; do time cmake ${ABSEIL_ROOT} \ -GXcode \ -DBUILD_SHARED_LIBS=${build_shared} \ - -DBUILD_TESTING=ON \ + -DABSL_BUILD_TESTING=ON \ -DCMAKE_BUILD_TYPE=${compilation_mode} \ -DCMAKE_CXX_STANDARD=11 \ -DCMAKE_MODULE_LINKER_FLAGS="-Wl,--no-undefined" \ diff --git a/create_lts.py b/create_lts.py index a98c76b4..56170806 100755 --- a/create_lts.py +++ b/create_lts.py @@ -1,4 +1,4 @@ -#!/usr/bin/env python +#!/usr/bin/env python3 # # Copyright 2021 The Abseil Authors. # @@ -96,6 +96,13 @@ def main(argv): # Replacement directives go here. ReplaceStringsInFile( + 'absl/base/config.h', { + '#undef ABSL_LTS_RELEASE_VERSION': + '#define ABSL_LTS_RELEASE_VERSION {}'.format(datestamp), + '#undef ABSL_LTS_RELEASE_PATCH_LEVEL': + '#define ABSL_LTS_RELEASE_PATCH_LEVEL 0' + }) + ReplaceStringsInFile( 'absl/base/options.h', { '#define ABSL_OPTION_USE_INLINE_NAMESPACE 0': '#define ABSL_OPTION_USE_INLINE_NAMESPACE 1', @@ -108,11 +115,16 @@ def main(argv): 'project(absl LANGUAGES CXX)': 'project(absl LANGUAGES CXX VERSION {})'.format(datestamp) }) - # Set the SOVERSION to YYYYMMDD.0.0 - The first 0 means we only have - # ABI compatible changes, and the second 0 means we can increment it - # to mark changes as ABI-compatible, for patch releases. - ReplaceStringsInFile('CMake/AbseilHelpers.cmake', - {'SOVERSION 0': 'SOVERSION "{}.0.0"'.format(datestamp)}) + # Set the SOVERSION to YYMM.0.0 - The first 0 means we only have ABI + # compatible changes, and the second 0 means we can increment it to + # mark changes as ABI-compatible, for patch releases. Note that we + # only use the last two digits of the year and the month because the + # MacOS linker requires the first part of the SOVERSION to fit into + # 16 bits. + # https://www.sicpers.info/2013/03/how-to-version-a-mach-o-library/ + ReplaceStringsInFile( + 'CMake/AbseilHelpers.cmake', + {'SOVERSION 0': 'SOVERSION "{}.0.0"'.format(datestamp[2:6])}) StripContentBetweenTags('CMakeLists.txt', '# absl:lts-remove-begin', '# absl:lts-remove-end') diff --git a/debian/changelog b/debian/changelog index 791e8091..1b8382c1 100644 --- a/debian/changelog +++ b/debian/changelog @@ -1,3 +1,37 @@ +abseil (20220623.1-1) unstable; urgency=medium + + * New upstream release. + + -- Benjamin Barenblat <bbaren@debian.org> Tue, 18 Oct 2022 10:02:49 -0400 + +abseil (0~20220623.0-2) unstable; urgency=medium + + * Backport an upstream patch to correct pkg-config file generation. + + -- Benjamin Barenblat <bbaren@debian.org> Tue, 30 Aug 2022 22:54:45 -0400 + +abseil (0~20220623.0-1) unstable; urgency=medium + + * New upstream release. (Closes: #1008730, #1012194) + + -- Benjamin Barenblat <bbaren@debian.org> Mon, 22 Aug 2022 22:17:36 -0400 + +abseil (0~20210324.2-4) unstable; urgency=medium + + * Fix "spurious -Wl flag in some pkg-config entries" by backporting a + patch from upstream that corrects CMake pkg-config generation. + (Closes: #1011294) + + -- Benjamin Barenblat <bbaren@debian.org> Fri, 27 May 2022 16:58:38 -0400 + +abseil (0~20210324.2-3) unstable; urgency=medium + + * Backport an upstream patch to disable a problematic unit test. + (Closes: #1007136) + * Reenable unit tests on hppa. + + -- Benjamin Barenblat <bbaren@debian.org> Thu, 14 Apr 2022 13:20:16 -0400 + abseil (0~20210324.2-2~bpo11+1) bullseye-backports-staging; urgency=medium * Rebuild for bullseye-backports-staging. diff --git a/debian/control b/debian/control index dde19493..b0038378 100644 --- a/debian/control +++ b/debian/control @@ -18,9 +18,9 @@ Maintainer: Benjamin Barenblat <bbaren@debian.org> Build-Depends: cmake (>= 3.5), debhelper-compat (= 12), - googletest (>= 1.10.0.20200926) [!hppa !mipsel !ppc64] + googletest (>= 1.12) [!mipsel !ppc64] <!nocheck> Rules-Requires-Root: no -Standards-Version: 4.6.0 +Standards-Version: 4.6.1 Section: libs Homepage: https://abseil.io/ Vcs-Browser: https://salsa.debian.org/debian/abseil @@ -37,7 +37,7 @@ Architecture: any Multi-Arch: same Section: libdevel Depends: - libabsl20210324 (= ${binary:Version}), + libabsl20220623 (= ${binary:Version}), ${misc:Depends}, Recommends: cmake (>= 2.6) | pkg-config, g++ (>= 5.1) Description: ${source:Synopsis} (development files) @@ -46,7 +46,7 @@ Description: ${source:Synopsis} (development files) This package contains header files and other data necessary for developing with Abseil. -Package: libabsl20210324 +Package: libabsl20220623 Architecture: any Multi-Arch: same Depends: ${shlibs:Depends}, ${misc:Depends} diff --git a/debian/copyright b/debian/copyright index 986a4261..260a1b8e 100644 --- a/debian/copyright +++ b/debian/copyright @@ -12,7 +12,7 @@ Copyright: License: Apache-2.0 Files: debian/* -Copyright: 2020 Google LLC +Copyright: 2020-2022 Google LLC License: Apache-2.0 License: Apache-2.0 diff --git a/debian/gbp.conf b/debian/gbp.conf index 64a42303..aea1f7a0 100644 --- a/debian/gbp.conf +++ b/debian/gbp.conf @@ -13,4 +13,4 @@ # the License. [DEFAULT] -upstream-tag = 20210324.2 +upstream-tag = 20220623.1 diff --git a/debian/libabsl20210324.lintian-overrides b/debian/libabsl20210324.lintian-overrides deleted file mode 100644 index 1ee8b551..00000000 --- a/debian/libabsl20210324.lintian-overrides +++ /dev/null @@ -1,40 +0,0 @@ -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. - -libabsl20210324: hardening-no-fortify-functions [usr/lib/*/libabsl_base.so*] -libabsl20210324: hardening-no-fortify-functions [usr/lib/*/libabsl_debugging_internal.so*] -libabsl20210324: hardening-no-fortify-functions [usr/lib/*/libabsl_random_internal_seed_material.so*] -libabsl20210324: library-not-linked-against-libc usr/lib/*/libabsl_bad_any_cast_impl.so* -libabsl20210324: library-not-linked-against-libc usr/lib/*/libabsl_bad_optional_access.so* -libabsl20210324: library-not-linked-against-libc usr/lib/*/libabsl_bad_variant_access.so* -libabsl20210324: library-not-linked-against-libc usr/lib/*/libabsl_exponential_biased.so* -libabsl20210324: library-not-linked-against-libc usr/lib/*/libabsl_flags_commandlineflag.so* -libabsl20210324: library-not-linked-against-libc usr/lib/*/libabsl_flags_commandlineflag_internal.so* -libabsl20210324: library-not-linked-against-libc usr/lib/*/libabsl_hash.so* -libabsl20210324: library-not-linked-against-libc usr/lib/*/libabsl_hashtablez_sampler.so* -libabsl20210324: library-not-linked-against-libc usr/lib/*/libabsl_log_severity.so* -libabsl20210324: library-not-linked-against-libc usr/lib/*/libabsl_periodic_sampler.so* -libabsl20210324: library-not-linked-against-libc usr/lib/*/libabsl_random_internal_randen.so* -libabsl20210324: library-not-linked-against-libc usr/lib/*/libabsl_random_internal_randen_hwaes_impl.so* -libabsl20210324: library-not-linked-against-libc usr/lib/*/libabsl_random_internal_randen_slow.so* -libabsl20210324: library-not-linked-against-libc usr/lib/*/libabsl_stacktrace.so* -libabsl20210324: library-not-linked-against-libc usr/lib/*/libabsl_throw_delegate.so* -libabsl20210324: no-symbols-control-file usr/lib/*/libabsl_* -libabsl20210324: package-name-doesnt-match-sonames libabsl-* -libabsl20210324: shared-library-lacks-prerequisites usr/lib/*/libabsl_city.so* -libabsl20210324: shared-library-lacks-prerequisites usr/lib/*/libabsl_leak_check.so* -libabsl20210324: shared-library-lacks-prerequisites usr/lib/*/libabsl_leak_check_disable.so* -libabsl20210324: shared-library-lacks-prerequisites usr/lib/*/libabsl_random_internal_platform.so* -libabsl20210324: shared-library-lacks-prerequisites usr/lib/*/libabsl_random_internal_randen_hwaes.so* -libabsl20210324: shared-library-lacks-prerequisites usr/lib/*/libabsl_wyhash.so* diff --git a/debian/libabsl20210324.shlibs b/debian/libabsl20210324.shlibs deleted file mode 100644 index f1ecca62..00000000 --- a/debian/libabsl20210324.shlibs +++ /dev/null @@ -1,74 +0,0 @@ -# Copyright 2020 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); you may not -# use this file except in compliance with the License. You may obtain a copy of -# the License at -# -# https://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -# License for the specific language governing permissions and limitations under -# the License. -# -libabsl_bad_any_cast_impl 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_bad_optional_access 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_bad_variant_access 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_base 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_city 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_civil_time 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_cord 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_debugging_internal 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_demangle_internal 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_examine_stack 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_exponential_biased 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_failure_signal_handler 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_flags_commandlineflag 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_flags_commandlineflag_internal 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_flags_config 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_flags_internal 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_flags_marshalling 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_flags_parse 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_flags_private_handle_accessor 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_flags_program_name 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_flags_reflection 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_flags_usage 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_flags_usage_internal 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_graphcycles_internal 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_hash 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_hashtablez_sampler 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_int128 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_leak_check 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_leak_check_disable 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_log_severity 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_malloc_internal 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_periodic_sampler 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_random_distributions 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_random_internal_distribution_test_util 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_random_internal_platform 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_random_internal_pool_urbg 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_random_internal_randen 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_random_internal_randen_hwaes 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_random_internal_randen_hwaes_impl 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_random_internal_randen_slow 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_random_internal_seed_material 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_random_seed_gen_exception 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_random_seed_sequences 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_raw_hash_set 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_raw_logging_internal 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_scoped_set_env 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_spinlock_wait 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_stacktrace 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_status 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_statusor 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_str_format_internal 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_strerror 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_strings 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_strings_internal 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_symbolize 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_synchronization 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_throw_delegate 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_time 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_time_zone 20210324 libabsl20210324 (>= 0~20210324.2-1) -libabsl_wyhash 20210324 libabsl20210324 (>= 0~20210324.2-1) diff --git a/debian/libabsl20210324.install b/debian/libabsl20220623.install index ab3017e5..ab3017e5 100644 --- a/debian/libabsl20210324.install +++ b/debian/libabsl20220623.install diff --git a/debian/libabsl20220623.lintian-overrides b/debian/libabsl20220623.lintian-overrides new file mode 100644 index 00000000..e00d857e --- /dev/null +++ b/debian/libabsl20220623.lintian-overrides @@ -0,0 +1,40 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +libabsl20220623: hardening-no-fortify-functions [usr/lib/*/libabsl_base.so*] +libabsl20220623: hardening-no-fortify-functions [usr/lib/*/libabsl_debugging_internal.so*] +libabsl20220623: hardening-no-fortify-functions [usr/lib/*/libabsl_random_internal_seed_material.so*] +libabsl20220623: hardening-no-fortify-functions [usr/lib/*/libabsl_time_zone.so*] +libabsl20220623: library-not-linked-against-libc [usr/lib/*/libabsl_bad_any_cast_impl.so*] +libabsl20220623: library-not-linked-against-libc [usr/lib/*/libabsl_bad_optional_access.so*] +libabsl20220623: library-not-linked-against-libc [usr/lib/*/libabsl_bad_variant_access.so*] +libabsl20220623: library-not-linked-against-libc [usr/lib/*/libabsl_cordz_functions.so*] +libabsl20220623: library-not-linked-against-libc [usr/lib/*/libabsl_cordz_sample_token.so*] +libabsl20220623: library-not-linked-against-libc [usr/lib/*/libabsl_exponential_biased.so*] +libabsl20220623: library-not-linked-against-libc [usr/lib/*/libabsl_flags_commandlineflag.so*] +libabsl20220623: library-not-linked-against-libc [usr/lib/*/libabsl_flags_commandlineflag_internal.so*] +libabsl20220623: library-not-linked-against-libc [usr/lib/*/libabsl_hash.so*] +libabsl20220623: library-not-linked-against-libc [usr/lib/*/libabsl_log_severity.so*] +libabsl20220623: library-not-linked-against-libc [usr/lib/*/libabsl_periodic_sampler.so*] +libabsl20220623: library-not-linked-against-libc [usr/lib/*/libabsl_random_internal_randen.so*] +libabsl20220623: library-not-linked-against-libc [usr/lib/*/libabsl_random_internal_randen_hwaes_impl.so*] +libabsl20220623: library-not-linked-against-libc [usr/lib/*/libabsl_random_internal_randen_slow.so*] +libabsl20220623: library-not-linked-against-libc [usr/lib/*/libabsl_throw_delegate.so*] +libabsl20220623: no-symbols-control-file usr/lib/*/libabsl_* +libabsl20220623: package-name-doesnt-match-sonames libabsl-* +libabsl20220623: shared-library-lacks-prerequisites [usr/lib/*/libabsl_city.so*] +libabsl20220623: shared-library-lacks-prerequisites [usr/lib/*/libabsl_leak_check.so*] +libabsl20220623: shared-library-lacks-prerequisites [usr/lib/*/libabsl_low_level_hash.so*] +libabsl20220623: shared-library-lacks-prerequisites [usr/lib/*/libabsl_random_internal_platform.so*] +libabsl20220623: shared-library-lacks-prerequisites [usr/lib/*/libabsl_random_internal_randen_hwaes.so*] diff --git a/debian/libabsl20220623.shlibs b/debian/libabsl20220623.shlibs new file mode 100644 index 00000000..b5e77c0c --- /dev/null +++ b/debian/libabsl20220623.shlibs @@ -0,0 +1,78 @@ +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. +# +libabsl_bad_any_cast_impl 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_bad_optional_access 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_bad_variant_access 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_base 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_city 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_civil_time 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_cord 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_cord_internal 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_cordz_functions 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_cordz_handle 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_cordz_info 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_cordz_sample_token 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_debugging_internal 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_demangle_internal 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_examine_stack 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_exponential_biased 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_failure_signal_handler 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_flags_commandlineflag 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_flags_commandlineflag_internal 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_flags_config 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_flags_internal 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_flags_marshalling 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_flags_parse 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_flags_private_handle_accessor 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_flags_program_name 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_flags_reflection 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_flags_usage 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_flags_usage_internal 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_graphcycles_internal 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_hash 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_hashtablez_sampler 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_int128 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_leak_check 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_log_severity 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_low_level_hash 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_malloc_internal 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_periodic_sampler 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_random_distributions 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_random_internal_distribution_test_util 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_random_internal_platform 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_random_internal_pool_urbg 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_random_internal_randen 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_random_internal_randen_hwaes 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_random_internal_randen_hwaes_impl 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_random_internal_randen_slow 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_random_internal_seed_material 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_random_seed_gen_exception 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_random_seed_sequences 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_raw_hash_set 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_raw_logging_internal 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_scoped_set_env 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_spinlock_wait 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_stacktrace 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_status 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_statusor 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_str_format_internal 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_strerror 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_strings 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_strings_internal 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_symbolize 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_synchronization 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_throw_delegate 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_time 20220623 libabsl20220623 (>= 0~20220623.0-1) +libabsl_time_zone 20220623 libabsl20220623 (>= 0~20220623.0-1) diff --git a/debian/patches/DiscreteDistributionTest-irrelevant-destination-buckets.diff b/debian/patches/DiscreteDistributionTest-irrelevant-destination-buckets.diff deleted file mode 100644 index f4cae4c7..00000000 --- a/debian/patches/DiscreteDistributionTest-irrelevant-destination-buckets.diff +++ /dev/null @@ -1,63 +0,0 @@ -From: Benjamin Barenblat <bbaren@google.com> -Subject: Don’t examine irrelevant destination buckets in DiscreteDistributionTest -Forwarded: yes -Applied-Upstream: https://github.com/abseil/abseil-cpp/commit/7ba826e50dff1878e6ecc6b9af44097c040c8968 - -Abseil generates discrete distributions using Walker’s aliasing -algorithm. This creates uniformly distributed buckets, each with a -probability of sending traffic to a different bucket. Abseil represents -a bucket as a pair - - (probability of retaining traffic × - alternate bucket if traffic is passed) - -and a distribution as a vector of such pairs. For example, {(0.3, 1), -(1.0, 1)} represents a distribution with two buckets, the zeroth of -which passes 70% of its traffic to bucket 1 and the first of which holds -on to all its traffic. - -This representation is not unique: When a bucket retains traffic with -probability 1, the alternate bucket is irrelevant. Continuing the -example above, {(0.3, 1), (1.0, 0)} _also_ represents a two-bucket -distribution where the zeroth bucket passes 70% of its traffic to the -first and the first hangs on to all traffic. Exactly what representation -Abseil generates for a given input is related to how much precision is -used in intermediate floating-point operations, which is an -architectural implementation detail. Remove sensitivity to that detail -by not examining the alternate bucket when the retention probability is -1.0. - -The author works at Google. Upstream applied this patch as Piper -revision 372993410 and exported it to GitHub; the Applied-Upstream URL -above points to the exported commit. - ---- a/absl/random/discrete_distribution_test.cc -+++ b/absl/random/discrete_distribution_test.cc -@@ -99,6 +99,7 @@ - } - - TEST(DiscreteDistributionTest, InitDiscreteDistribution) { -+ using testing::_; - using testing::Pair; - - { -@@ -111,8 +112,8 @@ TEST(DiscreteDistributionTest, InitDiscreteDistribution) { - // Each bucket is p=1/3, so bucket 0 will send half it's traffic - // to bucket 2, while the rest will retain all of their traffic. - EXPECT_THAT(q, testing::ElementsAre(Pair(0.5, 2), // -- Pair(1.0, 1), // -- Pair(1.0, 2))); -+ Pair(1.0, _), // -+ Pair(1.0, _))); - } - - { -@@ -135,7 +136,7 @@ TEST(DiscreteDistributionTest, InitDiscreteDistribution) { - - EXPECT_THAT(q, testing::ElementsAre(Pair(b0, 3), // - Pair(b1, 3), // -- Pair(1.0, 2), // -+ Pair(1.0, _), // - Pair(b3, 2), // - Pair(b1, 3))); - } diff --git a/debian/patches/arm-multiarch.diff b/debian/patches/arm-multiarch.diff deleted file mode 100644 index c976f730..00000000 --- a/debian/patches/arm-multiarch.diff +++ /dev/null @@ -1,25 +0,0 @@ -From: Benjamin Barenblat <bbaren@google.com> -Subject: Disable SysinfoTest.NominalCPUFrequency on armel/armhf - -NominalCPUFrequency has different behavior on 32-bit and 64-bit ARM -kernels. The Debian arm64 buildds assume they can build 32-bit ARM -packages, but if they do, the NominalCPUFrequency test will fail. -Disable the test when building for 32-bit ARM. - ---- a/absl/base/internal/sysinfo_test.cc -+++ b/absl/base/internal/sysinfo_test.cc -@@ -43,7 +43,13 @@ - // POWER is particularly problematic here; some Linux kernels expose the CPU - // frequency, while others do not. Since we can't predict a priori what a given - // machine is going to do, just disable this test on POWER on Linux. -+// -+// Debian also disables this test on armel and armhf, since tests for those -+// platforms could either be run on a 32-bit ARM system (where -+// NominalCPUFrequency returns a reasonable value) or a 64-bit ARM system (where -+// it does not). --#if !(defined(__linux) && (defined(__ppc64__) || defined(__PPC64__))) -+#if !(defined(__linux) && \ -+ (defined(__ppc64__) || defined(__PPC64__) || defined(__arm__))) - TEST(SysinfoTest, NominalCPUFrequency) { - // Linux only exposes the CPU frequency on certain architectures, and - // Emscripten doesn't expose it at all. diff --git a/debian/patches/big-endian-hash.diff b/debian/patches/big-endian-hash.diff deleted file mode 100644 index 52888e74..00000000 --- a/debian/patches/big-endian-hash.diff +++ /dev/null @@ -1,725 +0,0 @@ -From: Benjamin Barenblat <bbaren@google.com> -Subject: Restructure wyhash_test.cc to separate golden values -Forwarded: yes -Origin: backport, https://github.com/abseil/abseil-cpp/commit/a05366d851c5cb88065272f951e03955197e7c11 - -This patch is a subset of the referenced commit, which originally also changed -the mix function on arm64. The original message is as follows: - - Alternative bit mixer for LowLevelHash on ARM - - LowLevelHash's bit-mixer is inefficient on ARM because it calculates - a 128-bit product of two 64-bit numbers. On ARM, this requires a - sequence of two instructions with a high combined latency and poor - throughput. This change provides alternative bit-mixing code for ARM - that uses only 64-bit arithmetic (multiplication, xor, and - left-shifts) and speeds things up considerably. - - The bit-mixing code for ARM was inspired by by Woothash[1] and - xxh3[1]. Once I landed on a sequence of operations that provided - good mixing, I used a test harness to search for the combination of - shift / rotate factors that provided the best mixing, as indicated - by SMHasher hash quality tests. The new mixing code passes 13 out of - 15 of the hash quality test suites in SMHasher, with the two - failures being in the noise range: e.g. 1 collision vs. zero - expected in a keyset of ~8m keys. - - [1]: https://github.com/tommyettinger/waterhash/blob/49f5cf0b63b9/woothash.h#L16-L20 - [2]: https://github.com/Cyan4973/xxHash/blob/6853ddc36e46/xxhash.h#L3240-L3265 - - PiperOrigin-RevId: 391833008 - -This restructuring is a prerequisite for big-endian-hash2.diff. - ---- a/absl/hash/internal/wyhash_test.cc -+++ b/absl/hash/internal/wyhash_test.cc -@@ -14,473 +14,469 @@ - - #include "absl/hash/internal/wyhash.h" - -+#include <cinttypes> -+ --#include "absl/strings/escaping.h" - #include "gmock/gmock.h" - #include "gtest/gtest.h" -+#include "absl/strings/escaping.h" -+ -+#define UPDATE_GOLDEN 0 - - namespace { - --static const uint64_t kCurrentSeed = 0; - static const uint64_t kSalt[5] = {0xa0761d6478bd642f, 0xe7037ed1a0b428dbl, - 0x8ebc6af09c88c6e3, 0x589965cc75374cc3l, - 0x1d8e4e27c47d124f}; - --// Note: We don't account for endianness, so the values here are only correct if --// you're also running on a little endian platform. -- --TEST(WyhashTest, EmptyString) { -- const std::string s = ""; -- EXPECT_EQ( -- absl::hash_internal::Wyhash(s.c_str(), s.length(), kCurrentSeed, kSalt), -- 4808886099364463827); --} -- --TEST(WyhashTest, Spaces) { -- const std::string s = " "; -- EXPECT_EQ( -- absl::hash_internal::Wyhash(s.c_str(), s.length(), kCurrentSeed, kSalt), -- 1686201463024549249); --} -- --TEST(WyhashTest, RepeatingString) { -- const std::string s = "aaaa"; -- EXPECT_EQ( -- absl::hash_internal::Wyhash(s.c_str(), s.length(), kCurrentSeed, kSalt), -- 6646112255271966632); --} -- --TEST(WyhashTest, HexString) { -- const std::string small = "\x01\x02\x03"; -- const std::string med = "\x01\x02\x03\x04"; -- -- EXPECT_EQ(absl::hash_internal::Wyhash(small.c_str(), small.length(), -- kCurrentSeed, kSalt), -- 11989428023081740911ULL); -- EXPECT_EQ(absl::hash_internal::Wyhash(med.c_str(), med.length(), kCurrentSeed, -- kSalt), -- 9765997711188871556ULL); --} -- --TEST(WyhashTest, Words) { -- const std::string s = "third_party|wyhash|64"; -- EXPECT_EQ( -- absl::hash_internal::Wyhash(s.c_str(), s.length(), kCurrentSeed, kSalt), -- 3702018632387611330); --} -- --TEST(WyhashTest, LongString) { -- const std::string s = -- "AbCdEfGhIjKlMnOpQrStUvWxYz0123456789AbCdEfGhIjKlMnOpQrStUvWxYz" -- "0123456789AbCdEfGhIjKlMnOpQrStUvWxYz0123456789AbCdEfGhIjKlMnOp" -- "QrStUvWxYz0123456789AbCdEfGhIjKlMnOpQrStUvWxYz0123456789AbCdEf" -- "GhIjKlMnOpQrStUvWxYz0123456789AbCdEfGhIjKlMnOpQrStUvWxYz012345" -- "6789AbCdEfGhIjKlMnOpQrStUvWxYz0123456789"; -- -- EXPECT_EQ( -- absl::hash_internal::Wyhash(s.c_str(), s.length(), kCurrentSeed, kSalt), -- 9245411362605796064ULL); --} -- --TEST(WyhashTest, BigReference) { -+TEST(WyhashTest, VerifyGolden) { -+ constexpr size_t kNumGoldenOutputs = 134; -- struct ExpectedResult { -+ static struct { - absl::string_view base64_data; - uint64_t seed; -- uint64_t hash; -- } expected_results[] = { -+ } cases[] = { -- {"", uint64_t{0xec42b7ab404b8acb}, uint64_t{0xe5a40d39ab796423}}, -+ {"", uint64_t{0xec42b7ab404b8acb}}, -+ {"ICAg", uint64_t{0}}, -+ {"YWFhYQ==", uint64_t{0}}, -+ {"AQID", uint64_t{0}}, -+ {"AQIDBA==", uint64_t{0}}, -+ {"dGhpcmRfcGFydHl8d3loYXNofDY0", uint64_t{0}}, -- {"Zw==", uint64_t{0xeeee074043a3ee0f}, uint64_t{0xa6564b468248c683}}, -+ {"Zw==", uint64_t{0xeeee074043a3ee0f}}, -- {"xmk=", uint64_t{0x857902089c393de}, uint64_t{0xef192f401b116e1c}}, -+ {"xmk=", uint64_t{0x857902089c393de}}, -- {"c1H/", uint64_t{0x993df040024ca3af}, uint64_t{0xbe8dc0c54617639d}}, -+ {"c1H/", uint64_t{0x993df040024ca3af}}, -- {"SuwpzQ==", uint64_t{0xc4e4c2acea740e96}, uint64_t{0x93d7f665b5521c8e}}, -+ {"SuwpzQ==", uint64_t{0xc4e4c2acea740e96}}, -- {"uqvy++M=", uint64_t{0x6a214b3db872d0cf}, uint64_t{0x646d70bb42445f28}}, -+ {"uqvy++M=", uint64_t{0x6a214b3db872d0cf}}, -- {"RnzCVPgb", uint64_t{0x44343db6a89dba4d}, uint64_t{0x96a7b1e3cc9bd426}}, -+ {"RnzCVPgb", uint64_t{0x44343db6a89dba4d}}, -- {"6OeNdlouYw==", uint64_t{0x77b5d6d1ae1dd483}, -- uint64_t{0x76020289ab0790c4}}, -+ {"6OeNdlouYw==", uint64_t{0x77b5d6d1ae1dd483}}, -- {"M5/JmmYyDbc=", uint64_t{0x89ab8ecb44d221f1}, -- uint64_t{0x39f842e4133b9b44}}, -+ {"M5/JmmYyDbc=", uint64_t{0x89ab8ecb44d221f1}}, -- {"MVijWiVdBRdY", uint64_t{0x60244b17577ca81b}, -- uint64_t{0x2b8d7047be4bcaab}}, -+ {"MVijWiVdBRdY", uint64_t{0x60244b17577ca81b}}, -- {"6V7Uq7LNxpu0VA==", uint64_t{0x59a08dcee0717067}, -- uint64_t{0x99628abef6716a97}}, -+ {"6V7Uq7LNxpu0VA==", uint64_t{0x59a08dcee0717067}}, -- {"EQ6CdEEhPdyHcOk=", uint64_t{0xf5f20db3ade57396}, -- uint64_t{0x4432e02ba42b2740}}, -+ {"EQ6CdEEhPdyHcOk=", uint64_t{0xf5f20db3ade57396}}, -- {"PqFB4fxnPgF+l+rc", uint64_t{0xbf8dee0751ad3efb}, -- uint64_t{0x74d810efcad7918a}}, -+ {"PqFB4fxnPgF+l+rc", uint64_t{0xbf8dee0751ad3efb}}, -- {"a5aPOFwq7LA7+zKvPA==", uint64_t{0x6b7a06b268d63e30}, -- uint64_t{0x88c84e986002507f}}, -+ {"a5aPOFwq7LA7+zKvPA==", uint64_t{0x6b7a06b268d63e30}}, -- {"VOwY21wCGv5D+/qqOvs=", uint64_t{0xb8c37f0ae0f54c82}, -- uint64_t{0x4f99acf193cf39b9}}, -+ {"VOwY21wCGv5D+/qqOvs=", uint64_t{0xb8c37f0ae0f54c82}}, -- {"KdHmBTx8lHXYvmGJ+Vy7", uint64_t{0x9fcbed0c38e50eef}, -- uint64_t{0xd90e7a3655891e37}}, -+ {"KdHmBTx8lHXYvmGJ+Vy7", uint64_t{0x9fcbed0c38e50eef}}, -- {"qJkPlbHr8bMF7/cA6aE65Q==", uint64_t{0x2af4bade1d8e3a1d}, -- uint64_t{0x3bb378b1d4df8fcf}}, -+ {"qJkPlbHr8bMF7/cA6aE65Q==", uint64_t{0x2af4bade1d8e3a1d}}, -- {"ygvL0EhHZL0fIx6oHHtkxRQ=", uint64_t{0x714e3aa912da2f2c}, -- uint64_t{0xf78e94045c052d47}}, -+ {"ygvL0EhHZL0fIx6oHHtkxRQ=", uint64_t{0x714e3aa912da2f2c}}, -- {"c1rFXkt5YztwZCQRngncqtSs", uint64_t{0xf5ee75e3cbb82c1c}, -- uint64_t{0x26da0b2130da6b40}}, -+ {"c1rFXkt5YztwZCQRngncqtSs", uint64_t{0xf5ee75e3cbb82c1c}}, -- {"8hsQrzszzeNQSEcVXLtvIhm6mw==", uint64_t{0x620e7007321b93b9}, -- uint64_t{0x30b4d426af8c6986}}, -+ {"8hsQrzszzeNQSEcVXLtvIhm6mw==", uint64_t{0x620e7007321b93b9}}, -- {"ffUL4RocfyP4KfikGxO1yk7omDI=", uint64_t{0xc08528cac2e551fc}, -- uint64_t{0x5413b4aaf3baaeae}}, -+ {"ffUL4RocfyP4KfikGxO1yk7omDI=", uint64_t{0xc08528cac2e551fc}}, -- {"OOB5TT00vF9Od/rLbAWshiErqhpV", uint64_t{0x6a1debf9cc3ad39}, -- uint64_t{0x756ab265370a1597}}, -+ {"OOB5TT00vF9Od/rLbAWshiErqhpV", uint64_t{0x6a1debf9cc3ad39}}, -- {"or5wtXM7BFzTNpSzr+Lw5J5PMhVJ/Q==", uint64_t{0x7e0a3c88111fc226}, -- uint64_t{0xdaf5f4b7d09814fb}}, -+ {"or5wtXM7BFzTNpSzr+Lw5J5PMhVJ/Q==", uint64_t{0x7e0a3c88111fc226}}, -- {"gk6pCHDUsoopVEiaCrzVDhioRKxb844=", uint64_t{0x1301fef15df39edb}, -- uint64_t{0x8f874ae37742b75e}}, -+ {"gk6pCHDUsoopVEiaCrzVDhioRKxb844=", uint64_t{0x1301fef15df39edb}}, -- {"TNctmwlC5QbEM6/No4R/La3UdkfeMhzs", uint64_t{0x64e181f3d5817ab}, -- uint64_t{0x8fecd03956121ce8}}, -+ {"TNctmwlC5QbEM6/No4R/La3UdkfeMhzs", uint64_t{0x64e181f3d5817ab}}, -- {"SsQw9iAjhWz7sgcE9OwLuSC6hsM+BfHs2Q==", uint64_t{0xafafc44961078ecb}, -- uint64_t{0x229c292ea7a08285}}, -+ {"SsQw9iAjhWz7sgcE9OwLuSC6hsM+BfHs2Q==", uint64_t{0xafafc44961078ecb}}, -- {"ZzO3mVCj4xTT2TT3XqDyEKj2BZQBvrS8RHg=", uint64_t{0x4f7bb45549250094}, -- uint64_t{0xbb4bf0692d14bae}}, -+ {"ZzO3mVCj4xTT2TT3XqDyEKj2BZQBvrS8RHg=", uint64_t{0x4f7bb45549250094}}, -- {"+klp5iPQGtppan5MflEls0iEUzqU+zGZkDJX", uint64_t{0xa30061abaa2818c}, -- uint64_t{0x207b24ca3bdac1db}}, -+ {"+klp5iPQGtppan5MflEls0iEUzqU+zGZkDJX", uint64_t{0xa30061abaa2818c}}, -- {"RO6bvOnlJc8I9eniXlNgqtKy0IX6VNg16NRmgg==", uint64_t{0xd902ee3e44a5705f}, -- uint64_t{0x64f6cd6745d3825b}}, -+ {"RO6bvOnlJc8I9eniXlNgqtKy0IX6VNg16NRmgg==", -+ uint64_t{0xd902ee3e44a5705f}}, -- {"ZJjZqId1ZXBaij9igClE3nyliU5XWdNRrayGlYA=", uint64_t{0x316d36da516f583}, -- uint64_t{0xa2b2e1656b58df1e}}, -+ {"ZJjZqId1ZXBaij9igClE3nyliU5XWdNRrayGlYA=", uint64_t{0x316d36da516f583}}, -- {"7BfkhfGMDGbxfMB8uyL85GbaYQtjr2K8g7RpLzr/", uint64_t{0x402d83f9f834f616}, -- uint64_t{0xd01d30d9ee7a148}}, -+ {"7BfkhfGMDGbxfMB8uyL85GbaYQtjr2K8g7RpLzr/", -+ uint64_t{0x402d83f9f834f616}}, - {"rycWk6wHH7htETQtje9PidS2YzXBx+Qkg2fY7ZYS7A==", -- uint64_t{0x9c604164c016b72c}, uint64_t{0x1cb4cd00ab804e3b}}, -+ uint64_t{0x9c604164c016b72c}}, - {"RTkC2OUK+J13CdGllsH0H5WqgspsSa6QzRZouqx6pvI=", -- uint64_t{0x3f4507e01f9e73ba}, uint64_t{0x4697f2637fd90999}}, -+ uint64_t{0x3f4507e01f9e73ba}}, - {"tKjKmbLCNyrLCM9hycOAXm4DKNpM12oZ7dLTmUx5iwAi", -- uint64_t{0xc3fe0d5be8d2c7c7}, uint64_t{0x8383a756b5688c07}}, -+ uint64_t{0xc3fe0d5be8d2c7c7}}, - {"VprUGNH+5NnNRaORxgH/ySrZFQFDL+4VAodhfBNinmn8cg==", -- uint64_t{0x531858a40bfa7ea1}, uint64_t{0x695c29cb3696a975}}, -+ uint64_t{0x531858a40bfa7ea1}}, - {"gc1xZaY+q0nPcUvOOnWnT3bqfmT/geth/f7Dm2e/DemMfk4=", -- uint64_t{0x86689478a7a7e8fa}, uint64_t{0xda2e5a5a5e971521}}, -+ uint64_t{0x86689478a7a7e8fa}}, - {"Mr35fIxqx1ukPAL0su1yFuzzAU3wABCLZ8+ZUFsXn47UmAph", -- uint64_t{0x4ec948b8e7f27288}, uint64_t{0x7935d4befa056b2b}}, -+ uint64_t{0x4ec948b8e7f27288}}, - {"A9G8pw2+m7+rDtWYAdbl8tb2fT7FFo4hLi2vAsa5Y8mKH3CX3g==", -- uint64_t{0xce46c7213c10032}, uint64_t{0x38dd541ca95420fe}}, -+ uint64_t{0xce46c7213c10032}}, - {"DFaJGishGwEHDdj9ixbCoaTjz9KS0phLNWHVVdFsM93CvPft3hM=", -- uint64_t{0xf63e96ee6f32a8b6}, uint64_t{0xcc06c7a4963f967f}}, -+ uint64_t{0xf63e96ee6f32a8b6}}, - {"7+Ugx+Kr3aRNgYgcUxru62YkTDt5Hqis+2po81hGBkcrJg4N0uuy", -- uint64_t{0x1cfe85e65fc5225}, uint64_t{0xbf0f6f66e232fb20}}, -+ uint64_t{0x1cfe85e65fc5225}}, - {"H2w6O8BUKqu6Tvj2xxaecxEI2wRgIgqnTTG1WwOgDSINR13Nm4d4Vg==", -- uint64_t{0x45c474f1cee1d2e8}, uint64_t{0xf7efb32d373fe71a}}, -+ uint64_t{0x45c474f1cee1d2e8}}, - {"1XBMnIbqD5jy65xTDaf6WtiwtdtQwv1dCVoqpeKj+7cTR1SaMWMyI04=", -- uint64_t{0x6e024e14015f329c}, uint64_t{0xe2e64634b1c12660}}, -+ uint64_t{0x6e024e14015f329c}}, - {"znZbdXG2TSFrKHEuJc83gPncYpzXGbAebUpP0XxzH0rpe8BaMQ17nDbt", -- uint64_t{0x760c40502103ae1c}, uint64_t{0x285b8fd1638e306d}}, -+ uint64_t{0x760c40502103ae1c}}, - {"ylu8Atu13j1StlcC1MRMJJXIl7USgDDS22HgVv0WQ8hx/8pNtaiKB17hCQ==", -- uint64_t{0x17fd05c3c560c320}, uint64_t{0x658e8a4e3b714d6c}}, -+ uint64_t{0x17fd05c3c560c320}}, - {"M6ZVVzsd7vAvbiACSYHioH/440dp4xG2mLlBnxgiqEvI/aIEGpD0Sf4VS0g=", -- uint64_t{0x8b34200a6f8e90d9}, uint64_t{0xf391fb968e0eb398}}, -+ uint64_t{0x8b34200a6f8e90d9}}, - {"li3oFSXLXI+ubUVGJ4blP6mNinGKLHWkvGruun85AhVn6iuMtocbZPVhqxzn", -- uint64_t{0x6be89e50818bdf69}, uint64_t{0x744a9ea0cc144bf2}}, -+ uint64_t{0x6be89e50818bdf69}}, - {"kFuQHuUCqBF3Tc3hO4dgdIp223ShaCoog48d5Do5zMqUXOh5XpGK1t5XtxnfGA==", -- uint64_t{0xfb389773315b47d8}, uint64_t{0x12636f2be11012f1}}, -+ uint64_t{0xfb389773315b47d8}}, - {"jWmOad0v0QhXVJd1OdGuBZtDYYS8wBVHlvOeTQx9ZZnm8wLEItPMeihj72E0nWY=", -- uint64_t{0x4f2512a23f61efee}, uint64_t{0x29c57de825948f80}}, -+ uint64_t{0x4f2512a23f61efee}}, - {"z+DHU52HaOQdW4JrZwDQAebEA6rm13Zg/9lPYA3txt3NjTBqFZlOMvTRnVzRbl23", -- uint64_t{0x59ccd92fc16c6fda}, uint64_t{0x58c6f99ab0d1c021}}, -+ uint64_t{0x59ccd92fc16c6fda}}, - {"MmBiGDfYeTayyJa/tVycg+rN7f9mPDFaDc+23j0TlW9094er0ADigsl4QX7V3gG/qw==", -- uint64_t{0x25c5a7f5bd330919}, uint64_t{0x13e7b5a7b82fe3bb}}, -+ uint64_t{0x25c5a7f5bd330919}}, - {"774RK+9rOL4iFvs1q2qpo/JVc/I39buvNjqEFDtDvyoB0FXxPI2vXqOrk08VPfIHkmU=", -- uint64_t{0x51df4174d34c97d7}, uint64_t{0x10fbc87901e02b63}}, -+ uint64_t{0x51df4174d34c97d7}}, - {"+slatXiQ7/2lK0BkVUI1qzNxOOLP3I1iK6OfHaoxgqT63FpzbElwEXSwdsryq3UlHK0I", -- uint64_t{0x80ce6d76f89cb57}, uint64_t{0xa24c9184901b748b}}, -+ uint64_t{0x80ce6d76f89cb57}}, - {"64mVTbQ47dHjHlOHGS/hjJwr/" - "K2frCNpn87exOqMzNUVYiPKmhCbfS7vBUce5tO6Ec9osQ==", -- uint64_t{0x20961c911965f684}, uint64_t{0xcac4fd4c5080e581}}, -+ uint64_t{0x20961c911965f684}}, - {"fIsaG1r530SFrBqaDj1kqE0AJnvvK8MNEZbII2Yw1OK77v0V59xabIh0B5axaz/" - "+a2V5WpA=", -- uint64_t{0x4e5b926ec83868e7}, uint64_t{0xc38bdb7483ba68e1}}, -+ uint64_t{0x4e5b926ec83868e7}}, - {"PGih0zDEOWCYGxuHGDFu9Ivbff/" - "iE7BNUq65tycTR2R76TerrXALRosnzaNYO5fjFhTi+CiS", -- uint64_t{0x3927b30b922eecef}, uint64_t{0xdb2a8069b2ceaffa}}, -+ uint64_t{0x3927b30b922eecef}}, - {"RnpA/" - "zJnEnnLjmICORByRVb9bCOgxF44p3VMiW10G7PvW7IhwsWajlP9kIwNA9FjAD2GoQHk2Q=" - "=", -- uint64_t{0xbd0291284a49b61c}, uint64_t{0xdf9fe91d0d1c7887}}, -+ uint64_t{0xbd0291284a49b61c}}, - {"qFklMceaTHqJpy2qavJE+EVBiNFOi6OxjOA3LeIcBop1K7w8xQi3TrDk+" - "BrWPRIbfprszSaPfrI=", -- uint64_t{0x73a77c575bcc956}, uint64_t{0xe83f49e96e2e6a08}}, -+ uint64_t{0x73a77c575bcc956}}, - {"cLbfUtLl3EcQmITWoTskUR8da/VafRDYF/ylPYwk7/" - "zazk6ssyrzxMN3mmSyvrXR2yDGNZ3WDrTT", -- uint64_t{0x766a0e2ade6d09a6}, uint64_t{0xc69e61b62ca2b62}}, -+ uint64_t{0x766a0e2ade6d09a6}}, - {"s/" - "Jf1+" - "FbsbCpXWPTUSeWyMH6e4CvTFvPE5Fs6Z8hvFITGyr0dtukHzkI84oviVLxhM1xMxrMAy1db" - "w==", -- uint64_t{0x2599f4f905115869}, uint64_t{0xb4a4f3f85f8298fe}}, -+ uint64_t{0x2599f4f905115869}}, - {"FvyQ00+j7nmYZVQ8hI1Edxd0AWplhTfWuFGiu34AK5X8u2hLX1bE97sZM0CmeLe+" - "7LgoUT1fJ/axybE=", -- uint64_t{0xd8256e5444d21e53}, uint64_t{0x167a1b39e1e95f41}}, -+ uint64_t{0xd8256e5444d21e53}}, - {"L8ncxMaYLBH3g9buPu8hfpWZNlOF7nvWLNv9IozH07uQsIBWSKxoPy8+" - "LW4tTuzC6CIWbRGRRD1sQV/4", -- uint64_t{0xf664a91333fb8dfd}, uint64_t{0xf8a2a5649855ee41}}, -+ uint64_t{0xf664a91333fb8dfd}}, - {"CDK0meI07yrgV2kQlZZ+" - "wuVqhc2NmzqeLH7bmcA6kchsRWFPeVF5Wqjjaj556ABeUoUr3yBmfU3kWOakkg==", -- uint64_t{0x9625b859be372cd1}, uint64_t{0x27992565b595c498}}, -+ uint64_t{0x9625b859be372cd1}}, - {"d23/vc5ONh/" - "HkMiq+gYk4gaCNYyuFKwUkvn46t+dfVcKfBTYykr4kdvAPNXGYLjM4u1YkAEFpJP+" - "nX7eOvs=", -- uint64_t{0x7b99940782e29898}, uint64_t{0x3e08cca5b71f9346}}, -+ uint64_t{0x7b99940782e29898}}, - {"NUR3SRxBkxTSbtQORJpu/GdR6b/h6sSGfsMj/KFd99ahbh+9r7LSgSGmkGVB/" - "mGoT0pnMTQst7Lv2q6QN6Vm", -- uint64_t{0x4fe12fa5383b51a8}, uint64_t{0xad406b10c770a6d2}}, -+ uint64_t{0x4fe12fa5383b51a8}}, - {"2BOFlcI3Z0RYDtS9T9Ie9yJoXlOdigpPeeT+CRujb/" - "O39Ih5LPC9hP6RQk1kYESGyaLZZi3jtabHs7DiVx/VDg==", -- uint64_t{0xe2ccb09ac0f5b4b6}, uint64_t{0xd1713ce6e552bcf2}}, -+ uint64_t{0xe2ccb09ac0f5b4b6}}, - {"FF2HQE1FxEvWBpg6Z9zAMH+Zlqx8S1JD/" - "wIlViL6ZDZY63alMDrxB0GJQahmAtjlm26RGLnjW7jmgQ4Ie3I+014=", -- uint64_t{0x7d0a37adbd7b753b}, uint64_t{0x753b287194c73ad3}}, -+ uint64_t{0x7d0a37adbd7b753b}}, - {"tHmO7mqVL/PX11nZrz50Hc+M17Poj5lpnqHkEN+4bpMx/" - "YGbkrGOaYjoQjgmt1X2QyypK7xClFrjeWrCMdlVYtbW", -- uint64_t{0xd3ae96ef9f7185f2}, uint64_t{0x5ae41a95f600af1c}}, -+ uint64_t{0xd3ae96ef9f7185f2}}, - {"/WiHi9IQcxRImsudkA/KOTqGe8/" - "gXkhKIHkjddv5S9hi02M049dIK3EUyAEjkjpdGLUs+BN0QzPtZqjIYPOgwsYE9g==", -- uint64_t{0x4fb88ea63f79a0d8}, uint64_t{0x4a61163b86a8bb4c}}, -+ uint64_t{0x4fb88ea63f79a0d8}}, - {"qds+1ExSnU11L4fTSDz/QE90g4Jh6ioqSh3KDOTOAo2pQGL1k/" - "9CCC7J23YF27dUTzrWsCQA2m4epXoCc3yPHb3xElA=", -- uint64_t{0xed564e259bb5ebe9}, uint64_t{0x42eeaa79e760c7e4}}, -+ uint64_t{0xed564e259bb5ebe9}}, - {"8FVYHx40lSQPTHheh08Oq0/" - "pGm2OlG8BEf8ezvAxHuGGdgCkqpXIueJBF2mQJhTfDy5NncO8ntS7vaKs7sCNdDaNGOEi", -- uint64_t{0x3e3256b60c428000}, uint64_t{0x698df622ef465b0a}}, -+ uint64_t{0x3e3256b60c428000}}, - {"4ZoEIrJtstiCkeew3oRzmyJHVt/pAs2pj0HgHFrBPztbQ10NsQ/" - "lM6DM439QVxpznnBSiHMgMQJhER+70l72LqFTO1JiIQ==", -- uint64_t{0xfb05bad59ec8705}, uint64_t{0x157583111e1a6026}}, -+ uint64_t{0xfb05bad59ec8705}}, - {"hQPtaYI+wJyxXgwD5n8jGIKFKaFA/" - "P83KqCKZfPthnjwdOFysqEOYwAaZuaaiv4cDyi9TyS8hk5cEbNP/jrI7q6pYGBLbsM=", -- uint64_t{0xafdc251dbf97b5f8}, uint64_t{0xaa1388f078e793e0}}, -+ uint64_t{0xafdc251dbf97b5f8}}, - {"S4gpMSKzMD7CWPsSfLeYyhSpfWOntyuVZdX1xSBjiGvsspwOZcxNKCRIOqAA0moUfOh3I5+" - "juQV4rsqYElMD/gWfDGpsWZKQ", -- uint64_t{0x10ec9c92ddb5dcbc}, uint64_t{0xf10d68d0f3309360}}, -+ uint64_t{0x10ec9c92ddb5dcbc}}, - {"oswxop+" - "bthuDLT4j0PcoSKby4LhF47ZKg8K17xxHf74UsGCzTBbOz0MM8hQEGlyqDT1iUiAYnaPaUp" - "L2mRK0rcIUYA4qLt5uOw==", -- uint64_t{0x9a767d5822c7dac4}, uint64_t{0x2af056184457a3de}}, -+ uint64_t{0x9a767d5822c7dac4}}, - {"0II/" - "697p+" - "BtLSjxj5989OXI004TogEb94VUnDzOVSgMXie72cuYRvTFNIBgtXlKfkiUjeqVpd4a+" - "n5bxNOD1TGrjQtzKU5r7obo=", -- uint64_t{0xee46254080d6e2db}, uint64_t{0x6d0058e1590b2489}}, -+ uint64_t{0xee46254080d6e2db}}, - {"E84YZW2qipAlMPmctrg7TKlwLZ68l4L+c0xRDUfyyFrA4MAti0q9sHq3TDFviH0Y+" - "Kq3tEE5srWFA8LM9oomtmvm5PYxoaarWPLc", -- uint64_t{0xbbb669588d8bf398}, uint64_t{0x638f287f68817f12}}, -+ uint64_t{0xbbb669588d8bf398}}, - {"x3pa4HIElyZG0Nj7Vdy9IdJIR4izLmypXw5PCmZB5y68QQ4uRaVVi3UthsoJROvbjDJkP2D" - "Q6L/eN8pFeLFzNPKBYzcmuMOb5Ull7w==", -- uint64_t{0xdc2afaa529beef44}, uint64_t{0xc46b71fecefd5467}}, -+ uint64_t{0xdc2afaa529beef44}}, - {"jVDKGYIuWOP/" - "QKLdd2wi8B2VJA8Wh0c8PwrXJVM8FOGM3voPDVPyDJOU6QsBDPseoR8uuKd19OZ/" - "zAvSCB+zlf6upAsBlheUKgCfKww=", -- uint64_t{0xf1f67391d45013a8}, uint64_t{0x2c8e94679d964e0a}}, -+ uint64_t{0xf1f67391d45013a8}}, - {"mkquunhmYe1aR2wmUz4vcvLEcKBoe6H+kjUok9VUn2+eTSkWs4oDDtJvNCWtY5efJwg/" - "j4PgjRYWtqnrCkhaqJaEvkkOwVfgMIwF3e+d", -- uint64_t{0x16fce2b8c65a3429}, uint64_t{0x8612b797ce22503a}}, -+ uint64_t{0x16fce2b8c65a3429}}, - {"fRelvKYonTQ+s+rnnvQw+JzGfFoPixtna0vzcSjiDqX5s2Kg2//" - "UGrK+AVCyMUhO98WoB1DDbrsOYSw2QzrcPe0+3ck9sePvb+Q/IRaHbw==", -- uint64_t{0xf4b096699f49fe67}, uint64_t{0x59f929babfba7170}}, -+ uint64_t{0xf4b096699f49fe67}}, - {"DUwXFJzagljo44QeJ7/" - "6ZKw4QXV18lhkYT2jglMr8WB3CHUU4vdsytvw6AKv42ZcG6fRkZkq9fpnmXy6xG0aO3WPT1" - "eHuyFirAlkW+zKtwg=", -- uint64_t{0xca584c4bc8198682}, uint64_t{0x9527556923fb49a0}}, -+ uint64_t{0xca584c4bc8198682}}, - {"cYmZCrOOBBongNTr7e4nYn52uQUy2mfe48s50JXx2AZ6cRAt/" - "xRHJ5QbEoEJOeOHsJyM4nbzwFm++SlT6gFZZHJpkXJ92JkR86uS/eV1hJUR", -- uint64_t{0xed269fc3818b6aad}, uint64_t{0x1039ab644f5e150b}}, -+ uint64_t{0xed269fc3818b6aad}}, - {"EXeHBDfhwzAKFhsMcH9+2RHwV+mJaN01+9oacF6vgm8mCXRd6jeN9U2oAb0of5c5cO4i+" - "Vb/LlHZSMI490SnHU0bejhSCC2gsC5d2K30ER3iNA==", -- uint64_t{0x33f253cbb8fe66a8}, uint64_t{0x7816c83f3aa05e6d}}, -+ uint64_t{0x33f253cbb8fe66a8}}, - {"FzkzRYoNjkxFhZDso94IHRZaJUP61nFYrh5MwDwv9FNoJ5jyNCY/" - "eazPZk+tbmzDyJIGw2h3GxaWZ9bSlsol/vK98SbkMKCQ/wbfrXRLcDzdd/8=", -- uint64_t{0xd0b76b2c1523d99c}, uint64_t{0xf51d2f564518c619}}, -+ uint64_t{0xd0b76b2c1523d99c}}, - {"Re4aXISCMlYY/XsX7zkIFR04ta03u4zkL9dVbLXMa/q6hlY/CImVIIYRN3VKP4pnd0AUr/" - "ugkyt36JcstAInb4h9rpAGQ7GMVOgBniiMBZ/MGU7H", -- uint64_t{0xfd28f0811a2a237f}, uint64_t{0x67d494cff03ac004}}, -+ uint64_t{0xfd28f0811a2a237f}}, - {"ueLyMcqJXX+MhO4UApylCN9WlTQ+" - "ltJmItgG7vFUtqs2qNwBMjmAvr5u0sAKd8jpzV0dDPTwchbIeAW5zbtkA2NABJV6hFM48ib" - "4/J3A5mseA3cS8w==", -- uint64_t{0x6261fb136482e84}, uint64_t{0x2802d636ced1cfbb}}, -+ uint64_t{0x6261fb136482e84}}, - {"6Si7Yi11L+jZMkwaN+GUuzXMrlvEqviEkGOilNq0h8TdQyYKuFXzkYc/" - "q74gP3pVCyiwz9KpVGMM9vfnq36riMHRknkmhQutxLZs5fbmOgEO69HglCU=", -- uint64_t{0x458efc750bca7c3a}, uint64_t{0xf64e20bad771cb12}}, -+ uint64_t{0x458efc750bca7c3a}}, - {"Q6AbOofGuTJOegPh9Clm/" - "9crtUMQqylKrTc1fhfJo1tqvpXxhU4k08kntL1RG7woRnFrVh2UoMrL1kjin+s9CanT+" - "y4hHwLqRranl9FjvxfVKm3yvg68", -- uint64_t{0xa7e69ff84e5e7c27}, uint64_t{0xb9a6cf84a83e15e}}, -+ uint64_t{0xa7e69ff84e5e7c27}}, - {"ieQEbIPvqY2YfIjHnqfJiO1/MIVRk0RoaG/WWi3kFrfIGiNLCczYoklgaecHMm/" - "1sZ96AjO+a5stQfZbJQwS7Sc1ODABEdJKcTsxeW2hbh9A6CFzpowP1A==", -- uint64_t{0x3c59bfd0c29efe9e}, uint64_t{0x8da6630319609301}}, -+ uint64_t{0x3c59bfd0c29efe9e}}, - {"zQUv8hFB3zh2GGl3KTvCmnfzE+" - "SUgQPVaSVIELFX5H9cE3FuVFGmymkPQZJLAyzC90Cmi8GqYCvPqTuAAB//" - "XTJxy4bCcVArgZG9zJXpjowpNBfr3ngWrSE=", -- uint64_t{0x10befacc6afd298d}, uint64_t{0x40946a86e2a996f3}}, -+ uint64_t{0x10befacc6afd298d}}, - {"US4hcC1+op5JKGC7eIs8CUgInjKWKlvKQkapulxW262E/" - "B2ye79QxOexf188u2mFwwe3WTISJHRZzS61IwljqAWAWoBAqkUnW8SHmIDwHUP31J0p5sGd" - "P47L", -- uint64_t{0x41d5320b0a38efa7}, uint64_t{0xcab7f5997953fa76}}, -+ uint64_t{0x41d5320b0a38efa7}}, - {"9bHUWFna2LNaGF6fQLlkx1Hkt24nrkLE2CmFdWgTQV3FFbUe747SSqYw6ebpTa07MWSpWRP" - "sHesVo2B9tqHbe7eQmqYebPDFnNqrhSdZwFm9arLQVs+7a3Ic6A==", -- uint64_t{0x58db1c7450fe17f3}, uint64_t{0x39129ca0e04fc465}}, -+ uint64_t{0x58db1c7450fe17f3}}, - {"Kb3DpHRUPhtyqgs3RuXjzA08jGb59hjKTOeFt1qhoINfYyfTt2buKhD6YVffRCPsgK9SeqZ" - "qRPJSyaqsa0ovyq1WnWW8jI/NhvAkZTVHUrX2pC+cD3OPYT05Dag=", -- uint64_t{0x6098c055a335b7a6}, uint64_t{0x5238221fd685e1b8}}, -+ uint64_t{0x6098c055a335b7a6}}, - {"gzxyMJIPlU+bJBwhFUCHSofZ/" - "319LxqMoqnt3+L6h2U2+ZXJCSsYpE80xmR0Ta77Jq54o92SMH87HV8dGOaCTuAYF+" - "lDL42SY1P316Cl0sZTS2ow3ZqwGbcPNs/1", -- uint64_t{0x1bbacec67845a801}, uint64_t{0x175130c407dbcaab}}, -+ uint64_t{0x1bbacec67845a801}}, - {"uR7V0TW+FGVMpsifnaBAQ3IGlr1wx5sKd7TChuqRe6OvUXTlD4hKWy8S+" - "8yyOw8lQabism19vOQxfmocEOW/" - "vzY0pEa87qHrAZy4s9fH2Bltu8vaOIe+agYohhYORQ==", -- uint64_t{0xc419cfc7442190}, uint64_t{0x2f20e7536c0b0df}}, -+ uint64_t{0xc419cfc7442190}}, - {"1UR5eoo2aCwhacjZHaCh9bkOsITp6QunUxHQ2SfeHv0imHetzt/" - "Z70mhyWZBalv6eAx+YfWKCUib2SHDtz/" - "A2dc3hqUWX5VfAV7FQsghPUAtu6IiRatq4YSLpDvKZBQ=", -- uint64_t{0xc95e510d94ba270c}, uint64_t{0x2742cb488a04ad56}}, -+ uint64_t{0xc95e510d94ba270c}}, - {"opubR7H63BH7OtY+Avd7QyQ25UZ8kLBdFDsBTwZlY6gA/" - "u+x+" - "czC9AaZMgmQrUy15DH7YMGsvdXnviTtI4eVI4aF1H9Rl3NXMKZgwFOsdTfdcZeeHVRzBBKX" - "8jUfh1il", -- uint64_t{0xff1ae05c98089c3f}, uint64_t{0xd6afb593879ff93b}}, -+ uint64_t{0xff1ae05c98089c3f}}, - {"DC0kXcSXtfQ9FbSRwirIn5tgPri0sbzHSa78aDZVDUKCMaBGyFU6BmrulywYX8yzvwprdLs" - "oOwTWN2wMjHlPDqrvVHNEjnmufRDblW+nSS+xtKNs3N5xsxXdv6JXDrAB/Q==", -- uint64_t{0x90c02b8dceced493}, uint64_t{0xf50ad64caac0ca7f}}, -+ uint64_t{0x90c02b8dceced493}}, - {"BXRBk+3wEP3Lpm1y75wjoz+PgB0AMzLe8tQ1AYU2/" - "oqrQB2YMC6W+9QDbcOfkGbeH+b7IBkt/" - "gwCMw2HaQsRFEsurXtcQ3YwRuPz5XNaw5NAvrNa67Fm7eRzdE1+hWLKtA8=", -- uint64_t{0x9f8a76697ab1aa36}, uint64_t{0x2ade95c4261364ae}}, -+ uint64_t{0x9f8a76697ab1aa36}}, - {"RRBSvEGYnzR9E45Aps/+WSnpCo/X7gJLO4DRnUqFrJCV/kzWlusLE/" - "6ZU6RoUf2ROwcgEvUiXTGjLs7ts3t9SXnJHxC1KiOzxHdYLMhVvgNd3hVSAXODpKFSkVXND" - "55G2L1W", -- uint64_t{0x6ba1bf3d811a531d}, uint64_t{0x5c4f3299faacd07a}}, -+ uint64_t{0x6ba1bf3d811a531d}}, - {"jeh6Qazxmdi57pa9S3XSnnZFIRrnc6s8QLrah5OX3SB/V2ErSPoEAumavzQPkdKF1/" - "SfvmdL+qgF1C+Yawy562QaFqwVGq7+tW0yxP8FStb56ZRgNI4IOmI30s1Ei7iops9Uuw==", -- uint64_t{0x6a418974109c67b4}, uint64_t{0xfffe3bff0ae5e9bc}}, -+ uint64_t{0x6a418974109c67b4}}, - {"6QO5nnDrY2/" - "wrUXpltlKy2dSBcmK15fOY092CR7KxAjNfaY+" - "aAmtWbbzQk3MjBg03x39afSUN1fkrWACdyQKRaGxgwq6MGNxI6W+8DLWJBHzIXrntrE/" - "ml6fnNXEpxplWJ1vEs4=", -- uint64_t{0x8472f1c2b3d230a3}, uint64_t{0x1db785c0005166e4}}, -+ uint64_t{0x8472f1c2b3d230a3}}, - {"0oPxeEHhqhcFuwonNfLd5jF3RNATGZS6NPoS0WklnzyokbTqcl4BeBkMn07+fDQv83j/" - "BpGUwcWO05f3+DYzocfnizpFjLJemFGsls3gxcBYxcbqWYev51tG3lN9EvRE+X9+Pwww", -- uint64_t{0x5e06068f884e73a7}, uint64_t{0xea000d962ad18418}}, -+ uint64_t{0x5e06068f884e73a7}}, - {"naSBSjtOKgAOg8XVbR5cHAW3Y+QL4Pb/JO9/" - "oy6L08wvVRZqo0BrssMwhzBP401Um7A4ppAupbQeJFdMrysY34AuSSNvtNUy5VxjNECwiNt" - "gwYHw7yakDUv8WvonctmnoSPKENegQg==", -- uint64_t{0x55290b1a8f170f59}, uint64_t{0xe42aef38359362d9}}, -+ uint64_t{0x55290b1a8f170f59}}, - {"vPyl8DxVeRe1OpilKb9KNwpGkQRtA94UpAHetNh+" - "95V7nIW38v7PpzhnTWIml5kw3So1Si0TXtIUPIbsu32BNhoH7QwFvLM+" - "JACgSpc5e3RjsL6Qwxxi11npwxRmRUqATDeMUfRAjxg=", -- uint64_t{0x5501cfd83dfe706a}, uint64_t{0xc8e95657348a3891}}, -+ uint64_t{0x5501cfd83dfe706a}}, - {"QC9i2GjdTMuNC1xQJ74ngKfrlA4w3o58FhvNCltdIpuMhHP1YsDA78scQPLbZ3OCUgeQguY" - "f/vw6zAaVKSgwtaykqg5ka/4vhz4hYqWU5ficdXqClHl+zkWEY26slCNYOM5nnDlly8Cj", -- uint64_t{0xe43ed13d13a66990}, uint64_t{0xc162eca864f238c6}}, -+ uint64_t{0xe43ed13d13a66990}}, - {"7CNIgQhAHX27nxI0HeB5oUTnTdgKpRDYDKwRcXfSFGP1XeT9nQF6WKCMjL1tBV6x7KuJ91G" - "Zz11F4c+8s+MfqEAEpd4FHzamrMNjGcjCyrVtU6y+7HscMVzr7Q/" - "ODLcPEFztFnwjvCjmHw==", -- uint64_t{0xdf43bc375cf5283f}, uint64_t{0xbe1fb373e20579ad}}, -+ uint64_t{0xdf43bc375cf5283f}}, - {"Qa/hC2RPXhANSospe+gUaPfjdK/yhQvfm4cCV6/pdvCYWPv8p1kMtKOX3h5/" - "8oZ31fsmx4Axphu5qXJokuhZKkBUJueuMpxRyXpwSWz2wELx5glxF7CM0Fn+" - "OevnkhUn5jsPlG2r5jYlVn8=", -- uint64_t{0x8112b806d288d7b5}, uint64_t{0x628a1d4f40aa6ffd}}, -+ uint64_t{0x8112b806d288d7b5}}, - {"kUw/0z4l3a89jTwN5jpG0SHY5km/" - "IVhTjgM5xCiPRLncg40aqWrJ5vcF891AOq5hEpSq0bUCJUMFXgct7kvnys905HjerV7Vs1G" - "y84tgVJ70/2+pAZTsB/PzNOE/G6sOj4+GbTzkQu819OLB", -- uint64_t{0xd52a18abb001cb46}, uint64_t{0xa87bdb7456340f90}}, -+ uint64_t{0xd52a18abb001cb46}}, - {"VDdfSDbO8Tdj3T5W0XM3EI7iHh5xpIutiM6dvcJ/fhe23V/srFEkDy5iZf/" - "VnA9kfi2C79ENnFnbOReeuZW1b3MUXB9lgC6U4pOTuC+" - "jHK3Qnpyiqzj7h3ISJSuo2pob7vY6VHZo6Fn7exEqHg==", -- uint64_t{0xe12b76a2433a1236}, uint64_t{0x5960ef3ba982c801}}, -+ uint64_t{0xe12b76a2433a1236}}, - {"Ldfvy3ORdquM/R2fIkhH/ONi69mcP1AEJ6n/" - "oropwecAsLJzQSgezSY8bEiEs0VnFTBBsW+RtZY6tDj03fnb3amNUOq1b7jbqyQkL9hpl+" - "2Z2J8IaVSeownWl+bQcsR5/xRktIMckC5AtF4YHfU=", -- uint64_t{0x175bf7319cf1fa00}, uint64_t{0x5026586df9a431ec}}, -+ uint64_t{0x175bf7319cf1fa00}}, - {"BrbNpb42+" - "VzZAjJw6QLirXzhweCVRfwlczzZ0VX2xluskwBqyfnGovz5EuX79JJ31VNXa5hTkAyQat3l" - "YKRADTdAdwE5PqM1N7YaMqqsqoAAAeuYVXuk5eWCykYmClNdSspegwgCuT+403JigBzi", -- uint64_t{0xd63d57b3f67525ae}, uint64_t{0xfe4b8a20fdf0840b}}, -+ uint64_t{0xd63d57b3f67525ae}}, - {"gB3NGHJJvVcuPyF0ZSvHwnWSIfmaI7La24VMPQVoIIWF7Z74NltPZZpx2f+cocESM+" - "ILzQW9p+BC8x5IWz7N4Str2WLGKMdgmaBfNkEhSHQDU0IJEOnpUt0HmjhFaBlx0/" - "LTmhua+rQ6Wup8ezLwfg==", -- uint64_t{0x933faea858832b73}, uint64_t{0xdcb761867da7072f}}, -+ uint64_t{0x933faea858832b73}}, - {"hTKHlRxx6Pl4gjG+6ksvvj0CWFicUg3WrPdSJypDpq91LUWRni2KF6+" - "81ZoHBFhEBrCdogKqeK+hy9bLDnx7g6rAFUjtn1+cWzQ2YjiOpz4+" - "ROBB7lnwjyTGWzJD1rXtlso1g2qVH8XJVigC5M9AIxM=", -- uint64_t{0x53d061e5f8e7c04f}, uint64_t{0xc10d4653667275b7}}, -+ uint64_t{0x53d061e5f8e7c04f}}, - {"IWQBelSQnhrr0F3BhUpXUIDauhX6f95Qp+A0diFXiUK7irwPG1oqBiqHyK/SH/" - "9S+" - "rln9DlFROAmeFdH0OCJi2tFm4afxYzJTFR4HnR4cG4x12JqHaZLQx6iiu6CE3rtWBVz99oA" - "wCZUOEXIsLU24o2Y", -- uint64_t{0xdb4124556dd515e0}, uint64_t{0x727720deec13110b}}, -+ uint64_t{0xdb4124556dd515e0}}, - {"TKo+l+" - "1dOXdLvIrFqeLaHdm0HZnbcdEgOoLVcGRiCbAMR0j5pIFw8D36tefckAS1RCFOH5IgP8yiF" - "T0Gd0a2hI3+" - "fTKA7iK96NekxWeoeqzJyctc6QsoiyBlkZerRxs5RplrxoeNg29kKDTM0K94mnhD9g==", -- uint64_t{0x4fb31a0dd681ee71}, uint64_t{0x710b009662858dc9}}, -+ uint64_t{0x4fb31a0dd681ee71}}, - {"YU4e7G6EfQYvxCFoCrrT0EFgVLHFfOWRTJQJ5gxM3G2b+" - "1kJf9YPrpsxF6Xr6nYtS8reEEbDoZJYqnlk9lXSkVArm88Cqn6d25VCx3+" - "49MqC0trIlXtb7SXUUhwpJK16T0hJUfPH7s5cMZXc6YmmbFuBNPE=", -- uint64_t{0x27cc72eefa138e4c}, uint64_t{0xfbf8f7a3ecac1eb7}}, -+ uint64_t{0x27cc72eefa138e4c}}, - {"/I/" - "eImMwPo1U6wekNFD1Jxjk9XQVi1D+" - "FPdqcHifYXQuP5aScNQfxMAmaPR2XhuOQhADV5tTVbBKwCDCX4E3jcDNHzCiPvViZF1W27t" - "xaf2BbFQdwKrNCmrtzcluBFYu0XZfc7RU1RmxK/RtnF1qHsq/O4pp", -- uint64_t{0x44bc2dfba4bd3ced}, uint64_t{0xb6fc4fcd0722e3df}}, -+ uint64_t{0x44bc2dfba4bd3ced}}, - {"CJTT9WGcY2XykTdo8KodRIA29qsqY0iHzWZRjKHb9alwyJ7RZAE3V5Juv4MY3MeYEr1EPCC" - "MxO7yFXqT8XA8YTjaMp3bafRt17Pw8JC4iKJ1zN+WWKOESrj+" - "3aluGQqn8z1EzqY4PH7rLG575PYeWsP98BugdA==", -- uint64_t{0x242da1e3a439bed8}, uint64_t{0x7cb86dcc55104aac}}, -+ uint64_t{0x242da1e3a439bed8}}, - {"ZlhyQwLhXQyIUEnMH/" - "AEW27vh9xrbNKJxpWGtrEmKhd+nFqAfbeNBQjW0SfG1YI0xQkQMHXjuTt4P/" - "EpZRtA47ibZDVS8TtaxwyBjuIDwqcN09eCtpC+Ls+" - "vWDTLmBeDM3u4hmzz4DQAYsLiZYSJcldg9Q3wszw=", -- uint64_t{0xdc559c746e35c139}, uint64_t{0x19e71e9b45c3a51e}}, -+ uint64_t{0xdc559c746e35c139}}, - {"v2KU8y0sCrBghmnm8lzGJlwo6D6ObccAxCf10heoDtYLosk4ztTpLlpSFEyu23MLA1tJkcg" - "Rko04h19QMG0mOw/" - "wc93EXAweriBqXfvdaP85sZABwiKO+6rtS9pacRVpYYhHJeVTQ5NzrvBvi1huxAr+" - "xswhVMfL", -- uint64_t{0xd0b0350275b9989}, uint64_t{0x51de38573c2bea48}}, -+ uint64_t{0xd0b0350275b9989}}, - {"QhKlnIS6BuVCTQsnoE67E/" - "yrgogE8EwO7xLaEGei26m0gEU4OksefJgppDh3X0x0Cs78Dr9IHK5b977CmZlrTRmwhlP8p" - "M+UzXPNRNIZuN3ntOum/QhUWP8SGpirheXENWsXMQ/" - "nxtxakyEtrNkKk471Oov9juP8oQ==", -- uint64_t{0xb04489e41d17730c}, uint64_t{0xa73ab6996d6df158}}, -+ uint64_t{0xb04489e41d17730c}}, - {"/ZRMgnoRt+Uo6fUPr9FqQvKX7syhgVqWu+" - "WUSsiQ68UlN0efSP6Eced5gJZL6tg9gcYJIkhjuQNITU0Q3TjVAnAcobgbJikCn6qZ6pRxK" - "BY4MTiAlfGD3T7R7hwJwx554MAy++Zb/YUFlnCaCJiwQMnowF7aQzwYFCo=", -- uint64_t{0x2217285eb4572156}, uint64_t{0x55ef2b8c930817b2}}, -+ uint64_t{0x2217285eb4572156}}, - {"NB7tU5fNE8nI+SXGfipc7sRkhnSkUF1krjeo6k+8FITaAtdyz+" - "o7mONgXmGLulBPH9bEwyYhKNVY0L+njNQrZ9YC2aXsFD3PdZsxAFaBT3VXEzh+" - "NGBTjDASNL3mXyS8Yv1iThGfHoY7T4aR0NYGJ+k+pR6f+KrPC96M", -- uint64_t{0x12c2e8e68aede73b}, uint64_t{0xb2850bf5fae87157}}, -+ uint64_t{0x12c2e8e68aede73b}}, - {"8T6wrqCtEO6/rwxF6lvMeyuigVOLwPipX/FULvwyu+1wa5sQGav/" - "2FsLHUVn6cGSi0LlFwLewGHPFJDLR0u4t7ZUyM//" - "x6da0sWgOa5hzDqjsVGmjxEHXiaXKW3i4iSZNuxoNbMQkIbVML+" - "DkYu9ND0O2swg4itGeVSzXA==", -- uint64_t{0x4d612125bdc4fd00}, uint64_t{0xecf3de1acd04651f}}, -+ uint64_t{0x4d612125bdc4fd00}}, - {"Ntf1bMRdondtMv1CYr3G80iDJ4WSAlKy5H34XdGruQiCrnRGDBa+" - "eUi7vKp4gp3BBcVGl8eYSasVQQjn7MLvb3BjtXx6c/" - "bCL7JtpzQKaDnPr9GWRxpBXVxKREgMM7d8lm35EODv0w+" - "hQLfVSh8OGs7fsBb68nNWPLeeSOo=", -- uint64_t{0x81826b553954464e}, uint64_t{0xcc0a40552559ff32}}, -+ uint64_t{0x81826b553954464e}}, - {"VsSAw72Ro6xks02kaiLuiTEIWBC5bgqr4WDnmP8vglXzAhixk7td926rm9jNimL+" - "kroPSygZ9gl63aF5DCPOACXmsbmhDrAQuUzoh9ZKhWgElLQsrqo1KIjWoZT5b5QfVUXY9lS" - "IBg3U75SqORoTPq7HalxxoIT5diWOcJQi", -- uint64_t{0xc2e5d345dc0ddd2d}, uint64_t{0xc385c374f20315b1}}, -+ uint64_t{0xc2e5d345dc0ddd2d}}, - {"j+loZ+C87+" - "bJxNVebg94gU0mSLeDulcHs84tQT7BZM2rzDSLiCNxUedHr1ZWJ9ejTiBa0dqy2I2ABc++" - "xzOLcv+//YfibtjKtYggC6/3rv0XCc7xu6d/" - "O6xO+XOBhOWAQ+IHJVHf7wZnDxIXB8AUHsnjEISKj7823biqXjyP3g==", -- uint64_t{0x3da6830a9e32631e}, uint64_t{0xb90208a4c7234183}}, -+ uint64_t{0x3da6830a9e32631e}}, - {"f3LlpcPElMkspNtDq5xXyWU62erEaKn7RWKlo540gR6mZsNpK1czV/" - "sOmqaq8XAQLEn68LKj6/" - "cFkJukxRzCa4OF1a7cCAXYFp9+wZDu0bw4y63qbpjhdCl8GO6Z2lkcXy7KOzbPE01ukg7+" - "gN+7uKpoohgAhIwpAKQXmX5xtd0=", -- uint64_t{0xc9ae5c8759b4877a}, uint64_t{0x58aa1ca7a4c075d9}}, -+ uint64_t{0xc9ae5c8759b4877a}}, -+ }; -+ -+ constexpr uint64_t kGolden[kNumGoldenOutputs] = { -+ 0xe5a40d39ab796423, 0x1766974bf7527d81, 0x5c3bbbe230db17a8, -+ 0xa6630143a7e6aa6f, 0x8787cb2d04b0c984, 0x33603654ff574ac2, -+ 0xa6564b468248c683, 0xef192f401b116e1c, 0xbe8dc0c54617639d, -+ 0x93d7f665b5521c8e, 0x646d70bb42445f28, 0x96a7b1e3cc9bd426, -+ 0x76020289ab0790c4, 0x39f842e4133b9b44, 0x2b8d7047be4bcaab, -+ 0x99628abef6716a97, 0x4432e02ba42b2740, 0x74d810efcad7918a, -+ 0x88c84e986002507f, 0x4f99acf193cf39b9, 0xd90e7a3655891e37, -+ 0x3bb378b1d4df8fcf, 0xf78e94045c052d47, 0x26da0b2130da6b40, -+ 0x30b4d426af8c6986, 0x5413b4aaf3baaeae, 0x756ab265370a1597, -+ 0xdaf5f4b7d09814fb, 0x8f874ae37742b75e, 0x8fecd03956121ce8, -+ 0x229c292ea7a08285, 0x0bb4bf0692d14bae, 0x207b24ca3bdac1db, -+ 0x64f6cd6745d3825b, 0xa2b2e1656b58df1e, 0x0d01d30d9ee7a148, -+ 0x1cb4cd00ab804e3b, 0x4697f2637fd90999, 0x8383a756b5688c07, -+ 0x695c29cb3696a975, 0xda2e5a5a5e971521, 0x7935d4befa056b2b, -+ 0x38dd541ca95420fe, 0xcc06c7a4963f967f, 0xbf0f6f66e232fb20, -+ 0xf7efb32d373fe71a, 0xe2e64634b1c12660, 0x285b8fd1638e306d, -+ 0x658e8a4e3b714d6c, 0xf391fb968e0eb398, 0x744a9ea0cc144bf2, -+ 0x12636f2be11012f1, 0x29c57de825948f80, 0x58c6f99ab0d1c021, -+ 0x13e7b5a7b82fe3bb, 0x10fbc87901e02b63, 0xa24c9184901b748b, -+ 0xcac4fd4c5080e581, 0xc38bdb7483ba68e1, 0xdb2a8069b2ceaffa, -+ 0xdf9fe91d0d1c7887, 0xe83f49e96e2e6a08, 0x0c69e61b62ca2b62, -+ 0xb4a4f3f85f8298fe, 0x167a1b39e1e95f41, 0xf8a2a5649855ee41, -+ 0x27992565b595c498, 0x3e08cca5b71f9346, 0xad406b10c770a6d2, -+ 0xd1713ce6e552bcf2, 0x753b287194c73ad3, 0x5ae41a95f600af1c, -+ 0x4a61163b86a8bb4c, 0x42eeaa79e760c7e4, 0x698df622ef465b0a, -+ 0x157583111e1a6026, 0xaa1388f078e793e0, 0xf10d68d0f3309360, -+ 0x2af056184457a3de, 0x6d0058e1590b2489, 0x638f287f68817f12, -+ 0xc46b71fecefd5467, 0x2c8e94679d964e0a, 0x8612b797ce22503a, -+ 0x59f929babfba7170, 0x9527556923fb49a0, 0x1039ab644f5e150b, -+ 0x7816c83f3aa05e6d, 0xf51d2f564518c619, 0x67d494cff03ac004, -+ 0x2802d636ced1cfbb, 0xf64e20bad771cb12, 0x0b9a6cf84a83e15e, -+ 0x8da6630319609301, 0x40946a86e2a996f3, 0xcab7f5997953fa76, -+ 0x39129ca0e04fc465, 0x5238221fd685e1b8, 0x175130c407dbcaab, -+ 0x02f20e7536c0b0df, 0x2742cb488a04ad56, 0xd6afb593879ff93b, -+ 0xf50ad64caac0ca7f, 0x2ade95c4261364ae, 0x5c4f3299faacd07a, -+ 0xfffe3bff0ae5e9bc, 0x1db785c0005166e4, 0xea000d962ad18418, -+ 0xe42aef38359362d9, 0xc8e95657348a3891, 0xc162eca864f238c6, -+ 0xbe1fb373e20579ad, 0x628a1d4f40aa6ffd, 0xa87bdb7456340f90, -+ 0x5960ef3ba982c801, 0x5026586df9a431ec, 0xfe4b8a20fdf0840b, -+ 0xdcb761867da7072f, 0xc10d4653667275b7, 0x727720deec13110b, -+ 0x710b009662858dc9, 0xfbf8f7a3ecac1eb7, 0xb6fc4fcd0722e3df, -+ 0x7cb86dcc55104aac, 0x19e71e9b45c3a51e, 0x51de38573c2bea48, -+ 0xa73ab6996d6df158, 0x55ef2b8c930817b2, 0xb2850bf5fae87157, -+ 0xecf3de1acd04651f, 0xcc0a40552559ff32, 0xc385c374f20315b1, -+ 0xb90208a4c7234183, 0x58aa1ca7a4c075d9, - }; - -- for (const auto& expected_result : expected_results) { -+#if UPDATE_GOLDEN -+ (void)kGolden; // Silence warning. -+ for (size_t i = 0; i < kNumGoldenOutputs; ++i) { -+ std::string str; -+ ASSERT_TRUE(absl::Base64Unescape(cases[i].base64_data, &str)); -+ uint64_t h = absl::hash_internal::Wyhash(str.data(), str.size(), -+ cases[i].seed, kSalt); -+ printf("0x%016" PRIx64 ", ", h); -+ if (i % 3 == 2) { -+ printf("\n"); -+ } -+ } -+ printf("\n\n\n"); -+ EXPECT_FALSE(true); -+#else -+ for (size_t i = 0; i < kNumGoldenOutputs; ++i) { -+ SCOPED_TRACE(::testing::Message() -+ << "i = " << i << "; input = " << cases[i].base64_data); - std::string str; -- ASSERT_TRUE(absl::Base64Unescape(expected_result.base64_data, &str)); -- EXPECT_EQ(absl::hash_internal::Wyhash(str.data(), str.size(), -- expected_result.seed, kSalt), -- expected_result.hash); -+ ASSERT_TRUE(absl::Base64Unescape(cases[i].base64_data, &str)); -+ EXPECT_EQ(absl::hash_internal::Wyhash(str.data(), str.size(), cases[i].seed, -+ kSalt), -+ kGolden[i]); - } -+#endif - } - - } // namespace diff --git a/debian/patches/big-endian-hash2.diff b/debian/patches/big-endian-hash2.diff deleted file mode 100644 index 69ffa8a5..00000000 --- a/debian/patches/big-endian-hash2.diff +++ /dev/null @@ -1,164 +0,0 @@ -From: Milad Fa <46688537+miladfarca@users.noreply.github.com> -Subject: Fix hashing on big endian platforms (#1028) -Forwarded: https://github.com/abseil/abseil-cpp/pull/1028 -Origin: backport, https://github.com/abseil/abseil-cpp/commit/ae0f4c266095c9003786cd571bc1fb72544104a1 -Bug-Debian: https://bugs.debian.org/977638 - -Avoid using libstdc++'s implementation of std::hash<std::bitset> and -std::hash<std::vector> on big endian platforms in the implementation -of absl::Hash. - -This is a workaround for a buggy implementation that results in many -collisions. - -https://gcc.gnu.org/bugzilla/show_bug.cgi?id=102531 -https://gcc.gnu.org/bugzilla/show_bug.cgi?id=98731 - ---- a/absl/hash/internal/hash.h -+++ b/absl/hash/internal/hash.h -@@ -21,6 +21,7 @@ - - #include <algorithm> - #include <array> -+#include <bitset> - #include <cmath> - #include <cstring> - #include <deque> -@@ -489,8 +490,9 @@ - - // AbslHashValue for hashing std::vector - // --// Do not use this for vector<bool>. It does not have a .data(), and a fallback --// for std::hash<> is most likely faster. -+// Do not use this for vector<bool> on platforms that have a working -+// implementation of std::hash. It does not have a .data(), and a fallback for -+// std::hash<> is most likely faster. - template <typename H, typename T, typename Allocator> - typename std::enable_if<is_hashable<T>::value && !std::is_same<T, bool>::value, - H>::type -@@ -500,6 +502,27 @@ - vector.size()); - } - -+#if defined(ABSL_IS_BIG_ENDIAN) && \ -+ (defined(__GLIBCXX__) || defined(__GLIBCPP__)) -+// AbslHashValue for hashing std::vector<bool> -+// -+// std::hash in libstdc++ does not work correctly with vector<bool> on Big -+// Endian platforms therefore we need to implement a custom AbslHashValue for -+// it. More details on the bug: -+// https://gcc.gnu.org/bugzilla/show_bug.cgi?id=102531 -+template <typename H, typename T, typename Allocator> -+typename std::enable_if<is_hashable<T>::value && std::is_same<T, bool>::value, -+ H>::type -+AbslHashValue(H hash_state, const std::vector<T, Allocator>& vector) { -+ typename H::AbslInternalPiecewiseCombiner combiner; -+ for (const auto& i : vector) { -+ unsigned char c = static_cast<unsigned char>(i); -+ hash_state = combiner.add_buffer(std::move(hash_state), &c, sizeof(c)); -+ } -+ return H::combine(combiner.finalize(std::move(hash_state)), vector.size()); -+} -+#endif -+ - // ----------------------------------------------------------------------------- - // AbslHashValue for Ordered Associative Containers - // ----------------------------------------------------------------------------- -@@ -592,9 +615,28 @@ - // AbslHashValue for Other Types - // ----------------------------------------------------------------------------- - --// AbslHashValue for hashing std::bitset is not defined, for the same reason as --// for vector<bool> (see std::vector above): It does not expose the raw bytes, --// and a fallback to std::hash<> is most likely faster. -+// AbslHashValue for hashing std::bitset is not defined on Little Endian -+// platforms, for the same reason as for vector<bool> (see std::vector above): -+// It does not expose the raw bytes, and a fallback to std::hash<> is most -+// likely faster. -+ -+#if defined(ABSL_IS_BIG_ENDIAN) && \ -+ (defined(__GLIBCXX__) || defined(__GLIBCPP__)) -+// AbslHashValue for hashing std::bitset -+// -+// std::hash in libstdc++ does not work correctly with std::bitset on Big Endian -+// platforms therefore we need to implement a custom AbslHashValue for it. More -+// details on the bug: https://gcc.gnu.org/bugzilla/show_bug.cgi?id=102531 -+template <typename H, size_t N> -+H AbslHashValue(H hash_state, const std::bitset<N>& set) { -+ typename H::AbslInternalPiecewiseCombiner combiner; -+ for (int i = 0; i < N; i++) { -+ unsigned char c = static_cast<unsigned char>(set[i]); -+ hash_state = combiner.add_buffer(std::move(hash_state), &c, sizeof(c)); -+ } -+ return H::combine(combiner.finalize(std::move(hash_state)), N); -+} -+#endif - - // ----------------------------------------------------------------------------- - ---- a/absl/hash/internal/wyhash_test.cc -+++ b/absl/hash/internal/wyhash_test.cc -@@ -404,6 +404,55 @@ - uint64_t{0xc9ae5c8759b4877a}}, - }; - -+#if defined(ABSL_IS_BIG_ENDIAN) -+ constexpr uint64_t kGolden[kNumGoldenOutputs] = { -+ 0xe5a40d39ab796423, 0x1766974bf7527d81, 0x5c3bbbe230db17a8, -+ 0xa6630143a7e6aa6f, 0x17645cb7318b86b, 0x218b175f30ba61f8, -+ 0xa6564b468248c683, 0xef192f401b116e1c, 0xbe8dc0c54617639d, -+ 0xe7b01610fc22dbb8, 0x99d9f694404af913, 0xf4eecd37464b45c5, -+ 0x7d2c653d63596d9b, 0x3f15c8544ec5393a, 0x6b9dc0c1704f796c, -+ 0xf1ded7a7eae5ed5a, 0x2db2fd7c6dd4641b, 0x151ca2d3d4cd33ab, -+ 0xa5af5994ac2ccd64, 0x2b2a4ca3191d2fce, 0xf89e68c9364e7c05, -+ 0x71724c70b799c21, 0x70536fabfd157369, 0xdee92794c3c3082b, -+ 0xac033a6743d3b3eb, 0xed2956b506cd5151, 0xbd669644755264b6, -+ 0x6ab1ff5d5f549a63, 0xf6bd551a2e3e04e, 0x7b5a8cef6875ea73, -+ 0x22bccf4d4db0a91c, 0x4f2bc07754c7c7eb, 0xfb6b8342a86725db, -+ 0x13a1a0d4c5854da, 0x5f6e44655f7dedac, 0x54a9198dff2bdf85, -+ 0xdb17e6915d4e4042, 0xa69926cf5c3b89f, 0xf77f031bfd74c096, -+ 0x1d6f916fdd50ec3c, 0x334ac76013ade393, 0x99370f899111de15, -+ 0x352457a03ada6de, 0x341974d4f42d854d, 0xda89ab02872aeb5, -+ 0x6ec2b74e143b10d9, 0x6f284c0b5cd60522, 0xf9670de353438f88, -+ 0xde920913adf0a2b4, 0xb7a07d7c0c17a8ec, 0x879a69f558ba3a98, -+ 0x360cf6d802df20f9, 0x53530f8046673738, 0xbd8f5f2bcf35e483, -+ 0x3f171f047144b983, 0x644d04e820823465, 0x50e44773a20b2702, -+ 0xe584ed4c05c745dd, 0x9a825c85b95ab6c0, 0xbce2931deb74e775, -+ 0x10468e9e705c7cfe, 0x12e01de3104141e2, 0x5c11ae2ee3713abd, -+ 0x6ac5ffb0860319e6, 0xc1e6da1849d30fc9, 0xa0e4d247a458b447, -+ 0x4530d4615c32b89b, 0x116aa09107a76505, 0xf941339d00d9bb73, -+ 0x573a0fc1615afb33, 0xa975c81dc868b258, 0x3ab2c5250ab54bda, -+ 0x37f99f208a3e3b11, 0x4b49b0ff706689d, 0x30bafa0b8f0a87fe, -+ 0xea6787a65cc20cdd, 0x55861729f1fc3ab8, 0xea38e009c5be9b72, -+ 0xcb8522cba33c3c66, 0x352e77653fe306f3, 0xe0bb760793bac064, -+ 0xf66ec59322662956, 0x637aa320455d56f8, 0x46ee546be5824a89, -+ 0x9e6842421e83d8a4, 0xf98ac2bc96b9fb8c, 0xf2c1002fd9a70b99, -+ 0x4c2b62b1e39e9405, 0x3248555fa3ade9c4, 0xd4d04c37f6417c21, -+ 0xf40cd506b1bf5653, 0x6c45d6005c760d2f, 0x61d88a7e61ff0d7e, -+ 0x131591e8a53cc967, 0xdae85cb9bc29bab6, 0xe98835334905e626, -+ 0x7cce50a2b66b8754, 0x5b0b3d0c5ac498ae, 0xd35a218c974d1756, -+ 0xfce436ddc1d003c, 0xd183901de90bb741, 0x9378f8f34974a66, -+ 0x21f11ae0a0402368, 0xf2fbd7c94ef89cb6, 0xc329c69d0f0d080b, -+ 0xf2841cba16216a61, 0x47aba97b44916df1, 0x724d4e00a8019fcf, -+ 0x2df9005c2a728d63, 0xc788892a1a5d7515, 0x9e993a65f9df0480, -+ 0x76876721ff49f969, 0xbe7a796cfba15bf5, 0xa4c8bd54586f5488, -+ 0xb390a325275501ab, 0x893f11317427ccf1, 0x92f2bb57da5695b9, -+ 0x30985b90da88269f, 0x2c690e268e086de8, 0x1c02df6097997196, -+ 0x1f9778f8bbdf6455, 0x7d57378c7bf8416d, 0xba8582a5f8d84d38, -+ 0xe8ca43b85050be4e, 0x5048cf6bed8a5d9f, 0xfbc5ba80917d0ea4, -+ 0x8011026525bf1691, 0x26b8dc6aed9fb50d, 0x191f5bfee77c1fe3, -+ 0xdd497891465a2cc1, 0x6f1fe8c57a33072e, 0x2c9f4ec078c460c0, -+ 0x9a725bde8f6a1437, 0x6ce545fa3ef61e4d, -+ }; -+#else - constexpr uint64_t kGolden[kNumGoldenOutputs] = { - 0xe5a40d39ab796423, 0x1766974bf7527d81, 0x5c3bbbe230db17a8, - 0xa6630143a7e6aa6f, 0x8787cb2d04b0c984, 0x33603654ff574ac2, -@@ -451,6 +500,7 @@ - 0xecf3de1acd04651f, 0xcc0a40552559ff32, 0xc385c374f20315b1, - 0xb90208a4c7234183, 0x58aa1ca7a4c075d9, - }; -+#endif - - #if UPDATE_GOLDEN - (void)kGolden; // Silence warning. diff --git a/debian/patches/big-endian-random.diff b/debian/patches/big-endian-random.diff deleted file mode 100644 index a859aca5..00000000 --- a/debian/patches/big-endian-random.diff +++ /dev/null @@ -1,698 +0,0 @@ -From: Benjamin Barenblat <bbaren@google.com> -Subject: Use absl::uint128 for AES random number generator -Forwarded: yes -Applied-Upstream: https://github.com/abseil/abseil-cpp/commit/b06e719ee985ecd63e0dffbc68499549216f817f - -Replace randen’s internal 128-bit integer struct, u64x2, with -absl::uint128. This eliminates some code and improves support for -big-endian platforms. - -The author works at Google. Upstream applied this patch as Piper -revision 383475671 and exported it to GitHub; the Applied-Upstream URL -above points to the exported commit. - ---- a/absl/random/internal/BUILD.bazel -+++ b/absl/random/internal/BUILD.bazel -@@ -296,6 +296,7 @@ cc_library(name = "randen_slow", - ":platform", - "//absl/base:config", - "//absl/base:core_headers", -+ "//absl/numeric:int128", - ], - ) - -@@ -336,6 +337,7 @@ cc_library(name="randen_hwaes_impl", - ":platform", - "//absl/base:config", - "//absl/base:core_headers", -+ "//absl/numeric:int128", - ], - ) - ---- a/absl/random/internal/randen_hwaes.cc -+++ b/absl/random/internal/randen_hwaes.cc -@@ -23,6 +23,7 @@ - #include <cstring> - - #include "absl/base/attributes.h" -+#include "absl/numeric/int128.h" - #include "absl/random/internal/platform.h" - #include "absl/random/internal/randen_traits.h" - -@@ -120,11 +121,6 @@ - - using absl::random_internal::RandenTraits; - --// Randen operates on 128-bit vectors. --struct alignas(16) u64x2 { -- uint64_t data[2]; --}; -- - } // namespace - - // TARGET_CRYPTO defines a crypto attribute for each architecture. -@@ -186,7 +182,7 @@ - } - - // Enables native loads in the round loop by pre-swapping. --inline ABSL_TARGET_CRYPTO void SwapEndian(u64x2* state) { -+inline ABSL_TARGET_CRYPTO void SwapEndian(absl::uint128* state) { - for (uint32_t block = 0; block < RandenTraits::kFeistelBlocks; ++block) { - Vector128Store(ReverseBytes(Vector128Load(state + block)), state + block); - } -@@ -327,7 +323,7 @@ - - // Block shuffles applies a shuffle to the entire state between AES rounds. - // Improved odd-even shuffle from "New criterion for diffusion property". --inline ABSL_TARGET_CRYPTO void BlockShuffle(u64x2* state) { -+inline ABSL_TARGET_CRYPTO void BlockShuffle(absl::uint128* state) { - static_assert(RandenTraits::kFeistelBlocks == 16, - "Expecting 16 FeistelBlocks."); - -@@ -374,8 +370,9 @@ - // per 16 bytes (vs. 10 for AES-CTR). Computing eight round functions in - // parallel hides the 7-cycle AESNI latency on HSW. Note that the Feistel - // XORs are 'free' (included in the second AES instruction). --inline ABSL_TARGET_CRYPTO const u64x2* FeistelRound( -- u64x2* state, const u64x2* ABSL_RANDOM_INTERNAL_RESTRICT keys) { -+inline ABSL_TARGET_CRYPTO const absl::uint128* FeistelRound( -+ absl::uint128* state, -+ const absl::uint128* ABSL_RANDOM_INTERNAL_RESTRICT keys) { - static_assert(RandenTraits::kFeistelBlocks == 16, - "Expecting 16 FeistelBlocks."); - -@@ -436,7 +433,8 @@ - // 2^64 queries if the round function is a PRF. This is similar to the b=8 case - // of Simpira v2, but more efficient than its generic construction for b=16. - inline ABSL_TARGET_CRYPTO void Permute( -- u64x2* state, const u64x2* ABSL_RANDOM_INTERNAL_RESTRICT keys) { -+ absl::uint128* state, -+ const absl::uint128* ABSL_RANDOM_INTERNAL_RESTRICT keys) { - // (Successfully unrolled; the first iteration jumps into the second half) - #ifdef __clang__ - #pragma clang loop unroll_count(2) -@@ -473,10 +471,11 @@ void ABSL_TARGET_CRYPTO RandenHwAes::Absorb(const void* seed_void, - static_assert(RandenTraits::kStateBytes / sizeof(Vector128) == 16, - "Unexpected Randen kStateBlocks"); - -- auto* state = -- reinterpret_cast<u64x2 * ABSL_RANDOM_INTERNAL_RESTRICT>(state_void); -+ auto* state = reinterpret_cast<absl::uint128 * ABSL_RANDOM_INTERNAL_RESTRICT>( -+ state_void); - const auto* seed = -- reinterpret_cast<const u64x2 * ABSL_RANDOM_INTERNAL_RESTRICT>(seed_void); -+ reinterpret_cast<const absl::uint128 * ABSL_RANDOM_INTERNAL_RESTRICT>( -+ seed_void); - - Vector128 b1 = Vector128Load(state + 1); - b1 ^= Vector128Load(seed + 0); -@@ -545,8 +544,8 @@ void ABSL_TARGET_CRYPTO RandenHwAes::Generate(const void* keys_void, - static_assert(RandenTraits::kCapacityBytes == sizeof(Vector128), - "Capacity mismatch"); - -- auto* state = reinterpret_cast<u64x2*>(state_void); -- const auto* keys = reinterpret_cast<const u64x2*>(keys_void); -+ auto* state = reinterpret_cast<absl::uint128*>(state_void); -+ const auto* keys = reinterpret_cast<const absl::uint128*>(keys_void); - - const Vector128 prev_inner = Vector128Load(state); - ---- a/absl/random/internal/randen_slow.cc -+++ b/absl/random/internal/randen_slow.cc -@@ -19,6 +19,7 @@ - #include <cstring> - - #include "absl/base/attributes.h" -+#include "absl/numeric/int128.h" - #include "absl/random/internal/platform.h" - #include "absl/random/internal/randen_traits.h" - -@@ -38,192 +39,193 @@ - namespace { - - // AES portions based on rijndael-alg-fst.c, --// https://fastcrypto.org/front/misc/rijndael-alg-fst.c -+// https://fastcrypto.org/front/misc/rijndael-alg-fst.c, and modified for -+// little-endianness. - // - // Implementation of - // http://www.csrc.nist.gov/publications/fips/fips197/fips-197.pdf - constexpr uint32_t te0[256] = { -- 0xc66363a5, 0xf87c7c84, 0xee777799, 0xf67b7b8d, 0xfff2f20d, 0xd66b6bbd, -- 0xde6f6fb1, 0x91c5c554, 0x60303050, 0x02010103, 0xce6767a9, 0x562b2b7d, -- 0xe7fefe19, 0xb5d7d762, 0x4dababe6, 0xec76769a, 0x8fcaca45, 0x1f82829d, -- 0x89c9c940, 0xfa7d7d87, 0xeffafa15, 0xb25959eb, 0x8e4747c9, 0xfbf0f00b, -- 0x41adadec, 0xb3d4d467, 0x5fa2a2fd, 0x45afafea, 0x239c9cbf, 0x53a4a4f7, -- 0xe4727296, 0x9bc0c05b, 0x75b7b7c2, 0xe1fdfd1c, 0x3d9393ae, 0x4c26266a, -- 0x6c36365a, 0x7e3f3f41, 0xf5f7f702, 0x83cccc4f, 0x6834345c, 0x51a5a5f4, -- 0xd1e5e534, 0xf9f1f108, 0xe2717193, 0xabd8d873, 0x62313153, 0x2a15153f, -- 0x0804040c, 0x95c7c752, 0x46232365, 0x9dc3c35e, 0x30181828, 0x379696a1, -- 0x0a05050f, 0x2f9a9ab5, 0x0e070709, 0x24121236, 0x1b80809b, 0xdfe2e23d, -- 0xcdebeb26, 0x4e272769, 0x7fb2b2cd, 0xea75759f, 0x1209091b, 0x1d83839e, -- 0x582c2c74, 0x341a1a2e, 0x361b1b2d, 0xdc6e6eb2, 0xb45a5aee, 0x5ba0a0fb, -- 0xa45252f6, 0x763b3b4d, 0xb7d6d661, 0x7db3b3ce, 0x5229297b, 0xdde3e33e, -- 0x5e2f2f71, 0x13848497, 0xa65353f5, 0xb9d1d168, 0x00000000, 0xc1eded2c, -- 0x40202060, 0xe3fcfc1f, 0x79b1b1c8, 0xb65b5bed, 0xd46a6abe, 0x8dcbcb46, -- 0x67bebed9, 0x7239394b, 0x944a4ade, 0x984c4cd4, 0xb05858e8, 0x85cfcf4a, -- 0xbbd0d06b, 0xc5efef2a, 0x4faaaae5, 0xedfbfb16, 0x864343c5, 0x9a4d4dd7, -- 0x66333355, 0x11858594, 0x8a4545cf, 0xe9f9f910, 0x04020206, 0xfe7f7f81, -- 0xa05050f0, 0x783c3c44, 0x259f9fba, 0x4ba8a8e3, 0xa25151f3, 0x5da3a3fe, -- 0x804040c0, 0x058f8f8a, 0x3f9292ad, 0x219d9dbc, 0x70383848, 0xf1f5f504, -- 0x63bcbcdf, 0x77b6b6c1, 0xafdada75, 0x42212163, 0x20101030, 0xe5ffff1a, -- 0xfdf3f30e, 0xbfd2d26d, 0x81cdcd4c, 0x180c0c14, 0x26131335, 0xc3ecec2f, -- 0xbe5f5fe1, 0x359797a2, 0x884444cc, 0x2e171739, 0x93c4c457, 0x55a7a7f2, -- 0xfc7e7e82, 0x7a3d3d47, 0xc86464ac, 0xba5d5de7, 0x3219192b, 0xe6737395, -- 0xc06060a0, 0x19818198, 0x9e4f4fd1, 0xa3dcdc7f, 0x44222266, 0x542a2a7e, -- 0x3b9090ab, 0x0b888883, 0x8c4646ca, 0xc7eeee29, 0x6bb8b8d3, 0x2814143c, -- 0xa7dede79, 0xbc5e5ee2, 0x160b0b1d, 0xaddbdb76, 0xdbe0e03b, 0x64323256, -- 0x743a3a4e, 0x140a0a1e, 0x924949db, 0x0c06060a, 0x4824246c, 0xb85c5ce4, -- 0x9fc2c25d, 0xbdd3d36e, 0x43acacef, 0xc46262a6, 0x399191a8, 0x319595a4, -- 0xd3e4e437, 0xf279798b, 0xd5e7e732, 0x8bc8c843, 0x6e373759, 0xda6d6db7, -- 0x018d8d8c, 0xb1d5d564, 0x9c4e4ed2, 0x49a9a9e0, 0xd86c6cb4, 0xac5656fa, -- 0xf3f4f407, 0xcfeaea25, 0xca6565af, 0xf47a7a8e, 0x47aeaee9, 0x10080818, -- 0x6fbabad5, 0xf0787888, 0x4a25256f, 0x5c2e2e72, 0x381c1c24, 0x57a6a6f1, -- 0x73b4b4c7, 0x97c6c651, 0xcbe8e823, 0xa1dddd7c, 0xe874749c, 0x3e1f1f21, -- 0x964b4bdd, 0x61bdbddc, 0x0d8b8b86, 0x0f8a8a85, 0xe0707090, 0x7c3e3e42, -- 0x71b5b5c4, 0xcc6666aa, 0x904848d8, 0x06030305, 0xf7f6f601, 0x1c0e0e12, -- 0xc26161a3, 0x6a35355f, 0xae5757f9, 0x69b9b9d0, 0x17868691, 0x99c1c158, -- 0x3a1d1d27, 0x279e9eb9, 0xd9e1e138, 0xebf8f813, 0x2b9898b3, 0x22111133, -- 0xd26969bb, 0xa9d9d970, 0x078e8e89, 0x339494a7, 0x2d9b9bb6, 0x3c1e1e22, -- 0x15878792, 0xc9e9e920, 0x87cece49, 0xaa5555ff, 0x50282878, 0xa5dfdf7a, -- 0x038c8c8f, 0x59a1a1f8, 0x09898980, 0x1a0d0d17, 0x65bfbfda, 0xd7e6e631, -- 0x844242c6, 0xd06868b8, 0x824141c3, 0x299999b0, 0x5a2d2d77, 0x1e0f0f11, -- 0x7bb0b0cb, 0xa85454fc, 0x6dbbbbd6, 0x2c16163a, -+ 0xa56363c6, 0x847c7cf8, 0x997777ee, 0x8d7b7bf6, 0x0df2f2ff, 0xbd6b6bd6, -+ 0xb16f6fde, 0x54c5c591, 0x50303060, 0x03010102, 0xa96767ce, 0x7d2b2b56, -+ 0x19fefee7, 0x62d7d7b5, 0xe6abab4d, 0x9a7676ec, 0x45caca8f, 0x9d82821f, -+ 0x40c9c989, 0x877d7dfa, 0x15fafaef, 0xeb5959b2, 0xc947478e, 0x0bf0f0fb, -+ 0xecadad41, 0x67d4d4b3, 0xfda2a25f, 0xeaafaf45, 0xbf9c9c23, 0xf7a4a453, -+ 0x967272e4, 0x5bc0c09b, 0xc2b7b775, 0x1cfdfde1, 0xae93933d, 0x6a26264c, -+ 0x5a36366c, 0x413f3f7e, 0x02f7f7f5, 0x4fcccc83, 0x5c343468, 0xf4a5a551, -+ 0x34e5e5d1, 0x08f1f1f9, 0x937171e2, 0x73d8d8ab, 0x53313162, 0x3f15152a, -+ 0x0c040408, 0x52c7c795, 0x65232346, 0x5ec3c39d, 0x28181830, 0xa1969637, -+ 0x0f05050a, 0xb59a9a2f, 0x0907070e, 0x36121224, 0x9b80801b, 0x3de2e2df, -+ 0x26ebebcd, 0x6927274e, 0xcdb2b27f, 0x9f7575ea, 0x1b090912, 0x9e83831d, -+ 0x742c2c58, 0x2e1a1a34, 0x2d1b1b36, 0xb26e6edc, 0xee5a5ab4, 0xfba0a05b, -+ 0xf65252a4, 0x4d3b3b76, 0x61d6d6b7, 0xceb3b37d, 0x7b292952, 0x3ee3e3dd, -+ 0x712f2f5e, 0x97848413, 0xf55353a6, 0x68d1d1b9, 0x00000000, 0x2cededc1, -+ 0x60202040, 0x1ffcfce3, 0xc8b1b179, 0xed5b5bb6, 0xbe6a6ad4, 0x46cbcb8d, -+ 0xd9bebe67, 0x4b393972, 0xde4a4a94, 0xd44c4c98, 0xe85858b0, 0x4acfcf85, -+ 0x6bd0d0bb, 0x2aefefc5, 0xe5aaaa4f, 0x16fbfbed, 0xc5434386, 0xd74d4d9a, -+ 0x55333366, 0x94858511, 0xcf45458a, 0x10f9f9e9, 0x06020204, 0x817f7ffe, -+ 0xf05050a0, 0x443c3c78, 0xba9f9f25, 0xe3a8a84b, 0xf35151a2, 0xfea3a35d, -+ 0xc0404080, 0x8a8f8f05, 0xad92923f, 0xbc9d9d21, 0x48383870, 0x04f5f5f1, -+ 0xdfbcbc63, 0xc1b6b677, 0x75dadaaf, 0x63212142, 0x30101020, 0x1affffe5, -+ 0x0ef3f3fd, 0x6dd2d2bf, 0x4ccdcd81, 0x140c0c18, 0x35131326, 0x2fececc3, -+ 0xe15f5fbe, 0xa2979735, 0xcc444488, 0x3917172e, 0x57c4c493, 0xf2a7a755, -+ 0x827e7efc, 0x473d3d7a, 0xac6464c8, 0xe75d5dba, 0x2b191932, 0x957373e6, -+ 0xa06060c0, 0x98818119, 0xd14f4f9e, 0x7fdcdca3, 0x66222244, 0x7e2a2a54, -+ 0xab90903b, 0x8388880b, 0xca46468c, 0x29eeeec7, 0xd3b8b86b, 0x3c141428, -+ 0x79dedea7, 0xe25e5ebc, 0x1d0b0b16, 0x76dbdbad, 0x3be0e0db, 0x56323264, -+ 0x4e3a3a74, 0x1e0a0a14, 0xdb494992, 0x0a06060c, 0x6c242448, 0xe45c5cb8, -+ 0x5dc2c29f, 0x6ed3d3bd, 0xefacac43, 0xa66262c4, 0xa8919139, 0xa4959531, -+ 0x37e4e4d3, 0x8b7979f2, 0x32e7e7d5, 0x43c8c88b, 0x5937376e, 0xb76d6dda, -+ 0x8c8d8d01, 0x64d5d5b1, 0xd24e4e9c, 0xe0a9a949, 0xb46c6cd8, 0xfa5656ac, -+ 0x07f4f4f3, 0x25eaeacf, 0xaf6565ca, 0x8e7a7af4, 0xe9aeae47, 0x18080810, -+ 0xd5baba6f, 0x887878f0, 0x6f25254a, 0x722e2e5c, 0x241c1c38, 0xf1a6a657, -+ 0xc7b4b473, 0x51c6c697, 0x23e8e8cb, 0x7cdddda1, 0x9c7474e8, 0x211f1f3e, -+ 0xdd4b4b96, 0xdcbdbd61, 0x868b8b0d, 0x858a8a0f, 0x907070e0, 0x423e3e7c, -+ 0xc4b5b571, 0xaa6666cc, 0xd8484890, 0x05030306, 0x01f6f6f7, 0x120e0e1c, -+ 0xa36161c2, 0x5f35356a, 0xf95757ae, 0xd0b9b969, 0x91868617, 0x58c1c199, -+ 0x271d1d3a, 0xb99e9e27, 0x38e1e1d9, 0x13f8f8eb, 0xb398982b, 0x33111122, -+ 0xbb6969d2, 0x70d9d9a9, 0x898e8e07, 0xa7949433, 0xb69b9b2d, 0x221e1e3c, -+ 0x92878715, 0x20e9e9c9, 0x49cece87, 0xff5555aa, 0x78282850, 0x7adfdfa5, -+ 0x8f8c8c03, 0xf8a1a159, 0x80898909, 0x170d0d1a, 0xdabfbf65, 0x31e6e6d7, -+ 0xc6424284, 0xb86868d0, 0xc3414182, 0xb0999929, 0x772d2d5a, 0x110f0f1e, -+ 0xcbb0b07b, 0xfc5454a8, 0xd6bbbb6d, 0x3a16162c, - }; - - constexpr uint32_t te1[256] = { -- 0xa5c66363, 0x84f87c7c, 0x99ee7777, 0x8df67b7b, 0x0dfff2f2, 0xbdd66b6b, -- 0xb1de6f6f, 0x5491c5c5, 0x50603030, 0x03020101, 0xa9ce6767, 0x7d562b2b, -- 0x19e7fefe, 0x62b5d7d7, 0xe64dabab, 0x9aec7676, 0x458fcaca, 0x9d1f8282, -- 0x4089c9c9, 0x87fa7d7d, 0x15effafa, 0xebb25959, 0xc98e4747, 0x0bfbf0f0, -- 0xec41adad, 0x67b3d4d4, 0xfd5fa2a2, 0xea45afaf, 0xbf239c9c, 0xf753a4a4, -- 0x96e47272, 0x5b9bc0c0, 0xc275b7b7, 0x1ce1fdfd, 0xae3d9393, 0x6a4c2626, -- 0x5a6c3636, 0x417e3f3f, 0x02f5f7f7, 0x4f83cccc, 0x5c683434, 0xf451a5a5, -- 0x34d1e5e5, 0x08f9f1f1, 0x93e27171, 0x73abd8d8, 0x53623131, 0x3f2a1515, -- 0x0c080404, 0x5295c7c7, 0x65462323, 0x5e9dc3c3, 0x28301818, 0xa1379696, -- 0x0f0a0505, 0xb52f9a9a, 0x090e0707, 0x36241212, 0x9b1b8080, 0x3ddfe2e2, -- 0x26cdebeb, 0x694e2727, 0xcd7fb2b2, 0x9fea7575, 0x1b120909, 0x9e1d8383, -- 0x74582c2c, 0x2e341a1a, 0x2d361b1b, 0xb2dc6e6e, 0xeeb45a5a, 0xfb5ba0a0, -- 0xf6a45252, 0x4d763b3b, 0x61b7d6d6, 0xce7db3b3, 0x7b522929, 0x3edde3e3, -- 0x715e2f2f, 0x97138484, 0xf5a65353, 0x68b9d1d1, 0x00000000, 0x2cc1eded, -- 0x60402020, 0x1fe3fcfc, 0xc879b1b1, 0xedb65b5b, 0xbed46a6a, 0x468dcbcb, -- 0xd967bebe, 0x4b723939, 0xde944a4a, 0xd4984c4c, 0xe8b05858, 0x4a85cfcf, -- 0x6bbbd0d0, 0x2ac5efef, 0xe54faaaa, 0x16edfbfb, 0xc5864343, 0xd79a4d4d, -- 0x55663333, 0x94118585, 0xcf8a4545, 0x10e9f9f9, 0x06040202, 0x81fe7f7f, -- 0xf0a05050, 0x44783c3c, 0xba259f9f, 0xe34ba8a8, 0xf3a25151, 0xfe5da3a3, -- 0xc0804040, 0x8a058f8f, 0xad3f9292, 0xbc219d9d, 0x48703838, 0x04f1f5f5, -- 0xdf63bcbc, 0xc177b6b6, 0x75afdada, 0x63422121, 0x30201010, 0x1ae5ffff, -- 0x0efdf3f3, 0x6dbfd2d2, 0x4c81cdcd, 0x14180c0c, 0x35261313, 0x2fc3ecec, -- 0xe1be5f5f, 0xa2359797, 0xcc884444, 0x392e1717, 0x5793c4c4, 0xf255a7a7, -- 0x82fc7e7e, 0x477a3d3d, 0xacc86464, 0xe7ba5d5d, 0x2b321919, 0x95e67373, -- 0xa0c06060, 0x98198181, 0xd19e4f4f, 0x7fa3dcdc, 0x66442222, 0x7e542a2a, -- 0xab3b9090, 0x830b8888, 0xca8c4646, 0x29c7eeee, 0xd36bb8b8, 0x3c281414, -- 0x79a7dede, 0xe2bc5e5e, 0x1d160b0b, 0x76addbdb, 0x3bdbe0e0, 0x56643232, -- 0x4e743a3a, 0x1e140a0a, 0xdb924949, 0x0a0c0606, 0x6c482424, 0xe4b85c5c, -- 0x5d9fc2c2, 0x6ebdd3d3, 0xef43acac, 0xa6c46262, 0xa8399191, 0xa4319595, -- 0x37d3e4e4, 0x8bf27979, 0x32d5e7e7, 0x438bc8c8, 0x596e3737, 0xb7da6d6d, -- 0x8c018d8d, 0x64b1d5d5, 0xd29c4e4e, 0xe049a9a9, 0xb4d86c6c, 0xfaac5656, -- 0x07f3f4f4, 0x25cfeaea, 0xafca6565, 0x8ef47a7a, 0xe947aeae, 0x18100808, -- 0xd56fbaba, 0x88f07878, 0x6f4a2525, 0x725c2e2e, 0x24381c1c, 0xf157a6a6, -- 0xc773b4b4, 0x5197c6c6, 0x23cbe8e8, 0x7ca1dddd, 0x9ce87474, 0x213e1f1f, -- 0xdd964b4b, 0xdc61bdbd, 0x860d8b8b, 0x850f8a8a, 0x90e07070, 0x427c3e3e, -- 0xc471b5b5, 0xaacc6666, 0xd8904848, 0x05060303, 0x01f7f6f6, 0x121c0e0e, -- 0xa3c26161, 0x5f6a3535, 0xf9ae5757, 0xd069b9b9, 0x91178686, 0x5899c1c1, -- 0x273a1d1d, 0xb9279e9e, 0x38d9e1e1, 0x13ebf8f8, 0xb32b9898, 0x33221111, -- 0xbbd26969, 0x70a9d9d9, 0x89078e8e, 0xa7339494, 0xb62d9b9b, 0x223c1e1e, -- 0x92158787, 0x20c9e9e9, 0x4987cece, 0xffaa5555, 0x78502828, 0x7aa5dfdf, -- 0x8f038c8c, 0xf859a1a1, 0x80098989, 0x171a0d0d, 0xda65bfbf, 0x31d7e6e6, -- 0xc6844242, 0xb8d06868, 0xc3824141, 0xb0299999, 0x775a2d2d, 0x111e0f0f, -- 0xcb7bb0b0, 0xfca85454, 0xd66dbbbb, 0x3a2c1616, -+ 0x6363c6a5, 0x7c7cf884, 0x7777ee99, 0x7b7bf68d, 0xf2f2ff0d, 0x6b6bd6bd, -+ 0x6f6fdeb1, 0xc5c59154, 0x30306050, 0x01010203, 0x6767cea9, 0x2b2b567d, -+ 0xfefee719, 0xd7d7b562, 0xabab4de6, 0x7676ec9a, 0xcaca8f45, 0x82821f9d, -+ 0xc9c98940, 0x7d7dfa87, 0xfafaef15, 0x5959b2eb, 0x47478ec9, 0xf0f0fb0b, -+ 0xadad41ec, 0xd4d4b367, 0xa2a25ffd, 0xafaf45ea, 0x9c9c23bf, 0xa4a453f7, -+ 0x7272e496, 0xc0c09b5b, 0xb7b775c2, 0xfdfde11c, 0x93933dae, 0x26264c6a, -+ 0x36366c5a, 0x3f3f7e41, 0xf7f7f502, 0xcccc834f, 0x3434685c, 0xa5a551f4, -+ 0xe5e5d134, 0xf1f1f908, 0x7171e293, 0xd8d8ab73, 0x31316253, 0x15152a3f, -+ 0x0404080c, 0xc7c79552, 0x23234665, 0xc3c39d5e, 0x18183028, 0x969637a1, -+ 0x05050a0f, 0x9a9a2fb5, 0x07070e09, 0x12122436, 0x80801b9b, 0xe2e2df3d, -+ 0xebebcd26, 0x27274e69, 0xb2b27fcd, 0x7575ea9f, 0x0909121b, 0x83831d9e, -+ 0x2c2c5874, 0x1a1a342e, 0x1b1b362d, 0x6e6edcb2, 0x5a5ab4ee, 0xa0a05bfb, -+ 0x5252a4f6, 0x3b3b764d, 0xd6d6b761, 0xb3b37dce, 0x2929527b, 0xe3e3dd3e, -+ 0x2f2f5e71, 0x84841397, 0x5353a6f5, 0xd1d1b968, 0x00000000, 0xededc12c, -+ 0x20204060, 0xfcfce31f, 0xb1b179c8, 0x5b5bb6ed, 0x6a6ad4be, 0xcbcb8d46, -+ 0xbebe67d9, 0x3939724b, 0x4a4a94de, 0x4c4c98d4, 0x5858b0e8, 0xcfcf854a, -+ 0xd0d0bb6b, 0xefefc52a, 0xaaaa4fe5, 0xfbfbed16, 0x434386c5, 0x4d4d9ad7, -+ 0x33336655, 0x85851194, 0x45458acf, 0xf9f9e910, 0x02020406, 0x7f7ffe81, -+ 0x5050a0f0, 0x3c3c7844, 0x9f9f25ba, 0xa8a84be3, 0x5151a2f3, 0xa3a35dfe, -+ 0x404080c0, 0x8f8f058a, 0x92923fad, 0x9d9d21bc, 0x38387048, 0xf5f5f104, -+ 0xbcbc63df, 0xb6b677c1, 0xdadaaf75, 0x21214263, 0x10102030, 0xffffe51a, -+ 0xf3f3fd0e, 0xd2d2bf6d, 0xcdcd814c, 0x0c0c1814, 0x13132635, 0xececc32f, -+ 0x5f5fbee1, 0x979735a2, 0x444488cc, 0x17172e39, 0xc4c49357, 0xa7a755f2, -+ 0x7e7efc82, 0x3d3d7a47, 0x6464c8ac, 0x5d5dbae7, 0x1919322b, 0x7373e695, -+ 0x6060c0a0, 0x81811998, 0x4f4f9ed1, 0xdcdca37f, 0x22224466, 0x2a2a547e, -+ 0x90903bab, 0x88880b83, 0x46468cca, 0xeeeec729, 0xb8b86bd3, 0x1414283c, -+ 0xdedea779, 0x5e5ebce2, 0x0b0b161d, 0xdbdbad76, 0xe0e0db3b, 0x32326456, -+ 0x3a3a744e, 0x0a0a141e, 0x494992db, 0x06060c0a, 0x2424486c, 0x5c5cb8e4, -+ 0xc2c29f5d, 0xd3d3bd6e, 0xacac43ef, 0x6262c4a6, 0x919139a8, 0x959531a4, -+ 0xe4e4d337, 0x7979f28b, 0xe7e7d532, 0xc8c88b43, 0x37376e59, 0x6d6ddab7, -+ 0x8d8d018c, 0xd5d5b164, 0x4e4e9cd2, 0xa9a949e0, 0x6c6cd8b4, 0x5656acfa, -+ 0xf4f4f307, 0xeaeacf25, 0x6565caaf, 0x7a7af48e, 0xaeae47e9, 0x08081018, -+ 0xbaba6fd5, 0x7878f088, 0x25254a6f, 0x2e2e5c72, 0x1c1c3824, 0xa6a657f1, -+ 0xb4b473c7, 0xc6c69751, 0xe8e8cb23, 0xdddda17c, 0x7474e89c, 0x1f1f3e21, -+ 0x4b4b96dd, 0xbdbd61dc, 0x8b8b0d86, 0x8a8a0f85, 0x7070e090, 0x3e3e7c42, -+ 0xb5b571c4, 0x6666ccaa, 0x484890d8, 0x03030605, 0xf6f6f701, 0x0e0e1c12, -+ 0x6161c2a3, 0x35356a5f, 0x5757aef9, 0xb9b969d0, 0x86861791, 0xc1c19958, -+ 0x1d1d3a27, 0x9e9e27b9, 0xe1e1d938, 0xf8f8eb13, 0x98982bb3, 0x11112233, -+ 0x6969d2bb, 0xd9d9a970, 0x8e8e0789, 0x949433a7, 0x9b9b2db6, 0x1e1e3c22, -+ 0x87871592, 0xe9e9c920, 0xcece8749, 0x5555aaff, 0x28285078, 0xdfdfa57a, -+ 0x8c8c038f, 0xa1a159f8, 0x89890980, 0x0d0d1a17, 0xbfbf65da, 0xe6e6d731, -+ 0x424284c6, 0x6868d0b8, 0x414182c3, 0x999929b0, 0x2d2d5a77, 0x0f0f1e11, -+ 0xb0b07bcb, 0x5454a8fc, 0xbbbb6dd6, 0x16162c3a, - }; - - constexpr uint32_t te2[256] = { -- 0x63a5c663, 0x7c84f87c, 0x7799ee77, 0x7b8df67b, 0xf20dfff2, 0x6bbdd66b, -- 0x6fb1de6f, 0xc55491c5, 0x30506030, 0x01030201, 0x67a9ce67, 0x2b7d562b, -- 0xfe19e7fe, 0xd762b5d7, 0xabe64dab, 0x769aec76, 0xca458fca, 0x829d1f82, -- 0xc94089c9, 0x7d87fa7d, 0xfa15effa, 0x59ebb259, 0x47c98e47, 0xf00bfbf0, -- 0xadec41ad, 0xd467b3d4, 0xa2fd5fa2, 0xafea45af, 0x9cbf239c, 0xa4f753a4, -- 0x7296e472, 0xc05b9bc0, 0xb7c275b7, 0xfd1ce1fd, 0x93ae3d93, 0x266a4c26, -- 0x365a6c36, 0x3f417e3f, 0xf702f5f7, 0xcc4f83cc, 0x345c6834, 0xa5f451a5, -- 0xe534d1e5, 0xf108f9f1, 0x7193e271, 0xd873abd8, 0x31536231, 0x153f2a15, -- 0x040c0804, 0xc75295c7, 0x23654623, 0xc35e9dc3, 0x18283018, 0x96a13796, -- 0x050f0a05, 0x9ab52f9a, 0x07090e07, 0x12362412, 0x809b1b80, 0xe23ddfe2, -- 0xeb26cdeb, 0x27694e27, 0xb2cd7fb2, 0x759fea75, 0x091b1209, 0x839e1d83, -- 0x2c74582c, 0x1a2e341a, 0x1b2d361b, 0x6eb2dc6e, 0x5aeeb45a, 0xa0fb5ba0, -- 0x52f6a452, 0x3b4d763b, 0xd661b7d6, 0xb3ce7db3, 0x297b5229, 0xe33edde3, -- 0x2f715e2f, 0x84971384, 0x53f5a653, 0xd168b9d1, 0x00000000, 0xed2cc1ed, -- 0x20604020, 0xfc1fe3fc, 0xb1c879b1, 0x5bedb65b, 0x6abed46a, 0xcb468dcb, -- 0xbed967be, 0x394b7239, 0x4ade944a, 0x4cd4984c, 0x58e8b058, 0xcf4a85cf, -- 0xd06bbbd0, 0xef2ac5ef, 0xaae54faa, 0xfb16edfb, 0x43c58643, 0x4dd79a4d, -- 0x33556633, 0x85941185, 0x45cf8a45, 0xf910e9f9, 0x02060402, 0x7f81fe7f, -- 0x50f0a050, 0x3c44783c, 0x9fba259f, 0xa8e34ba8, 0x51f3a251, 0xa3fe5da3, -- 0x40c08040, 0x8f8a058f, 0x92ad3f92, 0x9dbc219d, 0x38487038, 0xf504f1f5, -- 0xbcdf63bc, 0xb6c177b6, 0xda75afda, 0x21634221, 0x10302010, 0xff1ae5ff, -- 0xf30efdf3, 0xd26dbfd2, 0xcd4c81cd, 0x0c14180c, 0x13352613, 0xec2fc3ec, -- 0x5fe1be5f, 0x97a23597, 0x44cc8844, 0x17392e17, 0xc45793c4, 0xa7f255a7, -- 0x7e82fc7e, 0x3d477a3d, 0x64acc864, 0x5de7ba5d, 0x192b3219, 0x7395e673, -- 0x60a0c060, 0x81981981, 0x4fd19e4f, 0xdc7fa3dc, 0x22664422, 0x2a7e542a, -- 0x90ab3b90, 0x88830b88, 0x46ca8c46, 0xee29c7ee, 0xb8d36bb8, 0x143c2814, -- 0xde79a7de, 0x5ee2bc5e, 0x0b1d160b, 0xdb76addb, 0xe03bdbe0, 0x32566432, -- 0x3a4e743a, 0x0a1e140a, 0x49db9249, 0x060a0c06, 0x246c4824, 0x5ce4b85c, -- 0xc25d9fc2, 0xd36ebdd3, 0xacef43ac, 0x62a6c462, 0x91a83991, 0x95a43195, -- 0xe437d3e4, 0x798bf279, 0xe732d5e7, 0xc8438bc8, 0x37596e37, 0x6db7da6d, -- 0x8d8c018d, 0xd564b1d5, 0x4ed29c4e, 0xa9e049a9, 0x6cb4d86c, 0x56faac56, -- 0xf407f3f4, 0xea25cfea, 0x65afca65, 0x7a8ef47a, 0xaee947ae, 0x08181008, -- 0xbad56fba, 0x7888f078, 0x256f4a25, 0x2e725c2e, 0x1c24381c, 0xa6f157a6, -- 0xb4c773b4, 0xc65197c6, 0xe823cbe8, 0xdd7ca1dd, 0x749ce874, 0x1f213e1f, -- 0x4bdd964b, 0xbddc61bd, 0x8b860d8b, 0x8a850f8a, 0x7090e070, 0x3e427c3e, -- 0xb5c471b5, 0x66aacc66, 0x48d89048, 0x03050603, 0xf601f7f6, 0x0e121c0e, -- 0x61a3c261, 0x355f6a35, 0x57f9ae57, 0xb9d069b9, 0x86911786, 0xc15899c1, -- 0x1d273a1d, 0x9eb9279e, 0xe138d9e1, 0xf813ebf8, 0x98b32b98, 0x11332211, -- 0x69bbd269, 0xd970a9d9, 0x8e89078e, 0x94a73394, 0x9bb62d9b, 0x1e223c1e, -- 0x87921587, 0xe920c9e9, 0xce4987ce, 0x55ffaa55, 0x28785028, 0xdf7aa5df, -- 0x8c8f038c, 0xa1f859a1, 0x89800989, 0x0d171a0d, 0xbfda65bf, 0xe631d7e6, -- 0x42c68442, 0x68b8d068, 0x41c38241, 0x99b02999, 0x2d775a2d, 0x0f111e0f, -- 0xb0cb7bb0, 0x54fca854, 0xbbd66dbb, 0x163a2c16, -+ 0x63c6a563, 0x7cf8847c, 0x77ee9977, 0x7bf68d7b, 0xf2ff0df2, 0x6bd6bd6b, -+ 0x6fdeb16f, 0xc59154c5, 0x30605030, 0x01020301, 0x67cea967, 0x2b567d2b, -+ 0xfee719fe, 0xd7b562d7, 0xab4de6ab, 0x76ec9a76, 0xca8f45ca, 0x821f9d82, -+ 0xc98940c9, 0x7dfa877d, 0xfaef15fa, 0x59b2eb59, 0x478ec947, 0xf0fb0bf0, -+ 0xad41ecad, 0xd4b367d4, 0xa25ffda2, 0xaf45eaaf, 0x9c23bf9c, 0xa453f7a4, -+ 0x72e49672, 0xc09b5bc0, 0xb775c2b7, 0xfde11cfd, 0x933dae93, 0x264c6a26, -+ 0x366c5a36, 0x3f7e413f, 0xf7f502f7, 0xcc834fcc, 0x34685c34, 0xa551f4a5, -+ 0xe5d134e5, 0xf1f908f1, 0x71e29371, 0xd8ab73d8, 0x31625331, 0x152a3f15, -+ 0x04080c04, 0xc79552c7, 0x23466523, 0xc39d5ec3, 0x18302818, 0x9637a196, -+ 0x050a0f05, 0x9a2fb59a, 0x070e0907, 0x12243612, 0x801b9b80, 0xe2df3de2, -+ 0xebcd26eb, 0x274e6927, 0xb27fcdb2, 0x75ea9f75, 0x09121b09, 0x831d9e83, -+ 0x2c58742c, 0x1a342e1a, 0x1b362d1b, 0x6edcb26e, 0x5ab4ee5a, 0xa05bfba0, -+ 0x52a4f652, 0x3b764d3b, 0xd6b761d6, 0xb37dceb3, 0x29527b29, 0xe3dd3ee3, -+ 0x2f5e712f, 0x84139784, 0x53a6f553, 0xd1b968d1, 0x00000000, 0xedc12ced, -+ 0x20406020, 0xfce31ffc, 0xb179c8b1, 0x5bb6ed5b, 0x6ad4be6a, 0xcb8d46cb, -+ 0xbe67d9be, 0x39724b39, 0x4a94de4a, 0x4c98d44c, 0x58b0e858, 0xcf854acf, -+ 0xd0bb6bd0, 0xefc52aef, 0xaa4fe5aa, 0xfbed16fb, 0x4386c543, 0x4d9ad74d, -+ 0x33665533, 0x85119485, 0x458acf45, 0xf9e910f9, 0x02040602, 0x7ffe817f, -+ 0x50a0f050, 0x3c78443c, 0x9f25ba9f, 0xa84be3a8, 0x51a2f351, 0xa35dfea3, -+ 0x4080c040, 0x8f058a8f, 0x923fad92, 0x9d21bc9d, 0x38704838, 0xf5f104f5, -+ 0xbc63dfbc, 0xb677c1b6, 0xdaaf75da, 0x21426321, 0x10203010, 0xffe51aff, -+ 0xf3fd0ef3, 0xd2bf6dd2, 0xcd814ccd, 0x0c18140c, 0x13263513, 0xecc32fec, -+ 0x5fbee15f, 0x9735a297, 0x4488cc44, 0x172e3917, 0xc49357c4, 0xa755f2a7, -+ 0x7efc827e, 0x3d7a473d, 0x64c8ac64, 0x5dbae75d, 0x19322b19, 0x73e69573, -+ 0x60c0a060, 0x81199881, 0x4f9ed14f, 0xdca37fdc, 0x22446622, 0x2a547e2a, -+ 0x903bab90, 0x880b8388, 0x468cca46, 0xeec729ee, 0xb86bd3b8, 0x14283c14, -+ 0xdea779de, 0x5ebce25e, 0x0b161d0b, 0xdbad76db, 0xe0db3be0, 0x32645632, -+ 0x3a744e3a, 0x0a141e0a, 0x4992db49, 0x060c0a06, 0x24486c24, 0x5cb8e45c, -+ 0xc29f5dc2, 0xd3bd6ed3, 0xac43efac, 0x62c4a662, 0x9139a891, 0x9531a495, -+ 0xe4d337e4, 0x79f28b79, 0xe7d532e7, 0xc88b43c8, 0x376e5937, 0x6ddab76d, -+ 0x8d018c8d, 0xd5b164d5, 0x4e9cd24e, 0xa949e0a9, 0x6cd8b46c, 0x56acfa56, -+ 0xf4f307f4, 0xeacf25ea, 0x65caaf65, 0x7af48e7a, 0xae47e9ae, 0x08101808, -+ 0xba6fd5ba, 0x78f08878, 0x254a6f25, 0x2e5c722e, 0x1c38241c, 0xa657f1a6, -+ 0xb473c7b4, 0xc69751c6, 0xe8cb23e8, 0xdda17cdd, 0x74e89c74, 0x1f3e211f, -+ 0x4b96dd4b, 0xbd61dcbd, 0x8b0d868b, 0x8a0f858a, 0x70e09070, 0x3e7c423e, -+ 0xb571c4b5, 0x66ccaa66, 0x4890d848, 0x03060503, 0xf6f701f6, 0x0e1c120e, -+ 0x61c2a361, 0x356a5f35, 0x57aef957, 0xb969d0b9, 0x86179186, 0xc19958c1, -+ 0x1d3a271d, 0x9e27b99e, 0xe1d938e1, 0xf8eb13f8, 0x982bb398, 0x11223311, -+ 0x69d2bb69, 0xd9a970d9, 0x8e07898e, 0x9433a794, 0x9b2db69b, 0x1e3c221e, -+ 0x87159287, 0xe9c920e9, 0xce8749ce, 0x55aaff55, 0x28507828, 0xdfa57adf, -+ 0x8c038f8c, 0xa159f8a1, 0x89098089, 0x0d1a170d, 0xbf65dabf, 0xe6d731e6, -+ 0x4284c642, 0x68d0b868, 0x4182c341, 0x9929b099, 0x2d5a772d, 0x0f1e110f, -+ 0xb07bcbb0, 0x54a8fc54, 0xbb6dd6bb, 0x162c3a16, - }; - - constexpr uint32_t te3[256] = { -- 0x6363a5c6, 0x7c7c84f8, 0x777799ee, 0x7b7b8df6, 0xf2f20dff, 0x6b6bbdd6, -- 0x6f6fb1de, 0xc5c55491, 0x30305060, 0x01010302, 0x6767a9ce, 0x2b2b7d56, -- 0xfefe19e7, 0xd7d762b5, 0xababe64d, 0x76769aec, 0xcaca458f, 0x82829d1f, -- 0xc9c94089, 0x7d7d87fa, 0xfafa15ef, 0x5959ebb2, 0x4747c98e, 0xf0f00bfb, -- 0xadadec41, 0xd4d467b3, 0xa2a2fd5f, 0xafafea45, 0x9c9cbf23, 0xa4a4f753, -- 0x727296e4, 0xc0c05b9b, 0xb7b7c275, 0xfdfd1ce1, 0x9393ae3d, 0x26266a4c, -- 0x36365a6c, 0x3f3f417e, 0xf7f702f5, 0xcccc4f83, 0x34345c68, 0xa5a5f451, -- 0xe5e534d1, 0xf1f108f9, 0x717193e2, 0xd8d873ab, 0x31315362, 0x15153f2a, -- 0x04040c08, 0xc7c75295, 0x23236546, 0xc3c35e9d, 0x18182830, 0x9696a137, -- 0x05050f0a, 0x9a9ab52f, 0x0707090e, 0x12123624, 0x80809b1b, 0xe2e23ddf, -- 0xebeb26cd, 0x2727694e, 0xb2b2cd7f, 0x75759fea, 0x09091b12, 0x83839e1d, -- 0x2c2c7458, 0x1a1a2e34, 0x1b1b2d36, 0x6e6eb2dc, 0x5a5aeeb4, 0xa0a0fb5b, -- 0x5252f6a4, 0x3b3b4d76, 0xd6d661b7, 0xb3b3ce7d, 0x29297b52, 0xe3e33edd, -- 0x2f2f715e, 0x84849713, 0x5353f5a6, 0xd1d168b9, 0x00000000, 0xeded2cc1, -- 0x20206040, 0xfcfc1fe3, 0xb1b1c879, 0x5b5bedb6, 0x6a6abed4, 0xcbcb468d, -- 0xbebed967, 0x39394b72, 0x4a4ade94, 0x4c4cd498, 0x5858e8b0, 0xcfcf4a85, -- 0xd0d06bbb, 0xefef2ac5, 0xaaaae54f, 0xfbfb16ed, 0x4343c586, 0x4d4dd79a, -- 0x33335566, 0x85859411, 0x4545cf8a, 0xf9f910e9, 0x02020604, 0x7f7f81fe, -- 0x5050f0a0, 0x3c3c4478, 0x9f9fba25, 0xa8a8e34b, 0x5151f3a2, 0xa3a3fe5d, -- 0x4040c080, 0x8f8f8a05, 0x9292ad3f, 0x9d9dbc21, 0x38384870, 0xf5f504f1, -- 0xbcbcdf63, 0xb6b6c177, 0xdada75af, 0x21216342, 0x10103020, 0xffff1ae5, -- 0xf3f30efd, 0xd2d26dbf, 0xcdcd4c81, 0x0c0c1418, 0x13133526, 0xecec2fc3, -- 0x5f5fe1be, 0x9797a235, 0x4444cc88, 0x1717392e, 0xc4c45793, 0xa7a7f255, -- 0x7e7e82fc, 0x3d3d477a, 0x6464acc8, 0x5d5de7ba, 0x19192b32, 0x737395e6, -- 0x6060a0c0, 0x81819819, 0x4f4fd19e, 0xdcdc7fa3, 0x22226644, 0x2a2a7e54, -- 0x9090ab3b, 0x8888830b, 0x4646ca8c, 0xeeee29c7, 0xb8b8d36b, 0x14143c28, -- 0xdede79a7, 0x5e5ee2bc, 0x0b0b1d16, 0xdbdb76ad, 0xe0e03bdb, 0x32325664, -- 0x3a3a4e74, 0x0a0a1e14, 0x4949db92, 0x06060a0c, 0x24246c48, 0x5c5ce4b8, -- 0xc2c25d9f, 0xd3d36ebd, 0xacacef43, 0x6262a6c4, 0x9191a839, 0x9595a431, -- 0xe4e437d3, 0x79798bf2, 0xe7e732d5, 0xc8c8438b, 0x3737596e, 0x6d6db7da, -- 0x8d8d8c01, 0xd5d564b1, 0x4e4ed29c, 0xa9a9e049, 0x6c6cb4d8, 0x5656faac, -- 0xf4f407f3, 0xeaea25cf, 0x6565afca, 0x7a7a8ef4, 0xaeaee947, 0x08081810, -- 0xbabad56f, 0x787888f0, 0x25256f4a, 0x2e2e725c, 0x1c1c2438, 0xa6a6f157, -- 0xb4b4c773, 0xc6c65197, 0xe8e823cb, 0xdddd7ca1, 0x74749ce8, 0x1f1f213e, -- 0x4b4bdd96, 0xbdbddc61, 0x8b8b860d, 0x8a8a850f, 0x707090e0, 0x3e3e427c, -- 0xb5b5c471, 0x6666aacc, 0x4848d890, 0x03030506, 0xf6f601f7, 0x0e0e121c, -- 0x6161a3c2, 0x35355f6a, 0x5757f9ae, 0xb9b9d069, 0x86869117, 0xc1c15899, -- 0x1d1d273a, 0x9e9eb927, 0xe1e138d9, 0xf8f813eb, 0x9898b32b, 0x11113322, -- 0x6969bbd2, 0xd9d970a9, 0x8e8e8907, 0x9494a733, 0x9b9bb62d, 0x1e1e223c, -- 0x87879215, 0xe9e920c9, 0xcece4987, 0x5555ffaa, 0x28287850, 0xdfdf7aa5, -- 0x8c8c8f03, 0xa1a1f859, 0x89898009, 0x0d0d171a, 0xbfbfda65, 0xe6e631d7, -- 0x4242c684, 0x6868b8d0, 0x4141c382, 0x9999b029, 0x2d2d775a, 0x0f0f111e, -- 0xb0b0cb7b, 0x5454fca8, 0xbbbbd66d, 0x16163a2c, -+ 0xc6a56363, 0xf8847c7c, 0xee997777, 0xf68d7b7b, 0xff0df2f2, 0xd6bd6b6b, -+ 0xdeb16f6f, 0x9154c5c5, 0x60503030, 0x02030101, 0xcea96767, 0x567d2b2b, -+ 0xe719fefe, 0xb562d7d7, 0x4de6abab, 0xec9a7676, 0x8f45caca, 0x1f9d8282, -+ 0x8940c9c9, 0xfa877d7d, 0xef15fafa, 0xb2eb5959, 0x8ec94747, 0xfb0bf0f0, -+ 0x41ecadad, 0xb367d4d4, 0x5ffda2a2, 0x45eaafaf, 0x23bf9c9c, 0x53f7a4a4, -+ 0xe4967272, 0x9b5bc0c0, 0x75c2b7b7, 0xe11cfdfd, 0x3dae9393, 0x4c6a2626, -+ 0x6c5a3636, 0x7e413f3f, 0xf502f7f7, 0x834fcccc, 0x685c3434, 0x51f4a5a5, -+ 0xd134e5e5, 0xf908f1f1, 0xe2937171, 0xab73d8d8, 0x62533131, 0x2a3f1515, -+ 0x080c0404, 0x9552c7c7, 0x46652323, 0x9d5ec3c3, 0x30281818, 0x37a19696, -+ 0x0a0f0505, 0x2fb59a9a, 0x0e090707, 0x24361212, 0x1b9b8080, 0xdf3de2e2, -+ 0xcd26ebeb, 0x4e692727, 0x7fcdb2b2, 0xea9f7575, 0x121b0909, 0x1d9e8383, -+ 0x58742c2c, 0x342e1a1a, 0x362d1b1b, 0xdcb26e6e, 0xb4ee5a5a, 0x5bfba0a0, -+ 0xa4f65252, 0x764d3b3b, 0xb761d6d6, 0x7dceb3b3, 0x527b2929, 0xdd3ee3e3, -+ 0x5e712f2f, 0x13978484, 0xa6f55353, 0xb968d1d1, 0x00000000, 0xc12ceded, -+ 0x40602020, 0xe31ffcfc, 0x79c8b1b1, 0xb6ed5b5b, 0xd4be6a6a, 0x8d46cbcb, -+ 0x67d9bebe, 0x724b3939, 0x94de4a4a, 0x98d44c4c, 0xb0e85858, 0x854acfcf, -+ 0xbb6bd0d0, 0xc52aefef, 0x4fe5aaaa, 0xed16fbfb, 0x86c54343, 0x9ad74d4d, -+ 0x66553333, 0x11948585, 0x8acf4545, 0xe910f9f9, 0x04060202, 0xfe817f7f, -+ 0xa0f05050, 0x78443c3c, 0x25ba9f9f, 0x4be3a8a8, 0xa2f35151, 0x5dfea3a3, -+ 0x80c04040, 0x058a8f8f, 0x3fad9292, 0x21bc9d9d, 0x70483838, 0xf104f5f5, -+ 0x63dfbcbc, 0x77c1b6b6, 0xaf75dada, 0x42632121, 0x20301010, 0xe51affff, -+ 0xfd0ef3f3, 0xbf6dd2d2, 0x814ccdcd, 0x18140c0c, 0x26351313, 0xc32fecec, -+ 0xbee15f5f, 0x35a29797, 0x88cc4444, 0x2e391717, 0x9357c4c4, 0x55f2a7a7, -+ 0xfc827e7e, 0x7a473d3d, 0xc8ac6464, 0xbae75d5d, 0x322b1919, 0xe6957373, -+ 0xc0a06060, 0x19988181, 0x9ed14f4f, 0xa37fdcdc, 0x44662222, 0x547e2a2a, -+ 0x3bab9090, 0x0b838888, 0x8cca4646, 0xc729eeee, 0x6bd3b8b8, 0x283c1414, -+ 0xa779dede, 0xbce25e5e, 0x161d0b0b, 0xad76dbdb, 0xdb3be0e0, 0x64563232, -+ 0x744e3a3a, 0x141e0a0a, 0x92db4949, 0x0c0a0606, 0x486c2424, 0xb8e45c5c, -+ 0x9f5dc2c2, 0xbd6ed3d3, 0x43efacac, 0xc4a66262, 0x39a89191, 0x31a49595, -+ 0xd337e4e4, 0xf28b7979, 0xd532e7e7, 0x8b43c8c8, 0x6e593737, 0xdab76d6d, -+ 0x018c8d8d, 0xb164d5d5, 0x9cd24e4e, 0x49e0a9a9, 0xd8b46c6c, 0xacfa5656, -+ 0xf307f4f4, 0xcf25eaea, 0xcaaf6565, 0xf48e7a7a, 0x47e9aeae, 0x10180808, -+ 0x6fd5baba, 0xf0887878, 0x4a6f2525, 0x5c722e2e, 0x38241c1c, 0x57f1a6a6, -+ 0x73c7b4b4, 0x9751c6c6, 0xcb23e8e8, 0xa17cdddd, 0xe89c7474, 0x3e211f1f, -+ 0x96dd4b4b, 0x61dcbdbd, 0x0d868b8b, 0x0f858a8a, 0xe0907070, 0x7c423e3e, -+ 0x71c4b5b5, 0xccaa6666, 0x90d84848, 0x06050303, 0xf701f6f6, 0x1c120e0e, -+ 0xc2a36161, 0x6a5f3535, 0xaef95757, 0x69d0b9b9, 0x17918686, 0x9958c1c1, -+ 0x3a271d1d, 0x27b99e9e, 0xd938e1e1, 0xeb13f8f8, 0x2bb39898, 0x22331111, -+ 0xd2bb6969, 0xa970d9d9, 0x07898e8e, 0x33a79494, 0x2db69b9b, 0x3c221e1e, -+ 0x15928787, 0xc920e9e9, 0x8749cece, 0xaaff5555, 0x50782828, 0xa57adfdf, -+ 0x038f8c8c, 0x59f8a1a1, 0x09808989, 0x1a170d0d, 0x65dabfbf, 0xd731e6e6, -+ 0x84c64242, 0xd0b86868, 0x82c34141, 0x29b09999, 0x5a772d2d, 0x1e110f0f, -+ 0x7bcbb0b0, 0xa8fc5454, 0x6dd6bbbb, 0x2c3a1616, - }; - - // Software implementation of the Vector128 class, using uint32_t -@@ -235,45 +237,13 @@ - inline ABSL_RANDOM_INTERNAL_ATTRIBUTE_ALWAYS_INLINE Vector128 - Vector128Load(const void* from) { - Vector128 result; -- const uint8_t* src = reinterpret_cast<const uint8_t*>(from); -- result.s[0] = static_cast<uint32_t>(src[0]) << 24 | -- static_cast<uint32_t>(src[1]) << 16 | -- static_cast<uint32_t>(src[2]) << 8 | -- static_cast<uint32_t>(src[3]); -- result.s[1] = static_cast<uint32_t>(src[4]) << 24 | -- static_cast<uint32_t>(src[5]) << 16 | -- static_cast<uint32_t>(src[6]) << 8 | -- static_cast<uint32_t>(src[7]); -- result.s[2] = static_cast<uint32_t>(src[8]) << 24 | -- static_cast<uint32_t>(src[9]) << 16 | -- static_cast<uint32_t>(src[10]) << 8 | -- static_cast<uint32_t>(src[11]); -- result.s[3] = static_cast<uint32_t>(src[12]) << 24 | -- static_cast<uint32_t>(src[13]) << 16 | -- static_cast<uint32_t>(src[14]) << 8 | -- static_cast<uint32_t>(src[15]); -+ std::memcpy(result.s, from, sizeof(Vector128)); - return result; - } - - inline ABSL_RANDOM_INTERNAL_ATTRIBUTE_ALWAYS_INLINE void Vector128Store( - const Vector128& v, void* to) { -- uint8_t* dst = reinterpret_cast<uint8_t*>(to); -- dst[0] = static_cast<uint8_t>(v.s[0] >> 24); -- dst[1] = static_cast<uint8_t>(v.s[0] >> 16); -- dst[2] = static_cast<uint8_t>(v.s[0] >> 8); -- dst[3] = static_cast<uint8_t>(v.s[0]); -- dst[4] = static_cast<uint8_t>(v.s[1] >> 24); -- dst[5] = static_cast<uint8_t>(v.s[1] >> 16); -- dst[6] = static_cast<uint8_t>(v.s[1] >> 8); -- dst[7] = static_cast<uint8_t>(v.s[1]); -- dst[8] = static_cast<uint8_t>(v.s[2] >> 24); -- dst[9] = static_cast<uint8_t>(v.s[2] >> 16); -- dst[10] = static_cast<uint8_t>(v.s[2] >> 8); -- dst[11] = static_cast<uint8_t>(v.s[2]); -- dst[12] = static_cast<uint8_t>(v.s[3] >> 24); -- dst[13] = static_cast<uint8_t>(v.s[3] >> 16); -- dst[14] = static_cast<uint8_t>(v.s[3] >> 8); -- dst[15] = static_cast<uint8_t>(v.s[3]); -+ std::memcpy(to, v.s, sizeof(Vector128)); - } - - // One round of AES. "round_key" is a public constant for breaking the -@@ -282,38 +252,33 @@ - AesRound(const Vector128& state, const Vector128& round_key) { - Vector128 result; - result.s[0] = round_key.s[0] ^ // -- te0[uint8_t(state.s[0] >> 24)] ^ // -- te1[uint8_t(state.s[1] >> 16)] ^ // -- te2[uint8_t(state.s[2] >> 8)] ^ // -- te3[uint8_t(state.s[3])]; -+ te0[uint8_t(state.s[0])] ^ // -+ te1[uint8_t(state.s[1] >> 8)] ^ // -+ te2[uint8_t(state.s[2] >> 16)] ^ // -+ te3[uint8_t(state.s[3] >> 24)]; - result.s[1] = round_key.s[1] ^ // -- te0[uint8_t(state.s[1] >> 24)] ^ // -- te1[uint8_t(state.s[2] >> 16)] ^ // -- te2[uint8_t(state.s[3] >> 8)] ^ // -- te3[uint8_t(state.s[0])]; -+ te0[uint8_t(state.s[1])] ^ // -+ te1[uint8_t(state.s[2] >> 8)] ^ // -+ te2[uint8_t(state.s[3] >> 16)] ^ // -+ te3[uint8_t(state.s[0] >> 24)]; - result.s[2] = round_key.s[2] ^ // -- te0[uint8_t(state.s[2] >> 24)] ^ // -- te1[uint8_t(state.s[3] >> 16)] ^ // -- te2[uint8_t(state.s[0] >> 8)] ^ // -- te3[uint8_t(state.s[1])]; -+ te0[uint8_t(state.s[2])] ^ // -+ te1[uint8_t(state.s[3] >> 8)] ^ // -+ te2[uint8_t(state.s[0] >> 16)] ^ // -+ te3[uint8_t(state.s[1] >> 24)]; - result.s[3] = round_key.s[3] ^ // -- te0[uint8_t(state.s[3] >> 24)] ^ // -- te1[uint8_t(state.s[0] >> 16)] ^ // -- te2[uint8_t(state.s[1] >> 8)] ^ // -- te3[uint8_t(state.s[2])]; -+ te0[uint8_t(state.s[3])] ^ // -+ te1[uint8_t(state.s[0] >> 8)] ^ // -+ te2[uint8_t(state.s[1] >> 16)] ^ // -+ te3[uint8_t(state.s[2] >> 24)]; - return result; - } - - using ::absl::random_internal::RandenTraits; - --// Randen operates on 128-bit vectors. --struct alignas(16) u64x2 { -- uint64_t data[2]; --}; -- - // The improved Feistel block shuffle function for 16 blocks. - inline ABSL_RANDOM_INTERNAL_ATTRIBUTE_ALWAYS_INLINE void BlockShuffle( -- u64x2* state) { -+ absl::uint128* state) { - static_assert(RandenTraits::kFeistelBlocks == 16, - "Feistel block shuffle only works for 16 blocks."); - -@@ -323,31 +288,31 @@ inline ABSL_RANDOM_INTERNAL_ATTRIBUTE_ALWAYS_INLINE void BlockShuffle( - // The fully unrolled loop without the memcpy improves the speed by about - // 30% over the equivalent: - #if 0 -- u64x2 source[RandenTraits::kFeistelBlocks]; -+ absl::uint128 source[RandenTraits::kFeistelBlocks]; - std::memcpy(source, state, sizeof(source)); - for (size_t i = 0; i < RandenTraits::kFeistelBlocks; i++) { -- const u64x2 v0 = source[shuffle[i]]; -+ const absl::uint128 v0 = source[shuffle[i]]; - state[i] = v0; - } - return; - #endif - -- const u64x2 v0 = state[shuffle[0]]; -- const u64x2 v1 = state[shuffle[1]]; -- const u64x2 v2 = state[shuffle[2]]; -- const u64x2 v3 = state[shuffle[3]]; -- const u64x2 v4 = state[shuffle[4]]; -- const u64x2 v5 = state[shuffle[5]]; -- const u64x2 v6 = state[shuffle[6]]; -- const u64x2 v7 = state[shuffle[7]]; -- const u64x2 w0 = state[shuffle[8]]; -- const u64x2 w1 = state[shuffle[9]]; -- const u64x2 w2 = state[shuffle[10]]; -- const u64x2 w3 = state[shuffle[11]]; -- const u64x2 w4 = state[shuffle[12]]; -- const u64x2 w5 = state[shuffle[13]]; -- const u64x2 w6 = state[shuffle[14]]; -- const u64x2 w7 = state[shuffle[15]]; -+ const absl::uint128 v0 = state[shuffle[0]]; -+ const absl::uint128 v1 = state[shuffle[1]]; -+ const absl::uint128 v2 = state[shuffle[2]]; -+ const absl::uint128 v3 = state[shuffle[3]]; -+ const absl::uint128 v4 = state[shuffle[4]]; -+ const absl::uint128 v5 = state[shuffle[5]]; -+ const absl::uint128 v6 = state[shuffle[6]]; -+ const absl::uint128 v7 = state[shuffle[7]]; -+ const absl::uint128 w0 = state[shuffle[8]]; -+ const absl::uint128 w1 = state[shuffle[9]]; -+ const absl::uint128 w2 = state[shuffle[10]]; -+ const absl::uint128 w3 = state[shuffle[11]]; -+ const absl::uint128 w4 = state[shuffle[12]]; -+ const absl::uint128 w5 = state[shuffle[13]]; -+ const absl::uint128 w6 = state[shuffle[14]]; -+ const absl::uint128 w7 = state[shuffle[15]]; - state[0] = v0; - state[1] = v1; - state[2] = v2; -@@ -371,9 +336,9 @@ - // per 16 bytes (vs. 10 for AES-CTR). Computing eight round functions in - // parallel hides the 7-cycle AESNI latency on HSW. Note that the Feistel - // XORs are 'free' (included in the second AES instruction). --inline ABSL_RANDOM_INTERNAL_ATTRIBUTE_ALWAYS_INLINE const u64x2* FeistelRound( -- u64x2* ABSL_RANDOM_INTERNAL_RESTRICT state, -- const u64x2* ABSL_RANDOM_INTERNAL_RESTRICT keys) { -+inline ABSL_RANDOM_INTERNAL_ATTRIBUTE_ALWAYS_INLINE const absl::uint128* -+FeistelRound(absl::uint128* ABSL_RANDOM_INTERNAL_RESTRICT state, -+ const absl::uint128* ABSL_RANDOM_INTERNAL_RESTRICT keys) { - for (size_t branch = 0; branch < RandenTraits::kFeistelBlocks; branch += 4) { - const Vector128 s0 = Vector128Load(state + branch); - const Vector128 s1 = Vector128Load(state + branch + 1); -@@ -398,7 +363,8 @@ - // 2^64 queries if the round function is a PRF. This is similar to the b=8 case - // of Simpira v2, but more efficient than its generic construction for b=16. - inline ABSL_RANDOM_INTERNAL_ATTRIBUTE_ALWAYS_INLINE void Permute( -- u64x2* state, const u64x2* ABSL_RANDOM_INTERNAL_RESTRICT keys) { -+ absl::uint128* state, -+ const absl::uint128* ABSL_RANDOM_INTERNAL_RESTRICT keys) { - for (size_t round = 0; round < RandenTraits::kFeistelRounds; ++round) { - keys = FeistelRound(state, keys); - BlockShuffle(state); -@@ -437,19 +403,18 @@ - } - - void RandenSlow::Generate(const void* keys_void, void* state_void) { -- static_assert(RandenTraits::kCapacityBytes == sizeof(u64x2), -+ static_assert(RandenTraits::kCapacityBytes == sizeof(absl::uint128), - "Capacity mismatch"); - -- auto* state = reinterpret_cast<u64x2*>(state_void); -- const auto* keys = reinterpret_cast<const u64x2*>(keys_void); -+ auto* state = reinterpret_cast<absl::uint128*>(state_void); -+ const auto* keys = reinterpret_cast<const absl::uint128*>(keys_void); - -- const u64x2 prev_inner = state[0]; -+ const absl::uint128 prev_inner = state[0]; - - Permute(state, keys); - - // Ensure backtracking resistance. -- state[0].data[0] ^= prev_inner.data[0]; -- state[0].data[1] ^= prev_inner.data[1]; -+ *state ^= prev_inner; - } - - } // namespace random_internal diff --git a/debian/patches/big-endian-random2.diff b/debian/patches/big-endian-random2.diff deleted file mode 100644 index 12893c9a..00000000 --- a/debian/patches/big-endian-random2.diff +++ /dev/null @@ -1,90 +0,0 @@ -From: Benjamin Barenblat <bbaren@google.com> -Subject: Make randen_slow endian-correct -Forwarded: yes -Applied-Upstream: https://github.com/abseil/abseil-cpp/commit/33541e751039a8c4bd3a395dd1a3a0928885814a - -Pay attention to the platform endianness when pulling bytes out of each -AES block, and use platform-endian round keys. - -The author works at Google. Upstream applied this patch as Piper -revision 383878281 and exported it to GitHub; the Applied-Upstream URL -above points to the exported commit. - ---- a/absl/random/internal/BUILD.bazel -+++ b/absl/random/internal/BUILD.bazel -@@ -296,6 +296,7 @@ - ":platform", - "//absl/base:config", - "//absl/base:core_headers", -+ "//absl/base:endian", - "//absl/numeric:int128", - ], - ) ---- a/absl/random/internal/randen_slow.cc -+++ b/absl/random/internal/randen_slow.cc -@@ -19,6 +19,7 @@ - #include <cstring> - - #include "absl/base/attributes.h" -+#include "absl/base/internal/endian.h" - #include "absl/numeric/int128.h" - #include "absl/random/internal/platform.h" - #include "absl/random/internal/randen_traits.h" -@@ -40,7 +41,7 @@ - - // AES portions based on rijndael-alg-fst.c, - // https://fastcrypto.org/front/misc/rijndael-alg-fst.c, and modified for --// little-endianness. -+// platform-endianness. - // - // Implementation of - // http://www.csrc.nist.gov/publications/fips/fips197/fips-197.pdf -@@ -251,6 +252,7 @@ - inline ABSL_RANDOM_INTERNAL_ATTRIBUTE_ALWAYS_INLINE Vector128 - AesRound(const Vector128& state, const Vector128& round_key) { - Vector128 result; -+#ifdef ABSL_IS_LITTLE_ENDIAN - result.s[0] = round_key.s[0] ^ // - te0[uint8_t(state.s[0])] ^ // - te1[uint8_t(state.s[1] >> 8)] ^ // -@@ -271,6 +273,28 @@ - te1[uint8_t(state.s[0] >> 8)] ^ // - te2[uint8_t(state.s[1] >> 16)] ^ // - te3[uint8_t(state.s[2] >> 24)]; -+#else -+ result.s[0] = round_key.s[0] ^ // -+ te0[uint8_t(state.s[0])] ^ // -+ te1[uint8_t(state.s[3] >> 8)] ^ // -+ te2[uint8_t(state.s[2] >> 16)] ^ // -+ te3[uint8_t(state.s[1] >> 24)]; -+ result.s[1] = round_key.s[1] ^ // -+ te0[uint8_t(state.s[1])] ^ // -+ te1[uint8_t(state.s[0] >> 8)] ^ // -+ te2[uint8_t(state.s[3] >> 16)] ^ // -+ te3[uint8_t(state.s[2] >> 24)]; -+ result.s[2] = round_key.s[2] ^ // -+ te0[uint8_t(state.s[2])] ^ // -+ te1[uint8_t(state.s[1] >> 8)] ^ // -+ te2[uint8_t(state.s[0] >> 16)] ^ // -+ te3[uint8_t(state.s[3] >> 24)]; -+ result.s[3] = round_key.s[3] ^ // -+ te0[uint8_t(state.s[3])] ^ // -+ te1[uint8_t(state.s[2] >> 8)] ^ // -+ te2[uint8_t(state.s[1] >> 16)] ^ // -+ te3[uint8_t(state.s[0] >> 24)]; -+#endif - return result; - } - -@@ -380,7 +404,11 @@ - const void* RandenSlow::GetKeys() { - // Round keys for one AES per Feistel round and branch. - // The canonical implementation uses first digits of Pi. -+#ifdef ABSL_IS_LITTLE_ENDIAN - return kRandenRoundKeys; -+#else -+ return kRandenRoundKeysBE; -+#endif - } - - void RandenSlow::Absorb(const void* seed_void, void* state_void) { diff --git a/debian/patches/big-endian-random3.diff b/debian/patches/big-endian-random3.diff deleted file mode 100644 index a7847a00..00000000 --- a/debian/patches/big-endian-random3.diff +++ /dev/null @@ -1,82 +0,0 @@ -From: Laramie Leavitt <lar@google.com> -Subject: Also use uint8_t golden values in randen_test.cc -Forwarded: yes -Origin: upstream, https://github.com/abseil/abseil-cpp/commit/cc413f8b674d61e3aa948386432e526e051afca0 - -Also use uint8_t golden values in randen_test.cc - -This makes randen_test, randen_slow_test, and randen_hwaes_test essentially -identical, as is the intent. - -The author works at Google. Upstream applied this patch as Piper -revision 405484423 and exported it to GitHub; the Applied-Upstream URL -above points to the exported commit. - ---- a/absl/random/internal/randen_test.cc -+++ b/absl/random/internal/randen_test.cc -@@ -23,9 +23,6 @@ - - using absl::random_internal::Randen; - --// Local state parameters. --constexpr size_t kStateSizeT = Randen::kStateBytes / sizeof(uint64_t); -- - TEST(RandenTest, CopyAndMove) { - static_assert(std::is_copy_constructible<Randen>::value, - "Randen must be copy constructible"); -@@ -41,30 +38,38 @@ - } - - TEST(RandenTest, Default) { -- constexpr uint64_t kGolden[] = { -- 0x6c6534090ee6d3ee, 0x044e2b9b9d5333c6, 0xc3c14f134e433977, -- 0xdda9f47cd90410ee, 0x887bf3087fd8ca10, 0xf0b780f545c72912, -- 0x15dbb1d37696599f, 0x30ec63baff3c6d59, 0xb29f73606f7f20a6, -- 0x02808a316f49a54c, 0x3b8feaf9d5c8e50e, 0x9cbf605e3fd9de8a, -- 0xc970ae1a78183bbb, 0xd8b2ffd356301ed5, 0xf4b327fe0fc73c37, -- 0xcdfd8d76eb8f9a19, 0xc3a506eb91420c9d, 0xd5af05dd3eff9556, -- 0x48db1bb78f83c4a1, 0x7023920e0d6bfe8c, 0x58d3575834956d42, -- 0xed1ef4c26b87b840, 0x8eef32a23e0b2df3, 0x497cabf3431154fc, -- 0x4e24370570029a8b, 0xd88b5749f090e5ea, 0xc651a582a970692f, -- 0x78fcec2cbb6342f5, 0x463cb745612f55db, 0x352ee4ad1816afe3, -- 0x026ff374c101da7e, 0x811ef0821c3de851, -+ constexpr uint8_t kGolden[] = { -+ 0xee, 0xd3, 0xe6, 0x0e, 0x09, 0x34, 0x65, 0x6c, 0xc6, 0x33, 0x53, 0x9d, -+ 0x9b, 0x2b, 0x4e, 0x04, 0x77, 0x39, 0x43, 0x4e, 0x13, 0x4f, 0xc1, 0xc3, -+ 0xee, 0x10, 0x04, 0xd9, 0x7c, 0xf4, 0xa9, 0xdd, 0x10, 0xca, 0xd8, 0x7f, -+ 0x08, 0xf3, 0x7b, 0x88, 0x12, 0x29, 0xc7, 0x45, 0xf5, 0x80, 0xb7, 0xf0, -+ 0x9f, 0x59, 0x96, 0x76, 0xd3, 0xb1, 0xdb, 0x15, 0x59, 0x6d, 0x3c, 0xff, -+ 0xba, 0x63, 0xec, 0x30, 0xa6, 0x20, 0x7f, 0x6f, 0x60, 0x73, 0x9f, 0xb2, -+ 0x4c, 0xa5, 0x49, 0x6f, 0x31, 0x8a, 0x80, 0x02, 0x0e, 0xe5, 0xc8, 0xd5, -+ 0xf9, 0xea, 0x8f, 0x3b, 0x8a, 0xde, 0xd9, 0x3f, 0x5e, 0x60, 0xbf, 0x9c, -+ 0xbb, 0x3b, 0x18, 0x78, 0x1a, 0xae, 0x70, 0xc9, 0xd5, 0x1e, 0x30, 0x56, -+ 0xd3, 0xff, 0xb2, 0xd8, 0x37, 0x3c, 0xc7, 0x0f, 0xfe, 0x27, 0xb3, 0xf4, -+ 0x19, 0x9a, 0x8f, 0xeb, 0x76, 0x8d, 0xfd, 0xcd, 0x9d, 0x0c, 0x42, 0x91, -+ 0xeb, 0x06, 0xa5, 0xc3, 0x56, 0x95, 0xff, 0x3e, 0xdd, 0x05, 0xaf, 0xd5, -+ 0xa1, 0xc4, 0x83, 0x8f, 0xb7, 0x1b, 0xdb, 0x48, 0x8c, 0xfe, 0x6b, 0x0d, -+ 0x0e, 0x92, 0x23, 0x70, 0x42, 0x6d, 0x95, 0x34, 0x58, 0x57, 0xd3, 0x58, -+ 0x40, 0xb8, 0x87, 0x6b, 0xc2, 0xf4, 0x1e, 0xed, 0xf3, 0x2d, 0x0b, 0x3e, -+ 0xa2, 0x32, 0xef, 0x8e, 0xfc, 0x54, 0x11, 0x43, 0xf3, 0xab, 0x7c, 0x49, -+ 0x8b, 0x9a, 0x02, 0x70, 0x05, 0x37, 0x24, 0x4e, 0xea, 0xe5, 0x90, 0xf0, -+ 0x49, 0x57, 0x8b, 0xd8, 0x2f, 0x69, 0x70, 0xa9, 0x82, 0xa5, 0x51, 0xc6, -+ 0xf5, 0x42, 0x63, 0xbb, 0x2c, 0xec, 0xfc, 0x78, 0xdb, 0x55, 0x2f, 0x61, -+ 0x45, 0xb7, 0x3c, 0x46, 0xe3, 0xaf, 0x16, 0x18, 0xad, 0xe4, 0x2e, 0x35, -+ 0x7e, 0xda, 0x01, 0xc1, 0x74, 0xf3, 0x6f, 0x02, 0x51, 0xe8, 0x3d, 0x1c, -+ 0x82, 0xf0, 0x1e, 0x81, - }; - -- alignas(16) uint64_t state[kStateSizeT]; -+ alignas(16) uint8_t state[Randen::kStateBytes]; - std::memset(state, 0, sizeof(state)); - - Randen r; - r.Generate(state); - -- auto id = std::begin(state); -- for (const auto& elem : kGolden) { -- EXPECT_EQ(elem, *id++); -- } -+ EXPECT_EQ(0, std::memcmp(state, kGolden, sizeof(state))); - } - - } // namespace diff --git a/debian/patches/big-endian-random4.diff b/debian/patches/big-endian-random4.diff deleted file mode 100644 index 15110401..00000000 --- a/debian/patches/big-endian-random4.diff +++ /dev/null @@ -1,71 +0,0 @@ -From: Milad Fa <46688537+miladfarca@users.noreply.github.com> -Subject: Fix Randen and PCG on Big Endian platforms (#1031) -Forwarded: https://github.com/abseil/abseil-cpp/pull/1031 -Origin: upstream, https://github.com/abseil/abseil-cpp/commit/022527c50e0e2bc937f9fa3c516e3e36cbba0845 - ---- a/absl/random/internal/explicit_seed_seq.h -+++ b/absl/random/internal/explicit_seed_seq.h -@@ -74,7 +74,7 @@ - template <typename OutIterator> - void generate(OutIterator begin, OutIterator end) { - for (size_t index = 0; begin != end; begin++) { -- *begin = state_.empty() ? 0 : little_endian::FromHost32(state_[index++]); -+ *begin = state_.empty() ? 0 : state_[index++]; - if (index >= state_.size()) { - index = 0; - } ---- a/absl/random/internal/randen_engine.h -+++ b/absl/random/internal/randen_engine.h -@@ -121,6 +121,13 @@ void reseed(SeedSequence& seq) { - const size_t requested_entropy = (entropy_size == 0) ? 8u : entropy_size; - std::fill(std::begin(buffer) + requested_entropy, std::end(buffer), 0); - seq.generate(std::begin(buffer), std::begin(buffer) + requested_entropy); -+#ifdef ABSL_IS_BIG_ENDIAN -+ // Randen expects the seed buffer to be in Little Endian; reverse it on -+ // Big Endian platforms. -+ for (sequence_result_type& e : buffer) { -+ e = absl::little_endian::FromHost(e); -+ } -+#endif - // The Randen paper suggests preferentially initializing even-numbered - // 128-bit vectors of the randen state (there are 16 such vectors). - // The seed data is merged into the state offset by 128-bits, which ---- a/absl/random/internal/randen_slow.cc -+++ b/absl/random/internal/randen_slow.cc -@@ -395,6 +395,23 @@ - } - } - -+// Enables native loads in the round loop by pre-swapping. -+inline ABSL_RANDOM_INTERNAL_ATTRIBUTE_ALWAYS_INLINE void SwapEndian( -+ absl::uint128* state) { -+#ifdef ABSL_IS_BIG_ENDIAN -+ for (uint32_t block = 0; block < RandenTraits::kFeistelBlocks; ++block) { -+ uint64_t new_lo = absl::little_endian::ToHost64( -+ static_cast<uint64_t>(state[block] >> 64)); -+ uint64_t new_hi = absl::little_endian::ToHost64( -+ static_cast<uint64_t>((state[block] << 64) >> 64)); -+ state[block] = (static_cast<absl::uint128>(new_hi) << 64) | new_lo; -+ } -+#else -+ // Avoid warning about unused variable. -+ (void)state; -+#endif -+} -+ - } // namespace - - namespace absl { -@@ -439,8 +456,12 @@ void RandenSlow::Generate(const void* keys_void, void* state_void) { - - const absl::uint128 prev_inner = state[0]; - -+ SwapEndian(state); -+ - Permute(state, keys); - -+ SwapEndian(state); -+ - // Ensure backtracking resistance. - *state ^= prev_inner; - } diff --git a/debian/patches/configure.diff b/debian/patches/configure.diff index 8138899f..ff0c64e9 100644 --- a/debian/patches/configure.diff +++ b/debian/patches/configure.diff @@ -13,22 +13,15 @@ Configure Abseil for Debian. - Enable upstream's hardened build mode. - - Disable Intel SSE2 on i386, since Debian supports some i386 processors - without that extension. Keep it enabled on amd64, since all amd64 processors - have it. - - - Disable Intel SSSE3 entirely, since no i386 processor supports it and Debian - supports amd64 processors without it. - --- a/CMake/AbseilHelpers.cmake +++ b/CMake/AbseilHelpers.cmake -@@ -263,7 +263,8 @@ +@@ -281,7 +281,8 @@ if(ABSL_ENABLE_INSTALL) set_target_properties(${_NAME} PROPERTIES OUTPUT_NAME "absl_${_NAME}" -- SOVERSION "2103.0.1" -+ SOVERSION 20210324 -+ VERSION "20210324.0.0" +- SOVERSION "2206.0.0" ++ SOVERSION 20220623 ++ VERSION "20220623.0.0" ) endif() else() @@ -74,8 +67,8 @@ Configure Abseil for Debian. // allowed. #define ABSL_OPTION_USE_INLINE_NAMESPACE 1 --#define ABSL_OPTION_INLINE_NAMESPACE_NAME lts_20210324 -+#define ABSL_OPTION_INLINE_NAMESPACE_NAME debian2 +-#define ABSL_OPTION_INLINE_NAMESPACE_NAME lts_20220623 ++#define ABSL_OPTION_INLINE_NAMESPACE_NAME debian3 // ABSL_OPTION_HARDENED // @@ -87,29 +80,3 @@ Configure Abseil for Debian. +#define ABSL_OPTION_HARDENED 1 #endif // ABSL_BASE_OPTIONS_H_ ---- a/absl/container/internal/have_sse.h -+++ b/absl/container/internal/have_sse.h -@@ -17,22 +17,14 @@ - #define ABSL_CONTAINER_INTERNAL_HAVE_SSE_H_ - - #ifndef ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSE2 --#if defined(__SSE2__) || \ -- (defined(_MSC_VER) && \ -- (defined(_M_X64) || (defined(_M_IX86) && _M_IX86_FP >= 2))) -+#if defined(__x86_64__) || (defined(_MSC_VER) && defined(_M_X64)) - #define ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSE2 1 - #else - #define ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSE2 0 - #endif - #endif - --#ifndef ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSSE3 --#ifdef __SSSE3__ --#define ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSSE3 1 --#else - #define ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSSE3 0 --#endif --#endif - - #if ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSSE3 && \ - !ABSL_INTERNAL_RAW_HASH_SET_HAVE_SSE2 diff --git a/debian/patches/cordrepring-typo.diff b/debian/patches/cordrepring-typo.diff deleted file mode 100644 index 6f21896b..00000000 --- a/debian/patches/cordrepring-typo.diff +++ /dev/null @@ -1,20 +0,0 @@ -From: Benjamin Barenblat <bbaren@google.com> -Subject: Fix typo in CordRepRing error message -Forwarded: yes -Applied-Upstream: https://github.com/abseil/abseil-cpp/commit/b97a1ecda869ca8754d467a56c50275cebfeb328 - -The author works at Google. Upstream applied this patch as Piper -revision 367481280 and exported it to GitHub; the Applied-Upstream URL -above points to the exported commit. - ---- a/absl/strings/internal/cord_rep_ring.cc -+++ b/absl/strings/internal/cord_rep_ring.cc -@@ -301,7 +301,7 @@ - if (offset >= child->length || entry_length > child->length - offset) { - output << "entry[" << head << "] has offset " << offset - << " and entry length " << entry_length -- << " which are outside of the childs length of " << child->length; -+ << " which are outside of the child's length of " << child->length; - return false; - } - diff --git a/debian/patches/cordz-info-statistics-test.diff b/debian/patches/cordz-info-statistics-test.diff new file mode 100644 index 00000000..6e0bfcce --- /dev/null +++ b/debian/patches/cordz-info-statistics-test.diff @@ -0,0 +1,11 @@ +--- a/absl/strings/internal/cordz_info_statistics_test.cc ++++ b/absl/strings/internal/cordz_info_statistics_test.cc +@@ -466,6 +466,8 @@ + } + + TEST(CordzInfoStatisticsTest, ThreadSafety) { ++ GTEST_SKIP() << "Skipping test; see https://bugs.debian.org/1018804"; ++ + Notification stop; + static constexpr int kNumThreads = 8; + int64_t sampled_node_count = 0; diff --git a/debian/patches/cpu-features.diff b/debian/patches/cpu-features.diff new file mode 100644 index 00000000..7610e6b1 --- /dev/null +++ b/debian/patches/cpu-features.diff @@ -0,0 +1,54 @@ +From: Benjamin Barenblat <bbaren@google.com> +Subject: Canonicalize supported CPU feature set +Forwarded: not-needed + +Explicitly set supported CPU features. + + - Disable Intel SSE and SSE2 on i386, since Debian supports some i386 + processors without those extensions. Keep them enabled on amd64, since all + amd64 processors have them. + + - Disable Intel SSSE3 entirely, since no i386 processor supports it and Debian + supports amd64 processors without it. + + - Disable NEON on armel and armhf, since no armel processor supports NEON and + Debian supports some armhf processors without it. Keep it enabled on arm64, + since all arm64 processors have it. + +--- a/absl/base/config.h ++++ b/absl/base/config.h +@@ -862,7 +862,7 @@ + // which architectures support the various x86 instruction sets. + #ifdef ABSL_INTERNAL_HAVE_SSE + #error ABSL_INTERNAL_HAVE_SSE cannot be directly set +-#elif defined(__SSE__) ++#elif defined(__x86_64__) + #define ABSL_INTERNAL_HAVE_SSE 1 + #elif defined(_M_X64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 1) + // MSVC only defines _M_IX86_FP for x86 32-bit code, and _M_IX86_FP >= 1 +@@ -877,7 +877,7 @@ + // which architectures support the various x86 instruction sets. + #ifdef ABSL_INTERNAL_HAVE_SSE2 + #error ABSL_INTERNAL_HAVE_SSE2 cannot be directly set +-#elif defined(__SSE2__) ++#elif defined(__x86_64__) + #define ABSL_INTERNAL_HAVE_SSE2 1 + #elif defined(_M_X64) || (defined(_M_IX86_FP) && _M_IX86_FP >= 2) + // MSVC only defines _M_IX86_FP for x86 32-bit code, and _M_IX86_FP >= 2 +@@ -898,15 +898,13 @@ + // by the CPU. + #ifdef ABSL_INTERNAL_HAVE_SSSE3 + #error ABSL_INTERNAL_HAVE_SSSE3 cannot be directly set +-#elif defined(__SSSE3__) +-#define ABSL_INTERNAL_HAVE_SSSE3 1 + #endif + + // ABSL_INTERNAL_HAVE_ARM_NEON is used for compile-time detection of NEON (ARM + // SIMD). + #ifdef ABSL_INTERNAL_HAVE_ARM_NEON + #error ABSL_INTERNAL_HAVE_ARM_NEON cannot be directly set +-#elif defined(__ARM_NEON) ++#elif defined(__aarch64__) + #define ABSL_INTERNAL_HAVE_ARM_NEON 1 + #endif + diff --git a/debian/patches/empty-flags-library.diff b/debian/patches/empty-flags-library.diff index fe1d3962..8c26a6d7 100644 --- a/debian/patches/empty-flags-library.diff +++ b/debian/patches/empty-flags-library.diff @@ -8,7 +8,7 @@ library header-only. --- a/absl/flags/CMakeLists.txt +++ b/absl/flags/CMakeLists.txt -@@ -197,8 +197,6 @@ +@@ -199,8 +199,6 @@ absl_cc_library( NAME flags diff --git a/debian/patches/float-rounding.diff b/debian/patches/float-rounding.diff deleted file mode 100644 index 005e08be..00000000 --- a/debian/patches/float-rounding.diff +++ /dev/null @@ -1,46 +0,0 @@ -From: Benjamin Barenblat <bbaren@google.com> -Subject: Round floats using round(x), not static_cast<int>(x + 0.5) -Forwarded: yes -Applied-Upstream: https://github.com/abseil/abseil-cpp/commit/d96e287417766deddbff2d01b96321288c59491e - -Adding 0.5 to an IEEE float may cause one bit of precision loss, which -is enough to change the result in certain cases. For example, - - static_cast<int>(std::round(0.49999999999999994)) == 0 - static_cast<int>(0.49999999999999994 + 0.5) == 1 - -The author works at Google. Upstream applied this patch as Piper -revision 369926519 and exported it to GitHub; the Applied-Upstream URL -above points to the exported commit. - ---- a/absl/time/duration_test.cc -+++ b/absl/time/duration_test.cc -@@ -1320,7 +1320,7 @@ TEST(Duration, SmallConversions) { - - EXPECT_EQ(absl::ZeroDuration(), absl::Seconds(0)); - // TODO(bww): Is the next one OK? -- EXPECT_EQ(absl::ZeroDuration(), absl::Seconds(0.124999999e-9)); -+ EXPECT_EQ(absl::ZeroDuration(), absl::Seconds(std::nextafter(0.125e-9, 0))); - EXPECT_EQ(absl::Nanoseconds(1) / 4, absl::Seconds(0.125e-9)); - EXPECT_EQ(absl::Nanoseconds(1) / 4, absl::Seconds(0.250e-9)); - EXPECT_EQ(absl::Nanoseconds(1) / 2, absl::Seconds(0.375e-9)); -@@ -1330,7 +1330,7 @@ TEST(Duration, SmallConversions) { - EXPECT_EQ(absl::Nanoseconds(1), absl::Seconds(0.875e-9)); - EXPECT_EQ(absl::Nanoseconds(1), absl::Seconds(1.000e-9)); - -- EXPECT_EQ(absl::ZeroDuration(), absl::Seconds(-0.124999999e-9)); -+ EXPECT_EQ(absl::ZeroDuration(), absl::Seconds(std::nextafter(-0.125e-9, 0))); - EXPECT_EQ(-absl::Nanoseconds(1) / 4, absl::Seconds(-0.125e-9)); - EXPECT_EQ(-absl::Nanoseconds(1) / 4, absl::Seconds(-0.250e-9)); - EXPECT_EQ(-absl::Nanoseconds(1) / 2, absl::Seconds(-0.375e-9)); ---- a/absl/time/time.h -+++ b/absl/time/time.h -@@ -1352,7 +1352,7 @@ - inline Duration MakePosDoubleDuration(double n) { - const int64_t int_secs = static_cast<int64_t>(n); - const uint32_t ticks = static_cast<uint32_t>( -- (n - static_cast<double>(int_secs)) * kTicksPerSecond + 0.5); -+ std::round((n - static_cast<double>(int_secs)) * kTicksPerSecond)); - return ticks < kTicksPerSecond - ? MakeDuration(int_secs, ticks) - : MakeDuration(int_secs + 1, ticks - kTicksPerSecond); diff --git a/debian/patches/float-tests-disable-i386.diff b/debian/patches/float-tests-disable-i386.diff deleted file mode 100644 index 0aef2805..00000000 --- a/debian/patches/float-tests-disable-i386.diff +++ /dev/null @@ -1,141 +0,0 @@ -From: Benjamin Barenblat <bbaren@google.com> -Subject: Skip floating-point edge-case tests when using an x87 -Forwarded: yes -Applied-Upstream: https://github.com/abseil/abseil-cpp/commit/311bbd2e50ea35e921a08186840d3b6ca279e880 - -32-bit Intel CPUs use 80-bit floats for intermediate values, which can -change the results of floating point computations from what we normally -expect. Identify tests that are sensitive to the x87, and skip them when -we’re on 32-bit Intel. - -The author works at Google. Upstream applied this patch as Piper -revision 378722613 and exported it to GitHub; the Applied-Upstream URL -above points to the exported commit. - ---- a/absl/random/beta_distribution_test.cc -+++ b/absl/random/beta_distribution_test.cc -@@ -15,6 +15,7 @@ - #include "absl/random/beta_distribution.h" - - #include <algorithm> -+#include <cfloat> - #include <cstddef> - #include <cstdint> - #include <iterator> -@@ -558,6 +559,14 @@ - // dependencies of the distribution change, such as RandU64ToDouble, then this - // is also likely to change. - TEST(BetaDistributionTest, AlgorithmBounds) { -+#if (defined(__i386__) || defined(_M_IX86)) && FLT_EVAL_METHOD != 0 -+ // We're using an x87-compatible FPU, and intermediate operations are -+ // performed with 80-bit floats. This produces slightly different results from -+ // what we expect below. -+ GTEST_SKIP() -+ << "Skipping the test because we detected x87 floating-point semantics"; -+#endif -+ - { - absl::random_internal::sequence_urbg urbg( - {0x7fbe76c8b4395800ull, 0x8000000000000000ull}); ---- a/absl/random/distributions_test.cc -+++ b/absl/random/distributions_test.cc -@@ -14,6 +14,7 @@ - - #include "absl/random/distributions.h" - -+#include <cfloat> - #include <cmath> - #include <cstdint> - #include <random> -@@ -224,6 +225,15 @@ - TEST_F(RandomDistributionsTest, UniformNonsenseRanges) { - // The ranges used in this test are undefined behavior. - // The results are arbitrary and subject to future changes. -+ -+#if (defined(__i386__) || defined(_M_IX86)) && FLT_EVAL_METHOD != 0 -+ // We're using an x87-compatible FPU, and intermediate operations can be -+ // performed with 80-bit floats. This produces slightly different results from -+ // what we expect below. -+ GTEST_SKIP() -+ << "Skipping the test because we detected x87 floating-point semantics"; -+#endif -+ - absl::InsecureBitGen gen; - - // <uint> ---- a/absl/random/exponential_distribution_test.cc -+++ b/absl/random/exponential_distribution_test.cc -@@ -15,6 +15,7 @@ - #include "absl/random/exponential_distribution.h" - - #include <algorithm> -+#include <cfloat> - #include <cmath> - #include <cstddef> - #include <cstdint> -@@ -384,6 +385,15 @@ - TEST(ExponentialDistributionTest, AlgorithmBounds) { - // Relies on absl::uniform_real_distribution, so some of these comments - // reference that. -+ -+#if (defined(__i386__) || defined(_M_IX86)) && FLT_EVAL_METHOD != 0 -+ // We're using an x87-compatible FPU, and intermediate operations can be -+ // performed with 80-bit floats. This produces slightly different results from -+ // what we expect below. -+ GTEST_SKIP() -+ << "Skipping the test because we detected x87 floating-point semantics"; -+#endif -+ - absl::exponential_distribution<double> dist; - - { ---- a/absl/random/uniform_real_distribution_test.cc -+++ b/absl/random/uniform_real_distribution_test.cc -@@ -14,6 +14,7 @@ - - #include "absl/random/uniform_real_distribution.h" - -+#include <cfloat> - #include <cmath> - #include <cstdint> - #include <iterator> -@@ -70,6 +71,14 @@ - TYPED_TEST_SUITE(UniformRealDistributionTest, RealTypes); - - TYPED_TEST(UniformRealDistributionTest, ParamSerializeTest) { -+#if (defined(__i386__) || defined(_M_IX86)) && FLT_EVAL_METHOD != 0 -+ // We're using an x87-compatible FPU, and intermediate operations are -+ // performed with 80-bit floats. This produces slightly different results from -+ // what we expect below. -+ GTEST_SKIP() -+ << "Skipping the test because we detected x87 floating-point semantics"; -+#endif -+ - using param_type = - typename absl::uniform_real_distribution<TypeParam>::param_type; - ---- a/absl/time/duration_test.cc -+++ b/absl/time/duration_test.cc -@@ -17,6 +17,7 @@ - #endif - - #include <chrono> // NOLINT(build/c++11) -+#include <cfloat> - #include <cmath> - #include <cstdint> - #include <ctime> -@@ -1390,6 +1391,14 @@ - // Seconds(point) returns a duration near point * Seconds(1.0). (They may - // not be exactly equal due to fused multiply/add contraction.) - TEST(Duration, ToDoubleSecondsCheckEdgeCases) { -+#if (defined(__i386__) || defined(_M_IX86)) && FLT_EVAL_METHOD != 0 -+ // We're using an x87-compatible FPU, and intermediate operations can be -+ // performed with 80-bit floats. This means the edge cases are different than -+ // what we expect here, so just skip this test. -+ GTEST_SKIP() -+ << "Skipping the test because we detected x87 floating-point semantics"; -+#endif -+ - constexpr uint32_t kTicksPerSecond = absl::time_internal::kTicksPerSecond; - constexpr auto duration_tick = absl::time_internal::MakeDuration(0, 1u); - int misses = 0; diff --git a/debian/patches/latomic.diff b/debian/patches/latomic.diff index 861f20bf..71cf7959 100644 --- a/debian/patches/latomic.diff +++ b/debian/patches/latomic.diff @@ -9,7 +9,7 @@ if anything needs libatomic and add the DT_NEEDED entry where necessary. --- a/absl/copts/AbseilConfigureCopts.cmake +++ b/absl/copts/AbseilConfigureCopts.cmake -@@ -62,4 +62,8 @@ +@@ -94,4 +94,8 @@ set(ABSL_TEST_COPTS "") endif() diff --git a/debian/patches/leaky-pkgconfig-cflags.diff b/debian/patches/leaky-pkgconfig-cflags.diff new file mode 100644 index 00000000..cf1479d4 --- /dev/null +++ b/debian/patches/leaky-pkgconfig-cflags.diff @@ -0,0 +1,16 @@ +From: Bruno Pitrus <brunopitrus@hotmail.com> +Subject: Do not leak -maes -msse4.1 into pkgconfig +Forwarded: https://github.com/abseil/abseil-cpp/pull/1216 +Origin: upstream, https://github.com/abseil/abseil-cpp/commit/09e96049995584c3489e4bd1467313e3e85af99c + +--- a/CMake/AbseilHelpers.cmake ++++ b/CMake/AbseilHelpers.cmake +@@ -166,6 +166,8 @@ + set(PC_CFLAGS "${PC_CFLAGS} ${cflag}") + elseif(${cflag} MATCHES "^(-W|/w[1234eo])") + # Don't impose our warnings on others. ++ elseif(${cflag} MATCHES "^-m") ++ # Don't impose CPU instruction requirements on others, as the code performs feature detection on runtime. + else() + set(PC_CFLAGS "${PC_CFLAGS} ${cflag}") + endif() diff --git a/debian/patches/missing-rint.diff b/debian/patches/missing-rint.diff deleted file mode 100644 index d9865364..00000000 --- a/debian/patches/missing-rint.diff +++ /dev/null @@ -1,45 +0,0 @@ -From: Benjamin Barenblat <bbaren@google.com> -Subject: Round a double multiplication before casting it to integer -Forwarded: yes -Applied-Upstream: https://github.com/abseil/abseil-cpp/commit/60be12ed9822078970f05f3c560324184302df6b - -The code - - static_cast<int>(x * y) - -(for double x and y) performs a double multiplication into a temporary -that, by standard, may have excess precision. The subsequent cast to int -discards the excess precision. However, the cast may examine the excess -precision during conversion, producing surprising results like - - static_cast<int>(1.7 * 10) == 16 - -on certain systems. Correct this case by explicitly rounding 1.7 * 10 -before casting it. - -The author works at Google. Upstream applied this patch as Piper -revision 378922064 and exported it to GitHub; the Applied-Upstream URL -above points to the exported commit. - ---- a/absl/random/mocking_bit_gen_test.cc -+++ b/absl/random/mocking_bit_gen_test.cc -@@ -15,6 +15,7 @@ - // - #include "absl/random/mocking_bit_gen.h" - -+#include <cmath> - #include <numeric> - #include <random> - -@@ -328,8 +329,9 @@ TEST(BasicMocking, WillByDefaultWithArgs) { - - absl::MockingBitGen gen; - ON_CALL(absl::MockPoisson<int>(), Call(gen, _)) -- .WillByDefault( -- [](double lambda) { return static_cast<int>(lambda * 10); }); -+ .WillByDefault([](double lambda) { -+ return static_cast<int>(std::rint(lambda * 10)); -+ }); - EXPECT_EQ(absl::Poisson<int>(gen, 1.7), 17); - EXPECT_EQ(absl::Poisson<int>(gen, 0.03), 0); - } diff --git a/debian/patches/series b/debian/patches/series index f68ea75c..277aba50 100644 --- a/debian/patches/series +++ b/debian/patches/series @@ -1,16 +1,6 @@ configure.diff +cpu-features.diff latomic.diff -arm-multiarch.diff empty-flags-library.diff -cordrepring-typo.diff -thumb-function-bounds.diff -float-rounding.diff -DiscreteDistributionTest-irrelevant-destination-buckets.diff -float-tests-disable-i386.diff -missing-rint.diff -big-endian-hash.diff -big-endian-hash2.diff -big-endian-random.diff -big-endian-random2.diff -big-endian-random3.diff -big-endian-random4.diff +leaky-pkgconfig-cflags.diff +cordz-info-statistics-test.diff diff --git a/debian/patches/thumb-function-bounds.diff b/debian/patches/thumb-function-bounds.diff deleted file mode 100644 index 1fd8c729..00000000 --- a/debian/patches/thumb-function-bounds.diff +++ /dev/null @@ -1,96 +0,0 @@ -From: Benjamin Barenblat <bbaren@google.com> -Subject: Correct Thumb function bound computation in the symbolizer -Forwarded: yes -Applied-Upstream: https://github.com/abseil/abseil-cpp/commit/1ae9b71c474628d60eb251a3f62967fe64151bb2 - -On 32-bit ARM, all functions are aligned to multiples of two bytes, and -the lowest-order bit in a function’s address is ignored by the CPU when -computing branch targets. That bit is still present in instructions and -ELF symbol tables, though; it’s repurposed to indicate whether the -function contains ARM or Thumb code. If the symbolizer doesn’t ignore -that bit, it will believe Thumb functions have boundaries that are off -by one byte, so instruct the symbolizer to null out the lowest-order bit -after retrieving it from the symbol table. - -The author works at Google. Upstream applied this patch as Piper -revision 369254082 and exported it to GitHub; the Applied-Upstream URL -above points to the exported commit. - ---- a/absl/debugging/symbolize_elf.inc -+++ b/absl/debugging/symbolize_elf.inc -@@ -701,6 +701,16 @@ - const char *start_address = - ComputeOffset(original_start_address, relocation); - -+#ifdef __arm__ -+ // ARM functions are always aligned to multiples of two bytes; the -+ // lowest-order bit in start_address is ignored by the CPU and indicates -+ // whether the function contains ARM (0) or Thumb (1) code. We don't care -+ // about what encoding is being used; we just want the real start address -+ // of the function. -+ start_address = reinterpret_cast<const char *>( -+ reinterpret_cast<uintptr_t>(start_address) & ~1); -+#endif -+ - if (deref_function_descriptor_pointer && - InSection(original_start_address, opd)) { - // The opd section is mapped into memory. Just dereference ---- a/absl/debugging/symbolize_test.cc -+++ b/absl/debugging/symbolize_test.cc -@@ -477,6 +477,46 @@ - #endif - } - -+#if defined(__arm__) && ABSL_HAVE_ATTRIBUTE(target) -+// Test that we correctly identify bounds of Thumb functions on ARM. -+// -+// Thumb functions have the lowest-order bit set in their addresses in the ELF -+// symbol table. This requires some extra logic to properly compute function -+// bounds. To test this logic, nudge a Thumb function right up against an ARM -+// function and try to symbolize the ARM function. -+// -+// A naive implementation will simply use the Thumb function's entry point as -+// written in the symbol table and will therefore treat the Thumb function as -+// extending one byte further in the instruction stream than it actually does. -+// When asked to symbolize the start of the ARM function, it will identify an -+// overlap between the Thumb and ARM functions, and it will return the name of -+// the Thumb function. -+// -+// A correct implementation, on the other hand, will null out the lowest-order -+// bit in the Thumb function's entry point. It will correctly compute the end of -+// the Thumb function, it will find no overlap between the Thumb and ARM -+// functions, and it will return the name of the ARM function. -+ -+__attribute__((target("thumb"))) int ArmThumbOverlapThumb(int x) { -+ return x * x * x; -+} -+ -+__attribute__((target("arm"))) int ArmThumbOverlapArm(int x) { -+ return x * x * x; -+} -+ -+void ABSL_ATTRIBUTE_NOINLINE TestArmThumbOverlap() { -+#if defined(ABSL_HAVE_ATTRIBUTE_NOINLINE) -+ const char *symbol = TrySymbolize((void *)&ArmThumbOverlapArm); -+ ABSL_RAW_CHECK(symbol != nullptr, "TestArmThumbOverlap failed"); -+ ABSL_RAW_CHECK(strcmp("ArmThumbOverlapArm()", symbol) == 0, -+ "TestArmThumbOverlap failed"); -+ std::cout << "TestArmThumbOverlap passed" << std::endl; -+#endif -+} -+ -+#endif // defined(__arm__) && ABSL_HAVE_ATTRIBUTE(target) -+ - #elif defined(_WIN32) - #if !defined(ABSL_CONSUME_DLL) - -@@ -551,6 +591,9 @@ - TestWithPCInsideInlineFunction(); - TestWithPCInsideNonInlineFunction(); - TestWithReturnAddress(); -+#if defined(__arm__) && ABSL_HAVE_ATTRIBUTE(target) -+ TestArmThumbOverlap(); -+#endif - #endif - - return RUN_ALL_TESTS(); diff --git a/debian/rules b/debian/rules index ef27843d..0dbb698e 100755 --- a/debian/rules +++ b/debian/rules @@ -19,9 +19,12 @@ export DEB_BUILD_MAINT_OPTIONS = hardening=+bindnow reproducible=+fixfilepath # Unit tests require more than 2 GB of RAM, so disable them on mipsel. # -# Unit tests are not yet passing on hppa or ppc64, so disable them there as -# well. -ifneq ($(filter $(DEB_HOST_ARCH),hppa mipsel ppc64),) +# Unit tests are not yet passing on ppc64, so disable them there as well. +# +# Disable unit tests unconditionally if nocheck is set. +ifneq ($(filter $(DEB_HOST_ARCH),mipsel ppc64),) +ABSL_RUN_TESTS=OFF +else ifneq ($(filter nocheck,$(DEB_BUILD_OPTIONS)),) ABSL_RUN_TESTS=OFF else ABSL_RUN_TESTS=ON @@ -35,11 +38,11 @@ override_dh_auto_clean: $(RM) -r $(CURDIR)/shared override_dh_auto_configure: - dh_auto_configure -Bstatic -- -DCMAKE_CXX_STANDARD=14 -DBUILD_SHARED_LIBS=OFF + dh_auto_configure -Bstatic -- -DCMAKE_CXX_STANDARD=17 -DBUILD_SHARED_LIBS=OFF ifeq ($(ABSL_RUN_TESTS),ON) - dh_auto_configure -Bshared -- -DCMAKE_CXX_STANDARD=14 -DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=ON -DABSL_USE_GOOGLETEST_HEAD=OFF + dh_auto_configure -Bshared -- -DCMAKE_CXX_STANDARD=17 -DBUILD_SHARED_LIBS=ON -DBUILD_TESTING=ON -DABSL_BUILD_TESTING=ON -DABSL_USE_GOOGLETEST_HEAD=OFF else - dh_auto_configure -Bshared -- -DCMAKE_CXX_STANDARD=14 -DBUILD_SHARED_LIBS=ON + dh_auto_configure -Bshared -- -DCMAKE_CXX_STANDARD=17 -DBUILD_SHARED_LIBS=ON endif override_dh_auto_build: diff --git a/debian/tests/bug1011294 b/debian/tests/bug1011294 new file mode 100755 index 00000000..302bcf73 --- /dev/null +++ b/debian/tests/bug1011294 @@ -0,0 +1,24 @@ +#!/bin/sh +# Copyright 2022 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not +# use this file except in compliance with the License. You may obtain a copy of +# the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations under +# the License. + +set -eu + +readonly TMP="$(mktemp -d)" +trap "rm -rf \"$TMP\"" EXIT +cd "$TMP" + +echo 'int main() {}' >noop.cc + +g++ -o noop noop.cc $(pkg-config --cflags absl_base) $(pkg-config --libs absl_base) diff --git a/debian/tests/cmake b/debian/tests/cmake index 1eb04de9..43eb3408 100755 --- a/debian/tests/cmake +++ b/debian/tests/cmake @@ -32,7 +32,7 @@ EOF cat >CMakeLists.txt <<EOF cmake_minimum_required(VERSION 3.5) project(test CXX) -set(CMAKE_CXX_STANDARD 14) +set(CMAKE_CXX_STANDARD 17) add_executable(test test.cc) find_package(absl REQUIRED) target_link_libraries(test absl::strings) diff --git a/debian/tests/control b/debian/tests/control index 43dc87cb..161ef695 100644 --- a/debian/tests/control +++ b/debian/tests/control @@ -17,3 +17,6 @@ Depends: @, g++, libgtest-dev Tests: cmake Depends: @, cmake (>= 3.5), g++, make + +Tests: bug1011294 +Depends: @, g++, pkg-config |