diff options
| -rw-r--r-- | BUILD | 6 | ||||
| -rw-r--r-- | Makefile | 38 | ||||
| -rw-r--r-- | binding.gyp | 1 | ||||
| -rw-r--r-- | build.yaml | 12 | ||||
| -rw-r--r-- | gRPC.podspec | 3 | ||||
| -rwxr-xr-x | grpc.gemspec | 2 | ||||
| -rw-r--r-- | package.json | 2 | ||||
| -rw-r--r-- | src/core/census/log.c | 610 | ||||
| -rw-r--r-- | src/core/census/log.h | 93 | ||||
| -rw-r--r-- | src/python/grpcio/grpc_core_dependencies.py | 1 | ||||
| -rw-r--r-- | test/core/census/log_test.c | 585 | ||||
| -rw-r--r-- | tools/doxygen/Doxyfile.core.internal | 2 | ||||
| -rw-r--r-- | tools/run_tests/sources_and_headers.json | 20 | ||||
| -rw-r--r-- | tools/run_tests/tests.json | 20 | ||||
| -rw-r--r-- | vsprojects/buildtests_c.sln | 27 | ||||
| -rw-r--r-- | vsprojects/vcxproj/grpc/grpc.vcxproj | 3 | ||||
| -rw-r--r-- | vsprojects/vcxproj/grpc/grpc.vcxproj.filters | 6 | ||||
| -rw-r--r-- | vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj | 3 | ||||
| -rw-r--r-- | vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters | 6 | ||||
| -rw-r--r-- | vsprojects/vcxproj/test/census_log_test/census_log_test.vcxproj | 199 | ||||
| -rw-r--r-- | vsprojects/vcxproj/test/census_log_test/census_log_test.vcxproj.filters | 21 |
21 files changed, 1660 insertions, 0 deletions
@@ -288,6 +288,7 @@ cc_library( "src/core/transport/transport_impl.h", "src/core/census/aggregation.h", "src/core/census/context.h", + "src/core/census/log.h", "src/core/census/rpc_metric_id.h", "src/core/httpcli/httpcli_security_connector.c", "src/core/security/base64.c", @@ -439,6 +440,7 @@ cc_library( "src/core/transport/transport_op_string.c", "src/core/census/context.c", "src/core/census/initialize.c", + "src/core/census/log.c", "src/core/census/operation.c", "src/core/census/placeholders.c", "src/core/census/tag_set.c", @@ -586,6 +588,7 @@ cc_library( "src/core/transport/transport_impl.h", "src/core/census/aggregation.h", "src/core/census/context.h", + "src/core/census/log.h", "src/core/census/rpc_metric_id.h", "src/core/surface/init_unsecure.c", "src/core/census/grpc_context.c", @@ -717,6 +720,7 @@ cc_library( "src/core/transport/transport_op_string.c", "src/core/census/context.c", "src/core/census/initialize.c", + "src/core/census/log.c", "src/core/census/operation.c", "src/core/census/placeholders.c", "src/core/census/tag_set.c", @@ -1394,6 +1398,7 @@ objc_library( "src/core/transport/transport_op_string.c", "src/core/census/context.c", "src/core/census/initialize.c", + "src/core/census/log.c", "src/core/census/operation.c", "src/core/census/placeholders.c", "src/core/census/tag_set.c", @@ -1536,6 +1541,7 @@ objc_library( "src/core/transport/transport_impl.h", "src/core/census/aggregation.h", "src/core/census/context.h", + "src/core/census/log.h", "src/core/census/rpc_metric_id.h", ], includes = [ @@ -829,6 +829,7 @@ algorithm_test: $(BINDIR)/$(CONFIG)/algorithm_test alloc_test: $(BINDIR)/$(CONFIG)/alloc_test alpn_test: $(BINDIR)/$(CONFIG)/alpn_test bin_encoder_test: $(BINDIR)/$(CONFIG)/bin_encoder_test +census_log_test: $(BINDIR)/$(CONFIG)/census_log_test channel_create_test: $(BINDIR)/$(CONFIG)/channel_create_test chttp2_hpack_encoder_test: $(BINDIR)/$(CONFIG)/chttp2_hpack_encoder_test chttp2_status_conversion_test: $(BINDIR)/$(CONFIG)/chttp2_status_conversion_test @@ -1138,6 +1139,7 @@ buildtests_c: privatelibs_c \ $(BINDIR)/$(CONFIG)/alloc_test \ $(BINDIR)/$(CONFIG)/alpn_test \ $(BINDIR)/$(CONFIG)/bin_encoder_test \ + $(BINDIR)/$(CONFIG)/census_log_test \ $(BINDIR)/$(CONFIG)/channel_create_test \ $(BINDIR)/$(CONFIG)/chttp2_hpack_encoder_test \ $(BINDIR)/$(CONFIG)/chttp2_status_conversion_test \ @@ -1371,6 +1373,8 @@ test_c: buildtests_c $(Q) $(BINDIR)/$(CONFIG)/alpn_test || ( echo test alpn_test failed ; exit 1 ) $(E) "[RUN] Testing bin_encoder_test" $(Q) $(BINDIR)/$(CONFIG)/bin_encoder_test || ( echo test bin_encoder_test failed ; exit 1 ) + $(E) "[RUN] Testing census_log_test" + $(Q) $(BINDIR)/$(CONFIG)/census_log_test || ( echo test census_log_test failed ; exit 1 ) $(E) "[RUN] Testing channel_create_test" $(Q) $(BINDIR)/$(CONFIG)/channel_create_test || ( echo test channel_create_test failed ; exit 1 ) $(E) "[RUN] Testing chttp2_hpack_encoder_test" @@ -2510,6 +2514,7 @@ LIBGRPC_SRC = \ src/core/transport/transport_op_string.c \ src/core/census/context.c \ src/core/census/initialize.c \ + src/core/census/log.c \ src/core/census/operation.c \ src/core/census/placeholders.c \ src/core/census/tag_set.c \ @@ -2816,6 +2821,7 @@ LIBGRPC_UNSECURE_SRC = \ src/core/transport/transport_op_string.c \ src/core/census/context.c \ src/core/census/initialize.c \ + src/core/census/log.c \ src/core/census/operation.c \ src/core/census/placeholders.c \ src/core/census/tag_set.c \ @@ -5919,6 +5925,38 @@ endif endif +CENSUS_LOG_TEST_SRC = \ + test/core/census/log_test.c \ + +CENSUS_LOG_TEST_OBJS = $(addprefix $(OBJDIR)/$(CONFIG)/, $(addsuffix .o, $(basename $(CENSUS_LOG_TEST_SRC)))) +ifeq ($(NO_SECURE),true) + +# You can't build secure targets if you don't have OpenSSL. + +$(BINDIR)/$(CONFIG)/census_log_test: openssl_dep_error + +else + + + +$(BINDIR)/$(CONFIG)/census_log_test: $(CENSUS_LOG_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + $(E) "[LD] Linking $@" + $(Q) mkdir -p `dirname $@` + $(Q) $(LD) $(LDFLAGS) $(CENSUS_LOG_TEST_OBJS) $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a $(LDLIBS) $(LDLIBS_SECURE) -o $(BINDIR)/$(CONFIG)/census_log_test + +endif + +$(OBJDIR)/$(CONFIG)/test/core/census/log_test.o: $(LIBDIR)/$(CONFIG)/libgrpc_test_util.a $(LIBDIR)/$(CONFIG)/libgrpc.a $(LIBDIR)/$(CONFIG)/libgpr_test_util.a $(LIBDIR)/$(CONFIG)/libgpr.a + +deps_census_log_test: $(CENSUS_LOG_TEST_OBJS:.o=.dep) + +ifneq ($(NO_SECURE),true) +ifneq ($(NO_DEPS),true) +-include $(CENSUS_LOG_TEST_OBJS:.o=.dep) +endif +endif + + CHANNEL_CREATE_TEST_SRC = \ test/core/surface/channel_create_test.c \ diff --git a/binding.gyp b/binding.gyp index 133f8f4862..f1c4ef0b4f 100644 --- a/binding.gyp +++ b/binding.gyp @@ -700,6 +700,7 @@ 'src/core/transport/transport_op_string.c', 'src/core/census/context.c', 'src/core/census/initialize.c', + 'src/core/census/log.c', 'src/core/census/operation.c', 'src/core/census/placeholders.c', 'src/core/census/tag_set.c', diff --git a/build.yaml b/build.yaml index 2209605f24..fe053b0917 100644 --- a/build.yaml +++ b/build.yaml @@ -15,10 +15,12 @@ filegroups: headers: - src/core/census/aggregation.h - src/core/census/context.h + - src/core/census/log.h - src/core/census/rpc_metric_id.h src: - src/core/census/context.c - src/core/census/initialize.c + - src/core/census/log.c - src/core/census/operation.c - src/core/census/placeholders.c - src/core/census/tag_set.c @@ -935,6 +937,16 @@ targets: - grpc - gpr_test_util - gpr +- name: census_log_test + build: test + language: c + src: + - test/core/census/log_test.c + deps: + - grpc_test_util + - grpc + - gpr_test_util + - gpr - name: channel_create_test build: test language: c diff --git a/gRPC.podspec b/gRPC.podspec index 30b278798a..fe7c8c96e5 100644 --- a/gRPC.podspec +++ b/gRPC.podspec @@ -292,6 +292,7 @@ Pod::Spec.new do |s| 'src/core/transport/transport_impl.h', 'src/core/census/aggregation.h', 'src/core/census/context.h', + 'src/core/census/log.h', 'src/core/census/rpc_metric_id.h', 'include/grpc/grpc_security.h', 'include/grpc/byte_buffer.h', @@ -450,6 +451,7 @@ Pod::Spec.new do |s| 'src/core/transport/transport_op_string.c', 'src/core/census/context.c', 'src/core/census/initialize.c', + 'src/core/census/log.c', 'src/core/census/operation.c', 'src/core/census/placeholders.c', 'src/core/census/tag_set.c', @@ -594,6 +596,7 @@ Pod::Spec.new do |s| 'src/core/transport/transport_impl.h', 'src/core/census/aggregation.h', 'src/core/census/context.h', + 'src/core/census/log.h', 'src/core/census/rpc_metric_id.h' ss.header_mappings_dir = '.' diff --git a/grpc.gemspec b/grpc.gemspec index 585d3ec909..e4a0a49676 100755 --- a/grpc.gemspec +++ b/grpc.gemspec @@ -282,6 +282,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/transport/transport_impl.h ) s.files += %w( src/core/census/aggregation.h ) s.files += %w( src/core/census/context.h ) + s.files += %w( src/core/census/log.h ) s.files += %w( src/core/census/rpc_metric_id.h ) s.files += %w( src/core/httpcli/httpcli_security_connector.c ) s.files += %w( src/core/security/base64.c ) @@ -433,6 +434,7 @@ Gem::Specification.new do |s| s.files += %w( src/core/transport/transport_op_string.c ) s.files += %w( src/core/census/context.c ) s.files += %w( src/core/census/initialize.c ) + s.files += %w( src/core/census/log.c ) s.files += %w( src/core/census/operation.c ) s.files += %w( src/core/census/placeholders.c ) s.files += %w( src/core/census/tag_set.c ) diff --git a/package.json b/package.json index 0a3440e6e4..f9bc5b9bb3 100644 --- a/package.json +++ b/package.json @@ -222,6 +222,7 @@ "src/core/transport/transport_impl.h", "src/core/census/aggregation.h", "src/core/census/context.h", + "src/core/census/log.h", "src/core/census/rpc_metric_id.h", "src/core/httpcli/httpcli_security_connector.c", "src/core/security/base64.c", @@ -373,6 +374,7 @@ "src/core/transport/transport_op_string.c", "src/core/census/context.c", "src/core/census/initialize.c", + "src/core/census/log.c", "src/core/census/operation.c", "src/core/census/placeholders.c", "src/core/census/tag_set.c", diff --git a/src/core/census/log.c b/src/core/census/log.c new file mode 100644 index 0000000000..a76f51036a --- /dev/null +++ b/src/core/census/log.c @@ -0,0 +1,610 @@ +/* + * + * Copyright 2015-2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +/* Available log space is divided up in blocks of + CENSUS_LOG_2_MAX_RECORD_SIZE bytes. A block can be in one of the + following three data structures: + - Free blocks (free_block_list) + - Blocks with unread data (dirty_block_list) + - Blocks currently attached to cores (core_local_blocks[]) + + census_log_start_write() moves a block from core_local_blocks[] to the + end of dirty_block_list when block: + - is out-of-space OR + - has an incomplete record (an incomplete record occurs when a thread calls + census_log_start_write() and is context-switched before calling + census_log_end_write() + So, blocks in dirty_block_list are ordered, from oldest to newest, by time + when block is detached from the core. + + census_log_read_next() first iterates over dirty_block_list and then + core_local_blocks[]. It moves completely read blocks from dirty_block_list + to free_block_list. Blocks in core_local_blocks[] are not freed, even when + completely read. + + If log is configured to discard old records and free_block_list is empty, + census_log_start_write() iterates over dirty_block_list to allocate a + new block. It moves the oldest available block (no pending read/write) to + core_local_blocks[]. + + core_local_block_struct is used to implement a map from core id to the block + associated with that core. This mapping is advisory. It is possible that the + block returned by this mapping is no longer associated with that core. This + mapping is updated, lazily, by census_log_start_write(). + + Locking in block struct: + + Exclusive g_log.lock must be held before calling any functions operatong on + block structs except census_log_start_write() and + census_log_end_write(). + + Writes to a block are serialized via writer_lock. + census_log_start_write() acquires this lock and + census_log_end_write() releases it. On failure to acquire the lock, + writer allocates a new block for the current core and updates + core_local_block accordingly. + + Simultaneous read and write access is allowed. Readers can safely read up to + committed bytes (bytes_committed). + + reader_lock protects the block, currently being read, from getting recycled. + start_read() acquires reader_lock and end_read() releases the lock. + + Read/write access to a block is disabled via try_disable_access(). It returns + with both writer_lock and reader_lock held. These locks are subsequently + released by enable_access() to enable access to the block. + + A note on naming: Most function/struct names are prepended by cl_ + (shorthand for census_log). Further, functions that manipulate structures + include the name of the structure, which will be passed as the first + argument. E.g. cl_block_initialize() will initialize a cl_block. +*/ + +#include "src/core/census/log.h" +#include <grpc/support/alloc.h> +#include <grpc/support/atm.h> +#include <grpc/support/cpu.h> +#include <grpc/support/log.h> +#include <grpc/support/sync.h> +#include <grpc/support/useful.h> +#include <stdbool.h> +#include <string.h> + +/* End of platform specific code */ + +typedef struct census_log_block_list_struct { + struct census_log_block_list_struct* next; + struct census_log_block_list_struct* prev; + struct census_log_block* block; +} cl_block_list_struct; + +typedef struct census_log_block { + /* Pointer to underlying buffer */ + char* buffer; + gpr_atm writer_lock; + gpr_atm reader_lock; + /* Keeps completely written bytes. Declared atomic because accessed + simultaneously by reader and writer. */ + gpr_atm bytes_committed; + /* Bytes already read */ + size_t bytes_read; + /* Links for list */ + cl_block_list_struct link; +/* We want this structure to be cacheline aligned. We assume the following + sizes for the various parts on 32/64bit systems: + type 32b size 64b size + char* 4 8 + 3x gpr_atm 12 24 + size_t 4 8 + cl_block_list_struct 12 24 + TOTAL 32 64 + + Depending on the size of our cacheline and the architecture, we + selectively add char buffering to this structure. The size is checked + via assert in census_log_initialize(). */ +#if defined(GPR_ARCH_64) +#define CL_BLOCK_PAD_SIZE (GPR_CACHELINE_SIZE - 64) +#else +#if defined(GPR_ARCH_32) +#define CL_BLOCK_PAD_SIZE (GPR_CACHELINE_SIZE - 32) +#else +#error "Unknown architecture" +#endif +#endif +#if CL_BLOCK_PAD_SIZE > 0 + char padding[CL_BLOCK_PAD_SIZE]; +#endif +} cl_block; + +/* A list of cl_blocks, doubly-linked through cl_block::link. */ +typedef struct census_log_block_list { + int32_t count; /* Number of items in list. */ + cl_block_list_struct ht; /* head/tail of linked list. */ +} cl_block_list; + +/* Cacheline aligned block pointers to avoid false sharing. Block pointer must + be initialized via set_block(), before calling other functions */ +typedef struct census_log_core_local_block { + gpr_atm block; +/* Ensure cachline alignment: we assume sizeof(gpr_atm) == 4 or 8 */ +#if defined(GPR_ARCH_64) +#define CL_CORE_LOCAL_BLOCK_PAD_SIZE (GPR_CACHELINE_SIZE - 8) +#else +#if defined(GPR_ARCH_32) +#define CL_CORE_LOCAL_BLOCK_PAD_SIZE (GPR_CACHELINE_SIZE - 4) +#else +#error "Unknown architecture" +#endif +#endif +#if CL_CORE_LOCAL_BLOCK_PAD_SIZE > 0 + char padding[CL_CORE_LOCAL_BLOCK_PAD_SIZE]; +#endif +} cl_core_local_block; + +struct census_log { + int discard_old_records; + /* Number of cores (aka hardware-contexts) */ + unsigned num_cores; + /* number of CENSUS_LOG_2_MAX_RECORD_SIZE blocks in log */ + uint32_t num_blocks; + cl_block* blocks; /* Block metadata. */ + cl_core_local_block* core_local_blocks; /* Keeps core to block mappings. */ + gpr_mu lock; + int initialized; /* has log been initialized? */ + /* Keeps the state of the reader iterator. A value of 0 indicates that + iterator has reached the end. census_log_init_reader() resets the + value to num_core to restart iteration. */ + uint32_t read_iterator_state; + /* Points to the block being read. If non-NULL, the block is locked for + reading (block_being_read_->reader_lock is held). */ + cl_block* block_being_read; + /* A non-zero value indicates that log is full. */ + gpr_atm is_full; + char* buffer; + cl_block_list free_block_list; + cl_block_list dirty_block_list; + gpr_atm out_of_space_count; +}; + +/* Single internal log */ +static struct census_log g_log; + +/* Functions that operate on an atomic memory location used as a lock */ + +/* Returns non-zero if lock is acquired */ +static int cl_try_lock(gpr_atm* lock) { return gpr_atm_acq_cas(lock, 0, 1); } + +static void cl_unlock(gpr_atm* lock) { gpr_atm_rel_store(lock, 0); } + +/* Functions that operate on cl_core_local_block's */ + +static void cl_core_local_block_set_block(cl_core_local_block* clb, + cl_block* block) { + gpr_atm_rel_store(&clb->block, (gpr_atm)block); +} + +static cl_block* cl_core_local_block_get_block(cl_core_local_block* clb) { + return (cl_block*)gpr_atm_acq_load(&clb->block); +} + +/* Functions that operate on cl_block_list_struct's */ + +static void cl_block_list_struct_initialize(cl_block_list_struct* bls, + cl_block* block) { + bls->next = bls->prev = bls; + bls->block = block; +} + +/* Functions that operate on cl_block_list's */ + +static void cl_block_list_initialize(cl_block_list* list) { + list->count = 0; + cl_block_list_struct_initialize(&list->ht, NULL); +} + +/* Returns head of *this, or NULL if empty. */ +static cl_block* cl_block_list_head(cl_block_list* list) { + return list->ht.next->block; +} + +/* Insert element *e after *pos. */ +static void cl_block_list_insert(cl_block_list* list, cl_block_list_struct* pos, + cl_block_list_struct* e) { + list->count++; + e->next = pos->next; + e->prev = pos; + e->next->prev = e; + e->prev->next = e; +} + +/* Insert block at the head of the list */ +static void cl_block_list_insert_at_head(cl_block_list* list, cl_block* block) { + cl_block_list_insert(list, &list->ht, &block->link); +} + +/* Insert block at the tail of the list */ +static void cl_block_list_insert_at_tail(cl_block_list* list, cl_block* block) { + cl_block_list_insert(list, list->ht.prev, &block->link); +} + +/* Removes block *b. Requires *b be in the list. */ +static void cl_block_list_remove(cl_block_list* list, cl_block* b) { + list->count--; + b->link.next->prev = b->link.prev; + b->link.prev->next = b->link.next; +} + +/* Functions that operate on cl_block's */ + +static void cl_block_initialize(cl_block* block, char* buffer) { + block->buffer = buffer; + gpr_atm_rel_store(&block->writer_lock, 0); + gpr_atm_rel_store(&block->reader_lock, 0); + gpr_atm_rel_store(&block->bytes_committed, 0); + block->bytes_read = 0; + cl_block_list_struct_initialize(&block->link, block); +} + +/* Guards against exposing partially written buffer to the reader. */ +static void cl_block_set_bytes_committed(cl_block* block, + size_t bytes_committed) { + gpr_atm_rel_store(&block->bytes_committed, (gpr_atm)bytes_committed); +} + +static size_t cl_block_get_bytes_committed(cl_block* block) { + return (size_t)gpr_atm_acq_load(&block->bytes_committed); +} + +/* Tries to disable future read/write access to this block. Succeeds if: + - no in-progress write AND + - no in-progress read AND + - 'discard_data' set to true OR no unread data + On success, clears the block state and returns with writer_lock_ and + reader_lock_ held. These locks are released by a subsequent + cl_block_access_enable() call. */ +static bool cl_block_try_disable_access(cl_block* block, int discard_data) { + if (!cl_try_lock(&block->writer_lock)) { + return false; + } + if (!cl_try_lock(&block->reader_lock)) { + cl_unlock(&block->writer_lock); + return false; + } + if (!discard_data && + (block->bytes_read != cl_block_get_bytes_committed(block))) { + cl_unlock(&block->reader_lock); + cl_unlock(&block->writer_lock); + return false; + } + cl_block_set_bytes_committed(block, 0); + block->bytes_read = 0; + return true; +} + +static void cl_block_enable_access(cl_block* block) { + cl_unlock(&block->reader_lock); + cl_unlock(&block->writer_lock); +} + +/* Returns with writer_lock held. */ +static void* cl_block_start_write(cl_block* block, size_t size) { + size_t bytes_committed; + if (!cl_try_lock(&block->writer_lock)) { + return NULL; + } + bytes_committed = cl_block_get_bytes_committed(block); + if (bytes_committed + size > CENSUS_LOG_MAX_RECORD_SIZE) { + cl_unlock(&block->writer_lock); + return NULL; + } + return block->buffer + bytes_committed; +} + +/* Releases writer_lock and increments committed bytes by 'bytes_written'. + 'bytes_written' must be <= 'size' specified in the corresponding + StartWrite() call. This function is thread-safe. */ +static void cl_block_end_write(cl_block* block, size_t bytes_written) { + cl_block_set_bytes_committed( + block, cl_block_get_bytes_committed(block) + bytes_written); + cl_unlock(&block->writer_lock); +} + +/* Returns a pointer to the first unread byte in buffer. The number of bytes + available are returned in 'bytes_available'. Acquires reader lock that is + released by a subsequent cl_block_end_read() call. Returns NULL if: + - read in progress + - no data available */ +static void* cl_block_start_read(cl_block* block, size_t* bytes_available) { + if (!cl_try_lock(&block->reader_lock)) { + return NULL; + } + /* bytes_committed may change from under us. Use bytes_available to update + bytes_read below. */ + size_t bytes_committed = cl_block_get_bytes_committed(block); + GPR_ASSERT(bytes_committed >= block->bytes_read); + *bytes_available = bytes_committed - block->bytes_read; + if (*bytes_available == 0) { + cl_unlock(&block->reader_lock); + return NULL; + } + void* record = block->buffer + block->bytes_read; + block->bytes_read += *bytes_available; + return record; +} + +static void cl_block_end_read(cl_block* block) { + cl_unlock(&block->reader_lock); +} + +/* Internal functions operating on g_log */ + +/* Allocates a new free block (or recycles an available dirty block if log is + configured to discard old records). Returns NULL if out-of-space. */ +static cl_block* cl_allocate_block(void) { + cl_block* block = cl_block_list_head(&g_log.free_block_list); + if (block != NULL) { + cl_block_list_remove(&g_log.free_block_list, block); + return block; + } + if (!g_log.discard_old_records) { + /* No free block and log is configured to keep old records. */ + return NULL; + } + /* Recycle dirty block. Start from the oldest. */ + for (block = cl_block_list_head(&g_log.dirty_block_list); block != NULL; + block = block->link.next->block) { + if (cl_block_try_disable_access(block, 1 /* discard data */)) { + cl_block_list_remove(&g_log.dirty_block_list, block); + return block; + } + } + return NULL; +} + +/* Allocates a new block and updates core id => block mapping. 'old_block' + points to the block that the caller thinks is attached to + 'core_id'. 'old_block' may be NULL. Returns true if: + - allocated a new block OR + - 'core_id' => 'old_block' mapping changed (another thread allocated a + block before lock was acquired). */ +static bool cl_allocate_core_local_block(uint32_t core_id, + cl_block* old_block) { + /* Now that we have the lock, check if core-local mapping has changed. */ + cl_core_local_block* core_local_block = &g_log.core_local_blocks[core_id]; + cl_block* block = cl_core_local_block_get_block(core_local_block); + if ((block != NULL) && (block != old_block)) { + return true; + } + if (block != NULL) { + cl_core_local_block_set_block(core_local_block, NULL); + cl_block_list_insert_at_tail(&g_log.dirty_block_list, block); + } + block = cl_allocate_block(); + if (block == NULL) { + gpr_atm_rel_store(&g_log.is_full, 1); + return false; + } + cl_core_local_block_set_block(core_local_block, block); + cl_block_enable_access(block); + return true; +} + +static cl_block* cl_get_block(void* record) { + uintptr_t p = (uintptr_t)((char*)record - g_log.buffer); + uintptr_t index = p >> CENSUS_LOG_2_MAX_RECORD_SIZE; + return &g_log.blocks[index]; +} + +/* Gets the next block to read and tries to free 'prev' block (if not NULL). + Returns NULL if reached the end. */ +static cl_block* cl_next_block_to_read(cl_block* prev) { + cl_block* block = NULL; + if (g_log.read_iterator_state == g_log.num_cores) { + /* We are traversing dirty list; find the next dirty block. */ + if (prev != NULL) { + /* Try to free the previous block if there is no unread data. This block + may have unread data if previously incomplete record completed between + read_next() calls. */ + block = prev->link.next->block; + if (cl_block_try_disable_access(prev, 0 /* do not discard data */)) { + cl_block_list_remove(&g_log.dirty_block_list, prev); + cl_block_list_insert_at_head(&g_log.free_block_list, prev); + gpr_atm_rel_store(&g_log.is_full, 0); + } + } else { + block = cl_block_list_head(&g_log.dirty_block_list); + } + if (block != NULL) { + return block; + } + /* We are done with the dirty list; moving on to core-local blocks. */ + } + while (g_log.read_iterator_state > 0) { + g_log.read_iterator_state--; + block = cl_core_local_block_get_block( + &g_log.core_local_blocks[g_log.read_iterator_state]); + if (block != NULL) { + return block; + } + } + return NULL; +} + +#define CL_LOG_2_MB 20 /* 2^20 = 1MB */ + +/* External functions: primary stats_log interface */ +void census_log_initialize(size_t size_in_mb, int discard_old_records) { + /* Check cacheline alignment. */ + GPR_ASSERT(sizeof(cl_block) % GPR_CACHELINE_SIZE == 0); + GPR_ASSERT(sizeof(cl_core_local_block) % GPR_CACHELINE_SIZE == 0); + GPR_ASSERT(!g_log.initialized); + g_log.discard_old_records = discard_old_records; + g_log.num_cores = gpr_cpu_num_cores(); + /* Ensure that we will not get any overflow in calaculating num_blocks */ + GPR_ASSERT(CL_LOG_2_MB >= CENSUS_LOG_2_MAX_RECORD_SIZE); + GPR_ASSERT(size_in_mb < 1000); + /* Ensure at least as many blocks as there are cores. */ + g_log.num_blocks = + (uint32_t)GPR_MAX(g_log.num_cores, (size_in_mb << CL_LOG_2_MB) >> + CENSUS_LOG_2_MAX_RECORD_SIZE); + gpr_mu_init(&g_log.lock); + g_log.read_iterator_state = 0; + g_log.block_being_read = NULL; + gpr_atm_rel_store(&g_log.is_full, 0); + g_log.core_local_blocks = (cl_core_local_block*)gpr_malloc_aligned( + g_log.num_cores * sizeof(cl_core_local_block), GPR_CACHELINE_SIZE_LOG); + memset(g_log.core_local_blocks, 0, + g_log.num_cores * sizeof(cl_core_local_block)); + g_log.blocks = (cl_block*)gpr_malloc_aligned( + g_log.num_blocks * sizeof(cl_block), GPR_CACHELINE_SIZE_LOG); + memset(g_log.blocks, 0, g_log.num_blocks * sizeof(cl_block)); + g_log.buffer = gpr_malloc(g_log.num_blocks * CENSUS_LOG_MAX_RECORD_SIZE); + memset(g_log.buffer, 0, g_log.num_blocks * CENSUS_LOG_MAX_RECORD_SIZE); + cl_block_list_initialize(&g_log.free_block_list); + cl_block_list_initialize(&g_log.dirty_block_list); + for (uint32_t i = 0; i < g_log.num_blocks; ++i) { + cl_block* block = g_log.blocks + i; + cl_block_initialize(block, g_log.buffer + (CENSUS_LOG_MAX_RECORD_SIZE * i)); + cl_block_try_disable_access(block, 1 /* discard data */); + cl_block_list_insert_at_tail(&g_log.free_block_list, block); + } + gpr_atm_rel_store(&g_log.out_of_space_count, 0); + g_log.initialized = 1; +} + +void census_log_shutdown(void) { + GPR_ASSERT(g_log.initialized); + gpr_mu_destroy(&g_log.lock); + gpr_free_aligned(g_log.core_local_blocks); + g_log.core_local_blocks = NULL; + gpr_free_aligned(g_log.blocks); + g_log.blocks = NULL; + gpr_free(g_log.buffer); + g_log.buffer = NULL; + g_log.initialized = 0; +} + +void* census_log_start_write(size_t size) { + /* Used to bound number of times block allocation is attempted. */ + GPR_ASSERT(size > 0); + GPR_ASSERT(g_log.initialized); + if (size > CENSUS_LOG_MAX_RECORD_SIZE) { + return NULL; + } + uint32_t attempts_remaining = g_log.num_blocks; + uint32_t core_id = gpr_cpu_current_cpu(); + do { + void* record = NULL; + cl_block* block = + cl_core_local_block_get_block(&g_log.core_local_blocks[core_id]); + if (block && (record = cl_block_start_write(block, size))) { + return record; + } + /* Need to allocate a new block. We are here if: + - No block associated with the core OR + - Write in-progress on the block OR + - block is out of space */ + if (gpr_atm_acq_load(&g_log.is_full)) { + gpr_atm_no_barrier_fetch_add(&g_log.out_of_space_count, 1); + return NULL; + } + gpr_mu_lock(&g_log.lock); + bool allocated = cl_allocate_core_local_block(core_id, block); + gpr_mu_unlock(&g_log.lock); + if (!allocated) { + gpr_atm_no_barrier_fetch_add(&g_log.out_of_space_count, 1); + return NULL; + } + } while (attempts_remaining--); + /* Give up. */ + gpr_atm_no_barrier_fetch_add(&g_log.out_of_space_count, 1); + return NULL; +} + +void census_log_end_write(void* record, size_t bytes_written) { + GPR_ASSERT(g_log.initialized); + cl_block_end_write(cl_get_block(record), bytes_written); +} + +void census_log_init_reader(void) { + GPR_ASSERT(g_log.initialized); + gpr_mu_lock(&g_log.lock); + /* If a block is locked for reading unlock it. */ + if (g_log.block_being_read != NULL) { + cl_block_end_read(g_log.block_being_read); + g_log.block_being_read = NULL; + } + g_log.read_iterator_state = g_log.num_cores; + gpr_mu_unlock(&g_log.lock); +} + +const void* census_log_read_next(size_t* bytes_available) { + GPR_ASSERT(g_log.initialized); + gpr_mu_lock(&g_log.lock); + if (g_log.block_being_read != NULL) { + cl_block_end_read(g_log.block_being_read); + } + do { + g_log.block_being_read = cl_next_block_to_read(g_log.block_being_read); + if (g_log.block_being_read != NULL) { + void* record = + cl_block_start_read(g_log.block_being_read, bytes_available); + if (record != NULL) { + gpr_mu_unlock(&g_log.lock); + return record; + } + } + } while (g_log.block_being_read != NULL); + gpr_mu_unlock(&g_log.lock); + return NULL; +} + +size_t census_log_remaining_space(void) { + GPR_ASSERT(g_log.initialized); + size_t space = 0; + gpr_mu_lock(&g_log.lock); + if (g_log.discard_old_records) { + /* Remaining space is not meaningful; just return the entire log space. */ + space = g_log.num_blocks << CENSUS_LOG_2_MAX_RECORD_SIZE; + } else { + GPR_ASSERT(g_log.free_block_list.count >= 0); + space = (size_t)g_log.free_block_list.count * CENSUS_LOG_MAX_RECORD_SIZE; + } + gpr_mu_unlock(&g_log.lock); + return space; +} + +int64_t census_log_out_of_space_count(void) { + GPR_ASSERT(g_log.initialized); + return gpr_atm_acq_load(&g_log.out_of_space_count); +} diff --git a/src/core/census/log.h b/src/core/census/log.h new file mode 100644 index 0000000000..05daea066f --- /dev/null +++ b/src/core/census/log.h @@ -0,0 +1,93 @@ +/* + * + * Copyright 2015-2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#ifndef GRPC_INTERNAL_CORE_CENSUS_LOG_H +#define GRPC_INTERNAL_CORE_CENSUS_LOG_H + +#include <grpc/support/port_platform.h> +#include <stddef.h> + +/* Maximum record size, in bytes. */ +#define CENSUS_LOG_2_MAX_RECORD_SIZE 14 /* 2^14 = 16KB */ +#define CENSUS_LOG_MAX_RECORD_SIZE (1 << CENSUS_LOG_2_MAX_RECORD_SIZE) + +/* Initialize the statistics logging subsystem with the given log size. A log + size of 0 will result in the smallest possible log for the platform + (approximately CENSUS_LOG_MAX_RECORD_SIZE * gpr_cpu_num_cores()). If + discard_old_records is non-zero, then new records will displace older ones + when the log is full. This function must be called before any other + census_log functions. +*/ +void census_log_initialize(size_t size_in_mb, int discard_old_records); + +/* Shutdown the logging subsystem. Caller must ensure that: + - no in progress or future call to any census_log functions + - no incomplete records +*/ +void census_log_shutdown(void); + +/* Allocates and returns a 'size' bytes record and marks it in use. A + subsequent census_log_end_write() marks the record complete. The + 'bytes_written' census_log_end_write() argument must be <= + 'size'. Returns NULL if out-of-space AND: + - log is configured to keep old records OR + - all blocks are pinned by incomplete records. +*/ +void* census_log_start_write(size_t size); + +void census_log_end_write(void* record, size_t bytes_written); + +void census_log_init_reader(void); + +/* census_log_read_next() iterates over blocks with data and for each block + returns a pointer to the first unread byte. The number of bytes that can be + read are returned in 'bytes_available'. Reader is expected to read all + available data. Reading the data consumes it i.e. it cannot be read again. + census_log_read_next() returns NULL if the end is reached i.e last block + is read. census_log_init_reader() starts the iteration or aborts the + current iteration. +*/ +const void* census_log_read_next(size_t* bytes_available); + +/* Returns estimated remaining space across all blocks, in bytes. If log is + configured to discard old records, returns total log space. Otherwise, + returns space available in empty blocks (partially filled blocks are + treated as full). +*/ +size_t census_log_remaining_space(void); + +/* Returns the number of times gprc_stats_log_start_write() failed due to + out-of-space. */ +int64_t census_log_out_of_space_count(void); + +#endif /* GRPC_INTERNAL_CORE_CENSUS_LOG_H */ diff --git a/src/python/grpcio/grpc_core_dependencies.py b/src/python/grpcio/grpc_core_dependencies.py index 708091df9f..ea63b41a76 100644 --- a/src/python/grpcio/grpc_core_dependencies.py +++ b/src/python/grpcio/grpc_core_dependencies.py @@ -222,6 +222,7 @@ CORE_SOURCE_FILES = [ 'src/core/transport/transport_op_string.c', 'src/core/census/context.c', 'src/core/census/initialize.c', + 'src/core/census/log.c', 'src/core/census/operation.c', 'src/core/census/placeholders.c', 'src/core/census/tag_set.c', diff --git a/test/core/census/log_test.c b/test/core/census/log_test.c new file mode 100644 index 0000000000..054b897369 --- /dev/null +++ b/test/core/census/log_test.c @@ -0,0 +1,585 @@ +/* + * + * Copyright 2015-2016, Google Inc. + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions are + * met: + * + * * Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * * Redistributions in binary form must reproduce the above + * copyright notice, this list of conditions and the following disclaimer + * in the documentation and/or other materials provided with the + * distribution. + * * Neither the name of Google Inc. nor the names of its + * contributors may be used to endorse or promote products derived from + * this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS + * "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT + * LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR + * A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT + * OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, + * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT + * LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE + * OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + * + */ + +#include "src/core/census/log.h" +#include <grpc/support/cpu.h> +#include <grpc/support/log.h> +#include <grpc/support/port_platform.h> +#include <grpc/support/sync.h> +#include <grpc/support/thd.h> +#include <grpc/support/time.h> +#include <grpc/support/useful.h> +#include <stdio.h> +#include <stdlib.h> +#include <string.h> +#include "test/core/util/test_config.h" + +/* Change this to non-zero if you want more output. */ +#define VERBOSE 0 + +/* Log size to use for all tests. */ +#define LOG_SIZE_IN_MB 1 +#define LOG_SIZE_IN_BYTES (LOG_SIZE_IN_MB << 20) + +/* Fills in 'record' of size 'size'. Each byte in record is filled in with the + same value. The value is extracted from 'record' pointer. */ +static void write_record(char* record, size_t size) { + char data = (char)((uintptr_t)record % 255); + memset(record, data, size); +} + +/* Reads fixed size records. Returns the number of records read in + 'num_records'. */ +static void read_records(size_t record_size, const char* buffer, + size_t buffer_size, int* num_records) { + GPR_ASSERT(buffer_size >= record_size); + GPR_ASSERT(buffer_size % record_size == 0); + *num_records = (int)(buffer_size / record_size); + for (int i = 0; i < *num_records; ++i) { + const char* record = buffer + (record_size * (size_t)i); + char data = (char)((uintptr_t)record % 255); + for (size_t j = 0; j < record_size; ++j) { + GPR_ASSERT(data == record[j]); + } + } +} + +/* Tries to write the specified number of records. Stops when the log gets + full. Returns the number of records written. Spins for random + number of times, up to 'max_spin_count', between writes. */ +static int write_records_to_log(int writer_id, size_t record_size, + int num_records, int max_spin_count) { + int counter = 0; + for (int i = 0; i < num_records; ++i) { + int spin_count = max_spin_count ? rand() % max_spin_count : 0; + if (VERBOSE && (counter++ == num_records / 10)) { + printf(" Writer %d: %d out of %d written\n", writer_id, i, num_records); + counter = 0; + } + char* record = (char*)(census_log_start_write(record_size)); + if (record == NULL) { + return i; + } + write_record(record, record_size); + census_log_end_write(record, record_size); + for (int j = 0; j < spin_count; ++j) { + GPR_ASSERT(j >= 0); + } + } + return num_records; +} + +/* Performs a single read iteration. Returns the number of records read. */ +static int perform_read_iteration(size_t record_size) { + const void* read_buffer = NULL; + size_t bytes_available; + int records_read = 0; + census_log_init_reader(); + while ((read_buffer = census_log_read_next(&bytes_available))) { + int num_records = 0; + read_records(record_size, (const char*)read_buffer, bytes_available, + &num_records); + records_read += num_records; + } + return records_read; +} + +/* Asserts that the log is empty. */ +static void assert_log_empty(void) { + census_log_init_reader(); + size_t bytes_available; + GPR_ASSERT(census_log_read_next(&bytes_available) == NULL); +} + +/* Fills the log and verifies data. If 'no fragmentation' is true, records + are sized such that CENSUS_LOG_2_MAX_RECORD_SIZE is a multiple of record + size. If not a circular log, verifies that the number of records written + match the number of records read. */ +static void fill_log(size_t log_size, int no_fragmentation, int circular_log) { + size_t size; + if (no_fragmentation) { + int log2size = rand() % (CENSUS_LOG_2_MAX_RECORD_SIZE + 1); + size = ((size_t)1 << log2size); + } else { + while (1) { + size = 1 + ((size_t)rand() % CENSUS_LOG_MAX_RECORD_SIZE); + if (CENSUS_LOG_MAX_RECORD_SIZE % size) { + break; + } + } + } + int records_written = + write_records_to_log(0 /* writer id */, size, + (int)((log_size / size) * 2), 0 /* spin count */); + int records_read = perform_read_iteration(size); + if (!circular_log) { + GPR_ASSERT(records_written == records_read); + } + assert_log_empty(); +} + +/* Structure to pass args to writer_thread */ +typedef struct writer_thread_args { + /* Index of this thread in the writers vector. */ + int index; + /* Record size. */ + size_t record_size; + /* Number of records to write. */ + int num_records; + /* Used to signal when writer is complete */ + gpr_cv* done; + gpr_mu* mu; + int* count; +} writer_thread_args; + +/* Writes the given number of records of random size (up to kMaxRecordSize) and + random data to the specified log. */ +static void writer_thread(void* arg) { + writer_thread_args* args = (writer_thread_args*)arg; + /* Maximum number of times to spin between writes. */ + static const int MAX_SPIN_COUNT = 50; + int records_written = 0; + if (VERBOSE) { + printf(" Writer %d starting\n", args->index); + } + while (records_written < args->num_records) { + records_written += write_records_to_log(args->index, args->record_size, + args->num_records - records_written, + MAX_SPIN_COUNT); + if (records_written < args->num_records) { + /* Ran out of log space. Sleep for a bit and let the reader catch up. + This should never happen for circular logs. */ + if (VERBOSE) { + printf(" Writer stalled due to out-of-space: %d out of %d written\n", + records_written, args->num_records); + } + gpr_sleep_until(GRPC_TIMEOUT_MILLIS_TO_DEADLINE(10)); + } + } + /* Done. Decrement count and signal. */ + gpr_mu_lock(args->mu); + (*args->count)--; + gpr_cv_broadcast(args->done); + if (VERBOSE) { + printf(" Writer %d done\n", args->index); + } + gpr_mu_unlock(args->mu); +} + +/* struct to pass args to reader_thread */ +typedef struct reader_thread_args { + /* Record size. */ + size_t record_size; + /* Interval between read iterations. */ + int read_iteration_interval_in_msec; + /* Total number of records. */ + int total_records; + /* Signalled when reader should stop. */ + gpr_cv stop; + int stop_flag; + /* Used to signal when reader has finished */ + gpr_cv* done; + gpr_mu* mu; + int running; +} reader_thread_args; + +/* Reads and verifies the specified number of records. Reader can also be + stopped via gpr_cv_signal(&args->stop). Sleeps for 'read_interval_in_msec' + between read iterations. */ +static void reader_thread(void* arg) { + reader_thread_args* args = (reader_thread_args*)arg; + if (VERBOSE) { + printf(" Reader starting\n"); + } + gpr_timespec interval = gpr_time_from_micros( + args->read_iteration_interval_in_msec * 1000, GPR_TIMESPAN); + gpr_mu_lock(args->mu); + int records_read = 0; + int num_iterations = 0; + int counter = 0; + while (!args->stop_flag && records_read < args->total_records) { + gpr_cv_wait(&args->stop, args->mu, interval); + if (!args->stop_flag) { + records_read += perform_read_iteration(args->record_size); + GPR_ASSERT(records_read <= args->total_records); + if (VERBOSE && (counter++ == 100000)) { + printf(" Reader: %d out of %d read\n", records_read, + args->total_records); + counter = 0; + } + ++num_iterations; + } + } + /* Done */ + args->running = 0; + gpr_cv_broadcast(args->done); + if (VERBOSE) { + printf(" Reader: records: %d, iterations: %d\n", records_read, + num_iterations); + } + gpr_mu_unlock(args->mu); +} + +/* Creates NUM_WRITERS writers where each writer writes NUM_RECORDS_PER_WRITER + records. Also, starts a reader that iterates over and reads blocks every + READ_ITERATION_INTERVAL_IN_MSEC. */ +/* Number of writers. */ +#define NUM_WRITERS 5 +static void multiple_writers_single_reader(int circular_log) { + /* Sleep interval between read iterations. */ + static const int READ_ITERATION_INTERVAL_IN_MSEC = 10; + /* Maximum record size. */ + static const size_t MAX_RECORD_SIZE = 20; + /* Number of records written by each writer. This is sized such that we + will write through the entire log ~10 times. */ + const int NUM_RECORDS_PER_WRITER = + (int)((10 * census_log_remaining_space()) / (MAX_RECORD_SIZE / 2)) / + NUM_WRITERS; + size_t record_size = ((size_t)rand() % MAX_RECORD_SIZE) + 1; + /* Create and start writers. */ + writer_thread_args writers[NUM_WRITERS]; + int writers_count = NUM_WRITERS; + gpr_cv writers_done; + gpr_mu writers_mu; /* protects writers_done and writers_count */ + gpr_cv_init(&writers_done); + gpr_mu_init(&writers_mu); + gpr_thd_id id; + for (int i = 0; i < NUM_WRITERS; ++i) { + writers[i].index = i; + writers[i].record_size = record_size; + writers[i].num_records = NUM_RECORDS_PER_WRITER; + writers[i].done = &writers_done; + writers[i].count = &writers_count; + writers[i].mu = &writers_mu; + gpr_thd_new(&id, &writer_thread, &writers[i], NULL); + } + /* Start reader. */ + gpr_cv reader_done; + gpr_mu reader_mu; /* protects reader_done and reader.running */ + reader_thread_args reader; + reader.record_size = record_size; + reader.read_iteration_interval_in_msec = READ_ITERATION_INTERVAL_IN_MSEC; + reader.total_records = NUM_WRITERS * NUM_RECORDS_PER_WRITER; + reader.stop_flag = 0; + gpr_cv_init(&reader.stop); + gpr_cv_init(&reader_done); + reader.done = &reader_done; + gpr_mu_init(&reader_mu); + reader.mu = &reader_mu; + reader.running = 1; + gpr_thd_new(&id, &reader_thread, &reader, NULL); + /* Wait for writers to finish. */ + gpr_mu_lock(&writers_mu); + while (writers_count != 0) { + gpr_cv_wait(&writers_done, &writers_mu, gpr_inf_future(GPR_CLOCK_REALTIME)); + } + gpr_mu_unlock(&writers_mu); + gpr_mu_destroy(&writers_mu); + gpr_cv_destroy(&writers_done); + gpr_mu_lock(&reader_mu); + if (circular_log) { + /* Stop reader. */ + reader.stop_flag = 1; + gpr_cv_signal(&reader.stop); + } + /* wait for reader to finish */ + while (reader.running) { + gpr_cv_wait(&reader_done, &reader_mu, gpr_inf_future(GPR_CLOCK_REALTIME)); + } + if (circular_log) { + /* Assert that there were no out-of-space errors. */ + GPR_ASSERT(0 == census_log_out_of_space_count()); + } + gpr_mu_unlock(&reader_mu); + gpr_mu_destroy(&reader_mu); + gpr_cv_destroy(&reader_done); + if (VERBOSE) { + printf(" Reader: finished\n"); + } +} + +static void setup_test(int circular_log) { + census_log_initialize(LOG_SIZE_IN_MB, circular_log); + GPR_ASSERT(census_log_remaining_space() == LOG_SIZE_IN_BYTES); +} + +/* Attempts to create a record of invalid size (size > + CENSUS_LOG_MAX_RECORD_SIZE). */ +void test_invalid_record_size(void) { + static const size_t INVALID_SIZE = CENSUS_LOG_MAX_RECORD_SIZE + 1; + static const size_t VALID_SIZE = 1; + printf("Starting test: invalid record size\n"); + setup_test(0); + void* record = census_log_start_write(INVALID_SIZE); + GPR_ASSERT(record == NULL); + /* Now try writing a valid record. */ + record = census_log_start_write(VALID_SIZE); + GPR_ASSERT(record != NULL); + census_log_end_write(record, VALID_SIZE); + /* Verifies that available space went down by one block. In theory, this + check can fail if the thread is context switched to a new CPU during the + start_write execution (multiple blocks get allocated), but this has not + been observed in practice. */ + GPR_ASSERT(LOG_SIZE_IN_BYTES - CENSUS_LOG_MAX_RECORD_SIZE == + census_log_remaining_space()); + census_log_shutdown(); +} + +/* Tests end_write() with a different size than what was specified in + start_write(). */ +void test_end_write_with_different_size(void) { + static const size_t START_WRITE_SIZE = 10; + static const size_t END_WRITE_SIZE = 7; + printf("Starting test: end write with different size\n"); + setup_test(0); + void* record_written = census_log_start_write(START_WRITE_SIZE); + GPR_ASSERT(record_written != NULL); + census_log_end_write(record_written, END_WRITE_SIZE); + census_log_init_reader(); + size_t bytes_available; + const void* record_read = census_log_read_next(&bytes_available); + GPR_ASSERT(record_written == record_read); + GPR_ASSERT(END_WRITE_SIZE == bytes_available); + assert_log_empty(); + census_log_shutdown(); +} + +/* Verifies that pending records are not available via read_next(). */ +void test_read_pending_record(void) { + static const size_t PR_RECORD_SIZE = 1024; + printf("Starting test: read pending record\n"); + setup_test(0); + /* Start a write. */ + void* record_written = census_log_start_write(PR_RECORD_SIZE); + GPR_ASSERT(record_written != NULL); + /* As write is pending, read should fail. */ + census_log_init_reader(); + size_t bytes_available; + const void* record_read = census_log_read_next(&bytes_available); + GPR_ASSERT(record_read == NULL); + /* A read followed by end_write() should succeed. */ + census_log_end_write(record_written, PR_RECORD_SIZE); + census_log_init_reader(); + record_read = census_log_read_next(&bytes_available); + GPR_ASSERT(record_written == record_read); + GPR_ASSERT(PR_RECORD_SIZE == bytes_available); + assert_log_empty(); + census_log_shutdown(); +} + +/* Tries reading beyond pending write. */ +void test_read_beyond_pending_record(void) { + printf("Starting test: read beyond pending record\n"); + setup_test(0); + /* Start a write. */ + const size_t incomplete_record_size = 10; + void* incomplete_record = census_log_start_write(incomplete_record_size); + GPR_ASSERT(incomplete_record != NULL); + const size_t complete_record_size = 20; + void* complete_record = census_log_start_write(complete_record_size); + GPR_ASSERT(complete_record != NULL); + GPR_ASSERT(complete_record != incomplete_record); + census_log_end_write(complete_record, complete_record_size); + /* Now iterate over blocks to read completed records. */ + census_log_init_reader(); + size_t bytes_available; + const void* record_read = census_log_read_next(&bytes_available); + GPR_ASSERT(complete_record == record_read); + GPR_ASSERT(complete_record_size == bytes_available); + /* Complete first record. */ + census_log_end_write(incomplete_record, incomplete_record_size); + /* Have read past the incomplete record, so read_next() should return NULL. */ + /* NB: this test also assumes our thread did not get switched to a different + CPU between the two start_write calls */ + record_read = census_log_read_next(&bytes_available); + GPR_ASSERT(record_read == NULL); + /* Reset reader to get the newly completed record. */ + census_log_init_reader(); + record_read = census_log_read_next(&bytes_available); + GPR_ASSERT(incomplete_record == record_read); + GPR_ASSERT(incomplete_record_size == bytes_available); + assert_log_empty(); + census_log_shutdown(); +} + +/* Tests scenario where block being read is detached from a core and put on the + dirty list. */ +void test_detached_while_reading(void) { + printf("Starting test: detached while reading\n"); + setup_test(0); + /* Start a write. */ + static const size_t DWR_RECORD_SIZE = 10; + void* record_written = census_log_start_write(DWR_RECORD_SIZE); + GPR_ASSERT(record_written != NULL); + census_log_end_write(record_written, DWR_RECORD_SIZE); + /* Read this record. */ + census_log_init_reader(); + size_t bytes_available; + const void* record_read = census_log_read_next(&bytes_available); + GPR_ASSERT(record_read != NULL); + GPR_ASSERT(DWR_RECORD_SIZE == bytes_available); + /* Now fill the log. This will move the block being read from core-local + array to the dirty list. */ + while ((record_written = census_log_start_write(DWR_RECORD_SIZE))) { + census_log_end_write(record_written, DWR_RECORD_SIZE); + } + + /* In this iteration, read_next() should only traverse blocks in the + core-local array. Therefore, we expect at most gpr_cpu_num_cores() more + blocks. As log is full, if read_next() is traversing the dirty list, we + will get more than gpr_cpu_num_cores() blocks. */ + int block_read = 0; + while ((record_read = census_log_read_next(&bytes_available))) { + ++block_read; + GPR_ASSERT(block_read <= (int)gpr_cpu_num_cores()); + } + census_log_shutdown(); +} + +/* Fills non-circular log with records sized such that size is a multiple of + CENSUS_LOG_MAX_RECORD_SIZE (no per-block fragmentation). */ +void test_fill_log_no_fragmentation(void) { + printf("Starting test: fill log no fragmentation\n"); + const int circular = 0; + setup_test(circular); + fill_log(LOG_SIZE_IN_BYTES, 1 /* no fragmentation */, circular); + census_log_shutdown(); +} + +/* Fills circular log with records sized such that size is a multiple of + CENSUS_LOG_MAX_RECORD_SIZE (no per-block fragmentation). */ +void test_fill_circular_log_no_fragmentation(void) { + printf("Starting test: fill circular log no fragmentation\n"); + const int circular = 1; + setup_test(circular); + fill_log(LOG_SIZE_IN_BYTES, 1 /* no fragmentation */, circular); + census_log_shutdown(); +} + +/* Fills non-circular log with records that may straddle end of a block. */ +void test_fill_log_with_straddling_records(void) { + printf("Starting test: fill log with straddling records\n"); + const int circular = 0; + setup_test(circular); + fill_log(LOG_SIZE_IN_BYTES, 0 /* block straddling records */, circular); + census_log_shutdown(); +} + +/* Fills circular log with records that may straddle end of a block. */ +void test_fill_circular_log_with_straddling_records(void) { + printf("Starting test: fill circular log with straddling records\n"); + const int circular = 1; + setup_test(circular); + fill_log(LOG_SIZE_IN_BYTES, 0 /* block straddling records */, circular); + census_log_shutdown(); +} + +/* Tests scenario where multiple writers and a single reader are using a log + that is configured to discard old records. */ +void test_multiple_writers_circular_log(void) { + printf("Starting test: multiple writers circular log\n"); + const int circular = 1; + setup_test(circular); + multiple_writers_single_reader(circular); + census_log_shutdown(); +} + +/* Tests scenario where multiple writers and a single reader are using a log + that is configured to discard old records. */ +void test_multiple_writers(void) { + printf("Starting test: multiple writers\n"); + const int circular = 0; + setup_test(circular); + multiple_writers_single_reader(circular); + census_log_shutdown(); +} + +/* Repeat the straddling records and multiple writers tests with a small log. */ +void test_small_log(void) { + printf("Starting test: small log\n"); + const int circular = 0; + census_log_initialize(0, circular); + size_t log_size = census_log_remaining_space(); + GPR_ASSERT(log_size > 0); + fill_log(log_size, 0, circular); + census_log_shutdown(); + census_log_initialize(0, circular); + multiple_writers_single_reader(circular); + census_log_shutdown(); +} + +void test_performance(void) { + for (size_t write_size = 1; write_size < CENSUS_LOG_MAX_RECORD_SIZE; + write_size *= 2) { + setup_test(0); + gpr_timespec start_time = gpr_now(GPR_CLOCK_REALTIME); + int nrecords = 0; + while (1) { + void* record = census_log_start_write(write_size); + if (record == NULL) { + break; + } + census_log_end_write(record, write_size); + nrecords++; + } + gpr_timespec write_time = + gpr_time_sub(gpr_now(GPR_CLOCK_REALTIME), start_time); + double write_time_micro = + (double)write_time.tv_sec * 1000000 + (double)write_time.tv_nsec / 1000; + census_log_shutdown(); + printf( + "Wrote %d %d byte records in %.3g microseconds: %g records/us " + "(%g ns/record), %g gigabytes/s\n", + nrecords, (int)write_size, write_time_micro, + nrecords / write_time_micro, 1000 * write_time_micro / nrecords, + (double)((int)write_size * nrecords) / write_time_micro / 1000); + } +} + +int main(int argc, char** argv) { + grpc_test_init(argc, argv); + srand((unsigned)gpr_now(GPR_CLOCK_REALTIME).tv_nsec); + test_invalid_record_size(); + test_end_write_with_different_size(); + test_read_pending_record(); + test_read_beyond_pending_record(); + test_detached_while_reading(); + test_fill_log_no_fragmentation(); + test_fill_circular_log_no_fragmentation(); + test_fill_log_with_straddling_records(); + test_fill_circular_log_with_straddling_records(); + test_small_log(); + test_multiple_writers(); + test_multiple_writers_circular_log(); + return 0; +} diff --git a/tools/doxygen/Doxyfile.core.internal b/tools/doxygen/Doxyfile.core.internal index cd49a1980d..782cfd67f0 100644 --- a/tools/doxygen/Doxyfile.core.internal +++ b/tools/doxygen/Doxyfile.core.internal @@ -896,6 +896,7 @@ src/core/transport/transport.h \ src/core/transport/transport_impl.h \ src/core/census/aggregation.h \ src/core/census/context.h \ +src/core/census/log.h \ src/core/census/rpc_metric_id.h \ src/core/httpcli/httpcli_security_connector.c \ src/core/security/base64.c \ @@ -1047,6 +1048,7 @@ src/core/transport/transport.c \ src/core/transport/transport_op_string.c \ src/core/census/context.c \ src/core/census/initialize.c \ +src/core/census/log.c \ src/core/census/operation.c \ src/core/census/placeholders.c \ src/core/census/tag_set.c \ diff --git a/tools/run_tests/sources_and_headers.json b/tools/run_tests/sources_and_headers.json index 172385675f..9800ead205 100644 --- a/tools/run_tests/sources_and_headers.json +++ b/tools/run_tests/sources_and_headers.json @@ -78,6 +78,20 @@ ], "headers": [], "language": "c", + "name": "census_log_test", + "src": [ + "test/core/census/log_test.c" + ] + }, + { + "deps": [ + "gpr", + "gpr_test_util", + "grpc", + "grpc_test_util" + ], + "headers": [], + "language": "c", "name": "channel_create_test", "src": [ "test/core/surface/channel_create_test.c" @@ -2961,6 +2975,7 @@ "src/core/census/aggregation.h", "src/core/census/context.h", "src/core/census/grpc_filter.h", + "src/core/census/log.h", "src/core/census/rpc_metric_id.h", "src/core/channel/channel_args.h", "src/core/channel/channel_stack.h", @@ -3106,6 +3121,8 @@ "src/core/census/grpc_filter.c", "src/core/census/grpc_filter.h", "src/core/census/initialize.c", + "src/core/census/log.c", + "src/core/census/log.h", "src/core/census/operation.c", "src/core/census/placeholders.c", "src/core/census/rpc_metric_id.h", @@ -3477,6 +3494,7 @@ "src/core/census/aggregation.h", "src/core/census/context.h", "src/core/census/grpc_filter.h", + "src/core/census/log.h", "src/core/census/rpc_metric_id.h", "src/core/channel/channel_args.h", "src/core/channel/channel_stack.h", @@ -3607,6 +3625,8 @@ "src/core/census/grpc_filter.c", "src/core/census/grpc_filter.h", "src/core/census/initialize.c", + "src/core/census/log.c", + "src/core/census/log.h", "src/core/census/operation.c", "src/core/census/placeholders.c", "src/core/census/rpc_metric_id.h", diff --git a/tools/run_tests/tests.json b/tools/run_tests/tests.json index a23a340b6c..46fbccba63 100644 --- a/tools/run_tests/tests.json +++ b/tools/run_tests/tests.json @@ -113,6 +113,26 @@ "exclude_configs": [], "flaky": false, "language": "c", + "name": "census_log_test", + "platforms": [ + "linux", + "mac", + "posix", + "windows" + ] + }, + { + "args": [], + "ci_platforms": [ + "linux", + "mac", + "posix", + "windows" + ], + "cpu_cost": 1.0, + "exclude_configs": [], + "flaky": false, + "language": "c", "name": "channel_create_test", "platforms": [ "linux", diff --git a/vsprojects/buildtests_c.sln b/vsprojects/buildtests_c.sln index 30f3bf5fc5..5421588457 100644 --- a/vsprojects/buildtests_c.sln +++ b/vsprojects/buildtests_c.sln @@ -167,6 +167,17 @@ Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "bin_encoder_test", "vcxproj {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} = {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} EndProjectSection EndProject +Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "census_log_test", "vcxproj\test\census_log_test\census_log_test.vcxproj", "{C27CEE16-2BEC-5572-3956-677E9F6F8BED}" + ProjectSection(myProperties) = preProject + lib = "False" + EndProjectSection + ProjectSection(ProjectDependencies) = postProject + {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} = {17BCAFC0-5FDC-4C94-AEB9-95F3E220614B} + {29D16885-7228-4C31-81ED-5F9187C7F2A9} = {29D16885-7228-4C31-81ED-5F9187C7F2A9} + {EAB0A629-17A9-44DB-B5FF-E91A721FE037} = {EAB0A629-17A9-44DB-B5FF-E91A721FE037} + {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} = {B23D3D1A-9438-4EDA-BEB6-9A0A03D17792} + EndProjectSection +EndProject Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "channel_create_test", "vcxproj\test\channel_create_test\channel_create_test.vcxproj", "{AFC88484-3A2E-32BC-25B2-23DF741D4F3D}" ProjectSection(myProperties) = preProject lib = "False" @@ -1590,6 +1601,22 @@ Global {D5C70922-D68E-0E9D-9988-995E0F9A79AE}.Release-DLL|Win32.Build.0 = Release|Win32 {D5C70922-D68E-0E9D-9988-995E0F9A79AE}.Release-DLL|x64.ActiveCfg = Release|x64 {D5C70922-D68E-0E9D-9988-995E0F9A79AE}.Release-DLL|x64.Build.0 = Release|x64 + {C27CEE16-2BEC-5572-3956-677E9F6F8BED}.Debug|Win32.ActiveCfg = Debug|Win32 + {C27CEE16-2BEC-5572-3956-677E9F6F8BED}.Debug|x64.ActiveCfg = Debug|x64 + {C27CEE16-2BEC-5572-3956-677E9F6F8BED}.Release|Win32.ActiveCfg = Release|Win32 + {C27CEE16-2BEC-5572-3956-677E9F6F8BED}.Release|x64.ActiveCfg = Release|x64 + {C27CEE16-2BEC-5572-3956-677E9F6F8BED}.Debug|Win32.Build.0 = Debug|Win32 + {C27CEE16-2BEC-5572-3956-677E9F6F8BED}.Debug|x64.Build.0 = Debug|x64 + {C27CEE16-2BEC-5572-3956-677E9F6F8BED}.Release|Win32.Build.0 = Release|Win32 + {C27CEE16-2BEC-5572-3956-677E9F6F8BED}.Release|x64.Build.0 = Release|x64 + {C27CEE16-2BEC-5572-3956-677E9F6F8BED}.Debug-DLL|Win32.ActiveCfg = Debug|Win32 + {C27CEE16-2BEC-5572-3956-677E9F6F8BED}.Debug-DLL|Win32.Build.0 = Debug|Win32 + {C27CEE16-2BEC-5572-3956-677E9F6F8BED}.Debug-DLL|x64.ActiveCfg = Debug|x64 + {C27CEE16-2BEC-5572-3956-677E9F6F8BED}.Debug-DLL|x64.Build.0 = Debug|x64 + {C27CEE16-2BEC-5572-3956-677E9F6F8BED}.Release-DLL|Win32.ActiveCfg = Release|Win32 + {C27CEE16-2BEC-5572-3956-677E9F6F8BED}.Release-DLL|Win32.Build.0 = Release|Win32 + {C27CEE16-2BEC-5572-3956-677E9F6F8BED}.Release-DLL|x64.ActiveCfg = Release|x64 + {C27CEE16-2BEC-5572-3956-677E9F6F8BED}.Release-DLL|x64.Build.0 = Release|x64 {AFC88484-3A2E-32BC-25B2-23DF741D4F3D}.Debug|Win32.ActiveCfg = Debug|Win32 {AFC88484-3A2E-32BC-25B2-23DF741D4F3D}.Debug|x64.ActiveCfg = Debug|x64 {AFC88484-3A2E-32BC-25B2-23DF741D4F3D}.Release|Win32.ActiveCfg = Release|Win32 diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj b/vsprojects/vcxproj/grpc/grpc.vcxproj index b0c76870fd..5b762108ba 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj @@ -405,6 +405,7 @@ <ClInclude Include="$(SolutionDir)\..\src\core\transport\transport_impl.h" /> <ClInclude Include="$(SolutionDir)\..\src\core\census\aggregation.h" /> <ClInclude Include="$(SolutionDir)\..\src\core\census\context.h" /> + <ClInclude Include="$(SolutionDir)\..\src\core\census\log.h" /> <ClInclude Include="$(SolutionDir)\..\src\core\census\rpc_metric_id.h" /> </ItemGroup> <ItemGroup> @@ -708,6 +709,8 @@ </ClCompile> <ClCompile Include="$(SolutionDir)\..\src\core\census\initialize.c"> </ClCompile> + <ClCompile Include="$(SolutionDir)\..\src\core\census\log.c"> + </ClCompile> <ClCompile Include="$(SolutionDir)\..\src\core\census\operation.c"> </ClCompile> <ClCompile Include="$(SolutionDir)\..\src\core\census\placeholders.c"> diff --git a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters index c0577c3eaf..d9a250da62 100644 --- a/vsprojects/vcxproj/grpc/grpc.vcxproj.filters +++ b/vsprojects/vcxproj/grpc/grpc.vcxproj.filters @@ -451,6 +451,9 @@ <ClCompile Include="$(SolutionDir)\..\src\core\census\initialize.c"> <Filter>src\core\census</Filter> </ClCompile> + <ClCompile Include="$(SolutionDir)\..\src\core\census\log.c"> + <Filter>src\core\census</Filter> + </ClCompile> <ClCompile Include="$(SolutionDir)\..\src\core\census\operation.c"> <Filter>src\core\census</Filter> </ClCompile> @@ -875,6 +878,9 @@ <ClInclude Include="$(SolutionDir)\..\src\core\census\context.h"> <Filter>src\core\census</Filter> </ClInclude> + <ClInclude Include="$(SolutionDir)\..\src\core\census\log.h"> + <Filter>src\core\census</Filter> + </ClInclude> <ClInclude Include="$(SolutionDir)\..\src\core\census\rpc_metric_id.h"> <Filter>src\core\census</Filter> </ClInclude> diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj index 3eac9314e5..858a6e888f 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj @@ -381,6 +381,7 @@ <ClInclude Include="$(SolutionDir)\..\src\core\transport\transport_impl.h" /> <ClInclude Include="$(SolutionDir)\..\src\core\census\aggregation.h" /> <ClInclude Include="$(SolutionDir)\..\src\core\census\context.h" /> + <ClInclude Include="$(SolutionDir)\..\src\core\census\log.h" /> <ClInclude Include="$(SolutionDir)\..\src\core\census\rpc_metric_id.h" /> </ItemGroup> <ItemGroup> @@ -644,6 +645,8 @@ </ClCompile> <ClCompile Include="$(SolutionDir)\..\src\core\census\initialize.c"> </ClCompile> + <ClCompile Include="$(SolutionDir)\..\src\core\census\log.c"> + </ClCompile> <ClCompile Include="$(SolutionDir)\..\src\core\census\operation.c"> </ClCompile> <ClCompile Include="$(SolutionDir)\..\src\core\census\placeholders.c"> diff --git a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters index 64f0407763..42b7298b9c 100644 --- a/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters +++ b/vsprojects/vcxproj/grpc_unsecure/grpc_unsecure.vcxproj.filters @@ -391,6 +391,9 @@ <ClCompile Include="$(SolutionDir)\..\src\core\census\initialize.c"> <Filter>src\core\census</Filter> </ClCompile> + <ClCompile Include="$(SolutionDir)\..\src\core\census\log.c"> + <Filter>src\core\census</Filter> + </ClCompile> <ClCompile Include="$(SolutionDir)\..\src\core\census\operation.c"> <Filter>src\core\census</Filter> </ClCompile> @@ -770,6 +773,9 @@ <ClInclude Include="$(SolutionDir)\..\src\core\census\context.h"> <Filter>src\core\census</Filter> </ClInclude> + <ClInclude Include="$(SolutionDir)\..\src\core\census\log.h"> + <Filter>src\core\census</Filter> + </ClInclude> <ClInclude Include="$(SolutionDir)\..\src\core\census\rpc_metric_id.h"> <Filter>src\core\census</Filter> </ClInclude> diff --git a/vsprojects/vcxproj/test/census_log_test/census_log_test.vcxproj b/vsprojects/vcxproj/test/census_log_test/census_log_test.vcxproj new file mode 100644 index 0000000000..851086d663 --- /dev/null +++ b/vsprojects/vcxproj/test/census_log_test/census_log_test.vcxproj @@ -0,0 +1,199 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project DefaultTargets="Build" ToolsVersion="12.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <Import Project="$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.1.0.204.1\build\native\grpc.dependencies.openssl.props" Condition="Exists('$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.1.0.204.1\build\native\1.0.204.1.props')" /> + <ItemGroup Label="ProjectConfigurations"> + <ProjectConfiguration Include="Debug|Win32"> + <Configuration>Debug</Configuration> + <Platform>Win32</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="Debug|x64"> + <Configuration>Debug</Configuration> + <Platform>x64</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="Release|Win32"> + <Configuration>Release</Configuration> + <Platform>Win32</Platform> + </ProjectConfiguration> + <ProjectConfiguration Include="Release|x64"> + <Configuration>Release</Configuration> + <Platform>x64</Platform> + </ProjectConfiguration> + </ItemGroup> + <PropertyGroup Label="Globals"> + <ProjectGuid>{C27CEE16-2BEC-5572-3956-677E9F6F8BED}</ProjectGuid> + <IgnoreWarnIntDirInTempDetected>true</IgnoreWarnIntDirInTempDetected> + <IntDir>$(SolutionDir)IntDir\$(MSBuildProjectName)\</IntDir> + </PropertyGroup> + <Import Project="$(VCTargetsPath)\Microsoft.Cpp.Default.props" /> + <PropertyGroup Condition="'$(VisualStudioVersion)' == '10.0'" Label="Configuration"> + <PlatformToolset>v100</PlatformToolset> + </PropertyGroup> + <PropertyGroup Condition="'$(VisualStudioVersion)' == '11.0'" Label="Configuration"> + <PlatformToolset>v110</PlatformToolset> + </PropertyGroup> + <PropertyGroup Condition="'$(VisualStudioVersion)' == '12.0'" Label="Configuration"> + <PlatformToolset>v120</PlatformToolset> + </PropertyGroup> + <PropertyGroup Condition="'$(VisualStudioVersion)' == '14.0'" Label="Configuration"> + <PlatformToolset>v140</PlatformToolset> + </PropertyGroup> + <PropertyGroup Condition="'$(Configuration)'=='Debug'" Label="Configuration"> + <ConfigurationType>Application</ConfigurationType> + <UseDebugLibraries>true</UseDebugLibraries> + <CharacterSet>Unicode</CharacterSet> + </PropertyGroup> + <PropertyGroup Condition="'$(Configuration)'=='Release'" Label="Configuration"> + <ConfigurationType>Application</ConfigurationType> + <UseDebugLibraries>false</UseDebugLibraries> + <WholeProgramOptimization>true</WholeProgramOptimization> + <CharacterSet>Unicode</CharacterSet> + </PropertyGroup> + <Import Project="$(VCTargetsPath)\Microsoft.Cpp.props" /> + <ImportGroup Label="ExtensionSettings"> + </ImportGroup> + <ImportGroup Label="PropertySheets"> + <Import Project="$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props" Condition="exists('$(UserRootDir)\Microsoft.Cpp.$(Platform).user.props')" Label="LocalAppDataPlatform" /> + <Import Project="$(SolutionDir)\..\vsprojects\global.props" /> + <Import Project="$(SolutionDir)\..\vsprojects\openssl.props" /> + <Import Project="$(SolutionDir)\..\vsprojects\winsock.props" /> + <Import Project="$(SolutionDir)\..\vsprojects\zlib.props" /> + </ImportGroup> + <PropertyGroup Label="UserMacros" /> + <PropertyGroup Condition="'$(Configuration)'=='Debug'"> + <TargetName>census_log_test</TargetName> + <Linkage-grpc_dependencies_zlib>static</Linkage-grpc_dependencies_zlib> + <Configuration-grpc_dependencies_zlib>Debug</Configuration-grpc_dependencies_zlib> + <Linkage-grpc_dependencies_openssl>static</Linkage-grpc_dependencies_openssl> + <Configuration-grpc_dependencies_openssl>Debug</Configuration-grpc_dependencies_openssl> + </PropertyGroup> + <PropertyGroup Condition="'$(Configuration)'=='Release'"> + <TargetName>census_log_test</TargetName> + <Linkage-grpc_dependencies_zlib>static</Linkage-grpc_dependencies_zlib> + <Configuration-grpc_dependencies_zlib>Release</Configuration-grpc_dependencies_zlib> + <Linkage-grpc_dependencies_openssl>static</Linkage-grpc_dependencies_openssl> + <Configuration-grpc_dependencies_openssl>Release</Configuration-grpc_dependencies_openssl> + </PropertyGroup> + <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|Win32'"> + <ClCompile> + <PrecompiledHeader>NotUsing</PrecompiledHeader> + <WarningLevel>Level3</WarningLevel> + <Optimization>Disabled</Optimization> + <PreprocessorDefinitions>WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <SDLCheck>true</SDLCheck> + <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> + <TreatWarningAsError>true</TreatWarningAsError> + <DebugInformationFormat Condition="$(Jenkins)">None</DebugInformationFormat> + <MinimalRebuild Condition="$(Jenkins)">false</MinimalRebuild> + </ClCompile> + <Link> + <SubSystem>Console</SubSystem> + <GenerateDebugInformation Condition="!$(Jenkins)">true</GenerateDebugInformation> + <GenerateDebugInformation Condition="$(Jenkins)">false</GenerateDebugInformation> + </Link> + </ItemDefinitionGroup> + + <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Debug|x64'"> + <ClCompile> + <PrecompiledHeader>NotUsing</PrecompiledHeader> + <WarningLevel>Level3</WarningLevel> + <Optimization>Disabled</Optimization> + <PreprocessorDefinitions>WIN32;_DEBUG;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <SDLCheck>true</SDLCheck> + <RuntimeLibrary>MultiThreadedDebug</RuntimeLibrary> + <TreatWarningAsError>true</TreatWarningAsError> + <DebugInformationFormat Condition="$(Jenkins)">None</DebugInformationFormat> + <MinimalRebuild Condition="$(Jenkins)">false</MinimalRebuild> + </ClCompile> + <Link> + <SubSystem>Console</SubSystem> + <GenerateDebugInformation Condition="!$(Jenkins)">true</GenerateDebugInformation> + <GenerateDebugInformation Condition="$(Jenkins)">false</GenerateDebugInformation> + </Link> + </ItemDefinitionGroup> + + <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|Win32'"> + <ClCompile> + <PrecompiledHeader>NotUsing</PrecompiledHeader> + <WarningLevel>Level3</WarningLevel> + <Optimization>MaxSpeed</Optimization> + <PreprocessorDefinitions>WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <FunctionLevelLinking>true</FunctionLevelLinking> + <IntrinsicFunctions>true</IntrinsicFunctions> + <SDLCheck>true</SDLCheck> + <RuntimeLibrary>MultiThreaded</RuntimeLibrary> + <TreatWarningAsError>true</TreatWarningAsError> + <DebugInformationFormat Condition="$(Jenkins)">None</DebugInformationFormat> + <MinimalRebuild Condition="$(Jenkins)">false</MinimalRebuild> + </ClCompile> + <Link> + <SubSystem>Console</SubSystem> + <GenerateDebugInformation Condition="!$(Jenkins)">true</GenerateDebugInformation> + <GenerateDebugInformation Condition="$(Jenkins)">false</GenerateDebugInformation> + <EnableCOMDATFolding>true</EnableCOMDATFolding> + <OptimizeReferences>true</OptimizeReferences> + </Link> + </ItemDefinitionGroup> + + <ItemDefinitionGroup Condition="'$(Configuration)|$(Platform)'=='Release|x64'"> + <ClCompile> + <PrecompiledHeader>NotUsing</PrecompiledHeader> + <WarningLevel>Level3</WarningLevel> + <Optimization>MaxSpeed</Optimization> + <PreprocessorDefinitions>WIN32;NDEBUG;_LIB;%(PreprocessorDefinitions)</PreprocessorDefinitions> + <FunctionLevelLinking>true</FunctionLevelLinking> + <IntrinsicFunctions>true</IntrinsicFunctions> + <SDLCheck>true</SDLCheck> + <RuntimeLibrary>MultiThreaded</RuntimeLibrary> + <TreatWarningAsError>true</TreatWarningAsError> + <DebugInformationFormat Condition="$(Jenkins)">None</DebugInformationFormat> + <MinimalRebuild Condition="$(Jenkins)">false</MinimalRebuild> + </ClCompile> + <Link> + <SubSystem>Console</SubSystem> + <GenerateDebugInformation Condition="!$(Jenkins)">true</GenerateDebugInformation> + <GenerateDebugInformation Condition="$(Jenkins)">false</GenerateDebugInformation> + <EnableCOMDATFolding>true</EnableCOMDATFolding> + <OptimizeReferences>true</OptimizeReferences> + </Link> + </ItemDefinitionGroup> + + <ItemGroup> + <ClCompile Include="$(SolutionDir)\..\test\core\census\log_test.c"> + </ClCompile> + </ItemGroup> + <ItemGroup> + <ProjectReference Include="$(SolutionDir)\..\vsprojects\vcxproj\.\grpc_test_util\grpc_test_util.vcxproj"> + <Project>{17BCAFC0-5FDC-4C94-AEB9-95F3E220614B}</Project> + </ProjectReference> + <ProjectReference Include="$(SolutionDir)\..\vsprojects\vcxproj\.\grpc\grpc.vcxproj"> + <Project>{29D16885-7228-4C31-81ED-5F9187C7F2A9}</Project> + </ProjectReference> + <ProjectReference Include="$(SolutionDir)\..\vsprojects\vcxproj\.\gpr_test_util\gpr_test_util.vcxproj"> + <Project>{EAB0A629-17A9-44DB-B5FF-E91A721FE037}</Project> + </ProjectReference> + <ProjectReference Include="$(SolutionDir)\..\vsprojects\vcxproj\.\gpr\gpr.vcxproj"> + <Project>{B23D3D1A-9438-4EDA-BEB6-9A0A03D17792}</Project> + </ProjectReference> + </ItemGroup> + <ItemGroup> + <None Include="packages.config" /> + </ItemGroup> + <Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" /> + <ImportGroup Label="ExtensionTargets"> + <Import Project="$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.zlib.redist.1.2.8.10\build\native\grpc.dependencies.zlib.redist.targets" Condition="Exists('$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.zlib.redist.1.2.8.10\build\native\grpc.dependencies\grpc.dependencies.zlib.targets')" /> + <Import Project="$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.zlib.1.2.8.10\build\native\grpc.dependencies.zlib.targets" Condition="Exists('$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.zlib.1.2.8.10\build\native\grpc.dependencies\grpc.dependencies.zlib.targets')" /> + <Import Project="$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.redist.1.0.204.1\build\native\grpc.dependencies.openssl.redist.targets" Condition="Exists('$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.redist.1.0.204.1\build\native\grpc.dependencies\grpc.dependencies.openssl.targets')" /> + <Import Project="$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.1.0.204.1\build\native\grpc.dependencies.openssl.targets" Condition="Exists('$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.1.0.204.1\build\native\grpc.dependencies\grpc.dependencies.openssl.targets')" /> + </ImportGroup> + <Target Name="EnsureNuGetPackageBuildImports" BeforeTargets="PrepareForBuild"> + <PropertyGroup> + <ErrorText>This project references NuGet package(s) that are missing on this computer. Enable NuGet Package Restore to download them. For more information, see http://go.microsoft.com/fwlink/?LinkID=322105. The missing file is {0}.</ErrorText> + </PropertyGroup> + <Error Condition="!Exists('$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.zlib.redist.1.2.8.10\build\native\grpc.dependencies.zlib.redist.targets')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.zlib.redist.1.2.8.10\build\native\grpc.dependencies.zlib.redist.targets')" /> + <Error Condition="!Exists('$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.zlib.1.2.8.10\build\native\grpc.dependencies.zlib.targets')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.zlib.1.2.8.10\build\native\grpc.dependencies.zlib.targets')" /> + <Error Condition="!Exists('$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.redist.1.0.204.1\build\native\grpc.dependencies.openssl.redist.targets')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.redist.1.0.204.1\build\native\grpc.dependencies.openssl.redist.targets')" /> + <Error Condition="!Exists('$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.1.0.204.1\build\native\grpc.dependencies.openssl.props')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.1.0.204.1\build\native\grpc.dependencies.openssl.props')" /> + <Error Condition="!Exists('$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.1.0.204.1\build\native\grpc.dependencies.openssl.targets')" Text="$([System.String]::Format('$(ErrorText)', '$(SolutionDir)\..\vsprojects\packages\grpc.dependencies.openssl.1.0.204.1\build\native\grpc.dependencies.openssl.targets')" /> + </Target> +</Project> + diff --git a/vsprojects/vcxproj/test/census_log_test/census_log_test.vcxproj.filters b/vsprojects/vcxproj/test/census_log_test/census_log_test.vcxproj.filters new file mode 100644 index 0000000000..135c77847f --- /dev/null +++ b/vsprojects/vcxproj/test/census_log_test/census_log_test.vcxproj.filters @@ -0,0 +1,21 @@ +<?xml version="1.0" encoding="utf-8"?> +<Project ToolsVersion="4.0" xmlns="http://schemas.microsoft.com/developer/msbuild/2003"> + <ItemGroup> + <ClCompile Include="$(SolutionDir)\..\test\core\census\log_test.c"> + <Filter>test\core\census</Filter> + </ClCompile> + </ItemGroup> + + <ItemGroup> + <Filter Include="test"> + <UniqueIdentifier>{4d0aae38-6975-cafb-30a6-a7c2c87d22ff}</UniqueIdentifier> + </Filter> + <Filter Include="test\core"> + <UniqueIdentifier>{fb85321f-d3b5-ef2f-c5aa-34660a5e0c7b}</UniqueIdentifier> + </Filter> + <Filter Include="test\core\census"> + <UniqueIdentifier>{f23141da-cbe2-70fa-8207-858af868eb18}</UniqueIdentifier> + </Filter> + </ItemGroup> +</Project> + |