Merge pull request #14751 from ncteisen/channel-tracing

[Fix Forward]: Channel Tracing Implementation; Part 1

14 files changed, 593 insertions, 10 deletions

diff --git a/src/core/ext/filters/client_channel/client_channel.cc b/src/core/ext/filters/client_channel/client_channel.cc
index bbc5160bec..bf3911e5ee 100644
--- a/src/core/ext/filters/client_channel/client_channel.cc
+++ b/src/core/ext/filters/client_channel/client_channel.cc
@@ -38,12 +38,12 @@
 #include "src/core/ext/filters/client_channel/proxy_mapper_registry.h"
 #include "src/core/ext/filters/client_channel/resolver_registry.h"
 #include "src/core/ext/filters/client_channel/retry_throttle.h"
-#include "src/core/ext/filters/client_channel/status_util.h"
 #include "src/core/ext/filters/client_channel/subchannel.h"
 #include "src/core/ext/filters/deadline/deadline_filter.h"
 #include "src/core/lib/backoff/backoff.h"
 #include "src/core/lib/channel/channel_args.h"
 #include "src/core/lib/channel/connected_channel.h"
+#include "src/core/lib/channel/status_util.h"
 #include "src/core/lib/gpr/string.h"
 #include "src/core/lib/gprpp/inlined_vector.h"
 #include "src/core/lib/gprpp/manual_constructor.h"
diff --git a/src/core/ext/filters/client_channel/method_params.cc b/src/core/ext/filters/client_channel/method_params.cc
index 374b87e170..1f116bb67d 100644
--- a/src/core/ext/filters/client_channel/method_params.cc
+++ b/src/core/ext/filters/client_channel/method_params.cc
@@ -26,7 +26,7 @@
 #include <grpc/support/string_util.h>
 
 #include "src/core/ext/filters/client_channel/method_params.h"
-#include "src/core/ext/filters/client_channel/status_util.h"
+#include "src/core/lib/channel/status_util.h"
 #include "src/core/lib/gpr/string.h"
 #include "src/core/lib/gprpp/memory.h"
 
diff --git a/src/core/ext/filters/client_channel/method_params.h b/src/core/ext/filters/client_channel/method_params.h
index 48ece29867..099924edf3 100644
--- a/src/core/ext/filters/client_channel/method_params.h
+++ b/src/core/ext/filters/client_channel/method_params.h
@@ -21,7 +21,7 @@
 
 #include <grpc/support/port_platform.h>
 
-#include "src/core/ext/filters/client_channel/status_util.h"
+#include "src/core/lib/channel/status_util.h"
 #include "src/core/lib/gprpp/ref_counted.h"
 #include "src/core/lib/gprpp/ref_counted_ptr.h"
 #include "src/core/lib/iomgr/exec_ctx.h"  // for grpc_millis
diff --git a/src/core/ext/filters/client_channel/subchannel.cc b/src/core/ext/filters/client_channel/subchannel.cc
index cae7cc35e3..d7815fb7e1 100644
--- a/src/core/ext/filters/client_channel/subchannel.cc
+++ b/src/core/ext/filters/client_channel/subchannel.cc
@@ -40,6 +40,7 @@
 #include "src/core/lib/debug/stats.h"
 #include "src/core/lib/gprpp/debug_location.h"
 #include "src/core/lib/gprpp/manual_constructor.h"
+#include "src/core/lib/gprpp/ref_counted_ptr.h"
 #include "src/core/lib/iomgr/sockaddr_utils.h"
 #include "src/core/lib/iomgr/timer.h"
 #include "src/core/lib/profiling/timers.h"
diff --git a/src/core/lib/channel/channel_trace.cc b/src/core/lib/channel/channel_trace.cc
new file mode 100644
index 0000000000..654300cd32
--- /dev/null
+++ b/src/core/lib/channel/channel_trace.cc
@@ -0,0 +1,239 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <grpc/impl/codegen/port_platform.h>
+
+#include "src/core/lib/channel/channel_trace.h"
+
+#include <grpc/grpc.h>
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+#include <grpc/support/string_util.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+
+#include "src/core/lib/channel/channel_trace_registry.h"
+#include "src/core/lib/channel/status_util.h"
+#include "src/core/lib/gpr/string.h"
+#include "src/core/lib/gpr/useful.h"
+#include "src/core/lib/gprpp/memory.h"
+#include "src/core/lib/iomgr/error.h"
+#include "src/core/lib/slice/slice_internal.h"
+#include "src/core/lib/surface/channel.h"
+#include "src/core/lib/transport/connectivity_state.h"
+#include "src/core/lib/transport/error_utils.h"
+
+namespace grpc_core {
+
+ChannelTrace::TraceEvent::TraceEvent(
+    Severity severity, grpc_slice data,
+    RefCountedPtr<ChannelTrace> referenced_tracer, ReferencedType type)
+    : severity_(severity),
+      data_(data),
+      timestamp_(grpc_millis_to_timespec(grpc_core::ExecCtx::Get()->Now(),
+                                         GPR_CLOCK_REALTIME)),
+      next_(nullptr),
+      referenced_tracer_(std::move(referenced_tracer)),
+      referenced_type_(type) {}
+
+ChannelTrace::TraceEvent::TraceEvent(Severity severity, grpc_slice data)
+    : severity_(severity),
+      data_(data),
+      timestamp_(grpc_millis_to_timespec(grpc_core::ExecCtx::Get()->Now(),
+                                         GPR_CLOCK_REALTIME)),
+      next_(nullptr) {}
+
+ChannelTrace::TraceEvent::~TraceEvent() { grpc_slice_unref_internal(data_); }
+
+ChannelTrace::ChannelTrace(size_t max_events)
+    : channel_uuid_(-1),
+      num_events_logged_(0),
+      list_size_(0),
+      max_list_size_(max_events),
+      head_trace_(nullptr),
+      tail_trace_(nullptr) {
+  if (max_list_size_ == 0) return;  // tracing is disabled if max_events == 0
+  gpr_mu_init(&tracer_mu_);
+  channel_uuid_ = grpc_channel_trace_registry_register_channel_trace(this);
+  time_created_ = grpc_millis_to_timespec(grpc_core::ExecCtx::Get()->Now(),
+                                          GPR_CLOCK_REALTIME);
+}
+
+ChannelTrace::~ChannelTrace() {
+  if (max_list_size_ == 0) return;  // tracing is disabled if max_events == 0
+  TraceEvent* it = head_trace_;
+  while (it != nullptr) {
+    TraceEvent* to_free = it;
+    it = it->next();
+    Delete<TraceEvent>(to_free);
+  }
+  grpc_channel_trace_registry_unregister_channel_trace(channel_uuid_);
+  gpr_mu_destroy(&tracer_mu_);
+}
+
+intptr_t ChannelTrace::GetUuid() const { return channel_uuid_; }
+
+void ChannelTrace::AddTraceEventHelper(TraceEvent* new_trace_event) {
+  ++num_events_logged_;
+  // first event case
+  if (head_trace_ == nullptr) {
+    head_trace_ = tail_trace_ = new_trace_event;
+  }
+  // regular event add case
+  else {
+    tail_trace_->set_next(new_trace_event);
+    tail_trace_ = tail_trace_->next();
+  }
+  ++list_size_;
+  // maybe garbage collect the end
+  if (list_size_ > max_list_size_) {
+    TraceEvent* to_free = head_trace_;
+    head_trace_ = head_trace_->next();
+    Delete<TraceEvent>(to_free);
+    --list_size_;
+  }
+}
+
+void ChannelTrace::AddTraceEvent(Severity severity, grpc_slice data) {
+  if (max_list_size_ == 0) return;  // tracing is disabled if max_events == 0
+  AddTraceEventHelper(New<TraceEvent>(severity, data));
+}
+
+void ChannelTrace::AddTraceEventReferencingChannel(
+    Severity severity, grpc_slice data,
+    RefCountedPtr<ChannelTrace> referenced_tracer) {
+  if (max_list_size_ == 0) return;  // tracing is disabled if max_events == 0
+  // create and fill up the new event
+  AddTraceEventHelper(
+      New<TraceEvent>(severity, data, std::move(referenced_tracer), Channel));
+}
+
+void ChannelTrace::AddTraceEventReferencingSubchannel(
+    Severity severity, grpc_slice data,
+    RefCountedPtr<ChannelTrace> referenced_tracer) {
+  if (max_list_size_ == 0) return;  // tracing is disabled if max_events == 0
+  // create and fill up the new event
+  AddTraceEventHelper(New<TraceEvent>(
+      severity, data, std::move(referenced_tracer), Subchannel));
+}
+
+namespace {
+
+// returns an allocated string that represents tm according to RFC-3339, and,
+// more specifically, follows:
+// https://developers.google.com/protocol-buffers/docs/proto3#json
+//
+// "Uses RFC 3339, where generated output will always be Z-normalized and uses
+// 0, 3, 6 or 9 fractional digits."
+char* fmt_time(gpr_timespec tm) {
+  char time_buffer[35];
+  char ns_buffer[11];  // '.' + 9 digits of precision
+  struct tm* tm_info = localtime((const time_t*)&tm.tv_sec);
+  strftime(time_buffer, sizeof(time_buffer), "%Y-%m-%dT%H:%M:%S", tm_info);
+  snprintf(ns_buffer, 11, ".%09d", tm.tv_nsec);
+  // This loop trims off trailing zeros by inserting a null character that the
+  // right point. We iterate in chunks of three because we want 0, 3, 6, or 9
+  // fractional digits.
+  for (int i = 7; i >= 1; i -= 3) {
+    if (ns_buffer[i] == '0' && ns_buffer[i + 1] == '0' &&
+        ns_buffer[i + 2] == '0') {
+      ns_buffer[i] = '\0';
+      // Edge case in which all fractional digits were 0.
+      if (i == 1) {
+        ns_buffer[0] = '\0';
+      }
+    } else {
+      break;
+    }
+  }
+  char* full_time_str;
+  gpr_asprintf(&full_time_str, "%s%sZ", time_buffer, ns_buffer);
+  return full_time_str;
+}
+
+const char* severity_string(ChannelTrace::Severity severity) {
+  switch (severity) {
+    case ChannelTrace::Severity::Info:
+      return "CT_INFO";
+    case ChannelTrace::Severity::Warning:
+      return "CT_WARNING";
+    case ChannelTrace::Severity::Error:
+      return "CT_ERROR";
+    default:
+      GPR_UNREACHABLE_CODE(return "CT_UNKNOWN");
+  }
+}
+
+}  // anonymous namespace
+
+void ChannelTrace::TraceEvent::RenderTraceEvent(grpc_json* json) const {
+  grpc_json* json_iterator = nullptr;
+  json_iterator = grpc_json_create_child(json_iterator, json, "description",
+                                         grpc_slice_to_c_string(data_),
+                                         GRPC_JSON_STRING, true);
+  json_iterator = grpc_json_create_child(json_iterator, json, "severity",
+                                         severity_string(severity_),
+                                         GRPC_JSON_STRING, false);
+  json_iterator =
+      grpc_json_create_child(json_iterator, json, "timestamp",
+                             fmt_time(timestamp_), GRPC_JSON_STRING, true);
+  if (referenced_tracer_ != nullptr) {
+    char* uuid_str;
+    gpr_asprintf(&uuid_str, "%" PRIdPTR, referenced_tracer_->channel_uuid_);
+    grpc_json* child_ref = grpc_json_create_child(
+        json_iterator, json,
+        (referenced_type_ == Channel) ? "channelRef" : "subchannelRef", nullptr,
+        GRPC_JSON_OBJECT, false);
+    json_iterator = grpc_json_create_child(
+        nullptr, child_ref,
+        (referenced_type_ == Channel) ? "channelId" : "subchannelId", uuid_str,
+        GRPC_JSON_STRING, true);
+    json_iterator = child_ref;
+  }
+}
+
+char* ChannelTrace::RenderTrace() const {
+  if (!max_list_size_)
+    return nullptr;  // tracing is disabled if max_events == 0
+  grpc_json* json = grpc_json_create(GRPC_JSON_OBJECT);
+  char* num_events_logged_str;
+  gpr_asprintf(&num_events_logged_str, "%" PRId64, num_events_logged_);
+  grpc_json* json_iterator = nullptr;
+  json_iterator =
+      grpc_json_create_child(json_iterator, json, "numEventsLogged",
+                             num_events_logged_str, GRPC_JSON_STRING, true);
+  json_iterator =
+      grpc_json_create_child(json_iterator, json, "creationTime",
+                             fmt_time(time_created_), GRPC_JSON_STRING, true);
+  grpc_json* events = grpc_json_create_child(json_iterator, json, "events",
+                                             nullptr, GRPC_JSON_ARRAY, false);
+  json_iterator = nullptr;
+  TraceEvent* it = head_trace_;
+  while (it != nullptr) {
+    json_iterator = grpc_json_create_child(json_iterator, events, nullptr,
+                                           nullptr, GRPC_JSON_OBJECT, false);
+    it->RenderTraceEvent(json_iterator);
+    it = it->next();
+  }
+  char* json_str = grpc_json_dump_to_string(json, 0);
+  grpc_json_destroy(json);
+  return json_str;
+}
+
+}  // namespace grpc_core
diff --git a/src/core/lib/channel/channel_trace.h b/src/core/lib/channel/channel_trace.h
new file mode 100644
index 0000000000..1df1e585f2
--- /dev/null
+++ b/src/core/lib/channel/channel_trace.h
@@ -0,0 +1,133 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_CHANNEL_CHANNEL_TRACE_H
+#define GRPC_CORE_LIB_CHANNEL_CHANNEL_TRACE_H
+
+#include <grpc/impl/codegen/port_platform.h>
+
+#include <grpc/grpc.h>
+#include "src/core/lib/gprpp/ref_counted.h"
+#include "src/core/lib/gprpp/ref_counted_ptr.h"
+#include "src/core/lib/iomgr/error.h"
+#include "src/core/lib/json/json.h"
+
+namespace grpc_core {
+
+// Object used to hold live data for a channel. This data is exposed via the
+// channelz service:
+// https://github.com/grpc/proposal/blob/master/A14-channelz.md
+class ChannelTrace : public RefCounted<ChannelTrace> {
+ public:
+  ChannelTrace(size_t max_events);
+  ~ChannelTrace();
+
+  // returns the tracer's uuid
+  intptr_t GetUuid() const;
+
+  enum Severity {
+    Unset = 0,  // never to be used
+    Info,       // we start at 1 to avoid using proto default values
+    Warning,
+    Error
+  };
+
+  // Adds a new trace event to the tracing object
+  //
+  // TODO(ncteisen): as this call is used more and more throughout the gRPC
+  // stack, determine if it makes more sense to accept a char* instead of a
+  // slice.
+  void AddTraceEvent(Severity severity, grpc_slice data);
+
+  // Adds a new trace event to the tracing object. This trace event refers to a
+  // an event on a child of the channel. For example, if this channel has
+  // created a new subchannel, then it would record that with a TraceEvent
+  // referencing the new subchannel.
+  //
+  // TODO(ncteisen): Once channelz is implemented, the events should reference
+  // the overall channelz object, not just the ChannelTrace object.
+  // TODO(ncteisen): as this call is used more and more throughout the gRPC
+  // stack, determine if it makes more sense to accept a char* instead of a
+  // slice.
+  void AddTraceEventReferencingChannel(
+      Severity severity, grpc_slice data,
+      RefCountedPtr<ChannelTrace> referenced_tracer);
+  void AddTraceEventReferencingSubchannel(
+      Severity severity, grpc_slice data,
+      RefCountedPtr<ChannelTrace> referenced_tracer);
+
+  // Returns the tracing data rendered as a grpc json string.
+  // The string is owned by the caller and must be freed.
+  char* RenderTrace() const;
+
+ private:
+  // Types of objects that can be references by trace events.
+  enum ReferencedType { Channel, Subchannel };
+  // Private class to encapsulate all the data and bookkeeping needed for a
+  // a trace event.
+  class TraceEvent {
+   public:
+    // Constructor for a TraceEvent that references a different channel.
+    // TODO(ncteisen): once channelz is implemented, this should reference the
+    // overall channelz object, not just the ChannelTrace object
+    TraceEvent(Severity severity, grpc_slice data,
+               RefCountedPtr<ChannelTrace> referenced_tracer,
+               ReferencedType type);
+
+    // Constructor for a TraceEvent that does not reverence a different
+    // channel.
+    TraceEvent(Severity severity, grpc_slice data);
+
+    ~TraceEvent();
+
+    // Renders the data inside of this TraceEvent into a json object. This is
+    // used by the ChannelTrace, when it is rendering itself.
+    void RenderTraceEvent(grpc_json* json) const;
+
+    // set and get for the next_ pointer.
+    TraceEvent* next() const { return next_; }
+    void set_next(TraceEvent* next) { next_ = next; }
+
+   private:
+    Severity severity_;
+    grpc_slice data_;
+    gpr_timespec timestamp_;
+    TraceEvent* next_;
+    // the tracer object for the (sub)channel that this trace event refers to.
+    RefCountedPtr<ChannelTrace> referenced_tracer_;
+    // the type that the referenced tracer points to. Unused if this trace
+    // does not point to any channel or subchannel
+    ReferencedType referenced_type_;
+  };  // TraceEvent
+
+  // Internal helper to add and link in a trace event
+  void AddTraceEventHelper(TraceEvent* new_trace_event);
+
+  gpr_mu tracer_mu_;
+  intptr_t channel_uuid_;
+  uint64_t num_events_logged_;
+  size_t list_size_;
+  size_t max_list_size_;
+  TraceEvent* head_trace_;
+  TraceEvent* tail_trace_;
+  gpr_timespec time_created_;
+};
+
+}  // namespace grpc_core
+
+#endif /* GRPC_CORE_LIB_CHANNEL_CHANNEL_TRACE_H */
diff --git a/src/core/lib/channel/channel_trace_registry.cc b/src/core/lib/channel/channel_trace_registry.cc
new file mode 100644
index 0000000000..6c82431467
--- /dev/null
+++ b/src/core/lib/channel/channel_trace_registry.cc
@@ -0,0 +1,80 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#include <grpc/impl/codegen/port_platform.h>
+
+#include "src/core/lib/avl/avl.h"
+#include "src/core/lib/channel/channel_trace.h"
+#include "src/core/lib/channel/channel_trace_registry.h"
+#include "src/core/lib/gpr/useful.h"
+
+#include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
+
+// file global lock and avl.
+static gpr_mu g_mu;
+static grpc_avl g_avl;
+static gpr_atm g_uuid = 0;
+
+// avl vtable for uuid (intptr_t) -> ChannelTrace
+// this table is only looking, it does not own anything.
+static void destroy_intptr(void* not_used, void* user_data) {}
+static void* copy_intptr(void* key, void* user_data) { return key; }
+static long compare_intptr(void* key1, void* key2, void* user_data) {
+  return GPR_ICMP(key1, key2);
+}
+
+static void destroy_channel_trace(void* trace, void* user_data) {}
+static void* copy_channel_trace(void* trace, void* user_data) { return trace; }
+static const grpc_avl_vtable avl_vtable = {
+    destroy_intptr, copy_intptr, compare_intptr, destroy_channel_trace,
+    copy_channel_trace};
+
+void grpc_channel_trace_registry_init() {
+  gpr_mu_init(&g_mu);
+  g_avl = grpc_avl_create(&avl_vtable);
+}
+
+void grpc_channel_trace_registry_shutdown() {
+  grpc_avl_unref(g_avl, nullptr);
+  gpr_mu_destroy(&g_mu);
+}
+
+intptr_t grpc_channel_trace_registry_register_channel_trace(
+    grpc_core::ChannelTrace* channel_trace) {
+  intptr_t prior = gpr_atm_no_barrier_fetch_add(&g_uuid, 1);
+  gpr_mu_lock(&g_mu);
+  g_avl = grpc_avl_add(g_avl, (void*)prior, channel_trace, nullptr);
+  gpr_mu_unlock(&g_mu);
+  return prior;
+}
+
+void grpc_channel_trace_registry_unregister_channel_trace(intptr_t uuid) {
+  gpr_mu_lock(&g_mu);
+  g_avl = grpc_avl_remove(g_avl, (void*)uuid, nullptr);
+  gpr_mu_unlock(&g_mu);
+}
+
+grpc_core::ChannelTrace* grpc_channel_trace_registry_get_channel_trace(
+    intptr_t uuid) {
+  gpr_mu_lock(&g_mu);
+  grpc_core::ChannelTrace* ret = static_cast<grpc_core::ChannelTrace*>(
+      grpc_avl_get(g_avl, (void*)uuid, nullptr));
+  gpr_mu_unlock(&g_mu);
+  return ret;
+}
diff --git a/src/core/lib/channel/channel_trace_registry.h b/src/core/lib/channel/channel_trace_registry.h
new file mode 100644
index 0000000000..391ecba7de
--- /dev/null
+++ b/src/core/lib/channel/channel_trace_registry.h
@@ -0,0 +1,43 @@
+/*
+ *
+ * Copyright 2017 gRPC authors.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ *
+ */
+
+#ifndef GRPC_CORE_LIB_CHANNEL_CHANNEL_TRACE_REGISTRY_H
+#define GRPC_CORE_LIB_CHANNEL_CHANNEL_TRACE_REGISTRY_H
+
+#include <grpc/impl/codegen/port_platform.h>
+
+#include "src/core/lib/channel/channel_trace.h"
+
+#include <stdint.h>
+
+// TODO(ncteisen): convert this file to C++
+
+void grpc_channel_trace_registry_init();
+void grpc_channel_trace_registry_shutdown();
+
+// globally registers a ChannelTrace. Returns its unique uuid
+intptr_t grpc_channel_trace_registry_register_channel_trace(
+    grpc_core::ChannelTrace* channel_trace);
+// globally unregisters the ChannelTrace that is associated to uuid.
+void grpc_channel_trace_registry_unregister_channel_trace(intptr_t uuid);
+// if object with uuid has previously been registered, returns the ChannelTrace
+// associated with that uuid. Else returns nullptr.
+grpc_core::ChannelTrace* grpc_channel_trace_registry_get_channel_trace(
+    intptr_t uuid);
+
+#endif /* GRPC_CORE_LIB_CHANNEL_CHANNEL_TRACE_REGISTRY_H */
diff --git a/src/core/ext/filters/client_channel/status_util.cc b/src/core/lib/channel/status_util.cc
index 11f732ab44..563db40846 100644
--- a/src/core/ext/filters/client_channel/status_util.cc
+++ b/src/core/lib/channel/status_util.cc
@@ -18,7 +18,7 @@
 
 #include <grpc/support/port_platform.h>
 
-#include "src/core/ext/filters/client_channel/status_util.h"
+#include "src/core/lib/channel/status_util.h"
 
 #include "src/core/lib/gpr/useful.h"
 
diff --git a/src/core/ext/filters/client_channel/status_util.h b/src/core/lib/channel/status_util.h
index e018709730..5409de6b3c 100644
--- a/src/core/ext/filters/client_channel/status_util.h
+++ b/src/core/lib/channel/status_util.h
@@ -16,8 +16,8 @@
  *
  */
 
-#ifndef GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_STATUS_UTIL_H
-#define GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_STATUS_UTIL_H
+#ifndef GRPC_CORE_LIB_CHANNEL_STATUS_UTIL_H
+#define GRPC_CORE_LIB_CHANNEL_STATUS_UTIL_H
 
 #include <grpc/support/port_platform.h>
 
@@ -55,4 +55,4 @@ class StatusCodeSet {
 }  // namespace internal
 }  // namespace grpc_core
 
-#endif /* GRPC_CORE_EXT_FILTERS_CLIENT_CHANNEL_STATUS_UTIL_H */
+#endif /* GRPC_CORE_LIB_CHANNEL_STATUS_UTIL_H */
diff --git a/src/core/lib/json/json.cc b/src/core/lib/json/json.cc
index 2141db4c5b..816241bbf0 100644
--- a/src/core/lib/json/json.cc
+++ b/src/core/lib/json/json.cc
@@ -21,6 +21,7 @@
 #include <string.h>
 
 #include <grpc/support/alloc.h>
+#include <grpc/support/log.h>
 
 #include "src/core/lib/json/json.h"
 
@@ -46,5 +47,40 @@ void grpc_json_destroy(grpc_json* json) {
     json->parent->child = json->next;
   }
 
+  if (json->owns_value) {
+    gpr_free((void*)json->value);
+  }
+
   gpr_free(json);
 }
+
+grpc_json* grpc_json_link_child(grpc_json* parent, grpc_json* child,
+                                grpc_json* sibling) {
+  // first child case.
+  if (parent->child == nullptr) {
+    GPR_ASSERT(sibling == nullptr);
+    parent->child = child;
+    return child;
+  }
+  if (sibling == nullptr) {
+    sibling = parent->child;
+  }
+  // always find the right most sibling.
+  while (sibling->next != nullptr) {
+    sibling = sibling->next;
+  }
+  sibling->next = child;
+  return child;
+}
+
+grpc_json* grpc_json_create_child(grpc_json* sibling, grpc_json* parent,
+                                  const char* key, const char* value,
+                                  grpc_json_type type, bool owns_value) {
+  grpc_json* child = grpc_json_create(type);
+  grpc_json_link_child(parent, child, sibling);
+  child->owns_value = owns_value;
+  child->parent = parent;
+  child->value = value;
+  child->key = key;
+  return child;
+}
diff --git a/src/core/lib/json/json.h b/src/core/lib/json/json.h
index 3a62ef9cfb..f93b43048b 100644
--- a/src/core/lib/json/json.h
+++ b/src/core/lib/json/json.h
@@ -21,6 +21,7 @@
 
 #include <grpc/support/port_platform.h>
 
+#include <stdbool.h>
 #include <stdlib.h>
 
 #include "src/core/lib/json/json_common.h"
@@ -37,6 +38,9 @@ typedef struct grpc_json {
   grpc_json_type type;
   const char* key;
   const char* value;
+
+  /* if set, destructor will free value */
+  bool owns_value;
 } grpc_json;
 
 /* The next two functions are going to parse the input string, and
@@ -67,9 +71,24 @@ char* grpc_json_dump_to_string(grpc_json* json, int indent);
 
 /* Use these to create or delete a grpc_json object.
  * Deletion is recursive. We will not attempt to free any of the strings
- * in any of the objects of that tree.
+ * in any of the objects of that tree, unless the boolean, owns_value,
+ * is true.
  */
 grpc_json* grpc_json_create(grpc_json_type type);
 void grpc_json_destroy(grpc_json* json);
 
+/* Links the child json object into the parent's json tree. If the parent
+ * already has children, then passing in the most recently added child as the
+ * sibling parameter is an optimization. For if sibling is NULL, this function
+ * will manually traverse the tree in order to find the right most sibling.
+ */
+grpc_json* grpc_json_link_child(grpc_json* parent, grpc_json* child,
+                                grpc_json* sibling);
+
+/* Creates a child json object into the parent's json tree then links it in
+ * as described above. */
+grpc_json* grpc_json_create_child(grpc_json* sibling, grpc_json* parent,
+                                  const char* key, const char* value,
+                                  grpc_json_type type, bool owns_value);
+
 #endif /* GRPC_CORE_LIB_JSON_JSON_H */
diff --git a/src/core/lib/surface/channel.cc b/src/core/lib/surface/channel.cc
index 03353d6beb..cecc15b2df 100644
--- a/src/core/lib/surface/channel.cc
+++ b/src/core/lib/surface/channel.cc
@@ -21,6 +21,7 @@
 #include "src/core/lib/surface/channel.h"
 
 #include <inttypes.h>
+#include <limits.h>
 #include <stdlib.h>
 #include <string.h>
 
@@ -30,8 +31,12 @@
 #include <grpc/support/string_util.h>
 
 #include "src/core/lib/channel/channel_args.h"
+#include "src/core/lib/channel/channel_trace.h"
 #include "src/core/lib/debug/stats.h"
 #include "src/core/lib/gpr/string.h"
+#include "src/core/lib/gprpp/manual_constructor.h"
+#include "src/core/lib/gprpp/memory.h"
+#include "src/core/lib/gprpp/ref_counted_ptr.h"
 #include "src/core/lib/iomgr/iomgr.h"
 #include "src/core/lib/slice/slice_internal.h"
 #include "src/core/lib/surface/api_trace.h"
@@ -62,6 +67,8 @@ struct grpc_channel {
   gpr_mu registered_call_mu;
   registered_call* registered_calls;
 
+  grpc_core::RefCountedPtr<grpc_core::ChannelTrace> tracer;
+
   char* target;
 };
 
@@ -93,12 +100,14 @@ grpc_channel* grpc_channel_create_with_builder(
             grpc_error_string(error));
     GRPC_ERROR_UNREF(error);
     gpr_free(target);
-    goto done;
+    grpc_channel_args_destroy(args);
+    return channel;
   }
 
   memset(channel, 0, sizeof(*channel));
   channel->target = target;
   channel->is_client = grpc_channel_stack_type_is_client(channel_stack_type);
+  size_t channel_tracer_max_nodes = 0;  // default to off
   gpr_mu_init(&channel->registered_call_mu);
   channel->registered_calls = nullptr;
 
@@ -161,14 +170,33 @@ grpc_channel* grpc_channel_create_with_builder(
       channel->compression_options.enabled_algorithms_bitset =
           static_cast<uint32_t>(args->args[i].value.integer) |
           0x1; /* always support no compression */
+    } else if (0 == strcmp(args->args[i].key,
+                           GRPC_ARG_MAX_CHANNEL_TRACE_EVENTS_PER_NODE)) {
+      GPR_ASSERT(channel_tracer_max_nodes == 0);
+      // max_nodes defaults to 0 (which is off), clamped between 0 and INT_MAX
+      const grpc_integer_options options = {0, 0, INT_MAX};
+      channel_tracer_max_nodes =
+          (size_t)grpc_channel_arg_get_integer(&args->args[i], options);
     }
   }
 
-done:
   grpc_channel_args_destroy(args);
+  channel->tracer = grpc_core::MakeRefCounted<grpc_core::ChannelTrace>(
+      channel_tracer_max_nodes);
+  channel->tracer->AddTraceEvent(
+      grpc_core::ChannelTrace::Severity::Info,
+      grpc_slice_from_static_string("Channel created"));
   return channel;
 }
 
+char* grpc_channel_get_trace(grpc_channel* channel) {
+  return channel->tracer->RenderTrace();
+}
+
+intptr_t grpc_channel_get_uuid(grpc_channel* channel) {
+  return channel->tracer->GetUuid();
+}
+
 grpc_channel* grpc_channel_create(const char* target,
                                   const grpc_channel_args* input_args,
                                   grpc_channel_stack_type channel_stack_type,
@@ -377,6 +405,7 @@ static void destroy_channel(void* arg, grpc_error* error) {
     GRPC_MDELEM_UNREF(rc->authority);
     gpr_free(rc);
   }
+  channel->tracer.reset();
   GRPC_MDELEM_UNREF(channel->default_authority);
   gpr_mu_destroy(&channel->registered_call_mu);
   gpr_free(channel->target);
diff --git a/src/core/lib/surface/init.cc b/src/core/lib/surface/init.cc
index ac9f9e6066..bd436d6857 100644
--- a/src/core/lib/surface/init.cc
+++ b/src/core/lib/surface/init.cc
@@ -27,6 +27,7 @@
 #include <grpc/support/log.h>
 #include <grpc/support/time.h>
 #include "src/core/lib/channel/channel_stack.h"
+#include "src/core/lib/channel/channel_trace_registry.h"
 #include "src/core/lib/channel/connected_channel.h"
 #include "src/core/lib/channel/handshaker_registry.h"
 #include "src/core/lib/debug/stats.h"
@@ -128,6 +129,7 @@ void grpc_init(void) {
     grpc_slice_intern_init();
     grpc_mdctx_global_init();
     grpc_channel_init_init();
+    grpc_channel_trace_registry_init();
     grpc_security_pre_init();
     grpc_core::ExecCtx::GlobalInit();
     grpc_iomgr_init();
@@ -176,6 +178,7 @@ void grpc_shutdown(void) {
       grpc_mdctx_global_shutdown();
       grpc_handshaker_factory_registry_shutdown();
       grpc_slice_intern_shutdown();
+      grpc_channel_trace_registry_shutdown();
       grpc_stats_shutdown();
     }
     grpc_core::ExecCtx::GlobalShutdown();