Add Python support for server SSL cert reloading

Previously, a secure server is configured with SSL credentials during
initialization, and those credentials will be used for the lifetime of
the server. If the user wants the server to use new credentials, the
user has to restart the server, resulting in server downtime. This
change enables the user to optionally configure the server with a
"certificiate config fetcher," such that on every new client
connection, the server will call the config fetcher before performing
the handshake, allowing the user application to optionally specify new
certificate configuration for the server to use (the fetcher can
return a "no change" and the server continues to use its current
certificate configuration).

17 files changed, 973 insertions, 10 deletions

diff --git a/src/python/grpcio_tests/tests/unit/_api_test.py b/src/python/grpcio_tests/tests/unit/_api_test.py
index a3351aab50..d877e0f993 100644
--- a/src/python/grpcio_tests/tests/unit/_api_test.py
+++ b/src/python/grpcio_tests/tests/unit/_api_test.py
@@ -30,19 +30,22 @@ class AllTest(unittest.TestCase):
             'ChannelConnectivity', 'StatusCode', 'RpcError', 'RpcContext',
             'Call', 'ChannelCredentials', 'CallCredentials',
             'AuthMetadataContext', 'AuthMetadataPluginCallback',
-            'AuthMetadataPlugin', 'ServerCredentials',
-            'UnaryUnaryMultiCallable', 'UnaryStreamMultiCallable',
-            'StreamUnaryMultiCallable', 'StreamStreamMultiCallable', 'Channel',
-            'ServicerContext', 'RpcMethodHandler', 'HandlerCallDetails',
-            'GenericRpcHandler', 'ServiceRpcHandler', 'Server',
-            'unary_unary_rpc_method_handler', 'unary_stream_rpc_method_handler',
+            'AuthMetadataPlugin', 'ServerCertificateConfig',
+            'ServerCredentials', 'UnaryUnaryMultiCallable',
+            'UnaryStreamMultiCallable', 'StreamUnaryMultiCallable',
+            'StreamStreamMultiCallable', 'Channel', 'ServicerContext',
+            'RpcMethodHandler', 'HandlerCallDetails', 'GenericRpcHandler',
+            'ServiceRpcHandler', 'Server', 'unary_unary_rpc_method_handler',
+            'unary_stream_rpc_method_handler',
             'stream_unary_rpc_method_handler',
             'stream_stream_rpc_method_handler',
             'method_handlers_generic_handler', 'ssl_channel_credentials',
             'metadata_call_credentials', 'access_token_call_credentials',
             'composite_call_credentials', 'composite_channel_credentials',
-            'ssl_server_credentials', 'channel_ready_future',
-            'insecure_channel', 'secure_channel', 'server',)
+            'ssl_server_credentials', 'ssl_server_certificate_config',
+            'ssl_server_credentials_dynamic_cert_config',
+            'channel_ready_future', 'insecure_channel', 'secure_channel',
+            'server',)
 
         six.assertCountEqual(self, expected_grpc_code_elements,
                              _from_grpc_import_star.GRPC_ELEMENTS)
diff --git a/src/python/grpcio_tests/tests/unit/_server_ssl_cert_config_test.py b/src/python/grpcio_tests/tests/unit/_server_ssl_cert_config_test.py
new file mode 100644
index 0000000000..d2f9f114a5
--- /dev/null
+++ b/src/python/grpcio_tests/tests/unit/_server_ssl_cert_config_test.py
@@ -0,0 +1,520 @@
+# Copyright 2017 gRPC authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+This tests server certificate rotation support.
+
+Here we test various aspects of gRPC Python, and in some cases C-core
+by extension, support for server certificate rotation.
+
+* ServerSSLCertReloadTestWithClientAuth: test ability to rotate
+  server's SSL cert for use in future channels with clients while not
+  affecting any existing channel. The server requires client
+  authentication.
+
+* ServerSSLCertReloadTestWithoutClientAuth: like
+  ServerSSLCertReloadTestWithClientAuth except that the server does
+  not authenticate the client.
+
+* ServerSSLCertReloadTestCertConfigReuse: tests gRPC Python's ability
+  to deal with user's reuse of ServerCertificateConfig instances.
+"""
+
+import abc
+import collections
+import os
+import six
+import threading
+import unittest
+
+from concurrent import futures
+
+import grpc
+from tests.unit import resources
+from tests.testing import _application_common
+from tests.testing import _server_application
+from tests.testing.proto import services_pb2_grpc
+
+CA_1_PEM = resources.cert_hier_1_root_ca_cert()
+CA_2_PEM = resources.cert_hier_2_root_ca_cert()
+
+CLIENT_KEY_1_PEM = resources.cert_hier_1_client_1_key()
+CLIENT_CERT_CHAIN_1_PEM = (resources.cert_hier_1_client_1_cert() +
+                           resources.cert_hier_1_intermediate_ca_cert())
+
+CLIENT_KEY_2_PEM = resources.cert_hier_2_client_1_key()
+CLIENT_CERT_CHAIN_2_PEM = (resources.cert_hier_2_client_1_cert() +
+                           resources.cert_hier_2_intermediate_ca_cert())
+
+SERVER_KEY_1_PEM = resources.cert_hier_1_server_1_key()
+SERVER_CERT_CHAIN_1_PEM = (resources.cert_hier_1_server_1_cert() +
+                           resources.cert_hier_1_intermediate_ca_cert())
+
+SERVER_KEY_2_PEM = resources.cert_hier_2_server_1_key()
+SERVER_CERT_CHAIN_2_PEM = (resources.cert_hier_2_server_1_cert() +
+                           resources.cert_hier_2_intermediate_ca_cert())
+
+# for use with the CertConfigFetcher. Roughly a simple custom mock
+# implementation
+Call = collections.namedtuple('Call', ['did_raise', 'returned_cert_config'])
+
+
+def _create_client_stub(
+        port,
+        expect_success,
+        root_certificates=None,
+        private_key=None,
+        certificate_chain=None,):
+    channel = grpc.secure_channel('localhost:{}'.format(port),
+                                  grpc.ssl_channel_credentials(
+                                      root_certificates=root_certificates,
+                                      private_key=private_key,
+                                      certificate_chain=certificate_chain))
+    if expect_success:
+        # per Nathaniel: there's some robustness issue if we start
+        # using a channel without waiting for it to be actually ready
+        grpc.channel_ready_future(channel).result(timeout=10)
+    return services_pb2_grpc.FirstServiceStub(channel)
+
+
+class CertConfigFetcher(object):
+
+    def __init__(self):
+        self._lock = threading.Lock()
+        self._calls = []
+        self._should_raise = False
+        self._cert_config = None
+
+    def reset(self):
+        with self._lock:
+            self._calls = []
+            self._should_raise = False
+            self._cert_config = None
+
+    def configure(self, should_raise, cert_config):
+        assert not (should_raise and cert_config), (
+            "should not specify both should_raise and a cert_config at the same time"
+        )
+        with self._lock:
+            self._should_raise = should_raise
+            self._cert_config = cert_config
+
+    def getCalls(self):
+        with self._lock:
+            return self._calls
+
+    def __call__(self):
+        with self._lock:
+            if self._should_raise:
+                self._calls.append(Call(True, None))
+                raise ValueError('just for fun, should not affect the test')
+            else:
+                self._calls.append(Call(False, self._cert_config))
+                return self._cert_config
+
+
+class _ServerSSLCertReloadTest(
+        six.with_metaclass(abc.ABCMeta, unittest.TestCase)):
+
+    def __init__(self, *args, **kwargs):
+        super(_ServerSSLCertReloadTest, self).__init__(*args, **kwargs)
+        self.server = None
+        self.port = None
+
+    @abc.abstractmethod
+    def require_client_auth(self):
+        raise NotImplementedError()
+
+    def setUp(self):
+        self.server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
+        services_pb2_grpc.add_FirstServiceServicer_to_server(
+            _server_application.FirstServiceServicer(), self.server)
+        switch_cert_on_client_num = 10
+        initial_cert_config = grpc.ssl_server_certificate_config(
+            [(SERVER_KEY_1_PEM, SERVER_CERT_CHAIN_1_PEM)],
+            root_certificates=CA_2_PEM)
+        self.cert_config_fetcher = CertConfigFetcher()
+        server_credentials = grpc.ssl_server_credentials_dynamic_cert_config(
+            initial_cert_config,
+            self.cert_config_fetcher,
+            require_client_auth=self.require_client_auth())
+        self.port = self.server.add_secure_port('[::]:0', server_credentials)
+        self.server.start()
+
+    def tearDown(self):
+        if self.server:
+            self.server.stop(None)
+
+    def _perform_rpc(self, client_stub, expect_success):
+        # we don't care about the actual response of the rpc; only
+        # whether we can perform it or not, and if not, the status
+        # code must be UNAVAILABLE
+        request = _application_common.UNARY_UNARY_REQUEST
+        if expect_success:
+            response = client_stub.UnUn(request)
+            self.assertEqual(response, _application_common.UNARY_UNARY_RESPONSE)
+        else:
+            with self.assertRaises(grpc.RpcError) as exception_context:
+                client_stub.UnUn(request)
+            self.assertEqual(exception_context.exception.code(),
+                             grpc.StatusCode.UNAVAILABLE)
+
+    def _do_one_shot_client_rpc(self,
+                                expect_success,
+                                root_certificates=None,
+                                private_key=None,
+                                certificate_chain=None):
+        client_stub = _create_client_stub(
+            self.port,
+            expect_success,
+            root_certificates=root_certificates,
+            private_key=private_key,
+            certificate_chain=certificate_chain)
+        self._perform_rpc(client_stub, expect_success)
+        del client_stub
+
+    def _test(self):
+        # things should work...
+        self.cert_config_fetcher.configure(False, None)
+        self._do_one_shot_client_rpc(
+            True,
+            root_certificates=CA_1_PEM,
+            private_key=CLIENT_KEY_2_PEM,
+            certificate_chain=CLIENT_CERT_CHAIN_2_PEM)
+        actual_calls = self.cert_config_fetcher.getCalls()
+        self.assertEqual(len(actual_calls), 1)
+        self.assertFalse(actual_calls[0].did_raise)
+        self.assertIsNone(actual_calls[0].returned_cert_config)
+
+        # client should reject server...
+        # fails because client trusts ca2 and so will reject server
+        self.cert_config_fetcher.reset()
+        self.cert_config_fetcher.configure(False, None)
+        self._do_one_shot_client_rpc(
+            False,
+            root_certificates=CA_2_PEM,
+            private_key=CLIENT_KEY_2_PEM,
+            certificate_chain=CLIENT_CERT_CHAIN_2_PEM)
+        actual_calls = self.cert_config_fetcher.getCalls()
+        self.assertGreaterEqual(len(actual_calls), 1)
+        self.assertFalse(actual_calls[0].did_raise)
+        for i, call in enumerate(actual_calls):
+            self.assertFalse(call.did_raise, 'i= {}'.format(i))
+            self.assertIsNone(call.returned_cert_config, 'i= {}'.format(i))
+
+        # should work again...
+        self.cert_config_fetcher.reset()
+        self.cert_config_fetcher.configure(True, None)
+        self._do_one_shot_client_rpc(
+            True,
+            root_certificates=CA_1_PEM,
+            private_key=CLIENT_KEY_2_PEM,
+            certificate_chain=CLIENT_CERT_CHAIN_2_PEM)
+        actual_calls = self.cert_config_fetcher.getCalls()
+        self.assertEqual(len(actual_calls), 1)
+        self.assertTrue(actual_calls[0].did_raise)
+        self.assertIsNone(actual_calls[0].returned_cert_config)
+
+        # if with_client_auth, then client should be rejected by
+        # server because client uses key/cert1, but server trusts ca2,
+        # so server will reject
+        self.cert_config_fetcher.reset()
+        self.cert_config_fetcher.configure(False, None)
+        self._do_one_shot_client_rpc(
+            not self.require_client_auth(),
+            root_certificates=CA_1_PEM,
+            private_key=CLIENT_KEY_1_PEM,
+            certificate_chain=CLIENT_CERT_CHAIN_1_PEM)
+        actual_calls = self.cert_config_fetcher.getCalls()
+        self.assertGreaterEqual(len(actual_calls), 1)
+        for i, call in enumerate(actual_calls):
+            self.assertFalse(call.did_raise, 'i= {}'.format(i))
+            self.assertIsNone(call.returned_cert_config, 'i= {}'.format(i))
+
+        # should work again...
+        self.cert_config_fetcher.reset()
+        self.cert_config_fetcher.configure(False, None)
+        self._do_one_shot_client_rpc(
+            True,
+            root_certificates=CA_1_PEM,
+            private_key=CLIENT_KEY_2_PEM,
+            certificate_chain=CLIENT_CERT_CHAIN_2_PEM)
+        actual_calls = self.cert_config_fetcher.getCalls()
+        self.assertEqual(len(actual_calls), 1)
+        self.assertFalse(actual_calls[0].did_raise)
+        self.assertIsNone(actual_calls[0].returned_cert_config)
+
+        # now create the "persistent" clients
+        self.cert_config_fetcher.reset()
+        self.cert_config_fetcher.configure(False, None)
+        persistent_client_stub_A = _create_client_stub(
+            self.port,
+            True,
+            root_certificates=CA_1_PEM,
+            private_key=CLIENT_KEY_2_PEM,
+            certificate_chain=CLIENT_CERT_CHAIN_2_PEM)
+        self._perform_rpc(persistent_client_stub_A, True)
+        actual_calls = self.cert_config_fetcher.getCalls()
+        self.assertEqual(len(actual_calls), 1)
+        self.assertFalse(actual_calls[0].did_raise)
+        self.assertIsNone(actual_calls[0].returned_cert_config)
+
+        self.cert_config_fetcher.reset()
+        self.cert_config_fetcher.configure(False, None)
+        persistent_client_stub_B = _create_client_stub(
+            self.port,
+            True,
+            root_certificates=CA_1_PEM,
+            private_key=CLIENT_KEY_2_PEM,
+            certificate_chain=CLIENT_CERT_CHAIN_2_PEM)
+        self._perform_rpc(persistent_client_stub_B, True)
+        actual_calls = self.cert_config_fetcher.getCalls()
+        self.assertEqual(len(actual_calls), 1)
+        self.assertFalse(actual_calls[0].did_raise)
+        self.assertIsNone(actual_calls[0].returned_cert_config)
+
+        # moment of truth!! client should reject server because the
+        # server switch cert...
+        cert_config = grpc.ssl_server_certificate_config(
+            [(SERVER_KEY_2_PEM, SERVER_CERT_CHAIN_2_PEM)],
+            root_certificates=CA_1_PEM)
+        self.cert_config_fetcher.reset()
+        self.cert_config_fetcher.configure(False, cert_config)
+        self._do_one_shot_client_rpc(
+            False,
+            root_certificates=CA_1_PEM,
+            private_key=CLIENT_KEY_2_PEM,
+            certificate_chain=CLIENT_CERT_CHAIN_2_PEM)
+        actual_calls = self.cert_config_fetcher.getCalls()
+        self.assertGreaterEqual(len(actual_calls), 1)
+        self.assertFalse(actual_calls[0].did_raise)
+        for i, call in enumerate(actual_calls):
+            self.assertFalse(call.did_raise, 'i= {}'.format(i))
+            self.assertEqual(call.returned_cert_config, cert_config,
+                             'i= {}'.format(i))
+
+        # now should work again...
+        self.cert_config_fetcher.reset()
+        self.cert_config_fetcher.configure(False, None)
+        self._do_one_shot_client_rpc(
+            True,
+            root_certificates=CA_2_PEM,
+            private_key=CLIENT_KEY_1_PEM,
+            certificate_chain=CLIENT_CERT_CHAIN_1_PEM)
+        actual_calls = self.cert_config_fetcher.getCalls()
+        self.assertEqual(len(actual_calls), 1)
+        self.assertFalse(actual_calls[0].did_raise)
+        self.assertIsNone(actual_calls[0].returned_cert_config)
+
+        # client should be rejected by server if with_client_auth
+        self.cert_config_fetcher.reset()
+        self.cert_config_fetcher.configure(False, None)
+        self._do_one_shot_client_rpc(
+            not self.require_client_auth(),
+            root_certificates=CA_2_PEM,
+            private_key=CLIENT_KEY_2_PEM,
+            certificate_chain=CLIENT_CERT_CHAIN_2_PEM)
+        actual_calls = self.cert_config_fetcher.getCalls()
+        self.assertGreaterEqual(len(actual_calls), 1)
+        for i, call in enumerate(actual_calls):
+            self.assertFalse(call.did_raise, 'i= {}'.format(i))
+            self.assertIsNone(call.returned_cert_config, 'i= {}'.format(i))
+
+        # here client should reject server...
+        self.cert_config_fetcher.reset()
+        self.cert_config_fetcher.configure(False, None)
+        self._do_one_shot_client_rpc(
+            False,
+            root_certificates=CA_1_PEM,
+            private_key=CLIENT_KEY_2_PEM,
+            certificate_chain=CLIENT_CERT_CHAIN_2_PEM)
+        actual_calls = self.cert_config_fetcher.getCalls()
+        self.assertGreaterEqual(len(actual_calls), 1)
+        for i, call in enumerate(actual_calls):
+            self.assertFalse(call.did_raise, 'i= {}'.format(i))
+            self.assertIsNone(call.returned_cert_config, 'i= {}'.format(i))
+
+        # persistent clients should continue to work
+        self.cert_config_fetcher.reset()
+        self.cert_config_fetcher.configure(False, None)
+        self._perform_rpc(persistent_client_stub_A, True)
+        actual_calls = self.cert_config_fetcher.getCalls()
+        self.assertEqual(len(actual_calls), 0)
+
+        self.cert_config_fetcher.reset()
+        self.cert_config_fetcher.configure(False, None)
+        self._perform_rpc(persistent_client_stub_B, True)
+        actual_calls = self.cert_config_fetcher.getCalls()
+        self.assertEqual(len(actual_calls), 0)
+
+
+class ServerSSLCertConfigFetcherParamsChecks(unittest.TestCase):
+
+    def test_check_on_initial_config(self):
+        with self.assertRaises(TypeError):
+            grpc.ssl_server_credentials_dynamic_cert_config(None, str)
+        with self.assertRaises(TypeError):
+            grpc.ssl_server_credentials_dynamic_cert_config(1, str)
+
+    def test_check_on_config_fetcher(self):
+        cert_config = grpc.ssl_server_certificate_config(
+            [(SERVER_KEY_2_PEM, SERVER_CERT_CHAIN_2_PEM)],
+            root_certificates=CA_1_PEM)
+        with self.assertRaises(TypeError):
+            grpc.ssl_server_credentials_dynamic_cert_config(cert_config, None)
+        with self.assertRaises(TypeError):
+            grpc.ssl_server_credentials_dynamic_cert_config(cert_config, 1)
+
+
+class ServerSSLCertReloadTestWithClientAuth(_ServerSSLCertReloadTest):
+
+    def require_client_auth(self):
+        return True
+
+    test = _ServerSSLCertReloadTest._test
+
+
+class ServerSSLCertReloadTestWithoutClientAuth(_ServerSSLCertReloadTest):
+
+    def require_client_auth(self):
+        return False
+
+    test = _ServerSSLCertReloadTest._test
+
+
+class ServerSSLCertReloadTestCertConfigReuse(_ServerSSLCertReloadTest):
+    """Ensures that `ServerCertificateConfig` instances can be reused.
+
+    Because C-core takes ownership of the
+    `grpc_ssl_server_certificate_config` encapsulated by
+    `ServerCertificateConfig`, this test reuses the same
+    `ServerCertificateConfig` instances multiple times to make sure
+    gRPC Python takes care of maintaining the validity of
+    `ServerCertificateConfig` instances, so that such instances can be
+    re-used by user application.
+    """
+
+    def require_client_auth(self):
+        return True
+
+    def setUp(self):
+        self.server = grpc.server(futures.ThreadPoolExecutor(max_workers=10))
+        services_pb2_grpc.add_FirstServiceServicer_to_server(
+            _server_application.FirstServiceServicer(), self.server)
+        self.cert_config_A = grpc.ssl_server_certificate_config(
+            [(SERVER_KEY_1_PEM, SERVER_CERT_CHAIN_1_PEM)],
+            root_certificates=CA_2_PEM)
+        self.cert_config_B = grpc.ssl_server_certificate_config(
+            [(SERVER_KEY_2_PEM, SERVER_CERT_CHAIN_2_PEM)],
+            root_certificates=CA_1_PEM)
+        self.cert_config_fetcher = CertConfigFetcher()
+        server_credentials = grpc.ssl_server_credentials_dynamic_cert_config(
+            self.cert_config_A,
+            self.cert_config_fetcher,
+            require_client_auth=True)
+        self.port = self.server.add_secure_port('[::]:0', server_credentials)
+        self.server.start()
+
+    def test_cert_config_reuse(self):
+
+        # succeed with A
+        self.cert_config_fetcher.reset()
+        self.cert_config_fetcher.configure(False, self.cert_config_A)
+        self._do_one_shot_client_rpc(
+            True,
+            root_certificates=CA_1_PEM,
+            private_key=CLIENT_KEY_2_PEM,
+            certificate_chain=CLIENT_CERT_CHAIN_2_PEM)
+        actual_calls = self.cert_config_fetcher.getCalls()
+        self.assertEqual(len(actual_calls), 1)
+        self.assertFalse(actual_calls[0].did_raise)
+        self.assertEqual(actual_calls[0].returned_cert_config,
+                         self.cert_config_A)
+
+        # fail with A
+        self.cert_config_fetcher.reset()
+        self.cert_config_fetcher.configure(False, self.cert_config_A)
+        self._do_one_shot_client_rpc(
+            False,
+            root_certificates=CA_2_PEM,
+            private_key=CLIENT_KEY_1_PEM,
+            certificate_chain=CLIENT_CERT_CHAIN_1_PEM)
+        actual_calls = self.cert_config_fetcher.getCalls()
+        self.assertGreaterEqual(len(actual_calls), 1)
+        self.assertFalse(actual_calls[0].did_raise)
+        for i, call in enumerate(actual_calls):
+            self.assertFalse(call.did_raise, 'i= {}'.format(i))
+            self.assertEqual(call.returned_cert_config, self.cert_config_A,
+                             'i= {}'.format(i))
+
+        # succeed again with A
+        self.cert_config_fetcher.reset()
+        self.cert_config_fetcher.configure(False, self.cert_config_A)
+        self._do_one_shot_client_rpc(
+            True,
+            root_certificates=CA_1_PEM,
+            private_key=CLIENT_KEY_2_PEM,
+            certificate_chain=CLIENT_CERT_CHAIN_2_PEM)
+        actual_calls = self.cert_config_fetcher.getCalls()
+        self.assertEqual(len(actual_calls), 1)
+        self.assertFalse(actual_calls[0].did_raise)
+        self.assertEqual(actual_calls[0].returned_cert_config,
+                         self.cert_config_A)
+
+        # succeed with B
+        self.cert_config_fetcher.reset()
+        self.cert_config_fetcher.configure(False, self.cert_config_B)
+        self._do_one_shot_client_rpc(
+            True,
+            root_certificates=CA_2_PEM,
+            private_key=CLIENT_KEY_1_PEM,
+            certificate_chain=CLIENT_CERT_CHAIN_1_PEM)
+        actual_calls = self.cert_config_fetcher.getCalls()
+        self.assertEqual(len(actual_calls), 1)
+        self.assertFalse(actual_calls[0].did_raise)
+        self.assertEqual(actual_calls[0].returned_cert_config,
+                         self.cert_config_B)
+
+        # fail with B
+        self.cert_config_fetcher.reset()
+        self.cert_config_fetcher.configure(False, self.cert_config_B)
+        self._do_one_shot_client_rpc(
+            False,
+            root_certificates=CA_1_PEM,
+            private_key=CLIENT_KEY_2_PEM,
+            certificate_chain=CLIENT_CERT_CHAIN_2_PEM)
+        actual_calls = self.cert_config_fetcher.getCalls()
+        self.assertGreaterEqual(len(actual_calls), 1)
+        self.assertFalse(actual_calls[0].did_raise)
+        for i, call in enumerate(actual_calls):
+            self.assertFalse(call.did_raise, 'i= {}'.format(i))
+            self.assertEqual(call.returned_cert_config, self.cert_config_B,
+                             'i= {}'.format(i))
+
+        # succeed again with B
+        self.cert_config_fetcher.reset()
+        self.cert_config_fetcher.configure(False, self.cert_config_B)
+        self._do_one_shot_client_rpc(
+            True,
+            root_certificates=CA_2_PEM,
+            private_key=CLIENT_KEY_1_PEM,
+            certificate_chain=CLIENT_CERT_CHAIN_1_PEM)
+        actual_calls = self.cert_config_fetcher.getCalls()
+        self.assertEqual(len(actual_calls), 1)
+        self.assertFalse(actual_calls[0].did_raise)
+        self.assertEqual(actual_calls[0].returned_cert_config,
+                         self.cert_config_B)
+
+
+if __name__ == '__main__':
+    unittest.main(verbosity=2)
diff --git a/src/python/grpcio_tests/tests/unit/credentials/README b/src/python/grpcio_tests/tests/unit/credentials/README
deleted file mode 100644
index cb20dcb49f..0000000000
--- a/src/python/grpcio_tests/tests/unit/credentials/README
+++ /dev/null
@@ -1 +0,0 @@
-These are test keys *NOT* to be used in production.
diff --git a/src/python/grpcio_tests/tests/unit/credentials/README.md b/src/python/grpcio_tests/tests/unit/credentials/README.md
new file mode 100644
index 0000000000..100b43c1aa
--- /dev/null
+++ b/src/python/grpcio_tests/tests/unit/credentials/README.md
@@ -0,0 +1,15 @@
+These are test keys *NOT* to be used in production.
+
+The `certificate_hierarchy_1` and `certificate_hierarchy_2` contain
+two disjoint but similarly organized certificate hierarchies. Each
+contains:
+
+* The respective root CA cert in `certs/ca.cert.pem`
+
+* The intermediate CA cert in
+  `intermediate/certs/intermediate.cert.pem`, signed by the root CA
+
+* A client cert and a server cert--both signed by the intermediate
+  CA--in `intermediate/certs/client.cert.pem` and
+  `intermediate/certs/localhost-1.cert.pem`; the corresponding keys
+  are in `intermediate/private`
diff --git a/src/python/grpcio_tests/tests/unit/credentials/certificate_hierarchy_1/certs/ca.cert.pem b/src/python/grpcio_tests/tests/unit/credentials/certificate_hierarchy_1/certs/ca.cert.pem
new file mode 100644
index 0000000000..604b86fdff
--- /dev/null
+++ b/src/python/grpcio_tests/tests/unit/credentials/certificate_hierarchy_1/certs/ca.cert.pem
@@ -0,0 +1,31 @@
+-----BEGIN CERTIFICATE-----
+MIIFZDCCA0ygAwIBAgIJAKfkDFZ6+Ly/MA0GCSqGSIb3DQEBCwUAMD8xCzAJBgNV
+BAYTAnVzMQ4wDAYDVQQIDAVkdW1teTEOMAwGA1UECgwFZHVtbXkxEDAOBgNVBAMM
+B3Jvb3QgY2EwHhcNMTcxMTAyMDAzNzA1WhcNMzcxMDI4MDAzNzA1WjA/MQswCQYD
+VQQGEwJ1czEOMAwGA1UECAwFZHVtbXkxDjAMBgNVBAoMBWR1bW15MRAwDgYDVQQD
+DAdyb290IGNhMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEAxlSUuSbi
+o66tT2ZCqu9wNqSX8VhAJkmrAT5y6m2V0VlQ8Gz7ddynW5UVSmtvDNTebZ15FrvO
+6Ng7QnwXXNs/dEzl6oMe6AKDZpuWScVkiqH1UYWBkMLRygWCTEYpSTWTpZWk1zxj
+DJ2LlIoO1X/ufLyLOfy2a2XEz8ICzJePmqVca6fmfEtCTj1/8FcwCBF6YlUWVzlR
+wewjanQo/lorTYbub+Q6LGxPXZ8W0qoKZzLDSD9cnj4pcJzGGFeu9KkNaW4rldZG
+t7mTGQqIRc98dDRc9Jb7PqL8tMPLidw1KErUi05ofxggc5vqNnj4xBl6aX6b/EYN
+rBLzO2e0FazX6TwNKwwg68vbOanpDq5LVmIUH8bY1zNZ+JPBGO9pXlAA0YwLx86r
+R7YhQ431ZpJ2KGnYjVhYnZ2L3NjV3UYX3x5Z3OrDj9hybhucJB48DMQ1+loEabwK
+fSUJtcSPc8dCIibxVKidBFgaTPXtHy2MPXuhMhR7PCtMpE7RPUoYmdZLr9FNN1ty
+/RAbwBfuhGLbRI2qqJgbOzHJHaOY/FtShfooLz7lt4LIjPTARaNsulG2rbv+m3z9
+mhNjL+peV8gni/xyOYYTbdzZagLrtSHeTWsITvmVt0flMHkjHyv35rw23+hBlSjp
+6+S+0MmwuwxqBBccBSlZ9t3Xh1N+vFkb2UkCAwEAAaNjMGEwHQYDVR0OBBYEFJWE
+tQwTbTCgZWNN08VSxjdNA0oaMB8GA1UdIwQYMBaAFJWEtQwTbTCgZWNN08VSxjdN
+A0oaMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3DQEB
+CwUAA4ICAQCNjPv/e3ozX1PuN5Tluf0yOmKCxKVCK3Pp98WkDzH4Rp1urEeYrGJL
+vBNcl17avOJ0e+zTVYPXFviFbsBsU/zaf+TqEujXabsdL+nvvCJ2mMqYn4wyDFjS
+zDNbGH6O0ENZz5NSY0/UGSOHYrYnYB94QRFLbbf0Y3PmBS2eRNjIUnv7ytPZNMi/
+piM+QhPb0Ebyk0rHQZ0RAJaC/wsEtqP8TGV/fx+AzG7zW/zxgPTrgIThy138tLQ+
+xCVDP9H2c17nVP6vjYzKnMZ94uGrGqUzV9vU7EqYl0uZflIf98pLfdKHnQ3heqds
+8KQPNKRxVvcc92qv2pQY951wb1fkhLutjHn7TUvrenyAngz+Vs19NxbqLPys1CTw
+iaL7vZ8VE/aEDm1tjt5SLM474tpATjk1+qMRaWnii8J5rTodYHP+Zu2GxyIrMiGq
+tfNZMYI0tETK1XmEo75E/3s9pmIeQNGKLFp+qL7xrVyN/2ffNv0at8kkqXluunK9
+/Ki0gKYlGFm4Eu8t/nHMqhBx/njYg6pLDuarLW6ftUV7aHd7qKcCWOWqK6gnH/vX
+3Apv31eltZBBVN69p3CFy2oMnjrom2Yn/DUXFwrJLBiNJ1dd1JyDxpqpJ74ZQy+/
+pSRWMTRM5SuC7lYARx5rYPmp6cZJWyWRH/3r7bwS699/W965pa5nug==
+-----END CERTIFICATE-----
diff --git a/src/python/grpcio_tests/tests/unit/credentials/certificate_hierarchy_1/intermediate/certs/client.cert.pem b/src/python/grpcio_tests/tests/unit/credentials/certificate_hierarchy_1/intermediate/certs/client.cert.pem
new file mode 100644
index 0000000000..44bc562599
--- /dev/null
+++ b/src/python/grpcio_tests/tests/unit/credentials/certificate_hierarchy_1/intermediate/certs/client.cert.pem
@@ -0,0 +1,28 @@
+-----BEGIN CERTIFICATE-----
+MIIExzCCAq+gAwIBAgICEAMwDQYJKoZIhvcNAQELBQAwRzELMAkGA1UEBhMCdXMx
+DjAMBgNVBAgMBWR1bW15MQ4wDAYDVQQKDAVkdW1teTEYMBYGA1UEAwwPaW50ZXJt
+ZWRpYXRlIGNhMB4XDTE3MTEwMjAwMzcwNloXDTI3MTAzMTAwMzcwNlowPjELMAkG
+A1UEBhMCdXMxDjAMBgNVBAgMBWR1bW15MQ4wDAYDVQQKDAVkdW1teTEPMA0GA1UE
+AwwGY2xpZW50MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAwquL6gtP
+R7P9xJK76FTj8fI5TSJa3cAMt1p6CmessjHQq7nQ6DWLGVi4XIt9Sc/1C3rXupOe
+90Ok4L0tsuVZH78Wn0EBmBH7S4IbhU9P+aJ9mcigepj1lnxWqoVblgeJYKMOOwAf
+pAKUNMWDSm+nCfwE+R5d8d8cfA41Awq1jTRjOVpiJq6aoKfs791a1ZkZde3kFrNV
+AVjC06GgA1lZd3sHf94hmLeC+xJztRXVE9e+7dcc7nFDH0t5DIKYBAklsHg77mZa
+3IK4aOZew7Lm6diPoMnAzXh2rWpJU6RrEE29gIkJBsF8CL1Ndg9MzssCg6KBjoai
+Vt5dJ+4TSEGCOwIDAQABo4HFMIHCMAkGA1UdEwQCMAAwEQYJYIZIAYb4QgEBBAQD
+AgWgMDMGCWCGSAGG+EIBDQQmFiRPcGVuU1NMIEdlbmVyYXRlZCBDbGllbnQgQ2Vy
+dGlmaWNhdGUwHQYDVR0OBBYEFPeuKDCswk8jaH9tl6X+uXjo+WM1MB8GA1UdIwQY
+MBaAFCoqYgmKh3CUafVp+paXxfz+He+FMA4GA1UdDwEB/wQEAwIF4DAdBgNVHSUE
+FjAUBggrBgEFBQcDAgYIKwYBBQUHAwQwDQYJKoZIhvcNAQELBQADggIBADYAp8XS
+UjMEpX/zVjRWpAAT4HNEJylCV1QNyhBIAyx38A6xJYuFIx966Htd6W9/Rw4sUY6y
+F4pOmnLCRxIPqFzzMYcBHSpynlcu5G7zqIod3wYIk7BNlB0AzkZn6yD8bM1y5Plf
+myzQVDEGggrDtaW2EehhNIB+wOmbRGITjIcZUEr8V4BlLXkCqOuxauxl82d5/k2w
+LAhjOb9d1VW6RT8+Lcn6drhHZdvtSCe8Z27BcXhaQLL8366mhbigKYJt5adD0KOx
+pl0MQcoL1Rth5cJEj+1/lgUaxcnvh7TaIIGEx0h3olQXsTxSTypU/nww2Ic41xdG
+xl3xvHsxe20IvOOAMRfS/LPW7MCtQ3k0BqB/rAQvmB0r5YITLlMJuBqg+zjYrG/j
+s5szSGAz9r0leFuPraeuZA41d9UBTAJMoVrrQZ4xVHMXQi1oz9E9KlIdbO9+spvC
+ulfO+D+Z4a9trYSWhnQL2dSHT0+kHqJ/8GipiUNP/yAC76dRpDVR3xtYNr73iw0j
+hyDsVjihTD8JBebs3axnt+Bc+FwoCCd6CVcsggfGUNhu/N5LS78b13PcaRzrUNjU
+Eh+8cJvMLst+UQzePlyazzpn7jjN3KsBzWUkbnXCtUs2qRMn8f2gZqliDo7JSFvy
+WtBSCYpikOivuJSQUlrHQ8NaXeddyWQzLY79
+-----END CERTIFICATE-----
diff --git a/src/python/grpcio_tests/tests/unit/credentials/certificate_hierarchy_1/intermediate/certs/intermediate.cert.pem b/src/python/grpcio_tests/tests/unit/credentials/certificate_hierarchy_1/intermediate/certs/intermediate.cert.pem
new file mode 100644
index 0000000000..98e13669c0
--- /dev/null
+++ b/src/python/grpcio_tests/tests/unit/credentials/certificate_hierarchy_1/intermediate/certs/intermediate.cert.pem
@@ -0,0 +1,31 @@
+-----BEGIN CERTIFICATE-----
+MIIFaDCCA1CgAwIBAgICEAAwDQYJKoZIhvcNAQELBQAwPzELMAkGA1UEBhMCdXMx
+DjAMBgNVBAgMBWR1bW15MQ4wDAYDVQQKDAVkdW1teTEQMA4GA1UEAwwHcm9vdCBj
+YTAeFw0xNzExMDIwMDM3MDZaFw0yNzEwMzEwMDM3MDZaMEcxCzAJBgNVBAYTAnVz
+MQ4wDAYDVQQIDAVkdW1teTEOMAwGA1UECgwFZHVtbXkxGDAWBgNVBAMMD2ludGVy
+bWVkaWF0ZSBjYTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAOOxzve7
+CG2P9SvKfXkJVTXkj4y79JSZ77Kud/TiPfDbHTqZWKuLTXkOCkCCxfpuJvWXnnj5
+1AeLCdKx9hEwJQeU23EXDt1K+RsRyl09SXtPNnJnqHD1mUHRQR28vGX5ctrQzK8J
+Sa6/mHW4bX8ol100npbgVMDnM4IDfLYcsv4BXMICGkSHOW6Gn0zJaeHzRVPpmnK/
+0k/GQAcIrU2sZ39kVlVQkWq3HJC28cNL/P04hjh4gAf0evo/k9VrEtxPWYMfiPDt
+kOAKueoPv/VTA/zL5t8lyzfhrhxvsJxFg/klapPXK0gLLbhsHyOhnkbrzvmSR4Rw
+xubYJ2dDK0DKx+BIZqlFznjP9BvOtvtuVVMyqg9cfgc7J/OjvAguO0f93MLSfIWP
+uISqv7Llt/Blvy/xI5owvOKVc/hm3d+5pqjWBC1HkVwo4qugpWmM49dFWl4kc4c7
+ayYUjTmcgoj1ZR89w4Off/bPd1A6gXqSkw2VQfgFF+uOos84fP1V+zPWhp3UDY3P
+bFeJtuTdv1gR5w1jCIq6xVJ+UsyDZBaYP7yBBRiNzS1/yXJpnXrvHmDfUeQHLBPR
+N0nbMjqXJ1dVpZwydiI0Qx9DnJtOaq/spUreXr8+PU2jeQdCCAN21MB1umr2gZBJ
+8MZBStTgE7SDByfGmGfp7B5/s/r4O/rNc4WzAgMBAAGjZjBkMB0GA1UdDgQWBBQq
+KmIJiodwlGn1afqWl8X8/h3vhTAfBgNVHSMEGDAWgBSVhLUME20woGVjTdPFUsY3
+TQNKGjASBgNVHRMBAf8ECDAGAQH/AgEAMA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG
+9w0BAQsFAAOCAgEAvzLu/Jc8DlfCltVufC54UZ8DVwUfxdGapNBGv4icrs1wMV3S
+xqdaLO+vSp9NeEufi724+/hj4URapW9kSt2TMD7BNJ61QSATZFJajxTFgGa0Zz95
+RBDw8/b5Arz/2pOF4VX+FJ+wqHvoH/2A0T+fwz8hLORhxZHv/cUN6kif4FKCwryQ
+s89e694kXkEiJfquvu7DR9hYCLOJwzMOOJiTnjz3hlQg4WGu7Z8ZvqzCM+how1hr
+nYbUx6a+HfoUf79AHJB0N1EsEEetJ+omvTdrrayCvy1bHA3QgHlJ28QZIJ7MzX9E
+n11/xQ95iTuSp8iWurzjTjbrm7eHnGUh+5QubYLXOzbqKzNZu72w0uvWv6ptIudU
+usttltiwW8H9kP0ArWTcZDPhhPfS9impFlhiPDk1wUv2/7g+Zz1OaOb7IiSH0s8y
+FG72AB8ucJ5dNa/2q5dJiM8Gm5CbiVw5RXTBjlfTTkNeM6LBI3dRghpPdU7Kbfhn
+xYs9vnRZeRMJHrcodLuwVcpY/gyeJ0k5LD6eIPCJmatkYZ122isYyMX8lL2P5aR+
+7d2hhqcOCproOrtThjp6nW2jWTB+R/O2+s6mhKSPgfbY2cdky1Y9FSJxSNayb9B8
+eQ+A29iOHrGVAA0R/rvw119rLAYxjXzToM28owx7IyXKrBaU4RMv5yokgag=
+-----END CERTIFICATE-----
diff --git a/src/python/grpcio_tests/tests/unit/credentials/certificate_hierarchy_1/intermediate/certs/localhost-1.cert.pem b/src/python/grpcio_tests/tests/unit/credentials/certificate_hierarchy_1/intermediate/certs/localhost-1.cert.pem
new file mode 100644
index 0000000000..f15f1cf5c2
--- /dev/null
+++ b/src/python/grpcio_tests/tests/unit/credentials/certificate_hierarchy_1/intermediate/certs/localhost-1.cert.pem
@@ -0,0 +1,30 @@
+-----BEGIN CERTIFICATE-----
+MIIFFzCCAv+gAwIBAgICEAAwDQYJKoZIhvcNAQELBQAwRzELMAkGA1UEBhMCdXMx
+DjAMBgNVBAgMBWR1bW15MQ4wDAYDVQQKDAVkdW1teTEYMBYGA1UEAwwPaW50ZXJt
+ZWRpYXRlIGNhMB4XDTE3MTEwMjAwMzcwNloXDTI3MTAzMTAwMzcwNlowTTELMAkG
+A1UEBhMCdXMxDjAMBgNVBAgMBWR1bW15MQ4wDAYDVQQKDAVkdW1teTEKMAgGA1UE
+CwwBMTESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A
+MIIBCgKCAQEArRAy0Nim9P883BAisXdFoKmgHGTtcLH/SzwkkPWTFHz0rHU1Klwz
+w8u3OkRyvgoQp7DqkohboNMDwg5VrOOcfKwtM2GZ5jixo+YKvJ25oj8Jfr+40baz
+nyWTmOcfoviKrb7u2T9BPEEz5og+lXRDAsTFATGaQDX2LN3Dd9KIw+7sWY+gc3Zi
+13HHaWYhtmfJjzFbH1vDxHKCdSdgtPyEhqcJ4OC6wbgp/mQ01VlPAr08kRfkC8mT
+TS7atqc410irKViF3sWi4YNPf7LuBrjo75FIIOp+sQgZE6xwOuZ/9bT2Zx/IUtCC
+TqzVgZI0s5NVlINtWR6eyyxQ1uDKTs4xrQIDAQABo4IBBTCCAQEwCQYDVR0TBAIw
+ADARBglghkgBhvhCAQEEBAMCBkAwMwYJYIZIAYb4QgENBCYWJE9wZW5TU0wgR2Vu
+ZXJhdGVkIFNlcnZlciBDZXJ0aWZpY2F0ZTAdBgNVHQ4EFgQUDE8pwi7aELJjvyNT
+ed81/KIgGfowaAYDVR0jBGEwX4AUKipiCYqHcJRp9Wn6lpfF/P4d74WhQ6RBMD8x
+CzAJBgNVBAYTAnVzMQ4wDAYDVQQIDAVkdW1teTEOMAwGA1UECgwFZHVtbXkxEDAO
+BgNVBAMMB3Jvb3QgY2GCAhAAMA4GA1UdDwEB/wQEAwIFoDATBgNVHSUEDDAKBggr
+BgEFBQcDATANBgkqhkiG9w0BAQsFAAOCAgEA2cvXxJw120Z9oWXyGwR6CH7TcXoy
+1i77B1M5j0Krvkjh2/MkEU+JxpZcrhAgZODK9wMPeIUIpJNw2t6Hg+vigpInu7pY
+MXR4IA5XLnhGV/hueXa0JLia5FG1TISxr4piW6Jd9P2pOt3ECm3Url/F0OeFF/74
+jGaAlWkbhqWJ9M7Gd4QP2wUNm0P4CwAqS9DC6dnMz+JXTakEUirOpmq7U8UKT+5N
+QS1K4WuH671n4MiYye3+UoRYt4zPjOzN+QxzvAMtkUBspPmWD6txmD5tKUYDECqn
+0sSbY6ytD30OTHIbICFp40arOffmEEJSriL+uQNPPmvqMxX1G2kUFGm15NLPs8Xa
+J7ChrAaJzssN5J3myZUbDfCuxmTkWg+hGvGmxLraVNWc3fzKFmdszSkXrGIdf2HR
+gZeFI3w6M4Ktx3KctXlsjwqQTYZI/WwLOEpsrHQBPBLQhISyNw4xjZ4MxK8SFZuQ
+IiGps/do0eEgeQ+o3gD1dIXt8YxFIxrgk0pzJONpXGgv/cZrukbLNTBdkTSkIwtx
+TXKdiJbO17H24MvW+UxFdsIoJXmfQZWdQC3p+Dl0iP+K80aI6WbaysmToHuOi216
+e49nmiM72Izul2zmBi7Cq2nRQbHAETsFfqC34FzJlx0aP8WS953IBD0jNi1BB+AX
+BxwiZ1rPjeMvekI=
+-----END CERTIFICATE-----
diff --git a/src/python/grpcio_tests/tests/unit/credentials/certificate_hierarchy_1/intermediate/private/client.key.pem b/src/python/grpcio_tests/tests/unit/credentials/certificate_hierarchy_1/intermediate/private/client.key.pem
new file mode 100644
index 0000000000..d8a21632ab
--- /dev/null
+++ b/src/python/grpcio_tests/tests/unit/credentials/certificate_hierarchy_1/intermediate/private/client.key.pem
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEAwquL6gtPR7P9xJK76FTj8fI5TSJa3cAMt1p6CmessjHQq7nQ
+6DWLGVi4XIt9Sc/1C3rXupOe90Ok4L0tsuVZH78Wn0EBmBH7S4IbhU9P+aJ9mcig
+epj1lnxWqoVblgeJYKMOOwAfpAKUNMWDSm+nCfwE+R5d8d8cfA41Awq1jTRjOVpi
+Jq6aoKfs791a1ZkZde3kFrNVAVjC06GgA1lZd3sHf94hmLeC+xJztRXVE9e+7dcc
+7nFDH0t5DIKYBAklsHg77mZa3IK4aOZew7Lm6diPoMnAzXh2rWpJU6RrEE29gIkJ
+BsF8CL1Ndg9MzssCg6KBjoaiVt5dJ+4TSEGCOwIDAQABAoIBABECKAFU56JeKYfp
+Qh20fQ4Amd0RaVsCkpnaf9s037PaAl9eptADDZozVDhRv6qZTtGn8/1LNJJqCJfS
+L5H30+egLHvRlDATMh+QyJLHMTegaNTs4IiVoK97QZ84c54SHoCg/ndNNXaA+y35
+K9VvF+sZZ93UN2UQl06Hdz5Cy0YA7L5HIIH3Ezk0ArAw4AarLil5mv4yEz2ApZhm
+Tw4I4yNfxB7tZeP+ekNg0XXRL1quA0tGblp+A5fAFfVMDplqqB2d3/KxPR9FSEOi
+4PzBZ5Mq2wQBPIaNog5um9qkw6VKxjl5sQGhP1GGTA8iZqR9iM2+xh57xdCZm3g3
+jcr+aPECgYEA42mXTsF/4oBQtU6hh/sOCMWHhxAPstKpQHFMKGYLHKEJ/V1qq0Sd
+d0kswAYCmH5G9ookzu5p7pNf0hUUHO5EwelpSZ3FEmtIM+oBwSnDk3vGuadYXN5X
+fPuVUla65B1F9SSwapYNBUAiRgrY69Knca2rkTSdcZQaBuWmo684UQcCgYEA2yRE
+P23I/9N6AVhKB/zTRtil1AxnTW8o+j7AE4q1o+xly7DS7DT34INaLKLiuG6ylV1F
+UoTiqmWqH3A7m3o3Id2AnVf/oDoKV78LCXRF3dJJWvzrPdob2fLlwyjgqXYvmD3O
+UH/OFY2blYcAHOYib1Y1AAhHPlXiHA52BYZtnC0CgYAVjjitWmII0ijURrPA8+cM
+pcyG3NrgFF++n/6cBbAf8pPD1Er8GPDkEaeQPAGa+r03OTjr9GVOG+IFQ8I4S81w
+o/M66x129XxOj2vDJ3ZGUIExr88MXnbkfeRVfasRXET5S5T9RWPOj5mwEe8lyz3b
+5J5SkS4rSeJ9rN7yvPUVmQKBgAvrrB67LRzldxSNpfFLSn7nGBYx2oi2zEbYlQA7
+ImhZWqw64S5iLz2yR3x4G9cmhmZjnXrAqcfVIez14PgzLL6V2wI0ID6qCZf+V25b
+OdW4M69UZMOHks5HTUJRfe8Z87rXWdq9KQu5GUaIAnSP/D2MNfPbf2yfpV4bV0Yz
+qtC9AoGAD3/XXaeGCdV5DPomEmehp84JXU2q/YECRvph46tr4jArG67PCvx2m84B
++W6my4Yi7QJcW4gC0gsdAuxbJl4Y7MCZBnTtNIRCRnHEIciKITJ/+brFln5QUgyn
+WnXEPN8q7VjSVXGrljFuLWkzi2Vh8iZDgourNfW+iYDGCJjx1H0=
+-----END RSA PRIVATE KEY-----
diff --git a/src/python/grpcio_tests/tests/unit/credentials/certificate_hierarchy_1/intermediate/private/localhost-1.key.pem b/src/python/grpcio_tests/tests/unit/credentials/certificate_hierarchy_1/intermediate/private/localhost-1.key.pem
new file mode 100644
index 0000000000..aa83f1a4a2
--- /dev/null
+++ b/src/python/grpcio_tests/tests/unit/credentials/certificate_hierarchy_1/intermediate/private/localhost-1.key.pem
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEogIBAAKCAQEArRAy0Nim9P883BAisXdFoKmgHGTtcLH/SzwkkPWTFHz0rHU1
+Klwzw8u3OkRyvgoQp7DqkohboNMDwg5VrOOcfKwtM2GZ5jixo+YKvJ25oj8Jfr+4
+0baznyWTmOcfoviKrb7u2T9BPEEz5og+lXRDAsTFATGaQDX2LN3Dd9KIw+7sWY+g
+c3Zi13HHaWYhtmfJjzFbH1vDxHKCdSdgtPyEhqcJ4OC6wbgp/mQ01VlPAr08kRfk
+C8mTTS7atqc410irKViF3sWi4YNPf7LuBrjo75FIIOp+sQgZE6xwOuZ/9bT2Zx/I
+UtCCTqzVgZI0s5NVlINtWR6eyyxQ1uDKTs4xrQIDAQABAoIBAC56mDswxH4uAmlT
+yA2Da+a/R6n4jTBkDZ1mFKf93Dd3a7rZa6Lpylk+YAI9GdfiGiD/SbB7AKjLo0m9
+0dKx+ngdQbJ39v42obbT9HQ9o/poFaO91+QyvkDytZYuFHgPaidJjRo5e8qz9D1o
+v+4hoFGhCQvOB5BRLcFU+cc3etWr5t61sNL/kKCWEDd+MWWsOCHpdhEoWC+o25pC
+bhD3FG5xoz+8zL7WdNfke/4twfKoBJ/kq89bfIkl8eKpg387WBQY44RJF7/zVr7a
+9dsUuW2y/wVXslCHChjSrxhRlOyy5ssv3EgKh8gPkZ+oeKuONqAGw27nyKyvpjxS
+i62K+WECgYEA4oKpIS2D77RCC6rpYIK6KYfbUcNSOtHFvcbf0RH9Oi8vSRYum2ZA
+/ITdWSFgWkhT6iOSPuvZlu/EvueWDgNgW1ZVsTMFeapz1+Jwk7JRoBKF1dUEwELh
+jdAswdh0MLbgBYs6NXtVVkeK2ocgZtosmt1PUktl566NlyIyhOjH6vkCgYEAw5g0
+cteTpz+noKsfWcbnjyesuQy0caICfZIE01nKv9rKTF8BtCO6Qxj10iM2o00jW7Vl
+tZa/igjuqvozXAHBI3xegtrWV05urkjj3FB/Pyuqsx3wxhAdSNchQjdTjwUBQEzp
+3ztGSlDTRPpijnpW28lg8Kkr3weryaHvl0xM1VUCgYBqnTN8QU8rgT3g/gYw/fcf
+2ylY98V5mAkqBTSN1JjLTTBFh2JSlLOb5/HDpRkUBZ0xxKJuaVaWW67QaHLRj7dH
+5oAZErnOBXPXNmbkrfcLkAxclJJS6Gf/9u9KIla2Iy2YjmrMh4uoO65Yo2eV4bVD
+A031nzWM8jUE4PzEYEjRCQKBgHDdTj6KiQg0Yg0DUabjcNEZasCpRSJhAyDkdmZi
+5OzKWnuxQvFowF1hdM/aQ/f9Vg7gYJ1lLIeBWf9NOv+3f3RzmrHVh2N/vbxSETIb
+PSH9l5WeDEauG8fhY66q8EuR7sPk3ftTX98YPqEJ/n8Ktz5COO8GH2umKInEKNXc
+UGW1AoGAfENy7vInNv0tzFWPSYdFgesvzo7e8mXyVO8hCyWvY3rxW2on7qfLF3Z9
+fHjd7P9gULja0n1kvmxwUC3u20RrvpY59F4hfi+ji2EiubS9Yhszd2e1CLeRMkln
+ojDjnflN32ZbWVHX/i6g3Dabq9JOD0FsOaOlriLMuofdA6jTUFE=
+-----END RSA PRIVATE KEY-----
diff --git a/src/python/grpcio_tests/tests/unit/credentials/certificate_hierarchy_2/certs/ca.cert.pem b/src/python/grpcio_tests/tests/unit/credentials/certificate_hierarchy_2/certs/ca.cert.pem
new file mode 100644
index 0000000000..212b5862cb
--- /dev/null
+++ b/src/python/grpcio_tests/tests/unit/credentials/certificate_hierarchy_2/certs/ca.cert.pem
@@ -0,0 +1,31 @@
+-----BEGIN CERTIFICATE-----
+MIIFZDCCA0ygAwIBAgIJALhSfZ8i0rWTMA0GCSqGSIb3DQEBCwUAMD8xCzAJBgNV
+BAYTAnVzMQ4wDAYDVQQIDAVkdW1teTEOMAwGA1UECgwFZHVtbXkxEDAOBgNVBAMM
+B3Jvb3QgY2EwHhcNMTcxMTAyMDAzNzU4WhcNMzcxMDI4MDAzNzU4WjA/MQswCQYD
+VQQGEwJ1czEOMAwGA1UECAwFZHVtbXkxDjAMBgNVBAoMBWR1bW15MRAwDgYDVQQD
+DAdyb290IGNhMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIICCgKCAgEArHaQ3uyp
+wVaVPZDYvy/EJbnP7KbZNPBvKpQCDEqg9B2TPaC8WVjiqT6I88c+KcGuB8nADJdk
+o10iZC5AwbrXK4ELSCOvBpxYI5sjoFcv3lZ/oe4e650QO2L/ADmtwLaLYK6rZkwW
+Sd90yCGF7ZOTZTJZDwmWEl+ohi+2ow6sRMHKcSKUNfx9G5BB7TOzoqUxqH+moEds
+YpjVMEcKzQi2FmbRd+8Dlg2eGqA2V4faprGQwoYz8NqJZGa/KPpRvXE2VjSTDN6b
+rJ7mmui6eYN53mZEBRYogyoQHdFXhK02FgyoPEgR/wQlLLbQ+xxOcv02YsOljtza
+hl5LjeNUYPMjyhef0QpONp+5NcFhZf38DsSq5EWZLLxPScxwl0lBQkJTjo5ARuFl
+Mrv50RYrLwv4ImsiO2ftE7gAX4vNsgcixnCHd6rNzoGimf1+DSvDVJ9ujWo7HPN3
+7ONuoyjsU4mUJJpYXs8zHx5WSxaYiPJRcmG3LjcU5/A+Fs7bkqSrlEjJsG29xDrO
+vKR7hH+m6MwcIcXSh9wjjAIvHxAALdU9xaYE3hmVkoxew1mRBsYq34h2vpwGOY5r
+0njRQyGGZnVa8qkQd6P3U5fcvLOM8v9QImZqRDS2jAGZXYruo/RIgJpklVX7ZY0+
+CnGdz4YxgLyOBJCDu3aEgL1oON3mg2SsrVMCAwEAAaNjMGEwHQYDVR0OBBYEFOBO
+9R6yEY6KOE+aSClwD2BQtWXKMB8GA1UdIwQYMBaAFOBO9R6yEY6KOE+aSClwD2BQ
+tWXKMA8GA1UdEwEB/wQFMAMBAf8wDgYDVR0PAQH/BAQDAgGGMA0GCSqGSIb3DQEB
+CwUAA4ICAQBElio7lLZ2eNt0tDPVSkChksc0IJ2/DpjYjqJzqxF/jZ2oToaAn2Er
+9iHl8ggTLB5WrTQxO1ev7qOwQsk9hrgvZ+EQCBTbyDyfNwVjgftH5jdQGdbyrnuJ
+yaks1mnd8is5ZZQmmQSd7GVOMTYi/7yH0xI4DHQ386dwnf5eKdoOI7yrVufEMxRx
+tB3Wv8KrX4z47zsIO28H/O0T26oUkrd4PEqFwDa5HQr6iG7QQgoGD/DPLgbBudlO
+kEos9fmXxiX60RLziKCE/DAxI3YWPvG3WhIWnVj22Oz6apz2pYWpOKwlaihNYrhq
+8xc02vIFwKh+t7D+wF4KHfduyMJ/wKVc5dzpNbTgkZePPKSB7QgbsMeRqbdPoXQF
+pMuzfj8VCWzpqBeHqE/adSCZhzeTrnyiYavF4T2vkSC5KJu+MHmbZ3nU9bcnnEy+
+24oEv9cEAiYNkvftbD+5ByEtkcBB2uT47sbiGrAeco+GxFGUVqi1IjObqrkIrPzV
+OjQhTZV6qgYCOuniJiGfoiMeHqdaDybpqo1bIrxSlvGRNcVoOsKt2/KP1DzW4ARZ
+hoRvayU2apHz/T5TAailqAW2MsrjGRaVHQTmeZKag8CKtAcjWzui0J2DnfXxUMn8
+R3ruFu3xJduOT1VghT9L9udvX9YhPCIKVL9+B5eFX9eMV6N7hUnVug==
+-----END CERTIFICATE-----
diff --git a/src/python/grpcio_tests/tests/unit/credentials/certificate_hierarchy_2/intermediate/certs/client.cert.pem b/src/python/grpcio_tests/tests/unit/credentials/certificate_hierarchy_2/intermediate/certs/client.cert.pem
new file mode 100644
index 0000000000..b6f4280168
--- /dev/null
+++ b/src/python/grpcio_tests/tests/unit/credentials/certificate_hierarchy_2/intermediate/certs/client.cert.pem
@@ -0,0 +1,28 @@
+-----BEGIN CERTIFICATE-----
+MIIExzCCAq+gAwIBAgICEAMwDQYJKoZIhvcNAQELBQAwRzELMAkGA1UEBhMCdXMx
+DjAMBgNVBAgMBWR1bW15MQ4wDAYDVQQKDAVkdW1teTEYMBYGA1UEAwwPaW50ZXJt
+ZWRpYXRlIGNhMB4XDTE3MTEwMjAwMzgwMFoXDTI3MTAzMTAwMzgwMFowPjELMAkG
+A1UEBhMCdXMxDjAMBgNVBAgMBWR1bW15MQ4wDAYDVQQKDAVkdW1teTEPMA0GA1UE
+AwwGY2xpZW50MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAyxZFTLqv
+Gd9SpFAykyRyLQgHcR5hgD55mz+9fl1OfnMoAc7yTdPVLksDLmeFUlxcvCtLHysJ
+klIBX62c6LzbsVcfLg/DPJlQxFnkhJCRKen4fp7x9h62qqJkDFVXsiEFza9L1lsN
+4OwqU8i4RRgZ/xggM/s/wVBtynioeW9QADNmKZ1n6HVKkYwdOynbFSggYfFrL3HL
+54bC9roZUETin0G5wZ9QU+srgivT0a/KC3ourBYHXAI40iHuuOBf3syDVJ6xId/r
+3UO3qkiQ5q7pwglg+8Nx7Q3CFtGZY3ewxSSSDo6BOyweGYMsBaxMO3EyTqecyfXn
+3n4XPqwmDalWYQIDAQABo4HFMIHCMAkGA1UdEwQCMAAwEQYJYIZIAYb4QgEBBAQD
+AgWgMDMGCWCGSAGG+EIBDQQmFiRPcGVuU1NMIEdlbmVyYXRlZCBDbGllbnQgQ2Vy
+dGlmaWNhdGUwHQYDVR0OBBYEFP2bodoNQ1tCNEOALPnygGMUfNI+MB8GA1UdIwQY
+MBaAFOWzLd7eBJwSNbzRqNsD7MQDCHg/MA4GA1UdDwEB/wQEAwIF4DAdBgNVHSUE
+FjAUBggrBgEFBQcDAgYIKwYBBQUHAwQwDQYJKoZIhvcNAQELBQADggIBAHqUuCLt
+olOdR9p/g+KgGPnKuVgMn15Wc2VLCrbbl2P0fuCcNWmnBKqHHgQ1EJEpgnQ2N8m6
+tOGucX7IAzlZj36RP4lN3gZqFRSO/OiTOUYpE6Uv1hYRxeMzAYo5sBdCiiypjV9z
+H0Ew5NuWRf2/0nFWoywB9ktHcfD8lRFI3o8zUFXmE2JSUPQtKhW3tBkPPjYBlgzD
+RD8cq8dVK9P7i3tUENP+MNHJToNLFBqfA9De6bKnhCWHhZkfB0VeeSm4Ja9HkCg/
+DB+PAKMfbLCH5T8gCpEWxNlvj09r9mn37fNjtJPO/goAcNZNO2AURmb/ZQ4ggdry
+xb6lm832qplMUMWx//Ore0faEodlEc5d2kEtmcjj79gAypcLmm74q7CPt7xmniyd
+XvNT33S2tkh4dSirpCVwq0xyqOP3ZqTsTjudTveTBaTZNhTbCjDbaV7ga47TcH9/
++OZ3fQKjt2LAC6162wgEFZf10nUgaAXvSlI74gru93vEwWd8Pd3sWfGwuAFX3oKI
+JuwL2kxEuoZQmeRiVJu6KQb+Im7d5CIoWViDmfxcSDJfdtSePTqmDURIx87fw14Z
+XBWJP4PiK5PRmG/L0cGiDckmDKm/MuD13Z2I/NMl81GNY/q3WY2O7BmddPpAG5dr
+sc5hOqA9+jX08XbxKnfBPYllK5skYMkFH5tN
+-----END CERTIFICATE-----
diff --git a/src/python/grpcio_tests/tests/unit/credentials/certificate_hierarchy_2/intermediate/certs/intermediate.cert.pem b/src/python/grpcio_tests/tests/unit/credentials/certificate_hierarchy_2/intermediate/certs/intermediate.cert.pem
new file mode 100644
index 0000000000..4305e5333f
--- /dev/null
+++ b/src/python/grpcio_tests/tests/unit/credentials/certificate_hierarchy_2/intermediate/certs/intermediate.cert.pem
@@ -0,0 +1,31 @@
+-----BEGIN CERTIFICATE-----
+MIIFaDCCA1CgAwIBAgICEAAwDQYJKoZIhvcNAQELBQAwPzELMAkGA1UEBhMCdXMx
+DjAMBgNVBAgMBWR1bW15MQ4wDAYDVQQKDAVkdW1teTEQMA4GA1UEAwwHcm9vdCBj
+YTAeFw0xNzExMDIwMDM3NTlaFw0yNzEwMzEwMDM3NTlaMEcxCzAJBgNVBAYTAnVz
+MQ4wDAYDVQQIDAVkdW1teTEOMAwGA1UECgwFZHVtbXkxGDAWBgNVBAMMD2ludGVy
+bWVkaWF0ZSBjYTCCAiIwDQYJKoZIhvcNAQEBBQADggIPADCCAgoCggIBAKuM2iFz
+CCKmbs4uLj/8wjtxf+jFmkwzN4Pps4hqJ3NJqWB326LhzWyqM4WiTAJWE02wSJbS
+16RPfbjkVC77OI4+PUdwqxU9vNAP/95w0h6hBSFtkysuT5VVUt5jiY7wnUKgqTCi
+MYhYOl+HEP32O4cnxAazkUKKvtyrd4/PvejJ9zz+iYexRnaaGfOFR3co7jQ5QKar
+oK4UgJC3mVDZQEMBV0oljkpgVQMAVb4XQU7e+o25JOUkOoK6LdK/b/95khR0jTyD
+OBLqd4mCEzxNi+jZ0jLTLPk0c+DnGmRfoNUxnFb40R8QnEIEKwf+JKyl6p89oqOl
+pvIZFLZlUWIS4qL+993l1SCqPkWJOAdTg+s/Zh6DeAOhrUn9/wk0aQwZrK7wQQLJ
+4GGhxC/FfuUGsLqZszAVkP8jDEWnzhN2rw3V+C7v6lj4qHhUwqGHuYzCx2Hxl+B8
+UyBmZb9gXKVUtAvaZjaL2PDj1ZAxT0KVMlw1ZVrZu45OsHNQuBx/4uIAt6Rga8yt
+av1lsq+hFqwI4bU/oZC/oPMacOsB4qEkAA1101WjMc5bg6JOPWobwIqmUXQR1WJE
+j30e99HCpk1Cc2+9sUCzNu8KvU5kUY2K90zwqProvj5IfMuDetAVXsEjgW+ZqSho
+UMIpJ2M/hzAFl8Z5IRlG+YNfZNXl0FqJ5LzLAgMBAAGjZjBkMB0GA1UdDgQWBBTl
+sy3e3gScEjW80ajbA+zEAwh4PzAfBgNVHSMEGDAWgBTgTvUeshGOijhPmkgpcA9g
+ULVlyjASBgNVHRMBAf8ECDAGAQH/AgEAMA4GA1UdDwEB/wQEAwIBhjANBgkqhkiG
+9w0BAQsFAAOCAgEAOS7DtliOUPcVosRfyx9dHcSUc3O+uY8uKjSHhdIrxDJm4lwP
+Q6lKg5j8CdMVb+sDQmyBkqQIA/6E13corP6R283jO6W8D4A8kjOiQWpXfjW6OcP3
+4rrDEWhCdeLFSNJIYOFkr2qWJpI/k0VpyDnmY0YluS5WbNjg6zTzGelzhFbV7/S1
+cteNAZD0vHD8NmbLVDJjjIY3E/iwzoUzBncLYbDwqyVS1g6utWdSy8LEJxzzqqWJ
+pBKlNYILAdh8efBgvotafaxsn2nfjmVmekPn3KcQZuE4Kzv1EQ2PrHpGeJKwh6up
+YBL2tav5cAki8bWoGPr2oGmWUf9L2tB57SdWdaY60ifzmQaeGiWPZBSmAz7PRSrz
+sR9SMIkBfYVRxXgWwlvr8JYnd2h/Ef5K9fI32nGfje+7/0kPEjNyjehri7sV4Sjt
+zzkDiFO+JklrRuLBPMFYOokq6Pcko32FKlE82pe8QkMDS8Sk//9PqCTK9ceB7y6E
+NYLNBW/X9SAw/TR5kdRinHHgHyEug7N4+DCU3lU1wl72ZjoiGE7V6c2AssFC2VcE
+E+WYxJT1ROJ1/5+U6BKdaIpTwMtRIFRomOEI66iOwOSEwqLIztkqxwpQ7THraWKm
+2W5e54u/efapIDcQFnP3E8r7TD0PdIeU6mD28o0+WiK3uL/OZpvyKaHPeFU=
+-----END CERTIFICATE-----
diff --git a/src/python/grpcio_tests/tests/unit/credentials/certificate_hierarchy_2/intermediate/certs/localhost-1.cert.pem b/src/python/grpcio_tests/tests/unit/credentials/certificate_hierarchy_2/intermediate/certs/localhost-1.cert.pem
new file mode 100644
index 0000000000..2850e42fe9
--- /dev/null
+++ b/src/python/grpcio_tests/tests/unit/credentials/certificate_hierarchy_2/intermediate/certs/localhost-1.cert.pem
@@ -0,0 +1,30 @@
+-----BEGIN CERTIFICATE-----
+MIIFFzCCAv+gAwIBAgICEAAwDQYJKoZIhvcNAQELBQAwRzELMAkGA1UEBhMCdXMx
+DjAMBgNVBAgMBWR1bW15MQ4wDAYDVQQKDAVkdW1teTEYMBYGA1UEAwwPaW50ZXJt
+ZWRpYXRlIGNhMB4XDTE3MTEwMjAwMzc1OVoXDTI3MTAzMTAwMzc1OVowTTELMAkG
+A1UEBhMCdXMxDjAMBgNVBAgMBWR1bW15MQ4wDAYDVQQKDAVkdW1teTEKMAgGA1UE
+CwwBMTESMBAGA1UEAwwJbG9jYWxob3N0MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A
+MIIBCgKCAQEAycY/7H1xk3/XHZRopULV7YsOzPIrMG25zoACbpDZxjS0I+2r1c7V
+wnvE8TszAkloLi+Skku5CYC7IvVEEEuKuIuV+8M48FJEwlCPge8LPiy18C+npCEd
+fgDzCV/O9DfJj6UaiCUayVE7UujXoke7AlKQEJcqvnD/CoTv2Y8jV1A6mPf6CTEI
+Sl1BMeFSmeFyvZll+xJ8Up1KfQZxKhtpP1s/rp6ZNlqSs1LM5+vcDHHZ6COTbq7t
+2vvcmGDTqeCLsqicBg1kJyMPRtqa0bNPj2bcVtcK0Ndfn6eL2hi+EoBy2nIXi6aG
+PpXf85b9bCLd5pZI80nHzFlhdvV+SxqrfwIDAQABo4IBBTCCAQEwCQYDVR0TBAIw
+ADARBglghkgBhvhCAQEEBAMCBkAwMwYJYIZIAYb4QgENBCYWJE9wZW5TU0wgR2Vu
+ZXJhdGVkIFNlcnZlciBDZXJ0aWZpY2F0ZTAdBgNVHQ4EFgQUoYjECaDz/ZELru/r
+jfTB1ShlVrAwaAYDVR0jBGEwX4AU5bMt3t4EnBI1vNGo2wPsxAMIeD+hQ6RBMD8x
+CzAJBgNVBAYTAnVzMQ4wDAYDVQQIDAVkdW1teTEOMAwGA1UECgwFZHVtbXkxEDAO
+BgNVBAMMB3Jvb3QgY2GCAhAAMA4GA1UdDwEB/wQEAwIFoDATBgNVHSUEDDAKBggr
+BgEFBQcDATANBgkqhkiG9w0BAQsFAAOCAgEAiiR2knMMym4O+3fD1KlYSnc2UR3v
+0FlRVAsr8wvTlVjJhx7DbRusBNJHWX66mUgK9x5OLnhyvyqlFVhR9AwlnxgfLWz9
+nnACeXzcjQZnKWFQWu8bJSC6Ene6rd1g2acK6SOjxavVbj7JVFnmlHF/naZUzvMl
+mJivYta4k7ob8UcX0I5TlJpzglU3UHyyJd5d9zhbF8wqbBq63zR2ovWci4pYCg+F
+jYcTGYVZJti3SHO+9/EqTC9x2KDNs3o0+rreJ3GuoonkInKZMQQZJQ6qILvkxlhT
+jyU5xlcaJ+0tSaiFK3eF0nXIpFYdZbIHYPCdLjh9AZ2dkFcAgSa/L8+tsVt60k8D
+HTO0Hz6dW5D2ckeebZvz5LACMN89gVzrc/rVkeg7QmpSbjkTSLC2KJS53hJzWcEI
+3KB73B9iY+ZYytcYBTYLizsAxd5g7j9z8UXrmVQ4mWbh2+xKiG+9aVOzCZ09AYi6
+WVK2aRcMQshgkkqPOloN9OeQNCE8Exf7N/zHsBhygorJXoD/PFgnV1VZm8xkOdiJ
+zTb3bpGdmL5+bzzS6wP8Q7pGZGYdlnB7JNO8oMYPPtzX8OOx92BTkPnqJnnRWTpR
+SjMEEdQe8K7iXxejQkjaAq5BlwaAOjCjPTqYomECcYjC0WaXsmrPcnZwSqpnHZZ2
+OiINYJub5cvBLNo=
+-----END CERTIFICATE-----
diff --git a/src/python/grpcio_tests/tests/unit/credentials/certificate_hierarchy_2/intermediate/private/client.key.pem b/src/python/grpcio_tests/tests/unit/credentials/certificate_hierarchy_2/intermediate/private/client.key.pem
new file mode 100644
index 0000000000..a4c5fd4a56
--- /dev/null
+++ b/src/python/grpcio_tests/tests/unit/credentials/certificate_hierarchy_2/intermediate/private/client.key.pem
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAyxZFTLqvGd9SpFAykyRyLQgHcR5hgD55mz+9fl1OfnMoAc7y
+TdPVLksDLmeFUlxcvCtLHysJklIBX62c6LzbsVcfLg/DPJlQxFnkhJCRKen4fp7x
+9h62qqJkDFVXsiEFza9L1lsN4OwqU8i4RRgZ/xggM/s/wVBtynioeW9QADNmKZ1n
+6HVKkYwdOynbFSggYfFrL3HL54bC9roZUETin0G5wZ9QU+srgivT0a/KC3ourBYH
+XAI40iHuuOBf3syDVJ6xId/r3UO3qkiQ5q7pwglg+8Nx7Q3CFtGZY3ewxSSSDo6B
+OyweGYMsBaxMO3EyTqecyfXn3n4XPqwmDalWYQIDAQABAoIBAFhIOR3OtVlw3BLz
+jdiq6jsrF1kUFNxTzDcxsSUiWIHde1G17Vzpre0uzJY6iBkyb1mZFFHbOpDxtwkp
+hmEh3/qqXbJ/RaatGxAP56e81G28+LnKTHJqDYwFhapa2wFjG4u7HSN0d4cEAq5j
+Pb9DZ+GdUjpmiON3HBL8+ne3bLZ42uI+DSVe8d3irbqg2rqsiANf0gdimMW4nuI4
+rVxf8HrY43PdQn/Vby+7qLRE3tmIlpbTqJGRtWRjdeBBI91APCrRljjXrKqT6Zpa
+E6Daz3YIQvXkIT0q+WkeN1VmQbtRnk7kRsPNp15kSwpHfmv6o/vkO9OUb1n71P2F
+wnB0WDECgYEA8iltnKxXnjqwZ/vzIWzcd94j+mdZg/K2/JCOqjwMvpSGCvx2zUmq
+Y2nxO2K85AVeOm/Yt87SMODB6AQ9CsrVGEUAzzacvCJDb8oUhaOL5gypnyvZiGCy
+snzXfgB+v/xuGekIjs2y7E8h3GG40j0aNQnUY1Fuc6iaeJG4BtjkuQUCgYEA1rE4
+DrTSsUh3hLYQusIHZR8Lecrrd4QUZSMKLkWjobiSTw3m4mglx1s2G4eZ3WuzOyFq
+Dp3/b3yfT8prdPBGA6shHNFf+1TO1q1/pIt15dc3sFwxMkuunai8N4QZJRqZLbYq
+FkNFkZ20hFHcH/NHDsAsRL/0tJdEmJ2ruP+Qdq0CgYBsdPGKwgVb8J0hdU4nIkJ7
+zRoABFmrJwGdjIDY7Zwnnw2JzhjHSL7vV3ubRVWkKmNReNZvPEoXahJuf7d3JfDa
+tczvAV6hRBc/8hnO4Li/h9xQVatP0T83gYJiBIbAJaaKJDyY+Lex7p8TvRCx2Hvs
+VUKyWL5HPrQwW9M3/dwyoQKBgQCNQoPA4Wcz8Jt7PZQaXaoh9eBGHab6t3P366s6
+MOXudZQG4f3FgINC/ZfHW1x43PFL+btfrMOyJkxoYqZ7hdB7f3DFFlpR80Y46GVw
+7bYAKbBhoPdZwYQ+BhT5bjhhOnQJKK/egBrZKevpmDb+6sIZSYaXIbovzMv8otmn
+WrhB7QKBgAdl+KYBQULCUBp8qCQH5sAQoWErpyuD2FNN6LGknpPqn4DdujvwEP0Z
+OSvbauLkI0Qc9/MezKPTeYXlFqdbpItwyySJsUkiI3HhVYlBgDkZ7xb6uHIH5E6I
+bKgIW5JEf5I7Eu1iurORkXxCCGMkiQmEs4X5kSXXRYgXfNgAD0FX
+-----END RSA PRIVATE KEY-----
diff --git a/src/python/grpcio_tests/tests/unit/credentials/certificate_hierarchy_2/intermediate/private/localhost-1.key.pem b/src/python/grpcio_tests/tests/unit/credentials/certificate_hierarchy_2/intermediate/private/localhost-1.key.pem
new file mode 100644
index 0000000000..8cba174841
--- /dev/null
+++ b/src/python/grpcio_tests/tests/unit/credentials/certificate_hierarchy_2/intermediate/private/localhost-1.key.pem
@@ -0,0 +1,27 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIEpAIBAAKCAQEAycY/7H1xk3/XHZRopULV7YsOzPIrMG25zoACbpDZxjS0I+2r
+1c7VwnvE8TszAkloLi+Skku5CYC7IvVEEEuKuIuV+8M48FJEwlCPge8LPiy18C+n
+pCEdfgDzCV/O9DfJj6UaiCUayVE7UujXoke7AlKQEJcqvnD/CoTv2Y8jV1A6mPf6
+CTEISl1BMeFSmeFyvZll+xJ8Up1KfQZxKhtpP1s/rp6ZNlqSs1LM5+vcDHHZ6COT
+bq7t2vvcmGDTqeCLsqicBg1kJyMPRtqa0bNPj2bcVtcK0Ndfn6eL2hi+EoBy2nIX
+i6aGPpXf85b9bCLd5pZI80nHzFlhdvV+SxqrfwIDAQABAoIBAQC022161aoTEtjH
+m7n8v56vUCCRFVQfEYsljFohrtZ0sdLyDVwjxkSWEYiizXRYTWIDXALd/N+7o9aZ
+bAx5Kq0J45wpUYBc8PDO15T6W0DRlxPxWVXDaSddRQ6TTXxcLREPH2dbtx5+asBo
+/Woi/Haki0q0hDr8/p2sWSH/+SwtWpOezGVlrWrkMeIhlBwHZfdHVoZvSx65Uv7x
+WU07vsjrbXNDwf+2fmklAQrzhedCeh8loGyjtN3cfrTjrE1zqpEsHnlZcJxe6sRB
+1nOqpoUnpZXklDDIYC8EmeubmDJ0jnXOQCDDep3MzVcnZGyF5E/+szaa1NL70Ayj
+rbKk1Y3ZAoGBAPy/1ym7Cjl4OGHN2fdkR6iL68ebJozpr+eTSxDNLuBSi5IJxJyG
+1+B4+v1u0RwZ3DjrSQsO5DCbZ+DHU6O/DAJK2CxUED+M+G2kRyffailRQmNzjpRG
+75dIhSkSRYH8vdvEOnGpeQBZwBcCRH/2YUMlZeSfx9fHJhk1nyUxJeHjAoGBAMxe
+k+cBb0zYok+Ww1xTwOdq0PwKj0oDsEg8hOdWc8pH0SlOAB4BI5kmfd1JDMHfRc49
+7tpNqjsPrnlb9xd8l0281Lj2NoVSE5KX1JtsOsKecQsvHH5zRk4eJ3h/mNixpjfe
+79Zc/O40T4rWpQRqhat+WHveJC0/ON4AH4uT0BK1AoGBAPcTioCu6YXYsjVaCJPB
+IhPwBGOylfL2lxDoel9IVWTRDMOMbPkfEHXNjn6lECJKXW//Af6fZg7mPJwN/wN5
+xYGQLNbYrrGRW2HDUBP4YU1WtHGIC3+EAL+BEztdMzmpGuh1YTSvmSvwkMltXA1D
+iz0amArw72lOsz29n3+6FfBFAoGAIpRqMC8k9vq80/yth6TAQifnvo3G2v4uyLo8
+vqv5IaPvNy70hB8rN9G0gEnI99Dgjdoa3SNBB4dKvUwbTgUN0OB/meBHL13I5Af+
+uGGiu6V1eS/6gUbeAX/Gq/PjF99PQareKAZJ4cBGKTbSayHfBjp1nFflBSbqZ13b
++JEFJvUCgYBOs2J2XXamPbI7gu7B2TE9j/62v0SJyoHq2LHMmYUDRuPdPk3eKCt3
+283w+E8XUIFbctaxsbo8msNjjvV22D/Nci3d87aPe8bn1SVto3GnTuwnOpRq3E+3
+wAarqrhiZbGZSCcAkEOk7FlxAwYnCM6paqMxDEMCJ4qChMM42E9ZyQ==
+-----END RSA PRIVATE KEY-----
diff --git a/src/python/grpcio_tests/tests/unit/resources.py b/src/python/grpcio_tests/tests/unit/resources.py
index 823d2307d3..11ef9e8565 100644
--- a/src/python/grpcio_tests/tests/unit/resources.py
+++ b/src/python/grpcio_tests/tests/unit/resources.py
@@ -11,7 +11,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
-"""Constants and functions for data used in interoperability testing."""
+"""Constants and functions for data used in testing."""
 
 import os
 
@@ -34,3 +34,81 @@ def private_key():
 def certificate_chain():
     return pkg_resources.resource_string(__name__,
                                          _CERTIFICATE_CHAIN_RESOURCE_PATH)
+
+
+def cert_hier_1_root_ca_cert():
+    return pkg_resources.resource_string(
+        __name__, 'credentials/certificate_hierarchy_1/certs/ca.cert.pem')
+
+
+def cert_hier_1_intermediate_ca_cert():
+    return pkg_resources.resource_string(
+        __name__,
+        'credentials/certificate_hierarchy_1/intermediate/certs/intermediate.cert.pem'
+    )
+
+
+def cert_hier_1_client_1_key():
+    return pkg_resources.resource_string(
+        __name__,
+        'credentials/certificate_hierarchy_1/intermediate/private/client.key.pem'
+    )
+
+
+def cert_hier_1_client_1_cert():
+    return pkg_resources.resource_string(
+        __name__,
+        'credentials/certificate_hierarchy_1/intermediate/certs/client.cert.pem')
+
+
+def cert_hier_1_server_1_key():
+    return pkg_resources.resource_string(
+        __name__,
+        'credentials/certificate_hierarchy_1/intermediate/private/localhost-1.key.pem'
+    )
+
+
+def cert_hier_1_server_1_cert():
+    return pkg_resources.resource_string(
+        __name__,
+        'credentials/certificate_hierarchy_1/intermediate/certs/localhost-1.cert.pem'
+    )
+
+
+def cert_hier_2_root_ca_cert():
+    return pkg_resources.resource_string(
+        __name__, 'credentials/certificate_hierarchy_2/certs/ca.cert.pem')
+
+
+def cert_hier_2_intermediate_ca_cert():
+    return pkg_resources.resource_string(
+        __name__,
+        'credentials/certificate_hierarchy_2/intermediate/certs/intermediate.cert.pem'
+    )
+
+
+def cert_hier_2_client_1_key():
+    return pkg_resources.resource_string(
+        __name__,
+        'credentials/certificate_hierarchy_2/intermediate/private/client.key.pem'
+    )
+
+
+def cert_hier_2_client_1_cert():
+    return pkg_resources.resource_string(
+        __name__,
+        'credentials/certificate_hierarchy_2/intermediate/certs/client.cert.pem')
+
+
+def cert_hier_2_server_1_key():
+    return pkg_resources.resource_string(
+        __name__,
+        'credentials/certificate_hierarchy_2/intermediate/private/localhost-1.key.pem'
+    )
+
+
+def cert_hier_2_server_1_cert():
+    return pkg_resources.resource_string(
+        __name__,
+        'credentials/certificate_hierarchy_2/intermediate/certs/localhost-1.cert.pem'
+    )