diff options
| -rw-r--r-- | .gitignore | 1 | ||||
| -rw-r--r-- | package.json | 192 | ||||
| -rw-r--r-- | src/csharp/Grpc.Core/Internal/DefaultSslRootsOverride.cs | 2 | ||||
| -rw-r--r-- | test/cpp/util/test_credentials_provider.cc | 118 | ||||
| -rw-r--r-- | test/cpp/util/test_credentials_provider.h | 16 | ||||
| -rwxr-xr-x | tools/buildgen/generate_projects.py | 31 | ||||
| -rwxr-xr-x | tools/buildgen/mako_renderer.py | 40 |
7 files changed, 253 insertions, 147 deletions
diff --git a/.gitignore b/.gitignore index 9a1dc00850..502483f456 100644 --- a/.gitignore +++ b/.gitignore @@ -35,6 +35,7 @@ coverage # cache for run_tests.py .run_tests_cache +.preprocessed_build # emacs temp files *~ diff --git a/package.json b/package.json index 4e7e83b02b..db7f299e13 100644 --- a/package.json +++ b/package.json @@ -421,102 +421,6 @@ "third_party/zlib/trees.c", "third_party/zlib/uncompr.c", "third_party/zlib/zutil.c", - "include/grpc/support/alloc.h", - "include/grpc/support/atm.h", - "include/grpc/support/atm_gcc_atomic.h", - "include/grpc/support/atm_gcc_sync.h", - "include/grpc/support/atm_win32.h", - "include/grpc/support/avl.h", - "include/grpc/support/cmdline.h", - "include/grpc/support/cpu.h", - "include/grpc/support/histogram.h", - "include/grpc/support/host_port.h", - "include/grpc/support/log.h", - "include/grpc/support/log_win32.h", - "include/grpc/support/port_platform.h", - "include/grpc/support/slice.h", - "include/grpc/support/slice_buffer.h", - "include/grpc/support/string_util.h", - "include/grpc/support/subprocess.h", - "include/grpc/support/sync.h", - "include/grpc/support/sync_generic.h", - "include/grpc/support/sync_posix.h", - "include/grpc/support/sync_win32.h", - "include/grpc/support/thd.h", - "include/grpc/support/time.h", - "include/grpc/support/tls.h", - "include/grpc/support/tls_gcc.h", - "include/grpc/support/tls_msvc.h", - "include/grpc/support/tls_pthread.h", - "include/grpc/support/useful.h", - "include/grpc/impl/codegen/alloc.h", - "include/grpc/impl/codegen/atm.h", - "include/grpc/impl/codegen/atm_gcc_atomic.h", - "include/grpc/impl/codegen/atm_gcc_sync.h", - "include/grpc/impl/codegen/atm_win32.h", - "include/grpc/impl/codegen/log.h", - "include/grpc/impl/codegen/port_platform.h", - "include/grpc/impl/codegen/slice.h", - "include/grpc/impl/codegen/slice_buffer.h", - "include/grpc/impl/codegen/sync.h", - "include/grpc/impl/codegen/sync_generic.h", - "include/grpc/impl/codegen/sync_posix.h", - "include/grpc/impl/codegen/sync_win32.h", - "include/grpc/impl/codegen/time.h", - "src/core/profiling/timers.h", - "src/core/support/block_annotate.h", - "src/core/support/env.h", - "src/core/support/load_file.h", - "src/core/support/murmur_hash.h", - "src/core/support/stack_lockfree.h", - "src/core/support/string.h", - "src/core/support/string_win32.h", - "src/core/support/thd_internal.h", - "src/core/support/time_precise.h", - "src/core/support/tmpfile.h", - "src/core/profiling/basic_timers.c", - "src/core/profiling/stap_timers.c", - "src/core/support/alloc.c", - "src/core/support/avl.c", - "src/core/support/cmdline.c", - "src/core/support/cpu_iphone.c", - "src/core/support/cpu_linux.c", - "src/core/support/cpu_posix.c", - "src/core/support/cpu_windows.c", - "src/core/support/env_linux.c", - "src/core/support/env_posix.c", - "src/core/support/env_win32.c", - "src/core/support/histogram.c", - "src/core/support/host_port.c", - "src/core/support/load_file.c", - "src/core/support/log.c", - "src/core/support/log_android.c", - "src/core/support/log_linux.c", - "src/core/support/log_posix.c", - "src/core/support/log_win32.c", - "src/core/support/murmur_hash.c", - "src/core/support/slice.c", - "src/core/support/slice_buffer.c", - "src/core/support/stack_lockfree.c", - "src/core/support/string.c", - "src/core/support/string_posix.c", - "src/core/support/string_win32.c", - "src/core/support/subprocess_posix.c", - "src/core/support/subprocess_windows.c", - "src/core/support/sync.c", - "src/core/support/sync_posix.c", - "src/core/support/sync_win32.c", - "src/core/support/thd.c", - "src/core/support/thd_posix.c", - "src/core/support/thd_win32.c", - "src/core/support/time.c", - "src/core/support/time_posix.c", - "src/core/support/time_precise.c", - "src/core/support/time_win32.c", - "src/core/support/tls_pthread.c", - "src/core/support/tmpfile_posix.c", - "src/core/support/tmpfile_win32.c", - "src/core/support/wrap_memcpy.c", "third_party/boringssl/crypto/aes/internal.h", "third_party/boringssl/crypto/asn1/asn1_locl.h", "third_party/boringssl/crypto/bio/internal.h", @@ -918,6 +822,102 @@ "third_party/boringssl/ssl/t1_enc.c", "third_party/boringssl/ssl/t1_lib.c", "third_party/boringssl/ssl/tls_record.c", + "include/grpc/support/alloc.h", + "include/grpc/support/atm.h", + "include/grpc/support/atm_gcc_atomic.h", + "include/grpc/support/atm_gcc_sync.h", + "include/grpc/support/atm_win32.h", + "include/grpc/support/avl.h", + "include/grpc/support/cmdline.h", + "include/grpc/support/cpu.h", + "include/grpc/support/histogram.h", + "include/grpc/support/host_port.h", + "include/grpc/support/log.h", + "include/grpc/support/log_win32.h", + "include/grpc/support/port_platform.h", + "include/grpc/support/slice.h", + "include/grpc/support/slice_buffer.h", + "include/grpc/support/string_util.h", + "include/grpc/support/subprocess.h", + "include/grpc/support/sync.h", + "include/grpc/support/sync_generic.h", + "include/grpc/support/sync_posix.h", + "include/grpc/support/sync_win32.h", + "include/grpc/support/thd.h", + "include/grpc/support/time.h", + "include/grpc/support/tls.h", + "include/grpc/support/tls_gcc.h", + "include/grpc/support/tls_msvc.h", + "include/grpc/support/tls_pthread.h", + "include/grpc/support/useful.h", + "include/grpc/impl/codegen/alloc.h", + "include/grpc/impl/codegen/atm.h", + "include/grpc/impl/codegen/atm_gcc_atomic.h", + "include/grpc/impl/codegen/atm_gcc_sync.h", + "include/grpc/impl/codegen/atm_win32.h", + "include/grpc/impl/codegen/log.h", + "include/grpc/impl/codegen/port_platform.h", + "include/grpc/impl/codegen/slice.h", + "include/grpc/impl/codegen/slice_buffer.h", + "include/grpc/impl/codegen/sync.h", + "include/grpc/impl/codegen/sync_generic.h", + "include/grpc/impl/codegen/sync_posix.h", + "include/grpc/impl/codegen/sync_win32.h", + "include/grpc/impl/codegen/time.h", + "src/core/profiling/timers.h", + "src/core/support/block_annotate.h", + "src/core/support/env.h", + "src/core/support/load_file.h", + "src/core/support/murmur_hash.h", + "src/core/support/stack_lockfree.h", + "src/core/support/string.h", + "src/core/support/string_win32.h", + "src/core/support/thd_internal.h", + "src/core/support/time_precise.h", + "src/core/support/tmpfile.h", + "src/core/profiling/basic_timers.c", + "src/core/profiling/stap_timers.c", + "src/core/support/alloc.c", + "src/core/support/avl.c", + "src/core/support/cmdline.c", + "src/core/support/cpu_iphone.c", + "src/core/support/cpu_linux.c", + "src/core/support/cpu_posix.c", + "src/core/support/cpu_windows.c", + "src/core/support/env_linux.c", + "src/core/support/env_posix.c", + "src/core/support/env_win32.c", + "src/core/support/histogram.c", + "src/core/support/host_port.c", + "src/core/support/load_file.c", + "src/core/support/log.c", + "src/core/support/log_android.c", + "src/core/support/log_linux.c", + "src/core/support/log_posix.c", + "src/core/support/log_win32.c", + "src/core/support/murmur_hash.c", + "src/core/support/slice.c", + "src/core/support/slice_buffer.c", + "src/core/support/stack_lockfree.c", + "src/core/support/string.c", + "src/core/support/string_posix.c", + "src/core/support/string_win32.c", + "src/core/support/subprocess_posix.c", + "src/core/support/subprocess_windows.c", + "src/core/support/sync.c", + "src/core/support/sync_posix.c", + "src/core/support/sync_win32.c", + "src/core/support/thd.c", + "src/core/support/thd_posix.c", + "src/core/support/thd_win32.c", + "src/core/support/time.c", + "src/core/support/time_posix.c", + "src/core/support/time_precise.c", + "src/core/support/time_win32.c", + "src/core/support/tls_pthread.c", + "src/core/support/tmpfile_posix.c", + "src/core/support/tmpfile_win32.c", + "src/core/support/wrap_memcpy.c", "binding.gyp" ], "main": "src/node/index.js", diff --git a/src/csharp/Grpc.Core/Internal/DefaultSslRootsOverride.cs b/src/csharp/Grpc.Core/Internal/DefaultSslRootsOverride.cs index eeaa7add81..dfaee5d9d7 100644 --- a/src/csharp/Grpc.Core/Internal/DefaultSslRootsOverride.cs +++ b/src/csharp/Grpc.Core/Internal/DefaultSslRootsOverride.cs @@ -56,7 +56,7 @@ namespace Grpc.Core.Internal { lock (staticLock) { - var stream = Assembly.GetExecutingAssembly().GetManifestResourceStream(RootsPemResourceName); + var stream = typeof(DefaultSslRootsOverride).GetTypeInfo().Assembly.GetManifestResourceStream(RootsPemResourceName); if (stream == null) { throw new IOException(string.Format("Error loading the embedded resource \"{0}\"", RootsPemResourceName)); diff --git a/test/cpp/util/test_credentials_provider.cc b/test/cpp/util/test_credentials_provider.cc index 1086e14258..cfd3ebbb11 100644 --- a/test/cpp/util/test_credentials_provider.cc +++ b/test/cpp/util/test_credentials_provider.cc @@ -34,48 +34,112 @@ #include "test/cpp/util/test_credentials_provider.h" +#include <grpc/support/sync.h> +#include <grpc++/impl/sync.h> + #include "test/core/end2end/data/ssl_test_data.h" +namespace { + +using grpc::ChannelArguments; +using grpc::ChannelCredentials; +using grpc::InsecureChannelCredentials; +using grpc::InsecureServerCredentials; +using grpc::ServerCredentials; +using grpc::SslCredentialsOptions; +using grpc::SslServerCredentialsOptions; +using grpc::testing::CredentialsProvider; + +class DefaultCredentialsProvider : public CredentialsProvider { + public: + ~DefaultCredentialsProvider() override {} + + std::shared_ptr<ChannelCredentials> GetChannelCredentials( + const grpc::string& type, ChannelArguments* args) override { + if (type == grpc::testing::kInsecureCredentialsType) { + return InsecureChannelCredentials(); + } else if (type == grpc::testing::kTlsCredentialsType) { + SslCredentialsOptions ssl_opts = {test_root_cert, "", ""}; + args->SetSslTargetNameOverride("foo.test.google.fr"); + return SslCredentials(ssl_opts); + } else { + gpr_log(GPR_ERROR, "Unsupported credentials type %s.", type.c_str()); + } + return nullptr; + } + + std::shared_ptr<ServerCredentials> GetServerCredentials( + const grpc::string& type) override { + if (type == grpc::testing::kInsecureCredentialsType) { + return InsecureServerCredentials(); + } else if (type == grpc::testing::kTlsCredentialsType) { + SslServerCredentialsOptions::PemKeyCertPair pkcp = {test_server1_key, + test_server1_cert}; + SslServerCredentialsOptions ssl_opts; + ssl_opts.pem_root_certs = ""; + ssl_opts.pem_key_cert_pairs.push_back(pkcp); + return SslServerCredentials(ssl_opts); + } else { + gpr_log(GPR_ERROR, "Unsupported credentials type %s.", type.c_str()); + } + return nullptr; + } + std::vector<grpc::string> GetSecureCredentialsTypeList() override { + std::vector<grpc::string> types; + types.push_back(grpc::testing::kTlsCredentialsType); + return types; + } +}; + +gpr_once g_once_init_provider_mu = GPR_ONCE_INIT; +grpc::mutex* g_provider_mu = nullptr; +CredentialsProvider* g_provider = nullptr; + +void InitProviderMu() { + g_provider_mu = new grpc::mutex; +} + +grpc::mutex& GetMu() { + gpr_once_init(&g_once_init_provider_mu, &InitProviderMu); + return *g_provider_mu; +} + +CredentialsProvider* GetProvider() { + grpc::unique_lock<grpc::mutex> lock(GetMu()); + if (g_provider == nullptr) { + g_provider = new DefaultCredentialsProvider; + } + return g_provider; +} + +} // namespace + namespace grpc { namespace testing { -const char kTlsCredentialsType[] = "TLS_CREDENTIALS"; +// Note that it is not thread-safe to set a provider while concurrently using +// the previously set provider, as this deletes and replaces it. nullptr may be +// given to reset to the default. +void SetTestCredentialsProvider(std::unique_ptr<CredentialsProvider> provider) { + grpc::unique_lock<grpc::mutex> lock(GetMu()); + if (g_provider != nullptr) { + delete g_provider; + } + g_provider = provider.release(); +} std::shared_ptr<ChannelCredentials> GetChannelCredentials( const grpc::string& type, ChannelArguments* args) { - if (type == kInsecureCredentialsType) { - return InsecureChannelCredentials(); - } else if (type == kTlsCredentialsType) { - SslCredentialsOptions ssl_opts = {test_root_cert, "", ""}; - args->SetSslTargetNameOverride("foo.test.google.fr"); - return SslCredentials(ssl_opts); - } else { - gpr_log(GPR_ERROR, "Unsupported credentials type %s.", type.c_str()); - } - return nullptr; + return GetProvider()->GetChannelCredentials(type, args); } std::shared_ptr<ServerCredentials> GetServerCredentials( const grpc::string& type) { - if (type == kInsecureCredentialsType) { - return InsecureServerCredentials(); - } else if (type == kTlsCredentialsType) { - SslServerCredentialsOptions::PemKeyCertPair pkcp = {test_server1_key, - test_server1_cert}; - SslServerCredentialsOptions ssl_opts; - ssl_opts.pem_root_certs = ""; - ssl_opts.pem_key_cert_pairs.push_back(pkcp); - return SslServerCredentials(ssl_opts); - } else { - gpr_log(GPR_ERROR, "Unsupported credentials type %s.", type.c_str()); - } - return nullptr; + return GetProvider()->GetServerCredentials(type); } std::vector<grpc::string> GetSecureCredentialsTypeList() { - std::vector<grpc::string> types; - types.push_back(kTlsCredentialsType); - return types; + return GetProvider()->GetSecureCredentialsTypeList(); } } // namespace testing diff --git a/test/cpp/util/test_credentials_provider.h b/test/cpp/util/test_credentials_provider.h index f7253051a9..a6b547cb07 100644 --- a/test/cpp/util/test_credentials_provider.h +++ b/test/cpp/util/test_credentials_provider.h @@ -44,6 +44,22 @@ namespace grpc { namespace testing { const char kInsecureCredentialsType[] = "INSECURE_CREDENTIALS"; +const char kTlsCredentialsType[] = "TLS_CREDENTIALS"; + +class CredentialsProvider { + public: + virtual ~CredentialsProvider() {} + + virtual std::shared_ptr<ChannelCredentials> GetChannelCredentials( + const grpc::string& type, ChannelArguments* args) = 0; + virtual std::shared_ptr<ServerCredentials> GetServerCredentials( + const grpc::string& type) = 0; + virtual std::vector<grpc::string> GetSecureCredentialsTypeList() = 0; +}; + +// Set the CredentialsProvider used by the other functions in this file. If this +// is not set, a default provider will be used. +void SetTestCredentialsProvider(std::unique_ptr<CredentialsProvider> provider); // Provide channel credentials according to the given type. Alter the channel // arguments if needed. diff --git a/tools/buildgen/generate_projects.py b/tools/buildgen/generate_projects.py index 965dd292af..0602d93e56 100755 --- a/tools/buildgen/generate_projects.py +++ b/tools/buildgen/generate_projects.py @@ -45,12 +45,12 @@ import jobset os.chdir(os.path.join(os.path.dirname(sys.argv[0]), '..', '..')) argp = argparse.ArgumentParser() -argp.add_argument('json', nargs='+') +argp.add_argument('build_files', nargs='+', default=[]) argp.add_argument('--templates', nargs='+', default=[]) argp.add_argument('--jobs', '-j', default=multiprocessing.cpu_count(), type=int) args = argp.parse_args() -json = args.json +json = args.build_files test = {} if 'TEST' in os.environ else None @@ -62,21 +62,31 @@ if not templates: for f in files: templates.append(os.path.join(root, f)) +pre_jobs = [] +base_cmd = ['python2.7', 'tools/buildgen/mako_renderer.py'] +cmd = base_cmd[:] +for plugin in plugins: + cmd.append('-p') + cmd.append(plugin) +for js in json: + cmd.append('-d') + cmd.append(js) +cmd.append('-w') +preprocessed_build = '.preprocessed_build' +cmd.append(preprocessed_build) +pre_jobs.append(jobset.JobSpec(cmd, shortname='preprocess', timeout_seconds=None)) + jobs = [] -for template in templates: +for template in reversed(sorted(templates)): root, f = os.path.split(template) if os.path.splitext(f)[1] == '.template': out_dir = '.' + root[len('templates'):] out = out_dir + '/' + os.path.splitext(f)[0] if not os.path.exists(out_dir): os.makedirs(out_dir) - cmd = ['python2.7', 'tools/buildgen/mako_renderer.py'] - for plugin in plugins: - cmd.append('-p') - cmd.append(plugin) - for js in json: - cmd.append('-d') - cmd.append(js) + cmd = base_cmd[:] + cmd.append('-P') + cmd.append(preprocessed_build) cmd.append('-o') if test is None: cmd.append(out) @@ -88,6 +98,7 @@ for template in templates: cmd.append(root + '/' + f) jobs.append(jobset.JobSpec(cmd, shortname=out, timeout_seconds=None)) +jobset.run(pre_jobs, maxjobs=args.jobs) jobset.run(jobs, maxjobs=args.jobs) if test is not None: diff --git a/tools/buildgen/mako_renderer.py b/tools/buildgen/mako_renderer.py index f1b28d352e..f629e68eb9 100755 --- a/tools/buildgen/mako_renderer.py +++ b/tools/buildgen/mako_renderer.py @@ -38,6 +38,7 @@ Just a wrapper around the mako rendering library. import getopt import imp import os +import cPickle as pickle import shutil import sys @@ -66,21 +67,23 @@ def out(msg): def showhelp(): - out('mako-renderer.py [-o out] [-m cache] [-d dict] [-d dict...] template') + out('mako-renderer.py [-o out] [-m cache] [-P preprocessed_input] [-d dict] [-d dict...]' + ' [-t template] [-w preprocessed_output]') def main(argv): got_input = False module_directory = None + preprocessed_output = None dictionary = {} json_dict = {} got_output = False - output_file = sys.stdout plugins = [] output_name = None + got_preprocessed_input = False try: - opts, args = getopt.getopt(argv, 'hm:d:o:p:') + opts, args = getopt.getopt(argv, 'hm:d:o:p:t:P:w:') except getopt.GetoptError: out('Unknown option') showhelp() @@ -104,18 +107,31 @@ def main(argv): showhelp() sys.exit(4) module_directory = arg + elif opt == '-P': + assert not got_preprocessed_input + assert json_dict == {} + sys.path.insert(0, os.path.abspath(os.path.join(os.path.dirname(sys.argv[0]), 'plugins'))) + with open(arg, 'r') as dict_file: + dictionary = pickle.load(dict_file) + got_preprocessed_input = True elif opt == '-d': - dict_file = open(arg, 'r') - bunch.merge_json(json_dict, yaml.load(dict_file.read())) - dict_file.close() + assert not got_preprocessed_input + with open(arg, 'r') as dict_file: + bunch.merge_json(json_dict, yaml.load(dict_file.read())) elif opt == '-p': plugins.append(import_plugin(arg)) + elif opt == '-w': + preprocessed_output = arg - for plugin in plugins: - plugin.mako_plugin(json_dict) + if not got_preprocessed_input: + for plugin in plugins: + plugin.mako_plugin(json_dict) + for k, v in json_dict.items(): + dictionary[k] = bunch.to_bunch(v) - for k, v in json_dict.items(): - dictionary[k] = bunch.to_bunch(v) + if preprocessed_output: + with open(preprocessed_output, 'w') as dict_file: + pickle.dump(dictionary, dict_file) cleared_dir = False for arg in args: @@ -168,11 +184,9 @@ def main(argv): with open(item_output_name, 'w') as output_file: template.render_context(Context(output_file, **args)) - if not got_input: + if not got_input and not preprocessed_output: out('Got nothing to do') showhelp() - output_file.close() - if __name__ == '__main__': main(sys.argv[1:]) |