From 3f0d3b68a273250b1bf70c278d0700a1ed30058c Mon Sep 17 00:00:00 2001 From: Jingwen Chen Date: Thu, 22 Sep 2016 17:46:49 +0000 Subject: Implementation of maven_jar rule in Skylark. **Experimental** This is an initial implementation of the maven_jar rule in Skylark, targeted at the FRs in issue #1410. Implemented a wrapper around the maven binary to pull dependencies from remote repositories into a directory under {output_base}/external. Attributes `name`, `artifact`, `repository`, `sha1` have been implemented, but not `server`. Caveat: this rule assumes that the Maven dependency is installed in the system. Hence, the maven_skylark_test integration tests are tagged with "manual" and commented out because the Bazel CI isn't configured with the Maven binary yet. Added a serve_not_found helper for 404 response tests. Usage: ``` load("@bazel_tools//tools/build_defs/repo:maven_rules.bzl", "maven_jar") maven_jar( name = "com_google_guava_guava", artifact = "com.google.guava:guava:18.0", sha1 = "cce0823396aa693798f8882e64213b1772032b09", repository = "http://uk.maven.org/maven2", ) ``` With regards to server, there are some limitations with retrieving a maven_server's attribute at Loading Phase without the use of hacky macros (issue #1704), and even if macros are used, the maven_server is not treated as an actual dependency by maven_jar. There is a test (`test_unimplemented_server_attr`) to ensure that the error message to shown to users if they use the server attribute with this rule. -- Change-Id: I167f9d13835c30be971928b4cc60167a8e396893 Reviewed-on: https://bazel-review.googlesource.com/c/5770 MOS_MIGRATED_REVID=133971809 --- WORKSPACE | 5 + src/test/shell/bazel/BUILD | 12 + src/test/shell/bazel/external_integration_test.sh | 10 +- src/test/shell/bazel/maven_skylark_test.sh | 194 ++++++++++++ src/test/shell/bazel/maven_test.sh | 15 +- src/test/shell/bazel/remote_helpers.sh | 15 + tools/build_defs/repo/maven_rules.bzl | 370 ++++++++++++++++++++++ 7 files changed, 601 insertions(+), 20 deletions(-) create mode 100755 src/test/shell/bazel/maven_skylark_test.sh create mode 100644 tools/build_defs/repo/maven_rules.bzl diff --git a/WORKSPACE b/WORKSPACE index ec8dff751f..b06d57f180 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -52,3 +52,8 @@ bind(name = "xcrunwrapper", actual = "@bazel_tools//tools/objc:xcrunwrapper") bind(name = "protobuf/java_runtime", actual = "//third_party/protobuf:protobuf") bind(name = "protobuf/javalite_runtime", actual = "//third_party/protobuf:protobuf-lite") + +# For Skylark tests at //src/test/shell/bazel:maven_skylark_test +# Uncomment the following lines, and the test in src/test/shell/bazel/BUILD to run it. +# load("//tools/build_defs/repo:maven_rules.bzl", "maven_dependency_plugin") +# maven_dependency_plugin() diff --git a/src/test/shell/bazel/BUILD b/src/test/shell/bazel/BUILD index b9a406d174..115474149c 100644 --- a/src/test/shell/bazel/BUILD +++ b/src/test/shell/bazel/BUILD @@ -279,6 +279,18 @@ sh_test( data = [":test-deps"], ) +# To run this test, ensure that the maven_dependency_plugin() in WORKSPACE is uncommented as well. +sh_test( + name = "maven_skylark_test", + size = "medium", + srcs = ["maven_skylark_test.sh"], + data = [ + ":test-deps", + "@m2//:files", + ], + tags = ["manual"], +) + sh_test( name = "generate_workspace_test", size = "large", diff --git a/src/test/shell/bazel/external_integration_test.sh b/src/test/shell/bazel/external_integration_test.sh index 4873710775..33f2573c31 100755 --- a/src/test/shell/bazel/external_integration_test.sh +++ b/src/test/shell/bazel/external_integration_test.sh @@ -349,15 +349,7 @@ EOF } function test_http_404() { - http_response=$TEST_TMPDIR/http_response - cat > $http_response < WORKSPACE <&2; exit 1; } +source $src/remote_helpers.sh \ + || { echo "remote_helpers.sh not found!" >&2; exit 1; } + +function setup_zoo() { + mkdir -p zoo + cat > zoo/BUILD < zoo/BallPit.java < WORKSPACE <& $TEST_log || fail "Expected run to succeed" + expect_log "Tra-la!" +} + +# Same as test_maven_jar, except omit sha1 implying "we don't care". +function test_maven_jar_no_sha1_skylark() { + setup_zoo + version="1.22" + serve_artifact com.example.carnivore carnivore $version + + cat > WORKSPACE <& $TEST_log || fail "Expected run to succeed" + expect_log "Tra-la!" +} + +function test_maven_jar_404_skylark() { + setup_zoo + version="1.23" + serve_not_found + + cat > WORKSPACE <& $TEST_log && echo "Expected build to fail" + kill_nc + expect_log "Failed to fetch Maven dependency" +} + +function test_maven_jar_mismatched_sha1_skylark() { + setup_zoo + version="1.24" + serve_artifact com.example.carnivore carnivore 1.24 + + wrong_sha1="0123456789012345678901234567890123456789" + cat > WORKSPACE <& $TEST_log && echo "Expected fetch to fail" + expect_log "has SHA-1 of $sha1, does not match expected SHA-1 ($wrong_sha1)" +} + +function test_unimplemented_server_attr_skylark() { + setup_zoo + version="1.25" + serve_jar + + cat > WORKSPACE <& $TEST_log && echo "Expected build to fail" + kill_nc + expect_log "specifies a 'server' attribute which is currently not supported." +} + +run_suite "maven skylark tests" diff --git a/src/test/shell/bazel/maven_test.sh b/src/test/shell/bazel/maven_test.sh index c3d9293a33..9768bdab0e 100755 --- a/src/test/shell/bazel/maven_test.sh +++ b/src/test/shell/bazel/maven_test.sh @@ -1,6 +1,6 @@ #!/bin/bash # -# Copyright 2015 The Bazel Authors. All rights reserved. +# Copyright 2016 The Bazel Authors. All rights reserved. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -87,14 +87,8 @@ EOF function test_maven_jar_404() { setup_zoo - http_response=$TEST_TMPDIR/http_response - cat > $http_response < WORKSPACE < WORKSPACE <& $TEST_log && echo "Expected fetch to fail" - kill_nc expect_log "has SHA-1 of $sha1, does not match expected SHA-1 ($wrong_sha1)" } diff --git a/src/test/shell/bazel/remote_helpers.sh b/src/test/shell/bazel/remote_helpers.sh index 87732339ef..ce701d0801 100755 --- a/src/test/shell/bazel/remote_helpers.sh +++ b/src/test/shell/bazel/remote_helpers.sh @@ -94,6 +94,21 @@ EOF redirect_pid=$! } +# Serves a HTTP 404 Not Found response with an optional parameter for the +# response body. +function serve_not_found() { + RESPONSE_BODY=${1:-} + http_response=$TEST_TMPDIR/http_response + cat > $http_response < + + {localRepository} + + + central + {mirror} + *,default + + + +""".format( + rule_name = ctx.name, + mirror = repository, # Required because the Bazel test environment has no internet access + localRepository = paths.m2_repo_dir, + ) + + # Overwrite default settings file + ctx.file("%s" % paths.m2_settings_path, _SETTINGS_XML) + + +def _file_exists(ctx, filename): + return _execute(ctx, "[[ -f %s ]] && exit 0 || exit 1" % filename).return_code == 0 + + +# Constructs the maven command to retrieve the dependencies from remote +# repositories using the dependency plugin, and executes it. +def _mvn_download(ctx, paths, fully_qualified_name): + repository = ctx.attr.repository + if repository == "": + repository = MAVEN_CENTRAL_URL + + # If a custom settings file exists, we'll use that. If not, Maven will use the default settings. + mvn_flags = "" + if _file_exists(ctx, paths.m2_settings_path): + mvn_flags += "-s %s" % paths.m2_settings_path + + # dependency:get step. Downloads the artifact into the local repository. + mvn_get = MVN_PLUGIN + ":get" + mvn_artifact = "-Dartifact=%s" % fully_qualified_name + mvn_transitive = "-Dtransitive=false" + mvn_remote_repo = "-Dmaven.repo.remote=%s" % repository + command = " ".join(["mvn", mvn_flags, mvn_get, mvn_transitive, mvn_remote_repo, mvn_artifact]) + exec_result = _execute(ctx, command) + if exec_result.return_code != 0: + fail("%s\n%s\nFailed to fetch Maven dependency" % (exec_result.stdout, exec_result.stderr)) + + # dependency:copy step. Moves the artifact from the local repository into //external. + mvn_copy = MVN_PLUGIN + ":copy" + mvn_output_dir = "-DoutputDirectory=%s" % paths.jar_dir + command = " ".join(["mvn", mvn_flags, mvn_copy, mvn_artifact, mvn_output_dir]) + exec_result = _execute(ctx, command) + if exec_result.return_code != 0: + fail("%s\n%s\nFailed to fetch Maven dependency" % (exec_result.stdout, exec_result.stderr)) + + +def _check_sha1(ctx, paths, sha1): + actual_sha1 = _execute(ctx, "openssl sha1 %s | awk '{printf $2}'" % paths.jar_path).stdout + + if sha1.lower() != actual_sha1.lower(): + fail(("{rule_name} has SHA-1 of {actual_sha1}, " + + "does not match expected SHA-1 ({expected_sha1})").format( + rule_name = ctx.name, + expected_sha1 = sha1, + actual_sha1 = actual_sha1)) + else: + _execute(ctx, "echo %s %s > %s" % (sha1, paths.jar_path, paths.sha1_path)) + + +def _maven_jar_impl(ctx): + # Ensure that we have all of the dependencies installed + _check_dependencies(ctx) + + # Provide warnings and errors about attributes + _validate_attr(ctx) + + # Create a struct to contain the different parts of the artifact FQN + coordinates = _create_coordinates(ctx.attr.artifact) + + # Create a struct to store the relative and absolute paths needed for this rule + paths = _create_paths(ctx, coordinates) + + _generate_build_file( + ctx = ctx, + paths = paths, + ) + + # Initialize local settings.xml files and symlink the dependency plugin + # artifact to the local repository + if ctx.attr.local_repository: + _mvn_init( + ctx = ctx, + paths = paths, + repository = ctx.attr.repository + ) + + if _execute(ctx, "mkdir -p %s" % paths.symlink_dir).return_code != 0: + fail("%s: Failed to create dirs in execution root.\n" % ctx.name) + + # Download the artifact + _mvn_download( + ctx = ctx, + paths = paths, + fully_qualified_name = coordinates.fully_qualified_name + ) + + if (ctx.attr.sha1 != ""): + _check_sha1( + ctx = ctx, + paths = paths, + sha1 = ctx.attr.sha1, + ) + + ctx.symlink(paths.jar_path, paths.symlink_jar_path) + + +_maven_jar_attrs = { + "artifact": attr.string( + default = "", + mandatory = True, + ), + "repository": attr.string(default = ""), + "server": attr.label(default = None), + "sha1": attr.string(default = ""), + "local_repository": attr.label( + default = None, + allow_single_file = True, + ) +} + + +maven_jar = repository_rule( + implementation=_maven_jar_impl, + attrs=_maven_jar_attrs, + local=False, +) + + +def _maven_dependency_plugin_impl(ctx): + _BUILD_FILE = """ +# DO NOT EDIT: automatically generated BUILD file for maven_dependency_plugin + +filegroup( + name = 'files', + srcs = glob(['**']), + visibility = ['//visibility:public'] +) +""" + ctx.file("BUILD", _BUILD_FILE, False) + + _SETTINGS_XML = """ + + + {localRepository} + + + central + {mirror} + *,default + + + +""".format( + localRepository = ctx.path("repository"), + mirror = MAVEN_CENTRAL_URL, + ) + settings_path = ctx.path("settings.xml") + ctx.file("%s" % settings_path, _SETTINGS_XML, False) + + # Download the plugin with transitive dependencies + mvn_flags = "-s %s" % settings_path + mvn_get = MVN_PLUGIN + ":get" + mvn_artifact = "-Dartifact=%s" % MVN_PLUGIN + command = " ".join(["mvn", mvn_flags, mvn_get, mvn_artifact]) + + exec_result = _execute(ctx, command) + if exec_result.return_code != 0: + fail("%s\nFailed to fetch Maven dependency" % exec_result.stderr) + + +_maven_dependency_plugin = repository_rule( + implementation=_maven_dependency_plugin_impl, +) + + +def maven_dependency_plugin(): + _maven_dependency_plugin(name = "m2") -- cgit v1.2.3