diff options
author | 2018-07-23 11:58:01 -0700 | |
---|---|---|
committer | 2018-07-23 11:58:01 -0700 | |
commit | 2eaba913d9bcc6baac813d3c9fb26e4939c3d7dd (patch) | |
tree | d06228cc90647cdf87d9ab9641c4d60fca34d58e | |
parent | e08b9b79b8facf947c6ff46633fe140fc1c6aee5 (diff) | |
parent | efe02c0dfb575ec4c0e13fd152c01c70d45df995 (diff) |
Merge pull request #16100 from mehrdada/master
Enhance nightly builds infrastructure
-rw-r--r-- | tools/internal_ci/linux/grpc_publish_packages.sh | 100 |
1 files changed, 93 insertions, 7 deletions
diff --git a/tools/internal_ci/linux/grpc_publish_packages.sh b/tools/internal_ci/linux/grpc_publish_packages.sh index 24bb4880ed..fecb9a5e09 100644 --- a/tools/internal_ci/linux/grpc_publish_packages.sh +++ b/tools/internal_ci/linux/grpc_publish_packages.sh @@ -39,30 +39,57 @@ mkdir -p "$LOCAL_BUILD_ROOT" find "$INPUT_ARTIFACTS" -type f +# protoc Plugins +PROTOC_PLUGINS_ZIPPED_PACKAGES=$(mktemp -d) +for zip_dir in protoc_windows_{x86,x64} +do + zip -jr "$PROTOC_PLUGINS_ZIPPED_PACKAGES/$zip_dir.zip" "$INPUT_ARTIFACTS/$zip_dir/"* +done +for tar_dir in protoc_{linux,macos}_{x86,x64} +do + chmod +x "$INPUT_ARTIFACTS/$tar_dir"/* + tar -cvzf "$PROTOC_PLUGINS_ZIPPED_PACKAGES/$tar_dir.tar.gz" -C "$INPUT_ARTIFACTS/$tar_dir" . +done + +PROTOC_PACKAGES=( + "$PROTOC_PLUGINS_ZIPPED_PACKAGES"/protoc_windows_{x86,x64}.zip + "$PROTOC_PLUGINS_ZIPPED_PACKAGES"/protoc_{linux,macos}_{x86,x64}.tar.gz +) + +# C# UNZIPPED_CSHARP_PACKAGES=$(mktemp -d) unzip "$INPUT_ARTIFACTS/csharp_nugets_windows_dotnetcli.zip" -d "$UNZIPPED_CSHARP_PACKAGES" CSHARP_PACKAGES=( "$UNZIPPED_CSHARP_PACKAGES"/* ) -PYTHON_PACKAGES=( +# Python +PYTHON_GRPCIO_PACKAGES=( "$INPUT_ARTIFACTS"/grpcio-[0-9]*.tar.gz "$INPUT_ARTIFACTS"/grpcio-[0-9]*.whl "$INPUT_ARTIFACTS"/python_linux_extra_arm*/grpcio-[0-9]*.whl - +) +PYTHON_GRPCIO_TOOLS_PACKAGES=( "$INPUT_ARTIFACTS"/grpcio-tools-[0-9]*.tar.gz "$INPUT_ARTIFACTS"/grpcio_tools-[0-9]*.whl "$INPUT_ARTIFACTS"/python_linux_extra_arm*/grpcio_tools-[0-9]*.whl - +) +PYTHON_GRPCIO_HEALTH_CHECKING_PACKAGES=( "$INPUT_ARTIFACTS"/grpcio-health-checking-[0-9]*.tar.gz +) +PYTHON_GRPCIO_REFLECTION_PACKAGES=( "$INPUT_ARTIFACTS"/grpcio-reflection-[0-9]*.tar.gz +) +PYTHON_GRPCIO_TESTING_PACKAGES=( "$INPUT_ARTIFACTS"/grpcio-testing-[0-9]*.tar.gz ) +# PHP PHP_PACKAGES=( "$INPUT_ARTIFACTS"/grpc-[0-9]*.tgz ) +# Ruby RUBY_PACKAGES=( "$INPUT_ARTIFACTS"/grpc-[0-9]*.gem "$INPUT_ARTIFACTS"/grpc-tools-[0-9]*.gem @@ -71,17 +98,21 @@ RUBY_PACKAGES=( function add_to_manifest() { local artifact_type=$1 local artifact_file=$2 + local artifact_prefix=$3 local artifact_name artifact_name=$(basename "$artifact_file") + local artifact_size + artifact_size=$(stat -c%s "$artifact_file") local artifact_sha256 artifact_sha256=$(openssl sha256 -r "$artifact_file" | cut -d " " -f 1) - local artifact_target=$LOCAL_BUILD_ROOT/$artifact_type + local artifact_target=$LOCAL_BUILD_ROOT/$artifact_type/$artifact_prefix mkdir -p "$artifact_target" cp "$artifact_file" "$artifact_target" cat <<EOF <artifact name='$artifact_name' type='$artifact_type' - path='$artifact_type/$artifact_name' + path='$artifact_type/$artifact_prefix$artifact_name' + size='$artifact_size' sha256='$artifact_sha256' /> EOF } @@ -90,7 +121,7 @@ EOF cat <<EOF <?xml version="1.0"?> <?xml-stylesheet href="/web-assets/build-201807.xsl" type="text/xsl"?> -<build id='$BUILD_ID' timestamp='$BUILD_TIMESTAMP'> +<build id='$BUILD_ID' timestamp='$BUILD_TIMESTAMP' version="201807"> <metadata> <project>gRPC</project> <repository>https://github.com/grpc/grpc</repository> @@ -100,9 +131,14 @@ EOF <artifacts> EOF + for pkg in "${PROTOC_PACKAGES[@]}"; do add_to_manifest protoc "$pkg"; done for pkg in "${CSHARP_PACKAGES[@]}"; do add_to_manifest csharp "$pkg"; done for pkg in "${PHP_PACKAGES[@]}"; do add_to_manifest php "$pkg"; done - for pkg in "${PYTHON_PACKAGES[@]}"; do add_to_manifest python "$pkg"; done + for pkg in "${PYTHON_GRPCIO_PACKAGES[@]}"; do add_to_manifest python "$pkg" grpcio/; done + for pkg in "${PYTHON_GRPCIO_TOOLS_PACKAGES[@]}"; do add_to_manifest python "$pkg" grpcio-tools/; done + for pkg in "${PYTHON_GRPCIO_HEALTH_CHECKING_PACKAGES[@]}"; do add_to_manifest python "$pkg" grpcio-health-checking/; done + for pkg in "${PYTHON_GRPCIO_REFLECTION_PACKAGES[@]}"; do add_to_manifest python "$pkg" grpcio-reflection/; done + for pkg in "${PYTHON_GRPCIO_TESTING_PACKAGES[@]}"; do add_to_manifest python "$pkg" grpcio-testing/; done for pkg in "${RUBY_PACKAGES[@]}"; do add_to_manifest ruby "$pkg"; done cat <<EOF @@ -111,6 +147,7 @@ EOF EOF }> "$LOCAL_BUILD_INDEX" +LOCAL_BUILD_INDEX_SIZE=$(stat -c%s "$LOCAL_BUILD_INDEX") LOCAL_BUILD_INDEX_SHA256=$(openssl sha256 -r "$LOCAL_BUILD_INDEX" | cut -d " " -f 1) OLD_INDEX=$(mktemp) @@ -133,12 +170,61 @@ gsutil cp "$GCS_INDEX" "$OLD_INDEX" branch='$BUILD_BRANCH_NAME' commit='$BUILD_GIT_COMMIT' path='$GCS_ARCHIVE_PREFIX$BUILD_RELPATH$INDEX_FILENAME' + size='$LOCAL_BUILD_INDEX_SIZE' sha256='$LOCAL_BUILD_INDEX_SHA256' /> EOF tail --lines=+5 "$OLD_INDEX" }> "$NEW_INDEX" + +function generate_directory_index() +{ + local target_dir=$1 + local current_directory_name + current_directory_name=$(basename "$target_dir") + cat <<EOF +<!DOCTYPE html> +<html xmlns="http://www.w3.org/1999/xhtml" lang="" xml:lang=""> + <head> + <meta http-equiv="Content-Type" content="text/html; charset=UTF-8" /> + <title>Index of $current_directory_name - packages.grpc.io</title> + <link rel="stylesheet" type="text/css" href="/web-assets/dirindex.css" /> + </head> + <body> + <h1>Index of <a href="#"><code>$current_directory_name</code></a></h1> + <ul> + <li><a href="#">.</a></li> + <li><a href="..">..</a></li> +EOF + +( + cd "$target_dir" + find * -maxdepth 0 -type d -print | sort | while read -r line + do + echo " <li><a href='$line/'>$line/</a></li>" + done + find * -maxdepth 0 -type f -print | sort | while read -r line + do + echo " <li><a href='$line'>$line</a></li>" + done +) + +cat <<EOF + </ul> + </body> +</html> +EOF +} + # Upload the current build artifacts gsutil -m cp -r "$LOCAL_STAGING_TEMPDIR/${BUILD_RELPATH%%/*}" "$GCS_ARCHIVE_ROOT" +# Upload directory indicies for subdirectories +( + cd "$LOCAL_BUILD_ROOT" + find * -type d | while read -r directory + do + generate_directory_index "$directory" | gsutil -h 'Content-Type:text/html' cp - "$GCS_ARCHIVE_ROOT$BUILD_RELPATH$directory/$INDEX_FILENAME" + done +) # Upload the new /index.xml gsutil -h "Content-Type:application/xml" cp "$NEW_INDEX" "$GCS_INDEX" |