Merge pull request #16088 from mehrdada/refactor-grpc_publish_packages

Refactor and enhance grpc_publish_packages.sh

1 files changed, 98 insertions, 64 deletions

diff --git a/tools/internal_ci/linux/grpc_publish_packages.sh b/tools/internal_ci/linux/grpc_publish_packages.sh
index 89da36987e..24bb4880ed 100644
--- a/tools/internal_ci/linux/grpc_publish_packages.sh
+++ b/tools/internal_ci/linux/grpc_publish_packages.sh
@@ -17,94 +17,128 @@ set -ex
 
 shopt -s nullglob
 
-export GOOGLE_APPLICATION_CREDENTIALS=${KOKORO_GFILE_DIR}/GrpcTesting-d0eeee2db331.json
-
-GCS_ROOT=gs://packages.grpc.io
-MANIFEST_FILE=index.xml
-ARCHIVE_UUID=${KOKORO_BUILD_ID:-$(uuidgen)}
-GIT_BRANCH_NAME=master #${KOKORO_GITHUB_COMMIT:-master}
-GIT_COMMIT=${KOKORO_GIT_COMMIT:-unknown}
-ARCHIVE_TIMESTAMP=$(date -Iseconds)
-TARGET_DIR=$(mktemp -d grpc_publish_packages.sh.XXXX)
-YEAR_MONTH_PREFIX=$(date "+%Y/%m")
-YEAR_PREFIX=${YEAR_MONTH_PREFIX%%/*}
-UPLOAD_ROOT=$TARGET_DIR/$YEAR_PREFIX
-RELATIVE_PATH=$YEAR_MONTH_PREFIX/$ARCHIVE_UUID
-BUILD_ROOT=$TARGET_DIR/$RELATIVE_PATH
-
-LINUX_PACKAGES=$KOKORO_GFILE_DIR/github/grpc/artifacts
-WINDOWS_PACKAGES=$KOKORO_GFILE_DIR/github/grpc/artifacts
-# TODO(mmx): enable linux_extra
-# LINUX_EXTRA_PACKAGES=$KOKORO_GFILE_DIR/github/grpc/artifacts
+INPUT_ARTIFACTS=$KOKORO_GFILE_DIR/github/grpc/artifacts
+INDEX_FILENAME=index.xml
+
+BUILD_ID=${KOKORO_BUILD_ID:-$(uuidgen)}
+BUILD_BRANCH_NAME=master
+BUILD_GIT_COMMIT=${KOKORO_GIT_COMMIT:-unknown}
+BUILD_TIMESTAMP=$(date -Iseconds)
+BUILD_RELPATH=$(date "+%Y/%m")/$BUILD_ID/
+
+GCS_ROOT=gs://packages.grpc.io/
+GCS_ARCHIVE_PREFIX=archive/
+GCS_ARCHIVE_ROOT=$GCS_ROOT$GCS_ARCHIVE_PREFIX
+GCS_INDEX=$GCS_ROOT$INDEX_FILENAME
+
+LOCAL_STAGING_TEMPDIR=$(mktemp -d)
+LOCAL_BUILD_ROOT=$LOCAL_STAGING_TEMPDIR/$BUILD_RELPATH
+LOCAL_BUILD_INDEX=$LOCAL_BUILD_ROOT$INDEX_FILENAME
+
+mkdir -p "$LOCAL_BUILD_ROOT"
+
+find "$INPUT_ARTIFACTS" -type f
+
+UNZIPPED_CSHARP_PACKAGES=$(mktemp -d)
+unzip "$INPUT_ARTIFACTS/csharp_nugets_windows_dotnetcli.zip" -d "$UNZIPPED_CSHARP_PACKAGES"
+CSHARP_PACKAGES=(
+  "$UNZIPPED_CSHARP_PACKAGES"/*
+)
 
 PYTHON_PACKAGES=(
-  "$LINUX_PACKAGES"/grpcio-[0-9]*.whl
-  "$LINUX_PACKAGES"/grpcio-[0-9]*.tar.gz
-  "$LINUX_PACKAGES"/grpcio_tools-[0-9]*.whl
-  "$LINUX_PACKAGES"/grpcio-tools-[0-9]*.tar.gz
-  "$LINUX_PACKAGES"/grpcio-health-checking-[0-9]*.tar.gz
-  "$LINUX_PACKAGES"/grpcio-reflection-[0-9]*.tar.gz
-  "$LINUX_PACKAGES"/grpcio-testing-[0-9]*.tar.gz
-  #"$LINUX_EXTRA_PACKAGES"/grpcio-[0-9]*.whl
-  #"$LINUX_EXTRA_PACKAGES"/grpcio_tools-[0-9]*.whl
+  "$INPUT_ARTIFACTS"/grpcio-[0-9]*.tar.gz
+  "$INPUT_ARTIFACTS"/grpcio-[0-9]*.whl
+  "$INPUT_ARTIFACTS"/python_linux_extra_arm*/grpcio-[0-9]*.whl
+
+  "$INPUT_ARTIFACTS"/grpcio-tools-[0-9]*.tar.gz
+  "$INPUT_ARTIFACTS"/grpcio_tools-[0-9]*.whl
+  "$INPUT_ARTIFACTS"/python_linux_extra_arm*/grpcio_tools-[0-9]*.whl
+
+  "$INPUT_ARTIFACTS"/grpcio-health-checking-[0-9]*.tar.gz
+  "$INPUT_ARTIFACTS"/grpcio-reflection-[0-9]*.tar.gz
+  "$INPUT_ARTIFACTS"/grpcio-testing-[0-9]*.tar.gz
 )
 
 PHP_PACKAGES=(
-  "$LINUX_PACKAGES"/grpc-[0-9]*.tgz
+  "$INPUT_ARTIFACTS"/grpc-[0-9]*.tgz
 )
 
 RUBY_PACKAGES=(
-  "$LINUX_PACKAGES"/grpc-[0-9]*.gem
-  "$LINUX_PACKAGES"/grpc-tools-[0-9]*.gem
-)
-
-CSHARP_PACKAGES=(
-  "$WINDOWS_PACKAGES"/csharp_nugets_windows_dotnetcli.zip
+  "$INPUT_ARTIFACTS"/grpc-[0-9]*.gem
+  "$INPUT_ARTIFACTS"/grpc-tools-[0-9]*.gem
 )
 
 function add_to_manifest() {
-  local xml_type=$1
-  local xml_name
-  xml_name=$(basename "$2")
-  local xml_sha256
-  xml_sha256=$(openssl sha256 -r "$2" | cut -d " " -f 1)
-  cp "$2" "$BUILD_ROOT"
-  echo "<artifact type='$xml_type' name='$xml_name' sha256='$xml_sha256' />"
+  local artifact_type=$1
+  local artifact_file=$2
+  local artifact_name
+  artifact_name=$(basename "$artifact_file")
+  local artifact_sha256
+  artifact_sha256=$(openssl sha256 -r "$artifact_file" | cut -d " " -f 1)
+  local artifact_target=$LOCAL_BUILD_ROOT/$artifact_type
+  mkdir -p "$artifact_target"
+  cp "$artifact_file" "$artifact_target"
+  cat <<EOF
+    <artifact name='$artifact_name'
+              type='$artifact_type'
+              path='$artifact_type/$artifact_name'
+              sha256='$artifact_sha256' />
+EOF
 }
 
-mkdir -p "$BUILD_ROOT"
-
 {
   cat <<EOF
 <?xml version="1.0"?>
-<?xml-stylesheet href="/web-assets/build.xsl" type="text/xsl"?>
+<?xml-stylesheet href="/web-assets/build-201807.xsl" type="text/xsl"?>
+<build id='$BUILD_ID' timestamp='$BUILD_TIMESTAMP'>
+  <metadata>
+    <project>gRPC</project>
+    <repository>https://github.com/grpc/grpc</repository>
+    <branch>$BUILD_BRANCH_NAME</branch>
+    <commit>$BUILD_GIT_COMMIT</commit>
+  </metadata>
+  <artifacts>
 EOF
-  echo "<build id='$ARCHIVE_UUID' timestamp='$ARCHIVE_TIMESTAMP'>"
-  echo "<metadata>"
-  echo "<branch>$GIT_BRANCH_NAME</branch>"
-  echo "<commit>$GIT_COMMIT</commit>"
-  echo "</metadata><artifacts>"
 
-  for pkg in "${PYTHON_PACKAGES[@]}"; do add_to_manifest python "$pkg"; done
   for pkg in "${CSHARP_PACKAGES[@]}"; do add_to_manifest csharp "$pkg"; done
   for pkg in "${PHP_PACKAGES[@]}"; do add_to_manifest php "$pkg"; done
+  for pkg in "${PYTHON_PACKAGES[@]}"; do add_to_manifest python "$pkg"; done
   for pkg in "${RUBY_PACKAGES[@]}"; do add_to_manifest ruby "$pkg"; done
 
-  echo "</artifacts></build>"
-}> "$BUILD_ROOT/$MANIFEST_FILE"
+  cat <<EOF
+  </artifacts>
+</build>
+EOF
+}> "$LOCAL_BUILD_INDEX"
+
+LOCAL_BUILD_INDEX_SHA256=$(openssl sha256 -r "$LOCAL_BUILD_INDEX" | cut -d " " -f 1)
 
-BUILD_XML_SHA=$(openssl sha256 -r "$BUILD_ROOT/$MANIFEST_FILE" | cut -d " " -f 1)
+OLD_INDEX=$(mktemp)
+NEW_INDEX=$(mktemp)
 
-PREV_HOME=$(mktemp old-XXXXX-$MANIFEST_FILE)
-NEW_HOME=$(mktemp new-XXXXX-$MANIFEST_FILE)
-gsutil cp "$GCS_ROOT/$MANIFEST_FILE" "$PREV_HOME"
+# Download the current /index.xml into $OLD_INDEX
+gsutil cp "$GCS_INDEX" "$OLD_INDEX"
 
 {
-  head --lines=4 "$PREV_HOME"
-  echo "<build id='$ARCHIVE_UUID' timestamp='$ARCHIVE_TIMESTAMP' branch='$GIT_BRANCH_NAME' commit='$GIT_COMMIT' manifest='archive/$RELATIVE_PATH/$MANIFEST_FILE' manifest-sha256='$BUILD_XML_SHA' />"
-  tail --lines=+5 "$PREV_HOME"
-}> "$NEW_HOME"
-
-gsutil -m cp -r "$UPLOAD_ROOT" "$GCS_ROOT/archive"
-gsutil -h "Content-Type:application/xml" cp "$NEW_HOME" "$GCS_ROOT/$MANIFEST_FILE"
+  # we want to add an entry as the first child under <builds> tag
+  # we can get by without a real XML parser by rewriting the header,
+  # injecting our new tag, and then dumping the rest of the file as is.
+  cat <<EOF
+<?xml version="1.0"?>
+<?xml-stylesheet href="/web-assets/home.xsl" type="text/xsl"?>
+<packages>
+  <builds>
+    <build id='$BUILD_ID'
+           timestamp='$BUILD_TIMESTAMP'
+           branch='$BUILD_BRANCH_NAME'
+           commit='$BUILD_GIT_COMMIT'
+           path='$GCS_ARCHIVE_PREFIX$BUILD_RELPATH$INDEX_FILENAME'
+           sha256='$LOCAL_BUILD_INDEX_SHA256' />
+EOF
+  tail --lines=+5 "$OLD_INDEX"
+}> "$NEW_INDEX"
 
+# Upload the current build artifacts
+gsutil -m cp -r "$LOCAL_STAGING_TEMPDIR/${BUILD_RELPATH%%/*}" "$GCS_ARCHIVE_ROOT"
+# Upload the new /index.xml
+gsutil -h "Content-Type:application/xml" cp "$NEW_INDEX" "$GCS_INDEX"