Update 'docker_build' to support 1.10 image format

Docker 1.10 updated the format of images moving layers to just being
tarballs referenced by a configuration file. A new manifest.json file
aggregates images and handles parent and tagging references. Layers and
images are now identified by their sha256 hash. An image configuration
file must reference all layers that belong to it by this identifier,
including all layers in any parent images.

Image configuration is generated the same way but now allows multiple
layer sha256 hashes to be provided. The base image configuration is read
to find config defaults and the layer identifiers that need to be
present.

Image creation now requires the layer identifier and file and can accept
multiple layers. A manifest with a single entry is created that points
at the image configuration, its layers and tags. If a base image is
provided its layers are added to the begining of the layer section and a
parent reference to the base image is added. Multiple tags can be
provided which are applied when the image is loaded.

The joining of partial images now consists of merging their contents
minus the manifest which is concatentated together.

These changes have been made in a backwards compatible way so versions
of docker below 1.10 will still work as before.

Fixes #1113

--
Change-Id: I0075decc48d8846ad16431948192db196ad702ee
Reviewed-on: https://bazel-review.googlesource.com/3730
MOS_MIGRATED_REVID=124339578

1 files changed, 120 insertions, 44 deletions

diff --git a/tools/build_defs/docker/create_image.py b/tools/build_defs/docker/create_image.py
index 040f1b907e..5dda217765 100644
--- a/tools/build_defs/docker/create_image.py
+++ b/tools/build_defs/docker/create_image.py
@@ -13,9 +13,12 @@
 # limitations under the License.
 """This tool creates a docker image from a layer and the various metadata."""
 
+import json
+import re
 import sys
 import tarfile
 
+from tools.build_defs.docker import utils
 from tools.build_defs.pkg import archive
 from third_party.py import gflags
 
@@ -27,24 +30,30 @@ gflags.DEFINE_string(
     'The output file, mandatory')
 gflags.MarkFlagAsRequired('output')
 
-gflags.DEFINE_string(
-    'metadata', None,
-    'The JSON metadata file for this image, mandatory.')
-gflags.MarkFlagAsRequired('metadata')
-
-gflags.DEFINE_string(
-    'layer', None,
-    'The tar file for the top layer of this image, mandatory.')
-gflags.MarkFlagAsRequired('layer')
+gflags.DEFINE_multistring(
+    'layer', [],
+    'Layer tar files and their identifiers that make up this image')
 
 gflags.DEFINE_string(
     'id', None,
     'The hex identifier of this image (hexstring or @filename), mandatory.')
 gflags.MarkFlagAsRequired('id')
 
+gflags.DEFINE_string('config', None,
+                     'The JSON configuration file for this image, mandatory.')
+gflags.MarkFlagAsRequired('config')
+
+gflags.DEFINE_string('base', None, 'The base image file for this image.')
+
 gflags.DEFINE_string(
-    'base', None,
-    'The base image file for this image.')
+    'legacy_id', None,
+    'The legacy hex identifier of this layer (hexstring or @filename).')
+
+gflags.DEFINE_string('metadata', None,
+                     'The legacy JSON metadata file for this layer.')
+
+gflags.DEFINE_string('legacy_base', None,
+                     'The legacy base image file for this image.')
 
 gflags.DEFINE_string(
     'repository', None,
@@ -54,49 +63,108 @@ gflags.DEFINE_string(
     'name', None,
     'The symbolic name of this image.')
 
+gflags.DEFINE_multistring('tag', None,
+                          'The repository tags to apply to the image')
+
 FLAGS = gflags.FLAGS
 
 
 def _base_name_filter(name):
   """Do not add multiple times 'top' and 'repositories' when merging images."""
-  filter_names = ['top', 'repositories']
+  filter_names = ['top', 'repositories', 'manifest.json']
   return all([not name.endswith(s) for s in filter_names])
 
 
-def create_image(output, identifier,
-                 base=None, layer=None, metadata=None,
-                 name=None, repository=None):
+def create_image(output,
+                 identifier,
+                 layers,
+                 config,
+                 tags=None,
+                 base=None,
+                 legacy_base=None,
+                 metadata_id=None,
+                 metadata=None,
+                 name=None,
+                 repository=None):
   """Creates a Docker image.
 
   Args:
     output: the name of the docker image file to create.
-    identifier: the identifier of the top layer for this image.
-    base: a base layer (optional) to merge to current layer.
-    layer: the layer content (a tar file).
+    identifier: the identifier for this image (sha256 of the metadata).
+    layers: the layer content (a sha256 and a tar file).
+    config: the configuration file for the image.
+    tags: tags that apply to this image.
+    base: a base layer (optional) to build on top of.
+    legacy_base: a base layer (optional) to build on top of.
+    metadata_id: the identifier of the top layer for this image.
     metadata: the json metadata file for the top layer.
     name: symbolic name for this docker image.
     repository: repository name for this docker image.
   """
   tar = archive.TarFileWriter(output)
-  # Write our id to 'top' as we are now the topmost layer.
-  tar.add_file('top', content=identifier)
-  # Each layer is encoded as a directory in the larger tarball of the form:
-  #  {id}\
-  #    layer.tar
-  #    VERSION
-  #    json
-  # Create the directory for us to now fill in.
-  tar.add_file(identifier + '/', tarfile.DIRTYPE)
-  # VERSION generally seems to contain 1.0, not entirely sure
-  # what the point of this is.
-  tar.add_file(identifier + '/VERSION', content=DATA_FORMAT_VERSION)
-  # Add the layer file
-  tar.add_file(identifier + '/layer.tar', file_content=layer)
-  # Now the json metadata
-  tar.add_file(identifier + '/json', file_content=metadata)
-  # Merge the base if any
+
+  # add the image config referenced by the Config section in the manifest
+  # the name can be anything but docker uses the format below
+  config_file_name = identifier + '.json'
+  tar.add_file(config_file_name, file_content=config)
+
+  layer_file_names = []
+
+  if metadata_id:
+    # Write our id to 'top' as we are now the topmost layer.
+    tar.add_file('top', content=metadata_id)
+
+    # Each layer is encoded as a directory in the larger tarball of the form:
+    #  {id}\
+    #    layer.tar
+    #    VERSION
+    #    json
+    # Create the directory for us to now fill in.
+    tar.add_file(metadata_id + '/', tarfile.DIRTYPE)
+    # VERSION generally seems to contain 1.0, not entirely sure
+    # what the point of this is.
+    tar.add_file(metadata_id + '/VERSION', content=DATA_FORMAT_VERSION)
+    # Add the layer file
+    layer_file_name = metadata_id + '/layer.tar'
+    layer_file_names.append(layer_file_name)
+    tar.add_file(layer_file_name, file_content=layers[0]['layer'])
+    # Now the json metadata
+    tar.add_file(metadata_id + '/json', file_content=metadata)
+
+    # Merge the base if any
+    if legacy_base:
+      tar.add_tar(legacy_base, name_filter=_base_name_filter)
+  else:
+    for layer in layers:
+      # layers can be called anything, so just name them by their sha256
+      layer_file_name = identifier + '/' + layer['name'] + '.tar'
+      layer_file_names.append(layer_file_name)
+      tar.add_file(layer_file_name, file_content=layer['layer'])
+
+  base_layer_file_names = []
+  parent = None
   if base:
-    tar.add_tar(base, name_filter=_base_name_filter)
+    latest_item = utils.GetLatestManifestFromTar(base)
+    if latest_item:
+      base_layer_file_names = latest_item.get('Layers', [])
+      config_file = latest_item['Config']
+      parent_search = re.search('^(.+)\\.json$', config_file)
+      if parent_search:
+        parent = parent_search.group(1)
+
+  manifest_item = {
+      'Config': config_file_name,
+      'Layers': base_layer_file_names + layer_file_names,
+      'RepoTags': tags or []
+  }
+  if parent:
+    manifest_item['Parent'] = 'sha256:' + parent
+
+  manifest = [manifest_item]
+
+  manifest_content = json.dumps(manifest, sort_keys=True)
+  tar.add_file('manifest.json', content=manifest_content)
+
   # In addition to N layers of the form described above, there is
   # a single file at the top of the image called repositories.
   # This file contains a JSON blob of the form:
@@ -120,17 +188,25 @@ def create_image(output, identifier,
 # create_image --output=output_file \
 #              --id=@identifier \
 #              [--base=base] \
-#              --layer=layer.tar \
+#              --layer=@identifier=layer.tar \
 #              --metadata=metadata.json \
-#              --name=myname --repository=repositoryName
+#              --name=myname --repository=repositoryName \
+#              --tag=repo/image:tag
 # See the gflags declaration about the flags argument details.
 def main(unused_argv):
-  identifier = FLAGS.id
-  if identifier.startswith('@'):
-    with open(identifier[1:], 'r') as f:
-      identifier = f.read()
-  create_image(FLAGS.output, identifier, FLAGS.base,
-               FLAGS.layer, FLAGS.metadata,
+  identifier = utils.ExtractValue(FLAGS.id)
+  legacy_id = utils.ExtractValue(FLAGS.legacy_id)
+
+  layers = []
+  for kv in FLAGS.layer:
+    (k, v) = kv.split('=', 1)
+    layers.append({
+        'name': utils.ExtractValue(k),
+        'layer': v,
+    })
+
+  create_image(FLAGS.output, identifier, layers, FLAGS.config, FLAGS.tag,
+               FLAGS.base, FLAGS.legacy_base, legacy_id, FLAGS.metadata,
                FLAGS.name, FLAGS.repository)
 
 if __name__ == '__main__':