From a7b7aa856f34bf2e44fbeb91d817742c61483618 Mon Sep 17 00:00:00 2001 From: Yong Tang Date: Mon, 2 Jul 2018 07:41:42 -0700 Subject: Add KinesisDataset support for tensorflow Dataset (#19712) * Add KinesisDataset support for tensorflow Dataset This fix is an attempt to add Kinesis support for tensorflow's Dataset. Kinesis is provided by AWS as a managed data streaming service. It is similiar to Apache Kafka, often used in places where maintaining a independent Kafka cluster on AWS is not desirable or possible. This fix adds the Kinesis support for tensorflow Dataset. Similiar to the Kafka integration in tensorflow, KinesisDataset outputs tf.string for records. Test cases have also been added, which could be invoked manually. Signed-off-by: Yong Tang * Expose KinesisDataset in dataset_ops.cc Signed-off-by: Yong Tang * Expose KinesisDataset in python wrapper Signed-off-by: Yong Tang * Add test cases for KinesisDataset Signed-off-by: Yong Tang * Update AWS library include files Signed-off-by: Yong Tang * Add Bazel BUILD files Signed-off-by: Yong Tang * Rename s3_crypto to aws_crypto Signed-off-by: Yong Tang * Rename with_s3_support to with_aws_support Signed-off-by: Yong Tang * Selectively add kinesis to tensorflow/contrib/BUILD Signed-off-by: Yong Tang * Set different partition key and pylint fix. Signed-off-by: Yong Tang * Add missing modules in cmake's python_modules.txt Signed-off-by: Yong Tang * Address review feedback Signed-off-by: Yong Tang --- tensorflow/BUILD | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'tensorflow/BUILD') diff --git a/tensorflow/BUILD b/tensorflow/BUILD index f362900387..51eea94847 100644 --- a/tensorflow/BUILD +++ b/tensorflow/BUILD @@ -216,8 +216,8 @@ config_setting( ) config_setting( - name = "with_s3_support", - define_values = {"with_s3_support": "true"}, + name = "with_aws_support", + define_values = {"with_aws_support": "true"}, visibility = ["//visibility:public"], ) @@ -244,8 +244,8 @@ config_setting( ) config_setting( - name = "with_s3_support_windows_override", - define_values = {"with_s3_support": "true"}, + name = "with_aws_support_windows_override", + define_values = {"with_aws_support": "true"}, values = {"cpu": "x64_windows"}, visibility = ["//visibility:public"], ) @@ -279,8 +279,8 @@ config_setting( ) config_setting( - name = "with_s3_support_android_override", - define_values = {"with_s3_support": "true"}, + name = "with_aws_support_android_override", + define_values = {"with_aws_support": "true"}, values = {"crosstool_top": "//external:android/crosstool"}, visibility = ["//visibility:public"], ) @@ -300,8 +300,8 @@ config_setting( ) config_setting( - name = "with_s3_support_ios_override", - define_values = {"with_s3_support": "true"}, + name = "with_aws_support_ios_override", + define_values = {"with_aws_support": "true"}, values = {"crosstool_top": "//tools/osx/crosstool:crosstool"}, visibility = ["//visibility:public"], ) -- cgit v1.2.3