Merge changes from github.

Change: 125080305

10 files changed, 279 insertions, 4 deletions

diff --git a/tensorflow/contrib/cmake/CMakeLists.txt b/tensorflow/contrib/cmake/CMakeLists.txt
index c72a31665c..c449c803fa 100644
--- a/tensorflow/contrib/cmake/CMakeLists.txt
+++ b/tensorflow/contrib/cmake/CMakeLists.txt
@@ -43,6 +43,10 @@ include(png)
 include(jpeg)
 include(re2)
 include(eigen)
+include(jsoncpp)
+include(boringssl)
+include(farmhash)
+include(highwayhash)
 
 # Let's get to work!
 include(tf_core_framework.cmake)
@@ -54,6 +58,7 @@ include(tf_core_direct_session.cmake)
 include(tf_core_kernels.cmake)
 include(tf_cc_ops.cmake)
 include(tf_tutorials.cmake)
+include(tf_tools.cmake)
 
 if (tensorflow_BUILD_TESTS)
   include(tests.cmake)
diff --git a/tensorflow/contrib/cmake/external/boringssl.cmake b/tensorflow/contrib/cmake/external/boringssl.cmake
new file mode 100644
index 0000000000..ca86e1f2fd
--- /dev/null
+++ b/tensorflow/contrib/cmake/external/boringssl.cmake
@@ -0,0 +1,31 @@
+include (ExternalProject)
+
+set(boringssl_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/boringssl/src/boringssl/include)
+#set(boringssl_EXTRA_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/boringssl/src)
+set(boringssl_URL https://boringssl.googlesource.com/boringssl)
+set(boringssl_TAG e72df93)
+set(boringssl_BUILD ${CMAKE_BINARY_DIR}/boringssl/src/boringssl-build)
+#set(boringssl_LIBRARIES ${boringssl_BUILD}/obj/so/libboringssl.so)
+set(boringssl_STATIC_LIBRARIES
+    ${boringssl_BUILD}/ssl/libssl.a
+    ${boringssl_BUILD}/crypto/libcrypto.a
+    ${boringssl_BUILD}/decrepit/libdecrepit.a
+)
+set(boringssl_INCLUDES ${boringssl_BUILD})
+
+set(boringssl_HEADERS
+    "${boringssl_INCLUDE_DIR}/include/*.h"
+)
+
+ExternalProject_Add(boringssl
+    PREFIX boringssl
+    GIT_REPOSITORY ${boringssl_URL}
+    GIT_TAG ${boringssl_TAG}
+    DOWNLOAD_DIR "${DOWNLOAD_LOCATION}"
+    # BUILD_IN_SOURCE 1
+    INSTALL_COMMAND ""
+    CMAKE_CACHE_ARGS
+        -DCMAKE_BUILD_TYPE:STRING=Release
+        -DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF
+)
+
diff --git a/tensorflow/contrib/cmake/external/farmhash.cmake b/tensorflow/contrib/cmake/external/farmhash.cmake
new file mode 100644
index 0000000000..3cf8d5a50a
--- /dev/null
+++ b/tensorflow/contrib/cmake/external/farmhash.cmake
@@ -0,0 +1,43 @@
+include (ExternalProject)
+
+set(farmhash_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/external/farmhash_archive)
+#set(farmhash_EXTRA_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/farmhash/src)
+set(farmhash_URL https://github.com/google/farmhash/archive/34c13ddfab0e35422f4c3979f360635a8c050260.zip)
+set(farmhash_HASH SHA256=e3d37a59101f38fd58fb799ed404d630f0eee18bfc2a2433910977cc8fea9c28)
+set(farmhash_BUILD ${CMAKE_BINARY_DIR}/farmhash/src/farmhash)
+set(farmhash_INSTALL ${CMAKE_BINARY_DIR}/farmhash/install)
+#set(farmhash_LIBRARIES ${farmhash_BUILD}/obj/so/libfarmhash.so)
+set(farmhash_STATIC_LIBRARIES
+    ${farmhash_INSTALL}/lib/libfarmhash.a
+)
+set(farmhash_INCLUDES ${farmhash_BUILD})
+
+set(farmhash_HEADERS
+    "${farmhash_BUILD}/src/farmhash.h"
+)
+
+ExternalProject_Add(farmhash
+    PREFIX farmhash
+    URL ${farmhash_URL}
+    URL_HASH ${farmhash_HASH}
+    DOWNLOAD_DIR "${DOWNLOAD_LOCATION}"
+    BUILD_COMMAND $(MAKE)
+    INSTALL_COMMAND $(MAKE) install
+    CONFIGURE_COMMAND
+        ${farmhash_BUILD}/configure
+        --prefix=${farmhash_INSTALL}
+        --enable-shared=yes
+)
+
+# put farmhash includes in the directory where they are expected
+add_custom_target(farmhash_create_destination_dir
+    COMMAND ${CMAKE_COMMAND} -E make_directory ${farmhash_INCLUDE_DIR}/farmhash-34c13ddfab0e35422f4c3979f360635a8c050260/src
+    DEPENDS farmhash)
+
+add_custom_target(farmhash_copy_headers_to_destination
+    DEPENDS farmhash_create_destination_dir)
+
+foreach(header_file ${farmhash_HEADERS})
+    add_custom_command(TARGET farmhash_copy_headers_to_destination PRE_BUILD
+    COMMAND ${CMAKE_COMMAND} -E copy ${header_file} ${farmhash_INCLUDE_DIR}/farmhash-34c13ddfab0e35422f4c3979f360635a8c050260/src)
+endforeach()
diff --git a/tensorflow/contrib/cmake/external/highwayhash.cmake b/tensorflow/contrib/cmake/external/highwayhash.cmake
new file mode 100644
index 0000000000..2213534bf8
--- /dev/null
+++ b/tensorflow/contrib/cmake/external/highwayhash.cmake
@@ -0,0 +1,41 @@
+include (ExternalProject)
+
+set(highwayhash_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/external/highwayhash)
+#set(highwayhash_EXTRA_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/highwayhash/src)
+set(highwayhash_URL https://github.com/google/highwayhash.git)
+set(highwayhash_TAG be5edafc2e1a455768e260ccd68ae7317b6690ee)
+set(highwayhash_BUILD ${CMAKE_BINARY_DIR}/highwayhash/src/highwayhash)
+set(highwayhash_INSTALL ${CMAKE_BINARY_DIR}/highwayhash/install)
+#set(highwayhash_LIBRARIES ${highwayhash_BUILD}/obj/so/libhighwayhash.so)
+set(highwayhash_STATIC_LIBRARIES
+    ${highwayhash_INSTALL}/lib/libhighwayhash.a
+)
+set(highwayhash_INCLUDES ${highwayhash_BUILD})
+
+set(highwayhash_HEADERS
+    "${highwayhash_BUILD}/highwayhash/*.h"
+)
+
+ExternalProject_Add(highwayhash
+    PREFIX highwayhash
+    GIT_REPOSITORY ${highwayhash_URL}
+    GIT_TAG ${highwayhash_HASH}
+    DOWNLOAD_DIR "${DOWNLOAD_LOCATION}"
+    BUILD_IN_SOURCE 1
+    BUILD_COMMAND $(MAKE)
+    CONFIGURE_COMMAND ""
+    INSTALL_COMMAND ""
+)
+
+# put highwayhash includes in the directory where they are expected
+add_custom_target(highwayhash_create_destination_dir
+    COMMAND ${CMAKE_COMMAND} -E make_directory ${highwayhash_INCLUDE_DIR}/highwayhash
+    DEPENDS highwayhash)
+
+add_custom_target(highwayhash_copy_headers_to_destination
+    DEPENDS highwayhash_create_destination_dir)
+
+foreach(header_file ${highwayhash_HEADERS})
+    add_custom_command(TARGET highwayhash_copy_headers_to_destination PRE_BUILD
+    COMMAND ${CMAKE_COMMAND} -E copy ${header_file} ${highwayhash_INCLUDE_DIR}/highwayhash)
+endforeach()
diff --git a/tensorflow/contrib/cmake/external/jsoncpp.cmake b/tensorflow/contrib/cmake/external/jsoncpp.cmake
new file mode 100644
index 0000000000..b392f07088
--- /dev/null
+++ b/tensorflow/contrib/cmake/external/jsoncpp.cmake
@@ -0,0 +1,29 @@
+include (ExternalProject)
+
+set(jsoncpp_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/jsoncpp/src/jsoncpp)
+#set(jsoncpp_EXTRA_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/jsoncpp/src)
+set(jsoncpp_URL https://github.com/open-source-parsers/jsoncpp.git)
+set(jsoncpp_TAG 4356d9b)
+set(jsoncpp_BUILD ${CMAKE_BINARY_DIR}/jsoncpp/src/jsoncpp/src/lib_json)
+set(jsoncpp_LIBRARIES ${jsoncpp_BUILD}/obj/so/libjsoncpp.so)
+get_filename_component(jsoncpp_STATIC_LIBRARIES ${jsoncpp_BUILD}/libjsoncpp.a ABSOLUTE)
+set(jsoncpp_INCLUDES ${jsoncpp_BUILD})
+
+# We only need jsoncpp.h in external/jsoncpp/jsoncpp/jsoncpp.h
+# For the rest, we'll just add the build dir as an include dir.
+set(jsoncpp_HEADERS
+    "${jsoncpp_INCLUDE_DIR}/include/json/json.h"
+)
+
+ExternalProject_Add(jsoncpp
+    PREFIX jsoncpp
+    GIT_REPOSITORY ${jsoncpp_URL}
+    GIT_TAG ${jsoncpp_TAG}
+    DOWNLOAD_DIR "${DOWNLOAD_LOCATION}"
+    BUILD_IN_SOURCE 1
+    INSTALL_COMMAND ""
+    CMAKE_CACHE_ARGS
+        -DCMAKE_BUILD_TYPE:STRING=Release
+        -DCMAKE_VERBOSE_MAKEFILE:BOOL=OFF
+)
+
diff --git a/tensorflow/contrib/cmake/external/re2.cmake b/tensorflow/contrib/cmake/external/re2.cmake
index b96d90533e..2a193c92db 100644
--- a/tensorflow/contrib/cmake/external/re2.cmake
+++ b/tensorflow/contrib/cmake/external/re2.cmake
@@ -1,7 +1,8 @@
 include (ExternalProject)
 
-set(re2_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/external/re2/re2)
-set(re2_EXTRA_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/re2/src)
+set(re2_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/external/re2/re2
+    ${CMAKE_CURRENT_BINARY_DIR}/external/re2)
+set(re2_EXTRA_INCLUDE_DIR ${CMAKE_CURRENT_BINARY_DIR}/re2/src/re2)
 set(re2_URL https://github.com/google/re2.git)
 set(re2_TAG 791beff)
 set(re2_BUILD ${CMAKE_BINARY_DIR}/re2/src/re2)
diff --git a/tensorflow/contrib/cmake/tf_cc_ops.cmake b/tensorflow/contrib/cmake/tf_cc_ops.cmake
index 6c1da54b48..6ea9860992 100644
--- a/tensorflow/contrib/cmake/tf_cc_ops.cmake
+++ b/tensorflow/contrib/cmake/tf_cc_ops.cmake
@@ -123,6 +123,8 @@ foreach(tf_cc_op_lib_name ${tf_cc_op_lib_names})
         ${jpeg_STATIC_LIBRARIES}
         ${png_STATIC_LIBRARIES}
         ${ZLIB_LIBRARIES}
+        ${jsoncpp_STATIC_LIBRARIES}
+        ${boringssl_STATIC_LIBRARIES}
         ${CMAKE_DL_LIBS}
     )
 
diff --git a/tensorflow/contrib/cmake/tf_core_framework.cmake b/tensorflow/contrib/cmake/tf_core_framework.cmake
index 989a176fd0..78aa9169dd 100644
--- a/tensorflow/contrib/cmake/tf_core_framework.cmake
+++ b/tensorflow/contrib/cmake/tf_core_framework.cmake
@@ -35,6 +35,37 @@ function(RELATIVE_PROTOBUF_GENERATE_CPP SRCS HDRS ROOT_DIR)
   set(${HDRS} ${${HDRS}} PARENT_SCOPE)
 endfunction()
 
+function(RELATIVE_PROTOBUF_TEXT_GENERATE_CPP SRCS HDRS ROOT_DIR)
+  if(NOT ARGN)
+      message(SEND_ERROR "Error: RELATIVE_PROTOBUF_TEXT_GENERATE_CPP() called without any proto files")
+    return()
+  endif()
+
+  set(${SRCS})
+  set(${HDRS})
+  foreach(FIL ${ARGN})
+    set(ABS_FIL ${ROOT_DIR}/${FIL})
+    get_filename_component(FIL_WE ${FIL} NAME_WE)
+    get_filename_component(FIL_DIR ${ABS_FIL} PATH)
+    file(RELATIVE_PATH REL_DIR ${ROOT_DIR} ${FIL_DIR})
+
+    list(APPEND ${SRCS} "${CMAKE_CURRENT_BINARY_DIR}/${REL_DIR}/${FIL_WE}.pb_text.cc")
+    list(APPEND ${HDRS} "${CMAKE_CURRENT_BINARY_DIR}/${REL_DIR}/${FIL_WE}.pb_text.h")
+
+    add_custom_command(
+      OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/${REL_DIR}/${FIL_WE}.pb_text.cc"
+             "${CMAKE_CURRENT_BINARY_DIR}/${REL_DIR}/${FIL_WE}.pb_text.h"
+      COMMAND ${PROTO_TEXT_EXE}
+      ARGS "${CMAKE_CURRENT_BINARY_DIR}/${REL_DIR}" ${REL_DIR} ${ABS_FIL} "${ROOT_DIR}/tensorflow/tools/proto_text/placeholder.txt"
+      DEPENDS ${ABS_FIL} ${PROTO_TEXT_EXE}
+      COMMENT "Running C++ protocol buffer text compiler (${PROTO_TEXT_EXE}) on ${FIL}"
+      VERBATIM )
+  endforeach()
+
+  set_source_files_properties(${${SRCS}} ${${HDRS}} PROPERTIES GENERATED TRUE)
+  set(${SRCS} ${${SRCS}} PARENT_SCOPE)
+  set(${HDRS} ${${HDRS}} PARENT_SCOPE)
+endfunction()
 
 ########################################################
 # tf_protos_cc library
@@ -52,6 +83,37 @@ RELATIVE_PROTOBUF_GENERATE_CPP(PROTO_SRCS PROTO_HDRS
     ${tensorflow_source_dir} ${tf_protos_cc_srcs}
 )
 
+set(PROTO_TEXT_EXE "proto_text")
+set(tf_proto_text_srcs
+    "tensorflow/core/example/example.proto"
+    "tensorflow/core/example/feature.proto"
+    "tensorflow/core/framework/allocation_description.proto"
+    "tensorflow/core/framework/attr_value.proto"
+    "tensorflow/core/framework/cost_graph.proto"
+    "tensorflow/core/framework/device_attributes.proto"
+    "tensorflow/core/framework/function.proto"
+    "tensorflow/core/framework/graph.proto"
+    "tensorflow/core/framework/kernel_def.proto"
+    "tensorflow/core/framework/log_memory.proto"
+    "tensorflow/core/framework/op_def.proto"
+    "tensorflow/core/framework/step_stats.proto"
+    "tensorflow/core/framework/summary.proto"
+    "tensorflow/core/framework/tensor.proto"
+    "tensorflow/core/framework/tensor_description.proto"
+    "tensorflow/core/framework/tensor_shape.proto"
+    "tensorflow/core/framework/tensor_slice.proto"
+    "tensorflow/core/framework/types.proto"
+    "tensorflow/core/framework/versions.proto"
+    "tensorflow/core/lib/core/error_codes.proto"
+    "tensorflow/core/protobuf/config.proto"
+    "tensorflow/core/protobuf/saver.proto"
+    "tensorflow/core/util/memmapped_file_system.proto"
+    "tensorflow/core/util/saved_tensor_slice.proto"
+)
+RELATIVE_PROTOBUF_TEXT_GENERATE_CPP(PROTO_TEXT_SRCS PROTO_TEXT_HDRS
+    ${tensorflow_source_dir} ${tf_proto_text_srcs}
+)
+
 add_library(tf_protos_cc ${PROTO_SRCS} ${PROTO_HDRS})
 target_include_directories(tf_protos_cc PUBLIC
      ${CMAKE_CURRENT_BINARY_DIR}
@@ -87,6 +149,10 @@ target_include_directories(tf_core_lib PUBLIC
     ${tensorflow_source_dir}
     ${jpeg_INCLUDE_DIR}
     ${png_INCLUDE_DIR}
+    ${eigen_INCLUDE_DIRS}
+    ${re2_EXTRA_INCLUDE_DIR}
+    ${jsoncpp_INCLUDE_DIR}
+    ${boringssl_INCLUDE_DIR}
 )
 #target_link_libraries(tf_core_lib
 #    ${CMAKE_THREAD_LIBS_INIT}
@@ -109,6 +175,8 @@ add_dependencies(tf_core_lib
     re2_copy_headers_to_destination
     eigen
     tf_protos_cc
+    jsoncpp
+    boringssl
 )
 
 
@@ -136,7 +204,7 @@ file(GLOB_RECURSE tf_core_framework_test_srcs
 
 list(REMOVE_ITEM tf_core_framework_srcs ${tf_core_framework_test_srcs})
 
-add_library(tf_core_framework OBJECT ${tf_core_framework_srcs})
+add_library(tf_core_framework OBJECT ${tf_core_framework_srcs} ${PROTO_TEXT_HDRS})
 target_include_directories(tf_core_framework PUBLIC
     ${tensorflow_source_dir}
     ${eigen_INCLUDE_DIRS}
@@ -154,6 +222,7 @@ target_include_directories(tf_core_framework PUBLIC
 #)
 add_dependencies(tf_core_framework
     tf_core_lib
+    proto_text
 )
 target_compile_options(tf_core_framework PRIVATE
     -fno-exceptions
diff --git a/tensorflow/contrib/cmake/tf_core_kernels.cmake b/tensorflow/contrib/cmake/tf_core_kernels.cmake
index c8f77f625c..5a22d88103 100644
--- a/tensorflow/contrib/cmake/tf_core_kernels.cmake
+++ b/tensorflow/contrib/cmake/tf_core_kernels.cmake
@@ -19,12 +19,14 @@ list(REMOVE_ITEM tf_core_kernels_srcs ${tf_core_kernels_exclude_srcs})
 
 add_library(tf_core_kernels OBJECT ${tf_core_kernels_srcs})
 
-add_dependencies(tf_core_kernels tf_core_cpu)
+add_dependencies(tf_core_kernels tf_core_cpu farmhash highwayhash)
 
 target_include_directories(tf_core_kernels PRIVATE
    ${tensorflow_source_dir}
    ${png_INCLUDE_DIR}
    ${eigen_INCLUDE_DIRS}
+   ${farmhash_INCLUDE_DIR}
+   ${highwayhash_INCLUDE_DIR}
 )
 
 #target_link_libraries(tf_core_kernels
diff --git a/tensorflow/contrib/cmake/tf_tools.cmake b/tensorflow/contrib/cmake/tf_tools.cmake
new file mode 100644
index 0000000000..3c1fe5eac7
--- /dev/null
+++ b/tensorflow/contrib/cmake/tf_tools.cmake
@@ -0,0 +1,52 @@
+set(tf_tools_proto_text_src_dir "${tensorflow_source_dir}/tensorflow/tools/proto_text")
+
+file(GLOB tf_tools_srcs
+    "${tf_tools_proto_text_src_dir}/gen_proto_text_functions.cc"
+    "${tf_tools_proto_text_src_dir}/gen_proto_text_functions_lib.h"
+    "${tf_tools_proto_text_src_dir}/gen_proto_text_functions_lib.cc"
+)
+
+set(proto_text "proto_text")
+
+add_executable(${proto_text}
+    ${tf_tools_srcs}
+    $<TARGET_OBJECTS:tf_core_lib>
+)
+
+target_include_directories(${proto_text} PUBLIC
+    ${tensorflow_source_dir}
+)
+
+target_link_libraries(${proto_text} PUBLIC
+    ${CMAKE_THREAD_LIBS_INIT}
+    ${PROTOBUF_LIBRARIES}
+    # tf_protos_cc
+    # re2_lib
+    ${jpeg_STATIC_LIBRARIES}
+    ${png_STATIC_LIBRARIES}
+    ${ZLIB_LIBRARIES}
+    ${jsoncpp_STATIC_LIBRARIES}
+    ${boringssl_STATIC_LIBRARIES}
+    ${CMAKE_DL_LIBS}
+)
+
+add_dependencies(${proto_text}
+    tf_core_lib
+    # jpeg_copy_headers_to_destination
+    # png_copy_headers_to_destination
+    # re2_copy_headers_to_destination
+    # eigen
+    # tf_protos_cc
+    # jsoncpp
+    # boringssl
+)
+
+target_compile_options(${proto_text} PRIVATE
+    -fno-exceptions
+    -DEIGEN_AVOID_STL_ARRAY
+)
+
+# C++11
+target_compile_features(${proto_text} PRIVATE
+    cxx_rvalue_references
+)