[tensorflow] add new port for linux (#7568)

* [+] Add tensorflow-cc port (Linux only) * [~] Refactor installation of tensorflow/external 1. Install tensorflow/external to tensorflow-etc/external; 2. Fix TensorflowCCConfig.cmake accordingly. * [tensorflow] Work in progress * fix tensorflow linux config * [tensorflow] partial support for windows * fix config paths * use environmental variables instead * remove files * add python_path * add verbose error messages * review fixes * [tensorflow] refactor config file * minor changes * set arch for CI
2024-11-24 08:39:01 +08:00 · 2019-08-20 12:13:26 -07:00 · 2019-08-20 12:13:26 -07:00 · 0433989bad
commit 0433989bad
parent 7f165907fc
5 changed files with 182 additions and 3 deletions
--- a/ports/tensorflow-cc/CONTROL
+++ b/ports/tensorflow-cc/CONTROL
@ -0,0 +1,4 @@
+Source: tensorflow-cc
+Version: 1.14
+Description: Library for computation using data flow graphs for scalable machine learning
+Build-Depends: c-ares
--- a/ports/tensorflow-cc/TensorflowCCConfig.cmake
+++ b/ports/tensorflow-cc/TensorflowCCConfig.cmake
@ -0,0 +1,28 @@
+set(tensorflow_cc_INCLUDE_DIR "${CMAKE_CURRENT_LIST_DIR}/../../include")
+
+message(WARNING "Tensorflow has vendored dependencies. You may need to manually include files from tensorflow-external")
+set(tensorflow_cc_INCLUDE_DIRS
+	${tensorflow_cc_INCLUDE_DIR}
+	${tensorflow_cc_INCLUDE_DIR}/tensorflow-external/
+	${tensorflow_cc_INCLUDE_DIR}/tensorflow-external/tensorflow/
+	${tensorflow_cc_INCLUDE_DIR}/tensorflow-external/external/com_google_absl
+	${tensorflow_cc_INCLUDE_DIR}/tensorflow-external/bazel-out/k8-opt/bin/
+	${tensorflow_cc_INCLUDE_DIR}/tensorflow-external/external/protobuf_archive/src/
+)
+
+add_library(tensorflow_cc::tensorflow_framework SHARED IMPORTED)
+set_target_properties(tensorflow_cc::tensorflow_framework 
+	PROPERTIES
+	IMPORTED_LOCATION ${CMAKE_CURRENT_LIST_DIR}/../../lib/libtensorflow_framework.so.1.14.0
+	INTERFACE_INCLUDE_DIRECTORIES "${tensorflow_cc_INCLUDE_DIRS}"
+)
+
+add_library(tensorflow_cc::tensorflow_cc SHARED IMPORTED)
+set_target_properties(tensorflow_cc::tensorflow_cc
+	PROPERTIES 
+	IMPORTED_LOCATION ${CMAKE_CURRENT_LIST_DIR}/../../lib/libtensorflow_cc.so.1.14.0
+	INTERFACE_INCLUDE_DIRECTORIES "${tensorflow_cc_INCLUDE_DIRS}"
+)
+
+set(tensorflow_cc_FOUND TRUE)
+set(tensorflow_framework_FOUND TRUE)
--- a/ports/tensorflow-cc/file-exists.patch
+++ b/ports/tensorflow-cc/file-exists.patch
@ -0,0 +1,13 @@
+diff --git a/configure.py b/configure.py
+index 43af22d..7989b1f 100644
+--- a/configure.py
+++ b/configure.py
+@@ -183,7 +183,7 @@ def get_python_path(environ_cp, python_bin_path):
+ 
+   paths = []
+   for path in all_paths:
+-    if os.path.isdir(path):
+    if os.path.isdir(path) or True:
+       paths.append(path)
+   return paths
+ 
--- a/ports/tensorflow-cc/portfile.cmake
+++ b/ports/tensorflow-cc/portfile.cmake
@ -0,0 +1,116 @@
+include(vcpkg_common_functions)
+
+message(WARNING "This tensorflow port currently is experimental on Windows and Linux platforms.")
+
+vcpkg_from_github(
+    OUT_SOURCE_PATH SOURCE_PATH
+    REPO tensorflow/tensorflow
+    REF v1.14.0
+    SHA512 ac9ea5a2d1c761aaafbdc335259e29c128127b8d069ec5b206067935180490aa95e93c7e13de57f7f54ce4ba4f34a822face22b4a028f60185edb380e5cd4787
+    HEAD_REF master
+    PATCHES
+        file-exists.patch # required or otherwise it cant find python lib path on windows
+)
+
+# due to https://github.com/bazelbuild/bazel/issues/8028, bazel must be version 25.0 or higher
+vcpkg_find_acquire_program(BAZEL)
+get_filename_component(BAZEL_DIR "${BAZEL}" DIRECTORY)
+vcpkg_add_to_path(PREPEND ${BAZEL_DIR})
+set(ENV{BAZEL_BIN_PATH} "${BAZEL}")
+
+vcpkg_find_acquire_program(PYTHON3)
+get_filename_component(PYTHON3_DIR "${PYTHON3}" DIRECTORY)
+vcpkg_add_to_path(PREPEND ${PYTHON3_DIR})
+set(ENV{PYTHON_BIN_PATH} "${PYTHON3}")
+
+function(tensorflow_try_remove_recurse_wait PATH_TO_REMOVE)
+    file(REMOVE_RECURSE ${PATH_TO_REMOVE})
+    if (EXISTS "${PATH_TO_REMOVE}")
+        execute_process(COMMAND ${CMAKE_COMMAND} -E sleep 5)
+        file(REMOVE_RECURSE ${PATH_TO_REMOVE})
+    endif()
+endfunction()
+
+# we currently only support the release version
+tensorflow_try_remove_recurse_wait(${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel)
+file(MAKE_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel)
+file(GLOB SOURCES ${SOURCE_PATH}/*)
+file(COPY ${SOURCES} DESTINATION ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel)
+
+if(CMAKE_HOST_WIN32)
+    vcpkg_acquire_msys(MSYS_ROOT PACKAGES unzip patch diffutils git)
+    set(BASH ${MSYS_ROOT}/usr/bin/bash.exe)
+    set(ENV{BAZEL_SH} ${MSYS_ROOT}/usr/bin/bash.exe)
+
+    set(ENV{BAZEL_VS} $ENV{VSInstallDir})
+    set(ENV{BAZEL_VC} $ENV{VCInstallDir})
+endif()
+
+# tensorflow has long file names, which will not work on windows
+set(ENV{TEST_TMPDIR} ${CURRENT_BUILDTREES_DIR}/../.bzl)
+
+set(ENV{USE_DEFAULT_PYTHON_LIB_PATH} 1)
+set(ENV{TF_NEED_KAFKA} 0)
+set(ENV{TF_NEED_OPENCL_SYCL} 0)
+set(ENV{TF_NEED_AWS} 0)
+set(ENV{TF_NEED_GCP} 0)
+set(ENV{TF_NEED_HDFS} 0)
+set(ENV{TF_NEED_S3} 0)
+set(ENV{TF_ENABLE_XLA} 0)
+set(ENV{TF_NEED_GDR} 0)
+set(ENV{TF_NEED_VERBS} 0)
+set(ENV{TF_NEED_OPENCL} 0)
+set(ENV{TF_NEED_MPI} 0)
+set(ENV{TF_NEED_TENSORRT} 0)
+set(ENV{TF_NEED_NGRAPH} 0)
+set(ENV{TF_NEED_IGNITE} 0)
+set(ENV{TF_NEED_ROCM} 0)
+set(ENV{TF_SET_ANDROID_WORKSPACE} 0)
+set(ENV{TF_DOWNLOAD_CLANG} 0)
+set(ENV{TF_NCCL_VERSION} 2.3)
+set(ENV{NCCL_INSTALL_PATH} "")
+set(ENV{CC_OPT_FLAGS} "/arch:AVX")
+set(ENV{TF_NEED_CUDA} 0)
+
+message(STATUS "Configuring TensorFlow")
+
+vcpkg_execute_required_process(
+    COMMAND ${PYTHON3} ${SOURCE_PATH}/configure.py
+    WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel
+    LOGNAME config-${TARGET_TRIPLET}-rel
+)
+message(STATUS "Warning: Building TensorFlow can take an hour or more.")
+
+if(CMAKE_HOST_WIN32)
+    vcpkg_execute_build_process(
+        COMMAND ${BASH} --noprofile --norc -c "${BAZEL} build --verbose_failures -c opt --python_path=${PYTHON3} --incompatible_disable_deprecated_attr_params=false --define=no_tensorflow_py_deps=true ///tensorflow:libtensorflow_cc.so ///tensorflow:install_headers"
+        WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel
+        LOGNAME build-${TARGET_TRIPLET}-rel
+    )
+else()
+    vcpkg_execute_build_process(
+        COMMAND ${BAZEL} build --verbose_failures -c opt --python_path=${PYTHON3} --incompatible_disable_deprecated_attr_params=false --define=no_tensorflow_py_deps=true //tensorflow:libtensorflow_cc.so //tensorflow:install_headers
+        WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel
+        LOGNAME build-${TARGET_TRIPLET}-rel
+    )
+endif()
+
+file(COPY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel/bazel-genfiles/tensorflow/include/ DESTINATION ${CURRENT_PACKAGES_DIR}/include/tensorflow-external)
+
+if(CMAKE_HOST_WIN32)
+    file(COPY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel/bazel-bin/tensorflow/libtensorflow_cc.so.1.14.0 DESTINATION ${CURRENT_PACKAGES_DIR}/lib)
+    file(COPY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel/bazel-bin/tensorflow/libtensorflow_cc.so.1.14.0.if.lib DESTINATION ${CURRENT_PACKAGES_DIR}/lib)
+    file(COPY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel/bazel-bin/tensorflow/libtensorflow_cc.so.1.14.0 DESTINATION ${CURRENT_PACKAGES_DIR}/debug/lib)
+    file(COPY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel/bazel-bin/tensorflow/libtensorflow_cc.so.1.14.0.if.lib DESTINATION ${CURRENT_PACKAGES_DIR}/debug/lib)
+else()
+    file(COPY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel/bazel-bin/tensorflow/libtensorflow_cc.so.1.14.0 DESTINATION ${CURRENT_PACKAGES_DIR}/lib)
+    file(COPY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel/bazel-bin/tensorflow/libtensorflow_framework.so.1.14.0 DESTINATION ${CURRENT_PACKAGES_DIR}/lib)
+    file(COPY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel/bazel-bin/tensorflow/libtensorflow_cc.so.1.14.0 DESTINATION ${CURRENT_PACKAGES_DIR}/debug/lib)
+    file(COPY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel/bazel-bin/tensorflow/libtensorflow_framework.so.1.14.0 DESTINATION ${CURRENT_PACKAGES_DIR}/debug/lib)
+endif()
+
+file(COPY ${SOURCE_PATH}/LICENSE DESTINATION ${CURRENT_PACKAGES_DIR}/share/tensorflow-cc)
+file(RENAME ${CURRENT_PACKAGES_DIR}/share/tensorflow-cc/LICENSE ${CURRENT_PACKAGES_DIR}/share/tensorflow-cc/copyright)
+
+file(COPY ${CMAKE_CURRENT_LIST_DIR}/TensorflowCCConfig.cmake DESTINATION ${CURRENT_PACKAGES_DIR}/share/unofficial-tensorflow-cc)
+file(RENAME ${CURRENT_PACKAGES_DIR}/share/unofficial-tensorflow-cc/TensorflowCCConfig.cmake ${CURRENT_PACKAGES_DIR}/share/unofficial-tensorflow-cc/unofficial-tensorflow-cc-config.cmake)
--- a/scripts/cmake/vcpkg_find_acquire_program.cmake
+++ b/scripts/cmake/vcpkg_find_acquire_program.cmake
@ -45,6 +45,7 @@ function(vcpkg_find_acquire_program VAR)
  unset(_vfa_RENAME)
  unset(SUBDIR)
  unset(REQUIRED_INTERPRETER)
+  unset(_vfa_SUPPORTED)
  unset(POST_INSTALL_COMMAND)

  vcpkg_get_program_files_platform_bitness(PROGRAM_FILES_PLATFORM_BITNESS)
@ -231,6 +232,23 @@ function(vcpkg_find_acquire_program VAR)
    set(URL "http://doxygen.nl/files/doxygen-1.8.15.windows.bin.zip")
    set(ARCHIVE "doxygen-1.8.15.windows.bin.zip")
    set(HASH 89482dcb1863d381d47812c985593e736d703931d49994e09c7c03ef67e064115d0222b8de1563a7930404c9bc2d3be323f3d13a01ef18861be584db3d5a953c)
+  elseif(VAR MATCHES "BAZEL")
+    set(PROGNAME bazel)
+    set(BAZEL_VERSION 0.25.2)
+    set(SUBDIR ${BAZEL_VERSION})
+    set(PATHS ${DOWNLOADS}/tools/bazel/${SUBDIR})
+    set(_vfa_RENAME "bazel")
+    if(CMAKE_HOST_SYSTEM_NAME STREQUAL "Linux")
+      set(_vfa_SUPPORTED ON)
+      set(URL "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-linux-x86_64")
+      set(ARCHIVE "bazel-${BAZEL_VERSION}-linux-x86_64")
+      set(NOEXTRACT ON)
+      set(HASH db4a583cf2996aeb29fd008261b12fe39a4a5faf0fbf96f7124e6d3ffeccf6d9655d391378e68dd0915bc91c9e146a51fd9661963743857ca25179547feceab1)
+    else()
+      set(URL "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-windows-x86_64.zip") 
+      set(ARCHIVE "bazel-${BAZEL_VERSION}-windows-x86_64.zip")
+      set(HASH 6482f99a0896f55ef65739e7b53452fd9c0adf597b599d0022a5e0c5fa4374f4a958d46f98e8ba25af4b065adacc578bfedced483d8c169ea5cb1777a99eea53)
+    endif()
  # Download Tools
  elseif(VAR MATCHES "ARIA2")
    set(PROGNAME aria2c)
@ -254,7 +272,7 @@ function(vcpkg_find_acquire_program VAR)

  do_find()
  if("${${VAR}}" MATCHES "-NOTFOUND")
-    if(NOT CMAKE_HOST_SYSTEM_NAME STREQUAL "Windows")
+    if(NOT CMAKE_HOST_SYSTEM_NAME STREQUAL "Windows" AND NOT _vfa_SUPPORTED)
      set(EXAMPLE ".")
      if(DEFINED BREW_PACKAGE_NAME AND CMAKE_HOST_SYSTEM_NAME STREQUAL "Darwin")
        set(EXAMPLE ":\n    brew install ${BREW_PACKAGE_NAME}")
@ -274,9 +292,9 @@ function(vcpkg_find_acquire_program VAR)
    file(MAKE_DIRECTORY ${PROG_PATH_SUBDIR})
    if(DEFINED NOEXTRACT)
      if(DEFINED _vfa_RENAME)
-        file(INSTALL ${ARCHIVE_PATH} DESTINATION ${PROG_PATH_SUBDIR} RENAME ${_vfa_RENAME})
+        file(INSTALL ${ARCHIVE_PATH} DESTINATION ${PROG_PATH_SUBDIR} RENAME ${_vfa_RENAME} PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
      else()
-        file(COPY ${ARCHIVE_PATH} DESTINATION ${PROG_PATH_SUBDIR})
+        file(COPY ${ARCHIVE_PATH} DESTINATION ${PROG_PATH_SUBDIR} PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE)
      endif()
    else()
      get_filename_component(ARCHIVE_EXTENSION ${ARCHIVE} EXT)