From 0433989bad71f765a17c60d81d4093148ed547c6 Mon Sep 17 00:00:00 2001 From: dan-shaw <51385773+dan-shaw@users.noreply.github.com> Date: Tue, 20 Aug 2019 12:13:26 -0700 Subject: [PATCH] [tensorflow] add new port for linux (#7568) * [+] Add tensorflow-cc port (Linux only) * [~] Refactor installation of tensorflow/external 1. Install tensorflow/external to tensorflow-etc/external; 2. Fix TensorflowCCConfig.cmake accordingly. * [tensorflow] Work in progress * fix tensorflow linux config * [tensorflow] partial support for windows * fix config paths * use environmental variables instead * remove files * add python_path * add verbose error messages * review fixes * [tensorflow] refactor config file * minor changes * set arch for CI --- ports/tensorflow-cc/CONTROL | 4 + ports/tensorflow-cc/TensorflowCCConfig.cmake | 28 +++++ ports/tensorflow-cc/file-exists.patch | 13 ++ ports/tensorflow-cc/portfile.cmake | 116 ++++++++++++++++++ .../cmake/vcpkg_find_acquire_program.cmake | 24 +++- 5 files changed, 182 insertions(+), 3 deletions(-) create mode 100644 ports/tensorflow-cc/CONTROL create mode 100644 ports/tensorflow-cc/TensorflowCCConfig.cmake create mode 100644 ports/tensorflow-cc/file-exists.patch create mode 100644 ports/tensorflow-cc/portfile.cmake diff --git a/ports/tensorflow-cc/CONTROL b/ports/tensorflow-cc/CONTROL new file mode 100644 index 0000000000..05e336e536 --- /dev/null +++ b/ports/tensorflow-cc/CONTROL @@ -0,0 +1,4 @@ +Source: tensorflow-cc +Version: 1.14 +Description: Library for computation using data flow graphs for scalable machine learning +Build-Depends: c-ares diff --git a/ports/tensorflow-cc/TensorflowCCConfig.cmake b/ports/tensorflow-cc/TensorflowCCConfig.cmake new file mode 100644 index 0000000000..db12811d5c --- /dev/null +++ b/ports/tensorflow-cc/TensorflowCCConfig.cmake @@ -0,0 +1,28 @@ +set(tensorflow_cc_INCLUDE_DIR "${CMAKE_CURRENT_LIST_DIR}/../../include") + +message(WARNING "Tensorflow has vendored dependencies. You may need to manually include files from tensorflow-external") +set(tensorflow_cc_INCLUDE_DIRS + ${tensorflow_cc_INCLUDE_DIR} + ${tensorflow_cc_INCLUDE_DIR}/tensorflow-external/ + ${tensorflow_cc_INCLUDE_DIR}/tensorflow-external/tensorflow/ + ${tensorflow_cc_INCLUDE_DIR}/tensorflow-external/external/com_google_absl + ${tensorflow_cc_INCLUDE_DIR}/tensorflow-external/bazel-out/k8-opt/bin/ + ${tensorflow_cc_INCLUDE_DIR}/tensorflow-external/external/protobuf_archive/src/ +) + +add_library(tensorflow_cc::tensorflow_framework SHARED IMPORTED) +set_target_properties(tensorflow_cc::tensorflow_framework + PROPERTIES + IMPORTED_LOCATION ${CMAKE_CURRENT_LIST_DIR}/../../lib/libtensorflow_framework.so.1.14.0 + INTERFACE_INCLUDE_DIRECTORIES "${tensorflow_cc_INCLUDE_DIRS}" +) + +add_library(tensorflow_cc::tensorflow_cc SHARED IMPORTED) +set_target_properties(tensorflow_cc::tensorflow_cc + PROPERTIES + IMPORTED_LOCATION ${CMAKE_CURRENT_LIST_DIR}/../../lib/libtensorflow_cc.so.1.14.0 + INTERFACE_INCLUDE_DIRECTORIES "${tensorflow_cc_INCLUDE_DIRS}" +) + +set(tensorflow_cc_FOUND TRUE) +set(tensorflow_framework_FOUND TRUE) \ No newline at end of file diff --git a/ports/tensorflow-cc/file-exists.patch b/ports/tensorflow-cc/file-exists.patch new file mode 100644 index 0000000000..8cefb4828c --- /dev/null +++ b/ports/tensorflow-cc/file-exists.patch @@ -0,0 +1,13 @@ +diff --git a/configure.py b/configure.py +index 43af22d..7989b1f 100644 +--- a/configure.py ++++ b/configure.py +@@ -183,7 +183,7 @@ def get_python_path(environ_cp, python_bin_path): + + paths = [] + for path in all_paths: +- if os.path.isdir(path): ++ if os.path.isdir(path) or True: + paths.append(path) + return paths + diff --git a/ports/tensorflow-cc/portfile.cmake b/ports/tensorflow-cc/portfile.cmake new file mode 100644 index 0000000000..0e40f859bd --- /dev/null +++ b/ports/tensorflow-cc/portfile.cmake @@ -0,0 +1,116 @@ +include(vcpkg_common_functions) + +message(WARNING "This tensorflow port currently is experimental on Windows and Linux platforms.") + +vcpkg_from_github( + OUT_SOURCE_PATH SOURCE_PATH + REPO tensorflow/tensorflow + REF v1.14.0 + SHA512 ac9ea5a2d1c761aaafbdc335259e29c128127b8d069ec5b206067935180490aa95e93c7e13de57f7f54ce4ba4f34a822face22b4a028f60185edb380e5cd4787 + HEAD_REF master + PATCHES + file-exists.patch # required or otherwise it cant find python lib path on windows +) + +# due to https://github.com/bazelbuild/bazel/issues/8028, bazel must be version 25.0 or higher +vcpkg_find_acquire_program(BAZEL) +get_filename_component(BAZEL_DIR "${BAZEL}" DIRECTORY) +vcpkg_add_to_path(PREPEND ${BAZEL_DIR}) +set(ENV{BAZEL_BIN_PATH} "${BAZEL}") + +vcpkg_find_acquire_program(PYTHON3) +get_filename_component(PYTHON3_DIR "${PYTHON3}" DIRECTORY) +vcpkg_add_to_path(PREPEND ${PYTHON3_DIR}) +set(ENV{PYTHON_BIN_PATH} "${PYTHON3}") + +function(tensorflow_try_remove_recurse_wait PATH_TO_REMOVE) + file(REMOVE_RECURSE ${PATH_TO_REMOVE}) + if (EXISTS "${PATH_TO_REMOVE}") + execute_process(COMMAND ${CMAKE_COMMAND} -E sleep 5) + file(REMOVE_RECURSE ${PATH_TO_REMOVE}) + endif() +endfunction() + +# we currently only support the release version +tensorflow_try_remove_recurse_wait(${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel) +file(MAKE_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel) +file(GLOB SOURCES ${SOURCE_PATH}/*) +file(COPY ${SOURCES} DESTINATION ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel) + +if(CMAKE_HOST_WIN32) + vcpkg_acquire_msys(MSYS_ROOT PACKAGES unzip patch diffutils git) + set(BASH ${MSYS_ROOT}/usr/bin/bash.exe) + set(ENV{BAZEL_SH} ${MSYS_ROOT}/usr/bin/bash.exe) + + set(ENV{BAZEL_VS} $ENV{VSInstallDir}) + set(ENV{BAZEL_VC} $ENV{VCInstallDir}) +endif() + +# tensorflow has long file names, which will not work on windows +set(ENV{TEST_TMPDIR} ${CURRENT_BUILDTREES_DIR}/../.bzl) + +set(ENV{USE_DEFAULT_PYTHON_LIB_PATH} 1) +set(ENV{TF_NEED_KAFKA} 0) +set(ENV{TF_NEED_OPENCL_SYCL} 0) +set(ENV{TF_NEED_AWS} 0) +set(ENV{TF_NEED_GCP} 0) +set(ENV{TF_NEED_HDFS} 0) +set(ENV{TF_NEED_S3} 0) +set(ENV{TF_ENABLE_XLA} 0) +set(ENV{TF_NEED_GDR} 0) +set(ENV{TF_NEED_VERBS} 0) +set(ENV{TF_NEED_OPENCL} 0) +set(ENV{TF_NEED_MPI} 0) +set(ENV{TF_NEED_TENSORRT} 0) +set(ENV{TF_NEED_NGRAPH} 0) +set(ENV{TF_NEED_IGNITE} 0) +set(ENV{TF_NEED_ROCM} 0) +set(ENV{TF_SET_ANDROID_WORKSPACE} 0) +set(ENV{TF_DOWNLOAD_CLANG} 0) +set(ENV{TF_NCCL_VERSION} 2.3) +set(ENV{NCCL_INSTALL_PATH} "") +set(ENV{CC_OPT_FLAGS} "/arch:AVX") +set(ENV{TF_NEED_CUDA} 0) + +message(STATUS "Configuring TensorFlow") + +vcpkg_execute_required_process( + COMMAND ${PYTHON3} ${SOURCE_PATH}/configure.py + WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel + LOGNAME config-${TARGET_TRIPLET}-rel +) +message(STATUS "Warning: Building TensorFlow can take an hour or more.") + +if(CMAKE_HOST_WIN32) + vcpkg_execute_build_process( + COMMAND ${BASH} --noprofile --norc -c "${BAZEL} build --verbose_failures -c opt --python_path=${PYTHON3} --incompatible_disable_deprecated_attr_params=false --define=no_tensorflow_py_deps=true ///tensorflow:libtensorflow_cc.so ///tensorflow:install_headers" + WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel + LOGNAME build-${TARGET_TRIPLET}-rel + ) +else() + vcpkg_execute_build_process( + COMMAND ${BAZEL} build --verbose_failures -c opt --python_path=${PYTHON3} --incompatible_disable_deprecated_attr_params=false --define=no_tensorflow_py_deps=true //tensorflow:libtensorflow_cc.so //tensorflow:install_headers + WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel + LOGNAME build-${TARGET_TRIPLET}-rel + ) +endif() + +file(COPY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel/bazel-genfiles/tensorflow/include/ DESTINATION ${CURRENT_PACKAGES_DIR}/include/tensorflow-external) + +if(CMAKE_HOST_WIN32) + file(COPY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel/bazel-bin/tensorflow/libtensorflow_cc.so.1.14.0 DESTINATION ${CURRENT_PACKAGES_DIR}/lib) + file(COPY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel/bazel-bin/tensorflow/libtensorflow_cc.so.1.14.0.if.lib DESTINATION ${CURRENT_PACKAGES_DIR}/lib) + file(COPY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel/bazel-bin/tensorflow/libtensorflow_cc.so.1.14.0 DESTINATION ${CURRENT_PACKAGES_DIR}/debug/lib) + file(COPY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel/bazel-bin/tensorflow/libtensorflow_cc.so.1.14.0.if.lib DESTINATION ${CURRENT_PACKAGES_DIR}/debug/lib) +else() + file(COPY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel/bazel-bin/tensorflow/libtensorflow_cc.so.1.14.0 DESTINATION ${CURRENT_PACKAGES_DIR}/lib) + file(COPY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel/bazel-bin/tensorflow/libtensorflow_framework.so.1.14.0 DESTINATION ${CURRENT_PACKAGES_DIR}/lib) + file(COPY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel/bazel-bin/tensorflow/libtensorflow_cc.so.1.14.0 DESTINATION ${CURRENT_PACKAGES_DIR}/debug/lib) + file(COPY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-rel/bazel-bin/tensorflow/libtensorflow_framework.so.1.14.0 DESTINATION ${CURRENT_PACKAGES_DIR}/debug/lib) +endif() + +file(COPY ${SOURCE_PATH}/LICENSE DESTINATION ${CURRENT_PACKAGES_DIR}/share/tensorflow-cc) +file(RENAME ${CURRENT_PACKAGES_DIR}/share/tensorflow-cc/LICENSE ${CURRENT_PACKAGES_DIR}/share/tensorflow-cc/copyright) + +file(COPY ${CMAKE_CURRENT_LIST_DIR}/TensorflowCCConfig.cmake DESTINATION ${CURRENT_PACKAGES_DIR}/share/unofficial-tensorflow-cc) +file(RENAME ${CURRENT_PACKAGES_DIR}/share/unofficial-tensorflow-cc/TensorflowCCConfig.cmake ${CURRENT_PACKAGES_DIR}/share/unofficial-tensorflow-cc/unofficial-tensorflow-cc-config.cmake) \ No newline at end of file diff --git a/scripts/cmake/vcpkg_find_acquire_program.cmake b/scripts/cmake/vcpkg_find_acquire_program.cmake index 84ebdbfa8c..fed016c7bf 100644 --- a/scripts/cmake/vcpkg_find_acquire_program.cmake +++ b/scripts/cmake/vcpkg_find_acquire_program.cmake @@ -45,6 +45,7 @@ function(vcpkg_find_acquire_program VAR) unset(_vfa_RENAME) unset(SUBDIR) unset(REQUIRED_INTERPRETER) + unset(_vfa_SUPPORTED) unset(POST_INSTALL_COMMAND) vcpkg_get_program_files_platform_bitness(PROGRAM_FILES_PLATFORM_BITNESS) @@ -231,6 +232,23 @@ function(vcpkg_find_acquire_program VAR) set(URL "http://doxygen.nl/files/doxygen-1.8.15.windows.bin.zip") set(ARCHIVE "doxygen-1.8.15.windows.bin.zip") set(HASH 89482dcb1863d381d47812c985593e736d703931d49994e09c7c03ef67e064115d0222b8de1563a7930404c9bc2d3be323f3d13a01ef18861be584db3d5a953c) + elseif(VAR MATCHES "BAZEL") + set(PROGNAME bazel) + set(BAZEL_VERSION 0.25.2) + set(SUBDIR ${BAZEL_VERSION}) + set(PATHS ${DOWNLOADS}/tools/bazel/${SUBDIR}) + set(_vfa_RENAME "bazel") + if(CMAKE_HOST_SYSTEM_NAME STREQUAL "Linux") + set(_vfa_SUPPORTED ON) + set(URL "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-linux-x86_64") + set(ARCHIVE "bazel-${BAZEL_VERSION}-linux-x86_64") + set(NOEXTRACT ON) + set(HASH db4a583cf2996aeb29fd008261b12fe39a4a5faf0fbf96f7124e6d3ffeccf6d9655d391378e68dd0915bc91c9e146a51fd9661963743857ca25179547feceab1) + else() + set(URL "https://github.com/bazelbuild/bazel/releases/download/${BAZEL_VERSION}/bazel-${BAZEL_VERSION}-windows-x86_64.zip") + set(ARCHIVE "bazel-${BAZEL_VERSION}-windows-x86_64.zip") + set(HASH 6482f99a0896f55ef65739e7b53452fd9c0adf597b599d0022a5e0c5fa4374f4a958d46f98e8ba25af4b065adacc578bfedced483d8c169ea5cb1777a99eea53) + endif() # Download Tools elseif(VAR MATCHES "ARIA2") set(PROGNAME aria2c) @@ -254,7 +272,7 @@ function(vcpkg_find_acquire_program VAR) do_find() if("${${VAR}}" MATCHES "-NOTFOUND") - if(NOT CMAKE_HOST_SYSTEM_NAME STREQUAL "Windows") + if(NOT CMAKE_HOST_SYSTEM_NAME STREQUAL "Windows" AND NOT _vfa_SUPPORTED) set(EXAMPLE ".") if(DEFINED BREW_PACKAGE_NAME AND CMAKE_HOST_SYSTEM_NAME STREQUAL "Darwin") set(EXAMPLE ":\n brew install ${BREW_PACKAGE_NAME}") @@ -274,9 +292,9 @@ function(vcpkg_find_acquire_program VAR) file(MAKE_DIRECTORY ${PROG_PATH_SUBDIR}) if(DEFINED NOEXTRACT) if(DEFINED _vfa_RENAME) - file(INSTALL ${ARCHIVE_PATH} DESTINATION ${PROG_PATH_SUBDIR} RENAME ${_vfa_RENAME}) + file(INSTALL ${ARCHIVE_PATH} DESTINATION ${PROG_PATH_SUBDIR} RENAME ${_vfa_RENAME} PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) else() - file(COPY ${ARCHIVE_PATH} DESTINATION ${PROG_PATH_SUBDIR}) + file(COPY ${ARCHIVE_PATH} DESTINATION ${PROG_PATH_SUBDIR} PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE GROUP_READ GROUP_EXECUTE WORLD_READ WORLD_EXECUTE) endif() else() get_filename_component(ARCHIVE_EXTENSION ${ARCHIVE} EXT)