[vcpkg baseline][tensorflow] setup python venv and update Linux VMs (#21912)

* Revert "incorporate changes from microsoft:master" * Revert "Revert "incorporate changes from microsoft:master"" * preliminary approach to solve pip issue * fix python venv * fix macOS build command again * bump version * x-add-version * Fix spelling of prerequisites. * Add python3-venv * Update linux pool. Co-authored-by: Joachim Gehweiler <joachim@Joachims-iMac.local> Co-authored-by: Billy Robert O'Neal III <bion@microsoft.com>
2025-01-19 04:43:02 +08:00 · 2021-12-14 02:46:40 +01:00 · 2021-12-14 02:46:40 +01:00 · a9ff505c5b
commit a9ff505c5b
parent d31efec576
9 changed files with 33 additions and 19 deletions
--- a/ports/tensorflow-common/generate_static_link_cmd_linux.py
+++ b/ports/tensorflow-common/generate_static_link_cmd_linux.py
@ -1,8 +1,9 @@
 import re
 import sys

-version = sys.argv[2]
-lib_suffix = "" if len(sys.argv) < 4 else sys.argv[3]
+params_path = sys.argv[2]
+version = sys.argv[3]
+lib_suffix = "" if len(sys.argv) < 5 else sys.argv[4]

 with open(sys.argv[1], "r") as f_in:
    with open("static_link.sh", "w") as f_out:
--- a/ports/tensorflow-common/generate_static_link_cmd_macos.py
+++ b/ports/tensorflow-common/generate_static_link_cmd_macos.py
@ -2,12 +2,13 @@ import os.path
 import re
 import sys

-version = sys.argv[2]
-lib_suffix = "" if len(sys.argv) < 4 else sys.argv[3]
+params_path = sys.argv[2]
+version = sys.argv[3]
+lib_suffix = "" if len(sys.argv) < 5 else sys.argv[4]

 with open(sys.argv[1], "r") as f_in:
    with open("static_link.sh", "w") as f_out:
-        if os.path.isfile(f"libtensorflow_framework.{version}.dylib-2.params"):
+        if os.path.isfile(f"{params_path}/libtensorflow_framework.{version}.dylib-2.params"):
            p_cd = re.compile(r"^\((cd .*) && \\$")
            p_linker = re.compile(fr"^\s*.+cc_wrapper.sh.+(@bazel-out\S+libtensorflow{lib_suffix}\.\d\.\d\.\d\.dylib-2\.params).*")
            f_out.write("#!/bin/bash\n# note: ar/binutils version 2.27 required to support output files > 4GB\n")
--- a/ports/tensorflow-common/generate_static_link_cmd_windows.py
+++ b/ports/tensorflow-common/generate_static_link_cmd_windows.py
@ -2,8 +2,9 @@ import os.path
 import re
 import sys

-version = sys.argv[2]
-lib_suffix = "" if len(sys.argv) < 4 else sys.argv[3]
+params_path = sys.argv[2]
+version = sys.argv[3]
+lib_suffix = "" if len(sys.argv) < 5 else sys.argv[4]

 with open(sys.argv[1], "r") as f_in:
    with open("static_link.bat", "w") as f_out:
--- a/ports/tensorflow-common/tensorflow-common.cmake
+++ b/ports/tensorflow-common/tensorflow-common.cmake
@ -9,7 +9,7 @@ set(ENV{BAZEL_BIN_PATH} "${BAZEL}")
 function(tensorflow_try_remove_recurse_wait PATH_TO_REMOVE)
 	file(REMOVE_RECURSE ${PATH_TO_REMOVE})
 	if(EXISTS "${PATH_TO_REMOVE}")
-		vcpkg_execute_required_process(COMMAND ${CMAKE_COMMAND} -E sleep 5 WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR} LOGNAME prerequesits-sleep-${TARGET_TRIPLET})
+		vcpkg_execute_required_process(COMMAND ${CMAKE_COMMAND} -E sleep 5 WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR} LOGNAME prerequisites-sleep-${TARGET_TRIPLET})
 		file(REMOVE_RECURSE ${PATH_TO_REMOVE})
 	endif()
 endfunction()
@ -39,29 +39,34 @@ if(CMAKE_HOST_WIN32)
 	set(ENV{BAZEL_VC_FULL_VERSION} $ENV{VCToolsVersion})

 	set(PYTHON3 "${MSYS_ROOT}/mingw64/bin/python3.exe")
-	vcpkg_execute_required_process(COMMAND ${PYTHON3} -c "import site; print(site.getsitepackages()[0])" WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR} LOGNAME prerequesits-pypath-${TARGET_TRIPLET} OUTPUT_VARIABLE PYTHON_LIB_PATH)
+	vcpkg_execute_required_process(COMMAND ${PYTHON3} -c "import site; print(site.getsitepackages()[0])" WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR} LOGNAME prerequisites-pypath-${TARGET_TRIPLET} OUTPUT_VARIABLE PYTHON_LIB_PATH)
 else()
 	vcpkg_find_acquire_program(PYTHON3)
-	get_filename_component(PYTHON3_DIR "${PYTHON3}" DIRECTORY)
-	vcpkg_add_to_path(PREPEND ${PYTHON3_DIR})
+
+	vcpkg_execute_required_process(COMMAND ${PYTHON3} -m venv --symlinks "${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-venv"  WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR} LOGNAME prerequisites-venv-${TARGET_TRIPLET})
+	vcpkg_add_to_path(PREPEND ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-venv/bin)
+	set(PYTHON3 ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-venv/bin/python3)
+	set(ENV{VIRTUAL_ENV} ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-venv)

 	if(VCPKG_TARGET_IS_OSX)
+		vcpkg_execute_required_process(COMMAND ${PYTHON3} -m pip install -U pip WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR} LOGNAME prerequisites-pip-${TARGET_TRIPLET})
 		# acceleration libs currently broken on macOS => force numpy user space reinstall without BLAS/LAPACK/ATLAS
 		# remove this work-around again, i.e. default to "else" branch, once acceleration libs are fixed upstream
 		set(ENV{BLAS} "None")
 		set(ENV{LAPACK} "None")
 		set(ENV{ATLAS} "None")
-		vcpkg_execute_required_process(COMMAND ${PYTHON3} -m pip install --user -U --force-reinstall numpy WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR} LOGNAME prerequesits-pip-${TARGET_TRIPLET})
+		vcpkg_execute_required_process(COMMAND ${PYTHON3} -m pip install -U --force-reinstall numpy WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR} LOGNAME prerequistes-pip-${TARGET_TRIPLET})
 	else()
-		vcpkg_execute_required_process(COMMAND ${PYTHON3} -m pip install --user -U numpy WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR} LOGNAME prerequesits-pip-${TARGET_TRIPLET})
+		vcpkg_execute_required_process(COMMAND ${PYTHON3} -m pip install -U pip numpy WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR} LOGNAME prerequisites-pip-${TARGET_TRIPLET})
 	endif()
-	vcpkg_execute_required_process(COMMAND ${PYTHON3} -c "import site; print(site.getusersitepackages())" WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR} LOGNAME prerequesits-pypath-${TARGET_TRIPLET} OUTPUT_VARIABLE PYTHON_LIB_PATH)
+
+	vcpkg_execute_required_process(COMMAND ${PYTHON3} -c "import site; print(site.getusersitepackages())" WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR} LOGNAME prerequisites-pypath-${TARGET_TRIPLET} OUTPUT_VARIABLE PYTHON_LIB_PATH)
 endif()
 set(ENV{PYTHON_BIN_PATH} "${PYTHON3}")
 set(ENV{PYTHON_LIB_PATH} "${PYTHON_LIB_PATH}")

 # check if numpy can be loaded
-vcpkg_execute_required_process(COMMAND ${PYTHON3} -c "import numpy" WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR} LOGNAME prerequesits-numpy-${TARGET_TRIPLET})
+vcpkg_execute_required_process(COMMAND ${PYTHON3} -c "import numpy" WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR} LOGNAME prerequisites-numpy-${TARGET_TRIPLET})

 set(ENV{USE_DEFAULT_PYTHON_LIB_PATH} 1)
 set(ENV{TF_NEED_KAFKA} 0)
@ -286,7 +291,7 @@ foreach(BUILD_TYPE dbg rel)
 		endif()
 		# for some reason stdout of bazel ends up in stderr, so use err log file in the following command
 		vcpkg_execute_build_process(
-			COMMAND ${PYTHON3} "${CMAKE_CURRENT_LIST_DIR}/generate_static_link_cmd_${PLATFORM_SUFFIX}.py" "${CURRENT_BUILDTREES_DIR}/build-${TARGET_TRIPLET}-${BUILD_TYPE}-err.log" ${TF_VERSION} ${TF_LIB_SUFFIX}
+			COMMAND ${PYTHON3} "${CMAKE_CURRENT_LIST_DIR}/generate_static_link_cmd_${PLATFORM_SUFFIX}.py" "${CURRENT_BUILDTREES_DIR}/build-${TARGET_TRIPLET}-${BUILD_TYPE}-err.log" "${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${BUILD_TYPE}/bazel-bin/tensorflow" ${TF_VERSION} ${TF_LIB_SUFFIX}
 			WORKING_DIRECTORY ${CURRENT_BUILDTREES_DIR}/${TARGET_TRIPLET}-${BUILD_TYPE}/bazel-${TARGET_TRIPLET}-${BUILD_TYPE}
 			LOGNAME postbuild2-${TARGET_TRIPLET}-${BUILD_TYPE}
 		)
--- a/ports/tensorflow-common/vcpkg.json
+++ b/ports/tensorflow-common/vcpkg.json
@ -1,6 +1,7 @@
 {
  "name": "tensorflow-common",
  "version-semver": "2.6.0",
+  "port-version": 1,
  "description": "This meta package holds common files for the C [tensorflow] and the C++ [tensorflow-cc] API version of TensorFlow but is not installable on its own.",
  "homepage": "https://github.com/tensorflow/tensorflow"
 }
--- a/scripts/azure-pipelines/azure-pipelines.yml
+++ b/scripts/azure-pipelines/azure-pipelines.yml
@ -3,7 +3,7 @@
 #
 variables:
  windows-pool: 'PrWin-2021-11-10'
-  linux-pool: 'PrLin-2021-11-10'
+  linux-pool: 'PrLin-2021-12-13'
  osx-pool: 'PrOsx-2021-07-27'

 jobs:
--- a/scripts/azure-pipelines/linux/provision-image.sh
+++ b/scripts/azure-pipelines/linux/provision-image.sh
@ -34,7 +34,7 @@ APT_PACKAGES="$APT_PACKAGES libxcb-res0-dev"
 APT_PACKAGES="$APT_PACKAGES python3-setuptools python3-mako"

 # Additionally required by some packages to install additional python packages
-APT_PACKAGES="$APT_PACKAGES python3-pip"
+APT_PACKAGES="$APT_PACKAGES python3-pip python3-venv"

 # Additionally required by qtwebengine
 APT_PACKAGES="$APT_PACKAGES nodejs"
--- a/versions/baseline.json
+++ b/versions/baseline.json
@ -6694,7 +6694,7 @@
    },
    "tensorflow-common": {
      "baseline": "2.6.0",
-      "port-version": 0
+      "port-version": 1
    },
    "tensorpipe": {
      "baseline": "2021-04-26",
--- a/versions/t-/tensorflow-common.json
+++ b/versions/t-/tensorflow-common.json
@ -1,5 +1,10 @@
 {
  "versions": [
+    {
+      "git-tree": "e766afec982de89398663f587da490fc8996474f",
+      "version-semver": "2.6.0",
+      "port-version": 1
+    },
    {
      "git-tree": "84662f791eb9d38350a4ffe99cd80d49f562b10f",
      "version-semver": "2.6.0",