diff --git a/.ci/scripts/utils.sh b/.ci/scripts/utils.sh index e8b8bbc0d7f..ce42cb59662 100644 --- a/.ci/scripts/utils.sh +++ b/.ci/scripts/utils.sh @@ -46,8 +46,8 @@ install_pip_dependencies() { install_domains() { echo "Install torchvision and torchaudio" - pip install --no-use-pep517 --user "git+https://github.com/pytorch/audio.git@${TORCHAUDIO_VERSION}" - pip install --no-use-pep517 --user "git+https://github.com/pytorch/vision.git@${TORCHVISION_VERSION}" + pip install --no-build-isolation --user "git+https://github.com/pytorch/audio.git@${TORCHAUDIO_VERSION}" + pip install --no-build-isolation --user "git+https://github.com/pytorch/vision.git@${TORCHVISION_VERSION}" } install_pytorch_and_domains() { diff --git a/backends/apple/coreml/scripts/build_tests.sh b/backends/apple/coreml/scripts/build_tests.sh index 190adf1f65a..0203e5027a2 100755 --- a/backends/apple/coreml/scripts/build_tests.sh +++ b/backends/apple/coreml/scripts/build_tests.sh @@ -30,7 +30,8 @@ rm -rf "$CMAKE_EXECUTORCH_BUILD_DIR_PATH" cmake "$EXECUTORCH_ROOT_PATH" -B"$CMAKE_EXECUTORCH_BUILD_DIR_PATH" \ -DCMAKE_TOOLCHAIN_FILE="$IOS_TOOLCHAIN_PATH" \ --DPLATFORM=MAC_UNIVERSAL \ +-DPLATFORM=MAC_ARM64 \ +-DCMAKE_OSX_ARCHITECTURES=arm64 \ -DDEPLOYMENT_TARGET=13.0 \ -DEXECUTORCH_BUILD_EXECUTOR_RUNNER=OFF \ -DEXECUTORCH_BUILD_XNNPACK=OFF @@ -44,7 +45,8 @@ rm -rf "$CMAKE_PROTOBUF_BUILD_DIR_PATH" cmake "$PROTOBUF_DIR_PATH/cmake" -B"$CMAKE_PROTOBUF_BUILD_DIR_PATH" \ -DCMAKE_TOOLCHAIN_FILE="$IOS_TOOLCHAIN_PATH" \ --DPLATFORM=MAC_UNIVERSAL \ +-DPLATFORM=MAC_ARM64 \ +-DCMAKE_OSX_ARCHITECTURES=arm64 \ -DDEPLOYMENT_TARGET=13.0 \ -Dprotobuf_BUILD_TESTS=OFF \ -Dprotobuf_BUILD_EXAMPLES=OFF \ @@ -55,7 +57,8 @@ cmake --build "$CMAKE_PROTOBUF_BUILD_DIR_PATH" -j9 -t libprotobuf-lite # Copy required libraries echo "ExecuTorch: Copying libraries" -mkdir "$LIBRARIES_DIR_PATH" +rm -rf $LIBRARIES_DIR_PATH +mkdir -p "$LIBRARIES_DIR_PATH" cp -f "$CMAKE_EXECUTORCH_BUILD_DIR_PATH/libexecutorch.a" "$LIBRARIES_DIR_PATH" cp -f "$CMAKE_EXECUTORCH_BUILD_DIR_PATH/libexecutorch_core.a" "$LIBRARIES_DIR_PATH" cp -f "$CMAKE_PROTOBUF_BUILD_DIR_PATH/libprotobuf-lite.a" "$LIBRARIES_DIR_PATH" diff --git a/backends/apple/coreml/scripts/generate_test_models.sh b/backends/apple/coreml/scripts/generate_test_models.sh index 6a73d697379..bb5de781b5e 100755 --- a/backends/apple/coreml/scripts/generate_test_models.sh +++ b/backends/apple/coreml/scripts/generate_test_models.sh @@ -15,7 +15,9 @@ COREML_DIR_PATH="$EXECUTORCH_ROOT_PATH/backends/apple/coreml" cd "$EXECUTORCH_ROOT_PATH" -mkdir "$COREML_DIR_PATH/runtime/test/models/" +rm -rf "$COREML_DIR_PATH/runtime/test/models/" +mkdir -p "$COREML_DIR_PATH/runtime/test/models/" + #Generate models cd "$EXECUTORCH_ROOT_PATH" diff --git a/backends/apple/coreml/scripts/install_requirements.sh b/backends/apple/coreml/scripts/install_requirements.sh index 5ec1ea6a1de..f57df535d86 100755 --- a/backends/apple/coreml/scripts/install_requirements.sh +++ b/backends/apple/coreml/scripts/install_requirements.sh @@ -5,6 +5,8 @@ # This source code is licensed under the BSD-style license found in the # LICENSE file in the root directory of this source tree. +set -euo pipefail + SCRIPT_DIR_PATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 pwd -P @@ -12,10 +14,16 @@ SCRIPT_DIR_PATH="$( # TODO(jathu): remove the need to fetch coremltools to build deps for coreml_executor_runner. # Keep this version in sync with: pyproject.toml -COREMLTOOLS_VERSION="9.0b1" +COREMLTOOLS_VERSION="9.0" -red=`tput setaf 1` -green=`tput setaf 2` +# Safe colors (no TERM noise in CI) +if command -v tput >/dev/null 2>&1 && [ -t 1 ] && [ -n "${TERM:-}" ]; then + red="$(tput setaf 1)" + green="$(tput setaf 2)" + reset="$(tput sgr0)" +else + red=""; green=""; reset="" +fi EXECUTORCH_ROOT_PATH=$(realpath "$SCRIPT_DIR_PATH/../../../../") COREML_DIR_PATH="$EXECUTORCH_ROOT_PATH/backends/apple/coreml" @@ -25,30 +33,79 @@ PROTOBUF_FILES_DIR_PATH="$COREMLTOOLS_DIR_PATH/build/mlmodel/format/" cd "$EXECUTORCH_ROOT_PATH" rm -rf "$COREML_DIR_PATH/third-party" -mkdir "$COREML_DIR_PATH/third-party" +mkdir -p "$COREML_DIR_PATH/third-party" -echo "${green}ExecuTorch: Cloning coremltools." -git clone --depth 1 --branch "${COREMLTOOLS_VERSION}" "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/apple/coremltools.git" $COREMLTOOLS_DIR_PATH -cd $COREMLTOOLS_DIR_PATH +echo "${green}ExecuTorch: Cloning coremltools.${reset}" +git clone --depth 1 --branch "${COREMLTOOLS_VERSION}" "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/apple/coremltools.git" "$COREMLTOOLS_DIR_PATH" +cd "$COREMLTOOLS_DIR_PATH" STATUS=$? if [ $STATUS -ne 0 ]; then - echo "${red}ExecuTorch: Failed to clone coremltools." + echo "${red}ExecuTorch: Failed to clone coremltools.${reset}" exit 1 fi -echo "${green}ExecuTorch: Installing coremltools dependencies." -pip install -r "$COREMLTOOLS_DIR_PATH/reqs/build.pip" +# --------------------------------------------------------------------- +# Host toolchain / SDK setup JUST for coremltools build +# --------------------------------------------------------------------- +HOST_SDKROOT="${SDKROOT:-}" +HOST_CC="${CC:-}" +HOST_CXX="${CXX:-}" +HOST_CFLAGS="${CFLAGS:-}" +HOST_CXXFLAGS="${CXXFLAGS:-}" + +if [[ "$(uname)" == "Darwin" ]]; then + # Only pick macOS SDK if nothing else is specified + if [[ -z "$HOST_SDKROOT" ]]; then + HOST_SDKROOT="$(xcrun --sdk macosx --show-sdk-path)" + fi + if [[ -z "$HOST_CC" ]]; then + HOST_CC="$(xcrun --find clang)" + fi + if [[ -z "$HOST_CXX" ]]; then + HOST_CXX="$(xcrun --find clang++)" + fi + # Only add -isysroot if caller didn't already set CFLAGS/CXXFLAGS + if [[ -z "$HOST_CFLAGS" && -n "$HOST_SDKROOT" ]]; then + HOST_CFLAGS="-isysroot ${HOST_SDKROOT}" + fi + if [[ -z "$HOST_CXXFLAGS" && -n "$HOST_SDKROOT" ]]; then + HOST_CXXFLAGS="-isysroot ${HOST_SDKROOT}" + fi +fi + +echo "${green}ExecuTorch: Installing coremltools dependencies.${reset}" +SDKROOT="$HOST_SDKROOT" \ +CC="$HOST_CC" \ +CXX="$HOST_CXX" \ +CFLAGS="$HOST_CFLAGS" \ +CXXFLAGS="$HOST_CXXFLAGS" \ +python -m pip install -r "$COREMLTOOLS_DIR_PATH/reqs/build.pip" STATUS=$? if [ $STATUS -ne 0 ]; then - echo "${red}ExecuTorch: Failed to install coremltools dependencies." + echo "${red}ExecuTorch: Failed to install coremltools dependencies.${reset}" exit 1 fi -mkdir "$COREMLTOOLS_DIR_PATH/build" +mkdir -p "$COREMLTOOLS_DIR_PATH/build" + +echo "${green}ExecuTorch: Configuring coremltools CMake build.${reset}" +SDKROOT="$HOST_SDKROOT" \ +CC="$HOST_CC" \ +CXX="$HOST_CXX" \ +CFLAGS="$HOST_CFLAGS" \ +CXXFLAGS="$HOST_CXXFLAGS" \ cmake -S "$COREMLTOOLS_DIR_PATH" -B "$COREMLTOOLS_DIR_PATH/build" + +echo "${green}ExecuTorch: Building mlmodel target.${reset}" +SDKROOT="$HOST_SDKROOT" \ +CC="$HOST_CC" \ +CXX="$HOST_CXX" \ +CFLAGS="$HOST_CFLAGS" \ +CXXFLAGS="$HOST_CXXFLAGS" \ cmake --build "$COREMLTOOLS_DIR_PATH/build" --parallel --target mlmodel -echo "${green}ExecuTorch: Copying protobuf files." +echo "${green}ExecuTorch: Copying protobuf files.${reset}" +rm -rf "$COREML_DIR_PATH/runtime/sdk/format/" mkdir -p "$COREML_DIR_PATH/runtime/sdk/format/" cp -rf "$PROTOBUF_FILES_DIR_PATH" "$COREML_DIR_PATH/runtime/sdk/format/"