Use pinned commit PyTorch on CI instead of nightly (pytorch#6564)

* Use pinned commit PyTorch on CI instead of nightly

* Need to use a default value

* Fix CI failures

* Update PT pin

* Fix libstdc++ installation

* Use a higher timeout for some jobs

* Fix review comments

* Minor format fix

* Add back the comment

.ci/docker/ci_commit_pins/pytorch.txt

@@ -1 +1 @@
-bd5482c7c3e1197e10c46ff739027f917d9c1fcc
+23d590e518688f96e1d1947a08e9ca27df3e67e4

.ci/docker/common/install_clang.sh

@@ -13,7 +13,7 @@ install_ubuntu() {
   apt-get install -y --no-install-recommends clang-"$CLANG_VERSION"
   apt-get install -y --no-install-recommends llvm-"$CLANG_VERSION"
   # Also require LLD linker from llvm and libomp to build PyTorch from source
-  apt-get install -y lld "libomp-${CLANG_VERSION}-dev"
+  apt-get install -y lld "libomp-${CLANG_VERSION}-dev" "libc++-${CLANG_VERSION}-dev"
 
   # Use update-alternatives to make this version the default
   update-alternatives --install /usr/bin/clang clang /usr/bin/clang-"$CLANG_VERSION" 50

.ci/docker/requirements-ci.txt

@@ -1,5 +1,5 @@
 mpmath==1.3.0
-numpy==1.22.0; python_version == '3.10'
+numpy==1.21.3; python_version == '3.10'
 numpy==1.23.2; python_version == '3.11'
 numpy; python_version >= '3.12'
 PyYAML==6.0.1

.ci/scripts/setup-linux.sh

@@ -19,6 +19,7 @@ else
 fi
 
 # As Linux job is running inside a Docker container, all of its dependencies
-# have already been installed
-install_executorch
+# have already been installed, so we use PyTorch build from source here instead
+# of nightly. This allows CI to test against latest commits from PyTorch
+install_executorch "use-pt-pinned-commit"
 build_executorch_runner "${BUILD_TOOL}"

.ci/scripts/setup-qnn-deps.sh

@@ -31,8 +31,9 @@ install_qnn() {
 }
 
 setup_libc++() {
+  clang_version=$1
   sudo apt-get update
-  pkgs_to_check=('libc++-dev')
+  pkgs_to_check=("libc++-${clang_version}-dev")
   j=0
   while [ $j -lt ${#pkgs_to_check[*]} ]; do
     install_status=$(verify_pkg_installed ${pkgs_to_check[$j]})
@@ -47,5 +48,6 @@ setup_libc++() {
   done
 }
 
-setup_libc++
+# This needs to match with the clang version from the Docker image
+setup_libc++ 12
 install_qnn

.ci/scripts/utils.sh

@@ -20,8 +20,11 @@ install_executorch() {
   which pip
   # Install executorch, this assumes that Executorch is checked out in the
   # current directory.
-  # TODO(T199538337): clean up install scripts to use install_requirements.sh
-  ./install_requirements.sh --pybind xnnpack
+  if [[ "${1:-}" == "use-pt-pinned-commit" ]]; then
+    ./install_requirements.sh --pybind xnnpack --use-pt-pinned-commit
+  else
+    ./install_requirements.sh --pybind xnnpack
+  fi
   # Just print out the list of packages for debugging
   pip list
 }

.github/workflows/trunk.yml

@@ -137,6 +137,7 @@ jobs:
       docker-image: executorch-ubuntu-22.04-arm-sdk
       submodules: 'true'
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
+      timeout: 90
       script: |
         # The generic Linux job chooses to use base env, not the one setup by the image
         CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")
@@ -162,6 +163,7 @@ jobs:
       docker-image: executorch-ubuntu-22.04-arm-sdk
       submodules: 'true'
       ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }}
+      timeout: 90
       script: |
         # The generic Linux job chooses to use base env, not the one setup by the image
         CONDA_ENV=$(conda env list --json | jq -r ".envs | .[-1]")

install_requirements.py

@@ -69,6 +69,7 @@ def python_is_compatible():
 EXECUTORCH_BUILD_PYBIND = "OFF"
 CMAKE_ARGS = os.getenv("CMAKE_ARGS", "")
 CMAKE_BUILD_ARGS = os.getenv("CMAKE_BUILD_ARGS", "")
+USE_PYTORCH_NIGHTLY = True
 
 for arg in sys.argv[1:]:
     if arg == "--pybind":
@@ -90,6 +91,11 @@ def python_is_compatible():
             shutil.rmtree(d, ignore_errors=True)
         print("Done cleaning build artifacts.")
         sys.exit(0)
+    elif arg == "--use-pt-pinned-commit":
+        # This option is used in CI to make sure that PyTorch build from the pinned commit
+        # is used instead of nightly. CI jobs wouldn't be able to catch regression from the
+        # latest PT commit otherwise
+        USE_PYTORCH_NIGHTLY = False
     else:
         print(f"Error: Unknown option {arg}")
         sys.exit(1)
@@ -113,11 +119,27 @@ def python_is_compatible():
 
 # pip packages needed by exir.
 EXIR_REQUIREMENTS = [
-    f"torch==2.6.0.{NIGHTLY_VERSION}",
-    f"torchvision==0.20.0.{NIGHTLY_VERSION}",  # For testing.
+    # Setting USE_PYTORCH_NIGHTLY to false to test the pinned PyTorch commit. Note
+    # that we don't need to set any version number there because they have already
+    # been installed on CI before this step, so pip won't reinstall them
+    f"torch==2.6.0.{NIGHTLY_VERSION}" if USE_PYTORCH_NIGHTLY else "torch",
+    (
+        f"torchvision==0.20.0.{NIGHTLY_VERSION}"
+        if USE_PYTORCH_NIGHTLY
+        else "torchvision"
+    ),  # For testing.
     "typing-extensions",
 ]
 
+# pip packages needed to run examples.
+# TODO: Make each example publish its own requirements.txt
+EXAMPLES_REQUIREMENTS = [
+    "timm==1.0.7",
+    f"torchaudio==2.5.0.{NIGHTLY_VERSION}" if USE_PYTORCH_NIGHTLY else "torchaudio",
+    "torchsr==1.0.4",
+    "transformers==4.42.4",
+]
+
 # pip packages needed for development.
 DEVEL_REQUIREMENTS = [
     "cmake",  # For building binary targets.
@@ -129,15 +151,6 @@ def python_is_compatible():
     "zstd",  # Imported by resolve_buck.py.
 ]
 
-# pip packages needed to run examples.
-# TODO: Make each example publish its own requirements.txt
-EXAMPLES_REQUIREMENTS = [
-    "timm==1.0.7",
-    f"torchaudio==2.5.0.{NIGHTLY_VERSION}",
-    "torchsr==1.0.4",
-    "transformers==4.42.4",
-]
-
 # Assemble the list of requirements to actually install.
 # TODO: Add options for reducing the number of requirements.
 REQUIREMENTS_TO_INSTALL = EXIR_REQUIREMENTS + DEVEL_REQUIREMENTS + EXAMPLES_REQUIREMENTS