diff --git a/xla/python/BUILD b/xla/python/BUILD
index e848931e3f42b..8da625a4a4ace 100644
--- a/xla/python/BUILD
+++ b/xla/python/BUILD
@@ -333,9 +333,6 @@ cc_library(
         "-fexceptions",
         "-fno-strict-aliasing",
     ],
-    defines = if_cuda(["GOOGLE_CUDA=1"]) + if_rocm([
-        "TENSORFLOW_USE_ROCM=1",
-    ]),
     features = ["-use_header_modules"],
     deps = [
         ":aggregate_profile",
diff --git a/xla/python/py_client.cc b/xla/python/py_client.cc
index da4eca022ed73..72c3d9cb95929 100644
--- a/xla/python/py_client.cc
+++ b/xla/python/py_client.cc
@@ -94,10 +94,6 @@ limitations under the License.
 #include "tsl/platform/status.h"
 #include "tsl/platform/statusor.h"
 
-#if GOOGLE_CUDA || TENSORFLOW_USE_ROCM || TENSORFLOW_USE_SYCL
-#include "xla/python/py_client_gpu.h"
-#endif  // GOOGLE_CUDA || TENSORFLOW_USE_ROCM || TENSORFLOW_USE_SYCL
-
 namespace xla {
 
 namespace nb = nanobind;