[ROCm] HIP stream priority fix post pytorch#101956 (pytorch#106157)

PR pytorch#101956 introduced additional stream priorities for cuda streams. HIP streams have slightly different semantics.
- HIP: 1=low, 0=default, -1=high
- CUDA: 0=default, -1=high, -2=higher, etc.

This PR forces HIP stream priority to just 0 and -1 to match the pytorch semantics.

This fixes a broken unit test.

```
python3 test_cuda_multigpu.py TestCudaMultiGPU.test_streams_priority -v

Test results will be stored in test-reports/python-unittest/test_cuda_multigpu

Running tests...
----------------------------------------------------------------------
  test_streams_priority (__main__.TestCudaMultiGPU) ... ERROR (0.200s)

======================================================================
ERROR [0.200s]: test_streams_priority (__main__.TestCudaMultiGPU)
----------------------------------------------------------------------
Traceback (most recent call last):
  File "/opt/conda/envs/py_3.8/lib/python3.8/site-packages/torch/testing/_internal/common_utils.py", line 2354, in wrapper
    method(*args, **kwargs)
  File "test_cuda_multigpu.py", line 656, in test_streams_priority
    low, high = torch.cuda.Stream.priority_range()
RuntimeError: least_priority == 0 INTERNAL ASSERT FAILED at "/var/lib/jenkins/pytorch-upstream/c10/hip/HIPStream.h":184, please report a bug to PyTorch. Unexpected HIP stream priority range
```

Pull Request resolved: pytorch#106157
Approved by: https://github.com/malfet

c10/cuda/CUDAStream.cpp

@@ -163,6 +163,13 @@ static void initGlobalStreamState() {
   int leastPriority = -1, greatestPriority = -1;
   C10_CUDA_CHECK(
       cudaDeviceGetStreamPriorityRange(&leastPriority, &greatestPriority));
+  // Note [HIP stream priorities]
+  // HIP stream priorities are 1=low, 0=default, -1=high which differs from CUDA
+  // which is 0=default, -1=high, -2=higher etc.
+  // Clamp leastPriority to 0 for HIP.
+#ifdef USE_ROCM
+  leastPriority = 0;
+#endif
   // greatestPriority is negative
   auto range = leastPriority - greatestPriority + 1;
   max_stream_priorities = range >= c10::cuda::max_compile_time_stream_priorities

c10/cuda/CUDAStream.h

@@ -180,8 +180,15 @@ class C10_CUDA_API CUDAStream {
     int least_priority, greatest_priority;
     C10_CUDA_CHECK(
         cudaDeviceGetStreamPriorityRange(&least_priority, &greatest_priority));
+#ifdef USE_ROCM
+    // See Note [HIP stream priorities]
+    TORCH_INTERNAL_ASSERT(
+        least_priority == 1, "Unexpected HIP stream priority range");
+    least_priority = 0;
+#else
     TORCH_INTERNAL_ASSERT(
         least_priority == 0, "Unexpected CUDA stream priority range");
+#endif
     TORCH_INTERNAL_ASSERT(
         greatest_priority <= -1, "Unexpected CUDA stream priority range");
     greatest_priority = std::max(