Remove dev comments + reset logging levels

.buildkite/pipeline.build.yml

@@ -14,7 +14,6 @@
   commands:
     # Build the wheels and jars
     - UPLOAD_WHEELS_AS_ARTIFACTS=1 LINUX_WHEELS=1 LINUX_JARS=1 ./ci/ci.sh build
-    - bash ./java/build-jar-multiplatform.sh linux
     # Upload the wheels and jars
     # We don't want to push on PRs, in fact, the copy_files will fail because unauthenticated.
     - if [ "$BUILDKITE_PULL_REQUEST" != "false" ]; then exit 0; fi
@@ -578,7 +577,8 @@
     # Horovod needs to be installed separately (needed for API ref imports)
     - ./ci/env/install-horovod.sh
     # See https://stackoverflow.com/questions/63383400/error-cannot-uninstall-ruamel-yaml-while-creating-docker-image-for-azure-ml-a
-    - pip install mosaicml==0.12.1 --ignore-installed
+    # Pin urllib to avoid downstream ssl incompatibility issues. This matches requirements-doc.txt.
+    - pip install "mosaicml==0.12.1" "urllib3<1.27" --ignore-installed
     - ./ci/ci.sh build
 
 - label: ":octopus: Tune multinode tests"

.buildkite/pipeline.gpu_large.yml

@@ -48,7 +48,11 @@
     - DOC_TESTING=1 TRAIN_TESTING=1 TUNE_TESTING=1 ./ci/env/install-dependencies.sh
     - pip install -Ur ./python/requirements/ml/requirements_ml_docker.txt
     - ./ci/env/env_info.sh
-    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=gpu,-timeseries_libs,-py37,-post_wheel_build doc/...
+    # Test examples with newer version of `transformers`
+    # TODO(amogkam): Remove when https://github.com/ray-project/ray/issues/36011 
+    # is resolved.
+    - pip install -U transformers
+    - bazel test --config=ci $(./ci/run/bazel_export_options) --build_tests_only --test_tag_filters=gpu,-timeseries_libs,-py37,-post_wheel_build,-doctest doc/...
 
 - label: ":zap: :python: Lightning 2.0 Train GPU tests"
   conditions:

BUILD.bazel

@@ -2619,6 +2619,31 @@ cc_test(
     ],
 )
 
+cc_test(
+    name = "chaos_redis_store_client_test",
+    size = "small",
+    srcs = ["src/ray/gcs/store_client/test/redis_store_client_test.cc"],
+    args = [
+        "$(location redis-server)",
+        "$(location redis-cli)",
+    ],
+    copts = COPTS,
+    data = [
+        "//:redis-cli",
+        "//:redis-server",
+    ],
+    env = {"REDIS_CHAOS": "1"},
+    tags = ["team:core"],
+    target_compatible_with = [
+        "@platforms//os:linux",
+    ],
+    deps = [
+        ":redis_store_client",
+        ":store_client_test_lib",
+        "@com_google_googletest//:gtest_main",
+    ],
+)
+
 cc_test(
     name = "in_memory_store_client_test",
     size = "small",

build-docker.sh

@@ -8,7 +8,7 @@ set -x
 
 GPU=""
 BASE_IMAGE="ubuntu:focal"
-WHEEL_URL="https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-3.0.0.dev0-cp37-cp37m-manylinux2014_x86_64.whl"
+WHEEL_URL="https://s3-us-west-2.amazonaws.com/ray-wheels/latest/ray-2.5.1-cp37-cp37m-manylinux2014_x86_64.whl"
 PYTHON_VERSION="3.7.16"
 
 

ci/build/build-docker-images.py

@@ -6,7 +6,6 @@
 import itertools
 import os
 import platform
-import re
 import shutil
 import subprocess
 import sys
@@ -97,16 +96,11 @@ def _get_branch():
 
 def _release_build():
     branch = _get_branch()
-    if branch is None:
-        return False
-    return branch != "master" and branch.startswith("releases")
+    return branch and branch.startswith("releases/")
 
 
 def _valid_branch():
-    branch = _get_branch()
-    if branch is None:
-        return False
-    return branch == "master" or _release_build()
+    return _get_branch() == "master" or _release_build()
 
 
 def _get_curr_dir():
@@ -601,13 +595,12 @@ def push_and_tag_images(
     image_list: Optional[List[str]] = None,
     suffix: Optional[str] = None,
 ):
-
     date_tag = datetime.datetime.now().strftime("%Y-%m-%d")
     sha_tag = _get_commit_sha()
     if _release_build():
-        release_name = re.search("[0-9]+\.[0-9]+\.[0-9].*", _get_branch()).group(0)
-        date_tag = release_name
-        sha_tag = release_name
+        release_name = _get_branch()[len("releases/") :]
+        date_tag = release_name + "." + date_tag
+        sha_tag = release_name + "." + sha_tag
 
     for image_name in image_list:
         full_image_name = f"rayproject/{image_name}"

ci/ci.sh

@@ -192,6 +192,7 @@ test_python() {
       -python/ray/tests:test_job
       -python/ray/tests:test_memstat
       -python/ray/tests:test_multi_node_3
+      -python/ray/tests:test_multiprocessing_client_mode # Flaky on Windows
       -python/ray/tests:test_object_manager # OOM on test_object_directory_basic
       -python/ray/tests:test_resource_demand_scheduler
       -python/ray/tests:test_stress  # timeout
@@ -434,7 +435,7 @@ validate_wheels_commit_str() {
   echo "All wheels passed the sanity check and have the correct wheel commit set."
 }
 
-build_wheels() {
+build_wheels_and_jars() {
   # Create wheel output directory and empty contents
   # If buildkite runners are re-used, wheels from previous builds might be here, so we delete them.
   mkdir -p .whl
@@ -465,11 +466,18 @@ build_wheels() {
       IMAGE_NAME="quay.io/pypa/manylinux2014_${HOSTTYPE}"
       IMAGE_TAG="2022-12-20-b4884d9"
 
+      local MOUNT_ENV=()
+      if [ "${LINUX_JARS-}" == "1" ]; then
+        MOUNT_ENV+=(
+          -e "BUILD_JAR=1"
+        )
+      fi
+
       if [ -z "${BUILDKITE-}" ]; then
         # This command should be kept in sync with ray/python/README-building-wheels.md,
         # except the "${MOUNT_BAZEL_CACHE[@]}" part.
         docker run --rm -w /ray -v "${PWD}":/ray "${MOUNT_BAZEL_CACHE[@]}" \
-        "${IMAGE_NAME}:${IMAGE_TAG}" /ray/python/build-wheel-manylinux2014.sh
+          "${MOUNT_ENV[@]}" "${IMAGE_NAME}:${IMAGE_TAG}" /ray/python/build-wheel-manylinux2014.sh
       else
         rm -rf /ray-mount/*
         rm -rf /ray-mount/.whl || true
@@ -479,7 +487,7 @@ build_wheels() {
         docker run --rm -v /ray:/ray-mounted ubuntu:focal ls /
         docker run --rm -v /ray:/ray-mounted ubuntu:focal ls /ray-mounted
         docker run --rm -w /ray -v /ray:/ray "${MOUNT_BAZEL_CACHE[@]}" \
-          "${IMAGE_NAME}:${IMAGE_TAG}" /ray/python/build-wheel-manylinux2014.sh
+          "${MOUNT_ENV[@]}" "${IMAGE_NAME}:${IMAGE_TAG}" /ray/python/build-wheel-manylinux2014.sh
         cp -rT /ray-mount /ray # copy new files back here
         find . | grep whl # testing
 
@@ -767,7 +775,7 @@ build() {
   fi
 
   if [[ "${NEED_WHEELS}" == "true" ]]; then
-    build_wheels
+    build_wheels_and_jars
   fi
 }
 

cpp/src/ray/runtime/task/local_mode_task_submitter.cc

@@ -61,6 +61,7 @@ ObjectID LocalModeTaskSubmitter::Submit(InvocationSpec &invocation,
                             address,
                             1,
                             /*returns_dynamic=*/false,
+                            /*is_streaming_generator*/ false,
                             required_resources,
                             required_placement_resources,
                             "",

cpp/src/ray/runtime/task/task_executor.cc

@@ -135,7 +135,8 @@ Status TaskExecutor::ExecuteTask(
     std::string *application_error,
     const std::vector<ConcurrencyGroup> &defined_concurrency_groups,
     const std::string name_of_concurrency_group_to_execute,
-    bool is_reattempt) {
+    bool is_reattempt,
+    bool is_streaming_generator) {
   RAY_LOG(DEBUG) << "Execute task type: " << TaskType_Name(task_type)
                  << " name:" << task_name;
   RAY_CHECK(ray_function.GetLanguage() == ray::Language::CPP);

cpp/src/ray/runtime/task/task_executor.h

@@ -91,7 +91,8 @@ class TaskExecutor {
       std::string *application_error,
       const std::vector<ConcurrencyGroup> &defined_concurrency_groups,
       const std::string name_of_concurrency_group_to_execute,
-      bool is_reattempt);
+      bool is_reattempt,
+      bool is_streaming_generator);
 
   virtual ~TaskExecutor(){};
 

dashboard/client/src/pages/job/JobDetailInfoPage.tsx

@@ -99,7 +99,18 @@ export const JobMetadataSection = ({ job }: JobMetadataSectionProps) => {
         },
         {
           label: "Status",
-          content: <JobStatusWithIcon job={job} />,
+          content: (
+            <React.Fragment>
+              <JobStatusWithIcon job={job} />{" "}
+              {job.message && (
+                <CodeDialogButton
+                  title="Status details"
+                  code={job.message}
+                  buttonText="View details"
+                />
+              )}
+            </React.Fragment>
+          ),
         },
         {
           label: "Job ID",

dashboard/client/src/pages/job/JobRow.tsx

@@ -2,6 +2,7 @@ import { Link, TableCell, TableRow, Tooltip } from "@material-ui/core";
 import { makeStyles } from "@material-ui/core/styles";
 import React from "react";
 import { Link as RouterLink } from "react-router-dom";
+import { CodeDialogButtonWithPreview } from "../../common/CodeDialogButton";
 import { DurationText } from "../../common/DurationText";
 import { formatDateFromTimeMs } from "../../common/formatUtils";
 import { JobStatusWithIcon } from "../../common/JobStatus";
@@ -22,6 +23,10 @@ const useStyles = makeStyles((theme) => ({
     overflow: "hidden",
     whiteSpace: "nowrap",
   },
+  statusMessage: {
+    maxWidth: 250,
+    display: "inline-flex",
+  },
 }));
 
 type JobRowProps = {
@@ -34,6 +39,7 @@ export const JobRow = ({ job }: JobRowProps) => {
     submission_id,
     driver_info,
     status,
+    message,
     start_time,
     end_time,
     entrypoint,
@@ -87,6 +93,17 @@ export const JobRow = ({ job }: JobRowProps) => {
       <TableCell align="center">
         <JobStatusWithIcon job={job} />
       </TableCell>
+      <TableCell align="center">
+        {message ? (
+          <CodeDialogButtonWithPreview
+            className={classes.statusMessage}
+            title="Status message"
+            code={message}
+          />
+        ) : (
+          "-"
+        )}
+      </TableCell>
       <TableCell align="center">
         {start_time && start_time > 0 ? (
           <DurationText startTime={start_time} endTime={end_time} />

dashboard/client/src/pages/job/index.tsx

@@ -39,6 +39,7 @@ const columns = [
   { label: "Submission ID" },
   { label: "Entrypoint" },
   { label: "Status" },
+  { label: "Status message" },
   { label: "Duration" },
   {
     label: "Tasks",

dashboard/client/src/pages/serve/ServeApplicationDetailPage.tsx

@@ -36,6 +36,10 @@ const useStyles = makeStyles((theme) =>
     helpInfo: {
       marginLeft: theme.spacing(1),
     },
+    statusMessage: {
+      display: "inline-flex",
+      maxWidth: "100%",
+    },
   }),
 );
 
@@ -91,7 +95,19 @@ export const ServeApplicationDetailPage = () => {
           {
             label: "Status",
             content: (
-              <StatusChip type="serveApplication" status={application.status} />
+              <React.Fragment>
+                <StatusChip
+                  type="serveApplication"
+                  status={application.status}
+                />{" "}
+                {application.message && (
+                  <CodeDialogButton
+                    title="Status details"
+                    code={application.message}
+                    buttonText="View details"
+                  />
+                )}
+              </React.Fragment>
             ),
           },
           {

dashboard/client/src/pages/serve/ServeApplicationsListPage.component.test.tsx

@@ -1,5 +1,4 @@
 import { render, screen } from "@testing-library/react";
-import userEvent from "@testing-library/user-event";
 import React from "react";
 import { getActor } from "../../service/actor";
 import { getServeApplications } from "../../service/serve";
@@ -19,7 +18,7 @@ const mockGetActor = jest.mocked(getActor);
 
 describe("ServeApplicationsListPage", () => {
   it("renders list", async () => {
-    expect.assertions(15);
+    expect.assertions(14);
 
     // Mock ServeController actor fetch
     mockGetActor.mockResolvedValue({
@@ -81,8 +80,6 @@ describe("ServeApplicationsListPage", () => {
 
     render(<ServeApplicationsListPage />, { wrapper: TEST_APP_WRAPPER });
 
-    const user = userEvent.setup();
-
     await screen.findByText("System");
     expect(screen.getByText("System")).toBeVisible();
     expect(screen.getByText("1.2.3.4")).toBeVisible();
@@ -106,11 +103,6 @@ describe("ServeApplicationsListPage", () => {
     expect(screen.getByText("/second-app")).toBeVisible();
     expect(screen.getByText("DEPLOYING")).toBeVisible();
 
-    // Config dialog
-    await user.click(screen.getAllByText("View")[0]);
-    await screen.findByText(/import_path: home:graph/);
-    expect(screen.getByText(/import_path: home:graph/)).toBeVisible();
-
     expect(screen.getByText("Metrics")).toBeVisible();
   });
 });

dashboard/client/src/pages/serve/ServeDeploymentRow.tsx

@@ -33,6 +33,10 @@ const useStyles = makeStyles((theme) =>
       fontSize: "1.5em",
       verticalAlign: "middle",
     },
+    statusMessage: {
+      maxWidth: 400,
+      display: "inline-flex",
+    },
   }),
 );
 
@@ -100,6 +104,7 @@ export const ServeDeploymentRow = ({
         <TableCell align="center">
           {message ? (
             <CodeDialogButtonWithPreview
+              className={classes.statusMessage}
               title="Message details"
               code={message}
             />

dashboard/client/src/pages/serve/ServeSystemActorDetailPage.tsx

@@ -145,7 +145,10 @@ export const ServeSystemActorDetail = ({
             label: "Status",
             content:
               actor.type === "httpProxy" ? (
-                <StatusChip type="serveReplica" status={actor.detail.status} />
+                <StatusChip
+                  type="serveHttpProxy"
+                  status={actor.detail.status}
+                />
               ) : fetchedActor ? (
                 <StatusChip
                   type="serveController"