feat(ui): Add stackdriver links to logs tab (#383)

* Add stackdriver panel to logs tabs

* Add missing stackdriver url util method

* Add file ref to point the source of the code to Merlin

* Remove leftover print statement

* Fix difficult to read alignment of if conditions

* Expose appConfigs allow utils to use them

ui/src/components/pod_logs_viewer/PodLogsViewer.js

@@ -4,6 +4,9 @@ import {
   EuiFlexItem,
   EuiSearchBar,
   EuiSpacer,
+  EuiPanel,
+  EuiText,
+  EuiLink
 } from "@elastic/eui";
 import { LazyLog, ScrollFollow } from "react-lazylog";
 import { slugify } from "@caraml-dev/ui-lib";
@@ -17,6 +20,7 @@ export const PodLogsViewer = ({
   query,
   onQueryChange,
   batchSize,
+  stackdriverUrls,
 }) => {
   const filters = useMemo(
     () => [
@@ -110,33 +114,59 @@ export const PodLogsViewer = ({
   };
 
   return (
-    <EuiFlexGroup
-      direction="column"
-      gutterSize="none"
-      className="euiFlexGroup---logsContainer">
-      <EuiFlexItem grow={false}>
-        <EuiSearchBar {...search} />
-      </EuiFlexItem>
-      <EuiFlexItem grow={false}>
-        <EuiSpacer size="s" />
-      </EuiFlexItem>
-      <EuiFlexItem grow={true}>
-        <ScrollFollow
-          startFollowing={true}
-          render={({ onScroll, follow }) => (
-            <LazyLog
-              key={slugify(searchQuery)}
-              eventSource={emitter}
-              extraLines={1}
-              onScroll={onScroll}
-              follow={follow}
-              caseInsensitive
-              enableSearch
-              selectableLines
+    <>
+      {
+        Object.keys(stackdriverUrls).length !== 0 &&
+        (
+          <>
+            <EuiPanel>
+              <EuiFlexGroup direction="row" alignItems="center">
+                <EuiFlexItem style={{marginTop:0, marginBottom:0}} grow={false}>
+                  <EuiText  style={{ fontSize: '14px', fontWeight:"bold"}}>Stackdriver Logs</EuiText>
+                </EuiFlexItem>
+                {Object.entries(stackdriverUrls).map(([component,url])=> (
+                  <EuiFlexItem style={{marginTop:0, marginBottom:0, paddingLeft:"10px", textTransform: "capitalize"}} key={component} grow={false}>
+                    <EuiText size="xs" >
+                      <EuiLink href={url} target="_blank" external>{component.replace(new RegExp("_", "g"), " ")}</EuiLink>
+                    </EuiText>
+                  </EuiFlexItem>
+                ))}
+              </EuiFlexGroup>
+            </EuiPanel>
+            <EuiSpacer size="s" />
+          </>
+        )
+      }
+      <EuiPanel>
+        <EuiFlexGroup
+          direction="column"
+          gutterSize="none"
+          className="euiFlexGroup---logsContainer">
+          <EuiFlexItem grow={false}>
+            <EuiSearchBar {...search} />
+          </EuiFlexItem>
+          <EuiFlexItem grow={false}>
+            <EuiSpacer size="s" />
+          </EuiFlexItem>
+          <EuiFlexItem grow={true}>
+            <ScrollFollow
+              startFollowing={true}
+              render={({ onScroll, follow }) => (
+                <LazyLog
+                  key={slugify(searchQuery)}
+                  eventSource={emitter}
+                  extraLines={1}
+                  onScroll={onScroll}
+                  follow={follow}
+                  caseInsensitive
+                  enableSearch
+                  selectableLines
+                />
+              )}
             />
-          )}
-        />
-      </EuiFlexItem>
-    </EuiFlexGroup>
+          </EuiFlexItem>
+        </EuiFlexGroup>
+      </EuiPanel>
+    </>
   );
 };

ui/src/config.js

@@ -31,7 +31,7 @@ const authConfig = {
   oauthClientId: process.env.REACT_APP_OAUTH_CLIENT_ID,
 };
 
-const appConfig = {
+export const appConfig = {
   environment: process.env.REACT_APP_ENVIRONMENT,
   homepage: process.env.PUBLIC_URL,
   appIcon: "graphApp",
@@ -60,6 +60,11 @@ const appConfig = {
     // Default number of tail log entries to be fetched
     defaultTailLines: 1000,
   },
+  imagebuilder: {
+    cluster:  process.env.REACT_APP_IMAGE_BUILDER_CLUSTER,
+    gcp_project: process.env.REACT_APP_IMAGE_BUILDER_GCP_PROJECT,
+    namespace: process.env.REACT_APP_IMAGE_BUILDER_NAMESPACE,
+  },
   // Specifies a set of page templating configurations that will soon be controlled by the mlp-ui package
   // TO-DO: to review if these set of specifications are still needed after the update to mlp-ui
   pageTemplate: {

ui/src/router/details/RouterDetailsView.js

@@ -24,6 +24,7 @@ import { RouterLogsView } from "./logs/RouterLogsView";
 import { VersionComparisonView } from "../versions/comparison/VersionComparisonView";
 import { TuringRouter } from "../../services/router/TuringRouter";
 import { useConfig } from "../../config";
+import { EnsemblersContextProvider } from "../../providers/ensemblers/context";
 
 export const RouterDetailsView = () => {
   const { projectId, routerId, "*": section } = useParams();
@@ -131,7 +132,13 @@ export const RouterDetailsView = () => {
               {/* ALERTS */}
               <Route path="alerts/*" element={<RouterAlertsView router={router} />} />
               {/* LOGS */}
-              <Route path="logs" element={<RouterLogsView router={router} />} />
+              <Route path="logs" element={
+                <EnsemblersContextProvider
+                  projectId={router.project_id}
+                  ensemblerType={"pyfunc"}>
+                  <RouterLogsView router={router} />
+                </EnsemblersContextProvider>
+              } />
             </Routes>
           </EuiPageTemplate.Section>
         </Fragment>

ui/src/router/details/logs/RouterLogsView.js

@@ -1,12 +1,14 @@
-import React, { useEffect, useState } from "react";
+import React, { useContext, useEffect, useState } from "react";
 import { ConfigSection } from "../../../components/config_section";
 import { LogEntry } from "../../../services/logs/LogEntry";
 import { get } from "../../../components/form/utils";
-import { replaceBreadcrumbs } from "@caraml-dev/ui-lib";
+import { ProjectsContext, replaceBreadcrumbs } from "@caraml-dev/ui-lib";
 import { PodLogsViewer } from "../../../components/pod_logs_viewer/PodLogsViewer";
-import { EuiPanel } from "@elastic/eui";
 import { useConfig } from "../../../config";
 import { useLogsEmitter } from "../../../components/pod_logs_viewer/hooks/useLogsEmitter";
+import { createStackdriverUrl } from "../../../utils/createStackdriverUrl";
+import EnsemblersContext from "../../../providers/ensemblers/context";
+import EnvironmentsContext from "../../../providers/environments/context";
 
 const components = [
   {
@@ -24,9 +26,19 @@ const components = [
 ];
 
 export const RouterLogsView = ({ router }) => {
-  const {
-    appConfig: { podLogs: configOptions },
-  } = useConfig();
+  const { appConfig } = useConfig();
+
+  const { currentProject } = useContext(ProjectsContext);
+
+  const { ensemblers } = useContext(EnsemblersContext);
+  const ensembler = Object.values(ensemblers)
+      .find((value) => value.id === router?.config?.ensembler?.pyfunc_config?.ensembler_id)
+
+  const environments = useContext(EnvironmentsContext);
+  const environment = Object.values(environments)
+      .find((value) => value.name === router?.environment_name)
+
+  const [stackdriverUrls, setStackdriverUrls] = useState({});
 
   useEffect(() => {
     replaceBreadcrumbs([
@@ -46,7 +58,7 @@ export const RouterLogsView = ({ router }) => {
 
   const [query, setQuery] = useState({
     component_type: "router",
-    tail_lines: configOptions.defaultTailLines,
+    tail_lines: appConfig.podLogs.defaultTailLines,
   });
 
   const { emitter } = useLogsEmitter(
@@ -55,24 +67,77 @@ export const RouterLogsView = ({ router }) => {
     (entries) =>
       !!entries.length ? entries[entries.length - 1].timestamp : undefined,
     (entries) => entries.map((entry) => LogEntry.fromJson(entry).toString()),
-    configOptions
+      appConfig.podLogs,
   );
 
   const availableComponents = components.filter(
     (c) => c.value === "router" || !!get(router, `config.${c.value}`)
   );
 
+  useEffect(
+    () => {
+      let urls = {}
+      if (
+          appConfig.imagebuilder.cluster &&
+          appConfig.imagebuilder.gcp_project &&
+          appConfig.imagebuilder.namespace &&
+          currentProject
+      ) {
+        // set router url
+        urls["router"] = createStackdriverUrl({
+          gcp_project: environment.gcp_project,
+          cluster: environment.cluster,
+          namespace: currentProject.name,
+          pod_name: router.name + "-turing-router-"  + router.config.version,
+          start_time: router.updated_at,
+        }, "router")
+
+        // set enricher url
+        if (router.config.enricher.type === "docker") {
+          urls["enricher"] = createStackdriverUrl({
+            gcp_project: environment.gcp_project,
+            cluster: environment.cluster,
+            namespace: currentProject.name,
+            pod_name: router.name + "-turing-enricher-" + router.config.version,
+            start_time: router.updated_at,
+          }, "enricher")
+        }
+
+        // set ensembler url
+        if (router.config.ensembler.type === "docker" || router.config.ensembler.type === "pyfunc") {
+          urls["ensembler"] = createStackdriverUrl({
+            gcp_project: environment.gcp_project,
+            cluster: environment.cluster,
+            namespace: currentProject.name,
+            pod_name: router.name + "-turing-ensembler-" + router.config.version,
+            start_time: router.updated_at,
+          }, "ensembler")
+        }
+
+        // set image builder url
+        if (router.config.ensembler.type === "pyfunc" && ensembler) {
+          urls["ensembler_image_builder"] = createStackdriverUrl({
+            job_name: "service-" + currentProject.name + "-" + ensembler.name,
+            start_time: ensembler.updated_at,
+          }, "ensembler_image_builder")
+        }
+
+        setStackdriverUrls(urls);
+      }
+    },
+    [currentProject, ensembler, environment, router, appConfig.imagebuilder]
+  );
+
   return (
     <ConfigSection title="Logs">
-      <EuiPanel>
-        <PodLogsViewer
-          components={availableComponents}
-          emitter={emitter}
-          query={query}
-          onQueryChange={setQuery}
-          batchSize={configOptions.batchSize}
-        />
-      </EuiPanel>
+      <PodLogsViewer
+        components={availableComponents}
+        emitter={emitter}
+        query={query}
+        onQueryChange={setQuery}
+        batchSize={appConfig.podLogs.batchSize}
+        stackdriverUrls={stackdriverUrls}
+      />
     </ConfigSection>
   );
 };

ui/src/utils/createStackdriverUrl.js

@@ -0,0 +1,39 @@
+// This file in almost an exact copy of this file from Merlin
+// Ref: https://github.com/caraml-dev/merlin/blob/8edef22b29d0bfb2728d62b1f880f1f753f9509e/ui/src/utils/createStackdriverUrl.js
+import { appConfig } from "../config";
+
+const stackdriverAPI = "https://console.cloud.google.com/logs/viewer";
+
+const stackdriverFilter = query => {
+  return `resource.type:"k8s_query" OR "k8s_container" OR "k8s_pod"
+resource.labels.project_id:${query.gcp_project}
+resource.labels.cluster_name:${query.cluster}
+resource.labels.namespace_name:${query.namespace}
+resource.labels.pod_name:${query.pod_name}
+timestamp>"${query.start_time}"
+`;
+};
+
+const stackdriverImageBuilderFilter =  query => {
+  return `resource.type:"k8s_container"
+resource.labels.project_id:${appConfig.imagebuilder.gcp_project}
+resource.labels.cluster_name:${appConfig.imagebuilder.cluster}
+resource.labels.namespace_name:${appConfig.imagebuilder.namespace}
+labels.k8s-pod/job-name:${query.job_name}
+timestamp>"${query.start_time}"`;
+}
+
+export const createStackdriverUrl = (query, component) => {
+  const advanceFilter = component === "ensembler_image_builder" ? stackdriverImageBuilderFilter(query) : stackdriverFilter(query);
+
+  const url = {
+    project: query.gcp_project || appConfig.imagebuilder.gcp_project,
+    minLogLevel: 0,
+    expandAll: false,
+    advancedFilter: advanceFilter,
+  };
+
+  const stackdriverParams = new URLSearchParams(url).toString();
+  return stackdriverAPI + "?" + stackdriverParams;
+};
+