feat: [CDE-626]: Replaced the log polling api with stream api to fetch realtime logs (#3631)

* fix: UI changes for the button scroll and alignment fix
* fix: fixed lint issuews
* feat: Replaced the log polling api with stream api to fetch realtime logs

web/src/cde-gitness/pages/GitspaceDetails/GitspaceDetails.module.scss

@@ -13,6 +13,7 @@
  * See the License for the specific language governing permissions and
  * limitations under the License.
  */
+@import 'src/utils/utils';
 
 .customSubheader {
   height: 10vh;
@@ -111,3 +112,45 @@
   width: 130px;
   justify-content: space-between !important;
 }
+
+.scrollDownBtn {
+  position: absolute;
+  padding: 8px !important;
+  bottom: 35px;
+  right: 30px;
+
+  & > :global(.bp3-icon) {
+    padding: 0 !important;
+  }
+
+  & > :global(.bp3-button-text) {
+    width: 0;
+    padding-left: 0;
+    overflow: hidden;
+    display: inline-block;
+  }
+
+  &:hover > :global(.bp3-button-text) {
+    width: auto;
+    padding-left: 4px;
+  }
+}
+
+.consoleContainer {
+  min-height: 120px !important;
+  max-height: 70vh !important;
+  overflow: scroll;
+  align-items: start !important;
+  padding-top: var(--spacing-large) !important;
+  padding-bottom: var(--spacing-large) !important;
+  padding-left: var(--spacing-xlarge) !important;
+  background-color: var(--black) !important;
+  color: var(--white) !important;
+  pre {
+    background-color: var(--black) !important;
+    color: var(--white) !important;
+    text-align: start !important;
+    font-size: 12px;
+    font-family: var(--font-family-mono) !important;
+  }
+}

web/src/cde-gitness/pages/GitspaceDetails/GitspaceDetails.module.scss.d.ts

@@ -18,6 +18,7 @@
 // This is an auto-generated file
 export declare const accordionnCustomSummary: string
 export declare const cardContainer: string
+export declare const consoleContainer: string
 export declare const containerlogsTitle: string
 export declare const copyBtn: string
 export declare const customSubheader: string
@@ -25,4 +26,5 @@ export declare const gitspaceIcon: string
 export declare const gitspaceIdContainer: string
 export declare const pageMain: string
 export declare const popover: string
+export declare const scrollDownBtn: string
 export declare const titleContainer: string

web/src/cde-gitness/pages/GitspaceDetails/GitspaceDetails.tsx

@@ -26,7 +26,8 @@ import {
   Page,
   Text,
   useToaster,
-  AccordionHandle
+  AccordionHandle,
+  ButtonSize
 } from '@harnessio/uicore'
 import { Play } from 'iconoir-react'
 import { useHistory, useParams } from 'react-router-dom'
@@ -53,12 +54,10 @@ import { useGitspaceDetails } from 'cde-gitness/hooks/useGitspaceDetails'
 import { useGitspaceEvents } from 'cde-gitness/hooks/useGitspaceEvents'
 import { useGitspaceActions } from 'cde-gitness/hooks/useGitspaceActions'
 import { useDeleteGitspaces } from 'cde-gitness/hooks/useDeleteGitspaces'
-import { useGitspacesLogs } from 'cde-gitness/hooks/useGitspaceLogs'
 import { useOpenVSCodeBrowserURL } from 'cde-gitness/hooks/useOpenVSCodeBrowserURL'
 import { ErrorCard } from 'cde-gitness/components/ErrorCard/ErrorCard'
 import CopyButton from 'cde-gitness/components/CopyButton/CopyButton'
-import ContainerLogs from '../../components/ContainerLogs/ContainerLogs'
-import { useGetLogStream } from '../../hooks/useGetLogStream'
+import Logger from './Logger/Logger'
 import css from './GitspaceDetails.module.scss'
 
 const GitspaceDetails = () => {
@@ -67,11 +66,15 @@ const GitspaceDetails = () => {
   const { routes, standalone } = useAppContext()
   const { showError, showSuccess } = useToaster()
   const history = useHistory()
+  const containerRef = useRef<HTMLDivElement | null>(null)
   const [startTriggred, setStartTriggred] = useState<boolean>(false)
   const [triggerPollingOnStart, setTriggerPollingOnStart] = useState<EnumGitspaceStateType>()
   const { gitspaceId = '' } = useParams<{ gitspaceId?: string }>()
 
+  const logCardId = 'logsCard'
+  const [expandedTab, setExpandedTab] = useState('')
   const [isStreamingLogs, setIsStreamingLogs] = useState(false)
+  const [isBottom, setIsBottom] = useState(false)
 
   const [startPolling, setStartPolling] = useState<GitspaceActionType | undefined>(undefined)
 
@@ -79,14 +82,6 @@ const GitspaceDetails = () => {
 
   const { data: eventData, refetch: refetchEventData } = useGitspaceEvents({ gitspaceId })
 
-  const {
-    data: responseData,
-    refetch: refetchLogsData,
-    response,
-    error: streamLogsError,
-    loading: logsLoading
-  } = useGitspacesLogs({ gitspaceId })
-
   const { mutate: actionMutate, loading: mutateLoading } = useGitspaceActions({ gitspaceId })
 
   const { mutate: deleteGitspace, loading: deleteLoading } = useDeleteGitspaces({ gitspaceId })
@@ -111,15 +106,13 @@ const GitspaceDetails = () => {
         defaultTo(item?.timestamp, 0) >= defaultTo(data?.instance?.updated, 0)
     )
     if (disabledActionButtons && filteredEvent?.length && !isStreamingLogs) {
-      refetchLogsData()
       setIsStreamingLogs(true)
-    } else if (
-      (filteredEvent?.length && !disabledActionButtons && isStreamingLogs) ||
-      (isStreamingLogs && streamLogsError)
-    ) {
+      viewLogs()
+    } else if (filteredEvent?.length && !disabledActionButtons && isStreamingLogs) {
       setIsStreamingLogs(false)
+      viewLogs()
     }
-  }, [eventData, data?.instance?.updated, disabledActionButtons, streamLogsError])
+  }, [eventData, data?.instance?.updated, disabledActionButtons])
 
   usePolling(
     async () => {
@@ -135,19 +128,6 @@ const GitspaceDetails = () => {
     }
   )
 
-  usePolling(
-    async () => {
-      if (!standalone) {
-        await refetchLogsData()
-      }
-    },
-    {
-      pollingInterval: 10000,
-      startCondition: (eventData?.[eventData?.length - 1]?.event as string) === 'agent_gitspace_creation_start',
-      stopCondition: pollingCondition
-    }
-  )
-
   useEffect(() => {
     const startTrigger = async () => {
       if (redirectFrom && !startTriggred && !mutateLoading) {
@@ -171,8 +151,6 @@ const GitspaceDetails = () => {
     }
   }, [data?.state, redirectFrom, mutateLoading, startTriggred])
 
-  const formattedlogsdata = useGetLogStream(standalone ? { response } : { response: undefined })
-
   const confirmDelete = useConfirmAct()
 
   const handleDelete = async (e: React.MouseEvent<HTMLDivElement, MouseEvent>) => {
@@ -203,14 +181,6 @@ const GitspaceDetails = () => {
   const myRef = useRef<any | null>(null)
   const selectedIde = getIDEOption(data?.ide, getString)
 
-  useEffect(() => {
-    if (standalone ? formattedlogsdata.data : responseData) {
-      accordionRef.current?.open('logsCard')
-    } else {
-      accordionRef.current?.close('logsCard')
-    }
-  }, [standalone, responseData, formattedlogsdata.data])
-
   const triggerGitspace = async () => {
     try {
       setStartPolling(GitspaceActionType.START)
@@ -227,6 +197,19 @@ const GitspaceDetails = () => {
   const viewLogs = () => {
     myRef.current?.scrollIntoView()
     accordionRef.current?.open('logsCard')
+    setExpandedTab('logsCard')
+  }
+
+  const handleClick = () => {
+    const logContainer = containerRef.current as HTMLDivElement
+    const scrollParent = logContainer?.parentElement as HTMLDivElement
+    if (!isBottom) {
+      scrollParent.scrollTop = scrollParent.scrollHeight
+      setIsBottom(true)
+    } else if (isBottom) {
+      scrollParent.scrollTop = 0
+      setIsBottom(false)
+    }
   }
 
   return (
@@ -451,16 +434,20 @@ const GitspaceDetails = () => {
           <Card className={css.cardContainer}>
             <EventTimelineAccordion data={eventData as TypesGitspaceEventResponse[]} />
           </Card>
-
           <Card className={css.cardContainer}>
             <Container ref={myRef}>
-              <Accordion activeId={''} ref={accordionRef}>
+              <Accordion
+                activeId={expandedTab}
+                ref={accordionRef}
+                onChange={(e: string) => {
+                  setExpandedTab(e)
+                }}>
                 <Accordion.Panel
                   className={css.accordionnCustomSummary}
                   summary={
                     <Layout.Vertical spacing="small">
                       <Text
-                        rightIcon={isStreamingLogs || logsLoading ? 'steps-spinner' : undefined}
+                        rightIcon={isStreamingLogs ? 'steps-spinner' : undefined}
                         className={css.containerlogsTitle}
                         font={{ variation: FontVariation.CARD_TITLE }}
                         margin={{ left: 'large' }}>
@@ -469,10 +456,27 @@ const GitspaceDetails = () => {
                       <Text margin={{ left: 'large' }}>{getString('cde.details.containerLogsSubText')} </Text>
                     </Layout.Vertical>
                   }
-                  id="logsCard"
+                  id={logCardId}
                   details={
-                    <Container width="100%">
-                      <ContainerLogs data={standalone ? formattedlogsdata.data : responseData} />
+                    <Container width="100%" className={css.consoleContainer}>
+                      <Logger
+                        value={data?.name ?? ''}
+                        state={data?.state ?? ''}
+                        logKey={data?.log_key ?? ''}
+                        isStreaming={isStreamingLogs}
+                        expanded={true}
+                        localRef={containerRef}
+                        setIsBottom={setIsBottom}
+                      />
+                      <Button
+                        size={ButtonSize.SMALL}
+                        variation={ButtonVariation.PRIMARY}
+                        text={isBottom ? getString('top') : getString('bottom')}
+                        icon={isBottom ? 'arrow-up' : 'arrow-down'}
+                        iconProps={{ size: 10 }}
+                        onClick={handleClick}
+                        className={css.scrollDownBtn}
+                      />
                     </Container>
                   }
                 />

web/src/cde-gitness/pages/GitspaceDetails/Logger/LogStreaming.tsx

@@ -0,0 +1,16 @@
+import React, { useEffect } from 'react'
+import { useAppContext } from 'AppContext'
+
+const LogStreaming: React.FC<any> = ({ logKeyList, onMessageStreaming, onError }: any) => {
+  const { hooks } = useAppContext()
+  const { subscribe, closeStream } = hooks?.useLogsStreaming(logKeyList, onMessageStreaming, onError)
+
+  useEffect(() => {
+    subscribe()
+
+    return () => closeStream()
+  }, [])
+  return <></>
+}
+
+export default LogStreaming

web/src/cde-gitness/pages/GitspaceDetails/Logger/Logger.module.scss

@@ -0,0 +1,106 @@
+@import 'src/utils/utils';
+
+.main {
+  flex-shrink: 0;
+
+  .line {
+    margin: 0;
+    padding: 0;
+    cursor: text;
+    min-height: 20px;
+    display: block;
+
+    @include mono-font;
+    color: var(--white);
+
+    word-wrap: break-word !important;
+    white-space: pre-wrap !important;
+  }
+}
+
+.pipelineSteps {
+  padding: 10px 20px 0 !important;
+  display: flex;
+  flex-direction: column;
+  gap: 5px;
+
+  &::before {
+    content: '';
+    height: 10px;
+    width: 100%;
+    background-color: var(--black);
+    position: absolute;
+    top: 64px;
+    z-index: 1;
+  }
+
+  .stepContainer {
+    display: flex;
+    flex-direction: column;
+    word-break: break-all;
+  }
+
+  .stepHeader {
+    display: flex;
+    align-items: center;
+    min-height: 34px;
+    border-radius: 6px;
+    padding: 0 10px 0 6px;
+    position: sticky;
+    top: 74px;
+    z-index: 2;
+    background-color: var(--black);
+
+    &.expanded {
+      .chevron {
+        transform: rotate(90deg);
+      }
+    }
+
+    .chevron {
+      transition: transform 0.2s ease;
+    }
+
+    &:hover {
+      background-color: #22222aa9;
+    }
+
+    &.selected {
+      background-color: #22222a;
+    }
+
+    &.selected .name {
+      color: var(--primary-7) !important;
+      font-weight: 600 !important;
+    }
+
+    .name {
+      color: #b0b1c3 !important;
+      font-weight: 400 !important;
+      font-size: 14px !important;
+      font-family: var(--font-family-mono);
+    }
+  }
+
+  .stepLogContainer {
+    padding: 15px 10px 15px 36px;
+    flex-shrink: 0;
+
+    .consoleLine {
+      color: var(--white);
+
+      @include mono-font;
+
+      word-wrap: break-word !important;
+      white-space: pre-wrap !important;
+      cursor: text;
+      margin: 0;
+      padding: 0;
+
+      &:empty {
+        display: inline-block;
+        min-height: 20px;
+      }
+    }
+  }
+}

web/src/cde-gitness/pages/GitspaceDetails/Logger/Logger.module.scss.d.ts

@@ -0,0 +1,29 @@
+/*
+ * Copyright 2023 Harness, Inc.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *     http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* eslint-disable */
+// This is an auto-generated file
+export declare const chevron: string
+export declare const consoleLine: string
+export declare const expanded: string
+export declare const line: string
+export declare const main: string
+export declare const name: string
+export declare const pipelineSteps: string
+export declare const selected: string
+export declare const stepContainer: string
+export declare const stepHeader: string
+export declare const stepLogContainer: string

web/src/cde-gitness/pages/GitspaceDetails/Logger/Logger.tsx

@@ -0,0 +1,104 @@
+import React, { useCallback, useEffect, useState } from 'react'
+import { Container } from '@harnessio/uicore'
+import cx from 'classnames'
+import { GitspaceStatus } from 'cde-gitness/constants'
+import { lineElement } from 'components/LogViewer/LogViewer'
+import { useScheduleJob } from 'hooks/useScheduleJob'
+import { useAppContext } from 'AppContext'
+import LogStreaming from './LogStreaming'
+import css from './Logger.module.scss'
+
+interface LoggerProps {
+  stepNameLogKeyMap?: Map<string, string>
+  expanded?: boolean
+  logKey: string
+  state: string
+  value: string
+  isStreaming: boolean
+  localRef: any
+  setIsBottom: (val: boolean) => void
+}
+
+const Logger: React.FC<LoggerProps> = ({ expanded, logKey, value, state, isStreaming, localRef, setIsBottom }) => {
+  const logKeyList: string[] = [logKey]
+  const { hooks } = useAppContext()
+  const [startStreaming, setStartStreaming] = useState(false)
+  const { getBlobData, blobDataCur } = hooks?.useLogsContent(logKeyList)
+
+  const sendStreamLogToRenderer = useScheduleJob({
+    handler: useCallback((blocks: string[]) => {
+      const logContainer = localRef.current as HTMLDivElement
+
+      if (logContainer) {
+        const fragment = new DocumentFragment()
+
+        blocks.forEach((block: string) => {
+          const blockData = JSON.parse(block)
+          const linePos = blockData.pos + 1
+          const localDate = new Date(blockData.time)
+          const formattedDate = localDate.toLocaleString()
+
+          fragment.appendChild(
+            lineElement(`${linePos}  ${blockData.level}  ${formattedDate.replace(',', '')}  ${blockData.out}`)
+          )
+        })
+
+        logContainer?.appendChild(fragment)
+
+        const scrollParent = logContainer?.parentElement as HTMLDivElement
+        scrollParent.scrollTop = scrollParent?.scrollHeight
+        setIsBottom(true)
+      }
+    }, []),
+    isStreaming: true
+  })
+
+  // eslint-disable-next-line @typescript-eslint/no-explicit-any
+  const onMessageStreaming = (e: any) => {
+    if (e.data) {
+      sendStreamLogToRenderer(e.data || '')
+    }
+  }
+
+  const onError = (e: any) => {
+    // eslint-disable-next-line no-console
+    console.log(e)
+  }
+
+  const getLogData = async () => {
+    await getBlobData(logKeyList)
+  }
+
+  useEffect(() => {
+    if (expanded && (state === GitspaceStatus.RUNNING || state === GitspaceStatus.STOPPED)) {
+      // Fetch from blob
+      getLogData()
+    } else if (expanded && state !== GitspaceStatus.RUNNING && state !== GitspaceStatus.STOPPED) {
+      if (isStreaming) {
+        setStartStreaming(true)
+      } else {
+        setStartStreaming(false)
+      }
+    }
+  }, [state, isStreaming, expanded])
+
+  useEffect(() => {
+    if (blobDataCur && (state === GitspaceStatus.RUNNING || state === GitspaceStatus.STOPPED)) {
+      const logData = JSON.parse(blobDataCur)?.map((logs: { level: string; time: string }) => {
+        return JSON.stringify(logs)
+      })
+      sendStreamLogToRenderer(logData || '')
+    }
+  }, [blobDataCur])
+
+  return (
+    <>
+      {startStreaming ? (
+        <LogStreaming logKeyList={logKeyList} onMessageStreaming={onMessageStreaming} onError={onError} />
+      ) : null}
+      <Container key={`harnesslog_${value}`} ref={localRef} className={cx(css.main, css.stepLogContainer)} />
+    </>
+  )
+}
+
+export default Logger

web/src/cde-gitness/services/index.tsx

@@ -1034,6 +1034,7 @@ export interface TypesGitspaceConfig {
   state?: EnumGitspaceStateType
   updated?: number
   user_id?: string
+  log_key?: string
 }
 
 export interface TypesGitspaceEventResponse {