CODE-520: FE: optimize loading of file lists for large repos

.gitignore

@@ -6,7 +6,9 @@ _research
 *.sqlite
 *.sqlite3
 web/node_modules
-web/dist/files
+web/dist
+web/coverage
+yarn-error*
 release
 .idea
 .vscode/settings.json

web/.gitignore

@@ -1,6 +0,0 @@
-node_modules
-dist
-coverage
-.env
-yarn-error*
-.DS_Store

web/src/pages/Repository/RepositoryContent/FolderContent/FolderContent.tsx

@@ -1,15 +1,18 @@
-import React, { useMemo } from 'react'
+import React, { useCallback, useEffect, useMemo, useState } from 'react'
 import { Container, Color, TableV2 as Table, Text, Utils } from '@harness/uicore'
 import type { CellProps, Column } from 'react-table'
 import { Render } from 'react-jsx-match'
-import { sortBy } from 'lodash-es'
+import { chunk, clone, sortBy, throttle } from 'lodash-es'
+import { useMutate } from 'restful-react'
 import { useHistory } from 'react-router-dom'
 import { useAppContext } from 'AppContext'
-import type { OpenapiContentInfo, OpenapiDirContent } from 'services/code'
+import type { OpenapiContentInfo, OpenapiDirContent, TypesCommit } from 'services/code'
-import { formatDate } from 'utils/Utils'
+import { formatDate, LIST_FETCHING_LIMIT } from 'utils/Utils'
 import { findReadmeInfo, CodeIcon, GitInfoProps, isFile } from 'utils/GitUtils'
 import { LatestCommitForFolder } from 'components/LatestCommit/LatestCommit'
+import { useEventListener } from 'hooks/useEventListener'
 import { Readme } from './Readme'
+import repositoryCSS from '../../Repository.module.scss'
 import css from './FolderContent.module.scss'
 
 export function FolderContent({
@@ -18,7 +21,7 @@ export function FolderContent({
   gitRef
 }: Pick<GitInfoProps, 'repoMetadata' | 'resourceContent' | 'gitRef'>) {
   const history = useHistory()
-  const { routes } = useAppContext()
+  const { routes, standalone } = useAppContext()
   const columns: Column<OpenapiContentInfo>[] = useMemo(
     () => [
       {
@@ -27,6 +30,7 @@ export function FolderContent({
         Cell: ({ row }: CellProps<OpenapiContentInfo>) => {
           return (
             <Text
+              data-resource-path={row.original.path}
               lineClamp={1}
               className={css.rowText}
               color={Color.BLACK}
@@ -79,6 +83,91 @@ export function FolderContent({
     [] // eslint-disable-line react-hooks/exhaustive-deps
   )
   const readmeInfo = useMemo(() => findReadmeInfo(resourceContent), [resourceContent])
+  const scrollElement = useMemo(
+    () => (standalone ? document.querySelector(`.${repositoryCSS.main}`)?.parentElement : window) as HTMLElement,
+    [standalone]
+  )
+  const resourceEntries = useMemo(
+    () => sortBy((resourceContent.content as OpenapiDirContent)?.entries || [], ['type', 'name']),
+    [resourceContent.content]
+  )
+  const [pathsChunks, setPathsChunks] = useState<PathsChunks>([])
+  const { mutate: fetchLastCommitsForPaths } = useMutate<PathDetails>({
+    verb: 'POST',
+    path: `/api/v1/repos/${encodeURIComponent(repoMetadata.path as string)}/path-details`
+  })
+  const [lastCommitMapping, setLastCommitMapping] = useState<Record<string, TypesCommit>>({})
+  const mergedContentEntries = useMemo(
+    () => resourceEntries.map(entry => ({ ...entry, latest_commit: lastCommitMapping[entry.path as string] })),
+    [resourceEntries, lastCommitMapping]
+  )
+
+  // The idea is to fetch last commit details for chunks that has atleast one path which is
+  // rendered in the viewport
+  // eslint-disable-next-line react-hooks/exhaustive-deps
+  const scrollCallback = useCallback(
+    throttle(() => {
+      pathsChunks.forEach(pathsChunk => {
+        const { paths, loaded, loading } = pathsChunk
+
+        if (!loaded && !loading) {
+          for (let i = 0; i < paths.length; i++) {
+            const element = document.querySelector(`[data-resource-path="${paths[i]}"]`)
+
+            if (element && isInViewport(element)) {
+              pathsChunk.loading = true
+
+              setPathsChunks(pathsChunks.map(_chunk => (pathsChunk === _chunk ? pathsChunk : _chunk)))
+
+              fetchLastCommitsForPaths({ paths })
+                .then(response => {
+                  const pathMapping: Record<string, TypesCommit> = clone(lastCommitMapping)
+
+                  pathsChunk.loaded = true
+                  setPathsChunks(pathsChunks.map(_chunk => (pathsChunk === _chunk ? pathsChunk : _chunk)))
+
+                  response?.details?.forEach(({ path, last_commit }) => {
+                    pathMapping[path] = last_commit
+                  })
+                  setLastCommitMapping(pathMapping)
+                })
+                .catch(error => {
+                  pathsChunk.loaded = false
+                  pathsChunk.loading = false
+                  setPathsChunks(pathsChunks.map(_chunk => (pathsChunk === _chunk ? pathsChunk : _chunk)))
+                  console.log('Failed to fetch path commit details', error) // eslint-disable-line no-console
+                })
+
+              break
+            }
+          }
+        }
+      })
+    }, 100),
+    [pathsChunks, lastCommitMapping]
+  )
+
+  // Group all resourceEntries paths into chunks, each has LIST_FETCHING_LIMIT paths
+  useEffect(() => {
+    setPathsChunks(
+      chunk(resourceEntries.map(entry => entry.path as string) || [], LIST_FETCHING_LIMIT).map(paths => ({
+        paths,
+        loaded: false,
+        loading: false
+      }))
+    )
+  }, [resourceEntries])
+
+  useEventListener('scroll', scrollCallback, scrollElement)
+
+  // Trigger scroll event callback on mount and cancel it on unmount
+  useEffect(() => {
+    scrollCallback()
+
+    return () => {
+      scrollCallback.cancel()
+    }
+  }, [scrollCallback])
 
   return (
     <Container className={css.folderContent}>
@@ -88,13 +177,13 @@ export function FolderContent({
         className={css.table}
         hideHeaders
         columns={columns}
-        data={sortBy((resourceContent.content as OpenapiDirContent)?.entries || [], ['type', 'name'])}
+        data={mergedContentEntries}
-        onRowClick={data => {
+        onRowClick={entry => {
           history.push(
             routes.toCODERepository({
               repoPath: repoMetadata.path as string,
               gitRef,
-              resourcePath: data.path
+              resourcePath: entry.path
             })
           )
         }}
@@ -107,3 +196,19 @@ export function FolderContent({
     </Container>
   )
 }
+
+function isInViewport(element: Element) {
+  const rect = element.getBoundingClientRect()
+  return (
+    rect.top >= 0 &&
+    rect.left >= 0 &&
+    rect.bottom <= (window.innerHeight || document.documentElement.clientHeight) &&
+    rect.right <= (window.innerWidth || document.documentElement.clientWidth)
+  )
+}
+
+interface PathDetails {
+  details: Array<{ path: string; last_commit: TypesCommit }>
+}
+
+type PathsChunks = Array<{ paths: string[]; loaded: boolean; loading: boolean }>

web/src/utils/Utils.ts

@@ -127,11 +127,13 @@ const LOCALE = Intl.NumberFormat().resolvedOptions?.().locale || 'en-US'
  * @param timeStyle Optional DateTimeFormat's `timeStyle` option.
  */
 export function formatTime(timestamp: number | string, timeStyle = 'short'): string {
-  return new Intl.DateTimeFormat(LOCALE, {
+  return timestamp
-    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+    ? new Intl.DateTimeFormat(LOCALE, {
-    // @ts-ignore: TS built-in type for DateTimeFormat is not correct
+        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-    timeStyle
+        // @ts-ignore: TS built-in type for DateTimeFormat is not correct
-  }).format(new Date(timestamp))
+        timeStyle
+      }).format(new Date(timestamp))
+    : ''
 }
 
 /**
@@ -140,11 +142,13 @@ export function formatTime(timestamp: number | string, timeStyle = 'short'): str
  * @param dateStyle Optional DateTimeFormat's `dateStyle` option.
  */
 export function formatDate(timestamp: number | string, dateStyle = 'medium'): string {
-  return new Intl.DateTimeFormat(LOCALE, {
+  return timestamp
-    // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+    ? new Intl.DateTimeFormat(LOCALE, {
-    // @ts-ignore: TS built-in type for DateTimeFormat is not correct
+        // eslint-disable-next-line @typescript-eslint/ban-ts-comment
-    dateStyle
+        // @ts-ignore: TS built-in type for DateTimeFormat is not correct
-  }).format(new Date(timestamp))
+        dateStyle
+      }).format(new Date(timestamp))
+    : ''
 }
 
 /**
@@ -153,7 +157,7 @@ export function formatDate(timestamp: number | string, dateStyle = 'medium'): st
  * @returns Formatted string.
  */
 export function formatNumber(num: number | bigint): string {
-  return new Intl.NumberFormat(LOCALE).format(num)
+  return num ? new Intl.NumberFormat(LOCALE).format(num) : ''
 }
 
 /**