diff --git a/registry/app/api/controller/metadata/artifact_mapper.go b/registry/app/api/controller/metadata/artifact_mapper.go index 7ff441892..1faed9f13 100644 --- a/registry/app/api/controller/metadata/artifact_mapper.go +++ b/registry/app/api/controller/metadata/artifact_mapper.go @@ -56,14 +56,21 @@ func GetRegistryArtifactMetadata(artifacts []types.ArtifactMetadata) []artifacta return artifactMetadataList } -func GetArtifactDetail(image *types.Image, artifact *types.Artifact) artifactapi.ArtifactDetail { +func GetMavenArtifactDetail(image *types.Image, artifact *types.Artifact, + metadata database.MavenMetadata) artifactapi.ArtifactDetail { createdAt := GetTimeInMs(artifact.CreatedAt) modifiedAt := GetTimeInMs(artifact.UpdatedAt) + var size int64 + for _, file := range metadata.Files { + size += file.Size + } + sizeVal := GetSize(size) artifactDetail := &artifactapi.ArtifactDetail{ CreatedAt: &createdAt, ModifiedAt: &modifiedAt, Name: &image.Name, Version: artifact.Version, + Size: &sizeVal, } return *artifactDetail } diff --git a/registry/app/api/controller/metadata/get_artifact_detail.go b/registry/app/api/controller/metadata/get_artifact_detail.go index 00bb19daa..1ba77879b 100644 --- a/registry/app/api/controller/metadata/get_artifact_detail.go +++ b/registry/app/api/controller/metadata/get_artifact_detail.go @@ -97,7 +97,16 @@ func (c *APIController) GetArtifactDetails( var artifactDetails artifact.ArtifactDetail if artifact.PackageTypeMAVEN == registry.PackageType { - artifactDetails = GetArtifactDetail(img, art) + var metadata database.MavenMetadata + err := json.Unmarshal(art.Metadata, &metadata) + if err != nil { + return artifact.GetArtifactDetails500JSONResponse{ + InternalServerErrorJSONResponse: artifact.InternalServerErrorJSONResponse( + *GetErrorResponse(http.StatusInternalServerError, err.Error()), + ), + }, nil + } + artifactDetails = GetMavenArtifactDetail(img, art, metadata) } else if artifact.PackageTypeGENERIC == registry.PackageType { var metadata database.GenericMetadata err := json.Unmarshal(art.Metadata, &metadata) diff --git a/registry/app/api/handler/maven/base.go b/registry/app/api/handler/maven/base.go index e307dbf42..46ec965fd 100644 --- a/registry/app/api/handler/maven/base.go +++ b/registry/app/api/handler/maven/base.go @@ -86,7 +86,7 @@ func (h *Handler) GetArtifactInfo(r *http.Request, remoteSupport bool) (pkg.Mave if err != nil { return pkg.MavenArtifactInfo{}, err } - if err = metadata.ValidateIdentifier(rootIdentifier); err != nil { + if err = metadata.ValidateIdentifier(registryIdentifier); err != nil { return pkg.MavenArtifactInfo{}, err } diff --git a/registry/app/api/middleware/bandwidth_stats.go b/registry/app/api/middleware/bandwidth_stats.go index 250055654..9ad941e86 100644 --- a/registry/app/api/middleware/bandwidth_stats.go +++ b/registry/app/api/middleware/bandwidth_stats.go @@ -21,6 +21,7 @@ import ( "net/http" "github.com/harness/gitness/registry/app/api/handler/generic" + "github.com/harness/gitness/registry/app/api/handler/maven" "github.com/harness/gitness/registry/app/api/handler/oci" "github.com/harness/gitness/registry/app/api/router/utils" "github.com/harness/gitness/registry/app/dist_temp/errcode" @@ -28,6 +29,7 @@ import ( "github.com/harness/gitness/registry/app/pkg/commons" "github.com/harness/gitness/registry/app/pkg/docker" generic2 "github.com/harness/gitness/registry/app/pkg/generic" + maven2 "github.com/harness/gitness/registry/app/pkg/maven" "github.com/harness/gitness/registry/app/store/database" "github.com/harness/gitness/registry/types" "github.com/harness/gitness/store" @@ -141,8 +143,8 @@ func TrackBandwidthStatForGenericArtifacts(h *generic.Handler) func(http.Handler return } - err = dbBandwidthStatForGenericArtifact(ctx, h.Controller, info, bandwidthType) - if !commons.IsEmptyError(err) { + err2 := dbBandwidthStatForGenericArtifact(ctx, h.Controller, info, bandwidthType) + if !commons.IsEmptyError(err2) { log.Ctx(ctx).Error().Stack().Str("middleware", "TrackBandwidthStat").Err(err).Msgf("error while putting bandwidth stat for artifact [%s:%s], %v", info.RegIdentifier, info.Image, err) @@ -153,6 +155,55 @@ func TrackBandwidthStatForGenericArtifacts(h *generic.Handler) func(http.Handler } } +func TrackBandwidthStatForMavenArtifacts(h *maven.Handler) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + methodType := r.Method + + sw := &StatusWriter{ResponseWriter: w} + + var bandwidthType types.BandwidthType + //nolint:gocritic + if http.MethodGet == methodType { + next.ServeHTTP(sw, r) + bandwidthType = types.BandwidthTypeDOWNLOAD + } else if http.MethodPut == methodType { + bandwidthType = types.BandwidthTypeUPLOAD + next.ServeHTTP(sw, r) + } else { + next.ServeHTTP(w, r) + return + } + + if types.BandwidthTypeUPLOAD == bandwidthType && sw.StatusCode != http.StatusCreated { + return + } else if types.BandwidthTypeDOWNLOAD == bandwidthType && sw.StatusCode != http.StatusOK && + sw.StatusCode != http.StatusTemporaryRedirect { + return + } + ctx := r.Context() + + info, err := h.GetArtifactInfo(r, false) + if !commons.IsEmpty(err) { + log.Ctx(ctx).Error().Stack().Str("middleware", + "TrackBandwidthStat").Err(err).Msgf("error while putting bandwidth stat for artifact, %v", + err) + return + } + + err2 := dbBandwidthStatForMavenArtifact(ctx, h.Controller, info, bandwidthType) + if !commons.IsEmptyError(err2) { + log.Ctx(ctx).Error().Stack().Str("middleware", + "TrackBandwidthStat").Err(err).Msgf("error while putting bandwidth stat for artifact [%s:%s], %v", + info.RegIdentifier, info.GroupID+":"+info.ArtifactID, err) + return + } + }, + ) + } +} + func dbBandwidthStatForGenericArtifact( ctx context.Context, c *generic2.Controller, @@ -197,6 +248,51 @@ func dbBandwidthStatForGenericArtifact( return errcode.Error{} } +func dbBandwidthStatForMavenArtifact( + ctx context.Context, + c *maven2.Controller, + info pkg.MavenArtifactInfo, + bandwidthType types.BandwidthType, +) errcode.Error { + imageName := info.GroupID + ":" + info.ArtifactID + registry, err := c.DBStore.RegistryDao.GetByParentIDAndName(ctx, info.ParentID, info.RegIdentifier) + if err != nil { + return errcode.ErrCodeInvalidRequest.WithDetail(err) + } + + image, err := c.DBStore.ImageDao.GetByName(ctx, registry.ID, imageName) + if err != nil { + return errcode.ErrCodeInvalidRequest.WithDetail(err) + } + + art, err := c.DBStore.ArtifactDao.GetByName(ctx, image.ID, info.Version) + if err != nil { + return errcode.ErrCodeInvalidRequest.WithDetail(err) + } + + var metadata database.GenericMetadata + err = json.Unmarshal(art.Metadata, &metadata) + + if err != nil { + return errcode.ErrCodeNameUnknown.WithDetail(err) + } + + var size int64 + for _, files := range metadata.Files { + size += files.Size + } + bandwidthStat := &types.BandwidthStat{ + ImageID: image.ID, + Type: bandwidthType, + Bytes: size, + } + + if err := c.DBStore.BandwidthStatDao.Create(ctx, bandwidthStat); err != nil { + return errcode.ErrCodeNameUnknown.WithDetail(err) + } + return errcode.Error{} +} + func dbBandwidthStat( ctx context.Context, c *docker.Controller, diff --git a/registry/app/api/middleware/download_stats.go b/registry/app/api/middleware/download_stats.go index 229f44c3e..024870b03 100644 --- a/registry/app/api/middleware/download_stats.go +++ b/registry/app/api/middleware/download_stats.go @@ -20,6 +20,7 @@ import ( "net/http" "github.com/harness/gitness/registry/app/api/handler/generic" + "github.com/harness/gitness/registry/app/api/handler/maven" "github.com/harness/gitness/registry/app/api/handler/oci" "github.com/harness/gitness/registry/app/api/router/utils" "github.com/harness/gitness/registry/app/dist_temp/errcode" @@ -27,6 +28,8 @@ import ( "github.com/harness/gitness/registry/app/pkg/commons" "github.com/harness/gitness/registry/app/pkg/docker" generic2 "github.com/harness/gitness/registry/app/pkg/generic" + maven2 "github.com/harness/gitness/registry/app/pkg/maven" + mavenutils "github.com/harness/gitness/registry/app/pkg/maven/utils" "github.com/harness/gitness/registry/types" "github.com/harness/gitness/store" @@ -154,6 +157,49 @@ func TrackDownloadStatForGenericArtifact(h *generic.Handler) func(http.Handler) } } +func TrackDownloadStatForMavenArtifact(h *maven.Handler) func(http.Handler) http.Handler { + return func(next http.Handler) http.Handler { + return http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + methodType := r.Method + ctx := r.Context() + sw := &StatusWriter{ResponseWriter: w} + + if http.MethodGet == methodType { + next.ServeHTTP(sw, r) + } else { + next.ServeHTTP(w, r) + return + } + + if sw.StatusCode != http.StatusOK && sw.StatusCode != http.StatusTemporaryRedirect { + return + } + + info, err := h.GetArtifactInfo(r, true) + if !commons.IsEmpty(err) { + log.Ctx(ctx).Error().Stack().Str("middleware", + "TrackDownloadStat").Err(err).Msgf("error while putting download stat of artifact, %v", + err) + return + } + + if !mavenutils.IsMainArtifactFile(info) { + return + } + + err = dbDownloadStatForMavenArtifact(ctx, h.Controller, info) + if !commons.IsEmpty(err) { + log.Ctx(ctx).Error().Stack().Str("middleware", + "TrackDownloadStat").Err(err).Msgf("error while putting download stat of artifact, %v", + err) + return + } + }, + ) + } +} + func dbDownloadStatForGenericArtifact( ctx context.Context, c *generic2.Controller, @@ -183,3 +229,34 @@ func dbDownloadStatForGenericArtifact( } return errcode.Error{} } + +func dbDownloadStatForMavenArtifact( + ctx context.Context, + c *maven2.Controller, + info pkg.MavenArtifactInfo, +) errcode.Error { + imageName := info.GroupID + ":" + info.ArtifactID + registry, err := c.DBStore.RegistryDao.GetByParentIDAndName(ctx, info.ParentID, info.RegIdentifier) + if err != nil { + return errcode.ErrCodeInvalidRequest.WithDetail(err) + } + + image, err := c.DBStore.ImageDao.GetByName(ctx, registry.ID, imageName) + if err != nil { + return errcode.ErrCodeInvalidRequest.WithDetail(err) + } + + artifact, err := c.DBStore.ArtifactDao.GetByName(ctx, image.ID, info.Version) + if err != nil { + return errcode.ErrCodeInvalidRequest.WithDetail(err) + } + + downloadStat := &types.DownloadStat{ + ArtifactID: artifact.ID, + } + + if err := c.DBStore.DownloadStatDao.Create(ctx, downloadStat); err != nil { + return errcode.ErrCodeNameUnknown.WithDetail(err) + } + return errcode.Error{} +} diff --git a/registry/app/api/router/maven/route.go b/registry/app/api/router/maven/route.go index cb543caaf..ab037e7db 100644 --- a/registry/app/api/router/maven/route.go +++ b/registry/app/api/router/maven/route.go @@ -42,8 +42,8 @@ func NewMavenHandler(handler *maven.Handler) Handler { r.Use(middleware.StoreOriginalURL) r.Use(middlewareauthn.Attempt(handler.Authenticator)) r.Use(middleware.CheckMavenAuth()) - //todo: r.Use(middleware.TrackDownloadStat(handler)) - //todo: r.Use(middleware.TrackBandwidthStat(handler)) + r.Use(middleware.TrackDownloadStatForMavenArtifact(handler)) + r.Use(middleware.TrackBandwidthStatForMavenArtifacts(handler)) r.Handle("/*", http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) { methodType := req.Method diff --git a/registry/app/pkg/maven/local.go b/registry/app/pkg/maven/local.go index 0266e88e7..6861e787f 100644 --- a/registry/app/pkg/maven/local.go +++ b/registry/app/pkg/maven/local.go @@ -17,9 +17,11 @@ package maven import ( "context" "database/sql" + "encoding/json" "io" "net/http" "strings" + "time" "github.com/harness/gitness/registry/app/dist_temp/errcode" "github.com/harness/gitness/registry/app/pkg" @@ -27,6 +29,7 @@ import ( "github.com/harness/gitness/registry/app/pkg/filemanager" "github.com/harness/gitness/registry/app/pkg/maven/utils" "github.com/harness/gitness/registry/app/storage" + "github.com/harness/gitness/registry/app/store/database" "github.com/harness/gitness/registry/types" "github.com/harness/gitness/store/database/dbtx" ) @@ -114,7 +117,7 @@ func (r *LocalRegistry) FetchArtifact(ctx context.Context, info pkg.MavenArtifac func (r *LocalRegistry) PutArtifact(ctx context.Context, info pkg.MavenArtifactInfo, fileReader io.Reader) ( responseHeaders *commons.ResponseHeaders, errs []error) { filePath := utils.GetFilePath(info) - _, err := r.fileManager.UploadFile(ctx, filePath, info.RegIdentifier, + fileInfo, err := r.fileManager.UploadFile(ctx, filePath, info.RegIdentifier, info.RegistryID, info.RootParentID, info.RootIdentifier, nil, fileReader, info.FileName) if err != nil { return responseHeaders, []error{errcode.ErrCodeUnknown.WithDetail(err)} @@ -137,9 +140,29 @@ func (r *LocalRegistry) PutArtifact(ctx context.Context, info pkg.MavenArtifactI return nil } - dbArtifact := &types.Artifact{ - ImageID: dbImage.ID, - Version: info.Version, + metadata := &database.MavenMetadata{} + + dbArtifact, err3 := r.DBStore.ArtifactDao.GetByName(ctx, dbImage.ID, info.Version) + + if err3 != nil && !strings.Contains(err3.Error(), "resource not found") { + return err3 + } + + err3 = r.updateArtifactMetadata(dbArtifact, metadata, info, fileInfo) + if err3 != nil { + return err3 + } + + metadataJSON, err3 := json.Marshal(metadata) + + if err3 != nil { + return err3 + } + + dbArtifact = &types.Artifact{ + ImageID: dbImage.ID, + Version: info.Version, + Metadata: metadataJSON, } err2 = r.DBStore.ArtifactDao.CreateOrUpdate(ctx, dbArtifact) @@ -160,6 +183,36 @@ func (r *LocalRegistry) PutArtifact(ctx context.Context, info pkg.MavenArtifactI return responseHeaders, nil } +func (r *LocalRegistry) updateArtifactMetadata(dbArtifact *types.Artifact, metadata *database.MavenMetadata, + info pkg.MavenArtifactInfo, fileInfo pkg.FileInfo) error { + var files []database.File + if dbArtifact != nil { + err := json.Unmarshal(dbArtifact.Metadata, metadata) + if err != nil { + return err + } + fileExist := false + files = metadata.Files + for _, file := range files { + if file.Filename == info.FileName { + fileExist = true + } + } + if !fileExist { + files = append(files, database.File{Size: fileInfo.Size, Filename: fileInfo.Filename, + CreatedAt: time.Now().UnixMilli()}) + metadata.Files = files + metadata.FileCount++ + } + } else { + files = append(files, database.File{Size: fileInfo.Size, Filename: fileInfo.Filename, + CreatedAt: time.Now().UnixMilli()}) + metadata.Files = files + metadata.FileCount++ + } + return nil +} + func processError(err error) ( responseHeaders *commons.ResponseHeaders, body *storage.FileReader, readCloser io.ReadCloser, redirectURL string, errs []error) { diff --git a/registry/app/pkg/maven/utils/utils.go b/registry/app/pkg/maven/utils/utils.go index d914121e1..7709de91e 100644 --- a/registry/app/pkg/maven/utils/utils.go +++ b/registry/app/pkg/maven/utils/utils.go @@ -37,11 +37,50 @@ const ( contentTypePlainText = "text/plain" ) +const ( + Jar = ".jar" + War = ".war" + Ear = ".ear" + Zip = ".zip" + TarGz = ".tar.gz" + So = ".so" + Dll = ".dll" + Dylib = ".dylib" + Rpm = ".rpm" + Deb = ".deb" + Exe = ".exe" +) + +var MainArtifactFileExtensions = []string{ + Jar, + War, + Ear, + Zip, + TarGz, + So, + Dll, + Dylib, + Rpm, + Deb, + Exe, +} + func GetFilePath(info pkg.MavenArtifactInfo) string { groupIDPath := strings.ReplaceAll(info.GroupID, ".", "/") return "/" + groupIDPath + "/" + info.ArtifactID + "/" + info.Version + "/" + info.FileName } +func IsMainArtifactFile(info pkg.MavenArtifactInfo) bool { + filePath := GetFilePath(info) + fileExtension := strings.ToLower(filepath.Ext(filePath)) + for _, ext := range MainArtifactFileExtensions { + if ext == fileExtension { + return true + } + } + return false +} + func SetHeaders(info pkg.MavenArtifactInfo, fileInfo pkg.FileInfo, ) *commons.ResponseHeaders { diff --git a/registry/app/store/database/artifact.go b/registry/app/store/database/artifact.go index 474103f7d..8a4142ce4 100644 --- a/registry/app/store/database/artifact.go +++ b/registry/app/store/database/artifact.go @@ -757,6 +757,11 @@ type GenericMetadata struct { FileCount int64 `json:"file_count"` } +type MavenMetadata struct { + Files []File `json:"files"` + FileCount int64 `json:"file_count"` +} + type File struct { Size int64 `json:"size"` Filename string `json:"file_name"`