github/drone
5
0
Fork 0
mirror of https://github.com/harness/drone.git synced 2025-05-16 08:59:56 +08:00
Code Issues Packages Projects Releases Wiki Activity

[feat]: [AH-152]: fix maven files count (#3335)

Browse Source 
* [feat]: [AH-152]: pr check
* [feat]: [AH-152]: pr check
* [feat]: [AH-152]: pr check
* [feat]: [AH-152]: pr check
* [feat]: [AH-152]: pr check
* [feat]: [AH-152]: fix files count
* [feat]: [AH-152]: fix files count
* [feat]: [AH-152]: fix files count
This commit is contained in:
Pragyesh Mishra 2025-01-28 17:59:33 +00:00 committed by Harness
parent 1422bc12d2
commit 4eb00eae4c
9 changed files with 297 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

9
registry/app/api/controller/metadata/artifact_mapper.go
View File

@ -56,14 +56,21 @@ func GetRegistryArtifactMetadata(artifacts []types.ArtifactMetadata) []artifacta
return artifactMetadataList
}
func GetArtifactDetail(image *types.Image, artifact *types.Artifact) artifactapi.ArtifactDetail {
func GetMavenArtifactDetail(image *types.Image, artifact *types.Artifact,
metadata database.MavenMetadata) artifactapi.ArtifactDetail {
createdAt := GetTimeInMs(artifact.CreatedAt)
modifiedAt := GetTimeInMs(artifact.UpdatedAt)
var size int64
for _, file := range metadata.Files {
size += file.Size
}
sizeVal := GetSize(size)
artifactDetail := &artifactapi.ArtifactDetail{
CreatedAt: &createdAt,
ModifiedAt: &modifiedAt,
Name: &image.Name,
Version: artifact.Version,
Size: &sizeVal,
}
return *artifactDetail
}

11
registry/app/api/controller/metadata/get_artifact_detail.go
View File

@ -97,7 +97,16 @@ func (c *APIController) GetArtifactDetails(
var artifactDetails artifact.ArtifactDetail
if artifact.PackageTypeMAVEN == registry.PackageType {
artifactDetails = GetArtifactDetail(img, art)
var metadata database.MavenMetadata
err := json.Unmarshal(art.Metadata, &metadata)
if err != nil {
return artifact.GetArtifactDetails500JSONResponse{
InternalServerErrorJSONResponse: artifact.InternalServerErrorJSONResponse(
*GetErrorResponse(http.StatusInternalServerError, err.Error()),
),
}, nil
}
artifactDetails = GetMavenArtifactDetail(img, art, metadata)
} else if artifact.PackageTypeGENERIC == registry.PackageType {
var metadata database.GenericMetadata
err := json.Unmarshal(art.Metadata, &metadata)

2
registry/app/api/handler/maven/base.go
View File

@ -86,7 +86,7 @@ func (h *Handler) GetArtifactInfo(r *http.Request, remoteSupport bool) (pkg.Mave
if err != nil {
return pkg.MavenArtifactInfo{}, err
}
if err = metadata.ValidateIdentifier(rootIdentifier); err != nil {
if err = metadata.ValidateIdentifier(registryIdentifier); err != nil {
return pkg.MavenArtifactInfo{}, err
}

100
registry/app/api/middleware/bandwidth_stats.go
View File

@ -21,6 +21,7 @@ import (
"net/http"
"github.com/harness/gitness/registry/app/api/handler/generic"
"github.com/harness/gitness/registry/app/api/handler/maven"
"github.com/harness/gitness/registry/app/api/handler/oci"
"github.com/harness/gitness/registry/app/api/router/utils"
"github.com/harness/gitness/registry/app/dist_temp/errcode"
@ -28,6 +29,7 @@ import (
"github.com/harness/gitness/registry/app/pkg/commons"
"github.com/harness/gitness/registry/app/pkg/docker"
generic2 "github.com/harness/gitness/registry/app/pkg/generic"
maven2 "github.com/harness/gitness/registry/app/pkg/maven"
"github.com/harness/gitness/registry/app/store/database"
"github.com/harness/gitness/registry/types"
"github.com/harness/gitness/store"
@ -141,8 +143,8 @@ func TrackBandwidthStatForGenericArtifacts(h *generic.Handler) func(http.Handler
return
}
err = dbBandwidthStatForGenericArtifact(ctx, h.Controller, info, bandwidthType)
if !commons.IsEmptyError(err) {
err2 := dbBandwidthStatForGenericArtifact(ctx, h.Controller, info, bandwidthType)
if !commons.IsEmptyError(err2) {
log.Ctx(ctx).Error().Stack().Str("middleware",
"TrackBandwidthStat").Err(err).Msgf("error while putting bandwidth stat for artifact [%s:%s], %v",
info.RegIdentifier, info.Image, err)
@ -153,6 +155,55 @@ func TrackBandwidthStatForGenericArtifacts(h *generic.Handler) func(http.Handler
}
}
func TrackBandwidthStatForMavenArtifacts(h *maven.Handler) func(http.Handler) http.Handler {
return func(next http.Handler) http.Handler {
return http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
methodType := r.Method
sw := &StatusWriter{ResponseWriter: w}
var bandwidthType types.BandwidthType
//nolint:gocritic
if http.MethodGet == methodType {
next.ServeHTTP(sw, r)
bandwidthType = types.BandwidthTypeDOWNLOAD
} else if http.MethodPut == methodType {
bandwidthType = types.BandwidthTypeUPLOAD
next.ServeHTTP(sw, r)
} else {
next.ServeHTTP(w, r)
return
}
if types.BandwidthTypeUPLOAD == bandwidthType && sw.StatusCode != http.StatusCreated {
return
} else if types.BandwidthTypeDOWNLOAD == bandwidthType && sw.StatusCode != http.StatusOK &&
sw.StatusCode != http.StatusTemporaryRedirect {
return
}
ctx := r.Context()
info, err := h.GetArtifactInfo(r, false)
if !commons.IsEmpty(err) {
log.Ctx(ctx).Error().Stack().Str("middleware",
"TrackBandwidthStat").Err(err).Msgf("error while putting bandwidth stat for artifact, %v",
err)
return
}
err2 := dbBandwidthStatForMavenArtifact(ctx, h.Controller, info, bandwidthType)
if !commons.IsEmptyError(err2) {
log.Ctx(ctx).Error().Stack().Str("middleware",
"TrackBandwidthStat").Err(err).Msgf("error while putting bandwidth stat for artifact [%s:%s], %v",
info.RegIdentifier, info.GroupID+":"+info.ArtifactID, err)
return
}
},
)
}
}
func dbBandwidthStatForGenericArtifact(
ctx context.Context,
c *generic2.Controller,
@ -197,6 +248,51 @@ func dbBandwidthStatForGenericArtifact(
return errcode.Error{}
}
func dbBandwidthStatForMavenArtifact(
ctx context.Context,
c *maven2.Controller,
info pkg.MavenArtifactInfo,
bandwidthType types.BandwidthType,
) errcode.Error {
imageName := info.GroupID + ":" + info.ArtifactID
registry, err := c.DBStore.RegistryDao.GetByParentIDAndName(ctx, info.ParentID, info.RegIdentifier)
if err != nil {
return errcode.ErrCodeInvalidRequest.WithDetail(err)
}
image, err := c.DBStore.ImageDao.GetByName(ctx, registry.ID, imageName)
if err != nil {
return errcode.ErrCodeInvalidRequest.WithDetail(err)
}
art, err := c.DBStore.ArtifactDao.GetByName(ctx, image.ID, info.Version)
if err != nil {
return errcode.ErrCodeInvalidRequest.WithDetail(err)
}
var metadata database.GenericMetadata
err = json.Unmarshal(art.Metadata, &metadata)
if err != nil {
return errcode.ErrCodeNameUnknown.WithDetail(err)
}
var size int64
for _, files := range metadata.Files {
size += files.Size
}
bandwidthStat := &types.BandwidthStat{
ImageID: image.ID,
Type: bandwidthType,
Bytes: size,
}
if err := c.DBStore.BandwidthStatDao.Create(ctx, bandwidthStat); err != nil {
return errcode.ErrCodeNameUnknown.WithDetail(err)
}
return errcode.Error{}
}
func dbBandwidthStat(
ctx context.Context,
c *docker.Controller,

77
registry/app/api/middleware/download_stats.go
View File

@ -20,6 +20,7 @@ import (
"net/http"
"github.com/harness/gitness/registry/app/api/handler/generic"
"github.com/harness/gitness/registry/app/api/handler/maven"
"github.com/harness/gitness/registry/app/api/handler/oci"
"github.com/harness/gitness/registry/app/api/router/utils"
"github.com/harness/gitness/registry/app/dist_temp/errcode"
@ -27,6 +28,8 @@ import (
"github.com/harness/gitness/registry/app/pkg/commons"
"github.com/harness/gitness/registry/app/pkg/docker"
generic2 "github.com/harness/gitness/registry/app/pkg/generic"
maven2 "github.com/harness/gitness/registry/app/pkg/maven"
mavenutils "github.com/harness/gitness/registry/app/pkg/maven/utils"
"github.com/harness/gitness/registry/types"
"github.com/harness/gitness/store"
@ -154,6 +157,49 @@ func TrackDownloadStatForGenericArtifact(h *generic.Handler) func(http.Handler)
}
}
func TrackDownloadStatForMavenArtifact(h *maven.Handler) func(http.Handler) http.Handler {
return func(next http.Handler) http.Handler {
return http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
methodType := r.Method
ctx := r.Context()
sw := &StatusWriter{ResponseWriter: w}
if http.MethodGet == methodType {
next.ServeHTTP(sw, r)
} else {
next.ServeHTTP(w, r)
return
}
if sw.StatusCode != http.StatusOK && sw.StatusCode != http.StatusTemporaryRedirect {
return
}
info, err := h.GetArtifactInfo(r, true)
if !commons.IsEmpty(err) {
log.Ctx(ctx).Error().Stack().Str("middleware",
"TrackDownloadStat").Err(err).Msgf("error while putting download stat of artifact, %v",
err)
return
}
if !mavenutils.IsMainArtifactFile(info) {
return
}
err = dbDownloadStatForMavenArtifact(ctx, h.Controller, info)
if !commons.IsEmpty(err) {
log.Ctx(ctx).Error().Stack().Str("middleware",
"TrackDownloadStat").Err(err).Msgf("error while putting download stat of artifact, %v",
err)
return
}
},
)
}
}
func dbDownloadStatForGenericArtifact(
ctx context.Context,
c *generic2.Controller,
@ -183,3 +229,34 @@ func dbDownloadStatForGenericArtifact(
}
return errcode.Error{}
}
func dbDownloadStatForMavenArtifact(
ctx context.Context,
c *maven2.Controller,
info pkg.MavenArtifactInfo,
) errcode.Error {
imageName := info.GroupID + ":" + info.ArtifactID
registry, err := c.DBStore.RegistryDao.GetByParentIDAndName(ctx, info.ParentID, info.RegIdentifier)
if err != nil {
return errcode.ErrCodeInvalidRequest.WithDetail(err)
}
image, err := c.DBStore.ImageDao.GetByName(ctx, registry.ID, imageName)
if err != nil {
return errcode.ErrCodeInvalidRequest.WithDetail(err)
}
artifact, err := c.DBStore.ArtifactDao.GetByName(ctx, image.ID, info.Version)
if err != nil {
return errcode.ErrCodeInvalidRequest.WithDetail(err)
}
downloadStat := &types.DownloadStat{
ArtifactID: artifact.ID,
}
if err := c.DBStore.DownloadStatDao.Create(ctx, downloadStat); err != nil {
return errcode.ErrCodeNameUnknown.WithDetail(err)
}
return errcode.Error{}
}

4
registry/app/api/router/maven/route.go
View File

@ -42,8 +42,8 @@ func NewMavenHandler(handler *maven.Handler) Handler {
r.Use(middleware.StoreOriginalURL)
r.Use(middlewareauthn.Attempt(handler.Authenticator))
r.Use(middleware.CheckMavenAuth())
//todo: r.Use(middleware.TrackDownloadStat(handler))
//todo: r.Use(middleware.TrackBandwidthStat(handler))
r.Use(middleware.TrackDownloadStatForMavenArtifact(handler))
r.Use(middleware.TrackBandwidthStatForMavenArtifacts(handler))
r.Handle("/*", http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
methodType := req.Method

61
registry/app/pkg/maven/local.go
View File

@ -17,9 +17,11 @@ package maven
import (
"context"
"database/sql"
"encoding/json"
"io"
"net/http"
"strings"
"time"
"github.com/harness/gitness/registry/app/dist_temp/errcode"
"github.com/harness/gitness/registry/app/pkg"
@ -27,6 +29,7 @@ import (
"github.com/harness/gitness/registry/app/pkg/filemanager"
"github.com/harness/gitness/registry/app/pkg/maven/utils"
"github.com/harness/gitness/registry/app/storage"
"github.com/harness/gitness/registry/app/store/database"
"github.com/harness/gitness/registry/types"
"github.com/harness/gitness/store/database/dbtx"
)
@ -114,7 +117,7 @@ func (r *LocalRegistry) FetchArtifact(ctx context.Context, info pkg.MavenArtifac
func (r *LocalRegistry) PutArtifact(ctx context.Context, info pkg.MavenArtifactInfo, fileReader io.Reader) (
responseHeaders *commons.ResponseHeaders, errs []error) {
filePath := utils.GetFilePath(info)
_, err := r.fileManager.UploadFile(ctx, filePath, info.RegIdentifier,
fileInfo, err := r.fileManager.UploadFile(ctx, filePath, info.RegIdentifier,
info.RegistryID, info.RootParentID, info.RootIdentifier, nil, fileReader, info.FileName)
if err != nil {
return responseHeaders, []error{errcode.ErrCodeUnknown.WithDetail(err)}
@ -137,9 +140,29 @@ func (r *LocalRegistry) PutArtifact(ctx context.Context, info pkg.MavenArtifactI
return nil
}
dbArtifact := &types.Artifact{
ImageID: dbImage.ID,
Version: info.Version,
metadata := &database.MavenMetadata{}
dbArtifact, err3 := r.DBStore.ArtifactDao.GetByName(ctx, dbImage.ID, info.Version)
if err3 != nil && !strings.Contains(err3.Error(), "resource not found") {
return err3
}
err3 = r.updateArtifactMetadata(dbArtifact, metadata, info, fileInfo)
if err3 != nil {
return err3
}
metadataJSON, err3 := json.Marshal(metadata)
if err3 != nil {
return err3
}
dbArtifact = &types.Artifact{
ImageID: dbImage.ID,
Version: info.Version,
Metadata: metadataJSON,
}
err2 = r.DBStore.ArtifactDao.CreateOrUpdate(ctx, dbArtifact)
@ -160,6 +183,36 @@ func (r *LocalRegistry) PutArtifact(ctx context.Context, info pkg.MavenArtifactI
return responseHeaders, nil
}
func (r *LocalRegistry) updateArtifactMetadata(dbArtifact *types.Artifact, metadata *database.MavenMetadata,
info pkg.MavenArtifactInfo, fileInfo pkg.FileInfo) error {
var files []database.File
if dbArtifact != nil {
err := json.Unmarshal(dbArtifact.Metadata, metadata)
if err != nil {
return err
}
fileExist := false
files = metadata.Files
for _, file := range files {
if file.Filename == info.FileName {
fileExist = true
}
}
if !fileExist {
files = append(files, database.File{Size: fileInfo.Size, Filename: fileInfo.Filename,
CreatedAt: time.Now().UnixMilli()})
metadata.Files = files
metadata.FileCount++
}
} else {
files = append(files, database.File{Size: fileInfo.Size, Filename: fileInfo.Filename,
CreatedAt: time.Now().UnixMilli()})
metadata.Files = files
metadata.FileCount++
}
return nil
}
func processError(err error) (
responseHeaders *commons.ResponseHeaders, body *storage.FileReader, readCloser io.ReadCloser,
redirectURL string, errs []error) {

39
registry/app/pkg/maven/utils/utils.go
View File

@ -37,11 +37,50 @@ const (
contentTypePlainText = "text/plain"
)
const (
Jar = ".jar"
War = ".war"
Ear = ".ear"
Zip = ".zip"
TarGz = ".tar.gz"
So = ".so"
Dll = ".dll"
Dylib = ".dylib"
Rpm = ".rpm"
Deb = ".deb"
Exe = ".exe"
)
var MainArtifactFileExtensions = []string{
Jar,
War,
Ear,
Zip,
TarGz,
So,
Dll,
Dylib,
Rpm,
Deb,
Exe,
}
func GetFilePath(info pkg.MavenArtifactInfo) string {
groupIDPath := strings.ReplaceAll(info.GroupID, ".", "/")
return "/" + groupIDPath + "/" + info.ArtifactID + "/" + info.Version + "/" + info.FileName
}
func IsMainArtifactFile(info pkg.MavenArtifactInfo) bool {
filePath := GetFilePath(info)
fileExtension := strings.ToLower(filepath.Ext(filePath))
for _, ext := range MainArtifactFileExtensions {
if ext == fileExtension {
return true
}
}
return false
}
func SetHeaders(info pkg.MavenArtifactInfo,
fileInfo pkg.FileInfo,
) *commons.ResponseHeaders {

5
registry/app/store/database/artifact.go
View File

@ -757,6 +757,11 @@ type GenericMetadata struct {
FileCount int64 `json:"file_count"`
}
type MavenMetadata struct {
Files []File `json:"files"`
FileCount int64 `json:"file_count"`
}
type File struct {
Size int64 `json:"size"`
Filename string `json:"file_name"`