From 321a1cc8d254d3fd40b1bd45efb12f8a0e120c34 Mon Sep 17 00:00:00 2001 From: Sourabh Awashti Date: Fri, 24 Jan 2025 09:31:00 +0000 Subject: [PATCH] feat:[AH-400]: upload and download flow for generic artifacts (#3298) * feat:[AH-400]: fix checks * feat:[AH-400]: fix checks * Merge branch 'AH-400' of https://git0.harness.io/l7B_kbSEQD2wjrM7PShm5w/PROD/Harness_Commons/gitness into AH-400 * feat:[AH-400]: fix checks * Merge branch 'main' into AH-400 * feat:[AH-400]: review changes * feat:[AH-400]: upload and download flow for generic artifacts --- registry/app/pkg/context.go | 8 + registry/app/pkg/generic/controller.go | 222 ++++++++++++++++++++++++ registry/app/pkg/generic/wire.go | 50 ++++++ registry/app/store/database/artifact.go | 12 ++ registry/types/artifact.go | 14 ++ 5 files changed, 306 insertions(+) create mode 100644 registry/app/pkg/generic/controller.go create mode 100644 registry/app/pkg/generic/wire.go diff --git a/registry/app/pkg/context.go b/registry/app/pkg/context.go index 568828b06..af9212bec 100644 --- a/registry/app/pkg/context.go +++ b/registry/app/pkg/context.go @@ -71,6 +71,14 @@ type MavenArtifactInfo struct { Path string } +type GenericArtifactInfo struct { + *ArtifactInfo + FileName string + Version string + RegistryID int64 + Description string +} + func (a *MavenArtifactInfo) SetMavenRepoKey(key string) { a.RegIdentifier = key } diff --git a/registry/app/pkg/generic/controller.go b/registry/app/pkg/generic/controller.go new file mode 100644 index 000000000..aa0981a7d --- /dev/null +++ b/registry/app/pkg/generic/controller.go @@ -0,0 +1,222 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package generic + +import ( + "context" + "database/sql" + "encoding/json" + "errors" + "fmt" + "mime/multipart" + "net/http" + "time" + + "github.com/harness/gitness/app/auth/authz" + corestore "github.com/harness/gitness/app/store" + "github.com/harness/gitness/registry/app/dist_temp/errcode" + "github.com/harness/gitness/registry/app/pkg" + "github.com/harness/gitness/registry/app/pkg/commons" + "github.com/harness/gitness/registry/app/pkg/filemanager" + "github.com/harness/gitness/registry/app/storage" + "github.com/harness/gitness/registry/app/store" + "github.com/harness/gitness/registry/app/store/database" + "github.com/harness/gitness/registry/types" + "github.com/harness/gitness/store/database/dbtx" + "github.com/harness/gitness/types/enum" +) + +type Controller struct { + spaceStore corestore.SpaceStore + authorizer authz.Authorizer + DBStore *DBStore + fileManager filemanager.FileManager + tx dbtx.Transactor +} + +type DBStore struct { + RegistryDao store.RegistryRepository + ImageDao store.ImageRepository + ArtifactDao store.ArtifactRepository + TagDao store.TagRepository + BandwidthStatDao store.BandwidthStatRepository + DownloadStatDao store.DownloadStatRepository +} + +func NewController( + spaceStore corestore.SpaceStore, + authorizer authz.Authorizer, + fileManager filemanager.FileManager, + dBStore *DBStore, + tx dbtx.Transactor, +) *Controller { + return &Controller{ + spaceStore: spaceStore, + authorizer: authorizer, + fileManager: fileManager, + DBStore: dBStore, + tx: tx, + } +} + +func NewDBStore( + registryDao store.RegistryRepository, + imageDao store.ImageRepository, + artifactDao store.ArtifactRepository, + bandwidthStatDao store.BandwidthStatRepository, + downloadStatDao store.DownloadStatRepository, +) *DBStore { + return &DBStore{ + RegistryDao: registryDao, + ImageDao: imageDao, + ArtifactDao: artifactDao, + BandwidthStatDao: bandwidthStatDao, + DownloadStatDao: downloadStatDao, + } +} + +const regNameFormat = "registry : [%s]" + +func (c Controller) UploadArtifact(ctx context.Context, info pkg.GenericArtifactInfo, + file multipart.File) (*commons.ResponseHeaders, []error) { + responseHeaders := &commons.ResponseHeaders{ + Headers: make(map[string]string), + Code: 0, + } + err := pkg.GetRegistryCheckAccess( + ctx, c.DBStore.RegistryDao, c.authorizer, c.spaceStore, info.RegIdentifier, info.ParentID, + enum.PermissionArtifactsUpload, + ) + if err != nil { + return nil, []error{errcode.ErrCodeDenied, err} + } + + path := info.Image + "/" + info.Version + "/" + info.FileName + fileInfo, err := c.fileManager.UploadFile(ctx, path, info.RegIdentifier, info.RegistryID, + info.RootParentID, info.RootIdentifier, file, nil, info.FileName) + if err != nil { + return responseHeaders, []error{errcode.ErrCodeUnknown.WithDetail(err)} + } + err = c.tx.WithTx( + ctx, func(ctx context.Context) error { + image := &types.Image{ + Name: info.Image, + RegistryID: info.RegistryID, + Enabled: true, + } + err := c.DBStore.ImageDao.CreateOrUpdate(ctx, image) + if err != nil { + return fmt.Errorf("failed to create image for artifact : [%s] with "+ + regNameFormat, info.Image, info.RegIdentifier) + } + + dbArtifact, err := c.DBStore.ArtifactDao.GetByName(ctx, image.ID, info.Version) + + if err != nil && !errors.Is(err, sql.ErrNoRows) { + return fmt.Errorf("failed to fetch artifact : [%s] with "+ + regNameFormat, info.Image, info.RegIdentifier) + } + + metadata := &database.GenericMetadata{ + Description: info.Description, + } + err2 := c.updateMetadata(dbArtifact, metadata, info, fileInfo) + if err2 != nil { + return fmt.Errorf("failed to update metadata for artifact : [%s] with "+ + regNameFormat, info.Image, info.RegIdentifier) + } + + metadataJSON, err := json.Marshal(metadata) + + if err != nil { + return fmt.Errorf("failed to parse metadata for artifact : [%s] with "+ + regNameFormat, info.Image, info.RegIdentifier) + } + + err = c.DBStore.ArtifactDao.CreateOrUpdate(ctx, &types.Artifact{ + ImageID: image.ID, + Version: info.Version, + Metadata: metadataJSON, + }) + if err != nil { + return fmt.Errorf("failed to create artifact : [%s] with "+ + regNameFormat, info.Image, info.RegIdentifier) + } + return nil + }) + + if err != nil { + return responseHeaders, []error{errcode.ErrCodeUnknown.WithDetail(err)} + } + responseHeaders.Code = http.StatusCreated + return responseHeaders, nil +} + +func (c Controller) updateMetadata(dbArtifact *types.Artifact, metadata *database.GenericMetadata, + info pkg.GenericArtifactInfo, fileInfo pkg.FileInfo) error { + var files []database.File + if dbArtifact != nil { + err := json.Unmarshal(dbArtifact.Metadata, metadata) + if err != nil { + return fmt.Errorf("failed to get metadata for artifact : [%s] with registry : [%s]", info.Image, info.RegIdentifier) + } + fileExist := false + files = metadata.Files + for _, file := range files { + if file.Filename == info.FileName { + fileExist = true + } + } + if !fileExist { + files = append(files, database.File{Size: fileInfo.Size, Filename: fileInfo.Filename, + CreatedAt: time.Now().UnixMilli()}) + metadata.Files = files + metadata.FileCount++ + } + } else { + files = append(files, database.File{Size: fileInfo.Size, Filename: fileInfo.Filename, + CreatedAt: time.Now().UnixMilli()}) + metadata.Files = files + metadata.FileCount++ + } + return nil +} + +func (c Controller) PullArtifact(ctx context.Context, + info pkg.GenericArtifactInfo) (*commons.ResponseHeaders, + *storage.FileReader, []error) { + responseHeaders := &commons.ResponseHeaders{ + Headers: make(map[string]string), + Code: 0, + } + err := pkg.GetRegistryCheckAccess( + ctx, c.DBStore.RegistryDao, c.authorizer, c.spaceStore, info.RegIdentifier, info.ParentID, + enum.PermissionArtifactsDownload, + ) + if err != nil { + return nil, nil, []error{errcode.ErrCodeDenied} + } + + path := "/" + info.Image + "/" + info.Version + "/" + info.FileName + fileReader, _, err := c.fileManager.DownloadFile(ctx, path, types.Registry{ + ID: info.RegistryID, + Name: info.RootIdentifier, + }, info.RootIdentifier) + if err != nil { + return responseHeaders, nil, []error{errcode.ErrCodeUnknown.WithDetail(err)} + } + responseHeaders.Code = http.StatusOK + return responseHeaders, fileReader, nil +} diff --git a/registry/app/pkg/generic/wire.go b/registry/app/pkg/generic/wire.go new file mode 100644 index 000000000..19454f6fb --- /dev/null +++ b/registry/app/pkg/generic/wire.go @@ -0,0 +1,50 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package generic + +import ( + "github.com/harness/gitness/app/auth/authz" + gitnessstore "github.com/harness/gitness/app/store" + "github.com/harness/gitness/registry/app/pkg/filemanager" + "github.com/harness/gitness/registry/app/store" + "github.com/harness/gitness/store/database/dbtx" + + "github.com/google/wire" +) + +func DBStoreProvider( + imageDao store.ImageRepository, + artifactDao store.ArtifactRepository, + bandwidthStatDao store.BandwidthStatRepository, + downloadStatDao store.DownloadStatRepository, + registryDao store.RegistryRepository, +) *DBStore { + return NewDBStore(registryDao, imageDao, artifactDao, bandwidthStatDao, downloadStatDao) +} + +func ControllerProvider( + spaceStore gitnessstore.SpaceStore, + authorizer authz.Authorizer, + fileManager filemanager.FileManager, + dBStore *DBStore, + tx dbtx.Transactor, +) *Controller { + return NewController(spaceStore, authorizer, fileManager, dBStore, tx) +} + +var DBStoreSet = wire.NewSet(DBStoreProvider) +var ControllerSet = wire.NewSet(ControllerProvider) + +var WireSet = wire.NewSet(ControllerSet, DBStoreSet) diff --git a/registry/app/store/database/artifact.go b/registry/app/store/database/artifact.go index 96334d752..d076bd977 100644 --- a/registry/app/store/database/artifact.go +++ b/registry/app/store/database/artifact.go @@ -158,3 +158,15 @@ func (a ArtifactDao) mapToArtifact(_ context.Context, dst *artifactDB) (*types.A UpdatedBy: updatedBy, }, nil } + +type GenericMetadata struct { + Files []File `json:"files"` + Description string `json:"desc"` + FileCount int64 `json:"file_count"` +} + +type File struct { + Size int64 `json:"size"` + Filename string `json:"file_name"` + CreatedAt int64 `json:"created_at"` +} diff --git a/registry/types/artifact.go b/registry/types/artifact.go index 3886f8ffa..c4f722f0a 100644 --- a/registry/types/artifact.go +++ b/registry/types/artifact.go @@ -15,7 +15,10 @@ package types import ( + "encoding/json" "time" + + "github.com/harness/gitness/registry/app/api/openapi/contracts/artifact" ) // Artifact DTO object. @@ -23,8 +26,19 @@ type Artifact struct { ID int64 Version string ImageID int64 + Metadata json.RawMessage CreatedAt time.Time UpdatedAt time.Time CreatedBy int64 UpdatedBy int64 } + +type NonOCIArtifactMetadata struct { + Name string + Size string + PackageType artifact.PackageType + FileCount int64 + IsLatestVersion bool + ModifiedAt time.Time + DownloadCount int64 +}