mirror of
https://github.com/harness/drone.git
synced 2025-05-05 04:30:50 +08:00
fix: [AH-1153]: fix download count per image and per manifest (#3609)
* fix: [AH-1153]: fix query * fix: [AH-1153]: fix download count per image and per manifest
This commit is contained in:
parent
b5b463245f
commit
c58fdbc4d0
@ -0,0 +1 @@
|
|||||||
|
DROP INDEX IF EXISTS download_stat_artifact_id;
|
@ -0,0 +1 @@
|
|||||||
|
CREATE INDEX download_stat_artifact_id ON download_stats(download_stat_artifact_id);
|
@ -0,0 +1 @@
|
|||||||
|
DROP INDEX IF EXISTS download_stat_artifact_id;
|
@ -0,0 +1 @@
|
|||||||
|
CREATE INDEX download_stat_artifact_id ON download_stats(download_stat_artifact_id);
|
@ -518,7 +518,7 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
apiHandler := router.APIHandlerProvider(registryRepository, upstreamProxyConfigRepository, fileManager, tagRepository, manifestRepository, cleanupPolicyRepository, imageRepository, storageDriver, spaceFinder, transactor, authenticator, provider, authorizer, auditService, artifactRepository, webhooksRepository, webhooksExecutionRepository, service2, spacePathStore, reporter10)
|
apiHandler := router.APIHandlerProvider(registryRepository, upstreamProxyConfigRepository, fileManager, tagRepository, manifestRepository, cleanupPolicyRepository, imageRepository, storageDriver, spaceFinder, transactor, authenticator, provider, authorizer, auditService, artifactRepository, webhooksRepository, webhooksExecutionRepository, service2, spacePathStore, reporter10, downloadStatRepository)
|
||||||
mavenDBStore := maven.DBStoreProvider(registryRepository, imageRepository, artifactRepository, spaceStore, bandwidthStatRepository, downloadStatRepository, nodesRepository, upstreamProxyConfigRepository)
|
mavenDBStore := maven.DBStoreProvider(registryRepository, imageRepository, artifactRepository, spaceStore, bandwidthStatRepository, downloadStatRepository, nodesRepository, upstreamProxyConfigRepository)
|
||||||
mavenLocalRegistry := maven.LocalRegistryProvider(mavenDBStore, transactor, fileManager)
|
mavenLocalRegistry := maven.LocalRegistryProvider(mavenDBStore, transactor, fileManager)
|
||||||
mavenController := maven.ProvideProxyController(mavenLocalRegistry, secretService, spaceFinder)
|
mavenController := maven.ProvideProxyController(mavenLocalRegistry, secretService, spaceFinder)
|
||||||
|
@ -497,7 +497,7 @@ func GetPythonArtifactDetail(
|
|||||||
return *artifactDetail
|
return *artifactDetail
|
||||||
}
|
}
|
||||||
|
|
||||||
func GetArtifactSummary(artifact types.ArtifactMetadata) *artifactapi.ArtifactSummaryResponseJSONResponse {
|
func GetArtifactSummary(artifact types.ImageMetadata) *artifactapi.ArtifactSummaryResponseJSONResponse {
|
||||||
createdAt := GetTimeInMs(artifact.CreatedAt)
|
createdAt := GetTimeInMs(artifact.CreatedAt)
|
||||||
modifiedAt := GetTimeInMs(artifact.ModifiedAt)
|
modifiedAt := GetTimeInMs(artifact.ModifiedAt)
|
||||||
artifactVersionSummary := &artifactapi.ArtifactSummary{
|
artifactVersionSummary := &artifactapi.ArtifactSummary{
|
||||||
@ -505,7 +505,6 @@ func GetArtifactSummary(artifact types.ArtifactMetadata) *artifactapi.ArtifactSu
|
|||||||
ModifiedAt: &modifiedAt,
|
ModifiedAt: &modifiedAt,
|
||||||
DownloadsCount: &artifact.DownloadCount,
|
DownloadsCount: &artifact.DownloadCount,
|
||||||
ImageName: artifact.Name,
|
ImageName: artifact.Name,
|
||||||
Labels: &artifact.Labels,
|
|
||||||
PackageType: artifact.PackageType,
|
PackageType: artifact.PackageType,
|
||||||
}
|
}
|
||||||
response := &artifactapi.ArtifactSummaryResponseJSONResponse{
|
response := &artifactapi.ArtifactSummaryResponseJSONResponse{
|
||||||
|
@ -49,6 +49,7 @@ type APIController struct {
|
|||||||
RegistryMetadataHelper RegistryMetadataHelper
|
RegistryMetadataHelper RegistryMetadataHelper
|
||||||
WebhookService WebhookService
|
WebhookService WebhookService
|
||||||
ArtifactEventReporter registryevents.Reporter
|
ArtifactEventReporter registryevents.Reporter
|
||||||
|
DownloadStatRepository store.DownloadStatRepository
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewAPIController(
|
func NewAPIController(
|
||||||
@ -73,6 +74,7 @@ func NewAPIController(
|
|||||||
registryMetadataHelper RegistryMetadataHelper,
|
registryMetadataHelper RegistryMetadataHelper,
|
||||||
webhookService WebhookService,
|
webhookService WebhookService,
|
||||||
artifactEventReporter registryevents.Reporter,
|
artifactEventReporter registryevents.Reporter,
|
||||||
|
downloadStatRepository store.DownloadStatRepository,
|
||||||
) *APIController {
|
) *APIController {
|
||||||
return &APIController{
|
return &APIController{
|
||||||
fileManager: fileManager,
|
fileManager: fileManager,
|
||||||
@ -96,5 +98,6 @@ func NewAPIController(
|
|||||||
RegistryMetadataHelper: registryMetadataHelper,
|
RegistryMetadataHelper: registryMetadataHelper,
|
||||||
WebhookService: webhookService,
|
WebhookService: webhookService,
|
||||||
ArtifactEventReporter: artifactEventReporter,
|
ArtifactEventReporter: artifactEventReporter,
|
||||||
|
DownloadStatRepository: downloadStatRepository,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -75,11 +75,7 @@ func (c *APIController) GetDockerArtifactManifests(
|
|||||||
|
|
||||||
image := string(r.Artifact)
|
image := string(r.Artifact)
|
||||||
version := string(r.Version)
|
version := string(r.Version)
|
||||||
artifactMetadata, err := c.TagStore.GetLatestTagMetadata(ctx, regInfo.parentID, regInfo.RegistryIdentifier, image)
|
manifestDetailsList, err := c.ProcessManifest(ctx, regInfo, image, version)
|
||||||
if err != nil {
|
|
||||||
return artifactManifestsErrorRs(err), nil
|
|
||||||
}
|
|
||||||
manifestDetailsList, err := c.ProcessManifest(ctx, regInfo, image, version, artifactMetadata.DownloadCount)
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return artifactManifestsErrorRs(err), nil
|
return artifactManifestsErrorRs(err), nil
|
||||||
}
|
}
|
||||||
@ -98,7 +94,7 @@ func (c *APIController) GetDockerArtifactManifests(
|
|||||||
|
|
||||||
func (c *APIController) getManifestList(
|
func (c *APIController) getManifestList(
|
||||||
ctx context.Context, reqManifest *ml.DeserializedManifestList, registry *types.Registry, image string,
|
ctx context.Context, reqManifest *ml.DeserializedManifestList, registry *types.Registry, image string,
|
||||||
regInfo *RegistryRequestBaseInfo, downloadCount int64,
|
regInfo *RegistryRequestBaseInfo,
|
||||||
) ([]artifact.DockerManifestDetails, error) {
|
) ([]artifact.DockerManifestDetails, error) {
|
||||||
manifestDetailsList := []artifact.DockerManifestDetails{}
|
manifestDetailsList := []artifact.DockerManifestDetails{}
|
||||||
for _, manifestEntry := range reqManifest.Manifests {
|
for _, manifestEntry := range reqManifest.Manifests {
|
||||||
@ -123,7 +119,11 @@ func (c *APIController) getManifestList(
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
manifestDetailsList = append(manifestDetailsList, getManifestDetails(referencedManifest, mConfig, downloadCount))
|
md, err := c.getManifestDetails(ctx, referencedManifest, mConfig)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
manifestDetailsList = append(manifestDetailsList, md)
|
||||||
}
|
}
|
||||||
return manifestDetailsList, nil
|
return manifestDetailsList, nil
|
||||||
}
|
}
|
||||||
@ -136,22 +136,31 @@ func artifactManifestsErrorRs(err error) artifact.GetDockerArtifactManifestsResp
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func getManifestDetails(
|
func (c *APIController) getManifestDetails(
|
||||||
m *types.Manifest, mConfig *manifestConfig, downloadsCount int64,
|
ctx context.Context,
|
||||||
) artifact.DockerManifestDetails {
|
m *types.Manifest, mConfig *manifestConfig,
|
||||||
|
) (artifact.DockerManifestDetails, error) {
|
||||||
createdAt := GetTimeInMs(m.CreatedAt)
|
createdAt := GetTimeInMs(m.CreatedAt)
|
||||||
size := GetSize(m.TotalSize)
|
size := GetSize(m.TotalSize)
|
||||||
|
dgst, _ := types.NewDigest(m.Digest)
|
||||||
|
image, err := c.ImageStore.GetByName(ctx, m.RegistryID, m.ImageName)
|
||||||
|
if err != nil {
|
||||||
|
return artifact.DockerManifestDetails{}, err
|
||||||
|
}
|
||||||
manifestDetails := artifact.DockerManifestDetails{
|
manifestDetails := artifact.DockerManifestDetails{
|
||||||
Digest: m.Digest.String(),
|
Digest: m.Digest.String(),
|
||||||
CreatedAt: &createdAt,
|
CreatedAt: &createdAt,
|
||||||
Size: &size,
|
Size: &size,
|
||||||
DownloadsCount: &downloadsCount,
|
}
|
||||||
|
downloadCountMap, err := c.DownloadStatRepository.GetTotalDownloadsForManifests(ctx, []string{string(dgst)}, image.ID)
|
||||||
|
if err == nil && len(downloadCountMap) > 0 {
|
||||||
|
downloadCount := downloadCountMap[string(dgst)]
|
||||||
|
manifestDetails.DownloadsCount = &downloadCount
|
||||||
}
|
}
|
||||||
if mConfig != nil {
|
if mConfig != nil {
|
||||||
manifestDetails.OsArch = fmt.Sprintf("%s/%s", mConfig.Os, mConfig.Arch)
|
manifestDetails.OsArch = fmt.Sprintf("%s/%s", mConfig.Os, mConfig.Arch)
|
||||||
}
|
}
|
||||||
return manifestDetails
|
return manifestDetails, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// ProcessManifest processes a Docker artifact manifest by retrieving the manifest details from the database,
|
// ProcessManifest processes a Docker artifact manifest by retrieving the manifest details from the database,
|
||||||
@ -161,7 +170,7 @@ func getManifestDetails(
|
|||||||
func (c *APIController) ProcessManifest(
|
func (c *APIController) ProcessManifest(
|
||||||
ctx context.Context,
|
ctx context.Context,
|
||||||
regInfo *RegistryRequestBaseInfo,
|
regInfo *RegistryRequestBaseInfo,
|
||||||
image, version string, downloadCount int64,
|
image, version string,
|
||||||
) ([]artifact.DockerManifestDetails, error) {
|
) ([]artifact.DockerManifestDetails, error) {
|
||||||
registry, err := c.RegistryRepository.GetByParentIDAndName(ctx, regInfo.parentID, regInfo.RegistryIdentifier)
|
registry, err := c.RegistryRepository.GetByParentIDAndName(ctx, regInfo.parentID, regInfo.RegistryIdentifier)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -186,15 +195,23 @@ func (c *APIController) ProcessManifest(
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
manifestDetailsList = append(manifestDetailsList, getManifestDetails(m, mConfig, downloadCount))
|
md, err := c.getManifestDetails(ctx, m, mConfig)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
manifestDetailsList = append(manifestDetailsList, md)
|
||||||
case *ocischema.DeserializedManifest:
|
case *ocischema.DeserializedManifest:
|
||||||
mConfig, err := getManifestConfig(ctx, reqManifest.Config().Digest, regInfo.RootIdentifier, c.StorageDriver)
|
mConfig, err := getManifestConfig(ctx, reqManifest.Config().Digest, regInfo.RootIdentifier, c.StorageDriver)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
manifestDetailsList = append(manifestDetailsList, getManifestDetails(m, mConfig, downloadCount))
|
md, err := c.getManifestDetails(ctx, m, mConfig)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
manifestDetailsList = append(manifestDetailsList, md)
|
||||||
case *ml.DeserializedManifestList:
|
case *ml.DeserializedManifestList:
|
||||||
manifestDetailsList, err = c.getManifestList(ctx, reqManifest, registry, image, regInfo, downloadCount)
|
manifestDetailsList, err = c.getManifestList(ctx, reqManifest, registry, image, regInfo)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return nil, err
|
return nil, err
|
||||||
}
|
}
|
||||||
|
@ -74,29 +74,54 @@ func (c *APIController) GetArtifactSummary(
|
|||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
var metadata *types.ArtifactMetadata
|
metadata, err := c.getImageMetadata(ctx, registry, image)
|
||||||
if registry.PackageType == artifact.PackageTypeDOCKER || registry.PackageType == artifact.PackageTypeHELM {
|
if err != nil {
|
||||||
metadata, err = c.TagStore.GetLatestTagMetadata(ctx, regInfo.parentID, regInfo.RegistryIdentifier, image)
|
return artifact.GetArtifactSummary500JSONResponse{
|
||||||
|
InternalServerErrorJSONResponse: artifact.InternalServerErrorJSONResponse(
|
||||||
if err != nil {
|
*GetErrorResponse(http.StatusInternalServerError, err.Error()),
|
||||||
return artifact.GetArtifactSummary500JSONResponse{
|
),
|
||||||
InternalServerErrorJSONResponse: artifact.InternalServerErrorJSONResponse(
|
}, nil
|
||||||
*GetErrorResponse(http.StatusInternalServerError, err.Error()),
|
|
||||||
),
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
} else {
|
|
||||||
metadata, err = c.ArtifactStore.GetLatestArtifactMetadata(ctx, regInfo.parentID, regInfo.RegistryIdentifier, image)
|
|
||||||
|
|
||||||
if err != nil {
|
|
||||||
return artifact.GetArtifactSummary500JSONResponse{
|
|
||||||
InternalServerErrorJSONResponse: artifact.InternalServerErrorJSONResponse(
|
|
||||||
*GetErrorResponse(http.StatusInternalServerError, err.Error()),
|
|
||||||
),
|
|
||||||
}, nil
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
return artifact.GetArtifactSummary200JSONResponse{
|
return artifact.GetArtifactSummary200JSONResponse{
|
||||||
ArtifactSummaryResponseJSONResponse: *GetArtifactSummary(*metadata),
|
ArtifactSummaryResponseJSONResponse: *GetArtifactSummary(*metadata),
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (c *APIController) getImageMetadata(
|
||||||
|
ctx context.Context,
|
||||||
|
registry *types.Registry,
|
||||||
|
image string,
|
||||||
|
) (*types.ImageMetadata, error) {
|
||||||
|
img, err := c.ImageStore.GetByName(ctx, registry.ID, image)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
downloadCount, err := c.DownloadStatRepository.GetTotalDownloadsForImage(ctx, img.ID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
imgMetadata := &types.ImageMetadata{
|
||||||
|
Name: image,
|
||||||
|
DownloadCount: downloadCount,
|
||||||
|
RepoName: registry.Name,
|
||||||
|
PackageType: registry.PackageType,
|
||||||
|
CreatedAt: img.CreatedAt,
|
||||||
|
}
|
||||||
|
|
||||||
|
if registry.PackageType == artifact.PackageTypeDOCKER || registry.PackageType == artifact.PackageTypeHELM {
|
||||||
|
latestTag, err := c.TagStore.GetLatestTag(ctx, registry.ID, image)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
imgMetadata.LatestVersion = latestTag.Name
|
||||||
|
imgMetadata.ModifiedAt = latestTag.UpdatedAt
|
||||||
|
} else {
|
||||||
|
latestArtifact, err := c.ArtifactStore.GetLatestByImageID(ctx, img.ID)
|
||||||
|
if err != nil {
|
||||||
|
return nil, err
|
||||||
|
}
|
||||||
|
imgMetadata.LatestVersion = latestArtifact.Version
|
||||||
|
imgMetadata.ModifiedAt = latestArtifact.UpdatedAt
|
||||||
|
}
|
||||||
|
return imgMetadata, nil
|
||||||
|
}
|
||||||
|
@ -82,14 +82,38 @@ func (c *APIController) GetAllArtifactVersions(
|
|||||||
if err != nil {
|
if err != nil {
|
||||||
return throw500Error(err)
|
return throw500Error(err)
|
||||||
}
|
}
|
||||||
|
img, err := c.ImageStore.GetByName(ctx, registry.ID, image)
|
||||||
|
if err != nil {
|
||||||
|
return throw500Error(err)
|
||||||
|
}
|
||||||
|
//nolint:nestif
|
||||||
if registry.PackageType == artifact.PackageTypeDOCKER || registry.PackageType == artifact.PackageTypeHELM {
|
if registry.PackageType == artifact.PackageTypeDOCKER || registry.PackageType == artifact.PackageTypeHELM {
|
||||||
tags, err := c.TagStore.GetAllTagsByRepoAndImage(
|
tags, err := c.TagStore.GetAllTagsByRepoAndImage(
|
||||||
ctx, regInfo.parentID, regInfo.RegistryIdentifier,
|
ctx, regInfo.parentID, regInfo.RegistryIdentifier,
|
||||||
image, regInfo.sortByField, regInfo.sortByOrder, regInfo.limit, regInfo.offset, regInfo.searchTerm,
|
image, regInfo.sortByField, regInfo.sortByOrder, regInfo.limit, regInfo.offset, regInfo.searchTerm,
|
||||||
)
|
)
|
||||||
|
if err != nil {
|
||||||
|
return throw500Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
count, _ := c.TagStore.CountAllTagsByRepoAndImage(
|
var digests []string
|
||||||
|
for _, tag := range *tags {
|
||||||
|
if tag.Digest != "" {
|
||||||
|
digests = append(digests, tag.Digest)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
counts, err := c.DownloadStatRepository.GetTotalDownloadsForManifests(ctx, digests, img.ID)
|
||||||
|
if err != nil {
|
||||||
|
return throw500Error(err)
|
||||||
|
}
|
||||||
|
|
||||||
|
for i, tag := range *tags {
|
||||||
|
if tag.Digest != "" {
|
||||||
|
(*tags)[i].DownloadCount = counts[tag.Digest]
|
||||||
|
}
|
||||||
|
}
|
||||||
|
count, err := c.TagStore.CountAllTagsByRepoAndImage(
|
||||||
ctx, regInfo.parentID, regInfo.RegistryIdentifier,
|
ctx, regInfo.parentID, regInfo.RegistryIdentifier,
|
||||||
image, regInfo.searchTerm,
|
image, regInfo.searchTerm,
|
||||||
)
|
)
|
||||||
|
@ -47,8 +47,14 @@ func (c *APIController) GetAllRegistries(
|
|||||||
registryIDsParam: nil,
|
registryIDsParam: nil,
|
||||||
recursive: r.Params.Recursive != nil && bool(*r.Params.Recursive), // default is false
|
recursive: r.Params.Recursive != nil && bool(*r.Params.Recursive), // default is false
|
||||||
}
|
}
|
||||||
regInfo, _ := c.GetRegistryRequestInfo(ctx, *registryRequestParams)
|
regInfo, err := c.GetRegistryRequestInfo(ctx, *registryRequestParams)
|
||||||
|
if err != nil {
|
||||||
|
return artifact.GetAllRegistries400JSONResponse{
|
||||||
|
BadRequestJSONResponse: artifact.BadRequestJSONResponse(
|
||||||
|
*GetErrorResponse(http.StatusBadRequest, err.Error()),
|
||||||
|
),
|
||||||
|
}, nil
|
||||||
|
}
|
||||||
space, err := c.SpaceFinder.FindByRef(ctx, regInfo.ParentRef)
|
space, err := c.SpaceFinder.FindByRef(ctx, regInfo.ParentRef)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return artifact.GetAllRegistries400JSONResponse{
|
return artifact.GetAllRegistries400JSONResponse{
|
||||||
|
@ -94,6 +94,7 @@ func (c *APIController) UpdateArtifactLabels(
|
|||||||
return throwModifyArtifact400Error(err), nil
|
return throwModifyArtifact400Error(err), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// TODO: use the correct way to get download count if this endpoint is used
|
||||||
tag, err := c.TagStore.GetLatestTagMetadata(ctx, regInfo.parentID, regInfo.RegistryIdentifier, a)
|
tag, err := c.TagStore.GetLatestTagMetadata(ctx, regInfo.parentID, regInfo.RegistryIdentifier, a)
|
||||||
|
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
@ -76,6 +76,7 @@ func NewAPIHandler(
|
|||||||
webhookService registrywebhook.Service,
|
webhookService registrywebhook.Service,
|
||||||
spacePathStore corestore.SpacePathStore,
|
spacePathStore corestore.SpacePathStore,
|
||||||
artifactEventReporter registryevents.Reporter,
|
artifactEventReporter registryevents.Reporter,
|
||||||
|
downloadStatRepository store.DownloadStatRepository,
|
||||||
) APIHandler {
|
) APIHandler {
|
||||||
r := chi.NewRouter()
|
r := chi.NewRouter()
|
||||||
r.Use(audit.Middleware())
|
r.Use(audit.Middleware())
|
||||||
@ -104,6 +105,7 @@ func NewAPIHandler(
|
|||||||
registryMetadataHelper,
|
registryMetadataHelper,
|
||||||
&webhookService,
|
&webhookService,
|
||||||
artifactEventReporter,
|
artifactEventReporter,
|
||||||
|
downloadStatRepository,
|
||||||
)
|
)
|
||||||
|
|
||||||
handler := artifact.NewStrictHandler(apiController, []artifact.StrictMiddlewareFunc{})
|
handler := artifact.NewStrictHandler(apiController, []artifact.StrictMiddlewareFunc{})
|
||||||
|
@ -74,6 +74,7 @@ func APIHandlerProvider(
|
|||||||
webhookService *registrywebhook.Service,
|
webhookService *registrywebhook.Service,
|
||||||
spacePathStore corestore.SpacePathStore,
|
spacePathStore corestore.SpacePathStore,
|
||||||
artifactEventReporter *registryevents.Reporter,
|
artifactEventReporter *registryevents.Reporter,
|
||||||
|
downloadStatRepository store.DownloadStatRepository,
|
||||||
) harness.APIHandler {
|
) harness.APIHandler {
|
||||||
return harness.NewAPIHandler(
|
return harness.NewAPIHandler(
|
||||||
repoDao,
|
repoDao,
|
||||||
@ -97,6 +98,7 @@ func APIHandlerProvider(
|
|||||||
*webhookService,
|
*webhookService,
|
||||||
spacePathStore,
|
spacePathStore,
|
||||||
*artifactEventReporter,
|
*artifactEventReporter,
|
||||||
|
downloadStatRepository,
|
||||||
)
|
)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -486,10 +486,17 @@ type ArtifactRepository interface {
|
|||||||
*[]types.Artifact,
|
*[]types.Artifact,
|
||||||
error,
|
error,
|
||||||
)
|
)
|
||||||
|
GetLatestByImageID(ctx context.Context, imageID int64) (*types.Artifact, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
type DownloadStatRepository interface {
|
type DownloadStatRepository interface {
|
||||||
Create(ctx context.Context, downloadStat *types.DownloadStat) error
|
Create(ctx context.Context, downloadStat *types.DownloadStat) error
|
||||||
|
GetTotalDownloadsForImage(ctx context.Context, imageID int64) (int64, error)
|
||||||
|
GetTotalDownloadsForManifests(
|
||||||
|
ctx context.Context,
|
||||||
|
artifactVersion []string,
|
||||||
|
imageID int64,
|
||||||
|
) (map[string]int64, error)
|
||||||
}
|
}
|
||||||
|
|
||||||
type BandwidthStatRepository interface {
|
type BandwidthStatRepository interface {
|
||||||
|
@ -110,6 +110,25 @@ func (a ArtifactDao) GetByRegistryIDAndImage(ctx context.Context, registryID int
|
|||||||
return &artifacts, nil
|
return &artifacts, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func (a ArtifactDao) GetLatestByImageID(ctx context.Context, imageID int64) (*types.Artifact, error) {
|
||||||
|
q := databaseg.Builder.Select(util.ArrToStringByDelimiter(util.GetDBTagsFromStruct(artifactDB{}), ",")).
|
||||||
|
From("artifacts").
|
||||||
|
Where("artifact_image_id = ?", imageID).OrderBy("artifact_updated_at DESC").Limit(1)
|
||||||
|
|
||||||
|
sql, args, err := q.ToSql()
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.Wrap(err, "Failed to convert query to sql")
|
||||||
|
}
|
||||||
|
|
||||||
|
db := dbtx.GetAccessor(ctx, a.db)
|
||||||
|
|
||||||
|
dst := new(artifactDB)
|
||||||
|
if err = db.GetContext(ctx, dst, sql, args...); err != nil {
|
||||||
|
return nil, databaseg.ProcessSQLErrorf(ctx, err, "Failed to get artifact")
|
||||||
|
}
|
||||||
|
return a.mapToArtifact(ctx, dst)
|
||||||
|
}
|
||||||
|
|
||||||
func (a ArtifactDao) CreateOrUpdate(ctx context.Context, artifact *types.Artifact) error {
|
func (a ArtifactDao) CreateOrUpdate(ctx context.Context, artifact *types.Artifact) error {
|
||||||
const sqlQuery = `
|
const sqlQuery = `
|
||||||
INSERT INTO artifacts (
|
INSERT INTO artifacts (
|
||||||
|
@ -17,16 +17,19 @@ package database
|
|||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
"database/sql"
|
"database/sql"
|
||||||
"errors"
|
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
"github.com/harness/gitness/app/api/request"
|
"github.com/harness/gitness/app/api/request"
|
||||||
"github.com/harness/gitness/registry/app/store"
|
"github.com/harness/gitness/registry/app/store"
|
||||||
|
"github.com/harness/gitness/registry/app/store/database/util"
|
||||||
"github.com/harness/gitness/registry/types"
|
"github.com/harness/gitness/registry/types"
|
||||||
databaseg "github.com/harness/gitness/store/database"
|
databaseg "github.com/harness/gitness/store/database"
|
||||||
"github.com/harness/gitness/store/database/dbtx"
|
"github.com/harness/gitness/store/database/dbtx"
|
||||||
|
|
||||||
|
sq "github.com/Masterminds/squirrel"
|
||||||
"github.com/jmoiron/sqlx"
|
"github.com/jmoiron/sqlx"
|
||||||
|
"github.com/pkg/errors"
|
||||||
|
"github.com/rs/zerolog/log"
|
||||||
)
|
)
|
||||||
|
|
||||||
type DownloadStatDao struct {
|
type DownloadStatDao struct {
|
||||||
@ -81,8 +84,69 @@ func (d DownloadStatDao) Create(ctx context.Context, downloadStat *types.Downloa
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (d DownloadStatDao) mapToInternalDownloadStat(ctx context.Context,
|
func (d DownloadStatDao) GetTotalDownloadsForImage(ctx context.Context, imageID int64) (int64, error) {
|
||||||
in *types.DownloadStat) *downloadStatDB {
|
q := databaseg.Builder.Select(`count(*)`).
|
||||||
|
From("artifacts art").Where("art.artifact_image_id = ?", imageID).
|
||||||
|
Join("download_stats ds ON ds.download_stat_artifact_id = art.artifact_id")
|
||||||
|
|
||||||
|
sql, args, err := q.ToSql()
|
||||||
|
if err != nil {
|
||||||
|
return 0, errors.Wrap(err, "Failed to convert query to sql")
|
||||||
|
}
|
||||||
|
// Log the final sql query
|
||||||
|
finalQuery := util.FormatQuery(sql, args)
|
||||||
|
log.Ctx(ctx).Debug().Str("sql", finalQuery).Msg("Executing GetTotalDownloadsForImage query")
|
||||||
|
// Execute query
|
||||||
|
db := dbtx.GetAccessor(ctx, d.db)
|
||||||
|
|
||||||
|
var count int64
|
||||||
|
err = db.QueryRowContext(ctx, sql, args...).Scan(&count)
|
||||||
|
if err != nil {
|
||||||
|
return 0, databaseg.ProcessSQLErrorf(ctx, err, "Failed executing count query")
|
||||||
|
}
|
||||||
|
return count, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d DownloadStatDao) GetTotalDownloadsForManifests(
|
||||||
|
ctx context.Context,
|
||||||
|
artifactVersions []string,
|
||||||
|
imageID int64,
|
||||||
|
) (map[string]int64, error) {
|
||||||
|
q := databaseg.Builder.Select(`art.artifact_version, count(*)`).
|
||||||
|
From("artifacts art").
|
||||||
|
Join("download_stats ds ON ds.download_stat_artifact_id = art.artifact_id").Where(sq.And{
|
||||||
|
sq.Eq{"artifact_image_id": imageID},
|
||||||
|
sq.Eq{"artifact_version": artifactVersions},
|
||||||
|
}, "art").GroupBy("art.artifact_version")
|
||||||
|
|
||||||
|
sql, args, err := q.ToSql()
|
||||||
|
if err != nil {
|
||||||
|
return nil, errors.Wrap(err, "Failed to convert query to sql")
|
||||||
|
}
|
||||||
|
// Log the final sql query
|
||||||
|
finalQuery := util.FormatQuery(sql, args)
|
||||||
|
log.Ctx(ctx).Debug().Str("sql", finalQuery).Msg("Executing GetTotalDownloadsForManifests query")
|
||||||
|
// Execute query
|
||||||
|
db := dbtx.GetAccessor(ctx, d.db)
|
||||||
|
|
||||||
|
dst := []*versionsCountDB{}
|
||||||
|
if err = db.SelectContext(ctx, &dst, sql, args...); err != nil {
|
||||||
|
return nil, databaseg.ProcessSQLErrorf(ctx, err, "Failed executing query")
|
||||||
|
}
|
||||||
|
|
||||||
|
// Convert the slice to a map
|
||||||
|
result := make(map[string]int64)
|
||||||
|
for _, v := range dst {
|
||||||
|
result[v.Version] = v.Count
|
||||||
|
}
|
||||||
|
|
||||||
|
return result, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (d DownloadStatDao) mapToInternalDownloadStat(
|
||||||
|
ctx context.Context,
|
||||||
|
in *types.DownloadStat,
|
||||||
|
) *downloadStatDB {
|
||||||
session, _ := request.AuthSessionFrom(ctx)
|
session, _ := request.AuthSessionFrom(ctx)
|
||||||
if in.CreatedAt.IsZero() {
|
if in.CreatedAt.IsZero() {
|
||||||
in.CreatedAt = time.Now()
|
in.CreatedAt = time.Now()
|
||||||
@ -104,3 +168,8 @@ func (d DownloadStatDao) mapToInternalDownloadStat(ctx context.Context,
|
|||||||
UpdatedBy: session.Principal.ID,
|
UpdatedBy: session.Principal.ID,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type versionsCountDB struct {
|
||||||
|
Version string `db:"artifact_version"`
|
||||||
|
Count int64 `db:"count"`
|
||||||
|
}
|
||||||
|
@ -253,8 +253,10 @@ func (i ImageDao) CreateOrUpdate(ctx context.Context, image *types.Image) error
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (i ImageDao) GetLabelsByParentIDAndRepo(ctx context.Context, parentID int64, repo string,
|
func (i ImageDao) GetLabelsByParentIDAndRepo(
|
||||||
limit int, offset int, search string) (labels []string, err error) {
|
ctx context.Context, parentID int64, repo string,
|
||||||
|
limit int, offset int, search string,
|
||||||
|
) (labels []string, err error) {
|
||||||
q := databaseg.Builder.Select("a.image_labels as labels").
|
q := databaseg.Builder.Select("a.image_labels as labels").
|
||||||
From("images a").
|
From("images a").
|
||||||
Join("registries r ON r.registry_id = a.image_registry_id").
|
Join("registries r ON r.registry_id = a.image_registry_id").
|
||||||
@ -264,7 +266,8 @@ func (i ImageDao) GetLabelsByParentIDAndRepo(ctx context.Context, parentID int64
|
|||||||
q = q.Where("a.image_labels LIKE ?", "%"+search+"%")
|
q = q.Where("a.image_labels LIKE ?", "%"+search+"%")
|
||||||
}
|
}
|
||||||
|
|
||||||
q = q.OrderBy("a.image_labels ASC").Limit(uint64(limit)).Offset(uint64(offset))
|
q = q.OrderBy("a.image_labels ASC").
|
||||||
|
Limit(util.SafeIntToUInt64(limit)).Offset(util.SafeIntToUInt64(offset))
|
||||||
|
|
||||||
sql, args, err := q.ToSql()
|
sql, args, err := q.ToSql()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -282,8 +285,10 @@ func (i ImageDao) GetLabelsByParentIDAndRepo(ctx context.Context, parentID int64
|
|||||||
return i.mapToImageLabels(dst), nil
|
return i.mapToImageLabels(dst), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (i ImageDao) CountLabelsByParentIDAndRepo(ctx context.Context, parentID int64, repo,
|
func (i ImageDao) CountLabelsByParentIDAndRepo(
|
||||||
search string) (count int64, err error) {
|
ctx context.Context, parentID int64, repo,
|
||||||
|
search string,
|
||||||
|
) (count int64, err error) {
|
||||||
q := databaseg.Builder.Select("a.image_labels as labels").
|
q := databaseg.Builder.Select("a.image_labels as labels").
|
||||||
From("images a").
|
From("images a").
|
||||||
Join("registries r ON r.registry_id = a.image_registry_id").
|
Join("registries r ON r.registry_id = a.image_registry_id").
|
||||||
@ -309,8 +314,10 @@ func (i ImageDao) CountLabelsByParentIDAndRepo(ctx context.Context, parentID int
|
|||||||
return int64(len(dst)), nil
|
return int64(len(dst)), nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (i ImageDao) GetByRepoAndName(ctx context.Context, parentID int64,
|
func (i ImageDao) GetByRepoAndName(
|
||||||
repo string, name string) (*types.Image, error) {
|
ctx context.Context, parentID int64,
|
||||||
|
repo string, name string,
|
||||||
|
) (*types.Image, error) {
|
||||||
q := databaseg.Builder.Select("a.image_id, a.image_name, "+
|
q := databaseg.Builder.Select("a.image_id, a.image_name, "+
|
||||||
" a.image_registry_id, a.image_labels, a.image_created_at, "+
|
" a.image_registry_id, a.image_labels, a.image_created_at, "+
|
||||||
" a.image_updated_at, a.image_created_by, a.image_updated_by").
|
" a.image_updated_at, a.image_created_by, a.image_updated_by").
|
||||||
@ -449,8 +456,10 @@ func (i ImageDao) mapToImageLabels(dst []*imageLabelDB) []string {
|
|||||||
return res
|
return res
|
||||||
}
|
}
|
||||||
|
|
||||||
func (i ImageDao) mapToImageLabel(elements map[string]bool, res []string,
|
func (i ImageDao) mapToImageLabel(
|
||||||
dst *imageLabelDB) (map[string]bool, []string) {
|
elements map[string]bool, res []string,
|
||||||
|
dst *imageLabelDB,
|
||||||
|
) (map[string]bool, []string) {
|
||||||
if dst == nil {
|
if dst == nil {
|
||||||
return elements, res
|
return elements, res
|
||||||
}
|
}
|
||||||
|
@ -95,6 +95,7 @@ type tagMetadataDB struct {
|
|||||||
NonConformant bool `db:"manifest_non_conformant"`
|
NonConformant bool `db:"manifest_non_conformant"`
|
||||||
Payload []byte `db:"manifest_payload"`
|
Payload []byte `db:"manifest_payload"`
|
||||||
MediaType string `db:"mt_media_type"`
|
MediaType string `db:"mt_media_type"`
|
||||||
|
Digest []byte `db:"manifest_digest"`
|
||||||
DownloadCount int64 `db:"download_count"`
|
DownloadCount int64 `db:"download_count"`
|
||||||
}
|
}
|
||||||
|
|
||||||
@ -969,38 +970,16 @@ func (t tagDao) GetAllTagsByRepoAndImage(
|
|||||||
image string, sortByField string, sortByOrder string, limit int, offset int,
|
image string, sortByField string, sortByOrder string, limit int, offset int,
|
||||||
search string,
|
search string,
|
||||||
) (*[]types.TagMetadata, error) {
|
) (*[]types.TagMetadata, error) {
|
||||||
// Define download count subquery
|
q := databaseg.Builder.Select(
|
||||||
downloadCountSubquery := `
|
`t.tag_name as name, m.manifest_total_size as size,
|
||||||
SELECT
|
r.registry_package_type as package_type, t.tag_updated_at as modified_at,
|
||||||
a.artifact_image_id,
|
m.manifest_schema_version, m.manifest_non_conformant, m.manifest_payload,
|
||||||
COUNT(d.download_stat_id) AS download_count,
|
mt.mt_media_type, m.manifest_digest`,
|
||||||
i.image_name,
|
).
|
||||||
i.image_registry_id
|
|
||||||
FROM artifacts a
|
|
||||||
JOIN download_stats d ON d.download_stat_artifact_id = a.artifact_id
|
|
||||||
JOIN images i ON i.image_id = a.artifact_image_id
|
|
||||||
GROUP BY a.artifact_image_id, i.image_name, i.image_registry_id
|
|
||||||
`
|
|
||||||
|
|
||||||
// Build the main query
|
|
||||||
q := databaseg.Builder.
|
|
||||||
Select(`
|
|
||||||
t.tag_name AS name,
|
|
||||||
m.manifest_total_size AS size,
|
|
||||||
r.registry_package_type AS package_type,
|
|
||||||
t.tag_updated_at AS modified_at,
|
|
||||||
m.manifest_schema_version,
|
|
||||||
m.manifest_non_conformant,
|
|
||||||
m.manifest_payload,
|
|
||||||
mt.mt_media_type,
|
|
||||||
COALESCE(dc.download_count, 0) AS download_count
|
|
||||||
`).
|
|
||||||
From("tags t").
|
From("tags t").
|
||||||
Join("registries r ON t.tag_registry_id = r.registry_id").
|
Join("registries r ON t.tag_registry_id = r.registry_id").
|
||||||
Join("manifests m ON t.tag_manifest_id = m.manifest_id").
|
Join("manifests m ON t.tag_manifest_id = m.manifest_id").
|
||||||
Join("media_types mt ON mt.mt_id = m.manifest_media_type_id").
|
Join("media_types mt ON mt.mt_id = m.manifest_media_type_id").
|
||||||
LeftJoin(fmt.Sprintf("(%s) AS dc ON t.tag_image_name = dc.image_name "+
|
|
||||||
"AND t.tag_registry_id = dc.image_registry_id", downloadCountSubquery)).
|
|
||||||
Where(
|
Where(
|
||||||
"r.registry_parent_id = ? AND r.registry_name = ? AND t.tag_image_name = ?",
|
"r.registry_parent_id = ? AND r.registry_name = ? AND t.tag_image_name = ?",
|
||||||
parentID, repoKey, image,
|
parentID, repoKey, image,
|
||||||
@ -1009,11 +988,7 @@ func (t tagDao) GetAllTagsByRepoAndImage(
|
|||||||
if search != "" {
|
if search != "" {
|
||||||
q = q.Where("tag_name LIKE ?", sqlPartialMatch(search))
|
q = q.Where("tag_name LIKE ?", sqlPartialMatch(search))
|
||||||
}
|
}
|
||||||
sortField := "tag_" + sortByField
|
q = q.OrderBy("t.tag_" + sortByField + " " + sortByOrder).Limit(uint64(limit)).Offset(uint64(offset)) //nolint:gosec
|
||||||
if sortByField == downloadCount {
|
|
||||||
sortField = downloadCount
|
|
||||||
}
|
|
||||||
q = q.OrderBy(sortField + " " + sortByOrder).Limit(uint64(limit)).Offset(uint64(offset)) //nolint:gosec
|
|
||||||
|
|
||||||
sql, args, err := q.ToSql()
|
sql, args, err := q.ToSql()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
@ -1039,7 +1014,7 @@ func (t tagDao) CountAllTagsByRepoAndImage(
|
|||||||
Join("manifests ON tag_manifest_id = manifest_id").
|
Join("manifests ON tag_manifest_id = manifest_id").
|
||||||
Where(
|
Where(
|
||||||
"registry_parent_id = ? AND registry_name = ?"+
|
"registry_parent_id = ? AND registry_name = ?"+
|
||||||
"AND tag_image_name = ?", parentID, repoKey, image,
|
" AND tag_image_name = ?", parentID, repoKey, image,
|
||||||
)
|
)
|
||||||
|
|
||||||
if search != "" {
|
if search != "" {
|
||||||
@ -1222,7 +1197,7 @@ func (t tagDao) mapToTagMetadata(
|
|||||||
_ context.Context,
|
_ context.Context,
|
||||||
dst *tagMetadataDB,
|
dst *tagMetadataDB,
|
||||||
) (*types.TagMetadata, error) {
|
) (*types.TagMetadata, error) {
|
||||||
return &types.TagMetadata{
|
tagMetadata := &types.TagMetadata{
|
||||||
Name: dst.Name,
|
Name: dst.Name,
|
||||||
Size: dst.Size,
|
Size: dst.Size,
|
||||||
PackageType: dst.PackageType,
|
PackageType: dst.PackageType,
|
||||||
@ -1233,7 +1208,13 @@ func (t tagDao) mapToTagMetadata(
|
|||||||
MediaType: dst.MediaType,
|
MediaType: dst.MediaType,
|
||||||
Payload: dst.Payload,
|
Payload: dst.Payload,
|
||||||
DownloadCount: dst.DownloadCount,
|
DownloadCount: dst.DownloadCount,
|
||||||
}, nil
|
}
|
||||||
|
if dst.Digest != nil {
|
||||||
|
dgst := types.Digest(util.GetHexEncodedString(dst.Digest))
|
||||||
|
tagMetadata.Digest = string(dgst)
|
||||||
|
}
|
||||||
|
|
||||||
|
return tagMetadata, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
func (t tagDao) mapToTagDetail(
|
func (t tagDao) mapToTagDetail(
|
||||||
|
@ -45,6 +45,16 @@ type ArtifactMetadata struct {
|
|||||||
Version string
|
Version string
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type ImageMetadata struct {
|
||||||
|
Name string
|
||||||
|
RepoName string
|
||||||
|
DownloadCount int64
|
||||||
|
PackageType artifact.PackageType
|
||||||
|
LatestVersion string
|
||||||
|
CreatedAt time.Time
|
||||||
|
ModifiedAt time.Time
|
||||||
|
}
|
||||||
|
|
||||||
type TagMetadata struct {
|
type TagMetadata struct {
|
||||||
Name string
|
Name string
|
||||||
Size string
|
Size string
|
||||||
@ -55,6 +65,7 @@ type TagMetadata struct {
|
|||||||
NonConformant bool
|
NonConformant bool
|
||||||
Payload Payload
|
Payload Payload
|
||||||
MediaType string
|
MediaType string
|
||||||
|
Digest string
|
||||||
DownloadCount int64
|
DownloadCount int64
|
||||||
}
|
}
|
||||||
|
|
||||||
|
Loading…
Reference in New Issue
Block a user