mirror of
https://github.com/harness/drone.git
synced 2025-05-03 11:29:23 +08:00
fb65327419
Added support for: 1. Default library/ prefix for docker hub if required. 2. Implemented manifest list support for upstream proxies. New table introduced: `oci_image_index_mappings` 3. Fixed Secret issue on UI for both gitness and harness code 4. Code refactoring to bring controller inside wire and other minor fixes around linting 5. Fixed few bugs around upstream proxy.
1132 lines
36 KiB
Go
1132 lines
36 KiB
Go
// Source: https://gitlab.com/gitlab-org/container-registry
|
|
|
|
// Copyright 2019 Gitlab Inc.
|
|
//
|
|
// Licensed under the Apache License, Version 2.0 (the "License");
|
|
// you may not use this file except in compliance with the License.
|
|
// You may obtain a copy of the License at
|
|
//
|
|
// http://www.apache.org/licenses/LICENSE-2.0
|
|
//
|
|
// Unless required by applicable law or agreed to in writing, software
|
|
// distributed under the License is distributed on an "AS IS" BASIS,
|
|
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
// See the License for the specific language governing permissions and
|
|
// limitations under the License.
|
|
|
|
package docker
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"errors"
|
|
"fmt"
|
|
"time"
|
|
|
|
gas "github.com/harness/gitness/app/store"
|
|
"github.com/harness/gitness/registry/app/api/openapi/contracts/artifact"
|
|
"github.com/harness/gitness/registry/app/event"
|
|
"github.com/harness/gitness/registry/app/manifest"
|
|
"github.com/harness/gitness/registry/app/manifest/manifestlist"
|
|
"github.com/harness/gitness/registry/app/manifest/ocischema"
|
|
"github.com/harness/gitness/registry/app/manifest/schema2"
|
|
"github.com/harness/gitness/registry/app/pkg"
|
|
"github.com/harness/gitness/registry/app/pkg/commons"
|
|
"github.com/harness/gitness/registry/app/store"
|
|
"github.com/harness/gitness/registry/app/store/database/util"
|
|
"github.com/harness/gitness/registry/gc"
|
|
"github.com/harness/gitness/registry/types"
|
|
gitnessstore "github.com/harness/gitness/store"
|
|
db "github.com/harness/gitness/store/database"
|
|
"github.com/harness/gitness/store/database/dbtx"
|
|
|
|
"github.com/distribution/distribution/v3"
|
|
"github.com/distribution/distribution/v3/registry/api/errcode"
|
|
"github.com/opencontainers/go-digest"
|
|
v1 "github.com/opencontainers/image-spec/specs-go/v1"
|
|
"github.com/rs/zerolog/log"
|
|
)
|
|
|
|
type manifestService struct {
|
|
registryDao store.RegistryRepository
|
|
manifestDao store.ManifestRepository
|
|
layerDao store.LayerRepository
|
|
blobRepo store.BlobRepository
|
|
mtRepository store.MediaTypesRepository
|
|
tagDao store.TagRepository
|
|
imageDao store.ImageRepository
|
|
artifactDao store.ArtifactRepository
|
|
manifestRefDao store.ManifestReferenceRepository
|
|
ociImageIndexMappingDao store.OCIImageIndexMappingRepository
|
|
spacePathStore gas.SpacePathStore
|
|
gcService gc.Service
|
|
tx dbtx.Transactor
|
|
reporter event.Reporter
|
|
}
|
|
|
|
func NewManifestService(
|
|
registryDao store.RegistryRepository, manifestDao store.ManifestRepository,
|
|
blobRepo store.BlobRepository, mtRepository store.MediaTypesRepository, tagDao store.TagRepository,
|
|
imageDao store.ImageRepository, artifactDao store.ArtifactRepository,
|
|
layerDao store.LayerRepository, manifestRefDao store.ManifestReferenceRepository,
|
|
tx dbtx.Transactor, gcService gc.Service, reporter event.Reporter, spacePathStore gas.SpacePathStore,
|
|
ociImageIndexMappingDao store.OCIImageIndexMappingRepository,
|
|
) ManifestService {
|
|
return &manifestService{
|
|
registryDao: registryDao,
|
|
manifestDao: manifestDao,
|
|
layerDao: layerDao,
|
|
blobRepo: blobRepo,
|
|
mtRepository: mtRepository,
|
|
tagDao: tagDao,
|
|
artifactDao: artifactDao,
|
|
imageDao: imageDao,
|
|
manifestRefDao: manifestRefDao,
|
|
gcService: gcService,
|
|
tx: tx,
|
|
reporter: reporter,
|
|
spacePathStore: spacePathStore,
|
|
ociImageIndexMappingDao: ociImageIndexMappingDao,
|
|
}
|
|
}
|
|
|
|
type ManifestService interface {
|
|
// GetTags gets the tags of a repository
|
|
DBTag(
|
|
ctx context.Context,
|
|
mfst manifest.Manifest,
|
|
d digest.Digest,
|
|
tag string,
|
|
repoKey string,
|
|
headers *commons.ResponseHeaders,
|
|
info pkg.RegistryInfo,
|
|
) error
|
|
DBPut(
|
|
ctx context.Context,
|
|
mfst manifest.Manifest,
|
|
d digest.Digest,
|
|
repoKey string,
|
|
headers *commons.ResponseHeaders,
|
|
info pkg.RegistryInfo,
|
|
) error
|
|
DeleteTag(ctx context.Context, repoKey string, tag string, info pkg.RegistryInfo) (bool, error)
|
|
DeleteManifest(ctx context.Context, repoKey string, d digest.Digest, info pkg.RegistryInfo) error
|
|
AddManifestAssociation(ctx context.Context, repoKey string, digest digest.Digest, info pkg.RegistryInfo) error
|
|
DBFindRepositoryBlob(
|
|
ctx context.Context, desc manifest.Descriptor, repoID int64,
|
|
info pkg.RegistryInfo,
|
|
) (*types.Blob, error)
|
|
}
|
|
|
|
func (l *manifestService) DBTag(
|
|
ctx context.Context,
|
|
mfst manifest.Manifest,
|
|
d digest.Digest,
|
|
tag string,
|
|
repoKey string,
|
|
headers *commons.ResponseHeaders,
|
|
info pkg.RegistryInfo,
|
|
) error {
|
|
imageName := info.Image
|
|
|
|
if err := l.dbTagManifest(ctx, d, tag, imageName, info); err != nil {
|
|
log.Ctx(ctx).Error().Err(err).Msg("failed to create tag in database")
|
|
err2 := l.handleTagError(ctx, mfst, d, tag, repoKey, headers, info, err, imageName)
|
|
if err2 != nil {
|
|
return err2
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (l *manifestService) handleTagError(
|
|
ctx context.Context,
|
|
mfst manifest.Manifest,
|
|
d digest.Digest,
|
|
tag string,
|
|
repoKey string,
|
|
headers *commons.ResponseHeaders,
|
|
info pkg.RegistryInfo,
|
|
err error,
|
|
imageName string,
|
|
) error {
|
|
if errors.Is(err, util.ErrManifestNotFound) {
|
|
// If online GC was already reviewing the manifest that we want to tag, and that manifest had no
|
|
// tags before the review start, the API is unable to stop the GC from deleting the manifest (as
|
|
// the GC already acquired the lock on the corresponding queue row). This means that once the API
|
|
// is unblocked and tries to create the tag, a foreign key violation error will occur (because we're
|
|
// trying to create a tag for a manifest that no longer exists) and lead to this specific error.
|
|
// This should be extremely rare, if it ever occurs, but if it does, we should recreate the manifest
|
|
// and tag it, instead of returning a "manifest not found response" to clients. It's expected that
|
|
// this route handles the creation of a manifest if it doesn't exist already.
|
|
if err = l.DBPut(ctx, mfst, "", repoKey, headers, info); err != nil {
|
|
return fmt.Errorf("failed to recreate manifest in database: %w", err)
|
|
}
|
|
if err = l.dbTagManifest(ctx, d, tag, imageName, info); err != nil {
|
|
return fmt.Errorf("failed to create tag in database after manifest recreate: %w", err)
|
|
}
|
|
} else {
|
|
return fmt.Errorf("failed to create tag in database: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (l *manifestService) dbTagManifest(
|
|
ctx context.Context,
|
|
dgst digest.Digest,
|
|
tagName, imageName string,
|
|
info pkg.RegistryInfo,
|
|
) error {
|
|
dbRegistry, err := l.registryDao.GetByParentIDAndName(ctx, info.ParentID, info.RegIdentifier)
|
|
if err != nil {
|
|
return formatFailedToTagErr(err)
|
|
}
|
|
newDigest, err := types.NewDigest(dgst)
|
|
if err != nil {
|
|
return formatFailedToTagErr(err)
|
|
}
|
|
dbManifest, err := l.manifestDao.FindManifestByDigest(ctx, dbRegistry.ID, info.Image, newDigest)
|
|
if errors.Is(err, gitnessstore.ErrResourceNotFound) {
|
|
return fmt.Errorf("manifest %s not found in database", dgst)
|
|
}
|
|
if err != nil {
|
|
return formatFailedToTagErr(err)
|
|
}
|
|
|
|
err = l.tx.WithTx(ctx, func(ctx context.Context) error {
|
|
// Prevent long running transactions by setting an upper limit of manifestTagGCLockTimeout. If the GC is holding
|
|
// the lock of a related review record, the processing there should be fast enough to avoid this. Regardless, we
|
|
// should not let transactions open (and clients waiting) for too long. If this sensible timeout is exceeded, abort
|
|
// the tag creation and let the client retry. This will bubble up and lead to a 503 Service Unavailable response.
|
|
// Set timeout for the transaction to prevent long-running operations
|
|
ctx, cancel := context.WithTimeout(ctx, manifestTagGCLockTimeout)
|
|
defer cancel()
|
|
|
|
// Attempt to find and lock the manifest for GC review
|
|
if err := l.lockManifestForGC(ctx, dbRegistry.ID, dbManifest.ID); err != nil {
|
|
return formatFailedToTagErr(err)
|
|
}
|
|
|
|
// Create or update artifact and tag records
|
|
if err := l.upsertArtifactAndTag(ctx, dbRegistry.ID, dbManifest.ID, imageName, tagName,
|
|
dgst); err != nil {
|
|
return formatFailedToTagErr(err)
|
|
}
|
|
|
|
return nil
|
|
})
|
|
|
|
if err != nil {
|
|
return formatFailedToTagErr(err)
|
|
}
|
|
spacePath, packageType, err := l.getSpacePathAndPackageType(ctx, dbRegistry)
|
|
if err == nil {
|
|
l.reportEventAsync(ctx, info.RegIdentifier, imageName, tagName, packageType, spacePath)
|
|
} else {
|
|
log.Ctx(ctx).Err(err).Msg("Failed to find spacePath, not publishing event")
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func formatFailedToTagErr(err error) error {
|
|
return fmt.Errorf("failed to tag manifest: %w", err)
|
|
}
|
|
|
|
// Locks the manifest for GC review.
|
|
func (l *manifestService) lockManifestForGC(ctx context.Context, repoID, manifestID int64) error {
|
|
_, err := l.gcService.ManifestFindAndLockBefore(
|
|
ctx, repoID, manifestID,
|
|
time.Now().Add(manifestTagGCReviewWindow),
|
|
)
|
|
if err != nil && !errors.Is(err, sql.ErrNoRows) {
|
|
// Use ProcessSQLErrorf for handling the SQL error abstraction
|
|
return db.ProcessSQLErrorf(
|
|
ctx,
|
|
err,
|
|
"failed to lock manifest for GC review [repoID: %d, manifestID: %d]", repoID, manifestID,
|
|
)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Creates or updates artifact and tag records.
|
|
func (l *manifestService) upsertArtifactAndTag(
|
|
ctx context.Context,
|
|
registryID,
|
|
manifestID int64,
|
|
imageName,
|
|
tagName string,
|
|
dgst digest.Digest,
|
|
) error {
|
|
image := &types.Image{
|
|
Name: imageName,
|
|
RegistryID: registryID,
|
|
Enabled: true,
|
|
}
|
|
|
|
if err := l.imageDao.CreateOrUpdate(ctx, image); err != nil {
|
|
return err
|
|
}
|
|
|
|
digest, err := types.NewDigest(dgst)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
artifact := &types.Artifact{
|
|
ImageID: image.ID,
|
|
Version: digest.String(),
|
|
}
|
|
|
|
if err := l.artifactDao.CreateOrUpdate(ctx, artifact); err != nil {
|
|
return err
|
|
}
|
|
|
|
tag := &types.Tag{
|
|
Name: tagName,
|
|
ImageName: imageName,
|
|
RegistryID: registryID,
|
|
ManifestID: manifestID,
|
|
}
|
|
|
|
return l.tagDao.CreateOrUpdate(ctx, tag)
|
|
}
|
|
|
|
// Retrieves the spacePath and packageType.
|
|
func (l *manifestService) getSpacePathAndPackageType(
|
|
ctx context.Context,
|
|
dbRepo *types.Registry,
|
|
) (string, event.PackageType, error) {
|
|
spacePath, err := l.spacePathStore.FindPrimaryBySpaceID(ctx, dbRepo.ParentID)
|
|
if err != nil {
|
|
log.Ctx(ctx).Err(err).Msg("Failed to find spacePath")
|
|
return "", event.PackageType(0), err
|
|
}
|
|
|
|
packageType, err := event.GetPackageTypeFromString(string(dbRepo.PackageType))
|
|
if err != nil {
|
|
log.Ctx(ctx).Err(err).Msg("Failed to find packageType")
|
|
return "", event.PackageType(0), err
|
|
}
|
|
|
|
return spacePath.Value, packageType, nil
|
|
}
|
|
|
|
// Reports event asynchronously.
|
|
func (l *manifestService) reportEventAsync(
|
|
ctx context.Context,
|
|
regID,
|
|
imageName,
|
|
tagName string,
|
|
packageType event.PackageType,
|
|
spacePath string,
|
|
) {
|
|
go l.reporter.ReportEvent(ctx, &event.ArtifactDetails{
|
|
RegistryID: regID,
|
|
ImagePath: imageName + ":" + tagName,
|
|
PackageType: packageType,
|
|
}, spacePath)
|
|
}
|
|
|
|
func (l *manifestService) DBPut(
|
|
ctx context.Context,
|
|
mfst manifest.Manifest,
|
|
d digest.Digest,
|
|
repoKey string,
|
|
headers *commons.ResponseHeaders,
|
|
info pkg.RegistryInfo,
|
|
) error {
|
|
_, payload, err := mfst.Payload()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
err = l.dbPutManifest(ctx, mfst, payload, d, repoKey, headers, info)
|
|
var mtErr util.UnknownMediaTypeError
|
|
if errors.As(err, &mtErr) {
|
|
return errcode.ErrorCodeManifestInvalid.WithDetail(mtErr.Error())
|
|
}
|
|
return err
|
|
}
|
|
|
|
func (l *manifestService) dbPutManifest(
|
|
ctx context.Context,
|
|
manifest manifest.Manifest,
|
|
payload []byte,
|
|
d digest.Digest,
|
|
repoKey string,
|
|
headers *commons.ResponseHeaders,
|
|
info pkg.RegistryInfo,
|
|
) error {
|
|
switch reqManifest := manifest.(type) {
|
|
case *schema2.DeserializedManifest:
|
|
return l.dbPutManifestSchema2(ctx, reqManifest, payload, d, repoKey, headers, info)
|
|
case *ocischema.DeserializedManifest:
|
|
return l.dbPutManifestOCI(ctx, reqManifest, payload, d, repoKey, headers, info)
|
|
case *manifestlist.DeserializedManifestList:
|
|
return l.dbPutManifestList(ctx, reqManifest, payload, d, repoKey, headers, info)
|
|
case *ocischema.DeserializedImageIndex:
|
|
return l.dbPutImageIndex(ctx, reqManifest, payload, d, repoKey, headers, info)
|
|
default:
|
|
return errcode.ErrorCodeManifestInvalid.WithDetail("manifest type unsupported")
|
|
}
|
|
}
|
|
|
|
func (l *manifestService) dbPutManifestSchema2(
|
|
ctx context.Context,
|
|
manifest *schema2.DeserializedManifest,
|
|
payload []byte,
|
|
d digest.Digest,
|
|
repoKey string,
|
|
headers *commons.ResponseHeaders,
|
|
info pkg.RegistryInfo,
|
|
) error {
|
|
return l.dbPutManifestV2(ctx, manifest, payload, false, d, repoKey, headers, info)
|
|
}
|
|
|
|
func (l *manifestService) dbPutManifestV2(
|
|
ctx context.Context,
|
|
mfst manifest.ManifestV2,
|
|
payload []byte,
|
|
nonConformant bool,
|
|
digest digest.Digest,
|
|
repoKey string,
|
|
headers *commons.ResponseHeaders,
|
|
info pkg.RegistryInfo,
|
|
) error {
|
|
// find target repository
|
|
dbRepo, err := l.registryDao.GetByParentIDAndName(ctx, info.ParentID, repoKey)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if dbRepo == nil {
|
|
return errors.New("repository not found in database")
|
|
}
|
|
|
|
// Find the config now to ensure that the config's blob is associated with the repository.
|
|
dbCfgBlob, err := l.DBFindRepositoryBlob(ctx, mfst.Config(), dbRepo.ID, info)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
dgst, err := types.NewDigest(digest)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
dbManifest, err := l.manifestDao.FindManifestByDigest(ctx, dbRepo.ID, info.Image, dgst)
|
|
if err != nil && !errors.Is(err, gitnessstore.ErrResourceNotFound) {
|
|
return err
|
|
}
|
|
|
|
if dbManifest != nil {
|
|
return nil
|
|
}
|
|
|
|
log.Debug().Msgf("manifest %s not found in database", dgst.String())
|
|
|
|
cfg := &types.Configuration{
|
|
MediaType: mfst.Config().MediaType,
|
|
Digest: dbCfgBlob.Digest,
|
|
BlobID: dbCfgBlob.ID,
|
|
}
|
|
|
|
//TODO: check if we need to store the config payload in the database
|
|
|
|
// skip retrieval and caching of config payload if its size is over the limit
|
|
/*if dbCfgBlob.Size <= datastore.ConfigSizeLimit {
|
|
// Since filesystem writes may be optional, We cannot be sure that the
|
|
// repository scoped filesystem blob service will have a link to the
|
|
// configuration blob; however, since we check for repository scoped access
|
|
// via the database above, we may retrieve the blob directly common storage.
|
|
cfgPayload, err := imh.blobProvider.Get(imh, dbCfgBlob.Digest)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
cfg.Payload = cfgPayload
|
|
}*/
|
|
|
|
m := &types.Manifest{
|
|
RegistryID: dbRepo.ID,
|
|
TotalSize: mfst.TotalSize(),
|
|
SchemaVersion: mfst.Version().SchemaVersion,
|
|
MediaType: mfst.Version().MediaType,
|
|
Digest: digest,
|
|
Payload: payload,
|
|
Configuration: cfg,
|
|
NonConformant: nonConformant,
|
|
ImageName: info.Image,
|
|
}
|
|
|
|
var artifactMediaType sql.NullString
|
|
ocim, ok := mfst.(manifest.ManifestOCI)
|
|
if ok {
|
|
subjectHandlingError := l.handleSubject(
|
|
ctx, ocim.Subject(), ocim.ArtifactType(),
|
|
ocim.Annotations(), dbRepo, m, headers, info,
|
|
)
|
|
if subjectHandlingError != nil {
|
|
return subjectHandlingError
|
|
}
|
|
if ocim.ArtifactType() != "" {
|
|
artifactMediaType.Valid = true
|
|
artifactMediaType.String = ocim.ArtifactType()
|
|
m.ArtifactType = artifactMediaType
|
|
}
|
|
} else if mfst.Config().MediaType != "" {
|
|
artifactMediaType.Valid = true
|
|
artifactMediaType.String = mfst.Config().MediaType
|
|
m.ArtifactType = artifactMediaType
|
|
}
|
|
|
|
// check if the manifest references non-distributable layers and mark it as such on the DB
|
|
ll := mfst.DistributableLayers()
|
|
m.NonDistributableLayers = len(ll) < len(mfst.Layers())
|
|
|
|
// Use CreateOrFind to prevent race conditions while pushing the same manifest with digest for different tags
|
|
if err := l.manifestDao.CreateOrFind(ctx, m); err != nil {
|
|
return err
|
|
}
|
|
|
|
dbManifest = m
|
|
|
|
// find and associate distributable manifest layer blobs
|
|
for _, reqLayer := range mfst.DistributableLayers() {
|
|
dbBlob, err := l.DBFindRepositoryBlob(ctx, reqLayer, dbRepo.ID, info)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// Overwrite the media type from common blob storage with the one
|
|
// specified in the manifest json for the layer entity. The layer entity
|
|
// has a 1-1 relationship with with the manifest, so we want to reflect
|
|
// the manifest's description of the layer. Multiple manifest can reference
|
|
// the same blob, so the common blob storage should remain generic.
|
|
if ok2 := l.layerMediaTypeExists(ctx, reqLayer.MediaType); ok2 {
|
|
dbBlob.MediaType = reqLayer.MediaType
|
|
}
|
|
|
|
if err2 := l.layerDao.AssociateLayerBlob(ctx, dbManifest, dbBlob); err2 != nil {
|
|
return err2
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func (l *manifestService) DBFindRepositoryBlob(
|
|
ctx context.Context, desc manifest.Descriptor,
|
|
repoID int64, info pkg.RegistryInfo,
|
|
) (*types.Blob, error) {
|
|
image := info.Image
|
|
b, err := l.blobRepo.FindByDigestAndRepoID(ctx, desc.Digest, repoID, image)
|
|
if err != nil {
|
|
if errors.Is(err, gitnessstore.ErrResourceNotFound) {
|
|
return nil, fmt.Errorf("blob not found in database")
|
|
}
|
|
return nil, err
|
|
}
|
|
return b, nil
|
|
}
|
|
|
|
// AddManifestAssociation This updates the manifestRefs for all new childDigests to their already existing parent
|
|
// manifests. This is used when a manifest from a manifest list is pulled from the remote and manifest list already
|
|
// exists in the database.
|
|
func (l *manifestService) AddManifestAssociation(
|
|
ctx context.Context, repoKey string, childDigest digest.Digest, info pkg.RegistryInfo,
|
|
) error {
|
|
newDigest, err2 := types.NewDigest(childDigest)
|
|
if err2 != nil {
|
|
return fmt.Errorf("failed to create digest: %s %w", childDigest, err2)
|
|
}
|
|
r, err := l.registryDao.GetByParentIDAndName(ctx, info.ParentID, repoKey)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to get registry: %s %w", repoKey, err)
|
|
}
|
|
childManifest, err2 := l.manifestDao.FindManifestByDigest(ctx, r.ID, info.Image, newDigest)
|
|
if err2 != nil {
|
|
return fmt.Errorf("failed to find manifest by digest. Repo: %d Image: %s %w", r.ID, info.Image, err2)
|
|
}
|
|
mappings, err := l.ociImageIndexMappingDao.GetAllByChildDigest(ctx, r.ID, childManifest.ImageName, newDigest)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to get oci image index mappings. Repo: %d Image: %s %w",
|
|
r.ID,
|
|
childManifest.ImageName,
|
|
err)
|
|
}
|
|
for _, mapping := range mappings {
|
|
parentManifest, err := l.manifestDao.Get(ctx, mapping.ParentManifestID)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to get manifest with ID: %d %w", mapping.ParentManifestID, err)
|
|
}
|
|
if err := l.manifestRefDao.AssociateManifest(ctx, parentManifest, childManifest); err != nil {
|
|
if errors.Is(err, util.ErrRefManifestNotFound) {
|
|
// This can only happen if the online GC deleted one
|
|
// of the referenced manifests (because they were
|
|
// untagged/unreferenced) between the call to
|
|
// `FindAndLockNBefore` and `AssociateManifest`. For now
|
|
// we need to return this error to mimic the behaviour
|
|
// of the corresponding filesystem validation.
|
|
log.Error().
|
|
Msgf("Failed to associate manifest Ref Manifest not found. parentDigest:%s childDigest:%s %v",
|
|
parentManifest.Digest.String(),
|
|
childManifest.Digest.String(),
|
|
err)
|
|
return err
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (l *manifestService) handleSubject(
|
|
ctx context.Context, subject manifest.Descriptor,
|
|
artifactType string, annotations map[string]string, dbRepo *types.Registry,
|
|
m *types.Manifest, headers *commons.ResponseHeaders, info pkg.RegistryInfo,
|
|
) error {
|
|
if subject.Digest.String() != "" {
|
|
// Fetch subject_id from digest
|
|
subjectDigest, err := types.NewDigest(subject.Digest)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
dbSubject, err := l.manifestDao.FindManifestByDigest(ctx, dbRepo.ID, info.Image, subjectDigest)
|
|
if err != nil && !errors.Is(err, gitnessstore.ErrResourceNotFound) {
|
|
return err
|
|
}
|
|
|
|
if errors.Is(err, gitnessstore.ErrResourceNotFound) {
|
|
// in case something happened to the referenced manifest after validation
|
|
// return distribution.ManifestBlobUnknownError{Digest: subject.Digest}
|
|
log.Ctx(ctx).Warn().Msgf("subject manifest not found in database")
|
|
} else {
|
|
m.SubjectID.Int64 = dbSubject.ID
|
|
m.SubjectID.Valid = true
|
|
}
|
|
m.SubjectDigest = subject.Digest
|
|
headers.Headers["OCI-Subject"] = subject.Digest.String()
|
|
}
|
|
|
|
if artifactType != "" {
|
|
m.ArtifactType.String = artifactType
|
|
m.ArtifactType.Valid = true
|
|
}
|
|
m.Annotations = annotations
|
|
return nil
|
|
}
|
|
|
|
func (l *manifestService) dbPutManifestOCI(
|
|
ctx context.Context,
|
|
manifest *ocischema.DeserializedManifest,
|
|
payload []byte,
|
|
d digest.Digest,
|
|
repoKey string,
|
|
headers *commons.ResponseHeaders,
|
|
info pkg.RegistryInfo,
|
|
) error {
|
|
return l.dbPutManifestV2(ctx, manifest, payload, false, d, repoKey, headers, info)
|
|
}
|
|
|
|
func (l *manifestService) dbPutManifestList(
|
|
ctx context.Context,
|
|
manifestList *manifestlist.DeserializedManifestList,
|
|
payload []byte,
|
|
digest digest.Digest,
|
|
repoKey string,
|
|
headers *commons.ResponseHeaders,
|
|
info pkg.RegistryInfo,
|
|
) error {
|
|
if LikelyBuildxCache(manifestList) {
|
|
return l.dbPutBuildkitIndex(ctx, manifestList, payload, digest, repoKey, headers, info)
|
|
}
|
|
|
|
r, err := l.registryDao.GetByParentIDAndName(ctx, info.ParentID, repoKey)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if r == nil {
|
|
return errors.New("repository not found in database")
|
|
}
|
|
|
|
dgst, err := types.NewDigest(digest)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
ml, err := l.manifestDao.FindManifestByDigest(ctx, r.ID, info.Image, dgst)
|
|
if err != nil && !errors.Is(err, gitnessstore.ErrResourceNotFound) {
|
|
return err
|
|
}
|
|
|
|
// Media type can be either Docker (`application/vnd.docker.distribution.manifest.list.v2+json`)
|
|
// or OCI (empty).
|
|
// We need to make it explicit if empty, otherwise we're not able to distinguish between media types.
|
|
mediaType := manifestList.MediaType
|
|
if mediaType == "" {
|
|
mediaType = v1.MediaTypeImageIndex
|
|
}
|
|
|
|
ml = &types.Manifest{
|
|
RegistryID: r.ID,
|
|
SchemaVersion: manifestList.SchemaVersion,
|
|
MediaType: mediaType,
|
|
Digest: digest,
|
|
Payload: payload,
|
|
ImageName: info.Image,
|
|
}
|
|
|
|
mm, ids, err2 := l.validateManifestList(ctx, manifestList, r, info)
|
|
if err2 != nil {
|
|
return err2
|
|
}
|
|
|
|
err = l.tx.WithTx(
|
|
ctx, func(ctx context.Context) error {
|
|
// Prevent long running transactions by setting an upper limit of
|
|
// manifestListCreateGCLockTimeout. If the GC is
|
|
// holding the lock of a related review record, the processing
|
|
// there should be fast enough to avoid this.
|
|
// Regardless, we should not let transactions open (and clients waiting)
|
|
// for too long. If this sensible timeout
|
|
// is exceeded, abort the request and let the client retry.
|
|
// This will bubble up and lead to a 503 Service
|
|
// Unavailable response.
|
|
ctx, cancel := context.WithTimeout(ctx, manifestListCreateGCLockTimeout)
|
|
defer cancel()
|
|
|
|
if _, err := l.gcService.ManifestFindAndLockNBefore(
|
|
ctx, r.ID, ids,
|
|
time.Now().Add(manifestListCreateGCReviewWindow),
|
|
); err != nil {
|
|
return err
|
|
}
|
|
|
|
// use CreateOrFind to prevent race conditions when the same digest is used by different tags
|
|
// and pushed at the same time
|
|
if err := l.manifestDao.CreateOrFind(ctx, ml); err != nil {
|
|
return err
|
|
}
|
|
|
|
// Associate manifests to the manifest list.
|
|
for _, m := range mm {
|
|
if err := l.manifestRefDao.AssociateManifest(ctx, ml, m); err != nil {
|
|
if errors.Is(err, util.ErrRefManifestNotFound) {
|
|
// This can only happen if the online GC deleted one
|
|
// of the referenced manifests (because they were
|
|
// untagged/unreferenced) between the call to
|
|
// `FindAndLockNBefore` and `AssociateManifest`. For now
|
|
// we need to return this error to mimic the behaviour
|
|
// of the corresponding filesystem validation.
|
|
return distribution.ErrManifestVerification{
|
|
distribution.ErrManifestBlobUnknown{Digest: m.Digest},
|
|
}
|
|
}
|
|
return err
|
|
}
|
|
}
|
|
|
|
err = l.mapManifestList(ctx, ml.ID, manifestList, r)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to map manifest list: %w", err)
|
|
}
|
|
|
|
return nil
|
|
},
|
|
)
|
|
|
|
if err != nil {
|
|
log.Ctx(ctx).Error().Err(err).Msgf("failed to create manifest list in database: %v", err)
|
|
return fmt.Errorf("failed to create manifest list in database: %w", err)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func (l *manifestService) validateManifestIndex(
|
|
ctx context.Context, manifestList *ocischema.DeserializedImageIndex, r *types.Registry, info pkg.RegistryInfo,
|
|
) ([]*types.Manifest, []int64, error) {
|
|
mm := make([]*types.Manifest, 0, len(manifestList.Manifests))
|
|
ids := make([]int64, 0, len(manifestList.Manifests))
|
|
for _, desc := range manifestList.Manifests {
|
|
m, err := l.dbFindManifestListManifest(ctx, r, info.Image, desc.Digest)
|
|
if errors.Is(err, gitnessstore.ErrResourceNotFound) && r.Type == artifact.RegistryTypeUPSTREAM {
|
|
continue
|
|
}
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
mm = append(mm, m)
|
|
ids = append(ids, m.ID)
|
|
}
|
|
log.Ctx(ctx).Debug().Msgf("validated %d / %d manifests in index", len(mm), len(manifestList.Manifests))
|
|
return mm, ids, nil
|
|
}
|
|
|
|
func (l *manifestService) mapManifestIndex(
|
|
ctx context.Context, mi int64, manifestList *ocischema.DeserializedImageIndex, r *types.Registry,
|
|
) error {
|
|
if r.Type != artifact.RegistryTypeUPSTREAM {
|
|
return nil
|
|
}
|
|
for _, desc := range manifestList.Manifests {
|
|
err := l.ociImageIndexMappingDao.Create(ctx, &types.OCIImageIndexMapping{
|
|
ParentManifestID: mi,
|
|
ChildManifestDigest: desc.Digest,
|
|
})
|
|
if err != nil {
|
|
log.Ctx(ctx).Error().Err(err).
|
|
Msgf("failed to create oci image index manifest for digest %s", desc.Digest)
|
|
return fmt.Errorf("failed to create oci image index manifest: %w", err)
|
|
}
|
|
}
|
|
log.Ctx(ctx).Debug().Msgf("successfully mapped manifest index %d with its manifests", mi)
|
|
return nil
|
|
}
|
|
|
|
func (l *manifestService) validateManifestList(
|
|
ctx context.Context, manifestList *manifestlist.DeserializedManifestList, r *types.Registry, info pkg.RegistryInfo,
|
|
) ([]*types.Manifest, []int64, error) {
|
|
mm := make([]*types.Manifest, 0, len(manifestList.Manifests))
|
|
ids := make([]int64, 0, len(manifestList.Manifests))
|
|
for _, desc := range manifestList.Manifests {
|
|
m, err := l.dbFindManifestListManifest(ctx, r, info.Image, desc.Digest)
|
|
if errors.Is(err, gitnessstore.ErrResourceNotFound) && r.Type == artifact.RegistryTypeUPSTREAM {
|
|
continue
|
|
}
|
|
if err != nil {
|
|
return nil, nil, err
|
|
}
|
|
mm = append(mm, m)
|
|
ids = append(ids, m.ID)
|
|
}
|
|
log.Ctx(ctx).Debug().Msgf("validated %d / %d manifests in list", len(mm), len(manifestList.Manifests))
|
|
return mm, ids, nil
|
|
}
|
|
|
|
func (l *manifestService) mapManifestList(
|
|
ctx context.Context, mi int64, manifestList *manifestlist.DeserializedManifestList, r *types.Registry,
|
|
) error {
|
|
if r.Type != artifact.RegistryTypeUPSTREAM {
|
|
return nil
|
|
}
|
|
for _, desc := range manifestList.Manifests {
|
|
err := l.ociImageIndexMappingDao.Create(ctx, &types.OCIImageIndexMapping{
|
|
ParentManifestID: mi,
|
|
ChildManifestDigest: desc.Digest,
|
|
})
|
|
if err != nil {
|
|
log.Ctx(ctx).Error().Err(err).Msgf("failed to create oci image index manifest for digest %s", desc.Digest)
|
|
return fmt.Errorf("failed to create oci image index manifest: %w", err)
|
|
}
|
|
}
|
|
log.Ctx(ctx).Debug().Msgf("successfully mapped manifest list %d with its manifests", mi)
|
|
return nil
|
|
}
|
|
|
|
func (l *manifestService) dbPutImageIndex(
|
|
ctx context.Context,
|
|
imageIndex *ocischema.DeserializedImageIndex,
|
|
payload []byte,
|
|
digest digest.Digest,
|
|
repoKey string,
|
|
headers *commons.ResponseHeaders,
|
|
info pkg.RegistryInfo,
|
|
) error {
|
|
r, err := l.registryDao.GetByParentIDAndName(ctx, info.ParentID, repoKey)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if r == nil {
|
|
return errors.New("repository not found in database")
|
|
}
|
|
|
|
dgst, err := types.NewDigest(digest)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
mi, err := l.manifestDao.FindManifestByDigest(ctx, r.ID, info.Image, dgst)
|
|
if err != nil && !errors.Is(err, gitnessstore.ErrResourceNotFound) {
|
|
return err
|
|
}
|
|
|
|
// Media type can be either Docker (`application/vnd.docker.distribution.manifest.list.v2+json`)
|
|
// or OCI (empty).
|
|
// We need to make it explicit if empty, otherwise we're not able to distinguish
|
|
// between media types.
|
|
mediaType := imageIndex.MediaType
|
|
if mediaType == "" {
|
|
mediaType = v1.MediaTypeImageIndex
|
|
}
|
|
|
|
mi = &types.Manifest{
|
|
RegistryID: r.ID,
|
|
SchemaVersion: imageIndex.SchemaVersion,
|
|
MediaType: mediaType,
|
|
Digest: digest,
|
|
Payload: payload,
|
|
ImageName: info.Image,
|
|
}
|
|
|
|
subjectHandlingError := l.handleSubject(
|
|
ctx, imageIndex.Subject(), imageIndex.ArtifactType(),
|
|
imageIndex.Annotations(), r, mi, headers, info,
|
|
)
|
|
if subjectHandlingError != nil {
|
|
return subjectHandlingError
|
|
}
|
|
|
|
mm, ids, err := l.validateManifestIndex(ctx, imageIndex, r, info)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to map manifest index: %w", err)
|
|
}
|
|
|
|
err = l.tx.WithTx(
|
|
ctx, func(ctx context.Context) error {
|
|
// Prevent long running transactions by setting an upper limit of
|
|
// manifestListCreateGCLockTimeout. If the GC is
|
|
// holding the lock of a related review record, the processing
|
|
// there should be fast enough to avoid this.
|
|
// Regardless, we should not let transactions open (and clients waiting)
|
|
// for too long. If this sensible timeout
|
|
// is exceeded, abort the request and let the client retry.
|
|
// This will bubble up and lead to a 503 Service
|
|
// Unavailable response.
|
|
ctx, cancel := context.WithTimeout(ctx, manifestListCreateGCLockTimeout)
|
|
defer cancel()
|
|
|
|
if _, err := l.gcService.ManifestFindAndLockNBefore(
|
|
ctx, r.ID, ids,
|
|
time.Now().Add(manifestListCreateGCReviewWindow),
|
|
); err != nil {
|
|
return err
|
|
}
|
|
// use CreateOrFind to prevent race conditions when the same digest is used by different tags
|
|
// and pushed at the same time
|
|
if err := l.manifestDao.CreateOrFind(ctx, mi); err != nil {
|
|
return err
|
|
}
|
|
|
|
// Associate manifests to the manifest list.
|
|
for _, m := range mm {
|
|
if err := l.manifestRefDao.AssociateManifest(ctx, mi, m); err != nil {
|
|
if errors.Is(err, util.ErrRefManifestNotFound) {
|
|
// This can only happen if the online GC deleted one of the
|
|
// referenced manifests (because they were
|
|
// untagged/unreferenced) between the call to
|
|
// `FindAndLockNBefore` and `AssociateManifest`. For now
|
|
// we need to return this error to mimic the behaviour
|
|
// of the corresponding filesystem validation.
|
|
return distribution.ErrManifestVerification{
|
|
distribution.ErrManifestBlobUnknown{Digest: m.Digest},
|
|
}
|
|
}
|
|
return err
|
|
}
|
|
}
|
|
|
|
err = l.mapManifestIndex(ctx, mi.ID, imageIndex, r)
|
|
if err != nil {
|
|
return fmt.Errorf("failed to map manifest index: %w", err)
|
|
}
|
|
return nil
|
|
},
|
|
)
|
|
|
|
if err != nil {
|
|
log.Ctx(ctx).Error().Err(err).Msgf("failed to create image index in database")
|
|
}
|
|
return err
|
|
}
|
|
|
|
func (l *manifestService) dbPutBuildkitIndex(
|
|
ctx context.Context,
|
|
ml *manifestlist.DeserializedManifestList,
|
|
payload []byte,
|
|
digest digest.Digest,
|
|
repoKey string,
|
|
headers *commons.ResponseHeaders,
|
|
info pkg.RegistryInfo,
|
|
) error {
|
|
// convert to OCI manifest and process as if it was one
|
|
m, err := OCIManifestFromBuildkitIndex(ml)
|
|
if err != nil {
|
|
return fmt.Errorf("converting buildkit index to manifest: %w", err)
|
|
}
|
|
|
|
// Note that `payload` is not the deserialized manifest (`m`) payload but
|
|
// rather the index payload, untouched.
|
|
// Within dbPutManifestOCIOrSchema2 we use this value for the
|
|
// `manifests.payload` column and source the value for
|
|
// the `manifests.digest` column from `imh.Digest`, and not from `m`.
|
|
// Therefore, we keep behavioral consistency for
|
|
// the outside world by preserving the index payload and digest while
|
|
// storing things internally as an OCI manifest.
|
|
return l.dbPutManifestV2(ctx, m, payload, true, digest, repoKey, headers, info)
|
|
}
|
|
|
|
func (l *manifestService) dbFindManifestListManifest(
|
|
ctx context.Context,
|
|
repository *types.Registry, imageName string, digest digest.Digest,
|
|
) (*types.Manifest, error) {
|
|
dgst, err := types.NewDigest(digest)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
dbManifest, err := l.manifestDao.FindManifestByDigest(
|
|
ctx, repository.ID,
|
|
imageName, dgst,
|
|
)
|
|
if err != nil {
|
|
if errors.Is(err, gitnessstore.ErrResourceNotFound) {
|
|
return nil, fmt.Errorf(
|
|
"manifest %s not found for %s/%s: %w", digest.String(),
|
|
repository.Name, imageName, err,
|
|
)
|
|
}
|
|
return nil, err
|
|
}
|
|
|
|
return dbManifest, nil
|
|
}
|
|
|
|
func (l *manifestService) layerMediaTypeExists(ctx context.Context, mt string) bool {
|
|
exists, err := l.mtRepository.MediaTypeExists(ctx, mt)
|
|
if err != nil {
|
|
log.Ctx(ctx).Error().Stack().Err(err).Msgf("error checking for existence of media type: %v", err)
|
|
return false
|
|
}
|
|
|
|
if exists {
|
|
return true
|
|
}
|
|
|
|
log.Ctx(ctx).Warn().Msgf("unknown layer media type")
|
|
|
|
return false
|
|
}
|
|
|
|
func (l *manifestService) DeleteTag(
|
|
ctx context.Context,
|
|
repoKey string,
|
|
tag string,
|
|
info pkg.RegistryInfo,
|
|
) (bool, error) {
|
|
// Fetch the registry by parent ID and name
|
|
registry, err := l.registryDao.GetByParentIDAndName(ctx, info.ParentID, repoKey)
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
found, err := l.tagDao.DeleteTagByName(ctx, registry.ID, tag)
|
|
if err != nil {
|
|
return false, fmt.Errorf("failed to delete tag in database: %w", err)
|
|
}
|
|
if !found {
|
|
return false, distribution.ErrTagUnknown{Tag: tag}
|
|
}
|
|
|
|
return true, nil
|
|
}
|
|
|
|
func (l *manifestService) DeleteTagsByManifestID(
|
|
ctx context.Context,
|
|
repoKey string,
|
|
manifestID int64,
|
|
info pkg.RegistryInfo,
|
|
) (bool, error) {
|
|
registry, err := l.registryDao.GetByParentIDAndName(ctx, info.ParentID, repoKey)
|
|
|
|
if err != nil {
|
|
return false, err
|
|
}
|
|
|
|
return l.tagDao.DeleteTagByManifestID(ctx, registry.ID, manifestID)
|
|
}
|
|
|
|
func (l *manifestService) DeleteManifest(
|
|
ctx context.Context,
|
|
repoKey string,
|
|
d digest.Digest,
|
|
info pkg.RegistryInfo,
|
|
) error {
|
|
log.Ctx(ctx).Debug().Msg("deleting manifest from repository in database")
|
|
|
|
registry, err := l.registryDao.GetByParentIDAndName(ctx, info.ParentID, repoKey)
|
|
imageName := info.Image
|
|
|
|
if registry == nil || err != nil {
|
|
return fmt.Errorf("repository not found in database: %w", err)
|
|
}
|
|
|
|
// We need to find the manifest first and then lookup for any manifest
|
|
// it references (if it's a manifest list). This
|
|
// is needed to ensure we lock any related online GC tasks to prevent
|
|
// race conditions around the delete.
|
|
newDigest, err := types.NewDigest(d)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
m, err := l.manifestDao.FindManifestByDigest(ctx, registry.ID, imageName, newDigest)
|
|
if err != nil {
|
|
if errors.Is(err, gitnessstore.ErrResourceNotFound) {
|
|
return util.ErrManifestNotFound
|
|
}
|
|
return err
|
|
}
|
|
|
|
return l.tx.WithTx(
|
|
ctx, func(ctx context.Context) error {
|
|
switch m.MediaType {
|
|
case manifestlist.MediaTypeManifestList, v1.MediaTypeImageIndex:
|
|
mm, err := l.manifestDao.References(ctx, m)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
// This should never happen, as it's not possible to delete a
|
|
// child manifest if it's referenced by a list, which
|
|
// means that we'll always have at least one child manifest here.
|
|
// Nevertheless, log error if this ever happens.
|
|
if len(mm) == 0 {
|
|
log.Ctx(ctx).Error().Stack().Err(err).Msgf("stored manifest list has no references")
|
|
break
|
|
}
|
|
ids := make([]int64, 0, len(mm))
|
|
for _, m := range mm {
|
|
ids = append(ids, m.ID)
|
|
}
|
|
|
|
// Prevent long running transactions by setting an upper limit of
|
|
// manifestDeleteGCLockTimeout. If the GC is
|
|
// holding the lock of a related review record, the processing
|
|
// there should be fast enough to avoid this.
|
|
// Regardless, we should not let transactions open (and clients waiting)
|
|
// for too long. If this sensible timeout
|
|
// is exceeded, abort the manifest delete and let the client retry.
|
|
// This will bubble up and lead to a 503
|
|
// Service Unavailable response.
|
|
ctx, cancel := context.WithTimeout(ctx, manifestDeleteGCLockTimeout)
|
|
defer cancel()
|
|
|
|
if _, err := l.gcService.ManifestFindAndLockNBefore(
|
|
ctx, registry.ID,
|
|
ids, time.Now().Add(manifestDeleteGCReviewWindow),
|
|
); err != nil {
|
|
return err
|
|
}
|
|
}
|
|
|
|
found, err := l.manifestDao.DeleteManifest(ctx, registry.ID, imageName, d)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
if !found {
|
|
return util.ErrManifestNotFound
|
|
}
|
|
|
|
return nil
|
|
},
|
|
)
|
|
}
|