github/drone
5
0
Fork 0
mirror of https://github.com/harness/drone.git synced 2025-05-03 18:12:09 +08:00
Code Issues Packages Projects Releases Wiki Activity

feat: [AH-993]: Complete implementation of Upstream changes of Python Package (#3573)

Browse Source 
* [AH-993]: Review comments fixed
* [AH-993]: Merge commit
* [AH-993]: Updated upstream creation
* [AH-993]: Cleanup
* [AH-993]: Updated messages
* [AH-993]: Merge commit
* [AH-993]: Upstream flows support for Python Packages
* [AH-993]: Updated local file
* [AH-993]: Added support for local and created arch to support different package types
* Merge branch 'main' of https://git0.harness.io/l7B_kbSEQD2wjrM7PShm5w/PROD/Harness_Commons/gitness into AH-993-upstream-implementation
* [AH-993]: temp commit
* [AH-993]: Merge commit:
* [AH-993]: temp update
This commit is contained in:
Arvind Choudhary 2025-03-25 05:36:47 +00:00 committed by Harness
parent d84b5cc7e1
commit ced5ce2f65
50 changed files with 1424 additions and 632 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
cmd/gitness/wire_gen.go
View File

@ -519,7 +519,8 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro
packagesHandler := api2.NewPackageHandlerProvider(registryRepository, spaceStore, tokenStore, controller, authenticator, provider, authorizer)
localBase := base.LocalBaseProvider(registryRepository, fileManager, transactor, imageRepository, artifactRepository)
pythonLocalRegistry := python.LocalRegistryProvider(localBase, fileManager, upstreamProxyConfigRepository, transactor, registryRepository, imageRepository, artifactRepository, provider)
proxy := python.ProxyProvider(upstreamProxyConfigRepository, registryRepository, imageRepository, artifactRepository, fileManager, transactor, provider)
localRegistryHelper := python.LocalRegistryHelperProvider(pythonLocalRegistry, localBase)
proxy := python.ProxyProvider(upstreamProxyConfigRepository, registryRepository, imageRepository, artifactRepository, fileManager, transactor, provider, spaceFinder, secretService, localRegistryHelper)
pythonController := python2.ControllerProvider(upstreamProxyConfigRepository, registryRepository, imageRepository, artifactRepository, fileManager, transactor, provider, pythonLocalRegistry, proxy)
pythonHandler := api2.NewPythonHandlerProvider(pythonController, packagesHandler)
handler4 := router.PackageHandlerProvider(packagesHandler, mavenHandler, genericHandler, pythonHandler)

3
registry/app/api/controller/metadata/utils.go
View File

@ -180,7 +180,8 @@ func ValidateUpstream(config *a.RegistryConfig) error {
}
if !commons.IsEmpty(config.Type) && config.Type == a.RegistryTypeUPSTREAM &&
*upstreamConfig.Source != a.UpstreamConfigSourceDockerhub &&
*upstreamConfig.Source != a.UpstreamConfigSourceMavenCentral {
*upstreamConfig.Source != a.UpstreamConfigSourceMavenCentral &&
*upstreamConfig.Source != a.UpstreamConfigSourcePyPi {
if commons.IsEmpty(upstreamConfig.Url) {
return errors.New("URL is required for upstream repository")
}

5
registry/app/api/controller/pkg/python/controller.go
View File

@ -48,8 +48,9 @@ type controller struct {
imageDao store.ImageRepository
artifactDao store.ArtifactRepository
urlProvider urlprovider.Provider
local python.LocalRegistry
proxy python.Proxy
// TODO: Cleanup and initiate at other place
local python.LocalRegistry
proxy python.Proxy
}
// NewController creates a new Python controller.

4
registry/app/api/controller/pkg/python/download.go
View File

@ -40,10 +40,10 @@ func (c *controller) DownloadPackageFile(
nil, "", nil, nil,
}
}
headers, fileReader, redirectURL, errs := pythonRegistry.DownloadPackageFile(ctx, info)
headers, fileReader, readCloser, redirectURL, errs := pythonRegistry.DownloadPackageFile(ctx, info)
return &GetArtifactResponse{
errs, headers, redirectURL,
fileReader, nil,
fileReader, readCloser,
}
}

2
registry/app/api/controller/pkg/python/upload_package.go
View File

@ -47,7 +47,7 @@ func (c *controller) UploadPackageFile(
nil,
}
}
headers, sha256, err := pythonRegistry.UploadPackageFile(ctx, info, file, fileHeader)
headers, sha256, err := pythonRegistry.UploadPackageFile(ctx, info, file, fileHeader.Filename)
if commons.IsEmptyError(err) {
return &PutArtifactResponse{
sha256, []error{}, headers,

5
registry/app/api/handler/oci/get_manifest.go
View File

@ -20,6 +20,7 @@ import (
"github.com/harness/gitness/registry/app/dist_temp/errcode"
"github.com/harness/gitness/registry/app/pkg/commons"
"github.com/harness/gitness/registry/app/pkg/docker"
"github.com/harness/gitness/registry/app/storage"
"github.com/rs/zerolog/log"
)
@ -36,8 +37,8 @@ func (h *Handler) GetManifest(w http.ResponseWriter, r *http.Request) {
result := h.Controller.PullManifest(
ctx,
info,
r.Header[commons.HeaderAccept],
r.Header[commons.HeaderIfNoneMatch],
r.Header[storage.HeaderAccept],
r.Header[storage.HeaderIfNoneMatch],
)
if commons.IsEmpty(result.GetErrors()) {
response, ok := result.(*docker.GetManifestResponse)

12
registry/app/api/handler/oci/head_manifest.go
View File

@ -20,6 +20,7 @@ import (
"github.com/harness/gitness/registry/app/dist_temp/errcode"
"github.com/harness/gitness/registry/app/pkg/commons"
"github.com/harness/gitness/registry/app/pkg/docker"
"github.com/harness/gitness/registry/app/storage"
)
// HeadManifest fetches the image manifest from the storage backend, if it exists.
@ -33,11 +34,16 @@ func (h *Handler) HeadManifest(w http.ResponseWriter, r *http.Request) {
result := h.Controller.HeadManifest(
r.Context(),
info,
r.Header[commons.HeaderAccept],
r.Header[commons.HeaderIfNoneMatch],
r.Header[storage.HeaderAccept],
r.Header[storage.HeaderIfNoneMatch],
)
if commons.IsEmpty(result.GetErrors()) {
result.(*docker.GetManifestResponse).ResponseHeaders.WriteToResponse(w)
response, ok := result.(*docker.GetManifestResponse)
if !ok {
handleErrors(r.Context(), errcode.Errors{errcode.ErrCodeManifestUnknown}, w)
return
}
response.ResponseHeaders.WriteToResponse(w)
}
handleErrors(r.Context(), result.GetErrors(), w)
}

7
registry/app/api/handler/oci/patch_blob_upload.go
View File

@ -18,6 +18,7 @@ import (
"net/http"
"github.com/harness/gitness/registry/app/pkg/commons"
"github.com/harness/gitness/registry/app/storage"
)
func (h *Handler) PatchBlobUpload(w http.ResponseWriter, r *http.Request) {
@ -26,9 +27,9 @@ func (h *Handler) PatchBlobUpload(w http.ResponseWriter, r *http.Request) {
handleErrors(r.Context(), []error{err}, w)
return
}
ct := r.Header.Get(commons.HeaderContentType)
cr := r.Header.Get(commons.HeaderContentRange)
cl := r.Header.Get(commons.HeaderContentLength)
ct := r.Header.Get(storage.HeaderContentType)
cr := r.Header.Get(storage.HeaderContentRange)
cl := r.Header.Get(storage.HeaderContentLength)
length := r.ContentLength
if length > 0 {
r.Body = http.MaxBytesReader(w, r.Body, length)

29
registry/app/api/handler/python/download.go
View File

@ -29,31 +29,48 @@ func (h *handler) DownloadPackageFile(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
info, ok := request.ArtifactInfoFrom(ctx).(*pythontype.ArtifactInfo)
if !ok {
log.Ctx(ctx).Error().Msg("Failed to get python artifact info from context")
h.HandleErrors(r.Context(), []error{fmt.Errorf("failed to fetch python artifact info from context")}, w)
h.HandleErrors(ctx, []error{fmt.Errorf("failed to fetch info from context")}, w)
return
}
response := h.controller.DownloadPackageFile(ctx, *info)
if response == nil {
h.HandleErrors(ctx, []error{fmt.Errorf("failed to get response from controller")}, w)
return
}
defer func() {
if response.Body != nil {
err := response.Body.Close()
if err != nil {
log.Ctx(r.Context()).Error().Msgf("Failed to close body: %v", err)
log.Ctx(ctx).Error().Msgf("Failed to close body: %v", err)
}
}
if response.ReadCloser != nil {
err := response.ReadCloser.Close()
if err != nil {
log.Ctx(ctx).Error().Msgf("Failed to close read closer: %v", err)
}
}
}()
if !commons.IsEmpty(response.GetErrors()) {
h.HandleErrors(r.Context(), response.GetErrors(), w)
h.HandleErrors(ctx, response.GetErrors(), w)
return
}
w.Header().Set("Content-Disposition", "attachment; filename="+info.Filename)
if response.RedirectURL != "" {
http.Redirect(w, r, response.RedirectURL, http.StatusTemporaryRedirect)
return
}
h.ServeContent(w, r, response.Body, info.Filename)
w.WriteHeader(http.StatusOK)
err := commons.ServeContent(w, r, response.Body, info.Filename, response.ReadCloser)
if err != nil {
log.Ctx(ctx).Error().Msgf("Failed to serve content: %v", err)
h.HandleErrors(ctx, []error{err}, w)
return
}
response.ResponseHeaders.WriteToResponse(w)
}

73
registry/app/api/handler/python/handler.go
View File

@ -15,7 +15,11 @@
package python
import (
"fmt"
"net/http"
"regexp"
"strings"
"unicode"
"github.com/harness/gitness/registry/app/api/controller/pkg/python"
"github.com/harness/gitness/registry/app/api/handler/packages"
@ -24,10 +28,28 @@ import (
"github.com/harness/gitness/registry/app/pkg"
"github.com/harness/gitness/registry/app/pkg/commons"
pythontype "github.com/harness/gitness/registry/app/pkg/types/python"
"github.com/harness/gitness/registry/validation"
"github.com/go-chi/chi/v5"
"github.com/rs/zerolog/log"
)
// https://peps.python.org/pep-0426/#name
var (
normalizer = strings.NewReplacer(".", "-", "_", "-")
nameMatcher = regexp.MustCompile(`\A(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\.\-_]*[a-zA-Z0-9])\z`)
)
// https://peps.python.org/pep-0440/#appendix-b-parsing-version-strings-with-regular-expressions
var versionMatcher = regexp.MustCompile(`\Av?` +
`(?:[0-9]+!)?` + // epoch
`[0-9]+(?:\.[0-9]+)*` + // release segment
`(?:[-_\.]?(?:a|b|c|rc|alpha|beta|pre|preview)[-_\.]?[0-9]*)?` + // pre-release
`(?:-[0-9]+|[-_\.]?(?:post|rev|r)[-_\.]?[0-9]*)?` + // post release
`(?:[-_\.]?dev[-_\.]?[0-9]*)?` + // dev release
`(?:\+[a-z0-9]+(?:[-_\.][a-z0-9]+)*)?` + // local version
`\z`)
type Handler interface {
pkg.ArtifactInfoProvider
UploadPackageFile(writer http.ResponseWriter, request *http.Request)
@ -74,7 +96,15 @@ func (h *handler) GetPackageArtifactInfo(r *http.Request) (pkg.PackageArtifactIn
}
}
image = normalizer.Replace(image)
if image != "" && version != "" && !isValidNameAndVersion(image, version) {
log.Info().Msgf("Invalid image name/version: %s/%s", info.Image, version)
return nil, fmt.Errorf("invalid name or version")
}
md.HomePage = getHomePage(md)
info.Image = image
return &pythontype.ArtifactInfo{
ArtifactInfo: info,
Metadata: md,
@ -82,3 +112,46 @@ func (h *handler) GetPackageArtifactInfo(r *http.Request) (pkg.PackageArtifactIn
Version: version,
}, nil
}
func getHomePage(md python2.Metadata) string {
var homepageURL string
if len(md.ProjectURLs) > 0 {
for k, v := range md.ProjectURLs {
if normalizeLabel(k) != "homepage" {
continue
}
homepageURL = strings.TrimSpace(v)
break
}
}
if len(homepageURL) == 0 {
homepageURL = md.HomePage
}
if !validation.IsValidURL(homepageURL) {
homepageURL = ""
}
return homepageURL
}
func isValidNameAndVersion(image, version string) bool {
return nameMatcher.MatchString(image) && versionMatcher.MatchString(version)
}
// Normalizes a Project-URL label.
// See https://packaging.python.org/en/latest/specifications/well-known-project-urls/#label-normalization.
func normalizeLabel(label string) string {
var builder strings.Builder
// "A label is normalized by deleting all ASCII punctuation and whitespace, and then converting the result
// to lowercase."
for _, r := range label {
if unicode.IsPunct(r) || unicode.IsSpace(r) {
continue
}
builder.WriteRune(unicode.ToLower(r))
}
return builder.String()
}

104
registry/app/api/openapi/contracts/artifact/services.gen.go
View File

@ -1,6 +1,6 @@
// Package artifact provides primitives to interact with the openapi HTTP API.
//
// Code generated by github.com/deepmap/oapi-codegen/v2 version v2.1.0 DO NOT EDIT.
// Code generated by github.com/oapi-codegen/oapi-codegen/v2 version v2.4.1 DO NOT EDIT.
package artifact
import (
@ -346,7 +346,6 @@ type MiddlewareFunc func(http.Handler) http.Handler
// CreateRegistry operation middleware
func (siw *ServerInterfaceWrapper) CreateRegistry(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
var err error
@ -369,12 +368,11 @@ func (siw *ServerInterfaceWrapper) CreateRegistry(w http.ResponseWriter, r *http
handler = middleware(handler)
}
handler.ServeHTTP(w, r.WithContext(ctx))
handler.ServeHTTP(w, r)
}
// DeleteRegistry operation middleware
func (siw *ServerInterfaceWrapper) DeleteRegistry(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
var err error
@ -395,12 +393,11 @@ func (siw *ServerInterfaceWrapper) DeleteRegistry(w http.ResponseWriter, r *http
handler = middleware(handler)
}
handler.ServeHTTP(w, r.WithContext(ctx))
handler.ServeHTTP(w, r)
}
// GetRegistry operation middleware
func (siw *ServerInterfaceWrapper) GetRegistry(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
var err error
@ -421,12 +418,11 @@ func (siw *ServerInterfaceWrapper) GetRegistry(w http.ResponseWriter, r *http.Re
handler = middleware(handler)
}
handler.ServeHTTP(w, r.WithContext(ctx))
handler.ServeHTTP(w, r)
}
// ModifyRegistry operation middleware
func (siw *ServerInterfaceWrapper) ModifyRegistry(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
var err error
@ -447,12 +443,11 @@ func (siw *ServerInterfaceWrapper) ModifyRegistry(w http.ResponseWriter, r *http
handler = middleware(handler)
}
handler.ServeHTTP(w, r.WithContext(ctx))
handler.ServeHTTP(w, r)
}
// ListArtifactLabels operation middleware
func (siw *ServerInterfaceWrapper) ListArtifactLabels(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
var err error
@ -500,12 +495,11 @@ func (siw *ServerInterfaceWrapper) ListArtifactLabels(w http.ResponseWriter, r *
handler = middleware(handler)
}
handler.ServeHTTP(w, r.WithContext(ctx))
handler.ServeHTTP(w, r)
}
// GetArtifactStatsForRegistry operation middleware
func (siw *ServerInterfaceWrapper) GetArtifactStatsForRegistry(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
var err error
@ -545,12 +539,11 @@ func (siw *ServerInterfaceWrapper) GetArtifactStatsForRegistry(w http.ResponseWr
handler = middleware(handler)
}
handler.ServeHTTP(w, r.WithContext(ctx))
handler.ServeHTTP(w, r)
}
// DeleteArtifact operation middleware
func (siw *ServerInterfaceWrapper) DeleteArtifact(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
var err error
@ -580,12 +573,11 @@ func (siw *ServerInterfaceWrapper) DeleteArtifact(w http.ResponseWriter, r *http
handler = middleware(handler)
}
handler.ServeHTTP(w, r.WithContext(ctx))
handler.ServeHTTP(w, r)
}
// UpdateArtifactLabels operation middleware
func (siw *ServerInterfaceWrapper) UpdateArtifactLabels(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
var err error
@ -615,12 +607,11 @@ func (siw *ServerInterfaceWrapper) UpdateArtifactLabels(w http.ResponseWriter, r
handler = middleware(handler)
}
handler.ServeHTTP(w, r.WithContext(ctx))
handler.ServeHTTP(w, r)
}
// GetArtifactStats operation middleware
func (siw *ServerInterfaceWrapper) GetArtifactStats(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
var err error
@ -669,12 +660,11 @@ func (siw *ServerInterfaceWrapper) GetArtifactStats(w http.ResponseWriter, r *ht
handler = middleware(handler)
}
handler.ServeHTTP(w, r.WithContext(ctx))
handler.ServeHTTP(w, r)
}
// GetArtifactSummary operation middleware
func (siw *ServerInterfaceWrapper) GetArtifactSummary(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
var err error
@ -704,12 +694,11 @@ func (siw *ServerInterfaceWrapper) GetArtifactSummary(w http.ResponseWriter, r *
handler = middleware(handler)
}
handler.ServeHTTP(w, r.WithContext(ctx))
handler.ServeHTTP(w, r)
}
// DeleteArtifactVersion operation middleware
func (siw *ServerInterfaceWrapper) DeleteArtifactVersion(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
var err error
@ -748,12 +737,11 @@ func (siw *ServerInterfaceWrapper) DeleteArtifactVersion(w http.ResponseWriter,
handler = middleware(handler)
}
handler.ServeHTTP(w, r.WithContext(ctx))
handler.ServeHTTP(w, r)
}
// GetArtifactDetails operation middleware
func (siw *ServerInterfaceWrapper) GetArtifactDetails(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
var err error
@ -803,12 +791,11 @@ func (siw *ServerInterfaceWrapper) GetArtifactDetails(w http.ResponseWriter, r *
handler = middleware(handler)
}
handler.ServeHTTP(w, r.WithContext(ctx))
handler.ServeHTTP(w, r)
}
// GetDockerArtifactDetails operation middleware
func (siw *ServerInterfaceWrapper) GetDockerArtifactDetails(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
var err error
@ -865,12 +852,11 @@ func (siw *ServerInterfaceWrapper) GetDockerArtifactDetails(w http.ResponseWrite
handler = middleware(handler)
}
handler.ServeHTTP(w, r.WithContext(ctx))
handler.ServeHTTP(w, r)
}
// GetDockerArtifactLayers operation middleware
func (siw *ServerInterfaceWrapper) GetDockerArtifactLayers(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
var err error
@ -927,12 +913,11 @@ func (siw *ServerInterfaceWrapper) GetDockerArtifactLayers(w http.ResponseWriter
handler = middleware(handler)
}
handler.ServeHTTP(w, r.WithContext(ctx))
handler.ServeHTTP(w, r)
}
// GetDockerArtifactManifest operation middleware
func (siw *ServerInterfaceWrapper) GetDockerArtifactManifest(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
var err error
@ -989,12 +974,11 @@ func (siw *ServerInterfaceWrapper) GetDockerArtifactManifest(w http.ResponseWrit
handler = middleware(handler)
}
handler.ServeHTTP(w, r.WithContext(ctx))
handler.ServeHTTP(w, r)
}
// GetDockerArtifactManifests operation middleware
func (siw *ServerInterfaceWrapper) GetDockerArtifactManifests(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
var err error
@ -1033,12 +1017,11 @@ func (siw *ServerInterfaceWrapper) GetDockerArtifactManifests(w http.ResponseWri
handler = middleware(handler)
}
handler.ServeHTTP(w, r.WithContext(ctx))
handler.ServeHTTP(w, r)
}
// GetArtifactFiles operation middleware
func (siw *ServerInterfaceWrapper) GetArtifactFiles(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
var err error
@ -1120,12 +1103,11 @@ func (siw *ServerInterfaceWrapper) GetArtifactFiles(w http.ResponseWriter, r *ht
handler = middleware(handler)
}
handler.ServeHTTP(w, r.WithContext(ctx))
handler.ServeHTTP(w, r)
}
// GetHelmArtifactDetails operation middleware
func (siw *ServerInterfaceWrapper) GetHelmArtifactDetails(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
var err error
@ -1164,12 +1146,11 @@ func (siw *ServerInterfaceWrapper) GetHelmArtifactDetails(w http.ResponseWriter,
handler = middleware(handler)
}
handler.ServeHTTP(w, r.WithContext(ctx))
handler.ServeHTTP(w, r)
}
// GetHelmArtifactManifest operation middleware
func (siw *ServerInterfaceWrapper) GetHelmArtifactManifest(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
var err error
@ -1208,12 +1189,11 @@ func (siw *ServerInterfaceWrapper) GetHelmArtifactManifest(w http.ResponseWriter
handler = middleware(handler)
}
handler.ServeHTTP(w, r.WithContext(ctx))
handler.ServeHTTP(w, r)
}
// GetArtifactVersionSummary operation middleware
func (siw *ServerInterfaceWrapper) GetArtifactVersionSummary(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
var err error
@ -1252,12 +1232,11 @@ func (siw *ServerInterfaceWrapper) GetArtifactVersionSummary(w http.ResponseWrit
handler = middleware(handler)
}
handler.ServeHTTP(w, r.WithContext(ctx))
handler.ServeHTTP(w, r)
}
// GetAllArtifactVersions operation middleware
func (siw *ServerInterfaceWrapper) GetAllArtifactVersions(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
var err error
@ -1330,12 +1309,11 @@ func (siw *ServerInterfaceWrapper) GetAllArtifactVersions(w http.ResponseWriter,
handler = middleware(handler)
}
handler.ServeHTTP(w, r.WithContext(ctx))
handler.ServeHTTP(w, r)
}
// GetAllArtifactsByRegistry operation middleware
func (siw *ServerInterfaceWrapper) GetAllArtifactsByRegistry(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
var err error
@ -1407,12 +1385,11 @@ func (siw *ServerInterfaceWrapper) GetAllArtifactsByRegistry(w http.ResponseWrit
handler = middleware(handler)
}
handler.ServeHTTP(w, r.WithContext(ctx))
handler.ServeHTTP(w, r)
}
// GetClientSetupDetails operation middleware
func (siw *ServerInterfaceWrapper) GetClientSetupDetails(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
var err error
@ -1452,12 +1429,11 @@ func (siw *ServerInterfaceWrapper) GetClientSetupDetails(w http.ResponseWriter,
handler = middleware(handler)
}
handler.ServeHTTP(w, r.WithContext(ctx))
handler.ServeHTTP(w, r)
}
// ListWebhooks operation middleware
func (siw *ServerInterfaceWrapper) ListWebhooks(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
var err error
@ -1521,12 +1497,11 @@ func (siw *ServerInterfaceWrapper) ListWebhooks(w http.ResponseWriter, r *http.R
handler = middleware(handler)
}
handler.ServeHTTP(w, r.WithContext(ctx))
handler.ServeHTTP(w, r)
}
// CreateWebhook operation middleware
func (siw *ServerInterfaceWrapper) CreateWebhook(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
var err error
@ -1547,12 +1522,11 @@ func (siw *ServerInterfaceWrapper) CreateWebhook(w http.ResponseWriter, r *http.
handler = middleware(handler)
}
handler.ServeHTTP(w, r.WithContext(ctx))
handler.ServeHTTP(w, r)
}
// DeleteWebhook operation middleware
func (siw *ServerInterfaceWrapper) DeleteWebhook(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
var err error
@ -1582,12 +1556,11 @@ func (siw *ServerInterfaceWrapper) DeleteWebhook(w http.ResponseWriter, r *http.
handler = middleware(handler)
}
handler.ServeHTTP(w, r.WithContext(ctx))
handler.ServeHTTP(w, r)
}
// GetWebhook operation middleware
func (siw *ServerInterfaceWrapper) GetWebhook(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
var err error
@ -1617,12 +1590,11 @@ func (siw *ServerInterfaceWrapper) GetWebhook(w http.ResponseWriter, r *http.Req
handler = middleware(handler)
}
handler.ServeHTTP(w, r.WithContext(ctx))
handler.ServeHTTP(w, r)
}
// UpdateWebhook operation middleware
func (siw *ServerInterfaceWrapper) UpdateWebhook(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
var err error
@ -1652,12 +1624,11 @@ func (siw *ServerInterfaceWrapper) UpdateWebhook(w http.ResponseWriter, r *http.
handler = middleware(handler)
}
handler.ServeHTTP(w, r.WithContext(ctx))
handler.ServeHTTP(w, r)
}
// ListWebhookExecutions operation middleware
func (siw *ServerInterfaceWrapper) ListWebhookExecutions(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
var err error
@ -1706,12 +1677,11 @@ func (siw *ServerInterfaceWrapper) ListWebhookExecutions(w http.ResponseWriter,
handler = middleware(handler)
}
handler.ServeHTTP(w, r.WithContext(ctx))
handler.ServeHTTP(w, r)
}
// GetWebhookExecution operation middleware
func (siw *ServerInterfaceWrapper) GetWebhookExecution(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
var err error
@ -1750,12 +1720,11 @@ func (siw *ServerInterfaceWrapper) GetWebhookExecution(w http.ResponseWriter, r
handler = middleware(handler)
}
handler.ServeHTTP(w, r.WithContext(ctx))
handler.ServeHTTP(w, r)
}
// ReTriggerWebhookExecution operation middleware
func (siw *ServerInterfaceWrapper) ReTriggerWebhookExecution(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
var err error
@ -1794,12 +1763,11 @@ func (siw *ServerInterfaceWrapper) ReTriggerWebhookExecution(w http.ResponseWrit
handler = middleware(handler)
}
handler.ServeHTTP(w, r.WithContext(ctx))
handler.ServeHTTP(w, r)
}
// GetArtifactStatsForSpace operation middleware
func (siw *ServerInterfaceWrapper) GetArtifactStatsForSpace(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
var err error
@ -1839,12 +1807,11 @@ func (siw *ServerInterfaceWrapper) GetArtifactStatsForSpace(w http.ResponseWrite
handler = middleware(handler)
}
handler.ServeHTTP(w, r.WithContext(ctx))
handler.ServeHTTP(w, r)
}
// GetAllArtifacts operation middleware
func (siw *ServerInterfaceWrapper) GetAllArtifacts(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
var err error
@ -1932,12 +1899,11 @@ func (siw *ServerInterfaceWrapper) GetAllArtifacts(w http.ResponseWriter, r *htt
handler = middleware(handler)
}
handler.ServeHTTP(w, r.WithContext(ctx))
handler.ServeHTTP(w, r)
}
// GetAllRegistries operation middleware
func (siw *ServerInterfaceWrapper) GetAllRegistries(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
var err error
@ -2025,7 +1991,7 @@ func (siw *ServerInterfaceWrapper) GetAllRegistries(w http.ResponseWriter, r *ht
handler = middleware(handler)
}
handler.ServeHTTP(w, r.WithContext(ctx))
handler.ServeHTTP(w, r)
}
type UnescapedCookieParamError struct {

2
registry/app/api/openapi/contracts/artifact/types.gen.go
View File

@ -1,6 +1,6 @@
// Package artifact provides primitives to interact with the openapi HTTP API.
//
// Code generated by github.com/deepmap/oapi-codegen/v2 version v2.1.0 DO NOT EDIT.
// Code generated by github.com/oapi-codegen/oapi-codegen/v2 version v2.4.1 DO NOT EDIT.
package artifact
import (

58
registry/app/pkg/base/base.go
View File

@ -18,6 +18,7 @@ import (
"context"
"encoding/json"
"fmt"
"io"
"mime/multipart"
"net/http"
"strings"
@ -37,7 +38,10 @@ import (
var _ LocalBase = (*localBase)(nil)
type LocalBase interface {
Upload(
// UploadFile uploads the file to the storage.
// FIXME: Validate upload by given sha256 or any other checksums provided
UploadFile(
ctx context.Context,
info pkg.ArtifactInfo,
fileName string,
@ -50,12 +54,23 @@ type LocalBase interface {
// each package implementation should have their own.
headers *commons.ResponseHeaders, sha256 string, err errcode.Error,
)
Upload(
ctx context.Context,
info pkg.ArtifactInfo,
fileName string,
version string,
path string,
file io.ReadCloser,
metadata metadata.Metadata,
) (*commons.ResponseHeaders, string, errcode.Error)
Download(ctx context.Context, info pkg.ArtifactInfo, version string, fileName string) (
*commons.ResponseHeaders,
*storage.FileReader,
string,
[]error,
)
Exists(ctx context.Context, info pkg.ArtifactInfo, version string, fileName string) bool
}
type localBase struct {
@ -82,7 +97,7 @@ func NewLocalBase(
}
}
func (l *localBase) Upload(
func (l *localBase) UploadFile(
ctx context.Context,
info pkg.ArtifactInfo,
fileName string,
@ -91,6 +106,31 @@ func (l *localBase) Upload(
file multipart.File,
// TODO: Metadata shouldn't be provided as a parameter, it should be fetched or created.
metadata metadata.Metadata,
) (*commons.ResponseHeaders, string, errcode.Error) {
return l.uploadInternal(ctx, info, fileName, version, path, file, nil, metadata)
}
func (l *localBase) Upload(
ctx context.Context,
info pkg.ArtifactInfo,
fileName string,
version string,
path string,
file io.ReadCloser,
metadata metadata.Metadata,
) (*commons.ResponseHeaders, string, errcode.Error) {
return l.uploadInternal(ctx, info, fileName, version, path, nil, file, metadata)
}
func (l *localBase) uploadInternal(
ctx context.Context,
info pkg.ArtifactInfo,
fileName string,
version string,
path string,
file multipart.File,
fileReadCloser io.ReadCloser,
metadata metadata.Metadata,
) (*commons.ResponseHeaders, string, errcode.Error) {
responseHeaders := &commons.ResponseHeaders{
Headers: make(map[string]string),
@ -108,7 +148,7 @@ func (l *localBase) Upload(
return responseHeaders, "", errcode.ErrCodeUnknown.WithDetail(err)
}
fileInfo, err := l.fileManager.UploadFile(ctx, path, info.RegIdentifier, registry.ID,
info.RootParentID, info.RootIdentifier, file, nil, fileName)
info.RootParentID, info.RootIdentifier, file, fileReadCloser, fileName)
if err != nil {
return responseHeaders, "", errcode.ErrCodeUnknown.WithDetail(err)
}
@ -171,9 +211,10 @@ func (l *localBase) Download(ctx context.Context, info pkg.ArtifactInfo, version
}
path := "/" + info.Image + "/" + version + "/" + fileName
reg, _ := l.registryDao.GetByRootParentIDAndName(ctx, info.RootParentID, info.RegIdentifier)
fileReader, _, redirectURL, err := l.fileManager.DownloadFile(ctx, path, types.Registry{
ID: info.RegistryID,
ID: reg.ID,
Name: info.RegIdentifier,
}, info.RootIdentifier)
if err != nil {
@ -183,11 +224,18 @@ func (l *localBase) Download(ctx context.Context, info pkg.ArtifactInfo, version
return responseHeaders, fileReader, redirectURL, nil
}
func (l *localBase) Exists(ctx context.Context, info pkg.ArtifactInfo, version string, fileName string) bool {
filePath := "/" + info.Image + "/" + version + "/" + fileName
sha256, _ := l.fileManager.HeadFile(ctx, filePath, info.RegistryID)
//FIXME: err should be checked on if the record doesn't exist or there was DB call issue
return sha256 != ""
}
func (l *localBase) updateMetadata(
dbArtifact *types.Artifact,
inputMetadata metadata.Metadata,
info pkg.ArtifactInfo,
fileInfo pkg.FileInfo,
fileInfo types.FileInfo,
) error {
var files []metadata.File
if dbArtifact != nil {

35
registry/app/pkg/base/wrapper.go
View File

@ -1,16 +1,16 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package base
@ -52,7 +52,12 @@ func NoProxyWrapper(
}
log.Ctx(ctx).Info().Msgf("Using Repository: %s, Type: %s", registry.Name, registry.Type)
art := getArtifactRegistry(*registry)
art, ok := getArtifactRegistry(*registry).(pkg.Artifact)
if !ok {
log.Ctx(ctx).Error().Msgf("artifact %s is not a registry", registry.Name)
return result
}
result = f(*registry, art)
if pkg.IsEmpty(result.GetErrors()) {
return result
@ -74,9 +79,13 @@ func ProxyWrapper(
if repos, err := getOrderedRepos(ctx, registryDao, requestRepoKey, *info.BaseInfo); err == nil {
for _, registry := range repos {
log.Ctx(ctx).Info().Msgf("Using Repository: %s, Type: %s", registry.Name, registry.Type)
reg := getArtifactRegistry(registry)
if reg != nil {
response = f(registry, reg)
artifact, ok := getArtifactRegistry(registry).(pkg.Artifact)
if !ok {
log.Ctx(ctx).Warn().Msgf("artifact %s is not a registry", registry.Name)
continue
}
if artifact != nil {
response = f(registry, artifact)
if pkg.IsEmpty(response.GetErrors()) {
return response
}

44
registry/app/pkg/commons/request.go
View File

@ -15,28 +15,15 @@
package commons
import (
"errors"
"fmt"
"io"
"net/http"
"reflect"
"time"
"github.com/harness/gitness/registry/app/dist_temp/errcode"
)
const (
HeaderAccept = "Accept"
HeaderAuthorization = "Authorization"
HeaderCacheControl = "Cache-Control"
HeaderContentLength = "Content-Length"
HeaderContentRange = "Content-Range"
HeaderContentType = "Content-Type"
HeaderDockerContentDigest = "Docker-Content-Digest"
HeaderDockerUploadUUID = "Docker-Upload-UUID"
HeaderEtag = "Etag"
HeaderIfNoneMatch = "If-None-Match"
HeaderLink = "Link"
HeaderLocation = "Location"
HeaderOCIFiltersApplied = "OCI-Filters-Applied"
HeaderOCISubject = "OCI-Subject"
HeaderRange = "Range"
"github.com/harness/gitness/registry/app/storage"
)
type ResponseHeaders struct {
@ -95,3 +82,24 @@ func (r *ResponseHeaders) WriteHeadersToResponse(w http.ResponseWriter) {
}
}
}
func ServeContent(
w http.ResponseWriter,
r *http.Request,
body *storage.FileReader,
fileName string,
readCloser io.ReadCloser,
) error {
if body != nil {
http.ServeContent(w, r, fileName, time.Time{}, body)
return nil
}
if readCloser != nil {
_, err := io.Copy(w, readCloser)
if err != nil {
return fmt.Errorf("failed to copy content: %w", err)
}
return nil
}
return errors.New("no content to serve")
}

12
registry/app/pkg/context.go
View File

@ -15,8 +15,6 @@
package pkg
import (
"time"
"github.com/harness/gitness/registry/app/api/openapi/contracts/artifact"
v2 "github.com/distribution/distribution/v3/registry/api/v2"
@ -47,16 +45,6 @@ type RegistryInfo struct {
PackageType artifact.PackageType
}
type FileInfo struct {
Size int64
Sha1 string
Sha256 string
Sha512 string
MD5 string
Filename string
CreatedAt time.Time
}
func (r *RegistryInfo) SetReference(ref string) {
r.Reference = ref
}

19
registry/app/pkg/docker/local.go
View File

@ -383,12 +383,13 @@ func (r *LocalRegistry) fetchBlobInternal(
var dgst digest.Digest
blobs := ctx.OciBlobStore
if err := r.dbBlobLinkExists(ctx, ctx.Digest, info.RegIdentifier, info); err != nil {
if err := r.dbBlobLinkExists(ctx, ctx.Digest, info.RegIdentifier, info); err != nil { //nolint:contextcheck
errs = append(errs, errcode.FromUnknownError(err))
return responseHeaders, nil, -1, nil, "", errs
}
dgst = ctx.Digest
headers := make(map[string]string)
//nolint:contextcheck
fileReader, redirectURL, size, err := blobs.ServeBlobInternal(
ctx.Context,
info.RootIdentifier,
@ -882,7 +883,7 @@ func (r *LocalRegistry) InitBlobUpload(
}
digest := digest.Digest(mountDigest)
if mountDigest != "" && fromRepo != "" {
err := r.dbMountBlob(blobCtx, fromRepo, artInfo.RegIdentifier, digest, artInfo)
err := r.dbMountBlob(blobCtx, fromRepo, artInfo.RegIdentifier, digest, artInfo) //nolint:contextcheck
if err != nil {
e := fmt.Errorf("failed to mount blob in database: %w", err)
errList = append(errList, errcode.FromUnknownError(e))
@ -897,7 +898,7 @@ func (r *LocalRegistry) InitBlobUpload(
}
blobs := blobCtx.OciBlobStore
upload, err := blobs.Create(blobCtx.Context)
upload, err := blobs.Create(blobCtx.Context) //nolint:contextcheck
if err != nil {
if errors.Is(err, storage.ErrUnsupported) {
errList = append(errList, errcode.ErrCodeUnsupported)
@ -916,7 +917,7 @@ func (r *LocalRegistry) InitBlobUpload(
errList = append(errList, errcode.ErrCodeUnknown.WithDetail(err))
return responseHeaders, errList
}
responseHeaders.Headers[commons.HeaderDockerUploadUUID] = blobCtx.Upload.ID()
responseHeaders.Headers[storage.HeaderDockerUploadUUID] = blobCtx.Upload.ID()
responseHeaders.Code = http.StatusAccepted
return responseHeaders, nil
}
@ -1017,7 +1018,7 @@ func (r *LocalRegistry) PushBlob(
}
ctx := r.App.GetBlobsContext(ctx2, artInfo)
if ctx.UUID != "" {
resumeErrs := ResumeBlobUpload(ctx, stateToken)
resumeErrs := ResumeBlobUpload(ctx, stateToken) //nolint:contextcheck
errs = append(errs, resumeErrs...)
}
@ -1048,6 +1049,7 @@ func (r *LocalRegistry) PushBlob(
return responseHeaders, errs
}
//nolint:contextcheck
if err := copyFullPayload(
ctx, contentLength, body, ctx.Upload,
"blob PUT",
@ -1059,6 +1061,7 @@ func (r *LocalRegistry) PushBlob(
return responseHeaders, errs
}
//nolint:contextcheck
desc, err := ctx.Upload.Commit(
ctx, artInfo.RootIdentifier, manifest.Descriptor{
Digest: dgst,
@ -1089,11 +1092,12 @@ func (r *LocalRegistry) PushBlob(
errcode.ErrCodeBlobUploadInvalid.WithDetail(err),
)
default:
dcontext.GetLogger(ctx, log.Error()).Msgf("unknown error completing upload: %v", err)
//nolint:contextcheck
log.Ctx(ctx).Error().Msgf("unknown error completing upload: %v", err)
errs = append(errs, errcode.ErrCodeUnknown.WithDetail(err))
}
}
//nolint:contextcheck
// Clean up the backend blob data if there was an error.
if err := ctx.Upload.Cancel(ctx); err != nil {
// If the cleanup fails, all we can do is observe and report.
@ -1104,6 +1108,7 @@ func (r *LocalRegistry) PushBlob(
return responseHeaders, errs
}
//nolint:contextcheck
err = r.dbPutBlobUploadComplete(
ctx,
artInfo.RegIdentifier,

30
registry/app/pkg/filemanager/file_manager.go
View File

@ -22,7 +22,6 @@ import (
"path"
"strings"
"github.com/harness/gitness/registry/app/pkg"
"github.com/harness/gitness/registry/app/storage"
"github.com/harness/gitness/registry/app/store"
"github.com/harness/gitness/registry/types"
@ -40,7 +39,8 @@ const (
pathFormat = "for path: %s, with error %w"
)
func NewFileManager(app *App, registryDao store.RegistryRepository, genericBlobDao store.GenericBlobRepository,
func NewFileManager(
app *App, registryDao store.RegistryRepository, genericBlobDao store.GenericBlobRepository,
nodesDao store.NodesRepository,
tx dbtx.Transactor,
) FileManager {
@ -71,7 +71,7 @@ func (f *FileManager) UploadFile(
file multipart.File,
fileReader io.Reader,
filename string,
) (pkg.FileInfo, error) {
) (types.FileInfo, error) {
// uploading the file to temporary path in file storage
blobContext := f.App.GetBlobsContext(ctx, regName, rootIdentifier)
pathUUID := uuid.NewString()
@ -81,7 +81,7 @@ func (f *FileManager) UploadFile(
if err != nil {
log.Error().Msgf("failed to initiate the file upload for file with"+
" name : %s with error : %s", filename, err.Error())
return pkg.FileInfo{}, fmt.Errorf("failed to initiate the file upload "+
return types.FileInfo{}, fmt.Errorf("failed to initiate the file upload "+
"for file with name : %s with error : %w", filename, err)
}
defer fw.Close()
@ -90,7 +90,7 @@ func (f *FileManager) UploadFile(
if err != nil {
log.Error().Msgf("failed to upload the file on temparary location"+
" with name : %s with error : %s", filename, err.Error())
return pkg.FileInfo{}, fmt.Errorf("failed to upload the file on temparary "+
return types.FileInfo{}, fmt.Errorf("failed to upload the file on temparary "+
"location with name : %s with error : %w", filename, err)
}
fileInfo.Filename = filename
@ -102,7 +102,7 @@ func (f *FileManager) UploadFile(
if err != nil {
log.Error().Msgf("failed to Move the file on permanent location "+
"with name : %s with error : %s", filename, err.Error())
return pkg.FileInfo{}, fmt.Errorf("failed to Move the file on permanent"+
return types.FileInfo{}, fmt.Errorf("failed to Move the file on permanent"+
" location with name : %s with error : %w", filename, err)
}
@ -120,7 +120,7 @@ func (f *FileManager) UploadFile(
if err != nil {
log.Error().Msgf("failed to save generic blob in db with "+
"sha256 : %s, err: %s", fileInfo.Sha256, err.Error())
return pkg.FileInfo{}, fmt.Errorf("failed to save generic blob"+
return types.FileInfo{}, fmt.Errorf("failed to save generic blob"+
" in db with sha256 : %s, err: %w", fileInfo.Sha256, err)
}
blobID = gb.ID
@ -135,7 +135,7 @@ func (f *FileManager) UploadFile(
if err != nil {
log.Error().Msgf("failed to save nodes for file : %s, with "+
"path : %s, err: %s", filename, filePath, err)
return pkg.FileInfo{}, fmt.Errorf("failed to save nodes for"+
return types.FileInfo{}, fmt.Errorf("failed to save nodes for"+
" file : %s, with path : %s, err: %w", filename, filePath, err)
}
return fileInfo, nil
@ -175,8 +175,10 @@ func (f *FileManager) createNodes(ctx context.Context, filePath string, blobID s
return nil
}
func (f *FileManager) SaveNode(ctx context.Context, filePath string, blobID string, regID int64, segment string,
parentID string, nodePath string, isFile bool) (string, error) {
func (f *FileManager) SaveNode(
ctx context.Context, filePath string, blobID string, regID int64, segment string,
parentID string, nodePath string, isFile bool,
) (string, error) {
node := &types.Node{
Name: segment,
RegistryID: regID,
@ -261,20 +263,20 @@ func (f *FileManager) GetFileMetadata(
ctx context.Context,
filePath string,
regID int64,
) (pkg.FileInfo, error) {
) (types.FileInfo, error) {
node, err := f.nodesDao.GetByPathAndRegistryID(ctx, regID, filePath)
if err != nil {
return pkg.FileInfo{}, fmt.Errorf("failed to get the node path mapping "+
return types.FileInfo{}, fmt.Errorf("failed to get the node path mapping "+
pathFormat, filePath, err)
}
blob, err := f.genericBlobDao.FindByID(ctx, node.BlobID)
if err != nil {
return pkg.FileInfo{}, fmt.Errorf("failed to get the blob for path: %s, "+
return types.FileInfo{}, fmt.Errorf("failed to get the blob for path: %s, "+
"with blob id: %s, with error %s", filePath, node.BlobID, err)
}
return pkg.FileInfo{
return types.FileInfo{
Sha1: blob.Sha1,
Size: blob.Size,
Sha256: blob.Sha256,

2
registry/app/pkg/generic/controller.go
View File

@ -173,7 +173,7 @@ func (c Controller) UploadArtifact(
func (c Controller) updateMetadata(
dbArtifact *types.Artifact, metadataInput *metadata.GenericMetadata,
info pkg.GenericArtifactInfo, fileInfo pkg.FileInfo,
info pkg.GenericArtifactInfo, fileInfo types.FileInfo,
) error {
var files []metadata.File
if dbArtifact != nil {

2
registry/app/pkg/maven/local.go
View File

@ -190,7 +190,7 @@ func (r *LocalRegistry) PutArtifact(ctx context.Context, info pkg.MavenArtifactI
func (r *LocalRegistry) updateArtifactMetadata(
dbArtifact *types.Artifact, mavenMetadata *metadata.MavenMetadata,
info pkg.MavenArtifactInfo, fileInfo pkg.FileInfo,
info pkg.MavenArtifactInfo, fileInfo types.FileInfo,
) error {
var files []metadata.File
if dbArtifact != nil {

6
registry/app/pkg/maven/utils/utils.go
View File

@ -22,6 +22,7 @@ import (
"github.com/harness/gitness/registry/app/pkg"
"github.com/harness/gitness/registry/app/pkg/commons"
"github.com/harness/gitness/registry/types"
)
const (
@ -81,8 +82,9 @@ func IsMainArtifactFile(info pkg.MavenArtifactInfo) bool {
return false
}
func SetHeaders(info pkg.MavenArtifactInfo,
fileInfo pkg.FileInfo,
func SetHeaders(
info pkg.MavenArtifactInfo,
fileInfo types.FileInfo,
) *commons.ResponseHeaders {
responseHeaders := &commons.ResponseHeaders{
Headers: map[string]string{},

102
registry/app/pkg/python/local.go
View File

@ -18,9 +18,11 @@ import (
"context"
"encoding/json"
"fmt"
"io"
"mime/multipart"
"net/http"
"sort"
"strconv"
"strings"
urlprovider "github.com/harness/gitness/app/url"
"github.com/harness/gitness/registry/app/api/openapi/contracts/artifact"
@ -31,10 +33,12 @@ import (
"github.com/harness/gitness/registry/app/pkg/commons"
"github.com/harness/gitness/registry/app/pkg/filemanager"
pythontype "github.com/harness/gitness/registry/app/pkg/types/python"
"github.com/harness/gitness/registry/app/remote/adapter/commons/pypi"
"github.com/harness/gitness/registry/app/storage"
"github.com/harness/gitness/registry/app/store"
"github.com/harness/gitness/registry/types"
"github.com/harness/gitness/store/database/dbtx"
"github.com/rs/zerolog/log"
)
var _ pkg.Artifact = (*localRegistry)(nil)
@ -85,31 +89,19 @@ func (c *localRegistry) GetPackageTypes() []artifact.PackageType {
return []artifact.PackageType{artifact.PackageTypePYTHON}
}
func (c *localRegistry) DownloadPackageFile(ctx context.Context, info pythontype.ArtifactInfo) (
*commons.ResponseHeaders,
*storage.FileReader,
string,
[]error,
) {
responseHeaders := &commons.ResponseHeaders{
Headers: make(map[string]string),
Code: 0,
func (c *localRegistry) DownloadPackageFile(
ctx context.Context,
info pythontype.ArtifactInfo,
) (*commons.ResponseHeaders, *storage.FileReader, io.ReadCloser, string, []error) {
headers, fileReader, redirectURL, errors := c.localBase.Download(ctx, info.ArtifactInfo, info.Version,
info.Filename)
if len(errors) > 0 {
return nil, nil, nil, "", errors
}
path := "/" + info.Image + "/" + info.Version + "/" + info.Filename
fileReader, _, redirectURL, err := c.fileManager.DownloadFile(ctx, path, types.Registry{
ID: info.RegistryID,
Name: info.RegIdentifier,
}, info.RootIdentifier)
if err != nil {
return responseHeaders, nil, "", []error{err}
}
responseHeaders.Code = http.StatusOK
return responseHeaders, fileReader, redirectURL, nil
return headers, fileReader, nil, redirectURL, nil
}
// Metadata represents the metadata of a Python package.
// GetPackageMetadata Metadata represents the metadata of a Python package.
func (c *localRegistry) GetPackageMetadata(
ctx context.Context,
info pythontype.ArtifactInfo,
@ -152,22 +144,68 @@ func (c *localRegistry) GetPackageMetadata(
}
}
// Sort files by Name
sort.Slice(packageMetadata.Files, func(i, j int) bool {
return packageMetadata.Files[i].Name < packageMetadata.Files[j].Name
})
sortPackageMetadata(ctx, packageMetadata)
return packageMetadata, nil
}
func sortPackageMetadata(ctx context.Context, metadata pythontype.PackageMetadata) {
sort.Slice(metadata.Files, func(i, j int) bool {
version1 := pypi.GetPyPIVersion(metadata.Files[i].Name)
version2 := pypi.GetPyPIVersion(metadata.Files[j].Name)
if version1 == "" || version2 == "" || version1 == version2 {
return metadata.Files[i].Name < metadata.Files[j].Name
}
vi := parseVersion(ctx, version1)
vj := parseVersion(ctx, version2)
for k := 0; k < len(vi) && k < len(vj); k++ {
if vi[k] != vj[k] {
return vi[k] < vj[k]
}
}
return len(vi) < len(vj)
})
}
func parseVersion(ctx context.Context, version string) []int {
parts := strings.Split(version, ".")
result := make([]int, len(parts))
for i, part := range parts {
num, err := strconv.Atoi(part)
if err != nil {
log.Debug().Ctx(ctx).Msgf("failed to parse version %s: %v", part, err)
continue
}
result[i] = num
}
return result
}
func (c *localRegistry) UploadPackageFile(
ctx context.Context,
info pythontype.ArtifactInfo,
file multipart.File,
fileHeader *multipart.FileHeader,
filename string,
) (headers *commons.ResponseHeaders, sha256 string, err errcode.Error) {
path := info.Image + "/" + info.Metadata.Version + "/" + fileHeader.Filename
return c.localBase.Upload(ctx, info.ArtifactInfo, fileHeader.Filename, info.Metadata.Version, path, file,
defer file.Close()
path := pkg.JoinWithSeparator("/", info.Image, info.Metadata.Version, filename)
return c.localBase.UploadFile(ctx, info.ArtifactInfo, filename, info.Metadata.Version, path, file,
&pythonmetadata.PythonMetadata{
Metadata: info.Metadata,
})
}
func (c *localRegistry) UploadPackageFileReader(
ctx context.Context,
info pythontype.ArtifactInfo,
file io.ReadCloser,
filename string,
) (headers *commons.ResponseHeaders, sha256 string, err errcode.Error) {
defer file.Close()
path := pkg.JoinWithSeparator("/", info.Image, info.Metadata.Version, filename)
return c.localBase.Upload(ctx, info.ArtifactInfo, filename, info.Metadata.Version, path, file,
&pythonmetadata.PythonMetadata{
Metadata: info.Metadata,
})

73
registry/app/pkg/python/local_helper.go
View File

@ -1,25 +1,76 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package python
import (
"context"
"io"
"github.com/harness/gitness/registry/app/dist_temp/errcode"
"github.com/harness/gitness/registry/app/pkg/base"
"github.com/harness/gitness/registry/app/pkg/commons"
"github.com/harness/gitness/registry/app/pkg/types/python"
"github.com/harness/gitness/registry/app/storage"
)
type LocalRegistryHelper interface {
FileExists(ctx context.Context, info python.ArtifactInfo) bool
DownloadFile(ctx context.Context, info python.ArtifactInfo) (
*commons.ResponseHeaders,
*storage.FileReader,
string,
[]error,
)
UploadPackageFile(
ctx context.Context,
info python.ArtifactInfo,
fileReader io.ReadCloser,
filename string,
) (*commons.ResponseHeaders, string, errcode.Error)
}
type localRegistryHelper struct {
localRegistry LocalRegistry
localBase base.LocalBase
}
func NewLocalRegistryHelper() LocalRegistryHelper {
return &localRegistryHelper{}
func NewLocalRegistryHelper(localRegistry LocalRegistry, localBase base.LocalBase) LocalRegistryHelper {
return &localRegistryHelper{
localRegistry: localRegistry,
localBase: localBase,
}
}
func (h *localRegistryHelper) FileExists(ctx context.Context, info python.ArtifactInfo) bool {
return h.localBase.Exists(ctx, info.ArtifactInfo, info.Version, info.Filename)
}
func (h *localRegistryHelper) DownloadFile(ctx context.Context, info python.ArtifactInfo) (
*commons.ResponseHeaders,
*storage.FileReader,
string,
[]error,
) {
return h.localBase.Download(ctx, info.ArtifactInfo, info.Version, info.Filename)
}
func (h *localRegistryHelper) UploadPackageFile(
ctx context.Context,
info python.ArtifactInfo,
fileReader io.ReadCloser,
filename string,
) (*commons.ResponseHeaders, string, errcode.Error) {
return h.localRegistry.UploadPackageFileReader(ctx, info, fileReader, filename)
}

191
registry/app/pkg/python/proxy.go
View File

@ -20,6 +20,7 @@ import (
"io"
"mime/multipart"
"github.com/harness/gitness/app/services/refcache"
urlprovider "github.com/harness/gitness/app/url"
"github.com/harness/gitness/registry/app/api/openapi/contracts/artifact"
"github.com/harness/gitness/registry/app/dist_temp/errcode"
@ -27,26 +28,33 @@ import (
"github.com/harness/gitness/registry/app/pkg/commons"
"github.com/harness/gitness/registry/app/pkg/filemanager"
pythontype "github.com/harness/gitness/registry/app/pkg/types/python"
"github.com/harness/gitness/registry/app/remote/controller/proxy/python"
"github.com/harness/gitness/registry/app/remote/adapter/commons/pypi"
"github.com/harness/gitness/registry/app/storage"
"github.com/harness/gitness/registry/app/store"
cfg "github.com/harness/gitness/registry/config"
request2 "github.com/harness/gitness/registry/request"
"github.com/harness/gitness/secret"
"github.com/harness/gitness/store/database/dbtx"
"github.com/rs/zerolog/log"
_ "github.com/harness/gitness/registry/app/remote/adapter/pypi" // This is required to init pypi adapter
)
var _ pkg.Artifact = (*proxy)(nil)
var _ Registry = (*proxy)(nil)
type proxy struct {
fileManager filemanager.FileManager
proxyStore store.UpstreamProxyConfigRepository
tx dbtx.Transactor
registryDao store.RegistryRepository
imageDao store.ImageRepository
artifactDao store.ArtifactRepository
urlProvider urlprovider.Provider
proxyController python.Controller
fileManager filemanager.FileManager
proxyStore store.UpstreamProxyConfigRepository
tx dbtx.Transactor
registryDao store.RegistryRepository
imageDao store.ImageRepository
artifactDao store.ArtifactRepository
urlProvider urlprovider.Provider
spaceFinder refcache.SpaceFinder
service secret.Service
localRegistryHelper LocalRegistryHelper
}
type Proxy interface {
@ -61,15 +69,21 @@ func NewProxy(
imageDao store.ImageRepository,
artifactDao store.ArtifactRepository,
urlProvider urlprovider.Provider,
spaceFinder refcache.SpaceFinder,
service secret.Service,
localRegistryHelper LocalRegistryHelper,
) Proxy {
return &proxy{
proxyStore: proxyStore,
registryDao: registryDao,
imageDao: imageDao,
artifactDao: artifactDao,
fileManager: fileManager,
tx: tx,
urlProvider: urlProvider,
fileManager: fileManager,
proxyStore: proxyStore,
tx: tx,
registryDao: registryDao,
imageDao: imageDao,
artifactDao: artifactDao,
urlProvider: urlProvider,
spaceFinder: spaceFinder,
service: service,
localRegistryHelper: localRegistryHelper,
}
}
@ -84,53 +98,138 @@ func (r *proxy) GetPackageTypes() []artifact.PackageType {
func (r *proxy) DownloadPackageFile(ctx context.Context, info pythontype.ArtifactInfo) (
*commons.ResponseHeaders,
*storage.FileReader,
io.ReadCloser,
string,
[]error,
) {
headers, body, _, url, errs := r.fetchFile(ctx, info, true)
return headers, body, url, errs
upstreamProxy, err := r.proxyStore.GetByRegistryIdentifier(ctx, info.ParentID, info.RegIdentifier)
if err != nil {
return nil, nil, nil, "", []error{errcode.ErrCodeUnknown.WithDetail(err)}
}
// TODO: Extract out to Path Utils for all package types
exists := r.localRegistryHelper.FileExists(ctx, info)
if exists {
headers, fileReader, redirectURL, errors := r.localRegistryHelper.DownloadFile(ctx, info)
if len(errors) == 0 {
return headers, fileReader, nil, redirectURL, errors
}
// If file exists in local registry, but download failed, we should try to download from remote
log.Warn().Ctx(ctx).Msgf("failed to pull from local, attempting streaming from remote, %v", errors)
}
remote, err := NewRemoteRegistryHelper(ctx, r.spaceFinder, *upstreamProxy, r.service)
if err != nil {
return nil, nil, nil, "", []error{errcode.ErrCodeUnknown.WithDetail(err)}
}
file, err := remote.GetFile(ctx, info.Image, info.Filename)
if err != nil {
return nil, nil, nil, "", []error{errcode.ErrCodeUnknown.WithDetail(err)}
}
go func(info pythontype.ArtifactInfo) {
ctx2 := context.WithoutCancel(ctx)
ctx2 = context.WithValue(ctx2, cfg.GoRoutineKey, "goRoutine")
err = r.putFileToLocal(ctx2, info.Image, info.Filename, remote)
if err != nil {
log.Ctx(ctx2).Error().Stack().Err(err).Msgf("error while putting file to localRegistry, %v", err)
return
}
log.Ctx(ctx2).Info().Msgf("Successfully updated file: %s, registry: %s", info.Filename, info.RegIdentifier)
}(info)
return nil, nil, file, "", nil
}
// Metadata represents the metadata of a Python package.
// GetPackageMetadata Returns metadata from remote.
func (r *proxy) GetPackageMetadata(
_ context.Context,
_ pythontype.ArtifactInfo,
ctx context.Context,
info pythontype.ArtifactInfo,
) (pythontype.PackageMetadata, error) {
return pythontype.PackageMetadata{}, nil
upstreamProxy, err := r.proxyStore.GetByRegistryIdentifier(ctx, info.ParentID, info.RegIdentifier)
if err != nil {
return pythontype.PackageMetadata{}, err
}
helper, _ := NewRemoteRegistryHelper(ctx, r.spaceFinder, *upstreamProxy, r.service)
result, err := helper.GetMetadata(ctx, info.Image)
if err != nil {
return pythontype.PackageMetadata{}, err
}
var files []pythontype.File
for _, file := range result.Packages {
files = append(files, pythontype.File{
Name: file.Name,
FileURL: r.urlProvider.RegistryURL(ctx) + fmt.Sprintf(
"/pkg/%s/%s/python/files/%s/%s/%s",
info.RootIdentifier,
info.RegIdentifier,
info.Image,
file.Version(),
file.Name),
RequiresPython: file.RequiresPython(),
})
}
metadata := pythontype.PackageMetadata{
Name: info.Image,
Files: files,
}
sortPackageMetadata(ctx, metadata)
return metadata, nil
}
// UploadPackageFile FIXME: Extract this upload function for all types of packageTypes
func (r *proxy) putFileToLocal(ctx context.Context, pkg string, filename string, remote RemoteRegistryHelper) error {
version := pypi.GetPyPIVersion(filename)
metadata, err := remote.GetJSON(ctx, pkg, version)
if err != nil {
log.Ctx(ctx).Error().Stack().Err(err).Msgf("fetching metadata for %s failed, %v", filename, err)
return err
}
file, err := remote.GetFile(ctx, pkg, filename)
if err != nil {
log.Ctx(ctx).Error().Stack().Err(err).Msgf("fetching file %s failed, %v", filename, err)
return err
}
info, ok := request2.ArtifactInfoFrom(ctx).(*pythontype.ArtifactInfo)
if !ok {
log.Ctx(ctx).Error().Msgf("failed to cast artifact info to python artifact info")
return errcode.ErrCodeInvalidRequest.WithDetail(fmt.Errorf("failed to cast artifact info to python artifact info"))
}
info.Metadata = *metadata
info.Filename = filename
_, sha256, err2 := r.localRegistryHelper.UploadPackageFile(ctx, *info, file, filename)
if !commons.IsEmptyError(err2) {
log.Ctx(ctx).Error().Stack().Err(err2).Msgf("uploading file %s failed, %v", filename, err)
return err2
}
log.Info().Msgf("Successfully uploaded %s with SHA256: %s", filename, sha256)
return nil
}
// UploadPackageFile TODO: Extract this upload function for all types of packageTypes
// uploads the package file to the storage.
func (r *proxy) UploadPackageFile(
ctx context.Context,
_ pythontype.ArtifactInfo,
_ multipart.File,
_ *multipart.FileHeader,
_ string,
) (*commons.ResponseHeaders, string, errcode.Error) {
log.Error().Ctx(ctx).Msg("Not implemented")
return nil, "", errcode.ErrCodeInvalidRequest.WithDetail(fmt.Errorf("not implemented"))
}
func (r *proxy) fetchFile(ctx context.Context, info pythontype.ArtifactInfo, serveFile bool) (
responseHeaders *commons.ResponseHeaders, body *storage.FileReader, readCloser io.ReadCloser,
redirectURL string, errs []error,
) {
log.Ctx(ctx).Info().Msgf("Maven Proxy: %s", info.RegIdentifier)
responseHeaders, body, redirectURL, useLocal := r.proxyController.UseLocalFile(ctx, info)
if useLocal {
return responseHeaders, body, readCloser, redirectURL, errs
}
upstreamProxy, err := r.proxyStore.GetByRegistryIdentifier(ctx, info.ParentID, info.RegIdentifier)
if err != nil {
return responseHeaders, nil, nil, "", []error{errcode.ErrCodeUnknown.WithDetail(err)}
}
// This is start of proxy Code.
responseHeaders, readCloser, err = r.proxyController.ProxyFile(ctx, info, *upstreamProxy, serveFile)
if err != nil {
return responseHeaders, nil, nil, "", []error{errcode.ErrCodeUnknown.WithDetail(err)}
}
return responseHeaders, nil, readCloser, "", errs
// UploadPackageFile TODO: Extract this upload function for all types of packageTypes
// uploads the package file to the storage.
func (r *proxy) UploadPackageFileReader(
ctx context.Context,
_ pythontype.ArtifactInfo,
_ io.ReadCloser,
_ string,
) (*commons.ResponseHeaders, string, errcode.Error) {
log.Error().Ctx(ctx).Msg("Not implemented")
return nil, "", errcode.ErrCodeInvalidRequest.WithDetail(fmt.Errorf("not implemented"))
}

29
registry/app/pkg/python/registry.go
View File

@ -1,21 +1,22 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package python
import (
"context"
"io"
"mime/multipart"
"github.com/harness/gitness/registry/app/dist_temp/errcode"
@ -34,12 +35,20 @@ type Registry interface {
ctx context.Context,
info python.ArtifactInfo,
file multipart.File,
fileHeader *multipart.FileHeader,
filename string,
) (*commons.ResponseHeaders, string, errcode.Error)
UploadPackageFileReader(
ctx context.Context,
info python.ArtifactInfo,
file io.ReadCloser,
filename string,
) (*commons.ResponseHeaders, string, errcode.Error)
DownloadPackageFile(ctx context.Context, info python.ArtifactInfo) (
*commons.ResponseHeaders,
*storage.FileReader,
io.ReadCloser,
string,
[]error,
)

118
registry/app/pkg/python/remote_helper.go
View File

@ -1,25 +1,121 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package python
import (
"context"
"io"
"github.com/harness/gitness/app/services/refcache"
"github.com/harness/gitness/registry/app/api/openapi/contracts/artifact"
"github.com/harness/gitness/registry/app/metadata/python"
"github.com/harness/gitness/registry/app/remote/adapter"
pypi2 "github.com/harness/gitness/registry/app/remote/adapter/commons/pypi"
"github.com/harness/gitness/registry/app/remote/adapter/pypi"
"github.com/harness/gitness/registry/app/remote/registry"
"github.com/harness/gitness/registry/types"
"github.com/harness/gitness/secret"
"github.com/rs/zerolog/log"
)
type RemoteRegistryHelper interface {
// GetFile Downloads the file for the given package and filename
GetFile(ctx context.Context, pkg string, filename string) (io.ReadCloser, error)
// GetMetadata Fetches the metadata for the given package for all versions
GetMetadata(ctx context.Context, pkg string) (*pypi2.SimpleMetadata, error)
// GetJSON Fetches the metadata for the given package and specific version
GetJSON(ctx context.Context, pkg string, version string) (*python.Metadata, error)
}
type remoteRegistryHelper struct {
adapter registry.PythonRegistry
registry types.UpstreamProxy
}
func NewRemoteRegistryHelper() RemoteRegistryHelper {
return &remoteRegistryHelper{}
func NewRemoteRegistryHelper(
ctx context.Context,
spaceFinder refcache.SpaceFinder,
registry types.UpstreamProxy,
service secret.Service,
) (RemoteRegistryHelper, error) {
r := &remoteRegistryHelper{
registry: registry,
}
if err := r.init(ctx, spaceFinder, service); err != nil {
log.Ctx(ctx).Error().Err(err).Msgf("failed to init remote registry for remote: %s", registry.RepoKey)
return nil, err
}
return r, nil
}
func (r *remoteRegistryHelper) init(
ctx context.Context,
spaceFinder refcache.SpaceFinder,
service secret.Service,
) error {
key := string(artifact.UpstreamConfigSourcePyPi)
if r.registry.Source == string(artifact.UpstreamConfigSourcePyPi) {
r.registry.RepoURL = pypi.PyPiURL
}
factory, err := adapter.GetFactory(key)
if err != nil {
log.Ctx(ctx).Error().Err(err).Msg("failed to get factory " + key)
return err
}
adpt, err := factory.Create(ctx, spaceFinder, r.registry, service)
if err != nil {
log.Ctx(ctx).Error().Err(err).Msg("failed to create factory " + key)
return err
}
pythonReg, ok := adpt.(registry.PythonRegistry)
if !ok {
log.Ctx(ctx).Error().Msg("failed to cast factory to python registry")
return err
}
r.adapter = pythonReg
return nil
}
func (r *remoteRegistryHelper) GetFile(ctx context.Context, pkg string, filename string) (io.ReadCloser, error) {
v2, err := r.adapter.GetPackage(ctx, pkg, filename)
if err != nil {
log.Ctx(ctx).Error().Err(err).Msgf("failed to get pkg: %s, file: %s", pkg, filename)
}
return v2, err
}
func (r *remoteRegistryHelper) GetMetadata(ctx context.Context, pkg string) (*pypi2.SimpleMetadata, error) {
packages, err := r.adapter.GetMetadata(ctx, pkg)
if err != nil {
log.Ctx(ctx).Error().Err(err).Msgf("failed to get metadata for pkg: %s", pkg)
return nil, err
}
return packages, nil
}
func (r *remoteRegistryHelper) GetJSON(ctx context.Context, pkg string, version string) (*python.Metadata, error) {
metadata, err := r.adapter.GetJSON(ctx, pkg, version)
if err != nil {
log.Ctx(ctx).Error().Err(err).Msgf("failed to get JSON for pkg: %s, version: %s", pkg, version)
return nil, err
}
return metadata, nil
}

14
registry/app/pkg/python/wire.go
View File

@ -15,10 +15,12 @@
package python
import (
"github.com/harness/gitness/app/services/refcache"
urlprovider "github.com/harness/gitness/app/url"
"github.com/harness/gitness/registry/app/pkg/base"
"github.com/harness/gitness/registry/app/pkg/filemanager"
"github.com/harness/gitness/registry/app/store"
"github.com/harness/gitness/secret"
"github.com/harness/gitness/store/database/dbtx"
"github.com/google/wire"
@ -48,10 +50,18 @@ func ProxyProvider(
fileManager filemanager.FileManager,
tx dbtx.Transactor,
urlProvider urlprovider.Provider,
spaceFinder refcache.SpaceFinder,
service secret.Service,
localRegistryHelper LocalRegistryHelper,
) Proxy {
proxy := NewProxy(fileManager, proxyStore, tx, registryDao, imageDao, artifactDao, urlProvider)
proxy := NewProxy(fileManager, proxyStore, tx, registryDao, imageDao, artifactDao, urlProvider,
spaceFinder, service, localRegistryHelper)
base.Register(proxy)
return proxy
}
var WireSet = wire.NewSet(LocalRegistryProvider, ProxyProvider)
func LocalRegistryHelperProvider(localRegistry LocalRegistry, localBase base.LocalBase) LocalRegistryHelper {
return NewLocalRegistryHelper(localRegistry, localBase)
}
var WireSet = wire.NewSet(LocalRegistryProvider, ProxyProvider, LocalRegistryHelperProvider)

22
registry/app/pkg/types/python/types.go
View File

@ -1,16 +1,16 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package python
@ -19,6 +19,8 @@ import (
"github.com/harness/gitness/registry/app/pkg"
)
// Metadata represents the metadata for a Python package.
type ArtifactInfo struct {
pkg.ArtifactInfo
Version string
@ -26,7 +28,7 @@ type ArtifactInfo struct {
Metadata python.Metadata
}
// BaseArtifactInfo implements pkg.PackageArtifactInfo interface.
// BaseArtifactInfo implements pkg.PackageArtifactInfo interface
func (a ArtifactInfo) BaseArtifactInfo() pkg.ArtifactInfo {
return a.ArtifactInfo
}

5
registry/app/pkg/utils.go
View File

@ -16,6 +16,7 @@ package pkg
import (
"reflect"
"strings"
)
func IsEmpty(slice interface{}) bool {
@ -24,3 +25,7 @@ func IsEmpty(slice interface{}) bool {
}
return reflect.ValueOf(slice).Len() == 0
}
func JoinWithSeparator(sep string, args ...string) string {
return strings.Join(args, sep)
}

10
registry/app/remote/adapter/awsecr/auth.go
View File

@ -167,16 +167,18 @@ func getCreds(
return accessKey, secretKey, false, nil
}
func getSecretValue(ctx context.Context, spaceFinder refcache.SpaceFinder, secretService secret.Service,
secretSpaceID int64, secretSpacePath string) (string, error) {
func getSecretValue(
ctx context.Context, spaceFinder refcache.SpaceFinder, secretService secret.Service,
secretSpaceID int64, secretSpacePath string,
) (string, error) {
spacePath, err := spaceFinder.FindByID(ctx, secretSpaceID)
if err != nil {
log.Error().Msgf("failed to find space path: %v", err)
log.Error().Msgf("failed to find space path: %d, %v", secretSpaceID, err)
return "", err
}
decryptSecret, err := secretService.DecryptSecret(ctx, spacePath.Path, secretSpacePath)
if err != nil {
log.Error().Msgf("failed to decrypt secret: %v", err)
log.Error().Msgf("failed to decrypt secret at path: %s, secret: %s, %v", spacePath.Path, secretSpacePath, err)
return "", err
}
return decryptSecret, nil

109
registry/app/remote/adapter/commons/pypi/dto.go Normal file
View File

@ -0,0 +1,109 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pypi
import (
"errors"
"fmt"
"strings"
"golang.org/x/net/html"
)
var (
extensions = []string{
".tar.gz",
".tar.bz2",
".tar.xz",
".zip",
".whl",
".egg",
".exe",
".app",
".dmg",
}
)
type SimpleMetadata struct {
Title string
MetaName string
Content string
Packages []Package
}
type Package struct {
Name string
ATags map[string]string
}
// URL returns the "href" attribute from the package's map.
func (p Package) URL() string {
return p.ATags["href"]
}
func (p Package) Valid() bool {
return p.URL() != "" && p.Name != ""
}
// RequiresPython returns the "data-requires-python" attribute (unescaped) from the package's map.
func (p Package) RequiresPython() string {
val := p.ATags["data-requires-python"]
// unescape HTML entities like "&gt;"
return html.UnescapeString(val)
}
// Version Fetches version from format:
// The wheel filename is {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}.whl
// SRC: https://packaging.python.org/en/latest/specifications/binary-distribution-format/#file-name-convention
func (p Package) Version() string {
return GetPyPIVersion(p.Name)
}
func (p Package) String() string {
return fmt.Sprintf("Name: %s, Version: %s, URL: %s, RequiresPython: %s", p.Name, p.Version(), p.URL(),
p.RequiresPython())
}
func GetPyPIVersion(filename string) string {
base, ext, err := stripRecognizedExtension(filename)
if err != nil {
return ""
}
splits := strings.Split(base, "-")
if len(splits) < 2 {
return ""
}
switch ext {
case ".whl", ".egg":
return splits[1]
case ".tar.gz", ".tar.bz2", ".tar.xz", ".zip", ".dmg", ".app", ".exe":
return splits[len(splits)-1]
default:
return ""
}
}
func stripRecognizedExtension(filename string) (string, string, error) {
for _, x := range extensions {
if strings.HasSuffix(strings.ToLower(filename), x) {
base := filename[:len(filename)-len(x)]
return base, x, nil
}
}
return "", "", errors.New("unrecognized file extension")
}

78
registry/app/remote/adapter/commons/utils.go Normal file
View File

@ -0,0 +1,78 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package commons
import (
"context"
"fmt"
"github.com/harness/gitness/app/services/refcache"
api "github.com/harness/gitness/registry/app/api/openapi/contracts/artifact"
"github.com/harness/gitness/registry/types"
"github.com/harness/gitness/secret"
"github.com/rs/zerolog/log"
)
func GetCredentials(
ctx context.Context, spaceFinder refcache.SpaceFinder, secretService secret.Service, reg types.UpstreamProxy,
) (accessKey string, secretKey string, isAnonymous bool, err error) {
if api.AuthType(reg.RepoAuthType) == api.AuthTypeAnonymous {
return "", "", true, nil
}
if api.AuthType(reg.RepoAuthType) == api.AuthTypeUserPassword {
secretKey, err = getSecretValue(ctx, spaceFinder, secretService, reg.SecretSpaceID,
reg.SecretIdentifier)
if err != nil {
log.Error().Err(err).Msgf("failed to get secret for registry: %s", reg.RepoKey)
return "", "", false, fmt.Errorf("failed to get secret for registry: %s", reg.RepoKey)
}
return reg.UserName, secretKey, false, nil
}
if api.AuthType(reg.RepoAuthType) == api.AuthTypeAccessKeySecretKey {
accessKey, err = getSecretValue(ctx, spaceFinder, secretService, reg.UserNameSecretSpaceID,
reg.UserNameSecretIdentifier)
if err != nil {
log.Error().Err(err).Msgf("failed to get access secret for registry: %s", reg.RepoKey)
return "", "", false, fmt.Errorf("failed to get access key for registry: %s", reg.RepoKey)
}
secretKey, err = getSecretValue(ctx, spaceFinder, secretService, reg.SecretSpaceID,
reg.SecretIdentifier)
if err != nil {
log.Error().Err(err).Msgf("failed to get user secret for registry: %s", reg.RepoKey)
return "", "", false, fmt.Errorf("failed to get secret key for registry: %s", reg.RepoKey)
}
return accessKey, secretKey, false, nil
}
return "", "", false, fmt.Errorf("unsupported auth type: %s", reg.RepoAuthType)
}
func getSecretValue(
ctx context.Context, spaceFinder refcache.SpaceFinder, secretService secret.Service,
secretSpaceID int64, secretSpacePath string,
) (string, error) {
spacePath, err := spaceFinder.FindByID(ctx, secretSpaceID)
if err != nil {
log.Error().Msgf("failed to find space path: %v", err)
return "", err
}
decryptSecret, err := secretService.DecryptSecret(ctx, spacePath.Path, secretSpacePath)
if err != nil {
log.Error().Msgf("failed to decrypt secret: %v", err)
return "", err
}
return decryptSecret, nil
}

38
registry/app/remote/adapter/native/adapter.go
View File

@ -20,10 +20,10 @@ import (
"context"
"github.com/harness/gitness/app/services/refcache"
api "github.com/harness/gitness/registry/app/api/openapi/contracts/artifact"
"github.com/harness/gitness/registry/app/common/lib"
"github.com/harness/gitness/registry/app/common/lib/errors"
adp "github.com/harness/gitness/registry/app/remote/adapter"
"github.com/harness/gitness/registry/app/remote/adapter/commons"
"github.com/harness/gitness/registry/app/remote/clients/registry"
"github.com/harness/gitness/registry/types"
"github.com/harness/gitness/secret"
@ -52,10 +52,15 @@ func NewAdapter(
adapter := &Adapter{
proxy: reg,
}
// Get the password: lookup secrets.secret_data using secret_identifier & secret_space_id.
password := getPwd(ctx, spaceFinder, service, reg)
username, password, url := reg.UserName, password, reg.RepoURL
adapter.Client = registry.NewClient(url, username, password, false)
url := reg.RepoURL
accessKey, secretKey, _, err := commons.GetCredentials(ctx, spaceFinder, service, reg)
if err != nil {
log.Error().Err(err).Msgf("error getting credentials for registry: %s", reg.RepoKey)
return nil
}
adapter.Client = registry.NewClient(url, accessKey, secretKey, false)
return adapter
}
@ -67,29 +72,6 @@ func NewAdapterWithAuthorizer(reg types.UpstreamProxy, authorizer lib.Authorizer
}
}
// getPwd: lookup secrets.secret_data using secret_identifier & secret_space_id.
func getPwd(
ctx context.Context, spaceFinder refcache.SpaceFinder, secretService secret.Service, reg types.UpstreamProxy,
) string {
if api.AuthType(reg.RepoAuthType) == api.AuthTypeUserPassword {
secretSpaceID := reg.SecretSpaceID
secretIdentifier := reg.SecretIdentifier
spacePath, err := spaceFinder.FindByID(ctx, secretSpaceID)
if err != nil {
log.Error().Msgf("failed to find space path: %v", err)
return ""
}
decryptSecret, err := secretService.DecryptSecret(ctx, spacePath.Path, secretIdentifier)
if err != nil {
log.Error().Msgf("failed to decrypt secret: %v", err)
return ""
}
return decryptSecret
}
return ""
}
// HealthCheck checks health status of a proxy.
func (a *Adapter) HealthCheck() (string, error) {
return "Not implemented", nil

231
registry/app/remote/adapter/pypi/adapter.go Normal file
View File

@ -0,0 +1,231 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pypi
import (
"context"
"encoding/json"
"fmt"
"io"
"github.com/harness/gitness/app/services/refcache"
"github.com/harness/gitness/registry/app/api/openapi/contracts/artifact"
"github.com/harness/gitness/registry/app/metadata/python"
adp "github.com/harness/gitness/registry/app/remote/adapter"
"github.com/harness/gitness/registry/app/remote/adapter/commons/pypi"
"github.com/harness/gitness/registry/app/remote/adapter/native"
"github.com/harness/gitness/registry/app/remote/registry"
"github.com/harness/gitness/registry/types"
"github.com/harness/gitness/secret"
"github.com/rs/zerolog/log"
"golang.org/x/net/html"
)
var _ registry.PythonRegistry = (*adapter)(nil)
var _ adp.Adapter = (*adapter)(nil)
const (
PyPiURL = "https://pypi.org"
)
type adapter struct {
*native.Adapter
registry types.UpstreamProxy
client *client
}
func newAdapter(
ctx context.Context,
spaceFinder refcache.SpaceFinder,
registry types.UpstreamProxy,
service secret.Service,
) (adp.Adapter, error) {
nativeAdapter := native.NewAdapter(ctx, spaceFinder, service, registry)
c, err := newClient(ctx, registry, spaceFinder, service)
if err != nil {
return nil, err
}
return &adapter{
Adapter: nativeAdapter,
registry: registry,
client: c,
}, nil
}
type factory struct {
}
func (f *factory) Create(
ctx context.Context, spaceFinder refcache.SpaceFinder, record types.UpstreamProxy, service secret.Service,
) (adp.Adapter, error) {
return newAdapter(ctx, spaceFinder, record, service)
}
func init() {
adapterType := string(artifact.UpstreamConfigSourcePyPi)
if err := adp.RegisterFactory(adapterType, new(factory)); err != nil {
log.Error().Stack().Err(err).Msgf("Failed to register adapter factory for %s", adapterType)
return
}
log.Info().Stack().Msgf("Registered adapter factory for %s", adapterType)
}
func (a *adapter) GetMetadata(_ context.Context, pkg string) (*pypi.SimpleMetadata, error) {
_, readCloser, err := a.GetFile("simple/" + pkg)
if err != nil {
return nil, err
}
defer readCloser.Close()
response, err := ParsePyPISimple(readCloser)
if err != nil {
return nil, err
}
err = validateMetadata(response)
if err != nil {
return nil, err
}
return &response, nil
}
func validateMetadata(response pypi.SimpleMetadata) error {
for _, p := range response.Packages {
if !p.Valid() {
log.Error().Msgf("invalid package: %s", p.String())
return fmt.Errorf("invalid package: %s", p.String())
}
}
return nil
}
func (a *adapter) GetPackage(ctx context.Context, pkg string, filename string) (io.ReadCloser, error) {
metadata, err := a.GetMetadata(ctx, pkg)
if err != nil {
return nil, err
}
downloadURL := ""
for _, p := range metadata.Packages {
if p.Name == filename {
downloadURL = p.URL()
break
}
}
if downloadURL == "" {
return nil, fmt.Errorf("pkg: %s, filename: %s not found", pkg, filename)
}
log.Ctx(ctx).Info().Msgf("Download URL: %s", downloadURL)
_, closer, err := a.GetFileFromURL(downloadURL)
if err != nil {
log.Ctx(ctx).Error().Err(err).Msgf("Failed to get file from URL: %s", downloadURL)
return nil, err
}
return closer, nil
}
func (a *adapter) GetJSON(ctx context.Context, pkg string, version string) (*python.Metadata, error) {
_, readCloser, err := a.GetFile(fmt.Sprintf("pypi/%s/%s/json", pkg, version))
if err != nil {
return nil, err
}
defer readCloser.Close()
response, err := ParseMetadata(ctx, readCloser)
if err != nil {
return nil, err
}
return &response, nil
}
func ParseMetadata(ctx context.Context, body io.ReadCloser) (python.Metadata, error) {
bytes, err := io.ReadAll(body)
if err != nil {
return python.Metadata{}, err
}
var response Response
if err := json.Unmarshal(bytes, &response); err != nil {
// FIXME: This is known problem where if the response fields returns null, the null is not handled.
// For eg: {"keywords":null} is not handled where "keywords" is []string
log.Ctx(ctx).Warn().Err(err).Msgf("Failed to unmarshal response")
}
return response.Info, nil
}
// ParsePyPISimple parses the given HTML and returns a SimpleMetadata DTO.
func ParsePyPISimple(r io.ReadCloser) (pypi.SimpleMetadata, error) {
doc, err := html.Parse(r)
if err != nil {
return pypi.SimpleMetadata{}, err
}
var result pypi.SimpleMetadata
var packages []pypi.Package
// Recursive function to walk the HTML nodes
var traverse func(*html.Node)
traverse = func(n *html.Node) {
if n.Type == html.ElementNode {
switch n.Data {
case "meta":
// Check for meta tag name="pypi:repository-version"
var metaName, metaContent string
for _, attr := range n.Attr {
switch attr.Key {
case "name":
metaName = attr.Val
case "content":
metaContent = attr.Val
}
}
if metaName == "pypi:repository-version" {
result.MetaName = metaName
result.Content = metaContent
}
case "title":
if n.FirstChild != nil {
result.Title = n.FirstChild.Data
}
case "a":
// Capture all attributes in a map
aMap := make(map[string]string)
for _, attr := range n.Attr {
aMap[attr.Key] = attr.Val
}
linkText := ""
if n.FirstChild != nil {
linkText = n.FirstChild.Data
}
packages = append(packages, pypi.Package{
ATags: aMap,
Name: linkText,
})
}
}
for c := n.FirstChild; c != nil; c = c.NextSibling {
traverse(c)
}
}
traverse(doc)
result.Packages = packages
return result, nil
}

60
registry/app/remote/adapter/pypi/client.go Normal file
View File

@ -0,0 +1,60 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pypi
import (
"context"
"net/http"
"github.com/harness/gitness/app/services/refcache"
commonhttp "github.com/harness/gitness/registry/app/common/http"
"github.com/harness/gitness/registry/app/remote/adapter/commons"
"github.com/harness/gitness/registry/types"
"github.com/harness/gitness/secret"
"github.com/rs/zerolog/log"
)
type client struct {
client *http.Client
url string
username string
password string
}
// newClient creates a new PyPi client.
func newClient(
ctx context.Context,
registry types.UpstreamProxy,
finder refcache.SpaceFinder,
service secret.Service,
) (*client, error) {
accessKey, secretKey, _, err := commons.GetCredentials(ctx, finder, service, registry)
if err != nil {
log.Ctx(ctx).Err(err).Msgf("error getting credentials for registry: %s %v", registry.RepoKey, err)
return nil, err
}
c := &client{
url: registry.RepoURL,
client: &http.Client{
Transport: commonhttp.GetHTTPTransport(commonhttp.WithInsecure(true)),
},
username: accessKey,
password: secretKey,
}
return c, nil
}

21
registry/app/remote/adapter/pypi/dto.go Normal file
View File

@ -0,0 +1,21 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pypi
import "github.com/harness/gitness/registry/app/metadata/python"
type Response struct {
Info python.Metadata `json:"info"`
}

27
registry/app/remote/clients/registry/client.go
View File

@ -48,15 +48,6 @@ import (
)
var (
// Cli is the global registry client instance, it targets to the backend docker registry.
Cli = func() Client {
url := "myurl"
username, password := "myusername", "mypassword"
// url, _ := config.RegistryURL()
// username, password := config.RegistryCredential()
return NewClient(url, username, password, false)
}()
accepts = []string{
v1.MediaTypeImageIndex,
manifestlist.MediaTypeManifestList,
@ -158,6 +149,9 @@ type Client interface {
// HeadFile Check existence of file
HeadFile(filePath string) (*commons.ResponseHeaders, bool, error)
// GetFileFromURL Download the file from URL instead of provided endpoint. Authorizer still remains the same.
GetFileFromURL(url string) (*commons.ResponseHeaders, io.ReadCloser, error)
}
// NewClient creates a registry client with the default authorizer which determines the auth scheme
@ -847,6 +841,21 @@ func (c *client) GetFile(filePath string) (*commons.ResponseHeaders, io.ReadClos
return responseHeaders, resp.Body, nil
}
func (c *client) GetFileFromURL(url string) (*commons.ResponseHeaders, io.ReadCloser, error) {
req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet,
url, nil)
if err != nil {
return nil, nil, err
}
resp, err := c.Do(req)
if err != nil {
return nil, nil, err
}
responseHeaders := utils.ParseResponseHeaders(resp)
return responseHeaders, resp.Body, nil
}
func (c *client) HeadFile(filePath string) (*commons.ResponseHeaders, bool, error) {
req, err := http.NewRequestWithContext(context.TODO(), http.MethodHead,
buildFileURL(c.url, filePath), nil)

143
registry/app/remote/controller/proxy/python/controller.go
View File

@ -1,143 +0,0 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package python
import (
"context"
"io"
"strings"
"github.com/harness/gitness/app/api/request"
"github.com/harness/gitness/app/services/refcache"
"github.com/harness/gitness/registry/app/pkg/commons"
"github.com/harness/gitness/registry/app/pkg/types/python"
"github.com/harness/gitness/registry/app/storage"
cfg "github.com/harness/gitness/registry/config"
"github.com/harness/gitness/registry/types"
"github.com/harness/gitness/secret"
"github.com/rs/zerolog/log"
)
type controller struct {
localRegistry registryInterface
secretService secret.Service
spaceFinder refcache.SpaceFinder
}
type Controller interface {
UseLocalFile(ctx context.Context, info python.ArtifactInfo) (
responseHeaders *commons.ResponseHeaders, fileReader *storage.FileReader, redirectURL string, useLocal bool,
)
ProxyFile(
ctx context.Context, info python.ArtifactInfo, proxy types.UpstreamProxy, serveFile bool,
) (*commons.ResponseHeaders, io.ReadCloser, error)
}
// NewProxyController -- get the proxy controller instance.
func NewProxyController(
l registryInterface, secretService secret.Service,
spaceFinder refcache.SpaceFinder,
) Controller {
return &controller{
localRegistry: l,
secretService: secretService,
spaceFinder: spaceFinder,
}
}
func (c *controller) UseLocalFile(ctx context.Context, info python.ArtifactInfo) (
responseHeaders *commons.ResponseHeaders, fileReader *storage.FileReader, redirectURL string, useLocal bool,
) {
responseHeaders, body, _, redirectURL, e := c.localRegistry.GetArtifact(ctx, info)
return responseHeaders, body, redirectURL, len(e) == 0
}
func (c *controller) ProxyFile(
ctx context.Context, info python.ArtifactInfo, proxy types.UpstreamProxy, serveFile bool,
) (responseHeaders *commons.ResponseHeaders, body io.ReadCloser, errs error) {
responseHeaders = &commons.ResponseHeaders{
Headers: make(map[string]string),
}
rHelper, err := NewRemoteHelper(ctx, c.spaceFinder, c.secretService, proxy)
if err != nil {
return responseHeaders, nil, err
}
filePath := ""
// FIXME:URGENT:
//filePath := utils.GetFilePath(info)
filePath = strings.Trim(filePath, "/")
if serveFile {
responseHeaders, body, err = rHelper.GetFile(filePath)
} else {
responseHeaders, _, err = rHelper.HeadFile(filePath)
}
if err != nil {
return responseHeaders, nil, err
}
if !serveFile {
return responseHeaders, nil, nil
}
go func(info python.ArtifactInfo) {
// Cloning Context.
session, ok := request.AuthSessionFrom(ctx)
if !ok {
log.Error().Stack().Err(err).Msg("failed to get auth session from context")
return
}
ctx2 := request.WithAuthSession(ctx, session)
ctx2 = context.WithoutCancel(ctx2)
ctx2 = context.WithValue(ctx2, cfg.GoRoutineKey, "goRoutine")
err = c.putFileToLocal(ctx2, info, rHelper)
if err != nil {
log.Ctx(ctx2).Error().Stack().Err(err).Msgf("error while putting file to localRegistry, %v", err)
return
}
log.Ctx(ctx2).Info().Msgf("Successfully updated file "+
"to registry: %s with file path: %s",
info.RegIdentifier, filePath)
}(info)
return responseHeaders, body, nil
}
func (c *controller) putFileToLocal(
ctx context.Context,
info python.ArtifactInfo,
r RemoteInterface,
) error {
filePath := ""
// FIXME:URGENT:
//filePath := utils.GetFilePath(info)
filePath = strings.Trim(filePath, "/")
_, fileReader, err := r.GetFile(filePath)
if err != nil {
return err
}
defer fileReader.Close()
_, errs := c.localRegistry.PutArtifact(ctx, info, fileReader)
if len(errs) > 0 {
return errs[0]
}
return err
}

39
registry/app/remote/controller/proxy/python/local.go
View File

@ -1,39 +0,0 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package python
import (
"context"
"io"
"github.com/harness/gitness/registry/app/pkg/commons"
"github.com/harness/gitness/registry/app/pkg/types/python"
"github.com/harness/gitness/registry/app/storage"
)
type registryInterface interface {
HeadArtifact(ctx context.Context, info python.ArtifactInfo) (
responseHeaders *commons.ResponseHeaders, errs []error,
)
GetArtifact(ctx context.Context, info python.ArtifactInfo) (
responseHeaders *commons.ResponseHeaders, body *storage.FileReader, fileReader io.ReadCloser,
redirectURL string, errs []error,
)
PutArtifact(ctx context.Context, info python.ArtifactInfo, fileReader io.Reader) (
responseHeaders *commons.ResponseHeaders, errs []error,
)
}

96
registry/app/remote/controller/proxy/python/remote.go
View File

@ -1,96 +0,0 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package python
import (
"context"
"io"
"github.com/harness/gitness/app/services/refcache"
api "github.com/harness/gitness/registry/app/api/openapi/contracts/artifact"
"github.com/harness/gitness/registry/app/pkg/commons"
"github.com/harness/gitness/registry/app/remote/adapter"
"github.com/harness/gitness/registry/types"
"github.com/harness/gitness/secret"
"github.com/rs/zerolog/log"
_ "github.com/harness/gitness/registry/app/remote/adapter/maven" // This is required to init maven adapter
)
const MavenCentralURL = "https://repo1.maven.org/maven2"
// RemoteInterface defines operations related to remote repository under proxy.
type RemoteInterface interface {
// Download the file
GetFile(filePath string) (*commons.ResponseHeaders, io.ReadCloser, error)
// Check existence of file
HeadFile(filePath string) (*commons.ResponseHeaders, bool, error)
}
type remoteHelper struct {
registry adapter.ArtifactRegistry
upstreamProxy types.UpstreamProxy
URL string
secretService secret.Service
}
// NewRemoteHelper create a remote interface.
func NewRemoteHelper(
ctx context.Context, spaceFinder refcache.SpaceFinder, secretService secret.Service,
proxy types.UpstreamProxy,
) (RemoteInterface, error) {
if proxy.Source == string(api.UpstreamConfigSourceMavenCentral) {
proxy.RepoURL = MavenCentralURL
}
r := &remoteHelper{
upstreamProxy: proxy,
secretService: secretService,
}
if err := r.init(ctx, spaceFinder, string(api.UpstreamConfigSourceMavenCentral)); err != nil {
return nil, err
}
return r, nil
}
func (r *remoteHelper) init(ctx context.Context, spaceFinder refcache.SpaceFinder, proxyType string) error {
if r.registry != nil {
return nil
}
factory, err := adapter.GetFactory(proxyType)
if err != nil {
return err
}
adp, err := factory.Create(ctx, spaceFinder, r.upstreamProxy, r.secretService)
if err != nil {
return err
}
reg, ok := adp.(adapter.ArtifactRegistry)
if !ok {
log.Warn().Msgf("Error: adp is not of type adapter.ArtifactRegistry")
}
r.registry = reg
return nil
}
func (r *remoteHelper) GetFile(filePath string) (*commons.ResponseHeaders, io.ReadCloser, error) {
return r.registry.GetFile(filePath)
}
func (r *remoteHelper) HeadFile(filePath string) (*commons.ResponseHeaders, bool, error) {
return r.registry.HeadFile(filePath)
}

29
registry/app/remote/registry/python.go Normal file
View File

@ -0,0 +1,29 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package registry
import (
"context"
"io"
"github.com/harness/gitness/registry/app/metadata/python"
"github.com/harness/gitness/registry/app/remote/adapter/commons/pypi"
)
type PythonRegistry interface {
GetMetadata(ctx context.Context, pkg string) (*pypi.SimpleMetadata, error)
GetPackage(ctx context.Context, pkg string, filename string) (io.ReadCloser, error)
GetJSON(ctx context.Context, pkg string, version string) (*python.Metadata, error)
}

14
registry/app/storage/blobStore.go
View File

@ -27,7 +27,7 @@ import (
"github.com/harness/gitness/registry/app/dist_temp/dcontext"
"github.com/harness/gitness/registry/app/driver"
"github.com/harness/gitness/registry/app/pkg"
"github.com/harness/gitness/registry/types"
"github.com/rs/zerolog/log"
)
@ -96,8 +96,10 @@ func (bs *genericBlobStore) newBlobUpload(
// Write takes a file writer and a multipart form file or file reader,
// streams the file to the writer, and calculates hashes.
func (bs *genericBlobStore) Write(ctx context.Context, w driver.FileWriter, file multipart.File,
fileReader io.Reader) (pkg.FileInfo, error) {
func (bs *genericBlobStore) Write(
ctx context.Context, w driver.FileWriter, file multipart.File,
fileReader io.Reader,
) (types.FileInfo, error) {
// Create new hash.Hash instances for SHA256 and SHA512
sha1Hasher := sha1.New()
sha256Hasher := sha256.New()
@ -115,15 +117,15 @@ func (bs *genericBlobStore) Write(ctx context.Context, w driver.FileWriter, file
totalBytesWritten, err = io.Copy(mw, file)
}
if err != nil {
return pkg.FileInfo{}, fmt.Errorf("failed to copy file to s3: %w", err)
return types.FileInfo{}, fmt.Errorf("failed to copy file to s3: %w", err)
}
err = w.Commit(ctx)
if err != nil {
return pkg.FileInfo{}, err
return types.FileInfo{}, err
}
return pkg.FileInfo{
return types.FileInfo{
Sha1: fmt.Sprintf("%x", sha1Hasher.Sum(nil)),
Sha256: fmt.Sprintf("%x", sha256Hasher.Sum(nil)),
Sha512: fmt.Sprintf("%x", sha512Hasher.Sum(nil)),

5
registry/app/storage/blobs.go
View File

@ -25,7 +25,7 @@ import (
"github.com/harness/gitness/registry/app/driver"
"github.com/harness/gitness/registry/app/manifest"
"github.com/harness/gitness/registry/app/pkg"
"github.com/harness/gitness/registry/types"
"github.com/distribution/reference"
"github.com/opencontainers/go-digest"
@ -173,7 +173,8 @@ type GenericBlobStore interface {
// multiple times until the BlobWriter is committed or cancelled.
Create(ctx context.Context, filePath string) (driver.FileWriter, error)
Write(ctx context.Context, w driver.FileWriter, file multipart.File, fileReader io.Reader) (pkg.FileInfo, error)
// Write writes the file to the blob store. There are two ways to write the file and fileReader takes the precedence.
Write(ctx context.Context, w driver.FileWriter, file multipart.File, fileReader io.Reader) (types.FileInfo, error)
Move(ctx context.Context, srcPath string, dstPath string) error
Delete(ctx context.Context, filePath string) error

37
registry/app/storage/ociblobstore.go
View File

@ -26,13 +26,30 @@ import (
"github.com/harness/gitness/registry/app/dist_temp/dcontext"
"github.com/harness/gitness/registry/app/driver"
"github.com/harness/gitness/registry/app/manifest"
"github.com/harness/gitness/registry/app/pkg/commons"
"github.com/google/uuid"
"github.com/opencontainers/go-digest"
"github.com/rs/zerolog/log"
)
const (
HeaderAccept = "Accept"
HeaderAuthorization = "Authorization"
HeaderCacheControl = "Cache-Control"
HeaderContentLength = "Content-Length"
HeaderContentRange = "Content-Range"
HeaderContentType = "Content-Type"
HeaderDockerContentDigest = "Docker-Content-Digest"
HeaderDockerUploadUUID = "Docker-Upload-UUID"
HeaderEtag = "Etag"
HeaderIfNoneMatch = "If-None-Match"
HeaderLink = "Link"
HeaderLocation = "Location"
HeaderOCIFiltersApplied = "OCI-Filters-Applied"
HeaderOCISubject = "OCI-Subject"
HeaderRange = "Range"
)
const blobCacheControlMaxAge = 365 * 24 * time.Hour
type ociBlobStore struct {
@ -106,7 +123,7 @@ func (bs *ociBlobStore) ServeBlobInternal(
}
if desc.MediaType != "" {
// Set the repository local content type.
headers[commons.HeaderContentType] = desc.MediaType
headers[HeaderContentType] = desc.MediaType
}
size := desc.Size
path, err := bs.pathFn(pathPrefix, desc.Digest)
@ -135,25 +152,25 @@ func (bs *ociBlobStore) ServeBlobInternal(
return nil, "", size, err
}
headers[commons.HeaderEtag] = fmt.Sprintf(`"%s"`, desc.Digest)
headers[HeaderEtag] = fmt.Sprintf(`"%s"`, desc.Digest)
// If-None-Match handled by ServeContent
headers[commons.HeaderCacheControl] = fmt.Sprintf(
headers[HeaderCacheControl] = fmt.Sprintf(
"max-age=%.f",
blobCacheControlMaxAge.Seconds(),
)
if headers[commons.HeaderDockerContentDigest] == "" {
headers[commons.HeaderDockerContentDigest] = desc.Digest.String()
if headers[HeaderDockerContentDigest] == "" {
headers[HeaderDockerContentDigest] = desc.Digest.String()
}
if headers[commons.HeaderContentType] == "" {
if headers[HeaderContentType] == "" {
// Set the content type if not already set.
headers[commons.HeaderContentType] = desc.MediaType
headers[HeaderContentType] = desc.MediaType
}
if headers[commons.HeaderContentLength] == "" {
if headers[HeaderContentLength] == "" {
// Set the content length if not already set.
headers[commons.HeaderContentLength] = fmt.Sprint(desc.Size)
headers[HeaderContentLength] = fmt.Sprint(desc.Size)
}
return br, "", size, err

6
registry/app/store/database/util/mapper.go
View File

@ -20,7 +20,7 @@ import (
"strconv"
"strings"
"github.com/harness/gitness/registry/app/pkg/commons"
"github.com/harness/gitness/registry/utils"
)
const ID = ""
@ -44,7 +44,7 @@ func StringToInt64Arr(s string) []int64 {
func StringToArrByDelimiter(s string, delimiter string) []string {
var arr []string
if commons.IsEmpty(s) {
if utils.IsEmpty(s) {
return arr
}
return strings.Split(s, delimiter)
@ -64,7 +64,7 @@ func Int64ArrToStringByDelimiter(arr []int64, delimiter string) string {
func StringToInt64ArrByDelimiter(s string, delimiter string) []int64 {
var arr []int64
if commons.IsEmpty(s) {
if utils.IsEmpty(s) {
return arr
}
for _, i := range strings.Split(s, delimiter) {

4
registry/app/store/database/util/utils.go
View File

@ -19,11 +19,11 @@ import (
"fmt"
"strings"
"github.com/harness/gitness/registry/app/pkg/commons"
"github.com/harness/gitness/registry/utils"
)
func GetEmptySQLString(str string) sql.NullString {
if commons.IsEmpty(str) {
if utils.IsEmpty(str) {
return sql.NullString{String: str, Valid: false}
}
return sql.NullString{String: str, Valid: true}

9
registry/types/digest.go
View File

@ -15,11 +15,10 @@
package types
import (
"encoding/hex"
"errors"
"fmt"
"github.com/harness/gitness/registry/app/store/database/util"
"github.com/opencontainers/go-digest"
)
@ -42,13 +41,17 @@ func GetDigestBytes(dgst digest.Digest) ([]byte, error) {
return nil, err
}
digestBytes, err := util.GetHexDecodedBytes(string(newDigest))
digestBytes, err := GetHexDecodedBytes(string(newDigest))
if err != nil {
return nil, err
}
return digestBytes, nil
}
func GetHexDecodedBytes(s string) ([]byte, error) {
return hex.DecodeString(s)
}
// String implements the Stringer interface.
func (d Digest) String() string {
return string(d)

10
registry/types/generic_blob.go
View File

@ -27,3 +27,13 @@ type GenericBlob struct {
CreatedAt time.Time
CreatedBy int64
}
type FileInfo struct {
Size int64
Sha1 string
Sha256 string
Sha512 string
MD5 string
Filename string
CreatedAt time.Time
}

25
registry/utils/utils.go
View File

@ -14,7 +14,10 @@
package utils
import "strings"
import (
"reflect"
"strings"
)
func HasAnyPrefix(s string, prefixes []string) bool {
for _, prefix := range prefixes {
@ -40,3 +43,23 @@ func SafeUint64(n int) uint64 {
}
return uint64(n)
}
func IsEmpty(slice interface{}) bool {
if slice == nil {
return true
}
val := reflect.ValueOf(slice)
// Check if the input is a pointer
if val.Kind() == reflect.Ptr {
// Dereference the pointer
val = val.Elem()
}
// Check if the dereferenced value is nil
if !val.IsValid() {
return true
}
return val.Len() == 0
}

54
registry/validation/helpers.go Normal file
View File

@ -0,0 +1,54 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package validation
import (
"net/url"
"strings"
)
// IsValidURL checks if URL is valid.
func IsValidURL(uri string) bool {
if u, err := url.ParseRequestURI(uri); err != nil ||
(u.Scheme != "http" && u.Scheme != "https") ||
!validPort(portOnly(u.Host)) {
return false
}
return true
}
func validPort(p string) bool {
for _, r := range []byte(p) {
if r < '0' || r > '9' {
return false
}
}
return true
}
func portOnly(hostport string) string {
colon := strings.IndexByte(hostport, ':')
if colon == -1 {
return ""
}
if i := strings.Index(hostport, "]:"); i != -1 {
return hostport[i+len("]:"):]
}
if strings.Contains(hostport, "]") {
return ""
}
return hostport[colon+len(":"):]
}