diff --git a/cmd/gitness/wire_gen.go b/cmd/gitness/wire_gen.go index 358fe9afe..49d2c151b 100644 --- a/cmd/gitness/wire_gen.go +++ b/cmd/gitness/wire_gen.go @@ -519,7 +519,8 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro packagesHandler := api2.NewPackageHandlerProvider(registryRepository, spaceStore, tokenStore, controller, authenticator, provider, authorizer) localBase := base.LocalBaseProvider(registryRepository, fileManager, transactor, imageRepository, artifactRepository) pythonLocalRegistry := python.LocalRegistryProvider(localBase, fileManager, upstreamProxyConfigRepository, transactor, registryRepository, imageRepository, artifactRepository, provider) - proxy := python.ProxyProvider(upstreamProxyConfigRepository, registryRepository, imageRepository, artifactRepository, fileManager, transactor, provider) + localRegistryHelper := python.LocalRegistryHelperProvider(pythonLocalRegistry, localBase) + proxy := python.ProxyProvider(upstreamProxyConfigRepository, registryRepository, imageRepository, artifactRepository, fileManager, transactor, provider, spaceFinder, secretService, localRegistryHelper) pythonController := python2.ControllerProvider(upstreamProxyConfigRepository, registryRepository, imageRepository, artifactRepository, fileManager, transactor, provider, pythonLocalRegistry, proxy) pythonHandler := api2.NewPythonHandlerProvider(pythonController, packagesHandler) handler4 := router.PackageHandlerProvider(packagesHandler, mavenHandler, genericHandler, pythonHandler) diff --git a/registry/app/api/controller/metadata/utils.go b/registry/app/api/controller/metadata/utils.go index 64d10ccff..1bc0a8a53 100644 --- a/registry/app/api/controller/metadata/utils.go +++ b/registry/app/api/controller/metadata/utils.go @@ -180,7 +180,8 @@ func ValidateUpstream(config *a.RegistryConfig) error { } if !commons.IsEmpty(config.Type) && config.Type == a.RegistryTypeUPSTREAM && *upstreamConfig.Source != a.UpstreamConfigSourceDockerhub && - *upstreamConfig.Source != a.UpstreamConfigSourceMavenCentral { + *upstreamConfig.Source != a.UpstreamConfigSourceMavenCentral && + *upstreamConfig.Source != a.UpstreamConfigSourcePyPi { if commons.IsEmpty(upstreamConfig.Url) { return errors.New("URL is required for upstream repository") } diff --git a/registry/app/api/controller/pkg/python/controller.go b/registry/app/api/controller/pkg/python/controller.go index 8c4a0db0f..de8a7f78a 100644 --- a/registry/app/api/controller/pkg/python/controller.go +++ b/registry/app/api/controller/pkg/python/controller.go @@ -48,8 +48,9 @@ type controller struct { imageDao store.ImageRepository artifactDao store.ArtifactRepository urlProvider urlprovider.Provider - local python.LocalRegistry - proxy python.Proxy + // TODO: Cleanup and initiate at other place + local python.LocalRegistry + proxy python.Proxy } // NewController creates a new Python controller. diff --git a/registry/app/api/controller/pkg/python/download.go b/registry/app/api/controller/pkg/python/download.go index f08494c08..30baaca4d 100644 --- a/registry/app/api/controller/pkg/python/download.go +++ b/registry/app/api/controller/pkg/python/download.go @@ -40,10 +40,10 @@ func (c *controller) DownloadPackageFile( nil, "", nil, nil, } } - headers, fileReader, redirectURL, errs := pythonRegistry.DownloadPackageFile(ctx, info) + headers, fileReader, readCloser, redirectURL, errs := pythonRegistry.DownloadPackageFile(ctx, info) return &GetArtifactResponse{ errs, headers, redirectURL, - fileReader, nil, + fileReader, readCloser, } } diff --git a/registry/app/api/controller/pkg/python/upload_package.go b/registry/app/api/controller/pkg/python/upload_package.go index 220b8becf..00c6fad3b 100644 --- a/registry/app/api/controller/pkg/python/upload_package.go +++ b/registry/app/api/controller/pkg/python/upload_package.go @@ -47,7 +47,7 @@ func (c *controller) UploadPackageFile( nil, } } - headers, sha256, err := pythonRegistry.UploadPackageFile(ctx, info, file, fileHeader) + headers, sha256, err := pythonRegistry.UploadPackageFile(ctx, info, file, fileHeader.Filename) if commons.IsEmptyError(err) { return &PutArtifactResponse{ sha256, []error{}, headers, diff --git a/registry/app/api/handler/oci/get_manifest.go b/registry/app/api/handler/oci/get_manifest.go index d9f5cb3ec..18998b447 100644 --- a/registry/app/api/handler/oci/get_manifest.go +++ b/registry/app/api/handler/oci/get_manifest.go @@ -20,6 +20,7 @@ import ( "github.com/harness/gitness/registry/app/dist_temp/errcode" "github.com/harness/gitness/registry/app/pkg/commons" "github.com/harness/gitness/registry/app/pkg/docker" + "github.com/harness/gitness/registry/app/storage" "github.com/rs/zerolog/log" ) @@ -36,8 +37,8 @@ func (h *Handler) GetManifest(w http.ResponseWriter, r *http.Request) { result := h.Controller.PullManifest( ctx, info, - r.Header[commons.HeaderAccept], - r.Header[commons.HeaderIfNoneMatch], + r.Header[storage.HeaderAccept], + r.Header[storage.HeaderIfNoneMatch], ) if commons.IsEmpty(result.GetErrors()) { response, ok := result.(*docker.GetManifestResponse) diff --git a/registry/app/api/handler/oci/head_manifest.go b/registry/app/api/handler/oci/head_manifest.go index 32b07ab3b..d695bce36 100644 --- a/registry/app/api/handler/oci/head_manifest.go +++ b/registry/app/api/handler/oci/head_manifest.go @@ -20,6 +20,7 @@ import ( "github.com/harness/gitness/registry/app/dist_temp/errcode" "github.com/harness/gitness/registry/app/pkg/commons" "github.com/harness/gitness/registry/app/pkg/docker" + "github.com/harness/gitness/registry/app/storage" ) // HeadManifest fetches the image manifest from the storage backend, if it exists. @@ -33,11 +34,16 @@ func (h *Handler) HeadManifest(w http.ResponseWriter, r *http.Request) { result := h.Controller.HeadManifest( r.Context(), info, - r.Header[commons.HeaderAccept], - r.Header[commons.HeaderIfNoneMatch], + r.Header[storage.HeaderAccept], + r.Header[storage.HeaderIfNoneMatch], ) if commons.IsEmpty(result.GetErrors()) { - result.(*docker.GetManifestResponse).ResponseHeaders.WriteToResponse(w) + response, ok := result.(*docker.GetManifestResponse) + if !ok { + handleErrors(r.Context(), errcode.Errors{errcode.ErrCodeManifestUnknown}, w) + return + } + response.ResponseHeaders.WriteToResponse(w) } handleErrors(r.Context(), result.GetErrors(), w) } diff --git a/registry/app/api/handler/oci/patch_blob_upload.go b/registry/app/api/handler/oci/patch_blob_upload.go index 6619d4dbb..c62ae3d19 100644 --- a/registry/app/api/handler/oci/patch_blob_upload.go +++ b/registry/app/api/handler/oci/patch_blob_upload.go @@ -18,6 +18,7 @@ import ( "net/http" "github.com/harness/gitness/registry/app/pkg/commons" + "github.com/harness/gitness/registry/app/storage" ) func (h *Handler) PatchBlobUpload(w http.ResponseWriter, r *http.Request) { @@ -26,9 +27,9 @@ func (h *Handler) PatchBlobUpload(w http.ResponseWriter, r *http.Request) { handleErrors(r.Context(), []error{err}, w) return } - ct := r.Header.Get(commons.HeaderContentType) - cr := r.Header.Get(commons.HeaderContentRange) - cl := r.Header.Get(commons.HeaderContentLength) + ct := r.Header.Get(storage.HeaderContentType) + cr := r.Header.Get(storage.HeaderContentRange) + cl := r.Header.Get(storage.HeaderContentLength) length := r.ContentLength if length > 0 { r.Body = http.MaxBytesReader(w, r.Body, length) diff --git a/registry/app/api/handler/python/download.go b/registry/app/api/handler/python/download.go index d8a68838e..8666f534b 100644 --- a/registry/app/api/handler/python/download.go +++ b/registry/app/api/handler/python/download.go @@ -29,31 +29,48 @@ func (h *handler) DownloadPackageFile(w http.ResponseWriter, r *http.Request) { ctx := r.Context() info, ok := request.ArtifactInfoFrom(ctx).(*pythontype.ArtifactInfo) if !ok { - log.Ctx(ctx).Error().Msg("Failed to get python artifact info from context") - h.HandleErrors(r.Context(), []error{fmt.Errorf("failed to fetch python artifact info from context")}, w) + h.HandleErrors(ctx, []error{fmt.Errorf("failed to fetch info from context")}, w) return } response := h.controller.DownloadPackageFile(ctx, *info) + if response == nil { + h.HandleErrors(ctx, []error{fmt.Errorf("failed to get response from controller")}, w) + return + } defer func() { if response.Body != nil { err := response.Body.Close() if err != nil { - log.Ctx(r.Context()).Error().Msgf("Failed to close body: %v", err) + log.Ctx(ctx).Error().Msgf("Failed to close body: %v", err) + } + } + + if response.ReadCloser != nil { + err := response.ReadCloser.Close() + if err != nil { + log.Ctx(ctx).Error().Msgf("Failed to close read closer: %v", err) } } }() if !commons.IsEmpty(response.GetErrors()) { - h.HandleErrors(r.Context(), response.GetErrors(), w) + h.HandleErrors(ctx, response.GetErrors(), w) + return } - w.Header().Set("Content-Disposition", "attachment; filename="+info.Filename) if response.RedirectURL != "" { http.Redirect(w, r, response.RedirectURL, http.StatusTemporaryRedirect) return } - h.ServeContent(w, r, response.Body, info.Filename) + + w.WriteHeader(http.StatusOK) + err := commons.ServeContent(w, r, response.Body, info.Filename, response.ReadCloser) + if err != nil { + log.Ctx(ctx).Error().Msgf("Failed to serve content: %v", err) + h.HandleErrors(ctx, []error{err}, w) + return + } response.ResponseHeaders.WriteToResponse(w) } diff --git a/registry/app/api/handler/python/handler.go b/registry/app/api/handler/python/handler.go index 8fe1e90cb..5071a1abb 100644 --- a/registry/app/api/handler/python/handler.go +++ b/registry/app/api/handler/python/handler.go @@ -15,7 +15,11 @@ package python import ( + "fmt" "net/http" + "regexp" + "strings" + "unicode" "github.com/harness/gitness/registry/app/api/controller/pkg/python" "github.com/harness/gitness/registry/app/api/handler/packages" @@ -24,10 +28,28 @@ import ( "github.com/harness/gitness/registry/app/pkg" "github.com/harness/gitness/registry/app/pkg/commons" pythontype "github.com/harness/gitness/registry/app/pkg/types/python" + "github.com/harness/gitness/registry/validation" "github.com/go-chi/chi/v5" + "github.com/rs/zerolog/log" ) +// https://peps.python.org/pep-0426/#name +var ( + normalizer = strings.NewReplacer(".", "-", "_", "-") + nameMatcher = regexp.MustCompile(`\A(?:[a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\.\-_]*[a-zA-Z0-9])\z`) +) + +// https://peps.python.org/pep-0440/#appendix-b-parsing-version-strings-with-regular-expressions +var versionMatcher = regexp.MustCompile(`\Av?` + + `(?:[0-9]+!)?` + // epoch + `[0-9]+(?:\.[0-9]+)*` + // release segment + `(?:[-_\.]?(?:a|b|c|rc|alpha|beta|pre|preview)[-_\.]?[0-9]*)?` + // pre-release + `(?:-[0-9]+|[-_\.]?(?:post|rev|r)[-_\.]?[0-9]*)?` + // post release + `(?:[-_\.]?dev[-_\.]?[0-9]*)?` + // dev release + `(?:\+[a-z0-9]+(?:[-_\.][a-z0-9]+)*)?` + // local version + `\z`) + type Handler interface { pkg.ArtifactInfoProvider UploadPackageFile(writer http.ResponseWriter, request *http.Request) @@ -74,7 +96,15 @@ func (h *handler) GetPackageArtifactInfo(r *http.Request) (pkg.PackageArtifactIn } } + image = normalizer.Replace(image) + if image != "" && version != "" && !isValidNameAndVersion(image, version) { + log.Info().Msgf("Invalid image name/version: %s/%s", info.Image, version) + return nil, fmt.Errorf("invalid name or version") + } + + md.HomePage = getHomePage(md) info.Image = image + return &pythontype.ArtifactInfo{ ArtifactInfo: info, Metadata: md, @@ -82,3 +112,46 @@ func (h *handler) GetPackageArtifactInfo(r *http.Request) (pkg.PackageArtifactIn Version: version, }, nil } + +func getHomePage(md python2.Metadata) string { + var homepageURL string + if len(md.ProjectURLs) > 0 { + for k, v := range md.ProjectURLs { + if normalizeLabel(k) != "homepage" { + continue + } + homepageURL = strings.TrimSpace(v) + break + } + } + + if len(homepageURL) == 0 { + homepageURL = md.HomePage + } + + if !validation.IsValidURL(homepageURL) { + homepageURL = "" + } + return homepageURL +} + +func isValidNameAndVersion(image, version string) bool { + return nameMatcher.MatchString(image) && versionMatcher.MatchString(version) +} + +// Normalizes a Project-URL label. +// See https://packaging.python.org/en/latest/specifications/well-known-project-urls/#label-normalization. +func normalizeLabel(label string) string { + var builder strings.Builder + + // "A label is normalized by deleting all ASCII punctuation and whitespace, and then converting the result + // to lowercase." + for _, r := range label { + if unicode.IsPunct(r) || unicode.IsSpace(r) { + continue + } + builder.WriteRune(unicode.ToLower(r)) + } + + return builder.String() +} diff --git a/registry/app/api/openapi/contracts/artifact/services.gen.go b/registry/app/api/openapi/contracts/artifact/services.gen.go index bf4c671ba..276eaa1c2 100644 --- a/registry/app/api/openapi/contracts/artifact/services.gen.go +++ b/registry/app/api/openapi/contracts/artifact/services.gen.go @@ -1,6 +1,6 @@ // Package artifact provides primitives to interact with the openapi HTTP API. // -// Code generated by github.com/deepmap/oapi-codegen/v2 version v2.1.0 DO NOT EDIT. +// Code generated by github.com/oapi-codegen/oapi-codegen/v2 version v2.4.1 DO NOT EDIT. package artifact import ( @@ -346,7 +346,6 @@ type MiddlewareFunc func(http.Handler) http.Handler // CreateRegistry operation middleware func (siw *ServerInterfaceWrapper) CreateRegistry(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -369,12 +368,11 @@ func (siw *ServerInterfaceWrapper) CreateRegistry(w http.ResponseWriter, r *http handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // DeleteRegistry operation middleware func (siw *ServerInterfaceWrapper) DeleteRegistry(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -395,12 +393,11 @@ func (siw *ServerInterfaceWrapper) DeleteRegistry(w http.ResponseWriter, r *http handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // GetRegistry operation middleware func (siw *ServerInterfaceWrapper) GetRegistry(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -421,12 +418,11 @@ func (siw *ServerInterfaceWrapper) GetRegistry(w http.ResponseWriter, r *http.Re handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // ModifyRegistry operation middleware func (siw *ServerInterfaceWrapper) ModifyRegistry(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -447,12 +443,11 @@ func (siw *ServerInterfaceWrapper) ModifyRegistry(w http.ResponseWriter, r *http handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // ListArtifactLabels operation middleware func (siw *ServerInterfaceWrapper) ListArtifactLabels(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -500,12 +495,11 @@ func (siw *ServerInterfaceWrapper) ListArtifactLabels(w http.ResponseWriter, r * handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // GetArtifactStatsForRegistry operation middleware func (siw *ServerInterfaceWrapper) GetArtifactStatsForRegistry(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -545,12 +539,11 @@ func (siw *ServerInterfaceWrapper) GetArtifactStatsForRegistry(w http.ResponseWr handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // DeleteArtifact operation middleware func (siw *ServerInterfaceWrapper) DeleteArtifact(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -580,12 +573,11 @@ func (siw *ServerInterfaceWrapper) DeleteArtifact(w http.ResponseWriter, r *http handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // UpdateArtifactLabels operation middleware func (siw *ServerInterfaceWrapper) UpdateArtifactLabels(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -615,12 +607,11 @@ func (siw *ServerInterfaceWrapper) UpdateArtifactLabels(w http.ResponseWriter, r handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // GetArtifactStats operation middleware func (siw *ServerInterfaceWrapper) GetArtifactStats(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -669,12 +660,11 @@ func (siw *ServerInterfaceWrapper) GetArtifactStats(w http.ResponseWriter, r *ht handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // GetArtifactSummary operation middleware func (siw *ServerInterfaceWrapper) GetArtifactSummary(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -704,12 +694,11 @@ func (siw *ServerInterfaceWrapper) GetArtifactSummary(w http.ResponseWriter, r * handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // DeleteArtifactVersion operation middleware func (siw *ServerInterfaceWrapper) DeleteArtifactVersion(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -748,12 +737,11 @@ func (siw *ServerInterfaceWrapper) DeleteArtifactVersion(w http.ResponseWriter, handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // GetArtifactDetails operation middleware func (siw *ServerInterfaceWrapper) GetArtifactDetails(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -803,12 +791,11 @@ func (siw *ServerInterfaceWrapper) GetArtifactDetails(w http.ResponseWriter, r * handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // GetDockerArtifactDetails operation middleware func (siw *ServerInterfaceWrapper) GetDockerArtifactDetails(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -865,12 +852,11 @@ func (siw *ServerInterfaceWrapper) GetDockerArtifactDetails(w http.ResponseWrite handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // GetDockerArtifactLayers operation middleware func (siw *ServerInterfaceWrapper) GetDockerArtifactLayers(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -927,12 +913,11 @@ func (siw *ServerInterfaceWrapper) GetDockerArtifactLayers(w http.ResponseWriter handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // GetDockerArtifactManifest operation middleware func (siw *ServerInterfaceWrapper) GetDockerArtifactManifest(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -989,12 +974,11 @@ func (siw *ServerInterfaceWrapper) GetDockerArtifactManifest(w http.ResponseWrit handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // GetDockerArtifactManifests operation middleware func (siw *ServerInterfaceWrapper) GetDockerArtifactManifests(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -1033,12 +1017,11 @@ func (siw *ServerInterfaceWrapper) GetDockerArtifactManifests(w http.ResponseWri handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // GetArtifactFiles operation middleware func (siw *ServerInterfaceWrapper) GetArtifactFiles(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -1120,12 +1103,11 @@ func (siw *ServerInterfaceWrapper) GetArtifactFiles(w http.ResponseWriter, r *ht handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // GetHelmArtifactDetails operation middleware func (siw *ServerInterfaceWrapper) GetHelmArtifactDetails(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -1164,12 +1146,11 @@ func (siw *ServerInterfaceWrapper) GetHelmArtifactDetails(w http.ResponseWriter, handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // GetHelmArtifactManifest operation middleware func (siw *ServerInterfaceWrapper) GetHelmArtifactManifest(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -1208,12 +1189,11 @@ func (siw *ServerInterfaceWrapper) GetHelmArtifactManifest(w http.ResponseWriter handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // GetArtifactVersionSummary operation middleware func (siw *ServerInterfaceWrapper) GetArtifactVersionSummary(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -1252,12 +1232,11 @@ func (siw *ServerInterfaceWrapper) GetArtifactVersionSummary(w http.ResponseWrit handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // GetAllArtifactVersions operation middleware func (siw *ServerInterfaceWrapper) GetAllArtifactVersions(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -1330,12 +1309,11 @@ func (siw *ServerInterfaceWrapper) GetAllArtifactVersions(w http.ResponseWriter, handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // GetAllArtifactsByRegistry operation middleware func (siw *ServerInterfaceWrapper) GetAllArtifactsByRegistry(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -1407,12 +1385,11 @@ func (siw *ServerInterfaceWrapper) GetAllArtifactsByRegistry(w http.ResponseWrit handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // GetClientSetupDetails operation middleware func (siw *ServerInterfaceWrapper) GetClientSetupDetails(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -1452,12 +1429,11 @@ func (siw *ServerInterfaceWrapper) GetClientSetupDetails(w http.ResponseWriter, handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // ListWebhooks operation middleware func (siw *ServerInterfaceWrapper) ListWebhooks(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -1521,12 +1497,11 @@ func (siw *ServerInterfaceWrapper) ListWebhooks(w http.ResponseWriter, r *http.R handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // CreateWebhook operation middleware func (siw *ServerInterfaceWrapper) CreateWebhook(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -1547,12 +1522,11 @@ func (siw *ServerInterfaceWrapper) CreateWebhook(w http.ResponseWriter, r *http. handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // DeleteWebhook operation middleware func (siw *ServerInterfaceWrapper) DeleteWebhook(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -1582,12 +1556,11 @@ func (siw *ServerInterfaceWrapper) DeleteWebhook(w http.ResponseWriter, r *http. handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // GetWebhook operation middleware func (siw *ServerInterfaceWrapper) GetWebhook(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -1617,12 +1590,11 @@ func (siw *ServerInterfaceWrapper) GetWebhook(w http.ResponseWriter, r *http.Req handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // UpdateWebhook operation middleware func (siw *ServerInterfaceWrapper) UpdateWebhook(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -1652,12 +1624,11 @@ func (siw *ServerInterfaceWrapper) UpdateWebhook(w http.ResponseWriter, r *http. handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // ListWebhookExecutions operation middleware func (siw *ServerInterfaceWrapper) ListWebhookExecutions(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -1706,12 +1677,11 @@ func (siw *ServerInterfaceWrapper) ListWebhookExecutions(w http.ResponseWriter, handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // GetWebhookExecution operation middleware func (siw *ServerInterfaceWrapper) GetWebhookExecution(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -1750,12 +1720,11 @@ func (siw *ServerInterfaceWrapper) GetWebhookExecution(w http.ResponseWriter, r handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // ReTriggerWebhookExecution operation middleware func (siw *ServerInterfaceWrapper) ReTriggerWebhookExecution(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -1794,12 +1763,11 @@ func (siw *ServerInterfaceWrapper) ReTriggerWebhookExecution(w http.ResponseWrit handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // GetArtifactStatsForSpace operation middleware func (siw *ServerInterfaceWrapper) GetArtifactStatsForSpace(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -1839,12 +1807,11 @@ func (siw *ServerInterfaceWrapper) GetArtifactStatsForSpace(w http.ResponseWrite handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // GetAllArtifacts operation middleware func (siw *ServerInterfaceWrapper) GetAllArtifacts(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -1932,12 +1899,11 @@ func (siw *ServerInterfaceWrapper) GetAllArtifacts(w http.ResponseWriter, r *htt handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } // GetAllRegistries operation middleware func (siw *ServerInterfaceWrapper) GetAllRegistries(w http.ResponseWriter, r *http.Request) { - ctx := r.Context() var err error @@ -2025,7 +1991,7 @@ func (siw *ServerInterfaceWrapper) GetAllRegistries(w http.ResponseWriter, r *ht handler = middleware(handler) } - handler.ServeHTTP(w, r.WithContext(ctx)) + handler.ServeHTTP(w, r) } type UnescapedCookieParamError struct { diff --git a/registry/app/api/openapi/contracts/artifact/types.gen.go b/registry/app/api/openapi/contracts/artifact/types.gen.go index 282d67e36..b1d4f2eac 100644 --- a/registry/app/api/openapi/contracts/artifact/types.gen.go +++ b/registry/app/api/openapi/contracts/artifact/types.gen.go @@ -1,6 +1,6 @@ // Package artifact provides primitives to interact with the openapi HTTP API. // -// Code generated by github.com/deepmap/oapi-codegen/v2 version v2.1.0 DO NOT EDIT. +// Code generated by github.com/oapi-codegen/oapi-codegen/v2 version v2.4.1 DO NOT EDIT. package artifact import ( diff --git a/registry/app/pkg/base/base.go b/registry/app/pkg/base/base.go index 39bc5406f..2fc01d57e 100644 --- a/registry/app/pkg/base/base.go +++ b/registry/app/pkg/base/base.go @@ -18,6 +18,7 @@ import ( "context" "encoding/json" "fmt" + "io" "mime/multipart" "net/http" "strings" @@ -37,7 +38,10 @@ import ( var _ LocalBase = (*localBase)(nil) type LocalBase interface { - Upload( + + // UploadFile uploads the file to the storage. + // FIXME: Validate upload by given sha256 or any other checksums provided + UploadFile( ctx context.Context, info pkg.ArtifactInfo, fileName string, @@ -50,12 +54,23 @@ type LocalBase interface { // each package implementation should have their own. headers *commons.ResponseHeaders, sha256 string, err errcode.Error, ) + Upload( + ctx context.Context, + info pkg.ArtifactInfo, + fileName string, + version string, + path string, + file io.ReadCloser, + metadata metadata.Metadata, + ) (*commons.ResponseHeaders, string, errcode.Error) Download(ctx context.Context, info pkg.ArtifactInfo, version string, fileName string) ( *commons.ResponseHeaders, *storage.FileReader, string, []error, ) + + Exists(ctx context.Context, info pkg.ArtifactInfo, version string, fileName string) bool } type localBase struct { @@ -82,7 +97,7 @@ func NewLocalBase( } } -func (l *localBase) Upload( +func (l *localBase) UploadFile( ctx context.Context, info pkg.ArtifactInfo, fileName string, @@ -91,6 +106,31 @@ func (l *localBase) Upload( file multipart.File, // TODO: Metadata shouldn't be provided as a parameter, it should be fetched or created. metadata metadata.Metadata, +) (*commons.ResponseHeaders, string, errcode.Error) { + return l.uploadInternal(ctx, info, fileName, version, path, file, nil, metadata) +} + +func (l *localBase) Upload( + ctx context.Context, + info pkg.ArtifactInfo, + fileName string, + version string, + path string, + file io.ReadCloser, + metadata metadata.Metadata, +) (*commons.ResponseHeaders, string, errcode.Error) { + return l.uploadInternal(ctx, info, fileName, version, path, nil, file, metadata) +} + +func (l *localBase) uploadInternal( + ctx context.Context, + info pkg.ArtifactInfo, + fileName string, + version string, + path string, + file multipart.File, + fileReadCloser io.ReadCloser, + metadata metadata.Metadata, ) (*commons.ResponseHeaders, string, errcode.Error) { responseHeaders := &commons.ResponseHeaders{ Headers: make(map[string]string), @@ -108,7 +148,7 @@ func (l *localBase) Upload( return responseHeaders, "", errcode.ErrCodeUnknown.WithDetail(err) } fileInfo, err := l.fileManager.UploadFile(ctx, path, info.RegIdentifier, registry.ID, - info.RootParentID, info.RootIdentifier, file, nil, fileName) + info.RootParentID, info.RootIdentifier, file, fileReadCloser, fileName) if err != nil { return responseHeaders, "", errcode.ErrCodeUnknown.WithDetail(err) } @@ -171,9 +211,10 @@ func (l *localBase) Download(ctx context.Context, info pkg.ArtifactInfo, version } path := "/" + info.Image + "/" + version + "/" + fileName + reg, _ := l.registryDao.GetByRootParentIDAndName(ctx, info.RootParentID, info.RegIdentifier) fileReader, _, redirectURL, err := l.fileManager.DownloadFile(ctx, path, types.Registry{ - ID: info.RegistryID, + ID: reg.ID, Name: info.RegIdentifier, }, info.RootIdentifier) if err != nil { @@ -183,11 +224,18 @@ func (l *localBase) Download(ctx context.Context, info pkg.ArtifactInfo, version return responseHeaders, fileReader, redirectURL, nil } +func (l *localBase) Exists(ctx context.Context, info pkg.ArtifactInfo, version string, fileName string) bool { + filePath := "/" + info.Image + "/" + version + "/" + fileName + sha256, _ := l.fileManager.HeadFile(ctx, filePath, info.RegistryID) + //FIXME: err should be checked on if the record doesn't exist or there was DB call issue + return sha256 != "" +} + func (l *localBase) updateMetadata( dbArtifact *types.Artifact, inputMetadata metadata.Metadata, info pkg.ArtifactInfo, - fileInfo pkg.FileInfo, + fileInfo types.FileInfo, ) error { var files []metadata.File if dbArtifact != nil { diff --git a/registry/app/pkg/base/wrapper.go b/registry/app/pkg/base/wrapper.go index 77ccd8411..785ee8e74 100644 --- a/registry/app/pkg/base/wrapper.go +++ b/registry/app/pkg/base/wrapper.go @@ -1,16 +1,16 @@ // Copyright 2023 Harness, Inc. // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package base @@ -52,7 +52,12 @@ func NoProxyWrapper( } log.Ctx(ctx).Info().Msgf("Using Repository: %s, Type: %s", registry.Name, registry.Type) - art := getArtifactRegistry(*registry) + art, ok := getArtifactRegistry(*registry).(pkg.Artifact) + if !ok { + log.Ctx(ctx).Error().Msgf("artifact %s is not a registry", registry.Name) + return result + } + result = f(*registry, art) if pkg.IsEmpty(result.GetErrors()) { return result @@ -74,9 +79,13 @@ func ProxyWrapper( if repos, err := getOrderedRepos(ctx, registryDao, requestRepoKey, *info.BaseInfo); err == nil { for _, registry := range repos { log.Ctx(ctx).Info().Msgf("Using Repository: %s, Type: %s", registry.Name, registry.Type) - reg := getArtifactRegistry(registry) - if reg != nil { - response = f(registry, reg) + artifact, ok := getArtifactRegistry(registry).(pkg.Artifact) + if !ok { + log.Ctx(ctx).Warn().Msgf("artifact %s is not a registry", registry.Name) + continue + } + if artifact != nil { + response = f(registry, artifact) if pkg.IsEmpty(response.GetErrors()) { return response } diff --git a/registry/app/pkg/commons/request.go b/registry/app/pkg/commons/request.go index 13bfeb578..0808d45fc 100644 --- a/registry/app/pkg/commons/request.go +++ b/registry/app/pkg/commons/request.go @@ -15,28 +15,15 @@ package commons import ( + "errors" + "fmt" + "io" "net/http" "reflect" + "time" "github.com/harness/gitness/registry/app/dist_temp/errcode" -) - -const ( - HeaderAccept = "Accept" - HeaderAuthorization = "Authorization" - HeaderCacheControl = "Cache-Control" - HeaderContentLength = "Content-Length" - HeaderContentRange = "Content-Range" - HeaderContentType = "Content-Type" - HeaderDockerContentDigest = "Docker-Content-Digest" - HeaderDockerUploadUUID = "Docker-Upload-UUID" - HeaderEtag = "Etag" - HeaderIfNoneMatch = "If-None-Match" - HeaderLink = "Link" - HeaderLocation = "Location" - HeaderOCIFiltersApplied = "OCI-Filters-Applied" - HeaderOCISubject = "OCI-Subject" - HeaderRange = "Range" + "github.com/harness/gitness/registry/app/storage" ) type ResponseHeaders struct { @@ -95,3 +82,24 @@ func (r *ResponseHeaders) WriteHeadersToResponse(w http.ResponseWriter) { } } } + +func ServeContent( + w http.ResponseWriter, + r *http.Request, + body *storage.FileReader, + fileName string, + readCloser io.ReadCloser, +) error { + if body != nil { + http.ServeContent(w, r, fileName, time.Time{}, body) + return nil + } + if readCloser != nil { + _, err := io.Copy(w, readCloser) + if err != nil { + return fmt.Errorf("failed to copy content: %w", err) + } + return nil + } + return errors.New("no content to serve") +} diff --git a/registry/app/pkg/context.go b/registry/app/pkg/context.go index 464355590..c9fa99f6d 100644 --- a/registry/app/pkg/context.go +++ b/registry/app/pkg/context.go @@ -15,8 +15,6 @@ package pkg import ( - "time" - "github.com/harness/gitness/registry/app/api/openapi/contracts/artifact" v2 "github.com/distribution/distribution/v3/registry/api/v2" @@ -47,16 +45,6 @@ type RegistryInfo struct { PackageType artifact.PackageType } -type FileInfo struct { - Size int64 - Sha1 string - Sha256 string - Sha512 string - MD5 string - Filename string - CreatedAt time.Time -} - func (r *RegistryInfo) SetReference(ref string) { r.Reference = ref } diff --git a/registry/app/pkg/docker/local.go b/registry/app/pkg/docker/local.go index 0de40596b..1e87b2c68 100644 --- a/registry/app/pkg/docker/local.go +++ b/registry/app/pkg/docker/local.go @@ -383,12 +383,13 @@ func (r *LocalRegistry) fetchBlobInternal( var dgst digest.Digest blobs := ctx.OciBlobStore - if err := r.dbBlobLinkExists(ctx, ctx.Digest, info.RegIdentifier, info); err != nil { + if err := r.dbBlobLinkExists(ctx, ctx.Digest, info.RegIdentifier, info); err != nil { //nolint:contextcheck errs = append(errs, errcode.FromUnknownError(err)) return responseHeaders, nil, -1, nil, "", errs } dgst = ctx.Digest headers := make(map[string]string) + //nolint:contextcheck fileReader, redirectURL, size, err := blobs.ServeBlobInternal( ctx.Context, info.RootIdentifier, @@ -882,7 +883,7 @@ func (r *LocalRegistry) InitBlobUpload( } digest := digest.Digest(mountDigest) if mountDigest != "" && fromRepo != "" { - err := r.dbMountBlob(blobCtx, fromRepo, artInfo.RegIdentifier, digest, artInfo) + err := r.dbMountBlob(blobCtx, fromRepo, artInfo.RegIdentifier, digest, artInfo) //nolint:contextcheck if err != nil { e := fmt.Errorf("failed to mount blob in database: %w", err) errList = append(errList, errcode.FromUnknownError(e)) @@ -897,7 +898,7 @@ func (r *LocalRegistry) InitBlobUpload( } blobs := blobCtx.OciBlobStore - upload, err := blobs.Create(blobCtx.Context) + upload, err := blobs.Create(blobCtx.Context) //nolint:contextcheck if err != nil { if errors.Is(err, storage.ErrUnsupported) { errList = append(errList, errcode.ErrCodeUnsupported) @@ -916,7 +917,7 @@ func (r *LocalRegistry) InitBlobUpload( errList = append(errList, errcode.ErrCodeUnknown.WithDetail(err)) return responseHeaders, errList } - responseHeaders.Headers[commons.HeaderDockerUploadUUID] = blobCtx.Upload.ID() + responseHeaders.Headers[storage.HeaderDockerUploadUUID] = blobCtx.Upload.ID() responseHeaders.Code = http.StatusAccepted return responseHeaders, nil } @@ -1017,7 +1018,7 @@ func (r *LocalRegistry) PushBlob( } ctx := r.App.GetBlobsContext(ctx2, artInfo) if ctx.UUID != "" { - resumeErrs := ResumeBlobUpload(ctx, stateToken) + resumeErrs := ResumeBlobUpload(ctx, stateToken) //nolint:contextcheck errs = append(errs, resumeErrs...) } @@ -1048,6 +1049,7 @@ func (r *LocalRegistry) PushBlob( return responseHeaders, errs } + //nolint:contextcheck if err := copyFullPayload( ctx, contentLength, body, ctx.Upload, "blob PUT", @@ -1059,6 +1061,7 @@ func (r *LocalRegistry) PushBlob( return responseHeaders, errs } + //nolint:contextcheck desc, err := ctx.Upload.Commit( ctx, artInfo.RootIdentifier, manifest.Descriptor{ Digest: dgst, @@ -1089,11 +1092,12 @@ func (r *LocalRegistry) PushBlob( errcode.ErrCodeBlobUploadInvalid.WithDetail(err), ) default: - dcontext.GetLogger(ctx, log.Error()).Msgf("unknown error completing upload: %v", err) + //nolint:contextcheck + log.Ctx(ctx).Error().Msgf("unknown error completing upload: %v", err) errs = append(errs, errcode.ErrCodeUnknown.WithDetail(err)) } } - + //nolint:contextcheck // Clean up the backend blob data if there was an error. if err := ctx.Upload.Cancel(ctx); err != nil { // If the cleanup fails, all we can do is observe and report. @@ -1104,6 +1108,7 @@ func (r *LocalRegistry) PushBlob( return responseHeaders, errs } + //nolint:contextcheck err = r.dbPutBlobUploadComplete( ctx, artInfo.RegIdentifier, diff --git a/registry/app/pkg/filemanager/file_manager.go b/registry/app/pkg/filemanager/file_manager.go index 028d1168c..508888a68 100644 --- a/registry/app/pkg/filemanager/file_manager.go +++ b/registry/app/pkg/filemanager/file_manager.go @@ -22,7 +22,6 @@ import ( "path" "strings" - "github.com/harness/gitness/registry/app/pkg" "github.com/harness/gitness/registry/app/storage" "github.com/harness/gitness/registry/app/store" "github.com/harness/gitness/registry/types" @@ -40,7 +39,8 @@ const ( pathFormat = "for path: %s, with error %w" ) -func NewFileManager(app *App, registryDao store.RegistryRepository, genericBlobDao store.GenericBlobRepository, +func NewFileManager( + app *App, registryDao store.RegistryRepository, genericBlobDao store.GenericBlobRepository, nodesDao store.NodesRepository, tx dbtx.Transactor, ) FileManager { @@ -71,7 +71,7 @@ func (f *FileManager) UploadFile( file multipart.File, fileReader io.Reader, filename string, -) (pkg.FileInfo, error) { +) (types.FileInfo, error) { // uploading the file to temporary path in file storage blobContext := f.App.GetBlobsContext(ctx, regName, rootIdentifier) pathUUID := uuid.NewString() @@ -81,7 +81,7 @@ func (f *FileManager) UploadFile( if err != nil { log.Error().Msgf("failed to initiate the file upload for file with"+ " name : %s with error : %s", filename, err.Error()) - return pkg.FileInfo{}, fmt.Errorf("failed to initiate the file upload "+ + return types.FileInfo{}, fmt.Errorf("failed to initiate the file upload "+ "for file with name : %s with error : %w", filename, err) } defer fw.Close() @@ -90,7 +90,7 @@ func (f *FileManager) UploadFile( if err != nil { log.Error().Msgf("failed to upload the file on temparary location"+ " with name : %s with error : %s", filename, err.Error()) - return pkg.FileInfo{}, fmt.Errorf("failed to upload the file on temparary "+ + return types.FileInfo{}, fmt.Errorf("failed to upload the file on temparary "+ "location with name : %s with error : %w", filename, err) } fileInfo.Filename = filename @@ -102,7 +102,7 @@ func (f *FileManager) UploadFile( if err != nil { log.Error().Msgf("failed to Move the file on permanent location "+ "with name : %s with error : %s", filename, err.Error()) - return pkg.FileInfo{}, fmt.Errorf("failed to Move the file on permanent"+ + return types.FileInfo{}, fmt.Errorf("failed to Move the file on permanent"+ " location with name : %s with error : %w", filename, err) } @@ -120,7 +120,7 @@ func (f *FileManager) UploadFile( if err != nil { log.Error().Msgf("failed to save generic blob in db with "+ "sha256 : %s, err: %s", fileInfo.Sha256, err.Error()) - return pkg.FileInfo{}, fmt.Errorf("failed to save generic blob"+ + return types.FileInfo{}, fmt.Errorf("failed to save generic blob"+ " in db with sha256 : %s, err: %w", fileInfo.Sha256, err) } blobID = gb.ID @@ -135,7 +135,7 @@ func (f *FileManager) UploadFile( if err != nil { log.Error().Msgf("failed to save nodes for file : %s, with "+ "path : %s, err: %s", filename, filePath, err) - return pkg.FileInfo{}, fmt.Errorf("failed to save nodes for"+ + return types.FileInfo{}, fmt.Errorf("failed to save nodes for"+ " file : %s, with path : %s, err: %w", filename, filePath, err) } return fileInfo, nil @@ -175,8 +175,10 @@ func (f *FileManager) createNodes(ctx context.Context, filePath string, blobID s return nil } -func (f *FileManager) SaveNode(ctx context.Context, filePath string, blobID string, regID int64, segment string, - parentID string, nodePath string, isFile bool) (string, error) { +func (f *FileManager) SaveNode( + ctx context.Context, filePath string, blobID string, regID int64, segment string, + parentID string, nodePath string, isFile bool, +) (string, error) { node := &types.Node{ Name: segment, RegistryID: regID, @@ -261,20 +263,20 @@ func (f *FileManager) GetFileMetadata( ctx context.Context, filePath string, regID int64, -) (pkg.FileInfo, error) { +) (types.FileInfo, error) { node, err := f.nodesDao.GetByPathAndRegistryID(ctx, regID, filePath) if err != nil { - return pkg.FileInfo{}, fmt.Errorf("failed to get the node path mapping "+ + return types.FileInfo{}, fmt.Errorf("failed to get the node path mapping "+ pathFormat, filePath, err) } blob, err := f.genericBlobDao.FindByID(ctx, node.BlobID) if err != nil { - return pkg.FileInfo{}, fmt.Errorf("failed to get the blob for path: %s, "+ + return types.FileInfo{}, fmt.Errorf("failed to get the blob for path: %s, "+ "with blob id: %s, with error %s", filePath, node.BlobID, err) } - return pkg.FileInfo{ + return types.FileInfo{ Sha1: blob.Sha1, Size: blob.Size, Sha256: blob.Sha256, diff --git a/registry/app/pkg/generic/controller.go b/registry/app/pkg/generic/controller.go index abdf6abc2..2cae2fe74 100644 --- a/registry/app/pkg/generic/controller.go +++ b/registry/app/pkg/generic/controller.go @@ -173,7 +173,7 @@ func (c Controller) UploadArtifact( func (c Controller) updateMetadata( dbArtifact *types.Artifact, metadataInput *metadata.GenericMetadata, - info pkg.GenericArtifactInfo, fileInfo pkg.FileInfo, + info pkg.GenericArtifactInfo, fileInfo types.FileInfo, ) error { var files []metadata.File if dbArtifact != nil { diff --git a/registry/app/pkg/maven/local.go b/registry/app/pkg/maven/local.go index b3d905c5f..5b3a62bec 100644 --- a/registry/app/pkg/maven/local.go +++ b/registry/app/pkg/maven/local.go @@ -190,7 +190,7 @@ func (r *LocalRegistry) PutArtifact(ctx context.Context, info pkg.MavenArtifactI func (r *LocalRegistry) updateArtifactMetadata( dbArtifact *types.Artifact, mavenMetadata *metadata.MavenMetadata, - info pkg.MavenArtifactInfo, fileInfo pkg.FileInfo, + info pkg.MavenArtifactInfo, fileInfo types.FileInfo, ) error { var files []metadata.File if dbArtifact != nil { diff --git a/registry/app/pkg/maven/utils/utils.go b/registry/app/pkg/maven/utils/utils.go index 7709de91e..3b761f4c9 100644 --- a/registry/app/pkg/maven/utils/utils.go +++ b/registry/app/pkg/maven/utils/utils.go @@ -22,6 +22,7 @@ import ( "github.com/harness/gitness/registry/app/pkg" "github.com/harness/gitness/registry/app/pkg/commons" + "github.com/harness/gitness/registry/types" ) const ( @@ -81,8 +82,9 @@ func IsMainArtifactFile(info pkg.MavenArtifactInfo) bool { return false } -func SetHeaders(info pkg.MavenArtifactInfo, - fileInfo pkg.FileInfo, +func SetHeaders( + info pkg.MavenArtifactInfo, + fileInfo types.FileInfo, ) *commons.ResponseHeaders { responseHeaders := &commons.ResponseHeaders{ Headers: map[string]string{}, diff --git a/registry/app/pkg/python/local.go b/registry/app/pkg/python/local.go index 75eb4ab20..93ea2a914 100644 --- a/registry/app/pkg/python/local.go +++ b/registry/app/pkg/python/local.go @@ -18,9 +18,11 @@ import ( "context" "encoding/json" "fmt" + "io" "mime/multipart" - "net/http" "sort" + "strconv" + "strings" urlprovider "github.com/harness/gitness/app/url" "github.com/harness/gitness/registry/app/api/openapi/contracts/artifact" @@ -31,10 +33,12 @@ import ( "github.com/harness/gitness/registry/app/pkg/commons" "github.com/harness/gitness/registry/app/pkg/filemanager" pythontype "github.com/harness/gitness/registry/app/pkg/types/python" + "github.com/harness/gitness/registry/app/remote/adapter/commons/pypi" "github.com/harness/gitness/registry/app/storage" "github.com/harness/gitness/registry/app/store" - "github.com/harness/gitness/registry/types" "github.com/harness/gitness/store/database/dbtx" + + "github.com/rs/zerolog/log" ) var _ pkg.Artifact = (*localRegistry)(nil) @@ -85,31 +89,19 @@ func (c *localRegistry) GetPackageTypes() []artifact.PackageType { return []artifact.PackageType{artifact.PackageTypePYTHON} } -func (c *localRegistry) DownloadPackageFile(ctx context.Context, info pythontype.ArtifactInfo) ( - *commons.ResponseHeaders, - *storage.FileReader, - string, - []error, -) { - responseHeaders := &commons.ResponseHeaders{ - Headers: make(map[string]string), - Code: 0, +func (c *localRegistry) DownloadPackageFile( + ctx context.Context, + info pythontype.ArtifactInfo, +) (*commons.ResponseHeaders, *storage.FileReader, io.ReadCloser, string, []error) { + headers, fileReader, redirectURL, errors := c.localBase.Download(ctx, info.ArtifactInfo, info.Version, + info.Filename) + if len(errors) > 0 { + return nil, nil, nil, "", errors } - - path := "/" + info.Image + "/" + info.Version + "/" + info.Filename - - fileReader, _, redirectURL, err := c.fileManager.DownloadFile(ctx, path, types.Registry{ - ID: info.RegistryID, - Name: info.RegIdentifier, - }, info.RootIdentifier) - if err != nil { - return responseHeaders, nil, "", []error{err} - } - responseHeaders.Code = http.StatusOK - return responseHeaders, fileReader, redirectURL, nil + return headers, fileReader, nil, redirectURL, nil } -// Metadata represents the metadata of a Python package. +// GetPackageMetadata Metadata represents the metadata of a Python package. func (c *localRegistry) GetPackageMetadata( ctx context.Context, info pythontype.ArtifactInfo, @@ -152,22 +144,68 @@ func (c *localRegistry) GetPackageMetadata( } } - // Sort files by Name - sort.Slice(packageMetadata.Files, func(i, j int) bool { - return packageMetadata.Files[i].Name < packageMetadata.Files[j].Name - }) - + sortPackageMetadata(ctx, packageMetadata) return packageMetadata, nil } +func sortPackageMetadata(ctx context.Context, metadata pythontype.PackageMetadata) { + sort.Slice(metadata.Files, func(i, j int) bool { + version1 := pypi.GetPyPIVersion(metadata.Files[i].Name) + version2 := pypi.GetPyPIVersion(metadata.Files[j].Name) + if version1 == "" || version2 == "" || version1 == version2 { + return metadata.Files[i].Name < metadata.Files[j].Name + } + + vi := parseVersion(ctx, version1) + vj := parseVersion(ctx, version2) + + for k := 0; k < len(vi) && k < len(vj); k++ { + if vi[k] != vj[k] { + return vi[k] < vj[k] + } + } + + return len(vi) < len(vj) + }) +} + +func parseVersion(ctx context.Context, version string) []int { + parts := strings.Split(version, ".") + result := make([]int, len(parts)) + for i, part := range parts { + num, err := strconv.Atoi(part) + if err != nil { + log.Debug().Ctx(ctx).Msgf("failed to parse version %s: %v", part, err) + continue + } + result[i] = num + } + return result +} + func (c *localRegistry) UploadPackageFile( ctx context.Context, info pythontype.ArtifactInfo, file multipart.File, - fileHeader *multipart.FileHeader, + filename string, ) (headers *commons.ResponseHeaders, sha256 string, err errcode.Error) { - path := info.Image + "/" + info.Metadata.Version + "/" + fileHeader.Filename - return c.localBase.Upload(ctx, info.ArtifactInfo, fileHeader.Filename, info.Metadata.Version, path, file, + defer file.Close() + path := pkg.JoinWithSeparator("/", info.Image, info.Metadata.Version, filename) + return c.localBase.UploadFile(ctx, info.ArtifactInfo, filename, info.Metadata.Version, path, file, + &pythonmetadata.PythonMetadata{ + Metadata: info.Metadata, + }) +} + +func (c *localRegistry) UploadPackageFileReader( + ctx context.Context, + info pythontype.ArtifactInfo, + file io.ReadCloser, + filename string, +) (headers *commons.ResponseHeaders, sha256 string, err errcode.Error) { + defer file.Close() + path := pkg.JoinWithSeparator("/", info.Image, info.Metadata.Version, filename) + return c.localBase.Upload(ctx, info.ArtifactInfo, filename, info.Metadata.Version, path, file, &pythonmetadata.PythonMetadata{ Metadata: info.Metadata, }) diff --git a/registry/app/pkg/python/local_helper.go b/registry/app/pkg/python/local_helper.go index 1766d3da6..7a0ef436f 100644 --- a/registry/app/pkg/python/local_helper.go +++ b/registry/app/pkg/python/local_helper.go @@ -1,25 +1,76 @@ // Copyright 2023 Harness, Inc. // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package python +import ( + "context" + "io" + + "github.com/harness/gitness/registry/app/dist_temp/errcode" + "github.com/harness/gitness/registry/app/pkg/base" + "github.com/harness/gitness/registry/app/pkg/commons" + "github.com/harness/gitness/registry/app/pkg/types/python" + "github.com/harness/gitness/registry/app/storage" +) + type LocalRegistryHelper interface { + FileExists(ctx context.Context, info python.ArtifactInfo) bool + DownloadFile(ctx context.Context, info python.ArtifactInfo) ( + *commons.ResponseHeaders, + *storage.FileReader, + string, + []error, + ) + UploadPackageFile( + ctx context.Context, + info python.ArtifactInfo, + fileReader io.ReadCloser, + filename string, + ) (*commons.ResponseHeaders, string, errcode.Error) } type localRegistryHelper struct { + localRegistry LocalRegistry + localBase base.LocalBase } -func NewLocalRegistryHelper() LocalRegistryHelper { - return &localRegistryHelper{} +func NewLocalRegistryHelper(localRegistry LocalRegistry, localBase base.LocalBase) LocalRegistryHelper { + return &localRegistryHelper{ + localRegistry: localRegistry, + localBase: localBase, + } +} + +func (h *localRegistryHelper) FileExists(ctx context.Context, info python.ArtifactInfo) bool { + return h.localBase.Exists(ctx, info.ArtifactInfo, info.Version, info.Filename) +} + +func (h *localRegistryHelper) DownloadFile(ctx context.Context, info python.ArtifactInfo) ( + *commons.ResponseHeaders, + *storage.FileReader, + string, + []error, +) { + return h.localBase.Download(ctx, info.ArtifactInfo, info.Version, info.Filename) +} + +func (h *localRegistryHelper) UploadPackageFile( + ctx context.Context, + info python.ArtifactInfo, + fileReader io.ReadCloser, + filename string, +) (*commons.ResponseHeaders, string, errcode.Error) { + return h.localRegistry.UploadPackageFileReader(ctx, info, fileReader, filename) } diff --git a/registry/app/pkg/python/proxy.go b/registry/app/pkg/python/proxy.go index 8c9cc3e79..79afd3c0d 100644 --- a/registry/app/pkg/python/proxy.go +++ b/registry/app/pkg/python/proxy.go @@ -20,6 +20,7 @@ import ( "io" "mime/multipart" + "github.com/harness/gitness/app/services/refcache" urlprovider "github.com/harness/gitness/app/url" "github.com/harness/gitness/registry/app/api/openapi/contracts/artifact" "github.com/harness/gitness/registry/app/dist_temp/errcode" @@ -27,26 +28,33 @@ import ( "github.com/harness/gitness/registry/app/pkg/commons" "github.com/harness/gitness/registry/app/pkg/filemanager" pythontype "github.com/harness/gitness/registry/app/pkg/types/python" - "github.com/harness/gitness/registry/app/remote/controller/proxy/python" + "github.com/harness/gitness/registry/app/remote/adapter/commons/pypi" "github.com/harness/gitness/registry/app/storage" "github.com/harness/gitness/registry/app/store" + cfg "github.com/harness/gitness/registry/config" + request2 "github.com/harness/gitness/registry/request" + "github.com/harness/gitness/secret" "github.com/harness/gitness/store/database/dbtx" "github.com/rs/zerolog/log" + + _ "github.com/harness/gitness/registry/app/remote/adapter/pypi" // This is required to init pypi adapter ) var _ pkg.Artifact = (*proxy)(nil) var _ Registry = (*proxy)(nil) type proxy struct { - fileManager filemanager.FileManager - proxyStore store.UpstreamProxyConfigRepository - tx dbtx.Transactor - registryDao store.RegistryRepository - imageDao store.ImageRepository - artifactDao store.ArtifactRepository - urlProvider urlprovider.Provider - proxyController python.Controller + fileManager filemanager.FileManager + proxyStore store.UpstreamProxyConfigRepository + tx dbtx.Transactor + registryDao store.RegistryRepository + imageDao store.ImageRepository + artifactDao store.ArtifactRepository + urlProvider urlprovider.Provider + spaceFinder refcache.SpaceFinder + service secret.Service + localRegistryHelper LocalRegistryHelper } type Proxy interface { @@ -61,15 +69,21 @@ func NewProxy( imageDao store.ImageRepository, artifactDao store.ArtifactRepository, urlProvider urlprovider.Provider, + spaceFinder refcache.SpaceFinder, + service secret.Service, + localRegistryHelper LocalRegistryHelper, ) Proxy { return &proxy{ - proxyStore: proxyStore, - registryDao: registryDao, - imageDao: imageDao, - artifactDao: artifactDao, - fileManager: fileManager, - tx: tx, - urlProvider: urlProvider, + fileManager: fileManager, + proxyStore: proxyStore, + tx: tx, + registryDao: registryDao, + imageDao: imageDao, + artifactDao: artifactDao, + urlProvider: urlProvider, + spaceFinder: spaceFinder, + service: service, + localRegistryHelper: localRegistryHelper, } } @@ -84,53 +98,138 @@ func (r *proxy) GetPackageTypes() []artifact.PackageType { func (r *proxy) DownloadPackageFile(ctx context.Context, info pythontype.ArtifactInfo) ( *commons.ResponseHeaders, *storage.FileReader, + io.ReadCloser, string, []error, ) { - headers, body, _, url, errs := r.fetchFile(ctx, info, true) - return headers, body, url, errs + upstreamProxy, err := r.proxyStore.GetByRegistryIdentifier(ctx, info.ParentID, info.RegIdentifier) + if err != nil { + return nil, nil, nil, "", []error{errcode.ErrCodeUnknown.WithDetail(err)} + } + + // TODO: Extract out to Path Utils for all package types + exists := r.localRegistryHelper.FileExists(ctx, info) + if exists { + headers, fileReader, redirectURL, errors := r.localRegistryHelper.DownloadFile(ctx, info) + if len(errors) == 0 { + return headers, fileReader, nil, redirectURL, errors + } + // If file exists in local registry, but download failed, we should try to download from remote + log.Warn().Ctx(ctx).Msgf("failed to pull from local, attempting streaming from remote, %v", errors) + } + + remote, err := NewRemoteRegistryHelper(ctx, r.spaceFinder, *upstreamProxy, r.service) + if err != nil { + return nil, nil, nil, "", []error{errcode.ErrCodeUnknown.WithDetail(err)} + } + + file, err := remote.GetFile(ctx, info.Image, info.Filename) + if err != nil { + return nil, nil, nil, "", []error{errcode.ErrCodeUnknown.WithDetail(err)} + } + + go func(info pythontype.ArtifactInfo) { + ctx2 := context.WithoutCancel(ctx) + ctx2 = context.WithValue(ctx2, cfg.GoRoutineKey, "goRoutine") + err = r.putFileToLocal(ctx2, info.Image, info.Filename, remote) + if err != nil { + log.Ctx(ctx2).Error().Stack().Err(err).Msgf("error while putting file to localRegistry, %v", err) + return + } + log.Ctx(ctx2).Info().Msgf("Successfully updated file: %s, registry: %s", info.Filename, info.RegIdentifier) + }(info) + + return nil, nil, file, "", nil } -// Metadata represents the metadata of a Python package. +// GetPackageMetadata Returns metadata from remote. func (r *proxy) GetPackageMetadata( - _ context.Context, - _ pythontype.ArtifactInfo, + ctx context.Context, + info pythontype.ArtifactInfo, ) (pythontype.PackageMetadata, error) { - return pythontype.PackageMetadata{}, nil + upstreamProxy, err := r.proxyStore.GetByRegistryIdentifier(ctx, info.ParentID, info.RegIdentifier) + if err != nil { + return pythontype.PackageMetadata{}, err + } + + helper, _ := NewRemoteRegistryHelper(ctx, r.spaceFinder, *upstreamProxy, r.service) + result, err := helper.GetMetadata(ctx, info.Image) + if err != nil { + return pythontype.PackageMetadata{}, err + } + + var files []pythontype.File + for _, file := range result.Packages { + files = append(files, pythontype.File{ + Name: file.Name, + FileURL: r.urlProvider.RegistryURL(ctx) + fmt.Sprintf( + "/pkg/%s/%s/python/files/%s/%s/%s", + info.RootIdentifier, + info.RegIdentifier, + info.Image, + file.Version(), + file.Name), + RequiresPython: file.RequiresPython(), + }) + } + + metadata := pythontype.PackageMetadata{ + Name: info.Image, + Files: files, + } + sortPackageMetadata(ctx, metadata) + return metadata, nil } -// UploadPackageFile FIXME: Extract this upload function for all types of packageTypes +func (r *proxy) putFileToLocal(ctx context.Context, pkg string, filename string, remote RemoteRegistryHelper) error { + version := pypi.GetPyPIVersion(filename) + metadata, err := remote.GetJSON(ctx, pkg, version) + if err != nil { + log.Ctx(ctx).Error().Stack().Err(err).Msgf("fetching metadata for %s failed, %v", filename, err) + return err + } + file, err := remote.GetFile(ctx, pkg, filename) + if err != nil { + log.Ctx(ctx).Error().Stack().Err(err).Msgf("fetching file %s failed, %v", filename, err) + return err + } + info, ok := request2.ArtifactInfoFrom(ctx).(*pythontype.ArtifactInfo) + if !ok { + log.Ctx(ctx).Error().Msgf("failed to cast artifact info to python artifact info") + return errcode.ErrCodeInvalidRequest.WithDetail(fmt.Errorf("failed to cast artifact info to python artifact info")) + } + info.Metadata = *metadata + info.Filename = filename + + _, sha256, err2 := r.localRegistryHelper.UploadPackageFile(ctx, *info, file, filename) + if !commons.IsEmptyError(err2) { + log.Ctx(ctx).Error().Stack().Err(err2).Msgf("uploading file %s failed, %v", filename, err) + return err2 + } + log.Info().Msgf("Successfully uploaded %s with SHA256: %s", filename, sha256) + return nil +} + +// UploadPackageFile TODO: Extract this upload function for all types of packageTypes // uploads the package file to the storage. func (r *proxy) UploadPackageFile( ctx context.Context, _ pythontype.ArtifactInfo, _ multipart.File, - _ *multipart.FileHeader, + _ string, ) (*commons.ResponseHeaders, string, errcode.Error) { log.Error().Ctx(ctx).Msg("Not implemented") return nil, "", errcode.ErrCodeInvalidRequest.WithDetail(fmt.Errorf("not implemented")) } -func (r *proxy) fetchFile(ctx context.Context, info pythontype.ArtifactInfo, serveFile bool) ( - responseHeaders *commons.ResponseHeaders, body *storage.FileReader, readCloser io.ReadCloser, - redirectURL string, errs []error, -) { - log.Ctx(ctx).Info().Msgf("Maven Proxy: %s", info.RegIdentifier) - - responseHeaders, body, redirectURL, useLocal := r.proxyController.UseLocalFile(ctx, info) - if useLocal { - return responseHeaders, body, readCloser, redirectURL, errs - } - - upstreamProxy, err := r.proxyStore.GetByRegistryIdentifier(ctx, info.ParentID, info.RegIdentifier) - if err != nil { - return responseHeaders, nil, nil, "", []error{errcode.ErrCodeUnknown.WithDetail(err)} - } - - // This is start of proxy Code. - responseHeaders, readCloser, err = r.proxyController.ProxyFile(ctx, info, *upstreamProxy, serveFile) - if err != nil { - return responseHeaders, nil, nil, "", []error{errcode.ErrCodeUnknown.WithDetail(err)} - } - return responseHeaders, nil, readCloser, "", errs +// UploadPackageFile TODO: Extract this upload function for all types of packageTypes +// uploads the package file to the storage. +func (r *proxy) UploadPackageFileReader( + ctx context.Context, + _ pythontype.ArtifactInfo, + _ io.ReadCloser, + _ string, +) (*commons.ResponseHeaders, string, errcode.Error) { + log.Error().Ctx(ctx).Msg("Not implemented") + return nil, "", errcode.ErrCodeInvalidRequest.WithDetail(fmt.Errorf("not implemented")) } diff --git a/registry/app/pkg/python/registry.go b/registry/app/pkg/python/registry.go index 66720d08f..8c16c2e7c 100644 --- a/registry/app/pkg/python/registry.go +++ b/registry/app/pkg/python/registry.go @@ -1,21 +1,22 @@ // Copyright 2023 Harness, Inc. // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package python import ( "context" + "io" "mime/multipart" "github.com/harness/gitness/registry/app/dist_temp/errcode" @@ -34,12 +35,20 @@ type Registry interface { ctx context.Context, info python.ArtifactInfo, file multipart.File, - fileHeader *multipart.FileHeader, + filename string, + ) (*commons.ResponseHeaders, string, errcode.Error) + + UploadPackageFileReader( + ctx context.Context, + info python.ArtifactInfo, + file io.ReadCloser, + filename string, ) (*commons.ResponseHeaders, string, errcode.Error) DownloadPackageFile(ctx context.Context, info python.ArtifactInfo) ( *commons.ResponseHeaders, *storage.FileReader, + io.ReadCloser, string, []error, ) diff --git a/registry/app/pkg/python/remote_helper.go b/registry/app/pkg/python/remote_helper.go index 55902ce5a..778ad49ec 100644 --- a/registry/app/pkg/python/remote_helper.go +++ b/registry/app/pkg/python/remote_helper.go @@ -1,25 +1,121 @@ // Copyright 2023 Harness, Inc. // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package python +import ( + "context" + "io" + + "github.com/harness/gitness/app/services/refcache" + "github.com/harness/gitness/registry/app/api/openapi/contracts/artifact" + "github.com/harness/gitness/registry/app/metadata/python" + "github.com/harness/gitness/registry/app/remote/adapter" + pypi2 "github.com/harness/gitness/registry/app/remote/adapter/commons/pypi" + "github.com/harness/gitness/registry/app/remote/adapter/pypi" + "github.com/harness/gitness/registry/app/remote/registry" + "github.com/harness/gitness/registry/types" + "github.com/harness/gitness/secret" + + "github.com/rs/zerolog/log" +) + type RemoteRegistryHelper interface { + // GetFile Downloads the file for the given package and filename + GetFile(ctx context.Context, pkg string, filename string) (io.ReadCloser, error) + + // GetMetadata Fetches the metadata for the given package for all versions + GetMetadata(ctx context.Context, pkg string) (*pypi2.SimpleMetadata, error) + + // GetJSON Fetches the metadata for the given package and specific version + GetJSON(ctx context.Context, pkg string, version string) (*python.Metadata, error) } type remoteRegistryHelper struct { + adapter registry.PythonRegistry + registry types.UpstreamProxy } -func NewRemoteRegistryHelper() RemoteRegistryHelper { - return &remoteRegistryHelper{} +func NewRemoteRegistryHelper( + ctx context.Context, + spaceFinder refcache.SpaceFinder, + registry types.UpstreamProxy, + service secret.Service, +) (RemoteRegistryHelper, error) { + r := &remoteRegistryHelper{ + registry: registry, + } + if err := r.init(ctx, spaceFinder, service); err != nil { + log.Ctx(ctx).Error().Err(err).Msgf("failed to init remote registry for remote: %s", registry.RepoKey) + return nil, err + } + return r, nil +} + +func (r *remoteRegistryHelper) init( + ctx context.Context, + spaceFinder refcache.SpaceFinder, + service secret.Service, +) error { + key := string(artifact.UpstreamConfigSourcePyPi) + if r.registry.Source == string(artifact.UpstreamConfigSourcePyPi) { + r.registry.RepoURL = pypi.PyPiURL + } + + factory, err := adapter.GetFactory(key) + if err != nil { + log.Ctx(ctx).Error().Err(err).Msg("failed to get factory " + key) + return err + } + + adpt, err := factory.Create(ctx, spaceFinder, r.registry, service) + if err != nil { + log.Ctx(ctx).Error().Err(err).Msg("failed to create factory " + key) + return err + } + + pythonReg, ok := adpt.(registry.PythonRegistry) + if !ok { + log.Ctx(ctx).Error().Msg("failed to cast factory to python registry") + return err + } + r.adapter = pythonReg + return nil +} + +func (r *remoteRegistryHelper) GetFile(ctx context.Context, pkg string, filename string) (io.ReadCloser, error) { + v2, err := r.adapter.GetPackage(ctx, pkg, filename) + if err != nil { + log.Ctx(ctx).Error().Err(err).Msgf("failed to get pkg: %s, file: %s", pkg, filename) + } + return v2, err +} + +func (r *remoteRegistryHelper) GetMetadata(ctx context.Context, pkg string) (*pypi2.SimpleMetadata, error) { + packages, err := r.adapter.GetMetadata(ctx, pkg) + if err != nil { + log.Ctx(ctx).Error().Err(err).Msgf("failed to get metadata for pkg: %s", pkg) + return nil, err + } + return packages, nil +} + +func (r *remoteRegistryHelper) GetJSON(ctx context.Context, pkg string, version string) (*python.Metadata, error) { + metadata, err := r.adapter.GetJSON(ctx, pkg, version) + if err != nil { + log.Ctx(ctx).Error().Err(err).Msgf("failed to get JSON for pkg: %s, version: %s", pkg, version) + return nil, err + } + return metadata, nil } diff --git a/registry/app/pkg/python/wire.go b/registry/app/pkg/python/wire.go index 576c5b7af..e5c6637f3 100644 --- a/registry/app/pkg/python/wire.go +++ b/registry/app/pkg/python/wire.go @@ -15,10 +15,12 @@ package python import ( + "github.com/harness/gitness/app/services/refcache" urlprovider "github.com/harness/gitness/app/url" "github.com/harness/gitness/registry/app/pkg/base" "github.com/harness/gitness/registry/app/pkg/filemanager" "github.com/harness/gitness/registry/app/store" + "github.com/harness/gitness/secret" "github.com/harness/gitness/store/database/dbtx" "github.com/google/wire" @@ -48,10 +50,18 @@ func ProxyProvider( fileManager filemanager.FileManager, tx dbtx.Transactor, urlProvider urlprovider.Provider, + spaceFinder refcache.SpaceFinder, + service secret.Service, + localRegistryHelper LocalRegistryHelper, ) Proxy { - proxy := NewProxy(fileManager, proxyStore, tx, registryDao, imageDao, artifactDao, urlProvider) + proxy := NewProxy(fileManager, proxyStore, tx, registryDao, imageDao, artifactDao, urlProvider, + spaceFinder, service, localRegistryHelper) base.Register(proxy) return proxy } -var WireSet = wire.NewSet(LocalRegistryProvider, ProxyProvider) +func LocalRegistryHelperProvider(localRegistry LocalRegistry, localBase base.LocalBase) LocalRegistryHelper { + return NewLocalRegistryHelper(localRegistry, localBase) +} + +var WireSet = wire.NewSet(LocalRegistryProvider, ProxyProvider, LocalRegistryHelperProvider) diff --git a/registry/app/pkg/types/python/types.go b/registry/app/pkg/types/python/types.go index 1a8546c1d..537ffa192 100644 --- a/registry/app/pkg/types/python/types.go +++ b/registry/app/pkg/types/python/types.go @@ -1,16 +1,16 @@ // Copyright 2023 Harness, Inc. // -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at // -// http://www.apache.org/licenses/LICENSE-2.0 +// http://www.apache.org/licenses/LICENSE-2.0 // -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. package python @@ -19,6 +19,8 @@ import ( "github.com/harness/gitness/registry/app/pkg" ) +// Metadata represents the metadata for a Python package. + type ArtifactInfo struct { pkg.ArtifactInfo Version string @@ -26,7 +28,7 @@ type ArtifactInfo struct { Metadata python.Metadata } -// BaseArtifactInfo implements pkg.PackageArtifactInfo interface. +// BaseArtifactInfo implements pkg.PackageArtifactInfo interface func (a ArtifactInfo) BaseArtifactInfo() pkg.ArtifactInfo { return a.ArtifactInfo } diff --git a/registry/app/pkg/utils.go b/registry/app/pkg/utils.go index 79f914b41..9a932f39f 100644 --- a/registry/app/pkg/utils.go +++ b/registry/app/pkg/utils.go @@ -16,6 +16,7 @@ package pkg import ( "reflect" + "strings" ) func IsEmpty(slice interface{}) bool { @@ -24,3 +25,7 @@ func IsEmpty(slice interface{}) bool { } return reflect.ValueOf(slice).Len() == 0 } + +func JoinWithSeparator(sep string, args ...string) string { + return strings.Join(args, sep) +} diff --git a/registry/app/remote/adapter/awsecr/auth.go b/registry/app/remote/adapter/awsecr/auth.go index 45cd59478..d2620a149 100644 --- a/registry/app/remote/adapter/awsecr/auth.go +++ b/registry/app/remote/adapter/awsecr/auth.go @@ -167,16 +167,18 @@ func getCreds( return accessKey, secretKey, false, nil } -func getSecretValue(ctx context.Context, spaceFinder refcache.SpaceFinder, secretService secret.Service, - secretSpaceID int64, secretSpacePath string) (string, error) { +func getSecretValue( + ctx context.Context, spaceFinder refcache.SpaceFinder, secretService secret.Service, + secretSpaceID int64, secretSpacePath string, +) (string, error) { spacePath, err := spaceFinder.FindByID(ctx, secretSpaceID) if err != nil { - log.Error().Msgf("failed to find space path: %v", err) + log.Error().Msgf("failed to find space path: %d, %v", secretSpaceID, err) return "", err } decryptSecret, err := secretService.DecryptSecret(ctx, spacePath.Path, secretSpacePath) if err != nil { - log.Error().Msgf("failed to decrypt secret: %v", err) + log.Error().Msgf("failed to decrypt secret at path: %s, secret: %s, %v", spacePath.Path, secretSpacePath, err) return "", err } return decryptSecret, nil diff --git a/registry/app/remote/adapter/commons/pypi/dto.go b/registry/app/remote/adapter/commons/pypi/dto.go new file mode 100644 index 000000000..7df4b130f --- /dev/null +++ b/registry/app/remote/adapter/commons/pypi/dto.go @@ -0,0 +1,109 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pypi + +import ( + "errors" + "fmt" + "strings" + + "golang.org/x/net/html" +) + +var ( + extensions = []string{ + ".tar.gz", + ".tar.bz2", + ".tar.xz", + ".zip", + ".whl", + ".egg", + + ".exe", + ".app", + ".dmg", + } +) + +type SimpleMetadata struct { + Title string + MetaName string + Content string + Packages []Package +} + +type Package struct { + Name string + ATags map[string]string +} + +// URL returns the "href" attribute from the package's map. +func (p Package) URL() string { + return p.ATags["href"] +} +func (p Package) Valid() bool { + return p.URL() != "" && p.Name != "" +} + +// RequiresPython returns the "data-requires-python" attribute (unescaped) from the package's map. +func (p Package) RequiresPython() string { + val := p.ATags["data-requires-python"] + // unescape HTML entities like ">" + return html.UnescapeString(val) +} + +// Version Fetches version from format: +// The wheel filename is {distribution}-{version}(-{build tag})?-{python tag}-{abi tag}-{platform tag}.whl +// SRC: https://packaging.python.org/en/latest/specifications/binary-distribution-format/#file-name-convention +func (p Package) Version() string { + return GetPyPIVersion(p.Name) +} + +func (p Package) String() string { + return fmt.Sprintf("Name: %s, Version: %s, URL: %s, RequiresPython: %s", p.Name, p.Version(), p.URL(), + p.RequiresPython()) +} + +func GetPyPIVersion(filename string) string { + base, ext, err := stripRecognizedExtension(filename) + if err != nil { + return "" + } + + splits := strings.Split(base, "-") + if len(splits) < 2 { + return "" + } + + switch ext { + case ".whl", ".egg": + return splits[1] + case ".tar.gz", ".tar.bz2", ".tar.xz", ".zip", ".dmg", ".app", ".exe": + return splits[len(splits)-1] + default: + return "" + } +} + +func stripRecognizedExtension(filename string) (string, string, error) { + for _, x := range extensions { + if strings.HasSuffix(strings.ToLower(filename), x) { + base := filename[:len(filename)-len(x)] + return base, x, nil + } + } + + return "", "", errors.New("unrecognized file extension") +} diff --git a/registry/app/remote/adapter/commons/utils.go b/registry/app/remote/adapter/commons/utils.go new file mode 100644 index 000000000..390d68977 --- /dev/null +++ b/registry/app/remote/adapter/commons/utils.go @@ -0,0 +1,78 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package commons + +import ( + "context" + "fmt" + + "github.com/harness/gitness/app/services/refcache" + api "github.com/harness/gitness/registry/app/api/openapi/contracts/artifact" + "github.com/harness/gitness/registry/types" + "github.com/harness/gitness/secret" + + "github.com/rs/zerolog/log" +) + +func GetCredentials( + ctx context.Context, spaceFinder refcache.SpaceFinder, secretService secret.Service, reg types.UpstreamProxy, +) (accessKey string, secretKey string, isAnonymous bool, err error) { + if api.AuthType(reg.RepoAuthType) == api.AuthTypeAnonymous { + return "", "", true, nil + } + if api.AuthType(reg.RepoAuthType) == api.AuthTypeUserPassword { + secretKey, err = getSecretValue(ctx, spaceFinder, secretService, reg.SecretSpaceID, + reg.SecretIdentifier) + if err != nil { + log.Error().Err(err).Msgf("failed to get secret for registry: %s", reg.RepoKey) + return "", "", false, fmt.Errorf("failed to get secret for registry: %s", reg.RepoKey) + } + return reg.UserName, secretKey, false, nil + } + if api.AuthType(reg.RepoAuthType) == api.AuthTypeAccessKeySecretKey { + accessKey, err = getSecretValue(ctx, spaceFinder, secretService, reg.UserNameSecretSpaceID, + reg.UserNameSecretIdentifier) + if err != nil { + log.Error().Err(err).Msgf("failed to get access secret for registry: %s", reg.RepoKey) + return "", "", false, fmt.Errorf("failed to get access key for registry: %s", reg.RepoKey) + } + + secretKey, err = getSecretValue(ctx, spaceFinder, secretService, reg.SecretSpaceID, + reg.SecretIdentifier) + if err != nil { + log.Error().Err(err).Msgf("failed to get user secret for registry: %s", reg.RepoKey) + return "", "", false, fmt.Errorf("failed to get secret key for registry: %s", reg.RepoKey) + } + return accessKey, secretKey, false, nil + } + return "", "", false, fmt.Errorf("unsupported auth type: %s", reg.RepoAuthType) +} + +func getSecretValue( + ctx context.Context, spaceFinder refcache.SpaceFinder, secretService secret.Service, + secretSpaceID int64, secretSpacePath string, +) (string, error) { + spacePath, err := spaceFinder.FindByID(ctx, secretSpaceID) + if err != nil { + log.Error().Msgf("failed to find space path: %v", err) + return "", err + } + decryptSecret, err := secretService.DecryptSecret(ctx, spacePath.Path, secretSpacePath) + if err != nil { + log.Error().Msgf("failed to decrypt secret: %v", err) + return "", err + } + return decryptSecret, nil +} diff --git a/registry/app/remote/adapter/native/adapter.go b/registry/app/remote/adapter/native/adapter.go index 2e44fadbc..332b7f0f8 100644 --- a/registry/app/remote/adapter/native/adapter.go +++ b/registry/app/remote/adapter/native/adapter.go @@ -20,10 +20,10 @@ import ( "context" "github.com/harness/gitness/app/services/refcache" - api "github.com/harness/gitness/registry/app/api/openapi/contracts/artifact" "github.com/harness/gitness/registry/app/common/lib" "github.com/harness/gitness/registry/app/common/lib/errors" adp "github.com/harness/gitness/registry/app/remote/adapter" + "github.com/harness/gitness/registry/app/remote/adapter/commons" "github.com/harness/gitness/registry/app/remote/clients/registry" "github.com/harness/gitness/registry/types" "github.com/harness/gitness/secret" @@ -52,10 +52,15 @@ func NewAdapter( adapter := &Adapter{ proxy: reg, } - // Get the password: lookup secrets.secret_data using secret_identifier & secret_space_id. - password := getPwd(ctx, spaceFinder, service, reg) - username, password, url := reg.UserName, password, reg.RepoURL - adapter.Client = registry.NewClient(url, username, password, false) + + url := reg.RepoURL + accessKey, secretKey, _, err := commons.GetCredentials(ctx, spaceFinder, service, reg) + if err != nil { + log.Error().Err(err).Msgf("error getting credentials for registry: %s", reg.RepoKey) + return nil + } + + adapter.Client = registry.NewClient(url, accessKey, secretKey, false) return adapter } @@ -67,29 +72,6 @@ func NewAdapterWithAuthorizer(reg types.UpstreamProxy, authorizer lib.Authorizer } } -// getPwd: lookup secrets.secret_data using secret_identifier & secret_space_id. -func getPwd( - ctx context.Context, spaceFinder refcache.SpaceFinder, secretService secret.Service, reg types.UpstreamProxy, -) string { - if api.AuthType(reg.RepoAuthType) == api.AuthTypeUserPassword { - secretSpaceID := reg.SecretSpaceID - secretIdentifier := reg.SecretIdentifier - - spacePath, err := spaceFinder.FindByID(ctx, secretSpaceID) - if err != nil { - log.Error().Msgf("failed to find space path: %v", err) - return "" - } - decryptSecret, err := secretService.DecryptSecret(ctx, spacePath.Path, secretIdentifier) - if err != nil { - log.Error().Msgf("failed to decrypt secret: %v", err) - return "" - } - return decryptSecret - } - return "" -} - // HealthCheck checks health status of a proxy. func (a *Adapter) HealthCheck() (string, error) { return "Not implemented", nil diff --git a/registry/app/remote/adapter/pypi/adapter.go b/registry/app/remote/adapter/pypi/adapter.go new file mode 100644 index 000000000..4f7addc9a --- /dev/null +++ b/registry/app/remote/adapter/pypi/adapter.go @@ -0,0 +1,231 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pypi + +import ( + "context" + "encoding/json" + "fmt" + "io" + + "github.com/harness/gitness/app/services/refcache" + "github.com/harness/gitness/registry/app/api/openapi/contracts/artifact" + "github.com/harness/gitness/registry/app/metadata/python" + adp "github.com/harness/gitness/registry/app/remote/adapter" + "github.com/harness/gitness/registry/app/remote/adapter/commons/pypi" + "github.com/harness/gitness/registry/app/remote/adapter/native" + "github.com/harness/gitness/registry/app/remote/registry" + "github.com/harness/gitness/registry/types" + "github.com/harness/gitness/secret" + + "github.com/rs/zerolog/log" + "golang.org/x/net/html" +) + +var _ registry.PythonRegistry = (*adapter)(nil) +var _ adp.Adapter = (*adapter)(nil) + +const ( + PyPiURL = "https://pypi.org" +) + +type adapter struct { + *native.Adapter + registry types.UpstreamProxy + client *client +} + +func newAdapter( + ctx context.Context, + spaceFinder refcache.SpaceFinder, + registry types.UpstreamProxy, + service secret.Service, +) (adp.Adapter, error) { + nativeAdapter := native.NewAdapter(ctx, spaceFinder, service, registry) + c, err := newClient(ctx, registry, spaceFinder, service) + if err != nil { + return nil, err + } + + return &adapter{ + Adapter: nativeAdapter, + registry: registry, + client: c, + }, nil +} + +type factory struct { +} + +func (f *factory) Create( + ctx context.Context, spaceFinder refcache.SpaceFinder, record types.UpstreamProxy, service secret.Service, +) (adp.Adapter, error) { + return newAdapter(ctx, spaceFinder, record, service) +} + +func init() { + adapterType := string(artifact.UpstreamConfigSourcePyPi) + if err := adp.RegisterFactory(adapterType, new(factory)); err != nil { + log.Error().Stack().Err(err).Msgf("Failed to register adapter factory for %s", adapterType) + return + } + log.Info().Stack().Msgf("Registered adapter factory for %s", adapterType) +} + +func (a *adapter) GetMetadata(_ context.Context, pkg string) (*pypi.SimpleMetadata, error) { + _, readCloser, err := a.GetFile("simple/" + pkg) + if err != nil { + return nil, err + } + defer readCloser.Close() + response, err := ParsePyPISimple(readCloser) + if err != nil { + return nil, err + } + err = validateMetadata(response) + if err != nil { + return nil, err + } + + return &response, nil +} + +func validateMetadata(response pypi.SimpleMetadata) error { + for _, p := range response.Packages { + if !p.Valid() { + log.Error().Msgf("invalid package: %s", p.String()) + return fmt.Errorf("invalid package: %s", p.String()) + } + } + return nil +} + +func (a *adapter) GetPackage(ctx context.Context, pkg string, filename string) (io.ReadCloser, error) { + metadata, err := a.GetMetadata(ctx, pkg) + if err != nil { + return nil, err + } + + downloadURL := "" + + for _, p := range metadata.Packages { + if p.Name == filename { + downloadURL = p.URL() + break + } + } + + if downloadURL == "" { + return nil, fmt.Errorf("pkg: %s, filename: %s not found", pkg, filename) + } + + log.Ctx(ctx).Info().Msgf("Download URL: %s", downloadURL) + _, closer, err := a.GetFileFromURL(downloadURL) + if err != nil { + log.Ctx(ctx).Error().Err(err).Msgf("Failed to get file from URL: %s", downloadURL) + return nil, err + } + return closer, nil +} + +func (a *adapter) GetJSON(ctx context.Context, pkg string, version string) (*python.Metadata, error) { + _, readCloser, err := a.GetFile(fmt.Sprintf("pypi/%s/%s/json", pkg, version)) + if err != nil { + return nil, err + } + defer readCloser.Close() + response, err := ParseMetadata(ctx, readCloser) + if err != nil { + return nil, err + } + return &response, nil +} + +func ParseMetadata(ctx context.Context, body io.ReadCloser) (python.Metadata, error) { + bytes, err := io.ReadAll(body) + if err != nil { + return python.Metadata{}, err + } + + var response Response + if err := json.Unmarshal(bytes, &response); err != nil { + // FIXME: This is known problem where if the response fields returns null, the null is not handled. + // For eg: {"keywords":null} is not handled where "keywords" is []string + log.Ctx(ctx).Warn().Err(err).Msgf("Failed to unmarshal response") + } + + return response.Info, nil +} + +// ParsePyPISimple parses the given HTML and returns a SimpleMetadata DTO. +func ParsePyPISimple(r io.ReadCloser) (pypi.SimpleMetadata, error) { + doc, err := html.Parse(r) + if err != nil { + return pypi.SimpleMetadata{}, err + } + + var result pypi.SimpleMetadata + var packages []pypi.Package + + // Recursive function to walk the HTML nodes + var traverse func(*html.Node) + traverse = func(n *html.Node) { + if n.Type == html.ElementNode { + switch n.Data { + case "meta": + // Check for meta tag name="pypi:repository-version" + var metaName, metaContent string + for _, attr := range n.Attr { + switch attr.Key { + case "name": + metaName = attr.Val + case "content": + metaContent = attr.Val + } + } + if metaName == "pypi:repository-version" { + result.MetaName = metaName + result.Content = metaContent + } + case "title": + if n.FirstChild != nil { + result.Title = n.FirstChild.Data + } + case "a": + // Capture all attributes in a map + aMap := make(map[string]string) + for _, attr := range n.Attr { + aMap[attr.Key] = attr.Val + } + linkText := "" + if n.FirstChild != nil { + linkText = n.FirstChild.Data + } + packages = append(packages, pypi.Package{ + ATags: aMap, + Name: linkText, + }) + } + } + + for c := n.FirstChild; c != nil; c = c.NextSibling { + traverse(c) + } + } + traverse(doc) + + result.Packages = packages + return result, nil +} diff --git a/registry/app/remote/adapter/pypi/client.go b/registry/app/remote/adapter/pypi/client.go new file mode 100644 index 000000000..5b67c80b1 --- /dev/null +++ b/registry/app/remote/adapter/pypi/client.go @@ -0,0 +1,60 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pypi + +import ( + "context" + "net/http" + + "github.com/harness/gitness/app/services/refcache" + commonhttp "github.com/harness/gitness/registry/app/common/http" + "github.com/harness/gitness/registry/app/remote/adapter/commons" + "github.com/harness/gitness/registry/types" + "github.com/harness/gitness/secret" + + "github.com/rs/zerolog/log" +) + +type client struct { + client *http.Client + url string + username string + password string +} + +// newClient creates a new PyPi client. +func newClient( + ctx context.Context, + registry types.UpstreamProxy, + finder refcache.SpaceFinder, + service secret.Service, +) (*client, error) { + accessKey, secretKey, _, err := commons.GetCredentials(ctx, finder, service, registry) + if err != nil { + log.Ctx(ctx).Err(err).Msgf("error getting credentials for registry: %s %v", registry.RepoKey, err) + return nil, err + } + + c := &client{ + url: registry.RepoURL, + client: &http.Client{ + Transport: commonhttp.GetHTTPTransport(commonhttp.WithInsecure(true)), + }, + username: accessKey, + password: secretKey, + } + + return c, nil +} diff --git a/registry/app/remote/adapter/pypi/dto.go b/registry/app/remote/adapter/pypi/dto.go new file mode 100644 index 000000000..1b92e361f --- /dev/null +++ b/registry/app/remote/adapter/pypi/dto.go @@ -0,0 +1,21 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package pypi + +import "github.com/harness/gitness/registry/app/metadata/python" + +type Response struct { + Info python.Metadata `json:"info"` +} diff --git a/registry/app/remote/clients/registry/client.go b/registry/app/remote/clients/registry/client.go index 7b840be99..5b4f0389d 100644 --- a/registry/app/remote/clients/registry/client.go +++ b/registry/app/remote/clients/registry/client.go @@ -48,15 +48,6 @@ import ( ) var ( - // Cli is the global registry client instance, it targets to the backend docker registry. - Cli = func() Client { - url := "myurl" - username, password := "myusername", "mypassword" - // url, _ := config.RegistryURL() - // username, password := config.RegistryCredential() - return NewClient(url, username, password, false) - }() - accepts = []string{ v1.MediaTypeImageIndex, manifestlist.MediaTypeManifestList, @@ -158,6 +149,9 @@ type Client interface { // HeadFile Check existence of file HeadFile(filePath string) (*commons.ResponseHeaders, bool, error) + + // GetFileFromURL Download the file from URL instead of provided endpoint. Authorizer still remains the same. + GetFileFromURL(url string) (*commons.ResponseHeaders, io.ReadCloser, error) } // NewClient creates a registry client with the default authorizer which determines the auth scheme @@ -847,6 +841,21 @@ func (c *client) GetFile(filePath string) (*commons.ResponseHeaders, io.ReadClos return responseHeaders, resp.Body, nil } +func (c *client) GetFileFromURL(url string) (*commons.ResponseHeaders, io.ReadCloser, error) { + req, err := http.NewRequestWithContext(context.TODO(), http.MethodGet, + url, nil) + if err != nil { + return nil, nil, err + } + + resp, err := c.Do(req) + if err != nil { + return nil, nil, err + } + responseHeaders := utils.ParseResponseHeaders(resp) + return responseHeaders, resp.Body, nil +} + func (c *client) HeadFile(filePath string) (*commons.ResponseHeaders, bool, error) { req, err := http.NewRequestWithContext(context.TODO(), http.MethodHead, buildFileURL(c.url, filePath), nil) diff --git a/registry/app/remote/controller/proxy/python/controller.go b/registry/app/remote/controller/proxy/python/controller.go deleted file mode 100644 index 0c38eb0a3..000000000 --- a/registry/app/remote/controller/proxy/python/controller.go +++ /dev/null @@ -1,143 +0,0 @@ -// Copyright 2023 Harness, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package python - -import ( - "context" - "io" - "strings" - - "github.com/harness/gitness/app/api/request" - "github.com/harness/gitness/app/services/refcache" - "github.com/harness/gitness/registry/app/pkg/commons" - "github.com/harness/gitness/registry/app/pkg/types/python" - "github.com/harness/gitness/registry/app/storage" - cfg "github.com/harness/gitness/registry/config" - "github.com/harness/gitness/registry/types" - "github.com/harness/gitness/secret" - - "github.com/rs/zerolog/log" -) - -type controller struct { - localRegistry registryInterface - secretService secret.Service - spaceFinder refcache.SpaceFinder -} - -type Controller interface { - UseLocalFile(ctx context.Context, info python.ArtifactInfo) ( - responseHeaders *commons.ResponseHeaders, fileReader *storage.FileReader, redirectURL string, useLocal bool, - ) - - ProxyFile( - ctx context.Context, info python.ArtifactInfo, proxy types.UpstreamProxy, serveFile bool, - ) (*commons.ResponseHeaders, io.ReadCloser, error) -} - -// NewProxyController -- get the proxy controller instance. -func NewProxyController( - l registryInterface, secretService secret.Service, - spaceFinder refcache.SpaceFinder, -) Controller { - return &controller{ - localRegistry: l, - secretService: secretService, - spaceFinder: spaceFinder, - } -} - -func (c *controller) UseLocalFile(ctx context.Context, info python.ArtifactInfo) ( - responseHeaders *commons.ResponseHeaders, fileReader *storage.FileReader, redirectURL string, useLocal bool, -) { - responseHeaders, body, _, redirectURL, e := c.localRegistry.GetArtifact(ctx, info) - return responseHeaders, body, redirectURL, len(e) == 0 -} - -func (c *controller) ProxyFile( - ctx context.Context, info python.ArtifactInfo, proxy types.UpstreamProxy, serveFile bool, -) (responseHeaders *commons.ResponseHeaders, body io.ReadCloser, errs error) { - responseHeaders = &commons.ResponseHeaders{ - Headers: make(map[string]string), - } - rHelper, err := NewRemoteHelper(ctx, c.spaceFinder, c.secretService, proxy) - if err != nil { - return responseHeaders, nil, err - } - - filePath := "" - // FIXME:URGENT: - //filePath := utils.GetFilePath(info) - - filePath = strings.Trim(filePath, "/") - - if serveFile { - responseHeaders, body, err = rHelper.GetFile(filePath) - } else { - responseHeaders, _, err = rHelper.HeadFile(filePath) - } - - if err != nil { - return responseHeaders, nil, err - } - - if !serveFile { - return responseHeaders, nil, nil - } - - go func(info python.ArtifactInfo) { - // Cloning Context. - session, ok := request.AuthSessionFrom(ctx) - if !ok { - log.Error().Stack().Err(err).Msg("failed to get auth session from context") - return - } - ctx2 := request.WithAuthSession(ctx, session) - ctx2 = context.WithoutCancel(ctx2) - ctx2 = context.WithValue(ctx2, cfg.GoRoutineKey, "goRoutine") - err = c.putFileToLocal(ctx2, info, rHelper) - if err != nil { - log.Ctx(ctx2).Error().Stack().Err(err).Msgf("error while putting file to localRegistry, %v", err) - return - } - log.Ctx(ctx2).Info().Msgf("Successfully updated file "+ - "to registry: %s with file path: %s", - info.RegIdentifier, filePath) - }(info) - return responseHeaders, body, nil -} - -func (c *controller) putFileToLocal( - ctx context.Context, - info python.ArtifactInfo, - r RemoteInterface, -) error { - filePath := "" - // FIXME:URGENT: - //filePath := utils.GetFilePath(info) - - filePath = strings.Trim(filePath, "/") - - _, fileReader, err := r.GetFile(filePath) - if err != nil { - return err - } - defer fileReader.Close() - _, errs := c.localRegistry.PutArtifact(ctx, info, fileReader) - if len(errs) > 0 { - return errs[0] - } - return err -} diff --git a/registry/app/remote/controller/proxy/python/local.go b/registry/app/remote/controller/proxy/python/local.go deleted file mode 100644 index c4f12df8f..000000000 --- a/registry/app/remote/controller/proxy/python/local.go +++ /dev/null @@ -1,39 +0,0 @@ -// Copyright 2023 Harness, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package python - -import ( - "context" - "io" - - "github.com/harness/gitness/registry/app/pkg/commons" - "github.com/harness/gitness/registry/app/pkg/types/python" - "github.com/harness/gitness/registry/app/storage" -) - -type registryInterface interface { - HeadArtifact(ctx context.Context, info python.ArtifactInfo) ( - responseHeaders *commons.ResponseHeaders, errs []error, - ) - - GetArtifact(ctx context.Context, info python.ArtifactInfo) ( - responseHeaders *commons.ResponseHeaders, body *storage.FileReader, fileReader io.ReadCloser, - redirectURL string, errs []error, - ) - - PutArtifact(ctx context.Context, info python.ArtifactInfo, fileReader io.Reader) ( - responseHeaders *commons.ResponseHeaders, errs []error, - ) -} diff --git a/registry/app/remote/controller/proxy/python/remote.go b/registry/app/remote/controller/proxy/python/remote.go deleted file mode 100644 index 129680702..000000000 --- a/registry/app/remote/controller/proxy/python/remote.go +++ /dev/null @@ -1,96 +0,0 @@ -// Copyright 2023 Harness, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package python - -import ( - "context" - "io" - - "github.com/harness/gitness/app/services/refcache" - api "github.com/harness/gitness/registry/app/api/openapi/contracts/artifact" - "github.com/harness/gitness/registry/app/pkg/commons" - "github.com/harness/gitness/registry/app/remote/adapter" - "github.com/harness/gitness/registry/types" - "github.com/harness/gitness/secret" - - "github.com/rs/zerolog/log" - - _ "github.com/harness/gitness/registry/app/remote/adapter/maven" // This is required to init maven adapter -) - -const MavenCentralURL = "https://repo1.maven.org/maven2" - -// RemoteInterface defines operations related to remote repository under proxy. -type RemoteInterface interface { - // Download the file - GetFile(filePath string) (*commons.ResponseHeaders, io.ReadCloser, error) - - // Check existence of file - HeadFile(filePath string) (*commons.ResponseHeaders, bool, error) -} - -type remoteHelper struct { - registry adapter.ArtifactRegistry - upstreamProxy types.UpstreamProxy - URL string - secretService secret.Service -} - -// NewRemoteHelper create a remote interface. -func NewRemoteHelper( - ctx context.Context, spaceFinder refcache.SpaceFinder, secretService secret.Service, - proxy types.UpstreamProxy, -) (RemoteInterface, error) { - if proxy.Source == string(api.UpstreamConfigSourceMavenCentral) { - proxy.RepoURL = MavenCentralURL - } - r := &remoteHelper{ - upstreamProxy: proxy, - secretService: secretService, - } - if err := r.init(ctx, spaceFinder, string(api.UpstreamConfigSourceMavenCentral)); err != nil { - return nil, err - } - return r, nil -} - -func (r *remoteHelper) init(ctx context.Context, spaceFinder refcache.SpaceFinder, proxyType string) error { - if r.registry != nil { - return nil - } - - factory, err := adapter.GetFactory(proxyType) - if err != nil { - return err - } - adp, err := factory.Create(ctx, spaceFinder, r.upstreamProxy, r.secretService) - if err != nil { - return err - } - reg, ok := adp.(adapter.ArtifactRegistry) - if !ok { - log.Warn().Msgf("Error: adp is not of type adapter.ArtifactRegistry") - } - r.registry = reg - return nil -} - -func (r *remoteHelper) GetFile(filePath string) (*commons.ResponseHeaders, io.ReadCloser, error) { - return r.registry.GetFile(filePath) -} - -func (r *remoteHelper) HeadFile(filePath string) (*commons.ResponseHeaders, bool, error) { - return r.registry.HeadFile(filePath) -} diff --git a/registry/app/remote/registry/python.go b/registry/app/remote/registry/python.go new file mode 100644 index 000000000..7e36bfec7 --- /dev/null +++ b/registry/app/remote/registry/python.go @@ -0,0 +1,29 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package registry + +import ( + "context" + "io" + + "github.com/harness/gitness/registry/app/metadata/python" + "github.com/harness/gitness/registry/app/remote/adapter/commons/pypi" +) + +type PythonRegistry interface { + GetMetadata(ctx context.Context, pkg string) (*pypi.SimpleMetadata, error) + GetPackage(ctx context.Context, pkg string, filename string) (io.ReadCloser, error) + GetJSON(ctx context.Context, pkg string, version string) (*python.Metadata, error) +} diff --git a/registry/app/storage/blobStore.go b/registry/app/storage/blobStore.go index 5a4a24009..786c92b0b 100644 --- a/registry/app/storage/blobStore.go +++ b/registry/app/storage/blobStore.go @@ -27,7 +27,7 @@ import ( "github.com/harness/gitness/registry/app/dist_temp/dcontext" "github.com/harness/gitness/registry/app/driver" - "github.com/harness/gitness/registry/app/pkg" + "github.com/harness/gitness/registry/types" "github.com/rs/zerolog/log" ) @@ -96,8 +96,10 @@ func (bs *genericBlobStore) newBlobUpload( // Write takes a file writer and a multipart form file or file reader, // streams the file to the writer, and calculates hashes. -func (bs *genericBlobStore) Write(ctx context.Context, w driver.FileWriter, file multipart.File, - fileReader io.Reader) (pkg.FileInfo, error) { +func (bs *genericBlobStore) Write( + ctx context.Context, w driver.FileWriter, file multipart.File, + fileReader io.Reader, +) (types.FileInfo, error) { // Create new hash.Hash instances for SHA256 and SHA512 sha1Hasher := sha1.New() sha256Hasher := sha256.New() @@ -115,15 +117,15 @@ func (bs *genericBlobStore) Write(ctx context.Context, w driver.FileWriter, file totalBytesWritten, err = io.Copy(mw, file) } if err != nil { - return pkg.FileInfo{}, fmt.Errorf("failed to copy file to s3: %w", err) + return types.FileInfo{}, fmt.Errorf("failed to copy file to s3: %w", err) } err = w.Commit(ctx) if err != nil { - return pkg.FileInfo{}, err + return types.FileInfo{}, err } - return pkg.FileInfo{ + return types.FileInfo{ Sha1: fmt.Sprintf("%x", sha1Hasher.Sum(nil)), Sha256: fmt.Sprintf("%x", sha256Hasher.Sum(nil)), Sha512: fmt.Sprintf("%x", sha512Hasher.Sum(nil)), diff --git a/registry/app/storage/blobs.go b/registry/app/storage/blobs.go index b17197531..880e0c566 100644 --- a/registry/app/storage/blobs.go +++ b/registry/app/storage/blobs.go @@ -25,7 +25,7 @@ import ( "github.com/harness/gitness/registry/app/driver" "github.com/harness/gitness/registry/app/manifest" - "github.com/harness/gitness/registry/app/pkg" + "github.com/harness/gitness/registry/types" "github.com/distribution/reference" "github.com/opencontainers/go-digest" @@ -173,7 +173,8 @@ type GenericBlobStore interface { // multiple times until the BlobWriter is committed or cancelled. Create(ctx context.Context, filePath string) (driver.FileWriter, error) - Write(ctx context.Context, w driver.FileWriter, file multipart.File, fileReader io.Reader) (pkg.FileInfo, error) + // Write writes the file to the blob store. There are two ways to write the file and fileReader takes the precedence. + Write(ctx context.Context, w driver.FileWriter, file multipart.File, fileReader io.Reader) (types.FileInfo, error) Move(ctx context.Context, srcPath string, dstPath string) error Delete(ctx context.Context, filePath string) error diff --git a/registry/app/storage/ociblobstore.go b/registry/app/storage/ociblobstore.go index 78ecac2c5..1da3265ce 100644 --- a/registry/app/storage/ociblobstore.go +++ b/registry/app/storage/ociblobstore.go @@ -26,13 +26,30 @@ import ( "github.com/harness/gitness/registry/app/dist_temp/dcontext" "github.com/harness/gitness/registry/app/driver" "github.com/harness/gitness/registry/app/manifest" - "github.com/harness/gitness/registry/app/pkg/commons" "github.com/google/uuid" "github.com/opencontainers/go-digest" "github.com/rs/zerolog/log" ) +const ( + HeaderAccept = "Accept" + HeaderAuthorization = "Authorization" + HeaderCacheControl = "Cache-Control" + HeaderContentLength = "Content-Length" + HeaderContentRange = "Content-Range" + HeaderContentType = "Content-Type" + HeaderDockerContentDigest = "Docker-Content-Digest" + HeaderDockerUploadUUID = "Docker-Upload-UUID" + HeaderEtag = "Etag" + HeaderIfNoneMatch = "If-None-Match" + HeaderLink = "Link" + HeaderLocation = "Location" + HeaderOCIFiltersApplied = "OCI-Filters-Applied" + HeaderOCISubject = "OCI-Subject" + HeaderRange = "Range" +) + const blobCacheControlMaxAge = 365 * 24 * time.Hour type ociBlobStore struct { @@ -106,7 +123,7 @@ func (bs *ociBlobStore) ServeBlobInternal( } if desc.MediaType != "" { // Set the repository local content type. - headers[commons.HeaderContentType] = desc.MediaType + headers[HeaderContentType] = desc.MediaType } size := desc.Size path, err := bs.pathFn(pathPrefix, desc.Digest) @@ -135,25 +152,25 @@ func (bs *ociBlobStore) ServeBlobInternal( return nil, "", size, err } - headers[commons.HeaderEtag] = fmt.Sprintf(`"%s"`, desc.Digest) + headers[HeaderEtag] = fmt.Sprintf(`"%s"`, desc.Digest) // If-None-Match handled by ServeContent - headers[commons.HeaderCacheControl] = fmt.Sprintf( + headers[HeaderCacheControl] = fmt.Sprintf( "max-age=%.f", blobCacheControlMaxAge.Seconds(), ) - if headers[commons.HeaderDockerContentDigest] == "" { - headers[commons.HeaderDockerContentDigest] = desc.Digest.String() + if headers[HeaderDockerContentDigest] == "" { + headers[HeaderDockerContentDigest] = desc.Digest.String() } - if headers[commons.HeaderContentType] == "" { + if headers[HeaderContentType] == "" { // Set the content type if not already set. - headers[commons.HeaderContentType] = desc.MediaType + headers[HeaderContentType] = desc.MediaType } - if headers[commons.HeaderContentLength] == "" { + if headers[HeaderContentLength] == "" { // Set the content length if not already set. - headers[commons.HeaderContentLength] = fmt.Sprint(desc.Size) + headers[HeaderContentLength] = fmt.Sprint(desc.Size) } return br, "", size, err diff --git a/registry/app/store/database/util/mapper.go b/registry/app/store/database/util/mapper.go index f2d60f330..cf2d22948 100644 --- a/registry/app/store/database/util/mapper.go +++ b/registry/app/store/database/util/mapper.go @@ -20,7 +20,7 @@ import ( "strconv" "strings" - "github.com/harness/gitness/registry/app/pkg/commons" + "github.com/harness/gitness/registry/utils" ) const ID = "" @@ -44,7 +44,7 @@ func StringToInt64Arr(s string) []int64 { func StringToArrByDelimiter(s string, delimiter string) []string { var arr []string - if commons.IsEmpty(s) { + if utils.IsEmpty(s) { return arr } return strings.Split(s, delimiter) @@ -64,7 +64,7 @@ func Int64ArrToStringByDelimiter(arr []int64, delimiter string) string { func StringToInt64ArrByDelimiter(s string, delimiter string) []int64 { var arr []int64 - if commons.IsEmpty(s) { + if utils.IsEmpty(s) { return arr } for _, i := range strings.Split(s, delimiter) { diff --git a/registry/app/store/database/util/utils.go b/registry/app/store/database/util/utils.go index 81a7a22c3..42bdb4284 100644 --- a/registry/app/store/database/util/utils.go +++ b/registry/app/store/database/util/utils.go @@ -19,11 +19,11 @@ import ( "fmt" "strings" - "github.com/harness/gitness/registry/app/pkg/commons" + "github.com/harness/gitness/registry/utils" ) func GetEmptySQLString(str string) sql.NullString { - if commons.IsEmpty(str) { + if utils.IsEmpty(str) { return sql.NullString{String: str, Valid: false} } return sql.NullString{String: str, Valid: true} diff --git a/registry/types/digest.go b/registry/types/digest.go index f5a7d4083..e831fc13a 100644 --- a/registry/types/digest.go +++ b/registry/types/digest.go @@ -15,11 +15,10 @@ package types import ( + "encoding/hex" "errors" "fmt" - "github.com/harness/gitness/registry/app/store/database/util" - "github.com/opencontainers/go-digest" ) @@ -42,13 +41,17 @@ func GetDigestBytes(dgst digest.Digest) ([]byte, error) { return nil, err } - digestBytes, err := util.GetHexDecodedBytes(string(newDigest)) + digestBytes, err := GetHexDecodedBytes(string(newDigest)) if err != nil { return nil, err } return digestBytes, nil } +func GetHexDecodedBytes(s string) ([]byte, error) { + return hex.DecodeString(s) +} + // String implements the Stringer interface. func (d Digest) String() string { return string(d) diff --git a/registry/types/generic_blob.go b/registry/types/generic_blob.go index 1d1ced4f2..8300a7c5a 100644 --- a/registry/types/generic_blob.go +++ b/registry/types/generic_blob.go @@ -27,3 +27,13 @@ type GenericBlob struct { CreatedAt time.Time CreatedBy int64 } + +type FileInfo struct { + Size int64 + Sha1 string + Sha256 string + Sha512 string + MD5 string + Filename string + CreatedAt time.Time +} diff --git a/registry/utils/utils.go b/registry/utils/utils.go index e1ff5dc0d..d0223f6a3 100644 --- a/registry/utils/utils.go +++ b/registry/utils/utils.go @@ -14,7 +14,10 @@ package utils -import "strings" +import ( + "reflect" + "strings" +) func HasAnyPrefix(s string, prefixes []string) bool { for _, prefix := range prefixes { @@ -40,3 +43,23 @@ func SafeUint64(n int) uint64 { } return uint64(n) } + +func IsEmpty(slice interface{}) bool { + if slice == nil { + return true + } + val := reflect.ValueOf(slice) + + // Check if the input is a pointer + if val.Kind() == reflect.Ptr { + // Dereference the pointer + val = val.Elem() + } + + // Check if the dereferenced value is nil + if !val.IsValid() { + return true + } + + return val.Len() == 0 +} diff --git a/registry/validation/helpers.go b/registry/validation/helpers.go new file mode 100644 index 000000000..9f566403a --- /dev/null +++ b/registry/validation/helpers.go @@ -0,0 +1,54 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package validation + +import ( + "net/url" + "strings" +) + +// IsValidURL checks if URL is valid. +func IsValidURL(uri string) bool { + if u, err := url.ParseRequestURI(uri); err != nil || + (u.Scheme != "http" && u.Scheme != "https") || + !validPort(portOnly(u.Host)) { + return false + } + + return true +} + +func validPort(p string) bool { + for _, r := range []byte(p) { + if r < '0' || r > '9' { + return false + } + } + return true +} + +func portOnly(hostport string) string { + colon := strings.IndexByte(hostport, ':') + if colon == -1 { + return "" + } + if i := strings.Index(hostport, "]:"); i != -1 { + return hostport[i+len("]:"):] + } + if strings.Contains(hostport, "]") { + return "" + } + return hostport[colon+len(":"):] +}