diff --git a/cmd/gitness/wire_gen.go b/cmd/gitness/wire_gen.go index 44d295df0..39121cf0b 100644 --- a/cmd/gitness/wire_gen.go +++ b/cmd/gitness/wire_gen.go @@ -106,6 +106,10 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro if err != nil { return nil, err } + encrypter, err := encrypt.ProvideEncrypter(config) + if err != nil { + return nil, err + } jobStore := database.ProvideJobStore(db) pubsubConfig := pubsub.ProvideConfig(config) universalClient, err := server.ProvideRedis(config) @@ -120,7 +124,7 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro if err != nil { return nil, err } - repository, err := importer.ProvideRepoImporter(config, provider, gitrpcInterface, repoStore, jobScheduler, executor) + repository, err := importer.ProvideRepoImporter(config, provider, gitrpcInterface, repoStore, encrypter, jobScheduler, executor) if err != nil { return nil, err } @@ -146,10 +150,6 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro spaceController := space.ProvideController(db, provider, eventsStreamer, pathUID, authorizer, pathStore, pipelineStore, secretStore, connectorStore, templateStore, spaceStore, repoStore, principalStore, repoController, membershipStore, repository) triggerStore := database.ProvideTriggerStore(db) pipelineController := pipeline.ProvideController(db, pathUID, pathStore, repoStore, triggerStore, authorizer, pipelineStore) - encrypter, err := encrypt.ProvideEncrypter(config) - if err != nil { - return nil, err - } secretController := secret.ProvideController(db, pathUID, pathStore, encrypter, secretStore, authorizer, spaceStore) triggerController := trigger.ProvideController(db, authorizer, triggerStore, pathUID, pipelineStore, repoStore) connectorController := connector.ProvideController(db, pathUID, connectorStore, authorizer, spaceStore) diff --git a/internal/services/importer/provider.go b/internal/services/importer/provider.go index 3a50d25b9..6fcc675c3 100644 --- a/internal/services/importer/provider.go +++ b/internal/services/importer/provider.go @@ -23,18 +23,14 @@ import ( type ProviderType string const ( - ProviderTypeGitHub ProviderType = "github" - ProviderTypeGitHubEnterprise ProviderType = "github-enterprise" - ProviderTypeGitLab ProviderType = "gitlab" - ProviderTypeGitLabEnterprise ProviderType = "gitlab-enterprise" + ProviderTypeGitHub ProviderType = "github" + ProviderTypeGitLab ProviderType = "gitlab" ) func (p ProviderType) Enum() []any { return []any{ ProviderTypeGitHub, - ProviderTypeGitHubEnterprise, ProviderTypeGitLab, - ProviderTypeGitLabEnterprise, } } @@ -83,67 +79,48 @@ func (r *RepositoryInfo) ToRepo( } func getClient(provider Provider) (*scm.Client, error) { + if provider.Username == "" || provider.Password == "" { + return nil, usererror.BadRequest("scm provider authentication credentials missing") + } + + var c *scm.Client + var err error + switch provider.Type { case "": - return nil, usererror.BadRequest("provider can not be empty") + return nil, usererror.BadRequest("scm provider can not be empty") case ProviderTypeGitHub: - c := github.NewDefault() - if provider.Password != "" { - c.Client = &http.Client{ - Transport: &oauth2.Transport{ - Source: oauth2.StaticTokenSource(&scm.Token{Token: provider.Password}), - }, + if provider.Host != "" { + c, err = github.New(provider.Host) + if err != nil { + return nil, usererror.BadRequestf("scm provider Host invalid: %s", err.Error()) } + } else { + c = github.NewDefault() } - return c, nil - - case ProviderTypeGitHubEnterprise: - c, err := github.New(provider.Host) - if err != nil { - return nil, usererror.BadRequestf("provider Host invalid: %s", err.Error()) - } - - if provider.Password != "" { - c.Client = &http.Client{ - Transport: &oauth2.Transport{ - Source: oauth2.StaticTokenSource(&scm.Token{Token: provider.Password}), - }, - } - } - return c, nil case ProviderTypeGitLab: - c := gitlab.NewDefault() - if provider.Password != "" { - c.Client = &http.Client{ - Transport: &oauth2.Transport{ - Source: oauth2.StaticTokenSource(&scm.Token{Token: provider.Password}), - }, + if provider.Host != "" { + c, err = gitlab.New(provider.Host) + if err != nil { + return nil, usererror.BadRequestf("scm provider Host invalid: %s", err.Error()) } + } else { + c = gitlab.NewDefault() } - return c, nil - - case ProviderTypeGitLabEnterprise: - c, err := gitlab.New(provider.Host) - if err != nil { - return nil, usererror.BadRequestf("provider Host invalid: %s", err.Error()) - } - - if provider.Password != "" { - c.Client = &http.Client{ - Transport: &oauth2.Transport{ - Source: oauth2.StaticTokenSource(&scm.Token{Token: provider.Password}), - }, - } - } - - return c, nil - default: - return nil, usererror.BadRequestf("unsupported provider: %s", provider) + return nil, usererror.BadRequestf("unsupported scm provider: %s", provider) } + + c.Client = &http.Client{ + Transport: &oauth2.Transport{ + Source: oauth2.StaticTokenSource(&scm.Token{Token: provider.Password}), + }, + } + + return c, nil } func LoadRepositoryFromProvider(ctx context.Context, provider Provider, repoSlug string) (RepositoryInfo, error) { scmClient, err := getClient(provider) @@ -155,18 +132,24 @@ func LoadRepositoryFromProvider(ctx context.Context, provider Provider, repoSlug return RepositoryInfo{}, usererror.BadRequest("provider repository identifier is missing") } - scmRepo, _, err := scmClient.Repositories.Find(ctx, repoSlug) - if errors.Is(err, scm.ErrNotFound) { + var statusCode int + scmRepo, scmResp, err := scmClient.Repositories.Find(ctx, repoSlug) + if scmResp != nil { + statusCode = scmResp.Status + } + + if errors.Is(err, scm.ErrNotFound) || statusCode == http.StatusNotFound { return RepositoryInfo{}, usererror.BadRequestf("repository %s not found at %s", repoSlug, provider.Type) } - if errors.Is(err, scm.ErrNotAuthorized) { + if errors.Is(err, scm.ErrNotAuthorized) || statusCode == http.StatusUnauthorized { return RepositoryInfo{}, usererror.BadRequestf("bad credentials provided for %s at %s", repoSlug, provider.Type) } - if err != nil { + if err != nil || statusCode > 299 { return RepositoryInfo{}, - fmt.Errorf("failed to fetch repository %s from %s: %w", repoSlug, provider.Type, err) + fmt.Errorf("failed to fetch repository %s from %s, status=%d: %w", + repoSlug, provider.Type, statusCode, err) } return RepositoryInfo{ @@ -188,33 +171,38 @@ func LoadRepositoriesFromProviderSpace(ctx context.Context, provider Provider, s return nil, usererror.BadRequest("provider space identifier is missing") } - repos := make([]RepositoryInfo, 0) + const pageSize = 100 + opts := scm.ListOptions{Page: 0, Size: pageSize} - const pageSize = 50 - page := 1 + repos := make([]RepositoryInfo, 0) for { - scmRepos, scmResponse, err := scmClient.Repositories.ListV2(ctx, scm.RepoListOptions{ - ListOptions: scm.ListOptions{ - URL: "", - Page: page, - Size: pageSize, - }, - RepoSearchTerm: scm.RepoSearchTerm{ - RepoName: "", - User: spaceSlug, - }, - }) - if errors.Is(err, scm.ErrNotFound) { + opts.Page++ + + var statusCode int + scmRepos, scmResp, err := scmClient.Repositories.List(ctx, opts) + if scmResp != nil { + statusCode = scmResp.Status + } + + if errors.Is(err, scm.ErrNotFound) || statusCode == http.StatusNotFound { return nil, usererror.BadRequestf("space %s not found at %s", spaceSlug, provider.Type) } - if errors.Is(err, scm.ErrNotAuthorized) { + if errors.Is(err, scm.ErrNotAuthorized) || statusCode == http.StatusUnauthorized { return nil, usererror.BadRequestf("bad credentials provided for %s at %s", spaceSlug, provider.Type) } - if err != nil { - return nil, fmt.Errorf("failed to fetch space %s from %s: %w", spaceSlug, provider.Type, err) + if err != nil || statusCode > 299 { + return nil, fmt.Errorf("failed to fetch space %s from %s, status=%d: %w", + spaceSlug, provider.Type, statusCode, err) + } + + if len(scmRepos) == 0 { + break } for _, scmRepo := range scmRepos { + if scmRepo.Namespace != spaceSlug { + continue + } repos = append(repos, RepositoryInfo{ Space: scmRepo.Namespace, UID: scmRepo.Name, @@ -223,12 +211,6 @@ func LoadRepositoriesFromProviderSpace(ctx context.Context, provider Provider, s DefaultBranch: scmRepo.Branch, }) } - - if len(scmRepos) == 0 || page == scmResponse.Page.Last { - break - } - - page++ } return repos, nil diff --git a/internal/services/importer/repository.go b/internal/services/importer/repository.go index a6e9add4a..028fd175e 100644 --- a/internal/services/importer/repository.go +++ b/internal/services/importer/repository.go @@ -6,6 +6,7 @@ package importer import ( "context" + "encoding/base64" "encoding/json" "errors" "fmt" @@ -13,6 +14,7 @@ import ( "strings" "time" + "github.com/harness/gitness/encrypt" "github.com/harness/gitness/gitrpc" "github.com/harness/gitness/internal/bootstrap" "github.com/harness/gitness/internal/githook" @@ -35,6 +37,7 @@ type Repository struct { urlProvider *gitnessurl.Provider git gitrpc.Interface repoStore store.RepoStore + encrypter encrypt.Encrypter scheduler *job.Scheduler } @@ -55,27 +58,17 @@ func (r *Repository) Register(executor *job.Executor) error { // Run starts a background job that imports the provided repository from the provided clone URL. func (r *Repository) Run(ctx context.Context, provider Provider, repo *types.Repository, cloneURL string) error { - input := Input{ + jobDef, err := r.getJobDef(*repo.ImportingJobUID, Input{ RepoID: repo.ID, GitUser: provider.Username, GitPass: provider.Password, CloneURL: cloneURL, - } - - data, err := json.Marshal(input) - if err != nil { - return fmt.Errorf("failed to marshal job input json: %w", err) - } - - strData := strings.TrimSpace(string(data)) - - return r.scheduler.RunJob(ctx, job.Definition{ - UID: *repo.ImportingJobUID, - Type: jobType, - MaxRetries: importJobMaxRetries, - Timeout: importJobMaxDuration, - Data: strData, }) + if err != nil { + return err + } + + return r.scheduler.RunJob(ctx, jobDef) } // RunMany starts background jobs that import the provided repositories from the provided clone URLs. @@ -91,34 +84,23 @@ func (r *Repository) RunMany(ctx context.Context, } n := len(repos) - defs := make([]job.Definition, n) for k := 0; k < n; k++ { repo := repos[k] cloneURL := cloneURLs[k] - input := Input{ + jobDef, err := r.getJobDef(*repo.ImportingJobUID, Input{ RepoID: repo.ID, GitUser: provider.Username, GitPass: provider.Password, CloneURL: cloneURL, - } - - data, err := json.Marshal(input) + }) if err != nil { - return fmt.Errorf("failed to marshal job input json: %w", err) + return err } - strData := strings.TrimSpace(string(data)) - - defs[k] = job.Definition{ - UID: *repo.ImportingJobUID, - Type: jobType, - MaxRetries: importJobMaxRetries, - Timeout: importJobMaxDuration, - Data: strData, - } + defs[k] = jobDef } err := r.scheduler.RunJobs(ctx, groupID, defs) @@ -129,13 +111,56 @@ func (r *Repository) RunMany(ctx context.Context, return nil } +func (r *Repository) getJobDef(jobUID string, input Input) (job.Definition, error) { + data, err := json.Marshal(input) + if err != nil { + return job.Definition{}, fmt.Errorf("failed to marshal job input json: %w", err) + } + + strData := strings.TrimSpace(string(data)) + + encryptedData, err := r.encrypter.Encrypt(strData) + if err != nil { + return job.Definition{}, fmt.Errorf("failed to encrypt job input: %w", err) + } + + return job.Definition{ + UID: jobUID, + Type: jobType, + MaxRetries: importJobMaxRetries, + Timeout: importJobMaxDuration, + Data: base64.StdEncoding.EncodeToString(encryptedData), + }, nil +} + +func (r *Repository) getJobInput(data string) (Input, error) { + encrypted, err := base64.StdEncoding.DecodeString(data) + if err != nil { + return Input{}, fmt.Errorf("failed to base64 decode job input: %w", err) + } + + decrypted, err := r.encrypter.Decrypt(encrypted) + if err != nil { + return Input{}, fmt.Errorf("failed to decrypt job input: %w", err) + } + + var input Input + + err = json.NewDecoder(strings.NewReader(decrypted)).Decode(&input) + if err != nil { + return Input{}, fmt.Errorf("failed to unmarshal job input json: %w", err) + } + + return input, nil +} + // Handle is repository import background job handler. func (r *Repository) Handle(ctx context.Context, data string, _ job.ProgressReporter) (string, error) { systemPrincipal := bootstrap.NewSystemServiceSession().Principal - var input Input - if err := json.NewDecoder(strings.NewReader(data)).Decode(&input); err != nil { - return "", fmt.Errorf("failed to unmarshal job input: %w", err) + input, err := r.getJobInput(data) + if err != nil { + return "", err } if input.CloneURL == "" { diff --git a/internal/services/importer/wire.go b/internal/services/importer/wire.go index 44328b4bf..cdeb81623 100644 --- a/internal/services/importer/wire.go +++ b/internal/services/importer/wire.go @@ -5,6 +5,7 @@ package importer import ( + "github.com/harness/gitness/encrypt" "github.com/harness/gitness/gitrpc" "github.com/harness/gitness/internal/services/job" "github.com/harness/gitness/internal/store" @@ -23,6 +24,7 @@ func ProvideRepoImporter( urlProvider *url.Provider, git gitrpc.Interface, repoStore store.RepoStore, + encrypter encrypt.Encrypter, scheduler *job.Scheduler, executor *job.Executor, ) (*Repository, error) { @@ -31,6 +33,7 @@ func ProvideRepoImporter( urlProvider: urlProvider, git: git, repoStore: repoStore, + encrypter: encrypter, scheduler: scheduler, }