Merge branch 'mg/import/encrypt_and_goscm' of _OKE5H2PQKOUfzFFDuD4FA/default/CODE/gitness (#435)

cmd/gitness/wire_gen.go

@@ -106,6 +106,10 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro
 	if err != nil {
 		return nil, err
 	}
+	encrypter, err := encrypt.ProvideEncrypter(config)
+	if err != nil {
+		return nil, err
+	}
 	jobStore := database.ProvideJobStore(db)
 	pubsubConfig := pubsub.ProvideConfig(config)
 	universalClient, err := server.ProvideRedis(config)
@@ -120,7 +124,7 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro
 	if err != nil {
 		return nil, err
 	}
-	repository, err := importer.ProvideRepoImporter(config, provider, gitrpcInterface, repoStore, jobScheduler, executor)
+	repository, err := importer.ProvideRepoImporter(config, provider, gitrpcInterface, repoStore, encrypter, jobScheduler, executor)
 	if err != nil {
 		return nil, err
 	}
@@ -146,10 +150,6 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro
 	spaceController := space.ProvideController(db, provider, eventsStreamer, pathUID, authorizer, pathStore, pipelineStore, secretStore, connectorStore, templateStore, spaceStore, repoStore, principalStore, repoController, membershipStore, repository)
 	triggerStore := database.ProvideTriggerStore(db)
 	pipelineController := pipeline.ProvideController(db, pathUID, pathStore, repoStore, triggerStore, authorizer, pipelineStore)
-	encrypter, err := encrypt.ProvideEncrypter(config)
-	if err != nil {
-		return nil, err
-	}
 	secretController := secret.ProvideController(db, pathUID, pathStore, encrypter, secretStore, authorizer, spaceStore)
 	triggerController := trigger.ProvideController(db, authorizer, triggerStore, pathUID, pipelineStore, repoStore)
 	connectorController := connector.ProvideController(db, pathUID, connectorStore, authorizer, spaceStore)

internal/services/importer/provider.go

@@ -24,17 +24,13 @@ type ProviderType string
 
 const (
 	ProviderTypeGitHub ProviderType = "github"
-	ProviderTypeGitHubEnterprise ProviderType = "github-enterprise"
 	ProviderTypeGitLab ProviderType = "gitlab"
-	ProviderTypeGitLabEnterprise ProviderType = "gitlab-enterprise"
 )
 
 func (p ProviderType) Enum() []any {
 	return []any{
 		ProviderTypeGitHub,
-		ProviderTypeGitHubEnterprise,
 		ProviderTypeGitLab,
-		ProviderTypeGitLabEnterprise,
 	}
 }
 
@@ -83,67 +79,48 @@ func (r *RepositoryInfo) ToRepo(
 }
 
 func getClient(provider Provider) (*scm.Client, error) {
+	if provider.Username == "" || provider.Password == "" {
+		return nil, usererror.BadRequest("scm provider authentication credentials missing")
+	}
+
+	var c *scm.Client
+	var err error
+
 	switch provider.Type {
 	case "":
-		return nil, usererror.BadRequest("provider can not be empty")
+		return nil, usererror.BadRequest("scm provider can not be empty")
 
 	case ProviderTypeGitHub:
-		c := github.NewDefault()
+		if provider.Host != "" {
-		if provider.Password != "" {
+			c, err = github.New(provider.Host)
-			c.Client = &http.Client{
-				Transport: &oauth2.Transport{
-					Source: oauth2.StaticTokenSource(&scm.Token{Token: provider.Password}),
-				},
-			}
-		}
-		return c, nil
-
-	case ProviderTypeGitHubEnterprise:
-		c, err := github.New(provider.Host)
 			if err != nil {
-			return nil, usererror.BadRequestf("provider Host invalid: %s", err.Error())
+				return nil, usererror.BadRequestf("scm provider Host invalid: %s", err.Error())
 			}
-
+		} else {
-		if provider.Password != "" {
+			c = github.NewDefault()
-			c.Client = &http.Client{
-				Transport: &oauth2.Transport{
-					Source: oauth2.StaticTokenSource(&scm.Token{Token: provider.Password}),
-				},
 		}
-		}
-		return c, nil
 
 	case ProviderTypeGitLab:
-		c := gitlab.NewDefault()
+		if provider.Host != "" {
-		if provider.Password != "" {
+			c, err = gitlab.New(provider.Host)
-			c.Client = &http.Client{
-				Transport: &oauth2.Transport{
-					Source: oauth2.StaticTokenSource(&scm.Token{Token: provider.Password}),
-				},
-			}
-		}
-
-		return c, nil
-
-	case ProviderTypeGitLabEnterprise:
-		c, err := gitlab.New(provider.Host)
 			if err != nil {
-			return nil, usererror.BadRequestf("provider Host invalid: %s", err.Error())
+				return nil, usererror.BadRequestf("scm provider Host invalid: %s", err.Error())
 			}
-
+		} else {
-		if provider.Password != "" {
+			c = gitlab.NewDefault()
-			c.Client = &http.Client{
-				Transport: &oauth2.Transport{
-					Source: oauth2.StaticTokenSource(&scm.Token{Token: provider.Password}),
-				},
 		}
-		}
-
-		return c, nil
 
 	default:
-		return nil, usererror.BadRequestf("unsupported provider: %s", provider)
+		return nil, usererror.BadRequestf("unsupported scm provider: %s", provider)
 	}
+
+	c.Client = &http.Client{
+		Transport: &oauth2.Transport{
+			Source: oauth2.StaticTokenSource(&scm.Token{Token: provider.Password}),
+		},
+	}
+
+	return c, nil
 }
 func LoadRepositoryFromProvider(ctx context.Context, provider Provider, repoSlug string) (RepositoryInfo, error) {
 	scmClient, err := getClient(provider)
@@ -155,18 +132,24 @@ func LoadRepositoryFromProvider(ctx context.Context, provider Provider, repoSlug
 		return RepositoryInfo{}, usererror.BadRequest("provider repository identifier is missing")
 	}
 
-	scmRepo, _, err := scmClient.Repositories.Find(ctx, repoSlug)
+	var statusCode int
-	if errors.Is(err, scm.ErrNotFound) {
+	scmRepo, scmResp, err := scmClient.Repositories.Find(ctx, repoSlug)
+	if scmResp != nil {
+		statusCode = scmResp.Status
+	}
+
+	if errors.Is(err, scm.ErrNotFound) || statusCode == http.StatusNotFound {
 		return RepositoryInfo{},
 			usererror.BadRequestf("repository %s not found at %s", repoSlug, provider.Type)
 	}
-	if errors.Is(err, scm.ErrNotAuthorized) {
+	if errors.Is(err, scm.ErrNotAuthorized) || statusCode == http.StatusUnauthorized {
 		return RepositoryInfo{},
 			usererror.BadRequestf("bad credentials provided for %s at %s", repoSlug, provider.Type)
 	}
-	if err != nil {
+	if err != nil || statusCode > 299 {
 		return RepositoryInfo{},
-			fmt.Errorf("failed to fetch repository %s from %s: %w", repoSlug, provider.Type, err)
+			fmt.Errorf("failed to fetch repository %s from %s, status=%d: %w",
+				repoSlug, provider.Type, statusCode, err)
 	}
 
 	return RepositoryInfo{
@@ -188,33 +171,38 @@ func LoadRepositoriesFromProviderSpace(ctx context.Context, provider Provider, s
 		return nil, usererror.BadRequest("provider space identifier is missing")
 	}
 
-	repos := make([]RepositoryInfo, 0)
+	const pageSize = 100
+	opts := scm.ListOptions{Page: 0, Size: pageSize}
 
-	const pageSize = 50
+	repos := make([]RepositoryInfo, 0)
-	page := 1
 	for {
-		scmRepos, scmResponse, err := scmClient.Repositories.ListV2(ctx, scm.RepoListOptions{
+		opts.Page++
-			ListOptions: scm.ListOptions{
+
-				URL:  "",
+		var statusCode int
-				Page: page,
+		scmRepos, scmResp, err := scmClient.Repositories.List(ctx, opts)
-				Size: pageSize,
+		if scmResp != nil {
-			},
+			statusCode = scmResp.Status
-			RepoSearchTerm: scm.RepoSearchTerm{
+		}
-				RepoName: "",
+
-				User:     spaceSlug,
+		if errors.Is(err, scm.ErrNotFound) || statusCode == http.StatusNotFound {
-			},
-		})
-		if errors.Is(err, scm.ErrNotFound) {
 			return nil, usererror.BadRequestf("space %s not found at %s", spaceSlug, provider.Type)
 		}
-		if errors.Is(err, scm.ErrNotAuthorized) {
+		if errors.Is(err, scm.ErrNotAuthorized) || statusCode == http.StatusUnauthorized {
 			return nil, usererror.BadRequestf("bad credentials provided for %s at %s", spaceSlug, provider.Type)
 		}
-		if err != nil {
+		if err != nil || statusCode > 299 {
-			return nil, fmt.Errorf("failed to fetch space %s from %s: %w", spaceSlug, provider.Type, err)
+			return nil, fmt.Errorf("failed to fetch space %s from %s, status=%d: %w",
+				spaceSlug, provider.Type, statusCode, err)
+		}
+
+		if len(scmRepos) == 0 {
+			break
 		}
 
 		for _, scmRepo := range scmRepos {
+			if scmRepo.Namespace != spaceSlug {
+				continue
+			}
 			repos = append(repos, RepositoryInfo{
 				Space:         scmRepo.Namespace,
 				UID:           scmRepo.Name,
@@ -223,12 +211,6 @@ func LoadRepositoriesFromProviderSpace(ctx context.Context, provider Provider, s
 				DefaultBranch: scmRepo.Branch,
 			})
 		}
-
-		if len(scmRepos) == 0 || page == scmResponse.Page.Last {
-			break
-		}
-
-		page++
 	}
 
 	return repos, nil

internal/services/importer/repository.go

@@ -6,6 +6,7 @@ package importer
 
 import (
 	"context"
+	"encoding/base64"
 	"encoding/json"
 	"errors"
 	"fmt"
@@ -13,6 +14,7 @@ import (
 	"strings"
 	"time"
 
+	"github.com/harness/gitness/encrypt"
 	"github.com/harness/gitness/gitrpc"
 	"github.com/harness/gitness/internal/bootstrap"
 	"github.com/harness/gitness/internal/githook"
@@ -35,6 +37,7 @@ type Repository struct {
 	urlProvider   *gitnessurl.Provider
 	git           gitrpc.Interface
 	repoStore     store.RepoStore
+	encrypter     encrypt.Encrypter
 	scheduler     *job.Scheduler
 }
 
@@ -55,27 +58,17 @@ func (r *Repository) Register(executor *job.Executor) error {
 
 // Run starts a background job that imports the provided repository from the provided clone URL.
 func (r *Repository) Run(ctx context.Context, provider Provider, repo *types.Repository, cloneURL string) error {
-	input := Input{
+	jobDef, err := r.getJobDef(*repo.ImportingJobUID, Input{
 		RepoID:   repo.ID,
 		GitUser:  provider.Username,
 		GitPass:  provider.Password,
 		CloneURL: cloneURL,
-	}
-
-	data, err := json.Marshal(input)
-	if err != nil {
-		return fmt.Errorf("failed to marshal job input json: %w", err)
-	}
-
-	strData := strings.TrimSpace(string(data))
-
-	return r.scheduler.RunJob(ctx, job.Definition{
-		UID:        *repo.ImportingJobUID,
-		Type:       jobType,
-		MaxRetries: importJobMaxRetries,
-		Timeout:    importJobMaxDuration,
-		Data:       strData,
 	})
+	if err != nil {
+		return err
+	}
+
+	return r.scheduler.RunJob(ctx, jobDef)
 }
 
 // RunMany starts background jobs that import the provided repositories from the provided clone URLs.
@@ -91,34 +84,23 @@ func (r *Repository) RunMany(ctx context.Context,
 	}
 
 	n := len(repos)
-
 	defs := make([]job.Definition, n)
 
 	for k := 0; k < n; k++ {
 		repo := repos[k]
 		cloneURL := cloneURLs[k]
 
-		input := Input{
+		jobDef, err := r.getJobDef(*repo.ImportingJobUID, Input{
 			RepoID:   repo.ID,
 			GitUser:  provider.Username,
 			GitPass:  provider.Password,
 			CloneURL: cloneURL,
-		}
+		})
-
-		data, err := json.Marshal(input)
 		if err != nil {
-			return fmt.Errorf("failed to marshal job input json: %w", err)
+			return err
 		}
 
-		strData := strings.TrimSpace(string(data))
+		defs[k] = jobDef
-
-		defs[k] = job.Definition{
-			UID:        *repo.ImportingJobUID,
-			Type:       jobType,
-			MaxRetries: importJobMaxRetries,
-			Timeout:    importJobMaxDuration,
-			Data:       strData,
-		}
 	}
 
 	err := r.scheduler.RunJobs(ctx, groupID, defs)
@@ -129,13 +111,56 @@ func (r *Repository) RunMany(ctx context.Context,
 	return nil
 }
 
+func (r *Repository) getJobDef(jobUID string, input Input) (job.Definition, error) {
+	data, err := json.Marshal(input)
+	if err != nil {
+		return job.Definition{}, fmt.Errorf("failed to marshal job input json: %w", err)
+	}
+
+	strData := strings.TrimSpace(string(data))
+
+	encryptedData, err := r.encrypter.Encrypt(strData)
+	if err != nil {
+		return job.Definition{}, fmt.Errorf("failed to encrypt job input: %w", err)
+	}
+
+	return job.Definition{
+		UID:        jobUID,
+		Type:       jobType,
+		MaxRetries: importJobMaxRetries,
+		Timeout:    importJobMaxDuration,
+		Data:       base64.StdEncoding.EncodeToString(encryptedData),
+	}, nil
+}
+
+func (r *Repository) getJobInput(data string) (Input, error) {
+	encrypted, err := base64.StdEncoding.DecodeString(data)
+	if err != nil {
+		return Input{}, fmt.Errorf("failed to base64 decode job input: %w", err)
+	}
+
+	decrypted, err := r.encrypter.Decrypt(encrypted)
+	if err != nil {
+		return Input{}, fmt.Errorf("failed to decrypt job input: %w", err)
+	}
+
+	var input Input
+
+	err = json.NewDecoder(strings.NewReader(decrypted)).Decode(&input)
+	if err != nil {
+		return Input{}, fmt.Errorf("failed to unmarshal job input json: %w", err)
+	}
+
+	return input, nil
+}
+
 // Handle is repository import background job handler.
 func (r *Repository) Handle(ctx context.Context, data string, _ job.ProgressReporter) (string, error) {
 	systemPrincipal := bootstrap.NewSystemServiceSession().Principal
 
-	var input Input
+	input, err := r.getJobInput(data)
-	if err := json.NewDecoder(strings.NewReader(data)).Decode(&input); err != nil {
+	if err != nil {
-		return "", fmt.Errorf("failed to unmarshal job input: %w", err)
+		return "", err
 	}
 
 	if input.CloneURL == "" {

internal/services/importer/wire.go

@@ -5,6 +5,7 @@
 package importer
 
 import (
+	"github.com/harness/gitness/encrypt"
 	"github.com/harness/gitness/gitrpc"
 	"github.com/harness/gitness/internal/services/job"
 	"github.com/harness/gitness/internal/store"
@@ -23,6 +24,7 @@ func ProvideRepoImporter(
 	urlProvider *url.Provider,
 	git gitrpc.Interface,
 	repoStore store.RepoStore,
+	encrypter encrypt.Encrypter,
 	scheduler *job.Scheduler,
 	executor *job.Executor,
 ) (*Repository, error) {
@@ -31,6 +33,7 @@ func ProvideRepoImporter(
 		urlProvider:   urlProvider,
 		git:           git,
 		repoStore:     repoStore,
+		encrypter:     encrypter,
 		scheduler:     scheduler,
 	}