github/drone
5
0
Fork 0
mirror of https://github.com/harness/drone.git synced 2025-05-07 02:30:08 +08:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'mg/import/encrypt_and_goscm' of _OKE5H2PQKOUfzFFDuD4FA/default/CODE/gitness (#435)

Browse Source
This commit is contained in:
Marko Gacesa 2023-09-12 13:10:00 +00:00 committed by Harness
commit fd7ed1e496
4 changed files with 132 additions and 122 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
cmd/gitness/wire_gen.go
View File

@ -106,6 +106,10 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro
if err != nil {
return nil, err
}
encrypter, err := encrypt.ProvideEncrypter(config)
if err != nil {
return nil, err
}
jobStore := database.ProvideJobStore(db)
pubsubConfig := pubsub.ProvideConfig(config)
universalClient, err := server.ProvideRedis(config)
@ -120,7 +124,7 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro
if err != nil {
return nil, err
}
repository, err := importer.ProvideRepoImporter(config, provider, gitrpcInterface, repoStore, jobScheduler, executor)
repository, err := importer.ProvideRepoImporter(config, provider, gitrpcInterface, repoStore, encrypter, jobScheduler, executor)
if err != nil {
return nil, err
}
@ -146,10 +150,6 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro
spaceController := space.ProvideController(db, provider, eventsStreamer, pathUID, authorizer, pathStore, pipelineStore, secretStore, connectorStore, templateStore, spaceStore, repoStore, principalStore, repoController, membershipStore, repository)
triggerStore := database.ProvideTriggerStore(db)
pipelineController := pipeline.ProvideController(db, pathUID, pathStore, repoStore, triggerStore, authorizer, pipelineStore)
encrypter, err := encrypt.ProvideEncrypter(config)
if err != nil {
return nil, err
}
secretController := secret.ProvideController(db, pathUID, pathStore, encrypter, secretStore, authorizer, spaceStore)
triggerController := trigger.ProvideController(db, authorizer, triggerStore, pathUID, pipelineStore, repoStore)
connectorController := connector.ProvideController(db, pathUID, connectorStore, authorizer, spaceStore)

140
internal/services/importer/provider.go
View File

@ -24,17 +24,13 @@ type ProviderType string
const (
ProviderTypeGitHub ProviderType = "github"
ProviderTypeGitHubEnterprise ProviderType = "github-enterprise"
ProviderTypeGitLab ProviderType = "gitlab"
ProviderTypeGitLabEnterprise ProviderType = "gitlab-enterprise"
)
func (p ProviderType) Enum() []any {
return []any{
ProviderTypeGitHub,
ProviderTypeGitHubEnterprise,
ProviderTypeGitLab,
ProviderTypeGitLabEnterprise,
}
}
@ -83,67 +79,48 @@ func (r *RepositoryInfo) ToRepo(
}
func getClient(provider Provider) (*scm.Client, error) {
if provider.Username == "" || provider.Password == "" {
return nil, usererror.BadRequest("scm provider authentication credentials missing")
}
var c *scm.Client
var err error
switch provider.Type {
case "":
return nil, usererror.BadRequest("provider can not be empty")
return nil, usererror.BadRequest("scm provider can not be empty")
case ProviderTypeGitHub:
c := github.NewDefault()
if provider.Password != "" {
c.Client = &http.Client{
Transport: &oauth2.Transport{
Source: oauth2.StaticTokenSource(&scm.Token{Token: provider.Password}),
},
}
}
return c, nil
case ProviderTypeGitHubEnterprise:
c, err := github.New(provider.Host)
if provider.Host != "" {
c, err = github.New(provider.Host)
if err != nil {
return nil, usererror.BadRequestf("provider Host invalid: %s", err.Error())
return nil, usererror.BadRequestf("scm provider Host invalid: %s", err.Error())
}
if provider.Password != "" {
c.Client = &http.Client{
Transport: &oauth2.Transport{
Source: oauth2.StaticTokenSource(&scm.Token{Token: provider.Password}),
},
} else {
c = github.NewDefault()
}
}
return c, nil
case ProviderTypeGitLab:
c := gitlab.NewDefault()
if provider.Password != "" {
c.Client = &http.Client{
Transport: &oauth2.Transport{
Source: oauth2.StaticTokenSource(&scm.Token{Token: provider.Password}),
},
}
}
return c, nil
case ProviderTypeGitLabEnterprise:
c, err := gitlab.New(provider.Host)
if provider.Host != "" {
c, err = gitlab.New(provider.Host)
if err != nil {
return nil, usererror.BadRequestf("provider Host invalid: %s", err.Error())
return nil, usererror.BadRequestf("scm provider Host invalid: %s", err.Error())
}
if provider.Password != "" {
c.Client = &http.Client{
Transport: &oauth2.Transport{
Source: oauth2.StaticTokenSource(&scm.Token{Token: provider.Password}),
},
} else {
c = gitlab.NewDefault()
}
}
return c, nil
default:
return nil, usererror.BadRequestf("unsupported provider: %s", provider)
return nil, usererror.BadRequestf("unsupported scm provider: %s", provider)
}
c.Client = &http.Client{
Transport: &oauth2.Transport{
Source: oauth2.StaticTokenSource(&scm.Token{Token: provider.Password}),
},
}
return c, nil
}
func LoadRepositoryFromProvider(ctx context.Context, provider Provider, repoSlug string) (RepositoryInfo, error) {
scmClient, err := getClient(provider)
@ -155,18 +132,24 @@ func LoadRepositoryFromProvider(ctx context.Context, provider Provider, repoSlug
return RepositoryInfo{}, usererror.BadRequest("provider repository identifier is missing")
}
scmRepo, _, err := scmClient.Repositories.Find(ctx, repoSlug)
if errors.Is(err, scm.ErrNotFound) {
var statusCode int
scmRepo, scmResp, err := scmClient.Repositories.Find(ctx, repoSlug)
if scmResp != nil {
statusCode = scmResp.Status
}
if errors.Is(err, scm.ErrNotFound) || statusCode == http.StatusNotFound {
return RepositoryInfo{},
usererror.BadRequestf("repository %s not found at %s", repoSlug, provider.Type)
}
if errors.Is(err, scm.ErrNotAuthorized) {
if errors.Is(err, scm.ErrNotAuthorized) || statusCode == http.StatusUnauthorized {
return RepositoryInfo{},
usererror.BadRequestf("bad credentials provided for %s at %s", repoSlug, provider.Type)
}
if err != nil {
if err != nil || statusCode > 299 {
return RepositoryInfo{},
fmt.Errorf("failed to fetch repository %s from %s: %w", repoSlug, provider.Type, err)
fmt.Errorf("failed to fetch repository %s from %s, status=%d: %w",
repoSlug, provider.Type, statusCode, err)
}
return RepositoryInfo{
@ -188,33 +171,38 @@ func LoadRepositoriesFromProviderSpace(ctx context.Context, provider Provider, s
return nil, usererror.BadRequest("provider space identifier is missing")
}
repos := make([]RepositoryInfo, 0)
const pageSize = 100
opts := scm.ListOptions{Page: 0, Size: pageSize}
const pageSize = 50
page := 1
repos := make([]RepositoryInfo, 0)
for {
scmRepos, scmResponse, err := scmClient.Repositories.ListV2(ctx, scm.RepoListOptions{
ListOptions: scm.ListOptions{
URL: "",
Page: page,
Size: pageSize,
},
RepoSearchTerm: scm.RepoSearchTerm{
RepoName: "",
User: spaceSlug,
},
})
if errors.Is(err, scm.ErrNotFound) {
opts.Page++
var statusCode int
scmRepos, scmResp, err := scmClient.Repositories.List(ctx, opts)
if scmResp != nil {
statusCode = scmResp.Status
}
if errors.Is(err, scm.ErrNotFound) || statusCode == http.StatusNotFound {
return nil, usererror.BadRequestf("space %s not found at %s", spaceSlug, provider.Type)
}
if errors.Is(err, scm.ErrNotAuthorized) {
if errors.Is(err, scm.ErrNotAuthorized) || statusCode == http.StatusUnauthorized {
return nil, usererror.BadRequestf("bad credentials provided for %s at %s", spaceSlug, provider.Type)
}
if err != nil {
return nil, fmt.Errorf("failed to fetch space %s from %s: %w", spaceSlug, provider.Type, err)
if err != nil || statusCode > 299 {
return nil, fmt.Errorf("failed to fetch space %s from %s, status=%d: %w",
spaceSlug, provider.Type, statusCode, err)
}
if len(scmRepos) == 0 {
break
}
for _, scmRepo := range scmRepos {
if scmRepo.Namespace != spaceSlug {
continue
}
repos = append(repos, RepositoryInfo{
Space: scmRepo.Namespace,
UID: scmRepo.Name,
@ -223,12 +211,6 @@ func LoadRepositoriesFromProviderSpace(ctx context.Context, provider Provider, s
DefaultBranch: scmRepo.Branch,
})
}
if len(scmRepos) == 0 || page == scmResponse.Page.Last {
break
}
page++
}
return repos, nil

93
internal/services/importer/repository.go
View File

@ -6,6 +6,7 @@ package importer
import (
"context"
"encoding/base64"
"encoding/json"
"errors"
"fmt"
@ -13,6 +14,7 @@ import (
"strings"
"time"
"github.com/harness/gitness/encrypt"
"github.com/harness/gitness/gitrpc"
"github.com/harness/gitness/internal/bootstrap"
"github.com/harness/gitness/internal/githook"
@ -35,6 +37,7 @@ type Repository struct {
urlProvider *gitnessurl.Provider
git gitrpc.Interface
repoStore store.RepoStore
encrypter encrypt.Encrypter
scheduler *job.Scheduler
}
@ -55,27 +58,17 @@ func (r *Repository) Register(executor *job.Executor) error {
// Run starts a background job that imports the provided repository from the provided clone URL.
func (r *Repository) Run(ctx context.Context, provider Provider, repo *types.Repository, cloneURL string) error {
input := Input{
jobDef, err := r.getJobDef(*repo.ImportingJobUID, Input{
RepoID: repo.ID,
GitUser: provider.Username,
GitPass: provider.Password,
CloneURL: cloneURL,
}
data, err := json.Marshal(input)
if err != nil {
return fmt.Errorf("failed to marshal job input json: %w", err)
}
strData := strings.TrimSpace(string(data))
return r.scheduler.RunJob(ctx, job.Definition{
UID: *repo.ImportingJobUID,
Type: jobType,
MaxRetries: importJobMaxRetries,
Timeout: importJobMaxDuration,
Data: strData,
})
if err != nil {
return err
}
return r.scheduler.RunJob(ctx, jobDef)
}
// RunMany starts background jobs that import the provided repositories from the provided clone URLs.
@ -91,34 +84,23 @@ func (r *Repository) RunMany(ctx context.Context,
}
n := len(repos)
defs := make([]job.Definition, n)
for k := 0; k < n; k++ {
repo := repos[k]
cloneURL := cloneURLs[k]
input := Input{
jobDef, err := r.getJobDef(*repo.ImportingJobUID, Input{
RepoID: repo.ID,
GitUser: provider.Username,
GitPass: provider.Password,
CloneURL: cloneURL,
}
data, err := json.Marshal(input)
})
if err != nil {
return fmt.Errorf("failed to marshal job input json: %w", err)
return err
}
strData := strings.TrimSpace(string(data))
defs[k] = job.Definition{
UID: *repo.ImportingJobUID,
Type: jobType,
MaxRetries: importJobMaxRetries,
Timeout: importJobMaxDuration,
Data: strData,
}
defs[k] = jobDef
}
err := r.scheduler.RunJobs(ctx, groupID, defs)
@ -129,13 +111,56 @@ func (r *Repository) RunMany(ctx context.Context,
return nil
}
func (r *Repository) getJobDef(jobUID string, input Input) (job.Definition, error) {
data, err := json.Marshal(input)
if err != nil {
return job.Definition{}, fmt.Errorf("failed to marshal job input json: %w", err)
}
strData := strings.TrimSpace(string(data))
encryptedData, err := r.encrypter.Encrypt(strData)
if err != nil {
return job.Definition{}, fmt.Errorf("failed to encrypt job input: %w", err)
}
return job.Definition{
UID: jobUID,
Type: jobType,
MaxRetries: importJobMaxRetries,
Timeout: importJobMaxDuration,
Data: base64.StdEncoding.EncodeToString(encryptedData),
}, nil
}
func (r *Repository) getJobInput(data string) (Input, error) {
encrypted, err := base64.StdEncoding.DecodeString(data)
if err != nil {
return Input{}, fmt.Errorf("failed to base64 decode job input: %w", err)
}
decrypted, err := r.encrypter.Decrypt(encrypted)
if err != nil {
return Input{}, fmt.Errorf("failed to decrypt job input: %w", err)
}
var input Input
err = json.NewDecoder(strings.NewReader(decrypted)).Decode(&input)
if err != nil {
return Input{}, fmt.Errorf("failed to unmarshal job input json: %w", err)
}
return input, nil
}
// Handle is repository import background job handler.
func (r *Repository) Handle(ctx context.Context, data string, _ job.ProgressReporter) (string, error) {
systemPrincipal := bootstrap.NewSystemServiceSession().Principal
var input Input
if err := json.NewDecoder(strings.NewReader(data)).Decode(&input); err != nil {
return "", fmt.Errorf("failed to unmarshal job input: %w", err)
input, err := r.getJobInput(data)
if err != nil {
return "", err
}
if input.CloneURL == "" {

3
internal/services/importer/wire.go
View File

@ -5,6 +5,7 @@
package importer
import (
"github.com/harness/gitness/encrypt"
"github.com/harness/gitness/gitrpc"
"github.com/harness/gitness/internal/services/job"
"github.com/harness/gitness/internal/store"
@ -23,6 +24,7 @@ func ProvideRepoImporter(
urlProvider *url.Provider,
git gitrpc.Interface,
repoStore store.RepoStore,
encrypter encrypt.Encrypter,
scheduler *job.Scheduler,
executor *job.Executor,
) (*Repository, error) {
@ -31,6 +33,7 @@ func ProvideRepoImporter(
urlProvider: urlProvider,
git: git,
repoStore: repoStore,
encrypter: encrypter,
scheduler: scheduler,
}