From cde80d53a8032437dffd7ee65bb74971103463f8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marko=20Ga=C4=87e=C5=A1a?= Date: Fri, 8 Sep 2023 17:35:50 +0200 Subject: [PATCH] add space import --- cmd/gitness/wire_gen.go | 4 +- internal/api/controller/repo/blame.go | 10 +- internal/api/controller/repo/commit.go | 14 +- internal/api/controller/repo/content_get.go | 7 +- .../controller/repo/content_paths_details.go | 13 +- internal/api/controller/repo/controller.go | 32 +++ internal/api/controller/repo/create_branch.go | 14 +- .../api/controller/repo/create_commit_tag.go | 14 +- internal/api/controller/repo/create_path.go | 14 +- internal/api/controller/repo/delete.go | 13 + internal/api/controller/repo/delete_branch.go | 13 +- internal/api/controller/repo/delete_path.go | 13 +- internal/api/controller/repo/delete_tag.go | 14 +- internal/api/controller/repo/diff.go | 6 +- internal/api/controller/repo/find.go | 3 +- internal/api/controller/repo/get_branch.go | 16 +- internal/api/controller/repo/get_commit.go | 16 +- .../controller/repo/get_commit_divergences.go | 18 +- internal/api/controller/repo/import.go | 110 +------- internal/api/controller/repo/import_cancel.go | 41 +++ .../api/controller/repo/import_progress.go | 1 + internal/api/controller/repo/list_branches.go | 19 +- .../api/controller/repo/list_commit_tags.go | 19 +- internal/api/controller/repo/list_commits.go | 19 +- internal/api/controller/repo/list_paths.go | 18 +- .../api/controller/repo/list_pipelines.go | 11 +- .../controller/repo/list_service_accounts.go | 18 +- internal/api/controller/repo/merge_check.go | 7 +- internal/api/controller/repo/move.go | 12 +- internal/api/controller/repo/raw.go | 21 +- internal/api/controller/repo/update.go | 14 +- internal/api/controller/space/controller.go | 5 +- internal/api/controller/space/import.go | 180 ++++++++++++ internal/api/controller/space/wire.go | 9 +- internal/api/handler/repo/import_cancel.go | 33 +++ internal/api/handler/space/import.go | 36 +++ internal/router/api.go | 1 + internal/services/importer/provider.go | 151 ++++++++-- internal/services/importer/repository.go | 262 +++++++++++++++--- internal/services/importer/wire.go | 11 +- internal/services/job/scheduler.go | 27 +- internal/services/job/timer.go | 5 + internal/store/database/job.go | 6 +- ...0025_alter_table_job_add_group_id.down.sql | 1 + .../0025_alter_table_job_add_group_id.up.sql | 1 + ...0025_alter_table_job_add_group_id.down.sql | 1 + .../0025_alter_table_job_add_group_id.up.sql | 1 + internal/store/database/repo.go | 34 +-- types/job.go | 1 + 49 files changed, 912 insertions(+), 397 deletions(-) create mode 100644 internal/api/controller/repo/import_cancel.go create mode 100644 internal/api/controller/space/import.go create mode 100644 internal/api/handler/repo/import_cancel.go create mode 100644 internal/api/handler/space/import.go create mode 100644 internal/store/database/migrate/postgres/0025_alter_table_job_add_group_id.down.sql create mode 100644 internal/store/database/migrate/postgres/0025_alter_table_job_add_group_id.up.sql create mode 100644 internal/store/database/migrate/sqlite/0025_alter_table_job_add_group_id.down.sql create mode 100644 internal/store/database/migrate/sqlite/0025_alter_table_job_add_group_id.up.sql diff --git a/cmd/gitness/wire_gen.go b/cmd/gitness/wire_gen.go index 248028317..1a970f498 100644 --- a/cmd/gitness/wire_gen.go +++ b/cmd/gitness/wire_gen.go @@ -120,7 +120,7 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro if err != nil { return nil, err } - repository, err := importer.ProvideRepoImporter(provider, gitrpcInterface, repoStore, jobScheduler, executor) + repository, err := importer.ProvideRepoImporter(config, provider, gitrpcInterface, repoStore, jobScheduler, executor) if err != nil { return nil, err } @@ -143,7 +143,7 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro secretStore := database.ProvideSecretStore(db) connectorStore := database.ProvideConnectorStore(db) templateStore := database.ProvideTemplateStore(db) - spaceController := space.ProvideController(db, provider, eventsStreamer, pathUID, authorizer, pathStore, pipelineStore, secretStore, connectorStore, templateStore, spaceStore, repoStore, principalStore, repoController, membershipStore) + spaceController := space.ProvideController(db, provider, eventsStreamer, pathUID, authorizer, pathStore, pipelineStore, secretStore, connectorStore, templateStore, spaceStore, repoStore, principalStore, repoController, membershipStore, repository) pipelineController := pipeline.ProvideController(db, pathUID, pathStore, repoStore, authorizer, pipelineStore) encrypter, err := encrypt.ProvideEncrypter(config) if err != nil { diff --git a/internal/api/controller/repo/blame.go b/internal/api/controller/repo/blame.go index 1c217b9bd..b085c4f1c 100644 --- a/internal/api/controller/repo/blame.go +++ b/internal/api/controller/repo/blame.go @@ -9,14 +9,14 @@ import ( "strings" "github.com/harness/gitness/gitrpc" - apiauth "github.com/harness/gitness/internal/api/auth" "github.com/harness/gitness/internal/api/usererror" "github.com/harness/gitness/internal/auth" "github.com/harness/gitness/types" "github.com/harness/gitness/types/enum" ) -func (c *Controller) Blame(ctx context.Context, session *auth.Session, +func (c *Controller) Blame(ctx context.Context, + session *auth.Session, repoRef, gitRef, path string, lineFrom, lineTo int, ) (types.Stream[*gitrpc.BlamePart], error) { @@ -29,15 +29,11 @@ func (c *Controller) Blame(ctx context.Context, session *auth.Session, return nil, usererror.BadRequest("Line range must be valid.") } - repo, err := c.repoStore.FindByRef(ctx, repoRef) + repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoView, true) if err != nil { return nil, err } - if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoView, true); err != nil { - return nil, err - } - if gitRef == "" { gitRef = repo.DefaultBranch } diff --git a/internal/api/controller/repo/commit.go b/internal/api/controller/repo/commit.go index 19be78666..a435cc7db 100644 --- a/internal/api/controller/repo/commit.go +++ b/internal/api/controller/repo/commit.go @@ -11,7 +11,6 @@ import ( "time" "github.com/harness/gitness/gitrpc" - apiauth "github.com/harness/gitness/internal/api/auth" "github.com/harness/gitness/internal/auth" "github.com/harness/gitness/internal/bootstrap" "github.com/harness/gitness/types/enum" @@ -40,17 +39,16 @@ type CommitFilesResponse struct { CommitID string `json:"commit_id"` } -func (c *Controller) CommitFiles(ctx context.Context, session *auth.Session, - repoRef string, in *CommitFilesOptions) (CommitFilesResponse, error) { - repo, err := c.repoStore.FindByRef(ctx, repoRef) +func (c *Controller) CommitFiles(ctx context.Context, + session *auth.Session, + repoRef string, + in *CommitFilesOptions, +) (CommitFilesResponse, error) { + repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoPush, false) if err != nil { return CommitFilesResponse{}, err } - if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoPush, false); err != nil { - return CommitFilesResponse{}, err - } - actions := make([]gitrpc.CommitFileAction, len(in.Actions)) for i, action := range in.Actions { var rawPayload []byte diff --git a/internal/api/controller/repo/content_get.go b/internal/api/controller/repo/content_get.go index 5ca0f7d98..117bff5e5 100644 --- a/internal/api/controller/repo/content_get.go +++ b/internal/api/controller/repo/content_get.go @@ -11,7 +11,6 @@ import ( "io" "github.com/harness/gitness/gitrpc" - apiauth "github.com/harness/gitness/internal/api/auth" "github.com/harness/gitness/internal/api/controller" "github.com/harness/gitness/internal/auth" "github.com/harness/gitness/types" @@ -89,15 +88,11 @@ func (c *Controller) GetContent(ctx context.Context, repoPath string, includeLatestCommit bool, ) (*GetContentOutput, error) { - repo, err := c.repoStore.FindByRef(ctx, repoRef) + repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoView, true) if err != nil { return nil, err } - if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoView, true); err != nil { - return nil, err - } - // set gitRef to default branch in case an empty reference was provided if gitRef == "" { gitRef = repo.DefaultBranch diff --git a/internal/api/controller/repo/content_paths_details.go b/internal/api/controller/repo/content_paths_details.go index b42d9d71b..d7078b2ac 100644 --- a/internal/api/controller/repo/content_paths_details.go +++ b/internal/api/controller/repo/content_paths_details.go @@ -8,7 +8,6 @@ import ( "context" "github.com/harness/gitness/gitrpc" - apiauth "github.com/harness/gitness/internal/api/auth" "github.com/harness/gitness/internal/api/usererror" "github.com/harness/gitness/internal/auth" "github.com/harness/gitness/types/enum" @@ -30,21 +29,19 @@ func (c *Controller) PathsDetails(ctx context.Context, gitRef string, input PathsDetailsInput, ) (PathsDetailsOutput, error) { - repo, err := c.repoStore.FindByRef(ctx, repoRef) + repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoView, true) if err != nil { return PathsDetailsOutput{}, err } - if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoView, true); err != nil { - return PathsDetailsOutput{}, err - } - if len(input.Paths) == 0 { return PathsDetailsOutput{}, nil } - if len(input.Paths) > 50 { - return PathsDetailsOutput{}, usererror.BadRequest("maximum number of elements in the Paths array is 25") + const maxInputPaths = 50 + if len(input.Paths) > maxInputPaths { + return PathsDetailsOutput{}, + usererror.BadRequestf("maximum number of elements in the Paths array is %d", maxInputPaths) } // set gitRef to default branch in case an empty reference was provided diff --git a/internal/api/controller/repo/controller.go b/internal/api/controller/repo/controller.go index 684f11e31..fbcd03dc1 100644 --- a/internal/api/controller/repo/controller.go +++ b/internal/api/controller/repo/controller.go @@ -11,6 +11,8 @@ import ( "strings" "github.com/harness/gitness/gitrpc" + apiauth "github.com/harness/gitness/internal/api/auth" + "github.com/harness/gitness/internal/api/usererror" "github.com/harness/gitness/internal/auth" "github.com/harness/gitness/internal/auth/authz" "github.com/harness/gitness/internal/githook" @@ -19,6 +21,7 @@ import ( "github.com/harness/gitness/internal/url" "github.com/harness/gitness/types" "github.com/harness/gitness/types/check" + "github.com/harness/gitness/types/enum" "github.com/jmoiron/sqlx" ) @@ -68,6 +71,35 @@ func NewController( } } +// getRepoCheckAccess fetches an active repo (not one that is currently being imported) +// and checks if the current user has permission to access it. +func (c *Controller) getRepoCheckAccess( + ctx context.Context, + session *auth.Session, + repoRef string, + reqPermission enum.Permission, + orPublic bool, +) (*types.Repository, error) { + if repoRef == "" { + return nil, usererror.BadRequest("A valid repository reference must be provided.") + } + + repo, err := c.repoStore.FindByRef(ctx, repoRef) + if err != nil { + return nil, fmt.Errorf("failed to find repository: %w", err) + } + + if repo.Importing { + return nil, usererror.BadRequest("Repository import is in progress.") + } + + if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, reqPermission, orPublic); err != nil { + return nil, fmt.Errorf("access check failed: %w", err) + } + + return repo, nil +} + // CreateRPCWriteParams creates base write parameters for gitrpc write operations. // IMPORTANT: session & repo are assumed to be not nil! func CreateRPCWriteParams(ctx context.Context, urlProvider *url.Provider, diff --git a/internal/api/controller/repo/create_branch.go b/internal/api/controller/repo/create_branch.go index 3e78f36f2..a528853b6 100644 --- a/internal/api/controller/repo/create_branch.go +++ b/internal/api/controller/repo/create_branch.go @@ -9,7 +9,6 @@ import ( "fmt" "github.com/harness/gitness/gitrpc" - apiauth "github.com/harness/gitness/internal/api/auth" "github.com/harness/gitness/internal/auth" "github.com/harness/gitness/types/enum" ) @@ -24,17 +23,16 @@ type CreateBranchInput struct { } // CreateBranch creates a new branch for a repo. -func (c *Controller) CreateBranch(ctx context.Context, session *auth.Session, - repoRef string, in *CreateBranchInput) (*Branch, error) { - repo, err := c.repoStore.FindByRef(ctx, repoRef) +func (c *Controller) CreateBranch(ctx context.Context, + session *auth.Session, + repoRef string, + in *CreateBranchInput, +) (*Branch, error) { + repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoPush, false) if err != nil { return nil, err } - if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoPush, false); err != nil { - return nil, err - } - // set target to default branch in case no target was provided if in.Target == "" { in.Target = repo.DefaultBranch diff --git a/internal/api/controller/repo/create_commit_tag.go b/internal/api/controller/repo/create_commit_tag.go index ed474dcba..ad52754cd 100644 --- a/internal/api/controller/repo/create_commit_tag.go +++ b/internal/api/controller/repo/create_commit_tag.go @@ -10,7 +10,6 @@ import ( "time" "github.com/harness/gitness/gitrpc" - apiauth "github.com/harness/gitness/internal/api/auth" "github.com/harness/gitness/internal/auth" "github.com/harness/gitness/types/enum" ) @@ -28,17 +27,16 @@ type CreateCommitTagInput struct { } // CreateCommitTag creates a new tag for a repo. -func (c *Controller) CreateCommitTag(ctx context.Context, session *auth.Session, - repoRef string, in *CreateCommitTagInput) (*CommitTag, error) { - repo, err := c.repoStore.FindByRef(ctx, repoRef) +func (c *Controller) CreateCommitTag(ctx context.Context, + session *auth.Session, + repoRef string, + in *CreateCommitTagInput, +) (*CommitTag, error) { + repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoPush, false) if err != nil { return nil, err } - if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoPush, false); err != nil { - return nil, err - } - // set target to default branch in case no branch or commit was provided if in.Target == "" { in.Target = repo.DefaultBranch diff --git a/internal/api/controller/repo/create_path.go b/internal/api/controller/repo/create_path.go index 6fda432aa..f88cd9bc5 100644 --- a/internal/api/controller/repo/create_path.go +++ b/internal/api/controller/repo/create_path.go @@ -10,7 +10,6 @@ import ( "strings" "time" - apiauth "github.com/harness/gitness/internal/api/auth" "github.com/harness/gitness/internal/api/usererror" "github.com/harness/gitness/internal/auth" "github.com/harness/gitness/types" @@ -24,17 +23,16 @@ type CreatePathInput struct { } // CreatePath creates a new path for a repo. -func (c *Controller) CreatePath(ctx context.Context, session *auth.Session, - repoRef string, in *CreatePathInput) (*types.Path, error) { - repo, err := c.repoStore.FindByRef(ctx, repoRef) +func (c *Controller) CreatePath(ctx context.Context, + session *auth.Session, + repoRef string, + in *CreatePathInput, +) (*types.Path, error) { + repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoEdit, false) if err != nil { return nil, err } - if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoEdit, false); err != nil { - return nil, err - } - if err = c.sanitizeCreatePathInput(in, repo.Path); err != nil { return nil, fmt.Errorf("failed to sanitize input: %w", err) } diff --git a/internal/api/controller/repo/delete.go b/internal/api/controller/repo/delete.go index 44de47294..483655036 100644 --- a/internal/api/controller/repo/delete.go +++ b/internal/api/controller/repo/delete.go @@ -19,6 +19,7 @@ import ( // Delete deletes a repo. func (c *Controller) Delete(ctx context.Context, session *auth.Session, repoRef string) error { + // note: can't use c.getRepoCheckAccess because import job for repositories being imported must be cancelled. repo, err := c.repoStore.FindByRef(ctx, repoRef) if err != nil { return err @@ -27,9 +28,21 @@ func (c *Controller) Delete(ctx context.Context, session *auth.Session, repoRef if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoDelete, false); err != nil { return err } + + if repo.Importing { + err = c.importer.Cancel(ctx, repo) + if err != nil { + return fmt.Errorf("failed to cancel repository import") + } + + return c.DeleteNoAuth(ctx, session, repo) + } + log.Ctx(ctx).Info().Msgf("Delete request received for repo %s , id: %d", repo.Path, repo.ID) + // TODO: uncomment when soft delete is implemented // return c.DeleteNoAuth(ctx, session, repo) + return nil } diff --git a/internal/api/controller/repo/delete_branch.go b/internal/api/controller/repo/delete_branch.go index c02763940..f57acaede 100644 --- a/internal/api/controller/repo/delete_branch.go +++ b/internal/api/controller/repo/delete_branch.go @@ -9,23 +9,22 @@ import ( "fmt" "github.com/harness/gitness/gitrpc" - apiauth "github.com/harness/gitness/internal/api/auth" "github.com/harness/gitness/internal/api/usererror" "github.com/harness/gitness/internal/auth" "github.com/harness/gitness/types/enum" ) // DeleteBranch deletes a repo branch. -func (c *Controller) DeleteBranch(ctx context.Context, session *auth.Session, repoRef string, branchName string) error { - repo, err := c.repoStore.FindByRef(ctx, repoRef) +func (c *Controller) DeleteBranch(ctx context.Context, + session *auth.Session, + repoRef string, + branchName string, +) error { + repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoPush, false) if err != nil { return err } - if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoPush, false); err != nil { - return err - } - // make sure user isn't deleting the default branch // ASSUMPTION: lower layer calls explicit branch api // and 'refs/heads/branch1' would fail if 'branch1' exists. diff --git a/internal/api/controller/repo/delete_path.go b/internal/api/controller/repo/delete_path.go index 288df0806..1f4ca8316 100644 --- a/internal/api/controller/repo/delete_path.go +++ b/internal/api/controller/repo/delete_path.go @@ -8,7 +8,6 @@ import ( "context" "fmt" - apiauth "github.com/harness/gitness/internal/api/auth" "github.com/harness/gitness/internal/api/usererror" "github.com/harness/gitness/internal/auth" "github.com/harness/gitness/store/database/dbtx" @@ -17,16 +16,16 @@ import ( ) // DeletePath deletes a repo path. -func (c *Controller) DeletePath(ctx context.Context, session *auth.Session, repoRef string, pathID int64) error { - repo, err := c.repoStore.FindByRef(ctx, repoRef) +func (c *Controller) DeletePath(ctx context.Context, + session *auth.Session, + repoRef string, + pathID int64, +) error { + repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoEdit, false) if err != nil { return err } - if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoEdit, false); err != nil { - return err - } - err = dbtx.New(c.db).WithTx(ctx, func(ctx context.Context) error { var path *types.Path path, err = c.pathStore.FindWithLock(ctx, pathID) diff --git a/internal/api/controller/repo/delete_tag.go b/internal/api/controller/repo/delete_tag.go index 1029c6ae5..62afaa000 100644 --- a/internal/api/controller/repo/delete_tag.go +++ b/internal/api/controller/repo/delete_tag.go @@ -9,23 +9,21 @@ import ( "fmt" "github.com/harness/gitness/gitrpc" - apiauth "github.com/harness/gitness/internal/api/auth" "github.com/harness/gitness/internal/auth" "github.com/harness/gitness/types/enum" ) // DeleteTag deletes a tag from the repo. -func (c *Controller) DeleteTag(ctx context.Context, session *auth.Session, - repoRef, tagName string) error { - repo, err := c.repoStore.FindByRef(ctx, repoRef) +func (c *Controller) DeleteTag(ctx context.Context, + session *auth.Session, + repoRef, + tagName string, +) error { + repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoPush, false) if err != nil { return err } - if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoPush, false); err != nil { - return err - } - writeParams, err := CreateRPCWriteParams(ctx, c.urlProvider, session, repo) if err != nil { return fmt.Errorf("failed to create RPC write params: %w", err) diff --git a/internal/api/controller/repo/diff.go b/internal/api/controller/repo/diff.go index 680cb49ad..c4d6f28a4 100644 --- a/internal/api/controller/repo/diff.go +++ b/internal/api/controller/repo/diff.go @@ -25,15 +25,11 @@ func (c *Controller) RawDiff( path string, w io.Writer, ) error { - repo, err := c.repoStore.FindByRef(ctx, repoRef) + repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoView, true) if err != nil { return err } - if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoView, false); err != nil { - return err - } - info, err := parseDiffPath(path) if err != nil { return err diff --git a/internal/api/controller/repo/find.go b/internal/api/controller/repo/find.go index 883d52608..f37410ec7 100644 --- a/internal/api/controller/repo/find.go +++ b/internal/api/controller/repo/find.go @@ -15,6 +15,7 @@ import ( // Find finds a repo. func (c *Controller) Find(ctx context.Context, session *auth.Session, repoRef string) (*types.Repository, error) { + // note: can't use c.getRepoCheckAccess because even repositories that are currently being imported can be fetched. repo, err := c.repoStore.FindByRef(ctx, repoRef) if err != nil { return nil, err @@ -24,7 +25,7 @@ func (c *Controller) Find(ctx context.Context, session *auth.Session, repoRef st return nil, err } - // backfil clone url + // backfill clone url repo.GitURL = c.urlProvider.GenerateRepoCloneURL(repo.Path) return repo, nil diff --git a/internal/api/controller/repo/get_branch.go b/internal/api/controller/repo/get_branch.go index c8e337d80..dccd1cd7b 100644 --- a/internal/api/controller/repo/get_branch.go +++ b/internal/api/controller/repo/get_branch.go @@ -9,21 +9,19 @@ import ( "fmt" "github.com/harness/gitness/gitrpc" - apiauth "github.com/harness/gitness/internal/api/auth" "github.com/harness/gitness/internal/auth" "github.com/harness/gitness/types/enum" ) // GetBranch gets a repo branch. -func (c *Controller) GetBranch(ctx context.Context, session *auth.Session, - repoRef string, branchName string) (*Branch, error) { - repo, err := c.repoStore.FindByRef(ctx, repoRef) +func (c *Controller) GetBranch(ctx context.Context, + session *auth.Session, + repoRef string, + branchName string, +) (*Branch, error) { + repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoView, true) if err != nil { - return nil, fmt.Errorf("faild to find repo: %w", err) - } - - if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoView, false); err != nil { - return nil, fmt.Errorf("access check failed: %w", err) + return nil, err } rpcOut, err := c.gitRPCClient.GetBranch(ctx, &gitrpc.GetBranchParams{ diff --git a/internal/api/controller/repo/get_commit.go b/internal/api/controller/repo/get_commit.go index c9019deea..39f2cb872 100644 --- a/internal/api/controller/repo/get_commit.go +++ b/internal/api/controller/repo/get_commit.go @@ -9,7 +9,6 @@ import ( "fmt" "github.com/harness/gitness/gitrpc" - apiauth "github.com/harness/gitness/internal/api/auth" "github.com/harness/gitness/internal/api/controller" "github.com/harness/gitness/internal/auth" "github.com/harness/gitness/types" @@ -17,15 +16,14 @@ import ( ) // GetCommit gets a repo commit. -func (c *Controller) GetCommit(ctx context.Context, session *auth.Session, - repoRef string, sha string) (*types.Commit, error) { - repo, err := c.repoStore.FindByRef(ctx, repoRef) +func (c *Controller) GetCommit(ctx context.Context, + session *auth.Session, + repoRef string, + sha string, +) (*types.Commit, error) { + repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoView, true) if err != nil { - return nil, fmt.Errorf("faild to find repo: %w", err) - } - - if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoView, false); err != nil { - return nil, fmt.Errorf("access check failed: %w", err) + return nil, err } rpcOut, err := c.gitRPCClient.GetCommit(ctx, &gitrpc.GetCommitParams{ diff --git a/internal/api/controller/repo/get_commit_divergences.go b/internal/api/controller/repo/get_commit_divergences.go index 62925273a..b7b1ab284 100644 --- a/internal/api/controller/repo/get_commit_divergences.go +++ b/internal/api/controller/repo/get_commit_divergences.go @@ -8,7 +8,6 @@ import ( "context" "github.com/harness/gitness/gitrpc" - apiauth "github.com/harness/gitness/internal/api/auth" "github.com/harness/gitness/internal/api/request" "github.com/harness/gitness/internal/api/usererror" "github.com/harness/gitness/internal/auth" @@ -39,20 +38,17 @@ type CommitDivergence struct { Behind int32 `json:"behind"` } -/* -* GetCommitDivergences returns the commit divergences between reference pairs. - */ -func (c *Controller) GetCommitDivergences(ctx context.Context, session *auth.Session, - repoRef string, in *GetCommitDivergencesInput) ([]CommitDivergence, error) { - repo, err := c.repoStore.FindByRef(ctx, repoRef) +// GetCommitDivergences returns the commit divergences between reference pairs. +func (c *Controller) GetCommitDivergences(ctx context.Context, + session *auth.Session, + repoRef string, + in *GetCommitDivergencesInput, +) ([]CommitDivergence, error) { + repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoView, true) if err != nil { return nil, err } - if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoView, false); err != nil { - return nil, err - } - // if no requests were provided return an empty list if in == nil || len(in.Requests) == 0 { return []CommitDivergence{}, nil diff --git a/internal/api/controller/repo/import.go b/internal/api/controller/repo/import.go index 72a710ad1..cdad3bfde 100644 --- a/internal/api/controller/repo/import.go +++ b/internal/api/controller/repo/import.go @@ -7,13 +7,9 @@ package repo import ( "context" "fmt" - "time" - "github.com/harness/gitness/gitrpc" "github.com/harness/gitness/internal/api/usererror" "github.com/harness/gitness/internal/auth" - "github.com/harness/gitness/internal/bootstrap" - "github.com/harness/gitness/internal/githook" "github.com/harness/gitness/internal/paths" "github.com/harness/gitness/internal/services/importer" "github.com/harness/gitness/internal/services/job" @@ -25,16 +21,12 @@ import ( ) type ImportInput struct { - ParentRef string `json:"parent_ref"` - UID string `json:"uid"` - - Provider importer.ProviderType `json:"provider"` - ProviderURL string `json:"provider_url"` - RepoSlug string `json:"repo_slug"` - Username string `json:"username"` - Password string `json:"password"` - + ParentRef string `json:"parent_ref"` + UID string `json:"uid"` Description string `json:"description"` + + Provider importer.Provider `json:"provider"` + ProviderRepo string `json:"provider_repo"` } // Import creates a new empty repository and starts git import to it from a remote repository. @@ -49,14 +41,7 @@ func (c *Controller) Import(ctx context.Context, session *auth.Session, in *Impo return nil, fmt.Errorf("failed to sanitize input: %w", err) } - providerInfo := importer.ProviderInfo{ - Type: in.Provider, - Host: in.ProviderURL, - User: in.Username, - Pass: in.Password, - } - - repoInfo, err := importer.Repo(ctx, providerInfo, in.RepoSlug) + remoteRepository, err := importer.LoadRepositoryFromProvider(ctx, in.Provider, in.ProviderRepo) if err != nil { return nil, err } @@ -66,29 +51,7 @@ func (c *Controller) Import(ctx context.Context, session *auth.Session, in *Impo return nil, fmt.Errorf("error creating job UID: %w", err) } - gitRPCResp, err := c.createEmptyGitRepository(ctx, session) - if err != nil { - return nil, fmt.Errorf("error creating repository on GitRPC: %w", err) - } - - now := time.Now().UnixMilli() - repo := &types.Repository{ - Version: 0, - ParentID: parentSpace.ID, - UID: in.UID, - GitUID: gitRPCResp.UID, - Path: "", // the path is set in the DB transaction below - Description: in.Description, - IsPublic: repoInfo.IsPublic, - CreatedBy: session.Principal.ID, - Created: now, - Updated: now, - ForkID: 0, - DefaultBranch: repoInfo.DefaultBranch, - Importing: true, - ImportingJobUID: &jobUID, - } - + var repo *types.Repository err = dbtx.New(c.db).WithTx(ctx, func(ctx context.Context) error { // lock parent space path to ensure it doesn't get updated while we setup new repo spacePath, err := c.pathStore.FindPrimaryWithLock(ctx, enum.PathTargetTypeSpace, parentSpace.ID) @@ -96,7 +59,8 @@ func (c *Controller) Import(ctx context.Context, session *auth.Session, in *Impo return usererror.BadRequest("Parent not found'") } - repo.Path = paths.Concatinate(spacePath.Value, in.UID) + pathToRepo := paths.Concatinate(spacePath.Value, in.UID) + repo = remoteRepository.ToRepo(parentSpace.ID, pathToRepo, in.UID, in.Description, jobUID, &session.Principal) err = c.repoStore.Create(ctx, repo) if err != nil { @@ -122,19 +86,10 @@ func (c *Controller) Import(ctx context.Context, session *auth.Session, in *Impo return nil }) if err != nil { - if err := c.DeleteRepositoryRPC(ctx, session, repo); err != nil { - log.Ctx(ctx).Warn().Err(err).Msg("gitrpc failed to delete repo for cleanup") - } - return nil, err } - err = c.importer.Run(ctx, jobUID, importer.Input{ - RepoID: repo.ID, - GitUser: in.Username, - GitPass: in.Password, - CloneURL: repoInfo.CloneURL, - }) + err = c.importer.Run(ctx, in.Provider, repo, remoteRepository.CloneURL) if err != nil { log.Ctx(ctx).Err(err).Msg("failed to start import repository job") } @@ -153,50 +108,5 @@ func (c *Controller) sanitizeImportInput(in *ImportInput) error { return err } - if in.Provider == "" { - return usererror.BadRequest("provider must be provided") - } - - if in.RepoSlug == "" { - return usererror.BadRequest("repo slug must be provided") - } - return nil } - -func (c *Controller) createEmptyGitRepository( - ctx context.Context, - session *auth.Session, -) (*gitrpc.CreateRepositoryOutput, error) { - // generate envars (add everything githook CLI needs for execution) - envVars, err := githook.GenerateEnvironmentVariables( - ctx, - c.urlProvider.GetAPIBaseURLInternal(), - 0, - session.Principal.ID, - true, - ) - if err != nil { - return nil, fmt.Errorf("failed to generate git hook environment variables: %w", err) - } - - actor := rpcIdentityFromPrincipal(session.Principal) - committer := rpcIdentityFromPrincipal(bootstrap.NewSystemServiceSession().Principal) - now := time.Now() - - resp, err := c.gitRPCClient.CreateRepository(ctx, &gitrpc.CreateRepositoryParams{ - Actor: *actor, - EnvVars: envVars, - DefaultBranch: c.defaultBranch, - Files: nil, - Author: actor, - AuthorDate: &now, - Committer: committer, - CommitterDate: &now, - }) - if err != nil { - return nil, fmt.Errorf("failed to create repo on gitrpc: %w", err) - } - - return resp, nil -} diff --git a/internal/api/controller/repo/import_cancel.go b/internal/api/controller/repo/import_cancel.go new file mode 100644 index 000000000..99dd8e333 --- /dev/null +++ b/internal/api/controller/repo/import_cancel.go @@ -0,0 +1,41 @@ +// Copyright 2022 Harness Inc. All rights reserved. +// Use of this source code is governed by the Polyform Free Trial License +// that can be found in the LICENSE.md file for this repository. + +package repo + +import ( + "context" + "fmt" + + apiauth "github.com/harness/gitness/internal/api/auth" + "github.com/harness/gitness/internal/api/usererror" + "github.com/harness/gitness/internal/auth" + "github.com/harness/gitness/types/enum" +) + +// ImportCancel cancels a repository import. +func (c *Controller) ImportCancel(ctx context.Context, + session *auth.Session, + repoRef string, +) error { + // note: can't use c.getRepoCheckAccess because this needs to fetch a repo being imported. + repo, err := c.repoStore.FindByRef(ctx, repoRef) + if err != nil { + return err + } + + if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoDelete, false); err != nil { + return err + } + + if !repo.Importing { + return usererror.BadRequest("repository is not being imported") + } + + if err = c.importer.Cancel(ctx, repo); err != nil { + return fmt.Errorf("failed to cancel repository import") + } + + return c.DeleteNoAuth(ctx, session, repo) +} diff --git a/internal/api/controller/repo/import_progress.go b/internal/api/controller/repo/import_progress.go index 72564f88c..8b7cf03fa 100644 --- a/internal/api/controller/repo/import_progress.go +++ b/internal/api/controller/repo/import_progress.go @@ -18,6 +18,7 @@ func (c *Controller) ImportProgress(ctx context.Context, session *auth.Session, repoRef string, ) (types.JobProgress, error) { + // note: can't use c.getRepoCheckAccess because this needs to fetch a repo being imported. repo, err := c.repoStore.FindByRef(ctx, repoRef) if err != nil { return types.JobProgress{}, err diff --git a/internal/api/controller/repo/list_branches.go b/internal/api/controller/repo/list_branches.go index d94421206..4994bb86d 100644 --- a/internal/api/controller/repo/list_branches.go +++ b/internal/api/controller/repo/list_branches.go @@ -9,7 +9,6 @@ import ( "fmt" "github.com/harness/gitness/gitrpc" - apiauth "github.com/harness/gitness/internal/api/auth" "github.com/harness/gitness/internal/api/controller" "github.com/harness/gitness/internal/auth" "github.com/harness/gitness/types" @@ -22,20 +21,18 @@ type Branch struct { Commit *types.Commit `json:"commit,omitempty"` } -/* -* ListBranches lists the branches of a repo. - */ -func (c *Controller) ListBranches(ctx context.Context, session *auth.Session, - repoRef string, includeCommit bool, filter *types.BranchFilter) ([]Branch, error) { - repo, err := c.repoStore.FindByRef(ctx, repoRef) +// ListBranches lists the branches of a repo. +func (c *Controller) ListBranches(ctx context.Context, + session *auth.Session, + repoRef string, + includeCommit bool, + filter *types.BranchFilter, +) ([]Branch, error) { + repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoView, true) if err != nil { return nil, err } - if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoView, false); err != nil { - return nil, err - } - rpcOut, err := c.gitRPCClient.ListBranches(ctx, &gitrpc.ListBranchesParams{ ReadParams: CreateRPCReadParams(repo), IncludeCommit: includeCommit, diff --git a/internal/api/controller/repo/list_commit_tags.go b/internal/api/controller/repo/list_commit_tags.go index 9b1b44d14..04944f333 100644 --- a/internal/api/controller/repo/list_commit_tags.go +++ b/internal/api/controller/repo/list_commit_tags.go @@ -9,7 +9,6 @@ import ( "fmt" "github.com/harness/gitness/gitrpc" - apiauth "github.com/harness/gitness/internal/api/auth" "github.com/harness/gitness/internal/api/controller" "github.com/harness/gitness/internal/auth" "github.com/harness/gitness/types" @@ -26,20 +25,18 @@ type CommitTag struct { Commit *types.Commit `json:"commit,omitempty"` } -/* -* ListCommitTags lists the commit tags of a repo. - */ -func (c *Controller) ListCommitTags(ctx context.Context, session *auth.Session, - repoRef string, includeCommit bool, filter *types.TagFilter) ([]CommitTag, error) { - repo, err := c.repoStore.FindByRef(ctx, repoRef) +// ListCommitTags lists the commit tags of a repo. +func (c *Controller) ListCommitTags(ctx context.Context, + session *auth.Session, + repoRef string, + includeCommit bool, + filter *types.TagFilter, +) ([]CommitTag, error) { + repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoView, true) if err != nil { return nil, err } - if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoView, false); err != nil { - return nil, err - } - rpcOut, err := c.gitRPCClient.ListCommitTags(ctx, &gitrpc.ListCommitTagsParams{ ReadParams: CreateRPCReadParams(repo), IncludeCommit: includeCommit, diff --git a/internal/api/controller/repo/list_commits.go b/internal/api/controller/repo/list_commits.go index d4994034c..6539b005c 100644 --- a/internal/api/controller/repo/list_commits.go +++ b/internal/api/controller/repo/list_commits.go @@ -9,27 +9,24 @@ import ( "fmt" "github.com/harness/gitness/gitrpc" - apiauth "github.com/harness/gitness/internal/api/auth" "github.com/harness/gitness/internal/api/controller" "github.com/harness/gitness/internal/auth" "github.com/harness/gitness/types" "github.com/harness/gitness/types/enum" ) -/* -* ListCommits lists the commits of a repo. - */ -func (c *Controller) ListCommits(ctx context.Context, session *auth.Session, - repoRef string, gitRef string, filter *types.CommitFilter) (types.ListCommitResponse, error) { - repo, err := c.repoStore.FindByRef(ctx, repoRef) +// ListCommits lists the commits of a repo. +func (c *Controller) ListCommits(ctx context.Context, + session *auth.Session, + repoRef string, + gitRef string, + filter *types.CommitFilter, +) (types.ListCommitResponse, error) { + repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoView, true) if err != nil { return types.ListCommitResponse{}, err } - if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoView, false); err != nil { - return types.ListCommitResponse{}, err - } - // set gitRef to default branch in case an empty reference was provided if gitRef == "" { gitRef = repo.DefaultBranch diff --git a/internal/api/controller/repo/list_paths.go b/internal/api/controller/repo/list_paths.go index 94e637888..2b26e4abb 100644 --- a/internal/api/controller/repo/list_paths.go +++ b/internal/api/controller/repo/list_paths.go @@ -8,27 +8,23 @@ import ( "context" "fmt" - apiauth "github.com/harness/gitness/internal/api/auth" "github.com/harness/gitness/internal/auth" "github.com/harness/gitness/store/database/dbtx" "github.com/harness/gitness/types" "github.com/harness/gitness/types/enum" ) -/* -* ListPaths lists all paths of a repo. - */ -func (c *Controller) ListPaths(ctx context.Context, session *auth.Session, - repoRef string, filter *types.PathFilter) ([]*types.Path, int64, error) { - repo, err := c.repoStore.FindByRef(ctx, repoRef) +// ListPaths lists all paths of a repo. +func (c *Controller) ListPaths(ctx context.Context, + session *auth.Session, + repoRef string, + filter *types.PathFilter, +) ([]*types.Path, int64, error) { + repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoView, true) if err != nil { return nil, 0, err } - if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoView, false); err != nil { - return nil, 0, err - } - var ( paths []*types.Path count int64 diff --git a/internal/api/controller/repo/list_pipelines.go b/internal/api/controller/repo/list_pipelines.go index ae5e0dda5..0ae83f71a 100644 --- a/internal/api/controller/repo/list_pipelines.go +++ b/internal/api/controller/repo/list_pipelines.go @@ -1,13 +1,13 @@ // Copyright 2022 Harness Inc. All rights reserved. // Use of this source code is governed by the Polyform Free Trial License // that can be found in the LICENSE.md file for this repository. + package repo import ( "context" "fmt" - apiauth "github.com/harness/gitness/internal/api/auth" "github.com/harness/gitness/internal/auth" "github.com/harness/gitness/store/database/dbtx" "github.com/harness/gitness/types" @@ -22,14 +22,9 @@ func (c *Controller) ListPipelines( latest bool, filter types.ListQueryFilter, ) ([]*types.Pipeline, int64, error) { - repo, err := c.repoStore.FindByRef(ctx, repoRef) + repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoView, true) if err != nil { - return nil, 0, fmt.Errorf("failed to find repo: %w", err) - } - - err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionPipelineView, false) - if err != nil { - return nil, 0, fmt.Errorf("failed to authorize: %w", err) + return nil, 0, err } var count int64 diff --git a/internal/api/controller/repo/list_service_accounts.go b/internal/api/controller/repo/list_service_accounts.go index b19a61f68..2353b185b 100644 --- a/internal/api/controller/repo/list_service_accounts.go +++ b/internal/api/controller/repo/list_service_accounts.go @@ -7,23 +7,19 @@ package repo import ( "context" - apiauth "github.com/harness/gitness/internal/api/auth" "github.com/harness/gitness/internal/auth" "github.com/harness/gitness/types" "github.com/harness/gitness/types/enum" ) -/* -* ListServiceAccounts lists the service accounts of a repo. - */ -func (c *Controller) ListServiceAccounts(ctx context.Context, session *auth.Session, - repoRef string) ([]*types.ServiceAccount, error) { - repo, err := c.repoStore.FindByRef(ctx, repoRef) - if err != nil { - return nil, err - } +// ListServiceAccounts lists the service accounts of a repo. - if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoView, false); err != nil { +func (c *Controller) ListServiceAccounts(ctx context.Context, + session *auth.Session, + repoRef string, +) ([]*types.ServiceAccount, error) { + repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoView, false) + if err != nil { return nil, err } diff --git a/internal/api/controller/repo/merge_check.go b/internal/api/controller/repo/merge_check.go index 9e3e2a983..89ebc9c87 100644 --- a/internal/api/controller/repo/merge_check.go +++ b/internal/api/controller/repo/merge_check.go @@ -9,7 +9,6 @@ import ( "fmt" "github.com/harness/gitness/gitrpc" - apiauth "github.com/harness/gitness/internal/api/auth" "github.com/harness/gitness/internal/auth" "github.com/harness/gitness/types/enum" ) @@ -25,15 +24,11 @@ func (c *Controller) MergeCheck( repoRef string, diffPath string, ) (MergeCheck, error) { - repo, err := c.repoStore.FindByRef(ctx, repoRef) + repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoView, false) if err != nil { return MergeCheck{}, err } - if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoView, false); err != nil { - return MergeCheck{}, fmt.Errorf("access check failed: %w", err) - } - info, err := parseDiffPath(diffPath) if err != nil { return MergeCheck{}, err diff --git a/internal/api/controller/repo/move.go b/internal/api/controller/repo/move.go index f0cdf8bf2..e4edbcfa8 100644 --- a/internal/api/controller/repo/move.go +++ b/internal/api/controller/repo/move.go @@ -7,6 +7,7 @@ package repo import ( "context" "fmt" + "github.com/harness/gitness/internal/api/usererror" "strconv" "strings" "time" @@ -55,13 +56,20 @@ func (i *MoveInput) hasChanges(repo *types.Repository) bool { // Move moves a repository to a new space and/or uid. // //nolint:gocognit // refactor if needed -func (c *Controller) Move(ctx context.Context, session *auth.Session, - repoRef string, in *MoveInput) (*types.Repository, error) { +func (c *Controller) Move(ctx context.Context, + session *auth.Session, + repoRef string, + in *MoveInput, +) (*types.Repository, error) { repo, err := c.repoStore.FindByRef(ctx, repoRef) if err != nil { return nil, err } + if repo.Importing { + return nil, usererror.BadRequest("can't move a repo that is being imported") + } + permission := enum.PermissionRepoEdit var inParentSpaceID *int64 if in.ParentRef != nil { diff --git a/internal/api/controller/repo/raw.go b/internal/api/controller/repo/raw.go index 684d6dd55..26fa50e9b 100644 --- a/internal/api/controller/repo/raw.go +++ b/internal/api/controller/repo/raw.go @@ -10,27 +10,24 @@ import ( "io" "github.com/harness/gitness/gitrpc" - apiauth "github.com/harness/gitness/internal/api/auth" "github.com/harness/gitness/internal/api/usererror" "github.com/harness/gitness/internal/auth" "github.com/harness/gitness/types/enum" ) -/* - * Raw finds the file of the repo at the given path and returns its raw content. - * If no gitRef is provided, the content is retrieved from the default branch. - */ -func (c *Controller) Raw(ctx context.Context, session *auth.Session, repoRef string, - gitRef string, repoPath string) (io.Reader, int64, error) { - repo, err := c.repoStore.FindByRef(ctx, repoRef) +// Raw finds the file of the repo at the given path and returns its raw content. +// If no gitRef is provided, the content is retrieved from the default branch. +func (c *Controller) Raw(ctx context.Context, + session *auth.Session, + repoRef string, + gitRef string, + repoPath string, +) (io.Reader, int64, error) { + repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoView, true) if err != nil { return nil, 0, err } - if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoView, true); err != nil { - return nil, 0, err - } - // set gitRef to default branch in case an empty reference was provided if gitRef == "" { gitRef = repo.DefaultBranch diff --git a/internal/api/controller/repo/update.go b/internal/api/controller/repo/update.go index d7a2f70a8..7550cc19b 100644 --- a/internal/api/controller/repo/update.go +++ b/internal/api/controller/repo/update.go @@ -9,7 +9,6 @@ import ( "fmt" "strings" - apiauth "github.com/harness/gitness/internal/api/auth" "github.com/harness/gitness/internal/auth" "github.com/harness/gitness/types" "github.com/harness/gitness/types/check" @@ -28,17 +27,16 @@ func (in *UpdateInput) hasChanges(repo *types.Repository) bool { } // Update updates a repository. -func (c *Controller) Update(ctx context.Context, session *auth.Session, - repoRef string, in *UpdateInput) (*types.Repository, error) { - repo, err := c.repoStore.FindByRef(ctx, repoRef) +func (c *Controller) Update(ctx context.Context, + session *auth.Session, + repoRef string, + in *UpdateInput, +) (*types.Repository, error) { + repo, err := c.getRepoCheckAccess(ctx, session, repoRef, enum.PermissionRepoEdit, false) if err != nil { return nil, err } - if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoEdit, false); err != nil { - return nil, err - } - if !in.hasChanges(repo) { return repo, nil } diff --git a/internal/api/controller/space/controller.go b/internal/api/controller/space/controller.go index 6b35a95fe..a609cdcdb 100644 --- a/internal/api/controller/space/controller.go +++ b/internal/api/controller/space/controller.go @@ -8,6 +8,7 @@ import ( "github.com/harness/gitness/internal/api/controller/repo" "github.com/harness/gitness/internal/auth/authz" "github.com/harness/gitness/internal/pipeline/events" + "github.com/harness/gitness/internal/services/importer" "github.com/harness/gitness/internal/store" "github.com/harness/gitness/internal/url" "github.com/harness/gitness/types/check" @@ -31,6 +32,7 @@ type Controller struct { principalStore store.PrincipalStore repoCtrl *repo.Controller membershipStore store.MembershipStore + importer *importer.Repository } func NewController(db *sqlx.DB, urlProvider *url.Provider, eventsStream events.EventsStreamer, @@ -38,7 +40,7 @@ func NewController(db *sqlx.DB, urlProvider *url.Provider, eventsStream events.E pathStore store.PathStore, pipelineStore store.PipelineStore, secretStore store.SecretStore, connectorStore store.ConnectorStore, templateStore store.TemplateStore, spaceStore store.SpaceStore, repoStore store.RepoStore, principalStore store.PrincipalStore, repoCtrl *repo.Controller, - membershipStore store.MembershipStore, + membershipStore store.MembershipStore, importer *importer.Repository, ) *Controller { return &Controller{ db: db, @@ -56,5 +58,6 @@ func NewController(db *sqlx.DB, urlProvider *url.Provider, eventsStream events.E principalStore: principalStore, repoCtrl: repoCtrl, membershipStore: membershipStore, + importer: importer, } } diff --git a/internal/api/controller/space/import.go b/internal/api/controller/space/import.go new file mode 100644 index 000000000..7d23c85fa --- /dev/null +++ b/internal/api/controller/space/import.go @@ -0,0 +1,180 @@ +// Copyright 2022 Harness Inc. All rights reserved. +// Use of this source code is governed by the Polyform Free Trial License +// that can be found in the LICENSE.md file for this repository. + +package space + +import ( + "context" + "fmt" + "github.com/harness/gitness/types/check" + "github.com/rs/zerolog/log" + "strconv" + "strings" + "time" + + "github.com/harness/gitness/internal/api/usererror" + "github.com/harness/gitness/internal/auth" + "github.com/harness/gitness/internal/bootstrap" + "github.com/harness/gitness/internal/paths" + "github.com/harness/gitness/internal/services/importer" + "github.com/harness/gitness/internal/services/job" + "github.com/harness/gitness/store/database/dbtx" + "github.com/harness/gitness/types" + "github.com/harness/gitness/types/enum" +) + +type ImportInput struct { + CreateInput + Provider importer.Provider `json:"provider"` + ProviderSpace string `json:"provider_space"` +} + +// Import creates new space and starts import of all repositories from the remote provider's space into it. +func (c *Controller) Import(ctx context.Context, session *auth.Session, in *ImportInput) (*types.Space, error) { + parentSpace, err := c.getSpaceCheckAuthSpaceCreation(ctx, session, in.ParentRef) + if err != nil { + return nil, err + } + + if in.UID == "" { + in.UID = in.ProviderSpace + } + + err = c.sanitizeCreateInput(&in.CreateInput) + if err != nil { + return nil, fmt.Errorf("failed to sanitize input: %w", err) + } + + remoteRepositories, err := importer.LoadRepositoriesFromProviderSpace(ctx, in.Provider, in.ProviderSpace) + if err != nil { + return nil, err + } + + localRepositories := make([]*types.Repository, len(remoteRepositories)) + cloneURLs := make([]string, len(remoteRepositories)) + + var space *types.Space + err = dbtx.New(c.db).WithTx(ctx, func(ctx context.Context) error { + spacePath := in.UID + parentSpaceID := int64(0) + if parentSpace != nil { + parentSpaceID = parentSpace.ID + // lock parent space path to ensure it doesn't get updated while we setup new space + parentPath, err := c.pathStore.FindPrimaryWithLock(ctx, enum.PathTargetTypeSpace, parentSpaceID) + if err != nil { + return usererror.BadRequest("Parent not found") + } + spacePath = paths.Concatinate(parentPath.Value, in.UID) + + // ensure path is within accepted depth! + err = check.PathDepth(spacePath, true) + if err != nil { + return fmt.Errorf("path is invalid: %w", err) + } + } + + now := time.Now().UnixMilli() + space = &types.Space{ + Version: 0, + ParentID: parentSpaceID, + UID: in.UID, + Path: spacePath, + Description: in.Description, + IsPublic: in.IsPublic, + CreatedBy: session.Principal.ID, + Created: now, + Updated: now, + } + err = c.spaceStore.Create(ctx, space) + if err != nil { + return fmt.Errorf("space creation failed: %w", err) + } + + path := &types.Path{ + Version: 0, + Value: space.Path, + IsPrimary: true, + TargetType: enum.PathTargetTypeSpace, + TargetID: space.ID, + CreatedBy: space.CreatedBy, + Created: now, + Updated: now, + } + err = c.pathStore.Create(ctx, path) + if err != nil { + return fmt.Errorf("failed to create path: %w", err) + } + + // add space membership to top level space only (as the user doesn't have inherited permissions already) + parentRefAsID, err := strconv.ParseInt(in.ParentRef, 10, 64) + if (err == nil && parentRefAsID == 0) || (len(strings.TrimSpace(in.ParentRef)) == 0) { + membership := &types.Membership{ + MembershipKey: types.MembershipKey{ + SpaceID: space.ID, + PrincipalID: session.Principal.ID, + }, + Role: enum.MembershipRoleSpaceOwner, + + // membership has been created by the system + CreatedBy: bootstrap.NewSystemServiceSession().Principal.ID, + Created: now, + Updated: now, + } + err = c.membershipStore.Create(ctx, membership) + if err != nil { + return fmt.Errorf("failed to make user owner of the space: %w", err) + } + } + + for i, remoteRepository := range remoteRepositories { + var jobUID string + + jobUID, err = job.UID() + if err != nil { + return fmt.Errorf("error creating job UID: %w", err) + } + + pathToRepo := paths.Concatinate(path.Value, remoteRepository.UID) + repo := remoteRepository.ToRepo( + space.ID, pathToRepo, remoteRepository.UID, "", jobUID, &session.Principal) + + err = c.repoStore.Create(ctx, repo) + if err != nil { + return fmt.Errorf("failed to create repository in storage: %w", err) + } + + repoPath := &types.Path{ + Version: 0, + Value: repo.Path, + IsPrimary: true, + TargetType: enum.PathTargetTypeRepo, + TargetID: repo.ID, + CreatedBy: repo.CreatedBy, + Created: repo.Created, + Updated: repo.Updated, + } + + err = c.pathStore.Create(ctx, repoPath) + if err != nil { + return fmt.Errorf("failed to create path: %w", err) + } + + localRepositories[i] = repo + cloneURLs[i] = remoteRepository.CloneURL + } + + return nil + }) + if err != nil { + return nil, err + } + + jobGroupID := fmt.Sprintf("space-import-%d", space.ID) + err = c.importer.RunMany(ctx, jobGroupID, in.Provider, localRepositories, cloneURLs) + if err != nil { + log.Ctx(ctx).Err(err).Msg("failed to start import repository job") + } + + return space, nil +} diff --git a/internal/api/controller/space/wire.go b/internal/api/controller/space/wire.go index ea39967a2..a9b8b42d2 100644 --- a/internal/api/controller/space/wire.go +++ b/internal/api/controller/space/wire.go @@ -8,6 +8,7 @@ import ( "github.com/harness/gitness/internal/api/controller/repo" "github.com/harness/gitness/internal/auth/authz" "github.com/harness/gitness/internal/pipeline/events" + "github.com/harness/gitness/internal/services/importer" "github.com/harness/gitness/internal/store" "github.com/harness/gitness/internal/url" "github.com/harness/gitness/types/check" @@ -26,9 +27,11 @@ func ProvideController(db *sqlx.DB, urlProvider *url.Provider, eventsStream even pipelineStore store.PipelineStore, secretStore store.SecretStore, connectorStore store.ConnectorStore, templateStore store.TemplateStore, spaceStore store.SpaceStore, repoStore store.RepoStore, principalStore store.PrincipalStore, - repoCtrl *repo.Controller, membershipStore store.MembershipStore, + repoCtrl *repo.Controller, membershipStore store.MembershipStore, importer *importer.Repository, ) *Controller { return NewController(db, urlProvider, eventsStream, uidCheck, authorizer, - pathStore, pipelineStore, secretStore, connectorStore, templateStore, - spaceStore, repoStore, principalStore, repoCtrl, membershipStore) + pathStore, pipelineStore, secretStore, + connectorStore, templateStore, + spaceStore, repoStore, principalStore, + repoCtrl, membershipStore, importer) } diff --git a/internal/api/handler/repo/import_cancel.go b/internal/api/handler/repo/import_cancel.go new file mode 100644 index 000000000..ab0c8fb8b --- /dev/null +++ b/internal/api/handler/repo/import_cancel.go @@ -0,0 +1,33 @@ +// Copyright 2022 Harness Inc. All rights reserved. +// Use of this source code is governed by the Polyform Free Trial License +// that can be found in the LICENSE.md file for this repository. + +package repo + +import ( + "net/http" + + "github.com/harness/gitness/internal/api/controller/repo" + "github.com/harness/gitness/internal/api/render" + "github.com/harness/gitness/internal/api/request" +) + +func HandleImportCancel(repoCtrl *repo.Controller) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + session, _ := request.AuthSessionFrom(ctx) + repoRef, err := request.GetRepoRefFromPath(r) + if err != nil { + render.TranslatedUserError(w, err) + return + } + + err = repoCtrl.ImportCancel(ctx, session, repoRef) + if err != nil { + render.TranslatedUserError(w, err) + return + } + + render.DeleteSuccessful(w) + } +} diff --git a/internal/api/handler/space/import.go b/internal/api/handler/space/import.go new file mode 100644 index 000000000..f320fdcab --- /dev/null +++ b/internal/api/handler/space/import.go @@ -0,0 +1,36 @@ +// Copyright 2022 Harness Inc. All rights reserved. +// Use of this source code is governed by the Polyform Free Trial License +// that can be found in the LICENSE.md file for this repository. + +package space + +import ( + "encoding/json" + "net/http" + + "github.com/harness/gitness/internal/api/controller/space" + "github.com/harness/gitness/internal/api/render" + "github.com/harness/gitness/internal/api/request" +) + +func HandleImport(spaceCtrl *space.Controller) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + session, _ := request.AuthSessionFrom(ctx) + + in := new(space.ImportInput) + err := json.NewDecoder(r.Body).Decode(in) + if err != nil { + render.BadRequestf(w, "Invalid Request Body: %s.", err) + return + } + + space, err := spaceCtrl.Import(ctx, session, in) + if err != nil { + render.TranslatedUserError(w, err) + return + } + + render.JSON(w, http.StatusCreated, space) + } +} diff --git a/internal/router/api.go b/internal/router/api.go index c987315a3..d2fa4762f 100644 --- a/internal/router/api.go +++ b/internal/router/api.go @@ -181,6 +181,7 @@ func setupSpaces(r chi.Router, spaceCtrl *space.Controller) { r.Route("/spaces", func(r chi.Router) { // Create takes path and parentId via body, not uri r.Post("/", handlerspace.HandleCreate(spaceCtrl)) + r.Post("/import", handlerspace.HandleImport(spaceCtrl)) r.Route(fmt.Sprintf("/{%s}", request.PathParamSpaceRef), func(r chi.Router) { // space operations diff --git a/internal/services/importer/provider.go b/internal/services/importer/provider.go index ecfce9668..0665e1be2 100644 --- a/internal/services/importer/provider.go +++ b/internal/services/importer/provider.go @@ -9,25 +9,40 @@ import ( "errors" "fmt" "net/http" + "time" "github.com/harness/gitness/internal/api/usererror" + "github.com/harness/gitness/types" "github.com/drone/go-scm/scm" "github.com/drone/go-scm/scm/driver/github" + "github.com/drone/go-scm/scm/driver/gitlab" "github.com/drone/go-scm/scm/transport/oauth2" ) type ProviderType string const ( - ProviderTypeGitHub ProviderType = "github" + ProviderTypeGitHub ProviderType = "github" + ProviderTypeGitHubEnterprise ProviderType = "github-enterprise" + ProviderTypeGitLab ProviderType = "gitlab" + ProviderTypeGitLabEnterprise ProviderType = "gitlab-enterprise" ) -type ProviderInfo struct { - Type ProviderType - Host string - User string - Pass string +type Provider struct { + Type ProviderType `json:"type"` + Host string `json:"host"` + Username string `json:"username"` + Password string `json:"password"` +} + +func (p Provider) Enum() []any { + return []any{ + ProviderTypeGitHub, + ProviderTypeGitHubEnterprise, + ProviderTypeGitLab, + ProviderTypeGitLabEnterprise, + } } type RepositoryInfo struct { @@ -38,42 +53,120 @@ type RepositoryInfo struct { DefaultBranch string } -func getClient(provider ProviderInfo) (*scm.Client, error) { - var scmClient *scm.Client +// ToRepo converts the RepositoryInfo into the types.Repository object marked as being imported. +func (r *RepositoryInfo) ToRepo( + spaceID int64, + path string, + uid string, + description string, + jobUID string, + principal *types.Principal, +) *types.Repository { + now := time.Now().UnixMilli() + gitTempUID := "importing-" + jobUID + return &types.Repository{ + Version: 0, + ParentID: spaceID, + UID: uid, + GitUID: gitTempUID, // the correct git UID will be set by the job handler + Path: path, + Description: description, + IsPublic: r.IsPublic, + CreatedBy: principal.ID, + Created: now, + Updated: now, + ForkID: 0, + DefaultBranch: r.DefaultBranch, + Importing: true, + ImportingJobUID: &jobUID, + } +} +func getClient(provider Provider) (*scm.Client, error) { switch provider.Type { case "": return nil, usererror.BadRequest("provider can not be empty") + case ProviderTypeGitHub: - scmClient = github.NewDefault() - if provider.Pass != "" { - scmClient.Client = &http.Client{ + c := github.NewDefault() + if provider.Password != "" { + c.Client = &http.Client{ Transport: &oauth2.Transport{ - Source: oauth2.StaticTokenSource(&scm.Token{Token: provider.Pass}), + Source: oauth2.StaticTokenSource(&scm.Token{Token: provider.Password}), }, } } + return c, nil + + case ProviderTypeGitHubEnterprise: + c, err := github.New(provider.Host) + if err != nil { + return nil, usererror.BadRequestf("provider Host invalid: %s", err.Error()) + } + + if provider.Password != "" { + c.Client = &http.Client{ + Transport: &oauth2.Transport{ + Source: oauth2.StaticTokenSource(&scm.Token{Token: provider.Password}), + }, + } + } + return c, nil + + case ProviderTypeGitLab: + c := gitlab.NewDefault() + if provider.Password != "" { + c.Client = &http.Client{ + Transport: &oauth2.Transport{ + Source: oauth2.StaticTokenSource(&scm.Token{Token: provider.Password}), + }, + } + } + + return c, nil + + case ProviderTypeGitLabEnterprise: + c, err := gitlab.New(provider.Host) + if err != nil { + return nil, usererror.BadRequestf("provider Host invalid: %s", err.Error()) + } + + if provider.Password != "" { + c.Client = &http.Client{ + Transport: &oauth2.Transport{ + Source: oauth2.StaticTokenSource(&scm.Token{Token: provider.Password}), + }, + } + } + + return c, nil + default: return nil, usererror.BadRequestf("unsupported provider: %s", provider) } - - return scmClient, nil } -func Repo(ctx context.Context, provider ProviderInfo, repoSlug string) (RepositoryInfo, error) { +func LoadRepositoryFromProvider(ctx context.Context, provider Provider, repoSlug string) (RepositoryInfo, error) { scmClient, err := getClient(provider) if err != nil { return RepositoryInfo{}, err } + if repoSlug == "" { + return RepositoryInfo{}, usererror.BadRequest("provider repository identifier is missing") + } + scmRepo, _, err := scmClient.Repositories.Find(ctx, repoSlug) if errors.Is(err, scm.ErrNotFound) { - return RepositoryInfo{}, usererror.BadRequestf("repository %s not found at %s", repoSlug, provider) + return RepositoryInfo{}, + usererror.BadRequestf("repository %s not found at %s", repoSlug, provider.Type) } if errors.Is(err, scm.ErrNotAuthorized) { - return RepositoryInfo{}, usererror.BadRequestf("bad credentials provided for %s at %s", repoSlug, provider) + return RepositoryInfo{}, + usererror.BadRequestf("bad credentials provided for %s at %s", repoSlug, provider.Type) } if err != nil { - return RepositoryInfo{}, fmt.Errorf("failed to fetch repository %s from %s: %w", repoSlug, provider, err) + return RepositoryInfo{}, + fmt.Errorf("failed to fetch repository %s from %s: %w", repoSlug, provider.Type, err) } return RepositoryInfo{ @@ -85,13 +178,17 @@ func Repo(ctx context.Context, provider ProviderInfo, repoSlug string) (Reposito }, nil } -func Space(ctx context.Context, provider ProviderInfo, space string) (map[string]RepositoryInfo, error) { +func LoadRepositoriesFromProviderSpace(ctx context.Context, provider Provider, spaceSlug string) ([]RepositoryInfo, error) { scmClient, err := getClient(provider) if err != nil { return nil, err } - repoMap := make(map[string]RepositoryInfo) + if spaceSlug == "" { + return nil, usererror.BadRequest("provider space identifier is missing") + } + + repos := make([]RepositoryInfo, 0) const pageSize = 50 page := 1 @@ -104,17 +201,17 @@ func Space(ctx context.Context, provider ProviderInfo, space string) (map[string }, RepoSearchTerm: scm.RepoSearchTerm{ RepoName: "", - User: space, + User: spaceSlug, }, }) if errors.Is(err, scm.ErrNotFound) { - return nil, usererror.BadRequestf("space %s not found at %s", space, provider) + return nil, usererror.BadRequestf("space %s not found at %s", spaceSlug, provider.Type) } if errors.Is(err, scm.ErrNotAuthorized) { - return nil, usererror.BadRequestf("bad credentials provided for %s at %s", space, provider) + return nil, usererror.BadRequestf("bad credentials provided for %s at %s", spaceSlug, provider.Type) } if err != nil { - return nil, fmt.Errorf("failed to fetch space %s from %s: %w", space, provider, err) + return nil, fmt.Errorf("failed to fetch space %s from %s: %w", spaceSlug, provider.Type, err) } for _, scmRepo := range scmRepos { @@ -122,13 +219,13 @@ func Space(ctx context.Context, provider ProviderInfo, space string) (map[string continue } - repoMap[scmRepo.Name] = RepositoryInfo{ + repos = append(repos, RepositoryInfo{ Space: scmRepo.Namespace, UID: scmRepo.Name, CloneURL: scmRepo.Clone, IsPublic: !scmRepo.Private, DefaultBranch: scmRepo.Branch, - } + }) } if len(scmRepos) == 0 || page == scmResponse.Page.Last { @@ -138,5 +235,5 @@ func Space(ctx context.Context, provider ProviderInfo, space string) (map[string page++ } - return repoMap, nil + return repos, nil } diff --git a/internal/services/importer/repository.go b/internal/services/importer/repository.go index 60c347bc4..399168587 100644 --- a/internal/services/importer/repository.go +++ b/internal/services/importer/repository.go @@ -21,13 +21,21 @@ import ( gitnessurl "github.com/harness/gitness/internal/url" gitness_store "github.com/harness/gitness/store" "github.com/harness/gitness/types" + + "github.com/rs/zerolog/log" +) + +const ( + importJobMaxRetries = 0 + importJobMaxDuration = 45 * time.Minute ) type Repository struct { - urlProvider *gitnessurl.Provider - git gitrpc.Interface - repoStore store.RepoStore - scheduler *job.Scheduler + defaultBranch string + urlProvider *gitnessurl.Provider + git gitrpc.Interface + repoStore store.RepoStore + scheduler *job.Scheduler } var _ job.Handler = (*Repository)(nil) @@ -45,25 +53,84 @@ func (i *Repository) Register(executor *job.Executor) error { return executor.Register(jobType, i) } -func (i *Repository) Run(ctx context.Context, jobUID string, input Input) error { +func (i *Repository) Run(ctx context.Context, provider Provider, repo *types.Repository, cloneURL string) error { + input := Input{ + RepoID: repo.ID, + GitUser: provider.Username, + GitPass: provider.Password, + CloneURL: cloneURL, + } + data, err := json.Marshal(input) if err != nil { - return err + return fmt.Errorf("failed to marshal job input json: %w", err) } strData := strings.TrimSpace(string(data)) return i.scheduler.RunJob(ctx, job.Definition{ - UID: jobUID, + UID: *repo.ImportingJobUID, Type: jobType, - MaxRetries: 1, - Timeout: 30 * time.Minute, + MaxRetries: importJobMaxRetries, + Timeout: importJobMaxDuration, Data: strData, }) } +func (i *Repository) RunMany(ctx context.Context, + groupID string, + provider Provider, + repos []*types.Repository, + cloneURLs []string, +) error { + if len(repos) != len(cloneURLs) { + return fmt.Errorf("slice length mismatch: have %d repositories and %d clone URLs", + len(repos), len(cloneURLs)) + } + + n := len(repos) + + defs := make([]job.Definition, n) + + for k := 0; k < n; k++ { + repo := repos[k] + cloneURL := cloneURLs[k] + + input := Input{ + RepoID: repo.ID, + GitUser: provider.Username, + GitPass: provider.Password, + CloneURL: cloneURL, + } + + data, err := json.Marshal(input) + if err != nil { + return fmt.Errorf("failed to marshal job input json: %w", err) + } + + strData := strings.TrimSpace(string(data)) + + defs[k] = job.Definition{ + UID: *repo.ImportingJobUID, + Type: jobType, + MaxRetries: importJobMaxRetries, + Timeout: importJobMaxDuration, + Data: strData, + } + } + + err := i.scheduler.RunJobs(ctx, groupID, defs) + if err != nil { + return fmt.Errorf("failed to run jobs: %w", err) + } + + return nil +} + // Handle is repository import background job handler. func (i *Repository) Handle(ctx context.Context, data string, _ job.ProgressReporter) (string, error) { + systemPrincipal := bootstrap.NewSystemServiceSession().Principal + var input Input if err := json.NewDecoder(strings.NewReader(data)).Decode(&input); err != nil { return "", fmt.Errorf("failed to unmarshal job input: %w", err) @@ -92,26 +159,44 @@ func (i *Repository) Handle(ctx context.Context, data string, _ job.ProgressRepo return "", fmt.Errorf("repository %s is not being imported", repo.UID) } - writeParams, err := createRPCWriteParams(ctx, i.urlProvider, repo) + gitUID, err := i.createGitRepository(ctx, &systemPrincipal, repo.ID) if err != nil { - return "", fmt.Errorf("failed to create write params: %w", err) + return "", fmt.Errorf("failed to create empty git repository: %w", err) } - syncOut, err := i.git.SyncRepository(ctx, &gitrpc.SyncRepositoryParams{ - WriteParams: writeParams, - Source: input.CloneURL, - CreateIfNotExists: false, - }) - if err != nil { - return "", fmt.Errorf("failed to sync repositories: %w", err) - } + err = func() error { + repo.GitUID = gitUID - repo.Importing = false - repo.DefaultBranch = syncOut.DefaultBranch + defaultBranch, err := i.syncGitRepository(ctx, &systemPrincipal, repo, input.CloneURL) + if err != nil { + return fmt.Errorf("failed to sync git repository from '%s': %w", input.CloneURL, err) + } - err = i.repoStore.Update(ctx, repo) + repo, err = i.repoStore.UpdateOptLock(ctx, repo, func(repo *types.Repository) error { + if !repo.Importing { + return errors.New("repository has already finished importing") + } + + repo.GitUID = gitUID + repo.DefaultBranch = defaultBranch + repo.Importing = false + + return nil + }) + if err != nil { + return fmt.Errorf("failed to update repository after import: %w", err) + } + + return nil + }() if err != nil { - return "", fmt.Errorf("failed to update repository after import: %w", err) + if errDel := i.deleteGitRepository(ctx, &systemPrincipal, repo); errDel != nil { + log.Ctx(ctx).Err(err). + Str("gitUID", gitUID). + Msg("failed to delete git repository after failed import") + } + + return "", fmt.Errorf("failed to import repository: %w", err) } return "", err @@ -135,31 +220,130 @@ func (i *Repository) GetProgress(ctx context.Context, repo *types.Repository) (t return progress, nil } -// CreateRPCWriteParams creates base write parameters for gitrpc write operations. -func createRPCWriteParams(ctx context.Context, - urlProvider *gitnessurl.Provider, +func (i *Repository) Cancel(ctx context.Context, repo *types.Repository) error { + if repo.ImportingJobUID == nil || *repo.ImportingJobUID == "" { + return nil + } + + err := i.scheduler.CancelJob(ctx, *repo.ImportingJobUID) + if err != nil { + return fmt.Errorf("failed to cancel job: %w", err) + } + + return nil +} + +func (i *Repository) createGitRepository(ctx context.Context, + principal *types.Principal, + repoID int64, +) (string, error) { + now := time.Now() + + envVars, err := i.createEnvVars(ctx, principal, repoID) + if err != nil { + return "", err + } + + resp, err := i.git.CreateRepository(ctx, &gitrpc.CreateRepositoryParams{ + Actor: gitrpc.Identity{ + Name: principal.DisplayName, + Email: principal.Email, + }, + EnvVars: envVars, + DefaultBranch: i.defaultBranch, + Files: nil, + Author: &gitrpc.Identity{ + Name: principal.DisplayName, + Email: principal.Email, + }, + AuthorDate: &now, + Committer: &gitrpc.Identity{ + Name: principal.DisplayName, + Email: principal.Email, + }, + CommitterDate: &now, + }) + if err != nil { + return "", fmt.Errorf("failed to create empty git repository: %w", err) + } + + return resp.UID, nil +} + +func (i *Repository) syncGitRepository(ctx context.Context, + principal *types.Principal, + repo *types.Repository, + sourceCloneURL string, +) (string, error) { + writeParams, err := i.createRPCWriteParams(ctx, principal, repo) + if err != nil { + return "", err + } + + syncOut, err := i.git.SyncRepository(ctx, &gitrpc.SyncRepositoryParams{ + WriteParams: writeParams, + Source: sourceCloneURL, + CreateIfNotExists: false, + }) + if err != nil { + return "", fmt.Errorf("failed to sync repository: %w", err) + } + + return syncOut.DefaultBranch, nil +} + +func (i *Repository) deleteGitRepository(ctx context.Context, + principal *types.Principal, + repo *types.Repository, +) error { + writeParams, err := i.createRPCWriteParams(ctx, principal, repo) + if err != nil { + return err + } + + err = i.git.DeleteRepository(ctx, &gitrpc.DeleteRepositoryParams{ + WriteParams: writeParams, + }) + if err != nil { + return fmt.Errorf("failed to delete git repository: %w", err) + } + + return nil +} + +func (i *Repository) createRPCWriteParams(ctx context.Context, + principal *types.Principal, repo *types.Repository, ) (gitrpc.WriteParams, error) { - gitnessSession := bootstrap.NewSystemServiceSession() - - // generate envars (add everything githook CLI needs for execution) - envVars, err := githook.GenerateEnvironmentVariables( - ctx, - urlProvider.GetAPIBaseURLInternal(), - repo.ID, - gitnessSession.Principal.ID, - false, - ) + envVars, err := i.createEnvVars(ctx, principal, repo.ID) if err != nil { - return gitrpc.WriteParams{}, fmt.Errorf("failed to generate git hook environment variables: %w", err) + return gitrpc.WriteParams{}, err } return gitrpc.WriteParams{ Actor: gitrpc.Identity{ - Name: gitnessSession.Principal.DisplayName, - Email: gitnessSession.Principal.Email, + Name: principal.DisplayName, + Email: principal.Email, }, RepoUID: repo.GitUID, EnvVars: envVars, }, nil } + +func (i *Repository) createEnvVars(ctx context.Context, + principal *types.Principal, + repoID int64, +) (map[string]string, error) { + envVars, err := githook.GenerateEnvironmentVariables( + ctx, + i.urlProvider.GetAPIBaseURLInternal(), + repoID, + principal.ID, + false, + ) + if err != nil { + return nil, fmt.Errorf("failed to generate git hook environment variables: %w", err) + } + + return envVars, nil +} diff --git a/internal/services/importer/wire.go b/internal/services/importer/wire.go index da3e4aa1e..44328b4bf 100644 --- a/internal/services/importer/wire.go +++ b/internal/services/importer/wire.go @@ -9,6 +9,7 @@ import ( "github.com/harness/gitness/internal/services/job" "github.com/harness/gitness/internal/store" "github.com/harness/gitness/internal/url" + "github.com/harness/gitness/types" "github.com/google/wire" ) @@ -18,6 +19,7 @@ var WireSet = wire.NewSet( ) func ProvideRepoImporter( + config *types.Config, urlProvider *url.Provider, git gitrpc.Interface, repoStore store.RepoStore, @@ -25,10 +27,11 @@ func ProvideRepoImporter( executor *job.Executor, ) (*Repository, error) { importer := &Repository{ - urlProvider: urlProvider, - git: git, - repoStore: repoStore, - scheduler: scheduler, + defaultBranch: config.Git.DefaultBranch, + urlProvider: urlProvider, + git: git, + repoStore: repoStore, + scheduler: scheduler, } err := executor.Register(jobType, importer) diff --git a/internal/services/job/scheduler.go b/internal/services/job/scheduler.go index d82afae38..1ca8545cb 100644 --- a/internal/services/job/scheduler.go +++ b/internal/services/job/scheduler.go @@ -35,9 +35,12 @@ type Scheduler struct { maxRunning int purgeMinOldAge time.Duration + // global context + globalCtx context.Context + // synchronization stuff - globalCtx context.Context signal chan time.Time + signalEdgy chan struct{} done chan struct{} wgRunning sync.WaitGroup cancelJobMx sync.Mutex @@ -102,6 +105,7 @@ func (s *Scheduler) Run(ctx context.Context) error { defer close(s.done) s.signal = make(chan time.Time, 1) + s.signalEdgy = make(chan struct{}, 1) s.globalCtx = ctx timer := newSchedulerTimer() @@ -122,6 +126,10 @@ func (s *Scheduler) Run(ctx context.Context) error { case <-ctx.Done(): return ctx.Err() + case <-s.signalEdgy: + timer.MakeEdgy() + return nil + case newTime := <-s.signal: dur := timer.RescheduleEarlier(newTime) if dur > 0 { @@ -256,6 +264,13 @@ func (s *Scheduler) scheduleProcessing(scheduled time.Time) { }() } +func (s *Scheduler) makeTimerEdgy() { + select { + case s.signalEdgy <- struct{}{}: + default: + } +} + // scheduleIfHaveMoreJobs triggers processing of ready jobs if the timer is edgy. // The timer would be edgy if the previous iteration found more jobs that it could start (full capacity). // This should be run after a non-recurring job has finished. @@ -276,14 +291,14 @@ func (s *Scheduler) RunJob(ctx context.Context, def Definition) error { // RunJobs runs a several jobs. It's more efficient than calling RunJob several times // because it locks the DB only once. -// TODO: Add groupID parameter and use it for all jobs. -func (s *Scheduler) RunJobs(ctx context.Context, defs []Definition) error { +func (s *Scheduler) RunJobs(ctx context.Context, groupID string, defs []Definition) error { jobs := make([]*types.Job, len(defs)) for i, def := range defs { if err := def.Validate(); err != nil { return err } jobs[i] = def.toNewJob() + jobs[i].GroupID = groupID } return s.startNewJobs(ctx, jobs) @@ -310,6 +325,8 @@ func (s *Scheduler) startNewJobsNoLock(ctx context.Context, jobs []*types.Job) e return fmt.Errorf("failed to count available slots for job execution: %w", err) } + canRunAll := available >= len(jobs) + for _, job := range jobs { if available > 0 { available-- @@ -340,6 +357,10 @@ func (s *Scheduler) startNewJobsNoLock(ctx context.Context, jobs []*types.Job) e }(s.globalCtx) } + if !canRunAll { + s.makeTimerEdgy() + } + return nil } diff --git a/internal/services/job/timer.go b/internal/services/job/timer.go index dfc93f70a..819cb884b 100644 --- a/internal/services/job/timer.go +++ b/internal/services/job/timer.go @@ -31,6 +31,11 @@ func (t *schedulerTimer) ResetAt(next time.Time, edgy bool) time.Duration { return t.resetAt(time.Now(), next, edgy) } +// MakeEdgy makes the timer edgy which meant it will be triggered immediately on reschedule attempt. +func (t *schedulerTimer) MakeEdgy() { + t.edgy = true +} + func (t *schedulerTimer) resetAt(now, next time.Time, edgy bool) time.Duration { var dur time.Duration diff --git a/internal/store/database/job.go b/internal/store/database/job.go index 0e8d4e5a9..cb20bb0db 100644 --- a/internal/store/database/job.go +++ b/internal/store/database/job.go @@ -54,7 +54,8 @@ const ( ,job_is_recurring ,job_recurring_cron ,job_consecutive_failures - ,job_last_failure_error` + ,job_last_failure_error + ,job_group_id` jobSelectBase = ` SELECT` + jobColumns + ` @@ -101,6 +102,7 @@ func (s *JobStore) Create(ctx context.Context, job *types.Job) error { ,:job_recurring_cron ,:job_consecutive_failures ,:job_last_failure_error + ,:job_group_id )` db := dbtx.GetAccessor(ctx, s.db) @@ -143,6 +145,7 @@ func (s *JobStore) Upsert(ctx context.Context, job *types.Job) error { ,:job_recurring_cron ,:job_consecutive_failures ,:job_last_failure_error + ,:job_group_id ) ON CONFLICT (job_uid) DO UPDATE SET @@ -196,6 +199,7 @@ func (s *JobStore) UpdateDefinition(ctx context.Context, job *types.Job) error { ,job_scheduled = :job_scheduled ,job_is_recurring = :job_is_recurring ,job_recurring_cron = :job_recurring_cron + ,job_group_id = :job_group_id WHERE job_uid = :job_uid` db := dbtx.GetAccessor(ctx, s.db) diff --git a/internal/store/database/migrate/postgres/0025_alter_table_job_add_group_id.down.sql b/internal/store/database/migrate/postgres/0025_alter_table_job_add_group_id.down.sql new file mode 100644 index 000000000..a740594cc --- /dev/null +++ b/internal/store/database/migrate/postgres/0025_alter_table_job_add_group_id.down.sql @@ -0,0 +1 @@ +ALTER TABLE jobs DROP COLUMN job_group_id; diff --git a/internal/store/database/migrate/postgres/0025_alter_table_job_add_group_id.up.sql b/internal/store/database/migrate/postgres/0025_alter_table_job_add_group_id.up.sql new file mode 100644 index 000000000..86a19161b --- /dev/null +++ b/internal/store/database/migrate/postgres/0025_alter_table_job_add_group_id.up.sql @@ -0,0 +1 @@ +ALTER TABLE jobs ADD COLUMN job_group_id TEXT NOT NULL DEFAULT ''; diff --git a/internal/store/database/migrate/sqlite/0025_alter_table_job_add_group_id.down.sql b/internal/store/database/migrate/sqlite/0025_alter_table_job_add_group_id.down.sql new file mode 100644 index 000000000..a740594cc --- /dev/null +++ b/internal/store/database/migrate/sqlite/0025_alter_table_job_add_group_id.down.sql @@ -0,0 +1 @@ +ALTER TABLE jobs DROP COLUMN job_group_id; diff --git a/internal/store/database/migrate/sqlite/0025_alter_table_job_add_group_id.up.sql b/internal/store/database/migrate/sqlite/0025_alter_table_job_add_group_id.up.sql new file mode 100644 index 000000000..86a19161b --- /dev/null +++ b/internal/store/database/migrate/sqlite/0025_alter_table_job_add_group_id.up.sql @@ -0,0 +1 @@ +ALTER TABLE jobs ADD COLUMN job_group_id TEXT NOT NULL DEFAULT ''; diff --git a/internal/store/database/repo.go b/internal/store/database/repo.go index 3091f9f85..6770ce57d 100644 --- a/internal/store/database/repo.go +++ b/internal/store/database/repo.go @@ -170,21 +170,22 @@ func (s *RepoStore) Update(ctx context.Context, repo *types.Repository) error { const sqlQuery = ` UPDATE repositories SET - repo_version = :repo_version - ,repo_updated = :repo_updated - ,repo_parent_id = :repo_parent_id - ,repo_uid = :repo_uid - ,repo_description = :repo_description - ,repo_is_public = :repo_is_public - ,repo_default_branch = :repo_default_branch - ,repo_pullreq_seq = :repo_pullreq_seq - ,repo_num_forks = :repo_num_forks - ,repo_num_pulls = :repo_num_pulls - ,repo_num_closed_pulls = :repo_num_closed_pulls - ,repo_num_open_pulls = :repo_num_open_pulls - ,repo_num_merged_pulls = :repo_num_merged_pulls - ,repo_importing = :repo_importing - ,repo_importing_job_uid = :repo_importing_job_uid + repo_version = :repo_version + ,repo_updated = :repo_updated + ,repo_parent_id = :repo_parent_id + ,repo_uid = :repo_uid + ,repo_git_uid = :repo_git_uid + ,repo_description = :repo_description + ,repo_is_public = :repo_is_public + ,repo_default_branch = :repo_default_branch + ,repo_pullreq_seq = :repo_pullreq_seq + ,repo_num_forks = :repo_num_forks + ,repo_num_pulls = :repo_num_pulls + ,repo_num_closed_pulls = :repo_num_closed_pulls + ,repo_num_open_pulls = :repo_num_open_pulls + ,repo_num_merged_pulls = :repo_num_merged_pulls + ,repo_importing = :repo_importing + ,repo_importing_job_uid = :repo_importing_job_uid WHERE repo_id = :repo_id AND repo_version = :repo_version - 1` updatedAt := time.Now() @@ -219,7 +220,8 @@ func (s *RepoStore) Update(ctx context.Context, repo *types.Repository) error { // UpdateOptLock updates the repository using the optimistic locking mechanism. func (s *RepoStore) UpdateOptLock(ctx context.Context, repo *types.Repository, - mutateFn func(repository *types.Repository) error) (*types.Repository, error) { + mutateFn func(repository *types.Repository) error, +) (*types.Repository, error) { for { dup := *repo diff --git a/types/job.go b/types/job.go index 37ae01c50..1a6a32fac 100644 --- a/types/job.go +++ b/types/job.go @@ -27,6 +27,7 @@ type Job struct { RecurringCron string `db:"job_recurring_cron"` ConsecutiveFailures int `db:"job_consecutive_failures"` LastFailureError string `db:"job_last_failure_error"` + GroupID string `db:"job_group_id"` } type JobStateChange struct {