diff --git a/app/githook/client_controller.go b/app/api/controller/githook/client.go similarity index 93% rename from app/githook/client_controller.go rename to app/api/controller/githook/client.go index 30eef3428..d2631da03 100644 --- a/app/githook/client_controller.go +++ b/app/api/controller/githook/client.go @@ -19,7 +19,7 @@ import ( "errors" "fmt" - "github.com/harness/gitness/app/api/controller/githook" + "github.com/harness/gitness/app/githook" "github.com/harness/gitness/git" "github.com/harness/gitness/git/hook" "github.com/harness/gitness/store" @@ -33,12 +33,12 @@ var _ hook.Client = (*ControllerClient)(nil) // ControllerClientFactory creates clients that directly call the controller to execute githooks. type ControllerClientFactory struct { - githookCtrl *githook.Controller + githookCtrl *Controller git git.Interface } func (f *ControllerClientFactory) NewClient(_ context.Context, envVars map[string]string) (hook.Client, error) { - payload, err := hook.LoadPayloadFromMap[Payload](envVars) + payload, err := hook.LoadPayloadFromMap[githook.Payload](envVars) if err != nil { return nil, fmt.Errorf("failed to load payload from provided map of environment variables: %w", err) } @@ -53,7 +53,7 @@ func (f *ControllerClientFactory) NewClient(_ context.Context, envVars map[strin } return &ControllerClient{ - baseInput: GetInputBaseFromPayload(payload), + baseInput: githook.GetInputBaseFromPayload(payload), githookCtrl: f.githookCtrl, git: f.git, }, nil @@ -62,8 +62,8 @@ func (f *ControllerClientFactory) NewClient(_ context.Context, envVars map[strin // ControllerClient directly calls the controller to execute githooks. type ControllerClient struct { baseInput types.GithookInputBase - githookCtrl *githook.Controller - git githook.RestrictedGIT + githookCtrl *Controller + git RestrictedGIT } func (c *ControllerClient) PreReceive( diff --git a/app/api/controller/githook/controller.go b/app/api/controller/githook/controller.go index 9116e1196..51f16d1dd 100644 --- a/app/api/controller/githook/controller.go +++ b/app/api/controller/githook/controller.go @@ -23,6 +23,7 @@ import ( "github.com/harness/gitness/app/auth" "github.com/harness/gitness/app/auth/authz" eventsgit "github.com/harness/gitness/app/events/git" + eventsrepo "github.com/harness/gitness/app/events/repo" "github.com/harness/gitness/app/services/protection" "github.com/harness/gitness/app/services/settings" "github.com/harness/gitness/app/store" @@ -41,6 +42,8 @@ type Controller struct { principalStore store.PrincipalStore repoStore store.RepoStore gitReporter *eventsgit.Reporter + repoReporter *eventsrepo.Reporter + git git.Interface pullreqStore store.PullReqStore urlProvider url.Provider protectionManager *protection.Manager @@ -56,6 +59,8 @@ func NewController( principalStore store.PrincipalStore, repoStore store.RepoStore, gitReporter *eventsgit.Reporter, + repoReporter *eventsrepo.Reporter, + git git.Interface, pullreqStore store.PullReqStore, urlProvider url.Provider, protectionManager *protection.Manager, @@ -64,13 +69,14 @@ func NewController( preReceiveExtender PreReceiveExtender, updateExtender UpdateExtender, postReceiveExtender PostReceiveExtender, - ) *Controller { return &Controller{ authorizer: authorizer, principalStore: principalStore, repoStore: repoStore, gitReporter: gitReporter, + repoReporter: repoReporter, + git: git, pullreqStore: pullreqStore, urlProvider: urlProvider, protectionManager: protectionManager, diff --git a/app/api/controller/githook/post_receive.go b/app/api/controller/githook/post_receive.go index ebc29f267..5c633fdb9 100644 --- a/app/api/controller/githook/post_receive.go +++ b/app/api/controller/githook/post_receive.go @@ -19,13 +19,17 @@ import ( "fmt" "strings" + "github.com/harness/gitness/app/api/usererror" "github.com/harness/gitness/app/auth" + "github.com/harness/gitness/app/bootstrap" events "github.com/harness/gitness/app/events/git" + repoevents "github.com/harness/gitness/app/events/repo" "github.com/harness/gitness/git" "github.com/harness/gitness/git/hook" "github.com/harness/gitness/types" "github.com/harness/gitness/types/enum" + "github.com/gotidy/ptr" "github.com/rs/zerolog/log" ) @@ -51,13 +55,16 @@ func (c *Controller) PostReceive( if err != nil { return hook.Output{}, err } + // create output object and have following messages fill its messages + out := hook.Output{} + + // update default branch based on ref update info on empty repos. + // as the branch could be different than the configured default value. + c.handleEmptyRepoPush(ctx, repo, in.PostReceiveInput, &out) // report ref events (best effort) c.reportReferenceEvents(ctx, rgit, repo, in.PrincipalID, in.PostReceiveInput) - // create output object and have following messages fill its messages - out := hook.Output{} - // handle branch updates related to PRs - best effort c.handlePRMessaging(ctx, repo, in.PostReceiveInput, &out) @@ -250,3 +257,50 @@ func (c *Controller) suggestPullRequest( " "+c.urlProvider.GenerateUICompareURL(repo.Path, repo.DefaultBranch, branchName), ) } + +// handleEmptyRepoPush updates repo default branch on empty repos if push contains branches. +func (c *Controller) handleEmptyRepoPush( + ctx context.Context, + repo *types.Repository, + in hook.PostReceiveInput, + out *hook.Output, +) { + if !repo.IsEmpty { + return + } + + var branchName string + // we only care about one active branch that was pushed. + for _, refUpdate := range in.RefUpdates { + if strings.HasPrefix(refUpdate.Ref, gitReferenceNamePrefixBranch) && + refUpdate.New.String() != types.NilSHA { + branchName = refUpdate.Ref[len(gitReferenceNamePrefixBranch):] + break + } + } + if branchName == "" { + out.Error = ptr.String(usererror.ErrEmptyRepoNeedsBranch.Error()) + return + } + + oldName := repo.DefaultBranch + var err error + repo, err = c.repoStore.UpdateOptLock(ctx, repo, func(r *types.Repository) error { + r.IsEmpty = false + r.DefaultBranch = branchName + return nil + }) + if err != nil { + log.Ctx(ctx).Warn().Err(err).Msgf("failed to update the repo default branch to %s and is_empty to false", branchName) + return + } + + if repo.DefaultBranch != oldName { + c.repoReporter.DefaultBranchUpdated(ctx, &repoevents.DefaultBranchUpdatedPayload{ + RepoID: repo.ID, + PrincipalID: bootstrap.NewSystemServiceSession().Principal.ID, + OldName: oldName, + NewName: repo.DefaultBranch, + }) + } +} diff --git a/app/api/controller/githook/wire.go b/app/api/controller/githook/wire.go index ddb7f6464..a54202196 100644 --- a/app/api/controller/githook/wire.go +++ b/app/api/controller/githook/wire.go @@ -14,11 +14,76 @@ package githook -import "github.com/google/wire" +import ( + "github.com/harness/gitness/app/api/controller/limiter" + "github.com/harness/gitness/app/auth/authz" + eventsgit "github.com/harness/gitness/app/events/git" + eventsrepo "github.com/harness/gitness/app/events/repo" + "github.com/harness/gitness/app/services/protection" + "github.com/harness/gitness/app/services/settings" + "github.com/harness/gitness/app/store" + "github.com/harness/gitness/app/url" + "github.com/harness/gitness/git" + "github.com/harness/gitness/git/hook" -// Due to cyclic injection dependencies, wiring can be found at app/githook/wire.go + "github.com/google/wire" +) var WireSet = wire.NewSet( + ProvideController, + ProvideFactory, +) + +func ProvideFactory() hook.ClientFactory { + return &ControllerClientFactory{ + githookCtrl: nil, + } +} + +func ProvideController( + authorizer authz.Authorizer, + principalStore store.PrincipalStore, + repoStore store.RepoStore, + gitReporter *eventsgit.Reporter, + repoReporter *eventsrepo.Reporter, + git git.Interface, + pullreqStore store.PullReqStore, + urlProvider url.Provider, + protectionManager *protection.Manager, + githookFactory hook.ClientFactory, + limiter limiter.ResourceLimiter, + settings *settings.Service, + preReceiveExtender PreReceiveExtender, + updateExtender UpdateExtender, + postReceiveExtender PostReceiveExtender, +) *Controller { + ctrl := NewController( + authorizer, + principalStore, + repoStore, + gitReporter, + repoReporter, + git, + pullreqStore, + urlProvider, + protectionManager, + limiter, + settings, + preReceiveExtender, + updateExtender, + postReceiveExtender, + ) + + // TODO: improve wiring if possible + if fct, ok := githookFactory.(*ControllerClientFactory); ok { + fct.githookCtrl = ctrl + fct.git = git + } + + return ctrl +} + +var ExtenderWireSet = wire.NewSet( ProvidePreReceiveExtender, ProvideUpdateExtender, ProvidePostReceiveExtender, diff --git a/app/api/controller/pullreq/controller.go b/app/api/controller/pullreq/controller.go index 4bef91b2c..554933bd0 100644 --- a/app/api/controller/pullreq/controller.go +++ b/app/api/controller/pullreq/controller.go @@ -25,6 +25,7 @@ import ( pullreqevents "github.com/harness/gitness/app/events/pullreq" "github.com/harness/gitness/app/services/codecomments" "github.com/harness/gitness/app/services/codeowners" + locker "github.com/harness/gitness/app/services/locker" "github.com/harness/gitness/app/services/protection" "github.com/harness/gitness/app/services/pullreq" "github.com/harness/gitness/app/sse" @@ -33,7 +34,6 @@ import ( "github.com/harness/gitness/errors" "github.com/harness/gitness/git" gitenum "github.com/harness/gitness/git/enum" - "github.com/harness/gitness/lock" "github.com/harness/gitness/store/database/dbtx" "github.com/harness/gitness/types" "github.com/harness/gitness/types/enum" @@ -55,12 +55,12 @@ type Controller struct { checkStore store.CheckStore git git.Interface eventReporter *pullreqevents.Reporter - mtxManager lock.MutexManager codeCommentMigrator *codecomments.Migrator pullreqService *pullreq.Service protectionManager *protection.Manager sseStreamer sse.Streamer codeOwners *codeowners.Service + locker *locker.Locker } func NewController( @@ -79,12 +79,12 @@ func NewController( checkStore store.CheckStore, git git.Interface, eventReporter *pullreqevents.Reporter, - mtxManager lock.MutexManager, codeCommentMigrator *codecomments.Migrator, pullreqService *pullreq.Service, protectionManager *protection.Manager, sseStreamer sse.Streamer, codeowners *codeowners.Service, + locker *locker.Locker, ) *Controller { return &Controller{ tx: tx, @@ -103,11 +103,11 @@ func NewController( git: git, codeCommentMigrator: codeCommentMigrator, eventReporter: eventReporter, - mtxManager: mtxManager, pullreqService: pullreqService, protectionManager: protectionManager, sseStreamer: sseStreamer, codeOwners: codeowners, + locker: locker, } } diff --git a/app/api/controller/pullreq/locks.go b/app/api/controller/pullreq/locks.go deleted file mode 100644 index 5efa6cfa2..000000000 --- a/app/api/controller/pullreq/locks.go +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2023 Harness, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package pullreq - -import ( - "context" - "fmt" - "strconv" - "time" - - "github.com/harness/gitness/contextutil" - "github.com/harness/gitness/lock" - "github.com/harness/gitness/logging" - - "github.com/rs/zerolog" - "github.com/rs/zerolog/log" -) - -func (c *Controller) lockPR( - ctx context.Context, - repoID int64, - prNum int64, - expiry time.Duration, -) (func(), error) { - key := fmt.Sprintf("%d/pulls", repoID) - if prNum != 0 { - key += "/" + strconv.FormatInt(prNum, 10) - } - - // annotate logs for easier debugging of lock related merge issues - // TODO: refactor once common logging annotations are added - ctx = logging.NewContext(ctx, func(c zerolog.Context) zerolog.Context { - return c. - Str("pullreq_lock", key). - Int64("repo_id", repoID) - }) - - mutex, err := c.mtxManager.NewMutex( - key, - lock.WithNamespace("repo"), - lock.WithExpiry(expiry), - lock.WithTimeoutFactor(4/expiry.Seconds()), // 4s - ) - if err != nil { - return nil, fmt.Errorf("failed to create new mutex for pr %d in repo %d: %w", prNum, repoID, err) - } - err = mutex.Lock(ctx) - if err != nil { - return nil, fmt.Errorf("failed to lock mutex for pr %d in repo %d: %w", prNum, repoID, err) - } - - log.Ctx(ctx).Debug().Msgf("successfully locked PR (expiry: %s)", expiry) - - unlockFn := func() { - // always unlock independent of whether source context got canceled or not - ctx, cancel := context.WithTimeout( - contextutil.WithNewValues(context.Background(), ctx), - 30*time.Second, - ) - defer cancel() - - err := mutex.Unlock(ctx) - if err != nil { - log.Ctx(ctx).Warn().Err(err).Msg("failed to unlock PR") - } else { - log.Ctx(ctx).Debug().Msg("successfully unlocked PR") - } - } - - return unlockFn, nil -} diff --git a/app/api/controller/pullreq/merge.go b/app/api/controller/pullreq/merge.go index 9d7626ccf..a19ef7ccd 100644 --- a/app/api/controller/pullreq/merge.go +++ b/app/api/controller/pullreq/merge.go @@ -117,7 +117,7 @@ func (c *Controller) Merge( // first one and second one will wait, when first one is done then second one // continue with latest data from db with state merged and return error that // pr is already merged. - unlock, err := c.lockPR( + unlock, err := c.locker.LockPR( ctx, targetRepo.ID, 0, // 0 means locks all PRs for this repo diff --git a/app/api/controller/pullreq/wire.go b/app/api/controller/pullreq/wire.go index 52cb17aa0..5019471be 100644 --- a/app/api/controller/pullreq/wire.go +++ b/app/api/controller/pullreq/wire.go @@ -19,13 +19,13 @@ import ( pullreqevents "github.com/harness/gitness/app/events/pullreq" "github.com/harness/gitness/app/services/codecomments" "github.com/harness/gitness/app/services/codeowners" + "github.com/harness/gitness/app/services/locker" "github.com/harness/gitness/app/services/protection" "github.com/harness/gitness/app/services/pullreq" "github.com/harness/gitness/app/sse" "github.com/harness/gitness/app/store" "github.com/harness/gitness/app/url" "github.com/harness/gitness/git" - "github.com/harness/gitness/lock" "github.com/harness/gitness/store/database/dbtx" "github.com/google/wire" @@ -43,10 +43,9 @@ func ProvideController(tx dbtx.Transactor, urlProvider url.Provider, authorizer repoStore store.RepoStore, principalStore store.PrincipalStore, fileViewStore store.PullReqFileViewStore, membershipStore store.MembershipStore, checkStore store.CheckStore, - rpcClient git.Interface, eventReporter *pullreqevents.Reporter, - mtxManager lock.MutexManager, codeCommentMigrator *codecomments.Migrator, + rpcClient git.Interface, eventReporter *pullreqevents.Reporter, codeCommentMigrator *codecomments.Migrator, pullreqService *pullreq.Service, ruleManager *protection.Manager, sseStreamer sse.Streamer, - codeOwners *codeowners.Service, + codeOwners *codeowners.Service, locker *locker.Locker, ) *Controller { return NewController(tx, urlProvider, authorizer, pullReqStore, pullReqActivityStore, @@ -56,6 +55,6 @@ func ProvideController(tx dbtx.Transactor, urlProvider url.Provider, authorizer fileViewStore, membershipStore, checkStore, rpcClient, eventReporter, - mtxManager, codeCommentMigrator, - pullreqService, ruleManager, sseStreamer, codeOwners) + codeCommentMigrator, + pullreqService, ruleManager, sseStreamer, codeOwners, locker) } diff --git a/app/api/controller/repo/controller.go b/app/api/controller/repo/controller.go index b0f971389..24f04a7b7 100644 --- a/app/api/controller/repo/controller.go +++ b/app/api/controller/repo/controller.go @@ -29,6 +29,7 @@ import ( "github.com/harness/gitness/app/services/codeowners" "github.com/harness/gitness/app/services/importer" "github.com/harness/gitness/app/services/keywordsearch" + "github.com/harness/gitness/app/services/locker" "github.com/harness/gitness/app/services/protection" "github.com/harness/gitness/app/services/settings" "github.com/harness/gitness/app/store" @@ -66,6 +67,7 @@ type Controller struct { eventReporter *repoevents.Reporter indexer keywordsearch.Indexer resourceLimiter limiter.ResourceLimiter + locker *locker.Locker mtxManager lock.MutexManager identifierCheck check.RepoIdentifier repoCheck Check @@ -90,6 +92,7 @@ func NewController( eventReporter *repoevents.Reporter, indexer keywordsearch.Indexer, limiter limiter.ResourceLimiter, + locker *locker.Locker, mtxManager lock.MutexManager, identifierCheck check.RepoIdentifier, repoCheck Check, @@ -114,6 +117,7 @@ func NewController( eventReporter: eventReporter, indexer: indexer, resourceLimiter: limiter, + locker: locker, mtxManager: mtxManager, identifierCheck: identifierCheck, repoCheck: repoCheck, diff --git a/app/api/controller/repo/create.go b/app/api/controller/repo/create.go index e44aff627..9e0c5dd16 100644 --- a/app/api/controller/repo/create.go +++ b/app/api/controller/repo/create.go @@ -81,7 +81,7 @@ func (c *Controller) Create(ctx context.Context, session *auth.Session, in *Crea return fmt.Errorf("resource limit exceeded: %w", limiter.ErrMaxNumReposReached) } - gitResp, err := c.createGitRepository(ctx, session, in) + gitResp, isEmpty, err := c.createGitRepository(ctx, session, in) if err != nil { return fmt.Errorf("error creating repository on git: %w", err) } @@ -99,6 +99,7 @@ func (c *Controller) Create(ctx context.Context, session *auth.Session, in *Crea Updated: now, ForkID: in.ForkID, DefaultBranch: in.DefaultBranch, + IsEmpty: isEmpty, } err = c.repoStore.Create(ctx, repo) if err != nil { @@ -118,7 +119,7 @@ func (c *Controller) Create(ctx context.Context, session *auth.Session, in *Crea repo.GitURL = c.urlProvider.GenerateGITCloneURL(repo.Path) // index repository if files are created - if in.Readme || in.GitIgnore != "" || (in.License != "" && in.License != "none") { + if !repo.IsEmpty { err = c.indexer.Index(ctx, repo) if err != nil { log.Ctx(ctx).Warn().Err(err).Int64("repo_id", repo.ID).Msg("failed to index repo") @@ -186,7 +187,7 @@ func (c *Controller) sanitizeCreateInput(in *CreateInput) error { } func (c *Controller) createGitRepository(ctx context.Context, session *auth.Session, - in *CreateInput) (*git.CreateRepositoryOutput, error) { + in *CreateInput) (*git.CreateRepositoryOutput, bool, error) { var ( err error content []byte @@ -202,7 +203,7 @@ func (c *Controller) createGitRepository(ctx context.Context, session *auth.Sess if in.License != "" && in.License != "none" { content, err = resources.ReadLicense(in.License) if err != nil { - return nil, fmt.Errorf("failed to read license '%s': %w", in.License, err) + return nil, false, fmt.Errorf("failed to read license '%s': %w", in.License, err) } files = append(files, git.File{ Path: "LICENSE", @@ -212,7 +213,7 @@ func (c *Controller) createGitRepository(ctx context.Context, session *auth.Sess if in.GitIgnore != "" { content, err = resources.ReadGitIgnore(in.GitIgnore) if err != nil { - return nil, fmt.Errorf("failed to read git ignore '%s': %w", in.GitIgnore, err) + return nil, false, fmt.Errorf("failed to read git ignore '%s': %w", in.GitIgnore, err) } files = append(files, git.File{ Path: ".gitignore", @@ -230,7 +231,7 @@ func (c *Controller) createGitRepository(ctx context.Context, session *auth.Sess true, ) if err != nil { - return nil, fmt.Errorf("failed to generate git hook environment variables: %w", err) + return nil, false, fmt.Errorf("failed to generate git hook environment variables: %w", err) } actor := identityFromPrincipal(session.Principal) @@ -247,10 +248,10 @@ func (c *Controller) createGitRepository(ctx context.Context, session *auth.Sess CommitterDate: &now, }) if err != nil { - return nil, fmt.Errorf("failed to create repo on: %w", err) + return nil, false, fmt.Errorf("failed to create repo on: %w", err) } - return resp, nil + return resp, len(files) == 0, nil } func createReadme(name, description string) []byte { diff --git a/app/api/controller/repo/default_branch.go b/app/api/controller/repo/default_branch.go index c1710b34b..b78da4cd1 100644 --- a/app/api/controller/repo/default_branch.go +++ b/app/api/controller/repo/default_branch.go @@ -21,12 +21,12 @@ import ( "github.com/harness/gitness/app/api/controller" "github.com/harness/gitness/app/auth" + "github.com/harness/gitness/app/bootstrap" + repoevents "github.com/harness/gitness/app/events/repo" "github.com/harness/gitness/contextutil" "github.com/harness/gitness/git" "github.com/harness/gitness/types" "github.com/harness/gitness/types/enum" - - "github.com/rs/zerolog/log" ) type UpdateDefaultBranchInput struct { @@ -50,11 +50,11 @@ func (c *Controller) UpdateDefaultBranch( // lock concurrent requests for updating the default branch of a repo // requests will wait for previous ones to compelete before proceed - unlock, err := c.lockDefaultBranch( + unlock, err := c.locker.LockDefaultBranch( ctx, - repo.GitUID, - in.Name, // branch name only used for logging (lock is on repo) - timeout+30*time.Second, + repo.ID, + in.Name, // branch name only used for logging (lock is on repo) + timeout+30*time.Second, // add 30s to the lock to give enough time for updating default branch ) if err != nil { return nil, err @@ -82,6 +82,7 @@ func (c *Controller) UpdateDefaultBranch( return nil, fmt.Errorf("failed to update the repo default branch: %w", err) } + oldName := repo.DefaultBranch repo, err = c.repoStore.UpdateOptLock(ctx, repo, func(r *types.Repository) error { r.DefaultBranch = in.Name return nil @@ -90,11 +91,12 @@ func (c *Controller) UpdateDefaultBranch( return nil, fmt.Errorf("failed to update the repo default branch on db:%w", err) } - err = c.indexer.Index(ctx, repo) - if err != nil { - log.Ctx(ctx).Warn().Err(err).Int64("repo_id", repo.ID). - Msgf("failed to index repo with the updated default branch %s", in.Name) - } + c.eventReporter.DefaultBranchUpdated(ctx, &repoevents.DefaultBranchUpdatedPayload{ + RepoID: repo.ID, + PrincipalID: bootstrap.NewSystemServiceSession().Principal.ID, + OldName: oldName, + NewName: repo.DefaultBranch, + }) return repo, nil } diff --git a/app/api/controller/repo/wire.go b/app/api/controller/repo/wire.go index 288d77ae3..3b1caaf0e 100644 --- a/app/api/controller/repo/wire.go +++ b/app/api/controller/repo/wire.go @@ -21,6 +21,7 @@ import ( "github.com/harness/gitness/app/services/codeowners" "github.com/harness/gitness/app/services/importer" "github.com/harness/gitness/app/services/keywordsearch" + "github.com/harness/gitness/app/services/locker" "github.com/harness/gitness/app/services/protection" "github.com/harness/gitness/app/services/settings" "github.com/harness/gitness/app/store" @@ -58,6 +59,7 @@ func ProvideController( reporeporter *repoevents.Reporter, indexer keywordsearch.Indexer, limiter limiter.ResourceLimiter, + locker *locker.Locker, mtxManager lock.MutexManager, identifierCheck check.RepoIdentifier, repoChecks Check, @@ -65,8 +67,8 @@ func ProvideController( return NewController(config, tx, urlProvider, authorizer, repoStore, spaceStore, pipelineStore, - principalStore, ruleStore, settings, principalInfoCache, protectionManager, - rpcClient, importer, codeOwners, reporeporter, indexer, limiter, mtxManager, identifierCheck, repoChecks) + principalStore, ruleStore, settings, principalInfoCache, protectionManager, rpcClient, importer, + codeOwners, reporeporter, indexer, limiter, locker, mtxManager, identifierCheck, repoChecks) } func ProvideRepoCheck() Check { diff --git a/app/api/usererror/usererror.go b/app/api/usererror/usererror.go index 20f2cf28e..aaafb4832 100644 --- a/app/api/usererror/usererror.go +++ b/app/api/usererror/usererror.go @@ -88,6 +88,10 @@ var ( http.StatusLocked, "The requested resource is temporarily locked, please retry the operation.", ) + + // ErrEmptyRepoNeedsBranch is returned if no branch found on the githook post receieve for empty repositories. + ErrEmptyRepoNeedsBranch = New(http.StatusBadRequest, + "Pushing to an empty repository requires at least one branch with commits.") ) // Error represents a json-encoded API error. diff --git a/app/events/repo/events_repo.go b/app/events/repo/events_repo.go index 8c82247e4..9321f7075 100644 --- a/app/events/repo/events_repo.go +++ b/app/events/repo/events_repo.go @@ -45,3 +45,27 @@ func (r *Reader) RegisterRepoDeleted(fn events.HandlerFunc[*DeletedPayload], opts ...events.HandlerOption) error { return events.ReaderRegisterEvent(r.innerReader, DeletedEvent, fn, opts...) } + +const DefaultBranchUpdatedEvent events.EventType = "default-branch-updated" + +type DefaultBranchUpdatedPayload struct { + RepoID int64 `json:"repo_id"` + PrincipalID int64 `json:"principal_id"` + OldName string `json:"old_name"` + NewName string `json:"new_name"` +} + +func (r *Reporter) DefaultBranchUpdated(ctx context.Context, payload *DefaultBranchUpdatedPayload) { + eventID, err := events.ReporterSendEvent(r.innerReporter, ctx, DefaultBranchUpdatedEvent, payload) + if err != nil { + log.Ctx(ctx).Err(err).Msgf("failed to send default branch updated event") + return + } + + log.Ctx(ctx).Debug().Msgf("reported default branch updated event with id '%s'", eventID) +} + +func (r *Reader) RegisterDefaultBranchUpdated(fn events.HandlerFunc[*DefaultBranchUpdatedPayload], + opts ...events.HandlerOption) error { + return events.ReaderRegisterEvent(r.innerReader, DefaultBranchUpdatedEvent, fn, opts...) +} diff --git a/app/githook/wire.go b/app/githook/wire.go deleted file mode 100644 index 44a2a9d28..000000000 --- a/app/githook/wire.go +++ /dev/null @@ -1,83 +0,0 @@ -// Copyright 2023 Harness, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package githook - -import ( - "github.com/harness/gitness/app/api/controller/githook" - "github.com/harness/gitness/app/api/controller/limiter" - "github.com/harness/gitness/app/auth/authz" - eventsgit "github.com/harness/gitness/app/events/git" - "github.com/harness/gitness/app/services/protection" - "github.com/harness/gitness/app/services/settings" - "github.com/harness/gitness/app/store" - "github.com/harness/gitness/app/url" - "github.com/harness/gitness/git" - "github.com/harness/gitness/git/hook" - - "github.com/google/wire" -) - -// WireSet provides a wire set for this package. -var WireSet = wire.NewSet( - ProvideController, - ProvideFactory, -) - -func ProvideFactory() hook.ClientFactory { - return &ControllerClientFactory{ - // will be set in ProvideController (to break cyclic dependency during wiring) - githookCtrl: nil, - } -} - -func ProvideController( - authorizer authz.Authorizer, - principalStore store.PrincipalStore, - repoStore store.RepoStore, - gitReporter *eventsgit.Reporter, - git git.Interface, - pullreqStore store.PullReqStore, - urlProvider url.Provider, - protectionManager *protection.Manager, - githookFactory hook.ClientFactory, - limiter limiter.ResourceLimiter, - settings *settings.Service, - preReceiveExtender githook.PreReceiveExtender, - updateExtender githook.UpdateExtender, - postReceiveExtender githook.PostReceiveExtender, -) *githook.Controller { - ctrl := githook.NewController( - authorizer, - principalStore, - repoStore, - gitReporter, - pullreqStore, - urlProvider, - protectionManager, - limiter, - settings, - preReceiveExtender, - updateExtender, - postReceiveExtender, - ) - - // TODO: improve wiring if possible - if fct, ok := githookFactory.(*ControllerClientFactory); ok { - fct.githookCtrl = ctrl - fct.git = git - } - - return ctrl -} diff --git a/app/services/keywordsearch/handler_branch.go b/app/services/keywordsearch/handler_branch.go index d018e82d0..e43da42a5 100644 --- a/app/services/keywordsearch/handler_branch.go +++ b/app/services/keywordsearch/handler_branch.go @@ -20,6 +20,7 @@ import ( "strings" gitevents "github.com/harness/gitness/app/events/git" + repoevents "github.com/harness/gitness/app/events/repo" "github.com/harness/gitness/events" ) @@ -33,6 +34,21 @@ func (s *Service) handleEventBranchUpdated(ctx context.Context, return s.indexRepo(ctx, event.Payload.RepoID, event.Payload.Ref) } +func (s *Service) handleUpdateDefaultBranch(ctx context.Context, + event *events.Event[*repoevents.DefaultBranchUpdatedPayload]) error { + repo, err := s.repoStore.Find(ctx, event.Payload.RepoID) + if err != nil { + return fmt.Errorf("failed to find repository in db: %w", err) + } + + err = s.indexer.Index(ctx, repo) + if err != nil { + return fmt.Errorf("index update failed for repo %d: %w", repo.ID, err) + } + + return nil +} + func (s *Service) indexRepo( ctx context.Context, repoID int64, @@ -64,6 +80,7 @@ func (s *Service) indexRepo( func getBranchFromRef(ref string) (string, error) { const refPrefix = "refs/heads/" + if !strings.HasPrefix(ref, refPrefix) { return "", fmt.Errorf("failed to get branch name from branch ref %s", ref) } diff --git a/app/services/keywordsearch/service.go b/app/services/keywordsearch/service.go index fcab3d40b..f806c07d5 100644 --- a/app/services/keywordsearch/service.go +++ b/app/services/keywordsearch/service.go @@ -21,14 +21,13 @@ import ( "time" gitevents "github.com/harness/gitness/app/events/git" + repoevents "github.com/harness/gitness/app/events/repo" "github.com/harness/gitness/app/store" "github.com/harness/gitness/events" "github.com/harness/gitness/stream" ) -const ( - eventsReaderGroupName = "gitness:keywordsearch" -) +const groupGitEvents = "gitness:keywordsearch" type Config struct { EventReaderName string @@ -63,6 +62,7 @@ func NewService( ctx context.Context, config Config, gitReaderFactory *events.ReaderFactory[*gitevents.Reader], + repoReaderFactory *events.ReaderFactory[*repoevents.Reader], repoStore store.RepoStore, indexer Indexer, ) (*Service, error) { @@ -75,7 +75,7 @@ func NewService( indexer: indexer, } - _, err := gitReaderFactory.Launch(ctx, eventsReaderGroupName, config.EventReaderName, + _, err := gitReaderFactory.Launch(ctx, groupGitEvents, config.EventReaderName, func(r *gitevents.Reader) error { const idleTimeout = 1 * time.Minute r.Configure( @@ -95,5 +95,22 @@ func NewService( return nil, fmt.Errorf("failed to launch git event reader for webhooks: %w", err) } + _, err = repoReaderFactory.Launch(ctx, groupGitEvents, config.EventReaderName, + func(r *repoevents.Reader) error { + const idleTimeout = 1 * time.Minute + r.Configure( + stream.WithConcurrency(config.Concurrency), + stream.WithHandlerOptions( + stream.WithIdleTimeout(idleTimeout), + stream.WithMaxRetries(config.MaxRetries), + )) + + _ = r.RegisterDefaultBranchUpdated((service.handleUpdateDefaultBranch)) + return nil + }) + if err != nil { + return nil, fmt.Errorf("failed to launch reader factory for repo git group: %w", err) + } + return service, nil } diff --git a/app/services/keywordsearch/wire.go b/app/services/keywordsearch/wire.go index 9b0f0a7d9..67ccf1165 100644 --- a/app/services/keywordsearch/wire.go +++ b/app/services/keywordsearch/wire.go @@ -18,6 +18,7 @@ import ( "context" gitevents "github.com/harness/gitness/app/events/git" + repoevents "github.com/harness/gitness/app/events/repo" "github.com/harness/gitness/app/store" "github.com/harness/gitness/events" @@ -35,12 +36,14 @@ var WireSet = wire.NewSet( func ProvideService(ctx context.Context, config Config, gitReaderFactory *events.ReaderFactory[*gitevents.Reader], + repoReaderFactory *events.ReaderFactory[*repoevents.Reader], repoStore store.RepoStore, indexer Indexer, ) (*Service, error) { return NewService(ctx, config, gitReaderFactory, + repoReaderFactory, repoStore, indexer) } diff --git a/app/api/controller/repo/locks.go b/app/services/locker/locker.go similarity index 65% rename from app/api/controller/repo/locks.go rename to app/services/locker/locker.go index 983103b84..1bc5a3fe2 100644 --- a/app/api/controller/repo/locks.go +++ b/app/services/locker/locker.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package repo +package locker import ( "context" @@ -27,38 +27,50 @@ import ( "github.com/rs/zerolog/log" ) -func (c *Controller) lockDefaultBranch( +const namespaceRepo = "repo" + +type Locker struct { + mtxManager lock.MutexManager +} + +func NewLocker(mtxManager lock.MutexManager) *Locker { + return &Locker{ + mtxManager: mtxManager, + } +} + +func (l Locker) lock( ctx context.Context, - repoUID string, - branchName string, + namespace string, + key string, expiry time.Duration, ) (func(), error) { - key := repoUID + "/defaultBranch" - // annotate logs for easier debugging of lock related issues - // TODO: refactor once common logging annotations are added ctx = logging.NewContext(ctx, func(zc zerolog.Context) zerolog.Context { return zc. - Str("default_branch_lock", key). - Str("repo_uid", repoUID) + Str("key", key). + Str("namespace", namespaceRepo). + Str("expiry", expiry.String()) }) - mutext, err := c.mtxManager.NewMutex( + mutext, err := l.mtxManager.NewMutex( key, - lock.WithNamespace("repo"), + lock.WithNamespace(namespace), lock.WithExpiry(expiry), lock.WithTimeoutFactor(4/expiry.Seconds()), // 4s ) if err != nil { - return nil, fmt.Errorf("failed to create new mutex for repo %q with default branch %s: %w", repoUID, branchName, err) + return nil, fmt.Errorf("failed to create new mutex: %w", err) } + log.Ctx(ctx).Debug().Msg("attempting to acquire lock") + err = mutext.Lock(ctx) if err != nil { - return nil, fmt.Errorf("failed to lock the mutex for repo %q with default branch %s: %w", repoUID, branchName, err) + return nil, fmt.Errorf("failed to lock the mutex: %w", err) } - log.Ctx(ctx).Info().Msgf("successfully locked the repo default branch (expiry: %s)", expiry) + log.Ctx(ctx).Debug().Msgf("successfully locked (expiry: %s)", expiry) unlockFn := func() { // always unlock independent of whether source context got canceled or not @@ -70,10 +82,11 @@ func (c *Controller) lockDefaultBranch( err := mutext.Unlock(ctx) if err != nil { - log.Ctx(ctx).Warn().Err(err).Msg("failed to unlock repo default branch") + log.Ctx(ctx).Warn().Err(err).Msg("failed to unlock") } else { - log.Ctx(ctx).Info().Msg("successfully unlocked repo default branch") + log.Ctx(ctx).Debug().Msg("successfully unlocked") } } + return unlockFn, nil } diff --git a/app/services/locker/pullreq.go b/app/services/locker/pullreq.go new file mode 100644 index 000000000..8a8ed9167 --- /dev/null +++ b/app/services/locker/pullreq.go @@ -0,0 +1,41 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package locker + +import ( + "context" + "fmt" + "strconv" + "time" +) + +func (l Locker) LockPR( + ctx context.Context, + repoID int64, + prNum int64, + expiry time.Duration, +) (func(), error) { + key := fmt.Sprintf("%d/pulls", repoID) + if prNum != 0 { + key += "/" + strconv.FormatInt(prNum, 10) + } + + unlockFn, err := l.lock(ctx, namespaceRepo, key, expiry) + if err != nil { + return nil, fmt.Errorf("failed to lock mutex for pr %d in repo %d: %w", prNum, repoID, err) + } + + return unlockFn, nil +} diff --git a/app/services/locker/repo.go b/app/services/locker/repo.go new file mode 100644 index 000000000..67e8907ba --- /dev/null +++ b/app/services/locker/repo.go @@ -0,0 +1,42 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package locker + +import ( + "context" + "fmt" + "strconv" + "time" + + "github.com/rs/zerolog/log" +) + +func (l Locker) LockDefaultBranch( + ctx context.Context, + repoID int64, + branchName string, + expiry time.Duration, +) (func(), error) { + key := strconv.FormatInt(repoID, 10) + "/defaultBranch" + + log.Ctx(ctx).Info().Msg("attempting to lock to update the repo default branch") + + unlockFn, err := l.lock(ctx, namespaceRepo, key, expiry) + if err != nil { + return nil, fmt.Errorf("failed to lock repo to update default branch to %s: %w", branchName, err) + } + + return unlockFn, nil +} diff --git a/app/services/locker/wire.go b/app/services/locker/wire.go new file mode 100644 index 000000000..8c1fd1a29 --- /dev/null +++ b/app/services/locker/wire.go @@ -0,0 +1,29 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package locker + +import ( + "github.com/harness/gitness/lock" + + "github.com/google/wire" +) + +var WireSet = wire.NewSet( + ProvideLocker, +) + +func ProvideLocker(mtxManager lock.MutexManager) *Locker { + return NewLocker(mtxManager) +} diff --git a/app/services/repo/handlers_default_branch.go b/app/services/repo/handlers_default_branch.go new file mode 100644 index 000000000..0de633f28 --- /dev/null +++ b/app/services/repo/handlers_default_branch.go @@ -0,0 +1,93 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package repo + +import ( + "context" + "fmt" + "time" + + "github.com/harness/gitness/app/bootstrap" + repoevents "github.com/harness/gitness/app/events/repo" + "github.com/harness/gitness/app/githook" + "github.com/harness/gitness/events" + "github.com/harness/gitness/git" + + "github.com/rs/zerolog/log" +) + +// handleUpdateDefaultBranch handles git update default branch using branch name from db (not event payload). +func (s *Service) handleUpdateDefaultBranch( + ctx context.Context, + event *events.Event[*repoevents.DefaultBranchUpdatedPayload], +) error { + // the max time we give an update default branch to succeed + const timeout = 2 * time.Minute + unlock, err := s.locker.LockDefaultBranch( + ctx, + event.Payload.RepoID, + event.Payload.NewName, // only used for logging + timeout+30*time.Second, // add 30s to the lock to give enough time for updating default branch + ) + if err != nil { + return fmt.Errorf("failed to lock repo for updating default branch to %s", event.Payload.NewName) + } + defer unlock() + + repo, err := s.repoStore.Find(ctx, event.Payload.RepoID) + if err != nil { + return fmt.Errorf("update default branch handler failed to find the repo: %w", err) + } + + // create new, time-restricted context to guarantee update completion, even if request is canceled. + // TODO: a proper error handling solution required. + ctx, cancel := context.WithTimeout( + ctx, + timeout, + ) + defer cancel() + + systemPrincipal := bootstrap.NewSystemServiceSession().Principal + envVars, err := githook.GenerateEnvironmentVariables( + ctx, + s.urlProvider.GetInternalAPIURL(), + repo.ID, + systemPrincipal.ID, + true, + true, + ) + if err != nil { + return fmt.Errorf("failed to generate git hook env variables: %w", err) + } + + err = s.git.UpdateDefaultBranch(ctx, &git.UpdateDefaultBranchParams{ + WriteParams: git.WriteParams{ + Actor: git.Identity{ + Name: systemPrincipal.DisplayName, + Email: systemPrincipal.Email, + }, + RepoUID: repo.GitUID, + EnvVars: envVars, + }, + BranchName: repo.DefaultBranch, + }) + if err != nil { + return fmt.Errorf("failed to update the repo default branch to %s", repo.DefaultBranch) + } + + log.Ctx(ctx).Info().Msgf("git repo default branch updated to %s by default branch event handler", repo.DefaultBranch) + + return nil +} diff --git a/app/services/reposize/reposize.go b/app/services/repo/reposize.go similarity index 76% rename from app/services/reposize/reposize.go rename to app/services/repo/reposize.go index 6db85227e..6507e05de 100644 --- a/app/services/reposize/reposize.go +++ b/app/services/repo/reposize.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package reposize +package repo import ( "context" @@ -30,7 +30,7 @@ import ( const jobType = "repo-size-calculator" -type Calculator struct { +type SizeCalculator struct { enabled bool cron string maxDur time.Duration @@ -40,12 +40,12 @@ type Calculator struct { scheduler *job.Scheduler } -func (c *Calculator) Register(ctx context.Context) error { - if !c.enabled { +func (s *SizeCalculator) Register(ctx context.Context) error { + if !s.enabled { return nil } - err := c.scheduler.AddRecurring(ctx, jobType, jobType, c.cron, c.maxDur) + err := s.scheduler.AddRecurring(ctx, jobType, jobType, s.cron, s.maxDur) if err != nil { return fmt.Errorf("failed to register recurring job for calculator: %w", err) } @@ -53,17 +53,17 @@ func (c *Calculator) Register(ctx context.Context) error { return nil } -func (c *Calculator) Handle(ctx context.Context, _ string, _ job.ProgressReporter) (string, error) { - if !c.enabled { +func (s *SizeCalculator) Handle(ctx context.Context, _ string, _ job.ProgressReporter) (string, error) { + if !s.enabled { return "", nil } - sizeInfos, err := c.repoStore.ListSizeInfos(ctx) + sizeInfos, err := s.repoStore.ListSizeInfos(ctx) if err != nil { return "", fmt.Errorf("failed to get repository sizes: %w", err) } - expiredBefore := time.Now().Add(c.maxDur) + expiredBefore := time.Now().Add(s.maxDur) log.Ctx(ctx).Info().Msgf( "start repo size calculation (operation timeout: %s)", expiredBefore.Format(time.RFC3339Nano), @@ -71,9 +71,9 @@ func (c *Calculator) Handle(ctx context.Context, _ string, _ job.ProgressReporte var wg sync.WaitGroup taskCh := make(chan *types.RepositorySizeInfo) - for i := 0; i < c.numWorkers; i++ { + for i := 0; i < s.numWorkers; i++ { wg.Add(1) - go worker(ctx, c, &wg, taskCh) + go worker(ctx, s, &wg, taskCh) } for _, sizeInfo := range sizeInfos { select { @@ -88,7 +88,7 @@ func (c *Calculator) Handle(ctx context.Context, _ string, _ job.ProgressReporte return "", nil } -func worker(ctx context.Context, c *Calculator, wg *sync.WaitGroup, taskCh <-chan *types.RepositorySizeInfo) { +func worker(ctx context.Context, s *SizeCalculator, wg *sync.WaitGroup, taskCh <-chan *types.RepositorySizeInfo) { defer wg.Done() for sizeInfo := range taskCh { @@ -96,7 +96,7 @@ func worker(ctx context.Context, c *Calculator, wg *sync.WaitGroup, taskCh <-cha log.Debug().Msgf("previous repo size: %d", sizeInfo.Size) - sizeOut, err := c.git.GetRepositorySize( + sizeOut, err := s.git.GetRepositorySize( ctx, &git.GetRepositorySizeParams{ReadParams: git.ReadParams{RepoUID: sizeInfo.GitUID}}) if err != nil { @@ -108,7 +108,7 @@ func worker(ctx context.Context, c *Calculator, wg *sync.WaitGroup, taskCh <-cha continue } - if err := c.repoStore.UpdateSize(ctx, sizeInfo.ID, sizeOut.Size); err != nil { + if err := s.repoStore.UpdateSize(ctx, sizeInfo.ID, sizeOut.Size); err != nil { log.Error().Msgf("failed to update repo size: %s", err.Error()) continue } diff --git a/app/services/repo/service.go b/app/services/repo/service.go new file mode 100644 index 000000000..33bb28de5 --- /dev/null +++ b/app/services/repo/service.go @@ -0,0 +1,79 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package repo + +import ( + "context" + "fmt" + "time" + + repoevents "github.com/harness/gitness/app/events/repo" + "github.com/harness/gitness/app/services/locker" + "github.com/harness/gitness/app/store" + "github.com/harness/gitness/app/url" + "github.com/harness/gitness/events" + "github.com/harness/gitness/git" + "github.com/harness/gitness/stream" + "github.com/harness/gitness/types" +) + +const groupRepo = "gitness:repo" + +type Service struct { + repoEvReporter *repoevents.Reporter + repoStore store.RepoStore + urlProvider url.Provider + git git.Interface + locker *locker.Locker +} + +func NewService( + ctx context.Context, + config *types.Config, + repoEvReporter *repoevents.Reporter, + repoReaderFactory *events.ReaderFactory[*repoevents.Reader], + repoStore store.RepoStore, + urlProvider url.Provider, + git git.Interface, + locker *locker.Locker, +) (*Service, error) { + service := &Service{ + repoEvReporter: repoEvReporter, + repoStore: repoStore, + urlProvider: urlProvider, + git: git, + locker: locker, + } + + _, err := repoReaderFactory.Launch(ctx, groupRepo, config.InstanceID, + func(r *repoevents.Reader) error { + const idleTimeout = 15 * time.Second + r.Configure( + stream.WithConcurrency(1), + stream.WithHandlerOptions( + stream.WithIdleTimeout(idleTimeout), + stream.WithMaxRetries(3), + )) + + _ = r.RegisterDefaultBranchUpdated(service.handleUpdateDefaultBranch) + + return nil + }) + if err != nil { + return nil, fmt.Errorf("failed to launch reader factory for repo git group: %w", err) + } + + return service, nil +} diff --git a/app/services/reposize/wire.go b/app/services/repo/wire.go similarity index 65% rename from app/services/reposize/wire.go rename to app/services/repo/wire.go index 447073148..f993033d5 100644 --- a/app/services/reposize/wire.go +++ b/app/services/repo/wire.go @@ -12,10 +12,16 @@ // See the License for the specific language governing permissions and // limitations under the License. -package reposize +package repo import ( + "context" + + repoevents "github.com/harness/gitness/app/events/repo" + "github.com/harness/gitness/app/services/locker" "github.com/harness/gitness/app/store" + "github.com/harness/gitness/app/url" + "github.com/harness/gitness/events" "github.com/harness/gitness/git" "github.com/harness/gitness/job" "github.com/harness/gitness/types" @@ -25,6 +31,7 @@ import ( var WireSet = wire.NewSet( ProvideCalculator, + ProvideService, ) func ProvideCalculator( @@ -33,8 +40,8 @@ func ProvideCalculator( repoStore store.RepoStore, scheduler *job.Scheduler, executor *job.Executor, -) (*Calculator, error) { - job := &Calculator{ +) (*SizeCalculator, error) { + job := &SizeCalculator{ enabled: config.RepoSize.Enabled, cron: config.RepoSize.CRON, maxDur: config.RepoSize.MaxDuration, @@ -51,3 +58,16 @@ func ProvideCalculator( return job, nil } + +func ProvideService(ctx context.Context, + config *types.Config, + repoEvReporter *repoevents.Reporter, + repoReaderFactory *events.ReaderFactory[*repoevents.Reader], + repoStore store.RepoStore, + urlProvider url.Provider, + git git.Interface, + locker *locker.Locker, +) (*Service, error) { + return NewService(ctx, config, repoEvReporter, repoReaderFactory, + repoStore, urlProvider, git, locker) +} diff --git a/app/services/wire.go b/app/services/wire.go index 60720a950..f6745878b 100644 --- a/app/services/wire.go +++ b/app/services/wire.go @@ -20,7 +20,7 @@ import ( "github.com/harness/gitness/app/services/metric" "github.com/harness/gitness/app/services/notification" "github.com/harness/gitness/app/services/pullreq" - "github.com/harness/gitness/app/services/reposize" + "github.com/harness/gitness/app/services/repo" "github.com/harness/gitness/app/services/trigger" "github.com/harness/gitness/app/services/webhook" "github.com/harness/gitness/job" @@ -38,7 +38,8 @@ type Services struct { Trigger *trigger.Service JobScheduler *job.Scheduler MetricCollector *metric.Collector - RepoSizeCalculator *reposize.Calculator + RepoSizeCalculator *repo.SizeCalculator + Repo *repo.Service Cleanup *cleanup.Service Notification *notification.Service Keywordsearch *keywordsearch.Service @@ -50,7 +51,8 @@ func ProvideServices( triggerSvc *trigger.Service, jobScheduler *job.Scheduler, metricCollector *metric.Collector, - repoSizeCalculator *reposize.Calculator, + repoSizeCalculator *repo.SizeCalculator, + repo *repo.Service, cleanupSvc *cleanup.Service, notificationSvc *notification.Service, keywordsearchSvc *keywordsearch.Service, @@ -62,6 +64,7 @@ func ProvideServices( JobScheduler: jobScheduler, MetricCollector: metricCollector, RepoSizeCalculator: repoSizeCalculator, + Repo: repo, Cleanup: cleanupSvc, Notification: notificationSvc, Keywordsearch: keywordsearchSvc, diff --git a/app/store/database/migrate/postgres/0048_alter_table_repo_add_isempty.down.sql b/app/store/database/migrate/postgres/0048_alter_table_repo_add_isempty.down.sql new file mode 100644 index 000000000..71caf2ca9 --- /dev/null +++ b/app/store/database/migrate/postgres/0048_alter_table_repo_add_isempty.down.sql @@ -0,0 +1 @@ +ALTER TABLE repositories DROP COLUMN repo_is_empty; \ No newline at end of file diff --git a/app/store/database/migrate/postgres/0048_alter_table_repo_add_isempty.up.sql b/app/store/database/migrate/postgres/0048_alter_table_repo_add_isempty.up.sql new file mode 100644 index 000000000..68513d8e2 --- /dev/null +++ b/app/store/database/migrate/postgres/0048_alter_table_repo_add_isempty.up.sql @@ -0,0 +1 @@ +ALTER TABLE repositories ADD COLUMN repo_is_empty BOOLEAN NOT NULL DEFAULT false; \ No newline at end of file diff --git a/app/store/database/migrate/sqlite/0048_alter_table_repo_add_isempty.down.sql b/app/store/database/migrate/sqlite/0048_alter_table_repo_add_isempty.down.sql new file mode 100644 index 000000000..71caf2ca9 --- /dev/null +++ b/app/store/database/migrate/sqlite/0048_alter_table_repo_add_isempty.down.sql @@ -0,0 +1 @@ +ALTER TABLE repositories DROP COLUMN repo_is_empty; \ No newline at end of file diff --git a/app/store/database/migrate/sqlite/0048_alter_table_repo_add_isempty.up.sql b/app/store/database/migrate/sqlite/0048_alter_table_repo_add_isempty.up.sql new file mode 100644 index 000000000..68513d8e2 --- /dev/null +++ b/app/store/database/migrate/sqlite/0048_alter_table_repo_add_isempty.up.sql @@ -0,0 +1 @@ +ALTER TABLE repositories ADD COLUMN repo_is_empty BOOLEAN NOT NULL DEFAULT false; \ No newline at end of file diff --git a/app/store/database/repo.go b/app/store/database/repo.go index 3d10c0d3f..8fb599b1e 100644 --- a/app/store/database/repo.go +++ b/app/store/database/repo.go @@ -88,6 +88,7 @@ type repository struct { NumMergedPulls int `db:"repo_num_merged_pulls"` Importing bool `db:"repo_importing"` + IsEmpty bool `db:"repo_is_empty"` } const ( @@ -113,7 +114,8 @@ const ( ,repo_num_closed_pulls ,repo_num_open_pulls ,repo_num_merged_pulls - ,repo_importing` + ,repo_importing + ,repo_is_empty` ) // Find finds the repo by id. @@ -238,6 +240,7 @@ func (s *RepoStore) Create(ctx context.Context, repo *types.Repository) error { ,repo_num_open_pulls ,repo_num_merged_pulls ,repo_importing + ,repo_is_empty ) values ( :repo_version ,:repo_parent_id @@ -260,6 +263,7 @@ func (s *RepoStore) Create(ctx context.Context, repo *types.Repository) error { ,:repo_num_open_pulls ,:repo_num_merged_pulls ,:repo_importing + ,:repo_is_empty ) RETURNING repo_id` db := dbtx.GetAccessor(ctx, s.db) @@ -303,6 +307,7 @@ func (s *RepoStore) Update(ctx context.Context, repo *types.Repository) error { ,repo_num_open_pulls = :repo_num_open_pulls ,repo_num_merged_pulls = :repo_num_merged_pulls ,repo_importing = :repo_importing + ,repo_is_empty = :repo_is_empty WHERE repo_id = :repo_id AND repo_version = :repo_version - 1` dbRepo := mapToInternalRepo(repo) @@ -746,6 +751,7 @@ func (s *RepoStore) mapToRepo( NumOpenPulls: in.NumOpenPulls, NumMergedPulls: in.NumMergedPulls, Importing: in.Importing, + IsEmpty: in.IsEmpty, // Path: is set below } @@ -829,6 +835,7 @@ func mapToInternalRepo(in *types.Repository) *repository { NumOpenPulls: in.NumOpenPulls, NumMergedPulls: in.NumMergedPulls, Importing: in.Importing, + IsEmpty: in.IsEmpty, } } diff --git a/cmd/gitness/wire.go b/cmd/gitness/wire.go index 4f5d94964..53880c016 100644 --- a/cmd/gitness/wire.go +++ b/cmd/gitness/wire.go @@ -13,7 +13,7 @@ import ( checkcontroller "github.com/harness/gitness/app/api/controller/check" "github.com/harness/gitness/app/api/controller/connector" "github.com/harness/gitness/app/api/controller/execution" - ctrlgithook "github.com/harness/gitness/app/api/controller/githook" + githookCtrl "github.com/harness/gitness/app/api/controller/githook" controllerkeywordsearch "github.com/harness/gitness/app/api/controller/keywordsearch" "github.com/harness/gitness/app/api/controller/limiter" controllerlogs "github.com/harness/gitness/app/api/controller/logs" @@ -40,7 +40,6 @@ import ( gitevents "github.com/harness/gitness/app/events/git" pullreqevents "github.com/harness/gitness/app/events/pullreq" repoevents "github.com/harness/gitness/app/events/repo" - "github.com/harness/gitness/app/githook" "github.com/harness/gitness/app/pipeline/canceler" "github.com/harness/gitness/app/pipeline/commit" "github.com/harness/gitness/app/pipeline/converter" @@ -59,12 +58,13 @@ import ( "github.com/harness/gitness/app/services/exporter" "github.com/harness/gitness/app/services/importer" "github.com/harness/gitness/app/services/keywordsearch" + locker "github.com/harness/gitness/app/services/locker" "github.com/harness/gitness/app/services/metric" "github.com/harness/gitness/app/services/notification" "github.com/harness/gitness/app/services/notification/mailer" "github.com/harness/gitness/app/services/protection" pullreqservice "github.com/harness/gitness/app/services/pullreq" - "github.com/harness/gitness/app/services/reposize" + reposervice "github.com/harness/gitness/app/services/repo" "github.com/harness/gitness/app/services/settings" "github.com/harness/gitness/app/services/trigger" "github.com/harness/gitness/app/services/usergroup" @@ -142,10 +142,11 @@ func initSystem(ctx context.Context, config *types.Config) (*cliserver.System, e webhook.WireSet, cliserver.ProvideTriggerConfig, trigger.WireSet, - githook.WireSet, - ctrlgithook.WireSet, + githookCtrl.ExtenderWireSet, + githookCtrl.WireSet, cliserver.ProvideLockConfig, lock.WireSet, + locker.WireSet, cliserver.ProvidePubsubConfig, pubsub.WireSet, cliserver.ProvideJobsConfig, @@ -178,7 +179,7 @@ func initSystem(ctx context.Context, config *types.Config) (*cliserver.System, e canceler.WireSet, exporter.WireSet, metric.WireSet, - reposize.WireSet, + reposervice.WireSet, cliserver.ProvideCodeOwnerConfig, codeowners.WireSet, cliserver.ProvideKeywordSearchConfig, diff --git a/cmd/gitness/wire_gen.go b/cmd/gitness/wire_gen.go index 47f14ce0e..400e7b099 100644 --- a/cmd/gitness/wire_gen.go +++ b/cmd/gitness/wire_gen.go @@ -12,7 +12,7 @@ import ( check2 "github.com/harness/gitness/app/api/controller/check" "github.com/harness/gitness/app/api/controller/connector" "github.com/harness/gitness/app/api/controller/execution" - githook2 "github.com/harness/gitness/app/api/controller/githook" + "github.com/harness/gitness/app/api/controller/githook" keywordsearch2 "github.com/harness/gitness/app/api/controller/keywordsearch" "github.com/harness/gitness/app/api/controller/limiter" logs2 "github.com/harness/gitness/app/api/controller/logs" @@ -39,7 +39,6 @@ import ( events4 "github.com/harness/gitness/app/events/git" events3 "github.com/harness/gitness/app/events/pullreq" events2 "github.com/harness/gitness/app/events/repo" - "github.com/harness/gitness/app/githook" "github.com/harness/gitness/app/pipeline/canceler" "github.com/harness/gitness/app/pipeline/commit" "github.com/harness/gitness/app/pipeline/converter" @@ -58,12 +57,13 @@ import ( "github.com/harness/gitness/app/services/exporter" "github.com/harness/gitness/app/services/importer" "github.com/harness/gitness/app/services/keywordsearch" + "github.com/harness/gitness/app/services/locker" "github.com/harness/gitness/app/services/metric" "github.com/harness/gitness/app/services/notification" "github.com/harness/gitness/app/services/notification/mailer" "github.com/harness/gitness/app/services/protection" "github.com/harness/gitness/app/services/pullreq" - "github.com/harness/gitness/app/services/reposize" + repo2 "github.com/harness/gitness/app/services/repo" "github.com/harness/gitness/app/services/settings" trigger2 "github.com/harness/gitness/app/services/trigger" "github.com/harness/gitness/app/services/usergroup" @@ -192,9 +192,10 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro if err != nil { return nil, err } + lockerLocker := locker.ProvideLocker(mutexManager) repoIdentifier := check.ProvideRepoIdentifierCheck() repoCheck := repo.ProvideRepoCheck() - repoController := repo.ProvideController(config, transactor, provider, authorizer, repoStore, spaceStore, pipelineStore, principalStore, ruleStore, settingsService, principalInfoCache, protectionManager, gitInterface, repository, codeownersService, reporter, indexer, resourceLimiter, mutexManager, repoIdentifier, repoCheck) + repoController := repo.ProvideController(config, transactor, provider, authorizer, repoStore, spaceStore, pipelineStore, principalStore, ruleStore, settingsService, principalInfoCache, protectionManager, gitInterface, repository, codeownersService, reporter, indexer, resourceLimiter, lockerLocker, mutexManager, repoIdentifier, repoCheck) reposettingsController := reposettings.ProvideController(authorizer, repoStore, settingsService) executionStore := database.ProvideExecutionStore(db) checkStore := database.ProvideCheckStore(db, principalInfoCache) @@ -254,7 +255,7 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro if err != nil { return nil, err } - pullreqController := pullreq2.ProvideController(transactor, provider, authorizer, pullReqStore, pullReqActivityStore, codeCommentView, pullReqReviewStore, pullReqReviewerStore, repoStore, principalStore, pullReqFileViewStore, membershipStore, checkStore, gitInterface, eventsReporter, mutexManager, migrator, pullreqService, protectionManager, streamer, codeownersService) + pullreqController := pullreq2.ProvideController(transactor, provider, authorizer, pullReqStore, pullReqActivityStore, codeCommentView, pullReqReviewStore, pullReqReviewerStore, repoStore, principalStore, pullReqFileViewStore, membershipStore, checkStore, gitInterface, eventsReporter, migrator, pullreqService, protectionManager, streamer, codeownersService, lockerLocker) webhookConfig := server.ProvideWebhookConfig(config) webhookStore := database.ProvideWebhookStore(db) webhookExecutionStore := database.ProvideWebhookExecutionStore(db) @@ -267,19 +268,19 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro if err != nil { return nil, err } - preReceiveExtender, err := githook2.ProvidePreReceiveExtender() + preReceiveExtender, err := githook.ProvidePreReceiveExtender() if err != nil { return nil, err } - updateExtender, err := githook2.ProvideUpdateExtender() + updateExtender, err := githook.ProvideUpdateExtender() if err != nil { return nil, err } - postReceiveExtender, err := githook2.ProvidePostReceiveExtender() + postReceiveExtender, err := githook.ProvidePostReceiveExtender() if err != nil { return nil, err } - githookController := githook.ProvideController(authorizer, principalStore, repoStore, reporter2, gitInterface, pullReqStore, provider, protectionManager, clientFactory, resourceLimiter, settingsService, preReceiveExtender, updateExtender, postReceiveExtender) + githookController := githook.ProvideController(authorizer, principalStore, repoStore, reporter2, reporter, gitInterface, pullReqStore, provider, protectionManager, clientFactory, resourceLimiter, settingsService, preReceiveExtender, updateExtender, postReceiveExtender) serviceaccountController := serviceaccount.NewController(principalUID, authorizer, principalStore, spaceStore, repoStore, tokenStore) principalController := principal.ProvideController(principalStore) v := check2.ProvideCheckSanitizers() @@ -319,7 +320,15 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro if err != nil { return nil, err } - calculator, err := reposize.ProvideCalculator(config, gitInterface, repoStore, jobScheduler, executor) + sizeCalculator, err := repo2.ProvideCalculator(config, gitInterface, repoStore, jobScheduler, executor) + if err != nil { + return nil, err + } + readerFactory2, err := events2.ProvideReaderFactory(eventsSystem) + if err != nil { + return nil, err + } + repoService, err := repo2.ProvideService(ctx, config, reporter, readerFactory2, repoStore, provider, gitInterface, lockerLocker) if err != nil { return nil, err } @@ -336,11 +345,11 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro return nil, err } keywordsearchConfig := server.ProvideKeywordSearchConfig(config) - keywordsearchService, err := keywordsearch.ProvideService(ctx, keywordsearchConfig, readerFactory, repoStore, indexer) + keywordsearchService, err := keywordsearch.ProvideService(ctx, keywordsearchConfig, readerFactory, readerFactory2, repoStore, indexer) if err != nil { return nil, err } - servicesServices := services.ProvideServices(webhookService, pullreqService, triggerService, jobScheduler, collector, calculator, cleanupService, notificationService, keywordsearchService) + servicesServices := services.ProvideServices(webhookService, pullreqService, triggerService, jobScheduler, collector, sizeCalculator, repoService, cleanupService, notificationService, keywordsearchService) serverSystem := server.NewSystem(bootstrapBootstrap, serverServer, poller, resolverManager, servicesServices) return serverSystem, nil } diff --git a/types/repo.go b/types/repo.go index f021c8a51..23f0fd1cf 100644 --- a/types/repo.go +++ b/types/repo.go @@ -50,6 +50,7 @@ type Repository struct { NumMergedPulls int `json:"num_merged_pulls"` Importing bool `json:"importing"` + IsEmpty bool `json:"is_empty,omitempty"` // git urls GitURL string `json:"git_url"`