diff --git a/app/api/controller/repo/delete.go b/app/api/controller/repo/purge.go similarity index 72% rename from app/api/controller/repo/delete.go rename to app/api/controller/repo/purge.go index acf02a15b..0fc5f4666 100644 --- a/app/api/controller/repo/delete.go +++ b/app/api/controller/repo/purge.go @@ -20,6 +20,7 @@ import ( apiauth "github.com/harness/gitness/app/api/auth" "github.com/harness/gitness/app/api/controller" + "github.com/harness/gitness/app/api/usererror" "github.com/harness/gitness/app/auth" repoevents "github.com/harness/gitness/app/events/repo" "github.com/harness/gitness/errors" @@ -30,12 +31,11 @@ import ( "github.com/rs/zerolog/log" ) -// Delete deletes a repo. -func (c *Controller) Delete(ctx context.Context, session *auth.Session, repoRef string) error { - // note: can't use c.getRepoCheckAccess because import job for repositories being imported must be cancelled. - repo, err := c.repoStore.FindByRef(ctx, repoRef) +// Purge removes a repo permanently. +func (c *Controller) Purge(ctx context.Context, session *auth.Session, repoRef string, deletedAt int64) error { + repo, err := c.repoStore.FindByRefAndDeletedAt(ctx, repoRef, deletedAt) if err != nil { - return err + return fmt.Errorf("failed to find the repo (deleted at %d): %w", deletedAt, err) } if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoDelete, false); err != nil { @@ -45,25 +45,26 @@ func (c *Controller) Delete(ctx context.Context, session *auth.Session, repoRef log.Ctx(ctx).Info(). Int64("repo.id", repo.ID). Str("repo.path", repo.Path). - Msgf("deleting repository") + Msg("purging repository") - if repo.Importing { - err = c.importer.Cancel(ctx, repo) - if err != nil { - return fmt.Errorf("failed to cancel repository import") - } + if repo.Deleted == nil { + return usererror.BadRequest("Repository has to be deleted before it can be purged.") } - return c.DeleteNoAuth(ctx, session, repo) + return c.PurgeNoAuth(ctx, session, repo) } -func (c *Controller) DeleteNoAuth(ctx context.Context, session *auth.Session, repo *types.Repository) error { - if err := c.deleteGitRepository(ctx, session, repo); err != nil { - return fmt.Errorf("failed to delete git repository: %w", err) +func (c *Controller) PurgeNoAuth( + ctx context.Context, + session *auth.Session, + repo *types.Repository, +) error { + if err := c.repoStore.Purge(ctx, repo.ID, repo.Deleted); err != nil { + return fmt.Errorf("failed to delete repo from db: %w", err) } - if err := c.repoStore.Delete(ctx, repo.ID); err != nil { - return fmt.Errorf("failed to delete repo from db: %w", err) + if err := c.deleteGitRepository(ctx, session, repo); err != nil { + return fmt.Errorf("failed to delete git repository: %w", err) } c.eventReporter.Deleted( @@ -95,13 +96,12 @@ func (c *Controller) deleteGitRepository( WriteParams: writeParams, }) - // deletion should not fail if dir does not exist in repos dir + // deletion should not fail if repo dir does not exist. if errors.IsNotFound(err) { log.Ctx(ctx).Warn().Str("repo.git_uid", repo.GitUID). Msg("git repository directory does not exist") } else if err != nil { - // deletion has failed before removing(rename) the repo dir - return fmt.Errorf("failed to delete git repository directory %s: %w", repo.GitUID, err) + return fmt.Errorf("failed to remove git repository %s: %w", repo.GitUID, err) } return nil } diff --git a/app/api/controller/repo/restore.go b/app/api/controller/repo/restore.go new file mode 100644 index 000000000..aa3ba276e --- /dev/null +++ b/app/api/controller/repo/restore.go @@ -0,0 +1,58 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package repo + +import ( + "context" + "fmt" + + apiauth "github.com/harness/gitness/app/api/auth" + "github.com/harness/gitness/app/api/usererror" + "github.com/harness/gitness/app/auth" + "github.com/harness/gitness/types" + "github.com/harness/gitness/types/enum" +) + +type RestoreInput struct { + NewIdentifier string `json:"new_identifier,omitempty"` + DeletedAt int64 `json:"deleted_at"` +} + +func (c *Controller) Restore( + ctx context.Context, + session *auth.Session, + repoRef string, + in *RestoreInput, +) (*types.Repository, error) { + repo, err := c.repoStore.FindByRefAndDeletedAt(ctx, repoRef, in.DeletedAt) + if err != nil { + return nil, fmt.Errorf("failed to find repository: %w", err) + } + + if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoEdit, false); err != nil { + return nil, fmt.Errorf("access check failed: %w", err) + } + + if repo.Deleted == nil { + return nil, usererror.BadRequest("cannot restore a repo that hasn't been deleted") + } + + repo, err = c.repoStore.Restore(ctx, repo, in.NewIdentifier) + if err != nil { + return nil, fmt.Errorf("failed to restore the repo: %w", err) + } + + return repo, nil +} diff --git a/app/api/controller/repo/soft_delete.go b/app/api/controller/repo/soft_delete.go new file mode 100644 index 000000000..c09d5a969 --- /dev/null +++ b/app/api/controller/repo/soft_delete.go @@ -0,0 +1,75 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package repo + +import ( + "context" + "fmt" + "time" + + apiauth "github.com/harness/gitness/app/api/auth" + "github.com/harness/gitness/app/api/usererror" + "github.com/harness/gitness/app/auth" + "github.com/harness/gitness/types" + "github.com/harness/gitness/types/enum" + + "github.com/rs/zerolog/log" +) + +// SoftDelete soft deletes a repo (aka sets the deleted timestamp while keep the data). +func (c *Controller) SoftDelete(ctx context.Context, session *auth.Session, repoRef string) error { + // note: can't use c.getRepoCheckAccess because import job for repositories being imported must be cancelled. + repo, err := c.repoStore.FindByRef(ctx, repoRef) + if err != nil { + return fmt.Errorf("failed to find the repo for soft delete: %w", err) + } + + if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoDelete, false); err != nil { + return fmt.Errorf("access check failed: %w", err) + } + + log.Ctx(ctx).Info(). + Int64("repo.id", repo.ID). + Str("repo.path", repo.Path). + Msg("soft deleting repository") + + if repo.Deleted != nil { + return usererror.BadRequest("repository has been already deleted") + } + + if repo.Importing { + log.Ctx(ctx).Info().Msg("repository is importing. cancelling the import job and purge the repo.") + err = c.importer.Cancel(ctx, repo) + if err != nil { + return fmt.Errorf("failed to cancel repository import") + } + return c.PurgeNoAuth(ctx, session, repo) + } + + return c.SoftDeleteNoAuth(ctx, repo, time.Now().UnixMilli()) +} + +func (c *Controller) SoftDeleteNoAuth( + ctx context.Context, + repo *types.Repository, + deletedAt int64, +) error { + err := c.repoStore.SoftDelete(ctx, repo, &deletedAt) + if err != nil { + return fmt.Errorf("failed to soft delete repo from db: %w", err) + } + + return nil +} diff --git a/app/api/controller/space/delete.go b/app/api/controller/space/delete.go index ac9e978d0..151c55f77 100644 --- a/app/api/controller/space/delete.go +++ b/app/api/controller/space/delete.go @@ -18,6 +18,7 @@ import ( "context" "fmt" "math" + "time" apiauth "github.com/harness/gitness/app/api/auth" "github.com/harness/gitness/app/auth" @@ -73,18 +74,29 @@ func (c *Controller) DeleteNoAuth(ctx context.Context, session *auth.Session, sp // WARNING this is meant for internal calls only. func (c *Controller) deleteRepositoriesNoAuth(ctx context.Context, session *auth.Session, spaceID int64) error { filter := &types.RepoFilter{ - Page: 1, - Size: int(math.MaxInt), - Query: "", - Order: enum.OrderAsc, - Sort: enum.RepoAttrNone, + Page: 1, + Size: int(math.MaxInt), + Query: "", + Order: enum.OrderAsc, + Sort: enum.RepoAttrNone, + DeletedBefore: nil, } repos, _, err := c.ListRepositoriesNoAuth(ctx, spaceID, filter) if err != nil { return fmt.Errorf("failed to list space repositories: %w", err) } + + // TEMPORARY until we support space delete/restore CODE-1413 + recent := time.Now().Add(+time.Hour * 24).UnixMilli() + filter.DeletedBefore = &recent + alreadyDeletedRepos, _, err := c.ListRepositoriesNoAuth(ctx, spaceID, filter) + if err != nil { + return fmt.Errorf("failed to list delete repositories for space %d: %w", spaceID, err) + } + repos = append(repos, alreadyDeletedRepos...) + for _, repo := range repos { - err = c.repoCtrl.DeleteNoAuth(ctx, session, repo) + err = c.repoCtrl.PurgeNoAuth(ctx, session, repo) if err != nil { return fmt.Errorf("failed to delete repository %d: %w", repo.ID, err) } diff --git a/app/api/handler/repo/purge.go b/app/api/handler/repo/purge.go new file mode 100644 index 000000000..dc129ca3f --- /dev/null +++ b/app/api/handler/repo/purge.go @@ -0,0 +1,51 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package repo + +import ( + "net/http" + + "github.com/harness/gitness/app/api/controller/repo" + "github.com/harness/gitness/app/api/render" + "github.com/harness/gitness/app/api/request" +) + +func HandlePurge(repoCtrl *repo.Controller) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + session, _ := request.AuthSessionFrom(ctx) + + repoRef, err := request.GetRepoRefFromPath(r) + if err != nil { + render.TranslatedUserError(w, err) + return + } + + deletedAt, err := request.GetRepoDeletedAtFromQuery(r) + if err != nil { + render.TranslatedUserError(w, err) + return + } + + err = repoCtrl.Purge(ctx, session, repoRef, deletedAt) + if err != nil { + render.TranslatedUserError(w, err) + return + } + + render.DeleteSuccessful(w) + } +} diff --git a/app/api/handler/repo/restore.go b/app/api/handler/repo/restore.go new file mode 100644 index 000000000..c26b008ea --- /dev/null +++ b/app/api/handler/repo/restore.go @@ -0,0 +1,53 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package repo + +import ( + "encoding/json" + "net/http" + + "github.com/harness/gitness/app/api/controller/repo" + "github.com/harness/gitness/app/api/render" + "github.com/harness/gitness/app/api/request" +) + +func HandleRestore(repoCtrl *repo.Controller) http.HandlerFunc { + return func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + + session, _ := request.AuthSessionFrom(ctx) + + repoRef, err := request.GetRepoRefFromPath(r) + if err != nil { + render.TranslatedUserError(w, err) + return + } + + in := new(repo.RestoreInput) + err = json.NewDecoder(r.Body).Decode(in) + if err != nil { + render.BadRequestf(w, "Invalid request body: %s.", err) + return + } + + repo, err := repoCtrl.Restore(ctx, session, repoRef, in) + if err != nil { + render.TranslatedUserError(w, err) + return + } + + render.JSON(w, http.StatusOK, repo) + } +} diff --git a/app/api/handler/repo/delete.go b/app/api/handler/repo/soft_delete.go similarity index 88% rename from app/api/handler/repo/delete.go rename to app/api/handler/repo/soft_delete.go index e7e432550..b003cd316 100644 --- a/app/api/handler/repo/delete.go +++ b/app/api/handler/repo/soft_delete.go @@ -23,19 +23,21 @@ import ( ) /* - * Deletes a repository. + * Soft Deletes a repository. */ -func HandleDelete(repoCtrl *repo.Controller) http.HandlerFunc { +func HandleSoftDelete(repoCtrl *repo.Controller) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { ctx := r.Context() + session, _ := request.AuthSessionFrom(ctx) + repoRef, err := request.GetRepoRefFromPath(r) if err != nil { render.TranslatedUserError(w, err) return } - err = repoCtrl.Delete(ctx, session, repoRef) + err = repoCtrl.SoftDelete(ctx, session, repoRef) if err != nil { render.TranslatedUserError(w, err) return diff --git a/app/api/openapi/repo.go b/app/api/openapi/repo.go index 82caade20..39044045a 100644 --- a/app/api/openapi/repo.go +++ b/app/api/openapi/repo.go @@ -194,6 +194,11 @@ type rule struct { Pattern protection.Pattern `json:"pattern"` } +type restoreRequest struct { + repoRequest + repo.RestoreInput +} + var queryParameterGitRef = openapi3.ParameterOrRef{ Parameter: &openapi3.Parameter{ Name: request.QueryParamGitRef, @@ -453,6 +458,20 @@ var queryParameterBypassRules = openapi3.ParameterOrRef{ }, } +var queryParameterRepoDeletedAt = openapi3.ParameterOrRef{ + Parameter: &openapi3.Parameter{ + Name: request.QueryParamRepoDeletedAt, + In: openapi3.ParameterInQuery, + Description: ptr.String("The time repository was deleted at in epoch format."), + Required: ptr.Bool(true), + Schema: &openapi3.SchemaOrRef{ + Schema: &openapi3.Schema{ + Type: ptrSchemaType(openapi3.SchemaTypeInteger), + }, + }, + }, +} + //nolint:funlen func repoOperations(reflector *openapi3.Reflector) { createRepository := openapi3.Operation{} @@ -513,6 +532,30 @@ func repoOperations(reflector *openapi3.Reflector) { _ = reflector.SetJSONResponse(&opDelete, new(usererror.Error), http.StatusNotFound) _ = reflector.Spec.AddOperation(http.MethodDelete, "/repos/{repo_ref}", opDelete) + opPurge := openapi3.Operation{} + opPurge.WithTags("repository") + opPurge.WithMapOfAnything(map[string]interface{}{"operationId": "purgeRepository"}) + opPurge.WithParameters(queryParameterRepoDeletedAt) + _ = reflector.SetRequest(&opPurge, new(repoRequest), http.MethodPost) + _ = reflector.SetJSONResponse(&opPurge, nil, http.StatusNoContent) + _ = reflector.SetJSONResponse(&opPurge, new(usererror.Error), http.StatusInternalServerError) + _ = reflector.SetJSONResponse(&opPurge, new(usererror.Error), http.StatusUnauthorized) + _ = reflector.SetJSONResponse(&opPurge, new(usererror.Error), http.StatusForbidden) + _ = reflector.SetJSONResponse(&opPurge, new(usererror.Error), http.StatusNotFound) + _ = reflector.Spec.AddOperation(http.MethodPost, "/repos/{repo_ref}/purge", opPurge) + + opRestore := openapi3.Operation{} + opRestore.WithTags("repository") + opRestore.WithMapOfAnything(map[string]interface{}{"operationId": "restoreRepository"}) + _ = reflector.SetRequest(&opRestore, new(restoreRequest), http.MethodPost) + _ = reflector.SetJSONResponse(&opRestore, new(types.Repository), http.StatusOK) + _ = reflector.SetJSONResponse(&opRestore, new(usererror.Error), http.StatusBadRequest) + _ = reflector.SetJSONResponse(&opRestore, new(usererror.Error), http.StatusInternalServerError) + _ = reflector.SetJSONResponse(&opRestore, new(usererror.Error), http.StatusUnauthorized) + _ = reflector.SetJSONResponse(&opRestore, new(usererror.Error), http.StatusForbidden) + _ = reflector.SetJSONResponse(&opRestore, new(usererror.Error), http.StatusNotFound) + _ = reflector.Spec.AddOperation(http.MethodPost, "/repos/{repo_ref}/restore", opRestore) + opMove := openapi3.Operation{} opMove.WithTags("repository") opMove.WithMapOfAnything(map[string]interface{}{"operationId": "moveRepository"}) diff --git a/app/api/request/repo.go b/app/api/request/repo.go index 6edc46b55..04bbdee67 100644 --- a/app/api/request/repo.go +++ b/app/api/request/repo.go @@ -23,8 +23,9 @@ import ( ) const ( - PathParamRepoRef = "repo_ref" - QueryParamRepoID = "repo_id" + PathParamRepoRef = "repo_ref" + QueryParamRepoID = "repo_id" + QueryParamRepoDeletedAt = "repo_deleted_at" ) func GetRepoRefFromPath(r *http.Request) (string, error) { @@ -54,3 +55,8 @@ func ParseRepoFilter(r *http.Request) *types.RepoFilter { Size: ParseLimit(r), } } + +// GetRepoDeletedAtFromQuery extracts the repository deleted timestamp from the query. +func GetRepoDeletedAtFromQuery(r *http.Request) (int64, error) { + return QueryParamAsPositiveInt64(r, QueryParamRepoDeletedAt) +} diff --git a/app/router/api.go b/app/router/api.go index 1006235d9..efbb64481 100644 --- a/app/router/api.go +++ b/app/router/api.go @@ -260,7 +260,9 @@ func setupRepos(r chi.Router, // repo level operations r.Get("/", handlerrepo.HandleFind(repoCtrl)) r.Patch("/", handlerrepo.HandleUpdate(repoCtrl)) - r.Delete("/", handlerrepo.HandleDelete(repoCtrl)) + r.Delete("/", handlerrepo.HandleSoftDelete(repoCtrl)) + r.Post("/purge", handlerrepo.HandlePurge(repoCtrl)) + r.Post("/restore", handlerrepo.HandleRestore(repoCtrl)) r.Post("/move", handlerrepo.HandleMove(repoCtrl)) r.Get("/service-accounts", handlerrepo.HandleListServiceAccounts(repoCtrl)) diff --git a/app/services/cleanup/deleted_repos.go b/app/services/cleanup/deleted_repos.go new file mode 100644 index 000000000..719902bfb --- /dev/null +++ b/app/services/cleanup/deleted_repos.go @@ -0,0 +1,102 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cleanup + +import ( + "context" + "fmt" + "math" + "time" + + "github.com/harness/gitness/app/api/controller/repo" + "github.com/harness/gitness/app/bootstrap" + "github.com/harness/gitness/app/store" + "github.com/harness/gitness/job" + "github.com/harness/gitness/types" + "github.com/harness/gitness/types/enum" + + "github.com/rs/zerolog/log" +) + +const ( + jobTypeDeletedRepos = "gitness:cleanup:deleted-repos" + jobCronDeletedRepos = "50 0 * * *" // At minute 50 past midnight every day. + jobMaxDurationDeletedRepos = 10 * time.Minute +) + +type deletedReposCleanupJob struct { + retentionTime time.Duration + + repoStore store.RepoStore + repoCtrl *repo.Controller +} + +func newDeletedReposCleanupJob( + retentionTime time.Duration, + repoStore store.RepoStore, + repoCtrl *repo.Controller, +) *deletedReposCleanupJob { + return &deletedReposCleanupJob{ + retentionTime: retentionTime, + + repoStore: repoStore, + repoCtrl: repoCtrl, + } +} + +// Handle purges old deleted repositories that are past the retention time. +func (j *deletedReposCleanupJob) Handle(ctx context.Context, _ string, _ job.ProgressReporter) (string, error) { + olderThan := time.Now().Add(-j.retentionTime) + + log.Ctx(ctx).Info().Msgf( + "start purging deleted repositories older than %s (aka created before %s)", + j.retentionTime, + olderThan.Format(time.RFC3339Nano)) + + deletedBefore := olderThan.UnixMilli() + filter := &types.RepoFilter{ + Page: 1, + Size: int(math.MaxInt), + Query: "", + Order: enum.OrderAsc, + Sort: enum.RepoAttrDeleted, + DeletedBefore: &deletedBefore, + } + toBePurgedRepos, err := j.repoStore.List(ctx, 0, filter) + if err != nil { + return "", fmt.Errorf("failed to list ready-to-delete repositories: %w", err) + } + + session := bootstrap.NewSystemServiceSession() + purgedRepos := 0 + for _, r := range toBePurgedRepos { + err := j.repoCtrl.PurgeNoAuth(ctx, session, r) + if err != nil { + log.Warn().Err(err).Msgf("failed to purge repo uid: %s, path: %s, deleted at %d", + r.Identifier, r.Path, *r.Deleted) + continue + } + purgedRepos++ + } + + result := "no old deleted repositories found" + if purgedRepos > 0 { + result = fmt.Sprintf("purged %d deleted repositories", purgedRepos) + } + + log.Ctx(ctx).Info().Msg(result) + + return result, nil +} diff --git a/app/services/cleanup/service.go b/app/services/cleanup/service.go index 6f30b9786..65ec47589 100644 --- a/app/services/cleanup/service.go +++ b/app/services/cleanup/service.go @@ -20,12 +20,14 @@ import ( "fmt" "time" + "github.com/harness/gitness/app/api/controller/repo" "github.com/harness/gitness/app/store" "github.com/harness/gitness/job" ) type Config struct { - WebhookExecutionsRetentionTime time.Duration + WebhookExecutionsRetentionTime time.Duration + DeletedRepositoriesRetentionTime time.Duration } func (c *Config) Prepare() error { @@ -35,6 +37,10 @@ func (c *Config) Prepare() error { if c.WebhookExecutionsRetentionTime <= 0 { return errors.New("config.WebhookExecutionsRetentionTime has to be provided") } + + if c.DeletedRepositoriesRetentionTime <= 0 { + return errors.New("config.DeletedRepositoriesRetentionTime has to be provided") + } return nil } @@ -45,6 +51,8 @@ type Service struct { executor *job.Executor webhookExecutionStore store.WebhookExecutionStore tokenStore store.TokenStore + repoStore store.RepoStore + repoCtrl *repo.Controller } func NewService( @@ -53,6 +61,8 @@ func NewService( executor *job.Executor, webhookExecutionStore store.WebhookExecutionStore, tokenStore store.TokenStore, + repoStore store.RepoStore, + repoCtrl *repo.Controller, ) (*Service, error) { if err := config.Prepare(); err != nil { return nil, fmt.Errorf("provided cleanup config is invalid: %w", err) @@ -65,6 +75,8 @@ func NewService( executor: executor, webhookExecutionStore: webhookExecutionStore, tokenStore: tokenStore, + repoStore: repoStore, + repoCtrl: repoCtrl, }, nil } @@ -104,6 +116,16 @@ func (s *Service) scheduleRecurringCleanupJobs(ctx context.Context) error { return fmt.Errorf("failed to schedule token job: %w", err) } + err = s.scheduler.AddRecurring( + ctx, + jobTypeDeletedRepos, + jobTypeDeletedRepos, + jobCronDeletedRepos, + jobMaxDurationDeletedRepos, + ) + if err != nil { + return fmt.Errorf("failed to schedule deleted repo cleanup job: %w", err) + } return nil } @@ -128,5 +150,15 @@ func (s *Service) registerJobHandlers() error { return fmt.Errorf("failed to register job handler for token cleanup: %w", err) } + if err := s.executor.Register( + jobTypeDeletedRepos, + newDeletedReposCleanupJob( + s.config.DeletedRepositoriesRetentionTime, + s.repoStore, + s.repoCtrl, + ), + ); err != nil { + return fmt.Errorf("failed to register job handler for deleted repos cleanup: %w", err) + } return nil } diff --git a/app/services/cleanup/wire.go b/app/services/cleanup/wire.go index 605cbdada..905ff3736 100644 --- a/app/services/cleanup/wire.go +++ b/app/services/cleanup/wire.go @@ -15,6 +15,7 @@ package cleanup import ( + "github.com/harness/gitness/app/api/controller/repo" "github.com/harness/gitness/app/store" "github.com/harness/gitness/job" @@ -31,6 +32,8 @@ func ProvideService( executor *job.Executor, webhookExecutionStore store.WebhookExecutionStore, tokenStore store.TokenStore, + repoStore store.RepoStore, + repoCtrl *repo.Controller, ) (*Service, error) { return NewService( config, @@ -38,5 +41,7 @@ func ProvideService( executor, webhookExecutionStore, tokenStore, + repoStore, + repoCtrl, ) } diff --git a/app/store/database.go b/app/store/database.go index af32afb60..57aa12837 100644 --- a/app/store/database.go +++ b/app/store/database.go @@ -184,6 +184,9 @@ type ( // Find the repo by id. Find(ctx context.Context, id int64) (*types.Repository, error) + // FindByRefAndDeletedAt finds the repo using the repoRef and deleted timestamp. + FindByRefAndDeletedAt(ctx context.Context, repoRef string, deletedAt int64) (*types.Repository, error) + // FindByRef finds the repo using the repoRef as either the id or the repo path. FindByRef(ctx context.Context, repoRef string) (*types.Repository, error) @@ -194,28 +197,36 @@ type ( Update(ctx context.Context, repo *types.Repository) error // Update the repo size. - UpdateSize(ctx context.Context, repoID int64, repoSize int64) error + UpdateSize(ctx context.Context, id int64, repoSize int64) error // Get the repo size. - GetSize(ctx context.Context, repoID int64) (int64, error) + GetSize(ctx context.Context, id int64) (int64, error) // UpdateOptLock the repo details using the optimistic locking mechanism. UpdateOptLock(ctx context.Context, repo *types.Repository, mutateFn func(repository *types.Repository) error) (*types.Repository, error) - // Delete the repo. - Delete(ctx context.Context, id int64) error + // SoftDelete a repo. + SoftDelete(ctx context.Context, repo *types.Repository, deletedAt *int64) error - // Count of repos in a space. + // Purge the soft deleted repo permanently. + Purge(ctx context.Context, id int64, deletedAt *int64) error + + // Restore a deleted repo using the optimistic locking mechanism. + Restore(ctx context.Context, repo *types.Repository, + newIdentifier string) (*types.Repository, error) + + // Count of active repos in a space. With "DeletedBefore" filter, counts only deleted repos by opts.DeletedBefore. Count(ctx context.Context, parentID int64, opts *types.RepoFilter) (int64, error) - // Count all repos in a hierarchy of spaces. + // Count all active repos in a hierarchy of spaces. CountAll(ctx context.Context, spaceID int64) (int64, error) - // List returns a list of repos in a space. + // List returns a list of active repos in a space. + // With "DeletedBefore" filter, shows deleted repos by opts.DeletedBefore. List(ctx context.Context, parentID int64, opts *types.RepoFilter) ([]*types.Repository, error) - // ListSizeInfos returns a list of all repo sizes. + // ListSizeInfos returns a list of all active repo sizes. ListSizeInfos(ctx context.Context) ([]*types.RepositorySizeInfo, error) } diff --git a/app/store/database/migrate/postgres/0045_alter_table_repo_add_deletetimestamp.down.sql b/app/store/database/migrate/postgres/0045_alter_table_repo_add_deletetimestamp.down.sql new file mode 100644 index 000000000..0cbb0525e --- /dev/null +++ b/app/store/database/migrate/postgres/0045_alter_table_repo_add_deletetimestamp.down.sql @@ -0,0 +1,7 @@ +ALTER TABLE repositories DROP COLUMN repo_deleted; + +DROP INDEX repositories_parent_id_uid; +DROP INDEX repositories_deleted; + +CREATE UNIQUE INDEX repositories_parent_id_uid +ON repositories(repo_parent_id, LOWER(repo_uid)); diff --git a/app/store/database/migrate/postgres/0045_alter_table_repo_add_deletetimestamp.up.sql b/app/store/database/migrate/postgres/0045_alter_table_repo_add_deletetimestamp.up.sql new file mode 100644 index 000000000..324d699ab --- /dev/null +++ b/app/store/database/migrate/postgres/0045_alter_table_repo_add_deletetimestamp.up.sql @@ -0,0 +1,11 @@ +ALTER TABLE repositories ADD COLUMN repo_deleted BIGINT DEFAULT NULL; + +DROP INDEX repositories_parent_id_uid; + +CREATE UNIQUE INDEX repositories_parent_id_uid + ON repositories(repo_parent_id, LOWER(repo_uid)); + WHERE repo_deleted IS NULL; + +CREATE INDEX repositories_deleted + ON repositories(repo_deleted) + WHERE repo_deleted IS NOT NULL; \ No newline at end of file diff --git a/app/store/database/migrate/sqlite/0045_alter_table_repo_add_deletetimestamp.down.sql b/app/store/database/migrate/sqlite/0045_alter_table_repo_add_deletetimestamp.down.sql new file mode 100644 index 000000000..0cbb0525e --- /dev/null +++ b/app/store/database/migrate/sqlite/0045_alter_table_repo_add_deletetimestamp.down.sql @@ -0,0 +1,7 @@ +ALTER TABLE repositories DROP COLUMN repo_deleted; + +DROP INDEX repositories_parent_id_uid; +DROP INDEX repositories_deleted; + +CREATE UNIQUE INDEX repositories_parent_id_uid +ON repositories(repo_parent_id, LOWER(repo_uid)); diff --git a/app/store/database/migrate/sqlite/0045_alter_table_repo_add_deletetimestamp.up.sql b/app/store/database/migrate/sqlite/0045_alter_table_repo_add_deletetimestamp.up.sql new file mode 100644 index 000000000..de885ed0a --- /dev/null +++ b/app/store/database/migrate/sqlite/0045_alter_table_repo_add_deletetimestamp.up.sql @@ -0,0 +1,11 @@ +ALTER TABLE repositories ADD COLUMN repo_deleted BIGINT DEFAULT NULL; + +DROP INDEX repositories_parent_id_uid; + +CREATE UNIQUE INDEX repositories_parent_id_uid + ON repositories(repo_parent_id, LOWER(repo_uid)) + WHERE repo_deleted IS NULL; + +CREATE INDEX repositories_deleted + ON repositories(repo_deleted) + WHERE repo_deleted IS NOT NULL; \ No newline at end of file diff --git a/app/store/database/repo.go b/app/store/database/repo.go index f51a8aeed..bcf804323 100644 --- a/app/store/database/repo.go +++ b/app/store/database/repo.go @@ -29,6 +29,7 @@ import ( "github.com/harness/gitness/types" "github.com/harness/gitness/types/enum" + "github.com/guregu/null" "github.com/jmoiron/sqlx" "github.com/pkg/errors" ) @@ -57,15 +58,16 @@ type RepoStore struct { type repository struct { // TODO: int64 ID doesn't match DB - ID int64 `db:"repo_id"` - Version int64 `db:"repo_version"` - ParentID int64 `db:"repo_parent_id"` - Identifier string `db:"repo_uid"` - Description string `db:"repo_description"` - IsPublic bool `db:"repo_is_public"` - CreatedBy int64 `db:"repo_created_by"` - Created int64 `db:"repo_created"` - Updated int64 `db:"repo_updated"` + ID int64 `db:"repo_id"` + Version int64 `db:"repo_version"` + ParentID int64 `db:"repo_parent_id"` + Identifier string `db:"repo_uid"` + Description string `db:"repo_description"` + IsPublic bool `db:"repo_is_public"` + CreatedBy int64 `db:"repo_created_by"` + Created int64 `db:"repo_created"` + Updated int64 `db:"repo_updated"` + Deleted null.Int `db:"repo_deleted"` Size int64 `db:"repo_size"` SizeUpdated int64 `db:"repo_size_updated"` @@ -95,6 +97,7 @@ const ( ,repo_created_by ,repo_created ,repo_updated + ,repo_deleted ,repo_size ,repo_size_updated ,repo_git_uid @@ -115,40 +118,53 @@ const ( // Find finds the repo by id. func (s *RepoStore) Find(ctx context.Context, id int64) (*types.Repository, error) { - const sqlQuery = repoSelectBase + ` - WHERE repo_id = $1` + return s.find(ctx, id, nil) +} + +// find is a wrapper to find a repo by id w/o deleted timestamp. +func (s *RepoStore) find(ctx context.Context, id int64, deletedAt *int64) (*types.Repository, error) { + var sqlQuery = repoSelectBase + ` + WHERE repo_id = $1 AND repo_deleted IS NULL` + if deletedAt != nil { + sqlQuery = repoSelectBase + ` + WHERE repo_id = $1 AND repo_deleted = $2` + } db := dbtx.GetAccessor(ctx, s.db) dst := new(repository) - if err := db.GetContext(ctx, dst, sqlQuery, id); err != nil { + if err := db.GetContext(ctx, dst, sqlQuery, id, deletedAt); err != nil { return nil, database.ProcessSQLErrorf(err, "Failed to find repo") } return s.mapToRepo(ctx, dst) } -// Find finds the repo with the given identifier in the given space ID. -func (s *RepoStore) FindByIdentifier( +func (s *RepoStore) findByIdentifier( ctx context.Context, spaceID int64, identifier string, + deletedAt *int64, ) (*types.Repository, error) { - const sqlQuery = repoSelectBase + ` - WHERE repo_parent_id = $1 AND LOWER(repo_uid) = $2` + var sqlQuery = repoSelectBase + ` + WHERE repo_parent_id = $1 AND LOWER(repo_uid) = $2 AND repo_deleted IS NULL` + + if deletedAt != nil { + sqlQuery = repoSelectBase + ` + WHERE repo_parent_id = $1 AND LOWER(repo_uid) = $2 AND repo_deleted = $3` + } db := dbtx.GetAccessor(ctx, s.db) dst := new(repository) - if err := db.GetContext(ctx, dst, sqlQuery, spaceID, strings.ToLower(identifier)); err != nil { + if err := db.GetContext(ctx, dst, sqlQuery, spaceID, strings.ToLower(identifier), deletedAt); err != nil { return nil, database.ProcessSQLErrorf(err, "Failed to find repo") } return s.mapToRepo(ctx, dst) } -// FindByRef finds the repo using the repoRef as either the id or the repo path. -func (s *RepoStore) FindByRef(ctx context.Context, repoRef string) (*types.Repository, error) { +func (s *RepoStore) findByRef(ctx context.Context, repoRef string, deletedAt *int64) (*types.Repository, error) { // ASSUMPTION: digits only is not a valid repo path id, err := strconv.ParseInt(repoRef, 10, 64) if err != nil { @@ -161,10 +177,23 @@ func (s *RepoStore) FindByRef(ctx context.Context, repoRef string) (*types.Repos return nil, fmt.Errorf("failed to get space path: %w", err) } - return s.FindByIdentifier(ctx, pathObject.SpaceID, repoIdentifier) + return s.findByIdentifier(ctx, pathObject.SpaceID, repoIdentifier, deletedAt) } + return s.find(ctx, id, deletedAt) +} - return s.Find(ctx, id) +// FindByRef finds the repo using the repoRef as either the id or the repo path. +func (s *RepoStore) FindByRef(ctx context.Context, repoRef string) (*types.Repository, error) { + return s.findByRef(ctx, repoRef, nil) +} + +// FindByRefAndDeletedAt finds the repo using the repoRef and deleted timestamp. +func (s *RepoStore) FindByRefAndDeletedAt( + ctx context.Context, + repoRef string, + deletedAt int64, +) (*types.Repository, error) { + return s.findByRef(ctx, repoRef, &deletedAt) } // Create creates a new repository. @@ -179,6 +208,7 @@ func (s *RepoStore) Create(ctx context.Context, repo *types.Repository) error { ,repo_created_by ,repo_created ,repo_updated + ,repo_deleted ,repo_size ,repo_size_updated ,repo_git_uid @@ -200,6 +230,7 @@ func (s *RepoStore) Create(ctx context.Context, repo *types.Repository) error { ,:repo_created_by ,:repo_created ,:repo_updated + ,:repo_deleted ,:repo_size ,:repo_size_updated ,:repo_git_uid @@ -241,6 +272,7 @@ func (s *RepoStore) Update(ctx context.Context, repo *types.Repository) error { SET repo_version = :repo_version ,repo_updated = :repo_updated + ,repo_deleted = :repo_deleted ,repo_parent_id = :repo_parent_id ,repo_uid = :repo_uid ,repo_git_uid = :repo_git_uid @@ -296,12 +328,12 @@ func (s *RepoStore) Update(ctx context.Context, repo *types.Repository) error { } // UpdateSize updates the size of a specific repository in the database. -func (s *RepoStore) UpdateSize(ctx context.Context, repoID int64, size int64) error { +func (s *RepoStore) UpdateSize(ctx context.Context, id int64, size int64) error { stmt := database.Builder. Update("repositories"). Set("repo_size", size). Set("repo_size_updated", time.Now().UnixMilli()). - Where("repo_id = ?", repoID) + Where("repo_id = ? AND repo_deleted IS NULL", id) sqlQuery, args, err := stmt.ToSql() if err != nil { @@ -321,26 +353,62 @@ func (s *RepoStore) UpdateSize(ctx context.Context, repoID int64, size int64) er } if count == 0 { - return fmt.Errorf("repo %d size not updated: %w", repoID, gitness_store.ErrResourceNotFound) + return fmt.Errorf("repo %d size not updated: %w", id, gitness_store.ErrResourceNotFound) } return nil } // GetSize returns the repo size. -func (s *RepoStore) GetSize(ctx context.Context, repoID int64) (int64, error) { - query := "SELECT repo_size FROM repositories WHERE repo_id = $1;" +func (s *RepoStore) GetSize(ctx context.Context, id int64) (int64, error) { + query := "SELECT repo_size FROM repositories WHERE repo_id = $1 AND repo_deleted IS NULL;" db := dbtx.GetAccessor(ctx, s.db) var size int64 - if err := db.GetContext(ctx, &size, query, repoID); err != nil { + if err := db.GetContext(ctx, &size, query, id); err != nil { return 0, database.ProcessSQLErrorf(err, "failed to get repo size") } return size, nil } -// UpdateOptLock updates the repository using the optimistic locking mechanism. -func (s *RepoStore) UpdateOptLock(ctx context.Context, +// UpdateOptLock updates the active repository using the optimistic locking mechanism. +func (s *RepoStore) UpdateOptLock( + ctx context.Context, + repo *types.Repository, + mutateFn func(repository *types.Repository) error, +) (*types.Repository, error) { + return s.updateOptLock( + ctx, + repo, + func(r *types.Repository) error { + if repo.Deleted != nil { + return gitness_store.ErrResourceNotFound + } + return mutateFn(r) + }, + ) +} + +// UpdateDeletedOptLock updates a deleted repository using the optimistic locking mechanism. +func (s *RepoStore) updateDeletedOptLock(ctx context.Context, + repo *types.Repository, + mutateFn func(repository *types.Repository) error, +) (*types.Repository, error) { + return s.updateOptLock( + ctx, + repo, + func(r *types.Repository) error { + if repo.Deleted == nil { + return gitness_store.ErrResourceNotFound + } + return mutateFn(r) + }, + ) +} + +// updateOptLock updates the repository using the optimistic locking mechanism. +func (s *RepoStore) updateOptLock( + ctx context.Context, repo *types.Repository, mutateFn func(repository *types.Repository) error, ) (*types.Repository, error) { @@ -360,29 +428,61 @@ func (s *RepoStore) UpdateOptLock(ctx context.Context, return nil, err } - repo, err = s.Find(ctx, repo.ID) + repo, err = s.find(ctx, repo.ID, repo.Deleted) if err != nil { return nil, err } } } -// Delete the repository. -func (s *RepoStore) Delete(ctx context.Context, id int64) error { +// SoftDelete deletes a repo softly by setting the deleted timestamp. +func (s *RepoStore) SoftDelete(ctx context.Context, repo *types.Repository, deletedAt *int64) error { + _, err := s.UpdateOptLock(ctx, repo, func(r *types.Repository) error { + r.Deleted = deletedAt + return nil + }) + if err != nil { + return fmt.Errorf("failed to soft delete repo: %w", err) + } + return nil +} + +// Purge deletes the repo permanently. +func (s *RepoStore) Purge(ctx context.Context, id int64, deletedAt *int64) error { const repoDelete = ` DELETE FROM repositories - WHERE repo_id = $1` + WHERE repo_id = $1 AND repo_deleted = $2` db := dbtx.GetAccessor(ctx, s.db) - if _, err := db.ExecContext(ctx, repoDelete, id); err != nil { + if _, err := db.ExecContext(ctx, repoDelete, id, deletedAt); err != nil { return database.ProcessSQLErrorf(err, "the delete query failed") } return nil } -// Count of repos in a space. if parentID (space) is zero then it will count all repositories in the system. +// Restore restores a deleted repo. +func (s *RepoStore) Restore( + ctx context.Context, + repo *types.Repository, + newIdentifier string, +) (*types.Repository, error) { + repo, err := s.updateDeletedOptLock(ctx, repo, func(r *types.Repository) error { + r.Deleted = nil + if newIdentifier != "" { + r.Identifier = newIdentifier + } + return nil + }) + if err != nil { + return nil, database.ProcessSQLErrorf(err, "failed to restore the repo") + } + return repo, nil +} + +// Count of active repos in a space. if parentID (space) is zero then it will count all repositories in the system. +// With "DeletedBefore" filter, counts only deleted repos by opts.DeletedBefore. func (s *RepoStore) Count(ctx context.Context, parentID int64, opts *types.RepoFilter) (int64, error) { stmt := database.Builder. Select("count(*)"). @@ -396,6 +496,12 @@ func (s *RepoStore) Count(ctx context.Context, parentID int64, opts *types.RepoF stmt = stmt.Where("LOWER(repo_uid) LIKE ?", fmt.Sprintf("%%%s%%", strings.ToLower(opts.Query))) } + if opts.DeletedBefore != nil { + stmt = stmt.Where("repo_deleted < ?", opts.DeletedBefore) + } else { + stmt = stmt.Where("repo_deleted IS NULL") + } + sql, args, err := stmt.ToSql() if err != nil { return 0, errors.Wrap(err, "Failed to convert query to sql") @@ -411,7 +517,7 @@ func (s *RepoStore) Count(ctx context.Context, parentID int64, opts *types.RepoF return count, nil } -// Count all repos in a hierarchy of spaces. +// Count all active repos in a hierarchy of spaces. func (s *RepoStore) CountAll(ctx context.Context, spaceID int64) (int64, error) { query := `WITH RECURSIVE SpaceHierarchy AS ( SELECT space_id, space_parent_id @@ -435,7 +541,7 @@ FROM SpaceHierarchy h1;` } query = fmt.Sprintf( - "SELECT COUNT(repo_id) FROM repositories WHERE repo_parent_id IN (%s);", + "SELECT COUNT(repo_id) FROM repositories WHERE repo_parent_id IN (%s) AND repo_deleted IS NULL;", intsToCSV(spaceIDs), ) @@ -447,13 +553,20 @@ FROM SpaceHierarchy h1;` return numRepos, nil } -// List returns a list of repos in a space. +// List returns a list of active repos in a space. +// With "DeletedBefore" filter, shows deleted repos by opts.DeletedBefore. func (s *RepoStore) List(ctx context.Context, parentID int64, opts *types.RepoFilter) ([]*types.Repository, error) { stmt := database.Builder. Select(repoColumnsForJoin). From("repositories"). Where("repo_parent_id = ?", fmt.Sprint(parentID)) + if opts.DeletedBefore != nil { + stmt = stmt.Where("repo_deleted < ?", opts.DeletedBefore) + } else { + stmt = stmt.Where("repo_deleted IS NULL") + } + if opts.Query != "" { stmt = stmt.Where("LOWER(repo_uid) LIKE ?", fmt.Sprintf("%%%s%%", strings.ToLower(opts.Query))) } @@ -472,6 +585,8 @@ func (s *RepoStore) List(ctx context.Context, parentID int64, opts *types.RepoFi stmt = stmt.OrderBy("repo_created " + opts.Order.String()) case enum.RepoAttrUpdated: stmt = stmt.OrderBy("repo_updated " + opts.Order.String()) + case enum.RepoAttrDeleted: + stmt = stmt.OrderBy("repo_deleted " + opts.Order.String()) } sql, args, err := stmt.ToSql() @@ -499,7 +614,8 @@ type repoSize struct { func (s *RepoStore) ListSizeInfos(ctx context.Context) ([]*types.RepositorySizeInfo, error) { stmt := database.Builder. Select("repo_id", "repo_git_uid", "repo_size", "repo_size_updated"). - From("repositories") + From("repositories"). + Where("repo_deleted IS NULL") sql, args, err := stmt.ToSql() if err != nil { @@ -531,6 +647,7 @@ func (s *RepoStore) mapToRepo( Created: in.Created, CreatedBy: in.CreatedBy, Updated: in.Updated, + Deleted: in.Deleted.Ptr(), Size: in.Size, SizeUpdated: in.SizeUpdated, GitUID: in.GitUID, @@ -609,6 +726,7 @@ func mapToInternalRepo(in *types.Repository) *repository { Created: in.Created, CreatedBy: in.CreatedBy, Updated: in.Updated, + Deleted: null.IntFromPtr(in.Deleted), Size: in.Size, SizeUpdated: in.SizeUpdated, GitUID: in.GitUID, diff --git a/cli/operations/server/config.go b/cli/operations/server/config.go index e6034098b..a4e1b6ac8 100644 --- a/cli/operations/server/config.go +++ b/cli/operations/server/config.go @@ -343,7 +343,8 @@ func ProvidePubsubConfig(config *types.Config) pubsub.Config { // ProvideCleanupConfig loads the cleanup service config from the main config. func ProvideCleanupConfig(config *types.Config) cleanup.Config { return cleanup.Config{ - WebhookExecutionsRetentionTime: config.Webhook.RetentionTime, + WebhookExecutionsRetentionTime: config.Webhook.RetentionTime, + DeletedRepositoriesRetentionTime: config.Repos.DeletedRetentionTime, } } diff --git a/cmd/gitness/wire.go b/cmd/gitness/wire.go index cba70c2b1..aa31bdd36 100644 --- a/cmd/gitness/wire.go +++ b/cmd/gitness/wire.go @@ -143,11 +143,11 @@ func initSystem(ctx context.Context, config *types.Config) (*cliserver.System, e lock.WireSet, cliserver.ProvidePubsubConfig, pubsub.WireSet, + cliserver.ProvideJobsConfig, + job.WireSet, cliserver.ProvideCleanupConfig, cleanup.WireSet, codecomments.WireSet, - cliserver.ProvideJobsConfig, - job.WireSet, protection.WireSet, checkcontroller.WireSet, execution.WireSet, diff --git a/cmd/gitness/wire_gen.go b/cmd/gitness/wire_gen.go index c71cbd1b6..c15ceb42f 100644 --- a/cmd/gitness/wire_gen.go +++ b/cmd/gitness/wire_gen.go @@ -304,7 +304,7 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro return nil, err } cleanupConfig := server.ProvideCleanupConfig(config) - cleanupService, err := cleanup.ProvideService(cleanupConfig, jobScheduler, executor, webhookExecutionStore, tokenStore) + cleanupService, err := cleanup.ProvideService(cleanupConfig, jobScheduler, executor, webhookExecutionStore, tokenStore, repoStore, repoController) if err != nil { return nil, err } diff --git a/git/repo.go b/git/repo.go index a0c06c27e..4056d91fc 100644 --- a/git/repo.go +++ b/git/repo.go @@ -225,7 +225,7 @@ func (s *Service) DeleteRepositoryBestEffort(ctx context.Context, repoUID string repoPath := getFullPathForRepo(s.reposRoot, repoUID) tempPath := path.Join(s.reposGraveyard, repoUID) - // delete should not fail if repoGraveyard dir does not exist + // delete should not fail if repoGraveyard dir does not exist. if _, err := os.Stat(s.reposGraveyard); os.IsNotExist(err) { if errdir := os.MkdirAll(s.reposGraveyard, fileMode700); errdir != nil { return fmt.Errorf("clean up dir '%s' doesn't exist and can't be created: %w", s.reposGraveyard, errdir) diff --git a/types/config.go b/types/config.go index 55c36bde0..3cc760174 100644 --- a/types/config.go +++ b/types/config.go @@ -348,4 +348,9 @@ type Config struct { Concurrency int `envconfig:"GITNESS_KEYWORD_SEARCH_CONCURRENCY" default:"4"` MaxRetries int `envconfig:"GITNESS_KEYWORD_SEARCH_MAX_RETRIES" default:"3"` } + + Repos struct { + // DeletedRetentionTime is the duration after which deleted repositories will be purged. + DeletedRetentionTime time.Duration `envconfig:"GITNESS_REPOS_DELETED_RETENTION_TIME" default:"2160h"` // 90 days + } } diff --git a/types/enum/common.go b/types/enum/common.go index e89215f4f..fc5129a80 100644 --- a/types/enum/common.go +++ b/types/enum/common.go @@ -48,6 +48,8 @@ const ( updated = "updated" updatedAt = "updated_at" updatedBy = "updated_by" + deleted = "deleted" + deletedAt = "deleted_at" displayName = "display_name" date = "date" defaultString = "default" diff --git a/types/enum/repo.go b/types/enum/repo.go index 1259cba64..32ce3c018 100644 --- a/types/enum/repo.go +++ b/types/enum/repo.go @@ -29,6 +29,7 @@ const ( RepoAttrIdentifier RepoAttrCreated RepoAttrUpdated + RepoAttrDeleted ) // ParseRepoAttr parses the repo attribute string @@ -44,6 +45,8 @@ func ParseRepoAttr(s string) RepoAttr { return RepoAttrCreated case updated, updatedAt: return RepoAttrUpdated + case deleted, deletedAt: + return RepoAttrDeleted default: return RepoAttrNone } @@ -61,6 +64,8 @@ func (a RepoAttr) String() string { return created case RepoAttrUpdated: return updated + case RepoAttrDeleted: + return deleted case RepoAttrNone: return "" default: diff --git a/types/repo.go b/types/repo.go index ad333f224..00f72f487 100644 --- a/types/repo.go +++ b/types/repo.go @@ -33,6 +33,7 @@ type Repository struct { CreatedBy int64 `json:"created_by"` Created int64 `json:"created"` Updated int64 `json:"updated"` + Deleted *int64 `json:"deleted,omitempty"` Size int64 `json:"size"` SizeUpdated int64 `json:"size_updated"` @@ -80,11 +81,12 @@ func (r Repository) GetGitUID() string { // RepoFilter stores repo query parameters. type RepoFilter struct { - Page int `json:"page"` - Size int `json:"size"` - Query string `json:"query"` - Sort enum.RepoAttr `json:"sort"` - Order enum.Order `json:"order"` + Page int `json:"page"` + Size int `json:"size"` + Query string `json:"query"` + Sort enum.RepoAttr `json:"sort"` + Order enum.Order `json:"order"` + DeletedBefore *int64 `json:"deleted_before,omitempty"` } // RepositoryGitInfo holds git info for a repository.