github/drone
5
0
Fork 0
mirror of https://github.com/harness/drone.git synced 2025-05-21 11:29:52 +08:00
Code Issues Packages Projects Releases Wiki Activity

Support soft delete, restore and purge repos plus a cleanup job for old deleted repos (#1005)

Browse Source
This commit is contained in:
Atefeh Mohseni-Ejiyeh 2024-02-14 01:39:39 +00:00 committed by Harness
parent 143b0bf5ad
commit fc9e77c91c
27 changed files with 711 additions and 90 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

40
app/api/controller/repo/delete.go → app/api/controller/repo/purge.go
View File

@ -20,6 +20,7 @@ import (
apiauth "github.com/harness/gitness/app/api/auth"
"github.com/harness/gitness/app/api/controller"
"github.com/harness/gitness/app/api/usererror"
"github.com/harness/gitness/app/auth"
repoevents "github.com/harness/gitness/app/events/repo"
"github.com/harness/gitness/errors"
@ -30,12 +31,11 @@ import (
"github.com/rs/zerolog/log"
)
// Delete deletes a repo.
func (c *Controller) Delete(ctx context.Context, session *auth.Session, repoRef string) error {
// note: can't use c.getRepoCheckAccess because import job for repositories being imported must be cancelled.
repo, err := c.repoStore.FindByRef(ctx, repoRef)
// Purge removes a repo permanently.
func (c *Controller) Purge(ctx context.Context, session *auth.Session, repoRef string, deletedAt int64) error {
repo, err := c.repoStore.FindByRefAndDeletedAt(ctx, repoRef, deletedAt)
if err != nil {
return err
return fmt.Errorf("failed to find the repo (deleted at %d): %w", deletedAt, err)
}
if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoDelete, false); err != nil {
@ -45,27 +45,28 @@ func (c *Controller) Delete(ctx context.Context, session *auth.Session, repoRef
log.Ctx(ctx).Info().
Int64("repo.id", repo.ID).
Str("repo.path", repo.Path).
Msgf("deleting repository")
Msg("purging repository")
if repo.Importing {
err = c.importer.Cancel(ctx, repo)
if err != nil {
return fmt.Errorf("failed to cancel repository import")
}
if repo.Deleted == nil {
return usererror.BadRequest("Repository has to be deleted before it can be purged.")
}
return c.DeleteNoAuth(ctx, session, repo)
return c.PurgeNoAuth(ctx, session, repo)
}
func (c *Controller) PurgeNoAuth(
ctx context.Context,
session *auth.Session,
repo *types.Repository,
) error {
if err := c.repoStore.Purge(ctx, repo.ID, repo.Deleted); err != nil {
return fmt.Errorf("failed to delete repo from db: %w", err)
}
func (c *Controller) DeleteNoAuth(ctx context.Context, session *auth.Session, repo *types.Repository) error {
if err := c.deleteGitRepository(ctx, session, repo); err != nil {
return fmt.Errorf("failed to delete git repository: %w", err)
}
if err := c.repoStore.Delete(ctx, repo.ID); err != nil {
return fmt.Errorf("failed to delete repo from db: %w", err)
}
c.eventReporter.Deleted(
ctx,
&repoevents.DeletedPayload{
@ -95,13 +96,12 @@ func (c *Controller) deleteGitRepository(
WriteParams: writeParams,
})
// deletion should not fail if dir does not exist in repos dir
// deletion should not fail if repo dir does not exist.
if errors.IsNotFound(err) {
log.Ctx(ctx).Warn().Str("repo.git_uid", repo.GitUID).
Msg("git repository directory does not exist")
} else if err != nil {
// deletion has failed before removing(rename) the repo dir
return fmt.Errorf("failed to delete git repository directory %s: %w", repo.GitUID, err)
return fmt.Errorf("failed to remove git repository %s: %w", repo.GitUID, err)
}
return nil
}

58
app/api/controller/repo/restore.go Normal file
View File

@ -0,0 +1,58 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package repo
import (
"context"
"fmt"
apiauth "github.com/harness/gitness/app/api/auth"
"github.com/harness/gitness/app/api/usererror"
"github.com/harness/gitness/app/auth"
"github.com/harness/gitness/types"
"github.com/harness/gitness/types/enum"
)
type RestoreInput struct {
NewIdentifier string `json:"new_identifier,omitempty"`
DeletedAt int64 `json:"deleted_at"`
}
func (c *Controller) Restore(
ctx context.Context,
session *auth.Session,
repoRef string,
in *RestoreInput,
) (*types.Repository, error) {
repo, err := c.repoStore.FindByRefAndDeletedAt(ctx, repoRef, in.DeletedAt)
if err != nil {
return nil, fmt.Errorf("failed to find repository: %w", err)
}
if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoEdit, false); err != nil {
return nil, fmt.Errorf("access check failed: %w", err)
}
if repo.Deleted == nil {
return nil, usererror.BadRequest("cannot restore a repo that hasn't been deleted")
}
repo, err = c.repoStore.Restore(ctx, repo, in.NewIdentifier)
if err != nil {
return nil, fmt.Errorf("failed to restore the repo: %w", err)
}
return repo, nil
}

75
app/api/controller/repo/soft_delete.go Normal file
View File

@ -0,0 +1,75 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package repo
import (
"context"
"fmt"
"time"
apiauth "github.com/harness/gitness/app/api/auth"
"github.com/harness/gitness/app/api/usererror"
"github.com/harness/gitness/app/auth"
"github.com/harness/gitness/types"
"github.com/harness/gitness/types/enum"
"github.com/rs/zerolog/log"
)
// SoftDelete soft deletes a repo (aka sets the deleted timestamp while keep the data).
func (c *Controller) SoftDelete(ctx context.Context, session *auth.Session, repoRef string) error {
// note: can't use c.getRepoCheckAccess because import job for repositories being imported must be cancelled.
repo, err := c.repoStore.FindByRef(ctx, repoRef)
if err != nil {
return fmt.Errorf("failed to find the repo for soft delete: %w", err)
}
if err = apiauth.CheckRepo(ctx, c.authorizer, session, repo, enum.PermissionRepoDelete, false); err != nil {
return fmt.Errorf("access check failed: %w", err)
}
log.Ctx(ctx).Info().
Int64("repo.id", repo.ID).
Str("repo.path", repo.Path).
Msg("soft deleting repository")
if repo.Deleted != nil {
return usererror.BadRequest("repository has been already deleted")
}
if repo.Importing {
log.Ctx(ctx).Info().Msg("repository is importing. cancelling the import job and purge the repo.")
err = c.importer.Cancel(ctx, repo)
if err != nil {
return fmt.Errorf("failed to cancel repository import")
}
return c.PurgeNoAuth(ctx, session, repo)
}
return c.SoftDeleteNoAuth(ctx, repo, time.Now().UnixMilli())
}
func (c *Controller) SoftDeleteNoAuth(
ctx context.Context,
repo *types.Repository,
deletedAt int64,
) error {
err := c.repoStore.SoftDelete(ctx, repo, &deletedAt)
if err != nil {
return fmt.Errorf("failed to soft delete repo from db: %w", err)
}
return nil
}

14
app/api/controller/space/delete.go
View File

@ -18,6 +18,7 @@ import (
"context"
"fmt"
"math"
"time"
apiauth "github.com/harness/gitness/app/api/auth"
"github.com/harness/gitness/app/auth"
@ -78,13 +79,24 @@ func (c *Controller) deleteRepositoriesNoAuth(ctx context.Context, session *auth
Query: "",
Order: enum.OrderAsc,
Sort: enum.RepoAttrNone,
DeletedBefore: nil,
}
repos, _, err := c.ListRepositoriesNoAuth(ctx, spaceID, filter)
if err != nil {
return fmt.Errorf("failed to list space repositories: %w", err)
}
// TEMPORARY until we support space delete/restore CODE-1413
recent := time.Now().Add(+time.Hour * 24).UnixMilli()
filter.DeletedBefore = &recent
alreadyDeletedRepos, _, err := c.ListRepositoriesNoAuth(ctx, spaceID, filter)
if err != nil {
return fmt.Errorf("failed to list delete repositories for space %d: %w", spaceID, err)
}
repos = append(repos, alreadyDeletedRepos...)
for _, repo := range repos {
err = c.repoCtrl.DeleteNoAuth(ctx, session, repo)
err = c.repoCtrl.PurgeNoAuth(ctx, session, repo)
if err != nil {
return fmt.Errorf("failed to delete repository %d: %w", repo.ID, err)
}

51
app/api/handler/repo/purge.go Normal file
View File

@ -0,0 +1,51 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package repo
import (
"net/http"
"github.com/harness/gitness/app/api/controller/repo"
"github.com/harness/gitness/app/api/render"
"github.com/harness/gitness/app/api/request"
)
func HandlePurge(repoCtrl *repo.Controller) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
session, _ := request.AuthSessionFrom(ctx)
repoRef, err := request.GetRepoRefFromPath(r)
if err != nil {
render.TranslatedUserError(w, err)
return
}
deletedAt, err := request.GetRepoDeletedAtFromQuery(r)
if err != nil {
render.TranslatedUserError(w, err)
return
}
err = repoCtrl.Purge(ctx, session, repoRef, deletedAt)
if err != nil {
render.TranslatedUserError(w, err)
return
}
render.DeleteSuccessful(w)
}
}

53
app/api/handler/repo/restore.go Normal file
View File

@ -0,0 +1,53 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package repo
import (
"encoding/json"
"net/http"
"github.com/harness/gitness/app/api/controller/repo"
"github.com/harness/gitness/app/api/render"
"github.com/harness/gitness/app/api/request"
)
func HandleRestore(repoCtrl *repo.Controller) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
session, _ := request.AuthSessionFrom(ctx)
repoRef, err := request.GetRepoRefFromPath(r)
if err != nil {
render.TranslatedUserError(w, err)
return
}
in := new(repo.RestoreInput)
err = json.NewDecoder(r.Body).Decode(in)
if err != nil {
render.BadRequestf(w, "Invalid request body: %s.", err)
return
}
repo, err := repoCtrl.Restore(ctx, session, repoRef, in)
if err != nil {
render.TranslatedUserError(w, err)
return
}
render.JSON(w, http.StatusOK, repo)
}
}

8
app/api/handler/repo/delete.go → app/api/handler/repo/soft_delete.go
View File

@ -23,19 +23,21 @@ import (
)
/*
* Deletes a repository.
* Soft Deletes a repository.
*/
func HandleDelete(repoCtrl *repo.Controller) http.HandlerFunc {
func HandleSoftDelete(repoCtrl *repo.Controller) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
session, _ := request.AuthSessionFrom(ctx)
repoRef, err := request.GetRepoRefFromPath(r)
if err != nil {
render.TranslatedUserError(w, err)
return
}
err = repoCtrl.Delete(ctx, session, repoRef)
err = repoCtrl.SoftDelete(ctx, session, repoRef)
if err != nil {
render.TranslatedUserError(w, err)
return

43
app/api/openapi/repo.go
View File

@ -194,6 +194,11 @@ type rule struct {
Pattern protection.Pattern `json:"pattern"`
}
type restoreRequest struct {
repoRequest
repo.RestoreInput
}
var queryParameterGitRef = openapi3.ParameterOrRef{
Parameter: &openapi3.Parameter{
Name: request.QueryParamGitRef,
@ -453,6 +458,20 @@ var queryParameterBypassRules = openapi3.ParameterOrRef{
},
}
var queryParameterRepoDeletedAt = openapi3.ParameterOrRef{
Parameter: &openapi3.Parameter{
Name: request.QueryParamRepoDeletedAt,
In: openapi3.ParameterInQuery,
Description: ptr.String("The time repository was deleted at in epoch format."),
Required: ptr.Bool(true),
Schema: &openapi3.SchemaOrRef{
Schema: &openapi3.Schema{
Type: ptrSchemaType(openapi3.SchemaTypeInteger),
},
},
},
}
//nolint:funlen
func repoOperations(reflector *openapi3.Reflector) {
createRepository := openapi3.Operation{}
@ -513,6 +532,30 @@ func repoOperations(reflector *openapi3.Reflector) {
_ = reflector.SetJSONResponse(&opDelete, new(usererror.Error), http.StatusNotFound)
_ = reflector.Spec.AddOperation(http.MethodDelete, "/repos/{repo_ref}", opDelete)
opPurge := openapi3.Operation{}
opPurge.WithTags("repository")
opPurge.WithMapOfAnything(map[string]interface{}{"operationId": "purgeRepository"})
opPurge.WithParameters(queryParameterRepoDeletedAt)
_ = reflector.SetRequest(&opPurge, new(repoRequest), http.MethodPost)
_ = reflector.SetJSONResponse(&opPurge, nil, http.StatusNoContent)
_ = reflector.SetJSONResponse(&opPurge, new(usererror.Error), http.StatusInternalServerError)
_ = reflector.SetJSONResponse(&opPurge, new(usererror.Error), http.StatusUnauthorized)
_ = reflector.SetJSONResponse(&opPurge, new(usererror.Error), http.StatusForbidden)
_ = reflector.SetJSONResponse(&opPurge, new(usererror.Error), http.StatusNotFound)
_ = reflector.Spec.AddOperation(http.MethodPost, "/repos/{repo_ref}/purge", opPurge)
opRestore := openapi3.Operation{}
opRestore.WithTags("repository")
opRestore.WithMapOfAnything(map[string]interface{}{"operationId": "restoreRepository"})
_ = reflector.SetRequest(&opRestore, new(restoreRequest), http.MethodPost)
_ = reflector.SetJSONResponse(&opRestore, new(types.Repository), http.StatusOK)
_ = reflector.SetJSONResponse(&opRestore, new(usererror.Error), http.StatusBadRequest)
_ = reflector.SetJSONResponse(&opRestore, new(usererror.Error), http.StatusInternalServerError)
_ = reflector.SetJSONResponse(&opRestore, new(usererror.Error), http.StatusUnauthorized)
_ = reflector.SetJSONResponse(&opRestore, new(usererror.Error), http.StatusForbidden)
_ = reflector.SetJSONResponse(&opRestore, new(usererror.Error), http.StatusNotFound)
_ = reflector.Spec.AddOperation(http.MethodPost, "/repos/{repo_ref}/restore", opRestore)
opMove := openapi3.Operation{}
opMove.WithTags("repository")
opMove.WithMapOfAnything(map[string]interface{}{"operationId": "moveRepository"})

6
app/api/request/repo.go
View File

@ -25,6 +25,7 @@ import (
const (
PathParamRepoRef = "repo_ref"
QueryParamRepoID = "repo_id"
QueryParamRepoDeletedAt = "repo_deleted_at"
)
func GetRepoRefFromPath(r *http.Request) (string, error) {
@ -54,3 +55,8 @@ func ParseRepoFilter(r *http.Request) *types.RepoFilter {
Size: ParseLimit(r),
}
}
// GetRepoDeletedAtFromQuery extracts the repository deleted timestamp from the query.
func GetRepoDeletedAtFromQuery(r *http.Request) (int64, error) {
return QueryParamAsPositiveInt64(r, QueryParamRepoDeletedAt)
}

4
app/router/api.go
View File

@ -260,7 +260,9 @@ func setupRepos(r chi.Router,
// repo level operations
r.Get("/", handlerrepo.HandleFind(repoCtrl))
r.Patch("/", handlerrepo.HandleUpdate(repoCtrl))
r.Delete("/", handlerrepo.HandleDelete(repoCtrl))
r.Delete("/", handlerrepo.HandleSoftDelete(repoCtrl))
r.Post("/purge", handlerrepo.HandlePurge(repoCtrl))
r.Post("/restore", handlerrepo.HandleRestore(repoCtrl))
r.Post("/move", handlerrepo.HandleMove(repoCtrl))
r.Get("/service-accounts", handlerrepo.HandleListServiceAccounts(repoCtrl))

102
app/services/cleanup/deleted_repos.go Normal file
View File

@ -0,0 +1,102 @@
// Copyright 2023 Harness, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package cleanup
import (
"context"
"fmt"
"math"
"time"
"github.com/harness/gitness/app/api/controller/repo"
"github.com/harness/gitness/app/bootstrap"
"github.com/harness/gitness/app/store"
"github.com/harness/gitness/job"
"github.com/harness/gitness/types"
"github.com/harness/gitness/types/enum"
"github.com/rs/zerolog/log"
)
const (
jobTypeDeletedRepos = "gitness:cleanup:deleted-repos"
jobCronDeletedRepos = "50 0 * * *" // At minute 50 past midnight every day.
jobMaxDurationDeletedRepos = 10 * time.Minute
)
type deletedReposCleanupJob struct {
retentionTime time.Duration
repoStore store.RepoStore
repoCtrl *repo.Controller
}
func newDeletedReposCleanupJob(
retentionTime time.Duration,
repoStore store.RepoStore,
repoCtrl *repo.Controller,
) *deletedReposCleanupJob {
return &deletedReposCleanupJob{
retentionTime: retentionTime,
repoStore: repoStore,
repoCtrl: repoCtrl,
}
}
// Handle purges old deleted repositories that are past the retention time.
func (j *deletedReposCleanupJob) Handle(ctx context.Context, _ string, _ job.ProgressReporter) (string, error) {
olderThan := time.Now().Add(-j.retentionTime)
log.Ctx(ctx).Info().Msgf(
"start purging deleted repositories older than %s (aka created before %s)",
j.retentionTime,
olderThan.Format(time.RFC3339Nano))
deletedBefore := olderThan.UnixMilli()
filter := &types.RepoFilter{
Page: 1,
Size: int(math.MaxInt),
Query: "",
Order: enum.OrderAsc,
Sort: enum.RepoAttrDeleted,
DeletedBefore: &deletedBefore,
}
toBePurgedRepos, err := j.repoStore.List(ctx, 0, filter)
if err != nil {
return "", fmt.Errorf("failed to list ready-to-delete repositories: %w", err)
}
session := bootstrap.NewSystemServiceSession()
purgedRepos := 0
for _, r := range toBePurgedRepos {
err := j.repoCtrl.PurgeNoAuth(ctx, session, r)
if err != nil {
log.Warn().Err(err).Msgf("failed to purge repo uid: %s, path: %s, deleted at %d",
r.Identifier, r.Path, *r.Deleted)
continue
}
purgedRepos++
}
result := "no old deleted repositories found"
if purgedRepos > 0 {
result = fmt.Sprintf("purged %d deleted repositories", purgedRepos)
}
log.Ctx(ctx).Info().Msg(result)
return result, nil
}

32
app/services/cleanup/service.go
View File

@ -20,12 +20,14 @@ import (
"fmt"
"time"
"github.com/harness/gitness/app/api/controller/repo"
"github.com/harness/gitness/app/store"
"github.com/harness/gitness/job"
)
type Config struct {
WebhookExecutionsRetentionTime time.Duration
DeletedRepositoriesRetentionTime time.Duration
}
func (c *Config) Prepare() error {
@ -35,6 +37,10 @@ func (c *Config) Prepare() error {
if c.WebhookExecutionsRetentionTime <= 0 {
return errors.New("config.WebhookExecutionsRetentionTime has to be provided")
}
if c.DeletedRepositoriesRetentionTime <= 0 {
return errors.New("config.DeletedRepositoriesRetentionTime has to be provided")
}
return nil
}
@ -45,6 +51,8 @@ type Service struct {
executor *job.Executor
webhookExecutionStore store.WebhookExecutionStore
tokenStore store.TokenStore
repoStore store.RepoStore
repoCtrl *repo.Controller
}
func NewService(
@ -53,6 +61,8 @@ func NewService(
executor *job.Executor,
webhookExecutionStore store.WebhookExecutionStore,
tokenStore store.TokenStore,
repoStore store.RepoStore,
repoCtrl *repo.Controller,
) (*Service, error) {
if err := config.Prepare(); err != nil {
return nil, fmt.Errorf("provided cleanup config is invalid: %w", err)
@ -65,6 +75,8 @@ func NewService(
executor: executor,
webhookExecutionStore: webhookExecutionStore,
tokenStore: tokenStore,
repoStore: repoStore,
repoCtrl: repoCtrl,
}, nil
}
@ -104,6 +116,16 @@ func (s *Service) scheduleRecurringCleanupJobs(ctx context.Context) error {
return fmt.Errorf("failed to schedule token job: %w", err)
}
err = s.scheduler.AddRecurring(
ctx,
jobTypeDeletedRepos,
jobTypeDeletedRepos,
jobCronDeletedRepos,
jobMaxDurationDeletedRepos,
)
if err != nil {
return fmt.Errorf("failed to schedule deleted repo cleanup job: %w", err)
}
return nil
}
@ -128,5 +150,15 @@ func (s *Service) registerJobHandlers() error {
return fmt.Errorf("failed to register job handler for token cleanup: %w", err)
}
if err := s.executor.Register(
jobTypeDeletedRepos,
newDeletedReposCleanupJob(
s.config.DeletedRepositoriesRetentionTime,
s.repoStore,
s.repoCtrl,
),
); err != nil {
return fmt.Errorf("failed to register job handler for deleted repos cleanup: %w", err)
}
return nil
}

5
app/services/cleanup/wire.go
View File

@ -15,6 +15,7 @@
package cleanup
import (
"github.com/harness/gitness/app/api/controller/repo"
"github.com/harness/gitness/app/store"
"github.com/harness/gitness/job"
@ -31,6 +32,8 @@ func ProvideService(
executor *job.Executor,
webhookExecutionStore store.WebhookExecutionStore,
tokenStore store.TokenStore,
repoStore store.RepoStore,
repoCtrl *repo.Controller,
) (*Service, error) {
return NewService(
config,
@ -38,5 +41,7 @@ func ProvideService(
executor,
webhookExecutionStore,
tokenStore,
repoStore,
repoCtrl,
)
}

27
app/store/database.go
View File

@ -184,6 +184,9 @@ type (
// Find the repo by id.
Find(ctx context.Context, id int64) (*types.Repository, error)
// FindByRefAndDeletedAt finds the repo using the repoRef and deleted timestamp.
FindByRefAndDeletedAt(ctx context.Context, repoRef string, deletedAt int64) (*types.Repository, error)
// FindByRef finds the repo using the repoRef as either the id or the repo path.
FindByRef(ctx context.Context, repoRef string) (*types.Repository, error)
@ -194,28 +197,36 @@ type (
Update(ctx context.Context, repo *types.Repository) error
// Update the repo size.
UpdateSize(ctx context.Context, repoID int64, repoSize int64) error
UpdateSize(ctx context.Context, id int64, repoSize int64) error
// Get the repo size.
GetSize(ctx context.Context, repoID int64) (int64, error)
GetSize(ctx context.Context, id int64) (int64, error)
// UpdateOptLock the repo details using the optimistic locking mechanism.
UpdateOptLock(ctx context.Context, repo *types.Repository,
mutateFn func(repository *types.Repository) error) (*types.Repository, error)
// Delete the repo.
Delete(ctx context.Context, id int64) error
// SoftDelete a repo.
SoftDelete(ctx context.Context, repo *types.Repository, deletedAt *int64) error
// Count of repos in a space.
// Purge the soft deleted repo permanently.
Purge(ctx context.Context, id int64, deletedAt *int64) error
// Restore a deleted repo using the optimistic locking mechanism.
Restore(ctx context.Context, repo *types.Repository,
newIdentifier string) (*types.Repository, error)
// Count of active repos in a space. With "DeletedBefore" filter, counts only deleted repos by opts.DeletedBefore.
Count(ctx context.Context, parentID int64, opts *types.RepoFilter) (int64, error)
// Count all repos in a hierarchy of spaces.
// Count all active repos in a hierarchy of spaces.
CountAll(ctx context.Context, spaceID int64) (int64, error)
// List returns a list of repos in a space.
// List returns a list of active repos in a space.
// With "DeletedBefore" filter, shows deleted repos by opts.DeletedBefore.
List(ctx context.Context, parentID int64, opts *types.RepoFilter) ([]*types.Repository, error)
// ListSizeInfos returns a list of all repo sizes.
// ListSizeInfos returns a list of all active repo sizes.
ListSizeInfos(ctx context.Context) ([]*types.RepositorySizeInfo, error)
}

7
app/store/database/migrate/postgres/0045_alter_table_repo_add_deletetimestamp.down.sql Normal file
View File

@ -0,0 +1,7 @@
ALTER TABLE repositories DROP COLUMN repo_deleted;
DROP INDEX repositories_parent_id_uid;
DROP INDEX repositories_deleted;
CREATE UNIQUE INDEX repositories_parent_id_uid
ON repositories(repo_parent_id, LOWER(repo_uid));

11
app/store/database/migrate/postgres/0045_alter_table_repo_add_deletetimestamp.up.sql Normal file
View File

@ -0,0 +1,11 @@
ALTER TABLE repositories ADD COLUMN repo_deleted BIGINT DEFAULT NULL;
DROP INDEX repositories_parent_id_uid;
CREATE UNIQUE INDEX repositories_parent_id_uid
ON repositories(repo_parent_id, LOWER(repo_uid));
WHERE repo_deleted IS NULL;
CREATE INDEX repositories_deleted
ON repositories(repo_deleted)
WHERE repo_deleted IS NOT NULL;

7
app/store/database/migrate/sqlite/0045_alter_table_repo_add_deletetimestamp.down.sql Normal file
View File

@ -0,0 +1,7 @@
ALTER TABLE repositories DROP COLUMN repo_deleted;
DROP INDEX repositories_parent_id_uid;
DROP INDEX repositories_deleted;
CREATE UNIQUE INDEX repositories_parent_id_uid
ON repositories(repo_parent_id, LOWER(repo_uid));

11
app/store/database/migrate/sqlite/0045_alter_table_repo_add_deletetimestamp.up.sql Normal file
View File

@ -0,0 +1,11 @@
ALTER TABLE repositories ADD COLUMN repo_deleted BIGINT DEFAULT NULL;
DROP INDEX repositories_parent_id_uid;
CREATE UNIQUE INDEX repositories_parent_id_uid
ON repositories(repo_parent_id, LOWER(repo_uid))
WHERE repo_deleted IS NULL;
CREATE INDEX repositories_deleted
ON repositories(repo_deleted)
WHERE repo_deleted IS NOT NULL;

178
app/store/database/repo.go
View File

@ -29,6 +29,7 @@ import (
"github.com/harness/gitness/types"
"github.com/harness/gitness/types/enum"
"github.com/guregu/null"
"github.com/jmoiron/sqlx"
"github.com/pkg/errors"
)
@ -66,6 +67,7 @@ type repository struct {
CreatedBy int64 `db:"repo_created_by"`
Created int64 `db:"repo_created"`
Updated int64 `db:"repo_updated"`
Deleted null.Int `db:"repo_deleted"`
Size int64 `db:"repo_size"`
SizeUpdated int64 `db:"repo_size_updated"`
@ -95,6 +97,7 @@ const (
,repo_created_by
,repo_created
,repo_updated
,repo_deleted
,repo_size
,repo_size_updated
,repo_git_uid
@ -115,40 +118,53 @@ const (
// Find finds the repo by id.
func (s *RepoStore) Find(ctx context.Context, id int64) (*types.Repository, error) {
const sqlQuery = repoSelectBase + `
WHERE repo_id = $1`
return s.find(ctx, id, nil)
}
// find is a wrapper to find a repo by id w/o deleted timestamp.
func (s *RepoStore) find(ctx context.Context, id int64, deletedAt *int64) (*types.Repository, error) {
var sqlQuery = repoSelectBase + `
WHERE repo_id = $1 AND repo_deleted IS NULL`
if deletedAt != nil {
sqlQuery = repoSelectBase + `
WHERE repo_id = $1 AND repo_deleted = $2`
}
db := dbtx.GetAccessor(ctx, s.db)
dst := new(repository)
if err := db.GetContext(ctx, dst, sqlQuery, id); err != nil {
if err := db.GetContext(ctx, dst, sqlQuery, id, deletedAt); err != nil {
return nil, database.ProcessSQLErrorf(err, "Failed to find repo")
}
return s.mapToRepo(ctx, dst)
}
// Find finds the repo with the given identifier in the given space ID.
func (s *RepoStore) FindByIdentifier(
func (s *RepoStore) findByIdentifier(
ctx context.Context,
spaceID int64,
identifier string,
deletedAt *int64,
) (*types.Repository, error) {
const sqlQuery = repoSelectBase + `
WHERE repo_parent_id = $1 AND LOWER(repo_uid) = $2`
var sqlQuery = repoSelectBase + `
WHERE repo_parent_id = $1 AND LOWER(repo_uid) = $2 AND repo_deleted IS NULL`
if deletedAt != nil {
sqlQuery = repoSelectBase + `
WHERE repo_parent_id = $1 AND LOWER(repo_uid) = $2 AND repo_deleted = $3`
}
db := dbtx.GetAccessor(ctx, s.db)
dst := new(repository)
if err := db.GetContext(ctx, dst, sqlQuery, spaceID, strings.ToLower(identifier)); err != nil {
if err := db.GetContext(ctx, dst, sqlQuery, spaceID, strings.ToLower(identifier), deletedAt); err != nil {
return nil, database.ProcessSQLErrorf(err, "Failed to find repo")
}
return s.mapToRepo(ctx, dst)
}
// FindByRef finds the repo using the repoRef as either the id or the repo path.
func (s *RepoStore) FindByRef(ctx context.Context, repoRef string) (*types.Repository, error) {
func (s *RepoStore) findByRef(ctx context.Context, repoRef string, deletedAt *int64) (*types.Repository, error) {
// ASSUMPTION: digits only is not a valid repo path
id, err := strconv.ParseInt(repoRef, 10, 64)
if err != nil {
@ -161,10 +177,23 @@ func (s *RepoStore) FindByRef(ctx context.Context, repoRef string) (*types.Repos
return nil, fmt.Errorf("failed to get space path: %w", err)
}
return s.FindByIdentifier(ctx, pathObject.SpaceID, repoIdentifier)
return s.findByIdentifier(ctx, pathObject.SpaceID, repoIdentifier, deletedAt)
}
return s.find(ctx, id, deletedAt)
}
return s.Find(ctx, id)
// FindByRef finds the repo using the repoRef as either the id or the repo path.
func (s *RepoStore) FindByRef(ctx context.Context, repoRef string) (*types.Repository, error) {
return s.findByRef(ctx, repoRef, nil)
}
// FindByRefAndDeletedAt finds the repo using the repoRef and deleted timestamp.
func (s *RepoStore) FindByRefAndDeletedAt(
ctx context.Context,
repoRef string,
deletedAt int64,
) (*types.Repository, error) {
return s.findByRef(ctx, repoRef, &deletedAt)
}
// Create creates a new repository.
@ -179,6 +208,7 @@ func (s *RepoStore) Create(ctx context.Context, repo *types.Repository) error {
,repo_created_by
,repo_created
,repo_updated
,repo_deleted
,repo_size
,repo_size_updated
,repo_git_uid
@ -200,6 +230,7 @@ func (s *RepoStore) Create(ctx context.Context, repo *types.Repository) error {
,:repo_created_by
,:repo_created
,:repo_updated
,:repo_deleted
,:repo_size
,:repo_size_updated
,:repo_git_uid
@ -241,6 +272,7 @@ func (s *RepoStore) Update(ctx context.Context, repo *types.Repository) error {
SET
repo_version = :repo_version
,repo_updated = :repo_updated
,repo_deleted = :repo_deleted
,repo_parent_id = :repo_parent_id
,repo_uid = :repo_uid
,repo_git_uid = :repo_git_uid
@ -296,12 +328,12 @@ func (s *RepoStore) Update(ctx context.Context, repo *types.Repository) error {
}
// UpdateSize updates the size of a specific repository in the database.
func (s *RepoStore) UpdateSize(ctx context.Context, repoID int64, size int64) error {
func (s *RepoStore) UpdateSize(ctx context.Context, id int64, size int64) error {
stmt := database.Builder.
Update("repositories").
Set("repo_size", size).
Set("repo_size_updated", time.Now().UnixMilli()).
Where("repo_id = ?", repoID)
Where("repo_id = ? AND repo_deleted IS NULL", id)
sqlQuery, args, err := stmt.ToSql()
if err != nil {
@ -321,26 +353,62 @@ func (s *RepoStore) UpdateSize(ctx context.Context, repoID int64, size int64) er
}
if count == 0 {
return fmt.Errorf("repo %d size not updated: %w", repoID, gitness_store.ErrResourceNotFound)
return fmt.Errorf("repo %d size not updated: %w", id, gitness_store.ErrResourceNotFound)
}
return nil
}
// GetSize returns the repo size.
func (s *RepoStore) GetSize(ctx context.Context, repoID int64) (int64, error) {
query := "SELECT repo_size FROM repositories WHERE repo_id = $1;"
func (s *RepoStore) GetSize(ctx context.Context, id int64) (int64, error) {
query := "SELECT repo_size FROM repositories WHERE repo_id = $1 AND repo_deleted IS NULL;"
db := dbtx.GetAccessor(ctx, s.db)
var size int64
if err := db.GetContext(ctx, &size, query, repoID); err != nil {
if err := db.GetContext(ctx, &size, query, id); err != nil {
return 0, database.ProcessSQLErrorf(err, "failed to get repo size")
}
return size, nil
}
// UpdateOptLock updates the repository using the optimistic locking mechanism.
func (s *RepoStore) UpdateOptLock(ctx context.Context,
// UpdateOptLock updates the active repository using the optimistic locking mechanism.
func (s *RepoStore) UpdateOptLock(
ctx context.Context,
repo *types.Repository,
mutateFn func(repository *types.Repository) error,
) (*types.Repository, error) {
return s.updateOptLock(
ctx,
repo,
func(r *types.Repository) error {
if repo.Deleted != nil {
return gitness_store.ErrResourceNotFound
}
return mutateFn(r)
},
)
}
// UpdateDeletedOptLock updates a deleted repository using the optimistic locking mechanism.
func (s *RepoStore) updateDeletedOptLock(ctx context.Context,
repo *types.Repository,
mutateFn func(repository *types.Repository) error,
) (*types.Repository, error) {
return s.updateOptLock(
ctx,
repo,
func(r *types.Repository) error {
if repo.Deleted == nil {
return gitness_store.ErrResourceNotFound
}
return mutateFn(r)
},
)
}
// updateOptLock updates the repository using the optimistic locking mechanism.
func (s *RepoStore) updateOptLock(
ctx context.Context,
repo *types.Repository,
mutateFn func(repository *types.Repository) error,
) (*types.Repository, error) {
@ -360,29 +428,61 @@ func (s *RepoStore) UpdateOptLock(ctx context.Context,
return nil, err
}
repo, err = s.Find(ctx, repo.ID)
repo, err = s.find(ctx, repo.ID, repo.Deleted)
if err != nil {
return nil, err
}
}
}
// Delete the repository.
func (s *RepoStore) Delete(ctx context.Context, id int64) error {
// SoftDelete deletes a repo softly by setting the deleted timestamp.
func (s *RepoStore) SoftDelete(ctx context.Context, repo *types.Repository, deletedAt *int64) error {
_, err := s.UpdateOptLock(ctx, repo, func(r *types.Repository) error {
r.Deleted = deletedAt
return nil
})
if err != nil {
return fmt.Errorf("failed to soft delete repo: %w", err)
}
return nil
}
// Purge deletes the repo permanently.
func (s *RepoStore) Purge(ctx context.Context, id int64, deletedAt *int64) error {
const repoDelete = `
DELETE FROM repositories
WHERE repo_id = $1`
WHERE repo_id = $1 AND repo_deleted = $2`
db := dbtx.GetAccessor(ctx, s.db)
if _, err := db.ExecContext(ctx, repoDelete, id); err != nil {
if _, err := db.ExecContext(ctx, repoDelete, id, deletedAt); err != nil {
return database.ProcessSQLErrorf(err, "the delete query failed")
}
return nil
}
// Count of repos in a space. if parentID (space) is zero then it will count all repositories in the system.
// Restore restores a deleted repo.
func (s *RepoStore) Restore(
ctx context.Context,
repo *types.Repository,
newIdentifier string,
) (*types.Repository, error) {
repo, err := s.updateDeletedOptLock(ctx, repo, func(r *types.Repository) error {
r.Deleted = nil
if newIdentifier != "" {
r.Identifier = newIdentifier
}
return nil
})
if err != nil {
return nil, database.ProcessSQLErrorf(err, "failed to restore the repo")
}
return repo, nil
}
// Count of active repos in a space. if parentID (space) is zero then it will count all repositories in the system.
// With "DeletedBefore" filter, counts only deleted repos by opts.DeletedBefore.
func (s *RepoStore) Count(ctx context.Context, parentID int64, opts *types.RepoFilter) (int64, error) {
stmt := database.Builder.
Select("count(*)").
@ -396,6 +496,12 @@ func (s *RepoStore) Count(ctx context.Context, parentID int64, opts *types.RepoF
stmt = stmt.Where("LOWER(repo_uid) LIKE ?", fmt.Sprintf("%%%s%%", strings.ToLower(opts.Query)))
}
if opts.DeletedBefore != nil {
stmt = stmt.Where("repo_deleted < ?", opts.DeletedBefore)
} else {
stmt = stmt.Where("repo_deleted IS NULL")
}
sql, args, err := stmt.ToSql()
if err != nil {
return 0, errors.Wrap(err, "Failed to convert query to sql")
@ -411,7 +517,7 @@ func (s *RepoStore) Count(ctx context.Context, parentID int64, opts *types.RepoF
return count, nil
}
// Count all repos in a hierarchy of spaces.
// Count all active repos in a hierarchy of spaces.
func (s *RepoStore) CountAll(ctx context.Context, spaceID int64) (int64, error) {
query := `WITH RECURSIVE SpaceHierarchy AS (
SELECT space_id, space_parent_id
@ -435,7 +541,7 @@ FROM SpaceHierarchy h1;`
}
query = fmt.Sprintf(
"SELECT COUNT(repo_id) FROM repositories WHERE repo_parent_id IN (%s);",
"SELECT COUNT(repo_id) FROM repositories WHERE repo_parent_id IN (%s) AND repo_deleted IS NULL;",
intsToCSV(spaceIDs),
)
@ -447,13 +553,20 @@ FROM SpaceHierarchy h1;`
return numRepos, nil
}
// List returns a list of repos in a space.
// List returns a list of active repos in a space.
// With "DeletedBefore" filter, shows deleted repos by opts.DeletedBefore.
func (s *RepoStore) List(ctx context.Context, parentID int64, opts *types.RepoFilter) ([]*types.Repository, error) {
stmt := database.Builder.
Select(repoColumnsForJoin).
From("repositories").
Where("repo_parent_id = ?", fmt.Sprint(parentID))
if opts.DeletedBefore != nil {
stmt = stmt.Where("repo_deleted < ?", opts.DeletedBefore)
} else {
stmt = stmt.Where("repo_deleted IS NULL")
}
if opts.Query != "" {
stmt = stmt.Where("LOWER(repo_uid) LIKE ?", fmt.Sprintf("%%%s%%", strings.ToLower(opts.Query)))
}
@ -472,6 +585,8 @@ func (s *RepoStore) List(ctx context.Context, parentID int64, opts *types.RepoFi
stmt = stmt.OrderBy("repo_created " + opts.Order.String())
case enum.RepoAttrUpdated:
stmt = stmt.OrderBy("repo_updated " + opts.Order.String())
case enum.RepoAttrDeleted:
stmt = stmt.OrderBy("repo_deleted " + opts.Order.String())
}
sql, args, err := stmt.ToSql()
@ -499,7 +614,8 @@ type repoSize struct {
func (s *RepoStore) ListSizeInfos(ctx context.Context) ([]*types.RepositorySizeInfo, error) {
stmt := database.Builder.
Select("repo_id", "repo_git_uid", "repo_size", "repo_size_updated").
From("repositories")
From("repositories").
Where("repo_deleted IS NULL")
sql, args, err := stmt.ToSql()
if err != nil {
@ -531,6 +647,7 @@ func (s *RepoStore) mapToRepo(
Created: in.Created,
CreatedBy: in.CreatedBy,
Updated: in.Updated,
Deleted: in.Deleted.Ptr(),
Size: in.Size,
SizeUpdated: in.SizeUpdated,
GitUID: in.GitUID,
@ -609,6 +726,7 @@ func mapToInternalRepo(in *types.Repository) *repository {
Created: in.Created,
CreatedBy: in.CreatedBy,
Updated: in.Updated,
Deleted: null.IntFromPtr(in.Deleted),
Size: in.Size,
SizeUpdated: in.SizeUpdated,
GitUID: in.GitUID,

1
cli/operations/server/config.go
View File

@ -344,6 +344,7 @@ func ProvidePubsubConfig(config *types.Config) pubsub.Config {
func ProvideCleanupConfig(config *types.Config) cleanup.Config {
return cleanup.Config{
WebhookExecutionsRetentionTime: config.Webhook.RetentionTime,
DeletedRepositoriesRetentionTime: config.Repos.DeletedRetentionTime,
}
}

4
cmd/gitness/wire.go
View File

@ -143,11 +143,11 @@ func initSystem(ctx context.Context, config *types.Config) (*cliserver.System, e
lock.WireSet,
cliserver.ProvidePubsubConfig,
pubsub.WireSet,
cliserver.ProvideJobsConfig,
job.WireSet,
cliserver.ProvideCleanupConfig,
cleanup.WireSet,
codecomments.WireSet,
cliserver.ProvideJobsConfig,
job.WireSet,
protection.WireSet,
checkcontroller.WireSet,
execution.WireSet,

2
cmd/gitness/wire_gen.go
View File

@ -304,7 +304,7 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro
return nil, err
}
cleanupConfig := server.ProvideCleanupConfig(config)
cleanupService, err := cleanup.ProvideService(cleanupConfig, jobScheduler, executor, webhookExecutionStore, tokenStore)
cleanupService, err := cleanup.ProvideService(cleanupConfig, jobScheduler, executor, webhookExecutionStore, tokenStore, repoStore, repoController)
if err != nil {
return nil, err
}

2
git/repo.go
View File

@ -225,7 +225,7 @@ func (s *Service) DeleteRepositoryBestEffort(ctx context.Context, repoUID string
repoPath := getFullPathForRepo(s.reposRoot, repoUID)
tempPath := path.Join(s.reposGraveyard, repoUID)
// delete should not fail if repoGraveyard dir does not exist
// delete should not fail if repoGraveyard dir does not exist.
if _, err := os.Stat(s.reposGraveyard); os.IsNotExist(err) {
if errdir := os.MkdirAll(s.reposGraveyard, fileMode700); errdir != nil {
return fmt.Errorf("clean up dir '%s' doesn't exist and can't be created: %w", s.reposGraveyard, errdir)

5
types/config.go
View File

@ -348,4 +348,9 @@ type Config struct {
Concurrency int `envconfig:"GITNESS_KEYWORD_SEARCH_CONCURRENCY" default:"4"`
MaxRetries int `envconfig:"GITNESS_KEYWORD_SEARCH_MAX_RETRIES" default:"3"`
}
Repos struct {
// DeletedRetentionTime is the duration after which deleted repositories will be purged.
DeletedRetentionTime time.Duration `envconfig:"GITNESS_REPOS_DELETED_RETENTION_TIME" default:"2160h"` // 90 days
}
}

2
types/enum/common.go
View File

@ -48,6 +48,8 @@ const (
updated = "updated"
updatedAt = "updated_at"
updatedBy = "updated_by"
deleted = "deleted"
deletedAt = "deleted_at"
displayName = "display_name"
date = "date"
defaultString = "default"

5
types/enum/repo.go
View File

@ -29,6 +29,7 @@ const (
RepoAttrIdentifier
RepoAttrCreated
RepoAttrUpdated
RepoAttrDeleted
)
// ParseRepoAttr parses the repo attribute string
@ -44,6 +45,8 @@ func ParseRepoAttr(s string) RepoAttr {
return RepoAttrCreated
case updated, updatedAt:
return RepoAttrUpdated
case deleted, deletedAt:
return RepoAttrDeleted
default:
return RepoAttrNone
}
@ -61,6 +64,8 @@ func (a RepoAttr) String() string {
return created
case RepoAttrUpdated:
return updated
case RepoAttrDeleted:
return deleted
case RepoAttrNone:
return ""
default:

2
types/repo.go
View File

@ -33,6 +33,7 @@ type Repository struct {
CreatedBy int64 `json:"created_by"`
Created int64 `json:"created"`
Updated int64 `json:"updated"`
Deleted *int64 `json:"deleted,omitempty"`
Size int64 `json:"size"`
SizeUpdated int64 `json:"size_updated"`
@ -85,6 +86,7 @@ type RepoFilter struct {
Query string `json:"query"`
Sort enum.RepoAttr `json:"sort"`
Order enum.Order `json:"order"`
DeletedBefore *int64 `json:"deleted_before,omitempty"`
}
// RepositoryGitInfo holds git info for a repository.