From 4c8302845d3721a9b1e3f63228aaab85a63c5d4c Mon Sep 17 00:00:00 2001 From: Vistaar Juneja Date: Wed, 6 Sep 2023 14:24:01 +0100 Subject: [PATCH] add logic to run executions --- build/commit/gitness.go | 51 ++ build/commit/service.go | 24 + build/commit/wire.go | 22 + build/file/gitness.go | 64 +++ build/file/service.go | 34 ++ build/file/wire.go | 22 + build/manager/client.go | 157 ++++++ build/manager/convert.go | 277 ++++++++++ build/manager/manager.go | 408 ++++++++++++++ build/manager/setup.go | 101 ++++ build/manager/teardown.go | 148 +++++ build/manager/updater.go | 39 ++ build/manager/wire.go | 48 ++ build/runner/poller.go | 30 ++ build/runner/runner.go | 66 +++ build/runner/wire.go | 40 ++ build/scheduler/queue.go | 294 ++++++++++ build/scheduler/scheduler.go | 32 ++ build/scheduler/wire.go | 25 + build/triggerer/dag/dag.go | 137 +++++ build/triggerer/dag/dag_test.go | 209 ++++++++ build/triggerer/skip.go | 78 +++ build/triggerer/trigger.go | 506 ++++++++++++++++++ build/triggerer/wire.go | 33 ++ cli/server/server.go | 11 + cli/server/system.go | 5 +- cmd/gitness/wire.go | 12 + cmd/gitness/wire_gen.go | 42 +- go.mod | 29 +- go.sum | 84 ++- internal/api/auth/pipeline.go | 1 + .../api/controller/execution/controller.go | 8 + internal/api/controller/execution/create.go | 56 +- internal/api/controller/execution/find.go | 3 +- internal/api/controller/execution/update.go | 2 +- internal/api/controller/execution/wire.go | 8 +- internal/api/controller/logs/find.go | 25 +- internal/api/controller/logs/tail.go | 2 +- internal/api/handler/execution/create.go | 10 +- internal/api/handler/logs/find.go | 7 +- internal/api/openapi/pipeline.go | 22 +- internal/api/request/pipeline.go | 5 + internal/store/database.go | 31 +- internal/store/database/execution.go | 23 +- .../database/migrate/ci/ci_migrations.sql | 262 +-------- internal/store/database/pipeline.go | 2 + internal/store/database/pipeline_join.go | 4 + internal/store/database/secret.go | 22 + internal/store/database/stage.go | 206 ++++++- internal/store/database/stage_map.go | 3 + internal/store/database/step.go | 136 ++++- internal/store/logs/db.go | 3 +- internal/url/provider.go | 12 + types/config.go | 13 +- types/events.go | 12 + types/stage.go | 1 + types/stage_status.go | 19 + 57 files changed, 3549 insertions(+), 377 deletions(-) create mode 100644 build/commit/gitness.go create mode 100644 build/commit/service.go create mode 100644 build/commit/wire.go create mode 100644 build/file/gitness.go create mode 100644 build/file/service.go create mode 100644 build/file/wire.go create mode 100644 build/manager/client.go create mode 100644 build/manager/convert.go create mode 100644 build/manager/manager.go create mode 100644 build/manager/setup.go create mode 100644 build/manager/teardown.go create mode 100644 build/manager/updater.go create mode 100644 build/manager/wire.go create mode 100644 build/runner/poller.go create mode 100644 build/runner/runner.go create mode 100644 build/runner/wire.go create mode 100644 build/scheduler/queue.go create mode 100644 build/scheduler/scheduler.go create mode 100644 build/scheduler/wire.go create mode 100644 build/triggerer/dag/dag.go create mode 100644 build/triggerer/dag/dag_test.go create mode 100644 build/triggerer/skip.go create mode 100644 build/triggerer/trigger.go create mode 100644 build/triggerer/wire.go create mode 100644 types/events.go create mode 100644 types/stage_status.go diff --git a/build/commit/gitness.go b/build/commit/gitness.go new file mode 100644 index 000000000..819eed3ec --- /dev/null +++ b/build/commit/gitness.go @@ -0,0 +1,51 @@ +// Copyright 2022 Harness Inc. All rights reserved. +// Use of this source code is governed by the Polyform Free Trial License +// that can be found in the LICENSE.md file for this repository. + +package commit + +import ( + "context" + + "github.com/harness/gitness/gitrpc" + "github.com/harness/gitness/internal/api/controller" + "github.com/harness/gitness/types" +) + +type service struct { + gitRPCClient gitrpc.Interface +} + +func new(gitRPCClient gitrpc.Interface) CommitService { + return &service{gitRPCClient: gitRPCClient} +} + +// FindRef finds information about a commit in gitness for the git ref. +// This is using the branch only as the ref at the moment, can be changed +// when needed to take any ref (like sha, tag). +func (f *service) FindRef( + ctx context.Context, + repo *types.Repository, + branch string, +) (*types.Commit, error) { + readParams := gitrpc.ReadParams{ + RepoUID: repo.GitUID, + } + branchOutput, err := f.gitRPCClient.GetBranch(ctx, &gitrpc.GetBranchParams{ + ReadParams: readParams, + BranchName: branch, + }) + if err != nil { + return nil, err + } + commitOutput, err := f.gitRPCClient.GetCommit(ctx, &gitrpc.GetCommitParams{ + ReadParams: readParams, + SHA: branchOutput.Branch.Commit.SHA, + }) + if err != nil { + return nil, err + } + + // convert the RPC commit output to a types.Commit. + return controller.MapCommit(&commitOutput.Commit) +} diff --git a/build/commit/service.go b/build/commit/service.go new file mode 100644 index 000000000..fddc2c9ef --- /dev/null +++ b/build/commit/service.go @@ -0,0 +1,24 @@ +// Copyright 2022 Harness Inc. All rights reserved. +// Use of this source code is governed by the Polyform Free Trial License +// that can be found in the LICENSE.md file for this repository. + +package commit + +import ( + "context" + + "github.com/harness/gitness/types" +) + +type ( + // CommitService provides access to commit information via + // the SCM provider. Today, this is gitness but it can + // be extendible to any SCM provider. + // + // Arguments: + // repo: the repo to read content from + // ref: the ref to fetch the commit from, eg refs/heads/master + CommitService interface { + FindRef(ctx context.Context, repo *types.Repository, ref string) (*types.Commit, error) + } +) diff --git a/build/commit/wire.go b/build/commit/wire.go new file mode 100644 index 000000000..3a81fc325 --- /dev/null +++ b/build/commit/wire.go @@ -0,0 +1,22 @@ +// Copyright 2022 Harness Inc. All rights reserved. +// Use of this source code is governed by the Polyform Free Trial License +// that can be found in the LICENSE.md file for this repository. + +package commit + +import ( + "github.com/harness/gitness/gitrpc" + + "github.com/google/wire" +) + +// WireSet provides a wire set for this package. +var WireSet = wire.NewSet( + ProvideCommitService, +) + +// ProvideCommitService provides a service which can fetch commit +// information about a repository. +func ProvideCommitService(gitRPCClient gitrpc.Interface) CommitService { + return new(gitRPCClient) +} diff --git a/build/file/gitness.go b/build/file/gitness.go new file mode 100644 index 000000000..b6786732a --- /dev/null +++ b/build/file/gitness.go @@ -0,0 +1,64 @@ +// Copyright 2022 Harness Inc. All rights reserved. +// Use of this source code is governed by the Polyform Free Trial License +// that can be found in the LICENSE.md file for this repository. + +package file + +import ( + "context" + "fmt" + "io/ioutil" + + "github.com/harness/gitness/gitrpc" + "github.com/harness/gitness/types" +) + +type service struct { + gitRPCClient gitrpc.Interface +} + +func new(gitRPCClient gitrpc.Interface) FileService { + return &service{gitRPCClient: gitRPCClient} +} + +func (f *service) Get( + ctx context.Context, + repo *types.Repository, + path string, + ref string, +) (*File, error) { + readParams := gitrpc.ReadParams{ + RepoUID: repo.GitUID, + } + treeNodeOutput, err := f.gitRPCClient.GetTreeNode(ctx, &gitrpc.GetTreeNodeParams{ + ReadParams: readParams, + GitREF: ref, + Path: path, + IncludeLatestCommit: false, + }) + if err != nil { + return nil, err + } + // viewing Raw content is only supported for blob content + if treeNodeOutput.Node.Type != gitrpc.TreeNodeTypeBlob { + return nil, fmt.Errorf("path content is not of blob type: %s", treeNodeOutput.Node.Type) + } + + blobReader, err := f.gitRPCClient.GetBlob(ctx, &gitrpc.GetBlobParams{ + ReadParams: readParams, + SHA: treeNodeOutput.Node.SHA, + SizeLimit: 0, // no size limit, we stream whatever data there is + }) + if err != nil { + return nil, fmt.Errorf("failed to read blob from gitrpc: %w", err) + } + + buf, err := ioutil.ReadAll(blobReader.Content) + if err != nil { + return nil, err + } + + return &File{ + Data: buf, + }, nil +} diff --git a/build/file/service.go b/build/file/service.go new file mode 100644 index 000000000..dd248eac3 --- /dev/null +++ b/build/file/service.go @@ -0,0 +1,34 @@ +// Copyright 2022 Harness Inc. All rights reserved. +// Use of this source code is governed by the Polyform Free Trial License +// that can be found in the LICENSE.md file for this repository. + +package file + +import ( + "context" + + "github.com/harness/gitness/types" +) + +type ( + // File represents the raw file contents in the + // version control system. + File struct { + Data []byte + } + + // FileService provides access to contents of files in + // the SCM provider. Today, this is gitness but it should + // be extendible to any SCM provider. + // The plan is for all remote repos to be pointers inside gitness + // so a repo entry would always exist. If this changes, the interface + // can be updated. + // + // Arguments: + // repo: the repo to read content from + // path: path in the repo to read + // ref: git ref for the repository e.g. refs/heads/master + FileService interface { + Get(ctx context.Context, repo *types.Repository, path, ref string) (*File, error) + } +) diff --git a/build/file/wire.go b/build/file/wire.go new file mode 100644 index 000000000..c6754f1de --- /dev/null +++ b/build/file/wire.go @@ -0,0 +1,22 @@ +// Copyright 2022 Harness Inc. All rights reserved. +// Use of this source code is governed by the Polyform Free Trial License +// that can be found in the LICENSE.md file for this repository. + +package file + +import ( + "github.com/harness/gitness/gitrpc" + + "github.com/google/wire" +) + +// WireSet provides a wire set for this package. +var WireSet = wire.NewSet( + ProvideFileService, +) + +// ProvideFileService provides a service which can read file contents +// from a repository. +func ProvideFileService(gitRPCClient gitrpc.Interface) FileService { + return new(gitRPCClient) +} diff --git a/build/manager/client.go b/build/manager/client.go new file mode 100644 index 000000000..134e8b873 --- /dev/null +++ b/build/manager/client.go @@ -0,0 +1,157 @@ +// Copyright 2022 Harness Inc. All rights reserved. +// Use of this source code is governed by the Polyform Free Trial License +// that can be found in the LICENSE.md file for this repository. + +package manager + +import ( + "bytes" + "context" + "encoding/json" + "errors" + + "github.com/harness/gitness/types" + + "github.com/drone/drone-go/drone" + "github.com/drone/runner-go/client" +) + +type embedded struct { + config *types.Config + manager ExecutionManager +} + +var _ client.Client = (*embedded)(nil) + +func NewEmbeddedClient(manager ExecutionManager, config *types.Config) *embedded { + return &embedded{ + config: config, + manager: manager, + } +} + +// Join notifies the server the runner is joining the cluster. +// Since the runner is embedded, this can just return nil. +func (e *embedded) Join(ctx context.Context, machine string) error { + return nil +} + +// Leave notifies the server the runner is leaving the cluster. +// Since the runner is embedded, this can just return nil. +func (e *embedded) Leave(ctx context.Context, machine string) error { + return nil +} + +// Ping sends a ping message to the server to test connectivity. +// Since the runner is embedded, this can just return nil. +func (e *embedded) Ping(ctx context.Context, machine string) error { + return nil +} + +// Request requests the next available build stage for execution. +func (e *embedded) Request(ctx context.Context, args *client.Filter) (*drone.Stage, error) { + request := &Request{ + Kind: args.Kind, + Type: args.Type, + OS: args.OS, + Arch: args.Arch, + Variant: args.Variant, + Kernel: args.Kernel, + Labels: args.Labels, + } + stage, err := e.manager.Request(ctx, request) + if err != nil { + return nil, err + } + return convertToDroneStage(stage), nil +} + +// Accept accepts the build stage for execution. +func (e *embedded) Accept(ctx context.Context, s *drone.Stage) error { + stage, err := e.manager.Accept(ctx, s.ID, s.Machine) + *s = *convertToDroneStage(stage) + return err +} + +// Detail gets the build stage details for execution. +func (e *embedded) Detail(ctx context.Context, stage *drone.Stage) (*client.Context, error) { + details, err := e.manager.Details(ctx, stage.ID) + if err != nil { + return nil, err + } + return &client.Context{ + Build: convertToDroneBuild(details.Build), + Repo: convertToDroneRepo(details.Repo), + Stage: convertToDroneStage(details.Stage), + Secrets: convertToDroneSecrets(details.Secrets), + Config: convertToDroneFile(details.Config), + System: &drone.System{ + Proto: e.config.Server.HTTP.Proto, + Host: e.config.Server.HTTP.Network, + }, + }, nil +} + +// Update updates the build stage. +func (e *embedded) Update(ctx context.Context, stage *drone.Stage) error { + var err error + convertedStage := convertFromDroneStage(stage) + if stage.Status == types.StatusPending || stage.Status == types.StatusRunning { + err = e.manager.BeforeAll(ctx, convertedStage) + } else { + err = e.manager.AfterAll(ctx, convertedStage) + } + *stage = *convertToDroneStage(convertedStage) + return err +} + +// UpdateStep updates the build step. +func (e *embedded) UpdateStep(ctx context.Context, step *drone.Step) error { + var err error + convertedStep := convertFromDroneStep(step) + if step.Status == types.StatusPending || step.Status == types.StatusRunning { + err = e.manager.Before(ctx, convertedStep) + } else { + err = e.manager.After(ctx, convertedStep) + } + *step = *convertToDroneStep(convertedStep) + return err +} + +// Watch watches for build cancellation requests. +func (e *embedded) Watch(ctx context.Context, stage int64) (bool, error) { + // Implement Watch logic here + return false, errors.New("Not implemented") +} + +// Batch batch writes logs to the streaming logs. +func (e *embedded) Batch(ctx context.Context, step int64, lines []*drone.Line) error { + for _, l := range lines { + line := convertFromDroneLine(l) + err := e.manager.Write(ctx, step, line) + if err != nil { + return err + } + } + return nil +} + +// Upload uploads the full logs to the server. +func (e *embedded) Upload(ctx context.Context, step int64, lines []*drone.Line) error { + var buffer bytes.Buffer + out, err := json.Marshal(lines) + if err != nil { + return err + } + _, err = buffer.Write(out) + if err != nil { + return err + } + return e.manager.Upload(ctx, step, &buffer) +} + +// UploadCard uploads a card to drone server. +func (e *embedded) UploadCard(ctx context.Context, step int64, card *drone.CardInput) error { + // Implement UploadCard logic here + return nil // Replace with appropriate error handling and logic +} diff --git a/build/manager/convert.go b/build/manager/convert.go new file mode 100644 index 000000000..7ec3df249 --- /dev/null +++ b/build/manager/convert.go @@ -0,0 +1,277 @@ +// Copyright 2022 Harness Inc. All rights reserved. +// Use of this source code is governed by the Polyform Free Trial License +// that can be found in the LICENSE.md file for this repository. + +package manager + +import ( + "time" + + "github.com/harness/gitness/build/file" + "github.com/harness/gitness/livelog" + "github.com/harness/gitness/types" + + "github.com/drone/drone-go/drone" + "github.com/drone/runner-go/client" +) + +func convertToDroneStage(stage *types.Stage) *drone.Stage { + return &drone.Stage{ + ID: stage.ID, + BuildID: stage.ExecutionID, + Number: int(stage.Number), + Name: stage.Name, + Kind: stage.Kind, + Type: stage.Type, + Status: stage.Status, + Error: stage.Error, + ErrIgnore: stage.ErrIgnore, + ExitCode: stage.ExitCode, + Machine: stage.Machine, + OS: stage.OS, + Arch: stage.Arch, + Variant: stage.Variant, + Kernel: stage.Kernel, + Limit: stage.Limit, + LimitRepo: stage.LimitRepo, + Started: stage.Started, + Stopped: stage.Stopped, + Created: stage.Created, + Updated: stage.Updated, + Version: stage.Version, + OnSuccess: stage.OnSuccess, + OnFailure: stage.OnFailure, + DependsOn: stage.DependsOn, + Labels: stage.Labels, + Steps: convertToDroneSteps(stage.Steps), + } +} + +func convertToDroneSteps(steps []*types.Step) []*drone.Step { + droneSteps := make([]*drone.Step, len(steps)) + for i, step := range steps { + droneSteps[i] = convertToDroneStep(step) + } + return droneSteps +} + +func convertToDroneStep(step *types.Step) *drone.Step { + return &drone.Step{ + ID: step.ID, + StageID: step.StageID, + Number: int(step.Number), + Name: step.Name, + Status: step.Status, + Error: step.Error, + ErrIgnore: step.ErrIgnore, + ExitCode: step.ExitCode, + Started: step.Started, + Stopped: step.Stopped, + Version: step.Version, + DependsOn: step.DependsOn, + Image: step.Image, + Detached: step.Detached, + Schema: step.Schema, + } +} + +func convertFromDroneStep(step *drone.Step) *types.Step { + return &types.Step{ + ID: step.ID, + StageID: step.StageID, + Number: int64(step.Number), + Name: step.Name, + Status: step.Status, + Error: step.Error, + ErrIgnore: step.ErrIgnore, + ExitCode: step.ExitCode, + Started: step.Started, + Stopped: step.Stopped, + Version: step.Version, + DependsOn: step.DependsOn, + Image: step.Image, + Detached: step.Detached, + Schema: step.Schema, + } +} + +func convertFromDroneSteps(steps []*drone.Step) []*types.Step { + typesSteps := make([]*types.Step, len(steps)) + for i, step := range steps { + typesSteps[i] = &types.Step{ + ID: step.ID, + StageID: step.StageID, // Assuming StageID maps to step_id + Number: int64(step.Number), + Name: step.Name, + Status: step.Status, + Error: step.Error, + ErrIgnore: step.ErrIgnore, + ExitCode: step.ExitCode, + Started: step.Started, + Stopped: step.Stopped, + Version: step.Version, + DependsOn: step.DependsOn, + Image: step.Image, + Detached: step.Detached, + Schema: step.Schema, + } + } + return typesSteps +} + +func convertFromDroneStage(stage *drone.Stage) *types.Stage { + return &types.Stage{ + ID: stage.ID, + ExecutionID: stage.BuildID, + Number: int64(stage.Number), + Name: stage.Name, + Kind: stage.Kind, + Type: stage.Type, + Status: stage.Status, + Error: stage.Error, + ErrIgnore: stage.ErrIgnore, + ExitCode: stage.ExitCode, + Machine: stage.Machine, + OS: stage.OS, + Arch: stage.Arch, + Variant: stage.Variant, + Kernel: stage.Kernel, + Limit: stage.Limit, + LimitRepo: stage.LimitRepo, + Started: stage.Started, + Stopped: stage.Stopped, + Version: stage.Version, + OnSuccess: stage.OnSuccess, + OnFailure: stage.OnFailure, + DependsOn: stage.DependsOn, + Labels: stage.Labels, + Steps: convertFromDroneSteps(stage.Steps), + } +} + +func convertFromDroneLine(l *drone.Line) *livelog.Line { + return &livelog.Line{ + Number: l.Number, + Message: l.Message, + Timestamp: l.Timestamp, + } +} + +func convertToDroneBuild(execution *types.Execution) *drone.Build { + return &drone.Build{ + ID: execution.ID, + RepoID: execution.RepoID, + Trigger: execution.Trigger, + Number: execution.Number, + Parent: execution.Parent, + Status: execution.Status, + Error: execution.Error, + Event: execution.Event, + Action: execution.Action, + Link: execution.Link, + Timestamp: execution.Timestamp, + Title: execution.Title, + Message: execution.Message, + Before: execution.Before, + After: execution.After, + Ref: execution.Ref, + Fork: execution.Fork, + Source: execution.Source, + Target: execution.Target, + Author: execution.Author, + AuthorName: execution.AuthorName, + AuthorEmail: execution.AuthorEmail, + AuthorAvatar: execution.AuthorAvatar, + Sender: execution.Sender, + Params: execution.Params, + Cron: execution.Cron, + Deploy: execution.Deploy, + DeployID: execution.DeployID, + Debug: execution.Debug, + Started: execution.Started, + Finished: execution.Finished, + Created: execution.Created, + Updated: execution.Updated, + Version: execution.Version, + } +} + +func convertFromDroneBuild(build *drone.Build) *types.Execution { + return &types.Execution{ + ID: build.ID, + PipelineID: build.RepoID, + RepoID: build.RepoID, + Trigger: build.Trigger, + Number: build.Number, + Parent: build.Parent, + Status: build.Status, + Error: build.Error, + Event: build.Event, + Action: build.Action, + Link: build.Link, + Timestamp: build.Timestamp, + Title: build.Title, + Message: build.Message, + Before: build.Before, + After: build.After, + Ref: build.Ref, + Fork: build.Fork, + Source: build.Source, + Target: build.Target, + Author: build.Author, + AuthorName: build.AuthorName, + AuthorEmail: build.AuthorEmail, + AuthorAvatar: build.AuthorAvatar, + Sender: build.Sender, + Params: build.Params, + Cron: build.Cron, + Deploy: build.Deploy, + DeployID: build.DeployID, + Debug: build.Debug, + Started: build.Started, + Finished: build.Finished, + Created: build.Created, + Updated: build.Updated, + Version: build.Version, + } +} + +func convertToDroneRepo(repo *types.Repository) *drone.Repo { + return &drone.Repo{ + ID: repo.ID, + UID: repo.UID, + UserID: repo.CreatedBy, + Name: repo.UID, + HTTPURL: repo.GitURL, + Link: repo.GitURL, + Private: !repo.IsPublic, + Created: repo.Created, + Updated: repo.Updated, + Version: repo.Version, + Branch: repo.DefaultBranch, + // TODO: We can get this from configuration once we start populating it. + // If this is not set drone runner cancels the build. + Timeout: int64(time.Duration(10 * time.Hour).Seconds()), + } +} + +func convertToDroneFile(file *file.File) *client.File { + return &client.File{ + Data: file.Data, + } +} + +func convertToDroneSecret(secret *types.Secret) *drone.Secret { + return &drone.Secret{ + Name: secret.UID, + Data: secret.Data, + } +} + +func convertToDroneSecrets(secrets []*types.Secret) []*drone.Secret { + ret := make([]*drone.Secret, len(secrets)) + for i, s := range secrets { + ret[i] = convertToDroneSecret(s) + } + return ret +} diff --git a/build/manager/manager.go b/build/manager/manager.go new file mode 100644 index 000000000..041b5e2f2 --- /dev/null +++ b/build/manager/manager.go @@ -0,0 +1,408 @@ +// Copyright 2022 Harness Inc. All rights reserved. +// Use of this source code is governed by the Polyform Free Trial License +// that can be found in the LICENSE.md file for this repository. + +package manager + +import ( + "bytes" + "context" + "errors" + "fmt" + "io" + "net/url" + "time" + + "github.com/harness/gitness/build/file" + "github.com/harness/gitness/build/scheduler" + "github.com/harness/gitness/internal/store" + urlprovider "github.com/harness/gitness/internal/url" + "github.com/harness/gitness/livelog" + gitness_store "github.com/harness/gitness/store" + "github.com/harness/gitness/types" + + "github.com/hashicorp/go-multierror" + "github.com/rs/zerolog/log" +) + +var noContext = context.Background() + +var _ ExecutionManager = (*Manager)(nil) + +type ( + // Request provides filters when requesting a pending + // build from the queue. This allows an agent, for example, + // to request a build that matches its architecture and kernel. + Request struct { + Kind string `json:"kind"` + Type string `json:"type"` + OS string `json:"os"` + Arch string `json:"arch"` + Variant string `json:"variant"` + Kernel string `json:"kernel"` + Labels map[string]string `json:"labels,omitempty"` + } + + // Config represents a pipeline config file. + Config struct { + Data string `json:"data"` + Kind string `json:"kind"` + } + + // Context represents the minimum amount of information + // required by the runner to execute a build. + Context struct { + Repo *types.Repository `json:"repository"` + Build *types.Execution `json:"build"` + Stage *types.Stage `json:"stage"` + Secrets []*types.Secret `json:"secrets"` + Config *file.File `json:"config"` + } + + // ExecutionManager encapsulates complex build operations and provides + // a simplified interface for build runners. + ExecutionManager interface { + // Request requests the next available build stage for execution. + Request(ctx context.Context, args *Request) (*types.Stage, error) + + // Accept accepts the build stage for execution. + Accept(ctx context.Context, stage int64, machine string) (*types.Stage, error) + + // Write writes a line to the build logs. + Write(ctx context.Context, step int64, line *livelog.Line) error + + // Details returns details about stage. + Details(ctx context.Context, stageID int64) (*Context, error) + + // Upload uploads the full logs. + Upload(ctx context.Context, step int64, r io.Reader) error + + // UploadBytes uploads the full logs. + UploadBytes(ctx context.Context, step int64, b []byte) error + + // Before signals the build step is about to start. + Before(ctx context.Context, step *types.Step) error + + // After signals the build step is complete. + After(ctx context.Context, step *types.Step) error + + // BeforeAll signals the build stage is about to start. + BeforeAll(ctx context.Context, stage *types.Stage) error + + // AfterAll signals the build stage is complete. + AfterAll(ctx context.Context, stage *types.Stage) error + } +) + +// Manager provides a simplified interface to the build runner so that it +// can more easily interact with the server. +type Manager struct { + Executions store.ExecutionStore + Config *types.Config + FileService file.FileService + Pipelines store.PipelineStore + urlProvider *urlprovider.Provider + // Converter store.ConvertService + // Events store.Pubsub + // Globals store.GlobalSecretStore + Logs store.LogStore + Logz livelog.LogStream + // Netrcs store.NetrcService + Repos store.RepoStore + Scheduler scheduler.Scheduler + Secrets store.SecretStore + // Status store.StatusService + Stages store.StageStore + Steps store.StepStore + // System *store.System + Users store.PrincipalStore + // Webhook store.WebhookSender +} + +func New( + config *types.Config, + executionStore store.ExecutionStore, + pipelineStore store.PipelineStore, + urlProvider *urlprovider.Provider, + fileService file.FileService, + logStore store.LogStore, + logStream livelog.LogStream, + repoStore store.RepoStore, + scheduler scheduler.Scheduler, + secretStore store.SecretStore, + stageStore store.StageStore, + stepStore store.StepStore, + userStore store.PrincipalStore, +) *Manager { + return &Manager{ + Config: config, + Executions: executionStore, + Pipelines: pipelineStore, + urlProvider: urlProvider, + FileService: fileService, + Logs: logStore, + Logz: logStream, + Repos: repoStore, + Scheduler: scheduler, + Secrets: secretStore, + Stages: stageStore, + Steps: stepStore, + Users: userStore, + } +} + +// Request requests the next available build stage for execution. +func (m *Manager) Request(ctx context.Context, args *Request) (*types.Stage, error) { + log := log.With(). + Str("kind", args.Kind). + Str("type", args.Type). + Str("os", args.OS). + Str("arch", args.Arch). + Str("kernel", args.Kernel). + Str("variant", args.Variant). + Logger() + log.Debug().Msg("manager: request queue item") + + stage, err := m.Scheduler.Request(ctx, scheduler.Filter{ + Kind: args.Kind, + Type: args.Type, + OS: args.OS, + Arch: args.Arch, + Kernel: args.Kernel, + Variant: args.Variant, + Labels: args.Labels, + }) + if err != nil && ctx.Err() != nil { + log.Debug().Err(err).Msg("manager: context canceled") + return nil, err + } + if err != nil { + log.Warn().Err(err).Msg("manager: request queue item error") + return nil, err + } + return stage, nil +} + +// Accept accepts the build stage for execution. It is possible for multiple +// agents to pull the same stage from the queue. +func (m *Manager) Accept(ctx context.Context, id int64, machine string) (*types.Stage, error) { + log := log.With(). + Int64("stage-id", id). + Str("machine", machine). + Logger() + log.Debug().Msg("manager: accept stage") + + stage, err := m.Stages.Find(noContext, id) + if err != nil { + log.Warn().Err(err).Msg("manager: cannot find stage") + return nil, err + } + if stage.Machine != "" { + log.Debug().Msg("manager: stage already assigned. abort.") + return nil, fmt.Errorf("stage already assigned, abort") + } + + stage.Machine = machine + stage.Status = types.StatusPending + stage.Updated = time.Now().Unix() + err = m.Stages.Update(noContext, stage) + if errors.Is(err, gitness_store.ErrVersionConflict) { + log.Debug().Err(err).Msg("manager: stage processed by another agent") + } else if err != nil { + log.Debug().Err(err).Msg("manager: cannot update stage") + } else { + log.Debug().Err(err).Msg("manager: stage accepted") + } + return stage, err +} + +// Write writes a line to the build logs. +func (m *Manager) Write(ctx context.Context, step int64, line *livelog.Line) error { + fmt.Println("line is: ", line) + err := m.Logz.Write(ctx, step, line) + if err != nil { + log.Warn().Int64("step-id", step).Err(err).Msg("manager: cannot write to log stream") + return err + } + return nil +} + +// Upload uploads the full logs. +func (m *Manager) Upload(ctx context.Context, step int64, r io.Reader) error { + err := m.Logs.Create(ctx, step, r) + if err != nil { + log.Error().Err(err).Int64("step-id", step).Msg("manager: cannot upload complete logs") + return err + } + return nil +} + +// UploadBytes uploads the full logs. +func (m *Manager) UploadBytes(ctx context.Context, step int64, data []byte) error { + buf := bytes.NewBuffer(data) + err := m.Logs.Create(ctx, step, buf) + if err != nil { + log.Error().Err(err).Int64("step-id", step).Msg("manager: cannot upload complete logs") + return err + } + return nil +} + +// Details provides details about the stage. +func (m *Manager) Details(ctx context.Context, stageID int64) (*Context, error) { + log := log.With(). + Int64("stage-id", stageID). + Logger() + log.Debug().Msg("manager: fetching stage details") + + stage, err := m.Stages.Find(noContext, stageID) + if err != nil { + log.Warn().Err(err).Msg("manager: cannot find stage") + return nil, err + } + execution, err := m.Executions.Find(noContext, stage.ExecutionID) + if err != nil { + log.Warn().Err(err).Msg("manager: cannot find build") + return nil, err + } + pipeline, err := m.Pipelines.Find(noContext, execution.PipelineID) + if err != nil { + log.Warn().Err(err).Msg("manager: cannot find pipeline") + return nil, err + } + repo, err := m.Repos.Find(noContext, execution.RepoID) + if err != nil { + log.Warn().Err(err).Msg("manager: cannot find repo") + return nil, err + } + // Backfill clone URL + repo.GitURL, err = m.createCustomCloneURL(repo.Path) + if err != nil { + log.Warn().Err(err).Msg("manager: could not create custom clone url") + return nil, err + } + stages, err := m.Stages.List(ctx, stage.ExecutionID) + if err != nil { + log.Warn().Err(err).Msg("manager: cannot list stages") + return nil, err + } + execution.Stages = stages + log = log.With(). + Int64("build", execution.Number). + Str("repo", repo.GetGitUID()). + Logger() + + // TODO: Currently we fetch all the secrets from the same space. + // This logic can be updated when needed. + secrets, err := m.Secrets.ListAll(noContext, repo.ParentID) + if err != nil { + log.Warn().Err(err).Msg("manager: cannot list secrets") + return nil, err + } + + // Fetch contents of YAML from the execution ref at the pipeline config path. + file, err := m.FileService.Get(ctx, repo, pipeline.ConfigPath, execution.After) + if err != nil { + log.Warn().Err(err).Msg("manager: cannot fetch file") + return nil, err + } + + return &Context{ + Repo: repo, + Build: execution, + Stage: stage, + Secrets: secrets, + Config: file, + }, nil +} + +// Before signals the build step is about to start. +func (m *Manager) Before(ctx context.Context, step *types.Step) error { + log := log.With(). + Str("step.status", step.Status). + Str("step.name", step.Name). + Int64("step.id", step.ID). + Logger() + + log.Debug().Msg("manager: updating step status") + + err := m.Logz.Create(noContext, step.ID) + if err != nil { + log.Warn().Err(err).Msg("manager: cannot create log stream") + return err + } + updater := &updater{ + Executions: m.Executions, + Repos: m.Repos, + Steps: m.Steps, + Stages: m.Stages, + } + return updater.do(ctx, step) +} + +// After signals the build step is complete. +func (m *Manager) After(ctx context.Context, step *types.Step) error { + log := log.With(). + Str("step.status", step.Status). + Str("step.name", step.Name). + Int64("step.id", step.ID). + Logger() + log.Debug().Msg("manager: updating step status") + + var errs error + updater := &updater{ + Executions: m.Executions, + Repos: m.Repos, + Steps: m.Steps, + Stages: m.Stages, + } + + if err := updater.do(ctx, step); err != nil { + errs = multierror.Append(errs, err) + log.Warn().Err(errs).Msg("manager: cannot update step") + } + + if err := m.Logz.Delete(noContext, step.ID); err != nil { + log.Warn().Err(err).Msg("manager: cannot teardown log stream") + } + return errs +} + +// BeforeAll signals the build stage is about to start. +func (m *Manager) BeforeAll(ctx context.Context, stage *types.Stage) error { + s := &setup{ + Executions: m.Executions, + Repos: m.Repos, + Steps: m.Steps, + Stages: m.Stages, + Users: m.Users, + } + + err := s.do(ctx, stage) + return err +} + +// AfterAll signals the build stage is complete. +func (m *Manager) AfterAll(ctx context.Context, stage *types.Stage) error { + t := &teardown{ + Executions: m.Executions, + Logs: m.Logz, + Repos: m.Repos, + Scheduler: m.Scheduler, + Steps: m.Steps, + Stages: m.Stages, + } + return t.do(ctx, stage) +} + +// createCustomCloneURL creates an endpoint to interact with gitness +// using the network (if provided) that gitness is running on. +func (m *Manager) createCustomCloneURL(repoPath string) (string, error) { + // We use http to interact with gitness from the build containers. + endpoint := "http://" + m.Config.Server.HTTP.Network + m.Config.Server.HTTP.Bind + url, err := url.Parse(endpoint) + if err != nil { + return "", err + } + return m.urlProvider.GenerateCustomRepoCloneURL(url, repoPath), nil +} diff --git a/build/manager/setup.go b/build/manager/setup.go new file mode 100644 index 000000000..ef79f6cf0 --- /dev/null +++ b/build/manager/setup.go @@ -0,0 +1,101 @@ +// Copyright 2022 Harness Inc. All rights reserved. +// Use of this source code is governed by the Polyform Free Trial License +// that can be found in the LICENSE.md file for this repository. + +package manager + +import ( + "context" + "errors" + "time" + + "github.com/harness/gitness/internal/store" + gitness_store "github.com/harness/gitness/store" + "github.com/harness/gitness/types" + "github.com/rs/zerolog/log" +) + +type setup struct { + Executions store.ExecutionStore + Repos store.RepoStore + Steps store.StepStore + Stages store.StageStore + Users store.PrincipalStore +} + +func (s *setup) do(ctx context.Context, stage *types.Stage) error { + execution, err := s.Executions.Find(noContext, stage.ExecutionID) + if err != nil { + log.Error().Err(err).Msg("manager: cannot find the execution") + return err + } + + log := log.With(). + Int64("execution.number", execution.Number). + Int64("execution.id", execution.ID). + Int64("stage.id", stage.ID). + Int64("repo.id", execution.RepoID). + Logger() + + _, err = s.Repos.Find(noContext, execution.RepoID) + if err != nil { + log.Error().Err(err).Msg("manager: cannot find the repository") + return err + } + + if len(stage.Error) > 500 { + stage.Error = stage.Error[:500] + } + stage.Updated = time.Now().Unix() + err = s.Stages.Update(noContext, stage) + if err != nil { + log.Error().Err(err). + Str("stage.status", stage.Status). + Msg("manager: cannot update the stage") + return err + } + + // TODO: create all the steps as part of a single transaction? + for _, step := range stage.Steps { + if len(step.Error) > 500 { + step.Error = step.Error[:500] + } + err := s.Steps.Create(noContext, step) + if err != nil { + log.Error().Err(err). + Str("stage.status", stage.Status). + Str("step.name", step.Name). + Int64("step.id", step.ID). + Msg("manager: cannot persist the step") + return err + } + } + + _, err = s.updateExecution(ctx, execution) + if err != nil { + log.Error().Err(err).Msg("manager: cannot update the execution") + return err + } + + return nil +} + +// helper function that updates the execution status from pending to running. +// This accounts for the fact that another agent may have already updated +// the execution status, which may happen if two stages execute concurrently. +func (s *setup) updateExecution(ctx context.Context, execution *types.Execution) (bool, error) { + if execution.Status != types.StatusPending { + return false, nil + } + execution.Started = time.Now().Unix() + execution.Updated = time.Now().Unix() + execution.Status = types.StatusRunning + err := s.Executions.Update(noContext, execution) + if errors.Is(err, gitness_store.ErrVersionConflict) { + return false, nil + } + if err != nil { + return false, err + } + return true, nil +} diff --git a/build/manager/teardown.go b/build/manager/teardown.go new file mode 100644 index 000000000..4e2f36cd6 --- /dev/null +++ b/build/manager/teardown.go @@ -0,0 +1,148 @@ +// Copyright 2022 Harness Inc. All rights reserved. +// Use of this source code is governed by the Polyform Free Trial License +// that can be found in the LICENSE.md file for this repository. + +package manager + +import ( + "context" + "time" + + "github.com/harness/gitness/build/scheduler" + "github.com/harness/gitness/internal/store" + "github.com/harness/gitness/livelog" + gitness_store "github.com/harness/gitness/store" + "github.com/harness/gitness/types" + "github.com/rs/zerolog/log" +) + +type teardown struct { + Executions store.ExecutionStore + Logs livelog.LogStream + Scheduler scheduler.Scheduler + Repos store.RepoStore + Steps store.StepStore + Stages store.StageStore +} + +func (t *teardown) do(ctx context.Context, stage *types.Stage) error { + log := log.With(). + Int64("stage.id", stage.ID). + Logger() + log.Debug().Msg("manager: stage is complete. teardown") + + execution, err := t.Executions.Find(noContext, stage.ExecutionID) + if err != nil { + log.Error().Err(err).Msg("manager: cannot find the execution") + return err + } + + log = log.With(). + Int64("execution.number", execution.Number). + Int64("execution.id", execution.ID). + Int64("repo.id", execution.RepoID). + Str("stage.status", stage.Status). + Logger() + + _, err = t.Repos.Find(noContext, execution.RepoID) + if err != nil { + log.Error().Err(err).Msg("manager: cannot find the repository") + return err + } + + for _, step := range stage.Steps { + if len(step.Error) > 500 { + step.Error = step.Error[:500] + } + err := t.Steps.Update(noContext, step) + if err != nil { + log = log.With(). + Str("step.name", step.Name). + Int64("step.id", step.ID). + Err(err). + Logger() + + log.Error().Msg("manager: cannot persist the step") + return err + } + } + + if len(stage.Error) > 500 { + stage.Error = stage.Error[:500] + } + + stage.Updated = time.Now().Unix() + err = t.Stages.Update(noContext, stage) + if err != nil { + log.Error().Err(err). + Msg("manager: cannot update the stage") + return err + } + + for _, step := range stage.Steps { + t.Logs.Delete(noContext, step.ID) + } + + stages, err := t.Stages.ListWithSteps(noContext, execution.ID) + if err != nil { + log.Warn().Err(err). + Msg("manager: cannot get stages") + return err + } + + if isexecutionComplete(stages) == false { + log.Warn().Err(err). + Msg("manager: execution pending completion of additional stages") + return nil + } + + log.Info().Msg("manager: execution is finished, teardown") + + execution.Status = types.StatusPassing + execution.Finished = time.Now().Unix() + for _, sibling := range stages { + if sibling.Status == types.StatusKilled { + execution.Status = types.StatusKilled + break + } + if sibling.Status == types.StatusFailing { + execution.Status = types.StatusFailing + break + } + if sibling.Status == types.StatusError { + execution.Status = types.StatusError + break + } + } + if execution.Started == 0 { + execution.Started = execution.Finished + } + + err = t.Executions.Update(noContext, execution) + if err == gitness_store.ErrVersionConflict { + log.Warn().Err(err). + Msg("manager: execution updated by another goroutine") + return nil + } + if err != nil { + log.Warn().Err(err). + Msg("manager: cannot update the execution") + return err + } + + return nil +} + +func isexecutionComplete(stages []*types.Stage) bool { + for _, stage := range stages { + switch stage.Status { + case types.StatusPending, + types.StatusRunning, + types.StatusWaiting, + types.StatusDeclined, + types.StatusBlocked: + return false + } + } + return true +} diff --git a/build/manager/updater.go b/build/manager/updater.go new file mode 100644 index 000000000..bb05d257e --- /dev/null +++ b/build/manager/updater.go @@ -0,0 +1,39 @@ +// Copyright 2022 Harness Inc. All rights reserved. +// Use of this source code is governed by the Polyform Free Trial License +// that can be found in the LICENSE.md file for this repository. + +package manager + +import ( + "context" + + "github.com/harness/gitness/internal/store" + "github.com/harness/gitness/types" + "github.com/rs/zerolog/log" +) + +type updater struct { + Executions store.ExecutionStore + Repos store.RepoStore + Steps store.StepStore + Stages store.StageStore +} + +func (u *updater) do(ctx context.Context, step *types.Step) error { + log := log.With(). + Str("step.name", step.Name). + Str("step.status", step.Status). + Int64("step.id", step.ID). + Logger() + + if len(step.Error) > 500 { + step.Error = step.Error[:500] + } + err := u.Steps.Update(noContext, step) + if err != nil { + log.Error().Err(err).Msg("manager: cannot update step") + return err + } + + return nil +} diff --git a/build/manager/wire.go b/build/manager/wire.go new file mode 100644 index 000000000..5f1e9c2f0 --- /dev/null +++ b/build/manager/wire.go @@ -0,0 +1,48 @@ +// Copyright 2022 Harness Inc. All rights reserved. +// Use of this source code is governed by the Polyform Free Trial License +// that can be found in the LICENSE.md file for this repository. + +package manager + +import ( + "github.com/harness/gitness/build/file" + "github.com/harness/gitness/build/scheduler" + "github.com/harness/gitness/internal/store" + "github.com/harness/gitness/internal/url" + "github.com/harness/gitness/livelog" + "github.com/harness/gitness/types" + + "github.com/drone/runner-go/client" + "github.com/google/wire" +) + +// WireSet provides a wire set for this package. +var WireSet = wire.NewSet( + ProvideExecutionManager, + ProvideExecutionClient, +) + +// ProvideExecutionManager provides an execution manager. +func ProvideExecutionManager( + config *types.Config, + executionStore store.ExecutionStore, + pipelineStore store.PipelineStore, + urlProvider *url.Provider, + fileService file.FileService, + logStore store.LogStore, + logStream livelog.LogStream, + repoStore store.RepoStore, + scheduler scheduler.Scheduler, + secretStore store.SecretStore, + stageStore store.StageStore, + stepStore store.StepStore, + userStore store.PrincipalStore) ExecutionManager { + return New(config, executionStore, pipelineStore, urlProvider, fileService, logStore, + logStream, repoStore, scheduler, secretStore, stageStore, stepStore, userStore) +} + +// ProvideExecutionClient provides a client implementation to interact with the execution manager. +// We use an embedded client here +func ProvideExecutionClient(manager ExecutionManager, config *types.Config) client.Client { + return NewEmbeddedClient(manager, config) +} diff --git a/build/runner/poller.go b/build/runner/poller.go new file mode 100644 index 000000000..2b4372e67 --- /dev/null +++ b/build/runner/poller.go @@ -0,0 +1,30 @@ +// Copyright 2022 Harness Inc. All rights reserved. +// Use of this source code is governed by the Polyform Free Trial License +// that can be found in the LICENSE.md file for this repository. + +package runner + +import ( + "github.com/harness/gitness/types" + + "github.com/drone-runners/drone-runner-docker/engine/resource" + runnerclient "github.com/drone/runner-go/client" + "github.com/drone/runner-go/pipeline/runtime" + "github.com/drone/runner-go/poller" +) + +func NewExecutionPoller( + runner *runtime.Runner, + config *types.Config, + client runnerclient.Client, +) *poller.Poller { + return &poller.Poller{ + Client: client, + Dispatch: runner.Run, + Filter: &runnerclient.Filter{ + Kind: resource.Kind, + Type: resource.Type, + // TODO: Check if other parameters are needed. + }, + } +} diff --git a/build/runner/runner.go b/build/runner/runner.go new file mode 100644 index 000000000..7f69ecdc1 --- /dev/null +++ b/build/runner/runner.go @@ -0,0 +1,66 @@ +// Copyright 2022 Harness Inc. All rights reserved. +// Use of this source code is governed by the Polyform Free Trial License +// that can be found in the LICENSE.md file for this repository. + +package runner + +import ( + "os" + + "github.com/google/uuid" + "github.com/harness/gitness/build/manager" + "github.com/harness/gitness/types" + + "github.com/drone-runners/drone-runner-docker/engine" + "github.com/drone-runners/drone-runner-docker/engine/compiler" + "github.com/drone-runners/drone-runner-docker/engine/linter" + "github.com/drone-runners/drone-runner-docker/engine/resource" + "github.com/drone/drone-go/drone" + runnerclient "github.com/drone/runner-go/client" + "github.com/drone/runner-go/environ/provider" + "github.com/drone/runner-go/pipeline/reporter/history" + "github.com/drone/runner-go/pipeline/reporter/remote" + "github.com/drone/runner-go/pipeline/runtime" + "github.com/drone/runner-go/pipeline/uploader" + "github.com/drone/runner-go/registry" + "github.com/drone/runner-go/secret" +) + +func NewExecutionRunner( + config *types.Config, + client runnerclient.Client, + m manager.ExecutionManager, +) (*runtime.Runner, error) { + compiler := &compiler.Compiler{ + Environ: provider.Static(map[string]string{}), + Registry: registry.Static([]*drone.Registry{}), + Secret: secret.Encrypted(), + } + + var host string + host, err := os.Hostname() + if err != nil { + host = uuid.New().String() + } + remote := remote.New(client) + upload := uploader.New(client) + tracer := history.New(remote) + engine, err := engine.NewEnv(engine.Opts{}) + if err != nil { + return nil, err + } + // TODO: Using the same parallel workers as the max concurrent step limit, + // this can be made configurable if needed later. + exec := runtime.NewExecer(tracer, remote, upload, + engine, int64(config.CI.ParallelWorkers)) + runner := &runtime.Runner{ + Machine: host, // TODO: Check whether this needs to be configurable + Client: client, + Reporter: tracer, + Lookup: resource.Lookup, + Lint: linter.New().Lint, + Compiler: compiler, + Exec: exec.Exec, + } + return runner, nil +} diff --git a/build/runner/wire.go b/build/runner/wire.go new file mode 100644 index 000000000..bc1252ac3 --- /dev/null +++ b/build/runner/wire.go @@ -0,0 +1,40 @@ +// Copyright 2022 Harness Inc. All rights reserved. +// Use of this source code is governed by the Polyform Free Trial License +// that can be found in the LICENSE.md file for this repository. + +package runner + +import ( + "github.com/harness/gitness/build/manager" + "github.com/harness/gitness/types" + + runnerclient "github.com/drone/runner-go/client" + "github.com/drone/runner-go/pipeline/runtime" + "github.com/drone/runner-go/poller" + "github.com/google/wire" +) + +// WireSet provides a wire set for this package. +var WireSet = wire.NewSet( + ProvideExecutionRunner, + ProvideExecutionPoller, +) + +// ProvideExecutionRunner provides an execution runner. +func ProvideExecutionRunner( + config *types.Config, + client runnerclient.Client, + manager manager.ExecutionManager, +) (*runtime.Runner, error) { + return NewExecutionRunner(config, client, manager) +} + +// ProvideExecutionPoller provides a poller which can poll the manager +// for new builds and execute them. +func ProvideExecutionPoller( + runner *runtime.Runner, + config *types.Config, + client runnerclient.Client, +) *poller.Poller { + return NewExecutionPoller(runner, config, client) +} diff --git a/build/scheduler/queue.go b/build/scheduler/queue.go new file mode 100644 index 000000000..28b1270b6 --- /dev/null +++ b/build/scheduler/queue.go @@ -0,0 +1,294 @@ +// Copyright 2022 Harness Inc. All rights reserved. +// Use of this source code is governed by the Polyform Free Trial License +// that can be found in the LICENSE.md file for this repository. + +package scheduler + +import ( + "context" + "sync" + "time" + + "github.com/harness/gitness/internal/store" + "github.com/harness/gitness/lock" + "github.com/harness/gitness/types" +) + +type queue struct { + sync.Mutex + globMx lock.Mutex + + ready chan struct{} + paused bool + interval time.Duration + throttle int + store store.StageStore + workers map[*worker]struct{} + ctx context.Context +} + +// newQueue returns a new Queue backed by the build datastore. +func newQueue(store store.StageStore, lock lock.MutexManager) (*queue, error) { + const lockKey = "build_queue" + mx, err := lock.NewMutex(lockKey) + if err != nil { + return nil, err + } + q := &queue{ + store: store, + globMx: mx, + ready: make(chan struct{}, 1), + workers: map[*worker]struct{}{}, + interval: time.Minute, + ctx: context.Background(), + } + go q.start() + return q, nil +} + +func (q *queue) Schedule(ctx context.Context, stage *types.Stage) error { + select { + case q.ready <- struct{}{}: + default: + } + return nil +} + +func (q *queue) Pause(ctx context.Context) error { + q.Lock() + q.paused = true + q.Unlock() + return nil +} + +func (q *queue) Request(ctx context.Context, params Filter) (*types.Stage, error) { + w := &worker{ + kind: params.Kind, + typ: params.Type, + os: params.OS, + arch: params.Arch, + kernel: params.Kernel, + variant: params.Variant, + labels: params.Labels, + channel: make(chan *types.Stage), + } + q.Lock() + q.workers[w] = struct{}{} + q.Unlock() + + select { + case q.ready <- struct{}{}: + default: + } + + select { + case <-ctx.Done(): + q.Lock() + delete(q.workers, w) + q.Unlock() + return nil, ctx.Err() + case b := <-w.channel: + return b, nil + } +} + +func (q *queue) signal(ctx context.Context) error { + if err := q.globMx.Lock(ctx); err != nil { + return err + } + defer q.globMx.Unlock(ctx) + + q.Lock() + count := len(q.workers) + pause := q.paused + q.Unlock() + if pause { + return nil + } + if count == 0 { + return nil + } + items, err := q.store.ListIncomplete(ctx) + if err != nil { + return err + } + + q.Lock() + defer q.Unlock() + for _, item := range items { + if item.Status == types.StatusRunning { + continue + } + if item.Machine != "" { + continue + } + + // if the stage defines concurrency limits we + // need to make sure those limits are not exceeded + // before proceeding. + if withinLimits(item, items) == false { + continue + } + + // if the system defines concurrency limits + // per repository we need to make sure those limits + // are not exceeded before proceeding. + if shouldThrottle(item, items, item.LimitRepo) == true { + continue + } + + loop: + for w := range q.workers { + // the worker must match the resource kind and type + if !matchResource(w.kind, w.typ, item.Kind, item.Type) { + continue + } + + if w.os != "" || w.arch != "" || w.variant != "" || w.kernel != "" { + // the worker is platform-specific. check to ensure + // the queue item matches the worker platform. + if w.os != item.OS { + continue + } + if w.arch != item.Arch { + continue + } + // if the pipeline defines a variant it must match + // the worker variant (e.g. arm6, arm7, etc). + if item.Variant != "" && item.Variant != w.variant { + continue + } + // if the pipeline defines a kernel version it must match + // the worker kernel version (e.g. 1709, 1803). + if item.Kernel != "" && item.Kernel != w.kernel { + continue + } + } + + if len(item.Labels) > 0 || len(w.labels) > 0 { + if !checkLabels(item.Labels, w.labels) { + continue + } + } + + select { + case w.channel <- item: + delete(q.workers, w) + break loop + } + } + } + return nil +} + +func (q *queue) start() error { + for { + select { + case <-q.ctx.Done(): + return q.ctx.Err() + case <-q.ready: + q.signal(q.ctx) + case <-time.After(q.interval): + q.signal(q.ctx) + } + } +} + +type worker struct { + kind string + typ string + os string + arch string + kernel string + variant string + labels map[string]string + channel chan *types.Stage +} + +type counter struct { + counts map[string]int +} + +func checkLabels(a, b map[string]string) bool { + if len(a) != len(b) { + return false + } + for k, v := range a { + if w, ok := b[k]; !ok || v != w { + return false + } + } + return true +} + +func withinLimits(stage *types.Stage, siblings []*types.Stage) bool { + if stage.Limit == 0 { + return true + } + count := 0 + for _, sibling := range siblings { + if sibling.RepoID != stage.RepoID { + continue + } + if sibling.ID == stage.ID { + continue + } + if sibling.Name != stage.Name { + continue + } + if sibling.ID < stage.ID || + sibling.Status == types.StatusRunning { + count++ + } + } + return count < stage.Limit +} + +func shouldThrottle(stage *types.Stage, siblings []*types.Stage, limit int) bool { + // if no throttle limit is defined (default) then + // return false to indicate no throttling is needed. + if limit == 0 { + return false + } + // if the repository is running it is too late + // to skip and we can exit + if stage.Status == types.StatusRunning { + return false + } + + count := 0 + // loop through running stages to count number of + // running stages for the parent repository. + for _, sibling := range siblings { + // ignore stages from other repository. + if sibling.RepoID != stage.RepoID { + continue + } + // ignore this stage and stages that were + // scheduled after this stage. + if sibling.ID >= stage.ID { + continue + } + count++ + } + // if the count of running stages exceeds the + // throttle limit return true. + return count >= limit +} + +// matchResource is a helper function that returns +func matchResource(kinda, typea, kindb, typeb string) bool { + if kinda == "" { + kinda = "pipeline" + } + if kindb == "" { + kindb = "pipeline" + } + if typea == "" { + typea = "docker" + } + if typeb == "" { + typeb = "docker" + } + return kinda == kindb && typea == typeb +} diff --git a/build/scheduler/scheduler.go b/build/scheduler/scheduler.go new file mode 100644 index 000000000..a6a6f9d8d --- /dev/null +++ b/build/scheduler/scheduler.go @@ -0,0 +1,32 @@ +// Copyright 2022 Harness Inc. All rights reserved. +// Use of this source code is governed by the Polyform Free Trial License +// that can be found in the LICENSE.md file for this repository. + +package scheduler + +import ( + "context" + + "github.com/harness/gitness/types" +) + +// Filter provides filter criteria to limit stages requested +// from the scheduler. +type Filter struct { + Kind string + Type string + OS string + Arch string + Kernel string + Variant string + Labels map[string]string +} + +// Scheduler schedules Build stages for execution. +type Scheduler interface { + // Schedule schedules the stage for execution. + Schedule(ctx context.Context, stage *types.Stage) error + + // Request requests the next stage scheduled for execution. + Request(ctx context.Context, filter Filter) (*types.Stage, error) +} diff --git a/build/scheduler/wire.go b/build/scheduler/wire.go new file mode 100644 index 000000000..bf4169dac --- /dev/null +++ b/build/scheduler/wire.go @@ -0,0 +1,25 @@ +// Copyright 2022 Harness Inc. All rights reserved. +// Use of this source code is governed by the Polyform Free Trial License +// that can be found in the LICENSE.md file for this repository. + +package scheduler + +import ( + "github.com/harness/gitness/internal/store" + "github.com/harness/gitness/lock" + + "github.com/google/wire" +) + +// WireSet provides a wire set for this package. +var WireSet = wire.NewSet( + ProvideScheduler, +) + +// ProvideScheduler provides a scheduler which can be used to schedule and request builds. +func ProvideScheduler( + stageStore store.StageStore, + lock lock.MutexManager, +) (Scheduler, error) { + return newQueue(stageStore, lock) +} diff --git a/build/triggerer/dag/dag.go b/build/triggerer/dag/dag.go new file mode 100644 index 000000000..a568789cc --- /dev/null +++ b/build/triggerer/dag/dag.go @@ -0,0 +1,137 @@ +// Copyright 2022 Harness Inc. All rights reserved. +// Use of this source code is governed by the Polyform Free Trial License +// that can be found in the LICENSE.md file for this repository. + +package dag + +// Dag is a directed acyclic graph. +type Dag struct { + graph map[string]*Vertex +} + +// Vertex is a vertex in the graph. +type Vertex struct { + Name string + Skip bool + graph []string +} + +// New creates a new directed acyclic graph (dag) that can +// determinate if a stage has dependencies. +func New() *Dag { + return &Dag{ + graph: make(map[string]*Vertex), + } +} + +// Add establishes a dependency between two vertices in the graph. +func (d *Dag) Add(from string, to ...string) *Vertex { + vertex := new(Vertex) + vertex.Name = from + vertex.Skip = false + vertex.graph = to + d.graph[from] = vertex + return vertex +} + +// Get returns the vertex from the graph. +func (d *Dag) Get(name string) (*Vertex, bool) { + vertex, ok := d.graph[name] + return vertex, ok +} + +// Dependencies returns the direct dependencies accounting for +// skipped dependencies. +func (d *Dag) Dependencies(name string) []string { + vertex := d.graph[name] + return d.dependencies(vertex) +} + +// Ancestors returns the ancestors of the vertex. +func (d *Dag) Ancestors(name string) []*Vertex { + vertex := d.graph[name] + return d.ancestors(vertex) +} + +// DetectCycles returns true if cycles are detected in the graph. +func (d *Dag) DetectCycles() bool { + visited := make(map[string]bool) + recStack := make(map[string]bool) + + for vertex := range d.graph { + if !visited[vertex] { + if d.detectCycles(vertex, visited, recStack) { + return true + } + } + } + return false +} + +// helper function returns the list of ancestors for the vertex. +func (d *Dag) ancestors(parent *Vertex) []*Vertex { + if parent == nil { + return nil + } + var combined []*Vertex + for _, name := range parent.graph { + vertex, found := d.graph[name] + if !found { + continue + } + if !vertex.Skip { + combined = append(combined, vertex) + } + combined = append(combined, d.ancestors(vertex)...) + } + return combined +} + +// helper function returns the list of dependencies for the, +// vertex taking into account skipped dependencies. +func (d *Dag) dependencies(parent *Vertex) []string { + if parent == nil { + return nil + } + var combined []string + for _, name := range parent.graph { + vertex, found := d.graph[name] + if !found { + continue + } + if vertex.Skip { + // if the vertex is skipped we should move up the + // graph and check direct ancestors. + combined = append(combined, d.dependencies(vertex)...) + } else { + combined = append(combined, vertex.Name) + } + } + return combined +} + +// helper function returns true if the vertex is cyclical. +func (d *Dag) detectCycles(name string, visited, recStack map[string]bool) bool { + visited[name] = true + recStack[name] = true + + vertex, ok := d.graph[name] + if !ok { + return false + } + for _, v := range vertex.graph { + // only check cycles on a vertex one time + if !visited[v] { + if d.detectCycles(v, visited, recStack) { + return true + } + // if we've visited this vertex in this recursion + // stack, then we have a cycle + } else if recStack[v] { + return true + } + + } + recStack[name] = false + return false +} diff --git a/build/triggerer/dag/dag_test.go b/build/triggerer/dag/dag_test.go new file mode 100644 index 000000000..2c96511f8 --- /dev/null +++ b/build/triggerer/dag/dag_test.go @@ -0,0 +1,209 @@ +// Copyright 2022 Harness Inc. All rights reserved. +// Use of this source code is governed by the Polyform Free Trial License +// that can be found in the LICENSE.md file for this repository. + +package dag + +import ( + "reflect" + "testing" +) + +func TestDag(t *testing.T) { + dag := New() + dag.Add("backend") + dag.Add("frontend") + dag.Add("notify", "backend", "frontend") + if dag.DetectCycles() { + t.Errorf("cycles detected") + } + + dag = New() + dag.Add("notify", "backend", "frontend") + if dag.DetectCycles() { + t.Errorf("cycles detected") + } + + dag = New() + dag.Add("backend", "frontend") + dag.Add("frontend", "backend") + dag.Add("notify", "backend", "frontend") + if dag.DetectCycles() == false { + t.Errorf("Expect cycles detected") + } + + dag = New() + dag.Add("backend", "backend") + dag.Add("frontend", "backend") + dag.Add("notify", "backend", "frontend") + if dag.DetectCycles() == false { + t.Errorf("Expect cycles detected") + } + + dag = New() + dag.Add("backend") + dag.Add("frontend") + dag.Add("notify", "backend", "frontend", "notify") + if dag.DetectCycles() == false { + t.Errorf("Expect cycles detected") + } +} + +func TestAncestors(t *testing.T) { + dag := New() + v := dag.Add("backend") + dag.Add("frontend", "backend") + dag.Add("notify", "frontend") + + ancestors := dag.Ancestors("frontend") + if got, want := len(ancestors), 1; got != want { + t.Errorf("Want %d ancestors, got %d", want, got) + } + if ancestors[0] != v { + t.Errorf("Unexpected ancestor") + } + + if v := dag.Ancestors("backend"); len(v) != 0 { + t.Errorf("Expect vertexes with no dependencies has zero ancestors") + } +} + +func TestAncestors_Skipped(t *testing.T) { + dag := New() + dag.Add("backend").Skip = true + dag.Add("frontend", "backend").Skip = true + dag.Add("notify", "frontend") + + if v := dag.Ancestors("frontend"); len(v) != 0 { + t.Errorf("Expect skipped vertexes excluded") + } + if v := dag.Ancestors("notify"); len(v) != 0 { + t.Errorf("Expect skipped vertexes excluded") + } +} + +func TestAncestors_NotFound(t *testing.T) { + dag := New() + dag.Add("backend") + dag.Add("frontend", "backend") + dag.Add("notify", "frontend") + if dag.DetectCycles() { + t.Errorf("cycles detected") + } + if v := dag.Ancestors("does-not-exist"); len(v) != 0 { + t.Errorf("Expect vertex not found does not panic") + } +} + +func TestAncestors_Malformed(t *testing.T) { + dag := New() + dag.Add("backend") + dag.Add("frontend", "does-not-exist") + dag.Add("notify", "frontend") + if dag.DetectCycles() { + t.Errorf("cycles detected") + } + if v := dag.Ancestors("frontend"); len(v) != 0 { + t.Errorf("Expect invalid dependency does not panic") + } +} + +func TestAncestors_Complex(t *testing.T) { + dag := New() + dag.Add("backend") + dag.Add("frontend") + dag.Add("publish", "backend", "frontend") + dag.Add("deploy", "publish") + last := dag.Add("notify", "deploy") + if dag.DetectCycles() { + t.Errorf("cycles detected") + } + + ancestors := dag.Ancestors("notify") + if got, want := len(ancestors), 4; got != want { + t.Errorf("Want %d ancestors, got %d", want, got) + return + } + for _, ancestor := range ancestors { + if ancestor == last { + t.Errorf("Unexpected ancestor") + } + } + + v, _ := dag.Get("publish") + v.Skip = true + ancestors = dag.Ancestors("notify") + if got, want := len(ancestors), 3; got != want { + t.Errorf("Want %d ancestors, got %d", want, got) + return + } +} + +func TestDependencies(t *testing.T) { + dag := New() + dag.Add("backend") + dag.Add("frontend") + dag.Add("publish", "backend", "frontend") + + if deps := dag.Dependencies("backend"); len(deps) != 0 { + t.Errorf("Expect zero dependencies") + } + if deps := dag.Dependencies("frontend"); len(deps) != 0 { + t.Errorf("Expect zero dependencies") + } + + got, want := dag.Dependencies("publish"), []string{"backend", "frontend"} + if !reflect.DeepEqual(got, want) { + t.Errorf("Unexpected dependencies, got %v", got) + } +} + +func TestDependencies_Skipped(t *testing.T) { + dag := New() + dag.Add("backend") + dag.Add("frontend").Skip = true + dag.Add("publish", "backend", "frontend") + + if deps := dag.Dependencies("backend"); len(deps) != 0 { + t.Errorf("Expect zero dependencies") + } + if deps := dag.Dependencies("frontend"); len(deps) != 0 { + t.Errorf("Expect zero dependencies") + } + + got, want := dag.Dependencies("publish"), []string{"backend"} + if !reflect.DeepEqual(got, want) { + t.Errorf("Unexpected dependencies, got %v", got) + } +} + +func TestDependencies_Complex(t *testing.T) { + dag := New() + dag.Add("clone") + dag.Add("backend") + dag.Add("frontend", "backend").Skip = true + dag.Add("publish", "frontend", "clone") + dag.Add("notify", "publish") + + if deps := dag.Dependencies("clone"); len(deps) != 0 { + t.Errorf("Expect zero dependencies for clone") + } + if deps := dag.Dependencies("backend"); len(deps) != 0 { + t.Errorf("Expect zero dependencies for backend") + } + + got, want := dag.Dependencies("frontend"), []string{"backend"} + if !reflect.DeepEqual(got, want) { + t.Errorf("Unexpected dependencies for frontend, got %v", got) + } + + got, want = dag.Dependencies("publish"), []string{"backend", "clone"} + if !reflect.DeepEqual(got, want) { + t.Errorf("Unexpected dependencies for publish, got %v", got) + } + + got, want = dag.Dependencies("notify"), []string{"publish"} + if !reflect.DeepEqual(got, want) { + t.Errorf("Unexpected dependencies for notify, got %v", got) + } +} diff --git a/build/triggerer/skip.go b/build/triggerer/skip.go new file mode 100644 index 000000000..7fe3c0749 --- /dev/null +++ b/build/triggerer/skip.go @@ -0,0 +1,78 @@ +// Copyright 2022 Harness Inc. All rights reserved. +// Use of this source code is governed by the Polyform Free Trial License +// that can be found in the LICENSE.md file for this repository. + +package triggerer + +import ( + "strings" + + "github.com/harness/gitness/types" + + "github.com/drone/drone-yaml/yaml" +) + +func skipBranch(document *yaml.Pipeline, branch string) bool { + return !document.Trigger.Branch.Match(branch) +} + +func skipRef(document *yaml.Pipeline, ref string) bool { + return !document.Trigger.Ref.Match(ref) +} + +func skipEvent(document *yaml.Pipeline, event string) bool { + return !document.Trigger.Event.Match(event) +} + +func skipAction(document *yaml.Pipeline, action string) bool { + return !document.Trigger.Action.Match(action) +} + +func skipInstance(document *yaml.Pipeline, instance string) bool { + return !document.Trigger.Instance.Match(instance) +} + +func skipTarget(document *yaml.Pipeline, env string) bool { + return !document.Trigger.Target.Match(env) +} + +func skipRepo(document *yaml.Pipeline, repo string) bool { + return !document.Trigger.Repo.Match(repo) +} + +func skipCron(document *yaml.Pipeline, cron string) bool { + return !document.Trigger.Cron.Match(cron) +} + +func skipMessage(hook *Hook) bool { + switch { + case hook.Event == types.EventTag: + return false + case hook.Event == types.EventCron: + return false + case hook.Event == types.EventCustom: + return false + case hook.Event == types.EventPromote: + return false + case hook.Event == types.EventRollback: + return false + case skipMessageEval(hook.Message): + return true + case skipMessageEval(hook.Title): + return true + default: + return false + } +} + +func skipMessageEval(str string) bool { + lower := strings.ToLower(str) + switch { + case strings.Contains(lower, "[ci skip]"), + strings.Contains(lower, "[skip ci]"), + strings.Contains(lower, "***no_ci***"): + return true + default: + return false + } +} diff --git a/build/triggerer/trigger.go b/build/triggerer/trigger.go new file mode 100644 index 000000000..45e86ee33 --- /dev/null +++ b/build/triggerer/trigger.go @@ -0,0 +1,506 @@ +// Copyright 2022 Harness Inc. All rights reserved. +// Use of this source code is governed by the Polyform Free Trial License +// that can be found in the LICENSE.md file for this repository. + +package triggerer + +import ( + "context" + "runtime/debug" + "time" + + "github.com/harness/gitness/build/file" + "github.com/harness/gitness/build/scheduler" + "github.com/harness/gitness/build/triggerer/dag" + "github.com/harness/gitness/internal/store" + "github.com/harness/gitness/store/database/dbtx" + "github.com/harness/gitness/types" + + "github.com/drone/drone-yaml/yaml" + "github.com/drone/drone-yaml/yaml/linter" + "github.com/jmoiron/sqlx" + "github.com/rs/zerolog/log" +) + +// Trigger types +const ( + TriggerHook = "@hook" + TriggerCron = "@cron" +) + +var _ Triggerer = (*triggerer)(nil) + +// Hook represents the payload of a post-commit hook. +type Hook struct { + Parent int64 `json:"parent"` + Trigger string `json:"trigger"` + Event string `json:"event"` + Action string `json:"action"` + Link string `json:"link"` + Timestamp int64 `json:"timestamp"` + Title string `json:"title"` + Message string `json:"message"` + Before string `json:"before"` + After string `json:"after"` + Ref string `json:"ref"` + Fork string `json:"hook"` + Source string `json:"source"` + Target string `json:"target"` + Author string `json:"author_login"` + AuthorName string `json:"author_name"` + AuthorEmail string `json:"author_email"` + AuthorAvatar string `json:"author_avatar"` + Deployment string `json:"deploy_to"` + DeploymentID int64 `json:"deploy_id"` + Debug bool `json:"debug"` + Cron string `json:"cron"` + Sender string `json:"sender"` + Params map[string]string `json:"params"` +} + +// Triggerer is responsible for triggering a Execution from an +// incoming drone. If an execution is skipped a nil value is +// returned. +type Triggerer interface { + Trigger(ctx context.Context, pipeline *types.Pipeline, hook *Hook) (*types.Execution, error) +} + +type triggerer struct { + executionStore store.ExecutionStore + stageStore store.StageStore + db *sqlx.DB + pipelineStore store.PipelineStore + fileService file.FileService + scheduler scheduler.Scheduler + repoStore store.RepoStore +} + +func New( + executionStore store.ExecutionStore, + stageStore store.StageStore, + pipelineStore store.PipelineStore, + db *sqlx.DB, + repoStore store.RepoStore, + scheduler scheduler.Scheduler, + fileService file.FileService, +) *triggerer { + return &triggerer{ + executionStore: executionStore, + stageStore: stageStore, + scheduler: scheduler, + db: db, + pipelineStore: pipelineStore, + fileService: fileService, + repoStore: repoStore, + } +} + +func (t *triggerer) Trigger( + ctx context.Context, + pipeline *types.Pipeline, + base *Hook, +) (*types.Execution, error) { + log := log.With(). + Str("ref", base.Ref). + Str("commit", base.After). + Logger() + + log.Debug().Msg("trigger: received") + defer func() { + // taking the paranoid approach to recover from + // a panic that should absolutely never happen. + if r := recover(); r != nil { + log.Error().Msgf("runner: unexpected panic: %s", r) + debug.PrintStack() + } + }() + + repo, err := t.repoStore.Find(ctx, pipeline.RepoID) + if err != nil { + log.Error().Err(err).Msg("could not find repo") + return nil, err + } + + // if skipMessage(base) { + // logger.Infoln("trigger: skipping hook. found skip directive") + // return nil, nil + // } + // if base.Event == core.EventPullRequest { + // if repo.IgnorePulls { + // logger.Infoln("trigger: skipping hook. project ignores pull requests") + // return nil, nil + // } + // if repo.IgnoreForks && !strings.EqualFold(base.Fork, repo.Slug) { + // logger.Infoln("trigger: skipping hook. project ignores forks") + // return nil, nil + // } + // } + + // user, err := t.users.Find(ctx, repo.UserID) + // if err != nil { + // logger = logger.WithError(err) + // logger.Warnln("trigger: cannot find repository owner") + // return nil, err + // } + + // if user.Active == false { + // logger.Infoln("trigger: skipping hook. repository owner is inactive") + // return nil, nil + // } + + // if the commit message is not included we should + // make an optional API call to the version control + // system to augment the available information. + // if base.Message == "" && base.After != "" { + // commit, err := t.commits.Find(ctx, user, repo.Slug, base.After) + // if err == nil && commit != nil { + // base.Message = commit.Message + // if base.AuthorEmail == "" { + // base.AuthorEmail = commit.Author.Email + // } + // if base.AuthorName == "" { + // base.AuthorName = commit.Author.Name + // } + // if base.AuthorAvatar == "" { + // base.AuthorAvatar = commit.Author.Avatar + // } + // } + // } + + var ref string + if base.After != "" { + ref = base.After + } else if base.Ref != "" { + ref = base.Ref + } else { + ref = pipeline.DefaultBranch + } + file, err := t.fileService.Get(ctx, repo, pipeline.ConfigPath, ref) + if err != nil { + log.Error().Err(err).Msg("trigger: could not find yaml") + return nil, err + } + + // // this code is temporarily in place to detect and convert + // // the legacy yaml configuration file to the new format. + // raw.Data, err = converter.ConvertString(raw.Data, converter.Metadata{ + // Filename: repo.Config, + // URL: repo.Link, + // Ref: base.Ref, + // }) + // if err != nil { + // logger = logger.WithError(err) + // logger.Warnln("trigger: cannot convert yaml") + // return t.createExecutionError(ctx, repo, base, err.Error()) + // } + + manifest, err := yaml.ParseString(string(file.Data)) + if err != nil { + log.Warn().Err(err).Msg("trigger: cannot parse yaml") + return t.createExecutionError(ctx, pipeline, base, err.Error()) + } + + // verr := t.validate.Validate(ctx, &core.ValidateArgs{ + // User: user, + // Repo: repo, + // Execution: tmpExecution, + // Config: raw, + // }) + // switch verr { + // case core.ErrValidatorBlock: + // logger.Debugln("trigger: yaml validation error: block pipeline") + // case core.ErrValidatorSkip: + // logger.Debugln("trigger: yaml validation error: skip pipeline") + // return nil, nil + // default: + // if verr != nil { + // logger = logger.WithError(err) + // logger.Warnln("trigger: yaml validation error") + // return t.createExecutionError(ctx, repo, base, verr.Error()) + // } + // } + + err = linter.Manifest(manifest, true) + if err != nil { + log.Warn().Err(err).Msg("trigger: yaml linting error") + return t.createExecutionError(ctx, pipeline, base, err.Error()) + } + + verified := true + // if repo.Protected && base.Trigger == core.TriggerHook { + // key := signer.KeyString(repo.Secret) + // val := []byte(raw.Data) + // verified, _ = signer.Verify(val, key) + // } + // // if pipeline validation failed with a block error, the + // // pipeline verification should be set to false, which will + // // force manual review and approval. + // if verr == core.ErrValidatorBlock { + // verified = false + // } + + var matched []*yaml.Pipeline + var dag = dag.New() + for _, document := range manifest.Resources { + pipeline, ok := document.(*yaml.Pipeline) + if !ok { + continue + } + // TODO add repo + // TODO add instance + // TODO add target + // TODO add ref + name := pipeline.Name + if name == "" { + name = "default" + } + node := dag.Add(pipeline.Name, pipeline.DependsOn...) + node.Skip = true + + if skipBranch(pipeline, base.Target) { + log.Info().Str("pipeline", pipeline.Name).Msg("trigger: skipping pipeline, does not match branch") + } else if skipEvent(pipeline, base.Event) { + log.Info().Str("pipeline", pipeline.Name).Msg("trigger: skipping pipeline, does not match event") + } else if skipAction(pipeline, base.Action) { + log.Info().Str("pipeline", pipeline.Name).Msg("trigger: skipping pipeline, does not match action") + } else if skipRef(pipeline, base.Ref) { + log.Info().Str("pipeline", pipeline.Name).Msg("trigger: skipping pipeline, does not match ref") + } else if skipRepo(pipeline, repo.Path) { + log.Info().Str("pipeline", pipeline.Name).Msg("trigger: skipping pipeline, does not match repo") + } else if skipTarget(pipeline, base.Deployment) { + log.Info().Str("pipeline", pipeline.Name).Msg("trigger: skipping pipeline, does not match deploy target") + } else if skipCron(pipeline, base.Cron) { + log.Info().Str("pipeline", pipeline.Name).Msg("trigger: skipping pipeline, does not match cron job") + } else { + matched = append(matched, pipeline) + node.Skip = false + } + } + + if dag.DetectCycles() { + return t.createExecutionError(ctx, pipeline, base, "Error: Dependency cycle detected in Pipeline") + } + + if len(matched) == 0 { + log.Info().Msg("trigger: skipping execution, no matching pipelines") + return nil, nil + } + + pipeline, err = t.pipelineStore.IncrementSeqNum(ctx, pipeline) + if err != nil { + log.Error().Err(err).Msg("trigger: cannot increment execution sequence number") + return nil, err + } + + execution := &types.Execution{ + RepoID: repo.ID, + PipelineID: pipeline.ID, + Trigger: base.Trigger, + Number: pipeline.Seq, + Parent: base.Parent, + Status: types.StatusPending, + Event: base.Event, + Action: base.Action, + Link: base.Link, + // Timestamp: base.Timestamp, + Title: trunc(base.Title, 2000), + Message: trunc(base.Message, 2000), + Before: base.Before, + After: base.After, + Ref: base.Ref, + Fork: base.Fork, + Source: base.Source, + Target: base.Target, + Author: base.Author, + AuthorName: base.AuthorName, + AuthorEmail: base.AuthorEmail, + AuthorAvatar: base.AuthorAvatar, + Params: base.Params, + Deploy: base.Deployment, + DeployID: base.DeploymentID, + Debug: base.Debug, + Sender: base.Sender, + Cron: base.Cron, + Created: time.Now().Unix(), + Updated: time.Now().Unix(), + } + + stages := make([]*types.Stage, len(matched)) + for i, match := range matched { + onSuccess := match.Trigger.Status.Match(types.StatusPassing) + onFailure := match.Trigger.Status.Match(types.StatusFailing) + if len(match.Trigger.Status.Include)+len(match.Trigger.Status.Exclude) == 0 { + onFailure = false + } + + stage := &types.Stage{ + RepoID: repo.ID, + Number: int64(i + 1), + Name: match.Name, + Kind: match.Kind, + Type: match.Type, + OS: match.Platform.OS, + Arch: match.Platform.Arch, + Variant: match.Platform.Variant, + Kernel: match.Platform.Version, + Limit: match.Concurrency.Limit, + Status: types.StatusWaiting, + DependsOn: match.DependsOn, + OnSuccess: onSuccess, + OnFailure: onFailure, + Labels: match.Node, + Created: time.Now().Unix(), + Updated: time.Now().Unix(), + } + if stage.Kind == "pipeline" && stage.Type == "" { + stage.Type = "docker" + } + if stage.OS == "" { + stage.OS = "linux" + } + if stage.Arch == "" { + stage.Arch = "amd64" + } + + if stage.Name == "" { + stage.Name = "default" + } + if verified == false { + stage.Status = types.StatusBlocked + } else if len(stage.DependsOn) == 0 { + stage.Status = types.StatusPending + } + stages[i] = stage + } + + for _, stage := range stages { + // here we re-work the dependencies for the stage to + // account for the fact that some steps may be skipped + // and may otherwise break the dependency chain. + stage.DependsOn = dag.Dependencies(stage.Name) + + // if the stage is pending dependencies, but those + // dependencies are skipped, the stage can be executed + // immediately. + if stage.Status == types.StatusWaiting && + len(stage.DependsOn) == 0 { + stage.Status = types.StatusPending + } + } + + err = t.createExecutionWithStages(ctx, execution, stages) + if err != nil { + log.Error().Err(err).Msg("trigger: cannot create execution") + return nil, err + } + + // err = t.status.Send(ctx, user, &core.StatusInput{ + // Repo: repo, + // Execution: execution, + // }) + // if err != nil { + // logger = logger.WithError(err) + // logger.Warnln("trigger: cannot create status") + // } + + for _, stage := range stages { + if stage.Status != types.StatusPending { + continue + } + err = t.scheduler.Schedule(ctx, stage) + if err != nil { + log.Error().Err(err).Msg("trigger: cannot enqueue execution") + return nil, err + } + } + + return execution, nil +} + +func trunc(s string, i int) string { + runes := []rune(s) + if len(runes) > i { + return string(runes[:i]) + } + return s +} + +// createExecutionWithStages writes an execution along with its stages in a single transaction. +func (t *triggerer) createExecutionWithStages( + ctx context.Context, + execution *types.Execution, + stages []*types.Stage, +) error { + return dbtx.New(t.db).WithTx(ctx, func(ctx context.Context) error { + err := t.executionStore.Create(ctx, execution) + if err != nil { + return err + } + + for _, stage := range stages { + stage.ExecutionID = execution.ID + err := t.stageStore.Create(ctx, stage) + if err != nil { + return err + } + } + return nil + }) +} + +// createExecutionError creates an execution with an error message. +func (t *triggerer) createExecutionError( + ctx context.Context, + pipeline *types.Pipeline, + base *Hook, + message string, +) (*types.Execution, error) { + log := log.With(). + Str("ref", base.Ref). + Str("commit", base.After). + Logger() + + pipeline, err := t.pipelineStore.IncrementSeqNum(ctx, pipeline) + if err != nil { + return nil, err + } + + execution := &types.Execution{ + RepoID: pipeline.RepoID, + Number: pipeline.Seq, + Parent: base.Parent, + Status: types.StatusError, + Error: message, + Event: base.Event, + Action: base.Action, + Link: base.Link, + Title: base.Title, + Message: base.Message, + Before: base.Before, + After: base.After, + Ref: base.Ref, + Fork: base.Fork, + Source: base.Source, + Target: base.Target, + Author: base.Author, + AuthorName: base.AuthorName, + AuthorEmail: base.AuthorEmail, + AuthorAvatar: base.AuthorAvatar, + Deploy: base.Deployment, + DeployID: base.DeploymentID, + Debug: base.Debug, + Sender: base.Sender, + Created: time.Now().Unix(), + Updated: time.Now().Unix(), + Started: time.Now().Unix(), + Finished: time.Now().Unix(), + } + + err = t.executionStore.Create(ctx, execution) + if err != nil { + log.Error().Err(err).Msg("trigger: cannot create execution error") + return nil, err + } + + return execution, err +} diff --git a/build/triggerer/wire.go b/build/triggerer/wire.go new file mode 100644 index 000000000..54738c7ff --- /dev/null +++ b/build/triggerer/wire.go @@ -0,0 +1,33 @@ +// Copyright 2022 Harness Inc. All rights reserved. +// Use of this source code is governed by the Polyform Free Trial License +// that can be found in the LICENSE.md file for this repository. + +package triggerer + +import ( + "github.com/harness/gitness/build/file" + "github.com/harness/gitness/build/scheduler" + "github.com/harness/gitness/internal/store" + + "github.com/google/wire" + "github.com/jmoiron/sqlx" +) + +// WireSet provides a wire set for this package. +var WireSet = wire.NewSet( + ProvideTriggerer, +) + +// ProvideTriggerer provides a triggerer which can execute builds. +func ProvideTriggerer( + executionStore store.ExecutionStore, + stageStore store.StageStore, + db *sqlx.DB, + pipelineStore store.PipelineStore, + fileService file.FileService, + scheduler scheduler.Scheduler, + repoStore store.RepoStore, +) Triggerer { + return New(executionStore, stageStore, pipelineStore, + db, repoStore, scheduler, fileService) +} diff --git a/cli/server/server.go b/cli/server/server.go index 779a5b5bb..e424646b9 100644 --- a/cli/server/server.go +++ b/cli/server/server.go @@ -16,10 +16,12 @@ import ( "github.com/harness/gitness/types" "github.com/harness/gitness/version" + "github.com/drone/runner-go/logger" "github.com/joho/godotenv" "github.com/mattn/go-isatty" "github.com/rs/zerolog" "github.com/rs/zerolog/log" + "github.com/sirupsen/logrus" "golang.org/x/sync/errgroup" "gopkg.in/alecthomas/kingpin.v2" ) @@ -80,6 +82,15 @@ func (c *command) run(*kingpin.ParseContext) error { // start server gHTTP, shutdownHTTP := system.server.ListenAndServe() g.Go(gHTTP.Wait) + g.Go(func() error { + // start poller for CI build executions. + log := logrus.New() + log.Out = os.Stdout + log.Level = logrus.DebugLevel // print all debug logs in common runner code. + ctx = logger.WithContext(ctx, logger.Logrus(log.WithContext(ctx))) + system.poller.Poll(ctx, config.CI.ParallelWorkers) + return nil + }) log.Info(). Str("port", config.Server.HTTP.Bind). Str("revision", version.GitCommit). diff --git a/cli/server/system.go b/cli/server/system.go index 239cb7913..29879c013 100644 --- a/cli/server/system.go +++ b/cli/server/system.go @@ -5,6 +5,7 @@ package server import ( + "github.com/drone/runner-go/poller" gitrpcserver "github.com/harness/gitness/gitrpc/server" gitrpccron "github.com/harness/gitness/gitrpc/server/cron" "github.com/harness/gitness/internal/bootstrap" @@ -17,16 +18,18 @@ type System struct { bootstrap bootstrap.Bootstrap server *server.Server gitRPCServer *gitrpcserver.GRPCServer + poller *poller.Poller services services.Services gitRPCCronMngr *gitrpccron.Manager } // NewSystem returns a new system structure. -func NewSystem(bootstrap bootstrap.Bootstrap, server *server.Server, gitRPCServer *gitrpcserver.GRPCServer, +func NewSystem(bootstrap bootstrap.Bootstrap, server *server.Server, poller *poller.Poller, gitRPCServer *gitrpcserver.GRPCServer, gitrpccron *gitrpccron.Manager, services services.Services) *System { return &System{ bootstrap: bootstrap, server: server, + poller: poller, gitRPCServer: gitRPCServer, services: services, gitRPCCronMngr: gitrpccron, diff --git a/cmd/gitness/wire.go b/cmd/gitness/wire.go index 54261520e..619c7b896 100644 --- a/cmd/gitness/wire.go +++ b/cmd/gitness/wire.go @@ -10,6 +10,12 @@ package main import ( "context" + "github.com/harness/gitness/build/commit" + "github.com/harness/gitness/build/file" + "github.com/harness/gitness/build/manager" + "github.com/harness/gitness/build/runner" + "github.com/harness/gitness/build/scheduler" + "github.com/harness/gitness/build/triggerer" cliserver "github.com/harness/gitness/cli/server" "github.com/harness/gitness/encrypt" "github.com/harness/gitness/events" @@ -114,6 +120,12 @@ func initSystem(ctx context.Context, config *types.Config) (*cliserver.System, e secret.WireSet, connector.WireSet, template.WireSet, + manager.WireSet, + triggerer.WireSet, + file.WireSet, + runner.WireSet, + scheduler.WireSet, + commit.WireSet, trigger.WireSet, plugin.WireSet, ) diff --git a/cmd/gitness/wire_gen.go b/cmd/gitness/wire_gen.go index 70f0e0692..5c0e391d3 100644 --- a/cmd/gitness/wire_gen.go +++ b/cmd/gitness/wire_gen.go @@ -8,6 +8,12 @@ package main import ( "context" + "github.com/harness/gitness/build/commit" + "github.com/harness/gitness/build/file" + "github.com/harness/gitness/build/manager" + "github.com/harness/gitness/build/runner" + "github.com/harness/gitness/build/scheduler" + "github.com/harness/gitness/build/triggerer" "github.com/harness/gitness/cli/server" "github.com/harness/gitness/encrypt" "github.com/harness/gitness/events" @@ -99,8 +105,21 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro } repoController := repo.ProvideController(config, db, provider, pathUID, authorizer, pathStore, repoStore, spaceStore, pipelineStore, principalStore, gitrpcInterface) executionStore := database.ProvideExecutionStore(db) + commitService := commit.ProvideCommitService(gitrpcInterface) stageStore := database.ProvideStageStore(db) - executionController := execution.ProvideController(db, authorizer, executionStore, repoStore, stageStore, pipelineStore) + fileService := file.ProvideFileService(gitrpcInterface) + lockConfig := server.ProvideLockConfig(config) + universalClient, err := server.ProvideRedis(config) + if err != nil { + return nil, err + } + mutexManager := lock.ProvideMutexManager(lockConfig, universalClient) + schedulerScheduler, err := scheduler.ProvideScheduler(stageStore, mutexManager) + if err != nil { + return nil, err + } + triggererTriggerer := triggerer.ProvideTriggerer(executionStore, stageStore, db, pipelineStore, fileService, schedulerScheduler, repoStore) + executionController := execution.ProvideController(db, authorizer, executionStore, commitService, triggererTriggerer, repoStore, stageStore, pipelineStore) stepStore := database.ProvideStepStore(db) logStore := logs.ProvideLogStore(db, config) logStream := livelog.ProvideLogStream(config) @@ -130,10 +149,6 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro if err != nil { return nil, err } - universalClient, err := server.ProvideRedis(config) - if err != nil { - return nil, err - } eventsSystem, err := events.ProvideSystem(eventsConfig, universalClient) if err != nil { return nil, err @@ -142,8 +157,6 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro if err != nil { return nil, err } - lockConfig := server.ProvideLockConfig(config) - mutexManager := lock.ProvideMutexManager(lockConfig, universalClient) migrator := codecomments.ProvideMigrator(gitrpcInterface) pullreqController := pullreq.ProvideController(db, provider, authorizer, pullReqStore, pullReqActivityStore, codeCommentView, pullReqReviewStore, pullReqReviewerStore, repoStore, principalStore, gitrpcInterface, reporter, mutexManager, migrator) webhookConfig, err := server.ProvideWebhookConfig() @@ -180,6 +193,13 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro webHandler := router.ProvideWebHandler(config) routerRouter := router.ProvideRouter(config, apiHandler, gitHandler, webHandler) serverServer := server2.ProvideServer(config, routerRouter) + executionManager := manager.ProvideExecutionManager(config, executionStore, pipelineStore, provider, fileService, logStore, logStream, repoStore, schedulerScheduler, secretStore, stageStore, stepStore, principalStore) + client := manager.ProvideExecutionClient(executionManager, config) + runtimeRunner, err := runner.ProvideExecutionRunner(config, client, executionManager) + if err != nil { + return nil, err + } + poller := runner.ProvideExecutionPoller(runtimeRunner, config, client) serverConfig, err := server.ProvideGitRPCServerConfig() if err != nil { return nil, err @@ -194,7 +214,7 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro if err != nil { return nil, err } - manager := cron.ProvideManager(serverConfig) + cronManager := cron.ProvideManager(serverConfig) repoGitInfoView := database.ProvideRepoGitInfoView(db) repoGitInfoCache := cache.ProvideRepoGitInfoCache(repoGitInfoView) pubsubConfig := pubsub.ProvideConfig(config) @@ -205,11 +225,11 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro } jobStore := database.ProvideJobStore(db) executor := job.ProvideExecutor(jobStore, pubSub) - scheduler, err := job.ProvideScheduler(jobStore, executor, mutexManager, pubSub, config) + jobScheduler, err := job.ProvideScheduler(jobStore, executor, mutexManager, pubSub, config) if err != nil { return nil, err } - servicesServices := services.ProvideServices(webhookService, pullreqService, executor, scheduler) - serverSystem := server.NewSystem(bootstrapBootstrap, serverServer, grpcServer, manager, servicesServices) + servicesServices := services.ProvideServices(webhookService, pullreqService, executor, jobScheduler) + serverSystem := server.NewSystem(bootstrapBootstrap, serverServer, poller, grpcServer, cronManager, servicesServices) return serverSystem, nil } diff --git a/go.mod b/go.mod index f960b03fd..0e45cc729 100644 --- a/go.mod +++ b/go.mod @@ -2,14 +2,22 @@ module github.com/harness/gitness go 1.19 +replace github.com/docker/docker => github.com/docker/engine v17.12.0-ce-rc1.0.20200309214505-aa6a9891b09c+incompatible + require ( code.gitea.io/gitea v1.17.2 github.com/Masterminds/squirrel v1.5.1 github.com/adrg/xdg v0.3.2 + github.com/aws/aws-sdk-go v1.44.322 github.com/coreos/go-semver v0.3.0 github.com/dchest/uniuri v0.0.0-20200228104902-7aecb25e1fe5 github.com/dgrijalva/jwt-go v3.2.0+incompatible + github.com/drone-runners/drone-runner-docker v1.8.3 + github.com/drone/drone-go v1.7.1 + github.com/drone/drone-yaml v1.2.3 github.com/drone/funcmap v0.0.0-20190918184546-d4ef6e88376d + github.com/drone/go-scm v1.31.2 + github.com/drone/runner-go v1.12.0 github.com/go-chi/chi v1.5.4 github.com/go-chi/cors v1.2.1 github.com/go-redis/redis/v8 v8.11.5 @@ -17,6 +25,7 @@ require ( github.com/golang-jwt/jwt v3.2.2+incompatible github.com/golang/mock v1.6.0 github.com/google/go-cmp v0.5.9 + github.com/google/uuid v1.3.1 github.com/google/wire v0.5.0 github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75 github.com/gotidy/ptr v1.3.0 @@ -38,6 +47,7 @@ require ( github.com/rs/xid v1.4.0 github.com/rs/zerolog v1.29.0 github.com/sercand/kuberesolver/v5 v5.1.0 + github.com/sirupsen/logrus v1.8.1 github.com/stretchr/testify v1.8.1 github.com/swaggest/openapi-go v0.2.23 github.com/swaggest/swgui v1.4.2 @@ -57,14 +67,27 @@ require ( cloud.google.com/go v0.110.0 // indirect cloud.google.com/go/compute v1.18.0 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect - github.com/aws/aws-sdk-go v1.44.322 // indirect + github.com/99designs/httpsignatures-go v0.0.0-20170731043157-88528bf4ca7e // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/bmatcuk/doublestar v1.1.1 // indirect + github.com/buildkite/yaml v2.1.0+incompatible // indirect + github.com/containerd/containerd v1.3.4 // indirect + github.com/docker/distribution v2.7.1+incompatible // indirect + github.com/docker/docker v0.0.0-00010101000000-000000000000 // indirect + github.com/docker/go-connections v0.3.0 // indirect + github.com/docker/go-units v0.4.0 // indirect + github.com/drone/envsubst v1.0.3 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect + github.com/ghodss/yaml v1.0.0 // indirect + github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect github.com/googleapis/gax-go/v2 v2.7.0 // indirect github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect + github.com/natessilva/dag v0.0.0-20180124060714-7194b8dcc5c4 // indirect + github.com/opencontainers/go-digest v1.0.0-rc1 // indirect + github.com/opencontainers/image-spec v1.0.1 // indirect github.com/prometheus/client_golang v1.15.1 // indirect github.com/prometheus/client_model v0.3.0 // indirect github.com/prometheus/common v0.42.0 // indirect @@ -99,7 +122,7 @@ require ( github.com/go-enry/go-oniguruma v1.2.1 // indirect github.com/go-git/gcfg v1.5.0 // indirect github.com/go-git/go-billy/v5 v5.3.1 // indirect - github.com/go-git/go-git/v5 v5.4.3-0.20210630082519-b4368b2a2ca4 // indirect + github.com/go-git/go-git/v5 v5.4.3-0.20210630082519-b4368b2a2ca4 github.com/gobwas/glob v0.2.3 // indirect github.com/golang-jwt/jwt/v4 v4.4.1 // indirect github.com/golang/protobuf v1.5.3 // indirect @@ -133,7 +156,7 @@ require ( google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect gopkg.in/ini.v1 v1.66.4 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect + gopkg.in/yaml.v2 v2.4.0 gopkg.in/yaml.v3 v3.0.1 // indirect strk.kbt.io/projects/go/libravatar v0.0.0-20191008002943-06d1c002b251 // indirect ) diff --git a/go.sum b/go.sum index d8c1eb41f..b03ebae09 100644 --- a/go.sum +++ b/go.sum @@ -13,11 +13,18 @@ cloud.google.com/go/profiler v0.3.1/go.mod h1:GsG14VnmcMFQ9b+kq71wh3EKMZr3WRMgLz cloud.google.com/go/storage v1.28.1 h1:F5QDG5ChchaAVQhINh24U99OWHURqrW8OmQcGKXcbgI= code.gitea.io/gitea v1.17.2 h1:NRcVr07jF+za4d0NZZlJXeCuQK5FfHMtjPDjq4u3UiY= code.gitea.io/gitea v1.17.2/go.mod h1:sovminOoSsc8IC2T29rX9+MmaboHTu8QDEvJjaSqIXg= +docker.io/go-docker v1.0.0/go.mod h1:7tiAn5a0LFmjbPDbyTPOaTTOuG1ZRNXdPA6RvKY+fpY= +github.com/99designs/basicauth-go v0.0.0-20160802081356-2a93ba0f464d/go.mod h1:3cARGAK9CfW3HoxCy1a0G4TKrdiKke8ftOMEOHyySYs= +github.com/99designs/httpsignatures-go v0.0.0-20170731043157-88528bf4ca7e h1:rl2Aq4ZODqTDkeSqQBy+fzpZPamacO1Srp8zq7jf2Sc= +github.com/99designs/httpsignatures-go v0.0.0-20170731043157-88528bf4ca7e/go.mod h1:Xa6lInWHNQnuWoF0YPSsx+INFA9qk7/7pTjwb3PInkY= +github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= +github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78/go.mod h1:LmzpDX56iTiv29bbRTIsUNlaFfuhWRQBWjQdVyAevI8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs= github.com/Masterminds/squirrel v1.5.1 h1:kWAKlLLJFxZG7N2E0mBMNWVp5AuUX+JUrnhFN74Eg+w= github.com/Masterminds/squirrel v1.5.1/go.mod h1:NNaOrjSoIDfDA40n7sr2tPNZRfjzjA400rg+riTZj10= +github.com/Microsoft/go-winio v0.4.11/go.mod h1:VhR8bwka0BXejwEJY73c50VrPtXAaKcyvVC4A4RozmA= github.com/Microsoft/go-winio v0.4.14/go.mod h1:qXqCSQ3Xa7+6tgxaGTIe4Kpcdsi+P8jBhyzoq1bpyYA= github.com/Microsoft/go-winio v0.4.16/go.mod h1:XB6nPKklQyQ7GC9LdcBEcBl8PF76WugXOPRXwdLnMv0= github.com/Microsoft/go-winio v0.5.0/go.mod h1:JPGBdM1cNvN/6ISo+n8V5iA4v8pBzdOpzfwIujj1a84= @@ -63,10 +70,14 @@ github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+Ce github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bmatcuk/doublestar v1.1.1 h1:YroD6BJCZBYx06yYFEWvUuKVWQn3vLLQAVmDmvTSaiQ= +github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= github.com/bool64/dev v0.1.41/go.mod h1:cTHiTDNc8EewrQPy3p1obNilpMpdmlUesDkFTF2zRWU= github.com/bool64/dev v0.1.42/go.mod h1:cTHiTDNc8EewrQPy3p1obNilpMpdmlUesDkFTF2zRWU= github.com/bool64/dev v0.2.22 h1:YJFKBRKplkt+0Emq/5Xk1Z5QRmMNzc1UOJkR3rxJksA= github.com/bool64/shared v0.1.5 h1:fp3eUhBsrSjNCQPcSdQqZxxh9bBwrYiZ+zOKFkM0/2E= +github.com/buildkite/yaml v2.1.0+incompatible h1:xirI+ql5GzfikVNDmt+yeiXpf/v1Gt03qXTtT5WXdr8= +github.com/buildkite/yaml v2.1.0+incompatible/go.mod h1:UoU8vbcwu1+vjZq01+KrpSeLBgQQIjL/H7Y6KwikUrI= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= @@ -83,6 +94,8 @@ github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGX github.com/cockroachdb/apd v1.1.0/go.mod h1:8Sl8LxpKi29FqWXR16WEFZRNSz3SoPzUzeMeY4+DwBQ= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= +github.com/containerd/containerd v1.3.4 h1:3o0smo5SKY7H6AJCmJhsnCjR2/V2T8VmiHt7seN2/kI= +github.com/containerd/containerd v1.3.4/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -97,6 +110,7 @@ github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ3 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dchest/uniuri v0.0.0-20160212164326-8902c56451e9/go.mod h1:GgB8SF9nRG+GqaDtLcwJZsQFhcogVCJ79j4EdT0c2V4= github.com/dchest/uniuri v0.0.0-20200228104902-7aecb25e1fe5 h1:RAV05c0xOkJ3dZGS0JFybxFKZ2WMLabgx3uXnd7rpGs= github.com/dchest/uniuri v0.0.0-20200228104902-7aecb25e1fe5/go.mod h1:GgB8SF9nRG+GqaDtLcwJZsQFhcogVCJ79j4EdT0c2V4= github.com/dgrijalva/jwt-go v3.2.0+incompatible h1:7qlOGliEKZXTDg6OTjfoBKDXWrumCAMpl/TFQ4/5kLM= @@ -108,8 +122,33 @@ github.com/djherbis/buffer v1.2.0 h1:PH5Dd2ss0C7CRRhQCZ2u7MssF+No9ide8Ye71nPHcrQ github.com/djherbis/buffer v1.2.0/go.mod h1:fjnebbZjCUpPinBRD+TDwXSOeNQ7fPQWLfGQqiAiUyE= github.com/djherbis/nio/v3 v3.0.1 h1:6wxhnuppteMa6RHA4L81Dq7ThkZH8SwnDzXDYy95vB4= github.com/djherbis/nio/v3 v3.0.1/go.mod h1:Ng4h80pbZFMla1yKzm61cF0tqqilXZYrogmWgZxOcmg= +github.com/docker/distribution v0.0.0-20170726174610-edc3ab29cdff/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/distribution v2.7.1+incompatible h1:a5mlkVzth6W5A4fOsS3D2EO5BUmsJpcB+cRlLU7cSug= +github.com/docker/distribution v2.7.1+incompatible/go.mod h1:J2gT2udsDAN96Uj4KfcMRqY0/ypR+oyYUYmja8H+y+w= +github.com/docker/engine v17.12.0-ce-rc1.0.20200309214505-aa6a9891b09c+incompatible h1:hx8H7MbcmXUXAmphQuA/XB7CfSzX4DRrNuHFvfK9aIQ= +github.com/docker/engine v17.12.0-ce-rc1.0.20200309214505-aa6a9891b09c+incompatible/go.mod h1:3CPr2caMgTHxxIAZgEMd3uLYPDlRvPqCpyeRf6ncPcY= +github.com/docker/go-connections v0.3.0 h1:3lOnM9cSzgGwx8VfK/NGOW5fLQ0GjIlCkaktF+n1M6o= +github.com/docker/go-connections v0.3.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= +github.com/docker/go-units v0.3.3/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/drone-runners/drone-runner-docker v1.8.3 h1:uUnC45C1JMSLW+9uy6RoKG5ugzeXWN89pygs9BMLObY= +github.com/drone-runners/drone-runner-docker v1.8.3/go.mod h1:JR3pZeVZKKpkbTajiq0YtAx9WutkODdVKZGNR83kEwE= +github.com/drone/drone-go v1.7.1 h1:ZX+3Rs8YHUSUQ5mkuMLmm1zr1ttiiE2YGNxF3AnyDKw= +github.com/drone/drone-go v1.7.1/go.mod h1:fxCf9jAnXDZV1yDr0ckTuWd1intvcQwfJmTRpTZ1mXg= +github.com/drone/drone-runtime v1.0.7-0.20190729202838-87c84080f4a1/go.mod h1:+osgwGADc/nyl40J0fdsf8Z09bgcBZXvXXnLOY48zYs= +github.com/drone/drone-yaml v1.2.3 h1:SWzLmzr8ARhbtw1WsVDENa8WFY2Pi9l0FVMfafVUWz8= +github.com/drone/drone-yaml v1.2.3/go.mod h1:QsqliFK8nG04AHFN9tTn9XJomRBQHD4wcejWW1uz/10= +github.com/drone/envsubst v1.0.2/go.mod h1:bkZbnc/2vh1M12Ecn7EYScpI4YGYU0etwLJICOWi8Z0= +github.com/drone/envsubst v1.0.3 h1:PCIBwNDYjs50AsLZPYdfhSATKaRg/FJmDc2D6+C2x8g= +github.com/drone/envsubst v1.0.3/go.mod h1:N2jZmlMufstn1KEqvbHjw40h1KyTmnVzHcSc9bFiJ2g= github.com/drone/funcmap v0.0.0-20190918184546-d4ef6e88376d h1:/IO7UVVu191Jc0DajV4cDVoO+91cuppvgxg2MZl+AXI= github.com/drone/funcmap v0.0.0-20190918184546-d4ef6e88376d/go.mod h1:Hph0/pT6ZxbujnE1Z6/08p5I0XXuOsppqF6NQlGOK0E= +github.com/drone/go-scm v1.31.2 h1:6hZxf0aETV17830fMCPrgcA4y8j/8Gdfy0xEdInUeqQ= +github.com/drone/go-scm v1.31.2/go.mod h1:DFIJJjhMj0TSXPz+0ni4nyZ9gtTtC40Vh/TGRugtyWw= +github.com/drone/runner-go v1.12.0 h1:zUjDj9ylsJ4n4Mvy4znddq/Z4EBzcUXzTltpzokKtgs= +github.com/drone/runner-go v1.12.0/go.mod h1:vu4pPPYDoeN6vdYQAY01GGGsAIW4aLganJNaa8Fx8zE= +github.com/drone/signal v1.0.0/go.mod h1:S8t92eFT0g4WUgEc/LxG+LCuiskpMNsG0ajAMGnyZpc= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= @@ -131,6 +170,7 @@ github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMo github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/ghodss/yaml v1.0.0 h1:wQHKEahhL6wmXdzwWG11gIVCkOv05bNOh+Rxn0yngAk= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.2.2/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/gliderlabs/ssh v0.3.4 h1:+AXBtim7MTKaLVPgvE+3mhewYRawNLTd+jEEz/wExZw= @@ -187,9 +227,11 @@ github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5x github.com/gofrs/uuid v3.2.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= +github.com/gogo/protobuf v0.0.0-20170307180453-100ba4e88506/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4= +github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= github.com/gogo/protobuf v1.3.2/go.mod h1:P1XiOD3dCwIKUDQYPy72D8LYyHL2YPYrpS2s69NZV8Q= github.com/golang-jwt/jwt v3.2.2+incompatible h1:IfV12K8xAKAnZqdXVzCZ+TOjboZ2keLg81eXfW3O+oY= github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= @@ -237,6 +279,7 @@ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20221103000818-d260c55eee4c h1:lvddKcYTQ545ADhBujtIJmqQrZBDsGo7XIMbAQe/sNY= @@ -246,22 +289,27 @@ github.com/google/subcommands v1.0.1 h1:/eqq+otEXm5vhfBrbREPCSVQbvofip6kIz+mX5TU github.com/google/subcommands v1.0.1/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/wire v0.5.0 h1:I7ELFeVBr3yfPIcc8+MWvrjk+3VjbcSzoXm3JVa+jD8= github.com/google/wire v0.5.0/go.mod h1:ngWDr9Qvq3yZA10YrxfyGELY/AFWGVpy9c1LTRi1EoU= github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k= github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ= github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= +github.com/googleapis/gnostic v0.2.0/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75 h1:f0n1xnMSmBLzVfsMMvriDyA75NB/oBgILX2GcHXIQzY= github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75/go.mod h1:g2644b03hfBX9Ov0ZBDgXXens4rxSxmqFBbhvKv2yVA= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gotidy/ptr v1.3.0 h1:5wdrH1G8X4txy6fbWWRznr7k974wMWtePWP3p6s1API= github.com/gotidy/ptr v1.3.0/go.mod h1:vpltyHhOZE+NGXUiwpVl3wV9AGEBlxhdnaimPDxRLxg= +github.com/gregjones/httpcache v0.0.0-20181110185634-c63ab54fda8f/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 h1:+9834+KizmvFV7pXQGSXQTsaWhq2GjuNUt0aUU0YBYw= github.com/grpc-ecosystem/go-grpc-middleware v1.3.0/go.mod h1:z0ButlSOZa5vEBq9m2m2hlwIgKw+rp3sdCBRoJY+30Y= @@ -269,6 +317,7 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgf github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/guregu/null v4.0.0+incompatible h1:4zw0ckM7ECd6FNNddc3Fu4aty9nTlpkkzH7dPn4/4Gw= github.com/guregu/null v4.0.0+incompatible/go.mod h1:ePGpQaN9cw0tj45IR5E5ehMvsFlLlQZAkkOXZurJ3NM= +github.com/h2non/gock v1.0.9/go.mod h1:CZMcB0Lg5IWnr9bF79pPMg9WeV6WumxQiUJ1UvdO1iE= github.com/harness/go-rbac v0.0.0-20230829014129-c9b217856ea2 h1:M1Jd2uEKl4YW9g/6vzN1qo06d5dshYYdwxlhOTUSnh4= github.com/harness/go-rbac v0.0.0-20230829014129-c9b217856ea2/go.mod h1:uGgBgSZPgyygG5rWzoYsKIQ8TM4zt5yQq9nreznWvOI= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= @@ -301,6 +350,7 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/iancoleman/orderedmap v0.2.0 h1:sq1N/TFpYH++aViPcaKjys3bDClUEU7s5B+z6jq8pNA= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= @@ -376,6 +426,7 @@ github.com/jmoiron/sqlx v1.3.3/go.mod h1:2BljVx/86SuTyjE+aPYlHCTNvZrnJXghYGpNiXL github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= +github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -470,7 +521,11 @@ github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lN github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0= github.com/modern-go/reflect2 v1.0.2 h1:xBagoLtFs94CBntxluKeaWgTMpvLxC4ur3nMaC9Gz0M= github.com/modern-go/reflect2 v1.0.2/go.mod h1:yWuevngMOJpCy52FWWMvUC8ws7m/LJsjYzDa0/r8luk= +github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= +github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/natessilva/dag v0.0.0-20180124060714-7194b8dcc5c4 h1:dnMxwus89s86tI8rcGVp2HwZzlz7c5o92VOy7dSckBQ= +github.com/natessilva/dag v0.0.0-20180124060714-7194b8dcc5c4/go.mod h1:cojhOHk1gbMeklOyDP2oKKLftefXoJreOQGOrXk+Z38= github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= github.com/nats-io/nats-server/v2 v2.1.2/go.mod h1:Afk+wRZqkMQs/p45uXdrVLuab3gwv3Z8C4HTBu8GD/k= @@ -478,6 +533,7 @@ github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzE github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= +github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/nxadm/tail v1.4.8 h1:nPr65rt6Y5JFSKQO7qToXr7pePgD6Gwiw05lkbyAQTE= @@ -504,6 +560,10 @@ github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9 github.com/onsi/gomega v1.20.0 h1:8W0cWlwFkflGPLltQvLRB7ZVD5HuP6ng320w2IS245Q= github.com/onsi/gomega v1.20.0/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= +github.com/opencontainers/go-digest v1.0.0-rc1 h1:WzifXhOVOEOuFYOJAW6aQqW0TooG2iki3E3Ii+WN7gQ= +github.com/opencontainers/go-digest v1.0.0-rc1/go.mod h1:cMLVZDEM3+U2I4VmLI6N8jQYUd2OVphdqWwCJHrFt2s= +github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= +github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= @@ -516,6 +576,8 @@ github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIw github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= +github.com/petar/GoLLRB v0.0.0-20130427215148-53be0d36a84c/go.mod h1:HUpKUBZnpzkdx0kD/+Yfuft+uD3zHGtXF/XJB14TUr4= +github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -570,6 +632,7 @@ github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdh github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= github.com/sercand/kuberesolver/v5 v5.1.0 h1:YLqreB1vUFbZHSidcqI5ChMp+RIRmop0brQOeUFWiRw= github.com/sercand/kuberesolver/v5 v5.1.0/go.mod h1:Fs1KbKhVRnB2aDWN12NjKCB+RgYMWZJ294T3BtmVCpQ= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= github.com/sergi/go-diff v1.2.0 h1:XU+rvMAioB0UC3q1MFrIQy4Vo5/4VsRDQQXHsEya6xQ= github.com/sergi/go-diff v1.2.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= @@ -584,12 +647,15 @@ github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPx github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= github.com/sony/gobreaker v0.4.1/go.mod h1:ZKptC7FHNvhBz7dN2LGjPVBz2sZJmc0/PkyDJOjmxWY= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= @@ -628,6 +694,7 @@ github.com/urfave/negroni v1.0.0 h1:kIimOitoypq34K7TG7DUaJ9kq/N4Ofuwi1sjz0KipXc= github.com/urfave/negroni v1.0.0/go.mod h1:Meg73S6kFm/4PpbYdq35yYWoCZ9mS/YSx+lKnmiohz4= github.com/vearutop/statigz v1.1.5 h1:qWvRgXFsseWVTFCkIvwHQPpaLNf9WI0+dDJE7I9432o= github.com/vearutop/statigz v1.1.5/go.mod h1:czAv7iXgPv/s+xsgXpVEhhD0NSOQ4wZPgmM/n7LANDI= +github.com/vinzenz/yaml v0.0.0-20170920082545-91409cdd725d/go.mod h1:mb5taDqMnJiZNRQ3+02W2IFG+oEz1+dTuCXkp4jpkfo= github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0= github.com/xanzy/ssh-agent v0.3.1 h1:AmzO1SSWxw73zxFZPRwaMN1MohDw8UyHnmuxyceTEGo= github.com/xanzy/ssh-agent v0.3.1/go.mod h1:QIE4lCeL7nkC25x+yA3LBIYfwCc1TFziCtG7cBAac6w= @@ -667,10 +734,12 @@ go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190411191339-88737f569e3a/go.mod h1:WFFai1msRO1wXaEeE5yQxYXgSfI8pQAWXbQop6sCtWE= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190621222207-cc06ce4a13d4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190820162420-60c769a6c586/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190911031432-227b76d455e7/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -706,6 +775,7 @@ golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -738,6 +808,7 @@ golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.6.0 h1:Lh8GPgSKBfWSwFvtuWOfeI3aAAnbXTSutYxJiOJFgIw= golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= @@ -756,6 +827,7 @@ golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20181005133103-4497e2df6f9e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -823,7 +895,9 @@ golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20220411224347-583f2d630306 h1:+gHMid33q6pen7kv9xvT+JRinntgeXO2AeZVd0AWD3w= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -862,6 +936,7 @@ google.golang.org/api v0.110.0 h1:l+rh0KYUooe9JGbGVx71tbFo4SMbMTXK3I3ia2QSEeU= google.golang.org/api v0.110.0/go.mod h1:7FC4Vvx1Mooxh8C5HWjzZHcavuS2f6pmJpZx60ca7iI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= @@ -916,6 +991,7 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/inconshreveable/log15.v2 v2.0.0-20180818164646-67afb5ed74ec/go.mod h1:aPpfJ7XW+gOuirDoZ8gHhLh3kZ1B08FtV2bbmy7Jv3s= +gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.66.4 h1:SsAcf+mM7mRZo2nJNGt8mZCjG8ZRaNGMURJw7BsIST4= gopkg.in/ini.v1 v1.66.4/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= @@ -935,10 +1011,16 @@ gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= +gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= +k8s.io/api v0.0.0-20181130031204-d04500c8c3dd/go.mod h1:iuAfoD4hCxJ8Onx9kaTIt30j7jUFS00AXQi6QMi99vA= +k8s.io/apimachinery v0.0.0-20181201231028-18a5ff3097b4/go.mod h1:ccL7Eh7zubPUSh9A3USN90/OzHNSVN6zxzde07TDCL0= +k8s.io/client-go v9.0.0+incompatible/go.mod h1:7vJpHMYJwNQCWgzmNV+VYUl1zCObLyodBc8nIyt8L5s= +k8s.io/klog v0.1.0/go.mod h1:Gq+BEi5rUBO/HRz0bTSXDUcqjScdoY3a9IHpCEIOOfk= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sourcegraph.com/sourcegraph/appdash v0.0.0-20190731080439-ebfcffb1b5c0/go.mod h1:hI742Nqp5OhwiqlzhgfbWU4mW4yO10fP+LoT9WOswdU= strk.kbt.io/projects/go/libravatar v0.0.0-20191008002943-06d1c002b251 h1:mUcz5b3FJbP5Cvdq7Khzn6J9OCUQJaBwgBkCR+MOwSs= diff --git a/internal/api/auth/pipeline.go b/internal/api/auth/pipeline.go index 8d012b8ca..38ce08f47 100644 --- a/internal/api/auth/pipeline.go +++ b/internal/api/auth/pipeline.go @@ -12,6 +12,7 @@ import ( "github.com/harness/gitness/internal/paths" "github.com/harness/gitness/types" "github.com/harness/gitness/types/enum" + "github.com/pkg/errors" ) diff --git a/internal/api/controller/execution/controller.go b/internal/api/controller/execution/controller.go index 00753c52a..6a180ff38 100644 --- a/internal/api/controller/execution/controller.go +++ b/internal/api/controller/execution/controller.go @@ -5,6 +5,8 @@ package execution import ( + "github.com/harness/gitness/build/commit" + "github.com/harness/gitness/build/triggerer" "github.com/harness/gitness/internal/auth/authz" "github.com/harness/gitness/internal/store" @@ -15,6 +17,8 @@ type Controller struct { db *sqlx.DB authorizer authz.Authorizer executionStore store.ExecutionStore + commitService commit.CommitService + triggerer triggerer.Triggerer repoStore store.RepoStore stageStore store.StageStore pipelineStore store.PipelineStore @@ -24,6 +28,8 @@ func NewController( db *sqlx.DB, authorizer authz.Authorizer, executionStore store.ExecutionStore, + commitService commit.CommitService, + triggerer triggerer.Triggerer, repoStore store.RepoStore, stageStore store.StageStore, pipelineStore store.PipelineStore, @@ -32,6 +38,8 @@ func NewController( db: db, authorizer: authorizer, executionStore: executionStore, + commitService: commitService, + triggerer: triggerer, repoStore: repoStore, stageStore: stageStore, pipelineStore: pipelineStore, diff --git a/internal/api/controller/execution/create.go b/internal/api/controller/execution/create.go index 662105c87..e5116e353 100644 --- a/internal/api/controller/execution/create.go +++ b/internal/api/controller/execution/create.go @@ -7,25 +7,22 @@ package execution import ( "context" "fmt" - "time" + "github.com/harness/gitness/build/triggerer" apiauth "github.com/harness/gitness/internal/api/auth" "github.com/harness/gitness/internal/auth" "github.com/harness/gitness/types" "github.com/harness/gitness/types/enum" -) -// TODO: Add more as needed. -type CreateInput struct { - Status string `json:"status"` -} + "github.com/drone/go-scm/scm" +) func (c *Controller) Create( ctx context.Context, session *auth.Session, repoRef string, pipelineUID string, - in *CreateInput, + branch string, ) (*types.Execution, error) { repo, err := c.repoStore.FindByRef(ctx, repoRef) if err != nil { @@ -42,25 +39,38 @@ func (c *Controller) Create( return nil, fmt.Errorf("failed to find pipeline: %w", err) } - pipeline, err = c.pipelineStore.IncrementSeqNum(ctx, pipeline) + // If the branch is empty, use the default branch specified in the pipeline. + if branch == "" { + branch = pipeline.DefaultBranch + } + // expand the branch to a git reference. + ref := scm.ExpandRef(branch, "refs/heads") + + // Fetch the commit information from the commits service. + commit, err := c.commitService.FindRef(ctx, repo, ref) if err != nil { - return nil, fmt.Errorf("failed to increment sequence number: %w", err) + return nil, fmt.Errorf("failed to fetch commit: %w", err) } - now := time.Now().UnixMilli() - execution := &types.Execution{ - Number: pipeline.Seq, - Status: in.Status, - RepoID: pipeline.RepoID, - PipelineID: pipeline.ID, - Created: now, - Updated: now, - Version: 0, - } - err = c.executionStore.Create(ctx, execution) - if err != nil { - return nil, fmt.Errorf("execution creation failed: %w", err) + // Create manual hook for execution. + hook := &triggerer.Hook{ + Trigger: session.Principal.UID, // who/what triggered the build, different from commit author + Author: commit.Author.Identity.Name, + AuthorName: commit.Author.Identity.Name, + AuthorEmail: commit.Author.Identity.Email, + Ref: ref, + Message: commit.Message, + Title: "", // we expect this to be empty. + Before: commit.SHA, + After: commit.SHA, + Sender: session.Principal.UID, + Source: branch, + Target: branch, + Action: types.EventCustom, + Params: map[string]string{}, + Timestamp: commit.Author.When.UnixMilli(), } - return execution, nil + // Trigger the execution + return c.triggerer.Trigger(ctx, pipeline, hook) } diff --git a/internal/api/controller/execution/find.go b/internal/api/controller/execution/find.go index 686c2c1f9..9600f3f5b 100644 --- a/internal/api/controller/execution/find.go +++ b/internal/api/controller/execution/find.go @@ -34,7 +34,8 @@ func (c *Controller) Find( if err != nil { return nil, fmt.Errorf("failed to find pipeline: %w", err) } - execution, err := c.executionStore.Find(ctx, pipeline.ID, executionNum) + + execution, err := c.executionStore.FindByNumber(ctx, pipeline.ID, executionNum) if err != nil { return nil, fmt.Errorf("failed to find execution %d: %w", executionNum, err) } diff --git a/internal/api/controller/execution/update.go b/internal/api/controller/execution/update.go index 7c2184b0a..ff5ac30fd 100644 --- a/internal/api/controller/execution/update.go +++ b/internal/api/controller/execution/update.go @@ -40,7 +40,7 @@ func (c *Controller) Update( return nil, fmt.Errorf("failed to find pipeline: %w", err) } - execution, err := c.executionStore.Find(ctx, pipeline.ID, executionNum) + execution, err := c.executionStore.FindByNumber(ctx, pipeline.ID, executionNum) if err != nil { return nil, fmt.Errorf("failed to find execution: %w", err) } diff --git a/internal/api/controller/execution/wire.go b/internal/api/controller/execution/wire.go index 29d6449ea..97b85b29c 100644 --- a/internal/api/controller/execution/wire.go +++ b/internal/api/controller/execution/wire.go @@ -5,6 +5,8 @@ package execution import ( + "github.com/harness/gitness/build/commit" + "github.com/harness/gitness/build/triggerer" "github.com/harness/gitness/internal/auth/authz" "github.com/harness/gitness/internal/store" @@ -20,10 +22,12 @@ var WireSet = wire.NewSet( func ProvideController(db *sqlx.DB, authorizer authz.Authorizer, executionStore store.ExecutionStore, + commitService commit.CommitService, + triggerer triggerer.Triggerer, repoStore store.RepoStore, stageStore store.StageStore, pipelineStore store.PipelineStore, ) *Controller { - return NewController(db, authorizer, executionStore, repoStore, stageStore, - pipelineStore) + return NewController(db, authorizer, executionStore, commitService, + triggerer, repoStore, stageStore, pipelineStore) } diff --git a/internal/api/controller/logs/find.go b/internal/api/controller/logs/find.go index 3ff681bf3..82e3d766e 100644 --- a/internal/api/controller/logs/find.go +++ b/internal/api/controller/logs/find.go @@ -5,12 +5,14 @@ package logs import ( + "bytes" "context" + "encoding/json" "fmt" - "io" apiauth "github.com/harness/gitness/internal/api/auth" "github.com/harness/gitness/internal/auth" + "github.com/harness/gitness/livelog" "github.com/harness/gitness/types/enum" ) @@ -22,7 +24,7 @@ func (c *Controller) Find( executionNum int64, stageNum int, stepNum int, -) (io.ReadCloser, error) { +) ([]*livelog.Line, error) { repo, err := c.repoStore.FindByRef(ctx, repoRef) if err != nil { return nil, fmt.Errorf("failed to find repo by ref: %w", err) @@ -37,7 +39,7 @@ func (c *Controller) Find( return nil, fmt.Errorf("failed to find pipeline: %w", err) } - execution, err := c.executionStore.Find(ctx, pipeline.ID, executionNum) + execution, err := c.executionStore.FindByNumber(ctx, pipeline.ID, executionNum) if err != nil { return nil, fmt.Errorf("failed to find execution: %w", err) } @@ -52,5 +54,20 @@ func (c *Controller) Find( return nil, fmt.Errorf("failed to find step: %w", err) } - return c.logStore.Find(ctx, step.ID) + rc, err := c.logStore.Find(ctx, step.ID) + if err != nil { + return nil, fmt.Errorf("could not find logs: %w", err) + } + defer rc.Close() + + lines := []*livelog.Line{} + buf := new(bytes.Buffer) + buf.ReadFrom(rc) + + err = json.Unmarshal(buf.Bytes(), &lines) + if err != nil { + return nil, fmt.Errorf("could not unmarshal logs: %w", err) + } + + return lines, nil } diff --git a/internal/api/controller/logs/tail.go b/internal/api/controller/logs/tail.go index 2e2a37467..edb477923 100644 --- a/internal/api/controller/logs/tail.go +++ b/internal/api/controller/logs/tail.go @@ -36,7 +36,7 @@ func (c *Controller) Tail( return nil, nil, fmt.Errorf("failed to find pipeline: %w", err) } - execution, err := c.executionStore.Find(ctx, pipeline.ID, executionNum) + execution, err := c.executionStore.FindByNumber(ctx, pipeline.ID, executionNum) if err != nil { return nil, nil, fmt.Errorf("failed to find execution: %w", err) } diff --git a/internal/api/handler/execution/create.go b/internal/api/handler/execution/create.go index 348972237..c274e4112 100644 --- a/internal/api/handler/execution/create.go +++ b/internal/api/handler/execution/create.go @@ -5,7 +5,6 @@ package execution import ( - "encoding/json" "net/http" "github.com/harness/gitness/internal/api/controller/execution" @@ -28,14 +27,9 @@ func HandleCreate(executionCtrl *execution.Controller) http.HandlerFunc { return } - in := new(execution.CreateInput) - err = json.NewDecoder(r.Body).Decode(in) - if err != nil { - render.BadRequestf(w, "Invalid Request Body: %s.", err) - return - } + branch := request.GetBranchFromQuery(r) - execution, err := executionCtrl.Create(ctx, session, repoRef, pipelineUID, in) + execution, err := executionCtrl.Create(ctx, session, repoRef, pipelineUID, branch) if err != nil { render.TranslatedUserError(w, err) return diff --git a/internal/api/handler/logs/find.go b/internal/api/handler/logs/find.go index b1e7b469c..ef61c7961 100644 --- a/internal/api/handler/logs/find.go +++ b/internal/api/handler/logs/find.go @@ -5,7 +5,6 @@ package logs import ( - "io" "net/http" "github.com/harness/gitness/internal/api/controller/logs" @@ -42,16 +41,14 @@ func HandleFind(logCtrl *logs.Controller) http.HandlerFunc { render.TranslatedUserError(w, err) return } - rc, err := logCtrl.Find( + lines, err := logCtrl.Find( ctx, session, repoRef, pipelineUID, executionNum, int(stageNum), int(stepNum)) if err != nil { render.TranslatedUserError(w, err) return } - defer rc.Close() - w.Header().Set("Content-Type", "text/plain") - io.Copy(w, rc) + render.JSON(w, http.StatusOK, lines) } } diff --git a/internal/api/openapi/pipeline.go b/internal/api/openapi/pipeline.go index 4b2d46489..d4bd22fb1 100644 --- a/internal/api/openapi/pipeline.go +++ b/internal/api/openapi/pipeline.go @@ -7,14 +7,15 @@ package openapi import ( "net/http" - "github.com/gotidy/ptr" "github.com/harness/gitness/internal/api/controller/execution" "github.com/harness/gitness/internal/api/controller/pipeline" "github.com/harness/gitness/internal/api/controller/trigger" "github.com/harness/gitness/internal/api/request" "github.com/harness/gitness/internal/api/usererror" + "github.com/harness/gitness/livelog" "github.com/harness/gitness/types" + "github.com/gotidy/ptr" "github.com/swaggest/openapi-go/openapi3" ) @@ -41,7 +42,6 @@ type logRequest struct { type createExecutionRequest struct { pipelineRequest - execution.CreateInput } type createTriggerRequest struct { @@ -95,6 +95,20 @@ var queryParameterLatest = openapi3.ParameterOrRef{ }, } +var queryParameterBranch = openapi3.ParameterOrRef{ + Parameter: &openapi3.Parameter{ + Name: request.QueryParamBranch, + In: openapi3.ParameterInQuery, + Description: ptr.String("Branch to run the execution for."), + Required: ptr.Bool(false), + Schema: &openapi3.SchemaOrRef{ + Schema: &openapi3.Schema{ + Type: ptrSchemaType(openapi3.SchemaTypeString), + }, + }, + }, +} + func pipelineOperations(reflector *openapi3.Reflector) { opCreate := openapi3.Operation{} opCreate.WithTags("pipeline") @@ -156,6 +170,7 @@ func pipelineOperations(reflector *openapi3.Reflector) { executionCreate := openapi3.Operation{} executionCreate.WithTags("pipeline") + executionCreate.WithParameters(queryParameterBranch) executionCreate.WithMapOfAnything(map[string]interface{}{"operationId": "createExecution"}) _ = reflector.SetRequest(&executionCreate, new(createExecutionRequest), http.MethodPost) _ = reflector.SetJSONResponse(&executionCreate, new(types.Execution), http.StatusCreated) @@ -282,7 +297,8 @@ func pipelineOperations(reflector *openapi3.Reflector) { logView.WithTags("pipeline") logView.WithMapOfAnything(map[string]interface{}{"operationId": "viewLogs"}) _ = reflector.SetRequest(&logView, new(logRequest), http.MethodGet) - _ = reflector.SetStringResponse(&logView, http.StatusOK, "text/plain") + _ = reflector.SetStringResponse(&logView, http.StatusOK, "application/json") + _ = reflector.SetJSONResponse(&logView, []*livelog.Line{}, http.StatusOK) _ = reflector.SetJSONResponse(&logView, new(usererror.Error), http.StatusInternalServerError) _ = reflector.SetJSONResponse(&logView, new(usererror.Error), http.StatusUnauthorized) _ = reflector.SetJSONResponse(&logView, new(usererror.Error), http.StatusForbidden) diff --git a/internal/api/request/pipeline.go b/internal/api/request/pipeline.go index 1aec54ed4..bbf8337fa 100644 --- a/internal/api/request/pipeline.go +++ b/internal/api/request/pipeline.go @@ -16,6 +16,7 @@ const ( PathParamStepNumber = "step_number" PathParamTriggerUID = "trigger_uid" QueryParamLatest = "latest" + QueryParamBranch = "branch" ) func GetPipelineUIDFromPath(r *http.Request) (string, error) { @@ -28,6 +29,10 @@ func GetPipelineUIDFromPath(r *http.Request) (string, error) { return url.PathUnescape(rawRef) } +func GetBranchFromQuery(r *http.Request) string { + return r.URL.Query().Get(QueryParamBranch) +} + func GetExecutionNumberFromPath(r *http.Request) (int64, error) { return PathParamAsPositiveInt64(r, PathParamExecutionNumber) } diff --git a/internal/store/database.go b/internal/store/database.go index d0db0f65f..954c5b0dc 100644 --- a/internal/store/database.go +++ b/internal/store/database.go @@ -537,16 +537,22 @@ type ( // Delete deletes a secret given an ID. Delete(ctx context.Context, id int64) error - // DeleteByUID deletes a secret given a space ID and a uid + // DeleteByUID deletes a secret given a space ID and a uid. DeleteByUID(ctx context.Context, spaceID int64, uid string) error - // List lists the secrets in a given space + // List lists the secrets in a given space. List(ctx context.Context, spaceID int64, filter types.ListQueryFilter) ([]*types.Secret, error) + + // ListAll lists all the secrets in a given space. + ListAll(ctx context.Context, parentID int64) ([]*types.Secret, error) } ExecutionStore interface { - // Find returns a execution given a pipeline and an execution number - Find(ctx context.Context, pipelineID int64, num int64) (*types.Execution, error) + // Find returns a execution given an execution ID. + Find(ctx context.Context, id int64) (*types.Execution, error) + + // FindByNumber returns a execution given a pipeline and an execution number + FindByNumber(ctx context.Context, pipelineID int64, num int64) (*types.Execution, error) // Create creates a new execution in the datastore. Create(ctx context.Context, execution *types.Execution) error @@ -573,6 +579,9 @@ type ( // where the stage is incomplete (pending or running). ListIncomplete(ctx context.Context) ([]*types.Stage, error) + // List returns a list of stages corresponding to an execution ID. + List(ctx context.Context, executionID int64) ([]*types.Stage, error) + // ListWithSteps returns a stage list from the datastore corresponding to an execution, // with the individual steps included. ListWithSteps(ctx context.Context, executionID int64) ([]*types.Stage, error) @@ -582,11 +591,25 @@ type ( // FindByNumber returns a stage from the datastore by number. FindByNumber(ctx context.Context, executionID int64, stageNum int) (*types.Stage, error) + + // Update tries to update a stage and returns an optimistic locking error if it was + // unable to do so. + Update(ctx context.Context, stage *types.Stage) error + + // Create creates a new stage. + Create(ctx context.Context, stage *types.Stage) error } StepStore interface { // FindByNumber returns a step from the datastore by number. FindByNumber(ctx context.Context, stageID int64, stepNum int) (*types.Step, error) + + // Create creates a new step. + Create(ctx context.Context, step *types.Step) error + + // Update tries to update a step and returns an optimistic locking error if it was + // unable to do so. + Update(ctx context.Context, e *types.Step) error } ConnectorStore interface { diff --git a/internal/store/database/execution.go b/internal/store/database/execution.go index 4bc586995..49533cc18 100644 --- a/internal/store/database/execution.go +++ b/internal/store/database/execution.go @@ -113,8 +113,23 @@ const ( ` ) -// Find returns an execution given a pipeline ID and an execution number. -func (s *executionStore) Find(ctx context.Context, pipelineID int64, executionNum int64) (*types.Execution, error) { +// Find returns an execution given an execution ID. +func (s *executionStore) Find(ctx context.Context, id int64) (*types.Execution, error) { + const findQueryStmt = ` + SELECT` + executionColumns + ` + FROM executions + WHERE execution_id = $1` + db := dbtx.GetAccessor(ctx, s.db) + + dst := new(execution) + if err := db.GetContext(ctx, dst, findQueryStmt, id); err != nil { + return nil, database.ProcessSQLErrorf(err, "Failed to find execution") + } + return mapInternalToExecution(dst) +} + +// FindByNumber returns an execution given a pipeline ID and an execution number. +func (s *executionStore) FindByNumber(ctx context.Context, pipelineID int64, executionNum int64) (*types.Execution, error) { const findQueryStmt = ` SELECT` + executionColumns + ` FROM executions @@ -230,6 +245,7 @@ func (s *executionStore) Update(ctx context.Context, e *types.Execution) error { ,execution_version = :execution_version WHERE execution_id = :execution_id AND execution_version = :execution_version - 1` updatedAt := time.Now() + stages := e.Stages execution := mapExecutionToInternal(e) @@ -264,6 +280,7 @@ func (s *executionStore) Update(ctx context.Context, e *types.Execution) error { *e = *m e.Version = execution.Version e.Updated = execution.Updated + e.Stages = stages // stages are not mapped in database. return nil } @@ -287,7 +304,7 @@ func (s *executionStore) UpdateOptLock(ctx context.Context, return nil, err } - execution, err = s.Find(ctx, execution.PipelineID, execution.Number) + execution, err = s.FindByNumber(ctx, execution.PipelineID, execution.Number) if err != nil { return nil, err } diff --git a/internal/store/database/migrate/ci/ci_migrations.sql b/internal/store/database/migrate/ci/ci_migrations.sql index 4f25e4b24..c580d0059 100644 --- a/internal/store/database/migrate/ci/ci_migrations.sql +++ b/internal/store/database/migrate/ci/ci_migrations.sql @@ -106,6 +106,7 @@ CREATE TABLE secrets ( CREATE TABLE stages ( stage_id INTEGER PRIMARY KEY AUTOINCREMENT ,stage_execution_id INTEGER NOT NULL + ,stage_repo_id INTEGER NOT NULL ,stage_number INTEGER NOT NULL ,stage_kind TEXT NOT NULL ,stage_type TEXT NOT NULL @@ -216,267 +217,6 @@ INSERT INTO pipelines ( 'develop', 'config_path_2', 1678932200000, 1678932300000, 1 ); --- Insert some executions -INSERT INTO executions ( - execution_id, execution_pipeline_id, execution_repo_id, execution_trigger, - execution_number, execution_parent, execution_status, execution_error, - execution_event, execution_action, execution_link, execution_timestamp, - execution_title, execution_message, execution_before, execution_after, - execution_ref, execution_source_repo, execution_source, execution_target, - execution_author, execution_author_name, execution_author_email, - execution_author_avatar, execution_sender, execution_params, execution_cron, - execution_deploy, execution_deploy_id, execution_debug, execution_started, - execution_finished, execution_created, execution_updated, execution_version -) VALUES ( - 1, 1, 1, 'manual', 1, 0, 'running', '', 'push', 'created', - 'https://example.com/pipelines/1', 1678932400, 'Pipeline Execution 1', - 'Pipeline execution message...', 'commit_hash_before', 'commit_hash_after', - 'refs/heads/main', 'source_repo_name', 'source_branch', 'target_branch', - 'author_login', 'Author Name', 'author@example.com', 'https://example.com/avatar.jpg', - 'sender_username', '{"param1": "value1", "param2": "value2"}', '0 0 * * *', - 'production', 5, 0, 1678932500, 1678932600, 1678932700, 1678932800, 1 -); - -INSERT INTO executions ( - execution_id, execution_pipeline_id, execution_repo_id, execution_trigger, - execution_number, execution_parent, execution_status, execution_error, - execution_event, execution_action, execution_link, execution_timestamp, - execution_title, execution_message, execution_before, execution_after, - execution_ref, execution_source_repo, execution_source, execution_target, - execution_author, execution_author_name, execution_author_email, - execution_author_avatar, execution_sender, execution_params, execution_cron, - execution_deploy, execution_deploy_id, execution_debug, execution_started, - execution_finished, execution_created, execution_updated, execution_version -) VALUES ( - 2, 1, 1, 'manual', 2, 0, 'running', '', 'push', 'created', - 'https://example.com/pipelines/1', 1678932400, 'Pipeline Execution 1', - 'Pipeline execution message...', 'commit_hash_before', 'commit_hash_after', - 'refs/heads/main', 'source_repo_name', 'source_branch', 'target_branch', - 'author_login', 'Author Name', 'author@example.com', 'https://example.com/avatar.jpg', - 'sender_username', '{"param1": "value1", "param2": "value2"}', '0 0 * * *', - 'production', 5, 0, 1678932500, 1678932600, 1678932700, 1678932800, 1 -); - -INSERT INTO executions ( - execution_id, execution_pipeline_id, execution_repo_id, execution_trigger, - execution_number, execution_parent, execution_status, execution_error, - execution_event, execution_action, execution_link, execution_timestamp, - execution_title, execution_message, execution_before, execution_after, - execution_ref, execution_source_repo, execution_source, execution_target, - execution_author, execution_author_name, execution_author_email, - execution_author_avatar, execution_sender, execution_params, execution_cron, - execution_deploy, execution_deploy_id, execution_debug, execution_started, - execution_finished, execution_created, execution_updated, execution_version -) VALUES ( - 3, 2, 1, 'manual', 1, 0, 'running', '', 'push', 'created', - 'https://example.com/pipelines/1', 1678932400, 'Pipeline Execution 1', - 'Pipeline execution message...', 'commit_hash_before', 'commit_hash_after', - 'refs/heads/main', 'source_repo_name', 'source_branch', 'target_branch', - 'author_login', 'Author Name', 'author@example.com', 'https://example.com/avatar.jpg', - 'sender_username', '{"param1": "value1", "param2": "value2"}', '0 0 * * *', - 'production', 5, 0, 1678932500, 1678932600, 1678932700, 1678932800, 1 -); - -INSERT INTO executions ( - execution_id, execution_pipeline_id, execution_repo_id, execution_trigger, - execution_number, execution_parent, execution_status, execution_error, - execution_event, execution_action, execution_link, execution_timestamp, - execution_title, execution_message, execution_before, execution_after, - execution_ref, execution_source_repo, execution_source, execution_target, - execution_author, execution_author_name, execution_author_email, - execution_author_avatar, execution_sender, execution_params, execution_cron, - execution_deploy, execution_deploy_id, execution_debug, execution_started, - execution_finished, execution_created, execution_updated, execution_version -) VALUES ( - 4, 2, 1, 'manual', 2, 0, 'running', '', 'push', 'created', - 'https://example.com/pipelines/1', 1678932400, 'Pipeline Execution 1', - 'Pipeline execution message...', 'commit_hash_before', 'commit_hash_after', - 'refs/heads/main', 'source_repo_name', 'source_branch', 'target_branch', - 'author_login', 'Author Name', 'author@example.com', 'https://example.com/avatar.jpg', - 'sender_username', '{"param1": "value1", "param2": "value2"}', '0 0 * * *', - 'production', 5, 0, 1678932500, 1678932600, 1678932700, 1678932800, 1 -); - -INSERT INTO executions ( - execution_id, execution_pipeline_id, execution_repo_id, execution_trigger, - execution_number, execution_parent, execution_status, execution_error, - execution_event, execution_action, execution_link, execution_timestamp, - execution_title, execution_message, execution_before, execution_after, - execution_ref, execution_source_repo, execution_source, execution_target, - execution_author, execution_author_name, execution_author_email, - execution_author_avatar, execution_sender, execution_params, execution_cron, - execution_deploy, execution_deploy_id, execution_debug, execution_started, - execution_finished, execution_created, execution_updated, execution_version -) VALUES ( - 5, 2, 1, 'manual', 3, 0, 'running', '', 'push', 'created', - 'https://example.com/pipelines/1', 1678932400, 'Pipeline Execution 1', - 'Pipeline execution message...', 'commit_hash_before', 'commit_hash_after', - 'refs/heads/main', 'source_repo_name', 'source_branch', 'target_branch', - 'author_login', 'Author Name', 'author@example.com', 'https://example.com/avatar.jpg', - 'sender_username', '{"param1": "value1", "param2": "value2"}', '0 0 * * *', - 'production', 5, 0, 1678932500, 1678932600, 1678932700, 1678932800, 1 -); - --- Insert some stages -INSERT INTO stages (stage_id, stage_execution_id, stage_number, stage_parent_group_id, stage_kind, stage_type, stage_name, stage_status, stage_error, stage_errignore, stage_exit_code, stage_limit, stage_os, stage_arch, stage_variant, stage_kernel, stage_machine, stage_started, stage_stopped, stage_created, stage_updated, stage_version, stage_on_success, stage_on_failure, stage_depends_on, stage_labels, stage_limit_repo) -VALUES ( - 1, -- stage_id - 1, -- stage_execution_id - 1, -- stage_number - 0, -- stage_parent_group_id - 'build', -- stage_kind - 'docker', -- stage_type - 'Build Stage', -- stage_name - 'Pending', -- stage_status - '', -- stage_error - 0, -- stage_errignore - 0, -- stage_exit_code - 2, -- stage_limit - 'linux', -- stage_os - 'x86_64', -- stage_arch - 'default', -- stage_variant - '4.18.0-305.7.1.el8_4.x86_64', -- stage_kernel - 'x86_64', -- stage_machine - 0, -- stage_started - 0, -- stage_stopped - 1679089460, -- stage_created - 1679089500, -- stage_updated - 1, -- stage_version - 1, -- stage_on_success - 0, -- stage_on_failure - '["1"]', -- stage_depends_on - '{}', -- stage_labels - 1 -- stage_limit_repo -); - --- Sample 2 -INSERT INTO stages (stage_id, stage_execution_id, stage_number, stage_parent_group_id, stage_kind, stage_type, stage_name, stage_status, stage_error, stage_errignore, stage_exit_code, stage_limit, stage_os, stage_arch, stage_variant, stage_kernel, stage_machine, stage_started, stage_stopped, stage_created, stage_updated, stage_version, stage_on_success, stage_on_failure, stage_depends_on, stage_labels, stage_limit_repo) -VALUES ( - 2, -- stage_id - 1, -- stage_execution_id - 2, -- stage_number - 0, -- stage_parent_group_id - 'test', -- stage_kind - 'pytest', -- stage_type - 'Test Stage', -- stage_name - 'Pending', -- stage_status - '', -- stage_error - 0, -- stage_errignore - 0, -- stage_exit_code - 1, -- stage_limit - 'linux', -- stage_os - 'x86_64', -- stage_arch - 'default', -- stage_variant - '4.18.0-305.7.1.el8_4.x86_64', -- stage_kernel - 'x86_64', -- stage_machine - 0, -- stage_started - 0, -- stage_stopped - 1679089560, -- stage_created - 1679089600, -- stage_updated - 1, -- stage_version - 1, -- stage_on_success - 1, -- stage_on_failure - '["1"]', -- stage_depends_on (referring to the first stage) - '{}', -- stage_labels - 0 -- stage_limit_repo (using default value) -); - -INSERT INTO steps (step_id, step_stage_id, step_number, step_parent_group_id, step_name, step_status, step_error, step_errignore, step_exit_code, step_started, step_stopped, step_version, step_depends_on, step_image, step_detached, step_schema) -VALUES ( - 1, -- step_id - 1, -- step_stage_id - 1, -- step_number - 0, -- step_parent_group_id - 'stage1step1', -- step_name - 'Pending', -- step_status - '', -- step_error - 0, -- step_errignore - 0, -- step_exit_code - 0, -- step_started - 0, -- step_stopped - 1, -- step_version - '["1"]', -- step_depends_on - 'sample_image', -- step_image - 0, -- step_detached - 'sample_schema' -- step_schema -); - -INSERT INTO steps (step_id, step_stage_id, step_number, step_parent_group_id, step_name, step_status, step_error, step_errignore, step_exit_code, step_started, step_stopped, step_version, step_depends_on, step_image, step_detached, step_schema) -VALUES ( - 2, -- step_id - 1, -- step_stage_id - 2, -- step_number - 0, -- step_parent_group_id - 'stage1step2', -- step_name - 'Success', -- step_status - '', -- step_error - 0, -- step_errignore - 0, -- step_exit_code - 0, -- step_started - 0, -- step_stopped - 1, -- step_version - '["1"]', -- step_depends_on - 'sample_image', -- step_image - 0, -- step_detached - 'sample_schema' -- step_schema -); - -INSERT INTO steps (step_id, step_stage_id, step_number, step_parent_group_id, step_name, step_status, step_error, step_errignore, step_exit_code, step_started, step_stopped, step_version, step_depends_on, step_image, step_detached, step_schema) -VALUES ( - 3, -- step_id - 2, -- step_stage_id - 1, -- step_number - 0, -- step_parent_group_id - 'stage2step1', -- step_name - 'Success', -- step_status - '', -- step_error - 0, -- step_errignore - 0, -- step_exit_code - 0, -- step_started - 0, -- step_stopped - 1, -- step_version - '["1"]', -- step_depends_on - 'sample_image', -- step_image - 0, -- step_detached - 'sample_schema' -- step_schema -); - -INSERT INTO steps (step_id, step_stage_id, step_number, step_parent_group_id, step_name, step_status, step_error, step_errignore, step_exit_code, step_started, step_stopped, step_version, step_depends_on, step_image, step_detached, step_schema) -VALUES ( - 4, -- step_id - 2, -- step_stage_id - 2, -- step_number - 0, -- step_parent_group_id - 'stage2step2', -- step_name - 'Success', -- step_status - '', -- step_error - 0, -- step_errignore - 0, -- step_exit_code - 0, -- step_started - 0, -- step_stopped - 1, -- step_version - '["1"]', -- step_depends_on - 'sample_image', -- step_image - 0, -- step_detached - 'sample_schema' -- step_schema -); - -INSERT INTO logs (log_id, log_data) VALUES (1, -'{"pos": 0, "out": "+git init", "time": 0} -{"pos": 0, "out": "echo Hi", "time": 2}'); - -INSERT INTO logs (log_id, log_data) VALUES (2, -'{"pos": 0, "out": "+git init", "time": 0} -{"pos": 0, "out": "echo Hi", "time": 2}'); - -INSERT INTO logs (log_id, log_data) VALUES (3, -'{"pos": 0, "out": "+git init", "time": 0} -{"pos": 0, "out": "echo Hi", "time": 2}'); - -INSERT INTO logs (log_id, log_data) VALUES (4, -'{"pos": 0, "out": "+git init", "time": 0} -{"pos": 0, "out": "echo Hi", "time": 2}'); - CREATE TABLE connectors ( connector_id INTEGER PRIMARY KEY AUTOINCREMENT ,connector_uid TEXT NOT NULL diff --git a/internal/store/database/pipeline.go b/internal/store/database/pipeline.go index 9b0c804af..0213ab1f5 100644 --- a/internal/store/database/pipeline.go +++ b/internal/store/database/pipeline.go @@ -211,6 +211,8 @@ func (s *pipelineStore) ListLatest( ,execution_status ,execution_error ,execution_link + ,execution_message + ,execution_after ,execution_timestamp ,execution_title ,execution_author diff --git a/internal/store/database/pipeline_join.go b/internal/store/database/pipeline_join.go index a7e212a2a..b179cf7d2 100644 --- a/internal/store/database/pipeline_join.go +++ b/internal/store/database/pipeline_join.go @@ -15,6 +15,8 @@ type pipelineExecutionJoin struct { *types.Pipeline ID sql.NullInt64 `db:"execution_id"` PipelineID sql.NullInt64 `db:"execution_pipeline_id"` + Message sql.NullString `db:"execution_message"` + After sql.NullString `db:"execution_after"` RepoID sql.NullInt64 `db:"execution_repo_id"` Trigger sql.NullString `db:"execution_trigger"` Number sql.NullInt64 `db:"execution_number"` @@ -56,6 +58,8 @@ func convertPipelineJoin(join *pipelineExecutionJoin) *types.Pipeline { RepoID: join.RepoID.Int64, Trigger: join.Trigger.String, Number: join.Number.Int64, + After: join.After.String, + Message: join.Message.String, Status: join.Status.String, Error: join.Error.String, Link: join.Link.String, diff --git a/internal/store/database/secret.go b/internal/store/database/secret.go index ac2b04c46..f49d085b4 100644 --- a/internal/store/database/secret.go +++ b/internal/store/database/secret.go @@ -209,6 +209,28 @@ func (s *secretStore) List(ctx context.Context, parentID int64, filter types.Lis return dst, nil } +// ListAll lists all the secrets present in a space. +func (s *secretStore) ListAll(ctx context.Context, parentID int64) ([]*types.Secret, error) { + stmt := database.Builder. + Select(secretColumns). + From("secrets"). + Where("secret_space_id = ?", fmt.Sprint(parentID)) + + sql, args, err := stmt.ToSql() + if err != nil { + return nil, errors.Wrap(err, "Failed to convert query to sql") + } + + db := dbtx.GetAccessor(ctx, s.db) + + dst := []*types.Secret{} + if err = db.SelectContext(ctx, &dst, sql, args...); err != nil { + return nil, database.ProcessSQLErrorf(err, "Failed executing custom list query") + } + + return dst, nil +} + // Delete deletes a secret given a secret ID. func (s *secretStore) Delete(ctx context.Context, id int64) error { const secretDeleteStmt = ` diff --git a/internal/store/database/stage.go b/internal/store/database/stage.go index ad6f8ebc1..4b2972985 100644 --- a/internal/store/database/stage.go +++ b/internal/store/database/stage.go @@ -6,8 +6,11 @@ package database import ( "context" + "fmt" + "time" "github.com/harness/gitness/internal/store" + gitness_store "github.com/harness/gitness/store" "github.com/harness/gitness/store/database" "github.com/harness/gitness/store/database/dbtx" "github.com/harness/gitness/types" @@ -22,6 +25,7 @@ const ( stageColumns = ` stage_id ,stage_execution_id + ,stage_repo_id ,stage_number ,stage_name ,stage_kind @@ -45,37 +49,39 @@ const ( ,stage_on_success ,stage_on_failure ,stage_depends_on - ,stage_labels + ,stage_labels ` ) type stage struct { - ID int64 `db:"stage_id"` - ExecutionID int64 `db:"stage_execution_id"` - Number int64 `db:"stage_number"` - Name string `db:"stage_name"` - Kind string `db:"stage_kind"` - Type string `db:"stage_type"` - Status string `db:"stage_status"` - Error string `db:"stage_error"` - ErrIgnore bool `db:"stage_errignore"` - ExitCode int `db:"stage_exit_code"` - Machine string `db:"stage_machine"` - OS string `db:"stage_os"` - Arch string `db:"stage_arch"` - Variant string `db:"stage_variant"` - Kernel string `db:"stage_kernel"` - Limit int `db:"stage_limit"` - LimitRepo int `db:"stage_limit_repo"` - Started int64 `db:"stage_started"` - Stopped int64 `db:"stage_stopped"` - Created int64 `db:"stage_created"` - Updated int64 `db:"stage_updated"` - Version int64 `db:"stage_version"` - OnSuccess bool `db:"stage_on_success"` - OnFailure bool `db:"stage_on_failure"` - DependsOn sqlxtypes.JSONText `db:"stage_depends_on"` - Labels sqlxtypes.JSONText `db:"stage_labels"` + ID int64 `db:"stage_id"` + ExecutionID int64 `db:"stage_execution_id"` + RepoID int64 `db:"stage_repo_id"` + Number int64 `db:"stage_number"` + Name string `db:"stage_name"` + Kind string `db:"stage_kind"` + Type string `db:"stage_type"` + Status string `db:"stage_status"` + Error string `db:"stage_error"` + ParentGroupID int64 `db:"stage_parent_group_id"` + ErrIgnore bool `db:"stage_errignore"` + ExitCode int `db:"stage_exit_code"` + Machine string `db:"stage_machine"` + OS string `db:"stage_os"` + Arch string `db:"stage_arch"` + Variant string `db:"stage_variant"` + Kernel string `db:"stage_kernel"` + Limit int `db:"stage_limit"` + LimitRepo int `db:"stage_limit_repo"` + Started int64 `db:"stage_started"` + Stopped int64 `db:"stage_stopped"` + Created int64 `db:"stage_created"` + Updated int64 `db:"stage_updated"` + Version int64 `db:"stage_version"` + OnSuccess bool `db:"stage_on_success"` + OnFailure bool `db:"stage_on_failure"` + DependsOn sqlxtypes.JSONText `db:"stage_depends_on"` + Labels sqlxtypes.JSONText `db:"stage_labels"` } // NewStageStore returns a new StageStore. @@ -104,6 +110,80 @@ func (s *stageStore) FindByNumber(ctx context.Context, executionID int64, stageN return mapInternalToStage(dst) } +// Create adds a stage in the database. +func (s *stageStore) Create(ctx context.Context, st *types.Stage) error { + const stageInsertStmt = ` + INSERT INTO stages ( + stage_execution_id + ,stage_repo_id + ,stage_number + ,stage_name + ,stage_kind + ,stage_type + ,stage_status + ,stage_error + ,stage_errignore + ,stage_exit_code + ,stage_machine + ,stage_parent_group_id + ,stage_os + ,stage_arch + ,stage_variant + ,stage_kernel + ,stage_limit + ,stage_limit_repo + ,stage_started + ,stage_stopped + ,stage_created + ,stage_updated + ,stage_version + ,stage_on_success + ,stage_on_failure + ,stage_depends_on + ,stage_labels + ) VALUES ( + :stage_execution_id + ,:stage_repo_id + ,:stage_number + ,:stage_name + ,:stage_kind + ,:stage_type + ,:stage_status + ,:stage_error + ,:stage_errignore + ,:stage_exit_code + ,:stage_machine + ,:stage_parent_group_id + ,:stage_os + ,:stage_arch + ,:stage_variant + ,:stage_kernel + ,:stage_limit + ,:stage_limit_repo + ,:stage_started + ,:stage_stopped + ,:stage_created + ,:stage_updated + ,:stage_version + ,:stage_on_success + ,:stage_on_failure + ,:stage_depends_on + ,:stage_labels + + ) RETURNING stage_id` + db := dbtx.GetAccessor(ctx, s.db) + + stage := mapStageToInternal(st) + query, arg, err := db.BindNamed(stageInsertStmt, stage) + if err != nil { + return database.ProcessSQLErrorf(err, "Failed to bind stage object") + } + if err = db.QueryRowContext(ctx, query, arg...).Scan(&stage.ID); err != nil { + return database.ProcessSQLErrorf(err, "Stage query failed") + } + return nil +} + // ListWithSteps returns a stage with information about all its containing steps. func (s *stageStore) ListWithSteps(ctx context.Context, executionID int64) ([]*types.Stage, error) { const queryNumberWithSteps = ` @@ -152,9 +232,79 @@ func (s *stageStore) ListIncomplete(ctx context.Context) ([]*types.Stage, error) db := dbtx.GetAccessor(ctx, s.db) dst := []*stage{} - if err := db.GetContext(ctx, dst, queryListIncomplete); err != nil { + if err := db.SelectContext(ctx, &dst, queryListIncomplete); err != nil { return nil, database.ProcessSQLErrorf(err, "Failed to find incomplete stages") } // map stages list return mapInternalToStageList(dst) } + +// List returns a list of stages corresponding to an execution ID. +func (s *stageStore) List(ctx context.Context, executionID int64) ([]*types.Stage, error) { + const queryList = ` + SELECT` + stageColumns + ` + FROM stages + WHERE stage_execution_id = $1 + ORDER BY stage_number ASC + ` + db := dbtx.GetAccessor(ctx, s.db) + + dst := []*stage{} + if err := db.SelectContext(ctx, &dst, queryList, executionID); err != nil { + return nil, database.ProcessSQLErrorf(err, "Failed to find stages") + } + // map stages list + return mapInternalToStageList(dst) +} + +// Update tries to update a stage in the datastore and returns a locking error +// if it was unable to do so. +func (s *stageStore) Update(ctx context.Context, e *types.Stage) error { + const stageUpdateStmt = ` + UPDATE stages + SET + stage_status = :stage_status + ,stage_machine = :stage_machine + ,stage_updated = :stage_updated + ,stage_version = :stage_version + ,stage_error = :stage_error + WHERE stage_id = :stage_id AND stage_version = :stage_version - 1` + updatedAt := time.Now() + steps := e.Steps + + stage := mapStageToInternal(e) + + stage.Version++ + stage.Updated = updatedAt.UnixMilli() + + db := dbtx.GetAccessor(ctx, s.db) + + query, arg, err := db.BindNamed(stageUpdateStmt, stage) + if err != nil { + return database.ProcessSQLErrorf(err, "Failed to bind stage object") + } + + result, err := db.ExecContext(ctx, query, arg...) + if err != nil { + return database.ProcessSQLErrorf(err, "Failed to update stage") + } + + count, err := result.RowsAffected() + if err != nil { + return database.ProcessSQLErrorf(err, "Failed to get number of updated rows") + } + + if count == 0 { + return gitness_store.ErrVersionConflict + } + + m, err := mapInternalToStage(stage) + if err != nil { + return fmt.Errorf("Could not map stage object: %w", err) + } + *e = *m + e.Version = stage.Version + e.Updated = stage.Updated + e.Steps = steps // steps is not mapped in database. + return nil +} diff --git a/internal/store/database/stage_map.go b/internal/store/database/stage_map.go index 8b68903fb..c27f44caa 100644 --- a/internal/store/database/stage_map.go +++ b/internal/store/database/stage_map.go @@ -28,6 +28,7 @@ func mapInternalToStage(in *stage) (*types.Stage, error) { return &types.Stage{ ID: in.ID, ExecutionID: in.ExecutionID, + RepoID: in.RepoID, Number: in.Number, Name: in.Name, Kind: in.Kind, @@ -59,6 +60,7 @@ func mapStageToInternal(in *types.Stage) *stage { return &stage{ ID: in.ID, ExecutionID: in.ExecutionID, + RepoID: in.RepoID, Number: in.Number, Name: in.Name, Kind: in.Kind, @@ -132,6 +134,7 @@ func scanRowStep(rows *sql.Rows, stage *types.Stage, step *types.Step) error { err := rows.Scan( &stage.ID, &stage.ExecutionID, + &stage.RepoID, &stage.Number, &stage.Name, &stage.Kind, diff --git a/internal/store/database/step.go b/internal/store/database/step.go index adfdf1fb8..2e04ed750 100644 --- a/internal/store/database/step.go +++ b/internal/store/database/step.go @@ -6,8 +6,10 @@ package database import ( "context" + "fmt" "github.com/harness/gitness/internal/store" + gitness_store "github.com/harness/gitness/store" "github.com/harness/gitness/store/database" "github.com/harness/gitness/store/database/dbtx" "github.com/harness/gitness/types" @@ -39,21 +41,22 @@ const ( ) type step struct { - ID int64 `db:"step_id"` - StageID int64 `db:"step_stage_id"` - Number int64 `db:"step_number"` - Name string `db:"step_name"` - Status string `db:"step_status"` - Error string `db:"step_error"` - ErrIgnore bool `db:"step_errignore"` - ExitCode int `db:"step_exit_code"` - Started int64 `db:"step_started"` - Stopped int64 `db:"step_stopped"` - Version int64 `db:"step_version"` - DependsOn sqlxtypes.JSONText `db:"step_depends_on"` - Image string `db:"step_image"` - Detached bool `db:"step_detached"` - Schema string `db:"step_schema"` + ID int64 `db:"step_id"` + StageID int64 `db:"step_stage_id"` + Number int64 `db:"step_number"` + ParentGroupID int64 `db:"step_parent_group_id"` + Name string `db:"step_name"` + Status string `db:"step_status"` + Error string `db:"step_error"` + ErrIgnore bool `db:"step_errignore"` + ExitCode int `db:"step_exit_code"` + Started int64 `db:"step_started"` + Stopped int64 `db:"step_stopped"` + Version int64 `db:"step_version"` + DependsOn sqlxtypes.JSONText `db:"step_depends_on"` + Image string `db:"step_image"` + Detached bool `db:"step_detached"` + Schema string `db:"step_schema"` } // NewStepStore returns a new StepStore. @@ -81,3 +84,106 @@ func (s *stepStore) FindByNumber(ctx context.Context, stageID int64, stepNum int } return mapInternalToStep(dst) } + +// Create creates a step. +func (s *stepStore) Create(ctx context.Context, step *types.Step) error { + const stepInsertStmt = ` + INSERT INTO steps ( + step_stage_id + ,step_number + ,step_name + ,step_status + ,step_error + ,step_parent_group_id + ,step_errignore + ,step_exit_code + ,step_started + ,step_stopped + ,step_version + ,step_depends_on + ,step_image + ,step_detached + ,step_schema + ) VALUES ( + :step_stage_id + ,:step_number + ,:step_name + ,:step_status + ,:step_error + ,:step_parent_group_id + ,:step_errignore + ,:step_exit_code + ,:step_started + ,:step_stopped + ,:step_version + ,:step_depends_on + ,:step_image + ,:step_detached + ,:step_schema + ) RETURNING step_id` + db := dbtx.GetAccessor(ctx, s.db) + + query, arg, err := db.BindNamed(stepInsertStmt, mapStepToInternal(step)) + if err != nil { + return database.ProcessSQLErrorf(err, "Failed to bind step object") + } + + if err = db.QueryRowContext(ctx, query, arg...).Scan(&step.ID); err != nil { + return database.ProcessSQLErrorf(err, "Step query failed") + } + + return nil +} + +// Update tries to update a step in the datastore and returns a locking error +// if it was unable to do so. +func (s *stepStore) Update(ctx context.Context, e *types.Step) error { + const stepUpdateStmt = ` + UPDATE steps + SET + step_name = :step_name + ,step_status = :step_status + ,step_error = :step_error + ,step_errignore = :step_errignore + ,step_exit_code = :step_exit_code + ,step_started = :step_started + ,step_stopped = :step_stopped + ,step_depends_on = :step_depends_on + ,step_image = :step_image + ,step_detached = :step_detached + ,step_schema = :step_schema + ,step_version = :step_version + WHERE step_id = :step_id AND step_version = :step_version - 1` + step := mapStepToInternal(e) + + step.Version++ + + db := dbtx.GetAccessor(ctx, s.db) + + query, arg, err := db.BindNamed(stepUpdateStmt, step) + if err != nil { + return database.ProcessSQLErrorf(err, "Failed to bind step object") + } + + result, err := db.ExecContext(ctx, query, arg...) + if err != nil { + return database.ProcessSQLErrorf(err, "Failed to update step") + } + + count, err := result.RowsAffected() + if err != nil { + return database.ProcessSQLErrorf(err, "Failed to get number of updated rows") + } + + if count == 0 { + return gitness_store.ErrVersionConflict + } + + m, err := mapInternalToStep(step) + if err != nil { + return fmt.Errorf("Could not map step object: %w", err) + } + *e = *m + e.Version = step.Version + return nil +} diff --git a/internal/store/logs/db.go b/internal/store/logs/db.go index 9156deab0..633931903 100644 --- a/internal/store/logs/db.go +++ b/internal/store/logs/db.go @@ -63,7 +63,8 @@ func (s *logStore) Create(ctx context.Context, stepID int64, r io.Reader) error ,log_data ) values ( :log_id - ,:log_data` + ,:log_data + )` data, err := io.ReadAll(r) if err != nil { return fmt.Errorf("could not read log data: %w", err) diff --git a/internal/url/provider.go b/internal/url/provider.go index a3a0e6289..4d6528419 100644 --- a/internal/url/provider.go +++ b/internal/url/provider.go @@ -68,3 +68,15 @@ func (p *Provider) GenerateRepoCloneURL(repoPath string) string { return p.gitURL.JoinPath(repoPath).String() } + +// GenerateCustomRepoCloneURL generates a custom clone URL given the base URL. +// Example: url is http://host.docker.internal:3000 and repoPath is test/gitness-demo +// it will return http://host.docker.internal:3000/git/test/gitness-demo.git +func (p *Provider) GenerateCustomRepoCloneURL(baseURL *url.URL, repoPath string) string { + repoPath = path.Clean(repoPath) + if !strings.HasSuffix(repoPath, ".git") { + repoPath += ".git" + } + + return baseURL.JoinPath("git").JoinPath(repoPath).String() +} diff --git a/types/config.go b/types/config.go index f8b5022ad..87fe47727 100644 --- a/types/config.go +++ b/types/config.go @@ -65,10 +65,16 @@ type Config struct { // HTTP defines the http configuration parameters HTTP struct { Bind string `envconfig:"GITNESS_HTTP_BIND" default:":3000"` - Proto string `envconfig:"GITNESS_HTTP_PROTO"` + Proto string `envconfig:"GITNESS_HTTP_PROTO" default:"http"` Host string `envconfig:"GITNESS_HTTP_HOST"` // GitHost is the host used to identify git traffic (OPTIONAL). GitHost string `envconfig:"GITNESS_HTTP_GIT_HOST" default:"git.localhost"` + // Network is the docker network on which the gitness container is running. + // One example of where it's used is CI executions to be able to communicate + // with gitness (for example while performing a clone on a local repo). + // In case gitness is running on the host, the build container can talk to + // localhost on the host using host.docker.internal. + Network string `envconfig:"GITNESS_CI_NETWORK" default:"host.docker.internal"` } // Acme defines Acme configuration parameters. @@ -79,6 +85,11 @@ type Config struct { } } + // CI defines configuration related to build executions. + CI struct { + ParallelWorkers int `envconfig:"GITNESS_CI_PARALLEL_WORKERS" default:"5"` + } + // Database defines the database configuration parameters. Database struct { Driver string `envconfig:"GITNESS_DATABASE_DRIVER" default:"sqlite3"` diff --git a/types/events.go b/types/events.go new file mode 100644 index 000000000..3954ca632 --- /dev/null +++ b/types/events.go @@ -0,0 +1,12 @@ +package types + +// Hook event constants. +const ( + EventCron = "cron" + EventCustom = "custom" + EventPush = "push" + EventPullRequest = "pull_request" + EventTag = "tag" + EventPromote = "promote" + EventRollback = "rollback" +) diff --git a/types/stage.go b/types/stage.go index d07218845..718a3e775 100644 --- a/types/stage.go +++ b/types/stage.go @@ -7,6 +7,7 @@ package types type Stage struct { ID int64 `json:"-"` ExecutionID int64 `json:"execution_id"` + RepoID int64 `json:"repo_id"` Number int64 `json:"number"` Name string `json:"name"` Kind string `json:"kind,omitempty"` diff --git a/types/stage_status.go b/types/stage_status.go new file mode 100644 index 000000000..35932b44f --- /dev/null +++ b/types/stage_status.go @@ -0,0 +1,19 @@ +// Copyright 2022 Harness Inc. All rights reserved. +// Use of this source code is governed by the Polyform Free Trial License +// that can be found in the LICENSE.md file for this repository. + +// Status types for a stage. +package types + +const ( + StatusSkipped = "skipped" + StatusBlocked = "blocked" + StatusDeclined = "declined" + StatusWaiting = "waiting_on_dependencies" + StatusPending = "pending" + StatusRunning = "running" + StatusPassing = "success" + StatusFailing = "failure" + StatusKilled = "killed" + StatusError = "error" +)