diff --git a/app/api/controller/execution/create.go b/app/api/controller/execution/create.go index 5e19bd275..2d25d75a1 100644 --- a/app/api/controller/execution/create.go +++ b/app/api/controller/execution/create.go @@ -75,7 +75,7 @@ func (c *Controller) Create( AuthorEmail: commit.Author.Identity.Email, Ref: ref, Message: commit.Message, - Title: "", // we expect this to be empty. + Title: commit.Title, Before: commit.SHA, After: commit.SHA, Sender: session.Principal.UID, diff --git a/app/pipeline/converter/converter.go b/app/pipeline/converter/converter.go new file mode 100644 index 000000000..87a61d114 --- /dev/null +++ b/app/pipeline/converter/converter.go @@ -0,0 +1,67 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package converter + +import ( + "context" + "strings" + + "github.com/harness/gitness/app/pipeline/converter/jsonnet" + "github.com/harness/gitness/app/pipeline/converter/starlark" + "github.com/harness/gitness/app/pipeline/file" +) + +const ( + jsonnetImportLimit = 1000 + starlarkStepLimit = 50000 + starlarkSizeLimit = 1000000 +) + +type converter struct { + fileService file.Service +} + +func newConverter(fileService file.Service) Service { + return &converter{fileService: fileService} +} + +func (c *converter) Convert(_ context.Context, args *ConvertArgs) (*file.File, error) { + path := args.Pipeline.ConfigPath + + if isJSONNet(path) { + str, err := jsonnet.Parse(args.Repo, args.Pipeline, args.Execution, args.File, c.fileService, jsonnetImportLimit) + if err != nil { + return nil, err + } + return &file.File{Data: []byte(str)}, nil + } else if isStarlark(path) { + str, err := starlark.Parse(args.Repo, args.Pipeline, args.Execution, args.File, starlarkStepLimit, starlarkSizeLimit) + if err != nil { + return nil, err + } + return &file.File{Data: []byte(str)}, nil + } + return args.File, nil +} + +func isJSONNet(path string) bool { + return strings.HasSuffix(path, ".drone.jsonnet") +} + +func isStarlark(path string) bool { + return strings.HasSuffix(path, ".drone.script") || + strings.HasSuffix(path, ".drone.star") || + strings.HasSuffix(path, ".drone.starlark") +} diff --git a/app/pipeline/converter/jsonnet/jsonnet.go b/app/pipeline/converter/jsonnet/jsonnet.go new file mode 100644 index 000000000..eb2f35207 --- /dev/null +++ b/app/pipeline/converter/jsonnet/jsonnet.go @@ -0,0 +1,219 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package jsonnet + +import ( + "bytes" + "context" + "errors" + "fmt" + "path" + "strconv" + "strings" + + "github.com/harness/gitness/app/pipeline/file" + "github.com/harness/gitness/types" + + "github.com/google/go-jsonnet" +) + +const repo = "repo." +const build = "build." +const param = "param." + +var noContext = context.Background() + +type importer struct { + repo *types.Repository + execution *types.Execution + + // jsonnet does not cache file imports and may request + // the same file multiple times. We cache the files to + // duplicate API calls. + cache map[string]jsonnet.Contents + + // limit the number of outbound requests. github limits + // the number of api requests per hour, so we should + // make sure that a single build does not abuse the api + // by importing dozens of files. + limit int + + // counts the number of outbound requests. if the count + // exceeds the limit, the importer will return errors. + count int + + fileService file.Service +} + +func (i *importer) Import(importedFrom, importedPath string) (contents jsonnet.Contents, foundAt string, err error) { + if i.cache == nil { + i.cache = map[string]jsonnet.Contents{} + } + + // the import is relative to the imported from path. the + // imported path must resolve to a filepath relative to + // the root of the repository. + importedPath = path.Join( + path.Dir(importedFrom), + importedPath, + ) + + if strings.HasPrefix(importedFrom, "../") { + err = fmt.Errorf("jsonnet: cannot resolve import: %s", importedPath) + return contents, foundAt, err + } + + // if the contents exist in the cache, return the + // cached item. + var ok bool + if contents, ok = i.cache[importedPath]; ok { + return contents, importedPath, nil + } + + defer func() { + i.count++ + }() + + // if the import limit is exceeded log an error message. + if i.limit > 0 && i.count >= i.limit { + return contents, foundAt, errors.New("jsonnet: import limit exceeded") + } + + find, err := i.fileService.Get(noContext, i.repo, importedPath, i.execution.Ref) + + if err != nil { + return contents, foundAt, err + } + + i.cache[importedPath] = jsonnet.MakeContents(string(find.Data)) + + return i.cache[importedPath], importedPath, err +} + +func Parse( + repo *types.Repository, + pipeline *types.Pipeline, + execution *types.Execution, + file *file.File, + fileService file.Service, + limit int, +) (string, error) { + vm := jsonnet.MakeVM() + vm.MaxStack = 500 + vm.StringOutput = false + vm.ErrorFormatter.SetMaxStackTraceSize(20) + if fileService != nil && limit > 0 { + vm.Importer( + &importer{ + repo: repo, + execution: execution, + limit: limit, + fileService: fileService, + }, + ) + } + + // map execution/repo/pipeline parameters + if execution != nil { + mapBuild(execution, vm) + } + if repo != nil { + mapRepo(repo, pipeline, vm) + } + + jsonnetFile := file + jsonnetFileName := pipeline.ConfigPath + + // convert the jsonnet file to yaml + buf := new(bytes.Buffer) + docs, err := vm.EvaluateAnonymousSnippetStream(jsonnetFileName, string(jsonnetFile.Data)) + if err != nil { + doc, err2 := vm.EvaluateAnonymousSnippet(jsonnetFileName, string(jsonnetFile.Data)) + if err2 != nil { + return "", err + } + docs = append(docs, doc) + } + + // the jsonnet vm returns a stream of yaml documents + // that need to be combined into a single yaml file. + for _, doc := range docs { + buf.WriteString("---") + buf.WriteString("\n") + buf.WriteString(doc) + } + + return buf.String(), nil +} + +// mapBuild populates build variables available to jsonnet templates. +// Since we want to maintain compatibility with drone, the older format +// needs to be maintained (even if the variables do not exist in gitness). +func mapBuild(v *types.Execution, vm *jsonnet.VM) { + vm.ExtVar(build+"event", v.Event) + vm.ExtVar(build+"action", v.Action) + vm.ExtVar(build+"environment", v.Deploy) + vm.ExtVar(build+"link", v.Link) + vm.ExtVar(build+"branch", v.Target) + vm.ExtVar(build+"source", v.Source) + vm.ExtVar(build+"before", v.Before) + vm.ExtVar(build+"after", v.After) + vm.ExtVar(build+"target", v.Target) + vm.ExtVar(build+"ref", v.Ref) + vm.ExtVar(build+"commit", v.After) + vm.ExtVar(build+"ref", v.Ref) + vm.ExtVar(build+"title", v.Title) + vm.ExtVar(build+"message", v.Message) + vm.ExtVar(build+"source_repo", v.Fork) + vm.ExtVar(build+"author_login", v.Author) + vm.ExtVar(build+"author_name", v.AuthorName) + vm.ExtVar(build+"author_email", v.AuthorEmail) + vm.ExtVar(build+"author_avatar", v.AuthorAvatar) + vm.ExtVar(build+"sender", v.Sender) + fromMap(v.Params, vm) +} + +// mapBuild populates repo level variables available to jsonnet templates. +// Since we want to maintain compatibility with drone 2.x, the older format +// needs to be maintained (even if the variables do not exist in gitness). +func mapRepo(v *types.Repository, p *types.Pipeline, vm *jsonnet.VM) { + namespace := v.Path + idx := strings.LastIndex(v.Path, "/") + if idx != -1 { + namespace = v.Path[:idx] + } + vm.ExtVar(repo+"uid", v.UID) + vm.ExtVar(repo+"name", v.UID) + vm.ExtVar(repo+"namespace", namespace) + vm.ExtVar(repo+"slug", v.Path) + vm.ExtVar(repo+"git_http_url", v.GitURL) + vm.ExtVar(repo+"git_ssh_url", v.GitURL) + vm.ExtVar(repo+"link", v.GitURL) + vm.ExtVar(repo+"branch", v.DefaultBranch) + vm.ExtVar(repo+"config", p.ConfigPath) + vm.ExtVar(repo+"private", strconv.FormatBool(!v.IsPublic)) + vm.ExtVar(repo+"visibility", "internal") + vm.ExtVar(repo+"active", strconv.FormatBool(true)) + vm.ExtVar(repo+"trusted", strconv.FormatBool(true)) + vm.ExtVar(repo+"protected", strconv.FormatBool(false)) + vm.ExtVar(repo+"ignore_forks", strconv.FormatBool(false)) + vm.ExtVar(repo+"ignore_pull_requests", strconv.FormatBool(false)) +} + +func fromMap(m map[string]string, vm *jsonnet.VM) { + for k, v := range m { + vm.ExtVar(build+param+k, v) + } +} diff --git a/app/pipeline/converter/service.go b/app/pipeline/converter/service.go new file mode 100644 index 000000000..7f84fb403 --- /dev/null +++ b/app/pipeline/converter/service.go @@ -0,0 +1,39 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package converter + +import ( + "context" + + "github.com/harness/gitness/app/pipeline/file" + "github.com/harness/gitness/types" +) + +type ( + // ConvertArgs represents a request to the pipeline + // conversion service. + ConvertArgs struct { + Repo *types.Repository `json:"repository,omitempty"` + Pipeline *types.Pipeline `json:"pipeline,omitempty"` + Execution *types.Execution `json:"execution,omitempty"` + File *file.File `json:"config,omitempty"` + } + + // Service converts a file which is in starlark/jsonnet form by looking + // at the extension and calling the appropriate parser. + Service interface { + Convert(ctx context.Context, args *ConvertArgs) (*file.File, error) + } +) diff --git a/app/pipeline/converter/starlark/args.go b/app/pipeline/converter/starlark/args.go new file mode 100644 index 000000000..f551a5db4 --- /dev/null +++ b/app/pipeline/converter/starlark/args.go @@ -0,0 +1,105 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package starlark + +import ( + "strings" + + "github.com/harness/gitness/types" + + "go.starlark.net/starlark" + "go.starlark.net/starlarkstruct" +) + +func createArgs( + repo *types.Repository, + pipeline *types.Pipeline, + execution *types.Execution, +) []starlark.Value { + args := []starlark.Value{ + starlarkstruct.FromStringDict( + starlark.String("context"), + starlark.StringDict{ + "repo": starlarkstruct.FromStringDict(starlark.String("repo"), fromRepo(repo, pipeline)), + "build": starlarkstruct.FromStringDict(starlark.String("build"), fromBuild(execution)), + }, + ), + } + return args +} + +func fromBuild(v *types.Execution) starlark.StringDict { + return starlark.StringDict{ + "event": starlark.String(v.Event), + "action": starlark.String(v.Action), + "cron": starlark.String(v.Cron), + "link": starlark.String(v.Link), + "branch": starlark.String(v.Target), + "source": starlark.String(v.Source), + "before": starlark.String(v.Before), + "after": starlark.String(v.After), + "target": starlark.String(v.Target), + "ref": starlark.String(v.Ref), + "commit": starlark.String(v.After), + "title": starlark.String(v.Title), + "message": starlark.String(v.Message), + "source_repo": starlark.String(v.Fork), + "author_login": starlark.String(v.Author), + "author_name": starlark.String(v.AuthorName), + "author_email": starlark.String(v.AuthorEmail), + "author_avatar": starlark.String(v.AuthorAvatar), + "sender": starlark.String(v.Sender), + "debug": starlark.Bool(v.Debug), + "params": fromMap(v.Params), + } +} + +func fromRepo(v *types.Repository, p *types.Pipeline) starlark.StringDict { + namespace := v.Path + idx := strings.LastIndex(v.Path, "/") + if idx != -1 { + namespace = v.Path[:idx] + } + return starlark.StringDict{ + "uid": starlark.String(v.UID), + "name": starlark.String(v.UID), + "namespace": starlark.String(namespace), + "slug": starlark.String(v.Path), + "git_http_url": starlark.String(v.GitURL), + "git_ssh_url": starlark.String(v.GitURL), + "link": starlark.String(v.GitURL), + "branch": starlark.String(v.DefaultBranch), + "config": starlark.String(p.ConfigPath), + "private": !starlark.Bool(v.IsPublic), + "visibility": starlark.String("internal"), + "active": starlark.Bool(true), + "trusted": starlark.Bool(true), + "protected": starlark.Bool(false), + "ignore_forks": starlark.Bool(false), + "ignore_pull_requests": starlark.Bool(false), + } +} + +func fromMap(m map[string]string) *starlark.Dict { + dict := new(starlark.Dict) + for k, v := range m { + //nolint: errcheck + dict.SetKey( + starlark.String(k), + starlark.String(v), + ) + } + return dict +} diff --git a/app/pipeline/converter/starlark/starlark.go b/app/pipeline/converter/starlark/starlark.go new file mode 100644 index 000000000..9269e1791 --- /dev/null +++ b/app/pipeline/converter/starlark/starlark.go @@ -0,0 +1,147 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package starlark + +import ( + "bytes" + "errors" + + "github.com/harness/gitness/app/pipeline/file" + "github.com/harness/gitness/types" + + "github.com/sirupsen/logrus" + "go.starlark.net/starlark" +) + +const ( + separator = "---" + newline = "\n" +) + +// default limit for generated configuration file size. +const defaultSizeLimit = 1000000 + +var ( + // ErrMainMissing indicates the starlark script is missing + // the main method. + ErrMainMissing = errors.New("starlark: missing main function") + + // ErrMainInvalid indicates the starlark script defines a + // global variable named main, however, it is not callable. + ErrMainInvalid = errors.New("starlark: main must be a function") + + // ErrMainReturn indicates the starlark script's main method + // returns an invalid or unexpected type. + ErrMainReturn = errors.New("starlark: main returns an invalid type") + + // ErrMaximumSize indicates the starlark script generated a + // file that exceeds the maximum allowed file size. + ErrMaximumSize = errors.New("starlark: maximum file size exceeded") + + // ErrCannotLoad indicates the starlark script is attempting to + // load an external file which is currently restricted. + ErrCannotLoad = errors.New("starlark: cannot load external scripts") +) + +func Parse( + repo *types.Repository, + pipeline *types.Pipeline, + execution *types.Execution, + file *file.File, + stepLimit uint64, + sizeLimit uint64, +) (string, error) { + thread := &starlark.Thread{ + Name: "drone", + Load: noLoad, + Print: func(_ *starlark.Thread, msg string) { + logrus.WithFields(logrus.Fields{ + "namespace": repo.Path, // TODO: update to just be the space + "name": repo.UID, + }).Traceln(msg) + }, + } + starlarkFile := file.Data + starlarkFileName := pipeline.ConfigPath + + globals, err := starlark.ExecFile(thread, starlarkFileName, starlarkFile, nil) + if err != nil { + return "", err + } + + // find the main method in the starlark script and + // cast to a callable type. If not callable the script + // is invalid. + mainVal, ok := globals["main"] + if !ok { + return "", ErrMainMissing + } + main, ok := mainVal.(starlark.Callable) + if !ok { + return "", ErrMainInvalid + } + + // create the input args and invoke the main method + // using the input args. + args := createArgs(repo, pipeline, execution) + + // set the maximum number of operations in the script. this + // mitigates long running scripts. + if stepLimit == 0 { + stepLimit = 50000 + } + thread.SetMaxExecutionSteps(stepLimit) + + // execute the main method in the script. + mainVal, err = starlark.Call(thread, main, args, nil) + if err != nil { + return "", err + } + + buf := new(bytes.Buffer) + switch v := mainVal.(type) { + case *starlark.List: + for i := 0; i < v.Len(); i++ { + item := v.Index(i) + buf.WriteString(separator) + buf.WriteString(newline) + if err := write(buf, item); err != nil { + return "", err + } + buf.WriteString(newline) + } + case *starlark.Dict: + if err := write(buf, v); err != nil { + return "", err + } + default: + return "", ErrMainReturn + } + + if sizeLimit == 0 { + sizeLimit = defaultSizeLimit + } + + // this is a temporary workaround until we + // implement a LimitWriter. + if b := buf.Bytes(); uint64(len(b)) > sizeLimit { + return "", ErrMaximumSize + } + return buf.String(), nil +} + +func noLoad(_ *starlark.Thread, _ string) (starlark.StringDict, error) { + return nil, ErrCannotLoad +} diff --git a/app/pipeline/converter/starlark/write.go b/app/pipeline/converter/starlark/write.go new file mode 100644 index 000000000..b7f067756 --- /dev/null +++ b/app/pipeline/converter/starlark/write.go @@ -0,0 +1,125 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package starlark + +import ( + "encoding/json" + "fmt" + "io" + + "go.starlark.net/starlark" +) + +type writer interface { + io.Writer + io.ByteWriter + io.StringWriter +} + +//nolint:gocognit,cyclop +func write(out writer, v starlark.Value) error { + if marshaler, ok := v.(json.Marshaler); ok { + jsonData, err := marshaler.MarshalJSON() + if err != nil { + return err + } + if _, err = out.Write(jsonData); err != nil { + return err + } + return nil + } + + switch v := v.(type) { + case starlark.NoneType: + if _, err := out.WriteString("null"); err != nil { + return err + } + case starlark.Bool: + fmt.Fprintf(out, "%t", v) + case starlark.Int: + if _, err := out.WriteString(v.String()); err != nil { + return err + } + case starlark.Float: + fmt.Fprintf(out, "%g", v) + case starlark.String: + s := string(v) + if isQuoteSafe(s) { + fmt.Fprintf(out, "%q", s) + } else { + data, err := json.Marshal(s) + if err != nil { + return err + } + if _, err = out.Write(data); err != nil { + return err + } + } + case starlark.Indexable: + if err := out.WriteByte('['); err != nil { + return err + } + for i, n := 0, starlark.Len(v); i < n; i++ { + if i > 0 { + if _, err := out.WriteString(", "); err != nil { + return err + } + } + if err := write(out, v.Index(i)); err != nil { + return err + } + } + if err := out.WriteByte(']'); err != nil { + return err + } + case *starlark.Dict: + if err := out.WriteByte('{'); err != nil { + return err + } + for i, itemPair := range v.Items() { + key := itemPair[0] + value := itemPair[1] + if i > 0 { + if _, err := out.WriteString(", "); err != nil { + return err + } + } + if err := write(out, key); err != nil { + return err + } + if _, err := out.WriteString(": "); err != nil { + return err + } + if err := write(out, value); err != nil { + return err + } + } + if err := out.WriteByte('}'); err != nil { + return err + } + default: + return fmt.Errorf("value %s (type `%s') can't be converted to JSON", v.String(), v.Type()) + } + return nil +} + +func isQuoteSafe(s string) bool { + for _, r := range s { + if r < 0x20 || r >= 0x10000 { + return false + } + } + return true +} diff --git a/app/pipeline/converter/wire.go b/app/pipeline/converter/wire.go new file mode 100644 index 000000000..2274eac35 --- /dev/null +++ b/app/pipeline/converter/wire.go @@ -0,0 +1,31 @@ +// Copyright 2023 Harness, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package converter + +import ( + "github.com/harness/gitness/app/pipeline/file" + + "github.com/google/wire" +) + +// WireSet provides a wire set for this package. +var WireSet = wire.NewSet( + ProvideService, +) + +// ProvideService provides a service which can convert templates. +func ProvideService(fileService file.Service) Service { + return newConverter(fileService) +} diff --git a/app/pipeline/manager/manager.go b/app/pipeline/manager/manager.go index 21d355bed..2dfdb2d54 100644 --- a/app/pipeline/manager/manager.go +++ b/app/pipeline/manager/manager.go @@ -24,6 +24,7 @@ import ( "github.com/harness/gitness/app/bootstrap" "github.com/harness/gitness/app/jwt" + "github.com/harness/gitness/app/pipeline/converter" "github.com/harness/gitness/app/pipeline/file" "github.com/harness/gitness/app/pipeline/scheduler" "github.com/harness/gitness/app/sse" @@ -125,12 +126,13 @@ type ( // Manager provides a simplified interface to the build runner so that it // can more easily interact with the server. type Manager struct { - Executions store.ExecutionStore - Config *types.Config - FileService file.Service - Pipelines store.PipelineStore - urlProvider urlprovider.Provider - Checks store.CheckStore + Executions store.ExecutionStore + Config *types.Config + FileService file.Service + ConverterService converter.Service + Pipelines store.PipelineStore + urlProvider urlprovider.Provider + Checks store.CheckStore // Converter store.ConvertService SSEStreamer sse.Streamer // Globals store.GlobalSecretStore @@ -155,6 +157,7 @@ func New( urlProvider urlprovider.Provider, sseStreamer sse.Streamer, fileService file.Service, + converterService converter.Service, logStore store.LogStore, logStream livelog.LogStream, checkStore store.CheckStore, @@ -166,21 +169,22 @@ func New( userStore store.PrincipalStore, ) *Manager { return &Manager{ - Config: config, - Executions: executionStore, - Pipelines: pipelineStore, - urlProvider: urlProvider, - SSEStreamer: sseStreamer, - FileService: fileService, - Logs: logStore, - Logz: logStream, - Checks: checkStore, - Repos: repoStore, - Scheduler: scheduler, - Secrets: secretStore, - Stages: stageStore, - Steps: stepStore, - Users: userStore, + Config: config, + Executions: executionStore, + Pipelines: pipelineStore, + urlProvider: urlProvider, + SSEStreamer: sseStreamer, + FileService: fileService, + ConverterService: converterService, + Logs: logStore, + Logz: logStream, + Checks: checkStore, + Repos: repoStore, + Scheduler: scheduler, + Secrets: secretStore, + Stages: stageStore, + Steps: stepStore, + Users: userStore, } } @@ -325,6 +329,19 @@ func (m *Manager) Details(_ context.Context, stageID int64) (*ExecutionContext, return nil, err } + // Convert file contents in case templates are being used. + args := &converter.ConvertArgs{ + Repo: repo, + Pipeline: pipeline, + Execution: execution, + File: file, + } + file, err = m.ConverterService.Convert(noContext, args) + if err != nil { + log.Warn().Err(err).Msg("manager: cannot convert template contents") + return nil, err + } + netrc, err := m.createNetrc(repo) if err != nil { log.Warn().Err(err).Msg("manager: failed to create netrc") diff --git a/app/pipeline/manager/wire.go b/app/pipeline/manager/wire.go index bb53ea8df..448deefd4 100644 --- a/app/pipeline/manager/wire.go +++ b/app/pipeline/manager/wire.go @@ -15,6 +15,7 @@ package manager import ( + "github.com/harness/gitness/app/pipeline/converter" "github.com/harness/gitness/app/pipeline/file" "github.com/harness/gitness/app/pipeline/scheduler" "github.com/harness/gitness/app/sse" @@ -41,6 +42,7 @@ func ProvideExecutionManager( urlProvider url.Provider, sseStreamer sse.Streamer, fileService file.Service, + converterService converter.Service, logStore store.LogStore, logStream livelog.LogStream, checkStore store.CheckStore, @@ -50,8 +52,8 @@ func ProvideExecutionManager( stageStore store.StageStore, stepStore store.StepStore, userStore store.PrincipalStore) ExecutionManager { - return New(config, executionStore, pipelineStore, urlProvider, sseStreamer, fileService, logStore, - logStream, checkStore, repoStore, scheduler, secretStore, stageStore, stepStore, userStore) + return New(config, executionStore, pipelineStore, urlProvider, sseStreamer, fileService, converterService, + logStore, logStream, checkStore, repoStore, scheduler, secretStore, stageStore, stepStore, userStore) } // ProvideExecutionClient provides a client implementation to interact with the execution manager. diff --git a/app/pipeline/triggerer/trigger.go b/app/pipeline/triggerer/trigger.go index 62d910943..194bc3dab 100644 --- a/app/pipeline/triggerer/trigger.go +++ b/app/pipeline/triggerer/trigger.go @@ -22,6 +22,7 @@ import ( "time" "github.com/harness/gitness/app/pipeline/checks" + "github.com/harness/gitness/app/pipeline/converter" "github.com/harness/gitness/app/pipeline/file" "github.com/harness/gitness/app/pipeline/manager" "github.com/harness/gitness/app/pipeline/scheduler" @@ -77,15 +78,16 @@ type Triggerer interface { } type triggerer struct { - executionStore store.ExecutionStore - checkStore store.CheckStore - stageStore store.StageStore - tx dbtx.Transactor - pipelineStore store.PipelineStore - fileService file.Service - urlProvider url.Provider - scheduler scheduler.Scheduler - repoStore store.RepoStore + executionStore store.ExecutionStore + checkStore store.CheckStore + stageStore store.StageStore + tx dbtx.Transactor + pipelineStore store.PipelineStore + fileService file.Service + converterService converter.Service + urlProvider url.Provider + scheduler scheduler.Scheduler + repoStore store.RepoStore } func New( @@ -98,17 +100,19 @@ func New( urlProvider url.Provider, scheduler scheduler.Scheduler, fileService file.Service, + converterService converter.Service, ) Triggerer { return &triggerer{ - executionStore: executionStore, - checkStore: checkStore, - stageStore: stageStore, - scheduler: scheduler, - urlProvider: urlProvider, - tx: tx, - pipelineStore: pipelineStore, - fileService: fileService, - repoStore: repoStore, + executionStore: executionStore, + checkStore: checkStore, + stageStore: stageStore, + scheduler: scheduler, + urlProvider: urlProvider, + tx: tx, + pipelineStore: pipelineStore, + fileService: fileService, + converterService: converterService, + repoStore: repoStore, } } @@ -186,6 +190,19 @@ func (t *triggerer) Trigger( stages := []*types.Stage{} //nolint:nestif // refactor if needed if !isV1Yaml(file.Data) { + // Convert from jsonnet/starlark to drone yaml + args := &converter.ConvertArgs{ + Repo: repo, + Pipeline: pipeline, + Execution: execution, + File: file, + } + file, err = t.converterService.Convert(ctx, args) + if err != nil { + log.Warn().Err(err).Msg("trigger: cannot convert from template") + return t.createExecutionWithError(ctx, pipeline, base, err.Error()) + } + manifest, err := yaml.ParseString(string(file.Data)) if err != nil { log.Warn().Err(err).Msg("trigger: cannot parse yaml") diff --git a/app/pipeline/triggerer/wire.go b/app/pipeline/triggerer/wire.go index 646c52442..9935036cd 100644 --- a/app/pipeline/triggerer/wire.go +++ b/app/pipeline/triggerer/wire.go @@ -15,6 +15,7 @@ package triggerer import ( + "github.com/harness/gitness/app/pipeline/converter" "github.com/harness/gitness/app/pipeline/file" "github.com/harness/gitness/app/pipeline/scheduler" "github.com/harness/gitness/app/store" @@ -37,10 +38,11 @@ func ProvideTriggerer( tx dbtx.Transactor, pipelineStore store.PipelineStore, fileService file.Service, + converterService converter.Service, scheduler scheduler.Scheduler, repoStore store.RepoStore, urlProvider url.Provider, ) Triggerer { return New(executionStore, checkStore, stageStore, pipelineStore, - tx, repoStore, urlProvider, scheduler, fileService) + tx, repoStore, urlProvider, scheduler, fileService, converterService) } diff --git a/cmd/gitness/wire.go b/cmd/gitness/wire.go index 984d44e24..86c69facd 100644 --- a/cmd/gitness/wire.go +++ b/cmd/gitness/wire.go @@ -40,6 +40,7 @@ import ( "github.com/harness/gitness/app/githook" "github.com/harness/gitness/app/pipeline/canceler" "github.com/harness/gitness/app/pipeline/commit" + "github.com/harness/gitness/app/pipeline/converter" "github.com/harness/gitness/app/pipeline/file" "github.com/harness/gitness/app/pipeline/manager" pluginmanager "github.com/harness/gitness/app/pipeline/plugin" @@ -159,6 +160,7 @@ func initSystem(ctx context.Context, config *types.Config) (*cliserver.System, e manager.WireSet, triggerer.WireSet, file.WireSet, + converter.WireSet, runner.WireSet, sse.WireSet, scheduler.WireSet, diff --git a/cmd/gitness/wire_gen.go b/cmd/gitness/wire_gen.go index d41564898..60c87e92e 100644 --- a/cmd/gitness/wire_gen.go +++ b/cmd/gitness/wire_gen.go @@ -39,6 +39,7 @@ import ( "github.com/harness/gitness/app/githook" "github.com/harness/gitness/app/pipeline/canceler" "github.com/harness/gitness/app/pipeline/commit" + "github.com/harness/gitness/app/pipeline/converter" "github.com/harness/gitness/app/pipeline/file" "github.com/harness/gitness/app/pipeline/manager" plugin2 "github.com/harness/gitness/app/pipeline/plugin" @@ -198,7 +199,8 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro cancelerCanceler := canceler.ProvideCanceler(executionStore, streamer, repoStore, schedulerScheduler, stageStore, stepStore) commitService := commit.ProvideService(gitInterface) fileService := file.ProvideService(gitInterface) - triggererTriggerer := triggerer.ProvideTriggerer(executionStore, checkStore, stageStore, transactor, pipelineStore, fileService, schedulerScheduler, repoStore, provider) + converterService := converter.ProvideService(fileService) + triggererTriggerer := triggerer.ProvideTriggerer(executionStore, checkStore, stageStore, transactor, pipelineStore, fileService, converterService, schedulerScheduler, repoStore, provider) executionController := execution.ProvideController(transactor, authorizer, executionStore, checkStore, cancelerCanceler, commitService, triggererTriggerer, repoStore, stageStore, pipelineStore) logStore := logs.ProvideLogStore(db, config) logStream := livelog.ProvideLogStream() @@ -278,7 +280,7 @@ func initSystem(ctx context.Context, config *types.Config) (*server.System, erro webHandler := router.ProvideWebHandler(config) routerRouter := router.ProvideRouter(apiHandler, gitHandler, webHandler, provider) serverServer := server2.ProvideServer(config, routerRouter) - executionManager := manager.ProvideExecutionManager(config, executionStore, pipelineStore, provider, streamer, fileService, logStore, logStream, checkStore, repoStore, schedulerScheduler, secretStore, stageStore, stepStore, principalStore) + executionManager := manager.ProvideExecutionManager(config, executionStore, pipelineStore, provider, streamer, fileService, converterService, logStore, logStream, checkStore, repoStore, schedulerScheduler, secretStore, stageStore, stepStore, principalStore) client := manager.ProvideExecutionClient(executionManager, provider, config) pluginManager := plugin2.ProvidePluginManager(config, pluginStore) runtimeRunner, err := runner.ProvideExecutionRunner(config, client, pluginManager) diff --git a/go.mod b/go.mod index 4ef911513..ecc6d7d3f 100644 --- a/go.mod +++ b/go.mod @@ -138,6 +138,7 @@ require ( github.com/golang/snappy v0.0.4 // indirect github.com/google/btree v1.0.1 // indirect github.com/google/certificate-transparency-go v1.1.2-0.20210511102531-373a877eec92 // indirect + github.com/google/go-jsonnet v0.20.0 // indirect github.com/google/s2a-go v0.1.4 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect github.com/googleapis/gax-go/v2 v2.12.0 // indirect @@ -212,6 +213,7 @@ require ( go.etcd.io/etcd/tests/v3 v3.5.0-alpha.0 // indirect go.etcd.io/etcd/v3 v3.5.0-alpha.0 // indirect go.opencensus.io v0.24.0 // indirect + go.starlark.net v0.0.0-20231121155337-90ade8b19d09 // indirect go.uber.org/zap v1.21.0 // indirect golang.org/x/oauth2 v0.10.0 // indirect golang.org/x/time v0.0.0-20220411224347-583f2d630306 // indirect diff --git a/go.sum b/go.sum index 12982394c..d4733e9c9 100644 --- a/go.sum +++ b/go.sum @@ -650,6 +650,8 @@ github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeN github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-github/v28 v28.1.1/go.mod h1:bsqJWQX05omyWVmc00nEUql9mhQyv38lDZ8kPZcQVoM= +github.com/google/go-jsonnet v0.20.0 h1:WG4TTSARuV7bSm4PMB4ohjxe33IHT5WVTrJSU33uT4g= +github.com/google/go-jsonnet v0.20.0/go.mod h1:VbgWF9JX7ztlv770x/TolZNGGFfiHEVx9G6ca2eUmeA= github.com/google/go-licenses v0.0.0-20210329231322-ce1d9163b77d/go.mod h1:+TYOmkVoJOpwnS0wfdsJCV9CoD5nJYsHoFk/0CrTK4M= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/go-replayers/grpcreplay v0.1.0/go.mod h1:8Ig2Idjpr6gifRd6pNVggX6TC1Zw6Jx74AKp7QNH2QE= @@ -1488,6 +1490,8 @@ go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= +go.starlark.net v0.0.0-20231121155337-90ade8b19d09 h1:hzy3LFnSN8kuQK8h9tHl4ndF6UruMj47OqwqsS+/Ai4= +go.starlark.net v0.0.0-20231121155337-90ade8b19d09/go.mod h1:LcLNIzVOMp4oV+uusnpk+VU+SzXaJakUuBjoCSWH5dM= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ=