diff --git a/agent/agent.go b/agent/agent.go index c16d11c29..5e82e625f 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -13,8 +13,8 @@ import ( "github.com/drone/drone/model" "github.com/drone/drone/version" "github.com/drone/drone/yaml" - "github.com/drone/drone/yaml/expander" "github.com/drone/drone/yaml/transform" + "github.com/drone/envsubst" ) type Logger interface { @@ -93,7 +93,14 @@ func (a *Agent) Run(payload *model.Work, cancel <-chan bool) error { func (a *Agent) prep(w *model.Work) (*yaml.Config, error) { envs := toEnv(w) - w.Yaml = expander.ExpandString(w.Yaml, envs) + + var err error + w.Yaml, err = envsubst.Eval(w.Yaml, func(s string) string { + return envs[s] + }) + if err != nil { + return nil, err + } // append secrets when verified or when a secret does not require // verification diff --git a/vendor/github.com/drone/envsubst/LICENSE b/vendor/github.com/drone/envsubst/LICENSE new file mode 100644 index 000000000..1de55b7f4 --- /dev/null +++ b/vendor/github.com/drone/envsubst/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2017 drone.io + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/vendor/github.com/drone/envsubst/README b/vendor/github.com/drone/envsubst/README new file mode 100644 index 000000000..7bbd20623 --- /dev/null +++ b/vendor/github.com/drone/envsubst/README @@ -0,0 +1,33 @@ +Go package emulates bash environment variable substitution in a string using ${var} syntax. Includes support for bash string replacement functions. + +Documentation: + + http://godoc.org/github.com/drone/env + +Supported Functions: + + ${var^} + ${var^^} + ${var,} + ${var,,} + ${var:position} + ${var:position:length} + ${var#substring} + ${var##substring} + ${var%substring} + ${var%%substring} + ${var/substring/replacement} + ${var//substring/replacement} + ${var/#substring/replacement} + ${var/%substring/replacement} + ${#var} + ${var=default} + ${var:=default} + ${var:-default} + +Unsupported Functions: + + ${var-default} + ${var+default} + ${var:?default} + ${var:+default} diff --git a/vendor/github.com/drone/envsubst/eval.go b/vendor/github.com/drone/envsubst/eval.go new file mode 100644 index 000000000..375ca4c9f --- /dev/null +++ b/vendor/github.com/drone/envsubst/eval.go @@ -0,0 +1,19 @@ +package envsubst + +import "os" + +// Eval replaces ${var} in the string based on the mapping function. +func Eval(s string, mapping func(string) string) (string, error) { + t, err := Parse(s) + if err != nil { + return s, err + } + return t.Execute(mapping) +} + +// EvalEnv replaces ${var} in the string according to the values of the +// current environment variables. References to undefined variables are +// replaced by the empty string. +func EvalEnv(s string) (string, error) { + return Eval(s, os.Getenv) +} diff --git a/vendor/github.com/drone/envsubst/funcs.go b/vendor/github.com/drone/envsubst/funcs.go new file mode 100644 index 000000000..a4cb00f1f --- /dev/null +++ b/vendor/github.com/drone/envsubst/funcs.go @@ -0,0 +1,177 @@ +package envsubst + +import ( + "strconv" + "strings" + "unicode" + "unicode/utf8" +) + +// defines a parameter substitution function. +type substituteFunc func(string, ...string) string + +// toLen returns the length of string s. +func toLen(s string, args ...string) string { + return strconv.Itoa(len(s)) +} + +// toLower returns a copy of the string s with all characters +// mapped to their lower case. +func toLower(s string, args ...string) string { + return strings.ToLower(s) +} + +// toUpper returns a copy of the string s with all characters +// mapped to their upper case. +func toUpper(s string, args ...string) string { + return strings.ToUpper(s) +} + +// toLowerFirst returns a copy of the string s with the first +// character mapped to its lower case. +func toLowerFirst(s string, args ...string) string { + if s == "" { + return s + } + r, n := utf8.DecodeRuneInString(s) + return string(unicode.ToLower(r)) + s[n:] +} + +// toUpperFirst returns a copy of the string s with the first +// character mapped to its upper case. +func toUpperFirst(s string, args ...string) string { + if s == "" { + return s + } + r, n := utf8.DecodeRuneInString(s) + return string(unicode.ToUpper(r)) + s[n:] +} + +// toDefault returns a copy of the string s if not empty, else +// returns a copy of the first string arugment. +func toDefault(s string, args ...string) string { + if len(s) == 0 && len(args) == 1 { + s = args[0] + } + return s +} + +// toSubstr returns a slice of the string s at the specified +// length and position. +func toSubstr(s string, args ...string) string { + if len(args) == 0 { + return s // should never happen + } + + pos, err := strconv.Atoi(args[0]) + if err != nil { + // bash returns the string if the position + // cannot be parsed. + return s + } + + if len(args) == 1 { + if pos < len(s) { + return s[pos:] + } + // if the position exceeds the length of the + // string an empty string is returned + return "" + } + + length, err := strconv.Atoi(args[1]) + if err != nil { + // bash returns the string if the length + // cannot be parsed. + return s + } + + if pos+length >= len(s) { + // if the position exceeds the length of the + // string an empty string is returned + return "" + } + + return s[pos : pos+length] +} + +// replaceAll returns a copy of the string s with all instances +// of the substring replaced with the replacement string. +func replaceAll(s string, args ...string) string { + switch len(args) { + case 0: + return s + case 1: + return strings.Replace(s, args[0], "", -1) + default: + return strings.Replace(s, args[0], args[1], -1) + } +} + +// replaceFirst returns a copy of the string s with the first +// instance of the substring replaced with the replacement string. +func replaceFirst(s string, args ...string) string { + switch len(args) { + case 0: + return s + case 1: + return strings.Replace(s, args[0], "", 1) + default: + return strings.Replace(s, args[0], args[1], 1) + } +} + +// replacePrefix returns a copy of the string s with the matching +// prefix replaced with the replacement string. +func replacePrefix(s string, args ...string) string { + if len(args) != 2 { + return s + } + if strings.HasPrefix(s, args[0]) { + return strings.Replace(s, args[0], args[1], 1) + } + return s +} + +// replaceSuffix returns a copy of the string s with the matching +// suffix replaced with the replacement string. +func replaceSuffix(s string, args ...string) string { + if len(args) != 2 { + return s + } + if strings.HasSuffix(s, args[0]) { + s = strings.TrimSuffix(s, args[0]) + s = s + args[1] + } + return s +} + +// TODO + +func trimShortestPrefix(s string, args ...string) string { + if len(args) != 0 { + s = strings.TrimPrefix(s, args[0]) + } + return s +} + +func trimShortestSuffix(s string, args ...string) string { + if len(args) != 0 { + s = strings.TrimSuffix(s, args[0]) + } + return s +} + +func trimLongestPrefix(s string, args ...string) string { + if len(args) != 0 { + s = strings.TrimPrefix(s, args[0]) + } + return s +} + +func trimLongestSuffix(s string, args ...string) string { + if len(args) != 0 { + s = strings.TrimSuffix(s, args[0]) + } + return s +} diff --git a/vendor/github.com/drone/envsubst/match.go b/vendor/github.com/drone/envsubst/match.go new file mode 100644 index 000000000..e2f94b580 --- /dev/null +++ b/vendor/github.com/drone/envsubst/match.go @@ -0,0 +1,20 @@ +package envsubst + +func matches() { + +} + +// alnum +// alpha +// ascii +// blank +// cntrl +// digit +// graph +// lower +// print +// punct +// space +// upper +// word +// xdigit diff --git a/vendor/github.com/drone/envsubst/template.go b/vendor/github.com/drone/envsubst/template.go new file mode 100644 index 000000000..b725d4f62 --- /dev/null +++ b/vendor/github.com/drone/envsubst/template.go @@ -0,0 +1,157 @@ +package envsubst + +import ( + "bytes" + "io" + "io/ioutil" + + "github.com/drone/envsubst/parse" +) + +// state represents the state of template execution. It is not part of the +// template so that multiple executions can run in parallel. +type state struct { + template *Template + writer io.Writer + node parse.Node // current node + + // maps variable names to values + mapper func(string) string +} + +// Template is the representation of a parsed shell format string. +type Template struct { + tree *parse.Tree +} + +// Parse creates a new shell format template and parses the template +// definition from string s. +func Parse(s string) (t *Template, err error) { + t = new(Template) + t.tree, err = parse.Parse(s) + if err != nil { + return nil, err + } + return t, nil +} + +// ParseFile creates a new shell format template and parses the template +// definition from the named file. +func ParseFile(path string) (*Template, error) { + b, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + return Parse(string(b)) +} + +// Execute applies a parsed template to the specified data mapping. +func (t *Template) Execute(mapping func(string) string) (str string, err error) { + b := new(bytes.Buffer) + s := new(state) + s.node = t.tree.Root + s.mapper = mapping + s.writer = b + err = t.eval(s) + if err != nil { + return + } + return b.String(), nil +} + +func (t *Template) eval(s *state) (err error) { + switch node := s.node.(type) { + case *parse.TextNode: + err = t.evalText(s, node) + case *parse.FuncNode: + err = t.evalFunc(s, node) + case *parse.ListNode: + err = t.evalList(s, node) + } + return err +} + +func (t *Template) evalText(s *state, node *parse.TextNode) error { + _, err := io.WriteString(s.writer, node.Value) + return err +} + +func (t *Template) evalList(s *state, node *parse.ListNode) (err error) { + for _, n := range node.Nodes { + s.node = n + err = t.eval(s) + if err != nil { + return err + } + } + return nil +} + +func (t *Template) evalFunc(s *state, node *parse.FuncNode) error { + var w = s.writer + var buf bytes.Buffer + var args []string + for _, n := range node.Args { + buf.Reset() + s.writer = &buf + s.node = n + err := t.eval(s) + if err != nil { + return err + } + args = append(args, buf.String()) + } + + // restore the origin writer + s.writer = w + s.node = node + + v := s.mapper(node.Param) + + fn := lookupFunc(node.Name, len(args)) + + _, err := io.WriteString(s.writer, fn(v, args...)) + return err +} + +// lookupFunc returns the parameters substitution function by name. If the +// named function does not exists, a default function is returned. +func lookupFunc(name string, args int) substituteFunc { + switch name { + case ",": + return toLowerFirst + case ",,": + return toLower + case "^": + return toUpperFirst + case "^^": + return toUpper + case "#": + if args == 0 { + return toLen + } + return trimShortestPrefix + case "##": + return trimLongestPrefix + case "%": + return trimShortestSuffix + case "%%": + return trimLongestSuffix + case ":": + return toSubstr + case "/#": + return replacePrefix + case "/%": + return replaceSuffix + case "/": + return replaceFirst + case "//": + return replaceAll + case "=", ":=", ":-": + return toDefault + case ":?", ":+", "-", "+": + return toDefault + default: + return toDefault + } +} diff --git a/vendor/vendor.json b/vendor/vendor.json index 7b2b54177..e5459cea2 100644 --- a/vendor/vendor.json +++ b/vendor/vendor.json @@ -53,6 +53,12 @@ "revision": "5d2041e26a699eaca682e2ea41c8f891e1060444", "revisionTime": "2016-01-25T09:48:45-08:00" }, + { + "checksumSHA1": "7tosn2Sxlubl+7ElXSZ6Mz8tAjY=", + "path": "github.com/drone/envsubst", + "revision": "3e65ae5fd2d944d56fdf52cb3f887247498d50e9", + "revisionTime": "2017-01-18T15:01:55Z" + }, { "path": "github.com/eknkc/amber", "revision": "144da19a9994994c069f0693294a66dd310e14a4", @@ -208,7 +214,6 @@ }, { "checksumSHA1": "+HvW+k8YkDaPKwF0Lwcz+Tf2A+E=", - "origin": "github.com/drone/drone/vendor/github.com/samalba/dockerclient", "path": "github.com/samalba/dockerclient", "revision": "91d7393ff85980ba3a8966405871a3d446ca28f2", "revisionTime": "2016-04-14T17:47:13Z" diff --git a/yaml/expander/expand.go b/yaml/expander/expand.go deleted file mode 100644 index fbf7af03e..000000000 --- a/yaml/expander/expand.go +++ /dev/null @@ -1,33 +0,0 @@ -package expander - -import "sort" - -// Expand expands variables into the Yaml configuration using a -// ${key} template parameter with limited support for bash string functions. -func Expand(config []byte, envs map[string]string) []byte { - return []byte( - ExpandString(string(config), envs), - ) -} - -// ExpandString injects the variables into the Yaml configuration string using -// a ${key} template parameter with limited support for bash string functions. -func ExpandString(config string, envs map[string]string) string { - if envs == nil || len(envs) == 0 { - return config - } - keys := []string{} - for k := range envs { - keys = append(keys, k) - } - sort.Sort(sort.Reverse(sort.StringSlice(keys))) - expanded := config - for _, k := range keys { - v := envs[k] - - for _, substitute := range substitutors { - expanded = substitute(expanded, k, v) - } - } - return expanded -} diff --git a/yaml/expander/expand_test.go b/yaml/expander/expand_test.go deleted file mode 100644 index 60a6ba29c..000000000 --- a/yaml/expander/expand_test.go +++ /dev/null @@ -1,48 +0,0 @@ -package expander - -import ( - "testing" - - "github.com/franela/goblin" -) - -func TestExpand(t *testing.T) { - - g := goblin.Goblin(t) - g.Describe("Expand params", func() { - - g.It("Should replace vars with ${key}", func() { - s := "echo ${FOO} $BAR" - m := map[string]string{} - m["FOO"] = "BAZ" - g.Assert("echo BAZ $BAR").Equal(ExpandString(s, m)) - }) - - g.It("Should not replace vars in nil map", func() { - s := "echo ${FOO} $BAR" - g.Assert(s).Equal(ExpandString(s, nil)) - }) - - g.It("Should escape quoted variables", func() { - s := `echo "${FOO}"` - m := map[string]string{} - m["FOO"] = "hello\nworld" - g.Assert(`echo "hello\nworld"`).Equal(ExpandString(s, m)) - }) - - g.It("Should replace variable prefix", func() { - s := `tag: ${TAG=${SHA:8}}` - m := map[string]string{} - m["TAG"] = "" - m["SHA"] = "f36cbf54ee1a1eeab264c8e388f386218ab1701b" - g.Assert("tag: f36cbf54").Equal(ExpandString(s, m)) - }) - - g.It("Should handle nested substitution operations", func() { - s := `echo "${TAG##v}"` - m := map[string]string{} - m["TAG"] = "v1.0.0" - g.Assert(`echo "1.0.0"`).Equal(ExpandString(s, m)) - }) - }) -} diff --git a/yaml/expander/func.go b/yaml/expander/func.go deleted file mode 100644 index d942ad903..000000000 --- a/yaml/expander/func.go +++ /dev/null @@ -1,182 +0,0 @@ -package expander - -import ( - "fmt" - "regexp" - "strconv" - "strings" -) - -// these are helper functions that bring bash-substitution to the drone yaml file. -// see http://tldp.org/LDP/abs/html/parameter-substitution.html - -type substituteFunc func(str, key, val string) string - -var substitutors = []substituteFunc{ - substituteQ, - substitute, - substitutePrefix, - substituteSuffix, - substituteDefault, - substituteReplace, - substituteLeft, - substituteSubstr, -} - -// substitute is a helper function that substitutes a simple parameter using -// ${parameter} notation. -func substitute(str, key, val string) string { - key = fmt.Sprintf("${%s}", key) - return strings.Replace(str, key, val, -1) -} - -// substituteQ is a helper function that substitutes a simple parameter using -// "${parameter}" notation with the escaped value, using %q. -func substituteQ(str, key, val string) string { - key = fmt.Sprintf(`"${%s}"`, key) - val = fmt.Sprintf("%q", val) - return strings.Replace(str, key, val, -1) -} - -// substitutePrefix is a helper function that substitutes parameters using -// ${parameter##prefix} notation with the parameter value minus the trimmed prefix. -func substitutePrefix(str, key, val string) string { - key = fmt.Sprintf("\\${%s##(.+)}", key) - reg, err := regexp.Compile(key) - if err != nil { - return str - } - for _, match := range reg.FindAllStringSubmatch(str, -1) { - if len(match) != 2 { - continue - } - val_ := strings.TrimPrefix(val, match[1]) - str = strings.Replace(str, match[0], val_, -1) - } - return str -} - -// substituteSuffix is a helper function that substitutes parameters using -// ${parameter%%suffix} notation with the parameter value minus the trimmed suffix. -func substituteSuffix(str, key, val string) string { - key = fmt.Sprintf("\\${%s%%%%(.+)}", key) - reg, err := regexp.Compile(key) - if err != nil { - return str - } - for _, match := range reg.FindAllStringSubmatch(str, -1) { - if len(match) != 2 { - continue - } - val_ := strings.TrimSuffix(val, match[1]) - str = strings.Replace(str, match[0], val_, -1) - } - return str -} - -// substituteDefault is a helper function that substitutes parameters using -// ${parameter=default} notation with the parameter value. When empty the -// default value is used. -func substituteDefault(str, key, val string) string { - key = fmt.Sprintf("\\${%s=(.+)}", key) - reg, err := regexp.Compile(key) - if err != nil { - return str - } - for _, match := range reg.FindAllStringSubmatch(str, -1) { - if len(match) != 2 { - continue - } - if len(val) == 0 { - str = strings.Replace(str, match[0], match[1], -1) - } else { - str = strings.Replace(str, match[0], val, -1) - } - } - return str -} - -// unescapeBackslash is a helper function to unescape any backslashes in str. -// Note that no actual literal conversions are done. -func unescapeBackslash(str string) string { - re := regexp.MustCompile(`\\(.)`) - - return string(re.ReplaceAll([]byte(str), []byte("$1"))) -} - -// substituteReplace is a helper function that substitutes parameters using -// ${parameter/old/new} notation with the parameter value. A find and replace -// is performed before injecting the strings, replacing the old pattern with -// the new value. -func substituteReplace(str, key, val string) string { - key = fmt.Sprintf(`\${%s/((?:\\.|[^\\])+)/(.+)}`, key) - reg, err := regexp.Compile(key) - if err != nil { - return str - } - match := reg.FindStringSubmatch(str) - if match == nil { - return str - } - - old := unescapeBackslash(match[1]) - new := unescapeBackslash(match[2]) - - with := strings.Replace(val, old, new, -1) - return strings.Replace(str, match[0], with, -1) -} - -// substituteLeft is a helper function that substitutes parameters using -// ${parameter:pos} notation with the parameter value, sliced up to the -// specified position. -func substituteLeft(str, key, val string) string { - key = fmt.Sprintf("\\${%s:([0-9]*)}", key) - reg, err := regexp.Compile(key) - if err != nil { - return str - } - for _, match := range reg.FindAllStringSubmatch(str, -1) { - if len(match) != 2 { - continue - } - index, err := strconv.Atoi(match[1]) - if err != nil { - continue // skip - } - if index > len(val)-1 { - continue // skip - } - - str = strings.Replace(str, match[0], val[:index], -1) - } - return str -} - -// substituteLeft is a helper function that substitutes parameters using -// ${parameter:pos:len} notation with the parameter value as a substring, -// starting at the specified position for the specified length. -func substituteSubstr(str, key, val string) string { - key = fmt.Sprintf("\\${%s:([0-9]*):([0-9]*)}", key) - reg, err := regexp.Compile(key) - if err != nil { - return str - } - for _, match := range reg.FindAllStringSubmatch(str, -1) { - if len(match) != 3 { - continue - } - pos, err := strconv.Atoi(match[1]) - if err != nil { - continue // skip - } - length, err := strconv.Atoi(match[2]) - if err != nil { - continue // skip - } - if pos+length > len(val)-1 { - continue // skip - } - str = strings.Replace(str, match[0], val[pos:pos+length], -1) - } - return str -} diff --git a/yaml/expander/func_test.go b/yaml/expander/func_test.go deleted file mode 100644 index 8618635b1..000000000 --- a/yaml/expander/func_test.go +++ /dev/null @@ -1,74 +0,0 @@ -package expander - -import ( - "testing" - - "github.com/franela/goblin" -) - -func TestSubstitution(t *testing.T) { - - g := goblin.Goblin(t) - g.Describe("Parameter Substitution", func() { - - g.It("Should substitute simple parameters", func() { - before := "echo ${GREETING} WORLD" - after := "echo HELLO WORLD" - g.Assert(substitute(before, "GREETING", "HELLO")).Equal(after) - }) - - g.It("Should substitute quoted parameters", func() { - before := "echo \"${GREETING}\" WORLD" - after := "echo \"HELLO\" WORLD" - g.Assert(substituteQ(before, "GREETING", "HELLO")).Equal(after) - }) - - g.It("Should substitute parameters and trim prefix", func() { - before := "echo ${GREETING##asdf} WORLD" - after := "echo HELLO WORLD" - g.Assert(substitutePrefix(before, "GREETING", "asdfHELLO")).Equal(after) - }) - - g.It("Should substitute parameters and trim suffix", func() { - before := "echo ${GREETING%%asdf} WORLD" - after := "echo HELLO WORLD" - g.Assert(substituteSuffix(before, "GREETING", "HELLOasdf")).Equal(after) - }) - - g.It("Should substitute parameters without using the default", func() { - before := "echo ${GREETING=HOLA} WORLD" - after := "echo HELLO WORLD" - g.Assert(substituteDefault(before, "GREETING", "HELLO")).Equal(after) - }) - - g.It("Should substitute parameters using the a default", func() { - before := "echo ${GREETING=HOLA} WORLD" - after := "echo HOLA WORLD" - g.Assert(substituteDefault(before, "GREETING", "")).Equal(after) - }) - - g.It("Should substitute parameters with replacement", func() { - before := "echo ${GREETING/HE/A} MONDE" - after := "echo ALLO MONDE" - g.Assert(substituteReplace(before, "GREETING", "HELLO")).Equal(after) - }) - - g.It("Should substitute parameters with replacement, containing slashes", func() { - before := `echo ${GREETING/HE\//A} MONDE` - after := "echo ALLO MONDE" - g.Assert(substituteReplace(before, "GREETING", "HE/LLO")).Equal(after) - }) - - g.It("Should substitute parameters with left substr", func() { - before := "echo ${FOO:4} IS COOL" - after := "echo THIS IS COOL" - g.Assert(substituteLeft(before, "FOO", "THIS IS A REALLY LONG STRING")).Equal(after) - }) - - g.It("Should substitute parameters with substr", func() { - before := "echo ${FOO:8:5} IS COOL" - after := "echo DRONE IS COOL" - g.Assert(substituteSubstr(before, "FOO", "THIS IS DRONE CI")).Equal(after) - }) - }) -}