From 110b9c3dd2252ff34118767dc7f25096a7de430b Mon Sep 17 00:00:00 2001 From: Mac Browning Date: Fri, 12 Feb 2016 13:17:30 -0500 Subject: [PATCH 1/3] resume waiting for the container to complete --- shared/docker/docker.go | 56 ++++++++++++++++++++++++++++++----------- 1 file changed, 41 insertions(+), 15 deletions(-) diff --git a/shared/docker/docker.go b/shared/docker/docker.go index 29ed10613..7bc2ed075 100644 --- a/shared/docker/docker.go +++ b/shared/docker/docker.go @@ -1,9 +1,11 @@ package docker import ( + "errors" "io" "io/ioutil" + log "github.com/Sirupsen/logrus" "github.com/samalba/dockerclient" ) @@ -80,24 +82,48 @@ func Wait(client dockerclient.Client, name string) (*dockerclient.ContainerInfo, errc := make(chan error, 1) infoc := make(chan *dockerclient.ContainerInfo, 1) go func() { - - // blocks and waits for the container to finish - // by streaming the logs (to /dev/null). Ideally - // we could use the `wait` function instead - rc, err := client.ContainerLogs(name, LogOptsTail) - if err != nil { - errc <- err - return + // options to fetch the stdout and stderr logs + // by tailing the output. + logOptsTail := &dockerclient.LogOptions{ + Follow: true, + Stdout: true, + Stderr: true, } - io.Copy(ioutil.Discard, rc) - rc.Close() - info, err := client.InspectContainer(name) - if err != nil { - errc <- err - return + for attempts := 0; attempts < 5; attempts++ { + if attempts > 0 { + // When resuming the stream, only grab the last line when starting + // the tailing. + logOptsTail.Tail = 1 + } + + // blocks and waits for the container to finish + // by streaming the logs (to /dev/null). Ideally + // we could use the `wait` function instead + rc, err := client.ContainerLogs(name, logOptsTail) + if err != nil { + errc <- err + return + } + io.Copy(ioutil.Discard, rc) + rc.Close() + + info, err := client.InspectContainer(name) + if err != nil { + errc <- err + return + } + + if !info.State.Running { + // The container is no longer running, there should be no more logs to tail. + infoc <- info + return + } + + log.Debugf("Attempting to resume log tailing after %d attempts.\n", attempts) } - infoc <- info + + errc <- errors.New("Maximum number of attempts made while tailing logs.") }() select { From 8df397db9f507952e5f624180dc831768df20252 Mon Sep 17 00:00:00 2001 From: Mac Browning Date: Fri, 12 Feb 2016 13:22:13 -0500 Subject: [PATCH 2/3] mark incomplete builds as errors --- engine/engine.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/engine/engine.go b/engine/engine.go index 29552f34a..f3d0c0c22 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -329,6 +329,11 @@ func (e *engine) runJob(c context.Context, r *Task, updater *updater, client doc info, builderr := docker.Wait(client, name) switch { + case info.State.Running: + // A build unblocked before actually being completed. + log.Errorf("incomplete build: %s", name) + r.Job.ExitCode = 1 + r.Job.Status = model.StatusError case info.State.ExitCode == 128: r.Job.ExitCode = info.State.ExitCode r.Job.Status = model.StatusKilled From 7a49d0324b1ec5190a5091cdc7947ec42f7f6daa Mon Sep 17 00:00:00 2001 From: Mac Browning Date: Fri, 12 Feb 2016 13:54:16 -0500 Subject: [PATCH 3/3] use dockerclient.Wait to block for in-progress builds --- shared/docker/docker.go | 62 ++++++++--------------------------------- 1 file changed, 12 insertions(+), 50 deletions(-) diff --git a/shared/docker/docker.go b/shared/docker/docker.go index 7bc2ed075..e08ec95e7 100644 --- a/shared/docker/docker.go +++ b/shared/docker/docker.go @@ -2,8 +2,6 @@ package docker import ( "errors" - "io" - "io/ioutil" log "github.com/Sirupsen/logrus" "github.com/samalba/dockerclient" @@ -79,57 +77,21 @@ func Wait(client dockerclient.Client, name string) (*dockerclient.ContainerInfo, client.KillContainer(name, "9") }() - errc := make(chan error, 1) - infoc := make(chan *dockerclient.ContainerInfo, 1) - go func() { - // options to fetch the stdout and stderr logs - // by tailing the output. - logOptsTail := &dockerclient.LogOptions{ - Follow: true, - Stdout: true, - Stderr: true, + for attempts := 0; attempts < 5; attempts++ { + done := client.Wait(name) + <-done + + info, err := client.InspectContainer(name) + if err != nil { + return nil, err } - for attempts := 0; attempts < 5; attempts++ { - if attempts > 0 { - // When resuming the stream, only grab the last line when starting - // the tailing. - logOptsTail.Tail = 1 - } - - // blocks and waits for the container to finish - // by streaming the logs (to /dev/null). Ideally - // we could use the `wait` function instead - rc, err := client.ContainerLogs(name, logOptsTail) - if err != nil { - errc <- err - return - } - io.Copy(ioutil.Discard, rc) - rc.Close() - - info, err := client.InspectContainer(name) - if err != nil { - errc <- err - return - } - - if !info.State.Running { - // The container is no longer running, there should be no more logs to tail. - infoc <- info - return - } - - log.Debugf("Attempting to resume log tailing after %d attempts.\n", attempts) + if !info.State.Running { + return info, nil } - errc <- errors.New("Maximum number of attempts made while tailing logs.") - }() - - select { - case info := <-infoc: - return info, nil - case err := <-errc: - return nil, err + log.Debugf("attempting to resume waiting after %d attempts.\n", attempts) } + + return nil, errors.New("reached maximum wait attempts") }