Add rudimentary grace period support

This is important to make the backend runner able to support partial traces; the
default CommandContext setup always sigkills, and we need to sigterm things like
rmem to get them to dump a partial result.

Seems to work; could do with some testing, but how?, and refactoring, but how?.

This isn't yet supported in the machine node, mainly because that doesn't quite
support the service.Runner idiom yet.

See golang/go#22757.

Author: MattWindsor91

internal/app/backend/backend.go

@@ -13,6 +13,9 @@ import (
 	"io"
 	"io/ioutil"
 	"os"
+	"time"
+
+	"github.com/MattWindsor91/c4t/internal/act"
 
 	"github.com/MattWindsor91/c4t/internal/helper/errhelp"
 
@@ -60,6 +63,12 @@ const (
 	flagDryRun                = "dry-run"
 	flagDryRunShort           = "d"
 	usageDryRun               = "if true, print any external commands run instead of running them"
+	flagTimeout               = "timeout"
+	flagTimeoutShort          = "t"
+	usageTimeout              = "`DURATION` to wait before trying to stop the backend"
+	flagGrace                 = "grace"
+	flagGraceShort            = "g"
+	usageGrace                = "`DURATION` to wait between sigterm and sigkill when timing out"
 )
 
 // App is the c4-backend app.
@@ -83,6 +92,8 @@ func flags() []c.Flag {
 		&c.GenericFlag{Name: flagBackendIDGlob, Aliases: []string{flagBackendIDGlobShort}, Usage: usageBackendIDGlob, Value: &id.ID{}},
 		&c.GenericFlag{Name: flagBackendStyleGlob, Aliases: []string{flagBackendStyleGlobShort}, Usage: usageBackendStyleGlob, Value: &id.ID{}},
 		&c.BoolFlag{Name: flagDryRun, Aliases: []string{flagDryRunShort}, Usage: usageDryRun},
+		&c.DurationFlag{Name: flagTimeout, Aliases: []string{flagTimeoutShort}, Usage: usageTimeout},
+		&c.DurationFlag{Name: flagGrace, Aliases: []string{flagGraceShort}, Usage: usageGrace},
 	}
 	return append(ownFlags, stdflag.ActRunnerCliFlags()...)
 }
@@ -93,50 +104,60 @@ func run(ctx *c.Context, outw io.Writer, errw io.Writer) error {
 		return fmt.Errorf("while getting config: %w", err)
 	}
 	c4f := stdflag.ActRunnerFromCli(ctx, errw)
-	cri := criteriaFromCli(ctx)
-	fn, err := inputNameFromCli(ctx)
+
+	td, err := ioutil.TempDir("", "c4t-backend")
 	if err != nil {
 		return err
 	}
-	arch := idFromCli(ctx, flagArchID)
 
-	bspec, b, err := getBackend(cfg, cri)
+	fn, err := inputNameFromCli(ctx)
 	if err != nil {
 		return err
 	}
 
-	in, err := backend.InputFromFile(ctx.Context, fn, c4f)
+	bspec, b, err := getBackend(cfg, criteriaFromCli(ctx))
 	if err != nil {
 		return err
 	}
 
-	td, err := ioutil.TempDir("", "c4t-backend")
+	j, err := jobFromCli(ctx, fn, c4f, bspec, td)
 	if err != nil {
 		return err
 	}
+	perr := runParseAndDump(ctx, outw, b, j, makeRunner(ctx, errw))
+	derr := os.RemoveAll(td)
+	return errhelp.FirstError(perr, derr)
+}
+
+func jobFromCli(ctx *c.Context, fn string, c4f *act.Runner, bspec *backend.Spec, td string) (backend.LiftJob, error) {
+	in, err := backend.InputFromFile(ctx.Context, fn, c4f)
+	if err != nil {
+		return backend.LiftJob{}, err
+	}
+
 	j := backend.LiftJob{
 		Backend: bspec,
-		Arch:    arch,
+		Arch:    idFromCli(ctx, flagArchID),
 		In:      in,
 		Out:     backend.LiftOutput{Dir: td, Target: backend.ToStandalone},
 	}
-	xr := makeRunner(ctx, errw)
-	perr := runParseAndDump(ctx, outw, b, j, xr)
-	derr := os.RemoveAll(td)
-	return errhelp.FirstError(perr, derr)
+	return j, nil
 }
 
 func makeRunner(ctx *c.Context, errw io.Writer) service.Runner {
 	// TODO(@MattWindsor91): the backend logic isn't very resilient against having external commands not run.
 	if ctx.Bool(flagDryRun) {
 		return srvrun.DryRunner{Writer: errw}
 	}
-	return srvrun.NewExecRunner(srvrun.StderrTo(errw))
+	// TODO(@MattWindsor91): use grace in the rest of c4t
+	return srvrun.NewExecRunner(srvrun.StderrTo(errw), srvrun.WithGrace(ctx.Duration(flagGrace)))
 }
 
 func runParseAndDump(ctx *c.Context, outw io.Writer, b backend2.Backend, j backend.LiftJob, xr service.Runner) error {
 	var o obs.Obs
-	if err := runAndParse(ctx.Context, b, j, &o, xr); err != nil {
+
+	to := ctx.Duration(flagTimeout)
+	if err := runAndParse(ctx.Context, to, b, j, &o, xr); err != nil {
 		return err
 	}
 
@@ -145,9 +166,10 @@ func runParseAndDump(ctx *c.Context, outw io.Writer, b backend2.Backend, j backe
 	return e.Encode(o)
 }
 
-func runAndParse(ctx context.Context, b backend2.Backend, j backend.LiftJob, o *obs.Obs, xr service.Runner) error {
-	// TODO(@MattWindsor91): deduplicate with runAndParseBin?.
-	r, err := b.Lift(ctx, j, xr)
+func runAndParse(ctx context.Context, to time.Duration, b backend2.Backend, j backend.LiftJob, o *obs.Obs, xr service.Runner) error {
+	// TODO(@MattWindsor91): clean this function up, eg making a separate struct...
+
+	r, err := liftWithTimeout(ctx, to, b, j, xr)
 	if err != nil {
 		return err
 	}
@@ -164,6 +186,17 @@ func runAndParse(ctx context.Context, b backend2.Backend, j backend.LiftJob, o *
 	return nil
 }
 
+func liftWithTimeout(ctx context.Context, to time.Duration, b backend2.Backend, j backend.LiftJob, xr service.Runner) (recipe.Recipe, error) {
+	cf := func() {}
+	if to != 0 {
+		ctx, cf = context.WithTimeout(ctx, to)
+	}
+	defer cf()
+
+	// TODO(@MattWindsor91): deduplicate with runAndParseBin?.
+	return b.Lift(ctx, j, xr)
+}
+
 func parseFile(ctx context.Context, b backend2.Backend, j backend.LiftJob, o *obs.Obs, fname string) error {
 	f, err := os.Open(fname)
 	if err != nil {

internal/helper/srvrun/dry.go

@@ -8,6 +8,7 @@ package srvrun
 import (
 	"context"
 	"io"
+	"time"
 
 	"github.com/MattWindsor91/c4t/internal/model/service"
 )
@@ -27,6 +28,11 @@ func (d DryRunner) WithStderr(_ io.Writer) service.Runner {
 	return d
 }
 
+// WithGrace just returns the same runner, ignoring the override.
+func (d DryRunner) WithGrace(_ time.Duration) service.Runner {
+	return d
+}
+
 // Run dumps r to the dry runner's writer.
 func (d DryRunner) Run(_ context.Context, r service.RunInfo) error {
 	_, err := io.WriteString(d, r.String()+"\n")

internal/helper/srvrun/exec.go

@@ -8,15 +8,19 @@ package srvrun
 import (
 	"context"
 	"io"
+	"os"
 	"os/exec"
+	"runtime"
+	"time"
 
 	"github.com/MattWindsor91/c4t/internal/model/service"
 )
 
 // ExecRunner represents runs of services using exec.
 type ExecRunner struct {
-	errw io.Writer
-	outw io.Writer
+	errw  io.Writer
+	outw  io.Writer
+	grace time.Duration
 }
 
 // NewExecRunner constructs an exec-based runner using the options in os.
@@ -28,22 +32,68 @@ func NewExecRunner(os ...ExecOption) *ExecRunner {
 
 // WithStderr returns a new ExecRunner with standard error rerouted to w.
 func (e ExecRunner) WithStderr(w io.Writer) service.Runner {
-	return ExecRunner{outw: e.outw, errw: w}
+	StderrTo(w)(&e)
+	return e
+}
+
+// WithGrace returns a new ExecRunner with the timeout grace period set as d.
+func (e ExecRunner) WithGrace(d time.Duration) service.Runner {
+	WithGrace(d)(&e)
+	return e
 }
 
 // WithStdout returns a new ExecRunner with standard output rerouted to w.
 func (e ExecRunner) WithStdout(w io.Writer) service.Runner {
-	return ExecRunner{outw: w, errw: e.errw}
+	e.outw = w
+	return e
+}
+
+func (e ExecRunner) hasGrace() bool {
+	return runtime.GOOS != "windows" && 0 < e.grace
 }
 
 // Run runs the command specified by r using exec, on context ctx.
 func (e ExecRunner) Run(ctx context.Context, r service.RunInfo) error {
+	if e.hasGrace() {
+		return e.runWithGrace(ctx, r)
+	}
 	c := exec.CommandContext(ctx, r.Cmd, r.Args...)
 	c.Stderr = e.errw
 	c.Stdout = e.outw
 	return c.Run()
 }
 
+func (e ExecRunner) runWithGrace(ctx context.Context, r service.RunInfo) error {
+	cdone := make(chan struct{})
+	defer close(cdone)
+
+	c := exec.Command(r.Cmd, r.Args...)
+	c.Stderr = e.errw
+	c.Stdout = e.outw
+	if err := c.Start(); err != nil {
+		return err
+	}
+
+	// https://github.com/golang/go/issues/22757#issuecomment-345009730
+	go func() {
+		select {
+		case <-cdone:
+			return
+		case <-ctx.Done():
+		}
+
+		t := time.AfterFunc(e.grace, func() {
+			_ = c.Process.Kill()
+		})
+		_ = c.Process.Signal(os.Interrupt)
+
+		<-cdone
+		t.Stop()
+	}()
+
+	return c.Wait()
+}
+
 // ExecOption is the type of options used when constructing a ExecRunner.
 type ExecOption func(*ExecRunner)
 
@@ -62,3 +112,13 @@ func StderrTo(w io.Writer) ExecOption {
 		l.errw = w
 	}
 }
+
+// WithGrace sets a timeout grace period of d.
+//
+// If an ExecRunner's context closes and it has a timeout grace period, and the OS supports it, it will SIGTERM the
+// program, wait d, and then SIGKILL.
+func WithGrace(d time.Duration) ExecOption {
+	return func(l *ExecRunner) {
+		l.grace = d
+	}
+}

internal/model/service/mocks/Runner.go

@@ -9,6 +9,7 @@ import (
 	"context"
 	"io"
 	"strings"
+	"time"
 )
 
 // Runner is the interface of things that can run, or pretend to run, services.
@@ -19,6 +20,9 @@ type Runner interface {
 	// WithStderr should return a new runner with the standard error overridden to w.
 	WithStderr(w io.Writer) Runner
 
+	// WithGrace should return a new runner with the timeout grace period set to d.
+	WithGrace(d time.Duration) Runner
+
 	// Run runs r using context ctx.
 	Run(ctx context.Context, r RunInfo) error
 }