Implement finalizers

- to ensure that nix-cli processes get SIGTERM when morph receives SIGTERM or SIGINT
- to prepare for better cleanup of tmp-directories in morph in the future

morph.go

@@ -50,10 +50,10 @@ var (
 	asJson              bool
 	execute             = executeCmd(app.Command("exec", "Execute arbitrary commands on machines"))
 	executeCommand      []string
+	keepGCRoot          = app.Flag("keep-result", "Keep latest build in .gcroots to prevent it from being garbage collected").Default("False").Bool()
 
 	tempDir, tempDirErr  = ioutil.TempDir("", "morph-")
 	assetRoot, assetsErr = assets.Setup()
-	keepGCRoot           = app.Flag("keep-result", "Keep latest build in .gcroots to prevent it from being garbage collected").Default("False").Bool()
 )
 
 func deploymentArg(cmd *kingpin.CmdClause) {
@@ -231,6 +231,12 @@ func main() {
 		fmt.Fprintln(os.Stderr, "Deprecation: The --build-arg flag will be removed in a future release.")
 	}
 
+	defer utils.RunFinalizers()
+	utils.AddFinalizer(func() {
+		assets.Teardown(assetRoot)
+	})
+	utils.SignalHandler()
+
 	hosts, err := getHosts(deployment)
 	if err != nil {
 		handleError(clause, hosts, err)
@@ -260,8 +266,6 @@ func main() {
 	if err != nil {
 		handleError(clause, hosts, err)
 	}
-
-	assets.Teardown(assetRoot)
 }
 
 func handleError(cmd string, hosts []nix.Host, err error) {

nix/nix.go

@@ -8,6 +8,7 @@ import (
 	"github.com/dbcdk/morph/healthchecks"
 	"github.com/dbcdk/morph/secrets"
 	"github.com/dbcdk/morph/ssh"
+	"github.com/dbcdk/morph/utils"
 	"io/ioutil"
 	"os"
 	"os/exec"
@@ -128,6 +129,11 @@ func (ctx *NixContext) GetMachines(deploymentPath string) (hosts []Host, err err
 	cmd.Stdout = &stdout
 	cmd.Stderr = os.Stderr
 
+	utils.AddFinalizer(func() {
+		if (cmd.ProcessState == nil || !cmd.ProcessState.Exited()) && cmd.Process != nil {
+			_ = cmd.Process.Signal(syscall.SIGTERM)
+		}
+	})
 	err = cmd.Run()
 	if err != nil {
 		errorMessage := fmt.Sprintf(
@@ -196,6 +202,11 @@ func (ctx *NixContext) BuildMachines(deploymentPath string, hosts []Host, nixArg
 	// show process output on attached stdout/stderr
 	cmd.Stdout = os.Stderr
 	cmd.Stderr = os.Stderr
+	utils.AddFinalizer(func() {
+		if (cmd.ProcessState == nil || !cmd.ProcessState.Exited()) && cmd.Process != nil {
+			_ = cmd.Process.Signal(syscall.SIGTERM)
+		}
+	})
 	err = cmd.Run()
 
 	if err != nil {

utils/finalizer.go

@@ -0,0 +1,52 @@
+package utils
+
+import (
+	"fmt"
+	"os"
+	"os/signal"
+	"syscall"
+)
+
+type finalizer struct {
+	function FinalizerFunc
+	executed bool
+}
+type FinalizerFunc func()
+
+var finalizers []*finalizer
+
+/*
+	Finalizers run sequentially at morph shutdown - both at clean shutdown and on errors.
+	Finalizers should be quick, simple and they need to ignore errors and don't panic.
+	Each finalizer will only run once and will _never_ be re-invoked.
+*/
+func (f *finalizer) Run() {
+	if !f.executed {
+		f.executed = true
+		f.function()
+	}
+}
+
+func RunFinalizers() {
+	for _, f := range finalizers {
+		f.Run()
+	}
+}
+
+func AddFinalizer(f FinalizerFunc) {
+	finalizers = append(finalizers, &finalizer{
+		function: f,
+		executed: false,
+	})
+}
+
+func SignalHandler() {
+	sigs := make(chan os.Signal, 1)
+	signal.Notify(sigs, syscall.SIGINT, syscall.SIGTERM)
+	go func() {
+		sig := <-sigs
+		fmt.Fprintf(os.Stderr, "Received signal: %s\n", sig.String())
+		RunFinalizers()
+		os.Exit(130) // reserved exit code for "Interrupted"
+	}()
+}