From 10508259e856abfa90db22ae93eceb62ac1ca1ec Mon Sep 17 00:00:00 2001
From: Finn <finn@entanglement.garden>
Date: Wed, 30 Oct 2019 22:12:58 -0700
Subject: [PATCH 01/10] Initial metadata server proof of concept

---
 cmd/rhyzome/main.go |  5 ++-
 metadata/server.go  | 82 +++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 86 insertions(+), 1 deletion(-)
 create mode 100644 metadata/server.go

diff --git a/cmd/rhyzome/main.go b/cmd/rhyzome/main.go
index 66d5b4d..9e51c8f 100644
--- a/cmd/rhyzome/main.go
+++ b/cmd/rhyzome/main.go
@@ -1,11 +1,14 @@
 package main
 
 import (
-	"git.callpipe.com/entanglement.garden/rhyzome/rest"
 	"git.callpipe.com/entanglement.garden/rhyzome/config"
+	"git.callpipe.com/entanglement.garden/rhyzome/metadata"
+	"git.callpipe.com/entanglement.garden/rhyzome/rest"
 )
 
 func main() {
 	config.Load()
+	// TODO: Some sort of shutdown signal
+	go metadata.ListenAndServe()
 	rest.ListenAndServe()
 }
diff --git a/metadata/server.go b/metadata/server.go
new file mode 100644
index 0000000..f8668a6
--- /dev/null
+++ b/metadata/server.go
@@ -0,0 +1,82 @@
+package metadata
+
+import (
+	"encoding/json"
+	"log"
+	"net"
+	"net/http"
+	"strings"
+
+	"github.com/gorilla/mux"
+	libvirt "github.com/libvirt/libvirt-go"
+	libvirtxml "github.com/libvirt/libvirt-go-xml"
+	"github.com/mostlygeek/arp"
+)
+
+func ListenAndServe() {
+  r := mux.NewRouter()
+	r.HandleFunc("/", index)
+	log.Fatal(http.ListenAndServe(":8081", r))
+}
+
+func index(w http.ResponseWriter, r *http.Request) {
+	// TODO: Can i wrap this in middleware?
+	conn, err := libvirt.NewConnect("qemu:///system")
+	if err != nil {
+		http.Error(w, err.Error(), 500)
+		return
+	}
+	defer conn.Close()
+
+	ip, _, err := net.SplitHostPort(r.RemoteAddr)
+	if err != nil {
+		http.Error(w, err.Error(), 404)
+		return
+	}
+
+	mac := strings.Trim(arp.Search(ip), "'")
+
+	log.Printf("%s Looking up metadata for %s", r.RemoteAddr, mac)
+
+	// iface, err := conn.LookupInterfaceByMACString(mac)
+	domainIDs, err := conn.ListDomains()
+	if err != nil {
+		http.Error(w, err.Error(), 404)
+		return
+	}
+
+	for _, domainID := range domainIDs {
+		dom, err := conn.LookupDomainById(domainID)
+		if err != nil {
+			log.Printf("Error looking up domain %d: %s", dom, err.Error())
+			continue
+		}
+		if dom == nil {
+			log.Printf("LookupDomainById(%d) returned nil", domainID)
+			continue
+		}
+
+		xmldoc, err := dom.GetXMLDesc(0)
+		if err != nil {
+			http.Error(w, err.Error(), 500)
+			return
+		}
+
+		domain := libvirtxml.Domain{}
+		if err := domain.Unmarshal(xmldoc); err != nil {
+			continue
+		}
+		// domains = append(domains, domcfg)
+
+		for _, iface := range domain.Devices.Interfaces {
+			if iface.MAC.Address == mac {
+				json.NewEncoder(w).Encode(domain)
+				log.Printf("%s found!", mac)
+				return
+			}
+
+			log.Printf("%s != %s", iface.MAC.Address, mac)
+		}
+	}
+	http.Error(w, "not found", 404)
+}
-- 
GitLab


From 159518c788cc6ee4a90e49f912b7d469a3e7f602 Mon Sep 17 00:00:00 2001
From: Finn <finn@entanglement.garden>
Date: Mon, 11 Nov 2019 16:25:29 -0800
Subject: [PATCH 02/10] WIP

---
 instances/instances.go |   2 -
 metadata/server.go     | 122 +++++++++++++++++++++++++----------------
 rest/router.go         |   3 +
 utils/middleware.go    |  34 ++++++++++++
 4 files changed, 111 insertions(+), 50 deletions(-)
 create mode 100644 utils/middleware.go

diff --git a/instances/instances.go b/instances/instances.go
index 3b66fa8..d275a1b 100644
--- a/instances/instances.go
+++ b/instances/instances.go
@@ -11,8 +11,6 @@ import (
 	"git.callpipe.com/entanglement.garden/rhyzome/jobs"
 )
 
-type contextKey int
-
 type Instance struct {
 	Name       string
 	UUID       string
diff --git a/metadata/server.go b/metadata/server.go
index f8668a6..c2c3149 100644
--- a/metadata/server.go
+++ b/metadata/server.go
@@ -1,6 +1,7 @@
 package metadata
 
 import (
+	"context"
 	"encoding/json"
 	"log"
 	"net"
@@ -11,72 +12,97 @@ import (
 	libvirt "github.com/libvirt/libvirt-go"
 	libvirtxml "github.com/libvirt/libvirt-go-xml"
 	"github.com/mostlygeek/arp"
+
+	"git.callpipe.com/entanglement.garden/rhyzome/utils"
+)
+
+const (
+	instanceContextKey utils.ContextKey = 0
+	libvirtConnecKey   utils.ContextKey = 1
 )
 
 func ListenAndServe() {
-  r := mux.NewRouter()
+	r := mux.NewRouter()
+	r.Use(utils.LoggingMiddleware)
+	r.Use(lookupRequester)
 	r.HandleFunc("/", index)
 	log.Fatal(http.ListenAndServe(":8081", r))
 }
 
-func index(w http.ResponseWriter, r *http.Request) {
-	// TODO: Can i wrap this in middleware?
-	conn, err := libvirt.NewConnect("qemu:///system")
-	if err != nil {
-		http.Error(w, err.Error(), 500)
-		return
-	}
-	defer conn.Close()
-
-	ip, _, err := net.SplitHostPort(r.RemoteAddr)
-	if err != nil {
-		http.Error(w, err.Error(), 404)
-		return
-	}
-
-	mac := strings.Trim(arp.Search(ip), "'")
-
-	log.Printf("%s Looking up metadata for %s", r.RemoteAddr, mac)
-
-	// iface, err := conn.LookupInterfaceByMACString(mac)
-	domainIDs, err := conn.ListDomains()
-	if err != nil {
-		http.Error(w, err.Error(), 404)
-		return
-	}
-
-	for _, domainID := range domainIDs {
-		dom, err := conn.LookupDomainById(domainID)
+func lookupRequester(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		conn, err := libvirt.NewConnect("qemu:///system")
 		if err != nil {
-			log.Printf("Error looking up domain %d: %s", dom, err.Error())
-			continue
-		}
-		if dom == nil {
-			log.Printf("LookupDomainById(%d) returned nil", domainID)
-			continue
+			http.Error(w, err.Error(), 500)
+			return
 		}
+		defer conn.Close()
 
-		xmldoc, err := dom.GetXMLDesc(0)
+		ip, _, err := net.SplitHostPort(r.RemoteAddr)
 		if err != nil {
-			http.Error(w, err.Error(), 500)
+			http.Error(w, err.Error(), 404)
 			return
 		}
 
-		domain := libvirtxml.Domain{}
-		if err := domain.Unmarshal(xmldoc); err != nil {
-			continue
+		mac := strings.Trim(arp.Search(ip), "'")
+
+		log.Printf("[%s] Looking up metadata for %s", ip, mac)
+
+		domainIDs, err := conn.ListDomains()
+		if err != nil {
+			http.Error(w, err.Error(), 404)
+			return
 		}
-		// domains = append(domains, domcfg)
 
-		for _, iface := range domain.Devices.Interfaces {
-			if iface.MAC.Address == mac {
-				json.NewEncoder(w).Encode(domain)
-				log.Printf("%s found!", mac)
+		ctx := r.Context()
+		found := false
+
+		for _, domainID := range domainIDs {
+			dom, err := conn.LookupDomainById(domainID)
+			if err != nil {
+				continue
+			}
+			if dom == nil {
+				log.Printf("LookupDomainById(%d) returned nil", domainID)
+				continue
+			}
+
+			xmldoc, err := dom.GetXMLDesc(0)
+			if err != nil {
+				http.Error(w, err.Error(), 500)
 				return
 			}
 
-			log.Printf("%s != %s", iface.MAC.Address, mac)
+			domain := libvirtxml.Domain{}
+			if err := domain.Unmarshal(xmldoc); err != nil {
+				continue
+			}
+			// domains = append(domains, domcfg)
+
+			for _, iface := range domain.Devices.Interfaces {
+				if iface.MAC.Address == mac {
+					ctx = context.WithValue(ctx, libvirtConnecKey, conn)
+					ctx = context.WithValue(ctx, instanceContextKey, domain)
+					found = true
+					break
+				}
+			}
+			if found {
+				break
+			}
 		}
-	}
-	http.Error(w, "not found", 404)
+
+		if !found {
+			http.Error(w, "unknown client", 401)
+			return
+		}
+
+		next.ServeHTTP(w, r.WithContext(ctx))
+	})
+}
+
+func index(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	domain := ctx.Value(instanceContextKey).(libvirt.Domain)
+	json.NewEncoder(w).Encode(domain)
 }
diff --git a/rest/router.go b/rest/router.go
index 450625e..3fb3ca3 100644
--- a/rest/router.go
+++ b/rest/router.go
@@ -6,10 +6,13 @@ import (
 	"net/http"
 
 	"github.com/gorilla/mux"
+
+	"git.callpipe.com/entanglement.garden/rhyzome/utils"
 )
 
 func ListenAndServe() {
 	r := mux.NewRouter()
+	r.Use(utils.LoggingMiddleware)
 	r.HandleFunc("/", renderRoot)
 	r.HandleFunc("/api/v1alpha1/instances", CreateInstance).Methods("POST")
 	r.HandleFunc("/api/v1alpha1/instances", ListInstances).Methods("GET")
diff --git a/utils/middleware.go b/utils/middleware.go
new file mode 100644
index 0000000..207fe23
--- /dev/null
+++ b/utils/middleware.go
@@ -0,0 +1,34 @@
+package utils
+
+import (
+	"log"
+	"net/http"
+)
+
+type ContextKey int
+
+type wrapper struct {
+	http.ResponseWriter
+	written int
+	status  int
+}
+
+func (w *wrapper) WriteHeader(code int) {
+	w.status = code
+	w.ResponseWriter.WriteHeader(code)
+}
+
+func (w *wrapper) Write(b []byte) (int, error) {
+	n, err := w.ResponseWriter.Write(b)
+	w.written += n
+	return n, err
+}
+
+// LoggingMiddleware is a middleware function that logs requests as they come in
+func LoggingMiddleware(next http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		res := &wrapper{w, 0, 0}
+		next.ServeHTTP(res, r)
+		log.Printf("%s %s %s %d %d", r.RemoteAddr, r.Method, r.RequestURI, res.status, res.written)
+	})
+}
-- 
GitLab


From 215ae8a790d5d476d516e872cc3c5eb49a22a93a Mon Sep 17 00:00:00 2001
From: Finn <finn@entanglement.garden>
Date: Mon, 27 Jan 2020 08:55:04 -0800
Subject: [PATCH 03/10] Configurable bind for metadata server

---
 config/config.go   | 14 ++++++++------
 metadata/server.go |  3 ++-
 2 files changed, 10 insertions(+), 7 deletions(-)

diff --git a/config/config.go b/config/config.go
index b78071e..fd60a31 100644
--- a/config/config.go
+++ b/config/config.go
@@ -12,12 +12,13 @@ var ConfigFiles = []string{"/etc/rhyzome.conf", "rhyzome.conf"}
 // Config describes all configurable keys
 type Config struct {
 	Bind            string // Bind is the address (and port) to bind the REST server to
+	BridgeInterface string // BridgeInterface is the bridge that all network interfaces are added to
 	DiskStoragePool string // DiskStoragePool is the name of the storage pool to use
-	ImageOwner      string // ImageOwner is the UID that should own volume images
+	ImageDir        string // ImageDir is the path to the local image pool
 	ImageGroup      string // ImageGroup is the GID that should own volume images
-	BridgeInterface string // BridgeInterface is the bridge that all network interfaces are added to
 	ImageHost       string // ImageHost is the base URL for the disk image server
-	ImageDir        string // ImageDir is the path to the local image pool
+	ImageOwner      string // ImageOwner is the UID that should own volume images
+	MetadataBind    string // MetadataBind is the port (and optionally IP) to bind the metadata server to
 }
 
 var (
@@ -27,12 +28,13 @@ var (
 	// Defaults are the default values for each config option, set in the Load() function
 	Defaults = Config{
 		Bind:            ":8080",
+		BridgeInterface: "brlan",
 		DiskStoragePool: "default",
-		ImageOwner:      "64055",
+		ImageDir:        "/var/lib/libvirt/images",
 		ImageGroup:      "64055",
-		BridgeInterface: "brlan",
 		ImageHost:       "http://image-host.fruit-0.entanglement.garden",
-		ImageDir:        "/var/lib/libvirt/images",
+		ImageOwner:      "64055",
+		MetadataBind:    ":8081",
 	}
 )
 
diff --git a/metadata/server.go b/metadata/server.go
index c2c3149..d31c285 100644
--- a/metadata/server.go
+++ b/metadata/server.go
@@ -13,6 +13,7 @@ import (
 	libvirtxml "github.com/libvirt/libvirt-go-xml"
 	"github.com/mostlygeek/arp"
 
+	"git.callpipe.com/entanglement.garden/rhyzome/config"
 	"git.callpipe.com/entanglement.garden/rhyzome/utils"
 )
 
@@ -26,7 +27,7 @@ func ListenAndServe() {
 	r.Use(utils.LoggingMiddleware)
 	r.Use(lookupRequester)
 	r.HandleFunc("/", index)
-	log.Fatal(http.ListenAndServe(":8081", r))
+	log.Fatal(http.ListenAndServe(config.C.MetadataBind, r))
 }
 
 func lookupRequester(next http.Handler) http.Handler {
-- 
GitLab


From 93866cb52a8b03aa413218a13f8faa97caece5e8 Mon Sep 17 00:00:00 2001
From: Finn <finn@entanglement.garden>
Date: Mon, 27 Jan 2020 08:56:35 -0800
Subject: [PATCH 04/10] Default bind the metadata server to 127.0.0.1

---
 config/config.go   | 2 +-
 metadata/server.go | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/config/config.go b/config/config.go
index fd60a31..6e1e2b0 100644
--- a/config/config.go
+++ b/config/config.go
@@ -34,7 +34,7 @@ var (
 		ImageGroup:      "64055",
 		ImageHost:       "http://image-host.fruit-0.entanglement.garden",
 		ImageOwner:      "64055",
-		MetadataBind:    ":8081",
+		MetadataBind:    "127.0.0.1:8081",
 	}
 )
 
diff --git a/metadata/server.go b/metadata/server.go
index d31c285..d1d5b88 100644
--- a/metadata/server.go
+++ b/metadata/server.go
@@ -104,6 +104,6 @@ func lookupRequester(next http.Handler) http.Handler {
 
 func index(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
-	domain := ctx.Value(instanceContextKey).(libvirt.Domain)
+	domain := ctx.Value(instanceContextKey).(libvirtxml.Domain)
 	json.NewEncoder(w).Encode(domain)
 }
-- 
GitLab


From 11207fc8d63c17dd8f0a87bf771725b61dc6917e Mon Sep 17 00:00:00 2001
From: Finn Herzfeld <finn@entanglement.garden>
Date: Wed, 5 Feb 2020 10:55:34 -0800
Subject: [PATCH 05/10] Initial attempt at vault role injection

doesn't work yet
---
 cmd/rhyzome/main.go      |   3 ++
 config/config.go         |  35 ++++++++++++-
 metadata/instanceInfo.go |  33 +++++++++++++
 metadata/server.go       |  29 ++++++-----
 utils/middleware.go      |   6 +++
 vault/approle.go         |  84 +++++++++++++++++++++++++++++++
 vault/vault.go           | 103 +++++++++++++++++++++++++++++++++++++++
 7 files changed, 277 insertions(+), 16 deletions(-)
 create mode 100644 metadata/instanceInfo.go
 create mode 100644 vault/approle.go
 create mode 100644 vault/vault.go

diff --git a/cmd/rhyzome/main.go b/cmd/rhyzome/main.go
index 9e51c8f..a1d2bc2 100644
--- a/cmd/rhyzome/main.go
+++ b/cmd/rhyzome/main.go
@@ -4,10 +4,13 @@ import (
 	"git.callpipe.com/entanglement.garden/rhyzome/config"
 	"git.callpipe.com/entanglement.garden/rhyzome/metadata"
 	"git.callpipe.com/entanglement.garden/rhyzome/rest"
+	"git.callpipe.com/entanglement.garden/rhyzome/vault"
 )
 
 func main() {
 	config.Load()
+	vault.Load()
+
 	// TODO: Some sort of shutdown signal
 	go metadata.ListenAndServe()
 	rest.ListenAndServe()
diff --git a/config/config.go b/config/config.go
index fd60a31..0067018 100644
--- a/config/config.go
+++ b/config/config.go
@@ -5,6 +5,9 @@ import (
 	"io/ioutil"
 	"log"
 	"os"
+
+	"github.com/hashicorp/vault/api"
+	libvirt "github.com/libvirt/libvirt-go"
 )
 
 var ConfigFiles = []string{"/etc/rhyzome.conf", "rhyzome.conf"}
@@ -14,11 +17,23 @@ type Config struct {
 	Bind            string // Bind is the address (and port) to bind the REST server to
 	BridgeInterface string // BridgeInterface is the bridge that all network interfaces are added to
 	DiskStoragePool string // DiskStoragePool is the name of the storage pool to use
+	Hostname        string // Hostname is the domain that guests will be created under
 	ImageDir        string // ImageDir is the path to the local image pool
 	ImageGroup      string // ImageGroup is the GID that should own volume images
 	ImageHost       string // ImageHost is the base URL for the disk image server
 	ImageOwner      string // ImageOwner is the UID that should own volume images
 	MetadataBind    string // MetadataBind is the port (and optionally IP) to bind the metadata server to
+	Vault           VaultConfig
+}
+
+type VaultConfig struct {
+	TLSConfig             api.TLSConfig // TLSConfig is the TLS configuration to use, if any
+	Address               string
+	InjectionCommand      string
+	CommandPollMS         int
+	CommandTimeoutSeconds libvirt.DomainQemuAgentCommandTimeout
+	RoleIDFilePath        string
+	SecretIDFilePath      string
 }
 
 var (
@@ -34,7 +49,13 @@ var (
 		ImageGroup:      "64055",
 		ImageHost:       "http://image-host.fruit-0.entanglement.garden",
 		ImageOwner:      "64055",
-		MetadataBind:    ":8081",
+		MetadataBind:    "127.0.0.1:8081",
+		Vault: VaultConfig{
+			TLSConfig:             api.TLSConfig{},
+			InjectionCommand:      "/opt/entanglement/vault-inject.sh",
+			CommandPollMS:         250,
+			CommandTimeoutSeconds: 5,
+		},
 	}
 )
 
@@ -59,5 +80,17 @@ func Load() {
 
 		json.Unmarshal(byteValue, &C)
 		log.Println("Successfully read config from", filename)
+		log.Printf("%+v", C)
+	}
+
+	if C.Hostname == "" {
+		log.Println("Hostname not specified in config, setting from system")
+		hostname, err := os.Hostname()
+		if err != nil {
+			log.Println("Error getting hostname from system", err)
+		} else {
+			C.Hostname = hostname
+			log.Println("Set hostname to", C.Hostname)
+		}
 	}
 }
diff --git a/metadata/instanceInfo.go b/metadata/instanceInfo.go
new file mode 100644
index 0000000..dce44f5
--- /dev/null
+++ b/metadata/instanceInfo.go
@@ -0,0 +1,33 @@
+package metadata
+
+import (
+	"net/http"
+
+	libvirt "github.com/libvirt/libvirt-go"
+
+	"git.callpipe.com/entanglement.garden/rhyzome/utils"
+)
+
+func guestInfo(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	domain := ctx.Value(utils.InstanceDomainContextKey).(*libvirt.Domain)
+	info, err := domain.QemuAgentCommand("{\"execute\":\"guest-info\"}", 1, 0)
+	if err != nil {
+		http.Error(w, err.Error(), 500)
+		return
+	}
+
+	w.Write([]byte(info))
+}
+
+func guestGetOSInfo(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	domain := ctx.Value(utils.InstanceDomainContextKey).(*libvirt.Domain)
+	info, err := domain.QemuAgentCommand("{\"execute\":\"guest-get-osinfo\"}", 1, 0)
+	if err != nil {
+		http.Error(w, err.Error(), 500)
+		return
+	}
+
+	w.Write([]byte(info))
+}
diff --git a/metadata/server.go b/metadata/server.go
index d31c285..f3913a6 100644
--- a/metadata/server.go
+++ b/metadata/server.go
@@ -15,11 +15,7 @@ import (
 
 	"git.callpipe.com/entanglement.garden/rhyzome/config"
 	"git.callpipe.com/entanglement.garden/rhyzome/utils"
-)
-
-const (
-	instanceContextKey utils.ContextKey = 0
-	libvirtConnecKey   utils.ContextKey = 1
+	"git.callpipe.com/entanglement.garden/rhyzome/vault"
 )
 
 func ListenAndServe() {
@@ -27,6 +23,9 @@ func ListenAndServe() {
 	r.Use(utils.LoggingMiddleware)
 	r.Use(lookupRequester)
 	r.HandleFunc("/", index)
+	r.HandleFunc("/guest-info", guestInfo)
+	r.HandleFunc("/guest-get-osinfo", guestGetOSInfo)
+	r.HandleFunc("/vault/inject-app-role", vault.InjectAppRole)
 	log.Fatal(http.ListenAndServe(config.C.MetadataBind, r))
 }
 
@@ -59,31 +58,31 @@ func lookupRequester(next http.Handler) http.Handler {
 		found := false
 
 		for _, domainID := range domainIDs {
-			dom, err := conn.LookupDomainById(domainID)
+			domain, err := conn.LookupDomainById(domainID)
 			if err != nil {
 				continue
 			}
-			if dom == nil {
+			if domain == nil {
 				log.Printf("LookupDomainById(%d) returned nil", domainID)
 				continue
 			}
 
-			xmldoc, err := dom.GetXMLDesc(0)
+			xmldoc, err := domain.GetXMLDesc(0)
 			if err != nil {
 				http.Error(w, err.Error(), 500)
 				return
 			}
 
-			domain := libvirtxml.Domain{}
-			if err := domain.Unmarshal(xmldoc); err != nil {
+			domainXML := libvirtxml.Domain{}
+			if err := domainXML.Unmarshal(xmldoc); err != nil {
 				continue
 			}
-			// domains = append(domains, domcfg)
 
-			for _, iface := range domain.Devices.Interfaces {
+			for _, iface := range domainXML.Devices.Interfaces {
 				if iface.MAC.Address == mac {
-					ctx = context.WithValue(ctx, libvirtConnecKey, conn)
-					ctx = context.WithValue(ctx, instanceContextKey, domain)
+					ctx = context.WithValue(ctx, utils.LibvirtConnecKey, conn)
+					ctx = context.WithValue(ctx, utils.InstanceDomainContextKey, domain)
+					ctx = context.WithValue(ctx, utils.InstanceDomainXMLContextKey, domainXML)
 					found = true
 					break
 				}
@@ -104,6 +103,6 @@ func lookupRequester(next http.Handler) http.Handler {
 
 func index(w http.ResponseWriter, r *http.Request) {
 	ctx := r.Context()
-	domain := ctx.Value(instanceContextKey).(libvirt.Domain)
+	domain := ctx.Value(utils.InstanceDomainXMLContextKey).(libvirtxml.Domain)
 	json.NewEncoder(w).Encode(domain)
 }
diff --git a/utils/middleware.go b/utils/middleware.go
index 207fe23..72e6353 100644
--- a/utils/middleware.go
+++ b/utils/middleware.go
@@ -13,6 +13,12 @@ type wrapper struct {
 	status  int
 }
 
+const (
+	InstanceDomainContextKey    ContextKey = 0
+	InstanceDomainXMLContextKey ContextKey = 1
+	LibvirtConnecKey            ContextKey = 2
+)
+
 func (w *wrapper) WriteHeader(code int) {
 	w.status = code
 	w.ResponseWriter.WriteHeader(code)
diff --git a/vault/approle.go b/vault/approle.go
new file mode 100644
index 0000000..008ae13
--- /dev/null
+++ b/vault/approle.go
@@ -0,0 +1,84 @@
+package vault
+
+import (
+	"encoding/json"
+	"fmt"
+	"log"
+	"net/http"
+	"time"
+
+	"github.com/hashicorp/vault/api"
+	libvirt "github.com/libvirt/libvirt-go"
+
+	"git.callpipe.com/entanglement.garden/qapi"
+	"git.callpipe.com/entanglement.garden/rhyzome/config"
+	"git.callpipe.com/entanglement.garden/rhyzome/utils"
+)
+
+// InjectAppRole injects the credentials for a given AppRole
+func InjectAppRole(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	domain := ctx.Value(utils.InstanceDomainContextKey).(*libvirt.Domain)
+
+	domainName, err := domain.GetName()
+	if err != nil {
+		log.Println("Error getting domain name")
+		http.Error(w, err.Error(), 500)
+		return
+	}
+
+	secret, err := vaultClient.Logical().Write(fmt.Sprintf("auth/approle/role/%s.%s/secret-id", domainName, config.C.Hostname), make(map[string]interface{}))
+	if err != nil {
+		log.Println("Error getting approle for VM", err)
+		http.Error(w, err.Error(), 500)
+		return
+	}
+
+	err = WriteSecret(domain, secret)
+
+	statusResponse, err := GuestExec(domain, config.C.Vault.InjectionCommand, domainName)
+	if err != nil {
+		http.Error(w, err.Error(), 500)
+		return
+	}
+
+	err = json.NewEncoder(w).Encode(statusResponse)
+	if err != nil {
+		http.Error(w, err.Error(), 500)
+	}
+}
+
+func WriteSecret(domain *libvirt.Domain, secret *api.Secret) error {
+	return nil
+}
+
+// GuestExec runs a command on the specified domain as root (via the qemu guest agent), waits for it to complete, and returns the response
+// TODO: Should this be a member function of instances.Instance?
+func GuestExec(domain *libvirt.Domain, path string, args ...string) (response qapi.GuestExecStatus, err error) {
+	commandResponse, err := qapi.ExecuteGuestExec(domain, qapi.GuestExecArg{
+		Path:          path,
+		Arg:           args,
+		CaptureOutput: true,
+	})
+	if err != nil {
+		log.Println("Error executing guest-exec command: %s", err.Error())
+		return
+	}
+
+	for {
+		response, err = qapi.ExecuteGuestExecStatus(domain, qapi.GuestExecStatusArg{PID: commandResponse.PID})
+		if err != nil {
+			log.Printf("Error checking status of command running on guest: %s", err.Error())
+			return
+		}
+
+		if response.Exited {
+			log.Printf("Command %s exited: %+v", path, response)
+			break
+		}
+
+		time.Sleep(time.Duration(config.C.Vault.CommandPollMS) * time.Millisecond)
+	}
+
+	return
+}
diff --git a/vault/vault.go b/vault/vault.go
new file mode 100644
index 0000000..aa2b102
--- /dev/null
+++ b/vault/vault.go
@@ -0,0 +1,103 @@
+package vault
+
+import (
+	"github.com/hashicorp/vault/api"
+	"io/ioutil"
+	"log"
+	"strings"
+	"time"
+
+	"git.callpipe.com/entanglement.garden/rhyzome/config"
+)
+
+var (
+	vaultClient  *api.Client
+	tokenWatcher *api.LifetimeWatcher
+	secret       *api.Secret
+)
+
+// Load initializes the vault client with the configured vaults from the config file and enviornment variables
+func Load() error {
+	c := api.DefaultConfig()
+	c.Address = config.C.Vault.Address
+
+	err := c.ConfigureTLS(&config.C.Vault.TLSConfig)
+	if err != nil {
+		log.Println("Error configuring Vault TLS")
+		return err
+	}
+
+	err = c.ReadEnvironment()
+	if err != nil {
+		log.Println("Error configuring Vault from enviornment")
+		return err
+	}
+
+	vaultClient, err = api.NewClient(c)
+	if err != nil {
+		log.Println("Error configuring vault client")
+		return err
+	}
+
+	roleID, err := ioutil.ReadFile(config.C.Vault.RoleIDFilePath)
+	if err != nil {
+		log.Println("Error reading role id:", err)
+		return err
+	}
+
+	secretID, err := ioutil.ReadFile(config.C.Vault.SecretIDFilePath)
+	if err != nil {
+		log.Println("Error reading secret id:", err)
+		return err
+	}
+
+	log.Println("Logging into vault at", c.Address)
+	secret, err = vaultClient.Logical().Write("/auth/approle/login", map[string]interface{}{
+		"role_id":   strings.TrimSpace(string(roleID)),
+		"secret_id": strings.TrimSpace(string(secretID)),
+	})
+	if err != nil {
+		log.Println("Error logging into vault:", err)
+		return err
+	}
+
+	err = startTokenWatcher()
+
+	return err
+}
+
+func startTokenWatcher() (err error) {
+	log.Println("Starting token watcher")
+	if vaultClient == nil {
+		log.Println("vaultClient is nil! We will now crash")
+	}
+	tokenWatcher, err = vaultClient.NewLifetimeWatcher(&api.LifetimeWatcherInput{Secret: secret})
+	if err != nil {
+		return err
+	}
+	go tokenWatcher.Start()
+	go watchToken()
+	return nil
+}
+
+func watchToken() {
+	if tokenWatcher == nil {
+		log.Println("WatchToken gonna fail because tokenWatcher is nil")
+	}
+	for {
+		select {
+		case err := <-tokenWatcher.DoneCh():
+			if err != nil {
+				log.Println("Error in vault watcher:", err)
+				time.Sleep(5 * time.Second)
+				tokenWatcher.Stop()
+				startTokenWatcher()
+				return
+			}
+
+		case renewal := <-tokenWatcher.RenewCh():
+			log.Printf("Successfully renewed vault client token %+v", renewal.Secret.Auth)
+			vaultClient.SetToken(renewal.Secret.Auth.ClientToken)
+		}
+	}
+}
-- 
GitLab


From 2bc1c7e26e0a661d6c8ce1312d9e68cb9c9cc519 Mon Sep 17 00:00:00 2001
From: Finn Herzfeld <finn@entanglement.garden>
Date: Wed, 5 Feb 2020 14:08:50 -0800
Subject: [PATCH 06/10] Actually write the secret to the guest

does not work yet
---
 vault/approle.go | 91 ++++++++++++++++++++++++++++++++++++++++++++----
 1 file changed, 85 insertions(+), 6 deletions(-)

diff --git a/vault/approle.go b/vault/approle.go
index 008ae13..aaca71a 100644
--- a/vault/approle.go
+++ b/vault/approle.go
@@ -1,13 +1,13 @@
 package vault
 
 import (
+	"encoding/base64"
 	"encoding/json"
 	"fmt"
 	"log"
 	"net/http"
 	"time"
 
-	"github.com/hashicorp/vault/api"
 	libvirt "github.com/libvirt/libvirt-go"
 
 	"git.callpipe.com/entanglement.garden/qapi"
@@ -27,14 +27,71 @@ func InjectAppRole(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	secret, err := vaultClient.Logical().Write(fmt.Sprintf("auth/approle/role/%s.%s/secret-id", domainName, config.C.Hostname), make(map[string]interface{}))
+	// Create the approle
+	appRolePath := fmt.Sprintf("auth/approle/role/%s.%s", domainName, config.C.Hostname)
+	_, err = vaultClient.Logical().Write(appRolePath, map[string]interface{}{
+		"secret_id_ttl":      "1h",
+		"token_policies":     "hypervisors,pki-and-read-own-cert",
+		"secret_id_num_uses": "1",
+		"token_max_ttl":      "24h",
+	})
 	if err != nil {
-		log.Println("Error getting approle for VM", err)
+		log.Println("Error creating vault role", appRolePath, err)
 		http.Error(w, err.Error(), 500)
 		return
 	}
 
-	err = WriteSecret(domain, secret)
+	roleID, err := vaultClient.Logical().Read(appRolePath + "/role-id")
+	if err != nil {
+		log.Println("Error getting role-id for role", appRolePath)
+		http.Error(w, err.Error(), 500)
+		return
+	}
+
+	roleIDstring, ok := roleID.Data["role_id"].(string)
+	if !ok {
+		msg := "Cannot convert role_id as received from vault to string"
+		log.Println(msg)
+		http.Error(w, msg, 500)
+		return
+	}
+
+	err = WriteSecret(domain, "/ec/entanglement/auth/role-id", roleIDstring)
+	if err != nil {
+		log.Println("Error writing role-id file to guest", err)
+		http.Error(w, err.Error(), 500)
+		return
+	}
+
+	metadata := map[string]string{"fqdn": domainName + "." + config.C.Hostname}
+	metadataString, err := json.Marshal(metadata)
+	if err != nil {
+		log.Println("Error generating metadata string to create secret-id", err)
+		http.Error(w, err.Error(), 500)
+		return
+	}
+
+	secretID, err := vaultClient.Logical().Write(appRolePath+"/secret-id", map[string]interface{}{"metadata": metadataString})
+	if err != nil {
+		log.Println("Error getting secret-id for role", appRolePath, err)
+		http.Error(w, err.Error(), 500)
+		return
+	}
+
+	secretIDstring, ok := secretID.Data["secret_id"].(string)
+	if !ok {
+		msg := "Cannot convert secret_id as received from vault to string"
+		log.Println(msg)
+		http.Error(w, msg, 500)
+		return
+	}
+
+	err = WriteSecret(domain, "/ec/entanglement/auth/secret-id", secretIDstring)
+	if err != nil {
+		log.Println("Error writing secret-id file to guest", err)
+		http.Error(w, err.Error(), 500)
+		return
+	}
 
 	statusResponse, err := GuestExec(domain, config.C.Vault.InjectionCommand, domainName)
 	if err != nil {
@@ -44,11 +101,33 @@ func InjectAppRole(w http.ResponseWriter, r *http.Request) {
 
 	err = json.NewEncoder(w).Encode(statusResponse)
 	if err != nil {
+		log.Println("Error encoding response", err)
 		http.Error(w, err.Error(), 500)
 	}
 }
 
-func WriteSecret(domain *libvirt.Domain, secret *api.Secret) error {
+func WriteSecret(domain *libvirt.Domain, path string, secret string) error {
+	fh, err := qapi.ExecuteGuestFileOpen(domain, qapi.GuestFileOpenArg{Path: path, Mode: "w"})
+	if err != nil {
+		log.Println("Error opening", path, "on guest", err)
+		return err
+	}
+
+	_, err = qapi.ExecuteGuestFileWrite(domain, qapi.GuestFileWriteArg{
+		Handle: fh,
+		BufB64: base64.StdEncoding.EncodeToString([]byte(secret)),
+	})
+	if err != nil {
+		log.Println("Error writing to", path, "on guest", err)
+		return err
+	}
+
+	err = qapi.ExecuteGuestFileClose(domain, qapi.GuestFileCloseArg{Handle: fh})
+	if err != nil {
+		log.Println("Error closing file", path, "on guest after writing secret", err)
+		return err
+	}
+
 	return nil
 }
 
@@ -61,7 +140,7 @@ func GuestExec(domain *libvirt.Domain, path string, args ...string) (response qa
 		CaptureOutput: true,
 	})
 	if err != nil {
-		log.Println("Error executing guest-exec command: %s", err.Error())
+		log.Println("Error executing guest-exec command:", err)
 		return
 	}
 
-- 
GitLab


From 5f64f979bb8acd200bc4190da5bcfaf8f75e0917 Mon Sep 17 00:00:00 2001
From: Finn <finn@entanglement.garden>
Date: Wed, 5 Feb 2020 18:29:36 -0800
Subject: [PATCH 07/10] Handle shutdown signals gracefully

I was trying something that didn't work out but we got signal handling out of it
---
 cmd/rhyzome/main.go    | 33 +++++++++++++++++++++++++++++++--
 instances/instances.go |  4 ++++
 metadata/server.go     | 19 ++++++++++++++++++-
 rest/router.go         | 20 +++++++++++++++++++-
 vault/approle.go       |  4 ++--
 5 files changed, 74 insertions(+), 6 deletions(-)

diff --git a/cmd/rhyzome/main.go b/cmd/rhyzome/main.go
index a1d2bc2..863d93b 100644
--- a/cmd/rhyzome/main.go
+++ b/cmd/rhyzome/main.go
@@ -1,17 +1,46 @@
 package main
 
 import (
+	"log"
+	"os"
+	"os/signal"
+	"syscall"
+
 	"git.callpipe.com/entanglement.garden/rhyzome/config"
 	"git.callpipe.com/entanglement.garden/rhyzome/metadata"
 	"git.callpipe.com/entanglement.garden/rhyzome/rest"
 	"git.callpipe.com/entanglement.garden/rhyzome/vault"
 )
 
+var (
+	signals = make(chan os.Signal, 1)
+)
+
 func main() {
+	signal.Notify(signals, syscall.SIGINT, syscall.SIGHUP)
+
 	config.Load()
 	vault.Load()
 
-	// TODO: Some sort of shutdown signal
 	go metadata.ListenAndServe()
-	rest.ListenAndServe()
+	go rest.ListenAndServe()
+
+	for {
+		signal := <-signals
+		log.Println("Received signal", signal)
+		switch signal {
+		case syscall.SIGHUP:
+			config.Load()
+		case syscall.SIGINT:
+			if err := rest.Shutdown(); err != nil {
+				log.Println("Error shutting down API server", err)
+			}
+
+			if err := metadata.Shutdown(); err != nil {
+				log.Println("Error shutting down metadata server", err)
+			}
+
+			return
+		}
+	}
 }
diff --git a/instances/instances.go b/instances/instances.go
index 47fc399..327ba87 100644
--- a/instances/instances.go
+++ b/instances/instances.go
@@ -43,6 +43,7 @@ func GetAll() (instances []Instance, err error) {
 	if err != nil {
 		return []Instance{}, err
 	}
+	defer conn.Close()
 
 	domainIDs, err := conn.ListDomains()
 	if err != nil {
@@ -97,6 +98,7 @@ func FromDomainID(domainID uint32) (Instance, error) {
 	if err != nil {
 		return Instance{}, err
 	}
+	defer conn.Close()
 	return FromDomainIDWithConnect(conn, domainID)
 }
 
@@ -119,6 +121,7 @@ func FromDomainUUID(domainUUID string) (Instance, error) {
 	if err != nil {
 		return Instance{}, err
 	}
+	defer conn.Close()
 
 	dom, err := conn.LookupDomainByUUIDString(domainUUID)
 	if err != nil {
@@ -140,6 +143,7 @@ func (i *Instance) CreateAsJob(j *jobs.Job) error {
 	if err != nil {
 		return err
 	}
+	defer conn.Close()
 
 	// Create volume for new instance
 	j.SetStatus("Creating volume")
diff --git a/metadata/server.go b/metadata/server.go
index f3913a6..9257ff7 100644
--- a/metadata/server.go
+++ b/metadata/server.go
@@ -18,6 +18,10 @@ import (
 	"git.callpipe.com/entanglement.garden/rhyzome/vault"
 )
 
+var (
+	server *http.Server
+)
+
 func ListenAndServe() {
 	r := mux.NewRouter()
 	r.Use(utils.LoggingMiddleware)
@@ -26,7 +30,15 @@ func ListenAndServe() {
 	r.HandleFunc("/guest-info", guestInfo)
 	r.HandleFunc("/guest-get-osinfo", guestGetOSInfo)
 	r.HandleFunc("/vault/inject-app-role", vault.InjectAppRole)
-	log.Fatal(http.ListenAndServe(config.C.MetadataBind, r))
+	server = &http.Server{
+		Addr:    config.C.MetadataBind,
+		Handler: r,
+	}
+	log.Println("Starting metadata server on", server.Addr)
+	err := server.ListenAndServe()
+	if err != http.ErrServerClosed {
+		panic(err)
+	}
 }
 
 func lookupRequester(next http.Handler) http.Handler {
@@ -106,3 +118,8 @@ func index(w http.ResponseWriter, r *http.Request) {
 	domain := ctx.Value(utils.InstanceDomainXMLContextKey).(libvirtxml.Domain)
 	json.NewEncoder(w).Encode(domain)
 }
+
+func Shutdown() error {
+	log.Println("Shutting down metadata server")
+	return server.Shutdown(context.Background())
+}
diff --git a/rest/router.go b/rest/router.go
index d659634..398560c 100644
--- a/rest/router.go
+++ b/rest/router.go
@@ -1,6 +1,7 @@
 package rest
 
 import (
+	"context"
 	"encoding/json"
 	"log"
 	"net/http"
@@ -11,6 +12,10 @@ import (
 	"git.callpipe.com/entanglement.garden/rhyzome/utils"
 )
 
+var (
+	server *http.Server
+)
+
 func ListenAndServe() {
 	r := mux.NewRouter()
 	r.Use(utils.LoggingMiddleware)
@@ -23,7 +28,15 @@ func ListenAndServe() {
 	r.HandleFunc("/api/v1alpha1/jobs/{id}", GetJob)
 
 	r.Use(loggingMiddleware)
-	log.Fatal(http.ListenAndServe(config.C.Bind, r))
+	server = &http.Server{
+		Addr:    config.C.Bind,
+		Handler: r,
+	}
+	log.Println("Starting API server on", server.Addr)
+	err := server.ListenAndServe()
+	if err != http.ErrServerClosed {
+		panic(err)
+	}
 }
 
 func renderRoot(w http.ResponseWriter, r *http.Request) {
@@ -37,3 +50,8 @@ func loggingMiddleware(next http.Handler) http.Handler {
 		log.Println(r.RemoteAddr, r.Method, r.RequestURI)
 	})
 }
+
+func Shutdown() error {
+	log.Println("Shutting down API server")
+	return server.Shutdown(context.Background())
+}
diff --git a/vault/approle.go b/vault/approle.go
index aaca71a..a84a3a8 100644
--- a/vault/approle.go
+++ b/vault/approle.go
@@ -56,7 +56,7 @@ func InjectAppRole(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	err = WriteSecret(domain, "/ec/entanglement/auth/role-id", roleIDstring)
+	err = WriteSecret(domain, "/etc/entanglement/auth/role-id", roleIDstring)
 	if err != nil {
 		log.Println("Error writing role-id file to guest", err)
 		http.Error(w, err.Error(), 500)
@@ -86,7 +86,7 @@ func InjectAppRole(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	err = WriteSecret(domain, "/ec/entanglement/auth/secret-id", secretIDstring)
+	err = WriteSecret(domain, "/etc/entanglement/auth/secret-id", secretIDstring)
 	if err != nil {
 		log.Println("Error writing secret-id file to guest", err)
 		http.Error(w, err.Error(), 500)
-- 
GitLab


From 1467866a4336cf3da47d706f0840b778cf1a35a2 Mon Sep 17 00:00:00 2001
From: Finn <finn@entanglement.garden>
Date: Wed, 12 Feb 2020 20:27:39 -0800
Subject: [PATCH 08/10] Create auth directory before writing files there

also make things more configurable
---
 config/config.go | 31 ++++++++++++++++++++-----------
 vault/approle.go | 20 ++++++++++++++------
 2 files changed, 34 insertions(+), 17 deletions(-)

diff --git a/config/config.go b/config/config.go
index 0067018..f5de331 100644
--- a/config/config.go
+++ b/config/config.go
@@ -27,13 +27,18 @@ type Config struct {
 }
 
 type VaultConfig struct {
-	TLSConfig             api.TLSConfig // TLSConfig is the TLS configuration to use, if any
-	Address               string
-	InjectionCommand      string
-	CommandPollMS         int
-	CommandTimeoutSeconds libvirt.DomainQemuAgentCommandTimeout
-	RoleIDFilePath        string
-	SecretIDFilePath      string
+	TLSConfig                  api.TLSConfig // TLSConfig is the TLS configuration to use, if any
+	Address                    string
+	InjectionCommand           string
+	CommandPollMS              int
+	GuestCommandTimeoutSeconds libvirt.DomainQemuAgentCommandTimeout
+	GuestAuthFolder            string
+	RoleIDFilePath             string
+	SecretIDFilePath           string
+	SecretIDTTL                string
+	TokenPolicies              []string
+	SecretIDNumUses            string
+	TokenMaxTTL                string
 }
 
 var (
@@ -51,10 +56,14 @@ var (
 		ImageOwner:      "64055",
 		MetadataBind:    "127.0.0.1:8081",
 		Vault: VaultConfig{
-			TLSConfig:             api.TLSConfig{},
-			InjectionCommand:      "/opt/entanglement/vault-inject.sh",
-			CommandPollMS:         250,
-			CommandTimeoutSeconds: 5,
+			TLSConfig:                  api.TLSConfig{},
+			InjectionCommand:           "/opt/entanglement/vault-inject.sh",
+			GuestAuthFolder:            "/etc/entanglement/auth",
+			CommandPollMS:              250,
+			GuestCommandTimeoutSeconds: 5,
+			SecretIDTTL:                "1h",
+			SecretIDNumUses:            "1",
+			TokenMaxTTL:                "24h",
 		},
 	}
 )
diff --git a/vault/approle.go b/vault/approle.go
index a84a3a8..dde4f3f 100644
--- a/vault/approle.go
+++ b/vault/approle.go
@@ -6,6 +6,7 @@ import (
 	"fmt"
 	"log"
 	"net/http"
+	"strings"
 	"time"
 
 	libvirt "github.com/libvirt/libvirt-go"
@@ -27,13 +28,20 @@ func InjectAppRole(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	_, err = GuestExec(domain, "mkdir", "-p", config.C.Vault.GuestAuthFolder)
+	if err != nil {
+		log.Println("Error creating", config.C.Vault.GuestAuthFolder, "on guest:", err)
+		http.Error(w, err.Error(), 500)
+		return
+	}
+
 	// Create the approle
 	appRolePath := fmt.Sprintf("auth/approle/role/%s.%s", domainName, config.C.Hostname)
 	_, err = vaultClient.Logical().Write(appRolePath, map[string]interface{}{
-		"secret_id_ttl":      "1h",
-		"token_policies":     "hypervisors,pki-and-read-own-cert",
-		"secret_id_num_uses": "1",
-		"token_max_ttl":      "24h",
+		"secret_id_ttl":      config.C.Vault.SecretIDTTL,
+		"token_policies":     strings.Join(config.C.Vault.TokenPolicies, ","),
+		"secret_id_num_uses": config.C.Vault.SecretIDNumUses,
+		"token_max_ttl":      config.C.Vault.TokenMaxTTL,
 	})
 	if err != nil {
 		log.Println("Error creating vault role", appRolePath, err)
@@ -56,7 +64,7 @@ func InjectAppRole(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	err = WriteSecret(domain, "/etc/entanglement/auth/role-id", roleIDstring)
+	err = WriteSecret(domain, fmt.Sprintf("%s/role-id", config.C.Vault.GuestAuthFolder), roleIDstring)
 	if err != nil {
 		log.Println("Error writing role-id file to guest", err)
 		http.Error(w, err.Error(), 500)
@@ -86,7 +94,7 @@ func InjectAppRole(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	err = WriteSecret(domain, "/etc/entanglement/auth/secret-id", secretIDstring)
+	err = WriteSecret(domain, fmt.Sprintf("%s/secret-id", config.C.Vault.GuestAuthFolder), secretIDstring)
 	if err != nil {
 		log.Println("Error writing secret-id file to guest", err)
 		http.Error(w, err.Error(), 500)
-- 
GitLab


From 8636db4cc047904123936e126ab1851551462249 Mon Sep 17 00:00:00 2001
From: Finn <finn@entanglement.garden>
Date: Thu, 13 Feb 2020 01:08:02 -0800
Subject: [PATCH 09/10] Add explicit cloud-init support to the metadata server

---
 config/config.go       |  4 ++++
 instances/instances.go |  6 ++++++
 metadata/cloudinit.go  | 41 +++++++++++++++++++++++++++++++++++++++++
 metadata/server.go     |  2 ++
 4 files changed, 53 insertions(+)
 create mode 100644 metadata/cloudinit.go

diff --git a/config/config.go b/config/config.go
index f5de331..6e7dc53 100644
--- a/config/config.go
+++ b/config/config.go
@@ -23,6 +23,8 @@ type Config struct {
 	ImageHost       string // ImageHost is the base URL for the disk image server
 	ImageOwner      string // ImageOwner is the UID that should own volume images
 	MetadataBind    string // MetadataBind is the port (and optionally IP) to bind the metadata server to
+	CloudInitSeed   string // CloudInitSeed is the seed to pass to cloud-init
+	CloudName       string // CloudName is the cloud provider name to tell cloud-init
 	Vault           VaultConfig
 }
 
@@ -55,6 +57,8 @@ var (
 		ImageHost:       "http://image-host.fruit-0.entanglement.garden",
 		ImageOwner:      "64055",
 		MetadataBind:    "127.0.0.1:8081",
+		CloudInitSeed:   "http://127.0.0.1:8081/cloud-init/",
+		CloudName:       "rhyzome",
 		Vault: VaultConfig{
 			TLSConfig:                  api.TLSConfig{},
 			InjectionCommand:           "/opt/entanglement/vault-inject.sh",
diff --git a/instances/instances.go b/instances/instances.go
index 327ba87..0171330 100644
--- a/instances/instances.go
+++ b/instances/instances.go
@@ -198,6 +198,12 @@ func (i *Instance) CreateAsJob(j *jobs.Job) error {
 			Graphics: []libvirtxml.DomainGraphic{libvirtxml.DomainGraphic{VNC: &libvirtxml.DomainGraphicVNC{}}},
 			Videos:   []libvirtxml.DomainVideo{libvirtxml.DomainVideo{Model: libvirtxml.DomainVideoModel{Type: "virtio"}}},
 		},
+		QEMUCommandline: &libvirtxml.DomainQEMUCommandline{
+			Args: []libvirtxml.DomainQEMUCommandlineArg{
+				libvirtxml.DomainQEMUCommandlineArg{Value: "-smbios"},
+				libvirtxml.DomainQEMUCommandlineArg{Value: "type=1,serial=ds=nocloud-net;s=" + config.C.CloudInitSeed},
+			},
+		},
 	}
 
 	domainXMLString, err := domainXML.Marshal()
diff --git a/metadata/cloudinit.go b/metadata/cloudinit.go
new file mode 100644
index 0000000..60d9933
--- /dev/null
+++ b/metadata/cloudinit.go
@@ -0,0 +1,41 @@
+package metadata
+
+import (
+	"log"
+	"net/http"
+
+	libvirtxml "github.com/libvirt/libvirt-go-xml"
+	"gopkg.in/yaml.v2"
+
+	"git.callpipe.com/entanglement.garden/rhyzome/config"
+	"git.callpipe.com/entanglement.garden/rhyzome/utils"
+)
+
+type CloudInitMetadata struct {
+	LocalHostname string `yaml:"local-hostname,omitempty"`
+	InstanceID    string `yaml:"instance-id,omitempty"`
+	CloudName     string `yaml:"cloud-name,omitempty"`
+}
+
+func cloudInitUserData(w http.ResponseWriter, r *http.Request) {
+	ctx := r.Context()
+	domain := ctx.Value(utils.InstanceDomainXMLContextKey).(libvirtxml.Domain)
+	w.Write([]byte(domain.Description))
+}
+
+func cloudInitMetaData(w http.ResponseWriter, r *http.Request) {
+	w.Header().Set("Content-Type", "application/yaml")
+	type cloudInitMetaDataResponse struct {
+	}
+	ctx := r.Context()
+	domain := ctx.Value(utils.InstanceDomainXMLContextKey).(libvirtxml.Domain)
+	err := yaml.NewEncoder(w).Encode(CloudInitMetadata{
+		LocalHostname: domain.Name,
+		InstanceID:    domain.UUID,
+		CloudName:     config.C.CloudName,
+	})
+	if err != nil {
+		log.Println("Error encoding response", err)
+		http.Error(w, err.Error(), 500)
+	}
+}
diff --git a/metadata/server.go b/metadata/server.go
index 9257ff7..b2b2177 100644
--- a/metadata/server.go
+++ b/metadata/server.go
@@ -27,6 +27,8 @@ func ListenAndServe() {
 	r.Use(utils.LoggingMiddleware)
 	r.Use(lookupRequester)
 	r.HandleFunc("/", index)
+	r.HandleFunc("/cloud-init/user-data", cloudInitUserData)
+	r.HandleFunc("/cloud-init/meta-data", cloudInitMetaData)
 	r.HandleFunc("/guest-info", guestInfo)
 	r.HandleFunc("/guest-get-osinfo", guestGetOSInfo)
 	r.HandleFunc("/vault/inject-app-role", vault.InjectAppRole)
-- 
GitLab


From 3abf27a7ba8e0452560a4f8b2a3a84d37b4b0c1c Mon Sep 17 00:00:00 2001
From: Finn Herzfeld <finn@entanglement.garden>
Date: Thu, 13 Feb 2020 12:05:34 -0800
Subject: [PATCH 10/10] Rearrange the config a little

---
 config/config.go       | 56 +++++++++++++++++++++++++-----------------
 instances/instances.go |  2 +-
 metadata/cloudinit.go  |  2 +-
 vault/approle.go       | 20 +++++++--------
 4 files changed, 46 insertions(+), 34 deletions(-)

diff --git a/config/config.go b/config/config.go
index 6e7dc53..be75d6e 100644
--- a/config/config.go
+++ b/config/config.go
@@ -14,29 +14,37 @@ var ConfigFiles = []string{"/etc/rhyzome.conf", "rhyzome.conf"}
 
 // Config describes all configurable keys
 type Config struct {
-	Bind            string // Bind is the address (and port) to bind the REST server to
-	BridgeInterface string // BridgeInterface is the bridge that all network interfaces are added to
-	DiskStoragePool string // DiskStoragePool is the name of the storage pool to use
-	Hostname        string // Hostname is the domain that guests will be created under
-	ImageDir        string // ImageDir is the path to the local image pool
-	ImageGroup      string // ImageGroup is the GID that should own volume images
-	ImageHost       string // ImageHost is the base URL for the disk image server
-	ImageOwner      string // ImageOwner is the UID that should own volume images
-	MetadataBind    string // MetadataBind is the port (and optionally IP) to bind the metadata server to
-	CloudInitSeed   string // CloudInitSeed is the seed to pass to cloud-init
-	CloudName       string // CloudName is the cloud provider name to tell cloud-init
-	Vault           VaultConfig
+	Bind                string                    // Bind is the address (and port) to bind the REST server to
+	BridgeInterface     string                    // BridgeInterface is the bridge that all network interfaces are added to
+	DiskStoragePool     string                    // DiskStoragePool is the name of the storage pool to use
+	Hostname            string                    // Hostname is the domain that guests will be created under
+	ImageDir            string                    // ImageDir is the path to the local image pool
+	ImageGroup          string                    // ImageGroup is the GID that should own volume images
+	ImageHost           string                    // ImageHost is the base URL for the disk image server
+	ImageOwner          string                    // ImageOwner is the UID that should own volume images
+	MetadataBind        string                    // MetadataBind is the port (and optionally IP) to bind the metadata server to
+	CloudInit           CloudInitConfig           // CloudInit contains settings that are specific to the cloud-init integration
+	Vault               VaultConfig               // Vault contains settings for connecting to the vault server
+	CredentialInjection CredentialInjectionConfig // CredentialInjection configures how credentials are injected into guests
 }
 
 type VaultConfig struct {
-	TLSConfig                  api.TLSConfig // TLSConfig is the TLS configuration to use, if any
-	Address                    string
-	InjectionCommand           string
+	TLSConfig        api.TLSConfig // TLSConfig is the TLS configuration to use, if any
+	Address          string
+	RoleIDFilePath   string
+	SecretIDFilePath string
+}
+
+type CloudInitConfig struct {
+	Seed      string // Seed is the seed URL to pass to cloud-init via qemu smbios flag. Should be http://<metadata server>/cloud-init/
+	CloudName string // CloudName is name of the cloud we tell cloud-init it's running on
+}
+
+type CredentialInjectionConfig struct {
+	PostInjectionCommand       string
 	CommandPollMS              int
 	GuestCommandTimeoutSeconds libvirt.DomainQemuAgentCommandTimeout
 	GuestAuthFolder            string
-	RoleIDFilePath             string
-	SecretIDFilePath           string
 	SecretIDTTL                string
 	TokenPolicies              []string
 	SecretIDNumUses            string
@@ -57,14 +65,18 @@ var (
 		ImageHost:       "http://image-host.fruit-0.entanglement.garden",
 		ImageOwner:      "64055",
 		MetadataBind:    "127.0.0.1:8081",
-		CloudInitSeed:   "http://127.0.0.1:8081/cloud-init/",
-		CloudName:       "rhyzome",
+		CloudInit: CloudInitConfig{
+			Seed:      "http://127.0.0.1:80801/cloud-init/",
+			CloudName: "rhyzome",
+		},
 		Vault: VaultConfig{
-			TLSConfig:                  api.TLSConfig{},
-			InjectionCommand:           "/opt/entanglement/vault-inject.sh",
+			TLSConfig: api.TLSConfig{},
+		},
+		CredentialInjection: CredentialInjectionConfig{
+			PostInjectionCommand:       "/opt/entanglement/vault-inject.sh",
+			GuestCommandTimeoutSeconds: 5,
 			GuestAuthFolder:            "/etc/entanglement/auth",
 			CommandPollMS:              250,
-			GuestCommandTimeoutSeconds: 5,
 			SecretIDTTL:                "1h",
 			SecretIDNumUses:            "1",
 			TokenMaxTTL:                "24h",
diff --git a/instances/instances.go b/instances/instances.go
index 0171330..5e28123 100644
--- a/instances/instances.go
+++ b/instances/instances.go
@@ -201,7 +201,7 @@ func (i *Instance) CreateAsJob(j *jobs.Job) error {
 		QEMUCommandline: &libvirtxml.DomainQEMUCommandline{
 			Args: []libvirtxml.DomainQEMUCommandlineArg{
 				libvirtxml.DomainQEMUCommandlineArg{Value: "-smbios"},
-				libvirtxml.DomainQEMUCommandlineArg{Value: "type=1,serial=ds=nocloud-net;s=" + config.C.CloudInitSeed},
+				libvirtxml.DomainQEMUCommandlineArg{Value: "type=1,serial=ds=nocloud-net;s=" + config.C.CloudInit.Seed},
 			},
 		},
 	}
diff --git a/metadata/cloudinit.go b/metadata/cloudinit.go
index 60d9933..c7112ea 100644
--- a/metadata/cloudinit.go
+++ b/metadata/cloudinit.go
@@ -32,7 +32,7 @@ func cloudInitMetaData(w http.ResponseWriter, r *http.Request) {
 	err := yaml.NewEncoder(w).Encode(CloudInitMetadata{
 		LocalHostname: domain.Name,
 		InstanceID:    domain.UUID,
-		CloudName:     config.C.CloudName,
+		CloudName:     config.C.CloudInit.CloudName,
 	})
 	if err != nil {
 		log.Println("Error encoding response", err)
diff --git a/vault/approle.go b/vault/approle.go
index dde4f3f..f9482b2 100644
--- a/vault/approle.go
+++ b/vault/approle.go
@@ -28,9 +28,9 @@ func InjectAppRole(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	_, err = GuestExec(domain, "mkdir", "-p", config.C.Vault.GuestAuthFolder)
+	_, err = GuestExec(domain, "mkdir", "-p", config.C.CredentialInjection.GuestAuthFolder)
 	if err != nil {
-		log.Println("Error creating", config.C.Vault.GuestAuthFolder, "on guest:", err)
+		log.Println("Error creating", config.C.CredentialInjection.GuestAuthFolder, "on guest:", err)
 		http.Error(w, err.Error(), 500)
 		return
 	}
@@ -38,10 +38,10 @@ func InjectAppRole(w http.ResponseWriter, r *http.Request) {
 	// Create the approle
 	appRolePath := fmt.Sprintf("auth/approle/role/%s.%s", domainName, config.C.Hostname)
 	_, err = vaultClient.Logical().Write(appRolePath, map[string]interface{}{
-		"secret_id_ttl":      config.C.Vault.SecretIDTTL,
-		"token_policies":     strings.Join(config.C.Vault.TokenPolicies, ","),
-		"secret_id_num_uses": config.C.Vault.SecretIDNumUses,
-		"token_max_ttl":      config.C.Vault.TokenMaxTTL,
+		"secret_id_ttl":      config.C.CredentialInjection.SecretIDTTL,
+		"token_policies":     strings.Join(config.C.CredentialInjection.TokenPolicies, ","),
+		"secret_id_num_uses": config.C.CredentialInjection.SecretIDNumUses,
+		"token_max_ttl":      config.C.CredentialInjection.TokenMaxTTL,
 	})
 	if err != nil {
 		log.Println("Error creating vault role", appRolePath, err)
@@ -64,7 +64,7 @@ func InjectAppRole(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	err = WriteSecret(domain, fmt.Sprintf("%s/role-id", config.C.Vault.GuestAuthFolder), roleIDstring)
+	err = WriteSecret(domain, fmt.Sprintf("%s/role-id", config.C.CredentialInjection.GuestAuthFolder), roleIDstring)
 	if err != nil {
 		log.Println("Error writing role-id file to guest", err)
 		http.Error(w, err.Error(), 500)
@@ -94,14 +94,14 @@ func InjectAppRole(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	err = WriteSecret(domain, fmt.Sprintf("%s/secret-id", config.C.Vault.GuestAuthFolder), secretIDstring)
+	err = WriteSecret(domain, fmt.Sprintf("%s/secret-id", config.C.CredentialInjection.GuestAuthFolder), secretIDstring)
 	if err != nil {
 		log.Println("Error writing secret-id file to guest", err)
 		http.Error(w, err.Error(), 500)
 		return
 	}
 
-	statusResponse, err := GuestExec(domain, config.C.Vault.InjectionCommand, domainName)
+	statusResponse, err := GuestExec(domain, config.C.CredentialInjection.PostInjectionCommand, domainName)
 	if err != nil {
 		http.Error(w, err.Error(), 500)
 		return
@@ -164,7 +164,7 @@ func GuestExec(domain *libvirt.Domain, path string, args ...string) (response qa
 			break
 		}
 
-		time.Sleep(time.Duration(config.C.Vault.CommandPollMS) * time.Millisecond)
+		time.Sleep(time.Duration(config.C.CredentialInjection.CommandPollMS) * time.Millisecond)
 	}
 
 	return
-- 
GitLab