From 10508259e856abfa90db22ae93eceb62ac1ca1ec Mon Sep 17 00:00:00 2001 From: Finn <finn@entanglement.garden> Date: Wed, 30 Oct 2019 22:12:58 -0700 Subject: [PATCH 01/10] Initial metadata server proof of concept --- cmd/rhyzome/main.go | 5 ++- metadata/server.go | 82 +++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 86 insertions(+), 1 deletion(-) create mode 100644 metadata/server.go diff --git a/cmd/rhyzome/main.go b/cmd/rhyzome/main.go index 66d5b4d..9e51c8f 100644 --- a/cmd/rhyzome/main.go +++ b/cmd/rhyzome/main.go @@ -1,11 +1,14 @@ package main import ( - "git.callpipe.com/entanglement.garden/rhyzome/rest" "git.callpipe.com/entanglement.garden/rhyzome/config" + "git.callpipe.com/entanglement.garden/rhyzome/metadata" + "git.callpipe.com/entanglement.garden/rhyzome/rest" ) func main() { config.Load() + // TODO: Some sort of shutdown signal + go metadata.ListenAndServe() rest.ListenAndServe() } diff --git a/metadata/server.go b/metadata/server.go new file mode 100644 index 0000000..f8668a6 --- /dev/null +++ b/metadata/server.go @@ -0,0 +1,82 @@ +package metadata + +import ( + "encoding/json" + "log" + "net" + "net/http" + "strings" + + "github.com/gorilla/mux" + libvirt "github.com/libvirt/libvirt-go" + libvirtxml "github.com/libvirt/libvirt-go-xml" + "github.com/mostlygeek/arp" +) + +func ListenAndServe() { + r := mux.NewRouter() + r.HandleFunc("/", index) + log.Fatal(http.ListenAndServe(":8081", r)) +} + +func index(w http.ResponseWriter, r *http.Request) { + // TODO: Can i wrap this in middleware? + conn, err := libvirt.NewConnect("qemu:///system") + if err != nil { + http.Error(w, err.Error(), 500) + return + } + defer conn.Close() + + ip, _, err := net.SplitHostPort(r.RemoteAddr) + if err != nil { + http.Error(w, err.Error(), 404) + return + } + + mac := strings.Trim(arp.Search(ip), "'") + + log.Printf("%s Looking up metadata for %s", r.RemoteAddr, mac) + + // iface, err := conn.LookupInterfaceByMACString(mac) + domainIDs, err := conn.ListDomains() + if err != nil { + http.Error(w, err.Error(), 404) + return + } + + for _, domainID := range domainIDs { + dom, err := conn.LookupDomainById(domainID) + if err != nil { + log.Printf("Error looking up domain %d: %s", dom, err.Error()) + continue + } + if dom == nil { + log.Printf("LookupDomainById(%d) returned nil", domainID) + continue + } + + xmldoc, err := dom.GetXMLDesc(0) + if err != nil { + http.Error(w, err.Error(), 500) + return + } + + domain := libvirtxml.Domain{} + if err := domain.Unmarshal(xmldoc); err != nil { + continue + } + // domains = append(domains, domcfg) + + for _, iface := range domain.Devices.Interfaces { + if iface.MAC.Address == mac { + json.NewEncoder(w).Encode(domain) + log.Printf("%s found!", mac) + return + } + + log.Printf("%s != %s", iface.MAC.Address, mac) + } + } + http.Error(w, "not found", 404) +} -- GitLab From 159518c788cc6ee4a90e49f912b7d469a3e7f602 Mon Sep 17 00:00:00 2001 From: Finn <finn@entanglement.garden> Date: Mon, 11 Nov 2019 16:25:29 -0800 Subject: [PATCH 02/10] WIP --- instances/instances.go | 2 - metadata/server.go | 122 +++++++++++++++++++++++++---------------- rest/router.go | 3 + utils/middleware.go | 34 ++++++++++++ 4 files changed, 111 insertions(+), 50 deletions(-) create mode 100644 utils/middleware.go diff --git a/instances/instances.go b/instances/instances.go index 3b66fa8..d275a1b 100644 --- a/instances/instances.go +++ b/instances/instances.go @@ -11,8 +11,6 @@ import ( "git.callpipe.com/entanglement.garden/rhyzome/jobs" ) -type contextKey int - type Instance struct { Name string UUID string diff --git a/metadata/server.go b/metadata/server.go index f8668a6..c2c3149 100644 --- a/metadata/server.go +++ b/metadata/server.go @@ -1,6 +1,7 @@ package metadata import ( + "context" "encoding/json" "log" "net" @@ -11,72 +12,97 @@ import ( libvirt "github.com/libvirt/libvirt-go" libvirtxml "github.com/libvirt/libvirt-go-xml" "github.com/mostlygeek/arp" + + "git.callpipe.com/entanglement.garden/rhyzome/utils" +) + +const ( + instanceContextKey utils.ContextKey = 0 + libvirtConnecKey utils.ContextKey = 1 ) func ListenAndServe() { - r := mux.NewRouter() + r := mux.NewRouter() + r.Use(utils.LoggingMiddleware) + r.Use(lookupRequester) r.HandleFunc("/", index) log.Fatal(http.ListenAndServe(":8081", r)) } -func index(w http.ResponseWriter, r *http.Request) { - // TODO: Can i wrap this in middleware? - conn, err := libvirt.NewConnect("qemu:///system") - if err != nil { - http.Error(w, err.Error(), 500) - return - } - defer conn.Close() - - ip, _, err := net.SplitHostPort(r.RemoteAddr) - if err != nil { - http.Error(w, err.Error(), 404) - return - } - - mac := strings.Trim(arp.Search(ip), "'") - - log.Printf("%s Looking up metadata for %s", r.RemoteAddr, mac) - - // iface, err := conn.LookupInterfaceByMACString(mac) - domainIDs, err := conn.ListDomains() - if err != nil { - http.Error(w, err.Error(), 404) - return - } - - for _, domainID := range domainIDs { - dom, err := conn.LookupDomainById(domainID) +func lookupRequester(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + conn, err := libvirt.NewConnect("qemu:///system") if err != nil { - log.Printf("Error looking up domain %d: %s", dom, err.Error()) - continue - } - if dom == nil { - log.Printf("LookupDomainById(%d) returned nil", domainID) - continue + http.Error(w, err.Error(), 500) + return } + defer conn.Close() - xmldoc, err := dom.GetXMLDesc(0) + ip, _, err := net.SplitHostPort(r.RemoteAddr) if err != nil { - http.Error(w, err.Error(), 500) + http.Error(w, err.Error(), 404) return } - domain := libvirtxml.Domain{} - if err := domain.Unmarshal(xmldoc); err != nil { - continue + mac := strings.Trim(arp.Search(ip), "'") + + log.Printf("[%s] Looking up metadata for %s", ip, mac) + + domainIDs, err := conn.ListDomains() + if err != nil { + http.Error(w, err.Error(), 404) + return } - // domains = append(domains, domcfg) - for _, iface := range domain.Devices.Interfaces { - if iface.MAC.Address == mac { - json.NewEncoder(w).Encode(domain) - log.Printf("%s found!", mac) + ctx := r.Context() + found := false + + for _, domainID := range domainIDs { + dom, err := conn.LookupDomainById(domainID) + if err != nil { + continue + } + if dom == nil { + log.Printf("LookupDomainById(%d) returned nil", domainID) + continue + } + + xmldoc, err := dom.GetXMLDesc(0) + if err != nil { + http.Error(w, err.Error(), 500) return } - log.Printf("%s != %s", iface.MAC.Address, mac) + domain := libvirtxml.Domain{} + if err := domain.Unmarshal(xmldoc); err != nil { + continue + } + // domains = append(domains, domcfg) + + for _, iface := range domain.Devices.Interfaces { + if iface.MAC.Address == mac { + ctx = context.WithValue(ctx, libvirtConnecKey, conn) + ctx = context.WithValue(ctx, instanceContextKey, domain) + found = true + break + } + } + if found { + break + } } - } - http.Error(w, "not found", 404) + + if !found { + http.Error(w, "unknown client", 401) + return + } + + next.ServeHTTP(w, r.WithContext(ctx)) + }) +} + +func index(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + domain := ctx.Value(instanceContextKey).(libvirt.Domain) + json.NewEncoder(w).Encode(domain) } diff --git a/rest/router.go b/rest/router.go index 450625e..3fb3ca3 100644 --- a/rest/router.go +++ b/rest/router.go @@ -6,10 +6,13 @@ import ( "net/http" "github.com/gorilla/mux" + + "git.callpipe.com/entanglement.garden/rhyzome/utils" ) func ListenAndServe() { r := mux.NewRouter() + r.Use(utils.LoggingMiddleware) r.HandleFunc("/", renderRoot) r.HandleFunc("/api/v1alpha1/instances", CreateInstance).Methods("POST") r.HandleFunc("/api/v1alpha1/instances", ListInstances).Methods("GET") diff --git a/utils/middleware.go b/utils/middleware.go new file mode 100644 index 0000000..207fe23 --- /dev/null +++ b/utils/middleware.go @@ -0,0 +1,34 @@ +package utils + +import ( + "log" + "net/http" +) + +type ContextKey int + +type wrapper struct { + http.ResponseWriter + written int + status int +} + +func (w *wrapper) WriteHeader(code int) { + w.status = code + w.ResponseWriter.WriteHeader(code) +} + +func (w *wrapper) Write(b []byte) (int, error) { + n, err := w.ResponseWriter.Write(b) + w.written += n + return n, err +} + +// LoggingMiddleware is a middleware function that logs requests as they come in +func LoggingMiddleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + res := &wrapper{w, 0, 0} + next.ServeHTTP(res, r) + log.Printf("%s %s %s %d %d", r.RemoteAddr, r.Method, r.RequestURI, res.status, res.written) + }) +} -- GitLab From 215ae8a790d5d476d516e872cc3c5eb49a22a93a Mon Sep 17 00:00:00 2001 From: Finn <finn@entanglement.garden> Date: Mon, 27 Jan 2020 08:55:04 -0800 Subject: [PATCH 03/10] Configurable bind for metadata server --- config/config.go | 14 ++++++++------ metadata/server.go | 3 ++- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/config/config.go b/config/config.go index b78071e..fd60a31 100644 --- a/config/config.go +++ b/config/config.go @@ -12,12 +12,13 @@ var ConfigFiles = []string{"/etc/rhyzome.conf", "rhyzome.conf"} // Config describes all configurable keys type Config struct { Bind string // Bind is the address (and port) to bind the REST server to + BridgeInterface string // BridgeInterface is the bridge that all network interfaces are added to DiskStoragePool string // DiskStoragePool is the name of the storage pool to use - ImageOwner string // ImageOwner is the UID that should own volume images + ImageDir string // ImageDir is the path to the local image pool ImageGroup string // ImageGroup is the GID that should own volume images - BridgeInterface string // BridgeInterface is the bridge that all network interfaces are added to ImageHost string // ImageHost is the base URL for the disk image server - ImageDir string // ImageDir is the path to the local image pool + ImageOwner string // ImageOwner is the UID that should own volume images + MetadataBind string // MetadataBind is the port (and optionally IP) to bind the metadata server to } var ( @@ -27,12 +28,13 @@ var ( // Defaults are the default values for each config option, set in the Load() function Defaults = Config{ Bind: ":8080", + BridgeInterface: "brlan", DiskStoragePool: "default", - ImageOwner: "64055", + ImageDir: "/var/lib/libvirt/images", ImageGroup: "64055", - BridgeInterface: "brlan", ImageHost: "http://image-host.fruit-0.entanglement.garden", - ImageDir: "/var/lib/libvirt/images", + ImageOwner: "64055", + MetadataBind: ":8081", } ) diff --git a/metadata/server.go b/metadata/server.go index c2c3149..d31c285 100644 --- a/metadata/server.go +++ b/metadata/server.go @@ -13,6 +13,7 @@ import ( libvirtxml "github.com/libvirt/libvirt-go-xml" "github.com/mostlygeek/arp" + "git.callpipe.com/entanglement.garden/rhyzome/config" "git.callpipe.com/entanglement.garden/rhyzome/utils" ) @@ -26,7 +27,7 @@ func ListenAndServe() { r.Use(utils.LoggingMiddleware) r.Use(lookupRequester) r.HandleFunc("/", index) - log.Fatal(http.ListenAndServe(":8081", r)) + log.Fatal(http.ListenAndServe(config.C.MetadataBind, r)) } func lookupRequester(next http.Handler) http.Handler { -- GitLab From 93866cb52a8b03aa413218a13f8faa97caece5e8 Mon Sep 17 00:00:00 2001 From: Finn <finn@entanglement.garden> Date: Mon, 27 Jan 2020 08:56:35 -0800 Subject: [PATCH 04/10] Default bind the metadata server to 127.0.0.1 --- config/config.go | 2 +- metadata/server.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/config/config.go b/config/config.go index fd60a31..6e1e2b0 100644 --- a/config/config.go +++ b/config/config.go @@ -34,7 +34,7 @@ var ( ImageGroup: "64055", ImageHost: "http://image-host.fruit-0.entanglement.garden", ImageOwner: "64055", - MetadataBind: ":8081", + MetadataBind: "127.0.0.1:8081", } ) diff --git a/metadata/server.go b/metadata/server.go index d31c285..d1d5b88 100644 --- a/metadata/server.go +++ b/metadata/server.go @@ -104,6 +104,6 @@ func lookupRequester(next http.Handler) http.Handler { func index(w http.ResponseWriter, r *http.Request) { ctx := r.Context() - domain := ctx.Value(instanceContextKey).(libvirt.Domain) + domain := ctx.Value(instanceContextKey).(libvirtxml.Domain) json.NewEncoder(w).Encode(domain) } -- GitLab From 11207fc8d63c17dd8f0a87bf771725b61dc6917e Mon Sep 17 00:00:00 2001 From: Finn Herzfeld <finn@entanglement.garden> Date: Wed, 5 Feb 2020 10:55:34 -0800 Subject: [PATCH 05/10] Initial attempt at vault role injection doesn't work yet --- cmd/rhyzome/main.go | 3 ++ config/config.go | 35 ++++++++++++- metadata/instanceInfo.go | 33 +++++++++++++ metadata/server.go | 29 ++++++----- utils/middleware.go | 6 +++ vault/approle.go | 84 +++++++++++++++++++++++++++++++ vault/vault.go | 103 +++++++++++++++++++++++++++++++++++++++ 7 files changed, 277 insertions(+), 16 deletions(-) create mode 100644 metadata/instanceInfo.go create mode 100644 vault/approle.go create mode 100644 vault/vault.go diff --git a/cmd/rhyzome/main.go b/cmd/rhyzome/main.go index 9e51c8f..a1d2bc2 100644 --- a/cmd/rhyzome/main.go +++ b/cmd/rhyzome/main.go @@ -4,10 +4,13 @@ import ( "git.callpipe.com/entanglement.garden/rhyzome/config" "git.callpipe.com/entanglement.garden/rhyzome/metadata" "git.callpipe.com/entanglement.garden/rhyzome/rest" + "git.callpipe.com/entanglement.garden/rhyzome/vault" ) func main() { config.Load() + vault.Load() + // TODO: Some sort of shutdown signal go metadata.ListenAndServe() rest.ListenAndServe() diff --git a/config/config.go b/config/config.go index fd60a31..0067018 100644 --- a/config/config.go +++ b/config/config.go @@ -5,6 +5,9 @@ import ( "io/ioutil" "log" "os" + + "github.com/hashicorp/vault/api" + libvirt "github.com/libvirt/libvirt-go" ) var ConfigFiles = []string{"/etc/rhyzome.conf", "rhyzome.conf"} @@ -14,11 +17,23 @@ type Config struct { Bind string // Bind is the address (and port) to bind the REST server to BridgeInterface string // BridgeInterface is the bridge that all network interfaces are added to DiskStoragePool string // DiskStoragePool is the name of the storage pool to use + Hostname string // Hostname is the domain that guests will be created under ImageDir string // ImageDir is the path to the local image pool ImageGroup string // ImageGroup is the GID that should own volume images ImageHost string // ImageHost is the base URL for the disk image server ImageOwner string // ImageOwner is the UID that should own volume images MetadataBind string // MetadataBind is the port (and optionally IP) to bind the metadata server to + Vault VaultConfig +} + +type VaultConfig struct { + TLSConfig api.TLSConfig // TLSConfig is the TLS configuration to use, if any + Address string + InjectionCommand string + CommandPollMS int + CommandTimeoutSeconds libvirt.DomainQemuAgentCommandTimeout + RoleIDFilePath string + SecretIDFilePath string } var ( @@ -34,7 +49,13 @@ var ( ImageGroup: "64055", ImageHost: "http://image-host.fruit-0.entanglement.garden", ImageOwner: "64055", - MetadataBind: ":8081", + MetadataBind: "127.0.0.1:8081", + Vault: VaultConfig{ + TLSConfig: api.TLSConfig{}, + InjectionCommand: "/opt/entanglement/vault-inject.sh", + CommandPollMS: 250, + CommandTimeoutSeconds: 5, + }, } ) @@ -59,5 +80,17 @@ func Load() { json.Unmarshal(byteValue, &C) log.Println("Successfully read config from", filename) + log.Printf("%+v", C) + } + + if C.Hostname == "" { + log.Println("Hostname not specified in config, setting from system") + hostname, err := os.Hostname() + if err != nil { + log.Println("Error getting hostname from system", err) + } else { + C.Hostname = hostname + log.Println("Set hostname to", C.Hostname) + } } } diff --git a/metadata/instanceInfo.go b/metadata/instanceInfo.go new file mode 100644 index 0000000..dce44f5 --- /dev/null +++ b/metadata/instanceInfo.go @@ -0,0 +1,33 @@ +package metadata + +import ( + "net/http" + + libvirt "github.com/libvirt/libvirt-go" + + "git.callpipe.com/entanglement.garden/rhyzome/utils" +) + +func guestInfo(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + domain := ctx.Value(utils.InstanceDomainContextKey).(*libvirt.Domain) + info, err := domain.QemuAgentCommand("{\"execute\":\"guest-info\"}", 1, 0) + if err != nil { + http.Error(w, err.Error(), 500) + return + } + + w.Write([]byte(info)) +} + +func guestGetOSInfo(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + domain := ctx.Value(utils.InstanceDomainContextKey).(*libvirt.Domain) + info, err := domain.QemuAgentCommand("{\"execute\":\"guest-get-osinfo\"}", 1, 0) + if err != nil { + http.Error(w, err.Error(), 500) + return + } + + w.Write([]byte(info)) +} diff --git a/metadata/server.go b/metadata/server.go index d31c285..f3913a6 100644 --- a/metadata/server.go +++ b/metadata/server.go @@ -15,11 +15,7 @@ import ( "git.callpipe.com/entanglement.garden/rhyzome/config" "git.callpipe.com/entanglement.garden/rhyzome/utils" -) - -const ( - instanceContextKey utils.ContextKey = 0 - libvirtConnecKey utils.ContextKey = 1 + "git.callpipe.com/entanglement.garden/rhyzome/vault" ) func ListenAndServe() { @@ -27,6 +23,9 @@ func ListenAndServe() { r.Use(utils.LoggingMiddleware) r.Use(lookupRequester) r.HandleFunc("/", index) + r.HandleFunc("/guest-info", guestInfo) + r.HandleFunc("/guest-get-osinfo", guestGetOSInfo) + r.HandleFunc("/vault/inject-app-role", vault.InjectAppRole) log.Fatal(http.ListenAndServe(config.C.MetadataBind, r)) } @@ -59,31 +58,31 @@ func lookupRequester(next http.Handler) http.Handler { found := false for _, domainID := range domainIDs { - dom, err := conn.LookupDomainById(domainID) + domain, err := conn.LookupDomainById(domainID) if err != nil { continue } - if dom == nil { + if domain == nil { log.Printf("LookupDomainById(%d) returned nil", domainID) continue } - xmldoc, err := dom.GetXMLDesc(0) + xmldoc, err := domain.GetXMLDesc(0) if err != nil { http.Error(w, err.Error(), 500) return } - domain := libvirtxml.Domain{} - if err := domain.Unmarshal(xmldoc); err != nil { + domainXML := libvirtxml.Domain{} + if err := domainXML.Unmarshal(xmldoc); err != nil { continue } - // domains = append(domains, domcfg) - for _, iface := range domain.Devices.Interfaces { + for _, iface := range domainXML.Devices.Interfaces { if iface.MAC.Address == mac { - ctx = context.WithValue(ctx, libvirtConnecKey, conn) - ctx = context.WithValue(ctx, instanceContextKey, domain) + ctx = context.WithValue(ctx, utils.LibvirtConnecKey, conn) + ctx = context.WithValue(ctx, utils.InstanceDomainContextKey, domain) + ctx = context.WithValue(ctx, utils.InstanceDomainXMLContextKey, domainXML) found = true break } @@ -104,6 +103,6 @@ func lookupRequester(next http.Handler) http.Handler { func index(w http.ResponseWriter, r *http.Request) { ctx := r.Context() - domain := ctx.Value(instanceContextKey).(libvirt.Domain) + domain := ctx.Value(utils.InstanceDomainXMLContextKey).(libvirtxml.Domain) json.NewEncoder(w).Encode(domain) } diff --git a/utils/middleware.go b/utils/middleware.go index 207fe23..72e6353 100644 --- a/utils/middleware.go +++ b/utils/middleware.go @@ -13,6 +13,12 @@ type wrapper struct { status int } +const ( + InstanceDomainContextKey ContextKey = 0 + InstanceDomainXMLContextKey ContextKey = 1 + LibvirtConnecKey ContextKey = 2 +) + func (w *wrapper) WriteHeader(code int) { w.status = code w.ResponseWriter.WriteHeader(code) diff --git a/vault/approle.go b/vault/approle.go new file mode 100644 index 0000000..008ae13 --- /dev/null +++ b/vault/approle.go @@ -0,0 +1,84 @@ +package vault + +import ( + "encoding/json" + "fmt" + "log" + "net/http" + "time" + + "github.com/hashicorp/vault/api" + libvirt "github.com/libvirt/libvirt-go" + + "git.callpipe.com/entanglement.garden/qapi" + "git.callpipe.com/entanglement.garden/rhyzome/config" + "git.callpipe.com/entanglement.garden/rhyzome/utils" +) + +// InjectAppRole injects the credentials for a given AppRole +func InjectAppRole(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + domain := ctx.Value(utils.InstanceDomainContextKey).(*libvirt.Domain) + + domainName, err := domain.GetName() + if err != nil { + log.Println("Error getting domain name") + http.Error(w, err.Error(), 500) + return + } + + secret, err := vaultClient.Logical().Write(fmt.Sprintf("auth/approle/role/%s.%s/secret-id", domainName, config.C.Hostname), make(map[string]interface{})) + if err != nil { + log.Println("Error getting approle for VM", err) + http.Error(w, err.Error(), 500) + return + } + + err = WriteSecret(domain, secret) + + statusResponse, err := GuestExec(domain, config.C.Vault.InjectionCommand, domainName) + if err != nil { + http.Error(w, err.Error(), 500) + return + } + + err = json.NewEncoder(w).Encode(statusResponse) + if err != nil { + http.Error(w, err.Error(), 500) + } +} + +func WriteSecret(domain *libvirt.Domain, secret *api.Secret) error { + return nil +} + +// GuestExec runs a command on the specified domain as root (via the qemu guest agent), waits for it to complete, and returns the response +// TODO: Should this be a member function of instances.Instance? +func GuestExec(domain *libvirt.Domain, path string, args ...string) (response qapi.GuestExecStatus, err error) { + commandResponse, err := qapi.ExecuteGuestExec(domain, qapi.GuestExecArg{ + Path: path, + Arg: args, + CaptureOutput: true, + }) + if err != nil { + log.Println("Error executing guest-exec command: %s", err.Error()) + return + } + + for { + response, err = qapi.ExecuteGuestExecStatus(domain, qapi.GuestExecStatusArg{PID: commandResponse.PID}) + if err != nil { + log.Printf("Error checking status of command running on guest: %s", err.Error()) + return + } + + if response.Exited { + log.Printf("Command %s exited: %+v", path, response) + break + } + + time.Sleep(time.Duration(config.C.Vault.CommandPollMS) * time.Millisecond) + } + + return +} diff --git a/vault/vault.go b/vault/vault.go new file mode 100644 index 0000000..aa2b102 --- /dev/null +++ b/vault/vault.go @@ -0,0 +1,103 @@ +package vault + +import ( + "github.com/hashicorp/vault/api" + "io/ioutil" + "log" + "strings" + "time" + + "git.callpipe.com/entanglement.garden/rhyzome/config" +) + +var ( + vaultClient *api.Client + tokenWatcher *api.LifetimeWatcher + secret *api.Secret +) + +// Load initializes the vault client with the configured vaults from the config file and enviornment variables +func Load() error { + c := api.DefaultConfig() + c.Address = config.C.Vault.Address + + err := c.ConfigureTLS(&config.C.Vault.TLSConfig) + if err != nil { + log.Println("Error configuring Vault TLS") + return err + } + + err = c.ReadEnvironment() + if err != nil { + log.Println("Error configuring Vault from enviornment") + return err + } + + vaultClient, err = api.NewClient(c) + if err != nil { + log.Println("Error configuring vault client") + return err + } + + roleID, err := ioutil.ReadFile(config.C.Vault.RoleIDFilePath) + if err != nil { + log.Println("Error reading role id:", err) + return err + } + + secretID, err := ioutil.ReadFile(config.C.Vault.SecretIDFilePath) + if err != nil { + log.Println("Error reading secret id:", err) + return err + } + + log.Println("Logging into vault at", c.Address) + secret, err = vaultClient.Logical().Write("/auth/approle/login", map[string]interface{}{ + "role_id": strings.TrimSpace(string(roleID)), + "secret_id": strings.TrimSpace(string(secretID)), + }) + if err != nil { + log.Println("Error logging into vault:", err) + return err + } + + err = startTokenWatcher() + + return err +} + +func startTokenWatcher() (err error) { + log.Println("Starting token watcher") + if vaultClient == nil { + log.Println("vaultClient is nil! We will now crash") + } + tokenWatcher, err = vaultClient.NewLifetimeWatcher(&api.LifetimeWatcherInput{Secret: secret}) + if err != nil { + return err + } + go tokenWatcher.Start() + go watchToken() + return nil +} + +func watchToken() { + if tokenWatcher == nil { + log.Println("WatchToken gonna fail because tokenWatcher is nil") + } + for { + select { + case err := <-tokenWatcher.DoneCh(): + if err != nil { + log.Println("Error in vault watcher:", err) + time.Sleep(5 * time.Second) + tokenWatcher.Stop() + startTokenWatcher() + return + } + + case renewal := <-tokenWatcher.RenewCh(): + log.Printf("Successfully renewed vault client token %+v", renewal.Secret.Auth) + vaultClient.SetToken(renewal.Secret.Auth.ClientToken) + } + } +} -- GitLab From 2bc1c7e26e0a661d6c8ce1312d9e68cb9c9cc519 Mon Sep 17 00:00:00 2001 From: Finn Herzfeld <finn@entanglement.garden> Date: Wed, 5 Feb 2020 14:08:50 -0800 Subject: [PATCH 06/10] Actually write the secret to the guest does not work yet --- vault/approle.go | 91 ++++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 85 insertions(+), 6 deletions(-) diff --git a/vault/approle.go b/vault/approle.go index 008ae13..aaca71a 100644 --- a/vault/approle.go +++ b/vault/approle.go @@ -1,13 +1,13 @@ package vault import ( + "encoding/base64" "encoding/json" "fmt" "log" "net/http" "time" - "github.com/hashicorp/vault/api" libvirt "github.com/libvirt/libvirt-go" "git.callpipe.com/entanglement.garden/qapi" @@ -27,14 +27,71 @@ func InjectAppRole(w http.ResponseWriter, r *http.Request) { return } - secret, err := vaultClient.Logical().Write(fmt.Sprintf("auth/approle/role/%s.%s/secret-id", domainName, config.C.Hostname), make(map[string]interface{})) + // Create the approle + appRolePath := fmt.Sprintf("auth/approle/role/%s.%s", domainName, config.C.Hostname) + _, err = vaultClient.Logical().Write(appRolePath, map[string]interface{}{ + "secret_id_ttl": "1h", + "token_policies": "hypervisors,pki-and-read-own-cert", + "secret_id_num_uses": "1", + "token_max_ttl": "24h", + }) if err != nil { - log.Println("Error getting approle for VM", err) + log.Println("Error creating vault role", appRolePath, err) http.Error(w, err.Error(), 500) return } - err = WriteSecret(domain, secret) + roleID, err := vaultClient.Logical().Read(appRolePath + "/role-id") + if err != nil { + log.Println("Error getting role-id for role", appRolePath) + http.Error(w, err.Error(), 500) + return + } + + roleIDstring, ok := roleID.Data["role_id"].(string) + if !ok { + msg := "Cannot convert role_id as received from vault to string" + log.Println(msg) + http.Error(w, msg, 500) + return + } + + err = WriteSecret(domain, "/ec/entanglement/auth/role-id", roleIDstring) + if err != nil { + log.Println("Error writing role-id file to guest", err) + http.Error(w, err.Error(), 500) + return + } + + metadata := map[string]string{"fqdn": domainName + "." + config.C.Hostname} + metadataString, err := json.Marshal(metadata) + if err != nil { + log.Println("Error generating metadata string to create secret-id", err) + http.Error(w, err.Error(), 500) + return + } + + secretID, err := vaultClient.Logical().Write(appRolePath+"/secret-id", map[string]interface{}{"metadata": metadataString}) + if err != nil { + log.Println("Error getting secret-id for role", appRolePath, err) + http.Error(w, err.Error(), 500) + return + } + + secretIDstring, ok := secretID.Data["secret_id"].(string) + if !ok { + msg := "Cannot convert secret_id as received from vault to string" + log.Println(msg) + http.Error(w, msg, 500) + return + } + + err = WriteSecret(domain, "/ec/entanglement/auth/secret-id", secretIDstring) + if err != nil { + log.Println("Error writing secret-id file to guest", err) + http.Error(w, err.Error(), 500) + return + } statusResponse, err := GuestExec(domain, config.C.Vault.InjectionCommand, domainName) if err != nil { @@ -44,11 +101,33 @@ func InjectAppRole(w http.ResponseWriter, r *http.Request) { err = json.NewEncoder(w).Encode(statusResponse) if err != nil { + log.Println("Error encoding response", err) http.Error(w, err.Error(), 500) } } -func WriteSecret(domain *libvirt.Domain, secret *api.Secret) error { +func WriteSecret(domain *libvirt.Domain, path string, secret string) error { + fh, err := qapi.ExecuteGuestFileOpen(domain, qapi.GuestFileOpenArg{Path: path, Mode: "w"}) + if err != nil { + log.Println("Error opening", path, "on guest", err) + return err + } + + _, err = qapi.ExecuteGuestFileWrite(domain, qapi.GuestFileWriteArg{ + Handle: fh, + BufB64: base64.StdEncoding.EncodeToString([]byte(secret)), + }) + if err != nil { + log.Println("Error writing to", path, "on guest", err) + return err + } + + err = qapi.ExecuteGuestFileClose(domain, qapi.GuestFileCloseArg{Handle: fh}) + if err != nil { + log.Println("Error closing file", path, "on guest after writing secret", err) + return err + } + return nil } @@ -61,7 +140,7 @@ func GuestExec(domain *libvirt.Domain, path string, args ...string) (response qa CaptureOutput: true, }) if err != nil { - log.Println("Error executing guest-exec command: %s", err.Error()) + log.Println("Error executing guest-exec command:", err) return } -- GitLab From 5f64f979bb8acd200bc4190da5bcfaf8f75e0917 Mon Sep 17 00:00:00 2001 From: Finn <finn@entanglement.garden> Date: Wed, 5 Feb 2020 18:29:36 -0800 Subject: [PATCH 07/10] Handle shutdown signals gracefully I was trying something that didn't work out but we got signal handling out of it --- cmd/rhyzome/main.go | 33 +++++++++++++++++++++++++++++++-- instances/instances.go | 4 ++++ metadata/server.go | 19 ++++++++++++++++++- rest/router.go | 20 +++++++++++++++++++- vault/approle.go | 4 ++-- 5 files changed, 74 insertions(+), 6 deletions(-) diff --git a/cmd/rhyzome/main.go b/cmd/rhyzome/main.go index a1d2bc2..863d93b 100644 --- a/cmd/rhyzome/main.go +++ b/cmd/rhyzome/main.go @@ -1,17 +1,46 @@ package main import ( + "log" + "os" + "os/signal" + "syscall" + "git.callpipe.com/entanglement.garden/rhyzome/config" "git.callpipe.com/entanglement.garden/rhyzome/metadata" "git.callpipe.com/entanglement.garden/rhyzome/rest" "git.callpipe.com/entanglement.garden/rhyzome/vault" ) +var ( + signals = make(chan os.Signal, 1) +) + func main() { + signal.Notify(signals, syscall.SIGINT, syscall.SIGHUP) + config.Load() vault.Load() - // TODO: Some sort of shutdown signal go metadata.ListenAndServe() - rest.ListenAndServe() + go rest.ListenAndServe() + + for { + signal := <-signals + log.Println("Received signal", signal) + switch signal { + case syscall.SIGHUP: + config.Load() + case syscall.SIGINT: + if err := rest.Shutdown(); err != nil { + log.Println("Error shutting down API server", err) + } + + if err := metadata.Shutdown(); err != nil { + log.Println("Error shutting down metadata server", err) + } + + return + } + } } diff --git a/instances/instances.go b/instances/instances.go index 47fc399..327ba87 100644 --- a/instances/instances.go +++ b/instances/instances.go @@ -43,6 +43,7 @@ func GetAll() (instances []Instance, err error) { if err != nil { return []Instance{}, err } + defer conn.Close() domainIDs, err := conn.ListDomains() if err != nil { @@ -97,6 +98,7 @@ func FromDomainID(domainID uint32) (Instance, error) { if err != nil { return Instance{}, err } + defer conn.Close() return FromDomainIDWithConnect(conn, domainID) } @@ -119,6 +121,7 @@ func FromDomainUUID(domainUUID string) (Instance, error) { if err != nil { return Instance{}, err } + defer conn.Close() dom, err := conn.LookupDomainByUUIDString(domainUUID) if err != nil { @@ -140,6 +143,7 @@ func (i *Instance) CreateAsJob(j *jobs.Job) error { if err != nil { return err } + defer conn.Close() // Create volume for new instance j.SetStatus("Creating volume") diff --git a/metadata/server.go b/metadata/server.go index f3913a6..9257ff7 100644 --- a/metadata/server.go +++ b/metadata/server.go @@ -18,6 +18,10 @@ import ( "git.callpipe.com/entanglement.garden/rhyzome/vault" ) +var ( + server *http.Server +) + func ListenAndServe() { r := mux.NewRouter() r.Use(utils.LoggingMiddleware) @@ -26,7 +30,15 @@ func ListenAndServe() { r.HandleFunc("/guest-info", guestInfo) r.HandleFunc("/guest-get-osinfo", guestGetOSInfo) r.HandleFunc("/vault/inject-app-role", vault.InjectAppRole) - log.Fatal(http.ListenAndServe(config.C.MetadataBind, r)) + server = &http.Server{ + Addr: config.C.MetadataBind, + Handler: r, + } + log.Println("Starting metadata server on", server.Addr) + err := server.ListenAndServe() + if err != http.ErrServerClosed { + panic(err) + } } func lookupRequester(next http.Handler) http.Handler { @@ -106,3 +118,8 @@ func index(w http.ResponseWriter, r *http.Request) { domain := ctx.Value(utils.InstanceDomainXMLContextKey).(libvirtxml.Domain) json.NewEncoder(w).Encode(domain) } + +func Shutdown() error { + log.Println("Shutting down metadata server") + return server.Shutdown(context.Background()) +} diff --git a/rest/router.go b/rest/router.go index d659634..398560c 100644 --- a/rest/router.go +++ b/rest/router.go @@ -1,6 +1,7 @@ package rest import ( + "context" "encoding/json" "log" "net/http" @@ -11,6 +12,10 @@ import ( "git.callpipe.com/entanglement.garden/rhyzome/utils" ) +var ( + server *http.Server +) + func ListenAndServe() { r := mux.NewRouter() r.Use(utils.LoggingMiddleware) @@ -23,7 +28,15 @@ func ListenAndServe() { r.HandleFunc("/api/v1alpha1/jobs/{id}", GetJob) r.Use(loggingMiddleware) - log.Fatal(http.ListenAndServe(config.C.Bind, r)) + server = &http.Server{ + Addr: config.C.Bind, + Handler: r, + } + log.Println("Starting API server on", server.Addr) + err := server.ListenAndServe() + if err != http.ErrServerClosed { + panic(err) + } } func renderRoot(w http.ResponseWriter, r *http.Request) { @@ -37,3 +50,8 @@ func loggingMiddleware(next http.Handler) http.Handler { log.Println(r.RemoteAddr, r.Method, r.RequestURI) }) } + +func Shutdown() error { + log.Println("Shutting down API server") + return server.Shutdown(context.Background()) +} diff --git a/vault/approle.go b/vault/approle.go index aaca71a..a84a3a8 100644 --- a/vault/approle.go +++ b/vault/approle.go @@ -56,7 +56,7 @@ func InjectAppRole(w http.ResponseWriter, r *http.Request) { return } - err = WriteSecret(domain, "/ec/entanglement/auth/role-id", roleIDstring) + err = WriteSecret(domain, "/etc/entanglement/auth/role-id", roleIDstring) if err != nil { log.Println("Error writing role-id file to guest", err) http.Error(w, err.Error(), 500) @@ -86,7 +86,7 @@ func InjectAppRole(w http.ResponseWriter, r *http.Request) { return } - err = WriteSecret(domain, "/ec/entanglement/auth/secret-id", secretIDstring) + err = WriteSecret(domain, "/etc/entanglement/auth/secret-id", secretIDstring) if err != nil { log.Println("Error writing secret-id file to guest", err) http.Error(w, err.Error(), 500) -- GitLab From 1467866a4336cf3da47d706f0840b778cf1a35a2 Mon Sep 17 00:00:00 2001 From: Finn <finn@entanglement.garden> Date: Wed, 12 Feb 2020 20:27:39 -0800 Subject: [PATCH 08/10] Create auth directory before writing files there also make things more configurable --- config/config.go | 31 ++++++++++++++++++++----------- vault/approle.go | 20 ++++++++++++++------ 2 files changed, 34 insertions(+), 17 deletions(-) diff --git a/config/config.go b/config/config.go index 0067018..f5de331 100644 --- a/config/config.go +++ b/config/config.go @@ -27,13 +27,18 @@ type Config struct { } type VaultConfig struct { - TLSConfig api.TLSConfig // TLSConfig is the TLS configuration to use, if any - Address string - InjectionCommand string - CommandPollMS int - CommandTimeoutSeconds libvirt.DomainQemuAgentCommandTimeout - RoleIDFilePath string - SecretIDFilePath string + TLSConfig api.TLSConfig // TLSConfig is the TLS configuration to use, if any + Address string + InjectionCommand string + CommandPollMS int + GuestCommandTimeoutSeconds libvirt.DomainQemuAgentCommandTimeout + GuestAuthFolder string + RoleIDFilePath string + SecretIDFilePath string + SecretIDTTL string + TokenPolicies []string + SecretIDNumUses string + TokenMaxTTL string } var ( @@ -51,10 +56,14 @@ var ( ImageOwner: "64055", MetadataBind: "127.0.0.1:8081", Vault: VaultConfig{ - TLSConfig: api.TLSConfig{}, - InjectionCommand: "/opt/entanglement/vault-inject.sh", - CommandPollMS: 250, - CommandTimeoutSeconds: 5, + TLSConfig: api.TLSConfig{}, + InjectionCommand: "/opt/entanglement/vault-inject.sh", + GuestAuthFolder: "/etc/entanglement/auth", + CommandPollMS: 250, + GuestCommandTimeoutSeconds: 5, + SecretIDTTL: "1h", + SecretIDNumUses: "1", + TokenMaxTTL: "24h", }, } ) diff --git a/vault/approle.go b/vault/approle.go index a84a3a8..dde4f3f 100644 --- a/vault/approle.go +++ b/vault/approle.go @@ -6,6 +6,7 @@ import ( "fmt" "log" "net/http" + "strings" "time" libvirt "github.com/libvirt/libvirt-go" @@ -27,13 +28,20 @@ func InjectAppRole(w http.ResponseWriter, r *http.Request) { return } + _, err = GuestExec(domain, "mkdir", "-p", config.C.Vault.GuestAuthFolder) + if err != nil { + log.Println("Error creating", config.C.Vault.GuestAuthFolder, "on guest:", err) + http.Error(w, err.Error(), 500) + return + } + // Create the approle appRolePath := fmt.Sprintf("auth/approle/role/%s.%s", domainName, config.C.Hostname) _, err = vaultClient.Logical().Write(appRolePath, map[string]interface{}{ - "secret_id_ttl": "1h", - "token_policies": "hypervisors,pki-and-read-own-cert", - "secret_id_num_uses": "1", - "token_max_ttl": "24h", + "secret_id_ttl": config.C.Vault.SecretIDTTL, + "token_policies": strings.Join(config.C.Vault.TokenPolicies, ","), + "secret_id_num_uses": config.C.Vault.SecretIDNumUses, + "token_max_ttl": config.C.Vault.TokenMaxTTL, }) if err != nil { log.Println("Error creating vault role", appRolePath, err) @@ -56,7 +64,7 @@ func InjectAppRole(w http.ResponseWriter, r *http.Request) { return } - err = WriteSecret(domain, "/etc/entanglement/auth/role-id", roleIDstring) + err = WriteSecret(domain, fmt.Sprintf("%s/role-id", config.C.Vault.GuestAuthFolder), roleIDstring) if err != nil { log.Println("Error writing role-id file to guest", err) http.Error(w, err.Error(), 500) @@ -86,7 +94,7 @@ func InjectAppRole(w http.ResponseWriter, r *http.Request) { return } - err = WriteSecret(domain, "/etc/entanglement/auth/secret-id", secretIDstring) + err = WriteSecret(domain, fmt.Sprintf("%s/secret-id", config.C.Vault.GuestAuthFolder), secretIDstring) if err != nil { log.Println("Error writing secret-id file to guest", err) http.Error(w, err.Error(), 500) -- GitLab From 8636db4cc047904123936e126ab1851551462249 Mon Sep 17 00:00:00 2001 From: Finn <finn@entanglement.garden> Date: Thu, 13 Feb 2020 01:08:02 -0800 Subject: [PATCH 09/10] Add explicit cloud-init support to the metadata server --- config/config.go | 4 ++++ instances/instances.go | 6 ++++++ metadata/cloudinit.go | 41 +++++++++++++++++++++++++++++++++++++++++ metadata/server.go | 2 ++ 4 files changed, 53 insertions(+) create mode 100644 metadata/cloudinit.go diff --git a/config/config.go b/config/config.go index f5de331..6e7dc53 100644 --- a/config/config.go +++ b/config/config.go @@ -23,6 +23,8 @@ type Config struct { ImageHost string // ImageHost is the base URL for the disk image server ImageOwner string // ImageOwner is the UID that should own volume images MetadataBind string // MetadataBind is the port (and optionally IP) to bind the metadata server to + CloudInitSeed string // CloudInitSeed is the seed to pass to cloud-init + CloudName string // CloudName is the cloud provider name to tell cloud-init Vault VaultConfig } @@ -55,6 +57,8 @@ var ( ImageHost: "http://image-host.fruit-0.entanglement.garden", ImageOwner: "64055", MetadataBind: "127.0.0.1:8081", + CloudInitSeed: "http://127.0.0.1:8081/cloud-init/", + CloudName: "rhyzome", Vault: VaultConfig{ TLSConfig: api.TLSConfig{}, InjectionCommand: "/opt/entanglement/vault-inject.sh", diff --git a/instances/instances.go b/instances/instances.go index 327ba87..0171330 100644 --- a/instances/instances.go +++ b/instances/instances.go @@ -198,6 +198,12 @@ func (i *Instance) CreateAsJob(j *jobs.Job) error { Graphics: []libvirtxml.DomainGraphic{libvirtxml.DomainGraphic{VNC: &libvirtxml.DomainGraphicVNC{}}}, Videos: []libvirtxml.DomainVideo{libvirtxml.DomainVideo{Model: libvirtxml.DomainVideoModel{Type: "virtio"}}}, }, + QEMUCommandline: &libvirtxml.DomainQEMUCommandline{ + Args: []libvirtxml.DomainQEMUCommandlineArg{ + libvirtxml.DomainQEMUCommandlineArg{Value: "-smbios"}, + libvirtxml.DomainQEMUCommandlineArg{Value: "type=1,serial=ds=nocloud-net;s=" + config.C.CloudInitSeed}, + }, + }, } domainXMLString, err := domainXML.Marshal() diff --git a/metadata/cloudinit.go b/metadata/cloudinit.go new file mode 100644 index 0000000..60d9933 --- /dev/null +++ b/metadata/cloudinit.go @@ -0,0 +1,41 @@ +package metadata + +import ( + "log" + "net/http" + + libvirtxml "github.com/libvirt/libvirt-go-xml" + "gopkg.in/yaml.v2" + + "git.callpipe.com/entanglement.garden/rhyzome/config" + "git.callpipe.com/entanglement.garden/rhyzome/utils" +) + +type CloudInitMetadata struct { + LocalHostname string `yaml:"local-hostname,omitempty"` + InstanceID string `yaml:"instance-id,omitempty"` + CloudName string `yaml:"cloud-name,omitempty"` +} + +func cloudInitUserData(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + domain := ctx.Value(utils.InstanceDomainXMLContextKey).(libvirtxml.Domain) + w.Write([]byte(domain.Description)) +} + +func cloudInitMetaData(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Content-Type", "application/yaml") + type cloudInitMetaDataResponse struct { + } + ctx := r.Context() + domain := ctx.Value(utils.InstanceDomainXMLContextKey).(libvirtxml.Domain) + err := yaml.NewEncoder(w).Encode(CloudInitMetadata{ + LocalHostname: domain.Name, + InstanceID: domain.UUID, + CloudName: config.C.CloudName, + }) + if err != nil { + log.Println("Error encoding response", err) + http.Error(w, err.Error(), 500) + } +} diff --git a/metadata/server.go b/metadata/server.go index 9257ff7..b2b2177 100644 --- a/metadata/server.go +++ b/metadata/server.go @@ -27,6 +27,8 @@ func ListenAndServe() { r.Use(utils.LoggingMiddleware) r.Use(lookupRequester) r.HandleFunc("/", index) + r.HandleFunc("/cloud-init/user-data", cloudInitUserData) + r.HandleFunc("/cloud-init/meta-data", cloudInitMetaData) r.HandleFunc("/guest-info", guestInfo) r.HandleFunc("/guest-get-osinfo", guestGetOSInfo) r.HandleFunc("/vault/inject-app-role", vault.InjectAppRole) -- GitLab From 3abf27a7ba8e0452560a4f8b2a3a84d37b4b0c1c Mon Sep 17 00:00:00 2001 From: Finn Herzfeld <finn@entanglement.garden> Date: Thu, 13 Feb 2020 12:05:34 -0800 Subject: [PATCH 10/10] Rearrange the config a little --- config/config.go | 56 +++++++++++++++++++++++++----------------- instances/instances.go | 2 +- metadata/cloudinit.go | 2 +- vault/approle.go | 20 +++++++-------- 4 files changed, 46 insertions(+), 34 deletions(-) diff --git a/config/config.go b/config/config.go index 6e7dc53..be75d6e 100644 --- a/config/config.go +++ b/config/config.go @@ -14,29 +14,37 @@ var ConfigFiles = []string{"/etc/rhyzome.conf", "rhyzome.conf"} // Config describes all configurable keys type Config struct { - Bind string // Bind is the address (and port) to bind the REST server to - BridgeInterface string // BridgeInterface is the bridge that all network interfaces are added to - DiskStoragePool string // DiskStoragePool is the name of the storage pool to use - Hostname string // Hostname is the domain that guests will be created under - ImageDir string // ImageDir is the path to the local image pool - ImageGroup string // ImageGroup is the GID that should own volume images - ImageHost string // ImageHost is the base URL for the disk image server - ImageOwner string // ImageOwner is the UID that should own volume images - MetadataBind string // MetadataBind is the port (and optionally IP) to bind the metadata server to - CloudInitSeed string // CloudInitSeed is the seed to pass to cloud-init - CloudName string // CloudName is the cloud provider name to tell cloud-init - Vault VaultConfig + Bind string // Bind is the address (and port) to bind the REST server to + BridgeInterface string // BridgeInterface is the bridge that all network interfaces are added to + DiskStoragePool string // DiskStoragePool is the name of the storage pool to use + Hostname string // Hostname is the domain that guests will be created under + ImageDir string // ImageDir is the path to the local image pool + ImageGroup string // ImageGroup is the GID that should own volume images + ImageHost string // ImageHost is the base URL for the disk image server + ImageOwner string // ImageOwner is the UID that should own volume images + MetadataBind string // MetadataBind is the port (and optionally IP) to bind the metadata server to + CloudInit CloudInitConfig // CloudInit contains settings that are specific to the cloud-init integration + Vault VaultConfig // Vault contains settings for connecting to the vault server + CredentialInjection CredentialInjectionConfig // CredentialInjection configures how credentials are injected into guests } type VaultConfig struct { - TLSConfig api.TLSConfig // TLSConfig is the TLS configuration to use, if any - Address string - InjectionCommand string + TLSConfig api.TLSConfig // TLSConfig is the TLS configuration to use, if any + Address string + RoleIDFilePath string + SecretIDFilePath string +} + +type CloudInitConfig struct { + Seed string // Seed is the seed URL to pass to cloud-init via qemu smbios flag. Should be http://<metadata server>/cloud-init/ + CloudName string // CloudName is name of the cloud we tell cloud-init it's running on +} + +type CredentialInjectionConfig struct { + PostInjectionCommand string CommandPollMS int GuestCommandTimeoutSeconds libvirt.DomainQemuAgentCommandTimeout GuestAuthFolder string - RoleIDFilePath string - SecretIDFilePath string SecretIDTTL string TokenPolicies []string SecretIDNumUses string @@ -57,14 +65,18 @@ var ( ImageHost: "http://image-host.fruit-0.entanglement.garden", ImageOwner: "64055", MetadataBind: "127.0.0.1:8081", - CloudInitSeed: "http://127.0.0.1:8081/cloud-init/", - CloudName: "rhyzome", + CloudInit: CloudInitConfig{ + Seed: "http://127.0.0.1:80801/cloud-init/", + CloudName: "rhyzome", + }, Vault: VaultConfig{ - TLSConfig: api.TLSConfig{}, - InjectionCommand: "/opt/entanglement/vault-inject.sh", + TLSConfig: api.TLSConfig{}, + }, + CredentialInjection: CredentialInjectionConfig{ + PostInjectionCommand: "/opt/entanglement/vault-inject.sh", + GuestCommandTimeoutSeconds: 5, GuestAuthFolder: "/etc/entanglement/auth", CommandPollMS: 250, - GuestCommandTimeoutSeconds: 5, SecretIDTTL: "1h", SecretIDNumUses: "1", TokenMaxTTL: "24h", diff --git a/instances/instances.go b/instances/instances.go index 0171330..5e28123 100644 --- a/instances/instances.go +++ b/instances/instances.go @@ -201,7 +201,7 @@ func (i *Instance) CreateAsJob(j *jobs.Job) error { QEMUCommandline: &libvirtxml.DomainQEMUCommandline{ Args: []libvirtxml.DomainQEMUCommandlineArg{ libvirtxml.DomainQEMUCommandlineArg{Value: "-smbios"}, - libvirtxml.DomainQEMUCommandlineArg{Value: "type=1,serial=ds=nocloud-net;s=" + config.C.CloudInitSeed}, + libvirtxml.DomainQEMUCommandlineArg{Value: "type=1,serial=ds=nocloud-net;s=" + config.C.CloudInit.Seed}, }, }, } diff --git a/metadata/cloudinit.go b/metadata/cloudinit.go index 60d9933..c7112ea 100644 --- a/metadata/cloudinit.go +++ b/metadata/cloudinit.go @@ -32,7 +32,7 @@ func cloudInitMetaData(w http.ResponseWriter, r *http.Request) { err := yaml.NewEncoder(w).Encode(CloudInitMetadata{ LocalHostname: domain.Name, InstanceID: domain.UUID, - CloudName: config.C.CloudName, + CloudName: config.C.CloudInit.CloudName, }) if err != nil { log.Println("Error encoding response", err) diff --git a/vault/approle.go b/vault/approle.go index dde4f3f..f9482b2 100644 --- a/vault/approle.go +++ b/vault/approle.go @@ -28,9 +28,9 @@ func InjectAppRole(w http.ResponseWriter, r *http.Request) { return } - _, err = GuestExec(domain, "mkdir", "-p", config.C.Vault.GuestAuthFolder) + _, err = GuestExec(domain, "mkdir", "-p", config.C.CredentialInjection.GuestAuthFolder) if err != nil { - log.Println("Error creating", config.C.Vault.GuestAuthFolder, "on guest:", err) + log.Println("Error creating", config.C.CredentialInjection.GuestAuthFolder, "on guest:", err) http.Error(w, err.Error(), 500) return } @@ -38,10 +38,10 @@ func InjectAppRole(w http.ResponseWriter, r *http.Request) { // Create the approle appRolePath := fmt.Sprintf("auth/approle/role/%s.%s", domainName, config.C.Hostname) _, err = vaultClient.Logical().Write(appRolePath, map[string]interface{}{ - "secret_id_ttl": config.C.Vault.SecretIDTTL, - "token_policies": strings.Join(config.C.Vault.TokenPolicies, ","), - "secret_id_num_uses": config.C.Vault.SecretIDNumUses, - "token_max_ttl": config.C.Vault.TokenMaxTTL, + "secret_id_ttl": config.C.CredentialInjection.SecretIDTTL, + "token_policies": strings.Join(config.C.CredentialInjection.TokenPolicies, ","), + "secret_id_num_uses": config.C.CredentialInjection.SecretIDNumUses, + "token_max_ttl": config.C.CredentialInjection.TokenMaxTTL, }) if err != nil { log.Println("Error creating vault role", appRolePath, err) @@ -64,7 +64,7 @@ func InjectAppRole(w http.ResponseWriter, r *http.Request) { return } - err = WriteSecret(domain, fmt.Sprintf("%s/role-id", config.C.Vault.GuestAuthFolder), roleIDstring) + err = WriteSecret(domain, fmt.Sprintf("%s/role-id", config.C.CredentialInjection.GuestAuthFolder), roleIDstring) if err != nil { log.Println("Error writing role-id file to guest", err) http.Error(w, err.Error(), 500) @@ -94,14 +94,14 @@ func InjectAppRole(w http.ResponseWriter, r *http.Request) { return } - err = WriteSecret(domain, fmt.Sprintf("%s/secret-id", config.C.Vault.GuestAuthFolder), secretIDstring) + err = WriteSecret(domain, fmt.Sprintf("%s/secret-id", config.C.CredentialInjection.GuestAuthFolder), secretIDstring) if err != nil { log.Println("Error writing secret-id file to guest", err) http.Error(w, err.Error(), 500) return } - statusResponse, err := GuestExec(domain, config.C.Vault.InjectionCommand, domainName) + statusResponse, err := GuestExec(domain, config.C.CredentialInjection.PostInjectionCommand, domainName) if err != nil { http.Error(w, err.Error(), 500) return @@ -164,7 +164,7 @@ func GuestExec(domain *libvirt.Domain, path string, args ...string) (response qa break } - time.Sleep(time.Duration(config.C.Vault.CommandPollMS) * time.Millisecond) + time.Sleep(time.Duration(config.C.CredentialInjection.CommandPollMS) * time.Millisecond) } return -- GitLab