Feature/tls nomad endpoint (#80)

* Add nomad TLS arguments and config. Signed-off-by: Andrew Cornies <[email protected]> * Fix spelling for argument. Signed-off-by: Andrew Cornies <[email protected]> * Nomad TLS endpoint update: - Updates to local Vagrant env - use Vault pki to generate local certs - add enable_nomad_tls arg - fix error handling issue Signed-off-by: Andrew Cornies <[email protected]>
hashicorp · Apr 20, 2019 · e516e7a · e516e7a
1 parent dbed1bb
commit e516e7a
Show file tree

Hide file tree

Showing 7 changed files with 98 additions and 33 deletions.
diff --git a/Vagrantfile b/Vagrantfile
@@ -82,7 +82,6 @@ Vagrant.configure("2") do |config|
       salt.verbose = true
       salt.salt_call_args = ["saltenv=dev", "pillarenv=dev"]
     end
-    override.vm.provision "shell", path: "provisioning/scripts/vault_populate.sh"
     override.vm.provision "shell", path: "provisioning/scripts/nomad_run.sh"
   end
 
@@ -97,14 +96,15 @@ Vagrant.configure("2") do |config|
       salt.verbose = true
       salt.salt_call_args = ["saltenv=dev", "pillarenv=dev"]
     end
-    override.vm.provision "shell", path: "provisioning/scripts/vault_populate.sh"
     override.vm.provision "shell", path: "provisioning/scripts/nomad_run.sh"
   end
 
   config.vm.provision :docker do |d|
     d.run 'dev-vault', image: 'vault:0.9.6', 
       args: '-p 8200:8200 -e "VAULT_DEV_ROOT_TOKEN_ID=vagrant" -v /vagrant:/vagrant'
   end
+
+  config.vm.provision "shell", path: "provisioning/scripts/vault_populate.sh"
   #
   # View the documentation for the provider you are using for more
   # information on available options.
@@ -120,7 +120,7 @@ Vagrant.configure("2") do |config|
     cd /vagrant/provisioning/saltstack/formulas
     git clone https://github.com/saltstack-formulas/nomad-formula.git
     git clone https://github.com/saltstack-formulas/consul-formula.git
-    git clone https://github.com/saltstack-formulas/vault-formula.git
+    # git clone https://github.com/saltstack-formulas/vault-formula.git
   SHELL
 
   # salt

diff --git a/main.go b/main.go
@@ -31,6 +31,11 @@ var (
 	statsdServer          = flag.String("statsd_addr", "localhost:8125", "Location for the statsd collector")
 	nodeURI               = flag.String("node_addr", "localhost", "URI of the current Nomad node, this address is used for reporting and logging")
 	nomadAddr             = flag.String("nomad_addr", "localhost:4646", "Address for Nomad API endpoint")
+	nomadTLSCA            = flag.String("nomad_tls_ca", "", "The TLS ca certificate file location")
+	nomadTLSCert          = flag.String("nomad_tls_cert", "", "The TLS client certifcate file location")
+	nomadTLSKey           = flag.String("nomad_tls_key", "", "The TLS private key file location")
+	nomadTLSSkipVerify    = flag.Bool("nomad_tls_skip_verify", false, "Skips TLS verification for Nomad API. Not recommend for production")
+	enableNomadTLS        = flag.Bool("enable_nomad_tls", false, "Toggles tls on Nomad endpoint/client")
 	nomadACL              = flag.String("nomad_acl", "", "The ACL token for faas-nomad if Nomad ACLs are enabled")
 	consulAddr            = flag.String("consul_addr", "http://localhost:8500", "Address for Consul API endpoint")
 	consulACL             = flag.String("consul_acl", "", "ACL token for Consul API, only required if ACL are enabled in Consul")
@@ -57,11 +62,19 @@ var (
 func main() {
 	flag.Parse()
 
+	nomadConfig := &fntypes.NomadConfig{
+		TLSEnabled:    *enableNomadTLS,
+		Address:       *nomadAddr,
+		ACLToken:      *nomadACL,
+		TLSCA:         *nomadTLSCA,
+		TLSCert:       *nomadTLSCert,
+		TLSPrivateKey: *nomadTLSKey,
+		TLSSkipVerify: *nomadTLSSkipVerify,
+	}
 	logger, stats, nomadClient, consulResolver := makeDependencies(
 		*statsdServer,
 		*nodeURI,
-		*nomadAddr,
-		*nomadACL,
+		*nomadConfig,
 		*consulAddr,
 		*consulACL,
 		*nomadRegion,
@@ -90,19 +103,20 @@ func createFaaSHandlers(nomadClient *api.Client, consulResolver *consul.Resolver
 
 	datacenter, err := nomadClient.Agent().Datacenter()
 	if err != nil {
-		logger.Error("Error returning the agent's datacenter", err)
+		logger.Error("Error returning the agent's datacenter", err.Error())
 		datacenter = "dc1"
 	}
 	logger.Info("Datacenter from agent: " + datacenter)
 
 	agentSelf, err := nomadClient.Agent().Self()
-	if err != nil {
-		logger.Error("/agent/self returned error. Unable to fetch Vault config.", err)
-	}
 	var vaultConfig fntypes.VaultConfig
-	mapstructure.Decode(agentSelf.Config["Vault"], &vaultConfig)
-	if len(*vaultAddrOverride) > 0 {
-		vaultConfig.Addr = *vaultAddrOverride
+	if err != nil {
+		logger.Error("/agent/self returned error. Unable to fetch Vault config.", err.Error())
+	} else {
+		mapstructure.Decode(agentSelf.Config["Vault"], &vaultConfig)
+		if len(*vaultAddrOverride) > 0 {
+			vaultConfig.Addr = *vaultAddrOverride
+		}
 	}
 
 	logger.Info("Vault address: " + vaultConfig.Addr)
@@ -142,7 +156,7 @@ func createFaaSHandlers(nomadClient *api.Client, consulResolver *consul.Resolver
 	}
 }
 
-func makeDependencies(statsDAddr, thisAddr, nomadAddr, nomadACL, consulAddr, consulACL, region string) (hclog.Logger, *statsd.Client, *api.Client, *consul.Resolver) {
+func makeDependencies(statsDAddr string, thisAddr string, nomadConfig fntypes.NomadConfig, consulAddr string, consulACL string, region string) (hclog.Logger, *statsd.Client, *api.Client, *consul.Resolver) {
 	logger := setupLogging()
 
 	logger.Info("Using StatsD server:" + statsDAddr)
@@ -156,9 +170,19 @@ func makeDependencies(statsDAddr, thisAddr, nomadAddr, nomadACL, consulAddr, con
 	stats.Tags = append(stats.Tags, "instance:"+strings.Replace(thisAddr, ":", "_", -1))
 
 	c := api.DefaultConfig()
-	logger.Info("create nomad client", "addr", nomadAddr)
-	clientConfig := c.ClientConfig(region, nomadAddr, false)
-	clientConfig.SecretID = nomadACL
+	logger.Info("create nomad client", "addr", nomadConfig.Address)
+	clientConfig := c.ClientConfig(region, nomadConfig.Address, nomadConfig.TLSEnabled)
+	clientConfig.SecretID = nomadConfig.ACLToken
+	if nomadConfig.TLSEnabled {
+		clientConfig.TLSConfig = &api.TLSConfig{
+			CACert:     nomadConfig.TLSCA,
+			ClientCert: nomadConfig.TLSCert,
+			ClientKey:  nomadConfig.TLSPrivateKey,
+			Insecure:   nomadConfig.TLSSkipVerify,
+		}
+		clientConfig.ConfigureTLS()
+	}
+
 	nomadClient, err := api.NewClient(clientConfig)
 	if err != nil {
 		logger.Error("Unable to create nomad client", err)

diff --git a/provisioning/saltstack/pillar/dev/init.sls b/provisioning/saltstack/pillar/dev/init.sls
@@ -5,6 +5,11 @@ include:
 nomad:
   config:
     datacenter: dc1
+    tls:
+      http: True
+      ca_file: /home/vagrant/placeholder-ca.crt
+      cert_file: /home/vagrant/placeholder.crt
+      key_file: /home/vagrant/placeholder.key
     advertise:
       http: {{ interface_address }}
     server:
@@ -38,6 +43,14 @@ nomad:
     - dc1
 consul:
   config:
+    connect:
+      enabled: True
+      ca_provider: vault
+      ca_config:
+        address: "http://127.0.0.1:8200"
+        token: vagrant
+        root_pki_path: pki
+        intermediate_pki_path: pki_int
     server: True
     advertise_addr: {{ interface_address }}
     addresses:
@@ -49,16 +62,4 @@ consul:
     enable_debug: True
     datacenter: dc1
     encrypt: "RIxqpNlOXqtr/j4BgvIMEw=="
-    bootstrap: true
-vault:
-  listen_protocol: tcp
-  listen_port: 8200
-  listen_address: 0.0.0.0
-  tls_disable: 0
-  default_lease_ttl: 24h
-  max_lease_ttl: 24h
-  self_signed_cert:
-    enabled: false
-  backend: {}
-  dev_mode: true
-  dev_root_token: vagrant
+    bootstrap: true
diff --git a/provisioning/saltstack/salt/base/init.sls b/provisioning/saltstack/salt/base/init.sls
@@ -1,4 +1,3 @@
 include:
   - consul
-  - vault
   - nomad
diff --git a/provisioning/scripts/nomad_run.sh b/provisioning/scripts/nomad_run.sh
@@ -9,6 +9,11 @@ do
     sleep 2
   fi
 done
+
+export NOMAD_ADDR=https://192.168.50.2:4646
+export NOMAD_CACERT=/home/vagrant/placeholder-ca.crt
+export NOMAD_CLIENT_CERT=/home/vagrant/placeholder.crt
+export NOMAD_CLIENT_KEY=/home/vagrant/placeholder.key
 echo 'Waiting for nomad...'
 while true
 do

diff --git a/provisioning/scripts/vault_populate.sh b/provisioning/scripts/vault_populate.sh
@@ -19,7 +19,6 @@ export VAULT_ADDR=${VAULT_URL}
 export VAULT_TOKEN=${TOKEN}
 
 vault auth enable approle
-
 vault policy write ${POLICY_NAME} /vagrant/provisioning/scripts/policy.hcl
 
 # create approle openfaas
@@ -31,9 +30,35 @@ curl -i \
 
 curl -i \
   --header "X-Vault-Token: ${TOKEN}" \
-  ${VAULT_URL}/v1/auth/approle/role/${POLICY_NAME}/role-id
+  ${VAULT_URL}/v1/auth/approle/role/${POLICY_NAME}/role-id -o ./role_id.json
 
 curl -i \
   --header "X-Vault-Token: ${TOKEN}" \
   --request POST \
-  ${VAULT_URL}/v1/auth/approle/role/${POLICY_NAME}/secret-id
+  ${VAULT_URL}/v1/auth/approle/role/${POLICY_NAME}/secret-id -o ./secret_id.json
+
+echo 'enabling pki backend...'
+curl -i -H "X-Vault-Token: ${TOKEN}" -H "Content-Type: application/json" \
+  -XPOST -d '{"type":"pki"}' ${VAULT_URL}/v1/sys/mounts/pki
+
+echo 'generate root ca...'
+curl -i -H "X-Vault-Token: ${TOKEN}" -H "Content-Type: application/json" \
+  -XPOST -d '{"common_name":"nomad.local", "ip_sans": "192.168.50.2"}' ${VAULT_URL}/v1/pki/root/generate/internal
+
+echo 'configure issuing urls...'
+curl -i -H "X-Vault-Token: ${TOKEN}" -H "Content-Type: application/json" \
+  -XPOST -d '{"issuing_certificates": ["http://localhost:8200/v1/pki/ca"], "crl_distribution_points": ["http://localhost:8200/v1/pki/crl"]}' ${VAULT_URL}/v1/pki/config/urls
+
+echo 'create role...'
+curl -i -H "X-Vault-Token: ${TOKEN}" -H "Content-Type: application/json" \
+  -XPOST -d '{"allowed_domains": ["nomad.local"], "allow_subdomains": true, "max_ttl": "72h"}' ${VAULT_URL}/v1/pki/roles/faas-nomad
+
+echo 'get certficates...'
+curl -H "X-Vault-Token: ${TOKEN}" -H "Content-Type: application/json" \
+  -XPOST -d '{"common_name": "server.nomad.local", "ip_sans": "192.168.50.2"}' ${VAULT_URL}/v1/pki/issue/faas-nomad -o ./output.json
+
+apt-get install jq -y
+
+jq -r '.data.issuing_ca' < ./output.json > ./placeholder-ca.crt
+jq -r '.data.certificate' < ./output.json > ./placeholder.crt
+jq -r '.data.private_key' < ./output.json > ./placeholder.key
diff --git a/types/nomad_config.go b/types/nomad_config.go
@@ -0,0 +1,11 @@
+package types
+
+type NomadConfig struct {
+	TLSEnabled    bool
+	Address       string
+	ACLToken      string
+	TLSCA         string
+	TLSCert       string
+	TLSPrivateKey string
+	TLSSkipVerify bool
+}