Merge pull request #46 from acornies/feature/vagrant

basic auth + vagrant environment for integration testing

.gitignore

@@ -23,5 +23,6 @@ consul.hcl
 nomad.hcl
 
 .vagrant
-debug
+provisioning/saltstack/formulas*
 ubuntu*.log
+debug

.vscode/launch.json

@@ -18,7 +18,9 @@
         "-nomad_region", "global",
         "-nomad_addr", "192.168.50.2:4646",
         "-consul_addr", "192.168.50.2:8500",
-        "-statsd_addr", "192.168.50.2:9125"
+        "-statsd_addr", "192.168.50.2:9125",
+        "-basic_auth_secret_path", "provisioning/secrets",
+        "-enable_basic_auth=true"
       ],
       "showLog": true
     }

Gopkg.toml

@@ -4,7 +4,7 @@
 
 [[constraint]]
   name = "github.com/openfaas/faas-provider"
-  version = "0.6.0"
+  version = "0.7.1"
 
 [[constraint]]
   name = "github.com/hashicorp/nomad"

README.md

@@ -48,6 +48,22 @@ NOMAD Running
 
 The startup script will set the advertised address to your primary local IP address and run both Nomad and Consul in the background redirecting the logs to your home folder.
 
+## Using Vagrant for Local Development
+Vagrant is a tool for provisioning dev environments. The `Vagrantfile` governs the Vagrant configuration:
+1) Install Vagrant via [download links](https://www.vagrantup.com/downloads.html) or package manager
+2) Install VirtualBox via [download links](https://www.virtualbox.org/wiki/Downloads) or preferred hypervisor of your choice (vagrant plugins may be required). VMWare Fusion is supported.
+3) `vagrant up` (default VirtualBox) or `vagrant up --provider vmware_fusion`
+
+The provisioners install Docker, Nomad, Consul, and Vault (via Saltstack) then launch OpenFaaS components with Nomad. If successful, the following services will be available over the private network (192.168.50.2):
+- Nomad (v0.8.4) 192.168.50.2:4646
+- Consul (v1.2.0) 192.168.50.2:8500
+- Vault (v0.10.4) 192.168.50.2:8200
+- FaaS Gateway (0.8.1) 192.168.50.2:8080
+
+This setup is intended to streamline local development of the faas-nomad provider with a more complete setup of the hashicorp ecosystem. Therefore, it is assumed that the faas-nomad source code is located on your workstation, and or is configured to listen on 0.0.0.0:8080 when debugging/running the Go process. There's a convenient VSCode debug configuration located at: `.vscode/launch.json`. The OpenFaaS gateway env var ends up looking like this in Docker:
+```
+functions_provider_url="http://192.168.50.1:8080/"
+
 ## Starting a remote Nomad / Consul environment
 If you would like to test OpenFaaS running on a cluster in AWS, a Terraform module and instructions can be found here:
 [faas-nomad/terraform at master · hashicorp/faas-nomad · GitHub](https://github.com/hashicorp/faas-nomad/tree/master/terraform)

Vagrantfile

@@ -0,0 +1,145 @@
+# -*- mode: ruby -*-
+# vi: set ft=ruby :
+
+# All Vagrant configuration is done below. The "2" in Vagrant.configure
+# configures the configuration version (we support older styles for
+# backwards compatibility). Please don't change it unless you know what
+# you're doing.
+Vagrant.configure("2") do |config|
+  # The most common configuration options are documented and commented below.
+  # For a complete reference, please see the online documentation at
+  # https://docs.vagrantup.com.
+
+  # Every Vagrant development environment requires a box. You can search for
+  # boxes at https://vagrantcloud.com/search.
+  config.vm.box = "ubuntu/xenial64"
+
+  # Disable automatic box update checking. If you disable this, then
+  # boxes will only be checked for updates when the user runs
+  # `vagrant box outdated`. This is not recommended.
+  # config.vm.box_check_update = false
+
+  # Create a forwarded port mapping which allows access to a specific port
+  # within the machine from a port on the host machine. In the example below,
+  # accessing "localhost:8080" will access port 80 on the guest machine.
+  # NOTE: This will enable public access to the opened port
+  # config.vm.network "forwarded_port", guest: 80, host: 8080
+
+  # Create a forwarded port mapping which allows access to a specific port
+  # within the machine from a port on the host machine and only allow access
+  # via 127.0.0.1 to disable public access
+  # config.vm.network "forwarded_port", guest: 8500, host: 8500
+
+  # Create a private network, which allows host-only access to the machine
+  # using a specific IP.
+  config.vm.network "private_network", ip: "192.168.50.2"
+
+  # Create a public network, which generally matched to bridged network.
+  # Bridged networks make the machine appear as another physical device on
+  # your network.
+  # config.vm.network "public_network"
+
+  # Share an additional folder to the guest VM. The first argument is
+  # the path on the host to the actual folder. The second argument is
+  # the path on the guest to mount the folder. And the optional third
+  # argument is a set of non-required options.
+  # config.vm.synced_folder "../data", "/vagrant_data"
+
+  ## For masterless, mount your salt file root
+  # config.vm.synced_folder "salt/roots/", "/srv/salt/"
+
+  config.vm.synced_folder "./provisioning", "/vagrant/provisioning"
+
+  # Provider-specific configuration so you can fine-tune various
+  # backing providers for Vagrant. These expose provider-specific options.
+  # Example for VirtualBox:
+  #
+  config.vm.provider "virtualbox" do |vb, override|
+    # Display the VirtualBox GUI when booting the machine
+    # vb.gui = false
+
+    # Customize the amount of memory on the VM:
+    vb.memory = "2048"
+    vb.cpus = 2
+    override.vm.provision :salt do |salt|
+      salt.minion_config = "provisioning/saltstack/etc/minion_virtualbox.yml"
+      salt.run_highstate = true
+      salt.verbose = true
+      salt.salt_call_args = ["saltenv=dev", "pillarenv=dev"]
+    end
+    override.vm.provision "shell", path: "provisioning/scripts/nomad_run.sh"
+  end
+
+  # vmware fusion
+  config.vm.provider "vmware_fusion" do |vmwf, override|
+    override.vm.box = "generic/ubuntu1604"
+    vmwf.memory = "2048"
+    vmwf.cpus = 2
+    override.vm.provision :salt do |salt|
+      salt.minion_config = "provisioning/saltstack/etc/minion_vmware.yml"
+      salt.run_highstate = true
+      salt.verbose = true
+      salt.salt_call_args = ["saltenv=dev", "pillarenv=dev"]
+    end
+    override.vm.provision "shell", path: "provisioning/scripts/nomad_run.sh"
+  end
+
+  # libvirt
+  config.vm.provider "libvirt" do |lv, override|
+    override.vm.box = "generic/ubuntu1604"
+    lv.memory = "2048"
+    lv.cpus = 2
+    override.vm.provision :salt do |salt|
+      salt.minion_config = "provisioning/saltstack/etc/minion_libvirt.yml"
+      salt.run_highstate = true
+      salt.verbose = true
+      salt.salt_call_args = ["saltenv=dev", "pillarenv=dev"]
+    end
+    override.vm.provision "shell", path: "provisioning/scripts/nomad_run.sh"
+  end
+  #
+  # View the documentation for the provider you are using for more
+  # information on available options.
+
+  # Enable provisioning with a shell script. Additional provisioners such as
+  # Puppet, Chef, Ansible, Salt, and Docker are also available. Please see the
+  # documentation for more information about their specific syntax and use.
+
+  # add dependent git forumlas
+  config.vm.provision "shell", inline: <<-SHELL
+    rm -r /vagrant/provisioning/saltstack/formulas
+    mkdir -p /vagrant/provisioning/saltstack/formulas
+    cd /vagrant/provisioning/saltstack/formulas
+    git clone https://github.com/tucowsinc/nomad-formula.git
+    git clone https://github.com/tucowsinc/consul-formula.git
+    git clone https://github.com/tucowsinc/docker-formula.git
+    git clone https://github.com/tucowsinc/vault-formula.git
+  SHELL
+
+  # salt
+  config.vm.provision :salt do |salt|
+
+    # Relative location of configuration file to use for minion
+    # since we need to tell our minion to run in masterless mode
+    # salt.minion_config = "provisioning/saltstack/etc/minion.yml"
+
+    # On provision, run state.highstate (which installs packages, services, etc).
+    # Highstate basicly means "comapre the VMs current machine state against 
+    # what it should be and make changes if necessary".
+    # salt.run_highstate = true
+
+    # What version of salt to install, and from where.
+    # Because by default it will install the latest, its better to explicetly
+    # choose when to upgrade what version of salt to use.
+
+    # I also prefer to install from git so I can specify a version.
+    salt.install_type = "git"
+    salt.install_args = "v2018.3.2"
+
+    # Run in verbose mode, so it will output all debug info to the console.
+    # This is nice to have when you are testing things out. Once you know they
+    # work well you can comment this line out.
+    # salt.verbose = true
+    # salt.salt_call_args = ["saltenv=dev", "pillarenv=dev"]
+  end
+end

main.go

@@ -2,6 +2,7 @@ package main
 
 import (
 	"flag"
+	"fmt"
 	"log"
 	"net/http"
 	"os"
@@ -23,12 +24,14 @@ import (
 var version = "notset"
 
 var (
-	port         = flag.Int("port", 8080, "Port to bind the server to")
-	statsdServer = flag.String("statsd_addr", "localhost:8125", "Location for the statsd collector")
-	nodeURI      = flag.String("node_addr", "localhost", "URI of the current Nomad node, this address is used for reporting and logging")
-	nomadAddr    = flag.String("nomad_addr", "localhost:4646", "Address for Nomad API endpoint")
-	consulAddr   = flag.String("consul_addr", "http://localhost:8500", "Address for Consul API endpoint")
-	nomadRegion  = flag.String("nomad_region", "global", "Default region to schedule functions in")
+	port                = flag.Int("port", 8080, "Port to bind the server to")
+	statsdServer        = flag.String("statsd_addr", "localhost:8125", "Location for the statsd collector")
+	nodeURI             = flag.String("node_addr", "localhost", "URI of the current Nomad node, this address is used for reporting and logging")
+	nomadAddr           = flag.String("nomad_addr", "localhost:4646", "Address for Nomad API endpoint")
+	consulAddr          = flag.String("consul_addr", "http://localhost:8500", "Address for Consul API endpoint")
+	nomadRegion         = flag.String("nomad_region", "global", "Default region to schedule functions in")
+	enableBasicAuth     = flag.Bool("enable_basic_auth", false, "Flag for enabling basic authentication on gateway endpoints")
+	basicAuthSecretPath = flag.String("basic_auth_secret_path", "/secrets", "The directory path to the basic auth secret file")
 )
 
 var functionTimeout = flag.Duration("function_timeout", 30*time.Second, "Timeout for function execution")
@@ -130,8 +133,12 @@ func main() {
 	config.WriteTimeout = *functionTimeout
 	config.TCPPort = port
 	config.EnableHealth = true
+	config.EnableBasicAuth = *enableBasicAuth
+	config.SecretMountPath = *basicAuthSecretPath
 
 	logger.Info("Started Nomad provider", "port", *config.TCPPort)
+	logger.Info("Basic authentication", "enabled", fmt.Sprintf("%t", config.EnableBasicAuth))
+
 	bootstrap.Serve(handlers, config)
 }
 

nomad_job_files/faas.hcl

@@ -23,20 +23,37 @@ job "faas-nomadd" {
       driver = "docker"
 
       config {
-        image = "quay.io/nicholasjackson/faas-nomad:v0.2.27"
+        image = "quay.io/nicholasjackson/faas-nomad:v0.2.28"
 
         args = [
           "-nomad_region", "${NOMAD_REGION}",
           "-nomad_addr", "${NOMAD_IP_http}:4646",
           "-consul_addr", "${NOMAD_IP_http}:8500",
           "-statsd_addr", "${NOMAD_ADDR_statsd_statsd}",
-          "-node_addr", "${NOMAD_IP_http}"
+          "-node_addr", "${NOMAD_IP_http}",
+          "-basic_auth_secret_path", "/secrets",
+          "-enable_basic_auth=false"
         ]
 
         port_map {
           http = 8080
         }
       }
+      // basic auth from vault example
+      // update -enable_basic_auth=true
+      // uncomment below if you have a Vault instance connected to Nomad
+//       template {
+//         destination   = "secrets/basic-auth-user"
+//         data = <<EOH
+// {{ with secret "secret/openfaas/auth/credentials" }}{{ .Data.username }}{{ end }}
+// EOH
+//       }
+//       template {
+//         destination   = "secrets/basic-auth-password"
+//         data = <<EOH
+// {{ with secret "secret/openfaas/auth/credentials" }}{{ .Data.password }}{{ end }}
+// EOH
+//       }
 
       resources {
         cpu    = 500 # 500 MHz
@@ -76,7 +93,7 @@ EOH
       }
 
       config {
-        image = "functions/gateway:0.8.1"
+        image = "openfaas/gateway:0.8.12"
 
         port_map {
           http = 8080

provisioning/saltstack/etc/minion_libvirt.yml

@@ -0,0 +1,20 @@
+file_client: local
+
+file_roots:
+  base:
+    - /vagrant/provisioning/saltstack/salt
+  dev:
+    - /vagrant/provisioning/saltstack/salt
+    - /vagrant/provisioning/saltstack/formulas/nomad-formula
+    - /vagrant/provisioning/saltstack/formulas/consul-formula
+    - /vagrant/provisioning/saltstack/formulas/docker-formula
+    - /vagrant/provisioning/saltstack/formulas/vault-formula
+
+pillar_roots:
+  base:
+    - /vagrant/provisioning/saltstack/pillar
+  dev:
+    - /vagrant/provisioning/saltstack/pillar
+
+grains:
+  provider: libvirt

provisioning/saltstack/etc/minion_virtualbox.yml

@@ -0,0 +1,20 @@
+file_client: local
+
+file_roots:
+  base:
+    - /vagrant/provisioning/saltstack/salt
+  dev:
+    - /vagrant/provisioning/saltstack/salt
+    - /vagrant/provisioning/saltstack/formulas/nomad-formula
+    - /vagrant/provisioning/saltstack/formulas/consul-formula
+    - /vagrant/provisioning/saltstack/formulas/docker-formula
+    - /vagrant/provisioning/saltstack/formulas/vault-formula
+
+pillar_roots:
+  base:
+    - /vagrant/provisioning/saltstack/pillar
+  dev:
+    - /vagrant/provisioning/saltstack/pillar
+
+grains:
+  provider: virtualbox

provisioning/saltstack/etc/minion_vmware.yml

@@ -0,0 +1,20 @@
+file_client: local
+
+file_roots:
+  base:
+    - /vagrant/provisioning/saltstack/salt
+  dev:
+    - /vagrant/provisioning/saltstack/salt
+    - /vagrant/provisioning/saltstack/formulas/nomad-formula
+    - /vagrant/provisioning/saltstack/formulas/consul-formula
+    - /vagrant/provisioning/saltstack/formulas/docker-formula
+    - /vagrant/provisioning/saltstack/formulas/vault-formula
+
+pillar_roots:
+  base:
+    - /vagrant/provisioning/saltstack/pillar
+  dev:
+    - /vagrant/provisioning/saltstack/pillar
+
+grains:
+  provider: vmware

provisioning/saltstack/pillar/base/init.sls

@@ -0,0 +1,12 @@
+nomad:
+  arch: amd64
+  version: 0.8.4
+  service_hash: fdc512a9e7e6a55303f12d4405676ec7c8116af6
+consul:
+  service: True
+  version: 1.2.0
+  user: root
+vault:
+  version: 0.10.4
+  service:
+    type: systemd