Upload 176 files

.gitattributes

@@ -1,35 +1,2 @@
-*.7z filter=lfs diff=lfs merge=lfs -text
-*.arrow filter=lfs diff=lfs merge=lfs -text
-*.bin filter=lfs diff=lfs merge=lfs -text
-*.bz2 filter=lfs diff=lfs merge=lfs -text
-*.ckpt filter=lfs diff=lfs merge=lfs -text
-*.ftz filter=lfs diff=lfs merge=lfs -text
-*.gz filter=lfs diff=lfs merge=lfs -text
-*.h5 filter=lfs diff=lfs merge=lfs -text
-*.joblib filter=lfs diff=lfs merge=lfs -text
-*.lfs.* filter=lfs diff=lfs merge=lfs -text
-*.mlmodel filter=lfs diff=lfs merge=lfs -text
-*.model filter=lfs diff=lfs merge=lfs -text
-*.msgpack filter=lfs diff=lfs merge=lfs -text
-*.npy filter=lfs diff=lfs merge=lfs -text
-*.npz filter=lfs diff=lfs merge=lfs -text
-*.onnx filter=lfs diff=lfs merge=lfs -text
-*.ot filter=lfs diff=lfs merge=lfs -text
-*.parquet filter=lfs diff=lfs merge=lfs -text
-*.pb filter=lfs diff=lfs merge=lfs -text
-*.pickle filter=lfs diff=lfs merge=lfs -text
-*.pkl filter=lfs diff=lfs merge=lfs -text
-*.pt filter=lfs diff=lfs merge=lfs -text
-*.pth filter=lfs diff=lfs merge=lfs -text
-*.rar filter=lfs diff=lfs merge=lfs -text
-*.safetensors filter=lfs diff=lfs merge=lfs -text
-saved_model/**/* filter=lfs diff=lfs merge=lfs -text
-*.tar.* filter=lfs diff=lfs merge=lfs -text
-*.tar filter=lfs diff=lfs merge=lfs -text
-*.tflite filter=lfs diff=lfs merge=lfs -text
-*.tgz filter=lfs diff=lfs merge=lfs -text
-*.wasm filter=lfs diff=lfs merge=lfs -text
-*.xz filter=lfs diff=lfs merge=lfs -text
-*.zip filter=lfs diff=lfs merge=lfs -text
-*.zst filter=lfs diff=lfs merge=lfs -text
-*tfevents* filter=lfs diff=lfs merge=lfs -text
+config/gen_bindata.go linguist-generated
+handlers/www/assets/swarm.png filter=lfs diff=lfs merge=lfs -text

.github/SECURITY.md

@@ -0,0 +1,16 @@
+# Security Policy
+
+## Reporting a Vulnerability
+
+If there are any vulnerabilities in this project or in [the playground](https://labs.play-with-docker.com), don't hesitate to _report them_.
+
+1. Write an email to marcosnils (at) gmail.com
+2. Describe the vulnerability.
+
+   If you have a fix, that is most welcome -- please attach or summarize it in your message!
+
+3. We will evaluate the vulnerability and, if necessary, release a fix or mitigating steps to address it. We will contact you to let you know the outcome, and will credit you in the report.
+
+   Please **do not disclose the vulnerability publicly** until a fix is released!
+
+4. Once we have either a) published a fix, or b) declined to address the vulnerability for whatever reason, you are free to publicly disclose it.

.github/workflows/test.yml

@@ -0,0 +1,30 @@
+on:
+  push:
+    branches:
+      - master
+  pull_request:
+    branches:
+      - '**'
+
+name: Go
+jobs:
+  test:
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        go_version: ["1.16.0"]
+    runs-on: ${{ matrix.os }}
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@v2
+    - name: Install Go
+      uses: actions/setup-go@v2
+      with:
+        go-version: ${{ matrix.go_version }}
+    - name: Generate
+      run: go generate ./...
+    - name: Test
+      run: go test ./...
+    - name: Verify clean commit
+      run: test -z "$(git status --porcelain)" || (git status; git diff; false)

.gitignore

@@ -0,0 +1,3 @@
+play-with-docker
+node_modules
+/vendor

Dockerfile

@@ -0,0 +1,23 @@
+FROM golang:1.16
+
+COPY . /go/src/github.com/play-with-docker/play-with-docker
+
+WORKDIR /go/src/github.com/play-with-docker/play-with-docker
+
+RUN ssh-keygen -N "" -t rsa -f /etc/ssh/ssh_host_rsa_key >/dev/null
+
+RUN CGO_ENABLED=0 go build -a -installsuffix nocgo -o /go/bin/play-with-docker .
+
+
+FROM alpine
+
+RUN apk --update add ca-certificates
+RUN mkdir -p /app/pwd
+
+COPY --from=0 /go/bin/play-with-docker /app/play-with-docker
+COPY --from=0 /etc/ssh/ssh_host_rsa_key /etc/ssh/ssh_host_rsa_key
+
+WORKDIR /app
+CMD ["./play-with-docker"]
+
+EXPOSE 3000

Dockerfile.l2

@@ -0,0 +1,27 @@
+FROM golang:1.9
+
+# Copy the runtime dockerfile into the context as Dockerfile
+COPY . /go/src/github.com/play-with-docker/play-with-docker
+
+WORKDIR /go/src/github.com/play-with-docker/play-with-docker
+
+
+RUN ssh-keygen -N "" -t rsa -f /etc/ssh/ssh_host_rsa_key >/dev/null
+
+WORKDIR /go/src/github.com/play-with-docker/play-with-docker/router/l2
+
+RUN CGO_ENABLED=0 go build -a -installsuffix nocgo -o /go/bin/play-with-docker-l2 .
+
+
+FROM alpine
+
+RUN apk --update add ca-certificates
+RUN mkdir /app
+
+COPY --from=0 /go/bin/play-with-docker-l2 /app/play-with-docker-l2
+COPY --from=0 /etc/ssh/ssh_host_rsa_key /etc/ssh/ssh_host_rsa_key
+
+WORKDIR /app
+CMD ["./play-with-docker-l2", "-ssh_key_path", "/etc/ssh/ssh_host_rsa_key"]
+
+EXPOSE 22 53 443 8080

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2016 Marcos Lilljedhal and Jonathan Leibiusky
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

api.go

@@ -0,0 +1,82 @@
+package main
+
+import (
+	"log"
+	"os"
+	"time"
+
+	"github.com/play-with-docker/play-with-docker/config"
+	"github.com/play-with-docker/play-with-docker/docker"
+	"github.com/play-with-docker/play-with-docker/event"
+	"github.com/play-with-docker/play-with-docker/handlers"
+	"github.com/play-with-docker/play-with-docker/id"
+	"github.com/play-with-docker/play-with-docker/k8s"
+	"github.com/play-with-docker/play-with-docker/provisioner"
+	"github.com/play-with-docker/play-with-docker/pwd"
+	"github.com/play-with-docker/play-with-docker/pwd/types"
+	"github.com/play-with-docker/play-with-docker/scheduler"
+	"github.com/play-with-docker/play-with-docker/scheduler/task"
+	"github.com/play-with-docker/play-with-docker/storage"
+)
+
+func main() {
+	config.ParseFlags()
+
+	e := initEvent()
+	s := initStorage()
+	df := initDockerFactory(s)
+	kf := initK8sFactory(s)
+
+	ipf := provisioner.NewInstanceProvisionerFactory(provisioner.NewWindowsASG(df, s), provisioner.NewDinD(id.XIDGenerator{}, df, s))
+	sp := provisioner.NewOverlaySessionProvisioner(df)
+
+	core := pwd.NewPWD(df, e, s, sp, ipf)
+
+	tasks := []scheduler.Task{
+		task.NewCheckPorts(e, df),
+		task.NewCheckSwarmPorts(e, df),
+		task.NewCheckSwarmStatus(e, df),
+		task.NewCollectStats(e, df, s),
+		task.NewCheckK8sClusterStatus(e, kf),
+		task.NewCheckK8sClusterExposedPorts(e, kf),
+	}
+	sch, err := scheduler.NewScheduler(tasks, s, e, core)
+	if err != nil {
+		log.Fatal("Error initializing the scheduler: ", err)
+	}
+
+	sch.Start()
+
+	d, err := time.ParseDuration("2h")
+	if err != nil {
+		log.Fatalf("Cannot parse duration Got: %v", err)
+	}
+
+	playground := types.Playground{Domain: config.PlaygroundDomain, DefaultDinDInstanceImage: "franela/dind", AvailableDinDInstanceImages: []string{"franela/dind"}, AllowWindowsInstances: config.NoWindows, DefaultSessionDuration: d, Extras: map[string]interface{}{"LoginRedirect": "http://localhost:3000"}, Privileged: true}
+	if _, err := core.PlaygroundNew(playground); err != nil {
+		log.Fatalf("Cannot create default playground. Got: %v", err)
+	}
+
+	handlers.Bootstrap(core, e)
+	handlers.Register(nil)
+}
+
+func initStorage() storage.StorageApi {
+	s, err := storage.NewFileStorage(config.SessionsFile)
+	if err != nil && !os.IsNotExist(err) {
+		log.Fatal("Error initializing StorageAPI: ", err)
+	}
+	return s
+}
+
+func initEvent() event.EventApi {
+	return event.NewLocalBroker()
+}
+
+func initDockerFactory(s storage.StorageApi) docker.FactoryApi {
+	return docker.NewLocalCachedFactory(s)
+}
+
+func initK8sFactory(s storage.StorageApi) k8s.FactoryApi {
+	return k8s.NewLocalCachedFactory(s)
+}

config/config.go

@@ -0,0 +1,78 @@
+package config
+
+import (
+	"flag"
+	"os"
+	"regexp"
+
+	"github.com/gorilla/securecookie"
+
+	"golang.org/x/oauth2"
+)
+
+const (
+	PWDHostnameRegex      = "[0-9]{1,3}-[0-9]{1,3}-[0-9]{1,3}-[0-9]{1,3}"
+	PortRegex             = "[0-9]{1,5}"
+	AliasnameRegex        = "[0-9|a-z|A-Z|-]*"
+	AliasSessionRegex     = "[0-9|a-z|A-Z]{8}"
+	AliasGroupRegex       = "(" + AliasnameRegex + ")-(" + AliasSessionRegex + ")"
+	PWDHostPortGroupRegex = "^.*ip(" + PWDHostnameRegex + ")(?:-?(" + PortRegex + "))?(?:\\..*)?$"
+	AliasPortGroupRegex   = "^.*pwd" + AliasGroupRegex + "(?:-?(" + PortRegex + "))?\\..*$"
+)
+
+var (
+	NameFilter  = regexp.MustCompile(PWDHostPortGroupRegex)
+	AliasFilter = regexp.MustCompile(AliasPortGroupRegex)
+)
+
+var (
+	PortNumber, SessionsFile, PWDContainerName, L2ContainerName, L2Subdomain, HashKey, SSHKeyPath, L2RouterIP, CookieHashKey, CookieBlockKey string
+	UseLetsEncrypt, ExternalDindVolume, NoWindows                                                                                            bool
+	LetsEncryptCertsDir                                                                                                                      string
+	MaxLoadAvg                                                                                                                               float64
+	ForceTLS                                                                                                                                 bool
+	SecureCookie                                                                                                                             *securecookie.SecureCookie
+	AdminToken                                                                                                                               string
+)
+
+// Unsafe enables a number of unsafe features when set. It is principally
+// intended to be used in development. For example, it allows the caller to
+// specify the Docker networks to join.
+var Unsafe bool
+
+var PlaygroundDomain string
+
+var SegmentId string
+
+// TODO move this to a sync map so it can be updated on demand when the configuration for a playground changes
+var Providers = map[string]map[string]*oauth2.Config{}
+
+func ParseFlags() {
+	flag.StringVar(&LetsEncryptCertsDir, "letsencrypt-certs-dir", "/certs", "Path where let's encrypt certs will be stored")
+	flag.BoolVar(&UseLetsEncrypt, "letsencrypt-enable", false, "Enabled let's encrypt tls certificates")
+	flag.BoolVar(&ForceTLS, "tls", false, "Use TLS to connect to docker daemons")
+	flag.StringVar(&PortNumber, "port", "3000", "Port number")
+	flag.StringVar(&SessionsFile, "save", "./pwd/sessions", "Tell where to store sessions file")
+	flag.StringVar(&PWDContainerName, "name", "pwd", "Container name used to run PWD (used to be able to connect it to the networks it creates)")
+	flag.StringVar(&L2ContainerName, "l2", "l2", "Container name used to run L2 Router")
+	flag.StringVar(&L2RouterIP, "l2-ip", "", "Host IP address for L2 router ping response")
+	flag.StringVar(&L2Subdomain, "l2-subdomain", "direct", "Subdomain to the L2 Router")
+	flag.StringVar(&HashKey, "hash_key", "salmonrosado", "Hash key to use for cookies")
+	flag.BoolVar(&NoWindows, "win-disable", false, "Disable windows instances")
+	flag.BoolVar(&ExternalDindVolume, "dind-external-volume", false, "Use external dind volume though XFS volume driver")
+	flag.Float64Var(&MaxLoadAvg, "maxload", 100, "Maximum allowed load average before failing ping requests")
+	flag.StringVar(&SSHKeyPath, "ssh_key_path", "", "SSH Private Key to use")
+	flag.StringVar(&CookieHashKey, "cookie-hash-key", "", "Hash key to use to validate cookies")
+	flag.StringVar(&CookieBlockKey, "cookie-block-key", "", "Block key to use to encrypt cookies")
+
+	flag.StringVar(&PlaygroundDomain, "playground-domain", "localhost", "Domain to use for the playground")
+	flag.StringVar(&AdminToken, "admin-token", "", "Token to validate admin user for admin endpoints")
+
+	flag.StringVar(&SegmentId, "segment-id", "", "Segment id to post metrics")
+
+	flag.BoolVar(&Unsafe, "unsafe", os.Getenv("PWD_UNSAFE") == "true", "Operate in unsafe mode")
+
+	flag.Parse()
+
+	SecureCookie = securecookie.New([]byte(CookieHashKey), []byte(CookieBlockKey))
+}

docker-compose.yml

@@ -0,0 +1,42 @@
+version: '3.2'
+services:
+    haproxy:
+        container_name: haproxy
+        image: haproxy
+        ports:
+            - "80:8080"
+        volumes:
+            - ./haproxy:/usr/local/etc/haproxy
+
+    pwd:
+        # pwd daemon container always needs to be named this way
+        container_name: pwd
+        # use the latest golang image
+        image: golang
+        # go to the right place and starts the app
+        command: /bin/sh -c 'ssh-keygen -N "" -t rsa -f /etc/ssh/ssh_host_rsa_key >/dev/null; cd /go/src/; if [ -e /runbin/pwd ]; then /runbin/pwd  -save /pwd/sessions -name l2; else go run api.go -save /pwd/sessions -name l2; fi'
+        environment:
+          - APPARMOR_PROFILE=docker-dind
+        volumes:
+            # since this app creates networks and launches containers, we need to talk to docker daemon
+            - /var/run/docker.sock:/var/run/docker.sock
+            # mount the box mounted shared folder to the container
+            - $PWD:/go/src
+            - sessions:/pwd
+    l2:
+        container_name: l2
+        # use the latest golang image
+        image: golang
+        # go to the right place and starts the app
+        command: /bin/sh -c 'ssh-keygen -N "" -t rsa -f /etc/ssh/ssh_host_rsa_key >/dev/null; cd /go/src/router/l2; if [ -e /runbin/l2 ]; then /runbin/l2 -ssh_key_path /etc/ssh/ssh_host_rsa_key -name l2 -save /pwd/networks; else go run l2.go -ssh_key_path /etc/ssh/ssh_host_rsa_key -name l2 -save /pwd/networks; fi'
+        volumes:
+            - /var/run/docker.sock:/var/run/docker.sock
+            - $PWD:/go/src
+            - networks:/pwd
+        ports:
+            - "8022:22"
+            - "8053:53"
+            - "443:443"
+volumes:
+    sessions:
+    networks:

docker/docker.go

@@ -0,0 +1,546 @@
+package docker
+
+import (
+	"archive/tar"
+	"bytes"
+	"context"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"net"
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/containerd/containerd/reference"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/network"
+	"github.com/docker/docker/api/types/swarm"
+	"github.com/docker/docker/api/types/volume"
+	"github.com/docker/docker/client"
+	"github.com/play-with-docker/play-with-docker/config"
+)
+
+const (
+	Byte     = 1
+	Kilobyte = 1024 * Byte
+	Megabyte = 1024 * Kilobyte
+)
+
+type DockerApi interface {
+	GetClient() *client.Client
+
+	NetworkCreate(id string, opts types.NetworkCreate) error
+	NetworkConnect(container, network, ip string) (string, error)
+	NetworkInspect(id string) (types.NetworkResource, error)
+	NetworkDelete(id string) error
+	NetworkDisconnect(containerId, networkId string) error
+
+	DaemonInfo() (types.Info, error)
+	DaemonHost() string
+
+	GetSwarmPorts() ([]string, []uint16, error)
+	GetPorts() ([]uint16, error)
+
+	ContainerStats(name string) (io.ReadCloser, error)
+	ContainerResize(name string, rows, cols uint) error
+	ContainerRename(old, new string) error
+	ContainerDelete(name string) error
+	ContainerCreate(opts CreateContainerOpts) error
+	ContainerIPs(id string) (map[string]string, error)
+	ExecAttach(instanceName string, command []string, out io.Writer) (int, error)
+	Exec(instanceName string, command []string) (int, error)
+
+	CreateAttachConnection(name string) (net.Conn, error)
+	CopyToContainer(containerName, destination, fileName string, content io.Reader) error
+	CopyFromContainer(containerName, filePath string) (io.Reader, error)
+	SwarmInit(advertiseAddr string) (*SwarmTokens, error)
+	SwarmJoin(addr, token string) error
+
+	ConfigCreate(name string, labels map[string]string, data []byte) error
+	ConfigDelete(name string) error
+}
+
+type SwarmTokens struct {
+	Manager string
+	Worker  string
+}
+
+type docker struct {
+	c *client.Client
+}
+
+func (d *docker) GetClient() *client.Client {
+	return d.c
+}
+
+func (d *docker) ConfigCreate(name string, labels map[string]string, data []byte) error {
+	config := swarm.ConfigSpec{}
+	config.Name = name
+	config.Labels = labels
+	config.Data = data
+	_, err := d.c.ConfigCreate(context.Background(), config)
+	return err
+}
+func (d *docker) ConfigDelete(name string) error {
+	return d.c.ConfigRemove(context.Background(), name)
+}
+
+func (d *docker) NetworkCreate(id string, opts types.NetworkCreate) error {
+	_, err := d.c.NetworkCreate(context.Background(), id, opts)
+
+	if err != nil {
+		log.Printf("Starting session err [%s]\n", err)
+
+		return err
+	}
+
+	return nil
+}
+
+func (d *docker) NetworkConnect(containerId, networkId, ip string) (string, error) {
+	settings := &network.EndpointSettings{}
+	if ip != "" {
+		settings.IPAddress = ip
+	}
+	err := d.c.NetworkConnect(context.Background(), networkId, containerId, settings)
+
+	if err != nil && !strings.Contains(err.Error(), "already exists") {
+		log.Printf("Connection container to network err [%s]\n", err)
+
+		return "", err
+	}
+
+	// Obtain the IP of the PWD container in this network
+	container, err := d.c.ContainerInspect(context.Background(), containerId)
+	if err != nil {
+		return "", err
+	}
+
+	n, found := container.NetworkSettings.Networks[networkId]
+	if !found {
+		return "", fmt.Errorf("Container [%s] connected to the network [%s] but couldn't obtain it's IP address", containerId, networkId)
+	}
+
+	return n.IPAddress, nil
+}
+
+func (d *docker) NetworkInspect(id string) (types.NetworkResource, error) {
+	return d.c.NetworkInspect(context.Background(), id, types.NetworkInspectOptions{})
+}
+
+func (d *docker) DaemonInfo() (types.Info, error) {
+	return d.c.Info(context.Background())
+}
+
+func (d *docker) DaemonHost() string {
+	return d.c.DaemonHost()
+}
+
+func (d *docker) GetSwarmPorts() ([]string, []uint16, error) {
+	hosts := []string{}
+	ports := []uint16{}
+
+	nodesIdx := map[string]string{}
+	nodes, nodesErr := d.c.NodeList(context.Background(), types.NodeListOptions{})
+	if nodesErr != nil {
+		return nil, nil, nodesErr
+	}
+	for _, n := range nodes {
+		nodesIdx[n.ID] = n.Description.Hostname
+		hosts = append(hosts, n.Description.Hostname)
+	}
+
+	services, err := d.c.ServiceList(context.Background(), types.ServiceListOptions{})
+	if err != nil {
+		return nil, nil, err
+	}
+	for _, service := range services {
+		for _, p := range service.Endpoint.Ports {
+			ports = append(ports, uint16(p.PublishedPort))
+		}
+	}
+
+	return hosts, ports, nil
+}
+
+func (d *docker) GetPorts() ([]uint16, error) {
+	opts := types.ContainerListOptions{}
+	containers, err := d.c.ContainerList(context.Background(), opts)
+	if err != nil {
+		return nil, err
+	}
+
+	openPorts := []uint16{}
+	for _, c := range containers {
+		for _, p := range c.Ports {
+			// When port is not published on the host docker return public port as 0, so we need to avoid it
+			if p.PublicPort != 0 {
+				openPorts = append(openPorts, p.PublicPort)
+			}
+		}
+	}
+
+	return openPorts, nil
+}
+
+func (d *docker) ContainerStats(name string) (io.ReadCloser, error) {
+	stats, err := d.c.ContainerStats(context.Background(), name, false)
+
+	return stats.Body, err
+}
+
+func (d *docker) ContainerResize(name string, rows, cols uint) error {
+	return d.c.ContainerResize(context.Background(), name, types.ResizeOptions{Height: rows, Width: cols})
+}
+
+func (d *docker) ContainerRename(old, new string) error {
+	return d.c.ContainerRename(context.Background(), old, new)
+}
+
+func (d *docker) CreateAttachConnection(name string) (net.Conn, error) {
+	ctx := context.Background()
+
+	conf := types.ContainerAttachOptions{true, true, true, true, "ctrl-^,ctrl-^", true}
+	conn, err := d.c.ContainerAttach(ctx, name, conf)
+	if err != nil {
+		return nil, err
+	}
+
+	return conn.Conn, nil
+}
+
+func (d *docker) CopyToContainer(containerName, destination, fileName string, content io.Reader) error {
+	contents, err := ioutil.ReadAll(content)
+	if err != nil {
+		return err
+	}
+	var buf bytes.Buffer
+	t := tar.NewWriter(&buf)
+	if err := t.WriteHeader(&tar.Header{Name: fileName, Mode: 0600, Size: int64(len(contents)), ModTime: time.Now()}); err != nil {
+		return err
+	}
+	if _, err := t.Write(contents); err != nil {
+		return err
+	}
+	if err := t.Close(); err != nil {
+		return err
+	}
+	return d.c.CopyToContainer(context.Background(), containerName, destination, &buf, types.CopyToContainerOptions{AllowOverwriteDirWithFile: true, CopyUIDGID: true})
+}
+
+func (d *docker) CopyFromContainer(containerName, filePath string) (io.Reader, error) {
+	rc, stat, err := d.c.CopyFromContainer(context.Background(), containerName, filePath)
+	if err != nil {
+		return nil, err
+	}
+	if stat.Mode.IsDir() {
+		return nil, fmt.Errorf("Copying directories is not supported")
+	}
+	tr := tar.NewReader(rc)
+	// advance to the only possible file in the tar archive
+	tr.Next()
+	return tr, nil
+}
+
+func (d *docker) ContainerDelete(name string) error {
+	err := d.c.ContainerRemove(context.Background(), name, types.ContainerRemoveOptions{Force: true, RemoveVolumes: true})
+	d.c.VolumeRemove(context.Background(), name, true)
+	return err
+}
+
+type CreateContainerOpts struct {
+	Image          string
+	SessionId      string
+	ContainerName  string
+	Hostname       string
+	ServerCert     []byte
+	ServerKey      []byte
+	CACert         []byte
+	Privileged     bool
+	HostFQDN       string
+	Labels         map[string]string
+	Networks       []string
+	DindVolumeSize string
+	Envs           []string
+}
+
+func (d *docker) ContainerCreate(opts CreateContainerOpts) (err error) {
+	// Make sure directories are available for the new instance container
+	containerDir := "/opt/pwd"
+	containerCertDir := fmt.Sprintf("%s/certs", containerDir)
+
+	env := append(opts.Envs, fmt.Sprintf("SESSION_ID=%s", opts.SessionId))
+
+	// Write certs to container cert dir
+	if len(opts.ServerCert) > 0 {
+		env = append(env, `DOCKER_TLSCERT=\/opt\/pwd\/certs\/cert.pem`)
+	}
+	if len(opts.ServerKey) > 0 {
+		env = append(env, `DOCKER_TLSKEY=\/opt\/pwd\/certs\/key.pem`)
+	}
+	if len(opts.CACert) > 0 {
+		// if ca cert is specified, verify that clients that connects present a certificate signed by the CA
+		env = append(env, `DOCKER_TLSCACERT=\/opt\/pwd\/certs\/ca.pem`)
+	}
+	if len(opts.ServerCert) > 0 || len(opts.ServerKey) > 0 || len(opts.CACert) > 0 {
+		// if any of the certs is specified, enable TLS
+		env = append(env, "DOCKER_TLSENABLE=true")
+	} else {
+		env = append(env, "DOCKER_TLSENABLE=false")
+	}
+
+	h := &container.HostConfig{
+		NetworkMode: container.NetworkMode(opts.SessionId),
+		Privileged:  opts.Privileged,
+		AutoRemove:  true,
+		LogConfig:   container.LogConfig{Config: map[string]string{"max-size": "10m", "max-file": "1"}},
+	}
+
+	if os.Getenv("APPARMOR_PROFILE") != "" {
+		h.SecurityOpt = []string{fmt.Sprintf("apparmor=%s", os.Getenv("APPARMOR_PROFILE"))}
+	}
+
+	if os.Getenv("STORAGE_SIZE") != "" {
+		// assing 10GB size FS for each container
+		h.StorageOpt = map[string]string{"size": os.Getenv("STORAGE_SIZE")}
+	}
+
+	var pidsLimit = int64(1000)
+	if envLimit := os.Getenv("MAX_PROCESSES"); envLimit != "" {
+		if i, err := strconv.Atoi(envLimit); err == nil {
+			pidsLimit = int64(i)
+		}
+	}
+	h.Resources.PidsLimit = &pidsLimit
+
+	if memLimit := os.Getenv("MAX_MEMORY_MB"); memLimit != "" {
+		if i, err := strconv.Atoi(memLimit); err == nil {
+			h.Resources.Memory = int64(i) * Megabyte
+		}
+	}
+
+	t := true
+	h.Resources.OomKillDisable = &t
+
+	env = append(env, fmt.Sprintf("PWD_HOST_FQDN=%s", opts.HostFQDN))
+	cf := &container.Config{
+		Hostname:     opts.Hostname,
+		Image:        opts.Image,
+		Tty:          true,
+		OpenStdin:    true,
+		AttachStdin:  true,
+		AttachStdout: true,
+		AttachStderr: true,
+		Env:          env,
+		Labels:       opts.Labels,
+	}
+
+	networkConf := &network.NetworkingConfig{
+		EndpointsConfig: map[string]*network.EndpointSettings{opts.Networks[0]: &network.EndpointSettings{}},
+	}
+
+	if config.ExternalDindVolume {
+		_, err = d.c.VolumeCreate(context.Background(), volume.VolumeCreateBody{
+			Driver: "xfsvol",
+			DriverOpts: map[string]string{
+				"size": opts.DindVolumeSize,
+			},
+			Name: opts.ContainerName,
+		})
+		if err != nil {
+			return
+		}
+		h.Binds = []string{fmt.Sprintf("%s:/var/lib/docker", opts.ContainerName)}
+
+		defer func() {
+			if err != nil {
+				d.c.VolumeRemove(context.Background(), opts.SessionId, true)
+			}
+		}()
+	}
+
+	container, err := d.c.ContainerCreate(context.Background(), cf, h, networkConf, opts.ContainerName)
+
+	if err != nil {
+		//if client.IsErrImageNotFound(err) {
+		//log.Printf("Unable to find image '%s' locally\n", opts.Image)
+		//if err = d.pullImage(context.Background(), opts.Image); err != nil {
+		//return "", err
+		//}
+		//container, err = d.c.ContainerCreate(context.Background(), cf, h, networkConf, opts.ContainerName)
+		//if err != nil {
+		//return "", err
+		//}
+		//} else {
+		return err
+		//}
+	}
+
+	//connect remaining networks if there are any
+	if len(opts.Networks) > 1 {
+		for _, nid := range opts.Networks {
+			err = d.c.NetworkConnect(context.Background(), nid, container.ID, &network.EndpointSettings{})
+			if err != nil {
+				return
+			}
+		}
+	}
+
+	if err = d.copyIfSet(opts.ServerCert, "cert.pem", containerCertDir, opts.ContainerName); err != nil {
+		return
+	}
+	if err = d.copyIfSet(opts.ServerKey, "key.pem", containerCertDir, opts.ContainerName); err != nil {
+		return
+	}
+	if err = d.copyIfSet(opts.CACert, "ca.pem", containerCertDir, opts.ContainerName); err != nil {
+		return
+	}
+
+	err = d.c.ContainerStart(context.Background(), container.ID, types.ContainerStartOptions{})
+	if err != nil {
+		return
+	}
+
+	return
+}
+
+func (d *docker) ContainerIPs(id string) (map[string]string, error) {
+	cinfo, err := d.c.ContainerInspect(context.Background(), id)
+	if err != nil {
+		return nil, err
+	}
+
+	ips := map[string]string{}
+	for networkId, conf := range cinfo.NetworkSettings.Networks {
+		ips[networkId] = conf.IPAddress
+	}
+	return ips, nil
+
+}
+
+func (d *docker) pullImage(ctx context.Context, image string) error {
+	_, err := reference.Parse(image)
+	if err != nil {
+		return err
+	}
+
+	options := types.ImageCreateOptions{}
+
+	responseBody, err := d.c.ImageCreate(ctx, image, options)
+	if err != nil {
+		return err
+	}
+	_, err = io.Copy(ioutil.Discard, responseBody)
+
+	return err
+}
+
+func (d *docker) copyIfSet(content []byte, fileName, path, containerName string) error {
+	if len(content) > 0 {
+		return d.CopyToContainer(containerName, path, fileName, bytes.NewReader(content))
+	}
+	return nil
+}
+
+func (d *docker) ExecAttach(instanceName string, command []string, out io.Writer) (int, error) {
+	e, err := d.c.ContainerExecCreate(context.Background(), instanceName, types.ExecConfig{Cmd: command, AttachStdout: true, AttachStderr: true, Tty: true})
+	if err != nil {
+		return 0, err
+	}
+	resp, err := d.c.ContainerExecAttach(context.Background(), e.ID, types.ExecStartCheck{
+		Tty: true,
+	})
+	if err != nil {
+		return 0, err
+	}
+	io.Copy(out, resp.Reader)
+	var ins types.ContainerExecInspect
+	for _ = range time.Tick(1 * time.Second) {
+		ins, err = d.c.ContainerExecInspect(context.Background(), e.ID)
+		if ins.Running {
+			continue
+		}
+		if err != nil {
+			return 0, err
+		}
+		break
+	}
+	return ins.ExitCode, nil
+
+}
+
+func (d *docker) Exec(instanceName string, command []string) (int, error) {
+	e, err := d.c.ContainerExecCreate(context.Background(), instanceName, types.ExecConfig{Cmd: command})
+	if err != nil {
+		return 0, err
+	}
+	err = d.c.ContainerExecStart(context.Background(), e.ID, types.ExecStartCheck{})
+	if err != nil {
+		return 0, err
+	}
+	var ins types.ContainerExecInspect
+	for _ = range time.Tick(1 * time.Second) {
+		ins, err = d.c.ContainerExecInspect(context.Background(), e.ID)
+		if ins.Running {
+			continue
+		}
+		if err != nil {
+			return 0, err
+		}
+		break
+	}
+	return ins.ExitCode, nil
+}
+
+func (d *docker) NetworkDisconnect(containerId, networkId string) error {
+	err := d.c.NetworkDisconnect(context.Background(), networkId, containerId, true)
+
+	if err != nil {
+		log.Printf("Disconnection of container from network err [%s]\n", err)
+
+		return err
+	}
+
+	return nil
+}
+
+func (d *docker) NetworkDelete(id string) error {
+	err := d.c.NetworkRemove(context.Background(), id)
+
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func (d *docker) SwarmInit(advertiseAddr string) (*SwarmTokens, error) {
+	req := swarm.InitRequest{AdvertiseAddr: advertiseAddr, ListenAddr: "0.0.0.0:2377"}
+	_, err := d.c.SwarmInit(context.Background(), req)
+
+	if err != nil {
+		return nil, err
+	}
+
+	swarmInfo, err := d.c.SwarmInspect(context.Background())
+	if err != nil {
+		return nil, err
+	}
+
+	return &SwarmTokens{
+		Worker:  swarmInfo.JoinTokens.Worker,
+		Manager: swarmInfo.JoinTokens.Manager,
+	}, nil
+}
+func (d *docker) SwarmJoin(addr, token string) error {
+	req := swarm.JoinRequest{RemoteAddrs: []string{addr}, JoinToken: token, ListenAddr: "0.0.0.0:2377", AdvertiseAddr: "eth0"}
+	return d.c.SwarmJoin(context.Background(), req)
+}
+
+func NewDocker(c *client.Client) *docker {
+	return &docker{c: c}
+}

docker/factory.go

@@ -0,0 +1,70 @@
+package docker
+
+import (
+	"crypto/tls"
+	"fmt"
+	"net"
+	"net/http"
+	"net/url"
+	"time"
+
+	"github.com/docker/docker/api"
+	"github.com/docker/docker/client"
+	"github.com/docker/go-connections/tlsconfig"
+	"github.com/play-with-docker/play-with-docker/pwd/types"
+	"github.com/play-with-docker/play-with-docker/router"
+)
+
+type FactoryApi interface {
+	GetForSession(session *types.Session) (DockerApi, error)
+	GetForInstance(instance *types.Instance) (DockerApi, error)
+}
+
+func NewClient(instance *types.Instance, proxyHost string) (*client.Client, error) {
+	var host string
+	var durl string
+
+	var tlsConfig *tls.Config
+	if (len(instance.Cert) > 0 && len(instance.Key) > 0) || instance.Tls {
+		host = router.EncodeHost(instance.SessionId, instance.RoutableIP, router.HostOpts{EncodedPort: 2376})
+		tlsConfig = tlsconfig.ClientDefault()
+		tlsConfig.InsecureSkipVerify = true
+		tlsConfig.ServerName = host
+		if len(instance.Cert) > 0 && len(instance.Key) > 0 {
+			tlsCert, err := tls.X509KeyPair(instance.Cert, instance.Key)
+			if err != nil {
+				return nil, fmt.Errorf("Could not load X509 key pair: %v. Make sure the key is not encrypted", err)
+			}
+			tlsConfig.Certificates = []tls.Certificate{tlsCert}
+		}
+	} else {
+		host = router.EncodeHost(instance.SessionId, instance.RoutableIP, router.HostOpts{EncodedPort: 2375})
+	}
+
+	transport := &http.Transport{
+		DialContext: (&net.Dialer{
+			Timeout:   1 * time.Second,
+			KeepAlive: 30 * time.Second,
+		}).DialContext,
+		MaxIdleConnsPerHost: 5,
+	}
+
+	if tlsConfig != nil {
+		transport.TLSClientConfig = tlsConfig
+		durl = fmt.Sprintf("https://%s", proxyHost)
+	} else {
+		transport.Proxy = http.ProxyURL(&url.URL{Host: proxyHost})
+		durl = fmt.Sprintf("http://%s", host)
+	}
+
+	cli := &http.Client{
+		Transport: transport,
+	}
+
+	dc, err := client.NewClient(durl, api.DefaultVersion, cli, nil)
+	if err != nil {
+		return nil, fmt.Errorf("Could not connect to DinD docker daemon: %v", err)
+	}
+
+	return dc, nil
+}

docker/factory_mock.go

@@ -0,0 +1,20 @@
+package docker
+
+import (
+	"github.com/play-with-docker/play-with-docker/pwd/types"
+	"github.com/stretchr/testify/mock"
+)
+
+type FactoryMock struct {
+	mock.Mock
+}
+
+func (m *FactoryMock) GetForSession(session *types.Session) (DockerApi, error) {
+	args := m.Called(session)
+	return args.Get(0).(DockerApi), args.Error(1)
+}
+
+func (m *FactoryMock) GetForInstance(instance *types.Instance) (DockerApi, error) {
+	args := m.Called(instance)
+	return args.Get(0).(DockerApi), args.Error(1)
+}

docker/local_cached_factory.go

@@ -0,0 +1,113 @@
+package docker
+
+import (
+	"context"
+	"fmt"
+	"log"
+	"sync"
+	"time"
+
+	"github.com/docker/docker/client"
+	"github.com/play-with-docker/play-with-docker/pwd/types"
+	"github.com/play-with-docker/play-with-docker/storage"
+)
+
+type localCachedFactory struct {
+	rw              sync.Mutex
+	irw             sync.Mutex
+	sessionClient   DockerApi
+	instanceClients map[string]*instanceEntry
+	storage         storage.StorageApi
+}
+
+type instanceEntry struct {
+	rw     sync.Mutex
+	client DockerApi
+}
+
+func (f *localCachedFactory) GetForSession(session *types.Session) (DockerApi, error) {
+	f.rw.Lock()
+	defer f.rw.Unlock()
+
+	if f.sessionClient != nil {
+		if err := f.check(f.sessionClient.GetClient()); err == nil {
+			return f.sessionClient, nil
+		} else {
+			f.sessionClient.GetClient().Close()
+		}
+	}
+
+	c, err := client.NewClientWithOpts()
+	if err != nil {
+		return nil, err
+	}
+	err = f.check(c)
+	if err != nil {
+		return nil, err
+	}
+	d := NewDocker(c)
+	f.sessionClient = d
+	return f.sessionClient, nil
+}
+
+func (f *localCachedFactory) GetForInstance(instance *types.Instance) (DockerApi, error) {
+	key := instance.Name
+
+	f.irw.Lock()
+	c, found := f.instanceClients[key]
+	if !found {
+		c := &instanceEntry{}
+		f.instanceClients[key] = c
+	}
+	c = f.instanceClients[key]
+	f.irw.Unlock()
+
+	c.rw.Lock()
+	defer c.rw.Unlock()
+
+	if c.client != nil {
+		if err := f.check(c.client.GetClient()); err == nil {
+			return c.client, nil
+		} else {
+			c.client.GetClient().Close()
+		}
+	}
+
+	dc, err := NewClient(instance, "l2:443")
+	if err != nil {
+		return nil, err
+	}
+	err = f.check(dc)
+	if err != nil {
+		return nil, err
+	}
+	dockerClient := NewDocker(dc)
+	c.client = dockerClient
+
+	return dockerClient, nil
+}
+
+func (f *localCachedFactory) check(c *client.Client) error {
+	ok := false
+	for i := 0; i < 5; i++ {
+		_, err := c.Ping(context.Background())
+		if err != nil {
+			log.Printf("Connection to [%s] has failed, maybe instance is not ready yet, sleeping and retrying in 1 second. Try #%d. Got: %v\n", c.DaemonHost(), i+1, err)
+			time.Sleep(time.Second)
+			continue
+		}
+		ok = true
+		break
+	}
+	if !ok {
+		return fmt.Errorf("Connection to docker daemon was not established.")
+	}
+	return nil
+}
+
+func NewLocalCachedFactory(s storage.StorageApi) *localCachedFactory {
+	return &localCachedFactory{
+		instanceClients: make(map[string]*instanceEntry),
+		storage:         s,
+	}
+}

docker/mock.go

@@ -0,0 +1,160 @@
+package docker
+
+import (
+	"io"
+	"net"
+	"time"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/client"
+	"github.com/stretchr/testify/mock"
+)
+
+type Mock struct {
+	mock.Mock
+}
+
+func (m *Mock) GetClient() *client.Client {
+	args := m.Called()
+	return args.Get(0).(*client.Client)
+}
+
+func (m *Mock) NetworkCreate(id string, opts types.NetworkCreate) error {
+	args := m.Called(id, opts)
+	return args.Error(0)
+}
+
+func (m *Mock) NetworkConnect(container, network, ip string) (string, error) {
+	args := m.Called(container, network, ip)
+	return args.String(0), args.Error(1)
+}
+
+func (m *Mock) NetworkInspect(id string) (types.NetworkResource, error) {
+	args := m.Called(id)
+	return args.Get(0).(types.NetworkResource), args.Error(1)
+}
+
+func (m *Mock) DaemonInfo() (types.Info, error) {
+	args := m.Called()
+	return args.Get(0).(types.Info), args.Error(1)
+}
+
+func (m *Mock) DaemonHost() string {
+	args := m.Called()
+	return args.String(0)
+}
+
+func (m *Mock) GetSwarmPorts() ([]string, []uint16, error) {
+	args := m.Called()
+	return args.Get(0).([]string), args.Get(1).([]uint16), args.Error(2)
+}
+
+func (m *Mock) GetPorts() ([]uint16, error) {
+	args := m.Called()
+	return args.Get(0).([]uint16), args.Error(1)
+}
+func (m *Mock) ContainerStats(name string) (io.ReadCloser, error) {
+	args := m.Called(name)
+	return args.Get(0).(io.ReadCloser), args.Error(1)
+}
+func (m *Mock) ContainerResize(name string, rows, cols uint) error {
+	args := m.Called(name, rows, cols)
+	return args.Error(0)
+}
+func (m *Mock) ContainerRename(old, new string) error {
+	args := m.Called(old, new)
+	return args.Error(0)
+}
+func (m *Mock) CreateAttachConnection(name string) (net.Conn, error) {
+	args := m.Called(name)
+	return args.Get(0).(net.Conn), args.Error(1)
+}
+func (m *Mock) CopyToContainer(containerName, destination, fileName string, content io.Reader) error {
+	args := m.Called(containerName, destination, fileName, content)
+	return args.Error(0)
+}
+
+func (m *Mock) CopyFromContainer(containerName, filePath string) (io.Reader, error) {
+	args := m.Called(containerName, filePath)
+	return args.Get(0).(io.Reader), args.Error(1)
+}
+func (m *Mock) ContainerDelete(id string) error {
+	args := m.Called(id)
+	return args.Error(0)
+}
+func (m *Mock) ContainerCreate(opts CreateContainerOpts) error {
+	args := m.Called(opts)
+	return args.Error(0)
+}
+func (m *Mock) ContainerIPs(id string) (map[string]string, error) {
+	args := m.Called(id)
+	return args.Get(0).(map[string]string), args.Error(1)
+}
+
+func (m *Mock) ExecAttach(instanceName string, command []string, out io.Writer) (int, error) {
+	args := m.Called(instanceName, command, out)
+	return args.Int(0), args.Error(1)
+}
+func (m *Mock) NetworkDisconnect(containerId, networkId string) error {
+	args := m.Called(containerId, networkId)
+	return args.Error(0)
+}
+func (m *Mock) NetworkDelete(id string) error {
+	args := m.Called(id)
+	return args.Error(0)
+}
+func (m *Mock) Exec(instanceName string, command []string) (int, error) {
+	args := m.Called(instanceName, command)
+	return args.Int(0), args.Error(1)
+}
+func (m *Mock) SwarmInit(advertiseAddr string) (*SwarmTokens, error) {
+	args := m.Called(advertiseAddr)
+	return args.Get(0).(*SwarmTokens), args.Error(1)
+}
+func (m *Mock) SwarmJoin(addr, token string) error {
+	args := m.Called(addr, token)
+	return args.Error(0)
+}
+func (m *Mock) ConfigCreate(name string, labels map[string]string, data []byte) error {
+	args := m.Called(name, labels, data)
+	return args.Error(0)
+}
+func (m *Mock) ConfigDelete(name string) error {
+	args := m.Called(name)
+	return args.Error(0)
+}
+
+type MockConn struct {
+}
+
+func (m *MockConn) Read(b []byte) (n int, err error) {
+	return len(b), nil
+}
+
+func (m *MockConn) Write(b []byte) (n int, err error) {
+	return len(b), nil
+}
+
+func (m *MockConn) Close() error {
+	return nil
+}
+
+func (m *MockConn) LocalAddr() net.Addr {
+	return &net.IPAddr{}
+}
+
+func (m *MockConn) RemoteAddr() net.Addr {
+	return &net.IPAddr{}
+}
+
+func (m *MockConn) SetDeadline(t time.Time) error {
+	return nil
+}
+
+func (m *MockConn) SetReadDeadline(t time.Time) error {
+	return nil
+}
+
+func (m *MockConn) SetWriteDeadline(t time.Time) error {
+	return nil
+}

dockerfiles/dind/.editorconfig

@@ -0,0 +1,14 @@
+# top-most EditorConfig file
+root = true
+
+# Unix-style newlines with a newline ending every file
+[*]
+end_of_line = lf
+insert_final_newline = true
+charset = utf-8
+indent_style = space
+indent_size = 4
+
+# Tab indentation (no size specified)
+[{Makefile,*.go}]
+indent_style = tab

dockerfiles/dind/.gitconfig

@@ -0,0 +1,2 @@
+[url "https://"]   
+    insteadOf = git://

dockerfiles/dind/.inputrc

@@ -0,0 +1,73 @@
+# /etc/inputrc - global inputrc for libreadline
+# See readline(3readline) and `info rluserman' for more information.
+
+# Be 8 bit clean.
+set input-meta on
+set output-meta on
+
+# To allow the use of 8bit-characters like the german umlauts, uncomment
+# the line below. However this makes the meta key not work as a meta key,
+# which is annoying to those which don't need to type in 8-bit characters.
+
+# set convert-meta off
+
+# try to enable the application keypad when it is called.  Some systems
+# need this to enable the arrow keys.
+# set enable-keypad on
+
+# see /usr/share/doc/bash/inputrc.arrows for other codes of arrow keys
+
+# do not bell on tab-completion
+# set bell-style none
+# set bell-style visible
+
+# some defaults / modifications for the emacs mode
+$if mode=emacs
+
+# allow the use of the Home/End keys
+"\e[1~": beginning-of-line
+"\e[4~": end-of-line
+
+# allow the use of the Delete/Insert keys
+"\e[3~": delete-char
+"\e[2~": quoted-insert
+
+# mappings for "page up" and "page down" to step to the beginning/end
+# of the history
+# "\e[5~": beginning-of-history
+# "\e[6~": end-of-history
+
+# alternate mappings for "page up" and "page down" to search the history
+# "\e[5~": history-search-backward
+# "\e[6~": history-search-forward
+
+# mappings for Ctrl-left-arrow and Ctrl-right-arrow for word moving
+"\e[1;5C": forward-word
+"\e[1;5D": backward-word
+"\e[5C": forward-word
+"\e[5D": backward-word
+"\e\e[C": forward-word
+"\e\e[D": backward-word
+
+$if term=rxvt
+"\e[7~": beginning-of-line
+"\e[8~": end-of-line
+"\eOc": forward-word
+"\eOd": backward-word
+$endif
+
+# for non RH/Debian xterm, can't hurt for RH/Debian xterm
+# "\eOH": beginning-of-line
+# "\eOF": end-of-line
+
+# for freebsd console
+# "\e[H": beginning-of-line
+# "\e[F": end-of-line
+
+$endif
+ 
+# faster completion
+set show-all-if-ambiguous on
+  
+"\e[A": history-search-backward
+"\e[B": history-search-forward

dockerfiles/dind/.profile

@@ -0,0 +1,6 @@
+export PS1='\e[1m\e[31m[\h] \e[32m($(docker-prompt)) \e[34m\u@$(hostname -i)\e[35m \w\e[0m\n$ '
+alias vi='vim'
+export PATH=$PATH:/root/go/bin
+export DOCKER_HOST=""
+cat /etc/motd
+echo $BASHPID > /var/run/cwd

dockerfiles/dind/.vimrc

@@ -0,0 +1,6 @@
+syntax on
+set autoindent
+set expandtab
+set number
+set shiftwidth=2
+set softtabstop=2

dockerfiles/dind/Dockerfile

@@ -0,0 +1,69 @@
+ARG VERSION=docker:dind
+FROM ${VERSION}
+
+RUN apk add --no-cache py-pip python3-dev libffi-dev openssl-dev git tmux apache2-utils vim build-base gettext-dev curl bash-completion bash util-linux jq openssh openssl tree \
+    && ln -s /usr/local/bin/docker /usr/bin/docker
+
+ENV GOPATH /root/go
+ENV IPTABLES_LEGACY /usr/local/sbin/.iptables-legacy/
+ENV PATH $IPTABLES_LEGACY:$GOPATH:$PATH
+
+
+ENV DOCKER_TLS_CERTDIR=""
+ENV DOCKER_CLI_EXPERIMENTAL=enabled
+
+# Install compose
+ENV COMPOSE_VERSION=2.18.1
+RUN mkdir -p /usr/lib/docker/cli-plugins \
+    && curl -LsS https://github.com/docker/compose/releases/download/v$COMPOSE_VERSION/docker-compose-linux-x86_64 -o /usr/lib/docker/cli-plugins/docker-compose \
+    && chmod +x /usr/lib/docker/cli-plugins/docker-compose
+
+
+# Install scout
+ENV SCOUT_VERSION=1.0.9
+RUN wget -O /tmp/scout.tar.gz https://github.com/docker/scout-cli/releases/download/v1.0.9/docker-scout_1.0.9_linux_amd64.tar.gz \
+    && tar -xvf /tmp/scout.tar.gz docker-scout -C /usr/local/bin \
+    && chmod +x /usr/local/bin/docker-scout \
+    && ln -s $(which docker-scout) /usr/lib/docker/cli-plugins \
+    && rm /tmp/scout.tar.gz
+
+
+
+# Add bash completion and set bash as default shell
+RUN curl -sS https://raw.githubusercontent.com/docker/cli/refs/heads/master/contrib/completion/bash/docker -o /etc/bash_completion.d/docker \
+    && sed -i "s/ash/bash/" /etc/passwd
+
+# Replace modprobe with a no-op to get rid of spurious warnings
+# (note: we can't just symlink to /bin/true because it might be busybox)
+RUN rm /sbin/modprobe && echo '#!/bin/true' >/sbin/modprobe && chmod +x /sbin/modprobe
+
+# Install a nice vimrc file and prompt (by soulshake)
+COPY ["docker-prompt", "sudo", "/usr/local/bin/"]
+COPY [".vimrc", ".profile", ".inputrc", ".gitconfig", "./root/"]
+COPY ["motd", "/etc/motd"]
+COPY ["daemon.json", "/etc/docker/"]
+
+
+# Move to our home
+WORKDIR /root
+
+# Setup certs and ssh keys
+RUN mkdir -p /var/run/pwd/certs && mkdir -p /var/run/pwd/uploads \
+    && ssh-keygen -N "" -t ed25519 -f  /etc/ssh/ssh_host_ed25519_key >/dev/null \
+    && mkdir ~/.ssh && ssh-keygen -N "" -t ed25519 -f ~/.ssh/id_rsa \
+    && cat ~/.ssh/id_rsa.pub > ~/.ssh/authorized_keys
+
+# Remove IPv6 alias for localhost and start docker in the background ...
+CMD cat /etc/hosts >/etc/hosts.bak && \
+    sed 's/^::1.*//' /etc/hosts.bak > /etc/hosts && \
+    sed -i "s/\PWD_IP_ADDRESS/$PWD_IP_ADDRESS/" /etc/docker/daemon.json && \
+    sed -i "s/\DOCKER_TLSENABLE/$DOCKER_TLSENABLE/" /etc/docker/daemon.json && \
+    sed -i "s/\DOCKER_TLSCACERT/$DOCKER_TLSCACERT/" /etc/docker/daemon.json && \
+    sed -i "s/\DOCKER_TLSCERT/$DOCKER_TLSCERT/" /etc/docker/daemon.json && \
+    sed -i "s/\DOCKER_TLSKEY/$DOCKER_TLSKEY/" /etc/docker/daemon.json && \
+    mount -t securityfs none /sys/kernel/security && \
+    echo "root:root" | chpasswd &> /dev/null && \
+    /usr/sbin/sshd -o PermitRootLogin=yes -o PrintMotd=no 2>/dev/null && \
+    dockerd &>/docker.log & \
+    while true ; do script -q -c "/bin/bash -l" /dev/null ; done
+# ... and then put a shell in the foreground, restarting it if it exits

dockerfiles/dind/Dockerfile.dind-ee

@@ -0,0 +1,54 @@
+ARG VERSION=franela/docker:ubuntu-19.03ee
+#ARG VERSION=franela/docker:18.09.2-ee-dind
+
+FROM ${VERSION}
+
+RUN apt-get update \ 
+    && apt-get install -y git tmux python-pip apache2-utils vim curl jq bash-completion screen tree zip \
+    && rm -rf /var/lib/apt/lists/*
+
+# Add kubectl client
+RUN curl -LO https://storage.googleapis.com/kubernetes-release/release/v1.11.7/bin/linux/amd64/kubectl \
+    && chmod +x ./kubectl \
+    && mv ./kubectl /usr/local/bin/kubectl 
+
+ENV COMPOSE_VERSION=1.22.0
+
+RUN pip install docker-compose==${COMPOSE_VERSION}
+RUN curl -L https://github.com/docker/machine/releases/download/${MACHINE_VERSION}/docker-machine-Linux-x86_64 \
+    -o /usr/bin/docker-machine && chmod +x /usr/bin/docker-machine
+ 
+
+# Install a nice vimrc file and prompt (by soulshake)
+COPY ["docker-prompt", "sudo", "ucp-beta.sh", "/usr/local/bin/"]
+COPY [".vimrc",".profile", ".inputrc", ".gitconfig", "workshop_beta.lic", "ucp-config.toml", "./root/"]
+COPY ["motd", "/etc/motd"]
+COPY ["ee/daemon.json", "/etc/docker/"]
+COPY ["ee/cert.pem", "ee/key.pem", "/opt/pwd/certs/"]
+COPY ["ee/ucp-key.pem", "./root/key.pem"]
+COPY ["ee/ucp-cert.pem", "./root/cert.pem"]
+
+# Move to our home
+WORKDIR /root
+
+# Setup certs and uploads folders
+RUN mkdir -p /opt/pwd/certs /opt/pwd/uploads
+
+VOLUME ["/var/lib/kubelet"]
+
+# Remove IPv6 alias for localhost and start docker in the background ...
+CMD cat /etc/hosts >/etc/hosts.bak && \
+    sed 's/^::1.*//' /etc/hosts.bak > /etc/hosts && \
+    sed -i "s/\PWD_IP_ADDRESS/$PWD_IP_ADDRESS/" /etc/docker/daemon.json && \
+    sed -i "s/\DOCKER_TLSENABLE/$DOCKER_TLSENABLE/" /etc/docker/daemon.json && \
+    sed -i "s/\DOCKER_TLSCACERT/$DOCKER_TLSCACERT/" /etc/docker/daemon.json && \
+    sed -i "s/\DOCKER_TLSCERT/$DOCKER_TLSCERT/" /etc/docker/daemon.json && \
+    sed -i "s/\DOCKER_TLSKEY/$DOCKER_TLSKEY/" /etc/docker/daemon.json && \
+    mount -t securityfs none /sys/kernel/security && \
+    mount --make-rshared / && \
+    #mount --make-rshared -t tmpfs tmpfs /run && \
+    #mount --make-rshared /var/lib/kubelet && \
+    #mount --make-rshared /var/lib/docker && \
+    dockerd > /docker.log 2>&1 & \
+    while true ; do script -q -c "/bin/bash -l" /dev/null ; done
+# ... and then put a shell in the foreground, restarting it if it exits

dockerfiles/dind/copy_certs.ps1

@@ -0,0 +1,114 @@
+param (
+        [Parameter(Mandatory = $true)]
+        [string] $Node,
+        [Parameter(Mandatory = $true)]
+        [string] $SessionId,
+        [Parameter(Mandatory = $true)]
+        [string] $FQDN
+)
+
+
+function GetDirectUrlFromIp ($ip) {
+        $ip_dash=$ip -replace "\.","-"
+        $url="https://ip${ip_dash}-${SessionId}.direct.${FQDN}"
+        return $url
+}
+
+function WaitForUrl ($url) {
+    write-host $url
+        do {
+                try{
+            invoke-webrequest -UseBasicParsing -uri $url | Out-Null
+        } catch {}
+        $status = $?
+        sleep 1
+        } until($status)
+}
+
+function GetNodeRoutableIp ($nodeName) {
+  $JQFilter='.instances[] | select (.hostname == \"{0}\") | .routable_ip' -f $nodeName
+  $rip = (invoke-webrequest -UseBasicParsing -uri "https://$FQDN/sessions/$SessionId").Content |  jq -r $JQFilter
+
+  IF([string]::IsNullOrEmpty($rip)) {
+    Write-Host "Could not fetch IP for node $nodeName"
+    exit 1
+  }
+  return $rip
+}
+
+function Set-UseUnsafeHeaderParsing
+{
+    param(
+        [Parameter(Mandatory,ParameterSetName='Enable')]
+        [switch]$Enable,
+
+        [Parameter(Mandatory,ParameterSetName='Disable')]
+        [switch]$Disable
+    )
+
+    $ShouldEnable = $PSCmdlet.ParameterSetName -eq 'Enable'
+
+    $netAssembly = [Reflection.Assembly]::GetAssembly([System.Net.Configuration.SettingsSection])
+
+    if($netAssembly)
+    {
+        $bindingFlags = [Reflection.BindingFlags] 'Static,GetProperty,NonPublic'
+        $settingsType = $netAssembly.GetType('System.Net.Configuration.SettingsSectionInternal')
+
+        $instance = $settingsType.InvokeMember('Section', $bindingFlags, $null, $null, @())
+
+        if($instance)
+        {
+            $bindingFlags = 'NonPublic','Instance'
+            $useUnsafeHeaderParsingField = $settingsType.GetField('useUnsafeHeaderParsing', $bindingFlags)
+
+            if($useUnsafeHeaderParsingField)
+            {
+              $useUnsafeHeaderParsingField.SetValue($instance, $ShouldEnable)
+            }
+        }
+    }
+}
+
+
+$ProgressPreference = 'SilentlyContinue'
+$ErrorActionPreference = 'Stop'
+
+Set-UseUnsafeHeaderParsing -Enable
+
+Start-Transcript -path ("C:\{0}.log" -f $MyInvocation.MyCommand.Name) -append
+
+add-type @"
+    using System.Net;
+    using System.Security.Cryptography.X509Certificates;
+
+    public class IDontCarePolicy : ICertificatePolicy {
+        public IDontCarePolicy() {}
+        public bool CheckValidationResult(
+            ServicePoint sPoint, X509Certificate cert,
+            WebRequest wRequest, int certProb) {
+            return true;
+        }
+    }
+"@
+
+[Net.ServicePointManager]::SecurityProtocol = [Net.SecurityProtocolType]::Tls12
+
+[System.Net.ServicePointManager]::CertificatePolicy = new-object IDontCarePolicy
+
+
+$dtr_ip = GetNodeRoutableIp $Node
+$dtr_url = GetDirectUrlFromIp $dtr_ip
+$dtr_hostname = $dtr_url -replace "https://",""
+
+WaitForUrl "${dtr_url}/ca"
+
+invoke-webrequest -UseBasicParsing -uri "$dtr_url/ca" -o c:\ca.crt
+
+$cert = new-object System.Security.Cryptography.X509Certificates.X509Certificate2 c:\ca.crt
+$store = new-object System.Security.Cryptography.X509Certificates.X509Store('Root','localmachine')
+$store.Open('ReadWrite')
+$store.Add($cert)
+$store.Close()
+
+Stop-Transcript

dockerfiles/dind/daemon.json

@@ -0,0 +1,17 @@
+{
+    "experimental": true,
+    "debug": true,
+    "log-level": "info",
+    "registry-mirrors": ["https://mirror.gcr.io"],
+    "insecure-registries": [
+        "127.0.0.1"
+    ],
+    "hosts": [
+        "unix:///var/run/docker.sock",
+        "tcp://0.0.0.0:2375"
+    ],
+    "tls": DOCKER_TLSENABLE,
+    "tlscacert": "DOCKER_TLSCACERT",
+    "tlscert": "DOCKER_TLSCERT",
+    "tlskey": "DOCKER_TLSKEY"
+}

dockerfiles/dind/docker-prompt

@@ -0,0 +1,22 @@
+
+#!/bin/sh
+case "$DOCKER_HOST" in
+*:3376)
+  echo swarm
+  ;;
+*:2376)
+  echo $DOCKER_MACHINE_NAME
+  ;;
+*:2375)
+  echo $DOCKER_MACHINE_NAME
+  ;;
+*:55555)
+  echo $DOCKER_MACHINE_NAME
+  ;;
+"")
+  echo local
+  ;;
+*)
+  echo unknown
+  ;;
+esac

dockerfiles/dind/ee/cert.pem

@@ -0,0 +1,29 @@
+-----BEGIN CERTIFICATE-----
+MIIE9jCCAt6gAwIBAgIQSCiXatddwed3bL9M9bierjANBgkqhkiG9w0BAQsFADAO
+MQwwCgYDVQQKEwNVQ1AwHhcNMTcwOTE1MjAzMzAwWhcNMjAwODMwMjAzMzAwWjAO
+MQwwCgYDVQQKEwNVQ1AwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQCq
+prmPRweArtZQ6HHDeYCSC3WxQOy6hakc3VZa6JEldbEoVjOc7MqZNPvTIp8b/W8H
+O10ibEGZ03vyeq10UsiFQiYmdhn1SqEilZnFSo892PSpGaN7VO325uUnIccJqc3O
+0YOdvNCdp9roZZ/K7z9nuC37cLy6+Lq2oLr1WYAxncJHedUi3LQCC+2qEBIVL+md
+9yE8amFrYbDhbNqmIcAJ2KmkqBPa0Pa+Qe1FxqQI5zJOT5rOJgF3JbWeqFpm0Zjx
+CPTt0cPY4lyQ2U9lyMXJmS4+R0wekkZXywaU1mJi3JJIlMSBMWmWoTrx5mLVWOLv
+u44hYerfOmN+ImXRWq4NAPLi4722/OLzCmFn81fdUHOFyxg2Tr23b6I6sMyUfLJ0
+lqS+thJ7N/tcQe3nTeQm9dcruDbJpjJQrQkjq9CFFsxNEXBT6EFMRp8oDutOAyHf
+guVeqdH5kz6vprNiLfSTqqZSEeQokRkHTyxpZ4grBBCiocsAxm8yLNqhcg3w44CN
+9G/3pylgu7xYSEXHYnnlxsk0MHxDFZ4NTo0UBuyIuozoePIS63GvsyBsBzKzO/RZ
+NsnPm3klZ4QnT3dIe0eRtCyu/prRmEMD/zC20fRcAuiG7jyV9NB/9mbLeDjAAngW
+1UhrWpAMiObQZN4h5+ofc0EXFHVvOWaqBmYXlNlEeQIDAQABo1AwTjAOBgNVHQ8B
+Af8EBAMCA6gwHQYDVR0lBBYwFAYIKwYBBQUHAwIGCCsGAQUFBwMBMAwGA1UdEwEB
+/wQCMAAwDwYDVR0RBAgwBocEfwAAATANBgkqhkiG9w0BAQsFAAOCAgEAFT1PFimD
+KGg8fVjuUO9IXf12LA7c+M6Eltyz22Ffxgopl1eHi4xHEfU94ueUAmODrag7Rc4E
+VmvrMFIsuFrX/xYjiu2gpHPOP2nQjNRAwKDU0gr2bZ+y98EBtlYO/aFMmYCxJr7B
+6esyA7I/cwLTxaNoTh67VTdPhfDmuEshoQn7Mtop38suevU5YBMTmUl7cp8bVdib
+j7UkTq0oRKmAchMAz3W0TgGw9ZKJzU6zEck/3Csz5RWlTI9HV7R7J8aGEIeHGf/i
+G+tfg0T8h+rQPkyCic5DIYuQzZ/P9pfJkedZuQU/mu0U/0IsNdkv9NX/4RQazu/Q
+OzQ71FOO2HR/S3hcLzS1Iy2zrHbARwji/Sr95gVE1Z4QCK2xSvyy9aqzHwRfc4SX
+AzaJhkACCnY7VDK6WJW7jnfkYco+l0tczDkyPjE7h3wP35tCuAZAvGkcrIbBL4oR
+8bnwYAOqiG0cPBmFDBYW7v19qIspw5XDjfMu4YEHon7pYdiKK0Brf0iL+Ep4b1oB
+8uvAysbc2Z/gIj1AsfnwSnrzcvzO6H1oCye277cSn2Z/ebiBaQi+kR3mubX96aPy
+bFc9Xb11/y0Y7kYmJ3ifHDJkWerpz5bWEm2KDq1qsFRH9zUMEVfJAXThITawqfuG
+3UBYWv8RePLnRbbnPuSaO9slNCoKl3NLqyk=
+-----END CERTIFICATE-----

dockerfiles/dind/ee/daemon.json

@@ -0,0 +1,10 @@
+{
+    "experimental": true,
+    "debug": true,
+    "log-level": "info",
+    "insecure-registries": ["127.0.0.1"],
+    "hosts": ["unix:///var/run/docker.sock", "tcp://0.0.0.0:2376"],
+    "tls": true,
+    "tlscert": "/opt/pwd/certs/cert.pem",
+    "tlskey": "/opt/pwd/certs/key.pem"
+}

dockerfiles/dind/ee/key.pem

@@ -0,0 +1,51 @@
+-----BEGIN RSA PRIVATE KEY-----
+MIIJJwIBAAKCAgEAqqa5j0cHgK7WUOhxw3mAkgt1sUDsuoWpHN1WWuiRJXWxKFYz
+nOzKmTT70yKfG/1vBztdImxBmdN78nqtdFLIhUImJnYZ9UqhIpWZxUqPPdj0qRmj
+e1Tt9ublJyHHCanNztGDnbzQnafa6GWfyu8/Z7gt+3C8uvi6tqC69VmAMZ3CR3nV
+Ity0AgvtqhASFS/pnfchPGpha2Gw4WzapiHACdippKgT2tD2vkHtRcakCOcyTk+a
+ziYBdyW1nqhaZtGY8Qj07dHD2OJckNlPZcjFyZkuPkdMHpJGV8sGlNZiYtySSJTE
+gTFplqE68eZi1Vji77uOIWHq3zpjfiJl0VquDQDy4uO9tvzi8wphZ/NX3VBzhcsY
+Nk69t2+iOrDMlHyydJakvrYSezf7XEHt503kJvXXK7g2yaYyUK0JI6vQhRbMTRFw
+U+hBTEafKA7rTgMh34LlXqnR+ZM+r6azYi30k6qmUhHkKJEZB08saWeIKwQQoqHL
+AMZvMizaoXIN8OOAjfRv96cpYLu8WEhFx2J55cbJNDB8QxWeDU6NFAbsiLqM6Hjy
+Eutxr7MgbAcyszv0WTbJz5t5JWeEJ093SHtHkbQsrv6a0ZhDA/8wttH0XALohu48
+lfTQf/Zmy3g4wAJ4FtVIa1qQDIjm0GTeIefqH3NBFxR1bzlmqgZmF5TZRHkCAwEA
+AQKCAgBDSNmBFJBwvH7kB8JTQGThMIOHEAJGyMyVBPA3h9sy2eSv8s0G4pY/MhTY
+ep4hext7znw6RlTXQfts79HUO4+0exBvucEiZfqCmFm44Fz6FcDhq6o5xpLM9t0D
+QN4pgToUgadTWk8m2jgFyYvnh82IJ6Z5rUm8rrVvrJAKjO9uoLUpWXAf/sU6yVk7
+5Ho8wFdsYTRJjeg7XplPSIwtVMFTIIpC0cKCVEH1YikbiebDW+UJ23k+Lt4FDGk/
+1UFPqPSUlON9oWeG7DlzIzua9j6F7k+9Xn80zpfNpc9CgATq1e0XkRCpn8HyEkAb
+gKsXU6SmwVyY7PKecXcpFIbwtMBK2zTG4VrmgsjwptK1S8lbqYftQeTxvNYdhjxA
+gdkBG5qIBkLcr8m796V2fDtJ6wvsVi+yDh+H7T8/vZuB9iaHJ3L1v36WiTODLTFW
+/OlgfimiBXuK8Z1EiB6+w522TdmhKOiWfjHdl7JSzsOla5i5cbcdeaD4AUzlmvGZ
+RCBE9Cd7RWGmDxnWz4NWFepwSfnOOQI9W95QkcRgwH61Y2axcdio0xJpQnUXiKHH
+rHhPTW0eDD7yoIqqKKK3evCOxpbJy6M/+fVqNZYWEfJ0cb7+Ska6aW3rUv8aeYFj
+xzitqKuL/0nFKpeppAkvXvoZf/mM0QtG+lgUHgOngwweYrrkwQKCAQEA3JtXFZDQ
+mIfkv0mAiwV5QbzQ63OxkO0MtPqSq50I8F6S+fIz+ILhxbMjcGq5dCbnJCFGJqn3
+7PXrT6nFXZ8j2/dcXmtxala2VAAq+GyA0TY/DQ6seTaKhsLq50vnzMXHT0pU8s/2
+4n7euf66lzQ1ByKrqXZCAyNajUXPoL37HFgFtCrEJlvi//K8x+tHr3QgF4Si8l31
+A1HLq2+KbppWXzc//knanstsCIxPvEelV0GZn3r5opiOczS30rYo87wKI9aCRgLZ
+GEKrMwlNVwwhScJd4msEYMsXUUxzDcNr5oi+iQmEDJpBKd98+3/Sp9XWVXUbik9a
+QfOvUcQMfDc1pwKCAQEAxgeiaYBb369Z6CW7rC3b3YnwOBJVK23PYcpN2DtnhRRI
+ARZgZBhwYKxDQ3djXZCiPEVtwO4WO8fCcY0GUFP2aVWuaokGjk1gNFwN6F046OdY
+WGETEe7AUCLuuwAv7Aqqug3Y6bxCtPGN3MNHT8qjTH99EMHx8L2+0UiIXnQreGmH
+VL/HEnpfDDZK4nfrwxdJOSueGdyOlflUIpDgmScIbKvIsyKhB2UstFBsCuDzhfE/
+a0VWDnZHgZPA/JhyhRy5eL9QGOqsdnzSxgvEbOyCR5p2jtO9otFw9fxpxF7uA0Yq
+EBye0gidmnF/FKDNK0iggtk34LTrDv2fz4tclXM43wKCAQBY79NC4XgHFjoVGBfX
+dCR3aRy8346Fg9uslpxMzWlOJoKQe6GSHYnRvy+ZYZ1Ug16KBVQXwEwwXtA39JSZ
+8s9tHaNCeYRmv4CQCuVH885XCcyPggvsbh2YyLoU91gDCPUaNThcD5VTqJw4VcZ5
+sNV0A/k6v29LfpRCAhP7lLvIqH/cK6WaZU71qrGK04K57FIHyTQ8C778UJyQh85C
+WrxZdJe696FIhXAPXinDGQtCSzMYxWYgs+ox7d3x9/g4kuVvn0oz2XAWRMJqN+TT
+JBPDfbWF02kXcKj84Jo9wTwd26Ec9BYlUobUz8G+TsDpYt8e4rBwqR8VGZ3jk+sI
+pOVfAoIBAA78xO33KPzk6IkJUgrV7a32opeby5Zd2TQte3bCCDOqNUjfyKvKrbaj
+UvPoNTz/lUe6eXQAkO41UCIH6lJqCFwwf+LQPA7JDF7qGKNdatE1sRn/PtI8n5Fx
+E2BTw0y6AfHS2nfWJ7ZKEdKDdQI08+b2PyDljMoLkkWEl82OPTv/wJ5JZWegm1Dx
+SvmY2d8KBCCvjGeoqaHwHM4A6P6uVZTj62yjUkyc+6Up8QNhwwyAFayosrqleQP1
+isWTRBeO9PqOgCFioWrWR511hog33iRNLGvi2pdYApSbZeXWyWy2Arj1cY+z1zm5
+HUUSZnTAKmW8yt3W03Nu/olWossszUECggEAD+dqDccmWF30yg82mxIMPb8pMV27
++ciQssiibGmhFvPcIfzish9FunXqLG7q+4M4M+O4WQ9unuaTH+z9TU7w3Foo4Xdf
+GePuwmZdpuYxClHAsNALuKWEJcjfFOdETLkAbk81+ghtyFblkPPI82wofs4K8OII
+1KPPDKoxeXmKXVF1UmOJX1KFyMnEjv0+Z1GrHnNV4703cNTMpDybaGpHsE77Vqd0
+ToZY9VG9eDLzaB6n5emSyFGBG73WQFU4EbLKjEBxtthgu8J9b17x96eF1NGZsEl1
+wEJvZpg7v6wyHK5XcYpwLY19+0khtvXwA7KKEr+sHqzF6arIqhl5hDLDAQ==
+-----END RSA PRIVATE KEY-----

dockerfiles/dind/ee/ucp-cert.pem

@@ -0,0 +1,63 @@
+-----BEGIN CERTIFICATE-----
+MIIGPDCCBSSgAwIBAgISA4MIK4JV9npV+QdQS7wVa48rMA0GCSqGSIb3DQEBCwUA
+MEoxCzAJBgNVBAYTAlVTMRYwFAYDVQQKEw1MZXQncyBFbmNyeXB0MSMwIQYDVQQD
+ExpMZXQncyBFbmNyeXB0IEF1dGhvcml0eSBYMzAeFw0xODAzMzEyMTQ3MjZaFw0x
+ODA2MjkyMTQ3MjZaMDQxMjAwBgNVBAMMKSouZGlyZWN0LmJldGEtaHlicmlkLnBs
+YXktd2l0aC1kb2NrZXIuY29tMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKC
+AQEA6PQCi9Rqr7Ka1KXSGCfBQVzgPyx/hh+uST1dz7PDw2epghYyaqNByaQEVKNR
+3ubPvOoASzhdJ1dZdyUzKUoU/jm8hgVK7HHdQDpFEX60az+r4Xo32R6WirG5+GXd
+hU3M0yRzbu0zZx7eVZognP/HcXJDhuf16hiHKmCr6MYXV4JY9xLMxExZOTB4fpGA
+Loiyvn2OEZAhREhiSX+6n4x7KJga8gYn/0f89o7up1DYQSwev+gQgRjTGlo1xrgu
+Oztekc3ydvbhGv7aL7Uj/zqPcVvXnDfnioQV7kEDcz8gupFyV7gZKolR1G8IQJdm
+TaYHguzFXF5Q3lKVWx19/CSZ8wIDAQABo4IDMDCCAywwDgYDVR0PAQH/BAQDAgWg
+MB0GA1UdJQQWMBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMBAf8EAjAAMB0G
+A1UdDgQWBBTVloZoUI5vKAN+D1PTgtYBgU184zAfBgNVHSMEGDAWgBSoSmpjBH3d
+uubRObemRWXv86jsoTBvBggrBgEFBQcBAQRjMGEwLgYIKwYBBQUHMAGGImh0dHA6
+Ly9vY3NwLmludC14My5sZXRzZW5jcnlwdC5vcmcwLwYIKwYBBQUHMAKGI2h0dHA6
+Ly9jZXJ0LmludC14My5sZXRzZW5jcnlwdC5vcmcvMDQGA1UdEQQtMCuCKSouZGly
+ZWN0LmJldGEtaHlicmlkLnBsYXktd2l0aC1kb2NrZXIuY29tMIH+BgNVHSAEgfYw
+gfMwCAYGZ4EMAQIBMIHmBgsrBgEEAYLfEwEBATCB1jAmBggrBgEFBQcCARYaaHR0
+cDovL2Nwcy5sZXRzZW5jcnlwdC5vcmcwgasGCCsGAQUFBwICMIGeDIGbVGhpcyBD
+ZXJ0aWZpY2F0ZSBtYXkgb25seSBiZSByZWxpZWQgdXBvbiBieSBSZWx5aW5nIFBh
+cnRpZXMgYW5kIG9ubHkgaW4gYWNjb3JkYW5jZSB3aXRoIHRoZSBDZXJ0aWZpY2F0
+ZSBQb2xpY3kgZm91bmQgYXQgaHR0cHM6Ly9sZXRzZW5jcnlwdC5vcmcvcmVwb3Np
+dG9yeS8wggEDBgorBgEEAdZ5AgQCBIH0BIHxAO8AdQDbdK/uyynssf7KPnFtLOW5
+qrs294Rxg8ddnU83th+/ZAAAAWJ+PniYAAAEAwBGMEQCIDngZdWcYWY0fPfUGTqX
+/Vt2qx+PRN5DN+m13TnA37e2AiBHIi5kMSxlvKNc3xzuJrvt/RKaj9xsBLmc8+uW
+ckaEdAB2ACk8UZZUyDlluqpQ/FgH1Ldvv1h6KXLcpMMM9OVFR/R4AAABYn4+eLUA
+AAQDAEcwRQIhAMkf8SYdt1egjzBE6nzOrY+f4WMS/N6XWN+gFl0mQIkhAiBn9+GG
+0XbLw33+WNJLUkau2ZdTo5kTw2qdUXdYpWJwrDANBgkqhkiG9w0BAQsFAAOCAQEA
+TAl62gFi+2l/yLItjNIrXeWh2ICH/epjeWlmF+rAb7Sb4iz9U8fsNBdDBQh25xJo
+6nLOlS2NG0hdUScylCYyGJZe6PeQvGO+qSLDamXf1DvXWvzbmQOCUkejgD7Uwbol
+5huuCAKoW4SsiaMku0J3545MEQx4Q5cPetsPawaByY5sgr2GZJzgM7lvtzr4hKWg
+x5QAns/bmcqe9LCJ2NLcgArliYu6dOHtS62kB7/Dz2DQRtCvpV553RaBe4k9Ruwl
+0ndHvjEC5OWa5sW1hwow5W3PC7Db7s0zqpt63EITkhrUOqtqtkwOMYBAkFIIe1eR
+T5fSFAdirKUOt5GnRJ40qw==
+-----END CERTIFICATE-----
+-----BEGIN CERTIFICATE-----
+MIIEkjCCA3qgAwIBAgIQCgFBQgAAAVOFc2oLheynCDANBgkqhkiG9w0BAQsFADA/
+MSQwIgYDVQQKExtEaWdpdGFsIFNpZ25hdHVyZSBUcnVzdCBDby4xFzAVBgNVBAMT
+DkRTVCBSb290IENBIFgzMB4XDTE2MDMxNzE2NDA0NloXDTIxMDMxNzE2NDA0Nlow
+SjELMAkGA1UEBhMCVVMxFjAUBgNVBAoTDUxldCdzIEVuY3J5cHQxIzAhBgNVBAMT
+GkxldCdzIEVuY3J5cHQgQXV0aG9yaXR5IFgzMIIBIjANBgkqhkiG9w0BAQEFAAOC
+AQ8AMIIBCgKCAQEAnNMM8FrlLke3cl03g7NoYzDq1zUmGSXhvb418XCSL7e4S0EF
+q6meNQhY7LEqxGiHC6PjdeTm86dicbp5gWAf15Gan/PQeGdxyGkOlZHP/uaZ6WA8
+SMx+yk13EiSdRxta67nsHjcAHJyse6cF6s5K671B5TaYucv9bTyWaN8jKkKQDIZ0
+Z8h/pZq4UmEUEz9l6YKHy9v6Dlb2honzhT+Xhq+w3Brvaw2VFn3EK6BlspkENnWA
+a6xK8xuQSXgvopZPKiAlKQTGdMDQMc2PMTiVFrqoM7hD8bEfwzB/onkxEz0tNvjj
+/PIzark5McWvxI0NHWQWM6r6hCm21AvA2H3DkwIDAQABo4IBfTCCAXkwEgYDVR0T
+AQH/BAgwBgEB/wIBADAOBgNVHQ8BAf8EBAMCAYYwfwYIKwYBBQUHAQEEczBxMDIG
+CCsGAQUFBzABhiZodHRwOi8vaXNyZy50cnVzdGlkLm9jc3AuaWRlbnRydXN0LmNv
+bTA7BggrBgEFBQcwAoYvaHR0cDovL2FwcHMuaWRlbnRydXN0LmNvbS9yb290cy9k
+c3Ryb290Y2F4My5wN2MwHwYDVR0jBBgwFoAUxKexpHsscfrb4UuQdf/EFWCFiRAw
+VAYDVR0gBE0wSzAIBgZngQwBAgEwPwYLKwYBBAGC3xMBAQEwMDAuBggrBgEFBQcC
+ARYiaHR0cDovL2Nwcy5yb290LXgxLmxldHNlbmNyeXB0Lm9yZzA8BgNVHR8ENTAz
+MDGgL6AthitodHRwOi8vY3JsLmlkZW50cnVzdC5jb20vRFNUUk9PVENBWDNDUkwu
+Y3JsMB0GA1UdDgQWBBSoSmpjBH3duubRObemRWXv86jsoTANBgkqhkiG9w0BAQsF
+AAOCAQEA3TPXEfNjWDjdGBX7CVW+dla5cEilaUcne8IkCJLxWh9KEik3JHRRHGJo
+uM2VcGfl96S8TihRzZvoroed6ti6WqEBmtzw3Wodatg+VyOeph4EYpr/1wXKtx8/
+wApIvJSwtmVi4MFU5aMqrSDE6ea73Mj2tcMyo5jMd6jmeWUHK8so/joWUoHOUgwu
+X4Po1QYz+3dszkDqMp4fklxBwXRsW10KXzPMTZ+sOPAveyxindmjkW8lGy+QsRlG
+PfZ+G6Z6h7mjem0Y+iWlkYcV4PIWL1iwBi8saCbGS5jN2p8M+X+Q7UNKEkROb3N6
+KOqkqm57TH2H3eDJAkSnh6/DNFu0Qg==
+-----END CERTIFICATE-----

dockerfiles/dind/ee/ucp-key.pem

@@ -0,0 +1,28 @@
+-----BEGIN PRIVATE KEY-----
+MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDo9AKL1GqvsprU
+pdIYJ8FBXOA/LH+GH65JPV3Ps8PDZ6mCFjJqo0HJpARUo1He5s+86gBLOF0nV1l3
+JTMpShT+ObyGBUrscd1AOkURfrRrP6vhejfZHpaKsbn4Zd2FTczTJHNu7TNnHt5V
+miCc/8dxckOG5/XqGIcqYKvoxhdXglj3EszETFk5MHh+kYAuiLK+fY4RkCFESGJJ
+f7qfjHsomBryBif/R/z2ju6nUNhBLB6/6BCBGNMaWjXGuC47O16RzfJ29uEa/tov
+tSP/Oo9xW9ecN+eKhBXuQQNzPyC6kXJXuBkqiVHUbwhAl2ZNpgeC7MVcXlDeUpVb
+HX38JJnzAgMBAAECggEAVqm4bMa4bea3HRcXYu8fQS7JKhdm1cHhd9PBm6yXzpE5
+CXEyjmNv7RD8n3Qm2BLsA67WLyWn2iPv35hSQTETQETAcudzKSVvFx7WZRzLB/8m
+9XofXsG3ZZ+avONAlwALjB1KaGEMN3fPZO8y5NVvIDBPGNggr1cyqbxPGAjh1Cav
+Laqki0rdPfr3FhxTyPBdmBFDcaMLc77Yl/7rmQJRYWb1qe+g4SEG4xXmEYpcpSUz
+zDJZAkY5XAO5cHU5EoKgKJedVBNxqAaRtaisO9yv+CKMqD83hAWhXqeK1bSphghs
+2qIkzNe134ZNUBbmK2FDsAbiPMHNcMKuI4ljfb78iQKBgQD5oZ/uzaYTt6ZQQzKq
+rQFA2DxSlBt4Ewae5n6JYzw0hIjRf7LvitZF9zKXcMkHP2QcL+5RiibyJ6ohGypa
+jpDP+m5e0B5tS6gEgFzBnrXWbjnrDxUR5Qj0lKg3uuOXw8OdwNxn+MulKkIfGyTW
+pCu7G1nh/kltwvN87s4cJycwnwKBgQDu5XUyIcok8nxcBwtxu3zFdtdNn+P4Yq1a
+W2sUEUEJUDwcUZqksPIxQhG/SMEEtBqii+EJj3nAlaWItBgTE37mzKGyKv16ZiM1
+hr+Rlv5AURxER+Eo4JLFqULZKwMaDlXDrFdV2ulF+6SXWOqKrp4/6sPYxtxHmKfs
+oBnXq/4yLQKBgCQFl5+NG2cC/EPevoP0fRbPXT0JVEFqdW0ek6ndoQVvDpM0myyH
+202zUyCZTNj348lRfVFU3zPYV2t5kQ4KPolUePLDk3BwF2m24CusbE7qDv+FaKPx
+ae5pOTD5jfgLbsHn36Y9N5240FvOve0fOZRBaSH8YLovBJXFnAZh+/y/AoGALZzQ
+CJddAjruNZ/+tmNmykkLiL2riERG9waXZkh5E28nWvzVuvYx9+e2fcBFYkGFCF4O
+xIWJaJTp+zTvl8zUIPsXMG524UTZGiI1N3YN63fRHtRekDB4tZbAtbg5qmLsSyT/
+s9vNSFhor6EBfyMiAfAwHpaxflYOUearqHslWK0CgYEAzi/B0azCOaDqzpp6RhAL
+rhTRFfu2HR8wN8EJLOSbBbUnlSSJHdnHJBwyyXe3shD/rETLV8dHx+6/k47e1l2d
+MUlsad/dOKQyL2pY7UodBzPJkIkmwknDnKzioGety8Tb98oUSTQ8oHfHMuRBOie9
+mq1MSTuZyZtsdSXnFhH3qNc=
+-----END PRIVATE KEY-----

dockerfiles/dind/motd

@@ -0,0 +1,8 @@
+###############################################################
+#                          WARNING!!!!                        #
+# This is a sandbox environment. Using personal credentials   #
+# is HIGHLY! discouraged. Any consequences of doing so are    #
+# completely the user's responsibilites.                      #
+#                                                             #
+# The PWD team.                                               #
+###############################################################

dockerfiles/dind/ssh_config

@@ -0,0 +1,2 @@
+Host *
+    StrictHostKeyChecking no

dockerfiles/dind/sudo

@@ -0,0 +1,5 @@
+#!/usr/bin/env bash
+
+# This is shim to help with the case were pasted commands from a readme assume you are not root. Since this isto be run by root, it should effectively be a dummy command that allows the parameters to pass through.
+
+exec "$@"

dockerfiles/dind/ucp-beta.sh

@@ -0,0 +1,98 @@
+#!/bin/bash
+
+set -e
+
+function wait_for_url {
+    # Wait for docker daemon to be ready
+    while ! curl -k -sS $1 > /dev/null; do
+        sleep 1;
+    done
+}
+
+function deploy_ucp {
+    wait_for_url "https://localhost:2376"
+
+    docker config create com.docker.ucp.config $HOME/ucp-config.toml
+
+    docker run --rm -i  --name ucp \
+        -v /var/run/docker.sock:/var/run/docker.sock \
+        docker/ucp:3.2.3 install --debug --force-insecure-tcp --skip-cloud-provider-check \
+        --san *.direct.${PWD_HOST_FQDN} \
+        --license $(cat $HOME/workshop_beta.lic) \
+        --swarm-port 2375 \
+        --existing-config \
+        --admin-username admin \
+        --admin-password admin1234
+
+    rm $HOME/workshop_beta.lic $HOME/ucp-config.toml
+    echo "Finished deploying UCP"
+}
+
+function get_instance_ip {
+    ip -o -4 a s eth1 | awk '{print $4}' | cut -d '/' -f1
+}
+
+function get_node_routable_ip {
+    curl -sS https://${PWD_HOST_FQDN}/sessions/${SESSION_ID} | jq -r '.instances[] | select(.hostname == "'$1'") | .routable_ip'
+}
+
+function get_direct_url_from_ip {
+    local ip_dash="${1//./-}"
+    local url="https://ip${ip_dash}-${SESSION_ID}.direct.${PWD_HOST_FQDN}"
+    echo $url
+}
+
+function deploy_dtr {
+    if [ $# -lt 1 ]; then
+        echo "DTR node hostname"
+        return
+    fi
+
+
+    local dtr_ip=$(get_node_routable_ip $1)
+    local ucp_ip=$(get_instance_ip)
+
+    local dtr_url=$(get_direct_url_from_ip $dtr_ip)
+    local ucp_url=$(get_direct_url_from_ip $ucp_ip)
+
+    docker run -i --rm docker/dtr:2.7.3 install \
+      --dtr-external-url $dtr_url \
+      --ucp-node $1 \
+      --ucp-username admin \
+      --ucp-password admin1234 \
+      --ucp-insecure-tls \
+      --ucp-url $ucp_url
+}
+
+function setup_dtr_certs {
+    if [ $# -lt 1 ]; then
+        echo "DTR node hostname is missing"
+        return
+    fi
+
+
+    local dtr_ip=$(get_node_routable_ip $1)
+    local dtr_url=$(get_direct_url_from_ip $dtr_ip)
+    local dtr_hostname="${dtr_url/https:\/\/}"
+
+    wait_for_url "$dtr_url/ca"
+
+    curl -kfsSL $dtr_url/ca -o /usr/local/share/ca-certificates/$dtr_hostname.crt
+    update-ca-certificates
+}
+
+
+case "$1" in
+    deploy)
+            deploy_ucp
+            deploy_dtr $2
+            setup_dtr_certs $2
+            ;;
+    setup-certs)
+            setup_dtr_certs $2
+            ;;
+    *)
+            echo "Illegal option $1"
+            ;;
+esac
+

dockerfiles/dind/ucp-config.toml

@@ -0,0 +1,2 @@
+[cluster_config]
+  custom_kubelet_flags = ["--http-check-frequency=20s", "--containerized=false"]

dockerfiles/dind/ucp.sh

@@ -0,0 +1,94 @@
+#!/bin/bash
+
+set -e
+
+function wait_for_url {
+    # Wait for docker daemon to be ready
+    while ! curl -k -sS $1 > /dev/null; do
+        sleep 1;
+    done
+}
+
+function deploy_ucp {
+    wait_for_url "https://localhost:2376"
+    docker run --rm -i  --name ucp \
+        -v /var/run/docker.sock:/var/run/docker.sock \
+        docker/ucp:3.2.3 install --debug --force-insecure-tcp --skip-cloud-provider-check \
+        --san *.direct.${PWD_HOST_FQDN} \
+        --license $(cat $HOME/workshop_beta.lic) \
+        --swarm-port 2375 \
+        --admin-username admin \
+        --admin-password admin1234
+
+    rm $HOME/workshop_beta.lic
+    echo "Finished deploying UCP"
+}
+
+function get_instance_ip {
+    ip -o -4 a s eth1 | awk '{print $4}' | cut -d '/' -f1
+}
+
+function get_node_routable_ip {
+    curl -sS https://${PWD_HOST_FQDN}/sessions/${SESSION_ID} | jq -r '.instances[] | select(.hostname == "'$1'") | .routable_ip'
+}
+
+function get_direct_url_from_ip {
+    local ip_dash="${1//./-}"
+    local url="https://ip${ip_dash}-${SESSION_ID}.direct.${PWD_HOST_FQDN}"
+    echo $url
+}
+
+function deploy_dtr {
+    if [ $# -lt 1 ]; then
+        echo "DTR node hostname"
+        return
+    fi
+
+
+    local dtr_ip=$(get_node_routable_ip $1)
+    local ucp_ip=$(get_instance_ip)
+
+    local dtr_url=$(get_direct_url_from_ip $dtr_ip)
+    local ucp_url=$(get_direct_url_from_ip $ucp_ip)
+
+    docker run -i --rm docker/dtr:2.7.3 install \
+      --dtr-external-url $dtr_url \
+      --ucp-node $1 \
+      --ucp-username admin \
+      --ucp-password admin1234 \
+      --ucp-insecure-tls \
+      --ucp-url $ucp_url
+}
+
+function setup_dtr_certs {
+    if [ $# -lt 1 ]; then
+        echo "DTR node hostname is missing"
+        return
+    fi
+
+
+    local dtr_ip=$(get_node_routable_ip $1)
+    local dtr_url=$(get_direct_url_from_ip $dtr_ip)
+    local dtr_hostname="${dtr_url/https:\/\/}"
+
+    wait_for_url "$dtr_url/ca"
+
+    curl -kfsSL $dtr_url/ca -o /usr/local/share/ca-certificates/$dtr_hostname.crt
+    update-ca-certificates
+}
+
+
+case "$1" in
+    deploy)
+            deploy_ucp
+            deploy_dtr $2
+            setup_dtr_certs $2
+            ;;
+    setup-certs)
+            setup_dtr_certs $2
+            ;;
+    *)
+            echo "Illegal option $1"
+            ;;
+esac
+

dockerfiles/dind/workshop.lic

@@ -0,0 +1 @@
+{"key_id":"B3T_Uirjs-tpcGd4Tql8HL--kDo1iTOUaVUFNMhEXM1Z","private_key":"RbtCEoNZ4OBu-yIHNM1mGCJ6R_4SxF-ThghAd-I3b6_N","authorization":"ewogICAicGF5bG9hZCI6ICJleUpsZUhCcGNtRjBhVzl1SWpvaU1qQXhPUzB3TkMweU5GUXhPRG93TkRvek5Gb2lMQ0owYjJ0bGJpSTZJbU16U1VnMllWSjFWak00WjBWSVIwWXRVV1l0ZGxGM2MwMHdlR05vYnpoWE4xSklPRzFLYVRaT1VUUTlJaXdpYldGNFJXNW5hVzVsY3lJNk1UQXNJbk5qWVc1dWFXNW5SVzVoWW14bFpDSTZkSEoxWlN3aWJHbGpaVzV6WlZSNWNHVWlPaUpQWm1ac2FXNWxJaXdpZEdsbGNpSTZJbEJ5YjJSMVkzUnBiMjRpZlEiLAogICAic2lnbmF0dXJlcyI6IFsKICAgICAgewogICAgICAgICAiaGVhZGVyIjogewogICAgICAgICAgICAiandrIjogewogICAgICAgICAgICAgICAiZSI6ICJBUUFCIiwKICAgICAgICAgICAgICAgImtleUlEIjogIko3TEQ6NjdWUjpMNUhaOlU3QkE6Mk80Rzo0QUwzOk9GMk46SkhHQjpFRlRIOjVDVlE6TUZFTzpBRUlUIiwKICAgICAgICAgICAgICAgImtpZCI6ICJKN0xEOjY3VlI6TDVIWjpVN0JBOjJPNEc6NEFMMzpPRjJOOkpIR0I6RUZUSDo1Q1ZROk1GRU86QUVJVCIsCiAgICAgICAgICAgICAgICJrdHkiOiAiUlNBIiwKICAgICAgICAgICAgICAgIm4iOiAieWRJeS1sVTdvN1BjZVktNC1zLUNRNU9FZ0N5RjhDeEljUUlXdUs4NHBJaVpjaVk2NzMweUNZbndMU0tUbHctVTZVQ19RUmVXUmlvTU5ORTVEczVUWUVYYkdHNm9sbTJxZFdiQndjQ2ctMlVVSF9PY0I5V3VQNmdSUEhwTUZNc3hEeld3dmF5OEpVdUhnWVVMVXBtMUl2LW1xN2xwNW5RX1J4clQwS1pSQVFUWUxFTUVmR3dtM2hNT19nZUxQUy1oZ0tQdElIbGtnNl9XY294VEdvS1A3OWRfd2FIWXhHTmw3V2hTbmVpQlN4YnBiUUFLazIxbGc3OThYYjd2WnlFQVRETXJSUjlNZUU2QWRqNUhKcFkzQ295UkFQQ21hS0dSQ0s0dW9aU29JdTBoRlZsS1VQeWJidzAwMEdPLXdhMktOOFV3Z0lJbTBpNUkxdVc5R2txNHpqQnk1emhncXVVWGJHOWJXUEFPWXJxNVFhODFEeEdjQmxKeUhZQXAtRERQRTlUR2c0elltWGpKbnhacUhFZHVHcWRldlo4WE1JMHVrZmtHSUkxNHdVT2lNSUlJclhsRWNCZl80Nkk4Z1FXRHp4eWNaZV9KR1gtTEF1YXlYcnlyVUZlaFZOVWRaVWw5d1hOYUpCLWthQ3F6NVF3YVI5M3NHdy1RU2Z0RDBOdkxlN0N5T0gtRTZ2ZzZTdF9OZVR2Z3Y4WW5oQ2lYSWxaOEhPZkl3TmU3dEVGX1VjejVPYlB5a20zdHlsck5VanQwVnlBbXR0YWNWSTJpR2loY1VQcm1rNGxWSVo3VkRfTFNXLWk3eW9TdXJ0cHNQWGNlMnBLRElvMzBsSkdoT18zS1VtbDJTVVpDcXpKMXlFbUtweXNINUhEVzljc0lGQ0EzZGVBamZaVXZON1UiCiAgICAgICAgICAgIH0sCiAgICAgICAgICAgICJhbGciOiAiUlMyNTYiCiAgICAgICAgIH0sCiAgICAgICAgICJzaWduYXR1cmUiOiAid2xrQUhLd0l1TUs5Y0N3YUdINVB1MW50dGNkLVk0SkNsRnpLeTZtcmJlTzR3eXFOenpwUi16TG4tMlhsYnJGZTdlczYtSklSREhmNzBGR3JRZl9MZEc3QVQ4bC1HRXVoUk1SaF8xVTlXd1BkOGdsWnNFem44VFUyeGtzU3lkWEw2WER3TUlqMHJUMFdpQm43T29YcEc0ZGJrUGgwLWxfY1VKQnphQzlwbEZ3ZXdmdF9Ocl80a0FpcUlNa3FJZHdQaU5XOVc4NERPUFdpZ2FrcTZTTnRtMVpqT2E1UG1ldHUydk1iUGpnTzFZM19tUVFsUldpakRwRUR1Rzl1dl9yNDFsN1I2LTdKQTB6SGpvdVVqdkxDREdIMUQ3eUxnR1RFMTlXN2FMRHI4ZE5FeXBjdi1vQzVmb3pqM19ISjUyWXVDS0RnazJzb3Y5YVFYOHhNTW5DWGU4Y3JIYjlEVG05eVcyd09FN0kxYVZLYXRKbjZrSGprM1FSWGVNbnRNQnJ6TGlzanBjZnlBYzdGNlc1YnBTSUtXaUQtd2o5QTRlY0FPbFNxc0NBS3lkaWxnR2lqQTNPY1dOZHhvV0NhV1MzaXFvakFBTE1JNHlsOFlpdG50ckVMVFNuUDFFS08wTGFaaTJxVURfU0lBSmFOUlRPTVIzblRqQUNwd1ZwYXAyU3lkOEZwc1pFVllTZFJVLWJVZDJybmN1ZHZfcC1XdFZpYWVsQ3BvTWstdURzWGhud2JyWFB6Y3dkVHVobmg0V2kxMmRTcjRUQ3ZMRktSMklCaklwam1VZWt4MFBTazlKUkNXc2R2bjY0dElCZnV6dVZSRkVkSVBidnBZd2pWOUZZc19VQWJvVE85a2E1OWZmNm1zOThHYXVTbE9sYnkwSWE0TlBxTTRKY2ZvSFUiLAogICAgICAgICAicHJvdGVjdGVkIjogImV5Sm1iM0p0WVhSTVpXNW5kR2dpT2pFM05Dd2labTl5YldGMFZHRnBiQ0k2SW1aUklpd2lkR2x0WlNJNklqSXdNVGd0TURVdE1UWlVNREU2TURNNk1qTmFJbjAiCiAgICAgIH0KICAgXQp9"}

dockerfiles/k8s/.bashrc

@@ -0,0 +1,3 @@
+export PS1="[\h \W]$ "
+cat /etc/motd
+echo $BASHPID > /var/run/cwd

dockerfiles/k8s/Dockerfile

@@ -0,0 +1,44 @@
+FROM centos:7
+
+COPY ./systemctl /usr/bin/systemctl
+COPY ./kubernetes.repo /etc/yum.repos.d/
+
+
+
+RUN yum install -y kubectl-1.27.2 kubeadm-1.27.2 kubelet-1.27.2 \
+    #&& mv -f /etc/systemd/system/kubelet.service.d/10-kubeadm.conf /etc/systemd/system/kubelet.service \
+    && yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo \
+    && yum install -y docker-ce git bash-completion \
+    && sed -i -e '4d;5d;8d' /lib/systemd/system/docker.service \
+    && yum clean all
+
+RUN curl -Lf -o /usr/bin/jq https://github.com/stedolan/jq/releases/download/jq-1.5/jq-linux64 \
+    && curl -Lf -o /usr/bin/docker-compose https://github.com/docker/compose/releases/download/1.21.0/docker-compose-$(uname -s)-$(uname -m) \
+    && chmod +x /usr/bin/jq /usr/bin/docker-compose
+
+
+VOLUME ["/var/lib/kubelet"]
+
+COPY ./kube* /etc/systemd/system/
+COPY ./wrapkubeadm.sh /usr/local/bin/kubeadm
+COPY ./tokens.csv /etc/pki/tokens.csv
+COPY ./daemon.json /etc/docker/
+COPY ./resolv.conf.override /etc/
+COPY ./docker.service /usr/lib/systemd/system/
+COPY ./.bashrc /root/
+
+COPY motd /etc/motd
+
+
+RUN echo 'source <(kubectl completion bash)' >>~/.bashrc \
+    && kubectl completion bash >> /etc/bash_completion.d/kubectl
+
+RUN mkdir -p /root/.kube && ln -s /etc/kubernetes/admin.conf /root/.kube/config \
+    && rm -f /etc/machine-id
+
+WORKDIR /root
+
+CMD mount --make-shared / \
+    && systemctl start docker \
+	&& systemctl start kubelet \
+    && while true; do script -q -c "/bin/bash -l" /dev/null; done

dockerfiles/k8s/daemon.json

@@ -0,0 +1,14 @@
+{
+  "experimental": true,
+  "debug": true,
+  "cri-containerd": true,
+  "log-level": "info",
+  "tls": false,
+  "insecure-registries": [
+    "127.0.0.1"
+  ],
+  "hosts": [
+    "unix:///var/run/docker.sock",
+    "tcp://0.0.0.0:2375"
+  ]
+}

dockerfiles/k8s/docker.service

@@ -0,0 +1,30 @@
+[Unit]
+Description=Docker Application Container Engine
+Documentation=https://docs.docker.com
+
+[Service]
+# the default is not to use systemd for cgroups because the delegate issues still
+# exists and systemd currently does not support the cgroup feature set required
+# for containers run by docker
+ExecStart=/usr/bin/dockerd
+ExecReload=/bin/kill -s HUP $MAINPID
+# Having non-zero Limit*s causes performance problems due to accounting overhead
+# in the kernel. We recommend using cgroups to do container-local accounting.
+LimitNOFILE=infinity
+LimitNPROC=infinity
+LimitCORE=infinity
+# Uncomment TasksMax if your systemd version supports it.
+# Only systemd 226 and above support this version.
+#TasksMax=infinity
+TimeoutStartSec=0
+# set delegate yes so that systemd does not reset the cgroups of docker containers
+Delegate=yes
+# kill only the docker process, not all processes in the cgroup
+KillMode=process
+# restart the docker process if it exits prematurely
+Restart=on-failure
+StartLimitBurst=3
+StartLimitInterval=60s
+
+[Install]
+WantedBy=multi-user.target

dockerfiles/k8s/kubelet.env

@@ -0,0 +1,6 @@
+KUBELET_KUBECONFIG_ARGS="--bootstrap-kubeconfig=/etc/kubernetes/bootstrap-kubelet.conf --kubeconfig=/etc/kubernetes/kubelet.conf"
+KUBELET_SYSTEM_PODS_ARGS="--pod-manifest-path=/etc/kubernetes/manifests"
+KUBELET_DNS_ARGS="--cluster-dns=10.96.0.10 --cluster-domain=cluster.local"
+KUBELET_AUTHZ_ARGS="--authorization-mode=Webhook --client-ca-file=/etc/kubernetes/pki/ca.crt"
+KUBELET_CGROUP_ARGS="--cgroup-driver=cgroupfs"
+KUBELET_EXTRA_ARGS="--fail-swap-on=false --resolv-conf=/etc/resolv.conf.override --container-runtime-endpoint /run/docker/containerd/containerd.sock "

dockerfiles/k8s/kubelet.service

@@ -0,0 +1,4 @@
+[Service]
+Restart=always
+EnvironmentFile=/etc/systemd/system/kubelet.env
+ExecStart=/usr/bin/kubelet $KUBELET_KUBECONFIG_ARGS $KUBELET_SYSTEM_PODS_ARGS $KUBELET_NETWORK_ARGS $KUBELET_DNS_ARGS $KUBELET_AUTHZ_ARGS $KUBELET_CGROUP_ARGS $KUBELET_EXTRA_ARGS

dockerfiles/k8s/kubernetes.repo

@@ -0,0 +1,7 @@
+[kubernetes]
+name=Kubernetes
+baseurl=https://pkgs.k8s.io/core:/stable:/v1.28/rpm/
+enabled=1
+gpgcheck=1
+gpgkey=https://pkgs.k8s.io/core:/stable:/v1.28/rpm/repodata/repomd.xml.key
+exclude=kubelet kubeadm kubectl cri-tools kubernetes-cni

dockerfiles/k8s/motd

@@ -0,0 +1,28 @@
+
+                          WARNING!!!!
+
+ This is a sandbox environment. Using personal credentials
+ is HIGHLY! discouraged. Any consequences of doing so, are
+ completely the user's responsibilites.
+
+ You can bootstrap a cluster as follows:
+
+ 1. Initializes cluster master node:
+
+ kubeadm init --apiserver-advertise-address $(hostname -i) --pod-network-cidr 10.5.0.0/16
+
+
+ 2. Initialize cluster networking:
+
+ kubectl apply -f https://raw.githubusercontent.com/cloudnativelabs/kube-router/master/daemonset/kubeadm-kuberouter.yaml
+
+
+ 3. (Optional) Create an nginx deployment:
+
+ kubectl apply -f https://raw.githubusercontent.com/kubernetes/website/master/content/en/examples/application/nginx-app.yaml
+
+
+                          The PWK team.
+
+
+

dockerfiles/k8s/resolv.conf.override

@@ -0,0 +1 @@
+nameserver 8.8.8.8

dockerfiles/k8s/systemctl

@@ -0,0 +1,281 @@
+#!/bin/bash
+
+function get_unit_file(){
+    
+     local UNIT=$1
+
+     for DIR in ${UNIT_PATHS[@]} ; do
+         if [ -f "${DIR}${UNIT}" ] ; then
+             echo "${DIR}${UNIT}"
+             break
+         fi
+     done
+
+}
+
+function read_option(){
+     local OPTION="$1"
+     local UNIT_FILE="$2"
+     local UNIT_INSTANCE="$3"
+
+     local UNIT=`basename $UNIT_FILE`
+     local UNIT_FULL=`echo $UNIT | sed "s/@/@$UNIT_INSTANCE/"`
+
+     VALUE="$(grep '^'$OPTION'[= ]' "$UNIT_FILE" | cut -d '=' -f2- | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//')"
+
+     VALUE="`
+         echo $VALUE |
+         sed -e "s/%[i]/$UNIT_INSTANCE/g" \
+             -e "s/%[I]/\"$UNIT_INSTANCE\"/g" \
+             -e "s/%[n]/$UNIT_FULL/g" \
+             -e "s/%[N]/\"$UNIT_FULL\"/g"
+     `"
+     # TODO: Add more options from:
+     # https://www.freedesktop.org/software/systemd/man/systemd.unit.html#Specifiers
+
+     echo $VALUE
+}
+
+function get_unit_wants() {
+
+     local UNIT_FILE=$1
+     local UNIT=`basename $UNIT_FILE`
+
+     sort -u <<< `(
+         # Print wants from UNIT_PATHS
+         for DIR in ${UNIT_PATHS[@]} ; do
+             if [ -d "${DIR}${UNIT}.wants" ] ; then
+                 ls -1 "${DIR}${UNIT}.wants/" | tr '\n' ' '
+             fi
+         done
+    
+         # Print wants from unit-file
+         read_option Wants $UNIT_FILE
+     )`
+}
+
+function action_start(){
+
+    # Find depended services
+    local UNIT_FILE=$1
+    local UNIT_WANTS=(`get_unit_wants $1`)
+    local UNIT_INSTANCE=$2
+
+    # Start depended services
+    for UNIT in ${UNIT_WANTS[@]}; do
+        exec_action start $UNIT
+    done
+
+    # Load options 
+    local User=`read_option User $UNIT_FILE $UNIT_INSTANCE`
+    local Type=`read_option Type $UNIT_FILE $UNIT_INSTANCE`
+    local EnvironmentFile=`read_option EnvironmentFile $UNIT_FILE $UNIT_INSTANCE`
+    local ExecStartPre=(`read_option ExecStartPre $UNIT_FILE $UNIT_INSTANCE`)
+    local ExecStart=`read_option ExecStart $UNIT_FILE $UNIT_INSTANCE`
+    local ExecStartPost=(`read_option ExecStartPost $UNIT_FILE $UNIT_INSTANCE`)
+    local Restart=(`read_option Restart $UNIT_FILE $UNIT_INSTANCE`)
+    local RestartSec=(`read_option RestartSec $UNIT_FILE $UNIT_INSTANCE`)
+	RestartSec=${RestartSec:=5}
+
+    [ -f "$EnvironmentFile" ] && source "$EnvironmentFile"
+
+    # Start service 
+    if [ -z $Type ] || [[ "${Type,,}" == *"simple"* ]] ; then
+		if [ "$Restart" == "always" ]; then
+			COMMAND='nohup bash -c "while true ; do '"$ExecStart"'; sleep $RestartSec; done" &>/dev/null &'
+		else
+			COMMAND='nohup '"$ExecStart"' >>/dev/null 2>&1 &'
+		fi
+    elif [[ "${Type,,}" == *"forking"* ]] || [[ "${Type,,}" == *"oneshot"* ]] ; then
+        COMMAND="$ExecStart"
+    else
+        >&2 echo "Unknown service type $Type"
+    fi
+
+    #[ -z $User ] || COMMAND="su $User -c \"$COMMAND\""
+
+    while IFS=$'\n' read -a i; do
+        eval $i
+    done  <<< "${ExecStartPre[@]}"
+
+    eval "$COMMAND"
+
+    while IFS=$'\n' read -a i; do
+        eval $i
+    done  <<< "${ExecStartPost[@]}"
+}
+
+function action_stop(){
+
+    # Find depended services
+    local UNIT_FILE=$1
+    local UNIT_WANTS=(`get_unit_wants $1`)
+    local UNIT_INSTANCE=$2
+
+    # Load options 
+    local User=`read_option User $UNIT_FILE $UNIT_INSTANCE`
+    local Type=`read_option Type $UNIT_FILE $UNIT_INSTANCE`
+    local EnvironmentFile=`read_option EnvironmentFile $UNIT_FILE $UNIT_INSTANCE`
+    local ExecStopPre=(`read_option ExecStartPre $UNIT_FILE $UNIT_INSTANCE`)
+    local ExecStop=`read_option ExecStop $UNIT_FILE $UNIT_INSTANCE`
+    local ExecStopPost=(`read_option ExecStartPost $UNIT_FILE $UNIT_INSTANCE`)
+    local ExecStart=`read_option ExecStart $UNIT_FILE $UNIT_INSTANCE`
+
+    [ -f "$EnvironmentFile" ] && source "$EnvironmentFile"
+
+    # Stop service 
+    if [ -z $ExecStop ] ; then
+        COMMAND="kill -TERM \$(pgrep -f \"$ExecStart\")"
+    else
+        COMMAND="$ExecStop"
+    fi
+
+    #[ -z $User ] || COMMAND="su $User -c \"$COMMAND\""
+
+    while IFS=$'\n' read -a i; do
+        eval $i
+    done  <<< "${ExecStopPre[@]}"
+
+    eval "$COMMAND"
+
+    while IFS=$'\n' read -a i; do
+        eval $i
+    done  <<< "${ExecStopPost[@]}"
+}
+
+function action_restart(){
+    local UNIT_FILE=$1
+    local UNIT_INSTANCE=$2
+
+    action_stop $UNIT_FILE $UNIT_INSTANCE
+    action_start $UNIT_FILE $UNIT_INSTANCE
+}
+
+
+function action_enable(){
+
+    local UNIT_FILE=$1
+    local UNIT=`basename $UNIT_FILE`
+    local UNIT_INSTANCE=$2
+    local UNIT_FULL=`echo $UNIT | sed "s/@/@$UNIT_INSTANCE/"`
+
+    local WantedBy=`read_option WantedBy $UNIT_FILE`
+
+    if [ -z $WantedBy ] ; then
+        >&2 echo "Unit $UNIT have no WantedBy option."
+        exit 1
+    fi
+    
+    local WANTEDBY_DIR="/etc/systemd/system/$WantedBy.wants"
+
+    if [ ! -f "$WANTEDBY_DIR/$UNIT_FULL" ] ; then
+        mkdir -p $WANTEDBY_DIR
+        echo Created symlink from $WANTEDBY_DIR/$UNIT_FULL to $UNIT_FILE.
+        ln -s $WANTEDBY_DIR/$UNIT_FULL $UNIT_FILE
+    fi
+    
+}
+    
+function action_disable(){
+
+    local UNIT_FILE=$1
+    local UNIT=`basename $UNIT_FILE`
+    local UNIT_INSTANCE=$2
+    local UNIT_FULL=`echo $UNIT | sed "s/@/@$UNIT_INSTANCE/"`
+
+    local WantedBy=`read_option WantedBy $UNIT_FILE`
+
+    if [ -z $WantedBy ] ; then
+        >&2 echo "Unit $UNIT have no WantedBy option."
+        exit 1
+    fi
+
+    local WANTEDBY_DIR="/etc/systemd/system/$WantedBy.wants"
+
+    if [ -f "$WANTEDBY_DIR/$UNIT_FULL" ] ; then
+        echo Removed $WANTEDBY_DIR/$UNIT_FULL.
+        rm -f $WANTEDBY_DIR/$UNIT_FULL.
+        rmdir --ignore-fail-on-non-empty $WANTEDBY_DIR
+    fi
+    
+}
+
+function action_status(){
+
+    # Find depended services
+    local UNIT_FILE=$1
+    local UNIT_WANTS=(`get_unit_wants $1`)
+    local UNIT_INSTANCE=$2
+
+    local ExecStart=`read_option ExecStart $UNIT_FILE $UNIT_INSTANCE`
+
+
+    COMMAND="pgrep -f \"$ExecStart\" &>/dev/null"
+
+
+    if eval "$COMMAND"; then
+     exit 0
+    fi
+
+    >&2 echo "Loaded: not-found"
+    exit 1
+}
+
+function action_is_enabled(){
+	exit 0
+}
+
+function action_is_active(){
+	local UNIT=`basename $1`
+	if systemctl status $UNIT ; then
+		>&2 echo "active"
+		exit 0
+	fi
+
+	exit 1
+
+}
+
+function exec_action(){
+
+    local ACTION=$1
+    local UNIT=$2
+    
+    [[ $UNIT =~ '.' ]] || UNIT="$UNIT.service"
+
+    if [[ $UNIT =~ '@' ]] ; then
+        local UNIT_INSTANCE=`echo $UNIT | cut -d'@' -f2- | cut -d. -f1`
+        local UNIT=`echo $UNIT | sed "s/$UNIT_INSTANCE//"`
+    fi
+
+    UNIT_FILE=`get_unit_file $UNIT`
+
+    if [ -z $UNIT_FILE ] ; then
+        >&2 echo "Failed to $ACTION $UNIT: Unit $UNIT not found."
+        exit 1
+    else
+        case "$ACTION" in
+            start )      action_start $UNIT_FILE $UNIT_INSTANCE ;;
+            stop )       action_stop $UNIT_FILE $UNIT_INSTANCE ;;
+            restart )    action_restart $UNIT_FILE $UNIT_INSTANCE ;;
+            enable )     action_enable $UNIT_FILE $UNIT_INSTANCE ;;
+            disable )    action_disable $UNIT_FILE $UNIT_INSTANCE ;;
+            status )     action_status $UNIT_FILE $UNIT_INSTANCE ;;
+            is-enabled ) action_is_enabled $UNIT_FILE $UNIT_INSTANCE ;;
+            is-active )  action_is_active $UNIT_FILE $UNIT_INSTANCE ;;
+            * ) >&2 echo "Unknown operation $ACTION." ; exit 1 ;;
+        esac
+    fi
+}
+
+ACTION="$1"
+UNITS="${@:2}"
+UNIT_PATHS=(
+    /etc/systemd/system/
+    /usr/lib/systemd/system/
+)
+
+
+for UNIT in ${UNITS[@]}; do
+    exec_action $ACTION $UNIT
+done

dockerfiles/k8s/tokens.csv

@@ -0,0 +1 @@
+31ada4fd-adec-460c-809a-9e56ceb75269,pwd,pwd,"system:admin,system:masters"

dockerfiles/k8s/wrapkubeadm.sh

@@ -0,0 +1,153 @@
+#!/bin/bash
+# Copyright 2017 Mirantis
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -o pipefail
+set -o errtrace
+
+apiserver_static_pod="/etc/kubernetes/manifests/kube-apiserver"
+
+# jq filters follow.
+
+# TODO: think about more secure possibilities
+apiserver_anonymous_auth='.spec.containers[0].command|=map(select(startswith("--token-auth-file")|not))+["--token-auth-file=/etc/pki/tokens.csv"]'
+
+# Sets etcd2 as backend
+apiserver_etcd2_backend='.spec.containers[0].command|=map(select(startswith("--storage-backend")|not))+["--storage-backend=etcd2"]'
+
+# Make apiserver accept insecure connections on port 8080
+# TODO: don't use insecure port
+#apiserver_insecure_bind_port='.spec.containers[0].command|=map(select(startswith("--insecure-port=")|not))+["--insecure-port=2375"]'
+
+
+# Update kube-proxy CIDR, enable --masquerade-all and disable conntrack (see dind::frob-proxy below)
+function dind::proxy-cidr-and-no-conntrack {
+  cluster_cidr="$(ip addr show docker0 | grep -w inet | awk '{ print $2; }')"
+  echo ".items[0].spec.template.spec.containers[0].command |= .+ [\"--cluster-cidr=${cluster_cidr}\", \"--masquerade-all\", \"--conntrack-max-per-core=0\"]"
+}
+
+
+# Adds route to defualt eth0 interface so 10.96.x.x can go through
+function dind::add-route {
+   ip route add 10.96.0.0/16 dev eth0
+}
+
+
+
+function dind::join-filters {
+  local IFS="|"
+  echo "$*"
+}
+
+function dind::frob-apiserver {
+  local -a filters=("${apiserver_anonymous_auth}")
+
+  dind::frob-file "${apiserver_static_pod}" "${filters[@]}"
+}
+
+function dind::frob-file {
+  local path_base="$1"
+  shift
+  local filter="$(dind::join-filters "$@")"
+  local status=0
+  if [[ -f ${path_base}.yaml ]]; then
+    dind::yq "${filter}" "${path_base}.yaml" || status=$?
+  else
+    echo "${path_base}.json or ${path_base}.yaml not found" >&2
+    return 1
+  fi
+  if [[ ${status} -ne 0 ]]; then
+    echo "Failed to frob ${path_base}.yaml" >&2
+    return 1
+  fi
+}
+
+function dind::yq {
+  local filter="$1"
+  local path="$2"
+  # We need to use a temp file here because if you feed an object to
+  # 'kubectl convert' via stdin, you'll get a List object because
+  # multiple input objects are implied
+  tmp="$(mktemp tmp-XXXXXXXXXX.json)"
+  kubectl convert -f "${path}" --local -o json 2>/dev/null |
+    jq "${filter}" > "${tmp}"
+  kubectl convert -f "${tmp}" --local -o yaml 2>/dev/null >"${path}"
+  rm -f "${tmp}"
+}
+
+function dind::frob-proxy {
+  # Trying to change conntrack settings fails even in priveleged containers,
+  # so we need to avoid it. Here's sample error message from kube-proxy:
+  # I1010 21:53:00.525940       1 conntrack.go:57] Setting conntrack hashsize to 49152
+  # Error: write /sys/module/nf_conntrack/parameters/hashsize: operation not supported
+  # write /sys/module/nf_conntrack/parameters/hashsize: operation not supported
+  #
+  # Recipe by @errordeveloper:
+  # https://github.com/kubernetes/kubernetes/pull/34522#issuecomment-253248985
+  local force_apply=--force
+  if ! kubectl version --short >&/dev/null; then
+    # kubectl 1.4 doesn't have version --short and also it doesn't support apply --force
+    force_apply=
+  fi
+  KUBECONFIG=/etc/kubernetes/admin.conf kubectl -n kube-system get ds -l k8s-app=kube-proxy -o json |
+    jq "$(dind::join-filters "$(dind::proxy-cidr-and-no-conntrack)")" | KUBECONFIG=/etc/kubernetes/admin.conf kubectl apply ${force_apply} -f -
+
+  KUBECONFIG=/etc/kubernetes/admin.conf kubectl -n kube-system delete pods --now -l "k8s-app=kube-proxy"
+}
+
+
+function dind::wait-for-apiserver {
+    echo -n "Waiting for api server to startup"
+    local url="https://localhost:6443/api"
+    local n=60
+    while true; do
+      if curl -k -s "${url}" >&/dev/null; then
+        break
+      fi
+      if ((--n == 0)); then
+        echo "Error: timed out waiting for apiserver to become available" >&2
+      fi
+      echo -n "."
+      sleep 0.5
+    done
+    echo ""
+}
+
+function dind::frob-cluster {
+  #dind::frob-apiserver
+  dind::wait-for-apiserver
+  dind::frob-proxy
+}
+
+# Weave depends on /etc/machine-id being unique
+if [[ ! -f /etc/machine-id ]]; then
+  rm -f /etc/machine-id
+  systemd-machine-id-setup
+fi
+
+if [[ "$@" == "init"* || "$@" == "join"* ]]; then
+# Call kubeadm with params and skip flag
+	/usr/bin/kubeadm "$@" --ignore-preflight-errors all --cri-socket /run/docker/containerd/containerd.sock
+else
+# Call kubeadm with params
+	/usr/bin/kubeadm "$@"
+fi
+
+# Frob cluster
+if [[ "$@" == "init"* && $? -eq 0 && ! "$@" == *"--help"* ]]; then
+   dind::frob-cluster
+else
+   dind::add-route
+fi
+

dockerfiles/pwm/.gitconfig

@@ -0,0 +1,2 @@
+[url "https://"]   
+    insteadOf = git://