add supplemental container support

README.md

@@ -2,6 +2,16 @@
 
 Flux is a lightweight self-hosted pseudo-PaaS for hosting Golang web apps with ease. Built on top of [Buildpacks](https://buildpacks.io/) and [Docker](https://docs.docker.com/get-docker/), Flux simplifies the deployment process with a focus on similicity, speed, and reliability.
 
+**Goals**:
+
+- Automatic deployment of Golang web apps, simply run `flux init`, chnage the app name, and run `flux deploy` and you're done!
+- Zero-downtime deployments with blue-green deployments
+- Simple but powerful configuration, flux should be able to handle most use cases, from a micro web app to a fullstack app with databases, caching layers, full text search, etc.
+
+**Limitations**:
+- Theoretically only supports up to 1023 containers (roughly 500 apps assuming 2 containers per app), this is because flux uses the same bridge network for all containers (this could theoretically be increased if flux was smart enough to create new networks once we hit the max, but this is not a priority)
+- Containers are not particularly isolated, if one malicious container wanted to scan all containers, or interact with other containers it tectically shouldnt, it totally just can (todo?)
+
 ## Features
 
 - **Blue-Green Deployments**: Deploy new versions of your app without downtime

cmd/flux/commands/delete.go

@@ -1,4 +1,4 @@
-package handlers
+package commands
 
 import (
 	"fmt"

cmd/flux/commands/deploy.go

@@ -1,4 +1,5 @@
-package handlers
+
+package commands
 
 import (
 	"archive/tar"

cmd/flux/commands/init.go

@@ -1,4 +1,4 @@
-package handlers
+package commands
 
 import (
 	"encoding/json"

cmd/flux/commands/list.go

@@ -1,4 +1,4 @@
-package handlers
+package commands
 
 import (
 	"encoding/json"

cmd/flux/commands/project.go

@@ -1,4 +1,4 @@
-package handlers
+package commands
 
 import (
 	"encoding/json"

cmd/flux/commands/start.go

@@ -1,4 +1,4 @@
-package handlers
+package commands
 
 import (
 	"fmt"

cmd/flux/commands/stop.go

@@ -1,4 +1,4 @@
-package handlers
+package commands
 
 import (
 	"fmt"

cmd/flux/main.go

@@ -13,7 +13,7 @@ import (
 
 	"github.com/agnivade/levenshtein"
 	"github.com/briandowns/spinner"
-	"github.com/juls0730/flux/cmd/flux/handlers"
+	"github.com/juls0730/flux/cmd/flux/commands"
 	"github.com/juls0730/flux/cmd/flux/models"
 	"github.com/juls0730/flux/pkg"
 )
@@ -23,6 +23,8 @@ var config []byte
 
 var configPath = filepath.Join(os.Getenv("HOME"), "/.config/flux")
 
+var version = pkg.Version
+
 var helpStr = `Usage:
   flux <command>
 
@@ -179,15 +181,20 @@ func main() {
 		os.Exit(1)
 	}
 
+	if info.Version != version {
+		fmt.Printf("Version mismatch, daemon is running version %s, but you are running version %s\n", info.Version, version)
+		os.Exit(1)
+	}
+
 	cmdHandler := CommandHandler{
 		commands: make(map[string]func(bool, models.Config, pkg.Info, *spinner.Spinner, *models.CustomSpinnerWriter, []string) error),
 	}
 
-	cmdHandler.RegisterCmd("deploy", handlers.DeployCommand)
+	cmdHandler.RegisterCmd("deploy", commands.DeployCommand)
-	cmdHandler.RegisterCmd("stop", handlers.StopCommand)
+	cmdHandler.RegisterCmd("stop", commands.StopCommand)
-	cmdHandler.RegisterCmd("start", handlers.StartCommand)
+	cmdHandler.RegisterCmd("start", commands.StartCommand)
-	cmdHandler.RegisterCmd("delete", handlers.DeleteCommand)
+	cmdHandler.RegisterCmd("delete", commands.DeleteCommand)
-	cmdHandler.RegisterCmd("init", handlers.InitCommand)
+	cmdHandler.RegisterCmd("init", commands.InitCommand)
 
 	err = runCommand(command, args, config, info, cmdHandler, 0)
 	if err != nil {

cmd/fluxd/main.go

@@ -4,7 +4,7 @@ import (
 	"net/http"
 	_ "net/http/pprof"
 
-	"github.com/juls0730/flux/server"
+	"github.com/juls0730/flux/internal/server"
 	"go.uber.org/zap"
 )
 

internal/server/app.go

@@ -18,7 +18,9 @@ type App struct {
 	DeploymentID int64       `json:"deployment_id,omitempty"`
 }
 
-func CreateApp(ctx context.Context, imageName string, projectPath string, projectConfig pkg.ProjectConfig) (*App, error) {
+// Create the initial app row in the database and create and start the deployment. The app is the overarching data
+// structure that contains all of the data for a project
+func CreateApp(ctx context.Context, imageName string, projectPath string, projectConfig *pkg.ProjectConfig) (*App, error) {
 	app := &App{
 		Name: projectConfig.Name,
 	}
@@ -31,16 +33,26 @@ func CreateApp(ctx context.Context, imageName string, projectPath string, projec
 		return nil, err
 	}
 
-	container, err := CreateContainer(ctx, imageName, projectPath, projectConfig, true, deployment)
+	for _, container := range projectConfig.Containers {
-	if err != nil || container == nil {
+		c, err := CreateContainer(ctx, &container, projectConfig.Name, false, deployment)
-		return nil, fmt.Errorf("failed to create container: %v", err)
+		if err != nil {
+			return nil, fmt.Errorf("failed to create container: %v", err)
+		}
+
+		c.Start(ctx, true)
 	}
 
-	if appInsertStmt == nil {
+	headContainer := &pkg.Container{
-		appInsertStmt, err = Flux.db.Prepare("INSERT INTO apps (name, deployment_id) VALUES ($1, $2) RETURNING id, name, deployment_id")
+		Name:        projectConfig.Name,
-		if err != nil {
+		ImageName:   imageName,
-			return nil, fmt.Errorf("failed to prepare statement: %v", err)
+		Volumes:     projectConfig.Volumes,
-		}
+		Environment: projectConfig.Environment,
+	}
+
+	// this call does a lot for us, see it's documentation for more info
+	_, err = CreateContainer(ctx, headContainer, projectConfig.Name, true, deployment)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create container: %v", err)
 	}
 
 	// create app in the database
@@ -59,7 +71,7 @@ func CreateApp(ctx context.Context, imageName string, projectPath string, projec
 	return app, nil
 }
 
-func (app *App) Upgrade(ctx context.Context, projectConfig pkg.ProjectConfig, imageName string, projectPath string) error {
+func (app *App) Upgrade(ctx context.Context, imageName string, projectPath string, projectConfig *pkg.ProjectConfig) error {
 	logger.Debugw("Upgrading deployment", zap.String("name", app.Name))
 
 	// if deploy is not started, start it
@@ -83,6 +95,7 @@ func (app *App) Upgrade(ctx context.Context, projectConfig pkg.ProjectConfig, im
 	return nil
 }
 
+// delete an app and deployment from the database, and its project files from disk.
 func (app *App) Remove(ctx context.Context) error {
 	Flux.appManager.RemoveApp(app.Name)
 
@@ -131,10 +144,12 @@ func (am *AppManager) GetAllApps() []*App {
 	return apps
 }
 
+// removes an app from the app manager
 func (am *AppManager) RemoveApp(name string) {
 	am.Delete(name)
 }
 
+// add a given app to the app manager
 func (am *AppManager) AddApp(name string, app *App) {
 	if app.Deployment.Containers == nil || app.Deployment.Head == nil || len(app.Deployment.Containers) == 0 {
 		panic("nil containers")
@@ -143,6 +158,7 @@ func (am *AppManager) AddApp(name string, app *App) {
 	am.Store(name, app)
 }
 
+// nukes an app completely
 func (am *AppManager) DeleteApp(name string) error {
 	app := am.GetApp(name)
 	if app == nil {
@@ -159,6 +175,7 @@ func (am *AppManager) DeleteApp(name string) error {
 	return nil
 }
 
+// Scan every app in the database, and create in memory structures if the deployment is already running
 func (am *AppManager) Init() {
 	logger.Info("Initializing deployments")
 
@@ -219,7 +236,7 @@ func (am *AppManager) Init() {
 			defer rows.Close()
 
 			for rows.Next() {
-				var volume Volume
+				volume := new(Volume)
 				rows.Scan(&volume.ID, &volume.VolumeID, &volume.ContainerID, &volume.Mountpoint)
 				container.Volumes = append(container.Volumes, volume)
 			}

internal/server/container.go

@@ -4,23 +4,20 @@ import (
 	"context"
 	"database/sql"
 	"fmt"
+	"io"
 	"net/http"
-	"os"
-	"path/filepath"
 	"strings"
 	"time"
 
 	"github.com/docker/docker/api/types/container"
+	"github.com/docker/docker/api/types/image"
 	"github.com/docker/docker/api/types/mount"
 	"github.com/docker/docker/api/types/volume"
-	"github.com/joho/godotenv"
 	"github.com/juls0730/flux/pkg"
 	"go.uber.org/zap"
 )
 
 var (
-	volumeInsertStmt    *sql.Stmt
-	volumeUpdateStmt    *sql.Stmt
 	containerInsertStmt *sql.Stmt
 )
 
@@ -34,12 +31,14 @@ type Volume struct {
 type Container struct {
 	ID           int64       `json:"id"`
 	Head         bool        `json:"head"` // if the container is the head of the deployment
+	Name         string      `json:"name"`
 	Deployment   *Deployment `json:"-"`
-	Volumes      []Volume    `json:"volumes"`
+	Volumes      []*Volume   `json:"volumes"`
 	ContainerID  [64]byte    `json:"container_id"`
 	DeploymentID int64       `json:"deployment_id"`
 }
 
+// Creates a volume in the docker daemon and returns the descriptor for the volume
 func CreateDockerVolume(ctx context.Context) (vol *Volume, err error) {
 	dockerVolume, err := Flux.dockerClient.VolumeCreate(ctx, volume.CreateOptions{
 		Driver:     "local",
@@ -58,45 +57,41 @@ func CreateDockerVolume(ctx context.Context) (vol *Volume, err error) {
 	return vol, nil
 }
 
-func CreateDockerContainer(ctx context.Context, imageName, projectPath string, projectConfig pkg.ProjectConfig, vol *Volume) (*Container, error) {
+// Creates a container in the docker daemon and returns the descriptor for the container
-	containerName := fmt.Sprintf("%s-%s", projectConfig.Name, time.Now().Format("20060102-150405"))
+func CreateDockerContainer(ctx context.Context, imageName string, projectName string, vols []*Volume, environment []string, hosts []string) (*Container, error) {
-
+	for _, host := range hosts {
-	if projectConfig.EnvFile != "" {
+		if host == ":" {
-		envBytes, err := os.Open(filepath.Join(projectPath, projectConfig.EnvFile))
+			return nil, fmt.Errorf("invalid host %s", host)
-		if err != nil {
-			return nil, fmt.Errorf("failed to open env file: %v", err)
-		}
-		defer envBytes.Close()
-
-		envVars, err := godotenv.Parse(envBytes)
-		if err != nil {
-			return nil, fmt.Errorf("failed to parse env file: %v", err)
-		}
-
-		for key, value := range envVars {
-			projectConfig.Environment = append(projectConfig.Environment, fmt.Sprintf("%s=%s", key, value))
 		}
 	}
 
+	safeImageName := strings.ReplaceAll(imageName, "/", "_")
+	containerName := fmt.Sprintf("flux_%s-%s-%s", safeImageName, projectName, time.Now().Format("20060102-150405"))
+
 	logger.Debugw("Creating container", zap.String("container_id", containerName))
+	mounts := make([]mount.Mount, len(vols))
+	volumes := make(map[string]struct{}, len(vols))
+	for i, volume := range vols {
+		volumes[volume.VolumeID] = struct{}{}
+
+		mounts[i] = mount.Mount{
+			Type:     mount.TypeVolume,
+			Source:   volume.VolumeID,
+			Target:   volume.Mountpoint,
+			ReadOnly: false,
+		}
+	}
+
 	resp, err := Flux.dockerClient.ContainerCreate(ctx, &container.Config{
-		Image: imageName,
+		Image:   imageName,
-		Env:   projectConfig.Environment,
+		Env:     environment,
-		Volumes: map[string]struct{}{
+		Volumes: volumes,
-			vol.VolumeID: {},
-		},
 	},
 		&container.HostConfig{
 			RestartPolicy: container.RestartPolicy{Name: container.RestartPolicyUnlessStopped},
 			NetworkMode:   "bridge",
-			Mounts: []mount.Mount{
+			Mounts:        mounts,
-				{
+			ExtraHosts:    hosts,
-					Type:     mount.TypeVolume,
-					Source:   vol.VolumeID,
-					Target:   vol.Mountpoint,
-					ReadOnly: false,
-				},
-			},
 		},
 		nil,
 		nil,
@@ -108,59 +103,91 @@ func CreateDockerContainer(ctx context.Context, imageName, projectPath string, p
 
 	c := &Container{
 		ContainerID: [64]byte([]byte(resp.ID)),
-		Volumes:     []Volume{*vol},
+		Volumes:     vols,
 	}
 
 	return c, nil
 }
 
-func CreateContainer(ctx context.Context, imageName, projectPath string, projectConfig pkg.ProjectConfig, head bool, deployment *Deployment) (c *Container, err error) {
+// Create a container given a container configuration and a deployment. This will do a few things:
-	logger.Debugw("Creating container with image", zap.String("image", imageName))
+// 1. Create the container in the docker daemon
+// 2. Create the volumes for the container
+// 3. Insert the container and volumes into the database
+func CreateContainer(ctx context.Context, container *pkg.Container, projectName string, head bool, deployment *Deployment) (c *Container, err error) {
+	if container.Name == "" {
+		return nil, fmt.Errorf("container name is empty")
+	}
 
-	if projectConfig.EnvFile != "" {
+	if container.ImageName == "" {
-		envBytes, err := os.Open(filepath.Join(projectPath, projectConfig.EnvFile))
+		return nil, fmt.Errorf("container image name is empty")
-		if err != nil {
+	}
-			return nil, fmt.Errorf("failed to open env file: %v", err)
-		}
-		defer envBytes.Close()
 
-		envVars, err := godotenv.Parse(envBytes)
+	logger.Debugw("Creating container with image", zap.String("image", container.ImageName))
+
+	var volumes []*Volume
+	// in the head container, we have a default volume where the project is mounted, this is important so that if the project uses sqlite for example,
+	// all the data will not be lost the second the containers turns off.
+	if head {
+		vol, err := CreateDockerVolume(ctx)
 		if err != nil {
-			return nil, fmt.Errorf("failed to parse env file: %v", err)
+			return nil, err
 		}
 
-		for key, value := range envVars {
+		vol.Mountpoint = "/workspace"
-			projectConfig.Environment = append(projectConfig.Environment, fmt.Sprintf("%s=%s", key, value))
+
+		volumes = append(volumes, vol)
+	}
+
+	for _, containerVolume := range container.Volumes {
+		vol, err := CreateDockerVolume(ctx)
+		if err != nil {
+			return nil, err
+		}
+
+		if containerVolume.Mountpoint == "" {
+			return nil, fmt.Errorf("mountpoint is empty")
+		}
+
+		if containerVolume.Mountpoint == "/workspace" || containerVolume.Mountpoint == "/" {
+			return nil, fmt.Errorf("invalid mountpoint")
+		}
+
+		vol.Mountpoint = containerVolume.Mountpoint
+		volumes = append(volumes, vol)
+	}
+
+	// if the container is the head, build a list of hostnames that the container can reach by name for this deployment
+	// TODO: this host list should be consistent across all containers in the deployment, not just the head
+	var hosts []string
+	if head {
+		for _, container := range deployment.Containers {
+			containerName, err := container.GetIp()
+			if err != nil {
+				return nil, err
+			}
+
+			hosts = append(hosts, fmt.Sprintf("%s:%s", container.Name, containerName))
 		}
 	}
 
-	var vol *Volume
+	// if the container is not the head, pull the image from docker hub
-	vol, err = CreateDockerVolume(ctx)
+	if !head {
+		image, err := Flux.dockerClient.ImagePull(ctx, container.ImageName, image.PullOptions{})
+		if err != nil {
+			logger.Errorw("Failed to pull image", zap.Error(err))
+			return nil, err
+		}
+
+		// blcok untile the image is pulled
+		io.Copy(io.Discard, image)
+	}
+
+	c, err = CreateDockerContainer(ctx, container.ImageName, projectName, volumes, container.Environment, hosts)
 	if err != nil {
 		return nil, err
 	}
 
-	vol.Mountpoint = "/workspace"
+	c.Name = container.Name
-
-	if volumeInsertStmt == nil {
-		volumeInsertStmt, err = Flux.db.Prepare("INSERT INTO volumes (volume_id, mountpoint, container_id) VALUES (?, ?, ?) RETURNING id, volume_id, mountpoint, container_id")
-		if err != nil {
-			logger.Errorw("Failed to prepare statement", zap.Error(err))
-			return nil, err
-		}
-	}
-
-	c, err = CreateDockerContainer(ctx, imageName, projectPath, projectConfig, vol)
-	if err != nil {
-		return nil, err
-	}
-
-	if containerInsertStmt == nil {
-		containerInsertStmt, err = Flux.db.Prepare("INSERT INTO containers (container_id, head, deployment_id) VALUES ($1, $2, $3) RETURNING id, container_id, head, deployment_id")
-		if err != nil {
-			return nil, err
-		}
-	}
 
 	var containerIDString string
 	err = containerInsertStmt.QueryRow(c.ContainerID[:], head, deployment.ID).Scan(&c.ID, &containerIDString, &c.Head, &c.DeploymentID)
@@ -169,11 +196,32 @@ func CreateContainer(ctx context.Context, imageName, projectPath string, project
 	}
 	copy(c.ContainerID[:], containerIDString)
 
-	err = volumeInsertStmt.QueryRow(vol.VolumeID, vol.Mountpoint, c.ContainerID[:]).Scan(&vol.ID, &vol.VolumeID, &vol.Mountpoint, &vol.ContainerID)
+	tx, err := Flux.db.Begin()
 	if err != nil {
 		return nil, err
 	}
 
+	volumeInsertStmt, err := tx.Prepare("INSERT INTO volumes (volume_id, mountpoint, container_id) VALUES (?, ?, ?) RETURNING id, volume_id, mountpoint, container_id")
+	if err != nil {
+		logger.Errorw("Failed to prepare statement", zap.Error(err))
+		tx.Rollback()
+		return nil, err
+	}
+
+	for _, vol := range c.Volumes {
+		err = volumeInsertStmt.QueryRow(vol.VolumeID, vol.Mountpoint, c.ContainerID[:]).Scan(&vol.ID, &vol.VolumeID, &vol.Mountpoint, &vol.ContainerID)
+		if err != nil {
+			tx.Rollback()
+			return nil, err
+		}
+	}
+
+	err = tx.Commit()
+	if err != nil {
+		tx.Rollback()
+		return nil, err
+	}
+
 	c.Deployment = deployment
 	if head {
 		deployment.Head = c
@@ -183,28 +231,29 @@ func CreateContainer(ctx context.Context, imageName, projectPath string, project
 	return c, nil
 }
 
-func (c *Container) Upgrade(ctx context.Context, imageName, projectPath string, projectConfig pkg.ProjectConfig) (*Container, error) {
+func (c *Container) Upgrade(ctx context.Context, imageName, projectPath string, projectConfig *pkg.ProjectConfig) (*Container, error) {
 	// Create new container with new image
 	logger.Debugw("Upgrading container", zap.ByteString("container_id", c.ContainerID[:12]))
 	if c.Volumes == nil {
 		return nil, fmt.Errorf("no volumes found for container %s", c.ContainerID[:12])
 	}
 
-	vol := &c.Volumes[0]
+	var hosts []string
+	for _, container := range c.Deployment.Containers {
+		containerJSON, err := Flux.dockerClient.ContainerInspect(context.Background(), string(container.ContainerID[:]))
+		if err != nil {
+			return nil, err
+		}
 
-	newContainer, err := CreateDockerContainer(ctx, imageName, projectPath, projectConfig, vol)
+		hosts = containerJSON.HostConfig.ExtraHosts
+	}
+
+	newContainer, err := CreateDockerContainer(ctx, imageName, projectConfig.Name, c.Volumes, projectConfig.Environment, hosts)
 	if err != nil {
 		return nil, err
 	}
 	newContainer.Deployment = c.Deployment
 
-	if containerInsertStmt == nil {
-		containerInsertStmt, err = Flux.db.Prepare("INSERT INTO containers (container_id, head, deployment_id) VALUES ($1, $2, $3) RETURNING id, container_id, head, deployment_id")
-		if err != nil {
-			return nil, err
-		}
-	}
-
 	var containerIDString string
 	err = containerInsertStmt.QueryRow(newContainer.ContainerID[:], c.Head, c.Deployment.ID).Scan(&newContainer.ID, &containerIDString, &newContainer.Head, &newContainer.DeploymentID)
 	if err != nil {
@@ -213,22 +262,102 @@ func (c *Container) Upgrade(ctx context.Context, imageName, projectPath string,
 	}
 	copy(newContainer.ContainerID[:], containerIDString)
 
-	if volumeUpdateStmt == nil {
+	tx, err := Flux.db.Begin()
-		volumeUpdateStmt, err = Flux.db.Prepare("UPDATE volumes SET container_id = ? WHERE id = ? RETURNING id, volume_id, mountpoint, container_id")
+	if err != nil {
+		logger.Errorw("Failed to begin transaction", zap.Error(err))
+		return nil, err
+	}
+
+	volumeUpdateStmt, err := tx.Prepare("UPDATE volumes SET container_id = ? WHERE id = ? RETURNING id, volume_id, mountpoint, container_id")
+	if err != nil {
+		tx.Rollback()
+		return nil, err
+	}
+
+	for _, vol := range newContainer.Volumes {
+		err = volumeUpdateStmt.QueryRow(newContainer.ContainerID[:], vol.ID).Scan(&vol.ID, &vol.VolumeID, &vol.Mountpoint, &vol.ContainerID)
 		if err != nil {
+			tx.Rollback()
+			logger.Error("Failed to update volume", zap.Error(err))
 			return nil, err
 		}
 	}
 
-	vol = &newContainer.Volumes[0]
+	err = tx.Commit()
-	volumeUpdateStmt.QueryRow(newContainer.ContainerID[:], vol.ID).Scan(&vol.ID, &vol.VolumeID, &vol.Mountpoint, &vol.ContainerID)
+	if err != nil {
+		tx.Rollback()
+		return nil, err
+	}
 
 	logger.Debug("Upgraded container")
 
 	return newContainer, nil
 }
 
-func (c *Container) Start(ctx context.Context) error {
+// initial indicates if the container was just created, because if not, we need to fix the extra hsots since it's not guaranteed that the supplemental containers have the same ip
+// as they had when the deployment was previously on
+func (c *Container) Start(ctx context.Context, initial bool) error {
+	if !initial && c.Head {
+		containerJSON, err := Flux.dockerClient.ContainerInspect(ctx, string(c.ContainerID[:]))
+		if err != nil {
+			return err
+		}
+
+		// remove yourself
+		Flux.dockerClient.ContainerRemove(ctx, string(c.ContainerID[:]), container.RemoveOptions{})
+
+		var volumes map[string]struct{} = make(map[string]struct{})
+		var hosts []string
+		var mounts []mount.Mount
+
+		for _, volume := range c.Volumes {
+			volumes[volume.VolumeID] = struct{}{}
+
+			mounts = append(mounts, mount.Mount{
+				Type:     mount.TypeVolume,
+				Source:   volume.VolumeID,
+				Target:   volume.Mountpoint,
+				ReadOnly: false,
+			})
+		}
+
+		for _, supplementalContainer := range c.Deployment.Containers {
+			if supplementalContainer.Head {
+				continue
+			}
+
+			ip, err := supplementalContainer.GetIp()
+			if err != nil {
+				return err
+			}
+
+			hosts = append(hosts, fmt.Sprintf("%s:%s", supplementalContainer.Name, ip))
+		}
+
+		// recreate yourself
+		resp, err := Flux.dockerClient.ContainerCreate(ctx, &container.Config{
+			Image:   containerJSON.Image,
+			Env:     containerJSON.Config.Env,
+			Volumes: volumes,
+		},
+			&container.HostConfig{
+				RestartPolicy: container.RestartPolicy{Name: container.RestartPolicyUnlessStopped},
+				NetworkMode:   "bridge",
+				Mounts:        mounts,
+				ExtraHosts:    hosts,
+			},
+			nil,
+			nil,
+			c.Name,
+		)
+		if err != nil {
+			return err
+		}
+
+		c.ContainerID = [64]byte([]byte(resp.ID))
+		Flux.db.Exec("UPDATE containers SET container_id = ? WHERE id = ?", c.ContainerID[:], c.ID)
+	}
+
 	return Flux.dockerClient.ContainerStart(ctx, string(c.ContainerID[:]), container.StartOptions{})
 }
 
@@ -236,6 +365,7 @@ func (c *Container) Stop(ctx context.Context) error {
 	return Flux.dockerClient.ContainerStop(ctx, string(c.ContainerID[:]), container.StopOptions{})
 }
 
+// Stop and remove a container and all of its volumes
 func (c *Container) Remove(ctx context.Context) error {
 	err := RemoveDockerContainer(ctx, string(c.ContainerID[:]))
 
@@ -280,16 +410,37 @@ func (c *Container) Wait(ctx context.Context, port uint16) error {
 	return WaitForDockerContainer(ctx, string(c.ContainerID[:]), port)
 }
 
-func (c *Container) Status(ctx context.Context) (string, error) {
+type ContainerStatus struct {
+	Status   string
+	ExitCode int
+}
+
+func (c *Container) Status(ctx context.Context) (*ContainerStatus, error) {
 	containerJSON, err := Flux.dockerClient.ContainerInspect(ctx, string(c.ContainerID[:]))
+	if err != nil {
+		return nil, err
+	}
+
+	containerStatus := &ContainerStatus{
+		Status:   containerJSON.State.Status,
+		ExitCode: containerJSON.State.ExitCode,
+	}
+
+	return containerStatus, nil
+}
+
+func (c *Container) GetIp() (string, error) {
+	containerJSON, err := Flux.dockerClient.ContainerInspect(context.Background(), string(c.ContainerID[:]))
 	if err != nil {
 		return "", err
 	}
 
-	return containerJSON.State.Status, nil
+	ip := containerJSON.NetworkSettings.IPAddress
+
+	return ip, nil
 }
 
-// RemoveContainer stops and removes a container, but be warned that this will not remove the container from the database
+// Stops and deletes a container from the docker daemon
 func RemoveDockerContainer(ctx context.Context, containerID string) error {
 	if err := Flux.dockerClient.ContainerStop(ctx, containerID, container.StopOptions{}); err != nil {
 		return fmt.Errorf("failed to stop container (%s): %v", containerID[:12], err)

internal/server/deploy.go

@@ -9,9 +9,12 @@ import (
 	"io"
 	"mime/multipart"
 	"net/http"
+	"os"
 	"os/exec"
+	"path/filepath"
 	"sync"
 
+	"github.com/joho/godotenv"
 	"github.com/juls0730/flux/pkg"
 	"go.uber.org/zap"
 )
@@ -103,7 +106,7 @@ func (s *FluxServer) DeployHandler(w http.ResponseWriter, r *http.Request) {
 	}
 	defer deployRequest.Config.Close()
 
-	var projectConfig pkg.ProjectConfig
+	projectConfig := new(pkg.ProjectConfig)
 	if err := json.NewDecoder(deployRequest.Config).Decode(&projectConfig); err != nil {
 		logger.Errorw("Failed to decode config", zap.Error(err))
 
@@ -221,12 +224,42 @@ func (s *FluxServer) DeployHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	// Streams the each line of the pipe into the eventChannel, this closes the pipe when the function exits
+	// We need to pre-process EnvFile since docker has no concept of where the file is, or anything like that, so we have to read from it,
-	var pipeGroup sync.WaitGroup
+	// and place all of it's content into the environment field so that docker can find it later
+	if projectConfig.EnvFile != "" {
+		envBytes, err := os.Open(filepath.Join(projectPath, projectConfig.EnvFile))
+		if err != nil {
+			logger.Errorw("Failed to open env file", zap.Error(err))
+			eventChannel <- DeploymentEvent{
+				Stage:      "error",
+				Message:    fmt.Sprintf("Failed to open env file: %v", err),
+				StatusCode: http.StatusInternalServerError,
+			}
+			return
+		}
+		defer envBytes.Close()
 
+		envVars, err := godotenv.Parse(envBytes)
+		if err != nil {
+			logger.Errorw("Failed to parse env file", zap.Error(err))
+			eventChannel <- DeploymentEvent{
+				Stage:      "error",
+				Message:    fmt.Sprintf("Failed to parse env file: %v", err),
+				StatusCode: http.StatusInternalServerError,
+			}
+			return
+		}
+
+		for key, value := range envVars {
+			projectConfig.Environment = append(projectConfig.Environment, fmt.Sprintf("%s=%s", key, value))
+		}
+	}
+
+	pipeGroup := sync.WaitGroup{}
 	streamPipe := func(pipe io.ReadCloser) {
 		pipeGroup.Add(1)
 		defer pipeGroup.Done()
+		defer pipe.Close()
 
 		scanner := bufio.NewScanner(pipe)
 		for scanner.Scan() {
@@ -252,29 +285,12 @@ func (s *FluxServer) DeployHandler(w http.ResponseWriter, r *http.Request) {
 		Message: "Preparing project",
 	}
 
+	reader, writer := io.Pipe()
+
 	prepareCmd := exec.Command("go", "generate")
 	prepareCmd.Dir = projectPath
-	cmdOut, err := prepareCmd.StdoutPipe()
+	prepareCmd.Stdout = writer
-	if err != nil {
+	prepareCmd.Stderr = writer
-		logger.Errorw("Failed to get stdout pipe", zap.Error(err))
-		eventChannel <- DeploymentEvent{
-			Stage:      "error",
-			Message:    fmt.Sprintf("Failed to get stdout pipe: %s", err),
-			StatusCode: http.StatusInternalServerError,
-		}
-
-		return
-	}
-	cmdErr, err := prepareCmd.StderrPipe()
-	if err != nil {
-		logger.Errorw("Failed to get stderr pipe", zap.Error(err))
-		eventChannel <- DeploymentEvent{
-			Stage:      "error",
-			Message:    fmt.Sprintf("Failed to get stderr pipe: %s", err),
-			StatusCode: http.StatusInternalServerError,
-		}
-		return
-	}
 
 	err = prepareCmd.Start()
 	if err != nil {
@@ -288,8 +304,7 @@ func (s *FluxServer) DeployHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	go streamPipe(cmdOut)
+	go streamPipe(reader)
-	go streamPipe(cmdErr)
 
 	pipeGroup.Wait()
 
@@ -305,37 +320,20 @@ func (s *FluxServer) DeployHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	writer.Close()
+
 	eventChannel <- DeploymentEvent{
 		Stage:   "building",
 		Message: "Building project image",
 	}
 
+	reader, writer = io.Pipe()
 	logger.Debugw("Building image for project", zap.String("name", projectConfig.Name))
 	imageName := fmt.Sprintf("flux_%s-image", projectConfig.Name)
 	buildCmd := exec.Command("pack", "build", imageName, "--builder", s.config.Builder)
 	buildCmd.Dir = projectPath
-	cmdOut, err = buildCmd.StdoutPipe()
+	buildCmd.Stdout = writer
-	if err != nil {
+	buildCmd.Stderr = writer
-		logger.Errorw("Failed to get stdout pipe", zap.Error(err))
-		eventChannel <- DeploymentEvent{
-			Stage:      "error",
-			Message:    fmt.Sprintf("Failed to get stdout pipe: %s", err),
-			StatusCode: http.StatusInternalServerError,
-		}
-
-		return
-	}
-	cmdErr, err = buildCmd.StderrPipe()
-	if err != nil {
-		logger.Errorw("Failed to get stderr pipe", zap.Error(err))
-		eventChannel <- DeploymentEvent{
-			Stage:      "error",
-			Message:    fmt.Sprintf("Failed to get stderr pipe: %s", err),
-			StatusCode: http.StatusInternalServerError,
-		}
-
-		return
-	}
 
 	err = buildCmd.Start()
 	if err != nil {
@@ -349,8 +347,7 @@ func (s *FluxServer) DeployHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	go streamPipe(cmdOut)
+	go streamPipe(reader)
-	go streamPipe(cmdErr)
 
 	pipeGroup.Wait()
 
@@ -375,28 +372,19 @@ func (s *FluxServer) DeployHandler(w http.ResponseWriter, r *http.Request) {
 
 	if app == nil {
 		app, err = CreateApp(ctx, imageName, projectPath, projectConfig)
-		if err != nil {
-			logger.Errorw("Failed to create app", zap.Error(err))
-			eventChannel <- DeploymentEvent{
-				Stage:      "error",
-				Message:    fmt.Sprintf("Failed to create app: %s", err),
-				StatusCode: http.StatusInternalServerError,
-			}
-
-			return
-		}
 	} else {
-		err = app.Upgrade(ctx, projectConfig, imageName, projectPath)
+		err = app.Upgrade(ctx, imageName, projectPath, projectConfig)
-		if err != nil {
+	}
-			logger.Errorw("Failed to upgrade app", zap.Error(err))
-			eventChannel <- DeploymentEvent{
-				Stage:      "error",
-				Message:    fmt.Sprintf("Failed to upgrade app: %s", err),
-				StatusCode: http.StatusInternalServerError,
-			}
 
-			return
+	if err != nil {
+		logger.Errorw("Failed to deploy app", zap.Error(err))
+		eventChannel <- DeploymentEvent{
+			Stage:      "error",
+			Message:    fmt.Sprintf("Failed to upgrade app: %s", err),
+			StatusCode: http.StatusInternalServerError,
 		}
+
+		return
 	}
 
 	eventChannel <- DeploymentEvent{
@@ -455,7 +443,7 @@ func (s *FluxServer) StopDeployHandler(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
-	if status == "stopped" {
+	if status == "stopped" || status == "failed" {
 		http.Error(w, "App is already stopped", http.StatusBadRequest)
 		return
 	}
@@ -525,5 +513,6 @@ func (s *FluxServer) DaemonInfoHandler(w http.ResponseWriter, r *http.Request) {
 	w.Header().Set("Content-Type", "application/json")
 	json.NewEncoder(w).Encode(pkg.Info{
 		Compression: s.config.Compression,
+		Version:     pkg.Version,
 	})
 }

internal/server/deployment.go

@@ -22,20 +22,12 @@ type Deployment struct {
 	Port       uint16           `json:"port"`
 }
 
-// Creates a deployment and containers in the database
+// Creates a deployment row in the database, containting the URL the app should be hosted on (it's public hostname)
+// and the port that the web server is listening on
 func CreateDeployment(port uint16, appUrl string, db *sql.DB) (*Deployment, error) {
 	var deployment Deployment
-	var err error
 
-	if deploymentInsertStmt == nil {
+	err := deploymentInsertStmt.QueryRow(appUrl, port).Scan(&deployment.ID, &deployment.URL, &deployment.Port)
-		deploymentInsertStmt, err = db.Prepare("INSERT INTO deployments (url, port) VALUES ($1, $2) RETURNING id, url, port")
-		if err != nil {
-			logger.Errorw("Failed to prepare statement", zap.Error(err))
-			return nil, err
-		}
-	}
-
-	err = deploymentInsertStmt.QueryRow(appUrl, port).Scan(&deployment.ID, &deployment.URL, &deployment.Port)
 	if err != nil {
 		logger.Errorw("Failed to insert deployment", zap.Error(err))
 		return nil, err
@@ -44,12 +36,14 @@ func CreateDeployment(port uint16, appUrl string, db *sql.DB) (*Deployment, erro
 	return &deployment, nil
 }
 
-func (deployment *Deployment) Upgrade(ctx context.Context, projectConfig pkg.ProjectConfig, imageName string, projectPath string) error {
+// Takes an existing deployment, and gracefully upgrades the app to a new image
+func (deployment *Deployment) Upgrade(ctx context.Context, projectConfig *pkg.ProjectConfig, imageName string, projectPath string) error {
 	existingContainers, err := findExistingDockerContainers(ctx, projectConfig.Name)
 	if err != nil {
 		return fmt.Errorf("failed to find existing containers: %v", err)
 	}
 
+	// we only upgrade the head container, in the future we might want to allow upgrading supplemental containers, but this should work just fine for now.
 	container, err := deployment.Head.Upgrade(ctx, imageName, projectPath, projectConfig)
 	if err != nil {
 		logger.Errorw("Failed to upgrade container", zap.Error(err))
@@ -61,7 +55,7 @@ func (deployment *Deployment) Upgrade(ctx context.Context, projectConfig pkg.Pro
 	deployment.Containers = append(deployment.Containers, container)
 
 	logger.Debugw("Starting container", zap.ByteString("container_id", container.ContainerID[:12]))
-	err = container.Start(ctx)
+	err = container.Start(ctx, true)
 	if err != nil {
 		logger.Errorw("Failed to start container", zap.Error(err))
 		return err
@@ -77,7 +71,7 @@ func (deployment *Deployment) Upgrade(ctx context.Context, projectConfig pkg.Pro
 		return err
 	}
 
-	// Create a new proxy that points to the new head, and replace the old one, but ensure that the old one is gracefully shutdown
+	// Create a new proxy that points to the new head, and replace the old one, but ensure that the old one is gracefully drained of connections
 	oldProxy := deployment.Proxy
 	deployment.Proxy, err = deployment.NewDeploymentProxy()
 	if err != nil {
@@ -93,6 +87,7 @@ func (deployment *Deployment) Upgrade(ctx context.Context, projectConfig pkg.Pro
 
 	var containers []*Container
 	var oldContainers []*Container
+	// delete the old head container from the database, and update the deployment's container list
 	for _, container := range deployment.Containers {
 		if existingContainers[string(container.ContainerID[:])] {
 			logger.Debugw("Deleting container from db", zap.ByteString("container_id", container.ContainerID[:12]))
@@ -117,6 +112,7 @@ func (deployment *Deployment) Upgrade(ctx context.Context, projectConfig pkg.Pro
 		return err
 	}
 
+	// gracefully shutdown the old proxy, or if it doesnt exist, just remove the containers
 	if oldProxy != nil {
 		go oldProxy.GracefulShutdown(oldContainers)
 	} else {
@@ -132,6 +128,7 @@ func (deployment *Deployment) Upgrade(ctx context.Context, projectConfig pkg.Pro
 	return nil
 }
 
+// Remove a deployment and all of it's containers
 func (d *Deployment) Remove(ctx context.Context) error {
 	for _, container := range d.Containers {
 		err := container.Remove(ctx)
@@ -154,7 +151,7 @@ func (d *Deployment) Remove(ctx context.Context) error {
 
 func (d *Deployment) Start(ctx context.Context) error {
 	for _, container := range d.Containers {
-		err := container.Start(ctx)
+		err := container.Start(ctx, false)
 		if err != nil {
 			logger.Errorf("Failed to start container (%s): %v\n", container.ContainerID[:12], err)
 			return err
@@ -184,8 +181,10 @@ func (d *Deployment) Stop(ctx context.Context) error {
 	return nil
 }
 
+// return the status of a deployment, either "running", "failed", "stopped", or "pending", errors if not all
+// containers are in the same state
 func (d *Deployment) Status(ctx context.Context) (string, error) {
-	var status string
+	var status *ContainerStatus
 	if d == nil {
 		return "", fmt.Errorf("deployment is nil")
 	}
@@ -202,17 +201,22 @@ func (d *Deployment) Status(ctx context.Context) (string, error) {
 		}
 
 		// if not all containers are in the same state
-		if status != "" && status != containerStatus {
+		if status != nil && status.Status != containerStatus.Status {
 			return "", fmt.Errorf("malformed deployment")
 		}
 
 		status = containerStatus
 	}
 
-	switch status {
+	switch status.Status {
 	case "running":
 		return "running", nil
 	case "exited":
+		if status.ExitCode != 0 {
+			// non-zero exit code in unix terminology means the program did no complete successfully
+			return "failed", nil
+		}
+
 		return "stopped", nil
 	default:
 		return "pending", nil

internal/server/proxy.go

@@ -14,18 +14,22 @@ import (
 )
 
 type Proxy struct {
+	// map[string]*Deployment
 	deployments sync.Map
 }
 
+// Stops forwarding traffic to a deployment
 func (p *Proxy) RemoveDeployment(deployment *Deployment) {
 	p.deployments.Delete(deployment.URL)
 }
 
+// Starts forwarding traffic to a deployment. The deployment must be ready to recieve requests before this is called.
 func (p *Proxy) AddDeployment(deployment *Deployment) {
 	logger.Debugw("Adding deployment", zap.String("url", deployment.URL))
 	p.deployments.Store(deployment.URL, deployment)
 }
 
+// This function is responsible for taking an http request and forwarding it to the correct deployment
 func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	host := r.Host
 
@@ -35,6 +39,7 @@ func (p *Proxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 		return
 	}
 
+	// on response from the server, this is decremented
 	atomic.AddInt64(&deployment.(*Deployment).Proxy.activeRequests, 1)
 
 	deployment.(*Deployment).Proxy.proxy.ServeHTTP(w, r)
@@ -47,6 +52,7 @@ type DeploymentProxy struct {
 	activeRequests int64
 }
 
+// Creates a proxy for a given deployment
 func (deployment *Deployment) NewDeploymentProxy() (*DeploymentProxy, error) {
 	if deployment == nil {
 		return nil, fmt.Errorf("deployment is nil")
@@ -68,7 +74,11 @@ func (deployment *Deployment) NewDeploymentProxy() (*DeploymentProxy, error) {
 
 	proxy := &httputil.ReverseProxy{
 		Director: func(req *http.Request) {
-			req.URL = containerUrl
+			req.URL = &url.URL{
+				Scheme: containerUrl.Scheme,
+				Host:   containerUrl.Host,
+				Path:   req.URL.Path,
+			}
 			req.Host = containerUrl.Host
 		},
 		Transport: &http.Transport{
@@ -90,6 +100,7 @@ func (deployment *Deployment) NewDeploymentProxy() (*DeploymentProxy, error) {
 	}, nil
 }
 
+// Drains connections from a proxy
 func (dp *DeploymentProxy) GracefulShutdown(oldContainers []*Container) {
 	ctx, cancel := context.WithTimeout(context.Background(), dp.gracePeriod)
 	defer cancel()

internal/server/server.go

@@ -77,6 +77,11 @@ func NewFluxServer() *FluxServer {
 		logger.Fatalw("Failed to create database schema", zap.Error(err))
 	}
 
+	err = PrepareDBStatements(db)
+	if err != nil {
+		logger.Fatalw("Failed to prepare database statements", zap.Error(err))
+	}
+
 	return &FluxServer{
 		db:           db,
 		proxy:        &Proxy{},
@@ -106,12 +111,12 @@ func NewServer() *FluxServer {
 	config.Level = zap.NewAtomicLevelAt(zapcore.Level(verbosity))
 
 	lameLogger, err := config.Build()
-	logger = lameLogger.Sugar()
-
 	if err != nil {
 		logger.Fatalw("Failed to create logger", zap.Error(err))
 	}
 
+	logger = lameLogger.Sugar()
+
 	Flux = NewFluxServer()
 	Flux.Logger = logger
 
@@ -152,7 +157,7 @@ func NewServer() *FluxServer {
 		logger.Fatalw("Failed to pull builder image", zap.Error(err))
 	}
 
-	// blocking wait for the iamge to be pulled
+	// blocking until the iamge is pulled
 	io.Copy(io.Discard, events)
 
 	logger.Infow("Successfully pulled builder image", zap.String("image", serverConfig.Builder))
@@ -170,15 +175,17 @@ func NewServer() *FluxServer {
 
 	go func() {
 		logger.Infof("Proxy server starting on http://127.0.0.1:%s", port)
-		if err := http.ListenAndServe(fmt.Sprintf(":%s", port), Flux.proxy); err != nil && err != http.ErrServerClosed {
+		if err := http.ListenAndServe(fmt.Sprintf(":%s", port), Flux.proxy); err != nil {
-			logger.Fatalw("Proxy server error", zap.Error(err))
+			logger.Fatalw("Failed to start proxy server", zap.Error(err))
 		}
 	}()
 
 	return Flux
 }
 
-func (s *FluxServer) UploadAppCode(code io.Reader, projectConfig pkg.ProjectConfig) (string, error) {
+// Handler for uploading a project to the server. We have to upload the entire project since we need to build the
+// project ourselves to work with the buildpacks
+func (s *FluxServer) UploadAppCode(code io.Reader, projectConfig *pkg.ProjectConfig) (string, error) {
 	var err error
 	projectPath := filepath.Join(s.rootDir, "apps", projectConfig.Name)
 	if err = os.MkdirAll(projectPath, 0755); err != nil {
@@ -251,3 +258,25 @@ func (s *FluxServer) UploadAppCode(code io.Reader, projectConfig pkg.ProjectConf
 
 	return projectPath, nil
 }
+
+// TODO: split each prepare statement into its coresponding module so the statememnts are easier to fine
+func PrepareDBStatements(db *sql.DB) error {
+	var err error
+	appInsertStmt, err = db.Prepare("INSERT INTO apps (name, deployment_id) VALUES ($1, $2) RETURNING id, name, deployment_id")
+	if err != nil {
+		return fmt.Errorf("failed to prepare statement: %v", err)
+	}
+
+	containerInsertStmt, err = db.Prepare("INSERT INTO containers (container_id, head, deployment_id) VALUES (?, ?, ?) RETURNING id, container_id, head, deployment_id")
+	if err != nil {
+		return err
+	}
+
+	deploymentInsertStmt, err = db.Prepare("INSERT INTO deployments (url, port) VALUES ($1, $2) RETURNING id, url, port")
+	if err != nil {
+		logger.Errorw("Failed to prepare statement", zap.Error(err))
+		return err
+	}
+
+	return nil
+}

pkg/config.go

@@ -1,9 +1,29 @@
 package pkg
 
-type ProjectConfig struct {
+type Volume struct {
+	Mountpoint string `json:"mountpoint,omitempty"`
+}
+
+type Container struct {
 	Name        string   `json:"name,omitempty"`
-	Url         string   `json:"url,omitempty"`
+	ImageName   string   `json:"image,omitempty"`
-	Port        uint16   `json:"port,omitempty"`
+	Volumes     []Volume `json:"volumes,omitempty"`
-	EnvFile     string   `json:"env_file,omitempty"`
 	Environment []string `json:"environment,omitempty"`
 }
+
+type ProjectConfig struct {
+	// name of the app
+	Name string `json:"name,omitempty"`
+	// public url of the app
+	// TODO: support multiple urls
+	Url string `json:"url,omitempty"`
+	// Port the web app listens on from the head container
+	Port    uint16 `json:"port,omitempty"`
+	EnvFile string `json:"env_file,omitempty"`
+	// additional environment variables
+	Environment []string `json:"environment,omitempty"`
+	// volumes for the head container
+	Volumes []Volume `json:"volumes,omitempty"`
+	// config for supplemental containersm
+	Containers []Container `json:"containers,omitempty"`
+}

pkg/responses.go

@@ -7,6 +7,7 @@ type App struct {
 	DeploymentStatus string `json:"deployment_status,omitempty"`
 }
 
+// TODO: this should be flattened to an int, where 0 = disabled and any other number is the level
 type Compression struct {
 	Enabled bool `json:"enabled"`
 	Level   int  `json:"level,omitempty"`
@@ -14,6 +15,7 @@ type Compression struct {
 
 type Info struct {
 	Compression Compression `json:"compression"`
+	Version     string      `json:"version"`
 }
 
 type DeploymentEvent struct {

pkg/version.go

@@ -0,0 +1,3 @@
+package pkg
+
+const Version = "2bd953d"

zqdgr.config.json

@@ -7,6 +7,7 @@
     "scripts": {
         "build:daemon": "go build -o fluxd cmd/fluxd/main.go",
         "build:cli": "go build -o flux cmd/flux/main.go",
+        "build:all": "go build -o fluxd cmd/fluxd/main.go && go build -o flux cmd/flux/main.go",
         "run:daemon": "go run cmd/fluxd/main.go",
         "run:cli": "go run cmd/flux/main.go"
     },