README update

README.md

@@ -8,11 +8,29 @@ Gocage can handle multiple datastores, so you can have jails on HDD storage and
 
 From v0.33b, due to multi ZFS pool support, gocage is no longer 100% compatible with iocage.  
 Zfs datasets now should be specified with the ZFS pool. e.g. :  
-<code>
+<pre><code>
 Config.Jail_zfs = 1
 Config.Jail_zfs_dataset = myzfspool/poudriere
 Config.Jail_zfs_mountpoint = none
-</code>
+</code></pre>
+
+Create jails
+------------
+For now, we can't pass config at creation time. We have to define config after creation:  
+<pre><code>
+gocage create jail1 -r 13.2-RELEASE
+gocage set Config.Ip4_addr="vnet0|192.168.1.91/24" Config.Vnet=1 jail1
+</code></pre>
+
+Create basejail (jail based on a template, system will be nullfs read-only mounted from the template):
+<pre><code>
+# First create basetpl from 13.2-RELEASE, set it as a template
+gocage create -r 13.2-RELEASE basetpl
+gocage set Config.Jailtype="template" basetpl
+# Then create basejail1 based on basetpl
+gocage create -b basetpl basejail1
+</code></pre>
+
 
 List jails
 ----------
@@ -132,6 +150,14 @@ To update jail patch version, use gocage update :
 `gocage update test`
 
 
+Upgrade jails  
+----------
+To upgrade jail to newer release, use gocage upgrade :  
+`gocage upgrade -r 13.2-RELEASE test`
+  
+A pre-upgrade snapshot wil be made so we can rollback if needed.  
+
+
 Delete jails  
 ----------
 `gocage destroy test`
@@ -217,7 +243,6 @@ gocage fetch -r 12.3 -o iocage --from file:/iocage/download
 
 TODO
 ----------
-gocage upgrade  
-gocage create  
+gocage create from templates  
 gocage init  
-  create default pool with defaults.json
+  create default pool with defaults.json  

cmd/console.go

@@ -12,7 +12,7 @@ import (
 func ShellJail(args []string) error {
 	// We cant shell more than one jail bc we replace gocage execution with jexec, so there wont be no return to gocage
 	if len(args) > 0 {
-		cj, err := getJailFromArray(args[0], gJails)
+		cj, err := getJailFromArray(args[0], []string{"basejail", "jail"}, gJails)
 		if err != nil {
 			fmt.Printf("Error getting jail %s: %v\n", args[0], err)
 			return err
@@ -30,10 +30,9 @@ func shellJail(jail *Jail) error {
 	if false == jail.Running {
 		return errors.New("Jail is not running")
 	}
-	
+
 	jid := strconv.Itoa(jail.JID)
 
-	//err := syscall.Exec("/usr/sbin/jexec", []string{"jexec", jid, "/bin/csh"}, os.Environ())
 	err := syscall.Exec("/usr/sbin/jexec", []string{"jexec", jid, "login", "-f", "root"}, os.Environ())
 
 	// We should never get here, as syscall.Exec replace the gocage binary execution with jexec
@@ -41,6 +40,6 @@ func shellJail(jail *Jail) error {
 	if err != nil {
 		log.Printf("Exec returned %v\n", err)
 	}
-	
+
 	return nil
 }

cmd/create.go

@@ -0,0 +1,285 @@
+package cmd
+
+import (
+	"os"
+	"fmt"
+	//"log"
+	"time"
+	"errors"
+	"strings"
+	cp "github.com/otiai10/copy"
+	log "github.com/sirupsen/logrus"
+)
+
+// TODO : Add a flag to specify which parts of freebsd base we want : Slim jail only need base.txz, neither lib32 nor src.txz
+func CreateJail(args []string) {
+	var err error
+	var jtype []string
+
+	if len(gCreateArgs.JailType) > 0 {
+		jtype = []string{gCreateArgs.JailType}
+	}
+	
+	for _, jname := range args {
+		// Check if jail exist and is distinctly named
+		_, err = getJailFromArray(jname, jtype, gJails)
+		if err != nil {
+			if strings.EqualFold(err.Error(), "Jail not found") {
+				
+			} else {
+				fmt.Printf("ERROR: %s\n", err.Error())
+				return
+			}
+		} else {
+			fmt.Printf("Jail exist: %s\n", jname)
+			continue
+		}
+
+		fmt.Printf("  > create jail %s\n", jname)
+
+		var ds *Datastore
+		if len(gCreateArgs.Datastore) > 0 {
+			fmt.Printf("DEBUG: Use %s datastore\n", gCreateArgs.Datastore)
+			ds, err = getDatastoreFromArray(gCreateArgs.Datastore, gDatastores)
+			if err != nil {
+				fmt.Printf("ERROR Getting datastore: %s\n", gCreateArgs.Datastore, err.Error())
+				return
+			}
+		} else {
+			ds = &gDatastores[0]
+		}
+
+		// Get base template if specified
+		if len(gCreateArgs.BaseTemplate) > 0 {
+			/**************************************************************************
+			*  Create based jail from a template
+			*/
+			log.Debugf("Jail will be created from a base template\n")
+			bj, err := getJailFromArray(gCreateArgs.BaseTemplate, []string{"template"}, gJails)
+			if err != nil {
+				if strings.EqualFold(err.Error(), "Jail not found") {
+					fmt.Printf("Template not found\n")
+					return
+				} else {
+					fmt.Printf("ERROR: %s\n", err.Error())
+					return
+				}
+			}
+			// Create jail datasets
+			dstDset := fmt.Sprintf("%s/jails/%s", ds.ZFSDataset, jname)
+			fmt.Printf("    > Initialize dataset %s\n", dstDset)
+			err = zfsCreateDataset(dstDset, "", "")
+			if err != nil {
+				fmt.Printf("ERROR creating dataset %s: %s\n", dstDset, err.Error())
+				return
+			}
+			// Create jail root datasets
+			dstRootDset := fmt.Sprintf("%s/jails/%s/root", ds.ZFSDataset, jname)
+			fmt.Printf("    > Initialize dataset %s\n", dstRootDset)
+			err = zfsCreateDataset(dstRootDset, "", "")
+			if err != nil {
+				fmt.Printf("ERROR creating dataset %s: %s\n", dstRootDset, err.Error())
+				return
+			}
+
+			// Create needed directories with basejail permissions
+			fmt.Printf("    > Create base read-only directories\n")
+			dstRootDir := fmt.Sprintf("%s/jails/%s/root", ds.Mountpoint, jname)
+			for _, d := range append(gBaseDirs, gEmptyDirs...) {
+				dstPath := dstRootDir
+				srcPath := bj.RootPath
+				for _, cd := range strings.Split(d, "/") {
+					srcPath = fmt.Sprintf("%s/%s", srcPath, cd)
+					dstPath = fmt.Sprintf("%s/%s", dstPath, cd)
+					_, err := os.Stat(dstPath)
+					if errors.Is(err, os.ErrNotExist) {
+						srcPerm, err := getPermissions(srcPath)
+						if err != nil {
+							fmt.Printf("ERROR getting permissions of %s: %s\n", srcPath, err.Error())
+							return
+						}
+						err = os.Mkdir(dstPath, srcPerm.Mode().Perm())
+						if err != nil {
+							fmt.Printf("ERROR creating directory %s: %s\n", dstPath, err.Error())
+							return
+						}
+					}
+				}
+			}
+
+			// Copy these from basejail
+			fmt.Printf("    > Create base writable directories\n")
+			for _, d := range gCopyDirs {
+				err := cp.Copy(fmt.Sprintf("%s/%s", bj.RootPath, d), fmt.Sprintf("%s/%s", dstRootDir, d))
+				if err != nil {
+					fmt.Printf("ERROR copying %s to %s: %s\n", fmt.Sprintf("%s/%s", bj.RootPath, d),
+						   fmt.Sprintf("%s/%s", dstRootDir, d), err.Error())
+					return
+				}
+			}
+
+			///////////////////////////////////////////////////////////////////////
+			// Copy defaults.json...
+			jailConfPath := fmt.Sprintf("%s/jails/%s/config.json", ds.Mountpoint, jname)
+			err = copyFile(fmt.Sprintf("%s/defaults.json", ds.Mountpoint),
+				       jailConfPath)
+			if err != nil {
+				fmt.Printf("ERROR creating config.json: %s\n", err.Error())
+				return
+			}
+			///////////////////////////////////////////////////////////////////////
+			// ... and update it
+			jailConf, err := getJailConfig(jailConfPath)
+			if err != nil {
+				log.Println("ERROR reading jail config from %s", jailConfPath)
+			}
+
+			// Build jail object from config
+			jailRootPath := fmt.Sprintf("%s/jails/%s/%s", ds.Mountpoint, jname, "root")
+			j := Jail{
+				Name:       jailConf.Host_hostuuid,
+				Config:     jailConf,
+				ConfigPath: jailConfPath,
+				Datastore:  ds.Name,
+				RootPath:   jailRootPath,
+				Running:    false,
+			}
+
+			// We need to store the basejail template. We could :
+			// 1. Use "origin" ?
+			// 2. Add a json item to config ("basejail_template" p.e.), but iocage would delete it once jail is started from iocage
+			// 3. Add a gocage specific config ("config.gocage.json" p.e.)
+			j.Config.Jailtype = "basejail"
+			j.Config.Origin = bj.Name
+			j.Config.Host_hostname = jname
+			j.Config.Host_hostuuid = jname
+
+			j.WriteConfigToDisk(false)
+
+			///////////////////////////////////////////////////////////////////////
+			// Create fstab
+			fstabHandle, err := os.Create(fmt.Sprintf("%s/jails/%s/fstab", ds.Mountpoint, jname))
+			if err != nil {
+				fmt.Printf("ERROR creating fstab: %s", err.Error())
+				return
+			}
+			defer fstabHandle.Close()
+
+			for _, d := range gBaseDirs {
+				fmt.Fprintf(fstabHandle, "%s\t%s\tnullfs\tro\t0\t0\n", fmt.Sprintf("%s/%s", bj.RootPath, d), fmt.Sprintf("%s/%s", dstRootDir, d))
+			}
+
+			fmt.Printf("  > Jail created!\n")
+		} else {
+			/**************************************************************************
+			 *  Create normal jail with its own freebsd base
+			 */
+			log.Debugf("Creating jail with its own freebsd base\n")
+
+			// First check if we got release on the same datastore
+			_, err := os.Stat(fmt.Sprintf("%s/releases/%s/root", ds.Mountpoint, gCreateArgs.Release))
+			if os.IsNotExist(err) {
+				fmt.Printf("ERROR: Release locally not available. Run \"gocage fetch\"\n")
+				return
+			}
+
+			///////////////////////////////////////////////////////////////////////
+			// Create and populate jail filesystem from release
+			dstDset := fmt.Sprintf("%s/jails/%s", ds.ZFSDataset, jname)
+			fmt.Printf("    > Initialize dataset %s\n", dstDset)
+			sNow := time.Now().Format("20060102150405")
+			reldset := fmt.Sprintf("%s/releases/%s", ds.ZFSDataset, gCreateArgs.Release)
+			err = zfsSnapshot(reldset, sNow)
+			if err != nil {
+				fmt.Printf("ERROR Creating snapshot of %s: %s\n", reldset, err.Error())
+				return
+			}
+
+			err = zfsCopy(fmt.Sprintf("%s@%s", reldset, sNow), dstDset)
+			if err != nil {
+				fmt.Printf("ERROR sending snapshot to %s: %s\n", dstDset, err.Error())
+				return
+			}
+			// Remove snapshot of release, then snapshot of destination dataset
+			err = zfsDestroy(fmt.Sprintf("%s@%s", reldset, sNow))
+			if err != nil {
+				fmt.Printf("ERROR destroying snapshot %s: %s\n", reldset, err.Error())
+				return
+			}
+			err = zfsDestroy(fmt.Sprintf("%s@%s", dstDset, sNow))
+			if err != nil {
+				fmt.Printf("ERROR destroying snapshot %s: %s\n", dstDset, err.Error())
+				return
+			}
+
+			dstRootDset := fmt.Sprintf("%s/jails/%s/root", ds.ZFSDataset, jname)
+			fmt.Printf("    > Initialize dataset %s\n", dstRootDset)
+			relrootdset := fmt.Sprintf("%s/releases/%s/root", ds.ZFSDataset, gCreateArgs.Release)
+			err = zfsSnapshot(relrootdset, sNow)
+			if err != nil {
+				fmt.Printf("ERROR Creating snapshot of %s: %s\n", relrootdset, err.Error())
+				return
+			}
+			err = zfsCopy(fmt.Sprintf("%s@%s", relrootdset, sNow), dstRootDset)
+			if err != nil {
+				fmt.Printf("ERROR sending snapshot to %s: %s\n", dstRootDset, err.Error())
+				return
+			}
+			// Remove snapshot of release, then snapshot of destination dataset
+			err = zfsDestroy(fmt.Sprintf("%s@%s", relrootdset, sNow))
+			if err != nil {
+				fmt.Printf("ERROR destroying snapshot %s: %s\n", relrootdset, err.Error())
+				return
+			}
+			err = zfsDestroy(fmt.Sprintf("%s@%s", dstRootDset, sNow))
+			if err != nil {
+				fmt.Printf("ERROR destroying snapshot %s: %s\n", dstRootDset, err.Error())
+				return
+			}
+
+			///////////////////////////////////////////////////////////////////////
+			// Copy defaults.json...
+			jailConfPath := fmt.Sprintf("%s/jails/%s/config.json", ds.Mountpoint, jname)
+			err = copyFile(fmt.Sprintf("%s/defaults.json", ds.Mountpoint), 
+					jailConfPath)
+			if err != nil {
+				fmt.Printf("ERROR creating config.json: %s\n", err.Error())
+				return
+			}
+			///////////////////////////////////////////////////////////////////////
+			// ... and update it
+			jailConf, err := getJailConfig(jailConfPath)
+			if err != nil {
+				log.Println("ERROR reading jail config from %s", jailConfPath)
+			}
+
+			// Build jail object from config
+			jailRootPath := fmt.Sprintf("%s/jails/%s/%s", ds.Mountpoint, jname, "root")
+			j := Jail{
+				Name:       jailConf.Host_hostuuid,
+				Config:     jailConf,
+				ConfigPath: jailConfPath,
+				Datastore:  ds.Name,
+				RootPath:   jailRootPath,
+				Running:    false,
+			}
+
+			j.Config.Release = gCreateArgs.Release
+			j.Config.Host_hostname = jname
+			j.Config.Host_hostuuid = jname
+			j.Config.Jailtype = "jail"
+						
+			j.WriteConfigToDisk(false)
+
+			///////////////////////////////////////////////////////////////////////
+			// Create fstab
+			fstabHandle, err := os.Create(fmt.Sprintf("%s/jails/%s/fstab", ds.Mountpoint, jname))
+			if err != nil {
+				fmt.Printf("ERROR creating fstab: %s", err.Error())
+				return
+			}
+			defer fstabHandle.Close()
+			fmt.Printf("  > Jail created!\n")
+		}
+	}
+}

cmd/destroy.go

@@ -3,13 +3,14 @@ package cmd
 import (
 	"fmt"
 	//"log"
+	"time"
 	//"errors"
 	"strings"
 )
 
 func DestroyJails(args []string) {
 	for _, a := range args {
-		cj, err := getJailFromArray(a, gJails)
+		cj, err := getJailFromArray(a, []string{""}, gJails)
 		if err != nil {
 			fmt.Printf("Error getting jail: %s\n", err)
 			return
@@ -27,6 +28,8 @@ func DestroyJails(args []string) {
 			}
 			fmt.Printf("Stopping jail %s\n", cj.Name)
 			StopJail([]string{fmt.Sprintf("%s/%s", cj.Datastore, cj.Name)})
+			// Give some time to the host OS to free all mounts accessing processes
+			time.Sleep(1 * time.Second)
 		}
 		
 		// Get root and config datasets, then destroy
@@ -35,7 +38,7 @@ func DestroyJails(args []string) {
 			fmt.Printf("Error getting root dataset: %s\n", err)
 			return
 		}
-		fmt.Printf("DEBUG: Prepare to zfs destroy %s\n", dsRootName)
+		//fmt.Printf("DEBUG: Prepare to zfs destroy %s\n", dsRootName)
 		if err = zfsDestroy(dsRootName); err != nil {
 			fmt.Printf("Error deleting root dataset: %s\n", err)
 			return
@@ -46,12 +49,11 @@ func DestroyJails(args []string) {
 			fmt.Printf("Error getting config dataset: %s\n", err)
 			return
 		}
-		fmt.Printf("DEBUG: Prepare to zfs destroy %s\n", dsConfName)
+		//fmt.Printf("DEBUG: Prepare to zfs destroy %s\n", dsConfName)
 		if err = zfsDestroy(dsConfName); err != nil {
 			fmt.Printf("Error deleting config dataset: %s\n", err)
 			return
 		}
-
-		//TODO: Delete jail named directory
+		fmt.Printf("Jail %s is no more!\n", cj.Name)
 	}
 }

cmd/fetch.go

@@ -23,7 +23,9 @@ const (
 )
 
 var (
-	FetchFiles = []string{"base.txz", "lib32.txz", "src.txz"}
+	// TODO : Make this a config/cmd line setting
+	//FetchFiles = []string{"base.txz", "lib32.txz", "src.txz"}
+	FetchFiles = []string{"base.txz"}
 )
 
 // TODO: Check if files already exist
@@ -66,9 +68,9 @@ func fetchRelease(release string, proto string, arch string, datastore string, f
 		}
 	}
 	
-	// Create download/XX.X-RELEASE dataset if necessary
-	thisDownloadDsName := fmt.Sprintf("%s/%s-RELEASE", downloadDsName, release)
-	thisDownloadDsMountPoint := fmt.Sprintf("%s/%s-RELEASE", downloadDsMountPoint, release)
+	// Create download/XX.X dataset if necessary
+	thisDownloadDsName := fmt.Sprintf("%s/%s", downloadDsName, release)
+	thisDownloadDsMountPoint := fmt.Sprintf("%s/%s", downloadDsMountPoint, release)
 	exist, err = doZfsDatasetExist(thisDownloadDsName)
 	if err != nil {
 		return fmt.Errorf("Error accessing dataset %s: %v\n", thisDownloadDsName, err)
@@ -82,9 +84,9 @@ func fetchRelease(release string, proto string, arch string, datastore string, f
 
 	var fetchUrl string
 	if len(fetchFrom) > 0 {
-		fetchUrl = fmt.Sprintf("%s/%s-RELEASE", fetchFrom, release)
+		fetchUrl = fmt.Sprintf("%s/%s", fetchFrom, release)
 	} else {
-		fetchUrl = fmt.Sprintf("%s://%s/%s/%s/%s-RELEASE", proto, ReleaseServer, ReleaseRootDir, arch, release)
+		fetchUrl = fmt.Sprintf("%s://%s/%s/%s/%s", proto, ReleaseServer, ReleaseRootDir, arch, release)
 	}
 	log.Debugf("FetchURL = %s", fetchUrl)
 	
@@ -153,9 +155,9 @@ func extractRelease(release string, datastore string) {
 		}
 	}
 	
-	// Create releases/XX.X-RELEASE dataset if necessary
-	thisReleaseDsName := fmt.Sprintf("%s/%s-RELEASE", releaseDsName, release)
-	thisReleaseDsMountPoint := fmt.Sprintf("%s/%s-RELEASE", releaseDsMountPoint, release)
+	// Create releases/XX.X dataset if necessary
+	thisReleaseDsName := fmt.Sprintf("%s/%s", releaseDsName, release)
+	thisReleaseDsMountPoint := fmt.Sprintf("%s/%s", releaseDsMountPoint, release)
 	exist, err = doZfsDatasetExist(thisReleaseDsName)
 	if err != nil {
 		fmt.Printf("Error accessing dataset %s: %v\n", thisReleaseDsName, err)
@@ -169,7 +171,7 @@ func extractRelease(release string, datastore string) {
 		}
 	}
 	
-	// Create releases/XX.X-RELEASE/root dataset if necessary
+	// Create releases/XX.X/root dataset if necessary
 	thisReleaseRootDsName := fmt.Sprintf("%s/root", thisReleaseDsName)
 	thisReleaseRootDsMountPoint := fmt.Sprintf("%s/root", thisReleaseDsMountPoint)
 	exist, err = doZfsDatasetExist(thisReleaseRootDsName)
@@ -185,9 +187,9 @@ func extractRelease(release string, datastore string) {
 		}
 	}
 	
-	// Now extract download/$RELEASE/*.txz to releases/XX.X-RELEASE/root
+	// Now extract download/$RELEASE/*.txz to releases/XX.X/root
 	downloadDsMountPoint := fmt.Sprintf("%s/download", ds.Mountpoint)
-	downloadDir := fmt.Sprintf("%s/%s-RELEASE", downloadDsMountPoint, release)
+	downloadDir := fmt.Sprintf("%s/%s", downloadDsMountPoint, release)
 	
 	d, err := os.Open(downloadDir)
 	defer d.Close()
@@ -201,67 +203,69 @@ func extractRelease(release string, datastore string) {
 		return
 	}
 	
-	// Extract every .txz files
+	// Extract every .txz files in FetchFiles
 	for _, fi := range files {
 		if false == fi.IsDir() {
 			if strings.HasSuffix(fi.Name(), ".txz") {
-				ar := fmt.Sprintf("%s/%s", downloadDir, fi.Name())
-				fmt.Printf("Extracting file %s... ", ar)
-				// pure Go method, sorry this is so slow. Also I did not handle permissions in this
-/*				f, err := os.Open(ar)
-				defer f.Close()
-				if err != nil {
-					fmt.Printf("Can not open %s: %v\n", ar, err)
-					return
-				}
-				// xz reader
-				r, err := xz.NewReader(f)
-				if err != nil {
-					fmt.Printf("Can not read %s: %v\n", ar, err)
-					return
-				}
-				// tar reader
-				tr := tar.NewReader(r)
-				// Iterate through the files in the archive.
-				for {
-					hdr, err := tr.Next()
-					if err == io.EOF {
-						// end of tar archive
-						break
-					}
+				if isStringInArray(FetchFiles, fi.Name()) {
+					ar := fmt.Sprintf("%s/%s", downloadDir, fi.Name())
+					fmt.Printf("Extracting file %s to %s... ", ar, thisReleaseRootDsMountPoint)
+					// pure Go method, sorry this is so slow. Also I did not handle permissions in this
+	/*				f, err := os.Open(ar)
+					defer f.Close()
 					if err != nil {
-						log.Fatal(err)
+						fmt.Printf("Can not open %s: %v\n", ar, err)
+						return
 					}
-					switch hdr.Typeflag {
-						case tar.TypeDir:
-							// create a directory
-							dest := fmt.Sprintf("%s/%s", thisReleaseRootDsMountPoint, hdr.Name)
-							// FIXME: Access rights?
-							err = os.MkdirAll(dest, 0777)
-							if err != nil {
-								log.Fatal(err)
-							}
-						case tar.TypeReg, tar.TypeRegA:
-							// write a file
-							dest := fmt.Sprintf("%s/%s", thisReleaseRootDsMountPoint, hdr.Name)
-							w, err := os.Create(dest)
-							defer w.Close()
-							if err != nil {
-								log.Fatal(err)
-							}
-							_, err = io.Copy(w, tr)
-							if err != nil {
-								log.Fatal(err)
-							}
+					// xz reader
+					r, err := xz.NewReader(f)
+					if err != nil {
+						fmt.Printf("Can not read %s: %v\n", ar, err)
+						return
+					}
+					// tar reader
+					tr := tar.NewReader(r)
+					// Iterate through the files in the archive.
+					for {
+						hdr, err := tr.Next()
+						if err == io.EOF {
+							// end of tar archive
+							break
+						}
+						if err != nil {
+							log.Fatal(err)
+						}
+						switch hdr.Typeflag {
+							case tar.TypeDir:
+								// create a directory
+								dest := fmt.Sprintf("%s/%s", thisReleaseRootDsMountPoint, hdr.Name)
+								// FIXME: Access rights?
+								err = os.MkdirAll(dest, 0777)
+								if err != nil {
+									log.Fatal(err)
+								}
+							case tar.TypeReg, tar.TypeRegA:
+								// write a file
+								dest := fmt.Sprintf("%s/%s", thisReleaseRootDsMountPoint, hdr.Name)
+								w, err := os.Create(dest)
+								defer w.Close()
+								if err != nil {
+									log.Fatal(err)
+								}
+								_, err = io.Copy(w, tr)
+								if err != nil {
+									log.Fatal(err)
+								}
+						}
+					}
+	*/
+					cmd := fmt.Sprintf("/usr/bin/tar xpf %s -C %s", ar, thisReleaseRootDsMountPoint)
+					out, err := executeCommand(cmd)
+					if err != nil && len(out) > 0 {
+						fmt.Printf("Error: %v: %s\n", err, out)
+					} else {
+						fmt.Printf("Done\n")
 					}
-				}
-*/
-				cmd := fmt.Sprintf("/usr/bin/tar xpf %s -C %s", ar, thisReleaseRootDsMountPoint)
-				out, err := executeCommand(cmd)
-				if err != nil && len(out) > 0 {
-					fmt.Printf("Error: %v: %s\n", err, out)
-				} else {
-					fmt.Printf("Done\n")
 				}
 			}
 		}

cmd/freebsd-update.conf.go

@@ -0,0 +1,85 @@
+package cmd
+
+const (
+	fbsdUpdateConfig = `
+	# $FreeBSD$
+	
+	# Trusted keyprint.  Changing this is a Bad Idea unless you've received
+	# a PGP-signed email from <security-officer@FreeBSD.org> telling you to
+	# change it and explaining why.
+	KeyPrint 800651ef4b4c71c27e60786d7b487188970f4b4169cc055784e21eb71d410cc5
+	
+	# Server or server pool from which to fetch updates.  You can change
+	# this to point at a specific server if you want, but in most cases
+	# using a "nearby" server won't provide a measurable improvement in
+	# performance.
+	ServerName update.FreeBSD.org
+	
+	# Components of the base system which should be kept updated.
+	Components world
+	
+	# Example for updating the userland and the kernel source code only:
+	# Components src/base src/sys world
+	
+	# Paths which start with anything matching an entry in an IgnorePaths
+	# statement will be ignored.
+	IgnorePaths
+	
+	# Paths which start with anything matching an entry in an IDSIgnorePaths
+	# statement will be ignored by "freebsd-update IDS".
+	IDSIgnorePaths /usr/share/man/cat
+	IDSIgnorePaths /usr/share/man/whatis
+	IDSIgnorePaths /var/db/locate.database
+	IDSIgnorePaths /var/log
+	
+	# Paths which start with anything matching an entry in an UpdateIfUnmodified
+	# statement will only be updated if the contents of the file have not been
+	# modified by the user (unless changes are merged; see below).
+	UpdateIfUnmodified /etc/ /var/ /root/ /.cshrc /.profile
+	
+	# When upgrading to a new FreeBSD release, files which match MergeChanges
+	# will have any local changes merged into the version from the new release.
+	MergeChanges /etc/
+	
+	### Default configuration options:
+	
+	# Directory in which to store downloaded updates and temporary
+	# files used by FreeBSD Update.
+	WorkDir /iocage/freebsd-update
+	
+	# Destination to send output of "freebsd-update cron" if an error
+	# occurs or updates have been downloaded.
+	# MailTo root
+	
+	# Is FreeBSD Update allowed to create new files?
+	# AllowAdd yes
+	
+	# Is FreeBSD Update allowed to delete files?
+	# AllowDelete yes
+	
+	# If the user has modified file ownership, permissions, or flags, should
+	# FreeBSD Update retain this modified metadata when installing a new version
+	# of that file?
+	# KeepModifiedMetadata yes
+	
+	# When upgrading between releases, should the list of Components be
+	# read strictly (StrictComponents yes) or merely as a list of components
+	# which *might* be installed of which FreeBSD Update should figure out
+	# which actually are installed and upgrade those (StrictComponents no)?
+	StrictComponents yes
+	
+	# When installing a new kernel perform a backup of the old one first
+	# so it is possible to boot the old kernel in case of problems.
+	BackupKernel no
+	
+	# If BackupKernel is enabled, the backup kernel is saved to this
+	# directory.
+	# BackupKernelDir /boot/kernel.old
+	
+	# When backing up a kernel also back up debug symbol files?
+	BackupKernelSymbolFiles no
+	
+	# Create a new boot environment when installing patches
+	CreateBootEnv no
+	`
+) 

cmd/list.go

@@ -52,7 +52,7 @@ func ListJails(args []string, display bool) {
 	for _, ds := range gDatastores {
 		listJailsFromDatastore(ds, args, display)
 	}
-	
+
 	// Only when displaying jails, we accept to process multiple same name jails
 	if false == display {
 		for _, j := range gJails {
@@ -68,11 +68,11 @@ func ListJails(args []string, display bool) {
 					break
 				}
 			}
-			
+
 			if true == skip {
 				continue
 			}
-			
+
 			// Initialize if not found in nameChecked
 			if false == found {
 				curCheck = &uniqueJailName{jail: j.Name,
@@ -81,11 +81,11 @@ func ListJails(args []string, display bool) {
 			} else {
 				found = false
 			}
-			
+
 			if countOfJailsWithThisName(j.Name) > 1 {
 				//fmt.Printf("DEBUG: Jail %s exist multiple times, now checking if specified with full name\n", j.Name)
 				curCheck.unique = false
-				
+
 				for _, a := range args {
 					//fmt.Printf("DEBUG: comparing %s/%s with %s\n", j.Datastore, j.Name, a)
 					if strings.EqualFold(a, fmt.Sprintf("%s/%s", j.Datastore, j.Name)) {
@@ -96,7 +96,7 @@ func ListJails(args []string, display bool) {
 			}
 			nameChecked = append(nameChecked, curCheck)
 		}
-		
+
 		// Now check
 		for _, a := range args {
 			for _, n := range nameChecked {
@@ -107,9 +107,9 @@ func ListJails(args []string, display bool) {
 			}
 		}
 	}
-	
+
 	fields := strings.Split(gDisplayJColumns, ",")
-	
+
 	// This is the structure we will filter, then display
 	var jails []Jail
 
@@ -249,7 +249,7 @@ func listJailsFromDirectory(dir string, dsname string) ([]Jail, error) {
 			jailConfPath := fmt.Sprintf("%s/%s/%s", dir, fi.Name(), "config.json")
 			jailConf, err := getJailConfig(jailConfPath)
 			if err != nil {
-				log.Println("ERROR reading jail config for %s", jailConfPath)
+				fmt.Printf("ERROR reading jail config from %s\n", jailConfPath)
 			}
 
 			// 2. Build jail object from config
@@ -275,6 +275,13 @@ func listJailsFromDirectory(dir string, dsname string) ([]Jail, error) {
 					// FIXME ??? Shouldn't be ioc-$Name ?
 					j.InternalName = rj.Name
 					j.Devfs_ruleset = rj.Devfs_ruleset
+					// Update release
+					r, err := getVersion(&j)
+					if err != nil {
+						fmt.Printf("ERROR getting jail %s version: %s\n", j.Name, err.Error())
+					} else {
+						j.Config.Release = r
+					}
 					break
 				}
 			}

cmd/migrate.go

@@ -26,7 +26,7 @@ func MigrateJail(args []string) {
 	}
 
 	for _, jn := range jailNames {
-		cj, err := getJailFromArray(jn, gJails)
+		cj, err := getJailFromArray(jn, []string{""}, gJails)
 		if cj == nil {
 			fmt.Printf("Error getting jail %s: Not found\n", jn)
 			return
@@ -177,7 +177,7 @@ func CleanMigrateMess(args []string) error {
 	}
 
 	for _, jn := range jailNames {
-		cj, err := getJailFromArray(jn, gJails)
+		cj, err := getJailFromArray(jn, []string{""}, gJails)
 		if cj == nil {
 			return errors.New(fmt.Sprintf("Error getting jail %s: Not found\n", jn))
 		}

cmd/properties.go

@@ -17,7 +17,7 @@ func GetJailProperties(args []string) {
 		for i, a := range args {
 			// Last arg is the jail name
 			if i == len(args)-1 {
-				jail, err = getJailFromArray(a, gJails)
+				jail, err = getJailFromArray(a, []string{""}, gJails)
 				if err != nil {
 					fmt.Printf("Error: %s\n", err.Error())
 					return
@@ -118,6 +118,7 @@ func SetJailProperties(args []string) {
 					gJails[i].ConfigUpdated = true
 				}
 			}
+			writeConfigToDisk(&gJails[i], false)
 		}
 	}
 }

cmd/root.go

@@ -6,7 +6,6 @@ import (
 	"sync"
 	"strings"
 	"io/ioutil"
-	"encoding/json"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -15,12 +14,19 @@ import (
 )
 
 const (
-	gVersion = "0.35"
+	gVersion = "0.38"
 	
 	// TODO : Get from $jail_zpool/defaults.json
 	MIN_DYN_DEVFS_RULESET = 1000
 )
 
+type createArgs struct {
+	Release      string
+	BaseTemplate string
+	Datastore    string
+	JailType     string
+}
+
 var (
 	gJailHost   JailHost
 	gJails      []Jail
@@ -29,6 +35,8 @@ var (
 	gUseSudo bool
 	gForce   bool
 	gDebug   bool
+	
+	gCreateArgs      createArgs
 
 	gConfigFile      string
 	gDisplayJColumns string
@@ -55,6 +63,15 @@ var (
 	gFetchRelease    string
 	gFetchIntoDS     string
 	gFetchFrom       string
+	gUpgradeRelease  string
+
+	// For a based jail, these are directories binded to basejail
+	gBaseDirs = []string{"bin", "boot", "lib", "libexec", "rescue", "sbin", "usr/bin", "usr/include",
+		"usr/lib", "usr/lib32", "usr/libdata", "usr/libexec", "usr/sbin", "usr/share"}
+	// These directories are to be created empty
+	gEmptyDirs = []string{"dev", "media", "mnt", "net", "proc"}
+	// Copy these from base template
+	gCopyDirs = []string{"etc", "root", "tmp", "var"}
 
 	gMdevfs          sync.Mutex
 
@@ -79,6 +96,20 @@ It support iocage jails and can coexist with iocage.`,
 			fmt.Printf("GoCage v.%s on FreeBSD %d.%d-%s\n", gVersion, fv.major, fv.minor, fv.flavor)
 		},
 	}
+	
+	/* TODO
+	Initialize datastore(s) /iocage, /iocage/jails
+	Put defaults.json, update it with hostid,interfaces, and maybe other necessary fields
+	Initialize bridge
+	initCmd = &cobra.Command{
+		Use:   "init",
+		Short: "Initialize GoCage",
+		//Long:  `Let this show you how much fail I had to get this *cough* perfect`,
+		Run: func(cmd *cobra.Command, args []string) {
+			fv, _ := getFreeBSDVersion()
+			fmt.Printf("GoCage v.%s on FreeBSD %d.%d-%s\n", gVersion, fv.major, fv.minor, fv.flavor)
+		},
+	}*/
 
 	listCmd = &cobra.Command{
 		Use:   "list",
@@ -136,12 +167,9 @@ ex: gocage list srv-db srv-web`,
 			} else {
 				StartJail(args)
 			}
-			WriteConfigToDisk("", false, false)
 		},
 	}
-	
-	
-	
+
 	restartCmd = &cobra.Command{
 		Use:   "restart",
 		Short: "restart jail",
@@ -150,10 +178,9 @@ ex: gocage list srv-db srv-web`,
 			ListJails(args, false)
 			StopJail(args)
 			StartJail(args)
-			WriteConfigToDisk("", false, false)
 		},
 	}
-	
+
 	shellCmd = &cobra.Command {
 		Use:   "console",
 		Short: "Execute shell on jail",
@@ -163,7 +190,7 @@ ex: gocage list srv-db srv-web`,
 			ShellJail(args)
 		},
 	}
-	
+
 	setCmd = &cobra.Command{
 		Use:   "set",
 		Short: "Set a jail property",
@@ -173,7 +200,6 @@ Multiples properties can be specified, separated with space (Ex: gocage set allo
 			// Load inventory
 			ListJails(args, false)
 			SetJailProperties(args)
-			WriteConfigToDisk("", true, false)
 		},
 	}
 
@@ -255,7 +281,6 @@ You can specify multiple jails.`,
 			// Load inventory
 			ListJails(args, false)
 			MigrateJail(args)
-			WriteConfigToDisk("", false, false)
 		},
 	}
 
@@ -291,7 +316,7 @@ You can specify multiple datastores.`,
 			ListDatastores(args, true)
 		},
 	}
-	
+
 	fetchCmd = &cobra.Command{
 		Use:   "fetch",
 		Short: "Fetch FreeBSD release to local datastore",
@@ -302,18 +327,36 @@ You can specify multiple datastores.`,
 			} else {
 				extractRelease(gFetchRelease, gFetchIntoDS)
 			}
-	 },
+		},
 	}
-	
-	UpdateCmd = &cobra.Command{
+
+	updateCmd = &cobra.Command{
 		Use:   "update",
-	 Short: "Update FreeBSD release",
-	 Run: func(cmd *cobra.Command, args []string) {
-		 ListJails(args, false)
-		 UpdateJail(args)
-	 },
+		Short: "Update FreeBSD release",
+		Run: func(cmd *cobra.Command, args []string) {
+			ListJails(args, false)
+			UpdateJail(args)
+		},
 	}
-	
+
+	upgradeCmd = &cobra.Command{
+		Use:   "upgrade",
+		Short: "Upgrade FreeBSD release",
+		Run: func(cmd *cobra.Command, args []string) {
+			ListJails(args, false)
+			UpgradeJail(args)
+		},
+	}
+
+	createCmd = &cobra.Command{
+		Use:   "create",
+		Short: "Create jail",
+		Run: func(cmd *cobra.Command, args []string) {
+			ListJails(args, false)
+			CreateJail(args)
+		},
+	}
+
 	testCmd = &cobra.Command{
 		Use:   "test",
 		Short: "temporary command to test some code snippet",
@@ -326,7 +369,7 @@ You can specify multiple datastores.`,
 // TODO : Init log level and log output
 func init() {
 	var err error
-	
+
 	cobra.OnInitialize(initConfig)
 
 	// Global switches
@@ -349,12 +392,12 @@ func init() {
 	snapshotListCmd.Flags().BoolVarP(&gNoSnapLineSep, "nolinesep", "l", false, "Do not display line separator between snapshots")
 	snapshotListCmd.Flags().StringVarP(&gFilterSnaps, "filter", "f", "none", "Only display snapshots with these values. Ex: \"gocage snapshot list -f Config.Boot=1\" will only list started on boot jails")
 	snapshotListCmd.Flags().StringVarP(&gSortSnapFields, "sort", "s", "none", "Display snapshots sorted by field values. Ex: \"gocage snapshot list -s +Jailname,-Used\" will sort snapshots by jail decreasing name, then increasing used space. 3 critera max supported.")
-	
+
 	datastoreListCmd.Flags().StringVarP(&gDisplayDColumns, "outcol", "o", "Name,Mountpoint,ZFSDataset,Available,Used,Referenced", "Show these columns in output")
 	datastoreListCmd.Flags().BoolVarP(&gNoDSLineSep, "nolinesep", "l", false, "Do not display line separator between datastores")
 	datastoreListCmd.Flags().StringVarP(&gFilterDS, "filter", "f", "none", "Only display datastores with these values. Ex: \"gocage datastore list -f Config.Boot=1\" will only list started on boot jails")
 	datastoreListCmd.Flags().StringVarP(&gSortDSFields, "sort", "s", "none", "Display datastores sorted by field values. Ex: \"gocage datastore list -s +Jailname,-Used\" will sort snapshots by jail decreasing name, then increasing used space. 3 critera max supported.")
-	
+
 	// This is local flag : Only available to gocage snapshot create command
 	snapshotCreateCmd.Flags().StringVarP(&gSnapshotName, "snapname", "n", "", "Name of the snapshot to create")
 	snapshotCreateCmd.MarkFlagRequired("snapname")
@@ -366,13 +409,19 @@ func init() {
 	migrateCmd.Flags().StringVarP(&gMigrateDestDatastore, "datastore", "d", "", "Path of destination datastore for jail (Ex: \"/iocage\")")
 	migrateCmd.Flags().BoolVarP(&gYesToAll, "yes", "y", false, "Answer yes to all questions")
 	migrateCmd.MarkFlagRequired("datastore")
-	
-	fetchCmd.Flags().StringVarP(&gFetchRelease, "release", "r", "", "Release to fetch (e.g.: \"13.1\"")
+
+	fetchCmd.Flags().StringVarP(&gFetchRelease, "release", "r", "", "Release to fetch (e.g.: \"13.1-RELEASE\"")
 	fetchCmd.Flags().StringVarP(&gFetchIntoDS, "datastore", "o", "", "Datastore release will be saved to")
 	fetchCmd.Flags().StringVarP(&gFetchFrom, "from", "d", "", "Repository to download from. Should contain XY.Z-RELEASE. File protocol supported")
 	fetchCmd.MarkFlagRequired("release")
 	fetchCmd.MarkFlagRequired("datastore")
-	
+
+	upgradeCmd.Flags().StringVarP(&gUpgradeRelease, "release", "r", "", "Release to upgrade to (e.g.: \"13.1-RELEASE\"")
+	upgradeCmd.MarkFlagRequired("release")
+
+	createCmd.Flags().StringVarP(&gCreateArgs.Release, "release", "r", "", "Release for the jail (e.g.: \"13.1-RELEASE\"")
+	createCmd.Flags().StringVarP(&gCreateArgs.BaseTemplate, "basetpl", "b", "", "Base template. This will create a jail based on basetpl, so every up(date|grade) made to basetpl will immediately propagate to new jail\n")
+	createCmd.Flags().StringVarP(&gCreateArgs.Datastore, "datastore", "d", "", "Datastore to create the jail on. Defaults to first declared in config.")
 
 	// Now declare commands
 	rootCmd.AddCommand(versionCmd)
@@ -389,7 +438,9 @@ func init() {
 	rootCmd.AddCommand(migrateCmd)
 	rootCmd.AddCommand(datastoreCmd)
 	rootCmd.AddCommand(fetchCmd)
-	rootCmd.AddCommand(UpdateCmd)
+	rootCmd.AddCommand(updateCmd)
+	rootCmd.AddCommand(upgradeCmd)
+	rootCmd.AddCommand(createCmd)
 	
 	rootCmd.AddCommand(testCmd)
 	
@@ -472,66 +523,6 @@ func initConfig() {
 	}
 }
 
-/********************************************************************************
- * Write jail(s) config which been updated to disk.
- * If name is specified, work on the jail. If name is empty string, work on all.
- * If changeauto not set, values which are in "auto" mode on disk
- *  won't be overwritten (p.ex defaultrouter wont be overwritten with current
- *  default route, so if route change on jailhost this will reflect on jail next
- *  start)
- *******************************************************************************/
-func WriteConfigToDisk(jailName string, changeauto bool, forceWrite bool) {
-	for _, j := range gJails {
-		if len(jailName) > 0 && j.Name == jailName || len(jailName) == 0 {
-			if j.ConfigUpdated || forceWrite {
-				log.Debug("%s config has changed, write changes to disk\n", j.Name)
-
-				// we will manipulate properties so get a copy
-				jc := j.Config
-
-				if changeauto == false {
-					// Overwrite "auto" properties
-					ondiskjc, err := getJailConfig(j.ConfigPath)
-					if err != nil {
-						panic(err)
-					}
-					// TODO : List all fields, then call getStructFieldValue to compare value with "auto"
-					// If "auto" then keep it that way before writing ondiskjc to disk
-					var properties []string
-					properties = getStructFieldNames(ondiskjc, properties, "")
-
-					for _, p := range properties {
-						v, _, err := getStructFieldValue(ondiskjc, p)
-						if err != nil {
-							panic(err)
-						}
-						if v.String() == "auto" {
-							err = setStructFieldValue(&jc, p, "auto")
-							if err != nil {
-								fmt.Printf("ERROR sanitizing config: %s\n", err.Error())
-								os.Exit(1)
-							}
-						}
-					}
-				}
-
-				marshaled, err := json.MarshalIndent(jc, "", "    ")
-				if err != nil {
-					fmt.Printf("ERROR marshaling config: %s\n", err.Error())
-				}
-				
-				//fmt.Printf("DEBUG: Will write config to disk, with content:\n")
-				//fmt.Printf(string(marshaled))
-				
-				if os.WriteFile(j.ConfigPath, []byte(marshaled), 0644); err != nil {
-					fmt.Printf("Error writing config file %s: %v\n", j.ConfigPath, err)
-					os.Exit(1)
-				}
-			}
-		}
-	}
-}
-
 
 func Execute() {
 	if err := rootCmd.Execute(); err != nil {

cmd/start.go

@@ -330,8 +330,10 @@ func configureDhcpOrAcceptRtadv(jail *Jail, ipproto int, enable bool) error {
 
 	for _, n := range nics {
 		// vnet0 is epair0b inside jail
-		if strings.Contains(n, "vnet") {
-			n = fmt.Sprintf("%sb", strings.Replace(n, "vnet", "epair", 1))
+		//if strings.Contains(n, "vnet") {
+		if strings.HasPrefix(n, "vnet") {
+			splitd := strings.Split(n, "|")
+			n = fmt.Sprintf("%sb", strings.Replace(splitd[0], "vnet", "epair", 1))
 		}
 		key := fmt.Sprintf("ifconfig_%s", n)
 		value := "SYNCDHCP"
@@ -342,12 +344,12 @@ func configureDhcpOrAcceptRtadv(jail *Jail, ipproto int, enable bool) error {
 		}
 
 		if enable == true {
-			err := enableRcKeyValue(jail.ConfigPath, key, value)
+			err := enableRcKeyValue(fmt.Sprintf("%s/etc/rc.conf", jail.RootPath), key, value)
 			if err != nil {
 				return fmt.Errorf("ERROR setting %s=%s with sysrc for jail %s: %s\n", key, value, jail.Name, err)
 			}
 		} else {
-			err := disableRcKey(jail.ConfigPath, key)
+			err := disableRcKey(fmt.Sprintf("%s/etc/rc.conf", jail.RootPath), key)
 			if err != nil {
 				return fmt.Errorf("ERROR deleting %s with sysrc for jail %s: %v\n", key, jail.Name, err)
 			}
@@ -508,7 +510,7 @@ func buildDevfsRuleSet(jail *Jail, m *sync.Mutex) (error, int) {
 		}
 	}
 	
-	log.Debug("buildDevfsRuleSet: Build ruleset %d\n", ruleset)
+	log.Debugf("buildDevfsRuleSet: Build ruleset %d\n", ruleset)
 	
 	// Get default devfs_ruleset for the datastore
 	// UPDATE: We don't need this as every jail have a default Devfs_ruleset value
@@ -893,11 +895,11 @@ func setupVnetInterfaceHostSide(jail *Jail) ([]string, error) {
 		}
 		epairs = append(epairs, hsepair)
 	}
-	
+	log.Debugf("setupVnetInterfaceHostSide: returning %v\n", epairs)
 	return epairs, nil
 }
 
-func setupVnetInterfaceJailSide(jail *Jail, hsepair string) error {
+func setupVnetInterfaceJailSide(jail *Jail, hostepairs []string) error {
 	var jsmac []byte
 	var err error
 
@@ -917,7 +919,7 @@ func setupVnetInterfaceJailSide(jail *Jail, hsepair string) error {
 	}
 
 	// Loop through configured interfaces
-	for _, nicCnf := range strings.Split(jail.Config.Interfaces, ",") {
+	for i, nicCnf := range strings.Split(jail.Config.Interfaces, ",") {
 		v := strings.Split(nicCnf, ":")
 		if len(v) != 2 {
 			return fmt.Errorf("Invalid value for Interfaces: %s\n", nicCnf)
@@ -928,7 +930,9 @@ func setupVnetInterfaceJailSide(jail *Jail, hsepair string) error {
 		// inside jail final nic name
 		jnic := strings.Replace(v[0], "vnet", "epair", 1)
 		jnic = jnic + "b"
-		
+		// host side associated jail nic name
+		jsepair := fmt.Sprintf("%sb", strings.TrimSuffix(hostepairs[i], "a"))
+
 		// Get jail side MAC
 		pname := fmt.Sprintf("Config.%s_mac", nic)
 		var val *reflect.Value
@@ -945,42 +949,39 @@ func setupVnetInterfaceJailSide(jail *Jail, hsepair string) error {
 		} else {
 			jsmac = val.Bytes()
 		}
-		
-		lasta := strings.LastIndex(hsepair, "a")
-		jsepair := hsepair[:lasta] + strings.Replace(hsepair[lasta:], "a", "b", 1)
-		
+
 		cmd := fmt.Sprintf("/sbin/ifconfig %s vnet %s", jsepair, jail.InternalName)
 		_, err := executeCommand(cmd)
 		if err != nil {
 			return fmt.Errorf("Error linking interface to jail: %v\n", err)
 		}
-		
+
 		// Get bridge MTU
 		mtu, err := gJailHost.GetBridgeMTU(bridge)
 		if err != nil {
 			return fmt.Errorf("Error getting bridge %s mtu: %v\n", bridge, err)
 		}
-				
+
 		cmd = fmt.Sprintf("/usr/sbin/jexec %d ifconfig %s mtu %d", jail.JID, jsepair, mtu)
 		_, err = executeCommand(cmd)
 		if err != nil {
 			return fmt.Errorf("Error setting mtu: %v\n", err)
 		}
-		
+
 		// rename epairXXb to epair0b (or opair1b, ...)
 		cmd = fmt.Sprintf("/usr/sbin/setfib %s jexec %d ifconfig %s name %s", jail.Config.Exec_fib, jail.JID, jsepair, jnic)
 		_, err = executeCommand(cmd)
 		if err != nil {
 			return fmt.Errorf("Error linking interface to jail: %v\n", err)
 		}
-		
+
 		cmd = fmt.Sprintf("/usr/sbin/setfib %s jexec %d ifconfig %s link %s", jail.Config.Exec_fib,
 						  jail.JID, jnic, hex.EncodeToString(jsmac))
 		_, err = executeCommand(cmd)
 		if err != nil {
 			return fmt.Errorf("Error setting mac: %v\n", err)
 		}
-		
+
 		// TODO: Move outside of this function
 		// add interface to bridge
 		if jail.Config.Nat == 0 {
@@ -990,16 +991,16 @@ func setupVnetInterfaceJailSide(jail *Jail, hsepair string) error {
 				return fmt.Errorf("Error adding member %s to %s: %v: %s\n", nic, bridge, err, out)
 			}
 		}
-		
+
 		// Check we have an IP for the nic, and set it into jail
 		if len(ip4s[nic]) > 0 {
 			err = setJailVnetIp(jail, jnic, ip4s[nic])
 		}
-		
+
 		if len(ip6s[nic]) > 0 {
 			err = setJailVnetIp(jail, jnic, ip6s[nic])
 		}
-		
+
 		// finally up interface
 		if jail.Config.Nat == 0 {
 			cmd := fmt.Sprintf("/sbin/ifconfig %s.%d up", nic, jail.JID)
@@ -1009,7 +1010,9 @@ func setupVnetInterfaceJailSide(jail *Jail, hsepair string) error {
 			}
 		}
 	}
-	
+
+	log.Debugf("setupVnetInterfaceJailSide: return with success\n")
+
 	return nil
 }
 
@@ -1069,9 +1072,9 @@ func StartJailsAtBoot() {
 	var curThNb int
 	var curPri int
 
-	// Get boot enabled jails
+	// Get boot enabled non-template jails
 	for _, j := range gJails {
-		if j.Config.Boot > 0 {
+		if j.Config.Boot > 0 && !strings.EqualFold(j.Config.Jailtype, "template") {
 			startList = append(startList, j)
 		}
 	}
@@ -1165,7 +1168,7 @@ func StartJail(args []string) {
 
 	for _, a := range args {
 		// Check if jail exist and is distinctly named
-		cj, err = getJailFromArray(a, gJails)
+		cj, err = getJailFromArray(a, []string{"basejail", "jail"}, gJails)
 		if err != nil {
 			fmt.Printf("Error getting jail: %s\n", err)
 			continue
@@ -1177,7 +1180,7 @@ func StartJail(args []string) {
 		}
 		
 		fmt.Printf("> Starting jail %s\n", cj.Name)
-		
+
 		// Set InternalName as it is used by some of these
 		cj.InternalName = fmt.Sprintf("ioc-%s", cj.Name)
 
@@ -1339,34 +1342,34 @@ func StartJail(args []string) {
 				cj.Config.Defaultrouter = ip4[0]
 			}
 		}
-		
+
 		// See https://github.com/iocage/iocage/blob/e94863d4c54f02523fb09e62e48be7db9ac92eda/iocage_lib/ioc_start.py:401
 		if cj.Config.Vnet == 0 {
 			// Not supported
 			fmt.Printf("Only VNet jails supported\n")
 			return
 		}
-		
+
 		var net []string
 		if false == strings.EqualFold(cj.Config.Vnet_interfaces, "none") {
 			net = append(net,  strings.Split(cj.Config.Vnet_interfaces, " ")...)
 		}
-		
+
 		err, dynrs := buildDevfsRuleSet(cj, &gMdevfs)
 		if err != nil {
 			fmt.Printf("%s\n", err.Error())
 			return
 		}
-		
+
 		err = buildJailParameters(cj, dynrs)
 		if err != nil {
 			fmt.Printf("%s\n", err.Error())
 			return
 		}
-		
+
 		// Synchronize jail config to disk
-		WriteConfigToDisk(cj.Name, false, false)
-		
+		cj.WriteConfigToDisk(false)
+
 		start_cmd := fmt.Sprintf("/usr/sbin/jail -f /var/run/jail.%s.conf -c", cj.InternalName)
 
 		//TODO: handle start_env & prestart_env, could be used by iocage plugins
@@ -1376,17 +1379,17 @@ func StartJail(args []string) {
 			fmt.Printf("Aborting jail start\n")
 			return
 		}
-		
+
 		fmt.Printf("  > Start jail:\n")
 		_, err = executeCommand(start_cmd)
 		if err != nil {
 			fmt.Printf("Error starting jail %s: %v\n", cj.Name, err)
 			return
 		}
-		
+
 		fmt.Printf("  > Start jail: OK\n")
 		fmt.Printf("  > With devfs ruleset %d\n", dynrs)
-		
+
 		// Update running state, JID and Devfs_ruleset
 		cj.Running = true
 		cj.Devfs_ruleset = dynrs
@@ -1400,13 +1403,13 @@ func StartJail(args []string) {
 				break
 			}
 		}
-		
+
 		hostInt, err := gJailHost.GetInterfaces()
 		if err != nil {
 			fmt.Printf("Error listing jail host interfaces: %v\n", err)
 			return
 		}
-		
+
 		if false == strings.EqualFold(cj.Config.Vnet_default_interface, "auto") &&
 			false == strings.EqualFold(cj.Config.Vnet_default_interface, "none") &&
 			false == isStringInArray(hostInt, cj.Config.Vnet_default_interface) {
@@ -1415,30 +1418,30 @@ func StartJail(args []string) {
 		}
 
 		fmt.Printf("  > Setup VNet network:\n")
-		hsepairs, err := setupVnetInterfaceHostSide(cj); 
+		hsepairs, err := setupVnetInterfaceHostSide(cj);
 		if err != nil {
 			fmt.Printf("Error setting VNet interface host side: %v\n", err)
 			return
 		}
-		
-		for _, ep := range hsepairs {
-			if err = setupVnetInterfaceJailSide(cj, ep); err != nil {
-				fmt.Printf("Error setting VNet interface jail side: %v\n", err)
-				return
-			}
+
+		if err = setupVnetInterfaceJailSide(cj, hsepairs); err != nil {
+			fmt.Printf("Error setting VNet interface jail side: %v\n", err)
+			return
 		}
 		fmt.Printf("  > Setup VNet network: OK\n")
 		
-		// TODO: Handle DHCP
-		fmt.Printf("  > Setup default ipv4 gateway:\n")
-		cmd := fmt.Sprintf("/usr/sbin/setfib %s /usr/sbin/jexec %d route add default %s", cj.Config.Exec_fib, cj.JID, cj.Config.Defaultrouter)
-		out, err := executeCommand(cmd)
-		if err != nil && len(out) > 0 {
-			fmt.Printf("Error: %v: %s\n", err, out)
-		} else {
-			fmt.Printf("  > Setup default ipv4 gateway: OK\n")
+		// Set default route, unless main network is dhcp
+		if ! cj.isFirstNetDhcp() {
+			fmt.Printf("  > Setup default ipv4 gateway:\n")
+			cmd := fmt.Sprintf("/usr/sbin/setfib %s /usr/sbin/jexec %d route add default %s", cj.Config.Exec_fib, cj.JID, cj.Config.Defaultrouter)
+			out, err := executeCommand(cmd)
+			if err != nil && len(out) > 0 {
+				fmt.Printf("Error: %v: %s\n", err, out)
+			} else {
+				fmt.Printf("  > Setup default ipv4 gateway: OK\n")
+			}
 		}
-		
+
 		if cj.Config.Ip6_addr != "none" {
 			fmt.Printf("  > Setup default ipv6 gateway:\n")
 			cmd := fmt.Sprintf("/usr/sbin/setfib %s /usr/sbin/jexec %d route add -6 default %s", cj.Config.Exec_fib, cj.JID, cj.Config.Defaultrouter6)
@@ -1449,7 +1452,7 @@ func StartJail(args []string) {
 				fmt.Printf("  > Setup default ipv6 gateway: OK\n")
 			}
 		}
-		
+
 		if cj.Config.Jail_zfs > 0 {
 			fmt.Printf("  > Jail ZFS datasets:\n")
 			err = jailZfsDatasets(cj)
@@ -1464,26 +1467,26 @@ func StartJail(args []string) {
 		if err != nil {
 			fmt.Printf("%s\n", err)
 		}
-		
+
 		if cj.Config.Host_time > 0 {
 			err = copyLocalTime(cj)
 			if err != nil {
 				fmt.Printf("%s\n", err)
 			}
 		}
-		
+
 		// Start services
 		if len(cj.Config.Exec_start) > 0 {
 			fmt.Printf("  > Start services:\n")
 			cmd := fmt.Sprintf("/usr/sbin/setfib %s /usr/sbin/jexec %d %s", cj.Config.Exec_fib, cj.JID, cj.Config.Exec_start)
 			err := executeCommandNonBlocking(cmd)
-			if err != nil && len(out) > 0 {
+			if err != nil {
 				fmt.Printf("Error: %v\n", err)
 			} else {
 				fmt.Printf("  > Start services: OK\n")
 			}
 		}
-		
+
 		if cj.Config.Rtsold > 0 || strings.EqualFold(cj.Config.Ip6_addr, "accept_rtadv") {
 			fmt.Printf("  > Start rtsold:\n")
 			cmd := fmt.Sprintf("/usr/sbin/setfib %s /usr/sbin/jexec %d service rtsold start", cj.Config.Exec_fib, cj.JID)
@@ -1494,7 +1497,7 @@ func StartJail(args []string) {
 				fmt.Printf("  > Start rtsold: OK\n")
 			}
 		}
-		
+
 		// TODO: Execute Exec_poststart
 		if len(cj.Config.Exec_poststart) > 0 {
 			fmt.Printf("  > Execute post-start:\n")
@@ -1506,18 +1509,18 @@ func StartJail(args []string) {
 				fmt.Printf("  > Execute post-start: OK\n")
 			}
 		}
-		
+
 		// WIP 10/07/2022 : https://github.com/iocage/iocage/blob/master/iocage_lib/ioc_start.py#L891
 		// TODO: Handle dhcp
 		// TODO: Apply rctl
-		
+
 		// Update last_started
+		// 23/07/2023 : This is not working, when writing to disk the old value is used
 		dt := time.Now()
 		curDate := fmt.Sprintf("%s", dt.Format("2006-01-02 15:04:05"))
-		fmt.Sprintf(cj.Config.Last_started, curDate)
-		WriteConfigToDisk(cj.Name, false, true)
-		
-		
+		cj.Config.Last_started = curDate
+		writeConfigToDisk(cj, false)
+
 		/*
 			out, err := executeCommand(fmt.Sprintf("rctl jail:%s", cj.InternalName))
 			if err == nil && len(out) > 0 {

cmd/stop.go

@@ -123,7 +123,7 @@ func deleteDevfsRuleset(ruleset int) error {
 	return nil
 }
 
-func umountJailFsFromHost(jail *Jail, mountpoint string) error {
+func umountFsFromHost(mountpoint string) error {
 	cmd := "mount -p"
 	out, err := executeCommand(cmd)
 	if err != nil {
@@ -134,11 +134,11 @@ func umountJailFsFromHost(jail *Jail, mountpoint string) error {
 	for _, l := range strings.Split(out, "\n") {
 		f := strings.Split(remSpPtrn.ReplaceAllString(l, " "), " ")
 		if len(f) > 2 {
-			if strings.EqualFold(f[1], fmt.Sprintf("%s%s", jail.RootPath, mountpoint)) {
-				cmd = fmt.Sprintf("umount %s%s", jail.RootPath, mountpoint)
+			if strings.EqualFold(f[1], mountpoint) {
+				cmd = fmt.Sprintf("umount %s", mountpoint)
 				_, err := executeCommand(cmd)
 				if err != nil {
-					return errors.New(fmt.Sprintf("Error umounting %s%s: %s", jail.RootPath, mountpoint, err.Error()))
+					return errors.New(fmt.Sprintf("Error umounting %s: %s", mountpoint, err.Error()))
 				}
 				return nil
 			}
@@ -148,6 +148,10 @@ func umountJailFsFromHost(jail *Jail, mountpoint string) error {
 	return nil
 }
 
+func umountJailFsFromHost(jail *Jail, mountpoint string) error {
+	return umountFsFromHost(fmt.Sprintf("%s%s", jail.RootPath, mountpoint))
+}
+
 // Internal usage only
 func stopJail(jail *Jail) error {
 	cmd := "jail -q"
@@ -270,7 +274,7 @@ func StopJail(args []string) {
 
 	for _, a := range args {
 		// Check if jail exist and is distinctly named
-		cj, err = getJailFromArray(a, gJails)
+		cj, err = getJailFromArray(a, []string{"basejail", "jail"}, gJails)
 		if err != nil {
 			fmt.Printf("Error getting jail: %s\n", err)
 			continue
@@ -281,25 +285,10 @@ func StopJail(args []string) {
 			continue
 		}
 
-		fmt.Printf("> Stopping jail %s\n", a)
+		fmt.Printf("> Stopping jail %s\n", cj.Name)
 
-		// Get current version to update config.json
-		cvers, err := executeCommandInJail(cj, "/bin/freebsd-version")
-		if err != nil {
-			fmt.Printf("ERROR: %s\n", err.Error())
-			return
-		}
-		cvers = strings.TrimRight(cvers, "\n")
-		
-		//fmt.Sprintf(cj.Config.Release, cvers)
-		//cj.Config.Release = cvers
-		//cj.ConfigUpdated = true
-		
-		// This is working in this context, but value is not available in WriteConfigToDisk context :/
-		setStructFieldValue(cj, "Config.Release", cvers)
-		fmt.Printf("DEBUG: release was set, now is : %s\n", cj.Config.Release)
-		
-		// We need to get the real Config object, not a copy of it		
+		// Get and write new release into config.json
+		updateVersion(cj)
 
 		out, err := executeCommand(fmt.Sprintf("rctl jail:%s", cj.InternalName))
 		if err == nil && len(out) > 0 {
@@ -419,7 +408,8 @@ func StopJail(args []string) {
 			fmt.Printf("  > Umount mountpoints from %s\n", fstab)
 			errs := 0
 			for _, m := range mounts {
-				err = umountJailFsFromHost(cj, m.Mountpoint)
+				log.Debugf("Umounting %s\n", m.Mountpoint)
+				err = umountFsFromHost(m.Mountpoint)
 				if err != nil {
 					fmt.Printf("ERROR: %s\n", err.Error())
 					errs += 1
@@ -429,7 +419,7 @@ func StopJail(args []string) {
 				fmt.Printf("  > Umount mountpoints from %s: OK\n", fstab)
 			}
 		}
-		
+
 		// TODO: Execute poststop
 		if len(cj.Config.Exec_poststop) > 0 {
 			fmt.Printf("  > Execute post-stop:\n")
@@ -440,13 +430,13 @@ func StopJail(args []string) {
 				fmt.Printf("  > Execute post-stop: OK\n")
 			}
 		}
-		
+
 		// Remove parameter file
 		pfile := fmt.Sprintf("/var/run/jail.%s.conf", cj.InternalName)
 		if err = os.Remove(pfile); err != nil {
 			fmt.Printf("Error deleting parameter file %s\n", pfile)
 		}
-		
+
 		// We need this to get a reference to cj.Running (bc cj.Running is just a copy of value in the scope of StopJail())
 		for i, j := range gJails {
 			if strings.EqualFold(j.Name, cj.Name) && strings.EqualFold(j.Datastore, cj.Datastore) {
@@ -464,9 +454,7 @@ func StopJail(args []string) {
 				}
 			}
 		}
-
-		fmt.Printf("DEBUG: release = %s\n", cj.Config.Release)
-		WriteConfigToDisk(cj.Name, false, true)
+		writeConfigToDisk(cj, false)
 
 	}
 }

cmd/update.go

@@ -7,117 +7,46 @@ import (
 	"time"
 )
 
-const (
-	fbsdUpdateConfig = `
-	# $FreeBSD: releng/12.2/usr.sbin/freebsd-update/freebsd-update.conf 337338 2018-08-04 22:25:41Z brd $
-	
-	# Trusted keyprint.  Changing this is a Bad Idea unless you've received
-	# a PGP-signed email from <security-officer@FreeBSD.org> telling you to
-	# change it and explaining why.
-	KeyPrint 800651ef4b4c71c27e60786d7b487188970f4b4169cc055784e21eb71d410cc5
-	
-	# Server or server pool from which to fetch updates.  You can change
-	# this to point at a specific server if you want, but in most cases
-	# using a "nearby" server won't provide a measurable improvement in
-	# performance.
-	ServerName update.FreeBSD.org
-	
-	# Components of the base system which should be kept updated.
-	Components world
-	
-	# Example for updating the userland and the kernel source code only:
-	# Components src/base src/sys world
-	
-	# Paths which start with anything matching an entry in an IgnorePaths
-	# statement will be ignored.
-	IgnorePaths
-	
-	# Paths which start with anything matching an entry in an IDSIgnorePaths
-	# statement will be ignored by "freebsd-update IDS".
-	IDSIgnorePaths /usr/share/man/cat
-	IDSIgnorePaths /usr/share/man/whatis
-	IDSIgnorePaths /var/db/locate.database
-	IDSIgnorePaths /var/log
-	
-	# Paths which start with anything matching an entry in an UpdateIfUnmodified
-	# statement will only be updated if the contents of the file have not been
-	# modified by the user (unless changes are merged; see below).
-	UpdateIfUnmodified /etc/ /var/ /root/ /.cshrc /.profile
-	
-	# When upgrading to a new FreeBSD release, files which match MergeChanges
-	# will have any local changes merged into the version from the new release.
-	MergeChanges /etc/
-	
-	### Default configuration options:
-	
-	# Directory in which to store downloaded updates and temporary
-	# files used by FreeBSD Update.
-	# WorkDir /var/db/freebsd-update
-	
-	# Destination to send output of "freebsd-update cron" if an error
-	# occurs or updates have been downloaded.
-	# MailTo root
-	
-	# Is FreeBSD Update allowed to create new files?
-	# AllowAdd yes
-	
-	# Is FreeBSD Update allowed to delete files?
-	# AllowDelete yes
-	
-	# If the user has modified file ownership, permissions, or flags, should
-	# FreeBSD Update retain this modified metadata when installing a new version
-	# of that file?
-	# KeepModifiedMetadata yes
-	
-	# When upgrading between releases, should the list of Components be
-	# read strictly (StrictComponents yes) or merely as a list of components
-	# which *might* be installed of which FreeBSD Update should figure out
-	# which actually are installed and upgrade those (StrictComponents no)?
-	# StrictComponents no
-	
-	# When installing a new kernel perform a backup of the old one first
-	# so it is possible to boot the old kernel in case of problems.
-	# BackupKernel yes
-	
-	# If BackupKernel is enabled, the backup kernel is saved to this
-	# directory.
-	# BackupKernelDir /boot/kernel.old
-	
-	# When backing up a kernel also back up debug symbol files?
-	# BackupKernelSymbolFiles no
-	
-	# Create a new boot environment when installing patches
-	# CreateBootEnv yes
-	`
-)
 
 // Internal usage only
 func updateJail(jail *Jail) error {
-	
 	// Create default config as temporary file
 	cfgFile, err := os.CreateTemp("", "gocage-jail-update-")
 	if err != nil {
 		return err
 	}
-	
-	cfgFile.Write([]byte(fbsdUpdateConfig))
-	
-	defer cfgFile.Close()
-	//defer os.Remove(cfgFile.Name())
-	
-	cmd := fmt.Sprintf("/usr/sbin/freebsd-update --not-running-from-cron -f %s -b %s --currently-running %s fetch install", 
-					   cfgFile.Name(), jail.RootPath, jail.Config.Release)
-	
-	fmt.Printf("DEBUG: Prepare to execute \"%s\"\n", cmd)
 
+	cfgFile.Write([]byte(fbsdUpdateConfig))
+
+	defer cfgFile.Close()
+	defer os.Remove(cfgFile.Name())
+
+	// Folder containing update/upgrade temporary files. Common so we save bandwith when upgrading multiple jails
+	// TODO: Variabilize /iocage/freebsd-update
+	_, err = os.Stat("/iocage/freebsd-update")
+	if os.IsNotExist(err) {
+		if err := os.Mkdir("/iocage/freebsd-update", 0755); err != nil {
+			return err
+		}
+	}
+
+	cmd := fmt.Sprintf("/usr/sbin/freebsd-update --not-running-from-cron -f %s -b %s --currently-running %s fetch",
+					   cfgFile.Name(), jail.RootPath, jail.Config.Release)
 	err = executeCommandWithOutputToStdout(cmd)
 	if err != nil {
 		return err
 	}
-	
+
+	cmd = fmt.Sprintf("/usr/sbin/freebsd-update --not-running-from-cron -f %s -b %s --currently-running %s install",
+			   cfgFile.Name(), jail.RootPath, jail.Config.Release)
+	err = executeCommandWithOutputToStdout(cmd)
+	if err != nil {
+		return err
+	}
+
 	// Get and write new release into config.json
-	
-	
+	updateVersion(jail)
+
 	return nil
 }
 
@@ -128,7 +57,7 @@ func UpdateJail(args []string) {
 
 	for _, a := range args {
 		// Check if jail exist and is distinctly named
-		cj, err = getJailFromArray(a, gJails)
+		cj, err = getJailFromArray(a, []string{""}, gJails)
 		if err != nil {
 			fmt.Printf("Error getting jail: %s\n", err)
 			continue

cmd/upgrade.go

@@ -0,0 +1,125 @@
+package cmd
+
+
+
+import (
+	"os"
+	"fmt"
+	//"log"
+	"time"
+	"strings"
+)
+
+// Internal usage only
+func upgradeJail(jail *Jail, version string) error {
+	// Create default config as temporary file
+	cfgFile, err := os.CreateTemp("", "gocage-jail-upgrade-")
+	if err != nil {
+		return err
+	}
+
+	cfgFile.Write([]byte(fbsdUpdateConfig))
+
+	defer cfgFile.Close()
+	defer os.Remove(cfgFile.Name())
+
+	// Folder containing update/uipgrade temporary files. Common so we save bandwith when upgrading multiple jails
+	// TODO: Variabilize /iocage/freebsd-update
+	_, err = os.Stat("/iocage/freebsd-update")
+	if os.IsNotExist(err) {
+		if err := os.Mkdir("/iocage/freebsd-update", 0755); err != nil {
+			return err
+		}
+	}
+
+	// Get current version. Won't work on stopped jail.
+	fbsdvers, err := executeCommandInJail(jail, "/bin/freebsd-version")
+	if err != nil {
+		fmt.Printf("ERROR executeCommandInJail: %s\n", err.Error())
+		return err
+	}
+	fbsdvers = strings.TrimRight(fbsdvers, "\n")
+	//fbsdvers := jail.Config.Release
+
+	cmd := fmt.Sprintf("/usr/sbin/freebsd-update -f %s -b %s --currently-running %s -r %s upgrade", 
+			   cfgFile.Name(), jail.RootPath, fbsdvers, version)
+
+	//fmt.Printf("DEBUG: Prepare to execute \"%s\"\n", cmd)
+
+	// Need to give user control, bc there could be merge edit needs
+	err = executeCommandWithStdinStdoutStderr(cmd)
+	if err != nil {
+		return err
+	}
+
+	cmd = fmt.Sprintf("/usr/sbin/freebsd-update -f %s -b %s --currently-running %s -r %s install", 
+			   cfgFile.Name(), jail.RootPath, fbsdvers, version)
+
+	//fmt.Printf("DEBUG: Prepare to execute \"%s\"\n", cmd)
+
+	err = executeCommandWithStdinStdoutStderr(cmd)
+	if err != nil {
+		return err
+	}
+
+	cmd = fmt.Sprintf("/usr/sbin/freebsd-update -f %s -b %s --currently-running %s -r %s install", 
+			  cfgFile.Name(), jail.RootPath, fbsdvers, version)
+
+	//fmt.Printf("DEBUG: Prepare to execute \"%s\"\n", cmd)
+
+	err = executeCommandWithStdinStdoutStderr(cmd)
+	if err != nil {
+		return err
+	}
+
+	cmd = fmt.Sprintf("/usr/local/sbin/pkg-static -j %d install -q -f -y pkg", jail.JID)
+	err = executeCommandWithStdinStdoutStderr(cmd)
+	if err != nil {
+		return err
+	}
+
+	// Get and write new release into config.json
+	updateVersion(jail)
+
+	return nil
+}
+
+func UpgradeJail(args []string) {
+	// Current jail were stopping
+	var cj *Jail
+	var err error
+
+	for _, a := range args {
+		// Check if jail exist and is distinctly named
+		cj, err = getJailFromArray(a, []string{""}, gJails)
+		if err != nil {
+			fmt.Printf("Error getting jail: %s\n", err)
+			continue
+		}
+
+		if cj.Running == false {
+			fmt.Printf("Error: jail must be running for upgrade.\n")
+			return
+		}
+
+		fmt.Printf("  > Snapshot jail %s\n", cj.Name)
+		// Set snapshot name
+		dt := time.Now()
+		curDate := fmt.Sprintf("%s", dt.Format("2006-01-02_15-04-05"))
+		gSnapshotName = fmt.Sprintf("goc_upgrade_%s_%s", cj.Config.Release, curDate)
+		err := createJailSnapshot(*cj)
+		if err != nil {
+			fmt.Printf("  > Snapshot jail %s: ERROR: %s\n", cj.Name, err.Error())
+			return
+		}
+		fmt.Printf("  > Snapshot jail %s: OK\n", cj.Name)
+
+		fmt.Printf("  > Upgrade jail %s to %s\n", cj.Name, gUpgradeRelease)
+		err = upgradeJail(cj, gUpgradeRelease)
+		if err != nil {
+			fmt.Printf("ERROR: %s\n", err.Error())
+		} else {
+			fmt.Printf("  > Upgrade jail %s: OK\n", cj.Name)
+		}
+	}
+}

cmd/utils.go

@@ -13,6 +13,7 @@ import (
 	"strconv"
 	"strings"
 	"io/ioutil"
+	"encoding/json"
 	"github.com/google/shlex"
 	"github.com/c2h5oh/datasize"
 	log "github.com/sirupsen/logrus"
@@ -23,6 +24,141 @@ const (
 	ifconfigipv4re = `inet[[:space:]](` + ipv4re + `)`
 	// Maximum thread qty for start/stop
 	gMaxThreads    = 4
+	
+	gDefaultsJson = ` {
+       "CONFIG_VERSION": "27",
+       "allow_chflags": 0,
+       "allow_mlock": 0,
+       "allow_mount": 0,
+       "allow_mount_devfs": 0,
+       "allow_mount_fusefs": 0,
+       "allow_mount_nullfs": 0,
+       "allow_mount_procfs": 0,
+       "allow_mount_tmpfs": 0,
+       "allow_mount_zfs": 0,
+       "allow_quotas": 0,
+       "allow_raw_sockets": 0,
+       "allow_set_hostname": 1,
+       "allow_socket_af": 0,
+       "allow_sysvipc": 0,
+       "allow_tun": 0,
+       "allow_vmm": 0,
+       "assign_localhost": 0,
+       "available": "readonly",
+       "basejail": 0,
+       "boot": 0,
+       "bpf": 0,
+       "children_max": "0",
+       "comment": "none",
+       "compression": "lz4",
+       "compressratio": "readonly",
+       "coredumpsize": "off",
+       "count": "1",
+       "cpuset": "off",
+       "cputime": "off",
+       "datasize": "off",
+       "dedup": "off",
+       "defaultrouter": "auto",
+       "defaultrouter6": "auto",
+       "depends": "none",
+       "devfs_ruleset": "4",
+       "dhcp": 0,
+       "enforce_statfs": "2",
+       "exec_clean": 1,
+       "exec_created": "/usr/bin/true",
+       "exec_fib": "0",
+       "exec_jail_user": "root",
+       "exec_poststart": "/usr/bin/true",
+       "exec_poststop": "/usr/bin/true",
+       "exec_prestart": "/usr/bin/true",
+       "exec_prestop": "/usr/bin/true",
+       "exec_start": "/bin/sh /etc/rc",
+       "exec_stop": "/bin/sh /etc/rc.shutdown",
+       "exec_system_jail_user": "0",
+       "exec_system_user": "root",
+       "exec_timeout": "60",
+       "host_domainname": "none",
+       "host_time": 1,
+       "hostid": "36353536-3135-5a43-4a34-313130315a56",
+       "hostid_strict_check": 0,
+       "interfaces": "vnet0:bridge0",
+       "ip4": "new",
+       "ip4_addr": "none",
+       "ip4_saddrsel": 1,
+       "ip6": "new",
+       "ip6_addr": "none",
+       "ip6_saddrsel": 1,
+       "ip_hostname": 0,
+       "jail_zfs": 0,
+       "jail_zfs_mountpoint": "none",
+       "last_started": "none",
+       "localhost_ip": "none",
+       "login_flags": "-f root",
+       "mac_prefix": "2c44fd",
+       "maxproc": "off",
+       "memorylocked": "off",
+       "memoryuse": "off",
+       "min_dyn_devfs_ruleset": "1000",
+       "mount_devfs": 1,
+       "mount_fdescfs": 1,
+       "mount_linprocfs": 0,
+       "mount_procfs": 0,
+       "mountpoint": "readonly",
+       "msgqqueued": "off",
+       "msgqsize": "off",
+       "nat": 0,
+       "nat_backend": "ipfw",
+       "nat_forwards": "none",
+       "nat_interface": "none",
+       "nat_prefix": "172.16",
+       "nmsgq": "off",
+       "notes": "none",
+       "nsem": "off",
+       "nsemop": "off",
+       "nshm": "off",
+       "nthr": "off",
+       "openfiles": "off",
+       "origin": "readonly",
+       "owner": "root",
+       "pcpu": "off",
+       "plugin_name": "none",
+       "plugin_repository": "none",
+       "priority": "99",
+       "pseudoterminals": "off",
+       "quota": "none",
+       "readbps": "off",
+       "readiops": "off",
+       "reservation": "none",
+       "resolver": "/etc/resolv.conf",
+       "rlimits": "off",
+       "rtsold": 0,
+       "securelevel": "2",
+       "shmsize": "off",
+       "stacksize": "off",
+       "stop_timeout": "30",
+       "swapuse": "off",
+       "sync_state": "none",
+       "sync_target": "none",
+       "sync_tgt_zpool": "none",
+       "sysvmsg": "new",
+       "sysvsem": "new",
+       "sysvshm": "new",
+       "template": 0,
+       "type": "jail",
+       "used": "readonly",
+       "vmemoryuse": "off",
+       "vnet": 0,
+       "vnet0_mac": "none",
+       "vnet1_mac": "none",
+       "vnet2_mac": "none",
+       "vnet3_mac": "none",
+       "vnet_default_interface": "auto",
+       "vnet_interfaces": "none",
+       "wallclock": "off",
+       "writebps": "off",
+       "writeiops": "off"
+       }
+`
 )
 
 /*****************************************************************************
@@ -217,6 +353,8 @@ func executeCommand(cmdline string) (string, error) {
 		// else
 		word = word + string(c)
 	}
+	
+	log.Debugf("executeCommand: %s\n", strings.Join(cmd, " "))
 
 	if len(cmd) > 1 {
 		out, err = exec.Command(cmd[0], cmd[1:]...).CombinedOutput()
@@ -336,6 +474,8 @@ func executeCommandWithOutputToStdout(cmdline string) (error) {
 		word = word + string(c)
 	}
 
+	log.Debugf("executeCommandWithOutputToStdout: will execute \"%s\"\n", strings.Join(cmd, " "))
+
 	var execHandle *exec.Cmd
 	if len(cmd) > 1 {
 		execHandle = exec.Command(cmd[0], cmd[1:]...)
@@ -365,9 +505,83 @@ func executeCommandWithOutputToStdout(cmdline string) (error) {
 	return fmt.Errorf("Unknown error: you shouldn't be here!\n")
 }
 
+/* Execute command plugging stdin and stdout to those of the running command.
+ * Blocking while the command run
+ */
+func executeCommandWithStdinStdoutStderr(cmdline string) (error) {
+	var cmd []string
+
+	if gUseSudo {
+		cmd = append(cmd, "sudo")
+	}
+
+	var word string
+	var in_escaped bool
+	// Split by words, or " enclosed words
+	for i, c := range (cmdline) {
+		if string(c) == "\"" {
+			if in_escaped {
+				// This is the closing "
+				cmd = append(cmd, word)
+				in_escaped = false
+			} else {
+				in_escaped = true
+			}
+			continue
+		}
+		if string(c) == " " {
+			if in_escaped {
+				word = word + string(c)
+				continue
+			} else {
+				cmd = append(cmd, word)
+				word = ""
+				continue
+			}
+		}
+		if i == (len(cmdline) - 1) {
+			word = word + string(c)
+			cmd = append(cmd, word)
+			break
+		}
+
+		// else
+		word = word + string(c)
+	}
+
+	var command *exec.Cmd
+	if len(cmd) > 1 {
+		command = exec.Command(cmd[0], cmd[1:]...)
+	} else {
+		command = exec.Command(cmd[0])
+	}
+
+	// Get environment
+	command.Env = os.Environ()
+
+	// Connect command to current stdin/out/err
+	command.Stdin = os.Stdin
+	command.Stdout = os.Stdout
+	command.Stderr = os.Stderr
+
+	if err := command.Start(); err != nil {
+		return err
+	}
+
+	err := command.Wait()
+
+	return err
+}
+
+
 func executeCommandInJail(jail *Jail, cmdline string) (string, error) {
 	var cmd []string
 
+	// We can't execute on non-running jail
+	if jail.Running == false {
+		return "", errors.New("Can't execute command on stopped jail")
+	}
+
 	if gUseSudo {
 		cmd = append(cmd, "sudo")
 	}
@@ -414,6 +628,10 @@ func executeCommandInJail(jail *Jail, cmdline string) (string, error) {
 		// else
 		word = word + string(c)
 	}
+	
+	if gDebug {
+		fmt.Printf("DEBUG: executeCommandInJail: prepare to execute \"%s\"\n", cmd)
+	}
 
 	out, err := exec.Command(cmd[0], cmd[1:]...).CombinedOutput()
 
@@ -466,8 +684,11 @@ func zfsSnapshot(dataset string, snapname string) error {
 	return nil
 }
 
+// Copy snapshot to a new dataset
+// TODO : Intercept death of sending process, then kill receiving
 func zfsCopy(src string, dest string) error {
 	// First, declare sending process & pipe
+	log.Debugf("Execute: zfs send %s\n", src)
 	cmd_send := exec.Command("zfs", "send", src)
 	stdout_send, err := cmd_send.StdoutPipe()
 	if err != nil {
@@ -476,6 +697,7 @@ func zfsCopy(src string, dest string) error {
 	}
 	
 	// then declare receiving process & pipe
+	log.Debugf("Execute: zfs receive %s\n", dest)
 	cmd_recv := exec.Command("zfs", "receive", dest)
 	stdin_recv, err := cmd_recv.StdinPipe()
 	if err != nil {
@@ -489,16 +711,19 @@ func zfsCopy(src string, dest string) error {
 	// then start processes and wait for finish
 	if err := cmd_recv.Start(); err != nil {
 		//fmt.Printf("Error: %v\n", err)
+		log.Debugf("zfs receive %s started: %v", dest, err)
 		return errors.New(fmt.Sprintf("Error starting receive process: %v\n", err))
 	}
 	//fmt.Printf("DEBUG: Start \"zfs send %s\"\n", dsconf)
 	if err := cmd_send.Start(); err != nil {
 		//fmt.Printf("Error: %v\n", err)
+		log.Debugf("zfs send %s started: %v", src, err)
 		return errors.New(fmt.Sprintf("Error starting send process: %v\n", err))
 	}
 	
 	//fmt.Printf("DEBUG: Wait for zfs send to finish\n")
 	if err := cmd_send.Wait(); err != nil {
+		log.Debugf("zfs send %s stopped with %v", err)
 		//fmt.Printf("Error: zfs send halted with %v\n", err)
 		return errors.New(fmt.Sprintf("send halted with: %v\n", err))
 	}
@@ -567,9 +792,20 @@ func doZfsDatasetExist(dataset string) (bool, error) {
 	return true, nil
 }
 
-// Create ZFS dataset. mountpoint can be "none", then the dataset won't be mounted
+/* Create ZFS dataset
+ * mountpoint can be "none", then the dataset won't be mounted
+ * mountpoint can be "", then it will be inherited
+ * compression can be "", then it wil be inherited
+ */
 func zfsCreateDataset(dataset, mountpoint, compression string) error {
-	cmd := fmt.Sprintf("zfs create -o mountpoint=%s -o compression=%s %s", mountpoint, compression, dataset)
+	cmd := "zfs create"
+	if len(mountpoint) > 0 {
+		cmd = fmt.Sprintf("%s -o mountpoint=%s", cmd, mountpoint)
+	}
+	if len(compression) > 0 {
+		cmd = fmt.Sprintf("%s -o compression=%s", cmd, compression)
+	}
+	cmd = fmt.Sprintf("%s %s", cmd, dataset)
 	out, err := executeCommand(cmd)
 	if err != nil {
 		return errors.New(fmt.Sprintf("%v; command returned \"%s\"", err, out))
@@ -589,6 +825,7 @@ func zfsGetDatasetByMountpoint(mountpoint string) (string, error) {
 
 // Delete a ZFS Dataset by name
 func zfsDestroy(dataset string) error {
+	log.Debugf("execute \"zfs destroy -r %s\"\n", dataset)
 	cmd := fmt.Sprintf("zfs destroy -r %s", dataset)
 	out, err := executeCommand(cmd)
 	if err != nil {
@@ -597,6 +834,40 @@ func zfsDestroy(dataset string) error {
 	return nil
 }
 
+/*****************************************************************************
+ *
+ * Filesystem operations
+ *
+ *****************************************************************************/
+/* Copy file */
+func copyFile(src, dst string) error {
+	srcfinfo, err := os.Stat(src)
+	if err != nil {
+		return fmt.Errorf("Cannot find source file: %s", err.Error())
+	}
+	if !srcfinfo.Mode().IsRegular() {
+		return fmt.Errorf("%s is not a regular file", src)
+	}
+
+	srcHandle, err := os.Open(src)
+	if err != nil {
+		return fmt.Errorf("Cannot open source file: %s", err.Error())
+	}
+	defer srcHandle.Close()
+	dstHandle, err := os.Create(dst)
+	if err != nil {
+		return fmt.Errorf("Cannot create destination file: %s", err.Error())
+	}
+	defer dstHandle.Close()
+	_, err = io.Copy(dstHandle, srcHandle)
+	return err
+}
+
+// Get permissions of file or folder
+func getPermissions(path string) (os.FileInfo, error) {
+	return os.Stat(path)
+}
+
 /*****************************************************************************
  *
  * rc.conf management
@@ -702,34 +973,6 @@ func copyDevfsRuleset(ruleset int, srcrs int) error {
 	return nil
 }
 
-/********************************************************************************
- * Returns value of parameter as read in /var/run/jail.$InternalName.conf
- * Directives without value will return "true" if found
- * Returns an error if parameter not found in file
-*******************************************************************************/
-func getValueFromRunningConfig(jname string, param string) (string, error) {
-	content, err := ioutil.ReadFile(fmt.Sprintf("/var/run/jail.%s.conf", jname))
-	if err != nil {
-		return "", err
-	}
-	
-	for _, line := range strings.Split(string(content), "\n") {
-		if strings.Contains(line, fmt.Sprintf("%s = ", param)) {
-			split := strings.Split(line, "=")
-			switch len(split) {
-				// directives without value
-				case 0:
-					return "true", nil
-				case 1:
-					return "", fmt.Errorf("Invalid format: %s", line)
-				case 2:
-					return strings.Trim(split[1], " ;\""), nil
-			}
-		}
-	}
-	
-	return "", fmt.Errorf("Parameter not found: %s", param)
-}
 /********************************************************************************
  * Add a rule to specified ruleset
  * Ex.: addDevfsRuleToRuleset("path bpf* unhide", 1002)
@@ -757,6 +1000,35 @@ func addDevfsRuleToRuleset(rule string, ruleset int) error {
 	return nil
 }
 
+/********************************************************************************
+ * Returns value of parameter as read in /var/run/jail.$InternalName.conf
+ * Directives without value will return "true" if found
+ * Returns an error if parameter not found in file
+*******************************************************************************/
+func getValueFromRunningConfig(jname string, param string) (string, error) {
+	content, err := ioutil.ReadFile(fmt.Sprintf("/var/run/jail.%s.conf", jname))
+	if err != nil {
+		return "", err
+	}
+
+	for _, line := range strings.Split(string(content), "\n") {
+		if strings.Contains(line, fmt.Sprintf("%s = ", param)) {
+			split := strings.Split(line, "=")
+			switch len(split) {
+				// directives without value
+				case 0:
+					return "true", nil
+				case 1:
+					return "", fmt.Errorf("Invalid format: %s", line)
+				case 2:
+					return strings.Trim(split[1], " ;\""), nil
+			}
+		}
+	}
+
+	return "", fmt.Errorf("Parameter not found: %s", param)
+}
+
 
 /******************************************************************************
  *
@@ -772,14 +1044,29 @@ func isStringInArray(strarr []string, searched string) bool {
 	return false
 }
 
+func (j Jail) isFirstNetDhcp() bool {
+	for _, n := range strings.Split(j.Config.Ip4_addr, ",") {
+		splitd := strings.Split(n, "|")
+		if len(splitd) > 1 && strings.EqualFold(splitd[1], "dhcp") {
+			return true
+		}
+	}
+	return false
+}
+
 /********************************************************************************
  * Get a specific jail reference, to update properties after a range loop
  * Name can be short or long form ("myjail" vs "mystore/myjail")
+ * An empty jailtype means "all types"
  *******************************************************************************/
-func getJailFromArray(name string, jarray []Jail) (*Jail, error) {
+func getJailFromArray(name string, jailtypes []string, jarray []Jail) (*Jail, error) {
 	var ds, jail string
 	var jails []Jail
-	
+
+	if (len(jailtypes) == 1 && len(jailtypes[0]) == 0) || len(jailtypes) == 0 {
+		jailtypes = []string{"basejail", "jail", "template"}
+	}
+
 	if strings.Contains(name, "/") {
 		split := strings.Split(name, "/")
 		if len(split) != 2 {
@@ -792,16 +1079,17 @@ func getJailFromArray(name string, jarray []Jail) (*Jail, error) {
 	}
 		
 	for i, j := range jarray {
-		//if jail == j.Name {
 		if strings.HasPrefix(j.Name, jail) {
-			if len(ds) > 0 {
-				if strings.EqualFold(ds, j.Datastore) {
-					return &jarray[i], nil
+			if isStringInArray(jailtypes, j.Config.Jailtype) {
+				if len(ds) > 0 {
+					if strings.EqualFold(ds, j.Datastore) {
+						return &jarray[i], nil
+					} else {
+						continue
+					}
 				} else {
-					continue
+					jails = append(jails, j)
 				}
-			} else {
-				jails = append(jails, j)
 			}
 		}
 	}
@@ -833,7 +1121,7 @@ func setJailConfigUpdated(jail *Jail) error {
 		return errors.New(fmt.Sprintf("No config path for jail %s", jail.Name))
 	}
 
-	j, err := getJailFromArray(jail.Name, gJails)
+	j, err := getJailFromArray(jail.Name, []string{""}, gJails)
 	if err != nil {
 		return err
 	}
@@ -842,6 +1130,121 @@ func setJailConfigUpdated(jail *Jail) error {
 	return nil
 }
 
+func getVersion(jail *Jail) (string, error) {
+	cvers, err := executeCommand(fmt.Sprintf("%s/bin/freebsd-version", jail.RootPath))
+	if err != nil {
+		fmt.Printf("ERROR: %s\n", err.Error())
+		return "", err
+	}
+	return strings.TrimRight(cvers, "\n"), nil
+}
+
+func updateVersion(jail *Jail) error {
+	cvers, err := executeCommand(fmt.Sprintf("%s/bin/freebsd-version", jail.RootPath))
+	if err != nil {
+		fmt.Printf("ERROR: %s\n", err.Error())
+		return err
+	}
+	cvers = strings.TrimRight(cvers, "\n")
+	jail.Config.Release = cvers
+	jail.WriteConfigToDisk(false)
+
+	return nil
+}
+
+/********************************************************************************
+ * Write jail(s) config which been updated to disk.
+ * If name is specified, work on the jail. If name is empty string, work on all.
+ * If changeauto not set, values which are in "auto" mode on disk
+ *  won't be overwritten (p.ex defaultrouter wont be overwritten with current
+ *  default route, so if route change on jailhost this will reflect on jail next
+ *  start)
+ *******************************************************************************/
+func writeConfigToDisk(j *Jail, changeauto bool) {
+	// we will manipulate properties so get a copy
+	jc := j.Config
+	
+	if changeauto == false {
+		// Overwrite "auto" properties
+		ondiskjc, err := getJailConfig(j.ConfigPath)
+		if err != nil {
+			panic(err)
+		}
+		// TODO : List all fields, then call getStructFieldValue to compare value with "auto"
+		// If "auto" then keep it that way before writing ondiskjc to disk
+		var properties []string
+		properties = getStructFieldNames(ondiskjc, properties, "")
+		
+		for _, p := range properties {
+			v, _, err := getStructFieldValue(ondiskjc, p)
+			if err != nil {
+				panic(err)
+			}
+			if v.String() == "auto" {
+				err = setStructFieldValue(&jc, p, "auto")
+				if err != nil {
+					fmt.Printf("ERROR sanitizing config: %s\n", err.Error())
+					os.Exit(1)
+				}
+			}
+		}
+	}
+	
+	marshaled, err := json.MarshalIndent(jc, "", "    ")
+	if err != nil {
+		fmt.Printf("ERROR marshaling config: %s\n", err.Error())
+	}
+	
+	//fmt.Printf("DEBUG: Will write config to disk, with content:\n")
+	//fmt.Printf(string(marshaled))
+	
+	if os.WriteFile(j.ConfigPath, []byte(marshaled), 0644); err != nil {
+		fmt.Printf("Error writing config file %s: %v\n", j.ConfigPath, err)
+		os.Exit(1)
+	}
+}
+
+func (j Jail) WriteConfigToDisk(changeauto bool) {
+	// we will manipulate properties so get a copy
+	jc := j.Config
+
+	if changeauto == false {
+		// Overwrite "auto" properties
+		ondiskjc, err := getJailConfig(j.ConfigPath)
+		if err != nil {
+			panic(err)
+		}
+		// TODO : List all fields, then call getStructFieldValue to compare value with "auto"
+		// If "auto" then keep it that way before writing ondiskjc to disk
+		var properties []string
+		properties = getStructFieldNames(ondiskjc, properties, "")
+		
+		for _, p := range properties {
+			v, _, err := getStructFieldValue(ondiskjc, p)
+			if err != nil {
+				panic(err)
+			}
+			if v.String() == "auto" {
+				err = setStructFieldValue(&jc, p, "auto")
+				if err != nil {
+					fmt.Printf("ERROR sanitizing config: %s\n", err.Error())
+					os.Exit(1)
+				}
+			}
+		}
+	}
+
+	marshaled, err := json.MarshalIndent(jc, "", "    ")
+	if err != nil {
+		fmt.Printf("ERROR marshaling config: %s\n", err.Error())
+	}
+
+	if os.WriteFile(j.ConfigPath, []byte(marshaled), 0644); err != nil {
+		fmt.Printf("Error writing config file %s: %v\n", j.ConfigPath, err)
+		os.Exit(1)
+	}
+}
+
 /******************************************************************************
  * Return the quantity of jails with the name passed as parameter
  *****************************************************************************/
@@ -857,7 +1260,7 @@ func countOfJailsWithThisName(name string) int {
 
 
 func isNameDistinctive(name string, jails []Jail) bool {
-	_, err := getJailFromArray(name, jails)
+	_, err := getJailFromArray(name, []string{""}, jails)
 	if err != nil {
 		return false
 	} else {

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/c-robinson/iplib v1.0.3
 	github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
+	github.com/otiai10/copy v1.12.0
 	github.com/sirupsen/logrus v1.8.1
 	github.com/spf13/cobra v1.2.1
 	github.com/spf13/viper v1.9.0
@@ -24,7 +25,7 @@ require (
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
 	github.com/subosito/gotenv v1.2.0 // indirect
-	golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf // indirect
+	golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8 // indirect
 	golang.org/x/text v0.3.6 // indirect
 	gopkg.in/ini.v1 v1.63.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect