v0.42h: finish previous commit

.gitignore

@@ -1,3 +1,2 @@
 gocage
 go.sum
-

CHANGELOG

@@ -0,0 +1,4 @@
+v.0.33b : Support jailing datasets on differents pools : jail_zfs_dataset now have to include the pool name  
+v.0.33c : Parallelize start/stop of jails with same priority  
+v.0.34 : jail name can be shortened 
+v.0.35 : One can now "gocage destroy"

README.md

@@ -6,18 +6,32 @@ Support iocage jails, so they can coexist.
 Gocage is meant to be a complete jail management tool with network, snapshots, jail cloning support and a web interface. This is the hypothetic future.  
 Gocage can handle multiple datastores, so you can have jails on HDD storage and jails on SSD storage.  
 
+From v0.33b, due to multi ZFS pool support, gocage is no longer 100% compatible with iocage.  
+Zfs datasets now should be specified with the ZFS pool. e.g. :  
+<pre><code>Config.Jail_zfs = 1
+Config.Jail_zfs_dataset = myzfspool/poudriere
+Config.Jail_zfs_mountpoint = none
+</code></pre>
+
+Create jails
+------------
+You need to specify release, and optional configuration:  
+<pre><code>gocage create jail1 -r 13.2-RELEASE -p "Config.Ip4_addr='vnet0|192.168.1.91/24',Config.Ip6=none,Config.Boot=1"
+</code></pre>
+
+Create basejail. A basejail is a jail based on a release: system will be nullfs read-only mounted from the release directory. Main advantage is that release updates will immediately apply to jails based on this release. Another advantage is that jail system is mounted read-only, a plus from a security perspective:  
+<pre><code>gocage create -b -r 14.0-RELEASE basejail1
+</code></pre>
 
 
 List jails
 ----------
-Nothing fancy, just use  
-`gocage list`
+<pre><code>gocage list</code></pre>
 
 ### Specify fields to display
 
 Use -o to specify which fields you want to display:  
-<pre><code>
-gocage list -o JID,Name,Running,Config.Boot,Config.Comment  
+<pre><code>gocage list -o JID,Name,Running,Config.Boot,Config.Comment  
 +=====+==========+=========+=============+================+  
 | JID | Name     | Running | Config.Boot | Config.Comment |  
 +=====+==========+=========+=============+================+  
@@ -31,7 +45,7 @@ gocage list -o JID,Name,Running,Config.Boot,Config.Comment
 +-----+----------+---------+-------------+----------------+  
 </code></pre>
 
-See [cmd/struct.go](https://git.nosd.in/yo/gocage/src/branch/master/cmd/struct.go) for field names.
+Use `gocage properties`to list available fields.
 
 
 Filter jails  
@@ -39,8 +53,7 @@ Filter jails
 
 ### By name
 Just add name on gocage list command :  
-<pre><code>
-gocage list srv-bdd srv-web
+<pre><code>gocage list srv-bdd srv-web
 +=====+=========+=================+=======================+=========+
 | JID | Name    | Config.Release  | Config.Ip4_addr       | Running |
 +=====+=========+=================+=======================+=========+
@@ -48,12 +61,11 @@ gocage list srv-bdd srv-web
 +-----+---------+-----------------+-----------------------+---------+
 | 41  | srv-web | 13.0-RELEASE-p4 | vnet0|192.168.1.26/24 | true    |
 +-----+---------+-----------------+-----------------------+---------+
-</pre></code>
+</code></pre>
 
 ### By field value
 You can filter jails with -f option, followed by key=value. Suppose you want to see only active at boot jails:  
-<pre><code>
-gocage list -f Config.Boot=1 -o JID,Name,Running,Config.Boot,Config.Comment
+<pre><code>gocage list -f Config.Boot=1 -o JID,Name,Running,Config.Boot,Config.Comment
 +=====+==========+=========+=============+================+
 | JID | Name     | Running | Config.Boot | Config.Comment |
 +=====+==========+=========+=============+================+
@@ -65,11 +77,10 @@ gocage list -f Config.Boot=1 -o JID,Name,Running,Config.Boot,Config.Comment
 +-----+----------+---------+-------------+----------------+
 | 22  | srv-dns1 | true    | 1           |                |
 +-----+----------+---------+-------------+----------------+
-</pre></code>
+</code></pre>
 
 Now, only active at boot and running :  
-<pre><code>
-gocage list -f Config.Boot=1,Running=true -o JID,Name,Running,Config.Boot
+<pre><code>gocage list -f Config.Boot=1,Running=true -o JID,Name,Running,Config.Boot
 +=====+==========+=========+=============+
 | JID | Name     | Running | Config.Boot |
 +=====+==========+=========+=============+
@@ -79,13 +90,12 @@ gocage list -f Config.Boot=1,Running=true -o JID,Name,Running,Config.Boot
 +-----+----------+---------+-------------+
 | 22  | srv-dns1 | true    | 1           |
 +-----+----------+---------+-------------+
-</pre></code>
+</code></pre>
 
 Sort jails  
 ----------
 Use -s switch followed by sort criteria. Criteria is a field name, prefixed with + or - for sort order (increase/decrease):  
-<pre><code>
-gocage list -f Config.Boot=1,Running=true -o JID,Name,Running,Config.Boot -s +JID
+<pre><code>gocage list -f Config.Boot=1,Running=true -o JID,Name,Running,Config.Boot -s +JID
 +=====+==========+=========+=============+
 | JID | Name     | Running | Config.Boot |
 +=====+==========+=========+=============+
@@ -95,12 +105,11 @@ gocage list -f Config.Boot=1,Running=true -o JID,Name,Running,Config.Boot -s +JI
 +-----+----------+---------+-------------+
 | 183 | test     | true    | 1           |
 +-----+----------+---------+-------------+
-</pre></code>
+</code></pre>
 
 You can use up to 3 criteria, delimited with comma.  
 As an example, you want to list boot priorities of automatically starting jails:  
-<pre><code>
-gocage list -o JID,Name,Config.Ip4_addr,Config.Priority,Config.Boot,Running -s -Config.Priority,-Config.Boot -f Running=true
+<pre><code>gocage list -o JID,Name,Config.Ip4_addr,Config.Priority,Config.Boot,Running -s -Config.Priority,-Config.Boot -f Running=true
 +=====+==============+=======================+=================+=============+=========+
 | JID | Name         | Config.Ip4_addr       | Config.Priority | Config.Boot | Running |
 +=====+==============+=======================+=================+=============+=========+
@@ -112,28 +121,62 @@ gocage list -o JID,Name,Config.Ip4_addr,Config.Priority,Config.Boot,Running -s -
 +-----+--------------+-----------------------+-----------------+-------------+---------+
 | 4   | coincoin     | vnet0|192.168.1.9/24  | 20              | 0           | true    |
 +-----+--------------+-----------------------+-----------------+-------------+---------+
-</pre></code>
+</code></pre>
 
 
-Stop jails
+Stop jails  
 ----------
-`gocage stop test`
+<pre><code>gocage stop test</code></pre>
+
+Update jails  
+----------
+To update jail patch version, use gocage update :  
+<pre><code>gocage update test</code></pre>
+
+Update basejails/releases  
+----------
+To update basejails, you need to update the release they are based on. Specify release with -r, and the datastore storing concerned release with -d :  
+<pre><code>gocage update -d fastgocage -r 14.1-RELEASE</code></pre>
+
+Upgrade jails  
+----------
+To upgrade jail to newer release, use gocage upgrade :  
+<pre><code>gocage upgrade -r 13.2-RELEASE test</code></pre>
+  
+A pre-upgrade snapshot wil be made so you can rollback if needed.  
+
+Upgrading basejail/release
+----------
+Upgrading basejails currently needs to be done manually, for each jail.  
+The idea is to stop the jail, change the content of its fstab file to point to the new release, then start jail.  
+If one change the fstab while the jail is running, its system directories won't be unmounted at stop time and this will provoke stop errors.  
+To minimize downtime, the change could be scripted:  
+<pre><code>gocage stop jail1
+sed -i .bak 's/14.0-RELEASE/14.1-RELEASE/' /iocage/jails/jail1/fstab
+# Avoid race-condition by waiting for the update in fstab
+until grep -q 14.1-RELEASE /iocage/jails/jail1/fstab; do sleep 0.2; done
+gocage start jail1
+</code></pre>
+
+You can now update ports.
+
+Delete jails  
+----------
+<pre><code>gocage destroy test</code></pre>
 
 
 Multi datastore  
 ----------
 A datastore is a ZFS dataset mounted. It should be declared in gocage.conf.yml, specifying its ZFS mountpoint :
-<pre><code>
-datastore:
+<pre><code>datastore:
   - /iocage
   - /fastiocage
-</pre></code>
+</code></pre>
 
 In gocage commands, datastore name is the mountpoint without its "/" prefix.  
 
 ### List datastores
-<pre><code>
-gocage datastore list 
+<pre><code>gocage datastore list 
 +============+=============+============+===========+==========+============+
 | Name       | Mountpoint  | ZFSDataset | Available | Used     | Referenced |
 +============+=============+============+===========+==========+============+
@@ -141,23 +184,21 @@ gocage datastore list
 +------------+-------------+------------+-----------+----------+------------+
 | fastiocage | /fastiocage | ssd/iocage | 1.5 TB    | 65.3 KB  | 34.6 KB    |
 +------------+-------------+------------+-----------+----------+------------+
-</pre></code>
+</code></pre>
 
-#### Filter datastores  
+### Filter datastores  
 As with jails and snapshots, you can filter by name:  
-<pre><code>
-gocage datastore list iocage
+<pre><code>gocage datastore list iocage
 +============+=============+============+===========+==========+============+
 | Name       | Mountpoint  | ZFSDataset | Available | Used     | Referenced |
 +============+=============+============+===========+==========+============+
 | iocage     | /iocage     | hdd/iocage | 1.6 TB    | 414.9 GB | 27.5 KB    |
 +------------+-------------+------------+-----------+----------+------------+
-</pre></code>
+</code></pre>
 
-#### Sort datastores  
+### Sort datastores  
 You can sort datastores:  
-<pre><code>
-gocage datastore list -s -Available
+<pre><code>gocage datastore list -s -Available
 +============+=============+============+===========+==========+============+
 | Name       | Mountpoint  | ZFSDataset | Available | Used     | Referenced |
 +============+=============+============+===========+==========+============+
@@ -165,9 +206,9 @@ gocage datastore list -s -Available
 +------------+-------------+------------+-----------+----------+------------+
 | fastiocage | /fastiocage | ssd/iocage | 1.5 TB    | 65.3 KB  | 34.6 KB    |
 +------------+-------------+------------+-----------+----------+------------+
-</pre></code>
+</code></pre>
 
-See [cmd/struct.go](https://git.nosd.in/yo/gocage/src/branch/master/cmd/struct.go) for field names.
+Use `gocage properties`to list available fields.
 
 Migrating jails  
 ----------
@@ -175,10 +216,28 @@ With multi datastore comes the need to migrate a jail between datastores.
 Migration can be done with a minimal downtime, using zfs differential send/receive.  
 Source jail datasets are sent to the destination datastore, jail is stopped and a last differential sync is done before starting jail on new datastore.  
 
-<pre><code>
-gocage migrate -d fastiocage srv-random
+### Warning 
+Be aware the moment you migrate a jail to another datastore than /iocage default, you lose compatibility with iocage.  
+Then you need to disable iocage service, and enable gocage so the jails will start automatically at boot.  
+Also make sure, if you don't destroy source jail, that it won't have the "boot" property set or you will have the 2 jails up at boot.  
+
+<pre><code>gocage migrate -d fastiocage srv-random
 Snapshot data/iocage/jails/srv-random: Done
 Snapshot data/iocage/jails/srv-random/root: Done
 Migrate jail config dataset to fastdata/iocage/jails/srv-random: Done
 Migrate jail filesystem dataset to fastdata/iocage/jails/srv-random/root: Done
-</pre></code>
+</code></pre>
+
+
+Fetch  
+----------
+Files can be fetched from custom repository, or from local directory with "from" option.  
+For example if you destroyed releases/12.3-RELEASE and still have the downloaded files in /iocage/download/12.3-RELEASE:  
+<pre><code>gocage fetch -r 12.3 -d iocage -f file:/iocage/download
+</code></pre>
+
+
+TODO
+----------
+gocage create from templates  
+

TODO.md

@@ -1,4 +1,19 @@
 Replicating jails between two servers (use zrepl)
+Manage remote jails :
+  - Make gocage a service
+  - All commands should become API endpoint
+  - How to handle authentication ?
 
+DEBUG: 
+- cmd/list.go:275:
+    // FIXME ??? Shouldn't be ioc-$Name ?
+    j.InternalName = rj.Name
+- WriteConfigToDisk don't write neither "release" in cmd stop neither "last_started" in cmd start
+  26/08/2023 : Last_started is updated
 
+BUGS:
+- unable to set values containing equal sign :
+  # gocage set Config.Exec_poststart="jail -m allow.mount.linprocfs=1 name=ioc-poudriere-noo" poudriere-noo
+  Error parsing args: Config.Exec_poststart=jail -m allow.mount.linprocfs=1 name=ioc-poudriere-noo
+- Fix fstab when migrating jail  
 

cmd/console.go

@@ -12,7 +12,7 @@ import (
 func ShellJail(args []string) error {
 	// We cant shell more than one jail bc we replace gocage execution with jexec, so there wont be no return to gocage
 	if len(args) > 0 {
-		cj, err := getJailFromArray(args[0], gJails)
+		cj, err := getJailFromArray(args[0], []string{"basejail", "jail"}, gJails)
 		if err != nil {
 			fmt.Printf("Error getting jail %s: %v\n", args[0], err)
 			return err
@@ -30,10 +30,9 @@ func shellJail(jail *Jail) error {
 	if false == jail.Running {
 		return errors.New("Jail is not running")
 	}
-	
+
 	jid := strconv.Itoa(jail.JID)
 
-	//err := syscall.Exec("/usr/sbin/jexec", []string{"jexec", jid, "/bin/csh"}, os.Environ())
 	err := syscall.Exec("/usr/sbin/jexec", []string{"jexec", jid, "login", "-f", "root"}, os.Environ())
 
 	// We should never get here, as syscall.Exec replace the gocage binary execution with jexec
@@ -41,6 +40,6 @@ func shellJail(jail *Jail) error {
 	if err != nil {
 		log.Printf("Exec returned %v\n", err)
 	}
-	
+
 	return nil
 }

cmd/create.go

@@ -0,0 +1,300 @@
+package cmd
+
+import (
+	"os"
+	"fmt"
+	//"log"
+	"time"
+	"errors"
+	"strings"
+	cp "github.com/otiai10/copy"
+	log "github.com/sirupsen/logrus"
+)
+
+// TODO : Add a flag to specify which parts of freebsd base we want : Slim jail only need base.txz, neither lib32 nor src.txz
+func CreateJail(args []string) {
+	var err error
+	var jtype []string
+
+	if gCreateArgs.BaseJail && gCreateArgs.Release == "" {
+		fmt.Println("Release should be set when creating basejail")
+		os.Exit(1)
+	}
+
+	if len(gCreateArgs.JailType) > 0 {
+		jtype = []string{gCreateArgs.JailType}
+	}
+	
+	for _, jname := range args {
+		// Check if jail exist and is distinctly named
+		_, err = getJailFromArray(jname, jtype, gJails)
+		if err != nil {
+			if strings.EqualFold(err.Error(), "Jail not found") {
+				
+			} else {
+				fmt.Printf("ERROR: %s\n", err.Error())
+				return
+			}
+		} else {
+			fmt.Printf("Jail exist: %s\n", jname)
+			continue
+		}
+
+		fmt.Printf("  > create jail %s\n", jname)
+
+		var ds *Datastore
+		if len(gCreateArgs.Datastore) > 0 {
+			log.Debugf("Use %s datastore\n", gCreateArgs.Datastore)
+			ds, err = getDatastoreFromArray(gCreateArgs.Datastore, gDatastores)
+			if err != nil {
+				fmt.Printf("ERROR Getting datastore: %s\n", gCreateArgs.Datastore, err.Error())
+				return
+			}
+		} else {
+			ds = &gDatastores[0]
+		}
+
+		// Get base template if specified
+		if gCreateArgs.BaseJail {
+			/**************************************************************************
+			*  Create based jail from a template
+			*/
+			log.Debugf("Jail will be created read-only from release %s\n", gCreateArgs.Release)
+
+			// First check if we got release on the same datastore
+			releasePath := fmt.Sprintf("%s/releases/%s/root", ds.Mountpoint, gCreateArgs.Release)
+			_, err := os.Stat(releasePath)
+			if os.IsNotExist(err) {
+				fmt.Printf("ERROR: Release locally not available. Run \"gocage fetch\"\n")
+				return
+			}
+
+			// Create jail datasets
+			dstDset := fmt.Sprintf("%s/jails/%s", ds.ZFSDataset, jname)
+			fmt.Printf("    > Initialize dataset %s\n", dstDset)
+			err = zfsCreateDataset(dstDset, "", "")
+			if err != nil {
+				fmt.Printf("ERROR creating dataset %s: %s\n", dstDset, err.Error())
+				return
+			}
+			// Create jail root datasets
+			dstRootDset := fmt.Sprintf("%s/jails/%s/root", ds.ZFSDataset, jname)
+			fmt.Printf("    > Initialize dataset %s\n", dstRootDset)
+			err = zfsCreateDataset(dstRootDset, "", "")
+			if err != nil {
+				fmt.Printf("ERROR creating dataset %s: %s\n", dstRootDset, err.Error())
+				return
+			}
+
+			// Create needed directories with basejail permissions
+			fmt.Printf("    > Create base read-only directories\n")
+			dstRootDir := fmt.Sprintf("%s/jails/%s/root", ds.Mountpoint, jname)
+			for _, d := range append(gBaseDirs, gEmptyDirs...) {
+				dstPath := dstRootDir
+				srcPath := releasePath
+				for _, cd := range strings.Split(d, "/") {
+					srcPath = fmt.Sprintf("%s/%s", srcPath, cd)
+					dstPath = fmt.Sprintf("%s/%s", dstPath, cd)
+					_, err := os.Stat(dstPath)
+					if errors.Is(err, os.ErrNotExist) {
+						srcPerm, err := getPermissions(srcPath)
+						if err != nil {
+							fmt.Printf("ERROR getting permissions of %s: %s\n", srcPath, err.Error())
+							return
+						}
+						err = os.Mkdir(dstPath, srcPerm.Mode().Perm())
+						if err != nil {
+							fmt.Printf("ERROR creating directory %s: %s\n", dstPath, err.Error())
+							return
+						}
+					}
+				}
+			}
+
+			// Copy these from basejail
+			fmt.Printf("    > Create base writable directories\n")
+			for _, d := range gCopyDirs {
+				err := cp.Copy(fmt.Sprintf("%s/%s", releasePath, d), fmt.Sprintf("%s/%s", dstRootDir, d))
+				if err != nil {
+					fmt.Printf("ERROR copying %s to %s: %s\n", fmt.Sprintf("%s/%s", releasePath, d),
+						   fmt.Sprintf("%s/%s", dstRootDir, d), err.Error())
+					return
+				}
+			}
+
+			///////////////////////////////////////////////////////////////////////
+			// Copy defaults.json...
+			jailConfPath := fmt.Sprintf("%s/jails/%s/config.json", ds.Mountpoint, jname)
+			err = copyFile(fmt.Sprintf("%s/defaults.json", ds.Mountpoint),
+				       jailConfPath)
+			if err != nil {
+				fmt.Printf("ERROR creating config.json: %s\n", err.Error())
+				return
+			}
+			///////////////////////////////////////////////////////////////////////
+			// ... and update it
+			jailConf, err := getJailConfig(jailConfPath)
+			if err != nil {
+				log.Println("ERROR reading jail config from %s", jailConfPath)
+			}
+
+			// Build jail object from config
+			jailRootPath := fmt.Sprintf("%s/jails/%s/%s", ds.Mountpoint, jname, "root")
+			j := Jail{
+				Name:       jailConf.Host_hostuuid,
+				Config:     jailConf,
+				ConfigPath: jailConfPath,
+				Datastore:  ds.Name,
+				RootPath:   jailRootPath,
+				Running:    false,
+			}
+
+			// We need to store the basejail template. We could :
+			// 1. Use "origin" ?
+			// 2. Add a json item to config ("basejail_template" p.e.), but iocage would delete it once jail is started from iocage
+			// 3. Add a gocage specific config ("config.gocage.json" p.e.)
+			j.Config.Jailtype = "basejail"
+			j.Config.Origin = gCreateArgs.Release
+			j.Config.Host_hostname = jname
+			j.Config.Host_hostuuid = jname
+
+			j.WriteConfigToDisk(false)
+
+			///////////////////////////////////////////////////////////////////////
+			// Create fstab
+			fstabHandle, err := os.Create(fmt.Sprintf("%s/jails/%s/fstab", ds.Mountpoint, jname))
+			if err != nil {
+				fmt.Printf("ERROR creating fstab: %s", err.Error())
+				return
+			}
+			defer fstabHandle.Close()
+
+			for _, d := range gBaseDirs {
+				fmt.Fprintf(fstabHandle, "%s\t%s\tnullfs\tro\t0\t0\n", fmt.Sprintf("%s/%s", releasePath, d), fmt.Sprintf("%s/%s", dstRootDir, d))
+			}
+
+			fmt.Printf("  > Jail created!\n")
+		} else {
+			/**************************************************************************
+			 *  Create normal jail with its own freebsd base
+			 */
+			log.Debugf("Creating jail with its own freebsd base\n")
+
+			// First check if we got release on the same datastore
+			_, err := os.Stat(fmt.Sprintf("%s/releases/%s/root", ds.Mountpoint, gCreateArgs.Release))
+			if os.IsNotExist(err) {
+				fmt.Printf("ERROR: Release locally not available. Run \"gocage fetch\"\n")
+				return
+			}
+
+			///////////////////////////////////////////////////////////////////////
+			// Create and populate jail filesystem from release
+			dstDset := fmt.Sprintf("%s/jails/%s", ds.ZFSDataset, jname)
+			fmt.Printf("    > Initialize dataset %s\n", dstDset)
+			sNow := time.Now().Format("20060102150405")
+			reldset := fmt.Sprintf("%s/releases/%s", ds.ZFSDataset, gCreateArgs.Release)
+			err = zfsSnapshot(reldset, sNow)
+			if err != nil {
+				fmt.Printf("ERROR Creating snapshot of %s: %s\n", reldset, err.Error())
+				return
+			}
+
+			err = zfsCopy(fmt.Sprintf("%s@%s", reldset, sNow), dstDset)
+			if err != nil {
+				fmt.Printf("ERROR sending snapshot to %s: %s\n", dstDset, err.Error())
+				return
+			}
+			// Remove snapshot of release, then snapshot of destination dataset
+			err = zfsDestroy(fmt.Sprintf("%s@%s", reldset, sNow))
+			if err != nil {
+				fmt.Printf("ERROR destroying snapshot %s: %s\n", reldset, err.Error())
+				return
+			}
+			err = zfsDestroy(fmt.Sprintf("%s@%s", dstDset, sNow))
+			if err != nil {
+				fmt.Printf("ERROR destroying snapshot %s: %s\n", dstDset, err.Error())
+				return
+			}
+
+			dstRootDset := fmt.Sprintf("%s/jails/%s/root", ds.ZFSDataset, jname)
+			fmt.Printf("    > Initialize dataset %s\n", dstRootDset)
+			relrootdset := fmt.Sprintf("%s/releases/%s/root", ds.ZFSDataset, gCreateArgs.Release)
+			err = zfsSnapshot(relrootdset, sNow)
+			if err != nil {
+				fmt.Printf("ERROR Creating snapshot of %s: %s\n", relrootdset, err.Error())
+				return
+			}
+			err = zfsCopy(fmt.Sprintf("%s@%s", relrootdset, sNow), dstRootDset)
+			if err != nil {
+				fmt.Printf("ERROR sending snapshot to %s: %s\n", dstRootDset, err.Error())
+				return
+			}
+			// Remove snapshot of release, then snapshot of destination dataset
+			err = zfsDestroy(fmt.Sprintf("%s@%s", relrootdset, sNow))
+			if err != nil {
+				fmt.Printf("ERROR destroying snapshot %s: %s\n", relrootdset, err.Error())
+				return
+			}
+			err = zfsDestroy(fmt.Sprintf("%s@%s", dstRootDset, sNow))
+			if err != nil {
+				fmt.Printf("ERROR destroying snapshot %s: %s\n", dstRootDset, err.Error())
+				return
+			}
+
+			///////////////////////////////////////////////////////////////////////
+			// Copy defaults.json...
+			jailConfPath := fmt.Sprintf("%s/jails/%s/config.json", ds.Mountpoint, jname)
+			err = copyFile(fmt.Sprintf("%s/defaults.json", ds.Mountpoint), 
+					jailConfPath)
+			if err != nil {
+				fmt.Printf("ERROR creating config.json: %s\n", err.Error())
+				return
+			}
+			///////////////////////////////////////////////////////////////////////
+			// ... and update it
+			jailConf, err := getJailConfig(jailConfPath)
+			if err != nil {
+				log.Println("ERROR reading jail config from %s", jailConfPath)
+			}
+
+			// Build jail object from config
+			jailRootPath := fmt.Sprintf("%s/jails/%s/%s", ds.Mountpoint, jname, "root")
+			j := Jail{
+				Name:       jailConf.Host_hostuuid,
+				Config:     jailConf,
+				ConfigPath: jailConfPath,
+				Datastore:  ds.Name,
+				RootPath:   jailRootPath,
+				Running:    false,
+			}
+
+			j.Config.Release = gCreateArgs.Release
+			j.Config.Host_hostname = jname
+			j.Config.Host_hostuuid = jname
+			j.Config.Jailtype = "jail"
+			j.WriteConfigToDisk(false)
+
+			///////////////////////////////////////////////////////////////////////
+			// Create fstab
+			fstabHandle, err := os.Create(fmt.Sprintf("%s/jails/%s/fstab", ds.Mountpoint, jname))
+			if err != nil {
+				fmt.Printf("ERROR creating fstab: %s", err.Error())
+				return
+			}
+			defer fstabHandle.Close()
+			fmt.Printf("  > Jail created!\n")
+		}
+
+		var cmdline []string
+		for _, props := range strings.Split(gCreateArgs.Properties, ",") {
+			cmdline = append(cmdline, props)
+		}
+
+		// Reload jail list so SetJailProperties will see it
+		ListJails(nil, false)
+
+		cmdline = append(cmdline, jname)
+		log.Debugf("cmdline: \"%v\"", cmdline)
+		SetJailProperties(cmdline)
+	}
+}

cmd/destroy.go

@@ -0,0 +1,59 @@
+package cmd
+
+import (
+	"fmt"
+	//"log"
+	"time"
+	//"errors"
+	"strings"
+)
+
+func DestroyJails(args []string) {
+	for _, a := range args {
+		cj, err := getJailFromArray(a, []string{""}, gJails)
+		if err != nil {
+			fmt.Printf("Error getting jail: %s\n", err)
+			return
+		}
+		
+		if cj.Running == true {
+			fmt.Printf("Jail %s is running\n", cj.Name)
+			if gForce == false {
+				var answer string
+				fmt.Printf("Stop jail and delete? (y/n) ")
+				fmt.Scanln(&answer)
+				if false == strings.EqualFold(answer, "y") {
+					return
+				}
+			}
+			fmt.Printf("Stopping jail %s\n", cj.Name)
+			StopJail([]string{fmt.Sprintf("%s/%s", cj.Datastore, cj.Name)})
+			// Give some time to the host OS to free all mounts accessing processes
+			time.Sleep(1 * time.Second)
+		}
+		
+		// Get root and config datasets, then destroy
+		dsRootName, err := zfsGetDatasetByMountpoint(cj.RootPath)
+		if err != nil {
+			fmt.Printf("Error getting root dataset: %s\n", err)
+			return
+		}
+		//fmt.Printf("DEBUG: Prepare to zfs destroy %s\n", dsRootName)
+		if err = zfsDestroy(dsRootName); err != nil {
+			fmt.Printf("Error deleting root dataset: %s\n", err)
+			return
+		}
+		
+		dsConfName, err := zfsGetDatasetByMountpoint(cj.ConfigPath)
+		if err != nil {
+			fmt.Printf("Error getting config dataset: %s\n", err)
+			return
+		}
+		//fmt.Printf("DEBUG: Prepare to zfs destroy %s\n", dsConfName)
+		if err = zfsDestroy(dsConfName); err != nil {
+			fmt.Printf("Error deleting config dataset: %s\n", err)
+			return
+		}
+		fmt.Printf("Jail %s is no more!\n", cj.Name)
+	}
+}

cmd/fetch.go

@@ -0,0 +1,378 @@
+package cmd
+
+import (
+	"io"
+	"os"
+	"fmt"
+	"bufio"
+	"bytes"
+	//"errors"
+	"strings"
+	"net/http"
+	//"archive/tar"
+	"encoding/hex"
+	"crypto/sha256"
+
+	//"github.com/ulikunitz/xz"
+	log "github.com/sirupsen/logrus"
+)
+
+const (
+	ReleaseServer = "download.freebsd.org"
+	ReleaseRootDir = "ftp/releases"
+)
+
+var (
+	// TODO : Make this a config/cmd line setting
+	//FetchFiles = []string{"base.txz", "lib32.txz", "src.txz"}
+	FetchFiles = []string{"base.txz"}
+)
+
+// TODO: Check if files already exist
+// Fetch release files, verify, put in datastore under ${datastore}/download
+// Only support http and file protocols
+func fetchRelease(release string, proto string, arch string, datastore string, fetchFrom string) error {
+	var ds Datastore
+	
+	log.SetReportCaller(true)
+	
+	if len(fetchFrom) > 0 {
+		proto = strings.Split(fetchFrom, ":")[0]
+	}
+	
+	if false == strings.EqualFold(proto, "http") &&
+		false == strings.EqualFold(proto, "file") {
+		return fmt.Errorf("Unsupported protocol: %s\n", proto)
+	}
+	
+	for _, ds = range gDatastores {
+		if strings.EqualFold(datastore, ds.Name) {
+			break
+		}
+	}
+	if false == strings.EqualFold(datastore, ds.Name) {
+		return fmt.Errorf("Datastore not found: %s\n", datastore)
+	}
+	
+	// Check datastore have a download dataset, and it is mounted
+	downloadDsName := fmt.Sprintf("%s/download", ds.ZFSDataset)
+	downloadDsMountPoint := fmt.Sprintf("%s/download", ds.Mountpoint)
+	exist, err := doZfsDatasetExist(downloadDsName)
+	if err != nil {
+		return fmt.Errorf("Error accessing dataset %s: %v\n", downloadDsName, err)
+	}
+	if false == exist {
+		// Then create dataset
+		if err := zfsCreateDataset(downloadDsName, downloadDsMountPoint, "lz4"); err != nil {
+			return fmt.Errorf("Error creating dataset %s: %v\n", downloadDsName, err)
+		}
+	}
+	
+	// Create download/XX.X dataset if necessary
+	thisDownloadDsName := fmt.Sprintf("%s/%s", downloadDsName, release)
+	thisDownloadDsMountPoint := fmt.Sprintf("%s/%s", downloadDsMountPoint, release)
+	exist, err = doZfsDatasetExist(thisDownloadDsName)
+	if err != nil {
+		return fmt.Errorf("Error accessing dataset %s: %v\n", thisDownloadDsName, err)
+	}
+	if false == exist {
+		// Then create dataset
+		if err := zfsCreateDataset(thisDownloadDsName, thisDownloadDsMountPoint, "lz4"); err != nil {
+			return fmt.Errorf("Error creating dataset %s: %v\n", thisDownloadDsName, err)
+		}
+	}
+
+	var fetchUrl string
+	if len(fetchFrom) > 0 {
+		fetchUrl = fmt.Sprintf("%s/%s", fetchFrom, release)
+	} else {
+		fetchUrl = fmt.Sprintf("%s://%s/%s/%s/%s", proto, ReleaseServer, ReleaseRootDir, arch, release)
+	}
+	log.Debugf("FetchURL = %s", fetchUrl)
+	
+	// check if proto/server/arch/release is available
+	if strings.EqualFold(proto, "http") {
+		resp, err := http.Get(fetchUrl)
+		if err != nil {
+			return fmt.Errorf("Can not get %s: %v\n", fetchUrl, err)
+		}
+		defer resp.Body.Close()
+		if resp.StatusCode != 200 {
+			return fmt.Errorf("Get %s returned %d, check release name\n", fetchUrl, resp.StatusCode)
+		}
+	}
+
+	// Fetch files
+	// Get MANIFEST so we get sha256 sums
+	if err := fetchFile(proto, fetchUrl, "MANIFEST", thisDownloadDsMountPoint, []byte{}); err != nil {
+		return fmt.Errorf("%v\n", err)
+	}
+	// Build an array of "file;checksum"
+	checksumMap, err := buildFileChecksumFromManifest(fmt.Sprintf("%s/MANIFEST", thisDownloadDsMountPoint), FetchFiles)
+	if err != nil {
+		return fmt.Errorf("%v\n", err)
+	}
+
+	// Fetch remaining files, verify integrity and write to disk
+	for f, c := range checksumMap {
+		if err := fetchFile(proto, fetchUrl, f, thisDownloadDsMountPoint, c); err != nil {
+			return fmt.Errorf("%v\n", err)
+		}
+	}
+	
+	return nil
+}
+
+// Extract release files stored in iocage/download/$RELEASE/ to iocage/releases/$RELEASE/root/
+func extractRelease(release string, datastore string) {
+	log.SetReportCaller(true)
+	
+	var ds Datastore
+	
+	for _, ds = range gDatastores {
+		if strings.EqualFold(datastore, ds.Name) {
+			break
+		}
+	}
+	if false == strings.EqualFold(datastore, ds.Name) {
+		fmt.Printf("Datastore not found: %s\n", datastore)
+		return
+	}
+	
+	// Check datastore have a releases dataset, and it is mounted
+	releaseDsName := fmt.Sprintf("%s/releases", ds.ZFSDataset)
+	releaseDsMountPoint := fmt.Sprintf("%s/releases", ds.Mountpoint)
+	exist, err := doZfsDatasetExist(releaseDsName)
+	if err != nil {
+		fmt.Printf("Error accessing dataset %s: %v\n", releaseDsName, err)
+		return
+	}
+	if false == exist {
+		// Then create dataset
+		if err := zfsCreateDataset(releaseDsName, releaseDsMountPoint, "lz4"); err != nil {
+			fmt.Printf("Error creating dataset %s: %v\n", releaseDsName, err)
+			return
+		}
+	}
+	
+	// Create releases/XX.X dataset if necessary
+	thisReleaseDsName := fmt.Sprintf("%s/%s", releaseDsName, release)
+	thisReleaseDsMountPoint := fmt.Sprintf("%s/%s", releaseDsMountPoint, release)
+	exist, err = doZfsDatasetExist(thisReleaseDsName)
+	if err != nil {
+		fmt.Printf("Error accessing dataset %s: %v\n", thisReleaseDsName, err)
+		return
+	}
+	if false == exist {
+		// Then create dataset
+		if err := zfsCreateDataset(thisReleaseDsName, thisReleaseDsMountPoint, "lz4"); err != nil {
+			fmt.Printf("Error creating dataset %s: %v\n", thisReleaseDsName, err)
+			return
+		}
+	}
+	
+	// Create releases/XX.X/root dataset if necessary
+	thisReleaseRootDsName := fmt.Sprintf("%s/root", thisReleaseDsName)
+	thisReleaseRootDsMountPoint := fmt.Sprintf("%s/root", thisReleaseDsMountPoint)
+	exist, err = doZfsDatasetExist(thisReleaseRootDsName)
+	if err != nil {
+		fmt.Printf("Error accessing dataset %s: %v\n", thisReleaseRootDsName, err)
+		return
+	}
+	if false == exist {
+		// Then create dataset
+		if err := zfsCreateDataset(thisReleaseRootDsName, thisReleaseRootDsMountPoint, "lz4"); err != nil {
+			fmt.Printf("Error creating dataset %s: %v\n", thisReleaseRootDsName, err)
+			return
+		}
+	}
+	
+	// Now extract download/$RELEASE/*.txz to releases/XX.X/root
+	downloadDsMountPoint := fmt.Sprintf("%s/download", ds.Mountpoint)
+	downloadDir := fmt.Sprintf("%s/%s", downloadDsMountPoint, release)
+	
+	d, err := os.Open(downloadDir)
+	defer d.Close()
+	if err != nil {
+		fmt.Printf("Can not read %s directory: %v\n", downloadDir, err)
+		return
+	}
+	files, err := d.Readdir(0)
+	if err != nil {
+		fmt.Printf("Can not browse %s directory: %v\n", downloadDir, err)
+		return
+	}
+	
+	// Extract every .txz files in FetchFiles
+	for _, fi := range files {
+		if false == fi.IsDir() {
+			if strings.HasSuffix(fi.Name(), ".txz") {
+				if isStringInArray(FetchFiles, fi.Name()) {
+					ar := fmt.Sprintf("%s/%s", downloadDir, fi.Name())
+					fmt.Printf("Extracting file %s to %s... ", ar, thisReleaseRootDsMountPoint)
+					// pure Go method, sorry this is so slow. Also I did not handle permissions in this
+	/*				f, err := os.Open(ar)
+					defer f.Close()
+					if err != nil {
+						fmt.Printf("Can not open %s: %v\n", ar, err)
+						return
+					}
+					// xz reader
+					r, err := xz.NewReader(f)
+					if err != nil {
+						fmt.Printf("Can not read %s: %v\n", ar, err)
+						return
+					}
+					// tar reader
+					tr := tar.NewReader(r)
+					// Iterate through the files in the archive.
+					for {
+						hdr, err := tr.Next()
+						if err == io.EOF {
+							// end of tar archive
+							break
+						}
+						if err != nil {
+							log.Fatal(err)
+						}
+						switch hdr.Typeflag {
+							case tar.TypeDir:
+								// create a directory
+								dest := fmt.Sprintf("%s/%s", thisReleaseRootDsMountPoint, hdr.Name)
+								// FIXME: Access rights?
+								err = os.MkdirAll(dest, 0777)
+								if err != nil {
+									log.Fatal(err)
+								}
+							case tar.TypeReg, tar.TypeRegA:
+								// write a file
+								dest := fmt.Sprintf("%s/%s", thisReleaseRootDsMountPoint, hdr.Name)
+								w, err := os.Create(dest)
+								defer w.Close()
+								if err != nil {
+									log.Fatal(err)
+								}
+								_, err = io.Copy(w, tr)
+								if err != nil {
+									log.Fatal(err)
+								}
+						}
+					}
+	*/
+					cmd := fmt.Sprintf("/usr/bin/tar xpf %s -C %s", ar, thisReleaseRootDsMountPoint)
+					out, err := executeCommand(cmd)
+					if err != nil && len(out) > 0 {
+						fmt.Printf("Error: %v: %s\n", err, out)
+					} else {
+						fmt.Printf("Done\n")
+					}
+				}
+			}
+		}
+	}
+}
+
+func fetchFile(proto, baseUrl, fileName, storeDir string, checksum []byte) error {
+	// Check storeDir exist
+	_, err := os.Stat(storeDir)
+	if os.IsNotExist(err) {
+		return fmt.Errorf("Directory does not exist: %s\n", storeDir)
+	}
+
+	url := fmt.Sprintf("%s/%s", baseUrl, fileName)
+	fmt.Printf("Fetching %s...", url)
+	
+	var body []byte
+	if strings.EqualFold(proto, "http") {
+		resp, err := http.Get(url)
+		if err != nil {
+			fmt.Printf(" Error\n")
+			return fmt.Errorf("Can not get %s: %v\n", url, err)
+		}
+		defer resp.Body.Close()
+		body, err = io.ReadAll(resp.Body)
+		if err != nil {
+			return fmt.Errorf("Can not read %s response body: %v\n", url, err)
+		}
+	} else if strings.EqualFold(proto, "file") {
+		url = strings.Replace(url, "file:", "", 1)
+		f, err := os.Open(url)
+		if err != nil {
+			return fmt.Errorf("Error accessing file %s", url)
+		}
+		defer f.Close()
+		body, err = io.ReadAll(f)
+		if err != nil {
+			return fmt.Errorf("Can not read file %s: %v\n", url, err)
+		}
+	}
+	
+	// Check integrity
+	if len(checksum) > 0 {
+		err = checkIntegrity(body, checksum)
+		if err != nil {
+			return fmt.Errorf("Error checking integrity")
+		}
+	}
+	
+	dest := fmt.Sprintf("%s/%s", storeDir, fileName)
+
+	f, err := os.Create(dest)       // creates a file at current directory
+	if err != nil {
+		return fmt.Errorf("Can not create file %s: %v\n", dest, err)
+	}
+	defer f.Close()
+	f.Write(body)
+	fmt.Printf(" Done\n")
+	return nil
+}
+
+
+func checkIntegrity(data []byte, checksum []byte) error {
+	sum := sha256.Sum256(data)
+	
+	if false == bytes.Equal(checksum[:],sum[:]) {
+		return fmt.Errorf("Invalid checksum: %x != %x", sum, checksum)
+	}
+	return nil
+}
+
+// Get checksum from manifest, for each file in fileList
+/* MANIFEST format:
+ * base-dbg.txz    a5b51f3d54686509e91ca9c30e9f1cd93dc757f25c643609b3c35e7119c0531d        1654    base_dbg        "Base system (Debugging)"       off
+ * base.txz        e85b256930a2fbc04b80334106afecba0f11e52e32ffa197a88d7319cf059840        26492   base    "Base system (MANDATORY)"       on
+ * kernel-dbg.txz  6b47a6cb83637af1f489aa8cdb802d9db936ea864887188cfc69d8075762214e        912     kernel_dbg      "Kernel (Debugging)"    on
+ */
+func buildFileChecksumFromManifest(manifest string, fileList []string) (map[string][]byte, error) {
+	var ckarr = make(map[string][]byte)
+
+	rm, err := os.Open(manifest)
+	if err != nil {
+		return ckarr, fmt.Errorf("Unable to open MANIFEST: %v", err)
+	}
+	fscan := bufio.NewScanner(rm)
+	fscan.Split(bufio.ScanLines)
+	
+	// For each MANIFEST line...
+	for fscan.Scan() {
+		fields := strings.Fields(fscan.Text())
+		fn := fields[0]
+		fck := fields[1]
+		hexSum, err := hex.DecodeString(fck)
+		if err != nil {
+			return ckarr, fmt.Errorf("Invalid value for checksum %s", fck)
+		}
+		// ... Find the corresponding file in fileList, then add to checksum array ckarr
+		for _, f := range fileList {
+			if strings.EqualFold(f, fn) {
+				ckarr[fn] = hexSum
+				break
+			}
+		}
+	}
+	
+	if len(ckarr) < len(fileList) {
+		return ckarr, fmt.Errorf("Missing file in MANIFEST")
+	}
+	return ckarr, nil
+}

cmd/init.go

@@ -0,0 +1,153 @@
+package cmd
+
+import (
+	"os"
+	"fmt"
+	"strings"
+
+	"github.com/spf13/viper"
+	log "github.com/sirupsen/logrus"
+)
+
+/********************************************************************************
+ * Initialize datastore(s) /iocage, /iocage/jails
+ * Put defaults.json,
+ * Update it with hostid, interfaces, and maybe other necessary fields
+ * Initialize bridge
+ *******************************************************************************/
+func InitGoCage(args []string) {
+	// Create datastores
+	for _, dstore := range viper.GetStringSlice("datastore") {
+		log.Debugf("Ranging over %v\n", dstore)
+		dset, err := zfsGetDatasetByMountpoint(dstore)
+		if err != nil && strings.HasSuffix(err.Error(), "No such file or directory\"") {
+			if len(gZPool) == 0 {
+				log.Errorf("Datastore mountpoint \"%s\" does not exist. Specify a pool if you want to create it.", dstore)
+				return
+			}
+			// Create dataset /iocage
+			rootDSName := fmt.Sprintf("%s%s", gZPool, dstore)
+			log.Debugf("Creating dataset %s mounted on %s\n", rootDSName, dstore)
+			if err = zfsCreateDataset(rootDSName, dstore, ""); err != nil {
+				log.Errorf("Error creating dataset %s: %v\n", rootDSName, err)
+				return
+			}
+			// Create /iocage/jail, releases, templates
+			for _, l := range []string{"jails","releases","templates"} {
+				cds := fmt.Sprintf("%s/%s", rootDSName, l)
+				cmp := fmt.Sprintf("%s/%s", dstore, l)
+				log.Debugf("Creating dataset %s mounted on %s\n", cds, cmp)
+				if err = zfsCreateDataset(cds, cmp, ""); err != nil {
+					log.Errorf("Error creating dataset %s: %v\n", cds, err)
+					return
+				}
+			}
+			
+			// Create /iocage/defaults.json
+			exists, err := doFileExist(fmt.Sprintf("%s/defaults.json", dstore))
+			if err != nil {
+				log.Errorf("Error checking defaults.json: %v\n", err)
+				return
+			}
+			if !exists {
+				if err = createDefaultsJson(dstore, gBridge); err != nil {
+					log.Errorf("%v\n", err)
+				}
+			}
+		} else if err != nil {
+			log.Errorf("Error checking datastore existence: %v\n", err)
+			return
+		} else {
+			log.Debugf("Datastore dataset exist: %s\n", dset)
+		}
+	}
+	// Check and create bridge
+	// FIXME: What if bridge name is invalid, as we already wrote it in defaults.json in dstore loop?
+	if len(gBridge) > 0 && len(gInterface) > 0 {
+		if err := initBridge(); err != nil {
+			log.Errorf("%v\n", err)
+		}
+	}
+}
+
+func createDefaultsJson(rootDirectory string, bridge string) error {
+	hostid, err := os.ReadFile("/etc/hostid")
+	if err != nil {
+		log.Fatalf("Unable to read /etc/hostid: %v\n", err)
+	}
+	json := strings.Replace(gDefaultsJson, "TO-BE-REPLACED-WITH-HOSTID", strings.Trim(string(hostid), "\n"), 1)
+	json = strings.Replace(json, "TO-BE-REPLACED-WITH-BRIDGE", bridge, 1)
+	if err := os.WriteFile(fmt.Sprintf("%s/defaults.json", rootDirectory), []byte(json), 0640); err != nil {
+		log.Fatal(err)
+	}
+	
+	return nil
+}
+
+func createInterface(iface string) error {
+	log.Debugf("creating interface \"%s\"\n", iface)
+	cmd := fmt.Sprintf("/sbin/ifconfig %s create", iface)
+	_, err := executeCommand(cmd)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func bringUpInterface(iface string) error {
+	log.Debugf("bringing up interface \"%s\"\n", iface)
+	cmd := fmt.Sprintf("/sbin/ifconfig %s up", iface)
+	_, err := executeCommand(cmd)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func addMemberToBridge(bridge string, iface string) error {
+	log.Debugf("adding member interface \"%s\" to bridge \"%s\"\n", iface, bridge)
+	cmd := fmt.Sprintf("/sbin/ifconfig %s addm %s", bridge, iface)
+	_, err := executeCommand(cmd)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func initBridge() error {
+	hostInt, err := gJailHost.GetInterfaces()
+	if err != nil {
+		return fmt.Errorf("Error listing interfaces: %v\n", err)
+	}
+	if !isStringInArray(hostInt, gInterface) {
+		return fmt.Errorf("Interface not found: %s\n", gInterface)
+	}
+	if !isStringInArray(hostInt, gBridge) {
+		if err := createInterface(gBridge); err != nil {
+			return fmt.Errorf("Error creating bridge: %v\n", err)
+		}
+		if err := bringUpInterface(gBridge); err != nil {
+			return fmt.Errorf("Error bringing up bridge: %v\n", err)
+		}
+		log.Infof("bridge was created, but it won't persist reboot. Configure rc.conf to persist. See https://docs.freebsd.org/en/books/handbook/advanced-networking/#network-bridging\n")
+		log.Infof("It is strongly suggested you move interface %s IP to bridge %s\n", gInterface, gBridge)
+	}
+	// FIXME: Need to check if not already member
+	members, err := getBridgeMembers(gBridge)
+	if err != nil {
+		return fmt.Errorf("Error getting bridge members: %v\n", err)
+	}
+	
+	// Return if interface already member of the bridge
+	for _, m := range members {
+		log.Debugf("Bridge member: %s\n", m)
+		if strings.EqualFold(m, gInterface) {
+			return nil
+		}
+	}
+	if err := addMemberToBridge(gBridge, gInterface); err != nil {
+		return fmt.Errorf("Error adding interface to bridge: %v\n", err)
+	}
+
+	return nil
+}

cmd/jailhost.go

@@ -4,7 +4,6 @@ import (
 	"fmt"
 	"net"
 	"regexp"
-	"strconv"
 	"strings"
 	"io/ioutil"
 	"golang.org/x/net/route"
@@ -179,36 +178,28 @@ func getHostId() (string, error) {
 	return strings.Split(string(content), "\n")[0], nil
 }
 
+func getArch() (string, error) {
+	out, err := executeCommand("/usr/bin/uname -p")
+	if err != nil {
+		return "", fmt.Errorf("Error executing \"/usr/bin/uname -p\": %v", err)
+	}
+	
+	return strings.Split(out, "\n")[0], nil
+}
+
 func getFreeBSDVersion() (FreeBSDVersion, error) {
 	var version FreeBSDVersion
-	regex := `([0-9]{1,2})(\.)?([0-9]{1,2})?\-([^\-]*)(\-)?(p[0-9]{1,2})?`
-	
-	re := regexp.MustCompile(regex)
 	
 	out, err := executeCommand("/bin/freebsd-version")
 	if err != nil {
 		return version, fmt.Errorf("Error executing \"/bin/freebsd-version\": %v", err)
 	}
 	
-	if re.MatchString(out) {
-		version.major, err = strconv.Atoi(re.FindStringSubmatch(out)[1])
-		if err != nil {
-			return version, err
-		}
-		version.minor, err = strconv.Atoi(re.FindStringSubmatch(out)[3])
-		if err != nil {
-			return version, err
-		}
-		version.flavor = strings.Trim(re.FindStringSubmatch(out)[4], "\n")
-		
-		// Skip the 'p' starting patch level
-		if len(re.FindStringSubmatch(out)[6]) > 0 {
-			version.patchLevel, err = strconv.Atoi(re.FindStringSubmatch(out)[6][1:])
-			if err != nil {
-				return version, err
-			}
-		}
+	version, err = freebsdVersionToStruct(out)
+	if err != nil {
+		return version, err
 	}
+
 	return version, nil
 }
 
@@ -219,6 +210,9 @@ func NewJailHost() (JailHost, error) {
 	if jh.hostname, err = getHostname(); err != nil {
 		return jh, err
 	}
+	if jh.arch, err = getArch(); err != nil {
+		return jh, err
+	}
 	if jh.hostid, err = getHostId(); err != nil {
 		return jh, err
 	}

cmd/list.go

@@ -52,7 +52,7 @@ func ListJails(args []string, display bool) {
 	for _, ds := range gDatastores {
 		listJailsFromDatastore(ds, args, display)
 	}
-	
+
 	// Only when displaying jails, we accept to process multiple same name jails
 	if false == display {
 		for _, j := range gJails {
@@ -68,11 +68,11 @@ func ListJails(args []string, display bool) {
 					break
 				}
 			}
-			
+
 			if true == skip {
 				continue
 			}
-			
+
 			// Initialize if not found in nameChecked
 			if false == found {
 				curCheck = &uniqueJailName{jail: j.Name,
@@ -81,11 +81,11 @@ func ListJails(args []string, display bool) {
 			} else {
 				found = false
 			}
-			
+
 			if countOfJailsWithThisName(j.Name) > 1 {
 				//fmt.Printf("DEBUG: Jail %s exist multiple times, now checking if specified with full name\n", j.Name)
 				curCheck.unique = false
-				
+
 				for _, a := range args {
 					//fmt.Printf("DEBUG: comparing %s/%s with %s\n", j.Datastore, j.Name, a)
 					if strings.EqualFold(a, fmt.Sprintf("%s/%s", j.Datastore, j.Name)) {
@@ -96,7 +96,7 @@ func ListJails(args []string, display bool) {
 			}
 			nameChecked = append(nameChecked, curCheck)
 		}
-		
+
 		// Now check
 		for _, a := range args {
 			for _, n := range nameChecked {
@@ -107,9 +107,9 @@ func ListJails(args []string, display bool) {
 			}
 		}
 	}
-	
+
 	fields := strings.Split(gDisplayJColumns, ",")
-	
+
 	// This is the structure we will filter, then display
 	var jails []Jail
 
@@ -249,7 +249,7 @@ func listJailsFromDirectory(dir string, dsname string) ([]Jail, error) {
 			jailConfPath := fmt.Sprintf("%s/%s/%s", dir, fi.Name(), "config.json")
 			jailConf, err := getJailConfig(jailConfPath)
 			if err != nil {
-				log.Println("ERROR reading jail config for %s", jailConfPath)
+				fmt.Printf("ERROR reading jail config from %s\n", jailConfPath)
 			}
 
 			// 2. Build jail object from config
@@ -272,7 +272,16 @@ func listJailsFromDirectory(dir string, dsname string) ([]Jail, error) {
 				if rj.Path == j.RootPath {
 					j.JID = rj.Jid
 					j.Running = true
+					// FIXME ??? Shouldn't be ioc-$Name ?
 					j.InternalName = rj.Name
+					j.Devfs_ruleset = rj.Devfs_ruleset
+					// Update release
+					r, err := getVersion(&j)
+					if err != nil {
+						fmt.Printf("ERROR getting jail %s version: %s\n", j.Name, err.Error())
+					} else {
+						j.Config.Release = r
+					}
 					break
 				}
 			}

cmd/migrate.go

@@ -26,7 +26,7 @@ func MigrateJail(args []string) {
 	}
 
 	for _, jn := range jailNames {
-		cj, err := getJailFromArray(jn, gJails)
+		cj, err := getJailFromArray(jn, []string{""}, gJails)
 		if cj == nil {
 			fmt.Printf("Error getting jail %s: Not found\n", jn)
 			return
@@ -177,7 +177,7 @@ func CleanMigrateMess(args []string) error {
 	}
 
 	for _, jn := range jailNames {
-		cj, err := getJailFromArray(jn, gJails)
+		cj, err := getJailFromArray(jn, []string{""}, gJails)
 		if cj == nil {
 			return errors.New(fmt.Sprintf("Error getting jail %s: Not found\n", jn))
 		}

cmd/properties.go

@@ -1,8 +1,9 @@
 package cmd
 
 import (
-	"errors"
+	"os"
 	"fmt"
+	"errors"
 	"reflect"
 	"strconv"
 	"strings"
@@ -17,10 +18,10 @@ func GetJailProperties(args []string) {
 		for i, a := range args {
 			// Last arg is the jail name
 			if i == len(args)-1 {
-				jail, err = getJailFromArray(a, gJails)
+				jail, err = getJailFromArray(a, []string{""}, gJails)
 				if err != nil {
 					fmt.Printf("Error: %s\n", err.Error())
-					return
+					os.Exit(1)
 				}
 			} else {
 				props = append(props, a)
@@ -105,19 +106,20 @@ func SetJailProperties(args []string) {
 		return
 	}
 
-	// Get jail by index to modify it
-	for i, _ := range gJails {
-		if gJails[i].Name == jail.Name {
-			for _, p := range props {
-				err := setStructFieldValue(&gJails[i], p.name, p.value)
-				if err != nil {
-					fmt.Printf("Error: %s\n", err.Error())
-					return
-				} else {
-					fmt.Printf("%s: %s set to %s\n", gJails[i].Name, p.name, p.value)
-					gJails[i].ConfigUpdated = true
-				}
-			}
+	cj, err := getJailFromArray(jail.Name, []string{""}, gJails)
+	if err != nil {
+		fmt.Printf("Error getting jail %s: %v\n", jail.Name, err)
+		return
+	}
+
+	for _, p := range props {
+		err := setStructFieldValue(cj, p.name, p.value)
+		if err != nil {
+			fmt.Printf("Error: %s\n", err.Error())
+			return
+		} else {
+			fmt.Printf("%s: %s set to %s\n", cj.Name, p.name, p.value)
 		}
 	}
+	cj.WriteConfigToDisk(false)
 }

cmd/root.go

@@ -1,11 +1,11 @@
 package cmd
 
 import (
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
 	"os"
+	"fmt"
+	"sync"
 	"strings"
+	"io/ioutil"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
@@ -14,12 +14,20 @@ import (
 )
 
 const (
-	gVersion = "0.30"
+	gVersion = "0.42h"
 	
 	// TODO : Get from $jail_zpool/defaults.json
 	MIN_DYN_DEVFS_RULESET = 1000
 )
 
+type createArgs struct {
+	Release      string
+	BaseJail     bool
+	Datastore    string
+	JailType     string
+	Properties   string
+}
+
 var (
 	gJailHost   JailHost
 	gJails      []Jail
@@ -28,6 +36,8 @@ var (
 	gUseSudo bool
 	gForce   bool
 	gDebug   bool
+	
+	gCreateArgs      createArgs
 
 	gConfigFile      string
 	gDisplayJColumns string
@@ -42,15 +52,36 @@ var (
 	gNoJailLineSep   bool
 	gNoSnapLineSep   bool
 	gNoDSLineSep     bool
+	gBridgeStaticMac bool
 
 	gHostVersion float64
 
 	gTimeZone     string
 	gSnapshotName string
+	gZPool        string
+	gBridge       string
+	gInterface    string
 
 	gMigrateDestDatastore string
 	gYesToAll             bool
 
+	gFetchRelease    string
+	gFetchIntoDS     string
+	gFetchFrom       string
+	gUpgradeRelease  string
+	gUpdateRelease   string
+	gUpdateReleaseDS string
+
+	// For a based jail, these are directories binded to basejail
+	gBaseDirs = []string{"bin", "boot", "lib", "libexec", "rescue", "sbin", "usr/bin", "usr/include",
+		"usr/lib", "usr/lib32", "usr/libdata", "usr/libexec", "usr/sbin", "usr/share"}
+	// These directories are to be created empty
+	gEmptyDirs = []string{"dev", "media", "mnt", "net", "proc"}
+	// Copy these from base template
+	gCopyDirs = []string{"etc", "root", "tmp", "var"}
+
+	gMdevfs          sync.Mutex
+
 	rootCmd = &cobra.Command{
 		Use:   "gocage",
 		Short: "GoCage is a FreeBSD Jail management tool",
@@ -69,7 +100,19 @@ It support iocage jails and can coexist with iocage.`,
 		Long:  `Let this show you how much fail I had to get this *cough* perfect`,
 		Run: func(cmd *cobra.Command, args []string) {
 			fv, _ := getFreeBSDVersion()
-			fmt.Printf("GoCage v.%s on FreeBSD %d.%d-%s\n", gVersion, fv.major, fv.minor, fv.flavor)
+			if fv.patchLevel > 0 {
+				fmt.Printf("GoCage v.%s on FreeBSD %d.%d-%s-p%d\n", gVersion, fv.major, fv.minor, fv.flavor, fv.patchLevel)
+			} else {
+				fmt.Printf("GoCage v.%s on FreeBSD %d.%d-%s\n", gVersion, fv.major, fv.minor, fv.flavor)
+			}
+		},
+	}
+
+	initCmd = &cobra.Command{
+		Use:   "init",
+		Short: "Initialize GoCage",
+		Run: func(cmd *cobra.Command, args []string) {
+			InitGoCage(args)
 		},
 	}
 
@@ -92,8 +135,8 @@ ex: gocage list srv-db srv-web`,
 			ListJailsProps(args)
 		},
 	}
-	
-/*	destroyCmd = &cobra.Command{
+
+	destroyCmd = &cobra.Command{
 		Use:   "destroy",
 		Short: "destroy jails",
 		Long: `Destroy jail filesystem, snapshots and configuration file.`,
@@ -102,7 +145,7 @@ ex: gocage list srv-db srv-web`,
 			DestroyJails(args)
 		},
 	}
-*/
+
 	stopCmd = &cobra.Command{
 		Use:   "stop",
 		Short: "stop jail",
@@ -129,12 +172,9 @@ ex: gocage list srv-db srv-web`,
 			} else {
 				StartJail(args)
 			}
-			WriteConfigToDisk(false)
 		},
 	}
-	
-	
-	
+
 	restartCmd = &cobra.Command{
 		Use:   "restart",
 		Short: "restart jail",
@@ -143,10 +183,9 @@ ex: gocage list srv-db srv-web`,
 			ListJails(args, false)
 			StopJail(args)
 			StartJail(args)
-			WriteConfigToDisk(false)
 		},
 	}
-	
+
 	shellCmd = &cobra.Command {
 		Use:   "console",
 		Short: "Execute shell on jail",
@@ -156,7 +195,7 @@ ex: gocage list srv-db srv-web`,
 			ShellJail(args)
 		},
 	}
-	
+
 	setCmd = &cobra.Command{
 		Use:   "set",
 		Short: "Set a jail property",
@@ -166,7 +205,6 @@ Multiples properties can be specified, separated with space (Ex: gocage set allo
 			// Load inventory
 			ListJails(args, false)
 			SetJailProperties(args)
-			WriteConfigToDisk(true)
 		},
 	}
 
@@ -248,7 +286,6 @@ You can specify multiple jails.`,
 			// Load inventory
 			ListJails(args, false)
 			MigrateJail(args)
-			WriteConfigToDisk(false)
 		},
 	}
 
@@ -284,7 +321,47 @@ You can specify multiple datastores.`,
 			ListDatastores(args, true)
 		},
 	}
-	
+
+	fetchCmd = &cobra.Command{
+		Use:   "fetch",
+		Short: "Fetch FreeBSD release to local datastore",
+		Run: func(cmd *cobra.Command, args []string) {
+			err := fetchRelease(gFetchRelease, "http", gJailHost.arch, gFetchIntoDS, gFetchFrom)
+			if err != nil {
+				fmt.Printf("%v\n", err)
+			} else {
+				extractRelease(gFetchRelease, gFetchIntoDS)
+			}
+		},
+	}
+
+	updateCmd = &cobra.Command{
+		Use:   "update",
+		Short: "Update FreeBSD release",
+		Run: func(cmd *cobra.Command, args []string) {
+			ListJails(args, false)
+			UpdateJail(args)
+		},
+	}
+
+	upgradeCmd = &cobra.Command{
+		Use:   "upgrade",
+		Short: "Upgrade FreeBSD release",
+		Run: func(cmd *cobra.Command, args []string) {
+			ListJails(args, false)
+			UpgradeJail(args)
+		},
+	}
+
+	createCmd = &cobra.Command{
+		Use:   "create",
+		Short: "Create jail",
+		Run: func(cmd *cobra.Command, args []string) {
+			ListJails(args, false)
+			CreateJail(args)
+		},
+	}
+
 	testCmd = &cobra.Command{
 		Use:   "test",
 		Short: "temporary command to test some code snippet",
@@ -297,16 +374,21 @@ You can specify multiple datastores.`,
 // TODO : Init log level and log output
 func init() {
 	var err error
-	
+
 	cobra.OnInitialize(initConfig)
 
 	// Global switches
 	rootCmd.PersistentFlags().StringVarP(&gConfigFile, "config", "c", "/usr/local/etc/gocage.conf.yml", "GoCage configuration file")
 	rootCmd.PersistentFlags().BoolVarP(&gUseSudo, "sudo", "u", false, "Use sudo to run commands")
 	rootCmd.PersistentFlags().StringVarP(&gTimeZone, "timezone", "t", "", "Specify timezone. Will get from /var/db/zoneinfo if not set.")
-	rootCmd.PersistentFlags().BoolVarP(&gDebug, "debug", "d", false, "Debug mode")
+	rootCmd.PersistentFlags().BoolVar(&gDebug, "debug", false, "Debug mode")
 
 	// Command dependant switches
+	initCmd.Flags().StringVarP(&gZPool, "pool", "p", "", "ZFS pool to create datastore on")
+	initCmd.Flags().StringVarP(&gBridge, "bridge", "b", "", "bridge to create for jails networking")
+	initCmd.Flags().StringVarP(&gInterface, "interface", "i", "", "interface to add as bridge member. This should be your main interface")
+	initCmd.MarkFlagRequired("bridge")
+	initCmd.MarkFlagsRequiredTogether("bridge", "interface")
 
 	// We reuse these flags in "gocage snapshot list myjail" and 'gocage datastore list" commands
 	listCmd.Flags().StringVarP(&gDisplayJColumns, "outcol", "o", "JID,Name,Config.Release,Config.Ip4_addr,Running", "Show these columns in output")
@@ -314,18 +396,18 @@ func init() {
 	listCmd.Flags().StringVarP(&gFilterJails, "filter", "f", "none", "Only display jails with these values. Ex: \"gocage list -f Config.Boot=1\" will only list started on boot jails")
 	listCmd.Flags().StringVarP(&gSortJailFields, "sort", "s", "none", "Display jails sorted by field values. Ex: \"gocage list -s +Name,-Config.Priority\" will sort jails by their decreasing name, then increasing start priority. 3 critera max supported.")
 	
-//	destroyCmd.Flags().BoolVarP(&gForce, "force", "f", false, "Force stop jail if running")
+	destroyCmd.Flags().BoolVarP(&gForce, "force", "f", false, "Force stop jail if running")
 
 	snapshotListCmd.Flags().StringVarP(&gDisplaySColumns, "outcol", "o", "Jailname,Name,Creation,Referenced,Used", "Show these columns in output")
 	snapshotListCmd.Flags().BoolVarP(&gNoSnapLineSep, "nolinesep", "l", false, "Do not display line separator between snapshots")
 	snapshotListCmd.Flags().StringVarP(&gFilterSnaps, "filter", "f", "none", "Only display snapshots with these values. Ex: \"gocage snapshot list -f Config.Boot=1\" will only list started on boot jails")
 	snapshotListCmd.Flags().StringVarP(&gSortSnapFields, "sort", "s", "none", "Display snapshots sorted by field values. Ex: \"gocage snapshot list -s +Jailname,-Used\" will sort snapshots by jail decreasing name, then increasing used space. 3 critera max supported.")
-	
+
 	datastoreListCmd.Flags().StringVarP(&gDisplayDColumns, "outcol", "o", "Name,Mountpoint,ZFSDataset,Available,Used,Referenced", "Show these columns in output")
 	datastoreListCmd.Flags().BoolVarP(&gNoDSLineSep, "nolinesep", "l", false, "Do not display line separator between datastores")
 	datastoreListCmd.Flags().StringVarP(&gFilterDS, "filter", "f", "none", "Only display datastores with these values. Ex: \"gocage datastore list -f Config.Boot=1\" will only list started on boot jails")
 	datastoreListCmd.Flags().StringVarP(&gSortDSFields, "sort", "s", "none", "Display datastores sorted by field values. Ex: \"gocage datastore list -s +Jailname,-Used\" will sort snapshots by jail decreasing name, then increasing used space. 3 critera max supported.")
-	
+
 	// This is local flag : Only available to gocage snapshot create command
 	snapshotCreateCmd.Flags().StringVarP(&gSnapshotName, "snapname", "n", "", "Name of the snapshot to create")
 	snapshotCreateCmd.MarkFlagRequired("snapname")
@@ -338,21 +420,43 @@ func init() {
 	migrateCmd.Flags().BoolVarP(&gYesToAll, "yes", "y", false, "Answer yes to all questions")
 	migrateCmd.MarkFlagRequired("datastore")
 
+	fetchCmd.Flags().StringVarP(&gFetchRelease, "release", "r", "", "Release to fetch (e.g.: \"13.1-RELEASE\"")
+	fetchCmd.Flags().StringVarP(&gFetchIntoDS, "datastore", "d", "", "Datastore release will be saved to")
+	fetchCmd.Flags().StringVarP(&gFetchFrom, "from", "f", "", "Repository to download from. Should contain XY.Z-RELEASE. File protocol supported")
+	fetchCmd.MarkFlagRequired("release")
+	fetchCmd.MarkFlagRequired("datastore")
+
+	upgradeCmd.Flags().StringVarP(&gUpgradeRelease, "release", "r", "", "Release to upgrade to (e.g.: \"13.1-RELEASE\"")
+	upgradeCmd.MarkFlagRequired("release")
+
+	updateCmd.Flags().StringVarP(&gUpdateRelease, "release", "r", "", "Release to update (e.g.: \"13.1-RELEASE\"")
+	updateCmd.Flags().StringVarP(&gUpdateReleaseDS, "datastore", "d", "", "Datastore release is stored on")
+	updateCmd.MarkFlagsRequiredTogether("release", "datastore")
+
+	createCmd.Flags().StringVarP(&gCreateArgs.Release, "release", "r", "", "Release for the jail (e.g.: \"13.1-RELEASE\"")
+	createCmd.Flags().BoolVarP(&gCreateArgs.BaseJail, "basejail", "b", false, "Basejail. This will create a jail mounted read only from a release, so every up(date|grade) made to this release will immediately propagate to new jail.\n")
+	createCmd.Flags().StringVarP(&gCreateArgs.Datastore, "datastore", "d", "", "Datastore to create the jail on. Defaults to first declared in config.")
+	createCmd.Flags().StringVarP(&gCreateArgs.Properties, "configuration", "p", "", "Configuration properties with format k1=v1,k2=v2 (Ex: \"Config.Ip4_addr=vnet0|192.168.1.2,Config.Ip6=none\")")
+
 	// Now declare commands
+	rootCmd.AddCommand(initCmd)
 	rootCmd.AddCommand(versionCmd)
 	rootCmd.AddCommand(listCmd)
-	listCmd.AddCommand(listPropsCmd)
+	rootCmd.AddCommand(listPropsCmd)
 	rootCmd.AddCommand(stopCmd)
 	rootCmd.AddCommand(startCmd)
 	rootCmd.AddCommand(restartCmd)
-//	rootCmd.AddCommand(destroyCmd)
+	rootCmd.AddCommand(destroyCmd)
 	rootCmd.AddCommand(shellCmd)
 	rootCmd.AddCommand(getCmd)
 	rootCmd.AddCommand(setCmd)
 	rootCmd.AddCommand(snapshotCmd)
 	rootCmd.AddCommand(migrateCmd)
 	rootCmd.AddCommand(datastoreCmd)
-	
+	rootCmd.AddCommand(fetchCmd)
+	rootCmd.AddCommand(updateCmd)
+	rootCmd.AddCommand(upgradeCmd)
+	rootCmd.AddCommand(createCmd)
 	rootCmd.AddCommand(testCmd)
 	
 	snapshotCmd.AddCommand(snapshotListCmd)
@@ -382,17 +486,6 @@ func initConfig() {
 		fmt.Printf("ERROR reading config file %s : %s\n", gConfigFile, err.Error())
 		os.Exit(1)
 	}
-	
-	// Load default configs from datastores
-	err := ListDatastores(viper.GetStringSlice("datastore"), false)
-	if err != nil {
-		fmt.Printf("ERROR: error checking datastores: %v\n", err)
-		os.Exit(1)
-	}
-
-	//	fmt.Println("Using config file:", viper.ConfigFileUsed())
-	//	fmt.Printf("datastore in config : %s\n", viper.GetStringSlice("datastore"))
-	//	fmt.Printf("datastore.0 in config : %s\n", viper.GetStringSlice("datastore.0"))
 
 	// Command line flags have priority on config file
 	if rootCmd.Flags().Lookup("sudo") != nil && false == rootCmd.Flags().Lookup("sudo").Changed {
@@ -426,68 +519,31 @@ func initConfig() {
 		fmt.Printf("More than 3 sort criteria is not supported!\n")
 		os.Exit(1)
 	}
-	
-	
+
+	gBridgeStaticMac = viper.GetBool("static-macs")
+
 	if gDebug {
 		log.SetLevel(log.DebugLevel)
 		log.Debugf("Debug mode enabled\n")
 	}
-}
 
-/********************************************************************************
- * Write jails config which been updated to disk.
- * If changeauto not set, values which are in "auto" mode on disk
- *  won't be overwritten (p.ex defaultrouter wont be overwritten with current
- *  default route, so if route change on jailhost this will reflect on jail next
- *  start)
- *******************************************************************************/
-func WriteConfigToDisk(changeauto bool) {
-	for _, j := range gJails {
-		if j.ConfigUpdated {
-			//log.Debug("%s config has changed, write changes to disk\n", j.Name)
-
-			// we will manipulate properties so get a copy
-			jc := j.Config
-
-			if changeauto == false {
-				// Overwrite "auto" properties
-				ondiskjc, err := getJailConfig(j.ConfigPath)
-				if err != nil {
-					panic(err)
-				}
-				// TODO : List all fields, then call getStructFieldValue to compare value with "auto"
-				// If "auto" then keep it that way before writing ondiskjc to disk
-				var properties []string
-				properties = getStructFieldNames(ondiskjc, properties, "")
-
-				for _, p := range properties {
-					v, _, err := getStructFieldValue(ondiskjc, p)
-					if err != nil {
-						panic(err)
-					}
-					if v.String() == "auto" {
-						err = setStructFieldValue(&jc, p, "auto")
-						if err != nil {
-							fmt.Printf("ERROR sanitizing config: %s\n", err.Error())
-							os.Exit(1)
-						}
-					}
-				}
-			}
-
-			marshaled, err := json.MarshalIndent(jc, "", "    ")
-			if err != nil {
-				fmt.Printf("ERROR marshaling config: %s\n", err.Error())
-			}
-			//fmt.Printf(string(marshaled))
-			if os.WriteFile(j.ConfigPath, []byte(marshaled), 0644); err != nil {
-				fmt.Printf("Error writing config file %s: %v\n", j.ConfigPath, err)
-				os.Exit(1)
-			}
+	// no need to check prerequesites if we are initializing gocage
+	for _, rc := range rootCmd.Commands() {
+		//fmt.Printf("DEBUG: rootCmd subcommand: %v. Was it called? %s\n", rc.Use, rootCmd.Commands()[i].CalledAs())
+		if len(rc.CalledAs()) > 0 && strings.EqualFold("init", rc.CalledAs()) {
+			return
 		}
 	}
+
+	// Load default configs from datastores
+	err := ListDatastores(viper.GetStringSlice("datastore"), false)
+	if err != nil {
+		fmt.Printf("ERROR: error checking datastores: %v\n", err)
+		os.Exit(1)
+	}
 }
 
+
 func Execute() {
 	if err := rootCmd.Execute(); err != nil {
 		fmt.Fprintln(os.Stderr, err)

cmd/start.go

@@ -4,6 +4,8 @@ import (
 	"os"
 	"fmt"
 	"net"
+	"sync"
+	"time"
 	"errors"
 	"regexp"
 	"reflect"
@@ -189,24 +191,26 @@ func prepareJailedZfsDatasets(jail *Jail) error {
 		}
 		for _, d := range strings.Split(jail.Config.Jail_zfs_dataset, " ") {
 			// Check if dataset exist, create if necessary
-			cmd := fmt.Sprintf("zfs get -H creation %s/%s", jail.Zpool, d)
+			// Support jailing datasets on differents pools : dataset should be specified with pool name
+			cmd := fmt.Sprintf("zfs get -H creation %s", d)
 			out, err := executeCommand(cmd)
 			if err != nil {
 				if strings.HasSuffix(out, "dataset does not exist") {
-					cmd = fmt.Sprintf("zfs create -o compression=lz4 -o mountpoint=none %s/%s", jail.Zpool, d)
+			        // Support jailing datasets on differents pools : dataset should be specified with pool name
+					cmd = fmt.Sprintf("zfs create -o compression=lz4 -o mountpoint=none %s", d)
 					_, err = executeCommand(cmd)
 					if err != nil {
-						return errors.New(fmt.Sprintf("Error creating dataset %s/%s: %s", jail.Zpool, d, err.Error()))
+						return errors.New(fmt.Sprintf("Error creating dataset %s: %s", d, err.Error()))
 					}
 				} else {
 					return errors.New(fmt.Sprintf("Error getting zfs dataset %s: %s", d, err.Error()))
 				}
 			}
 
-			cmd = fmt.Sprintf("zfs set jailed=on %s/%s", jail.Zpool, d)
+			cmd = fmt.Sprintf("zfs set jailed=on %s", d)
 			out, err = executeCommand(cmd)
 			if err != nil {
-				return errors.New(fmt.Sprintf("Error executing \"zfs set jailed=on %s/%s\": %s", jail.Zpool, d, err.Error()))
+				return errors.New(fmt.Sprintf("Error executing \"zfs set jailed=on %s\": %s", d, err.Error()))
 			}
 		}
 	}
@@ -217,27 +221,27 @@ func jailZfsDatasets(jail *Jail) error {
 	if jail.Config.Jail_zfs > 0 {
 		for _, d := range strings.Split(jail.Config.Jail_zfs_dataset, " ") {
 			// Jail dataset
-			cmd := fmt.Sprintf("zfs jail %d %s/%s", jail.JID, jail.Zpool, d)
+			// Support jailing datasets on differents pools : dataset should be specified with pool name
+			cmd := fmt.Sprintf("zfs jail %d %s", jail.JID, d)
 			out, err := executeCommand(cmd)
 			if err != nil {
 				return errors.New(fmt.Sprintf("Error jailling zfs dataset %s: %v: out", d, err, out))
 			}
 			
 			// Mount from inside jail if mountpoint is set
-			cmd = fmt.Sprintf("zfs get -H -o value mountpoint %s/%s", jail.Zpool, d)
+			cmd = fmt.Sprintf("zfs get -H -o value mountpoint %s", d)
 			out, err = executeCommand(cmd)
 			if err != nil {
-				return errors.New(fmt.Sprintf("Error getting zfs dataset %s/%s mountpoint: %v: %s", jail.Zpool, d, err, out))
+				return errors.New(fmt.Sprintf("Error getting zfs dataset %s mountpoint: %v: %s", d, err, out))
 			}
 			if len(out) > 0 && out != "-" && (false == strings.EqualFold(out, "none")) {
-				//cmd = fmt.Sprintf("zfs mount %s/%s", jail.Zpool, d)
-				cmd = fmt.Sprintf("zfs mount -a")
+				// Should we "mount -a" ? cmd = fmt.Sprintf("zfs mount -a")
+				cmd = fmt.Sprintf("zfs mount %s", d)
 				out, err = executeCommandInJail(jail, cmd)
 				if err != nil {
 					// If already mounted, continue processing
 					if ! strings.HasSuffix(out, "filesystem already mounted\n") {
-						//return errors.New(fmt.Sprintf("Error mounting zfs dataset %s/%s: %v: %s", jail.Zpool, d, err, out))
-						return errors.New(fmt.Sprintf("Error executing \"zfs mount -a\" from inside jail: %v: %s", err, out))
+						return errors.New(fmt.Sprintf("Error mounting zfs dataset %s from inside jail: %v: %s", d, err, out))
 					}
 				}
 			}
@@ -326,24 +330,26 @@ func configureDhcpOrAcceptRtadv(jail *Jail, ipproto int, enable bool) error {
 
 	for _, n := range nics {
 		// vnet0 is epair0b inside jail
-		if strings.Contains(n, "vnet") {
-			n = fmt.Sprintf("%sb", strings.Replace(n, "vnet", "epair", 1))
+		//if strings.Contains(n, "vnet") {
+		if strings.HasPrefix(n, "vnet") {
+			splitd := strings.Split(n, "|")
+			n = fmt.Sprintf("%sb", strings.Replace(splitd[0], "vnet", "epair", 1))
 		}
 		key := fmt.Sprintf("ifconfig_%s", n)
 		value := "SYNCDHCP"
 
 		if ipproto == IPv6 {
 			key = fmt.Sprintf("%s_ipv6", key)
-			value = "inet6 auto_linklocal accept_rtadv autoconf"
+			value = "\"inet6 auto_linklocal accept_rtadv autoconf\""
 		}
 
 		if enable == true {
-			err := enableRcKeyValue(jail.ConfigPath, key, value)
+			err := enableRcKeyValue(fmt.Sprintf("%s/etc/rc.conf", jail.RootPath), key, value)
 			if err != nil {
 				return fmt.Errorf("ERROR setting %s=%s with sysrc for jail %s: %s\n", key, value, jail.Name, err)
 			}
 		} else {
-			err := disableRcKey(jail.ConfigPath, key)
+			err := disableRcKey(fmt.Sprintf("%s/etc/rc.conf", jail.RootPath), key)
 			if err != nil {
 				return fmt.Errorf("ERROR deleting %s with sysrc for jail %s: %v\n", key, jail.Name, err)
 			}
@@ -357,7 +363,7 @@ func checkRtsold(jail *Jail) error {
 	if strings.Contains(jail.Config.Ip6_addr, "accept_rtadv") == false {
 		return fmt.Errorf("Must set at least one ip6_addr to accept_rtadv!\n")
 	}
-	err := enableRcKeyValue(jail.ConfigPath, "rtsold_enable", "yes")
+	err := enableRcKeyValue(fmt.Sprintf("%s/etc/rc.conf", jail.RootPath), "rtsold_enable", "yes")
 	if err != nil {
 		return fmt.Errorf("ERROR setting rtsold_enable=YES with sysrc for jail %s: %s\n", jail.Name, err)
 	}
@@ -472,12 +478,17 @@ func genNatIpv4(jail *Jail) ([]string, error) {
 	return ippair, nil
 }
 
-func buildDevfsRuleSet(jail *Jail) (error, int) {
+// FIXME : Must lock this function so parallel start do not 
+func buildDevfsRuleSet(jail *Jail, m *sync.Mutex) (error, int) {
 	rulesets := []int{}
 	
+	m.Lock()
+	//defer m.Unlock()
+
 	// Get known rulesets
 	out, err := executeCommand("devfs rule showsets")
 	if err != nil {
+		m.Unlock()
 		return errors.New(fmt.Sprintf("Error executing command \"devfs rule showsets\": %v; command returned: %s\n", err, out)), 0
 	}
 	srs := strings.Split(out, "\n")
@@ -499,25 +510,29 @@ func buildDevfsRuleSet(jail *Jail) (error, int) {
 		}
 	}
 	
-	log.Debug("buildDevfsRuleSet: Build ruleset %d\n", ruleset)
+	log.Debugf("buildDevfsRuleSet: Build ruleset %d\n", ruleset)
 	
 	// Get default devfs_ruleset for the datastore
 	// UPDATE: We don't need this as every jail have a default Devfs_ruleset value
 	/*ds, err := getDatastoreFromArray(jail.Datastore, gDatastores)
 	if err != nil {
+		m.Unlock()
 		return errors.New(fmt.Sprintf("Error getting datastore %s for jail %s", jail.Datastore, jail.Name)), 0
 	}
 	defaultrs, err := strconv.ParseInt(ds.DefaultJailConfig.Devfs_ruleset, 10, 64)
 	if err != nil {
+		m.Unlock()
 		return errors.New(fmt.Sprintf("Error parsing default devfs_ruleset for datastore %s", jail.Datastore)), 0
 	}*/
 	
 	// Clone configured devfs_ruleset to a dynamic ruleset
 	if false == isStringInArray(srs, jail.Config.Devfs_ruleset) {
+		m.Unlock()
 		return errors.New(fmt.Sprintf("Unknown ruleset: %s", jail.Config.Devfs_ruleset)), 0
 	}
 	rs, _ := strconv.Atoi(jail.Config.Devfs_ruleset)
 	err = copyDevfsRuleset(ruleset, rs)
+	m.Unlock()
 	if err != nil {
 		return err, 0
 	}
@@ -795,7 +810,9 @@ func generateMAC(jail *Jail, nic string) ([]byte, []byte, error) {
 	}
 	
 	hsmac := append(prefix, suffix...)
-	jsmac := append(hsmac[:5], hsmac[5]+1)
+	jsmac := make([]byte, 6)
+	copy(jsmac, hsmac)
+	jsmac[5] = jsmac[5] + 1
 	
 	// Save MACs to config
 	pname := fmt.Sprintf("Config.%s_mac", strings.Title(nic))
@@ -825,7 +842,7 @@ func setupVnetInterfaceHostSide(jail *Jail) ([]string, error) {
 		bridge := v[1]
 		
 		// Get host side MAC
-		pname := fmt.Sprintf("Config.%s_mac", nic)
+		pname := fmt.Sprintf("Config.%s_mac", strings.Title(nic))
 		var val *reflect.Value
 		val, pname, err = getStructFieldValue(jail, pname)
 		if err != nil {
@@ -838,13 +855,22 @@ func setupVnetInterfaceHostSide(jail *Jail) ([]string, error) {
 				return []string{}, err
 			}
 		} else {
-			hsmac = val.Bytes()
+			if strings.EqualFold(val.String(), "none") {
+				hsmac, _, err = generateMAC(jail, nic)
+				if err != nil {
+					return []string{}, err
+				}
+			}
+			hsmac, err = hex.DecodeString(strings.Split(val.String(), " ")[0])
+			if err != nil {
+				return []string{}, fmt.Errorf("Error converting %s to hex\n", strings.Split(val.String(), " ")[0])
+			}
 		}
 		
 		// Get bridge MTU
 		mtu, err := gJailHost.GetBridgeMTU(bridge)
 		if err != nil {
-			return []string{}, fmt.Errorf("Error getting bridge mtu: %v\n", err)
+			return []string{}, fmt.Errorf("Error getting bridge \"%s\" mtu: %v\n", bridge, err)
 		}
 		
 		// Create epair interface
@@ -880,11 +906,11 @@ func setupVnetInterfaceHostSide(jail *Jail) ([]string, error) {
 		}
 		epairs = append(epairs, hsepair)
 	}
-	
+	log.Debugf("setupVnetInterfaceHostSide: returning %v\n", epairs)
 	return epairs, nil
 }
 
-func setupVnetInterfaceJailSide(jail *Jail, hsepair string) error {
+func setupVnetInterfaceJailSide(jail *Jail, hostepairs []string) error {
 	var jsmac []byte
 	var err error
 
@@ -894,7 +920,9 @@ func setupVnetInterfaceJailSide(jail *Jail, hsepair string) error {
 	
 	for _, i := range strings.Split(jail.Config.Ip4_addr, ",") {
 		v := strings.Split(i, "|")
-		ip4s[v[0]] = v[1]
+		if len(v) > 1 {
+			ip4s[v[0]] = v[1]
+		}
 	}
 	for _, i := range strings.Split(jail.Config.Ip6_addr, ",") {
 		v := strings.Split(i, "|")
@@ -904,7 +932,7 @@ func setupVnetInterfaceJailSide(jail *Jail, hsepair string) error {
 	}
 
 	// Loop through configured interfaces
-	for _, nicCnf := range strings.Split(jail.Config.Interfaces, ",") {
+	for i, nicCnf := range strings.Split(jail.Config.Interfaces, ",") {
 		v := strings.Split(nicCnf, ":")
 		if len(v) != 2 {
 			return fmt.Errorf("Invalid value for Interfaces: %s\n", nicCnf)
@@ -915,9 +943,11 @@ func setupVnetInterfaceJailSide(jail *Jail, hsepair string) error {
 		// inside jail final nic name
 		jnic := strings.Replace(v[0], "vnet", "epair", 1)
 		jnic = jnic + "b"
-		
+		// host side associated jail nic name
+		jsepair := fmt.Sprintf("%sb", strings.TrimSuffix(hostepairs[i], "a"))
+
 		// Get jail side MAC
-		pname := fmt.Sprintf("Config.%s_mac", nic)
+		pname := fmt.Sprintf("Config.%s_mac", strings.Title(nic))
 		var val *reflect.Value
 		val, pname, err = getStructFieldValue(jail, pname)
 		if err != nil {
@@ -930,44 +960,45 @@ func setupVnetInterfaceJailSide(jail *Jail, hsepair string) error {
 				return err
 			}
 		} else {
-			jsmac = val.Bytes()
+			jsmac, err = hex.DecodeString(strings.Split(val.String(), " ")[1])
+                        if err != nil {
+                                return fmt.Errorf("Error converting %s to hex\n", strings.Split(val.String(), " ")[1])
+                        }
+
 		}
-		
-		lasta := strings.LastIndex(hsepair, "a")
-		jsepair := hsepair[:lasta] + strings.Replace(hsepair[lasta:], "a", "b", 1)
-		
+
 		cmd := fmt.Sprintf("/sbin/ifconfig %s vnet %s", jsepair, jail.InternalName)
 		_, err := executeCommand(cmd)
 		if err != nil {
 			return fmt.Errorf("Error linking interface to jail: %v\n", err)
 		}
-		
+
 		// Get bridge MTU
 		mtu, err := gJailHost.GetBridgeMTU(bridge)
 		if err != nil {
-			return fmt.Errorf("Error getting bridge mtu: %v\n", err)
+			return fmt.Errorf("Error getting bridge \"%s\" mtu: %v\n", bridge, err)
 		}
-				
+
 		cmd = fmt.Sprintf("/usr/sbin/jexec %d ifconfig %s mtu %d", jail.JID, jsepair, mtu)
 		_, err = executeCommand(cmd)
 		if err != nil {
 			return fmt.Errorf("Error setting mtu: %v\n", err)
 		}
-		
+
 		// rename epairXXb to epair0b (or opair1b, ...)
 		cmd = fmt.Sprintf("/usr/sbin/setfib %s jexec %d ifconfig %s name %s", jail.Config.Exec_fib, jail.JID, jsepair, jnic)
 		_, err = executeCommand(cmd)
 		if err != nil {
 			return fmt.Errorf("Error linking interface to jail: %v\n", err)
 		}
-		
+
 		cmd = fmt.Sprintf("/usr/sbin/setfib %s jexec %d ifconfig %s link %s", jail.Config.Exec_fib,
 						  jail.JID, jnic, hex.EncodeToString(jsmac))
 		_, err = executeCommand(cmd)
 		if err != nil {
 			return fmt.Errorf("Error setting mac: %v\n", err)
 		}
-		
+
 		// TODO: Move outside of this function
 		// add interface to bridge
 		if jail.Config.Nat == 0 {
@@ -976,17 +1007,30 @@ func setupVnetInterfaceJailSide(jail *Jail, hsepair string) error {
 			if err != nil {
 				return fmt.Errorf("Error adding member %s to %s: %v: %s\n", nic, bridge, err, out)
 			}
+			// Fix mac flapping and instabilities with bridged vnet network
+			if gBridgeStaticMac {
+				printablemac := hex.EncodeToString(jsmac)
+				for i := 2 ; i < len(printablemac) ; i += 3 {
+					printablemac = printablemac[:i] + ":" + printablemac[i:]
+				}
+				log.Debugf("Set %s.%d as static to %s in %s address cache\n", nic, jail.JID, printablemac, bridge)
+				cmd = fmt.Sprintf("/sbin/ifconfig %s static %s.%d %s", bridge, nic, jail.JID, printablemac)
+				out, err := executeCommand(cmd)
+				if err != nil {
+					return fmt.Errorf("Error setting %s.%d static with %s on %s: %v: %s\n", nic, jail.JID, printablemac, bridge, err, out)
+				}
+			}
 		}
-		
+
 		// Check we have an IP for the nic, and set it into jail
 		if len(ip4s[nic]) > 0 {
 			err = setJailVnetIp(jail, jnic, ip4s[nic])
 		}
-		
+
 		if len(ip6s[nic]) > 0 {
 			err = setJailVnetIp(jail, jnic, ip6s[nic])
 		}
-		
+
 		// finally up interface
 		if jail.Config.Nat == 0 {
 			cmd := fmt.Sprintf("/sbin/ifconfig %s.%d up", nic, jail.JID)
@@ -996,7 +1040,9 @@ func setupVnetInterfaceJailSide(jail *Jail, hsepair string) error {
 			}
 		}
 	}
-	
+
+	log.Debugf("setupVnetInterfaceJailSide: return with success\n")
+
 	return nil
 }
 
@@ -1010,7 +1056,7 @@ func generateResolvConf(jail *Jail) error {
 		for _, l := range strings.Split(jail.Config.Resolver, ";") {
 			f.WriteString(fmt.Sprintf("%s\n", l))
 		}
-	} else if jail.Config.Resolver == "none" {
+	} else if jail.Config.Resolver == "none" || jail.Config.Resolver == "/etc/resolv.conf" {
 		read, err := ioutil.ReadFile("/etc/resolv.conf")
 		if err != nil {
 			return fmt.Errorf("Error opening /etc/resolv.conf: %v", err)
@@ -1052,10 +1098,13 @@ func cleanAfterStartCrash() {
 // Start all jails with boot=true, in priority order
 func StartJailsAtBoot() {
 	var startList []Jail
+	var wg *sync.WaitGroup
+	var curThNb int
+	var curPri int
 
-	// Get boot enabled jails
+	// Get boot enabled non-template jails
 	for _, j := range gJails {
-		if j.Config.Boot > 0 {
+		if j.Config.Boot > 0 && !strings.EqualFold(j.Config.Jailtype, "template") {
 			startList = append(startList, j)
 		}
 	}
@@ -1069,11 +1118,51 @@ func StartJailsAtBoot() {
 	}
 	JailsOrderedBy(fct.Interface().(jailLessFunc)).Sort(startList)
 
-	for _, j := range startList {
+	wg = new(sync.WaitGroup)
+	curThNb = 0
+	for i, j := range startList {
 		jFullName := fmt.Sprintf("%s/%s", j.Datastore, j.Name)
 		log.Debugf("Starting %s with priority %s\n", jFullName, j.Config.Priority)
-		StartJail([]string{jFullName})
+		jailPri, err := strconv.Atoi(j.Config.Priority)
+		if err != nil {
+			panic(fmt.Sprintf("Invalid format for Priority (Jail %s)\n", jFullName))
+		}
+		
+		if (curThNb >= gMaxThreads || i == 0) {
+			// FIXME : Use a pool instead of waiting for all threads to run a new one
+			wg.Wait()
+			curThNb = 0
+			
+			wg.Add(1)
+			curThNb++
+			curPri = jailPri
+			go func(jailFullName string) {
+				defer wg.Done()
+				StartJail([]string{jailFullName})
+			}(jFullName)
+		} else {
+			if (curPri == jailPri) {
+				wg.Add(1)
+				curThNb++
+				go func(jailFullName string) {
+					defer wg.Done()
+					StartJail([]string{jailFullName})
+				}(jFullName)
+			} else {
+				wg.Wait()
+				curThNb = 0
+			
+				wg.Add(1)
+				curThNb++
+				curPri = jailPri
+				go func(jailFullName string) {
+					defer wg.Done()
+					StartJail([]string{jailFullName})
+				}(jFullName)
+			}
+		}
 	}
+	wg.Wait()
 }
 
 
@@ -1109,7 +1198,7 @@ func StartJail(args []string) {
 
 	for _, a := range args {
 		// Check if jail exist and is distinctly named
-		cj, err = getJailFromArray(a, gJails)
+		cj, err = getJailFromArray(a, []string{"basejail", "jail"}, gJails)
 		if err != nil {
 			fmt.Printf("Error getting jail: %s\n", err)
 			continue
@@ -1120,8 +1209,8 @@ func StartJail(args []string) {
 			continue
 		}
 		
-		fmt.Printf("> Starting jail %s\n", a)
-		
+		fmt.Printf("> Starting jail %s\n", cj.Name)
+
 		// Set InternalName as it is used by some of these
 		cj.InternalName = fmt.Sprintf("ioc-%s", cj.Name)
 
@@ -1283,34 +1372,34 @@ func StartJail(args []string) {
 				cj.Config.Defaultrouter = ip4[0]
 			}
 		}
-		
+
 		// See https://github.com/iocage/iocage/blob/e94863d4c54f02523fb09e62e48be7db9ac92eda/iocage_lib/ioc_start.py:401
 		if cj.Config.Vnet == 0 {
 			// Not supported
 			fmt.Printf("Only VNet jails supported\n")
 			return
 		}
-		
+
 		var net []string
 		if false == strings.EqualFold(cj.Config.Vnet_interfaces, "none") {
 			net = append(net,  strings.Split(cj.Config.Vnet_interfaces, " ")...)
 		}
-		
-		err, dynrs := buildDevfsRuleSet(cj)
+
+		err, dynrs := buildDevfsRuleSet(cj, &gMdevfs)
 		if err != nil {
 			fmt.Printf("%s\n", err.Error())
 			return
 		}
-		
+
 		err = buildJailParameters(cj, dynrs)
 		if err != nil {
 			fmt.Printf("%s\n", err.Error())
 			return
 		}
-		
+
 		// Synchronize jail config to disk
-		WriteConfigToDisk(false)
-		
+		cj.WriteConfigToDisk(false)
+
 		start_cmd := fmt.Sprintf("/usr/sbin/jail -f /var/run/jail.%s.conf -c", cj.InternalName)
 
 		//TODO: handle start_env & prestart_env, could be used by iocage plugins
@@ -1320,19 +1409,20 @@ func StartJail(args []string) {
 			fmt.Printf("Aborting jail start\n")
 			return
 		}
-		
+
 		fmt.Printf("  > Start jail:\n")
 		_, err = executeCommand(start_cmd)
 		if err != nil {
 			fmt.Printf("Error starting jail %s: %v\n", cj.Name, err)
 			return
 		}
-		
+
 		fmt.Printf("  > Start jail: OK\n")
 		fmt.Printf("  > With devfs ruleset %d\n", dynrs)
-		
-		// Update running state and JID
+
+		// Update running state, JID and Devfs_ruleset
 		cj.Running = true
+		cj.Devfs_ruleset = dynrs
 		rjails, err := jail.GetJails()
 		if err != nil {
 			fmt.Printf("Error: Unable to list running jails\n")
@@ -1343,13 +1433,13 @@ func StartJail(args []string) {
 				break
 			}
 		}
-		
+
 		hostInt, err := gJailHost.GetInterfaces()
 		if err != nil {
 			fmt.Printf("Error listing jail host interfaces: %v\n", err)
 			return
 		}
-		
+
 		if false == strings.EqualFold(cj.Config.Vnet_default_interface, "auto") &&
 			false == strings.EqualFold(cj.Config.Vnet_default_interface, "none") &&
 			false == isStringInArray(hostInt, cj.Config.Vnet_default_interface) {
@@ -1358,30 +1448,30 @@ func StartJail(args []string) {
 		}
 
 		fmt.Printf("  > Setup VNet network:\n")
-		hsepairs, err := setupVnetInterfaceHostSide(cj); 
+		hsepairs, err := setupVnetInterfaceHostSide(cj);
 		if err != nil {
 			fmt.Printf("Error setting VNet interface host side: %v\n", err)
 			return
 		}
-		
-		for _, ep := range hsepairs {
-			if err = setupVnetInterfaceJailSide(cj, ep); err != nil {
-				fmt.Printf("Error setting VNet interface jail side: %v\n", err)
-				return
-			}
+
+		if err = setupVnetInterfaceJailSide(cj, hsepairs); err != nil {
+			fmt.Printf("Error setting VNet interface jail side: %v\n", err)
+			return
 		}
 		fmt.Printf("  > Setup VNet network: OK\n")
 		
-		// TODO: Handle DHCP
-		fmt.Printf("  > Setup default ipv4 gateway:\n")
-		cmd := fmt.Sprintf("/usr/sbin/setfib %s /usr/sbin/jexec %d route add default %s", cj.Config.Exec_fib, cj.JID, cj.Config.Defaultrouter)
-		out, err := executeCommand(cmd)
-		if err != nil && len(out) > 0 {
-			fmt.Printf("Error: %v: %s\n", err, out)
-		} else {
-			fmt.Printf("  > Setup default ipv4 gateway: OK\n")
+		// Set default route, unless main network is dhcp
+		if ! cj.isFirstNetDhcp() && !strings.EqualFold(cj.Config.Ip4_addr, "none") {
+			fmt.Printf("  > Setup default ipv4 gateway:\n")
+			cmd := fmt.Sprintf("/usr/sbin/setfib %s /usr/sbin/jexec %d route add default %s", cj.Config.Exec_fib, cj.JID, cj.Config.Defaultrouter)
+			out, err := executeCommand(cmd)
+			if err != nil && len(out) > 0 {
+				fmt.Printf("Error: %v: %s\n", err, out)
+			} else {
+				fmt.Printf("  > Setup default ipv4 gateway: OK\n")
+			}
 		}
-		
+
 		if cj.Config.Ip6_addr != "none" {
 			fmt.Printf("  > Setup default ipv6 gateway:\n")
 			cmd := fmt.Sprintf("/usr/sbin/setfib %s /usr/sbin/jexec %d route add -6 default %s", cj.Config.Exec_fib, cj.JID, cj.Config.Defaultrouter6)
@@ -1392,7 +1482,7 @@ func StartJail(args []string) {
 				fmt.Printf("  > Setup default ipv6 gateway: OK\n")
 			}
 		}
-		
+
 		if cj.Config.Jail_zfs > 0 {
 			fmt.Printf("  > Jail ZFS datasets:\n")
 			err = jailZfsDatasets(cj)
@@ -1407,26 +1497,26 @@ func StartJail(args []string) {
 		if err != nil {
 			fmt.Printf("%s\n", err)
 		}
-		
+
 		if cj.Config.Host_time > 0 {
 			err = copyLocalTime(cj)
 			if err != nil {
 				fmt.Printf("%s\n", err)
 			}
 		}
-		
+
 		// Start services
 		if len(cj.Config.Exec_start) > 0 {
 			fmt.Printf("  > Start services:\n")
 			cmd := fmt.Sprintf("/usr/sbin/setfib %s /usr/sbin/jexec %d %s", cj.Config.Exec_fib, cj.JID, cj.Config.Exec_start)
-			out, err := executeCommand(cmd)
-			if err != nil && len(out) > 0 {
-				fmt.Printf("Error: %v: %s\n", err, out)
+			err := executeCommandNonBlocking(cmd)
+			if err != nil {
+				fmt.Printf("Error: %v\n", err)
 			} else {
 				fmt.Printf("  > Start services: OK\n")
 			}
 		}
-		
+
 		if cj.Config.Rtsold > 0 || strings.EqualFold(cj.Config.Ip6_addr, "accept_rtadv") {
 			fmt.Printf("  > Start rtsold:\n")
 			cmd := fmt.Sprintf("/usr/sbin/setfib %s /usr/sbin/jexec %d service rtsold start", cj.Config.Exec_fib, cj.JID)
@@ -1437,7 +1527,7 @@ func StartJail(args []string) {
 				fmt.Printf("  > Start rtsold: OK\n")
 			}
 		}
-		
+
 		// TODO: Execute Exec_poststart
 		if len(cj.Config.Exec_poststart) > 0 {
 			fmt.Printf("  > Execute post-start:\n")
@@ -1449,13 +1539,18 @@ func StartJail(args []string) {
 				fmt.Printf("  > Execute post-start: OK\n")
 			}
 		}
-		
+
 		// WIP 10/07/2022 : https://github.com/iocage/iocage/blob/master/iocage_lib/ioc_start.py#L891
 		// TODO: Handle dhcp
 		// TODO: Apply rctl
-		
-		
-		
+
+		// Update last_started
+		// 23/07/2023 : This is not working, when writing to disk the old value is used
+		dt := time.Now()
+		curDate := fmt.Sprintf("%s", dt.Format("2006-01-02 15:04:05"))
+		cj.Config.Last_started = curDate
+		writeConfigToDisk(cj, false)
+
 		/*
 			out, err := executeCommand(fmt.Sprintf("rctl jail:%s", cj.InternalName))
 			if err == nil && len(out) > 0 {

cmd/stop.go

@@ -4,6 +4,7 @@ import (
 	"os"
 	"fmt"
 	//"log"
+	"sync"
 	"errors"
 	"regexp"
 	"os/exec"
@@ -50,10 +51,10 @@ func umountAndUnjailZFS(jail *Jail) error {
 
 	for _, zd := range ds {
 		// 1. Get dataset and childs
-		cmd := fmt.Sprintf("zfs list -H -r -o name -S name %s/%s", jail.Zpool, zd)
+		cmd := fmt.Sprintf("zfs list -H -r -o name -S name %s", zd)
 		out, err := executeCommand(cmd)
 		if err != nil {
-			fmt.Printf(fmt.Sprintf("ERROR listing dataset %s/%s\n", jail.Zpool, zd))
+			fmt.Printf(fmt.Sprintf("ERROR listing dataset %s\n", zd))
 			os.Exit(1)
 		}
 		for _, c := range strings.Split(out, "\n") {
@@ -71,10 +72,10 @@ func umountAndUnjailZFS(jail *Jail) error {
 	}
 
 	// 2. Unjail dataset from the host
-	cmd := fmt.Sprintf("zfs unjail %s %s/%s", jail.InternalName, jail.Zpool, ds[len(ds)-1])
+	cmd := fmt.Sprintf("zfs unjail %s %s", jail.InternalName, ds[len(ds)-1])
 	_, err := executeCommand(cmd)
 	if err != nil {
-		fmt.Printf("ERROR unjailing %s/%s: %s\n", jail.Zpool, ds[len(ds)-1], err.Error())
+		fmt.Printf("ERROR unjailing %s: %s\n", ds[len(ds)-1], err.Error())
 		return err
 	}
 
@@ -82,15 +83,30 @@ func umountAndUnjailZFS(jail *Jail) error {
 }
 
 func destroyVNetInterfaces(jail *Jail) error {
-	for _, i := range strings.Split(jail.Config.Ip4_addr, ",") {
-		iname := fmt.Sprintf("%s.%d", strings.Split(i, "|")[0], jail.JID)
-		fmt.Printf("%s: ", iname)
-		_, err := executeCommand(fmt.Sprintf("ifconfig %s destroy", iname))
-		//_, err := executeScript(fmt.Sprintf("ifconfig %s destroy >/dev/null 2>&1", iname))
-		if err != nil {
-			return err
-		} else {
-			fmt.Printf("OK\n")
+	if !strings.EqualFold(jail.Config.Ip4_addr, "none") {
+		for _, i := range strings.Split(jail.Config.Ip4_addr, ",") {
+			iname := fmt.Sprintf("%s.%d", strings.Split(i, "|")[0], jail.JID)
+			fmt.Printf("%s: ", iname)
+			_, err := executeCommand(fmt.Sprintf("ifconfig %s destroy", iname))
+			//_, err := executeScript(fmt.Sprintf("ifconfig %s destroy >/dev/null 2>&1", iname))
+			if err != nil {
+				return err
+			} else {
+				fmt.Printf("OK\n")
+			}
+		}
+	}
+	if !strings.EqualFold(jail.Config.Ip6_addr, "none") {
+		for _, i := range strings.Split(jail.Config.Ip6_addr, ",") {
+			iname := fmt.Sprintf("%s.%d", strings.Split(i, "|")[0], jail.JID)
+			fmt.Printf("%s: ", iname)
+			_, err := executeCommand(fmt.Sprintf("ifconfig %s destroy", iname))
+			//_, err := executeScript(fmt.Sprintf("ifconfig %s destroy >/dev/null 2>&1", iname))
+			if err != nil {
+				return err
+			} else {
+				fmt.Printf("OK\n")
+			}
 		}
 	}
 
@@ -122,7 +138,7 @@ func deleteDevfsRuleset(ruleset int) error {
 	return nil
 }
 
-func umountJailFsFromHost(jail *Jail, mountpoint string) error {
+func umountFsFromHost(mountpoint string) error {
 	cmd := "mount -p"
 	out, err := executeCommand(cmd)
 	if err != nil {
@@ -133,11 +149,11 @@ func umountJailFsFromHost(jail *Jail, mountpoint string) error {
 	for _, l := range strings.Split(out, "\n") {
 		f := strings.Split(remSpPtrn.ReplaceAllString(l, " "), " ")
 		if len(f) > 2 {
-			if strings.EqualFold(f[1], fmt.Sprintf("%s%s", jail.RootPath, mountpoint)) {
-				cmd = fmt.Sprintf("umount %s%s", jail.RootPath, mountpoint)
+			if strings.EqualFold(f[1], mountpoint) {
+				cmd = fmt.Sprintf("umount %s", mountpoint)
 				_, err := executeCommand(cmd)
 				if err != nil {
-					return errors.New(fmt.Sprintf("Error umounting %s%s: %s", jail.RootPath, mountpoint, err.Error()))
+					return errors.New(fmt.Sprintf("Error umounting %s: %s", mountpoint, err.Error()))
 				}
 				return nil
 			}
@@ -147,6 +163,10 @@ func umountJailFsFromHost(jail *Jail, mountpoint string) error {
 	return nil
 }
 
+func umountJailFsFromHost(jail *Jail, mountpoint string) error {
+	return umountFsFromHost(fmt.Sprintf("%s%s", jail.RootPath, mountpoint))
+}
+
 // Internal usage only
 func stopJail(jail *Jail) error {
 	cmd := "jail -q"
@@ -169,8 +189,13 @@ func stopJail(jail *Jail) error {
 }
 
 // Stop all running jails by reverse priority
+// Parallelize up to gMaxThreads
+// Only parallelize same priority level jails
 func StopAllRunningJails() {
 	var stopList []Jail
+	var wg *sync.WaitGroup
+	var curThNb int
+	var curPri int
 	
 	// Get boot enabled jails
 	for _, j := range gJails {
@@ -187,12 +212,53 @@ func StopAllRunningJails() {
 		return
 	}
 	JailsOrderedBy(fct.Interface().(jailLessFunc)).Sort(stopList)
-	
-	for _, j := range stopList {
+
+
+	wg = new(sync.WaitGroup)
+	curThNb = 0
+	for i, j := range stopList {
 		jFullName := fmt.Sprintf("%s/%s", j.Datastore, j.Name)
 		log.Debugf("Stopping %s with priority %s\n", jFullName, j.Config.Priority)
-		StopJail([]string{jFullName})
+		jailPri, err := strconv.Atoi(j.Config.Priority)
+		if err != nil {
+			panic(fmt.Sprintf("Invalid format for Priority (Jail %s)\n", jFullName))
+		}
+
+		if (curThNb >= gMaxThreads || i == 0) {
+			// FIXME : Use a pool instead of waiting for all threads to run a new one
+			wg.Wait()
+			curThNb = 0
+			
+			wg.Add(1)
+			curThNb++
+			curPri = jailPri
+			go func(jailFullName string) {
+				defer wg.Done()
+				StopJail([]string{jailFullName})
+			}(jFullName)
+		} else {
+			if (curPri == jailPri) {
+				wg.Add(1)
+				curThNb++
+				go func(jailFullName string) {
+					defer wg.Done()
+					StopJail([]string{jailFullName})
+				}(jFullName)
+			} else {
+				wg.Wait()
+				curThNb = 0
+			
+				wg.Add(1)
+				curThNb++
+				curPri = jailPri
+				go func(jailFullName string) {
+					defer wg.Done()
+					StopJail([]string{jailFullName})
+				}(jFullName)
+			}
+		}
 	}
+	wg.Wait()
 }
 
 /*
@@ -223,19 +289,22 @@ func StopJail(args []string) {
 
 	for _, a := range args {
 		// Check if jail exist and is distinctly named
-		cj, err = getJailFromArray(a, gJails)
+		cj, err = getJailFromArray(a, []string{"basejail", "jail"}, gJails)
 		if err != nil {
 			fmt.Printf("Error getting jail: %s\n", err)
 			continue
 		}
-		
+
 		if cj.Running == false {
 			fmt.Printf("Jail %s is not running!\n", cj.Name)
 			continue
 		}
-		
-		fmt.Printf("> Stopping jail %s\n", a)
-		
+
+		fmt.Printf("> Stopping jail %s\n", cj.Name)
+
+		// Get and write new release into config.json
+		updateVersion(cj)
+
 		out, err := executeCommand(fmt.Sprintf("rctl jail:%s", cj.InternalName))
 		if err == nil && len(out) > 0 {
 			fmt.Printf("  > Remove RCTL rules:\n")
@@ -286,20 +355,13 @@ func StopJail(args []string) {
 				fmt.Printf("  > Destroy VNet interfaces: OK\n")
 			}
 		}
-		
-		// Get currently used ruleset from /var/run/jail.$internal_name.conf
-		ruleset, err := getValueFromRunningConfig(cj.InternalName, "devfs_ruleset")
-		if err != nil {
-			fmt.Printf("ERROR getting current devfs ruleset: %s\n", err.Error())
-			return
-		}
-		rsi, _ := strconv.Atoi(ruleset)
-		fmt.Printf("  > Remove devfs ruleset %d: \n", rsi)
-		err = deleteDevfsRuleset(rsi)
+
+		fmt.Printf("  > Remove devfs ruleset %d: \n", cj.Devfs_ruleset)
+		err = deleteDevfsRuleset(cj.Devfs_ruleset)
 		if err != nil {
 			fmt.Printf("ERROR: %s\n", err.Error())
 		} else {
-			fmt.Printf("  > Remove devfsruleset %d: OK\n", rsi)
+			fmt.Printf("  > Remove devfsruleset %d: OK\n", cj.Devfs_ruleset)
 		}
 
 		fmt.Printf("  > Stop jail %s:\n", cj.Name)
@@ -361,7 +423,8 @@ func StopJail(args []string) {
 			fmt.Printf("  > Umount mountpoints from %s\n", fstab)
 			errs := 0
 			for _, m := range mounts {
-				err = umountJailFsFromHost(cj, m.Mountpoint)
+				log.Debugf("Umounting %s\n", m.Mountpoint)
+				err = umountFsFromHost(m.Mountpoint)
 				if err != nil {
 					fmt.Printf("ERROR: %s\n", err.Error())
 					errs += 1
@@ -371,7 +434,7 @@ func StopJail(args []string) {
 				fmt.Printf("  > Umount mountpoints from %s: OK\n", fstab)
 			}
 		}
-		
+
 		// TODO: Execute poststop
 		if len(cj.Config.Exec_poststop) > 0 {
 			fmt.Printf("  > Execute post-stop:\n")
@@ -382,13 +445,13 @@ func StopJail(args []string) {
 				fmt.Printf("  > Execute post-stop: OK\n")
 			}
 		}
-		
+
 		// Remove parameter file
 		pfile := fmt.Sprintf("/var/run/jail.%s.conf", cj.InternalName)
 		if err = os.Remove(pfile); err != nil {
 			fmt.Printf("Error deleting parameter file %s\n", pfile)
 		}
-		
+
 		// We need this to get a reference to cj.Running (bc cj.Running is just a copy of value in the scope of StopJail())
 		for i, j := range gJails {
 			if strings.EqualFold(j.Name, cj.Name) && strings.EqualFold(j.Datastore, cj.Datastore) {
@@ -401,7 +464,12 @@ func StopJail(args []string) {
 				if err = setStructFieldValue(&gJails[i], "InternalName", ""); err != nil {
 					fmt.Printf("ERROR: clearing InternalName property: %s\n", err.Error())
 				}
+				if err = setStructFieldValue(&gJails[i], "Devfs_ruleset", "0"); err != nil {
+					fmt.Printf("ERROR: setting Devfs_ruleset property to 0: %s\n", err.Error())
+				}
 			}
 		}
+		writeConfigToDisk(cj, false)
+
 	}
 }

cmd/struct.go

@@ -27,6 +27,7 @@ type Jail struct {
 	Running       bool
 	// No need, Config.Release always represent what is running (plus it know release for non-running jails)
 	//Release					string
+	Devfs_ruleset int	// The effective devfs ruleset generated at runtime
 	Zpool         string
 	Datastore     string
 }
@@ -212,6 +213,7 @@ type FreeBSDVersion struct {
 type JailHost struct {
 	hostname          string
 	hostid            string
+	arch              string
 	default_gateway4  string
 	default_gateway6  string
 	default_interface string
@@ -247,6 +249,8 @@ type JailSort struct {
 	DatastoreDec    jailLessFunc
 	ZpoolInc        jailLessFunc
 	ZpoolDec        jailLessFunc
+	Devfs_rulesetInc jailLessFunc
+	Devfs_rulesetDec jailLessFunc
 	Config          JailConfigSort
 }
 

cmd/update.go

@@ -0,0 +1,144 @@
+package cmd
+
+import (
+	"os"
+	"fmt"
+	//"log"
+	"time"
+	"strings"
+	"github.com/spf13/viper"
+)
+
+
+// Internal usage only
+func updateJail(jail *Jail, doUpdateVersion bool) error {
+	// Create default config as temporary file
+	cfgFile, err := os.CreateTemp("", "gocage-jail-update-")
+	if err != nil {
+		return err
+	}
+
+	// Folder containing update/upgrade temporary files. Mutualized so we save bandwith when upgrading multiple jails
+	uwd := viper.GetString("updateWorkDir")
+	if len(uwd) == 0 {
+		return fmt.Errorf("updateWorkDir not set in configuration")
+	}
+	_, err = os.Stat(uwd)
+	if os.IsNotExist(err) {
+		if err := os.Mkdir(uwd, 0755); err != nil {
+			return err
+		}
+	}
+	cfgFile.Write([]byte(strings.Replace(fbsdUpdateConfig, "TO-BE-REPLACED-WITH-UPDATEWORKDIR", uwd, 1)))
+	defer cfgFile.Close()
+	defer os.Remove(cfgFile.Name())
+
+	cmd := fmt.Sprintf("/usr/sbin/freebsd-update --not-running-from-cron -f %s -b %s --currently-running %s fetch",
+					   cfgFile.Name(), jail.RootPath, jail.Config.Release)
+
+	err = executeCommandWithOutputToStdout(cmd)
+	if err != nil {
+		return err
+	}
+
+	cmd = fmt.Sprintf("/usr/sbin/freebsd-update --not-running-from-cron -f %s -b %s --currently-running %s install",
+			   cfgFile.Name(), jail.RootPath, jail.Config.Release)
+	err = executeCommandWithOutputToStdout(cmd)
+	if err != nil {
+		return err
+	}
+
+	// Get and write new release into config.json. Don't do that for fake jail (aka release updating)
+	if doUpdateVersion {
+		updateVersion(jail)
+	}
+
+	return nil
+}
+
+func UpdateJail(args []string) {
+	// Current jail were stopping
+	var cj *Jail
+	var err error
+
+	// User is updateing a release, fake a jail
+	if  len(gUpdateRelease) > 0 {
+		// get datastore mountpoing from datastore name
+		ds, err := getDatastoreFromArray(gUpdateReleaseDS, gDatastores)
+		if err != nil {
+			fmt.Printf("Error getting datastore %s: %v\n", gUpdateReleaseDS, err)
+			return
+		}
+		rp := fmt.Sprintf("%s/releases/%s/root", ds.Mountpoint, gUpdateRelease)
+		fakeJail := Jail{RootPath: rp}
+		v, err := getVersion(&fakeJail)
+		if err != nil {
+			fmt.Printf("Error getting version of release %s: %v\n", gUpdateRelease, err)
+			return
+		}
+		fakeJail.Config.Release = v
+
+		// Remove patch level from Release
+		fv, err := freebsdVersionToStruct(fakeJail.Config.Release)
+		if err != nil {
+			fmt.Printf("Error converting release %s: %v\n", fakeJail.Config.Release, err)
+			return
+		}
+		release := fmt.Sprintf("%d.%d-%s", fv.major, fv.minor, fv.flavor)
+		// Snapshot before updating
+		dt := time.Now()
+		curDate := fmt.Sprintf("%s", dt.Format("2006-01-02_15-04-05"))
+		snapshotName := fmt.Sprintf("gocage_update_%s_%s", v, curDate)
+		err = zfsSnapshot(fmt.Sprintf("%s/releases/%s", ds.ZFSDataset, release), snapshotName)
+		if err != nil {
+			fmt.Printf("Error snapshoting release %s: %v\n", gUpdateRelease, err)
+			return
+		}
+		err = zfsSnapshot(fmt.Sprintf("%s/releases/%s/root", ds.ZFSDataset, release), snapshotName)
+		if err != nil {
+			fmt.Printf("Error snapshoting release %s: %v\n", gUpdateRelease, err)
+		} else {
+			fmt.Printf("Release %s was snapshoted with success: %s\n", gUpdateRelease, snapshotName)
+		}
+
+		if err = updateJail(&fakeJail, false); err != nil {
+			fmt.Printf("Error updating release %s: %v\n", gUpdateRelease, err)
+		}
+		return
+	}
+
+	for _, a := range args {
+		// Check if jail exist and is distinctly named
+		cj, err = getJailFromArray(a, []string{""}, gJails)
+		if err != nil {
+			fmt.Printf("Error getting jail: %s\n", err)
+			continue
+		}
+
+		// We cant update basejail as system is readonly
+		if strings.EqualFold(cj.Config.Jailtype, "basejail") {
+			fmt.Printf("%s is a basejail using %s system files. Please update %s!\n", cj.Name, cj.Config.Origin, cj.Config.Origin)
+			continue
+		}
+
+		fmt.Printf("  > Snapshot jail %s\n", cj.Name)
+		// Set snapshot name
+		dt := time.Now()
+		curDate := fmt.Sprintf("%s", dt.Format("2006-01-02_15-04-05"))
+		gSnapshotName = fmt.Sprintf("gocage_update_%s_%s", cj.Config.Release, curDate)
+		err := createJailSnapshot(*cj)
+		if err != nil {
+			fmt.Printf("  > Snapshot jail %s: ERROR: %s\n", cj.Name, err.Error())
+			return
+		}
+		fmt.Printf("  > Snapshot jail %s: OK\n", cj.Name)
+		
+		fmt.Printf("  > Update jail %s\n", cj.Name)
+		err = updateJail(cj, true)
+		if err != nil {
+			fmt.Printf("ERROR: %s\n", err.Error())
+		} else {
+			fmt.Printf("  > Update jail %s: OK\n", cj.Name)
+		}
+	}
+}

cmd/upgrade.go

@@ -0,0 +1,126 @@
+package cmd
+
+
+
+import (
+	"os"
+	"fmt"
+	//"log"
+	"time"
+	"strings"
+	"github.com/spf13/viper"
+)
+
+// Internal usage only
+func upgradeJail(jail *Jail, version string) error {
+	// Create default config as temporary file
+	cfgFile, err := os.CreateTemp("", "gocage-jail-upgrade-")
+	if err != nil {
+		return err
+	}
+
+	// Folder containing update/upgrade temporary files. Mutualized so we save bandwith when upgrading multiple jails
+	uwd := viper.GetString("updateWorkDir")
+	if len(uwd) == 0 {
+		return fmt.Errorf("updateWorkDir not set in configuration")
+	}
+	_, err = os.Stat(uwd)
+	if os.IsNotExist(err) {
+		if err := os.Mkdir(uwd, 0755); err != nil {
+			return err
+		}
+	}
+	cfgFile.Write([]byte(strings.Replace(fbsdUpdateConfig, "TO-BE-REPLACED-WITH-UPDATEWORKDIR", uwd, 1)))
+	defer cfgFile.Close()
+	defer os.Remove(cfgFile.Name())
+
+	// Get current version. Won't work on stopped jail.
+	fbsdvers, err := executeCommandInJail(jail, "/bin/freebsd-version")
+	if err != nil {
+		fmt.Printf("ERROR executeCommandInJail: %s\n", err.Error())
+		return err
+	}
+	fbsdvers = strings.TrimRight(fbsdvers, "\n")
+
+	cmd := fmt.Sprintf("/usr/sbin/freebsd-update -f %s -b %s --currently-running %s -r %s upgrade", 
+			   cfgFile.Name(), jail.RootPath, fbsdvers, version)
+
+	//fmt.Printf("DEBUG: Prepare to execute \"%s\"\n", cmd)
+
+	// Need to give user control, bc there could be merge edit needs
+	err = executeCommandWithStdinStdoutStderr(cmd)
+	if err != nil {
+		return err
+	}
+
+	cmd = fmt.Sprintf("/usr/sbin/freebsd-update -f %s -b %s --currently-running %s -r %s install", 
+			   cfgFile.Name(), jail.RootPath, fbsdvers, version)
+
+	//fmt.Printf("DEBUG: Prepare to execute \"%s\"\n", cmd)
+
+	err = executeCommandWithStdinStdoutStderr(cmd)
+	if err != nil {
+		return err
+	}
+
+	cmd = fmt.Sprintf("/usr/sbin/freebsd-update -f %s -b %s --currently-running %s -r %s install", 
+			  cfgFile.Name(), jail.RootPath, fbsdvers, version)
+
+	//fmt.Printf("DEBUG: Prepare to execute \"%s\"\n", cmd)
+
+	err = executeCommandWithStdinStdoutStderr(cmd)
+	if err != nil {
+		return err
+	}
+
+	cmd = fmt.Sprintf("/usr/local/sbin/pkg-static -j %d install -q -f -y pkg", jail.JID)
+	err = executeCommandWithStdinStdoutStderr(cmd)
+	if err != nil {
+		return err
+	}
+
+	// Get and write new release into config.json
+	updateVersion(jail)
+
+	return nil
+}
+
+func UpgradeJail(args []string) {
+	// Current jail were stopping
+	var cj *Jail
+	var err error
+
+	for _, a := range args {
+		// Check if jail exist and is distinctly named
+		cj, err = getJailFromArray(a, []string{""}, gJails)
+		if err != nil {
+			fmt.Printf("Error getting jail: %s\n", err)
+			continue
+		}
+
+		if cj.Running == false {
+			fmt.Printf("Error: jail must be running for upgrade.\n")
+			return
+		}
+
+		fmt.Printf("  > Snapshot jail %s\n", cj.Name)
+		// Set snapshot name
+		dt := time.Now()
+		curDate := fmt.Sprintf("%s", dt.Format("2006-01-02_15-04-05"))
+		gSnapshotName = fmt.Sprintf("goc_upgrade_%s_%s", cj.Config.Release, curDate)
+		err := createJailSnapshot(*cj)
+		if err != nil {
+			fmt.Printf("  > Snapshot jail %s: ERROR: %s\n", cj.Name, err.Error())
+			return
+		}
+		fmt.Printf("  > Snapshot jail %s: OK\n", cj.Name)
+
+		fmt.Printf("  > Upgrade jail %s to %s\n", cj.Name, gUpgradeRelease)
+		err = upgradeJail(cj, gUpgradeRelease)
+		if err != nil {
+			fmt.Printf("ERROR: %s\n", err.Error())
+		} else {
+			fmt.Printf("  > Upgrade jail %s: OK\n", cj.Name)
+		}
+	}
+}

go.mod

@@ -6,26 +6,53 @@ require (
 	github.com/c-robinson/iplib v1.0.3
 	github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
+	github.com/otiai10/copy v1.12.0
 	github.com/sirupsen/logrus v1.8.1
-	github.com/spf13/cobra v1.2.1
-	github.com/spf13/viper v1.9.0
-	golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420
+	github.com/spf13/cobra v1.8.1
+	github.com/spf13/viper v1.19.0
+	golang.org/x/net v0.25.0
 )
 
 require (
-	github.com/fsnotify/fsnotify v1.5.1 // indirect
+	github.com/bytedance/sonic v1.11.6 // indirect
+	github.com/bytedance/sonic/loader v0.1.1 // indirect
+	github.com/cloudwego/base64x v0.1.4 // indirect
+	github.com/cloudwego/iasm v0.2.0 // indirect
+	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.3 // indirect
+	github.com/gin-contrib/sse v0.1.0 // indirect
+	github.com/go-playground/locales v0.14.1 // indirect
+	github.com/go-playground/universal-translator v0.18.1 // indirect
+	github.com/go-playground/validator/v10 v10.20.0 // indirect
+	github.com/goccy/go-json v0.10.2 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
-	github.com/inconshreveable/mousetrap v1.0.0 // indirect
-	github.com/magiconair/properties v1.8.5 // indirect
-	github.com/mitchellh/mapstructure v1.4.2 // indirect
-	github.com/pelletier/go-toml v1.9.4 // indirect
-	github.com/spf13/afero v1.6.0 // indirect
-	github.com/spf13/cast v1.4.1 // indirect
-	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.7 // indirect
+	github.com/leodido/go-urn v1.4.0 // indirect
+	github.com/magiconair/properties v1.8.7 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
+	github.com/sagikazarmark/locafero v0.4.0 // indirect
+	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
+	github.com/sourcegraph/conc v0.3.0 // indirect
+	github.com/spf13/afero v1.11.0 // indirect
+	github.com/spf13/cast v1.6.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/subosito/gotenv v1.2.0 // indirect
-	golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf // indirect
-	golang.org/x/text v0.3.6 // indirect
-	gopkg.in/ini.v1 v1.63.2 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
+	github.com/subosito/gotenv v1.6.0 // indirect
+	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
+	github.com/ugorji/go/codec v1.2.12 // indirect
+	go.uber.org/atomic v1.9.0 // indirect
+	go.uber.org/multierr v1.9.0 // indirect
+	golang.org/x/arch v0.8.0 // indirect
+	golang.org/x/crypto v0.23.0 // indirect
+	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
+	golang.org/x/sys v0.20.0 // indirect
+	golang.org/x/text v0.15.0 // indirect
+	google.golang.org/protobuf v1.34.1 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

gocage.conf.yml

@@ -5,6 +5,9 @@ datastore:
 # Prefix all commands with sudo
 sudo: false
 
+# Directory used to store update temporary files. Mutualized so we save bandwith
+updateWorkDir: /iocage/freebsd-updates
+
 # Columns to display when "gocage list". Column names are struct fields, see cmd/struct.go
 outcol: 'JID,Name,Config.Release,Config.Ip4_addr,Running'
 

jail/main.go

@@ -14,16 +14,17 @@ package jail
 */
 import "C"
 import (
-	"strconv"
+    "strconv"
 //    "syscall"
     "unsafe"
 )
 
 
 type Jail struct {
-    Name	string
-    Jid		int
-	Path	string
+    Name          string
+    Jid           int
+    Path          string
+    Devfs_ruleset int
 }
 
 
@@ -33,8 +34,8 @@ func GetJails() ([]Jail, error) {
     var jl Jail
     var err error
 
-	// Make "params" a list of 4 jails parameters
-    params := make([]C.struct_jailparam, 4)
+	// Make "params" a list of 5 jails parameters
+    params := make([]C.struct_jailparam, 5)
 
     // initialize parameter names
     csname := C.CString("name")
@@ -43,27 +44,31 @@ func GetJails() ([]Jail, error) {
     defer C.free(unsafe.Pointer(csjid))
     cspath := C.CString("path")
     defer C.free(unsafe.Pointer(cspath))
+	csdevfsrs := C.CString("devfs_ruleset")
+	defer C.free(unsafe.Pointer(csdevfsrs))
     cslastjid := C.CString("lastjid")
     defer C.free(unsafe.Pointer(cslastjid))
+	
 
     // initialize params struct with parameter names
     C.jailparam_init(&params[0], csname)
     C.jailparam_init(&params[1], csjid)
     C.jailparam_init(&params[2], cspath)
+	C.jailparam_init(&params[3], csdevfsrs)
 
     // The key to retrieve jail. lastjid = 0 returns first jail and its jid as jailparam_get return value
-    C.jailparam_init(&params[3], cslastjid)
+    C.jailparam_init(&params[4], cslastjid)
 
     lastjailid := 0
     cslastjidval := C.CString(strconv.Itoa(lastjailid))
     defer C.free(unsafe.Pointer(cslastjidval))
 
-    C.jailparam_import(&params[3], cslastjidval)
+    C.jailparam_import(&params[4], cslastjidval)
 
     // loop on existing jails
     for lastjailid >= 0 {
         // get parameter values
-        lastjailid = int(C.jailparam_get(&params[0], 4, 0))
+        lastjailid = int(C.jailparam_get(&params[0], 5, 0))
         if lastjailid > 0 {
             nametmp := C.jailparam_export(&params[0])
             jl.Name = C.GoString(nametmp)
@@ -75,23 +80,28 @@ func GetJails() ([]Jail, error) {
             // Memory mgmt : Non gere par Go
             C.free(unsafe.Pointer(jidtmp))
             
-			pathtmp := C.jailparam_export(&params[2])
+            pathtmp := C.jailparam_export(&params[2])
             jl.Path = C.GoString(pathtmp)
             // Memory mgmt : Non gere par Go
             C.free(unsafe.Pointer(pathtmp))
 
+            drstmp := C.jailparam_export(&params[3])
+            jl.Devfs_ruleset, _ = strconv.Atoi(C.GoString(drstmp))
+            // Memory mgmt : Non gere par Go
+            C.free(unsafe.Pointer(drstmp))
+
             jls = append(jls, jl)
             //log.Debug("Got jid " + strconv.Itoa(jl.jid) + " with name " + jl.name)
 
             // Prepare next loop iteration
             cslastjidval := C.CString(strconv.Itoa(lastjailid))
             defer C.free(unsafe.Pointer(cslastjidval))
-            C.jailparam_import(&params[3], cslastjidval)
+            C.jailparam_import(&params[4], cslastjidval)
         }
     }
 
-	// Free 4 items of params list
-    C.jailparam_free(&params[0], 4)
+	// Free 5 items of params list
+    C.jailparam_free(&params[0], 5)
 
     return jls, err
 }

service/gocage

@@ -0,0 +1,43 @@
+#!/bin/sh
+#
+# $FreeBSD$
+#
+
+# PROVIDE: gocage
+# REQUIRE: LOGIN cleanvar
+# KEYWORD: shutdown
+
+# Add the following lines to /etc/rc.conf to enable :
+#
+# gocage_enable="YES"
+#
+# gocage_conf="/usr/local/etc/gocage.conf.yml"
+#
+
+. /etc/rc.subr
+
+name="gocage"
+rcvar=gocage_enable
+
+# read configuration and set defaults
+load_rc_config "$name"
+
+: ${gocage_enable:="NO"}
+: ${gocage_conf="/usr/local/etc/gocage.conf.yml"}
+
+start_cmd=${name}_start
+stop_cmd=${name}_stop
+
+gocage_start()
+{
+    echo "Gocage starting jails... "
+    /usr/local/bin/gocage -c ${gocage_conf} start
+}
+
+gocage_stop()
+{
+    echo "Gocage stopping jails... "
+    /usr/local/bin/gocage  -c ${gocage_conf} stop
+}
+
+run_rc_command "$1"