v0.42h: finish previous commit

.gitignore

@@ -1,3 +1,2 @@
 gocage
 go.sum
-

CHANGELOG

@@ -0,0 +1,4 @@
+v.0.33b : Support jailing datasets on differents pools : jail_zfs_dataset now have to include the pool name  
+v.0.33c : Parallelize start/stop of jails with same priority  
+v.0.34 : jail name can be shortened 
+v.0.35 : One can now "gocage destroy"

README.md

@@ -4,19 +4,34 @@ GoCage
 Jail management tool for FreeBSD, written in Go.  
 Support iocage jails, so they can coexist.  
 Gocage is meant to be a complete jail management tool with network, snapshots, jail cloning support and a web interface. This is the hypothetic future.  
+Gocage can handle multiple datastores, so you can have jails on HDD storage and jails on SSD storage.  
+
+From v0.33b, due to multi ZFS pool support, gocage is no longer 100% compatible with iocage.  
+Zfs datasets now should be specified with the ZFS pool. e.g. :  
+<pre><code>Config.Jail_zfs = 1
+Config.Jail_zfs_dataset = myzfspool/poudriere
+Config.Jail_zfs_mountpoint = none
+</code></pre>
+
+Create jails
+------------
+You need to specify release, and optional configuration:  
+<pre><code>gocage create jail1 -r 13.2-RELEASE -p "Config.Ip4_addr='vnet0|192.168.1.91/24',Config.Ip6=none,Config.Boot=1"
+</code></pre>
+
+Create basejail. A basejail is a jail based on a release: system will be nullfs read-only mounted from the release directory. Main advantage is that release updates will immediately apply to jails based on this release. Another advantage is that jail system is mounted read-only, a plus from a security perspective:  
+<pre><code>gocage create -b -r 14.0-RELEASE basejail1
+</code></pre>
 
-At present time, it list and stops jails.  
 
 List jails
 ----------
-Nothing fancy, just use  
-`gocage list`
+<pre><code>gocage list</code></pre>
 
 ### Specify fields to display
 
 Use -o to specify which fields you want to display:  
-<pre><code>
-gocage list -o JID,Name,Running,Config.Boot,Config.Comment  
+<pre><code>gocage list -o JID,Name,Running,Config.Boot,Config.Comment  
 +=====+==========+=========+=============+================+  
 | JID | Name     | Running | Config.Boot | Config.Comment |  
 +=====+==========+=========+=============+================+  
@@ -30,7 +45,7 @@ gocage list -o JID,Name,Running,Config.Boot,Config.Comment
 +-----+----------+---------+-------------+----------------+  
 </code></pre>
 
-See [cmd/struct.go](https://git.nosd.in/yo/gocage/src/branch/master/cmd/struct.go) for field names.
+Use `gocage properties`to list available fields.
 
 
 Filter jails  
@@ -38,8 +53,7 @@ Filter jails
 
 ### By name
 Just add name on gocage list command :  
-<pre><code>
-gocage list srv-bdd srv-web
+<pre><code>gocage list srv-bdd srv-web
 +=====+=========+=================+=======================+=========+
 | JID | Name    | Config.Release  | Config.Ip4_addr       | Running |
 +=====+=========+=================+=======================+=========+
@@ -47,12 +61,11 @@ gocage list srv-bdd srv-web
 +-----+---------+-----------------+-----------------------+---------+
 | 41  | srv-web | 13.0-RELEASE-p4 | vnet0|192.168.1.26/24 | true    |
 +-----+---------+-----------------+-----------------------+---------+
-</pre></code>
+</code></pre>
 
 ### By field value
 You can filter jails with -f option, followed by key=value. Suppose you want to see only active at boot jails:  
-<pre><code>
-gocage list -f Config.Boot=1 -o JID,Name,Running,Config.Boot,Config.Comment
+<pre><code>gocage list -f Config.Boot=1 -o JID,Name,Running,Config.Boot,Config.Comment
 +=====+==========+=========+=============+================+
 | JID | Name     | Running | Config.Boot | Config.Comment |
 +=====+==========+=========+=============+================+
@@ -64,11 +77,10 @@ gocage list -f Config.Boot=1 -o JID,Name,Running,Config.Boot,Config.Comment
 +-----+----------+---------+-------------+----------------+
 | 22  | srv-dns1 | true    | 1           |                |
 +-----+----------+---------+-------------+----------------+
-</pre></code>
+</code></pre>
 
 Now, only active at boot and running :  
-<pre><code>
-gocage list -f Config.Boot=1,Running=true -o JID,Name,Running,Config.Boot
+<pre><code>gocage list -f Config.Boot=1,Running=true -o JID,Name,Running,Config.Boot
 +=====+==========+=========+=============+
 | JID | Name     | Running | Config.Boot |
 +=====+==========+=========+=============+
@@ -78,9 +90,154 @@ gocage list -f Config.Boot=1,Running=true -o JID,Name,Running,Config.Boot
 +-----+----------+---------+-------------+
 | 22  | srv-dns1 | true    | 1           |
 +-----+----------+---------+-------------+
-</pre></code>
+</code></pre>
 
-Stop jails
+Sort jails  
 ----------
-`gocage stop test`
+Use -s switch followed by sort criteria. Criteria is a field name, prefixed with + or - for sort order (increase/decrease):  
+<pre><code>gocage list -f Config.Boot=1,Running=true -o JID,Name,Running,Config.Boot -s +JID
++=====+==========+=========+=============+
+| JID | Name     | Running | Config.Boot |
++=====+==========+=========+=============+
+| 22  | srv-dns1 | true    | 1           |
++-----+----------+---------+-------------+
+| 29  | bdd-tst  | true    | 1           |
++-----+----------+---------+-------------+
+| 183 | test     | true    | 1           |
++-----+----------+---------+-------------+
+</code></pre>
+
+You can use up to 3 criteria, delimited with comma.  
+As an example, you want to list boot priorities of automatically starting jails:  
+<pre><code>gocage list -o JID,Name,Config.Ip4_addr,Config.Priority,Config.Boot,Running -s -Config.Priority,-Config.Boot -f Running=true
++=====+==============+=======================+=================+=============+=========+
+| JID | Name         | Config.Ip4_addr       | Config.Priority | Config.Boot | Running |
++=====+==============+=======================+=================+=============+=========+
+| 1   | srv-dhcp     | vnet0|192.168.1.2/24  | 99              | 1           | true    |
++-----+--------------+-----------------------+-----------------+-------------+---------+
+| 8   | srv-dns      | vnet0|192.168.1.1/24  | 80              | 1           | true    |
++-----+--------------+-----------------------+-----------------+-------------+---------+
+| 7   | srv-random   | vnet0|192.168.1.12/24 | 20              | 1           | true    |
++-----+--------------+-----------------------+-----------------+-------------+---------+
+| 4   | coincoin     | vnet0|192.168.1.9/24  | 20              | 0           | true    |
++-----+--------------+-----------------------+-----------------+-------------+---------+
+</code></pre>
+
+
+Stop jails  
+----------
+<pre><code>gocage stop test</code></pre>
+
+Update jails  
+----------
+To update jail patch version, use gocage update :  
+<pre><code>gocage update test</code></pre>
+
+Update basejails/releases  
+----------
+To update basejails, you need to update the release they are based on. Specify release with -r, and the datastore storing concerned release with -d :  
+<pre><code>gocage update -d fastgocage -r 14.1-RELEASE</code></pre>
+
+Upgrade jails  
+----------
+To upgrade jail to newer release, use gocage upgrade :  
+<pre><code>gocage upgrade -r 13.2-RELEASE test</code></pre>
+  
+A pre-upgrade snapshot wil be made so you can rollback if needed.  
+
+Upgrading basejail/release
+----------
+Upgrading basejails currently needs to be done manually, for each jail.  
+The idea is to stop the jail, change the content of its fstab file to point to the new release, then start jail.  
+If one change the fstab while the jail is running, its system directories won't be unmounted at stop time and this will provoke stop errors.  
+To minimize downtime, the change could be scripted:  
+<pre><code>gocage stop jail1
+sed -i .bak 's/14.0-RELEASE/14.1-RELEASE/' /iocage/jails/jail1/fstab
+# Avoid race-condition by waiting for the update in fstab
+until grep -q 14.1-RELEASE /iocage/jails/jail1/fstab; do sleep 0.2; done
+gocage start jail1
+</code></pre>
+
+You can now update ports.
+
+Delete jails  
+----------
+<pre><code>gocage destroy test</code></pre>
+
+
+Multi datastore  
+----------
+A datastore is a ZFS dataset mounted. It should be declared in gocage.conf.yml, specifying its ZFS mountpoint :
+<pre><code>datastore:
+  - /iocage
+  - /fastiocage
+</code></pre>
+
+In gocage commands, datastore name is the mountpoint without its "/" prefix.  
+
+### List datastores
+<pre><code>gocage datastore list 
++============+=============+============+===========+==========+============+
+| Name       | Mountpoint  | ZFSDataset | Available | Used     | Referenced |
++============+=============+============+===========+==========+============+
+| iocage     | /iocage     | hdd/iocage | 1.6 TB    | 414.9 GB | 27.5 KB    |
++------------+-------------+------------+-----------+----------+------------+
+| fastiocage | /fastiocage | ssd/iocage | 1.5 TB    | 65.3 KB  | 34.6 KB    |
++------------+-------------+------------+-----------+----------+------------+
+</code></pre>
+
+### Filter datastores  
+As with jails and snapshots, you can filter by name:  
+<pre><code>gocage datastore list iocage
++============+=============+============+===========+==========+============+
+| Name       | Mountpoint  | ZFSDataset | Available | Used     | Referenced |
++============+=============+============+===========+==========+============+
+| iocage     | /iocage     | hdd/iocage | 1.6 TB    | 414.9 GB | 27.5 KB    |
++------------+-------------+------------+-----------+----------+------------+
+</code></pre>
+
+### Sort datastores  
+You can sort datastores:  
+<pre><code>gocage datastore list -s -Available
++============+=============+============+===========+==========+============+
+| Name       | Mountpoint  | ZFSDataset | Available | Used     | Referenced |
++============+=============+============+===========+==========+============+
+| iocage     | /iocage     | hdd/iocage | 1.6 TB    | 415.0 GB | 27.5 KB    |
++------------+-------------+------------+-----------+----------+------------+
+| fastiocage | /fastiocage | ssd/iocage | 1.5 TB    | 65.3 KB  | 34.6 KB    |
++------------+-------------+------------+-----------+----------+------------+
+</code></pre>
+
+Use `gocage properties`to list available fields.
+
+Migrating jails  
+----------
+With multi datastore comes the need to migrate a jail between datastores.  
+Migration can be done with a minimal downtime, using zfs differential send/receive.  
+Source jail datasets are sent to the destination datastore, jail is stopped and a last differential sync is done before starting jail on new datastore.  
+
+### Warning 
+Be aware the moment you migrate a jail to another datastore than /iocage default, you lose compatibility with iocage.  
+Then you need to disable iocage service, and enable gocage so the jails will start automatically at boot.  
+Also make sure, if you don't destroy source jail, that it won't have the "boot" property set or you will have the 2 jails up at boot.  
+
+<pre><code>gocage migrate -d fastiocage srv-random
+Snapshot data/iocage/jails/srv-random: Done
+Snapshot data/iocage/jails/srv-random/root: Done
+Migrate jail config dataset to fastdata/iocage/jails/srv-random: Done
+Migrate jail filesystem dataset to fastdata/iocage/jails/srv-random/root: Done
+</code></pre>
+
+
+Fetch  
+----------
+Files can be fetched from custom repository, or from local directory with "from" option.  
+For example if you destroyed releases/12.3-RELEASE and still have the downloaded files in /iocage/download/12.3-RELEASE:  
+<pre><code>gocage fetch -r 12.3 -d iocage -f file:/iocage/download
+</code></pre>
+
+
+TODO
+----------
+gocage create from templates  
 

TODO.md

@@ -1,4 +1,19 @@
 Replicating jails between two servers (use zrepl)
+Manage remote jails :
+  - Make gocage a service
+  - All commands should become API endpoint
+  - How to handle authentication ?
 
+DEBUG: 
+- cmd/list.go:275:
+    // FIXME ??? Shouldn't be ioc-$Name ?
+    j.InternalName = rj.Name
+- WriteConfigToDisk don't write neither "release" in cmd stop neither "last_started" in cmd start
+  26/08/2023 : Last_started is updated
 
+BUGS:
+- unable to set values containing equal sign :
+  # gocage set Config.Exec_poststart="jail -m allow.mount.linprocfs=1 name=ioc-poudriere-noo" poudriere-noo
+  Error parsing args: Config.Exec_poststart=jail -m allow.mount.linprocfs=1 name=ioc-poudriere-noo
+- Fix fstab when migrating jail  
 

cmd/console.go

@@ -0,0 +1,45 @@
+package cmd
+
+import (
+	"os"
+	"fmt"
+	"log"
+	"errors"
+	"strconv"
+	"syscall"
+)
+
+func ShellJail(args []string) error {
+	// We cant shell more than one jail bc we replace gocage execution with jexec, so there wont be no return to gocage
+	if len(args) > 0 {
+		cj, err := getJailFromArray(args[0], []string{"basejail", "jail"}, gJails)
+		if err != nil {
+			fmt.Printf("Error getting jail %s: %v\n", args[0], err)
+			return err
+		}
+		if err := shellJail(cj); err != nil {
+			fmt.Printf("%v\n", err)
+			return err
+		}
+	}
+	
+	return nil
+}
+
+func shellJail(jail *Jail) error {
+	if false == jail.Running {
+		return errors.New("Jail is not running")
+	}
+
+	jid := strconv.Itoa(jail.JID)
+
+	err := syscall.Exec("/usr/sbin/jexec", []string{"jexec", jid, "login", "-f", "root"}, os.Environ())
+
+	// We should never get here, as syscall.Exec replace the gocage binary execution with jexec
+	// This means the moment syscall.Exec fires, gocage execution halt.
+	if err != nil {
+		log.Printf("Exec returned %v\n", err)
+	}
+
+	return nil
+}

cmd/create.go

@@ -0,0 +1,300 @@
+package cmd
+
+import (
+	"os"
+	"fmt"
+	//"log"
+	"time"
+	"errors"
+	"strings"
+	cp "github.com/otiai10/copy"
+	log "github.com/sirupsen/logrus"
+)
+
+// TODO : Add a flag to specify which parts of freebsd base we want : Slim jail only need base.txz, neither lib32 nor src.txz
+func CreateJail(args []string) {
+	var err error
+	var jtype []string
+
+	if gCreateArgs.BaseJail && gCreateArgs.Release == "" {
+		fmt.Println("Release should be set when creating basejail")
+		os.Exit(1)
+	}
+
+	if len(gCreateArgs.JailType) > 0 {
+		jtype = []string{gCreateArgs.JailType}
+	}
+	
+	for _, jname := range args {
+		// Check if jail exist and is distinctly named
+		_, err = getJailFromArray(jname, jtype, gJails)
+		if err != nil {
+			if strings.EqualFold(err.Error(), "Jail not found") {
+				
+			} else {
+				fmt.Printf("ERROR: %s\n", err.Error())
+				return
+			}
+		} else {
+			fmt.Printf("Jail exist: %s\n", jname)
+			continue
+		}
+
+		fmt.Printf("  > create jail %s\n", jname)
+
+		var ds *Datastore
+		if len(gCreateArgs.Datastore) > 0 {
+			log.Debugf("Use %s datastore\n", gCreateArgs.Datastore)
+			ds, err = getDatastoreFromArray(gCreateArgs.Datastore, gDatastores)
+			if err != nil {
+				fmt.Printf("ERROR Getting datastore: %s\n", gCreateArgs.Datastore, err.Error())
+				return
+			}
+		} else {
+			ds = &gDatastores[0]
+		}
+
+		// Get base template if specified
+		if gCreateArgs.BaseJail {
+			/**************************************************************************
+			*  Create based jail from a template
+			*/
+			log.Debugf("Jail will be created read-only from release %s\n", gCreateArgs.Release)
+
+			// First check if we got release on the same datastore
+			releasePath := fmt.Sprintf("%s/releases/%s/root", ds.Mountpoint, gCreateArgs.Release)
+			_, err := os.Stat(releasePath)
+			if os.IsNotExist(err) {
+				fmt.Printf("ERROR: Release locally not available. Run \"gocage fetch\"\n")
+				return
+			}
+
+			// Create jail datasets
+			dstDset := fmt.Sprintf("%s/jails/%s", ds.ZFSDataset, jname)
+			fmt.Printf("    > Initialize dataset %s\n", dstDset)
+			err = zfsCreateDataset(dstDset, "", "")
+			if err != nil {
+				fmt.Printf("ERROR creating dataset %s: %s\n", dstDset, err.Error())
+				return
+			}
+			// Create jail root datasets
+			dstRootDset := fmt.Sprintf("%s/jails/%s/root", ds.ZFSDataset, jname)
+			fmt.Printf("    > Initialize dataset %s\n", dstRootDset)
+			err = zfsCreateDataset(dstRootDset, "", "")
+			if err != nil {
+				fmt.Printf("ERROR creating dataset %s: %s\n", dstRootDset, err.Error())
+				return
+			}
+
+			// Create needed directories with basejail permissions
+			fmt.Printf("    > Create base read-only directories\n")
+			dstRootDir := fmt.Sprintf("%s/jails/%s/root", ds.Mountpoint, jname)
+			for _, d := range append(gBaseDirs, gEmptyDirs...) {
+				dstPath := dstRootDir
+				srcPath := releasePath
+				for _, cd := range strings.Split(d, "/") {
+					srcPath = fmt.Sprintf("%s/%s", srcPath, cd)
+					dstPath = fmt.Sprintf("%s/%s", dstPath, cd)
+					_, err := os.Stat(dstPath)
+					if errors.Is(err, os.ErrNotExist) {
+						srcPerm, err := getPermissions(srcPath)
+						if err != nil {
+							fmt.Printf("ERROR getting permissions of %s: %s\n", srcPath, err.Error())
+							return
+						}
+						err = os.Mkdir(dstPath, srcPerm.Mode().Perm())
+						if err != nil {
+							fmt.Printf("ERROR creating directory %s: %s\n", dstPath, err.Error())
+							return
+						}
+					}
+				}
+			}
+
+			// Copy these from basejail
+			fmt.Printf("    > Create base writable directories\n")
+			for _, d := range gCopyDirs {
+				err := cp.Copy(fmt.Sprintf("%s/%s", releasePath, d), fmt.Sprintf("%s/%s", dstRootDir, d))
+				if err != nil {
+					fmt.Printf("ERROR copying %s to %s: %s\n", fmt.Sprintf("%s/%s", releasePath, d),
+						   fmt.Sprintf("%s/%s", dstRootDir, d), err.Error())
+					return
+				}
+			}
+
+			///////////////////////////////////////////////////////////////////////
+			// Copy defaults.json...
+			jailConfPath := fmt.Sprintf("%s/jails/%s/config.json", ds.Mountpoint, jname)
+			err = copyFile(fmt.Sprintf("%s/defaults.json", ds.Mountpoint),
+				       jailConfPath)
+			if err != nil {
+				fmt.Printf("ERROR creating config.json: %s\n", err.Error())
+				return
+			}
+			///////////////////////////////////////////////////////////////////////
+			// ... and update it
+			jailConf, err := getJailConfig(jailConfPath)
+			if err != nil {
+				log.Println("ERROR reading jail config from %s", jailConfPath)
+			}
+
+			// Build jail object from config
+			jailRootPath := fmt.Sprintf("%s/jails/%s/%s", ds.Mountpoint, jname, "root")
+			j := Jail{
+				Name:       jailConf.Host_hostuuid,
+				Config:     jailConf,
+				ConfigPath: jailConfPath,
+				Datastore:  ds.Name,
+				RootPath:   jailRootPath,
+				Running:    false,
+			}
+
+			// We need to store the basejail template. We could :
+			// 1. Use "origin" ?
+			// 2. Add a json item to config ("basejail_template" p.e.), but iocage would delete it once jail is started from iocage
+			// 3. Add a gocage specific config ("config.gocage.json" p.e.)
+			j.Config.Jailtype = "basejail"
+			j.Config.Origin = gCreateArgs.Release
+			j.Config.Host_hostname = jname
+			j.Config.Host_hostuuid = jname
+
+			j.WriteConfigToDisk(false)
+
+			///////////////////////////////////////////////////////////////////////
+			// Create fstab
+			fstabHandle, err := os.Create(fmt.Sprintf("%s/jails/%s/fstab", ds.Mountpoint, jname))
+			if err != nil {
+				fmt.Printf("ERROR creating fstab: %s", err.Error())
+				return
+			}
+			defer fstabHandle.Close()
+
+			for _, d := range gBaseDirs {
+				fmt.Fprintf(fstabHandle, "%s\t%s\tnullfs\tro\t0\t0\n", fmt.Sprintf("%s/%s", releasePath, d), fmt.Sprintf("%s/%s", dstRootDir, d))
+			}
+
+			fmt.Printf("  > Jail created!\n")
+		} else {
+			/**************************************************************************
+			 *  Create normal jail with its own freebsd base
+			 */
+			log.Debugf("Creating jail with its own freebsd base\n")
+
+			// First check if we got release on the same datastore
+			_, err := os.Stat(fmt.Sprintf("%s/releases/%s/root", ds.Mountpoint, gCreateArgs.Release))
+			if os.IsNotExist(err) {
+				fmt.Printf("ERROR: Release locally not available. Run \"gocage fetch\"\n")
+				return
+			}
+
+			///////////////////////////////////////////////////////////////////////
+			// Create and populate jail filesystem from release
+			dstDset := fmt.Sprintf("%s/jails/%s", ds.ZFSDataset, jname)
+			fmt.Printf("    > Initialize dataset %s\n", dstDset)
+			sNow := time.Now().Format("20060102150405")
+			reldset := fmt.Sprintf("%s/releases/%s", ds.ZFSDataset, gCreateArgs.Release)
+			err = zfsSnapshot(reldset, sNow)
+			if err != nil {
+				fmt.Printf("ERROR Creating snapshot of %s: %s\n", reldset, err.Error())
+				return
+			}
+
+			err = zfsCopy(fmt.Sprintf("%s@%s", reldset, sNow), dstDset)
+			if err != nil {
+				fmt.Printf("ERROR sending snapshot to %s: %s\n", dstDset, err.Error())
+				return
+			}
+			// Remove snapshot of release, then snapshot of destination dataset
+			err = zfsDestroy(fmt.Sprintf("%s@%s", reldset, sNow))
+			if err != nil {
+				fmt.Printf("ERROR destroying snapshot %s: %s\n", reldset, err.Error())
+				return
+			}
+			err = zfsDestroy(fmt.Sprintf("%s@%s", dstDset, sNow))
+			if err != nil {
+				fmt.Printf("ERROR destroying snapshot %s: %s\n", dstDset, err.Error())
+				return
+			}
+
+			dstRootDset := fmt.Sprintf("%s/jails/%s/root", ds.ZFSDataset, jname)
+			fmt.Printf("    > Initialize dataset %s\n", dstRootDset)
+			relrootdset := fmt.Sprintf("%s/releases/%s/root", ds.ZFSDataset, gCreateArgs.Release)
+			err = zfsSnapshot(relrootdset, sNow)
+			if err != nil {
+				fmt.Printf("ERROR Creating snapshot of %s: %s\n", relrootdset, err.Error())
+				return
+			}
+			err = zfsCopy(fmt.Sprintf("%s@%s", relrootdset, sNow), dstRootDset)
+			if err != nil {
+				fmt.Printf("ERROR sending snapshot to %s: %s\n", dstRootDset, err.Error())
+				return
+			}
+			// Remove snapshot of release, then snapshot of destination dataset
+			err = zfsDestroy(fmt.Sprintf("%s@%s", relrootdset, sNow))
+			if err != nil {
+				fmt.Printf("ERROR destroying snapshot %s: %s\n", relrootdset, err.Error())
+				return
+			}
+			err = zfsDestroy(fmt.Sprintf("%s@%s", dstRootDset, sNow))
+			if err != nil {
+				fmt.Printf("ERROR destroying snapshot %s: %s\n", dstRootDset, err.Error())
+				return
+			}
+
+			///////////////////////////////////////////////////////////////////////
+			// Copy defaults.json...
+			jailConfPath := fmt.Sprintf("%s/jails/%s/config.json", ds.Mountpoint, jname)
+			err = copyFile(fmt.Sprintf("%s/defaults.json", ds.Mountpoint), 
+					jailConfPath)
+			if err != nil {
+				fmt.Printf("ERROR creating config.json: %s\n", err.Error())
+				return
+			}
+			///////////////////////////////////////////////////////////////////////
+			// ... and update it
+			jailConf, err := getJailConfig(jailConfPath)
+			if err != nil {
+				log.Println("ERROR reading jail config from %s", jailConfPath)
+			}
+
+			// Build jail object from config
+			jailRootPath := fmt.Sprintf("%s/jails/%s/%s", ds.Mountpoint, jname, "root")
+			j := Jail{
+				Name:       jailConf.Host_hostuuid,
+				Config:     jailConf,
+				ConfigPath: jailConfPath,
+				Datastore:  ds.Name,
+				RootPath:   jailRootPath,
+				Running:    false,
+			}
+
+			j.Config.Release = gCreateArgs.Release
+			j.Config.Host_hostname = jname
+			j.Config.Host_hostuuid = jname
+			j.Config.Jailtype = "jail"
+			j.WriteConfigToDisk(false)
+
+			///////////////////////////////////////////////////////////////////////
+			// Create fstab
+			fstabHandle, err := os.Create(fmt.Sprintf("%s/jails/%s/fstab", ds.Mountpoint, jname))
+			if err != nil {
+				fmt.Printf("ERROR creating fstab: %s", err.Error())
+				return
+			}
+			defer fstabHandle.Close()
+			fmt.Printf("  > Jail created!\n")
+		}
+
+		var cmdline []string
+		for _, props := range strings.Split(gCreateArgs.Properties, ",") {
+			cmdline = append(cmdline, props)
+		}
+
+		// Reload jail list so SetJailProperties will see it
+		ListJails(nil, false)
+
+		cmdline = append(cmdline, jname)
+		log.Debugf("cmdline: \"%v\"", cmdline)
+		SetJailProperties(cmdline)
+	}
+}

cmd/datastore.go

@@ -0,0 +1,108 @@
+package cmd
+
+import (
+	"fmt"
+	"errors"
+	"reflect"
+	"strconv"
+	"strings"
+) 
+
+func ListDatastores(args []string, display bool) error {
+	/***************************************************************
+	/   Filter datastores by names given on command line
+	/**************************************************************/
+	for _, d := range args {
+		cmd := fmt.Sprintf("zfs list -p -H -o name,used,available,referenced %s", d)
+		out, err := executeCommand(cmd)
+		if err != nil {
+			if strings.HasSuffix(err.Error(), "No such file or directory") {
+				return errors.New(fmt.Sprintf("Datastore does not exist: %s", d))
+			} else {
+				return errors.New(fmt.Sprintf("Error executing command %s: %v; command returned: %s\n", cmd, err, out))
+			}
+		}
+		
+		fields := strings.Fields(out)
+		if len(fields) < 4 {
+			return errors.New(fmt.Sprintf("Error parsing output of \"%s\": Not enough fields", cmd))
+		}
+		u, _ := strconv.ParseUint(fields[1], 10, 64)
+		a, _ := strconv.ParseUint(fields[2], 10, 64)
+		r, _ := strconv.ParseUint(fields[3], 10, 64)
+		ds := Datastore{Name: d[1:], Mountpoint: d, ZFSDataset: fields[0], Used: u, Referenced: r, Available: a}
+		
+		err = loadDefaultsForDatastore(&ds)
+		if err != nil {
+			return err
+		}
+		
+		gDatastores = append(gDatastores, ds)
+	}
+	
+	fields := strings.Split(gDisplayDColumns, ",")
+
+	if true == display {
+		/***************************************************************
+		/   Sort datastores
+		/     We support 3 sort criteria max
+		/**************************************************************/
+		if len(gSortDSFields) > 0 && gSortDSFields != "none" {		
+			ds := initDatastoreSortStruct()
+
+			// The way we manage criteria quantity is not very elegant...
+			var fct1, fct2, fct3 *reflect.Value
+			for i, c := range strings.Split(gSortDSFields, ",") {
+				var fctName string
+				if strings.HasPrefix(c, "-") {
+					fctName = fmt.Sprintf("%sDec", strings.Replace(c, "-", "", 1))
+				} else { // Par defaut (pas de prefix +/-) on considere un tri incremental
+					fctName = fmt.Sprintf("%sInc", strings.Replace(c, "+", "", 1))
+				}
+
+				// Get function by its name
+				fct, _, err := getStructFieldValue(ds, fctName)
+				
+				if err != nil {
+					fieldName := strings.Replace(strings.Replace(c, "-", "", 1), "+", "", 1)
+					return errors.New(fmt.Sprintf("ERROR getting DatastoreSort struct field %s. Please check the field name: %s\n", fctName, fieldName))
+				}
+				switch i + 1 {
+				case 1:
+					fct1 = fct
+				case 2:
+					fct2 = fct
+				case 3:
+					fct3 = fct
+				}
+			}
+
+			switch len(strings.Split(gSortDSFields, ",")) {
+			case 1:
+				DatastoresOrderedBy(fct1.Interface().(datastoreLessFunc)).Sort(gDatastores)
+			case 2:
+				DatastoresOrderedBy(fct1.Interface().(datastoreLessFunc), fct2.Interface().(datastoreLessFunc)).Sort(gDatastores)
+			case 3:
+				DatastoresOrderedBy(fct1.Interface().(datastoreLessFunc), fct2.Interface().(datastoreLessFunc), fct3.Interface().(datastoreLessFunc)).Sort(gDatastores)
+			}
+		}
+		displayDatastoresFields(gDatastores, fields)
+	}
+	
+	return nil
+}
+
+/********************************************************************************
+ * Load jails default config from $datastore/defaults.json
+ *******************************************************************************/
+func loadDefaultsForDatastore(ds *Datastore) error {
+	jc, err := getJailConfig(fmt.Sprintf("%s/defaults.json", ds.Mountpoint))
+	if err != nil {
+		return err
+	}
+
+	ds.DefaultJailConfig = jc
+	//gDefaultConfig = append(gDefaultConfig, jc)
+
+	return nil
+}

cmd/destroy.go

@@ -0,0 +1,59 @@
+package cmd
+
+import (
+	"fmt"
+	//"log"
+	"time"
+	//"errors"
+	"strings"
+)
+
+func DestroyJails(args []string) {
+	for _, a := range args {
+		cj, err := getJailFromArray(a, []string{""}, gJails)
+		if err != nil {
+			fmt.Printf("Error getting jail: %s\n", err)
+			return
+		}
+		
+		if cj.Running == true {
+			fmt.Printf("Jail %s is running\n", cj.Name)
+			if gForce == false {
+				var answer string
+				fmt.Printf("Stop jail and delete? (y/n) ")
+				fmt.Scanln(&answer)
+				if false == strings.EqualFold(answer, "y") {
+					return
+				}
+			}
+			fmt.Printf("Stopping jail %s\n", cj.Name)
+			StopJail([]string{fmt.Sprintf("%s/%s", cj.Datastore, cj.Name)})
+			// Give some time to the host OS to free all mounts accessing processes
+			time.Sleep(1 * time.Second)
+		}
+		
+		// Get root and config datasets, then destroy
+		dsRootName, err := zfsGetDatasetByMountpoint(cj.RootPath)
+		if err != nil {
+			fmt.Printf("Error getting root dataset: %s\n", err)
+			return
+		}
+		//fmt.Printf("DEBUG: Prepare to zfs destroy %s\n", dsRootName)
+		if err = zfsDestroy(dsRootName); err != nil {
+			fmt.Printf("Error deleting root dataset: %s\n", err)
+			return
+		}
+		
+		dsConfName, err := zfsGetDatasetByMountpoint(cj.ConfigPath)
+		if err != nil {
+			fmt.Printf("Error getting config dataset: %s\n", err)
+			return
+		}
+		//fmt.Printf("DEBUG: Prepare to zfs destroy %s\n", dsConfName)
+		if err = zfsDestroy(dsConfName); err != nil {
+			fmt.Printf("Error deleting config dataset: %s\n", err)
+			return
+		}
+		fmt.Printf("Jail %s is no more!\n", cj.Name)
+	}
+}

cmd/fetch.go

@@ -0,0 +1,378 @@
+package cmd
+
+import (
+	"io"
+	"os"
+	"fmt"
+	"bufio"
+	"bytes"
+	//"errors"
+	"strings"
+	"net/http"
+	//"archive/tar"
+	"encoding/hex"
+	"crypto/sha256"
+
+	//"github.com/ulikunitz/xz"
+	log "github.com/sirupsen/logrus"
+)
+
+const (
+	ReleaseServer = "download.freebsd.org"
+	ReleaseRootDir = "ftp/releases"
+)
+
+var (
+	// TODO : Make this a config/cmd line setting
+	//FetchFiles = []string{"base.txz", "lib32.txz", "src.txz"}
+	FetchFiles = []string{"base.txz"}
+)
+
+// TODO: Check if files already exist
+// Fetch release files, verify, put in datastore under ${datastore}/download
+// Only support http and file protocols
+func fetchRelease(release string, proto string, arch string, datastore string, fetchFrom string) error {
+	var ds Datastore
+	
+	log.SetReportCaller(true)
+	
+	if len(fetchFrom) > 0 {
+		proto = strings.Split(fetchFrom, ":")[0]
+	}
+	
+	if false == strings.EqualFold(proto, "http") &&
+		false == strings.EqualFold(proto, "file") {
+		return fmt.Errorf("Unsupported protocol: %s\n", proto)
+	}
+	
+	for _, ds = range gDatastores {
+		if strings.EqualFold(datastore, ds.Name) {
+			break
+		}
+	}
+	if false == strings.EqualFold(datastore, ds.Name) {
+		return fmt.Errorf("Datastore not found: %s\n", datastore)
+	}
+	
+	// Check datastore have a download dataset, and it is mounted
+	downloadDsName := fmt.Sprintf("%s/download", ds.ZFSDataset)
+	downloadDsMountPoint := fmt.Sprintf("%s/download", ds.Mountpoint)
+	exist, err := doZfsDatasetExist(downloadDsName)
+	if err != nil {
+		return fmt.Errorf("Error accessing dataset %s: %v\n", downloadDsName, err)
+	}
+	if false == exist {
+		// Then create dataset
+		if err := zfsCreateDataset(downloadDsName, downloadDsMountPoint, "lz4"); err != nil {
+			return fmt.Errorf("Error creating dataset %s: %v\n", downloadDsName, err)
+		}
+	}
+	
+	// Create download/XX.X dataset if necessary
+	thisDownloadDsName := fmt.Sprintf("%s/%s", downloadDsName, release)
+	thisDownloadDsMountPoint := fmt.Sprintf("%s/%s", downloadDsMountPoint, release)
+	exist, err = doZfsDatasetExist(thisDownloadDsName)
+	if err != nil {
+		return fmt.Errorf("Error accessing dataset %s: %v\n", thisDownloadDsName, err)
+	}
+	if false == exist {
+		// Then create dataset
+		if err := zfsCreateDataset(thisDownloadDsName, thisDownloadDsMountPoint, "lz4"); err != nil {
+			return fmt.Errorf("Error creating dataset %s: %v\n", thisDownloadDsName, err)
+		}
+	}
+
+	var fetchUrl string
+	if len(fetchFrom) > 0 {
+		fetchUrl = fmt.Sprintf("%s/%s", fetchFrom, release)
+	} else {
+		fetchUrl = fmt.Sprintf("%s://%s/%s/%s/%s", proto, ReleaseServer, ReleaseRootDir, arch, release)
+	}
+	log.Debugf("FetchURL = %s", fetchUrl)
+	
+	// check if proto/server/arch/release is available
+	if strings.EqualFold(proto, "http") {
+		resp, err := http.Get(fetchUrl)
+		if err != nil {
+			return fmt.Errorf("Can not get %s: %v\n", fetchUrl, err)
+		}
+		defer resp.Body.Close()
+		if resp.StatusCode != 200 {
+			return fmt.Errorf("Get %s returned %d, check release name\n", fetchUrl, resp.StatusCode)
+		}
+	}
+
+	// Fetch files
+	// Get MANIFEST so we get sha256 sums
+	if err := fetchFile(proto, fetchUrl, "MANIFEST", thisDownloadDsMountPoint, []byte{}); err != nil {
+		return fmt.Errorf("%v\n", err)
+	}
+	// Build an array of "file;checksum"
+	checksumMap, err := buildFileChecksumFromManifest(fmt.Sprintf("%s/MANIFEST", thisDownloadDsMountPoint), FetchFiles)
+	if err != nil {
+		return fmt.Errorf("%v\n", err)
+	}
+
+	// Fetch remaining files, verify integrity and write to disk
+	for f, c := range checksumMap {
+		if err := fetchFile(proto, fetchUrl, f, thisDownloadDsMountPoint, c); err != nil {
+			return fmt.Errorf("%v\n", err)
+		}
+	}
+	
+	return nil
+}
+
+// Extract release files stored in iocage/download/$RELEASE/ to iocage/releases/$RELEASE/root/
+func extractRelease(release string, datastore string) {
+	log.SetReportCaller(true)
+	
+	var ds Datastore
+	
+	for _, ds = range gDatastores {
+		if strings.EqualFold(datastore, ds.Name) {
+			break
+		}
+	}
+	if false == strings.EqualFold(datastore, ds.Name) {
+		fmt.Printf("Datastore not found: %s\n", datastore)
+		return
+	}
+	
+	// Check datastore have a releases dataset, and it is mounted
+	releaseDsName := fmt.Sprintf("%s/releases", ds.ZFSDataset)
+	releaseDsMountPoint := fmt.Sprintf("%s/releases", ds.Mountpoint)
+	exist, err := doZfsDatasetExist(releaseDsName)
+	if err != nil {
+		fmt.Printf("Error accessing dataset %s: %v\n", releaseDsName, err)
+		return
+	}
+	if false == exist {
+		// Then create dataset
+		if err := zfsCreateDataset(releaseDsName, releaseDsMountPoint, "lz4"); err != nil {
+			fmt.Printf("Error creating dataset %s: %v\n", releaseDsName, err)
+			return
+		}
+	}
+	
+	// Create releases/XX.X dataset if necessary
+	thisReleaseDsName := fmt.Sprintf("%s/%s", releaseDsName, release)
+	thisReleaseDsMountPoint := fmt.Sprintf("%s/%s", releaseDsMountPoint, release)
+	exist, err = doZfsDatasetExist(thisReleaseDsName)
+	if err != nil {
+		fmt.Printf("Error accessing dataset %s: %v\n", thisReleaseDsName, err)
+		return
+	}
+	if false == exist {
+		// Then create dataset
+		if err := zfsCreateDataset(thisReleaseDsName, thisReleaseDsMountPoint, "lz4"); err != nil {
+			fmt.Printf("Error creating dataset %s: %v\n", thisReleaseDsName, err)
+			return
+		}
+	}
+	
+	// Create releases/XX.X/root dataset if necessary
+	thisReleaseRootDsName := fmt.Sprintf("%s/root", thisReleaseDsName)
+	thisReleaseRootDsMountPoint := fmt.Sprintf("%s/root", thisReleaseDsMountPoint)
+	exist, err = doZfsDatasetExist(thisReleaseRootDsName)
+	if err != nil {
+		fmt.Printf("Error accessing dataset %s: %v\n", thisReleaseRootDsName, err)
+		return
+	}
+	if false == exist {
+		// Then create dataset
+		if err := zfsCreateDataset(thisReleaseRootDsName, thisReleaseRootDsMountPoint, "lz4"); err != nil {
+			fmt.Printf("Error creating dataset %s: %v\n", thisReleaseRootDsName, err)
+			return
+		}
+	}
+	
+	// Now extract download/$RELEASE/*.txz to releases/XX.X/root
+	downloadDsMountPoint := fmt.Sprintf("%s/download", ds.Mountpoint)
+	downloadDir := fmt.Sprintf("%s/%s", downloadDsMountPoint, release)
+	
+	d, err := os.Open(downloadDir)
+	defer d.Close()
+	if err != nil {
+		fmt.Printf("Can not read %s directory: %v\n", downloadDir, err)
+		return
+	}
+	files, err := d.Readdir(0)
+	if err != nil {
+		fmt.Printf("Can not browse %s directory: %v\n", downloadDir, err)
+		return
+	}
+	
+	// Extract every .txz files in FetchFiles
+	for _, fi := range files {
+		if false == fi.IsDir() {
+			if strings.HasSuffix(fi.Name(), ".txz") {
+				if isStringInArray(FetchFiles, fi.Name()) {
+					ar := fmt.Sprintf("%s/%s", downloadDir, fi.Name())
+					fmt.Printf("Extracting file %s to %s... ", ar, thisReleaseRootDsMountPoint)
+					// pure Go method, sorry this is so slow. Also I did not handle permissions in this
+	/*				f, err := os.Open(ar)
+					defer f.Close()
+					if err != nil {
+						fmt.Printf("Can not open %s: %v\n", ar, err)
+						return
+					}
+					// xz reader
+					r, err := xz.NewReader(f)
+					if err != nil {
+						fmt.Printf("Can not read %s: %v\n", ar, err)
+						return
+					}
+					// tar reader
+					tr := tar.NewReader(r)
+					// Iterate through the files in the archive.
+					for {
+						hdr, err := tr.Next()
+						if err == io.EOF {
+							// end of tar archive
+							break
+						}
+						if err != nil {
+							log.Fatal(err)
+						}
+						switch hdr.Typeflag {
+							case tar.TypeDir:
+								// create a directory
+								dest := fmt.Sprintf("%s/%s", thisReleaseRootDsMountPoint, hdr.Name)
+								// FIXME: Access rights?
+								err = os.MkdirAll(dest, 0777)
+								if err != nil {
+									log.Fatal(err)
+								}
+							case tar.TypeReg, tar.TypeRegA:
+								// write a file
+								dest := fmt.Sprintf("%s/%s", thisReleaseRootDsMountPoint, hdr.Name)
+								w, err := os.Create(dest)
+								defer w.Close()
+								if err != nil {
+									log.Fatal(err)
+								}
+								_, err = io.Copy(w, tr)
+								if err != nil {
+									log.Fatal(err)
+								}
+						}
+					}
+	*/
+					cmd := fmt.Sprintf("/usr/bin/tar xpf %s -C %s", ar, thisReleaseRootDsMountPoint)
+					out, err := executeCommand(cmd)
+					if err != nil && len(out) > 0 {
+						fmt.Printf("Error: %v: %s\n", err, out)
+					} else {
+						fmt.Printf("Done\n")
+					}
+				}
+			}
+		}
+	}
+}
+
+func fetchFile(proto, baseUrl, fileName, storeDir string, checksum []byte) error {
+	// Check storeDir exist
+	_, err := os.Stat(storeDir)
+	if os.IsNotExist(err) {
+		return fmt.Errorf("Directory does not exist: %s\n", storeDir)
+	}
+
+	url := fmt.Sprintf("%s/%s", baseUrl, fileName)
+	fmt.Printf("Fetching %s...", url)
+	
+	var body []byte
+	if strings.EqualFold(proto, "http") {
+		resp, err := http.Get(url)
+		if err != nil {
+			fmt.Printf(" Error\n")
+			return fmt.Errorf("Can not get %s: %v\n", url, err)
+		}
+		defer resp.Body.Close()
+		body, err = io.ReadAll(resp.Body)
+		if err != nil {
+			return fmt.Errorf("Can not read %s response body: %v\n", url, err)
+		}
+	} else if strings.EqualFold(proto, "file") {
+		url = strings.Replace(url, "file:", "", 1)
+		f, err := os.Open(url)
+		if err != nil {
+			return fmt.Errorf("Error accessing file %s", url)
+		}
+		defer f.Close()
+		body, err = io.ReadAll(f)
+		if err != nil {
+			return fmt.Errorf("Can not read file %s: %v\n", url, err)
+		}
+	}
+	
+	// Check integrity
+	if len(checksum) > 0 {
+		err = checkIntegrity(body, checksum)
+		if err != nil {
+			return fmt.Errorf("Error checking integrity")
+		}
+	}
+	
+	dest := fmt.Sprintf("%s/%s", storeDir, fileName)
+
+	f, err := os.Create(dest)       // creates a file at current directory
+	if err != nil {
+		return fmt.Errorf("Can not create file %s: %v\n", dest, err)
+	}
+	defer f.Close()
+	f.Write(body)
+	fmt.Printf(" Done\n")
+	return nil
+}
+
+
+func checkIntegrity(data []byte, checksum []byte) error {
+	sum := sha256.Sum256(data)
+	
+	if false == bytes.Equal(checksum[:],sum[:]) {
+		return fmt.Errorf("Invalid checksum: %x != %x", sum, checksum)
+	}
+	return nil
+}
+
+// Get checksum from manifest, for each file in fileList
+/* MANIFEST format:
+ * base-dbg.txz    a5b51f3d54686509e91ca9c30e9f1cd93dc757f25c643609b3c35e7119c0531d        1654    base_dbg        "Base system (Debugging)"       off
+ * base.txz        e85b256930a2fbc04b80334106afecba0f11e52e32ffa197a88d7319cf059840        26492   base    "Base system (MANDATORY)"       on
+ * kernel-dbg.txz  6b47a6cb83637af1f489aa8cdb802d9db936ea864887188cfc69d8075762214e        912     kernel_dbg      "Kernel (Debugging)"    on
+ */
+func buildFileChecksumFromManifest(manifest string, fileList []string) (map[string][]byte, error) {
+	var ckarr = make(map[string][]byte)
+
+	rm, err := os.Open(manifest)
+	if err != nil {
+		return ckarr, fmt.Errorf("Unable to open MANIFEST: %v", err)
+	}
+	fscan := bufio.NewScanner(rm)
+	fscan.Split(bufio.ScanLines)
+	
+	// For each MANIFEST line...
+	for fscan.Scan() {
+		fields := strings.Fields(fscan.Text())
+		fn := fields[0]
+		fck := fields[1]
+		hexSum, err := hex.DecodeString(fck)
+		if err != nil {
+			return ckarr, fmt.Errorf("Invalid value for checksum %s", fck)
+		}
+		// ... Find the corresponding file in fileList, then add to checksum array ckarr
+		for _, f := range fileList {
+			if strings.EqualFold(f, fn) {
+				ckarr[fn] = hexSum
+				break
+			}
+		}
+	}
+	
+	if len(ckarr) < len(fileList) {
+		return ckarr, fmt.Errorf("Missing file in MANIFEST")
+	}
+	return ckarr, nil
+}

cmd/init.go

@@ -0,0 +1,153 @@
+package cmd
+
+import (
+	"os"
+	"fmt"
+	"strings"
+
+	"github.com/spf13/viper"
+	log "github.com/sirupsen/logrus"
+)
+
+/********************************************************************************
+ * Initialize datastore(s) /iocage, /iocage/jails
+ * Put defaults.json,
+ * Update it with hostid, interfaces, and maybe other necessary fields
+ * Initialize bridge
+ *******************************************************************************/
+func InitGoCage(args []string) {
+	// Create datastores
+	for _, dstore := range viper.GetStringSlice("datastore") {
+		log.Debugf("Ranging over %v\n", dstore)
+		dset, err := zfsGetDatasetByMountpoint(dstore)
+		if err != nil && strings.HasSuffix(err.Error(), "No such file or directory\"") {
+			if len(gZPool) == 0 {
+				log.Errorf("Datastore mountpoint \"%s\" does not exist. Specify a pool if you want to create it.", dstore)
+				return
+			}
+			// Create dataset /iocage
+			rootDSName := fmt.Sprintf("%s%s", gZPool, dstore)
+			log.Debugf("Creating dataset %s mounted on %s\n", rootDSName, dstore)
+			if err = zfsCreateDataset(rootDSName, dstore, ""); err != nil {
+				log.Errorf("Error creating dataset %s: %v\n", rootDSName, err)
+				return
+			}
+			// Create /iocage/jail, releases, templates
+			for _, l := range []string{"jails","releases","templates"} {
+				cds := fmt.Sprintf("%s/%s", rootDSName, l)
+				cmp := fmt.Sprintf("%s/%s", dstore, l)
+				log.Debugf("Creating dataset %s mounted on %s\n", cds, cmp)
+				if err = zfsCreateDataset(cds, cmp, ""); err != nil {
+					log.Errorf("Error creating dataset %s: %v\n", cds, err)
+					return
+				}
+			}
+			
+			// Create /iocage/defaults.json
+			exists, err := doFileExist(fmt.Sprintf("%s/defaults.json", dstore))
+			if err != nil {
+				log.Errorf("Error checking defaults.json: %v\n", err)
+				return
+			}
+			if !exists {
+				if err = createDefaultsJson(dstore, gBridge); err != nil {
+					log.Errorf("%v\n", err)
+				}
+			}
+		} else if err != nil {
+			log.Errorf("Error checking datastore existence: %v\n", err)
+			return
+		} else {
+			log.Debugf("Datastore dataset exist: %s\n", dset)
+		}
+	}
+	// Check and create bridge
+	// FIXME: What if bridge name is invalid, as we already wrote it in defaults.json in dstore loop?
+	if len(gBridge) > 0 && len(gInterface) > 0 {
+		if err := initBridge(); err != nil {
+			log.Errorf("%v\n", err)
+		}
+	}
+}
+
+func createDefaultsJson(rootDirectory string, bridge string) error {
+	hostid, err := os.ReadFile("/etc/hostid")
+	if err != nil {
+		log.Fatalf("Unable to read /etc/hostid: %v\n", err)
+	}
+	json := strings.Replace(gDefaultsJson, "TO-BE-REPLACED-WITH-HOSTID", strings.Trim(string(hostid), "\n"), 1)
+	json = strings.Replace(json, "TO-BE-REPLACED-WITH-BRIDGE", bridge, 1)
+	if err := os.WriteFile(fmt.Sprintf("%s/defaults.json", rootDirectory), []byte(json), 0640); err != nil {
+		log.Fatal(err)
+	}
+	
+	return nil
+}
+
+func createInterface(iface string) error {
+	log.Debugf("creating interface \"%s\"\n", iface)
+	cmd := fmt.Sprintf("/sbin/ifconfig %s create", iface)
+	_, err := executeCommand(cmd)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func bringUpInterface(iface string) error {
+	log.Debugf("bringing up interface \"%s\"\n", iface)
+	cmd := fmt.Sprintf("/sbin/ifconfig %s up", iface)
+	_, err := executeCommand(cmd)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func addMemberToBridge(bridge string, iface string) error {
+	log.Debugf("adding member interface \"%s\" to bridge \"%s\"\n", iface, bridge)
+	cmd := fmt.Sprintf("/sbin/ifconfig %s addm %s", bridge, iface)
+	_, err := executeCommand(cmd)
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func initBridge() error {
+	hostInt, err := gJailHost.GetInterfaces()
+	if err != nil {
+		return fmt.Errorf("Error listing interfaces: %v\n", err)
+	}
+	if !isStringInArray(hostInt, gInterface) {
+		return fmt.Errorf("Interface not found: %s\n", gInterface)
+	}
+	if !isStringInArray(hostInt, gBridge) {
+		if err := createInterface(gBridge); err != nil {
+			return fmt.Errorf("Error creating bridge: %v\n", err)
+		}
+		if err := bringUpInterface(gBridge); err != nil {
+			return fmt.Errorf("Error bringing up bridge: %v\n", err)
+		}
+		log.Infof("bridge was created, but it won't persist reboot. Configure rc.conf to persist. See https://docs.freebsd.org/en/books/handbook/advanced-networking/#network-bridging\n")
+		log.Infof("It is strongly suggested you move interface %s IP to bridge %s\n", gInterface, gBridge)
+	}
+	// FIXME: Need to check if not already member
+	members, err := getBridgeMembers(gBridge)
+	if err != nil {
+		return fmt.Errorf("Error getting bridge members: %v\n", err)
+	}
+	
+	// Return if interface already member of the bridge
+	for _, m := range members {
+		log.Debugf("Bridge member: %s\n", m)
+		if strings.EqualFold(m, gInterface) {
+			return nil
+		}
+	}
+	if err := addMemberToBridge(gBridge, gInterface); err != nil {
+		return fmt.Errorf("Error adding interface to bridge: %v\n", err)
+	}
+
+	return nil
+}

cmd/jailhost.go

@@ -0,0 +1,223 @@
+package cmd
+
+import (
+	"fmt"
+	"net"
+	"regexp"
+	"strings"
+	"io/ioutil"
+	"golang.org/x/net/route"
+)
+
+var defaultRoute4 = [4]byte{0, 0, 0, 0}
+var defaultRoute6 = [16]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
+var local6 = [16]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}
+
+func Inet4AddrToString(ip4 route.Inet4Addr) string {
+	return fmt.Sprintf("%v.%v.%v.%v", ip4.IP[0], ip4.IP[1], ip4.IP[2], ip4.IP[3])
+}
+
+func Inet6AddrToString(ip6 route.Inet6Addr) string {
+	return fmt.Sprintf("%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x",
+		ip6.IP[0], ip6.IP[1], ip6.IP[2], ip6.IP[3], ip6.IP[4], ip6.IP[5], ip6.IP[6], ip6.IP[7],
+		ip6.IP[8], ip6.IP[9], ip6.IP[10], ip6.IP[11], ip6.IP[12], ip6.IP[13], ip6.IP[14], ip6.IP[15])
+}
+
+/*****************************************************************************
+* Initialize default_interface, default_gatway4, default_gateway6
+*****************************************************************************/
+func (jh *JailHost) InitNetworkProperties() {
+	rib, _ := route.FetchRIB(0, route.RIBTypeRoute, 0)
+	messages, err := route.ParseRIB(route.RIBTypeRoute, rib)
+	if err != nil {
+		panic(err)
+	}
+
+	for _, message := range messages {
+		route_message := message.(*route.RouteMessage)
+		addresses := route_message.Addrs
+		card_index := route_message.Index
+
+		if addresses[0].Family() == 2 {
+			var destination4, gateway4 *route.Inet4Addr
+			ok := false
+
+			if destination4, ok = addresses[0].(*route.Inet4Addr); !ok {
+				continue
+			}
+
+			if gateway4, ok = addresses[1].(*route.Inet4Addr); !ok {
+				continue
+			}
+
+			if destination4 == nil || gateway4 == nil {
+				continue
+			}
+
+			if destination4.IP == defaultRoute4 {
+				card, _ := net.InterfaceByIndex(card_index)
+				//fmt.Printf("Default IPv4 gateway is %v on card %s\n", Inet4AddrToString(*gateway4), card.Name)
+				jh.default_interface = card.Name
+				jh.default_gateway4 = Inet4AddrToString(*gateway4)
+			}
+		} else if addresses[0].Family() == 28 {
+			var destination6, gateway6 *route.Inet6Addr
+			ok := false
+
+			if destination6, ok = addresses[0].(*route.Inet6Addr); !ok {
+				continue
+			}
+
+			if gateway6, ok = addresses[1].(*route.Inet6Addr); !ok {
+				continue
+			}
+
+			if destination6 == nil || gateway6 == nil {
+				continue
+			}
+
+			if destination6.IP == defaultRoute6 && gateway6.IP != local6 {
+				card, _ := net.InterfaceByIndex(card_index)
+				//fmt.Printf("Default IPv6 gateway is %v on card %s\n", Inet6AddrToString(*gateway6), card.Name)
+				jh.default_interface = card.Name
+				jh.default_gateway6 = Inet6AddrToString(*gateway6)
+			}
+		}
+	}
+}
+
+func (jh *JailHost) GetDefaultInterface() string {
+	if len(jh.default_interface) == 0 {
+		jh.InitNetworkProperties()
+	}
+	return jh.default_interface
+}
+
+func (jh *JailHost) GetDefaultGateway4() string {
+	if len(jh.default_gateway4) == 0 {
+		jh.InitNetworkProperties()
+	}
+	return jh.default_gateway4
+}
+
+func (jh *JailHost) GetDefaultGateway6() string {
+	if len(jh.default_gateway6) == 0 {
+		jh.InitNetworkProperties()
+	}
+	return jh.default_gateway6
+}
+
+/*****************************************************************************
+ * Get all network interfaces
+ ****************************************************************************/
+func (jh *JailHost) GetInterfaces() ([]string, error) {
+	var names []string
+	
+	interfaces, err := net.Interfaces()
+	if err != nil {
+		return names, fmt.Errorf("Error listing network interfaces: %v", err)
+	}
+	for _, n := range interfaces {
+		names = append(names, n.Name)
+	}
+	
+	return names, nil
+}
+
+func (jh *JailHost) GetBridgeMTU(bridgeName string) (int, error) {
+	bridge, err := net.InterfaceByName(bridgeName)
+	if err != nil {
+		return 0, err
+	}
+	
+	return bridge.MTU, nil
+}
+
+/*****************************************************************************
+ * Get all IPv4 currently in use on host
+ ****************************************************************************/
+func getHostInUseIPv4() ([]string, error) {
+	var ips []string
+
+	re := regexp.MustCompile(ifconfigipv4re)
+
+	out, err := executeCommand("/sbin/ifconfig")
+	if err != nil {
+		return ips, fmt.Errorf("Error executing \"/sbin/ifconfig\": %s", err)
+	}
+
+	for _, line := range strings.Split(out, "\n") {
+		if re.MatchString(line) {
+			ips = append(ips, re.FindStringSubmatch(line)[1])
+		}
+	}
+
+	return ips, nil
+}
+
+func getHostname() (string, error) {
+	out, err := executeCommand("/bin/hostname")
+	if err != nil {
+		return "", fmt.Errorf("Error executing \"/bin/hostname\": %v", err)
+	}
+	
+	return strings.Split(out, "\n")[0], nil
+}
+
+func getHostId() (string, error) {
+	var content []byte
+	var err     error
+	// return empty string if file does not exist
+	if content, err = ioutil.ReadFile("/etc/hostid"); err != nil {
+		if strings.HasSuffix(err.Error(), "no such file or directory") {
+			return "", nil
+		}
+		return "", err
+	}
+	
+	return strings.Split(string(content), "\n")[0], nil
+}
+
+func getArch() (string, error) {
+	out, err := executeCommand("/usr/bin/uname -p")
+	if err != nil {
+		return "", fmt.Errorf("Error executing \"/usr/bin/uname -p\": %v", err)
+	}
+	
+	return strings.Split(out, "\n")[0], nil
+}
+
+func getFreeBSDVersion() (FreeBSDVersion, error) {
+	var version FreeBSDVersion
+	
+	out, err := executeCommand("/bin/freebsd-version")
+	if err != nil {
+		return version, fmt.Errorf("Error executing \"/bin/freebsd-version\": %v", err)
+	}
+	
+	version, err = freebsdVersionToStruct(out)
+	if err != nil {
+		return version, err
+	}
+
+	return version, nil
+}
+
+func NewJailHost() (JailHost, error) {
+	var jh  JailHost
+	var err error
+	
+	if jh.hostname, err = getHostname(); err != nil {
+		return jh, err
+	}
+	if jh.arch, err = getArch(); err != nil {
+		return jh, err
+	}
+	if jh.hostid, err = getHostId(); err != nil {
+		return jh, err
+	}
+	if jh.version, err = getFreeBSDVersion(); err != nil {
+		return jh, err
+	}
+	return jh, nil
+}

cmd/list.go

@@ -1,315 +1,121 @@
 package cmd
 
 import (
-	"os"
+	"encoding/json"
 	"fmt"
+	"gocage/jail"
+	"io/ioutil"
 	"log"
-	"errors"
+	"os"
 	"reflect"
 	"strings"
-	"io/ioutil"
-	"gocage/jail"
-	"encoding/json"
-	"github.com/spf13/viper"
 )
 
-
-// Recurse into structure, returning reflect.Value of wanted field.
-//  Nested fields are named with a dot (ex "MyStruct.MyField")
-func getStructFieldValue(parentStruct interface{}, fieldName string) (reflect.Value, string, error) {
-	v := reflect.ValueOf(parentStruct)
-
-	if v.Kind() == reflect.Ptr {
-		v = v.Elem()
-	}
-
-	if false {
-		for i := 0 ; i < v.NumField(); i++ {
-			f := v.Field(i)
-			if f.Kind() == reflect.String {
-				fmt.Printf("%v=%v\n", v.Type().Field(i).Name, f.Interface())
-			}
-		}
-	}
-
-	if strings.Contains(fieldName, ".") {
-		fs := strings.Split(fieldName, ".")
-		f := v.FieldByName(fs[0])
-		if f.Kind() == reflect.Struct {
-			return getStructFieldValue(f.Interface(), strings.Join(fs[1:], "."))
-		} else {
-			log.Fatalln(fmt.Sprintf("%s is not a struct: %s\n", fs[0], f.Kind().String()))
-		}
-	} else {
-		f := v.FieldByName(fieldName)
-		if f.IsValid() {
-			return f, fieldName, nil
-		} else {
-			return v, fieldName, errors.New(fmt.Sprintf("Field not found: %s", fieldName))
-		}
-	}
-
-	return v, fieldName, nil
-}
-
-
-// TODO : Replace by getStructFieldValue
-// Recurse into structure, returning reflect.Value of wanted field.
-//  Nested fields are named with a dot (ex "MyStruct.MyField")
-func getStructField(parentStruct interface{}, fieldName string) (reflect.Value, string) {
-	v := reflect.ValueOf(parentStruct)
-
-	if v.Kind() == reflect.Ptr {
-		v = v.Elem()
-	}
-
-	if false {
-		for i := 0 ; i < v.NumField(); i++ {
-			f := v.Field(i)
-			if f.Kind() == reflect.String {
-				fmt.Printf("%v=%v\n", v.Type().Field(i).Name, f.Interface())
-			}
-		}
-	}
-
-	if strings.Contains(fieldName, ".") {
-		fs := strings.Split(fieldName, ".")
-		f := v.FieldByName(fs[0])
-		if f.Kind() == reflect.Struct {
-			return getStructField(f.Interface(), strings.Join(fs[1:], "."))
-		} else {
-			log.Fatalln(fmt.Sprintf("%s is not a struct: %s\n", fs[0], f.Kind().String()))
-		}
-	}
-
-	return v, fieldName
-}
-
-
-/* Pretty display of jails field
-   Fields to show are given in a string array parameter
-   Ex. : displayJails(["Name", "JID", "RootPath"])
-*/
-func displayStructFields(jails []Jail, valsToDisplay []string) {
-	/* A line is defined by :
-		Nr of fields
-		For each field :
-			Its name
-			Its max length
-			Its value
-	*/
-	type Field struct {
-		Name	string
-		MaxLen	int
-		Value	string
-	}
-
-	type Line []Field
-
-	type Output []Line
-	var out Output
-
-	//fmt.Printf("%d fields in a %d items structure\n", len(valsToDisplay), len(jails))
-
-	// Browse structure to get max length and values
-	for _, j := range jails {
-		// Have to use a pointer, else reflect.Value.Elem() will panic : https://pkg.go.dev/reflect#Value.Elem
-		tj := &j
-		
-		line := make([]Field, len(valsToDisplay))
-		for i, f := range valsToDisplay {
-			a, f := getStructField(tj, f)
-
-			field := Field {
-				Name: f,
-			}
-			if a.FieldByName(f).IsValid() {
-				// For now this just contains this item length, will adjust later
-				//  We need to divide length by number of items in string fields (because we can have more than 1 ip4_addr)
-				if reflect.TypeOf(a.FieldByName(f).Interface()).Kind() == reflect.String {
-					itnr := len(strings.Split(string(a.FieldByName(f).Interface().(string)), ","))
-					field.MaxLen = len(fmt.Sprintf("%v", a.FieldByName(f).Interface())) / itnr
-				} else {
-					field.MaxLen = len(fmt.Sprintf("%v", a.FieldByName(f).Interface()))
-				}
-				field.Value = fmt.Sprintf("%v", a.FieldByName(f).Interface())
-			} else {
-				fmt.Printf("Invalid field name: %s\n", f)
-			}
-			line[i] = field
-			
-		}
-		out = append(out, line)
-	}
-
-	if len(out) == 0 {
-		fmt.Printf("Nothing to see here!\n")
+/********************************************************************************
+ * List all properties a jail have, with their internal name
+ * Only print properties name. To get name & values, use GetJailProperties()
+ *******************************************************************************/
+func ListJailsProps(args []string) {
+	var conf Jail
+	var result []string
+	// Mandatory constructor to init default values
+	jailconf, err := NewJailConfig()
+	if err != nil {
+		fmt.Printf("Error allocating JailConfig: %s\n", err.Error())
 		return
 	}
 
-	// Get real maximum length
-	maxlen := make([]int, len(valsToDisplay))
-	for i := 0; i< len(valsToDisplay); i++ {
-		maxlen[i] = len(valsToDisplay[i])
-	}
-	for _, l := range out {
-		for i, f := range l {
-			if f.MaxLen > maxlen[i] {
-				maxlen[i] = f.MaxLen
-			}
-		}
-	}
+	conf.Config = jailconf
 
-	// Align maxlen to the real maximum length
-	for io, l := range out {
-		for ii, _ := range l {
-			// We need to access slice by index if we want to modify content
-			out[io][ii].MaxLen = maxlen[ii]
-		}
-	}
+	result = getStructFieldNames(conf, result, "")
 
-	totalLen := 0
-	// For each field, add separator and 2 blank spaces 
-	for _, f := range out[0] {
-		totalLen += f.MaxLen + 3
+	for _, f := range result {
+		fmt.Printf("%s\n", f)
 	}
-	// Then add starting delimiter 
-	totalLen += 1
-
-	Debug := false
-	if Debug == true {
-		for _, f := range out[0] {
-			fmt.Printf("%s max length : %d\n", f.Name, f.MaxLen)
-		}
-	}
-
-	// Lets draw things on the screen!
-	// First, headers: 1st separator line
-	for i, f := range out[0] {
-		if i == 0 {	fmt.Printf("+")	}
-		for i := 0 ; i < f.MaxLen+2 ; i++ {	fmt.Printf("=")	}
-		fmt.Printf("+")
-	}
-	fmt.Printf("\n")
-
-
-	//  Column names
-	for i, f := range out[0] {
-		if i == 0 {
-			fmt.Printf("|")
-		}
-		/* Use vlsToDisplay to get real name (with "Config.")
-		fmt.Printf(" %s", f.Name)
-		for i := len(f.Name)+1 ; i < f.MaxLen+1 ; i++ {  */
-		fmt.Printf(" %s", valsToDisplay[i])
-		for i := len(valsToDisplay[i])+1 ; i < f.MaxLen+1 ; i++ {
-			fmt.Printf(" ")
-		}
-		fmt.Printf(" |")
-	}
-	//  Finally separator line
-	fmt.Printf("\n")
-	for i, f := range out[0] {
-		if i == 0 {	fmt.Printf("+")	}
-		for i := 0 ; i < f.MaxLen+2 ; i++ {	fmt.Printf("=")	}
-		fmt.Printf("+")
-	}
-	fmt.Printf("\n")
-
-	// Then display data
-	//  Loop through lines
-	for _, l := range out {
-		
-		// Loop through fields
-		//  In case we need to add a line for a 2nd IP, or whatever object
-		var supplines = make(map[string]string)
-		for i, f := range l {
-			if i == 0 {
-				fmt.Printf("|")
-			}
-			// Special cases of value displaying
-			if f.Name == "JID" && f.Value == "0" {
-				fmt.Printf("  ")
-			} else if f.Name == "Ip4_addr" {
-				ia := strings.Split(f.Value, ",")
-				// If we have more than 1 value we need to finish this line, and store value for writing at the end of line loop
-				for i, inter := range ia {
-					if i > 0 {
-						supplines[f.Name] = inter
-					} else {
-						fmt.Printf(" %s", inter)
-					}
-				}
-				//fmt.Printf(" %s", strings.Split(strings.Split(f.Value, "|")[1], "/")[0])
-			} else {
-				fmt.Printf(" %s", f.Value)
-			}
-			// Complete with spaces to the max length
-			for i := len(f.Value)+1 ; i < f.MaxLen+1 ; i++ {
-				fmt.Printf(" ")
-			}
-			fmt.Printf(" |")
-		}
-		// Draw supplementary lines
-		if len(supplines) > 0 {
-			for i, f := range l {
-				if i == 0 {
-					fmt.Printf("\n|")
-				}
-				// Overwrite value in this scope
-				v, exists := supplines[f.Name]
-				if exists {
-					fmt.Printf(" %s", v)
-				} else {
-					// 1st option : Do not redisplay precedent line values
-					fmt.Printf(" ")
-					// 2nd option : Redisplay precedenet line values
-					//fmt.Printf(" %s", f.Value)
-				}
-				// Complete with spaces to the max length
-				for i := len(v)+1 ; i < f.MaxLen+1 ; i++ {
-					fmt.Printf(" ")
-				}
-				fmt.Printf(" |")
-			}
-		}
-		// Draw line separator between jails
-		if !gNoLineSep {
-			fmt.Printf("\n")
-			for i, f := range out[0] {
-				if i == 0 {	fmt.Printf("+")	}
-				for i := 0 ; i < f.MaxLen+2 ; i++ {	fmt.Printf("-")	}
-				fmt.Printf("+")
-			}
-		}
-		fmt.Printf("\n")
-	}
-	if gNoLineSep {
-		for i, f := range out[0] {
-			if i == 0 {	fmt.Printf("+")	}
-			for i := 0 ; i < f.MaxLen+2 ; i++ {	fmt.Printf("-")	}
-			fmt.Printf("+")
-		}
-	}
-	
-	fmt.Printf("\n")
 }
 
-
-/* Get Jails from datastores. Store config and running metadata into gJails global var */
+/********************************************************************************
+ * Get Jails from datastores. Store config and running metadata
+ *   into gJails global var
+ *******************************************************************************/
 func ListJails(args []string, display bool) {
-	fields := strings.Split(gDisplayColumns, ",")
-
-	for _, d := range viper.GetStringSlice("datastore") {
-		listJailsFromDatastore(d)
+	type uniqueJailName struct {
+		jail       string
+		unique     bool
+		uniqueName bool
 	}
+	var nameChecked []*uniqueJailName
+	var curCheck *uniqueJailName
+	var found bool
+	var skip bool
+	
+	for _, ds := range gDatastores {
+		listJailsFromDatastore(ds, args, display)
+	}
+
+	// Only when displaying jails, we accept to process multiple same name jails
+	if false == display {
+		for _, j := range gJails {
+			// If already checked and was using distinctive name, skip this occurence
+			for i, n := range nameChecked {
+				if strings.EqualFold(n.jail, j.Name) {
+					found = true
+					if false == n.unique && true == n.uniqueName {
+						skip = true
+					} else {
+						curCheck = nameChecked[i]
+					}
+					break
+				}
+			}
+
+			if true == skip {
+				continue
+			}
+
+			// Initialize if not found in nameChecked
+			if false == found {
+				curCheck = &uniqueJailName{jail: j.Name,
+									unique: true,
+									uniqueName: false}
+			} else {
+				found = false
+			}
+
+			if countOfJailsWithThisName(j.Name) > 1 {
+				//fmt.Printf("DEBUG: Jail %s exist multiple times, now checking if specified with full name\n", j.Name)
+				curCheck.unique = false
+
+				for _, a := range args {
+					//fmt.Printf("DEBUG: comparing %s/%s with %s\n", j.Datastore, j.Name, a)
+					if strings.EqualFold(a, fmt.Sprintf("%s/%s", j.Datastore, j.Name)) {
+						//fmt.Printf("DEBUG: Was specified with full name, GG!\n")
+						curCheck.uniqueName = true
+					}
+				}
+			}
+			nameChecked = append(nameChecked, curCheck)
+		}
+
+		// Now check
+		for _, a := range args {
+			for _, n := range nameChecked {
+				if strings.EqualFold(n.jail, a) && false == n.unique && false == n.uniqueName {
+					fmt.Printf("There is more than one jail named \"%s\", please use datastore/jail format\n", n.jail)
+					os.Exit(1)
+				}
+			}
+		}
+	}
+
+	fields := strings.Split(gDisplayJColumns, ",")
 
 	// This is the structure we will filter, then display
 	var jails []Jail
 
-	/************************************************************************************
+	/***************************************************************
 	/   Filter jails with "filter" options
-	/***********************************************************************************/
+	/**************************************************************/
 	if len(gFilterJails) > 0 && gFilterJails != "none" {
 		flts := make(map[string]string)
 		for _, flt := range strings.Split(gFilterJails, ",") {
@@ -340,40 +146,38 @@ func ListJails(args []string, display bool) {
 		jails = gJails
 	}
 
-
-	/************************************************************************************
+	/***************************************************************
 	/   Filter jails by names given on command line
-	/***********************************************************************************/
+	/**************************************************************/
 	if len(args) > 0 {
 		var js []Jail
 		for _, a := range args {
 			for _, j := range jails {
 				if j.Name == a {
 					js = append(js, j)
-					break
 				}
 			}
 		}
 		jails = js
 	}
 
-	/************************************************************************************
+	/***************************************************************
 	/   Sort jails
 	/     We support 3 sort criteria max
-	/***********************************************************************************/
-	if len(gSortFields) > 0 {
-		js := initSortStruct()
+	/**************************************************************/
+	if len(gSortJailFields) > 0 && gSortJailFields != "none" {
+		js := initJailSortStruct()
 
 		// The way we manage criteria quantity is not very elegant...
-		var fct1, fct2, fct3 reflect.Value
-		for i, c := range strings.Split(gSortFields, ",") {
+		var fct1, fct2, fct3 *reflect.Value
+		for i, c := range strings.Split(gSortJailFields, ",") {
 			var fctName string
 			if strings.HasPrefix(c, "-") {
 				fctName = fmt.Sprintf("%sDec", strings.Replace(c, "-", "", 1))
-			} else {	// Par defaut (pas de prefix +/-) on considere un tri incremental	
+			} else { // Par defaut (pas de prefix +/-) on considere un tri incremental
 				fctName = fmt.Sprintf("%sInc", strings.Replace(c, "+", "", 1))
 			}
-			
+
 			// Get function by its name
 			fct, _, err := getStructFieldValue(js, fctName)
 			if err != nil {
@@ -381,82 +185,109 @@ func ListJails(args []string, display bool) {
 				fmt.Printf("ERROR getting JailSort struct field %s. Please check the field name: %s\n", fctName, fieldName)
 				return
 			}
-			switch i+1 {
-				case 1:	fct1 = fct
-				case 2: fct2 = fct
-				case 3: fct3 = fct
+			switch i + 1 {
+			case 1:
+				fct1 = fct
+			case 2:
+				fct2 = fct
+			case 3:
+				fct3 = fct
 			}
 		}
 
-		switch len(strings.Split(gSortFields, ",")) {
-			case 1:
-				OrderedBy(fct1.Interface().(lessFunc)).Sort(jails)
-			case 2:
-				OrderedBy(fct1.Interface().(lessFunc), fct2.Interface().(lessFunc)).Sort(jails)
-			case 3:
-				OrderedBy(fct1.Interface().(lessFunc), fct2.Interface().(lessFunc), fct3.Interface().(lessFunc)).Sort(jails)
+		switch len(strings.Split(gSortJailFields, ",")) {
+		case 1:
+			JailsOrderedBy(fct1.Interface().(jailLessFunc)).Sort(jails)
+		case 2:
+			JailsOrderedBy(fct1.Interface().(jailLessFunc), fct2.Interface().(jailLessFunc)).Sort(jails)
+		case 3:
+			JailsOrderedBy(fct1.Interface().(jailLessFunc), fct2.Interface().(jailLessFunc), fct3.Interface().(jailLessFunc)).Sort(jails)
 		}
 	}
 
-	/************************************************************************************
+	/***************************************************************
 	/   Finally, display jails
-	/***********************************************************************************/
+	/**************************************************************/
 	if display {
-		displayStructFields(jails, fields)
+		displayJailsFields(jails, fields)
 	}
 }
 
-
-func listJailsFromDatastore(datastore string) {
-	fileInfo, err := os.Stat(datastore)
-	if err != nil { log.Fatalln(fmt.Sprintf("Unable to access %s, check path and/or rights", datastore)) }
-	if fileInfo.IsDir() == false { log.Fatalln(fmt.Sprintf("%s is not a directory", datastore)) }
+func listJailsFromDatastore(ds Datastore, args []string, accept_multiple bool) {
+	
+	fileInfo, err := os.Stat(ds.Mountpoint)
+	if err != nil {
+		log.Fatalln(fmt.Sprintf("Unable to access %s, check path and/or rights", ds.Mountpoint))
+	}
+	if fileInfo.IsDir() == false {
+		log.Fatalln(fmt.Sprintf("%s is not a directory", ds.Mountpoint))
+	}
 
 	// A datastore have to contain a "jails" directory
-	jailsDir := fmt.Sprintf("%s/jails", datastore)
+	jailsDir := fmt.Sprintf("%s/jails", ds.Mountpoint)
 	fileInfo, err = os.Stat(jailsDir)
-	if err != nil { log.Fatalln(fmt.Sprintf("Unable to access %s, check path and/or rights", jailsDir)) }
-	if fileInfo.IsDir() == false { log.Fatalln(fmt.Sprintf("%s is not a directory", jailsDir)) }
+	if err != nil {
+		log.Fatalln(fmt.Sprintf("Unable to access %s, check path and/or rights", jailsDir))
+	}
+	if fileInfo.IsDir() == false {
+		log.Fatalln(fmt.Sprintf("%s is not a directory", jailsDir))
+	}
 
-	listJailsFromDirectory(jailsDir)
+	listJailsFromDirectory(jailsDir, ds.Name)
 }
 
-
-func listJailsFromDirectory(dir string) ([]Jail) {
+func listJailsFromDirectory(dir string, dsname string) ([]Jail, error) {
 	files, err := ioutil.ReadDir(dir)
-	if err != nil { log.Fatalln(fmt.Sprintf("Unable to browse %s, check path and/or rights", dir)) }
+	if err != nil {
+		log.Fatalln(fmt.Sprintf("Unable to browse %s, check path and/or rights", dir))
+	}
 
 	for _, fi := range files {
 		if fi.IsDir() == true {
 
-		// 1. Get conf from config.json
+			// 1. Get conf from config.json
 			jailConfPath := fmt.Sprintf("%s/%s/%s", dir, fi.Name(), "config.json")
 			jailConf, err := getJailConfig(jailConfPath)
-			if err != nil { log.Println("ERROR reading jail config for %s", jailConfPath) }
-	
-		// 2. Build jail object from config
-			jailRootPath := fmt.Sprintf("%s/%s/%s", dir, fi.Name(), "root")
-			j := Jail{
-				Name: jailConf.Host_hostname,
-				Config: jailConf,
-				ConfigPath: jailConfPath,
-				RootPath: jailRootPath,
-				Running: false,
+			if err != nil {
+				fmt.Printf("ERROR reading jail config from %s\n", jailConfPath)
 			}
 
-		// 3. Add current running informations
+			// 2. Build jail object from config
+			jailRootPath := fmt.Sprintf("%s/%s/%s", dir, fi.Name(), "root")
+			j := Jail{
+				Name:       jailConf.Host_hostuuid,
+				Config:     jailConf,
+				ConfigPath: jailConfPath,
+				Datastore:  dsname,
+				RootPath:   jailRootPath,
+				Running:    false,
+			}
+
+			// 3. Add current running informations
 			rjails, err := jail.GetJails()
-			if err != nil { log.Fatalln("Unable to list running jails") }
+			if err != nil {
+				log.Fatalln("Unable to list running jails")
+			}
 			for _, rj := range rjails {
 				if rj.Path == j.RootPath {
 					j.JID = rj.Jid
 					j.Running = true
+					// FIXME ??? Shouldn't be ioc-$Name ?
 					j.InternalName = rj.Name
+					j.Devfs_ruleset = rj.Devfs_ruleset
+					// Update release
+					r, err := getVersion(&j)
+					if err != nil {
+						fmt.Printf("ERROR getting jail %s version: %s\n", j.Name, err.Error())
+					} else {
+						j.Config.Release = r
+					}
 					break
 				}
 			}
-	
-/* This op take some 600ms for ~40 jails :^( */
+
+			// 4. Get Zpool
+			/* This op take some 600ms for ~40 jails :^( */
 			out, err := executeCommand(fmt.Sprintf("zfs list -H -o name %s", j.RootPath))
 			if err != nil {
 				fmt.Printf("ERROR getting dataset from %s: %s\n", j.RootPath, err.Error())
@@ -464,21 +295,40 @@ func listJailsFromDirectory(dir string) ([]Jail) {
 				j.Zpool = strings.Split(strings.TrimSuffix(out, "\n"), "/")[0]
 			}
 
+			// Check if jail with the same name already exist on another DS
+			for _, jj := range gJails {
+				if strings.EqualFold(jj.Name, j.Name) {
+					/*fmt.Printf("Warning: A jail with name %s exist on datastore %s, use full name to interact!\n", j.Name, jj.Datastore)
+					fmt.Printf("(Ex: gocage start %s/%s)\n", jj.Datastore, j.Name)*/
+					// Add Datastore  to avoid confusion
+					if false == strings.Contains(gDisplayJColumns, "Datastore") {
+						gDisplayJColumns += ",Datastore"
+					}
+				}
+			}
+			
 			gJails = append(gJails, j)
 		}
 	}
 
-	return gJails
+	return gJails, nil
 }
 
-
 func getJailConfig(jailConfigPath string) (JailConfig, error) {
 	content, err := ioutil.ReadFile(jailConfigPath)
-	if err != nil { log.Fatalln(fmt.Sprintf("Unable to read %s, check path and/or rights", jailConfigPath)) }
+	if err != nil {
+		log.Fatalln(fmt.Sprintf("Unable to read %s, check path and/or rights", jailConfigPath))
+	}
 
-	var jc JailConfig
+	// Mandatory constructor to init default values
+	jc, err := NewJailConfig()
+	if err != nil {
+		return jc, err
+	}
 	err = json.Unmarshal([]byte(content), &jc)
-	if err != nil { log.Fatalln(fmt.Sprintf("Error occured during unmarshaling %s: %s", jailConfigPath, err.Error())) }
+	if err != nil {
+		log.Fatalln(fmt.Sprintf("Error occured during unmarshaling %s: %s", jailConfigPath, err.Error()))
+	}
 
 	return jc, err
 }

cmd/migrate.go

@@ -0,0 +1,224 @@
+package cmd
+
+import (
+	"os"
+	"fmt"
+	"bufio"
+	"errors"
+	"strings"
+)
+
+const (
+	// 1MB
+	BUFFER_SIZE = 512
+)
+
+/********************************************************************************
+ * Migrate jail to another zfs pool
+ *******************************************************************************/
+func MigrateJail(args []string) {
+	var jailNames []string
+
+	if len(args) > 0 {
+		for _, a := range args {
+			jailNames = append(jailNames, a)
+		}
+	}
+
+	for _, jn := range jailNames {
+		cj, err := getJailFromArray(jn, []string{""}, gJails)
+		if cj == nil {
+			fmt.Printf("Error getting jail %s: Not found\n", jn)
+			return
+		}
+		
+		// Check if destination datastore exist & get current DS
+		destDS, err := getDatastoreFromArray(gMigrateDestDatastore, gDatastores)
+		if err != nil {
+			fmt.Printf("Error getting datastore \"%s\": %v\n", gMigrateDestDatastore, err)
+			return
+		}
+		curDS, err := getDatastoreFromArray(cj.Datastore, gDatastores)
+		if err != nil {
+			fmt.Printf("Error getting datastore \"%s\": %v\n", gMigrateDestDatastore, err)
+			return
+		}
+		
+		if cj.Running == true && gYesToAll == false {
+			fmt.Printf("WARNING: Jail %s is running\n", cj.Name)
+			fmt.Printf("Migration will stop it for data sync before starting on the new pool. You will be prompted for shutdown.\n")
+			fmt.Printf("Continue? (y/n) ")
+			scanr := bufio.NewScanner(os.Stdin)
+			scanr.Scan()
+			if false == strings.EqualFold(scanr.Text(), "y") {
+				fmt.Printf("Migration aborted\n")
+				return
+			}
+		}
+
+		/* TODO : Check dest pool (gMigrateDestDatastore) existence
+				zfs snapshot /iocage/jails/$jail@gocage_mig_first_snap
+				zfs snapshot /iocage/jails/$jail/root@gocage_mig_first_snap
+				zfs send jail@gocage_mig_first_snap | zfs receive destpool/jails/jail_name
+				zfs send jail/root@gocage_mig_first_snap | zfs receive destpool/jails/jail_name/root
+				shutdown jail if needed
+				if jail was shutdown
+					zfs snapshot /iocage/jails/$jail@gocage_mig_last_snap to get last data
+					zfs send -i jail@gocage_mig_first_snap jail/root@gocage_mig_last_snap | zfs receive -F destpool/jails/jail_name
+					start jail on new dest
+					zfs destroy destpool/jails/jail_name@gocage_mig_first_snap
+					zfs destroy destpool/jails/jail_name/root@gocage_mig_first_snap
+		*/
+
+		// Snapshot config
+		dsconf := strings.Join([]string{curDS.ZFSDataset, "jails", jn}, "/")
+		fmt.Printf("Snapshot %s: ", dsconf)
+		if err = zfsSnapshot(dsconf, "gocage_mig_init"); err != nil {
+			fmt.Printf("Error: %v\n", err)
+			return
+		}
+		fmt.Printf("Done\n")
+		
+		// Snapshot filesystem
+		dsdata := strings.Join([]string{curDS.ZFSDataset, "jails", jn, "root"}, "/")
+		fmt.Printf("Snapshot %s: ", dsdata)
+		if err := zfsSnapshot(dsdata, "gocage_mig_init"); err != nil {
+			fmt.Printf("Error: %v\n", err)
+			return
+		}
+		fmt.Printf("Done\n")
+		
+		
+		// TODO: Browse jailed ZFS dataset to migrate them (see poudriere for example)
+		// zfs send -R src/poudriere@init | zfs receive -Fu dest/poudriere
+		
+		dsconfdest := strings.Join([]string{destDS.ZFSDataset, "jails", jn}, "/")
+		fmt.Printf("Migrate jail config dataset to %s: ", dsconfdest)
+		if err := zfsCopy(fmt.Sprintf("%s@gocage_mig_init", dsconf), dsconfdest); err != nil {
+			fmt.Printf("Error: %v\n", err)
+		}
+		fmt.Printf("Done\n")
+		
+		dsdatadest := strings.Join([]string{destDS.ZFSDataset, "jails", jn, "root"}, "/")
+		fmt.Printf("Migrate jail filesystem dataset to %s: ", dsdatadest)
+		if err := zfsCopy(fmt.Sprintf("%s@gocage_mig_init", dsdata), dsdatadest); err != nil {
+			fmt.Printf("Error: %v\n", err)
+		}
+		fmt.Printf("Done\n")
+		
+		// Running jail needs a last snapshot for an incremental send/recv after shutting down.
+		if cj.Running == true {
+			fmt.Printf("Shutdown jail %s for last data sync, this could take some time.\n", cj.Name)
+			if gYesToAll == false {
+				fmt.Printf("Continue? (y/n) ")
+				scanr := bufio.NewScanner(os.Stdin)
+				scanr.Scan()
+				if false == strings.EqualFold(scanr.Text(), "y") {
+					fmt.Printf("Migration aborted. Now cleaning destination pool.\n")
+					if err := CleanMigrateMess([]string{cj.Name}); err != nil {
+						fmt.Printf("Error: %v\n", err)
+					}
+					// TODO : Remove destination datasets, or handle the cas at the beginning of current function
+					// (when snapshot already exist on source and dest)
+					return
+				}
+			}
+			
+			StopJail([]string{cj.Name})
+			
+			fmt.Printf("Snapshot %s: ", dsconf)
+			if err = zfsSnapshot(dsconf, "gocage_mig_last_sync"); err != nil {
+				fmt.Printf("Error: %v\n", err)
+				return
+			}
+			fmt.Printf("Done\n")
+			fmt.Printf("Snapshot %s: ", dsdata)
+			if err := zfsSnapshot(dsdata, "gocage_mig_last_sync"); err != nil {
+				fmt.Printf("Error: %v\n", err)
+				return
+			}
+			fmt.Printf("Done\n")
+			
+			fmt.Printf("Synchronize jail config to %s: ", dsconfdest)
+			if err := zfsCopyIncremental(fmt.Sprintf("%s@gocage_mig_init", dsconf), 
+										fmt.Sprintf("%s@gocage_mig_last_sync", dsconf),  
+										 dsconfdest); err != nil {
+				fmt.Printf("Error: %v\n", err)
+				return
+			}
+			fmt.Printf("Done\n")
+			
+			fmt.Printf("Synchronize jail filesystem dataset to %s: ", dsdatadest)
+			if err := zfsCopyIncremental(fmt.Sprintf("%s@gocage_mig_init", dsdata), 
+										fmt.Sprintf("%s@gocage_mig_last_sync", dsdata),  
+										dsdatadest); err != nil {
+				return
+				fmt.Printf("Error: %v\n", err)
+			}
+			fmt.Printf("Done\n")
+			
+			// TODO : Start jail on new datastore (! Currently ListJails won't support 2 jails with same name !)
+			
+			// TODO : zfs destroy destpool/jails/jail_name@gocage_mig_first_snap
+			// TODO : zfs destroy destpool/jails/jail_name/root@gocage_mig_first_snap
+			// TODO : zfs destroy -r srcpool/iocage/jails/$jail
+		}
+	}
+}
+
+// Clean snapshots from an aborted migration
+func CleanMigrateMess(args []string) error {
+	var jailNames []string
+
+	if len(args) > 0 {
+		for _, a := range args {
+			jailNames = append(jailNames, a)
+		}
+	}
+
+	for _, jn := range jailNames {
+		cj, err := getJailFromArray(jn, []string{""}, gJails)
+		if cj == nil {
+			return errors.New(fmt.Sprintf("Error getting jail %s: Not found\n", jn))
+		}
+		curDS, err := getDatastoreFromArray(cj.Datastore, gDatastores)
+		if err != nil {
+			return errors.New(fmt.Sprintf("Error getting datastore \"%s\": %v\n", cj.Datastore, err))
+		}
+		
+		cmd := fmt.Sprintf("zfs destroy %s@gocage_mig_init", strings.Join([]string{curDS.ZFSDataset, "jails", jn}, "/"))
+		out, err := executeCommand(cmd)
+		if err != nil {
+			if false == strings.HasSuffix(out, "could not find any snapshots to destroy; check snapshot names.\n") {
+				fmt.Printf("Error executing command %s: %v; command returned: %s\n", cmd, err, out)
+				return errors.New(fmt.Sprintf("Error executing command %s: %v; command returned: %s\n", cmd, err, out))
+			}
+		}
+		cmd = fmt.Sprintf("zfs destroy %s@gocage_mig_init", strings.Join([]string{curDS.ZFSDataset, "jails", jn, "root"}, "/"))
+		out, err = executeCommand(cmd)
+		if err != nil {
+			if false == strings.HasSuffix(out, "could not find any snapshots to destroy; check snapshot names.\n") {
+				fmt.Printf("Error executing command %s: %v; command returned: %s\n", cmd, err, out)
+				return errors.New(fmt.Sprintf("Error executing command %s: %v; command returned: %s\n", cmd, err, out))
+			}
+		}
+		cmd = fmt.Sprintf("zfs destroy %s@gocage_mig_last_sync", strings.Join([]string{curDS.ZFSDataset, "jails", jn}, "/"))
+		out, err = executeCommand(cmd)
+		if err != nil {
+			if false == strings.HasSuffix(out, "could not find any snapshots to destroy; check snapshot names.\n") {
+				fmt.Printf("Error executing command %s: %v; command returned: %s\n", cmd, err, out)
+				return errors.New(fmt.Sprintf("Error executing command %s: %v; command returned: %s\n", cmd, err, out))
+			}
+		}
+		cmd = fmt.Sprintf("zfs destroy %s@gocage_mig_last_sync", strings.Join([]string{curDS.ZFSDataset, "jails", jn, "root"}, "/"))
+		out, err = executeCommand(cmd)
+		if err != nil {
+			if false == strings.HasSuffix(out, "could not find any snapshots to destroy; check snapshot names.\n") {
+				fmt.Printf("Error executing command %s: %v; command returned: %s\n", cmd, err, out)
+				return errors.New(fmt.Sprintf("Error executing command %s: %v; command returned: %s\n", cmd, err, out))
+			}
+		}
+	}
+
+	return nil
+}

cmd/properties.go

@@ -0,0 +1,125 @@
+package cmd
+
+import (
+	"os"
+	"fmt"
+	"errors"
+	"reflect"
+	"strconv"
+	"strings"
+)
+
+func GetJailProperties(args []string) {
+	var props []string
+	var jail *Jail
+	var err error
+
+	if len(args) > 0 {
+		for i, a := range args {
+			// Last arg is the jail name
+			if i == len(args)-1 {
+				jail, err = getJailFromArray(a, []string{""}, gJails)
+				if err != nil {
+					fmt.Printf("Error: %s\n", err.Error())
+					os.Exit(1)
+				}
+			} else {
+				props = append(props, a)
+			}
+		}
+	}
+
+	if len(jail.Name) == 0 || len(args) == 0 {
+		// TODO : Show help
+		fmt.Printf("Error\n")
+		return
+	}
+	
+	if isStringInArray(props, "all") {
+		var result []string
+		result = getStructFieldNames(*jail, result, "")
+		props = result
+	}
+
+	for _, p := range props {
+		v, err := getJailProperty(jail, p)
+		if err != nil {
+			fmt.Printf("Error: %s\n", err.Error())
+			return
+		} else {
+			fmt.Printf("%s = %s\n", p, v)
+		}
+	}
+}
+
+func getJailProperty(jail *Jail, propName string) (string, error) {
+	val, _, err := getStructFieldValue(jail, propName)
+	if err != nil {
+		return "", err
+	}
+
+	switch val.Kind() {
+	case reflect.String:
+		return val.String(), nil
+	case reflect.Int:
+		return strconv.FormatInt(val.Int(), 10), nil
+	case reflect.Bool:
+		return strconv.FormatBool(val.Bool()), nil
+	default:
+		return "", errors.New(fmt.Sprintf("Error: Unknown type for property %s: %s\n", propName, val.Kind()))
+	}
+
+	return "", errors.New("Unknown error in getJailProperty")
+}
+
+func SetJailProperties(args []string) {
+	type properties struct {
+		name  string
+		value string
+	}
+
+	var jail Jail
+	var props []properties
+
+	if len(args) > 0 {
+		for i, a := range args {
+			// This is the jail name
+			if i == len(args)-1 {
+				jail.Name = a
+			} else {
+				kv := strings.Split(a, "=")
+				if len(kv) != 2 {
+					// TODO : Show help
+					fmt.Printf("Error parsing args: %s\n", a)
+					return
+				} else {
+					p := properties{name: kv[0], value: kv[1]}
+					props = append(props, p)
+				}
+			}
+		}
+	}
+
+	if len(jail.Name) == 0 || len(args) == 0 {
+		// TODO : Show help
+		fmt.Printf("Error\n")
+		return
+	}
+
+	cj, err := getJailFromArray(jail.Name, []string{""}, gJails)
+	if err != nil {
+		fmt.Printf("Error getting jail %s: %v\n", jail.Name, err)
+		return
+	}
+
+	for _, p := range props {
+		err := setStructFieldValue(cj, p.name, p.value)
+		if err != nil {
+			fmt.Printf("Error: %s\n", err.Error())
+			return
+		} else {
+			fmt.Printf("%s: %s set to %s\n", cj.Name, p.name, p.value)
+		}
+	}
+	cj.WriteConfigToDisk(false)
+}

cmd/root.go

@@ -3,90 +3,475 @@ package cmd
 import (
 	"os"
 	"fmt"
+	"sync"
 	"strings"
+	"io/ioutil"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
+	// TODO : Use log
+	log "github.com/sirupsen/logrus"
 )
 
 const (
-	version = "0.02"
+	gVersion = "0.42h"
+	
+	// TODO : Get from $jail_zpool/defaults.json
+	MIN_DYN_DEVFS_RULESET = 1000
 )
 
+type createArgs struct {
+	Release      string
+	BaseJail     bool
+	Datastore    string
+	JailType     string
+	Properties   string
+}
+
 var (
-	gJails			[]Jail
+	gJailHost   JailHost
+	gJails      []Jail
+	gDatastores []Datastore
 
-	gUseSudo		bool
+	gUseSudo bool
+	gForce   bool
+	gDebug   bool
+	
+	gCreateArgs      createArgs
 
-	gConfigFile		string
-	gDisplayColumns	string
-	gFilterJails	string
-	gSortFields		string
-	gNoLineSep		bool
+	gConfigFile      string
+	gDisplayJColumns string
+	gDisplaySColumns string
+	gDisplayDColumns string
+	gFilterJails     string
+	gFilterSnaps     string
+	gFilterDS        string
+	gSortJailFields  string
+	gSortSnapFields  string
+	gSortDSFields    string
+	gNoJailLineSep   bool
+	gNoSnapLineSep   bool
+	gNoDSLineSep     bool
+	gBridgeStaticMac bool
 
+	gHostVersion float64
 
-	rootCmd = & cobra.Command{
-    	Use:    "gocage",
-	    Short:  "GoCage is a FreeBSD Jail management tool",
-	    Long:   `GoCage is a jail management tool. It support VNET, host-only, NAT networks. Provides snapshots and cloning.
+	gTimeZone     string
+	gSnapshotName string
+	gZPool        string
+	gBridge       string
+	gInterface    string
+
+	gMigrateDestDatastore string
+	gYesToAll             bool
+
+	gFetchRelease    string
+	gFetchIntoDS     string
+	gFetchFrom       string
+	gUpgradeRelease  string
+	gUpdateRelease   string
+	gUpdateReleaseDS string
+
+	// For a based jail, these are directories binded to basejail
+	gBaseDirs = []string{"bin", "boot", "lib", "libexec", "rescue", "sbin", "usr/bin", "usr/include",
+		"usr/lib", "usr/lib32", "usr/libdata", "usr/libexec", "usr/sbin", "usr/share"}
+	// These directories are to be created empty
+	gEmptyDirs = []string{"dev", "media", "mnt", "net", "proc"}
+	// Copy these from base template
+	gCopyDirs = []string{"etc", "root", "tmp", "var"}
+
+	gMdevfs          sync.Mutex
+
+	rootCmd = &cobra.Command{
+		Use:   "gocage",
+		Short: "GoCage is a FreeBSD Jail management tool",
+		Long: `GoCage is a jail management tool. It support VNET, host-only, NAT networks. Provides snapshots and cloning.
 It support iocage jails and can coexist with iocage.`,
-
-	    Run: func(cmd *cobra.Command, args []string) {
-	        fmt.Println("Here we are in the Run")
-	    },
+		Run: func(cmd *cobra.Command, args []string) {
+			fv, _ := getFreeBSDVersion()
+			fmt.Printf("GoCage v.%s on FreeBSD %d.%d-%s\n", gVersion, fv.major, fv.minor, fv.flavor)
+			fmt.Printf("Use -h flag to display help\n")
+		},
 	}
 
 	versionCmd = &cobra.Command{
-	    Use:   "version",
-	    Short: "Print the version number of GoCage",
-	    Long:  `Let this show you how much fail I had to get this *cough* perfect`,
-	    Run: func(cmd *cobra.Command, args []string) {
-	        fmt.Printf("GoCage v.%s\n", version)
-	    },
+		Use:   "version",
+		Short: "Print the version number of GoCage",
+		Long:  `Let this show you how much fail I had to get this *cough* perfect`,
+		Run: func(cmd *cobra.Command, args []string) {
+			fv, _ := getFreeBSDVersion()
+			if fv.patchLevel > 0 {
+				fmt.Printf("GoCage v.%s on FreeBSD %d.%d-%s-p%d\n", gVersion, fv.major, fv.minor, fv.flavor, fv.patchLevel)
+			} else {
+				fmt.Printf("GoCage v.%s on FreeBSD %d.%d-%s\n", gVersion, fv.major, fv.minor, fv.flavor)
+			}
+		},
+	}
+
+	initCmd = &cobra.Command{
+		Use:   "init",
+		Short: "Initialize GoCage",
+		Run: func(cmd *cobra.Command, args []string) {
+			InitGoCage(args)
+		},
 	}
 
 	listCmd = &cobra.Command{
 		Use:   "list",
 		Short: "Print jails",
-		Long:  `Display jails, their IP and OS.
+		Long: `Display jails, their IP and OS.
 Jail list can be restricted by adding name on command line
 ex: gocage list srv-db srv-web`,
 		Run: func(cmd *cobra.Command, args []string) {
 			ListJails(args, true)
 		},
 	}
-	
+
+	listPropsCmd = &cobra.Command{
+		Use:   "properties",
+		Short: "Print jails properties",
+		Long:  "Display jails properties. You can use properties to filter, get or set them.",
+		Run: func(cmd *cobra.Command, args []string) {
+			ListJailsProps(args)
+		},
+	}
+
+	destroyCmd = &cobra.Command{
+		Use:   "destroy",
+		Short: "destroy jails",
+		Long: `Destroy jail filesystem, snapshots and configuration file.`,
+		Run: func(cmd *cobra.Command, args []string) {
+			ListJails(args, false)
+			DestroyJails(args)
+		},
+	}
+
 	stopCmd = &cobra.Command{
 		Use:   "stop",
 		Short: "stop jail",
-		Long:  `shutdown jail`,
+		Long:  "shutdown jail",
 		Run: func(cmd *cobra.Command, args []string) {
-			// Get the inventory
+			// Load inventory
+			ListJails(args, false)
+			if len(args) == 0 {
+				StopAllRunningJails()
+			} else {
+				StopJail(args)
+			}
+		},
+	}
+
+	startCmd = &cobra.Command{
+		Use:   "start",
+		Short: "start jail",
+		Run: func(cmd *cobra.Command, args []string) {
+			// Load inventory
+			ListJails(args, false)
+			if len(args) == 0 {
+				StartJailsAtBoot()
+			} else {
+				StartJail(args)
+			}
+		},
+	}
+
+	restartCmd = &cobra.Command{
+		Use:   "restart",
+		Short: "restart jail",
+		Run: func(cmd *cobra.Command, args []string) {
+			// Load inventory
 			ListJails(args, false)
 			StopJail(args)
+			StartJail(args)
+		},
+	}
+
+	shellCmd = &cobra.Command {
+		Use:   "console",
+		Short: "Execute shell on jail",
+		Run: func(cmd *cobra.Command, args []string) {
+			// Load inventory
+			ListJails(args, false)
+			ShellJail(args)
+		},
+	}
+
+	setCmd = &cobra.Command{
+		Use:   "set",
+		Short: "Set a jail property",
+		Long: `Set jail property value. Specify property=value, end command with jail name.
+Multiples properties can be specified, separated with space (Ex: gocage set allow_mlock=1 boot=1 myjail)`,
+		Run: func(cmd *cobra.Command, args []string) {
+			// Load inventory
+			ListJails(args, false)
+			SetJailProperties(args)
+		},
+	}
+
+	getCmd = &cobra.Command{
+		Use:   "get",
+		Short: "Get a jail property",
+		Long: `Get jail property value. Specify property, end command with jail name.
+Multiples properties can be specified, separated with space (Ex: gocage get allow_mlock boot myjail)
+For all properties specify "all" (Ex: gocage get all myjail)`,
+		Run: func(cmd *cobra.Command, args []string) {
+			// Load inventory
+			ListJails(args, false)
+			GetJailProperties(args)
+		},
+	}
+
+	snapshotCmd = &cobra.Command{
+		Use:   "snapshot",
+		Short: "snapshot jail",
+		Long:  "Commands to manage jail snapshots. If no arguments given, ",
+		Run: func(cmd *cobra.Command, args []string) {
+
+		},
+	}
+
+	snapshotListCmd = &cobra.Command{
+		Use:   "list",
+		Short: "list snapshots",
+		Long: `List snapshots of a jail by specifying its name. 
+List all snapshots if no jail name specified.
+You can specify multiple jails.`,
+		Run: func(cmd *cobra.Command, args []string) {
+			// Load inventory
+			ListJails(args, false)
+			ListJailsSnapshots(args)
+		},
+	}
+
+	snapshotCreateCmd = &cobra.Command{
+		Use:   "create",
+		Short: "create snapshots",
+		Long:  `Create snapshot of a jail by specifying snapshot name and jail name.`,
+		//		You can specify multiple jails.`,
+		Run: func(cmd *cobra.Command, args []string) {
+			// Load inventory
+			ListJails(args, false)
+			CreateJailSnapshot(args)
+		},
+	}
+
+	snapshotRollbackCmd = &cobra.Command{
+		Use:   "rollback",
+		Short: "Rollback snapshots",
+		Long:  `Rollback jail to specifyed snapshot.`,
+		//		You can specify multiple jails.`,
+		Run: func(cmd *cobra.Command, args []string) {
+			// Load inventory
+			ListJails(args, false)
+			RollbackJailSnapshot(args)
+		},
+	}
+
+	snapshotDeleteCmd = &cobra.Command{
+		Use:   "destroy",
+		Short: "destroy snapshots",
+		Long:  `Destroy snapshot of a jail by specifying snapshot name and jail name.
+				You can specify multiple snapshots separated by comma.`,
+		Run: func(cmd *cobra.Command, args []string) {
+			// Load inventory
+			ListJails(args, false)
+			DeleteJailSnapshot(args)
+		},
+	}
+
+	migrateCmd = &cobra.Command{
+		Use:   "migrate",
+		Short: "Migrate jail to another datastore",
+		Run: func(cmd *cobra.Command, args []string) {
+			// Load inventory
+			ListJails(args, false)
+			MigrateJail(args)
+		},
+	}
+
+	migrateCleanCmd = &cobra.Command{
+		Use:   "clean",
+		Short: "Clean previous aborted/in error jail migration",
+		Run: func(cmd *cobra.Command, args []string) {
+			// Load inventory
+			ListJails(args, false)
+			err := CleanMigrateMess(args)
+			if err != nil {
+				fmt.Printf("%v", err)
+			}
+		},
+	}
+	
+	datastoreCmd = &cobra.Command{
+		Use:   "datastore",
+		Short: "list datastores",
+		Long:  "Commands to manage datastores. If no arguments given, list them.",
+		Run: func(cmd *cobra.Command, args []string) {
+			ListDatastores(args, true)
+		},
+	}
+	
+	datastoreListCmd = &cobra.Command{
+		Use:   "list",
+		Short: "list datastores",
+		Long: `List datastore by specifying its name. 
+List all datastores if no name specified.
+You can specify multiple datastores.`,
+		Run: func(cmd *cobra.Command, args []string) {
+			ListDatastores(args, true)
+		},
+	}
+
+	fetchCmd = &cobra.Command{
+		Use:   "fetch",
+		Short: "Fetch FreeBSD release to local datastore",
+		Run: func(cmd *cobra.Command, args []string) {
+			err := fetchRelease(gFetchRelease, "http", gJailHost.arch, gFetchIntoDS, gFetchFrom)
+			if err != nil {
+				fmt.Printf("%v\n", err)
+			} else {
+				extractRelease(gFetchRelease, gFetchIntoDS)
+			}
+		},
+	}
+
+	updateCmd = &cobra.Command{
+		Use:   "update",
+		Short: "Update FreeBSD release",
+		Run: func(cmd *cobra.Command, args []string) {
+			ListJails(args, false)
+			UpdateJail(args)
+		},
+	}
+
+	upgradeCmd = &cobra.Command{
+		Use:   "upgrade",
+		Short: "Upgrade FreeBSD release",
+		Run: func(cmd *cobra.Command, args []string) {
+			ListJails(args, false)
+			UpgradeJail(args)
+		},
+	}
+
+	createCmd = &cobra.Command{
+		Use:   "create",
+		Short: "Create jail",
+		Run: func(cmd *cobra.Command, args []string) {
+			ListJails(args, false)
+			CreateJail(args)
+		},
+	}
+
+	testCmd = &cobra.Command{
+		Use:   "test",
+		Short: "temporary command to test some code snippet",
+		Run: func(cmd *cobra.Command, args []string) {
+			fmt.Printf("Nope\n")
 		},
 	}
 )
 
-
-
+// TODO : Init log level and log output
 func init() {
-    cobra.OnInitialize(initConfig)
+	var err error
 
-// Global switches
-    rootCmd.PersistentFlags().StringVarP(&gConfigFile, "config", "c", "/usr/local/etc/gocage.conf.yml", "GoCage configuration file")
-    rootCmd.PersistentFlags().BoolVarP(&gUseSudo, "sudo", "u", false, "Use sudo to run commands")
+	cobra.OnInitialize(initConfig)
 
-// Command dependant switches
-    listCmd.PersistentFlags().StringVarP(&gDisplayColumns, "outcol", "o", "JID,Name,Config.Release,Config.Ip4_addr,Running", "Show these columns in output")
-    listCmd.PersistentFlags().BoolVarP(&gNoLineSep, "nolinesep", "l", false, "Do not display line separator between jails")
-    listCmd.PersistentFlags().StringVarP(&gFilterJails, "filter", "f", "none", "Only display jails with these values. Ex: \"gocage list -f Config.Boot=1\" will only list started on boot jails")
-    listCmd.PersistentFlags().StringVarP(&gSortFields, "sort", "s", "none", "Display jails sorted by field values. Ex: \"gocage list -s +Name,-Config.Priority\" will sort jails by their decreasing name, then increasing start priority. 3 critera max supported.")
-    
-// Now declare commands
-    rootCmd.AddCommand(versionCmd)
-    rootCmd.AddCommand(listCmd)
-    rootCmd.AddCommand(stopCmd)
+	// Global switches
+	rootCmd.PersistentFlags().StringVarP(&gConfigFile, "config", "c", "/usr/local/etc/gocage.conf.yml", "GoCage configuration file")
+	rootCmd.PersistentFlags().BoolVarP(&gUseSudo, "sudo", "u", false, "Use sudo to run commands")
+	rootCmd.PersistentFlags().StringVarP(&gTimeZone, "timezone", "t", "", "Specify timezone. Will get from /var/db/zoneinfo if not set.")
+	rootCmd.PersistentFlags().BoolVar(&gDebug, "debug", false, "Debug mode")
+
+	// Command dependant switches
+	initCmd.Flags().StringVarP(&gZPool, "pool", "p", "", "ZFS pool to create datastore on")
+	initCmd.Flags().StringVarP(&gBridge, "bridge", "b", "", "bridge to create for jails networking")
+	initCmd.Flags().StringVarP(&gInterface, "interface", "i", "", "interface to add as bridge member. This should be your main interface")
+	initCmd.MarkFlagRequired("bridge")
+	initCmd.MarkFlagsRequiredTogether("bridge", "interface")
+
+	// We reuse these flags in "gocage snapshot list myjail" and 'gocage datastore list" commands
+	listCmd.Flags().StringVarP(&gDisplayJColumns, "outcol", "o", "JID,Name,Config.Release,Config.Ip4_addr,Running", "Show these columns in output")
+	listCmd.Flags().BoolVarP(&gNoJailLineSep, "nolinesep", "l", false, "Do not display line separator between jails")
+	listCmd.Flags().StringVarP(&gFilterJails, "filter", "f", "none", "Only display jails with these values. Ex: \"gocage list -f Config.Boot=1\" will only list started on boot jails")
+	listCmd.Flags().StringVarP(&gSortJailFields, "sort", "s", "none", "Display jails sorted by field values. Ex: \"gocage list -s +Name,-Config.Priority\" will sort jails by their decreasing name, then increasing start priority. 3 critera max supported.")
+	
+	destroyCmd.Flags().BoolVarP(&gForce, "force", "f", false, "Force stop jail if running")
+
+	snapshotListCmd.Flags().StringVarP(&gDisplaySColumns, "outcol", "o", "Jailname,Name,Creation,Referenced,Used", "Show these columns in output")
+	snapshotListCmd.Flags().BoolVarP(&gNoSnapLineSep, "nolinesep", "l", false, "Do not display line separator between snapshots")
+	snapshotListCmd.Flags().StringVarP(&gFilterSnaps, "filter", "f", "none", "Only display snapshots with these values. Ex: \"gocage snapshot list -f Config.Boot=1\" will only list started on boot jails")
+	snapshotListCmd.Flags().StringVarP(&gSortSnapFields, "sort", "s", "none", "Display snapshots sorted by field values. Ex: \"gocage snapshot list -s +Jailname,-Used\" will sort snapshots by jail decreasing name, then increasing used space. 3 critera max supported.")
+
+	datastoreListCmd.Flags().StringVarP(&gDisplayDColumns, "outcol", "o", "Name,Mountpoint,ZFSDataset,Available,Used,Referenced", "Show these columns in output")
+	datastoreListCmd.Flags().BoolVarP(&gNoDSLineSep, "nolinesep", "l", false, "Do not display line separator between datastores")
+	datastoreListCmd.Flags().StringVarP(&gFilterDS, "filter", "f", "none", "Only display datastores with these values. Ex: \"gocage datastore list -f Config.Boot=1\" will only list started on boot jails")
+	datastoreListCmd.Flags().StringVarP(&gSortDSFields, "sort", "s", "none", "Display datastores sorted by field values. Ex: \"gocage datastore list -s +Jailname,-Used\" will sort snapshots by jail decreasing name, then increasing used space. 3 critera max supported.")
+
+	// This is local flag : Only available to gocage snapshot create command
+	snapshotCreateCmd.Flags().StringVarP(&gSnapshotName, "snapname", "n", "", "Name of the snapshot to create")
+	snapshotCreateCmd.MarkFlagRequired("snapname")
+	snapshotDeleteCmd.Flags().StringVarP(&gSnapshotName, "snapname", "n", "", "Name of the snapshot to destroy")
+	snapshotDeleteCmd.MarkFlagRequired("snapname")
+	snapshotRollbackCmd.Flags().StringVarP(&gSnapshotName, "snapname", "n", "", "Name of the snapshot to rollback to")
+	snapshotRollbackCmd.MarkFlagRequired("snapname")
+
+	migrateCmd.Flags().StringVarP(&gMigrateDestDatastore, "datastore", "d", "", "Path of destination datastore for jail (Ex: \"/iocage\")")
+	migrateCmd.Flags().BoolVarP(&gYesToAll, "yes", "y", false, "Answer yes to all questions")
+	migrateCmd.MarkFlagRequired("datastore")
+
+	fetchCmd.Flags().StringVarP(&gFetchRelease, "release", "r", "", "Release to fetch (e.g.: \"13.1-RELEASE\"")
+	fetchCmd.Flags().StringVarP(&gFetchIntoDS, "datastore", "d", "", "Datastore release will be saved to")
+	fetchCmd.Flags().StringVarP(&gFetchFrom, "from", "f", "", "Repository to download from. Should contain XY.Z-RELEASE. File protocol supported")
+	fetchCmd.MarkFlagRequired("release")
+	fetchCmd.MarkFlagRequired("datastore")
+
+	upgradeCmd.Flags().StringVarP(&gUpgradeRelease, "release", "r", "", "Release to upgrade to (e.g.: \"13.1-RELEASE\"")
+	upgradeCmd.MarkFlagRequired("release")
+
+	updateCmd.Flags().StringVarP(&gUpdateRelease, "release", "r", "", "Release to update (e.g.: \"13.1-RELEASE\"")
+	updateCmd.Flags().StringVarP(&gUpdateReleaseDS, "datastore", "d", "", "Datastore release is stored on")
+	updateCmd.MarkFlagsRequiredTogether("release", "datastore")
+
+	createCmd.Flags().StringVarP(&gCreateArgs.Release, "release", "r", "", "Release for the jail (e.g.: \"13.1-RELEASE\"")
+	createCmd.Flags().BoolVarP(&gCreateArgs.BaseJail, "basejail", "b", false, "Basejail. This will create a jail mounted read only from a release, so every up(date|grade) made to this release will immediately propagate to new jail.\n")
+	createCmd.Flags().StringVarP(&gCreateArgs.Datastore, "datastore", "d", "", "Datastore to create the jail on. Defaults to first declared in config.")
+	createCmd.Flags().StringVarP(&gCreateArgs.Properties, "configuration", "p", "", "Configuration properties with format k1=v1,k2=v2 (Ex: \"Config.Ip4_addr=vnet0|192.168.1.2,Config.Ip6=none\")")
+
+	// Now declare commands
+	rootCmd.AddCommand(initCmd)
+	rootCmd.AddCommand(versionCmd)
+	rootCmd.AddCommand(listCmd)
+	rootCmd.AddCommand(listPropsCmd)
+	rootCmd.AddCommand(stopCmd)
+	rootCmd.AddCommand(startCmd)
+	rootCmd.AddCommand(restartCmd)
+	rootCmd.AddCommand(destroyCmd)
+	rootCmd.AddCommand(shellCmd)
+	rootCmd.AddCommand(getCmd)
+	rootCmd.AddCommand(setCmd)
+	rootCmd.AddCommand(snapshotCmd)
+	rootCmd.AddCommand(migrateCmd)
+	rootCmd.AddCommand(datastoreCmd)
+	rootCmd.AddCommand(fetchCmd)
+	rootCmd.AddCommand(updateCmd)
+	rootCmd.AddCommand(upgradeCmd)
+	rootCmd.AddCommand(createCmd)
+	rootCmd.AddCommand(testCmd)
+	
+	snapshotCmd.AddCommand(snapshotListCmd)
+	snapshotCmd.AddCommand(snapshotCreateCmd)
+	snapshotCmd.AddCommand(snapshotDeleteCmd)
+	snapshotCmd.AddCommand(snapshotRollbackCmd)
+	migrateCmd.AddCommand(migrateCleanCmd)
+	datastoreCmd.AddCommand(datastoreListCmd)
+
+	// Get FreeBSD version, hostname, hostid
+	gJailHost, err = NewJailHost()
+	if err != nil {
+		fmt.Printf("Error initializing JailHost properties: %v\n", err)
+		os.Exit(1)
+	}
 }
 
 func initConfig() {
@@ -95,8 +480,6 @@ func initConfig() {
 		os.Exit(1)
 	}
 
-//	fmt.Printf("We are in initConfig(), with config file %s\n", gConfigFile)
-
 	viper.SetConfigFile(gConfigFile)
 
 	if err := viper.ReadInConfig(); err != nil {
@@ -104,37 +487,66 @@ func initConfig() {
 		os.Exit(1)
 	}
 
-//	fmt.Println("Using config file:", viper.ConfigFileUsed())
-//	fmt.Printf("datastore in config : %s\n", viper.GetStringSlice("datastore"))
-//	fmt.Printf("datastore.0 in config : %s\n", viper.GetStringSlice("datastore.0"))
-	
 	// Command line flags have priority on config file
 	if rootCmd.Flags().Lookup("sudo") != nil && false == rootCmd.Flags().Lookup("sudo").Changed {
 		gUseSudo = viper.GetBool("sudo")
 	}
+	if rootCmd.Flags().Lookup("timezone") != nil && false == rootCmd.Flags().Lookup("timezone").Changed {
+		gTimeZone = viper.GetString("timezone")
+	}
+	// If neither on cmdline nor config file, get from /var/db/zoneinfo
+	if len(gTimeZone) == 0 {
+		tz, err := ioutil.ReadFile("/var/db/zoneinfo")
+		if err != nil {
+			fmt.Println("Error reading /var/db/zoneinfo: %s\n", err.Error())
+			os.Exit(1)
+		}
+		gTimeZone = strings.Trim(string(tz), "\n")
+	}
 	if listCmd.Flags().Lookup("outcol") != nil && false == listCmd.Flags().Lookup("outcol").Changed {
-		gDisplayColumns = viper.GetString("outcol")
+		gDisplayJColumns = viper.GetString("outcol")
 	}
 	if listCmd.Flags().Lookup("nolinesep") != nil && false == listCmd.Flags().Lookup("nolinesep").Changed {
-		gNoLineSep = viper.GetBool("nolinesep")
+		gNoJailLineSep = viper.GetBool("nolinesep")
 	}
 	if listCmd.Flags().Lookup("filter") != nil && false == listCmd.Flags().Lookup("filter").Changed {
 		gFilterJails = viper.GetString("filter")
 	}
 	if listCmd.Flags().Lookup("sort") != nil && false == listCmd.Flags().Lookup("sort").Changed {
-		gSortFields = viper.GetString("sort")
+		gSortJailFields = viper.GetString("sort")
 	}
-	if len(strings.Split(gSortFields, ",")) > 3 {
-		fmt.Printf("More than 3 sort criteria, this is not supported!\n")
+	if len(strings.Split(gSortJailFields, ",")) > 3 {
+		fmt.Printf("More than 3 sort criteria is not supported!\n")
+		os.Exit(1)
+	}
+
+	gBridgeStaticMac = viper.GetBool("static-macs")
+
+	if gDebug {
+		log.SetLevel(log.DebugLevel)
+		log.Debugf("Debug mode enabled\n")
+	}
+
+	// no need to check prerequesites if we are initializing gocage
+	for _, rc := range rootCmd.Commands() {
+		//fmt.Printf("DEBUG: rootCmd subcommand: %v. Was it called? %s\n", rc.Use, rootCmd.Commands()[i].CalledAs())
+		if len(rc.CalledAs()) > 0 && strings.EqualFold("init", rc.CalledAs()) {
+			return
+		}
+	}
+
+	// Load default configs from datastores
+	err := ListDatastores(viper.GetStringSlice("datastore"), false)
+	if err != nil {
+		fmt.Printf("ERROR: error checking datastores: %v\n", err)
 		os.Exit(1)
 	}
 }
 
+
 func Execute() {
-    if err := rootCmd.Execute(); err != nil {
-        fmt.Fprintln(os.Stderr, err)
-        os.Exit(1)
-    }
+	if err := rootCmd.Execute(); err != nil {
+		fmt.Fprintln(os.Stderr, err)
+		os.Exit(1)
+	}
 }
-
-

cmd/snapshots.go

@@ -0,0 +1,330 @@
+package cmd
+
+import (
+	"bufio"
+	"errors"
+	"fmt"
+	"os"
+	"regexp"
+	"strconv"
+	"strings"
+	"reflect"
+	"time"
+)
+
+/********************************************************************************
+ * List all snapshots jails have
+ *******************************************************************************/
+func ListJailsSnapshots(args []string) {
+	var jailNames []string
+	var snapshots []Snapshot
+
+	/***************************************************************
+	/   Filter snapshots by jailname
+	/**************************************************************/
+	if len(args) > 0 {
+		for _, a := range args {
+			/*if countOfJailsWithThisName(a) > 1 {
+				fmt.Printf("Nope")
+				return
+			}*/
+			jailNames = append(jailNames, a)
+		}
+	}
+
+	if len(jailNames) == 0 || len(args) == 0 {
+		for _, j := range gJails {
+			snapshots = append(snapshots, listJailSnapshots(j)...)
+		}
+	} else {
+		for _, cj := range gJails {
+			for _, jn := range jailNames {
+				if strings.EqualFold(cj.Name, jn) {
+					snapshots = append(snapshots, listJailSnapshots(cj)...)
+				}
+			}
+		}
+	}
+	
+	fields := strings.Split(gDisplaySColumns, ",")
+	
+	/***************************************************************
+	/   Sort snapshots
+	/     We support 3 sort criteria max
+	/**************************************************************/
+	if len(gSortSnapFields) > 0 && gSortSnapFields != "none" {		
+		ss := initSnapshotSortStruct()
+
+		// The way we manage criteria quantity is not very elegant...
+		var fct1, fct2, fct3 *reflect.Value
+		for i, c := range strings.Split(gSortSnapFields, ",") {
+			var fctName string
+			if strings.HasPrefix(c, "-") {
+				fctName = fmt.Sprintf("%sDec", strings.Replace(c, "-", "", 1))
+			} else { // Par defaut (pas de prefix +/-) on considere un tri incremental
+				fctName = fmt.Sprintf("%sInc", strings.Replace(c, "+", "", 1))
+			}
+
+			// Get function by its name
+			fct, _, err := getStructFieldValue(ss, fctName)
+			
+			if err != nil {
+				fieldName := strings.Replace(strings.Replace(c, "-", "", 1), "+", "", 1)
+				fmt.Printf("ERROR getting SnapshotSort struct field %s. Please check the field name: %s\n", fctName, fieldName)
+				return
+			}
+			switch i + 1 {
+			case 1:
+				fct1 = fct
+			case 2:
+				fct2 = fct
+			case 3:
+				fct3 = fct
+			}
+		}
+
+		switch len(strings.Split(gSortSnapFields, ",")) {
+		case 1:
+			SnapshotsOrderedBy(fct1.Interface().(snapshotLessFunc)).Sort(snapshots)
+		case 2:
+			SnapshotsOrderedBy(fct1.Interface().(snapshotLessFunc), fct2.Interface().(snapshotLessFunc)).Sort(snapshots)
+		case 3:
+			SnapshotsOrderedBy(fct1.Interface().(snapshotLessFunc), fct2.Interface().(snapshotLessFunc), fct3.Interface().(snapshotLessFunc)).Sort(snapshots)
+		}
+	}
+	
+	displaySnapshotsFields(snapshots, fields)
+}
+
+/********************************************************************************
+ * List all snapshots a jail have
+ *******************************************************************************/
+func listJailSnapshots(jail Jail) []Snapshot {
+	var snapshots []Snapshot
+
+	// 1. List all datasets
+	// TODO : Include mounted filesystems?
+	
+	curDS, err := getDatastoreFromArray(jail.Datastore, gDatastores)
+	if err != nil {
+		fmt.Printf("Error getting datastore \"%s\": %v\n", jail.Datastore, err)
+		return snapshots
+	}
+	
+	rootDataset := fmt.Sprintf("%s/%s/%s", curDS.ZFSDataset, "jails", jail.Name)
+	cmd := fmt.Sprintf("zfs list -p -r -H -o name,mountpoint,used,referenced,creation -t snapshot %s", rootDataset)
+	out, err := executeCommand(cmd)
+	if err != nil {
+		fmt.Printf("Error: %s\n", err.Error())
+		return snapshots
+	}
+	
+	for _, line := range strings.Split(out, "\n") {
+		if len(line) > 0 {
+			ls := strings.Split(line, "\t")
+			// Parse creation date so we can use it to sort snapshots
+			//creationts, err := time.ParseInLocation(dateLayout, ls[4], loc)
+			creationts, err := strconv.ParseInt(ls[4], 10, 64)
+			if err != nil {
+				fmt.Println("Error while parsing date %s:", ls[4], err)
+				return snapshots
+			}
+			// Get subdir to append to snapshot name
+			subdir := strings.Replace(strings.Split(ls[0], "@")[0], rootDataset, "", 1)
+
+			u, _ := strconv.ParseUint(ls[2], 10, 64)
+			r, _ := strconv.ParseUint(ls[3], 10, 64)
+			snapshots = append(snapshots, Snapshot{Datastore: curDS.Name,
+				Name:       fmt.Sprintf("%s%s", strings.Split(ls[0], "@")[1], subdir),
+				Jailname:   jail.Name,
+				Mountpoint: ls[1],
+				Used:       u,
+				Referenced: r,
+				Creation:   time.Unix(creationts, 0)})
+		}
+	}
+
+	// Sort snapshots by creation date
+	ss := initSnapshotSortStruct()
+	SnapshotsOrderedBy(ss.CreationInc).Sort(snapshots)
+
+	return snapshots
+}
+
+/********************************************************************************
+ * Create snapshot for jail(s)
+ *******************************************************************************/
+func CreateJailSnapshot(args []string) {
+	var jailNames []string
+
+	if len(args) > 0 {
+		for _, a := range args {
+			jailNames = append(jailNames, a)
+		}
+	}
+
+	for _, cj := range gJails {
+		for _, jn := range jailNames {
+			if strings.EqualFold(cj.Name, jn) {
+				createJailSnapshot(cj)
+			}
+		}
+	}
+}
+
+/********************************************************************************
+ * Create snapshot for a jail
+ *******************************************************************************/
+func createJailSnapshot(jail Jail) error {
+	curDS, _ := getDatastoreFromArray(jail.Datastore, gDatastores)
+	rootDataset := fmt.Sprintf("%s/%s/%s", curDS.ZFSDataset, "jails", jail.Name)
+	cmd := fmt.Sprintf("zfs snapshot -r %s@%s", rootDataset, gSnapshotName)
+
+	_, err := executeCommand(cmd)
+	if err != nil {
+		fmt.Printf("Error creating snapshot %s@%s: %s\n", rootDataset, gSnapshotName, err.Error())
+		return err
+	}
+	fmt.Printf("Snapshot %s@%s created\n", rootDataset, gSnapshotName)
+
+	return nil
+}
+
+/********************************************************************************
+ * Delete snapshot for jail(s)
+ *******************************************************************************/
+func DeleteJailSnapshot(args []string) {
+	var jailNames []string
+
+	if len(args) > 0 {
+		for _, a := range args {
+			jailNames = append(jailNames, a)
+		}
+	}
+
+	for _, cj := range gJails {
+		for _, jn := range jailNames {
+			if strings.EqualFold(cj.Name, jn) {
+				deleteJailSnapshot(cj)
+			}
+		}
+	}
+}
+
+/********************************************************************************
+ * Delete snapshot for a jail
+ *******************************************************************************/
+func deleteJailSnapshot(jail Jail) error {
+	var snaptodel []string
+
+	// Get all recursive snapshots
+	curDS, _ := getDatastoreFromArray(jail.Datastore, gDatastores)
+	rootDataset := fmt.Sprintf("%s/%s/%s", curDS.ZFSDataset, "jails", jail.Name)
+	cmd := fmt.Sprintf("zfs list -r -H -o name -t snapshot %s", rootDataset)
+	out, err := executeCommand(cmd)
+	if err != nil {
+		fmt.Printf("Error: listing snapshots: %s\n", err.Error())
+		return nil
+	}
+
+	for _, line := range strings.Split(out, "\n") {
+		if len(line) > 0 {
+			ls := strings.Split(line, "@")
+			for _, sn := range strings.Split(gSnapshotName, ",") {
+				matched, _ := regexp.Match(fmt.Sprintf("^%s(\\/.*)?$", sn), []byte(ls[1]))
+				if matched {
+					snaptodel = append(snaptodel, strings.Join(ls, "@"))
+				}
+			}
+		}
+	}
+
+	for _, s := range snaptodel {
+		cmd := fmt.Sprintf("zfs destroy %s", s)
+		_, err := executeCommand(cmd)
+		if err != nil {
+			fmt.Printf("Error deleting snapshot %s: %s\n", s, err.Error())
+			return nil
+		}
+		fmt.Printf("Snapshot %s deleted\n", s)
+	}
+
+	return nil
+}
+
+func RollbackJailSnapshot(args []string) error {
+	var jailNames []string
+
+	if len(args) > 0 {
+		for _, a := range args {
+			jailNames = append(jailNames, a)
+		}
+	}
+
+	for _, cj := range gJails {
+		for _, jn := range jailNames {
+			if strings.EqualFold(cj.Name, jn) {
+				rollbackJailSnapshot(cj)
+			}
+		}
+	}
+
+	return nil
+}
+
+/********************************************************************************
+ * rollback jail to snapshot gSnapshotName, destroy this snapshots and more
+ *   recents snapshots and bookmarks
+ *******************************************************************************/
+func rollbackJailSnapshot(jail Jail) error {
+	var snaptorb []string
+
+	if jail.Running {
+		fmt.Printf("Jail should be stoped to rollback, should we stop and rollback? (y/n)\n")
+		scanr := bufio.NewScanner(os.Stdin)
+		if scanr.Scan() {
+			if !strings.EqualFold(scanr.Text(), "y") {
+				return errors.New("Jail is running")
+			} else {
+				err := stopJail(&jail)
+				if err != nil {
+					return err
+				}
+			}
+		}
+	}
+
+	// We need to rollback parent and childs
+	// Get all recursive snapshots
+	curDS, _ := getDatastoreFromArray(jail.Datastore, gDatastores)
+	rootDataset := fmt.Sprintf("%s/%s/%s", curDS.ZFSDataset, "jails", jail.Name)
+	cmd := fmt.Sprintf("zfs list -r -H -o name -t snapshot %s", rootDataset)
+	out, err := executeCommand(cmd)
+	if err != nil {
+		fmt.Printf("Error: listing snapshots: %s\n", err.Error())
+		return err
+	}
+
+	for _, line := range strings.Split(out, "\n") {
+		if len(line) > 0 {
+			ls := strings.Split(line, "@")
+
+			matched, _ := regexp.Match(fmt.Sprintf("^%s(\\/.*)?$", gSnapshotName), []byte(ls[1]))
+			if matched {
+				snaptorb = append(snaptorb, strings.Join(ls, "@"))
+			}
+		}
+	}
+
+	for _, s := range snaptorb {
+		cmd := fmt.Sprintf("zfs rollback -r %s", s)
+		_, err := executeCommand(cmd)
+		if err != nil {
+			fmt.Printf("Error rolling back snapshot %s: %s\n", s, err.Error())
+			return err
+		}
+	}
+	fmt.Printf("Jail is back to %s\n", gSnapshotName)
+
+	return nil
+}

cmd/stop.go

@@ -3,12 +3,16 @@ package cmd
 import (
 	"os"
 	"fmt"
-//	"log"
+	//"log"
+	"sync"
 	"errors"
 	"regexp"
 	"os/exec"
-//	"reflect"
+	//"reflect"
+	"strconv"
 	"strings"
+	
+	log "github.com/sirupsen/logrus"
 )
 
 // TODO : Use SYS_RCTL_GET_RACCT syscall
@@ -38,7 +42,6 @@ func removeRctlRules(jail string, rules []string) error {
 	return nil
 }
 
-
 // TODO: Validate with  >1 dataset
 func umountAndUnjailZFS(jail *Jail) error {
 	var ds []string
@@ -48,10 +51,10 @@ func umountAndUnjailZFS(jail *Jail) error {
 
 	for _, zd := range ds {
 		// 1. Get dataset and childs
-		cmd := fmt.Sprintf("zfs list -H -r -o name -S name %s/%s", jail.Zpool, zd)
+		cmd := fmt.Sprintf("zfs list -H -r -o name -S name %s", zd)
 		out, err := executeCommand(cmd)
 		if err != nil {
-			fmt.Printf(fmt.Sprintf("ERROR listing dataset %s/%s\n", jail.Zpool, zd))
+			fmt.Printf(fmt.Sprintf("ERROR listing dataset %s\n", zd))
 			os.Exit(1)
 		}
 		for _, c := range strings.Split(out, "\n") {
@@ -69,43 +72,64 @@ func umountAndUnjailZFS(jail *Jail) error {
 	}
 
 	// 2. Unjail dataset from the host
-	cmd := fmt.Sprintf("zfs unjail %s %s/%s", jail.InternalName, jail.Zpool, ds[len(ds)-1])
+	cmd := fmt.Sprintf("zfs unjail %s %s", jail.InternalName, ds[len(ds)-1])
 	_, err := executeCommand(cmd)
 	if err != nil {
-		fmt.Printf("ERROR unjailing %s/%s: %s\n", jail.Zpool, ds[len(ds)-1], err.Error())
+		fmt.Printf("ERROR unjailing %s: %s\n", ds[len(ds)-1], err.Error())
 		return err
 	}
-	
+
 	return nil
 }
 
-
 func destroyVNetInterfaces(jail *Jail) error {
-	for _, i := range strings.Split(jail.Config.Ip4_addr, ",") {
-		iname := fmt.Sprintf("%s.%d", strings.Split(i, "|")[0], jail.JID)
-		fmt.Printf("%s: ", iname)
-		_, err := executeCommand(fmt.Sprintf("ifconfig %s destroy", iname))
-		if err != nil {
-			return err
-		} else {
-			fmt.Printf("OK\n")
+	if !strings.EqualFold(jail.Config.Ip4_addr, "none") {
+		for _, i := range strings.Split(jail.Config.Ip4_addr, ",") {
+			iname := fmt.Sprintf("%s.%d", strings.Split(i, "|")[0], jail.JID)
+			fmt.Printf("%s: ", iname)
+			_, err := executeCommand(fmt.Sprintf("ifconfig %s destroy", iname))
+			//_, err := executeScript(fmt.Sprintf("ifconfig %s destroy >/dev/null 2>&1", iname))
+			if err != nil {
+				return err
+			} else {
+				fmt.Printf("OK\n")
+			}
+		}
+	}
+	if !strings.EqualFold(jail.Config.Ip6_addr, "none") {
+		for _, i := range strings.Split(jail.Config.Ip6_addr, ",") {
+			iname := fmt.Sprintf("%s.%d", strings.Split(i, "|")[0], jail.JID)
+			fmt.Printf("%s: ", iname)
+			_, err := executeCommand(fmt.Sprintf("ifconfig %s destroy", iname))
+			//_, err := executeScript(fmt.Sprintf("ifconfig %s destroy >/dev/null 2>&1", iname))
+			if err != nil {
+				return err
+			} else {
+				fmt.Printf("OK\n")
+			}
 		}
 	}
 
 	return nil
 }
 
-
-func deleteDevfsRuleset(jail *Jail) error {
+// Jails copy the ruleset referenced as "devfs_ruleset" when starting, getting a new devsf_ruleset ID.
+// This new ID can be obtained with 'jls -j $JID devfs_ruleset'
+// This is the ID which needs to be removed. Original ID referenced is json should not be deleted
+//  or else it will require a restart of "devfs" service.
+// But, stoppign the jail already removes this >1000 ID.
+// So no need to call this function.
+func deleteDevfsRuleset(ruleset int) error {
 	cmd := "devfs rule showsets"
 	out, err := executeCommand(cmd)
 	if err != nil {
 		return errors.New(fmt.Sprintf("ERROR listing rulesets: %s", err.Error()))
 	}
-
+	
+	rs := strconv.Itoa(ruleset)
 	for _, r := range strings.Split(out, "\n") {
-		if r == jail.Config.Devfs_ruleset {
-			cmd := fmt.Sprintf("devfs rule -s %s delset", jail.Config.Devfs_ruleset)
+		if r == rs {
+			cmd := fmt.Sprintf("devfs rule -s %d delset", ruleset)
 			_, err := executeCommand(cmd)
 			return err
 		}
@@ -114,8 +138,7 @@ func deleteDevfsRuleset(jail *Jail) error {
 	return nil
 }
 
-
-func umountJailFsFromHost(jail *Jail, mountpoint string) error {
+func umountFsFromHost(mountpoint string) error {
 	cmd := "mount -p"
 	out, err := executeCommand(cmd)
 	if err != nil {
@@ -126,11 +149,11 @@ func umountJailFsFromHost(jail *Jail, mountpoint string) error {
 	for _, l := range strings.Split(out, "\n") {
 		f := strings.Split(remSpPtrn.ReplaceAllString(l, " "), " ")
 		if len(f) > 2 {
-			if strings.EqualFold(f[1], fmt.Sprintf("%s%s", jail.RootPath, mountpoint)) {
-				cmd = fmt.Sprintf("umount %s%s", jail.RootPath, mountpoint)
+			if strings.EqualFold(f[1], mountpoint) {
+				cmd = fmt.Sprintf("umount %s", mountpoint)
 				_, err := executeCommand(cmd)
 				if err != nil {
-					return errors.New(fmt.Sprintf("Error umounting %s/%s: %s", jail.RootPath, mountpoint, err.Error()))
+					return errors.New(fmt.Sprintf("Error umounting %s: %s", mountpoint, err.Error()))
 				}
 				return nil
 			}
@@ -140,6 +163,9 @@ func umountJailFsFromHost(jail *Jail, mountpoint string) error {
 	return nil
 }
 
+func umountJailFsFromHost(jail *Jail, mountpoint string) error {
+	return umountFsFromHost(fmt.Sprintf("%s%s", jail.RootPath, mountpoint))
+}
 
 // Internal usage only
 func stopJail(jail *Jail) error {
@@ -162,6 +188,79 @@ func stopJail(jail *Jail) error {
 	return nil
 }
 
+// Stop all running jails by reverse priority
+// Parallelize up to gMaxThreads
+// Only parallelize same priority level jails
+func StopAllRunningJails() {
+	var stopList []Jail
+	var wg *sync.WaitGroup
+	var curThNb int
+	var curPri int
+	
+	// Get boot enabled jails
+	for _, j := range gJails {
+		if j.Running == true {
+			stopList = append(stopList, j)
+		}
+	}
+	
+	// Order by priority
+	js := initJailSortStruct()
+	fct, _, err := getStructFieldValue(js, "Config.PriorityDec")
+	if err != nil {
+		log.Errorf("ERROR getting JailSort struct field \"Config.PriorityDec\"\n")
+		return
+	}
+	JailsOrderedBy(fct.Interface().(jailLessFunc)).Sort(stopList)
+
+
+	wg = new(sync.WaitGroup)
+	curThNb = 0
+	for i, j := range stopList {
+		jFullName := fmt.Sprintf("%s/%s", j.Datastore, j.Name)
+		log.Debugf("Stopping %s with priority %s\n", jFullName, j.Config.Priority)
+		jailPri, err := strconv.Atoi(j.Config.Priority)
+		if err != nil {
+			panic(fmt.Sprintf("Invalid format for Priority (Jail %s)\n", jFullName))
+		}
+
+		if (curThNb >= gMaxThreads || i == 0) {
+			// FIXME : Use a pool instead of waiting for all threads to run a new one
+			wg.Wait()
+			curThNb = 0
+			
+			wg.Add(1)
+			curThNb++
+			curPri = jailPri
+			go func(jailFullName string) {
+				defer wg.Done()
+				StopJail([]string{jailFullName})
+			}(jFullName)
+		} else {
+			if (curPri == jailPri) {
+				wg.Add(1)
+				curThNb++
+				go func(jailFullName string) {
+					defer wg.Done()
+					StopJail([]string{jailFullName})
+				}(jFullName)
+			} else {
+				wg.Wait()
+				curThNb = 0
+			
+				wg.Add(1)
+				curThNb++
+				curPri = jailPri
+				go func(jailFullName string) {
+					defer wg.Done()
+					StopJail([]string{jailFullName})
+				}(jFullName)
+			}
+		}
+	}
+	wg.Wait()
+}
+
 /*
 	Stop jail:
 		Remove rctl rules
@@ -186,25 +285,26 @@ func stopJail(jail *Jail) error {
 func StopJail(args []string) {
 	// Current jail were stopping
 	var cj *Jail
+	var err error
 
-	for _, j := range args {
-		fmt.Printf("> Stopping jail %s\n", j)
-
-		for _, rj := range gJails {
-			if rj.Name == j {
-				cj = &rj
-				break
-			}
-		}
-		if cj == nil {
-			fmt.Printf("Jail not found: %s\n", j)
+	for _, a := range args {
+		// Check if jail exist and is distinctly named
+		cj, err = getJailFromArray(a, []string{"basejail", "jail"}, gJails)
+		if err != nil {
+			fmt.Printf("Error getting jail: %s\n", err)
 			continue
 		}
+
 		if cj.Running == false {
 			fmt.Printf("Jail %s is not running!\n", cj.Name)
 			continue
 		}
 
+		fmt.Printf("> Stopping jail %s\n", cj.Name)
+
+		// Get and write new release into config.json
+		updateVersion(cj)
+
 		out, err := executeCommand(fmt.Sprintf("rctl jail:%s", cj.InternalName))
 		if err == nil && len(out) > 0 {
 			fmt.Printf("  > Remove RCTL rules:\n")
@@ -216,17 +316,17 @@ func StopJail(args []string) {
 			}
 		}
 
-		if len (cj.Config.Exec_prestop) > 0 {
-			fmt.Printf("  > Execute prestop:\n")
+		if len(cj.Config.Exec_prestop) > 0 {
+			fmt.Printf("  > Execute pre-stop:\n")
 			_, err := executeCommand(cj.Config.Exec_prestop)
 			if err != nil {
 				fmt.Printf("ERROR: %s\n", err.Error())
 			} else {
-				fmt.Printf("  > Execute prestop: OK\n")
+				fmt.Printf("  > Execute pre-stop: OK\n")
 			}
 		}
-		
-		if len (cj.Config.Exec_stop) > 0 {
+
+		if len(cj.Config.Exec_stop) > 0 {
 			fmt.Printf("  > Execute stop:\n")
 			_, err := executeCommandInJail(cj, cj.Config.Exec_stop)
 			if err != nil {
@@ -255,15 +355,15 @@ func StopJail(args []string) {
 				fmt.Printf("  > Destroy VNet interfaces: OK\n")
 			}
 		}
-		
-		fmt.Printf("  > Remove devfsruleset %s:\n", cj.Config.Devfs_ruleset)
-		err = deleteDevfsRuleset(cj)
+
+		fmt.Printf("  > Remove devfs ruleset %d: \n", cj.Devfs_ruleset)
+		err = deleteDevfsRuleset(cj.Devfs_ruleset)
 		if err != nil {
 			fmt.Printf("ERROR: %s\n", err.Error())
 		} else {
-			fmt.Printf("  > Remove devfsruleset %s: OK\n", cj.Config.Devfs_ruleset)
+			fmt.Printf("  > Remove devfsruleset %d: OK\n", cj.Devfs_ruleset)
 		}
-		
+
 		fmt.Printf("  > Stop jail %s:\n", cj.Name)
 		err = stopJail(cj)
 		if err != nil {
@@ -292,15 +392,16 @@ func StopJail(args []string) {
 			}
 		}
 
-		if cj.Config.Mount_fdescfs > 0 {
+		// FIXME: /dev/fd is mounted even with Mount_fdescfs = 0 ?!
+		//if cj.Config.Mount_fdescfs > 0 {
 			fmt.Printf("  > Umount fdescfs:\n")
-			err := umountJailFsFromHost(cj, "/dev/fd")
+			err = umountJailFsFromHost(cj, "/dev/fd")
 			if err != nil {
 				fmt.Printf("ERROR: %s\n", err.Error())
 			} else {
 				fmt.Printf("  > Umount fdescfs: OK\n")
 			}
-		}
+		//}
 
 		if cj.Config.Mount_devfs > 0 {
 			fmt.Printf("  > Umount devfs:\n")
@@ -311,6 +412,64 @@ func StopJail(args []string) {
 				fmt.Printf("  > Umount devfs: OK\n")
 			}
 		}
+
+		// Remove local mounts from $JAIL/fstab
+		fstab := strings.Replace(cj.ConfigPath, "config.json", "fstab", 1)
+		mounts, err := getFstab(fstab)
+		if err != nil {
+			fmt.Printf("ERROR: %s\n", err.Error())
+		}
+		if len(mounts) > 0 {
+			fmt.Printf("  > Umount mountpoints from %s\n", fstab)
+			errs := 0
+			for _, m := range mounts {
+				log.Debugf("Umounting %s\n", m.Mountpoint)
+				err = umountFsFromHost(m.Mountpoint)
+				if err != nil {
+					fmt.Printf("ERROR: %s\n", err.Error())
+					errs += 1
+				}
+			}
+			if errs == 0 {
+				fmt.Printf("  > Umount mountpoints from %s: OK\n", fstab)
+			}
+		}
+
+		// TODO: Execute poststop
+		if len(cj.Config.Exec_poststop) > 0 {
+			fmt.Printf("  > Execute post-stop:\n")
+			_, err := executeCommand(cj.Config.Exec_poststop)
+			if err != nil {
+				fmt.Printf("ERROR: %s\n", err.Error())
+			} else {
+				fmt.Printf("  > Execute post-stop: OK\n")
+			}
+		}
+
+		// Remove parameter file
+		pfile := fmt.Sprintf("/var/run/jail.%s.conf", cj.InternalName)
+		if err = os.Remove(pfile); err != nil {
+			fmt.Printf("Error deleting parameter file %s\n", pfile)
+		}
+
+		// We need this to get a reference to cj.Running (bc cj.Running is just a copy of value in the scope of StopJail())
+		for i, j := range gJails {
+			if strings.EqualFold(j.Name, cj.Name) && strings.EqualFold(j.Datastore, cj.Datastore) {
+				if err = setStructFieldValue(&gJails[i], "Running", "false"); err != nil {
+					fmt.Printf("ERROR: setting Running property to false: %s\n", err.Error())
+				}
+				if err = setStructFieldValue(&gJails[i], "JID", "0"); err != nil {
+					fmt.Printf("ERROR: setting JID property to 0: %s\n", err.Error())
+				}
+				if err = setStructFieldValue(&gJails[i], "InternalName", ""); err != nil {
+					fmt.Printf("ERROR: clearing InternalName property: %s\n", err.Error())
+				}
+				if err = setStructFieldValue(&gJails[i], "Devfs_ruleset", "0"); err != nil {
+					fmt.Printf("ERROR: setting Devfs_ruleset property to 0: %s\n", err.Error())
+				}
+			}
+		}
+		writeConfigToDisk(cj, false)
+
 	}
 }
-

cmd/struct.go

@@ -1,18 +1,35 @@
 package cmd
 
+import (
+	"time"
+)
+
+const (
+	IPv4 = 0
+	IPv6 = 1
+)
+
+type NatDesc struct {
+	Proto    string
+	JailPort string
+	HostPort string
+}
 
 // To allow sorting, just duplicate fields in JailSort below
 type Jail struct {
-	Name					string
-	InternalName			string
-	JID						int
-	Config					JailConfig
-	RootPath				string
-	ConfigPath				string
-	Running					bool
+	Name          string
+	InternalName  string
+	JID           int
+	Config        JailConfig
+	RootPath      string
+	ConfigPath    string
+	ConfigUpdated bool
+	Running       bool
 	// No need, Config.Release always represent what is running (plus it know release for non-running jails)
 	//Release					string
-	Zpool					string
+	Devfs_ruleset int	// The effective devfs ruleset generated at runtime
+	Zpool         string
+	Datastore     string
 }
 
 // iocage struct as stored in config.json
@@ -20,444 +37,527 @@ type Jail struct {
 //
 // Fields in this struct are acquired by their name using reflection
 //  So these char are forbidden for field name: -+.
+// Array should be forbidden, or else you'll need to rewrite setJailProperty()
 //
 // To allow sorting, just duplicate fields in JailConfigSort below
 type JailConfig struct {
-	Config_version			string	`json:"CONFIG_VERSION"`
-	Allow_chflags			int		`json:"allow_chflags"`
-	Allow_mlock				int		`json:"allow_mlock"`
-	Allow_mount				int		`json:"allow_mount"`
-	Allow_mount_devfs		int		`json:"allow_mount_devfs"`
-	Allow_mount_fusefs		int		`json:"allow_mount_fusefs"`
-	Allow_mount_nullfs		int		`json:"allow_mount_nullfs"`
-	Allow_mount_procfs		int		`json:"allow_mount_procfs"`
-	Allow_mount_tmpfs		int		`json:"allow_mount_tmpfs"`
-	Allow_mount_zfs			int		`json:"allow_mount_zfs"`
-	Allow_quotas			int		`json:"allow_quotas"`
-	Allow_raw_sockets		int		`json:"allow_raw_sockets"`
-	Allow_set_hostname		int		`json:"allow_set_hostname"`
-	Allow_socket_af			int		`json:"allow_socket_af"`
-	Allow_sysvipc			int		`json:"allow_sysvipc"`
-	Allow_tun				int		`json:"allow_tun"`
-	Allow_vmm				int		`json:"allow_vmm"`
-	Assign_localhost		int		`json:"assign_localhost"`
-	Available				string	`json:"available"`
-	Basejail				int		`json:"basejail"`
-	Boot					int		`json:"boot"`
-	Bpf						int		`json:"bpf"`
-	Children_max			string	`json:"children_max"`
-	Cloned_release			string	`json:"cloned_release"`
-	Comment					string	`json:"comment"`
-	Compression				string	`json:"compression"`
-	Compressratio			string	`json:"compressratio"`
-	Coredumpsize			string	`json:"coredumpsize"`
-	Count					string	`json:"count"`
-	Cpuset					string	`json:"cpuset"`
-	Cputime					string	`json:"cputime"`
-	Datasize				string	`json:"datasize"`
-	Dedup					string	`json:"dedup"`
-	Defaultrouter			string	`json:"defaultrouter"`
-	Defaultrouter6			string	`json:"defaultrouter6"`
-	Depends					string	`json:"depends"`
-	Devfs_ruleset			string	`json:"devfs_ruleset"`
-	Dhcp					int		`json:"dhcp"`
-	Enforce_statfs			string	`json:"enforce_statfs"`
-	Exec_clean				int		`json:"exec_clean"`
-	Exec_created			string	`json:"exec_created"`
-	Exec_fib				string	`json:"exec_fib"`
-	Exec_jail_user			string	`json:"exec_jail_user"`
-	Exec_poststart			string	`json:"exec_poststart"`
-	Exec_poststop			string	`json:"exec_poststop"`
-	Exec_prestart			string	`json:"exec_prestart"`
-	Exec_prestop			string	`json:"exec_prestop"`
-	Exec_start				string	`json:"exec_start"`
-	Exec_stop				string	`json:"exec_stop"`
-	Exec_system_jail_user	string	`json:"exec_system_jail_user"`
-	Exec_system_user		string	`json:"exec_system_user"`
-	Exec_timeout			string	`json:"exec_timeout"`
-	Host_domainname			string	`json:"host_domainname"`
-	Host_hostname			string	`json:"host_hostname"`
-	Host_hostuuid			string	`json:"host_hostuuid"`
-	Host_time				int		`json:"host_time"`
-	Hostid					string	`json:"hostid"`
-	Hostid_strict_check		int		`json:"hostid_strict_check"`
-	Interfaces				string	`json:"interfaces"`
-	Ip4						string	`json:"ip4"`
-	Ip4_addr				string	`json:"ip4_addr"`
-	Ip4_saddrsel			string	`json:"ip4_saddrsel"`
-	Ip6						string	`json:"ip6"`
-	Ip6_addr				string	`json:"ip6_addr"`
-	Ip6_saddrsel			string	`json:"ip4_saddrsel"`
-	Ip_hostname				int		`json:"ip_hostname"`
-	Jail_zfs				int		`json:"jail_zfs"`
-	Jail_zfs_dataset		string	`json:"jail_zfs_dataset"`
-	Jail_zfs_mountpoint		string	`json:"jail_zfs_mountpoint"`
-	Last_started			string	`json:"last_started"`
-	Localhost_ip			string	`json:"localhost_ip"`
-	Login_flags				string	`json:"login_flags"`
-	Mac_prefix				string	`json:"mac_prefix"`
-	Maxproc					string	`json:"maxproc"`
-	Memorylocked			string	`json:"memorylocked"`
-	Memoryuse				string	`json:"memoryuse"`
-	Min_dyn_devfs_ruleset	string	`json:"min_dyn_devfs_ruleset"`
-	Mount_devfs				int		`json:"mount_devfs"`
-	Mount_fdescfs			int		`json:"mount_fdescfs"`
-	Mount_linprocfs			int		`json:"mount_linprocfs"`
-	Mount_procfs			int		`json:"mount_procfs"`
-	Mountpoint				string	`json:"mountpoint"`
-	Msgqqueued				string	`json:"msgqqueued"`
-	Msgqsize				string	`json:"msgqsize"`
-	Nat						int		`json:"nat"`
-	Nat_backend				string	`json:"nat_backend"`
-	Nat_forwards			string	`json:"nat_forwards"`
-	Nat_interface			string	`json:"nat_interface"`
-	Nat_prefix				string	`json:"nat_prefix"`
-	Nmsgq					string	`json:"nmsgq"`
-	Notes					string	`json:"notes"`
-	Nsem					string	`json:"nsem"`
-	Nsemop					string	`json:"nsemop"`
-	Nshm					string	`json:"nshm"`
-	Nthr					string	`json:"nthr"`
-	Openfiles				string	`json:"openfiles"`
-	Origin					string	`json:"origin"`
-	Owner					string	`json:"owner"`
-	Pcpu					string	`json:"pcpu"`
-	Plugin_name				string	`json:"plugin_name"`
-	Plugin_repository		string	`json:"plugin_repository"`
-	Priority				string	`json:"priority"`
-	Pseudoterminals			string	`json:"pseudoterminals"`
-	Quota					string	`json:"quota"`
-	Readbps					string	`json:"readbps"`
-	Readiops				string	`json:"readiops"`
-	Release					string	`json:"release"`
-	Reservation				string	`json:"reservation"`
-	Resolver				string	`json:"resolver"`
-	Rlimits					string	`json:"rlimits"`
-	Rtsold					int		`json:"rtsold"`
-	Securelevel				string	`json:"securelevel"`
-	Shmsize					string	`json:"shmsize"`
-	Stacksize				string	`json:"stacksize"`
-	Stop_timeout			string	`json:"stop_timeout"`
-	Swapuse					string	`json:"swapuse"`
-	Sync_state				string	`json:"sync_state"`
-	Sync_target				string	`json:"sync_target"`
-	Sync_tgt_zpool			string	`json:"sync_tgt_zpool"`
-	Sysvmsg					string	`json:"sysvmsg"`
-	Sysvsem					string	`json:"sysvsem"`
-	Sysvshm					string	`json:"sysvshm"`
-	Template				int		`json:"template"`
+	Config_version        string `json:"CONFIG_VERSION"`
+	Allow_chflags         int    `json:"allow_chflags"`
+	Allow_mlock           int    `json:"allow_mlock"`
+	Allow_mount           int    `json:"allow_mount"`
+	Allow_mount_devfs     int    `json:"allow_mount_devfs"`
+	Allow_mount_fusefs    int    `json:"allow_mount_fusefs"`
+	Allow_mount_nullfs    int    `json:"allow_mount_nullfs"`
+	Allow_mount_procfs    int    `json:"allow_mount_procfs"`
+	Allow_mount_tmpfs     int    `json:"allow_mount_tmpfs"`
+	Allow_mount_zfs       int    `json:"allow_mount_zfs"`
+	Allow_quotas          int    `json:"allow_quotas"`
+	Allow_raw_sockets     int    `json:"allow_raw_sockets"`
+	Allow_set_hostname    int    `json:"allow_set_hostname"`
+	Allow_socket_af       int    `json:"allow_socket_af"`
+	Allow_sysvipc         int    `json:"allow_sysvipc"`
+	Allow_tun             int    `json:"allow_tun"`
+	Allow_vmm             int    `json:"allow_vmm"`
+	Assign_localhost      int    `json:"assign_localhost"`
+	Available             string `json:"available"`
+	Basejail              int    `json:"basejail"`
+	Boot                  int    `json:"boot"`
+	Bpf                   int    `json:"bpf"`
+	Children_max          string `json:"children_max"`
+	Cloned_release        string `json:"cloned_release"`
+	Comment               string `json:"comment"`
+	Compression           string `json:"compression"`
+	Compressratio         string `json:"compressratio"`
+	Coredumpsize          string `json:"coredumpsize"`
+	Count                 string `json:"count"`
+	Cpuset                string `json:"cpuset"`
+	Cputime               string `json:"cputime"`
+	Datasize              string `json:"datasize"`
+	Dedup                 string `json:"dedup"`
+	Defaultrouter         string `json:"defaultrouter"`
+	Defaultrouter6        string `json:"defaultrouter6"`
+	Depends               string `json:"depends"`
+	Devfs_ruleset         string `json:"devfs_ruleset"`
+	Dhcp                  int    `json:"dhcp"`
+	Enforce_statfs        string `json:"enforce_statfs"`
+	Exec_clean            int    `json:"exec_clean"`
+	Exec_created          string `json:"exec_created"`
+	Exec_fib              string `json:"exec_fib"`
+	Exec_jail_user        string `json:"exec_jail_user"`
+	Exec_poststart        string `json:"exec_poststart"`
+	Exec_poststop         string `json:"exec_poststop"`
+	Exec_prestart         string `json:"exec_prestart"`
+	Exec_prestop          string `json:"exec_prestop"`
+	Exec_start            string `json:"exec_start"`
+	Exec_stop             string `json:"exec_stop"`
+	Exec_system_jail_user string `json:"exec_system_jail_user"`
+	Exec_system_user      string `json:"exec_system_user"`
+	Exec_timeout          string `json:"exec_timeout"`
+	Host_domainname       string `json:"host_domainname"`
+	Host_hostname         string `json:"host_hostname"`
+	Host_hostuuid         string `json:"host_hostuuid"`
+	Host_time             int    `json:"host_time"`
+	Hostid                string `json:"hostid"`
+	Hostid_strict_check   int    `json:"hostid_strict_check"`
+// Specify multiple net cards with "vnet0:bridge0,vnet1:bridge1"
+	Interfaces            string `json:"interfaces"`
+	Ip4                   string `json:"ip4"`
+// Specify multiples IP with "vnet0|192.168.1.42,vnet1|10.0.0.3"
+	Ip4_addr              string `json:"ip4_addr"`
+	Ip4_saddrsel          string `json:"ip4_saddrsel"`
+	Ip6                   string `json:"ip6"`
+	Ip6_addr              string `json:"ip6_addr"`
+	Ip6_saddrsel          string `json:"ip4_saddrsel"`
+	Ip_hostname           int    `json:"ip_hostname"`
+	Jail_zfs              int    `json:"jail_zfs"`
+	Jail_zfs_dataset      string `json:"jail_zfs_dataset"`
+	Jail_zfs_mountpoint   string `json:"jail_zfs_mountpoint"`
+	Last_started          string `json:"last_started"`
+	Localhost_ip          string `json:"localhost_ip"`
+	Login_flags           string `json:"login_flags"`
+	Mac_prefix            string `json:"mac_prefix"`
+	Maxproc               string `json:"maxproc"`
+	Memorylocked          string `json:"memorylocked"`
+	Memoryuse             string `json:"memoryuse"`
+	Min_dyn_devfs_ruleset string `json:"min_dyn_devfs_ruleset"`
+	Mount_devfs           int    `json:"mount_devfs"`
+	Mount_fdescfs         int    `json:"mount_fdescfs"`
+	Mount_linprocfs       int    `json:"mount_linprocfs"`
+	Mount_procfs          int    `json:"mount_procfs"`
+	Mountpoint            string `json:"mountpoint"`
+	Msgqqueued            string `json:"msgqqueued"`
+	Msgqsize              string `json:"msgqsize"`
+	Nat                   int    `json:"nat"`
+	Nat_backend           string `json:"nat_backend"`
+	Nat_forwards          string `json:"nat_forwards"`
+	Nat_interface         string `json:"nat_interface"`
+	Nat_prefix            string `json:"nat_prefix"`
+	Nmsgq                 string `json:"nmsgq"`
+	Notes                 string `json:"notes"`
+	Nsem                  string `json:"nsem"`
+	Nsemop                string `json:"nsemop"`
+	Nshm                  string `json:"nshm"`
+	Nthr                  string `json:"nthr"`
+	Openfiles             string `json:"openfiles"`
+	Origin                string `json:"origin"`
+	Owner                 string `json:"owner"`
+	Pcpu                  string `json:"pcpu"`
+	Plugin_name           string `json:"plugin_name"`
+	Plugin_repository     string `json:"plugin_repository"`
+	Priority              string `json:"priority"`
+	Pseudoterminals       string `json:"pseudoterminals"`
+	Quota                 string `json:"quota"`
+	Readbps               string `json:"readbps"`
+	Readiops              string `json:"readiops"`
+	Release               string `json:"release"`
+	Reservation           string `json:"reservation"`
+	Resolver              string `json:"resolver"`
+	Rlimits               string `json:"rlimits"`
+	Rtsold                int    `json:"rtsold"`
+	Securelevel           string `json:"securelevel"`
+	Shmsize               string `json:"shmsize"`
+	Stacksize             string `json:"stacksize"`
+	Stop_timeout          string `json:"stop_timeout"`
+	Swapuse               string `json:"swapuse"`
+	Sync_state            string `json:"sync_state"`
+	Sync_target           string `json:"sync_target"`
+	Sync_tgt_zpool        string `json:"sync_tgt_zpool"`
+	Sysvmsg               string `json:"sysvmsg"`
+	Sysvsem               string `json:"sysvsem"`
+	Sysvshm               string `json:"sysvshm"`
+	Template              int    `json:"template"`
 	// Go don't like a variable named "type" (And i dont care about finding a cleaner way)
-	Jailtype				string	`json:"type"`
-	Used					string	`json:"used"`
-	Vmemoryuse				string	`json:"vmemoryuse"`
-	Vnet					int		`json:"vnet"`
-	Vnet0_mac				string	`json:"vnet0_mac"`
-	Vnet1_mac				string	`json:"vnet1_mac"`
-	Vnet2_mac				string	`json:"vnet2_mac"`
-	Vnet3_mac				string	`json:"vnet3_mac"`
-	Vnet_default_interface	string	`json:"vnet_default_interface"`
-	Vnet_interfaces			string	`json:"vnet_interfaces"`
-	Wallclock				string	`json:"wallclock"`
-	Writebps				string	`json:"writebps"`
-	Writeiops				string	`json:"writeiops"`
+	Jailtype               string `json:"type"`
+	Used                   string `json:"used"`
+	Vmemoryuse             string `json:"vmemoryuse"`
+	Vnet                   int    `json:"vnet"`
+	Vnet0_mac              string `json:"vnet0_mac"`
+	Vnet1_mac              string `json:"vnet1_mac"`
+	Vnet2_mac              string `json:"vnet2_mac"`
+	Vnet3_mac              string `json:"vnet3_mac"`
+	Vnet_default_interface string `json:"vnet_default_interface"`
+	Vnet_interfaces        string `json:"vnet_interfaces"`
+	Wallclock              string `json:"wallclock"`
+	Writebps               string `json:"writebps"`
+	Writeiops              string `json:"writeiops"`
 }
 
-// This struct hold "sort by jail fields" functions
-type lessFunc func(j1 *Jail, j2 *Jail) bool
+// Represent an fstab line
+type Mount struct {
+	Device     string
+	Mountpoint string
+	Type       string
+	Options    []string
+	Fs_Freq    int
+	Fs_Passno  int
+}
+
+type Snapshot struct {
+	// snapshot name is stored after '@' in dataset name
+	Name       string
+	Datastore  string
+	Jailname   string
+	Mountpoint string
+	Used       uint64
+	Referenced uint64
+	Creation   time.Time
+}
+
+type FreeBSDVersion struct {
+	major      int
+	minor      int
+	flavor     string
+	patchLevel int
+}
+
+type JailHost struct {
+	hostname          string
+	hostid            string
+	arch              string
+	default_gateway4  string
+	default_gateway6  string
+	default_interface string
+	version           FreeBSDVersion
+}
+
+type Datastore struct {
+	Name              string
+	Mountpoint        string
+	ZFSDataset        string
+	DefaultJailConfig JailConfig
+	Used              uint64
+	Referenced        uint64
+	Available         uint64
+}
 
 // Fields in this struct are acquired by their name using reflection
 //  So these char are forbidden for field name: -+.
-//
 type JailSort struct {
-    NameInc					lessFunc
-	NameDec					lessFunc
-    InternalNameInc			lessFunc
-    InternalNameDec			lessFunc
-    JIDInc					lessFunc 
-    JIDDec					lessFunc 
-    RootPathInc				lessFunc
-    RootPathDec				lessFunc
-	ConfigPathInc			lessFunc
-	ConfigPathDec			lessFunc
-    RunningInc				lessFunc
-    RunningDec				lessFunc
-    ZpoolInc				lessFunc
-    ZpoolDec				lessFunc
-	Config					JailConfigSort
+	NameInc         jailLessFunc
+	NameDec         jailLessFunc
+	InternalNameInc jailLessFunc
+	InternalNameDec jailLessFunc
+	JIDInc          jailLessFunc
+	JIDDec          jailLessFunc
+	RootPathInc     jailLessFunc
+	RootPathDec     jailLessFunc
+	ConfigPathInc   jailLessFunc
+	ConfigPathDec   jailLessFunc
+	RunningInc      jailLessFunc
+	RunningDec      jailLessFunc
+	DatastoreInc    jailLessFunc
+	DatastoreDec    jailLessFunc
+	ZpoolInc        jailLessFunc
+	ZpoolDec        jailLessFunc
+	Devfs_rulesetInc jailLessFunc
+	Devfs_rulesetDec jailLessFunc
+	Config          JailConfigSort
 }
 
-
 type JailConfigSort struct {
-	Config_versionInc		lessFunc
-	Config_versionDec		lessFunc
-	Allow_chflagsInc		lessFunc
-	Allow_chflagsDec		lessFunc
-	Allow_mlockInc			lessFunc
-	Allow_mlockDec			lessFunc
-	Allow_mountInc			lessFunc
-	Allow_mountDec			lessFunc
-	Allow_mount_devfsInc	lessFunc
-	Allow_mount_devfsDec	lessFunc
-	Allow_mount_fusefsInc	lessFunc
-	Allow_mount_fusefsDec	lessFunc
-	Allow_mount_nullfsInc	lessFunc
-	Allow_mount_nullfsDec	lessFunc
-	Allow_mount_procfsInc	lessFunc
-	Allow_mount_procfsDec	lessFunc
-	Allow_mount_tmpfsInc	lessFunc
-	Allow_mount_tmpfsDec	lessFunc
-	Allow_mount_zfsInc		lessFunc
-	Allow_mount_zfsDec		lessFunc
-	Allow_quotasInc			lessFunc
-	Allow_quotasDec			lessFunc
-	Allow_raw_socketsInc	lessFunc
-	Allow_raw_socketsDec	lessFunc
-	Allow_set_hostnameInc	lessFunc
-	Allow_set_hostnameDec	lessFunc
-	Allow_socket_afInc		lessFunc
-	Allow_socket_afDec		lessFunc
-	Allow_sysvipcInc		lessFunc
-	Allow_sysvipcDec		lessFunc
-	Allow_tunInc			lessFunc
-	Allow_tunDec			lessFunc
-	Allow_vmmInc			lessFunc
-	Allow_vmmDec			lessFunc
-	Assign_localhostInc		lessFunc
-	Assign_localhostDec		lessFunc
-	AvailableInc			lessFunc
-	AvailableDec			lessFunc
-	BasejailInc				lessFunc
-	BasejailDec				lessFunc
-	BootInc					lessFunc
-	BootDec					lessFunc
-	BpfInc					lessFunc
-	BpfDec					lessFunc
-	Children_maxInc			lessFunc
-	Children_maxDec			lessFunc
-	Cloned_releaseInc		lessFunc
-	Cloned_releaseDec		lessFunc
-	CommentInc				lessFunc
-	CommentDec				lessFunc
-	CompressionInc			lessFunc
-	CompressionDec			lessFunc
-	CompressratioInc		lessFunc
-	CompressratioDec		lessFunc
-	CoredumpsizeInc			lessFunc
-	CoredumpsizeDec			lessFunc
-	CountInc				lessFunc
-	CountDec				lessFunc
-	CpusetInc				lessFunc
-	CpusetDec				lessFunc
-	CputimeInc				lessFunc
-	CputimeDec				lessFunc
-	DatasizeInc				lessFunc
-	DatasizeDec				lessFunc
-	DedupInc				lessFunc
-	DedupDec				lessFunc
-	DefaultrouterInc		lessFunc
-	DefaultrouterDec		lessFunc
-	Defaultrouter6Inc		lessFunc
-	Defaultrouter6Dec		lessFunc
-	DependsInc				lessFunc
-	DependsDec				lessFunc
-	Devfs_rulesetInc		lessFunc
-	Devfs_rulesetDec		lessFunc
-	DhcpInc					lessFunc
-	DhcpDec					lessFunc
-	Enforce_statfsInc		lessFunc
-	Enforce_statfsDec		lessFunc
-	Exec_cleanInc			lessFunc
-	Exec_cleanDec			lessFunc
-	Exec_createdInc			lessFunc
-	Exec_createdDec			lessFunc
-	Exec_fibInc				lessFunc
-	Exec_fibDec				lessFunc
-	Exec_jail_userInc		lessFunc
-	Exec_jail_userDec		lessFunc
-	Exec_poststartInc		lessFunc
-	Exec_poststartDec		lessFunc
-	Exec_poststopInc		lessFunc
-	Exec_poststopDec		lessFunc
-	Exec_prestartInc		lessFunc
-	Exec_prestartDec		lessFunc
-	Exec_prestopInc			lessFunc
-	Exec_prestopDec			lessFunc
-	Exec_startInc			lessFunc
-	Exec_startDec			lessFunc
-	Exec_stopInc			lessFunc
-	Exec_stopDec			lessFunc
-	Exec_system_jail_userInc	lessFunc
-	Exec_system_jail_userDec	lessFunc
-	Exec_system_userInc		lessFunc
-	Exec_system_userDec		lessFunc
-	Exec_timeoutInc			lessFunc
-	Exec_timeoutDec			lessFunc
-	Host_domainnameInc		lessFunc
-	Host_domainnameDec		lessFunc
-	Host_hostnameInc		lessFunc
-	Host_hostnameDec		lessFunc
-	Host_hostuuidInc		lessFunc
-	Host_hostuuidDec		lessFunc
-	Host_timeInc			lessFunc
-	Host_timeDec			lessFunc
-	HostidInc				lessFunc
-	HostidDec				lessFunc
-	Hostid_strict_checkInc	lessFunc
-	Hostid_strict_checkDec	lessFunc
-	InterfacesInc			lessFunc
-	InterfacesDec			lessFunc
-	Ip4Inc					lessFunc
-	Ip4Dec					lessFunc
-	Ip4_addrInc				lessFunc
-	Ip4_addrDec				lessFunc
-	Ip4_saddrselInc			lessFunc
-	Ip4_saddrselDec			lessFunc
-	Ip6Inc					lessFunc
-	Ip6Dec					lessFunc
-	Ip6_addrInc				lessFunc
-	Ip6_addrDec				lessFunc
-	Ip6_saddrselInc			lessFunc
-	Ip6_saddrselDec			lessFunc
-	Ip_hostnameInc			lessFunc
-	Ip_hostnameDec			lessFunc
-	Jail_zfsInc				lessFunc
-	Jail_zfsDec				lessFunc
-	Jail_zfs_datasetInc		lessFunc
-	Jail_zfs_datasetDec		lessFunc
-	Jail_zfs_mountpointInc	lessFunc
-	Jail_zfs_mountpointDec	lessFunc
-	Last_startedInc			lessFunc
-	Last_startedDec			lessFunc
-	Localhost_ipInc			lessFunc
-	Localhost_ipDec			lessFunc
-	Login_flagsInc			lessFunc
-	Login_flagsDec			lessFunc
-	Mac_prefixInc			lessFunc
-	Mac_prefixDec			lessFunc
-	MaxprocInc				lessFunc
-	MaxprocDec				lessFunc
-	MemorylockedInc			lessFunc
-	MemorylockedDec			lessFunc
-	MemoryuseInc			lessFunc
-	MemoryuseDec			lessFunc
-	Min_dyn_devfs_rulesetInc	lessFunc
-	Min_dyn_devfs_rulesetDec	lessFunc
-	Mount_devfsInc			lessFunc
-	Mount_devfsDec			lessFunc
-	Mount_fdescfsInc		lessFunc
-	Mount_fdescfsDec		lessFunc
-	Mount_linprocfsInc		lessFunc
-	Mount_linprocfsDec		lessFunc
-	Mount_procfsInc			lessFunc
-	Mount_procfsDec			lessFunc
-	MountpointInc			lessFunc
-	MountpointDec			lessFunc
-	MsgqqueuedInc			lessFunc
-	MsgqqueuedDec			lessFunc
-	MsgqsizeInc				lessFunc
-	MsgqsizeDec				lessFunc
-	NatInc					lessFunc
-	NatDec					lessFunc
-	Nat_backendInc			lessFunc
-	Nat_backendDec			lessFunc
-	Nat_forwardsInc			lessFunc
-	Nat_forwardsDec			lessFunc
-	Nat_interfaceInc		lessFunc
-	Nat_interfaceDec		lessFunc
-	Nat_prefixInc			lessFunc
-	Nat_prefixDec			lessFunc
-	NmsgqInc				lessFunc
-	NmsgqDec				lessFunc
-	NotesInc				lessFunc
-	NotesDec				lessFunc
-	NsemInc					lessFunc
-	NsemDec					lessFunc
-	NsemopInc				lessFunc
-	NsemopDec				lessFunc
-	NshmInc					lessFunc
-	NshmDec					lessFunc
-	NthrInc					lessFunc
-	NthrDec					lessFunc
-	OpenfilesInc			lessFunc
-	OpenfilesDec			lessFunc
-	OriginInc				lessFunc
-	OriginDec				lessFunc
-	OwnerInc				lessFunc
-	OwnerDec				lessFunc
-	PcpuInc					lessFunc
-	PcpuDec					lessFunc
-	Plugin_nameInc			lessFunc
-	Plugin_nameDec			lessFunc
-	Plugin_repositoryInc	lessFunc
-	Plugin_repositoryDec	lessFunc
-	PriorityInc				lessFunc
-	PriorityDec				lessFunc
-	PseudoterminalsInc		lessFunc
-	PseudoterminalsDec		lessFunc
-	QuotaInc				lessFunc
-	QuotaDec				lessFunc
-	ReadbpsInc				lessFunc
-	ReadbpsDec				lessFunc
-	ReadiopsInc				lessFunc
-	ReadiopsDec				lessFunc
-	ReleaseInc				lessFunc
-	ReleaseDec				lessFunc
-	ReservationInc			lessFunc
-	ReservationDec			lessFunc
-	ResolverInc				lessFunc
-	ResolverDec				lessFunc
-	RlimitsInc				lessFunc
-	RlimitsDec				lessFunc
-	RtsoldInc				lessFunc
-	RtsoldDec				lessFunc
-	SecurelevelInc			lessFunc	
-	SecurelevelDec			lessFunc	
-	ShmsizeInc				lessFunc	
-	ShmsizeDec				lessFunc	
-	StacksizeInc			lessFunc
-	StacksizeDec			lessFunc
-	Stop_timeoutInc			lessFunc
-	Stop_timeoutDec			lessFunc
-	SwapuseInc				lessFunc
-	SwapuseDec				lessFunc
-	Sync_stateInc			lessFunc
-	Sync_stateDec			lessFunc
-	Sync_targetInc			lessFunc
-	Sync_targetDec			lessFunc
-	Sync_tgt_zpoolInc		lessFunc
-	Sync_tgt_zpoolDec		lessFunc
-	SysvmsgInc				lessFunc
-	SysvmsgDec				lessFunc
-	SysvsemInc				lessFunc	
-	SysvsemDec				lessFunc	
-	SysvshmInc				lessFunc	
-	SysvshmDec				lessFunc	
-	TemplateInc				lessFunc
-	TemplateDec				lessFunc
-	JailtypeInc				lessFunc
-	JailtypeDec				lessFunc
-	UsedInc					lessFunc
-	UsedDec					lessFunc
-	VmemoryuseInc			lessFunc
-	VmemoryuseDec			lessFunc
-	VnetInc					lessFunc
-	VnetDec					lessFunc
-	Vnet0_macInc			lessFunc
-	Vnet0_macDec			lessFunc
-	Vnet1_macInc			lessFunc
-	Vnet1_macDec			lessFunc
-	Vnet2_macInc			lessFunc
-	Vnet2_macDec			lessFunc
-	Vnet3_macInc			lessFunc
-	Vnet3_macDec			lessFunc
-	Vnet_default_interfaceInc	lessFunc
-	Vnet_default_interfaceDec	lessFunc
-	Vnet_interfacesInc		lessFunc
-	Vnet_interfacesDec		lessFunc
-	WallclockInc			lessFunc
-	WallclockDec			lessFunc
-	WritebpsInc				lessFunc
-	WritebpsDec				lessFunc
-	WriteiopsInc			lessFunc
-	WriteiopsDec			lessFunc
+	Config_versionInc         jailLessFunc
+	Config_versionDec         jailLessFunc
+	Allow_chflagsInc          jailLessFunc
+	Allow_chflagsDec          jailLessFunc
+	Allow_mlockInc            jailLessFunc
+	Allow_mlockDec            jailLessFunc
+	Allow_mountInc            jailLessFunc
+	Allow_mountDec            jailLessFunc
+	Allow_mount_devfsInc      jailLessFunc
+	Allow_mount_devfsDec      jailLessFunc
+	Allow_mount_fusefsInc     jailLessFunc
+	Allow_mount_fusefsDec     jailLessFunc
+	Allow_mount_nullfsInc     jailLessFunc
+	Allow_mount_nullfsDec     jailLessFunc
+	Allow_mount_procfsInc     jailLessFunc
+	Allow_mount_procfsDec     jailLessFunc
+	Allow_mount_tmpfsInc      jailLessFunc
+	Allow_mount_tmpfsDec      jailLessFunc
+	Allow_mount_zfsInc        jailLessFunc
+	Allow_mount_zfsDec        jailLessFunc
+	Allow_quotasInc           jailLessFunc
+	Allow_quotasDec           jailLessFunc
+	Allow_raw_socketsInc      jailLessFunc
+	Allow_raw_socketsDec      jailLessFunc
+	Allow_set_hostnameInc     jailLessFunc
+	Allow_set_hostnameDec     jailLessFunc
+	Allow_socket_afInc        jailLessFunc
+	Allow_socket_afDec        jailLessFunc
+	Allow_sysvipcInc          jailLessFunc
+	Allow_sysvipcDec          jailLessFunc
+	Allow_tunInc              jailLessFunc
+	Allow_tunDec              jailLessFunc
+	Allow_vmmInc              jailLessFunc
+	Allow_vmmDec              jailLessFunc
+	Assign_localhostInc       jailLessFunc
+	Assign_localhostDec       jailLessFunc
+	AvailableInc              jailLessFunc
+	AvailableDec              jailLessFunc
+	BasejailInc               jailLessFunc
+	BasejailDec               jailLessFunc
+	BootInc                   jailLessFunc
+	BootDec                   jailLessFunc
+	BpfInc                    jailLessFunc
+	BpfDec                    jailLessFunc
+	Children_maxInc           jailLessFunc
+	Children_maxDec           jailLessFunc
+	Cloned_releaseInc         jailLessFunc
+	Cloned_releaseDec         jailLessFunc
+	CommentInc                jailLessFunc
+	CommentDec                jailLessFunc
+	CompressionInc            jailLessFunc
+	CompressionDec            jailLessFunc
+	CompressratioInc          jailLessFunc
+	CompressratioDec          jailLessFunc
+	CoredumpsizeInc           jailLessFunc
+	CoredumpsizeDec           jailLessFunc
+	CountInc                  jailLessFunc
+	CountDec                  jailLessFunc
+	CpusetInc                 jailLessFunc
+	CpusetDec                 jailLessFunc
+	CputimeInc                jailLessFunc
+	CputimeDec                jailLessFunc
+	DatasizeInc               jailLessFunc
+	DatasizeDec               jailLessFunc
+	DedupInc                  jailLessFunc
+	DedupDec                  jailLessFunc
+	DefaultrouterInc          jailLessFunc
+	DefaultrouterDec          jailLessFunc
+	Defaultrouter6Inc         jailLessFunc
+	Defaultrouter6Dec         jailLessFunc
+	DependsInc                jailLessFunc
+	DependsDec                jailLessFunc
+	Devfs_rulesetInc          jailLessFunc
+	Devfs_rulesetDec          jailLessFunc
+	DhcpInc                   jailLessFunc
+	DhcpDec                   jailLessFunc
+	Enforce_statfsInc         jailLessFunc
+	Enforce_statfsDec         jailLessFunc
+	Exec_cleanInc             jailLessFunc
+	Exec_cleanDec             jailLessFunc
+	Exec_createdInc           jailLessFunc
+	Exec_createdDec           jailLessFunc
+	Exec_fibInc               jailLessFunc
+	Exec_fibDec               jailLessFunc
+	Exec_jail_userInc         jailLessFunc
+	Exec_jail_userDec         jailLessFunc
+	Exec_poststartInc         jailLessFunc
+	Exec_poststartDec         jailLessFunc
+	Exec_poststopInc          jailLessFunc
+	Exec_poststopDec          jailLessFunc
+	Exec_prestartInc          jailLessFunc
+	Exec_prestartDec          jailLessFunc
+	Exec_prestopInc           jailLessFunc
+	Exec_prestopDec           jailLessFunc
+	Exec_startInc             jailLessFunc
+	Exec_startDec             jailLessFunc
+	Exec_stopInc              jailLessFunc
+	Exec_stopDec              jailLessFunc
+	Exec_system_jail_userInc  jailLessFunc
+	Exec_system_jail_userDec  jailLessFunc
+	Exec_system_userInc       jailLessFunc
+	Exec_system_userDec       jailLessFunc
+	Exec_timeoutInc           jailLessFunc
+	Exec_timeoutDec           jailLessFunc
+	Host_domainnameInc        jailLessFunc
+	Host_domainnameDec        jailLessFunc
+	Host_hostnameInc          jailLessFunc
+	Host_hostnameDec          jailLessFunc
+	Host_hostuuidInc          jailLessFunc
+	Host_hostuuidDec          jailLessFunc
+	Host_timeInc              jailLessFunc
+	Host_timeDec              jailLessFunc
+	HostidInc                 jailLessFunc
+	HostidDec                 jailLessFunc
+	Hostid_strict_checkInc    jailLessFunc
+	Hostid_strict_checkDec    jailLessFunc
+	InterfacesInc             jailLessFunc
+	InterfacesDec             jailLessFunc
+	Ip4Inc                    jailLessFunc
+	Ip4Dec                    jailLessFunc
+	Ip4_addrInc               jailLessFunc
+	Ip4_addrDec               jailLessFunc
+	Ip4_saddrselInc           jailLessFunc
+	Ip4_saddrselDec           jailLessFunc
+	Ip6Inc                    jailLessFunc
+	Ip6Dec                    jailLessFunc
+	Ip6_addrInc               jailLessFunc
+	Ip6_addrDec               jailLessFunc
+	Ip6_saddrselInc           jailLessFunc
+	Ip6_saddrselDec           jailLessFunc
+	Ip_hostnameInc            jailLessFunc
+	Ip_hostnameDec            jailLessFunc
+	Jail_zfsInc               jailLessFunc
+	Jail_zfsDec               jailLessFunc
+	Jail_zfs_datasetInc       jailLessFunc
+	Jail_zfs_datasetDec       jailLessFunc
+	Jail_zfs_mountpointInc    jailLessFunc
+	Jail_zfs_mountpointDec    jailLessFunc
+	Last_startedInc           jailLessFunc
+	Last_startedDec           jailLessFunc
+	Localhost_ipInc           jailLessFunc
+	Localhost_ipDec           jailLessFunc
+	Login_flagsInc            jailLessFunc
+	Login_flagsDec            jailLessFunc
+	Mac_prefixInc             jailLessFunc
+	Mac_prefixDec             jailLessFunc
+	MaxprocInc                jailLessFunc
+	MaxprocDec                jailLessFunc
+	MemorylockedInc           jailLessFunc
+	MemorylockedDec           jailLessFunc
+	MemoryuseInc              jailLessFunc
+	MemoryuseDec              jailLessFunc
+	Min_dyn_devfs_rulesetInc  jailLessFunc
+	Min_dyn_devfs_rulesetDec  jailLessFunc
+	Mount_devfsInc            jailLessFunc
+	Mount_devfsDec            jailLessFunc
+	Mount_fdescfsInc          jailLessFunc
+	Mount_fdescfsDec          jailLessFunc
+	Mount_linprocfsInc        jailLessFunc
+	Mount_linprocfsDec        jailLessFunc
+	Mount_procfsInc           jailLessFunc
+	Mount_procfsDec           jailLessFunc
+	MountpointInc             jailLessFunc
+	MountpointDec             jailLessFunc
+	MsgqqueuedInc             jailLessFunc
+	MsgqqueuedDec             jailLessFunc
+	MsgqsizeInc               jailLessFunc
+	MsgqsizeDec               jailLessFunc
+	NatInc                    jailLessFunc
+	NatDec                    jailLessFunc
+	Nat_backendInc            jailLessFunc
+	Nat_backendDec            jailLessFunc
+	Nat_forwardsInc           jailLessFunc
+	Nat_forwardsDec           jailLessFunc
+	Nat_interfaceInc          jailLessFunc
+	Nat_interfaceDec          jailLessFunc
+	Nat_prefixInc             jailLessFunc
+	Nat_prefixDec             jailLessFunc
+	NmsgqInc                  jailLessFunc
+	NmsgqDec                  jailLessFunc
+	NotesInc                  jailLessFunc
+	NotesDec                  jailLessFunc
+	NsemInc                   jailLessFunc
+	NsemDec                   jailLessFunc
+	NsemopInc                 jailLessFunc
+	NsemopDec                 jailLessFunc
+	NshmInc                   jailLessFunc
+	NshmDec                   jailLessFunc
+	NthrInc                   jailLessFunc
+	NthrDec                   jailLessFunc
+	OpenfilesInc              jailLessFunc
+	OpenfilesDec              jailLessFunc
+	OriginInc                 jailLessFunc
+	OriginDec                 jailLessFunc
+	OwnerInc                  jailLessFunc
+	OwnerDec                  jailLessFunc
+	PcpuInc                   jailLessFunc
+	PcpuDec                   jailLessFunc
+	Plugin_nameInc            jailLessFunc
+	Plugin_nameDec            jailLessFunc
+	Plugin_repositoryInc      jailLessFunc
+	Plugin_repositoryDec      jailLessFunc
+	PriorityInc               jailLessFunc
+	PriorityDec               jailLessFunc
+	PseudoterminalsInc        jailLessFunc
+	PseudoterminalsDec        jailLessFunc
+	QuotaInc                  jailLessFunc
+	QuotaDec                  jailLessFunc
+	ReadbpsInc                jailLessFunc
+	ReadbpsDec                jailLessFunc
+	ReadiopsInc               jailLessFunc
+	ReadiopsDec               jailLessFunc
+	ReleaseInc                jailLessFunc
+	ReleaseDec                jailLessFunc
+	ReservationInc            jailLessFunc
+	ReservationDec            jailLessFunc
+	ResolverInc               jailLessFunc
+	ResolverDec               jailLessFunc
+	RlimitsInc                jailLessFunc
+	RlimitsDec                jailLessFunc
+	RtsoldInc                 jailLessFunc
+	RtsoldDec                 jailLessFunc
+	SecurelevelInc            jailLessFunc
+	SecurelevelDec            jailLessFunc
+	ShmsizeInc                jailLessFunc
+	ShmsizeDec                jailLessFunc
+	StacksizeInc              jailLessFunc
+	StacksizeDec              jailLessFunc
+	Stop_timeoutInc           jailLessFunc
+	Stop_timeoutDec           jailLessFunc
+	SwapuseInc                jailLessFunc
+	SwapuseDec                jailLessFunc
+	Sync_stateInc             jailLessFunc
+	Sync_stateDec             jailLessFunc
+	Sync_targetInc            jailLessFunc
+	Sync_targetDec            jailLessFunc
+	Sync_tgt_zpoolInc         jailLessFunc
+	Sync_tgt_zpoolDec         jailLessFunc
+	SysvmsgInc                jailLessFunc
+	SysvmsgDec                jailLessFunc
+	SysvsemInc                jailLessFunc
+	SysvsemDec                jailLessFunc
+	SysvshmInc                jailLessFunc
+	SysvshmDec                jailLessFunc
+	TemplateInc               jailLessFunc
+	TemplateDec               jailLessFunc
+	JailtypeInc               jailLessFunc
+	JailtypeDec               jailLessFunc
+	UsedInc                   jailLessFunc
+	UsedDec                   jailLessFunc
+	VmemoryuseInc             jailLessFunc
+	VmemoryuseDec             jailLessFunc
+	VnetInc                   jailLessFunc
+	VnetDec                   jailLessFunc
+	Vnet0_macInc              jailLessFunc
+	Vnet0_macDec              jailLessFunc
+	Vnet1_macInc              jailLessFunc
+	Vnet1_macDec              jailLessFunc
+	Vnet2_macInc              jailLessFunc
+	Vnet2_macDec              jailLessFunc
+	Vnet3_macInc              jailLessFunc
+	Vnet3_macDec              jailLessFunc
+	Vnet_default_interfaceInc jailLessFunc
+	Vnet_default_interfaceDec jailLessFunc
+	Vnet_interfacesInc        jailLessFunc
+	Vnet_interfacesDec        jailLessFunc
+	WallclockInc              jailLessFunc
+	WallclockDec              jailLessFunc
+	WritebpsInc               jailLessFunc
+	WritebpsDec               jailLessFunc
+	WriteiopsInc              jailLessFunc
+	WriteiopsDec              jailLessFunc
 }
+
+type SnapshotSort struct {
+	NameInc       snapshotLessFunc
+	NameDec       snapshotLessFunc
+	DatastoreInc  snapshotLessFunc
+	DatastoreDec  snapshotLessFunc
+	JailnameInc   snapshotLessFunc
+	JailnameDec   snapshotLessFunc
+	MountpointInc snapshotLessFunc
+	MountpointDec snapshotLessFunc
+	UsedInc       snapshotLessFunc
+	UsedDec       snapshotLessFunc
+	ReferencedInc snapshotLessFunc
+	ReferencedDec snapshotLessFunc
+	CreationInc   snapshotLessFunc
+	CreationDec   snapshotLessFunc
+}
+
+type DatastoreSort struct {
+	NameInc       datastoreLessFunc
+	NameDec       datastoreLessFunc
+	MountpointInc datastoreLessFunc
+	MountpointDec datastoreLessFunc
+	ZFSDatasetInc datastoreLessFunc
+	ZFSDatasetDec datastoreLessFunc
+	UsedInc       datastoreLessFunc
+	UsedDec       datastoreLessFunc
+	ReferencedInc datastoreLessFunc
+	ReferencedDec datastoreLessFunc
+	AvailableInc  datastoreLessFunc
+	AvailableDec  datastoreLessFunc
+}
+

cmd/update.go

@@ -0,0 +1,144 @@
+package cmd
+
+import (
+	"os"
+	"fmt"
+	//"log"
+	"time"
+	"strings"
+	"github.com/spf13/viper"
+)
+
+
+// Internal usage only
+func updateJail(jail *Jail, doUpdateVersion bool) error {
+	// Create default config as temporary file
+	cfgFile, err := os.CreateTemp("", "gocage-jail-update-")
+	if err != nil {
+		return err
+	}
+
+	// Folder containing update/upgrade temporary files. Mutualized so we save bandwith when upgrading multiple jails
+	uwd := viper.GetString("updateWorkDir")
+	if len(uwd) == 0 {
+		return fmt.Errorf("updateWorkDir not set in configuration")
+	}
+	_, err = os.Stat(uwd)
+	if os.IsNotExist(err) {
+		if err := os.Mkdir(uwd, 0755); err != nil {
+			return err
+		}
+	}
+	cfgFile.Write([]byte(strings.Replace(fbsdUpdateConfig, "TO-BE-REPLACED-WITH-UPDATEWORKDIR", uwd, 1)))
+	defer cfgFile.Close()
+	defer os.Remove(cfgFile.Name())
+
+	cmd := fmt.Sprintf("/usr/sbin/freebsd-update --not-running-from-cron -f %s -b %s --currently-running %s fetch",
+					   cfgFile.Name(), jail.RootPath, jail.Config.Release)
+
+	err = executeCommandWithOutputToStdout(cmd)
+	if err != nil {
+		return err
+	}
+
+	cmd = fmt.Sprintf("/usr/sbin/freebsd-update --not-running-from-cron -f %s -b %s --currently-running %s install",
+			   cfgFile.Name(), jail.RootPath, jail.Config.Release)
+	err = executeCommandWithOutputToStdout(cmd)
+	if err != nil {
+		return err
+	}
+
+	// Get and write new release into config.json. Don't do that for fake jail (aka release updating)
+	if doUpdateVersion {
+		updateVersion(jail)
+	}
+
+	return nil
+}
+
+func UpdateJail(args []string) {
+	// Current jail were stopping
+	var cj *Jail
+	var err error
+
+	// User is updateing a release, fake a jail
+	if  len(gUpdateRelease) > 0 {
+		// get datastore mountpoing from datastore name
+		ds, err := getDatastoreFromArray(gUpdateReleaseDS, gDatastores)
+		if err != nil {
+			fmt.Printf("Error getting datastore %s: %v\n", gUpdateReleaseDS, err)
+			return
+		}
+		rp := fmt.Sprintf("%s/releases/%s/root", ds.Mountpoint, gUpdateRelease)
+		fakeJail := Jail{RootPath: rp}
+		v, err := getVersion(&fakeJail)
+		if err != nil {
+			fmt.Printf("Error getting version of release %s: %v\n", gUpdateRelease, err)
+			return
+		}
+		fakeJail.Config.Release = v
+
+		// Remove patch level from Release
+		fv, err := freebsdVersionToStruct(fakeJail.Config.Release)
+		if err != nil {
+			fmt.Printf("Error converting release %s: %v\n", fakeJail.Config.Release, err)
+			return
+		}
+		release := fmt.Sprintf("%d.%d-%s", fv.major, fv.minor, fv.flavor)
+		// Snapshot before updating
+		dt := time.Now()
+		curDate := fmt.Sprintf("%s", dt.Format("2006-01-02_15-04-05"))
+		snapshotName := fmt.Sprintf("gocage_update_%s_%s", v, curDate)
+		err = zfsSnapshot(fmt.Sprintf("%s/releases/%s", ds.ZFSDataset, release), snapshotName)
+		if err != nil {
+			fmt.Printf("Error snapshoting release %s: %v\n", gUpdateRelease, err)
+			return
+		}
+		err = zfsSnapshot(fmt.Sprintf("%s/releases/%s/root", ds.ZFSDataset, release), snapshotName)
+		if err != nil {
+			fmt.Printf("Error snapshoting release %s: %v\n", gUpdateRelease, err)
+		} else {
+			fmt.Printf("Release %s was snapshoted with success: %s\n", gUpdateRelease, snapshotName)
+		}
+
+		if err = updateJail(&fakeJail, false); err != nil {
+			fmt.Printf("Error updating release %s: %v\n", gUpdateRelease, err)
+		}
+		return
+	}
+
+	for _, a := range args {
+		// Check if jail exist and is distinctly named
+		cj, err = getJailFromArray(a, []string{""}, gJails)
+		if err != nil {
+			fmt.Printf("Error getting jail: %s\n", err)
+			continue
+		}
+
+		// We cant update basejail as system is readonly
+		if strings.EqualFold(cj.Config.Jailtype, "basejail") {
+			fmt.Printf("%s is a basejail using %s system files. Please update %s!\n", cj.Name, cj.Config.Origin, cj.Config.Origin)
+			continue
+		}
+
+		fmt.Printf("  > Snapshot jail %s\n", cj.Name)
+		// Set snapshot name
+		dt := time.Now()
+		curDate := fmt.Sprintf("%s", dt.Format("2006-01-02_15-04-05"))
+		gSnapshotName = fmt.Sprintf("gocage_update_%s_%s", cj.Config.Release, curDate)
+		err := createJailSnapshot(*cj)
+		if err != nil {
+			fmt.Printf("  > Snapshot jail %s: ERROR: %s\n", cj.Name, err.Error())
+			return
+		}
+		fmt.Printf("  > Snapshot jail %s: OK\n", cj.Name)
+		
+		fmt.Printf("  > Update jail %s\n", cj.Name)
+		err = updateJail(cj, true)
+		if err != nil {
+			fmt.Printf("ERROR: %s\n", err.Error())
+		} else {
+			fmt.Printf("  > Update jail %s: OK\n", cj.Name)
+		}
+	}
+}

cmd/upgrade.go

@@ -0,0 +1,126 @@
+package cmd
+
+
+
+import (
+	"os"
+	"fmt"
+	//"log"
+	"time"
+	"strings"
+	"github.com/spf13/viper"
+)
+
+// Internal usage only
+func upgradeJail(jail *Jail, version string) error {
+	// Create default config as temporary file
+	cfgFile, err := os.CreateTemp("", "gocage-jail-upgrade-")
+	if err != nil {
+		return err
+	}
+
+	// Folder containing update/upgrade temporary files. Mutualized so we save bandwith when upgrading multiple jails
+	uwd := viper.GetString("updateWorkDir")
+	if len(uwd) == 0 {
+		return fmt.Errorf("updateWorkDir not set in configuration")
+	}
+	_, err = os.Stat(uwd)
+	if os.IsNotExist(err) {
+		if err := os.Mkdir(uwd, 0755); err != nil {
+			return err
+		}
+	}
+	cfgFile.Write([]byte(strings.Replace(fbsdUpdateConfig, "TO-BE-REPLACED-WITH-UPDATEWORKDIR", uwd, 1)))
+	defer cfgFile.Close()
+	defer os.Remove(cfgFile.Name())
+
+	// Get current version. Won't work on stopped jail.
+	fbsdvers, err := executeCommandInJail(jail, "/bin/freebsd-version")
+	if err != nil {
+		fmt.Printf("ERROR executeCommandInJail: %s\n", err.Error())
+		return err
+	}
+	fbsdvers = strings.TrimRight(fbsdvers, "\n")
+
+	cmd := fmt.Sprintf("/usr/sbin/freebsd-update -f %s -b %s --currently-running %s -r %s upgrade", 
+			   cfgFile.Name(), jail.RootPath, fbsdvers, version)
+
+	//fmt.Printf("DEBUG: Prepare to execute \"%s\"\n", cmd)
+
+	// Need to give user control, bc there could be merge edit needs
+	err = executeCommandWithStdinStdoutStderr(cmd)
+	if err != nil {
+		return err
+	}
+
+	cmd = fmt.Sprintf("/usr/sbin/freebsd-update -f %s -b %s --currently-running %s -r %s install", 
+			   cfgFile.Name(), jail.RootPath, fbsdvers, version)
+
+	//fmt.Printf("DEBUG: Prepare to execute \"%s\"\n", cmd)
+
+	err = executeCommandWithStdinStdoutStderr(cmd)
+	if err != nil {
+		return err
+	}
+
+	cmd = fmt.Sprintf("/usr/sbin/freebsd-update -f %s -b %s --currently-running %s -r %s install", 
+			  cfgFile.Name(), jail.RootPath, fbsdvers, version)
+
+	//fmt.Printf("DEBUG: Prepare to execute \"%s\"\n", cmd)
+
+	err = executeCommandWithStdinStdoutStderr(cmd)
+	if err != nil {
+		return err
+	}
+
+	cmd = fmt.Sprintf("/usr/local/sbin/pkg-static -j %d install -q -f -y pkg", jail.JID)
+	err = executeCommandWithStdinStdoutStderr(cmd)
+	if err != nil {
+		return err
+	}
+
+	// Get and write new release into config.json
+	updateVersion(jail)
+
+	return nil
+}
+
+func UpgradeJail(args []string) {
+	// Current jail were stopping
+	var cj *Jail
+	var err error
+
+	for _, a := range args {
+		// Check if jail exist and is distinctly named
+		cj, err = getJailFromArray(a, []string{""}, gJails)
+		if err != nil {
+			fmt.Printf("Error getting jail: %s\n", err)
+			continue
+		}
+
+		if cj.Running == false {
+			fmt.Printf("Error: jail must be running for upgrade.\n")
+			return
+		}
+
+		fmt.Printf("  > Snapshot jail %s\n", cj.Name)
+		// Set snapshot name
+		dt := time.Now()
+		curDate := fmt.Sprintf("%s", dt.Format("2006-01-02_15-04-05"))
+		gSnapshotName = fmt.Sprintf("goc_upgrade_%s_%s", cj.Config.Release, curDate)
+		err := createJailSnapshot(*cj)
+		if err != nil {
+			fmt.Printf("  > Snapshot jail %s: ERROR: %s\n", cj.Name, err.Error())
+			return
+		}
+		fmt.Printf("  > Snapshot jail %s: OK\n", cj.Name)
+
+		fmt.Printf("  > Upgrade jail %s to %s\n", cj.Name, gUpgradeRelease)
+		err = upgradeJail(cj, gUpgradeRelease)
+		if err != nil {
+			fmt.Printf("ERROR: %s\n", err.Error())
+		} else {
+			fmt.Printf("  > Upgrade jail %s: OK\n", cj.Name)
+		}
+	}
+}

go.mod

@@ -3,24 +3,56 @@ module gocage
 go 1.17
 
 require (
-	github.com/spf13/cobra v1.2.1
-	github.com/spf13/viper v1.9.0
+	github.com/c-robinson/iplib v1.0.3
+	github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
+	github.com/otiai10/copy v1.12.0
+	github.com/sirupsen/logrus v1.8.1
+	github.com/spf13/cobra v1.8.1
+	github.com/spf13/viper v1.19.0
+	golang.org/x/net v0.25.0
 )
 
 require (
-	github.com/fsnotify/fsnotify v1.5.1 // indirect
+	github.com/bytedance/sonic v1.11.6 // indirect
+	github.com/bytedance/sonic/loader v0.1.1 // indirect
+	github.com/cloudwego/base64x v0.1.4 // indirect
+	github.com/cloudwego/iasm v0.2.0 // indirect
+	github.com/fsnotify/fsnotify v1.7.0 // indirect
+	github.com/gabriel-vasile/mimetype v1.4.3 // indirect
+	github.com/gin-contrib/sse v0.1.0 // indirect
+	github.com/go-playground/locales v0.14.1 // indirect
+	github.com/go-playground/universal-translator v0.18.1 // indirect
+	github.com/go-playground/validator/v10 v10.20.0 // indirect
+	github.com/goccy/go-json v0.10.2 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
-	github.com/inconshreveable/mousetrap v1.0.0 // indirect
-	github.com/magiconair/properties v1.8.5 // indirect
-	github.com/mitchellh/mapstructure v1.4.2 // indirect
-	github.com/pelletier/go-toml v1.9.4 // indirect
-	github.com/spf13/afero v1.6.0 // indirect
-	github.com/spf13/cast v1.4.1 // indirect
-	github.com/spf13/jwalterweatherman v1.1.0 // indirect
+	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/json-iterator/go v1.1.12 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.7 // indirect
+	github.com/leodido/go-urn v1.4.0 // indirect
+	github.com/magiconair/properties v1.8.7 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mitchellh/mapstructure v1.5.0 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.2 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
+	github.com/sagikazarmark/locafero v0.4.0 // indirect
+	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
+	github.com/sourcegraph/conc v0.3.0 // indirect
+	github.com/spf13/afero v1.11.0 // indirect
+	github.com/spf13/cast v1.6.0 // indirect
 	github.com/spf13/pflag v1.0.5 // indirect
-	github.com/subosito/gotenv v1.2.0 // indirect
-	golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf // indirect
-	golang.org/x/text v0.3.6 // indirect
-	gopkg.in/ini.v1 v1.63.2 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
+	github.com/subosito/gotenv v1.6.0 // indirect
+	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
+	github.com/ugorji/go/codec v1.2.12 // indirect
+	go.uber.org/atomic v1.9.0 // indirect
+	go.uber.org/multierr v1.9.0 // indirect
+	golang.org/x/arch v0.8.0 // indirect
+	golang.org/x/crypto v0.23.0 // indirect
+	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
+	golang.org/x/sys v0.20.0 // indirect
+	golang.org/x/text v0.15.0 // indirect
+	google.golang.org/protobuf v1.34.1 // indirect
+	gopkg.in/ini.v1 v1.67.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 )

gocage.conf.yml

@@ -1,11 +1,15 @@
-# List datastore root directory (the one containing jails/ and defaults.json)
+# List datastore(s) root directory (the one containing jails/ and defaults.json)
 datastore:
     - /iocage
 
+# Prefix all commands with sudo
 sudo: false
 
+# Directory used to store update temporary files. Mutualized so we save bandwith
+updateWorkDir: /iocage/freebsd-updates
+
 # Columns to display when "gocage list". Column names are struct fields, see cmd/struct.go
 outcol: 'JID,Name,Config.Release,Config.Ip4_addr,Running'
 
-# Do not add line separator between jails
+# Do not add line separator between jails
 nolinesep: false

jail/main.go

@@ -14,16 +14,17 @@ package jail
 */
 import "C"
 import (
-	"strconv"
+    "strconv"
 //    "syscall"
     "unsafe"
 )
 
 
 type Jail struct {
-    Name	string
-    Jid		int
-	Path	string
+    Name          string
+    Jid           int
+    Path          string
+    Devfs_ruleset int
 }
 
 
@@ -33,8 +34,8 @@ func GetJails() ([]Jail, error) {
     var jl Jail
     var err error
 
-	// Make "params" a list of 4 jails parameters
-    params := make([]C.struct_jailparam, 4)
+	// Make "params" a list of 5 jails parameters
+    params := make([]C.struct_jailparam, 5)
 
     // initialize parameter names
     csname := C.CString("name")
@@ -43,27 +44,31 @@ func GetJails() ([]Jail, error) {
     defer C.free(unsafe.Pointer(csjid))
     cspath := C.CString("path")
     defer C.free(unsafe.Pointer(cspath))
+	csdevfsrs := C.CString("devfs_ruleset")
+	defer C.free(unsafe.Pointer(csdevfsrs))
     cslastjid := C.CString("lastjid")
     defer C.free(unsafe.Pointer(cslastjid))
+	
 
     // initialize params struct with parameter names
     C.jailparam_init(&params[0], csname)
     C.jailparam_init(&params[1], csjid)
     C.jailparam_init(&params[2], cspath)
+	C.jailparam_init(&params[3], csdevfsrs)
 
     // The key to retrieve jail. lastjid = 0 returns first jail and its jid as jailparam_get return value
-    C.jailparam_init(&params[3], cslastjid)
+    C.jailparam_init(&params[4], cslastjid)
 
     lastjailid := 0
     cslastjidval := C.CString(strconv.Itoa(lastjailid))
     defer C.free(unsafe.Pointer(cslastjidval))
 
-    C.jailparam_import(&params[3], cslastjidval)
+    C.jailparam_import(&params[4], cslastjidval)
 
     // loop on existing jails
     for lastjailid >= 0 {
         // get parameter values
-        lastjailid = int(C.jailparam_get(&params[0], 4, 0))
+        lastjailid = int(C.jailparam_get(&params[0], 5, 0))
         if lastjailid > 0 {
             nametmp := C.jailparam_export(&params[0])
             jl.Name = C.GoString(nametmp)
@@ -75,23 +80,28 @@ func GetJails() ([]Jail, error) {
             // Memory mgmt : Non gere par Go
             C.free(unsafe.Pointer(jidtmp))
             
-			pathtmp := C.jailparam_export(&params[2])
+            pathtmp := C.jailparam_export(&params[2])
             jl.Path = C.GoString(pathtmp)
             // Memory mgmt : Non gere par Go
             C.free(unsafe.Pointer(pathtmp))
 
+            drstmp := C.jailparam_export(&params[3])
+            jl.Devfs_ruleset, _ = strconv.Atoi(C.GoString(drstmp))
+            // Memory mgmt : Non gere par Go
+            C.free(unsafe.Pointer(drstmp))
+
             jls = append(jls, jl)
             //log.Debug("Got jid " + strconv.Itoa(jl.jid) + " with name " + jl.name)
 
             // Prepare next loop iteration
             cslastjidval := C.CString(strconv.Itoa(lastjailid))
             defer C.free(unsafe.Pointer(cslastjidval))
-            C.jailparam_import(&params[3], cslastjidval)
+            C.jailparam_import(&params[4], cslastjidval)
         }
     }
 
-	// Free 4 items of params list
-    C.jailparam_free(&params[0], 4)
+	// Free 5 items of params list
+    C.jailparam_free(&params[0], 5)
 
     return jls, err
 }

main.go

@@ -1,10 +1,9 @@
 package main
 
 import (
-    "gocage/cmd"
+	"gocage/cmd"
 )
 
-func main () {
-    cmd.Execute()
+func main() {
+	cmd.Execute()
 }
-

service/gocage

@@ -0,0 +1,43 @@
+#!/bin/sh
+#
+# $FreeBSD$
+#
+
+# PROVIDE: gocage
+# REQUIRE: LOGIN cleanvar
+# KEYWORD: shutdown
+
+# Add the following lines to /etc/rc.conf to enable :
+#
+# gocage_enable="YES"
+#
+# gocage_conf="/usr/local/etc/gocage.conf.yml"
+#
+
+. /etc/rc.subr
+
+name="gocage"
+rcvar=gocage_enable
+
+# read configuration and set defaults
+load_rc_config "$name"
+
+: ${gocage_enable:="NO"}
+: ${gocage_conf="/usr/local/etc/gocage.conf.yml"}
+
+start_cmd=${name}_start
+stop_cmd=${name}_stop
+
+gocage_start()
+{
+    echo "Gocage starting jails... "
+    /usr/local/bin/gocage -c ${gocage_conf} start
+}
+
+gocage_stop()
+{
+    echo "Gocage stopping jails... "
+    /usr/local/bin/gocage  -c ${gocage_conf} stop
+}
+
+run_rc_command "$1"