Add Devfs_ruleset property to reflect generated RS

README.md

@@ -4,8 +4,9 @@ GoCage
 Jail management tool for FreeBSD, written in Go.  
 Support iocage jails, so they can coexist.  
 Gocage is meant to be a complete jail management tool with network, snapshots, jail cloning support and a web interface. This is the hypothetic future.  
+Gocage can handle multiple datastores, so you can have jails on HDD storage and jails on SSD storage.  
+
 
-At present time, it list and stops jails.  
 
 List jails
 ----------
@@ -118,3 +119,72 @@ Stop jails
 ----------
 `gocage stop test`
 
+
+Multi datastore  
+----------
+A datastore is a ZFS dataset mounted. It should be declared in gocage.conf.yml, specifying its ZFS mountpoint :
+<pre><code>
+datastore:
+  - /iocage
+  - /fastiocage
+</pre></code>
+
+In gocage commands, datastore name is the mountpoint without its "/" prefix.  
+
+### List datastores
+<pre><code>
+gocage datastore list 
++============+=============+============+===========+==========+============+
+| Name       | Mountpoint  | ZFSDataset | Available | Used     | Referenced |
++============+=============+============+===========+==========+============+
+| iocage     | /iocage     | hdd/iocage | 1.6 TB    | 414.9 GB | 27.5 KB    |
++------------+-------------+------------+-----------+----------+------------+
+| fastiocage | /fastiocage | ssd/iocage | 1.5 TB    | 65.3 KB  | 34.6 KB    |
++------------+-------------+------------+-----------+----------+------------+
+</pre></code>
+
+### Filter datastores  
+As with jails and snapshots, you can filter by name:  
+<pre><code>
+gocage datastore list iocage
++============+=============+============+===========+==========+============+
+| Name       | Mountpoint  | ZFSDataset | Available | Used     | Referenced |
++============+=============+============+===========+==========+============+
+| iocage     | /iocage     | hdd/iocage | 1.6 TB    | 414.9 GB | 27.5 KB    |
++------------+-------------+------------+-----------+----------+------------+
+</pre></code>
+
+### Sort datastores  
+You can sort datastores:  
+<pre><code>
+gocage datastore list -s -Available
++============+=============+============+===========+==========+============+
+| Name       | Mountpoint  | ZFSDataset | Available | Used     | Referenced |
++============+=============+============+===========+==========+============+
+| iocage     | /iocage     | hdd/iocage | 1.6 TB    | 415.0 GB | 27.5 KB    |
++------------+-------------+------------+-----------+----------+------------+
+| fastiocage | /fastiocage | ssd/iocage | 1.5 TB    | 65.3 KB  | 34.6 KB    |
++------------+-------------+------------+-----------+----------+------------+
+</pre></code>
+
+See [cmd/struct.go](https://git.nosd.in/yo/gocage/src/branch/master/cmd/struct.go) for field names.
+
+Migrating jails  
+----------
+With multi datastore comes the need to migrate a jail between datastores.  
+Migration can be done with a minimal downtime, using zfs differential send/receive.  
+Source jail datasets are sent to the destination datastore, jail is stopped and a last differential sync is done before starting jail on new datastore.  
+
+### Warning 
+Be aware the moment you migrate a jail to another datastore than /iocage default, you lose compatibility with iocage.  
+Then you need to disable iocage service, and enable gocage so the jails will start automatically at boot.  
+Also make sure, if you don't destroy source jail, that it won't have the "boot" property set or you will have the 2 jails up at boot.  
+
+<pre><code>
+gocage migrate -d fastiocage srv-random
+Snapshot data/iocage/jails/srv-random: Done
+Snapshot data/iocage/jails/srv-random/root: Done
+Migrate jail config dataset to fastdata/iocage/jails/srv-random: Done
+Migrate jail filesystem dataset to fastdata/iocage/jails/srv-random/root: Done
+</pre></code>
+

cmd/console.go

@@ -0,0 +1,46 @@
+package cmd
+
+import (
+	"os"
+	"fmt"
+	"log"
+	"errors"
+	"strconv"
+	"syscall"
+)
+
+func ShellJail(args []string) error {
+	// We cant shell more than one jail bc we replace gocage execution with jexec, so there wont be no return to gocage
+	if len(args) > 0 {
+		cj, err := getJailFromArray(args[0], gJails)
+		if err != nil {
+			fmt.Printf("Error getting jail %s: %v\n", args[0], err)
+			return err
+		}
+		if err := shellJail(cj); err != nil {
+			fmt.Printf("%v\n", err)
+			return err
+		}
+	}
+	
+	return nil
+}
+
+func shellJail(jail *Jail) error {
+	if false == jail.Running {
+		return errors.New("Jail is not running")
+	}
+	
+	jid := strconv.Itoa(jail.JID)
+
+	//err := syscall.Exec("/usr/sbin/jexec", []string{"jexec", jid, "/bin/csh"}, os.Environ())
+	err := syscall.Exec("/usr/sbin/jexec", []string{"jexec", jid, "login", "-f", "root"}, os.Environ())
+
+	// We should never get here, as syscall.Exec replace the gocage binary execution with jexec
+	// This means the moment syscall.Exec fires, gocage execution halt.
+	if err != nil {
+		log.Printf("Exec returned %v\n", err)
+	}
+	
+	return nil
+}

cmd/datastore.go

@@ -0,0 +1,108 @@
+package cmd
+
+import (
+	"fmt"
+	"errors"
+	"reflect"
+	"strconv"
+	"strings"
+) 
+
+func ListDatastores(args []string, display bool) error {
+	/***************************************************************
+	/   Filter datastores by names given on command line
+	/**************************************************************/
+	for _, d := range args {
+		cmd := fmt.Sprintf("zfs list -p -H -o name,used,available,referenced %s", d)
+		out, err := executeCommand(cmd)
+		if err != nil {
+			if strings.HasSuffix(err.Error(), "No such file or directory") {
+				return errors.New(fmt.Sprintf("Datastore does not exist: %s", d))
+			} else {
+				return errors.New(fmt.Sprintf("Error executing command %s: %v; command returned: %s\n", cmd, err, out))
+			}
+		}
+		
+		fields := strings.Fields(out)
+		if len(fields) < 4 {
+			return errors.New(fmt.Sprintf("Error parsing output of \"%s\": Not enough fields", cmd))
+		}
+		u, _ := strconv.ParseUint(fields[1], 10, 64)
+		a, _ := strconv.ParseUint(fields[2], 10, 64)
+		r, _ := strconv.ParseUint(fields[3], 10, 64)
+		ds := Datastore{Name: d[1:], Mountpoint: d, ZFSDataset: fields[0], Used: u, Referenced: r, Available: a}
+		
+		err = loadDefaultsForDatastore(&ds)
+		if err != nil {
+			return err
+		}
+		
+		gDatastores = append(gDatastores, ds)
+	}
+	
+	fields := strings.Split(gDisplayDColumns, ",")
+
+	if true == display {
+		/***************************************************************
+		/   Sort datastores
+		/     We support 3 sort criteria max
+		/**************************************************************/
+		if len(gSortDSFields) > 0 && gSortDSFields != "none" {		
+			ds := initDatastoreSortStruct()
+
+			// The way we manage criteria quantity is not very elegant...
+			var fct1, fct2, fct3 *reflect.Value
+			for i, c := range strings.Split(gSortDSFields, ",") {
+				var fctName string
+				if strings.HasPrefix(c, "-") {
+					fctName = fmt.Sprintf("%sDec", strings.Replace(c, "-", "", 1))
+				} else { // Par defaut (pas de prefix +/-) on considere un tri incremental
+					fctName = fmt.Sprintf("%sInc", strings.Replace(c, "+", "", 1))
+				}
+
+				// Get function by its name
+				fct, _, err := getStructFieldValue(ds, fctName)
+				
+				if err != nil {
+					fieldName := strings.Replace(strings.Replace(c, "-", "", 1), "+", "", 1)
+					return errors.New(fmt.Sprintf("ERROR getting DatastoreSort struct field %s. Please check the field name: %s\n", fctName, fieldName))
+				}
+				switch i + 1 {
+				case 1:
+					fct1 = fct
+				case 2:
+					fct2 = fct
+				case 3:
+					fct3 = fct
+				}
+			}
+
+			switch len(strings.Split(gSortDSFields, ",")) {
+			case 1:
+				DatastoresOrderedBy(fct1.Interface().(datastoreLessFunc)).Sort(gDatastores)
+			case 2:
+				DatastoresOrderedBy(fct1.Interface().(datastoreLessFunc), fct2.Interface().(datastoreLessFunc)).Sort(gDatastores)
+			case 3:
+				DatastoresOrderedBy(fct1.Interface().(datastoreLessFunc), fct2.Interface().(datastoreLessFunc), fct3.Interface().(datastoreLessFunc)).Sort(gDatastores)
+			}
+		}
+		displayDatastoresFields(gDatastores, fields)
+	}
+	
+	return nil
+}
+
+/********************************************************************************
+ * Load jails default config from $datastore/defaults.json
+ *******************************************************************************/
+func loadDefaultsForDatastore(ds *Datastore) error {
+	jc, err := getJailConfig(fmt.Sprintf("%s/defaults.json", ds.Mountpoint))
+	if err != nil {
+		return err
+	}
+
+	ds.DefaultJailConfig = jc
+	//gDefaultConfig = append(gDefaultConfig, jc)
+
+	return nil
+}

cmd/jailhost.go

@@ -0,0 +1,229 @@
+package cmd
+
+import (
+	"fmt"
+	"net"
+	"regexp"
+	"strconv"
+	"strings"
+	"io/ioutil"
+	"golang.org/x/net/route"
+)
+
+var defaultRoute4 = [4]byte{0, 0, 0, 0}
+var defaultRoute6 = [16]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0}
+var local6 = [16]byte{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1}
+
+func Inet4AddrToString(ip4 route.Inet4Addr) string {
+	return fmt.Sprintf("%v.%v.%v.%v", ip4.IP[0], ip4.IP[1], ip4.IP[2], ip4.IP[3])
+}
+
+func Inet6AddrToString(ip6 route.Inet6Addr) string {
+	return fmt.Sprintf("%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x:%02x%02x",
+		ip6.IP[0], ip6.IP[1], ip6.IP[2], ip6.IP[3], ip6.IP[4], ip6.IP[5], ip6.IP[6], ip6.IP[7],
+		ip6.IP[8], ip6.IP[9], ip6.IP[10], ip6.IP[11], ip6.IP[12], ip6.IP[13], ip6.IP[14], ip6.IP[15])
+}
+
+/*****************************************************************************
+* Initialize default_interface, default_gatway4, default_gateway6
+*****************************************************************************/
+func (jh *JailHost) InitNetworkProperties() {
+	rib, _ := route.FetchRIB(0, route.RIBTypeRoute, 0)
+	messages, err := route.ParseRIB(route.RIBTypeRoute, rib)
+	if err != nil {
+		panic(err)
+	}
+
+	for _, message := range messages {
+		route_message := message.(*route.RouteMessage)
+		addresses := route_message.Addrs
+		card_index := route_message.Index
+
+		if addresses[0].Family() == 2 {
+			var destination4, gateway4 *route.Inet4Addr
+			ok := false
+
+			if destination4, ok = addresses[0].(*route.Inet4Addr); !ok {
+				continue
+			}
+
+			if gateway4, ok = addresses[1].(*route.Inet4Addr); !ok {
+				continue
+			}
+
+			if destination4 == nil || gateway4 == nil {
+				continue
+			}
+
+			if destination4.IP == defaultRoute4 {
+				card, _ := net.InterfaceByIndex(card_index)
+				//fmt.Printf("Default IPv4 gateway is %v on card %s\n", Inet4AddrToString(*gateway4), card.Name)
+				jh.default_interface = card.Name
+				jh.default_gateway4 = Inet4AddrToString(*gateway4)
+			}
+		} else if addresses[0].Family() == 28 {
+			var destination6, gateway6 *route.Inet6Addr
+			ok := false
+
+			if destination6, ok = addresses[0].(*route.Inet6Addr); !ok {
+				continue
+			}
+
+			if gateway6, ok = addresses[1].(*route.Inet6Addr); !ok {
+				continue
+			}
+
+			if destination6 == nil || gateway6 == nil {
+				continue
+			}
+
+			if destination6.IP == defaultRoute6 && gateway6.IP != local6 {
+				card, _ := net.InterfaceByIndex(card_index)
+				//fmt.Printf("Default IPv6 gateway is %v on card %s\n", Inet6AddrToString(*gateway6), card.Name)
+				jh.default_interface = card.Name
+				jh.default_gateway6 = Inet6AddrToString(*gateway6)
+			}
+		}
+	}
+}
+
+func (jh *JailHost) GetDefaultInterface() string {
+	if len(jh.default_interface) == 0 {
+		jh.InitNetworkProperties()
+	}
+	return jh.default_interface
+}
+
+func (jh *JailHost) GetDefaultGateway4() string {
+	if len(jh.default_gateway4) == 0 {
+		jh.InitNetworkProperties()
+	}
+	return jh.default_gateway4
+}
+
+func (jh *JailHost) GetDefaultGateway6() string {
+	if len(jh.default_gateway6) == 0 {
+		jh.InitNetworkProperties()
+	}
+	return jh.default_gateway6
+}
+
+/*****************************************************************************
+ * Get all network interfaces
+ ****************************************************************************/
+func (jh *JailHost) GetInterfaces() ([]string, error) {
+	var names []string
+	
+	interfaces, err := net.Interfaces()
+	if err != nil {
+		return names, fmt.Errorf("Error listing network interfaces: %v", err)
+	}
+	for _, n := range interfaces {
+		names = append(names, n.Name)
+	}
+	
+	return names, nil
+}
+
+func (jh *JailHost) GetBridgeMTU(bridgeName string) (int, error) {
+	bridge, err := net.InterfaceByName(bridgeName)
+	if err != nil {
+		return 0, err
+	}
+	
+	return bridge.MTU, nil
+}
+
+/*****************************************************************************
+ * Get all IPv4 currently in use on host
+ ****************************************************************************/
+func getHostInUseIPv4() ([]string, error) {
+	var ips []string
+
+	re := regexp.MustCompile(ifconfigipv4re)
+
+	out, err := executeCommand("/sbin/ifconfig")
+	if err != nil {
+		return ips, fmt.Errorf("Error executing \"/sbin/ifconfig\": %s", err)
+	}
+
+	for _, line := range strings.Split(out, "\n") {
+		if re.MatchString(line) {
+			ips = append(ips, re.FindStringSubmatch(line)[1])
+		}
+	}
+
+	return ips, nil
+}
+
+func getHostname() (string, error) {
+	out, err := executeCommand("/bin/hostname")
+	if err != nil {
+		return "", fmt.Errorf("Error executing \"/bin/hostname\": %v", err)
+	}
+	
+	return strings.Split(out, "\n")[0], nil
+}
+
+func getHostId() (string, error) {
+	var content []byte
+	var err     error
+	// return empty string if file does not exist
+	if content, err = ioutil.ReadFile("/etc/hostid"); err != nil {
+		if strings.HasSuffix(err.Error(), "no such file or directory") {
+			return "", nil
+		}
+		return "", err
+	}
+	
+	return strings.Split(string(content), "\n")[0], nil
+}
+
+func getFreeBSDVersion() (FreeBSDVersion, error) {
+	var version FreeBSDVersion
+	regex := `([0-9]{1,2})(\.)?([0-9]{1,2})?\-([^\-]*)(\-)?(p[0-9]{1,2})?`
+	
+	re := regexp.MustCompile(regex)
+	
+	out, err := executeCommand("/bin/freebsd-version")
+	if err != nil {
+		return version, fmt.Errorf("Error executing \"/bin/freebsd-version\": %v", err)
+	}
+	
+	if re.MatchString(out) {
+		version.major, err = strconv.Atoi(re.FindStringSubmatch(out)[1])
+		if err != nil {
+			return version, err
+		}
+		version.minor, err = strconv.Atoi(re.FindStringSubmatch(out)[3])
+		if err != nil {
+			return version, err
+		}
+		version.flavor = strings.Trim(re.FindStringSubmatch(out)[4], "\n")
+		
+		// Skip the 'p' starting patch level
+		if len(re.FindStringSubmatch(out)[6]) > 0 {
+			version.patchLevel, err = strconv.Atoi(re.FindStringSubmatch(out)[6][1:])
+			if err != nil {
+				return version, err
+			}
+		}
+	}
+	return version, nil
+}
+
+func NewJailHost() (JailHost, error) {
+	var jh  JailHost
+	var err error
+	
+	if jh.hostname, err = getHostname(); err != nil {
+		return jh, err
+	}
+	if jh.hostid, err = getHostId(); err != nil {
+		return jh, err
+	}
+	if jh.version, err = getFreeBSDVersion(); err != nil {
+		return jh, err
+	}
+	return jh, nil
+}

cmd/list.go

@@ -1,15 +1,14 @@
 package cmd
 
 import (
-	"os"
+	"encoding/json"
 	"fmt"
+	"gocage/jail"
+	"io/ioutil"
 	"log"
+	"os"
 	"reflect"
 	"strings"
-	"io/ioutil"
-	"gocage/jail"
-	"encoding/json"
-	"github.com/spf13/viper"
 )
 
 /********************************************************************************
@@ -18,29 +17,99 @@ import (
  *******************************************************************************/
 func ListJailsProps(args []string) {
 	var conf Jail
-	var jailconf JailConfig
 	var result []string
-	
+	// Mandatory constructor to init default values
+	jailconf, err := NewJailConfig()
+	if err != nil {
+		fmt.Printf("Error allocating JailConfig: %s\n", err.Error())
+		return
+	}
+
 	conf.Config = jailconf
-	
+
 	result = getStructFieldNames(conf, result, "")
-	
+
 	for _, f := range result {
 		fmt.Printf("%s\n", f)
 	}
 }
 
 /********************************************************************************
- * Get Jails from datastores. Store config and running metadata 
+ * Get Jails from datastores. Store config and running metadata
  *   into gJails global var
  *******************************************************************************/
 func ListJails(args []string, display bool) {
-	fields := strings.Split(gDisplayColumns, ",")
-
-	for _, d := range viper.GetStringSlice("datastore") {
-		listJailsFromDatastore(d)
+	type uniqueJailName struct {
+		jail       string
+		unique     bool
+		uniqueName bool
 	}
-
+	var nameChecked []*uniqueJailName
+	var curCheck *uniqueJailName
+	var found bool
+	var skip bool
+	
+	for _, ds := range gDatastores {
+		listJailsFromDatastore(ds, args, display)
+	}
+	
+	// Only when displaying jails, we accept to process multiple same name jails
+	if false == display {
+		for _, j := range gJails {
+			// If already checked and was using distinctive name, skip this occurence
+			for i, n := range nameChecked {
+				if strings.EqualFold(n.jail, j.Name) {
+					found = true
+					if false == n.unique && true == n.uniqueName {
+						skip = true
+					} else {
+						curCheck = nameChecked[i]
+					}
+					break
+				}
+			}
+			
+			if true == skip {
+				continue
+			}
+			
+			// Initialize if not found in nameChecked
+			if false == found {
+				curCheck = &uniqueJailName{jail: j.Name,
+									unique: true,
+									uniqueName: false}
+			} else {
+				found = false
+			}
+			
+			if countOfJailsWithThisName(j.Name) > 1 {
+				//fmt.Printf("DEBUG: Jail %s exist multiple times, now checking if specified with full name\n", j.Name)
+				curCheck.unique = false
+				
+				for _, a := range args {
+					//fmt.Printf("DEBUG: comparing %s/%s with %s\n", j.Datastore, j.Name, a)
+					if strings.EqualFold(a, fmt.Sprintf("%s/%s", j.Datastore, j.Name)) {
+						//fmt.Printf("DEBUG: Was specified with full name, GG!\n")
+						curCheck.uniqueName = true
+					}
+				}
+			}
+			nameChecked = append(nameChecked, curCheck)
+		}
+		
+		// Now check
+		for _, a := range args {
+			for _, n := range nameChecked {
+				if strings.EqualFold(n.jail, a) && false == n.unique && false == n.uniqueName {
+					fmt.Printf("There is more than one jail named \"%s\", please use datastore/jail format\n", n.jail)
+					os.Exit(1)
+				}
+			}
+		}
+	}
+	
+	fields := strings.Split(gDisplayJColumns, ",")
+	
 	// This is the structure we will filter, then display
 	var jails []Jail
 
@@ -77,7 +146,6 @@ func ListJails(args []string, display bool) {
 		jails = gJails
 	}
 
-
 	/***************************************************************
 	/   Filter jails by names given on command line
 	/**************************************************************/
@@ -87,7 +155,6 @@ func ListJails(args []string, display bool) {
 			for _, j := range jails {
 				if j.Name == a {
 					js = append(js, j)
-					break
 				}
 			}
 		}
@@ -98,19 +165,19 @@ func ListJails(args []string, display bool) {
 	/   Sort jails
 	/     We support 3 sort criteria max
 	/**************************************************************/
-	if len(gSortFields) > 0 && gSortFields != "none" {
+	if len(gSortJailFields) > 0 && gSortJailFields != "none" {
 		js := initJailSortStruct()
 
 		// The way we manage criteria quantity is not very elegant...
 		var fct1, fct2, fct3 *reflect.Value
-		for i, c := range strings.Split(gSortFields, ",") {
+		for i, c := range strings.Split(gSortJailFields, ",") {
 			var fctName string
 			if strings.HasPrefix(c, "-") {
 				fctName = fmt.Sprintf("%sDec", strings.Replace(c, "-", "", 1))
-			} else {	// Par defaut (pas de prefix +/-) on considere un tri incremental	
+			} else { // Par defaut (pas de prefix +/-) on considere un tri incremental
 				fctName = fmt.Sprintf("%sInc", strings.Replace(c, "+", "", 1))
 			}
-			
+
 			// Get function by its name
 			fct, _, err := getStructFieldValue(js, fctName)
 			if err != nil {
@@ -118,20 +185,23 @@ func ListJails(args []string, display bool) {
 				fmt.Printf("ERROR getting JailSort struct field %s. Please check the field name: %s\n", fctName, fieldName)
 				return
 			}
-			switch i+1 {
-				case 1:	fct1 = fct
-				case 2: fct2 = fct
-				case 3: fct3 = fct
+			switch i + 1 {
+			case 1:
+				fct1 = fct
+			case 2:
+				fct2 = fct
+			case 3:
+				fct3 = fct
 			}
 		}
 
-		switch len(strings.Split(gSortFields, ",")) {
-			case 1:
-				JailsOrderedBy(fct1.Interface().(jailLessFunc)).Sort(jails)
-			case 2:
-				JailsOrderedBy(fct1.Interface().(jailLessFunc), fct2.Interface().(jailLessFunc)).Sort(jails)
-			case 3:
-				JailsOrderedBy(fct1.Interface().(jailLessFunc), fct2.Interface().(jailLessFunc), fct3.Interface().(jailLessFunc)).Sort(jails)
+		switch len(strings.Split(gSortJailFields, ",")) {
+		case 1:
+			JailsOrderedBy(fct1.Interface().(jailLessFunc)).Sort(jails)
+		case 2:
+			JailsOrderedBy(fct1.Interface().(jailLessFunc), fct2.Interface().(jailLessFunc)).Sort(jails)
+		case 3:
+			JailsOrderedBy(fct1.Interface().(jailLessFunc), fct2.Interface().(jailLessFunc), fct3.Interface().(jailLessFunc)).Sort(jails)
 		}
 	}
 
@@ -143,57 +213,73 @@ func ListJails(args []string, display bool) {
 	}
 }
 
-
-func listJailsFromDatastore(datastore string) {
-	fileInfo, err := os.Stat(datastore)
-	if err != nil { log.Fatalln(fmt.Sprintf("Unable to access %s, check path and/or rights", datastore)) }
-	if fileInfo.IsDir() == false { log.Fatalln(fmt.Sprintf("%s is not a directory", datastore)) }
+func listJailsFromDatastore(ds Datastore, args []string, accept_multiple bool) {
+	
+	fileInfo, err := os.Stat(ds.Mountpoint)
+	if err != nil {
+		log.Fatalln(fmt.Sprintf("Unable to access %s, check path and/or rights", ds.Mountpoint))
+	}
+	if fileInfo.IsDir() == false {
+		log.Fatalln(fmt.Sprintf("%s is not a directory", ds.Mountpoint))
+	}
 
 	// A datastore have to contain a "jails" directory
-	jailsDir := fmt.Sprintf("%s/jails", datastore)
+	jailsDir := fmt.Sprintf("%s/jails", ds.Mountpoint)
 	fileInfo, err = os.Stat(jailsDir)
-	if err != nil { log.Fatalln(fmt.Sprintf("Unable to access %s, check path and/or rights", jailsDir)) }
-	if fileInfo.IsDir() == false { log.Fatalln(fmt.Sprintf("%s is not a directory", jailsDir)) }
+	if err != nil {
+		log.Fatalln(fmt.Sprintf("Unable to access %s, check path and/or rights", jailsDir))
+	}
+	if fileInfo.IsDir() == false {
+		log.Fatalln(fmt.Sprintf("%s is not a directory", jailsDir))
+	}
 
-	listJailsFromDirectory(jailsDir)
+	listJailsFromDirectory(jailsDir, ds.Name)
 }
 
-
-func listJailsFromDirectory(dir string) ([]Jail) {
+func listJailsFromDirectory(dir string, dsname string) ([]Jail, error) {
 	files, err := ioutil.ReadDir(dir)
-	if err != nil { log.Fatalln(fmt.Sprintf("Unable to browse %s, check path and/or rights", dir)) }
+	if err != nil {
+		log.Fatalln(fmt.Sprintf("Unable to browse %s, check path and/or rights", dir))
+	}
 
 	for _, fi := range files {
 		if fi.IsDir() == true {
 
-		// 1. Get conf from config.json
+			// 1. Get conf from config.json
 			jailConfPath := fmt.Sprintf("%s/%s/%s", dir, fi.Name(), "config.json")
 			jailConf, err := getJailConfig(jailConfPath)
-			if err != nil { log.Println("ERROR reading jail config for %s", jailConfPath) }
-	
-		// 2. Build jail object from config
-			jailRootPath := fmt.Sprintf("%s/%s/%s", dir, fi.Name(), "root")
-			j := Jail{
-				Name: jailConf.Host_hostname,
-				Config: jailConf,
-				ConfigPath: jailConfPath,
-				RootPath: jailRootPath,
-				Running: false,
+			if err != nil {
+				log.Println("ERROR reading jail config for %s", jailConfPath)
 			}
 
-		// 3. Add current running informations
+			// 2. Build jail object from config
+			jailRootPath := fmt.Sprintf("%s/%s/%s", dir, fi.Name(), "root")
+			j := Jail{
+				Name:       jailConf.Host_hostuuid,
+				Config:     jailConf,
+				ConfigPath: jailConfPath,
+				Datastore:  dsname,
+				RootPath:   jailRootPath,
+				Running:    false,
+			}
+
+			// 3. Add current running informations
 			rjails, err := jail.GetJails()
-			if err != nil { log.Fatalln("Unable to list running jails") }
+			if err != nil {
+				log.Fatalln("Unable to list running jails")
+			}
 			for _, rj := range rjails {
 				if rj.Path == j.RootPath {
 					j.JID = rj.Jid
 					j.Running = true
 					j.InternalName = rj.Name
+					j.Devfs_ruleset = rj.Devfs_ruleset
 					break
 				}
 			}
-	
-/* This op take some 600ms for ~40 jails :^( */
+
+			// 4. Get Zpool
+			/* This op take some 600ms for ~40 jails :^( */
 			out, err := executeCommand(fmt.Sprintf("zfs list -H -o name %s", j.RootPath))
 			if err != nil {
 				fmt.Printf("ERROR getting dataset from %s: %s\n", j.RootPath, err.Error())
@@ -201,21 +287,40 @@ func listJailsFromDirectory(dir string) ([]Jail) {
 				j.Zpool = strings.Split(strings.TrimSuffix(out, "\n"), "/")[0]
 			}
 
+			// Check if jail with the same name already exist on another DS
+			for _, jj := range gJails {
+				if strings.EqualFold(jj.Name, j.Name) {
+					/*fmt.Printf("Warning: A jail with name %s exist on datastore %s, use full name to interact!\n", j.Name, jj.Datastore)
+					fmt.Printf("(Ex: gocage start %s/%s)\n", jj.Datastore, j.Name)*/
+					// Add Datastore  to avoid confusion
+					if false == strings.Contains(gDisplayJColumns, "Datastore") {
+						gDisplayJColumns += ",Datastore"
+					}
+				}
+			}
+			
 			gJails = append(gJails, j)
 		}
 	}
 
-	return gJails
+	return gJails, nil
 }
 
-
 func getJailConfig(jailConfigPath string) (JailConfig, error) {
 	content, err := ioutil.ReadFile(jailConfigPath)
-	if err != nil { log.Fatalln(fmt.Sprintf("Unable to read %s, check path and/or rights", jailConfigPath)) }
+	if err != nil {
+		log.Fatalln(fmt.Sprintf("Unable to read %s, check path and/or rights", jailConfigPath))
+	}
 
-	var jc JailConfig
+	// Mandatory constructor to init default values
+	jc, err := NewJailConfig()
+	if err != nil {
+		return jc, err
+	}
 	err = json.Unmarshal([]byte(content), &jc)
-	if err != nil { log.Fatalln(fmt.Sprintf("Error occured during unmarshaling %s: %s", jailConfigPath, err.Error())) }
+	if err != nil {
+		log.Fatalln(fmt.Sprintf("Error occured during unmarshaling %s: %s", jailConfigPath, err.Error()))
+	}
 
 	return jc, err
 }

cmd/migrate.go

@@ -0,0 +1,224 @@
+package cmd
+
+import (
+	"os"
+	"fmt"
+	"bufio"
+	"errors"
+	"strings"
+)
+
+const (
+	// 1MB
+	BUFFER_SIZE = 512
+)
+
+/********************************************************************************
+ * Migrate jail to another zfs pool
+ *******************************************************************************/
+func MigrateJail(args []string) {
+	var jailNames []string
+
+	if len(args) > 0 {
+		for _, a := range args {
+			jailNames = append(jailNames, a)
+		}
+	}
+
+	for _, jn := range jailNames {
+		cj, err := getJailFromArray(jn, gJails)
+		if cj == nil {
+			fmt.Printf("Error getting jail %s: Not found\n", jn)
+			return
+		}
+		
+		// Check if destination datastore exist & get current DS
+		destDS, err := getDatastoreFromArray(gMigrateDestDatastore, gDatastores)
+		if err != nil {
+			fmt.Printf("Error getting datastore \"%s\": %v\n", gMigrateDestDatastore, err)
+			return
+		}
+		curDS, err := getDatastoreFromArray(cj.Datastore, gDatastores)
+		if err != nil {
+			fmt.Printf("Error getting datastore \"%s\": %v\n", gMigrateDestDatastore, err)
+			return
+		}
+		
+		if cj.Running == true && gYesToAll == false {
+			fmt.Printf("WARNING: Jail %s is running\n", cj.Name)
+			fmt.Printf("Migration will stop it for data sync before starting on the new pool. You will be prompted for shutdown.\n")
+			fmt.Printf("Continue? (y/n) ")
+			scanr := bufio.NewScanner(os.Stdin)
+			scanr.Scan()
+			if false == strings.EqualFold(scanr.Text(), "y") {
+				fmt.Printf("Migration aborted\n")
+				return
+			}
+		}
+
+		/* TODO : Check dest pool (gMigrateDestDatastore) existence
+				zfs snapshot /iocage/jails/$jail@gocage_mig_first_snap
+				zfs snapshot /iocage/jails/$jail/root@gocage_mig_first_snap
+				zfs send jail@gocage_mig_first_snap | zfs receive destpool/jails/jail_name
+				zfs send jail/root@gocage_mig_first_snap | zfs receive destpool/jails/jail_name/root
+				shutdown jail if needed
+				if jail was shutdown
+					zfs snapshot /iocage/jails/$jail@gocage_mig_last_snap to get last data
+					zfs send -i jail@gocage_mig_first_snap jail/root@gocage_mig_last_snap | zfs receive -F destpool/jails/jail_name
+					start jail on new dest
+					zfs destroy destpool/jails/jail_name@gocage_mig_first_snap
+					zfs destroy destpool/jails/jail_name/root@gocage_mig_first_snap
+		*/
+
+		// Snapshot config
+		dsconf := strings.Join([]string{curDS.ZFSDataset, "jails", jn}, "/")
+		fmt.Printf("Snapshot %s: ", dsconf)
+		if err = zfsSnapshot(dsconf, "gocage_mig_init"); err != nil {
+			fmt.Printf("Error: %v\n", err)
+			return
+		}
+		fmt.Printf("Done\n")
+		
+		// Snapshot filesystem
+		dsdata := strings.Join([]string{curDS.ZFSDataset, "jails", jn, "root"}, "/")
+		fmt.Printf("Snapshot %s: ", dsdata)
+		if err := zfsSnapshot(dsdata, "gocage_mig_init"); err != nil {
+			fmt.Printf("Error: %v\n", err)
+			return
+		}
+		fmt.Printf("Done\n")
+		
+		
+		// TODO: Browse jailed ZFS dataset to migrate them (see poudriere for example)
+		// zfs send -R src/poudriere@init | zfs receive -Fu dest/poudriere
+		
+		dsconfdest := strings.Join([]string{destDS.ZFSDataset, "jails", jn}, "/")
+		fmt.Printf("Migrate jail config dataset to %s: ", dsconfdest)
+		if err := zfsCopy(fmt.Sprintf("%s@gocage_mig_init", dsconf), dsconfdest); err != nil {
+			fmt.Printf("Error: %v\n", err)
+		}
+		fmt.Printf("Done\n")
+		
+		dsdatadest := strings.Join([]string{destDS.ZFSDataset, "jails", jn, "root"}, "/")
+		fmt.Printf("Migrate jail filesystem dataset to %s: ", dsdatadest)
+		if err := zfsCopy(fmt.Sprintf("%s@gocage_mig_init", dsdata), dsdatadest); err != nil {
+			fmt.Printf("Error: %v\n", err)
+		}
+		fmt.Printf("Done\n")
+		
+		// Running jail needs a last snapshot for an incremental send/recv after shutting down.
+		if cj.Running == true {
+			fmt.Printf("Shutdown jail %s for last data sync, this could take some time.\n", cj.Name)
+			if gYesToAll == false {
+				fmt.Printf("Continue? (y/n) ")
+				scanr := bufio.NewScanner(os.Stdin)
+				scanr.Scan()
+				if false == strings.EqualFold(scanr.Text(), "y") {
+					fmt.Printf("Migration aborted. Now cleaning destination pool.\n")
+					if err := CleanMigrateMess([]string{cj.Name}); err != nil {
+						fmt.Printf("Error: %v\n", err)
+					}
+					// TODO : Remove destination datasets, or handle the cas at the beginning of current function
+					// (when snapshot already exist on source and dest)
+					return
+				}
+			}
+			
+			StopJail([]string{cj.Name})
+			
+			fmt.Printf("Snapshot %s: ", dsconf)
+			if err = zfsSnapshot(dsconf, "gocage_mig_last_sync"); err != nil {
+				fmt.Printf("Error: %v\n", err)
+				return
+			}
+			fmt.Printf("Done\n")
+			fmt.Printf("Snapshot %s: ", dsdata)
+			if err := zfsSnapshot(dsdata, "gocage_mig_last_sync"); err != nil {
+				fmt.Printf("Error: %v\n", err)
+				return
+			}
+			fmt.Printf("Done\n")
+			
+			fmt.Printf("Synchronize jail config to %s: ", dsconfdest)
+			if err := zfsCopyIncremental(fmt.Sprintf("%s@gocage_mig_init", dsconf), 
+										fmt.Sprintf("%s@gocage_mig_last_sync", dsconf),  
+										 dsconfdest); err != nil {
+				fmt.Printf("Error: %v\n", err)
+				return
+			}
+			fmt.Printf("Done\n")
+			
+			fmt.Printf("Synchronize jail filesystem dataset to %s: ", dsdatadest)
+			if err := zfsCopyIncremental(fmt.Sprintf("%s@gocage_mig_init", dsdata), 
+										fmt.Sprintf("%s@gocage_mig_last_sync", dsdata),  
+										dsdatadest); err != nil {
+				return
+				fmt.Printf("Error: %v\n", err)
+			}
+			fmt.Printf("Done\n")
+			
+			// TODO : Start jail on new datastore (! Currently ListJails won't support 2 jails with same name !)
+			
+			// TODO : zfs destroy destpool/jails/jail_name@gocage_mig_first_snap
+			// TODO : zfs destroy destpool/jails/jail_name/root@gocage_mig_first_snap
+			// TODO : zfs destroy -r srcpool/iocage/jails/$jail
+		}
+	}
+}
+
+// Clean snapshots from an aborted migration
+func CleanMigrateMess(args []string) error {
+	var jailNames []string
+
+	if len(args) > 0 {
+		for _, a := range args {
+			jailNames = append(jailNames, a)
+		}
+	}
+
+	for _, jn := range jailNames {
+		cj, err := getJailFromArray(jn, gJails)
+		if cj == nil {
+			return errors.New(fmt.Sprintf("Error getting jail %s: Not found\n", jn))
+		}
+		curDS, err := getDatastoreFromArray(cj.Datastore, gDatastores)
+		if err != nil {
+			return errors.New(fmt.Sprintf("Error getting datastore \"%s\": %v\n", cj.Datastore, err))
+		}
+		
+		cmd := fmt.Sprintf("zfs destroy %s@gocage_mig_init", strings.Join([]string{curDS.ZFSDataset, "jails", jn}, "/"))
+		out, err := executeCommand(cmd)
+		if err != nil {
+			if false == strings.HasSuffix(out, "could not find any snapshots to destroy; check snapshot names.\n") {
+				fmt.Printf("Error executing command %s: %v; command returned: %s\n", cmd, err, out)
+				return errors.New(fmt.Sprintf("Error executing command %s: %v; command returned: %s\n", cmd, err, out))
+			}
+		}
+		cmd = fmt.Sprintf("zfs destroy %s@gocage_mig_init", strings.Join([]string{curDS.ZFSDataset, "jails", jn, "root"}, "/"))
+		out, err = executeCommand(cmd)
+		if err != nil {
+			if false == strings.HasSuffix(out, "could not find any snapshots to destroy; check snapshot names.\n") {
+				fmt.Printf("Error executing command %s: %v; command returned: %s\n", cmd, err, out)
+				return errors.New(fmt.Sprintf("Error executing command %s: %v; command returned: %s\n", cmd, err, out))
+			}
+		}
+		cmd = fmt.Sprintf("zfs destroy %s@gocage_mig_last_sync", strings.Join([]string{curDS.ZFSDataset, "jails", jn}, "/"))
+		out, err = executeCommand(cmd)
+		if err != nil {
+			if false == strings.HasSuffix(out, "could not find any snapshots to destroy; check snapshot names.\n") {
+				fmt.Printf("Error executing command %s: %v; command returned: %s\n", cmd, err, out)
+				return errors.New(fmt.Sprintf("Error executing command %s: %v; command returned: %s\n", cmd, err, out))
+			}
+		}
+		cmd = fmt.Sprintf("zfs destroy %s@gocage_mig_last_sync", strings.Join([]string{curDS.ZFSDataset, "jails", jn, "root"}, "/"))
+		out, err = executeCommand(cmd)
+		if err != nil {
+			if false == strings.HasSuffix(out, "could not find any snapshots to destroy; check snapshot names.\n") {
+				fmt.Printf("Error executing command %s: %v; command returned: %s\n", cmd, err, out)
+				return errors.New(fmt.Sprintf("Error executing command %s: %v; command returned: %s\n", cmd, err, out))
+			}
+		}
+	}
+
+	return nil
+}

cmd/properties.go

@@ -1,28 +1,33 @@
 package cmd
 
 import (
-	"fmt"
 	"errors"
+	"fmt"
 	"reflect"
-	"strings"
 	"strconv"
+	"strings"
 )
 
 func GetJailProperties(args []string) {
 	var props []string
-	var jail Jail
-	
+	var jail *Jail
+	var err error
+
 	if len(args) > 0 {
 		for i, a := range args {
 			// Last arg is the jail name
 			if i == len(args)-1 {
-				jail.Name = a
+				jail, err = getJailFromArray(a, gJails)
+				if err != nil {
+					fmt.Printf("Error: %s\n", err.Error())
+					return
+				}
 			} else {
 				props = append(props, a)
 			}
 		}
-	} 
-	
+	}
+
 	if len(jail.Name) == 0 || len(args) == 0 {
 		// TODO : Show help
 		fmt.Printf("Error\n")
@@ -31,12 +36,12 @@ func GetJailProperties(args []string) {
 	
 	if isStringInArray(props, "all") {
 		var result []string
-		result = getStructFieldNames(jail, result, "")
+		result = getStructFieldNames(*jail, result, "")
 		props = result
 	}
-	
+
 	for _, p := range props {
-		v, err := getJailProperty(&jail, p)
+		v, err := getJailProperty(jail, p)
 		if err != nil {
 			fmt.Printf("Error: %s\n", err.Error())
 			return
@@ -47,38 +52,34 @@ func GetJailProperties(args []string) {
 }
 
 func getJailProperty(jail *Jail, propName string) (string, error) {
-	for i, j := range gJails {
-		if j.Name == jail.Name {
-			val, _, err := getStructFieldValue(&gJails[i], propName)
-			if err != nil {
-				return "", err
-			}
-			
-			switch val.Kind() {
-				case reflect.String:
-					return val.String(), nil
-				case reflect.Int:
-					return strconv.FormatInt(val.Int(), 10), nil
-				case reflect.Bool:
-					return strconv.FormatBool(val.Bool()), nil
-				default:
-					return "", errors.New(fmt.Sprintf("Error: Unknown type for property %s: %s\n", propName, val.Kind()))
-			}
-		}
+	val, _, err := getStructFieldValue(jail, propName)
+	if err != nil {
+		return "", err
 	}
-	
-	return "", errors.New(fmt.Sprintf("Jail not found: %s", jail.Name))
+
+	switch val.Kind() {
+	case reflect.String:
+		return val.String(), nil
+	case reflect.Int:
+		return strconv.FormatInt(val.Int(), 10), nil
+	case reflect.Bool:
+		return strconv.FormatBool(val.Bool()), nil
+	default:
+		return "", errors.New(fmt.Sprintf("Error: Unknown type for property %s: %s\n", propName, val.Kind()))
+	}
+
+	return "", errors.New("Unknown error in getJailProperty")
 }
 
 func SetJailProperties(args []string) {
 	type properties struct {
-		name	string
-		value	string
+		name  string
+		value string
 	}
-	
+
 	var jail Jail
 	var props []properties
-	
+
 	if len(args) > 0 {
 		for i, a := range args {
 			// This is the jail name
@@ -96,61 +97,27 @@ func SetJailProperties(args []string) {
 				}
 			}
 		}
-	} 
-	
+	}
+
 	if len(jail.Name) == 0 || len(args) == 0 {
 		// TODO : Show help
 		fmt.Printf("Error\n")
 		return
 	}
-	
-	for _, p := range props {
-		err := setJailProperty(&jail, p.name, p.value)
-		if err != nil {
-			fmt.Printf("Error: %s\n", err.Error())
-			return
-		}
-	}
-}
 
-// setJailProperty takes a string as propValue, whatever the real property type is.
-//  It will be converted.
-func setJailProperty(jail *Jail, propName string, propValue string) error {
-	for i, j := range gJails {
-		if j.Name == jail.Name {
-			val, _, err := getStructFieldValue(&gJails[i], propName)
-			//val, _, err := getStructFieldValue(&gJails[i].Config, strings.Split(propName, ".")[1])
-			if err != nil {
-				return err
-			}
-			
-			if val.CanSet() {
-				switch val.Kind() {
-					case reflect.String:
-						val.SetString(propValue)
-					case reflect.Int:
-						ival, err := strconv.ParseInt(propValue, 10, 64)
-						if err != nil {
-							return err
-						}
-						val.SetInt(ival)
-					case reflect.Bool:
-						bval, err := strconv.ParseBool(propValue)
-						if err != nil {
-							return err
-						}
-						val.SetBool(bval)
-					default:
-						return errors.New(fmt.Sprintf("Field is an unkown type: %s: %s", propName, val.Kind().String()))
+	// Get jail by index to modify it
+	for i, _ := range gJails {
+		if gJails[i].Name == jail.Name {
+			for _, p := range props {
+				err := setStructFieldValue(&gJails[i], p.name, p.value)
+				if err != nil {
+					fmt.Printf("Error: %s\n", err.Error())
+					return
+				} else {
+					fmt.Printf("%s: %s set to %s\n", gJails[i].Name, p.name, p.value)
+					gJails[i].ConfigUpdated = true
 				}
-				fmt.Printf("%s: %s set to %s\n", jail.Name, propName, propValue)
-				gJails[i].ConfigUpdated = true
-			} else {
-				return errors.New(fmt.Sprintf("Field is not writable : %s", propName))
 			}
 		}
 	}
-	
-	return nil
 }
-

cmd/root.go

@@ -1,131 +1,189 @@
 package cmd
 
 import (
-	"os"
-	"fmt"
-	"strconv"
-	"strings"
-	"io/ioutil"
 	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"strings"
 
 	"github.com/spf13/cobra"
 	"github.com/spf13/viper"
+	// TODO : Use log
+	log "github.com/sirupsen/logrus"
 )
 
 const (
-	gVersion = "0.25"
+	gVersion = "0.31"
+	
+	// TODO : Get from $jail_zpool/defaults.json
+	MIN_DYN_DEVFS_RULESET = 1000
 )
 
 var (
-	gJails			[]Jail
+	gJailHost   JailHost
+	gJails      []Jail
+	gDatastores []Datastore
 
-	gUseSudo		bool
+	gUseSudo bool
+	gForce   bool
+	gDebug   bool
 
-	gConfigFile		string
-	gDisplayColumns	string
-	gFilterJails	string
-	gSortFields		string
-	gNoLineSep		bool
+	gConfigFile      string
+	gDisplayJColumns string
+	gDisplaySColumns string
+	gDisplayDColumns string
+	gFilterJails     string
+	gFilterSnaps     string
+	gFilterDS        string
+	gSortJailFields  string
+	gSortSnapFields  string
+	gSortDSFields    string
+	gNoJailLineSep   bool
+	gNoSnapLineSep   bool
+	gNoDSLineSep     bool
 
-	gHostVersion	float64
-	
-	gTimeZone		string
-	gSnapshotName	string
+	gHostVersion float64
 
-	rootCmd = & cobra.Command {
-		Use:    "gocage",
-		Short:  "GoCage is a FreeBSD Jail management tool",
-		Long:   `GoCage is a jail management tool. It support VNET, host-only, NAT networks. Provides snapshots and cloning.
+	gTimeZone     string
+	gSnapshotName string
+
+	gMigrateDestDatastore string
+	gYesToAll             bool
+
+	rootCmd = &cobra.Command{
+		Use:   "gocage",
+		Short: "GoCage is a FreeBSD Jail management tool",
+		Long: `GoCage is a jail management tool. It support VNET, host-only, NAT networks. Provides snapshots and cloning.
 It support iocage jails and can coexist with iocage.`,
 		Run: func(cmd *cobra.Command, args []string) {
-			fmt.Println("Here we are in the Run")
-			cleanAfterRun()
+			fv, _ := getFreeBSDVersion()
+			fmt.Printf("GoCage v.%s on FreeBSD %d.%d-%s\n", gVersion, fv.major, fv.minor, fv.flavor)
+			fmt.Printf("Use -h flag to display help\n")
 		},
 	}
-	
-	versionCmd = &cobra.Command {
+
+	versionCmd = &cobra.Command{
 		Use:   "version",
 		Short: "Print the version number of GoCage",
 		Long:  `Let this show you how much fail I had to get this *cough* perfect`,
 		Run: func(cmd *cobra.Command, args []string) {
-			fmt.Printf("GoCage v.%s on FreeBSD %.1f\n", gVersion, gHostVersion)
-			cleanAfterRun()
-	    },
+			fv, _ := getFreeBSDVersion()
+			fmt.Printf("GoCage v.%s on FreeBSD %d.%d-%s\n", gVersion, fv.major, fv.minor, fv.flavor)
+		},
 	}
 
-	listCmd = &cobra.Command {
+	listCmd = &cobra.Command{
 		Use:   "list",
 		Short: "Print jails",
-		Long:  `Display jails, their IP and OS.
+		Long: `Display jails, their IP and OS.
 Jail list can be restricted by adding name on command line
 ex: gocage list srv-db srv-web`,
 		Run: func(cmd *cobra.Command, args []string) {
 			ListJails(args, true)
-			cleanAfterRun()
 		},
 	}
-	
-	listPropsCmd = &cobra.Command {
+
+	listPropsCmd = &cobra.Command{
 		Use:   "properties",
 		Short: "Print jails properties",
 		Long:  "Display jails properties. You can use properties to filter, get or set them.",
 		Run: func(cmd *cobra.Command, args []string) {
 			ListJailsProps(args)
-			cleanAfterRun()
 		},
 	}
 	
-	stopCmd = &cobra.Command {
+/*	destroyCmd = &cobra.Command{
+		Use:   "destroy",
+		Short: "destroy jails",
+		Long: `Destroy jail filesystem, snapshots and configuration file.`,
+		Run: func(cmd *cobra.Command, args []string) {
+			ListJails(args, false)
+			DestroyJails(args)
+		},
+	}
+*/
+	stopCmd = &cobra.Command{
 		Use:   "stop",
 		Short: "stop jail",
 		Long:  "shutdown jail",
 		Run: func(cmd *cobra.Command, args []string) {
 			// Load inventory
 			ListJails(args, false)
-			StopJail(args)
-			cleanAfterRun()
+			if len(args) == 0 {
+				StopAllRunningJails()
+			} else {
+				StopJail(args)
+			}
 		},
 	}
 
-	startCmd = &cobra.Command {
+	startCmd = &cobra.Command{
 		Use:   "start",
 		Short: "start jail",
 		Run: func(cmd *cobra.Command, args []string) {
 			// Load inventory
 			ListJails(args, false)
-			StartJail(args)
-			cleanAfterRun()
+			if len(args) == 0 {
+				StartJailsAtBoot()
+			} else {
+				StartJail(args)
+			}
+			WriteConfigToDisk(false)
 		},
 	}
 	
-	setCmd = &cobra.Command {
+	
+	
+	restartCmd = &cobra.Command{
+		Use:   "restart",
+		Short: "restart jail",
+		Run: func(cmd *cobra.Command, args []string) {
+			// Load inventory
+			ListJails(args, false)
+			StopJail(args)
+			StartJail(args)
+			WriteConfigToDisk(false)
+		},
+	}
+	
+	shellCmd = &cobra.Command {
+		Use:   "console",
+		Short: "Execute shell on jail",
+		Run: func(cmd *cobra.Command, args []string) {
+			// Load inventory
+			ListJails(args, false)
+			ShellJail(args)
+		},
+	}
+	
+	setCmd = &cobra.Command{
 		Use:   "set",
 		Short: "Set a jail property",
-		Long:  `Set jail property value. Specify property=value, end command with jail name.
+		Long: `Set jail property value. Specify property=value, end command with jail name.
 Multiples properties can be specified, separated with space (Ex: gocage set allow_mlock=1 boot=1 myjail)`,
 		Run: func(cmd *cobra.Command, args []string) {
 			// Load inventory
 			ListJails(args, false)
 			SetJailProperties(args)
-			cleanAfterRun()
+			WriteConfigToDisk(true)
 		},
 	}
-	
-	getCmd = &cobra.Command {
+
+	getCmd = &cobra.Command{
 		Use:   "get",
 		Short: "Get a jail property",
-		Long:  `Get jail property value. Specify property, end command with jail name.
+		Long: `Get jail property value. Specify property, end command with jail name.
 Multiples properties can be specified, separated with space (Ex: gocage get allow_mlock boot myjail)
 For all properties specify "all" (Ex: gocage get all myjail)`,
 		Run: func(cmd *cobra.Command, args []string) {
 			// Load inventory
 			ListJails(args, false)
 			GetJailProperties(args)
-			cleanAfterRun()
 		},
 	}
-	
-	snapshotCmd = &cobra.Command {
+
+	snapshotCmd = &cobra.Command{
 		Use:   "snapshot",
 		Short: "snapshot jail",
 		Long:  "Commands to manage jail snapshots. If no arguments given, ",
@@ -133,11 +191,11 @@ For all properties specify "all" (Ex: gocage get all myjail)`,
 
 		},
 	}
-	
-	snapshotListCmd = &cobra.Command {
+
+	snapshotListCmd = &cobra.Command{
 		Use:   "list",
 		Short: "list snapshots",
-		Long:  `List snapshots of a jail by specifying its name. 
+		Long: `List snapshots of a jail by specifying its name. 
 List all snapshots if no jail name specified.
 You can specify multiple jails.`,
 		Run: func(cmd *cobra.Command, args []string) {
@@ -146,77 +204,170 @@ You can specify multiple jails.`,
 			ListJailsSnapshots(args)
 		},
 	}
-	 
-	 snapshotCreateCmd = &cobra.Command {
-		 Use:   "create",
+
+	snapshotCreateCmd = &cobra.Command{
+		Use:   "create",
 		Short: "create snapshots",
 		Long:  `Create snapshot of a jail by specifying snapshot name and jail name.`,
-//		You can specify multiple jails.`,
+		//		You can specify multiple jails.`,
 		Run: func(cmd *cobra.Command, args []string) {
 			// Load inventory
 			ListJails(args, false)
 			CreateJailSnapshot(args)
 		},
 	}
-	
-	snapshotDeleteCmd = &cobra.Command {
+
+	snapshotRollbackCmd = &cobra.Command{
+		Use:   "rollback",
+		Short: "Rollback snapshots",
+		Long:  `Rollback jail to specifyed snapshot.`,
+		//		You can specify multiple jails.`,
+		Run: func(cmd *cobra.Command, args []string) {
+			// Load inventory
+			ListJails(args, false)
+			RollbackJailSnapshot(args)
+		},
+	}
+
+	snapshotDeleteCmd = &cobra.Command{
 		Use:   "destroy",
 		Short: "destroy snapshots",
-		Long:  `Destroy snapshot of a jail by specifying snapshot name and jail name.`,
-		//		You can specify multiple jails.`,
+		Long:  `Destroy snapshot of a jail by specifying snapshot name and jail name.
+				You can specify multiple snapshots separated by comma.`,
 		Run: func(cmd *cobra.Command, args []string) {
 			// Load inventory
 			ListJails(args, false)
 			DeleteJailSnapshot(args)
 		},
 	}
+
+	migrateCmd = &cobra.Command{
+		Use:   "migrate",
+		Short: "Migrate jail to another datastore",
+		Run: func(cmd *cobra.Command, args []string) {
+			// Load inventory
+			ListJails(args, false)
+			MigrateJail(args)
+			WriteConfigToDisk(false)
+		},
+	}
+
+	migrateCleanCmd = &cobra.Command{
+		Use:   "clean",
+		Short: "Clean previous aborted/in error jail migration",
+		Run: func(cmd *cobra.Command, args []string) {
+			// Load inventory
+			ListJails(args, false)
+			err := CleanMigrateMess(args)
+			if err != nil {
+				fmt.Printf("%v", err)
+			}
+		},
+	}
+	
+	datastoreCmd = &cobra.Command{
+		Use:   "datastore",
+		Short: "list datastores",
+		Long:  "Commands to manage datastores. If no arguments given, list them.",
+		Run: func(cmd *cobra.Command, args []string) {
+			ListDatastores(args, true)
+		},
+	}
+	
+	datastoreListCmd = &cobra.Command{
+		Use:   "list",
+		Short: "list datastores",
+		Long: `List datastore by specifying its name. 
+List all datastores if no name specified.
+You can specify multiple datastores.`,
+		Run: func(cmd *cobra.Command, args []string) {
+			ListDatastores(args, true)
+		},
+	}
+	
+	testCmd = &cobra.Command{
+		Use:   "test",
+		Short: "temporary command to test some code snippet",
+		Run: func(cmd *cobra.Command, args []string) {
+			fmt.Printf("Nope\n")
+		},
+	}
 )
 
-
-
+// TODO : Init log level and log output
 func init() {
-    cobra.OnInitialize(initConfig)
+	var err error
+	
+	cobra.OnInitialize(initConfig)
 
-// Global switches
+	// Global switches
 	rootCmd.PersistentFlags().StringVarP(&gConfigFile, "config", "c", "/usr/local/etc/gocage.conf.yml", "GoCage configuration file")
 	rootCmd.PersistentFlags().BoolVarP(&gUseSudo, "sudo", "u", false, "Use sudo to run commands")
 	rootCmd.PersistentFlags().StringVarP(&gTimeZone, "timezone", "t", "", "Specify timezone. Will get from /var/db/zoneinfo if not set.")
+	rootCmd.PersistentFlags().BoolVarP(&gDebug, "debug", "d", false, "Debug mode")
 
-// Command dependant switches
+	// Command dependant switches
 
-	// These are persistent so we can reuse them in "gocage list snapshot myjail" command (TODO)
-	listCmd.PersistentFlags().StringVarP(&gDisplayColumns, "outcol", "o", "JID,Name,Config.Release,Config.Ip4_addr,Running", "Show these columns in output")
-	listCmd.PersistentFlags().BoolVarP(&gNoLineSep, "nolinesep", "l", false, "Do not display line separator between jails")
-	listCmd.PersistentFlags().StringVarP(&gFilterJails, "filter", "f", "none", "Only display jails with these values. Ex: \"gocage list -f Config.Boot=1\" will only list started on boot jails")
-	listCmd.PersistentFlags().StringVarP(&gSortFields, "sort", "s", "none", "Display jails sorted by field values. Ex: \"gocage list -s +Name,-Config.Priority\" will sort jails by their decreasing name, then increasing start priority. 3 critera max supported.")
+	// We reuse these flags in "gocage snapshot list myjail" and 'gocage datastore list" commands
+	listCmd.Flags().StringVarP(&gDisplayJColumns, "outcol", "o", "JID,Name,Config.Release,Config.Ip4_addr,Running", "Show these columns in output")
+	listCmd.Flags().BoolVarP(&gNoJailLineSep, "nolinesep", "l", false, "Do not display line separator between jails")
+	listCmd.Flags().StringVarP(&gFilterJails, "filter", "f", "none", "Only display jails with these values. Ex: \"gocage list -f Config.Boot=1\" will only list started on boot jails")
+	listCmd.Flags().StringVarP(&gSortJailFields, "sort", "s", "none", "Display jails sorted by field values. Ex: \"gocage list -s +Name,-Config.Priority\" will sort jails by their decreasing name, then increasing start priority. 3 critera max supported.")
+	
+//	destroyCmd.Flags().BoolVarP(&gForce, "force", "f", false, "Force stop jail if running")
+
+	snapshotListCmd.Flags().StringVarP(&gDisplaySColumns, "outcol", "o", "Jailname,Name,Creation,Referenced,Used", "Show these columns in output")
+	snapshotListCmd.Flags().BoolVarP(&gNoSnapLineSep, "nolinesep", "l", false, "Do not display line separator between snapshots")
+	snapshotListCmd.Flags().StringVarP(&gFilterSnaps, "filter", "f", "none", "Only display snapshots with these values. Ex: \"gocage snapshot list -f Config.Boot=1\" will only list started on boot jails")
+	snapshotListCmd.Flags().StringVarP(&gSortSnapFields, "sort", "s", "none", "Display snapshots sorted by field values. Ex: \"gocage snapshot list -s +Jailname,-Used\" will sort snapshots by jail decreasing name, then increasing used space. 3 critera max supported.")
+	
+	datastoreListCmd.Flags().StringVarP(&gDisplayDColumns, "outcol", "o", "Name,Mountpoint,ZFSDataset,Available,Used,Referenced", "Show these columns in output")
+	datastoreListCmd.Flags().BoolVarP(&gNoDSLineSep, "nolinesep", "l", false, "Do not display line separator between datastores")
+	datastoreListCmd.Flags().StringVarP(&gFilterDS, "filter", "f", "none", "Only display datastores with these values. Ex: \"gocage datastore list -f Config.Boot=1\" will only list started on boot jails")
+	datastoreListCmd.Flags().StringVarP(&gSortDSFields, "sort", "s", "none", "Display datastores sorted by field values. Ex: \"gocage datastore list -s +Jailname,-Used\" will sort snapshots by jail decreasing name, then increasing used space. 3 critera max supported.")
 	
 	// This is local flag : Only available to gocage snapshot create command
 	snapshotCreateCmd.Flags().StringVarP(&gSnapshotName, "snapname", "n", "", "Name of the snapshot to create")
 	snapshotCreateCmd.MarkFlagRequired("snapname")
 	snapshotDeleteCmd.Flags().StringVarP(&gSnapshotName, "snapname", "n", "", "Name of the snapshot to destroy")
 	snapshotDeleteCmd.MarkFlagRequired("snapname")
-    
-// Now declare commands
+	snapshotRollbackCmd.Flags().StringVarP(&gSnapshotName, "snapname", "n", "", "Name of the snapshot to rollback to")
+	snapshotRollbackCmd.MarkFlagRequired("snapname")
+
+	migrateCmd.Flags().StringVarP(&gMigrateDestDatastore, "datastore", "d", "", "Path of destination datastore for jail (Ex: \"/iocage\")")
+	migrateCmd.Flags().BoolVarP(&gYesToAll, "yes", "y", false, "Answer yes to all questions")
+	migrateCmd.MarkFlagRequired("datastore")
+
+	// Now declare commands
 	rootCmd.AddCommand(versionCmd)
 	rootCmd.AddCommand(listCmd)
 	listCmd.AddCommand(listPropsCmd)
 	rootCmd.AddCommand(stopCmd)
 	rootCmd.AddCommand(startCmd)
+	rootCmd.AddCommand(restartCmd)
+//	rootCmd.AddCommand(destroyCmd)
+	rootCmd.AddCommand(shellCmd)
 	rootCmd.AddCommand(getCmd)
 	rootCmd.AddCommand(setCmd)
 	rootCmd.AddCommand(snapshotCmd)
+	rootCmd.AddCommand(migrateCmd)
+	rootCmd.AddCommand(datastoreCmd)
+	
+	rootCmd.AddCommand(testCmd)
+	
 	snapshotCmd.AddCommand(snapshotListCmd)
 	snapshotCmd.AddCommand(snapshotCreateCmd)
 	snapshotCmd.AddCommand(snapshotDeleteCmd)
-	
-	// Get FreeBSD version
-	out, err := executeCommand("freebsd-version")
+	snapshotCmd.AddCommand(snapshotRollbackCmd)
+	migrateCmd.AddCommand(migrateCleanCmd)
+	datastoreCmd.AddCommand(datastoreListCmd)
+
+	// Get FreeBSD version, hostname, hostid
+	gJailHost, err = NewJailHost()
 	if err != nil {
-		fmt.Printf("Error running \"freebsd-version\": %s", err.Error())
+		fmt.Printf("Error initializing JailHost properties: %v\n", err)
 		os.Exit(1)
 	}
-	gHostVersion, _ = strconv.ParseFloat(strings.Split(out, "-")[0], 32)
-	
 }
 
 func initConfig() {
@@ -225,19 +376,24 @@ func initConfig() {
 		os.Exit(1)
 	}
 
-//	fmt.Printf("We are in initConfig(), with config file %s\n", gConfigFile)
-
 	viper.SetConfigFile(gConfigFile)
 
 	if err := viper.ReadInConfig(); err != nil {
 		fmt.Printf("ERROR reading config file %s : %s\n", gConfigFile, err.Error())
 		os.Exit(1)
 	}
-
-//	fmt.Println("Using config file:", viper.ConfigFileUsed())
-//	fmt.Printf("datastore in config : %s\n", viper.GetStringSlice("datastore"))
-//	fmt.Printf("datastore.0 in config : %s\n", viper.GetStringSlice("datastore.0"))
 	
+	// Load default configs from datastores
+	err := ListDatastores(viper.GetStringSlice("datastore"), false)
+	if err != nil {
+		fmt.Printf("ERROR: error checking datastores: %v\n", err)
+		os.Exit(1)
+	}
+
+	//	fmt.Println("Using config file:", viper.ConfigFileUsed())
+	//	fmt.Printf("datastore in config : %s\n", viper.GetStringSlice("datastore"))
+	//	fmt.Printf("datastore.0 in config : %s\n", viper.GetStringSlice("datastore.0"))
+
 	// Command line flags have priority on config file
 	if rootCmd.Flags().Lookup("sudo") != nil && false == rootCmd.Flags().Lookup("sudo").Changed {
 		gUseSudo = viper.GetBool("sudo")
@@ -255,28 +411,71 @@ func initConfig() {
 		gTimeZone = strings.Trim(string(tz), "\n")
 	}
 	if listCmd.Flags().Lookup("outcol") != nil && false == listCmd.Flags().Lookup("outcol").Changed {
-		gDisplayColumns = viper.GetString("outcol")
+		gDisplayJColumns = viper.GetString("outcol")
 	}
 	if listCmd.Flags().Lookup("nolinesep") != nil && false == listCmd.Flags().Lookup("nolinesep").Changed {
-		gNoLineSep = viper.GetBool("nolinesep")
+		gNoJailLineSep = viper.GetBool("nolinesep")
 	}
 	if listCmd.Flags().Lookup("filter") != nil && false == listCmd.Flags().Lookup("filter").Changed {
 		gFilterJails = viper.GetString("filter")
 	}
 	if listCmd.Flags().Lookup("sort") != nil && false == listCmd.Flags().Lookup("sort").Changed {
-		gSortFields = viper.GetString("sort")
+		gSortJailFields = viper.GetString("sort")
 	}
-	if len(strings.Split(gSortFields, ",")) > 3 {
-		fmt.Printf("More than 3 sort criteria, this is not supported!\n")
+	if len(strings.Split(gSortJailFields, ",")) > 3 {
+		fmt.Printf("More than 3 sort criteria is not supported!\n")
 		os.Exit(1)
 	}
+	
+	
+	if gDebug {
+		log.SetLevel(log.DebugLevel)
+		log.Debugf("Debug mode enabled\n")
+	}
 }
 
-// Called after execution
-func cleanAfterRun() {
+/********************************************************************************
+ * Write jails config which been updated to disk.
+ * If changeauto not set, values which are in "auto" mode on disk
+ *  won't be overwritten (p.ex defaultrouter wont be overwritten with current
+ *  default route, so if route change on jailhost this will reflect on jail next
+ *  start)
+ *******************************************************************************/
+func WriteConfigToDisk(changeauto bool) {
 	for _, j := range gJails {
 		if j.ConfigUpdated {
-			marshaled, err := json.MarshalIndent(j.Config, "", "    ")
+			//log.Debug("%s config has changed, write changes to disk\n", j.Name)
+
+			// we will manipulate properties so get a copy
+			jc := j.Config
+
+			if changeauto == false {
+				// Overwrite "auto" properties
+				ondiskjc, err := getJailConfig(j.ConfigPath)
+				if err != nil {
+					panic(err)
+				}
+				// TODO : List all fields, then call getStructFieldValue to compare value with "auto"
+				// If "auto" then keep it that way before writing ondiskjc to disk
+				var properties []string
+				properties = getStructFieldNames(ondiskjc, properties, "")
+
+				for _, p := range properties {
+					v, _, err := getStructFieldValue(ondiskjc, p)
+					if err != nil {
+						panic(err)
+					}
+					if v.String() == "auto" {
+						err = setStructFieldValue(&jc, p, "auto")
+						if err != nil {
+							fmt.Printf("ERROR sanitizing config: %s\n", err.Error())
+							os.Exit(1)
+						}
+					}
+				}
+			}
+
+			marshaled, err := json.MarshalIndent(jc, "", "    ")
 			if err != nil {
 				fmt.Printf("ERROR marshaling config: %s\n", err.Error())
 			}
@@ -295,5 +494,3 @@ func Execute() {
 		os.Exit(1)
 	}
 }
-
-

cmd/snapshots.go

@@ -1,10 +1,15 @@
 package cmd
 
 import (
+	"bufio"
+	"errors"
 	"fmt"
-	"time"
+	"os"
 	"regexp"
+	"strconv"
 	"strings"
+	"reflect"
+	"time"
 )
 
 /********************************************************************************
@@ -13,13 +18,20 @@ import (
 func ListJailsSnapshots(args []string) {
 	var jailNames []string
 	var snapshots []Snapshot
-	
+
+	/***************************************************************
+	/   Filter snapshots by jailname
+	/**************************************************************/
 	if len(args) > 0 {
 		for _, a := range args {
+			/*if countOfJailsWithThisName(a) > 1 {
+				fmt.Printf("Nope")
+				return
+			}*/
 			jailNames = append(jailNames, a)
 		}
-	} 
-	
+	}
+
 	if len(jailNames) == 0 || len(args) == 0 {
 		for _, j := range gJails {
 			snapshots = append(snapshots, listJailSnapshots(j)...)
@@ -33,72 +45,124 @@ func ListJailsSnapshots(args []string) {
 			}
 		}
 	}
-	displaySnapshotsFields(snapshots, []string{"Jailname","Name","Creation","Referenced","Used"})
-}
+	
+	fields := strings.Split(gDisplaySColumns, ",")
+	
+	/***************************************************************
+	/   Sort snapshots
+	/     We support 3 sort criteria max
+	/**************************************************************/
+	if len(gSortSnapFields) > 0 && gSortSnapFields != "none" {		
+		ss := initSnapshotSortStruct()
 
+		// The way we manage criteria quantity is not very elegant...
+		var fct1, fct2, fct3 *reflect.Value
+		for i, c := range strings.Split(gSortSnapFields, ",") {
+			var fctName string
+			if strings.HasPrefix(c, "-") {
+				fctName = fmt.Sprintf("%sDec", strings.Replace(c, "-", "", 1))
+			} else { // Par defaut (pas de prefix +/-) on considere un tri incremental
+				fctName = fmt.Sprintf("%sInc", strings.Replace(c, "+", "", 1))
+			}
+
+			// Get function by its name
+			fct, _, err := getStructFieldValue(ss, fctName)
+			
+			if err != nil {
+				fieldName := strings.Replace(strings.Replace(c, "-", "", 1), "+", "", 1)
+				fmt.Printf("ERROR getting SnapshotSort struct field %s. Please check the field name: %s\n", fctName, fieldName)
+				return
+			}
+			switch i + 1 {
+			case 1:
+				fct1 = fct
+			case 2:
+				fct2 = fct
+			case 3:
+				fct3 = fct
+			}
+		}
+
+		switch len(strings.Split(gSortSnapFields, ",")) {
+		case 1:
+			SnapshotsOrderedBy(fct1.Interface().(snapshotLessFunc)).Sort(snapshots)
+		case 2:
+			SnapshotsOrderedBy(fct1.Interface().(snapshotLessFunc), fct2.Interface().(snapshotLessFunc)).Sort(snapshots)
+		case 3:
+			SnapshotsOrderedBy(fct1.Interface().(snapshotLessFunc), fct2.Interface().(snapshotLessFunc), fct3.Interface().(snapshotLessFunc)).Sort(snapshots)
+		}
+	}
+	
+	displaySnapshotsFields(snapshots, fields)
+}
 
 /********************************************************************************
  * List all snapshots a jail have
  *******************************************************************************/
 func listJailSnapshots(jail Jail) []Snapshot {
 	var snapshots []Snapshot
-	
+
 	// 1. List all datasets
 	// TODO : Include mounted filesystems?
-	rs := strings.Split(jail.RootPath, "/")
-	rootDataset := fmt.Sprintf("%s%s", jail.Zpool, strings.Join(rs[:len(rs)-1], "/"))
-	cmd := fmt.Sprintf("zfs list -r -H -o name,mountpoint,used,referenced,creation -t snapshot %s", rootDataset)
+	
+	curDS, err := getDatastoreFromArray(jail.Datastore, gDatastores)
+	if err != nil {
+		fmt.Printf("Error getting datastore \"%s\": %v\n", jail.Datastore, err)
+		return snapshots
+	}
+	
+	rootDataset := fmt.Sprintf("%s/%s/%s", curDS.ZFSDataset, "jails", jail.Name)
+	cmd := fmt.Sprintf("zfs list -p -r -H -o name,mountpoint,used,referenced,creation -t snapshot %s", rootDataset)
 	out, err := executeCommand(cmd)
 	if err != nil {
 		fmt.Printf("Error: %s\n", err.Error())
 		return snapshots
 	}
 	
-	dateLayout := "Mon Jan _2 15:04 2006"
-	loc, _ := time.LoadLocation(gTimeZone)
-	
 	for _, line := range strings.Split(out, "\n") {
 		if len(line) > 0 {
 			ls := strings.Split(line, "\t")
 			// Parse creation date so we can use it to sort snapshots
-			creationts, err := time.ParseInLocation(dateLayout, ls[4], loc)
+			//creationts, err := time.ParseInLocation(dateLayout, ls[4], loc)
+			creationts, err := strconv.ParseInt(ls[4], 10, 64)
 			if err != nil {
 				fmt.Println("Error while parsing date %s:", ls[4], err)
 				return snapshots
 			}
 			// Get subdir to append to snapshot name
 			subdir := strings.Replace(strings.Split(ls[0], "@")[0], rootDataset, "", 1)
-			
-			snapshots = append(snapshots, Snapshot{Dsname: ls[0],
-				Name: fmt.Sprintf("%s%s", strings.Split(ls[0], "@")[1], subdir),
-				Jailname: jail.Name,
+
+			u, _ := strconv.ParseUint(ls[2], 10, 64)
+			r, _ := strconv.ParseUint(ls[3], 10, 64)
+			snapshots = append(snapshots, Snapshot{Datastore: curDS.Name,
+				Name:       fmt.Sprintf("%s%s", strings.Split(ls[0], "@")[1], subdir),
+				Jailname:   jail.Name,
 				Mountpoint: ls[1],
-				Used: ls[2],
-				Referenced: ls[3],
-				Creation: creationts})
+				Used:       u,
+				Referenced: r,
+				Creation:   time.Unix(creationts, 0)})
 		}
 	}
-	
+
 	// Sort snapshots by creation date
 	ss := initSnapshotSortStruct()
 	SnapshotsOrderedBy(ss.CreationInc).Sort(snapshots)
-	
+
 	return snapshots
 }
 
-
 /********************************************************************************
  * Create snapshot for jail(s)
  *******************************************************************************/
 func CreateJailSnapshot(args []string) {
 	var jailNames []string
-	
+
 	if len(args) > 0 {
 		for _, a := range args {
 			jailNames = append(jailNames, a)
 		}
-	} 
-	
+	}
+
 	for _, cj := range gJails {
 		for _, jn := range jailNames {
 			if strings.EqualFold(cj.Name, jn) {
@@ -112,8 +176,8 @@ func CreateJailSnapshot(args []string) {
  * Create snapshot for a jail
  *******************************************************************************/
 func createJailSnapshot(jail Jail) error {
-	rs := strings.Split(jail.RootPath, "/")
-	rootDataset := fmt.Sprintf("%s%s", jail.Zpool, strings.Join(rs[:len(rs)-1], "/"))
+	curDS, _ := getDatastoreFromArray(jail.Datastore, gDatastores)
+	rootDataset := fmt.Sprintf("%s/%s/%s", curDS.ZFSDataset, "jails", jail.Name)
 	cmd := fmt.Sprintf("zfs snapshot -r %s@%s", rootDataset, gSnapshotName)
 
 	_, err := executeCommand(cmd)
@@ -122,7 +186,7 @@ func createJailSnapshot(jail Jail) error {
 		return err
 	}
 	fmt.Printf("Snapshot %s@%s created\n", rootDataset, gSnapshotName)
-	
+
 	return nil
 }
 
@@ -131,13 +195,13 @@ func createJailSnapshot(jail Jail) error {
  *******************************************************************************/
 func DeleteJailSnapshot(args []string) {
 	var jailNames []string
-	
+
 	if len(args) > 0 {
 		for _, a := range args {
 			jailNames = append(jailNames, a)
 		}
-	} 
-	
+	}
+
 	for _, cj := range gJails {
 		for _, jn := range jailNames {
 			if strings.EqualFold(cj.Name, jn) {
@@ -152,28 +216,29 @@ func DeleteJailSnapshot(args []string) {
  *******************************************************************************/
 func deleteJailSnapshot(jail Jail) error {
 	var snaptodel []string
-	
+
 	// Get all recursive snapshots
-	rs := strings.Split(jail.RootPath, "/")
-	rootDataset := fmt.Sprintf("%s%s", jail.Zpool, strings.Join(rs[:len(rs)-1], "/"))
+	curDS, _ := getDatastoreFromArray(jail.Datastore, gDatastores)
+	rootDataset := fmt.Sprintf("%s/%s/%s", curDS.ZFSDataset, "jails", jail.Name)
 	cmd := fmt.Sprintf("zfs list -r -H -o name -t snapshot %s", rootDataset)
 	out, err := executeCommand(cmd)
 	if err != nil {
 		fmt.Printf("Error: listing snapshots: %s\n", err.Error())
 		return nil
 	}
-	
+
 	for _, line := range strings.Split(out, "\n") {
 		if len(line) > 0 {
 			ls := strings.Split(line, "@")
-			
-			matched, _ := regexp.Match(fmt.Sprintf("^%s(\\/.*)?$", gSnapshotName), []byte(ls[1]))
-			if matched {
-				snaptodel = append(snaptodel, strings.Join(ls, "@"))
+			for _, sn := range strings.Split(gSnapshotName, ",") {
+				matched, _ := regexp.Match(fmt.Sprintf("^%s(\\/.*)?$", sn), []byte(ls[1]))
+				if matched {
+					snaptodel = append(snaptodel, strings.Join(ls, "@"))
+				}
 			}
 		}
 	}
-	
+
 	for _, s := range snaptodel {
 		cmd := fmt.Sprintf("zfs destroy %s", s)
 		_, err := executeCommand(cmd)
@@ -183,6 +248,83 @@ func deleteJailSnapshot(jail Jail) error {
 		}
 		fmt.Printf("Snapshot %s deleted\n", s)
 	}
-	
+
+	return nil
+}
+
+func RollbackJailSnapshot(args []string) error {
+	var jailNames []string
+
+	if len(args) > 0 {
+		for _, a := range args {
+			jailNames = append(jailNames, a)
+		}
+	}
+
+	for _, cj := range gJails {
+		for _, jn := range jailNames {
+			if strings.EqualFold(cj.Name, jn) {
+				rollbackJailSnapshot(cj)
+			}
+		}
+	}
+
+	return nil
+}
+
+/********************************************************************************
+ * rollback jail to snapshot gSnapshotName, destroy this snapshots and more
+ *   recents snapshots and bookmarks
+ *******************************************************************************/
+func rollbackJailSnapshot(jail Jail) error {
+	var snaptorb []string
+
+	if jail.Running {
+		fmt.Printf("Jail should be stoped to rollback, should we stop and rollback? (y/n)\n")
+		scanr := bufio.NewScanner(os.Stdin)
+		if scanr.Scan() {
+			if !strings.EqualFold(scanr.Text(), "y") {
+				return errors.New("Jail is running")
+			} else {
+				err := stopJail(&jail)
+				if err != nil {
+					return err
+				}
+			}
+		}
+	}
+
+	// We need to rollback parent and childs
+	// Get all recursive snapshots
+	curDS, _ := getDatastoreFromArray(jail.Datastore, gDatastores)
+	rootDataset := fmt.Sprintf("%s/%s/%s", curDS.ZFSDataset, "jails", jail.Name)
+	cmd := fmt.Sprintf("zfs list -r -H -o name -t snapshot %s", rootDataset)
+	out, err := executeCommand(cmd)
+	if err != nil {
+		fmt.Printf("Error: listing snapshots: %s\n", err.Error())
+		return err
+	}
+
+	for _, line := range strings.Split(out, "\n") {
+		if len(line) > 0 {
+			ls := strings.Split(line, "@")
+
+			matched, _ := regexp.Match(fmt.Sprintf("^%s(\\/.*)?$", gSnapshotName), []byte(ls[1]))
+			if matched {
+				snaptorb = append(snaptorb, strings.Join(ls, "@"))
+			}
+		}
+	}
+
+	for _, s := range snaptorb {
+		cmd := fmt.Sprintf("zfs rollback -r %s", s)
+		_, err := executeCommand(cmd)
+		if err != nil {
+			fmt.Printf("Error rolling back snapshot %s: %s\n", s, err.Error())
+			return err
+		}
+	}
+	fmt.Printf("Jail is back to %s\n", gSnapshotName)
+
 	return nil
 }

cmd/stop.go

@@ -3,12 +3,15 @@ package cmd
 import (
 	"os"
 	"fmt"
-//	"log"
+	//"log"
 	"errors"
 	"regexp"
 	"os/exec"
-//	"reflect"
+	//"reflect"
+	"strconv"
 	"strings"
+	
+	log "github.com/sirupsen/logrus"
 )
 
 // TODO : Use SYS_RCTL_GET_RACCT syscall
@@ -38,7 +41,6 @@ func removeRctlRules(jail string, rules []string) error {
 	return nil
 }
 
-
 // TODO: Validate with  >1 dataset
 func umountAndUnjailZFS(jail *Jail) error {
 	var ds []string
@@ -75,16 +77,16 @@ func umountAndUnjailZFS(jail *Jail) error {
 		fmt.Printf("ERROR unjailing %s/%s: %s\n", jail.Zpool, ds[len(ds)-1], err.Error())
 		return err
 	}
-	
+
 	return nil
 }
 
-
 func destroyVNetInterfaces(jail *Jail) error {
 	for _, i := range strings.Split(jail.Config.Ip4_addr, ",") {
 		iname := fmt.Sprintf("%s.%d", strings.Split(i, "|")[0], jail.JID)
 		fmt.Printf("%s: ", iname)
 		_, err := executeCommand(fmt.Sprintf("ifconfig %s destroy", iname))
+		//_, err := executeScript(fmt.Sprintf("ifconfig %s destroy >/dev/null 2>&1", iname))
 		if err != nil {
 			return err
 		} else {
@@ -101,16 +103,17 @@ func destroyVNetInterfaces(jail *Jail) error {
 //  or else it will require a restart of "devfs" service.
 // But, stoppign the jail already removes this >1000 ID.
 // So no need to call this function.
-func deleteDevfsRuleset(jail *Jail) error {
+func deleteDevfsRuleset(ruleset int) error {
 	cmd := "devfs rule showsets"
 	out, err := executeCommand(cmd)
 	if err != nil {
 		return errors.New(fmt.Sprintf("ERROR listing rulesets: %s", err.Error()))
 	}
-
+	
+	rs := strconv.Itoa(ruleset)
 	for _, r := range strings.Split(out, "\n") {
-		if r == jail.Config.Devfs_ruleset {
-			cmd := fmt.Sprintf("devfs rule -s %s delset", jail.Config.Devfs_ruleset)
+		if r == rs {
+			cmd := fmt.Sprintf("devfs rule -s %d delset", ruleset)
 			_, err := executeCommand(cmd)
 			return err
 		}
@@ -119,7 +122,6 @@ func deleteDevfsRuleset(jail *Jail) error {
 	return nil
 }
 
-
 func umountJailFsFromHost(jail *Jail, mountpoint string) error {
 	cmd := "mount -p"
 	out, err := executeCommand(cmd)
@@ -135,7 +137,7 @@ func umountJailFsFromHost(jail *Jail, mountpoint string) error {
 				cmd = fmt.Sprintf("umount %s%s", jail.RootPath, mountpoint)
 				_, err := executeCommand(cmd)
 				if err != nil {
-					return errors.New(fmt.Sprintf("Error umounting %s/%s: %s", jail.RootPath, mountpoint, err.Error()))
+					return errors.New(fmt.Sprintf("Error umounting %s%s: %s", jail.RootPath, mountpoint, err.Error()))
 				}
 				return nil
 			}
@@ -145,7 +147,6 @@ func umountJailFsFromHost(jail *Jail, mountpoint string) error {
 	return nil
 }
 
-
 // Internal usage only
 func stopJail(jail *Jail) error {
 	cmd := "jail -q"
@@ -167,6 +168,33 @@ func stopJail(jail *Jail) error {
 	return nil
 }
 
+// Stop all running jails by reverse priority
+func StopAllRunningJails() {
+	var stopList []Jail
+	
+	// Get boot enabled jails
+	for _, j := range gJails {
+		if j.Running == true {
+			stopList = append(stopList, j)
+		}
+	}
+	
+	// Order by priority
+	js := initJailSortStruct()
+	fct, _, err := getStructFieldValue(js, "Config.PriorityDec")
+	if err != nil {
+		log.Errorf("ERROR getting JailSort struct field \"Config.PriorityDec\"\n")
+		return
+	}
+	JailsOrderedBy(fct.Interface().(jailLessFunc)).Sort(stopList)
+	
+	for _, j := range stopList {
+		jFullName := fmt.Sprintf("%s/%s", j.Datastore, j.Name)
+		log.Debugf("Stopping %s with priority %s\n", jFullName, j.Config.Priority)
+		StopJail([]string{jFullName})
+	}
+}
+
 /*
 	Stop jail:
 		Remove rctl rules
@@ -191,25 +219,23 @@ func stopJail(jail *Jail) error {
 func StopJail(args []string) {
 	// Current jail were stopping
 	var cj *Jail
+	var err error
 
-	for _, j := range args {
-		fmt.Printf("> Stopping jail %s\n", j)
-
-		for _, rj := range gJails {
-			if rj.Name == j {
-				cj = &rj
-				break
-			}
-		}
-		if cj == nil {
-			fmt.Printf("Jail not found: %s\n", j)
+	for _, a := range args {
+		// Check if jail exist and is distinctly named
+		cj, err = getJailFromArray(a, gJails)
+		if err != nil {
+			fmt.Printf("Error getting jail: %s\n", err)
 			continue
 		}
+		
 		if cj.Running == false {
 			fmt.Printf("Jail %s is not running!\n", cj.Name)
 			continue
 		}
-
+		
+		fmt.Printf("> Stopping jail %s\n", a)
+		
 		out, err := executeCommand(fmt.Sprintf("rctl jail:%s", cj.InternalName))
 		if err == nil && len(out) > 0 {
 			fmt.Printf("  > Remove RCTL rules:\n")
@@ -221,17 +247,17 @@ func StopJail(args []string) {
 			}
 		}
 
-		if len (cj.Config.Exec_prestop) > 0 {
-			fmt.Printf("  > Execute prestop:\n")
+		if len(cj.Config.Exec_prestop) > 0 {
+			fmt.Printf("  > Execute pre-stop:\n")
 			_, err := executeCommand(cj.Config.Exec_prestop)
 			if err != nil {
 				fmt.Printf("ERROR: %s\n", err.Error())
 			} else {
-				fmt.Printf("  > Execute prestop: OK\n")
+				fmt.Printf("  > Execute pre-stop: OK\n")
 			}
 		}
-		
-		if len (cj.Config.Exec_stop) > 0 {
+
+		if len(cj.Config.Exec_stop) > 0 {
 			fmt.Printf("  > Execute stop:\n")
 			_, err := executeCommandInJail(cj, cj.Config.Exec_stop)
 			if err != nil {
@@ -260,15 +286,15 @@ func StopJail(args []string) {
 				fmt.Printf("  > Destroy VNet interfaces: OK\n")
 			}
 		}
-		
-		/*fmt.Printf("  > Remove devfsruleset %s:\n", cj.Config.Devfs_ruleset)
-		err = deleteDevfsRuleset(cj)
+
+		fmt.Printf("  > Remove devfs ruleset %d: \n", cj.Devfs_ruleset)
+		err = deleteDevfsRuleset(cj.Devfs_ruleset)
 		if err != nil {
 			fmt.Printf("ERROR: %s\n", err.Error())
 		} else {
-			fmt.Printf("  > Remove devfsruleset %s: OK\n", cj.Config.Devfs_ruleset)
-		}*/
-		
+			fmt.Printf("  > Remove devfsruleset %d: OK\n", cj.Devfs_ruleset)
+		}
+
 		fmt.Printf("  > Stop jail %s:\n", cj.Name)
 		err = stopJail(cj)
 		if err != nil {
@@ -297,15 +323,16 @@ func StopJail(args []string) {
 			}
 		}
 
-		if cj.Config.Mount_fdescfs > 0 {
+		// FIXME: /dev/fd is mounted even with Mount_fdescfs = 0 ?!
+		//if cj.Config.Mount_fdescfs > 0 {
 			fmt.Printf("  > Umount fdescfs:\n")
-			err := umountJailFsFromHost(cj, "/dev/fd")
+			err = umountJailFsFromHost(cj, "/dev/fd")
 			if err != nil {
 				fmt.Printf("ERROR: %s\n", err.Error())
 			} else {
 				fmt.Printf("  > Umount fdescfs: OK\n")
 			}
-		}
+		//}
 
 		if cj.Config.Mount_devfs > 0 {
 			fmt.Printf("  > Umount devfs:\n")
@@ -318,7 +345,6 @@ func StopJail(args []string) {
 		}
 
 		// Remove local mounts from $JAIL/fstab
-		// The way we get fstab is presumptuous
 		fstab := strings.Replace(cj.ConfigPath, "config.json", "fstab", 1)
 		mounts, err := getFstab(fstab)
 		if err != nil {
@@ -338,6 +364,40 @@ func StopJail(args []string) {
 				fmt.Printf("  > Umount mountpoints from %s: OK\n", fstab)
 			}
 		}
+		
+		// TODO: Execute poststop
+		if len(cj.Config.Exec_poststop) > 0 {
+			fmt.Printf("  > Execute post-stop:\n")
+			_, err := executeCommand(cj.Config.Exec_poststop)
+			if err != nil {
+				fmt.Printf("ERROR: %s\n", err.Error())
+			} else {
+				fmt.Printf("  > Execute post-stop: OK\n")
+			}
+		}
+		
+		// Remove parameter file
+		pfile := fmt.Sprintf("/var/run/jail.%s.conf", cj.InternalName)
+		if err = os.Remove(pfile); err != nil {
+			fmt.Printf("Error deleting parameter file %s\n", pfile)
+		}
+		
+		// We need this to get a reference to cj.Running (bc cj.Running is just a copy of value in the scope of StopJail())
+		for i, j := range gJails {
+			if strings.EqualFold(j.Name, cj.Name) && strings.EqualFold(j.Datastore, cj.Datastore) {
+				if err = setStructFieldValue(&gJails[i], "Running", "false"); err != nil {
+					fmt.Printf("ERROR: setting Running property to false: %s\n", err.Error())
+				}
+				if err = setStructFieldValue(&gJails[i], "JID", "0"); err != nil {
+					fmt.Printf("ERROR: setting JID property to 0: %s\n", err.Error())
+				}
+				if err = setStructFieldValue(&gJails[i], "InternalName", ""); err != nil {
+					fmt.Printf("ERROR: clearing InternalName property: %s\n", err.Error())
+				}
+				if err = setStructFieldValue(&gJails[i], "Devfs_ruleset", "0"); err != nil {
+					fmt.Printf("ERROR: setting Devfs_ruleset property to 0: %s\n", err.Error())
+				}
+			}
+		}
 	}
 }
-

go.mod

@@ -3,8 +3,13 @@ module gocage
 go 1.17
 
 require (
+	github.com/c-robinson/iplib v1.0.3
+	github.com/c2h5oh/datasize v0.0.0-20220606134207-859f65c6625b
+	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
+	github.com/sirupsen/logrus v1.8.1
 	github.com/spf13/cobra v1.2.1
 	github.com/spf13/viper v1.9.0
+	golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420
 )
 
 require (

gocage.conf.yml

@@ -1,11 +1,12 @@
-# List datastore root directory (the one containing jails/ and defaults.json)
+# List datastore(s) root directory (the one containing jails/ and defaults.json)
 datastore:
     - /iocage
 
+# Prefix all commands with sudo
 sudo: false
 
 # Columns to display when "gocage list". Column names are struct fields, see cmd/struct.go
 outcol: 'JID,Name,Config.Release,Config.Ip4_addr,Running'
 
-# Do not add line separator between jails
+# Do not add line separator between jails
 nolinesep: false

jail/main.go

@@ -14,16 +14,17 @@ package jail
 */
 import "C"
 import (
-	"strconv"
+    "strconv"
 //    "syscall"
     "unsafe"
 )
 
 
 type Jail struct {
-    Name	string
-    Jid		int
-	Path	string
+    Name          string
+    Jid           int
+    Path          string
+    Devfs_ruleset int
 }
 
 
@@ -33,8 +34,8 @@ func GetJails() ([]Jail, error) {
     var jl Jail
     var err error
 
-	// Make "params" a list of 4 jails parameters
-    params := make([]C.struct_jailparam, 4)
+	// Make "params" a list of 5 jails parameters
+    params := make([]C.struct_jailparam, 5)
 
     // initialize parameter names
     csname := C.CString("name")
@@ -43,27 +44,31 @@ func GetJails() ([]Jail, error) {
     defer C.free(unsafe.Pointer(csjid))
     cspath := C.CString("path")
     defer C.free(unsafe.Pointer(cspath))
+	csdevfsrs := C.CString("devfs_ruleset")
+	defer C.free(unsafe.Pointer(csdevfsrs))
     cslastjid := C.CString("lastjid")
     defer C.free(unsafe.Pointer(cslastjid))
+	
 
     // initialize params struct with parameter names
     C.jailparam_init(&params[0], csname)
     C.jailparam_init(&params[1], csjid)
     C.jailparam_init(&params[2], cspath)
+	C.jailparam_init(&params[3], csdevfsrs)
 
     // The key to retrieve jail. lastjid = 0 returns first jail and its jid as jailparam_get return value
-    C.jailparam_init(&params[3], cslastjid)
+    C.jailparam_init(&params[4], cslastjid)
 
     lastjailid := 0
     cslastjidval := C.CString(strconv.Itoa(lastjailid))
     defer C.free(unsafe.Pointer(cslastjidval))
 
-    C.jailparam_import(&params[3], cslastjidval)
+    C.jailparam_import(&params[4], cslastjidval)
 
     // loop on existing jails
     for lastjailid >= 0 {
         // get parameter values
-        lastjailid = int(C.jailparam_get(&params[0], 4, 0))
+        lastjailid = int(C.jailparam_get(&params[0], 5, 0))
         if lastjailid > 0 {
             nametmp := C.jailparam_export(&params[0])
             jl.Name = C.GoString(nametmp)
@@ -75,23 +80,28 @@ func GetJails() ([]Jail, error) {
             // Memory mgmt : Non gere par Go
             C.free(unsafe.Pointer(jidtmp))
             
-			pathtmp := C.jailparam_export(&params[2])
+            pathtmp := C.jailparam_export(&params[2])
             jl.Path = C.GoString(pathtmp)
             // Memory mgmt : Non gere par Go
             C.free(unsafe.Pointer(pathtmp))
 
+            drstmp := C.jailparam_export(&params[3])
+            jl.Devfs_ruleset, _ = strconv.Atoi(C.GoString(drstmp))
+            // Memory mgmt : Non gere par Go
+            C.free(unsafe.Pointer(drstmp))
+
             jls = append(jls, jl)
             //log.Debug("Got jid " + strconv.Itoa(jl.jid) + " with name " + jl.name)
 
             // Prepare next loop iteration
             cslastjidval := C.CString(strconv.Itoa(lastjailid))
             defer C.free(unsafe.Pointer(cslastjidval))
-            C.jailparam_import(&params[3], cslastjidval)
+            C.jailparam_import(&params[4], cslastjidval)
         }
     }
 
-	// Free 4 items of params list
-    C.jailparam_free(&params[0], 4)
+	// Free 5 items of params list
+    C.jailparam_free(&params[0], 5)
 
     return jls, err
 }

main.go

@@ -1,10 +1,9 @@
 package main
 
 import (
-    "gocage/cmd"
+	"gocage/cmd"
 )
 
-func main () {
-    cmd.Execute()
+func main() {
+	cmd.Execute()
 }
-

service/gocage

@@ -0,0 +1,43 @@
+#!/bin/sh
+#
+# $FreeBSD$
+#
+
+# PROVIDE: gocage
+# REQUIRE: LOGIN cleanvar
+# KEYWORD: shutdown
+
+# Add the following lines to /etc/rc.conf to enable :
+#
+# gocage_enable="YES"
+#
+# gocage_conf="/usr/local/etc/gocage.conf.yml"
+#
+
+. /etc/rc.subr
+
+name="gocage"
+rcvar=gocage_enable
+
+# read configuration and set defaults
+load_rc_config "$name"
+
+: ${gocage_enable:="NO"}
+: ${gocage_conf="/usr/local/etc/gocage.conf.yml"}
+
+start_cmd=${name}_start
+stop_cmd=${name}_stop
+
+gocage_start()
+{
+    echo "Gocage starting jails... "
+    /usr/local/bin/gocage -c ${gocage_conf} start
+}
+
+gocage_stop()
+{
+    echo "Gocage stopping jails... "
+    /usr/local/bin/gocage  -c ${gocage_conf} stop
+}
+
+run_rc_command "$1"