gocage/cmd/start.go

package cmd

import (
	"os"
	"fmt"
	"net"
	"sync"
	"time"
	"errors"
	"regexp"
	"reflect"
	"strconv"
	"strings"
	"io/ioutil"
	"crypto/rand"
	"gocage/jail"
	"encoding/hex"
	
	"github.com/c-robinson/iplib"
	log "github.com/sirupsen/logrus"
)

func mountProcFs(jail *Jail) error {
	cmd := fmt.Sprintf("mount -t procfs proc %s/proc", jail.RootPath)
	_, err := executeCommand(cmd)
	if err != nil {
		return errors.New(fmt.Sprintf("Error mounting procfs on %s/proc: %s", jail.RootPath, err.Error()))
	}

	return nil
}

func mountLinProcFs(jail *Jail) error {
	ldir := fmt.Sprintf("%s/compat/linux/proc", jail.RootPath)
	_, err := os.Stat(ldir)
	if os.IsNotExist(err) {
		errDir := os.MkdirAll(ldir, 0755)
		if errDir != nil {
			return errors.New(fmt.Sprintf("Error creating directory %s: %s", ldir, errDir.Error()))
		}
	}
	cmd := fmt.Sprintf("mount -t linprocfs linproc %s", ldir)
	_, err = executeCommand(cmd)
	if err != nil {
		return errors.New(fmt.Sprintf("Error mounting linprocfs on %s: %s", ldir, err.Error()))
	}

	return nil
}

func mountDevFs(jail *Jail) error {
	cmd := fmt.Sprintf("mount -t devfs dev %s/dev", jail.RootPath)
	_, err := executeCommand(cmd)
	if err != nil {
		return errors.New(fmt.Sprintf("Error mounting devfs on %s/dev: %s", jail.RootPath, err.Error()))
	}

	return nil
}

func mountFdescFs(jail *Jail) error {
	// FreeBSD <= 9.3 do not support fdescfs
	if gJailHost.version.major < 9 || (gJailHost.version.major == 9 && gJailHost.version.minor <= 3) {
		//fmt.Printf("    FreeBSD <= 9.3 does not support fdescfs, disabling in config\n")
		jail.Config.Mount_fdescfs = 0
		// Tag config so it will be synced on disk
		if err := setJailConfigUpdated(jail); err != nil {
			//fmt.Printf(fmt.Sprintf("Error updating config for jail %s: %s", jail.Name, err.Error()))
			return err
		}

		// Should we consider this an error?
		return nil
	}

	cmd := fmt.Sprintf("mount -t fdescfs descfs %s/dev/fd", jail.RootPath)
	_, err := executeCommand(cmd)
	if err != nil {
		return errors.New(fmt.Sprintf("Error mounting fdescfs on %s/dev/fd: %s", jail.RootPath, err.Error()))
	}

	return nil
}

// devfs & fdescfs are automatically mounted by jail binary (seen on FreeBSD 13.1)
func mountAllJailFsFromHost(jail *Jail) error {
	procfsFound := false
	linProcfsFound := false

	cmd := "mount -p"
	out, err := executeCommand(cmd)
	if err != nil {
		return errors.New(fmt.Sprintf("Error executing mount: %s", err.Error()))
	}

	var outclean []string
	remSpPtrn := regexp.MustCompile(`\s+`)
	for _, l := range strings.Split(out, "\n") {
		outclean = append(outclean, remSpPtrn.ReplaceAllString(l, " "))
	}

	// Check if these FS are already mounted
	for _, l := range outclean {
		f := strings.Split(l, " ")
		if len(f) > 2 {

			if strings.EqualFold(f[1], fmt.Sprintf("%s/proc", jail.RootPath)) {
				procfsFound = true
			}
			if strings.EqualFold(f[1], fmt.Sprintf("%s/compat/linux/proc", jail.RootPath)) {
				linProcfsFound = true
			}
		}
	}

	// Mount wanted FS
	if jail.Config.Mount_procfs > 0 && procfsFound == false {
		err := mountProcFs(jail)
		if err != nil {
			return err
		}
	}

	if jail.Config.Mount_linprocfs > 0 && linProcfsFound == false {
		err = mountLinProcFs(jail)
		if err != nil {
			return err
		}
	}

	// Ces montages doivent-ils etre effectués une fois le jail démarré?

	// FreeBSD <= 9.3 do not support fdescfs
	if gJailHost.version.major < 9 || (gJailHost.version.major == 9 && gJailHost.version.minor <= 3) {
		if jail.Config.Allow_mount_tmpfs > 0 {
			//fmt.Printf("    FreeBSD <= 9.3 does not support tmpfs, disabling in config\n")
			jail.Config.Allow_mount_tmpfs = 0
			// Tag config so it will be synced on disk
			jail.ConfigUpdated = true
			err = setJailConfigUpdated(jail)
			if err != nil {
				fmt.Printf(fmt.Sprintf("Error updating config for jail %s: %s", jail.Name, err.Error()))
				return err
			}
		}
	}

	if gJailHost.version.major < 12 {
		if jail.Config.Allow_mlock > 0 {
			jail.Config.Allow_mlock = 0
			if err = setJailConfigUpdated(jail); err != nil {
				//fmt.Printf(fmt.Sprintf("Error updating config for jail %s: %s", jail.Name, err.Error()))
				return err
			}
		}
		if jail.Config.Allow_mount_fusefs > 0 {
			jail.Config.Allow_mount_fusefs = 0
			if err = setJailConfigUpdated(jail); err != nil {
				//fmt.Printf(fmt.Sprintf("Error updating config for jail %s: %s", jail.Name, err.Error()))
				return err
			}
		}
		if jail.Config.Allow_vmm > 0 {
			jail.Config.Allow_vmm = 0
			if err = setJailConfigUpdated(jail); err != nil {
				//fmt.Printf(fmt.Sprintf("Error updating config for jail %s: %s", jail.Name, err.Error()))
				return err
			}
		}
	}

	return nil
}

// TODO
func prepareJailedZfsDatasets(jail *Jail) error {
	if jail.Config.Jail_zfs > 0 {
		// For jail to mount filesystem, enforce_statfs should be 1 or lower (2 is the default)
		if jail.Config.Allow_mount != 1 {
			jail.Config.Allow_mount = 1
			setJailConfigUpdated(jail)
		}
		if jail.Config.Allow_mount_zfs != 1 {
			jail.Config.Allow_mount_zfs = 1
			setJailConfigUpdated(jail)
		}
		// TODO : Overload Json Unmarshalling to fix bad typed values, keeping iocage compatibility
		if jail.Config.Enforce_statfs > "1" {
			jail.Config.Enforce_statfs = "1"
			setJailConfigUpdated(jail)
		}
		for _, d := range strings.Split(jail.Config.Jail_zfs_dataset, " ") {
			// Check if dataset exist, create if necessary
			// Support jailing datasets on differents pools : dataset should be specified with pool name
			cmd := fmt.Sprintf("zfs get -H creation %s", d)
			out, err := executeCommand(cmd)
			if err != nil {
				if strings.HasSuffix(out, "dataset does not exist") {
			        // Support jailing datasets on differents pools : dataset should be specified with pool name
					cmd = fmt.Sprintf("zfs create -o compression=lz4 -o mountpoint=none %s", d)
					_, err = executeCommand(cmd)
					if err != nil {
						return errors.New(fmt.Sprintf("Error creating dataset %s: %s", d, err.Error()))
					}
				} else {
					return errors.New(fmt.Sprintf("Error getting zfs dataset %s: %s", d, err.Error()))
				}
			}

			cmd = fmt.Sprintf("zfs set jailed=on %s", d)
			out, err = executeCommand(cmd)
			if err != nil {
				return errors.New(fmt.Sprintf("Error executing \"zfs set jailed=on %s\": %s", d, err.Error()))
			}
		}
	}
	return nil
}

func jailZfsDatasets(jail *Jail) error {
	if jail.Config.Jail_zfs > 0 {
		for _, d := range strings.Split(jail.Config.Jail_zfs_dataset, " ") {
			// Jail dataset
			// Support jailing datasets on differents pools : dataset should be specified with pool name
			cmd := fmt.Sprintf("zfs jail %d %s", jail.JID, d)
			out, err := executeCommand(cmd)
			if err != nil {
				return errors.New(fmt.Sprintf("Error jailling zfs dataset %s: %v: out", d, err, out))
			}
			
			// Mount from inside jail if mountpoint is set
			cmd = fmt.Sprintf("zfs get -H -o value mountpoint %s", d)
			out, err = executeCommand(cmd)
			if err != nil {
				return errors.New(fmt.Sprintf("Error getting zfs dataset %s mountpoint: %v: %s", d, err, out))
			}
			if len(out) > 0 && out != "-" && (false == strings.EqualFold(out, "none")) {
				// Should we "mount -a" ? cmd = fmt.Sprintf("zfs mount -a")
				cmd = fmt.Sprintf("zfs mount %s", d)
				out, err = executeCommandInJail(jail, cmd)
				if err != nil {
					// If already mounted, continue processing
					if ! strings.HasSuffix(out, "filesystem already mounted\n") {
						return errors.New(fmt.Sprintf("Error mounting zfs dataset %s from inside jail: %v: %s", d, err, out))
					}
				}
			}
		}
	}
	
	return nil
}

// tcp(80:8080),tcp(3300-3310:33060-33070)
func getNatForwardsArray(nat_forwards string, decompose_range bool) ([]NatDesc, error) {
	var res []NatDesc

	regx := `(tcp|udp)\(([0-9]{1,5}(?:-[0-9]{1,5})?):([0-9]{1,5}(?:-[0-9]{1,5})?)\)`
	re := regexp.MustCompile(regx)

	for _, match := range re.FindAllStringSubmatch(nat_forwards, -1) {
		if strings.Contains(match[2], "-") == true && decompose_range == true {
			sjrange, err := strconv.Atoi(strings.Split(match[2], "-")[0])
			if err != nil {
				return res, err
			}
			ejrange, err := strconv.Atoi(strings.Split(match[2], "-")[1])
			if err != nil {
				return res, err
			}
			shrange, err := strconv.Atoi(strings.Split(match[3], "-")[0])
			if err != nil {
				return res, err
			}
			ehrange, err := strconv.Atoi(strings.Split(match[3], "-")[1])
			if err != nil {
				return res, err
			}
			if (ejrange - sjrange) != (ehrange - shrange) {
				return res, errors.New(fmt.Sprintf("Invalid port range in nat_forwards: %s", match[0]))
			}
			for i := sjrange; i <= ejrange; i++ {
				nd := NatDesc{Proto: match[1],
					JailPort: strconv.Itoa(i),
					HostPort: strconv.Itoa(shrange + (i - sjrange)),
				}
				res = append(res, nd)
			}
		} else {
			nd := NatDesc{Proto: match[1],
				JailPort: match[2],
				HostPort: match[3],
			}
			res = append(res, nd)
		}
	}

	return res, nil
}

/*****************************************************************************
 * Enable or disables DHCP or accept_rtadv for interfaces declared with this
 *   option
 ****************************************************************************/
func configureDhcpOrAcceptRtadv(jail *Jail, ipproto int, enable bool) error {
	var nics []string

	// Iocage legacy behavior to enable on all interfaces if Config.Dhcp is set...
	if ipproto == IPv4 && jail.Config.Dhcp > 0 || enable == false {
		nic_list := strings.Split(jail.Config.Interfaces, ",")
		for _, n := range nic_list {
			nics = append(nics, strings.Split(n, ":")[0])
		}
		//  ...else enable for selected interface in Config.IpX_addr
	} else {
		if ipproto == IPv4 {
			for _, i := range strings.Split(jail.Config.Ip4_addr, ",") {
				if strings.EqualFold(strings.ToLower(strings.Split(i, "|")[1]), "dhcp") {
					nics = append(nics, i)
				}
			}
		} else {
			for _, i := range strings.Split(jail.Config.Ip6_addr, ",") {
				if strings.EqualFold(strings.ToLower(strings.Split(i, "|")[1]), "accept_rtadv") {
					nics = append(nics, i)
				}
			}
		}
	}

	for _, n := range nics {
		// vnet0 is epair0b inside jail
		//if strings.Contains(n, "vnet") {
		if strings.HasPrefix(n, "vnet") {
			splitd := strings.Split(n, "|")
			n = fmt.Sprintf("%sb", strings.Replace(splitd[0], "vnet", "epair", 1))
		}
		key := fmt.Sprintf("ifconfig_%s", n)
		value := "SYNCDHCP"

		if ipproto == IPv6 {
			key = fmt.Sprintf("%s_ipv6", key)
			value = "\"inet6 auto_linklocal accept_rtadv autoconf\""
		}

		if enable == true {
			err := enableRcKeyValue(fmt.Sprintf("%s/etc/rc.conf", jail.RootPath), key, value)
			if err != nil {
				return fmt.Errorf("ERROR setting %s=%s with sysrc for jail %s: %s\n", key, value, jail.Name, err)
			}
		} else {
			err := disableRcKey(fmt.Sprintf("%s/etc/rc.conf", jail.RootPath), key)
			if err != nil {
				return fmt.Errorf("ERROR deleting %s with sysrc for jail %s: %v\n", key, jail.Name, err)
			}
		}
	}

	return nil
}

func checkRtsold(jail *Jail) error {
	if strings.Contains(jail.Config.Ip6_addr, "accept_rtadv") == false {
		return fmt.Errorf("Must set at least one ip6_addr to accept_rtadv!\n")
	}
	err := enableRcKeyValue(fmt.Sprintf("%s/etc/rc.conf", jail.RootPath), "rtsold_enable", "yes")
	if err != nil {
		return fmt.Errorf("ERROR setting rtsold_enable=YES with sysrc for jail %s: %s\n", jail.Name, err)
	}
	return nil
}

func checkNat(backend string) error {
	cmd := "/sbin/sysctl -q net.inet.ip.forwarding=1"
	_, err := executeCommand(cmd)
	if err != nil {
		return fmt.Errorf("ERROR executing \"/sbin/sysctl -q net.inet.ip.forwarding=1\": %s", err)
	}

	if strings.EqualFold(backend, "pf") {
		// Load module and enable pf
		out, err := executeCommand("/sbin/kldload -n pf")
		if err != nil {
			if false == strings.Contains(out, "module already loaded or in kernel") {
				return fmt.Errorf("ERROR executing \"/sbin/kldload pf\": %s", err)
			}
		}
		out, err = executeCommand("/sbin/pfctl -e")
		if err != nil {
			if false == strings.Contains(out, "pf already enabled") {
				return fmt.Errorf("ERROR executing \"/sbin/pfctl -e\": %s", err)
			}
		}
	} else if strings.EqualFold(backend, "ipwf") {
		// Check if module loaded
		out, err := executeCommand("/sbin/sysctl net.inet.ip.fw.enable=1")
		if err != nil {
			if false == strings.Contains(out, "unknown oid 'net.inet.ip.fw.enable'") {
				return fmt.Errorf("ERROR executing \"/sbin/sysctl net.inet.ip.fw.enable=1\": %s", err)
			}
		}

		_, _ = executeCommand("/bin/kenv net.inet.ip.fw.default_to_accept=1")
		_, _ = executeCommand("/sbin/kldload -n ipfw")
		_, _ = executeCommand("/sbin/kldload -n ipfw_nat")
		_, err = executeCommand("/sbin/sysctl -q net.inet.ip.fw.enable=1")
		if err != nil {
			return fmt.Errorf("ERROR executing \"/sbin/sysctl -q net.inet.ip.fw.enable=1\": %s", err)
		}
	}

	return nil
}

/*****************************************************************************
 * Get IPv4 in use in all running jails
 ****************************************************************************/
func getJailsInUseIPv4() ([]string, error) {
	var ips []string

	re := regexp.MustCompile(ifconfigipv4re)

	for _, j := range gJails {
		if j.Running == true {
			out, err := executeCommandInJail(&j, "/sbin/ifconfig")
			if err != nil {
				return ips, fmt.Errorf("ERROR executing \"/sbin/ifconfig\" in jail %s: %s", j.Name, err)
			}

			for _, line := range strings.Split(out, "\n") {
				if re.MatchString(line) {
					ips = append(ips, re.FindStringSubmatch(line)[1])
				}
			}
		}
	}

	return ips, nil
}

func genNatIpv4(jail *Jail) ([]string, error) {
	var ippair []string

	// Get all IP in use, host and jails
	inuseip4, err := getHostInUseIPv4()
	if err != nil {
		return ippair, err
	}
	ij, err := getJailsInUseIPv4()
	if err != nil {
		return ippair, err
	}
	inuseip4 = append(inuseip4, ij...)

	// TODO : Voir https://github.com/iocage/iocage/blob/e94863d4c54f02523fb09e62e48be7db9ac92eda/iocage_lib/ioc_common.py#L1026

	for i := 0; i < 256; i++ {
		for j := 0; j < 256; j += 4 {
			n := iplib.NewNet4(net.ParseIP(fmt.Sprintf("172.16.%d.%d", i, j)), 30)
			for _, ip := range n.Enumerate(0, 0) {
				ippair = append(ippair, ip.String())
			}

			found := false
			for _, ip := range inuseip4 {
				for _, ipn := range ippair {
					if ip == ipn {
						found = true
					}
				}
			}
			if found == false {
				return ippair, nil
			}
		}
	}

	return ippair, nil
}

// FIXME : Must lock this function so parallel start do not 
func buildDevfsRuleSet(jail *Jail, m *sync.Mutex) (error, int) {
	rulesets := []int{}
	
	m.Lock()
	//defer m.Unlock()

	// Get known rulesets
	out, err := executeCommand("devfs rule showsets")
	if err != nil {
		m.Unlock()
		return errors.New(fmt.Sprintf("Error executing command \"devfs rule showsets\": %v; command returned: %s\n", err, out)), 0
	}
	srs := strings.Split(out, "\n")
	for _, i := range srs {
		if len(i) > 0 {
			j, err := strconv.Atoi(i)
			if err != nil {
				panic(err)
			}
			rulesets = append(rulesets, j)
		}
	}
	
	// Build a dynamic ruleset
	ruleset := MIN_DYN_DEVFS_RULESET
	for _, r := range rulesets {
		if ruleset == r {
			ruleset++
		}
	}
	
	log.Debugf("buildDevfsRuleSet: Build ruleset %d\n", ruleset)
	
	// Get default devfs_ruleset for the datastore
	// UPDATE: We don't need this as every jail have a default Devfs_ruleset value
	/*ds, err := getDatastoreFromArray(jail.Datastore, gDatastores)
	if err != nil {
		m.Unlock()
		return errors.New(fmt.Sprintf("Error getting datastore %s for jail %s", jail.Datastore, jail.Name)), 0
	}
	defaultrs, err := strconv.ParseInt(ds.DefaultJailConfig.Devfs_ruleset, 10, 64)
	if err != nil {
		m.Unlock()
		return errors.New(fmt.Sprintf("Error parsing default devfs_ruleset for datastore %s", jail.Datastore)), 0
	}*/
	
	// Clone configured devfs_ruleset to a dynamic ruleset
	if false == isStringInArray(srs, jail.Config.Devfs_ruleset) {
		m.Unlock()
		return errors.New(fmt.Sprintf("Unknown ruleset: %s", jail.Config.Devfs_ruleset)), 0
	}
	rs, _ := strconv.Atoi(jail.Config.Devfs_ruleset)
	err = copyDevfsRuleset(ruleset, rs)
	m.Unlock()
	if err != nil {
		return err, 0
	}
	
	// Add rules for enabled options
	if jail.Config.Allow_mount_fusefs > 0 {
		err := addDevfsRuleToRuleset("path fuse unhide", ruleset)
		if err != nil {
			return err, 0
		}
	}
	if jail.Config.Bpf > 0 {
		err := addDevfsRuleToRuleset("path bpf* unhide", ruleset)
		if err != nil {
			return err, 0
		}
	}
	if jail.Config.Allow_tun > 0 {
		err := addDevfsRuleToRuleset("path tun* unhide", ruleset)
		if err != nil {
			return err, 0
		}
	}
		
	return nil, ruleset
}

/*****************************************************************************
 * Build the file containing jail start parameters, in 
 *   /var/run/jail.${jail_internal_name}.conf
 ****************************************************************************/
func buildJailParameters(jail *Jail, dynruleset int) error {
	parameterFilePath := fmt.Sprintf("/var/run/jail.%s.conf", jail.InternalName)
	
	pfile, err := os.Create(parameterFilePath)
	if err != nil {
		return errors.New(fmt.Sprintf("Unable to create file /var/run/jail.%d.conf", jail.InternalName))
	}
	defer pfile.Close()
	
	if _, err = pfile.WriteString(fmt.Sprintf("%s {\n", jail.InternalName)); err != nil {
		return errors.New(fmt.Sprintf("Unable to write to file /var/run/jail.%d.conf: %v", jail.InternalName, err))
	}
	
	if jail.Config.Vnet > 0 { 
		if _, err = pfile.WriteString("    vnet;\n"); err != nil {
			return errors.New(fmt.Sprintf("Unable to write to file /var/run/jail.%d.conf: %v", jail.InternalName, err))
		}
	}
	// FreeBSD 9.3 and under do not support these 2 options
	if gJailHost.version.major > 9 || (gJailHost.version.major == 9 && gJailHost.version.minor > 3) {
		if _, err = pfile.WriteString("    mount.fdescfs = \"1\";\n"); err != nil {
			return errors.New(fmt.Sprintf("Unable to write to file /var/run/jail.%d.conf: %v", jail.InternalName, err))
		}
		if jail.Config.Allow_mount_tmpfs > 0 {
			if _, err = pfile.WriteString("    allow.mount.tmpfs = \"1\";\n"); err != nil {
				return errors.New(fmt.Sprintf("Unable to write to file /var/run/jail.%d.conf: %v", jail.InternalName, err))
			}
		}
	}
	
	if gJailHost.version.major >= 12 {
		if jail.Config.Allow_mlock > 0 {
			if _, err = pfile.WriteString("    allow.mlock = \"1\";\n"); err != nil {
				return errors.New(fmt.Sprintf("Unable to write to file /var/run/jail.%d.conf: %v", jail.InternalName, err))
			}
		}
		if jail.Config.Allow_mount_fusefs > 0 {
			if _, err = pfile.WriteString("    allow.mount.fusefs = \"1\";\n"); err != nil {
				return errors.New(fmt.Sprintf("Unable to write to file /var/run/jail.%d.conf: %v", jail.InternalName, err))
			}
		}
		if jail.Config.Allow_vmm > 0 {
			if _, err = pfile.WriteString("    allow.vmm = \"1\";\n"); err != nil {
				return errors.New(fmt.Sprintf("Unable to write to file /var/run/jail.%d.conf: %v", jail.InternalName, err))
			}
		}
	}
	
	if jail.Config.Allow_set_hostname > 0 {
		if _, err = pfile.WriteString("    allow.set_hostname = \"1\";\n"); err != nil {
			return errors.New(fmt.Sprintf("Unable to write to file /var/run/jail.%d.conf: %v", jail.InternalName, err))
		}
	}
	if jail.Config.Mount_devfs > 0 {
		if _, err = pfile.WriteString("    mount.devfs = \"1\";\n"); err != nil {
			return errors.New(fmt.Sprintf("Unable to write to file /var/run/jail.%d.conf: %v", jail.InternalName, err))
		}
	}
	if jail.Config.Allow_raw_sockets > 0 {
		if _, err = pfile.WriteString("    allow.raw_sockets = \"1\";\n"); err != nil {
			return errors.New(fmt.Sprintf("Unable to write to file /var/run/jail.%d.conf: %v", jail.InternalName, err))
		}
	}
	if jail.Config.Allow_sysvipc > 0 {
		if _, err = pfile.WriteString("    allow.sysvipc = \"1\";\n"); err != nil {
			return errors.New(fmt.Sprintf("Unable to write to file /var/run/jail.%d.conf: %v", jail.InternalName, err))
		}
	}
	if jail.Config.Allow_quotas > 0 {
		if _, err = pfile.WriteString("    allow.quotas = \"1\";\n"); err != nil {
			return errors.New(fmt.Sprintf("Unable to write to file /var/run/jail.%d.conf: %v", jail.InternalName, err))
		}
	}
	if jail.Config.Allow_socket_af > 0 {
		if _, err = pfile.WriteString("    allow.socket_af = \"1\";\n"); err != nil {
			return errors.New(fmt.Sprintf("Unable to write to file /var/run/jail.%d.conf: %v", jail.InternalName, err))
		}
	}
	if jail.Config.Allow_chflags > 0 {
		if _, err = pfile.WriteString("    allow.chflags = \"1\";\n"); err != nil {
			return errors.New(fmt.Sprintf("Unable to write to file /var/run/jail.%d.conf: %v", jail.InternalName, err))
		}
	}
	if jail.Config.Allow_mount > 0 {
		if _, err = pfile.WriteString("    allow.mount = \"1\";\n"); err != nil {
			return errors.New(fmt.Sprintf("Unable to write to file /var/run/jail.%d.conf: %v", jail.InternalName, err))
		}
	}
	if jail.Config.Allow_mount_devfs > 0 {
		if _, err = pfile.WriteString("    allow.mount.devfs = \"1\";\n"); err != nil {
			return errors.New(fmt.Sprintf("Unable to write to file /var/run/jail.%d.conf: %v", jail.InternalName, err))
		}
	}
	if jail.Config.Allow_mount_nullfs > 0 {
		if _, err = pfile.WriteString("    allow.mount.nullfs = \"1\";\n"); err != nil {
			return errors.New(fmt.Sprintf("Unable to write to file /var/run/jail.%d.conf: %v", jail.InternalName, err))
		}
	}
	if jail.Config.Allow_mount_procfs > 0 {
		if _, err = pfile.WriteString("    allow.mount.procfs = \"1\";\n"); err != nil {
			return errors.New(fmt.Sprintf("Unable to write to file /var/run/jail.%d.conf: %v", jail.InternalName, err))
		}
	}
	if jail.Config.Allow_mount_zfs > 0 {
		if _, err = pfile.WriteString("    allow.mount.zfs = \"1\";\n"); err != nil {
			return errors.New(fmt.Sprintf("Unable to write to file /var/run/jail.%d.conf: %v", jail.InternalName, err))
		}
	}
	if gJailHost.version.major > 10 || (gJailHost.version.major == 10 && gJailHost.version.minor > 3) {
		if _, err = pfile.WriteString(fmt.Sprintf("    sysvmsg = \"%s\";\n", jail.Config.Sysvmsg)); err != nil {
			return errors.New(fmt.Sprintf("Unable to write to file /var/run/jail.%d.conf: %v", jail.InternalName, err))
		}
		if _, err = pfile.WriteString(fmt.Sprintf("    sysvsem = \"%s\";\n", jail.Config.Sysvsem)); err != nil {
			return errors.New(fmt.Sprintf("Unable to write to file /var/run/jail.%d.conf: %v", jail.InternalName, err))
		}
		if _, err = pfile.WriteString(fmt.Sprintf("    sysvshm = \"%s\";\n", jail.Config.Sysvshm)); err != nil {
			return errors.New(fmt.Sprintf("Unable to write to file /var/run/jail.%d.conf: %v", jail.InternalName, err))
		}
	}
	if gJailHost.version.major >= 12 {
		if _, err = pfile.WriteString(fmt.Sprintf("    exec.created = \"%s\";\n", jail.Config.Exec_created)); err != nil {
			return errors.New(fmt.Sprintf("Unable to write to file /var/run/jail.%d.conf: %v", jail.InternalName, err))
		}
	}
	if _, err = pfile.WriteString(fmt.Sprintf("    host.domainname = \"%s\";\n", jail.Config.Host_domainname)); err != nil {
		return errors.New(fmt.Sprintf("Unable to write to file /var/run/jail.%d.conf: %v", jail.InternalName, err))
	}
	if _, err = pfile.WriteString(fmt.Sprintf("    host.hostname = \"%s\";\n", jail.Config.Host_hostname)); err != nil {
		return errors.New(fmt.Sprintf("Unable to write to file /var/run/jail.%d.conf: %v", jail.InternalName, err))
	}
	if _, err = pfile.WriteString(fmt.Sprintf("    path = \"%s\";\n", jail.RootPath)); err != nil {
		return errors.New(fmt.Sprintf("Unable to write to file /var/run/jail.%d.conf: %v", jail.InternalName, err))
	}
	if _, err = pfile.WriteString(fmt.Sprintf("    securelevel = \"%s\";\n", jail.Config.Securelevel)); err != nil {
		return errors.New(fmt.Sprintf("Unable to write to file /var/run/jail.%d.conf: %v", jail.InternalName, err))
	}
	
	// FIXME : Handle the same name jail on different datastores
	if _, err = pfile.WriteString(fmt.Sprintf("    host.hostuuid = \"%s\";\n", jail.Config.Host_hostuuid)); err != nil {
		return errors.New(fmt.Sprintf("Unable to write to file /var/run/jail.%d.conf: %v", jail.InternalName, err))
	}
	
	if _, err = pfile.WriteString(fmt.Sprintf("    devfs_ruleset = \"%d\";\n", dynruleset)); err != nil {
		return errors.New(fmt.Sprintf("Unable to write to file /var/run/jail.%d.conf: %v", jail.InternalName, err))
	}
	if _, err = pfile.WriteString(fmt.Sprintf("    enforce_statfs = \"%s\";\n", jail.Config.Enforce_statfs)); err != nil {
		return errors.New(fmt.Sprintf("Unable to write to file /var/run/jail.%d.conf: %v", jail.InternalName, err))
	}
	if _, err = pfile.WriteString(fmt.Sprintf("    children.max = \"%s\";\n", jail.Config.Children_max)); err != nil {
		return errors.New(fmt.Sprintf("Unable to write to file /var/run/jail.%d.conf: %v", jail.InternalName, err))
	}
	if _, err = pfile.WriteString(fmt.Sprintf("    exec.clean = \"%d\";\n", jail.Config.Exec_clean)); err != nil {
		return errors.New(fmt.Sprintf("Unable to write to file /var/run/jail.%d.conf: %v", jail.InternalName, err))
	}
	if _, err = pfile.WriteString(fmt.Sprintf("    exec.timeout = \"%s\";\n", jail.Config.Exec_timeout)); err != nil {
		return errors.New(fmt.Sprintf("Unable to write to file /var/run/jail.%d.conf: %v", jail.InternalName, err))
	}
	if _, err = pfile.WriteString(fmt.Sprintf("    stop.timeout = \"%s\";\n", jail.Config.Stop_timeout)); err != nil {
		return errors.New(fmt.Sprintf("Unable to write to file /var/run/jail.%d.conf: %v", jail.InternalName, err))
	}
	splits := strings.Split(jail.RootPath, "/")
	fstabPath := strings.Join(append(splits[:len(splits)-1], 
									 strings.Replace(splits[len(splits)-1], "root", "fstab", 1)), "/")
	if _, err = pfile.WriteString(fmt.Sprintf("    mount.fstab = \"%s\";\n", fstabPath)); err != nil {
		return errors.New(fmt.Sprintf("Unable to write to file /var/run/jail.%d.conf: %v", jail.InternalName, err))
	}
	if _, err = pfile.WriteString("    allow.dying;\n"); err != nil {
		return errors.New(fmt.Sprintf("Unable to write to file /var/run/jail.%d.conf: %v", jail.InternalName, err))
	}
	// get current datastore to get mountpoint
	ds, err := getDatastoreFromArray(jail.Datastore, gDatastores)
	if err != nil {
		return errors.New(fmt.Sprintf("Unable to get datastore %s for jail %s", jail.Datastore, jail.Name))
	}
	consolelog := fmt.Sprintf("%s/log/%s-console.log", ds.Mountpoint, jail.InternalName)
	if _, err = pfile.WriteString(fmt.Sprintf("    exec.consolelog = \"%s\";\n", consolelog)); err != nil {
		return errors.New(fmt.Sprintf("Unable to write to file /var/run/jail.%d.conf: %v", jail.InternalName, err))
	}
	if jail.Config.Ip_hostname > 0 {
		if _, err = pfile.WriteString(fmt.Sprintf("    ip_hostname = \"%s\";\n", jail.Config.Ip_hostname)); err != nil {
			return errors.New(fmt.Sprintf("Unable to write to file /var/run/jail.%d.conf: %v", jail.InternalName, err))
		}
	}
	if _, err = pfile.WriteString("    persist;\n"); err != nil {
		return errors.New(fmt.Sprintf("Unable to write to file /var/run/jail.%d.conf: %v", jail.InternalName, err))
	}
	
	if _, err = pfile.WriteString("}\n"); err != nil {
		return errors.New(fmt.Sprintf("Unable to write to file /var/run/jail.%d.conf: %v", jail.InternalName, err))
	}
	return nil
}

func setJailVnetIp(jail *Jail, nic string, ip string) error {
	if strings.Contains(ip, ":") {
		if jail.Config.Dhcp == 0 && false == strings.EqualFold(ip, "dhcp") {
			cmd := fmt.Sprintf("/sbin/ifconfig %s inet6 %s up", nic, ip)
			_, err := executeCommandInJail(jail, cmd)
			if err != nil {
				return fmt.Errorf("Error setting network inside jail: %v", err)
			}
		}
	} else {
		if jail.Config.Dhcp == 0 && false == strings.EqualFold(ip, "dhcp") {
			// vnetX is epairXb inside the jail
			cmd := fmt.Sprintf("/sbin/ifconfig %s %s up", nic, ip)
			_, err := executeCommandInJail(jail, cmd)
			if err != nil {
				return fmt.Errorf("Error setting network inside jail: %v", err)
			}
		}
	}
	return nil
}

// Generate a MAC address for the jail and save in config
// Returns host-side MAC, jail-side MAC, error
func generateMAC(jail *Jail, nic string) ([]byte, []byte, error) {
	var prefix []byte
	var err error
	if len(jail.Config.Mac_prefix) > 0 {
		prefix, err = hex.DecodeString(jail.Config.Mac_prefix)
		if err != nil {
			return []byte{0,0,0,0,0,0}, []byte{0,0,0,0,0,0}, err
		}
	} else {
		ds, err := getDatastoreFromArray(jail.Datastore, gDatastores)
		if err != nil {
			return []byte{0,0,0,0,0,0}, []byte{0,0,0,0,0,0}, err
		}
		prefix, err = hex.DecodeString(ds.DefaultJailConfig.Mac_prefix)
		if err != nil {
			return []byte{0,0,0,0,0,0}, []byte{0,0,0,0,0,0}, err
		}
	}
	
	// Set the local bit, ensure unicast address
	//prefix[0] = (prefix[0] | 2) & 0xfe
    
	suffix := make([]byte, (6-len(prefix)))
	_, err = rand.Read(suffix)
	if err != nil {
		return []byte{0,0,0,0,0,0}, []byte{0,0,0,0,0,0}, err
	}
	
	hsmac := append(prefix, suffix...)
	jsmac := make([]byte, 6)
	copy(jsmac, hsmac)
	jsmac[5] = jsmac[5] + 1
	
	// Save MACs to config
	pname := fmt.Sprintf("Config.%s_mac", strings.Title(nic))
	err = setStructFieldValue(jail, pname, fmt.Sprintf("%s %s", hex.EncodeToString(hsmac), hex.EncodeToString(jsmac)))
	if err != nil {
		return []byte{0,0,0,0,0,0}, []byte{0,0,0,0,0,0}, fmt.Errorf("Error setting %s for jail %s: %s\n", pname, jail.Name, err.Error())
	}
	setJailConfigUpdated(jail)
    
	return hsmac, jsmac, nil
}

// Setup interface host side
// Returns array of epair names, error
func setupVnetInterfaceHostSide(jail *Jail) ([]string, error) {
	var epairs []string
	var jsnic string
	var hsmac []byte
	var err error
	
	for _, nicCnf := range strings.Split(jail.Config.Interfaces, ",") {
		v := strings.Split(nicCnf, ":")
		if len(v) != 2 {
			return []string{}, fmt.Errorf("Invalid value for Interfaces: %s\n", nicCnf)
		}
		nic := v[0]
		bridge := v[1]
		
		// Get host side MAC
		pname := fmt.Sprintf("Config.%s_mac", strings.Title(nic))
		var val *reflect.Value
		val, pname, err = getStructFieldValue(jail, pname)
		if err != nil {
			if strings.HasPrefix(err.Error(), "Field not found") {
				hsmac, _, err = generateMAC(jail, nic)
				if err != nil {
					return []string{}, err
				}
			} else {
				return []string{}, err
			}
		} else {
			if strings.EqualFold(val.String(), "none") {
				hsmac, _, err = generateMAC(jail, nic)
				if err != nil {
					return []string{}, err
				}
			}
			hsmac, err = hex.DecodeString(strings.Split(val.String(), " ")[0])
			if err != nil {
				return []string{}, fmt.Errorf("Error converting %s to hex\n", strings.Split(val.String(), " ")[0])
			}
		}
		
		// Get bridge MTU
		mtu, err := gJailHost.GetBridgeMTU(bridge)
		if err != nil {
			return []string{}, fmt.Errorf("Error getting bridge \"%s\" mtu: %v\n", bridge, err)
		}
		
		// Create epair interface
		out, err := executeCommand("/sbin/ifconfig epair create")
		if err != nil {
			return []string{}, fmt.Errorf("Error creating epair interface: %v\n", err)
		}
		hsepair := strings.TrimRight(out, "\n")
		
		if true == strings.Contains(nic, "vnet") {
			jsnic = strings.Replace(nic, "vnet", "epair", 1) + "b"
		} else {
			jsnic = nic
		}
		
		// Name epair and set MTU
		cmd := fmt.Sprintf("/sbin/ifconfig %s name %s.%d mtu %d", hsepair, nic, jail.JID, mtu)
		out, err = executeCommand(cmd)
		if err != nil {
			return []string{}, fmt.Errorf("Error creating epair interface host side: %v\n", err)
		}
		
		cmd = fmt.Sprintf("/sbin/ifconfig %s.%d link %s", nic, jail.JID, hex.EncodeToString(hsmac))
		out, err = executeCommand(cmd)
		if err != nil {
			return []string{}, fmt.Errorf("Error setting mac host side: %v\n", err)
		}
		
		cmd = fmt.Sprintf("/sbin/ifconfig %s.%d description \"associated with jail: %s as nic: %s\"", nic, jail.JID, jail.Name, jsnic)
		out, err = executeCommand(cmd)
		if err != nil {
			return []string{}, fmt.Errorf("Error setting nic description: %v\n", err)
		}
		epairs = append(epairs, hsepair)
	}
	log.Debugf("setupVnetInterfaceHostSide: returning %v\n", epairs)
	return epairs, nil
}

func setupVnetInterfaceJailSide(jail *Jail, hostepairs []string) error {
	var jsmac []byte
	var err error

	// Maps of IP by netcard (Ex: jailIp4s["vnet0"] = "192.168.1.1")
	var ip4s = make(map[string]string)
	var ip6s = make(map[string]string)
	
	for _, i := range strings.Split(jail.Config.Ip4_addr, ",") {
		v := strings.Split(i, "|")
		if len(v) > 1 {
			ip4s[v[0]] = v[1]
		}
	}
	for _, i := range strings.Split(jail.Config.Ip6_addr, ",") {
		v := strings.Split(i, "|")
		if len(v) > 1 {
			ip6s[v[0]] = v[1]
		}
	}

	// Loop through configured interfaces
	for i, nicCnf := range strings.Split(jail.Config.Interfaces, ",") {
		v := strings.Split(nicCnf, ":")
		if len(v) != 2 {
			return fmt.Errorf("Invalid value for Interfaces: %s\n", nicCnf)
		}
		// vnet nic name
		nic := v[0]
		bridge := v[1]
		// inside jail final nic name
		jnic := strings.Replace(v[0], "vnet", "epair", 1)
		jnic = jnic + "b"
		// host side associated jail nic name
		jsepair := fmt.Sprintf("%sb", strings.TrimSuffix(hostepairs[i], "a"))

		// Get jail side MAC
		pname := fmt.Sprintf("Config.%s_mac", strings.Title(nic))
		var val *reflect.Value
		val, pname, err = getStructFieldValue(jail, pname)
		if err != nil {
			if strings.HasPrefix(err.Error(), "Field not found") {
				_, jsmac, err = generateMAC(jail, nic)
				if err != nil {
					return err
				}
			} else {
				return err
			}
		} else {
			jsmac, err = hex.DecodeString(strings.Split(val.String(), " ")[1])
                        if err != nil {
                                return fmt.Errorf("Error converting %s to hex\n", strings.Split(val.String(), " ")[1])
                        }

		}

		cmd := fmt.Sprintf("/sbin/ifconfig %s vnet %s", jsepair, jail.InternalName)
		_, err := executeCommand(cmd)
		if err != nil {
			return fmt.Errorf("Error linking interface to jail: %v\n", err)
		}

		// Get bridge MTU
		mtu, err := gJailHost.GetBridgeMTU(bridge)
		if err != nil {
			return fmt.Errorf("Error getting bridge \"%s\" mtu: %v\n", bridge, err)
		}

		cmd = fmt.Sprintf("/usr/sbin/jexec %d ifconfig %s mtu %d", jail.JID, jsepair, mtu)
		_, err = executeCommand(cmd)
		if err != nil {
			return fmt.Errorf("Error setting mtu: %v\n", err)
		}

		// rename epairXXb to epair0b (or opair1b, ...)
		cmd = fmt.Sprintf("/usr/sbin/setfib %s jexec %d ifconfig %s name %s", jail.Config.Exec_fib, jail.JID, jsepair, jnic)
		_, err = executeCommand(cmd)
		if err != nil {
			return fmt.Errorf("Error linking interface to jail: %v\n", err)
		}

		cmd = fmt.Sprintf("/usr/sbin/setfib %s jexec %d ifconfig %s link %s", jail.Config.Exec_fib,
						  jail.JID, jnic, hex.EncodeToString(jsmac))
		_, err = executeCommand(cmd)
		if err != nil {
			return fmt.Errorf("Error setting mac: %v\n", err)
		}

		// TODO: Move outside of this function
		// add interface to bridge
		if jail.Config.Nat == 0 {
			cmd := fmt.Sprintf("/sbin/ifconfig %s addm %s.%d up", bridge, nic, jail.JID)
			out, err := executeCommand(cmd)
			if err != nil {
				return fmt.Errorf("Error adding member %s to %s: %v: %s\n", nic, bridge, err, out)
			}
		}

		// Check we have an IP for the nic, and set it into jail
		if len(ip4s[nic]) > 0 {
			err = setJailVnetIp(jail, jnic, ip4s[nic])
		}

		if len(ip6s[nic]) > 0 {
			err = setJailVnetIp(jail, jnic, ip6s[nic])
		}

		// finally up interface
		if jail.Config.Nat == 0 {
			cmd := fmt.Sprintf("/sbin/ifconfig %s.%d up", nic, jail.JID)
			out, err := executeCommand(cmd)
			if err != nil {
				return fmt.Errorf("Error upping interface %s.%d: %v: %s\n", nic, jail.JID, err, out)
			}
		}
	}

	log.Debugf("setupVnetInterfaceJailSide: return with success\n")

	return nil
}

func generateResolvConf(jail *Jail) error {
	if jail.Config.Resolver != "/etc/resolv.conf" && jail.Config.Resolver != "none" && jail.Config.Resolver != "/dev/null" {
		f, err := os.Create(fmt.Sprintf("%s/etc/resolv.conf", jail.RootPath))
		if err != nil {
			return err
		}
		defer f.Close()
		for _, l := range strings.Split(jail.Config.Resolver, ";") {
			f.WriteString(fmt.Sprintf("%s\n", l))
		}
	} else if jail.Config.Resolver == "none" || jail.Config.Resolver == "/etc/resolv.conf" {
		read, err := ioutil.ReadFile("/etc/resolv.conf")
		if err != nil {
			return fmt.Errorf("Error opening /etc/resolv.conf: %v", err)
		}
		err = ioutil.WriteFile(fmt.Sprintf("%s/etc/resolv.conf", jail.RootPath), read, 0644)
		if err != nil {
			return fmt.Errorf("Error writing to %s/etc/resolv.conf: %v", jail.RootPath, err)
		}
	} else if jail.Config.Resolver == "/dev/null" {
		// Just do nothing
	}
	
	return nil
}

func copyLocalTime(jail *Jail) error {
	if jail.Config.Host_time > 0 {
		read, err := ioutil.ReadFile("/etc/localtime")
		if err != nil {
			return fmt.Errorf("Error opening /etc/localtime: %v", err)
		}
		err = ioutil.WriteFile(fmt.Sprintf("%s/etc/localtime", jail.RootPath), read, 0644)
		if err != nil {
			return fmt.Errorf("Error writing to %s/etc/localtime: %v", jail.RootPath, err)
		}
	}
	
	return nil
}


// TODO: Umount all FS
// TODO: Delete devfs ruleset
// TODO: Remove /var/run/jail-${internalname}.conf
func cleanAfterStartCrash() {
	
}

// Start all jails with boot=true, in priority order
func StartJailsAtBoot() {
	var startList []Jail
	var wg *sync.WaitGroup
	var curThNb int
	var curPri int

	// Get boot enabled non-template jails
	for _, j := range gJails {
		if j.Config.Boot > 0 && !strings.EqualFold(j.Config.Jailtype, "template") {
			startList = append(startList, j)
		}
	}

	// Order by priority
	js := initJailSortStruct()
	fct, _, err := getStructFieldValue(js, "Config.PriorityInc")
	if err != nil {
		log.Errorf("ERROR getting JailSort struct field \"Config.PriorityInc\"\n")
		return
	}
	JailsOrderedBy(fct.Interface().(jailLessFunc)).Sort(startList)

	wg = new(sync.WaitGroup)
	curThNb = 0
	for i, j := range startList {
		jFullName := fmt.Sprintf("%s/%s", j.Datastore, j.Name)
		log.Debugf("Starting %s with priority %s\n", jFullName, j.Config.Priority)
		jailPri, err := strconv.Atoi(j.Config.Priority)
		if err != nil {
			panic(fmt.Sprintf("Invalid format for Priority (Jail %s)\n", jFullName))
		}
		
		if (curThNb >= gMaxThreads || i == 0) {
			// FIXME : Use a pool instead of waiting for all threads to run a new one
			wg.Wait()
			curThNb = 0
			
			wg.Add(1)
			curThNb++
			curPri = jailPri
			go func(jailFullName string) {
				defer wg.Done()
				StartJail([]string{jailFullName})
			}(jFullName)
		} else {
			if (curPri == jailPri) {
				wg.Add(1)
				curThNb++
				go func(jailFullName string) {
					defer wg.Done()
					StartJail([]string{jailFullName})
				}(jFullName)
			} else {
				wg.Wait()
				curThNb = 0
			
				wg.Add(1)
				curThNb++
				curPri = jailPri
				go func(jailFullName string) {
					defer wg.Done()
					StartJail([]string{jailFullName})
				}(jFullName)
			}
		}
	}
	wg.Wait()
}


/*
	Start jail:
		Check jail fstab?
		Mount procfs
		Mount linprocfs
		Mount devfs?
		Mount fdescfs?
		If jail_zfs, then check jail_zfs_dataset exist (and create otherwise)
		TODO : Check NAT settings and compatibility with other jails
		Generate devfsruleset from configured
		Write config file in /var/run/jails.ioc-$NAME.conf
		Execute PreStart (Exec_prestart)
		Start jail (With ENV VARS for network conf)
		Start networking
		Mount jail_zfs_datasets inside jail
		Generate resolv.Conf
		Copy /etc/localtime into jail (?)
		Configure NAT
		Execute Exec_start into jail
		Execute Exec_poststart
		If DHCP, check with ifconfig inside jail
		Set RCTL Rules

	Use setfib for each jail command
*/
func StartJail(args []string) {
	// jail we have to start
	var cj *Jail
	var err error

	for _, a := range args {
		// Check if jail exist and is distinctly named
		cj, err = getJailFromArray(a, []string{"basejail", "jail"}, gJails)
		if err != nil {
			fmt.Printf("Error getting jail: %s\n", err)
			continue
		}

		if cj.Running == true {
			fmt.Printf("Jail %s/%s is already running!\n", cj.Datastore, cj.Name)
			continue
		}
		
		fmt.Printf("> Starting jail %s\n", cj.Name)

		// Set InternalName as it is used by some of these
		cj.InternalName = fmt.Sprintf("ioc-%s", cj.Name)

		if len(cj.Config.Hostid) > 0 && cj.Config.Hostid_strict_check > 0 {
			if strings.EqualFold(gJailHost.hostid, cj.Config.Hostid) == false {
				fmt.Printf("hostid is not matching and hostid_strict_check is on. Not starting jail.\n")
				return
			}
		}

		var props_missing []string
		// DHCP can also be set with "dhcp" value in ip4_addr (Eg: "vnet0|dhcp")
		if cj.Config.Dhcp > 0 || strings.Contains(strings.ToLower(cj.Config.Ip4_addr), "dhcp") == true {
			if cj.Config.Bpf == 0 {
				props_missing = append(props_missing, fmt.Sprintf("%s: dhcp requires bpf", cj.Name))
			}
			if cj.Config.Vnet == 0 {
				props_missing = append(props_missing, fmt.Sprintf("%s: dhcp requires vnet", cj.Name))
			}
		}

		// tcp(80:8080),tcp(3300-3310:33000-33010)
		if cj.Config.Nat > 0 && strings.EqualFold(cj.Config.Nat_forwards, "none") == false {
			// If NAT && port forwarding is enabled, check that port does not conflict
			//   with another running jail
			for _, j := range gJails {
				if j.Running == false || strings.EqualFold(j.Config.Nat_forwards, "none") == false || j.Config.Nat != 1 {
					continue
				} else {
					jnd, err := getNatForwardsArray(j.Config.Nat_forwards, true)
					if err != nil {
						fmt.Printf(err.Error())
						return
					}
					cjnd, err := getNatForwardsArray(cj.Config.Nat_forwards, true)
					if err != nil {
						fmt.Printf(err.Error())
						return
					}
					for _, jn := range jnd {
						for _, cjn := range cjnd {
							if jn == cjn {
								fmt.Printf("nat_forwards rule \"%s\" is in conflict with jail %s, won't start\n",
									fmt.Sprintf("%s(%s:%s)", cjn.Proto, cjn.JailPort, cjn.HostPort), j.Name)
								return
							}
						}
					}
				}
			}
		}

		if cj.Config.Nat > 0 && strings.EqualFold(cj.Config.Nat_interface, "none") == true {
			cj.Config.Nat_interface = gJailHost.GetDefaultInterface()
			cj.ConfigUpdated = true
		}

		if cj.Config.Vnet > 0 && strings.EqualFold(cj.Config.Defaultrouter, "auto") == true {
			cj.Config.Defaultrouter = gJailHost.GetDefaultGateway4()
			// "auto" default Gateway should not be updated to support jailhost route change
		}

		if cj.Config.Vnet > 0 && strings.EqualFold(cj.Config.Defaultrouter6, "auto") == true {
			cj.Config.Defaultrouter6 = gJailHost.GetDefaultGateway6()
			// "auto" default Gateway should not be updated to support jailhost route change
		}

		if strings.EqualFold(cj.Config.Ip6_addr, "accept_rtadv") && cj.Config.Vnet == 0 {
			props_missing = append(props_missing, fmt.Sprintf("%s: accept_rtadv requires vnet", cj.Name))
		}

		if cj.Config.Bpf > 0 && cj.Config.Vnet == 0 {
			props_missing = append(props_missing, fmt.Sprintf("%s: bpf requires vnet", cj.Name))
		}

		if len(props_missing) > 0 {
			for _, m := range props_missing {
				fmt.Printf("%s\n", m)
			}
			return
		}

		if cj.Config.Dhcp > 0 || strings.Contains(strings.ToLower(cj.Config.Ip4_addr), "dhcp") == true {
			err = configureDhcpOrAcceptRtadv(cj, IPv4, true)
		} else {
			err = configureDhcpOrAcceptRtadv(cj, IPv4, false)
		}
		if err != nil {
			fmt.Printf(err.Error())
			return
		}

		if cj.Config.Rtsold > 0 {
			err = checkRtsold(cj)
		}
		if err != nil {
			fmt.Printf(err.Error())
			return
		}

		if strings.Contains(strings.ToLower(cj.Config.Ip6_addr), "accept_rtadv") == true {
			err = configureDhcpOrAcceptRtadv(cj, IPv6, true)
		} else {
			err = configureDhcpOrAcceptRtadv(cj, IPv6, false)
		}
		if err != nil {
			fmt.Printf(err.Error())
			return
		}

		fmt.Printf("  > Mount special filesystems:\n")
		err := mountAllJailFsFromHost(cj)
		if err != nil {
			fmt.Printf("ERROR: %s\n", err.Error())
		} else {
			fmt.Printf("  > Mount special filesystems: OK\n")
		}

		if cj.Config.Jail_zfs > 0 {
			fmt.Printf("  > Prepare ZFS Datasets:\n")
			err := prepareJailedZfsDatasets(cj)
			if err != nil {
				fmt.Printf("ERROR: %s\n", err.Error())
			} else {
				fmt.Printf("  > Prepare ZFS Datasets: OK\n")
			}
		}

		// Check NAT backend
		if cj.Config.Nat > 0 {
			log.Debug("Check NAT backend %s\n", cj.Config.Nat_backend)
			err = checkNat(cj.Config.Nat_backend)
			if err != nil {
				fmt.Printf(err.Error())
				return
			}

			if cj.Config.Vnet == 0 {
				log.Debug("Generate NAT IPv4 without VNet\n")
				ip4, err := genNatIpv4(cj)
				if err != nil {
					fmt.Printf("%s\n", err.Error())
					return
				}

				log.Debug("Configuring NAT : Set ip4_addr to %s\n", ip4[0])
				// This IP should not be saved into json
				cj.Config.Ip4_addr = fmt.Sprintf("%s|%s", cj.Config.Nat_interface, ip4[0])
			} else {
				log.Debug("Generate NAT IPv4 with VNet\n")
				ip4, err := genNatIpv4(cj)
				if err != nil {
					fmt.Printf("%s\n", err.Error())
					return
				}
				log.Debug("Configuring NAT : Set ip4_addr to %s, defaultrouter to %s\n", ip4[1], ip4[0])
				// This IP should not be saved into json
				cj.Config.Ip4_addr = fmt.Sprintf("vnet0|%s", ip4[1])
				cj.Config.Defaultrouter = ip4[0]
			}
		}

		// See https://github.com/iocage/iocage/blob/e94863d4c54f02523fb09e62e48be7db9ac92eda/iocage_lib/ioc_start.py:401
		if cj.Config.Vnet == 0 {
			// Not supported
			fmt.Printf("Only VNet jails supported\n")
			return
		}

		var net []string
		if false == strings.EqualFold(cj.Config.Vnet_interfaces, "none") {
			net = append(net,  strings.Split(cj.Config.Vnet_interfaces, " ")...)
		}

		err, dynrs := buildDevfsRuleSet(cj, &gMdevfs)
		if err != nil {
			fmt.Printf("%s\n", err.Error())
			return
		}

		err = buildJailParameters(cj, dynrs)
		if err != nil {
			fmt.Printf("%s\n", err.Error())
			return
		}

		// Synchronize jail config to disk
		cj.WriteConfigToDisk(false)

		start_cmd := fmt.Sprintf("/usr/sbin/jail -f /var/run/jail.%s.conf -c", cj.InternalName)

		//TODO: handle start_env & prestart_env, could be used by iocage plugins
		_, err = executeScript(cj.Config.Exec_prestart)
		if err != nil {
			fmt.Printf("Error executing exec_prestart script: %v\n", err)
			fmt.Printf("Aborting jail start\n")
			return
		}

		fmt.Printf("  > Start jail:\n")
		_, err = executeCommand(start_cmd)
		if err != nil {
			fmt.Printf("Error starting jail %s: %v\n", cj.Name, err)
			return
		}

		fmt.Printf("  > Start jail: OK\n")
		fmt.Printf("  > With devfs ruleset %d\n", dynrs)

		// Update running state, JID and Devfs_ruleset
		cj.Running = true
		cj.Devfs_ruleset = dynrs
		rjails, err := jail.GetJails()
		if err != nil {
			fmt.Printf("Error: Unable to list running jails\n")
		}
		for _, j := range rjails {
			if j.Name == cj.InternalName {
				cj.JID = j.Jid
				break
			}
		}

		// Get a reference to current jail, to update its properties in RAM so API will get fresh data
		for i, j := range gJails {
			if strings.EqualFold(j.Name, cj.Name) && strings.EqualFold(j.Datastore, cj.Datastore) {
				if err = setStructFieldValue(&gJails[i], "Running", "true"); err != nil {
					fmt.Printf("ERROR: setting Running property to true: %s\n", err.Error())
				}
				if err = setStructFieldValue(&gJails[i], "JID", fmt.Sprintf("%d", cj.JID)); err != nil {
					fmt.Printf("ERROR: setting JID property to %d: %s\n", cj.JID, err.Error())
				}
				if err = setStructFieldValue(&gJails[i], "InternalName", cj.InternalName); err != nil {
					fmt.Printf("ERROR: Setting InternalName property: %s\n", err.Error())
				}
				// FIXME: this value of devfs_ruleset should not go in Config, it should reside in Jail root properties as it is volatile (only valid while jail is running)
				/*if err = setStructFieldValue(&gJails[i], "Devfs_ruleset", "0"); err != nil {
					fmt.Printf("ERROR: setting Devfs_ruleset property to 0: %s\n", err.Error())
				}*/
			}
		}

		hostInt, err := gJailHost.GetInterfaces()
		if err != nil {
			fmt.Printf("Error listing jail host interfaces: %v\n", err)
			return
		}

		if false == strings.EqualFold(cj.Config.Vnet_default_interface, "auto") &&
			false == strings.EqualFold(cj.Config.Vnet_default_interface, "none") &&
			false == isStringInArray(hostInt, cj.Config.Vnet_default_interface) {
			fmt.Printf("vnet_default_interface can be one of \"none\", \"auto\" of a valid interface (e.g. \"lagg0\")\n")
			return
		}

		fmt.Printf("  > Setup VNet network:\n")
		hsepairs, err := setupVnetInterfaceHostSide(cj);
		if err != nil {
			fmt.Printf("Error setting VNet interface host side: %v\n", err)
			return
		}

		if err = setupVnetInterfaceJailSide(cj, hsepairs); err != nil {
			fmt.Printf("Error setting VNet interface jail side: %v\n", err)
			return
		}
		fmt.Printf("  > Setup VNet network: OK\n")
		
		// Set default route, unless main network is dhcp
		if ! cj.isFirstNetDhcp() && !strings.EqualFold(cj.Config.Ip4_addr, "none") {
			fmt.Printf("  > Setup default ipv4 gateway:\n")
			cmd := fmt.Sprintf("/usr/sbin/setfib %s /usr/sbin/jexec %d route add default %s", cj.Config.Exec_fib, cj.JID, cj.Config.Defaultrouter)
			out, err := executeCommand(cmd)
			if err != nil && len(out) > 0 {
				fmt.Printf("Error: %v: %s\n", err, out)
			} else {
				fmt.Printf("  > Setup default ipv4 gateway: OK\n")
			}
		}

		if cj.Config.Ip6_addr != "none" {
			fmt.Printf("  > Setup default ipv6 gateway:\n")
			cmd := fmt.Sprintf("/usr/sbin/setfib %s /usr/sbin/jexec %d route add -6 default %s", cj.Config.Exec_fib, cj.JID, cj.Config.Defaultrouter6)
			out, err := executeCommand(cmd)
			if err != nil && len(out) > 0 {
				fmt.Printf("Error: %v: %s\n", err, out)
			} else {
				fmt.Printf("  > Setup default ipv6 gateway: OK\n")
			}
		}

		if cj.Config.Jail_zfs > 0 {
			fmt.Printf("  > Jail ZFS datasets:\n")
			err = jailZfsDatasets(cj)
			if err != nil {
				fmt.Printf("%v\n", err)
				return
			}
			fmt.Printf("  > Jail ZFS datasets: OK\n")
		}

		err = generateResolvConf(cj)
		if err != nil {
			fmt.Printf("%s\n", err)
		}

		if cj.Config.Host_time > 0 {
			err = copyLocalTime(cj)
			if err != nil {
				fmt.Printf("%s\n", err)
			}
		}

		// Start services
		if len(cj.Config.Exec_start) > 0 {
			fmt.Printf("  > Start services:\n")
			cmd := fmt.Sprintf("/usr/sbin/setfib %s /usr/sbin/jexec %d %s", cj.Config.Exec_fib, cj.JID, cj.Config.Exec_start)
			err := executeCommandNonBlocking(cmd)
			if err != nil {
				fmt.Printf("Error: %v\n", err)
			} else {
				fmt.Printf("  > Start services: OK\n")
			}
		}

		if cj.Config.Rtsold > 0 || strings.EqualFold(cj.Config.Ip6_addr, "accept_rtadv") {
			fmt.Printf("  > Start rtsold:\n")
			cmd := fmt.Sprintf("/usr/sbin/setfib %s /usr/sbin/jexec %d service rtsold start", cj.Config.Exec_fib, cj.JID)
			out, err := executeCommand(cmd)
			if err != nil && len(out) > 0 {
				fmt.Printf("Error: %v: %s\n", err, out)
			} else {
				fmt.Printf("  > Start rtsold: OK\n")
			}
		}

		// TODO: Execute Exec_poststart
		if len(cj.Config.Exec_poststart) > 0 {
			fmt.Printf("  > Execute post-start:\n")
			cmd := fmt.Sprintf("%s", cj.Config.Exec_poststart)
			out, err := executeCommand(cmd)
			if err != nil && len(out) > 0 {
				fmt.Printf("Error: %v: %s\n", err, out)
			} else {
				fmt.Printf("  > Execute post-start: OK\n")
			}
		}

		// WIP 10/07/2022 : https://github.com/iocage/iocage/blob/master/iocage_lib/ioc_start.py#L891
		// TODO: Handle dhcp
		// TODO: Apply rctl

		// Update last_started
		// 23/07/2023 : This is not working, when writing to disk the old value is used
		dt := time.Now()
		curDate := fmt.Sprintf("%s", dt.Format("2006-01-02 15:04:05"))
		cj.Config.Last_started = curDate
		writeConfigToDisk(cj, false)

		/*
			out, err := executeCommand(fmt.Sprintf("rctl jail:%s", cj.InternalName))
			if err == nil && len(out) > 0 {
				fmt.Printf("  > Remove RCTL rules:\n")
				err := removeRctlRules(cj.InternalName, []string{""})
				if err != nil {
					fmt.Printf("ERROR: %s\n", err.Error())
				} else {
					fmt.Printf("  > Remove RCTL rules: OK\n")
				}
			}

			if len (cj.Config.Exec_prestop) > 0 {
				fmt.Printf("  > Execute prestop:\n")
				_, err := executeCommand(cj.Config.Exec_prestop)
				if err != nil {
					fmt.Printf("ERROR: %s\n", err.Error())
				} else {
					fmt.Printf("  > Execute prestop: OK\n")
				}
			}

			if len (cj.Config.Exec_stop) > 0 {
				fmt.Printf("  > Execute stop:\n")
				_, err := executeCommandInJail(cj, cj.Config.Exec_stop)
				if err != nil {
					fmt.Printf("ERROR: %s\n", err.Error())
				} else {
					fmt.Printf("  > Execute stop: OK\n")
				}
			}

			if cj.Config.Jail_zfs > 0 {
				fmt.Printf("  > Umount jailed ZFS:\n")
				err := umountAndUnjailZFS(cj)
				if err != nil {
					fmt.Printf("ERROR: %s\n", err.Error())
				} else {
					fmt.Printf("  > Umount jailed ZFS: OK\n")
				}
			}

			if cj.Config.Vnet > 0 && len(cj.Config.Ip4_addr) > 0 {
				fmt.Printf("  > Destroy VNet interfaces:\n")
				err := destroyVNetInterfaces(cj)
				if err != nil {
					fmt.Printf("ERROR: %s\n", err.Error())
				} else {
					fmt.Printf("  > Destroy VNet interfaces: OK\n")
				}
			}

			fmt.Printf("  > Remove devfsruleset %s:\n", cj.Config.Devfs_ruleset)
			err = deleteDevfsRuleset(cj)
			if err != nil {
				fmt.Printf("ERROR: %s\n", err.Error())
			} else {
				fmt.Printf("  > Remove devfsruleset %s: OK\n", cj.Config.Devfs_ruleset)
			}

			fmt.Printf("  > Stop jail %s:\n", cj.Name)
			err = stopJail(cj)
			if err != nil {
				fmt.Printf("ERROR: %s\n", err.Error())
			} else {
				fmt.Printf("  > Stop jail %s: OK\n", cj.Name)
			}
		*/

	}
}