VNet configuration, jail and mount ZFS datasets into jail

This commit is contained in:
yo 2022-06-26 20:02:28 +02:00
parent 7266496cac
commit 276d01ed4c

View File

@ -8,24 +8,14 @@ import (
"net"
"os"
"regexp"
"reflect"
"strconv"
"strings"
"crypto/rand"
"gocage/jail"
"encoding/hex"
)
func setJailConfigUpdated(jail *Jail) error {
if len(jail.ConfigPath) == 0 {
return errors.New(fmt.Sprintf("No config path for jail %s", jail.Name))
}
j, err := getJailFromArray(jail.Name, gJails)
if err != nil {
return err
}
j.ConfigUpdated = true
return nil
}
func mountProcFs(jail *Jail) error {
cmd := fmt.Sprintf("mount -t procfs proc %s/proc", jail.RootPath)
_, err := executeCommand(cmd)
@ -66,11 +56,14 @@ func mountDevFs(jail *Jail) error {
func mountFdescFs(jail *Jail) error {
// FreeBSD <= 9.3 do not support fdescfs
if gHostVersion <= 9.3 {
fmt.Printf(" FreeBSD <= 9.3 does not support fdescfs, disabling in config\n")
if gJailHost.version.major < 9 || (gJailHost.version.major == 9 && gJailHost.version.minor <= 3) {
//fmt.Printf(" FreeBSD <= 9.3 does not support fdescfs, disabling in config\n")
jail.Config.Mount_fdescfs = 0
// Tag config so it will be synced on disk
jail.ConfigUpdated = true
if err := setJailConfigUpdated(jail); err != nil {
//fmt.Printf(fmt.Sprintf("Error updating config for jail %s: %s", jail.Name, err.Error()))
return err
}
// Should we consider this an error?
return nil
@ -85,11 +78,10 @@ func mountFdescFs(jail *Jail) error {
return nil
}
// devfs & fdescfs are automatically mounted by jail binary (seen on FreeBSD 13.1)
func mountAllJailFsFromHost(jail *Jail) error {
procfsFound := false
linProcfsFound := false
devfsFound := false
fdescfsFound := false
cmd := "mount -p"
out, err := executeCommand(cmd)
@ -114,12 +106,6 @@ func mountAllJailFsFromHost(jail *Jail) error {
if strings.EqualFold(f[1], fmt.Sprintf("%s/compat/linux/proc", jail.RootPath)) {
linProcfsFound = true
}
if strings.EqualFold(f[1], fmt.Sprintf("%s/dev", jail.RootPath)) {
devfsFound = true
}
if strings.EqualFold(f[1], fmt.Sprintf("%s/dev/fd", jail.RootPath)) {
fdescfsFound = true
}
}
}
@ -138,48 +124,44 @@ func mountAllJailFsFromHost(jail *Jail) error {
}
}
if jail.Config.Mount_devfs > 0 && devfsFound == false {
err := mountDevFs(jail)
if err != nil {
return err
}
}
if jail.Config.Mount_fdescfs > 0 && fdescfsFound == false {
err := mountFdescFs(jail)
if err != nil {
return err
}
}
// Ces montages doivent-ils etre effectués une fois le jail démarré?
// FreeBSD <= 9.3 do not support fdescfs
//if gHostVersion <= 9.3 && jail.Config.Allow_mount_tmpfs > 0 {
if gHostVersion <= 9.3 && jail.Config.Allow_mount_tmpfs > 0 {
fmt.Printf(" FreeBSD <= 9.3 does not support tmpfs, disabling in config\n")
jail.Config.Allow_mount_tmpfs = 0
// Tag config so it will be synced on disk
jail.ConfigUpdated = true
err = setJailConfigUpdated(jail)
if err != nil {
fmt.Printf(fmt.Sprintf("Error updating config for jail %s: %s", jail.Name, err.Error()))
return err
if gJailHost.version.major < 9 || (gJailHost.version.major == 9 && gJailHost.version.minor <= 3) {
if jail.Config.Allow_mount_tmpfs > 0 {
//fmt.Printf(" FreeBSD <= 9.3 does not support tmpfs, disabling in config\n")
jail.Config.Allow_mount_tmpfs = 0
// Tag config so it will be synced on disk
jail.ConfigUpdated = true
err = setJailConfigUpdated(jail)
if err != nil {
fmt.Printf(fmt.Sprintf("Error updating config for jail %s: %s", jail.Name, err.Error()))
return err
}
}
}
if gHostVersion < 12 {
if gJailHost.version.major < 12 {
if jail.Config.Allow_mlock > 0 {
jail.Config.Allow_mlock = 0
jail.ConfigUpdated = true
/* WIP
err = setJailProperty(jail, "Config.Allow_mlock", "0")
if err != nil {
if err = setJailConfigUpdated(jail); err != nil {
//fmt.Printf(fmt.Sprintf("Error updating config for jail %s: %s", jail.Name, err.Error()))
return err
}*/
}
}
if jail.Config.Allow_mount_fusefs > 0 {
jail.Config.Allow_mount_fusefs = 0
if err = setJailConfigUpdated(jail); err != nil {
//fmt.Printf(fmt.Sprintf("Error updating config for jail %s: %s", jail.Name, err.Error()))
return err
}
}
if jail.Config.Allow_vmm > 0 {
jail.Config.Allow_vmm = 0
if err = setJailConfigUpdated(jail); err != nil {
//fmt.Printf(fmt.Sprintf("Error updating config for jail %s: %s", jail.Name, err.Error()))
return err
}
}
}
@ -224,12 +206,38 @@ func prepareJailedZfsDatasets(jail *Jail) error {
if err != nil {
return errors.New(fmt.Sprintf("Error executing \"zfs set jailed=on %s/%s\": %s", jail.Zpool, d, err.Error()))
}
// TODO : Execute "zfs jail $jailname $dataset" when jail will be up
}
}
return nil
}
func jailZfsDatasets(jail *Jail) error {
for _, d := range strings.Split(jail.Config.Jail_zfs_dataset, " ") {
// Jail dataset
cmd := fmt.Sprintf("zfs jail %d %s/%s", jail.JID, jail.Zpool, d)
out, err := executeCommand(cmd)
if err != nil {
return errors.New(fmt.Sprintf("Error jailling zfs dataset %s: %v: out", d, err, out))
}
// Mount from inside jail if mountpoint is set
cmd = fmt.Sprintf("zfs get -H -o value mountpoint %s/%s", jail.JID, jail.Zpool, d)
out, err = executeCommand(cmd)
if err != nil {
return errors.New(fmt.Sprintf("Error getting zfs dataset %s mountpoint: %v: out", d, err, out))
}
if len(out) > 0 && out != "-" && (false == strings.EqualFold(out, "none")) {
cmd = fmt.Sprintf("zfs mount %s", jail.Zpool, d)
out, err = executeCommandInJail(jail, cmd)
if err != nil {
return errors.New(fmt.Sprintf("Error mounting zfs dataset %s: %v: out", d, err, out))
}
}
}
return nil
}
// tcp(80:8080),tcp(3300-3310:33060-33070)
func getNatForwardsArray(nat_forwards string, decompose_range bool) ([]NatDesc, error) {
var res []NatDesc
@ -328,7 +336,7 @@ func configureDhcpOrAcceptRtadv(jail *Jail, ipproto int, enable bool) error {
} else {
err := disableRcKey(jail.ConfigPath, key)
if err != nil {
return fmt.Errorf("ERROR deleting %s with sysrc for jail %s: %s\n", key, value, jail.Name, err)
return fmt.Errorf("ERROR deleting %s with sysrc for jail %s: %v\n", key, jail.Name, err)
}
}
}
@ -389,20 +397,25 @@ func checkNat(backend string) error {
return nil
}
/*****************************************************************************
* Get IPv4 in use in all running jails
****************************************************************************/
func getJailsInUseIPv4() ([]string, error) {
var ips []string
re := regexp.MustCompile(ifconfigipv4re)
for _, j := range gJails {
out, err := executeCommandInJail(&j, "/sbin/ifconfig")
if err != nil {
return ips, fmt.Errorf("ERROR executing \"/sbin/ifconfig\" in jail %s: %s", j.Name, err)
}
if j.Running == true {
out, err := executeCommandInJail(&j, "/sbin/ifconfig")
if err != nil {
return ips, fmt.Errorf("ERROR executing \"/sbin/ifconfig\" in jail %s: %s", j.Name, err)
}
for _, line := range strings.Split(out, "\n") {
if re.MatchString(line) {
ips = append(ips, re.FindStringSubmatch(line)[1])
for _, line := range strings.Split(out, "\n") {
if re.MatchString(line) {
ips = append(ips, re.FindStringSubmatch(line)[1])
}
}
}
}
@ -460,11 +473,13 @@ func buildDevfsRuleSet(jail *Jail) (error, int) {
}
srs := strings.Split(out, "\n")
for _, i := range srs {
j, err := strconv.Atoi(i)
if err != nil {
panic(err)
if len(i) > 0 {
j, err := strconv.Atoi(i)
if err != nil {
panic(err)
}
rulesets = append(rulesets, j)
}
rulesets = append(rulesets, j)
}
// Build a dynamic ruleset
@ -718,6 +733,270 @@ func buildJailParameters(jail *Jail, dynruleset int) error {
return nil
}
func setJailVnetIp(jail *Jail, nic string, ip string) error {
if strings.Contains(ip, ":") {
if jail.Config.Dhcp == 0 && false == strings.EqualFold(ip, "dhcp") {
cmd := fmt.Sprintf("/sbin/ifconfig %s inet6 %s up", nic, ip)
_, err := executeCommandInJail(jail, cmd)
if err != nil {
return fmt.Errorf("Error setting network inside jail: %v", err)
}
}
} else {
if jail.Config.Dhcp == 0 && false == strings.EqualFold(ip, "dhcp") {
// vnetX is epairXb inside the jail
cmd := fmt.Sprintf("/sbin/ifconfig %s %s up", nic, ip)
_, err := executeCommandInJail(jail, cmd)
if err != nil {
return fmt.Errorf("Error setting network inside jail: %v", err)
}
}
}
return nil
}
// Generate a MAC address for the jail and save in config
// Returns host-side MAC, jail-side MAC, error
func generateMAC(jail *Jail, nic string) ([]byte, []byte, error) {
var prefix []byte
var err error
if len(jail.Config.Mac_prefix) > 0 {
prefix, err = hex.DecodeString(jail.Config.Mac_prefix)
if err != nil {
return []byte{0,0,0,0,0,0}, []byte{0,0,0,0,0,0}, err
}
} else {
ds, err := getDatastoreFromArray(jail.Datastore, gDatastores)
if err != nil {
return []byte{0,0,0,0,0,0}, []byte{0,0,0,0,0,0}, err
}
prefix, err = hex.DecodeString(ds.DefaultJailConfig.Mac_prefix)
if err != nil {
return []byte{0,0,0,0,0,0}, []byte{0,0,0,0,0,0}, err
}
}
// Set the local bit, ensure unicast address
//prefix[0] = (prefix[0] | 2) & 0xfe
suffix := make([]byte, (6-len(prefix)))
_, err = rand.Read(suffix)
if err != nil {
return []byte{0,0,0,0,0,0}, []byte{0,0,0,0,0,0}, err
}
hsmac := append(prefix, suffix...)
jsmac := append(hsmac[:5], hsmac[5]+1)
// Save MACs to config
pname := fmt.Sprintf("Config.%s_mac", strings.Title(nic))
err = setStructFieldValue(jail, pname, fmt.Sprintf("%s %s", hex.EncodeToString(hsmac), hex.EncodeToString(jsmac)))
if err != nil {
return []byte{0,0,0,0,0,0}, []byte{0,0,0,0,0,0}, fmt.Errorf("Error setting %s for jail %s: %s\n", pname, jail.Name, err.Error())
}
setJailConfigUpdated(jail)
return hsmac, jsmac, nil
}
// Setup interface host side
// Returns array of epair names, error
func setupVnetInterfaceHostSide(jail *Jail) ([]string, error) {
var epairs []string
var jsnic string
var hsmac []byte
var err error
for _, nicCnf := range strings.Split(jail.Config.Interfaces, ",") {
v := strings.Split(nicCnf, ":")
if len(v) != 2 {
return []string{}, fmt.Errorf("Invalid value for Interfaces: %s\n", nicCnf)
}
nic := v[0]
bridge := v[1]
// Get host side MAC
pname := fmt.Sprintf("Config.%s_mac", nic)
var val *reflect.Value
val, pname, err = getStructFieldValue(jail, pname)
if err != nil {
if strings.HasPrefix(err.Error(), "Field not found") {
hsmac, _, err = generateMAC(jail, nic)
if err != nil {
return []string{}, err
}
} else {
return []string{}, err
}
} else {
hsmac = val.Bytes()
}
// Get bridge MTU
mtu, err := gJailHost.GetBridgeMTU(bridge)
if err != nil {
return []string{}, fmt.Errorf("Error getting bridge mtu: %v\n", err)
}
// Create epair interface
out, err := executeCommand("/sbin/ifconfig epair create")
if err != nil {
return []string{}, fmt.Errorf("Error creating epair interface: %v\n", err)
}
hsepair := strings.TrimRight(out, "\n")
if true == strings.Contains(nic, "vnet") {
jsnic = strings.Replace(nic, "vnet", "epair", 1) + "b"
} else {
jsnic = nic
}
// Name epair and set MTU
cmd := fmt.Sprintf("/sbin/ifconfig %s name %s.%d mtu %d", hsepair, nic, jail.JID, mtu)
out, err = executeCommand(cmd)
if err != nil {
return []string{}, fmt.Errorf("Error creating epair interface host side: %v\n", err)
}
cmd = fmt.Sprintf("/sbin/ifconfig %s.%d link %s", nic, jail.JID, hex.EncodeToString(hsmac))
out, err = executeCommand(cmd)
if err != nil {
return []string{}, fmt.Errorf("Error setting mac host side: %v\n", err)
}
cmd = fmt.Sprintf("/sbin/ifconfig %s.%d description \"associated with jail: %s as nic: %s\"", nic, jail.JID, jail.Name, jsnic)
out, err = executeCommand(cmd)
if err != nil {
return []string{}, fmt.Errorf("Error setting nic description: %v\n", err)
}
epairs = append(epairs, hsepair)
}
return epairs, nil
}
func setupVnetInterfaceJailSide(jail *Jail, hsepair string) error {
var jsmac []byte
var err error
// Maps of IP by netcard (Ex: jailIp4s["vnet0"] = "192.168.1.1")
var ip4s = make(map[string]string)
var ip6s = make(map[string]string)
for _, i := range strings.Split(jail.Config.Ip4_addr, ",") {
v := strings.Split(i, "|")
ip4s[v[0]] = v[1]
}
for _, i := range strings.Split(jail.Config.Ip6_addr, ",") {
v := strings.Split(i, "|")
if len(v) > 1 {
ip6s[v[0]] = v[1]
}
}
// Loop through configured interfaces
for _, nicCnf := range strings.Split(jail.Config.Interfaces, ",") {
v := strings.Split(nicCnf, ":")
if len(v) != 2 {
return fmt.Errorf("Invalid value for Interfaces: %s\n", nicCnf)
}
// vnet nic name
nic := v[0]
bridge := v[1]
// inside jail final nic name
jnic := strings.Replace(v[0], "vnet", "epair", 1)
jnic = jnic + "b"
// Get jail side MAC
pname := fmt.Sprintf("Config.%s_mac", nic)
var val *reflect.Value
val, pname, err = getStructFieldValue(jail, pname)
if err != nil {
if strings.HasPrefix(err.Error(), "Field not found") {
_, jsmac, err = generateMAC(jail, nic)
if err != nil {
return err
}
} else {
return err
}
} else {
jsmac = val.Bytes()
}
lasta := strings.LastIndex(hsepair, "a")
jsepair := hsepair[:lasta] + strings.Replace(hsepair[lasta:], "a", "b", 1)
cmd := fmt.Sprintf("/sbin/ifconfig %s vnet %s", jsepair, jail.InternalName)
_, err := executeCommand(cmd)
if err != nil {
return fmt.Errorf("Error linking interface to jail: %v\n", err)
}
// Get bridge MTU
mtu, err := gJailHost.GetBridgeMTU(bridge)
if err != nil {
return fmt.Errorf("Error getting bridge mtu: %v\n", err)
}
cmd = fmt.Sprintf("/usr/sbin/jexec %d ifconfig %s mtu %d", jail.JID, jsepair, mtu)
_, err = executeCommand(cmd)
if err != nil {
return fmt.Errorf("Error setting mtu: %v\n", err)
}
// rename epairXXb to epair0b (or opair1b, ...)
cmd = fmt.Sprintf("/usr/sbin/setfib %s jexec %d ifconfig %s name %s", jail.Config.Exec_fib, jail.JID, jsepair, jnic)
_, err = executeCommand(cmd)
if err != nil {
return fmt.Errorf("Error linking interface to jail: %v\n", err)
}
cmd = fmt.Sprintf("/usr/sbin/setfib %s jexec %d ifconfig %s link %s", jail.Config.Exec_fib,
jail.JID, jnic, hex.EncodeToString(jsmac))
_, err = executeCommand(cmd)
if err != nil {
return fmt.Errorf("Error setting mac: %v\n", err)
}
// TODO: Move outside of this function
// add interface to bridge
if jail.Config.Nat == 0 {
cmd := fmt.Sprintf("/sbin/ifconfig %s addm %s.%d up", bridge, nic, jail.JID)
out, err := executeCommand(cmd)
if err != nil {
return fmt.Errorf("Error adding member %s to %s: %v: %s\n", nic, bridge, err, out)
}
}
// Check we have an IP for the nic, and set it into jail
if len(ip4s[nic]) > 0 {
err = setJailVnetIp(jail, jnic, ip4s[nic])
}
if len(ip6s[nic]) > 0 {
err = setJailVnetIp(jail, jnic, ip6s[nic])
}
// finally up interface
if jail.Config.Nat == 0 {
cmd := fmt.Sprintf("/sbin/ifconfig %s.%d up", nic, jail.JID)
out, err := executeCommand(cmd)
if err != nil {
return fmt.Errorf("Error upping interface %s.%d: %v: %s\n", nic, jail.JID, err, out)
}
}
}
return nil
}
// TODO: Umount all FS
// TODO: Delete devfs ruleset
// TODO: Remove /var/run/jail-${internalname}.conf
func cleanAfterStartCrash() {
}
/*
Start jail:
@ -768,6 +1047,9 @@ func StartJail(args []string) {
fmt.Printf("Jail %s is already running!\n", cj.Name)
continue
}
// Set InternalName as it is used by some of these
cj.InternalName = fmt.Sprintf("ioc-%s", cj.Name)
if len(cj.Config.Hostid) > 0 && cj.Config.Hostid_strict_check > 0 {
if strings.EqualFold(gJailHost.hostid, cj.Config.Hostid) == false {
@ -819,20 +1101,17 @@ func StartJail(args []string) {
}
if cj.Config.Nat > 0 && strings.EqualFold(cj.Config.Nat_interface, "none") == true {
var jhost JailHost
cj.Config.Nat_interface = jhost.GetDefaultInterface()
cj.Config.Nat_interface = gJailHost.GetDefaultInterface()
cj.ConfigUpdated = true
}
if cj.Config.Vnet > 0 && strings.EqualFold(cj.Config.Defaultrouter, "auto") == true {
var jhost JailHost
cj.Config.Defaultrouter = jhost.GetDefaultGateway4()
cj.Config.Defaultrouter = gJailHost.GetDefaultGateway4()
// "auto" default Gateway should not be updated to support jailhost route change
}
if cj.Config.Vnet > 0 && strings.EqualFold(cj.Config.Defaultrouter6, "auto") == true {
var jhost JailHost
cj.Config.Defaultrouter6 = jhost.GetDefaultGateway6()
cj.Config.Defaultrouter6 = gJailHost.GetDefaultGateway6()
// "auto" default Gateway should not be updated to support jailhost route change
}
@ -897,10 +1176,6 @@ func StartJail(args []string) {
}
}
// TODO : Check capabilites relative to FreeBSD Version when executing jail with all parameters
// See l.335 of https://github.com/iocage/iocage/blob/e94863d4c54f02523fb09e62e48be7db9ac92eda/iocage_lib/ioc_start.py
//checkCapabilities(cj)
// Check NAT backend
if cj.Config.Nat > 0 {
log.Debug("Check NAT backend %s\n", cj.Config.Nat_backend)
@ -953,7 +1228,6 @@ func StartJail(args []string) {
return
}
// CONTINUE HERE, around https://github.com/iocage/iocage/blob/master/iocage_lib/ioc_start.py:516
err = buildJailParameters(cj, dynrs)
if err != nil {
fmt.Printf("%s\n", err.Error())
@ -962,7 +1236,84 @@ func StartJail(args []string) {
// Synchronize jail config to disk
WriteConfigToDisk(false)
start_cmd := fmt.Sprintf("/usr/sbin/jail -f /var/run/jail.%s.conf -c", cj.InternalName)
//TODO: handle start_env & prestart_env, could be used by iocage plugins
_, err = executeScript(cj.Config.Exec_prestart)
if err != nil {
fmt.Printf("Error executing exec_prestart script: %v\n", err)
fmt.Printf("Aborting jail start\n")
return
}
fmt.Printf(" > Start jail:\n")
_, err = executeCommand(start_cmd)
if err != nil {
fmt.Printf("Error starting jail %s: %v\n", cj.Name, err)
return
}
fmt.Printf(" > Start jail: OK\n")
fmt.Printf(" > With devfs ruleset %d\n", dynrs)
// Update running state and JID
cj.Running = true
rjails, err := jail.GetJails()
if err != nil {
fmt.Printf("Error: Unable to list running jails\n")
}
for _, j := range rjails {
if j.Name == cj.InternalName {
cj.JID = j.Jid
break
}
}
hostInt, err := gJailHost.GetInterfaces()
if err != nil {
fmt.Printf("Error listing jail host interfaces: %v\n", err)
return
}
if false == strings.EqualFold(cj.Config.Vnet_default_interface, "auto") &&
false == strings.EqualFold(cj.Config.Vnet_default_interface, "none") &&
false == isStringInArray(hostInt, cj.Config.Vnet_default_interface) {
fmt.Printf("vnet_default_interface can be one of \"none\", \"auto\" of a valid interface (e.g. \"lagg0\")\n")
return
}
fmt.Printf(" > Setup VNet network:\n")
hsepairs, err := setupVnetInterfaceHostSide(cj);
if err != nil {
fmt.Printf("Error setting VNet interface host side: %v\n", err)
return
}
for _, ep := range hsepairs {
if err = setupVnetInterfaceJailSide(cj, ep); err != nil {
fmt.Printf("Error setting VNet interface jail side: %v\n", err)
return
}
}
fmt.Printf(" > Setup VNet network: OK\n")
err = jailZfsDatasets(cj)
if err != nil {
fmt.Printf("%v\n", err)
return
}
// WIP 26/06/2022 : https://github.com/iocage/iocage/blob/master/iocage_lib/ioc_start.py#L792
// TODO
//generateResolvConf(cj)
// TODO : Start services
/*
out, err := executeCommand(fmt.Sprintf("rctl jail:%s", cj.InternalName))
if err == nil && len(out) > 0 {