WIP: fetch command

.gitignore

@@ -1,3 +1,2 @@
 gocage
 go.sum
-

cmd/fetch.go

@@ -0,0 +1,206 @@
+package cmd
+
+import (
+	"io"
+	"os"
+	"fmt"
+	"bufio"
+	"bytes"
+	//"errors"
+	"strings"
+	"net/http"
+	"encoding/hex"
+	"crypto/sha256"
+
+	log "github.com/sirupsen/logrus"
+)
+
+const (
+	ReleaseServer = "download.freebsd.org"
+	ReleaseRootDir = "ftp/releases"
+)
+
+var (
+	FetchFiles = []string{"base.txz", "lib32.txz", "src.txz"}
+)
+
+// TODO: Check if files already exist
+// Fetch release files, verify, put in datastore under ${datastore}/releases
+// Only support http
+func fetchRelease(release string, proto string, arch string, datastore string) {
+	var ds Datastore
+	
+	log.SetReportCaller(true)
+	
+	if false == strings.EqualFold(proto, "http") {
+		fmt.Printf("Unsupported protocol: %s\n", proto)
+		return
+	}
+	
+	for _, ds = range gDatastores {
+		if strings.EqualFold(datastore, ds.Name) {
+			break
+		}
+	}
+	if false == strings.EqualFold(datastore, ds.Name) {
+		fmt.Printf("Datastore not found: %s\n", datastore)
+		return
+	}
+	
+	// Check datastore have a releases dataset, and it is mounted
+	releaseDsName := fmt.Sprintf("%s/releases", ds.ZFSDataset)
+	releaseDsMountPoint := fmt.Sprintf("%s/releases", ds.Mountpoint)
+	exist, err := doZfsDatasetExist(releaseDsName)
+	if err != nil {
+		fmt.Printf("Error accessing dataset %s: %v\n", releaseDsName, err)
+		return
+	}
+	if false == exist {
+		// Then create dataset
+		if err := createZfsDataset(releaseDsName, releaseDsMountPoint, "lz4"); err != nil {
+			fmt.Printf("Error creating dataset %s: %v\n", releaseDsName, err)
+			return
+		}
+	}
+	
+	// Create XX.X-RELEASE directory if necessary
+	fileDir := fmt.Sprintf("%s/%s-RELEASE", releaseDsMountPoint, release)
+	_, err = os.Stat(fileDir)
+	if os.IsNotExist(err) {
+		if err := os.Mkdir(fileDir, 0755); err != nil {
+			fmt.Printf("Error creating directory %s: %v\n", fileDir, err)
+			return
+		}
+	}
+
+
+	fetchUrl := fmt.Sprintf("%s://%s/%s/%s/%s-RELEASE", proto, ReleaseServer, ReleaseRootDir, arch, release)
+	log.Debugf("FetchURL = %s", fetchUrl)
+	
+	// check if proto/server/arch/release is available
+	if strings.EqualFold(proto, "http") {
+		resp, err := http.Get(fetchUrl)
+		if err != nil {
+			fmt.Printf("Can not get %s: %v\n", fetchUrl, err)
+			return
+		}
+		defer resp.Body.Close()
+		if resp.StatusCode != 200 {
+			fmt.Printf("Get %s returned %d, check release name\n", fetchUrl, resp.StatusCode)
+			return
+		}
+	}
+
+	// Fetch files
+	// Get MANIFEST so we get sha256 sums
+	if err := fetchHTTPFile(fetchUrl, "MANIFEST", fileDir, []byte{}); err != nil {
+		fmt.Printf("%v\n", err)
+		return
+	}
+	// Build an array of "file;checksum"
+	checksumMap, err := buildFileChecksumFromManifest(fmt.Sprintf("%s/MANIFEST", fileDir), FetchFiles)
+	if err != nil {
+		fmt.Printf("%v\n", err)
+		return
+	}
+
+	// Fetch remaining files, verify integrity and write to disk
+	for f, c := range checksumMap {
+		if strings.EqualFold(proto, "http") {
+			if err := fetchHTTPFile(fetchUrl, f, fileDir, c); err != nil {
+				fmt.Printf("%v\n", err)
+				return
+			}
+		}
+	}
+}
+
+func fetchHTTPFile(baseUrl, fileName, storeDir string, checksum []byte) error {
+	// Check storeDir exist
+	_, err := os.Stat(storeDir)
+	if os.IsNotExist(err) {
+		return fmt.Errorf("Directory does not exist: %s\n", storeDir)
+	}
+
+	url := fmt.Sprintf("%s/%s", baseUrl, fileName)
+	fmt.Printf("Fetching %s...", url)
+	resp, err := http.Get(url)
+	if err != nil {
+		fmt.Printf(" Error\n")
+		return fmt.Errorf("Can not get %s: %v\n", url, err)
+	}
+	defer resp.Body.Close()
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return fmt.Errorf("Can not read %s response body: %v\n", url, err)
+	}
+
+	// Check integrity
+	if len(checksum) > 0 {
+		err = checkIntegrity(body, checksum)
+		if err != nil {
+			return fmt.Errorf("Error checking integrity")
+		}
+	}
+	
+	dest := fmt.Sprintf("%s/%s", storeDir, fileName)
+
+	f, err := os.Create(dest)       // creates a file at current directory
+	if err != nil {
+		return fmt.Errorf("Can not create file %s: %v\n", dest, err)
+	}
+	defer f.Close()
+	f.Write(body)
+	fmt.Printf(" Done.\n")
+	return nil
+}
+
+
+func checkIntegrity(data []byte, checksum []byte) error {
+	sum := sha256.Sum256(data)
+	
+	if false == bytes.Equal(checksum[:],sum[:]) {
+		return fmt.Errorf("Invalid checksum: %x != %x", sum, checksum)
+	}
+	return nil
+}
+
+// Get checksum from manifest, for each file in fileList
+/* MANIFEST format:
+ * base-dbg.txz    a5b51f3d54686509e91ca9c30e9f1cd93dc757f25c643609b3c35e7119c0531d        1654    base_dbg        "Base system (Debugging)"       off
+ * base.txz        e85b256930a2fbc04b80334106afecba0f11e52e32ffa197a88d7319cf059840        26492   base    "Base system (MANDATORY)"       on
+ * kernel-dbg.txz  6b47a6cb83637af1f489aa8cdb802d9db936ea864887188cfc69d8075762214e        912     kernel_dbg      "Kernel (Debugging)"    on
+ */
+func buildFileChecksumFromManifest(manifest string, fileList []string) (map[string][]byte, error) {
+	var ckarr = make(map[string][]byte)
+
+	rm, err := os.Open(manifest)
+	if err != nil {
+		return ckarr, fmt.Errorf("Unable to open MANIFEST: %v", err)
+	}
+	fscan := bufio.NewScanner(rm)
+	fscan.Split(bufio.ScanLines)
+	
+	// For each MANIFEST line...
+	for fscan.Scan() {
+		fields := strings.Fields(fscan.Text())
+		fn := fields[0]
+		fck := fields[1]
+		hexSum, err := hex.DecodeString(fck)
+		if err != nil {
+			return ckarr, fmt.Errorf("Invalid value for checksum %s", fck)
+		}
+		// ... Find the corresponding file in fileList, then add to checksum array ckarr
+		for _, f := range fileList {
+			if strings.EqualFold(f, fn) {
+				ckarr[fn] = hexSum
+				break
+			}
+		}
+	}
+	
+	if len(ckarr) < len(fileList) {
+		return ckarr, fmt.Errorf("Missing file in MANIFEST")
+	}
+	return ckarr, nil
+}

cmd/jailhost.go

@@ -179,6 +179,15 @@ func getHostId() (string, error) {
 	return strings.Split(string(content), "\n")[0], nil
 }
 
+func getArch() (string, error) {
+	out, err := executeCommand("/usr/bin/uname -p")
+	if err != nil {
+		return "", fmt.Errorf("Error executing \"/usr/bin/uname -p\": %v", err)
+	}
+	
+	return strings.Split(out, "\n")[0], nil
+}
+
 func getFreeBSDVersion() (FreeBSDVersion, error) {
 	var version FreeBSDVersion
 	regex := `([0-9]{1,2})(\.)?([0-9]{1,2})?\-([^\-]*)(\-)?(p[0-9]{1,2})?`
@@ -219,6 +228,9 @@ func NewJailHost() (JailHost, error) {
 	if jh.hostname, err = getHostname(); err != nil {
 		return jh, err
 	}
+	if jh.arch, err = getArch(); err != nil {
+		return jh, err
+	}
 	if jh.hostid, err = getHostId(); err != nil {
 		return jh, err
 	}

cmd/root.go

@@ -51,6 +51,10 @@ var (
 	gMigrateDestDatastore string
 	gYesToAll             bool
 
+	gFetchRelease    string
+	gFetchIntoDS     string
+
+
 	rootCmd = &cobra.Command{
 		Use:   "gocage",
 		Short: "GoCage is a FreeBSD Jail management tool",
@@ -285,6 +289,14 @@ You can specify multiple datastores.`,
 		},
 	}
 	
+	fetchCmd = &cobra.Command{
+		Use:   "fetch",
+		Short: "Fetch FreeBSD release to local datastore",
+		Run: func(cmd *cobra.Command, args []string) {
+			fetchRelease(gFetchRelease, "http", gJailHost.arch, gFetchIntoDS)
+	 },
+	}
+	
 	testCmd = &cobra.Command{
 		Use:   "test",
 		Short: "temporary command to test some code snippet",
@@ -304,7 +316,7 @@ func init() {
 	rootCmd.PersistentFlags().StringVarP(&gConfigFile, "config", "c", "/usr/local/etc/gocage.conf.yml", "GoCage configuration file")
 	rootCmd.PersistentFlags().BoolVarP(&gUseSudo, "sudo", "u", false, "Use sudo to run commands")
 	rootCmd.PersistentFlags().StringVarP(&gTimeZone, "timezone", "t", "", "Specify timezone. Will get from /var/db/zoneinfo if not set.")
-	rootCmd.PersistentFlags().BoolVarP(&gDebug, "debug", "d", false, "Debug mode")
+	rootCmd.PersistentFlags().BoolVar(&gDebug, "debug", false, "Debug mode")
 
 	// Command dependant switches
 
@@ -337,6 +349,12 @@ func init() {
 	migrateCmd.Flags().StringVarP(&gMigrateDestDatastore, "datastore", "d", "", "Path of destination datastore for jail (Ex: \"/iocage\")")
 	migrateCmd.Flags().BoolVarP(&gYesToAll, "yes", "y", false, "Answer yes to all questions")
 	migrateCmd.MarkFlagRequired("datastore")
+	
+	fetchCmd.Flags().StringVarP(&gFetchRelease, "release", "r", "", "Release to fetch (e.g.: \"13.1\"")
+	fetchCmd.Flags().StringVarP(&gFetchIntoDS, "datastore", "o", "", "Datastore release will be saved to")
+	fetchCmd.MarkFlagRequired("release")
+	fetchCmd.MarkFlagRequired("datastore")
+	
 
 	// Now declare commands
 	rootCmd.AddCommand(versionCmd)
@@ -352,6 +370,7 @@ func init() {
 	rootCmd.AddCommand(snapshotCmd)
 	rootCmd.AddCommand(migrateCmd)
 	rootCmd.AddCommand(datastoreCmd)
+	rootCmd.AddCommand(fetchCmd)
 	
 	rootCmd.AddCommand(testCmd)
 	

cmd/struct.go

@@ -213,6 +213,7 @@ type FreeBSDVersion struct {
 type JailHost struct {
 	hostname          string
 	hostid            string
+	arch              string
 	default_gateway4  string
 	default_gateway6  string
 	default_interface string

cmd/utils.go

@@ -4,7 +4,7 @@ import (
 	"io"
 	"os"
 	"fmt"
-	"log"
+	//"log"
 	"sort"
 	"bufio"
 	"errors"
@@ -15,6 +15,7 @@ import (
 	"io/ioutil"
 	"github.com/google/shlex"
 	"github.com/c2h5oh/datasize"
+	log "github.com/sirupsen/logrus"
 )
 
 const (
@@ -411,6 +412,31 @@ func zfsCopyIncremental(firstsnap string, secondsnap string, dest string) error
 	
 	return nil
 }
+
+// Return true if dataset exist, false if not, false & error if anything else
+func doZfsDatasetExist(dataset string) (bool, error) {
+	cmd := fmt.Sprintf("zfs list %s", dataset)
+	out, err := executeCommand(cmd)
+	if err != nil {
+		if strings.HasSuffix(strings.TrimSuffix(out, "\n"), "dataset does not exist") {
+			return false, nil
+		} else {
+			return false, errors.New(fmt.Sprintf("%v; command returned \"%s\"", err, out))
+		}
+	}
+	return true, nil
+}
+
+// Create ZFS dataset. mountpoint can be "none", then the dataset won't be mounted
+func createZfsDataset(dataset, mountpoint, compression string) error {
+	cmd := fmt.Sprintf("zfs create -o mountpoint=%s -o compression=%s %s", mountpoint, compression, dataset)
+	out, err := executeCommand(cmd)
+	if err != nil {
+		return errors.New(fmt.Sprintf("%v; command returned \"%s\"", err, out))
+	}
+	return nil
+}
+
 /*****************************************************************************
  *
  * rc.conf management