WIP: fetch command

cmd/fetch.go

@@ -0,0 +1,206 @@
+package cmd
+
+import (
+	"io"
+	"os"
+	"fmt"
+	"bufio"
+	"bytes"
+	//"errors"
+	"strings"
+	"net/http"
+	"encoding/hex"
+	"crypto/sha256"
+
+	log "github.com/sirupsen/logrus"
+)
+
+const (
+	ReleaseServer = "download.freebsd.org"
+	ReleaseRootDir = "ftp/releases"
+)
+
+var (
+	FetchFiles = []string{"base.txz", "lib32.txz", "src.txz"}
+)
+
+// TODO: Check if files already exist
+// Fetch release files, verify, put in datastore under ${datastore}/releases
+// Only support http
+func fetchRelease(release string, proto string, arch string, datastore string) {
+	var ds Datastore
+	
+	log.SetReportCaller(true)
+	
+	if false == strings.EqualFold(proto, "http") {
+		fmt.Printf("Unsupported protocol: %s\n", proto)
+		return
+	}
+	
+	for _, ds = range gDatastores {
+		if strings.EqualFold(datastore, ds.Name) {
+			break
+		}
+	}
+	if false == strings.EqualFold(datastore, ds.Name) {
+		fmt.Printf("Datastore not found: %s\n", datastore)
+		return
+	}
+	
+	// Check datastore have a releases dataset, and it is mounted
+	releaseDsName := fmt.Sprintf("%s/releases", ds.ZFSDataset)
+	releaseDsMountPoint := fmt.Sprintf("%s/releases", ds.Mountpoint)
+	exist, err := doZfsDatasetExist(releaseDsName)
+	if err != nil {
+		fmt.Printf("Error accessing dataset %s: %v\n", releaseDsName, err)
+		return
+	}
+	if false == exist {
+		// Then create dataset
+		if err := createZfsDataset(releaseDsName, releaseDsMountPoint, "lz4"); err != nil {
+			fmt.Printf("Error creating dataset %s: %v\n", releaseDsName, err)
+			return
+		}
+	}
+	
+	// Create XX.X-RELEASE directory if necessary
+	fileDir := fmt.Sprintf("%s/%s-RELEASE", releaseDsMountPoint, release)
+	_, err = os.Stat(fileDir)
+	if os.IsNotExist(err) {
+		if err := os.Mkdir(fileDir, 0755); err != nil {
+			fmt.Printf("Error creating directory %s: %v\n", fileDir, err)
+			return
+		}
+	}
+
+
+	fetchUrl := fmt.Sprintf("%s://%s/%s/%s/%s-RELEASE", proto, ReleaseServer, ReleaseRootDir, arch, release)
+	log.Debugf("FetchURL = %s", fetchUrl)
+	
+	// check if proto/server/arch/release is available
+	if strings.EqualFold(proto, "http") {
+		resp, err := http.Get(fetchUrl)
+		if err != nil {
+			fmt.Printf("Can not get %s: %v\n", fetchUrl, err)
+			return
+		}
+		defer resp.Body.Close()
+		if resp.StatusCode != 200 {
+			fmt.Printf("Get %s returned %d, check release name\n", fetchUrl, resp.StatusCode)
+			return
+		}
+	}
+
+	// Fetch files
+	// Get MANIFEST so we get sha256 sums
+	if err := fetchHTTPFile(fetchUrl, "MANIFEST", fileDir, []byte{}); err != nil {
+		fmt.Printf("%v\n", err)
+		return
+	}
+	// Build an array of "file;checksum"
+	checksumMap, err := buildFileChecksumFromManifest(fmt.Sprintf("%s/MANIFEST", fileDir), FetchFiles)
+	if err != nil {
+		fmt.Printf("%v\n", err)
+		return
+	}
+
+	// Fetch remaining files, verify integrity and write to disk
+	for f, c := range checksumMap {
+		if strings.EqualFold(proto, "http") {
+			if err := fetchHTTPFile(fetchUrl, f, fileDir, c); err != nil {
+				fmt.Printf("%v\n", err)
+				return
+			}
+		}
+	}
+}
+
+func fetchHTTPFile(baseUrl, fileName, storeDir string, checksum []byte) error {
+	// Check storeDir exist
+	_, err := os.Stat(storeDir)
+	if os.IsNotExist(err) {
+		return fmt.Errorf("Directory does not exist: %s\n", storeDir)
+	}
+
+	url := fmt.Sprintf("%s/%s", baseUrl, fileName)
+	fmt.Printf("Fetching %s...", url)
+	resp, err := http.Get(url)
+	if err != nil {
+		fmt.Printf(" Error\n")
+		return fmt.Errorf("Can not get %s: %v\n", url, err)
+	}
+	defer resp.Body.Close()
+	body, err := io.ReadAll(resp.Body)
+	if err != nil {
+		return fmt.Errorf("Can not read %s response body: %v\n", url, err)
+	}
+
+	// Check integrity
+	if len(checksum) > 0 {
+		err = checkIntegrity(body, checksum)
+		if err != nil {
+			return fmt.Errorf("Error checking integrity")
+		}
+	}
+	
+	dest := fmt.Sprintf("%s/%s", storeDir, fileName)
+
+	f, err := os.Create(dest)       // creates a file at current directory
+	if err != nil {
+		return fmt.Errorf("Can not create file %s: %v\n", dest, err)
+	}
+	defer f.Close()
+	f.Write(body)
+	fmt.Printf(" Done.\n")
+	return nil
+}
+
+
+func checkIntegrity(data []byte, checksum []byte) error {
+	sum := sha256.Sum256(data)
+	
+	if false == bytes.Equal(checksum[:],sum[:]) {
+		return fmt.Errorf("Invalid checksum: %x != %x", sum, checksum)
+	}
+	return nil
+}
+
+// Get checksum from manifest, for each file in fileList
+/* MANIFEST format:
+ * base-dbg.txz    a5b51f3d54686509e91ca9c30e9f1cd93dc757f25c643609b3c35e7119c0531d        1654    base_dbg        "Base system (Debugging)"       off
+ * base.txz        e85b256930a2fbc04b80334106afecba0f11e52e32ffa197a88d7319cf059840        26492   base    "Base system (MANDATORY)"       on
+ * kernel-dbg.txz  6b47a6cb83637af1f489aa8cdb802d9db936ea864887188cfc69d8075762214e        912     kernel_dbg      "Kernel (Debugging)"    on
+ */
+func buildFileChecksumFromManifest(manifest string, fileList []string) (map[string][]byte, error) {
+	var ckarr = make(map[string][]byte)
+
+	rm, err := os.Open(manifest)
+	if err != nil {
+		return ckarr, fmt.Errorf("Unable to open MANIFEST: %v", err)
+	}
+	fscan := bufio.NewScanner(rm)
+	fscan.Split(bufio.ScanLines)
+	
+	// For each MANIFEST line...
+	for fscan.Scan() {
+		fields := strings.Fields(fscan.Text())
+		fn := fields[0]
+		fck := fields[1]
+		hexSum, err := hex.DecodeString(fck)
+		if err != nil {
+			return ckarr, fmt.Errorf("Invalid value for checksum %s", fck)
+		}
+		// ... Find the corresponding file in fileList, then add to checksum array ckarr
+		for _, f := range fileList {
+			if strings.EqualFold(f, fn) {
+				ckarr[fn] = hexSum
+				break
+			}
+		}
+	}
+	
+	if len(ckarr) < len(fileList) {
+		return ckarr, fmt.Errorf("Missing file in MANIFEST")
+	}
+	return ckarr, nil
+}