379 lines
11 KiB
Go
379 lines
11 KiB
Go
package cmd
|
|
|
|
import (
|
|
"io"
|
|
"os"
|
|
"fmt"
|
|
"bufio"
|
|
"bytes"
|
|
//"errors"
|
|
"strings"
|
|
"net/http"
|
|
//"archive/tar"
|
|
"encoding/hex"
|
|
"crypto/sha256"
|
|
|
|
//"github.com/ulikunitz/xz"
|
|
log "github.com/sirupsen/logrus"
|
|
)
|
|
|
|
const (
|
|
ReleaseServer = "download.freebsd.org"
|
|
ReleaseRootDir = "ftp/releases"
|
|
)
|
|
|
|
var (
|
|
// TODO : Make this a config/cmd line setting
|
|
//FetchFiles = []string{"base.txz", "lib32.txz", "src.txz"}
|
|
FetchFiles = []string{"base.txz"}
|
|
)
|
|
|
|
// TODO: Check if files already exist
|
|
// Fetch release files, verify, put in datastore under ${datastore}/download
|
|
// Only support http and file protocols
|
|
func fetchRelease(release string, proto string, arch string, datastore string, fetchFrom string) error {
|
|
var ds Datastore
|
|
|
|
log.SetReportCaller(true)
|
|
|
|
if len(fetchFrom) > 0 {
|
|
proto = strings.Split(fetchFrom, ":")[0]
|
|
}
|
|
|
|
if false == strings.EqualFold(proto, "http") &&
|
|
false == strings.EqualFold(proto, "file") {
|
|
return fmt.Errorf("Unsupported protocol: %s\n", proto)
|
|
}
|
|
|
|
for _, ds = range gDatastores {
|
|
if strings.EqualFold(datastore, ds.Name) {
|
|
break
|
|
}
|
|
}
|
|
if false == strings.EqualFold(datastore, ds.Name) {
|
|
return fmt.Errorf("Datastore not found: %s\n", datastore)
|
|
}
|
|
|
|
// Check datastore have a download dataset, and it is mounted
|
|
downloadDsName := fmt.Sprintf("%s/download", ds.ZFSDataset)
|
|
downloadDsMountPoint := fmt.Sprintf("%s/download", ds.Mountpoint)
|
|
exist, err := doZfsDatasetExist(downloadDsName)
|
|
if err != nil {
|
|
return fmt.Errorf("Error accessing dataset %s: %v\n", downloadDsName, err)
|
|
}
|
|
if false == exist {
|
|
// Then create dataset
|
|
if err := zfsCreateDataset(downloadDsName, downloadDsMountPoint, "lz4"); err != nil {
|
|
return fmt.Errorf("Error creating dataset %s: %v\n", downloadDsName, err)
|
|
}
|
|
}
|
|
|
|
// Create download/XX.X dataset if necessary
|
|
thisDownloadDsName := fmt.Sprintf("%s/%s", downloadDsName, release)
|
|
thisDownloadDsMountPoint := fmt.Sprintf("%s/%s", downloadDsMountPoint, release)
|
|
exist, err = doZfsDatasetExist(thisDownloadDsName)
|
|
if err != nil {
|
|
return fmt.Errorf("Error accessing dataset %s: %v\n", thisDownloadDsName, err)
|
|
}
|
|
if false == exist {
|
|
// Then create dataset
|
|
if err := zfsCreateDataset(thisDownloadDsName, thisDownloadDsMountPoint, "lz4"); err != nil {
|
|
return fmt.Errorf("Error creating dataset %s: %v\n", thisDownloadDsName, err)
|
|
}
|
|
}
|
|
|
|
var fetchUrl string
|
|
if len(fetchFrom) > 0 {
|
|
fetchUrl = fmt.Sprintf("%s/%s", fetchFrom, release)
|
|
} else {
|
|
fetchUrl = fmt.Sprintf("%s://%s/%s/%s/%s", proto, ReleaseServer, ReleaseRootDir, arch, release)
|
|
}
|
|
log.Debugf("FetchURL = %s", fetchUrl)
|
|
|
|
// check if proto/server/arch/release is available
|
|
if strings.EqualFold(proto, "http") {
|
|
resp, err := http.Get(fetchUrl)
|
|
if err != nil {
|
|
return fmt.Errorf("Can not get %s: %v\n", fetchUrl, err)
|
|
}
|
|
defer resp.Body.Close()
|
|
if resp.StatusCode != 200 {
|
|
return fmt.Errorf("Get %s returned %d, check release name\n", fetchUrl, resp.StatusCode)
|
|
}
|
|
}
|
|
|
|
// Fetch files
|
|
// Get MANIFEST so we get sha256 sums
|
|
if err := fetchFile(proto, fetchUrl, "MANIFEST", thisDownloadDsMountPoint, []byte{}); err != nil {
|
|
return fmt.Errorf("%v\n", err)
|
|
}
|
|
// Build an array of "file;checksum"
|
|
checksumMap, err := buildFileChecksumFromManifest(fmt.Sprintf("%s/MANIFEST", thisDownloadDsMountPoint), FetchFiles)
|
|
if err != nil {
|
|
return fmt.Errorf("%v\n", err)
|
|
}
|
|
|
|
// Fetch remaining files, verify integrity and write to disk
|
|
for f, c := range checksumMap {
|
|
if err := fetchFile(proto, fetchUrl, f, thisDownloadDsMountPoint, c); err != nil {
|
|
return fmt.Errorf("%v\n", err)
|
|
}
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// Extract release files stored in iocage/download/$RELEASE/ to iocage/releases/$RELEASE/root/
|
|
func extractRelease(release string, datastore string) {
|
|
log.SetReportCaller(true)
|
|
|
|
var ds Datastore
|
|
|
|
for _, ds = range gDatastores {
|
|
if strings.EqualFold(datastore, ds.Name) {
|
|
break
|
|
}
|
|
}
|
|
if false == strings.EqualFold(datastore, ds.Name) {
|
|
fmt.Printf("Datastore not found: %s\n", datastore)
|
|
return
|
|
}
|
|
|
|
// Check datastore have a releases dataset, and it is mounted
|
|
releaseDsName := fmt.Sprintf("%s/releases", ds.ZFSDataset)
|
|
releaseDsMountPoint := fmt.Sprintf("%s/releases", ds.Mountpoint)
|
|
exist, err := doZfsDatasetExist(releaseDsName)
|
|
if err != nil {
|
|
fmt.Printf("Error accessing dataset %s: %v\n", releaseDsName, err)
|
|
return
|
|
}
|
|
if false == exist {
|
|
// Then create dataset
|
|
if err := zfsCreateDataset(releaseDsName, releaseDsMountPoint, "lz4"); err != nil {
|
|
fmt.Printf("Error creating dataset %s: %v\n", releaseDsName, err)
|
|
return
|
|
}
|
|
}
|
|
|
|
// Create releases/XX.X dataset if necessary
|
|
thisReleaseDsName := fmt.Sprintf("%s/%s", releaseDsName, release)
|
|
thisReleaseDsMountPoint := fmt.Sprintf("%s/%s", releaseDsMountPoint, release)
|
|
exist, err = doZfsDatasetExist(thisReleaseDsName)
|
|
if err != nil {
|
|
fmt.Printf("Error accessing dataset %s: %v\n", thisReleaseDsName, err)
|
|
return
|
|
}
|
|
if false == exist {
|
|
// Then create dataset
|
|
if err := zfsCreateDataset(thisReleaseDsName, thisReleaseDsMountPoint, "lz4"); err != nil {
|
|
fmt.Printf("Error creating dataset %s: %v\n", thisReleaseDsName, err)
|
|
return
|
|
}
|
|
}
|
|
|
|
// Create releases/XX.X/root dataset if necessary
|
|
thisReleaseRootDsName := fmt.Sprintf("%s/root", thisReleaseDsName)
|
|
thisReleaseRootDsMountPoint := fmt.Sprintf("%s/root", thisReleaseDsMountPoint)
|
|
exist, err = doZfsDatasetExist(thisReleaseRootDsName)
|
|
if err != nil {
|
|
fmt.Printf("Error accessing dataset %s: %v\n", thisReleaseRootDsName, err)
|
|
return
|
|
}
|
|
if false == exist {
|
|
// Then create dataset
|
|
if err := zfsCreateDataset(thisReleaseRootDsName, thisReleaseRootDsMountPoint, "lz4"); err != nil {
|
|
fmt.Printf("Error creating dataset %s: %v\n", thisReleaseRootDsName, err)
|
|
return
|
|
}
|
|
}
|
|
|
|
// Now extract download/$RELEASE/*.txz to releases/XX.X/root
|
|
downloadDsMountPoint := fmt.Sprintf("%s/download", ds.Mountpoint)
|
|
downloadDir := fmt.Sprintf("%s/%s", downloadDsMountPoint, release)
|
|
|
|
d, err := os.Open(downloadDir)
|
|
defer d.Close()
|
|
if err != nil {
|
|
fmt.Printf("Can not read %s directory: %v\n", downloadDir, err)
|
|
return
|
|
}
|
|
files, err := d.Readdir(0)
|
|
if err != nil {
|
|
fmt.Printf("Can not browse %s directory: %v\n", downloadDir, err)
|
|
return
|
|
}
|
|
|
|
// Extract every .txz files in FetchFiles
|
|
for _, fi := range files {
|
|
if false == fi.IsDir() {
|
|
if strings.HasSuffix(fi.Name(), ".txz") {
|
|
if isStringInArray(FetchFiles, fi.Name()) {
|
|
ar := fmt.Sprintf("%s/%s", downloadDir, fi.Name())
|
|
fmt.Printf("Extracting file %s to %s... ", ar, thisReleaseRootDsMountPoint)
|
|
// pure Go method, sorry this is so slow. Also I did not handle permissions in this
|
|
/* f, err := os.Open(ar)
|
|
defer f.Close()
|
|
if err != nil {
|
|
fmt.Printf("Can not open %s: %v\n", ar, err)
|
|
return
|
|
}
|
|
// xz reader
|
|
r, err := xz.NewReader(f)
|
|
if err != nil {
|
|
fmt.Printf("Can not read %s: %v\n", ar, err)
|
|
return
|
|
}
|
|
// tar reader
|
|
tr := tar.NewReader(r)
|
|
// Iterate through the files in the archive.
|
|
for {
|
|
hdr, err := tr.Next()
|
|
if err == io.EOF {
|
|
// end of tar archive
|
|
break
|
|
}
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
switch hdr.Typeflag {
|
|
case tar.TypeDir:
|
|
// create a directory
|
|
dest := fmt.Sprintf("%s/%s", thisReleaseRootDsMountPoint, hdr.Name)
|
|
// FIXME: Access rights?
|
|
err = os.MkdirAll(dest, 0777)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
case tar.TypeReg, tar.TypeRegA:
|
|
// write a file
|
|
dest := fmt.Sprintf("%s/%s", thisReleaseRootDsMountPoint, hdr.Name)
|
|
w, err := os.Create(dest)
|
|
defer w.Close()
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
_, err = io.Copy(w, tr)
|
|
if err != nil {
|
|
log.Fatal(err)
|
|
}
|
|
}
|
|
}
|
|
*/
|
|
cmd := fmt.Sprintf("/usr/bin/tar xpf %s -C %s", ar, thisReleaseRootDsMountPoint)
|
|
out, err := executeCommand(cmd)
|
|
if err != nil && len(out) > 0 {
|
|
fmt.Printf("Error: %v: %s\n", err, out)
|
|
} else {
|
|
fmt.Printf("Done\n")
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func fetchFile(proto, baseUrl, fileName, storeDir string, checksum []byte) error {
|
|
// Check storeDir exist
|
|
_, err := os.Stat(storeDir)
|
|
if os.IsNotExist(err) {
|
|
return fmt.Errorf("Directory does not exist: %s\n", storeDir)
|
|
}
|
|
|
|
url := fmt.Sprintf("%s/%s", baseUrl, fileName)
|
|
fmt.Printf("Fetching %s...", url)
|
|
|
|
var body []byte
|
|
if strings.EqualFold(proto, "http") {
|
|
resp, err := http.Get(url)
|
|
if err != nil {
|
|
fmt.Printf(" Error\n")
|
|
return fmt.Errorf("Can not get %s: %v\n", url, err)
|
|
}
|
|
defer resp.Body.Close()
|
|
body, err = io.ReadAll(resp.Body)
|
|
if err != nil {
|
|
return fmt.Errorf("Can not read %s response body: %v\n", url, err)
|
|
}
|
|
} else if strings.EqualFold(proto, "file") {
|
|
url = strings.Replace(url, "file:", "", 1)
|
|
f, err := os.Open(url)
|
|
if err != nil {
|
|
return fmt.Errorf("Error accessing file %s", url)
|
|
}
|
|
defer f.Close()
|
|
body, err = io.ReadAll(f)
|
|
if err != nil {
|
|
return fmt.Errorf("Can not read file %s: %v\n", url, err)
|
|
}
|
|
}
|
|
|
|
// Check integrity
|
|
if len(checksum) > 0 {
|
|
err = checkIntegrity(body, checksum)
|
|
if err != nil {
|
|
return fmt.Errorf("Error checking integrity")
|
|
}
|
|
}
|
|
|
|
dest := fmt.Sprintf("%s/%s", storeDir, fileName)
|
|
|
|
f, err := os.Create(dest) // creates a file at current directory
|
|
if err != nil {
|
|
return fmt.Errorf("Can not create file %s: %v\n", dest, err)
|
|
}
|
|
defer f.Close()
|
|
f.Write(body)
|
|
fmt.Printf(" Done\n")
|
|
return nil
|
|
}
|
|
|
|
|
|
func checkIntegrity(data []byte, checksum []byte) error {
|
|
sum := sha256.Sum256(data)
|
|
|
|
if false == bytes.Equal(checksum[:],sum[:]) {
|
|
return fmt.Errorf("Invalid checksum: %x != %x", sum, checksum)
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Get checksum from manifest, for each file in fileList
|
|
/* MANIFEST format:
|
|
* base-dbg.txz a5b51f3d54686509e91ca9c30e9f1cd93dc757f25c643609b3c35e7119c0531d 1654 base_dbg "Base system (Debugging)" off
|
|
* base.txz e85b256930a2fbc04b80334106afecba0f11e52e32ffa197a88d7319cf059840 26492 base "Base system (MANDATORY)" on
|
|
* kernel-dbg.txz 6b47a6cb83637af1f489aa8cdb802d9db936ea864887188cfc69d8075762214e 912 kernel_dbg "Kernel (Debugging)" on
|
|
*/
|
|
func buildFileChecksumFromManifest(manifest string, fileList []string) (map[string][]byte, error) {
|
|
var ckarr = make(map[string][]byte)
|
|
|
|
rm, err := os.Open(manifest)
|
|
if err != nil {
|
|
return ckarr, fmt.Errorf("Unable to open MANIFEST: %v", err)
|
|
}
|
|
fscan := bufio.NewScanner(rm)
|
|
fscan.Split(bufio.ScanLines)
|
|
|
|
// For each MANIFEST line...
|
|
for fscan.Scan() {
|
|
fields := strings.Fields(fscan.Text())
|
|
fn := fields[0]
|
|
fck := fields[1]
|
|
hexSum, err := hex.DecodeString(fck)
|
|
if err != nil {
|
|
return ckarr, fmt.Errorf("Invalid value for checksum %s", fck)
|
|
}
|
|
// ... Find the corresponding file in fileList, then add to checksum array ckarr
|
|
for _, f := range fileList {
|
|
if strings.EqualFold(f, fn) {
|
|
ckarr[fn] = hexSum
|
|
break
|
|
}
|
|
}
|
|
}
|
|
|
|
if len(ckarr) < len(fileList) {
|
|
return ckarr, fmt.Errorf("Missing file in MANIFEST")
|
|
}
|
|
return ckarr, nil
|
|
}
|