diff --git a/README.md b/README.md index 8ba34e9..4761385 100644 --- a/README.md +++ b/README.md @@ -189,10 +189,20 @@ Migrate jail filesystem dataset to fastdata/iocage/jails/srv-random/root: Done +Fetch +---------- +Files can be fetched from custom repository, or from local directory with "from" option. +For example if you destroyed releases/12.3-RELEASE and still have the downloaded files in /iocage/download/12.3-RELEASE: +
+gocage fetch -r 12.3 -o iocage --from file:/iocage/download
+
+
+
TODO
----------
gocage update
gocage upgrade
gocage create
gocage destroy
-
+gocage init
+ create default pool with defaults.json
diff --git a/cmd/fetch.go b/cmd/fetch.go
index ccc519c..8a71d9b 100644
--- a/cmd/fetch.go
+++ b/cmd/fetch.go
@@ -9,9 +9,11 @@ import (
//"errors"
"strings"
"net/http"
+ //"archive/tar"
"encoding/hex"
"crypto/sha256"
+ //"github.com/ulikunitz/xz"
log "github.com/sirupsen/logrus"
)
@@ -25,18 +27,106 @@ var (
)
// TODO: Check if files already exist
-// Fetch release files, verify, put in datastore under ${datastore}/download, then extract into ${datastore}/releases
-// Only support http
-func fetchRelease(release string, proto string, arch string, datastore string) {
+// Fetch release files, verify, put in datastore under ${datastore}/download
+// Only support http and file protocols
+func fetchRelease(release string, proto string, arch string, datastore string, fetchFrom string) error {
var ds Datastore
log.SetReportCaller(true)
- if false == strings.EqualFold(proto, "http") {
- fmt.Printf("Unsupported protocol: %s\n", proto)
- return
+ if len(fetchFrom) > 0 {
+ proto = strings.Split(fetchFrom, ":")[0]
}
+ if false == strings.EqualFold(proto, "http") &&
+ false == strings.EqualFold(proto, "file") {
+ return fmt.Errorf("Unsupported protocol: %s\n", proto)
+ }
+
+ for _, ds = range gDatastores {
+ if strings.EqualFold(datastore, ds.Name) {
+ break
+ }
+ }
+ if false == strings.EqualFold(datastore, ds.Name) {
+ return fmt.Errorf("Datastore not found: %s\n", datastore)
+ }
+
+ // Check datastore have a download dataset, and it is mounted
+ downloadDsName := fmt.Sprintf("%s/download", ds.ZFSDataset)
+ downloadDsMountPoint := fmt.Sprintf("%s/download", ds.Mountpoint)
+ exist, err := doZfsDatasetExist(downloadDsName)
+ if err != nil {
+ return fmt.Errorf("Error accessing dataset %s: %v\n", downloadDsName, err)
+ }
+ if false == exist {
+ // Then create dataset
+ if err := createZfsDataset(downloadDsName, downloadDsMountPoint, "lz4"); err != nil {
+ return fmt.Errorf("Error creating dataset %s: %v\n", downloadDsName, err)
+ }
+ }
+
+ // Create download/XX.X-RELEASE dataset if necessary
+ thisDownloadDsName := fmt.Sprintf("%s/%s-RELEASE", downloadDsName, release)
+ thisDownloadDsMountPoint := fmt.Sprintf("%s/%s-RELEASE", downloadDsMountPoint, release)
+ exist, err = doZfsDatasetExist(thisDownloadDsName)
+ if err != nil {
+ return fmt.Errorf("Error accessing dataset %s: %v\n", thisDownloadDsName, err)
+ }
+ if false == exist {
+ // Then create dataset
+ if err := createZfsDataset(thisDownloadDsName, thisDownloadDsMountPoint, "lz4"); err != nil {
+ return fmt.Errorf("Error creating dataset %s: %v\n", thisDownloadDsName, err)
+ }
+ }
+
+ var fetchUrl string
+ if len(fetchFrom) > 0 {
+ fetchUrl = fmt.Sprintf("%s/%s-RELEASE", fetchFrom, release)
+ } else {
+ fetchUrl = fmt.Sprintf("%s://%s/%s/%s/%s-RELEASE", proto, ReleaseServer, ReleaseRootDir, arch, release)
+ }
+ log.Debugf("FetchURL = %s", fetchUrl)
+
+ // check if proto/server/arch/release is available
+ if strings.EqualFold(proto, "http") {
+ resp, err := http.Get(fetchUrl)
+ if err != nil {
+ return fmt.Errorf("Can not get %s: %v\n", fetchUrl, err)
+ }
+ defer resp.Body.Close()
+ if resp.StatusCode != 200 {
+ return fmt.Errorf("Get %s returned %d, check release name\n", fetchUrl, resp.StatusCode)
+ }
+ }
+
+ // Fetch files
+ // Get MANIFEST so we get sha256 sums
+ if err := fetchFile(proto, fetchUrl, "MANIFEST", thisDownloadDsMountPoint, []byte{}); err != nil {
+ return fmt.Errorf("%v\n", err)
+ }
+ // Build an array of "file;checksum"
+ checksumMap, err := buildFileChecksumFromManifest(fmt.Sprintf("%s/MANIFEST", thisDownloadDsMountPoint), FetchFiles)
+ if err != nil {
+ return fmt.Errorf("%v\n", err)
+ }
+
+ // Fetch remaining files, verify integrity and write to disk
+ for f, c := range checksumMap {
+ if err := fetchFile(proto, fetchUrl, f, thisDownloadDsMountPoint, c); err != nil {
+ return fmt.Errorf("%v\n", err)
+ }
+ }
+
+ return nil
+}
+
+// Extract release files stored in iocage/download/$RELEASE/ to iocage/releases/$RELEASE/root/
+func extractRelease(release string, datastore string) {
+ log.SetReportCaller(true)
+
+ var ds Datastore
+
for _, ds = range gDatastores {
if strings.EqualFold(datastore, ds.Name) {
break
@@ -47,77 +137,138 @@ func fetchRelease(release string, proto string, arch string, datastore string) {
return
}
- // Check datastore have a download dataset, and it is mounted
- downloadDsName := fmt.Sprintf("%s/download", ds.ZFSDataset)
- downloadDsMountPoint := fmt.Sprintf("%s/download", ds.Mountpoint)
- exist, err := doZfsDatasetExist(downloadDsName)
+ // Check datastore have a releases dataset, and it is mounted
+ releaseDsName := fmt.Sprintf("%s/releases", ds.ZFSDataset)
+ releaseDsMountPoint := fmt.Sprintf("%s/releases", ds.Mountpoint)
+ exist, err := doZfsDatasetExist(releaseDsName)
if err != nil {
- fmt.Printf("Error accessing dataset %s: %v\n", downloadDsName, err)
+ fmt.Printf("Error accessing dataset %s: %v\n", releaseDsName, err)
return
}
if false == exist {
// Then create dataset
- if err := createZfsDataset(downloadDsName, downloadDsMountPoint, "lz4"); err != nil {
- fmt.Printf("Error creating dataset %s: %v\n", downloadDsName, err)
+ if err := createZfsDataset(releaseDsName, releaseDsMountPoint, "lz4"); err != nil {
+ fmt.Printf("Error creating dataset %s: %v\n", releaseDsName, err)
return
}
}
- // Create XX.X-RELEASE directory if necessary
- fileDir := fmt.Sprintf("%s/%s-RELEASE", downloadDsMountPoint, release)
- _, err = os.Stat(fileDir)
- if os.IsNotExist(err) {
- if err := os.Mkdir(fileDir, 0755); err != nil {
- fmt.Printf("Error creating directory %s: %v\n", fileDir, err)
- return
- }
- }
-
-
- fetchUrl := fmt.Sprintf("%s://%s/%s/%s/%s-RELEASE", proto, ReleaseServer, ReleaseRootDir, arch, release)
- log.Debugf("FetchURL = %s", fetchUrl)
-
- // check if proto/server/arch/release is available
- if strings.EqualFold(proto, "http") {
- resp, err := http.Get(fetchUrl)
- if err != nil {
- fmt.Printf("Can not get %s: %v\n", fetchUrl, err)
- return
- }
- defer resp.Body.Close()
- if resp.StatusCode != 200 {
- fmt.Printf("Get %s returned %d, check release name\n", fetchUrl, resp.StatusCode)
- return
- }
- }
-
- // Fetch files
- // Get MANIFEST so we get sha256 sums
- if err := fetchHTTPFile(fetchUrl, "MANIFEST", fileDir, []byte{}); err != nil {
- fmt.Printf("%v\n", err)
- return
- }
- // Build an array of "file;checksum"
- checksumMap, err := buildFileChecksumFromManifest(fmt.Sprintf("%s/MANIFEST", fileDir), FetchFiles)
+ // Create releases/XX.X-RELEASE dataset if necessary
+ thisReleaseDsName := fmt.Sprintf("%s/%s-RELEASE", releaseDsName, release)
+ thisReleaseDsMountPoint := fmt.Sprintf("%s/%s-RELEASE", releaseDsMountPoint, release)
+ exist, err = doZfsDatasetExist(thisReleaseDsName)
if err != nil {
- fmt.Printf("%v\n", err)
+ fmt.Printf("Error accessing dataset %s: %v\n", thisReleaseDsName, err)
return
}
-
- // Fetch remaining files, verify integrity and write to disk
- for f, c := range checksumMap {
- if strings.EqualFold(proto, "http") {
- if err := fetchHTTPFile(fetchUrl, f, fileDir, c); err != nil {
- fmt.Printf("%v\n", err)
- return
+ if false == exist {
+ // Then create dataset
+ if err := createZfsDataset(thisReleaseDsName, thisReleaseDsMountPoint, "lz4"); err != nil {
+ fmt.Printf("Error creating dataset %s: %v\n", thisReleaseDsName, err)
+ return
+ }
+ }
+
+ // Create releases/XX.X-RELEASE/root dataset if necessary
+ thisReleaseRootDsName := fmt.Sprintf("%s/root", thisReleaseDsName)
+ thisReleaseRootDsMountPoint := fmt.Sprintf("%s/root", thisReleaseDsMountPoint)
+ exist, err = doZfsDatasetExist(thisReleaseRootDsName)
+ if err != nil {
+ fmt.Printf("Error accessing dataset %s: %v\n", thisReleaseRootDsName, err)
+ return
+ }
+ if false == exist {
+ // Then create dataset
+ if err := createZfsDataset(thisReleaseRootDsName, thisReleaseRootDsMountPoint, "lz4"); err != nil {
+ fmt.Printf("Error creating dataset %s: %v\n", thisReleaseRootDsName, err)
+ return
+ }
+ }
+
+ // Now extract download/$RELEASE/*.txz to releases/XX.X-RELEASE/root
+ downloadDsMountPoint := fmt.Sprintf("%s/download", ds.Mountpoint)
+ downloadDir := fmt.Sprintf("%s/%s-RELEASE", downloadDsMountPoint, release)
+
+ d, err := os.Open(downloadDir)
+ defer d.Close()
+ if err != nil {
+ fmt.Printf("Can not read %s directory: %v\n", downloadDir, err)
+ return
+ }
+ files, err := d.Readdir(0)
+ if err != nil {
+ fmt.Printf("Can not browse %s directory: %v\n", downloadDir, err)
+ return
+ }
+
+ // Extract every .txz files
+ for _, fi := range files {
+ if false == fi.IsDir() {
+ if strings.HasSuffix(fi.Name(), ".txz") {
+ ar := fmt.Sprintf("%s/%s", downloadDir, fi.Name())
+ fmt.Printf("Extracting file %s... ", ar)
+ // pure Go method, sorry this is so slow. Also I did not handle permissions in this
+/* f, err := os.Open(ar)
+ defer f.Close()
+ if err != nil {
+ fmt.Printf("Can not open %s: %v\n", ar, err)
+ return
+ }
+ // xz reader
+ r, err := xz.NewReader(f)
+ if err != nil {
+ fmt.Printf("Can not read %s: %v\n", ar, err)
+ return
+ }
+ // tar reader
+ tr := tar.NewReader(r)
+ // Iterate through the files in the archive.
+ for {
+ hdr, err := tr.Next()
+ if err == io.EOF {
+ // end of tar archive
+ break
+ }
+ if err != nil {
+ log.Fatal(err)
+ }
+ switch hdr.Typeflag {
+ case tar.TypeDir:
+ // create a directory
+ dest := fmt.Sprintf("%s/%s", thisReleaseRootDsMountPoint, hdr.Name)
+ // FIXME: Access rights?
+ err = os.MkdirAll(dest, 0777)
+ if err != nil {
+ log.Fatal(err)
+ }
+ case tar.TypeReg, tar.TypeRegA:
+ // write a file
+ dest := fmt.Sprintf("%s/%s", thisReleaseRootDsMountPoint, hdr.Name)
+ w, err := os.Create(dest)
+ defer w.Close()
+ if err != nil {
+ log.Fatal(err)
+ }
+ _, err = io.Copy(w, tr)
+ if err != nil {
+ log.Fatal(err)
+ }
+ }
+ }
+*/
+ cmd := fmt.Sprintf("/usr/bin/tar xpf %s -C %s", ar, thisReleaseRootDsMountPoint)
+ out, err := executeCommand(cmd)
+ if err != nil && len(out) > 0 {
+ fmt.Printf("Error: %v: %s\n", err, out)
+ } else {
+ fmt.Printf("Done\n")
+ }
}
}
}
-
- // TODO: Extract download/$RELEASE/files to releases/$RELEASE/
}
-func fetchHTTPFile(baseUrl, fileName, storeDir string, checksum []byte) error {
+func fetchFile(proto, baseUrl, fileName, storeDir string, checksum []byte) error {
// Check storeDir exist
_, err := os.Stat(storeDir)
if os.IsNotExist(err) {
@@ -126,17 +277,32 @@ func fetchHTTPFile(baseUrl, fileName, storeDir string, checksum []byte) error {
url := fmt.Sprintf("%s/%s", baseUrl, fileName)
fmt.Printf("Fetching %s...", url)
- resp, err := http.Get(url)
- if err != nil {
- fmt.Printf(" Error\n")
- return fmt.Errorf("Can not get %s: %v\n", url, err)
+
+ var body []byte
+ if strings.EqualFold(proto, "http") {
+ resp, err := http.Get(url)
+ if err != nil {
+ fmt.Printf(" Error\n")
+ return fmt.Errorf("Can not get %s: %v\n", url, err)
+ }
+ defer resp.Body.Close()
+ body, err = io.ReadAll(resp.Body)
+ if err != nil {
+ return fmt.Errorf("Can not read %s response body: %v\n", url, err)
+ }
+ } else if strings.EqualFold(proto, "file") {
+ url = strings.Replace(url, "file:", "", 1)
+ f, err := os.Open(url)
+ if err != nil {
+ return fmt.Errorf("Error accessing file %s", url)
+ }
+ defer f.Close()
+ body, err = io.ReadAll(f)
+ if err != nil {
+ return fmt.Errorf("Can not read file %s: %v\n", url, err)
+ }
}
- defer resp.Body.Close()
- body, err := io.ReadAll(resp.Body)
- if err != nil {
- return fmt.Errorf("Can not read %s response body: %v\n", url, err)
- }
-
+
// Check integrity
if len(checksum) > 0 {
err = checkIntegrity(body, checksum)
@@ -153,7 +319,7 @@ func fetchHTTPFile(baseUrl, fileName, storeDir string, checksum []byte) error {
}
defer f.Close()
f.Write(body)
- fmt.Printf(" Done.\n")
+ fmt.Printf(" Done\n")
return nil
}
diff --git a/cmd/root.go b/cmd/root.go
index 7b72bf8..a117a82 100644
--- a/cmd/root.go
+++ b/cmd/root.go
@@ -14,7 +14,7 @@ import (
)
const (
- gVersion = "0.31"
+ gVersion = "0.32a"
// TODO : Get from $jail_zpool/defaults.json
MIN_DYN_DEVFS_RULESET = 1000
@@ -53,6 +53,7 @@ var (
gFetchRelease string
gFetchIntoDS string
+ gFetchFrom string
rootCmd = &cobra.Command{
@@ -293,7 +294,12 @@ You can specify multiple datastores.`,
Use: "fetch",
Short: "Fetch FreeBSD release to local datastore",
Run: func(cmd *cobra.Command, args []string) {
- fetchRelease(gFetchRelease, "http", gJailHost.arch, gFetchIntoDS)
+ err := fetchRelease(gFetchRelease, "http", gJailHost.arch, gFetchIntoDS, gFetchFrom)
+ if err != nil {
+ fmt.Printf("%v\n", err)
+ } else {
+ extractRelease(gFetchRelease, gFetchIntoDS)
+ }
},
}
@@ -352,6 +358,7 @@ func init() {
fetchCmd.Flags().StringVarP(&gFetchRelease, "release", "r", "", "Release to fetch (e.g.: \"13.1\"")
fetchCmd.Flags().StringVarP(&gFetchIntoDS, "datastore", "o", "", "Datastore release will be saved to")
+ fetchCmd.Flags().StringVarP(&gFetchFrom, "from", "d", "", "Repository to download from. Should contain XY.Z-RELEASE. File protocol supported")
fetchCmd.MarkFlagRequired("release")
fetchCmd.MarkFlagRequired("datastore")