diff --git a/build/linux/x86_64/Linux-x86_64-lpmx b/build/linux/x86_64/Linux-x86_64-lpmx index 0835ec0..861a2f0 100755 Binary files a/build/linux/x86_64/Linux-x86_64-lpmx and b/build/linux/x86_64/Linux-x86_64-lpmx differ diff --git a/container/container.go b/container/container.go index fc4f862..d0da883 100644 --- a/container/container.go +++ b/container/container.go @@ -1751,6 +1751,157 @@ func SingularityLoad(file string, name string, tag string) *Error { } } +//name -> name with tag info +func SkopeoLoad(name, dir string) *Error { + currdir, err := GetConfigDir() + if err != nil { + return err + } + rootdir := fmt.Sprintf("%s/.lpmxdata", currdir) + sysdir := fmt.Sprintf("%s/.lpmxsys", currdir) + tempdir := fmt.Sprintf("%s/.temp", currdir) + //we delete temp dir if it exists at the end of the function + defer func() { + if FolderExist(tempdir) { + os.RemoveAll(tempdir) + } + }() + + //check if folder exists + adir, derr := filepath.Abs(dir) + if derr!= nil { + cerr := ErrNew(derr, fmt.Sprintf("could not parse the absolute path: %s", adir)) + return cerr + } + if !FolderExist(adir) { + cerr := ErrNew(ErrNExist, fmt.Sprintf("%s does exist", adir)) + return cerr + } + + //configure Docker related info locally + var doc Image + err = unmarshalObj(rootdir, &doc) + if err != nil && err.Err != ErrNExist { + return err + } + + if err != nil && err.Err == ErrNExist { + ret, err := MakeDir(rootdir) + doc.RootDir = rootdir + doc.Images = make(map[string]interface{}) + if !ret { + return err + } + } + + image_dir := fmt.Sprintf("%s/.image", rootdir) + ret, layer_order, lerr := LoadSkopeoTar(adir, image_dir) + if lerr != nil { + return lerr + } + if _, ok := doc.Images[name]; ok { + cerr := ErrNew(ErrExist, fmt.Sprintf("%s already exists", name)) + return cerr + } else { + tdata := strings.Split(name, ":") + tname := tdata[0] + ttag := tdata[1] + mdata := make(map[string]interface{}) + mdata["rootdir"] = fmt.Sprintf("%s/%s/%s", doc.RootDir, tname, ttag) + mdata["config"] = fmt.Sprintf("%s/setting.yml", mdata["rootdir"].(string)) + mdata["image"] = fmt.Sprintf("%s/.image", rootdir) + mdata["layer"] = ret + mdata["layer_order"] = strings.Join(layer_order, ":") + mdata["imagetype"] = "Docker" + + //add docker info file(.info) + if !FolderExist(mdata["rootdir"].(string)) { + merr := os.MkdirAll(mdata["rootdir"].(string), os.FileMode(FOLDER_MODE)) + if merr != nil { + cerr := ErrNew(merr, fmt.Sprintf("could not mkdir %s", mdata["rootdir"].(string))) + return cerr + } + } + var docinfo ImageInfo + docinfo.Name = name + docinfo.ImageType = "Docker" + // layer_order is absolute path + //docinfo layers map should remove absolute path of host + layersmap := make(map[string]int64) + for k, v := range ret { + layersmap[path.Base(k)] = v + } + docinfo.LayersMap = layersmap + + var layer_sha []string + for _, layer := range layer_order { + layer_sha = append(layer_sha, path.Base(layer)) + } + docinfo.Layers = strings.Join(layer_sha, ":") + + LOGGER.WithFields(logrus.Fields{ + "docinfo": docinfo, + }).Debug("Skopeo debug, docinfo debug") + + dinfodata, _ := StructMarshal(docinfo) + err = WriteToFile(dinfodata, fmt.Sprintf("%s/.info", mdata["rootdir"].(string))) + if err != nil { + return err + } + //end + + workspace := fmt.Sprintf("%s/workspace", mdata["rootdir"]) + if !FolderExist(workspace) { + MakeDir(workspace) + } + mdata["workspace"] = workspace + + //extract layers + base := fmt.Sprintf("%s/.base", rootdir) + if !FolderExist(base) { + MakeDir(base) + } + mdata["base"] = base + + for _, k := range layer_order { + k = path.Base(k) + tar_path := fmt.Sprintf("%s/%s", image_dir, k) + layerfolder := fmt.Sprintf("%s/%s", mdata["base"], k) + if !FolderExist(layerfolder) { + MakeDir(layerfolder) + } + + err := Untar(tar_path, layerfolder) + if err != nil { + return err + } + } + + //download setting from github + rdir, _ := mdata["rootdir"].(string) + + yaml := fmt.Sprintf("%s/distro.management.yml", sysdir) + err = DownloadFilefromGithubPlus(tname, ttag, "setting.yml", SETTING_URL, rdir, yaml) + if err != nil { + LOGGER.WithFields(logrus.Fields{ + "err": err, + "toPath": rdir, + }).Error("Download setting from github failure and could not rollback to default one") + return err + } + + //add map to this image + doc.Images[name] = mdata + + ddata, _ := StructMarshal(doc) + err = WriteToFile(ddata, fmt.Sprintf("%s/.info", doc.RootDir)) + if err != nil { + return err + } + } + return nil +} + func DockerLoad(file string) *Error { currdir, err := GetConfigDir() if err != nil { @@ -1767,8 +1918,13 @@ func DockerLoad(file string) *Error { }() //check if file exists - if !FileExist(file) { - cerr := ErrNew(ErrNExist, fmt.Sprintf("%s does exist", file)) + afile, ferr := filepath.Abs(file) + if ferr != nil { + cerr := ErrNew(ferr, fmt.Sprintf("could not parse to the absolute path: %s", file)) + return cerr + } + if !FileExist(afile) { + cerr := ErrNew(ErrNExist, fmt.Sprintf("%s does exist", afile)) return cerr } @@ -1795,7 +1951,7 @@ func DockerLoad(file string) *Error { } //untar tar ball - uerr := Untar(file, tmpdir) + uerr := Untar(afile, tmpdir) if uerr != nil { return uerr } diff --git a/docker/docker.go b/docker/docker.go index bcc70f6..0696f68 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -38,6 +38,20 @@ type DockerSaveInfo struct { Layers []string //layers included inside this image, from lower to higher layers } +//Skopeo manifest item structure +type SkopeoManifestItem struct { + MediaType string `json:"mediaType"` + Size uint `json:"size"` + Digest string `json:"digest"` +} +//Skopeo manifest structure +type SkopeoManifest struct { + SchemaVersion int `json:"schemaVersion"` + MediaType string `json:"mediaType"` + Config SkopeoManifestItem `json:"config"` + Layers []SkopeoManifestItem `json:"layers"` +} + func ListRepositories(username string, pass string) ([]string, *Error) { log.SetOutput(ioutil.Discard) hub, err := registry.New(DOCKER_URL, username, pass) @@ -342,6 +356,47 @@ func DeleteManifest(username string, pass string, name string, tag string) *Erro return nil } +func LoadSkopeoTar(dir, imagedir string) (map[string]int64, []string, *Error) { + if !FolderExist(dir) { + cerr := ErrNew(ErrNExist, fmt.Sprintf("%s does not exist", dir)) + return nil, nil, cerr + } + + var info SkopeoManifest + manifest_file := fmt.Sprintf("%s/manifest.json", dir) + b, berr := ioutil.ReadFile(manifest_file) + if berr != nil { + cerr := ErrNew(berr, fmt.Sprintf("could not read file: %s", manifest_file)) + return nil, nil, cerr + } + + jerr := json.Unmarshal(b, &info) + if jerr != nil { + cerr := ErrNew(jerr, "could not unmarshal json bytes to objects") + return nil, nil, cerr + } + + layer_data := make(map[string]int64) + var layers []string + for _, item := range info.Layers { + shavalue := strings.Split(item.Digest, ":")[1] + layer_path := fmt.Sprintf("%s/%s", dir, shavalue) + target_path := fmt.Sprintf("%s/%s.tar.gz", imagedir, shavalue) + _, rerr := CopyFile(layer_path, target_path) + if rerr != nil { + return nil, nil, rerr + } + file_length, ferr := GetFileLength(target_path) + if ferr != nil { + return nil, nil, ferr + } + layer_data[target_path] = file_length + layers = append(layers, target_path) + } + + return layer_data, layers, nil +} + //dir is temp dir used for tarball extraction, image dir is used for the storage of image data func LoadDockerTar(dir, imagedir string) (string, map[string]int64, []string, *Error) { if !FolderExist(dir) { diff --git a/main.go b/main.go index 4575303..dbf4684 100644 --- a/main.go +++ b/main.go @@ -20,7 +20,7 @@ var ( ) const ( - VERSION = "alpha-1.7.2" + VERSION = "alpha-1.8" ) func checkCompleteness() *Error { @@ -672,12 +672,39 @@ func main() { }, } + var SkopeoNameTag string + var skopeoLoadCmd = &cobra.Command{ + Use: "skopeoload", + Short: "load the skopeo directory", + Long: "skopeoload sub-command is one advanced command of lpmx, which is used for importing 'skopeo copy' generated directory to system", + Args: cobra.ExactArgs(1), + PreRun: func(cmd *cobra.Command, args []string) { + err := checkCompleteness() + if err != nil { + LOGGER.Fatal(err.Error()) + return + } + }, + Run: func(cmd *cobra.Command, args []string) { + err := SkopeoLoad(SkopeoNameTag, args[0]) + if err != nil { + LOGGER.Fatal(err.Error()) + return + } else { + LOGGER.Info("DONE") + return + } + }, + } + skopeoLoadCmd.Flags().StringVarP(&SkopeoNameTag, "nametag", "n", "", "required") + skopeoLoadCmd.MarkFlagRequired("nametag") + var dockerCmd = &cobra.Command{ Use: "docker", Short: "docker command", Long: "docker command is the advanced command of lpmx, which is used for executing docker related commands", } - dockerCmd.AddCommand(dockerCreateCmd, dockerSearchCmd, dockerListCmd, dockerDeleteCmd, dockerDownloadCmd, dockerResetCmd, dockerPackageCmd, dockerAddCmd, dockerCommitCmd, dockerLoadCmd, dockerRunCmd, dockerMergeCmd) + dockerCmd.AddCommand(dockerCreateCmd, dockerSearchCmd, dockerListCmd, dockerDeleteCmd, dockerDownloadCmd, dockerResetCmd, dockerPackageCmd, dockerAddCmd, dockerCommitCmd, dockerLoadCmd, dockerRunCmd, dockerMergeCmd, skopeoLoadCmd) var SingularityLoadName string var SingularityLoadTag string diff --git a/utils/utils.go b/utils/utils.go index 7db9f9f..0594b72 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -288,18 +288,8 @@ func RemoveFile(path string) (bool, *Error) { func Rename(old_path string, new_path string) *Error { if t, terr := FileType(old_path); terr == nil { - if t == TYPE_DIR { - parent_dir := filepath.Dir(new_path) - if !FolderExist(parent_dir) { - err := os.MkdirAll(parent_dir, 0777) - if err != nil { - cerr := ErrNew(err, fmt.Sprintf("could not mkdir %s", parent_dir)) - return cerr - } - } - } - if t == TYPE_REGULAR { - parent_dir := filepath.Dir(new_path) + parent_dir := filepath.Dir(new_path) + if t == TYPE_DIR || t == TYPE_REGULAR { if !FolderExist(parent_dir) { err := os.MkdirAll(parent_dir, 0777) if err != nil { @@ -411,6 +401,14 @@ func CopyFile(src string, dst string) (bool, *Error) { return false, cerr } defer in.Close() + parent_dir := filepath.Dir(dst) + if !FolderExist(parent_dir) { + err := os.MkdirAll(parent_dir, 0777) + if err != nil { + cerr := ErrNew(err, fmt.Sprintf("could not mkdir %s", parent_dir)) + return false, cerr + } + } out, oerr := os.Create(dst) if oerr != nil { cerr := ErrNew(oerr, fmt.Sprintf("can't open file %s", dst))