The following pull request was submitted through Github. It can be accessed and reviewed at: https://github.com/lxc/lxd/pull/6148
This e-mail was sent by the LXC bot, direct replies will not reach the author unless they happen to be subscribed to this list. === Description (from pull-request) === This brings the `simplestreams` package in line with some of our other packages, makes it `golint` clean, updates some of the structs to match the server side data and also makes it so that all images that are suitable to LXD are now returned.
From 8e40f883b4d3c18f437fb287b6d958280df0b42b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgra...@ubuntu.com> Date: Tue, 3 Sep 2019 15:50:51 -0400 Subject: [PATCH 01/11] shared/simplestreams: Split out sortedImages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber <stgra...@ubuntu.com> --- shared/simplestreams/simplestreams.go | 52 +------------------------ shared/simplestreams/sort.go | 56 +++++++++++++++++++++++++++ 2 files changed, 57 insertions(+), 51 deletions(-) create mode 100644 shared/simplestreams/sort.go diff --git a/shared/simplestreams/simplestreams.go b/shared/simplestreams/simplestreams.go index 030da803c3..1d84701667 100644 --- a/shared/simplestreams/simplestreams.go +++ b/shared/simplestreams/simplestreams.go @@ -20,56 +20,6 @@ import ( "github.com/lxc/lxd/shared/osarch" ) -type ssSortImage []api.Image - -func (a ssSortImage) Len() int { - return len(a) -} - -func (a ssSortImage) Swap(i, j int) { - a[i], a[j] = a[j], a[i] -} - -func (a ssSortImage) Less(i, j int) bool { - if a[i].Properties["os"] == a[j].Properties["os"] { - if a[i].Properties["release"] == a[j].Properties["release"] { - if !shared.TimeIsSet(a[i].CreatedAt) { - return true - } - - if !shared.TimeIsSet(a[j].CreatedAt) { - return false - } - - if a[i].CreatedAt == a[j].CreatedAt { - return a[i].Properties["serial"] > a[j].Properties["serial"] - } - - return a[i].CreatedAt.UTC().Unix() > a[j].CreatedAt.UTC().Unix() - } - - if a[i].Properties["release"] == "" { - return false - } - - if a[j].Properties["release"] == "" { - return true - } - - return a[i].Properties["release"] < a[j].Properties["release"] - } - - if a[i].Properties["os"] == "" { - return false - } - - if a[j].Properties["os"] == "" { - return true - } - - return a[i].Properties["os"] < a[j].Properties["os"] -} - var ssDefaultOS = map[string]string{ "https://cloud-images.ubuntu.com": "ubuntu", } @@ -440,7 +390,7 @@ func (s *SimpleStreams) parseManifest(path string) (*SimpleStreamsManifest, erro func (s *SimpleStreams) applyAliases(images []api.Image) ([]api.Image, map[string]*api.ImageAliasesEntry, error) { aliases := map[string]*api.ImageAliasesEntry{} - sort.Sort(ssSortImage(images)) + sort.Sort(sortedImages(images)) defaultOS := "" for k, v := range ssDefaultOS { diff --git a/shared/simplestreams/sort.go b/shared/simplestreams/sort.go new file mode 100644 index 0000000000..bb62abb422 --- /dev/null +++ b/shared/simplestreams/sort.go @@ -0,0 +1,56 @@ +package simplestreams + +import ( + "github.com/lxc/lxd/shared" + "github.com/lxc/lxd/shared/api" +) + +type sortedImages []api.Image + +func (a sortedImages) Len() int { + return len(a) +} + +func (a sortedImages) Swap(i, j int) { + a[i], a[j] = a[j], a[i] +} + +func (a sortedImages) Less(i, j int) bool { + if a[i].Properties["os"] == a[j].Properties["os"] { + if a[i].Properties["release"] == a[j].Properties["release"] { + if !shared.TimeIsSet(a[i].CreatedAt) { + return true + } + + if !shared.TimeIsSet(a[j].CreatedAt) { + return false + } + + if a[i].CreatedAt == a[j].CreatedAt { + return a[i].Properties["serial"] > a[j].Properties["serial"] + } + + return a[i].CreatedAt.UTC().Unix() > a[j].CreatedAt.UTC().Unix() + } + + if a[i].Properties["release"] == "" { + return false + } + + if a[j].Properties["release"] == "" { + return true + } + + return a[i].Properties["release"] < a[j].Properties["release"] + } + + if a[i].Properties["os"] == "" { + return false + } + + if a[j].Properties["os"] == "" { + return true + } + + return a[i].Properties["os"] < a[j].Properties["os"] +} From 9ce8681b2f9565eb1417d4d65ba6cfa617efab22 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgra...@ubuntu.com> Date: Tue, 3 Sep 2019 15:52:05 -0400 Subject: [PATCH 02/11] shared/simplestreams: Rename ssDefaultOS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber <stgra...@ubuntu.com> --- shared/simplestreams/simplestreams.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/shared/simplestreams/simplestreams.go b/shared/simplestreams/simplestreams.go index 1d84701667..f629a33093 100644 --- a/shared/simplestreams/simplestreams.go +++ b/shared/simplestreams/simplestreams.go @@ -20,7 +20,7 @@ import ( "github.com/lxc/lxd/shared/osarch" ) -var ssDefaultOS = map[string]string{ +var urlDefaultOS = map[string]string{ "https://cloud-images.ubuntu.com": "ubuntu", } @@ -393,7 +393,7 @@ func (s *SimpleStreams) applyAliases(images []api.Image) ([]api.Image, map[strin sort.Sort(sortedImages(images)) defaultOS := "" - for k, v := range ssDefaultOS { + for k, v := range urlDefaultOS { if strings.HasPrefix(s.url, k) { defaultOS = v break From bb110d7211cd683ec81266c10d67d47585834525 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgra...@ubuntu.com> Date: Tue, 3 Sep 2019 15:58:43 -0400 Subject: [PATCH 03/11] shared/simplestreams: Split index/manifest out MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber <stgra...@ubuntu.com> --- shared/simplestreams/index.go | 15 ++ shared/simplestreams/manifest.go | 250 +++++++++++++++++++++++++ shared/simplestreams/simplestreams.go | 254 -------------------------- 3 files changed, 265 insertions(+), 254 deletions(-) create mode 100644 shared/simplestreams/index.go create mode 100644 shared/simplestreams/manifest.go diff --git a/shared/simplestreams/index.go b/shared/simplestreams/index.go new file mode 100644 index 0000000000..2416c489c3 --- /dev/null +++ b/shared/simplestreams/index.go @@ -0,0 +1,15 @@ +package simplestreams + +type SimpleStreamsIndex struct { + Format string `json:"format"` + Index map[string]SimpleStreamsIndexStream `json:"index"` + Updated string `json:"updated,omitempty"` +} + +type SimpleStreamsIndexStream struct { + Updated string `json:"updated,omitempty"` + DataType string `json:"datatype"` + Path string `json:"path"` + Format string `json:"format,omitempty"` + Products []string `json:"products"` +} diff --git a/shared/simplestreams/manifest.go b/shared/simplestreams/manifest.go new file mode 100644 index 0000000000..c22b8721dc --- /dev/null +++ b/shared/simplestreams/manifest.go @@ -0,0 +1,250 @@ +package simplestreams + +import ( + "fmt" + "strings" + "time" + + "github.com/lxc/lxd/shared" + "github.com/lxc/lxd/shared/api" + "github.com/lxc/lxd/shared/osarch" +) + +type SimpleStreamsManifest struct { + Updated string `json:"updated,omitempty"` + DataType string `json:"datatype"` + Format string `json:"format"` + License string `json:"license,omitempty"` + Products map[string]SimpleStreamsManifestProduct `json:"products"` +} + +type SimpleStreamsManifestProduct struct { + Aliases string `json:"aliases"` + Architecture string `json:"arch"` + OperatingSystem string `json:"os"` + Release string `json:"release"` + ReleaseCodename string `json:"release_codename,omitempty"` + ReleaseTitle string `json:"release_title"` + Supported bool `json:"supported,omitempty"` + SupportedEOL string `json:"support_eol,omitempty"` + Version string `json:"version,omitempty"` + Versions map[string]SimpleStreamsManifestProductVersion `json:"versions"` +} + +type SimpleStreamsManifestProductVersion struct { + PublicName string `json:"pubname,omitempty"` + Label string `json:"label,omitempty"` + Items map[string]SimpleStreamsManifestProductVersionItem `json:"items"` +} + +type SimpleStreamsManifestProductVersionItem struct { + Path string `json:"path"` + FileType string `json:"ftype"` + HashMd5 string `json:"md5,omitempty"` + HashSha256 string `json:"sha256,omitempty"` + LXDHashSha256 string `json:"combined_sha256,omitempty"` + LXDHashSha256RootXz string `json:"combined_rootxz_sha256,omitempty"` + LXDHashSha256SquashFs string `json:"combined_squashfs_sha256,omitempty"` + Size int64 `json:"size"` + DeltaBase string `json:"delta_base,omitempty"` +} + +func (s *SimpleStreamsManifest) ToLXD() ([]api.Image, map[string][][]string) { + downloads := map[string][][]string{} + + images := []api.Image{} + nameLayout := "20060102" + eolLayout := "2006-01-02" + + for _, product := range s.Products { + // Skip unsupported architectures + architecture, err := osarch.ArchitectureId(product.Architecture) + if err != nil { + continue + } + + architectureName, err := osarch.ArchitectureName(architecture) + if err != nil { + continue + } + + for name, version := range product.Versions { + // Short of anything better, use the name as date (see format above) + if len(name) < 8 { + continue + } + + creationDate, err := time.Parse(nameLayout, name[0:8]) + if err != nil { + continue + } + + var meta SimpleStreamsManifestProductVersionItem + var rootTar SimpleStreamsManifestProductVersionItem + var rootSquash SimpleStreamsManifestProductVersionItem + deltas := []SimpleStreamsManifestProductVersionItem{} + + for _, item := range version.Items { + // Identify deltas + if item.FileType == "squashfs.vcdiff" { + deltas = append(deltas, item) + } + + // Skip the files we don't care about + if !shared.StringInSlice(item.FileType, []string{"root.tar.xz", "lxd.tar.xz", "lxd_combined.tar.gz", "squashfs"}) { + continue + } + + if item.FileType == "lxd.tar.xz" { + meta = item + } else if item.FileType == "squashfs" { + rootSquash = item + } else if item.FileType == "root.tar.xz" { + rootTar = item + } else if item.FileType == "lxd_combined.tar.gz" { + meta = item + rootTar = item + } + } + + if meta.FileType == "" || (rootTar.FileType == "" && rootSquash.FileType == "") { + // Invalid image + continue + } + + var rootfsSize int64 + metaPath := meta.Path + metaHash := meta.HashSha256 + metaSize := meta.Size + rootfsPath := "" + rootfsHash := "" + fields := strings.Split(meta.Path, "/") + filename := fields[len(fields)-1] + size := meta.Size + fingerprint := "" + + if rootSquash.FileType != "" { + if meta.LXDHashSha256SquashFs != "" { + fingerprint = meta.LXDHashSha256SquashFs + } else { + fingerprint = meta.LXDHashSha256 + } + size += rootSquash.Size + rootfsPath = rootSquash.Path + rootfsHash = rootSquash.HashSha256 + rootfsSize = rootSquash.Size + } else { + if meta == rootTar { + fingerprint = meta.HashSha256 + size = meta.Size + } else { + if meta.LXDHashSha256RootXz != "" { + fingerprint = meta.LXDHashSha256RootXz + } else { + fingerprint = meta.LXDHashSha256 + } + size += rootTar.Size + } + rootfsPath = rootTar.Path + rootfsHash = rootTar.HashSha256 + rootfsSize = rootTar.Size + } + + if size == 0 || filename == "" || fingerprint == "" { + // Invalid image + continue + } + + // Generate the actual image entry + description := fmt.Sprintf("%s %s %s", product.OperatingSystem, product.ReleaseTitle, product.Architecture) + if version.Label != "" { + description = fmt.Sprintf("%s (%s)", description, version.Label) + } + description = fmt.Sprintf("%s (%s)", description, name) + + image := api.Image{} + image.Architecture = architectureName + image.Public = true + image.Size = size + image.CreatedAt = creationDate + image.UploadedAt = creationDate + image.Filename = filename + image.Fingerprint = fingerprint + image.Properties = map[string]string{ + "os": product.OperatingSystem, + "release": product.Release, + "version": product.Version, + "architecture": product.Architecture, + "label": version.Label, + "serial": name, + "description": description, + } + + // Add the provided aliases + if product.Aliases != "" { + image.Aliases = []api.ImageAlias{} + for _, entry := range strings.Split(product.Aliases, ",") { + image.Aliases = append(image.Aliases, api.ImageAlias{Name: entry}) + } + } + + // Clear unset properties + for k, v := range image.Properties { + if v == "" { + delete(image.Properties, k) + } + } + + // Attempt to parse the EOL + image.ExpiresAt = time.Unix(0, 0).UTC() + if product.SupportedEOL != "" { + eolDate, err := time.Parse(eolLayout, product.SupportedEOL) + if err == nil { + image.ExpiresAt = eolDate + } + } + + var imgDownloads [][]string + if meta == rootTar { + imgDownloads = [][]string{{metaPath, metaHash, "meta", fmt.Sprintf("%d", metaSize)}} + } else { + imgDownloads = [][]string{ + {metaPath, metaHash, "meta", fmt.Sprintf("%d", metaSize)}, + {rootfsPath, rootfsHash, "root", fmt.Sprintf("%d", rootfsSize)}} + } + + // Add the deltas + for _, delta := range deltas { + srcImage, ok := product.Versions[delta.DeltaBase] + if !ok { + continue + } + + var srcFingerprint string + for _, item := range srcImage.Items { + if item.FileType != "lxd.tar.xz" { + continue + } + + srcFingerprint = item.LXDHashSha256SquashFs + break + } + + if srcFingerprint == "" { + continue + } + + imgDownloads = append(imgDownloads, []string{ + delta.Path, + delta.HashSha256, + fmt.Sprintf("root.delta-%s", srcFingerprint), + fmt.Sprintf("%d", delta.Size)}) + } + + downloads[fingerprint] = imgDownloads + images = append(images, image) + } + } + + return images, downloads +} diff --git a/shared/simplestreams/simplestreams.go b/shared/simplestreams/simplestreams.go index f629a33093..d1ac9cf3cc 100644 --- a/shared/simplestreams/simplestreams.go +++ b/shared/simplestreams/simplestreams.go @@ -12,7 +12,6 @@ import ( "sort" "strconv" "strings" - "time" "github.com/lxc/lxd/shared" "github.com/lxc/lxd/shared/api" @@ -24,259 +23,6 @@ var urlDefaultOS = map[string]string{ "https://cloud-images.ubuntu.com": "ubuntu", } -type SimpleStreamsManifest struct { - Updated string `json:"updated,omitempty"` - DataType string `json:"datatype"` - Format string `json:"format"` - License string `json:"license,omitempty"` - Products map[string]SimpleStreamsManifestProduct `json:"products"` -} - -func (s *SimpleStreamsManifest) ToLXD() ([]api.Image, map[string][][]string) { - downloads := map[string][][]string{} - - images := []api.Image{} - nameLayout := "20060102" - eolLayout := "2006-01-02" - - for _, product := range s.Products { - // Skip unsupported architectures - architecture, err := osarch.ArchitectureId(product.Architecture) - if err != nil { - continue - } - - architectureName, err := osarch.ArchitectureName(architecture) - if err != nil { - continue - } - - for name, version := range product.Versions { - // Short of anything better, use the name as date (see format above) - if len(name) < 8 { - continue - } - - creationDate, err := time.Parse(nameLayout, name[0:8]) - if err != nil { - continue - } - - var meta SimpleStreamsManifestProductVersionItem - var rootTar SimpleStreamsManifestProductVersionItem - var rootSquash SimpleStreamsManifestProductVersionItem - deltas := []SimpleStreamsManifestProductVersionItem{} - - for _, item := range version.Items { - // Identify deltas - if item.FileType == "squashfs.vcdiff" { - deltas = append(deltas, item) - } - - // Skip the files we don't care about - if !shared.StringInSlice(item.FileType, []string{"root.tar.xz", "lxd.tar.xz", "lxd_combined.tar.gz", "squashfs"}) { - continue - } - - if item.FileType == "lxd.tar.xz" { - meta = item - } else if item.FileType == "squashfs" { - rootSquash = item - } else if item.FileType == "root.tar.xz" { - rootTar = item - } else if item.FileType == "lxd_combined.tar.gz" { - meta = item - rootTar = item - } - } - - if meta.FileType == "" || (rootTar.FileType == "" && rootSquash.FileType == "") { - // Invalid image - continue - } - - var rootfsSize int64 - metaPath := meta.Path - metaHash := meta.HashSha256 - metaSize := meta.Size - rootfsPath := "" - rootfsHash := "" - fields := strings.Split(meta.Path, "/") - filename := fields[len(fields)-1] - size := meta.Size - fingerprint := "" - - if rootSquash.FileType != "" { - if meta.LXDHashSha256SquashFs != "" { - fingerprint = meta.LXDHashSha256SquashFs - } else { - fingerprint = meta.LXDHashSha256 - } - size += rootSquash.Size - rootfsPath = rootSquash.Path - rootfsHash = rootSquash.HashSha256 - rootfsSize = rootSquash.Size - } else { - if meta == rootTar { - fingerprint = meta.HashSha256 - size = meta.Size - } else { - if meta.LXDHashSha256RootXz != "" { - fingerprint = meta.LXDHashSha256RootXz - } else { - fingerprint = meta.LXDHashSha256 - } - size += rootTar.Size - } - rootfsPath = rootTar.Path - rootfsHash = rootTar.HashSha256 - rootfsSize = rootTar.Size - } - - if size == 0 || filename == "" || fingerprint == "" { - // Invalid image - continue - } - - // Generate the actual image entry - description := fmt.Sprintf("%s %s %s", product.OperatingSystem, product.ReleaseTitle, product.Architecture) - if version.Label != "" { - description = fmt.Sprintf("%s (%s)", description, version.Label) - } - description = fmt.Sprintf("%s (%s)", description, name) - - image := api.Image{} - image.Architecture = architectureName - image.Public = true - image.Size = size - image.CreatedAt = creationDate - image.UploadedAt = creationDate - image.Filename = filename - image.Fingerprint = fingerprint - image.Properties = map[string]string{ - "os": product.OperatingSystem, - "release": product.Release, - "version": product.Version, - "architecture": product.Architecture, - "label": version.Label, - "serial": name, - "description": description, - } - - // Add the provided aliases - if product.Aliases != "" { - image.Aliases = []api.ImageAlias{} - for _, entry := range strings.Split(product.Aliases, ",") { - image.Aliases = append(image.Aliases, api.ImageAlias{Name: entry}) - } - } - - // Clear unset properties - for k, v := range image.Properties { - if v == "" { - delete(image.Properties, k) - } - } - - // Attempt to parse the EOL - image.ExpiresAt = time.Unix(0, 0).UTC() - if product.SupportedEOL != "" { - eolDate, err := time.Parse(eolLayout, product.SupportedEOL) - if err == nil { - image.ExpiresAt = eolDate - } - } - - var imgDownloads [][]string - if meta == rootTar { - imgDownloads = [][]string{{metaPath, metaHash, "meta", fmt.Sprintf("%d", metaSize)}} - } else { - imgDownloads = [][]string{ - {metaPath, metaHash, "meta", fmt.Sprintf("%d", metaSize)}, - {rootfsPath, rootfsHash, "root", fmt.Sprintf("%d", rootfsSize)}} - } - - // Add the deltas - for _, delta := range deltas { - srcImage, ok := product.Versions[delta.DeltaBase] - if !ok { - continue - } - - var srcFingerprint string - for _, item := range srcImage.Items { - if item.FileType != "lxd.tar.xz" { - continue - } - - srcFingerprint = item.LXDHashSha256SquashFs - break - } - - if srcFingerprint == "" { - continue - } - - imgDownloads = append(imgDownloads, []string{ - delta.Path, - delta.HashSha256, - fmt.Sprintf("root.delta-%s", srcFingerprint), - fmt.Sprintf("%d", delta.Size)}) - } - - downloads[fingerprint] = imgDownloads - images = append(images, image) - } - } - - return images, downloads -} - -type SimpleStreamsManifestProduct struct { - Aliases string `json:"aliases"` - Architecture string `json:"arch"` - OperatingSystem string `json:"os"` - Release string `json:"release"` - ReleaseCodename string `json:"release_codename,omitempty"` - ReleaseTitle string `json:"release_title"` - Supported bool `json:"supported,omitempty"` - SupportedEOL string `json:"support_eol,omitempty"` - Version string `json:"version,omitempty"` - Versions map[string]SimpleStreamsManifestProductVersion `json:"versions"` -} - -type SimpleStreamsManifestProductVersion struct { - PublicName string `json:"pubname,omitempty"` - Label string `json:"label,omitempty"` - Items map[string]SimpleStreamsManifestProductVersionItem `json:"items"` -} - -type SimpleStreamsManifestProductVersionItem struct { - Path string `json:"path"` - FileType string `json:"ftype"` - HashMd5 string `json:"md5,omitempty"` - HashSha256 string `json:"sha256,omitempty"` - LXDHashSha256 string `json:"combined_sha256,omitempty"` - LXDHashSha256RootXz string `json:"combined_rootxz_sha256,omitempty"` - LXDHashSha256SquashFs string `json:"combined_squashfs_sha256,omitempty"` - Size int64 `json:"size"` - DeltaBase string `json:"delta_base,omitempty"` -} - -type SimpleStreamsIndex struct { - Format string `json:"format"` - Index map[string]SimpleStreamsIndexStream `json:"index"` - Updated string `json:"updated,omitempty"` -} - -type SimpleStreamsIndexStream struct { - Updated string `json:"updated,omitempty"` - DataType string `json:"datatype"` - Path string `json:"path"` - Format string `json:"format,omitempty"` - Products []string `json:"products"` -} - type SimpleStreamsFile struct { Path string Sha256 string From 2eb29c1dd3d922a9f4980a983ca7f058e9a89483 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgra...@ubuntu.com> Date: Tue, 3 Sep 2019 16:09:08 -0400 Subject: [PATCH 04/11] shared/simplestreams: Rename index structs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber <stgra...@ubuntu.com> --- shared/simplestreams/index.go | 16 +++++++++------- shared/simplestreams/simplestreams.go | 16 ++++++++-------- 2 files changed, 17 insertions(+), 15 deletions(-) diff --git a/shared/simplestreams/index.go b/shared/simplestreams/index.go index 2416c489c3..91f7c197fd 100644 --- a/shared/simplestreams/index.go +++ b/shared/simplestreams/index.go @@ -1,15 +1,17 @@ package simplestreams -type SimpleStreamsIndex struct { - Format string `json:"format"` - Index map[string]SimpleStreamsIndexStream `json:"index"` - Updated string `json:"updated,omitempty"` +// Stream represents the base structure of index.json +type Stream struct { + Index map[string]StreamIndex `json:"index"` + Updated string `json:"updated,omitempty"` + Format string `json:"format"` } -type SimpleStreamsIndexStream struct { - Updated string `json:"updated,omitempty"` +// StreamIndex represents the Index entry inside index.json +type StreamIndex struct { DataType string `json:"datatype"` Path string `json:"path"` - Format string `json:"format,omitempty"` + Updated string `json:"updated,omitempty"` Products []string `json:"products"` + Format string `json:"format,omitempty"` } diff --git a/shared/simplestreams/simplestreams.go b/shared/simplestreams/simplestreams.go index d1ac9cf3cc..81f4a68012 100644 --- a/shared/simplestreams/simplestreams.go +++ b/shared/simplestreams/simplestreams.go @@ -43,15 +43,15 @@ type SimpleStreams struct { url string useragent string - cachedIndex *SimpleStreamsIndex + cachedStream *Stream cachedManifest map[string]*SimpleStreamsManifest cachedImages []api.Image cachedAliases map[string]*api.ImageAliasesEntry } -func (s *SimpleStreams) parseIndex() (*SimpleStreamsIndex, error) { - if s.cachedIndex != nil { - return s.cachedIndex, nil +func (s *SimpleStreams) parseIndex() (*Stream, error) { + if s.cachedStream != nil { + return s.cachedStream, nil } url := fmt.Sprintf("%s/streams/v1/index.json", s.url) @@ -80,15 +80,15 @@ func (s *SimpleStreams) parseIndex() (*SimpleStreamsIndex, error) { } // Parse the idnex - ssIndex := SimpleStreamsIndex{} - err = json.Unmarshal(body, &ssIndex) + stream := Stream{} + err = json.Unmarshal(body, &stream) if err != nil { return nil, err } - s.cachedIndex = &ssIndex + s.cachedStream = &stream - return &ssIndex, nil + return &stream, nil } func (s *SimpleStreams) parseManifest(path string) (*SimpleStreamsManifest, error) { From 498666b87688190cbec7003c86e9622515efaca5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgra...@ubuntu.com> Date: Tue, 3 Sep 2019 16:16:17 -0400 Subject: [PATCH 05/11] shared/simplestreams: Rename product structs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber <stgra...@ubuntu.com> --- .../{manifest.go => products.go} | 69 ++++++++++--------- shared/simplestreams/simplestreams.go | 8 +-- 2 files changed, 42 insertions(+), 35 deletions(-) rename shared/simplestreams/{manifest.go => products.go} (74%) diff --git a/shared/simplestreams/manifest.go b/shared/simplestreams/products.go similarity index 74% rename from shared/simplestreams/manifest.go rename to shared/simplestreams/products.go index c22b8721dc..ab89eaea48 100644 --- a/shared/simplestreams/manifest.go +++ b/shared/simplestreams/products.go @@ -10,46 +10,53 @@ import ( "github.com/lxc/lxd/shared/osarch" ) -type SimpleStreamsManifest struct { - Updated string `json:"updated,omitempty"` - DataType string `json:"datatype"` - Format string `json:"format"` - License string `json:"license,omitempty"` - Products map[string]SimpleStreamsManifestProduct `json:"products"` +// Products represents the base of download.json +type Products struct { + ContentID string `json:"content_id"` + DataType string `json:"datatype"` + Format string `json:"format"` + License string `json:"license,omitempty"` + Products map[string]Product `json:"products"` + Updated string `json:"updated,omitempty"` } -type SimpleStreamsManifestProduct struct { - Aliases string `json:"aliases"` - Architecture string `json:"arch"` - OperatingSystem string `json:"os"` - Release string `json:"release"` - ReleaseCodename string `json:"release_codename,omitempty"` - ReleaseTitle string `json:"release_title"` - Supported bool `json:"supported,omitempty"` - SupportedEOL string `json:"support_eol,omitempty"` - Version string `json:"version,omitempty"` - Versions map[string]SimpleStreamsManifestProductVersion `json:"versions"` +// Product represents a single product inside download.json +type Product struct { + Aliases string `json:"aliases"` + Architecture string `json:"arch"` + OperatingSystem string `json:"os"` + Release string `json:"release"` + ReleaseCodename string `json:"release_codename,omitempty"` + ReleaseTitle string `json:"release_title"` + Supported bool `json:"supported,omitempty"` + SupportedEOL string `json:"support_eol,omitempty"` + Version string `json:"version,omitempty"` + Versions map[string]ProductVersion `json:"versions"` } -type SimpleStreamsManifestProductVersion struct { - PublicName string `json:"pubname,omitempty"` - Label string `json:"label,omitempty"` - Items map[string]SimpleStreamsManifestProductVersionItem `json:"items"` +// ProductVersion represents a particular version of a product +type ProductVersion struct { + Items map[string]ProductVersionItem `json:"items"` + Label string `json:"label,omitempty"` + PublicName string `json:"pubname,omitempty"` } -type SimpleStreamsManifestProductVersionItem struct { - Path string `json:"path"` +// ProductVersionItem represents a file/item of a particular ProductVersion +type ProductVersionItem struct { + LXDHashSha256RootXz string `json:"combined_rootxz_sha256,omitempty"` + LXDHashSha256 string `json:"combined_sha256,omitempty"` + LXDHashSha256SquashFs string `json:"combined_squashfs_sha256,omitempty"` FileType string `json:"ftype"` HashMd5 string `json:"md5,omitempty"` + Path string `json:"path"` HashSha256 string `json:"sha256,omitempty"` - LXDHashSha256 string `json:"combined_sha256,omitempty"` - LXDHashSha256RootXz string `json:"combined_rootxz_sha256,omitempty"` - LXDHashSha256SquashFs string `json:"combined_squashfs_sha256,omitempty"` Size int64 `json:"size"` DeltaBase string `json:"delta_base,omitempty"` } -func (s *SimpleStreamsManifest) ToLXD() ([]api.Image, map[string][][]string) { + +// ToLXD converts the products data into a list of LXD images and associated downloadable files +func (s *Products) ToLXD() ([]api.Image, map[string][][]string) { downloads := map[string][][]string{} images := []api.Image{} @@ -79,10 +86,10 @@ func (s *SimpleStreamsManifest) ToLXD() ([]api.Image, map[string][][]string) { continue } - var meta SimpleStreamsManifestProductVersionItem - var rootTar SimpleStreamsManifestProductVersionItem - var rootSquash SimpleStreamsManifestProductVersionItem - deltas := []SimpleStreamsManifestProductVersionItem{} + var meta ProductVersionItem + var rootTar ProductVersionItem + var rootSquash ProductVersionItem + deltas := []ProductVersionItem{} for _, item := range version.Items { // Identify deltas diff --git a/shared/simplestreams/simplestreams.go b/shared/simplestreams/simplestreams.go index 81f4a68012..6182b1c2e3 100644 --- a/shared/simplestreams/simplestreams.go +++ b/shared/simplestreams/simplestreams.go @@ -33,7 +33,7 @@ func NewClient(url string, httpClient http.Client, useragent string) *SimpleStre return &SimpleStreams{ http: &httpClient, url: url, - cachedManifest: map[string]*SimpleStreamsManifest{}, + cachedManifest: map[string]*Products{}, useragent: useragent, } } @@ -44,7 +44,7 @@ type SimpleStreams struct { useragent string cachedStream *Stream - cachedManifest map[string]*SimpleStreamsManifest + cachedManifest map[string]*Products cachedImages []api.Image cachedAliases map[string]*api.ImageAliasesEntry } @@ -91,7 +91,7 @@ func (s *SimpleStreams) parseIndex() (*Stream, error) { return &stream, nil } -func (s *SimpleStreams) parseManifest(path string) (*SimpleStreamsManifest, error) { +func (s *SimpleStreams) parseManifest(path string) (*Products, error) { if s.cachedManifest[path] != nil { return s.cachedManifest[path], nil } @@ -122,7 +122,7 @@ func (s *SimpleStreams) parseManifest(path string) (*SimpleStreamsManifest, erro } // Parse the idnex - ssManifest := SimpleStreamsManifest{} + ssManifest := Products{} err = json.Unmarshal(body, &ssManifest) if err != nil { return nil, err From c4d50f1249636cd73063b6399c41421f56762b4a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgra...@ubuntu.com> Date: Tue, 3 Sep 2019 16:19:31 -0400 Subject: [PATCH 06/11] shared/simplestreams: Rename SimpleStreamsFile MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber <stgra...@ubuntu.com> --- shared/simplestreams/products.go | 1 - shared/simplestreams/simplestreams.go | 8 ++++---- 2 files changed, 4 insertions(+), 5 deletions(-) diff --git a/shared/simplestreams/products.go b/shared/simplestreams/products.go index ab89eaea48..920dc01f84 100644 --- a/shared/simplestreams/products.go +++ b/shared/simplestreams/products.go @@ -54,7 +54,6 @@ type ProductVersionItem struct { DeltaBase string `json:"delta_base,omitempty"` } - // ToLXD converts the products data into a list of LXD images and associated downloadable files func (s *Products) ToLXD() ([]api.Image, map[string][][]string) { downloads := map[string][][]string{} diff --git a/shared/simplestreams/simplestreams.go b/shared/simplestreams/simplestreams.go index 6182b1c2e3..56d904c765 100644 --- a/shared/simplestreams/simplestreams.go +++ b/shared/simplestreams/simplestreams.go @@ -23,7 +23,7 @@ var urlDefaultOS = map[string]string{ "https://cloud-images.ubuntu.com": "ubuntu", } -type SimpleStreamsFile struct { +type DownloadableFile struct { Path string Sha256 string Size int64 @@ -244,7 +244,7 @@ func (s *SimpleStreams) getImages() ([]api.Image, map[string]*api.ImageAliasesEn return images, aliases, nil } -func (s *SimpleStreams) GetFiles(fingerprint string) (map[string]SimpleStreamsFile, error) { +func (s *SimpleStreams) GetFiles(fingerprint string) (map[string]DownloadableFile, error) { // Load the main index ssIndex, err := s.parseIndex() if err != nil { @@ -272,7 +272,7 @@ func (s *SimpleStreams) GetFiles(fingerprint string) (map[string]SimpleStreamsFi for _, image := range manifestImages { if strings.HasPrefix(image.Fingerprint, fingerprint) { - files := map[string]SimpleStreamsFile{} + files := map[string]DownloadableFile{} for _, path := range downloads[image.Fingerprint] { size, err := strconv.ParseInt(path[3], 10, 64) @@ -280,7 +280,7 @@ func (s *SimpleStreams) GetFiles(fingerprint string) (map[string]SimpleStreamsFi return nil, err } - files[path[2]] = SimpleStreamsFile{ + files[path[2]] = DownloadableFile{ Path: path[0], Sha256: path[1], Size: size} From 40e44f1ed21ab7d0a5c94db262ecdf338ce65464 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgra...@ubuntu.com> Date: Tue, 3 Sep 2019 16:25:26 -0400 Subject: [PATCH 07/11] shared/simplestreams: Rename internal functions MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber <stgra...@ubuntu.com> --- shared/simplestreams/simplestreams.go | 48 +++++++++++++-------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/shared/simplestreams/simplestreams.go b/shared/simplestreams/simplestreams.go index 56d904c765..7c85aae880 100644 --- a/shared/simplestreams/simplestreams.go +++ b/shared/simplestreams/simplestreams.go @@ -33,7 +33,7 @@ func NewClient(url string, httpClient http.Client, useragent string) *SimpleStre return &SimpleStreams{ http: &httpClient, url: url, - cachedManifest: map[string]*Products{}, + cachedProducts: map[string]*Products{}, useragent: useragent, } } @@ -44,12 +44,12 @@ type SimpleStreams struct { useragent string cachedStream *Stream - cachedManifest map[string]*Products + cachedProducts map[string]*Products cachedImages []api.Image cachedAliases map[string]*api.ImageAliasesEntry } -func (s *SimpleStreams) parseIndex() (*Stream, error) { +func (s *SimpleStreams) parseStream() (*Stream, error) { if s.cachedStream != nil { return s.cachedStream, nil } @@ -91,9 +91,9 @@ func (s *SimpleStreams) parseIndex() (*Stream, error) { return &stream, nil } -func (s *SimpleStreams) parseManifest(path string) (*Products, error) { - if s.cachedManifest[path] != nil { - return s.cachedManifest[path], nil +func (s *SimpleStreams) parseProducts(path string) (*Products, error) { + if s.cachedProducts[path] != nil { + return s.cachedProducts[path], nil } url := fmt.Sprintf("%s/%s", s.url, path) @@ -122,15 +122,15 @@ func (s *SimpleStreams) parseManifest(path string) (*Products, error) { } // Parse the idnex - ssManifest := Products{} - err = json.Unmarshal(body, &ssManifest) + products := Products{} + err = json.Unmarshal(body, &products) if err != nil { return nil, err } - s.cachedManifest[path] = &ssManifest + s.cachedProducts[path] = &products - return &ssManifest, nil + return &products, nil } func (s *SimpleStreams) applyAliases(images []api.Image) ([]api.Image, map[string]*api.ImageAliasesEntry, error) { @@ -202,14 +202,14 @@ func (s *SimpleStreams) getImages() ([]api.Image, map[string]*api.ImageAliasesEn images := []api.Image{} - // Load the main index - ssIndex, err := s.parseIndex() + // Load the stream data + stream, err := s.parseStream() if err != nil { return nil, nil, err } - // Iterate through the various image manifests - for _, entry := range ssIndex.Index { + // Iterate through the various indices + for _, entry := range stream.Index { // We only care about images if entry.DataType != "image-downloads" { continue @@ -220,14 +220,14 @@ func (s *SimpleStreams) getImages() ([]api.Image, map[string]*api.ImageAliasesEn continue } - manifest, err := s.parseManifest(entry.Path) + products, err := s.parseProducts(entry.Path) if err != nil { return nil, nil, err } - manifestImages, _ := manifest.ToLXD() + streamImages, _ := products.ToLXD() - for _, image := range manifestImages { + for _, image := range streamImages { images = append(images, image) } } @@ -245,14 +245,14 @@ func (s *SimpleStreams) getImages() ([]api.Image, map[string]*api.ImageAliasesEn } func (s *SimpleStreams) GetFiles(fingerprint string) (map[string]DownloadableFile, error) { - // Load the main index - ssIndex, err := s.parseIndex() + // Load the main stream + stream, err := s.parseStream() if err != nil { return nil, err } - // Iterate through the various image manifests - for _, entry := range ssIndex.Index { + // Iterate through the various indices + for _, entry := range stream.Index { // We only care about images if entry.DataType != "image-downloads" { continue @@ -263,14 +263,14 @@ func (s *SimpleStreams) GetFiles(fingerprint string) (map[string]DownloadableFil continue } - manifest, err := s.parseManifest(entry.Path) + products, err := s.parseProducts(entry.Path) if err != nil { return nil, err } - manifestImages, downloads := manifest.ToLXD() + images, downloads := products.ToLXD() - for _, image := range manifestImages { + for _, image := range images { if strings.HasPrefix(image.Fingerprint, fingerprint) { files := map[string]DownloadableFile{} From 3b594dffa599658947e872563873af5c3559a130 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgra...@ubuntu.com> Date: Tue, 3 Sep 2019 16:33:40 -0400 Subject: [PATCH 08/11] shared/simplestreams: Remove dead code MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber <stgra...@ubuntu.com> --- shared/simplestreams/simplestreams.go | 109 -------------------------- 1 file changed, 109 deletions(-) diff --git a/shared/simplestreams/simplestreams.go b/shared/simplestreams/simplestreams.go index 7c85aae880..e8e16dfafd 100644 --- a/shared/simplestreams/simplestreams.go +++ b/shared/simplestreams/simplestreams.go @@ -1,21 +1,15 @@ package simplestreams import ( - "crypto/sha256" "encoding/json" "fmt" - "io" "io/ioutil" "net/http" - "os" - "path/filepath" "sort" "strconv" "strings" - "github.com/lxc/lxd/shared" "github.com/lxc/lxd/shared/api" - "github.com/lxc/lxd/shared/ioprogress" "github.com/lxc/lxd/shared/osarch" ) @@ -294,72 +288,6 @@ func (s *SimpleStreams) GetFiles(fingerprint string) (map[string]DownloadableFil return nil, fmt.Errorf("Couldn't find the requested image") } -func (s *SimpleStreams) downloadFile(path string, hash string, target string, progress func(int64, int64)) error { - download := func(url string, hash string, target string) error { - out, err := os.Create(target) - if err != nil { - return err - } - defer out.Close() - - req, err := http.NewRequest("GET", url, nil) - if err != nil { - return err - } - - if s.useragent != "" { - req.Header.Set("User-Agent", s.useragent) - } - - r, err := s.http.Do(req) - if err != nil { - return err - } - defer r.Body.Close() - - if r.StatusCode != http.StatusOK { - return fmt.Errorf("Unable to fetch %s: %s", url, r.Status) - } - - body := &ioprogress.ProgressReader{ - ReadCloser: r.Body, - Tracker: &ioprogress.ProgressTracker{ - Length: r.ContentLength, - Handler: progress, - }, - } - - sha256 := sha256.New() - _, err = io.Copy(io.MultiWriter(out, sha256), body) - if err != nil { - return err - } - - result := fmt.Sprintf("%x", sha256.Sum(nil)) - if result != hash { - os.Remove(target) - return fmt.Errorf("Hash mismatch for %s: %s != %s", path, result, hash) - } - - return nil - } - - // Try http first - if strings.HasPrefix(s.url, "https://") { - err := download(fmt.Sprintf("http://%s/%s", strings.TrimPrefix(s.url, "https://"), path), hash, target) - if err == nil { - return nil - } - } - - err := download(fmt.Sprintf("%s/%s", s.url, path), hash, target) - if err != nil { - return err - } - - return nil -} - func (s *SimpleStreams) ListAliases() ([]api.ImageAliasesEntry, error) { _, aliasesMap, err := s.getImages() if err != nil { @@ -416,40 +344,3 @@ func (s *SimpleStreams) GetImage(fingerprint string) (*api.Image, error) { return &matches[0], nil } - -func (s *SimpleStreams) ExportImage(image string, target string) (string, error) { - if !shared.IsDir(target) { - return "", fmt.Errorf("Split images can only be written to a directory") - } - - files, err := s.GetFiles(image) - if err != nil { - return "", err - } - - for _, file := range files { - fields := strings.Split(file.Path, "/") - targetFile := filepath.Join(target, fields[len(fields)-1]) - - err := s.downloadFile(file.Path, file.Sha256, targetFile, nil) - if err != nil { - return "", err - } - } - - return target, nil -} - -func (s *SimpleStreams) Download(image string, fileType string, target string, progress func(int64, int64)) error { - files, err := s.GetFiles(image) - if err != nil { - return err - } - - file, ok := files[fileType] - if ok { - return s.downloadFile(file.Path, file.Sha256, target, progress) - } - - return fmt.Errorf("The file couldn't be found") -} From 50cb1720a930493faff47f427335e21ebf28dca9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgra...@ubuntu.com> Date: Tue, 3 Sep 2019 16:34:50 -0400 Subject: [PATCH 09/11] shared/simplestreams: Make golint clean MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber <stgra...@ubuntu.com> --- shared/simplestreams/simplestreams.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/shared/simplestreams/simplestreams.go b/shared/simplestreams/simplestreams.go index e8e16dfafd..f6f805894f 100644 --- a/shared/simplestreams/simplestreams.go +++ b/shared/simplestreams/simplestreams.go @@ -17,12 +17,14 @@ var urlDefaultOS = map[string]string{ "https://cloud-images.ubuntu.com": "ubuntu", } +// DownloadableFile represents a file with its URL, hash and size type DownloadableFile struct { Path string Sha256 string Size int64 } +// NewClient returns a simplestreams client for the provided stream URL func NewClient(url string, httpClient http.Client, useragent string) *SimpleStreams { return &SimpleStreams{ http: &httpClient, @@ -32,6 +34,7 @@ func NewClient(url string, httpClient http.Client, useragent string) *SimpleStre } } +// SimpleStreams represents a simplestream client type SimpleStreams struct { http *http.Client url string @@ -238,6 +241,7 @@ func (s *SimpleStreams) getImages() ([]api.Image, map[string]*api.ImageAliasesEn return images, aliases, nil } +// GetFiles returns a map of files for the provided image fingerprint func (s *SimpleStreams) GetFiles(fingerprint string) (map[string]DownloadableFile, error) { // Load the main stream stream, err := s.parseStream() @@ -288,6 +292,7 @@ func (s *SimpleStreams) GetFiles(fingerprint string) (map[string]DownloadableFil return nil, fmt.Errorf("Couldn't find the requested image") } +// ListAliases returns a list of image aliases for the provided image fingerprint func (s *SimpleStreams) ListAliases() ([]api.ImageAliasesEntry, error) { _, aliasesMap, err := s.getImages() if err != nil { @@ -303,11 +308,13 @@ func (s *SimpleStreams) ListAliases() ([]api.ImageAliasesEntry, error) { return aliases, nil } +// ListImages returns a list of LXD images func (s *SimpleStreams) ListImages() ([]api.Image, error) { images, _, err := s.getImages() return images, err } +// GetAlias returns a LXD ImageAliasesEntry for the provided alias name func (s *SimpleStreams) GetAlias(name string) (*api.ImageAliasesEntry, error) { _, aliasesMap, err := s.getImages() if err != nil { @@ -322,6 +329,7 @@ func (s *SimpleStreams) GetAlias(name string) (*api.ImageAliasesEntry, error) { return alias, nil } +// GetImage returns a LXD image for the provided image fingerprint func (s *SimpleStreams) GetImage(fingerprint string) (*api.Image, error) { images, _, err := s.getImages() if err != nil { From 72f42c01ce7cb714cf8f8ac4df596cbbf29c500a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgra...@ubuntu.com> Date: Tue, 3 Sep 2019 16:35:32 -0400 Subject: [PATCH 10/11] tests: Add simplestreams to golint MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Stéphane Graber <stgra...@ubuntu.com> --- test/suites/static_analysis.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/test/suites/static_analysis.sh b/test/suites/static_analysis.sh index cd2da54018..8f40270444 100644 --- a/test/suites/static_analysis.sh +++ b/test/suites/static_analysis.sh @@ -101,6 +101,7 @@ test_static_analysis() { golint -set_exit_status shared/log15/stack golint -set_exit_status shared/logger/ golint -set_exit_status shared/logging/ + golint -set_exit_status shared/simplestreams/ golint -set_exit_status shared/subtest/ golint -set_exit_status shared/termios/ golint -set_exit_status shared/version/ From 4c857306e1d866312ea13959f15e3a3171d8c176 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?St=C3=A9phane=20Graber?= <stgra...@ubuntu.com> Date: Tue, 3 Sep 2019 17:27:59 -0400 Subject: [PATCH 11/11] shared/simplestreams: Record all images MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Until now when both squashfs and tar.xz existed, only squashfs would make it to the final index, making it impossible to query information about the tar.xz image. This fixes that, while still prefering squashfs in results. Signed-off-by: Stéphane Graber <stgra...@ubuntu.com> --- shared/simplestreams/products.go | 277 ++++++++++++++++--------------- shared/simplestreams/sort.go | 6 + 2 files changed, 148 insertions(+), 135 deletions(-) diff --git a/shared/simplestreams/products.go b/shared/simplestreams/products.go index 920dc01f84..b1b06c0681 100644 --- a/shared/simplestreams/products.go +++ b/shared/simplestreams/products.go @@ -85,170 +85,177 @@ func (s *Products) ToLXD() ([]api.Image, map[string][][]string) { continue } - var meta ProductVersionItem - var rootTar ProductVersionItem - var rootSquash ProductVersionItem - deltas := []ProductVersionItem{} + // Image processing function + addImage := func(meta *ProductVersionItem, root *ProductVersionItem) error { + // Look for deltas (only on squashfs) + deltas := []ProductVersionItem{} + if root != nil && root.FileType == "squashfs" { + for _, item := range version.Items { + if item.FileType == "squashfs.vcdiff" { + deltas = append(deltas, item) + } + } + } - for _, item := range version.Items { - // Identify deltas - if item.FileType == "squashfs.vcdiff" { - deltas = append(deltas, item) + // Figure out the fingerprint + fingerprint := "" + if root != nil { + if root.FileType == "root.tar.xz" { + if meta.LXDHashSha256RootXz != "" { + fingerprint = meta.LXDHashSha256RootXz + } else { + fingerprint = meta.LXDHashSha256 + } + } else if root.FileType == "squashfs" { + fingerprint = meta.LXDHashSha256SquashFs + } + } else { + fingerprint = meta.HashSha256 } - // Skip the files we don't care about - if !shared.StringInSlice(item.FileType, []string{"root.tar.xz", "lxd.tar.xz", "lxd_combined.tar.gz", "squashfs"}) { - continue + if fingerprint == "" { + return fmt.Errorf("No LXD image fingerprint found") } - if item.FileType == "lxd.tar.xz" { - meta = item - } else if item.FileType == "squashfs" { - rootSquash = item - } else if item.FileType == "root.tar.xz" { - rootTar = item - } else if item.FileType == "lxd_combined.tar.gz" { - meta = item - rootTar = item + // Figure out the size + size := meta.Size + if root != nil { + size += root.Size } - } - if meta.FileType == "" || (rootTar.FileType == "" && rootSquash.FileType == "") { - // Invalid image - continue - } + // Determine filename + if meta.Path == "" { + return fmt.Errorf("Missing path field on metadata entry") + } - var rootfsSize int64 - metaPath := meta.Path - metaHash := meta.HashSha256 - metaSize := meta.Size - rootfsPath := "" - rootfsHash := "" - fields := strings.Split(meta.Path, "/") - filename := fields[len(fields)-1] - size := meta.Size - fingerprint := "" - - if rootSquash.FileType != "" { - if meta.LXDHashSha256SquashFs != "" { - fingerprint = meta.LXDHashSha256SquashFs - } else { - fingerprint = meta.LXDHashSha256 + fields := strings.Split(meta.Path, "/") + filename := fields[len(fields)-1] + + // Generate the actual image entry + description := fmt.Sprintf("%s %s %s", product.OperatingSystem, product.ReleaseTitle, product.Architecture) + if version.Label != "" { + description = fmt.Sprintf("%s (%s)", description, version.Label) } - size += rootSquash.Size - rootfsPath = rootSquash.Path - rootfsHash = rootSquash.HashSha256 - rootfsSize = rootSquash.Size - } else { - if meta == rootTar { - fingerprint = meta.HashSha256 - size = meta.Size - } else { - if meta.LXDHashSha256RootXz != "" { - fingerprint = meta.LXDHashSha256RootXz - } else { - fingerprint = meta.LXDHashSha256 - } - size += rootTar.Size + description = fmt.Sprintf("%s (%s)", description, name) + + image := api.Image{} + image.Architecture = architectureName + image.Public = true + image.Size = size + image.CreatedAt = creationDate + image.UploadedAt = creationDate + image.Filename = filename + image.Fingerprint = fingerprint + image.Properties = map[string]string{ + "os": product.OperatingSystem, + "release": product.Release, + "version": product.Version, + "architecture": product.Architecture, + "label": version.Label, + "serial": name, + "description": description, } - rootfsPath = rootTar.Path - rootfsHash = rootTar.HashSha256 - rootfsSize = rootTar.Size - } - if size == 0 || filename == "" || fingerprint == "" { - // Invalid image - continue - } + if root != nil { + image.Properties["type"] = root.FileType + } else { + image.Properties["type"] = "tar.gz" + } - // Generate the actual image entry - description := fmt.Sprintf("%s %s %s", product.OperatingSystem, product.ReleaseTitle, product.Architecture) - if version.Label != "" { - description = fmt.Sprintf("%s (%s)", description, version.Label) - } - description = fmt.Sprintf("%s (%s)", description, name) - - image := api.Image{} - image.Architecture = architectureName - image.Public = true - image.Size = size - image.CreatedAt = creationDate - image.UploadedAt = creationDate - image.Filename = filename - image.Fingerprint = fingerprint - image.Properties = map[string]string{ - "os": product.OperatingSystem, - "release": product.Release, - "version": product.Version, - "architecture": product.Architecture, - "label": version.Label, - "serial": name, - "description": description, - } + // Clear unset properties + for k, v := range image.Properties { + if v == "" { + delete(image.Properties, k) + } + } - // Add the provided aliases - if product.Aliases != "" { - image.Aliases = []api.ImageAlias{} - for _, entry := range strings.Split(product.Aliases, ",") { - image.Aliases = append(image.Aliases, api.ImageAlias{Name: entry}) + // Add the provided aliases + if product.Aliases != "" { + image.Aliases = []api.ImageAlias{} + for _, entry := range strings.Split(product.Aliases, ",") { + image.Aliases = append(image.Aliases, api.ImageAlias{Name: entry}) + } } - } - // Clear unset properties - for k, v := range image.Properties { - if v == "" { - delete(image.Properties, k) + // Attempt to parse the EOL + image.ExpiresAt = time.Unix(0, 0).UTC() + if product.SupportedEOL != "" { + eolDate, err := time.Parse(eolLayout, product.SupportedEOL) + if err == nil { + image.ExpiresAt = eolDate + } } - } - // Attempt to parse the EOL - image.ExpiresAt = time.Unix(0, 0).UTC() - if product.SupportedEOL != "" { - eolDate, err := time.Parse(eolLayout, product.SupportedEOL) - if err == nil { - image.ExpiresAt = eolDate + // Set the file list + var imgDownloads [][]string + if root == nil { + imgDownloads = [][]string{{meta.Path, meta.HashSha256, "meta", fmt.Sprintf("%d", meta.Size)}} + } else { + imgDownloads = [][]string{ + {meta.Path, meta.HashSha256, "meta", fmt.Sprintf("%d", meta.Size)}, + {root.Path, root.HashSha256, "root", fmt.Sprintf("%d", root.Size)}} } - } - var imgDownloads [][]string - if meta == rootTar { - imgDownloads = [][]string{{metaPath, metaHash, "meta", fmt.Sprintf("%d", metaSize)}} - } else { - imgDownloads = [][]string{ - {metaPath, metaHash, "meta", fmt.Sprintf("%d", metaSize)}, - {rootfsPath, rootfsHash, "root", fmt.Sprintf("%d", rootfsSize)}} - } + // Add the deltas + for _, delta := range deltas { + srcImage, ok := product.Versions[delta.DeltaBase] + if !ok { + // Delta for a since expired image + continue + } - // Add the deltas - for _, delta := range deltas { - srcImage, ok := product.Versions[delta.DeltaBase] - if !ok { - continue - } + // Locate source image fingerprint + var srcFingerprint string + for _, item := range srcImage.Items { + if item.FileType != "lxd.tar.xz" { + continue + } - var srcFingerprint string - for _, item := range srcImage.Items { - if item.FileType != "lxd.tar.xz" { + srcFingerprint = item.LXDHashSha256SquashFs + break + } + + if srcFingerprint == "" { + // Couldn't find the image continue } - srcFingerprint = item.LXDHashSha256SquashFs - break + // Add the delta + imgDownloads = append(imgDownloads, []string{ + delta.Path, + delta.HashSha256, + fmt.Sprintf("root.delta-%s", srcFingerprint), + fmt.Sprintf("%d", delta.Size)}) } - if srcFingerprint == "" { - continue - } + // Add the image + downloads[fingerprint] = imgDownloads + images = append(images, image) - imgDownloads = append(imgDownloads, []string{ - delta.Path, - delta.HashSha256, - fmt.Sprintf("root.delta-%s", srcFingerprint), - fmt.Sprintf("%d", delta.Size)}) + return nil } - downloads[fingerprint] = imgDownloads - images = append(images, image) + // Locate a valid LXD image + for _, item := range version.Items { + if item.FileType == "lxd_combined.tar.gz" { + err := addImage(&item, nil) + if err != nil { + continue + } + } + + if item.FileType == "lxd.tar.xz" { + // Locate the root files + for _, subItem := range version.Items { + if shared.StringInSlice(subItem.FileType, []string{"root.tar.xz", "squashfs"}) { + err := addImage(&item, &subItem) + if err != nil { + continue + } + } + } + } + } } } diff --git a/shared/simplestreams/sort.go b/shared/simplestreams/sort.go index bb62abb422..78872e4e7d 100644 --- a/shared/simplestreams/sort.go +++ b/shared/simplestreams/sort.go @@ -16,6 +16,12 @@ func (a sortedImages) Swap(i, j int) { } func (a sortedImages) Less(i, j int) bool { + if a[i].Properties["type"] != a[j].Properties["type"] { + if a[i].Properties["type"] == "squashfs" { + return true + } + } + if a[i].Properties["os"] == a[j].Properties["os"] { if a[i].Properties["release"] == a[j].Properties["release"] { if !shared.TimeIsSet(a[i].CreatedAt) {
_______________________________________________ lxc-devel mailing list lxc-devel@lists.linuxcontainers.org http://lists.linuxcontainers.org/listinfo/lxc-devel