The following pull request was submitted through Github. It can be accessed and reviewed at: https://github.com/lxc/lxd/pull/6806
This e-mail was sent by the LXC bot, direct replies will not reach the author unless they happen to be subscribed to this list. === Description (from pull-request) ===
From 27d49a01758945e477f4b379d9469ef3a381e7fc Mon Sep 17 00:00:00 2001 From: Free Ekanayaka <free.ekanay...@canonical.com> Date: Wed, 29 Jan 2020 13:13:45 +0000 Subject: [PATCH 1/5] api: Add api_filtering extension Signed-off-by: Free Ekanayaka <free.ekanay...@canonical.com> --- doc/api-extensions.md | 3 +++ shared/version/api.go | 1 + 2 files changed, 4 insertions(+) diff --git a/doc/api-extensions.md b/doc/api-extensions.md index a5dad365ea..08916acc3b 100644 --- a/doc/api-extensions.md +++ b/doc/api-extensions.md @@ -902,3 +902,6 @@ This adds the ability to use LVM stripes on normal volumes and thin pool volumes ## vm\_boot\_priority Adds a `boot.priority` property on nic and disk devices to control the boot order. + +## api\_filtering +Adds support for filtering the result of a GET request for instances and images. diff --git a/shared/version/api.go b/shared/version/api.go index b06864911b..2dc3ee9dab 100644 --- a/shared/version/api.go +++ b/shared/version/api.go @@ -184,6 +184,7 @@ var APIExtensions = []string{ "resources_disk_id", "storage_lvm_stripes", "vm_boot_priority", + "api_filtering", } // APIExtensionsCount returns the number of available API extensions. From 8033d75c664277c8881ffd635e9c6d215b971abe Mon Sep 17 00:00:00 2001 From: Free Ekanayaka <free.ekanay...@canonical.com> Date: Wed, 29 Jan 2020 12:25:15 +0000 Subject: [PATCH 2/5] lxd/filter: Add API filtering package Signed-off-by: Free Ekanayaka <free.ekanay...@canonical.com> --- lxd/filter/clause.go | 89 +++++++++++++++++++++++++++++++++++++++ lxd/filter/clause_test.go | 47 +++++++++++++++++++++ lxd/filter/doc.go | 3 ++ lxd/filter/match.go | 31 ++++++++++++++ lxd/filter/match_test.go | 53 +++++++++++++++++++++++ lxd/filter/value.go | 59 ++++++++++++++++++++++++++ lxd/filter/value_test.go | 52 +++++++++++++++++++++++ 7 files changed, 334 insertions(+) create mode 100644 lxd/filter/clause.go create mode 100644 lxd/filter/clause_test.go create mode 100644 lxd/filter/doc.go create mode 100644 lxd/filter/match.go create mode 100644 lxd/filter/match_test.go create mode 100644 lxd/filter/value.go create mode 100644 lxd/filter/value_test.go diff --git a/lxd/filter/clause.go b/lxd/filter/clause.go new file mode 100644 index 0000000000..4b6d0a836b --- /dev/null +++ b/lxd/filter/clause.go @@ -0,0 +1,89 @@ +package filter + +import ( + "fmt" + "strings" + + "github.com/lxc/lxd/shared" +) + +// Clause is a single filter clause in a filter string. +type Clause struct { + PrevLogical string + Not bool + Field string + Operator string + Value string +} + +// Parse a user-provided filter string. +func Parse(s string) ([]Clause, error) { + clauses := []Clause{} + + parts := strings.Fields(s) + + index := 0 + prevLogical := "and" + + for index < len(parts) { + clause := Clause{} + + if strings.EqualFold(parts[index], "not") { + clause.Not = true + index++ + if index == len(parts) { + return nil, fmt.Errorf("incomplete not clause") + } + } else { + clause.Not = false + } + + clause.Field = parts[index] + + index++ + if index == len(parts) { + return nil, fmt.Errorf("clause has no operator") + } + clause.Operator = parts[index] + + index++ + if index == len(parts) { + return nil, fmt.Errorf("clause has no value") + } + value := parts[index] + + // support strings with spaces that are quoted + if strings.HasPrefix(value, "\"") { + value = value[1:] + for { + index++ + if index == len(parts) { + return nil, fmt.Errorf("unterminated quote") + } + if strings.HasSuffix(parts[index], "\"") { + break + } + value += " " + parts[index] + } + end := parts[index] + value += " " + end[0:len(end)-1] + } + clause.Value = value + index++ + + clause.PrevLogical = prevLogical + if index < len(parts) { + prevLogical = parts[index] + if !shared.StringInSlice(prevLogical, []string{"and", "or"}) { + return nil, fmt.Errorf("invalid clause composition") + } + index++ + if index == len(parts) { + return nil, fmt.Errorf("unterminated compound clause") + } + } + clauses = append(clauses, clause) + } + + return clauses, nil +} diff --git a/lxd/filter/clause_test.go b/lxd/filter/clause_test.go new file mode 100644 index 0000000000..0cfa73f5f9 --- /dev/null +++ b/lxd/filter/clause_test.go @@ -0,0 +1,47 @@ +package filter_test + +import ( + "testing" + + "github.com/lxc/lxd/lxd/filter" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestParse_Error(t *testing.T) { + cases := map[string]string{ + "not": "incomplete not clause", + "foo": "clause has no operator", + "not foo": "clause has no operator", + "foo eq": "clause has no value", + "foo eq \"bar": "unterminated quote", + "foo eq bar and": "unterminated compound clause", + "foo eq \"bar egg\" and": "unterminated compound clause", + "foo eq bar xxx": "invalid clause composition", + } + for s, message := range cases { + t.Run(s, func(t *testing.T) { + clauses, err := filter.Parse(s) + assert.Nil(t, clauses) + assert.EqualError(t, err, message) + }) + } +} + +func TestParse(t *testing.T) { + clauses, err := filter.Parse("foo eq \"bar egg\" or not baz eq yuk") + require.NoError(t, err) + assert.Len(t, clauses, 2) + clause1 := clauses[0] + clause2 := clauses[1] + assert.False(t, clause1.Not) + assert.Equal(t, "and", clause1.PrevLogical) + assert.Equal(t, "foo", clause1.Field) + assert.Equal(t, "eq", clause1.Operator) + assert.Equal(t, "bar egg", clause1.Value) + assert.True(t, clause2.Not) + assert.Equal(t, "baz", clause2.Field) + assert.Equal(t, "or", clause2.PrevLogical) + assert.Equal(t, "eq", clause2.Operator) + assert.Equal(t, "yuk", clause2.Value) +} diff --git a/lxd/filter/doc.go b/lxd/filter/doc.go new file mode 100644 index 0000000000..9f13a18aed --- /dev/null +++ b/lxd/filter/doc.go @@ -0,0 +1,3 @@ +// API filtering. + +package filter diff --git a/lxd/filter/match.go b/lxd/filter/match.go new file mode 100644 index 0000000000..99525f284a --- /dev/null +++ b/lxd/filter/match.go @@ -0,0 +1,31 @@ +package filter + +// Match returns true if the given object matches the given filter. +func Match(obj interface{}, clauses []Clause) bool { + match := true + + for _, clause := range clauses { + value := ValueOf(obj, clause.Field) + clauseMatch := value == clause.Value + + if clause.Operator == "ne" { + clauseMatch = !clauseMatch + } + + // Finish out logic + if clause.Not { + clauseMatch = !clauseMatch + } + + switch clause.PrevLogical { + case "and": + match = match && clauseMatch + case "or": + match = match || clauseMatch + default: + panic("unexpected clause operator") + } + } + + return match +} diff --git a/lxd/filter/match_test.go b/lxd/filter/match_test.go new file mode 100644 index 0000000000..7c5c89f848 --- /dev/null +++ b/lxd/filter/match_test.go @@ -0,0 +1,53 @@ +package filter_test + +import ( + "testing" + "time" + + "github.com/lxc/lxd/lxd/filter" + "github.com/lxc/lxd/shared/api" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestMatch_Instance(t *testing.T) { + instance := api.Instance{ + InstancePut: api.InstancePut{ + Architecture: "x86_64", + Config: map[string]string{ + "image.os": "Busybox", + }, + Stateful: false, + }, + CreatedAt: time.Date(2020, 1, 29, 11, 10, 32, 0, time.UTC), + Name: "c1", + ExpandedConfig: map[string]string{ + "image.os": "Busybox", + }, + ExpandedDevices: map[string]map[string]string{ + "root": { + "path": "/", + "pool": "default", + "type": "disk", + }, + }, + Status: "Running", + } + cases := map[string]interface{}{ + "architecture eq x86_64": true, + "architecture eq i686": false, + "name eq c1 and status eq Running": true, + "config.image.os eq Busybox and expanded_devices.root.path eq /": true, + "name eq c2 or status eq Running": true, + "name eq c2 or name eq c3": false, + } + for s := range cases { + t.Run(s, func(t *testing.T) { + f, err := filter.Parse(s) + require.NoError(t, err) + match := filter.Match(instance, f) + assert.Equal(t, cases[s], match) + }) + } + +} diff --git a/lxd/filter/value.go b/lxd/filter/value.go new file mode 100644 index 0000000000..70b8ecaebb --- /dev/null +++ b/lxd/filter/value.go @@ -0,0 +1,59 @@ +package filter + +import ( + "reflect" + "strings" +) + +// ValueOf returns the value of the given field. +func ValueOf(obj interface{}, field string) interface{} { + value := reflect.ValueOf(obj) + typ := value.Type() + parts := strings.Split(field, ".") + + key := parts[0] + rest := strings.Join(parts[1:], ".") + + var parent interface{} + + if value.Kind() == reflect.Map { + switch reflect.TypeOf(obj).Elem().Kind() { + case reflect.String: + m := value.Interface().(map[string]string) + return m[field] + case reflect.Map: + for _, entry := range value.MapKeys() { + if entry.Interface() != key { + continue + } + m := value.MapIndex(entry) + return ValueOf(m.Interface(), rest) + } + } + return nil + } + + for i := 0; i < value.NumField(); i++ { + fieldValue := value.Field(i) + fieldType := typ.Field(i) + yaml := fieldType.Tag.Get("yaml") + + if yaml == ",inline" { + parent = fieldValue.Interface() + } + + if yaml == key { + v := fieldValue.Interface() + if len(parts) == 1 { + return v + } + return ValueOf(v, rest) + } + } + + if parent != nil { + return ValueOf(parent, field) + } + + return nil +} diff --git a/lxd/filter/value_test.go b/lxd/filter/value_test.go new file mode 100644 index 0000000000..5682f65195 --- /dev/null +++ b/lxd/filter/value_test.go @@ -0,0 +1,52 @@ +package filter_test + +import ( + "testing" + "time" + + "github.com/lxc/lxd/lxd/filter" + "github.com/lxc/lxd/shared/api" + "github.com/stretchr/testify/assert" +) + +func TestValueOf_Instance(t *testing.T) { + instance := api.Instance{ + InstancePut: api.InstancePut{ + Architecture: "x86_64", + Config: map[string]string{ + "image.os": "Busybox", + }, + Stateful: false, + }, + CreatedAt: time.Date(2020, 1, 29, 11, 10, 32, 0, time.UTC), + Name: "c1", + ExpandedConfig: map[string]string{ + "image.os": "Busybox", + }, + ExpandedDevices: map[string]map[string]string{ + "root": { + "path": "/", + "pool": "default", + "type": "disk", + }, + }, + Status: "Running", + } + cases := map[string]interface{}{ + "architecture": "x86_64", + "created_at": time.Date(2020, 1, 29, 11, 10, 32, 0, time.UTC), + "config.image.os": "Busybox", + "name": "c1", + "expanded_config.image.os": "Busybox", + "expanded_devices.root.pool": "default", + "status": "Running", + "stateful": false, + } + for field := range cases { + t.Run(field, func(t *testing.T) { + value := filter.ValueOf(instance, field) + assert.Equal(t, cases[field], value) + }) + } + +} From 1724463ec294ea350b2186f23b79e6c4e7bc73bf Mon Sep 17 00:00:00 2001 From: Free Ekanayaka <free.ekanay...@canonical.com> Date: Wed, 29 Jan 2020 13:11:43 +0000 Subject: [PATCH 3/5] lxd/instance: Add instance list filtering functions Signed-off-by: Free Ekanayaka <free.ekanay...@canonical.com> --- lxd/instance/filter.go | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) create mode 100644 lxd/instance/filter.go diff --git a/lxd/instance/filter.go b/lxd/instance/filter.go new file mode 100644 index 0000000000..495f6717b5 --- /dev/null +++ b/lxd/instance/filter.go @@ -0,0 +1,30 @@ +package instance + +import ( + "github.com/lxc/lxd/lxd/filter" + "github.com/lxc/lxd/shared/api" +) + +// Filter returns a filtered list of instances that match the given clauses. +func Filter(instances []*api.Instance, clauses []filter.Clause) []*api.Instance { + filtered := []*api.Instance{} + for _, instance := range instances { + if !filter.Match(*instance, clauses) { + continue + } + filtered = append(filtered, instance) + } + return filtered +} + +// FilterFull returns a filtered list of full instances that match the given clauses. +func FilterFull(instances []*api.InstanceFull, clauses []filter.Clause) []*api.InstanceFull { + filtered := []*api.InstanceFull{} + for _, instance := range instances { + if !filter.Match(*instance, clauses) { + continue + } + filtered = append(filtered, instance) + } + return filtered +} From e3ef0353c87cde52bfa4112e1ddc31941772fe0c Mon Sep 17 00:00:00 2001 From: Free Ekanayaka <free.ekanay...@canonical.com> Date: Wed, 29 Jan 2020 13:12:58 +0000 Subject: [PATCH 4/5] lxd: Make use of filtering for instances and images Signed-off-by: Free Ekanayaka <free.ekanay...@canonical.com> --- lxd/containers_get.go | 44 +++++++++++++++++++++++++++++++++++-------- lxd/images.go | 21 +++++++++++++++------ 2 files changed, 51 insertions(+), 14 deletions(-) diff --git a/lxd/containers_get.go b/lxd/containers_get.go index 5fe0015cf3..2c7635d093 100644 --- a/lxd/containers_get.go +++ b/lxd/containers_get.go @@ -15,6 +15,7 @@ import ( "github.com/lxc/lxd/lxd/cluster" "github.com/lxc/lxd/lxd/db" "github.com/lxc/lxd/lxd/db/query" + "github.com/lxc/lxd/lxd/filter" "github.com/lxc/lxd/lxd/instance" "github.com/lxc/lxd/lxd/instance/instancetype" "github.com/lxc/lxd/lxd/response" @@ -82,6 +83,16 @@ func doContainersGet(d *Daemon, r *http.Request) (interface{}, error) { recursion = 0 } + // Parse filter value + filterStr := r.FormValue("filter") + var clauses []filter.Clause + if filterStr != "" { + clauses, err = filter.Parse(filterStr) + if err != nil { + return nil, errors.Wrap(err, "Invalid filter") + } + } + // Parse the project field project := projectParam(r) @@ -109,7 +120,8 @@ func doContainersGet(d *Daemon, r *http.Request) (interface{}, error) { // Get the local instances nodeCts := map[string]instance.Instance{} - if recursion > 0 { + mustLoadObjects := recursion > 0 || (recursion == 0 && clauses != nil) + if mustLoadObjects { cts, err := instanceLoadNodeProjectAll(d.State(), project, instanceType) if err != nil { return nil, err @@ -160,9 +172,9 @@ func doContainersGet(d *Daemon, r *http.Request) (interface{}, error) { } // Mark containers on unavailable nodes as down - if recursion > 0 && address == "0.0.0.0" { + if mustLoadObjects && address == "0.0.0.0" { for _, container := range containers { - if recursion == 1 { + if recursion < 2 { resultListAppend(container, api.Instance{}, fmt.Errorf("unavailable")) } else { resultFullListAppend(container, api.InstanceFull{}, fmt.Errorf("unavailable")) @@ -174,7 +186,7 @@ func doContainersGet(d *Daemon, r *http.Request) (interface{}, error) { // For recursion requests we need to fetch the state of remote // containers from their respective nodes. - if recursion > 0 && address != "" && !isClusterNotification(r) { + if mustLoadObjects && address != "" && !isClusterNotification(r) { wg.Add(1) go func(address string, containers []string) { defer wg.Done() @@ -213,8 +225,7 @@ func doContainersGet(d *Daemon, r *http.Request) (interface{}, error) { continue } - - if recursion == 0 { + if !mustLoadObjects { for _, container := range containers { instancePath := "instances" if strings.HasPrefix(mux.CurrentRoute(r).GetName(), "container") { @@ -243,7 +254,7 @@ func doContainersGet(d *Daemon, r *http.Request) (interface{}, error) { break } - if recursion == 1 { + if recursion < 2 { c, _, err := nodeCts[container].Render() if err != nil { resultListAppend(container, api.Instance{}, err) @@ -276,6 +287,18 @@ func doContainersGet(d *Daemon, r *http.Request) (interface{}, error) { wg.Wait() if recursion == 0 { + if clauses != nil { + for _, container := range instance.Filter(resultList, clauses) { + instancePath := "instances" + if strings.HasPrefix(mux.CurrentRoute(r).GetName(), "container") { + instancePath = "containers" + } else if strings.HasPrefix(mux.CurrentRoute(r).GetName(), "vm") { + instancePath = "virtual-machines" + } + url := fmt.Sprintf("/%s/%s/%s", version.APIVersion, instancePath, container.Name) + resultString = append(resultString, url) + } + } return resultString, nil } @@ -284,7 +307,9 @@ func doContainersGet(d *Daemon, r *http.Request) (interface{}, error) { sort.Slice(resultList, func(i, j int) bool { return resultList[i].Name < resultList[j].Name }) - + if clauses != nil { + resultList = instance.Filter(resultList, clauses) + } return resultList, nil } @@ -293,6 +318,9 @@ func doContainersGet(d *Daemon, r *http.Request) (interface{}, error) { return resultFullList[i].Name < resultFullList[j].Name }) + if clauses != nil { + resultFullList = instance.FilterFull(resultFullList, clauses) + } return resultFullList, nil } diff --git a/lxd/images.go b/lxd/images.go index 277e3e00f6..78f1ea4ca0 100644 --- a/lxd/images.go +++ b/lxd/images.go @@ -916,12 +916,16 @@ func getImageMetadata(fname string) (*api.ImageMetadata, string, error) { return &result, imageType, nil } -func doImagesGet(d *Daemon, recursion bool, project string, public bool) (interface{}, error) { +func doImagesGet(d *Daemon, recursion bool, project string, public bool, filterStr string) (interface{}, error) { results, err := d.cluster.ImagesGet(project, public) if err != nil { return []string{}, err } + if filterStr != "" { + recursion = true + } + resultString := make([]string, len(results)) resultMap := make([]*api.Image, len(results)) i := 0 @@ -930,7 +934,7 @@ func doImagesGet(d *Daemon, recursion bool, project string, public bool) (interf url := fmt.Sprintf("/%s/images/%s", version.APIVersion, name) resultString[i] = url } else { - image, response := doImageGet(d.cluster, project, name, public) + image, response := doImageGet(d.cluster, project, name, public, filterStr) if response != nil { continue } @@ -943,15 +947,18 @@ func doImagesGet(d *Daemon, recursion bool, project string, public bool) (interf if !recursion { return resultString, nil } - + if filterStr != "" { + panic("TODO") + } return resultMap, nil } func imagesGet(d *Daemon, r *http.Request) response.Response { project := projectParam(r) + filter := r.FormValue("filter") public := d.checkTrustedClient(r) != nil || AllowProjectPermission("images", "view")(d, r) != response.EmptySyncResponse - result, err := doImagesGet(d, util.IsRecursionRequest(r), project, public) + result, err := doImagesGet(d, util.IsRecursionRequest(r), project, public, filter) if err != nil { return response.SmartError(err) } @@ -1516,8 +1523,9 @@ func imageDeleteFromDisk(fingerprint string) { } } -func doImageGet(db *db.Cluster, project, fingerprint string, public bool) (*api.Image, response.Response) { +func doImageGet(db *db.Cluster, project, fingerprint string, public bool, filter string) (*api.Image, response.Response) { _, imgInfo, err := db.ImageGet(project, fingerprint, public, false) + if err != nil { return nil, response.SmartError(err) } @@ -1560,8 +1568,9 @@ func imageGet(d *Daemon, r *http.Request) response.Response { fingerprint := mux.Vars(r)["fingerprint"] public := d.checkTrustedClient(r) != nil || AllowProjectPermission("images", "view")(d, r) != response.EmptySyncResponse secret := r.FormValue("secret") + filter := r.FormValue("filter") - info, resp := doImageGet(d.cluster, project, fingerprint, false) + info, resp := doImageGet(d.cluster, project, fingerprint, false, filter) if resp != nil { return resp } From 81d8b65afa11f81b676eb55a5d1c7d312e4ffa85 Mon Sep 17 00:00:00 2001 From: Free Ekanayaka <free.ekanay...@canonical.com> Date: Wed, 29 Jan 2020 13:13:54 +0000 Subject: [PATCH 5/5] doc/rest-api: Document filtering Signed-off-by: Free Ekanayaka <free.ekanay...@canonical.com> --- doc/rest-api.md | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/doc/rest-api.md b/doc/rest-api.md index 81012c0b3b..716274c630 100644 --- a/doc/rest-api.md +++ b/doc/rest-api.md @@ -157,6 +157,37 @@ they point to (typically a dict). Recursion is implemented by simply replacing any pointer to an job (URL) by the object itself. +## Filtering +To filter your results on certain values, filter is implemented for collections. +A `filter` argument can be passed to a GET query against a collection. + +Filtering is available for the instance and image endpoints. + +There is no default value for filter which means that all results found will +be returned. The following is the language used for the filter argument: + +?filter=field_name eq desired_field_assignment + +The language follows the OData conventions for structuring REST API filtering +logic. Logical operators are also supported for filtering: not(not), equals(eq), +not equals(ne), and(and), or(or). Filters are evaluated with left associativity. +Values with spaces can be surrounded with quotes. Nesting filtering is also supported. +For instance, to filter on a field in a config you would pass: + +?filter=config.field_name eq desired_field_assignment + +For filtering on device attributes you would pass: + +?filter=devices.device_name.field_name eq desired_field_assignment + +Here are a few GET query examples of the different filtering methods mentioned above: + +containers?filter=name eq "my container" and status eq Running + +containers?filter=config.image.os eq ubuntu or devices.eth0.nictype eq bridged + +images?filter=Properties.os eq Centos and not UpdateSource.Protocol eq simplestreams + ## Async operations Any operation which may take more than a second to be done must be done in the background, returning a background operation ID to the client.
_______________________________________________ lxc-devel mailing list lxc-devel@lists.linuxcontainers.org http://lists.linuxcontainers.org/listinfo/lxc-devel