This is an automated email from the ASF dual-hosted git repository.
spacewander pushed a commit to branch next
in repository https://gitbox.apache.org/repos/asf/apisix.git
The following commit(s) were added to refs/heads/next by this push:
new 8682493b6 feat: support paging to get a list of objects (#7379)
8682493b6 is described below
commit 8682493b6993c015fc54f19917f3398d868e1eb0
Author: tzssangglass <[email protected]>
AuthorDate: Wed Jul 6 13:50:27 2022 +0800
feat: support paging to get a list of objects (#7379)
---
apisix/admin/consumers.lua | 2 +
apisix/admin/global_rules.lua | 2 +
apisix/admin/plugin_config.lua | 2 +
apisix/admin/proto.lua | 2 +
apisix/admin/routes.lua | 2 +
apisix/admin/services.lua | 2 +
apisix/admin/ssl.lua | 2 +
apisix/admin/stream_routes.lua | 2 +
apisix/admin/upstreams.lua | 2 +
apisix/admin/v3_adapter.lua | 125 +++++++++++++++++++++
apisix/core/etcd.lua | 64 ++---------
t/admin/filter.t | 248 +++++++++++++++++++++++++++++++++++++++++
12 files changed, 398 insertions(+), 57 deletions(-)
diff --git a/apisix/admin/consumers.lua b/apisix/admin/consumers.lua
index 46b23de09..e9456a31b 100644
--- a/apisix/admin/consumers.lua
+++ b/apisix/admin/consumers.lua
@@ -18,6 +18,7 @@ local core = require("apisix.core")
local plugins = require("apisix.admin.plugins")
local utils = require("apisix.admin.utils")
local plugin = require("apisix.plugin")
+local v3_adapter = require("apisix.admin.v3_adapter")
local pairs = pairs
local _M = {
@@ -102,6 +103,7 @@ function _M.get(consumer_name)
end
utils.fix_count(res.body, consumer_name)
+ v3_adapter.filter(res.body)
return res.status, res.body
end
diff --git a/apisix/admin/global_rules.lua b/apisix/admin/global_rules.lua
index c4dd4ca93..5cec0604f 100644
--- a/apisix/admin/global_rules.lua
+++ b/apisix/admin/global_rules.lua
@@ -17,6 +17,7 @@
local core = require("apisix.core")
local utils = require("apisix.admin.utils")
local schema_plugin = require("apisix.admin.plugins").check_schema
+local v3_adapter = require("apisix.admin.v3_adapter")
local type = type
local tostring = tostring
@@ -97,6 +98,7 @@ function _M.get(id)
end
utils.fix_count(res.body, id)
+ v3_adapter.filter(res.body)
return res.status, res.body
end
diff --git a/apisix/admin/plugin_config.lua b/apisix/admin/plugin_config.lua
index bcf199fcd..06c100cdf 100644
--- a/apisix/admin/plugin_config.lua
+++ b/apisix/admin/plugin_config.lua
@@ -18,6 +18,7 @@ local core = require("apisix.core")
local get_routes = require("apisix.router").http_routes
local utils = require("apisix.admin.utils")
local schema_plugin = require("apisix.admin.plugins").check_schema
+local v3_adapter = require("apisix.admin.v3_adapter")
local type = type
local tostring = tostring
local ipairs = ipairs
@@ -97,6 +98,7 @@ function _M.get(id)
end
utils.fix_count(res.body, id)
+ v3_adapter.filter(res.body)
return res.status, res.body
end
diff --git a/apisix/admin/proto.lua b/apisix/admin/proto.lua
index 132db68a1..abe161f07 100644
--- a/apisix/admin/proto.lua
+++ b/apisix/admin/proto.lua
@@ -21,6 +21,7 @@ local utils = require("apisix.admin.utils")
local get_routes = require("apisix.router").http_routes
local get_services = require("apisix.http.service").services
local compile_proto =
require("apisix.plugins.grpc-transcode.proto").compile_proto
+local v3_adapter = require("apisix.admin.v3_adapter")
local tostring = tostring
@@ -99,6 +100,7 @@ function _M.get(id)
end
utils.fix_count(res.body, id)
+ v3_adapter.filter(res.body)
return res.status, res.body
end
diff --git a/apisix/admin/routes.lua b/apisix/admin/routes.lua
index 877f6cf5e..ccfe0bf95 100644
--- a/apisix/admin/routes.lua
+++ b/apisix/admin/routes.lua
@@ -19,6 +19,7 @@ local core = require("apisix.core")
local apisix_upstream = require("apisix.upstream")
local schema_plugin = require("apisix.admin.plugins").check_schema
local utils = require("apisix.admin.utils")
+local v3_adapter = require("apisix.admin.v3_adapter")
local tostring = tostring
local type = type
local loadstring = loadstring
@@ -203,6 +204,7 @@ function _M.get(id)
end
utils.fix_count(res.body, id)
+ v3_adapter.filter(res.body)
return res.status, res.body
end
diff --git a/apisix/admin/services.lua b/apisix/admin/services.lua
index 59c53eec3..1872d5bb7 100644
--- a/apisix/admin/services.lua
+++ b/apisix/admin/services.lua
@@ -19,6 +19,7 @@ local get_routes = require("apisix.router").http_routes
local apisix_upstream = require("apisix.upstream")
local schema_plugin = require("apisix.admin.plugins").check_schema
local utils = require("apisix.admin.utils")
+local v3_adapter = require("apisix.admin.v3_adapter")
local tostring = tostring
local ipairs = ipairs
local type = type
@@ -146,6 +147,7 @@ function _M.get(id)
end
utils.fix_count(res.body, id)
+ v3_adapter.filter(res.body)
return res.status, res.body
end
diff --git a/apisix/admin/ssl.lua b/apisix/admin/ssl.lua
index 9a73107c9..70d868dc9 100644
--- a/apisix/admin/ssl.lua
+++ b/apisix/admin/ssl.lua
@@ -17,6 +17,7 @@
local core = require("apisix.core")
local utils = require("apisix.admin.utils")
local apisix_ssl = require("apisix.ssl")
+local v3_adapter = require("apisix.admin.v3_adapter")
local tostring = tostring
local type = type
@@ -107,6 +108,7 @@ function _M.get(id)
end
utils.fix_count(res.body, id)
+ v3_adapter.filter(res.body)
return res.status, res.body
end
diff --git a/apisix/admin/stream_routes.lua b/apisix/admin/stream_routes.lua
index 6770830ac..625911c04 100644
--- a/apisix/admin/stream_routes.lua
+++ b/apisix/admin/stream_routes.lua
@@ -17,6 +17,7 @@
local core = require("apisix.core")
local utils = require("apisix.admin.utils")
local stream_route_checker =
require("apisix.stream.router.ip_port").stream_route_checker
+local v3_adapter = require("apisix.admin.v3_adapter")
local tostring = tostring
@@ -114,6 +115,7 @@ function _M.get(id)
end
utils.fix_count(res.body, id)
+ v3_adapter.filter(res.body)
return res.status, res.body
end
diff --git a/apisix/admin/upstreams.lua b/apisix/admin/upstreams.lua
index 5aec65269..d262f3977 100644
--- a/apisix/admin/upstreams.lua
+++ b/apisix/admin/upstreams.lua
@@ -19,6 +19,7 @@ local get_routes = require("apisix.router").http_routes
local get_services = require("apisix.http.service").services
local apisix_upstream = require("apisix.upstream")
local utils = require("apisix.admin.utils")
+local v3_adapter = require("apisix.admin.v3_adapter")
local tostring = tostring
local ipairs = ipairs
local type = type
@@ -99,6 +100,7 @@ function _M.get(id)
end
utils.fix_count(res.body, id)
+ v3_adapter.filter(res.body)
return res.status, res.body
end
diff --git a/apisix/admin/v3_adapter.lua b/apisix/admin/v3_adapter.lua
new file mode 100644
index 000000000..f55df73a4
--- /dev/null
+++ b/apisix/admin/v3_adapter.lua
@@ -0,0 +1,125 @@
+--
+-- Licensed to the Apache Software Foundation (ASF) under one or more
+-- contributor license agreements. See the NOTICE file distributed with
+-- this work for additional information regarding copyright ownership.
+-- The ASF licenses this file to You under the Apache License, Version 2.0
+-- (the "License"); you may not use this file except in compliance with
+-- the License. You may obtain a copy of the License at
+--
+-- http://www.apache.org/licenses/LICENSE-2.0
+--
+-- Unless required by applicable law or agreed to in writing, software
+-- distributed under the License is distributed on an "AS IS" BASIS,
+-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+-- See the License for the specific language governing permissions and
+-- limitations under the License.
+--
+
+local fetch_local_conf = require("apisix.core.config_local").local_conf
+local try_read_attr = require("apisix.core.table").try_read_attr
+local log = require("apisix.core.log")
+local request = require("apisix.core.request")
+local response = require("apisix.core.response")
+local table = require("apisix.core.table")
+local tonumber = tonumber
+
+local _M = {}
+
+
+local admin_api_version
+local function enable_v3()
+ if admin_api_version then
+ if admin_api_version == "v3" then
+ return true
+ end
+
+ if admin_api_version == "default" then
+ return false
+ end
+ end
+
+ local local_conf, err = fetch_local_conf()
+ if not local_conf then
+ admin_api_version = "default"
+ log.error("failed to fetch local conf: ", err)
+ return false
+ end
+
+ local api_ver = try_read_attr(local_conf, "apisix", "admin_api_version")
+ if api_ver ~= "v3" then
+ admin_api_version = "default"
+ return false
+ end
+
+ admin_api_version = api_ver
+ return true
+end
+_M.enable_v3 = enable_v3
+
+
+function _M.to_v3(body, action)
+ if not enable_v3() then
+ body.action = action
+ end
+end
+
+
+function _M.to_v3_list(body)
+ if not enable_v3() then
+ return
+ end
+
+ if body.node.dir then
+ body.list = body.node.nodes
+ body.node = nil
+ end
+end
+
+
+local function sort(l, r)
+ return l.createdIndex < r.createdIndex
+end
+
+
+function _M.filter(body)
+ if not enable_v3() then
+ return
+ end
+
+ local args = request.get_uri_args()
+ args.page = tonumber(args.page)
+ args.page_size = tonumber(args.page_size)
+ if not args.page or not args.page_size then
+ return
+ end
+
+ if args.page_size < 10 or args.page_size > 500 then
+ return response.exit(400, "page_size must be between 10 and 500")
+ end
+
+ if not args.page or args.page < 1 then
+ -- default page is 1
+ args.page = 1
+ end
+
+ local list = body.list
+
+ -- sort nodes by there createdIndex
+ table.sort(list, sort)
+
+ local to = args.page * args.page_size
+ local from = to - args.page_size + 1
+
+ local res = table.new(20, 0)
+
+ for i = from, to do
+ if list[i] then
+ res[i - from + 1] = list[i]
+ end
+ end
+
+ body.list = res
+end
+
+
+return _M
diff --git a/apisix/core/etcd.lua b/apisix/core/etcd.lua
index fd9eecc4a..ada913934 100644
--- a/apisix/core/etcd.lua
+++ b/apisix/core/etcd.lua
@@ -21,8 +21,7 @@
local fetch_local_conf = require("apisix.core.config_local").local_conf
local array_mt = require("apisix.core.json").array_mt
-local try_read_attr = require("apisix.core.table").try_read_attr
-local log = require("apisix.core.log")
+local v3_adapter = require("apisix.admin.v3_adapter")
local etcd = require("resty.etcd")
local clone_tab = require("table.clone")
local health_check = require("resty.etcd.health_check")
@@ -37,55 +36,6 @@ local is_http = ngx.config.subsystem == "http"
local _M = {}
-local admin_api_version
-local function is_v3()
- if admin_api_version then
- if admin_api_version == "v3" then
- return true
- end
-
- if admin_api_version == "default" then
- return false
- end
- end
-
- local local_conf, err = fetch_local_conf()
- if not local_conf then
- admin_api_version = "default"
- log.error("failed to fetch local conf: ", err)
- return false
- end
-
- local api_ver = try_read_attr(local_conf, "apisix", "admin_api_version")
- if api_ver ~= "v3" then
- admin_api_version = "default"
- return false
- end
-
- admin_api_version = api_ver
- return true
-end
-
-
-local function to_v3(body, action)
- if not is_v3() then
- body.action = action
- end
-end
-
-
-local function to_v3_list(body)
- if not is_v3() then
- return
- end
-
- if body.node.dir then
- body.list = body.node.nodes
- body.node = nil
- end
-end
-
-
-- this function create the etcd client instance used in the Admin API
local function new()
local local_conf, err = fetch_local_conf()
@@ -219,7 +169,7 @@ function _M.get_format(res, real_key, is_dir, formatter)
return not_found(res)
end
- to_v3(res.body, "get")
+ v3_adapter.to_v3(res.body, "get")
if formatter then
return formatter(res)
@@ -247,7 +197,7 @@ function _M.get_format(res, real_key, is_dir, formatter)
end
res.body.kvs = nil
- to_v3_list(res.body)
+ v3_adapter.to_v3_list(res.body)
return res
end
@@ -324,7 +274,7 @@ local function set(key, value, ttl)
res.headers["X-Etcd-Index"] = res.body.header.revision
-- etcd v3 set would not return kv info
- to_v3(res.body, "set")
+ v3_adapter.to_v3(res.body, "set")
res.body.node = {}
res.body.node.key = prefix .. key
res.body.node.value = value
@@ -387,7 +337,7 @@ function _M.atomic_set(key, value, ttl, mod_revision)
res.headers["X-Etcd-Index"] = res.body.header.revision
-- etcd v3 set would not return kv info
- to_v3(res.body, "compareAndSwap")
+ v3_adapter.to_v3(res.body, "compareAndSwap")
res.body.node = {
key = key,
value = value,
@@ -425,7 +375,7 @@ function _M.push(key, value, ttl)
return nil, err
end
- to_v3(res.body, "create")
+ v3_adapter.to_v3(res.body, "create")
return res, nil
end
@@ -449,7 +399,7 @@ function _M.delete(key)
end
-- etcd v3 set would not return kv info
- to_v3(res.body, "delete")
+ v3_adapter.to_v3(res.body, "delete")
res.body.node = {}
res.body.key = prefix .. key
diff --git a/t/admin/filter.t b/t/admin/filter.t
new file mode 100644
index 000000000..decf463d3
--- /dev/null
+++ b/t/admin/filter.t
@@ -0,0 +1,248 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+use t::APISIX 'no_plan';
+
+repeat_each(1);
+worker_connections(1024);
+no_root_location();
+no_shuffle();
+
+add_block_preprocessor(sub {
+ my ($block) = @_;
+
+ my $user_yaml_config = <<_EOC_;
+apisix:
+ node_listen: 1984
+ admin_key: null
+ admin_api_version: v3
+_EOC_
+ $block->set_value("yaml_config", $user_yaml_config);
+
+ if (!$block->request) {
+ $block->set_value("request", "GET /t");
+ }
+
+ if ((!defined $block->error_log) && (!defined $block->no_error_log)) {
+ $block->set_value("no_error_log", "[error]");
+ }
+});
+
+run_tests();
+
+__DATA__
+
+=== TEST 1: bad page_size(page_size must be between 10 and 500)
+--- config
+ location /t {
+ content_by_lua_block {
+ local t = require("lib.test_admin").test
+
+ ngx.sleep(0.5)
+
+ local code, body = t('/apisix/admin/routes/?page=1&page_size=2',
+ ngx.HTTP_GET
+ )
+ ngx.status = code
+ ngx.say(body)
+ }
+ }
+--- error_code: 400
+--- response_body
+page_size must be between 10 and 500
+
+
+
+=== TEST 2: ignore bad page and would use default value 1
+--- config
+ location /t {
+ content_by_lua_block {
+ local json = require("toolkit.json")
+ local t = require("lib.test_admin").test
+
+ for i = 1, 11 do
+ local code, body = t('/apisix/admin/routes/' .. i,
+ ngx.HTTP_PUT,
+ [[{
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:1980": 1
+ },
+ "type": "roundrobin"
+ },
+ "uri": "/hello]] .. i .. [["
+ }]]
+ )
+ end
+
+ ngx.sleep(0.5)
+
+ local code, body, res =
t('/apisix/admin/routes/?page=-1&page_size=10',
+ ngx.HTTP_GET
+ )
+ res = json.decode(res)
+ assert(#res.list == 10)
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 3: sort by createdIndex
+# the smaller the createdIndex, the higher the ranking
+--- config
+ location /t {
+ content_by_lua_block {
+ local json = require("toolkit.json")
+ local t = require("lib.test_admin").test
+
+ for i = 1, 11 do
+ local code, body = t('/apisix/admin/routes/' .. i,
+ ngx.HTTP_PUT,
+ [[{
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:1980": 1
+ },
+ "type": "roundrobin"
+ },
+ "uri": "/hello]] .. i .. [["
+ }]]
+ )
+ end
+
+ ngx.sleep(0.5)
+
+ local code, body, res =
t('/apisix/admin/routes/?page=1&page_size=10',
+ ngx.HTTP_GET
+ )
+ res = json.decode(res)
+
+ for i = 1, #res.list - 1 do
+ assert(res.list[i].createdIndex < res.list[i + 1].createdIndex)
+ end
+ if code >= 300 then
+ ngx.status = code
+ end
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 4: routes pagination
+--- config
+ location /t {
+ content_by_lua_block {
+ local json = require("toolkit.json")
+ local t = require("lib.test_admin").test
+
+ for i = 1, 11 do
+ local code, body = t('/apisix/admin/routes/' .. i,
+ ngx.HTTP_PUT,
+ [[{
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:1980": 1
+ },
+ "type": "roundrobin"
+ },
+ "uri": "/hello]] .. i .. [["
+ }]]
+ )
+ end
+
+ ngx.sleep(0.5)
+
+ local code, body, res =
t('/apisix/admin/routes/?page=1&page_size=10',
+ ngx.HTTP_GET
+ )
+ res = json.decode(res)
+ assert(#res.list == 10)
+
+ code, body, res = t('/apisix/admin/routes/?page=2&page_size=10',
+ ngx.HTTP_GET
+ )
+ res = json.decode(res)
+ assert(#res.list == 1)
+
+ code, body, res = t('/apisix/admin/routes/?page=3&page_size=10',
+ ngx.HTTP_GET
+ )
+ res = json.decode(res)
+ assert(#res.list == 0)
+
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
+
+
+
+=== TEST 5: services pagination
+--- config
+ location /t {
+ content_by_lua_block {
+ local json = require("toolkit.json")
+ local t = require("lib.test_admin").test
+
+ for i = 1, 11 do
+ local code, body = t('/apisix/admin/services/' .. i,
+ ngx.HTTP_PUT,
+ [[{
+ "upstream": {
+ "nodes": {
+ "127.0.0.1:1980": 1
+ },
+ "type": "roundrobin"
+ }
+ }]]
+ )
+ end
+
+ ngx.sleep(0.5)
+
+ local code, body, res =
t('/apisix/admin/services/?page=1&page_size=10',
+ ngx.HTTP_GET
+ )
+ res = json.decode(res)
+ assert(#res.list == 10)
+
+ code, body, res = t('/apisix/admin/services/?page=2&page_size=10',
+ ngx.HTTP_GET
+ )
+ res = json.decode(res)
+ assert(#res.list == 1)
+
+ code, body, res = t('/apisix/admin/services/?page=3&page_size=10',
+ ngx.HTTP_GET
+ )
+ res = json.decode(res)
+ assert(#res.list == 0)
+
+ ngx.say(body)
+ }
+ }
+--- response_body
+passed
