membphis commented on code in PR #11499: URL: https://github.com/apache/apisix/pull/11499#discussion_r1726400126
########## apisix/plugins/ai-proxy.lua: ########## @@ -0,0 +1,103 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local core = require("apisix.core") +local schema = require("apisix.plugins.ai-proxy.schema") +local constants = require("apisix.constants") +local require = require + +local ngx_req = ngx.req +local ngx = ngx + +local plugin_name = "ai-proxy" +local _M = { + version = 0.5, + priority = 1004, + name = plugin_name, + schema = schema, +} + + +function _M.check_schema(conf) + return core.schema.check(schema.plugin_schema, conf) +end + + +local CONTENT_TYPE_JSON = "application/json" + + +local function get_request_table() + local req_body, err = core.request.get_body() + if not req_body then + return nil, "failed to get request body: " .. (err or "request body is empty") + end + req_body, err = req_body:gsub("\\\"", "\"") -- remove escaping in JSON Review Comment: it seems unsafe. I do not think we need to call this step ########## apisix/plugins/ai-proxy.lua: ########## @@ -0,0 +1,103 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local core = require("apisix.core") +local schema = require("apisix.plugins.ai-proxy.schema") +local constants = require("apisix.constants") +local require = require + +local ngx_req = ngx.req +local ngx = ngx + +local plugin_name = "ai-proxy" +local _M = { + version = 0.5, + priority = 1004, + name = plugin_name, + schema = schema, +} + + +function _M.check_schema(conf) + return core.schema.check(schema.plugin_schema, conf) +end + + +local CONTENT_TYPE_JSON = "application/json" + + +local function get_request_table() + local req_body, err = core.request.get_body() + if not req_body then + return nil, "failed to get request body: " .. (err or "request body is empty") + end + req_body, err = req_body:gsub("\\\"", "\"") -- remove escaping in JSON + if not req_body then + return nil, "failed to remove escaping from body: " .. req_body .. ". err: " .. err + end + return core.json.decode(req_body) +end + +function _M.access(conf, ctx) + local route_type = conf.route_type + ctx.ai_proxy = {} + + local content_type = core.request.header(ctx, "Content-Type") or CONTENT_TYPE_JSON + if content_type ~= CONTENT_TYPE_JSON then + return 400, "unsupported content-type: " .. content_type + end + + local request_table, err = get_request_table() + if not request_table then + return 400, err + end + + local req_schema = schema.chat_request_schema + if route_type == constants.COMPLETION then + req_schema = schema.chat_completion_request_schema + end + local ok, err = core.schema.check(req_schema, request_table) + if not ok then + return 400, "request format doesn't match schema: " .. err + end + + if conf.model.options and conf.model.options.response_streaming then + request_table.stream = true + ngx.ctx.disable_proxy_buffering = true + end + + if conf.model.name then + request_table.model = conf.model.name + end + + local ai_driver = require("apisix.plugins.ai-proxy.drivers." .. conf.model.provider) Review Comment: it may fail, we need to check the return value ########## apisix/plugins/ai-proxy/drivers/openai.lua: ########## @@ -0,0 +1,67 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local _M = {} + +local core = require("apisix.core") +local test_scheme = os.getenv("AI_PROXY_TEST_SCHEME") +local ngx = ngx +local pairs = pairs + +-- globals +local DEFAULT_HOST = "api.openai.com" +local DEFAULT_PORT = 443 + +local path_mapper = { + ["llm/completions"] = "/v1/completions", + ["llm/chat"] = "/v1/chat/completions", +} + + +function _M.configure_request(conf, request_table, ctx) + local ip, err = core.resolver.parse_domain(conf.model.options.upstream_host or DEFAULT_HOST) + if not ip then + core.log.error("failed to resolve ai_proxy upstream host: ", err) + return core.response.exit(500) + end + ctx.custom_upstream_ip = ip + ctx.custom_upstream_port = conf.model.options.upstream_port or DEFAULT_PORT + + local ups_path = (conf.model.options and conf.model.options.upstream_path) + or path_mapper[conf.route_type] + ngx.var.upstream_uri = ups_path + ngx.var.upstream_scheme = test_scheme or "https" + ngx.req.set_method(ngx.HTTP_POST) + ngx.var.upstream_host = conf.model.options.upstream_host or DEFAULT_HOST + ctx.custom_balancer_host = conf.model.options.upstream_host or DEFAULT_HOST + ctx.custom_balancer_port = conf.model.options.port or DEFAULT_PORT + if conf.auth.source == "header" then + core.request.set_header(ctx, conf.auth.name, conf.auth.value) + else + local args = core.request.get_uri_args(ctx) + args[conf.auth.name] = conf.auth.value + core.request.set_uri_args(ctx, args) + end + + if conf.model.options then + for opt, val in pairs(conf.model.options) do + request_table[opt] = val + end + end + return true, nil Review Comment: remove `, nil` ########## apisix/plugins/ai-proxy.lua: ########## @@ -0,0 +1,103 @@ +-- +-- Licensed to the Apache Software Foundation (ASF) under one or more +-- contributor license agreements. See the NOTICE file distributed with +-- this work for additional information regarding copyright ownership. +-- The ASF licenses this file to You under the Apache License, Version 2.0 +-- (the "License"); you may not use this file except in compliance with +-- the License. You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- +local core = require("apisix.core") +local schema = require("apisix.plugins.ai-proxy.schema") +local constants = require("apisix.constants") +local require = require + +local ngx_req = ngx.req +local ngx = ngx + +local plugin_name = "ai-proxy" +local _M = { + version = 0.5, + priority = 1004, + name = plugin_name, + schema = schema, +} + + +function _M.check_schema(conf) + return core.schema.check(schema.plugin_schema, conf) +end + + +local CONTENT_TYPE_JSON = "application/json" + + +local function get_request_table() + local req_body, err = core.request.get_body() + if not req_body then + return nil, "failed to get request body: " .. (err or "request body is empty") + end + req_body, err = req_body:gsub("\\\"", "\"") -- remove escaping in JSON + if not req_body then + return nil, "failed to remove escaping from body: " .. req_body .. ". err: " .. err + end + return core.json.decode(req_body) +end + +function _M.access(conf, ctx) + local route_type = conf.route_type + ctx.ai_proxy = {} + + local content_type = core.request.header(ctx, "Content-Type") or CONTENT_TYPE_JSON + if content_type ~= CONTENT_TYPE_JSON then + return 400, "unsupported content-type: " .. content_type + end + + local request_table, err = get_request_table() + if not request_table then + return 400, err + end + + local req_schema = schema.chat_request_schema + if route_type == constants.COMPLETION then + req_schema = schema.chat_completion_request_schema + end + local ok, err = core.schema.check(req_schema, request_table) + if not ok then + return 400, "request format doesn't match schema: " .. err + end + + if conf.model.options and conf.model.options.response_streaming then + request_table.stream = true + ngx.ctx.disable_proxy_buffering = true Review Comment: why do we need this line? `ngx.ctx.disable_proxy_buffering = true` -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
