bzp2010 commented on code in PR #2474:
URL: https://github.com/apache/apisix-dashboard/pull/2474#discussion_r903407771
##########
api/internal/handler/data_loader/route_import.go:
##########
@@ -19,535 +19,317 @@ package data_loader
import (
"bytes"
"context"
- "encoding/json"
"fmt"
- "net/http"
"path"
"reflect"
- "regexp"
- "strings"
- "github.com/getkin/kin-openapi/openapi3"
"github.com/gin-gonic/gin"
+ "github.com/juliangruber/go-intersect"
+ "github.com/pkg/errors"
"github.com/shiningrush/droplet"
- "github.com/shiningrush/droplet/data"
"github.com/shiningrush/droplet/wrapper"
wgin "github.com/shiningrush/droplet/wrapper/gin"
"github.com/apisix/manager-api/internal/conf"
"github.com/apisix/manager-api/internal/core/entity"
"github.com/apisix/manager-api/internal/core/store"
"github.com/apisix/manager-api/internal/handler"
- "github.com/apisix/manager-api/internal/log"
- "github.com/apisix/manager-api/internal/utils"
- "github.com/apisix/manager-api/internal/utils/consts"
+ loader
"github.com/apisix/manager-api/internal/handler/data_loader/loader"
+
"github.com/apisix/manager-api/internal/handler/data_loader/loader/openapi3"
)
type ImportHandler struct {
- routeStore *store.GenericStore
- svcStore store.Interface
- upstreamStore store.Interface
+ routeStore store.Interface
+ upstreamStore store.Interface
+ serviceStore store.Interface
+ consumerStore store.Interface
+ sslStore store.Interface
+ streamRouteStore store.Interface
+ globalPluginStore store.Interface
+ pluginConfigStore store.Interface
+ protoStore store.Interface
}
func NewImportHandler() (handler.RouteRegister, error) {
return &ImportHandler{
- routeStore: store.GetStore(store.HubKeyRoute),
- svcStore: store.GetStore(store.HubKeyService),
- upstreamStore: store.GetStore(store.HubKeyUpstream),
+ routeStore: store.GetStore(store.HubKeyRoute),
+ upstreamStore: store.GetStore(store.HubKeyUpstream),
+ serviceStore: store.GetStore(store.HubKeyService),
+ consumerStore: store.GetStore(store.HubKeyConsumer),
+ sslStore: store.GetStore(store.HubKeySsl),
+ streamRouteStore: store.GetStore(store.HubKeyStreamRoute),
+ globalPluginStore: store.GetStore(store.HubKeyGlobalRule),
+ pluginConfigStore: store.GetStore(store.HubKeyPluginConfig),
+ protoStore: store.GetStore(store.HubKeyProto),
}, nil
}
-var regPathVar = regexp.MustCompile(`{[\w.]*}`)
-var regPathRepeat = regexp.MustCompile(`-APISIX-REPEAT-URI-[\d]*`)
-
func (h *ImportHandler) ApplyRoute(r *gin.Engine) {
r.POST("/apisix/admin/import/routes", wgin.Wraps(h.Import,
wrapper.InputType(reflect.TypeOf(ImportInput{}))))
}
+type ImportResult struct {
+ Total int `json:"total"`
+ Failed int `json:"failed"`
+ Errors []string `json:"errors"`
+}
+
+type LoaderType string
+
type ImportInput struct {
- Force bool `auto_read:"force,query"`
+ Type string `auto_read:"type"`
+ TaskName string `auto_read:"task_name"`
FileName string `auto_read:"_file"`
FileContent []byte `auto_read:"file"`
+
+ MergeMethod string `auto_read:"merge_method"`
}
+const (
+ LoaderTypeOpenAPI3 LoaderType = "openapi3"
+)
+
func (h *ImportHandler) Import(c droplet.Context) (interface{}, error) {
input := c.Input().(*ImportInput)
- Force := input.Force
- // file check
+ // input file content check
suffix := path.Ext(input.FileName)
if suffix != ".json" && suffix != ".yaml" && suffix != ".yml" {
- return nil, fmt.Errorf("required file type is .yaml, .yml or
.json but got: %s", suffix)
+ return nil, errors.Errorf("required file type is .yaml, .yml or
.json but got: %s", suffix)
}
-
contentLen := bytes.Count(input.FileContent, nil) - 1
- if contentLen > conf.ImportSizeLimit {
- log.Warnf("upload file size exceeds limit: %d", contentLen)
- return nil, fmt.Errorf("the file size exceeds the limit; limit
%d", conf.ImportSizeLimit)
+ if contentLen <= 0 {
+ return nil, errors.New("uploaded file is empty")
}
-
- swagger, err :=
openapi3.NewSwaggerLoader().LoadSwaggerFromData(input.FileContent)
- if err != nil {
- return nil, err
+ if contentLen > conf.ImportSizeLimit {
+ return nil, errors.Errorf("uploaded file size exceeds the
limit, limit is %d", conf.ImportSizeLimit)
}
- if len(swagger.Paths) < 1 {
- return &data.SpecCodeResponse{StatusCode:
http.StatusBadRequest},
- consts.ErrImportFile
+ var l loader.Loader
+ switch LoaderType(input.Type) {
+ case LoaderTypeOpenAPI3:
+ l = &openapi3.Loader{
+ MergeMethod: input.MergeMethod == "true",
+ TaskName: input.TaskName,
+ }
+ break
+ default:
+ return nil, fmt.Errorf("unsupported data loader type: %s",
input.Type)
}
- routes, err := OpenAPI3ToRoute(swagger)
+ dataSets, err := l.Import(input.FileContent)
if err != nil {
return nil, err
}
- // check route
- for _, route := range routes {
- err := checkRouteExist(c.Context(), h.routeStore, route)
- if err != nil && !Force {
- log.Warnf("import duplicate: %s, route: %#v", err,
route)
- return &data.SpecCodeResponse{StatusCode:
http.StatusBadRequest},
- fmt.Errorf("route(uris:%v) conflict, %s",
route.Uris, err)
- }
- if route.ServiceID != nil {
- _, err := h.svcStore.Get(c.Context(),
utils.InterfaceToString(route.ServiceID))
- if err != nil {
- if err == data.ErrNotFound {
- return
&data.SpecCodeResponse{StatusCode: http.StatusBadRequest},
- fmt.Errorf(consts.IDNotFound,
"service", route.ServiceID)
- }
- return &data.SpecCodeResponse{StatusCode:
http.StatusBadRequest}, err
- }
- }
- if route.UpstreamID != nil {
- _, err := h.upstreamStore.Get(c.Context(),
utils.InterfaceToString(route.UpstreamID))
- if err != nil {
- if err == data.ErrNotFound {
- return
&data.SpecCodeResponse{StatusCode: http.StatusBadRequest},
- fmt.Errorf(consts.IDNotFound,
"upstream", route.UpstreamID)
- }
- return &data.SpecCodeResponse{StatusCode:
http.StatusBadRequest}, err
- }
- }
-
- if _, err := h.routeStore.CreateCheck(route); err != nil {
- return handler.SpecCodeResponse(err),
- fmt.Errorf("create route(uris:%v) failed: %s",
route.Uris, err)
- }
+ // Pre-checking for route duplication
+ preCheckErrs := h.preCheck(c.Context(), dataSets)
+ if _, ok := preCheckErrs[store.HubKeyRoute]; ok &&
len(preCheckErrs[store.HubKeyRoute]) > 0 {
+ return h.convertToImportResult(dataSets, preCheckErrs), nil
}
- // merge route
- idRoute := make(map[string]*entity.Route)
- for _, route := range routes {
- if existRoute, ok := idRoute[route.ID.(string)]; ok {
- uris := append(existRoute.Uris, route.Uris...)
- existRoute.Uris = uris
- } else {
- idRoute[route.ID.(string)] = route
- }
- }
-
- routes = make([]*entity.Route, 0, len(idRoute))
- for _, route := range idRoute {
- routes = append(routes, route)
- }
-
- // create route
- for _, route := range routes {
- if Force && route.ID != nil {
- if _, err := h.routeStore.Update(c.Context(), route,
true); err != nil {
- return handler.SpecCodeResponse(err),
- fmt.Errorf("update route(uris:%v)
failed: %s", route.Uris, err)
- }
- } else {
- if _, err := h.routeStore.Create(c.Context(), route);
err != nil {
- return handler.SpecCodeResponse(err),
- fmt.Errorf("create route(uris:%v)
failed: %s", route.Uris, err)
- }
- }
- }
-
- return map[string]int{
- "paths": len(swagger.Paths),
- "routes": len(routes),
- }, nil
+ // Create APISIX resources
+ createErrs := h.createEntities(c.Context(), dataSets)
+ return h.convertToImportResult(dataSets, createErrs), nil
}
-func checkRouteExist(ctx context.Context, routeStore *store.GenericStore,
route *entity.Route) error {
- //routeStore := store.GetStore(store.HubKeyRoute)
- ret, err := routeStore.List(ctx, store.ListInput{
- Predicate: func(obj interface{}) bool {
- id := utils.InterfaceToString(route.ID)
- item := obj.(*entity.Route)
- if id != "" && id != utils.InterfaceToString(item.ID) {
- return false
- }
-
- itemUris := item.Uris
- if item.URI != "" {
- if itemUris == nil {
- itemUris = []string{item.URI}
- } else {
- itemUris = append(itemUris, item.URI)
- }
- }
-
- routeUris := route.Uris
- if route.URI != "" {
- if routeUris == nil {
- routeUris = []string{route.URI}
- } else {
- routeUris = append(routeUris, route.URI)
+// Pre-check imported data for duplicates
+// The main problem facing duplication is routing, so here
+// we mainly check the duplication of routes, based on
+// domain name and uri.
+func (h *ImportHandler) preCheck(ctx context.Context, data *loader.DataSets)
map[store.HubKey][]string {
+ errs := make(map[store.HubKey][]string)
+ for _, route := range data.Routes {
+ errs[store.HubKeyRoute] = make([]string, 0)
+ o, err := h.routeStore.List(ctx, store.ListInput{
+ // The check logic here is that if when a duplicate
HOST or URI
+ // has been found, the HTTP method is checked for
overlap, and
+ // if there is overlap it is determined to be a
duplicate route
+ // and the import is rejected.
+ Predicate: func(obj interface{}) bool {
+ r := obj.(*entity.Route)
+ isMethodDuplicated :=
len(intersect.Hash(r.Methods, route.Methods)) > 0
+
+ // Check URI and host duplication
+ // First check for duplicate URIs
+ if (r.URI != "" && route.URI != "" && r.URI ==
route.URI) || len(intersect.Hash(r.Uris, route.Uris)) > 0 {
Review Comment:
yes, let me try
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
