This is an automated email from the ASF dual-hosted git repository.
kriszu pushed a commit to branch dev
in repository https://gitbox.apache.org/repos/asf/incubator-streampark.git
The following commit(s) were added to refs/heads/dev by this push:
new 8b580f106 [Improve] openapi request parameter schema improvement
(#3916)
8b580f106 is described below
commit 8b580f1065222b6e0c37d56bfebc42155d94df5e
Author: benjobs <[email protected]>
AuthorDate: Thu Jul 25 23:34:03 2024 +0800
[Improve] openapi request parameter schema improvement (#3916)
* [Improve] remove swagger
* [Improve] REST API improvements
* [Improve] checkstyle issue improvement
* [Improve] class not found fixed
* [Improve] known-dependencies.txt minor improvement
* [Improve] known-dependencies.txt updated
* [Improve] known-dependencies.txt updated
* [Improve] swagger improvement
* [Improve] OpenAPI schema bean improvement
* [Improve] serviceHelper minor improvement
* [Improve] shell script improvement
* [Improve] remove unused plugins dir
* [Improve] shell script minor improvement
* [Improve] openapi improvements
* [Improve] swagger minor improvement
* [Improve] openapi minor improvement
* [Improve] minor improvements
* [Improve] openapi minor improvement
* [Improve] openapi minor improvement
* [Improve] openAPI improvement
* [Improve] openapi.white support
* [Improve] openapi minor improvements
---
.../src/main/assembly/conf/config.yaml | 2 +
.../console/core/aspect/StreamParkAspect.java | 38 ++++++-
.../console/core/bean/OpenAPISchema.java | 2 +
.../console/core/component/OpenAPIComponent.java | 126 ++++++++++++++-------
4 files changed, 122 insertions(+), 46 deletions(-)
diff --git
a/streampark-console/streampark-console-service/src/main/assembly/conf/config.yaml
b/streampark-console/streampark-console-service/src/main/assembly/conf/config.yaml
index ec9a5fd33..86f9394b6 100644
---
a/streampark-console/streampark-console-service/src/main/assembly/conf/config.yaml
+++
b/streampark-console/streampark-console-service/src/main/assembly/conf/config.yaml
@@ -64,6 +64,8 @@ streampark:
http-auth: 'simple' # default simple, or kerberos
# flink on yarn or spark on yarn, HADOOP_USER_NAME
hadoop-user-name: hdfs
+ # openapi whitelist,
+ openapi.white-list:
# flink on yarn or spark on yarn, when the hadoop cluster enable kerberos
authentication, it is necessary to set Kerberos authentication parameters.
security:
diff --git
a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/aspect/StreamParkAspect.java
b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/aspect/StreamParkAspect.java
index 48b30ba5c..a17e7e23d 100644
---
a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/aspect/StreamParkAspect.java
+++
b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/aspect/StreamParkAspect.java
@@ -42,11 +42,20 @@ import org.aspectj.lang.annotation.Pointcut;
import org.aspectj.lang.reflect.MethodSignature;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.core.DefaultParameterNameDiscoverer;
+import org.springframework.core.SpringProperties;
import org.springframework.expression.EvaluationContext;
import org.springframework.expression.Expression;
import org.springframework.expression.spel.standard.SpelExpressionParser;
import org.springframework.expression.spel.support.StandardEvaluationContext;
import org.springframework.stereotype.Component;
+import org.springframework.web.context.request.RequestContextHolder;
+import org.springframework.web.context.request.ServletRequestAttributes;
+
+import javax.annotation.PostConstruct;
+import javax.servlet.http.HttpServletRequest;
+
+import java.util.HashSet;
+import java.util.Set;
@Slf4j
@Component
@@ -62,9 +71,27 @@ public class StreamParkAspect {
@Autowired
private ApplicationManageService applicationManageService;
+ private final Set<String> openapiWhitelist = new HashSet<>();
+
+ @PostConstruct
+ public void initOpenapiWhitelist() {
+ String whiteLists =
SpringProperties.getProperty("streampark.openapi.white-list");
+ if (StringUtils.isNotBlank(whiteLists)) {
+ String[] whiteList = whiteLists.trim().split("\\s");
+ for (String order : whiteList) {
+ if (StringUtils.isNotBlank(order)) {
+ if (!order.startsWith("/")) {
+ order = "/" + order;
+ }
+ openapiWhitelist.add(order);
+ }
+ }
+ }
+ }
+
@Pointcut("execution(public"
+ " org.apache.streampark.console.base.domain.RestResponse"
- + " org.apache.streampark.console.*.controller.*.*(..))")
+ + " org.apache.streampark.console.core.controller.*.*(..))")
public void openAPI() {
}
@@ -77,7 +104,14 @@ public class StreamParkAspect {
if (isApi != null && isApi) {
OpenAPI openAPI =
methodSignature.getMethod().getAnnotation(OpenAPI.class);
if (openAPI == null) {
- throw new ApiAlertException("current api unsupported!");
+ HttpServletRequest request =
+ ((ServletRequestAttributes)
RequestContextHolder.getRequestAttributes()).getRequest();
+ String url = request.getRequestURI();
+ if (openapiWhitelist.contains(url)) {
+ log.info("request by openapi white-list: {} ", url);
+ } else {
+ throw new ApiAlertException("current api unsupported!");
+ }
}
}
return (RestResponse) joinPoint.proceed();
diff --git
a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/bean/OpenAPISchema.java
b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/bean/OpenAPISchema.java
index 08477168d..d714ee480 100644
---
a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/bean/OpenAPISchema.java
+++
b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/bean/OpenAPISchema.java
@@ -29,6 +29,8 @@ public class OpenAPISchema {
private String url;
+ private String method;
+
private List<Schema> header;
private List<Schema> schema;
diff --git
a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/component/OpenAPIComponent.java
b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/component/OpenAPIComponent.java
index 2e6675cdf..b9507ebb4 100644
---
a/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/component/OpenAPIComponent.java
+++
b/streampark-console/streampark-console-service/src/main/java/org/apache/streampark/console/core/component/OpenAPIComponent.java
@@ -19,6 +19,7 @@ package org.apache.streampark.console.core.component;
import org.apache.streampark.common.util.CURLBuilder;
import org.apache.streampark.common.util.ReflectUtils;
+import org.apache.streampark.console.base.util.Tuple2;
import org.apache.streampark.console.core.annotation.OpenAPI;
import org.apache.streampark.console.core.bean.OpenAPISchema;
import org.apache.streampark.console.core.controller.OpenAPIController;
@@ -29,6 +30,7 @@ import org.apache.commons.lang3.StringUtils;
import lombok.extern.slf4j.Slf4j;
import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.http.HttpMethod;
import org.springframework.stereotype.Component;
import org.springframework.web.bind.annotation.DeleteMapping;
import org.springframework.web.bind.annotation.GetMapping;
@@ -50,6 +52,8 @@ public class OpenAPIComponent {
@Autowired
private AccessTokenService accessTokenService;
+ private final Map<String, String> types = new HashMap<>();
+
private final Map<String, OpenAPISchema> schemas = new HashMap<>();
public synchronized OpenAPISchema getOpenAPISchema(String name) {
@@ -63,20 +67,46 @@ public class OpenAPIComponent {
return schemas.get(name);
}
+ public String getOpenApiCUrl(String baseUrl, Long appId, Long teamId,
String name) {
+ OpenAPISchema schema = this.getOpenAPISchema(name);
+ if (schema == null) {
+ throw new UnsupportedOperationException("Unsupported OpenAPI: " +
name);
+ }
+
+ String url = schema.getUrl();
+ if (StringUtils.isNoneBlank(baseUrl)) {
+ url = baseUrl + url;
+ }
+ CURLBuilder curlBuilder = new CURLBuilder(url);
+ curlBuilder
+ .addHeader("Content-Type", "application/x-www-form-urlencoded;
charset=UTF-8")
+ .addHeader(
+ "Authorization",
+
accessTokenService.getByUserId(ServiceHelper.getUserId()).getToken());
+
+ schema.getSchema().forEach(c -> {
+ if (c.isRequired()) {
+ if ("appId".equals(c.getBindFor())) {
+ curlBuilder.addFormData(c.getName(), appId);
+ } else if ("teamId".equals(c.getBindFor())) {
+ curlBuilder.addFormData(c.getName(), teamId);
+ }
+ } else {
+ curlBuilder.addFormData(c.getName(), c.getDefaultValue());
+ }
+ });
+ return curlBuilder.build();
+ }
+
private void initOpenAPISchema() {
+ initTypes();
Class<?> clazz = OpenAPIController.class;
RequestMapping requestMapping =
clazz.getDeclaredAnnotation(RequestMapping.class);
String basePath = requestMapping.value()[0];
List<Method> methodList = ReflectUtils.getMethodsByAnnotation(clazz,
OpenAPI.class);
for (Method method : methodList) {
- String[] subUriPath = getMethodUriPath(method);
- String subPath = (subUriPath != null && subUriPath.length > 0) ?
subUriPath[0] : "";
- String restUrl = "/" + basePath;
- if (subPath != null) {
- restUrl += "/" + subPath;
- }
- restUrl = restUrl.replaceAll("/+", "/").replaceAll("/$", "");
+ OpenAPISchema detail = new OpenAPISchema();
List<OpenAPISchema.Schema> headerList = new ArrayList<>();
OpenAPI openAPI = method.getDeclaredAnnotation(OpenAPI.class);
@@ -89,11 +119,19 @@ public class OpenAPIComponent {
paramList.add(paramToSchema(param));
}
- OpenAPISchema detail = new OpenAPISchema();
- detail.setUrl(restUrl);
detail.setSchema(paramList);
detail.setHeader(headerList);
+ Tuple2<String, String[]> methodURI =
getMethodAndRequestURI(method);
+ String[] requestURI = methodURI.t2;
+ String uri = (requestURI != null && requestURI.length > 0) ?
requestURI[0] : "";
+ String restURI = "/" + basePath;
+ if (uri != null) {
+ restURI += "/" + uri;
+ }
+ restURI = restURI.replaceAll("/+", "/").replaceAll("/$", "");
+ detail.setUrl(restURI);
+ detail.setMethod(methodURI.t1);
schemas.put(openAPI.name(), detail);
}
}
@@ -106,71 +144,71 @@ public class OpenAPIComponent {
} else {
schema.setBindFor(param.bindFor());
}
- schema.setType(param.type().getName());
schema.setRequired(param.required());
schema.setDescription(param.description());
schema.setDefaultValue(param.defaultValue());
+ String type = types.get(param.type().getSimpleName());
+ if (type != null) {
+ schema.setType(type);
+ } else {
+ schema.setType("string(" + param.type().getSimpleName() + ")");
+ }
return schema;
}
- private String[] getMethodUriPath(Method method) {
+ private Tuple2<String, String[]> getMethodAndRequestURI(Method method) {
method.setAccessible(true);
GetMapping getMapping = method.getDeclaredAnnotation(GetMapping.class);
if (getMapping != null) {
- return getMapping.value();
+ return Tuple2.of(HttpMethod.GET.name(), getMapping.value());
}
PostMapping postMapping =
method.getDeclaredAnnotation(PostMapping.class);
if (postMapping != null) {
- return postMapping.value();
+ return Tuple2.of(HttpMethod.POST.name(), postMapping.value());
}
DeleteMapping deleteMapping =
method.getDeclaredAnnotation(DeleteMapping.class);
if (deleteMapping != null) {
- return deleteMapping.value();
+ return Tuple2.of(HttpMethod.DELETE.name(), deleteMapping.value());
}
PatchMapping patchMapping =
method.getDeclaredAnnotation(PatchMapping.class);
if (patchMapping != null) {
- return patchMapping.value();
+ return Tuple2.of(HttpMethod.PATCH.name(), patchMapping.value());
}
PutMapping putMapping = method.getDeclaredAnnotation(PutMapping.class);
if (putMapping != null) {
- return putMapping.value();
+ return Tuple2.of(HttpMethod.PUT.name(), putMapping.value());
}
- return null;
+
+ throw new IllegalArgumentException("get http method and requestURI
failed: " + method.getName());
}
- public String getOpenApiCUrl(String baseUrl, Long appId, Long teamId,
String name) {
- OpenAPISchema schema = this.getOpenAPISchema(name);
- if (schema == null) {
- throw new UnsupportedOperationException("Unsupported OpenAPI: " +
name);
- }
+ private void initTypes() {
+ types.put("String", "string");
- String url = schema.getUrl();
- if (StringUtils.isNoneBlank(baseUrl)) {
- url = baseUrl + url;
- }
- CURLBuilder curlBuilder = new CURLBuilder(url);
- curlBuilder
- .addHeader("Content-Type", "application/x-www-form-urlencoded;
charset=UTF-8")
- .addHeader(
- "Authorization",
-
accessTokenService.getByUserId(ServiceHelper.getUserId()).getToken());
+ types.put("int", "integer(int32)");
+ types.put("Integer", "integer(int32)");
+ types.put("Short", "integer(int32)");
- schema.getSchema().forEach(c -> {
- if (c.isRequired()) {
- if ("appId".equals(c.getBindFor())) {
- curlBuilder.addFormData(c.getName(), appId);
- } else if ("teamId".equals(c.getBindFor())) {
- curlBuilder.addFormData(c.getName(), teamId);
- }
- } else {
- curlBuilder.addFormData(c.getName(), c.getDefaultValue());
- }
- });
- return curlBuilder.build();
+ types.put("long", "integer(int64)");
+ types.put("Long", "integer(int64)");
+
+ types.put("double", "number(double)");
+ types.put("Double", "number(double)");
+
+ types.put("float", "number(float)");
+ types.put("Float", "number(float)");
+ types.put("boolean", "boolean");
+ types.put("Boolean", "boolean");
+
+ types.put("byte", "string(byte)");
+ types.put("Byte", "string(byte)");
+
+ types.put("Date", "string(date)");
+ types.put("DateTime", "string(datetime)");
}
}
