caishunfeng commented on code in PR #10442: URL: https://github.com/apache/dolphinscheduler/pull/10442#discussion_r897478087
########## dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/utils/JSONUtils.java: ########## @@ -0,0 +1,384 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one or more + * contributor license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright ownership. + * The ASF licenses this file to You under the Apache License, Version 2.0 + * (the "License"); you may not use this file except in compliance with + * the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.dolphinscheduler.api.test.utils; +import static java.nio.charset.StandardCharsets.UTF_8; + +import static com.fasterxml.jackson.databind.DeserializationFeature.ACCEPT_EMPTY_ARRAY_AS_NULL_OBJECT; +import static com.fasterxml.jackson.databind.DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES; +import static com.fasterxml.jackson.databind.DeserializationFeature.READ_UNKNOWN_ENUM_VALUES_AS_NULL; +import static com.fasterxml.jackson.databind.MapperFeature.REQUIRE_SETTERS_FOR_GETTERS; + +import java.io.IOException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.TimeZone; + +import org.apache.dolphinscheduler.api.test.core.Constants; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.core.type.TypeReference; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.JsonDeserializer; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.JsonSerializer; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.ObjectWriter; +import com.fasterxml.jackson.databind.SerializationFeature; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.node.ArrayNode; +import com.fasterxml.jackson.databind.node.ObjectNode; +import com.fasterxml.jackson.databind.node.TextNode; +import com.fasterxml.jackson.databind.type.CollectionType; +import org.testcontainers.shaded.org.apache.commons.lang.StringUtils; + +/** + * json utils + */ +public class JSONUtils { Review Comment: Is it necessary to create this util class again? ########## dolphinscheduler-api-test/dolphinscheduler-api-test-case/pom.xml: ########## @@ -0,0 +1,40 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to Apache Software Foundation (ASF) under one or more contributor + ~ license agreements. See the NOTICE file distributed with + ~ this work for additional information regarding copyright + ~ ownership. Apache Software Foundation (ASF) licenses this file to you under + ~ the Apache License, Version 2.0 (the "License"); you may + ~ not use this file except in compliance with the License. + ~ You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, + ~ software distributed under the License is distributed on an + ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + ~ KIND, either express or implied. See the License for the + ~ specific language governing permissions and limitations + ~ under the License. + --> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <parent> + <artifactId>dolphinscheduler-api-test</artifactId> + <groupId>org.apache.dolphinscheduler</groupId> + <version>1.0-SNAPSHOT</version> Review Comment: I think we should keep the version consistent ########## dolphinscheduler-api-test/README.md: ########## @@ -0,0 +1,44 @@ +# DolphinScheduler Backend API Test + +## Page Object Model + +DolphinScheduler API test respects +the [Page Object Model (POM)](https://www.selenium.dev/documentation/guidelines/page_object_models/) design pattern. +Every page of DolphinScheduler's api is abstracted into a class for better maintainability. + +### Example + +The login page's api is abstracted +as [`LoginPage`](dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/pages/LoginPage.java), with the +following fields, + +```java +public HttpResponse login(String username, String password) { + Map<String, Object> params = new HashMap<>(); + + params.put("userName", username); + params.put("userPassword", password); + + RequestClient requestClient = new RequestClient(); + + return requestClient.post("/login", null, params); +} +``` + +where `userName`, `userPassword` are the main elements on UI that we are interested in. + +## Test Environment Setup + +DolphinScheduler End-to-End test uses [testcontainers](https://www.testcontainers.org) to set up the testing Review Comment: Is api-test related to e2e? ########## dolphinscheduler-api-test/dolphinscheduler-api-test-core/src/main/java/org/apache/dolphinscheduler/api/test/core/DolphinSchedulerExtension.java: ########## @@ -0,0 +1,79 @@ +/* + * Licensed to Apache Software Foundation (ASF) under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Apache Software Foundation (ASF) licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.dolphinscheduler.api.test.core; + +import java.io.File; +import java.net.URL; +import java.time.Duration; +import java.util.List; +import java.util.Objects; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +import org.junit.jupiter.api.extension.AfterAllCallback; +import org.junit.jupiter.api.extension.BeforeAllCallback; +import org.junit.jupiter.api.extension.ExtensionContext; +import org.testcontainers.containers.DockerComposeContainer; +import org.testcontainers.containers.wait.strategy.Wait; + +import lombok.extern.slf4j.Slf4j; + +@Slf4j +final class DolphinSchedulerExtension implements BeforeAllCallback, AfterAllCallback { + private final boolean LOCAL_MODE = Objects.equals(System.getProperty("local"), "true"); + + private final String SERVICE_NAME = "dolphinscheduler_1"; + + private DockerComposeContainer<?> compose; + + @Override + public void beforeAll(ExtensionContext context) { + if (!LOCAL_MODE) { + compose = createDockerCompose(context); + compose.start(); + } + } + + @Override + public void afterAll(ExtensionContext context) { + if (compose != null) { + compose.stop(); + } + } + + private DockerComposeContainer<?> createDockerCompose(ExtensionContext context) { + final Class<?> clazz = context.getRequiredTestClass(); + final DolphinScheduler annotation = clazz.getAnnotation(DolphinScheduler.class); + final List<File> files = Stream.of(annotation.composeFiles()) + .map(it -> DolphinScheduler.class.getClassLoader().getResource(it)) + .filter(Objects::nonNull) + .map(URL::getPath) + .map(File::new) + .collect(Collectors.toList()); + + compose = new DockerComposeContainer<>(files) + .withPull(true) + .withTailChildContainers(true) + .withLogConsumer(SERVICE_NAME, outputFrame -> LOGGER.info(outputFrame.getUtf8String())) + .waitingFor(SERVICE_NAME, Wait.forHealthcheck().withStartupTimeout(Duration.ofSeconds(180))); Review Comment: use constant replace 180 ########## dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/utils/RequestClient.java: ########## @@ -0,0 +1,171 @@ +/* + * Licensed to Apache Software Foundation (ASF) under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Apache Software Foundation (ASF) licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.dolphinscheduler.api.test.utils; + +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.apache.dolphinscheduler.api.test.core.Constants; +import org.apache.dolphinscheduler.api.test.entity.HttpResponse; +import org.apache.dolphinscheduler.api.test.entity.HttpResponseBody; +import org.testcontainers.shaded.okhttp3.FormBody; +import org.testcontainers.shaded.okhttp3.Headers; +import org.testcontainers.shaded.okhttp3.MediaType; +import org.testcontainers.shaded.okhttp3.OkHttpClient; +import org.testcontainers.shaded.okhttp3.Request; +import org.testcontainers.shaded.okhttp3.RequestBody; +import org.testcontainers.shaded.okhttp3.Response; + +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +@Slf4j +public class RequestClient { + + private OkHttpClient httpClient = null; + + public RequestClient() { + this.httpClient = new OkHttpClient(); + } + + @SneakyThrows + public HttpResponse get(String url, Map<String, String> headers, Map<String, Object> params) { + String requestUrl = String.format("%s%s%s", Constants.DOLPHINSCHEDULER_API_URL, url, getParams(params)); + + Headers headersBuilder = new Headers.Builder().build(); + if (headers != null) { + headersBuilder = Headers.of(headers); + } + + LOGGER.info("GET request to {}, Headers: {}", requestUrl, headersBuilder); + Request request = new Request.Builder() + .url(requestUrl) + .headers(headersBuilder) + .get() + .build(); + + Response response = this.httpClient.newCall(request).execute(); + + HttpResponseBody responseData = null; + int responseCode = response.code(); + if (response.body() != null) { + responseData = JSONUtils.parseObject(response.body().string(), HttpResponseBody.class); + } + response.close(); + + HttpResponse httpResponse = new HttpResponse(responseCode, responseData); + + LOGGER.info("GET response: {}", httpResponse); + + return httpResponse; + } + + public static String getParams(Map<String, Object> params) { + StringBuilder sb = new StringBuilder("?"); + if (params.size() > 0) { + for (Map.Entry<String, Object> item : params.entrySet()) { + Object value = item.getValue(); + if (Objects.nonNull(value)) { + sb.append("&"); + sb.append(item.getKey()); + sb.append("="); Review Comment: same here ########## dolphinscheduler-api-test/dolphinscheduler-api-test-core/pom.xml: ########## @@ -0,0 +1,32 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + ~ Licensed to Apache Software Foundation (ASF) under one or more contributor + ~ license agreements. See the NOTICE file distributed with + ~ this work for additional information regarding copyright + ~ ownership. Apache Software Foundation (ASF) licenses this file to you under + ~ the Apache License, Version 2.0 (the "License"); you may + ~ not use this file except in compliance with the License. + ~ You may obtain a copy of the License at + ~ + ~ http://www.apache.org/licenses/LICENSE-2.0 + ~ + ~ Unless required by applicable law or agreed to in writing, + ~ software distributed under the License is distributed on an + ~ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + ~ KIND, either express or implied. See the License for the + ~ specific language governing permissions and limitations + ~ under the License. + --> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <parent> + <artifactId>dolphinscheduler-api-test</artifactId> + <groupId>org.apache.dolphinscheduler</groupId> + <version>1.0-SNAPSHOT</version> Review Comment: same here ########## dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/java/org/apache/dolphinscheduler/api.test/utils/RequestClient.java: ########## @@ -0,0 +1,171 @@ +/* + * Licensed to Apache Software Foundation (ASF) under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Apache Software Foundation (ASF) licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.dolphinscheduler.api.test.utils; + +import lombok.SneakyThrows; +import lombok.extern.slf4j.Slf4j; +import org.apache.dolphinscheduler.api.test.core.Constants; +import org.apache.dolphinscheduler.api.test.entity.HttpResponse; +import org.apache.dolphinscheduler.api.test.entity.HttpResponseBody; +import org.testcontainers.shaded.okhttp3.FormBody; +import org.testcontainers.shaded.okhttp3.Headers; +import org.testcontainers.shaded.okhttp3.MediaType; +import org.testcontainers.shaded.okhttp3.OkHttpClient; +import org.testcontainers.shaded.okhttp3.Request; +import org.testcontainers.shaded.okhttp3.RequestBody; +import org.testcontainers.shaded.okhttp3.Response; + +import java.util.HashMap; +import java.util.Map; +import java.util.Objects; + +@Slf4j +public class RequestClient { + + private OkHttpClient httpClient = null; + + public RequestClient() { + this.httpClient = new OkHttpClient(); + } + + @SneakyThrows + public HttpResponse get(String url, Map<String, String> headers, Map<String, Object> params) { + String requestUrl = String.format("%s%s%s", Constants.DOLPHINSCHEDULER_API_URL, url, getParams(params)); + + Headers headersBuilder = new Headers.Builder().build(); + if (headers != null) { + headersBuilder = Headers.of(headers); + } + + LOGGER.info("GET request to {}, Headers: {}", requestUrl, headersBuilder); + Request request = new Request.Builder() + .url(requestUrl) + .headers(headersBuilder) + .get() + .build(); + + Response response = this.httpClient.newCall(request).execute(); + + HttpResponseBody responseData = null; + int responseCode = response.code(); + if (response.body() != null) { + responseData = JSONUtils.parseObject(response.body().string(), HttpResponseBody.class); + } + response.close(); + + HttpResponse httpResponse = new HttpResponse(responseCode, responseData); + + LOGGER.info("GET response: {}", httpResponse); + + return httpResponse; + } + + public static String getParams(Map<String, Object> params) { + StringBuilder sb = new StringBuilder("?"); + if (params.size() > 0) { + for (Map.Entry<String, Object> item : params.entrySet()) { + Object value = item.getValue(); + if (Objects.nonNull(value)) { + sb.append("&"); Review Comment: use constant. ########## dolphinscheduler-api-test/dolphinscheduler-api-test-case/src/test/resources/docker/file-manage/common.properties: ########## @@ -0,0 +1,90 @@ +# Review Comment: If the original file has changed, does it need to be kept in sync here? -- This is an automated message from the Apache Git Service. To respond to the message, please log on to GitHub and use the URL above to go to the specific comment. To unsubscribe, e-mail: [email protected] For queries about this service, please contact Infrastructure at: [email protected]
