zentol commented on a change in pull request #12701:
URL: https://github.com/apache/flink/pull/12701#discussion_r442031268
##########
File path:
flink-end-to-end-tests/flink-end-to-end-tests-common/src/main/java/org/apache/flink/tests/util/TestUtils.java
##########
@@ -52,10 +54,11 @@
* @throws RuntimeException if none or multiple jars could be found
*/
public static Path getResourceJar(final String jarNameRegex) {
- String moduleDirProp = System.getProperty("moduleDir");
- Preconditions.checkNotNull(moduleDirProp, "The moduleDir
property was not set, You can set it when running maven via
-DmoduleDir=<path>");
+ // if the property is not set then we are most likely running
in the IDE, where the working directory is the
+ // module of the test that is currently running, which is
exactly what we want
+ Path moduleDirectory =
MODULE_DIRECTORY.get(Paths.get("").toAbsolutePath());
Review comment:
When you're taking commits from the master branch and backport them to
1.11, then it is customary to retain the authorship.
##########
File path:
flink-end-to-end-tests/flink-end-to-end-tests-hbase/src/test/java/org/apache/flink/tests/util/hbase/SQLClientHBaseITCase.java
##########
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.tests.util.hbase;
+
+import org.apache.flink.api.common.time.Deadline;
+import org.apache.flink.tests.util.TestUtils;
+import org.apache.flink.tests.util.cache.DownloadCache;
+import org.apache.flink.tests.util.categories.PreCommit;
+import org.apache.flink.tests.util.categories.TravisGroup1;
+import org.apache.flink.tests.util.flink.ClusterController;
+import org.apache.flink.tests.util.flink.FlinkResource;
+import org.apache.flink.tests.util.flink.FlinkResourceSetup;
+import org.apache.flink.tests.util.flink.LocalStandaloneFlinkResourceFactory;
+import org.apache.flink.tests.util.flink.SQLJobSubmission;
+import org.apache.flink.util.FileUtils;
+import org.apache.flink.util.TestLogger;
+
+import org.hamcrest.CoreMatchers;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.time.Duration;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static
org.apache.flink.tests.util.flink.LocalStandaloneFlinkResourceFactory.loadProjectRootDirectory;
+import static org.hamcrest.Matchers.arrayContainingInAnyOrder;
+import static org.hamcrest.Matchers.containsString;
+import static org.junit.Assert.assertThat;
+
+/**
+ * End-to-end test for the HBase connectors.
+ */
+@Category(value = {TravisGroup1.class, PreCommit.class})
+public class SQLClientHBaseITCase extends TestLogger {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(SQLClientHBaseITCase.class);
+
+ private static final String HBASE_E2E_SQL = "hbase_e2e.sql";
+ private static final String YARN_CLASSPATH =
"/flink-yarn-tests/target/yarn.classpath";
+
+ @Rule
+ public final HBaseResource hbase;
+
+ @Rule
+ public final FlinkResource flink = new
LocalStandaloneFlinkResourceFactory()
+ .create(FlinkResourceSetup.builder().build());
+
+ @Rule
+ public final TemporaryFolder tmp = new TemporaryFolder();
+
+ @ClassRule
+ public static final DownloadCache DOWNLOAD_CACHE = DownloadCache.get();
+
+ private static final Path sqlToolBoxJar =
TestUtils.getResourceJar(".*SqlToolbox.jar");
+ private static final Path sqlConnectorHBaseJar =
TestUtils.getResourceJar(".*hbase.jar");
+ private List<Path> hadoopClasspathJars;
+
+ public SQLClientHBaseITCase() {
+ this.hbase = HBaseResource.get();
+ }
+
+ @Before
+ public void before() throws Exception {
+ DOWNLOAD_CACHE.before();
+ Path tmpPath = tmp.getRoot().toPath();
+ LOG.info("The current temporary path: {}", tmpPath);
+
+ // Prepare all hadoop jars under HADOOP_CLASSPATH, use
yarn.classpath which contains all hadoop jars
+ Path currentModulePath = Paths.get("").toAbsolutePath();
+ Path flinkHomePath =
loadProjectRootDirectory(currentModulePath);
+ String hadoopClasspath = Paths.get(flinkHomePath.toString(),
YARN_CLASSPATH).toString();
Review comment:
Instead of re-using another modules classpath for convenience, please
instead declare a set of hadop dependencies that this test required, and use
the dependency-plugin to generate a distinct classpath file.
##########
File path: flink-end-to-end-tests/run-nightly-tests.sh
##########
@@ -236,16 +236,16 @@ printf
"\n\n====================================================================
printf "Running Java end-to-end tests\n"
printf
"==============================================================================\n"
-
-LOG4J_PROPERTIES=${END_TO_END_DIR}/../tools/log4j-travis.properties
+FLINK_PROJECT_ROOT_DIR="`(cd \"$END_TO_END_DIR\"/.. && pwd -P)`" # get flink
project root directory
+LOG4J_PROPERTIES=${END_TO_END_DIR}/../tools/ci/log4j.properties
MVN_LOGGING_OPTIONS="-Dlog.dir=${ARTIFACTS_DIR}
-DlogBackupDir=${ARTIFACTS_DIR}
-Dlog4j.configurationFile=file://$LOG4J_PROPERTIES"
MVN_COMMON_OPTIONS="-Dflink.forkCount=2 -Dflink.forkCountTestPackage=2 -Dfast
-Pskip-webui-build"
e2e_modules=$(find flink-end-to-end-tests -mindepth 2 -maxdepth 5 -name
'pom.xml' -printf '%h\n' | sort -u | tr '\n' ',')
e2e_modules="${e2e_modules},$(find flink-walkthroughs -mindepth 2 -maxdepth 2
-name 'pom.xml' -printf '%h\n' | sort -u | tr '\n' ',')"
PROFILE="$PROFILE -Pe2e-travis1 -Pe2e-travis2 -Pe2e-travis3 -Pe2e-travis4
-Pe2e-travis5 -Pe2e-travis6"
-run_mvn ${MVN_COMMON_OPTIONS} ${MVN_LOGGING_OPTIONS} ${PROFILE} verify -pl
${e2e_modules} -DdistDir=$(readlink -e build-target)
+run_mvn ${MVN_COMMON_OPTIONS} ${MVN_LOGGING_OPTIONS} ${PROFILE} verify -pl
${e2e_modules} -DdistDir=$(readlink -e build-target)
-DrootDir=${FLINK_PROJECT_ROOT_DIR} -Dcache-dir=$E2E_CACHE_FOLDER
-Dcache-ttl=P1M
Review comment:
same story as above in regards to retaining authorship
##########
File path:
flink-end-to-end-tests/flink-end-to-end-tests-hbase/src/test/java/org/apache/flink/tests/util/hbase/SQLClientHBaseITCase.java
##########
@@ -0,0 +1,193 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.tests.util.hbase;
+
+import org.apache.flink.api.common.time.Deadline;
+import org.apache.flink.tests.util.TestUtils;
+import org.apache.flink.tests.util.cache.DownloadCache;
+import org.apache.flink.tests.util.categories.PreCommit;
+import org.apache.flink.tests.util.categories.TravisGroup1;
+import org.apache.flink.tests.util.flink.ClusterController;
+import org.apache.flink.tests.util.flink.FlinkResource;
+import org.apache.flink.tests.util.flink.FlinkResourceSetup;
+import org.apache.flink.tests.util.flink.LocalStandaloneFlinkResourceFactory;
+import org.apache.flink.tests.util.flink.SQLJobSubmission;
+import org.apache.flink.util.FileUtils;
+import org.apache.flink.util.TestLogger;
+
+import org.hamcrest.CoreMatchers;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URL;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.time.Duration;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static
org.apache.flink.tests.util.flink.LocalStandaloneFlinkResourceFactory.loadProjectRootDirectory;
+import static org.hamcrest.Matchers.arrayContainingInAnyOrder;
+import static org.hamcrest.Matchers.containsString;
+import static org.junit.Assert.assertThat;
+
+/**
+ * End-to-end test for the HBase connectors.
+ */
+@Category(value = {TravisGroup1.class, PreCommit.class})
+public class SQLClientHBaseITCase extends TestLogger {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(SQLClientHBaseITCase.class);
+
+ private static final String HBASE_E2E_SQL = "hbase_e2e.sql";
+ private static final String YARN_CLASSPATH =
"/flink-yarn-tests/target/yarn.classpath";
+
+ @Rule
+ public final HBaseResource hbase;
+
+ @Rule
+ public final FlinkResource flink = new
LocalStandaloneFlinkResourceFactory()
+ .create(FlinkResourceSetup.builder().build());
+
+ @Rule
+ public final TemporaryFolder tmp = new TemporaryFolder();
+
+ @ClassRule
+ public static final DownloadCache DOWNLOAD_CACHE = DownloadCache.get();
+
+ private static final Path sqlToolBoxJar =
TestUtils.getResourceJar(".*SqlToolbox.jar");
+ private static final Path sqlConnectorHBaseJar =
TestUtils.getResourceJar(".*hbase.jar");
+ private List<Path> hadoopClasspathJars;
+
+ public SQLClientHBaseITCase() {
+ this.hbase = HBaseResource.get();
+ }
+
+ @Before
+ public void before() throws Exception {
+ DOWNLOAD_CACHE.before();
+ Path tmpPath = tmp.getRoot().toPath();
+ LOG.info("The current temporary path: {}", tmpPath);
+
+ // Prepare all hadoop jars under HADOOP_CLASSPATH, use
yarn.classpath which contains all hadoop jars
+ Path currentModulePath = Paths.get("").toAbsolutePath();
+ Path flinkHomePath =
loadProjectRootDirectory(currentModulePath);
+ String hadoopClasspath = Paths.get(flinkHomePath.toString(),
YARN_CLASSPATH).toString();
+ File hadoopClasspathFile = new File(hadoopClasspath);
+
+ if (!hadoopClasspathFile.exists()) {
+ throw new FileNotFoundException(hadoopClasspath);
Review comment:
we need a better error message
##########
File path:
flink-end-to-end-tests/flink-end-to-end-tests-hbase/src/main/java/org/apache/flink/tests/util/hbase/LocalStandaloneHBaseResource.java
##########
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.flink.tests.util.hbase;
+
+import org.apache.flink.tests.util.AutoClosableProcess;
+import org.apache.flink.tests.util.CommandLineWrapper;
+import org.apache.flink.tests.util.activation.OperatingSystemRestriction;
+import org.apache.flink.tests.util.cache.DownloadCache;
+import org.apache.flink.util.OperatingSystem;
+
+import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.function.Consumer;
+import java.util.stream.Collectors;
+
+/**
+ * {@link HBaseResource} that downloads hbase and set up a local hbase cluster.
+ */
+public class LocalStandaloneHBaseResource implements HBaseResource {
+
+ private static final Logger LOG =
LoggerFactory.getLogger(LocalStandaloneHBaseResource.class);
+
+ private final TemporaryFolder tmp = new TemporaryFolder();
+
+ private final DownloadCache downloadCache = DownloadCache.get();
+ private Path hbaseDir;
+
+ LocalStandaloneHBaseResource() {
+ OperatingSystemRestriction.forbid(
+ String.format("The %s relies on UNIX utils and shell
scripts.", getClass().getSimpleName()),
+ OperatingSystem.WINDOWS);
+ }
+
+ private static String getHBaseDownloadUrl() {
+ return
"https://archive.apache.org/dist/hbase/1.4.3/hbase-1.4.3-bin.tar.gz";;
+ }
+
+ @Override
+ public void before() throws Exception {
+ tmp.create();
+ downloadCache.before();
+
+ this.hbaseDir =
tmp.newFolder("hbase").toPath().toAbsolutePath();
+ setupHBaseDist();
+ setupHBaseCluster();
+ }
+
+ private void setupHBaseDist() throws IOException {
+ final Path downloadDirectory =
tmp.newFolder("getOrDownload").toPath();
+ final Path hbaseArchive =
downloadCache.getOrDownload(getHBaseDownloadUrl(), downloadDirectory);
+
+ LOG.info("HBase localtion: {}", hbaseDir.toAbsolutePath());
Review comment:
```suggestion
LOG.info("HBase location: {}", hbaseDir.toAbsolutePath());
```
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
