This is an automated email from the ASF dual-hosted git repository.
shaofengshi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/kylin.git
The following commit(s) were added to refs/heads/master by this push:
new 8ecaba5 KYLIN-3454 #5525 fix potential thread-safe problem
8ecaba5 is described below
commit 8ecaba51a97e13458d2dd1a94dee656dcd75e20d
Author: Xinbei Fu <[email protected]>
AuthorDate: Wed Jul 18 13:41:03 2018 +0800
KYLIN-3454 #5525 fix potential thread-safe problem
---
.../kylin/common/persistence/ResourceTool.java | 28 +++++----
.../kylin/common/persistence/ResourceToolTest.java | 68 ++++++++++++++++++++++
2 files changed, 85 insertions(+), 11 deletions(-)
diff --git
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
index 8803320..450eb57 100644
---
a/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
+++
b/core-common/src/main/java/org/apache/kylin/common/persistence/ResourceTool.java
@@ -40,7 +40,6 @@ public class ResourceTool {
private static String[] includes = null;
private static String[] excludes = null;
- private static final TreeSet<String> pathsSkipChildrenCheck = new
TreeSet<>();
private static final Logger logger =
LoggerFactory.getLogger(ResourceTool.class);
@@ -173,13 +172,7 @@ public class ResourceTool {
logger.info("Copy from {} to {}", src, dst);
- for (String resourceRoot : SKIP_CHILDREN_CHECK_RESOURCE_ROOT) {
- NavigableSet<String> all =
src.listResourcesRecursively(resourceRoot);
- if (all != null) {
-
pathsSkipChildrenCheck.addAll(src.listResourcesRecursively(resourceRoot));
- }
- }
- copyR(src, dst, path, copyImmutableResource);
+ copyR(src, dst, path, getPathsSkipChildren(src),
copyImmutableResource);
}
public static void copy(KylinConfig srcConfig, KylinConfig dstConfig,
List<String> paths) throws IOException {
@@ -195,7 +188,7 @@ public class ResourceTool {
logger.info("Copy from {} to {}", src, dst);
for (String path : paths) {
- copyR(src, dst, path, copyImmutableResource);
+ copyR(src, dst, path, getPathsSkipChildren(src),
copyImmutableResource);
}
}
@@ -209,7 +202,7 @@ public class ResourceTool {
copy(srcConfig, dstConfig, "/", copyImmutableResource);
}
- public static void copyR(ResourceStore src, ResourceStore dst, String
path, boolean copyImmutableResource)
+ public static void copyR(ResourceStore src, ResourceStore dst, String
path, TreeSet<String> pathsSkipChildrenCheck, boolean copyImmutableResource)
throws IOException {
if (!copyImmutableResource && IMMUTABLE_PREFIX.contains(path)) {
@@ -241,9 +234,22 @@ public class ResourceTool {
} else {
// case of folder
for (String child : children)
- copyR(src, dst, child, copyImmutableResource);
+ copyR(src, dst, child, pathsSkipChildrenCheck,
copyImmutableResource);
+ }
+
+ }
+
+ private static TreeSet<String> getPathsSkipChildren(ResourceStore src)
throws IOException {
+ TreeSet<String> pathsSkipChildrenCheck = new TreeSet<>();
+
+ for (String resourceRoot : SKIP_CHILDREN_CHECK_RESOURCE_ROOT) {
+ NavigableSet<String> all =
src.listResourcesRecursively(resourceRoot);
+ if (all != null) {
+
pathsSkipChildrenCheck.addAll(src.listResourcesRecursively(resourceRoot));
+ }
}
+ return pathsSkipChildrenCheck;
}
private static boolean matchFilter(String path) {
diff --git
a/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceToolTest.java
b/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceToolTest.java
new file mode 100644
index 0000000..a6361c5
--- /dev/null
+++
b/core-common/src/test/java/org/apache/kylin/common/persistence/ResourceToolTest.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.kylin.common.persistence;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.kylin.common.KylinConfig;
+import org.apache.kylin.common.util.LocalFileMetadataTestCase;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.File;
+import java.io.IOException;
+
+public class ResourceToolTest extends LocalFileMetadataTestCase {
+ private static final String dstPath = "../examples/test_metadata2/";
+
+ @Before
+ public void setup() throws Exception {
+ this.createTestMetadata();
+ FileUtils.forceMkdir(new File(dstPath));
+ FileUtils.cleanDirectory(new File(dstPath));
+ }
+
+ @After
+ public void after() throws Exception {
+ File directory = new File(dstPath);
+ try {
+ FileUtils.deleteDirectory(directory);
+ } catch (IOException e) {
+ if (directory.exists() && directory.list().length > 0)
+ throw new IllegalStateException("Can't delete directory " +
directory, e);
+ }
+ this.cleanupTestMetadata();
+ }
+
+ @Test
+ public void testCopy() throws IOException {
+ KylinConfig dstConfig = KylinConfig.createInstanceFromUri(dstPath);
+ ResourceStore srcStore =
ResourceStore.getStore(KylinConfig.getInstanceFromEnv());
+ ResourceStore dstStore = ResourceStore.getStore(dstConfig);
+
+ //metadata under source path and destination path are not equal before
copy
+ Assert.assertNotEquals(srcStore.listResources("/"),
dstStore.listResources("/"));
+
+ ResourceTool.copy(KylinConfig.getInstanceFromEnv(), dstConfig, "/");
+
+ //After copy, two paths have same metadata
+ Assert.assertEquals(srcStore.listResources("/"),
dstStore.listResources("/"));
+ }
+}
