Repository: tajo Updated Branches: refs/heads/master 3b9a2e5f2 -> 513a8c308
TAJO-1384: Duplicated output file path problem. (jihoon) Closes #406 Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/513a8c30 Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/513a8c30 Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/513a8c30 Branch: refs/heads/master Commit: 513a8c3086b889d9b701d5c97eda3fb8c5453834 Parents: 3b9a2e5 Author: Jihoon Son <[email protected]> Authored: Wed Mar 11 10:07:01 2015 +0900 Committer: Jihoon Son <[email protected]> Committed: Wed Mar 11 10:07:40 2015 +0900 ---------------------------------------------------------------------- CHANGES | 2 + .../org/apache/tajo/storage/StorageUtil.java | 30 ++++------ .../apache/tajo/storage/TestStorageUtil.java | 58 ++++++++++++++++++++ 3 files changed, 72 insertions(+), 18 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/513a8c30/CHANGES ---------------------------------------------------------------------- diff --git a/CHANGES b/CHANGES index dc1a5f7..68aabf5 100644 --- a/CHANGES +++ b/CHANGES @@ -16,6 +16,8 @@ Release 0.11.0 - unreleased BUG FIXES + TAJO-1384: Duplicated output file path problem. (jihoon) + TAJO-1386: CURRENT_DATE generates parsing errors sometimes. (Contributed by navis, Committed by hyunsik) http://git-wip-us.apache.org/repos/asf/tajo/blob/513a8c30/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageUtil.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageUtil.java b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageUtil.java index 68e96d8..926b5d3 100644 --- a/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageUtil.java +++ b/tajo-storage/tajo-storage-common/src/main/java/org/apache/tajo/storage/StorageUtil.java @@ -18,15 +18,10 @@ package org.apache.tajo.storage; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileStatus; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.tajo.catalog.Column; -import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.TableMeta; -import org.apache.tajo.util.FileUtil; import sun.nio.ch.DirectBuffer; import java.io.DataInput; @@ -34,8 +29,6 @@ import java.io.EOFException; import java.io.IOException; import java.io.InputStream; import java.nio.ByteBuffer; -import java.util.ArrayList; -import java.util.List; public class StorageUtil extends StorageConstants { @@ -119,32 +112,33 @@ public class StorageUtil extends StorageConstants { } int maxValue = -1; - List<Path> fileNamePatternMatchedList = new ArrayList<Path>(); for (FileStatus eachFile: files) { // In the case of partition table, return largest value within all partition dirs. + int value; if (eachFile.isDirectory() && recursive) { - int value = getMaxFileSequence(fs, eachFile.getPath(), recursive); + value = getMaxFileSequence(fs, eachFile.getPath(), recursive); if (value > maxValue) { maxValue = value; } } else { if (eachFile.getPath().getName().matches(fileNamePatternV08) || eachFile.getPath().getName().matches(fileNamePatternV09)) { - fileNamePatternMatchedList.add(eachFile.getPath()); + value = getSequence(eachFile.getPath().getName()); + if (value > maxValue) { + maxValue = value; + } } } } - if (fileNamePatternMatchedList.isEmpty()) { - return maxValue; - } - Path lastFile = fileNamePatternMatchedList.get(fileNamePatternMatchedList.size() - 1); - String pathName = lastFile.getName(); + return maxValue; + } - // 0.8: pathName = part-<ExecutionBlockId.seq>-<TaskId.seq> - // 0.9: pathName = part-<ExecutionBlockId.seq>-<TaskId.seq>-<Sequence> - String[] pathTokens = pathName.split("-"); + // 0.8: pathName = part-<ExecutionBlockId.seq>-<TaskId.seq> + // 0.9: pathName = part-<ExecutionBlockId.seq>-<TaskId.seq>-<Sequence> + private static int getSequence(String name) { + String[] pathTokens = name.split("-"); if (pathTokens.length == 3) { return -1; } else if(pathTokens.length == 4) { http://git-wip-us.apache.org/repos/asf/tajo/blob/513a8c30/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorageUtil.java ---------------------------------------------------------------------- diff --git a/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorageUtil.java b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorageUtil.java new file mode 100644 index 0000000..4b7c929 --- /dev/null +++ b/tajo-storage/tajo-storage-hdfs/src/test/java/org/apache/tajo/storage/TestStorageUtil.java @@ -0,0 +1,58 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.storage; + +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.util.CommonTestingUtil; +import org.junit.After; +import org.junit.Before; +import org.junit.Test; + +import java.io.IOException; + +import static org.junit.Assert.assertEquals; + +public class TestStorageUtil { + private TajoConf conf; + private static String TEST_PATH = "target/test-data/TestStorageUtil"; + private Path testDir; + private FileSystem fs; + + @Before + public void setUp() throws Exception { + conf = new TajoConf(); + testDir = CommonTestingUtil.getTestDir(TEST_PATH); + fs = testDir.getFileSystem(conf); + } + + @After + public void tearDown() throws Exception { + } + + @Test + public final void testGetMaxFileSequence() throws IOException { + for (int i = 0; i < 7; i++) { + fs.create(new Path(testDir, "part-00-00000-00"+i), false); + } + + assertEquals(6, StorageUtil.getMaxFileSequence(fs, testDir, true)); + } +}
