Repository: tajo
Updated Branches:
refs/heads/master 101413e31 -> 29db645ad
TAJO-909: {SortBased, Col}PartitionStoreExec should not write partition keys to
files. (Hyoungjun Kim via hyunsik)
Closes #64
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/29db645a
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/29db645a
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/29db645a
Branch: refs/heads/master
Commit: 29db645ad02067464ed3e9b94b3e62d44ab2adf4
Parents: 101413e
Author: Hyunsik Choi <[email protected]>
Authored: Fri Jul 11 14:05:11 2014 +0900
Committer: Hyunsik Choi <[email protected]>
Committed: Fri Jul 11 14:05:11 2014 +0900
----------------------------------------------------------------------
CHANGES | 3 ++
.../planner/physical/ColPartitionStoreExec.java | 9 ++++-
.../java/org/apache/tajo/QueryTestCaseBase.java | 35 ++++++++++++--------
.../tajo/engine/query/TestTablePartitions.java | 10 +++++-
.../querymaster/TestQueryUnitStatusUpdate.java | 4 +--
5 files changed, 43 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/29db645a/CHANGES
----------------------------------------------------------------------
diff --git a/CHANGES b/CHANGES
index c0208aa..f73bf1b 100644
--- a/CHANGES
+++ b/CHANGES
@@ -82,6 +82,9 @@ Release 0.9.0 - unreleased
BUG FIXES
+ TAJO-909: {SortBased, Col}PartitionStoreExec should not write partition
+ keys to files. (Hyoungjun Kim via hyunsik)
+
TAJO-912: Tsql prints wrong version. (Mai Hai Thanh via hyunsik)
TAJO-902: Unicode delimiter does not work correctly. (jinho)
http://git-wip-us.apache.org/repos/asf/tajo/blob/29db645a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java
----------------------------------------------------------------------
diff --git
a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java
b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java
index fe36905..12eb0e0 100644
---
a/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java
+++
b/tajo-core/src/main/java/org/apache/tajo/engine/planner/physical/ColPartitionStoreExec.java
@@ -23,9 +23,10 @@ import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.tajo.catalog.CatalogUtil;
import org.apache.tajo.catalog.Column;
+import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
-import org.apache.tajo.engine.planner.logical.InsertNode;
import org.apache.tajo.engine.planner.logical.CreateTableNode;
+import org.apache.tajo.engine.planner.logical.InsertNode;
import org.apache.tajo.engine.planner.logical.NodeType;
import org.apache.tajo.engine.planner.logical.StoreTableNode;
import org.apache.tajo.storage.StorageUtil;
@@ -60,6 +61,12 @@ public abstract class ColPartitionStoreExec extends
UnaryPhysicalExec {
// Find column index to name subpartition directory path
keyNum = this.plan.getPartitionMethod().getExpressionSchema().size();
+ if (plan.getType() == NodeType.INSERT && keyNum > 0) {
+ Column[] removedPartitionColumns = new Column[this.outSchema.size() -
keyNum];
+ System.arraycopy(this.outSchema.toArray(), 0, removedPartitionColumns,
0, removedPartitionColumns.length);
+ this.outSchema = new Schema(removedPartitionColumns);
+ }
+
keyIds = new int[keyNum];
keyNames = new String[keyNum];
for (int i = 0; i < keyNum; i++) {
http://git-wip-us.apache.org/repos/asf/tajo/blob/29db645a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java
----------------------------------------------------------------------
diff --git a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java
b/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java
index 70c73f9..29613cb 100644
--- a/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java
+++ b/tajo-core/src/test/java/org/apache/tajo/QueryTestCaseBase.java
@@ -574,23 +574,14 @@ public class QueryTestCaseBase {
public String getTableFileContents(Path path) throws Exception {
FileSystem fs = path.getFileSystem(conf);
- FileStatus[] files = fs.listStatus(path);
-
- if (files == null || files.length == 0) {
- return null;
- }
-
StringBuilder sb = new StringBuilder();
- byte[] buf = new byte[1024];
-
- for (FileStatus file: files) {
- if (file.isDirectory()) {
- continue;
- }
- InputStream in = fs.open(file.getPath());
+ List<Path> paths = listFiles(fs, path);
+ for (Path eachPath: paths) {
+ InputStream in = fs.open(eachPath);
try {
while (true) {
+ byte[] buf = new byte[1024];
int readBytes = in.read(buf);
if (readBytes <= 0) {
break;
@@ -602,7 +593,6 @@ public class QueryTestCaseBase {
in.close();
}
}
-
return sb.toString();
}
@@ -621,4 +611,21 @@ public class QueryTestCaseBase {
Path path = tableDesc.getPath();
return getTableFileContents(path);
}
+
+ private List<Path> listFiles(FileSystem fs, Path path) throws Exception {
+ List<Path> result = new ArrayList<Path>();
+ FileStatus[] files = fs.listStatus(path);
+ if (files == null || files.length == 0) {
+ return result;
+ }
+
+ for (FileStatus eachFile: files) {
+ if (eachFile.isDirectory()) {
+ result.addAll(listFiles(fs, eachFile.getPath()));
+ } else {
+ result.add(eachFile.getPath());
+ }
+ }
+ return result;
+ }
}
http://git-wip-us.apache.org/repos/asf/tajo/blob/29db645a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java
----------------------------------------------------------------------
diff --git
a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java
b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java
index 8c989b5..c34c3f4 100644
---
a/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java
+++
b/tajo-core/src/test/java/org/apache/tajo/engine/query/TestTablePartitions.java
@@ -240,13 +240,21 @@ public class TestTablePartitions extends
QueryTestCaseBase {
assertEquals(5, desc.getStats().getNumRows().intValue());
}
+ String expected = "N\n" +
+ "N\n" +
+ "N\n" +
+ "R\n" +
+ "R\n";
+
+ String tableData = getTableFileContents(desc.getPath());
+ assertEquals(expected, tableData);
+
res = executeString("select * from " + tableName + " where col2 = 2");
Map<Double, int []> resultRows1 = Maps.newHashMap();
resultRows1.put(45.0d, new int[]{3, 2});
resultRows1.put(38.0d, new int[]{2, 2});
-
for (int i = 0; i < 2; i++) {
assertTrue(res.next());
assertEquals(resultRows1.get(res.getDouble(4))[0], res.getInt(2));
http://git-wip-us.apache.org/repos/asf/tajo/blob/29db645a/tajo-core/src/test/java/org/apache/tajo/master/querymaster/TestQueryUnitStatusUpdate.java
----------------------------------------------------------------------
diff --git
a/tajo-core/src/test/java/org/apache/tajo/master/querymaster/TestQueryUnitStatusUpdate.java
b/tajo-core/src/test/java/org/apache/tajo/master/querymaster/TestQueryUnitStatusUpdate.java
index 7b9d830..e340953 100644
---
a/tajo-core/src/test/java/org/apache/tajo/master/querymaster/TestQueryUnitStatusUpdate.java
+++
b/tajo-core/src/test/java/org/apache/tajo/master/querymaster/TestQueryUnitStatusUpdate.java
@@ -105,8 +105,8 @@ public class TestQueryUnitStatusUpdate extends
QueryTestCaseBase {
// in/out * subquery(4)
long[] expectedNumRows = new long[]{2, 2, 5, 5, 7, 2, 2, 2};
- long[] expectedNumBytes = new long[]{18, 34, 45, 75, 109, 34, 34, 18};
- long[] expectedReadBytes = new long[]{18, 0, 45, 0, 109, 0, 34, 0};
+ long[] expectedNumBytes = new long[]{8, 34, 20, 75, 109, 34, 34, 18};
+ long[] expectedReadBytes = new long[]{8, 0, 20, 0, 109, 0, 34, 0};
assertStatus(4, expectedNumRows, expectedNumBytes, expectedReadBytes);
} finally {
