Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/tajo into
hbase_storage
Conflicts:
tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java
tajo-storage/src/main/java/org/apache/tajo/storage/text/DelimitedTextFile.java
tajo-storage/src/main/java/org/apache/tajo/storage/trevni/TrevniAppender.java
tajo-storage/src/main/java/org/apache/tajo/storage/trevni/TrevniScanner.java
tajo-storage/src/main/resources/storage-default.xml
Project: http://git-wip-us.apache.org/repos/asf/tajo/repo
Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/337677be
Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/337677be
Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/337677be
Branch: refs/heads/hbase_storage
Commit: 337677be7f32cb4971ca5d9c80fbe1c57b12eb8a
Parents: 6937387 a56f8b5
Author: HyoungJun Kim <[email protected]>
Authored: Tue Nov 25 15:58:11 2014 +0900
Committer: HyoungJun Kim <[email protected]>
Committed: Tue Nov 25 15:58:11 2014 +0900
----------------------------------------------------------------------
CHANGES | 42 ++
.../org/apache/tajo/algebra/CreateTable.java | 4 +-
.../main/java/org/apache/tajo/algebra/Expr.java | 5 +-
.../org/apache/tajo/catalog/CatalogUtil.java | 2 -
.../tajo/catalog/json/FunctionAdapter.java | 5 +-
.../tajo/catalog/json/TableMetaAdapter.java | 12 +-
.../src/main/proto/CatalogProtos.proto | 3 +-
.../tajo/catalog/store/HCatalogStore.java | 3 +-
.../java/org/apache/tajo/cli/tsql/TajoCli.java | 5 +
tajo-common/pom.xml | 4 +
.../java/org/apache/tajo/conf/TajoConf.java | 2 +-
.../java/org/apache/tajo/datum/BlobDatum.java | 2 -
.../java/org/apache/tajo/datum/DateDatum.java | 20 +-
.../java/org/apache/tajo/datum/TextDatum.java | 5 +-
.../org/apache/tajo/datum/TimestampDatum.java | 11 +-
.../org/apache/tajo/json/CommonGsonHelper.java | 24 +-
.../org/apache/tajo/json/DataTypeAdapter.java | 2 +-
.../java/org/apache/tajo/json/DatumAdapter.java | 14 +-
.../java/org/apache/tajo/util/KeyValueSet.java | 6 +-
.../java/org/apache/tajo/util/NumberUtil.java | 700 +++++++++++++++++--
.../java/org/apache/tajo/util/StringUtils.java | 57 ++
.../java/org/apache/tajo/util/UnsafeUtil.java | 4 +-
.../org/apache/tajo/util/datetime/TimeMeta.java | 14 +-
.../org/apache/tajo/datum/TestDateDatum.java | 17 +-
.../org/apache/tajo/datum/TestTextDatum.java | 14 +
.../apache/tajo/datum/TestTimestampDatum.java | 17 +-
.../org/apache/tajo/util/TestStringUtil.java | 24 +
.../java/org/apache/tajo/master/TajoMaster.java | 29 +-
.../tajo/master/TajoMasterClientService.java | 32 +-
.../master/querymaster/QueryJobManager.java | 70 +-
.../master/querymaster/QueryMasterTask.java | 1 +
.../org/apache/tajo/net/CachedDNSResolver.java | 64 --
.../tajo/scheduler/SimpleFifoScheduler.java | 2 +-
.../apache/tajo/util/history/HistoryWriter.java | 2 +
.../tajo/webapp/QueryExecutorServlet.java | 24 +-
.../java/org/apache/tajo/worker/TajoWorker.java | 12 +-
.../main/java/org/apache/tajo/worker/Task.java | 4 +-
.../apache/tajo/worker/TaskRunnerManager.java | 7 +-
.../src/main/resources/webapps/admin/index.jsp | 42 +-
.../resources/webapps/admin/query_executor.jsp | 4 +-
.../org/apache/tajo/TajoTestingCluster.java | 20 +
.../org/apache/tajo/cli/tsql/TestTajoCli.java | 22 +-
.../org/apache/tajo/client/TestTajoClient.java | 9 +-
.../tajo/master/ha/TestHAServiceHDFSImpl.java | 41 +-
.../org/apache/tajo/worker/TestHistory.java | 6 +-
.../tajo/plan/serder/EvalNodeAdapter.java | 3 +-
.../tajo/plan/serder/LogicalNodeAdapter.java | 3 +-
tajo-project/pom.xml | 2 +-
tajo-storage/pom.xml | 11 +-
.../org/apache/tajo/storage/avro/AvroUtil.java | 35 +-
.../tajo/storage/text/DelimitedTextFile.java | 25 +-
.../text/TextFieldSerializerDeserializer.java | 56 +-
.../tajo/storage/trevni/TrevniAppender.java | 204 ------
.../tajo/storage/trevni/TrevniScanner.java | 193 -----
.../apache/tajo/tuple/offheap/HeapTuple.java | 7 +-
.../apache/tajo/tuple/offheap/UnSafeTuple.java | 7 +-
.../src/main/resources/storage-default.xml | 20 +-
.../java/org/apache/tajo/HttpFileServer.java | 84 +++
.../org/apache/tajo/HttpFileServerHandler.java | 184 +++++
.../tajo/HttpFileServerPipelineFactory.java | 54 ++
.../apache/tajo/storage/TestMergeScanner.java | 4 +-
.../org/apache/tajo/storage/TestStorages.java | 1 -
.../apache/tajo/storage/avro/TestAvroUtil.java | 108 +++
.../src/test/resources/storage-default.xml | 18 +-
.../tajo/pullserver/TajoPullServerService.java | 2 +-
65 files changed, 1636 insertions(+), 794 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/337677be/CHANGES
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/337677be/tajo-catalog/tajo-catalog-common/src/main/java/org/apache/tajo/catalog/CatalogUtil.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/337677be/tajo-catalog/tajo-catalog-common/src/main/proto/CatalogProtos.proto
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/337677be/tajo-client/src/main/java/org/apache/tajo/cli/tsql/TajoCli.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/337677be/tajo-common/src/main/java/org/apache/tajo/conf/TajoConf.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/337677be/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java
----------------------------------------------------------------------
diff --cc tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java
index 47b42ec,6e585af..d015efe
--- a/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java
+++ b/tajo-core/src/main/java/org/apache/tajo/master/TajoMaster.java
@@@ -49,12 -49,11 +49,12 @@@ import org.apache.tajo.master.rm.TajoWo
import org.apache.tajo.master.rm.WorkerResourceManager;
import org.apache.tajo.master.session.SessionManager;
import org.apache.tajo.rpc.RpcChannelFactory;
+import org.apache.tajo.storage.FileStorageManager;
import org.apache.tajo.storage.StorageManager;
import org.apache.tajo.util.*;
- import org.apache.tajo.util.metrics.TajoSystemMetrics;
import org.apache.tajo.util.history.HistoryReader;
import org.apache.tajo.util.history.HistoryWriter;
+ import org.apache.tajo.util.metrics.TajoSystemMetrics;
import org.apache.tajo.webapp.QueryExecutorServlet;
import org.apache.tajo.webapp.StaticHttpServer;
http://git-wip-us.apache.org/repos/asf/tajo/blob/337677be/tajo-core/src/main/java/org/apache/tajo/master/TajoMasterClientService.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/337677be/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java
----------------------------------------------------------------------
diff --cc
tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java
index 6a55de6,5cf3df5..6b61d04
---
a/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java
+++
b/tajo-core/src/main/java/org/apache/tajo/master/querymaster/QueryMasterTask.java
@@@ -361,29 -354,8 +361,30 @@@ public class QueryMasterTask extends Co
LogicalPlanner planner = new LogicalPlanner(catalog);
LogicalOptimizer optimizer = new LogicalOptimizer(systemConf);
Expr expr = JsonHelper.fromJson(jsonExpr, Expr.class);
+ jsonExpr = null; // remove the possible OOM
- LogicalPlan plan = planner.createPlan(queryContext, expr);
+ plan = planner.createPlan(queryContext, expr);
+
+ StoreType storeType = PlannerUtil.getStoreType(plan);
+ if (storeType != null) {
+ sm = StorageManager.getStorageManager(systemConf, storeType);
+ StorageProperty storageProperty = sm.getStorageProperty();
+ if (storageProperty.isSortedInsert()) {
+ String tableName = PlannerUtil.getStoreTableName(plan);
+ LogicalRootNode rootNode = plan.getRootBlock().getRoot();
+ TableDesc tableDesc = PlannerUtil.getTableDesc(catalog,
rootNode.getChild());
+ if (tableDesc == null) {
+ throw new VerifyException("Can't get table meta data from
catalog: " + tableName);
+ }
+ List<RewriteRule> storageSpecifiedRewriteRules = sm.getRewriteRules(
+ getQueryTaskContext().getQueryContext(), tableDesc);
+ if (storageSpecifiedRewriteRules != null) {
+ for (RewriteRule eachRule: storageSpecifiedRewriteRules) {
+ optimizer.addRuleAfterToJoinOpt(eachRule);
+ }
+ }
+ }
+ }
+
optimizer.optimize(queryContext, plan);
GlobalEngine.DistributedQueryHookManager hookManager = new
GlobalEngine.DistributedQueryHookManager();
http://git-wip-us.apache.org/repos/asf/tajo/blob/337677be/tajo-core/src/main/java/org/apache/tajo/worker/Task.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/337677be/tajo-core/src/test/java/org/apache/tajo/TajoTestingCluster.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/337677be/tajo-project/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/337677be/tajo-storage/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tajo/blob/337677be/tajo-storage/src/main/java/org/apache/tajo/storage/text/DelimitedTextFile.java
----------------------------------------------------------------------
diff --cc
tajo-storage/src/main/java/org/apache/tajo/storage/text/DelimitedTextFile.java
index 29c6987,68d89e7..a337509
---
a/tajo-storage/src/main/java/org/apache/tajo/storage/text/DelimitedTextFile.java
+++
b/tajo-storage/src/main/java/org/apache/tajo/storage/text/DelimitedTextFile.java
@@@ -32,8 -32,6 +32,7 @@@ import org.apache.hadoop.io.compress.Co
import org.apache.hadoop.io.compress.CompressionCodecFactory;
import org.apache.hadoop.io.compress.CompressionOutputStream;
import org.apache.hadoop.io.compress.Compressor;
- import org.apache.hadoop.util.ReflectionUtils;
+import org.apache.tajo.QueryUnitAttemptId;
import org.apache.tajo.catalog.Schema;
import org.apache.tajo.catalog.TableMeta;
import org.apache.tajo.catalog.statistics.TableStats;
http://git-wip-us.apache.org/repos/asf/tajo/blob/337677be/tajo-storage/src/main/resources/storage-default.xml
----------------------------------------------------------------------
diff --cc tajo-storage/src/main/resources/storage-default.xml
index 0f05536,064f250..85e5f41
--- a/tajo-storage/src/main/resources/storage-default.xml
+++ b/tajo-storage/src/main/resources/storage-default.xml
@@@ -35,7 -35,7 +35,7 @@@
<!--- Registered Scanner Handler -->
<property>
<name>tajo.storage.scanner-handler</name>
-
<value>textfile,csv,raw,rcfile,row,trevni,parquet,sequencefile,avro,hbase</value>
- <value>textfile,csv,raw,rcfile,row,parquet,sequencefile,avro</value>
++ <value>textfile,csv,raw,rcfile,row,parquet,sequencefile,avro,hbase</value>
</property>
<!--- Fragment Class Configurations -->
@@@ -134,9 -116,9 +125,9 @@@
<!--- Appender Handler -->
<property>
<name>tajo.storage.appender-handler</name>
-
<value>textfile,csv,raw,rcfile,row,trevni,parquet,sequencefile,avro,hbase,hfile</value>
- <value>textfile,csv,raw,rcfile,row,parquet,sequencefile,avro</value>
++ <value>textfile,csv,raw,rcfile,row,parquet,sequencefile,avro,hbase</value>
</property>
-
+
<property>
<name>tajo.storage.appender-handler.textfile.class</name>
<value>org.apache.tajo.storage.text.DelimitedTextFile$DelimitedTextFileAppender</value>
http://git-wip-us.apache.org/repos/asf/tajo/blob/337677be/tajo-storage/src/test/java/org/apache/tajo/storage/TestMergeScanner.java
----------------------------------------------------------------------
diff --cc
tajo-storage/src/test/java/org/apache/tajo/storage/TestMergeScanner.java
index b70bba9,e6714b5..cc4aa51
--- a/tajo-storage/src/test/java/org/apache/tajo/storage/TestMergeScanner.java
+++ b/tajo-storage/src/test/java/org/apache/tajo/storage/TestMergeScanner.java
@@@ -93,10 -91,10 +92,10 @@@ public class TestMergeScanner
public void setup() throws Exception {
conf = new TajoConf();
conf.setVar(ConfVars.ROOT_DIR, TEST_PATH);
- conf.setStrings("tajo.storage.projectable-scanner", "rcfile", "trevni",
"parquet", "avro");
+ conf.setStrings("tajo.storage.projectable-scanner", "rcfile", "parquet",
"avro");
testDir = CommonTestingUtil.getTestDir(TEST_PATH);
fs = testDir.getFileSystem(conf);
- sm = StorageManager.getStorageManager(conf, testDir);
+ sm = StorageManager.getFileStorageManager(conf, testDir);
}
@Test
http://git-wip-us.apache.org/repos/asf/tajo/blob/337677be/tajo-storage/src/test/java/org/apache/tajo/storage/TestStorages.java
----------------------------------------------------------------------
