Merge remote-tracking branch 'gerrit/master' into HEAD Change-Id: I1b56e4dbb67889bbe4ff8462ccc2b49968821a1f
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/eae4f307 Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/eae4f307 Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/eae4f307 Branch: refs/heads/hadoop-next Commit: eae4f307d880d98293ef85633cfe13a2ebb3b06b Parents: 3cb3f34 13455b5 Author: Tim Armstrong <[email protected]> Authored: Tue Oct 25 13:12:51 2016 -0700 Committer: Tim Armstrong <[email protected]> Committed: Tue Oct 25 13:13:53 2016 -0700 ---------------------------------------------------------------------- .gitignore | 2 +- LICENSE.txt | 130 ++- README.md | 10 +- be/CMakeLists.txt | 2 +- be/src/benchmarks/bitmap-benchmark.cc | 6 +- be/src/benchmarks/bloom-filter-benchmark.cc | 198 ++-- be/src/codegen/CMakeLists.txt | 7 + be/src/codegen/codegen-anyval.cc | 8 +- be/src/codegen/llvm-codegen.cc | 10 +- be/src/codegen/llvm-codegen.h | 9 +- be/src/exec/aggregation-node.cc | 61 +- be/src/exec/aggregation-node.h | 11 +- be/src/exec/analytic-eval-node.cc | 88 +- be/src/exec/analytic-eval-node.h | 28 +- be/src/exec/data-sink.cc | 5 + be/src/exec/exchange-node.cc | 17 +- be/src/exec/exchange-node.h | 5 + be/src/exec/exec-node.cc | 21 +- be/src/exec/exec-node.h | 13 +- be/src/exec/hash-join-node.cc | 51 +- be/src/exec/hash-join-node.h | 6 +- be/src/exec/hash-table.cc | 58 +- be/src/exec/hash-table.h | 12 +- be/src/exec/hbase-table-sink.cc | 4 +- be/src/exec/hdfs-avro-scanner.cc | 18 +- be/src/exec/hdfs-avro-scanner.h | 2 +- be/src/exec/hdfs-parquet-scanner.cc | 12 +- be/src/exec/hdfs-rcfile-scanner.cc | 2 +- be/src/exec/hdfs-scan-node-base.cc | 30 +- be/src/exec/hdfs-scan-node-base.h | 1 + be/src/exec/hdfs-scanner.cc | 66 +- be/src/exec/hdfs-scanner.h | 14 +- be/src/exec/hdfs-sequence-scanner.cc | 12 +- be/src/exec/hdfs-table-sink.cc | 4 +- be/src/exec/hdfs-text-scanner.cc | 10 +- be/src/exec/kudu-scan-node.cc | 3 - be/src/exec/kudu-scan-node.h | 3 - be/src/exec/kudu-scanner.cc | 4 +- be/src/exec/kudu-table-sink.cc | 141 ++- be/src/exec/kudu-table-sink.h | 53 +- be/src/exec/nested-loop-join-node.cc | 12 +- be/src/exec/old-hash-table.cc | 62 +- be/src/exec/old-hash-table.h | 6 +- be/src/exec/partitioned-aggregation-node.cc | 76 +- be/src/exec/partitioned-aggregation-node.h | 13 +- be/src/exec/partitioned-hash-join-builder.cc | 83 +- be/src/exec/partitioned-hash-join-builder.h | 19 +- be/src/exec/partitioned-hash-join-node.cc | 56 +- be/src/exec/partitioned-hash-join-node.h | 4 +- be/src/exec/plan-root-sink.cc | 8 +- be/src/exec/sort-node.cc | 23 +- be/src/exec/sort-node.h | 4 + be/src/exec/topn-node.cc | 94 +- be/src/exec/topn-node.h | 4 +- be/src/exprs/agg-fn-evaluator.h | 27 +- be/src/exprs/aggregate-functions-ir.cc | 11 +- be/src/exprs/case-expr.cc | 6 +- be/src/exprs/case-expr.h | 2 +- be/src/exprs/compound-predicates.cc | 37 +- be/src/exprs/compound-predicates.h | 10 +- be/src/exprs/conditional-functions.cc | 4 +- be/src/exprs/conditional-functions.h | 8 +- be/src/exprs/expr.cc | 14 +- be/src/exprs/expr.h | 8 +- be/src/exprs/hive-udf-call.cc | 4 +- be/src/exprs/hive-udf-call.h | 2 +- be/src/exprs/is-not-empty-predicate.cc | 4 +- be/src/exprs/is-not-empty-predicate.h | 2 +- be/src/exprs/literal.cc | 4 +- be/src/exprs/literal.h | 2 +- be/src/exprs/null-literal.cc | 4 +- be/src/exprs/null-literal.h | 2 +- be/src/exprs/scalar-fn-call.cc | 246 ++--- be/src/exprs/scalar-fn-call.h | 4 +- be/src/exprs/slot-ref.cc | 5 +- be/src/exprs/slot-ref.h | 2 +- be/src/exprs/tuple-is-null-predicate.cc | 6 +- be/src/exprs/tuple-is-null-predicate.h | 2 +- be/src/runtime/coordinator.cc | 14 +- be/src/runtime/coordinator.h | 1 - be/src/runtime/plan-fragment-executor.cc | 16 +- be/src/runtime/runtime-state.cc | 8 - be/src/runtime/runtime-state.h | 27 +- be/src/runtime/sorted-run-merger.h | 2 +- be/src/runtime/sorter.cc | 2 +- be/src/runtime/sorter.h | 2 +- be/src/runtime/tuple.cc | 6 +- be/src/runtime/tuple.h | 2 +- be/src/scheduling/simple-scheduler-test-util.cc | 6 +- be/src/scheduling/simple-scheduler-test-util.h | 5 +- be/src/scheduling/simple-scheduler-test.cc | 19 + be/src/scheduling/simple-scheduler.cc | 8 +- be/src/service/fe-support.cc | 36 +- be/src/service/frontend.cc | 9 +- be/src/service/impala-beeswax-server.cc | 4 +- be/src/service/query-exec-state.cc | 4 +- be/src/service/query-options.cc | 1 + be/src/testutil/mem-util.h | 2 + be/src/testutil/test-udfs.cc | 29 + be/src/udf/udf.cc | 3 +- be/src/util/bit-util.h | 4 + be/src/util/bitmap-test.cc | 82 +- be/src/util/bitmap.cc | 8 +- be/src/util/bitmap.h | 42 +- be/src/util/bloom-filter.cc | 49 +- be/src/util/cpu-info.cc | 1 + be/src/util/cpu-info.h | 3 +- be/src/util/tuple-row-compare.cc | 13 +- be/src/util/tuple-row-compare.h | 2 +- bin/bootstrap_toolchain.py | 9 +- bin/distcc/.gitignore | 1 + bin/distcc/README.md | 106 +++ bin/distcc/distcc.sh | 62 ++ bin/distcc/distcc_env.sh | 160 ++++ bin/impala-config.sh | 12 +- bin/rat_exclude_files.txt | 1 - bin/start-catalogd.sh | 30 +- bin/start-impala-cluster.py | 11 +- buildall.sh | 3 + common/thrift/CatalogObjects.thrift | 39 +- common/thrift/ImpalaInternalService.thrift | 4 +- common/thrift/ImpalaService.thrift | 8 +- common/thrift/JniCatalog.thrift | 44 +- common/thrift/PlanNodes.thrift | 7 +- common/thrift/generate_error_codes.py | 9 + fe/src/main/cup/sql-parser.cup | 407 ++++---- .../apache/impala/analysis/AnalysisUtils.java | 43 + .../apache/impala/analysis/AnalyticExpr.java | 10 +- .../org/apache/impala/analysis/ColumnDef.java | 66 +- .../analysis/CreateTableAsSelectStmt.java | 41 +- .../impala/analysis/CreateTableDataSrcStmt.java | 30 +- .../analysis/CreateTableLikeFileStmt.java | 29 +- .../impala/analysis/CreateTableLikeStmt.java | 19 +- .../apache/impala/analysis/CreateTableStmt.java | 480 +++++----- .../apache/impala/analysis/DistributeParam.java | 211 +++-- .../java/org/apache/impala/analysis/Expr.java | 6 +- .../apache/impala/analysis/FunctionParams.java | 3 +- .../org/apache/impala/analysis/ModifyStmt.java | 4 +- .../apache/impala/analysis/TableDataLayout.java | 56 ++ .../org/apache/impala/analysis/TableDef.java | 316 +++++++ .../org/apache/impala/analysis/ToSqlUtils.java | 68 +- .../org/apache/impala/catalog/BuiltinsDb.java | 4 +- .../java/org/apache/impala/catalog/Catalog.java | 4 + .../impala/catalog/CatalogServiceCatalog.java | 7 +- .../main/java/org/apache/impala/catalog/Db.java | 5 + .../apache/impala/catalog/HdfsFileFormat.java | 15 +- .../org/apache/impala/catalog/HdfsTable.java | 29 +- .../apache/impala/catalog/ImpaladCatalog.java | 8 +- .../org/apache/impala/catalog/KuduTable.java | 318 ++++--- .../java/org/apache/impala/catalog/Table.java | 5 + .../org/apache/impala/catalog/TableLoader.java | 1 - .../java/org/apache/impala/catalog/Type.java | 9 + .../impala/catalog/delegates/DdlDelegate.java | 75 -- .../catalog/delegates/KuduDdlDelegate.java | 190 ---- .../delegates/UnsupportedOpDelegate.java | 35 - .../apache/impala/common/FileSystemUtil.java | 2 +- .../impala/planner/HdfsPartitionFilter.java | 2 +- .../org/apache/impala/planner/HdfsScanNode.java | 40 - .../org/apache/impala/planner/KuduScanNode.java | 2 +- .../impala/service/CatalogOpExecutor.java | 416 ++++---- .../org/apache/impala/service/Frontend.java | 9 +- .../org/apache/impala/service/JniCatalog.java | 2 +- .../org/apache/impala/service/JniFrontend.java | 5 +- .../impala/service/KuduCatalogOpExecutor.java | 240 +++++ .../java/org/apache/impala/util/KuduUtil.java | 106 +-- fe/src/main/jflex/sql-scanner.flex | 2 + .../apache/impala/analysis/AnalyzeDDLTest.java | 409 ++++---- .../impala/analysis/AnalyzeExprsTest.java | 11 +- .../org/apache/impala/analysis/ParserTest.java | 32 +- .../org/apache/impala/service/JdbcTest.java | 6 +- .../impala/testutil/ImpaladTestCatalog.java | 6 +- fe/src/test/resources/hbase-site.xml.template | 2 - infra/python/deps/download_requirements | 2 +- infra/python/deps/requirements.txt | 6 +- llvm-ir/test-loop.bc | Bin 1828 -> 0 bytes shell/impala_client.py | 2 +- testdata/bin/create-load-data.sh | 4 +- testdata/bin/generate-schema-statements.py | 20 +- testdata/bin/load-tpc-kudu.py | 119 +++ testdata/bin/run-step.sh | 3 +- .../common/etc/hadoop/conf/hadoop-policy.xml | 2 - .../etc/hadoop/conf/log4j.properties.tmpl | 2 - .../functional/functional_schema_template.sql | 156 +-- testdata/datasets/tpcds/tpcds_kudu_template.sql | 945 +++++++++++++++++++ testdata/datasets/tpch/tpch_kudu_template.sql | 198 ++++ testdata/datasets/tpch/tpch_schema_template.sql | 120 +-- .../queries/PlannerTest/kudu.test | 28 +- .../queries/PlannerTest/subquery-rewrite.test | 23 + .../queries/QueryTest/alter-table.test | 38 +- .../queries/QueryTest/analytic-fns.test | 14 + .../queries/QueryTest/create_kudu.test | 90 -- .../queries/QueryTest/joins.test | 10 + .../queries/QueryTest/kudu-scan-node.test | 34 +- .../queries/QueryTest/kudu-show-create.test | 16 - .../queries/QueryTest/kudu_alter.test | 21 +- .../queries/QueryTest/kudu_create.test | 105 +++ .../queries/QueryTest/kudu_crud.test | 161 ++-- .../queries/QueryTest/kudu_partition_ddl.test | 103 +- .../queries/QueryTest/kudu_stats.test | 12 +- .../queries/QueryTest/mt-dop.test | 9 + .../queries/QueryTest/udf-errors.test | 48 + .../functional-query/queries/QueryTest/udf.test | 14 + .../workloads/tpcds/queries/tpcds-kudu-q19.test | 39 + .../workloads/tpcds/queries/tpcds-kudu-q27.test | 36 + .../workloads/tpcds/queries/tpcds-kudu-q3.test | 32 + .../workloads/tpcds/queries/tpcds-kudu-q34.test | 47 + .../workloads/tpcds/queries/tpcds-kudu-q42.test | 29 + .../workloads/tpcds/queries/tpcds-kudu-q43.test | 37 + .../workloads/tpcds/queries/tpcds-kudu-q46.test | 81 ++ .../workloads/tpcds/queries/tpcds-kudu-q47.test | 53 ++ .../workloads/tpcds/queries/tpcds-kudu-q52.test | 28 + .../workloads/tpcds/queries/tpcds-kudu-q53.test | 38 + .../workloads/tpcds/queries/tpcds-kudu-q55.test | 25 + .../workloads/tpcds/queries/tpcds-kudu-q59.test | 84 ++ .../workloads/tpcds/queries/tpcds-kudu-q6.test | 28 + .../workloads/tpcds/queries/tpcds-kudu-q61.test | 43 + .../workloads/tpcds/queries/tpcds-kudu-q63.test | 38 + .../workloads/tpcds/queries/tpcds-kudu-q65.test | 63 ++ .../workloads/tpcds/queries/tpcds-kudu-q68.test | 62 ++ .../workloads/tpcds/queries/tpcds-kudu-q7.test | 32 + .../workloads/tpcds/queries/tpcds-kudu-q73.test | 51 + .../workloads/tpcds/queries/tpcds-kudu-q79.test | 61 ++ .../workloads/tpcds/queries/tpcds-kudu-q8.test | 71 ++ .../workloads/tpcds/queries/tpcds-kudu-q88.test | 93 ++ .../workloads/tpcds/queries/tpcds-kudu-q89.test | 33 + .../workloads/tpcds/queries/tpcds-kudu-q96.test | 17 + .../workloads/tpcds/queries/tpcds-kudu-q98.test | 34 + tests/beeswax/impala_beeswax.py | 4 +- tests/common/__init__.py | 4 +- tests/common/kudu_test_suite.py | 148 +++ tests/comparison/db_connection.py | 5 + tests/conftest.py | 10 +- tests/custom_cluster/test_client_ssl.py | 1 + tests/custom_cluster/test_kudu.py | 53 ++ tests/metadata/test_ddl.py | 3 +- tests/metadata/test_show_create_table.py | 5 - tests/query_test/test_cancellation.py | 49 +- tests/query_test/test_kudu.py | 560 ++++++++--- tests/query_test/test_mt_dop.py | 47 + tests/query_test/test_udfs.py | 18 + tests/stress/concurrent_select.py | 30 +- 241 files changed, 7624 insertions(+), 3492 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/eae4f307/bin/impala-config.sh ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/eae4f307/fe/src/main/java/org/apache/impala/analysis/ColumnDef.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/eae4f307/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/eae4f307/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/eae4f307/fe/src/main/java/org/apache/impala/service/Frontend.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/eae4f307/fe/src/main/java/org/apache/impala/service/JniFrontend.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/eae4f307/fe/src/test/java/org/apache/impala/analysis/ParserTest.java ----------------------------------------------------------------------
