Merge remote-tracking branch 'origin/master' into hadoop-next Change-Id: I493d811bb72949f315fd131ae7cbf886a3d0efd5
Project: http://git-wip-us.apache.org/repos/asf/incubator-impala/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-impala/commit/32dfcebf Tree: http://git-wip-us.apache.org/repos/asf/incubator-impala/tree/32dfcebf Diff: http://git-wip-us.apache.org/repos/asf/incubator-impala/diff/32dfcebf Branch: refs/heads/hadoop-next Commit: 32dfcebfa2316316daa9ec901009084e5b3a7ad3 Parents: eae4f30 0a654b3 Author: Tim Armstrong <[email protected]> Authored: Wed Nov 16 13:23:01 2016 -0800 Committer: Tim Armstrong <[email protected]> Committed: Thu Nov 17 08:07:40 2016 -0800 ---------------------------------------------------------------------- .clang-tidy | 94 ++ .gitignore | 1 - CMakeLists.txt | 3 +- EXPORT_CONTROL.md | 45 + README.md | 7 +- be/CMakeLists.txt | 20 +- be/generated-sources/gen-cpp/CMakeLists.txt | 2 + be/src/benchmarks/bloom-filter-benchmark.cc | 179 +-- be/src/benchmarks/expr-benchmark.cc | 2 +- be/src/benchmarks/hash-benchmark.cc | 2 +- be/src/benchmarks/in-predicate-benchmark.cc | 8 +- be/src/benchmarks/parse-timestamp-benchmark.cc | 15 - be/src/catalog/catalog-server.cc | 2 +- be/src/catalog/catalog-server.h | 2 +- be/src/catalog/catalog.cc | 23 +- be/src/codegen/codegen-anyval.cc | 64 +- be/src/codegen/codegen-anyval.h | 55 +- be/src/codegen/gen_ir_descriptions.py | 8 +- be/src/codegen/impala-ir.cc | 6 + be/src/codegen/llvm-codegen-test.cc | 16 +- be/src/codegen/llvm-codegen.cc | 56 +- be/src/codegen/llvm-codegen.h | 43 +- be/src/common/compiler-util.h | 3 - be/src/common/global-flags.cc | 14 + be/src/common/init.cc | 2 +- be/src/common/logging.h | 5 - be/src/common/status.h | 10 +- be/src/exec/aggregation-node.cc | 78 +- be/src/exec/analytic-eval-node.cc | 32 +- be/src/exec/analytic-eval-node.h | 6 +- be/src/exec/base-sequence-scanner.cc | 8 +- be/src/exec/data-sink.cc | 17 +- be/src/exec/data-sink.h | 9 +- be/src/exec/delimited-text-parser.cc | 12 +- be/src/exec/delimited-text-parser.h | 62 +- be/src/exec/exec-node.cc | 4 +- be/src/exec/exec-node.h | 8 +- be/src/exec/external-data-source-executor.h | 2 +- be/src/exec/hash-join-node.cc | 12 +- be/src/exec/hash-table-test.cc | 2 +- be/src/exec/hash-table.cc | 42 +- be/src/exec/hbase-scan-node.h | 2 +- be/src/exec/hdfs-avro-scanner.cc | 45 +- be/src/exec/hdfs-parquet-scanner.cc | 91 +- be/src/exec/hdfs-parquet-scanner.h | 7 +- be/src/exec/hdfs-parquet-table-writer.cc | 4 +- be/src/exec/hdfs-rcfile-scanner.cc | 9 +- be/src/exec/hdfs-rcfile-scanner.h | 4 +- be/src/exec/hdfs-scan-node-base.cc | 28 +- be/src/exec/hdfs-scan-node-base.h | 11 +- be/src/exec/hdfs-scan-node.cc | 1 - be/src/exec/hdfs-scan-node.h | 5 - be/src/exec/hdfs-scanner.cc | 15 +- be/src/exec/hdfs-scanner.h | 13 +- be/src/exec/hdfs-sequence-scanner.cc | 2 +- be/src/exec/hdfs-table-sink.cc | 2 +- be/src/exec/hdfs-table-writer.h | 2 +- be/src/exec/hdfs-text-scanner.cc | 10 +- be/src/exec/kudu-scan-node.cc | 10 +- be/src/exec/kudu-scan-node.h | 4 +- be/src/exec/kudu-scanner.cc | 188 +-- be/src/exec/kudu-scanner.h | 28 - be/src/exec/kudu-table-sink.cc | 112 +- be/src/exec/kudu-table-sink.h | 36 +- be/src/exec/kudu-util.cc | 11 + be/src/exec/kudu-util.h | 5 + be/src/exec/nested-loop-join-builder.cc | 2 +- be/src/exec/nested-loop-join-node.cc | 2 +- be/src/exec/old-hash-table.cc | 10 +- be/src/exec/parquet-column-readers.h | 2 +- be/src/exec/partitioned-aggregation-node-ir.cc | 2 +- be/src/exec/partitioned-aggregation-node.cc | 670 ++++++----- be/src/exec/partitioned-aggregation-node.h | 25 +- be/src/exec/partitioned-hash-join-builder-ir.cc | 4 +- be/src/exec/partitioned-hash-join-builder.cc | 14 +- be/src/exec/partitioned-hash-join-node.cc | 46 +- be/src/exec/plan-root-sink.cc | 5 +- be/src/exec/plan-root-sink.h | 25 +- be/src/exec/scanner-context.cc | 2 +- be/src/exec/scanner-context.inline.h | 3 +- be/src/exec/text-converter.cc | 12 +- be/src/exec/union-node.cc | 192 ++- be/src/exec/union-node.h | 36 +- be/src/exec/write-stream.h | 2 +- .../experiments/bit-stream-utils.8byte.inline.h | 4 +- be/src/experiments/tuple-splitter-test.cc | 1 - be/src/exprs/agg-fn-evaluator.cc | 29 +- be/src/exprs/aggregate-functions-ir.cc | 60 +- be/src/exprs/aggregate-functions.h | 9 +- be/src/exprs/anyval-util.cc | 37 +- be/src/exprs/anyval-util.h | 44 +- be/src/exprs/case-expr.cc | 17 +- be/src/exprs/compound-predicates.cc | 7 +- be/src/exprs/compound-predicates.h | 2 +- be/src/exprs/expr-codegen-test.cc | 2 +- be/src/exprs/expr-context.h | 11 +- be/src/exprs/expr-test.cc | 8 +- be/src/exprs/expr.cc | 93 +- be/src/exprs/expr.h | 20 +- be/src/exprs/hive-udf-call.cc | 29 +- be/src/exprs/in-predicate-ir.cc | 110 -- be/src/exprs/in-predicate.h | 152 ++- be/src/exprs/like-predicate.cc | 30 +- be/src/exprs/literal.cc | 9 +- be/src/exprs/null-literal.cc | 2 +- be/src/exprs/scalar-fn-call.cc | 232 ++-- be/src/exprs/scalar-fn-call.h | 24 +- be/src/exprs/slot-ref.cc | 51 +- be/src/exprs/string-functions-ir.cc | 6 +- be/src/exprs/utility-functions-ir.cc | 1 + be/src/rpc/TAcceptQueueServer.h | 6 +- be/src/rpc/authentication.cc | 1 + be/src/rpc/authentication.h | 5 +- be/src/rpc/thrift-server.h | 2 + be/src/runtime/CMakeLists.txt | 1 + be/src/runtime/buffered-block-mgr.cc | 27 +- be/src/runtime/buffered-block-mgr.h | 2 +- be/src/runtime/buffered-tuple-stream-test.cc | 81 +- be/src/runtime/buffered-tuple-stream.cc | 39 +- be/src/runtime/buffered-tuple-stream.h | 58 +- be/src/runtime/coordinator.cc | 478 ++------ be/src/runtime/coordinator.h | 87 +- be/src/runtime/data-stream-sender.cc | 4 +- be/src/runtime/data-stream-sender.h | 1 - be/src/runtime/decimal-test.cc | 18 + be/src/runtime/descriptors.cc | 133 ++- be/src/runtime/descriptors.h | 49 +- be/src/runtime/disk-io-mgr-internal.h | 3 +- be/src/runtime/disk-io-mgr-scan-range.cc | 93 +- be/src/runtime/disk-io-mgr-stress-test.cc | 1 + be/src/runtime/disk-io-mgr-stress.cc | 16 +- be/src/runtime/disk-io-mgr-stress.h | 8 +- be/src/runtime/disk-io-mgr-test.cc | 102 +- be/src/runtime/disk-io-mgr.cc | 160 +-- be/src/runtime/disk-io-mgr.h | 163 ++- be/src/runtime/exec-env.h | 2 +- be/src/runtime/free-pool-test.cc | 41 +- be/src/runtime/free-pool.h | 7 +- be/src/runtime/hbase-table.cc | 2 +- be/src/runtime/hdfs-fs-cache.h | 2 +- be/src/runtime/mem-pool-test.cc | 38 +- be/src/runtime/mem-pool.cc | 8 +- be/src/runtime/mem-pool.h | 72 +- be/src/runtime/multi-precision-test.cc | 2 +- be/src/runtime/plan-fragment-executor.cc | 42 +- be/src/runtime/plan-fragment-executor.h | 6 +- be/src/runtime/row-batch-test.cc | 18 +- be/src/runtime/row-batch.cc | 53 +- be/src/runtime/row-batch.h | 92 +- be/src/runtime/runtime-filter-bank.h | 2 +- be/src/runtime/runtime-filter.h | 2 +- be/src/runtime/scoped-buffer.h | 2 +- be/src/runtime/sorted-run-merger.cc | 5 +- be/src/runtime/sorted-run-merger.h | 6 +- be/src/runtime/sorter.cc | 15 +- be/src/runtime/string-buffer.h | 7 +- be/src/runtime/string-compare-test.cc | 67 ++ be/src/runtime/string-value.inline.h | 4 +- be/src/runtime/timestamp-parse-util.cc | 2 +- be/src/runtime/timestamp-value.cc | 4 +- be/src/runtime/tmp-file-mgr-test.cc | 2 +- be/src/runtime/tmp-file-mgr.cc | 10 +- be/src/runtime/tmp-file-mgr.h | 2 +- be/src/runtime/tuple.cc | 5 +- be/src/runtime/types.h | 29 +- be/src/scheduling/admission-controller.cc | 2 + be/src/scheduling/query-schedule.cc | 175 +-- be/src/scheduling/query-schedule.h | 77 +- be/src/scheduling/simple-scheduler-test-util.cc | 10 +- be/src/scheduling/simple-scheduler-test-util.h | 14 +- be/src/scheduling/simple-scheduler-test.cc | 2 +- be/src/scheduling/simple-scheduler.cc | 443 +++---- be/src/scheduling/simple-scheduler.h | 81 +- be/src/service/fe-support.cc | 19 +- be/src/service/frontend.cc | 35 +- be/src/service/impala-beeswax-server.cc | 3 +- be/src/service/impala-hs2-server.cc | 3 +- be/src/service/impala-http-handler.cc | 28 +- be/src/service/impala-server.cc | 38 +- be/src/service/impala-server.h | 80 +- be/src/service/query-exec-state.cc | 62 +- be/src/service/query-exec-state.h | 33 +- be/src/service/query-options.cc | 5 + be/src/service/query-options.h | 5 +- be/src/service/query-result-set.cc | 2 +- be/src/statestore/failure-detector.h | 2 +- be/src/statestore/statestore-subscriber.h | 2 +- be/src/statestore/statestore.cc | 4 +- be/src/statestore/statestore.h | 31 +- be/src/testutil/test-macros.h | 55 - be/src/testutil/test-udas.cc | 51 +- be/src/testutil/test-udas.h | 2 +- be/src/testutil/test-udfs.cc | 6 +- be/src/thirdparty/mustache/mustache.cc | 2 + be/src/transport/TSasl.h | 9 +- be/src/transport/TSaslClientTransport.h | 8 +- be/src/transport/TSaslServerTransport.h | 6 +- be/src/transport/TSaslTransport.h | 6 +- be/src/udf/uda-test-harness.h | 6 +- be/src/udf/uda-test.cc | 2 +- be/src/udf/udf-internal.h | 63 +- be/src/udf/udf-ir.cc | 2 - be/src/udf/udf-test-harness.cc | 2 +- be/src/udf/udf.cc | 30 +- be/src/udf/udf.h | 17 +- be/src/udf_samples/uda-sample.cc | 2 +- be/src/util/CMakeLists.txt | 1 + be/src/util/aligned-new.h | 55 + be/src/util/avro-util.cc | 42 +- be/src/util/avro-util.h | 7 +- be/src/util/backend-gflag-util.cc | 76 ++ be/src/util/backend-gflag-util.h | 31 + be/src/util/benchmark-test.cc | 2 +- be/src/util/bit-stream-utils.inline.h | 2 + be/src/util/bit-util-test.cc | 7 +- be/src/util/bit-util.h | 2 +- be/src/util/blocking-queue-test.cc | 2 +- be/src/util/blocking-queue.h | 5 +- be/src/util/bloom-filter-test.cc | 57 +- be/src/util/bloom-filter.cc | 40 +- be/src/util/buffer-builder.h | 22 +- be/src/util/compress.cc | 2 +- be/src/util/decompress.cc | 2 +- be/src/util/metrics.h | 6 +- be/src/util/minidump.cc | 10 +- be/src/util/network-util.cc | 1 + be/src/util/parquet-reader.cc | 4 +- be/src/util/periodic-counter-updater.h | 2 +- be/src/util/pprof-path-handlers.cc | 2 +- be/src/util/progress-updater.cc | 8 +- be/src/util/rle-test.cc | 4 +- be/src/util/runtime-profile-counters.h | 74 +- be/src/util/runtime-profile-test.cc | 59 + be/src/util/runtime-profile.cc | 123 +- be/src/util/runtime-profile.h | 17 +- be/src/util/thread-pool.h | 3 +- be/src/util/tuple-row-compare.cc | 2 +- be/src/util/webserver.cc | 6 + bin/bootstrap_development.sh | 2 +- bin/check-rat-report.py | 13 +- bin/distcc/README.md | 16 +- bin/generate_minidump_collection_testdata.py | 6 +- bin/impala-config.sh | 4 +- bin/load-data.py | 12 + bin/make_impala.sh | 6 +- bin/rat_exclude_files.txt | 2 + bin/remote_data_load.py | 560 +++++++++ bin/run-backend-tests.sh | 1 + bin/run_clang_tidy.sh | 54 + buildall.sh | 9 + cmake_modules/asan_toolchain.cmake | 44 - cmake_modules/clang_toolchain.cmake | 54 + common/thrift/BackendGflags.thrift | 55 + common/thrift/CMakeLists.txt | 11 +- common/thrift/CatalogObjects.thrift | 40 +- common/thrift/CatalogService.thrift | 2 +- common/thrift/DataSinks.thrift | 4 +- common/thrift/Descriptors.thrift | 1 + common/thrift/Frontend.thrift | 61 +- common/thrift/ImpalaInternalService.thrift | 30 +- common/thrift/ImpalaService.thrift | 4 + common/thrift/JniCatalog.thrift | 24 +- common/thrift/Planner.thrift | 4 +- common/thrift/RuntimeProfile.thrift | 13 + common/thrift/cli_service.thrift | 1126 ------------------ common/thrift/generate_error_codes.py | 12 +- common/thrift/generate_metrics.py | 2 +- fe/src/main/cup/sql-parser.cup | 229 ++-- .../analysis/AlterTableDropPartitionStmt.java | 22 +- .../analysis/AlterTableSetCachedStmt.java | 30 +- .../analysis/AlterTableSetFileFormatStmt.java | 12 +- .../analysis/AlterTableSetLocationStmt.java | 50 +- .../impala/analysis/AlterTableSetStmt.java | 18 +- .../analysis/AlterTableSetTblProperties.java | 22 +- .../apache/impala/analysis/AlterTableStmt.java | 24 +- .../apache/impala/analysis/AlterViewStmt.java | 3 +- .../apache/impala/analysis/AnalysisContext.java | 72 +- .../org/apache/impala/analysis/Analyzer.java | 52 +- .../apache/impala/analysis/BaseTableRef.java | 2 +- .../impala/analysis/BetweenPredicate.java | 111 +- .../impala/analysis/CollectionTableRef.java | 2 +- .../org/apache/impala/analysis/ColumnDef.java | 7 +- .../impala/analysis/CompoundPredicate.java | 26 +- .../impala/analysis/ComputeStatsStmt.java | 115 +- .../analysis/CreateOrAlterViewStmtBase.java | 8 +- .../analysis/CreateTableAsSelectStmt.java | 23 +- .../apache/impala/analysis/CreateTableStmt.java | 15 +- .../apache/impala/analysis/CreateViewStmt.java | 3 +- .../org/apache/impala/analysis/DeleteStmt.java | 14 +- .../apache/impala/analysis/DescriptorTable.java | 80 +- .../apache/impala/analysis/DistributeParam.java | 150 +-- .../apache/impala/analysis/DropStatsStmt.java | 55 +- .../apache/impala/analysis/ExistsPredicate.java | 10 +- .../java/org/apache/impala/analysis/Expr.java | 21 +- .../apache/impala/analysis/InlineViewRef.java | 19 +- .../org/apache/impala/analysis/InsertStmt.java | 305 +++-- .../org/apache/impala/analysis/LiteralExpr.java | 2 +- .../org/apache/impala/analysis/ModifyStmt.java | 27 +- .../apache/impala/analysis/PartitionSet.java | 199 ++++ .../apache/impala/analysis/PartitionSpec.java | 62 +- .../impala/analysis/PartitionSpecBase.java | 94 ++ .../org/apache/impala/analysis/QueryStmt.java | 41 +- .../apache/impala/analysis/RangePartition.java | 227 ++++ .../org/apache/impala/analysis/SelectList.java | 11 + .../apache/impala/analysis/SelectListItem.java | 3 +- .../org/apache/impala/analysis/SelectStmt.java | 32 +- .../apache/impala/analysis/ShowFilesStmt.java | 44 +- .../apache/impala/analysis/SlotDescriptor.java | 3 +- .../org/apache/impala/analysis/SortInfo.java | 63 +- .../apache/impala/analysis/StatementBase.java | 13 +- .../apache/impala/analysis/StmtRewriter.java | 30 +- .../org/apache/impala/analysis/Subquery.java | 17 +- .../org/apache/impala/analysis/TableRef.java | 20 +- .../apache/impala/analysis/TupleDescriptor.java | 83 +- .../org/apache/impala/analysis/UnionStmt.java | 187 +-- .../org/apache/impala/analysis/UpdateStmt.java | 21 +- .../org/apache/impala/authorization/User.java | 26 +- .../org/apache/impala/catalog/BuiltinsDb.java | 20 +- .../impala/catalog/CatalogServiceCatalog.java | 52 +- .../java/org/apache/impala/catalog/Column.java | 2 +- .../apache/impala/catalog/DataSourceTable.java | 9 +- .../org/apache/impala/catalog/HBaseTable.java | 8 +- .../org/apache/impala/catalog/HdfsTable.java | 73 +- .../apache/impala/catalog/IncompleteTable.java | 18 +- .../org/apache/impala/catalog/KuduTable.java | 58 +- .../java/org/apache/impala/catalog/Table.java | 35 +- .../java/org/apache/impala/catalog/TableId.java | 42 - .../org/apache/impala/catalog/TableLoader.java | 9 +- .../apache/impala/catalog/TableLoadingMgr.java | 4 +- .../java/org/apache/impala/catalog/View.java | 12 +- .../apache/impala/common/FileSystemUtil.java | 13 + .../main/java/org/apache/impala/common/Id.java | 2 - .../org/apache/impala/common/RuntimeEnv.java | 12 - .../impala/planner/DataSourceScanNode.java | 4 +- .../apache/impala/planner/HBaseScanNode.java | 6 +- .../apache/impala/planner/HBaseTableSink.java | 3 +- .../impala/planner/HdfsPartitionPruner.java | 50 +- .../org/apache/impala/planner/HdfsScanNode.java | 6 +- .../apache/impala/planner/HdfsTableSink.java | 3 +- .../org/apache/impala/planner/JoinTableId.java | 6 +- .../org/apache/impala/planner/KuduScanNode.java | 22 +- .../apache/impala/planner/KuduTableSink.java | 20 +- .../org/apache/impala/planner/PlanNode.java | 8 +- .../java/org/apache/impala/planner/Planner.java | 55 +- .../org/apache/impala/planner/ScanNode.java | 6 +- .../impala/planner/SingleNodePlanner.java | 6 +- .../org/apache/impala/planner/TableSink.java | 16 +- .../org/apache/impala/planner/UnionNode.java | 17 +- .../impala/rewrite/BetweenToCompoundRule.java | 63 + .../apache/impala/rewrite/ExprRewriteRule.java | 39 + .../org/apache/impala/rewrite/ExprRewriter.java | 97 ++ .../rewrite/ExtractCommonConjunctRule.java | 102 ++ .../apache/impala/service/BackendConfig.java | 62 +- .../impala/service/CatalogOpExecutor.java | 404 ++++--- .../org/apache/impala/service/FeSupport.java | 18 - .../org/apache/impala/service/Frontend.java | 222 +--- .../org/apache/impala/service/JniCatalog.java | 40 +- .../org/apache/impala/service/JniFrontend.java | 24 +- .../impala/service/KuduCatalogOpExecutor.java | 71 +- .../java/org/apache/impala/util/KuduUtil.java | 180 ++- fe/src/main/jflex/sql-scanner.flex | 1 + .../apache/impala/analysis/AnalyzeDDLTest.java | 521 +++++--- .../impala/analysis/AnalyzeModifyStmtsTest.java | 2 - .../impala/analysis/AnalyzeStmtsTest.java | 26 +- .../impala/analysis/AnalyzeSubqueriesTest.java | 10 +- .../impala/analysis/AnalyzeUpsertStmtTest.java | 132 ++ .../apache/impala/analysis/AnalyzerTest.java | 8 + .../impala/analysis/ExprRewriteRulesTest.java | 201 ++++ .../impala/analysis/ExprRewriterTest.java | 175 +++ .../org/apache/impala/analysis/ExprTest.java | 3 +- .../org/apache/impala/analysis/ParserTest.java | 198 +-- .../org/apache/impala/analysis/ToSqlTest.java | 30 + .../apache/impala/common/FrontendTestBase.java | 14 +- .../impala/hive/executor/UdfExecutorTest.java | 188 +-- .../org/apache/impala/planner/PlannerTest.java | 6 + .../apache/impala/planner/PlannerTestBase.java | 232 ++-- .../testutil/CatalogServiceTestCatalog.java | 3 +- .../org/apache/impala/util/KuduUtilTest.java | 120 -- infra/python/deps/download_requirements | 2 +- infra/python/deps/pip_download.py | 10 +- infra/python/deps/requirements.txt | 2 +- shell/impala_client.py | 2 +- testdata/bad_avro_snap/README | 13 +- .../bad_avro_snap/invalid_decimal_schema.avro | Bin 0 -> 328 bytes testdata/bin/compute-table-stats.sh | 8 +- testdata/bin/copy-udfs-udas.sh | 8 +- testdata/bin/create-load-data.sh | 176 ++- testdata/bin/create-table-many-blocks.sh | 14 +- testdata/bin/generate-schema-statements.py | 3 +- testdata/bin/load-test-warehouse-snapshot.sh | 37 +- testdata/bin/load_nested.py | 13 +- testdata/bin/run-step.sh | 1 + testdata/bin/setup-hdfs-env.sh | 32 +- testdata/bin/wait-for-hiveserver2.py | 12 +- testdata/bin/wait-for-metastore.py | 2 +- .../functional/functional_schema_template.sql | 39 +- .../datasets/functional/schema_constraints.csv | 1 + testdata/datasets/tpcds/tpcds_kudu_template.sql | 246 ++-- testdata/datasets/tpch/tpch_kudu_template.sql | 102 +- .../impala/datagenerator/TestDataGenerator.java | 5 + .../queries/PlannerTest/insert.test | 135 +++ .../queries/PlannerTest/kudu-delete.test | 18 - .../queries/PlannerTest/kudu-update.test | 14 - .../queries/PlannerTest/kudu-upsert.test | 92 ++ .../queries/PlannerTest/kudu.test | 97 +- .../queries/PlannerTest/mt-dop-validation.test | 50 +- .../queries/PlannerTest/subquery-rewrite.test | 34 +- .../queries/PlannerTest/tpcds-all.test | 12 +- .../queries/PlannerTest/tpch-all.test | 12 +- .../queries/PlannerTest/tpch-kudu.test | 5 +- .../queries/PlannerTest/tpch-nested.test | 12 +- .../queries/PlannerTest/tpch-views.test | 4 +- .../queries/DataErrorsTest/avro-errors.test | 8 + .../queries/QueryTest/alter-table.test | 45 + .../queries/QueryTest/avro-schema-changes.test | 6 +- .../queries/QueryTest/compute-stats.test | 84 ++ .../queries/QueryTest/exprs.test | 29 + .../queries/QueryTest/hdfs-caching.test | 2 +- .../queries/QueryTest/insert.test | 14 + .../queries/QueryTest/java-udf.test | 125 +- .../queries/QueryTest/kudu-scan-node.test | 8 + .../QueryTest/kudu-timeouts-catalogd.test | 25 + .../QueryTest/kudu-timeouts-impalad.test | 13 + .../queries/QueryTest/kudu_alter.test | 6 + .../queries/QueryTest/kudu_create.test | 35 +- .../queries/QueryTest/kudu_crud.test | 271 ++++- .../queries/QueryTest/kudu_partition_ddl.test | 216 +++- .../queries/QueryTest/kudu_stats.test | 4 +- .../QueryTest/libs_with_same_filenames.test | 11 +- .../queries/QueryTest/load-java-udfs.test | 79 +- .../queries/QueryTest/mt-dop-compute-stats.test | 8 + .../queries/QueryTest/mt-dop-parquet.test | 7 + .../QueryTest/partition-ddl-predicates.test | 156 +++ .../queries/QueryTest/scanners.test | 26 + .../queries/QueryTest/uda-mem-limit.test | 3 - .../functional-query/queries/QueryTest/uda.test | 35 + .../queries/QueryTest/udf-codegen-required.test | 10 + .../queries/QueryTest/udf-errors.test | 37 +- .../queries/QueryTest/udf-mem-limit.test | 5 - .../functional-query/queries/QueryTest/udf.test | 7 - .../queries/QueryTest/union.test | 28 + .../queries/primitive_filter_in_predicate.test | 10 + tests/benchmark/report_benchmark_results.py | 2 +- tests/common/impala_service.py | 4 + tests/common/impala_test_suite.py | 24 +- tests/common/kudu_test_suite.py | 4 +- tests/common/test_result_verifier.py | 22 + tests/comparison/POSTGRES.txt | 84 ++ tests/comparison/README | 4 + tests/comparison/common.py | 46 +- tests/comparison/db_connection.py | 146 ++- tests/comparison/leopard/controller.py | 5 +- tests/comparison/leopard/impala_docker_env.py | 4 +- tests/comparison/tests/fake_query.py | 9 +- tests/comparison/tests/test_cursor.py | 184 +++ tests/conftest.py | 51 +- .../custom_cluster/test_admission_controller.py | 40 + .../test_hive_parquet_timestamp_conversion.py | 6 +- tests/custom_cluster/test_kudu.py | 24 + tests/custom_cluster/test_permanent_udfs.py | 9 + tests/failure/test_failpoints.py | 4 + tests/metadata/test_ddl.py | 4 + .../metadata/test_metadata_query_statements.py | 47 +- tests/metadata/test_show_create_table.py | 63 +- tests/performance/query_exec_functions.py | 10 +- tests/performance/query_executor.py | 1 + tests/query_test/test_aggregation.py | 124 +- tests/query_test/test_kudu.py | 62 +- tests/query_test/test_mt_dop.py | 59 +- tests/query_test/test_scanners.py | 18 + tests/query_test/test_udfs.py | 88 +- tests/run-custom-cluster-tests.sh | 5 +- tests/run-process-failure-tests.sh | 12 +- .../org/apache/impala/ReplaceStringUdf.java | 31 + .../java/org/apache/impala/UnresolvedUdf.java | 25 +- www/queries.tmpl | 6 + www/sessions.tmpl | 94 +- 477 files changed, 12455 insertions(+), 7916 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/32dfcebf/bin/impala-config.sh ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/32dfcebf/fe/src/main/java/org/apache/impala/analysis/ColumnDef.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/32dfcebf/fe/src/main/java/org/apache/impala/analysis/Subquery.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/32dfcebf/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java ---------------------------------------------------------------------- diff --cc fe/src/main/java/org/apache/impala/catalog/HdfsTable.java index 7326670,21b5359..6530502 --- a/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java +++ b/fe/src/main/java/org/apache/impala/catalog/HdfsTable.java @@@ -45,9 -47,9 +45,6 @@@ import org.apache.hadoop.hive.metastore import org.apache.hadoop.hive.metastore.api.StorageDescriptor; import org.apache.hadoop.hive.serde.serdeConstants; import org.apache.hadoop.util.StringUtils; --import org.slf4j.Logger; --import org.slf4j.LoggerFactory; -- import org.apache.impala.analysis.ColumnDef; import org.apache.impala.analysis.Expr; import org.apache.impala.analysis.LiteralExpr; @@@ -67,7 -70,7 +65,6 @@@ import org.apache.impala.thrift.TCatalo import org.apache.impala.thrift.TColumn; import org.apache.impala.thrift.THdfsFileBlock; import org.apache.impala.thrift.THdfsPartition; --import org.apache.impala.thrift.THdfsPartitionLocation; import org.apache.impala.thrift.THdfsTable; import org.apache.impala.thrift.TNetworkAddress; import org.apache.impala.thrift.TPartitionKeyValue; @@@ -86,14 -89,44 +83,14 @@@ import org.apache.impala.util.ListMap import org.apache.impala.util.MetaStoreUtil; import org.apache.impala.util.TAccessLevelUtil; import org.apache.impala.util.TResultRowBuilder; ++import org.slf4j.Logger; ++import org.slf4j.LoggerFactory; ++ import com.google.common.base.Preconditions; - import com.google.common.base.Strings; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; - import com.google.common.collect.ImmutableSet; - import com.google.common.collect.ImmutableMap; -import org.apache.avro.Schema; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.fs.BlockLocation; -import org.apache.hadoop.fs.BlockStorageLocation; -import org.apache.hadoop.fs.FileStatus; -import org.apache.hadoop.fs.FileSystem; -import org.apache.hadoop.fs.Path; -import org.apache.hadoop.fs.VolumeId; -import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hdfs.DFSConfigKeys; -import org.apache.hadoop.hdfs.DistributedFileSystem; -import org.apache.hadoop.hive.metastore.IMetaStoreClient; -import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.metastore.api.Partition; -import org.apache.hadoop.hive.metastore.api.StorageDescriptor; -import org.apache.hadoop.hive.serde.serdeConstants; -import org.apache.hadoop.util.StringUtils; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.IOException; -import java.net.URI; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.Collections; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.TreeMap; - /** * Internal representation of table-related metadata of a file-resident table on a * Hadoop filesystem. The table data can be accessed through libHDFS (which is more of http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/32dfcebf/fe/src/main/java/org/apache/impala/service/CatalogOpExecutor.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/32dfcebf/fe/src/main/java/org/apache/impala/service/Frontend.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/32dfcebf/fe/src/main/java/org/apache/impala/service/JniFrontend.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-impala/blob/32dfcebf/fe/src/test/java/org/apache/impala/analysis/ParserTest.java ----------------------------------------------------------------------
