Author: gunther Date: Thu Aug 28 03:15:13 2014 New Revision: 1621031 URL: http://svn.apache.org/r1621031 Log: Merge latest trunk into cbo branch. (Gunther Hagleitner)
Added: hive/branches/cbo/data/files/extrapolate_stats_full.txt - copied unchanged from r1621023, hive/trunk/data/files/extrapolate_stats_full.txt hive/branches/cbo/data/files/extrapolate_stats_partial.txt - copied unchanged from r1621023, hive/trunk/data/files/extrapolate_stats_partial.txt hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/IExtrapolatePartStatus.java - copied unchanged from r1621023, hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/IExtrapolatePartStatus.java hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/LinearExtrapolatePartStatus.java - copied unchanged from r1621023, hive/trunk/metastore/src/java/org/apache/hadoop/hive/metastore/LinearExtrapolatePartStatus.java hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/TimestampColumnStatistics.java - copied unchanged from r1621023, hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/io/orc/TimestampColumnStatistics.java hive/branches/cbo/ql/src/test/queries/clientnegative/authorization_create_tbl.q - copied unchanged from r1621023, hive/trunk/ql/src/test/queries/clientnegative/authorization_create_tbl.q hive/branches/cbo/ql/src/test/queries/clientnegative/authorization_create_view.q - copied unchanged from r1621023, hive/trunk/ql/src/test/queries/clientnegative/authorization_create_view.q hive/branches/cbo/ql/src/test/queries/clientnegative/authorization_ctas2.q - copied unchanged from r1621023, hive/trunk/ql/src/test/queries/clientnegative/authorization_ctas2.q hive/branches/cbo/ql/src/test/queries/clientpositive/extrapolate_part_stats_full.q - copied unchanged from r1621023, hive/trunk/ql/src/test/queries/clientpositive/extrapolate_part_stats_full.q hive/branches/cbo/ql/src/test/queries/clientpositive/extrapolate_part_stats_partial.q - copied unchanged from r1621023, hive/trunk/ql/src/test/queries/clientpositive/extrapolate_part_stats_partial.q hive/branches/cbo/ql/src/test/queries/clientpositive/orc_merge_incompat1.q - copied unchanged from r1621023, hive/trunk/ql/src/test/queries/clientpositive/orc_merge_incompat1.q hive/branches/cbo/ql/src/test/queries/clientpositive/orc_merge_incompat2.q - copied unchanged from r1621023, hive/trunk/ql/src/test/queries/clientpositive/orc_merge_incompat2.q hive/branches/cbo/ql/src/test/queries/clientpositive/orc_ppd_timestamp.q - copied unchanged from r1621023, hive/trunk/ql/src/test/queries/clientpositive/orc_ppd_timestamp.q hive/branches/cbo/ql/src/test/results/clientnegative/authorization_create_tbl.q.out - copied unchanged from r1621023, hive/trunk/ql/src/test/results/clientnegative/authorization_create_tbl.q.out hive/branches/cbo/ql/src/test/results/clientnegative/authorization_create_view.q.out - copied unchanged from r1621023, hive/trunk/ql/src/test/results/clientnegative/authorization_create_view.q.out hive/branches/cbo/ql/src/test/results/clientnegative/authorization_ctas2.q.out - copied unchanged from r1621023, hive/trunk/ql/src/test/results/clientnegative/authorization_ctas2.q.out hive/branches/cbo/ql/src/test/results/clientpositive/extrapolate_part_stats_full.q.out - copied unchanged from r1621023, hive/trunk/ql/src/test/results/clientpositive/extrapolate_part_stats_full.q.out hive/branches/cbo/ql/src/test/results/clientpositive/extrapolate_part_stats_partial.q.out - copied unchanged from r1621023, hive/trunk/ql/src/test/results/clientpositive/extrapolate_part_stats_partial.q.out hive/branches/cbo/ql/src/test/results/clientpositive/orc_merge_incompat1.q.out - copied unchanged from r1621023, hive/trunk/ql/src/test/results/clientpositive/orc_merge_incompat1.q.out hive/branches/cbo/ql/src/test/results/clientpositive/orc_merge_incompat2.q.out - copied unchanged from r1621023, hive/trunk/ql/src/test/results/clientpositive/orc_merge_incompat2.q.out hive/branches/cbo/ql/src/test/results/clientpositive/orc_ppd_timestamp.q.out - copied unchanged from r1621023, hive/trunk/ql/src/test/results/clientpositive/orc_ppd_timestamp.q.out hive/branches/cbo/service/src/java/org/apache/hive/service/cli/FetchType.java - copied unchanged from r1621023, hive/trunk/service/src/java/org/apache/hive/service/cli/FetchType.java hive/branches/cbo/service/src/java/org/apache/hive/service/cli/operation/LogDivertAppender.java - copied unchanged from r1621023, hive/trunk/service/src/java/org/apache/hive/service/cli/operation/LogDivertAppender.java hive/branches/cbo/service/src/java/org/apache/hive/service/cli/operation/OperationLog.java - copied unchanged from r1621023, hive/trunk/service/src/java/org/apache/hive/service/cli/operation/OperationLog.java hive/branches/cbo/service/src/java/org/apache/hive/service/server/ThreadFactoryWithGarbageCleanup.java - copied unchanged from r1621023, hive/trunk/service/src/java/org/apache/hive/service/server/ThreadFactoryWithGarbageCleanup.java hive/branches/cbo/service/src/java/org/apache/hive/service/server/ThreadWithGarbageCleanup.java - copied unchanged from r1621023, hive/trunk/service/src/java/org/apache/hive/service/server/ThreadWithGarbageCleanup.java hive/branches/cbo/service/src/test/org/apache/hive/service/cli/operation/ - copied from r1621023, hive/trunk/service/src/test/org/apache/hive/service/cli/operation/ Removed: hive/branches/cbo/ql/src/test/queries/clientnegative/authorize_create_tbl.q hive/branches/cbo/ql/src/test/queries/clientnegative/orc_merge1.q hive/branches/cbo/ql/src/test/queries/clientnegative/orc_merge2.q hive/branches/cbo/ql/src/test/queries/clientnegative/orc_merge3.q hive/branches/cbo/ql/src/test/queries/clientnegative/orc_merge4.q hive/branches/cbo/ql/src/test/queries/clientnegative/orc_merge5.q hive/branches/cbo/ql/src/test/results/clientnegative/authorize_create_tbl.q.out Modified: hive/branches/cbo/ (props changed) hive/branches/cbo/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java hive/branches/cbo/contrib/src/test/results/clientnegative/serde_regex.q.out hive/branches/cbo/contrib/src/test/results/clientpositive/fileformat_base64.q.out hive/branches/cbo/contrib/src/test/results/clientpositive/serde_regex.q.out hive/branches/cbo/hbase-handler/src/test/results/negative/cascade_dbdrop.q.out hive/branches/cbo/hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/CreateTableHook.java hive/branches/cbo/hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/HCatSemanticAnalyzer.java hive/branches/cbo/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestSemanticAnalysis.java hive/branches/cbo/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatPartition.java hive/branches/cbo/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java hive/branches/cbo/itests/src/test/resources/testconfiguration.properties hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java hive/branches/cbo/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java hive/branches/cbo/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java hive/branches/cbo/ql/src/gen/protobuf/gen-java/org/apache/hadoop/hive/ql/io/orc/OrcProto.java hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/FileSinkOperator.java hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/Utilities.java hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/TezTask.java hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/exec/vector/VectorFileSinkOperator.java hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/hooks/Entity.java hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/hooks/UpdateInputAccessTimeHook.java hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/index/IndexMetadataChangeTask.java hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/index/bitmap/BitmapIndexHandler.java hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/index/compact/CompactIndexHandler.java hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/merge/MergeMapper.java hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/ColumnStatisticsImpl.java hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/FileDump.java hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/OrcFileMergeMapper.java hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/RecordReaderImpl.java hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/orc/WriterImpl.java hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/PredicateLeaf.java hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/io/sarg/SearchArgumentImpl.java hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/metadata/Partition.java hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/BaseSemanticAnalyzer.java hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/DDLSemanticAnalyzer.java hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/HiveParser.g hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/IndexUpdater.java hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzer.java hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/SemanticAnalyzerFactory.java hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/parse/TaskCompiler.java hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/AlterTableDesc.java hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/HiveOperation.java hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/plan/PlanUtils.java hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/HiveOperationType.java hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/security/authorization/plugin/sqlstd/Operation2Privilege.java hive/branches/cbo/ql/src/java/org/apache/hadoop/hive/ql/stats/StatsUtils.java hive/branches/cbo/ql/src/protobuf/org/apache/hadoop/hive/ql/io/orc/orc_proto.proto hive/branches/cbo/ql/src/test/org/apache/hadoop/hive/ql/io/orc/TestFileDump.java hive/branches/cbo/ql/src/test/queries/clientpositive/add_part_exist.q hive/branches/cbo/ql/src/test/queries/clientpositive/alter1.q hive/branches/cbo/ql/src/test/queries/clientpositive/alter_char1.q hive/branches/cbo/ql/src/test/queries/clientpositive/alter_index.q hive/branches/cbo/ql/src/test/queries/clientpositive/alter_partition_coltype.q hive/branches/cbo/ql/src/test/queries/clientpositive/alter_skewed_table.q hive/branches/cbo/ql/src/test/queries/clientpositive/alter_varchar1.q hive/branches/cbo/ql/src/test/queries/clientpositive/alter_view_as_select.q hive/branches/cbo/ql/src/test/queries/clientpositive/alter_view_rename.q hive/branches/cbo/ql/src/test/queries/clientpositive/archive_multi.q hive/branches/cbo/ql/src/test/queries/clientpositive/create_or_replace_view.q hive/branches/cbo/ql/src/test/queries/clientpositive/drop_multi_partitions.q hive/branches/cbo/ql/src/test/queries/clientpositive/exchange_partition.q hive/branches/cbo/ql/src/test/queries/clientpositive/index_auto_empty.q hive/branches/cbo/ql/src/test/queries/clientpositive/input46.q hive/branches/cbo/ql/src/test/queries/clientpositive/optimize_nullscan.q hive/branches/cbo/ql/src/test/queries/clientpositive/touch.q hive/branches/cbo/ql/src/test/queries/clientpositive/unset_table_view_property.q hive/branches/cbo/ql/src/test/resources/orc-file-dump-dictionary-threshold.out hive/branches/cbo/ql/src/test/resources/orc-file-dump.out hive/branches/cbo/ql/src/test/results/clientnegative/alter_view_as_select_not_exist.q.out hive/branches/cbo/ql/src/test/results/clientnegative/alter_view_as_select_with_partition.q.out hive/branches/cbo/ql/src/test/results/clientnegative/alter_view_failure.q.out hive/branches/cbo/ql/src/test/results/clientnegative/alter_view_failure2.q.out hive/branches/cbo/ql/src/test/results/clientnegative/alter_view_failure4.q.out hive/branches/cbo/ql/src/test/results/clientnegative/alter_view_failure5.q.out hive/branches/cbo/ql/src/test/results/clientnegative/alter_view_failure6.q.out hive/branches/cbo/ql/src/test/results/clientnegative/alter_view_failure7.q.out hive/branches/cbo/ql/src/test/results/clientnegative/alter_view_failure9.q.out hive/branches/cbo/ql/src/test/results/clientnegative/analyze_view.q.out hive/branches/cbo/ql/src/test/results/clientnegative/authorization_drop_db_cascade.q.out hive/branches/cbo/ql/src/test/results/clientnegative/authorization_not_owner_drop_view.q.out hive/branches/cbo/ql/src/test/results/clientnegative/authorization_part.q.out hive/branches/cbo/ql/src/test/results/clientnegative/authorization_select_view.q.out hive/branches/cbo/ql/src/test/results/clientnegative/create_or_replace_view1.q.out hive/branches/cbo/ql/src/test/results/clientnegative/create_or_replace_view2.q.out hive/branches/cbo/ql/src/test/results/clientnegative/create_or_replace_view3.q.out hive/branches/cbo/ql/src/test/results/clientnegative/create_or_replace_view4.q.out hive/branches/cbo/ql/src/test/results/clientnegative/create_or_replace_view5.q.out hive/branches/cbo/ql/src/test/results/clientnegative/create_or_replace_view6.q.out hive/branches/cbo/ql/src/test/results/clientnegative/create_or_replace_view7.q.out hive/branches/cbo/ql/src/test/results/clientnegative/create_or_replace_view8.q.out hive/branches/cbo/ql/src/test/results/clientnegative/create_view_failure1.q.out hive/branches/cbo/ql/src/test/results/clientnegative/create_view_failure2.q.out hive/branches/cbo/ql/src/test/results/clientnegative/create_view_failure4.q.out hive/branches/cbo/ql/src/test/results/clientnegative/desc_failure3.q.out hive/branches/cbo/ql/src/test/results/clientnegative/drop_table_failure2.q.out hive/branches/cbo/ql/src/test/results/clientnegative/insert_view_failure.q.out hive/branches/cbo/ql/src/test/results/clientnegative/invalidate_view1.q.out hive/branches/cbo/ql/src/test/results/clientnegative/load_view_failure.q.out hive/branches/cbo/ql/src/test/results/clientnegative/recursive_view.q.out hive/branches/cbo/ql/src/test/results/clientnegative/unset_view_property.q.out hive/branches/cbo/ql/src/test/results/clientpositive/add_part_exist.q.out hive/branches/cbo/ql/src/test/results/clientpositive/allcolref_in_udf.q.out hive/branches/cbo/ql/src/test/results/clientpositive/alter1.q.out hive/branches/cbo/ql/src/test/results/clientpositive/alter_char1.q.out hive/branches/cbo/ql/src/test/results/clientpositive/alter_index.q.out hive/branches/cbo/ql/src/test/results/clientpositive/alter_partition_coltype.q.out hive/branches/cbo/ql/src/test/results/clientpositive/alter_rename_partition_authorization.q.out hive/branches/cbo/ql/src/test/results/clientpositive/alter_rename_table.q.out hive/branches/cbo/ql/src/test/results/clientpositive/alter_skewed_table.q.out hive/branches/cbo/ql/src/test/results/clientpositive/alter_table_update_status.q.out hive/branches/cbo/ql/src/test/results/clientpositive/alter_varchar1.q.out hive/branches/cbo/ql/src/test/results/clientpositive/alter_view_as_select.q.out hive/branches/cbo/ql/src/test/results/clientpositive/alter_view_rename.q.out hive/branches/cbo/ql/src/test/results/clientpositive/annotate_stats_part.q.out hive/branches/cbo/ql/src/test/results/clientpositive/archive_multi.q.out hive/branches/cbo/ql/src/test/results/clientpositive/authorization_1.q.out hive/branches/cbo/ql/src/test/results/clientpositive/authorization_2.q.out hive/branches/cbo/ql/src/test/results/clientpositive/authorization_3.q.out hive/branches/cbo/ql/src/test/results/clientpositive/authorization_4.q.out hive/branches/cbo/ql/src/test/results/clientpositive/authorization_6.q.out hive/branches/cbo/ql/src/test/results/clientpositive/authorization_8.q.out hive/branches/cbo/ql/src/test/results/clientpositive/authorization_cli_createtab.q.out hive/branches/cbo/ql/src/test/results/clientpositive/authorization_create_temp_table.q.out hive/branches/cbo/ql/src/test/results/clientpositive/authorization_owner_actions.q.out hive/branches/cbo/ql/src/test/results/clientpositive/authorization_view_sqlstd.q.out hive/branches/cbo/ql/src/test/results/clientpositive/autogen_colalias.q.out hive/branches/cbo/ql/src/test/results/clientpositive/cbo_correctness.q.out hive/branches/cbo/ql/src/test/results/clientpositive/char_nested_types.q.out hive/branches/cbo/ql/src/test/results/clientpositive/colstats_all_nulls.q.out hive/branches/cbo/ql/src/test/results/clientpositive/create_big_view.q.out hive/branches/cbo/ql/src/test/results/clientpositive/create_default_prop.q.out hive/branches/cbo/ql/src/test/results/clientpositive/create_like_tbl_props.q.out hive/branches/cbo/ql/src/test/results/clientpositive/create_like_view.q.out hive/branches/cbo/ql/src/test/results/clientpositive/create_or_replace_view.q.out hive/branches/cbo/ql/src/test/results/clientpositive/create_union_table.q.out hive/branches/cbo/ql/src/test/results/clientpositive/create_view.q.out hive/branches/cbo/ql/src/test/results/clientpositive/create_view_partitioned.q.out hive/branches/cbo/ql/src/test/results/clientpositive/create_view_translate.q.out hive/branches/cbo/ql/src/test/results/clientpositive/cross_product_check_1.q.out hive/branches/cbo/ql/src/test/results/clientpositive/cross_product_check_2.q.out hive/branches/cbo/ql/src/test/results/clientpositive/ctas.q.out hive/branches/cbo/ql/src/test/results/clientpositive/ctas_char.q.out hive/branches/cbo/ql/src/test/results/clientpositive/ctas_colname.q.out hive/branches/cbo/ql/src/test/results/clientpositive/ctas_date.q.out hive/branches/cbo/ql/src/test/results/clientpositive/ctas_uses_database_location.q.out hive/branches/cbo/ql/src/test/results/clientpositive/ctas_varchar.q.out hive/branches/cbo/ql/src/test/results/clientpositive/cte_2.q.out hive/branches/cbo/ql/src/test/results/clientpositive/database.q.out hive/branches/cbo/ql/src/test/results/clientpositive/database_drop.q.out hive/branches/cbo/ql/src/test/results/clientpositive/dbtxnmgr_ddl1.q.out hive/branches/cbo/ql/src/test/results/clientpositive/dbtxnmgr_query5.q.out hive/branches/cbo/ql/src/test/results/clientpositive/decimal_6.q.out hive/branches/cbo/ql/src/test/results/clientpositive/decimal_serde.q.out hive/branches/cbo/ql/src/test/results/clientpositive/describe_formatted_view_partitioned.q.out hive/branches/cbo/ql/src/test/results/clientpositive/describe_formatted_view_partitioned_json.q.out hive/branches/cbo/ql/src/test/results/clientpositive/describe_syntax.q.out hive/branches/cbo/ql/src/test/results/clientpositive/drop_multi_partitions.q.out hive/branches/cbo/ql/src/test/results/clientpositive/exchange_partition.q.out hive/branches/cbo/ql/src/test/results/clientpositive/explain_dependency.q.out hive/branches/cbo/ql/src/test/results/clientpositive/explain_logical.q.out hive/branches/cbo/ql/src/test/results/clientpositive/fileformat_sequencefile.q.out hive/branches/cbo/ql/src/test/results/clientpositive/fileformat_text.q.out hive/branches/cbo/ql/src/test/results/clientpositive/global_limit.q.out hive/branches/cbo/ql/src/test/results/clientpositive/groupby_duplicate_key.q.out hive/branches/cbo/ql/src/test/results/clientpositive/index_auto_empty.q.out hive/branches/cbo/ql/src/test/results/clientpositive/input15.q.out hive/branches/cbo/ql/src/test/results/clientpositive/input46.q.out hive/branches/cbo/ql/src/test/results/clientpositive/inputddl1.q.out hive/branches/cbo/ql/src/test/results/clientpositive/inputddl2.q.out hive/branches/cbo/ql/src/test/results/clientpositive/inputddl3.q.out hive/branches/cbo/ql/src/test/results/clientpositive/insert1.q.out hive/branches/cbo/ql/src/test/results/clientpositive/insert2_overwrite_partitions.q.out hive/branches/cbo/ql/src/test/results/clientpositive/join41.q.out hive/branches/cbo/ql/src/test/results/clientpositive/join_filters_overlap.q.out hive/branches/cbo/ql/src/test/results/clientpositive/join_view.q.out hive/branches/cbo/ql/src/test/results/clientpositive/lateral_view_noalias.q.out hive/branches/cbo/ql/src/test/results/clientpositive/lateral_view_outer.q.out hive/branches/cbo/ql/src/test/results/clientpositive/merge3.q.out hive/branches/cbo/ql/src/test/results/clientpositive/multi_insert_lateral_view.q.out hive/branches/cbo/ql/src/test/results/clientpositive/multi_join_union.q.out hive/branches/cbo/ql/src/test/results/clientpositive/nestedvirtual.q.out hive/branches/cbo/ql/src/test/results/clientpositive/nonmr_fetch.q.out hive/branches/cbo/ql/src/test/results/clientpositive/nullformat.q.out hive/branches/cbo/ql/src/test/results/clientpositive/nullformatCTAS.q.out hive/branches/cbo/ql/src/test/results/clientpositive/optimize_nullscan.q.out hive/branches/cbo/ql/src/test/results/clientpositive/orc_analyze.q.out hive/branches/cbo/ql/src/test/results/clientpositive/orc_createas1.q.out hive/branches/cbo/ql/src/test/results/clientpositive/parallel_orderby.q.out hive/branches/cbo/ql/src/test/results/clientpositive/parquet_ctas.q.out hive/branches/cbo/ql/src/test/results/clientpositive/parquet_join.q.out hive/branches/cbo/ql/src/test/results/clientpositive/partition_decode_name.q.out hive/branches/cbo/ql/src/test/results/clientpositive/partition_special_char.q.out hive/branches/cbo/ql/src/test/results/clientpositive/ppd_field_garbage.q.out hive/branches/cbo/ql/src/test/results/clientpositive/ppd_union_view.q.out hive/branches/cbo/ql/src/test/results/clientpositive/ptf.q.out hive/branches/cbo/ql/src/test/results/clientpositive/query_result_fileformat.q.out hive/branches/cbo/ql/src/test/results/clientpositive/quotedid_basic.q.out hive/branches/cbo/ql/src/test/results/clientpositive/rcfile_createas1.q.out hive/branches/cbo/ql/src/test/results/clientpositive/rcfile_default_format.q.out hive/branches/cbo/ql/src/test/results/clientpositive/sample_islocalmode_hook.q.out hive/branches/cbo/ql/src/test/results/clientpositive/semijoin.q.out hive/branches/cbo/ql/src/test/results/clientpositive/serde_regex.q.out hive/branches/cbo/ql/src/test/results/clientpositive/show_create_table_db_table.q.out hive/branches/cbo/ql/src/test/results/clientpositive/show_create_table_temp_table.q.out hive/branches/cbo/ql/src/test/results/clientpositive/show_create_table_view.q.out hive/branches/cbo/ql/src/test/results/clientpositive/skewjoin_noskew.q.out hive/branches/cbo/ql/src/test/results/clientpositive/smb_mapjoin9.q.out hive/branches/cbo/ql/src/test/results/clientpositive/stats5.q.out hive/branches/cbo/ql/src/test/results/clientpositive/stats_counter.q.out hive/branches/cbo/ql/src/test/results/clientpositive/str_to_map.q.out hive/branches/cbo/ql/src/test/results/clientpositive/subquery_exists.q.out hive/branches/cbo/ql/src/test/results/clientpositive/subquery_exists_having.q.out hive/branches/cbo/ql/src/test/results/clientpositive/subquery_notin.q.out hive/branches/cbo/ql/src/test/results/clientpositive/subquery_views.q.out hive/branches/cbo/ql/src/test/results/clientpositive/symlink_text_input_format.q.out hive/branches/cbo/ql/src/test/results/clientpositive/temp_table.q.out hive/branches/cbo/ql/src/test/results/clientpositive/temp_table_gb1.q.out hive/branches/cbo/ql/src/test/results/clientpositive/temp_table_join1.q.out hive/branches/cbo/ql/src/test/results/clientpositive/temp_table_names.q.out hive/branches/cbo/ql/src/test/results/clientpositive/temp_table_precedence.q.out hive/branches/cbo/ql/src/test/results/clientpositive/temp_table_subquery1.q.out hive/branches/cbo/ql/src/test/results/clientpositive/tez/cross_product_check_1.q.out hive/branches/cbo/ql/src/test/results/clientpositive/tez/cross_product_check_2.q.out hive/branches/cbo/ql/src/test/results/clientpositive/tez/ctas.q.out hive/branches/cbo/ql/src/test/results/clientpositive/tez/insert1.q.out hive/branches/cbo/ql/src/test/results/clientpositive/tez/optimize_nullscan.q.out hive/branches/cbo/ql/src/test/results/clientpositive/tez/orc_analyze.q.out hive/branches/cbo/ql/src/test/results/clientpositive/tez/ptf.q.out hive/branches/cbo/ql/src/test/results/clientpositive/tez/stats_counter.q.out hive/branches/cbo/ql/src/test/results/clientpositive/tez/subquery_exists.q.out hive/branches/cbo/ql/src/test/results/clientpositive/tez/temp_table.q.out hive/branches/cbo/ql/src/test/results/clientpositive/tez/tez_dml.q.out hive/branches/cbo/ql/src/test/results/clientpositive/tez/tez_union.q.out hive/branches/cbo/ql/src/test/results/clientpositive/tez/vector_decimal_aggregate.q.out hive/branches/cbo/ql/src/test/results/clientpositive/tez/vectorization_part_project.q.out hive/branches/cbo/ql/src/test/results/clientpositive/tez/vectorized_ptf.q.out hive/branches/cbo/ql/src/test/results/clientpositive/tez/vectorized_timestamp_funcs.q.out hive/branches/cbo/ql/src/test/results/clientpositive/touch.q.out hive/branches/cbo/ql/src/test/results/clientpositive/union24.q.out hive/branches/cbo/ql/src/test/results/clientpositive/union25.q.out hive/branches/cbo/ql/src/test/results/clientpositive/union27.q.out hive/branches/cbo/ql/src/test/results/clientpositive/union31.q.out hive/branches/cbo/ql/src/test/results/clientpositive/union32.q.out hive/branches/cbo/ql/src/test/results/clientpositive/union_top_level.q.out hive/branches/cbo/ql/src/test/results/clientpositive/unset_table_view_property.q.out hive/branches/cbo/ql/src/test/results/clientpositive/updateAccessTime.q.out hive/branches/cbo/ql/src/test/results/clientpositive/varchar_nested_types.q.out hive/branches/cbo/ql/src/test/results/clientpositive/vector_between_in.q.out hive/branches/cbo/ql/src/test/results/clientpositive/vector_decimal_aggregate.q.out hive/branches/cbo/ql/src/test/results/clientpositive/vector_decimal_expressions.q.out hive/branches/cbo/ql/src/test/results/clientpositive/vector_decimal_mapjoin.q.out hive/branches/cbo/ql/src/test/results/clientpositive/vector_decimal_math_funcs.q.out hive/branches/cbo/ql/src/test/results/clientpositive/vectorization_decimal_date.q.out hive/branches/cbo/ql/src/test/results/clientpositive/vectorized_ptf.q.out hive/branches/cbo/ql/src/test/results/clientpositive/view.q.out hive/branches/cbo/ql/src/test/results/clientpositive/view_cast.q.out hive/branches/cbo/ql/src/test/results/clientpositive/view_inputs.q.out hive/branches/cbo/ql/src/test/results/clientpositive/windowing.q.out hive/branches/cbo/ql/src/test/results/clientpositive/windowing_navfn.q.out hive/branches/cbo/ql/src/test/results/clientpositive/windowing_streaming.q.out hive/branches/cbo/service/if/TCLIService.thrift hive/branches/cbo/service/src/gen/thrift/gen-cpp/TCLIService_types.cpp hive/branches/cbo/service/src/gen/thrift/gen-cpp/TCLIService_types.h hive/branches/cbo/service/src/gen/thrift/gen-javabean/org/apache/hive/service/cli/thrift/TFetchResultsReq.java hive/branches/cbo/service/src/gen/thrift/gen-py/TCLIService/ttypes.py hive/branches/cbo/service/src/gen/thrift/gen-rb/t_c_l_i_service_types.rb hive/branches/cbo/service/src/java/org/apache/hive/service/cli/CLIService.java hive/branches/cbo/service/src/java/org/apache/hive/service/cli/CLIServiceClient.java hive/branches/cbo/service/src/java/org/apache/hive/service/cli/EmbeddedCLIServiceClient.java hive/branches/cbo/service/src/java/org/apache/hive/service/cli/ICLIService.java hive/branches/cbo/service/src/java/org/apache/hive/service/cli/operation/GetCatalogsOperation.java hive/branches/cbo/service/src/java/org/apache/hive/service/cli/operation/GetColumnsOperation.java hive/branches/cbo/service/src/java/org/apache/hive/service/cli/operation/GetFunctionsOperation.java hive/branches/cbo/service/src/java/org/apache/hive/service/cli/operation/GetSchemasOperation.java hive/branches/cbo/service/src/java/org/apache/hive/service/cli/operation/GetTableTypesOperation.java hive/branches/cbo/service/src/java/org/apache/hive/service/cli/operation/GetTablesOperation.java hive/branches/cbo/service/src/java/org/apache/hive/service/cli/operation/GetTypeInfoOperation.java hive/branches/cbo/service/src/java/org/apache/hive/service/cli/operation/HiveCommandOperation.java hive/branches/cbo/service/src/java/org/apache/hive/service/cli/operation/MetadataOperation.java hive/branches/cbo/service/src/java/org/apache/hive/service/cli/operation/Operation.java hive/branches/cbo/service/src/java/org/apache/hive/service/cli/operation/OperationManager.java hive/branches/cbo/service/src/java/org/apache/hive/service/cli/operation/SQLOperation.java hive/branches/cbo/service/src/java/org/apache/hive/service/cli/session/HiveSession.java hive/branches/cbo/service/src/java/org/apache/hive/service/cli/session/HiveSessionBase.java hive/branches/cbo/service/src/java/org/apache/hive/service/cli/session/HiveSessionImpl.java hive/branches/cbo/service/src/java/org/apache/hive/service/cli/session/SessionManager.java hive/branches/cbo/service/src/java/org/apache/hive/service/cli/thrift/ThriftBinaryCLIService.java hive/branches/cbo/service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIService.java hive/branches/cbo/service/src/java/org/apache/hive/service/cli/thrift/ThriftCLIServiceClient.java hive/branches/cbo/service/src/java/org/apache/hive/service/cli/thrift/ThriftHttpCLIService.java Propchange: hive/branches/cbo/ ------------------------------------------------------------------------------ Merged /hive/trunk:r1619930-1621023 Modified: hive/branches/cbo/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java?rev=1621031&r1=1621030&r2=1621031&view=diff ============================================================================== --- hive/branches/cbo/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java (original) +++ hive/branches/cbo/common/src/java/org/apache/hadoop/hive/conf/HiveConf.java Thu Aug 28 03:15:13 2014 @@ -54,7 +54,6 @@ import org.apache.hive.common.HiveCompat * Hive Configuration. */ public class HiveConf extends Configuration { - protected String hiveJar; protected Properties origProp; protected String auxJars; @@ -1408,6 +1407,14 @@ public class HiveConf extends Configurat "to construct a list exception handlers to handle exceptions thrown\n" + "by record readers"), + // operation log configuration + HIVE_SERVER2_LOGGING_OPERATION_ENABLED("hive.server2.logging.operation.enabled", true, + "When true, HS2 will save operation logs"), + HIVE_SERVER2_LOGGING_OPERATION_LOG_LOCATION("hive.server2.logging.operation.log.location", + "${system:java.io.tmpdir}" + File.separator + "${system:user.name}" + File.separator + + "operation_logs", + "Top level directory where operation logs are stored if logging functionality is enabled"), + // logging configuration HIVE_LOG4J_FILE("hive.log4j.file", "", "Hive log4j configuration file.\n" + @@ -1479,8 +1486,11 @@ public class HiveConf extends Configurat "Minimum number of worker threads when in HTTP mode."), HIVE_SERVER2_THRIFT_HTTP_MAX_WORKER_THREADS("hive.server2.thrift.http.max.worker.threads", 500, "Maximum number of worker threads when in HTTP mode."), - HIVE_SERVER2_THRIFT_HTTP_MAX_IDLE_TIME("hive.server2.thrift.http.max.idle.time", 1800000, + HIVE_SERVER2_THRIFT_HTTP_MAX_IDLE_TIME("hive.server2.thrift.http.max.idle.time", 1800000, "Maximum idle time in milliseconds for a connection on the server when in HTTP mode."), + HIVE_SERVER2_THRIFT_HTTP_WORKER_KEEPALIVE_TIME("hive.server2.thrift.http.worker.keepalive.time", 60, + "Keepalive time (in seconds) for an idle http worker thread. When number of workers > min workers, " + + "excess threads are killed after this time interval."), // binary transport settings HIVE_SERVER2_THRIFT_PORT("hive.server2.thrift.port", 10000, @@ -1503,7 +1513,9 @@ public class HiveConf extends Configurat "Minimum number of Thrift worker threads"), HIVE_SERVER2_THRIFT_MAX_WORKER_THREADS("hive.server2.thrift.max.worker.threads", 500, "Maximum number of Thrift worker threads"), - + HIVE_SERVER2_THRIFT_WORKER_KEEPALIVE_TIME("hive.server2.thrift.worker.keepalive.time", 60, + "Keepalive time (in seconds) for an idle worker thread. When number of workers > min workers, " + + "excess threads are killed after this time interval."), // Configuration for async thread pool in SessionManager HIVE_SERVER2_ASYNC_EXEC_THREADS("hive.server2.async.exec.threads", 100, "Number of threads in the async thread pool for HiveServer2"), Modified: hive/branches/cbo/contrib/src/test/results/clientnegative/serde_regex.q.out URL: http://svn.apache.org/viewvc/hive/branches/cbo/contrib/src/test/results/clientnegative/serde_regex.q.out?rev=1621031&r1=1621030&r2=1621031&view=diff ============================================================================== --- hive/branches/cbo/contrib/src/test/results/clientnegative/serde_regex.q.out (original) +++ hive/branches/cbo/contrib/src/test/results/clientnegative/serde_regex.q.out Thu Aug 28 03:15:13 2014 @@ -56,7 +56,7 @@ STAGE PLANS: serde properties: input.regex ([^ ]*) ([^ ]*) ([^ ]*) (-|\[[^\]]*\]) ([^ "]*|"[^"]*") (-|[0-9]*) (-|[0-9]*)(?: ([^ "]*|"[^"]*") ([^ "]*|"[^"]*"))? output.format.string %1$s %2$s %3$s %4$s %5$s %6$s %7$s %8$s %9$s - name: serde_regex + name: default.serde_regex PREHOOK: query: CREATE TABLE serde_regex( host STRING, Modified: hive/branches/cbo/contrib/src/test/results/clientpositive/fileformat_base64.q.out URL: http://svn.apache.org/viewvc/hive/branches/cbo/contrib/src/test/results/clientpositive/fileformat_base64.q.out?rev=1621031&r1=1621030&r2=1621031&view=diff ============================================================================== --- hive/branches/cbo/contrib/src/test/results/clientpositive/fileformat_base64.q.out (original) +++ hive/branches/cbo/contrib/src/test/results/clientpositive/fileformat_base64.q.out Thu Aug 28 03:15:13 2014 @@ -22,7 +22,7 @@ STAGE PLANS: columns: key int, value string input format: org.apache.hadoop.hive.contrib.fileformat.base64.Base64TextInputFormat output format: org.apache.hadoop.hive.contrib.fileformat.base64.Base64TextOutputFormat - name: base64_test + name: default.base64_test PREHOOK: query: CREATE TABLE base64_test(key INT, value STRING) STORED AS INPUTFORMAT 'org.apache.hadoop.hive.contrib.fileformat.base64.Base64TextInputFormat' Modified: hive/branches/cbo/contrib/src/test/results/clientpositive/serde_regex.q.out URL: http://svn.apache.org/viewvc/hive/branches/cbo/contrib/src/test/results/clientpositive/serde_regex.q.out?rev=1621031&r1=1621030&r2=1621031&view=diff ============================================================================== --- hive/branches/cbo/contrib/src/test/results/clientpositive/serde_regex.q.out (original) +++ hive/branches/cbo/contrib/src/test/results/clientpositive/serde_regex.q.out Thu Aug 28 03:15:13 2014 @@ -48,7 +48,7 @@ STAGE PLANS: serde properties: input.regex ([^ ]*) ([^ ]*) ([^ ]*) (-|\[[^\]]*\]) ([^ "]*|"[^"]*") (-|[0-9]*) (-|[0-9]*)(?: ([^ "]*|"[^"]*") ([^ "]*|"[^"]*"))? output.format.string %1$s %2$s %3$s %4$s %5$s %6$s %7$s %8$s %9$s - name: serde_regex + name: default.serde_regex PREHOOK: query: CREATE TABLE serde_regex( host STRING, Modified: hive/branches/cbo/hbase-handler/src/test/results/negative/cascade_dbdrop.q.out URL: http://svn.apache.org/viewvc/hive/branches/cbo/hbase-handler/src/test/results/negative/cascade_dbdrop.q.out?rev=1621031&r1=1621030&r2=1621031&view=diff ============================================================================== --- hive/branches/cbo/hbase-handler/src/test/results/negative/cascade_dbdrop.q.out (original) +++ hive/branches/cbo/hbase-handler/src/test/results/negative/cascade_dbdrop.q.out Thu Aug 28 03:15:13 2014 @@ -20,7 +20,7 @@ WITH SERDEPROPERTIES ("hbase.columns.map TBLPROPERTIES ("hbase.table.name" = "hbase_table_0") PREHOOK: type: CREATETABLE PREHOOK: Output: database:hbasedb -PREHOOK: Output: hbaseDB@hbaseDB.hbase_table_0 +PREHOOK: Output: hbaseDB@hbase_table_0 POSTHOOK: query: -- EXCLUDE_HADOOP_MAJOR_VERSIONS(0.20, 0.20S) -- Hadoop 0.23 changes the behavior FsShell on Exit Codes -- In Hadoop 0.20 @@ -37,7 +37,6 @@ WITH SERDEPROPERTIES ("hbase.columns.map TBLPROPERTIES ("hbase.table.name" = "hbase_table_0") POSTHOOK: type: CREATETABLE POSTHOOK: Output: database:hbasedb -POSTHOOK: Output: hbaseDB@hbaseDB.hbase_table_0 POSTHOOK: Output: hbaseDB@hbase_table_0 Found 3 items #### A masked pattern was here #### Modified: hive/branches/cbo/hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/CreateTableHook.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/CreateTableHook.java?rev=1621031&r1=1621030&r2=1621031&view=diff ============================================================================== --- hive/branches/cbo/hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/CreateTableHook.java (original) +++ hive/branches/cbo/hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/CreateTableHook.java Thu Aug 28 03:15:13 2014 @@ -86,7 +86,7 @@ final class CreateTableHook extends HCat "Operation not supported. Create table as " + "Select is not a valid operation."); - case HiveParser.TOK_TABLEBUCKETS: + case HiveParser.TOK_ALTERTABLE_BUCKETS: break; case HiveParser.TOK_LIKETABLE: Modified: hive/branches/cbo/hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/HCatSemanticAnalyzer.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/HCatSemanticAnalyzer.java?rev=1621031&r1=1621030&r2=1621031&view=diff ============================================================================== --- hive/branches/cbo/hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/HCatSemanticAnalyzer.java (original) +++ hive/branches/cbo/hcatalog/core/src/main/java/org/apache/hive/hcatalog/cli/SemanticAnalysis/HCatSemanticAnalyzer.java Thu Aug 28 03:15:13 2014 @@ -71,7 +71,7 @@ public class HCatSemanticAnalyzer extend hook = new CreateDatabaseHook(); return hook.preAnalyze(context, ast); - case HiveParser.TOK_ALTERTABLE_PARTITION: + case HiveParser.TOK_ALTERTABLE: if (((ASTNode) ast.getChild(1)).getToken().getType() == HiveParser.TOK_ALTERTABLE_FILEFORMAT) { return ast; } else if (((ASTNode) ast.getChild(1)).getToken().getType() == HiveParser.TOK_ALTERTABLE_MERGEFILES) { @@ -163,7 +163,6 @@ public class HCatSemanticAnalyzer extend case HiveParser.TOK_CREATETABLE: case HiveParser.TOK_CREATEDATABASE: - case HiveParser.TOK_ALTERTABLE_PARTITION: // HCat will allow these operations to be performed. // Database DDL @@ -178,12 +177,20 @@ public class HCatSemanticAnalyzer extend case HiveParser.TOK_CREATEINDEX: case HiveParser.TOK_DROPINDEX: case HiveParser.TOK_SHOWINDEXES: + break; // View DDL //case HiveParser.TOK_ALTERVIEW_ADDPARTS: - case HiveParser.TOK_ALTERVIEW_DROPPARTS: - case HiveParser.TOK_ALTERVIEW_PROPERTIES: - case HiveParser.TOK_ALTERVIEW_RENAME: + case HiveParser.TOK_ALTERVIEW: + switch (ast.getChild(1).getType()) { + case HiveParser.TOK_ALTERVIEW_ADDPARTS: + case HiveParser.TOK_ALTERVIEW_DROPPARTS: + case HiveParser.TOK_ALTERVIEW_RENAME: + case HiveParser.TOK_ALTERVIEW_PROPERTIES: + case HiveParser.TOK_ALTERVIEW_DROPPROPERTIES: + } + break; + case HiveParser.TOK_CREATEVIEW: case HiveParser.TOK_DROPVIEW: @@ -205,20 +212,39 @@ public class HCatSemanticAnalyzer extend case HiveParser.TOK_DESCFUNCTION: case HiveParser.TOK_SHOWFUNCTIONS: case HiveParser.TOK_EXPLAIN: + break; // Table DDL - case HiveParser.TOK_ALTERTABLE_ADDPARTS: - case HiveParser.TOK_ALTERTABLE_ADDCOLS: - case HiveParser.TOK_ALTERTABLE_CHANGECOL_AFTER_POSITION: - case HiveParser.TOK_ALTERTABLE_SERDEPROPERTIES: - case HiveParser.TOK_ALTERTABLE_CLUSTER_SORT: - case HiveParser.TOK_ALTERTABLE_DROPPARTS: - case HiveParser.TOK_ALTERTABLE_PROPERTIES: - case HiveParser.TOK_ALTERTABLE_RENAME: - case HiveParser.TOK_ALTERTABLE_RENAMECOL: - case HiveParser.TOK_ALTERTABLE_REPLACECOLS: - case HiveParser.TOK_ALTERTABLE_SERIALIZER: - case HiveParser.TOK_ALTERTABLE_TOUCH: + case HiveParser.TOK_ALTERTABLE: + switch (ast.getChild(1).getType()) { + case HiveParser.TOK_ALTERTABLE_ADDPARTS: + case HiveParser.TOK_ALTERTABLE_ADDCOLS: + case HiveParser.TOK_ALTERTABLE_CHANGECOL_AFTER_POSITION: + case HiveParser.TOK_ALTERTABLE_SERDEPROPERTIES: + case HiveParser.TOK_ALTERTABLE_CLUSTER_SORT: + case HiveParser.TOK_ALTERTABLE_DROPPARTS: + case HiveParser.TOK_ALTERTABLE_PROPERTIES: + case HiveParser.TOK_ALTERTABLE_DROPPROPERTIES: + case HiveParser.TOK_ALTERTABLE_RENAME: + case HiveParser.TOK_ALTERTABLE_RENAMECOL: + case HiveParser.TOK_ALTERTABLE_REPLACECOLS: + case HiveParser.TOK_ALTERTABLE_SERIALIZER: + case HiveParser.TOK_ALTERTABLE_TOUCH: + case HiveParser.TOK_ALTERTABLE_ARCHIVE: + case HiveParser.TOK_ALTERTABLE_UNARCHIVE: + case HiveParser.TOK_ALTERTABLE_EXCHANGEPARTITION: + case HiveParser.TOK_ALTERTABLE_SKEWED: + case HiveParser.TOK_ALTERTABLE_FILEFORMAT: + case HiveParser.TOK_ALTERTABLE_PROTECTMODE: + case HiveParser.TOK_ALTERTABLE_LOCATION: + case HiveParser.TOK_ALTERTABLE_MERGEFILES: + case HiveParser.TOK_ALTERTABLE_RENAMEPART: + case HiveParser.TOK_ALTERTABLE_SKEWED_LOCATION: + case HiveParser.TOK_ALTERTABLE_BUCKETS: + case HiveParser.TOK_ALTERTABLE_COMPACT: + } + break; + case HiveParser.TOK_DESCTABLE: case HiveParser.TOK_DROPTABLE: case HiveParser.TOK_SHOW_TABLESTATUS: Modified: hive/branches/cbo/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestSemanticAnalysis.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestSemanticAnalysis.java?rev=1621031&r1=1621030&r2=1621031&view=diff ============================================================================== --- hive/branches/cbo/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestSemanticAnalysis.java (original) +++ hive/branches/cbo/hcatalog/core/src/test/java/org/apache/hive/hcatalog/cli/TestSemanticAnalysis.java Thu Aug 28 03:15:13 2014 @@ -156,7 +156,7 @@ public class TestSemanticAnalysis extend public void testCreateTableIfNotExists() throws MetaException, TException, NoSuchObjectException, CommandNeedRetryException { hcatDriver.run("drop table " + TBL_NAME); - hcatDriver.run("create table junit_sem_analysis (a int) stored as RCFILE"); + hcatDriver.run("create table " + TBL_NAME + " (a int) stored as RCFILE"); Table tbl = client.getTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, TBL_NAME); List<FieldSchema> cols = tbl.getSd().getCols(); assertEquals(1, cols.size()); Modified: hive/branches/cbo/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatPartition.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatPartition.java?rev=1621031&r1=1621030&r2=1621031&view=diff ============================================================================== --- hive/branches/cbo/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatPartition.java (original) +++ hive/branches/cbo/hcatalog/webhcat/java-client/src/main/java/org/apache/hive/hcatalog/api/HCatPartition.java Thu Aug 28 03:15:13 2014 @@ -51,6 +51,7 @@ public class HCatPartition { private int createTime; private int lastAccessTime; private StorageDescriptor sd; + private List<HCatFieldSchema> columns; // Cache column-list from this.sd. private Map<String, String> parameters; // For use from within HCatClient.getPartitions(). @@ -68,6 +69,7 @@ public class HCatPartition { } this.sd = partition.getSd(); + this.columns = getColumns(this.sd); } // For constructing HCatPartitions afresh, as an argument to HCatClient.addPartitions(). @@ -77,6 +79,7 @@ public class HCatPartition { this.dbName = hcatTable.getDbName(); this.sd = new StorageDescriptor(hcatTable.getSd()); this.sd.setLocation(location); + this.columns = getColumns(this.sd); this.createTime = (int)(System.currentTimeMillis()/1000); this.lastAccessTime = -1; this.values = new ArrayList<String>(hcatTable.getPartCols().size()); @@ -98,7 +101,7 @@ public class HCatPartition { this.dbName = rhs.dbName; this.sd = new StorageDescriptor(rhs.sd); this.sd.setLocation(location); - + this.columns = getColumns(this.sd); this.createTime = (int) (System.currentTimeMillis() / 1000); this.lastAccessTime = -1; this.values = new ArrayList<String>(hcatTable.getPartCols().size()); @@ -112,6 +115,14 @@ public class HCatPartition { } } + private static List<HCatFieldSchema> getColumns(StorageDescriptor sd) throws HCatException { + ArrayList<HCatFieldSchema> columns = new ArrayList<HCatFieldSchema>(sd.getColsSize()); + for (FieldSchema fieldSchema : sd.getCols()) { + columns.add(HCatSchemaUtils.getHCatFieldSchema(fieldSchema)); + } + return columns; + } + // For use from HCatClient.addPartitions(), to construct from user-input. Partition toHivePartition() throws HCatException { Partition hivePtn = new Partition(); @@ -172,11 +183,7 @@ public class HCatPartition { * * @return the columns */ - public List<HCatFieldSchema> getColumns() throws HCatException { - ArrayList<HCatFieldSchema> columns = new ArrayList<HCatFieldSchema>(sd.getColsSize()); - for (FieldSchema fieldSchema : sd.getCols()) { - columns.add(HCatSchemaUtils.getHCatFieldSchema(fieldSchema)); - } + public List<HCatFieldSchema> getColumns() { return columns; } Modified: hive/branches/cbo/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java?rev=1621031&r1=1621030&r2=1621031&view=diff ============================================================================== --- hive/branches/cbo/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java (original) +++ hive/branches/cbo/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/security/authorization/plugin/TestHiveAuthorizerCheckInvocation.java Thu Aug 28 03:15:13 2014 @@ -21,6 +21,7 @@ package org.apache.hadoop.hive.ql.securi import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNull; import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; import static org.mockito.Matchers.any; import static org.mockito.Mockito.reset; import static org.mockito.Mockito.verify; @@ -141,6 +142,47 @@ public class TestHiveAuthorizerCheckInvo } @Test + public void testCreateTableWithDb() throws HiveAuthzPluginException, HiveAccessControlException, + CommandNeedRetryException { + final String newTable = "ctTableWithDb"; + checkCreateViewOrTableWithDb(newTable, "create table " + dbName + "." + newTable + "(i int)"); + } + + @Test + public void testCreateViewWithDb() throws HiveAuthzPluginException, HiveAccessControlException, + CommandNeedRetryException { + final String newTable = "ctViewWithDb"; + checkCreateViewOrTableWithDb(newTable, "create table " + dbName + "." + newTable + "(i int)"); + } + + private void checkCreateViewOrTableWithDb(String newTable, String cmd) + throws HiveAuthzPluginException, HiveAccessControlException { + reset(mockedAuthorizer); + int status = driver.compile(cmd); + assertEquals(0, status); + + List<HivePrivilegeObject> outputs = getHivePrivilegeObjectInputs().getRight(); + assertEquals("num outputs", 2, outputs.size()); + for (HivePrivilegeObject output : outputs) { + switch (output.getType()) { + case DATABASE: + assertTrue("database name", output.getDbname().equalsIgnoreCase(dbName)); + break; + case TABLE_OR_VIEW: + assertTrue("database name", output.getDbname().equalsIgnoreCase(dbName)); + assertEqualsIgnoreCase("table name", output.getObjectName(), newTable); + break; + default: + fail("Unexpected type : " + output.getType()); + } + } + } + + private void assertEqualsIgnoreCase(String msg, String expected, String actual) { + assertEquals(msg, expected.toLowerCase(), actual.toLowerCase()); + } + + @Test public void testInputNoColumnsUsed() throws HiveAuthzPluginException, HiveAccessControlException, CommandNeedRetryException { Modified: hive/branches/cbo/itests/src/test/resources/testconfiguration.properties URL: http://svn.apache.org/viewvc/hive/branches/cbo/itests/src/test/resources/testconfiguration.properties?rev=1621031&r1=1621030&r2=1621031&view=diff ============================================================================== --- hive/branches/cbo/itests/src/test/resources/testconfiguration.properties (original) +++ hive/branches/cbo/itests/src/test/resources/testconfiguration.properties Thu Aug 28 03:15:13 2014 @@ -87,6 +87,7 @@ minitez.query.files.shared=alter_merge_2 mapreduce2.q,\ merge1.q,\ merge2.q,\ + metadataonly1.q,\ metadata_only_queries.q,\ optimize_nullscan.q,\ orc_analyze.q,\ Modified: hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java?rev=1621031&r1=1621030&r2=1621031&view=diff ============================================================================== --- hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java (original) +++ hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/HiveMetaStore.java Thu Aug 28 03:15:13 2014 @@ -250,7 +250,7 @@ public class HiveMetaStore extends Thrif private static String currentUrl; private Warehouse wh; // hdfs warehouse - private final ThreadLocal<RawStore> threadLocalMS = + private static final ThreadLocal<RawStore> threadLocalMS = new ThreadLocal<RawStore>() { @Override protected synchronized RawStore initialValue() { @@ -265,6 +265,14 @@ public class HiveMetaStore extends Thrif } }; + public static RawStore getRawStore() { + return threadLocalMS.get(); + } + + public static void removeRawStore() { + threadLocalMS.remove(); + } + // Thread local configuration is needed as many threads could make changes // to the conf using the connection hook private final ThreadLocal<Configuration> threadLocalConf = @@ -384,6 +392,7 @@ public class HiveMetaStore extends Thrif } } + @Override public void init() throws MetaException { rawStoreClassName = hiveConf.getVar(HiveConf.ConfVars.METASTORE_RAW_STORE_IMPL); initListeners = MetaStoreUtils.getMetaStoreListeners( @@ -5024,12 +5033,8 @@ public class HiveMetaStore extends Thrif startFunction("get_aggr_stats_for: db=" + request.getDbName() + " table=" + request.getTblName()); AggrStats aggrStats = null; try { - //TODO: We are setting partitionCnt for which we were able to retrieve stats same as - // incoming number from request. This is not correct, but currently no users of this api - // rely on this. Only, current user StatsAnnotation don't care for it. StatsOptimizer - // will care for it, so before StatsOptimizer begin using it, we need to fix this. aggrStats = new AggrStats(getMS().get_aggr_stats_for(request.getDbName(), - request.getTblName(), request.getPartNames(), request.getColNames()), request.getPartNames().size()); + request.getTblName(), request.getPartNames(), request.getColNames())); return aggrStats; } finally { endFunction("get_partitions_statistics_req: ", aggrStats == null, null, request.getTblName()); Modified: hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java?rev=1621031&r1=1621030&r2=1621031&view=diff ============================================================================== --- hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java (original) +++ hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/MetaStoreDirectSql.java Thu Aug 28 03:15:13 2014 @@ -21,19 +21,18 @@ package org.apache.hadoop.hive.metastore import static org.apache.commons.lang.StringUtils.join; import static org.apache.commons.lang.StringUtils.repeat; +import java.math.BigDecimal; +import java.nio.ByteBuffer; import java.sql.Connection; import java.sql.SQLException; -import java.sql.Statement; import java.text.ParseException; -import java.text.SimpleDateFormat; import java.util.ArrayList; -import java.util.Date; +import java.util.Arrays; import java.util.HashMap; import java.util.Iterator; import java.util.List; import java.util.Map; import java.util.TreeMap; -import java.util.concurrent.atomic.AtomicLong; import javax.jdo.PersistenceManager; import javax.jdo.Query; @@ -43,10 +42,12 @@ import javax.jdo.datastore.JDOConnection import org.apache.commons.lang.StringUtils; import org.apache.commons.logging.Log; import org.apache.commons.logging.LogFactory; +import org.apache.hadoop.hive.metastore.api.AggrStats; import org.apache.hadoop.hive.metastore.api.ColumnStatistics; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; +import org.apache.hadoop.hive.metastore.api.Decimal; import org.apache.hadoop.hive.metastore.api.FieldSchema; import org.apache.hadoop.hive.metastore.api.MetaException; import org.apache.hadoop.hive.metastore.api.Order; @@ -65,9 +66,8 @@ import org.apache.hadoop.hive.metastore. import org.apache.hadoop.hive.metastore.parser.ExpressionTree.Operator; import org.apache.hadoop.hive.metastore.parser.ExpressionTree.TreeNode; import org.apache.hadoop.hive.metastore.parser.ExpressionTree.TreeVisitor; -import org.apache.hadoop.hive.metastore.parser.FilterLexer; import org.apache.hadoop.hive.serde.serdeConstants; -import org.datanucleus.store.schema.SchemaTool; +import org.datanucleus.store.rdbms.query.ForwardQueryResult; import com.google.common.collect.Lists; @@ -97,7 +97,7 @@ class MetaStoreDirectSql { * Whether direct SQL can be used with the current datastore backing {@link #pm}. */ private final boolean isCompatibleDatastore; - + public MetaStoreDirectSql(PersistenceManager pm) { this.pm = pm; Transaction tx = pm.currentTransaction(); @@ -893,33 +893,247 @@ class MetaStoreDirectSql { return result; } - public List<ColumnStatisticsObj> aggrColStatsForPartitions(String dbName, String tableName, + public AggrStats aggrColStatsForPartitions(String dbName, String tableName, List<String> partNames, List<String> colNames) throws MetaException { - String qText = "select \"COLUMN_NAME\", \"COLUMN_TYPE\", " - + "min(\"LONG_LOW_VALUE\"), max(\"LONG_HIGH_VALUE\"), min(\"DOUBLE_LOW_VALUE\"), max(\"DOUBLE_HIGH_VALUE\"), " - + "min(\"BIG_DECIMAL_LOW_VALUE\"), max(\"BIG_DECIMAL_HIGH_VALUE\"), sum(\"NUM_NULLS\"), max(\"NUM_DISTINCTS\"), " - + "max(\"AVG_COL_LEN\"), max(\"MAX_COL_LEN\"), sum(\"NUM_TRUES\"), sum(\"NUM_FALSES\") from \"PART_COL_STATS\"" - + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ? and \"COLUMN_NAME\" in (" - + makeParams(colNames.size()) + ") AND \"PARTITION_NAME\" in (" - + makeParams(partNames.size()) + ") group by \"COLUMN_NAME\", \"COLUMN_TYPE\""; + long partsFound = partsFoundForPartitions(dbName, tableName, partNames, + colNames); + List<ColumnStatisticsObj> stats = columnStatisticsObjForPartitions(dbName, + tableName, partNames, colNames, partsFound); + return new AggrStats(stats, partsFound); + } + private long partsFoundForPartitions(String dbName, String tableName, + List<String> partNames, List<String> colNames) throws MetaException { + long partsFound = 0; boolean doTrace = LOG.isDebugEnabled(); + String qText = "select count(\"COLUMN_NAME\") from \"PART_COL_STATS\"" + + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ? " + + " and \"COLUMN_NAME\" in (" + makeParams(colNames.size()) + ")" + + " and \"PARTITION_NAME\" in (" + makeParams(partNames.size()) + ")" + + " group by \"PARTITION_NAME\""; long start = doTrace ? System.nanoTime() : 0; Query query = pm.newQuery("javax.jdo.query.SQL", qText); - Object qResult = query.executeWithArray(prepareParams(dbName, tableName, partNames, colNames)); - if (qResult == null) { - query.closeAll(); - return Lists.newArrayList(); - } - List<Object[]> list = ensureList(qResult); - List<ColumnStatisticsObj> colStats = new ArrayList<ColumnStatisticsObj>(list.size()); - for (Object[] row : list) { - colStats.add(prepareCSObj(row,0)); - } + Object qResult = query.executeWithArray(prepareParams(dbName, tableName, + partNames, colNames)); long end = doTrace ? System.nanoTime() : 0; timingTrace(doTrace, qText, start, end); - query.closeAll(); - return colStats; + ForwardQueryResult fqr = (ForwardQueryResult) qResult; + List<Integer> colnumbers = new ArrayList<Integer>(); + colnumbers.addAll(fqr); + for (Integer colnumber : colnumbers) { + if (colnumber == colNames.size()) + partsFound++; + } + return partsFound; + } + + private List<ColumnStatisticsObj> columnStatisticsObjForPartitions( + String dbName, String tableName, List<String> partNames, + List<String> colNames, long partsFound) throws MetaException { + String commonPrefix = "select \"COLUMN_NAME\", \"COLUMN_TYPE\", " + + "min(\"LONG_LOW_VALUE\"), max(\"LONG_HIGH_VALUE\"), min(\"DOUBLE_LOW_VALUE\"), max(\"DOUBLE_HIGH_VALUE\"), " + + "min(\"BIG_DECIMAL_LOW_VALUE\"), max(\"BIG_DECIMAL_HIGH_VALUE\"), sum(\"NUM_NULLS\"), max(\"NUM_DISTINCTS\"), " + + "max(\"AVG_COL_LEN\"), max(\"MAX_COL_LEN\"), sum(\"NUM_TRUES\"), sum(\"NUM_FALSES\") from \"PART_COL_STATS\"" + + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ? "; + String qText = null; + long start = 0; + long end = 0; + Query query = null; + boolean doTrace = LOG.isDebugEnabled(); + Object qResult = null; + ForwardQueryResult fqr = null; + // Check if the status of all the columns of all the partitions exists + // Extrapolation is not needed. + if (partsFound == partNames.size()) { + qText = commonPrefix + + " and \"COLUMN_NAME\" in (" + makeParams(colNames.size()) + ")" + + " and \"PARTITION_NAME\" in (" + makeParams(partNames.size()) + ")" + + " group by \"COLUMN_NAME\", \"COLUMN_TYPE\""; + start = doTrace ? System.nanoTime() : 0; + query = pm.newQuery("javax.jdo.query.SQL", qText); + qResult = query.executeWithArray(prepareParams(dbName, tableName, + partNames, colNames)); + if (qResult == null) { + query.closeAll(); + return Lists.newArrayList(); + } + end = doTrace ? System.nanoTime() : 0; + timingTrace(doTrace, qText, start, end); + List<Object[]> list = ensureList(qResult); + List<ColumnStatisticsObj> colStats = new ArrayList<ColumnStatisticsObj>( + list.size()); + for (Object[] row : list) { + colStats.add(prepareCSObj(row, 0)); + } + query.closeAll(); + return colStats; + } else { + // Extrapolation is needed for some columns. + // In this case, at least a column status for a partition is missing. + // We need to extrapolate this partition based on the other partitions + List<ColumnStatisticsObj> colStats = new ArrayList<ColumnStatisticsObj>( + colNames.size()); + qText = "select \"COLUMN_NAME\", \"COLUMN_TYPE\", count(\"PARTITION_NAME\") " + + " from \"PART_COL_STATS\"" + + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ? " + + " and \"COLUMN_NAME\" in (" + makeParams(colNames.size()) + ")" + + " and \"PARTITION_NAME\" in (" + makeParams(partNames.size()) + ")" + + " group by \"COLUMN_NAME\", \"COLUMN_TYPE\""; + start = doTrace ? System.nanoTime() : 0; + query = pm.newQuery("javax.jdo.query.SQL", qText); + qResult = query.executeWithArray(prepareParams(dbName, tableName, + partNames, colNames)); + end = doTrace ? System.nanoTime() : 0; + timingTrace(doTrace, qText, start, end); + if (qResult == null) { + query.closeAll(); + return Lists.newArrayList(); + } + List<String> noExtraColumnNames = new ArrayList<String>(); + Map<String, String[]> extraColumnNameTypeParts = new HashMap<String, String[]>(); + List<Object[]> list = ensureList(qResult); + for (Object[] row : list) { + String colName = (String) row[0]; + String colType = (String) row[1]; + if ((Integer) row[2] == partNames.size() || (Integer) row[2] < 2) { + // Extrapolation is not needed for this column if + // count(\"PARTITION_NAME\")==partNames.size() + // Or, extrapolation is not possible for this column if + // count(\"PARTITION_NAME\")<2 + noExtraColumnNames.add(colName); + } else { + extraColumnNameTypeParts.put(colName, + new String[] { colType, String.valueOf((Integer) row[2]) }); + } + } + query.closeAll(); + // Extrapolation is not needed for columns noExtraColumnNames + if (noExtraColumnNames.size() != 0) { + qText = commonPrefix + + " and \"COLUMN_NAME\" in ("+ makeParams(noExtraColumnNames.size()) + ")" + + " and \"PARTITION_NAME\" in ("+ makeParams(partNames.size()) +")" + + " group by \"COLUMN_NAME\", \"COLUMN_TYPE\""; + start = doTrace ? System.nanoTime() : 0; + query = pm.newQuery("javax.jdo.query.SQL", qText); + qResult = query.executeWithArray(prepareParams(dbName, tableName, + partNames, noExtraColumnNames)); + if (qResult == null) { + query.closeAll(); + return Lists.newArrayList(); + } + list = ensureList(qResult); + for (Object[] row : list) { + colStats.add(prepareCSObj(row, 0)); + } + end = doTrace ? System.nanoTime() : 0; + timingTrace(doTrace, qText, start, end); + query.closeAll(); + } + // Extrapolation is needed for extraColumnNames. + // give a sequence number for all the partitions + if (extraColumnNameTypeParts.size() != 0) { + Map<String, Integer> indexMap = new HashMap<String, Integer>(); + for (int index = 0; index < partNames.size(); index++) { + indexMap.put(partNames.get(index), index); + } + // get sum for all columns to reduce the number of queries + Map<String, Map<Integer, Object>> sumMap = new HashMap<String, Map<Integer, Object>>(); + qText = "select \"COLUMN_NAME\", sum(\"NUM_NULLS\"), sum(\"NUM_TRUES\"), sum(\"NUM_FALSES\")" + + " from \"PART_COL_STATS\"" + + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ? " + + " and \"COLUMN_NAME\" in (" +makeParams(extraColumnNameTypeParts.size())+ ")" + + " and \"PARTITION_NAME\" in (" + makeParams(partNames.size()) + ")" + + " group by \"COLUMN_NAME\""; + start = doTrace ? System.nanoTime() : 0; + query = pm.newQuery("javax.jdo.query.SQL", qText); + List<String> extraColumnNames = new ArrayList<String>(); + extraColumnNames.addAll(extraColumnNameTypeParts.keySet()); + qResult = query.executeWithArray(prepareParams(dbName, tableName, + partNames, extraColumnNames)); + if (qResult == null) { + query.closeAll(); + return Lists.newArrayList(); + } + list = ensureList(qResult); + // see the indexes for colstats in IExtrapolatePartStatus + Integer[] sumIndex = new Integer[] { 6, 10, 11 }; + for (Object[] row : list) { + Map<Integer, Object> indexToObject = new HashMap<Integer, Object>(); + for (int ind = 1; ind < row.length; ind++) { + indexToObject.put(sumIndex[ind - 1], row[ind]); + } + sumMap.put((String) row[0], indexToObject); + } + end = doTrace ? System.nanoTime() : 0; + timingTrace(doTrace, qText, start, end); + query.closeAll(); + for (Map.Entry<String, String[]> entry : extraColumnNameTypeParts + .entrySet()) { + Object[] row = new Object[IExtrapolatePartStatus.colStatNames.length + 2]; + String colName = entry.getKey(); + String colType = entry.getValue()[0]; + Long sumVal = Long.parseLong(entry.getValue()[1]); + // fill in colname + row[0] = colName; + // fill in coltype + row[1] = colType; + // use linear extrapolation. more complicated one can be added in the future. + IExtrapolatePartStatus extrapolateMethod = new LinearExtrapolatePartStatus(); + // fill in colstatus + Integer[] index = IExtrapolatePartStatus.indexMaps.get(colType + .toLowerCase()); + //if the colType is not the known type, long, double, etc, then get all index. + if (index == null) { + index = IExtrapolatePartStatus.indexMaps.get("default"); + } + for (int colStatIndex : index) { + String colStatName = IExtrapolatePartStatus.colStatNames[colStatIndex]; + // if the aggregation type is sum, we do a scale-up + if (IExtrapolatePartStatus.aggrTypes[colStatIndex] == IExtrapolatePartStatus.AggrType.Sum) { + Long val = (Long) sumMap.get(colName).get(colStatIndex); + if (val == null) { + row[2 + colStatIndex] = null; + } else { + row[2 + colStatIndex] = (Long) (val / sumVal * (partNames + .size())); + } + } else { + // if the aggregation type is min/max, we extrapolate from the + // left/right borders + qText = "select \"" + + colStatName + + "\",\"PARTITION_NAME\" from \"PART_COL_STATS\"" + + " where \"DB_NAME\" = ? and \"TABLE_NAME\" = ?" + + " and \"COLUMN_NAME\" in (" +makeParams(1)+ ")" + + " and \"PARTITION_NAME\" in (" + makeParams(partNames.size()) + ")" + + " order by \'" + colStatName + "\'"; + start = doTrace ? System.nanoTime() : 0; + query = pm.newQuery("javax.jdo.query.SQL", qText); + qResult = query.executeWithArray(prepareParams(dbName, + tableName, partNames, Arrays.asList(colName))); + if (qResult == null) { + query.closeAll(); + return Lists.newArrayList(); + } + fqr = (ForwardQueryResult) qResult; + Object[] min = (Object[]) (fqr.get(0)); + Object[] max = (Object[]) (fqr.get(fqr.size() - 1)); + end = doTrace ? System.nanoTime() : 0; + timingTrace(doTrace, qText, start, end); + query.closeAll(); + if (min[0] == null || max[0] == null) { + row[2 + colStatIndex] = null; + } else { + row[2 + colStatIndex] = extrapolateMethod.extrapolate(min, max, + colStatIndex, indexMap); + } + } + } + colStats.add(prepareCSObj(row, 0)); + } + } + return colStats; + } } private ColumnStatisticsObj prepareCSObj (Object[] row, int i) throws MetaException { @@ -949,7 +1163,7 @@ class MetaStoreDirectSql { return params; } - + public List<ColumnStatistics> getPartitionStats(String dbName, String tableName, List<String> partNames, List<String> colNames) throws MetaException { if (colNames.isEmpty() || partNames.isEmpty()) { Modified: hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java?rev=1621031&r1=1621030&r2=1621031&view=diff ============================================================================== --- hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java (original) +++ hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/ObjectStore.java Thu Aug 28 03:15:13 2014 @@ -60,6 +60,7 @@ import org.apache.hadoop.hive.common.cla import org.apache.hadoop.hive.common.classification.InterfaceStability; import org.apache.hadoop.hive.conf.HiveConf; import org.apache.hadoop.hive.conf.HiveConf.ConfVars; +import org.apache.hadoop.hive.metastore.api.AggrStats; import org.apache.hadoop.hive.metastore.api.ColumnStatistics; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; @@ -251,6 +252,8 @@ public class ObjectStore implements RawS expressionProxy = createExpressionProxy(hiveConf); directSql = new MetaStoreDirectSql(pm); } + LOG.debug("RawStore: " + this + ", with PersistenceManager: " + pm + + " created in the thread with id: " + Thread.currentThread().getId()); } /** @@ -342,6 +345,8 @@ public class ObjectStore implements RawS @Override public void shutdown() { if (pm != null) { + LOG.debug("RawStore: " + this + ", with PersistenceManager: " + pm + + " will be shutdown"); pm.close(); } } @@ -5904,25 +5909,28 @@ public class ObjectStore implements RawS @Override - public List<ColumnStatisticsObj> get_aggr_stats_for(String dbName, String tblName, + public AggrStats get_aggr_stats_for(String dbName, String tblName, final List<String> partNames, final List<String> colNames) throws MetaException, NoSuchObjectException { - - return new GetListHelper<ColumnStatisticsObj>(dbName, tblName, true, false) { + return new GetHelper<AggrStats>(dbName, tblName, true, false) { @Override - protected List<ColumnStatisticsObj> getSqlResult( - GetHelper<List<ColumnStatisticsObj>> ctx) throws MetaException { - return directSql.aggrColStatsForPartitions(dbName, tblName, partNames, colNames); + protected AggrStats getSqlResult(GetHelper<AggrStats> ctx) + throws MetaException { + return directSql.aggrColStatsForPartitions(dbName, tblName, partNames, + colNames); } - @Override - protected List<ColumnStatisticsObj> getJdoResult( - GetHelper<List<ColumnStatisticsObj>> ctx) throws MetaException, - NoSuchObjectException { - // This is fast path for query optimizations, if we can find this info quickly using + protected AggrStats getJdoResult(GetHelper<AggrStats> ctx) + throws MetaException, NoSuchObjectException { + // This is fast path for query optimizations, if we can find this info + // quickly using // directSql, do it. No point in failing back to slow path here. throw new MetaException("Jdo path is not implemented for stats aggr."); } - }.run(true); + @Override + protected String describeResult() { + return null; + } + }.run(true); } private List<MPartitionColumnStatistics> getMPartitionColumnStatistics( Modified: hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java?rev=1621031&r1=1621030&r2=1621031&view=diff ============================================================================== --- hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java (original) +++ hive/branches/cbo/metastore/src/java/org/apache/hadoop/hive/metastore/RawStore.java Thu Aug 28 03:15:13 2014 @@ -26,6 +26,7 @@ import java.util.List; import java.util.Map; import org.apache.hadoop.conf.Configurable; +import org.apache.hadoop.hive.metastore.api.AggrStats; import org.apache.hadoop.hive.metastore.api.ColumnStatistics; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.Database; @@ -548,6 +549,6 @@ public interface RawStore extends Config */ public List<String> getFunctions(String dbName, String pattern) throws MetaException; - public List<ColumnStatisticsObj> get_aggr_stats_for(String dbName, String tblName, + public AggrStats get_aggr_stats_for(String dbName, String tblName, List<String> partNames, List<String> colNames) throws MetaException, NoSuchObjectException; } Modified: hive/branches/cbo/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java?rev=1621031&r1=1621030&r2=1621031&view=diff ============================================================================== --- hive/branches/cbo/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java (original) +++ hive/branches/cbo/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreControlledCommit.java Thu Aug 28 03:15:13 2014 @@ -24,6 +24,7 @@ import java.util.Map; import org.apache.hadoop.conf.Configurable; import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hive.metastore.api.AggrStats; import org.apache.hadoop.hive.metastore.api.ColumnStatistics; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.Database; @@ -712,7 +713,7 @@ public class DummyRawStoreControlledComm } @Override - public List<ColumnStatisticsObj> get_aggr_stats_for(String dbName, + public AggrStats get_aggr_stats_for(String dbName, String tblName, List<String> partNames, List<String> colNames) throws MetaException { return null; Modified: hive/branches/cbo/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java URL: http://svn.apache.org/viewvc/hive/branches/cbo/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java?rev=1621031&r1=1621030&r2=1621031&view=diff ============================================================================== --- hive/branches/cbo/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java (original) +++ hive/branches/cbo/metastore/src/test/org/apache/hadoop/hive/metastore/DummyRawStoreForJdoConnection.java Thu Aug 28 03:15:13 2014 @@ -25,6 +25,7 @@ import junit.framework.Assert; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hive.conf.HiveConf; +import org.apache.hadoop.hive.metastore.api.AggrStats; import org.apache.hadoop.hive.metastore.api.ColumnStatistics; import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; import org.apache.hadoop.hive.metastore.api.Database; @@ -729,7 +730,7 @@ public class DummyRawStoreForJdoConnecti } @Override - public List<ColumnStatisticsObj> get_aggr_stats_for(String dbName, + public AggrStats get_aggr_stats_for(String dbName, String tblName, List<String> partNames, List<String> colNames) throws MetaException { return null;