http://git-wip-us.apache.org/repos/asf/hive/blob/133d3c47/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/DecimalColumnStatsMerger.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/DecimalColumnStatsMerger.java b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/DecimalColumnStatsMerger.java deleted file mode 100644 index 54099f6..0000000 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/DecimalColumnStatsMerger.java +++ /dev/null @@ -1,61 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hive.metastore.columnstats.merge; - -import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator; -import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; -import org.apache.hadoop.hive.metastore.api.Decimal; -import org.apache.hadoop.hive.metastore.columnstats.cache.DecimalColumnStatsDataInspector; - -public class DecimalColumnStatsMerger extends ColumnStatsMerger { - @Override - public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) { - DecimalColumnStatsDataInspector aggregateData = - (DecimalColumnStatsDataInspector) aggregateColStats.getStatsData().getDecimalStats(); - DecimalColumnStatsDataInspector newData = - (DecimalColumnStatsDataInspector) newColStats.getStatsData().getDecimalStats(); - Decimal lowValue = aggregateData.getLowValue() != null - && (aggregateData.getLowValue().compareTo(newData.getLowValue()) > 0) ? aggregateData - .getLowValue() : newData.getLowValue(); - aggregateData.setLowValue(lowValue); - Decimal highValue = aggregateData.getHighValue() != null - && (aggregateData.getHighValue().compareTo(newData.getHighValue()) > 0) ? aggregateData - .getHighValue() : newData.getHighValue(); - aggregateData.setHighValue(highValue); - aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls()); - if (aggregateData.getNdvEstimator() == null || newData.getNdvEstimator() == null) { - aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs())); - } else { - NumDistinctValueEstimator oldEst = aggregateData.getNdvEstimator(); - NumDistinctValueEstimator newEst = newData.getNdvEstimator(); - long ndv = -1; - if (oldEst.canMerge(newEst)) { - oldEst.mergeEstimators(newEst); - ndv = oldEst.estimateNumDistinctValues(); - aggregateData.setNdvEstimator(oldEst); - } else { - ndv = Math.max(aggregateData.getNumDVs(), newData.getNumDVs()); - } - LOG.debug("Use bitvector to merge column " + aggregateColStats.getColName() + "'s ndvs of " - + aggregateData.getNumDVs() + " and " + newData.getNumDVs() + " to be " + ndv); - aggregateData.setNumDVs(ndv); - } - } -}
http://git-wip-us.apache.org/repos/asf/hive/blob/133d3c47/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/DoubleColumnStatsMerger.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/DoubleColumnStatsMerger.java b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/DoubleColumnStatsMerger.java deleted file mode 100644 index 817a55d..0000000 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/DoubleColumnStatsMerger.java +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hive.metastore.columnstats.merge; - -import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator; -import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; -import org.apache.hadoop.hive.metastore.columnstats.cache.DoubleColumnStatsDataInspector; - -public class DoubleColumnStatsMerger extends ColumnStatsMerger { - @Override - public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) { - DoubleColumnStatsDataInspector aggregateData = - (DoubleColumnStatsDataInspector) aggregateColStats.getStatsData().getDoubleStats(); - DoubleColumnStatsDataInspector newData = - (DoubleColumnStatsDataInspector) newColStats.getStatsData().getDoubleStats(); - aggregateData.setLowValue(Math.min(aggregateData.getLowValue(), newData.getLowValue())); - aggregateData.setHighValue(Math.max(aggregateData.getHighValue(), newData.getHighValue())); - aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls()); - if (aggregateData.getNdvEstimator() == null || newData.getNdvEstimator() == null) { - aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs())); - } else { - NumDistinctValueEstimator oldEst = aggregateData.getNdvEstimator(); - NumDistinctValueEstimator newEst = newData.getNdvEstimator(); - long ndv = -1; - if (oldEst.canMerge(newEst)) { - oldEst.mergeEstimators(newEst); - ndv = oldEst.estimateNumDistinctValues(); - aggregateData.setNdvEstimator(oldEst); - } else { - ndv = Math.max(aggregateData.getNumDVs(), newData.getNumDVs()); - } - LOG.debug("Use bitvector to merge column " + aggregateColStats.getColName() + "'s ndvs of " - + aggregateData.getNumDVs() + " and " + newData.getNumDVs() + " to be " + ndv); - aggregateData.setNumDVs(ndv); - } - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/133d3c47/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/LongColumnStatsMerger.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/LongColumnStatsMerger.java b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/LongColumnStatsMerger.java deleted file mode 100644 index dc048e0..0000000 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/LongColumnStatsMerger.java +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hive.metastore.columnstats.merge; - -import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator; -import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; -import org.apache.hadoop.hive.metastore.columnstats.cache.LongColumnStatsDataInspector; - -public class LongColumnStatsMerger extends ColumnStatsMerger { - @Override - public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) { - LongColumnStatsDataInspector aggregateData = - (LongColumnStatsDataInspector) aggregateColStats.getStatsData().getLongStats(); - LongColumnStatsDataInspector newData = - (LongColumnStatsDataInspector) newColStats.getStatsData().getLongStats(); - aggregateData.setLowValue(Math.min(aggregateData.getLowValue(), newData.getLowValue())); - aggregateData.setHighValue(Math.max(aggregateData.getHighValue(), newData.getHighValue())); - aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls()); - if (aggregateData.getNdvEstimator() == null || newData.getNdvEstimator() == null) { - aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs())); - } else { - NumDistinctValueEstimator oldEst = aggregateData.getNdvEstimator(); - NumDistinctValueEstimator newEst = newData.getNdvEstimator(); - long ndv = -1; - if (oldEst.canMerge(newEst)) { - oldEst.mergeEstimators(newEst); - ndv = oldEst.estimateNumDistinctValues(); - aggregateData.setNdvEstimator(oldEst); - } else { - ndv = Math.max(aggregateData.getNumDVs(), newData.getNumDVs()); - } - LOG.debug("Use bitvector to merge column " + aggregateColStats.getColName() + "'s ndvs of " - + aggregateData.getNumDVs() + " and " + newData.getNumDVs() + " to be " + ndv); - aggregateData.setNumDVs(ndv); - } - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/133d3c47/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/StringColumnStatsMerger.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/StringColumnStatsMerger.java b/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/StringColumnStatsMerger.java deleted file mode 100644 index e353b8f..0000000 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/columnstats/merge/StringColumnStatsMerger.java +++ /dev/null @@ -1,54 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hive.metastore.columnstats.merge; - -import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator; -import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; -import org.apache.hadoop.hive.metastore.columnstats.cache.StringColumnStatsDataInspector; - -public class StringColumnStatsMerger extends ColumnStatsMerger { - @Override - public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) { - StringColumnStatsDataInspector aggregateData = - (StringColumnStatsDataInspector) aggregateColStats.getStatsData().getStringStats(); - StringColumnStatsDataInspector newData = - (StringColumnStatsDataInspector) newColStats.getStatsData().getStringStats(); - aggregateData.setMaxColLen(Math.max(aggregateData.getMaxColLen(), newData.getMaxColLen())); - aggregateData.setAvgColLen(Math.max(aggregateData.getAvgColLen(), newData.getAvgColLen())); - aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls()); - if (aggregateData.getNdvEstimator() == null || newData.getNdvEstimator() == null) { - aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs())); - } else { - NumDistinctValueEstimator oldEst = aggregateData.getNdvEstimator(); - NumDistinctValueEstimator newEst = newData.getNdvEstimator(); - long ndv = -1; - if (oldEst.canMerge(newEst)) { - oldEst.mergeEstimators(newEst); - ndv = oldEst.estimateNumDistinctValues(); - aggregateData.setNdvEstimator(oldEst); - } else { - ndv = Math.max(aggregateData.getNumDVs(), newData.getNumDVs()); - } - LOG.debug("Use bitvector to merge column " + aggregateColStats.getColName() + "'s ndvs of " - + aggregateData.getNumDVs() + " and " + newData.getNumDVs() + " to be " + ndv); - aggregateData.setNumDVs(ndv); - } - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/133d3c47/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/MetadataStore.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/MetadataStore.java b/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/MetadataStore.java deleted file mode 100644 index d427fef..0000000 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/hbase/MetadataStore.java +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.metastore.hbase; - -import java.io.IOException; -import java.nio.ByteBuffer; -import java.util.List; - -public interface MetadataStore { - /** - * @param fileIds file ID list. - * @param result The ref parameter, used to return the serialized file metadata. - */ - void getFileMetadata(List<Long> fileIds, ByteBuffer[] result) throws IOException; - - /** - * @param fileIds file ID list. - * @param metadataBuffers Serialized file metadata, one per file ID. - * @param addedCols The column names for additional columns created by file-format-specific - * metadata handler, to be stored in the cache. - * @param addedVals The values for addedCols; one value per file ID per added column. - */ - void storeFileMetadata(List<Long> fileIds, List<ByteBuffer> metadataBuffers, - ByteBuffer[] addedCols, ByteBuffer[][] addedVals) throws IOException, InterruptedException; - - /** - * @param fileId The file ID. - * @param metadata Serialized file metadata. - * @param addedCols The column names for additional columns created by file-format-specific - * metadata handler, to be stored in the cache. - * @param addedVals The values for addedCols; one value per added column. - */ - void storeFileMetadata(long fileId, ByteBuffer metadata, ByteBuffer[] addedCols, - ByteBuffer[] addedVals) throws IOException, InterruptedException; - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/133d3c47/metastore/src/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java b/metastore/src/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java deleted file mode 100644 index 12773ac..0000000 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/parser/ExpressionTree.java +++ /dev/null @@ -1,605 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.metastore.parser; - -import java.util.Date; -import java.util.HashMap; -import java.util.Map; -import java.util.Set; -import java.util.Stack; - -import org.antlr.runtime.ANTLRStringStream; -import org.antlr.runtime.CharStream; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.metastore.ColumnType; -import org.apache.hadoop.hive.metastore.HiveMetaStore; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.Warehouse; -import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; - -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Sets; - -/** - * The Class representing the filter as a binary tree. The tree has TreeNode's - * at intermediate level and the leaf level nodes are of type LeafNode. - */ -public class ExpressionTree { - /** The empty tree that can be returned for an empty filter. */ - public static final ExpressionTree EMPTY_TREE = new ExpressionTree(); - - /** The logical operations supported. */ - public enum LogicalOperator { - AND, - OR - } - - /** The operators supported. */ - public enum Operator { - EQUALS ("=", "==", "="), - GREATERTHAN (">"), - LESSTHAN ("<"), - LESSTHANOREQUALTO ("<="), - GREATERTHANOREQUALTO (">="), - LIKE ("LIKE", "matches", "like"), - NOTEQUALS2 ("!=", "!=", "<>"), - NOTEQUALS ("<>", "!=", "<>"); - - private final String op; - private final String jdoOp; - private final String sqlOp; - - // private constructor - private Operator(String op){ - this.op = op; - this.jdoOp = op; - this.sqlOp = op; - } - - private Operator(String op, String jdoOp, String sqlOp){ - this.op = op; - this.jdoOp = jdoOp; - this.sqlOp = sqlOp; - } - - public String getOp() { - return op; - } - - public String getJdoOp() { - return jdoOp; - } - - public String getSqlOp() { - return sqlOp; - } - - public static Operator fromString(String inputOperator) { - for(Operator op : Operator.values()) { - if(op.getOp().equals(inputOperator)){ - return op; - } - } - - throw new Error("Invalid value " + inputOperator + - " for " + Operator.class.getSimpleName()); - } - - @Override - public String toString() { - return op; - } - - } - - /** - * Depth first traversal of ExpressionTree. - * The users should override the subset of methods to do their stuff. - */ - public static class TreeVisitor { - private void visit(TreeNode node) throws MetaException { - if (shouldStop()) return; - assert node != null && node.getLhs() != null && node.getRhs() != null; - beginTreeNode(node); - node.lhs.accept(this); - midTreeNode(node); - node.rhs.accept(this); - endTreeNode(node); - } - - protected void beginTreeNode(TreeNode node) throws MetaException {} - protected void midTreeNode(TreeNode node) throws MetaException {} - protected void endTreeNode(TreeNode node) throws MetaException {} - protected void visit(LeafNode node) throws MetaException {} - protected boolean shouldStop() { - return false; - } - } - - /** - * Helper class that wraps the stringbuilder used to build the filter over the tree, - * as well as error propagation in two modes - expect errors, i.e. filter might as well - * be unbuildable and that's not a failure condition; or don't expect errors, i.e. filter - * must be buildable. - */ - public static class FilterBuilder { - private final StringBuilder result = new StringBuilder(); - private String errorMessage = null; - private boolean expectNoErrors = false; - - public FilterBuilder(boolean expectNoErrors) { - this.expectNoErrors = expectNoErrors; - } - - public String getFilter() throws MetaException { - assert errorMessage == null; - if (errorMessage != null) { - throw new MetaException("Trying to get result after error: " + errorMessage); - } - return result.toString(); - } - - @Override - public String toString() { - try { - return getFilter(); - } catch (MetaException ex) { - throw new RuntimeException(ex); - } - } - - public String getErrorMessage() { - return errorMessage; - } - - public boolean hasError() { - return errorMessage != null; - } - - public FilterBuilder append(String filterPart) { - this.result.append(filterPart); - return this; - } - - public void setError(String errorMessage) throws MetaException { - this.errorMessage = errorMessage; - if (expectNoErrors) { - throw new MetaException(errorMessage); - } - } - } - - /** - * The Class representing a Node in the ExpressionTree. - */ - public static class TreeNode { - private TreeNode lhs; - private LogicalOperator andOr; - private TreeNode rhs; - - public TreeNode() { - } - - public TreeNode(TreeNode lhs, LogicalOperator andOr, TreeNode rhs) { - this.lhs = lhs; - this.andOr = andOr; - this.rhs = rhs; - } - - public TreeNode getLhs() { - return lhs; - } - - public LogicalOperator getAndOr() { - return andOr; - } - - public TreeNode getRhs() { - return rhs; - } - - /** Double dispatch for TreeVisitor. */ - protected void accept(TreeVisitor visitor) throws MetaException { - visitor.visit(this); - } - - /** - * Generates a JDO filter statement - * @param table - * The table on which the filter is applied. If table is not null, - * then this method generates a JDO statement to get all partitions - * of the table that match the filter. - * If table is null, then this method generates a JDO statement to get all - * tables that match the filter. - * @param params - * A map of parameter key to values for the filter statement. - * @param filterBuffer The filter builder that is used to build filter. - * @throws MetaException - */ - public void generateJDOFilter(Configuration conf, Table table, - Map<String, Object> params, FilterBuilder filterBuffer) throws MetaException { - if (filterBuffer.hasError()) return; - if (lhs != null) { - filterBuffer.append (" ("); - lhs.generateJDOFilter(conf, table, params, filterBuffer); - - if (rhs != null) { - if( andOr == LogicalOperator.AND ) { - filterBuffer.append(" && "); - } else { - filterBuffer.append(" || "); - } - - rhs.generateJDOFilter(conf, table, params, filterBuffer); - } - filterBuffer.append (") "); - } - } - } - - /** - * The Class representing the leaf level nodes in the ExpressionTree. - */ - public static class LeafNode extends TreeNode { - public String keyName; - public Operator operator; - /** Constant expression side of the operator. Can currently be a String or a Long. */ - public Object value; - public boolean isReverseOrder = false; - private static final String PARAM_PREFIX = "hive_filter_param_"; - - @Override - protected void accept(TreeVisitor visitor) throws MetaException { - visitor.visit(this); - } - - @Override - public void generateJDOFilter(Configuration conf, Table table, Map<String, Object> params, - FilterBuilder filterBuilder) throws MetaException { - if (table != null) { - generateJDOFilterOverPartitions(conf, table, params, filterBuilder); - } else { - generateJDOFilterOverTables(params, filterBuilder); - } - } - - //can only support "=" and "!=" for now, because our JDO lib is buggy when - // using objects from map.get() - private static final Set<Operator> TABLE_FILTER_OPS = Sets.newHashSet( - Operator.EQUALS, Operator.NOTEQUALS, Operator.NOTEQUALS2, Operator.LIKE); - - private void generateJDOFilterOverTables(Map<String, Object> params, - FilterBuilder filterBuilder) throws MetaException { - if (keyName.equals(hive_metastoreConstants.HIVE_FILTER_FIELD_OWNER)) { - keyName = "this.owner"; - } else if (keyName.equals(hive_metastoreConstants.HIVE_FILTER_FIELD_LAST_ACCESS)) { - //lastAccessTime expects an integer, so we cannot use the "like operator" - if (operator == Operator.LIKE) { - filterBuilder.setError("Like is not supported for HIVE_FILTER_FIELD_LAST_ACCESS"); - return; - } - keyName = "this.lastAccessTime"; - } else if (keyName.startsWith(hive_metastoreConstants.HIVE_FILTER_FIELD_PARAMS)) { - if (!TABLE_FILTER_OPS.contains(operator)) { - filterBuilder.setError("Only " + TABLE_FILTER_OPS + " are supported " + - "operators for HIVE_FILTER_FIELD_PARAMS"); - return; - } - String paramKeyName = keyName.substring(hive_metastoreConstants.HIVE_FILTER_FIELD_PARAMS.length()); - keyName = "this.parameters.get(\"" + paramKeyName + "\")"; - //value is persisted as a string in the db, so make sure it's a string here - // in case we get a long. - value = value.toString(); - } else { - filterBuilder.setError("Invalid key name in filter. " + - "Use constants from org.apache.hadoop.hive.metastore.api"); - return; - } - generateJDOFilterGeneral(params, filterBuilder); - } - - /** - * Generates a general filter. Given a map of <key, value>, - * generates a statement of the form: - * key1 operator value2 (&& | || ) key2 operator value2 ... - * - * Currently supported types for value are String and Long. - * The LIKE operator for Longs is unsupported. - */ - private void generateJDOFilterGeneral(Map<String, Object> params, - FilterBuilder filterBuilder) throws MetaException { - String paramName = PARAM_PREFIX + params.size(); - params.put(paramName, value); - - if (isReverseOrder) { - if (operator == Operator.LIKE) { - filterBuilder.setError("Value should be on the RHS for LIKE operator : " + - "Key <" + keyName + ">"); - } else { - filterBuilder.append(paramName + " " + operator.getJdoOp() + " " + keyName); - } - } else { - if (operator == Operator.LIKE) { - filterBuilder.append(" " + keyName + "." + operator.getJdoOp() + "(" + paramName + ") "); - } else { - filterBuilder.append(" " + keyName + " " + operator.getJdoOp() + " " + paramName); - } - } - } - - private void generateJDOFilterOverPartitions(Configuration conf, Table table, - Map<String, Object> params, FilterBuilder filterBuilder) throws MetaException { - int partitionColumnCount = table.getPartitionKeys().size(); - int partitionColumnIndex = getPartColIndexForFilter(table, filterBuilder); - if (filterBuilder.hasError()) return; - - boolean canPushDownIntegral = - HiveConf.getBoolVar(conf, HiveConf.ConfVars.METASTORE_INTEGER_JDO_PUSHDOWN); - String valueAsString = getJdoFilterPushdownParam( - table, partitionColumnIndex, filterBuilder, canPushDownIntegral); - if (filterBuilder.hasError()) return; - - String paramName = PARAM_PREFIX + params.size(); - params.put(paramName, valueAsString); - - boolean isOpEquals = operator == Operator.EQUALS; - if (isOpEquals || operator == Operator.NOTEQUALS || operator == Operator.NOTEQUALS2) { - makeFilterForEquals(keyName, valueAsString, paramName, params, - partitionColumnIndex, partitionColumnCount, isOpEquals, filterBuilder); - return; - } - //get the value for a partition key form MPartition.values (PARTITION_KEY_VALUES) - String valString = "values.get(" + partitionColumnIndex + ")"; - - if (operator == Operator.LIKE) { - if (isReverseOrder) { - // For LIKE, the value should be on the RHS. - filterBuilder.setError( - "Value should be on the RHS for LIKE operator : Key <" + keyName + ">"); - } - // TODO: in all likelihood, this won't actually work. Keep it for backward compat. - filterBuilder.append(" " + valString + "." + operator.getJdoOp() + "(" + paramName + ") "); - } else { - filterBuilder.append(isReverseOrder - ? paramName + " " + operator.getJdoOp() + " " + valString - : " " + valString + " " + operator.getJdoOp() + " " + paramName); - } - } - - /** - * @return true iff filter pushdown for this operator can be done for integral types. - */ - public boolean canJdoUseStringsWithIntegral() { - return (operator == Operator.EQUALS) - || (operator == Operator.NOTEQUALS) - || (operator == Operator.NOTEQUALS2); - } - - /** - * Get partition column index in the table partition column list that - * corresponds to the key that is being filtered on by this tree node. - * @param table The table. - * @param filterBuilder filter builder used to report error, if any. - * @return The index. - */ - public int getPartColIndexForFilter( - Table table, FilterBuilder filterBuilder) throws MetaException { - int partitionColumnIndex; - assert (table.getPartitionKeys().size() > 0); - for (partitionColumnIndex = 0; partitionColumnIndex < table.getPartitionKeys().size(); - ++partitionColumnIndex) { - if (table.getPartitionKeys().get(partitionColumnIndex).getName(). - equalsIgnoreCase(keyName)) { - break; - } - } - if( partitionColumnIndex == table.getPartitionKeys().size()) { - filterBuilder.setError("Specified key <" + keyName + - "> is not a partitioning key for the table"); - return -1; - } - - return partitionColumnIndex; - } - - /** - * Validates and gets the query parameter for JDO filter pushdown based on the column - * and the constant stored in this node. - * @param table The table. - * @param partColIndex The index of the column to check. - * @param filterBuilder filter builder used to report error, if any. - * @return The parameter string. - */ - private String getJdoFilterPushdownParam(Table table, int partColIndex, - FilterBuilder filterBuilder, boolean canPushDownIntegral) throws MetaException { - boolean isIntegralSupported = canPushDownIntegral && canJdoUseStringsWithIntegral(); - String colType = table.getPartitionKeys().get(partColIndex).getType(); - // Can only support partitions whose types are string, or maybe integers - if (!colType.equals(ColumnType.STRING_TYPE_NAME) - && (!isIntegralSupported || !ColumnType.IntegralTypes.contains(colType))) { - filterBuilder.setError("Filtering is supported only on partition keys of type " + - "string" + (isIntegralSupported ? ", or integral types" : "")); - return null; - } - - // There's no support for date cast in JDO. Let's convert it to string; the date - // columns have been excluded above, so it will either compare w/string or fail. - Object val = value; - if (value instanceof Date) { - val = HiveMetaStore.PARTITION_DATE_FORMAT.get().format((Date)value); - } - boolean isStringValue = val instanceof String; - if (!isStringValue && (!isIntegralSupported || !(val instanceof Long))) { - filterBuilder.setError("Filtering is supported only on partition keys of type " + - "string" + (isIntegralSupported ? ", or integral types" : "")); - return null; - } - - return isStringValue ? (String)val : Long.toString((Long)val); - } - } - - public void accept(TreeVisitor treeVisitor) throws MetaException { - if (this.root != null) { - this.root.accept(treeVisitor); - } - } - - /** - * For equals and not-equals, we can make the JDO query much faster by filtering - * based on the partition name. For a condition like ds="2010-10-01", we can see - * if there are any partitions with a name that contains the substring "ds=2010-10-01/" - * False matches aren't possible since "=" is escaped for partition names - * and the trailing '/' ensures that we won't get a match with ds=2010-10-011 - * Note that filters on integral type equality also work correctly by virtue of - * comparing them as part of ds=1234 string. - * - * Two cases to keep in mind: Case with only one partition column (no '/'s) - * Case where the partition key column is at the end of the name. (no - * tailing '/') - * - * @param keyName name of the partition column e.g. ds. - * @param value The value to compare to. - * @param paramName name of the parameter to use for JDOQL. - * @param params a map from the parameter name to their values. - * @param keyPos The index of the requisite partition column in the list of such columns. - * @param keyCount Partition column count for the table. - * @param isEq whether the operator is equals, or not-equals. - * @param fltr Filter builder used to append the filter, or report errors. - */ - private static void makeFilterForEquals(String keyName, String value, String paramName, - Map<String, Object> params, int keyPos, int keyCount, boolean isEq, FilterBuilder fltr) - throws MetaException { - Map<String, String> partKeyToVal = new HashMap<String, String>(); - partKeyToVal.put(keyName, value); - // If a partition has multiple partition keys, we make the assumption that - // makePartName with one key will return a substring of the name made - // with both all the keys. - String escapedNameFragment = Warehouse.makePartName(partKeyToVal, false); - if (keyCount == 1) { - // Case where this is no other partition columns - params.put(paramName, escapedNameFragment); - fltr.append("partitionName ").append(isEq ? "== " : "!= ").append(paramName); - } else if (keyPos + 1 == keyCount) { - // Case where the partition column is at the end of the name. There will - // be a leading '/' but no trailing '/' - params.put(paramName, "/" + escapedNameFragment); - fltr.append(isEq ? "" : "!").append("partitionName.endsWith(") - .append(paramName).append(")"); - } else if (keyPos == 0) { - // Case where the partition column is at the beginning of the name. There will - // be a trailing '/' but no leading '/' - params.put(paramName, escapedNameFragment + "/"); - fltr.append(isEq ? "" : "!").append("partitionName.startsWith(") - .append(paramName).append(")"); - } else { - // Case where the partition column is in the middle of the name. There will - // be a leading '/' and an trailing '/' - params.put(paramName, "/" + escapedNameFragment + "/"); - fltr.append("partitionName.indexOf(").append(paramName).append(")") - .append(isEq ? ">= 0" : "< 0"); - } - } - - /** - * The root node for the tree. - */ - private TreeNode root = null; - - /** - * The node stack used to keep track of the tree nodes during parsing. - */ - private final Stack<TreeNode> nodeStack = new Stack<TreeNode>(); - - public TreeNode getRoot() { - return this.root; - } - - @VisibleForTesting - public void setRootForTest(TreeNode tn) { - this.root = tn; - } - - - /** - * Adds a intermediate node of either type(AND/OR). Pops last two nodes from - * the stack and sets them as children of the new node and pushes itself - * onto the stack. - * @param andOr the operator type - */ - public void addIntermediateNode(LogicalOperator andOr) { - - TreeNode rhs = nodeStack.pop(); - TreeNode lhs = nodeStack.pop(); - TreeNode newNode = new TreeNode(lhs, andOr, rhs); - nodeStack.push(newNode); - root = newNode; - } - - /** - * Adds a leaf node, pushes the new node onto the stack. - * @param newNode the new node - */ - public void addLeafNode(LeafNode newNode) { - if( root == null ) { - root = newNode; - } - nodeStack.push(newNode); - } - - /** Generate the JDOQL filter for the given expression tree - * @param table the table being queried - * @param params the input map which is updated with the - * the parameterized values. Keys are the parameter names and values - * are the parameter values - * @param filterBuilder the filter builder to append to. - */ - public void generateJDOFilterFragment(Configuration conf, Table table, - Map<String, Object> params, FilterBuilder filterBuilder) throws MetaException { - if (root == null) { - return; - } - - filterBuilder.append(" && ( "); - root.generateJDOFilter(conf, table, params, filterBuilder); - filterBuilder.append(" )"); - } - - /** Case insensitive ANTLR string stream */ - public static class ANTLRNoCaseStringStream extends ANTLRStringStream { - public ANTLRNoCaseStringStream (String input) { - super(input); - } - - @Override - public int LA (int i) { - int returnChar = super.LA (i); - - if (returnChar == CharStream.EOF) { - return returnChar; - } - else if (returnChar == 0) { - return returnChar; - } - - return Character.toUpperCase ((char) returnChar); - } - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/133d3c47/metastore/src/java/org/apache/hadoop/hive/metastore/parser/Filter.g ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/parser/Filter.g b/metastore/src/java/org/apache/hadoop/hive/metastore/parser/Filter.g deleted file mode 100644 index 81111a0..0000000 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/parser/Filter.g +++ /dev/null @@ -1,484 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -grammar Filter; - -options -{ - k=3; -} - - -// Package headers -@header { -package org.apache.hadoop.hive.metastore.parser; - -import org.apache.hadoop.hive.metastore.parser.ExpressionTree; -import org.apache.hadoop.hive.metastore.parser.ExpressionTree.LeafNode; -import org.apache.hadoop.hive.metastore.parser.ExpressionTree.Operator; -import org.apache.hadoop.hive.metastore.parser.ExpressionTree.LogicalOperator; -} - -@lexer::header { -package org.apache.hadoop.hive.metastore.parser; - -import java.sql.Date; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.regex.Matcher; -import java.util.regex.Pattern; -} - -@lexer::members { - public String errorMsg; - - private static final Pattern datePattern = Pattern.compile(".*(\\d\\d\\d\\d-\\d\\d-\\d\\d).*"); - private static final ThreadLocal<SimpleDateFormat> dateFormat = - new ThreadLocal<SimpleDateFormat>() { - @Override - protected SimpleDateFormat initialValue() { - SimpleDateFormat val = new SimpleDateFormat("yyyy-MM-dd"); - val.setLenient(false); // Without this, 2020-20-20 becomes 2021-08-20. - return val; - }; - }; - - public static java.sql.Date ExtractDate (String input) { - Matcher m = datePattern.matcher(input); - if (!m.matches()) { - return null; - } - try { - return new java.sql.Date(dateFormat.get().parse(m.group(1)).getTime()); - } catch (ParseException pe) { - return null; - } - } - - @Override - public void emitErrorMessage(String msg) { - // save for caller to detect invalid filter - errorMsg = msg; - } -} - -@members { - public ExpressionTree tree = new ExpressionTree(); - - public static String TrimQuotes (String input) { - if (input.length () > 1) { - if ((input.charAt (0) == '"' && input.charAt (input.length () - 1) == '"') - || (input.charAt (0) == '\'' && input.charAt (input.length () - 1) == '\'')) { - return input.substring (1, input.length () - 1); - } - } - return input; - } -} - -@rulecatch{ - catch (RecognitionException e){ - throw e; - } -} - -//main rule -filter - : - orExpression - ; - -orExpression - : - andExpression (KW_OR andExpression { tree.addIntermediateNode(LogicalOperator.OR); } )* - ; - -andExpression - : - expression (KW_AND expression { tree.addIntermediateNode(LogicalOperator.AND); } )* - ; - -expression - : - LPAREN orExpression RPAREN - | - operatorExpression - ; - -operatorExpression - : - betweenExpression - | - inExpression - | - multiColInExpression - | - binOpExpression - ; - -binOpExpression -@init { - boolean isReverseOrder = false; - Object val = null; -} - : - ( - ( - (key = Identifier op = operator value = DateLiteral) - | - (value = DateLiteral op = operator key = Identifier) { isReverseOrder = true; } - ) { val = FilterLexer.ExtractDate(value.getText()); } - | - ( - (key = Identifier op = operator value = StringLiteral) - | - (value = StringLiteral op = operator key = Identifier) { isReverseOrder = true; } - ) { val = TrimQuotes(value.getText()); } - | - ( - (key = Identifier op = operator value = IntegralLiteral) - | - (value = IntegralLiteral op = operator key = Identifier) { isReverseOrder = true; } - ) { val = Long.parseLong(value.getText()); } - ) - { - LeafNode node = new LeafNode(); - node.keyName = key.getText(); - node.value = val; - node.operator = op; - node.isReverseOrder = isReverseOrder; - - tree.addLeafNode(node); - }; - -operator returns [Operator op] - : - t = (LESSTHAN | LESSTHANOREQUALTO | GREATERTHAN | GREATERTHANOREQUALTO | KW_LIKE | EQUAL | NOTEQUAL) - { - $op = Operator.fromString(t.getText().toUpperCase()); - }; - -betweenExpression -@init { - Object leftV = null; - Object rightV = null; - boolean isPositive = true; -} - : - ( - key = Identifier (KW_NOT { isPositive = false; } )? BETWEEN - ( - (left = DateLiteral KW_AND right = DateLiteral) { - leftV = FilterLexer.ExtractDate(left.getText()); - rightV = FilterLexer.ExtractDate(right.getText()); - } - | - (left = StringLiteral KW_AND right = StringLiteral) { leftV = TrimQuotes(left.getText()); - rightV = TrimQuotes(right.getText()); - } - | - (left = IntegralLiteral KW_AND right = IntegralLiteral) { leftV = Long.parseLong(left.getText()); - rightV = Long.parseLong(right.getText()); - } - ) - ) - { - LeafNode leftNode = new LeafNode(), rightNode = new LeafNode(); - leftNode.keyName = rightNode.keyName = key.getText(); - leftNode.value = leftV; - rightNode.value = rightV; - leftNode.operator = isPositive ? Operator.GREATERTHANOREQUALTO : Operator.LESSTHAN; - rightNode.operator = isPositive ? Operator.LESSTHANOREQUALTO : Operator.GREATERTHAN; - tree.addLeafNode(leftNode); - tree.addLeafNode(rightNode); - tree.addIntermediateNode(isPositive ? LogicalOperator.AND : LogicalOperator.OR); - }; - -inExpression -@init { - List constants = new ArrayList(); - Object constantV = null; - boolean isPositive = true; -} - : - ( - LPAREN key = Identifier RPAREN ( KW_NOT { isPositive = false; } )? IN LPAREN - ( - ( - constant = DateLiteral - { - constantV = FilterLexer.ExtractDate(constant.getText()); - constants.add(constantV); - } - ( - COMMA constant = DateLiteral - { - constantV = FilterLexer.ExtractDate(constant.getText()); - constants.add(constantV); - } - )* - ) - | - ( - constant = StringLiteral - { - constantV = TrimQuotes(constant.getText()); - constants.add(constantV); - } - ( - COMMA constant = StringLiteral - { - constantV = TrimQuotes(constant.getText()); - constants.add(constantV); - } - )* - ) - | - ( - constant = IntegralLiteral - { - constantV = Long.parseLong(constant.getText()); - constants.add(constantV); - } - ( - COMMA constant = IntegralLiteral - { - constantV = Long.parseLong(constant.getText()); - constants.add(constantV); - } - )* - ) - ) RPAREN - ) - { - for (int i = 0; i < constants.size(); i++) { - Object value = constants.get(i); - LeafNode leaf = new LeafNode(); - leaf.keyName = key.getText(); - leaf.value = value; - leaf.operator = isPositive ? Operator.EQUALS : Operator.NOTEQUALS2; - tree.addLeafNode(leaf); - if (i != 0) { - tree.addIntermediateNode(isPositive ? LogicalOperator.OR : LogicalOperator.AND); - } - } - }; - -multiColInExpression -@init { - List<String> keyNames = new ArrayList<String>(); - List constants = new ArrayList(); - List partialConstants; - String keyV = null; - Object constantV = null; - boolean isPositive = true; -} - : - ( - LPAREN - ( - KW_STRUCT LPAREN key = Identifier - { - keyV = key.getText(); - keyNames.add(keyV); - } - ( - COMMA key = Identifier - { - keyV = key.getText(); - keyNames.add(keyV); - } - )* RPAREN - ) RPAREN ( KW_NOT { isPositive = false; } )? IN LPAREN KW_CONST KW_STRUCT LPAREN - { - partialConstants = new ArrayList(); - } - ( - constant = DateLiteral - { - constantV = FilterLexer.ExtractDate(constant.getText()); - partialConstants.add(constantV); - } - | constant = StringLiteral - { - constantV = TrimQuotes(constant.getText()); - partialConstants.add(constantV); - } - | constant = IntegralLiteral - { - constantV = Long.parseLong(constant.getText()); - partialConstants.add(constantV); - } - ) - ( - COMMA - ( - constant = DateLiteral - { - constantV = FilterLexer.ExtractDate(constant.getText()); - partialConstants.add(constantV); - } - | constant = StringLiteral - { - constantV = TrimQuotes(constant.getText()); - partialConstants.add(constantV); - } - | constant = IntegralLiteral - { - constantV = Long.parseLong(constant.getText()); - partialConstants.add(constantV); - } - ) - )* - { - constants.add(partialConstants); - } - RPAREN - ( - COMMA KW_CONST KW_STRUCT LPAREN - { - partialConstants = new ArrayList(); - } - ( - constant = DateLiteral - { - constantV = FilterLexer.ExtractDate(constant.getText()); - partialConstants.add(constantV); - } - | constant = StringLiteral - { - constantV = TrimQuotes(constant.getText()); - partialConstants.add(constantV); - } - | constant = IntegralLiteral - { - constantV = Long.parseLong(constant.getText()); - partialConstants.add(constantV); - } - ) - ( - COMMA - ( - constant = DateLiteral - { - constantV = FilterLexer.ExtractDate(constant.getText()); - partialConstants.add(constantV); - } - | constant = StringLiteral - { - constantV = TrimQuotes(constant.getText()); - partialConstants.add(constantV); - } - | constant = IntegralLiteral - { - constantV = Long.parseLong(constant.getText()); - partialConstants.add(constantV); - } - ) - )* - { - constants.add(partialConstants); - } - RPAREN - )* RPAREN - ) - { - for (int i = 0; i < constants.size(); i++) { - List list = (List) constants.get(i); - assert keyNames.size() == list.size(); - for (int j=0; j < list.size(); j++) { - String keyName = keyNames.get(j); - Object value = list.get(j); - LeafNode leaf = new LeafNode(); - leaf.keyName = keyName; - leaf.value = value; - leaf.operator = isPositive ? Operator.EQUALS : Operator.NOTEQUALS2; - tree.addLeafNode(leaf); - if (j != 0) { - tree.addIntermediateNode(isPositive ? LogicalOperator.AND : LogicalOperator.OR); - } - } - if (i != 0) { - tree.addIntermediateNode(isPositive ? LogicalOperator.OR : LogicalOperator.AND); - } - } - }; - -// Keywords -KW_NOT : 'NOT'; -KW_AND : 'AND'; -KW_OR : 'OR'; -KW_LIKE : 'LIKE'; -KW_DATE : 'date'; -KW_CONST : 'CONST'; -KW_STRUCT : 'STRUCT'; - -// Operators -LPAREN : '(' ; -RPAREN : ')' ; -COMMA : ',' ; -EQUAL : '='; -NOTEQUAL : '<>' | '!='; -LESSTHANOREQUALTO : '<='; -LESSTHAN : '<'; -GREATERTHANOREQUALTO : '>='; -GREATERTHAN : '>'; -BETWEEN : 'BETWEEN'; -IN : 'IN'; - -// LITERALS -fragment -Letter - : 'a'..'z' | 'A'..'Z' - ; - -fragment -Digit - : - '0'..'9' - ; - -fragment DateString - : - (Digit)(Digit)(Digit)(Digit) '-' (Digit)(Digit) '-' (Digit)(Digit) - ; - -/* When I figure out how to make lexer backtrack after validating predicate, dates would be able -to support single quotes [( '\'' DateString '\'' ) |]. For now, what we do instead is have a hack -to parse the string in metastore code from StringLiteral. */ -DateLiteral - : - KW_DATE? DateString { ExtractDate(getText()) != null }? - ; - -StringLiteral - : - ( '\'' ( ~('\''|'\\') | ('\\' .) )* '\'' - | '\"' ( ~('\"'|'\\') | ('\\' .) )* '\"' - ) - ; - -IntegralLiteral - : - ('-')? (Digit)+ - ; - -Identifier - : - (Letter | Digit) (Letter | Digit | '_')* - ; - -WS : (' '|'\r'|'\t'|'\n')+ { skip(); } ; http://git-wip-us.apache.org/repos/asf/hive/blob/133d3c47/metastore/src/java/org/apache/hadoop/hive/metastore/parser/package-info.java ---------------------------------------------------------------------- diff --git a/metastore/src/java/org/apache/hadoop/hive/metastore/parser/package-info.java b/metastore/src/java/org/apache/hadoop/hive/metastore/parser/package-info.java deleted file mode 100644 index 29f1f0b..0000000 --- a/metastore/src/java/org/apache/hadoop/hive/metastore/parser/package-info.java +++ /dev/null @@ -1,23 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/** - * This package implements the parser for parsing the string filter - * for the listPartitionsByFilter API. - */ -package org.apache.hadoop.hive.metastore.parser; http://git-wip-us.apache.org/repos/asf/hive/blob/133d3c47/metastore/src/test/org/apache/hadoop/hive/metastore/TestDeadline.java ---------------------------------------------------------------------- diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/TestDeadline.java b/metastore/src/test/org/apache/hadoop/hive/metastore/TestDeadline.java deleted file mode 100644 index ea11ed5..0000000 --- a/metastore/src/test/org/apache/hadoop/hive/metastore/TestDeadline.java +++ /dev/null @@ -1,127 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.hadoop.hive.metastore; - -import org.apache.hadoop.hive.metastore.api.MetaException; -import org.junit.Assert; -import org.junit.Test; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -/** - * Test the Deadline class used for long running requests. - */ -public class TestDeadline { - - private static final Logger LOG = LoggerFactory.getLogger(TestDeadline.class); - private long timeout = 1000; - private long resetTimeout = 200; - private long duration = 500; - private boolean isFailed; - private String errorMsg; - - @Test - public void testDeadline() throws Exception { - isFailed = false; - errorMsg = ""; - - Thread threadTimeout = new Thread(createRunnable()); - threadTimeout.setDaemon(true); - threadTimeout.start(); - threadTimeout.join(60000); - - if (isFailed) { - Assert.fail(errorMsg); - } - } - - private Runnable createRunnable() { - return new Runnable() { - @Override - public void run() { - Deadline.registerIfNot(timeout); - try { - // normal - start(); - try { - Deadline.checkTimeout(); - } catch (MetaException e) { - failInThread("should not timeout", e); - return; - } - Deadline.stopTimer(); - - // normal. Check stopTimer() works. - start(); - try { - Deadline.checkTimeout(); - } catch (MetaException e) { - failInThread("should not timeout", e); - return; - } - Deadline.stopTimer(); - - // reset - Deadline.resetTimeout(resetTimeout); - - // timeout - start(); - try { - Deadline.checkTimeout(); - failInThread("should timeout.", null); - return; - } catch (MetaException e) { - if (e.getCause() instanceof DeadlineException) { - Deadline.clear(); - } else { - failInThread("new MetaException failed.", e); - return; - } - } - if (Deadline.getCurrentDeadline() != null) { - failInThread("the threadlocal object should be removed after timeout.", null); - } - } catch (MetaException e) { - failInThread("error happens in start, end, or reset. Check the exception.", e); - } - } - }; - } - - private void start() throws MetaException { - Deadline.startTimer("test"); - try { - Thread.sleep(duration); - } catch (InterruptedException e) { - failInThread("Odd. Sleeping thread is interrupted.", e); - } - } - - private void failInThread(String msg, Exception e) { - isFailed = true; - - if (e != null) { - LOG.error(msg, e); - errorMsg = msg + ": " + e.getMessage(); - } else { - LOG.error(msg); - errorMsg = msg; - } - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/133d3c47/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java ---------------------------------------------------------------------- diff --git a/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java b/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java deleted file mode 100644 index 67b798f..0000000 --- a/metastore/src/test/org/apache/hadoop/hive/metastore/TestObjectStore.java +++ /dev/null @@ -1,583 +0,0 @@ -/* - Licensed to the Apache Software Foundation (ASF) under one - or more contributor license agreements. See the NOTICE file - distributed with this work for additional information - regarding copyright ownership. The ASF licenses this file - to you under the Apache License, Version 2.0 (the - "License"); you may not use this file except in compliance - with the License. You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - - Unless required by applicable law or agreed to in writing, software - distributed under the License is distributed on an "AS IS" BASIS, - WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - See the License for the specific language governing permissions and - limitations under the License. - */ -package org.apache.hadoop.hive.metastore; - - -import com.codahale.metrics.Counter; -import com.google.common.base.Supplier; -import com.google.common.collect.ImmutableList; -import org.apache.hadoop.hive.conf.HiveConf; -import org.apache.hadoop.hive.metastore.ObjectStore.RetryingExecutor; -import org.apache.hadoop.hive.metastore.api.CurrentNotificationEventId; -import org.apache.hadoop.hive.metastore.api.Database; -import org.apache.hadoop.hive.metastore.api.FieldSchema; -import org.apache.hadoop.hive.metastore.api.FileMetadataExprType; -import org.apache.hadoop.hive.metastore.api.Function; -import org.apache.hadoop.hive.metastore.api.Index; -import org.apache.hadoop.hive.metastore.api.InvalidInputException; -import org.apache.hadoop.hive.metastore.api.InvalidObjectException; -import org.apache.hadoop.hive.metastore.api.MetaException; -import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; -import org.apache.hadoop.hive.metastore.api.NotificationEvent; -import org.apache.hadoop.hive.metastore.api.NotificationEventRequest; -import org.apache.hadoop.hive.metastore.api.NotificationEventResponse; -import org.apache.hadoop.hive.metastore.api.Partition; -import org.apache.hadoop.hive.metastore.api.PrincipalType; -import org.apache.hadoop.hive.metastore.api.Role; -import org.apache.hadoop.hive.metastore.api.SQLForeignKey; -import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey; -import org.apache.hadoop.hive.metastore.api.SerDeInfo; -import org.apache.hadoop.hive.metastore.api.StorageDescriptor; -import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.hive.metastore.messaging.EventMessage; -import org.apache.hadoop.hive.metastore.metrics.Metrics; -import org.apache.hadoop.hive.metastore.metrics.MetricsConstants; -import org.apache.hadoop.hive.metastore.model.MNotificationLog; -import org.apache.hadoop.hive.metastore.model.MNotificationNextId; -import org.apache.hadoop.hive.ql.io.sarg.SearchArgument; -import org.junit.After; -import org.junit.Assert; -import org.junit.Before; -import org.junit.Ignore; -import org.junit.Test; -import org.mockito.Mockito; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import javax.jdo.Query; -import java.util.Arrays; -import java.util.HashMap; -import java.util.HashSet; -import java.util.List; -import java.util.Set; -import java.util.concurrent.BrokenBarrierException; -import java.util.concurrent.CyclicBarrier; -import java.util.concurrent.ExecutionException; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; -import java.util.concurrent.TimeUnit; - -import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertTrue; - -public class TestObjectStore { - private ObjectStore objectStore = null; - - private static final String DB1 = "testobjectstoredb1"; - private static final String DB2 = "testobjectstoredb2"; - private static final String TABLE1 = "testobjectstoretable1"; - private static final String KEY1 = "testobjectstorekey1"; - private static final String KEY2 = "testobjectstorekey2"; - private static final String OWNER = "testobjectstoreowner"; - private static final String USER1 = "testobjectstoreuser1"; - private static final String ROLE1 = "testobjectstorerole1"; - private static final String ROLE2 = "testobjectstorerole2"; - private static final Logger LOG = LoggerFactory.getLogger(TestObjectStore.class.getName()); - - private static final class LongSupplier implements Supplier<Long> { - public long value = 0; - - @Override - public Long get() { - return value; - } - } - - public static class MockPartitionExpressionProxy implements PartitionExpressionProxy { - @Override - public String convertExprToFilter(byte[] expr) throws MetaException { - return null; - } - - @Override - public boolean filterPartitionsByExpr(List<FieldSchema> partColumns, - byte[] expr, String defaultPartitionName, List<String> partitionNames) - throws MetaException { - return false; - } - - @Override - public FileMetadataExprType getMetadataType(String inputFormat) { - return null; - } - - @Override - public SearchArgument createSarg(byte[] expr) { - return null; - } - - @Override - public FileFormatProxy getFileFormatProxy(FileMetadataExprType type) { - return null; - } - } - - @Before - public void setUp() throws Exception { - HiveConf conf = new HiveConf(); - conf.setVar(HiveConf.ConfVars.METASTORE_EXPRESSION_PROXY_CLASS, MockPartitionExpressionProxy.class.getName()); - - objectStore = new ObjectStore(); - objectStore.setConf(conf); - dropAllStoreObjects(objectStore); - } - - @After - public void tearDown() { - } - - /** - * Test notification operations - */ - @Test - public void testNotificationOps() throws InterruptedException { - final int NO_EVENT_ID = 0; - final int FIRST_EVENT_ID = 1; - final int SECOND_EVENT_ID = 2; - - NotificationEvent event = - new NotificationEvent(0, 0, EventMessage.EventType.CREATE_DATABASE.toString(), ""); - NotificationEventResponse eventResponse; - CurrentNotificationEventId eventId; - - // Verify that there is no notifications available yet - eventId = objectStore.getCurrentNotificationEventId(); - Assert.assertEquals(NO_EVENT_ID, eventId.getEventId()); - - // Verify that addNotificationEvent() updates the NotificationEvent with the new event ID - objectStore.addNotificationEvent(event); - Assert.assertEquals(FIRST_EVENT_ID, event.getEventId()); - objectStore.addNotificationEvent(event); - Assert.assertEquals(SECOND_EVENT_ID, event.getEventId()); - - // Verify that objectStore fetches the latest notification event ID - eventId = objectStore.getCurrentNotificationEventId(); - Assert.assertEquals(SECOND_EVENT_ID, eventId.getEventId()); - - // Verify that getNextNotification() returns all events - eventResponse = objectStore.getNextNotification(new NotificationEventRequest()); - Assert.assertEquals(2, eventResponse.getEventsSize()); - Assert.assertEquals(FIRST_EVENT_ID, eventResponse.getEvents().get(0).getEventId()); - Assert.assertEquals(SECOND_EVENT_ID, eventResponse.getEvents().get(1).getEventId()); - - // Verify that getNextNotification(last) returns events after a specified event - eventResponse = objectStore.getNextNotification(new NotificationEventRequest(FIRST_EVENT_ID)); - Assert.assertEquals(1, eventResponse.getEventsSize()); - Assert.assertEquals(SECOND_EVENT_ID, eventResponse.getEvents().get(0).getEventId()); - - // Verify that getNextNotification(last) returns zero events if there are no more notifications available - eventResponse = objectStore.getNextNotification(new NotificationEventRequest(SECOND_EVENT_ID)); - Assert.assertEquals(0, eventResponse.getEventsSize()); - - // Verify that cleanNotificationEvents() cleans up all old notifications - Thread.sleep(1); - objectStore.cleanNotificationEvents(1); - eventResponse = objectStore.getNextNotification(new NotificationEventRequest()); - Assert.assertEquals(0, eventResponse.getEventsSize()); - } - - /** - * Test database operations - */ - @Test - public void testDatabaseOps() throws MetaException, InvalidObjectException, NoSuchObjectException { - Database db1 = new Database(DB1, "description", "locationurl", null); - Database db2 = new Database(DB2, "description", "locationurl", null); - objectStore.createDatabase(db1); - objectStore.createDatabase(db2); - - List<String> databases = objectStore.getAllDatabases(); - LOG.info("databases: " + databases); - Assert.assertEquals(2, databases.size()); - Assert.assertEquals(DB1, databases.get(0)); - Assert.assertEquals(DB2, databases.get(1)); - - objectStore.dropDatabase(DB1); - databases = objectStore.getAllDatabases(); - Assert.assertEquals(1, databases.size()); - Assert.assertEquals(DB2, databases.get(0)); - - objectStore.dropDatabase(DB2); - } - - /** - * Test table operations - */ - @Test - public void testTableOps() throws MetaException, InvalidObjectException, NoSuchObjectException, InvalidInputException { - Database db1 = new Database(DB1, "description", "locationurl", null); - objectStore.createDatabase(db1); - StorageDescriptor sd1 = new StorageDescriptor(ImmutableList.of(new FieldSchema("pk_col", "double", null)), - "location", null, null, false, 0, new SerDeInfo("SerDeName", "serializationLib", null), - null, null, null); - HashMap<String, String> params = new HashMap<>(); - params.put("EXTERNAL", "false"); - Table tbl1 = new Table(TABLE1, DB1, "owner", 1, 2, 3, sd1, null, params, null, null, "MANAGED_TABLE"); - objectStore.createTable(tbl1); - - List<String> tables = objectStore.getAllTables(DB1); - Assert.assertEquals(1, tables.size()); - Assert.assertEquals(TABLE1, tables.get(0)); - - StorageDescriptor sd2 = new StorageDescriptor(ImmutableList.of(new FieldSchema("fk_col", "double", null)), - "location", null, null, false, 0, new SerDeInfo("SerDeName", "serializationLib", null), - null, null, null); - Table newTbl1 = new Table("new" + TABLE1, DB1, "owner", 1, 2, 3, sd2, null, params, null, null, "MANAGED_TABLE"); - objectStore.alterTable(DB1, TABLE1, newTbl1); - tables = objectStore.getTables(DB1, "new*"); - Assert.assertEquals(1, tables.size()); - Assert.assertEquals("new" + TABLE1, tables.get(0)); - - objectStore.createTable(tbl1); - tables = objectStore.getAllTables(DB1); - Assert.assertEquals(2, tables.size()); - - List<SQLForeignKey> foreignKeys = objectStore.getForeignKeys(DB1, TABLE1, null, null); - Assert.assertEquals(0, foreignKeys.size()); - - SQLPrimaryKey pk = new SQLPrimaryKey(DB1, TABLE1, "pk_col", 1, - "pk_const_1", false, false, false); - objectStore.addPrimaryKeys(ImmutableList.of(pk)); - SQLForeignKey fk = new SQLForeignKey(DB1, TABLE1, "pk_col", - DB1, "new" + TABLE1, "fk_col", 1, - 0, 0, "fk_const_1", "pk_const_1", false, false, false); - objectStore.addForeignKeys(ImmutableList.of(fk)); - - // Retrieve from PK side - foreignKeys = objectStore.getForeignKeys(null, null, DB1, "new" + TABLE1); - Assert.assertEquals(1, foreignKeys.size()); - - List<SQLForeignKey> fks = objectStore.getForeignKeys(null, null, DB1, "new" + TABLE1); - if (fks != null) { - for (SQLForeignKey fkcol : fks) { - objectStore.dropConstraint(fkcol.getFktable_db(), fkcol.getFktable_name(), fkcol.getFk_name()); - } - } - // Retrieve from FK side - foreignKeys = objectStore.getForeignKeys(DB1, TABLE1, null, null); - Assert.assertEquals(0, foreignKeys.size()); - // Retrieve from PK side - foreignKeys = objectStore.getForeignKeys(null, null, DB1, "new" + TABLE1); - Assert.assertEquals(0, foreignKeys.size()); - - objectStore.dropTable(DB1, TABLE1); - tables = objectStore.getAllTables(DB1); - Assert.assertEquals(1, tables.size()); - - objectStore.dropTable(DB1, "new" + TABLE1); - tables = objectStore.getAllTables(DB1); - Assert.assertEquals(0, tables.size()); - - objectStore.dropDatabase(DB1); - } - - private StorageDescriptor createFakeSd(String location) { - return new StorageDescriptor(null, location, null, null, false, 0, - new SerDeInfo("SerDeName", "serializationLib", null), null, null, null); - } - - - /** - * Tests partition operations - */ - @Test - public void testPartitionOps() throws MetaException, InvalidObjectException, NoSuchObjectException, InvalidInputException { - Database db1 = new Database(DB1, "description", "locationurl", null); - objectStore.createDatabase(db1); - StorageDescriptor sd = createFakeSd("location"); - HashMap<String, String> tableParams = new HashMap<>(); - tableParams.put("EXTERNAL", "false"); - FieldSchema partitionKey1 = new FieldSchema("Country", ColumnType.STRING_TYPE_NAME, ""); - FieldSchema partitionKey2 = new FieldSchema("State", ColumnType.STRING_TYPE_NAME, ""); - Table tbl1 = new Table(TABLE1, DB1, "owner", 1, 2, 3, sd, Arrays.asList(partitionKey1, partitionKey2), tableParams, null, null, "MANAGED_TABLE"); - objectStore.createTable(tbl1); - HashMap<String, String> partitionParams = new HashMap<>(); - partitionParams.put("PARTITION_LEVEL_PRIVILEGE", "true"); - List<String> value1 = Arrays.asList("US", "CA"); - Partition part1 = new Partition(value1, DB1, TABLE1, 111, 111, sd, partitionParams); - objectStore.addPartition(part1); - List<String> value2 = Arrays.asList("US", "MA"); - Partition part2 = new Partition(value2, DB1, TABLE1, 222, 222, sd, partitionParams); - objectStore.addPartition(part2); - - Deadline.startTimer("getPartition"); - List<Partition> partitions = objectStore.getPartitions(DB1, TABLE1, 10); - Assert.assertEquals(2, partitions.size()); - Assert.assertEquals(111, partitions.get(0).getCreateTime()); - Assert.assertEquals(222, partitions.get(1).getCreateTime()); - - int numPartitions = objectStore.getNumPartitionsByFilter(DB1, TABLE1, ""); - Assert.assertEquals(partitions.size(), numPartitions); - - numPartitions = objectStore.getNumPartitionsByFilter(DB1, TABLE1, "country = \"US\""); - Assert.assertEquals(2, numPartitions); - - objectStore.dropPartition(DB1, TABLE1, value1); - partitions = objectStore.getPartitions(DB1, TABLE1, 10); - Assert.assertEquals(1, partitions.size()); - Assert.assertEquals(222, partitions.get(0).getCreateTime()); - - objectStore.dropPartition(DB1, TABLE1, value2); - objectStore.dropTable(DB1, TABLE1); - objectStore.dropDatabase(DB1); - } - - /** - * Test master keys operation - */ - @Test - public void testMasterKeyOps() throws MetaException, NoSuchObjectException { - int id1 = objectStore.addMasterKey(KEY1); - int id2 = objectStore.addMasterKey(KEY2); - - String[] keys = objectStore.getMasterKeys(); - Assert.assertEquals(2, keys.length); - Assert.assertEquals(KEY1, keys[0]); - Assert.assertEquals(KEY2, keys[1]); - - objectStore.updateMasterKey(id1, "new" + KEY1); - objectStore.updateMasterKey(id2, "new" + KEY2); - keys = objectStore.getMasterKeys(); - Assert.assertEquals(2, keys.length); - Assert.assertEquals("new" + KEY1, keys[0]); - Assert.assertEquals("new" + KEY2, keys[1]); - - objectStore.removeMasterKey(id1); - keys = objectStore.getMasterKeys(); - Assert.assertEquals(1, keys.length); - Assert.assertEquals("new" + KEY2, keys[0]); - - objectStore.removeMasterKey(id2); - } - - /** - * Test role operation - */ - @Test - public void testRoleOps() throws InvalidObjectException, MetaException, NoSuchObjectException { - objectStore.addRole(ROLE1, OWNER); - objectStore.addRole(ROLE2, OWNER); - List<String> roles = objectStore.listRoleNames(); - Assert.assertEquals(2, roles.size()); - Assert.assertEquals(ROLE2, roles.get(1)); - Role role1 = objectStore.getRole(ROLE1); - Assert.assertEquals(OWNER, role1.getOwnerName()); - objectStore.grantRole(role1, USER1, PrincipalType.USER, OWNER, PrincipalType.ROLE, true); - objectStore.revokeRole(role1, USER1, PrincipalType.USER, false); - objectStore.removeRole(ROLE1); - } - - @Test - public void testDirectSqlErrorMetrics() throws Exception { - HiveConf conf = new HiveConf(); - conf.setBoolVar(HiveConf.ConfVars.HIVE_SERVER2_METRICS_ENABLED, true); - Metrics.initialize(conf); - conf.setVar(HiveConf.ConfVars.HIVE_CODAHALE_METRICS_REPORTER_CLASSES, - "org.apache.hadoop.hive.common.metrics.metrics2.JsonFileMetricsReporter, " + - "org.apache.hadoop.hive.common.metrics.metrics2.JmxMetricsReporter" - ); - - // recall setup so that we get an object store with the metrics initalized - setUp(); - Counter directSqlErrors = - Metrics.getRegistry().getCounters().get(MetricsConstants.DIRECTSQL_ERRORS); - - objectStore.new GetDbHelper("foo", true, true) { - @Override - protected Database getSqlResult(ObjectStore.GetHelper<Database> ctx) throws MetaException { - return null; - } - - @Override - protected Database getJdoResult(ObjectStore.GetHelper<Database> ctx) throws MetaException, - NoSuchObjectException { - return null; - } - }.run(false); - - Assert.assertEquals(0, directSqlErrors.getCount()); - - objectStore.new GetDbHelper("foo", true, true) { - @Override - protected Database getSqlResult(ObjectStore.GetHelper<Database> ctx) throws MetaException { - throw new RuntimeException(); - } - - @Override - protected Database getJdoResult(ObjectStore.GetHelper<Database> ctx) throws MetaException, - NoSuchObjectException { - return null; - } - }.run(false); - - Assert.assertEquals(1, directSqlErrors.getCount()); - } - - private static void dropAllStoreObjects(RawStore store) - throws MetaException, InvalidObjectException, InvalidInputException { - try { - Deadline.registerIfNot(100000); - List<Function> functions = store.getAllFunctions(); - for (Function func : functions) { - store.dropFunction(func.getDbName(), func.getFunctionName()); - } - List<String> dbs = store.getAllDatabases(); - for (String db : dbs) { - List<String> tbls = store.getAllTables(db); - for (String tbl : tbls) { - List<Index> indexes = store.getIndexes(db, tbl, 100); - for (Index index : indexes) { - store.dropIndex(db, tbl, index.getIndexName()); - } - } - for (String tbl : tbls) { - Deadline.startTimer("getPartition"); - List<Partition> parts = store.getPartitions(db, tbl, 100); - for (Partition part : parts) { - store.dropPartition(db, tbl, part.getValues()); - } - // Find any constraints and drop them - Set<String> constraints = new HashSet<>(); - List<SQLPrimaryKey> pk = store.getPrimaryKeys(db, tbl); - if (pk != null) { - for (SQLPrimaryKey pkcol : pk) { - constraints.add(pkcol.getPk_name()); - } - } - List<SQLForeignKey> fks = store.getForeignKeys(null, null, db, tbl); - if (fks != null) { - for (SQLForeignKey fkcol : fks) { - constraints.add(fkcol.getFk_name()); - } - } - for (String constraint : constraints) { - store.dropConstraint(db, tbl, constraint); - } - store.dropTable(db, tbl); - } - store.dropDatabase(db); - } - List<String> roles = store.listRoleNames(); - for (String role : roles) { - store.removeRole(role); - } - } catch (NoSuchObjectException e) { - } - } - - @Test - public void testQueryCloseOnError() throws Exception { - ObjectStore spy = Mockito.spy(objectStore); - spy.getAllDatabases(); - spy.getAllFunctions(); - spy.getAllTables(DB1); - spy.getPartitionCount(); - Mockito.verify(spy, Mockito.times(3)) - .rollbackAndCleanup(Mockito.anyBoolean(), Mockito.<Query>anyObject()); - } - - @Ignore( - "This test is here to allow testing with other databases like mysql / postgres etc\n" - + " with user changes to the code. This cannot be run on apache derby because of\n" - + " https://db.apache.org/derby/docs/10.10/devguide/cdevconcepts842385.html"; - ) - @Test - public void testConcurrentAddNotifications() throws ExecutionException, InterruptedException { - - final int NUM_THREADS = 10; - CyclicBarrier cyclicBarrier = new CyclicBarrier(NUM_THREADS, - () -> LoggerFactory.getLogger("test") - .debug(NUM_THREADS + " threads going to add notification")); - - HiveConf conf = new HiveConf(); - conf.setVar(HiveConf.ConfVars.METASTORE_EXPRESSION_PROXY_CLASS, - MockPartitionExpressionProxy.class.getName()); - /* - Below are the properties that need to be set based on what database this test is going to be run - */ - -// conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_DRIVER, "com.mysql.jdbc.Driver"); -// conf.setVar(HiveConf.ConfVars.METASTORECONNECTURLKEY, -// "jdbc:mysql://localhost:3306/metastore_db"); -// conf.setVar(HiveConf.ConfVars.METASTORE_CONNECTION_USER_NAME, ""); -// conf.setVar(HiveConf.ConfVars.METASTOREPWD, ""); - - /* - we have to add this one manually as for tests the db is initialized via the metastoreDiretSQL - and we don't run the schema creation sql that includes the an insert for notification_sequence - which can be locked. the entry in notification_sequence happens via notification_event insertion. - */ - objectStore.getPersistenceManager().newQuery(MNotificationLog.class, "eventType==''").execute(); - objectStore.getPersistenceManager().newQuery(MNotificationNextId.class, "nextEventId==-1").execute(); - - objectStore.addNotificationEvent( - new NotificationEvent(0, 0, - EventMessage.EventType.CREATE_DATABASE.toString(), - "CREATE DATABASE DB initial")); - - ExecutorService executorService = Executors.newFixedThreadPool(NUM_THREADS); - for (int i = 0; i < NUM_THREADS; i++) { - final int n = i; - - executorService.execute( - () -> { - ObjectStore store = new ObjectStore(); - store.setConf(conf); - - String eventType = EventMessage.EventType.CREATE_DATABASE.toString(); - NotificationEvent dbEvent = - new NotificationEvent(0, 0, eventType, - "CREATE DATABASE DB" + n); - System.out.println("ADDING NOTIFICATION"); - - try { - cyclicBarrier.await(); - } catch (InterruptedException | BrokenBarrierException e) { - throw new RuntimeException(e); - } - store.addNotificationEvent(dbEvent); - System.out.println("FINISH NOTIFICATION"); - }); - } - executorService.shutdown(); - assertTrue(executorService.awaitTermination(15, TimeUnit.SECONDS)); - - // we have to setup this again as the underlying PMF keeps getting reinitialized with original - // reference closed - ObjectStore store = new ObjectStore(); - store.setConf(conf); - - NotificationEventResponse eventResponse = store.getNextNotification( - new NotificationEventRequest()); - assertEquals(NUM_THREADS + 1, eventResponse.getEventsSize()); - long previousId = 0; - for (NotificationEvent event : eventResponse.getEvents()) { - assertTrue("previous:" + previousId + " current:" + event.getEventId(), - previousId < event.getEventId()); - assertTrue(previousId + 1 == event.getEventId()); - previousId = event.getEventId(); - } - } - - @Test - public void testRetryingExecutorSleep() throws Exception { - RetryingExecutor re = new ObjectStore.RetryingExecutor(new HiveConf(), null); - assertTrue("invalid sleep value", re.getSleepInterval() >= 0); - } - -}
