http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/cache/StringColumnStatsDataInspector.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/cache/StringColumnStatsDataInspector.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/cache/StringColumnStatsDataInspector.java deleted file mode 100644 index 0876410..0000000 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/cache/StringColumnStatsDataInspector.java +++ /dev/null @@ -1,125 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.metastore.columnstats.cache; - -import java.nio.ByteBuffer; - -import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator; -import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimatorFactory; -import org.apache.hadoop.hive.metastore.api.StringColumnStatsData; - -@SuppressWarnings("serial") -public class StringColumnStatsDataInspector extends StringColumnStatsData { - - private NumDistinctValueEstimator ndvEstimator; - - public StringColumnStatsDataInspector() { - super(); - } - - public StringColumnStatsDataInspector(long maxColLen, double avgColLen, - long numNulls, long numDVs) { - super(maxColLen, avgColLen, numNulls, numDVs); - } - - public StringColumnStatsDataInspector(StringColumnStatsDataInspector other) { - super(other); - if (other.ndvEstimator != null) { - super.setBitVectors(ndvEstimator.serialize()); - } - } - - @Override - public StringColumnStatsDataInspector deepCopy() { - return new StringColumnStatsDataInspector(this); - } - - @Override - public byte[] getBitVectors() { - if (ndvEstimator != null) { - updateBitVectors(); - } - return super.getBitVectors(); - } - - @Override - public ByteBuffer bufferForBitVectors() { - if (ndvEstimator != null) { - updateBitVectors(); - } - return super.bufferForBitVectors(); - } - - @Override - public void setBitVectors(byte[] bitVectors) { - super.setBitVectors(bitVectors); - this.ndvEstimator = null; - } - - @Override - public void setBitVectors(ByteBuffer bitVectors) { - super.setBitVectors(bitVectors); - this.ndvEstimator = null; - } - - @Override - public void unsetBitVectors() { - super.unsetBitVectors(); - this.ndvEstimator = null; - } - - @Override - public boolean isSetBitVectors() { - if (ndvEstimator != null) { - updateBitVectors(); - } - return super.isSetBitVectors(); - } - - @Override - public void setBitVectorsIsSet(boolean value) { - if (ndvEstimator != null) { - updateBitVectors(); - } - super.setBitVectorsIsSet(value); - } - - public NumDistinctValueEstimator getNdvEstimator() { - if (isSetBitVectors() && getBitVectors().length != 0) { - updateNdvEstimator(); - } - return ndvEstimator; - } - - public void setNdvEstimator(NumDistinctValueEstimator ndvEstimator) { - super.unsetBitVectors(); - this.ndvEstimator = ndvEstimator; - } - - private void updateBitVectors() { - super.setBitVectors(ndvEstimator.serialize()); - this.ndvEstimator = null; - } - - private void updateNdvEstimator() { - this.ndvEstimator = NumDistinctValueEstimatorFactory - .getNumDistinctValueEstimator(super.getBitVectors()); - super.unsetBitVectors(); - } - -}
http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/BinaryColumnStatsMerger.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/BinaryColumnStatsMerger.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/BinaryColumnStatsMerger.java deleted file mode 100644 index 1c2402f..0000000 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/BinaryColumnStatsMerger.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hive.metastore.columnstats.merge; - -import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData; -import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; - -public class BinaryColumnStatsMerger extends ColumnStatsMerger { - - @Override - public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) { - BinaryColumnStatsData aggregateData = aggregateColStats.getStatsData().getBinaryStats(); - BinaryColumnStatsData newData = newColStats.getStatsData().getBinaryStats(); - aggregateData.setMaxColLen(Math.max(aggregateData.getMaxColLen(), newData.getMaxColLen())); - aggregateData.setAvgColLen(Math.max(aggregateData.getAvgColLen(), newData.getAvgColLen())); - aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls()); - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/BooleanColumnStatsMerger.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/BooleanColumnStatsMerger.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/BooleanColumnStatsMerger.java deleted file mode 100644 index fd6b87a..0000000 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/BooleanColumnStatsMerger.java +++ /dev/null @@ -1,35 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hive.metastore.columnstats.merge; - -import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData; -import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; - -public class BooleanColumnStatsMerger extends ColumnStatsMerger { - - @Override - public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) { - BooleanColumnStatsData aggregateData = aggregateColStats.getStatsData().getBooleanStats(); - BooleanColumnStatsData newData = newColStats.getStatsData().getBooleanStats(); - aggregateData.setNumTrues(aggregateData.getNumTrues() + newData.getNumTrues()); - aggregateData.setNumFalses(aggregateData.getNumFalses() + newData.getNumFalses()); - aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls()); - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/ColumnStatsMerger.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/ColumnStatsMerger.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/ColumnStatsMerger.java deleted file mode 100644 index ce55756..0000000 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/ColumnStatsMerger.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hive.metastore.columnstats.merge; - -import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -public abstract class ColumnStatsMerger { - protected final Logger LOG = LoggerFactory.getLogger(ColumnStatsMerger.class.getName()); - - public abstract void merge(ColumnStatisticsObj aggregateColStats, - ColumnStatisticsObj newColStats); -} http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/ColumnStatsMergerFactory.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/ColumnStatsMergerFactory.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/ColumnStatsMergerFactory.java deleted file mode 100644 index 64d07c7..0000000 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/ColumnStatsMergerFactory.java +++ /dev/null @@ -1,120 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hive.metastore.columnstats.merge; - -import org.apache.hadoop.hive.metastore.api.BinaryColumnStatsData; -import org.apache.hadoop.hive.metastore.api.BooleanColumnStatsData; -import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData; -import org.apache.hadoop.hive.metastore.api.ColumnStatisticsData._Fields; -import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; -import org.apache.hadoop.hive.metastore.columnstats.cache.DateColumnStatsDataInspector; -import org.apache.hadoop.hive.metastore.columnstats.cache.DecimalColumnStatsDataInspector; -import org.apache.hadoop.hive.metastore.columnstats.cache.DoubleColumnStatsDataInspector; -import org.apache.hadoop.hive.metastore.columnstats.cache.LongColumnStatsDataInspector; -import org.apache.hadoop.hive.metastore.columnstats.cache.StringColumnStatsDataInspector; - -public class ColumnStatsMergerFactory { - - private ColumnStatsMergerFactory() { - } - - public static ColumnStatsMerger getColumnStatsMerger(ColumnStatisticsObj statsObjNew, - ColumnStatisticsObj statsObjOld) { - ColumnStatsMerger agg; - _Fields typeNew = statsObjNew.getStatsData().getSetField(); - _Fields typeOld = statsObjOld.getStatsData().getSetField(); - // make sure that they have the same type - typeNew = typeNew == typeOld ? typeNew : null; - switch (typeNew) { - case BOOLEAN_STATS: - agg = new BooleanColumnStatsMerger(); - break; - case LONG_STATS: { - agg = new LongColumnStatsMerger(); - break; - } - case DOUBLE_STATS: { - agg = new DoubleColumnStatsMerger(); - break; - } - case STRING_STATS: { - agg = new StringColumnStatsMerger(); - break; - } - case BINARY_STATS: - agg = new BinaryColumnStatsMerger(); - break; - case DECIMAL_STATS: { - agg = new DecimalColumnStatsMerger(); - break; - } - case DATE_STATS: { - agg = new DateColumnStatsMerger(); - break; - } - default: - throw new IllegalArgumentException("Unknown stats type " + statsObjNew.getStatsData().getSetField()); - } - return agg; - } - - public static ColumnStatisticsObj newColumnStaticsObj(String colName, String colType, _Fields type) { - ColumnStatisticsObj cso = new ColumnStatisticsObj(); - ColumnStatisticsData csd = new ColumnStatisticsData(); - cso.setColName(colName); - cso.setColType(colType); - switch (type) { - case BOOLEAN_STATS: - csd.setBooleanStats(new BooleanColumnStatsData()); - break; - - case LONG_STATS: - csd.setLongStats(new LongColumnStatsDataInspector()); - break; - - case DOUBLE_STATS: - csd.setDoubleStats(new DoubleColumnStatsDataInspector()); - break; - - case STRING_STATS: - csd.setStringStats(new StringColumnStatsDataInspector()); - break; - - case BINARY_STATS: - csd.setBinaryStats(new BinaryColumnStatsData()); - break; - - case DECIMAL_STATS: - csd.setDecimalStats(new DecimalColumnStatsDataInspector()); - break; - - case DATE_STATS: - csd.setDateStats(new DateColumnStatsDataInspector()); - break; - - default: - throw new IllegalArgumentException("Unknown stats type"); - } - - cso.setStatsData(csd); - return cso; - } - -} http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DateColumnStatsMerger.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DateColumnStatsMerger.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DateColumnStatsMerger.java deleted file mode 100644 index 5baebbb..0000000 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DateColumnStatsMerger.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hive.metastore.columnstats.merge; - -import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator; -import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; -import org.apache.hadoop.hive.metastore.api.Date; -import org.apache.hadoop.hive.metastore.columnstats.cache.DateColumnStatsDataInspector; - -public class DateColumnStatsMerger extends ColumnStatsMerger { - @Override - public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) { - DateColumnStatsDataInspector aggregateData = - (DateColumnStatsDataInspector) aggregateColStats.getStatsData().getDateStats(); - DateColumnStatsDataInspector newData = - (DateColumnStatsDataInspector) newColStats.getStatsData().getDateStats(); - Date lowValue = aggregateData.getLowValue().compareTo(newData.getLowValue()) < 0 ? aggregateData - .getLowValue() : newData.getLowValue(); - aggregateData.setLowValue(lowValue); - Date highValue = aggregateData.getHighValue().compareTo(newData.getHighValue()) >= 0 ? aggregateData - .getHighValue() : newData.getHighValue(); - aggregateData.setHighValue(highValue); - aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls()); - if (aggregateData.getNdvEstimator() == null || newData.getNdvEstimator() == null) { - aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs())); - } else { - NumDistinctValueEstimator oldEst = aggregateData.getNdvEstimator(); - NumDistinctValueEstimator newEst = newData.getNdvEstimator(); - long ndv = -1; - if (oldEst.canMerge(newEst)) { - oldEst.mergeEstimators(newEst); - ndv = oldEst.estimateNumDistinctValues(); - aggregateData.setNdvEstimator(oldEst); - } else { - ndv = Math.max(aggregateData.getNumDVs(), newData.getNumDVs()); - } - LOG.debug("Use bitvector to merge column " + aggregateColStats.getColName() + "'s ndvs of " - + aggregateData.getNumDVs() + " and " + newData.getNumDVs() + " to be " + ndv); - aggregateData.setNumDVs(ndv); - } - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DecimalColumnStatsMerger.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DecimalColumnStatsMerger.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DecimalColumnStatsMerger.java deleted file mode 100644 index 517ca72..0000000 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DecimalColumnStatsMerger.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hive.metastore.columnstats.merge; - -import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator; -import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; -import org.apache.hadoop.hive.metastore.api.Decimal; -import org.apache.hadoop.hive.metastore.columnstats.cache.DecimalColumnStatsDataInspector; - -public class DecimalColumnStatsMerger extends ColumnStatsMerger { - @Override - public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) { - DecimalColumnStatsDataInspector aggregateData = - (DecimalColumnStatsDataInspector) aggregateColStats.getStatsData().getDecimalStats(); - DecimalColumnStatsDataInspector newData = - (DecimalColumnStatsDataInspector) newColStats.getStatsData().getDecimalStats(); - - Decimal lowValue = getMin(aggregateData.getLowValue(), newData.getLowValue()); - aggregateData.setLowValue(lowValue); - - Decimal highValue = getMax(aggregateData.getHighValue(), newData.getHighValue()); - aggregateData.setHighValue(highValue); - - aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls()); - - if (aggregateData.getNdvEstimator() == null || newData.getNdvEstimator() == null) { - aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs())); - } else { - NumDistinctValueEstimator oldEst = aggregateData.getNdvEstimator(); - NumDistinctValueEstimator newEst = newData.getNdvEstimator(); - long ndv = -1; - if (oldEst.canMerge(newEst)) { - oldEst.mergeEstimators(newEst); - ndv = oldEst.estimateNumDistinctValues(); - aggregateData.setNdvEstimator(oldEst); - } else { - ndv = Math.max(aggregateData.getNumDVs(), newData.getNumDVs()); - } - LOG.debug("Use bitvector to merge column " + aggregateColStats.getColName() + "'s ndvs of " - + aggregateData.getNumDVs() + " and " + newData.getNumDVs() + " to be " + ndv); - aggregateData.setNumDVs(ndv); - } - } - - Decimal getMax(Decimal firstValue, Decimal secondValue) { - if (firstValue == null && secondValue == null) { - return null; - } - - if (firstValue != null && secondValue != null) { - return firstValue.compareTo(secondValue) > 0 ? firstValue : secondValue; - } - - return firstValue == null ? secondValue : firstValue; - } - - Decimal getMin(Decimal firstValue, Decimal secondValue) { - if (firstValue == null && secondValue == null) { - return null; - } - - if (firstValue != null && secondValue != null) { - return firstValue.compareTo(secondValue) > 0 ? secondValue : firstValue; - } - - return firstValue == null ? secondValue : firstValue; - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DoubleColumnStatsMerger.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DoubleColumnStatsMerger.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DoubleColumnStatsMerger.java deleted file mode 100644 index 6a95751..0000000 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/DoubleColumnStatsMerger.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hive.metastore.columnstats.merge; - -import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator; -import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; -import org.apache.hadoop.hive.metastore.columnstats.cache.DoubleColumnStatsDataInspector; - -public class DoubleColumnStatsMerger extends ColumnStatsMerger { - @Override - public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) { - DoubleColumnStatsDataInspector aggregateData = - (DoubleColumnStatsDataInspector) aggregateColStats.getStatsData().getDoubleStats(); - DoubleColumnStatsDataInspector newData = - (DoubleColumnStatsDataInspector) newColStats.getStatsData().getDoubleStats(); - aggregateData.setLowValue(Math.min(aggregateData.getLowValue(), newData.getLowValue())); - aggregateData.setHighValue(Math.max(aggregateData.getHighValue(), newData.getHighValue())); - aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls()); - if (aggregateData.getNdvEstimator() == null || newData.getNdvEstimator() == null) { - aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs())); - } else { - NumDistinctValueEstimator oldEst = aggregateData.getNdvEstimator(); - NumDistinctValueEstimator newEst = newData.getNdvEstimator(); - long ndv = -1; - if (oldEst.canMerge(newEst)) { - oldEst.mergeEstimators(newEst); - ndv = oldEst.estimateNumDistinctValues(); - aggregateData.setNdvEstimator(oldEst); - } else { - ndv = Math.max(aggregateData.getNumDVs(), newData.getNumDVs()); - } - LOG.debug("Use bitvector to merge column " + aggregateColStats.getColName() + "'s ndvs of " - + aggregateData.getNumDVs() + " and " + newData.getNumDVs() + " to be " + ndv); - aggregateData.setNumDVs(ndv); - } - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/LongColumnStatsMerger.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/LongColumnStatsMerger.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/LongColumnStatsMerger.java deleted file mode 100644 index ca1a912..0000000 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/LongColumnStatsMerger.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hive.metastore.columnstats.merge; - -import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator; -import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; -import org.apache.hadoop.hive.metastore.columnstats.cache.LongColumnStatsDataInspector; - -public class LongColumnStatsMerger extends ColumnStatsMerger { - @Override - public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) { - LongColumnStatsDataInspector aggregateData = - (LongColumnStatsDataInspector) aggregateColStats.getStatsData().getLongStats(); - LongColumnStatsDataInspector newData = - (LongColumnStatsDataInspector) newColStats.getStatsData().getLongStats(); - aggregateData.setLowValue(Math.min(aggregateData.getLowValue(), newData.getLowValue())); - aggregateData.setHighValue(Math.max(aggregateData.getHighValue(), newData.getHighValue())); - aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls()); - if (aggregateData.getNdvEstimator() == null || newData.getNdvEstimator() == null) { - aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs())); - } else { - NumDistinctValueEstimator oldEst = aggregateData.getNdvEstimator(); - NumDistinctValueEstimator newEst = newData.getNdvEstimator(); - long ndv = -1; - if (oldEst.canMerge(newEst)) { - oldEst.mergeEstimators(newEst); - ndv = oldEst.estimateNumDistinctValues(); - aggregateData.setNdvEstimator(oldEst); - } else { - ndv = Math.max(aggregateData.getNumDVs(), newData.getNumDVs()); - } - LOG.debug("Use bitvector to merge column " + aggregateColStats.getColName() + "'s ndvs of " - + aggregateData.getNumDVs() + " and " + newData.getNumDVs() + " to be " + ndv); - aggregateData.setNumDVs(ndv); - } - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/StringColumnStatsMerger.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/StringColumnStatsMerger.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/StringColumnStatsMerger.java deleted file mode 100644 index d6b4478..0000000 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/columnstats/merge/StringColumnStatsMerger.java +++ /dev/null @@ -1,54 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hive.metastore.columnstats.merge; - -import org.apache.hadoop.hive.common.ndv.NumDistinctValueEstimator; -import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; -import org.apache.hadoop.hive.metastore.columnstats.cache.StringColumnStatsDataInspector; - -public class StringColumnStatsMerger extends ColumnStatsMerger { - @Override - public void merge(ColumnStatisticsObj aggregateColStats, ColumnStatisticsObj newColStats) { - StringColumnStatsDataInspector aggregateData = - (StringColumnStatsDataInspector) aggregateColStats.getStatsData().getStringStats(); - StringColumnStatsDataInspector newData = - (StringColumnStatsDataInspector) newColStats.getStatsData().getStringStats(); - aggregateData.setMaxColLen(Math.max(aggregateData.getMaxColLen(), newData.getMaxColLen())); - aggregateData.setAvgColLen(Math.max(aggregateData.getAvgColLen(), newData.getAvgColLen())); - aggregateData.setNumNulls(aggregateData.getNumNulls() + newData.getNumNulls()); - if (aggregateData.getNdvEstimator() == null || newData.getNdvEstimator() == null) { - aggregateData.setNumDVs(Math.max(aggregateData.getNumDVs(), newData.getNumDVs())); - } else { - NumDistinctValueEstimator oldEst = aggregateData.getNdvEstimator(); - NumDistinctValueEstimator newEst = newData.getNdvEstimator(); - long ndv = -1; - if (oldEst.canMerge(newEst)) { - oldEst.mergeEstimators(newEst); - ndv = oldEst.estimateNumDistinctValues(); - aggregateData.setNdvEstimator(oldEst); - } else { - ndv = Math.max(aggregateData.getNumDVs(), newData.getNumDVs()); - } - LOG.debug("Use bitvector to merge column " + aggregateColStats.getColName() + "'s ndvs of " - + aggregateData.getNumDVs() + " and " + newData.getNumDVs() + " to be " + ndv); - aggregateData.setNumDVs(ndv); - } - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/ConfTemplatePrinter.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/ConfTemplatePrinter.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/ConfTemplatePrinter.java deleted file mode 100644 index f57e2ce..0000000 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/conf/ConfTemplatePrinter.java +++ /dev/null @@ -1,150 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.metastore.conf; - -import org.w3c.dom.Document; -import org.w3c.dom.Element; -import org.w3c.dom.Text; - -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.parsers.ParserConfigurationException; -import javax.xml.transform.OutputKeys; -import javax.xml.transform.Transformer; -import javax.xml.transform.TransformerException; -import javax.xml.transform.TransformerFactory; -import javax.xml.transform.dom.DOMSource; -import javax.xml.transform.stream.StreamResult; -import java.io.File; - -public class ConfTemplatePrinter { - - private Document generateTemplate() throws ParserConfigurationException { - DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); - DocumentBuilder docBuilder = dbf.newDocumentBuilder(); - Document doc = docBuilder.newDocument(); - doc.appendChild(doc.createProcessingInstruction( - "xml-stylesheet", "type=\"text/xsl\" href=\"configuration.xsl\"")); - - doc.appendChild(doc.createComment("\n" + - " Licensed to the Apache Software Foundation (ASF) under one or more\n" + - " contributor license agreements. See the NOTICE file distributed with\n" + - " this work for additional information regarding copyright ownership.\n" + - " The ASF licenses this file to You under the Apache License, Version 2.0\n" + - " (the \"License\"); you may not use this file except in compliance with\n" + - " the License. You may obtain a copy of the License at\n" + - "\n" + - " http://www.apache.org/licenses/LICENSE-2.0\n"; + - "\n" + - " Unless required by applicable law or agreed to in writing, software\n" + - " distributed under the License is distributed on an \"AS IS\" BASIS,\n" + - " WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.\n" + - " See the License for the specific language governing permissions and\n" + - " limitations under the License.\n")); - - Element root = doc.createElement("configuration"); - doc.appendChild(root); - - root.appendChild(doc.createComment( - " WARNING!!! This file is auto generated for documentation purposes ONLY! ")); - root.appendChild(doc.createComment( - " WARNING!!! Any changes you make to this file will be ignored by the metastore. ")); - root.appendChild(doc.createComment( - " WARNING!!! You must make your changes in metastore-site.xml instead. ")); - - root.appendChild(doc.createComment(" Metastore Execution Parameters ")); - - root.appendChild(doc.createComment("================================")); - root.appendChild(doc.createComment("All time unit values have a time unit abbreviation suffix")); - root.appendChild(doc.createComment("Any time value can take any of the units")); - root.appendChild(doc.createComment("d = day")); - root.appendChild(doc.createComment("h = hour")); - root.appendChild(doc.createComment("m = minute")); - root.appendChild(doc.createComment("s = second")); - root.appendChild(doc.createComment("ms = millisecond")); - root.appendChild(doc.createComment("us = microsecond")); - root.appendChild(doc.createComment("ns = nanosecond")); - root.appendChild(doc.createComment("================================")); - - for (MetastoreConf.ConfVars confVars : MetastoreConf.ConfVars.values()) { - Element property = appendElement(root, "property", null); - appendElement(property, "name", confVars.getVarname()); - appendElement(property, "value", confVars.getDefaultVal().toString()); - appendElement(property, "description", normalize(confVars.getDescription())); - // wish to add new line here. - } - return doc; - - } - - private String normalize(String description) { - int index = description.indexOf('\n'); - if (index < 0) { - return description; - } - int prev = 0; - StringBuilder builder = new StringBuilder(description.length() << 1); - for (;index > 0; index = description.indexOf('\n', prev = index + 1)) { - builder.append("\n ").append(description.substring(prev, index)); - } - if (prev < description.length()) { - builder.append("\n ").append(description.substring(prev)); - } - builder.append("\n "); - return builder.toString(); - } - - private void writeToFile(File template, Document document) throws TransformerException { - Transformer transformer = TransformerFactory.newInstance().newTransformer(); - transformer.setOutputProperty(OutputKeys.INDENT, "yes"); - transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount";, "2"); - DOMSource source = new DOMSource(document); - StreamResult result = new StreamResult(template); - transformer.transform(source, result); - } - - private Element appendElement(Element parent, String name, String text) { - Document document = parent.getOwnerDocument(); - Element child = document.createElement(name); - parent.appendChild(child); - if (text != null) { - Text textNode = document.createTextNode(text); - child.appendChild(textNode); - } - return child; - } - - private void print(String fileName) throws ParserConfigurationException, TransformerException { - Document doc = generateTemplate(); - File file = new File(fileName); - File dir = file.getParentFile(); - // Make certain the target directory exists. - dir.mkdirs(); - writeToFile(file, doc); - } - - public static void main(String[] args) throws Exception { - if (args.length != 1) { - String msg = "Usage: ConfTemplatePrinter filename"; - System.err.println(msg); - throw new RuntimeException(msg); - } - ConfTemplatePrinter printer = new ConfTemplatePrinter(); - printer.print(args[0]); - } -}
