http://git-wip-us.apache.org/repos/asf/hive/blob/93b9cdd6/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/StatsSetupConst.java ---------------------------------------------------------------------- diff --cc standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/StatsSetupConst.java index 0000000,a7ca05a..af9b0b1 mode 000000,100644..100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/StatsSetupConst.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/common/StatsSetupConst.java @@@ -1,0 -1,335 +1,339 @@@ + /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.hadoop.hive.common; + + import java.io.IOException; + import java.util.List; + import java.util.Map; + import java.util.TreeMap; + + import com.google.common.collect.ImmutableList; + import org.apache.hadoop.conf.Configuration; + import org.apache.hadoop.hive.metastore.conf.MetastoreConf; + import org.apache.hadoop.hive.metastore.conf.MetastoreConf.ConfVars; + import org.slf4j.Logger; + import org.slf4j.LoggerFactory; + + import com.fasterxml.jackson.annotation.JsonInclude; + import com.fasterxml.jackson.annotation.JsonProperty; + import com.fasterxml.jackson.core.JsonGenerator; + import com.fasterxml.jackson.core.JsonParser; + import com.fasterxml.jackson.core.JsonProcessingException; + import com.fasterxml.jackson.databind.DeserializationContext; + import com.fasterxml.jackson.databind.JsonDeserializer; + import com.fasterxml.jackson.databind.JsonSerializer; + import com.fasterxml.jackson.databind.ObjectMapper; + import com.fasterxml.jackson.databind.ObjectReader; + import com.fasterxml.jackson.databind.ObjectWriter; + import com.fasterxml.jackson.databind.SerializerProvider; + import com.fasterxml.jackson.databind.annotation.JsonDeserialize; + import com.fasterxml.jackson.databind.annotation.JsonSerialize; + + + /** + * A class that defines the constant strings used by the statistics implementation. + */ + + public class StatsSetupConst { + + protected static final Logger LOG = LoggerFactory.getLogger(StatsSetupConst.class.getName()); + + public enum StatDB { + fs { + @Override + public String getPublisher(Configuration conf) { + return "org.apache.hadoop.hive.ql.stats.fs.FSStatsPublisher"; + } + + @Override + public String getAggregator(Configuration conf) { + return "org.apache.hadoop.hive.ql.stats.fs.FSStatsAggregator"; + } + }, + custom { + @Override + public String getPublisher(Configuration conf) { + return MetastoreConf.getVar(conf, ConfVars.STATS_DEFAULT_PUBLISHER); } + @Override + public String getAggregator(Configuration conf) { + return MetastoreConf.getVar(conf, ConfVars.STATS_DEFAULT_AGGREGATOR); } + }; + public abstract String getPublisher(Configuration conf); + public abstract String getAggregator(Configuration conf); + } + + // statistics stored in metastore + /** + * The name of the statistic Num Files to be published or gathered. + */ + public static final String NUM_FILES = "numFiles"; + + /** + * The name of the statistic Num Partitions to be published or gathered. + */ + public static final String NUM_PARTITIONS = "numPartitions"; + + /** + * The name of the statistic Total Size to be published or gathered. + */ + public static final String TOTAL_SIZE = "totalSize"; + + /** + * The name of the statistic Row Count to be published or gathered. + */ + public static final String ROW_COUNT = "numRows"; + + public static final String RUN_TIME_ROW_COUNT = "runTimeNumRows"; + + /** + * The name of the statistic Raw Data Size to be published or gathered. + */ + public static final String RAW_DATA_SIZE = "rawDataSize"; + + /** + * The name of the statistic for Number of Erasure Coded Files - to be published or gathered. + */ + public static final String NUM_ERASURE_CODED_FILES = "numFilesErasureCoded"; + + /** + * Temp dir for writing stats from tasks. + */ + public static final String STATS_TMP_LOC = "hive.stats.tmp.loc"; + + public static final String STATS_FILE_PREFIX = "tmpstats-"; + /** + * List of all supported statistics + */ + public static final List<String> SUPPORTED_STATS = ImmutableList.of( + NUM_FILES, ROW_COUNT, TOTAL_SIZE, RAW_DATA_SIZE, NUM_ERASURE_CODED_FILES); + + /** + * List of all statistics that need to be collected during query execution. These are + * statistics that inherently require a scan of the data. + */ + public static final List<String> STATS_REQUIRE_COMPUTE = ImmutableList.of(ROW_COUNT, RAW_DATA_SIZE); + + /** + * List of statistics that can be collected quickly without requiring a scan of the data. + */ + public static final List<String> FAST_STATS = ImmutableList.of( + NUM_FILES, TOTAL_SIZE, NUM_ERASURE_CODED_FILES); + + // This string constant is used to indicate to AlterHandler that + // alterPartition/alterTable is happening via statsTask or via user. + public static final String STATS_GENERATED = "STATS_GENERATED"; + + public static final String TASK = "TASK"; + + public static final String USER = "USER"; + + // This string constant is used by AlterHandler to figure out that it should not attempt to + // update stats. It is set by any client-side task which wishes to signal that no stats + // update should take place, such as with replication. + public static final String DO_NOT_UPDATE_STATS = "DO_NOT_UPDATE_STATS"; + + //This string constant will be persisted in metastore to indicate whether corresponding + //table or partition's statistics and table or partition's column statistics are accurate or not. + public static final String COLUMN_STATS_ACCURATE = "COLUMN_STATS_ACCURATE"; + + public static final String COLUMN_STATS = "COLUMN_STATS"; + + public static final String BASIC_STATS = "BASIC_STATS"; + + public static final String CASCADE = "CASCADE"; + ++ // TODO: when alter calls are switched to req/resp models, replace these and the above with fields. ++ public static final String TXN_ID = "WRITER_TXN_ID"; ++ public static final String VALID_WRITE_IDS = "WRITER_WRITE_ID"; ++ + public static final String TRUE = "true"; + + public static final String FALSE = "false"; + + // The parameter keys for the table statistics. Those keys are excluded from 'show create table' command output. + public static final List<String> TABLE_PARAMS_STATS_KEYS = ImmutableList.of( + COLUMN_STATS_ACCURATE, NUM_FILES, TOTAL_SIZE, ROW_COUNT, RAW_DATA_SIZE, NUM_PARTITIONS, + NUM_ERASURE_CODED_FILES); + + private static class ColumnStatsAccurate { + private static ObjectReader objectReader; + private static ObjectWriter objectWriter; + + static { + ObjectMapper objectMapper = new ObjectMapper(); + objectReader = objectMapper.readerFor(ColumnStatsAccurate.class); + objectWriter = objectMapper.writerFor(ColumnStatsAccurate.class); + } + + static class BooleanSerializer extends JsonSerializer<Boolean> { + + @Override + public void serialize(Boolean value, JsonGenerator jsonGenerator, + SerializerProvider serializerProvider) throws IOException { + jsonGenerator.writeString(value.toString()); + } + } + + static class BooleanDeserializer extends JsonDeserializer<Boolean> { + + public Boolean deserialize(JsonParser jsonParser, + DeserializationContext deserializationContext) + throws IOException { + return Boolean.valueOf(jsonParser.getValueAsString()); + } + } + + @JsonInclude(JsonInclude.Include.NON_DEFAULT) + @JsonSerialize(using = BooleanSerializer.class) + @JsonDeserialize(using = BooleanDeserializer.class) + @JsonProperty(BASIC_STATS) + boolean basicStats; + + @JsonInclude(JsonInclude.Include.NON_EMPTY) + @JsonProperty(COLUMN_STATS) + @JsonSerialize(contentUsing = BooleanSerializer.class) + @JsonDeserialize(contentUsing = BooleanDeserializer.class) + TreeMap<String, Boolean> columnStats = new TreeMap<>(); + + } + + public static boolean areBasicStatsUptoDate(Map<String, String> params) { + if (params == null) { + return false; + } + ColumnStatsAccurate stats = parseStatsAcc(params.get(COLUMN_STATS_ACCURATE)); + return stats.basicStats; + } + + public static boolean areColumnStatsUptoDate(Map<String, String> params, String colName) { + if (params == null) { + return false; + } + ColumnStatsAccurate stats = parseStatsAcc(params.get(COLUMN_STATS_ACCURATE)); + return stats.columnStats.containsKey(colName); + } + + // It will only throw JSONException when stats.put(BASIC_STATS, TRUE) + // has duplicate key, which is not possible + // note that set basic stats false will wipe out column stats too. + public static void setBasicStatsState(Map<String, String> params, String setting) { + if (setting.equals(FALSE)) { + if (params!=null && params.containsKey(COLUMN_STATS_ACCURATE)) { + params.remove(COLUMN_STATS_ACCURATE); + } + return; + } + if (params == null) { + throw new RuntimeException("params are null...cant set columnstatstate!"); + } + ColumnStatsAccurate stats = parseStatsAcc(params.get(COLUMN_STATS_ACCURATE)); + stats.basicStats = true; + try { + params.put(COLUMN_STATS_ACCURATE, ColumnStatsAccurate.objectWriter.writeValueAsString(stats)); + } catch (JsonProcessingException e) { + throw new RuntimeException("can't serialize column stats", e); + } + } + + public static void setColumnStatsState(Map<String, String> params, List<String> colNames) { + if (params == null) { + throw new RuntimeException("params are null...cant set columnstatstate!"); + } + if (colNames == null) { + return; + } + ColumnStatsAccurate stats = parseStatsAcc(params.get(COLUMN_STATS_ACCURATE)); + + for (String colName : colNames) { + if (!stats.columnStats.containsKey(colName)) { + stats.columnStats.put(colName, true); + } + } + try { + params.put(COLUMN_STATS_ACCURATE, ColumnStatsAccurate.objectWriter.writeValueAsString(stats)); + } catch (JsonProcessingException e) { + LOG.trace(e.getMessage()); + } + } + + public static boolean canColumnStatsMerge(Map<String, String> params, String colName) { + if (params == null) { + return false; + } + ColumnStatsAccurate stats = parseStatsAcc(params.get(COLUMN_STATS_ACCURATE)); + return stats.columnStats.containsKey(colName); + } + + public static void clearColumnStatsState(Map<String, String> params) { + if (params == null) { + return; + } + + ColumnStatsAccurate stats = parseStatsAcc(params.get(COLUMN_STATS_ACCURATE)); + stats.columnStats.clear(); + + try { + params.put(COLUMN_STATS_ACCURATE, ColumnStatsAccurate.objectWriter.writeValueAsString(stats)); + } catch (JsonProcessingException e) { + LOG.trace(e.getMessage()); + } + } + + public static void removeColumnStatsState(Map<String, String> params, List<String> colNames) { + if (params == null) { + return; + } + try { + ColumnStatsAccurate stats = parseStatsAcc(params.get(COLUMN_STATS_ACCURATE)); + for (String string : colNames) { + stats.columnStats.remove(string); + } + params.put(COLUMN_STATS_ACCURATE, ColumnStatsAccurate.objectWriter.writeValueAsString(stats)); + } catch (JsonProcessingException e) { + LOG.trace(e.getMessage()); + } + } + + public static void setStatsStateForCreateTable(Map<String, String> params, + List<String> cols, String setting) { + if (TRUE.equals(setting)) { + for (String stat : StatsSetupConst.SUPPORTED_STATS) { + params.put(stat, "0"); + } + } + setBasicStatsState(params, setting); + if (TRUE.equals(setting)) { + setColumnStatsState(params, cols); + } + } + + private static ColumnStatsAccurate parseStatsAcc(String statsAcc) { + if (statsAcc == null) { + return new ColumnStatsAccurate(); + } + try { + return ColumnStatsAccurate.objectReader.readValue(statsAcc); + } catch (Exception e) { + ColumnStatsAccurate ret = new ColumnStatsAccurate(); + if (TRUE.equalsIgnoreCase(statsAcc)) { + ret.basicStats = true; + } + return ret; + } + } + }
http://git-wip-us.apache.org/repos/asf/hive/blob/93b9cdd6/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/AlterHandler.java ---------------------------------------------------------------------- diff --cc standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/AlterHandler.java index 0000000,050dca9..e7cf07f mode 000000,100644..100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/AlterHandler.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/AlterHandler.java @@@ -1,0 -1,202 +1,203 @@@ + /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.hadoop.hive.metastore; + + import java.util.List; + + import org.apache.hadoop.conf.Configurable; + import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; + import org.apache.hadoop.hive.metastore.api.EnvironmentContext; + import org.apache.hadoop.hive.metastore.api.InvalidObjectException; + import org.apache.hadoop.hive.metastore.api.InvalidOperationException; + import org.apache.hadoop.hive.metastore.api.MetaException; + import org.apache.hadoop.hive.metastore.api.Partition; + import org.apache.hadoop.hive.metastore.api.Table; + + /** + * Interface for Alter Table and Alter Partition code + */ + public interface AlterHandler extends Configurable { + + /** + * @deprecated As of release 2.2.0. Replaced by {@link #alterTable(RawStore, Warehouse, String, + * String, String, Table, EnvironmentContext, IHMSHandler)} + * + * handles alter table, the changes could be cascaded to partitions if applicable + * + * @param msdb + * object to get metadata + * @param wh + * Hive Warehouse where table data is stored + * @param catName + * catalog of the table being altered + * @param dbname + * database of the table being altered + * @param name + * original name of the table being altered. same as + * <i>newTable.tableName</i> if alter op is not a rename. + * @param newTable + * new table object + * @throws InvalidOperationException + * thrown if the newTable object is invalid + * @throws MetaException + * thrown if there is any other error + */ + @Deprecated + default void alterTable(RawStore msdb, Warehouse wh, String catName, String dbname, + String name, Table newTable, EnvironmentContext envContext) + throws InvalidOperationException, MetaException { + alterTable(msdb, wh, catName, dbname, name, newTable, envContext, null); + } + + /** + * handles alter table, the changes could be cascaded to partitions if applicable + * + * @param msdb + * object to get metadata + * @param wh + * Hive Warehouse where table data is stored + * @param catName catalog of the table being altered + * @param dbname + * database of the table being altered + * @param name + * original name of the table being altered. same as + * <i>newTable.tableName</i> if alter op is not a rename. + * @param newTable + * new table object + * @param handler + * HMSHandle object (required to log event notification) + * @throws InvalidOperationException + * thrown if the newTable object is invalid + * @throws MetaException + * thrown if there is any other error + */ + void alterTable(RawStore msdb, Warehouse wh, String catName, String dbname, + String name, Table newTable, EnvironmentContext envContext, + IHMSHandler handler) throws InvalidOperationException, MetaException; + + /** + * @deprecated As of release 2.2.0. Replaced by {@link #alterPartition(RawStore, Warehouse, String, + * String, List, Partition, EnvironmentContext, IHMSHandler)} + * + * handles alter partition + * + * @param msdb + * object to get metadata + * @param wh + * @param dbname + * database of the partition being altered + * @param name + * table of the partition being altered + * @param part_vals + * original values of the partition being altered + * @param new_part + * new partition object + * @return the altered partition + * @throws InvalidOperationException + * @throws InvalidObjectException + * @throws AlreadyExistsException + * @throws MetaException + */ + @Deprecated + Partition alterPartition(final RawStore msdb, Warehouse wh, final String dbname, + final String name, final List<String> part_vals, final Partition new_part, + EnvironmentContext environmentContext) + throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException; + + /** + * handles alter partition + * + * @param msdb + * object to get metadata + * @param wh physical warehouse class + * @param catName catalog name + * @param dbname + * database of the partition being altered + * @param name + * table of the partition being altered + * @param part_vals + * original values of the partition being altered + * @param new_part + * new partition object + * @param handler + * HMSHandle object (required to log event notification) + * @return the altered partition + * @throws InvalidOperationException + * @throws InvalidObjectException + * @throws AlreadyExistsException + * @throws MetaException + */ + Partition alterPartition(final RawStore msdb, Warehouse wh, final String catName, + final String dbname, final String name, final List<String> part_vals, + final Partition new_part, EnvironmentContext environmentContext, + IHMSHandler handler) + throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException; + + /** + * @deprecated As of release 3.0.0. Replaced by {@link #alterPartitions(RawStore, Warehouse, String, + * String, String, List, EnvironmentContext, IHMSHandler)} + * + * handles alter partitions + * + * @param msdb + * object to get metadata + * @param wh + * @param dbname + * database of the partition being altered + * @param name + * table of the partition being altered + * @param new_parts + * new partition list + * @return the altered partition list + * @throws InvalidOperationException + * @throws InvalidObjectException + * @throws AlreadyExistsException + * @throws MetaException + */ + @Deprecated + List<Partition> alterPartitions(final RawStore msdb, Warehouse wh, + final String dbname, final String name, final List<Partition> new_parts, + EnvironmentContext environmentContext) + throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException; + + /** + * handles alter partitions + * + * @param msdb + * object to get metadata + * @param wh + * @param dbname + * database of the partition being altered + * @param name + * table of the partition being altered + * @param new_parts + * new partition list + * @param handler + * HMSHandle object (required to log event notification) + * @return the altered partition list + * @throws InvalidOperationException + * @throws InvalidObjectException + * @throws AlreadyExistsException + * @throws MetaException + */ + List<Partition> alterPartitions(final RawStore msdb, Warehouse wh, final String catName, + final String dbname, final String name, final List<Partition> new_parts, - EnvironmentContext environmentContext,IHMSHandler handler) ++ EnvironmentContext environmentContext, long txnId, String writeIdList, long writeId, ++ IHMSHandler handler) + throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException; + } http://git-wip-us.apache.org/repos/asf/hive/blob/93b9cdd6/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java ---------------------------------------------------------------------- diff --cc standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java index 0000000,93ac74c..8b2a6ba mode 000000,100644..100644 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java +++ b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/HiveAlterHandler.java @@@ -1,0 -1,948 +1,974 @@@ + /* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + package org.apache.hadoop.hive.metastore; + + import com.google.common.annotations.VisibleForTesting; + import com.google.common.collect.Lists; + + import org.apache.commons.lang.StringUtils; + import org.apache.hadoop.hive.common.TableName; + import org.apache.hadoop.hive.metastore.conf.MetastoreConf; + import org.apache.hadoop.hive.metastore.events.AlterPartitionEvent; + import org.apache.hadoop.hive.metastore.events.AlterTableEvent; + import org.apache.hadoop.hive.metastore.messaging.EventMessage; + import org.apache.hadoop.hive.metastore.utils.FileUtils; + import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; + import org.slf4j.Logger; + import org.slf4j.LoggerFactory; + import org.apache.hadoop.conf.Configuration; + import org.apache.hadoop.fs.FileSystem; + import org.apache.hadoop.fs.Path; + import org.apache.hadoop.hive.common.StatsSetupConst; + import org.apache.hadoop.hive.metastore.api.AlreadyExistsException; + import org.apache.hadoop.hive.metastore.api.ColumnStatistics; + import org.apache.hadoop.hive.metastore.api.ColumnStatisticsDesc; + import org.apache.hadoop.hive.metastore.api.ColumnStatisticsObj; + import org.apache.hadoop.hive.metastore.api.Database; + import org.apache.hadoop.hive.metastore.api.EnvironmentContext; + import org.apache.hadoop.hive.metastore.api.FieldSchema; + import org.apache.hadoop.hive.metastore.api.InvalidInputException; + import org.apache.hadoop.hive.metastore.api.InvalidObjectException; + import org.apache.hadoop.hive.metastore.api.InvalidOperationException; + import org.apache.hadoop.hive.metastore.api.MetaException; + import org.apache.hadoop.hive.metastore.api.NoSuchObjectException; + import org.apache.hadoop.hive.metastore.api.Partition; + import org.apache.hadoop.hive.metastore.api.Table; + import org.apache.hadoop.hive.metastore.api.hive_metastoreConstants; + + import java.io.IOException; + import java.net.URI; + import java.util.ArrayList; + import java.util.Collections; + import java.util.HashMap; + import java.util.Iterator; + import java.util.List; + import java.util.Map; + import java.util.Map.Entry; + + import static org.apache.hadoop.hive.metastore.Warehouse.DEFAULT_CATALOG_NAME; + import static org.apache.hadoop.hive.metastore.utils.MetaStoreUtils.getDefaultCatalog; + import static org.apache.hadoop.hive.metastore.utils.StringUtils.normalizeIdentifier; + + /** + * Hive specific implementation of alter + */ + public class HiveAlterHandler implements AlterHandler { + + protected Configuration conf; + private static final Logger LOG = LoggerFactory.getLogger(HiveAlterHandler.class + .getName()); + + // hiveConf, getConf and setConf are in this class because AlterHandler extends Configurable. + // Always use the configuration from HMS Handler. Making AlterHandler not extend Configurable + // is not in the scope of the fix for HIVE-17942. + @Override + public Configuration getConf() { + return conf; + } + + @Override + @SuppressWarnings("nls") + public void setConf(Configuration conf) { + this.conf = conf; + } + + @Override + public void alterTable(RawStore msdb, Warehouse wh, String catName, String dbname, + String name, Table newt, EnvironmentContext environmentContext, + IHMSHandler handler) throws InvalidOperationException, MetaException { + catName = normalizeIdentifier(catName); + name = name.toLowerCase(); + dbname = dbname.toLowerCase(); + + final boolean cascade = environmentContext != null + && environmentContext.isSetProperties() + && StatsSetupConst.TRUE.equals(environmentContext.getProperties().get( + StatsSetupConst.CASCADE)); + if (newt == null) { + throw new InvalidOperationException("New table is null"); + } + + String newTblName = newt.getTableName().toLowerCase(); + String newDbName = newt.getDbName().toLowerCase(); + + if (!MetaStoreUtils.validateName(newTblName, handler.getConf())) { + throw new InvalidOperationException(newTblName + " is not a valid object name"); + } + String validate = MetaStoreUtils.validateTblColumns(newt.getSd().getCols()); + if (validate != null) { + throw new InvalidOperationException("Invalid column " + validate); + } + + Path srcPath = null; + FileSystem srcFs; + Path destPath = null; + FileSystem destFs = null; + + boolean success = false; + boolean dataWasMoved = false; + boolean isPartitionedTable = false; + + Table oldt = null; + + List<TransactionalMetaStoreEventListener> transactionalListeners = handler.getTransactionalListeners(); + List<MetaStoreEventListener> listeners = handler.getListeners(); + Map<String, String> txnAlterTableEventResponses = Collections.emptyMap(); + + try { + boolean rename = false; + List<Partition> parts; + + // Switching tables between catalogs is not allowed. + if (!catName.equalsIgnoreCase(newt.getCatName())) { + throw new InvalidOperationException("Tables cannot be moved between catalogs, old catalog" + + catName + ", new catalog " + newt.getCatName()); + } + + // check if table with the new name already exists + if (!newTblName.equals(name) || !newDbName.equals(dbname)) { - if (msdb.getTable(catName, newDbName, newTblName) != null) { ++ if (msdb.getTable(catName, newDbName, newTblName, -1, null) != null) { + throw new InvalidOperationException("new table " + newDbName + + "." + newTblName + " already exists"); + } + rename = true; + } + + msdb.openTransaction(); + // get old table - oldt = msdb.getTable(catName, dbname, name); ++ // Note: we don't verify stats here; it's done below in alterTableUpdateTableColumnStats. ++ oldt = msdb.getTable(catName, dbname, name, -1, null); + if (oldt == null) { + throw new InvalidOperationException("table " + + TableName.getQualified(catName, dbname, name) + " doesn't exist"); + } + + if (oldt.getPartitionKeysSize() != 0) { + isPartitionedTable = true; + } + + // Views derive the column type from the base table definition. So the view definition + // can be altered to change the column types. The column type compatibility checks should + // be done only for non-views. + if (MetastoreConf.getBoolVar(handler.getConf(), + MetastoreConf.ConfVars.DISALLOW_INCOMPATIBLE_COL_TYPE_CHANGES) && + !oldt.getTableType().equals(TableType.VIRTUAL_VIEW.toString())) { + // Throws InvalidOperationException if the new column types are not + // compatible with the current column types. + checkColTypeChangeCompatible(oldt.getSd().getCols(), newt.getSd().getCols()); + } + + //check that partition keys have not changed, except for virtual views + //however, allow the partition comments to change + boolean partKeysPartiallyEqual = checkPartialPartKeysEqual(oldt.getPartitionKeys(), + newt.getPartitionKeys()); + + if(!oldt.getTableType().equals(TableType.VIRTUAL_VIEW.toString())){ + if (!partKeysPartiallyEqual) { + throw new InvalidOperationException("partition keys can not be changed."); + } + } + + // rename needs change the data location and move the data to the new location corresponding + // to the new name if: + // 1) the table is not a virtual view, and + // 2) the table is not an external table, and + // 3) the user didn't change the default location (or new location is empty), and + // 4) the table was not initially created with a specified location + if (rename + && !oldt.getTableType().equals(TableType.VIRTUAL_VIEW.toString()) + && (oldt.getSd().getLocation().compareTo(newt.getSd().getLocation()) == 0 + || StringUtils.isEmpty(newt.getSd().getLocation())) + && !MetaStoreUtils.isExternalTable(oldt)) { + Database olddb = msdb.getDatabase(catName, dbname); + // if a table was created in a user specified location using the DDL like + // create table tbl ... location ...., it should be treated like an external table + // in the table rename, its data location should not be changed. We can check + // if the table directory was created directly under its database directory to tell + // if it is such a table + srcPath = new Path(oldt.getSd().getLocation()); + String oldtRelativePath = (new Path(olddb.getLocationUri()).toUri()) + .relativize(srcPath.toUri()).toString(); + boolean tableInSpecifiedLoc = !oldtRelativePath.equalsIgnoreCase(name) + && !oldtRelativePath.equalsIgnoreCase(name + Path.SEPARATOR); + + if (!tableInSpecifiedLoc) { + srcFs = wh.getFs(srcPath); + + // get new location + Database db = msdb.getDatabase(catName, newDbName); + Path databasePath = constructRenamedPath(wh.getDatabasePath(db), srcPath); + destPath = new Path(databasePath, newTblName); + destFs = wh.getFs(destPath); + + newt.getSd().setLocation(destPath.toString()); + + // check that destination does not exist otherwise we will be + // overwriting data + // check that src and dest are on the same file system + if (!FileUtils.equalsFileSystem(srcFs, destFs)) { + throw new InvalidOperationException("table new location " + destPath + + " is on a different file system than the old location " + + srcPath + ". This operation is not supported"); + } + + try { + if (destFs.exists(destPath)) { + throw new InvalidOperationException("New location for this table " + + TableName.getQualified(catName, newDbName, newTblName) + + " already exists : " + destPath); + } + // check that src exists and also checks permissions necessary, rename src to dest + if (srcFs.exists(srcPath) && wh.renameDir(srcPath, destPath, + ReplChangeManager.isSourceOfReplication(olddb))) { + dataWasMoved = true; + } + } catch (IOException | MetaException e) { + LOG.error("Alter Table operation for " + dbname + "." + name + " failed.", e); + throw new InvalidOperationException("Alter Table operation for " + dbname + "." + name + + " failed to move data due to: '" + getSimpleMessage(e) + + "' See hive log file for details."); + } + + if (!HiveMetaStore.isRenameAllowed(olddb, db)) { + LOG.error("Alter Table operation for " + TableName.getQualified(catName, dbname, name) + + "to new table = " + TableName.getQualified(catName, newDbName, newTblName) + " failed "); + throw new MetaException("Alter table not allowed for table " + + TableName.getQualified(catName, dbname, name) + + "to new table = " + TableName.getQualified(catName, newDbName, newTblName)); + } + } + + if (isPartitionedTable) { + String oldTblLocPath = srcPath.toUri().getPath(); + String newTblLocPath = dataWasMoved ? destPath.toUri().getPath() : null; + + // also the location field in partition + parts = msdb.getPartitions(catName, dbname, name, -1); + Map<Partition, ColumnStatistics> columnStatsNeedUpdated = new HashMap<>(); + for (Partition part : parts) { + String oldPartLoc = part.getSd().getLocation(); + if (dataWasMoved && oldPartLoc.contains(oldTblLocPath)) { + URI oldUri = new Path(oldPartLoc).toUri(); + String newPath = oldUri.getPath().replace(oldTblLocPath, newTblLocPath); + Path newPartLocPath = new Path(oldUri.getScheme(), oldUri.getAuthority(), newPath); + part.getSd().setLocation(newPartLocPath.toString()); + } + part.setDbName(newDbName); + part.setTableName(newTblName); + ColumnStatistics colStats = updateOrGetPartitionColumnStats(msdb, catName, dbname, name, + part.getValues(), part.getSd().getCols(), oldt, part, null); + if (colStats != null) { + columnStatsNeedUpdated.put(part, colStats); + } + } - msdb.alterTable(catName, dbname, name, newt); ++ // Do not verify stats parameters on a partitioned table. ++ msdb.alterTable(catName, dbname, name, newt, -1, null); + // alterPartition is only for changing the partition location in the table rename + if (dataWasMoved) { + + int partsToProcess = parts.size(); + int partitionBatchSize = MetastoreConf.getIntVar(handler.getConf(), + MetastoreConf.ConfVars.BATCH_RETRIEVE_MAX); + int batchStart = 0; + while (partsToProcess > 0) { + int batchEnd = Math.min(batchStart + partitionBatchSize, parts.size()); + List<Partition> partBatch = parts.subList(batchStart, batchEnd); + int partBatchSize = partBatch.size(); + partsToProcess -= partBatchSize; + batchStart += partBatchSize; + List<List<String>> partValues = new ArrayList<>(partBatchSize); + for (Partition part : partBatch) { + partValues.add(part.getValues()); + } - msdb.alterPartitions(catName, newDbName, newTblName, partValues, partBatch); ++ msdb.alterPartitions(catName, newDbName, newTblName, partValues, ++ partBatch, -1, -1, null); + } + } + + for (Entry<Partition, ColumnStatistics> partColStats : columnStatsNeedUpdated.entrySet()) { + ColumnStatistics newPartColStats = partColStats.getValue(); + newPartColStats.getStatsDesc().setDbName(newDbName); + newPartColStats.getStatsDesc().setTableName(newTblName); + msdb.updatePartitionColumnStatistics(newPartColStats, partColStats.getKey().getValues()); + } + } else { - alterTableUpdateTableColumnStats(msdb, oldt, newt); ++ alterTableUpdateTableColumnStats(msdb, oldt, newt, environmentContext); + } + } else { + // operations other than table rename + + if (MetaStoreUtils.requireCalStats(null, null, newt, environmentContext) && + !isPartitionedTable) { + Database db = msdb.getDatabase(catName, newDbName); + // Update table stats. For partitioned table, we update stats in alterPartition() + MetaStoreUtils.updateTableStatsSlow(db, newt, wh, false, true, environmentContext); + } + + if (isPartitionedTable) { + //Currently only column related changes can be cascaded in alter table + if(!MetaStoreUtils.areSameColumns(oldt.getSd().getCols(), newt.getSd().getCols())) { + parts = msdb.getPartitions(catName, dbname, name, -1); + for (Partition part : parts) { + Partition oldPart = new Partition(part); + List<FieldSchema> oldCols = part.getSd().getCols(); + part.getSd().setCols(newt.getSd().getCols()); + ColumnStatistics colStats = updateOrGetPartitionColumnStats(msdb, catName, dbname, name, + part.getValues(), oldCols, oldt, part, null); + assert(colStats == null); ++ // Note: we don't do txn stats validation here; this can only delete stats? + if (cascade) { - msdb.alterPartition(catName, dbname, name, part.getValues(), part); ++ msdb.alterPartition(catName, dbname, name, part.getValues(), part, -1, null); + } else { + // update changed properties (stats) + oldPart.setParameters(part.getParameters()); - msdb.alterPartition(catName, dbname, name, part.getValues(), oldPart); ++ msdb.alterPartition(catName, dbname, name, part.getValues(), oldPart, -1, null); + } + } - msdb.alterTable(catName, dbname, name, newt); ++ // Don't validate table-level stats for a partitoned table. ++ msdb.alterTable(catName, dbname, name, newt, -1, null); + } else { + LOG.warn("Alter table not cascaded to partitions."); - alterTableUpdateTableColumnStats(msdb, oldt, newt); ++ alterTableUpdateTableColumnStats(msdb, oldt, newt, environmentContext); + } + } else { - alterTableUpdateTableColumnStats(msdb, oldt, newt); ++ alterTableUpdateTableColumnStats(msdb, oldt, newt, environmentContext); + } + } + + if (transactionalListeners != null && !transactionalListeners.isEmpty()) { + txnAlterTableEventResponses = MetaStoreListenerNotifier.notifyEvent(transactionalListeners, + EventMessage.EventType.ALTER_TABLE, + new AlterTableEvent(oldt, newt, false, true, handler), + environmentContext); + } + // commit the changes + success = msdb.commitTransaction(); + } catch (InvalidObjectException e) { + LOG.debug("Failed to get object from Metastore ", e); + throw new InvalidOperationException( + "Unable to change partition or table." + + " Check metastore logs for detailed stack." + e.getMessage()); + } catch (InvalidInputException e) { + LOG.debug("Accessing Metastore failed due to invalid input ", e); + throw new InvalidOperationException( + "Unable to change partition or table." + + " Check metastore logs for detailed stack." + e.getMessage()); + } catch (NoSuchObjectException e) { + LOG.debug("Object not found in metastore ", e); + throw new InvalidOperationException( + "Unable to change partition or table. Database " + dbname + " does not exist" + + " Check metastore logs for detailed stack." + e.getMessage()); + } finally { + if (!success) { + LOG.error("Failed to alter table " + TableName.getQualified(catName, dbname, name)); + msdb.rollbackTransaction(); + if (dataWasMoved) { + try { + if (destFs.exists(destPath)) { + if (!destFs.rename(destPath, srcPath)) { + LOG.error("Failed to restore data from " + destPath + " to " + srcPath + + " in alter table failure. Manual restore is needed."); + } + } + } catch (IOException e) { + LOG.error("Failed to restore data from " + destPath + " to " + srcPath + + " in alter table failure. Manual restore is needed."); + } + } + } + } + + if (!listeners.isEmpty()) { + // I don't think event notifications in case of failures are necessary, but other HMS operations + // make this call whether the event failed or succeeded. To make this behavior consistent, + // this call is made for failed events also. + MetaStoreListenerNotifier.notifyEvent(listeners, EventMessage.EventType.ALTER_TABLE, + new AlterTableEvent(oldt, newt, false, success, handler), + environmentContext, txnAlterTableEventResponses, msdb); + } + } + + /** + * MetaException that encapsulates error message from RemoteException from hadoop RPC which wrap + * the stack trace into e.getMessage() which makes logs/stack traces confusing. + * @param ex + * @return + */ + String getSimpleMessage(Exception ex) { + if(ex instanceof MetaException) { + String msg = ex.getMessage(); + if(msg == null || !msg.contains("\n")) { + return msg; + } + return msg.substring(0, msg.indexOf('\n')); + } + return ex.getMessage(); + } + + @Override + public Partition alterPartition(final RawStore msdb, Warehouse wh, final String dbname, + final String name, final List<String> part_vals, final Partition new_part, + EnvironmentContext environmentContext) + throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException { + return alterPartition(msdb, wh, DEFAULT_CATALOG_NAME, dbname, name, part_vals, new_part, + environmentContext, null); + } + + @Override + public Partition alterPartition(final RawStore msdb, Warehouse wh, final String catName, + final String dbname, final String name, + final List<String> part_vals, final Partition new_part, + EnvironmentContext environmentContext, IHMSHandler handler) + throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException { + boolean success = false; + Partition oldPart; + List<TransactionalMetaStoreEventListener> transactionalListeners = null; + if (handler != null) { + transactionalListeners = handler.getTransactionalListeners(); + } + + // Set DDL time to now if not specified + if (new_part.getParameters() == null || + new_part.getParameters().get(hive_metastoreConstants.DDL_TIME) == null || + Integer.parseInt(new_part.getParameters().get(hive_metastoreConstants.DDL_TIME)) == 0) { + new_part.putToParameters(hive_metastoreConstants.DDL_TIME, Long.toString(System + .currentTimeMillis() / 1000)); + } ++ long txnId = -1; ++ String validWriteIds = null; ++ if (environmentContext != null && environmentContext.isSetProperties() ++ && environmentContext.getProperties().containsKey(StatsSetupConst.VALID_WRITE_IDS)) { ++ txnId = Long.parseLong(environmentContext.getProperties().get(StatsSetupConst.TXN_ID)); ++ validWriteIds = environmentContext.getProperties().get(StatsSetupConst.VALID_WRITE_IDS); ++ } + + //alter partition + if (part_vals == null || part_vals.size() == 0) { + try { + msdb.openTransaction(); + - Table tbl = msdb.getTable(catName, dbname, name); ++ Table tbl = msdb.getTable(catName, dbname, name, -1, null); + if (tbl == null) { + throw new InvalidObjectException( + "Unable to alter partition because table or database does not exist."); + } + oldPart = msdb.getPartition(catName, dbname, name, new_part.getValues()); + if (MetaStoreUtils.requireCalStats(oldPart, new_part, tbl, environmentContext)) { + // if stats are same, no need to update + if (MetaStoreUtils.isFastStatsSame(oldPart, new_part)) { + MetaStoreUtils.updateBasicState(environmentContext, new_part.getParameters()); + } else { + MetaStoreUtils.updatePartitionStatsFast( + new_part, tbl, wh, false, true, environmentContext, false); + } + } + + // PartitionView does not have SD. We do not need update its column stats + if (oldPart.getSd() != null) { + updateOrGetPartitionColumnStats(msdb, catName, dbname, name, new_part.getValues(), + oldPart.getSd().getCols(), tbl, new_part, null); + } - msdb.alterPartition(catName, dbname, name, new_part.getValues(), new_part); ++ msdb.alterPartition( ++ catName, dbname, name, new_part.getValues(), new_part, txnId, validWriteIds); + if (transactionalListeners != null && !transactionalListeners.isEmpty()) { + MetaStoreListenerNotifier.notifyEvent(transactionalListeners, + EventMessage.EventType.ALTER_PARTITION, + new AlterPartitionEvent(oldPart, new_part, tbl, false, true, handler), + environmentContext); + + + } + success = msdb.commitTransaction(); + } catch (InvalidObjectException e) { + LOG.warn("Alter failed", e); + throw new InvalidOperationException("alter is not possible: " + e.getMessage()); + } catch (NoSuchObjectException e) { + //old partition does not exist + throw new InvalidOperationException("alter is not possible: " + e.getMessage()); + } finally { + if(!success) { + msdb.rollbackTransaction(); + } + } + return oldPart; + } + + //rename partition + String oldPartLoc; + String newPartLoc; + Path srcPath = null; + Path destPath = null; + FileSystem srcFs; + FileSystem destFs = null; + boolean dataWasMoved = false; + Database db; + try { + msdb.openTransaction(); - Table tbl = msdb.getTable(DEFAULT_CATALOG_NAME, dbname, name); ++ Table tbl = msdb.getTable(DEFAULT_CATALOG_NAME, dbname, name, -1, null); + if (tbl == null) { + throw new InvalidObjectException( + "Unable to alter partition because table or database does not exist."); + } + try { + oldPart = msdb.getPartition(catName, dbname, name, part_vals); + } catch (NoSuchObjectException e) { + // this means there is no existing partition + throw new InvalidObjectException( + "Unable to rename partition because old partition does not exist"); + } + + Partition check_part; + try { + check_part = msdb.getPartition(catName, dbname, name, new_part.getValues()); + } catch(NoSuchObjectException e) { + // this means there is no existing partition + check_part = null; + } + + if (check_part != null) { + throw new AlreadyExistsException("Partition already exists:" + dbname + "." + name + "." + + new_part.getValues()); + } + + // when renaming a partition, we should update + // 1) partition SD Location + // 2) partition column stats if there are any because of part_name field in HMS table PART_COL_STATS + // 3) rename the partition directory if it is not an external table + if (!tbl.getTableType().equals(TableType.EXTERNAL_TABLE.toString())) { + try { + db = msdb.getDatabase(catName, dbname); + + // if tbl location is available use it + // else derive the tbl location from database location + destPath = wh.getPartitionPath(db, tbl, new_part.getValues()); + destPath = constructRenamedPath(destPath, new Path(new_part.getSd().getLocation())); + } catch (NoSuchObjectException e) { + LOG.debug("Didn't find object in metastore ", e); + throw new InvalidOperationException( + "Unable to change partition or table. Database " + dbname + " does not exist" + + " Check metastore logs for detailed stack." + e.getMessage()); + } + + if (destPath != null) { + newPartLoc = destPath.toString(); + oldPartLoc = oldPart.getSd().getLocation(); + LOG.info("srcPath:" + oldPartLoc); + LOG.info("descPath:" + newPartLoc); + srcPath = new Path(oldPartLoc); + srcFs = wh.getFs(srcPath); + destFs = wh.getFs(destPath); + // check that src and dest are on the same file system + if (!FileUtils.equalsFileSystem(srcFs, destFs)) { + throw new InvalidOperationException("New table location " + destPath + + " is on a different file system than the old location " + + srcPath + ". This operation is not supported."); + } + + try { + if (srcFs.exists(srcPath)) { + if (newPartLoc.compareTo(oldPartLoc) != 0 && destFs.exists(destPath)) { + throw new InvalidOperationException("New location for this table " + + tbl.getDbName() + "." + tbl.getTableName() + + " already exists : " + destPath); + } + //if destPath's parent path doesn't exist, we should mkdir it + Path destParentPath = destPath.getParent(); + if (!wh.mkdirs(destParentPath)) { + throw new MetaException("Unable to create path " + destParentPath); + } + + //rename the data directory + wh.renameDir(srcPath, destPath, ReplChangeManager.isSourceOfReplication(db)); + LOG.info("Partition directory rename from " + srcPath + " to " + destPath + " done."); + dataWasMoved = true; + } + } catch (IOException e) { + LOG.error("Cannot rename partition directory from " + srcPath + " to " + destPath, e); + throw new InvalidOperationException("Unable to access src or dest location for partition " + + tbl.getDbName() + "." + tbl.getTableName() + " " + new_part.getValues()); + } catch (MetaException me) { + LOG.error("Cannot rename partition directory from " + srcPath + " to " + destPath, me); + throw me; + } + new_part.getSd().setLocation(newPartLoc); + } + } else { + new_part.getSd().setLocation(oldPart.getSd().getLocation()); + } + + if (MetaStoreUtils.requireCalStats(oldPart, new_part, tbl, environmentContext)) { + MetaStoreUtils.updatePartitionStatsFast( + new_part, tbl, wh, false, true, environmentContext, false); + } + + String newPartName = Warehouse.makePartName(tbl.getPartitionKeys(), new_part.getValues()); + ColumnStatistics cs = updateOrGetPartitionColumnStats(msdb, catName, dbname, name, oldPart.getValues(), + oldPart.getSd().getCols(), tbl, new_part, null); - msdb.alterPartition(catName, dbname, name, part_vals, new_part); ++ msdb.alterPartition(catName, dbname, name, part_vals, new_part, txnId, validWriteIds); + if (cs != null) { + cs.getStatsDesc().setPartName(newPartName); + try { + msdb.updatePartitionColumnStatistics(cs, new_part.getValues()); + } catch (InvalidInputException iie) { + throw new InvalidOperationException("Unable to update partition stats in table rename." + iie); + } catch (NoSuchObjectException nsoe) { + // It is ok, ignore + } + } + + if (transactionalListeners != null && !transactionalListeners.isEmpty()) { + MetaStoreListenerNotifier.notifyEvent(transactionalListeners, + EventMessage.EventType.ALTER_PARTITION, + new AlterPartitionEvent(oldPart, new_part, tbl, false, true, handler), + environmentContext); + } + + success = msdb.commitTransaction(); + } finally { + if (!success) { + LOG.error("Failed to rename a partition. Rollback transaction"); + msdb.rollbackTransaction(); + if (dataWasMoved) { + LOG.error("Revert the data move in renaming a partition."); + try { + if (destFs.exists(destPath)) { + wh.renameDir(destPath, srcPath, false); + } + } catch (MetaException me) { + LOG.error("Failed to restore partition data from " + destPath + " to " + srcPath + + " in alter partition failure. Manual restore is needed."); + } catch (IOException ioe) { + LOG.error("Failed to restore partition data from " + destPath + " to " + srcPath + + " in alter partition failure. Manual restore is needed."); + } + } + } + } + return oldPart; + } + ++ @Deprecated + @Override + public List<Partition> alterPartitions(final RawStore msdb, Warehouse wh, final String dbname, + final String name, final List<Partition> new_parts, + EnvironmentContext environmentContext) + throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException { + return alterPartitions(msdb, wh, DEFAULT_CATALOG_NAME, dbname, name, new_parts, - environmentContext, null); ++ environmentContext, -1, null, -1, null); + } + + @Override + public List<Partition> alterPartitions(final RawStore msdb, Warehouse wh, final String catName, + final String dbname, final String name, + final List<Partition> new_parts, - EnvironmentContext environmentContext, IHMSHandler handler) ++ EnvironmentContext environmentContext, ++ long txnId, String writeIdList, long writeId, ++ IHMSHandler handler) + throws InvalidOperationException, InvalidObjectException, AlreadyExistsException, MetaException { + List<Partition> oldParts = new ArrayList<>(); + List<List<String>> partValsList = new ArrayList<>(); + List<TransactionalMetaStoreEventListener> transactionalListeners = null; + if (handler != null) { + transactionalListeners = handler.getTransactionalListeners(); + } + + boolean success = false; + try { + msdb.openTransaction(); + - Table tbl = msdb.getTable(catName, dbname, name); ++ // Note: should we pass in write ID here? We only update stats on parts so probably not. ++ Table tbl = msdb.getTable(catName, dbname, name, -1, null); + if (tbl == null) { + throw new InvalidObjectException( + "Unable to alter partitions because table or database does not exist."); + } + for (Partition tmpPart: new_parts) { + // Set DDL time to now if not specified + if (tmpPart.getParameters() == null || + tmpPart.getParameters().get(hive_metastoreConstants.DDL_TIME) == null || + Integer.parseInt(tmpPart.getParameters().get(hive_metastoreConstants.DDL_TIME)) == 0) { + tmpPart.putToParameters(hive_metastoreConstants.DDL_TIME, Long.toString(System + .currentTimeMillis() / 1000)); + } + + Partition oldTmpPart = msdb.getPartition(catName, dbname, name, tmpPart.getValues()); + oldParts.add(oldTmpPart); + partValsList.add(tmpPart.getValues()); + + if (MetaStoreUtils.requireCalStats(oldTmpPart, tmpPart, tbl, environmentContext)) { + // Check if stats are same, no need to update + if (MetaStoreUtils.isFastStatsSame(oldTmpPart, tmpPart)) { + MetaStoreUtils.updateBasicState(environmentContext, tmpPart.getParameters()); + } else { + MetaStoreUtils.updatePartitionStatsFast( + tmpPart, tbl, wh, false, true, environmentContext, false); + } + } + + // PartitionView does not have SD and we do not need to update its column stats + if (oldTmpPart.getSd() != null) { + updateOrGetPartitionColumnStats(msdb, catName, dbname, name, oldTmpPart.getValues(), + oldTmpPart.getSd().getCols(), tbl, tmpPart, null); + } + } + - msdb.alterPartitions(catName, dbname, name, partValsList, new_parts); ++ msdb.alterPartitions(catName, dbname, name, partValsList, new_parts, writeId, txnId, writeIdList); + Iterator<Partition> oldPartsIt = oldParts.iterator(); + for (Partition newPart : new_parts) { + Partition oldPart; + if (oldPartsIt.hasNext()) { + oldPart = oldPartsIt.next(); + } else { + throw new InvalidOperationException("Missing old partition corresponding to new partition " + + "when invoking MetaStoreEventListener for alterPartitions event."); + } + + if (transactionalListeners != null && !transactionalListeners.isEmpty()) { + MetaStoreListenerNotifier.notifyEvent(transactionalListeners, + EventMessage.EventType.ALTER_PARTITION, + new AlterPartitionEvent(oldPart, newPart, tbl, false, true, handler)); + } + } + + success = msdb.commitTransaction(); + } catch (InvalidObjectException | NoSuchObjectException e) { + throw new InvalidOperationException("Alter partition operation failed: " + e); + } finally { + if(!success) { + msdb.rollbackTransaction(); + } + } + + return oldParts; + } + + private boolean checkPartialPartKeysEqual(List<FieldSchema> oldPartKeys, + List<FieldSchema> newPartKeys) { + //return true if both are null, or false if one is null and the other isn't + if (newPartKeys == null || oldPartKeys == null) { + return oldPartKeys == newPartKeys; + } + if (oldPartKeys.size() != newPartKeys.size()) { + return false; + } + Iterator<FieldSchema> oldPartKeysIter = oldPartKeys.iterator(); + Iterator<FieldSchema> newPartKeysIter = newPartKeys.iterator(); + FieldSchema oldFs; + FieldSchema newFs; + while (oldPartKeysIter.hasNext()) { + oldFs = oldPartKeysIter.next(); + newFs = newPartKeysIter.next(); + // Alter table can change the type of partition key now. + // So check the column name only. + if (!oldFs.getName().equals(newFs.getName())) { + return false; + } + } + + return true; + } + + /** + * Uses the scheme and authority of the object's current location and the path constructed + * using the object's new name to construct a path for the object's new location. + */ + private Path constructRenamedPath(Path defaultNewPath, Path currentPath) { + URI currentUri = currentPath.toUri(); + + return new Path(currentUri.getScheme(), currentUri.getAuthority(), + defaultNewPath.toUri().getPath()); + } + + @VisibleForTesting - void alterTableUpdateTableColumnStats(RawStore msdb, Table oldTable, Table newTable) ++ void alterTableUpdateTableColumnStats(RawStore msdb, Table oldTable, Table newTable, ++ EnvironmentContext ec) + throws MetaException, InvalidObjectException { + String catName = normalizeIdentifier(oldTable.isSetCatName() ? oldTable.getCatName() : + getDefaultCatalog(conf)); + String dbName = oldTable.getDbName().toLowerCase(); + String tableName = normalizeIdentifier(oldTable.getTableName()); + String newDbName = newTable.getDbName().toLowerCase(); + String newTableName = normalizeIdentifier(newTable.getTableName()); ++ long txnId = -1; ++ String validWriteIds = null; ++ if (ec != null && ec.isSetProperties() && ec.getProperties().containsKey( ++ StatsSetupConst.VALID_WRITE_IDS)) { ++ txnId = Long.parseLong(ec.getProperties().get(StatsSetupConst.TXN_ID)); ++ validWriteIds = ec.getProperties().get(StatsSetupConst.VALID_WRITE_IDS); ++ } + + try { + List<FieldSchema> oldCols = oldTable.getSd().getCols(); + List<FieldSchema> newCols = newTable.getSd().getCols(); + List<ColumnStatisticsObj> newStatsObjs = new ArrayList<>(); + ColumnStatistics colStats = null; + boolean updateColumnStats = true; + + // Nothing to update if everything is the same + if (newDbName.equals(dbName) && + newTableName.equals(tableName) && + MetaStoreUtils.columnsIncludedByNameType(oldCols, newCols)) { + updateColumnStats = false; + } + + if (updateColumnStats) { + List<String> oldColNames = new ArrayList<>(oldCols.size()); + for (FieldSchema oldCol : oldCols) { + oldColNames.add(oldCol.getName()); + } + - // Collect column stats which need to be rewritten and remove old stats ++ // Collect column stats which need to be rewritten and remove old stats. + colStats = msdb.getTableColumnStatistics(catName, dbName, tableName, oldColNames); + if (colStats == null) { + updateColumnStats = false; + } else { + List<ColumnStatisticsObj> statsObjs = colStats.getStatsObj(); + if (statsObjs != null) { + List<String> deletedCols = new ArrayList<>(); + for (ColumnStatisticsObj statsObj : statsObjs) { + boolean found = false; + for (FieldSchema newCol : newCols) { + if (statsObj.getColName().equalsIgnoreCase(newCol.getName()) + && statsObj.getColType().equalsIgnoreCase(newCol.getType())) { + found = true; + break; + } + } + + if (found) { + if (!newDbName.equals(dbName) || !newTableName.equals(tableName)) { + msdb.deleteTableColumnStatistics(catName, dbName, tableName, statsObj.getColName()); + newStatsObjs.add(statsObj); + deletedCols.add(statsObj.getColName()); + } + } else { + msdb.deleteTableColumnStatistics(catName, dbName, tableName, statsObj.getColName()); + deletedCols.add(statsObj.getColName()); + } + } + StatsSetupConst.removeColumnStatsState(newTable.getParameters(), deletedCols); + } + } + } + + // Change to new table and append stats for the new table - msdb.alterTable(catName, dbName, tableName, newTable); ++ msdb.alterTable(catName, dbName, tableName, newTable, txnId, validWriteIds); + if (updateColumnStats && !newStatsObjs.isEmpty()) { + ColumnStatisticsDesc statsDesc = colStats.getStatsDesc(); + statsDesc.setDbName(newDbName); + statsDesc.setTableName(newTableName); + colStats.setStatsObj(newStatsObjs); + msdb.updateTableColumnStatistics(colStats); + } + } catch (NoSuchObjectException nsoe) { + LOG.debug("Could not find db entry." + nsoe); + } catch (InvalidInputException e) { + //should not happen since the input were verified before passed in + throw new InvalidObjectException("Invalid inputs to update table column stats: " + e); + } + } + + private ColumnStatistics updateOrGetPartitionColumnStats( + RawStore msdb, String catName, String dbname, String tblname, List<String> partVals, + List<FieldSchema> oldCols, Table table, Partition part, List<FieldSchema> newCols) + throws MetaException, InvalidObjectException { + ColumnStatistics newPartsColStats = null; + try { + // if newCols are not specified, use default ones. + if (newCols == null) { + newCols = part.getSd() == null ? new ArrayList<>() : part.getSd().getCols(); + } + String oldPartName = Warehouse.makePartName(table.getPartitionKeys(), partVals); + String newPartName = Warehouse.makePartName(table.getPartitionKeys(), part.getValues()); + boolean rename = !part.getDbName().equals(dbname) || !part.getTableName().equals(tblname) + || !oldPartName.equals(newPartName); + + // do not need to update column stats if alter partition is not for rename or changing existing columns + if (!rename && MetaStoreUtils.columnsIncludedByNameType(oldCols, newCols)) { + return newPartsColStats; + } + List<String> oldColNames = new ArrayList<>(oldCols.size()); + for (FieldSchema oldCol : oldCols) { + oldColNames.add(oldCol.getName()); + } + List<String> oldPartNames = Lists.newArrayList(oldPartName); ++ // Note: doesn't take txn stats into account. This method can only remove stats. + List<ColumnStatistics> partsColStats = msdb.getPartitionColumnStatistics(catName, dbname, tblname, + oldPartNames, oldColNames); + assert (partsColStats.size() <= 1); + for (ColumnStatistics partColStats : partsColStats) { //actually only at most one loop + List<ColumnStatisticsObj> newStatsObjs = new ArrayList<>(); + List<ColumnStatisticsObj> statsObjs = partColStats.getStatsObj(); + List<String> deletedCols = new ArrayList<>(); + for (ColumnStatisticsObj statsObj : statsObjs) { + boolean found =false; + for (FieldSchema newCol : newCols) { + if (statsObj.getColName().equalsIgnoreCase(newCol.getName()) + && statsObj.getColType().equalsIgnoreCase(newCol.getType())) { + found = true; + break; + } + } + if (found) { + if (rename) { + msdb.deletePartitionColumnStatistics(catName, dbname, tblname, partColStats.getStatsDesc().getPartName(), + partVals, statsObj.getColName()); + newStatsObjs.add(statsObj); + } + } else { + msdb.deletePartitionColumnStatistics(catName, dbname, tblname, partColStats.getStatsDesc().getPartName(), + partVals, statsObj.getColName()); + deletedCols.add(statsObj.getColName()); + } + } + StatsSetupConst.removeColumnStatsState(part.getParameters(), deletedCols); + if (!newStatsObjs.isEmpty()) { + partColStats.setStatsObj(newStatsObjs); + newPartsColStats = partColStats; + } + } + } catch (NoSuchObjectException nsoe) { + // ignore this exception, actually this exception won't be thrown from getPartitionColumnStatistics + } catch (InvalidInputException iie) { + throw new InvalidObjectException("Invalid input to delete partition column stats." + iie); + } + + return newPartsColStats; + } + + private void checkColTypeChangeCompatible(List<FieldSchema> oldCols, List<FieldSchema> newCols) + throws InvalidOperationException { + List<String> incompatibleCols = new ArrayList<>(); + int maxCols = Math.min(oldCols.size(), newCols.size()); + for (int i = 0; i < maxCols; i++) { + if (!ColumnType.areColTypesCompatible( + ColumnType.getTypeName(oldCols.get(i).getType()), + ColumnType.getTypeName(newCols.get(i).getType()))) { + incompatibleCols.add(newCols.get(i).getName()); + } + } + if (!incompatibleCols.isEmpty()) { + throw new InvalidOperationException( + "The following columns have types incompatible with the existing " + + "columns in their respective positions :\n" + + org.apache.commons.lang.StringUtils.join(incompatibleCols, ',') + ); + } + } + + }