Repository: hive Updated Branches: refs/heads/master 87f8eccf9 -> ef7c3963b
http://git-wip-us.apache.org/repos/asf/hive/blob/ef7c3963/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/events/UpdatePartitionColumnStatEvent.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/events/UpdatePartitionColumnStatEvent.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/events/UpdatePartitionColumnStatEvent.java new file mode 100644 index 0000000..a61b98c --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/events/UpdatePartitionColumnStatEvent.java @@ -0,0 +1,93 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.events; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; +import org.apache.hadoop.hive.metastore.api.ColumnStatistics; + +import java.util.List; +import java.util.Map; + +/** + * UpdatePartitionColumnStatEvent + * Event generated for partition column stat update event. + */ [email protected] [email protected] +public class UpdatePartitionColumnStatEvent extends ListenerEvent { + private ColumnStatistics partColStats; + private String validWriteIds; + private long writeId; + private Map<String, String> parameters; + private List<String> partVals; + + /** + * @param statsObj Columns statistics Info. + * @param partVals partition names + * @param parameters table parameters to be updated after stats are updated. + * @param validWriteIds valid write id list for the query. + * @param writeId writeId for the query. + * @param handler handler that is firing the event + */ + public UpdatePartitionColumnStatEvent(ColumnStatistics statsObj, List<String> partVals, Map<String, String> parameters, + String validWriteIds, long writeId, IHMSHandler handler) { + super(true, handler); + this.partColStats = statsObj; + this.validWriteIds = validWriteIds; + this.writeId = writeId; + this.parameters = parameters; + this.partVals = partVals; + } + + /** + * @param statsObj Columns statistics Info. + * @param partVals partition names + * @param handler handler that is firing the event + */ + public UpdatePartitionColumnStatEvent(ColumnStatistics statsObj, List<String> partVals, IHMSHandler handler) { + super(true, handler); + this.partColStats = statsObj; + this.partVals = partVals; + this.validWriteIds = null; + this.writeId = 0; + this.parameters = null; + } + + public ColumnStatistics getPartColStats() { + return partColStats; + } + + public String getValidWriteIds() { + return validWriteIds; + } + + public long getWriteId() { + return writeId; + } + + public Map<String, String> getPartParameters() { + return parameters; + } + + public List<String> getPartVals() { + return partVals; + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/ef7c3963/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/events/UpdateTableColumnStatEvent.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/events/UpdateTableColumnStatEvent.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/events/UpdateTableColumnStatEvent.java new file mode 100644 index 0000000..cf23617 --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/events/UpdateTableColumnStatEvent.java @@ -0,0 +1,84 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.events; + +import org.apache.hadoop.classification.InterfaceAudience; +import org.apache.hadoop.classification.InterfaceStability; +import org.apache.hadoop.hive.metastore.IHMSHandler; +import org.apache.hadoop.hive.metastore.api.ColumnStatistics; + +import java.util.List; +import java.util.Map; + +/** + * UpdateTableColumnStatEvent + * Event generated for table column stat update event. + */ [email protected] [email protected] +public class UpdateTableColumnStatEvent extends ListenerEvent { + private ColumnStatistics colStats; + private String validWriteIds; + private long writeId; + private Map<String, String> parameters; + + /** + * @param colStats Columns statistics Info. + * @param parameters table parameters to be updated after stats are updated. + * @param validWriteIds valid write id list for the query. + * @param colStats writeId for the query. + * @param handler handler that is firing the event + */ + public UpdateTableColumnStatEvent(ColumnStatistics colStats, Map<String, String> parameters, String validWriteIds, + long writeId, IHMSHandler handler) { + super(true, handler); + this.colStats = colStats; + this.validWriteIds = validWriteIds; + this.writeId = writeId; + this.parameters = parameters; + } + + /** + * @param colStats Columns statistics Info. + * @param handler handler that is firing the event + */ + public UpdateTableColumnStatEvent(ColumnStatistics colStats, IHMSHandler handler) { + super(true, handler); + this.colStats = colStats; + this.validWriteIds = null; + this.writeId = 0; + this.parameters = null; + } + + public ColumnStatistics getColStats() { + return colStats; + } + + public String getValidWriteIds() { + return validWriteIds; + } + + public long getWriteId() { + return writeId; + } + + public Map<String, String> getTableParameters() { + return parameters; + } +} http://git-wip-us.apache.org/repos/asf/hive/blob/ef7c3963/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/DeletePartitionColumnStatMessage.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/DeletePartitionColumnStatMessage.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/DeletePartitionColumnStatMessage.java new file mode 100644 index 0000000..d8166c3 --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/DeletePartitionColumnStatMessage.java @@ -0,0 +1,36 @@ +/* * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.messaging; + +import java.util.List; + +/** + * HCat message sent when an partition column statistics is deleted. + */ +public abstract class DeletePartitionColumnStatMessage extends EventMessage { + + protected DeletePartitionColumnStatMessage() { + super(EventType.DELETE_PARTITION_COLUMN_STAT); + } + + public abstract String getColName(); + + public abstract String getPartName(); + + public abstract List<String> getPartValues(); +} http://git-wip-us.apache.org/repos/asf/hive/blob/ef7c3963/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/DeleteTableColumnStatMessage.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/DeleteTableColumnStatMessage.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/DeleteTableColumnStatMessage.java new file mode 100644 index 0000000..61893b2 --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/DeleteTableColumnStatMessage.java @@ -0,0 +1,30 @@ +/* * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.messaging; + +/** + * HCat message sent when an table table statistics is deleted. + */ +public abstract class DeleteTableColumnStatMessage extends EventMessage { + + protected DeleteTableColumnStatMessage() { + super(EventType.DELETE_TABLE_COLUMN_STAT); + } + + public abstract String getColName(); +} http://git-wip-us.apache.org/repos/asf/hive/blob/ef7c3963/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/EventMessage.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/EventMessage.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/EventMessage.java index 1262c12..8b3e474 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/EventMessage.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/EventMessage.java @@ -61,7 +61,11 @@ public abstract class EventMessage { ABORT_TXN(MessageBuilder.ABORT_TXN_EVENT), ALLOC_WRITE_ID(MessageBuilder.ALLOC_WRITE_ID_EVENT), ALTER_CATALOG(MessageBuilder.ALTER_CATALOG_EVENT), - ACID_WRITE(MessageBuilder.ACID_WRITE_EVENT); + ACID_WRITE(MessageBuilder.ACID_WRITE_EVENT), + UPDATE_TABLE_COLUMN_STAT(MessageBuilder.UPDATE_TBL_COL_STAT_EVENT), + DELETE_TABLE_COLUMN_STAT(MessageBuilder.DELETE_TBL_COL_STAT_EVENT), + UPDATE_PARTITION_COLUMN_STAT(MessageBuilder.UPDATE_PART_COL_STAT_EVENT), + DELETE_PARTITION_COLUMN_STAT(MessageBuilder.DELETE_PART_COL_STAT_EVENT); private String typeString; http://git-wip-us.apache.org/repos/asf/hive/blob/ef7c3963/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/MessageBuilder.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/MessageBuilder.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/MessageBuilder.java index 787b9b2..6add6c2 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/MessageBuilder.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/MessageBuilder.java @@ -46,6 +46,7 @@ import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey; import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint; import org.apache.hadoop.hive.metastore.api.Table; import org.apache.hadoop.hive.metastore.api.TxnToWriteId; +import org.apache.hadoop.hive.metastore.api.ColumnStatistics; import org.apache.hadoop.hive.metastore.conf.MetastoreConf; import org.apache.hadoop.hive.metastore.events.AcidWriteEvent; import org.apache.hadoop.hive.metastore.messaging.json.JSONAbortTxnMessage; @@ -73,6 +74,10 @@ import org.apache.hadoop.hive.metastore.messaging.json.JSONDropPartitionMessage; import org.apache.hadoop.hive.metastore.messaging.json.JSONDropTableMessage; import org.apache.hadoop.hive.metastore.messaging.json.JSONInsertMessage; import org.apache.hadoop.hive.metastore.messaging.json.JSONOpenTxnMessage; +import org.apache.hadoop.hive.metastore.messaging.json.JSONUpdateTableColumnStatMessage; +import org.apache.hadoop.hive.metastore.messaging.json.JSONUpdatePartitionColumnStatMessage; +import org.apache.hadoop.hive.metastore.messaging.json.JSONDeleteTableColumnStatMessage; +import org.apache.hadoop.hive.metastore.messaging.json.JSONDeletePartitionColumnStatMessage; import org.apache.hadoop.hive.metastore.utils.MetaStoreUtils; import org.apache.thrift.TBase; import org.apache.thrift.TDeserializer; @@ -118,6 +123,10 @@ public class MessageBuilder { public static final String ALLOC_WRITE_ID_EVENT = "ALLOC_WRITE_ID_EVENT"; public static final String ALTER_CATALOG_EVENT = "ALTER_CATALOG"; public static final String ACID_WRITE_EVENT = "ACID_WRITE_EVENT"; + public static final String UPDATE_TBL_COL_STAT_EVENT = "UPDATE_TBL_COL_STAT_EVENT"; + public static final String DELETE_TBL_COL_STAT_EVENT = "DELETE_TBL_COL_STAT_EVENT"; + public static final String UPDATE_PART_COL_STAT_EVENT = "UPDATE_PART_COL_STAT_EVENT"; + public static final String DELETE_PART_COL_STAT_EVENT = "DELETE_PART_COL_STAT_EVENT"; protected static final Configuration conf = MetastoreConf.newMetastoreConf(); @@ -277,6 +286,30 @@ public class MessageBuilder { files); } + public JSONUpdateTableColumnStatMessage buildUpdateTableColumnStatMessage(ColumnStatistics colStats, + Map<String, String> parameters, + String validWriteIds, long writeId) { + return new JSONUpdateTableColumnStatMessage(MS_SERVER_URL, MS_SERVICE_PRINCIPAL, now(), colStats, parameters, + validWriteIds, writeId); + } + + public JSONDeleteTableColumnStatMessage buildDeleteTableColumnStatMessage(String dbName, String colName) { + return new JSONDeleteTableColumnStatMessage(MS_SERVER_URL, MS_SERVICE_PRINCIPAL, now(), dbName, colName); + } + + public JSONUpdatePartitionColumnStatMessage buildUpdatePartitionColumnStatMessage(ColumnStatistics colStats, + List<String> partVals, Map<String, String> parameters, + String validWriteIds, long writeId) { + return new JSONUpdatePartitionColumnStatMessage(MS_SERVER_URL, MS_SERVICE_PRINCIPAL, now(), colStats, partVals, + parameters, validWriteIds, writeId); + } + + public JSONDeletePartitionColumnStatMessage buildDeletePartitionColumnStatMessage(String dbName, String colName, + String partName, List<String> partValues) { + return new JSONDeletePartitionColumnStatMessage(MS_SERVER_URL, MS_SERVICE_PRINCIPAL, now(), dbName, + colName, partName, partValues); + } + private long now() { return System.currentTimeMillis() / 1000; } @@ -342,6 +375,11 @@ public class MessageBuilder { return tableObj; } + public static String createTableColumnStatJson(ColumnStatistics tableColumnStat) throws TException { + TSerializer serializer = new TSerializer(new TJSONProtocol.Factory()); + return serializer.toString(tableColumnStat, "UTF-8"); + } + /* * TODO: Some thoughts here : We have a current todo to move some of these methods over to * MessageFactory instead of being here, so we can override them, but before we move them over, http://git-wip-us.apache.org/repos/asf/hive/blob/ef7c3963/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/MessageDeserializer.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/MessageDeserializer.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/MessageDeserializer.java index b701d84..8a7031f 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/MessageDeserializer.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/MessageDeserializer.java @@ -74,6 +74,14 @@ public abstract class MessageDeserializer { return getAllocWriteIdMessage(messageBody); case ACID_WRITE: return getAcidWriteMessage(messageBody); + case UPDATE_TABLE_COLUMN_STAT: + return getUpdateTableColumnStatMessage(messageBody); + case DELETE_TABLE_COLUMN_STAT: + return getDeleteTableColumnStatMessage(messageBody); + case UPDATE_PARTITION_COLUMN_STAT: + return getUpdatePartitionColumnStatMessage(messageBody); + case DELETE_PARTITION_COLUMN_STAT: + return getDeletePartitionColumnStatMessage(messageBody); default: throw new IllegalArgumentException("Unsupported event-type: " + eventTypeString); } @@ -195,6 +203,26 @@ public abstract class MessageDeserializer { */ public abstract AcidWriteMessage getAcidWriteMessage(String messageBody); + /** + * Method to de-serialize UpdateTableColumnStatMessage instance. + */ + public abstract UpdateTableColumnStatMessage getUpdateTableColumnStatMessage(String messageBody); + + /** + * Method to de-serialize DeleteTableColumnStatMessage instance. + */ + public abstract DeleteTableColumnStatMessage getDeleteTableColumnStatMessage(String messageBody); + + /** + * Method to de-serialize UpdatePartitionColumnStatMessage instance. + */ + public abstract UpdatePartitionColumnStatMessage getUpdatePartitionColumnStatMessage(String messageBody); + + /** + * Method to de-serialize DeletePartitionColumnStatMessage instance. + */ + public abstract DeletePartitionColumnStatMessage getDeletePartitionColumnStatMessage(String messageBody); + // Protection against construction. protected MessageDeserializer() {} } http://git-wip-us.apache.org/repos/asf/hive/blob/ef7c3963/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/UpdatePartitionColumnStatMessage.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/UpdatePartitionColumnStatMessage.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/UpdatePartitionColumnStatMessage.java new file mode 100644 index 0000000..d15c6e6 --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/UpdatePartitionColumnStatMessage.java @@ -0,0 +1,42 @@ +/* * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.messaging; + +import org.apache.hadoop.hive.metastore.api.ColumnStatistics; +import java.util.List; +import java.util.Map; + +/** + * HCat message sent when an table partition statistics update is done. + */ +public abstract class UpdatePartitionColumnStatMessage extends EventMessage { + + protected UpdatePartitionColumnStatMessage() { + super(EventType.UPDATE_TABLE_COLUMN_STAT); + } + + public abstract ColumnStatistics getColumnStatistics(); + + public abstract String getValidWriteIds(); + + public abstract Long getWriteId(); + + public abstract Map<String, String> getParameters(); + + public abstract List<String> getPartVals(); +} http://git-wip-us.apache.org/repos/asf/hive/blob/ef7c3963/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/UpdateTableColumnStatMessage.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/UpdateTableColumnStatMessage.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/UpdateTableColumnStatMessage.java new file mode 100644 index 0000000..ed8944b --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/UpdateTableColumnStatMessage.java @@ -0,0 +1,39 @@ +/* * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.hadoop.hive.metastore.messaging; + +import org.apache.hadoop.hive.metastore.api.ColumnStatistics; +import java.util.Map; + +/** + * HCat message sent when an table column statistics update is done. + */ +public abstract class UpdateTableColumnStatMessage extends EventMessage { + + protected UpdateTableColumnStatMessage() { + super(EventType.UPDATE_TABLE_COLUMN_STAT); + } + + public abstract ColumnStatistics getColumnStatistics(); + + public abstract String getValidWriteIds(); + + public abstract Long getWriteId(); + + public abstract Map<String, String> getParameters(); +} http://git-wip-us.apache.org/repos/asf/hive/blob/ef7c3963/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDeletePartitionColumnStatMessage.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDeletePartitionColumnStatMessage.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDeletePartitionColumnStatMessage.java new file mode 100644 index 0000000..a06348e --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDeletePartitionColumnStatMessage.java @@ -0,0 +1,102 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.hive.metastore.messaging.json; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.hadoop.hive.metastore.messaging.DeletePartitionColumnStatMessage; +import java.util.List; + +/** + * JSON implementation of JSONDeletePartitionColumnStatMessage + */ +public class JSONDeletePartitionColumnStatMessage extends DeletePartitionColumnStatMessage { + + @JsonProperty + private Long timestamp; + + @JsonProperty + private String server, servicePrincipal, database, colName, partName; + + @JsonProperty + private List<String> partValues; + + /** + * Default constructor, needed for Jackson. + */ + public JSONDeletePartitionColumnStatMessage() { + } + + public JSONDeletePartitionColumnStatMessage(String server, String servicePrincipal, Long timestamp, + String dbName, String colName, String partName, + List<String> partValues) { + this.timestamp = timestamp; + this.server = server; + this.servicePrincipal = servicePrincipal; + this.database = dbName; + this.partValues = partValues; + this.partName = partName; + this.colName = colName; + } + + @Override + public Long getTimestamp() { + return timestamp; + } + + @Override + public String getDB() { + return database; + } + + @Override + public String getServicePrincipal() { + return servicePrincipal; + } + + @Override + public String getServer() { + return server; + } + + @Override + public String getColName() { + return colName; + } + + @Override + public String getPartName() { + return partName; + } + + @Override + public List<String> getPartValues() { + return partValues; + } + + @Override + public String toString() { + try { + return JSONMessageDeserializer.mapper.writeValueAsString(this); + } catch (Exception exception) { + throw new IllegalArgumentException("Could not serialize: ", exception); + } + } +} + http://git-wip-us.apache.org/repos/asf/hive/blob/ef7c3963/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDeleteTableColumnStatMessage.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDeleteTableColumnStatMessage.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDeleteTableColumnStatMessage.java new file mode 100644 index 0000000..4fdba25 --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDeleteTableColumnStatMessage.java @@ -0,0 +1,85 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.hive.metastore.messaging.json; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.hadoop.hive.metastore.messaging.DeleteTableColumnStatMessage; + +/** + * JSON implementation of JSONDeleteTableColumnStatMessage + */ +public class JSONDeleteTableColumnStatMessage extends DeleteTableColumnStatMessage { + + @JsonProperty + private Long timestamp; + + @JsonProperty + private String server, servicePrincipal, database, colName; + + /** + * Default constructor, needed for Jackson. + */ + public JSONDeleteTableColumnStatMessage() { + } + + public JSONDeleteTableColumnStatMessage(String server, String servicePrincipal, Long timestamp, + String dbName, String colName) { + this.timestamp = timestamp; + this.server = server; + this.servicePrincipal = servicePrincipal; + this.database = dbName; + this.colName = colName; + } + + @Override + public Long getTimestamp() { + return timestamp; + } + + @Override + public String getDB() { + return database; + } + + @Override + public String getServicePrincipal() { + return servicePrincipal; + } + + @Override + public String getServer() { + return server; + } + + @Override + public String getColName() { + return colName; + } + + @Override + public String toString() { + try { + return JSONMessageDeserializer.mapper.writeValueAsString(this); + } catch (Exception exception) { + throw new IllegalArgumentException("Could not serialize: ", exception); + } + } +} + http://git-wip-us.apache.org/repos/asf/hive/blob/ef7c3963/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializer.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializer.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializer.java index 3a480aa..ef7a947 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializer.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializer.java @@ -42,6 +42,10 @@ import org.apache.hadoop.hive.metastore.messaging.InsertMessage; import org.apache.hadoop.hive.metastore.messaging.MessageDeserializer; import org.apache.hadoop.hive.metastore.messaging.OpenTxnMessage; import org.apache.hadoop.hive.metastore.messaging.AcidWriteMessage; +import org.apache.hadoop.hive.metastore.messaging.UpdateTableColumnStatMessage; +import org.apache.hadoop.hive.metastore.messaging.DeleteTableColumnStatMessage; +import org.apache.hadoop.hive.metastore.messaging.UpdatePartitionColumnStatMessage; +import org.apache.hadoop.hive.metastore.messaging.DeletePartitionColumnStatMessage; import com.fasterxml.jackson.databind.DeserializationFeature; import com.fasterxml.jackson.databind.MapperFeature; import com.fasterxml.jackson.databind.ObjectMapper; @@ -270,4 +274,40 @@ public class JSONMessageDeserializer extends MessageDeserializer { throw new IllegalArgumentException("Could not construct AcidWriteMessage", e); } } + + @Override + public UpdateTableColumnStatMessage getUpdateTableColumnStatMessage(String messageBody) { + try { + return mapper.readValue(messageBody, JSONUpdateTableColumnStatMessage.class); + } catch (Exception e) { + throw new IllegalArgumentException("Could not construct UpdateTableColumnStatMessage", e); + } + } + + @Override + public DeleteTableColumnStatMessage getDeleteTableColumnStatMessage(String messageBody) { + try { + return mapper.readValue(messageBody, JSONDeleteTableColumnStatMessage.class); + } catch (Exception e) { + throw new IllegalArgumentException("Could not construct UpdateTableColumnStatMessage", e); + } + } + + @Override + public UpdatePartitionColumnStatMessage getUpdatePartitionColumnStatMessage(String messageBody) { + try { + return mapper.readValue(messageBody, JSONUpdatePartitionColumnStatMessage.class); + } catch (Exception e) { + throw new IllegalArgumentException("Could not construct UpdatePartitionColumnStatMessage", e); + } + } + + @Override + public DeletePartitionColumnStatMessage getDeletePartitionColumnStatMessage(String messageBody) { + try { + return mapper.readValue(messageBody, JSONDeletePartitionColumnStatMessage.class); + } catch (Exception e) { + throw new IllegalArgumentException("Could not construct UpdatePartitionColumnStatMessage", e); + } + } } http://git-wip-us.apache.org/repos/asf/hive/blob/ef7c3963/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONUpdatePartitionColumnStatMessage.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONUpdatePartitionColumnStatMessage.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONUpdatePartitionColumnStatMessage.java new file mode 100644 index 0000000..3f20091 --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONUpdatePartitionColumnStatMessage.java @@ -0,0 +1,133 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.hive.metastore.messaging.json; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.hadoop.hive.metastore.api.ColumnStatistics; +import org.apache.hadoop.hive.metastore.messaging.MessageBuilder; +import org.apache.hadoop.hive.metastore.messaging.UpdatePartitionColumnStatMessage; +import org.apache.thrift.TException; + +import java.util.List; +import java.util.Map; + +/** + * JSON implementation of JSONUpdatePartitionColumnStatMessage + */ +public class JSONUpdatePartitionColumnStatMessage extends UpdatePartitionColumnStatMessage { + + @JsonProperty + private Long writeId, timestamp; + + @JsonProperty + private String validWriteIds, server, servicePrincipal, database; + + @JsonProperty + private String colStatsJson; + + @JsonProperty + private Map<String, String> parameters; + + @JsonProperty + private List<String> partVals; + + /** + * Default constructor, needed for Jackson. + */ + public JSONUpdatePartitionColumnStatMessage() { + } + + public JSONUpdatePartitionColumnStatMessage(String server, String servicePrincipal, Long timestamp, + ColumnStatistics colStats, List<String> partVals, + Map<String, String> parameters, String validWriteIds, long writeId) { + this.timestamp = timestamp; + this.server = server; + this.servicePrincipal = servicePrincipal; + this.writeId = writeId; + this.validWriteIds = validWriteIds; + this.database = colStats.getStatsDesc().getDbName(); + this.partVals = partVals; + try { + this.colStatsJson = MessageBuilder.createTableColumnStatJson(colStats); + } catch (TException e) { + throw new IllegalArgumentException("Could not serialize JSONUpdatePartitionColumnStatMessage : ", e); + } + this.parameters = parameters; + } + + @Override + public Long getTimestamp() { + return timestamp; + } + + @Override + public String getDB() { + return database; + } + + @Override + public String getServicePrincipal() { + return servicePrincipal; + } + + @Override + public String getServer() { + return server; + } + + @Override + public ColumnStatistics getColumnStatistics() { + try { + return (ColumnStatistics) MessageBuilder.getTObj(colStatsJson, ColumnStatistics.class); + } catch (Exception e) { + throw new RuntimeException("failed to get the ColumnStatistics object ", e); + } + } + + @Override + public String getValidWriteIds() { + return validWriteIds; + } + + @Override + public Long getWriteId() { + return writeId; + } + + @Override + public Map<String, String> getParameters() { + return parameters; + } + + @Override + public List<String> getPartVals() { + return partVals; + } + + @Override + public String toString() { + try { + return JSONMessageDeserializer.mapper.writeValueAsString(this); + } catch (Exception exception) { + throw new IllegalArgumentException("Could not serialize: ", exception); + } + } +} + http://git-wip-us.apache.org/repos/asf/hive/blob/ef7c3963/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONUpdateTableColumnStatMessage.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONUpdateTableColumnStatMessage.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONUpdateTableColumnStatMessage.java new file mode 100644 index 0000000..d99ef11 --- /dev/null +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONUpdateTableColumnStatMessage.java @@ -0,0 +1,121 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.hadoop.hive.metastore.messaging.json; + +import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.hadoop.hive.metastore.api.ColumnStatistics; +import org.apache.hadoop.hive.metastore.messaging.MessageBuilder; +import org.apache.hadoop.hive.metastore.messaging.UpdateTableColumnStatMessage; +import org.apache.thrift.TException; +import java.util.Map; + +/** + * JSON implementation of JSONUpdateTableColumnStatMessage + */ +public class JSONUpdateTableColumnStatMessage extends UpdateTableColumnStatMessage { + + @JsonProperty + private Long writeId, timestamp; + + @JsonProperty + private String validWriteIds, server, servicePrincipal, database; + + @JsonProperty + private String colStatsJson; + + @JsonProperty + Map<String, String> parameters; + + /** + * Default constructor, needed for Jackson. + */ + public JSONUpdateTableColumnStatMessage() { + } + + public JSONUpdateTableColumnStatMessage(String server, String servicePrincipal, Long timestamp, + ColumnStatistics colStats, Map<String, String> parameters, String validWriteIds, long writeId) { + this.timestamp = timestamp; + this.server = server; + this.servicePrincipal = servicePrincipal; + this.writeId = writeId; + this.validWriteIds = validWriteIds; + this.database = colStats.getStatsDesc().getDbName(); + try { + this.colStatsJson = MessageBuilder.createTableColumnStatJson(colStats); + } catch (TException e) { + throw new IllegalArgumentException("Could not serialize JSONUpdateTableColumnStatMessage : ", e); + } + this.parameters = parameters; + } + + @Override + public Long getTimestamp() { + return timestamp; + } + + @Override + public String getDB() { + return database; + } + + @Override + public String getServicePrincipal() { + return servicePrincipal; + } + + @Override + public String getServer() { + return server; + } + + @Override + public ColumnStatistics getColumnStatistics() { + try { + return (ColumnStatistics) MessageBuilder.getTObj(colStatsJson, ColumnStatistics.class); + } catch (Exception e) { + throw new RuntimeException("failed to get the ColumnStatistics object ", e); + } + } + + @Override + public String getValidWriteIds() { + return validWriteIds; + } + + @Override + public Long getWriteId() { + return writeId; + } + + @Override + public Map<String, String> getParameters() { + return parameters; + } + + @Override + public String toString() { + try { + return JSONMessageDeserializer.mapper.writeValueAsString(this); + } catch (Exception exception) { + throw new IllegalArgumentException("Could not serialize: ", exception); + } + } +} + http://git-wip-us.apache.org/repos/asf/hive/blob/ef7c3963/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/gzip/DeSerializer.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/gzip/DeSerializer.java b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/gzip/DeSerializer.java index 49596b5..e951481 100644 --- a/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/gzip/DeSerializer.java +++ b/standalone-metastore/metastore-server/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/gzip/DeSerializer.java @@ -42,6 +42,10 @@ import org.apache.hadoop.hive.metastore.messaging.DropPartitionMessage; import org.apache.hadoop.hive.metastore.messaging.DropTableMessage; import org.apache.hadoop.hive.metastore.messaging.InsertMessage; import org.apache.hadoop.hive.metastore.messaging.OpenTxnMessage; +import org.apache.hadoop.hive.metastore.messaging.UpdateTableColumnStatMessage; +import org.apache.hadoop.hive.metastore.messaging.UpdatePartitionColumnStatMessage; +import org.apache.hadoop.hive.metastore.messaging.DeleteTableColumnStatMessage; +import org.apache.hadoop.hive.metastore.messaging.DeletePartitionColumnStatMessage; import org.apache.hadoop.hive.metastore.messaging.json.JSONMessageDeserializer; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -196,4 +200,24 @@ public class DeSerializer extends JSONMessageDeserializer { public AcidWriteMessage getAcidWriteMessage(String messageBody) { return super.getAcidWriteMessage(deCompress(messageBody)); } + + @Override + public UpdateTableColumnStatMessage getUpdateTableColumnStatMessage(String messageBody) { + return super.getUpdateTableColumnStatMessage(deCompress(messageBody)); + } + + @Override + public UpdatePartitionColumnStatMessage getUpdatePartitionColumnStatMessage(String messageBody) { + return super.getUpdatePartitionColumnStatMessage(deCompress(messageBody)); + } + + @Override + public DeleteTableColumnStatMessage getDeleteTableColumnStatMessage(String messageBody) { + return super.getDeleteTableColumnStatMessage(deCompress(messageBody)); + } + + @Override + public DeletePartitionColumnStatMessage getDeletePartitionColumnStatMessage(String messageBody) { + return super.getDeletePartitionColumnStatMessage(deCompress(messageBody)); + } } http://git-wip-us.apache.org/repos/asf/hive/blob/ef7c3963/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveAlterHandler.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveAlterHandler.java b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveAlterHandler.java index 93b2f23..88d5e71 100644 --- a/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveAlterHandler.java +++ b/standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/TestHiveAlterHandler.java @@ -60,7 +60,7 @@ public class TestHiveAlterHandler { getDefaultCatalog(conf), oldTable.getDbName(), oldTable.getTableName(), Arrays.asList("col1", "col2", "col3")); HiveAlterHandler handler = new HiveAlterHandler(); handler.setConf(conf); - handler.alterTableUpdateTableColumnStats(msdb, oldTable, newTable, null, null); + handler.alterTableUpdateTableColumnStats(msdb, oldTable, newTable, null, null, conf, null); } @Test @@ -86,7 +86,7 @@ public class TestHiveAlterHandler { HiveAlterHandler handler = new HiveAlterHandler(); handler.setConf(conf); try { - handler.alterTableUpdateTableColumnStats(msdb, oldTable, newTable, null, null); + handler.alterTableUpdateTableColumnStats(msdb, oldTable, newTable, null, null, conf, null); } catch (Throwable t) { System.err.println(t); t.printStackTrace(System.err); @@ -121,7 +121,7 @@ public class TestHiveAlterHandler { getDefaultCatalog(conf), oldTable.getDbName(), oldTable.getTableName(), Arrays.asList("col1", "col2", "col3", "col4")); HiveAlterHandler handler = new HiveAlterHandler(); handler.setConf(conf); - handler.alterTableUpdateTableColumnStats(msdb, oldTable, newTable, null, null); + handler.alterTableUpdateTableColumnStats(msdb, oldTable, newTable, null, null, conf, null); } }
