http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateDatabaseMessage.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateDatabaseMessage.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateDatabaseMessage.java deleted file mode 100644 index 761ff99..0000000 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateDatabaseMessage.java +++ /dev/null @@ -1,85 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hive.metastore.messaging.json; - -import org.apache.hadoop.hive.metastore.api.Database; -import org.apache.hadoop.hive.metastore.messaging.CreateDatabaseMessage; -import org.apache.thrift.TException; - -import com.fasterxml.jackson.annotation.JsonProperty; - -/** - * JSON Implementation of CreateDatabaseMessage. - */ -public class JSONCreateDatabaseMessage extends CreateDatabaseMessage { - - @JsonProperty - String server, servicePrincipal, db, dbJson; - - @JsonProperty - Long timestamp; - - /** - * Default constructor, required for Jackson. - */ - public JSONCreateDatabaseMessage() {} - - public JSONCreateDatabaseMessage(String server, String servicePrincipal, Database db, - Long timestamp) { - this.server = server; - this.servicePrincipal = servicePrincipal; - this.db = db.getName(); - this.timestamp = timestamp; - try { - this.dbJson = JSONMessageFactory.createDatabaseObjJson(db); - } catch (TException ex) { - throw new IllegalArgumentException("Could not serialize Function object", ex); - } - checkValid(); - } - - @Override - public Database getDatabaseObject() throws Exception { - return (Database) JSONMessageFactory.getTObj(dbJson, Database.class); - } - - @Override - public String getDB() { return db; } - - @Override - public String getServer() { return server; } - - @Override - public String getServicePrincipal() { return servicePrincipal; } - - @Override - public Long getTimestamp() { return timestamp; } - - @Override - public String toString() { - try { - return JSONMessageDeserializer.mapper.writeValueAsString(this); - } - catch (Exception exception) { - throw new IllegalArgumentException("Could not serialize: ", exception); - } - } - -}
http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateFunctionMessage.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateFunctionMessage.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateFunctionMessage.java deleted file mode 100644 index f7287df..0000000 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateFunctionMessage.java +++ /dev/null @@ -1,87 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hive.metastore.messaging.json; - -import org.apache.hadoop.hive.metastore.api.Function; -import org.apache.hadoop.hive.metastore.messaging.CreateFunctionMessage; -import org.apache.thrift.TException; - -import com.fasterxml.jackson.annotation.JsonProperty; - -/** - * JSON Implementation of CreateFunctionMessage. - */ -public class JSONCreateFunctionMessage extends CreateFunctionMessage { - - @JsonProperty - String server, servicePrincipal, db, functionObjJson; - - @JsonProperty - Long timestamp; - - /** - * Default constructor, required for Jackson. - */ - public JSONCreateFunctionMessage() {} - - public JSONCreateFunctionMessage(String server, String servicePrincipal, Function fn, Long timestamp) { - this.server = server; - this.servicePrincipal = servicePrincipal; - this.db = fn.getDbName(); - this.timestamp = timestamp; - try { - this.functionObjJson = JSONMessageFactory.createFunctionObjJson(fn); - } catch (TException ex) { - throw new IllegalArgumentException("Could not serialize Function object", ex); - } - checkValid(); - } - - @Override - public String getDB() { return db; } - - @Override - public String getServer() { return server; } - - @Override - public String getServicePrincipal() { return servicePrincipal; } - - @Override - public Long getTimestamp() { return timestamp; } - - public String getFunctionObjJson() { - return functionObjJson; - } - - @Override - public Function getFunctionObj() throws Exception { - return (Function) JSONMessageFactory.getTObj(functionObjJson,Function.class); - } - - @Override - public String toString() { - try { - return JSONMessageDeserializer.mapper.writeValueAsString(this); - } - catch (Exception exception) { - throw new IllegalArgumentException("Could not serialize: ", exception); - } - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateTableMessage.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateTableMessage.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateTableMessage.java deleted file mode 100644 index d0f4b86..0000000 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONCreateTableMessage.java +++ /dev/null @@ -1,134 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hive.metastore.messaging.json; - -import java.util.Iterator; -import java.util.List; - -import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.hive.metastore.messaging.CreateTableMessage; -import org.apache.thrift.TException; - -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.collect.Lists; - -/** - * JSON implementation of CreateTableMessage. - */ -public class JSONCreateTableMessage extends CreateTableMessage { - - @JsonProperty - String server, servicePrincipal, db, table, tableType, tableObjJson; - @JsonProperty - Long timestamp; - @JsonProperty - List<String> files; - - /** - * Default constructor, needed for Jackson. - */ - public JSONCreateTableMessage() { - } - - public JSONCreateTableMessage(String server, String servicePrincipal, String db, String table, - String tableType, Long timestamp) { - this.server = server; - this.servicePrincipal = servicePrincipal; - this.db = db; - this.table = table; - this.tableType = tableType; - this.timestamp = timestamp; - checkValid(); - } - - public JSONCreateTableMessage(String server, String servicePrincipal, String db, String table, - Long timestamp) { - this(server, servicePrincipal, db, table, null, timestamp); - } - - public JSONCreateTableMessage(String server, String servicePrincipal, Table tableObj, - Iterator<String> fileIter, Long timestamp) { - this(server, servicePrincipal, tableObj.getDbName(), tableObj.getTableName(), - tableObj.getTableType(), timestamp); - try { - this.tableObjJson = JSONMessageFactory.createTableObjJson(tableObj); - } catch (TException e) { - throw new IllegalArgumentException("Could not serialize: ", e); - } - this.files = Lists.newArrayList(fileIter); - } - - @Override - public String getServer() { - return server; - } - - @Override - public String getServicePrincipal() { - return servicePrincipal; - } - - @Override - public String getDB() { - return db; - } - - @Override - public Long getTimestamp() { - return timestamp; - } - - @Override - public String getTable() { - return table; - } - - @Override - public String getTableType() { - if (tableType != null) { - return tableType; - } else { - return ""; - } - } - - @Override - public Table getTableObj() throws Exception { - return (Table) JSONMessageFactory.getTObj(tableObjJson,Table.class); - } - - public String getTableObjJson() { - return tableObjJson; - } - - @Override - public String toString() { - try { - return JSONMessageDeserializer.mapper.writeValueAsString(this); - } catch (Exception exception) { - throw new IllegalArgumentException("Could not serialize: ", exception); - } - } - - @Override - public Iterable<String> getFiles() { - return files; - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropCatalogMessage.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropCatalogMessage.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropCatalogMessage.java deleted file mode 100644 index 3576351..0000000 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropCatalogMessage.java +++ /dev/null @@ -1,67 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.metastore.messaging.json; - -import org.apache.hadoop.hive.metastore.messaging.DropCatalogMessage; - -import com.fasterxml.jackson.annotation.JsonProperty; - -public class JSONDropCatalogMessage extends DropCatalogMessage { - - @JsonProperty - String server, servicePrincipal, catalog; - - @JsonProperty - Long timestamp; - - public JSONDropCatalogMessage() { - - } - - public JSONDropCatalogMessage(String server, String servicePrincipal, String catalog, - Long timestamp) { - this.server = server; - this.servicePrincipal = servicePrincipal; - this.catalog = catalog; - this.timestamp = timestamp; - } - - @Override - public String getDB() { - return null; - } - - @Override - public String getServer() { - return server; - } - - @Override - public String getServicePrincipal() { - return servicePrincipal; - } - - public String getCatalog() { - return catalog; - } - - @Override - public Long getTimestamp() { - return timestamp; - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropConstraintMessage.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropConstraintMessage.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropConstraintMessage.java deleted file mode 100644 index 8dcfd8b..0000000 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropConstraintMessage.java +++ /dev/null @@ -1,91 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hive.metastore.messaging.json; - -import org.apache.hadoop.hive.metastore.messaging.DropConstraintMessage; - -import com.fasterxml.jackson.annotation.JsonProperty; - -/** - * JSON implementation of DropConstraintMessage - */ -public class JSONDropConstraintMessage extends DropConstraintMessage { - - @JsonProperty - String server, servicePrincipal, dbName, tableName, constraintName; - - @JsonProperty - Long timestamp; - - /** - * Default constructor, needed for Jackson. - */ - public JSONDropConstraintMessage() { - } - - public JSONDropConstraintMessage(String server, String servicePrincipal, String dbName, - String tableName, String constraintName, Long timestamp) { - this.server = server; - this.servicePrincipal = servicePrincipal; - this.timestamp = timestamp; - this.dbName = dbName; - this.tableName = tableName; - this.constraintName = constraintName; - } - - @Override - public String getServer() { - return server; - } - - @Override - public String getServicePrincipal() { - return servicePrincipal; - } - - @Override - public String getDB() { - return dbName; - } - - @Override - public String getTable() { - return tableName; - } - - @Override - public String getConstraint() { - return constraintName; - } - - @Override - public Long getTimestamp() { - return timestamp; - } - - @Override - public String toString() { - try { - return JSONMessageDeserializer.mapper.writeValueAsString(this); - } catch (Exception exception) { - throw new IllegalArgumentException("Could not serialize: ", exception); - } - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropDatabaseMessage.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropDatabaseMessage.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropDatabaseMessage.java deleted file mode 100644 index d2a75bf..0000000 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropDatabaseMessage.java +++ /dev/null @@ -1,72 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hive.metastore.messaging.json; - -import org.apache.hadoop.hive.metastore.messaging.DropDatabaseMessage; - -import com.fasterxml.jackson.annotation.JsonProperty; - -/** - * JSON implementation of DropDatabaseMessage. - */ -public class JSONDropDatabaseMessage extends DropDatabaseMessage { - - @JsonProperty - String server, servicePrincipal, db; - - @JsonProperty - Long timestamp; - - /** - * Default constructor, required for Jackson. - */ - public JSONDropDatabaseMessage() {} - - public JSONDropDatabaseMessage(String server, String servicePrincipal, String db, Long timestamp) { - this.server = server; - this.servicePrincipal = servicePrincipal; - this.db = db; - this.timestamp = timestamp; - checkValid(); - } - - - @Override - public String getServer() { return server; } - - @Override - public String getServicePrincipal() { return servicePrincipal; } - - @Override - public String getDB() { return db; } - - @Override - public Long getTimestamp() { return timestamp; } - - @Override - public String toString() { - try { - return JSONMessageDeserializer.mapper.writeValueAsString(this); - } - catch (Exception exception) { - throw new IllegalArgumentException("Could not serialize: ", exception); - } - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropFunctionMessage.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropFunctionMessage.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropFunctionMessage.java deleted file mode 100644 index a7a7a1f..0000000 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropFunctionMessage.java +++ /dev/null @@ -1,79 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hive.metastore.messaging.json; - -import org.apache.hadoop.hive.metastore.api.Function; -import org.apache.hadoop.hive.metastore.messaging.DropFunctionMessage; - -import com.fasterxml.jackson.annotation.JsonProperty; - -/** - * JSON Implementation of CreateDatabaseMessage. - */ -public class JSONDropFunctionMessage extends DropFunctionMessage { - - @JsonProperty - String server, servicePrincipal, db, functionName; - - @JsonProperty - Long timestamp; - - /** - * Default constructor, required for Jackson. - */ - public JSONDropFunctionMessage() {} - - public JSONDropFunctionMessage(String server, String servicePrincipal, Function fn, Long timestamp) { - this.server = server; - this.servicePrincipal = servicePrincipal; - this.db = fn.getDbName(); - this.functionName = fn.getFunctionName(); - this.timestamp = timestamp; - checkValid(); - } - - @Override - public String getDB() { return db; } - - @Override - public String getServer() { return server; } - - @Override - public String getServicePrincipal() { return servicePrincipal; } - - @Override - public Long getTimestamp() { return timestamp; } - - @Override - public String toString() { - try { - return JSONMessageDeserializer.mapper.writeValueAsString(this); - } - catch (Exception exception) { - throw new IllegalArgumentException("Could not serialize: ", exception); - } - } - - @Override - public String getFunctionName() { - return functionName; - } - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropPartitionMessage.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropPartitionMessage.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropPartitionMessage.java deleted file mode 100644 index 957d595..0000000 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropPartitionMessage.java +++ /dev/null @@ -1,135 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hive.metastore.messaging.json; - -import java.util.List; -import java.util.Map; - -import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.hive.metastore.messaging.DropPartitionMessage; -import org.apache.thrift.TException; - -import com.fasterxml.jackson.annotation.JsonProperty; - -/** - * JSON implementation of DropPartitionMessage. - */ -public class JSONDropPartitionMessage extends DropPartitionMessage { - - @JsonProperty - String server, servicePrincipal, db, table, tableType, tableObjJson; - - @JsonProperty - Long timestamp; - - @JsonProperty - List<Map<String, String>> partitions; - - /** - * Default Constructor. Required for Jackson. - */ - public JSONDropPartitionMessage() { - } - - public JSONDropPartitionMessage(String server, String servicePrincipal, String db, String table, - List<Map<String, String>> partitions, Long timestamp) { - this(server, servicePrincipal, db, table, null, partitions, timestamp); - } - - public JSONDropPartitionMessage(String server, String servicePrincipal, String db, String table, - String tableType, List<Map<String, String>> partitions, Long timestamp) { - this.server = server; - this.servicePrincipal = servicePrincipal; - this.db = db; - this.table = table; - this.tableType = tableType; - this.partitions = partitions; - this.timestamp = timestamp; - checkValid(); - } - - public JSONDropPartitionMessage(String server, String servicePrincipal, Table tableObj, - List<Map<String, String>> partitionKeyValues, long timestamp) { - this(server, servicePrincipal, tableObj.getDbName(), tableObj.getTableName(), - tableObj.getTableType(), partitionKeyValues, timestamp); - try { - this.tableObjJson = JSONMessageFactory.createTableObjJson(tableObj); - } catch (TException e) { - throw new IllegalArgumentException("Could not serialize: ", e); - } - } - - @Override - public String getServer() { - return server; - } - - @Override - public String getServicePrincipal() { - return servicePrincipal; - } - - @Override - public String getDB() { - return db; - } - - @Override - public String getTable() { - return table; - } - - @Override - public String getTableType() { - if (tableType != null) { - return tableType; - } else { - return ""; - } - } - - @Override - public Long getTimestamp() { - return timestamp; - } - - @Override - public List<Map<String, String>> getPartitions() { - return partitions; - } - - @Override - public Table getTableObj() throws Exception { - return (Table) JSONMessageFactory.getTObj(tableObjJson, Table.class); - } - - public String getTableObjJson() { - return tableObjJson; - } - - @Override - public String toString() { - try { - return JSONMessageDeserializer.mapper.writeValueAsString(this); - } catch (Exception exception) { - throw new IllegalArgumentException("Could not serialize: ", exception); - } - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropTableMessage.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropTableMessage.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropTableMessage.java deleted file mode 100644 index 88374ec..0000000 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONDropTableMessage.java +++ /dev/null @@ -1,121 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hive.metastore.messaging.json; - -import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.hive.metastore.messaging.DropTableMessage; -import org.apache.thrift.TException; - -import com.fasterxml.jackson.annotation.JsonProperty; - -/** - * JSON implementation of DropTableMessage. - */ -public class JSONDropTableMessage extends DropTableMessage { - - @JsonProperty - String server, servicePrincipal, db, table, tableType, tableObjJson; - - @JsonProperty - Long timestamp; - - /** - * Default constructor, needed for Jackson. - */ - public JSONDropTableMessage() { - } - - public JSONDropTableMessage(String server, String servicePrincipal, String db, String table, - Long timestamp) { - this(server, servicePrincipal, db, table, null, timestamp); - } - - public JSONDropTableMessage(String server, String servicePrincipal, String db, String table, - String tableType, Long timestamp) { - this.server = server; - this.servicePrincipal = servicePrincipal; - this.db = db; - this.table = table; - this.tableType = tableType; - this.timestamp = timestamp; - checkValid(); - } - - public JSONDropTableMessage(String server, String servicePrincipal, Table tableObj, - Long timestamp) { - this(server, servicePrincipal, tableObj.getDbName(), tableObj.getTableName(), - tableObj.getTableType(), timestamp); - try { - this.tableObjJson = JSONMessageFactory.createTableObjJson(tableObj); - } catch (TException e) { - throw new IllegalArgumentException("Could not serialize: ", e); - } - checkValid(); - } - - @Override - public String getTable() { - return table; - } - - @Override - public String getTableType() { - if (tableType != null) { - return tableType; - } else { - return ""; - } - } - - @Override - public Table getTableObj() throws Exception { - return (Table) JSONMessageFactory.getTObj(tableObjJson,Table.class); - } - - @Override - public String getServer() { - return server; - } - - @Override - public String getServicePrincipal() { - return servicePrincipal; - } - - @Override - public String getDB() { - return db; - } - - @Override - public Long getTimestamp() { - return timestamp; - } - - @Override - public String toString() { - try { - return JSONMessageDeserializer.mapper.writeValueAsString(this); - } catch (Exception exception) { - throw new IllegalArgumentException("Could not serialize: ", exception); - } - } - -} http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONInsertMessage.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONInsertMessage.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONInsertMessage.java deleted file mode 100644 index 2318a67..0000000 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONInsertMessage.java +++ /dev/null @@ -1,148 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hive.metastore.messaging.json; - -import java.util.Iterator; -import java.util.List; - -import org.apache.hadoop.hive.metastore.api.Partition; -import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.hive.metastore.messaging.InsertMessage; -import org.apache.thrift.TException; - -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.collect.Lists; - -/** - * JSON implementation of InsertMessage - */ -public class JSONInsertMessage extends InsertMessage { - - @JsonProperty - String server, servicePrincipal, db, table, tableType, tableObjJson, ptnObjJson; - - @JsonProperty - Long timestamp; - - @JsonProperty - String replace; - - @JsonProperty - List<String> files; - - /** - * Default constructor, needed for Jackson. - */ - public JSONInsertMessage() { - } - - public JSONInsertMessage(String server, String servicePrincipal, Table tableObj, Partition ptnObj, - boolean replace, Iterator<String> fileIter, Long timestamp) { - this.server = server; - this.servicePrincipal = servicePrincipal; - - if (null == tableObj) { - throw new IllegalArgumentException("Table not valid."); - } - - this.db = tableObj.getDbName(); - this.table = tableObj.getTableName(); - this.tableType = tableObj.getTableType(); - - try { - this.tableObjJson = JSONMessageFactory.createTableObjJson(tableObj); - if (null != ptnObj) { - this.ptnObjJson = JSONMessageFactory.createPartitionObjJson(ptnObj); - } else { - this.ptnObjJson = null; - } - } catch (TException e) { - throw new IllegalArgumentException("Could not serialize: ", e); - } - - this.timestamp = timestamp; - this.replace = Boolean.toString(replace); - this.files = Lists.newArrayList(fileIter); - - checkValid(); - } - - @Override - public String getTable() { - return table; - } - - @Override - public String getTableType() { - if (tableType != null) { - return tableType; - } else { - return ""; - } - } - - @Override - public String getServer() { - return server; - } - - @Override - public Iterable<String> getFiles() { - return files; - } - - @Override - public String getServicePrincipal() { - return servicePrincipal; - } - - @Override - public String getDB() { - return db; - } - - @Override - public Long getTimestamp() { - return timestamp; - } - - @Override - public boolean isReplace() { return Boolean.parseBoolean(replace); } - - @Override - public Table getTableObj() throws Exception { - return (Table) JSONMessageFactory.getTObj(tableObjJson,Table.class); - } - - @Override - public Partition getPtnObj() throws Exception { - return ((null == ptnObjJson) ? null : (Partition) JSONMessageFactory.getTObj(ptnObjJson, Partition.class)); - } - - @Override - public String toString() { - try { - return JSONMessageDeserializer.mapper.writeValueAsString(this); - } catch (Exception exception) { - throw new IllegalArgumentException("Could not serialize: ", exception); - } - } - -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializer.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializer.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializer.java deleted file mode 100644 index 3a480aa..0000000 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageDeserializer.java +++ /dev/null @@ -1,273 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hive.metastore.messaging.json; - -import org.apache.hadoop.hive.metastore.messaging.AbortTxnMessage; -import org.apache.hadoop.hive.metastore.messaging.AddForeignKeyMessage; -import org.apache.hadoop.hive.metastore.messaging.AddNotNullConstraintMessage; -import org.apache.hadoop.hive.metastore.messaging.AddPartitionMessage; -import org.apache.hadoop.hive.metastore.messaging.AddPrimaryKeyMessage; -import org.apache.hadoop.hive.metastore.messaging.AddUniqueConstraintMessage; -import org.apache.hadoop.hive.metastore.messaging.AllocWriteIdMessage; -import org.apache.hadoop.hive.metastore.messaging.AlterDatabaseMessage; -import org.apache.hadoop.hive.metastore.messaging.AlterPartitionMessage; -import org.apache.hadoop.hive.metastore.messaging.AlterTableMessage; -import org.apache.hadoop.hive.metastore.messaging.CommitTxnMessage; -import org.apache.hadoop.hive.metastore.messaging.CreateDatabaseMessage; -import org.apache.hadoop.hive.metastore.messaging.CreateFunctionMessage; -import org.apache.hadoop.hive.metastore.messaging.CreateTableMessage; -import org.apache.hadoop.hive.metastore.messaging.DropConstraintMessage; -import org.apache.hadoop.hive.metastore.messaging.DropDatabaseMessage; -import org.apache.hadoop.hive.metastore.messaging.DropFunctionMessage; -import org.apache.hadoop.hive.metastore.messaging.DropPartitionMessage; -import org.apache.hadoop.hive.metastore.messaging.DropTableMessage; -import org.apache.hadoop.hive.metastore.messaging.InsertMessage; -import org.apache.hadoop.hive.metastore.messaging.MessageDeserializer; -import org.apache.hadoop.hive.metastore.messaging.OpenTxnMessage; -import org.apache.hadoop.hive.metastore.messaging.AcidWriteMessage; -import com.fasterxml.jackson.databind.DeserializationFeature; -import com.fasterxml.jackson.databind.MapperFeature; -import com.fasterxml.jackson.databind.ObjectMapper; - -/** - * MessageDeserializer implementation, for deserializing from JSON strings. - */ -public class JSONMessageDeserializer extends MessageDeserializer { - - static ObjectMapper mapper = new ObjectMapper(); // Thread-safe. - - static { - mapper.configure(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES, false); - mapper.configure(MapperFeature.AUTO_DETECT_GETTERS, false); - mapper.configure(MapperFeature.AUTO_DETECT_IS_GETTERS, false); - mapper.configure(MapperFeature.AUTO_DETECT_FIELDS, false); - } - - @Override - public CreateDatabaseMessage getCreateDatabaseMessage(String messageBody) { - try { - return mapper.readValue(messageBody, JSONCreateDatabaseMessage.class); - } - catch (Exception exception) { - throw new IllegalArgumentException("Could not construct JSONCreateDatabaseMessage.", - exception); - } - } - - @Override - public AlterDatabaseMessage getAlterDatabaseMessage(String messageBody) { - try { - return mapper.readValue(messageBody, JSONAlterDatabaseMessage.class); - } catch (Exception exception) { - throw new IllegalArgumentException("Could not construct JSONAlterDatabaseMessage.", - exception); - } - } - - @Override - public DropDatabaseMessage getDropDatabaseMessage(String messageBody) { - try { - return mapper.readValue(messageBody, JSONDropDatabaseMessage.class); - } - catch (Exception exception) { - throw new IllegalArgumentException("Could not construct JSONDropDatabaseMessage.", exception); - } - } - - @Override - public CreateTableMessage getCreateTableMessage(String messageBody) { - try { - return mapper.readValue(messageBody, JSONCreateTableMessage.class); - } - catch (Exception exception) { - throw new IllegalArgumentException("Could not construct JSONCreateTableMessage.", exception); - } - } - - @Override - public AlterTableMessage getAlterTableMessage(String messageBody) { - try { - return mapper.readValue(messageBody, JSONAlterTableMessage.class); - } - catch (Exception exception) { - throw new IllegalArgumentException("Could not construct appropriate alter table type.", - exception); - } - } - - @Override - public DropTableMessage getDropTableMessage(String messageBody) { - try { - return mapper.readValue(messageBody, JSONDropTableMessage.class); - } - catch (Exception exception) { - throw new IllegalArgumentException("Could not construct JSONDropTableMessage.", exception); - } - } - - @Override - public AddPartitionMessage getAddPartitionMessage(String messageBody) { - try { - return mapper.readValue(messageBody, JSONAddPartitionMessage.class); - } - catch (Exception exception) { - throw new IllegalArgumentException("Could not construct AddPartitionMessage.", exception); - } - } - - @Override - public AlterPartitionMessage getAlterPartitionMessage(String messageBody) { - try { - return mapper.readValue(messageBody, JSONAlterPartitionMessage.class); - } catch (Exception e) { - throw new IllegalArgumentException("Could not construct AlterPartitionMessage.", e); - } - } - - @Override - public DropPartitionMessage getDropPartitionMessage(String messageBody) { - try { - return mapper.readValue(messageBody, JSONDropPartitionMessage.class); - } - catch (Exception exception) { - throw new IllegalArgumentException("Could not construct DropPartitionMessage.", exception); - } - } - - @Override - public CreateFunctionMessage getCreateFunctionMessage(String messageBody) { - try { - return mapper.readValue(messageBody, JSONCreateFunctionMessage.class); - } - catch (Exception exception) { - throw new IllegalArgumentException("Could not construct JSONCreateFunctionMessage.", - exception); - } - } - - @Override - public DropFunctionMessage getDropFunctionMessage(String messageBody) { - try { - return mapper.readValue(messageBody, JSONDropFunctionMessage.class); - } - catch (Exception exception) { - throw new IllegalArgumentException("Could not construct JSONDropDatabaseMessage.", exception); - } - } - - @Override - public InsertMessage getInsertMessage(String messageBody) { - try { - return mapper.readValue(messageBody, JSONInsertMessage.class); - } catch (Exception e) { - throw new IllegalArgumentException("Could not construct InsertMessage", e); - } - } - - @Override - public AddPrimaryKeyMessage getAddPrimaryKeyMessage(String messageBody) { - try { - return mapper.readValue(messageBody, JSONAddPrimaryKeyMessage.class); - } catch (Exception e) { - throw new IllegalArgumentException("Could not construct AddPrimaryKeyMessage", e); - } - } - - @Override - public AddForeignKeyMessage getAddForeignKeyMessage(String messageBody) { - try { - return mapper.readValue(messageBody, JSONAddForeignKeyMessage.class); - } catch (Exception e) { - throw new IllegalArgumentException("Could not construct AddForeignKeyMessage", e); - } - } - - @Override - public AddUniqueConstraintMessage getAddUniqueConstraintMessage(String messageBody) { - try { - return mapper.readValue(messageBody, JSONAddUniqueConstraintMessage.class); - } catch (Exception e) { - throw new IllegalArgumentException("Could not construct AddUniqueConstraintMessage", e); - } - } - - @Override - public AddNotNullConstraintMessage getAddNotNullConstraintMessage(String messageBody) { - try { - return mapper.readValue(messageBody, JSONAddNotNullConstraintMessage.class); - } catch (Exception e) { - throw new IllegalArgumentException("Could not construct AddNotNullConstraintMessage", e); - } - } - - @Override - public DropConstraintMessage getDropConstraintMessage(String messageBody) { - try { - return mapper.readValue(messageBody, JSONDropConstraintMessage.class); - } catch (Exception e) { - throw new IllegalArgumentException("Could not construct DropConstraintMessage", e); - } - } - - @Override - public OpenTxnMessage getOpenTxnMessage(String messageBody) { - try { - return mapper.readValue(messageBody, JSONOpenTxnMessage.class); - } catch (Exception e) { - throw new IllegalArgumentException("Could not construct OpenTxnMessage", e); - } - } - - @Override - public CommitTxnMessage getCommitTxnMessage(String messageBody) { - try { - return mapper.readValue(messageBody, JSONCommitTxnMessage.class); - } catch (Exception e) { - throw new IllegalArgumentException("Could not construct CommitTxnMessage", e); - } - } - - @Override - public AbortTxnMessage getAbortTxnMessage(String messageBody) { - try { - return mapper.readValue(messageBody, JSONAbortTxnMessage.class); - } catch (Exception e) { - throw new IllegalArgumentException("Could not construct AbortTxnMessage", e); - } - } - - @Override - public AllocWriteIdMessage getAllocWriteIdMessage(String messageBody) { - try { - return mapper.readValue(messageBody, JSONAllocWriteIdMessage.class); - } catch (Exception e) { - throw new IllegalArgumentException("Could not construct AllocWriteIdMessage", e); - } - } - - @Override - public AcidWriteMessage getAcidWriteMessage(String messageBody) { - try { - return mapper.readValue(messageBody, JSONAcidWriteMessage.class); - } catch (Exception e) { - throw new IllegalArgumentException("Could not construct AcidWriteMessage", e); - } - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageFactory.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageFactory.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageFactory.java deleted file mode 100644 index 6412674..0000000 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONMessageFactory.java +++ /dev/null @@ -1,402 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hive.metastore.messaging.json; - -import java.util.Iterator; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -import javax.annotation.Nullable; - -import org.apache.hadoop.hive.metastore.api.Catalog; -import org.apache.hadoop.hive.metastore.api.Database; -import org.apache.hadoop.hive.metastore.api.Function; -import org.apache.hadoop.hive.metastore.api.NotificationEvent; -import org.apache.hadoop.hive.metastore.api.Partition; -import org.apache.hadoop.hive.metastore.api.SQLForeignKey; -import org.apache.hadoop.hive.metastore.api.SQLNotNullConstraint; -import org.apache.hadoop.hive.metastore.api.SQLPrimaryKey; -import org.apache.hadoop.hive.metastore.api.SQLUniqueConstraint; -import org.apache.hadoop.hive.metastore.api.Table; -import org.apache.hadoop.hive.metastore.api.TxnToWriteId; -import org.apache.hadoop.hive.metastore.events.AcidWriteEvent; -import org.apache.hadoop.hive.metastore.messaging.AbortTxnMessage; -import org.apache.hadoop.hive.metastore.messaging.AddForeignKeyMessage; -import org.apache.hadoop.hive.metastore.messaging.AddNotNullConstraintMessage; -import org.apache.hadoop.hive.metastore.messaging.AddPartitionMessage; -import org.apache.hadoop.hive.metastore.messaging.AddPrimaryKeyMessage; -import org.apache.hadoop.hive.metastore.messaging.AddUniqueConstraintMessage; -import org.apache.hadoop.hive.metastore.messaging.AllocWriteIdMessage; -import org.apache.hadoop.hive.metastore.messaging.AlterCatalogMessage; -import org.apache.hadoop.hive.metastore.messaging.AlterDatabaseMessage; -import org.apache.hadoop.hive.metastore.messaging.AlterPartitionMessage; -import org.apache.hadoop.hive.metastore.messaging.AlterTableMessage; -import org.apache.hadoop.hive.metastore.messaging.CommitTxnMessage; -import org.apache.hadoop.hive.metastore.messaging.CreateCatalogMessage; -import org.apache.hadoop.hive.metastore.messaging.CreateDatabaseMessage; -import org.apache.hadoop.hive.metastore.messaging.CreateFunctionMessage; -import org.apache.hadoop.hive.metastore.messaging.CreateTableMessage; -import org.apache.hadoop.hive.metastore.messaging.DropCatalogMessage; -import org.apache.hadoop.hive.metastore.messaging.DropConstraintMessage; -import org.apache.hadoop.hive.metastore.messaging.DropDatabaseMessage; -import org.apache.hadoop.hive.metastore.messaging.DropFunctionMessage; -import org.apache.hadoop.hive.metastore.messaging.DropPartitionMessage; -import org.apache.hadoop.hive.metastore.messaging.DropTableMessage; -import org.apache.hadoop.hive.metastore.messaging.InsertMessage; -import org.apache.hadoop.hive.metastore.messaging.MessageDeserializer; -import org.apache.hadoop.hive.metastore.messaging.MessageFactory; -import org.apache.hadoop.hive.metastore.messaging.OpenTxnMessage; -import org.apache.hadoop.hive.metastore.messaging.AcidWriteMessage; -import org.apache.hadoop.hive.metastore.messaging.PartitionFiles; -import org.apache.thrift.TBase; -import org.apache.thrift.TDeserializer; -import org.apache.thrift.TException; -import org.apache.thrift.TSerializer; -import org.apache.thrift.protocol.TJSONProtocol; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import com.fasterxml.jackson.core.JsonFactory; -import com.fasterxml.jackson.core.JsonParser; -import com.fasterxml.jackson.databind.JsonNode; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.node.ObjectNode; -import com.google.common.collect.Iterables; -import com.google.common.collect.Iterators; -import com.google.common.collect.Lists; - -/** - * The JSON implementation of the MessageFactory. Constructs JSON implementations of each - * message-type. - */ -public class JSONMessageFactory extends MessageFactory { - - private static final Logger LOG = LoggerFactory.getLogger(JSONMessageFactory.class.getName()); - - private static JSONMessageDeserializer deserializer = new JSONMessageDeserializer(); - - @Override - public MessageDeserializer getDeserializer() { - return deserializer; - } - - @Override - public String getMessageFormat() { - return "json-0.2"; - } - - @Override - public CreateDatabaseMessage buildCreateDatabaseMessage(Database db) { - return new JSONCreateDatabaseMessage(MS_SERVER_URL, MS_SERVICE_PRINCIPAL, db, now()); - } - - @Override - public AlterDatabaseMessage buildAlterDatabaseMessage(Database beforeDb, Database afterDb) { - return new JSONAlterDatabaseMessage(MS_SERVER_URL, MS_SERVICE_PRINCIPAL, - beforeDb, afterDb, now()); - } - - @Override - public DropDatabaseMessage buildDropDatabaseMessage(Database db) { - return new JSONDropDatabaseMessage(MS_SERVER_URL, MS_SERVICE_PRINCIPAL, db.getName(), now()); - } - - @Override - public CreateTableMessage buildCreateTableMessage(Table table, Iterator<String> fileIter) { - return new JSONCreateTableMessage(MS_SERVER_URL, MS_SERVICE_PRINCIPAL, table, fileIter, now()); - } - - @Override - public AlterTableMessage buildAlterTableMessage(Table before, Table after, boolean isTruncateOp) { - return new JSONAlterTableMessage(MS_SERVER_URL, MS_SERVICE_PRINCIPAL, - before, after, isTruncateOp, now()); - } - - @Override - public DropTableMessage buildDropTableMessage(Table table) { - return new JSONDropTableMessage(MS_SERVER_URL, MS_SERVICE_PRINCIPAL, table, now()); - } - - @Override - public AddPartitionMessage buildAddPartitionMessage(Table table, - Iterator<Partition> partitionsIterator, Iterator<PartitionFiles> partitionFileIter) { - return new JSONAddPartitionMessage(MS_SERVER_URL, MS_SERVICE_PRINCIPAL, table, - partitionsIterator, partitionFileIter, now()); - } - - @Override - public AlterPartitionMessage buildAlterPartitionMessage(Table table, Partition before, - Partition after, boolean isTruncateOp) { - return new JSONAlterPartitionMessage(MS_SERVER_URL, MS_SERVICE_PRINCIPAL, - table, before, after, isTruncateOp, now()); - } - - @Override - public DropPartitionMessage buildDropPartitionMessage(Table table, - Iterator<Partition> partitionsIterator) { - return new JSONDropPartitionMessage(MS_SERVER_URL, MS_SERVICE_PRINCIPAL, table, - getPartitionKeyValues(table, partitionsIterator), now()); - } - - @Override - public CreateFunctionMessage buildCreateFunctionMessage(Function fn) { - return new JSONCreateFunctionMessage(MS_SERVER_URL, MS_SERVICE_PRINCIPAL, fn, now()); - } - - @Override - public DropFunctionMessage buildDropFunctionMessage(Function fn) { - return new JSONDropFunctionMessage(MS_SERVER_URL, MS_SERVICE_PRINCIPAL, fn, now()); - } - - @Override - public InsertMessage buildInsertMessage(Table tableObj, Partition partObj, - boolean replace, Iterator<String> fileIter) { - return new JSONInsertMessage(MS_SERVER_URL, MS_SERVICE_PRINCIPAL, - tableObj, partObj, replace, fileIter, now()); - } - - @Override - public AddPrimaryKeyMessage buildAddPrimaryKeyMessage(List<SQLPrimaryKey> pks) { - return new JSONAddPrimaryKeyMessage(MS_SERVER_URL, MS_SERVICE_PRINCIPAL, pks, now()); - } - - @Override - public AddForeignKeyMessage buildAddForeignKeyMessage(List<SQLForeignKey> fks) { - return new JSONAddForeignKeyMessage(MS_SERVER_URL, MS_SERVICE_PRINCIPAL, fks, now()); - } - - @Override - public AddUniqueConstraintMessage buildAddUniqueConstraintMessage(List<SQLUniqueConstraint> uks) { - return new JSONAddUniqueConstraintMessage(MS_SERVER_URL, MS_SERVICE_PRINCIPAL, uks, now()); - } - - @Override - public AddNotNullConstraintMessage buildAddNotNullConstraintMessage(List<SQLNotNullConstraint> nns) { - return new JSONAddNotNullConstraintMessage(MS_SERVER_URL, MS_SERVICE_PRINCIPAL, nns, now()); - } - - @Override - public DropConstraintMessage buildDropConstraintMessage(String dbName, String tableName, - String constraintName) { - return new JSONDropConstraintMessage(MS_SERVER_URL, MS_SERVICE_PRINCIPAL, dbName, tableName, - constraintName, now()); - } - - @Override - public CreateCatalogMessage buildCreateCatalogMessage(Catalog catalog) { - return new JSONCreateCatalogMessage(MS_SERVER_URL, MS_SERVICE_PRINCIPAL, catalog.getName(), now()); - } - - @Override - public AlterCatalogMessage buildAlterCatalogMessage(Catalog beforeCat, Catalog afterCat) { - return new JSONAlterCatalogMessage(MS_SERVER_URL, MS_SERVICE_PRINCIPAL, - beforeCat, afterCat, now()); - } - - @Override - public DropCatalogMessage buildDropCatalogMessage(Catalog catalog) { - return new JSONDropCatalogMessage(MS_SERVER_URL, MS_SERVICE_PRINCIPAL, catalog.getName(), now()); - } - - @Override - public OpenTxnMessage buildOpenTxnMessage(Long fromTxnId, Long toTxnId) { - return new JSONOpenTxnMessage(MS_SERVER_URL, MS_SERVICE_PRINCIPAL, fromTxnId, toTxnId, now()); - } - - @Override - public CommitTxnMessage buildCommitTxnMessage(Long txnId) { - return new JSONCommitTxnMessage(MS_SERVER_URL, MS_SERVICE_PRINCIPAL, txnId, now()); - } - - @Override - public AbortTxnMessage buildAbortTxnMessage(Long txnId) { - return new JSONAbortTxnMessage(MS_SERVER_URL, MS_SERVICE_PRINCIPAL, txnId, now()); - } - - @Override - public AllocWriteIdMessage buildAllocWriteIdMessage(List<TxnToWriteId> txnToWriteIdList, - String dbName, String tableName) { - return new JSONAllocWriteIdMessage(MS_SERVER_URL, MS_SERVICE_PRINCIPAL, txnToWriteIdList, dbName, tableName, now()); - } - - @Override - public AcidWriteMessage buildAcidWriteMessage(AcidWriteEvent acidWriteEvent, Iterator<String> files) { - return new JSONAcidWriteMessage(MS_SERVER_URL, MS_SERVICE_PRINCIPAL, now(), acidWriteEvent, files); - } - - private long now() { - return System.currentTimeMillis() / 1000; - } - - static Map<String, String> getPartitionKeyValues(Table table, Partition partition) { - Map<String, String> partitionKeys = new LinkedHashMap<>(); - for (int i = 0; i < table.getPartitionKeysSize(); ++i) { - partitionKeys.put(table.getPartitionKeys().get(i).getName(), partition.getValues().get(i)); - } - return partitionKeys; - } - - static List<Map<String, String>> getPartitionKeyValues(final Table table, - Iterator<Partition> iterator) { - return Lists.newArrayList(Iterators.transform(iterator, - new com.google.common.base.Function<Partition, Map<String, String>>() { - @Override - public Map<String, String> apply(@Nullable Partition partition) { - return getPartitionKeyValues(table, partition); - } - })); - } - - static String createPrimaryKeyObjJson(SQLPrimaryKey primaryKeyObj) throws TException { - TSerializer serializer = new TSerializer(new TJSONProtocol.Factory()); - return serializer.toString(primaryKeyObj, "UTF-8"); - } - - static String createForeignKeyObjJson(SQLForeignKey foreignKeyObj) throws TException { - TSerializer serializer = new TSerializer(new TJSONProtocol.Factory()); - return serializer.toString(foreignKeyObj, "UTF-8"); - } - - static String createUniqueConstraintObjJson(SQLUniqueConstraint uniqueConstraintObj) throws TException { - TSerializer serializer = new TSerializer(new TJSONProtocol.Factory()); - return serializer.toString(uniqueConstraintObj, "UTF-8"); - } - - static String createNotNullConstraintObjJson(SQLNotNullConstraint notNullConstaintObj) throws TException { - TSerializer serializer = new TSerializer(new TJSONProtocol.Factory()); - return serializer.toString(notNullConstaintObj, "UTF-8"); - } - - static String createDatabaseObjJson(Database dbObj) throws TException { - TSerializer serializer = new TSerializer(new TJSONProtocol.Factory()); - return serializer.toString(dbObj, "UTF-8"); - } - - static String createCatalogObjJson(Catalog catObj) throws TException { - TSerializer serializer = new TSerializer(new TJSONProtocol.Factory()); - return serializer.toString(catObj, "UTF-8"); - } - - static String createTableObjJson(Table tableObj) throws TException { - TSerializer serializer = new TSerializer(new TJSONProtocol.Factory()); - return serializer.toString(tableObj, "UTF-8"); - } - - static String createPartitionObjJson(Partition partitionObj) throws TException { - TSerializer serializer = new TSerializer(new TJSONProtocol.Factory()); - return serializer.toString(partitionObj, "UTF-8"); - } - - static String createFunctionObjJson(Function functionObj) throws TException { - TSerializer serializer = new TSerializer(new TJSONProtocol.Factory()); - return serializer.toString(functionObj, "UTF-8"); - } - - public static ObjectNode getJsonTree(NotificationEvent event) throws Exception { - return getJsonTree(event.getMessage()); - } - - public static ObjectNode getJsonTree(String eventMessage) throws Exception { - JsonParser jsonParser = (new JsonFactory()).createJsonParser(eventMessage); - ObjectMapper mapper = new ObjectMapper(); - return mapper.readValue(jsonParser, ObjectNode.class); - } - - public static Table getTableObj(ObjectNode jsonTree) throws Exception { - TDeserializer deSerializer = new TDeserializer(new TJSONProtocol.Factory()); - Table tableObj = new Table(); - String tableJson = jsonTree.get("tableObjJson").asText(); - deSerializer.deserialize(tableObj, tableJson, "UTF-8"); - return tableObj; - } - - /* - * TODO: Some thoughts here : We have a current todo to move some of these methods over to - * MessageFactory instead of being here, so we can override them, but before we move them over, - * we should keep the following in mind: - * - * a) We should return Iterables, not Lists. That makes sure that we can be memory-safe when - * implementing it rather than forcing ourselves down a path wherein returning List is part of - * our interface, and then people use .size() or somesuch which makes us need to materialize - * the entire list and not change. Also, returning Iterables allows us to do things like - * Iterables.transform for some of these. - * b) We should not have "magic" names like "tableObjJson", because that breaks expectation of a - * couple of things - firstly, that of serialization format, although that is fine for this - * JSONMessageFactory, and secondly, that makes us just have a number of mappings, one for each - * obj type, and sometimes, as the case is with alter, have multiples. Also, any event-specific - * item belongs in that event message / event itself, as opposed to in the factory. It's okay to - * have utility accessor methods here that are used by each of the messages to provide accessors. - * I'm adding a couple of those here. - * - */ - - public static TBase getTObj(String tSerialized, Class<? extends TBase> objClass) throws Exception{ - TDeserializer thriftDeSerializer = new TDeserializer(new TJSONProtocol.Factory()); - TBase obj = objClass.newInstance(); - thriftDeSerializer.deserialize(obj, tSerialized, "UTF-8"); - return obj; - } - - public static Iterable<? extends TBase> getTObjs( - Iterable<String> objRefStrs, final Class<? extends TBase> objClass) throws Exception { - - try { - return Iterables.transform(objRefStrs, new com.google.common.base.Function<String,TBase>(){ - @Override - public TBase apply(@Nullable String objStr){ - try { - return getTObj(objStr, objClass); - } catch (Exception e) { - throw new RuntimeException(e); - } - } - }); - } catch (RuntimeException re){ - // We have to add this bit of exception handling here, because Function.apply does not allow us to throw - // the actual exception that might be a checked exception, so we wind up needing to throw a RuntimeException - // with the previously thrown exception as its cause. However, since RuntimeException.getCause() returns - // a throwable instead of an Exception, we have to account for the possibility that the underlying code - // might have thrown a Throwable that we wrapped instead, in which case, continuing to throw the - // RuntimeException is the best thing we can do. - Throwable t = re.getCause(); - if (t instanceof Exception){ - throw (Exception) t; - } else { - throw re; - } - } - } - - // If we do not need this format of accessor using ObjectNode, this is a candidate for removal as well - public static Iterable<? extends TBase> getTObjs( - ObjectNode jsonTree, String objRefListName, final Class<? extends TBase> objClass) throws Exception { - Iterable<JsonNode> jsonArrayIterator = jsonTree.get(objRefListName); - com.google.common.base.Function<JsonNode,String> textExtractor = - new com.google.common.base.Function<JsonNode, String>() { - @Nullable - @Override - public String apply(@Nullable JsonNode input) { - return input.asText(); - } - }; - return getTObjs(Iterables.transform(jsonArrayIterator, textExtractor), objClass); - } -} http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONOpenTxnMessage.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONOpenTxnMessage.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONOpenTxnMessage.java deleted file mode 100644 index 0c4615b..0000000 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/messaging/json/JSONOpenTxnMessage.java +++ /dev/null @@ -1,106 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.hadoop.hive.metastore.messaging.json; -import java.util.List; -import java.util.stream.Collectors; -import java.util.stream.LongStream; - -import org.apache.hadoop.hive.metastore.messaging.OpenTxnMessage; - -import com.fasterxml.jackson.annotation.JsonProperty; - -/** - * JSON implementation of OpenTxnMessage - */ -public class JSONOpenTxnMessage extends OpenTxnMessage { - - @JsonProperty - private List<Long> txnIds; - - @JsonProperty - private Long timestamp, fromTxnId, toTxnId; - - @JsonProperty - private String server; - - @JsonProperty - private String servicePrincipal; - - /** - * Default constructor, needed for Jackson. - */ - public JSONOpenTxnMessage() { - } - - public JSONOpenTxnMessage(String server, String servicePrincipal, List<Long> txnIds, Long timestamp) { - this.timestamp = timestamp; - this.txnIds = txnIds; - this.server = server; - this.servicePrincipal = servicePrincipal; - } - - public JSONOpenTxnMessage(String server, String servicePrincipal, Long fromTxnId, Long toTxnId, Long timestamp) { - this.timestamp = timestamp; - this.txnIds = null; - this.server = server; - this.servicePrincipal = servicePrincipal; - this.fromTxnId = fromTxnId; - this.toTxnId = toTxnId; - } - - @Override - public List<Long> getTxnIds() { - if (txnIds != null) { - return txnIds; - } - return LongStream.rangeClosed(fromTxnId, toTxnId) - .boxed().collect(Collectors.toList()); - } - - @Override - public Long getTimestamp() { - return timestamp; - } - - @Override - public String getDB() { - return null; - } - - @Override - public String getServicePrincipal() { - return servicePrincipal; - } - - @Override - public String getServer() { - return server; - } - - @Override - public String toString() { - try { - return JSONMessageDeserializer.mapper.writeValueAsString(this); - } catch (Exception exception) { - throw new IllegalArgumentException("Could not serialize: ", exception); - } - } -} - http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/metrics/JsonReporter.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/metrics/JsonReporter.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/metrics/JsonReporter.java deleted file mode 100644 index e198fbc..0000000 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/metrics/JsonReporter.java +++ /dev/null @@ -1,223 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * <p> - * http://www.apache.org/licenses/LICENSE-2.0 - * <p> - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.metastore.metrics; - -import com.codahale.metrics.Counter; -import com.codahale.metrics.Gauge; -import com.codahale.metrics.Histogram; -import com.codahale.metrics.Meter; -import com.codahale.metrics.MetricFilter; -import com.codahale.metrics.MetricRegistry; -import com.codahale.metrics.ScheduledReporter; -import com.codahale.metrics.Timer; -import com.codahale.metrics.json.MetricsModule; -import com.fasterxml.jackson.core.JsonProcessingException; -import com.fasterxml.jackson.databind.ObjectMapper; -import com.fasterxml.jackson.databind.ObjectWriter; -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.hive.metastore.conf.MetastoreConf; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.io.BufferedWriter; -import java.io.FileWriter; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.StandardCopyOption; -import java.nio.file.attribute.FileAttribute; -import java.nio.file.attribute.PosixFilePermission; -import java.nio.file.attribute.PosixFilePermissions; -import java.util.Set; -import java.util.SortedMap; -import java.util.concurrent.TimeUnit; - -/** - * A metrics reporter for Metrics that dumps metrics periodically into - * a file in JSON format. - */ -public class JsonReporter extends ScheduledReporter { - // - // Implementation notes. - // - // 1. Since only local file systems are supported, there is no need to use Hadoop - // version of Path class. - // 2. java.nio package provides modern implementation of file and directory operations - // which is better then the traditional java.io, so we are using it here. - // In particular, it supports atomic creation of temporary files with specified - // permissions in the specified directory. This also avoids various attacks possible - // when temp file name is generated first, followed by file creation. - // See http://www.oracle.com/technetwork/articles/javase/nio-139333.html for - // the description of NIO API and - // http://docs.oracle.com/javase/tutorial/essential/io/legacy.html for the - // description of interoperability between legacy IO api vs NIO API. - // 3. To avoid race conditions with readers of the metrics file, the implementation - // dumps metrics to a temporary file in the same directory as the actual metrics - // file and then renames it to the destination. Since both are located on the same - // filesystem, this rename is likely to be atomic (as long as the underlying OS - // support atomic renames. - // - // NOTE: This reporter is very similar to - // org.apache.hadoop.hive.common.metrics.metrics2.JsonFileMetricsReporter. - // org.apache.hadoop.hive.metastore.metrics.JsonReporter. - // It would be good to unify the two. - // - private static final Logger LOG = LoggerFactory.getLogger(JsonReporter.class); - - private static final FileAttribute<Set<PosixFilePermission>> FILE_ATTRS = - PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rw-r--r--")); - // Permissions for metric directory - private static final FileAttribute<Set<PosixFilePermission>> DIR_ATTRS = - PosixFilePermissions.asFileAttribute(PosixFilePermissions.fromString("rwxr-xr-x")); - - private final MetricRegistry registry; - private ObjectWriter jsonWriter; - // Location of JSON file - private final Path path; - // Directory where path resides - private final Path metricsDir; - - private JsonReporter(MetricRegistry registry, String name, MetricFilter filter, - TimeUnit rateUnit, TimeUnit durationUnit, Configuration conf) { - super(registry, name, filter, rateUnit, durationUnit); - String pathString = MetastoreConf.getVar(conf, MetastoreConf.ConfVars .METRICS_JSON_FILE_LOCATION); - path = Paths.get(pathString).toAbsolutePath(); - LOG.info("Reporting metrics to {}", path); - // We want to use metricsDir in the same directory as the destination file to support atomic - // move of temp file to the destination metrics file - metricsDir = path.getParent(); - this.registry = registry; - } - - @Override - public void start(long period, TimeUnit unit) { - // Create metrics directory if it is not present - if (!metricsDir.toFile().exists()) { - LOG.warn("Metrics directory {} does not exist, creating one", metricsDir); - try { - // createDirectories creates all non-existent parent directories - Files.createDirectories(metricsDir, DIR_ATTRS); - } catch (IOException e) { - LOG.warn("Failed to initialize JSON reporter: failed to create directory {}: {}", metricsDir, e.getMessage()); - return; - } - } - jsonWriter = new ObjectMapper().registerModule(new MetricsModule(TimeUnit.MILLISECONDS, - TimeUnit.MILLISECONDS, false)).writerWithDefaultPrettyPrinter(); - super.start(period, unit); - } - - @Override - public void report(SortedMap<String, Gauge> sortedMap, SortedMap<String, Counter> sortedMap1, - SortedMap<String, Histogram> sortedMap2, SortedMap<String, Meter> sortedMap3, - SortedMap<String, Timer> sortedMap4) { - - String json; - try { - json = jsonWriter.writeValueAsString(registry); - } catch (JsonProcessingException e) { - LOG.error("Unable to convert json to string ", e); - return; - } - - // Metrics are first dumped to a temp file which is then renamed to the destination - Path tmpFile = null; - try { - tmpFile = Files.createTempFile(metricsDir, "hmsmetrics", "json", FILE_ATTRS); - } catch (IOException e) { - LOG.error("failed to create temp file for JSON metrics", e); - return; - } catch (SecurityException e) { - // This shouldn't ever happen - LOG.error("failed to create temp file for JSON metrics: no permissions", e); - return; - } catch (UnsupportedOperationException e) { - // This shouldn't ever happen - LOG.error("failed to create temp file for JSON metrics: operartion not supported", e); - return; - } - - // Use try .. finally to cleanup temp file if something goes wrong - try { - // Write json to the temp file - try (BufferedWriter bw = new BufferedWriter(new FileWriter(tmpFile.toFile()))) { - bw.write(json); - } catch (IOException e) { - LOG.error("Unable to write to temp file {}" + tmpFile, e); - return; - } - - // Atomically move temp file to the destination file - try { - Files.move(tmpFile, path, StandardCopyOption.ATOMIC_MOVE); - } catch (Exception e) { - LOG.error("Unable to rename temp file {} to {}", tmpFile, path); - LOG.error("Exception during rename", e); - } - } finally { - // If something happened and we were not able to rename the temp file, attempt to remove it - if (tmpFile.toFile().exists()) { - // Attempt to delete temp file, if this fails, not much can be done about it. - try { - Files.delete(tmpFile); - } catch (Exception e) { - LOG.error("failed to delete temporary metrics file " + tmpFile, e); - } - } - } - } - - public static Builder forRegistry(MetricRegistry registry, Configuration conf) { - return new Builder(registry, conf); - } - - public static class Builder { - private final MetricRegistry registry; - private final Configuration conf; - private TimeUnit rate = TimeUnit.SECONDS; - private TimeUnit duration = TimeUnit.MILLISECONDS; - private MetricFilter filter = MetricFilter.ALL; - - private Builder(MetricRegistry registry, Configuration conf) { - this.registry = registry; - this.conf = conf; - } - - public Builder convertRatesTo(TimeUnit rate) { - this.rate = rate; - return this; - } - - public Builder convertDurationsTo(TimeUnit duration) { - this.duration = duration; - return this; - } - - public Builder filter(MetricFilter filter) { - this.filter = filter; - return this; - } - - public JsonReporter build() { - return new JsonReporter(registry, "json-reporter", filter, rate, duration, conf); - } - - } - -} http://git-wip-us.apache.org/repos/asf/hive/blob/081fa368/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/metrics/JvmPauseMonitor.java ---------------------------------------------------------------------- diff --git a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/metrics/JvmPauseMonitor.java b/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/metrics/JvmPauseMonitor.java deleted file mode 100644 index 9a9e91b..0000000 --- a/standalone-metastore/metastore-common/src/main/java/org/apache/hadoop/hive/metastore/metrics/JvmPauseMonitor.java +++ /dev/null @@ -1,222 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package org.apache.hadoop.hive.metastore.metrics; - -import com.codahale.metrics.Counter; -import com.google.common.base.Joiner; -import com.google.common.base.Preconditions; -import com.google.common.base.Stopwatch; -import com.google.common.collect.Lists; -import com.google.common.collect.Maps; -import com.google.common.collect.Sets; - -import org.apache.hadoop.conf.Configuration; -import org.apache.hadoop.util.Daemon; -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; - -import java.lang.management.GarbageCollectorMXBean; -import java.lang.management.ManagementFactory; -import java.util.List; -import java.util.Map; -import java.util.Set; -import java.util.concurrent.TimeUnit; - -/** - * Based on the JvmPauseMonitor from Hadoop. - */ -public class JvmPauseMonitor { - private static final Logger LOG = LoggerFactory.getLogger( - JvmPauseMonitor.class); - - /** The target sleep time */ - private static final long SLEEP_INTERVAL_MS = 500; - - /** log WARN if we detect a pause longer than this threshold */ - private final long warnThresholdMs; - private static final String WARN_THRESHOLD_KEY = - "jvm.pause.warn-threshold.ms"; - private static final long WARN_THRESHOLD_DEFAULT = 10000; - - /** log INFO if we detect a pause longer than this threshold */ - private final long infoThresholdMs; - private static final String INFO_THRESHOLD_KEY = - "jvm.pause.info-threshold.ms"; - private static final long INFO_THRESHOLD_DEFAULT = 1000; - - private long numGcWarnThresholdExceeded = 0; - private long numGcInfoThresholdExceeded = 0; - private long totalGcExtraSleepTime = 0; - - private Thread monitorThread; - private volatile boolean shouldRun = true; - - public JvmPauseMonitor(Configuration conf) { - this.warnThresholdMs = conf.getLong(WARN_THRESHOLD_KEY, WARN_THRESHOLD_DEFAULT); - this.infoThresholdMs = conf.getLong(INFO_THRESHOLD_KEY, INFO_THRESHOLD_DEFAULT); - } - - public void start() { - Preconditions.checkState(monitorThread == null, - "JvmPauseMonitor thread is Already started"); - monitorThread = new Daemon(new Monitor()); - monitorThread.start(); - } - - public void stop() { - shouldRun = false; - if (isStarted()) { - monitorThread.interrupt(); - try { - monitorThread.join(); - } catch (InterruptedException e) { - Thread.currentThread().interrupt(); - } - } - } - - public boolean isStarted() { - return monitorThread != null; - } - - public long getNumGcWarnThreadholdExceeded() { - return numGcWarnThresholdExceeded; - } - - public long getNumGcInfoThresholdExceeded() { - return numGcInfoThresholdExceeded; - } - - public long getTotalGcExtraSleepTime() { - return totalGcExtraSleepTime; - } - - private String formatMessage(long extraSleepTime, - Map<String, GcTimes> gcTimesAfterSleep, - Map<String, GcTimes> gcTimesBeforeSleep) { - - Set<String> gcBeanNames = Sets.intersection( - gcTimesAfterSleep.keySet(), - gcTimesBeforeSleep.keySet()); - List<String> gcDiffs = Lists.newArrayList(); - for (String name : gcBeanNames) { - GcTimes diff = gcTimesAfterSleep.get(name).subtract( - gcTimesBeforeSleep.get(name)); - if (diff.gcCount != 0) { - gcDiffs.add("GC pool '" + name + "' had collection(s): " + - diff.toString()); - } - } - - String ret = "Detected pause in JVM or host machine (eg GC): " + - "pause of approximately " + extraSleepTime + "ms\n"; - if (gcDiffs.isEmpty()) { - ret += "No GCs detected"; - } else { - ret += Joiner.on("\n").join(gcDiffs); - } - return ret; - } - - private Map<String, GcTimes> getGcTimes() { - Map<String, GcTimes> map = Maps.newHashMap(); - List<GarbageCollectorMXBean> gcBeans = - ManagementFactory.getGarbageCollectorMXBeans(); - for (GarbageCollectorMXBean gcBean : gcBeans) { - map.put(gcBean.getName(), new GcTimes(gcBean)); - } - return map; - } - - private static class GcTimes { - private GcTimes(GarbageCollectorMXBean gcBean) { - gcCount = gcBean.getCollectionCount(); - gcTimeMillis = gcBean.getCollectionTime(); - } - - private GcTimes(long count, long time) { - this.gcCount = count; - this.gcTimeMillis = time; - } - - private GcTimes subtract(GcTimes other) { - return new GcTimes(this.gcCount - other.gcCount, - this.gcTimeMillis - other.gcTimeMillis); - } - - @Override - public String toString() { - return "count=" + gcCount + " time=" + gcTimeMillis + "ms"; - } - - private final long gcCount; - private final long gcTimeMillis; - } - - private class Monitor implements Runnable { - @Override - public void run() { - Stopwatch sw = Stopwatch.createUnstarted(); - Map<String, GcTimes> gcTimesBeforeSleep = getGcTimes(); - Counter jvmPauseWarnCnt = Metrics.getOrCreateCounter(MetricsConstants.JVM_PAUSE_WARN); - Counter jvmPauseInfoCnt = Metrics.getOrCreateCounter(MetricsConstants.JVM_PAUSE_INFO); - Counter jvmExtraSleepCnt = Metrics.getOrCreateCounter(MetricsConstants.JVM_EXTRA_SLEEP); - while (shouldRun) { - sw.reset().start(); - try { - Thread.sleep(SLEEP_INTERVAL_MS); - } catch (InterruptedException ie) { - return; - } - long extraSleepTime = sw.elapsed(TimeUnit.MILLISECONDS) - SLEEP_INTERVAL_MS; - Map<String, GcTimes> gcTimesAfterSleep = getGcTimes(); - - if (extraSleepTime > warnThresholdMs) { - ++numGcWarnThresholdExceeded; - LOG.warn(formatMessage( - extraSleepTime, gcTimesAfterSleep, gcTimesBeforeSleep)); - if (jvmPauseWarnCnt != null) jvmPauseWarnCnt.inc(); - } else if (extraSleepTime > infoThresholdMs) { - ++numGcInfoThresholdExceeded; - LOG.info(formatMessage( - extraSleepTime, gcTimesAfterSleep, gcTimesBeforeSleep)); - if (jvmPauseInfoCnt != null) jvmPauseInfoCnt.inc(); - } - if (jvmExtraSleepCnt != null) jvmExtraSleepCnt.inc(extraSleepTime); - totalGcExtraSleepTime += extraSleepTime; - gcTimesBeforeSleep = gcTimesAfterSleep; - } - } - } - - /** - * Simple 'main' to facilitate manual testing of the pause monitor. - * - * This main function just leaks memory into a list. Running this class - * with a 1GB heap will very quickly go into "GC hell" and result in - * log messages about the GC pauses. - */ - public static void main(String []args) throws Exception { - new JvmPauseMonitor(new Configuration()).start(); - List<String> list = Lists.newArrayList(); - int i = 0; - while (true) { - list.add(String.valueOf(i++)); - } - } -}
