[
https://issues.apache.org/jira/browse/HIVE-27186?focusedWorklogId=858726&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-858726
]
ASF GitHub Bot logged work on HIVE-27186:
-----------------------------------------
Author: ASF GitHub Bot
Created on: 24/Apr/23 14:45
Start Date: 24/Apr/23 14:45
Worklog Time Spent: 10m
Work Description: henrib commented on code in PR #4194:
URL: https://github.com/apache/hive/pull/4194#discussion_r1175396173
##########
standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSPropertyStoreRemoteTest.java:
##########
@@ -0,0 +1,233 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.properties;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.jexl3.JxltEngine;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.HMSHandler;
+import org.apache.hadoop.hive.metastore.HiveMetaStore;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.JdoPropertyStore;
+import org.apache.hadoop.hive.metastore.MetaStoreTestUtils;
+import org.apache.hadoop.hive.metastore.ObjectStore;
+import org.apache.hadoop.hive.metastore.TestObjectStore;
+import org.apache.hadoop.hive.metastore.Warehouse;
+import org.apache.hadoop.hive.metastore.api.InvalidInputException;
+import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
+import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
+import org.apache.hadoop.hive.metastore.api.MaintenanceOpStatus;
+import org.apache.hadoop.hive.metastore.api.MaintenanceOpType;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.StringWriter;
+import java.util.Map;
+import java.util.TreeMap;
+
+import static
org.apache.hadoop.hive.metastore.properties.HMSPropertyManager.JEXL;
+import static
org.apache.hadoop.hive.metastore.properties.HMSPropertyManager.MAINTENANCE_OPERATION;
+import static
org.apache.hadoop.hive.metastore.properties.HMSPropertyManager.MAINTENANCE_STATUS;
+import static org.apache.hadoop.hive.metastore.properties.PropertyType.BOOLEAN;
+import static
org.apache.hadoop.hive.metastore.properties.PropertyType.DATETIME;
+import static org.apache.hadoop.hive.metastore.properties.PropertyType.DOUBLE;
+import static org.apache.hadoop.hive.metastore.properties.PropertyType.INTEGER;
+import static org.apache.hadoop.hive.metastore.properties.PropertyType.JSON;
+import static org.apache.hadoop.hive.metastore.properties.PropertyType.STRING;
+
+public class HMSPropertyStoreRemoteTest extends HMSPropertyStoreTest {
+ //private static final String NS = "hms";
+ protected HiveMetaStoreClient client;
+ private boolean isServerStarted = false;
+ protected int port;
+
+ boolean createStore(Configuration conf, Warehouse wh) {
+ try {
+ MetaStoreTestUtils.setConfForStandloneMode(conf);
+ objectStore = new ObjectStore();
+ objectStore.setConf(conf);
+ //TestObjectStore.dropAllStoreObjects(objectStore);
+ HMSHandler.createDefaultCatalog(objectStore, wh);
+ // configure object store
+ objectStore.createDatabase(new DatabaseBuilder()
+ .setCatalogName("hive")
+ .setName(DB1)
+ .setDescription("description")
+ .setLocation("locationurl")
+ .build(conf));
+ } catch(InvalidObjectException | MetaException | InvalidOperationException
xmeta) {
+ throw new PropertyException("unable to initialize server", xmeta);
+ }
+ return true;
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ conf = MetastoreConf.newMetastoreConf();
+ MetastoreConf.setBoolVar(conf, MetastoreConf.ConfVars.HIVE_IN_TEST, true);
+ // Events that get cleaned happen in batches of 1 to exercise batching code
+ MetastoreConf.setLongVar(conf,
MetastoreConf.ConfVars.EVENT_CLEAN_MAX_EVENTS, 1L);
+ MetaStoreTestUtils.setConfForStandloneMode(conf);
+ if (isServerStarted) {
+ Assert.assertNotNull("Unable to connect to the MetaStore server",
client);
+ return;
+ }
+ port =
MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge(),
conf);
+ System.out.println("Starting MetaStore Server on port " + port);
+ isServerStarted = true;
+
+ Warehouse wh = new Warehouse(conf);
+ boolean inited = createStore(conf, wh);
+ LOG.info("MetaStore Thrift Server test initialization " + (inited?
"successful":"failed"));
+ // This is default case with setugi off for both client and server
+ client = createClient();
+ }
+
+
+ protected HiveMetaStoreClient createClient() throws Exception {
+ MetastoreConf.setVar(conf, MetastoreConf.ConfVars.THRIFT_URIS,
"thrift://localhost:" + port);
+ MetastoreConf.setBoolVar(conf, MetastoreConf.ConfVars.EXECUTE_SET_UGI,
false);
+ HiveMetaStoreClient client = new HiveMetaStoreClient(conf);
+ return client;
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ MetaStoreTestUtils.close(port);
+ super.tearDown();
+ }
+
+ @Test
+ public void testHMSProperties() throws Exception {
Review Comment:
Refactored the unit tests (and added a Servlet skeleton to test another
transport).
Issue Time Tracking
-------------------
Worklog Id: (was: 858726)
Time Spent: 11h 40m (was: 11.5h)
> A persistent property store
> ----------------------------
>
> Key: HIVE-27186
> URL: https://issues.apache.org/jira/browse/HIVE-27186
> Project: Hive
> Issue Type: Improvement
> Components: Metastore
> Affects Versions: 4.0.0-alpha-2
> Reporter: Henri Biestro
> Assignee: Henri Biestro
> Priority: Major
> Labels: pull-request-available
> Time Spent: 11h 40m
> Remaining Estimate: 0h
>
> WHAT
> A persistent property store usable as a support facility for any metadata
> augmentation feature.
> WHY
> When adding new meta-data oriented features, we usually need to persist
> information linking the feature data and the HiveMetaStore objects it applies
> to. Any information related to a database, a table or the cluster - like
> statistics for example or any operational data state or data (think rolling
> backup) - fall in this use-case.
> Typically, accommodating such a feature requires modifying the Metastore
> database schema by adding or altering a table. It also usually implies
> modifying the thrift APIs to expose such meta-data to consumers.
> The proposed feature wants to solve the persistence and query/transport for
> these types of use-cases by exposing a 'key/(meta)value' store exposed as a
> property system.
> HOW
> A property-value model is the simple and generic exposed API.
> To provision for several usage scenarios, the model entry point is a
> 'namespace' that qualifies the feature-component property manager. For
> example, 'stats' could be the namespace for all properties related to the
> 'statistics' feature.
> The namespace identifies a manager that handles property-groups persisted as
> property-maps. For instance, all statistics pertaining to a given table would
> be collocated in the same property-group. As such, all properties (say number
> of 'unique_values' per columns) for a given HMS table 'relation0' would all
> be stored and persisted in the same property-map instance.
> Property-maps may be decorated by an (optional) schema that may declare the
> name and value-type of allowed properties (and their optional default value).
> Each property is addressed by a name, a path uniquely identifying the
> property in a given property map.
> The manager also handles transforming property-map names to the property-map
> keys used to persist them in the DB.
> The API provides inserting/updating properties in bulk transactionally. It
> also provides selection/projection to help reduce the volume of exchange
> between client/server; selection can use (JEXL expression) predicates to
> filter maps.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
