[
https://issues.apache.org/jira/browse/HIVE-27186?focusedWorklogId=858525&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-858525
]
ASF GitHub Bot logged work on HIVE-27186:
-----------------------------------------
Author: ASF GitHub Bot
Created on: 21/Apr/23 23:53
Start Date: 21/Apr/23 23:53
Worklog Time Spent: 10m
Work Description: saihemanth-cloudera commented on code in PR #4194:
URL: https://github.com/apache/hive/pull/4194#discussion_r1174228581
##########
standalone-metastore/metastore-server/src/test/java/org/apache/hadoop/hive/metastore/properties/HMSPropertyStoreRemoteTest.java:
##########
@@ -0,0 +1,233 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hive.metastore.properties;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.jexl3.JxltEngine;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hive.metastore.HMSHandler;
+import org.apache.hadoop.hive.metastore.HiveMetaStore;
+import org.apache.hadoop.hive.metastore.HiveMetaStoreClient;
+import org.apache.hadoop.hive.metastore.JdoPropertyStore;
+import org.apache.hadoop.hive.metastore.MetaStoreTestUtils;
+import org.apache.hadoop.hive.metastore.ObjectStore;
+import org.apache.hadoop.hive.metastore.TestObjectStore;
+import org.apache.hadoop.hive.metastore.Warehouse;
+import org.apache.hadoop.hive.metastore.api.InvalidInputException;
+import org.apache.hadoop.hive.metastore.api.InvalidObjectException;
+import org.apache.hadoop.hive.metastore.api.InvalidOperationException;
+import org.apache.hadoop.hive.metastore.api.MaintenanceOpStatus;
+import org.apache.hadoop.hive.metastore.api.MaintenanceOpType;
+import org.apache.hadoop.hive.metastore.api.MetaException;
+import org.apache.hadoop.hive.metastore.client.builder.DatabaseBuilder;
+import org.apache.hadoop.hive.metastore.conf.MetastoreConf;
+import org.apache.hadoop.hive.metastore.security.HadoopThriftAuthBridge;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.io.StringWriter;
+import java.util.Map;
+import java.util.TreeMap;
+
+import static
org.apache.hadoop.hive.metastore.properties.HMSPropertyManager.JEXL;
+import static
org.apache.hadoop.hive.metastore.properties.HMSPropertyManager.MAINTENANCE_OPERATION;
+import static
org.apache.hadoop.hive.metastore.properties.HMSPropertyManager.MAINTENANCE_STATUS;
+import static org.apache.hadoop.hive.metastore.properties.PropertyType.BOOLEAN;
+import static
org.apache.hadoop.hive.metastore.properties.PropertyType.DATETIME;
+import static org.apache.hadoop.hive.metastore.properties.PropertyType.DOUBLE;
+import static org.apache.hadoop.hive.metastore.properties.PropertyType.INTEGER;
+import static org.apache.hadoop.hive.metastore.properties.PropertyType.JSON;
+import static org.apache.hadoop.hive.metastore.properties.PropertyType.STRING;
+
+public class HMSPropertyStoreRemoteTest extends HMSPropertyStoreTest {
+ //private static final String NS = "hms";
+ protected HiveMetaStoreClient client;
+ private boolean isServerStarted = false;
+ protected int port;
+
+ boolean createStore(Configuration conf, Warehouse wh) {
+ try {
+ MetaStoreTestUtils.setConfForStandloneMode(conf);
+ objectStore = new ObjectStore();
+ objectStore.setConf(conf);
+ //TestObjectStore.dropAllStoreObjects(objectStore);
+ HMSHandler.createDefaultCatalog(objectStore, wh);
+ // configure object store
+ objectStore.createDatabase(new DatabaseBuilder()
+ .setCatalogName("hive")
+ .setName(DB1)
+ .setDescription("description")
+ .setLocation("locationurl")
+ .build(conf));
+ } catch(InvalidObjectException | MetaException | InvalidOperationException
xmeta) {
+ throw new PropertyException("unable to initialize server", xmeta);
+ }
+ return true;
+ }
+
+ @Before
+ public void setUp() throws Exception {
+ conf = MetastoreConf.newMetastoreConf();
+ MetastoreConf.setBoolVar(conf, MetastoreConf.ConfVars.HIVE_IN_TEST, true);
+ // Events that get cleaned happen in batches of 1 to exercise batching code
+ MetastoreConf.setLongVar(conf,
MetastoreConf.ConfVars.EVENT_CLEAN_MAX_EVENTS, 1L);
+ MetaStoreTestUtils.setConfForStandloneMode(conf);
+ if (isServerStarted) {
+ Assert.assertNotNull("Unable to connect to the MetaStore server",
client);
+ return;
+ }
+ port =
MetaStoreTestUtils.startMetaStoreWithRetry(HadoopThriftAuthBridge.getBridge(),
conf);
+ System.out.println("Starting MetaStore Server on port " + port);
+ isServerStarted = true;
+
+ Warehouse wh = new Warehouse(conf);
+ boolean inited = createStore(conf, wh);
+ LOG.info("MetaStore Thrift Server test initialization " + (inited?
"successful":"failed"));
+ // This is default case with setugi off for both client and server
+ client = createClient();
+ }
+
+
+ protected HiveMetaStoreClient createClient() throws Exception {
+ MetastoreConf.setVar(conf, MetastoreConf.ConfVars.THRIFT_URIS,
"thrift://localhost:" + port);
+ MetastoreConf.setBoolVar(conf, MetastoreConf.ConfVars.EXECUTE_SET_UGI,
false);
+ HiveMetaStoreClient client = new HiveMetaStoreClient(conf);
+ return client;
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ MetaStoreTestUtils.close(port);
+ super.tearDown();
+ }
+
+ @Test
+ public void testHMSProperties() throws Exception {
Review Comment:
I think we can just setup the config to use remote HMS and then run the
tests from HMSPropertyStoreTest class. We currently do that for few HMS unit
tests.
Issue Time Tracking
-------------------
Worklog Id: (was: 858525)
Time Spent: 7h (was: 6h 50m)
> A persistent property store
> ----------------------------
>
> Key: HIVE-27186
> URL: https://issues.apache.org/jira/browse/HIVE-27186
> Project: Hive
> Issue Type: Improvement
> Components: Metastore
> Affects Versions: 4.0.0-alpha-2
> Reporter: Henri Biestro
> Assignee: Henri Biestro
> Priority: Major
> Labels: pull-request-available
> Time Spent: 7h
> Remaining Estimate: 0h
>
> WHAT
> A persistent property store usable as a support facility for any metadata
> augmentation feature.
> WHY
> When adding new meta-data oriented features, we usually need to persist
> information linking the feature data and the HiveMetaStore objects it applies
> to. Any information related to a database, a table or the cluster - like
> statistics for example or any operational data state or data (think rolling
> backup) - fall in this use-case.
> Typically, accommodating such a feature requires modifying the Metastore
> database schema by adding or altering a table. It also usually implies
> modifying the thrift APIs to expose such meta-data to consumers.
> The proposed feature wants to solve the persistence and query/transport for
> these types of use-cases by exposing a 'key/(meta)value' store exposed as a
> property system.
> HOW
> A property-value model is the simple and generic exposed API.
> To provision for several usage scenarios, the model entry point is a
> 'namespace' that qualifies the feature-component property manager. For
> example, 'stats' could be the namespace for all properties related to the
> 'statistics' feature.
> The namespace identifies a manager that handles property-groups persisted as
> property-maps. For instance, all statistics pertaining to a given table would
> be collocated in the same property-group. As such, all properties (say number
> of 'unique_values' per columns) for a given HMS table 'relation0' would all
> be stored and persisted in the same property-map instance.
> Property-maps may be decorated by an (optional) schema that may declare the
> name and value-type of allowed properties (and their optional default value).
> Each property is addressed by a name, a path uniquely identifying the
> property in a given property map.
> The manager also handles transforming property-map names to the property-map
> keys used to persist them in the DB.
> The API provides inserting/updating properties in bulk transactionally. It
> also provides selection/projection to help reduce the volume of exchange
> between client/server; selection can use (JEXL expression) predicates to
> filter maps.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
