[
https://issues.apache.org/jira/browse/DRILL-8259?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17598227#comment-17598227
]
ASF GitHub Bot commented on DRILL-8259:
---------------------------------------
Z0ltrix commented on code in PR #2596:
URL: https://github.com/apache/drill/pull/2596#discussion_r959257458
##########
contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/config/HBasePersistentStoreProvider.java:
##########
@@ -20,116 +20,249 @@
import java.io.IOException;
import java.util.Map;
+import org.apache.drill.common.AutoCloseables;
import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.exec.exception.StoreException;
import org.apache.drill.exec.store.hbase.DrillHBaseConstants;
import org.apache.drill.exec.store.sys.PersistentStore;
import org.apache.drill.exec.store.sys.PersistentStoreConfig;
import org.apache.drill.exec.store.sys.PersistentStoreRegistry;
import
org.apache.drill.exec.store.sys.store.provider.BasePersistentStoreProvider;
+import
org.apache.drill.shaded.guava.com.google.common.annotations.VisibleForTesting;
+import org.apache.drill.shaded.guava.com.google.common.collect.Maps;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
+import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.util.Bytes;
-import
org.apache.drill.shaded.guava.com.google.common.annotations.VisibleForTesting;
-
public class HBasePersistentStoreProvider extends BasePersistentStoreProvider {
private static final org.slf4j.Logger logger =
org.slf4j.LoggerFactory.getLogger(HBasePersistentStoreProvider.class);
- static final byte[] FAMILY = Bytes.toBytes("s");
+ public static final byte[] DEFAULT_FAMILY_NAME = Bytes.toBytes("s");
- static final byte[] QUALIFIER = Bytes.toBytes("d");
+ public static final byte[] QUALIFIER_NAME = Bytes.toBytes("d");
+
+ private static final String HBASE_CLIENT_ID =
"drill-hbase-persistent-store-client";
private final TableName hbaseTableName;
+ private final byte[] family;
+
+ private Table hbaseTable;
+
private Configuration hbaseConf;
- private Connection connection;
+ private final Map<String, Object> tableConfig;
- private Table hbaseTable;
+ private final Map<String, Object> columnConfig;
+ private Connection connection;
+
+ @SuppressWarnings("unchecked")
public HBasePersistentStoreProvider(PersistentStoreRegistry registry) {
- @SuppressWarnings("unchecked")
- final Map<String, Object> config = (Map<String, Object>)
registry.getConfig().getAnyRef(DrillHBaseConstants.SYS_STORE_PROVIDER_HBASE_CONFIG);
- this.hbaseConf = HBaseConfiguration.create();
- this.hbaseConf.set(HConstants.HBASE_CLIENT_INSTANCE_ID,
"drill-hbase-persistent-store-client");
- if (config != null) {
- for (Map.Entry<String, Object> entry : config.entrySet()) {
- this.hbaseConf.set(entry.getKey(), String.valueOf(entry.getValue()));
+ final Map<String, Object> hbaseConfig = (Map<String, Object>)
registry.getConfig().getAnyRef(DrillHBaseConstants.SYS_STORE_PROVIDER_HBASE_CONFIG);
+ if
(registry.getConfig().hasPath(DrillHBaseConstants.SYS_STORE_PROVIDER_HBASE_TABLE_CONFIG))
{
+ tableConfig = (Map<String, Object>)
registry.getConfig().getAnyRef(DrillHBaseConstants.SYS_STORE_PROVIDER_HBASE_TABLE_CONFIG);
+ } else {
+ tableConfig = Maps.newHashMap();
+ }
+ if
(registry.getConfig().hasPath(DrillHBaseConstants.SYS_STORE_PROVIDER_HBASE_COLUMN_CONFIG))
{
+ columnConfig = (Map<String, Object>)
registry.getConfig().getAnyRef(DrillHBaseConstants.SYS_STORE_PROVIDER_HBASE_COLUMN_CONFIG);
+ } else {
+ columnConfig = Maps.newHashMap();
+ }
+ hbaseConf = HBaseConfiguration.create();
Review Comment:
> As you know, HBase is a nightmare for operational services due to the
complexity of the settings. The actual value in the above example is not a
recommended value, no unique value is appropriate for every case, but is simply
the type of value that this parameter has to fill, is "true/false", not "0/1".
hi @luocooong im still worried about the defaults, escpecially when drill
creates the table on his own...
am i correcth that you dont set any defaults except
SYS_STORE_PROVIDER_HBASE_TABLE, SYS_STORE_PROVIDER_HBASE_NAMESPACE and
SYS_STORE_PROVIDER_HBASE_FAMILY?
> Support advanced HBase persistence storage options
> --------------------------------------------------
>
> Key: DRILL-8259
> URL: https://issues.apache.org/jira/browse/DRILL-8259
> Project: Apache Drill
> Issue Type: New Feature
> Components: Storage - HBase
> Reporter: Cong Luo
> Assignee: Cong Luo
> Priority: Major
> Fix For: 2.0.0
>
>
> Its contents are as follows
> {code:java}
> sys.store.provider: {
> class: "org.apache.drill.exec.store.hbase.config.HBasePStoreProvider",
> hbase: {
> table : "drill_store",
> config: {
> "hbase.zookeeper.quorum": "zk_host3,zk_host2,zk_host1",
> "hbase.zookeeper.property.clientPort": "2181",
> "zookeeper.znode.parent": "/hbase-test"
> },
> table_config : {
> "durability": "ASYNC_WAL",
> "compaction_enabled": false,
> "split_enabled": false,
> "max_filesize": 10737418240,
> "memstore_flushsize": 536870912
> },
> column_config : {
> "versions": 1,
> "ttl": 2626560,
> "compression": "SNAPPY",
> "blockcache": true,
> "blocksize": 131072,
> "data_block_encoding": "FAST_DIFF",
> "in_memory": true,
> "dfs_replication": 3
> }
> }
> }{code}
>
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
