Z0ltrix commented on code in PR #2596:
URL: https://github.com/apache/drill/pull/2596#discussion_r959257458
##########
contrib/storage-hbase/src/main/java/org/apache/drill/exec/store/hbase/config/HBasePersistentStoreProvider.java:
##########
@@ -20,116 +20,249 @@
import java.io.IOException;
import java.util.Map;
+import org.apache.drill.common.AutoCloseables;
import org.apache.drill.common.exceptions.DrillRuntimeException;
import org.apache.drill.exec.exception.StoreException;
import org.apache.drill.exec.store.hbase.DrillHBaseConstants;
import org.apache.drill.exec.store.sys.PersistentStore;
import org.apache.drill.exec.store.sys.PersistentStoreConfig;
import org.apache.drill.exec.store.sys.PersistentStoreRegistry;
import
org.apache.drill.exec.store.sys.store.provider.BasePersistentStoreProvider;
+import
org.apache.drill.shaded.guava.com.google.common.annotations.VisibleForTesting;
+import org.apache.drill.shaded.guava.com.google.common.collect.Maps;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Table;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
+import org.apache.hadoop.hbase.io.compress.Compression.Algorithm;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.util.Bytes;
-import
org.apache.drill.shaded.guava.com.google.common.annotations.VisibleForTesting;
-
public class HBasePersistentStoreProvider extends BasePersistentStoreProvider {
private static final org.slf4j.Logger logger =
org.slf4j.LoggerFactory.getLogger(HBasePersistentStoreProvider.class);
- static final byte[] FAMILY = Bytes.toBytes("s");
+ public static final byte[] DEFAULT_FAMILY_NAME = Bytes.toBytes("s");
- static final byte[] QUALIFIER = Bytes.toBytes("d");
+ public static final byte[] QUALIFIER_NAME = Bytes.toBytes("d");
+
+ private static final String HBASE_CLIENT_ID =
"drill-hbase-persistent-store-client";
private final TableName hbaseTableName;
+ private final byte[] family;
+
+ private Table hbaseTable;
+
private Configuration hbaseConf;
- private Connection connection;
+ private final Map<String, Object> tableConfig;
- private Table hbaseTable;
+ private final Map<String, Object> columnConfig;
+ private Connection connection;
+
+ @SuppressWarnings("unchecked")
public HBasePersistentStoreProvider(PersistentStoreRegistry registry) {
- @SuppressWarnings("unchecked")
- final Map<String, Object> config = (Map<String, Object>)
registry.getConfig().getAnyRef(DrillHBaseConstants.SYS_STORE_PROVIDER_HBASE_CONFIG);
- this.hbaseConf = HBaseConfiguration.create();
- this.hbaseConf.set(HConstants.HBASE_CLIENT_INSTANCE_ID,
"drill-hbase-persistent-store-client");
- if (config != null) {
- for (Map.Entry<String, Object> entry : config.entrySet()) {
- this.hbaseConf.set(entry.getKey(), String.valueOf(entry.getValue()));
+ final Map<String, Object> hbaseConfig = (Map<String, Object>)
registry.getConfig().getAnyRef(DrillHBaseConstants.SYS_STORE_PROVIDER_HBASE_CONFIG);
+ if
(registry.getConfig().hasPath(DrillHBaseConstants.SYS_STORE_PROVIDER_HBASE_TABLE_CONFIG))
{
+ tableConfig = (Map<String, Object>)
registry.getConfig().getAnyRef(DrillHBaseConstants.SYS_STORE_PROVIDER_HBASE_TABLE_CONFIG);
+ } else {
+ tableConfig = Maps.newHashMap();
+ }
+ if
(registry.getConfig().hasPath(DrillHBaseConstants.SYS_STORE_PROVIDER_HBASE_COLUMN_CONFIG))
{
+ columnConfig = (Map<String, Object>)
registry.getConfig().getAnyRef(DrillHBaseConstants.SYS_STORE_PROVIDER_HBASE_COLUMN_CONFIG);
+ } else {
+ columnConfig = Maps.newHashMap();
+ }
+ hbaseConf = HBaseConfiguration.create();
Review Comment:
> As you know, HBase is a nightmare for operational services due to the
complexity of the settings. The actual value in the above example is not a
recommended value, no unique value is appropriate for every case, but is simply
the type of value that this parameter has to fill, is "true/false", not "0/1".
hi @luocooong im still worried about the defaults, escpecially when drill
creates the table on his own...
am i correcth that you dont set any defaults except
SYS_STORE_PROVIDER_HBASE_TABLE, SYS_STORE_PROVIDER_HBASE_NAMESPACE and
SYS_STORE_PROVIDER_HBASE_FAMILY?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
