Repository: trafodion Updated Branches: refs/heads/master dcbc9f2c9 -> 072708f1b
[TRAFODION-3025] add create option storage policy Project: http://git-wip-us.apache.org/repos/asf/trafodion/repo Commit: http://git-wip-us.apache.org/repos/asf/trafodion/commit/5724227c Tree: http://git-wip-us.apache.org/repos/asf/trafodion/tree/5724227c Diff: http://git-wip-us.apache.org/repos/asf/trafodion/diff/5724227c Branch: refs/heads/master Commit: 5724227c48f1fa33e993a5642803792fb0478867 Parents: 027c7f5 Author: Liu Ming <ovis_p...@sina.com> Authored: Sat Apr 21 11:26:52 2018 -0400 Committer: Liu Ming <ovis_p...@sina.com> Committed: Sat Apr 21 11:26:52 2018 -0400 ---------------------------------------------------------------------- core/sql/executor/HBaseClient_JNI.cpp | 5 + core/sql/exp/ExpHbaseDefs.h | 1 + core/sql/regress/seabase/EXPECTED002 | 2 + core/sql/regress/seabase/TEST002 | 1 + core/sql/sqlcomp/CmpSeabaseDDLcommon.cpp | 5 + .../java/org/trafodion/sql/HBaseClient.java | 104 ++++++++++++++++++- 6 files changed, 116 insertions(+), 2 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/trafodion/blob/5724227c/core/sql/executor/HBaseClient_JNI.cpp ---------------------------------------------------------------------- diff --git a/core/sql/executor/HBaseClient_JNI.cpp b/core/sql/executor/HBaseClient_JNI.cpp index 8726fb8..a231241 100644 --- a/core/sql/executor/HBaseClient_JNI.cpp +++ b/core/sql/executor/HBaseClient_JNI.cpp @@ -535,10 +535,14 @@ HBC_RetCode HBaseClient_JNI::create(const char* fileName, HBASE_NAMELIST& colFam if (jresult == false) { +printf("LMDBG: jresult is false\n"); logError(CAT_SQL_HBASE, "HBaseClient_JNI::create()", getLastError()); jenv_->PopLocalFrame(NULL); return HBC_ERROR_CREATE_EXCEPTION; } + else +printf("LMDBG: jresult is true\n"); + jenv_->PopLocalFrame(NULL); return HBC_OK; } @@ -604,6 +608,7 @@ HBC_RetCode HBaseClient_JNI::create(const char* fileName, if (jresult == false) { logError(CAT_SQL_HBASE, "HBaseClient_JNI::create()", getLastError()); + GetCliGlobals()->setJniErrorStr(getLastError()); jenv_->PopLocalFrame(NULL); return HBC_ERROR_CREATE_EXCEPTION; } http://git-wip-us.apache.org/repos/asf/trafodion/blob/5724227c/core/sql/exp/ExpHbaseDefs.h ---------------------------------------------------------------------- diff --git a/core/sql/exp/ExpHbaseDefs.h b/core/sql/exp/ExpHbaseDefs.h index 598328c..b3f8475 100644 --- a/core/sql/exp/ExpHbaseDefs.h +++ b/core/sql/exp/ExpHbaseDefs.h @@ -74,6 +74,7 @@ enum HbaseOptionEnum HBASE_SPLIT_POLICY = 22, // " HBASE_CACHE_DATA_IN_L1 = 23, // column family HBASE_PREFETCH_BLOCKS_ON_OPEN = 24, // " + HBASE_HDFS_STORAGE_POLICY = 25, // " HBASE_MAX_OPTIONS }; http://git-wip-us.apache.org/repos/asf/trafodion/blob/5724227c/core/sql/regress/seabase/EXPECTED002 ---------------------------------------------------------------------- diff --git a/core/sql/regress/seabase/EXPECTED002 b/core/sql/regress/seabase/EXPECTED002 index 286b157..a98ae97 100644 --- a/core/sql/regress/seabase/EXPECTED002 +++ b/core/sql/regress/seabase/EXPECTED002 @@ -1313,6 +1313,7 @@ FS_DATA_TYPE SQL_DATA_TYPE DISPLAY_DATA_TYPE +>memstore_flush_size = '2000000', +>split_policy = 'org.apache.hadoop.hbase.regionserver.KeyPrefixRegionSplitPolicy', +>CACHE_DATA_IN_L1 = 'false', ++>HDFS_STORAGE_POLICY = 'hot' , +>prefetch_blocks_on_open = 'false' +>); @@ -1350,6 +1351,7 @@ CREATE TABLE TRAFODION.T002SCH.CREATEOPTIONS MEMSTORE_FLUSH_SIZE = '2000000', SPLIT_POLICY = 'org.apache.hadoop.hbase.regionserver.KeyPrefixRegionSplitPolicy', CACHE_DATA_IN_L1 = 'false', + HDFS_STORAGE_POLICY = 'hot', PREFETCH_BLOCKS_ON_OPEN = 'false' ) ; http://git-wip-us.apache.org/repos/asf/trafodion/blob/5724227c/core/sql/regress/seabase/TEST002 ---------------------------------------------------------------------- diff --git a/core/sql/regress/seabase/TEST002 b/core/sql/regress/seabase/TEST002 index c7bc291..081793f 100644 --- a/core/sql/regress/seabase/TEST002 +++ b/core/sql/regress/seabase/TEST002 @@ -261,6 +261,7 @@ durability = 'async_wal', memstore_flush_size = '2000000', split_policy = 'org.apache.hadoop.hbase.regionserver.KeyPrefixRegionSplitPolicy', CACHE_DATA_IN_L1 = 'false', +HDFS_STORAGE_POLICY = 'hot' , prefetch_blocks_on_open = 'false' ); http://git-wip-us.apache.org/repos/asf/trafodion/blob/5724227c/core/sql/sqlcomp/CmpSeabaseDDLcommon.cpp ---------------------------------------------------------------------- diff --git a/core/sql/sqlcomp/CmpSeabaseDDLcommon.cpp b/core/sql/sqlcomp/CmpSeabaseDDLcommon.cpp index fa43caa..14f07c2 100644 --- a/core/sql/sqlcomp/CmpSeabaseDDLcommon.cpp +++ b/core/sql/sqlcomp/CmpSeabaseDDLcommon.cpp @@ -2488,6 +2488,11 @@ short CmpSeabaseDDL::generateHbaseOptionsArray( else isError = TRUE; } + else if (hbaseOption->key() == "HDFS_STORAGE_POLICY") + { + hbaseCreateOptionsArray[HBASE_HDFS_STORAGE_POLICY] = + hbaseOption->val(); + } else isError = TRUE; http://git-wip-us.apache.org/repos/asf/trafodion/blob/5724227c/core/sql/src/main/java/org/trafodion/sql/HBaseClient.java ---------------------------------------------------------------------- diff --git a/core/sql/src/main/java/org/trafodion/sql/HBaseClient.java b/core/sql/src/main/java/org/trafodion/sql/HBaseClient.java index ffb42a2..94517bf 100644 --- a/core/sql/src/main/java/org/trafodion/sql/HBaseClient.java +++ b/core/sql/src/main/java/org/trafodion/sql/HBaseClient.java @@ -34,6 +34,8 @@ import java.util.Map; import java.util.Arrays; import java.net.URI; import java.net.URISyntaxException; +import java.lang.reflect.InvocationTargetException; +import java.lang.reflect.Method; import org.apache.log4j.PropertyConfigurator; import org.apache.log4j.Logger; @@ -93,6 +95,8 @@ import java.util.TreeSet; import org.apache.hadoop.fs.Path; import org.apache.hadoop.fs.FileStatus; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.LocatedFileStatus; import org.apache.hadoop.fs.permission.AclEntry; import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.io.hfile.HFile; @@ -161,6 +165,7 @@ public class HBaseClient { public static final int HBASE_SPLIT_POLICY = 22; public static final int HBASE_CACHE_DATA_IN_L1 = 23; public static final int HBASE_PREFETCH_BLOCKS_ON_OPEN = 24; + public static final int HBASE_HDFS_STORAGE_POLICY= 25; private static Connection connection; @@ -237,10 +242,12 @@ public class HBaseClient { private class ChangeFlags { boolean tableDescriptorChanged; boolean columnDescriptorChanged; + boolean storagePolicyChanged; ChangeFlags() { tableDescriptorChanged = false; columnDescriptorChanged = false; + storagePolicyChanged = false; } void setTableDescriptorChanged() { @@ -258,6 +265,19 @@ public class HBaseClient { boolean columnDescriptorChanged() { return columnDescriptorChanged; } + + void setStoragePolicyChanged(String str) { + storagePolicy_ = str; + storagePolicyChanged = true; + } + + boolean storagePolicyChanged() { + return storagePolicyChanged; + } + + String storagePolicy_; + + } private ChangeFlags setDescriptors(Object[] tableOptions, @@ -477,6 +497,11 @@ public class HBaseClient { colDesc.setPrefetchBlocksOnOpen(false); returnStatus.setColumnDescriptorChanged(); break ; + case HBASE_HDFS_STORAGE_POLICY: + //TODO HBase 2.0 support this + //So when come to HBase 2.0, no need to do this via HDFS, just set here + returnStatus.setStoragePolicyChanged(tableOption); + break ; case HBASE_SPLIT_POLICY: // This method not yet available in earlier versions // desc.setRegionSplitPolicyClassName(tableOption)); @@ -498,6 +523,7 @@ public class HBaseClient { throws IOException, MasterNotRunningException { if (logger.isDebugEnabled()) logger.debug("HBaseClient.createk(" + tblName + ") called."); String trueStr = "TRUE"; + ChangeFlags setDescRet = null; HTableDescriptor desc = new HTableDescriptor(tblName); addCoprocessor(desc); int defaultVersionsValue = 0; @@ -518,7 +544,7 @@ public class HBaseClient { HColumnDescriptor colDesc = new HColumnDescriptor(colFam); // change the descriptors based on the tableOptions; - setDescriptors(tableOptions,desc /*out*/,colDesc /*out*/, defaultVersionsValue); + setDescRet = setDescriptors(tableOptions,desc /*out*/,colDesc /*out*/, defaultVersionsValue); desc.addFamily(colDesc); } @@ -552,10 +578,84 @@ public class HBaseClient { admin.createTable(desc); } } - admin.close(); + + if(setDescRet!= null) + if(setDescRet.storagePolicyChanged()) + { + //change the HDFS storage policy + //get the HBase table path + String hbaseRoot = config.get("hbase.rootdir"); + FileSystem fs = FileSystem.get(config); + //Construct the HDFS dir + //find out if namespace is there + String[] parts = tblName.split(":"); + String namespacestr=""; + + //guess the path pattern + //different HBase version may have different path pattern + //There is no interface to get this information using HBase User API + //Since it is HBase internal behavior + //At present, before HBase 2.0 release and before HBASE-19858 released in HBase 1.5.0 + //Trafodion here need a trick to guess + String fullPath = hbaseRoot + "/data/" ; + String fullPath2 = hbaseRoot + "/data/default/"; + + //check if fullPath2 exist + if(fs.exists(new Path(fullPath2))) + fullPath = fullPath2; + + if(parts.length >1) //have namespace + fullPath = fullPath + parts[0] + "/" + parts[1]; + else + fullPath = fullPath + tblName; + + if (logger.isDebugEnabled()) logger.debug("createk table fullPath is " + fullPath); + + String invokeret = invokeSetStoragePolicy(fs, fullPath, setDescRet.storagePolicy_ ) ; + + if( invokeret != null) + { + //error handling + admin.close(); + throw new IOException(invokeret); + } + } + + admin.close(); return true; } + private static String invokeSetStoragePolicy(final FileSystem fs, final String pathstr, + final String storagePolicy) { + String ret = null; + Path path = new Path(pathstr); + Method m = null; + try { + m = fs.getClass().getDeclaredMethod("setStoragePolicy", + new Class<?>[] { Path.class, String.class }); + m.setAccessible(true); + } catch (NoSuchMethodException e) { + ret = "FileSystem doesn't support setStoragePolicy"; + m = null; + } catch (SecurityException e) { + ret = "No access to setStoragePolicy on FileSystem from the SecurityManager"; + m = null; // could happen on setAccessible() or getDeclaredMethod() + } + if (m != null) { + try { + m.invoke(fs, path, storagePolicy); + if (logger.isDebugEnabled()) { + logger.debug("Set storagePolicy=" + storagePolicy + " for path=" + path); + } + } catch (Exception e) { + logger.error("invoke set storage policy error : " + e); + ret = "invoke set storage policy error : " + e.getMessage(); + } + } + + return ret; + } + public boolean registerTruncateOnAbort(String tblName, long transID) throws MasterNotRunningException, IOException {
