This is an automated email from the ASF dual-hosted git repository. stoty pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/master by this push: new a3a19af PHOENIX-6280 Support HBase 2.4 a3a19af is described below commit a3a19afcdc06f4834051f33b3a2adc4577b454c0 Author: Istvan Toth <st...@apache.org> AuthorDate: Thu Dec 24 11:25:31 2020 +0100 PHOENIX-6280 Support HBase 2.4 --- Jenkinsfile | 2 +- phoenix-core/pom.xml | 19 ++ .../it/resources/compatible_client_versions.json | 3 +- .../hadoop/hbase/ipc/PhoenixRpcScheduler.java | 2 +- .../IndexHalfStoreFileReaderGenerator.java | 3 +- .../org/apache/phoenix/execute/DelegateHTable.java | 5 - .../hbase/index/parallel/ThreadPoolManager.java | 5 +- .../apache/phoenix/schema/MetaDataSplitPolicy.java | 47 +++-- .../schema/SplitOnLeadingVarCharColumnsPolicy.java | 4 +- .../phoenix/transaction/OmidTransactionTable.java | 5 - .../phoenix/compat/hbase/CompatDelegateHTable.java | 8 + .../compat/hbase/CompatOmidTransactionTable.java | 8 + .../compat/hbase/CompatPhoenixRpcScheduler.java | 2 +- ...onTable.java => CompatSteppingSplitPolicy.java} | 10 +- .../apache/phoenix/compat/hbase/CompatUtil.java | 9 +- .../phoenix/compat/hbase/CompatDelegateHTable.java | 7 + .../compat/hbase/CompatOmidTransactionTable.java | 8 + .../compat/hbase/CompatPhoenixRpcScheduler.java | 2 +- .../compat/hbase/CompatSteppingSplitPolicy.java | 10 +- .../apache/phoenix/compat/hbase/CompatUtil.java | 8 + .../phoenix/compat/hbase/CompatDelegateHTable.java | 6 + .../compat/hbase/CompatOmidTransactionTable.java | 6 + .../compat/hbase/CompatPhoenixRpcScheduler.java | 2 +- .../compat/hbase/CompatSteppingSplitPolicy.java | 10 +- .../apache/phoenix/compat/hbase/CompatUtil.java | 8 + phoenix-hbase-compat-2.4.0/pom.xml | 102 +++++++++++ .../phoenix/compat/hbase/CompatDelegateHTable.java | 7 + .../compat/hbase/CompatOmidTransactionTable.java | 7 + .../phoenix/compat/hbase/CompatPermissionUtil.java | 59 ++++++ .../compat/hbase/CompatPhoenixRpcScheduler.java | 2 +- .../compat/hbase/CompatSteppingSplitPolicy.java | 5 +- .../compat/hbase/CompatStoreFileReader.java | 48 +++++ .../apache/phoenix/compat/hbase/CompatUtil.java | 8 + .../compat/hbase/HbaseCompatCapabilities.java | 26 +-- .../apache/phoenix/compat/hbase/OffsetCell.java | 136 ++++++++++++++ .../CompatBaseScannerRegionObserver.java | 197 +++++++++++++++++++++ .../coprocessor/CompatIndexRegionObserver.java | 25 +-- .../phoenix/compat/hbase/test/DelegateCell.java | 139 +++++++++++++++ phoenix-server/pom.xml | 17 ++ pom.xml | 25 +++ 40 files changed, 931 insertions(+), 71 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 5bb1ebd..1b4e1f5 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -39,7 +39,7 @@ pipeline { axes { axis { name 'HBASE_PROFILE' - values '2.1', '2.2', '2.3' + values '2.1', '2.2', '2.3', '2.4' } } diff --git a/phoenix-core/pom.xml b/phoenix-core/pom.xml index 9f8efca..fd25797 100644 --- a/phoenix-core/pom.xml +++ b/phoenix-core/pom.xml @@ -70,6 +70,9 @@ || ("${hbase.compat.version}".equals("2.3.0") && hbaseMinor == 3 && hbasePatch >=0) + || ("${hbase.compat.version}".equals("2.4.0") + && hbaseMinor == 4 + && hbasePatch >=0) ) </condition> </evaluateBeanshell> @@ -590,5 +593,21 @@ </dependency> </dependencies> </profile> + <profile> + <id>phoenix-hbase-compat-2.4.0</id> + <!-- keep dependency plugin happy --> + <activation> + <property> + <name>hbase.profile</name> + <value>2.4</value> + </property> + </activation> + <dependencies> + <dependency> + <groupId>org.apache.zookeeper</groupId> + <artifactId>zookeeper-jute</artifactId> + </dependency> + </dependencies> + </profile> </profiles> </project> diff --git a/phoenix-core/src/it/resources/compatible_client_versions.json b/phoenix-core/src/it/resources/compatible_client_versions.json index 692cb38..aa94642 100644 --- a/phoenix-core/src/it/resources/compatible_client_versions.json +++ b/phoenix-core/src/it/resources/compatible_client_versions.json @@ -22,5 +22,6 @@ "1.5": ["4.15.0"], "2.1": ["5.1.0"], "2.2": ["5.1.0"], - "2.3": ["5.1.0"] + "2.3": ["5.1.0"], + "2.4": ["5.1.0"] } diff --git a/phoenix-core/src/main/java/org/apache/hadoop/hbase/ipc/PhoenixRpcScheduler.java b/phoenix-core/src/main/java/org/apache/hadoop/hbase/ipc/PhoenixRpcScheduler.java index 4508a2d..9c56316 100644 --- a/phoenix-core/src/main/java/org/apache/hadoop/hbase/ipc/PhoenixRpcScheduler.java +++ b/phoenix-core/src/main/java/org/apache/hadoop/hbase/ipc/PhoenixRpcScheduler.java @@ -28,7 +28,7 @@ import org.apache.phoenix.query.QueryServicesOptions; import org.apache.phoenix.thirdparty.com.google.common.annotations.VisibleForTesting; /** - * {@link RpcScheduler} that first checks to see if this is an index or metedata update before passing off the + * {@link RpcScheduler} that first checks to see if this is an index or metadata update before passing off the * call to the delegate {@link RpcScheduler}. */ public class PhoenixRpcScheduler extends CompatPhoenixRpcScheduler { diff --git a/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/IndexHalfStoreFileReaderGenerator.java b/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/IndexHalfStoreFileReaderGenerator.java index 5e59136..427bf3b 100644 --- a/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/IndexHalfStoreFileReaderGenerator.java +++ b/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/IndexHalfStoreFileReaderGenerator.java @@ -53,6 +53,7 @@ import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker; import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.phoenix.compat.hbase.CompatUtil; import org.apache.phoenix.index.IndexMaintainer; import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.query.QueryConstants; @@ -97,7 +98,7 @@ public class IndexHalfStoreFileReaderGenerator implements RegionObserver, Region byte[] regionStartKeyInHFile = null; try (Connection hbaseConn = ConnectionFactory.createConnection(ctx.getEnvironment().getConfiguration())) { - Scan scan = MetaTableAccessor.getScanForTableName(hbaseConn, tableName); + Scan scan = CompatUtil.getScanForTableName(hbaseConn, tableName); SingleColumnValueFilter scvf = null; if (Reference.isTopFileRegion(r.getFileRegion())) { scvf = new SingleColumnValueFilter(HConstants.CATALOG_FAMILY, diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/DelegateHTable.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/DelegateHTable.java index 5b0b268..88e46d1 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/execute/DelegateHTable.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/DelegateHTable.java @@ -163,11 +163,6 @@ public class DelegateHTable extends CompatDelegateHTable implements Table { } @Override - public void mutateRow(RowMutations rm) throws IOException { - delegate.mutateRow(rm); - } - - @Override public Result append(Append append) throws IOException { return delegate.append(append); } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/parallel/ThreadPoolManager.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/parallel/ThreadPoolManager.java index 3dbe439..ea62cc5 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/parallel/ThreadPoolManager.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/parallel/ThreadPoolManager.java @@ -32,6 +32,7 @@ import org.slf4j.LoggerFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; import org.apache.hadoop.hbase.util.Threads; +import org.apache.phoenix.thirdparty.com.google.common.util.concurrent.ThreadFactoryBuilder; /** * Manage access to thread pools @@ -89,7 +90,9 @@ public class ThreadPoolManager { ShutdownOnUnusedThreadPoolExecutor pool = new ShutdownOnUnusedThreadPoolExecutor(maxThreads, maxThreads, keepAliveTime, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>(), - Threads.newDaemonThreadFactory(builder.getName() + "-"), builder.getName()); + new ThreadFactoryBuilder().setNameFormat(builder.getName()+"-pool-%d") + .setUncaughtExceptionHandler(Threads.LOGGING_EXCEPTION_HANDLER).build(), + builder.getName()); pool.allowCoreThreadTimeOut(true); return pool; } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataSplitPolicy.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataSplitPolicy.java index 365ff5c..e5a0c33 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataSplitPolicy.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/MetaDataSplitPolicy.java @@ -23,24 +23,35 @@ import org.apache.phoenix.query.QueryServicesOptions; public class MetaDataSplitPolicy extends SplitOnLeadingVarCharColumnsPolicy { - @Override - protected boolean shouldSplit() { - Configuration conf = getConf(); - boolean allowSplittableSystemCatalogRollback = conf.getBoolean - (QueryServices.ALLOW_SPLITTABLE_SYSTEM_CATALOG_ROLLBACK, - QueryServicesOptions.DEFAULT_ALLOW_SPLITTABLE_SYSTEM_CATALOG_ROLLBACK); - boolean allowSystemCatalogToSplit = conf.getBoolean(QueryServices.SYSTEM_CATALOG_SPLITTABLE, - QueryServicesOptions.DEFAULT_SYSTEM_CATALOG_SPLITTABLE); - return super.shouldSplit() && allowSystemCatalogToSplit && !allowSplittableSystemCatalogRollback; - } + private boolean allowSystemCatalogToSplit() { + Configuration conf = getConf(); + boolean allowSplittableSystemCatalogRollback = + conf.getBoolean(QueryServices.ALLOW_SPLITTABLE_SYSTEM_CATALOG_ROLLBACK, + QueryServicesOptions.DEFAULT_ALLOW_SPLITTABLE_SYSTEM_CATALOG_ROLLBACK); + boolean allowSystemCatalogToSplit = + conf.getBoolean(QueryServices.SYSTEM_CATALOG_SPLITTABLE, + QueryServicesOptions.DEFAULT_SYSTEM_CATALOG_SPLITTABLE); + return allowSystemCatalogToSplit && !allowSplittableSystemCatalogRollback; + } - @Override - protected int getColumnToSplitAt() { - // SYSTEM.CATALOG rowkey is (tenant id, schema name, table name, column name, - // column family) ensure all meta data rows for a given schema are in the same - // region (indexes and tables are in the same schema as we lock the parent table - // when modifying an index) - return 2; - } + //This only exists in HBase 2.4+ + @Override + protected boolean canSplit() { + return super.canSplit() && allowSystemCatalogToSplit(); + } + + @Override + protected boolean shouldSplit() { + return super.shouldSplit() && allowSystemCatalogToSplit(); + } + + @Override + protected int getColumnToSplitAt() { + // SYSTEM.CATALOG rowkey is (tenant id, schema name, table name, column name, + // column family) ensure all meta data rows for a given schema are in the same + // region (indexes and tables are in the same schema as we lock the parent table + // when modifying an index) + return 2; + } } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/schema/SplitOnLeadingVarCharColumnsPolicy.java b/phoenix-core/src/main/java/org/apache/phoenix/schema/SplitOnLeadingVarCharColumnsPolicy.java index 80551f5..8eb22d7 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/schema/SplitOnLeadingVarCharColumnsPolicy.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/schema/SplitOnLeadingVarCharColumnsPolicy.java @@ -17,10 +17,10 @@ */ package org.apache.phoenix.schema; -import org.apache.hadoop.hbase.regionserver.SteppingSplitPolicy; +import org.apache.phoenix.compat.hbase.CompatSteppingSplitPolicy; import org.apache.phoenix.util.SchemaUtil; -public abstract class SplitOnLeadingVarCharColumnsPolicy extends SteppingSplitPolicy { +public abstract class SplitOnLeadingVarCharColumnsPolicy extends CompatSteppingSplitPolicy { abstract protected int getColumnToSplitAt(); protected final byte[] getSplitPoint(byte[] splitPoint) { diff --git a/phoenix-core/src/main/java/org/apache/phoenix/transaction/OmidTransactionTable.java b/phoenix-core/src/main/java/org/apache/phoenix/transaction/OmidTransactionTable.java index 886ca60..cdd8349 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/transaction/OmidTransactionTable.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/transaction/OmidTransactionTable.java @@ -215,11 +215,6 @@ public class OmidTransactionTable extends CompatOmidTransactionTable implements } @Override - public void mutateRow(RowMutations rm) throws IOException { - throw new UnsupportedOperationException(); - } - - @Override public Result append(Append append) throws IOException { throw new UnsupportedOperationException(); } diff --git a/phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatDelegateHTable.java b/phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatDelegateHTable.java index a55ae5b..089e44a 100644 --- a/phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatDelegateHTable.java +++ b/phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatDelegateHTable.java @@ -17,6 +17,9 @@ */ package org.apache.phoenix.compat.hbase; +import java.io.IOException; + +import org.apache.hadoop.hbase.client.RowMutations; import org.apache.hadoop.hbase.client.Table; public abstract class CompatDelegateHTable implements Table { @@ -27,4 +30,9 @@ public abstract class CompatDelegateHTable implements Table { this.delegate = delegate; } + @Override + public void mutateRow(RowMutations rm) throws IOException { + delegate.mutateRow(rm); + } + } diff --git a/phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatOmidTransactionTable.java b/phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatOmidTransactionTable.java index d01868b..573196e 100644 --- a/phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatOmidTransactionTable.java +++ b/phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatOmidTransactionTable.java @@ -17,7 +17,15 @@ */ package org.apache.phoenix.compat.hbase; +import java.io.IOException; + +import org.apache.hadoop.hbase.client.RowMutations; import org.apache.hadoop.hbase.client.Table; public abstract class CompatOmidTransactionTable implements Table { + + @Override + public void mutateRow(RowMutations rm) throws IOException { + throw new UnsupportedOperationException(); + } } diff --git a/phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatPhoenixRpcScheduler.java b/phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatPhoenixRpcScheduler.java index ff296f3..194c4c3 100644 --- a/phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatPhoenixRpcScheduler.java +++ b/phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatPhoenixRpcScheduler.java @@ -20,7 +20,7 @@ package org.apache.phoenix.compat.hbase; import org.apache.hadoop.hbase.ipc.RpcScheduler; /** - * {@link RpcScheduler} that first checks to see if this is an index or metedata update before + * {@link RpcScheduler} that first checks to see if this is an index or metadata update before * passing off the call to the delegate {@link RpcScheduler}. */ public abstract class CompatPhoenixRpcScheduler extends RpcScheduler { diff --git a/phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatOmidTransactionTable.java b/phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatSteppingSplitPolicy.java similarity index 79% copy from phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatOmidTransactionTable.java copy to phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatSteppingSplitPolicy.java index d01868b..5f700b0 100644 --- a/phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatOmidTransactionTable.java +++ b/phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatSteppingSplitPolicy.java @@ -17,7 +17,13 @@ */ package org.apache.phoenix.compat.hbase; -import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.regionserver.SteppingSplitPolicy; + +public class CompatSteppingSplitPolicy extends SteppingSplitPolicy { + + protected boolean canSplit() { + //dummy + return false; + } -public abstract class CompatOmidTransactionTable implements Table { } diff --git a/phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatUtil.java b/phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatUtil.java index 7b9d36a..a7d9ced 100644 --- a/phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatUtil.java +++ b/phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatUtil.java @@ -20,7 +20,10 @@ package org.apache.phoenix.compat.hbase; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellComparatorImpl; - +import org.apache.hadoop.hbase.MetaTableAccessor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; import org.apache.hadoop.hbase.regionserver.StoreFileWriter; @@ -52,4 +55,8 @@ public class CompatUtil { CellComparatorImpl cellComparator) { return builder.withComparator(cellComparator); } + + public static Scan getScanForTableName(Connection conn, TableName tableName) { + return MetaTableAccessor.getScanForTableName(conn, tableName); + } } diff --git a/phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatDelegateHTable.java b/phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatDelegateHTable.java index a55ae5b..a862073 100644 --- a/phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatDelegateHTable.java +++ b/phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatDelegateHTable.java @@ -17,6 +17,9 @@ */ package org.apache.phoenix.compat.hbase; +import java.io.IOException; + +import org.apache.hadoop.hbase.client.RowMutations; import org.apache.hadoop.hbase.client.Table; public abstract class CompatDelegateHTable implements Table { @@ -27,4 +30,8 @@ public abstract class CompatDelegateHTable implements Table { this.delegate = delegate; } + @Override + public void mutateRow(RowMutations rm) throws IOException { + delegate.mutateRow(rm); + } } diff --git a/phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatOmidTransactionTable.java b/phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatOmidTransactionTable.java index d01868b..573196e 100644 --- a/phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatOmidTransactionTable.java +++ b/phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatOmidTransactionTable.java @@ -17,7 +17,15 @@ */ package org.apache.phoenix.compat.hbase; +import java.io.IOException; + +import org.apache.hadoop.hbase.client.RowMutations; import org.apache.hadoop.hbase.client.Table; public abstract class CompatOmidTransactionTable implements Table { + + @Override + public void mutateRow(RowMutations rm) throws IOException { + throw new UnsupportedOperationException(); + } } diff --git a/phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatPhoenixRpcScheduler.java b/phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatPhoenixRpcScheduler.java index 13d69a7..b749a25 100644 --- a/phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatPhoenixRpcScheduler.java +++ b/phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatPhoenixRpcScheduler.java @@ -20,7 +20,7 @@ package org.apache.phoenix.compat.hbase; import org.apache.hadoop.hbase.ipc.RpcScheduler; /** - * {@link RpcScheduler} that first checks to see if this is an index or metedata update before + * {@link RpcScheduler} that first checks to see if this is an index or metadata update before * passing off the call to the delegate {@link RpcScheduler}. */ public abstract class CompatPhoenixRpcScheduler extends RpcScheduler { diff --git a/phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatOmidTransactionTable.java b/phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatSteppingSplitPolicy.java similarity index 79% copy from phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatOmidTransactionTable.java copy to phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatSteppingSplitPolicy.java index d01868b..5f700b0 100644 --- a/phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatOmidTransactionTable.java +++ b/phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatSteppingSplitPolicy.java @@ -17,7 +17,13 @@ */ package org.apache.phoenix.compat.hbase; -import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.regionserver.SteppingSplitPolicy; + +public class CompatSteppingSplitPolicy extends SteppingSplitPolicy { + + protected boolean canSplit() { + //dummy + return false; + } -public abstract class CompatOmidTransactionTable implements Table { } diff --git a/phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatUtil.java b/phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatUtil.java index b2cefe4..b11e8e7 100644 --- a/phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatUtil.java +++ b/phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatUtil.java @@ -20,6 +20,10 @@ package org.apache.phoenix.compat.hbase; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellComparatorImpl; +import org.apache.hadoop.hbase.MetaTableAccessor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; import org.apache.hadoop.hbase.regionserver.StoreFileWriter; @@ -51,4 +55,8 @@ public class CompatUtil { CellComparatorImpl cellComparator) { return builder.withComparator(cellComparator); } + + public static Scan getScanForTableName(Connection conn, TableName tableName) { + return MetaTableAccessor.getScanForTableName(conn, tableName); + } } diff --git a/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatDelegateHTable.java b/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatDelegateHTable.java index 770e858..d7ae605 100644 --- a/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatDelegateHTable.java +++ b/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatDelegateHTable.java @@ -20,6 +20,7 @@ package org.apache.phoenix.compat.hbase; import java.io.IOException; import org.apache.hadoop.hbase.client.RegionLocator; +import org.apache.hadoop.hbase.client.RowMutations; import org.apache.hadoop.hbase.client.Table; public abstract class CompatDelegateHTable implements Table { @@ -34,4 +35,9 @@ public abstract class CompatDelegateHTable implements Table { public RegionLocator getRegionLocator() throws IOException { return delegate.getRegionLocator(); } + + @Override + public void mutateRow(RowMutations rm) throws IOException { + delegate.mutateRow(rm); + } } diff --git a/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatOmidTransactionTable.java b/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatOmidTransactionTable.java index e1396ca..8baa2f3 100644 --- a/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatOmidTransactionTable.java +++ b/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatOmidTransactionTable.java @@ -20,6 +20,7 @@ package org.apache.phoenix.compat.hbase; import java.io.IOException; import org.apache.hadoop.hbase.client.RegionLocator; +import org.apache.hadoop.hbase.client.RowMutations; import org.apache.hadoop.hbase.client.Table; public abstract class CompatOmidTransactionTable implements Table { @@ -28,4 +29,9 @@ public abstract class CompatOmidTransactionTable implements Table { public RegionLocator getRegionLocator() throws IOException { throw new UnsupportedOperationException(); } + + @Override + public void mutateRow(RowMutations rm) throws IOException { + throw new UnsupportedOperationException(); + } } diff --git a/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatPhoenixRpcScheduler.java b/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatPhoenixRpcScheduler.java index 13d69a7..b749a25 100644 --- a/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatPhoenixRpcScheduler.java +++ b/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatPhoenixRpcScheduler.java @@ -20,7 +20,7 @@ package org.apache.phoenix.compat.hbase; import org.apache.hadoop.hbase.ipc.RpcScheduler; /** - * {@link RpcScheduler} that first checks to see if this is an index or metedata update before + * {@link RpcScheduler} that first checks to see if this is an index or metadata update before * passing off the call to the delegate {@link RpcScheduler}. */ public abstract class CompatPhoenixRpcScheduler extends RpcScheduler { diff --git a/phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatOmidTransactionTable.java b/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatSteppingSplitPolicy.java similarity index 79% copy from phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatOmidTransactionTable.java copy to phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatSteppingSplitPolicy.java index d01868b..5f700b0 100644 --- a/phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatOmidTransactionTable.java +++ b/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatSteppingSplitPolicy.java @@ -17,7 +17,13 @@ */ package org.apache.phoenix.compat.hbase; -import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.regionserver.SteppingSplitPolicy; + +public class CompatSteppingSplitPolicy extends SteppingSplitPolicy { + + protected boolean canSplit() { + //dummy + return false; + } -public abstract class CompatOmidTransactionTable implements Table { } diff --git a/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatUtil.java b/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatUtil.java index efe6efb..e2ee693 100644 --- a/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatUtil.java +++ b/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatUtil.java @@ -21,6 +21,10 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.CellComparatorImpl; +import org.apache.hadoop.hbase.MetaTableAccessor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.io.compress.Compression.Algorithm; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; @@ -69,4 +73,8 @@ public class CompatUtil { CellComparatorImpl cellComparator) { return builder; } + + public static Scan getScanForTableName(Connection conn, TableName tableName) { + return MetaTableAccessor.getScanForTableName(conn, tableName); + } } diff --git a/phoenix-hbase-compat-2.4.0/pom.xml b/phoenix-hbase-compat-2.4.0/pom.xml new file mode 100644 index 0000000..72a7cf2 --- /dev/null +++ b/phoenix-hbase-compat-2.4.0/pom.xml @@ -0,0 +1,102 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project + xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation= + "http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.phoenix</groupId> + <artifactId>phoenix</artifactId> + <version>5.1.0-SNAPSHOT</version> + </parent> + + <artifactId>phoenix-hbase-compat-2.4.0</artifactId> + <name>Phoenix Hbase 2.4.0 compatibility</name> + <description>Compatibility module for HBase 2.4.0+</description> + + <properties> + <hbase24.compat.version>2.4.0</hbase24.compat.version> + </properties> + + <dependencies> + <!-- HBase dependencies --> + <dependency> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-client</artifactId> + <version>${hbase24.compat.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-common</artifactId> + <version>${hbase24.compat.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-server</artifactId> + <version>${hbase24.compat.version}</version> + <scope>provided</scope> + </dependency> + <!-- Override parent dependencyManagement for transitive HBase dependencies --> + <dependency> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-hadoop-compat</artifactId> + <version>${hbase24.compat.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-hadoop2-compat</artifactId> + <version>${hbase24.compat.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-protocol</artifactId> + <version>${hbase24.compat.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-protocol-shaded</artifactId> + <version>${hbase24.compat.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-zookeeper</artifactId> + <version>${hbase24.compat.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-metrics</artifactId> + <version>${hbase24.compat.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-metrics-api</artifactId> + <version>${hbase24.compat.version}</version> + <scope>provided</scope> + </dependency> + </dependencies> + +</project> diff --git a/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatDelegateHTable.java b/phoenix-hbase-compat-2.4.0/src/main/java/org/apache/phoenix/compat/hbase/CompatDelegateHTable.java similarity index 85% copy from phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatDelegateHTable.java copy to phoenix-hbase-compat-2.4.0/src/main/java/org/apache/phoenix/compat/hbase/CompatDelegateHTable.java index 770e858..a255732 100644 --- a/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatDelegateHTable.java +++ b/phoenix-hbase-compat-2.4.0/src/main/java/org/apache/phoenix/compat/hbase/CompatDelegateHTable.java @@ -20,6 +20,8 @@ package org.apache.phoenix.compat.hbase; import java.io.IOException; import org.apache.hadoop.hbase.client.RegionLocator; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.RowMutations; import org.apache.hadoop.hbase.client.Table; public abstract class CompatDelegateHTable implements Table { @@ -34,4 +36,9 @@ public abstract class CompatDelegateHTable implements Table { public RegionLocator getRegionLocator() throws IOException { return delegate.getRegionLocator(); } + + @Override + public Result mutateRow(RowMutations rm) throws IOException { + return delegate.mutateRow(rm); + } } diff --git a/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatOmidTransactionTable.java b/phoenix-hbase-compat-2.4.0/src/main/java/org/apache/phoenix/compat/hbase/CompatOmidTransactionTable.java similarity index 83% copy from phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatOmidTransactionTable.java copy to phoenix-hbase-compat-2.4.0/src/main/java/org/apache/phoenix/compat/hbase/CompatOmidTransactionTable.java index e1396ca..ae7992f 100644 --- a/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatOmidTransactionTable.java +++ b/phoenix-hbase-compat-2.4.0/src/main/java/org/apache/phoenix/compat/hbase/CompatOmidTransactionTable.java @@ -20,6 +20,8 @@ package org.apache.phoenix.compat.hbase; import java.io.IOException; import org.apache.hadoop.hbase.client.RegionLocator; +import org.apache.hadoop.hbase.client.Result; +import org.apache.hadoop.hbase.client.RowMutations; import org.apache.hadoop.hbase.client.Table; public abstract class CompatOmidTransactionTable implements Table { @@ -28,4 +30,9 @@ public abstract class CompatOmidTransactionTable implements Table { public RegionLocator getRegionLocator() throws IOException { throw new UnsupportedOperationException(); } + + @Override + public Result mutateRow(RowMutations rm) throws IOException { + throw new UnsupportedOperationException(); + } } diff --git a/phoenix-hbase-compat-2.4.0/src/main/java/org/apache/phoenix/compat/hbase/CompatPermissionUtil.java b/phoenix-hbase-compat-2.4.0/src/main/java/org/apache/phoenix/compat/hbase/CompatPermissionUtil.java new file mode 100644 index 0000000..80a99b3 --- /dev/null +++ b/phoenix-hbase-compat-2.4.0/src/main/java/org/apache/phoenix/compat/hbase/CompatPermissionUtil.java @@ -0,0 +1,59 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.compat.hbase; + +import java.io.IOException; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.security.User; +import org.apache.hadoop.hbase.security.access.AccessChecker; +import org.apache.hadoop.hbase.security.access.Permission; +import org.apache.hadoop.hbase.security.access.UserPermission; +import org.apache.hadoop.hbase.zookeeper.ZKWatcher; + +public class CompatPermissionUtil { + + private CompatPermissionUtil() { + //Not to be instantiated + } + + public static AccessChecker newAccessChecker(final Configuration conf, ZKWatcher zk) { + //Ignore ZK parameter + return new AccessChecker(conf); + } + + public static void stopAccessChecker(AccessChecker accessChecker) throws IOException { + //NOOP + } + + public static String getUserFromUP(UserPermission userPermission) { + return userPermission.getUser(); + } + + public static Permission getPermissionFromUP(UserPermission userPermission) { + return userPermission.getPermission(); + } + + public static boolean authorizeUserTable(AccessChecker accessChecker, User user, + TableName table, Permission.Action action) { + // This also checks for group access + return accessChecker.getAuthManager().authorizeUserTable(user, table, action); + } + +} diff --git a/phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatPhoenixRpcScheduler.java b/phoenix-hbase-compat-2.4.0/src/main/java/org/apache/phoenix/compat/hbase/CompatPhoenixRpcScheduler.java similarity index 98% copy from phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatPhoenixRpcScheduler.java copy to phoenix-hbase-compat-2.4.0/src/main/java/org/apache/phoenix/compat/hbase/CompatPhoenixRpcScheduler.java index 13d69a7..b749a25 100644 --- a/phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatPhoenixRpcScheduler.java +++ b/phoenix-hbase-compat-2.4.0/src/main/java/org/apache/phoenix/compat/hbase/CompatPhoenixRpcScheduler.java @@ -20,7 +20,7 @@ package org.apache.phoenix.compat.hbase; import org.apache.hadoop.hbase.ipc.RpcScheduler; /** - * {@link RpcScheduler} that first checks to see if this is an index or metedata update before + * {@link RpcScheduler} that first checks to see if this is an index or metadata update before * passing off the call to the delegate {@link RpcScheduler}. */ public abstract class CompatPhoenixRpcScheduler extends RpcScheduler { diff --git a/phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatOmidTransactionTable.java b/phoenix-hbase-compat-2.4.0/src/main/java/org/apache/phoenix/compat/hbase/CompatSteppingSplitPolicy.java similarity index 86% copy from phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatOmidTransactionTable.java copy to phoenix-hbase-compat-2.4.0/src/main/java/org/apache/phoenix/compat/hbase/CompatSteppingSplitPolicy.java index d01868b..f185153 100644 --- a/phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatOmidTransactionTable.java +++ b/phoenix-hbase-compat-2.4.0/src/main/java/org/apache/phoenix/compat/hbase/CompatSteppingSplitPolicy.java @@ -17,7 +17,8 @@ */ package org.apache.phoenix.compat.hbase; -import org.apache.hadoop.hbase.client.Table; +import org.apache.hadoop.hbase.regionserver.SteppingSplitPolicy; + +public class CompatSteppingSplitPolicy extends SteppingSplitPolicy { -public abstract class CompatOmidTransactionTable implements Table { } diff --git a/phoenix-hbase-compat-2.4.0/src/main/java/org/apache/phoenix/compat/hbase/CompatStoreFileReader.java b/phoenix-hbase-compat-2.4.0/src/main/java/org/apache/phoenix/compat/hbase/CompatStoreFileReader.java new file mode 100644 index 0000000..03aa257 --- /dev/null +++ b/phoenix-hbase-compat-2.4.0/src/main/java/org/apache/phoenix/compat/hbase/CompatStoreFileReader.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.compat.hbase; + +import java.io.IOException; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.fs.HFileSystem; +import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.io.hfile.HFileInfo; +import org.apache.hadoop.hbase.io.hfile.ReaderContext; +import org.apache.hadoop.hbase.io.hfile.ReaderContext.ReaderType; +import org.apache.hadoop.hbase.regionserver.StoreFileReader; + +public class CompatStoreFileReader extends StoreFileReader { + + public CompatStoreFileReader(final FileSystem fs, final Path p, + final FSDataInputStreamWrapper in, long size, final CacheConfig cacheConf, + boolean primaryReplicaStoreFile, AtomicInteger refCount, final Configuration conf) + throws IOException { + super(new ReaderContext(p, in, size, new HFileSystem(fs), primaryReplicaStoreFile, + ReaderType.STREAM), + new HFileInfo(new ReaderContext(p, in, size, new HFileSystem(fs), + primaryReplicaStoreFile, ReaderType.STREAM), conf), + cacheConf, refCount, conf); + getHFileReader().getHFileInfo().initMetaAndIndex(getHFileReader()); + } + +} \ No newline at end of file diff --git a/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatUtil.java b/phoenix-hbase-compat-2.4.0/src/main/java/org/apache/phoenix/compat/hbase/CompatUtil.java similarity index 89% copy from phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatUtil.java copy to phoenix-hbase-compat-2.4.0/src/main/java/org/apache/phoenix/compat/hbase/CompatUtil.java index efe6efb..bb2dcf5 100644 --- a/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatUtil.java +++ b/phoenix-hbase-compat-2.4.0/src/main/java/org/apache/phoenix/compat/hbase/CompatUtil.java @@ -21,6 +21,10 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.CellComparator; import org.apache.hadoop.hbase.CellComparatorImpl; +import org.apache.hadoop.hbase.MetaTableAccessor; +import org.apache.hadoop.hbase.TableName; +import org.apache.hadoop.hbase.client.Connection; +import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.exceptions.DeserializationException; import org.apache.hadoop.hbase.io.compress.Compression.Algorithm; import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; @@ -69,4 +73,8 @@ public class CompatUtil { CellComparatorImpl cellComparator) { return builder; } + + public static Scan getScanForTableName(Connection conn, TableName tableName) { + return MetaTableAccessor.getScanForTableName(conn.getConfiguration(), tableName); + } } diff --git a/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatDelegateHTable.java b/phoenix-hbase-compat-2.4.0/src/main/java/org/apache/phoenix/compat/hbase/HbaseCompatCapabilities.java similarity index 51% copy from phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatDelegateHTable.java copy to phoenix-hbase-compat-2.4.0/src/main/java/org/apache/phoenix/compat/hbase/HbaseCompatCapabilities.java index 770e858..47b151a 100644 --- a/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatDelegateHTable.java +++ b/phoenix-hbase-compat-2.4.0/src/main/java/org/apache/phoenix/compat/hbase/HbaseCompatCapabilities.java @@ -15,23 +15,25 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.phoenix.compat.hbase; -import java.io.IOException; +public class HbaseCompatCapabilities { -import org.apache.hadoop.hbase.client.RegionLocator; -import org.apache.hadoop.hbase.client.Table; + public static boolean isMaxLookbackTimeSupported() { + return true; + } -public abstract class CompatDelegateHTable implements Table { + //In HBase 2.1 and 2.2, a lookback query won't return any results if covered by a future delete, + //but in 2.3 and later we have the preSoreScannerOpen hook that overrides that behavior + public static boolean isLookbackBeyondDeletesSupported() { return true; } - protected final Table delegate; + //HBase 2.1 does not have HBASE-22710, which is necessary for raw scan skip scan and + // AllVersionsIndexRebuild filters to + // show all versions properly. HBase 2.2.5+ and HBase 2.3.0+ have this fix. + public static boolean isRawFilterSupported() { return true; } - public CompatDelegateHTable(Table delegate) { - this.delegate = delegate; - } + //HBase 2.3+ has preWALAppend() on RegionObserver (HBASE-22623) + public static boolean hasPreWALAppend() { return false; } - @Override - public RegionLocator getRegionLocator() throws IOException { - return delegate.getRegionLocator(); - } } diff --git a/phoenix-hbase-compat-2.4.0/src/main/java/org/apache/phoenix/compat/hbase/OffsetCell.java b/phoenix-hbase-compat-2.4.0/src/main/java/org/apache/phoenix/compat/hbase/OffsetCell.java new file mode 100644 index 0000000..c5485a5 --- /dev/null +++ b/phoenix-hbase-compat-2.4.0/src/main/java/org/apache/phoenix/compat/hbase/OffsetCell.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.compat.hbase; + +import org.apache.hadoop.hbase.Cell; + +public class OffsetCell implements Cell { + + private Cell cell; + private int offset; + + public OffsetCell(Cell cell, int offset) { + this.cell = cell; + this.offset = offset; + } + + @Override + public byte[] getRowArray() { + return cell.getRowArray(); + } + + @Override + public int getRowOffset() { + return cell.getRowOffset() + offset; + } + + @Override + public short getRowLength() { + return (short) (cell.getRowLength() - offset); + } + + @Override + public byte[] getFamilyArray() { + return cell.getFamilyArray(); + } + + @Override + public int getFamilyOffset() { + return cell.getFamilyOffset(); + } + + @Override + public byte getFamilyLength() { + return cell.getFamilyLength(); + } + + @Override + public byte[] getQualifierArray() { + return cell.getQualifierArray(); + } + + @Override + public int getQualifierOffset() { + return cell.getQualifierOffset(); + } + + @Override + public int getQualifierLength() { + return cell.getQualifierLength(); + } + + @Override + public long getTimestamp() { + return cell.getTimestamp(); + } + + @Override + public byte getTypeByte() { + return cell.getTypeByte(); + } + + @Override public long getSequenceId() { + return cell.getSequenceId(); + } + + @Override + public byte[] getValueArray() { + return cell.getValueArray(); + } + + @Override + public int getValueOffset() { + return cell.getValueOffset(); + } + + @Override + public int getValueLength() { + return cell.getValueLength(); + } + + @Override + public byte[] getTagsArray() { + return cell.getTagsArray(); + } + + @Override + public int getTagsOffset() { + return cell.getTagsOffset(); + } + + @Override + public int getTagsLength() { + return cell.getTagsLength(); + } + + @Override + public Type getType() { + return cell.getType(); + } + + @Override + public long heapSize() { + return cell.heapSize(); + } + + @Override + public int getSerializedSize() { + return cell.getSerializedSize() - offset; + } + +} diff --git a/phoenix-hbase-compat-2.4.0/src/main/java/org/apache/phoenix/compat/hbase/coprocessor/CompatBaseScannerRegionObserver.java b/phoenix-hbase-compat-2.4.0/src/main/java/org/apache/phoenix/compat/hbase/coprocessor/CompatBaseScannerRegionObserver.java new file mode 100644 index 0000000..cd1a7f5 --- /dev/null +++ b/phoenix-hbase-compat-2.4.0/src/main/java/org/apache/phoenix/compat/hbase/coprocessor/CompatBaseScannerRegionObserver.java @@ -0,0 +1,197 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.compat.hbase.coprocessor; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HConstants; +import org.apache.hadoop.hbase.KeepDeletedCells; +import org.apache.hadoop.hbase.MemoryCompactionPolicy; +import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor; +import org.apache.hadoop.hbase.client.Scan; +import org.apache.hadoop.hbase.coprocessor.ObserverContext; +import org.apache.hadoop.hbase.coprocessor.RegionCoprocessorEnvironment; +import org.apache.hadoop.hbase.coprocessor.RegionObserver; +import org.apache.hadoop.hbase.regionserver.FlushLifeCycleTracker; +import org.apache.hadoop.hbase.regionserver.ScanOptions; +import org.apache.hadoop.hbase.regionserver.ScanType; +import org.apache.hadoop.hbase.regionserver.Store; +import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker; +import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest; +import org.apache.hadoop.hbase.util.EnvironmentEdgeManager; + +import java.io.IOException; + +public class CompatBaseScannerRegionObserver implements RegionObserver { + + public static final String PHOENIX_MAX_LOOKBACK_AGE_CONF_KEY = + "phoenix.max.lookback.age.seconds"; + public static final int DEFAULT_PHOENIX_MAX_LOOKBACK_AGE = 0; + + public void preCompactScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c, Store store, + ScanType scanType, ScanOptions options, CompactionLifeCycleTracker tracker, + CompactionRequest request) throws IOException { + Configuration conf = c.getEnvironment().getConfiguration(); + if (isMaxLookbackTimeEnabled(conf)) { + setScanOptionsForFlushesAndCompactions(conf, options, store, scanType); + } + } + + public void preFlushScannerOpen(ObserverContext<RegionCoprocessorEnvironment> c, Store store, + ScanOptions options, FlushLifeCycleTracker tracker) throws IOException { + Configuration conf = c.getEnvironment().getConfiguration(); + if (isMaxLookbackTimeEnabled(conf)) { + setScanOptionsForFlushesAndCompactions(conf, options, store, ScanType.COMPACT_RETAIN_DELETES); + } + } + + public void preMemStoreCompactionCompactScannerOpen( + ObserverContext<RegionCoprocessorEnvironment> c, Store store, ScanOptions options) + throws IOException { + Configuration conf = c.getEnvironment().getConfiguration(); + if (isMaxLookbackTimeEnabled(conf)) { + MemoryCompactionPolicy inMemPolicy = + store.getColumnFamilyDescriptor().getInMemoryCompaction(); + ScanType scanType; + //the eager and adaptive in-memory compaction policies can purge versions; the others + // can't. (Eager always does; adaptive sometimes does) + if (inMemPolicy.equals(MemoryCompactionPolicy.EAGER) || + inMemPolicy.equals(MemoryCompactionPolicy.ADAPTIVE)) { + scanType = ScanType.COMPACT_DROP_DELETES; + } else { + scanType = ScanType.COMPACT_RETAIN_DELETES; + } + setScanOptionsForFlushesAndCompactions(conf, options, store, scanType); + } + } + + public void preStoreScannerOpen(ObserverContext<RegionCoprocessorEnvironment> ctx, Store store, + ScanOptions options) throws IOException { + + if (!storeFileScanDoesntNeedAlteration(options)) { + //PHOENIX-4277 -- When doing a point-in-time (SCN) Scan, HBase by default will hide + // mutations that happen before a delete marker. This overrides that behavior. + options.setMinVersions(options.getMinVersions()); + KeepDeletedCells keepDeletedCells = KeepDeletedCells.TRUE; + if (store.getColumnFamilyDescriptor().getTimeToLive() != HConstants.FOREVER) { + keepDeletedCells = KeepDeletedCells.TTL; + } + options.setKeepDeletedCells(keepDeletedCells); + } + } + + private boolean storeFileScanDoesntNeedAlteration(ScanOptions options) { + Scan scan = options.getScan(); + boolean isRaw = scan.isRaw(); + //true if keep deleted cells is either TRUE or TTL + boolean keepDeletedCells = options.getKeepDeletedCells().equals(KeepDeletedCells.TRUE) || + options.getKeepDeletedCells().equals(KeepDeletedCells.TTL); + boolean timeRangeIsLatest = scan.getTimeRange().getMax() == HConstants.LATEST_TIMESTAMP; + boolean timestampIsTransactional = + isTransactionalTimestamp(scan.getTimeRange().getMax()); + return isRaw + || keepDeletedCells + || timeRangeIsLatest + || timestampIsTransactional; + } + + private boolean isTransactionalTimestamp(long ts) { + //have to use the HBase edge manager because the Phoenix one is in phoenix-core + return ts > (long) (EnvironmentEdgeManager.currentTime() * 1.1); + } + + /* + * If KeepDeletedCells.FALSE, KeepDeletedCells.TTL , + * let delete markers age once lookback age is done. + */ + public KeepDeletedCells getKeepDeletedCells(ScanOptions options, ScanType scanType) { + //if we're doing a minor compaction or flush, always set keep deleted cells + //to true. Otherwise, if keep deleted cells is false or TTL, use KeepDeletedCells TTL, + //where the value of the ttl might be overriden to the max lookback age elsewhere + return (options.getKeepDeletedCells() == KeepDeletedCells.TRUE + || scanType.equals(ScanType.COMPACT_RETAIN_DELETES)) ? + KeepDeletedCells.TRUE : KeepDeletedCells.TTL; + } + + /* + * if the user set a TTL we should leave MIN_VERSIONS at the default (0 in most of the cases). + * Otherwise the data (1st version) will not be removed after the TTL. If no TTL, we want + * Math.max(maxVersions, minVersions, 1) + */ + public int getMinVersions(ScanOptions options, ColumnFamilyDescriptor cfDescriptor) { + return cfDescriptor.getTimeToLive() != HConstants.FOREVER ? options.getMinVersions() + : Math.max(Math.max(options.getMinVersions(), + cfDescriptor.getMaxVersions()),1); + } + + /** + * + * @param conf HBase Configuration + * @param columnDescriptor ColumnFamilyDescriptor for the store being compacted + * @param options ScanOptions of overrides to the compaction scan + * @return Time to live in milliseconds, based on both HBase TTL and Phoenix max lookback age + */ + public long getTimeToLiveForCompactions(Configuration conf, + ColumnFamilyDescriptor columnDescriptor, + ScanOptions options) { + long ttlConfigured = columnDescriptor.getTimeToLive(); + long ttlInMillis = ttlConfigured * 1000; + long maxLookbackTtl = getMaxLookbackInMillis(conf); + if (isMaxLookbackTimeEnabled(maxLookbackTtl)) { + if (ttlConfigured == HConstants.FOREVER + && columnDescriptor.getKeepDeletedCells() != KeepDeletedCells.TRUE) { + // If user configured default TTL(FOREVER) and keep deleted cells to false or + // TTL then to remove unwanted delete markers we should change ttl to max lookback age + ttlInMillis = maxLookbackTtl; + } else { + //if there is a TTL, use TTL instead of max lookback age. + // Max lookback age should be more recent or equal to TTL + ttlInMillis = Math.max(ttlInMillis, maxLookbackTtl); + } + } + + return ttlInMillis; + } + + public void setScanOptionsForFlushesAndCompactions(Configuration conf, + ScanOptions options, + final Store store, + ScanType type) { + ColumnFamilyDescriptor cfDescriptor = store.getColumnFamilyDescriptor(); + options.setTTL(getTimeToLiveForCompactions(conf, cfDescriptor, + options)); + options.setKeepDeletedCells(getKeepDeletedCells(options, type)); + options.setMaxVersions(Integer.MAX_VALUE); + options.setMinVersions(getMinVersions(options, cfDescriptor)); + } + + public static long getMaxLookbackInMillis(Configuration conf){ + //config param is in seconds, switch to millis + return conf.getLong(PHOENIX_MAX_LOOKBACK_AGE_CONF_KEY, + DEFAULT_PHOENIX_MAX_LOOKBACK_AGE) * 1000; + } + + public static boolean isMaxLookbackTimeEnabled(Configuration conf){ + return isMaxLookbackTimeEnabled(conf.getLong(PHOENIX_MAX_LOOKBACK_AGE_CONF_KEY, + DEFAULT_PHOENIX_MAX_LOOKBACK_AGE)); + } + + public static boolean isMaxLookbackTimeEnabled(long maxLookbackTime){ + return maxLookbackTime > 0L; + } + +} diff --git a/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatDelegateHTable.java b/phoenix-hbase-compat-2.4.0/src/main/java/org/apache/phoenix/compat/hbase/coprocessor/CompatIndexRegionObserver.java similarity index 53% copy from phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatDelegateHTable.java copy to phoenix-hbase-compat-2.4.0/src/main/java/org/apache/phoenix/compat/hbase/coprocessor/CompatIndexRegionObserver.java index 770e858..f887ed3 100644 --- a/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatDelegateHTable.java +++ b/phoenix-hbase-compat-2.4.0/src/main/java/org/apache/phoenix/compat/hbase/coprocessor/CompatIndexRegionObserver.java @@ -15,23 +15,26 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -package org.apache.phoenix.compat.hbase; +package org.apache.phoenix.compat.hbase.coprocessor; -import java.io.IOException; +import org.apache.hadoop.hbase.coprocessor.RegionObserver; +import org.apache.hadoop.hbase.wal.WALKey; -import org.apache.hadoop.hbase.client.RegionLocator; -import org.apache.hadoop.hbase.client.Table; +import java.util.HashMap; +import java.util.Map; -public abstract class CompatDelegateHTable implements Table { +public class CompatIndexRegionObserver implements RegionObserver { - protected final Table delegate; + public static void appendToWALKey(WALKey key, String attrKey, byte[] attrValue) { + key.addExtendedAttribute(attrKey, attrValue); + } - public CompatDelegateHTable(Table delegate) { - this.delegate = delegate; + public static byte[] getAttributeValueFromWALKey(WALKey key, String attrKey) { + return key.getExtendedAttribute(attrKey); } - @Override - public RegionLocator getRegionLocator() throws IOException { - return delegate.getRegionLocator(); + public static Map<String, byte[]> getAttributeValuesFromWALKey(WALKey key) { + return new HashMap<String, byte[]>(key.getExtendedAttributes()); } + } diff --git a/phoenix-hbase-compat-2.4.0/src/main/java/org/apache/phoenix/compat/hbase/test/DelegateCell.java b/phoenix-hbase-compat-2.4.0/src/main/java/org/apache/phoenix/compat/hbase/test/DelegateCell.java new file mode 100644 index 0000000..0c7dfd8 --- /dev/null +++ b/phoenix-hbase-compat-2.4.0/src/main/java/org/apache/phoenix/compat/hbase/test/DelegateCell.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.compat.hbase.test; + +import org.apache.hadoop.hbase.Cell; + +public class DelegateCell implements Cell { + private final Cell delegate; + private final String name; + public DelegateCell(Cell delegate, String name) { + this.delegate = delegate; + this.name = name; + } + + @Override + public int getValueOffset() { + return delegate.getValueOffset(); + } + + @Override + public int getValueLength() { + return delegate.getValueLength(); + } + + @Override + public byte[] getValueArray() { + return delegate.getValueArray(); + } + + @Override + public byte getTypeByte() { + return delegate.getTypeByte(); + } + + @Override + public long getTimestamp() { + return delegate.getTimestamp(); + } + + @Override + public int getTagsOffset() { + return delegate.getTagsOffset(); + } + + @Override + public byte[] getTagsArray() { + return delegate.getTagsArray(); + } + + @Override + public int getRowOffset() { + return delegate.getRowOffset(); + } + + @Override + public short getRowLength() { + return delegate.getRowLength(); + } + + @Override + public byte[] getRowArray() { + return delegate.getRowArray(); + } + + @Override + public int getQualifierOffset() { + return delegate.getQualifierOffset(); + } + + @Override + public int getQualifierLength() { + return delegate.getQualifierLength(); + } + + @Override + public byte[] getQualifierArray() { + return delegate.getQualifierArray(); + } + + @Override + public int getFamilyOffset() { + return delegate.getFamilyOffset(); + } + + @Override + public byte getFamilyLength() { + return delegate.getFamilyLength(); + } + + @Override + public byte[] getFamilyArray() { + return delegate.getFamilyArray(); + } + + @Override + public String toString() { + return name; + } + + @Override + public long getSequenceId() { + return delegate.getSequenceId(); + } + + @Override + public int getTagsLength() { + return delegate.getTagsLength(); + } + + @Override + public Type getType() { + return delegate.getType(); + } + + @Override + public long heapSize() { + return delegate.heapSize(); + } + + @Override + public int getSerializedSize() { + return delegate.getSerializedSize(); + } +} diff --git a/phoenix-server/pom.xml b/phoenix-server/pom.xml index d189cb1..54f924f 100644 --- a/phoenix-server/pom.xml +++ b/phoenix-server/pom.xml @@ -426,5 +426,22 @@ </dependency> </dependencies> </profile> + <profile> + <id>phoenix-hbase-compat-2.4.0</id> + <!-- keep dependency plugin happy --> + <activation> + <property> + <name>hbase.profile</name> + <value>2.4</value> + </property> + </activation> + <dependencies> + <dependency> + <groupId>org.apache.zookeeper</groupId> + <artifactId>zookeeper-jute</artifactId> + <scope>provided</scope> + </dependency> + </dependencies> + </profile> </profiles> </project> diff --git a/pom.xml b/pom.xml index d6a10cf..df6d2c7 100644 --- a/pom.xml +++ b/pom.xml @@ -40,6 +40,7 @@ </organization> <modules> + <module>phoenix-hbase-compat-2.4.0</module> <module>phoenix-hbase-compat-2.3.0</module> <module>phoenix-hbase-compat-2.2.1</module> <module>phoenix-hbase-compat-2.1.6</module> @@ -629,6 +630,11 @@ <artifactId>phoenix-hbase-compat-2.3.0</artifactId> <version>${project.version}</version> </dependency> + <dependency> + <groupId>org.apache.phoenix</groupId> + <artifactId>phoenix-hbase-compat-2.4.0</artifactId> + <version>${project.version}</version> + </dependency> <!-- Intra-project test dependencies --> <dependency> <groupId>org.apache.phoenix</groupId> @@ -1526,6 +1532,25 @@ </properties> </profile> <profile> + <!-- PHOENIX-5993 This won't work with the public HBase artifacts --> + <id>phoenix-hbase-compat-2.4.0</id> + <activation> + <property> + <name>hbase.profile</name> + <value>2.4</value> + </property> + </activation> + <properties> + <hbase.profile>2.4</hbase.profile> + <hbase.compat.version>2.4.0</hbase.compat.version> + <hbase.version>2.4.0</hbase.version> + <!-- PHOENIX-6010 3.1.3 doesn't work because of Guava rebase --> + <hadoop.version>3.1.2</hadoop.version> + <hbase.thirdparty.version>3.4.1</hbase.thirdparty.version> + <zookeeper.version>3.5.7</zookeeper.version> + </properties> + </profile> + <profile> <id>owasp-dependency-check</id> <activation> <property>