This is an automated email from the ASF dual-hosted git repository. stoty pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/phoenix.git
The following commit(s) were added to refs/heads/master by this push: new a8d6ba0 PHOENIX-6016 Support HBase 2.3.0 a8d6ba0 is described below commit a8d6ba036bc15eb66004a99e88a5d8a4fd1cab4c Author: Istvan Toth <st...@apache.org> AuthorDate: Fri Jul 17 07:51:20 2020 +0200 PHOENIX-6016 Support HBase 2.3.0 --- phoenix-core/pom.xml | 25 ++++ ...WALReplayWithIndexWritesAndCompressedWALIT.java | 17 +-- .../phoenix/end2end/BackwardCompatibilityIT.java | 3 + .../apache/phoenix/end2end/PermissionsCacheIT.java | 4 +- .../apache/phoenix/end2end/index/LocalIndexIT.java | 29 +++-- .../it/resources/compatible_client_versions.json | 4 +- .../regionserver/IndexHalfStoreFileReader.java | 6 +- .../IndexHalfStoreFileReaderGenerator.java | 4 +- .../coprocessor/PhoenixAccessController.java | 3 +- .../org/apache/phoenix/execute/DelegateHTable.java | 6 +- .../iterate/TableSnapshotResultIterator.java | 4 +- .../phoenix/mapreduce/MultiHfileOutputFormat.java | 24 ++-- .../phoenix/mapreduce/index/IndexScrutinyTool.java | 3 +- .../apache/phoenix/mapreduce/index/IndexTool.java | 11 +- .../phoenix/transaction/OmidTransactionTable.java | 4 +- .../recovery/TestPerRegionIndexWriteCache.java | 4 +- .../phoenix/jdbc/PhoenixEmbeddedDriverTest.java | 13 +- .../phoenix/compat/hbase/CompatDelegateHTable.java | 20 +-- .../compat/hbase/CompatOmidTransactionTable.java | 13 +- .../phoenix/compat/hbase/CompatPermissionUtil.java | 8 +- .../compat/hbase/CompatPhoenixRpcScheduler.java | 23 ++-- .../compat/hbase/CompatStoreFileReader.java | 29 ++--- .../apache/phoenix/compat/hbase/CompatUtil.java | 37 ++++-- .../phoenix/compat/hbase/CompatDelegateHTable.java | 20 +-- .../compat/hbase/CompatOmidTransactionTable.java | 13 +- .../phoenix/compat/hbase/CompatPermissionUtil.java | 6 + .../compat/hbase/CompatPhoenixRpcScheduler.java | 23 ++-- ...{CompatUtil.java => CompatStoreFileReader.java} | 29 ++--- .../apache/phoenix/compat/hbase/CompatUtil.java | 36 ++++-- phoenix-hbase-compat-2.3.0/pom.xml | 88 +++++++++++++ .../phoenix/compat/hbase/CompatDelegateHTable.java | 27 ++-- .../compat/hbase/CompatOmidTransactionTable.java | 21 ++-- .../phoenix/compat/hbase/CompatPermissionUtil.java | 9 +- .../compat/hbase/CompatPhoenixRpcScheduler.java | 23 ++-- .../compat/hbase/CompatStoreFileReader.java | 48 +++++++ .../apache/phoenix/compat/hbase/CompatUtil.java | 72 +++++++++++ .../apache/phoenix/compat/hbase/OffsetCell.java | 136 ++++++++++++++++++++ .../phoenix/compat/hbase/test/DelegateCell.java | 139 +++++++++++++++++++++ pom.xml | 55 ++++++-- 39 files changed, 837 insertions(+), 202 deletions(-) diff --git a/phoenix-core/pom.xml b/phoenix-core/pom.xml index 66386ad..1eece7f 100644 --- a/phoenix-core/pom.xml +++ b/phoenix-core/pom.xml @@ -140,6 +140,13 @@ <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-dependency-plugin</artifactId> + <configuration> + <ignoredUsedUndeclaredDependencies> + <ignoredUsedUndeclaredDependency> + dnsjava:dnsjava + </ignoredUsedUndeclaredDependency> + </ignoredUsedUndeclaredDependencies> + </configuration> <executions> <execution> <!-- generates the file that will be used by the sandbox script in the dev env --> @@ -493,6 +500,24 @@ </dependencies> +<profiles> + <profile> + <id>phoenix-hbase-compat-2.3.0</id> + <!-- keep dependency plugin happy --> + <activation> + <property> + <name>hbase.profile</name> + <value>2.3</value> + </property> + </activation> + <dependencies> + <dependency> + <groupId>org.apache.zookeeper</groupId> + <artifactId>zookeeper-jute</artifactId> + </dependency> + </dependencies> + </profile> +</profiles> <reporting> <plugins> <plugin> diff --git a/phoenix-core/src/it/java/org/apache/hadoop/hbase/regionserver/wal/WALReplayWithIndexWritesAndCompressedWALIT.java b/phoenix-core/src/it/java/org/apache/hadoop/hbase/regionserver/wal/WALReplayWithIndexWritesAndCompressedWALIT.java index 0314289..12618dd 100644 --- a/phoenix-core/src/it/java/org/apache/hadoop/hbase/regionserver/wal/WALReplayWithIndexWritesAndCompressedWALIT.java +++ b/phoenix-core/src/it/java/org/apache/hadoop/hbase/regionserver/wal/WALReplayWithIndexWritesAndCompressedWALIT.java @@ -52,7 +52,7 @@ import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.regionserver.HRegionServer; import org.apache.hadoop.hbase.regionserver.RegionServerAccounting; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALFactory; import org.apache.hadoop.hbase.wal.WALSplitter; @@ -176,11 +176,14 @@ public class WALReplayWithIndexWritesAndCompressedWALIT { */ @Test public void testReplayEditsWrittenViaHRegion() throws Exception { - final String tableNameStr = "testReplayEditsWrittenViaHRegion"; - final RegionInfo hri = RegionInfoBuilder.newBuilder(org.apache.hadoop.hbase.TableName.valueOf(tableNameStr)).setSplit(false).build(); - final Path basedir = FSUtils.getTableDir(hbaseRootDir, org.apache.hadoop.hbase.TableName.valueOf(tableNameStr)); - deleteDir(basedir); - final TableDescriptor htd = createBasic3FamilyHTD(tableNameStr); + final String tableNameStr = "testReplayEditsWrittenViaHRegion"; + final RegionInfo hri = + RegionInfoBuilder.newBuilder(TableName.valueOf(tableNameStr)) + .setSplit(false).build(); + final Path basedir = + CommonFSUtils.getTableDir(hbaseRootDir, TableName.valueOf(tableNameStr)); + deleteDir(basedir); + final TableDescriptor htd = createBasic3FamilyHTD(tableNameStr); //setup basic indexing for the table // enable indexing to a non-existant index table @@ -234,7 +237,7 @@ public class WALReplayWithIndexWritesAndCompressedWALIT { ConnectionFactory.createConnection(UTIL.getConfiguration()); // now check to ensure that we wrote to the index table - Table index = hbaseConn.getTable(org.apache.hadoop.hbase.TableName.valueOf(INDEX_TABLE_NAME)); + Table index = hbaseConn.getTable(TableName.valueOf(INDEX_TABLE_NAME)); int indexSize = getKeyValueCount(index); assertEquals("Index wasn't propertly updated from WAL replay!", 1, indexSize); Get g = new Get(rowkey); diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityIT.java index fde2f69..8dae498 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/BackwardCompatibilityIT.java @@ -106,6 +106,7 @@ public class BackwardCompatibilityIT { private static HBaseTestingUtility hbaseTestUtil; private static String zkQuorum; private static String url; + private String tmpDir; public BackwardCompatibilityIT(String compatibleClientVersion) { this.compatibleClientVersion = compatibleClientVersion; @@ -118,6 +119,7 @@ public class BackwardCompatibilityIT { @Before public synchronized void doSetup() throws Exception { + tmpDir = System.getProperty("java.io.tmpdir"); conf = HBaseConfiguration.create(); hbaseTestUtil = new HBaseTestingUtility(conf); setUpConfigForMiniCluster(conf); @@ -136,6 +138,7 @@ public class BackwardCompatibilityIT { } finally { hbaseTestUtil.shutdownMiniCluster(); } + System.setProperty("java.io.tmpdir", tmpDir); } private static List<String> computeClientVersions() throws Exception { diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/PermissionsCacheIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/PermissionsCacheIT.java index 2833a23..fb3e550 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/PermissionsCacheIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/PermissionsCacheIT.java @@ -27,13 +27,13 @@ import java.util.List; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.AuthUtil; import org.apache.hadoop.hbase.HBaseTestingUtility; -import org.apache.hadoop.hbase.security.access.AccessControlLists; import org.apache.hadoop.hbase.security.access.Permission; import org.apache.hadoop.hbase.security.access.Permission.Action; import org.apache.hadoop.hbase.zookeeper.ZKUtil; import org.apache.hadoop.hbase.zookeeper.ZKWatcher; import org.apache.hadoop.hbase.zookeeper.ZNodePaths; import org.apache.hbase.thirdparty.com.google.common.collect.ListMultimap; +import org.apache.phoenix.compat.hbase.CompatUtil; import org.apache.phoenix.util.SchemaUtil; import org.junit.BeforeClass; import org.junit.Test; @@ -89,7 +89,7 @@ public class PermissionsCacheIT extends BasePermissionsIT { String tableZNode = ZNodePaths.joinZNode(aclZNode, "@" + schema); byte[] data = ZKUtil.getData(zkw, tableZNode); ListMultimap<String, ? extends Permission> userPermissions = - AccessControlLists.readPermissions(data, conf); + CompatUtil.readPermissions(data, conf); assertTrue("User permissions not found in cache:", userPermissions.containsKey(regularUser1.getName())); List<? extends Permission> tablePermissions = diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/LocalIndexIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/LocalIndexIT.java index 0be30c2..724da6e 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/LocalIndexIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/index/LocalIndexIT.java @@ -53,7 +53,7 @@ import org.apache.hadoop.hbase.client.Table; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.Pair; import org.apache.phoenix.compile.QueryPlan; import org.apache.phoenix.coprocessor.BaseScannerRegionObserver; @@ -1039,17 +1039,22 @@ public class LocalIndexIT extends BaseLocalIndexIT { conn1.close(); } - private void copyLocalIndexHFiles(Configuration conf, RegionInfo fromRegion, RegionInfo toRegion, boolean move) - throws IOException { - Path root = FSUtils.getRootDir(conf); - - Path seondRegion = new Path(FSUtils.getTableDir(root, fromRegion.getTable()) + Path.SEPARATOR - + fromRegion.getEncodedName() + Path.SEPARATOR + "L#0/"); - Path hfilePath = FSUtils.getCurrentFileSystem(conf).listFiles(seondRegion, true).next().getPath(); - Path firstRegionPath = new Path(FSUtils.getTableDir(root, toRegion.getTable()) + Path.SEPARATOR - + toRegion.getEncodedName() + Path.SEPARATOR + "L#0/"); - FileSystem currentFileSystem = FSUtils.getCurrentFileSystem(conf); - assertTrue(FileUtil.copy(currentFileSystem, hfilePath, currentFileSystem, firstRegionPath, move, conf)); + private void copyLocalIndexHFiles(Configuration conf, RegionInfo fromRegion, + RegionInfo toRegion, boolean move) throws IOException { + Path root = CommonFSUtils.getRootDir(conf); + + Path seondRegion = + new Path(CommonFSUtils.getTableDir(root, fromRegion.getTable()) + Path.SEPARATOR + + fromRegion.getEncodedName() + Path.SEPARATOR + "L#0/"); + Path hfilePath = + CommonFSUtils.getCurrentFileSystem(conf).listFiles(seondRegion, true).next() + .getPath(); + Path firstRegionPath = + new Path(CommonFSUtils.getTableDir(root, toRegion.getTable()) + Path.SEPARATOR + + toRegion.getEncodedName() + Path.SEPARATOR + "L#0/"); + FileSystem currentFileSystem = CommonFSUtils.getCurrentFileSystem(conf); + assertTrue(FileUtil.copy(currentFileSystem, hfilePath, currentFileSystem, firstRegionPath, + move, conf)); } private int getCount(PhoenixConnection conn, String tableName, String columnFamily) diff --git a/phoenix-core/src/it/resources/compatible_client_versions.json b/phoenix-core/src/it/resources/compatible_client_versions.json index 810265c..692cb38 100644 --- a/phoenix-core/src/it/resources/compatible_client_versions.json +++ b/phoenix-core/src/it/resources/compatible_client_versions.json @@ -20,7 +20,7 @@ "1.3": ["4.14.3", "4.15.0"], "1.4": ["4.14.3", "4.15.0"], "1.5": ["4.15.0"], - "2.0": ["5.1.0"], "2.1": ["5.1.0"], - "2.2": ["5.1.0"] + "2.2": ["5.1.0"], + "2.3": ["5.1.0"] } diff --git a/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/IndexHalfStoreFileReader.java b/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/IndexHalfStoreFileReader.java index f4e9354..99ba0ff 100644 --- a/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/IndexHalfStoreFileReader.java +++ b/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/IndexHalfStoreFileReader.java @@ -35,6 +35,7 @@ import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.io.Reference; import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.phoenix.compat.hbase.CompatStoreFileReader; import org.apache.phoenix.index.IndexMaintainer; /** @@ -52,7 +53,7 @@ import org.apache.phoenix.index.IndexMaintainer; * This file is not splitable. Calls to {@link #midkey()} return null. */ -public class IndexHalfStoreFileReader extends StoreFileReader { +public class IndexHalfStoreFileReader extends CompatStoreFileReader { private final boolean top; // This is the key we split around. Its the first possible entry on a row: // i.e. empty column and a timestamp of LATEST_TIMESTAMP. @@ -88,8 +89,7 @@ public class IndexHalfStoreFileReader extends StoreFileReader { final byte[][] viewConstants, final RegionInfo regionInfo, byte[] regionStartKeyInHFile, byte[] splitKey, boolean primaryReplicaStoreFile, AtomicInteger refCount, RegionInfo currentRegion) throws IOException { - super(fs, p, in, size, cacheConf, primaryReplicaStoreFile, refCount, false, - conf); + super(fs, p, in, size, cacheConf, primaryReplicaStoreFile, refCount, conf); this.splitkey = splitKey == null ? r.getSplitKey() : splitKey; // Is it top or bottom half? this.top = Reference.isTopFileRegion(r.getFileRegion()); diff --git a/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/IndexHalfStoreFileReaderGenerator.java b/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/IndexHalfStoreFileReaderGenerator.java index 680eb3f..cdf0a2b 100644 --- a/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/IndexHalfStoreFileReaderGenerator.java +++ b/phoenix-core/src/main/java/org/apache/hadoop/hbase/regionserver/IndexHalfStoreFileReaderGenerator.java @@ -53,8 +53,6 @@ import org.apache.hadoop.hbase.io.hfile.CacheConfig; import org.apache.hadoop.hbase.regionserver.compactions.CompactionLifeCycleTracker; import org.apache.hadoop.hbase.regionserver.compactions.CompactionRequest; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.Pair; -import org.apache.phoenix.compat.hbase.CompatUtil; import org.apache.phoenix.index.IndexMaintainer; import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.query.QueryConstants; @@ -118,7 +116,7 @@ public class IndexHalfStoreFileReaderGenerator implements RegionObserver, Region } if (result == null || result.isEmpty()) { List<RegionInfo> mergeRegions = - CompatUtil.getMergeRegions(ctx.getEnvironment().getConnection(), + MetaTableAccessor.getMergeRegions(ctx.getEnvironment().getConnection(), region.getRegionInfo().getRegionName()); if (mergeRegions == null || mergeRegions.isEmpty()){ return reader; diff --git a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/PhoenixAccessController.java b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/PhoenixAccessController.java index 8acc42f..18787ec 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/PhoenixAccessController.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/coprocessor/PhoenixAccessController.java @@ -153,12 +153,13 @@ public class PhoenixAccessController extends BaseMetaDataEndpointObserver { "Not a valid environment, should be loaded by PhoenixMetaDataControllerEnvironment"); } + //2.3+ doesn't need to access ZK object. ZKWatcher zk = null; RegionCoprocessorEnvironment regionEnv = this.env.getRegionCoprocessorEnvironment(); if (regionEnv instanceof HasRegionServerServices) { zk = ((HasRegionServerServices) regionEnv).getRegionServerServices().getZooKeeper(); } - accessChecker = new AccessChecker(env.getConfiguration(), zk); + accessChecker = CompatPermissionUtil.newAccessChecker(env.getConfiguration(), zk); // set the user-provider. this.userProvider = UserProvider.instantiate(env.getConfiguration()); // init superusers and add the server principal (if using security) diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/DelegateHTable.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/DelegateHTable.java index c953c8d..5b0b268 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/execute/DelegateHTable.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/DelegateHTable.java @@ -43,17 +43,17 @@ import org.apache.hadoop.hbase.client.coprocessor.Batch.Call; import org.apache.hadoop.hbase.client.coprocessor.Batch.Callback; import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel; +import org.apache.phoenix.compat.hbase.CompatDelegateHTable; import com.google.protobuf.Descriptors.MethodDescriptor; import com.google.protobuf.Message; import com.google.protobuf.Service; import com.google.protobuf.ServiceException; -public class DelegateHTable implements Table { - protected final Table delegate; +public class DelegateHTable extends CompatDelegateHTable implements Table { public DelegateHTable(Table delegate) { - this.delegate = delegate; + super(delegate); } @Override diff --git a/phoenix-core/src/main/java/org/apache/phoenix/iterate/TableSnapshotResultIterator.java b/phoenix-core/src/main/java/org/apache/phoenix/iterate/TableSnapshotResultIterator.java index 5063ae7..09355bb 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/iterate/TableSnapshotResultIterator.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/iterate/TableSnapshotResultIterator.java @@ -33,7 +33,7 @@ import org.apache.hadoop.hbase.client.Scan; import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.io.ImmutableBytesWritable; import org.apache.hadoop.hbase.snapshot.RestoreSnapshotHelper; -import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil; import org.apache.phoenix.monitoring.ScanMetricsHolder; import org.apache.phoenix.schema.tuple.Tuple; @@ -80,7 +80,7 @@ public class TableSnapshotResultIterator implements ResultIterator { UUID.randomUUID().toString()); this.snapshotName = configuration.get( PhoenixConfigurationUtil.SNAPSHOT_NAME_KEY); - this.rootDir = FSUtils.getRootDir(configuration); + this.rootDir = CommonFSUtils.getRootDir(configuration); this.fs = rootDir.getFileSystem(configuration); init(); } diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/MultiHfileOutputFormat.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/MultiHfileOutputFormat.java index f4458ce..3216dcf 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/MultiHfileOutputFormat.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/MultiHfileOutputFormat.java @@ -72,6 +72,7 @@ import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.hadoop.mapreduce.lib.output.FileOutputCommitter; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; import org.apache.hadoop.mapreduce.lib.partition.TotalOrderPartitioner; +import org.apache.phoenix.compat.hbase.CompatUtil; import org.apache.phoenix.mapreduce.bulkload.TableRowkeyPair; import org.apache.phoenix.mapreduce.bulkload.TargetTableRef; import org.apache.phoenix.mapreduce.bulkload.TargetTableRefFunctions; @@ -227,14 +228,14 @@ public class MultiHfileOutputFormat extends FileOutputFormat<TableRowkeyPair, Ce WriterLength wl = new WriterLength(); Path tableOutputPath = CsvBulkImportUtil.getOutputPath(outputdir, tableName); Path familydir = new Path(tableOutputPath, Bytes.toString(family)); - + // phoenix-2216: start : fetching the configuration properties that were set to the table. // create a map from column family to the compression algorithm for the table. final Map<byte[], Algorithm> compressionMap = createFamilyCompressionMap(conf,tableName); final Map<byte[], BloomType> bloomTypeMap = createFamilyBloomTypeMap(conf,tableName); final Map<byte[], Integer> blockSizeMap = createFamilyBlockSizeMap(conf,tableName); // phoenix-2216: end - + String dataBlockEncodingStr = conf.get(DATABLOCK_ENCODING_OVERRIDE_CONF_KEY); final Map<byte[], DataBlockEncoding> datablockEncodingMap = createFamilyDataBlockEncodingMap(conf,tableName); final DataBlockEncoding overriddenEncoding; @@ -243,7 +244,7 @@ public class MultiHfileOutputFormat extends FileOutputFormat<TableRowkeyPair, Ce } else { overriddenEncoding = null; } - + Algorithm compression = compressionMap.get(family); compression = compression == null ? defaultCompression : compression; BloomType bloomType = bloomTypeMap.get(family); @@ -259,14 +260,17 @@ public class MultiHfileOutputFormat extends FileOutputFormat<TableRowkeyPair, Ce .withCompression(compression) .withChecksumType(HStore.getChecksumType(conf)) .withBytesPerCheckSum(HStore.getBytesPerChecksum(conf)) - .withBlockSize(blockSize); - contextBuilder.withDataBlockEncoding(encoding); + .withBlockSize(blockSize) + .withDataBlockEncoding(encoding); + CompatUtil.withComparator(contextBuilder, CellComparatorImpl.COMPARATOR); HFileContext hFileContext = contextBuilder.build(); - - wl.writer = new StoreFileWriter.Builder(conf, new CacheConfig(tempConf), fs) - .withOutputDir(familydir).withBloomType(bloomType) - .withComparator(CellComparatorImpl.COMPARATOR) - .withFileContext(hFileContext).build(); + + StoreFileWriter.Builder storeFileWriterBuilder = + new StoreFileWriter.Builder(conf, new CacheConfig(tempConf), fs) + .withOutputDir(familydir).withBloomType(bloomType) + .withFileContext(hFileContext); + CompatUtil.withComparator(storeFileWriterBuilder, CellComparatorImpl.COMPARATOR); + wl.writer = storeFileWriterBuilder.build(); // join and put it in the writers map . // phoenix-2216: start : holds a map of writers where the diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexScrutinyTool.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexScrutinyTool.java index 29fddf7..fd856fb 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexScrutinyTool.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexScrutinyTool.java @@ -42,6 +42,7 @@ import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.Admin; import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; +import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.util.FSUtils; import org.apache.hadoop.io.Text; import org.apache.hadoop.mapreduce.Job; @@ -327,7 +328,7 @@ public class IndexScrutinyTool extends Configured implements Tool { } // root dir not a subdirectory of hbase dir Path rootDir = new Path("hdfs:///index-snapshot-dir"); - FSUtils.setRootDir(configuration, rootDir); + CommonFSUtils.setRootDir(configuration, rootDir); // set input for map reduce job using hbase snapshots //PhoenixMapReduceUtil.setInput(job, PhoenixIndexDBWritable.class, snapshotName, diff --git a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java index 55eccba..0e3e5d8 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/mapreduce/index/IndexTool.java @@ -51,15 +51,12 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.conf.Configured; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; -import org.apache.hadoop.hbase.Cell; import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.hbase.HTableDescriptor; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.client.ConnectionFactory; import org.apache.hadoop.hbase.client.Admin; -import org.apache.hadoop.hbase.client.HBaseAdmin; -import org.apache.hadoop.hbase.client.HTable; import org.apache.hadoop.hbase.client.RegionLocator; import org.apache.hadoop.hbase.client.ResultScanner; import org.apache.hadoop.hbase.client.Scan; @@ -69,7 +66,7 @@ import org.apache.hadoop.hbase.mapreduce.TableInputFormat; import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil; import org.apache.hadoop.hbase.mapreduce.TableOutputFormat; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.io.IntWritable; import org.apache.hadoop.io.NullWritable; import org.apache.hadoop.mapreduce.Job; @@ -94,8 +91,6 @@ import org.apache.phoenix.mapreduce.util.PhoenixConfigurationUtil; import org.apache.phoenix.mapreduce.util.PhoenixMapReduceUtil; import org.apache.phoenix.parse.HintNode.Hint; import org.apache.phoenix.query.ConnectionQueryServices; -import org.apache.phoenix.query.HConnectionFactory; -import org.apache.phoenix.query.QueryConstants; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.query.QueryServicesOptions; import org.apache.phoenix.schema.PIndexState; @@ -658,8 +653,8 @@ public class IndexTool extends Configured implements Tool { } // root dir not a subdirectory of hbase dir Path rootDir = new Path("hdfs:///index-snapshot-dir"); - FSUtils.setRootDir(configuration, rootDir); - Path restoreDir = new Path(FSUtils.getRootDir(configuration), "restore-dir"); + CommonFSUtils.setRootDir(configuration, rootDir); + Path restoreDir = new Path(CommonFSUtils.getRootDir(configuration), "restore-dir"); // set input for map reduce job using hbase snapshots PhoenixMapReduceUtil diff --git a/phoenix-core/src/main/java/org/apache/phoenix/transaction/OmidTransactionTable.java b/phoenix-core/src/main/java/org/apache/phoenix/transaction/OmidTransactionTable.java index 5749d83..886ca60 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/transaction/OmidTransactionTable.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/transaction/OmidTransactionTable.java @@ -47,6 +47,7 @@ import org.apache.hadoop.hbase.filter.CompareFilter.CompareOp; import org.apache.hadoop.hbase.ipc.CoprocessorRpcChannel; import org.apache.omid.transaction.TTable; import org.apache.omid.transaction.Transaction; +import org.apache.phoenix.compat.hbase.CompatOmidTransactionTable; import org.apache.phoenix.exception.SQLExceptionCode; import org.apache.phoenix.exception.SQLExceptionInfo; @@ -55,7 +56,7 @@ import com.google.protobuf.Message; import com.google.protobuf.Service; import com.google.protobuf.ServiceException; -public class OmidTransactionTable implements Table { +public class OmidTransactionTable extends CompatOmidTransactionTable implements Table { // Copied from HBase ProtobufUtil since it's not accessible final static Result EMPTY_RESULT_EXISTS_TRUE = Result.create(null, true); @@ -375,4 +376,5 @@ public class OmidTransactionTable implements Table { public long getOperationTimeout(TimeUnit unit) { throw new UnsupportedOperationException(); } + } diff --git a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/recovery/TestPerRegionIndexWriteCache.java b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/recovery/TestPerRegionIndexWriteCache.java index 0ff7f29..d043f2c 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/recovery/TestPerRegionIndexWriteCache.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/hbase/index/write/recovery/TestPerRegionIndexWriteCache.java @@ -40,7 +40,7 @@ import org.apache.hadoop.hbase.client.TableDescriptor; import org.apache.hadoop.hbase.client.TableDescriptorBuilder; import org.apache.hadoop.hbase.regionserver.HRegion; import org.apache.hadoop.hbase.util.Bytes; -import org.apache.hadoop.hbase.util.FSUtils; +import org.apache.hadoop.hbase.util.CommonFSUtils; import org.apache.hadoop.hbase.wal.WAL; import org.apache.hadoop.hbase.wal.WALFactory; import org.apache.hadoop.hdfs.MiniDFSCluster; @@ -102,7 +102,7 @@ public class TestPerRegionIndexWriteCache { FileSystem newFS = miniDfs.getFileSystem(); RegionInfo hri = RegionInfoBuilder.newBuilder(tableName).setStartKey(null).setEndKey(null).setSplit(false).build(); - Path basedir = FSUtils.getTableDir(hbaseRootDir, tableName); + Path basedir = CommonFSUtils.getTableDir(hbaseRootDir, tableName); Random rn = new Random(); tableName = TableName.valueOf("TestPerRegion" + rn.nextInt()); WALFactory walFactory = new WALFactory(TEST_UTIL.getConfiguration(), getClass().getSimpleName()); diff --git a/phoenix-core/src/test/java/org/apache/phoenix/jdbc/PhoenixEmbeddedDriverTest.java b/phoenix-core/src/test/java/org/apache/phoenix/jdbc/PhoenixEmbeddedDriverTest.java index ba64892..1cc3f3f 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/jdbc/PhoenixEmbeddedDriverTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/jdbc/PhoenixEmbeddedDriverTest.java @@ -26,13 +26,20 @@ import java.sql.Driver; import java.sql.DriverManager; import java.sql.SQLException; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.HConstants; import org.apache.phoenix.exception.SQLExceptionCode; import org.apache.phoenix.jdbc.PhoenixEmbeddedDriver.ConnectionInfo; +import org.apache.phoenix.query.HBaseFactoryProvider; import org.junit.Test; public class PhoenixEmbeddedDriverTest { @Test public void testGetConnectionInfo() throws SQLException { + Configuration config = + HBaseFactoryProvider.getConfigurationFactory().getConfiguration(); + String defaultQuorum = config.get(HConstants.ZOOKEEPER_QUORUM); + String[] urls = new String[] { null, "", @@ -73,9 +80,9 @@ public class PhoenixEmbeddedDriverTest { "jdbc:phoenix:v1,v2,v3:345:/hbase:user1:C:\\Documents and Settings\\user1\\user1.keytab;test=false", }; ConnectionInfo[] infos = new ConnectionInfo[] { - new ConnectionInfo("localhost",2181,"/hbase"), - new ConnectionInfo("localhost",2181,"/hbase"), - new ConnectionInfo("localhost",2181,"/hbase"), + new ConnectionInfo(defaultQuorum, 2181, "/hbase"), + new ConnectionInfo(defaultQuorum, 2181, "/hbase"), + new ConnectionInfo(defaultQuorum, 2181, "/hbase"), new ConnectionInfo(null,null,null), new ConnectionInfo("localhost",null,null), new ConnectionInfo("localhost",null,null), diff --git a/phoenix-core/src/it/resources/compatible_client_versions.json b/phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatDelegateHTable.java similarity index 67% copy from phoenix-core/src/it/resources/compatible_client_versions.json copy to phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatDelegateHTable.java index 810265c..a55ae5b 100644 --- a/phoenix-core/src/it/resources/compatible_client_versions.json +++ b/phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatDelegateHTable.java @@ -15,12 +15,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -{ - "_comment": "Lists all phoenix compatible client versions against the current branch version for a given hbase profile If hbase profile is 1.3, phoenix client versions 4.14.3 and 4.15.0 are tested against current branch version", - "1.3": ["4.14.3", "4.15.0"], - "1.4": ["4.14.3", "4.15.0"], - "1.5": ["4.15.0"], - "2.0": ["5.1.0"], - "2.1": ["5.1.0"], - "2.2": ["5.1.0"] +package org.apache.phoenix.compat.hbase; + +import org.apache.hadoop.hbase.client.Table; + +public abstract class CompatDelegateHTable implements Table { + + protected final Table delegate; + + public CompatDelegateHTable(Table delegate) { + this.delegate = delegate; + } + } diff --git a/phoenix-core/src/it/resources/compatible_client_versions.json b/phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatOmidTransactionTable.java similarity index 67% copy from phoenix-core/src/it/resources/compatible_client_versions.json copy to phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatOmidTransactionTable.java index 810265c..d01868b 100644 --- a/phoenix-core/src/it/resources/compatible_client_versions.json +++ b/phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatOmidTransactionTable.java @@ -15,12 +15,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -{ - "_comment": "Lists all phoenix compatible client versions against the current branch version for a given hbase profile If hbase profile is 1.3, phoenix client versions 4.14.3 and 4.15.0 are tested against current branch version", - "1.3": ["4.14.3", "4.15.0"], - "1.4": ["4.14.3", "4.15.0"], - "1.5": ["4.15.0"], - "2.0": ["5.1.0"], - "2.1": ["5.1.0"], - "2.2": ["5.1.0"] +package org.apache.phoenix.compat.hbase; + +import org.apache.hadoop.hbase.client.Table; + +public abstract class CompatOmidTransactionTable implements Table { } diff --git a/phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatPermissionUtil.java b/phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatPermissionUtil.java index fdf572c..78068ab 100644 --- a/phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatPermissionUtil.java +++ b/phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatPermissionUtil.java @@ -19,6 +19,7 @@ package org.apache.phoenix.compat.hbase; import java.io.IOException; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.security.access.AccessChecker; @@ -26,11 +27,16 @@ import org.apache.hadoop.hbase.security.access.Permission; import org.apache.hadoop.hbase.security.access.TableAuthManager; import org.apache.hadoop.hbase.security.access.UserPermission; import org.apache.hadoop.hbase.util.Bytes; +import org.apache.hadoop.hbase.zookeeper.ZKWatcher; public class CompatPermissionUtil { private CompatPermissionUtil() { - //Nott to be instantiated + //Not to be instantiated + } + + public static AccessChecker newAccessChecker(final Configuration conf, ZKWatcher zk) { + return new AccessChecker(conf, zk); } public static void stopAccessChecker(AccessChecker accessChecker) throws IOException { diff --git a/phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatPhoenixRpcScheduler.java b/phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatPhoenixRpcScheduler.java index af25975..ff296f3 100644 --- a/phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatPhoenixRpcScheduler.java +++ b/phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatPhoenixRpcScheduler.java @@ -1,12 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. You may obtain a - * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable - * law or agreed to in writing, software distributed under the License is distributed on an "AS IS" - * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License - * for the specific language governing permissions and limitations under the License. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.apache.phoenix.compat.hbase; diff --git a/phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatUtil.java b/phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatStoreFileReader.java similarity index 55% copy from phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatUtil.java copy to phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatStoreFileReader.java index 4a9090c..9867ac6 100644 --- a/phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatUtil.java +++ b/phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatStoreFileReader.java @@ -18,25 +18,22 @@ package org.apache.phoenix.compat.hbase; import java.io.IOException; -import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.MetaTableAccessor; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.regionserver.StoreFileReader; -public class CompatUtil { +public class CompatStoreFileReader extends StoreFileReader { - private CompatUtil() { - //Not to be instantiated - } - - public static List<RegionInfo> getMergeRegions(Connection conn, byte[] regionName) + public CompatStoreFileReader(final FileSystem fs, final Path p, + final FSDataInputStreamWrapper in, long size, final CacheConfig cacheConf, + boolean primaryReplicaStoreFile, AtomicInteger refCount, final Configuration conf) throws IOException { - return MetaTableAccessor.getMergeRegions(conn, regionName); + super(fs, p, in, size, cacheConf, primaryReplicaStoreFile, refCount, false, conf); } - public static int getCellSerializedSize(Cell cell) { - return cell.getSerializedSize(); - } -} +} \ No newline at end of file diff --git a/phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatUtil.java b/phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatUtil.java index 61dfeec..7b9d36a 100644 --- a/phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatUtil.java +++ b/phoenix-hbase-compat-2.1.6/src/main/java/org/apache/phoenix/compat/hbase/CompatUtil.java @@ -17,26 +17,39 @@ */ package org.apache.phoenix.compat.hbase; -import java.io.IOException; -import java.util.List; - +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.MetaTableAccessor; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.CellComparatorImpl; + +import org.apache.hadoop.hbase.exceptions.DeserializationException; +import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; +import org.apache.hadoop.hbase.regionserver.StoreFileWriter; +import org.apache.hadoop.hbase.security.access.AccessControlLists; +import org.apache.hadoop.hbase.security.access.Permission; +import org.apache.hbase.thirdparty.com.google.common.collect.ListMultimap; public class CompatUtil { private CompatUtil() { - //Not to be instantiated - } - - public static List<RegionInfo> getMergeRegions(Connection conn, byte[] regionName) - throws IOException { - return MetaTableAccessor.getMergeRegions(conn, regionName); + // Not to be instantiated } public static int getCellSerializedSize(Cell cell) { return org.apache.hadoop.hbase.KeyValueUtil.length(cell); } + + public static ListMultimap<String, ? extends Permission> readPermissions(byte[] data, + Configuration conf) throws DeserializationException { + return AccessControlLists.readPermissions(data, conf); + } + + public static HFileContextBuilder withComparator(HFileContextBuilder contextBuilder, + CellComparatorImpl cellComparator) { + return contextBuilder; + } + + public static StoreFileWriter.Builder withComparator(StoreFileWriter.Builder builder, + CellComparatorImpl cellComparator) { + return builder.withComparator(cellComparator); + } } diff --git a/phoenix-core/src/it/resources/compatible_client_versions.json b/phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatDelegateHTable.java similarity index 67% copy from phoenix-core/src/it/resources/compatible_client_versions.json copy to phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatDelegateHTable.java index 810265c..a55ae5b 100644 --- a/phoenix-core/src/it/resources/compatible_client_versions.json +++ b/phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatDelegateHTable.java @@ -15,12 +15,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -{ - "_comment": "Lists all phoenix compatible client versions against the current branch version for a given hbase profile If hbase profile is 1.3, phoenix client versions 4.14.3 and 4.15.0 are tested against current branch version", - "1.3": ["4.14.3", "4.15.0"], - "1.4": ["4.14.3", "4.15.0"], - "1.5": ["4.15.0"], - "2.0": ["5.1.0"], - "2.1": ["5.1.0"], - "2.2": ["5.1.0"] +package org.apache.phoenix.compat.hbase; + +import org.apache.hadoop.hbase.client.Table; + +public abstract class CompatDelegateHTable implements Table { + + protected final Table delegate; + + public CompatDelegateHTable(Table delegate) { + this.delegate = delegate; + } + } diff --git a/phoenix-core/src/it/resources/compatible_client_versions.json b/phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatOmidTransactionTable.java similarity index 67% copy from phoenix-core/src/it/resources/compatible_client_versions.json copy to phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatOmidTransactionTable.java index 810265c..d01868b 100644 --- a/phoenix-core/src/it/resources/compatible_client_versions.json +++ b/phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatOmidTransactionTable.java @@ -15,12 +15,9 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -{ - "_comment": "Lists all phoenix compatible client versions against the current branch version for a given hbase profile If hbase profile is 1.3, phoenix client versions 4.14.3 and 4.15.0 are tested against current branch version", - "1.3": ["4.14.3", "4.15.0"], - "1.4": ["4.14.3", "4.15.0"], - "1.5": ["4.15.0"], - "2.0": ["5.1.0"], - "2.1": ["5.1.0"], - "2.2": ["5.1.0"] +package org.apache.phoenix.compat.hbase; + +import org.apache.hadoop.hbase.client.Table; + +public abstract class CompatOmidTransactionTable implements Table { } diff --git a/phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatPermissionUtil.java b/phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatPermissionUtil.java index 84b304b..4a13cfd 100644 --- a/phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatPermissionUtil.java +++ b/phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatPermissionUtil.java @@ -19,11 +19,13 @@ package org.apache.phoenix.compat.hbase; import java.io.IOException; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.security.access.AccessChecker; import org.apache.hadoop.hbase.security.access.Permission; import org.apache.hadoop.hbase.security.access.UserPermission; +import org.apache.hadoop.hbase.zookeeper.ZKWatcher; public class CompatPermissionUtil { @@ -31,6 +33,10 @@ public class CompatPermissionUtil { //Not to be instantiated } + public static AccessChecker newAccessChecker(final Configuration conf, ZKWatcher zk) { + return new AccessChecker(conf, zk); + } + public static void stopAccessChecker(AccessChecker accessChecker) throws IOException { accessChecker.stop(); } diff --git a/phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatPhoenixRpcScheduler.java b/phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatPhoenixRpcScheduler.java index 059a39e..13d69a7 100644 --- a/phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatPhoenixRpcScheduler.java +++ b/phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatPhoenixRpcScheduler.java @@ -1,12 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. You may obtain a - * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable - * law or agreed to in writing, software distributed under the License is distributed on an "AS IS" - * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License - * for the specific language governing permissions and limitations under the License. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.apache.phoenix.compat.hbase; diff --git a/phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatUtil.java b/phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatStoreFileReader.java similarity index 55% copy from phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatUtil.java copy to phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatStoreFileReader.java index 4a9090c..9867ac6 100644 --- a/phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatUtil.java +++ b/phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatStoreFileReader.java @@ -18,25 +18,22 @@ package org.apache.phoenix.compat.hbase; import java.io.IOException; -import java.util.List; +import java.util.concurrent.atomic.AtomicInteger; -import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.MetaTableAccessor; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.regionserver.StoreFileReader; -public class CompatUtil { +public class CompatStoreFileReader extends StoreFileReader { - private CompatUtil() { - //Not to be instantiated - } - - public static List<RegionInfo> getMergeRegions(Connection conn, byte[] regionName) + public CompatStoreFileReader(final FileSystem fs, final Path p, + final FSDataInputStreamWrapper in, long size, final CacheConfig cacheConf, + boolean primaryReplicaStoreFile, AtomicInteger refCount, final Configuration conf) throws IOException { - return MetaTableAccessor.getMergeRegions(conn, regionName); + super(fs, p, in, size, cacheConf, primaryReplicaStoreFile, refCount, false, conf); } - public static int getCellSerializedSize(Cell cell) { - return cell.getSerializedSize(); - } -} +} \ No newline at end of file diff --git a/phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatUtil.java b/phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatUtil.java index 4a9090c..b2cefe4 100644 --- a/phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatUtil.java +++ b/phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatUtil.java @@ -17,26 +17,38 @@ */ package org.apache.phoenix.compat.hbase; -import java.io.IOException; -import java.util.List; - +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.Cell; -import org.apache.hadoop.hbase.MetaTableAccessor; -import org.apache.hadoop.hbase.client.Connection; -import org.apache.hadoop.hbase.client.RegionInfo; +import org.apache.hadoop.hbase.CellComparatorImpl; +import org.apache.hadoop.hbase.exceptions.DeserializationException; +import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; +import org.apache.hadoop.hbase.regionserver.StoreFileWriter; +import org.apache.hadoop.hbase.security.access.AccessControlLists; +import org.apache.hadoop.hbase.security.access.Permission; +import org.apache.hbase.thirdparty.com.google.common.collect.ListMultimap; public class CompatUtil { private CompatUtil() { - //Not to be instantiated - } - - public static List<RegionInfo> getMergeRegions(Connection conn, byte[] regionName) - throws IOException { - return MetaTableAccessor.getMergeRegions(conn, regionName); + // Not to be instantiated } public static int getCellSerializedSize(Cell cell) { return cell.getSerializedSize(); } + + public static ListMultimap<String, ? extends Permission> readPermissions(byte[] data, + Configuration conf) throws DeserializationException { + return AccessControlLists.readPermissions(data, conf); + } + + public static HFileContextBuilder withComparator(HFileContextBuilder contextBuilder, + CellComparatorImpl cellComparator) { + return contextBuilder; + } + + public static StoreFileWriter.Builder withComparator(StoreFileWriter.Builder builder, + CellComparatorImpl cellComparator) { + return builder.withComparator(cellComparator); + } } diff --git a/phoenix-hbase-compat-2.3.0/pom.xml b/phoenix-hbase-compat-2.3.0/pom.xml new file mode 100644 index 0000000..045c793 --- /dev/null +++ b/phoenix-hbase-compat-2.3.0/pom.xml @@ -0,0 +1,88 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software Foundation (ASF) under one or more + contributor license agreements. See the NOTICE file distributed with + this work for additional information regarding copyright ownership. + The ASF licenses this file to You under the Apache License, Version 2.0 + (the "License"); you may not use this file except in compliance with + the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +--> +<project + xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" + xsi:schemaLocation= + "http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache</groupId> + <artifactId>apache</artifactId> + <version>21</version> + </parent> + <groupId>org.apache.phoenix</groupId> + <artifactId>phoenix-hbase-compat-2.3.0</artifactId> + <version>5.1.0-SNAPSHOT</version> + <name>Phoenix Hbase 2.3.0 compatibility</name> + <description>Compatibility module for HBase 2.3.0+</description> + + <build> + <plugins> + <!-- Setup eclipse --> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-eclipse-plugin</artifactId> + <version>2.9</version> + <configuration> + <buildcommands> + <buildcommand>org.eclipse.jdt.core.javabuilder</buildcommand> + </buildcommands> + </configuration> + </plugin> + <plugin> + <groupId>org.apache.rat</groupId> + <artifactId>apache-rat-plugin</artifactId> + </plugin> + </plugins> + </build> + + <dependencies> + <!-- HBase dependencies --> + <dependency> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-client</artifactId> + <version>2.3.0</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-common</artifactId> + <version>2.3.0</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-server</artifactId> + <version>2.3.0</version> + <scope>provided</scope> + </dependency> + </dependencies> + + <reporting> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-project-info-reports-plugin</artifactId> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>findbugs-maven-plugin</artifactId> + </plugin> + </plugins> + </reporting> +</project> diff --git a/phoenix-core/src/it/resources/compatible_client_versions.json b/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatDelegateHTable.java similarity index 62% copy from phoenix-core/src/it/resources/compatible_client_versions.json copy to phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatDelegateHTable.java index 810265c..770e858 100644 --- a/phoenix-core/src/it/resources/compatible_client_versions.json +++ b/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatDelegateHTable.java @@ -15,12 +15,23 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -{ - "_comment": "Lists all phoenix compatible client versions against the current branch version for a given hbase profile If hbase profile is 1.3, phoenix client versions 4.14.3 and 4.15.0 are tested against current branch version", - "1.3": ["4.14.3", "4.15.0"], - "1.4": ["4.14.3", "4.15.0"], - "1.5": ["4.15.0"], - "2.0": ["5.1.0"], - "2.1": ["5.1.0"], - "2.2": ["5.1.0"] +package org.apache.phoenix.compat.hbase; + +import java.io.IOException; + +import org.apache.hadoop.hbase.client.RegionLocator; +import org.apache.hadoop.hbase.client.Table; + +public abstract class CompatDelegateHTable implements Table { + + protected final Table delegate; + + public CompatDelegateHTable(Table delegate) { + this.delegate = delegate; + } + + @Override + public RegionLocator getRegionLocator() throws IOException { + return delegate.getRegionLocator(); + } } diff --git a/phoenix-core/src/it/resources/compatible_client_versions.json b/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatOmidTransactionTable.java similarity index 67% copy from phoenix-core/src/it/resources/compatible_client_versions.json copy to phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatOmidTransactionTable.java index 810265c..e1396ca 100644 --- a/phoenix-core/src/it/resources/compatible_client_versions.json +++ b/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatOmidTransactionTable.java @@ -15,12 +15,17 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -{ - "_comment": "Lists all phoenix compatible client versions against the current branch version for a given hbase profile If hbase profile is 1.3, phoenix client versions 4.14.3 and 4.15.0 are tested against current branch version", - "1.3": ["4.14.3", "4.15.0"], - "1.4": ["4.14.3", "4.15.0"], - "1.5": ["4.15.0"], - "2.0": ["5.1.0"], - "2.1": ["5.1.0"], - "2.2": ["5.1.0"] +package org.apache.phoenix.compat.hbase; + +import java.io.IOException; + +import org.apache.hadoop.hbase.client.RegionLocator; +import org.apache.hadoop.hbase.client.Table; + +public abstract class CompatOmidTransactionTable implements Table { + + @Override + public RegionLocator getRegionLocator() throws IOException { + throw new UnsupportedOperationException(); + } } diff --git a/phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatPermissionUtil.java b/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatPermissionUtil.java similarity index 87% copy from phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatPermissionUtil.java copy to phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatPermissionUtil.java index 84b304b..80a99b3 100644 --- a/phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatPermissionUtil.java +++ b/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatPermissionUtil.java @@ -19,11 +19,13 @@ package org.apache.phoenix.compat.hbase; import java.io.IOException; +import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.hbase.TableName; import org.apache.hadoop.hbase.security.User; import org.apache.hadoop.hbase.security.access.AccessChecker; import org.apache.hadoop.hbase.security.access.Permission; import org.apache.hadoop.hbase.security.access.UserPermission; +import org.apache.hadoop.hbase.zookeeper.ZKWatcher; public class CompatPermissionUtil { @@ -31,8 +33,13 @@ public class CompatPermissionUtil { //Not to be instantiated } + public static AccessChecker newAccessChecker(final Configuration conf, ZKWatcher zk) { + //Ignore ZK parameter + return new AccessChecker(conf); + } + public static void stopAccessChecker(AccessChecker accessChecker) throws IOException { - accessChecker.stop(); + //NOOP } public static String getUserFromUP(UserPermission userPermission) { diff --git a/phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatPhoenixRpcScheduler.java b/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatPhoenixRpcScheduler.java similarity index 57% copy from phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatPhoenixRpcScheduler.java copy to phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatPhoenixRpcScheduler.java index 059a39e..13d69a7 100644 --- a/phoenix-hbase-compat-2.2.1/src/main/java/org/apache/phoenix/compat/hbase/CompatPhoenixRpcScheduler.java +++ b/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatPhoenixRpcScheduler.java @@ -1,12 +1,19 @@ /* - * Licensed to the Apache Software Foundation (ASF) under one or more contributor license - * agreements. See the NOTICE file distributed with this work for additional information regarding - * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance with the License. You may obtain a - * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable - * law or agreed to in writing, software distributed under the License is distributed on an "AS IS" - * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License - * for the specific language governing permissions and limitations under the License. + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. */ package org.apache.phoenix.compat.hbase; diff --git a/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatStoreFileReader.java b/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatStoreFileReader.java new file mode 100644 index 0000000..03aa257 --- /dev/null +++ b/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatStoreFileReader.java @@ -0,0 +1,48 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.compat.hbase; + +import java.io.IOException; +import java.util.concurrent.atomic.AtomicInteger; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.fs.FileSystem; +import org.apache.hadoop.fs.Path; +import org.apache.hadoop.hbase.fs.HFileSystem; +import org.apache.hadoop.hbase.io.FSDataInputStreamWrapper; +import org.apache.hadoop.hbase.io.hfile.CacheConfig; +import org.apache.hadoop.hbase.io.hfile.HFileInfo; +import org.apache.hadoop.hbase.io.hfile.ReaderContext; +import org.apache.hadoop.hbase.io.hfile.ReaderContext.ReaderType; +import org.apache.hadoop.hbase.regionserver.StoreFileReader; + +public class CompatStoreFileReader extends StoreFileReader { + + public CompatStoreFileReader(final FileSystem fs, final Path p, + final FSDataInputStreamWrapper in, long size, final CacheConfig cacheConf, + boolean primaryReplicaStoreFile, AtomicInteger refCount, final Configuration conf) + throws IOException { + super(new ReaderContext(p, in, size, new HFileSystem(fs), primaryReplicaStoreFile, + ReaderType.STREAM), + new HFileInfo(new ReaderContext(p, in, size, new HFileSystem(fs), + primaryReplicaStoreFile, ReaderType.STREAM), conf), + cacheConf, refCount, conf); + getHFileReader().getHFileInfo().initMetaAndIndex(getHFileReader()); + } + +} \ No newline at end of file diff --git a/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatUtil.java b/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatUtil.java new file mode 100644 index 0000000..efe6efb --- /dev/null +++ b/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/CompatUtil.java @@ -0,0 +1,72 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.compat.hbase; + +import org.apache.hadoop.conf.Configuration; +import org.apache.hadoop.hbase.Cell; +import org.apache.hadoop.hbase.CellComparator; +import org.apache.hadoop.hbase.CellComparatorImpl; +import org.apache.hadoop.hbase.exceptions.DeserializationException; +import org.apache.hadoop.hbase.io.compress.Compression.Algorithm; +import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding; +import org.apache.hadoop.hbase.io.hfile.HFileContext; +import org.apache.hadoop.hbase.io.hfile.HFileContextBuilder; +import org.apache.hadoop.hbase.regionserver.HStore; +import org.apache.hadoop.hbase.regionserver.StoreFileWriter; +import org.apache.hadoop.hbase.security.access.Permission; +import org.apache.hadoop.hbase.security.access.PermissionStorage; +import org.apache.hbase.thirdparty.com.google.common.collect.ListMultimap; + + +public class CompatUtil { + + private CompatUtil() { + //Not to be instantiated + } + + public static int getCellSerializedSize(Cell cell) { + return cell.getSerializedSize(); + } + + public static ListMultimap<String, ? extends Permission> readPermissions( + byte[] data, Configuration conf) throws DeserializationException { + return PermissionStorage.readPermissions(data, conf); + } + + public static HFileContext createHFileContext(Configuration conf, Algorithm compression, + Integer blockSize, DataBlockEncoding encoding, CellComparator comparator) { + + return new HFileContextBuilder() + .withCompression(compression) + .withChecksumType(HStore.getChecksumType(conf)) + .withBytesPerCheckSum(HStore.getBytesPerChecksum(conf)) + .withBlockSize(blockSize) + .withDataBlockEncoding(encoding) + .build(); + } + + public static HFileContextBuilder withComparator(HFileContextBuilder contextBuilder, + CellComparatorImpl cellComparator) { + return contextBuilder.withCellComparator(cellComparator); + } + + public static StoreFileWriter.Builder withComparator(StoreFileWriter.Builder builder, + CellComparatorImpl cellComparator) { + return builder; + } +} diff --git a/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/OffsetCell.java b/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/OffsetCell.java new file mode 100644 index 0000000..c5485a5 --- /dev/null +++ b/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/OffsetCell.java @@ -0,0 +1,136 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.compat.hbase; + +import org.apache.hadoop.hbase.Cell; + +public class OffsetCell implements Cell { + + private Cell cell; + private int offset; + + public OffsetCell(Cell cell, int offset) { + this.cell = cell; + this.offset = offset; + } + + @Override + public byte[] getRowArray() { + return cell.getRowArray(); + } + + @Override + public int getRowOffset() { + return cell.getRowOffset() + offset; + } + + @Override + public short getRowLength() { + return (short) (cell.getRowLength() - offset); + } + + @Override + public byte[] getFamilyArray() { + return cell.getFamilyArray(); + } + + @Override + public int getFamilyOffset() { + return cell.getFamilyOffset(); + } + + @Override + public byte getFamilyLength() { + return cell.getFamilyLength(); + } + + @Override + public byte[] getQualifierArray() { + return cell.getQualifierArray(); + } + + @Override + public int getQualifierOffset() { + return cell.getQualifierOffset(); + } + + @Override + public int getQualifierLength() { + return cell.getQualifierLength(); + } + + @Override + public long getTimestamp() { + return cell.getTimestamp(); + } + + @Override + public byte getTypeByte() { + return cell.getTypeByte(); + } + + @Override public long getSequenceId() { + return cell.getSequenceId(); + } + + @Override + public byte[] getValueArray() { + return cell.getValueArray(); + } + + @Override + public int getValueOffset() { + return cell.getValueOffset(); + } + + @Override + public int getValueLength() { + return cell.getValueLength(); + } + + @Override + public byte[] getTagsArray() { + return cell.getTagsArray(); + } + + @Override + public int getTagsOffset() { + return cell.getTagsOffset(); + } + + @Override + public int getTagsLength() { + return cell.getTagsLength(); + } + + @Override + public Type getType() { + return cell.getType(); + } + + @Override + public long heapSize() { + return cell.heapSize(); + } + + @Override + public int getSerializedSize() { + return cell.getSerializedSize() - offset; + } + +} diff --git a/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/test/DelegateCell.java b/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/test/DelegateCell.java new file mode 100644 index 0000000..0c7dfd8 --- /dev/null +++ b/phoenix-hbase-compat-2.3.0/src/main/java/org/apache/phoenix/compat/hbase/test/DelegateCell.java @@ -0,0 +1,139 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.phoenix.compat.hbase.test; + +import org.apache.hadoop.hbase.Cell; + +public class DelegateCell implements Cell { + private final Cell delegate; + private final String name; + public DelegateCell(Cell delegate, String name) { + this.delegate = delegate; + this.name = name; + } + + @Override + public int getValueOffset() { + return delegate.getValueOffset(); + } + + @Override + public int getValueLength() { + return delegate.getValueLength(); + } + + @Override + public byte[] getValueArray() { + return delegate.getValueArray(); + } + + @Override + public byte getTypeByte() { + return delegate.getTypeByte(); + } + + @Override + public long getTimestamp() { + return delegate.getTimestamp(); + } + + @Override + public int getTagsOffset() { + return delegate.getTagsOffset(); + } + + @Override + public byte[] getTagsArray() { + return delegate.getTagsArray(); + } + + @Override + public int getRowOffset() { + return delegate.getRowOffset(); + } + + @Override + public short getRowLength() { + return delegate.getRowLength(); + } + + @Override + public byte[] getRowArray() { + return delegate.getRowArray(); + } + + @Override + public int getQualifierOffset() { + return delegate.getQualifierOffset(); + } + + @Override + public int getQualifierLength() { + return delegate.getQualifierLength(); + } + + @Override + public byte[] getQualifierArray() { + return delegate.getQualifierArray(); + } + + @Override + public int getFamilyOffset() { + return delegate.getFamilyOffset(); + } + + @Override + public byte getFamilyLength() { + return delegate.getFamilyLength(); + } + + @Override + public byte[] getFamilyArray() { + return delegate.getFamilyArray(); + } + + @Override + public String toString() { + return name; + } + + @Override + public long getSequenceId() { + return delegate.getSequenceId(); + } + + @Override + public int getTagsLength() { + return delegate.getTagsLength(); + } + + @Override + public Type getType() { + return delegate.getType(); + } + + @Override + public long heapSize() { + return delegate.heapSize(); + } + + @Override + public int getSerializedSize() { + return delegate.getSerializedSize(); + } +} diff --git a/pom.xml b/pom.xml index c6c755a..63d1def 100644 --- a/pom.xml +++ b/pom.xml @@ -40,6 +40,7 @@ </organization> <modules> + <module>phoenix-hbase-compat-2.3.0</module> <module>phoenix-hbase-compat-2.2.1</module> <module>phoenix-hbase-compat-2.1.6</module> <module>phoenix-core</module> @@ -436,20 +437,15 @@ hbasePatch = Integer.parseInt(versionMatcher.group(3)); hbaseMajor == 2 && ( - ("${hbase.compat.version}".equals("2.0.1") - && hbaseMinor == 0 - && hbasePatch >=1 - && hbasePatch <=4) - || ("${hbase.compat.version}".equals("2.0.1") - && hbaseMinor == 1 - && hbasePatch >=1 - && hbasePatch <=2) - || ("${hbase.compat.version}".equals("2.1.6") + ("${hbase.compat.version}".equals("2.1.6") && hbaseMinor == 1 && hbasePatch >=6) || ("${hbase.compat.version}".equals("2.2.1") && hbaseMinor == 2 && hbasePatch >=1) + || ("${hbase.compat.version}".equals("2.3.0") + && hbaseMinor == 3 + && hbasePatch >=0) ) </condition> </evaluateBeanshell> @@ -635,7 +631,11 @@ <artifactId>phoenix-hbase-compat-2.2.1</artifactId> <version>${project.version}</version> </dependency> - + <dependency> + <groupId>org.apache.phoenix</groupId> + <artifactId>phoenix-hbase-compat-2.3.0</artifactId> + <version>${project.version}</version> + </dependency> <!-- Intra-project test dependencies --> <dependency> <groupId>org.apache.phoenix</groupId> @@ -1205,6 +1205,11 @@ <version>${zookeeper.version}</version> </dependency> <dependency> + <groupId>org.apache.zookeeper</groupId> + <artifactId>zookeeper-jute</artifactId> + <version>${zookeeper.version}</version> + </dependency> + <dependency> <groupId>org.antlr</groupId> <artifactId>antlr-runtime</artifactId> <version>${antlr.version}</version> @@ -1471,8 +1476,31 @@ <properties> <hbase.profile>2.2</hbase.profile> <hbase.compat.version>2.2.1</hbase.compat.version> - <!-- PHOENIX-5993 HBase 2.2.5 public maven artifacts are incompatible with Hadoop 3 --> <hbase.version>2.2.4</hbase.version> + <!-- PHOENIX-5993 We should use this, if we could, + as 2.2.4 is not the latest version, Hadoop 3.0.3 is unsupported with Hbase 2.2.x + <hbase.version>2.2.5</hbase.version> + <hadoop.version>3.1.2</hadoop.version> + --> + </properties> + </profile> + <profile> + <!-- PHOENIX-5993 This won't work with the public HBase artifacts --> + <id>phoenix-hbase-compat-2.3.0</id> + <activation> + <property> + <name>hbase.profile</name> + <value>2.3</value> + </property> + </activation> + <properties> + <hbase.profile>2.3</hbase.profile> + <hbase.compat.version>2.3.0</hbase.compat.version> + <hbase.version>2.3.0</hbase.version> + <!-- PHOENIX-6010 3.1.3 doesn't work because of Guava rebase --> + <hadoop.version>3.1.2</hadoop.version> + <hbase.thirdparty.version>3.3.0</hbase.thirdparty.version> + <zookeeper.version>3.5.7</zookeeper.version> </properties> </profile> <profile> @@ -1486,8 +1514,11 @@ <properties> <hbase.profile>2.2</hbase.profile> <hbase.compat.version>2.2.1</hbase.compat.version> - <!-- PHOENIX-5993 HBase 2.2.5 public maven artifacts are incompatible with Hadoop 3 --> <hbase.version>2.2.4</hbase.version> + <!-- PHOENIX-5993 We should use this, if we could + <hbase.version>2.2.5</hbase.version> + <hadoop.version>3.1.2</hadoop.version> + --> </properties> </profile> <profile>