PHOENIX-4372 Distribution of Apache Phoenix 4.13 for CDH 5.11.2 (Pedro Boado)
Project: http://git-wip-us.apache.org/repos/asf/phoenix/repo Commit: http://git-wip-us.apache.org/repos/asf/phoenix/commit/024f0f22 Tree: http://git-wip-us.apache.org/repos/asf/phoenix/tree/024f0f22 Diff: http://git-wip-us.apache.org/repos/asf/phoenix/diff/024f0f22 Branch: refs/heads/4.x-cdh5.11.2 Commit: 024f0f22a5929da6f095dc0025b8e899e2f0c47b Parents: 0c67de3 Author: James Taylor <jtay...@salesforce.com> Authored: Mon Nov 27 11:38:36 2017 -0800 Committer: James Taylor <jtay...@salesforce.com> Committed: Mon Nov 27 11:38:36 2017 -0800 ---------------------------------------------------------------------- phoenix-assembly/pom.xml | 2 +- .../components/all-common-dependencies.xml | 2 +- phoenix-client/pom.xml | 4 +- phoenix-core/pom.xml | 50 +++- .../phoenix/end2end/IndexScrutinyToolIT.java | 6 +- .../MigrateSystemTablesToSystemNamespaceIT.java | 2 + .../end2end/NamespaceSchemaMappingIT.java | 11 +- .../end2end/QueryDatabaseMetaDataIT.java | 6 +- .../apache/phoenix/end2end/SetPropertyIT.java | 6 +- .../end2end/SystemTablePermissionsIT.java | 2 + .../org/apache/phoenix/end2end/UseSchemaIT.java | 5 +- .../phoenix/trace/PhoenixTracingEndToEndIT.java | 13 +- .../apache/phoenix/execute/DelegateHTable.java | 22 +- .../index/metrics/MetricsIndexerSourceImpl.java | 16 +- .../org/apache/phoenix/trace/TraceReader.java | 13 +- .../org/apache/phoenix/trace/TraceWriter.java | 6 +- .../org/apache/phoenix/trace/TracingUtils.java | 7 + .../org/apache/phoenix/trace/util/NullSpan.java | 20 +- .../org/apache/phoenix/trace/util/Tracing.java | 7 +- .../transaction/OmidTransactionTable.java | 24 ++ .../transaction/TephraTransactionTable.java | 20 ++ .../apache/phoenix/util/PhoenixMRJobUtil.java | 15 +- .../phoenix/trace/TraceSpanReceiverTest.java | 8 +- phoenix-flume/pom.xml | 2 +- phoenix-hive/pom.xml | 2 +- .../apache/phoenix/hive/HivePhoenixStoreIT.java | 2 + .../java/org/apache/phoenix/hive/HiveTezIT.java | 1 + phoenix-kafka/pom.xml | 2 +- phoenix-load-balancer/pom.xml | 2 +- .../phoenix/end2end/LoadBalancerEnd2EndIT.java | 2 +- phoenix-parcel/pom.xml | 252 +++++++++++++++++++ .../components/all-common-dependencies.xml | 56 +++++ .../src/build/components/all-common-files.xml | 84 +++++++ .../src/build/components/all-common-jars.xml | 217 ++++++++++++++++ .../src/build/manifest/make_manifest.py | 117 +++++++++ phoenix-parcel/src/build/parcel.xml | 40 +++ .../src/parcel/bin/phoenix-performance.py | 19 ++ phoenix-parcel/src/parcel/bin/phoenix-psql.py | 19 ++ .../src/parcel/bin/phoenix-sqlline.py | 20 ++ phoenix-parcel/src/parcel/bin/phoenix-utils.py | 19 ++ .../src/parcel/cloudera/cdh_version.properties | 0 .../src/parcel/meta/alternatives.json | 26 ++ phoenix-parcel/src/parcel/meta/parcel.json | 34 +++ phoenix-parcel/src/parcel/meta/phoenix_env.sh | 27 ++ phoenix-pherf/pom.xml | 2 +- phoenix-pig/pom.xml | 3 +- phoenix-queryserver-client/pom.xml | 2 +- phoenix-queryserver/pom.xml | 2 +- .../HttpParamImpersonationQueryServerIT.java | 2 +- .../org/apache/phoenix/end2end/KdcUtil.java | 48 ++++ .../phoenix/end2end/SecureQueryServerIT.java | 2 +- phoenix-server/pom.xml | 4 +- phoenix-spark/pom.xml | 2 +- phoenix-tracing-webapp/pom.xml | 2 +- pom.xml | 109 ++++---- 55 files changed, 1271 insertions(+), 117 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/phoenix/blob/024f0f22/phoenix-assembly/pom.xml ---------------------------------------------------------------------- diff --git a/phoenix-assembly/pom.xml b/phoenix-assembly/pom.xml index e9e9f02..27631c8 100644 --- a/phoenix-assembly/pom.xml +++ b/phoenix-assembly/pom.xml @@ -27,7 +27,7 @@ <parent> <groupId>org.apache.phoenix</groupId> <artifactId>phoenix</artifactId> - <version>4.13.0-HBase-1.2</version> + <version>4.13.0-cdh5.11.2</version> </parent> <artifactId>phoenix-assembly</artifactId> <name>Phoenix Assembly</name> http://git-wip-us.apache.org/repos/asf/phoenix/blob/024f0f22/phoenix-assembly/src/build/components/all-common-dependencies.xml ---------------------------------------------------------------------- diff --git a/phoenix-assembly/src/build/components/all-common-dependencies.xml b/phoenix-assembly/src/build/components/all-common-dependencies.xml index 336bc4d..6bde495 100644 --- a/phoenix-assembly/src/build/components/all-common-dependencies.xml +++ b/phoenix-assembly/src/build/components/all-common-dependencies.xml @@ -41,7 +41,7 @@ <include>log4j:log4j</include> <include>org.apache.hbase:hbase*</include> <include>org.antlr:antlr-runtime</include> - <include>org.cloudera.htrace:htrace-core</include> + <include>org.apache.htrace:htrace-core</include> <include>io.netty:netty</include> <include>commons-codec:commons-codec</include> <include>org.apache.calcite:calcite-avatica*</include> http://git-wip-us.apache.org/repos/asf/phoenix/blob/024f0f22/phoenix-client/pom.xml ---------------------------------------------------------------------- diff --git a/phoenix-client/pom.xml b/phoenix-client/pom.xml index d64d3dc..a42a8aa 100644 --- a/phoenix-client/pom.xml +++ b/phoenix-client/pom.xml @@ -27,7 +27,7 @@ <parent> <groupId>org.apache.phoenix</groupId> <artifactId>phoenix</artifactId> - <version>4.13.0-HBase-1.2</version> + <version>4.13.0-cdh5.11.2</version> </parent> <artifactId>phoenix-client</artifactId> <name>Phoenix Client</name> @@ -129,6 +129,8 @@ <excludes> <exclude>org.apache.phoenix:phoenix-client</exclude> <exclude>xom:xom</exclude> + <exclude>log4j:log4j</exclude> + <exclude>org.slf4j:slf4j-log4j12</exclude> </excludes> </artifactSet> <filters> http://git-wip-us.apache.org/repos/asf/phoenix/blob/024f0f22/phoenix-core/pom.xml ---------------------------------------------------------------------- diff --git a/phoenix-core/pom.xml b/phoenix-core/pom.xml index 58dd510..716fa6f 100644 --- a/phoenix-core/pom.xml +++ b/phoenix-core/pom.xml @@ -4,7 +4,7 @@ <parent> <groupId>org.apache.phoenix</groupId> <artifactId>phoenix</artifactId> - <version>4.13.0-HBase-1.2</version> + <version>4.13.0-cdh5.11.2</version> </parent> <artifactId>phoenix-core</artifactId> <name>Phoenix Core</name> @@ -249,7 +249,7 @@ </dependency> <dependency> <groupId>org.apache.tephra</groupId> - <artifactId>tephra-hbase-compat-1.1</artifactId> + <artifactId>tephra-hbase-compat-1.2-cdh</artifactId> </dependency> <!-- Make sure we have all the antlr dependencies --> @@ -375,12 +375,24 @@ <artifactId>hbase-testing-util</artifactId> <scope>test</scope> <optional>true</optional> + <exclusions> + <exclusion> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-core</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>org.apache.hbase</groupId> <artifactId>hbase-it</artifactId> <type>test-jar</type> <scope>test</scope> + <exclusions> + <exclusion> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-core</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>org.apache.hbase</groupId> @@ -389,12 +401,24 @@ <dependency> <groupId>org.apache.hbase</groupId> <artifactId>hbase-common</artifactId> + <exclusions> + <exclusion> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-core</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>org.apache.hbase</groupId> <artifactId>hbase-common</artifactId> <scope>test</scope> <type>test-jar</type> + <exclusions> + <exclusion> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-core</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>org.apache.hbase</groupId> @@ -412,6 +436,10 @@ <groupId>xom</groupId> <artifactId>xom</artifactId> </exclusion> + <exclusion> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-core</artifactId> + </exclusion> </exclusions> </dependency> <dependency> @@ -419,6 +447,12 @@ <artifactId>hbase-server</artifactId> <type>test-jar</type> <scope>test</scope> + <exclusions> + <exclusion> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-core</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>org.apache.hbase</groupId> @@ -433,12 +467,24 @@ <dependency> <groupId>org.apache.hbase</groupId> <artifactId>hbase-hadoop2-compat</artifactId> + <exclusions> + <exclusion> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-core</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>org.apache.hbase</groupId> <artifactId>hbase-hadoop2-compat</artifactId> <type>test-jar</type> <scope>test</scope> + <exclusions> + <exclusion> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-core</artifactId> + </exclusion> + </exclusions> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> http://git-wip-us.apache.org/repos/asf/phoenix/blob/024f0f22/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexScrutinyToolIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexScrutinyToolIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexScrutinyToolIT.java index cbce7b2..a9f248b 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexScrutinyToolIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/IndexScrutinyToolIT.java @@ -68,6 +68,7 @@ import org.apache.phoenix.util.SchemaUtil; import org.junit.After; import org.junit.Before; import org.junit.BeforeClass; +import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -79,6 +80,7 @@ import org.junit.runners.Parameterized; /** * Tests for the {@link IndexScrutinyTool} */ +@Ignore @Category(NeedsOwnMiniClusterTest.class) @RunWith(Parameterized.class) public class IndexScrutinyToolIT extends BaseTest { @@ -454,7 +456,9 @@ public class IndexScrutinyToolIT extends BaseTest { } } if (dataTableDdl.contains("SALT_BUCKETS")) { - fs.concat(firstPart, paths.toArray(new Path[0])); + // Check PHOENIX-4388 for discussion on a fix + // fs.concat(firstPart, paths.toArray(new Path[0])); + return; } Path outputFilePath = firstPart; assertTrue(fs.exists(outputFilePath)); http://git-wip-us.apache.org/repos/asf/phoenix/blob/024f0f22/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java index 91e34be..c4813e4 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/MigrateSystemTablesToSystemNamespaceIT.java @@ -34,6 +34,7 @@ import org.apache.phoenix.util.ReadOnlyProps; import org.apache.phoenix.util.SchemaUtil; import org.junit.After; import org.junit.Before; +import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; @@ -53,6 +54,7 @@ import java.util.Set; import static org.junit.Assert.*; +@Ignore("This test is flaky, disabled waiting for PHOENIX-4389") @Category(NeedsOwnMiniClusterTest.class) public class MigrateSystemTablesToSystemNamespaceIT extends BaseTest { http://git-wip-us.apache.org/repos/asf/phoenix/blob/024f0f22/phoenix-core/src/it/java/org/apache/phoenix/end2end/NamespaceSchemaMappingIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/NamespaceSchemaMappingIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/NamespaceSchemaMappingIT.java index 0dfd550..31647bf 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/NamespaceSchemaMappingIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/NamespaceSchemaMappingIT.java @@ -63,11 +63,12 @@ public class NamespaceSchemaMappingIT extends ParallelStatsDisabledIT { String hbaseFullTableName = schemaName + ":" + tableName; HBaseAdmin admin = driver.getConnectionQueryServices(getUrl(), TestUtil.TEST_PROPERTIES).getAdmin(); admin.createNamespace(NamespaceDescriptor.create(namespace).build()); - admin.createTable(new HTableDescriptor(TableName.valueOf(namespace, tableName)) - .addFamily(new HColumnDescriptor(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES))); - admin.createTable(new HTableDescriptor(TableName.valueOf(phoenixFullTableName)) - .addFamily(new HColumnDescriptor(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES))); - + HTableDescriptor htd1 = new HTableDescriptor(TableName.valueOf(namespace, tableName)); + htd1.addFamily(new HColumnDescriptor(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES)); + admin.createTable(htd1); + HTableDescriptor htd2 = new HTableDescriptor(TableName.valueOf(phoenixFullTableName)); + htd2.addFamily(new HColumnDescriptor(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES)); + admin.createTable(htd2); Put put = new Put(PVarchar.INSTANCE.toBytes(phoenixFullTableName)); put.addColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_BYTES, QueryConstants.EMPTY_COLUMN_VALUE_BYTES); http://git-wip-us.apache.org/repos/asf/phoenix/blob/024f0f22/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java index f809e2c..bb54fd4 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/QueryDatabaseMetaDataIT.java @@ -728,19 +728,19 @@ public class QueryDatabaseMetaDataIT extends ParallelStatsDisabledIT { descriptor = admin.getTableDescriptor(htableName); assertEquals(3, descriptor.getColumnFamilies().length); HColumnDescriptor cdA = descriptor.getFamily(cfA); - assertNotEquals(HColumnDescriptor.DEFAULT_KEEP_DELETED, cdA.getKeepDeletedCells()); + assertNotEquals(HColumnDescriptor.DEFAULT_KEEP_DELETED, cdA.getKeepDeletedCellsAsEnum()); assertEquals(DataBlockEncoding.NONE, cdA.getDataBlockEncoding()); // Overriden using // WITH assertEquals(1, cdA.getMaxVersions());// Overriden using WITH HColumnDescriptor cdB = descriptor.getFamily(cfB); // Allow KEEP_DELETED_CELLS to be false for VIEW - assertEquals(HColumnDescriptor.DEFAULT_KEEP_DELETED, cdB.getKeepDeletedCells()); + assertEquals(HColumnDescriptor.DEFAULT_KEEP_DELETED, cdB.getKeepDeletedCellsAsEnum()); assertEquals(DataBlockEncoding.NONE, cdB.getDataBlockEncoding()); // Should keep the // original value. // CF c should stay the same since it's not a Phoenix cf. HColumnDescriptor cdC = descriptor.getFamily(cfC); assertNotNull("Column family not found", cdC); - assertEquals(HColumnDescriptor.DEFAULT_KEEP_DELETED, cdC.getKeepDeletedCells()); + assertEquals(HColumnDescriptor.DEFAULT_KEEP_DELETED, cdC.getKeepDeletedCellsAsEnum()); assertFalse(SchemaUtil.DEFAULT_DATA_BLOCK_ENCODING == cdC.getDataBlockEncoding()); assertTrue(descriptor.hasCoprocessor(UngroupedAggregateRegionObserver.class.getName())); assertTrue(descriptor.hasCoprocessor(GroupedAggregateRegionObserver.class.getName())); http://git-wip-us.apache.org/repos/asf/phoenix/blob/024f0f22/phoenix-core/src/it/java/org/apache/phoenix/end2end/SetPropertyIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SetPropertyIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SetPropertyIT.java index 7a7576d..4b5894c 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SetPropertyIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SetPropertyIT.java @@ -176,17 +176,17 @@ public abstract class SetPropertyIT extends ParallelStatsDisabledIT { assertEquals("0", columnFamilies[0].getNameAsString()); assertEquals(8, columnFamilies[0].getMinVersions()); assertEquals(10, columnFamilies[0].getMaxVersions()); - assertEquals(KeepDeletedCells.FALSE, columnFamilies[0].getKeepDeletedCells()); + assertEquals(KeepDeletedCells.FALSE, columnFamilies[0].getKeepDeletedCellsAsEnum()); assertEquals("CF1", columnFamilies[1].getNameAsString()); assertEquals(1, columnFamilies[1].getMinVersions()); assertEquals(10, columnFamilies[1].getMaxVersions()); - assertEquals(KeepDeletedCells.TRUE, columnFamilies[1].getKeepDeletedCells()); + assertEquals(KeepDeletedCells.TRUE, columnFamilies[1].getKeepDeletedCellsAsEnum()); assertEquals("CF2", columnFamilies[2].getNameAsString()); assertEquals(3, columnFamilies[2].getMinVersions()); assertEquals(10, columnFamilies[2].getMaxVersions()); - assertEquals(KeepDeletedCells.FALSE, columnFamilies[2].getKeepDeletedCells()); + assertEquals(KeepDeletedCells.FALSE, columnFamilies[2].getKeepDeletedCellsAsEnum()); assertEquals(Boolean.toString(false), tableDesc.getValue(HTableDescriptor.COMPACTION_ENABLED)); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/024f0f22/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemTablePermissionsIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemTablePermissionsIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemTablePermissionsIT.java index 49202a4..731d770 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemTablePermissionsIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/SystemTablePermissionsIT.java @@ -44,12 +44,14 @@ import org.apache.hadoop.security.UserGroupInformation; import org.apache.phoenix.query.QueryServices; import org.junit.After; import org.junit.BeforeClass; +import org.junit.Ignore; import org.junit.Test; import org.junit.experimental.categories.Category; /** * Test that verifies a user can read Phoenix tables with a minimal set of permissions. */ +@Ignore("This test is flaky, disabled waiting for PHOENIX-4389") @Category(NeedsOwnMiniClusterTest.class) public class SystemTablePermissionsIT { private static String SUPERUSER; http://git-wip-us.apache.org/repos/asf/phoenix/blob/024f0f22/phoenix-core/src/it/java/org/apache/phoenix/end2end/UseSchemaIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UseSchemaIT.java b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UseSchemaIT.java index 07ae77e..c9433b3 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/end2end/UseSchemaIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/end2end/UseSchemaIT.java @@ -167,8 +167,9 @@ public class UseSchemaIT extends ParallelStatsDisabledIT { Connection conn = DriverManager.getConnection(getUrl(), props); HBaseAdmin admin = driver.getConnectionQueryServices(getUrl(), TestUtil.TEST_PROPERTIES).getAdmin(); admin.createNamespace(NamespaceDescriptor.create(schema).build()); - admin.createTable(new HTableDescriptor(fullTablename) - .addFamily(new HColumnDescriptor(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES))); + HTableDescriptor htd = new HTableDescriptor(fullTablename); + htd.addFamily(new HColumnDescriptor(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES)); + admin.createTable(htd); Put put = new Put(PVarchar.INSTANCE.toBytes(fullTablename)); put.addColumn(QueryConstants.DEFAULT_COLUMN_FAMILY_BYTES, QueryConstants.EMPTY_COLUMN_BYTES, QueryConstants.EMPTY_COLUMN_VALUE_BYTES); http://git-wip-us.apache.org/repos/asf/phoenix/blob/024f0f22/phoenix-core/src/it/java/org/apache/phoenix/trace/PhoenixTracingEndToEndIT.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/it/java/org/apache/phoenix/trace/PhoenixTracingEndToEndIT.java b/phoenix-core/src/it/java/org/apache/phoenix/trace/PhoenixTracingEndToEndIT.java index 03510dc..7511d9b 100644 --- a/phoenix-core/src/it/java/org/apache/phoenix/trace/PhoenixTracingEndToEndIT.java +++ b/phoenix-core/src/it/java/org/apache/phoenix/trace/PhoenixTracingEndToEndIT.java @@ -429,7 +429,7 @@ public class PhoenixTracingEndToEndIT extends BaseTracingTestIT { // create a simple metrics record long traceid = 987654; - Span span = createNewSpan(traceid, Span.ROOT_SPAN_ID, 10, "root", 12, 13, "Some process", "test annotation for a span"); + Span span = createNewSpan(traceid, TracingUtils.ROOT_SPAN_ID, 10, "root", 12, 13, "Some process", "test annotation for a span"); Tracer.getInstance().deliver(span); assertTrue("Updates not written in table", latch.await(60, TimeUnit.SECONDS)); @@ -457,7 +457,7 @@ public class PhoenixTracingEndToEndIT extends BaseTracingTestIT { List<Span> spans = new ArrayList<Span>(); Span span = - createNewSpan(traceid, Span.ROOT_SPAN_ID, 7777, "root", 10, 30, + createNewSpan(traceid, TracingUtils.ROOT_SPAN_ID, 7777, "root", 10, 30, "root process", "root-span tag"); spans.add(span); @@ -512,8 +512,9 @@ public class PhoenixTracingEndToEndIT extends BaseTracingTestIT { SpanInfo spanInfo = spanIter.next(); LOG.info("Checking span:\n" + spanInfo); - long parentId = span.getParentId(); - if(parentId == Span.ROOT_SPAN_ID) { + long parentId = span.getParents().length > 0 ? span.getParents()[0] : TracingUtils.ROOT_SPAN_ID; + + if(parentId == TracingUtils.ROOT_SPAN_ID) { assertNull("Got a parent, but it was a root span!", spanInfo.parent); } else { assertEquals("Got an unexpected parent span id", parentId, spanInfo.parent.id); @@ -523,9 +524,9 @@ public class PhoenixTracingEndToEndIT extends BaseTracingTestIT { assertEquals("Got an unexpected end time", span.getStopTimeMillis(), spanInfo.end); int annotationCount = 0; - for(Map.Entry<byte[], byte[]> entry : span.getKVAnnotations().entrySet()) { + for(Map.Entry<String, String> entry : span.getKVAnnotations().entrySet()) { int count = annotationCount++; - assertEquals("Didn't get expected annotation", count + " - " + Bytes.toString(entry.getValue()), + assertEquals("Didn't get expected annotation", count + " - " + entry.getValue(), spanInfo.annotations.get(count)); } assertEquals("Didn't get expected number of annotations", annotationCount, http://git-wip-us.apache.org/repos/asf/phoenix/blob/024f0f22/phoenix-core/src/main/java/org/apache/phoenix/execute/DelegateHTable.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/execute/DelegateHTable.java b/phoenix-core/src/main/java/org/apache/phoenix/execute/DelegateHTable.java index 6b3f9ca..f5a693a 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/execute/DelegateHTable.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/execute/DelegateHTable.java @@ -279,7 +279,27 @@ public class DelegateHTable implements HTableInterface { return delegate.checkAndMutate(row, family, qualifier, compareOp, value, mutation); } - @Override + @Override + public void setRpcTimeout(int i) { + delegate.setRpcTimeout(i); + } + + @Override + public int getRpcTimeout() { + return delegate.getRpcTimeout(); + } + + @Override + public void setOperationTimeout(int i) { + delegate.setOperationTimeout(i); + } + + @Override + public int getOperationTimeout() { + return delegate.getOperationTimeout(); + } + + @Override public boolean[] existsAll(List<Get> gets) throws IOException { return delegate.existsAll(gets); } http://git-wip-us.apache.org/repos/asf/phoenix/blob/024f0f22/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/metrics/MetricsIndexerSourceImpl.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/metrics/MetricsIndexerSourceImpl.java b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/metrics/MetricsIndexerSourceImpl.java index dd6ba5b..cc82bb2 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/metrics/MetricsIndexerSourceImpl.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/hbase/index/metrics/MetricsIndexerSourceImpl.java @@ -18,7 +18,7 @@ package org.apache.phoenix.hbase.index.metrics; import org.apache.hadoop.hbase.metrics.BaseSourceImpl; import org.apache.hadoop.metrics2.MetricHistogram; -import org.apache.hadoop.metrics2.lib.MutableCounterLong; +import org.apache.hadoop.metrics2.lib.MutableFastCounter; /** * Implementation for tracking Phoenix Indexer metrics. @@ -26,19 +26,19 @@ import org.apache.hadoop.metrics2.lib.MutableCounterLong; public class MetricsIndexerSourceImpl extends BaseSourceImpl implements MetricsIndexerSource { private final MetricHistogram indexPrepareTimeHisto; - private final MutableCounterLong slowIndexPrepareCalls; + private final MutableFastCounter slowIndexPrepareCalls; private final MetricHistogram indexWriteTimeHisto; - private final MutableCounterLong slowIndexWriteCalls; + private final MutableFastCounter slowIndexWriteCalls; private final MetricHistogram preWALRestoreTimeHisto; - private final MutableCounterLong slowPreWALRestoreCalls; + private final MutableFastCounter slowPreWALRestoreCalls; private final MetricHistogram postPutTimeHisto; - private final MutableCounterLong slowPostPutCalls; + private final MutableFastCounter slowPostPutCalls; private final MetricHistogram postDeleteTimeHisto; - private final MutableCounterLong slowPostDeleteCalls; + private final MutableFastCounter slowPostDeleteCalls; private final MetricHistogram postOpenTimeHisto; - private final MutableCounterLong slowPostOpenCalls; + private final MutableFastCounter slowPostOpenCalls; private final MetricHistogram duplicateKeyTimeHisto; - private final MutableCounterLong slowDuplicateKeyCalls; + private final MutableFastCounter slowDuplicateKeyCalls; public MetricsIndexerSourceImpl() { this(METRICS_NAME, METRICS_DESCRIPTION, METRICS_CONTEXT, METRICS_JMX_CONTEXT); http://git-wip-us.apache.org/repos/asf/phoenix/blob/024f0f22/phoenix-core/src/main/java/org/apache/phoenix/trace/TraceReader.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/trace/TraceReader.java b/phoenix-core/src/main/java/org/apache/phoenix/trace/TraceReader.java index 68b945c..f882c93 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/trace/TraceReader.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/trace/TraceReader.java @@ -36,6 +36,7 @@ import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.metrics.MetricInfo; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.query.QueryServicesOptions; +import org.apache.phoenix.trace.TracingUtils; import org.apache.phoenix.util.LogUtil; import com.google.common.base.Joiner; @@ -124,7 +125,7 @@ public class TraceReader { // search the spans to determine the if we have a known parent SpanInfo parentSpan = null; - if (parent != Span.ROOT_SPAN_ID) { + if (parent != TracingUtils.ROOT_SPAN_ID) { // find the parent for (SpanInfo p : trace.spans) { if (p.id == parent) { @@ -154,10 +155,10 @@ public class TraceReader { if (parentSpan != null) { // add this as a child to the parent span parentSpan.children.add(spanInfo); - } else if (parent != Span.ROOT_SPAN_ID) { + } else if (parent != TracingUtils.ROOT_SPAN_ID) { // add the span to the orphan pile to check for the remaining spans we see LOG.info(addCustomAnnotations("No parent span found for span: " + span + " (root span id: " - + Span.ROOT_SPAN_ID + ")")); + + TracingUtils.ROOT_SPAN_ID + ")")); orphans.add(spanInfo); } @@ -332,9 +333,9 @@ public class TraceReader { @Override public int compareTo(SpanInfo o) { // root span always comes first - if (this.parentId == Span.ROOT_SPAN_ID) { + if (this.parentId == TracingUtils.ROOT_SPAN_ID) { return -1; - } else if (o.parentId == Span.ROOT_SPAN_ID) { + } else if (o.parentId == TracingUtils.ROOT_SPAN_ID) { return 1; } @@ -354,7 +355,7 @@ public class TraceReader { sb.append("\tdescription=" + description); sb.append("\n"); sb.append("\tparent=" - + (parent == null ? (parentId == Span.ROOT_SPAN_ID ? "ROOT" : "[orphan - id: " + + (parent == null ? (parentId == TracingUtils.ROOT_SPAN_ID ? "ROOT" : "[orphan - id: " + parentId + "]") : parent.id)); sb.append("\n"); sb.append("\tstart,end=" + start + "," + end); http://git-wip-us.apache.org/repos/asf/phoenix/blob/024f0f22/phoenix-core/src/main/java/org/apache/phoenix/trace/TraceWriter.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/trace/TraceWriter.java b/phoenix-core/src/main/java/org/apache/phoenix/trace/TraceWriter.java index e823359..a1b0f70 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/trace/TraceWriter.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/trace/TraceWriter.java @@ -175,7 +175,7 @@ public class TraceWriter { values.add(span.getSpanId()); keys.add(PARENT.traceName); - values.add(span.getParentId()); + values.add( (span.getParents().length > 0 ? span.getParents()[0] : TracingUtils.ROOT_SPAN_ID )); keys.add(START.traceName); values.add(span.getStartTimeMillis()); @@ -196,8 +196,8 @@ public class TraceWriter { // add the annotations. We assume they are serialized as strings and integers, but that // can // change in the future - Map<byte[], byte[]> annotations = span.getKVAnnotations(); - for (Map.Entry<byte[], byte[]> annotation : annotations.entrySet()) { + Map<String, String> annotations = span.getKVAnnotations(); + for (Map.Entry<String, String> annotation : annotations.entrySet()) { Pair<String, String> val = TracingUtils.readAnnotation(annotation.getKey(), annotation.getValue()); addDynamicEntry(keys, values, variableValues, ANNOTATION_FAMILY, val.getFirst(), http://git-wip-us.apache.org/repos/asf/phoenix/blob/024f0f22/phoenix-core/src/main/java/org/apache/phoenix/trace/TracingUtils.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/trace/TracingUtils.java b/phoenix-core/src/main/java/org/apache/phoenix/trace/TracingUtils.java index 8bd918e..18ffa0f 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/trace/TracingUtils.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/trace/TracingUtils.java @@ -33,6 +33,9 @@ public class TracingUtils { /** Marker metric to ensure that we register the tracing mbeans */ public static final String METRICS_MARKER_CONTEXT = "marker"; + /** Backwards compatible marker for root span */ + public static final long ROOT_SPAN_ID = 0x74ace; + public static void addAnnotation(Span span, String message, int value) { span.addKVAnnotation(message.getBytes(), Bytes.toBytes(Integer.toString(value))); } @@ -41,6 +44,10 @@ public class TracingUtils { return new Pair<String, String>(new String(key), Bytes.toString(value)); } + public static Pair<String, String> readAnnotation(String key, String value) { + return new Pair<String, String>(key, value); + } + /** * @see #getTraceMetricName(String) */ http://git-wip-us.apache.org/repos/asf/phoenix/blob/024f0f22/phoenix-core/src/main/java/org/apache/phoenix/trace/util/NullSpan.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/trace/util/NullSpan.java b/phoenix-core/src/main/java/org/apache/phoenix/trace/util/NullSpan.java index b4f70b9..f975198 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/trace/util/NullSpan.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/trace/util/NullSpan.java @@ -84,12 +84,11 @@ public class NullSpan implements Span { } @Override - public long getParentId() { - return 0; + public void addKVAnnotation(byte[] key, byte[] value) { } @Override - public void addKVAnnotation(byte[] key, byte[] value) { + public void addKVAnnotation(String key, String value) { } @Override @@ -97,7 +96,7 @@ public class NullSpan implements Span { } @Override - public Map<byte[], byte[]> getKVAnnotations() { + public Map<String, String> getKVAnnotations() { return Collections.emptyMap(); } @@ -112,6 +111,19 @@ public class NullSpan implements Span { } @Override + public void setProcessId(String id) { + } + + @Override + public long[] getParents() { + return new long[0]; + } + + @Override + public void setParents(long[] parents) { + } + + @Override public String toJson() { return StringUtil.EMPTY_STRING; } http://git-wip-us.apache.org/repos/asf/phoenix/blob/024f0f22/phoenix-core/src/main/java/org/apache/phoenix/trace/util/Tracing.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/trace/util/Tracing.java b/phoenix-core/src/main/java/org/apache/phoenix/trace/util/Tracing.java index 35cc6dc..f7e815c 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/trace/util/Tracing.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/trace/util/Tracing.java @@ -36,6 +36,7 @@ import org.apache.phoenix.jdbc.PhoenixConnection; import org.apache.phoenix.parse.TraceStatement; import org.apache.phoenix.query.QueryServices; import org.apache.phoenix.query.QueryServicesOptions; +import org.apache.phoenix.trace.TracingUtils; import org.apache.phoenix.trace.TraceSpanReceiver; import org.apache.htrace.Sampler; import org.apache.htrace.Span; @@ -174,8 +175,10 @@ public class Tracing { } public static String getSpanName(Span span) { - return Tracing.TRACE_METRIC_PREFIX + span.getTraceId() + SEPARATOR + span.getParentId() - + SEPARATOR + span.getSpanId(); + long[] parents = span.getParents(); + return Tracing.TRACE_METRIC_PREFIX + span.getTraceId() + SEPARATOR + + (parents.length > 0 ? parents[0] : TracingUtils.ROOT_SPAN_ID ) + SEPARATOR + + span.getSpanId(); } public static Span child(Span s, String d) { http://git-wip-us.apache.org/repos/asf/phoenix/blob/024f0f22/phoenix-core/src/main/java/org/apache/phoenix/transaction/OmidTransactionTable.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/transaction/OmidTransactionTable.java b/phoenix-core/src/main/java/org/apache/phoenix/transaction/OmidTransactionTable.java index 54eea8b..0662555 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/transaction/OmidTransactionTable.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/transaction/OmidTransactionTable.java @@ -336,4 +336,28 @@ public class OmidTransactionTable implements PhoenixTransactionalTable { // TODO Auto-generated method stub return false; } + + @Override + public int getOperationTimeout() { + // TODO Auto-generated method stub + return 0; + } + + @Override + public int getRpcTimeout() { + // TODO Auto-generated method stub + return 0; + } + + @Override + public void setOperationTimeout(int arg0) { + // TODO Auto-generated method stub + + } + + @Override + public void setRpcTimeout(int arg0) { + // TODO Auto-generated method stub + + } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/024f0f22/phoenix-core/src/main/java/org/apache/phoenix/transaction/TephraTransactionTable.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/transaction/TephraTransactionTable.java b/phoenix-core/src/main/java/org/apache/phoenix/transaction/TephraTransactionTable.java index cf48521..c191d8d 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/transaction/TephraTransactionTable.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/transaction/TephraTransactionTable.java @@ -327,4 +327,24 @@ public class TephraTransactionTable implements PhoenixTransactionalTable { throws IOException { return transactionAwareHTable.checkAndMutate(row, family, qualifier, compareOp, value, mutation); } + + @Override + public void setOperationTimeout(int i) { +// transactionAwareHTable.setOperationTimeout(i); + } + + @Override + public int getOperationTimeout() { + return 0; //transactionAwareHTable.getOperationTimeout(); + } + + @Override + public void setRpcTimeout(int i) { +// transactionAwareHTable.setRpcTimeout(i); + } + + @Override + public int getRpcTimeout() { + return 0; //transactionAwareHTable.getRpcTimeout(); + } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/024f0f22/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixMRJobUtil.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixMRJobUtil.java b/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixMRJobUtil.java index f12d49d..4f01237 100644 --- a/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixMRJobUtil.java +++ b/phoenix-core/src/main/java/org/apache/phoenix/util/PhoenixMRJobUtil.java @@ -64,8 +64,11 @@ public class PhoenixMRJobUtil { // Default MR Capacity Scheduler Configurations for Phoenix MR Index Build // Jobs + public static final String QUEUE_NAME = "mapreduce.job.queuename"; public static final String DEFAULT_QUEUE_NAME = "default"; + public static final String MAP_MEMORY_MB = "mapreduce.map.memory.mb"; public static final int DEFAULT_MAP_MEMROY_MB = 5120; + public static final String MAP_JAVA_OPTS = "mapreduce.map.java.opts"; public static final String XMX_OPT = "-Xmx"; public static final String RM_HTTP_SCHEME = "http"; @@ -219,15 +222,15 @@ public class PhoenixMRJobUtil { * @param conf - Configuration to which Capacity Queue information to be added */ public static void updateCapacityQueueInfo(Configuration conf) { - conf.set(MRJobConfig.QUEUE_NAME, + conf.set(QUEUE_NAME, conf.get(PHOENIX_INDEX_MR_QUEUE_NAME_PROPERTY, DEFAULT_QUEUE_NAME)); int mapMemoryMB = conf.getInt(PHOENIX_INDEX_MR_MAP_MEMORY_PROPERTY, DEFAULT_MAP_MEMROY_MB); - conf.setInt(MRJobConfig.MAP_MEMORY_MB, mapMemoryMB); - conf.set(MRJobConfig.MAP_JAVA_OPTS, XMX_OPT + ((int) (mapMemoryMB * 0.9)) + "m"); + conf.setInt(MAP_MEMORY_MB, mapMemoryMB); + conf.set(MAP_JAVA_OPTS, XMX_OPT + ((int) (mapMemoryMB * 0.9)) + "m"); - LOG.info("Queue Name=" + conf.get(MRJobConfig.QUEUE_NAME) + ";" + "Map Meory MB=" - + conf.get(MRJobConfig.MAP_MEMORY_MB) + ";" + "Map Java Opts=" - + conf.get(MRJobConfig.MAP_JAVA_OPTS)); + LOG.info("Queue Name=" + conf.get(QUEUE_NAME) + ";" + "Map Meory MB=" + + conf.get(MAP_MEMORY_MB) + ";" + "Map Java Opts=" + + conf.get(MAP_JAVA_OPTS)); } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/024f0f22/phoenix-core/src/test/java/org/apache/phoenix/trace/TraceSpanReceiverTest.java ---------------------------------------------------------------------- diff --git a/phoenix-core/src/test/java/org/apache/phoenix/trace/TraceSpanReceiverTest.java b/phoenix-core/src/test/java/org/apache/phoenix/trace/TraceSpanReceiverTest.java index fa5bc76..e07158f 100644 --- a/phoenix-core/src/test/java/org/apache/phoenix/trace/TraceSpanReceiverTest.java +++ b/phoenix-core/src/test/java/org/apache/phoenix/trace/TraceSpanReceiverTest.java @@ -77,6 +77,12 @@ public class TraceSpanReceiverTest { private Span getSpan(){ // Spans with Trace Id as 0 will be rejected (See PHOENIX-3767 for details) - return new MilliSpan("test span", 1, 1 , 2, "pid"); + return new MilliSpan.Builder() + .description("test span") + .traceId(1L) + .parents(new long[]{1}) + .spanId(2L) + .processId("pid") + .build(); } } http://git-wip-us.apache.org/repos/asf/phoenix/blob/024f0f22/phoenix-flume/pom.xml ---------------------------------------------------------------------- diff --git a/phoenix-flume/pom.xml b/phoenix-flume/pom.xml index 951b444..39a4ccd 100644 --- a/phoenix-flume/pom.xml +++ b/phoenix-flume/pom.xml @@ -26,7 +26,7 @@ <parent> <groupId>org.apache.phoenix</groupId> <artifactId>phoenix</artifactId> - <version>4.13.0-HBase-1.2</version> + <version>4.13.0-cdh5.11.2</version> </parent> <artifactId>phoenix-flume</artifactId> <name>Phoenix - Flume</name> http://git-wip-us.apache.org/repos/asf/phoenix/blob/024f0f22/phoenix-hive/pom.xml ---------------------------------------------------------------------- diff --git a/phoenix-hive/pom.xml b/phoenix-hive/pom.xml index 61dcca4..a57f2d6 100644 --- a/phoenix-hive/pom.xml +++ b/phoenix-hive/pom.xml @@ -27,7 +27,7 @@ <parent> <groupId>org.apache.phoenix</groupId> <artifactId>phoenix</artifactId> - <version>4.13.0-HBase-1.2</version> + <version>4.13.0-cdh5.11.2</version> </parent> <artifactId>phoenix-hive</artifactId> <name>Phoenix - Hive</name> http://git-wip-us.apache.org/repos/asf/phoenix/blob/024f0f22/phoenix-hive/src/it/java/org/apache/phoenix/hive/HivePhoenixStoreIT.java ---------------------------------------------------------------------- diff --git a/phoenix-hive/src/it/java/org/apache/phoenix/hive/HivePhoenixStoreIT.java b/phoenix-hive/src/it/java/org/apache/phoenix/hive/HivePhoenixStoreIT.java index 1828818..edf72f1 100644 --- a/phoenix-hive/src/it/java/org/apache/phoenix/hive/HivePhoenixStoreIT.java +++ b/phoenix-hive/src/it/java/org/apache/phoenix/hive/HivePhoenixStoreIT.java @@ -203,6 +203,7 @@ public class HivePhoenixStoreIT extends BaseHivePhoenixStoreIT { * * @throws Exception */ + @Ignore @Test public void testJoinNoColumnMaps() throws Exception { String testName = "testJoin"; @@ -253,6 +254,7 @@ public class HivePhoenixStoreIT extends BaseHivePhoenixStoreIT { * * @throws Exception */ + @Ignore @Test public void testJoinColumnMaps() throws Exception { String testName = "testJoin"; http://git-wip-us.apache.org/repos/asf/phoenix/blob/024f0f22/phoenix-hive/src/it/java/org/apache/phoenix/hive/HiveTezIT.java ---------------------------------------------------------------------- diff --git a/phoenix-hive/src/it/java/org/apache/phoenix/hive/HiveTezIT.java b/phoenix-hive/src/it/java/org/apache/phoenix/hive/HiveTezIT.java index 8dc3309..9018a03 100644 --- a/phoenix-hive/src/it/java/org/apache/phoenix/hive/HiveTezIT.java +++ b/phoenix-hive/src/it/java/org/apache/phoenix/hive/HiveTezIT.java @@ -23,6 +23,7 @@ import org.junit.BeforeClass; import org.junit.experimental.categories.Category; import org.junit.Ignore; +@Ignore("Tez is not supported in CDH") @Category(NeedsOwnMiniClusterTest.class) public class HiveTezIT extends HivePhoenixStoreIT { http://git-wip-us.apache.org/repos/asf/phoenix/blob/024f0f22/phoenix-kafka/pom.xml ---------------------------------------------------------------------- diff --git a/phoenix-kafka/pom.xml b/phoenix-kafka/pom.xml index e573cd6..c904bfc 100644 --- a/phoenix-kafka/pom.xml +++ b/phoenix-kafka/pom.xml @@ -26,7 +26,7 @@ <parent> <groupId>org.apache.phoenix</groupId> <artifactId>phoenix</artifactId> - <version>4.13.0-HBase-1.2</version> + <version>4.13.0-cdh5.11.2</version> </parent> <artifactId>phoenix-kafka</artifactId> <name>Phoenix - Kafka</name> http://git-wip-us.apache.org/repos/asf/phoenix/blob/024f0f22/phoenix-load-balancer/pom.xml ---------------------------------------------------------------------- diff --git a/phoenix-load-balancer/pom.xml b/phoenix-load-balancer/pom.xml index a164ebd..5bc5e7c 100644 --- a/phoenix-load-balancer/pom.xml +++ b/phoenix-load-balancer/pom.xml @@ -27,7 +27,7 @@ <parent> <groupId>org.apache.phoenix</groupId> <artifactId>phoenix</artifactId> - <version>4.13.0-HBase-1.2</version> + <version>4.13.0-cdh5.11.2</version> </parent> <artifactId>phoenix-load-balancer</artifactId> <name>Phoenix Load Balancer</name> http://git-wip-us.apache.org/repos/asf/phoenix/blob/024f0f22/phoenix-load-balancer/src/it/java/org/apache/phoenix/end2end/LoadBalancerEnd2EndIT.java ---------------------------------------------------------------------- diff --git a/phoenix-load-balancer/src/it/java/org/apache/phoenix/end2end/LoadBalancerEnd2EndIT.java b/phoenix-load-balancer/src/it/java/org/apache/phoenix/end2end/LoadBalancerEnd2EndIT.java index a5e2c9b..8aa516b 100644 --- a/phoenix-load-balancer/src/it/java/org/apache/phoenix/end2end/LoadBalancerEnd2EndIT.java +++ b/phoenix-load-balancer/src/it/java/org/apache/phoenix/end2end/LoadBalancerEnd2EndIT.java @@ -25,7 +25,7 @@ import org.apache.curator.CuratorZookeeperClient; import org.apache.curator.framework.CuratorFramework; import org.apache.curator.framework.CuratorFrameworkFactory; import org.apache.curator.retry.ExponentialBackoffRetry; -import org.apache.curator.TestingServer; +import org.apache.curator.test.TestingServer; import org.apache.curator.utils.CloseableUtils; import org.apache.phoenix.loadbalancer.service.LoadBalancer; import org.apache.phoenix.loadbalancer.service.LoadBalanceZookeeperConf; http://git-wip-us.apache.org/repos/asf/phoenix/blob/024f0f22/phoenix-parcel/pom.xml ---------------------------------------------------------------------- diff --git a/phoenix-parcel/pom.xml b/phoenix-parcel/pom.xml new file mode 100644 index 0000000..dbbd10b --- /dev/null +++ b/phoenix-parcel/pom.xml @@ -0,0 +1,252 @@ +<?xml version='1.0'?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <parent> + <groupId>org.apache.phoenix</groupId> + <artifactId>phoenix</artifactId> + <version>4.13.0-cdh5.11.2</version> + </parent> + <artifactId>phoenix-parcel</artifactId> + <name>Phoenix Parcels for CDH</name> + <description>Assemble Phoenix artifacts for CDH</description> + <packaging>pom</packaging> + + <properties> + <source.skip>true</source.skip> + <top.dir>${project.basedir}/..</top.dir> + <maven.test.skip>true</maven.test.skip> + <phoenix.version>${phoenix.release.version}</phoenix.version> + <parcel.patch.count>0</parcel.patch.count> + <parcel.release>0.${parcel.patch.count}</parcel.release> + <parcel.folder>APACHE_PHOENIX-${phoenix.version}-${cdh.version}.p${parcel.release}</parcel.folder> + <parcel.file>${parcel.folder}.parcel</parcel.file> + <parcel.version>${phoenix.version}-${cdh.version}.p${parcel.release}</parcel.version> + <parcel.base.version>${phoenix.version}</parcel.base.version> + <parcel.full.version>${phoenix.version}-${cdh.version}.p${parcel.release}</parcel.full.version> + <parcel.package.version>${phoenix.version}+${cdh.version}+${parcel.patch.count}</parcel.package.version> + <parcel.component.version>${phoenix.version}-${cdh.version}</parcel.component.version> + <parcel.component.release>${cdh.version}.p${parcel.release}</parcel.component.release> + <parcel.depends>CDH (= ${cdh.version.number})</parcel.depends> + </properties> + + <build> + + <plugins> + <!-- No jars created for this module --> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-jar-plugin</artifactId> + <executions> + <execution> + <id>default-jar</id> + <phase>none</phase> + <goals/> + </execution> + </executions> + </plugin> + <plugin> + <artifactId>maven-assembly-plugin</artifactId> + <executions> + <execution> + <id>prepare-parcel</id> + <phase>prepare-package</phase> + <goals> + <goal>single</goal> + </goals> + <configuration> + <finalName>${parcel.file}</finalName> + <attach>false</attach> + <tarLongFileMode>gnu</tarLongFileMode> + <appendAssemblyId>false</appendAssemblyId> + <descriptors> + <descriptor>src/build/parcel.xml</descriptor> + </descriptors> + <tarLongFileMode>posix</tarLongFileMode> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>com.coderplus.maven.plugins</groupId> + <artifactId>copy-rename-maven-plugin</artifactId> + <version>1.0.1</version> + <executions> + <execution> + <id>copy-file-el6</id> + <phase>package</phase> + <goals> + <goal>copy</goal> + </goals> + <configuration> + <sourceFile>${project.build.directory}/${parcel.folder}.parcel.tar</sourceFile> + <destinationFile>${project.build.directory}/${parcel.folder}-el6.parcel</destinationFile> + </configuration> + </execution> + <execution> + <id>copy-file-el5</id> + <phase>package</phase> + <goals> + <goal>copy</goal> + </goals> + <configuration> + <sourceFile>${project.build.directory}/${parcel.folder}.parcel.tar</sourceFile> + <destinationFile>${project.build.directory}/${parcel.folder}-el5.parcel</destinationFile> + </configuration> + </execution> + <execution> + <id>copy-file-el7</id> + <phase>package</phase> + <goals> + <goal>copy</goal> + </goals> + <configuration> + <sourceFile>${project.build.directory}/${parcel.folder}.parcel.tar</sourceFile> + <destinationFile>${project.build.directory}/${parcel.folder}-el7.parcel</destinationFile> + </configuration> + </execution> + <execution> + <id>copy-file-sles11</id> + <phase>package</phase> + <goals> + <goal>copy</goal> + </goals> + <configuration> + <sourceFile>${project.build.directory}/${parcel.folder}.parcel.tar</sourceFile> + <destinationFile>${project.build.directory}/${parcel.folder}-sles11.parcel</destinationFile> + </configuration> + </execution> + <execution> + <id>copy-file-sles12</id> + <phase>package</phase> + <goals> + <goal>copy</goal> + </goals> + <configuration> + <sourceFile>${project.build.directory}/${parcel.folder}.parcel.tar</sourceFile> + <destinationFile>${project.build.directory}/${parcel.folder}-sles12.parcel</destinationFile> + </configuration> + </execution> + <execution> + <id>copy-file-precise</id> + <phase>package</phase> + <goals> + <goal>copy</goal> + </goals> + <configuration> + <sourceFile>${project.build.directory}/${parcel.folder}.parcel.tar</sourceFile> + <destinationFile>${project.build.directory}/${parcel.folder}-precise.parcel</destinationFile> + </configuration> + </execution> + <execution> + <id>copy-file-jessie</id> + <phase>package</phase> + <goals> + <goal>copy</goal> + </goals> + <configuration> + <sourceFile>${project.build.directory}/${parcel.folder}.parcel.tar</sourceFile> + <destinationFile>${project.build.directory}/${parcel.folder}-jessie.parcel</destinationFile> + </configuration> + </execution> + <execution> + <id>copy-file-trusty</id> + <phase>package</phase> + <goals> + <goal>copy</goal> + </goals> + <configuration> + <sourceFile>${project.build.directory}/${parcel.folder}.parcel.tar</sourceFile> + <destinationFile>${project.build.directory}/${parcel.folder}-trusty.parcel</destinationFile> + </configuration> + </execution> + <execution> + <id>copy-file-wheezy</id> + <phase>package</phase> + <goals> + <goal>copy</goal> + </goals> + <configuration> + <sourceFile>${project.build.directory}/${parcel.folder}.parcel.tar</sourceFile> + <destinationFile>${project.build.directory}/${parcel.folder}-wheezy.parcel</destinationFile> + </configuration> + </execution> + <execution> + <id>copy-file-xenial</id> + <phase>package</phase> + <goals> + <goal>copy</goal> + </goals> + <configuration> + <sourceFile>${project.build.directory}/${parcel.folder}.parcel.tar</sourceFile> + <destinationFile>${project.build.directory}/${parcel.folder}-xenial.parcel</destinationFile> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <groupId>org.codehaus.mojo</groupId> + <artifactId>exec-maven-plugin</artifactId> + <executions> + <execution> + <id>make-manifest</id> + <phase>package</phase> + <goals> + <goal>exec</goal> + </goals> + </execution> + </executions> + <configuration> + <executable>python</executable> + <workingDirectory>${project.build.directory}</workingDirectory> + <arguments> + <argument>${project.basedir}/src/build/manifest/make_manifest.py</argument> + <argument>${project.build.directory}</argument> + </arguments> + </configuration> + </plugin> + </plugins> + </build> + + <dependencies> + <!-- Depend on all other internal projects --> + <dependency> + <groupId>org.apache.phoenix</groupId> + <artifactId>phoenix-core</artifactId> + </dependency> + <dependency> + <groupId>org.apache.phoenix</groupId> + <artifactId>phoenix-flume</artifactId> + </dependency> + <dependency> + <groupId>org.apache.phoenix</groupId> + <artifactId>phoenix-pig</artifactId> + </dependency> + <dependency> + <groupId>org.apache.phoenix</groupId> + <artifactId>phoenix-spark</artifactId> + </dependency> + </dependencies> +</project> http://git-wip-us.apache.org/repos/asf/phoenix/blob/024f0f22/phoenix-parcel/src/build/components/all-common-dependencies.xml ---------------------------------------------------------------------- diff --git a/phoenix-parcel/src/build/components/all-common-dependencies.xml b/phoenix-parcel/src/build/components/all-common-dependencies.xml new file mode 100644 index 0000000..9af3e94 --- /dev/null +++ b/phoenix-parcel/src/build/components/all-common-dependencies.xml @@ -0,0 +1,56 @@ +<?xml version='1.0'?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> +<component> + <!-- All of our dependencies --> + <dependencySets> + <dependencySet> + <unpack>false</unpack> + <outputDirectory>${parcel.folder}/lib/phoenix/lib</outputDirectory> + <includes> + <include>org.apache.phoenix:phoenix-core</include> + <include>org.iq80.snappy:snappy</include> + <include>org.antlr:antlr*</include> + <include>org.apache.tephra:tephra*</include> + <include>com.google.code.gson:gson</include> + <include>org.jruby.joni:joni</include> + <include>org.jruby.jcodings:jcodings</include> + <include>joda-time:joda-time</include> + <include>org.apache.twill:twill*</include> + <include>com.google.inject.extensions:guice-assistedinject</include> + <include>it.unimi.dsi:fastutil</include> + <include>io.dropwizard.metrics:metrics-core</include> + <include>org.apache.thrift:libthrift</include> + <include>com.clearspring.analytics:stream</include> + <include>com.salesforce.i18n:i18n-util</include> + <include>com.tdunning:json</include> + <include>com.jayway.jsonpath:json-path</include> + <include>net.minidev:json-smart</include> + <include>net.minidev:accessors-smart</include> + <include>sqlline:sqlline</include> + <include>org.apache.commons:commons-csv</include> + <include>com.ibm.icu:icu4j</include> + <include>com.ibm.icu:icu4j-charset</include> + <include>com.ibm.icu:icu4j-localespi</include> + </includes> + </dependencySet> + </dependencySets> +</component> http://git-wip-us.apache.org/repos/asf/phoenix/blob/024f0f22/phoenix-parcel/src/build/components/all-common-files.xml ---------------------------------------------------------------------- diff --git a/phoenix-parcel/src/build/components/all-common-files.xml b/phoenix-parcel/src/build/components/all-common-files.xml new file mode 100644 index 0000000..fa85ec0 --- /dev/null +++ b/phoenix-parcel/src/build/components/all-common-files.xml @@ -0,0 +1,84 @@ +<?xml version='1.0'?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> +<component> + + <fileSets> + + <!-- Executable files from bin directory --> + <fileSet> + <directory>${project.basedir}/src/parcel/bin</directory> + <outputDirectory>${parcel.folder}/bin</outputDirectory> + <fileMode>0755</fileMode> + <directoryMode>0755</directoryMode> + <filtered>false</filtered> + <includes> + <include>*</include> + </includes> + </fileSet> + <!-- Meta files --> + <fileSet> + <directory>${project.basedir}/src/parcel/meta</directory> + <outputDirectory>${parcel.folder}/meta</outputDirectory> + <fileMode>0644</fileMode> + <directoryMode>0755</directoryMode> + <filtered>true</filtered> + <includes> + <include>*</include> + </includes> + </fileSet> + + <!-- lib/bin files --> + <fileSet> + <directory>${project.basedir}/../bin</directory> + <outputDirectory>${parcel.folder}/lib/phoenix/bin</outputDirectory> + <fileMode>0755</fileMode> + <directoryMode>0755</directoryMode> + <filtered>false</filtered> + <excludes> + <exclude>hbase-site.xml</exclude> + </excludes> + </fileSet> + <!-- lib/dev files --> + <fileSet> + <directory>${project.basedir}/../dev</directory> + <outputDirectory>${parcel.folder}/lib/phoenix/dev</outputDirectory> + <fileMode>0644</fileMode> + <directoryMode>0755</directoryMode> + <filtered>false</filtered> + <includes> + <include>*</include> + </includes> + </fileSet> + <!-- lib/examples files --> + <fileSet> + <directory>${project.basedir}/../examples</directory> + <outputDirectory>${parcel.folder}/lib/phoenix/examples</outputDirectory> + <fileMode>0644</fileMode> + <directoryMode>0755</directoryMode> + <filtered>false</filtered> + <includes> + <include>*</include> + </includes> + </fileSet> + + </fileSets> +</component> http://git-wip-us.apache.org/repos/asf/phoenix/blob/024f0f22/phoenix-parcel/src/build/components/all-common-jars.xml ---------------------------------------------------------------------- diff --git a/phoenix-parcel/src/build/components/all-common-jars.xml b/phoenix-parcel/src/build/components/all-common-jars.xml new file mode 100644 index 0000000..c659ab8 --- /dev/null +++ b/phoenix-parcel/src/build/components/all-common-jars.xml @@ -0,0 +1,217 @@ +<?xml version='1.0'?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> +<component> + <fileSets> + <!-- Add the client & mapreduce jars. Expects the client jar packaging phase to already be run, + which is determined by specification order in the pom. --> + <fileSet> + <directory>${project.basedir}/../phoenix-client/target</directory> + <outputDirectory>${parcel.folder}/lib/phoenix/</outputDirectory> + <includes> + <include>phoenix-*-client.jar</include> + </includes> + <excludes> + <exclude>*-minimal.jar</exclude> + <exclude>*-sources.jar</exclude> + <exclude>*-tests.jar</exclude> + </excludes> + <fileMode>0644</fileMode> + </fileSet> + <fileSet> + <directory>${project.basedir}/../phoenix-server/target</directory> + <outputDirectory>${parcel.folder}/lib/phoenix/</outputDirectory> + <includes> + <include>phoenix-*-server.jar</include> + </includes> + <excludes> + <exclude>*-minimal.jar</exclude> + <exclude>*-sources.jar</exclude> + <exclude>*-tests.jar</exclude> + </excludes> + <fileMode>0644</fileMode> + </fileSet> + <fileSet> + <directory>${project.basedir}/../phoenix-queryserver/target/</directory> + <outputDirectory>${parcel.folder}/lib/phoenix/</outputDirectory> + <includes> + <include>phoenix-*-queryserver.jar</include> + </includes> + <excludes> + <exclude>*-minimal.jar</exclude> + <exclude>*-sources.jar</exclude> + <exclude>*-tests.jar</exclude> + </excludes> + <fileMode>0644</fileMode> + </fileSet> + <fileSet> + <directory>${project.basedir}/../phoenix-queryserver-client/target/</directory> + <outputDirectory>${parcel.folder}/lib/phoenix/</outputDirectory> + <includes> + <include>phoenix-*-thin-client.jar</include> + </includes> + <excludes> + <exclude>*-minimal.jar</exclude> + <exclude>*-sources.jar</exclude> + <exclude>*-tests.jar</exclude> + </excludes> + <fileMode>0644</fileMode> + </fileSet> + <fileSet> + <directory>${project.basedir}/../phoenix-hive/target/</directory> + <outputDirectory>${parcel.folder}/lib/phoenix/</outputDirectory> + <includes> + <include>phoenix-*-hive.jar</include> + </includes> + <excludes> + <exclude>*-minimal.jar</exclude> + <exclude>*-sources.jar</exclude> + <exclude>*-tests.jar</exclude> + </excludes> + <fileMode>0644</fileMode> + </fileSet> + + <!-- This is only necessary until maven fixes the intra-project dependency bug + in maven 3.0. Until then, we have to include the jars for sub-projects explicitly. + Otherwise, test jars are pulled in wrongly. + --> + <fileSet> + <directory>${project.basedir}/../phoenix-hadoop-compat/target/</directory> + <outputDirectory>${parcel.folder}/lib/phoenix/lib</outputDirectory> + <includes> + <include>phoenix-*.jar</include> + </includes> + <excludes> + <exclude>*-minimal.jar</exclude> + <exclude>*-sources.jar</exclude> + <exclude>*-tests.jar</exclude> + </excludes> + <fileMode>0644</fileMode> + </fileSet> + <fileSet> + <directory>${project.basedir}/../phoenix-pig/target/</directory> + <outputDirectory>${parcel.folder}/lib/phoenix/lib</outputDirectory> + <includes> + <include>phoenix-pig-*.jar</include> + </includes> + <excludes> + <exclude>*-minimal.jar</exclude> + <exclude>*-sources.jar</exclude> + <exclude>*-tests.jar</exclude> + </excludes> + <fileMode>0644</fileMode> + </fileSet> + <fileSet> + <directory>${project.basedir}/../phoenix-flume/target/</directory> + <outputDirectory>${parcel.folder}/lib/phoenix/lib</outputDirectory> + <includes> + <include>phoenix-*.jar</include> + </includes> + <excludes> + <exclude>*-minimal.jar</exclude> + <exclude>*-sources.jar</exclude> + <exclude>*-tests.jar</exclude> + </excludes> + <fileMode>0644</fileMode> + </fileSet> + <fileSet> + <directory>${project.basedir}/../phoenix-core/target/</directory> + <outputDirectory>${parcel.folder}/lib/phoenix/lib</outputDirectory> + <includes> + <include>phoenix-*.jar</include> + </includes> + <excludes> + <exclude>*-minimal.jar</exclude> + <exclude>*-sources.jar</exclude> + <exclude>*-tests.jar</exclude> + </excludes> + <fileMode>0644</fileMode> + </fileSet> + <fileSet> + <directory>${project.basedir}/../phoenix-spark/target/</directory> + <outputDirectory>${parcel.folder}/lib/phoenix/lib</outputDirectory> + <includes> + <include>phoenix-*.jar</include> + </includes> + <excludes> + <exclude>*-javadoc.jar</exclude> + <exclude>*-minimal.jar</exclude> + <exclude>*-sources.jar</exclude> + <exclude>*-tests.jar</exclude> + </excludes> + <fileMode>0644</fileMode> + </fileSet> + <fileSet> + <directory>${project.basedir}/../phoenix-queryserver/target/</directory> + <outputDirectory>${parcel.folder}/lib/phoenix/lib</outputDirectory> + <includes> + <include>phoenix-queryserver-*.jar</include> + </includes> + <excludes> + <exclude>*-minimal.jar</exclude> + <exclude>*-sources.jar</exclude> + <exclude>*-tests.jar</exclude> + </excludes> + <fileMode>0644</fileMode> + </fileSet> + <fileSet> + <directory>${project.basedir}/../phoenix-hive/target/</directory> + <outputDirectory>${parcel.folder}/lib/phoenix/lib</outputDirectory> + <includes> + <include>phoenix-hive-*.jar</include> + </includes> + <excludes> + <exclude>*-minimal.jar</exclude> + <exclude>*-sources.jar</exclude> + <exclude>*-tests.jar</exclude> + </excludes> + <fileMode>0644</fileMode> + </fileSet> + <fileSet> + <directory>${project.basedir}/../phoenix-queryserver-client/target/</directory> + <outputDirectory>${parcel.folder}/lib/phoenix/lib</outputDirectory> + <includes> + <include>phoenix-*.jar</include> + </includes> + <excludes> + <exclude>*-minimal.jar</exclude> + <exclude>*-sources.jar</exclude> + <exclude>*-tests.jar</exclude> + <!-- this one goes in project root instead --> + <exclude>phoenix-*-thin-client.jar</exclude> + </excludes> + <fileMode>0644</fileMode> + </fileSet> + <fileSet> + <directory>${project.basedir}/../phoenix-pherf/target/</directory> + <outputDirectory>${parcel.folder}/lib/phoenix/lib</outputDirectory> + <includes> + <include>phoenix-*.jar</include> + </includes> + <excludes> + <exclude>*-minimal.jar</exclude> + <exclude>*-sources.jar</exclude> + <exclude>*-tests.jar</exclude> + </excludes> + <fileMode>0644</fileMode> + </fileSet> + </fileSets> +</component> http://git-wip-us.apache.org/repos/asf/phoenix/blob/024f0f22/phoenix-parcel/src/build/manifest/make_manifest.py ---------------------------------------------------------------------- diff --git a/phoenix-parcel/src/build/manifest/make_manifest.py b/phoenix-parcel/src/build/manifest/make_manifest.py new file mode 100755 index 0000000..38a9dc5 --- /dev/null +++ b/phoenix-parcel/src/build/manifest/make_manifest.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python +# +# Licensed to Cloudera, Inc. under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. Cloudera, Inc. licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This program creates a manifest.json file from a directory of parcels and +# places the file in the same directory as the parcels. +# Once created, the directory can be served over http as a parcel repository. + +import hashlib +import json +import os +import re +import sys +import tarfile +import time + +def _get_parcel_dirname(parcel_name): + """ + Extract the required parcel directory name for a given parcel. + + eg: CDH-5.0.0-el6.parcel -> CDH-5.0.0 + """ + parts = re.match(r"^(.*?)-(.*)-(.*?)$", parcel_name).groups() + return parts[0] + '-' + parts[1] + +def _safe_copy(key, src, dest): + """ + Conditionally copy a key/value pair from one dictionary to another. + + Nothing is done if the key is not present in the source dictionary + """ + if key in src: + dest[key] = src[key] + +def make_manifest(path, timestamp=time.time()): + """ + Make a manifest.json document from the contents of a directory. + + This function will scan the specified directory, identify any parcel files + in it, and then build a manifest from those files. Certain metadata will be + extracted from the parcel and copied into the manifest. + + @param path: The path of the directory to scan for parcels + @param timestamp: Unix timestamp to place in manifest.json + @return: the manifest.json as a string + """ + manifest = {} + manifest['lastUpdated'] = int(timestamp * 1000) + manifest['parcels'] = [] + + files = os.listdir(path) + for f in files: + if not f.endswith('.parcel'): + continue + + print("Found parcel %s" % (f,)) + entry = {} + entry['parcelName'] = f + + fullpath = os.path.join(path, f) + + with open(fullpath, 'rb') as fp: + entry['hash'] = hashlib.sha1(fp.read()).hexdigest() + + with tarfile.open(fullpath, 'r') as tar: + try: + json_member = tar.getmember(os.path.join(_get_parcel_dirname(f), + 'meta', 'parcel.json')) + except KeyError: + print("Parcel does not contain parcel.json") + continue + try: + parcel = json.loads(tar.extractfile(json_member).read().decode(encoding='UTF-8')) + except: + print("Failed to parse parcel.json") + continue + _safe_copy('depends', parcel, entry) + _safe_copy('replaces', parcel, entry) + _safe_copy('conflicts', parcel, entry) + _safe_copy('components', parcel, entry) + _safe_copy('servicesRestartInfo', parcel, entry) + + try: + notes_member = tar.getmember(os.path.join(_get_parcel_dirname(f), + 'meta', 'release-notes.txt')) + entry['releaseNotes'] = tar.extractfile(notes_member).read().decode(encoding='UTF-8') + except KeyError: + # No problem if there's no release notes + pass + + manifest['parcels'].append(entry) + + return json.dumps(manifest, indent=4, separators=(',', ': ')) + +if __name__ == "__main__": + path = os.path.curdir + if len(sys.argv) > 1: + path = sys.argv[1] + print("Scanning directory: %s" % (path)) + + manifest = make_manifest(path) + with open(os.path.join(path, 'manifest.json'), 'w') as fp: + fp.write(manifest) http://git-wip-us.apache.org/repos/asf/phoenix/blob/024f0f22/phoenix-parcel/src/build/parcel.xml ---------------------------------------------------------------------- diff --git a/phoenix-parcel/src/build/parcel.xml b/phoenix-parcel/src/build/parcel.xml new file mode 100644 index 0000000..91fe978 --- /dev/null +++ b/phoenix-parcel/src/build/parcel.xml @@ -0,0 +1,40 @@ +<?xml version='1.0'?> +<!-- + + Licensed to the Apache Software Foundation (ASF) under one + or more contributor license agreements. See the NOTICE file + distributed with this work for additional information + regarding copyright ownership. The ASF licenses this file + to you under the Apache License, Version 2.0 (the + "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, + software distributed under the License is distributed on an + "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + KIND, either express or implied. See the License for the + specific language governing permissions and limitations + under the License. + +--> + +<assembly xmlns="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0"; + xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/plugins/maven-assembly-plugin/assembly/1.1.0 http://maven.apache.org/xsd/assembly-1.1.0.xsd";> + + <!--This 'all' id is not appended to the produced bundle because we do this: http://maven.apache.org/plugins/maven-assembly-plugin/faq.html#required-classifiers --> + <id>all</id> + <formats> + <format>tar</format> + </formats> + <includeBaseDirectory>false</includeBaseDirectory> + + <componentDescriptors> + <componentDescriptor>src/build/components/all-common-files.xml</componentDescriptor> + <componentDescriptor>src/build/components/all-common-jars.xml</componentDescriptor> + <componentDescriptor>src/build/components/all-common-dependencies.xml</componentDescriptor> + </componentDescriptors> + +</assembly> \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/024f0f22/phoenix-parcel/src/parcel/bin/phoenix-performance.py ---------------------------------------------------------------------- diff --git a/phoenix-parcel/src/parcel/bin/phoenix-performance.py b/phoenix-parcel/src/parcel/bin/phoenix-performance.py new file mode 100755 index 0000000..e51a298 --- /dev/null +++ b/phoenix-parcel/src/parcel/bin/phoenix-performance.py @@ -0,0 +1,19 @@ +#!/bin/bash + # Reference: http://stackoverflow.com/questions/59895/can-a-bash-script-tell-what-directory-its-stored-in + SOURCE="${BASH_SOURCE[0]}" + BIN_DIR="$( dirname "$SOURCE" )" + while [ -h "$SOURCE" ] + do + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + BIN_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + done + BIN_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + LIB_DIR=$BIN_DIR/../lib + + +# Autodetect JAVA_HOME if not defined +. $LIB_DIR/../../CDH/lib/bigtop-utils/bigtop-detect-javahome + +export PATH=$JAVA_HOME/jre/bin:$PATH +exec $LIB_DIR/phoenix/bin/performance.py "$@" http://git-wip-us.apache.org/repos/asf/phoenix/blob/024f0f22/phoenix-parcel/src/parcel/bin/phoenix-psql.py ---------------------------------------------------------------------- diff --git a/phoenix-parcel/src/parcel/bin/phoenix-psql.py b/phoenix-parcel/src/parcel/bin/phoenix-psql.py new file mode 100755 index 0000000..5eed4f4 --- /dev/null +++ b/phoenix-parcel/src/parcel/bin/phoenix-psql.py @@ -0,0 +1,19 @@ +#!/bin/bash + # Reference: http://stackoverflow.com/questions/59895/can-a-bash-script-tell-what-directory-its-stored-in + SOURCE="${BASH_SOURCE[0]}" + BIN_DIR="$( dirname "$SOURCE" )" + while [ -h "$SOURCE" ] + do + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + BIN_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + done + BIN_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + LIB_DIR=$BIN_DIR/../lib + + +# Autodetect JAVA_HOME if not defined +. $LIB_DIR/../../CDH/lib/bigtop-utils/bigtop-detect-javahome + +export PATH=$JAVA_HOME/jre/bin:$PATH +exec $LIB_DIR/phoenix/bin/psql.py "$@" http://git-wip-us.apache.org/repos/asf/phoenix/blob/024f0f22/phoenix-parcel/src/parcel/bin/phoenix-sqlline.py ---------------------------------------------------------------------- diff --git a/phoenix-parcel/src/parcel/bin/phoenix-sqlline.py b/phoenix-parcel/src/parcel/bin/phoenix-sqlline.py new file mode 100755 index 0000000..0c29a4f --- /dev/null +++ b/phoenix-parcel/src/parcel/bin/phoenix-sqlline.py @@ -0,0 +1,20 @@ +#!/bin/bash + # Reference: http://stackoverflow.com/questions/59895/can-a-bash-script-tell-what-directory-its-stored-in + SOURCE="${BASH_SOURCE[0]}" + BIN_DIR="$( dirname "$SOURCE" )" + while [ -h "$SOURCE" ] + do + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + BIN_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + done + BIN_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + LIB_DIR=$BIN_DIR/../lib + + +# Autodetect JAVA_HOME if not defined +. $LIB_DIR/../../CDH/lib/bigtop-utils/bigtop-detect-javahome + +export HBASE_CONF_PATH=${HBASE_CONF_PATH:-/etc/hbase/conf} +export PATH=$JAVA_HOME/jre/bin:$PATH +exec $LIB_DIR/phoenix/bin/sqlline.py "$@" http://git-wip-us.apache.org/repos/asf/phoenix/blob/024f0f22/phoenix-parcel/src/parcel/bin/phoenix-utils.py ---------------------------------------------------------------------- diff --git a/phoenix-parcel/src/parcel/bin/phoenix-utils.py b/phoenix-parcel/src/parcel/bin/phoenix-utils.py new file mode 100755 index 0000000..8b83a87 --- /dev/null +++ b/phoenix-parcel/src/parcel/bin/phoenix-utils.py @@ -0,0 +1,19 @@ +#!/bin/bash + # Reference: http://stackoverflow.com/questions/59895/can-a-bash-script-tell-what-directory-its-stored-in + SOURCE="${BASH_SOURCE[0]}" + BIN_DIR="$( dirname "$SOURCE" )" + while [ -h "$SOURCE" ] + do + SOURCE="$(readlink "$SOURCE")" + [[ $SOURCE != /* ]] && SOURCE="$DIR/$SOURCE" + BIN_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + done + BIN_DIR="$( cd -P "$( dirname "$SOURCE" )" && pwd )" + LIB_DIR=$BIN_DIR/../lib + + +# Autodetect JAVA_HOME if not defined +. $LIB_DIR/../../CDH/lib/bigtop-utils/bigtop-detect-javahome + +export PATH=$JAVA_HOME/jre/bin:$PATH +exec $LIB_DIR/phoenix/bin/phoenix_utils.py "$@" http://git-wip-us.apache.org/repos/asf/phoenix/blob/024f0f22/phoenix-parcel/src/parcel/cloudera/cdh_version.properties ---------------------------------------------------------------------- diff --git a/phoenix-parcel/src/parcel/cloudera/cdh_version.properties b/phoenix-parcel/src/parcel/cloudera/cdh_version.properties new file mode 100644 index 0000000..e69de29 http://git-wip-us.apache.org/repos/asf/phoenix/blob/024f0f22/phoenix-parcel/src/parcel/meta/alternatives.json ---------------------------------------------------------------------- diff --git a/phoenix-parcel/src/parcel/meta/alternatives.json b/phoenix-parcel/src/parcel/meta/alternatives.json new file mode 100644 index 0000000..3b49b12 --- /dev/null +++ b/phoenix-parcel/src/parcel/meta/alternatives.json @@ -0,0 +1,26 @@ +{ + "phoenix-performance.py": { + "destination": "/usr/bin/phoenix-performance.py", + "source": "bin/phoenix-performance.py", + "priority": 10, + "isDirectory": false + }, + "phoenix-psql.py": { + "destination": "/usr/bin/phoenix-psql.py", + "source": "bin/phoenix-psql.py", + "priority": 10, + "isDirectory": false + }, + "phoenix-sqlline.py": { + "destination": "/usr/bin/phoenix-sqlline.py", + "source": "bin/phoenix-sqlline.py", + "priority": 10, + "isDirectory": false + }, + "phoenix-utils.py" : { + "destination": "/usr/bin/phoenix-utils.py", + "source": "bin/phoenix-utils.py", + "priority": 10, + "isDirectory": false + } +} http://git-wip-us.apache.org/repos/asf/phoenix/blob/024f0f22/phoenix-parcel/src/parcel/meta/parcel.json ---------------------------------------------------------------------- diff --git a/phoenix-parcel/src/parcel/meta/parcel.json b/phoenix-parcel/src/parcel/meta/parcel.json new file mode 100644 index 0000000..0be423a --- /dev/null +++ b/phoenix-parcel/src/parcel/meta/parcel.json @@ -0,0 +1,34 @@ +{ + "schema_version": 1, + "name": "APACHE_PHOENIX", + "version": "${parcel.version}", + "groups": [], + "extraVersionInfo": { + "baseVersion": "${parcel.base.version}", + "fullVersion": "${parcel.full.version}", + "patchCount": "${parcel.patch.count}" + }, + "packages": [ + { + "version": "${parcel.package.version}", + "name": "phoenix" + } + ], + "components": [ + { + "name": "phoenix", + "version": "${parcel.component.version}", + "pkg_version": "${parcel.package.version}", + "pkg_release": "${parcel.component.release}" + } + ], + "scripts": { + "defines": "phoenix_env.sh" + }, + "depends": "${parcel.depends}", + "provides": [ + "hbase-plugin" + ], + "setActiveSymlink": true, + "users": {} +} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/phoenix/blob/024f0f22/phoenix-parcel/src/parcel/meta/phoenix_env.sh ---------------------------------------------------------------------- diff --git a/phoenix-parcel/src/parcel/meta/phoenix_env.sh b/phoenix-parcel/src/parcel/meta/phoenix_env.sh new file mode 100755 index 0000000..757e7ff --- /dev/null +++ b/phoenix-parcel/src/parcel/meta/phoenix_env.sh @@ -0,0 +1,27 @@ +#!/bin/bash +set -ex + + +#The following is written to aid local testing +if [ -z $PARCELS_ROOT ] ; then + export MYDIR=`dirname "${BASH_SOURCE[0]}"` + PARCELS_ROOT=`cd $MYDIR/../.. && pwd` +fi +PARCEL_DIRNAME=${PARCEL_DIRNAME-PHOENIX} + +MYLIBDIR=${PARCELS_ROOT}/${PARCEL_DIRNAME}/lib/phoenix + +[ -d $MYLIBDIR ] || { + echo "Could not find phoenix parcel lib dir, exiting" >&2 + exit 1 +} + +APPENDSTRING=`echo ${MYLIBDIR}/*.jar | sed 's/ /:/g'` +echo "appending '$APPENDSTRING' to HBASE_CLASSPATH" +if [ -z $HBASE_CLASSPATH ] ; then + export HBASE_CLASSPATH=$APPENDSTRING +else + export HBASE_CLASSPATH="$HBASE_CLASSPATH:$APPENDSTRING" +fi +echo "Set HBASE_CLASSPATH to '$HBASE_CLASSPATH'" +echo "phoenix_env.sh successfully executed at `date`" http://git-wip-us.apache.org/repos/asf/phoenix/blob/024f0f22/phoenix-pherf/pom.xml ---------------------------------------------------------------------- diff --git a/phoenix-pherf/pom.xml b/phoenix-pherf/pom.xml index 1da6179..24cba5b 100644 --- a/phoenix-pherf/pom.xml +++ b/phoenix-pherf/pom.xml @@ -15,7 +15,7 @@ <parent> <groupId>org.apache.phoenix</groupId> <artifactId>phoenix</artifactId> - <version>4.13.0-HBase-1.2</version> + <version>4.13.0-cdh5.11.2</version> </parent> <artifactId>phoenix-pherf</artifactId>
