Ottomata has submitted this change and it was merged. Change subject: Recreate wmf branch from HEAD of master, depend on cdh5.4.0, bump to version wmf6 ......................................................................
Recreate wmf branch from HEAD of master, depend on cdh5.4.0, bump to version wmf6 Bug: T109860 Change-Id: I1d898beb916b00e0fd9a4234d351e8d3c1cd6f28 --- M README.md M camus-api/pom.xml M camus-etl-kafka/pom.xml M camus-example/pom.xml M camus-kafka-coders/pom.xml M camus-schema-registry-avro/pom.xml M camus-schema-registry/pom.xml M camus-sweeper/pom.xml A camus-wmf/pom.xml M pom.xml 10 files changed, 112 insertions(+), 21 deletions(-) Approvals: Ottomata: Verified; Looks good to me, approved diff --git a/README.md b/README.md index b5ec313..df28266 100644 --- a/README.md +++ b/README.md @@ -12,6 +12,29 @@ There is a [Google Groups mailing list](https://groups.google.com/forum/#!forum/camus_etl "Google Groups mailing list") that you can email or search if you have any questions. For a more detailed documentation on the main Camus components, please see [Camus InputFormat and OutputFormat Behavior](https://github.com/linkedin/camus/wiki/Camus-InputFormat-and-OutputFormat-Behavior "Camus InputFormat and OutputFormat Behavior") + +## Wikimedia Fork & Branches +Wikimedia uses Camus, but we use different versions of Hadoop and +Kafka than LinkedIn. Wikimedia maintains 2 branches in our Camus repository. + +The wmf branch tracks either wikimedia or master, but has modifications +that will not be upstreamed to LinkedIn. Notably, Wikimedia uses +Cloudera's Hadoop and Kafka from Maven Central. We also use this +branch to roll Camus builds versioned with 'wmf' in the name. + +The wikimedia branch is meant to track LinkedIn's master, and is used +to submit Github pull requests to LinkedIn. It should only be out of sync +with LinkedIn's master if there is an outstanding pull request. + +Wikimedia primarily uses Gerrit, not Github, and replicates Gerrit hosted +repositories to Github. Since Github cannot track forks of replicated +repositories (they are pushed directly to Github), they cannot be used +to submit pull requests. Wikimedia maintains a manual Github repository +at https://github.com/wikimedia/camus from which pull requests are sent. +the 'wmf' branch which is used for building Camus for Wikimedia is +only present in Gerrit and its replicated Github repository at +https://github.com/wikimedia/analytics-camus. + # Brief Overview All work is done within a single Hadoop job divided into three stages: diff --git a/camus-api/pom.xml b/camus-api/pom.xml index e3582bd..dbab7c6 100644 --- a/camus-api/pom.xml +++ b/camus-api/pom.xml @@ -5,7 +5,7 @@ <parent> <groupId>com.linkedin.camus</groupId> <artifactId>camus-parent</artifactId> - <version>0.1.0-SNAPSHOT</version> + <version>0.1.0-wmf6</version> </parent> <artifactId>camus-api</artifactId> diff --git a/camus-etl-kafka/pom.xml b/camus-etl-kafka/pom.xml index eec17c2..4b874ee 100644 --- a/camus-etl-kafka/pom.xml +++ b/camus-etl-kafka/pom.xml @@ -5,7 +5,7 @@ <parent> <groupId>com.linkedin.camus</groupId> <artifactId>camus-parent</artifactId> - <version>0.1.0-SNAPSHOT</version> + <version>0.1.0-wmf6</version> </parent> <artifactId>camus-etl-kafka</artifactId> diff --git a/camus-example/pom.xml b/camus-example/pom.xml index 86b47e8..85cfb86 100644 --- a/camus-example/pom.xml +++ b/camus-example/pom.xml @@ -5,7 +5,7 @@ <parent> <groupId>com.linkedin.camus</groupId> <artifactId>camus-parent</artifactId> - <version>0.1.0-SNAPSHOT</version> + <version>0.1.0-wmf6</version> </parent> <artifactId>camus-example</artifactId> diff --git a/camus-kafka-coders/pom.xml b/camus-kafka-coders/pom.xml index ffd6869..59c9126 100644 --- a/camus-kafka-coders/pom.xml +++ b/camus-kafka-coders/pom.xml @@ -5,7 +5,7 @@ <parent> <groupId>com.linkedin.camus</groupId> <artifactId>camus-parent</artifactId> - <version>0.1.0-SNAPSHOT</version> + <version>0.1.0-wmf6</version> </parent> <artifactId>camus-kafka-coders</artifactId> diff --git a/camus-schema-registry-avro/pom.xml b/camus-schema-registry-avro/pom.xml index 7ec13bb..6a95632 100644 --- a/camus-schema-registry-avro/pom.xml +++ b/camus-schema-registry-avro/pom.xml @@ -5,7 +5,7 @@ <parent> <groupId>com.linkedin.camus</groupId> <artifactId>camus-parent</artifactId> - <version>0.1.0-SNAPSHOT</version> + <version>0.1.0-wmf6</version> </parent> <artifactId>camus-schema-registry-avro</artifactId> diff --git a/camus-schema-registry/pom.xml b/camus-schema-registry/pom.xml index 1927f46..702a80f 100644 --- a/camus-schema-registry/pom.xml +++ b/camus-schema-registry/pom.xml @@ -5,7 +5,7 @@ <parent> <groupId>com.linkedin.camus</groupId> <artifactId>camus-parent</artifactId> - <version>0.1.0-SNAPSHOT</version> + <version>0.1.0-wmf6</version> </parent> <artifactId>camus-schema-registry</artifactId> diff --git a/camus-sweeper/pom.xml b/camus-sweeper/pom.xml index 3e87ef1..bd337bb 100644 --- a/camus-sweeper/pom.xml +++ b/camus-sweeper/pom.xml @@ -4,12 +4,12 @@ <groupId>com.linkedin.camus</groupId> <artifactId>camus-sweeper</artifactId> <name>Camus compaction code small files produced Camus</name> - <version>0.1.0-SNAPSHOT</version> - + <version>0.1.0-wmf6</version> + <parent> <groupId>com.linkedin.camus</groupId> <artifactId>camus-parent</artifactId> - <version>0.1.0-SNAPSHOT</version> + <version>0.1.0-wmf6</version> </parent> <dependencies> @@ -47,7 +47,7 @@ <artifactId>paranamer</artifactId> <version>2.2</version> </dependency> - + <dependency> <groupId>org.easymock</groupId> <artifactId>easymock</artifactId> diff --git a/camus-wmf/pom.xml b/camus-wmf/pom.xml new file mode 100644 index 0000000..4dafa8f --- /dev/null +++ b/camus-wmf/pom.xml @@ -0,0 +1,55 @@ +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + + <parent> + <groupId>com.linkedin.camus</groupId> + <artifactId>camus-parent</artifactId> + <version>0.1.0-wmf6</version> + </parent> + + <groupId>org.wikimedia.analytics</groupId> + <artifactId>camus-wmf</artifactId> + <version>0.1.0-wmf6</version> + <name>Camus Wikimedia</name> + <description> + This subproject exists to build a shaded jar + that is usesable on Wikimedia's Analytics cluster. + It depends only on what is needed from other + Camus subprojects to generate a shaded jar. + </description> + <packaging>jar</packaging> + + <dependencies> + <dependency> + <groupId>com.linkedin.camus</groupId> + <artifactId>camus-etl-kafka</artifactId> + <version>0.1.0-wmf6</version> + <scope>compile</scope> + </dependency> + </dependencies> + + <build> + <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-shade-plugin</artifactId> + <version>2.0</version> + <configuration> + <shadedArtifactAttached>false</shadedArtifactAttached> + </configuration> + <executions> + <execution> + <phase>package</phase> + <goals> + <goal>shade</goal> + </goals> + <configuration> + <createDependencyReducedPom>false</createDependencyReducedPom> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> +</project> \ No newline at end of file diff --git a/pom.xml b/pom.xml index ddddf4e..23eebf5 100644 --- a/pom.xml +++ b/pom.xml @@ -9,7 +9,7 @@ <groupId>com.linkedin.camus</groupId> <artifactId>camus-parent</artifactId> - <version>0.1.0-SNAPSHOT</version> + <version>0.1.0-wmf6</version> <packaging>pom</packaging> <name>Camus Parent</name> <description> @@ -26,27 +26,27 @@ <dependency> <groupId>com.linkedin.camus</groupId> <artifactId>camus-api</artifactId> - <version>0.1.0-SNAPSHOT</version> + <version>0.1.0-wmf6</version> </dependency> <dependency> <groupId>com.linkedin.camus</groupId> <artifactId>camus-etl-kafka</artifactId> - <version>0.1.0-SNAPSHOT</version> + <version>0.1.0-wmf6</version> </dependency> <dependency> <groupId>com.linkedin.camus</groupId> <artifactId>camus-kafka-coders</artifactId> - <version>0.1.0-SNAPSHOT</version> + <version>0.1.0-wmf6</version> </dependency> <dependency> <groupId>com.linkedin.camus</groupId> <artifactId>camus-schema-registry</artifactId> - <version>0.1.0-SNAPSHOT</version> + <version>0.1.0-wmf6</version> </dependency> <dependency> <groupId>com.linkedin.camus</groupId> <artifactId>camus-schema-registry</artifactId> - <version>0.1.0-SNAPSHOT</version> + <version>0.1.0-wmf6</version> <type>test-jar</type> <scope>test</scope> </dependency> @@ -78,17 +78,17 @@ <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-mapreduce-client-core</artifactId> - <version>2.3.0</version> + <version>2.6.0-cdh5.4.0</version> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-mapreduce-client-common</artifactId> - <version>2.3.0</version> + <version>2.6.0-cdh5.4.0</version> </dependency> <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> - <version>2.3.0</version> + <version>2.6.0-cdh5.4.0</version> </dependency> <dependency> <groupId>org.easymock</groupId> @@ -166,6 +166,7 @@ <module>camus-schema-registry-avro</module> <module>camus-example</module> <module>camus-sweeper</module> + <module>camus-wmf</module> </modules> <licenses> @@ -189,8 +190,19 @@ <repositories> <repository> - <id>apache-releases</id> - <url>https://repository.apache.org/content/groups/public</url> + <id>central</id> + <url>http://repo1.maven.org/maven2</url> + <releases> + <enabled>false</enabled> + </releases> + <snapshots> + <enabled>false</enabled> + </snapshots> + </repository> + + <repository> + <id>wikimedia.mirrored</id> + <url>http://archiva.wikimedia.org/repository/mirrored/</url> </repository> </repositories> @@ -253,4 +265,5 @@ </plugin> </plugins> </build> + </project> -- To view, visit https://gerrit.wikimedia.org/r/233036 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: I1d898beb916b00e0fd9a4234d351e8d3c1cd6f28 Gerrit-PatchSet: 3 Gerrit-Project: analytics/camus Gerrit-Branch: wmf Gerrit-Owner: Ottomata <[email protected]> Gerrit-Reviewer: Ottomata <[email protected]> _______________________________________________ MediaWiki-commits mailing list [email protected] https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
