Nuria has submitted this change and it was merged. Change subject: Update casssandra loading classes for new AQS ......................................................................
Update casssandra loading classes for new AQS Bug: T147841 Change-Id: Ic3653201fbb8fdf095ad99329ba1a3d01387b2a8 --- M refinery-cassandra/pom.xml M refinery-cassandra/src/main/java/org/wikimedia/analytics/refinery/cassandra/CassandraXSVLoader.java M refinery-cassandra/src/test/java/org/wikimedia/analytics/refinery/cassandra/TestCassandraXSVLoader.java 3 files changed, 18 insertions(+), 8 deletions(-) Approvals: Nuria: Looks good to me, approved jenkins-bot: Verified diff --git a/refinery-cassandra/pom.xml b/refinery-cassandra/pom.xml index e875015..8a87da5 100644 --- a/refinery-cassandra/pom.xml +++ b/refinery-cassandra/pom.xml @@ -23,6 +23,10 @@ <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> </exclusion> + <exclusion> + <groupId>com.google.collections</groupId> + <artifactId>google-collections</artifactId> + </exclusion> </exclusions> <scope>provided</scope> </dependency> @@ -35,6 +39,10 @@ <exclusion> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> + </exclusion> + <exclusion> + <groupId>com.google.collections</groupId> + <artifactId>google-collections</artifactId> </exclusion> </exclusions> <scope>provided</scope> @@ -55,7 +63,7 @@ <dependency> <groupId>org.apache.cassandra</groupId> <artifactId>cassandra-all</artifactId> - <version>2.2.2</version> + <version>2.2.6</version> </dependency> <dependency> diff --git a/refinery-cassandra/src/main/java/org/wikimedia/analytics/refinery/cassandra/CassandraXSVLoader.java b/refinery-cassandra/src/main/java/org/wikimedia/analytics/refinery/cassandra/CassandraXSVLoader.java index 5dfc5af..ee2bf83 100644 --- a/refinery-cassandra/src/main/java/org/wikimedia/analytics/refinery/cassandra/CassandraXSVLoader.java +++ b/refinery-cassandra/src/main/java/org/wikimedia/analytics/refinery/cassandra/CassandraXSVLoader.java @@ -41,10 +41,12 @@ * Map-Reduce job loading data into a Cassandra cluster from separated-value files. * * Usage: - * hadoop jar /path/to/refinery-job.jar org.wikimedia.analytics.refinery.job.CassandraXSVLoader \ + * hadoop jar /path/to/refinery-job.jar org.wikimedia.analytics.refinery.cassandra.CassandraXSVLoader \ + * -D mapreduce.job.user.classpath.first=true \ + * -D mapreduce.job.reduces=X \ * -D cassandra_host=your_host \ - * -D cassandra_user=cassandra \ - * -D cassandra_passwd=cassandra \ + * -D cassandra_username=aqsloader \ + * -D cassandra_password=cassandra \ * -D input_path=/path/to/xsv/file/or/folder \ * -D input_separator=, \ * -D input_fields=text_field,,int_field,double_field \ @@ -280,6 +282,9 @@ job.setOutputKeyClass(Map.class); job.setOutputValueClass(List.class); + // Use cassandra cql output format + // This is where the actual connection and data push + // to cassandra is made job.setOutputFormatClass(CqlOutputFormat.class); ConfigHelper.setOutputColumnFamily( diff --git a/refinery-cassandra/src/test/java/org/wikimedia/analytics/refinery/cassandra/TestCassandraXSVLoader.java b/refinery-cassandra/src/test/java/org/wikimedia/analytics/refinery/cassandra/TestCassandraXSVLoader.java index 29081ab..7e40052 100644 --- a/refinery-cassandra/src/test/java/org/wikimedia/analytics/refinery/cassandra/TestCassandraXSVLoader.java +++ b/refinery-cassandra/src/test/java/org/wikimedia/analytics/refinery/cassandra/TestCassandraXSVLoader.java @@ -10,11 +10,8 @@ import java.io.IOException; import static org.junit.Assert.*; -import static org.mockito.Mockito.when; -/** - * Created by jo on 8/5/15. - */ + public class TestCassandraXSVLoader { private Configuration configurationMock = Mockito.mock(Configuration.class); -- To view, visit https://gerrit.wikimedia.org/r/295663 To unsubscribe, visit https://gerrit.wikimedia.org/r/settings Gerrit-MessageType: merged Gerrit-Change-Id: Ic3653201fbb8fdf095ad99329ba1a3d01387b2a8 Gerrit-PatchSet: 5 Gerrit-Project: analytics/refinery/source Gerrit-Branch: master Gerrit-Owner: Joal <j...@wikimedia.org> Gerrit-Reviewer: Elukey <ltosc...@wikimedia.org> Gerrit-Reviewer: Joal <j...@wikimedia.org> Gerrit-Reviewer: Nuria <nu...@wikimedia.org> Gerrit-Reviewer: Ottomata <o...@wikimedia.org> Gerrit-Reviewer: jenkins-bot <> _______________________________________________ MediaWiki-commits mailing list MediaWiki-commits@lists.wikimedia.org https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits