Nuria has submitted this change and it was merged.
Change subject: Update casssandra loading classes for new AQS
......................................................................
Update casssandra loading classes for new AQS
Bug: T147841
Change-Id: Ic3653201fbb8fdf095ad99329ba1a3d01387b2a8
---
M refinery-cassandra/pom.xml
M
refinery-cassandra/src/main/java/org/wikimedia/analytics/refinery/cassandra/CassandraXSVLoader.java
M
refinery-cassandra/src/test/java/org/wikimedia/analytics/refinery/cassandra/TestCassandraXSVLoader.java
3 files changed, 18 insertions(+), 8 deletions(-)
Approvals:
Nuria: Looks good to me, approved
jenkins-bot: Verified
diff --git a/refinery-cassandra/pom.xml b/refinery-cassandra/pom.xml
index e875015..8a87da5 100644
--- a/refinery-cassandra/pom.xml
+++ b/refinery-cassandra/pom.xml
@@ -23,6 +23,10 @@
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
</exclusion>
+ <exclusion>
+ <groupId>com.google.collections</groupId>
+ <artifactId>google-collections</artifactId>
+ </exclusion>
</exclusions>
<scope>provided</scope>
</dependency>
@@ -35,6 +39,10 @@
<exclusion>
<groupId>com.google.guava</groupId>
<artifactId>guava</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.collections</groupId>
+ <artifactId>google-collections</artifactId>
</exclusion>
</exclusions>
<scope>provided</scope>
@@ -55,7 +63,7 @@
<dependency>
<groupId>org.apache.cassandra</groupId>
<artifactId>cassandra-all</artifactId>
- <version>2.2.2</version>
+ <version>2.2.6</version>
</dependency>
<dependency>
diff --git
a/refinery-cassandra/src/main/java/org/wikimedia/analytics/refinery/cassandra/CassandraXSVLoader.java
b/refinery-cassandra/src/main/java/org/wikimedia/analytics/refinery/cassandra/CassandraXSVLoader.java
index 5dfc5af..ee2bf83 100644
---
a/refinery-cassandra/src/main/java/org/wikimedia/analytics/refinery/cassandra/CassandraXSVLoader.java
+++
b/refinery-cassandra/src/main/java/org/wikimedia/analytics/refinery/cassandra/CassandraXSVLoader.java
@@ -41,10 +41,12 @@
* Map-Reduce job loading data into a Cassandra cluster from separated-value
files.
*
* Usage:
- * hadoop jar /path/to/refinery-job.jar
org.wikimedia.analytics.refinery.job.CassandraXSVLoader \
+ * hadoop jar /path/to/refinery-job.jar
org.wikimedia.analytics.refinery.cassandra.CassandraXSVLoader \
+ * -D mapreduce.job.user.classpath.first=true \
+ * -D mapreduce.job.reduces=X \
* -D cassandra_host=your_host \
- * -D cassandra_user=cassandra \
- * -D cassandra_passwd=cassandra \
+ * -D cassandra_username=aqsloader \
+ * -D cassandra_password=cassandra \
* -D input_path=/path/to/xsv/file/or/folder \
* -D input_separator=, \
* -D input_fields=text_field,,int_field,double_field \
@@ -280,6 +282,9 @@
job.setOutputKeyClass(Map.class);
job.setOutputValueClass(List.class);
+ // Use cassandra cql output format
+ // This is where the actual connection and data push
+ // to cassandra is made
job.setOutputFormatClass(CqlOutputFormat.class);
ConfigHelper.setOutputColumnFamily(
diff --git
a/refinery-cassandra/src/test/java/org/wikimedia/analytics/refinery/cassandra/TestCassandraXSVLoader.java
b/refinery-cassandra/src/test/java/org/wikimedia/analytics/refinery/cassandra/TestCassandraXSVLoader.java
index 29081ab..7e40052 100644
---
a/refinery-cassandra/src/test/java/org/wikimedia/analytics/refinery/cassandra/TestCassandraXSVLoader.java
+++
b/refinery-cassandra/src/test/java/org/wikimedia/analytics/refinery/cassandra/TestCassandraXSVLoader.java
@@ -10,11 +10,8 @@
import java.io.IOException;
import static org.junit.Assert.*;
-import static org.mockito.Mockito.when;
-/**
- * Created by jo on 8/5/15.
- */
+
public class TestCassandraXSVLoader {
private Configuration configurationMock =
Mockito.mock(Configuration.class);
--
To view, visit https://gerrit.wikimedia.org/r/295663
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: merged
Gerrit-Change-Id: Ic3653201fbb8fdf095ad99329ba1a3d01387b2a8
Gerrit-PatchSet: 5
Gerrit-Project: analytics/refinery/source
Gerrit-Branch: master
Gerrit-Owner: Joal <j...@wikimedia.org>
Gerrit-Reviewer: Elukey <ltosc...@wikimedia.org>
Gerrit-Reviewer: Joal <j...@wikimedia.org>
Gerrit-Reviewer: Nuria <nu...@wikimedia.org>
Gerrit-Reviewer: Ottomata <o...@wikimedia.org>
Gerrit-Reviewer: jenkins-bot <>
_______________________________________________
MediaWiki-commits mailing list
MediaWiki-commits@lists.wikimedia.org
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
