Joal has uploaded a new change for review.
https://gerrit.wikimedia.org/r/315241
Change subject: Update cassandra load jobs to match new aqs needs
......................................................................
Update cassandra load jobs to match new aqs needs
Old aqs nodes are being removed, new cassandra loading
configuration is to be applied in production for new-aqs.
Bug: T147841
Change-Id: Ib966c4cf3726f1892409d98dad4126967238839e
---
M oozie/cassandra/bundle.properties
M oozie/cassandra/bundle.xml
M oozie/cassandra/coord_per_article_daily.properties
M oozie/cassandra/coord_per_project_daily.properties
M oozie/cassandra/coord_per_project_hourly.properties
M oozie/cassandra/coord_per_project_monthly.properties
M oozie/cassandra/coord_top_articles_daily.properties
M oozie/cassandra/coord_top_articles_monthly.properties
M oozie/cassandra/coord_unique_devices_daily.properties
M oozie/cassandra/coord_unique_devices_monthly.properties
10 files changed, 42 insertions(+), 42 deletions(-)
git pull ssh://gerrit.wikimedia.org:29418/analytics/refinery
refs/changes/41/315241/1
diff --git a/oozie/cassandra/bundle.properties
b/oozie/cassandra/bundle.properties
index 332cd67..34bf4a1 100644
--- a/oozie/cassandra/bundle.properties
+++ b/oozie/cassandra/bundle.properties
@@ -21,7 +21,7 @@
refinery_directory = ${name_node}/wmf/refinery/current
# HDFS path to the refinery job jar that will be used by this job.
-refinery_cassandra_jar_path =
${refinery_directory}/artifacts/org/wikimedia/analytics/refinery/refinery-cassandra-0.0.30.jar
+refinery_cassandra_jar_path =
${refinery_directory}/artifacts/org/wikimedia/analytics/refinery/refinery-cassandra-0.0.36.jar
cassandra_reducer_class =
org.wikimedia.analytics.refinery.cassandra.ReducerToCassandra
cassandra_output_format_class =
org.wikimedia.analytics.refinery.cassandra.CqlOutputFormat
@@ -62,11 +62,11 @@
temporary_directory = ${name_node}/tmp
# Cassandra cluster info
-cassandra_host = aqs1001.eqiad.wmnet
+cassandra_host = aqs1004-a.eqiad.wmnet
cassandra_port = 9042
-cassandra_username = cassandra
+cassandra_username = aqsloader
cassandra_password = cassandra
-cassandra_nodes = 3
+cassandra_nodes = 6
batch_size = 1024
# Hive value separator
diff --git a/oozie/cassandra/bundle.xml b/oozie/cassandra/bundle.xml
index f509ade..926f6fd 100644
--- a/oozie/cassandra/bundle.xml
+++ b/oozie/cassandra/bundle.xml
@@ -75,7 +75,7 @@
</property>
<property>
<name>per_article_flat_parallel_loaders</name>
- <value>3</value>
+ <value>6</value>
</property>
<property>
<name>per_article_flat_keyspace</name>
@@ -91,7 +91,7 @@
</property>
<property>
<name>per_article_flat_hive_fields_types</name>
-
<value>text,text,text,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int</value>
+
<value>text,text,text,int_nullable,int_nullable,int_nullable,int_nullable,int_nullable,int_nullable,int_nullable,int_nullable,int_nullable,int_nullable,int_nullable,int_nullable,int_nullable,int_nullable,int_nullable,int_nullable</value>
</property>
<property>
<name>per_article_flat_cassandra_fields</name>
diff --git a/oozie/cassandra/coord_per_article_daily.properties
b/oozie/cassandra/coord_per_article_daily.properties
index ddbc978..f1e6ab6 100644
--- a/oozie/cassandra/coord_per_article_daily.properties
+++ b/oozie/cassandra/coord_per_article_daily.properties
@@ -21,7 +21,7 @@
refinery_directory = ${name_node}/wmf/refinery/current
# HDFS path to the refinery job jar that will be used by this job.
-refinery_cassandra_jar_path =
${refinery_directory}/artifacts/org/wikimedia/analytics/refinery/refinery-cassandra-0.0.30.jar
+refinery_cassandra_jar_path =
${refinery_directory}/artifacts/org/wikimedia/analytics/refinery/refinery-cassandra-0.0.36.jar
cassandra_reducer_class =
org.wikimedia.analytics.refinery.cassandra.ReducerToCassandra
cassandra_output_format_class =
org.wikimedia.analytics.refinery.cassandra.CqlOutputFormat
@@ -61,11 +61,11 @@
temporary_directory = ${name_node}/tmp
# Cassandra cluster info
-cassandra_host = aqs1001.eqiad.wmnet
+cassandra_host = aqs1004-a.eqiad.wmnet
cassandra_port = 9042
-cassandra_username = cassandra
+cassandra_username = aqsloader
cassandra_password = cassandra
-cassandra_nodes = 3
+cassandra_nodes = 6
batch_size = 1024
# Hive value separator
@@ -85,11 +85,11 @@
dataset_name = pageview_hourly
hive_script = pageview_per_article.hql
source_table = wmf.pageview_hourly
-cassandra_parallel_loaders = 3
+cassandra_parallel_loaders = 6
cassandra_keyspace =
local_group_default_T_pageviews_per_article_flat
cassandra_cql = UPDATE "${cassandra_keyspace}"."data" SET
"aa" = ?, "ab" = ?, "as" = ?, "au" = ?, "da" = ?, "db" = ?, "ds" = ?, "du" = ?,
"maa" = ?, "mab" = ?, "mas" = ?, "mau" = ?, "mwa" = ?, "mwb" = ?, "mws" = ?,
"mwu" = ?
hive_fields =
project,article,timestamp,aa,ab,as,au,da,db,ds,du,maa,mab,mas,mau,mwa,mwb,mws,mwu
-hive_fields_types =
text,text,text,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int,int
+hive_fields_types =
text,text,text,int_nullable,int_nullable,int_nullable,int_nullable,int_nullable,int_nullable,int_nullable,int_nullable,int_nullable,int_nullable,int_nullable,int_nullable,int_nullable,int_nullable,int_nullable,int_nullable
cassandra_fields =
aa,ab,as,au,da,db,ds,du,maa,mab,mas,mau,mwa,mwb,mws,mwu
cassandra_primary_keys =
_domain,project,article,granularity,timestamp,_tid
diff --git a/oozie/cassandra/coord_per_project_daily.properties
b/oozie/cassandra/coord_per_project_daily.properties
index 020024b..9340762 100644
--- a/oozie/cassandra/coord_per_project_daily.properties
+++ b/oozie/cassandra/coord_per_project_daily.properties
@@ -21,7 +21,7 @@
refinery_directory = ${name_node}/wmf/refinery/current
# HDFS path to the refinery job jar that will be used by this job.
-refinery_cassandra_jar_path =
${refinery_directory}/artifacts/org/wikimedia/analytics/refinery/refinery-cassandra-0.0.30.jar
+refinery_cassandra_jar_path =
${refinery_directory}/artifacts/org/wikimedia/analytics/refinery/refinery-cassandra-0.0.36.jar
cassandra_reducer_class =
org.wikimedia.analytics.refinery.cassandra.ReducerToCassandra
cassandra_output_format_class =
org.wikimedia.analytics.refinery.cassandra.CqlOutputFormat
@@ -61,11 +61,11 @@
temporary_directory = ${name_node}/tmp
# Cassandra cluster info
-cassandra_host = aqs1001.eqiad.wmnet
+cassandra_host = aqs1004-a.eqiad.wmnet
cassandra_port = 9042
-cassandra_username = cassandra
+cassandra_username = aqsloader
cassandra_password = cassandra
-cassandra_nodes = 3
+cassandra_nodes = 6
batch_size = 1024
# Hive value separator
diff --git a/oozie/cassandra/coord_per_project_hourly.properties
b/oozie/cassandra/coord_per_project_hourly.properties
index f47c64f..095d8ef 100644
--- a/oozie/cassandra/coord_per_project_hourly.properties
+++ b/oozie/cassandra/coord_per_project_hourly.properties
@@ -21,7 +21,7 @@
refinery_directory = ${name_node}/wmf/refinery/current
# HDFS path to the refinery job jar that will be used by this job.
-refinery_cassandra_jar_path =
${refinery_directory}/artifacts/org/wikimedia/analytics/refinery/refinery-cassandra-0.0.30.jar
+refinery_cassandra_jar_path =
${refinery_directory}/artifacts/org/wikimedia/analytics/refinery/refinery-cassandra-0.0.36.jar
cassandra_reducer_class =
org.wikimedia.analytics.refinery.cassandra.ReducerToCassandra
cassandra_output_format_class =
org.wikimedia.analytics.refinery.cassandra.CqlOutputFormat
@@ -60,11 +60,11 @@
temporary_directory = ${name_node}/tmp
# Cassandra cluster info
-cassandra_host = aqs1001.eqiad.wmnet
+cassandra_host = aqs1004-a.eqiad.wmnet
cassandra_port = 9042
-cassandra_username = cassandra
+cassandra_username = aqsloader
cassandra_password = cassandra
-cassandra_nodes = 3
+cassandra_nodes = 6
batch_size = 1024
# Hive value separator
diff --git a/oozie/cassandra/coord_per_project_monthly.properties
b/oozie/cassandra/coord_per_project_monthly.properties
index a261a2b..3529d26 100644
--- a/oozie/cassandra/coord_per_project_monthly.properties
+++ b/oozie/cassandra/coord_per_project_monthly.properties
@@ -21,7 +21,7 @@
refinery_directory = ${name_node}/wmf/refinery/current
# HDFS path to the refinery job jar that will be used by this job.
-refinery_cassandra_jar_path =
${refinery_directory}/artifacts/org/wikimedia/analytics/refinery/refinery-cassandra-0.0.30.jar
+refinery_cassandra_jar_path =
${refinery_directory}/artifacts/org/wikimedia/analytics/refinery/refinery-cassandra-0.0.36.jar
cassandra_reducer_class =
org.wikimedia.analytics.refinery.cassandra.ReducerToCassandra
cassandra_output_format_class =
org.wikimedia.analytics.refinery.cassandra.CqlOutputFormat
@@ -63,11 +63,11 @@
temporary_directory = ${name_node}/tmp
# Cassandra cluster info
-cassandra_host = aqs1001.eqiad.wmnet
+cassandra_host = aqs1004-a.eqiad.wmnet
cassandra_port = 9042
-cassandra_username = cassandra
+cassandra_username = aqsloader
cassandra_password = cassandra
-cassandra_nodes = 3
+cassandra_nodes = 6
batch_size = 1024
# Hive value separator
diff --git a/oozie/cassandra/coord_top_articles_daily.properties
b/oozie/cassandra/coord_top_articles_daily.properties
index 9882e01..ebd5dcd 100644
--- a/oozie/cassandra/coord_top_articles_daily.properties
+++ b/oozie/cassandra/coord_top_articles_daily.properties
@@ -21,7 +21,7 @@
refinery_directory = ${name_node}/wmf/refinery/current
# HDFS path to the refinery job jar that will be used by this job.
-refinery_cassandra_jar_path =
${refinery_directory}/artifacts/org/wikimedia/analytics/refinery/refinery-cassandra-0.0.30.jar
+refinery_cassandra_jar_path =
${refinery_directory}/artifacts/org/wikimedia/analytics/refinery/refinery-cassandra-0.0.36.jar
cassandra_reducer_class =
org.wikimedia.analytics.refinery.cassandra.ReducerToCassandra
cassandra_output_format_class =
org.wikimedia.analytics.refinery.cassandra.CqlOutputFormat
@@ -61,11 +61,11 @@
temporary_directory = ${name_node}/tmp
# Cassandra cluster info
-cassandra_host = aqs1001.eqiad.wmnet
+cassandra_host = aqs1004-a.eqiad.wmnet
cassandra_port = 9042
-cassandra_username = cassandra
+cassandra_username = aqsloader
cassandra_password = cassandra
-cassandra_nodes = 3
+cassandra_nodes = 6
batch_size = 1024
# Hive value separator
@@ -85,7 +85,7 @@
dataset_name = pageview_hourly
hive_script = pageview_top_articles.hql
source_table = wmf.pageview_hourly
-cassandra_parallel_loaders = 3
+cassandra_parallel_loaders = 6
cassandra_keyspace = local_group_default_T_top_pageviews
cassandra_cql = UPDATE "${cassandra_keyspace}"."data" SET
"articlesJSON" = ?
hive_fields = project,access,year,month,day,articlesJSON
diff --git a/oozie/cassandra/coord_top_articles_monthly.properties
b/oozie/cassandra/coord_top_articles_monthly.properties
index 2896d4b..e3674e0 100644
--- a/oozie/cassandra/coord_top_articles_monthly.properties
+++ b/oozie/cassandra/coord_top_articles_monthly.properties
@@ -21,7 +21,7 @@
refinery_directory = ${name_node}/wmf/refinery/current
# HDFS path to the refinery job jar that will be used by this job.
-refinery_cassandra_jar_path =
${refinery_directory}/artifacts/org/wikimedia/analytics/refinery/refinery-cassandra-0.0.30.jar
+refinery_cassandra_jar_path =
${refinery_directory}/artifacts/org/wikimedia/analytics/refinery/refinery-cassandra-0.0.36.jar
cassandra_reducer_class =
org.wikimedia.analytics.refinery.cassandra.ReducerToCassandra
cassandra_output_format_class =
org.wikimedia.analytics.refinery.cassandra.CqlOutputFormat
@@ -63,11 +63,11 @@
temporary_directory = ${name_node}/tmp
# Cassandra cluster info
-cassandra_host = aqs1001.eqiad.wmnet
+cassandra_host = aqs1004-a.eqiad.wmnet
cassandra_port = 9042
-cassandra_username = cassandra
+cassandra_username = aqsloader
cassandra_password = cassandra
-cassandra_nodes = 3
+cassandra_nodes = 6
batch_size = 1024
# Hive value separator
@@ -87,7 +87,7 @@
dataset_name = pageview_hourly
hive_script = pageview_top_articles.hql
source_table = wmf.pageview_hourly
-cassandra_parallel_loaders = 3
+cassandra_parallel_loaders = 6
cassandra_keyspace = local_group_default_T_top_pageviews
cassandra_cql = UPDATE "${cassandra_keyspace}"."data" SET
"articlesJSON" = ?
hive_fields = project,access,year,month,day,articlesJSON
diff --git a/oozie/cassandra/coord_unique_devices_daily.properties
b/oozie/cassandra/coord_unique_devices_daily.properties
index d59b80b..6be277a 100644
--- a/oozie/cassandra/coord_unique_devices_daily.properties
+++ b/oozie/cassandra/coord_unique_devices_daily.properties
@@ -21,7 +21,7 @@
refinery_directory = ${name_node}/wmf/refinery/current
# HDFS path to the refinery job jar that will be used by this job.
-refinery_cassandra_jar_path =
${refinery_directory}/artifacts/org/wikimedia/analytics/refinery/refinery-cassandra-0.0.30.jar
+refinery_cassandra_jar_path =
${refinery_directory}/artifacts/org/wikimedia/analytics/refinery/refinery-cassandra-0.0.36.jar
cassandra_reducer_class =
org.wikimedia.analytics.refinery.cassandra.ReducerToCassandra
cassandra_output_format_class =
org.wikimedia.analytics.refinery.cassandra.CqlOutputFormat
@@ -59,11 +59,11 @@
temporary_directory = ${name_node}/tmp
# Cassandra cluster info
-cassandra_host = aqs1001.eqiad.wmnet
+cassandra_host = aqs1004-a.eqiad.wmnet
cassandra_port = 9042
-cassandra_username = cassandra
+cassandra_username = aqsloader
cassandra_password = cassandra
-cassandra_nodes = 3
+cassandra_nodes = 6
batch_size = 1024
# Hive value separator
diff --git a/oozie/cassandra/coord_unique_devices_monthly.properties
b/oozie/cassandra/coord_unique_devices_monthly.properties
index 9840d19..af8b2d0 100644
--- a/oozie/cassandra/coord_unique_devices_monthly.properties
+++ b/oozie/cassandra/coord_unique_devices_monthly.properties
@@ -21,7 +21,7 @@
refinery_directory = ${name_node}/wmf/refinery/current
# HDFS path to the refinery job jar that will be used by this job.
-refinery_cassandra_jar_path =
${refinery_directory}/artifacts/org/wikimedia/analytics/refinery/refinery-cassandra-0.0.30.jar
+refinery_cassandra_jar_path =
${refinery_directory}/artifacts/org/wikimedia/analytics/refinery/refinery-cassandra-0.0.36.jar
cassandra_reducer_class =
org.wikimedia.analytics.refinery.cassandra.ReducerToCassandra
cassandra_output_format_class =
org.wikimedia.analytics.refinery.cassandra.CqlOutputFormat
@@ -61,11 +61,11 @@
temporary_directory = ${name_node}/tmp
# Cassandra cluster info
-cassandra_host = aqs1001.eqiad.wmnet
+cassandra_host = aqs1004-a.eqiad.wmnet
cassandra_port = 9042
-cassandra_username = cassandra
+cassandra_username = aqsloader
cassandra_password = cassandra
-cassandra_nodes = 3
+cassandra_nodes = 6
batch_size = 1024
# Hive value separator
--
To view, visit https://gerrit.wikimedia.org/r/315241
To unsubscribe, visit https://gerrit.wikimedia.org/r/settings
Gerrit-MessageType: newchange
Gerrit-Change-Id: Ib966c4cf3726f1892409d98dad4126967238839e
Gerrit-PatchSet: 1
Gerrit-Project: analytics/refinery
Gerrit-Branch: master
Gerrit-Owner: Joal <[email protected]>
_______________________________________________
MediaWiki-commits mailing list
[email protected]
https://lists.wikimedia.org/mailman/listinfo/mediawiki-commits
