This is an automated email from the ASF dual-hosted git repository.
arvindsh pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/fluo-muchos.git
The following commit(s) were added to refs/heads/master by this push:
new 282a7cf Fix issues with setting up Spark (#322)
282a7cf is described below
commit 282a7cf92c72415877e2956fd78803a74bc57394
Author: Arvind Shyamsundar <[email protected]>
AuthorDate: Tue Feb 11 14:16:40 2020 -0800
Fix issues with setting up Spark (#322)
The Ansible task to copy the Spark YARN shuffle JAR is now sequenced
to run as part of the Spark role. In addition, Spark 2.4.5 is the
current latest production release so defaulting to that.
---
ansible/roles/hadoop/tasks/main.yml | 3 ---
ansible/roles/spark/tasks/main.yml | 3 +++
conf/checksums | 1 +
conf/muchos.props.example | 2 +-
4 files changed, 5 insertions(+), 4 deletions(-)
diff --git a/ansible/roles/hadoop/tasks/main.yml
b/ansible/roles/hadoop/tasks/main.yml
index 45d72ec..2126c8c 100644
--- a/ansible/roles/hadoop/tasks/main.yml
+++ b/ansible/roles/hadoop/tasks/main.yml
@@ -41,9 +41,6 @@
synchronize: src={{ user_home }}/mvn_dep/ dest={{ hadoop_home
}}/share/hadoop/common/lib/
when: hadoop_major_version == '3' and java_product_version == 11
-- name: "copy spark yarn shuffle jar to hadoop lib"
- command: cp {{ spark_home }}/yarn/spark-{{ spark_version }}-yarn-shuffle.jar
{{ hadoop_home }}/share/hadoop/yarn/lib/ creates={{ hadoop_home
}}/share/hadoop/yarn/lib/spark-{{ spark_version }}-yarn-shuffle.jar
- when: "'spark' in groups"
- name: "setup hadoop short circuit socket dir"
file: path=/var/lib/hadoop-hdfs state=directory owner={{ cluster_user }}
group={{ cluster_group }} mode=0755
become: yes
diff --git a/ansible/roles/spark/tasks/main.yml
b/ansible/roles/spark/tasks/main.yml
index 4310754..9ccfe81 100644
--- a/ansible/roles/spark/tasks/main.yml
+++ b/ansible/roles/spark/tasks/main.yml
@@ -22,3 +22,6 @@
with_items:
- spark-defaults.conf
- spark-env.sh
+- name: "copy spark yarn shuffle jar to hadoop lib"
+ command: cp {{ spark_home }}/yarn/spark-{{ spark_version }}-yarn-shuffle.jar
{{ hadoop_home }}/share/hadoop/yarn/lib/ creates={{ hadoop_home
}}/share/hadoop/yarn/lib/spark-{{ spark_version }}-yarn-shuffle.jar
+ when: "'spark' in groups"
diff --git a/conf/checksums b/conf/checksums
index 3f4c3d4..68c6eb4 100644
--- a/conf/checksums
+++ b/conf/checksums
@@ -22,6 +22,7 @@
hadoop:2.8.3:e8bf9a53337b1dca3b152b0a5b5e277dc734e76520543e525c301a050bb27eae
hadoop:2.7.6:f2327ea93f4bc5a5d7150dee8e0ede196d3a77ff8526a7dd05a48a09aae25669
hadoop:2.7.5:0bfc4d9b04be919be2fdf36f67fa3b4526cdbd406c512a7a1f5f1b715661f831
hadoop:2.6.5:001ad18d4b6d0fe542b15ddadba2d092bc97df1c4d2d797381c8d12887691898
+spark:2.4.5:17A8B3BD0151E506E2B3C09FFE6F1010FFE1C6B7B906F3A19D357B37098EBD0C3AD91E6501545AE87834B81D2EF07556E5FECCFB7275D1C74533777BC7A5DF3B
spark:2.4.4:A3E4FDEAD2409C8D1E0F213145ACC3B43788285715EB100DD29B171E6B24012673DDD94BDC612D946764B71CBC8F78BCD93889A4BC60B3136A33CD7E6C4DBC26
spark:2.4.3:4db62e110c8080f28ac8f1d701b7477cede23e60b51231ab63687b2cc6150faa
spark:2.3.4:0241cbe73d3ad4fec1c655d839facc16e647ff9a3825f51c78d9fb270a753da4
diff --git a/conf/muchos.props.example b/conf/muchos.props.example
index 63898be..84e42e0 100644
--- a/conf/muchos.props.example
+++ b/conf/muchos.props.example
@@ -38,7 +38,7 @@ accumulo_password = secret
# Software versions (make sure you have a corresponding entry for the checksum
in conf/checksums)
hadoop_version = 3.2.1
zookeeper_version = 3.5.6
-spark_version = 2.3.2
+spark_version = 2.4.5
fluo_version = 1.2.0
fluo_yarn_version = 1.0.0
accumulo_version = 2.0.0
