This is an automated email from the ASF dual-hosted git repository. bertty pushed a commit to branch assembly in repository https://gitbox.apache.org/repos/asf/incubator-wayang.git
commit b889d40e6681deeeb310796416b0b1e535871ddd Author: Bertty Contreras-Rojas <[email protected]> AuthorDate: Fri Jul 8 19:19:48 2022 +0200 [WAYANG-assembly] mirror issues Signed-off-by: bertty <[email protected]> --- README.md | 2 +- bin/wayang-submit | 22 +++++++++++++++++++--- wayang-assembly/README.md | 1 + 3 files changed, 21 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 314d39ec..b8b11b36 100644 --- a/README.md +++ b/README.md @@ -31,7 +31,7 @@ You can download wayang from [here](https://github.com/apache/incubator-wayang/r ```shell tar -xvf wayang-0.6.1-snapshot.tar.gz -cd wayang-0.6.1-snapshot +cd wayang-0.6.1-SNAPSHOT ``` In linux diff --git a/bin/wayang-submit b/bin/wayang-submit index 9cb4d48b..aa85c86b 100755 --- a/bin/wayang-submit +++ b/bin/wayang-submit @@ -71,22 +71,37 @@ fi # Find Spark jars. if [ -d "${SPARK_HOME}" ]; then SPARK_JARS_DIR="${SPARK_HOME}/jars" + ls ${SPARK_JARS_DIR}/ fi # Find Hadoop jars. if [ -d "${HADOOP_HOME}" ]; then HADOOP_JARS_DIR="${HADOOP_HOME}/share/hadoop/common/*:${HADOOP_HOME}/share/hadoop/common/lib/*" + ls ${HADOOP_HOME}/share/hadoop/common/ + ls ${HADOOP_HOME}/share/hadoop/common/lib/ +fi + +if [ "$(ls ${SPARK_JARS_DIR} | grep ^hadoop | wc -l)" == "0" ]; then + + HADOOP_JARS_DIR="${HADOOP_JARS_DIR}:${HADOOP_HOME}/share/hadoop/mapreduce/$(ls ${HADOOP_HOME}/share/hadoop/mapreduce | grep ^hadoop-mapreduce-client-common | grep -v tests | head -n 1)" + HADOOP_JARS_DIR="${HADOOP_JARS_DIR}:${HADOOP_HOME}/share/hadoop/mapreduce/$(ls ${HADOOP_HOME}/share/hadoop/mapreduce | grep ^hadoop-mapreduce-client-core | grep -v tests | head -n 1)" + HADOOP_JARS_DIR="${HADOOP_JARS_DIR}:${HADOOP_HOME}/share/hadoop/mapreduce/$(ls ${HADOOP_HOME}/share/hadoop/mapreduce | grep ^hadoop-mapreduce-client-jobclient | grep -v tests | head -n 1)" + HADOOP_JARS_DIR="${HADOOP_JARS_DIR}:${HADOOP_HOME}/share/hadoop/hdfs/$(ls ${HADOOP_HOME}/share/hadoop/hdfs | grep ^hadoop-hdfs-client | grep -v tests | head -n 1)" + HADOOP_JARS_DIR="${HADOOP_JARS_DIR}:${HADOOP_HOME}/share/hadoop/hdfs/lib/$(ls ${HADOOP_HOME}/share/hadoop/hdfs/lib | grep ^hadoop-annotations | grep -v tests | head -n 1)" + HADOOP_JARS_DIR="${HADOOP_JARS_DIR}:${HADOOP_HOME}/share/hadoop/hdfs/lib/$(ls ${HADOOP_HOME}/share/hadoop/hdfs/lib | grep ^hadoop-auth | grep -v tests | head -n 1)" + fi WAYANG_CODE="${WAYANG_HOME}/jars" +ls ${WAYANG_CODE} WAYANG_LIBS="${WAYANG_HOME}/libs" +ls ${WAYANG_LIBS} WAYANG_CONF="${WAYANG_HOME}/conf" - +ls ${WAYANG_CONF} # Bootstrap the classpath. WAYANG_CLASSPATH="${WAYANG_CONF}/*:${WAYANG_CODE}/*:${WAYANG_LIBS}/*" - -WAYANG_CLASSPATH="${WAYANG_CLASSPATH}:${SPARK_JARS_DIR}/*:${HADOOP_JARS_DIR}" +WAYANG_CLASSPATH="${SPARK_JARS_DIR}/*:${WAYANG_CLASSPATH}:${HADOOP_JARS_DIR}" FLAGS="" if [ "${FLAG_LOG}" = "true" ]; then @@ -101,6 +116,7 @@ if [ -n "${OTHER_FLAGS}" ]; then FLAGS="${FLAGS} ${OTHER_FLAGS}" fi +echo "lala" echo "$RUNNER $FLAGS -cp "${WAYANG_CLASSPATH}" $CLASS ${@:2}" eval "$RUNNER $FLAGS -cp "${WAYANG_CLASSPATH}" $CLASS ${@:2}" diff --git a/wayang-assembly/README.md b/wayang-assembly/README.md index 4f78e1db..c29cc46b 100644 --- a/wayang-assembly/README.md +++ b/wayang-assembly/README.md @@ -19,5 +19,6 @@ except for the jars in the list To execute the Wayang Assembly you need to execute the following command in the project root ```shell +./mvnw clean install -DskipTests ./mvnw clean package -pl :wayang-assembly -Pdistribution ``` \ No newline at end of file
