[systemds] branch master updated: [SYSTEMDS-2896] Spark Log4j propagation

[baunsgaard]

This is an automated email from the ASF dual-hosted git repository.

baunsgaard pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/systemds.git


The following commit(s) were added to refs/heads/master by this push:
     new d575d31  [SYSTEMDS-2896] Spark Log4j propagation
d575d31 is described below

commit d575d3190e65cff72e50b5e42311552620c78278
Author: baunsgaard <baunsga...@tugraz.at>
AuthorDate: Thu Mar 11 12:41:27 2021 +0100

    [SYSTEMDS-2896] Spark Log4j propagation
    
    This commit modify out bin/systemds to properly propagate our
    log4j settings to workers and controller in spark to allow control over
    the log output from spark execution in systemds.
---
 bin/systemds                                       | 8 +++++++-
 src/main/java/org/apache/sysds/conf/DMLConfig.java | 2 +-
 2 files changed, 8 insertions(+), 2 deletions(-)

diff --git a/bin/systemds b/bin/systemds
index 3b1dc37..2eeaa3d 100755
--- a/bin/systemds
+++ b/bin/systemds
@@ -246,6 +246,7 @@ if [ -z "$LOG4JPROP" ] ; then
   if [ -z "${LOG4JPROP}" ]; then
     LOG4JPROP=""
   else
+    SPARK_LOG4J_PATH="$LOG4JPROP"
     LOG4JPROP="-Dlog4j.configuration=file:$LOG4JPROP"
   fi
 else
@@ -254,8 +255,10 @@ else
   if [ -z "${LOG4JPROP2}" ]; then
     LOG4JPROP=""
   else
-      LOG4JPROP="-Dlog4j.configuration=file:$LOG4JPROP2"
+    SPARK_LOG4J_PATH="$LOG4JPROP2"
+    LOG4JPROP="-Dlog4j.configuration=file:$LOG4JPROP2"
   fi
+
 fi
 
 if [[ "$*" == *-config* ]]; then
@@ -366,6 +369,9 @@ else
   export SPARK_MAJOR_VERSION=2
   CMD=" \
   spark-submit $SYSTEMDS_DISTRIBUTED_OPTS \
+  --conf 
spark.driver.extraJavaOptions=-Dlog4j.configuration=file:$SPARK_LOG4J_PATH \
+  --conf 
spark.executor.extraJavaOptions=-Dlog4j.configuration=file:$SPARK_LOG4J_PATH \
+  --files $SPARK_LOG4J_PATH \
   $SYSTEMDS_JAR_FILE \
   -f $SCRIPT_FILE \
   $*"
diff --git a/src/main/java/org/apache/sysds/conf/DMLConfig.java 
b/src/main/java/org/apache/sysds/conf/DMLConfig.java
index 280bad7..5f90fb4 100644
--- a/src/main/java/org/apache/sysds/conf/DMLConfig.java
+++ b/src/main/java/org/apache/sysds/conf/DMLConfig.java
@@ -126,7 +126,7 @@ public class DMLConfig
                _defaultVals.put(CP_PARALLEL_IO,         "true" );
                _defaultVals.put(COMPRESSED_LINALG,      
Compression.CompressConfig.FALSE.name() );
                _defaultVals.put(COMPRESSED_LOSSY,       "false" );
-               _defaultVals.put(COMPRESSED_VALID_COMPRESSIONS, "SDC,DDC,RLE");
+               _defaultVals.put(COMPRESSED_VALID_COMPRESSIONS, "SDC,DDC");
                _defaultVals.put(COMPRESSED_OVERLAPPING, "true" );
                _defaultVals.put(COMPRESSED_SAMPLING_RATIO, "0.01");
                _defaultVals.put(COMPRESSED_COCODE,      "COST");