This is an automated email from the ASF dual-hosted git repository. jiayu pushed a commit to branch SEDONA-651 in repository https://gitbox.apache.org/repos/asf/sedona.git
commit 9abe9ed43f7a5671b292785047af7a32a07a6004 Author: Jia Yu <[email protected]> AuthorDate: Thu Sep 5 00:35:10 2024 -0700 Add --- docs/api/sql/Parameter.md | 10 ++++++ .../org/apache/sedona/core/utils/SedonaConf.java | 41 +++++++++++++++------- 2 files changed, 38 insertions(+), 13 deletions(-) diff --git a/docs/api/sql/Parameter.md b/docs/api/sql/Parameter.md index d242aec19..f80058ac6 100644 --- a/docs/api/sql/Parameter.md +++ b/docs/api/sql/Parameter.md @@ -25,6 +25,16 @@ println(sedonaConf) sparkSession.conf.set("sedona.global.index","false") ``` +In addition, you can also add `spark` prefix to the parameter name, for example: + +```scala +sparkSession.conf.set("spark.sedona.global.index","false") +``` + +However, any parameter set through `spark` prefix will be honored by Spark, which means you can set these parameters before hand via `spark-defaults.conf` or Spark on Kubernetes configuration. + +If you set the same parameter through both `sedona` and `spark.sedona` prefixes, the parameter set through `sedona` prefix will override the parameter set through `spark.sedona` prefix. + ## Explanation * sedona.global.index diff --git a/spark/common/src/main/java/org/apache/sedona/core/utils/SedonaConf.java b/spark/common/src/main/java/org/apache/sedona/core/utils/SedonaConf.java index ee228ed9c..442a73ce8 100644 --- a/spark/common/src/main/java/org/apache/sedona/core/utils/SedonaConf.java +++ b/spark/common/src/main/java/org/apache/sedona/core/utils/SedonaConf.java @@ -64,38 +64,53 @@ public class SedonaConf implements Serializable { } public SedonaConf(RuntimeConfig runtimeConfig) { - this.useIndex = Boolean.parseBoolean(runtimeConfig.get("sedona.global.index", "true")); - this.indexType = IndexType.getIndexType(runtimeConfig.get("sedona.global.indextype", "rtree")); + this.useIndex = Boolean.parseBoolean(getConfigValue(runtimeConfig, "global.index", "true")); + this.indexType = + IndexType.getIndexType(getConfigValue(runtimeConfig, "global.indextype", "rtree")); this.joinApproximateTotalCount = - Long.parseLong(runtimeConfig.get("sedona.join.approxcount", "-1")); - String[] boundaryString = runtimeConfig.get("sedona.join.boundary", "0,0,0,0").split(","); + Long.parseLong(getConfigValue(runtimeConfig, "join.approxcount", "-1")); + String[] boundaryString = getConfigValue(runtimeConfig, "join.boundary", "0,0,0,0").split(","); this.datasetBoundary = new Envelope( Double.parseDouble(boundaryString[0]), Double.parseDouble(boundaryString[1]), Double.parseDouble(boundaryString[2]), Double.parseDouble(boundaryString[3])); - this.joinGridType = GridType.getGridType(runtimeConfig.get("sedona.join.gridtype", "kdbtree")); + this.joinGridType = + GridType.getGridType(getConfigValue(runtimeConfig, "join.gridtype", "kdbtree")); this.joinBuildSide = - JoinBuildSide.getBuildSide(runtimeConfig.get("sedona.join.indexbuildside", "left")); + JoinBuildSide.getBuildSide(getConfigValue(runtimeConfig, "join.indexbuildside", "left")); this.joinSparitionDominantSide = JoinSparitionDominantSide.getJoinSparitionDominantSide( - runtimeConfig.get("sedona.join.spatitionside", "left")); + getConfigValue(runtimeConfig, "join.spatitionside", "left")); this.fallbackPartitionNum = - Integer.parseInt(runtimeConfig.get("sedona.join.numpartition", "-1")); + Integer.parseInt(getConfigValue(runtimeConfig, "join.numpartition", "-1")); this.autoBroadcastJoinThreshold = bytesFromString( - runtimeConfig.get( - "sedona.join.autoBroadcastJoinThreshold", + getConfigValue( + runtimeConfig, + "join.autoBroadcastJoinThreshold", runtimeConfig.get("spark.sql.autoBroadcastJoinThreshold"))); this.spatialJoinOptimizationMode = SpatialJoinOptimizationMode.getSpatialJoinOptimizationMode( - runtimeConfig.get("sedona.join.optimizationmode", "nonequi")); + getConfigValue(runtimeConfig, "join.optimizationmode", "nonequi")); // Parameters for knn joins this.includeTieBreakersInKNNJoins = - Boolean.parseBoolean( - runtimeConfig.get("spark.sedona.join.knn.includeTieBreakers", "false")); + Boolean.parseBoolean(getConfigValue(runtimeConfig, "join.knn.includeTieBreakers", "false")); + } + + // Helper method to prioritize `sedona.*` over `spark.sedona.*` + private String getConfigValue( + RuntimeConfig runtimeConfig, String keySuffix, String defaultValue) { + String sedonaKey = "sedona." + keySuffix; + String sparkSedonaKey = "spark.sedona." + keySuffix; + + if (runtimeConfig.contains(sedonaKey)) { + return runtimeConfig.get(sedonaKey, defaultValue); + } else { + return runtimeConfig.get(sparkSedonaKey, defaultValue); + } } public boolean getUseIndex() {
