This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new b7eb65625dd [SPARK-45413][CORE] Add warning for prepare drop LevelDB
support
b7eb65625dd is described below
commit b7eb65625ddfa0ed168db1c77d7013b6e79815e4
Author: Jia Fan <[email protected]>
AuthorDate: Sun Oct 8 09:33:25 2023 -0700
[SPARK-45413][CORE] Add warning for prepare drop LevelDB support
### What changes were proposed in this pull request?
This PR add warning deprecated message for use LEVELDB in
`spark.history.store.hybridStore.diskBackend` or
`spark.shuffle.service.db.backend`.
### Why are the changes needed?
Add warning for LEVELDB users in Spark, so we can remove it safety in the
future.
### Does this PR introduce _any_ user-facing change?
NO
### How was this patch tested?
exist test.
### Was this patch authored or co-authored using generative AI tooling?
No
Closes #43217 from Hisoka-X/SPARK-45413-remove-leveldb-for-hybridStore.
Authored-by: Jia Fan <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../src/main/java/org/apache/spark/network/util/DBProvider.java | 8 +++++++-
.../src/main/scala/org/apache/spark/internal/config/History.scala | 2 +-
.../src/main/scala/org/apache/spark/internal/config/package.scala | 2 +-
core/src/main/scala/org/apache/spark/status/KVUtils.scala | 4 +++-
docs/configuration.md | 2 +-
docs/monitoring.md | 2 +-
docs/running-on-yarn.md | 4 ++--
docs/spark-standalone.md | 4 ++--
8 files changed, 18 insertions(+), 10 deletions(-)
diff --git
a/common/network-common/src/main/java/org/apache/spark/network/util/DBProvider.java
b/common/network-common/src/main/java/org/apache/spark/network/util/DBProvider.java
index 2a4afa73622..6d77608f7ac 100644
---
a/common/network-common/src/main/java/org/apache/spark/network/util/DBProvider.java
+++
b/common/network-common/src/main/java/org/apache/spark/network/util/DBProvider.java
@@ -21,6 +21,8 @@ import java.io.IOException;
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.annotations.VisibleForTesting;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.spark.network.shuffledb.DB;
import org.apache.spark.network.shuffledb.DBBackend;
@@ -29,6 +31,7 @@ import org.apache.spark.network.shuffledb.RocksDB;
import org.apache.spark.network.shuffledb.StoreVersion;
public class DBProvider {
+ private static final Logger logger =
LoggerFactory.getLogger(DBProvider.class);
public static DB initDB(
DBBackend dbBackend,
File dbFile,
@@ -38,6 +41,7 @@ public class DBProvider {
switch (dbBackend) {
case LEVELDB:
org.iq80.leveldb.DB levelDB = LevelDBProvider.initLevelDB(dbFile,
version, mapper);
+ logger.warn("The LEVELDB is deprecated. Please use ROCKSDB
instead.");
return levelDB != null ? new LevelDB(levelDB) : null;
case ROCKSDB:
org.rocksdb.RocksDB rocksDB = RocksDBProvider.initRockDB(dbFile,
version, mapper);
@@ -53,7 +57,9 @@ public class DBProvider {
public static DB initDB(DBBackend dbBackend, File file) throws IOException
{
if (file != null) {
switch (dbBackend) {
- case LEVELDB: return new LevelDB(LevelDBProvider.initLevelDB(file));
+ case LEVELDB:
+ logger.warn("The LEVELDB is deprecated. Please use ROCKSDB
instead.");
+ return new LevelDB(LevelDBProvider.initLevelDB(file));
case ROCKSDB: return new RocksDB(RocksDBProvider.initRocksDB(file));
default:
throw new IllegalArgumentException("Unsupported DBBackend: " +
dbBackend);
diff --git a/core/src/main/scala/org/apache/spark/internal/config/History.scala
b/core/src/main/scala/org/apache/spark/internal/config/History.scala
index 7101318eb56..2306856f933 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/History.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/History.scala
@@ -243,7 +243,7 @@ private[spark] object History {
}
val HYBRID_STORE_DISK_BACKEND =
ConfigBuilder("spark.history.store.hybridStore.diskBackend")
- .doc("Specifies a disk-based store used in hybrid store; LEVELDB or
ROCKSDB.")
+ .doc("Specifies a disk-based store used in hybrid store; ROCKSDB or
LEVELDB (deprecated).")
.version("3.3.0")
.stringConf
.transform(_.toUpperCase(Locale.ROOT))
diff --git a/core/src/main/scala/org/apache/spark/internal/config/package.scala
b/core/src/main/scala/org/apache/spark/internal/config/package.scala
index e4e461dd340..4f2c33d2a19 100644
--- a/core/src/main/scala/org/apache/spark/internal/config/package.scala
+++ b/core/src/main/scala/org/apache/spark/internal/config/package.scala
@@ -725,7 +725,7 @@ package object config {
private[spark] val SHUFFLE_SERVICE_DB_BACKEND =
ConfigBuilder(Constants.SHUFFLE_SERVICE_DB_BACKEND)
.doc("Specifies a disk-based store used in shuffle service local db. " +
- "LEVELDB or ROCKSDB.")
+ "ROCKSDB or LEVELDB (deprecated).")
.version("3.4.0")
.stringConf
.transform(_.toUpperCase(Locale.ROOT))
diff --git a/core/src/main/scala/org/apache/spark/status/KVUtils.scala
b/core/src/main/scala/org/apache/spark/status/KVUtils.scala
index 14aa7db6e06..821686803f4 100644
--- a/core/src/main/scala/org/apache/spark/status/KVUtils.scala
+++ b/core/src/main/scala/org/apache/spark/status/KVUtils.scala
@@ -95,7 +95,9 @@ private[spark] object KVUtils extends Logging {
val kvSerializer = serializer(conf, live)
val db = backend(conf, live) match {
- case LEVELDB => new LevelDB(path, kvSerializer)
+ case LEVELDB =>
+ logWarning("The LEVELDB is deprecated. Please use ROCKSDB instead.")
+ new LevelDB(path, kvSerializer)
case ROCKSDB => new RocksDB(path, kvSerializer)
}
val dbMeta = db.getMetadata(classTag[M].runtimeClass)
diff --git a/docs/configuration.md b/docs/configuration.md
index 90a3fb32933..b95a063dbf8 100644
--- a/docs/configuration.md
+++ b/docs/configuration.md
@@ -1282,7 +1282,7 @@ Apart from these, the following properties are also
available, and may be useful
<td><code>spark.shuffle.service.db.backend</code></td>
<td>ROCKSDB</td>
<td>
- Specifies a disk-based store used in shuffle service local db. Setting as
LEVELDB or ROCKSDB.
+ Specifies a disk-based store used in shuffle service local db. Setting as
ROCKSDB or LEVELDB (deprecated).
</td>
<td>3.4.0</td>
</tr>
diff --git a/docs/monitoring.md b/docs/monitoring.md
index a3f372bf56c..7336be9bb67 100644
--- a/docs/monitoring.md
+++ b/docs/monitoring.md
@@ -416,7 +416,7 @@ Security options for the Spark History Server are covered
more detail in the
<td>spark.history.store.hybridStore.diskBackend</td>
<td>ROCKSDB</td>
<td>
- Specifies a disk-based store used in hybrid store; LEVELDB or ROCKSDB.
+ Specifies a disk-based store used in hybrid store; ROCKSDB or LEVELDB
(deprecated).
</td>
<td>3.3.0</td>
</tr>
diff --git a/docs/running-on-yarn.md b/docs/running-on-yarn.md
index ef919bb7928..11ed7e9e877 100644
--- a/docs/running-on-yarn.md
+++ b/docs/running-on-yarn.md
@@ -917,8 +917,8 @@ The following extra configuration options are available
when the shuffle service
<td>ROCKSDB</td>
<td>
When work-preserving restart is enabled in YARN, this is used to specify
the disk-base store used
- in shuffle service state store, supports `LEVELDB` and `ROCKSDB` with
`ROCKSDB` as default value.
- The original data store in `LevelDB/RocksDB` will not be automatically
converted to another kind
+ in shuffle service state store, supports `ROCKSDB` and `LEVELDB`
(deprecated) with `ROCKSDB` as default value.
+ The original data store in `RocksDB/LevelDB` will not be automatically
converted to another kind
of storage now. The original data store will be retained and the new type
data store will be
created when switching storage types.
</td>
diff --git a/docs/spark-standalone.md b/docs/spark-standalone.md
index a91090d65cc..93f0818b6ce 100644
--- a/docs/spark-standalone.md
+++ b/docs/spark-standalone.md
@@ -335,8 +335,8 @@ SPARK_WORKER_OPTS supports the following system properties:
<td>ROCKSDB</td>
<td>
When <code>spark.shuffle.service.db.enabled</code> is true, user can use
this to specify the kind of disk-based
- store used in shuffle service state store. This supports `LEVELDB` and
`ROCKSDB` now and `ROCKSDB` as default value.
- The original data store in `LevelDB/RocksDB` will not be automatically
convert to another kind of storage now.
+ store used in shuffle service state store. This supports `ROCKSDB` and
`LEVELDB` (deprecated) now and `ROCKSDB` as default value.
+ The original data store in `RocksDB/LevelDB` will not be automatically
convert to another kind of storage now.
</td>
<td>3.4.0</td>
</tr>
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
