zhengruifeng commented on PR #39151:
URL: https://github.com/apache/spark/pull/39151#issuecomment-1360941255
quick tests:
1, cache (after make some private ones public)
```
scala> val df = spark.range(0, 100)
df: org.apache.spark.sql.Dataset[Long] = [id: bigint]
scala> df.cache()
res0: df.type = [id: bigint]
scala> val plan = df.logicalPlan
plan: org.apache.spark.sql.catalyst.plans.logical.LogicalPlan =
Range (0, 100, step=1, splits=Some(10))
scala> import org.apache.spark.sql.{Column, Dataset, SparkSession}
import org.apache.spark.sql.{Column, Dataset, SparkSession}
scala> val df2 = Dataset.ofRows(spark, plan)
df2: org.apache.spark.sql.DataFrame = [id: bigint]
scala> df.storageLevel
res1: org.apache.spark.storage.StorageLevel = StorageLevel(disk, memory,
deserialized, 1 replicas)
scala> df2.storageLevel
res2: org.apache.spark.storage.StorageLevel = StorageLevel(disk, memory,
deserialized, 1 replicas)
scala> df == df2
res3: Boolean = false
scala> df2.explain
== Physical Plan ==
InMemoryTableScan [id#0L]
+- InMemoryRelation [id#0L], StorageLevel(disk, memory, deserialized, 1
replicas)
+- *(1) Range (0, 100, step=1, splits=10)
scala> df.explain
== Physical Plan ==
InMemoryTableScan [id#0L]
+- InMemoryRelation [id#0L], StorageLevel(disk, memory, deserialized, 1
replicas)
+- *(1) Range (0, 100, step=1, splits=10)
```
2, relation
```
scala> val df = spark.range(0, 100)
df: org.apache.spark.sql.Dataset[Long] = [id: bigint]
scala> df.write.parquet("/tmp/1.pq")
scala> val df2 = spark.read.parquet("/tmp/1.pq")
df2: org.apache.spark.sql.DataFrame = [id: bigint]
scala> df2.queryExecution.logical.deterministic
res2: Boolean = true
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
