This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/master by this push:
new e1c630a98c4 [SPARK-42114][SQL][TESTS] Add uniform parquet encryption
test case
e1c630a98c4 is described below
commit e1c630a98c45ae07c43c8cf95979532b51bf59ec
Author: Gidon Gershinsky <[email protected]>
AuthorDate: Fri Jan 20 01:53:41 2023 -0800
[SPARK-42114][SQL][TESTS] Add uniform parquet encryption test case
### What changes were proposed in this pull request?
Unitest of the new parquet encryption mode, available since parquet 1.12.3.
This mode is triggered by setting a hadoop parameter (/df write option)
`parquet.encryption.uniform.key` : master key ID for uniform encryption of all
columns and footer.
### Why are the changes needed?
To verify end to end Spark SQL encryption and decryption of dataframes in
uniform mode.
### Does this PR introduce _any_ user-facing change?
No user-facing changes.
### How was this patch tested?
This patch is the unitest
Closes #39665 from ggershinsky/uniform-unitest1.
Authored-by: Gidon Gershinsky <[email protected]>
Signed-off-by: Dongjoon Hyun <[email protected]>
---
.../spark/sql/hive/ParquetEncryptionSuite.scala | 26 ++++++++++++++++++++++
1 file changed, 26 insertions(+)
diff --git
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/ParquetEncryptionSuite.scala
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/ParquetEncryptionSuite.scala
index 24107f0c626..549431ef4f4 100644
---
a/sql/hive/src/test/scala/org/apache/spark/sql/hive/ParquetEncryptionSuite.scala
+++
b/sql/hive/src/test/scala/org/apache/spark/sql/hive/ParquetEncryptionSuite.scala
@@ -91,6 +91,32 @@ class ParquetEncryptionSuite extends QueryTest with
TestHiveSingleton {
}
}
+ test("SPARK-42114: Test of uniform parquet encryption") {
+ withTempDir { dir =>
+ withSQLConf(
+ "parquet.crypto.factory.class" ->
+ "org.apache.parquet.crypto.keytools.PropertiesDrivenCryptoFactory",
+ "parquet.encryption.kms.client.class" ->
+ "org.apache.parquet.crypto.keytools.mocks.InMemoryKMS",
+ "parquet.encryption.key.list" ->
+ s"key1: ${key1}") {
+
+ val inputDF = Seq((1, 22, 333)).toDF("a", "b", "c")
+ val parquetDir = new File(dir, "parquet").getCanonicalPath
+ inputDF.write
+ .option("parquet.encryption.uniform.key", "key1")
+ .parquet(parquetDir)
+
+ verifyParquetEncrypted(parquetDir)
+
+ val parquetDF = spark.read.parquet(parquetDir)
+ assert(parquetDF.inputFiles.nonEmpty)
+ val readDataset = parquetDF.select("a", "b", "c")
+ checkAnswer(readDataset, inputDF)
+ }
+ }
+ }
+
/**
* Verify that the directory contains an encrypted parquet in
* encrypted footer mode by means of checking for all the parquet part files
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
