liujiawinds commented on code in PR #19844:
URL: https://github.com/apache/flink/pull/19844#discussion_r927578181
##########
flink-formats/flink-orc/src/test/java/org/apache/flink/orc/writer/OrcBulkWriterTest.java:
##########
@@ -75,8 +78,105 @@ public void testOrcBulkWriter() throws Exception {
testHarness.snapshot(1, ++time);
testHarness.notifyOfCompletedCheckpoint(1);
+ }
+ }
- OrcBulkWriterTestUtil.validate(outDir, input);
+ @Test
+ public void testOrcBulkWriter() throws Exception {
+ final File outDir = TEMPORARY_FOLDER.newFolder();
+ final Properties writerProps = new Properties();
+ writerProps.setProperty("orc.compress", CompressionKind.LZ4.name());
+
+ OrcBulkWriterFactory<Record> writer =
+ new OrcBulkWriterFactory<>(
+ new RecordVectorizer(schema), writerProps, new
Configuration());
+
+ writeRecordsIntoOrcFile(outDir, writer);
+
+ // validate records and compression kind
+ OrcBulkWriterTestUtil.validate(outDir, input, CompressionKind.LZ4);
+ }
+
+ @Test
+ public void testZstdCompression() throws Exception {
+ final File outDir = TEMPORARY_FOLDER.newFolder();
+ final Properties writerProps = new Properties();
+ writerProps.setProperty("orc.compress", CompressionKind.ZSTD.name());
+
+ OrcBulkWriterFactory<Record> writer =
+ new OrcBulkWriterFactory<>(
+ new RecordVectorizer(schema), writerProps, new
Configuration());
+
+ writeRecordsIntoOrcFile(outDir, writer);
+
+ // validate records and compression kind
+ OrcBulkWriterTestUtil.validate(outDir, input, CompressionKind.ZSTD);
+ }
+
+ @Test
+ public void testOrcColumnEncryption() throws Exception {
+ final File outDir = TEMPORARY_FOLDER.newFolder();
+
+ // Simple configuration for column encryption.
+ Configuration conf = new Configuration();
+ conf.set("orc.encrypt", "pii:_col0");
+ conf.set("orc.mask", "sha256:_col0");
+ OrcFile.WriterOptions writerOptions =
+ OrcFile.writerOptions(conf)
+ .encrypt(OrcConf.ENCRYPTION.getString(conf))
+ .masks(OrcConf.DATA_MASK.getString(conf))
+ .setKeyProvider(new DummyInMemoryKeystore());
+
+ OrcBulkWriterFactory<Record> writer =
+ new OrcBulkWriterFactory<>(new RecordVectorizer(schema),
writerOptions);
+
+ writeRecordsIntoOrcFile(outDir, writer);
+
+ // Validate bucket count and file count.
+ final File[] buckets = outDir.listFiles();
+ assertThat(buckets).isNotNull();
+ assertThat(buckets.length).isEqualTo(1);
+ final File[] partFiles = buckets[0].listFiles();
+ assertThat(partFiles).isNotNull();
+
+ // Validate data in orc and file schema attribute value.
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscr...@flink.apache.org
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org
