alamb commented on code in PR #7244:
URL: https://github.com/apache/arrow-datafusion/pull/7244#discussion_r1292765245
##########
datafusion/core/src/datasource/file_format/parquet.rs:
##########
@@ -543,6 +575,318 @@ async fn fetch_statistics(
Ok(statistics)
}
+/// Implements [`DataSink`] for writing to a parquet file.
+struct ParquetSink {
+ /// Config options for writing data
+ config: FileSinkConfig,
+}
+
+impl Debug for ParquetSink {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ f.debug_struct("ParquetSink").finish()
+ }
+}
+
+impl DisplayAs for ParquetSink {
+ fn fmt_as(&self, t: DisplayFormatType, f: &mut fmt::Formatter<'_>) ->
fmt::Result {
+ match t {
+ DisplayFormatType::Default | DisplayFormatType::Verbose => {
+ write!(
+ f,
+ "ParquetSink(writer_mode={:?}, file_groups=",
+ self.config.writer_mode
+ )?;
+ FileGroupDisplay(&self.config.file_groups).fmt_as(t, f)?;
+ write!(f, ")")
+ }
+ }
+ }
+}
+
+/// Parses datafusion.execution.parquet.encoding String to a
parquet::basic::Encoding
+fn parse_encoding_string(str_setting: &str) ->
Result<parquet::basic::Encoding> {
+ let str_setting_lower: &str = &str_setting.to_lowercase();
+ match str_setting_lower {
+ "plain" => Ok(parquet::basic::Encoding::PLAIN),
Review Comment:
Filed https://github.com/apache/arrow-rs/issues/4693 and
https://github.com/apache/arrow-datafusion/pull/7275 to leave a reference here
##########
datafusion/core/src/datasource/file_format/parquet.rs:
##########
@@ -543,6 +575,318 @@ async fn fetch_statistics(
Ok(statistics)
}
+/// Implements [`DataSink`] for writing to a parquet file.
+struct ParquetSink {
+ /// Config options for writing data
+ config: FileSinkConfig,
+}
+
+impl Debug for ParquetSink {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ f.debug_struct("ParquetSink").finish()
+ }
+}
+
+impl DisplayAs for ParquetSink {
+ fn fmt_as(&self, t: DisplayFormatType, f: &mut fmt::Formatter<'_>) ->
fmt::Result {
+ match t {
+ DisplayFormatType::Default | DisplayFormatType::Verbose => {
+ write!(
+ f,
+ "ParquetSink(writer_mode={:?}, file_groups=",
+ self.config.writer_mode
+ )?;
+ FileGroupDisplay(&self.config.file_groups).fmt_as(t, f)?;
+ write!(f, ")")
+ }
+ }
+ }
+}
+
+/// Parses datafusion.execution.parquet.encoding String to a
parquet::basic::Encoding
+fn parse_encoding_string(str_setting: &str) ->
Result<parquet::basic::Encoding> {
+ let str_setting_lower: &str = &str_setting.to_lowercase();
+ match str_setting_lower {
+ "plain" => Ok(parquet::basic::Encoding::PLAIN),
Review Comment:
Filed https://github.com/apache/arrow-rs/issues/4693 and
https://github.com/apache/arrow-datafusion/pull/7275 to leave a reference in
DataFusion
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
