blackmwk commented on code in PR #2453:
URL: https://github.com/apache/iceberg-rust/pull/2453#discussion_r3328004206
##########
crates/iceberg/src/table.rs:
##########
@@ -356,9 +421,58 @@ impl StaticTable {
}
}
+/// If the table metadata sets the `encryption.key-id` property, build an
+/// [`EncryptionManager`] for the table.
+///
+/// Returns `Ok(None)` if the property is not set. Returns an error if the
+/// property is set but no [`KeyManagementClient`] was provided.
+fn maybe_configure_encryption(
+ kms_client: Option<&Arc<dyn KeyManagementClient>>,
+ metadata: &TableMetadataRef,
+) -> Result<Option<Arc<EncryptionManager>>> {
+ let table_properties = metadata.table_properties()?;
+ let Some(table_key_id) = table_properties.encryption_key_id else {
+ return Ok(None);
+ };
+
+ // Encryption is a v3 feature: `encryption-keys` table metadata and the
+ // snapshot `key-id` field are introduced in format version 3.
+ if metadata.format_version() < FormatVersion::V3 {
Review Comment:
This logic is odd to me. I think the correct one should be like:
```
if format_version < v3 {
return Ok(None)
}
// Now we are v3
if encryption_key_id is none {
return None
}
```
##########
crates/iceberg/src/table.rs:
##########
@@ -264,12 +302,39 @@ impl Table {
self.metadata.current_schema().clone()
}
+ /// Creates a [`ManifestListLoader`] for the given snapshot.
+ pub fn manifest_list_loader<'a>(&'a self, snapshot: &'a Snapshot) ->
ManifestListLoader<'a> {
+ ManifestListLoader {
+ snapshot,
+ file_io: &self.file_io,
+ table_metadata: &self.metadata,
+ encryption_manager: self.encryption_manager.as_deref(),
+ }
+ }
+
/// Create a reader for the table.
pub fn reader_builder(&self) -> ArrowReaderBuilder {
ArrowReaderBuilder::new(self.file_io.clone(), self.runtime().clone())
}
}
+/// Loads a [`ManifestList`] for a snapshot.
+pub struct ManifestListLoader<'a> {
Review Comment:
This should be moved to `manifest_list` module.
##########
crates/iceberg/src/table.rs:
##########
@@ -356,9 +392,58 @@ impl StaticTable {
}
}
+/// If the table metadata sets the `encryption.key-id` property, build an
+/// [`EncryptionManager`] for the table.
+///
+/// Returns `Ok(None)` if the property is not set. Returns an error if the
+/// property is set but no [`KeyManagementClient`] was provided.
+fn maybe_configure_encryption(
+ kms_client: Option<&Arc<dyn KeyManagementClient>>,
+ metadata: &TableMetadataRef,
+) -> Result<Option<Arc<EncryptionManager>>> {
+ let table_properties = metadata.table_properties()?;
+ let Some(table_key_id) = table_properties.encryption_key_id else {
+ return Ok(None);
+ };
+
+ // Encryption is a v3 feature: `encryption-keys` table metadata and the
+ // snapshot `key-id` field are introduced in format version 3.
+ if metadata.format_version() < FormatVersion::V3 {
+ return Err(Error::new(
+ ErrorKind::PreconditionFailed,
+ format!(
+ "Table encryption requires format version 3, found {}",
+ metadata.format_version()
+ ),
+ ));
+ }
+
+ let kms_client = kms_client.ok_or_else(|| {
Review Comment:
Sounds reasonable to me.
##########
crates/iceberg/src/table.rs:
##########
@@ -356,9 +392,58 @@ impl StaticTable {
}
}
+/// If the table metadata sets the `encryption.key-id` property, build an
+/// [`EncryptionManager`] for the table.
+///
+/// Returns `Ok(None)` if the property is not set. Returns an error if the
+/// property is set but no [`KeyManagementClient`] was provided.
+fn maybe_configure_encryption(
+ kms_client: Option<&Arc<dyn KeyManagementClient>>,
+ metadata: &TableMetadataRef,
+) -> Result<Option<Arc<EncryptionManager>>> {
+ let table_properties = metadata.table_properties()?;
+ let Some(table_key_id) = table_properties.encryption_key_id else {
+ return Ok(None);
+ };
+
+ // Encryption is a v3 feature: `encryption-keys` table metadata and the
+ // snapshot `key-id` field are introduced in format version 3.
+ if metadata.format_version() < FormatVersion::V3 {
+ return Err(Error::new(
+ ErrorKind::PreconditionFailed,
+ format!(
+ "Table encryption requires format version 3, found {}",
+ metadata.format_version()
+ ),
+ ));
+ }
+
+ let kms_client = kms_client.ok_or_else(|| {
Review Comment:
We should log a warning if encryption key is not set, but kms client is
provided.
##########
crates/iceberg/src/table.rs:
##########
@@ -264,12 +302,39 @@ impl Table {
self.metadata.current_schema().clone()
}
+ /// Creates a [`ManifestListLoader`] for the given snapshot.
+ pub fn manifest_list_loader<'a>(&'a self, snapshot: &'a Snapshot) ->
ManifestListLoader<'a> {
+ ManifestListLoader {
+ snapshot,
+ file_io: &self.file_io,
+ table_metadata: &self.metadata,
+ encryption_manager: self.encryption_manager.as_deref(),
+ }
+ }
+
/// Create a reader for the table.
pub fn reader_builder(&self) -> ArrowReaderBuilder {
ArrowReaderBuilder::new(self.file_io.clone(), self.runtime().clone())
}
}
+/// Loads a [`ManifestList`] for a snapshot.
+pub struct ManifestListLoader<'a> {
+ snapshot: &'a Snapshot,
+ file_io: &'a FileIO,
+ table_metadata: &'a TableMetadataRef,
+ encryption_manager: Option<&'a EncryptionManager>,
+}
+
+impl ManifestListLoader<'_> {
+ /// Loads and returns the [`ManifestList`] for this snapshot.
+ pub async fn load(&self) -> Result<ManifestList> {
+ self.snapshot
+ .load_manifest_list(self.file_io, self.table_metadata,
self.encryption_manager)
Review Comment:
We should move the logic of loading manifest list into this method, and
deprecate the `load_manifest_list` in `Snapshot`.
##########
crates/iceberg/src/spec/snapshot.rs:
##########
@@ -199,8 +200,22 @@ impl Snapshot {
&self,
file_io: &FileIO,
table_metadata: &TableMetadata,
+ encryption_manager: Option<&EncryptionManager>,
Review Comment:
I prefer to move it into a standalone pr.
##########
crates/iceberg/src/table.rs:
##########
@@ -264,12 +302,39 @@ impl Table {
self.metadata.current_schema().clone()
}
+ /// Creates a [`ManifestListLoader`] for the given snapshot.
+ pub fn manifest_list_loader<'a>(&'a self, snapshot: &'a Snapshot) ->
ManifestListLoader<'a> {
+ ManifestListLoader {
+ snapshot,
+ file_io: &self.file_io,
+ table_metadata: &self.metadata,
+ encryption_manager: self.encryption_manager.as_deref(),
+ }
+ }
+
/// Create a reader for the table.
pub fn reader_builder(&self) -> ArrowReaderBuilder {
ArrowReaderBuilder::new(self.file_io.clone(), self.runtime().clone())
}
}
+/// Loads a [`ManifestList`] for a snapshot.
+pub struct ManifestListLoader<'a> {
Review Comment:
Sorry, I made a mistake, we already have a
[`ManifestListWriter`](https://github.com/apache/iceberg-rust/blob/bc336579e9377c293c4c3ec241a40da156432e53/crates/iceberg/src/spec/manifest_list.rs#L94),
maybe we should call this `ManifestListReader`?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
