xushiyan commented on code in PR #395:
URL: https://github.com/apache/hudi-rs/pull/395#discussion_r2296780637


##########
crates/core/src/timeline/mod.rs:
##########
@@ -54,75 +61,70 @@ pub const DEFAULT_LOADING_ACTIONS: &[Action] =
     &[Action::Commit, Action::DeltaCommit, Action::ReplaceCommit];
 
 impl Timeline {
-    #[cfg(test)]
-    pub(crate) async fn new_from_completed_commits(
+    pub(crate) fn new(
         hudi_configs: Arc<HudiConfigs>,
-        storage_options: Arc<HashMap<String, String>>,
-        completed_commits: Vec<Instant>,
-    ) -> Result<Self> {
-        let storage = Storage::new(storage_options.clone(), 
hudi_configs.clone())?;
-        Ok(Self {
+        storage: Arc<Storage>,
+        active_loader: TimelineLoader,
+        archived_loader: Option<TimelineLoader>,
+    ) -> Self {
+        Self {
             hudi_configs,
             storage,
-            completed_commits,
-        })
+            active_loader,
+            archived_loader,
+            completed_commits: Vec::new(),
+        }
     }
 
     pub(crate) async fn new_from_storage(
         hudi_configs: Arc<HudiConfigs>,
         storage_options: Arc<HashMap<String, String>>,
     ) -> Result<Self> {
         let storage = Storage::new(storage_options.clone(), 
hudi_configs.clone())?;
+        let mut timeline = TimelineBuilder::new(hudi_configs, 
storage).build().await?;
         let selector = TimelineSelector::completed_actions_in_range(
             DEFAULT_LOADING_ACTIONS,
-            hudi_configs.clone(),
+            timeline.hudi_configs.clone(),
             None,
             None,
         )?;
-        let completed_commits = Self::load_instants(&selector, &storage, 
false).await?;
-        Ok(Self {
-            hudi_configs,
-            storage,
-            completed_commits,
-        })
+        timeline.completed_commits = timeline.load_instants(&selector, 
false).await?;
+        Ok(timeline)
+    }
+
+    pub async fn load_instants(
+        &self,
+        selector: &TimelineSelector,
+        desc: bool,
+    ) -> Result<Vec<Instant>> {
+        self.active_loader.load_instants(selector, desc).await
     }
 
-    async fn load_instants(
+    pub async fn load_instants_with_archive(
+        &self,
         selector: &TimelineSelector,
-        storage: &Storage,
         desc: bool,
+        include_archived: bool,

Review Comment:
   I understand the current TimelineSelector has this property too, which was 
my initial thought. But let's try to go with simple UX for the API: callers do 
not need to know if they need to load archived or not, which is timeline's 
internal impl - they just ask for a list of instants for a given time range 
(everything else that a timeline selector holds):
   - if no specific time given, load active instants
   - if a start or end time given, and falls into archived range, then load 
relevant archived + active instants
   
   Timeline loader can internally decide what to load.
   
   The only case that we might need users to set `include_archived` is to set 
it `false` when they might load for a time range that could be in archived but 
for perf reason they choose to skip archived. This is a rare advanced usage we 
should be good to ignore now. 
   



##########
crates/core/src/timeline/loader.rs:
##########
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use crate::error::CoreError;
+use crate::metadata::HUDI_METADATA_DIR;
+use crate::storage::Storage;
+use crate::timeline::instant::Instant;
+use crate::timeline::lsm_tree::LSM_TIMELINE_DIR;
+use crate::timeline::selector::TimelineSelector;
+use crate::Result;
+use log::debug;
+use std::sync::Arc;
+
+#[derive(Debug, Clone)]
+pub enum TimelineLoader {
+    LayoutOneActive(Arc<Storage>),
+    LayoutOneArchived(Arc<Storage>),
+    LayoutTwoActive(Arc<Storage>),
+    LayoutTwoCompacted(Arc<Storage>),

Review Comment:
   ```suggestion
       LayoutTwoArchived(Arc<Storage>),
   ```
   
   i wasn't thinking thoroughly. i think we should stick with 
`LayoutTwoArchived` like for layout 1, since it's still archived timeline, just 
in the form of LSM tree and compacted, which is internal impl.



##########
crates/core/src/timeline/loader.rs:
##########
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use crate::error::CoreError;
+use crate::metadata::HUDI_METADATA_DIR;
+use crate::storage::Storage;
+use crate::timeline::instant::Instant;
+use crate::timeline::lsm_tree::LSM_TIMELINE_DIR;
+use crate::timeline::selector::TimelineSelector;
+use crate::Result;
+use log::debug;
+use std::sync::Arc;
+
+#[derive(Debug, Clone)]
+pub enum TimelineLoader {
+    LayoutOneActive(Arc<Storage>),
+    LayoutOneArchived(Arc<Storage>),
+    LayoutTwoActive(Arc<Storage>),
+    LayoutTwoCompacted(Arc<Storage>),
+}
+
+impl TimelineLoader {
+    pub async fn load_instants(
+        &self,
+        selector: &TimelineSelector,
+        desc: bool,
+    ) -> Result<Vec<Instant>> {
+        match self {
+            TimelineLoader::LayoutOneActive(storage) => {
+                let files = storage.list_files(Some(HUDI_METADATA_DIR)).await?;
+                let mut instants = Vec::with_capacity(files.len() / 3);
+
+                for file_info in files {
+                    match selector.try_create_instant(file_info.name.as_str()) 
{
+                        Ok(instant) => instants.push(instant),
+                        Err(e) => {
+                            debug!(
+                                "Instant not created from file {:?} due to: 
{:?}",
+                                file_info, e
+                            );
+                        }
+                    }
+                }
+
+                instants.sort_unstable();
+                instants.shrink_to_fit();
+
+                if desc {
+                    Ok(instants.into_iter().rev().collect())
+                } else {
+                    Ok(instants)
+                }
+            }
+            TimelineLoader::LayoutTwoActive(storage) => {
+                let files = storage.list_files(Some(LSM_TIMELINE_DIR)).await?;
+                let mut instants = Vec::new();
+
+                for file_info in files {
+                    if file_info.name.starts_with("history/") || 
file_info.name.starts_with('_') {
+                        continue;
+                    }
+                    match selector.try_create_instant(file_info.name.as_str()) 
{
+                        Ok(instant) => instants.push(instant),
+                        Err(e) => {
+                            debug!(
+                                "Instant not created from file {:?} due to: 
{:?}",
+                                file_info, e
+                            );
+                        }
+                    }
+                }
+                Ok(instants)
+            }
+            _ => Err(CoreError::Unsupported(
+                "Loading from this timeline layout is not implemented 
yet.".to_string(),
+            )),
+        }
+    }
+
+    pub async fn load_archived_instants(
+        &self,
+        _selector: &TimelineSelector,
+        _desc: bool,
+    ) -> Result<Vec<Instant>> {
+        match self {
+            TimelineLoader::LayoutOneArchived(_) => Err(CoreError::Unsupported(
+                "Archived timeline for layout v1 not implemented 
yet".to_string(),
+            )),
+            TimelineLoader::LayoutTwoCompacted(_) => 
Err(CoreError::Unsupported(
+                "Compacted timeline for layout v2 not implemented 
yet".to_string(),

Review Comment:
   totally ok to do archived timeline support separately. reminder to fix the 
name as it's still archived in V2. also this method may not be pub for hiding 
the impl.



##########
crates/core/src/timeline/lsm_tree.rs:
##########
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use crate::error::CoreError;
+use crate::storage::Storage;
+use crate::Result;
+use serde::{Deserialize, Serialize};
+use std::sync::Arc;
+
+pub const LSM_TIMELINE_DIR: &str = ".hoodie/timeline";

Review Comment:
   we need to respect `"hoodie.timeline.history.path"`, and 
`"hoodie.timeline.history.path"` right? same for v6, we need to handle 
`"hoodie.archivelog.folder"`. but we can deal with these in PR for archived 
timeline. the corresponding timeline loader should be responsible to take these 
from configs and own it



##########
crates/core/src/timeline/lsm_tree.rs:
##########
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use crate::error::CoreError;
+use crate::storage::Storage;
+use crate::Result;
+use serde::{Deserialize, Serialize};
+use std::sync::Arc;
+
+pub const LSM_TIMELINE_DIR: &str = ".hoodie/timeline";
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct TimelineManifest {
+    pub version: i64,
+    pub entries: Vec<ManifestEntry>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ManifestEntry {
+    pub file_name: String,
+    pub min_instant: String,
+    pub max_instant: String,
+    pub level: i32,
+    pub file_size: i64,
+}
+
+pub struct LSMTree {
+    storage: Arc<Storage>,

Review Comment:
   need to take any hudi configs for LSM tree specific settings?



##########
crates/core/src/timeline/loader.rs:
##########
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use crate::error::CoreError;
+use crate::metadata::HUDI_METADATA_DIR;
+use crate::storage::Storage;
+use crate::timeline::instant::Instant;
+use crate::timeline::lsm_tree::LSM_TIMELINE_DIR;
+use crate::timeline::selector::TimelineSelector;
+use crate::Result;
+use log::debug;
+use std::sync::Arc;
+
+#[derive(Debug, Clone)]
+pub enum TimelineLoader {
+    LayoutOneActive(Arc<Storage>),
+    LayoutOneArchived(Arc<Storage>),
+    LayoutTwoActive(Arc<Storage>),
+    LayoutTwoCompacted(Arc<Storage>),
+}
+
+impl TimelineLoader {
+    pub async fn load_instants(
+        &self,
+        selector: &TimelineSelector,
+        desc: bool,
+    ) -> Result<Vec<Instant>> {
+        match self {
+            TimelineLoader::LayoutOneActive(storage) => {
+                let files = storage.list_files(Some(HUDI_METADATA_DIR)).await?;
+                let mut instants = Vec::with_capacity(files.len() / 3);
+
+                for file_info in files {
+                    match selector.try_create_instant(file_info.name.as_str()) 
{
+                        Ok(instant) => instants.push(instant),
+                        Err(e) => {
+                            debug!(
+                                "Instant not created from file {:?} due to: 
{:?}",
+                                file_info, e
+                            );
+                        }
+                    }
+                }
+
+                instants.sort_unstable();
+                instants.shrink_to_fit();
+
+                if desc {
+                    Ok(instants.into_iter().rev().collect())
+                } else {
+                    Ok(instants)
+                }
+            }
+            TimelineLoader::LayoutTwoActive(storage) => {
+                let files = storage.list_files(Some(LSM_TIMELINE_DIR)).await?;
+                let mut instants = Vec::new();
+
+                for file_info in files {
+                    if file_info.name.starts_with("history/") || 
file_info.name.starts_with('_') {

Review Comment:
   if we can enhance the `storage.list_files()` to support a listing for only 
immediate files under the prefix, then we dont need explicit check and just 
grab the instant files. What is the case with having `_` prefix?



##########
crates/core/src/timeline/builder.rs:
##########
@@ -0,0 +1,101 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use crate::config::table::HudiTableConfig::{TableVersion, 
TimelineLayoutVersion};
+use crate::config::HudiConfigs;
+use crate::error::CoreError;
+use crate::storage::Storage;
+use crate::timeline::loader::TimelineLoader;
+use crate::timeline::Timeline;
+use crate::Result;
+use std::sync::Arc;
+
+pub struct TimelineBuilder {
+    hudi_configs: Arc<HudiConfigs>,
+    storage: Arc<Storage>,
+}
+
+impl TimelineBuilder {
+    pub fn new(hudi_configs: Arc<HudiConfigs>, storage: Arc<Storage>) -> Self {
+        Self {
+            hudi_configs,
+            storage,
+        }
+    }
+
+    pub async fn build(self) -> Result<Timeline> {
+        let (active_loader, archived_loader) = self.resolve_loader_config()?;
+        let timeline = Timeline::new(
+            self.hudi_configs,
+            self.storage,
+            active_loader,
+            archived_loader,
+        );
+        Ok(timeline)
+    }
+
+    fn resolve_loader_config(&self) -> Result<(TimelineLoader, 
Option<TimelineLoader>)> {
+        let table_version = match self.hudi_configs.get(TableVersion) {
+            Ok(v) => v.to::<isize>(),
+            Err(_) => {
+                return Ok((
+                    TimelineLoader::LayoutOneActive(self.storage.clone()),
+                    
Some(TimelineLoader::LayoutOneArchived(self.storage.clone())),
+                ))
+            }
+        };
+
+        let layout_version = self
+            .hudi_configs
+            .try_get(TimelineLayoutVersion)
+            .map(|v| v.to::<isize>())
+            .unwrap_or_else(|| if table_version == 8 { 2 } else { 1 });
+
+        match layout_version {
+            1 => {
+                if !(5..=8).contains(&table_version) {

Review Comment:
   reminder- master version dropped v5 support



##########
crates/core/src/timeline/loader.rs:
##########
@@ -0,0 +1,111 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use crate::error::CoreError;
+use crate::metadata::HUDI_METADATA_DIR;
+use crate::storage::Storage;
+use crate::timeline::instant::Instant;
+use crate::timeline::lsm_tree::LSM_TIMELINE_DIR;
+use crate::timeline::selector::TimelineSelector;
+use crate::Result;
+use log::debug;
+use std::sync::Arc;
+
+#[derive(Debug, Clone)]
+pub enum TimelineLoader {
+    LayoutOneActive(Arc<Storage>),
+    LayoutOneArchived(Arc<Storage>),
+    LayoutTwoActive(Arc<Storage>),
+    LayoutTwoCompacted(Arc<Storage>),
+}
+
+impl TimelineLoader {
+    pub async fn load_instants(
+        &self,
+        selector: &TimelineSelector,
+        desc: bool,
+    ) -> Result<Vec<Instant>> {
+        match self {
+            TimelineLoader::LayoutOneActive(storage) => {
+                let files = storage.list_files(Some(HUDI_METADATA_DIR)).await?;
+                let mut instants = Vec::with_capacity(files.len() / 3);
+
+                for file_info in files {
+                    match selector.try_create_instant(file_info.name.as_str()) 
{
+                        Ok(instant) => instants.push(instant),
+                        Err(e) => {
+                            debug!(
+                                "Instant not created from file {:?} due to: 
{:?}",
+                                file_info, e
+                            );
+                        }
+                    }
+                }
+
+                instants.sort_unstable();
+                instants.shrink_to_fit();
+
+                if desc {
+                    Ok(instants.into_iter().rev().collect())
+                } else {
+                    Ok(instants)
+                }
+            }
+            TimelineLoader::LayoutTwoActive(storage) => {
+                let files = storage.list_files(Some(LSM_TIMELINE_DIR)).await?;
+                let mut instants = Vec::new();
+
+                for file_info in files {
+                    if file_info.name.starts_with("history/") || 
file_info.name.starts_with('_') {

Review Comment:
   it also occurs to me that we may need to enhance the listing by ignoring 
`.crc` files by default.. we'll track this separately.



##########
crates/core/src/timeline/lsm_tree.rs:
##########
@@ -0,0 +1,70 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use crate::error::CoreError;
+use crate::storage::Storage;
+use crate::Result;
+use serde::{Deserialize, Serialize};
+use std::sync::Arc;
+
+pub const LSM_TIMELINE_DIR: &str = ".hoodie/timeline";
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct TimelineManifest {
+    pub version: i64,
+    pub entries: Vec<ManifestEntry>,
+}
+
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ManifestEntry {

Review Comment:
   maybe some docs help explain this struct



##########
crates/core/src/timeline/mod.rs:
##########
@@ -54,75 +61,70 @@ pub const DEFAULT_LOADING_ACTIONS: &[Action] =
     &[Action::Commit, Action::DeltaCommit, Action::ReplaceCommit];
 
 impl Timeline {
-    #[cfg(test)]
-    pub(crate) async fn new_from_completed_commits(
+    pub(crate) fn new(
         hudi_configs: Arc<HudiConfigs>,
-        storage_options: Arc<HashMap<String, String>>,
-        completed_commits: Vec<Instant>,
-    ) -> Result<Self> {
-        let storage = Storage::new(storage_options.clone(), 
hudi_configs.clone())?;
-        Ok(Self {
+        storage: Arc<Storage>,
+        active_loader: TimelineLoader,
+        archived_loader: Option<TimelineLoader>,
+    ) -> Self {
+        Self {
             hudi_configs,
             storage,
-            completed_commits,
-        })
+            active_loader,
+            archived_loader,
+            completed_commits: Vec::new(),
+        }
     }
 
     pub(crate) async fn new_from_storage(
         hudi_configs: Arc<HudiConfigs>,
         storage_options: Arc<HashMap<String, String>>,
     ) -> Result<Self> {
         let storage = Storage::new(storage_options.clone(), 
hudi_configs.clone())?;
+        let mut timeline = TimelineBuilder::new(hudi_configs, 
storage).build().await?;
         let selector = TimelineSelector::completed_actions_in_range(
             DEFAULT_LOADING_ACTIONS,
-            hudi_configs.clone(),
+            timeline.hudi_configs.clone(),
             None,
             None,
         )?;
-        let completed_commits = Self::load_instants(&selector, &storage, 
false).await?;
-        Ok(Self {
-            hudi_configs,
-            storage,
-            completed_commits,
-        })
+        timeline.completed_commits = timeline.load_instants(&selector, 
false).await?;
+        Ok(timeline)
+    }
+
+    pub async fn load_instants(
+        &self,
+        selector: &TimelineSelector,
+        desc: bool,
+    ) -> Result<Vec<Instant>> {
+        self.active_loader.load_instants(selector, desc).await
     }
 
-    async fn load_instants(
+    pub async fn load_instants_with_archive(
+        &self,
         selector: &TimelineSelector,
-        storage: &Storage,
         desc: bool,
+        include_archived: bool,
     ) -> Result<Vec<Instant>> {
-        let files = storage.list_files(Some(HUDI_METADATA_DIR)).await?;
-
-        // For most cases, we load completed instants, so we can pre-allocate 
the vector with a
-        // capacity of 1/3 of the total number of listed files,
-        // ignoring requested and inflight instants.
-        let mut instants = Vec::with_capacity(files.len() / 3);
-
-        for file_info in files {
-            match selector.try_create_instant(file_info.name.as_str()) {
-                Ok(instant) => instants.push(instant),
-                Err(e) => {
-                    // Ignore files that are not valid or desired instants.
-                    debug!(
-                        "Instant not created from file {:?} due to: {:?}",
-                        file_info, e
-                    );
+        // Always try active first
+        let mut instants = self.active_loader.load_instants(selector, 
desc).await?;
+
+        // Load archived/compacted if flag is set
+        if include_archived {
+            if let Some(archived_loader) = &self.archived_loader {
+                let archived_instants = archived_loader
+                    .load_archived_instants(selector, desc)
+                    .await?;
+                instants.extend(archived_instants);
+
+                // Re-sort after merging
+                instants.sort_unstable();

Review Comment:
   can we be sure about loading results' order- archived instants already 
sorted right? then we can just concat with active ones?



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: [email protected]

For queries about this service, please contact Infrastructure at:
[email protected]

Reply via email to