[
https://issues.apache.org/jira/browse/GOBBLIN-1709?focusedWorklogId=809366&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-809366
]
ASF GitHub Bot logged work on GOBBLIN-1709:
-------------------------------------------
Author: ASF GitHub Bot
Created on: 16/Sep/22 01:18
Start Date: 16/Sep/22 01:18
Worklog Time Spent: 10m
Work Description: meethngala commented on code in PR #3560:
URL: https://github.com/apache/gobblin/pull/3560#discussion_r972531688
##########
gobblin-data-management/src/test/java/org/apache/gobblin/data/management/copy/iceberg/IcebergDatasetTest.java:
##########
@@ -77,80 +74,112 @@ public void testGetFilePaths() throws IOException {
}
/**
- * Test case to copy all the file paths for a mocked iceberg table. This is
a full copy overwriting everything on the destination
+ * Test case to generate copy entities for all the file paths for a mocked
iceberg table.
+ * The assumption here is that we create copy entities for all the matching
file paths,
+ * without calculating any difference between the source and destination
*/
@Test
public void testGenerateCopyEntitiesForTableFileSet() throws IOException,
URISyntaxException {
FileSystem fs = Mockito.mock(FileSystem.class);
String test_db_name = "test_db_name";
String test_table_name = "test_tbl_name";
- Set<String> setOfFilePaths = new HashSet<>(Arrays.asList(METADATA_PATH,
MANIFEST_PATH, MANIFEST_FILE_PATH1, MANIFEST_FILE_PATH2));
-
+ String test_qualified_path =
"/root/iceberg/test/destination/sub_path_destination";
+ String test_uri_path = "/root/iceberg/test/output";
Properties properties = new Properties();
properties.setProperty("data.publisher.final.dir", "/test");
+ List<String> expected = new ArrayList<>(Arrays.asList(METADATA_PATH,
MANIFEST_PATH, MANIFEST_LIST_PATH, MANIFEST_FILE_PATH1, MANIFEST_FILE_PATH2));
CopyConfiguration copyConfiguration = CopyConfiguration.builder(null,
properties)
.preserve(PreserveAttributes.fromMnemonicString(""))
.copyContext(new CopyContext())
.build();
- TableOperations tableOperations = Mockito.mock(TableOperations.class);
- IcebergTable icebergTable = new MockedIcebergTable(tableOperations);
+ IcebergTable icebergTable = new MockedIcebergTable(METADATA_PATH,
MANIFEST_PATH, MANIFEST_LIST_PATH, new
ArrayList<>(Arrays.asList(MANIFEST_FILE_PATH1, MANIFEST_FILE_PATH2)));
IcebergDataset icebergDataset = new IcebergDataset(test_db_name,
test_table_name, icebergTable, new Properties(), fs);
+ Path path1 = new Path(METADATA_PATH);
+ Path path2 = new Path(MANIFEST_PATH);
+ Path path3 = new Path(MANIFEST_LIST_PATH);
+ Path path4 = new Path(MANIFEST_FILE_PATH1);
+ Path path5 = new Path(MANIFEST_FILE_PATH2);
+
FileStatus fileStatus1 = new FileStatus();
- fileStatus1.setPath(new Path(METADATA_PATH));
+ fileStatus1.setPath(path1);
FileStatus fileStatus2 = new FileStatus();
- fileStatus2.setPath(new Path(MANIFEST_PATH));
+ fileStatus2.setPath(path2);
FileStatus fileStatus3 = new FileStatus();
- fileStatus3.setPath(new Path(MANIFEST_FILE_PATH1));
+ fileStatus3.setPath(path3);
FileStatus fileStatus4 = new FileStatus();
- fileStatus4.setPath(new Path(MANIFEST_FILE_PATH2));
-
- Path path1 = new Path(METADATA_PATH);
- Path path2 = new Path(MANIFEST_PATH);
- Path path3 = new Path(MANIFEST_FILE_PATH1);
- Path path4 = new Path(MANIFEST_FILE_PATH2);
+ fileStatus4.setPath(path4);
+ FileStatus fileStatus5 = new FileStatus();
+ fileStatus5.setPath(path5);
+ Map<Path, FileStatus> mapOfPathAndFileStatus = Maps.newHashMap();
+ mapOfPathAndFileStatus.put(path1, fileStatus1);
+ mapOfPathAndFileStatus.put(path2, fileStatus2);
+ mapOfPathAndFileStatus.put(path3, fileStatus3);
+ mapOfPathAndFileStatus.put(path4, fileStatus4);
+ mapOfPathAndFileStatus.put(path5, fileStatus5);
Review Comment:
that's a good idea... pushed the implementation as part of the latest commit
Issue Time Tracking
-------------------
Worklog Id: (was: 809366)
Time Spent: 5h 40m (was: 5.5h)
> Create work units for Hive Catalog based Iceberg Datasets to support Distcp
> for Iceberg
> ---------------------------------------------------------------------------------------
>
> Key: GOBBLIN-1709
> URL: https://issues.apache.org/jira/browse/GOBBLIN-1709
> Project: Apache Gobblin
> Issue Type: New Feature
> Components: distcp-ng
> Reporter: Meeth Gala
> Assignee: Issac Buenrostro
> Priority: Major
> Time Spent: 5h 40m
> Remaining Estimate: 0h
>
> We want to support Distcp for Iceberg based datasets.
> As a pilot, we are starting with Hive Catalog and will expand the
> functionality to cover all Iceberg based datasets.
>
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
