[
https://issues.apache.org/jira/browse/GOBBLIN-1720?focusedWorklogId=824108&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-824108
]
ASF GitHub Bot logged work on GOBBLIN-1720:
-------------------------------------------
Author: ASF GitHub Bot
Created on: 08/Nov/22 06:28
Start Date: 08/Nov/22 06:28
Worklog Time Spent: 10m
Work Description: meethngala commented on code in PR #3577:
URL: https://github.com/apache/gobblin/pull/3577#discussion_r1016188914
##########
gobblin-data-management/src/test/java/org/apache/gobblin/data/management/copy/iceberg/IcebergDatasetTest.java:
##########
@@ -246,52 +246,74 @@ public void
testGenerateCopyEntitiesMultiSnapshotWhenDestEmpty() throws IOExcept
MANIFEST_LIST_PATH_1, MANIFEST_PATH_1, MANIFEST_DATA_PATH_1A,
MANIFEST_DATA_PATH_1B);
MockFileSystemBuilder sourceBuilder = new
MockFileSystemBuilder(SRC_FS_URI);
- sourceBuilder.addPaths(expectedPaths);
+ sourceBuilder.addPathsAndFileStatuses(expectedPaths, false);
FileSystem sourceFs = sourceBuilder.build();
IcebergTable icebergTable =
MockIcebergTable.withSnapshots(Arrays.asList(SNAPSHOT_PATHS_1,
SNAPSHOT_PATHS_0));
- IcebergDataset icebergDataset = new TrickIcebergDataset(testDbName,
testTblName, icebergTable, new Properties(), sourceFs);
+ IcebergDataset icebergDataset =
+ new TrickIcebergDataset(testDbName, testTblName, icebergTable, new
Properties(), sourceFs);
MockFileSystemBuilder destBuilder = new MockFileSystemBuilder(DEST_FS_URI);
FileSystem destFs = destBuilder.build();
- CopyConfiguration copyConfiguration = CopyConfiguration.builder(destFs,
copyConfigProperties)
- .preserve(PreserveAttributes.fromMnemonicString(""))
- .copyContext(new CopyContext())
- .build();
+ CopyConfiguration copyConfiguration =
+ CopyConfiguration.builder(destFs,
copyConfigProperties).preserve(PreserveAttributes.fromMnemonicString(""))
+ .copyContext(new CopyContext()).build();
Collection<CopyEntity> copyEntities =
icebergDataset.generateCopyEntities(destFs, copyConfiguration);
verifyCopyEntities(copyEntities, expectedPaths);
}
+ @Test
+ public void testFsOwnershipAndPermissionPreservationWhenDestEmpty() throws
IOException {
+ List<String> expectedPaths = Arrays.asList(METADATA_PATH,
MANIFEST_LIST_PATH_0,
+ MANIFEST_PATH_0, MANIFEST_DATA_PATH_0A, MANIFEST_DATA_PATH_0B);
+ MockFileSystemBuilder sourceBuilder = new
MockFileSystemBuilder(SRC_FS_URI);
+ sourceBuilder.addPathsAndFileStatuses(expectedPaths, true);
+ FileSystem sourceFs = sourceBuilder.build();
+
+ IcebergTable icebergTable =
MockIcebergTable.withSnapshots(Arrays.asList(SNAPSHOT_PATHS_0));
+ IcebergDataset icebergDataset = new TrickIcebergDataset(testDbName,
testTblName, icebergTable, new Properties(), sourceFs);
+
+ MockFileSystemBuilder destBuilder = new MockFileSystemBuilder(DEST_FS_URI);
+ FileSystem destFs = destBuilder.build();
+
+ CopyConfiguration copyConfiguration =
+ CopyConfiguration.builder(destFs, copyConfigProperties)
+ // preserving attributes for owner, group and permissions
respectively
+ .preserve(PreserveAttributes.fromMnemonicString("ugp"))
+ .copyContext(new CopyContext()).build();
+
+ Collection<CopyEntity> copyEntities =
icebergDataset.generateCopyEntities(destFs, copyConfiguration);
+ Map<Path, FileStatus> expectedPathsAndFileStatuses =
sourceBuilder.getPathsAndFileStatuses();
+ verifyFsOwnershipAndPermissionPreservation(copyEntities,
expectedPathsAndFileStatuses);
+ }
+
/**
* exercise {@link IcebergDataset::getFilePaths} and validate the result
* @return {@link IcebergTable} (mock!), for behavioral verification
*/
- protected IcebergTable validateGetFilePathsGivenDestState(
- List<MockIcebergTable.SnapshotPaths> sourceSnapshotPathSets,
- List<String> existingDestPaths,
- Set<Path> expectedResultPaths) throws IOException {
- return validateGetFilePathsGivenDestState(sourceSnapshotPathSets,
Optional.empty(),existingDestPaths, expectedResultPaths);
+ protected IcebergTable
validateGetFilePathsGivenDestState(List<MockIcebergTable.SnapshotPaths>
sourceSnapshotPathSets,
+ List<String> existingDestPaths, Set<Path> expectedResultPaths) throws
IOException {
+ return validateGetFilePathsGivenDestState(sourceSnapshotPathSets,
Optional.empty(), existingDestPaths,
+ expectedResultPaths);
}
/**
* exercise {@link IcebergDataset::getFilePaths} and validate the result
* @return {@link IcebergTable} (mock!), for behavioral verification
*/
- protected IcebergTable validateGetFilePathsGivenDestState(
- List<MockIcebergTable.SnapshotPaths> sourceSnapshotPathSets,
- Optional<List<String>> optExistingSourcePaths,
- List<String> existingDestPaths,
- Set<Path> expectedResultPaths) throws IOException {
+ protected IcebergTable
validateGetFilePathsGivenDestState(List<MockIcebergTable.SnapshotPaths>
sourceSnapshotPathSets,
+ Optional<List<String>> optExistingSourcePaths, List<String>
existingDestPaths, Set<Path> expectedResultPaths) throws IOException {
IcebergTable icebergTable =
MockIcebergTable.withSnapshots(sourceSnapshotPathSets);
MockFileSystemBuilder sourceFsBuilder = new
MockFileSystemBuilder(SRC_FS_URI, !optExistingSourcePaths.isPresent());
- optExistingSourcePaths.ifPresent(sourceFsBuilder::addPaths);
+ optExistingSourcePaths.ifPresent(sourceFsBuilder :: addPaths);
Review Comment:
removed the additional whitespaces!
Issue Time Tracking
-------------------
Worklog Id: (was: 824108)
Time Spent: 1h 40m (was: 1.5h)
> Preserve Ancestor Owner and Permissions for Fs between Src and Dest for
> Iceberg Distcp
> --------------------------------------------------------------------------------------
>
> Key: GOBBLIN-1720
> URL: https://issues.apache.org/jira/browse/GOBBLIN-1720
> Project: Apache Gobblin
> Issue Type: Improvement
> Reporter: Meeth Gala
> Priority: Major
> Time Spent: 1h 40m
> Remaining Estimate: 0h
>
> We want to preserve the Fs ownership and permissions between src and dest
> while performing an Iceberg based distcp. Currently, we are preserving all
> the permissions up to root dir for Iceberg tables.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
