[
https://issues.apache.org/jira/browse/GOBBLIN-1720?focusedWorklogId=814232&page=com.atlassian.jira.plugin.system.issuetabpanels:worklog-tabpanel#worklog-814232
]
ASF GitHub Bot logged work on GOBBLIN-1720:
-------------------------------------------
Author: ASF GitHub Bot
Created on: 06/Oct/22 05:43
Start Date: 06/Oct/22 05:43
Worklog Time Spent: 10m
Work Description: phet commented on code in PR #3577:
URL: https://github.com/apache/gobblin/pull/3577#discussion_r988569180
##########
gobblin-data-management/src/test/java/org/apache/gobblin/data/management/copy/iceberg/IcebergDatasetTest.java:
##########
@@ -176,22 +176,72 @@ public void
testGenerateCopyEntitiesMultiSnapshotWhenDestEmpty() throws IOExcept
verifyCopyEntities(copyEntities, expectedPaths);
}
+ @Test
+ public void testFsOwnershipAndPermissionPreservationWhenDestEmpty() throws
IOException {
+ List<String> expectedPaths = Arrays.asList(METADATA_PATH,
MANIFEST_LIST_PATH_0,
+ MANIFEST_PATH_0, MANIFEST_DATA_PATH_0A, MANIFEST_DATA_PATH_0B);
+
+ MockFileSystemBuilder sourceBuilder = new
MockFileSystemBuilder(SRC_FS_URI);
+ sourceBuilder.addPaths(expectedPaths);
+ FileSystem sourceFs = sourceBuilder.build();
+
+ IcebergTable icebergTable = new
MockedIcebergTable(Arrays.asList(SNAPSHOT_PATHS_0));
+ IcebergDataset icebergDataset = new TrickIcebergDataset(testDbName,
testTblName, icebergTable, new Properties(), sourceFs);
+
+ MockFileSystemBuilder destBuilder = new MockFileSystemBuilder(DEST_FS_URI);
+ FileSystem destFs = destBuilder.build();
+
+ CopyConfiguration copyConfiguration = CopyConfiguration.builder(destFs,
copyConfigProperties)
+ .preserve(PreserveAttributes.fromMnemonicString("ugp"))
+ .copyContext(new CopyContext())
+ .build();
+ Collection<CopyEntity> copyEntities =
icebergDataset.generateCopyEntities(destFs, copyConfiguration);
+ verifyFsOwnershipAndPermissionPreservation(copyEntities);
+ }
+
private void verifyCopyEntities(Collection<CopyEntity> copyEntities,
List<String> expected) {
List<String> actual = new ArrayList<>();
for (CopyEntity copyEntity : copyEntities) {
String json = copyEntity.toString();
- String filepath = new Gson().fromJson(json, JsonObject.class)
- .getAsJsonObject("object-data").getAsJsonObject("origin")
- .getAsJsonObject("object-data").getAsJsonObject("path")
- .getAsJsonObject("object-data").getAsJsonObject("uri")
- .getAsJsonPrimitive("object-data").getAsString();
+ String filepath = getFilePathAsStringFromJson(json);
actual.add(filepath);
}
Assert.assertEquals(actual.size(), expected.size(),
"Set" + actual.toString() + " vs Set" + expected.toString());
Assert.assertEqualsNoOrder(actual.toArray(), expected.toArray());
}
+ private void
verifyFsOwnershipAndPermissionPreservation(Collection<CopyEntity> copyEntities)
{
+ List<JsonObject> ancestorsOwnerAndPermissionsList = new ArrayList<>();
+ List<JsonObject> destinationOwnerAndPermissionsList = new ArrayList<>();
+
+ for(CopyEntity copyEntity : copyEntities) {
+ String copyEntityJson = copyEntity.toString();
+
+ JsonArray ancestorsOwnerAndPermissions = new
Gson().fromJson(copyEntityJson, JsonObject.class)
+
.getAsJsonObject("object-data").getAsJsonArray("ancestorsOwnerAndPermission");
+ JsonObject rootAncestorOwnerAndPermissions =
ancestorsOwnerAndPermissions.get(ancestorsOwnerAndPermissions.size()-1)
+ .getAsJsonObject().getAsJsonObject();
+ ancestorsOwnerAndPermissionsList.add(rootAncestorOwnerAndPermissions);
+ String filePath = getFilePathAsStringFromJson(copyEntityJson);
+ Assert.assertEquals(ancestorsOwnerAndPermissions.size(), new
Path(filePath).getParent().depth() -1);
+
+ JsonObject destinationOwnerAndPermissions = new
Gson().fromJson(copyEntityJson, JsonObject.class)
+
.getAsJsonObject("object-data").getAsJsonObject("destinationOwnerAndPermission");
+ destinationOwnerAndPermissionsList.add(destinationOwnerAndPermissions);
+ }
+ Assert.assertEqualsNoOrder(ancestorsOwnerAndPermissionsList.toArray(),
destinationOwnerAndPermissionsList.toArray());
Review Comment:
both of these were initialized from the `CopyEntity` JSON... so,
perhaps--maybe?--that demonstrates some level of internal coherence in the
result... nonetheless it doesn't verify that the particular values of either
are anything in particular (e.g. what they're expect to be, if the impl's
correct).
Issue Time Tracking
-------------------
Worklog Id: (was: 814232)
Time Spent: 20m (was: 10m)
> Preserve Ancestor Owner and Permissions for Fs between Src and Dest for
> Iceberg Distcp
> --------------------------------------------------------------------------------------
>
> Key: GOBBLIN-1720
> URL: https://issues.apache.org/jira/browse/GOBBLIN-1720
> Project: Apache Gobblin
> Issue Type: Improvement
> Reporter: Meeth Gala
> Priority: Major
> Time Spent: 20m
> Remaining Estimate: 0h
>
> We want to preserve the Fs ownership and permissions between src and dest
> while performing an Iceberg based distcp. Currently, we are preserving all
> the permissions up to root dir for Iceberg tables.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
