phet commented on code in PR #3864:
URL: https://github.com/apache/gobblin/pull/3864#discussion_r1468125008
##########
gobblin-service/src/test/java/org/apache/gobblin/service/GobblinServiceManagerTest.java:
##########
@@ -339,60 +342,83 @@ public void testRunOnceJob() throws Exception {
AssertWithBackoff.create().maxSleepMs(200L).timeoutMs(2000L).backoffFactor(1)
.assertTrue(input ->
this.gobblinServiceManager.getFlowCatalog().getSpecs().size() == 0,
"Waiting for job to get orchestrated...");
-
AssertWithBackoff.create().maxSleepMs(100L).timeoutMs(1000L).backoffFactor(1)
- .assertTrue(input ->
!this.gobblinServiceManager.getScheduler().getScheduledFlowSpecs().containsKey(TEST_URI.toString()),
- "Waiting for job to get orchestrated...");
+
AssertWithBackoff.create().maxSleepMs(100L).timeoutMs(2000L).backoffFactor(1)
+ .assertTrue(input ->
!this.gobblinServiceManager.getScheduler().getScheduledFlowSpecs().containsKey(uri.toString()),
Review Comment:
"not present" tests can mislead us. e.g. wouldn't we find success here even
if `uri.equals("what, me worry?")`?
should we first verify prior that the scheduler actually at one point had
this URI?
##########
gobblin-service/src/test/java/org/apache/gobblin/service/GobblinServiceManagerTest.java:
##########
@@ -403,94 +429,145 @@ public void testCreateAgain() throws Exception {
Assert.assertEquals(exception.getStatus(), HttpStatus.CONFLICT_409);
}
+ /*
+ This test adds a new flowConfig to the client checks the config retrieved
using the getFlowConfig endpoint matches it
+ */
@Test (dependsOnMethods = "testCreateAgain")
public void testGet() throws Exception {
- FlowConfig flowConfig = this.flowConfigClient.getFlowConfig(TEST_FLOW_ID);
+ FlowId flowId = createFlowIdWithUniqueName(TEST_GROUP_NAME);
+ FlowConfig flowConfig = new FlowConfig().setId(flowId)
+ .setTemplateUris(TEST_TEMPLATE_URI).setSchedule(new
Schedule().setCronSchedule(TEST_SCHEDULE).setRunImmediately(true))
+ .setProperties(new StringMap(flowProperties));
+ this.flowConfigClient.createFlowConfig(flowConfig);
- Assert.assertEquals(flowConfig.getId().getFlowGroup(), TEST_GROUP_NAME);
- Assert.assertEquals(flowConfig.getId().getFlowName(), TEST_FLOW_NAME);
- Assert.assertEquals(flowConfig.getSchedule().getCronSchedule(),
TEST_SCHEDULE);
- Assert.assertEquals(flowConfig.getTemplateUris(), TEST_TEMPLATE_URI);
- Assert.assertTrue(flowConfig.getSchedule().isRunImmediately());
+ FlowConfig getFlowConfig = this.flowConfigClient.getFlowConfig(flowId);
+ Assert.assertEquals(getFlowConfig.getId().getFlowGroup(), TEST_GROUP_NAME);
+ Assert.assertEquals(getFlowConfig.getId().getFlowName(),
flowId.getFlowName());
+ Assert.assertEquals(getFlowConfig.getSchedule().getCronSchedule(),
TEST_SCHEDULE);
+ Assert.assertEquals(getFlowConfig.getTemplateUris(), TEST_TEMPLATE_URI);
+ Assert.assertTrue(getFlowConfig.getSchedule().isRunImmediately());
// Add this assert back when getFlowSpec() is changed to return the raw
flow spec
- //Assert.assertEquals(flowConfig.getProperties().size(), 1);
- Assert.assertEquals(flowConfig.getProperties().get("param1"), "value1");
+ //Assert.assertEquals(getFlowConfig.getProperties().size(), 1);
+ Assert.assertEquals(getFlowConfig.getProperties().get("param1"), "value1");
}
- @Test (dependsOnMethods = "testCreateAgain")
+ /*
+ Adds one more flowConfig to the flowCatalog, checks that the size
increased, and then deletes the one it just added
+ */
+ @Test (dependsOnMethods = "testGet")
public void testGetAll() throws Exception {
- FlowConfig flowConfig2 = new FlowConfig().setId(TEST_FLOW_ID2)
+ int sizeBefore = this.flowConfigClient.getAllFlowConfigs().size();
+ FlowId flowId = createFlowIdWithUniqueName(TEST_GROUP_NAME);
+ FlowConfig flowConfig = new FlowConfig().setId(flowId)
.setTemplateUris(TEST_TEMPLATE_URI).setSchedule(new
Schedule().setCronSchedule(TEST_SCHEDULE))
.setProperties(new StringMap(flowProperties));
- this.flowConfigClient.createFlowConfig(flowConfig2);
+ this.flowConfigClient.createFlowConfig(flowConfig);
Collection<FlowConfig> flowConfigs =
this.flowConfigClient.getAllFlowConfigs();
- Assert.assertEquals(flowConfigs.size(), 2);
+ Assert.assertEquals(flowConfigs.size(), sizeBefore + 1);
- this.flowConfigClient.deleteFlowConfig(TEST_FLOW_ID2);
+ this.flowConfigClient.deleteFlowConfig(flowId);
}
- @Test (dependsOnMethods = "testCreateAgain", enabled = false)
+ /*
+ This tests the getAll method using one filter (group name). Note that to
remove dependency on other tests, we
+ do not check for number of items from the group name used for other
tests.
+ */
+ @Test (dependsOnMethods = "testGetAll")//, enabled = false, groups =
{"disabledOnCI"})
public void testGetFilteredFlows() throws Exception {
// Not implemented for FsSpecStore
- Collection<FlowConfig> flowConfigs =
this.flowConfigClient.getFlowConfigs(TEST_GROUP_NAME, null, null, null, null,
null,
-null, null, null, null);
- Assert.assertEquals(flowConfigs.size(), 2);
+ // Add 1 config with group name 2 & check size
+ FlowId flowId2 = createFlowIdWithUniqueName(TEST_GROUP_NAME2);
+ FlowConfig flowConfig = new FlowConfig().setId(flowId2)
+ .setTemplateUris(TEST_TEMPLATE_URI).setSchedule(new
Schedule().setCronSchedule(TEST_SCHEDULE))
+ .setProperties(new StringMap(flowProperties));
+ this.flowConfigClient.createFlowConfig(flowConfig);
- flowConfigs = this.flowConfigClient.getFlowConfigs(TEST_GROUP_NAME,
TEST_FLOW_NAME2, null, null, null, null,
+ Collection<FlowConfig> flowConfigs =
this.flowConfigClient.getFlowConfigs(TEST_GROUP_NAME2, null, null, null, null,
null,
null, null, null, null);
Assert.assertEquals(flowConfigs.size(), 1);
- flowConfigs = this.flowConfigClient.getFlowConfigs(null, null, null, null,
null, null,
- TEST_SCHEDULE, null, null, null);
+ // Add another and check size
+ FlowId flowId3 = createFlowIdWithUniqueName(TEST_GROUP_NAME2);
+ flowConfig = new FlowConfig().setId(flowId3)
+ .setTemplateUris(TEST_TEMPLATE_URI).setSchedule(new
Schedule().setCronSchedule(TEST_SCHEDULE))
+ .setProperties(new StringMap(flowProperties));
+ this.flowConfigClient.createFlowConfig(flowConfig);
+
+ // Check for flow with flowId3's name
+ flowConfigs = this.flowConfigClient.getFlowConfigs(TEST_GROUP_NAME2, null,
null, null, null, null,
+ null, null, null, null);
Assert.assertEquals(flowConfigs.size(), 2);
+
+ flowConfigs = this.flowConfigClient.getFlowConfigs(null,
flowId3.getFlowName(), null, null, null, null,
+ TEST_SCHEDULE, null, null, null);
+ Assert.assertEquals(flowConfigs.size(), 1);
}
- @Test (dependsOnMethods = "testGet")
+ @Test (dependsOnMethods = "testGetFilteredFlows") // depends on testGetAll
until testGetFilteredFlows is enabled
public void testUpdate() throws Exception {
- FlowId flowId = new
FlowId().setFlowGroup(TEST_GROUP_NAME).setFlowName(TEST_FLOW_NAME);
+ FlowId flowId = createFlowIdWithUniqueName(TEST_GROUP_NAME);
+ // Original flow config
Map<String, String> flowProperties = Maps.newHashMap();
flowProperties.put("param1", "value1b");
flowProperties.put("param2", "value2b");
flowProperties.put(ServiceConfigKeys.FLOW_SOURCE_IDENTIFIER_KEY,
TEST_SOURCE_NAME);
flowProperties.put(ServiceConfigKeys.FLOW_DESTINATION_IDENTIFIER_KEY,
TEST_SINK_NAME);
- FlowConfig flowConfig = new FlowConfig().setId(TEST_FLOW_ID)
+ FlowConfig flowConfig = new FlowConfig().setId(flowId)
.setTemplateUris(TEST_TEMPLATE_URI).setSchedule(new
Schedule().setCronSchedule(TEST_SCHEDULE))
.setProperties(new StringMap(flowProperties));
- this.flowConfigClient.updateFlowConfig(flowConfig);
+ this.flowConfigClient.createFlowConfig(flowConfig);
+
+ // Updated flow config
+ flowProperties = Maps.newHashMap();
+ String newValue1 = "updated1b";
+ String newValue2 = "updated2b";
+ flowProperties.put("param1", newValue1);
+ flowProperties.put("param2", newValue2);
+ flowProperties.put(ServiceConfigKeys.FLOW_SOURCE_IDENTIFIER_KEY,
TEST_SOURCE_NAME);
+ flowProperties.put(ServiceConfigKeys.FLOW_DESTINATION_IDENTIFIER_KEY,
TEST_SINK_NAME);
+
+ FlowConfig updatedFlowConfig = new FlowConfig().setId(flowId)
+ .setTemplateUris(TEST_TEMPLATE_URI).setSchedule(new
Schedule().setCronSchedule(TEST_SCHEDULE))
+ .setProperties(new StringMap(flowProperties));
+ this.flowConfigClient.updateFlowConfig(updatedFlowConfig);
FlowConfig retrievedFlowConfig =
this.flowConfigClient.getFlowConfig(flowId);
-
Assert.assertTrue(this.gobblinServiceManager.getScheduler().getScheduledFlowSpecs().containsKey(TEST_URI.toString()));
+ // // TODO: remove scheduler related assertions until they are debugged
+ //
Assert.assertTrue(this.gobblinServiceManager.getScheduler().getScheduledFlowSpecs().containsKey(TEST_URI.toString()));
Assert.assertEquals(retrievedFlowConfig.getId().getFlowGroup(),
TEST_GROUP_NAME);
- Assert.assertEquals(retrievedFlowConfig.getId().getFlowName(),
TEST_FLOW_NAME);
+ Assert.assertEquals(retrievedFlowConfig.getId().getFlowName(),
flowId.getFlowName());
Assert.assertEquals(retrievedFlowConfig.getSchedule().getCronSchedule(),
TEST_SCHEDULE);
Assert.assertEquals(retrievedFlowConfig.getTemplateUris(),
TEST_TEMPLATE_URI);
- // Add this asssert when getFlowSpec() is changed to return the raw flow
spec
+ // Add this assert when getFlowSpec() is changed to return the raw flow
spec
//Assert.assertEquals(flowConfig.getProperties().size(), 2);
Review Comment:
same Q here about advice
##########
gobblin-service/src/test/java/org/apache/gobblin/service/GobblinServiceManagerTest.java:
##########
@@ -708,4 +788,11 @@ public void testGetFilteredFlowsPaginated() throws
Exception {
this.flowConfigClient.deleteFlowConfig(TEST_FLOW_ID6);
this.flowConfigClient.deleteFlowConfig(TEST_FLOW_ID7);
}
+
+ /*
+ Creates a unique flowId for a given group using the current timestamp as the
flowName.
+ */
+ public FlowId createFlowIdWithUniqueName(String groupName) {
Review Comment:
good call to add--I like this!
is there somewhere configured to ensure the tests are not run in parallel
(such that timestamp is no longer unique)?
also are we certain it will always take >1ms between successive calls? we
could always use nanos...
##########
gobblin-service/src/test/java/org/apache/gobblin/service/GobblinServiceManagerTest.java:
##########
@@ -501,28 +578,29 @@ public void testDelete() throws Exception {
@Test (dependsOnMethods = "testDelete")
public void testGitCreate() throws Exception {
+ URI uri = FlowSpec.Utils.createFlowSpecUri(new
FlowId().setFlowGroup(TEST_GROUP_NAME).setFlowName("testGitFlow"));
Review Comment:
is the rename in this PR to ensure it's unique so no collisions w/ other
tests? if so, shall we put the name as a constant up where `TEST_GROUP_NAME`
and others are defined?
##########
gobblin-service/src/test/java/org/apache/gobblin/service/GobblinServiceManagerTest.java:
##########
@@ -403,94 +429,145 @@ public void testCreateAgain() throws Exception {
Assert.assertEquals(exception.getStatus(), HttpStatus.CONFLICT_409);
}
+ /*
+ This test adds a new flowConfig to the client checks the config retrieved
using the getFlowConfig endpoint matches it
+ */
@Test (dependsOnMethods = "testCreateAgain")
public void testGet() throws Exception {
- FlowConfig flowConfig = this.flowConfigClient.getFlowConfig(TEST_FLOW_ID);
+ FlowId flowId = createFlowIdWithUniqueName(TEST_GROUP_NAME);
+ FlowConfig flowConfig = new FlowConfig().setId(flowId)
+ .setTemplateUris(TEST_TEMPLATE_URI).setSchedule(new
Schedule().setCronSchedule(TEST_SCHEDULE).setRunImmediately(true))
+ .setProperties(new StringMap(flowProperties));
+ this.flowConfigClient.createFlowConfig(flowConfig);
- Assert.assertEquals(flowConfig.getId().getFlowGroup(), TEST_GROUP_NAME);
- Assert.assertEquals(flowConfig.getId().getFlowName(), TEST_FLOW_NAME);
- Assert.assertEquals(flowConfig.getSchedule().getCronSchedule(),
TEST_SCHEDULE);
- Assert.assertEquals(flowConfig.getTemplateUris(), TEST_TEMPLATE_URI);
- Assert.assertTrue(flowConfig.getSchedule().isRunImmediately());
+ FlowConfig getFlowConfig = this.flowConfigClient.getFlowConfig(flowId);
+ Assert.assertEquals(getFlowConfig.getId().getFlowGroup(), TEST_GROUP_NAME);
+ Assert.assertEquals(getFlowConfig.getId().getFlowName(),
flowId.getFlowName());
+ Assert.assertEquals(getFlowConfig.getSchedule().getCronSchedule(),
TEST_SCHEDULE);
+ Assert.assertEquals(getFlowConfig.getTemplateUris(), TEST_TEMPLATE_URI);
+ Assert.assertTrue(getFlowConfig.getSchedule().isRunImmediately());
// Add this assert back when getFlowSpec() is changed to return the raw
flow spec
- //Assert.assertEquals(flowConfig.getProperties().size(), 1);
- Assert.assertEquals(flowConfig.getProperties().get("param1"), "value1");
+ //Assert.assertEquals(getFlowConfig.getProperties().size(), 1);
Review Comment:
is this still up-to-date maintenance advice re the todo?
##########
gobblin-service/src/test/java/org/apache/gobblin/service/GobblinServiceManagerTest.java:
##########
@@ -403,94 +429,145 @@ public void testCreateAgain() throws Exception {
Assert.assertEquals(exception.getStatus(), HttpStatus.CONFLICT_409);
}
+ /*
+ This test adds a new flowConfig to the client checks the config retrieved
using the getFlowConfig endpoint matches it
+ */
@Test (dependsOnMethods = "testCreateAgain")
public void testGet() throws Exception {
- FlowConfig flowConfig = this.flowConfigClient.getFlowConfig(TEST_FLOW_ID);
+ FlowId flowId = createFlowIdWithUniqueName(TEST_GROUP_NAME);
+ FlowConfig flowConfig = new FlowConfig().setId(flowId)
+ .setTemplateUris(TEST_TEMPLATE_URI).setSchedule(new
Schedule().setCronSchedule(TEST_SCHEDULE).setRunImmediately(true))
+ .setProperties(new StringMap(flowProperties));
+ this.flowConfigClient.createFlowConfig(flowConfig);
- Assert.assertEquals(flowConfig.getId().getFlowGroup(), TEST_GROUP_NAME);
- Assert.assertEquals(flowConfig.getId().getFlowName(), TEST_FLOW_NAME);
- Assert.assertEquals(flowConfig.getSchedule().getCronSchedule(),
TEST_SCHEDULE);
- Assert.assertEquals(flowConfig.getTemplateUris(), TEST_TEMPLATE_URI);
- Assert.assertTrue(flowConfig.getSchedule().isRunImmediately());
+ FlowConfig getFlowConfig = this.flowConfigClient.getFlowConfig(flowId);
+ Assert.assertEquals(getFlowConfig.getId().getFlowGroup(), TEST_GROUP_NAME);
+ Assert.assertEquals(getFlowConfig.getId().getFlowName(),
flowId.getFlowName());
+ Assert.assertEquals(getFlowConfig.getSchedule().getCronSchedule(),
TEST_SCHEDULE);
+ Assert.assertEquals(getFlowConfig.getTemplateUris(), TEST_TEMPLATE_URI);
+ Assert.assertTrue(getFlowConfig.getSchedule().isRunImmediately());
// Add this assert back when getFlowSpec() is changed to return the raw
flow spec
- //Assert.assertEquals(flowConfig.getProperties().size(), 1);
- Assert.assertEquals(flowConfig.getProperties().get("param1"), "value1");
+ //Assert.assertEquals(getFlowConfig.getProperties().size(), 1);
+ Assert.assertEquals(getFlowConfig.getProperties().get("param1"), "value1");
}
- @Test (dependsOnMethods = "testCreateAgain")
+ /*
+ Adds one more flowConfig to the flowCatalog, checks that the size
increased, and then deletes the one it just added
+ */
+ @Test (dependsOnMethods = "testGet")
public void testGetAll() throws Exception {
- FlowConfig flowConfig2 = new FlowConfig().setId(TEST_FLOW_ID2)
+ int sizeBefore = this.flowConfigClient.getAllFlowConfigs().size();
+ FlowId flowId = createFlowIdWithUniqueName(TEST_GROUP_NAME);
+ FlowConfig flowConfig = new FlowConfig().setId(flowId)
.setTemplateUris(TEST_TEMPLATE_URI).setSchedule(new
Schedule().setCronSchedule(TEST_SCHEDULE))
.setProperties(new StringMap(flowProperties));
- this.flowConfigClient.createFlowConfig(flowConfig2);
+ this.flowConfigClient.createFlowConfig(flowConfig);
Collection<FlowConfig> flowConfigs =
this.flowConfigClient.getAllFlowConfigs();
- Assert.assertEquals(flowConfigs.size(), 2);
+ Assert.assertEquals(flowConfigs.size(), sizeBefore + 1);
- this.flowConfigClient.deleteFlowConfig(TEST_FLOW_ID2);
+ this.flowConfigClient.deleteFlowConfig(flowId);
}
- @Test (dependsOnMethods = "testCreateAgain", enabled = false)
+ /*
+ This tests the getAll method using one filter (group name). Note that to
remove dependency on other tests, we
+ do not check for number of items from the group name used for other
tests.
+ */
+ @Test (dependsOnMethods = "testGetAll")//, enabled = false, groups =
{"disabledOnCI"})
public void testGetFilteredFlows() throws Exception {
// Not implemented for FsSpecStore
- Collection<FlowConfig> flowConfigs =
this.flowConfigClient.getFlowConfigs(TEST_GROUP_NAME, null, null, null, null,
null,
-null, null, null, null);
- Assert.assertEquals(flowConfigs.size(), 2);
+ // Add 1 config with group name 2 & check size
+ FlowId flowId2 = createFlowIdWithUniqueName(TEST_GROUP_NAME2);
+ FlowConfig flowConfig = new FlowConfig().setId(flowId2)
+ .setTemplateUris(TEST_TEMPLATE_URI).setSchedule(new
Schedule().setCronSchedule(TEST_SCHEDULE))
+ .setProperties(new StringMap(flowProperties));
+ this.flowConfigClient.createFlowConfig(flowConfig);
- flowConfigs = this.flowConfigClient.getFlowConfigs(TEST_GROUP_NAME,
TEST_FLOW_NAME2, null, null, null, null,
+ Collection<FlowConfig> flowConfigs =
this.flowConfigClient.getFlowConfigs(TEST_GROUP_NAME2, null, null, null, null,
null,
null, null, null, null);
Assert.assertEquals(flowConfigs.size(), 1);
- flowConfigs = this.flowConfigClient.getFlowConfigs(null, null, null, null,
null, null,
- TEST_SCHEDULE, null, null, null);
+ // Add another and check size
+ FlowId flowId3 = createFlowIdWithUniqueName(TEST_GROUP_NAME2);
+ flowConfig = new FlowConfig().setId(flowId3)
+ .setTemplateUris(TEST_TEMPLATE_URI).setSchedule(new
Schedule().setCronSchedule(TEST_SCHEDULE))
+ .setProperties(new StringMap(flowProperties));
+ this.flowConfigClient.createFlowConfig(flowConfig);
+
+ // Check for flow with flowId3's name
+ flowConfigs = this.flowConfigClient.getFlowConfigs(TEST_GROUP_NAME2, null,
null, null, null, null,
+ null, null, null, null);
Assert.assertEquals(flowConfigs.size(), 2);
+
+ flowConfigs = this.flowConfigClient.getFlowConfigs(null,
flowId3.getFlowName(), null, null, null, null,
+ TEST_SCHEDULE, null, null, null);
+ Assert.assertEquals(flowConfigs.size(), 1);
}
- @Test (dependsOnMethods = "testGet")
+ @Test (dependsOnMethods = "testGetFilteredFlows") // depends on testGetAll
until testGetFilteredFlows is enabled
Review Comment:
comment may be inaccurate
##########
gobblin-service/src/test/java/org/apache/gobblin/service/GobblinServiceManagerTest.java:
##########
@@ -603,43 +683,41 @@ public void testGetAllPaginated() throws Exception {
.setProperties(new StringMap(flowProperties));
this.flowConfigClient.createFlowConfig(flowConfig4);
- // Check that there are a total of 4 flowConfigs by using the default
getAll call
+ // Check that the size of flowConfigs has increased by 4 using the default
getAll call
Collection<FlowConfig> flowConfigs =
this.flowConfigClient.getAllFlowConfigs();
- Assert.assertEquals(flowConfigs.size(), 4);
+ Assert.assertEquals(flowConfigs.size(), sizeBefore + 4);
- // Check that there are a total of 4 flowConfigs using new getAll call
+ // Check that the size of flowConfigs has increased by 4 using new getAll
call
flowConfigs = this.flowConfigClient.getAllFlowConfigs(0,20);
- Assert.assertEquals(flowConfigs.size(), 4);
+ Assert.assertEquals(flowConfigs.size(), sizeBefore + 4);
// Attempt pagination with one element from the start of the specStore
configurations stored
// Start at index 0 and return 1 element
- flowConfigs = this.flowConfigClient.getAllFlowConfigs(0,1);
+ flowConfigs = this.flowConfigClient.getAllFlowConfigs(sizeBefore,1);
Assert.assertEquals(flowConfigs.size(), 1);
-
Assert.assertEquals(((FlowConfig)(flowConfigs.toArray()[0])).getId().getFlowName(),
"testFlow");
+ // TODO: remove these assertions which are flaky because the flows above
all show up with the same modification time
+//
Assert.assertEquals(((FlowConfig)(flowConfigs.toArray()[0])).getId().getFlowName(),
TEST_FLOW_ID.getFlowName());
Review Comment:
if they're not totally ordered, do you want to check that the name is
one/any of the four possibilities you added?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]