sv2000 commented on a change in pull request #2633: GOBBLIN-759: Added feature 
to support DistCP to copy files that were …
URL: https://github.com/apache/incubator-gobblin/pull/2633#discussion_r324484386
 
 

 ##########
 File path: 
gobblin-data-management/src/test/java/org/apache/gobblin/data/management/copy/TimestampBasedCopyableDatasetTest.java
 ##########
 @@ -91,12 +110,82 @@ public void testConfigOptions() {
         TimeBasedCopyPolicyForTest.class.getName());
   }
 
+  @Test
+  public void testCopyWithFilter() throws IOException {
+
+    /** source setup **/
+    Path srcRoot = new Path(this.testTempPath, "src/slt/eqp/daily");
+
+    if (this.localFs.exists(srcRoot)) {
+      this.localFs.delete(srcRoot, true);
+    }
+
+    List<DateTime> dateTimeList = Lists.newArrayList();
+    IntStream.range(0, 4)
+        .forEach(
+            i -> dateTimeList.add(new 
DateTime(DateTimeZone.forID(ConfigurationKeys.PST_TIMEZONE_NAME)).minusDays(i)));
+
+    String datePattern = "yyyy/MM/dd";
+    DateTimeFormatter formatter = DateTimeFormat.forPattern(datePattern);
+
+    for (DateTime dt : dateTimeList) {
+      String srcVersionPathStr = formatter.print(dt);
+      Path srcVersionPath = new Path(srcRoot, srcVersionPathStr);
+      this.localFs.mkdirs(srcVersionPath);
+
+      Path srcfile = new Path(srcVersionPath, "file1.avro");
+      this.localFs.create(srcfile);
+    }
+
+    /** destination setup **/
+    Path destRoot = new Path(this.testTempPath, "dest/slt/eqp");
 
 Review comment:
   Change "dest/slt/eqp" pathname to some other dummy path.

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
us...@infra.apache.org


With regards,
Apache Git Services

Reply via email to