This is an automated email from the ASF dual-hosted git repository.
zihanli58 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/gobblin.git
The following commit(s) were added to refs/heads/master by this push:
new cb75ecc GOBBLIN-1579 Fail job on hive existing target table location
mismatch (#3433)
cb75ecc is described below
commit cb75ecc25e00b1920b996b8b564a0068f15a532b
Author: vgnanasekaran <[email protected]>
AuthorDate: Wed Jan 19 11:46:00 2022 -0800
GOBBLIN-1579 Fail job on hive existing target table location mismatch
(#3433)
Co-authored-by: Gnanasekaran <[email protected]>
---
.../copy/hive/UnpartitionedTableFileSet.java | 8 ++--
.../copy/hive/UnpartitionedTableFileSetTest.java | 54 ++++++++++++++++++++++
2 files changed, 57 insertions(+), 5 deletions(-)
diff --git
a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/UnpartitionedTableFileSet.java
b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/UnpartitionedTableFileSet.java
index 89ebe7e..86cc490 100644
---
a/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/UnpartitionedTableFileSet.java
+++
b/gobblin-data-management/src/main/java/org/apache/gobblin/data/management/copy/hive/UnpartitionedTableFileSet.java
@@ -81,12 +81,10 @@ public class UnpartitionedTableFileSet extends HiveFileSet {
existingTargetTable = Optional.absent();
break ;
default:
- log.error("Source and target table are not compatible. Aborting
copy of table " + this.helper.getTargetTable(),
- new
HiveTableLocationNotMatchException(this.helper.getTargetTable().getDataLocation(),
- existingTargetTable.get().getDataLocation()));
+ log.error("Source and target table are not compatible. Aborting
copy of table " + this.helper.getTargetTable());
multiTimer.close();
-
- return Lists.newArrayList();
+ throw new
HiveTableLocationNotMatchException(this.helper.getTargetTable().getDataLocation(),
+ existingTargetTable.get().getDataLocation());
}
}
}
diff --git
a/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/copy/hive/UnpartitionedTableFileSetTest.java
b/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/copy/hive/UnpartitionedTableFileSetTest.java
new file mode 100644
index 0000000..b3102e4
--- /dev/null
+++
b/gobblin-data-management/src/test/java/org/apache/gobblin/data/management/copy/hive/UnpartitionedTableFileSetTest.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.gobblin.data.management.copy.hive;
+
+import com.google.common.base.Optional;
+import org.apache.gobblin.data.management.copy.CopyEntity;
+import org.apache.gobblin.metrics.MetricContext;
+import org.apache.gobblin.metrics.event.EventSubmitter;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hive.ql.metadata.Table;
+import org.mockito.Mockito;
+import org.testng.annotations.Test;
+
+import java.util.List;
+
+public class UnpartitionedTableFileSetTest {
+
+ @Test(expectedExceptions = { HiveTableLocationNotMatchException.class })
+ public void testHiveTableLocationNotMatchException() throws Exception {
+ Path testPath = new Path("/testPath/db/table");
+ Path existingTablePath = new Path("/existing/testPath/db/table");
+ Table table = new Table("testDb","table1");
+ table.setDataLocation(testPath);
+ Table existingTargetTable = new Table("testDb","table1");
+ existingTargetTable.setDataLocation(existingTablePath);
+ HiveDataset hiveDataset = Mockito.mock(HiveDataset.class);
+ Mockito.when(hiveDataset.getTable()).thenReturn(table);
+ HiveCopyEntityHelper helper = Mockito.mock(HiveCopyEntityHelper.class);
+ Mockito.when(helper.getDataset()).thenReturn(hiveDataset);
+
Mockito.when(helper.getExistingTargetTable()).thenReturn(Optional.of(existingTargetTable));
+ Mockito.when(helper.getTargetTable()).thenReturn(table);
+
Mockito.when(helper.getExistingEntityPolicy()).thenReturn(HiveCopyEntityHelper.ExistingEntityPolicy.ABORT);
+ MetricContext metricContext =
MetricContext.builder("testUnpartitionedTableFileSet").build();
+ EventSubmitter eventSubmitter = new
EventSubmitter.Builder(metricContext,"loc.nomatch.exp").build();
+ Mockito.when(helper.getEventSubmitter()).thenReturn(eventSubmitter);
+ UnpartitionedTableFileSet upts = new
UnpartitionedTableFileSet("testLocationMatch",hiveDataset,helper);
+ List<CopyEntity> copyEntities =
(List<CopyEntity>)upts.generateCopyEntities();
+ }
+}
