Repository: hive Updated Branches: refs/heads/branch-3 0084f9288 -> 2bf4fef22
HIVE-19639 : a transactional Hive table cannot be imported as an external table (Sergey Shelukhin, reviewed by Thejas M Nair) Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2bf4fef2 Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2bf4fef2 Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2bf4fef2 Branch: refs/heads/branch-3 Commit: 2bf4fef22babce90ade5670ff5ce49143be37457 Parents: 0084f92 Author: sergey <ser...@apache.org> Authored: Tue May 22 12:08:50 2018 -0700 Committer: sergey <ser...@apache.org> Committed: Tue May 22 14:35:46 2018 -0700 ---------------------------------------------------------------------- .../org/apache/hadoop/hive/ql/io/AcidUtils.java | 5 ++ .../hive/ql/parse/ImportSemanticAnalyzer.java | 31 +++++---- ql/src/test/queries/clientpositive/mm_exim.q | 8 +++ .../results/clientpositive/llap/mm_exim.q.out | 70 ++++++++++++++++++++ 4 files changed, 102 insertions(+), 12 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/2bf4fef2/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java index 183515a..4b9a8a1 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/io/AcidUtils.java @@ -1932,4 +1932,9 @@ public class AcidUtils { } return writeId; } + + public static void setNonTransactional(Map<String, String> tblProps) { + tblProps.put(hive_metastoreConstants.TABLE_IS_TRANSACTIONAL, "false"); + tblProps.remove(hive_metastoreConstants.TABLE_TRANSACTIONAL_PROPERTIES); + } } http://git-wip-us.apache.org/repos/asf/hive/blob/2bf4fef2/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java index eb6708b..1a3cef9 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/parse/ImportSemanticAnalyzer.java @@ -255,16 +255,13 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer { } if (isExternalSet) { - if (AcidUtils.isInsertOnlyTable(tblDesc.getTblProps())) { - throw new SemanticException("Cannot import an MM table as external"); - } tblDesc.setExternal(isExternalSet); // This condition-check could have been avoided, but to honour the old // default of not calling if it wasn't set, we retain that behaviour. // TODO:cleanup after verification that the outer if isn't really needed here } - if (isLocationSet){ + if (isLocationSet) { tblDesc.setLocation(parsedLocation); x.getInputs().add(toReadEntity(new Path(parsedLocation), x.getConf())); } @@ -320,11 +317,16 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer { boolean tableExists = false; if (table != null) { - checkTable(table, tblDesc,replicationSpec, x.getConf()); + checkTable(table, tblDesc, replicationSpec, x.getConf()); x.getLOG().debug("table " + tblDesc.getTableName() + " exists: metadata checked"); tableExists = true; } + if (!tableExists && isExternalSet) { + // If the user is explicitly importing a new external table, clear txn flags from the spec. + AcidUtils.setNonTransactional(tblDesc.getTblProps()); + } + Long writeId = 0L; // Initialize with 0 for non-ACID and non-MM tables. int stmtId = 0; if (!replicationSpec.isInReplicationScope() @@ -862,7 +864,7 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer { x.getLOG().debug("table " + tblDesc.getTableName() + " does not exist"); Task<?> t = createTableTask(tblDesc, x); - table = createNewTableMetadataObject(tblDesc); + table = createNewTableMetadataObject(tblDesc, false); Database parentDb = x.getHive().getDatabase(tblDesc.getDatabaseName()); @@ -898,14 +900,19 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer { } } - private static Table createNewTableMetadataObject(ImportTableDesc tblDesc) + private static Table createNewTableMetadataObject(ImportTableDesc tblDesc, boolean isRepl) throws SemanticException { Table newTable = new Table(tblDesc.getDatabaseName(), tblDesc.getTableName()); //so that we know the type of table we are creating: acid/MM to match what was exported newTable.setParameters(tblDesc.getTblProps()); if(tblDesc.isExternal() && AcidUtils.isTransactionalTable(newTable)) { - throw new SemanticException("External tables may not be transactional: " + - Warehouse.getQualifiedName(tblDesc.getDatabaseName(), tblDesc.getTableName())); + if (isRepl) { + throw new SemanticException("External tables may not be transactional: " + + Warehouse.getQualifiedName(tblDesc.getDatabaseName(), tblDesc.getTableName())); + } else { + throw new AssertionError("Internal error: transactional properties not set properly" + + tblDesc.getTblProps()); + } } return newTable; } @@ -1000,7 +1007,7 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer { } Task t = createTableTask(tblDesc, x); - table = createNewTableMetadataObject(tblDesc); + table = createNewTableMetadataObject(tblDesc, true); if (!replicationSpec.isMetadataOnly()) { if (isPartitioned(tblDesc)) { @@ -1033,7 +1040,7 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer { // create the dummy table object for adding repl tasks. boolean isOldTableValid = true; if (table.isPartitioned() != isPartitioned(tblDesc)) { - table = createNewTableMetadataObject(tblDesc); + table = createNewTableMetadataObject(tblDesc, true); isOldTableValid = false; } @@ -1051,7 +1058,7 @@ public class ImportSemanticAnalyzer extends BaseSemanticAnalyzer { ptn = x.getHive().getPartition(table, partSpec, false); } catch (HiveException ex) { ptn = null; - table = createNewTableMetadataObject(tblDesc); + table = createNewTableMetadataObject(tblDesc, true); isOldTableValid = false; } } http://git-wip-us.apache.org/repos/asf/hive/blob/2bf4fef2/ql/src/test/queries/clientpositive/mm_exim.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/mm_exim.q b/ql/src/test/queries/clientpositive/mm_exim.q index d084125..5d82a13 100644 --- a/ql/src/test/queries/clientpositive/mm_exim.q +++ b/ql/src/test/queries/clientpositive/mm_exim.q @@ -95,4 +95,12 @@ import table import7_mm from 'ql/test/data/exports/intermmediate_part'; select * from import7_mm order by key, p; drop table import7_mm; +-- import MM as external + +drop table import8_mm; +import external table import8_mm from 'ql/test/data/exports/intermmediate_nonpart'; +desc formatted import8_mm; +select * from import8_mm order by key, p; +drop table import8_mm; + set hive.exim.test.mode=false; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hive/blob/2bf4fef2/ql/src/test/results/clientpositive/llap/mm_exim.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/llap/mm_exim.q.out b/ql/src/test/results/clientpositive/llap/mm_exim.q.out index 1f40754..8ba815c 100644 --- a/ql/src/test/results/clientpositive/llap/mm_exim.q.out +++ b/ql/src/test/results/clientpositive/llap/mm_exim.q.out @@ -554,3 +554,73 @@ POSTHOOK: query: drop table import7_mm POSTHOOK: type: DROPTABLE POSTHOOK: Input: default@import7_mm POSTHOOK: Output: default@import7_mm +PREHOOK: query: drop table import8_mm +PREHOOK: type: DROPTABLE +POSTHOOK: query: drop table import8_mm +POSTHOOK: type: DROPTABLE +PREHOOK: query: import external table import8_mm from 'ql/test/data/exports/intermmediate_nonpart' +PREHOOK: type: IMPORT +#### A masked pattern was here #### +PREHOOK: Output: database:default +POSTHOOK: query: import external table import8_mm from 'ql/test/data/exports/intermmediate_nonpart' +POSTHOOK: type: IMPORT +#### A masked pattern was here #### +POSTHOOK: Output: database:default +POSTHOOK: Output: default@import8_mm +PREHOOK: query: desc formatted import8_mm +PREHOOK: type: DESCTABLE +PREHOOK: Input: default@import8_mm +POSTHOOK: query: desc formatted import8_mm +POSTHOOK: type: DESCTABLE +POSTHOOK: Input: default@import8_mm +# col_name data_type comment +key int +p int + +# Detailed Table Information +Database: default +#### A masked pattern was here #### +Retention: 0 +#### A masked pattern was here #### +Table Type: EXTERNAL_TABLE +Table Parameters: + EXTERNAL TRUE + bucketing_version 2 + numFiles 3 + numRows 6 + rawDataSize 37 + totalSize 43 +#### A masked pattern was here #### + +# Storage Information +SerDe Library: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe +InputFormat: org.apache.hadoop.mapred.TextInputFormat +OutputFormat: org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat +Compressed: No +Num Buckets: -1 +Bucket Columns: [] +Sort Columns: [] +Storage Desc Params: + serialization.format 1 +PREHOOK: query: select * from import8_mm order by key, p +PREHOOK: type: QUERY +PREHOOK: Input: default@import8_mm +#### A masked pattern was here #### +POSTHOOK: query: select * from import8_mm order by key, p +POSTHOOK: type: QUERY +POSTHOOK: Input: default@import8_mm +#### A masked pattern was here #### +0 456 +10 456 +97 455 +98 455 +100 457 +103 457 +PREHOOK: query: drop table import8_mm +PREHOOK: type: DROPTABLE +PREHOOK: Input: default@import8_mm +PREHOOK: Output: default@import8_mm +POSTHOOK: query: drop table import8_mm +POSTHOOK: type: DROPTABLE +POSTHOOK: Input: default@import8_mm +POSTHOOK: Output: default@import8_mm
