HIVE-11834: Lineage doesn't work with dynamic partitioning query (Jimmy, reviewed by Szehon)
Project: http://git-wip-us.apache.org/repos/asf/hive/repo Commit: http://git-wip-us.apache.org/repos/asf/hive/commit/2278548e Tree: http://git-wip-us.apache.org/repos/asf/hive/tree/2278548e Diff: http://git-wip-us.apache.org/repos/asf/hive/diff/2278548e Branch: refs/heads/beeline-cli Commit: 2278548e41bf7b51e7b604fe9c91905b1ca198f1 Parents: 3cf7bd9 Author: Jimmy Xiang <[email protected]> Authored: Wed Sep 16 08:09:41 2015 -0700 Committer: Jimmy Xiang <[email protected]> Committed: Fri Sep 18 07:19:23 2015 -0700 ---------------------------------------------------------------------- .../hadoop/hive/ql/hooks/LineageLogger.java | 93 +++++++++++--------- .../hive/ql/optimizer/lineage/LineageCtx.java | 8 +- ql/src/test/queries/clientpositive/lineage3.q | 26 ++++++ .../test/results/clientpositive/lineage3.q.out | 68 +++++++++++++- 4 files changed, 153 insertions(+), 42 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hive/blob/2278548e/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageLogger.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageLogger.java b/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageLogger.java index f615d81..9988c79 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageLogger.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/hooks/LineageLogger.java @@ -215,8 +215,7 @@ public class LineageLogger implements ExecuteWithHookContext { private List<Edge> getEdges(QueryPlan plan, Index index) { LinkedHashMap<String, ObjectPair<SelectOperator, org.apache.hadoop.hive.ql.metadata.Table>> finalSelOps = index.getFinalSelectOps(); - Set<Vertex> allTargets = new LinkedHashSet<Vertex>(); - Map<String, Vertex> allSources = new LinkedHashMap<String, Vertex>(); + Map<String, Vertex> vertexCache = new LinkedHashMap<String, Vertex>(); List<Edge> edges = new ArrayList<Edge>(); for (ObjectPair<SelectOperator, org.apache.hadoop.hive.ql.metadata.Table> pair: finalSelOps.values()) { @@ -244,41 +243,46 @@ public class LineageLogger implements ExecuteWithHookContext { } } } - int fields = fieldSchemas.size(); Map<ColumnInfo, Dependency> colMap = index.getDependencies(finalSelOp); List<Dependency> dependencies = colMap != null ? Lists.newArrayList(colMap.values()) : null; + int fields = fieldSchemas.size(); + if (t != null && colMap != null && fields < colMap.size()) { + // Dynamic partition keys should be added to field schemas. + List<FieldSchema> partitionKeys = t.getPartitionKeys(); + int dynamicKeyCount = colMap.size() - fields; + int keyOffset = partitionKeys.size() - dynamicKeyCount; + if (keyOffset >= 0) { + fields += dynamicKeyCount; + for (int i = 0; i < dynamicKeyCount; i++) { + FieldSchema field = partitionKeys.get(keyOffset + i); + fieldSchemas.add(field); + if (colNames != null) { + colNames.add(field.getName()); + } + } + } + } if (dependencies == null || dependencies.size() != fields) { log("Result schema has " + fields + " fields, but we don't get as many dependencies"); } else { // Go through each target column, generate the lineage edges. + Set<Vertex> targets = new LinkedHashSet<Vertex>(); for (int i = 0; i < fields; i++) { - Vertex target = new Vertex( - getTargetFieldName(i, destTableName, colNames, fieldSchemas)); - allTargets.add(target); + Vertex target = getOrCreateVertex(vertexCache, + getTargetFieldName(i, destTableName, colNames, fieldSchemas), + Vertex.Type.COLUMN); + targets.add(target); Dependency dep = dependencies.get(i); - String expr = dep.getExpr(); - Set<Vertex> sources = createSourceVertices(allSources, dep.getBaseCols()); - Edge edge = findSimilarEdgeBySources(edges, sources, expr, Edge.Type.PROJECTION); - if (edge == null) { - Set<Vertex> targets = new LinkedHashSet<Vertex>(); - targets.add(target); - edges.add(new Edge(sources, targets, expr, Edge.Type.PROJECTION)); - } else { - edge.targets.add(target); - } + addEdge(vertexCache, edges, dep.getBaseCols(), target, + dep.getExpr(), Edge.Type.PROJECTION); } Set<Predicate> conds = index.getPredicates(finalSelOp); if (conds != null && !conds.isEmpty()) { for (Predicate cond: conds) { - String expr = cond.getExpr(); - Set<Vertex> sources = createSourceVertices(allSources, cond.getBaseCols()); - Edge edge = findSimilarEdgeByTargets(edges, allTargets, expr, Edge.Type.PREDICATE); - if (edge == null) { - edges.add(new Edge(sources, allTargets, expr, Edge.Type.PREDICATE)); - } else { - edge.sources.addAll(sources); - } + addEdge(vertexCache, edges, cond.getBaseCols(), + new LinkedHashSet<Vertex>(targets), cond.getExpr(), + Edge.Type.PREDICATE); } } } @@ -286,12 +290,35 @@ public class LineageLogger implements ExecuteWithHookContext { return edges; } + private void addEdge(Map<String, Vertex> vertexCache, List<Edge> edges, + Set<BaseColumnInfo> srcCols, Vertex target, String expr, Edge.Type type) { + Set<Vertex> targets = new LinkedHashSet<Vertex>(); + targets.add(target); + addEdge(vertexCache, edges, srcCols, targets, expr, type); + } + + /** + * Find an edge from all edges that has the same source vertices. + * If found, add the more targets to this edge's target vertex list. + * Otherwise, create a new edge and add to edge list. + */ + private void addEdge(Map<String, Vertex> vertexCache, List<Edge> edges, + Set<BaseColumnInfo> srcCols, Set<Vertex> targets, String expr, Edge.Type type) { + Set<Vertex> sources = createSourceVertices(vertexCache, srcCols); + Edge edge = findSimilarEdgeBySources(edges, sources, expr, type); + if (edge == null) { + edges.add(new Edge(sources, targets, expr, type)); + } else { + edge.targets.addAll(targets); + } + } + /** * Convert a list of columns to a set of vertices. * Use cached vertices if possible. */ private Set<Vertex> createSourceVertices( - Map<String, Vertex> srcVertexCache, Collection<BaseColumnInfo> baseCols) { + Map<String, Vertex> vertexCache, Collection<BaseColumnInfo> baseCols) { Set<Vertex> sources = new LinkedHashSet<Vertex>(); if (baseCols != null && !baseCols.isEmpty()) { for(BaseColumnInfo col: baseCols) { @@ -308,7 +335,7 @@ public class LineageLogger implements ExecuteWithHookContext { type = Vertex.Type.COLUMN; label = tableName + "." + fieldSchema.getName(); } - sources.add(getOrCreateVertex(srcVertexCache, label, type)); + sources.add(getOrCreateVertex(vertexCache, label, type)); } } return sources; @@ -342,20 +369,6 @@ public class LineageLogger implements ExecuteWithHookContext { } /** - * Find an edge that has the same type, expression, and targets. - */ - private Edge findSimilarEdgeByTargets( - List<Edge> edges, Set<Vertex> targets, String expr, Edge.Type type) { - for (Edge edge: edges) { - if (edge.type == type && StringUtils.equals(edge.expr, expr) - && SetUtils.isEqualSet(edge.targets, targets)) { - return edge; - } - } - return null; - } - - /** * Generate normalized name for a given target column. */ private String getTargetFieldName(int fieldIndex, http://git-wip-us.apache.org/repos/asf/hive/blob/2278548e/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/LineageCtx.java ---------------------------------------------------------------------- diff --git a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/LineageCtx.java b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/LineageCtx.java index c33d775..2d8b9e3 100644 --- a/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/LineageCtx.java +++ b/ql/src/java/org/apache/hadoop/hive/ql/optimizer/lineage/LineageCtx.java @@ -18,13 +18,13 @@ package org.apache.hadoop.hive.ql.optimizer.lineage; -import java.util.ArrayList; import java.util.HashMap; import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.Map; import java.util.Set; +import org.apache.commons.lang.StringUtils; import org.apache.hadoop.hive.common.ObjectPair; import org.apache.hadoop.hive.ql.exec.ColumnInfo; import org.apache.hadoop.hive.ql.exec.FileSinkOperator; @@ -171,6 +171,12 @@ public class LineageCtx implements NodeProcessorCtx { conds = new LinkedHashSet<Predicate>(); condMap.put(op, conds); } + for (Predicate p: conds) { + if (StringUtils.equals(cond.getExpr(), p.getExpr())) { + p.getBaseCols().addAll(cond.getBaseCols()); + return; + } + } conds.add(cond); } http://git-wip-us.apache.org/repos/asf/hive/blob/2278548e/ql/src/test/queries/clientpositive/lineage3.q ---------------------------------------------------------------------- diff --git a/ql/src/test/queries/clientpositive/lineage3.q b/ql/src/test/queries/clientpositive/lineage3.q index c24ff7d..70d4e57 100644 --- a/ql/src/test/queries/clientpositive/lineage3.q +++ b/ql/src/test/queries/clientpositive/lineage3.q @@ -176,3 +176,29 @@ alter view dest_v3 as select * from dest_v3 limit 2; +drop table if exists src_dp; +create table src_dp (first string, word string, year int, month int, day int); +drop table if exists dest_dp1; +create table dest_dp1 (first string, word string) partitioned by (year int); +drop table if exists dest_dp2; +create table dest_dp2 (first string, word string) partitioned by (y int, m int); +drop table if exists dest_dp3; +create table dest_dp3 (first string, word string) partitioned by (y int, m int, d int); + +set hive.exec.dynamic.partition.mode=nonstrict; + +insert into dest_dp1 partition (year) select first, word, year from src_dp; +insert into dest_dp2 partition (y, m) select first, word, year, month from src_dp; +insert into dest_dp2 partition (y=0, m) select first, word, month from src_dp where year=0; +insert into dest_dp3 partition (y=0, m, d) select first, word, month m, day d from src_dp where year=0; + +drop table if exists src_dp1; +create table src_dp1 (f string, w string, m int); + +from src_dp, src_dp1 +insert into dest_dp1 partition (year) select first, word, year +insert into dest_dp2 partition (y, m) select first, word, year, month +insert into dest_dp3 partition (y=2, m, d) select first, word, month m, day d where year=2 +insert into dest_dp2 partition (y=1, m) select f, w, m +insert into dest_dp1 partition (year=0) select f, w; + http://git-wip-us.apache.org/repos/asf/hive/blob/2278548e/ql/src/test/results/clientpositive/lineage3.q.out ---------------------------------------------------------------------- diff --git a/ql/src/test/results/clientpositive/lineage3.q.out b/ql/src/test/results/clientpositive/lineage3.q.out index 708abee..ad965c8 100644 --- a/ql/src/test/results/clientpositive/lineage3.q.out +++ b/ql/src/test/results/clientpositive/lineage3.q.out @@ -25,7 +25,7 @@ PREHOOK: type: QUERY PREHOOK: Input: default@alltypesorc PREHOOK: Output: default@d1 PREHOOK: Output: default@d2 -{"version":"1.0","engine":"mr","hash":"8703e4091ebd4c96afd3cac83e3a2957","queryText":"from (select a.ctinyint x, b.cstring1 y\nfrom alltypesorc a join alltypesorc b on a.cint = b.cbigint) t\ninsert into table d1 select x where y is null\ninsert into table d2 select y where x > 0","edges":[{"sources":[2],"targets":[0],"expression":"UDFToInteger(x)","edgeType":"PROJECTION"},{"sources":[3,4],"targets":[0,1],"expression":"(UDFToLong(a.cint) = b.cbigint)","edgeType":"PREDICATE"},{"sources":[5],"targets":[0,1],"expression":"t.y is null","edgeType":"PREDICATE"},{"sources":[5],"targets":[1],"expression":"CAST( y AS varchar(128))","edgeType":"PROJECTION"},{"sources":[2],"targets":[0,1],"expression":"(t.x > 0)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.d1.a"},{"id":1,"vertexType":"COLUMN","vertexId":"default.d2.b"},{"id":2,"vertexType":"COLUMN","vertexId":"default.alltypesorc.ctinyint"},{"id":3,"vertexType":"COLUMN","vertexId":"default.alltypesorc. cint"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cbigint"},{"id":5,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cstring1"}]} +{"version":"1.0","engine":"mr","hash":"8703e4091ebd4c96afd3cac83e3a2957","queryText":"from (select a.ctinyint x, b.cstring1 y\nfrom alltypesorc a join alltypesorc b on a.cint = b.cbigint) t\ninsert into table d1 select x where y is null\ninsert into table d2 select y where x > 0","edges":[{"sources":[2],"targets":[0],"expression":"UDFToInteger(x)","edgeType":"PROJECTION"},{"sources":[3,4],"targets":[0,1],"expression":"(UDFToLong(a.cint) = b.cbigint)","edgeType":"PREDICATE"},{"sources":[5],"targets":[0],"expression":"t.y is null","edgeType":"PREDICATE"},{"sources":[5],"targets":[1],"expression":"CAST( y AS varchar(128))","edgeType":"PROJECTION"},{"sources":[2],"targets":[1],"expression":"(t.x > 0)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.d1.a"},{"id":1,"vertexType":"COLUMN","vertexId":"default.d2.b"},{"id":2,"vertexType":"COLUMN","vertexId":"default.alltypesorc.ctinyint"},{"id":3,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cint "},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cbigint"},{"id":5,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cstring1"}]} PREHOOK: query: drop table if exists t PREHOOK: type: DROPTABLE PREHOOK: query: create table t as @@ -320,3 +320,69 @@ PREHOOK: Input: default@dest_v3 {"version":"1.0","engine":"mr","hash":"40bccc0722002f798d0548b59e369e83","queryText":"select * from dest_v3 limit 2","edges":[{"sources":[3,4,5,6,7],"targets":[0],"expression":"(tok_function sum (. (tok_table_or_col $hdt$_0) ctinyint) (tok_windowspec (tok_partitioningspec (tok_distributeby (. (tok_table_or_col $hdt$_0) csmallint)) (tok_orderby (tok_tabsortcolnameasc (. (tok_table_or_col $hdt$_0) csmallint)))) (tok_windowvalues (preceding 2147483647) current)))","edgeType":"PROJECTION"},{"sources":[6],"targets":[1],"expression":"count(default.alltypesorc.cstring1)","edgeType":"PROJECTION"},{"sources":[5],"targets":[2],"edgeType":"PROJECTION"},{"sources":[8],"targets":[0,1,2],"expression":"(a.cboolean2 = true)","edgeType":"PREDICATE"},{"sources":[7],"targets":[0,1,2],"expression":"(a.cint = a.cint)","edgeType":"PREDICATE"},{"sources":[9],"targets":[0,1,2],"expression":"(a.cfloat > 0.0)","edgeType":"PREDICATE"},{"sources":[7],"targets":[0,1,2],"expression":"(count(default.alltypesorc.c int) > 10)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"dest_v3.a"},{"id":1,"vertexType":"COLUMN","vertexId":"dest_v3.x"},{"id":2,"vertexType":"COLUMN","vertexId":"dest_v3.cboolean1"},{"id":3,"vertexType":"COLUMN","vertexId":"default.alltypesorc.ctinyint"},{"id":4,"vertexType":"COLUMN","vertexId":"default.alltypesorc.csmallint"},{"id":5,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cboolean1"},{"id":6,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cstring1"},{"id":7,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cint"},{"id":8,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cboolean2"},{"id":9,"vertexType":"COLUMN","vertexId":"default.alltypesorc.cfloat"}]} 38 216 false 38 229 true +PREHOOK: query: drop table if exists src_dp +PREHOOK: type: DROPTABLE +PREHOOK: query: create table src_dp (first string, word string, year int, month int, day int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@src_dp +PREHOOK: query: drop table if exists dest_dp1 +PREHOOK: type: DROPTABLE +PREHOOK: query: create table dest_dp1 (first string, word string) partitioned by (year int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dest_dp1 +PREHOOK: query: drop table if exists dest_dp2 +PREHOOK: type: DROPTABLE +PREHOOK: query: create table dest_dp2 (first string, word string) partitioned by (y int, m int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dest_dp2 +PREHOOK: query: drop table if exists dest_dp3 +PREHOOK: type: DROPTABLE +PREHOOK: query: create table dest_dp3 (first string, word string) partitioned by (y int, m int, d int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@dest_dp3 +PREHOOK: query: insert into dest_dp1 partition (year) select first, word, year from src_dp +PREHOOK: type: QUERY +PREHOOK: Input: default@src_dp +PREHOOK: Output: default@dest_dp1 +{"version":"1.0","engine":"mr","hash":"b2d38401a3281e74a003d9650df97060","queryText":"insert into dest_dp1 partition (year) select first, word, year from src_dp","edges":[{"sources":[3],"targets":[0],"edgeType":"PROJECTION"},{"sources":[4],"targets":[1],"edgeType":"PROJECTION"},{"sources":[5],"targets":[2],"edgeType":"PROJECTION"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.dest_dp1.first"},{"id":1,"vertexType":"COLUMN","vertexId":"default.dest_dp1.word"},{"id":2,"vertexType":"COLUMN","vertexId":"default.dest_dp1.year"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src_dp.first"},{"id":4,"vertexType":"COLUMN","vertexId":"default.src_dp.word"},{"id":5,"vertexType":"COLUMN","vertexId":"default.src_dp.year"}]} +PREHOOK: query: insert into dest_dp2 partition (y, m) select first, word, year, month from src_dp +PREHOOK: type: QUERY +PREHOOK: Input: default@src_dp +PREHOOK: Output: default@dest_dp2 +{"version":"1.0","engine":"mr","hash":"237302d8ffd62b5b71d9544b22de7770","queryText":"insert into dest_dp2 partition (y, m) select first, word, year, month from src_dp","edges":[{"sources":[4],"targets":[0],"edgeType":"PROJECTION"},{"sources":[5],"targets":[1],"edgeType":"PROJECTION"},{"sources":[6],"targets":[2],"edgeType":"PROJECTION"},{"sources":[7],"targets":[3],"edgeType":"PROJECTION"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.dest_dp2.first"},{"id":1,"vertexType":"COLUMN","vertexId":"default.dest_dp2.word"},{"id":2,"vertexType":"COLUMN","vertexId":"default.dest_dp2.y"},{"id":3,"vertexType":"COLUMN","vertexId":"default.dest_dp2.m"},{"id":4,"vertexType":"COLUMN","vertexId":"default.src_dp.first"},{"id":5,"vertexType":"COLUMN","vertexId":"default.src_dp.word"},{"id":6,"vertexType":"COLUMN","vertexId":"default.src_dp.year"},{"id":7,"vertexType":"COLUMN","vertexId":"default.src_dp.month"}]} +PREHOOK: query: insert into dest_dp2 partition (y=0, m) select first, word, month from src_dp where year=0 +PREHOOK: type: QUERY +PREHOOK: Input: default@src_dp +PREHOOK: Output: default@dest_dp2@y=0 +{"version":"1.0","engine":"mr","hash":"63e990b47e7ab4eb6f2ea09dfb7453ff","queryText":"insert into dest_dp2 partition (y=0, m) select first, word, month from src_dp where year=0","edges":[{"sources":[3],"targets":[0],"edgeType":"PROJECTION"},{"sources":[4],"targets":[1],"edgeType":"PROJECTION"},{"sources":[5],"targets":[2],"edgeType":"PROJECTION"},{"sources":[6],"targets":[0,1,2],"expression":"(src_dp.year = 0)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.dest_dp2.first"},{"id":1,"vertexType":"COLUMN","vertexId":"default.dest_dp2.word"},{"id":2,"vertexType":"COLUMN","vertexId":"default.dest_dp2.m"},{"id":3,"vertexType":"COLUMN","vertexId":"default.src_dp.first"},{"id":4,"vertexType":"COLUMN","vertexId":"default.src_dp.word"},{"id":5,"vertexType":"COLUMN","vertexId":"default.src_dp.month"},{"id":6,"vertexType":"COLUMN","vertexId":"default.src_dp.year"}]} +PREHOOK: query: insert into dest_dp3 partition (y=0, m, d) select first, word, month m, day d from src_dp where year=0 +PREHOOK: type: QUERY +PREHOOK: Input: default@src_dp +PREHOOK: Output: default@dest_dp3@y=0 +{"version":"1.0","engine":"mr","hash":"6bf71a9d02c0612c63b6f40b15c1e8b3","queryText":"insert into dest_dp3 partition (y=0, m, d) select first, word, month m, day d from src_dp where year=0","edges":[{"sources":[4],"targets":[0],"edgeType":"PROJECTION"},{"sources":[5],"targets":[1],"edgeType":"PROJECTION"},{"sources":[6],"targets":[2],"edgeType":"PROJECTION"},{"sources":[7],"targets":[3],"edgeType":"PROJECTION"},{"sources":[8],"targets":[0,1,2,3],"expression":"(src_dp.year = 0)","edgeType":"PREDICATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.dest_dp3.first"},{"id":1,"vertexType":"COLUMN","vertexId":"default.dest_dp3.word"},{"id":2,"vertexType":"COLUMN","vertexId":"default.dest_dp3.m"},{"id":3,"vertexType":"COLUMN","vertexId":"default.dest_dp3.d"},{"id":4,"vertexType":"COLUMN","vertexId":"default.src_dp.first"},{"id":5,"vertexType":"COLUMN","vertexId":"default.src_dp.word"},{"id":6,"vertexType":"COLUMN","vertexId":"default.src_dp.month"},{"id":7,"vertexType":"CO LUMN","vertexId":"default.src_dp.day"},{"id":8,"vertexType":"COLUMN","vertexId":"default.src_dp.year"}]} +PREHOOK: query: drop table if exists src_dp1 +PREHOOK: type: DROPTABLE +PREHOOK: query: create table src_dp1 (f string, w string, m int) +PREHOOK: type: CREATETABLE +PREHOOK: Output: database:default +PREHOOK: Output: default@src_dp1 +Warning: Shuffle Join JOIN[4][tables = [src_dp, src_dp1]] in Stage 'Stage-5:MAPRED' is a cross product +PREHOOK: query: from src_dp, src_dp1 +insert into dest_dp1 partition (year) select first, word, year +insert into dest_dp2 partition (y, m) select first, word, year, month +insert into dest_dp3 partition (y=2, m, d) select first, word, month m, day d where year=2 +insert into dest_dp2 partition (y=1, m) select f, w, m +insert into dest_dp1 partition (year=0) select f, w +PREHOOK: type: QUERY +PREHOOK: Input: default@src_dp +PREHOOK: Input: default@src_dp1 +PREHOOK: Output: default@dest_dp1 +PREHOOK: Output: default@dest_dp1@year=0 +PREHOOK: Output: default@dest_dp2 +PREHOOK: Output: default@dest_dp2@y=1 +PREHOOK: Output: default@dest_dp3@y=2 +{"version":"1.0","engine":"mr","hash":"44f16edbf35cfeaf3d4f7b0113a69b74","queryText":"from src_dp, src_dp1\ninsert into dest_dp1 partition (year) select first, word, year\ninsert into dest_dp2 partition (y, m) select first, word, year, month\ninsert into dest_dp3 partition (y=2, m, d) select first, word, month m, day d where year=2\ninsert into dest_dp2 partition (y=1, m) select f, w, m\ninsert into dest_dp1 partition (year=0) select f, w","edges":[{"sources":[11],"targets":[0,1,2],"edgeType":"PROJECTION"},{"sources":[12],"targets":[3,4,5],"edgeType":"PROJECTION"},{"sources":[13],"targets":[6,7],"edgeType":"PROJECTION"},{"sources":[14],"targets":[8,9],"edgeType":"PROJECTION"},{"sources":[15],"targets":[1,0],"edgeType":"PROJECTION"},{"sources":[16],"targets":[4,3],"edgeType":"PROJECTION"},{"sources":[17],"targets":[8],"edgeType":"PROJECTION"},{"sources":[18],"targets":[10],"edgeType":"PROJECTION"},{"sources":[13],"targets":[2,5,9,10],"expression":"(src_dp.year = 2)","edgeType":"PREDI CATE"}],"vertices":[{"id":0,"vertexType":"COLUMN","vertexId":"default.dest_dp1.first"},{"id":1,"vertexType":"COLUMN","vertexId":"default.dest_dp2.first"},{"id":2,"vertexType":"COLUMN","vertexId":"default.dest_dp3.first"},{"id":3,"vertexType":"COLUMN","vertexId":"default.dest_dp1.word"},{"id":4,"vertexType":"COLUMN","vertexId":"default.dest_dp2.word"},{"id":5,"vertexType":"COLUMN","vertexId":"default.dest_dp3.word"},{"id":6,"vertexType":"COLUMN","vertexId":"default.dest_dp1.year"},{"id":7,"vertexType":"COLUMN","vertexId":"default.dest_dp2.y"},{"id":8,"vertexType":"COLUMN","vertexId":"default.dest_dp2.m"},{"id":9,"vertexType":"COLUMN","vertexId":"default.dest_dp3.m"},{"id":10,"vertexType":"COLUMN","vertexId":"default.dest_dp3.d"},{"id":11,"vertexType":"COLUMN","vertexId":"default.src_dp.first"},{"id":12,"vertexType":"COLUMN","vertexId":"default.src_dp.word"},{"id":13,"vertexType":"COLUMN","vertexId":"default.src_dp.year"},{"id":14,"vertexType":"COLUMN","vertexId":"default.src_dp.month "},{"id":15,"vertexType":"COLUMN","vertexId":"default.src_dp1.f"},{"id":16,"vertexType":"COLUMN","vertexId":"default.src_dp1.w"},{"id":17,"vertexType":"COLUMN","vertexId":"default.src_dp1.m"},{"id":18,"vertexType":"COLUMN","vertexId":"default.src_dp.day"}]}
