Author: vikram
Date: Thu Jul 31 20:00:57 2014
New Revision: 1614973
URL: http://svn.apache.org/r1614973
Log:
HIVE-7096: Support grouped splits in Tez partitioned broadcast join (Vikram
Dixit, reviewed by Gunther Hagleitner)
Added:
hive/branches/tez/ql/src/test/queries/clientpositive/tez_bmj_schema_evolution.q
hive/branches/tez/ql/src/test/results/clientpositive/tez/tez_bmj_schema_evolution.q.out
Modified:
hive/branches/tez/itests/qtest/testconfiguration.properties
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinKey.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionEdge.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/DefaultHivePartitioner.java
Modified: hive/branches/tez/itests/qtest/testconfiguration.properties
URL:
http://svn.apache.org/viewvc/hive/branches/tez/itests/qtest/testconfiguration.properties?rev=1614973&r1=1614972&r2=1614973&view=diff
==============================================================================
--- hive/branches/tez/itests/qtest/testconfiguration.properties (original)
+++ hive/branches/tez/itests/qtest/testconfiguration.properties Thu Jul 31
20:00:57 2014
@@ -1,5 +1,5 @@
minimr.query.files=stats_counter_partitioned.q,list_bucket_dml_10.q,input16_cc.q,scriptfile1.q,scriptfile1_win.q,bucket4.q,bucketmapjoin6.q,disable_merge_for_bucketing.q,reduce_deduplicate.q,smb_mapjoin_8.q,join1.q,groupby2.q,bucketizedhiveinputformat.q,bucketmapjoin7.q,optrstat_groupby.q,bucket_num_reducers.q,bucket5.q,load_fs2.q,bucket_num_reducers2.q,infer_bucket_sort_merge.q,infer_bucket_sort_reducers_power_two.q,infer_bucket_sort_dyn_part.q,infer_bucket_sort_bucketed_table.q,infer_bucket_sort_map_operators.q,infer_bucket_sort_num_buckets.q,leftsemijoin_mr.q,schemeAuthority.q,schemeAuthority2.q,truncate_column_buckets.q,remote_script.q,,load_hdfs_file_with_space_in_the_name.q,parallel_orderby.q,import_exported_table.q,stats_counter.q,auto_sortmerge_join_16.q,quotedid_smb.q,file_with_header_footer.q,external_table_with_space_in_location_path.q,root_dir_external_table.q,index_bitmap3.q,ql_rewrite_gbtoidx.q,index_bitmap_auto.q,udf_using.q,empty_dir_in_table.q,temp_table_external.q
minimr.query.negative.files=cluster_tasklog_retrieval.q,minimr_broken_pipe.q,mapreduce_stack_trace.q,mapreduce_stack_trace_turnoff.q,mapreduce_stack_trace_hadoop20.q,mapreduce_stack_trace_turnoff_hadoop20.q,file_with_header_footer_negative.q,udf_local_resource.q
-minitez.query.files=tez_fsstat.q,mapjoin_decimal.q,tez_join_tests.q,tez_joins_explain.q,mrr.q,tez_dml.q,tez_insert_overwrite_local_directory_1.q,tez_union.q,bucket_map_join_tez1.q,bucket_map_join_tez2.q,tez_schema_evolution.q,tez_join_hash.q,tez_bmj_schema_evolution.q
+minitez.query.files=tez_fsstat.q,mapjoin_decimal.q,tez_join_tests.q,tez_joins_explain.q,mrr.q,tez_dml.q,tez_insert_overwrite_local_directory_1.q,tez_union.q,bucket_map_join_tez1.q,bucket_map_join_tez2.q,tez_schema_evolution.q,tez_join_hash.q
minitez.query.files.shared=cross_product_check_1.q,cross_product_check_2.q,dynpart_sort_opt_vectorization.q,dynpart_sort_optimization.q,orc_analyze.q,join0.q,join1.q,auto_join0.q,auto_join1.q,bucket2.q,bucket3.q,bucket4.q,count.q,create_merge_compressed.q,cross_join.q,ctas.q,custom_input_output_format.q,disable_merge_for_bucketing.q,enforce_order.q,filter_join_breaktask.q,filter_join_breaktask2.q,groupby1.q,groupby2.q,groupby3.q,having.q,insert1.q,insert_into1.q,insert_into2.q,leftsemijoin.q,limit_pushdown.q,load_dyn_part1.q,load_dyn_part2.q,load_dyn_part3.q,mapjoin_mapjoin.q,mapreduce1.q,mapreduce2.q,merge1.q,merge2.q,metadata_only_queries.q,sample1.q,subquery_in.q,subquery_exists.q,vectorization_15.q,ptf.q,stats_counter.q,stats_noscan_1.q,stats_counter_partitioned.q,union2.q,union3.q,union4.q,union5.q,union6.q,union7.q,union8.q,union9.q,transform1.q,transform2.q,transform_ppr1.q,transform_ppr2.q,script_env_var1.q,script_env_var2.q,script_pipe.q,scriptfile1.q,metadataonly1.q,temp_t
able.q,vectorized_ptf.q,optimize_nullscan.q
beeline.positive.exclude=add_part_exist.q,alter1.q,alter2.q,alter4.q,alter5.q,alter_rename_partition.q,alter_rename_partition_authorization.q,archive.q,archive_corrupt.q,archive_multi.q,archive_mr_1806.q,archive_multi_mr_1806.q,authorization_1.q,authorization_2.q,authorization_4.q,authorization_5.q,authorization_6.q,authorization_7.q,ba_table1.q,ba_table2.q,ba_table3.q,ba_table_udfs.q,binary_table_bincolserde.q,binary_table_colserde.q,cluster.q,columnarserde_create_shortcut.q,combine2.q,constant_prop.q,create_nested_type.q,create_or_replace_view.q,create_struct_table.q,create_union_table.q,database.q,database_location.q,database_properties.q,ddltime.q,describe_database_json.q,drop_database_removes_partition_dirs.q,escape1.q,escape2.q,exim_00_nonpart_empty.q,exim_01_nonpart.q,exim_02_00_part_empty.q,exim_02_part.q,exim_03_nonpart_over_compat.q,exim_04_all_part.q,exim_04_evolved_parts.q,exim_05_some_part.q,exim_06_one_part.q,exim_07_all_part_over_nonoverlap.q,exim_08_nonpart_rename.q,
exim_09_part_spec_nonoverlap.q,exim_10_external_managed.q,exim_11_managed_external.q,exim_12_external_location.q,exim_13_managed_location.q,exim_14_managed_location_over_existing.q,exim_15_external_part.q,exim_16_part_external.q,exim_17_part_managed.q,exim_18_part_external.q,exim_19_00_part_external_location.q,exim_19_part_external_location.q,exim_20_part_managed_location.q,exim_21_export_authsuccess.q,exim_22_import_exist_authsuccess.q,exim_23_import_part_authsuccess.q,exim_24_import_nonexist_authsuccess.q,global_limit.q,groupby_complex_types.q,groupby_complex_types_multi_single_reducer.q,index_auth.q,index_auto.q,index_auto_empty.q,index_bitmap.q,index_bitmap1.q,index_bitmap2.q,index_bitmap3.q,index_bitmap_auto.q,index_bitmap_rc.q,index_compact.q,index_compact_1.q,index_compact_2.q,index_compact_3.q,index_stale_partitioned.q,init_file.q,input16.q,input16_cc.q,input46.q,input_columnarserde.q,input_dynamicserde.q,input_lazyserde.q,input_testxpath3.q,input_testxpath4.q,insert2_overwr
ite_partitions.q,insertexternal1.q,join_thrift.q,lateral_view.q,load_binary_data.q,load_exist_part_authsuccess.q,load_nonpart_authsuccess.q,load_part_authsuccess.q,loadpart_err.q,lock1.q,lock2.q,lock3.q,lock4.q,merge_dynamic_partition.q,multi_insert.q,multi_insert_move_tasks_share_dependencies.q,null_column.q,ppd_clusterby.q,query_with_semi.q,rename_column.q,sample6.q,sample_islocalmode_hook.q,set_processor_namespaces.q,show_tables.q,source.q,split_sample.q,str_to_map.q,transform1.q,udaf_collect_set.q,udaf_context_ngrams.q,udaf_histogram_numeric.q,udaf_ngrams.q,udaf_percentile_approx.q,udf_array.q,udf_bitmap_and.q,udf_bitmap_or.q,udf_explode.q,udf_format_number.q,udf_map.q,udf_map_keys.q,udf_map_values.q,udf_max.q,udf_min.q,udf_named_struct.q,udf_percentile.q,udf_printf.q,udf_sentences.q,udf_sort_array.q,udf_split.q,udf_struct.q,udf_substr.q,udf_translate.q,udf_union.q,udf_xpath.q,udtf_stack.q,view.q,virtual_column.q
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java?rev=1614973&r1=1614972&r2=1614973&view=diff
==============================================================================
---
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
(original)
+++
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/MapJoinOperator.java
Thu Jul 31 20:00:57 2014
@@ -186,7 +186,7 @@ public class MapJoinOperator extends Abs
* process different buckets and if the container is reused to join a
different bucket,
* join results can be incorrect. The cache is keyed on operator id and
for bucket map join
* the operator does not change but data needed is different. For a
proper fix, this
- * requires changes in the Tez API with regard to finding bucket id and
+ * requires changes in the Tez API with regard to finding bucket id and
* also ability to schedule tasks to re-use containers that have cached
the specific bucket.
*/
LOG.info("This is not bucket map join, so cache");
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java?rev=1614973&r1=1614972&r2=1614973&view=diff
==============================================================================
---
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java
(original)
+++
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/mr/MapRedTask.java
Thu Jul 31 20:00:57 2014
@@ -130,7 +130,7 @@ public class MapRedTask extends ExecDriv
runningViaChild = conf.getBoolVar(HiveConf.ConfVars.SUBMITVIACHILD);
- if (!runningViaChild) {
+ if(!runningViaChild) {
// we are not running this mapred task via child jvm
// so directly invoke ExecDriver
return super.execute(driverContext);
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java?rev=1614973&r1=1614972&r2=1614973&view=diff
==============================================================================
---
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java
(original)
+++
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinBytesTableContainer.java
Thu Jul 31 20:00:57 2014
@@ -14,10 +14,10 @@ import org.apache.hadoop.hive.ql.exec.ve
import org.apache.hadoop.hive.ql.exec.vector.VectorHashKeyWrapperBatch;
import
org.apache.hadoop.hive.ql.exec.vector.expressions.VectorExpressionWriter;
import org.apache.hadoop.hive.ql.metadata.HiveException;
-import org.apache.hadoop.hive.serde2.ByteStream.Output;
-import org.apache.hadoop.hive.serde2.ByteStream.RandomAccessOutput;
import org.apache.hadoop.hive.serde2.SerDe;
import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.ByteStream.Output;
+import org.apache.hadoop.hive.serde2.ByteStream.RandomAccessOutput;
import org.apache.hadoop.hive.serde2.WriteBuffers;
import org.apache.hadoop.hive.serde2.binarysortable.BinarySortableSerDe;
import org.apache.hadoop.hive.serde2.io.ShortWritable;
@@ -57,7 +57,7 @@ public class MapJoinBytesTableContainer
private boolean[] sortableSortOrders;
private KeyValueHelper writeHelper;
- private final List<Object> EMPTY_LIST = new ArrayList<Object>(0);
+ private List<Object> EMPTY_LIST = new ArrayList<Object>(0);
public MapJoinBytesTableContainer(Configuration hconf,
MapJoinObjectSerDeContext valCtx)
throws SerDeException {
@@ -474,7 +474,6 @@ public class MapJoinBytesTableContainer
return valueStruct.getFieldsAsList(); // TODO: should we unset bytes
after that?
}
- @Override
public void addRow(List<Object> t) {
if (dummyRow != null || !refs.isEmpty()) {
throw new RuntimeException("Cannot add rows when not empty");
@@ -483,11 +482,9 @@ public class MapJoinBytesTableContainer
}
// Various unsupported methods.
- @Override
public void addRow(Object[] value) {
throw new RuntimeException(this.getClass().getCanonicalName() + " cannot
add arrays");
}
- @Override
public void write(MapJoinObjectSerDeContext valueContext,
ObjectOutputStream out) {
throw new RuntimeException(this.getClass().getCanonicalName() + " cannot
be serialized");
}
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinKey.java
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinKey.java?rev=1614973&r1=1614972&r2=1614973&view=diff
==============================================================================
---
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinKey.java
(original)
+++
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/persistence/MapJoinKey.java
Thu Jul 31 20:00:57 2014
@@ -42,6 +42,7 @@ import org.apache.hadoop.hive.serde2.obj
import
org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+
import org.apache.hadoop.io.Writable;
/**
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionEdge.java
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionEdge.java?rev=1614973&r1=1614972&r2=1614973&view=diff
==============================================================================
---
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionEdge.java
(original)
+++
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionEdge.java
Thu Jul 31 20:00:57 2014
@@ -19,8 +19,8 @@
package org.apache.hadoop.hive.ql.exec.tez;
import java.io.IOException;
-import java.util.Collections;
import java.util.List;
+import java.util.ArrayList;
import java.util.Map;
import org.apache.commons.logging.Log;
@@ -31,6 +31,8 @@ import org.apache.tez.dag.api.EdgeManage
import org.apache.tez.runtime.api.events.DataMovementEvent;
import org.apache.tez.runtime.api.events.InputReadErrorEvent;
+import com.google.common.collect.Multimap;
+
public class CustomPartitionEdge extends EdgeManager {
private static final Log LOG =
LogFactory.getLog(CustomPartitionEdge.class.getName());
@@ -39,22 +41,22 @@ public class CustomPartitionEdge extends
EdgeManagerContext context = null;
// used by the framework at runtime. initialize is the real initializer at
runtime
- public CustomPartitionEdge() {
+ public CustomPartitionEdge() {
}
@Override
- public int getNumDestinationTaskPhysicalInputs(int destinationTaskIndex) {
- return context.getSourceVertexNumTasks();
+ public int getNumDestinationTaskPhysicalInputs(int numSourceTasks) {
+ return numSourceTasks;
}
@Override
- public int getNumSourceTaskPhysicalOutputs(int sourceTaskIndex) {
+ public int getNumSourceTaskPhysicalOutputs(int numDestinationTasks) {
return conf.getNumBuckets();
}
@Override
public int getNumDestinationConsumerTasks(int sourceTaskIndex) {
- return context.getDestinationVertexNumTasks();
+ return sourceTaskIndex;
}
// called at runtime to initialize the custom edge.
@@ -81,25 +83,30 @@ public class CustomPartitionEdge extends
@Override
public void routeDataMovementEventToDestination(DataMovementEvent event,
- int sourceTaskIndex, int sourceOutputIndex, Map<Integer, List<Integer>>
mapDestTaskIndices) {
- List<Integer> outputIndices = Collections.singletonList(sourceTaskIndex);
- for (Integer destIndex : conf.getRoutingTable().get(sourceOutputIndex)) {
- mapDestTaskIndices.put(destIndex, outputIndices);
- }
+ int sourceTaskIndex, int numDestinationTasks, Map<Integer,
List<Integer>> mapDestTaskIndices) {
+ int srcIndex = event.getSourceIndex();
+ List<Integer> destTaskIndices = new ArrayList<Integer>();
+ destTaskIndices.addAll(conf.getRoutingTable().get(srcIndex));
+ mapDestTaskIndices.put(new Integer(sourceTaskIndex), destTaskIndices);
}
@Override
- public void routeInputSourceTaskFailedEventToDestination(int sourceTaskIndex,
- Map<Integer, List<Integer>> mapDestTaskIndices) {
- List<Integer> outputIndices = Collections.singletonList(sourceTaskIndex);
- for (int i = 0; i < context.getDestinationVertexNumTasks(); i++) {
- mapDestTaskIndices.put(i, outputIndices);
- }
+ public void routeInputSourceTaskFailedEventToDestination(int
+ sourceTaskIndex, Map<Integer, List<Integer>> mapDestTaskIndices) {
+ List<Integer> destTaskIndices = new ArrayList<Integer>();
+ addAllDestinationTaskIndices(context.getDestinationVertexNumTasks(),
destTaskIndices);
+ mapDestTaskIndices.put(new Integer(sourceTaskIndex), destTaskIndices);
}
@Override
- public int routeInputErrorEventToSource(InputReadErrorEvent event,
+ public int routeInputErrorEventToSource(InputReadErrorEvent event,
int destinationTaskIndex, int destinationFailedInputIndex) {
return event.getIndex();
}
+
+ void addAllDestinationTaskIndices(int numDestinationTasks, List<Integer>
taskIndices) {
+ for(int i=0; i<numDestinationTasks; ++i) {
+ taskIndices.add(new Integer(i));
+ }
+ }
}
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java?rev=1614973&r1=1614972&r2=1614973&view=diff
==============================================================================
---
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java
(original)
+++
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/CustomPartitionVertex.java
Thu Jul 31 20:00:57 2014
@@ -22,7 +22,6 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collection;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -31,12 +30,11 @@ import java.util.TreeMap;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hive.shims.ShimLoader;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.io.serializer.SerializationFactory;
import org.apache.hadoop.mapred.FileSplit;
import org.apache.hadoop.mapred.InputSplit;
-import org.apache.hadoop.mapred.JobConf;
import org.apache.hadoop.mapred.split.TezGroupedSplitsInputFormat;
import org.apache.tez.dag.api.EdgeManagerDescriptor;
import org.apache.tez.dag.api.EdgeProperty;
@@ -46,12 +44,10 @@ import org.apache.tez.dag.api.TezConfigu
import org.apache.tez.dag.api.VertexLocationHint;
import org.apache.tez.dag.api.VertexManagerPlugin;
import org.apache.tez.dag.api.VertexManagerPluginContext;
-import org.apache.tez.dag.api.VertexManagerPluginContext.TaskWithLocationHint;
import org.apache.tez.mapreduce.hadoop.MRHelpers;
import org.apache.tez.mapreduce.protos.MRRuntimeProtos.MRInputUserPayloadProto;
import org.apache.tez.mapreduce.protos.MRRuntimeProtos.MRSplitProto;
import org.apache.tez.runtime.api.Event;
-import org.apache.tez.runtime.api.RootInputSpecUpdate;
import org.apache.tez.runtime.api.events.RootInputConfigureVertexTasksEvent;
import org.apache.tez.runtime.api.events.RootInputDataInformationEvent;
import org.apache.tez.runtime.api.events.RootInputUpdatePayloadEvent;
@@ -80,7 +76,6 @@ public class CustomPartitionVertex exten
private Configuration conf = null;
private boolean rootVertexInitialized = false;
private final SplitGrouper grouper = new SplitGrouper();
- private int taskCount = 0;
public CustomPartitionVertex() {
}
@@ -95,8 +90,7 @@ public class CustomPartitionVertex exten
@Override
public void onVertexStarted(Map<String, List<Integer>> completions) {
int numTasks = context.getVertexNumTasks(context.getVertexName());
- List<VertexManagerPluginContext.TaskWithLocationHint> scheduledTasks =
- new ArrayList<VertexManagerPluginContext.TaskWithLocationHint>(numTasks);
+ List<VertexManagerPluginContext.TaskWithLocationHint> scheduledTasks = new
ArrayList<VertexManagerPluginContext.TaskWithLocationHint>(numTasks);
for (int i = 0; i < numTasks; ++i) {
scheduledTasks.add(new
VertexManagerPluginContext.TaskWithLocationHint(new Integer(i), null));
}
@@ -120,7 +114,6 @@ public class CustomPartitionVertex exten
// ensure this method is called only once. Tez will call it once per Root
// Input.
Preconditions.checkState(rootVertexInitialized == false);
- LOG.info("Root vertex not initialized");
rootVertexInitialized = true;
try {
// This is using the payload from the RootVertexInitializer corresponding
@@ -158,7 +151,7 @@ public class CustomPartitionVertex exten
}
boolean dataInformationEventSeen = false;
- Map<String, List<FileSplit>> pathFileSplitsMap = new TreeMap<String,
List<FileSplit>>();
+ Map<Path, List<FileSplit>> pathFileSplitsMap = new TreeMap<Path,
List<FileSplit>>();
for (Event event : events) {
if (event instanceof RootInputConfigureVertexTasksEvent) {
@@ -189,10 +182,10 @@ public class CustomPartitionVertex exten
} catch (IOException e) {
throw new RuntimeException("Failed to get file split for event: " +
diEvent);
}
- List<FileSplit> fsList =
pathFileSplitsMap.get(fileSplit.getPath().getName());
+ List<FileSplit> fsList = pathFileSplitsMap.get(fileSplit.getPath());
if (fsList == null) {
fsList = new ArrayList<FileSplit>();
- pathFileSplitsMap.put(fileSplit.getPath().getName(), fsList);
+ pathFileSplitsMap.put(fileSplit.getPath(), fsList);
}
fsList.add(fileSplit);
}
@@ -211,23 +204,12 @@ public class CustomPartitionVertex exten
int availableSlots = totalResource / taskResource;
LOG.info("Grouping splits. " + availableSlots + " available slots, " +
waves + " waves.");
- JobConf jobConf = new JobConf(conf);
- ShimLoader.getHadoopShims().getMergedCredentials(jobConf);
Multimap<Integer, InputSplit> bucketToGroupedSplitMap =
- HashMultimap.<Integer, InputSplit> create();
- for (Integer key : bucketToInitialSplitMap.keySet()) {
- InputSplit[] inputSplitArray =
- (bucketToInitialSplitMap.get(key).toArray(new InputSplit[0]));
- Multimap<Integer, InputSplit> groupedSplit =
- HiveSplitGenerator.generateGroupedSplits(jobConf, conf,
inputSplitArray, waves,
- availableSlots);
- bucketToGroupedSplitMap.putAll(key, groupedSplit.values());
- }
+ grouper.group(conf, bucketToInitialSplitMap, availableSlots, waves);
- LOG.info("We have grouped the splits into " +
bucketToGroupedSplitMap.size() + " tasks");
processAllEvents(inputName, bucketToGroupedSplitMap);
- } catch (Exception e) {
+ } catch (IOException e) {
throw new RuntimeException(e);
}
}
@@ -237,6 +219,7 @@ public class CustomPartitionVertex exten
Multimap<Integer, Integer> bucketToTaskMap = HashMultimap.<Integer,
Integer> create();
List<InputSplit> finalSplits = Lists.newLinkedList();
+ int taskCount = 0;
for (Entry<Integer, Collection<InputSplit>> entry :
bucketToGroupedSplitMap.asMap().entrySet()) {
int bucketNum = entry.getKey();
Collection<InputSplit> initialSplits = entry.getValue();
@@ -281,15 +264,10 @@ public class CustomPartitionVertex exten
}
// Replace the Edge Managers
- Map<String, RootInputSpecUpdate> rootInputSpecUpdate =
- new HashMap<String, RootInputSpecUpdate>();
- rootInputSpecUpdate.put(
- inputName,
- RootInputSpecUpdate.getDefaultSinglePhysicalInputSpecUpdate());
context.setVertexParallelism(
taskCount,
new VertexLocationHint(grouper.createTaskLocationHints(finalSplits
- .toArray(new InputSplit[finalSplits.size()]))), emMap,
rootInputSpecUpdate);
+ .toArray(new InputSplit[finalSplits.size()]))), emMap, null);
// Set the actual events for the tasks.
context.addRootInputEvents(inputName, taskEvents);
@@ -326,7 +304,7 @@ public class CustomPartitionVertex exten
* This method generates the map of bucket to file splits.
*/
private Multimap<Integer, InputSplit> getBucketSplitMapForPath(
- Map<String, List<FileSplit>> pathFileSplitsMap) {
+ Map<Path, List<FileSplit>> pathFileSplitsMap) {
int bucketNum = 0;
int fsCount = 0;
@@ -334,7 +312,7 @@ public class CustomPartitionVertex exten
Multimap<Integer, InputSplit> bucketToInitialSplitMap =
ArrayListMultimap.<Integer, InputSplit> create();
- for (Map.Entry<String, List<FileSplit>> entry :
pathFileSplitsMap.entrySet()) {
+ for (Map.Entry<Path, List<FileSplit>> entry :
pathFileSplitsMap.entrySet()) {
int bucketId = bucketNum % numBuckets;
for (FileSplit fsplit : entry.getValue()) {
fsCount++;
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java?rev=1614973&r1=1614972&r2=1614973&view=diff
==============================================================================
---
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
(original)
+++
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/DagUtils.java
Thu Jul 31 20:00:57 2014
@@ -95,14 +95,12 @@ import org.apache.tez.dag.api.Vertex;
import org.apache.tez.dag.api.VertexGroup;
import org.apache.tez.dag.api.VertexManagerPluginDescriptor;
import org.apache.tez.dag.library.vertexmanager.ShuffleVertexManager;
-import org.apache.tez.mapreduce.common.MRInputAMSplitGenerator;
import org.apache.tez.mapreduce.hadoop.InputSplitInfo;
import org.apache.tez.mapreduce.hadoop.MRHelpers;
import org.apache.tez.mapreduce.hadoop.MRJobConfig;
import org.apache.tez.mapreduce.input.MRInputLegacy;
import org.apache.tez.mapreduce.output.MROutput;
import org.apache.tez.mapreduce.partition.MRPartitioner;
-import org.apache.tez.runtime.api.TezRootInputInitializer;
import org.apache.tez.runtime.library.conf.OrderedPartitionedKVEdgeConfigurer;
import
org.apache.tez.runtime.library.conf.UnorderedPartitionedKVEdgeConfigurer;
import
org.apache.tez.runtime.library.conf.UnorderedUnpartitionedKVEdgeConfigurer;
@@ -417,7 +415,7 @@ public class DagUtils {
boolean useTezGroupedSplits = false;
int numTasks = -1;
- Class<? extends TezRootInputInitializer> amSplitGeneratorClass = null;
+ Class<HiveSplitGenerator> amSplitGeneratorClass = null;
InputSplitInfo inputSplitInfo = null;
Class inputFormatClass = conf.getClass("mapred.input.format.class",
InputFormat.class);
@@ -453,11 +451,7 @@ public class DagUtils {
&& !mapWork.isUseOneNullRowInputFormat()) {
// if we're generating the splits in the AM, we just need to set
// the correct plugin.
- if (useTezGroupedSplits) {
- amSplitGeneratorClass = HiveSplitGenerator.class;
- } else {
- amSplitGeneratorClass = MRInputAMSplitGenerator.class;
- }
+ amSplitGeneratorClass = HiveSplitGenerator.class;
} else {
// client side split generation means we have to compute them now
inputSplitInfo = MRHelpers.generateInputSplits(conf,
@@ -488,7 +482,6 @@ public class DagUtils {
} else {
mrInput = MRHelpers.createMRInputPayload(serializedConf, null);
}
-
map.addInput(alias,
new InputDescriptor(MRInputLegacy.class.getName()).
setUserPayload(mrInput), amSplitGeneratorClass);
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java?rev=1614973&r1=1614972&r2=1614973&view=diff
==============================================================================
---
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java
(original)
+++
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HashTableLoader.java
Thu Jul 31 20:00:57 2014
@@ -18,29 +18,45 @@
package org.apache.hadoop.hive.ql.exec.tez;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
import java.util.Map;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hive.conf.HiveConf;
+import org.apache.hadoop.hive.ql.debug.Utils;
import org.apache.hadoop.hive.ql.exec.MapJoinOperator;
import org.apache.hadoop.hive.ql.exec.MapredContext;
import org.apache.hadoop.hive.ql.exec.mr.ExecMapperContext;
import org.apache.hadoop.hive.ql.exec.persistence.HashMapWrapper;
import org.apache.hadoop.hive.ql.exec.persistence.MapJoinBytesTableContainer;
import org.apache.hadoop.hive.ql.exec.persistence.MapJoinKey;
+import org.apache.hadoop.hive.ql.exec.persistence.MapJoinKeyObject;
+import org.apache.hadoop.hive.ql.exec.persistence.LazyFlatRowContainer;
+import org.apache.hadoop.hive.ql.exec.persistence.MapJoinKey;
import org.apache.hadoop.hive.ql.exec.persistence.MapJoinObjectSerDeContext;
import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainer;
import org.apache.hadoop.hive.ql.exec.persistence.MapJoinTableContainerSerDe;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.plan.MapJoinDesc;
+import org.apache.hadoop.hive.serde2.ByteStream.Output;
import org.apache.hadoop.hive.serde2.SerDeException;
+import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryFactory;
+import org.apache.hadoop.hive.serde2.lazybinary.LazyBinarySerDe;
+import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryStruct;
+import org.apache.hadoop.hive.serde2.lazybinary.LazyBinaryUtils;
import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector;
-import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
import org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector;
import org.apache.hadoop.hive.serde2.objectinspector.StructField;
import org.apache.hadoop.hive.serde2.objectinspector.StructObjectInspector;
+import org.apache.hadoop.hive.serde2.objectinspector.ObjectInspector.Category;
+import
org.apache.hadoop.hive.serde2.objectinspector.PrimitiveObjectInspector.PrimitiveCategory;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfo;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoFactory;
+import org.apache.hadoop.hive.serde2.typeinfo.TypeInfoUtils;
+import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.Writable;
import org.apache.tez.runtime.api.LogicalInput;
import org.apache.tez.runtime.library.api.KeyValueReader;
@@ -57,7 +73,6 @@ public class HashTableLoader implements
private Configuration hconf;
private MapJoinDesc desc;
private MapJoinKey lastKey = null;
- private int rowCount = 0;
@Override
public void init(ExecMapperContext context, Configuration hconf,
MapJoinOperator joinOp) {
@@ -106,7 +121,6 @@ public class HashTableLoader implements
? new MapJoinBytesTableContainer(hconf, valCtx) : new
HashMapWrapper(hconf);
while (kvReader.next()) {
- rowCount++;
lastKey = tableContainer.putRow(keyCtx,
(Writable)kvReader.getCurrentKey(),
valCtx, (Writable)kvReader.getCurrentValue());
}
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java?rev=1614973&r1=1614972&r2=1614973&view=diff
==============================================================================
---
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java
(original)
+++
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/exec/tez/HiveSplitGenerator.java
Thu Jul 31 20:00:57 2014
@@ -45,12 +45,11 @@ import org.apache.tez.mapreduce.protos.M
import org.apache.tez.mapreduce.protos.MRRuntimeProtos.MRSplitsProto;
import org.apache.tez.runtime.api.Event;
import org.apache.tez.runtime.api.RootInputSpecUpdate;
+import org.apache.tez.runtime.api.TezRootInputInitializer;
+import org.apache.tez.runtime.api.TezRootInputInitializerContext;
import org.apache.tez.runtime.api.events.RootInputConfigureVertexTasksEvent;
import org.apache.tez.runtime.api.events.RootInputDataInformationEvent;
import org.apache.tez.runtime.api.events.RootInputInitializerEvent;
-import org.apache.tez.runtime.api.RootInputSpecUpdate;
-import org.apache.tez.runtime.api.TezRootInputInitializer;
-import org.apache.tez.runtime.api.TezRootInputInitializerContext;
import com.google.common.collect.ArrayListMultimap;
import com.google.common.collect.Lists;
@@ -66,7 +65,7 @@ public class HiveSplitGenerator implemen
private static final Log LOG = LogFactory.getLog(HiveSplitGenerator.class);
- private static final SplitGrouper grouper = new SplitGrouper();
+ private final SplitGrouper grouper = new SplitGrouper();
@Override
public List<Event> initialize(TezRootInputInitializerContext
rootInputContext) throws Exception {
@@ -87,36 +86,7 @@ public class HiveSplitGenerator implemen
InputSplitInfoMem inputSplitInfo = null;
String realInputFormatName = userPayloadProto.getInputFormatName();
if (realInputFormatName != null && !realInputFormatName.isEmpty()) {
- // Need to instantiate the realInputFormat
- InputFormat<?, ?> inputFormat =
- (InputFormat<?, ?>)
ReflectionUtils.newInstance(Class.forName(realInputFormatName),
- jobConf);
-
- int totalResource =
rootInputContext.getTotalAvailableResource().getMemory();
- int taskResource = rootInputContext.getVertexTaskResource().getMemory();
- int availableSlots = totalResource / taskResource;
-
- // Create the un-grouped splits
- float waves =
- conf.getFloat(TezConfiguration.TEZ_AM_GROUPING_SPLIT_WAVES,
- TezConfiguration.TEZ_AM_GROUPING_SPLIT_WAVES_DEFAULT);
-
- InputSplit[] splits = inputFormat.getSplits(jobConf, (int)
(availableSlots * waves));
- LOG.info("Number of input splits: " + splits.length + ". " +
availableSlots
- + " available slots, " + waves + " waves. Input format is: " +
realInputFormatName);
-
- Multimap<Integer, InputSplit> groupedSplits =
- generateGroupedSplits(jobConf, conf, splits, waves, availableSlots);
- // And finally return them in a flat array
- InputSplit[] flatSplits = groupedSplits.values().toArray(new
InputSplit[0]);
- LOG.info("Number of grouped splits: " + flatSplits.length);
-
- List<TaskLocationHint> locationHints =
grouper.createTaskLocationHints(flatSplits);
-
- Utilities.clearWork(jobConf);
-
- inputSplitInfo =
- new InputSplitInfoMem(flatSplits, locationHints, flatSplits.length,
null, jobConf);
+ inputSplitInfo = generateGroupedSplits(rootInputContext, jobConf, conf,
realInputFormatName);
} else {
inputSplitInfo = MRHelpers.generateInputSplitsToMem(jobConf);
}
@@ -124,12 +94,31 @@ public class HiveSplitGenerator implemen
return createEventList(sendSerializedEvents, inputSplitInfo);
}
- public static Multimap<Integer, InputSplit> generateGroupedSplits(JobConf
jobConf,
- Configuration conf, InputSplit[] splits, float waves, int availableSlots)
- throws Exception {
+ private InputSplitInfoMem
generateGroupedSplits(TezRootInputInitializerContext context,
+ JobConf jobConf, Configuration conf, String realInputFormatName) throws
Exception {
+
+ int totalResource = context.getTotalAvailableResource().getMemory();
+ int taskResource = context.getVertexTaskResource().getMemory();
+ int availableSlots = totalResource / taskResource;
+
+ float waves =
+ conf.getFloat(TezConfiguration.TEZ_AM_GROUPING_SPLIT_WAVES,
+ TezConfiguration.TEZ_AM_GROUPING_SPLIT_WAVES_DEFAULT);
MapWork work = Utilities.getMapWork(jobConf);
+ LOG.info("Grouping splits for " + work.getName() + ". " + availableSlots +
" available slots, "
+ + waves + " waves. Input format is: " + realInputFormatName);
+
+ // Need to instantiate the realInputFormat
+ InputFormat<?, ?> inputFormat =
+ (InputFormat<?, ?>) ReflectionUtils
+ .newInstance(Class.forName(realInputFormatName), jobConf);
+
+ // Create the un-grouped splits
+ InputSplit[] splits = inputFormat.getSplits(jobConf, (int) (availableSlots
* waves));
+ LOG.info("Number of input splits: " + splits.length);
+
Multimap<Integer, InputSplit> bucketSplitMultiMap =
ArrayListMultimap.<Integer, InputSplit> create();
@@ -172,7 +161,15 @@ public class HiveSplitGenerator implemen
Multimap<Integer, InputSplit> groupedSplits =
grouper.group(jobConf, bucketSplitMultiMap, availableSlots, waves);
- return groupedSplits;
+ // And finally return them in a flat array
+ InputSplit[] flatSplits = groupedSplits.values().toArray(new
InputSplit[0]);
+ LOG.info("Number of grouped splits: " + flatSplits.length);
+
+ List<TaskLocationHint> locationHints =
grouper.createTaskLocationHints(flatSplits);
+
+ Utilities.clearWork(jobConf);
+
+ return new InputSplitInfoMem(flatSplits, locationHints, flatSplits.length,
null, jobConf);
}
private List<Event> createEventList(boolean sendSerializedEvents,
InputSplitInfoMem inputSplitInfo) {
@@ -181,8 +178,7 @@ public class HiveSplitGenerator implemen
RootInputConfigureVertexTasksEvent configureVertexEvent = new
RootInputConfigureVertexTasksEvent(inputSplitInfo.getNumTasks(),
- inputSplitInfo.getTaskLocationHints(),
- RootInputSpecUpdate.getDefaultSinglePhysicalInputSpecUpdate());
+ inputSplitInfo.getTaskLocationHints(),
RootInputSpecUpdate.getDefaultSinglePhysicalInputSpecUpdate());
events.add(configureVertexEvent);
if (sendSerializedEvents) {
Modified:
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/DefaultHivePartitioner.java
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/DefaultHivePartitioner.java?rev=1614973&r1=1614972&r2=1614973&view=diff
==============================================================================
---
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/DefaultHivePartitioner.java
(original)
+++
hive/branches/tez/ql/src/java/org/apache/hadoop/hive/ql/io/DefaultHivePartitioner.java
Thu Jul 31 20:00:57 2014
@@ -18,13 +18,14 @@
package org.apache.hadoop.hive.ql.io;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.mapred.lib.HashPartitioner;
/** Partition keys by their {@link Object#hashCode()}. */
public class DefaultHivePartitioner<K2, V2> extends HashPartitioner<K2, V2>
implements HivePartitioner<K2, V2> {
/** Use {@link Object#hashCode()} to partition. */
- @Override
public int getBucket(K2 key, V2 value, int numBuckets) {
return (key.hashCode() & Integer.MAX_VALUE) % numBuckets;
}
Added:
hive/branches/tez/ql/src/test/queries/clientpositive/tez_bmj_schema_evolution.q
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/queries/clientpositive/tez_bmj_schema_evolution.q?rev=1614973&view=auto
==============================================================================
---
hive/branches/tez/ql/src/test/queries/clientpositive/tez_bmj_schema_evolution.q
(added)
+++
hive/branches/tez/ql/src/test/queries/clientpositive/tez_bmj_schema_evolution.q
Thu Jul 31 20:00:57 2014
@@ -0,0 +1,23 @@
+set hive.enforce.bucketing=true;
+set hive.enforce.sorting = true;
+set hive.optimize.bucketingsorting=false;
+set hive.auto.convert.join.noconditionaltask.size=10000;
+
+create table test (key int, value string) partitioned by (p int) clustered by
(key) into 2 buckets stored as textfile;
+create table test1 (key int, value string) stored as textfile;
+
+insert into table test partition (p=1) select * from src;
+
+alter table test set fileformat orc;
+
+insert into table test partition (p=2) select * from src;
+insert into table test1 select * from src;
+
+describe test;
+set hive.auto.convert.join = true;
+set hive.convert.join.bucket.mapjoin.tez = true;
+
+explain select test.key, test.value from test join test1 on (test.key =
test1.key) order by test.key;
+
+select test.key, test.value from test join test1 on (test.key = test1.key)
order by test.key;
+
Added:
hive/branches/tez/ql/src/test/results/clientpositive/tez/tez_bmj_schema_evolution.q.out
URL:
http://svn.apache.org/viewvc/hive/branches/tez/ql/src/test/results/clientpositive/tez/tez_bmj_schema_evolution.q.out?rev=1614973&view=auto
==============================================================================
---
hive/branches/tez/ql/src/test/results/clientpositive/tez/tez_bmj_schema_evolution.q.out
(added)
+++
hive/branches/tez/ql/src/test/results/clientpositive/tez/tez_bmj_schema_evolution.q.out
Thu Jul 31 20:00:57 2014
@@ -0,0 +1,2213 @@
+PREHOOK: query: create table test (key int, value string) partitioned by (p
int) clustered by (key) into 2 buckets stored as textfile
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+POSTHOOK: query: create table test (key int, value string) partitioned by (p
int) clustered by (key) into 2 buckets stored as textfile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@test
+PREHOOK: query: create table test1 (key int, value string) stored as textfile
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+POSTHOOK: query: create table test1 (key int, value string) stored as textfile
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@test1
+PREHOOK: query: insert into table test partition (p=1) select * from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@test@p=1
+POSTHOOK: query: insert into table test partition (p=1) select * from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@test@p=1
+POSTHOOK: Lineage: test PARTITION(p=1).key EXPRESSION
[(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: test PARTITION(p=1).value SIMPLE
[(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: alter table test set fileformat orc
+PREHOOK: type: ALTERTABLE_FILEFORMAT
+PREHOOK: Input: default@test
+PREHOOK: Output: default@test
+POSTHOOK: query: alter table test set fileformat orc
+POSTHOOK: type: ALTERTABLE_FILEFORMAT
+POSTHOOK: Input: default@test
+POSTHOOK: Output: default@test
+PREHOOK: query: insert into table test partition (p=2) select * from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@test@p=2
+POSTHOOK: query: insert into table test partition (p=2) select * from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@test@p=2
+POSTHOOK: Lineage: test PARTITION(p=2).key EXPRESSION
[(src)src.FieldSchema(name:key, type:string, comment:default), ]
+POSTHOOK: Lineage: test PARTITION(p=2).value SIMPLE
[(src)src.FieldSchema(name:value, type:string, comment:default), ]
+PREHOOK: query: insert into table test1 select * from src
+PREHOOK: type: QUERY
+PREHOOK: Input: default@src
+PREHOOK: Output: default@test1
+POSTHOOK: query: insert into table test1 select * from src
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@src
+POSTHOOK: Output: default@test1
+POSTHOOK: Lineage: test1.key EXPRESSION [(src)src.FieldSchema(name:key,
type:string, comment:default), ]
+POSTHOOK: Lineage: test1.value SIMPLE [(src)src.FieldSchema(name:value,
type:string, comment:default), ]
+PREHOOK: query: describe test
+PREHOOK: type: DESCTABLE
+PREHOOK: Input: default@test
+POSTHOOK: query: describe test
+POSTHOOK: type: DESCTABLE
+POSTHOOK: Input: default@test
+key int
+value string
+p int
+
+# Partition Information
+# col_name data_type comment
+
+p int
+PREHOOK: query: explain select test.key, test.value from test join test1 on
(test.key = test1.key) order by test.key
+PREHOOK: type: QUERY
+POSTHOOK: query: explain select test.key, test.value from test join test1 on
(test.key = test1.key) order by test.key
+POSTHOOK: type: QUERY
+STAGE DEPENDENCIES:
+ Stage-1 is a root stage
+ Stage-0 depends on stages: Stage-1
+
+STAGE PLANS:
+ Stage: Stage-1
+ Tez
+ Edges:
+ Map 2 <- Map 1 (CUSTOM_EDGE)
+ Reducer 3 <- Map 2 (SIMPLE_EDGE)
+#### A masked pattern was here ####
+ Vertices:
+ Map 1
+ Map Operator Tree:
+ TableScan
+ alias: test1
+ Statistics: Num rows: 500 Data size: 5312 Basic stats:
COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats:
COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: key (type: int)
+ sort order: +
+ Map-reduce partition columns: key (type: int)
+ Statistics: Num rows: 250 Data size: 2656 Basic stats:
COMPLETE Column stats: NONE
+ Map 2
+ Map Operator Tree:
+ TableScan
+ alias: test
+ Statistics: Num rows: 1000 Data size: 52312 Basic stats:
COMPLETE Column stats: NONE
+ Filter Operator
+ predicate: key is not null (type: boolean)
+ Statistics: Num rows: 500 Data size: 26156 Basic stats:
COMPLETE Column stats: NONE
+ Map Join Operator
+ condition map:
+ Inner Join 0 to 1
+ condition expressions:
+ 0 {key} {value}
+ 1
+ keys:
+ 0 key (type: int)
+ 1 key (type: int)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 550 Data size: 28771 Basic stats:
COMPLETE Column stats: NONE
+ Select Operator
+ expressions: _col0 (type: int), _col1 (type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 550 Data size: 28771 Basic
stats: COMPLETE Column stats: NONE
+ Reduce Output Operator
+ key expressions: _col0 (type: int)
+ sort order: +
+ Statistics: Num rows: 550 Data size: 28771 Basic
stats: COMPLETE Column stats: NONE
+ value expressions: _col1 (type: string)
+ Reducer 3
+ Reduce Operator Tree:
+ Select Operator
+ expressions: KEY.reducesinkkey0 (type: int), VALUE._col0
(type: string)
+ outputColumnNames: _col0, _col1
+ Statistics: Num rows: 550 Data size: 28771 Basic stats:
COMPLETE Column stats: NONE
+ File Output Operator
+ compressed: false
+ Statistics: Num rows: 550 Data size: 28771 Basic stats:
COMPLETE Column stats: NONE
+ table:
+ input format: org.apache.hadoop.mapred.TextInputFormat
+ output format:
org.apache.hadoop.hive.ql.io.HiveIgnoreKeyTextOutputFormat
+ serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
+
+ Stage: Stage-0
+ Fetch Operator
+ limit: -1
+ Processor Tree:
+ ListSink
+
+PREHOOK: query: select test.key, test.value from test join test1 on (test.key
= test1.key) order by test.key
+PREHOOK: type: QUERY
+PREHOOK: Input: default@test
+PREHOOK: Input: default@test1
+PREHOOK: Input: default@test@p=1
+PREHOOK: Input: default@test@p=2
+#### A masked pattern was here ####
+POSTHOOK: query: select test.key, test.value from test join test1 on (test.key
= test1.key) order by test.key
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@test
+POSTHOOK: Input: default@test1
+POSTHOOK: Input: default@test@p=1
+POSTHOOK: Input: default@test@p=2
+#### A masked pattern was here ####
+0 val_0
+0 val_0
+0 val_0
+0 val_0
+0 val_0
+0 val_0
+0 val_0
+0 val_0
+0 val_0
+0 val_0
+0 val_0
+0 val_0
+0 val_0
+0 val_0
+0 val_0
+0 val_0
+0 val_0
+0 val_0
+2 val_2
+2 val_2
+4 val_4
+4 val_4
+5 val_5
+5 val_5
+5 val_5
+5 val_5
+5 val_5
+5 val_5
+5 val_5
+5 val_5
+5 val_5
+5 val_5
+5 val_5
+5 val_5
+5 val_5
+5 val_5
+5 val_5
+5 val_5
+5 val_5
+5 val_5
+8 val_8
+8 val_8
+9 val_9
+9 val_9
+10 val_10
+10 val_10
+11 val_11
+11 val_11
+12 val_12
+12 val_12
+12 val_12
+12 val_12
+12 val_12
+12 val_12
+12 val_12
+12 val_12
+15 val_15
+15 val_15
+15 val_15
+15 val_15
+15 val_15
+15 val_15
+15 val_15
+15 val_15
+17 val_17
+17 val_17
+18 val_18
+18 val_18
+18 val_18
+18 val_18
+18 val_18
+18 val_18
+18 val_18
+18 val_18
+19 val_19
+19 val_19
+20 val_20
+20 val_20
+24 val_24
+24 val_24
+24 val_24
+24 val_24
+24 val_24
+24 val_24
+24 val_24
+24 val_24
+26 val_26
+26 val_26
+26 val_26
+26 val_26
+26 val_26
+26 val_26
+26 val_26
+26 val_26
+27 val_27
+27 val_27
+28 val_28
+28 val_28
+30 val_30
+30 val_30
+33 val_33
+33 val_33
+34 val_34
+34 val_34
+35 val_35
+35 val_35
+35 val_35
+35 val_35
+35 val_35
+35 val_35
+35 val_35
+35 val_35
+35 val_35
+35 val_35
+35 val_35
+35 val_35
+35 val_35
+35 val_35
+35 val_35
+35 val_35
+35 val_35
+35 val_35
+37 val_37
+37 val_37
+37 val_37
+37 val_37
+37 val_37
+37 val_37
+37 val_37
+37 val_37
+41 val_41
+41 val_41
+42 val_42
+42 val_42
+42 val_42
+42 val_42
+42 val_42
+42 val_42
+42 val_42
+42 val_42
+43 val_43
+43 val_43
+44 val_44
+44 val_44
+47 val_47
+47 val_47
+51 val_51
+51 val_51
+51 val_51
+51 val_51
+51 val_51
+51 val_51
+51 val_51
+51 val_51
+53 val_53
+53 val_53
+54 val_54
+54 val_54
+57 val_57
+57 val_57
+58 val_58
+58 val_58
+58 val_58
+58 val_58
+58 val_58
+58 val_58
+58 val_58
+58 val_58
+64 val_64
+64 val_64
+65 val_65
+65 val_65
+66 val_66
+66 val_66
+67 val_67
+67 val_67
+67 val_67
+67 val_67
+67 val_67
+67 val_67
+67 val_67
+67 val_67
+69 val_69
+69 val_69
+70 val_70
+70 val_70
+70 val_70
+70 val_70
+70 val_70
+70 val_70
+70 val_70
+70 val_70
+70 val_70
+70 val_70
+70 val_70
+70 val_70
+70 val_70
+70 val_70
+70 val_70
+70 val_70
+70 val_70
+70 val_70
+72 val_72
+72 val_72
+72 val_72
+72 val_72
+72 val_72
+72 val_72
+72 val_72
+72 val_72
+74 val_74
+74 val_74
+76 val_76
+76 val_76
+76 val_76
+76 val_76
+76 val_76
+76 val_76
+76 val_76
+76 val_76
+77 val_77
+77 val_77
+78 val_78
+78 val_78
+80 val_80
+80 val_80
+82 val_82
+82 val_82
+83 val_83
+83 val_83
+83 val_83
+83 val_83
+83 val_83
+83 val_83
+83 val_83
+83 val_83
+84 val_84
+84 val_84
+84 val_84
+84 val_84
+84 val_84
+84 val_84
+84 val_84
+84 val_84
+85 val_85
+85 val_85
+86 val_86
+86 val_86
+87 val_87
+87 val_87
+90 val_90
+90 val_90
+90 val_90
+90 val_90
+90 val_90
+90 val_90
+90 val_90
+90 val_90
+90 val_90
+90 val_90
+90 val_90
+90 val_90
+90 val_90
+90 val_90
+90 val_90
+90 val_90
+90 val_90
+90 val_90
+92 val_92
+92 val_92
+95 val_95
+95 val_95
+95 val_95
+95 val_95
+95 val_95
+95 val_95
+95 val_95
+95 val_95
+96 val_96
+96 val_96
+97 val_97
+97 val_97
+97 val_97
+97 val_97
+97 val_97
+97 val_97
+97 val_97
+97 val_97
+98 val_98
+98 val_98
+98 val_98
+98 val_98
+98 val_98
+98 val_98
+98 val_98
+98 val_98
+100 val_100
+100 val_100
+100 val_100
+100 val_100
+100 val_100
+100 val_100
+100 val_100
+100 val_100
+103 val_103
+103 val_103
+103 val_103
+103 val_103
+103 val_103
+103 val_103
+103 val_103
+103 val_103
+104 val_104
+104 val_104
+104 val_104
+104 val_104
+104 val_104
+104 val_104
+104 val_104
+104 val_104
+105 val_105
+105 val_105
+111 val_111
+111 val_111
+113 val_113
+113 val_113
+113 val_113
+113 val_113
+113 val_113
+113 val_113
+113 val_113
+113 val_113
+114 val_114
+114 val_114
+116 val_116
+116 val_116
+118 val_118
+118 val_118
+118 val_118
+118 val_118
+118 val_118
+118 val_118
+118 val_118
+118 val_118
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+119 val_119
+120 val_120
+120 val_120
+120 val_120
+120 val_120
+120 val_120
+120 val_120
+120 val_120
+120 val_120
+125 val_125
+125 val_125
+125 val_125
+125 val_125
+125 val_125
+125 val_125
+125 val_125
+125 val_125
+126 val_126
+126 val_126
+128 val_128
+128 val_128
+128 val_128
+128 val_128
+128 val_128
+128 val_128
+128 val_128
+128 val_128
+128 val_128
+128 val_128
+128 val_128
+128 val_128
+128 val_128
+128 val_128
+128 val_128
+128 val_128
+128 val_128
+128 val_128
+129 val_129
+129 val_129
+129 val_129
+129 val_129
+129 val_129
+129 val_129
+129 val_129
+129 val_129
+131 val_131
+131 val_131
+133 val_133
+133 val_133
+134 val_134
+134 val_134
+134 val_134
+134 val_134
+134 val_134
+134 val_134
+134 val_134
+134 val_134
+136 val_136
+136 val_136
+137 val_137
+137 val_137
+137 val_137
+137 val_137
+137 val_137
+137 val_137
+137 val_137
+137 val_137
+138 val_138
+138 val_138
+138 val_138
+138 val_138
+138 val_138
+138 val_138
+138 val_138
+138 val_138
+138 val_138
+138 val_138
+138 val_138
+138 val_138
+138 val_138
+138 val_138
+138 val_138
+138 val_138
+138 val_138
+138 val_138
+138 val_138
+138 val_138
+138 val_138
+138 val_138
+138 val_138
+138 val_138
+138 val_138
+138 val_138
+138 val_138
+138 val_138
+138 val_138
+138 val_138
+138 val_138
+138 val_138
+143 val_143
+143 val_143
+145 val_145
+145 val_145
+146 val_146
+146 val_146
+146 val_146
+146 val_146
+146 val_146
+146 val_146
+146 val_146
+146 val_146
+149 val_149
+149 val_149
+149 val_149
+149 val_149
+149 val_149
+149 val_149
+149 val_149
+149 val_149
+150 val_150
+150 val_150
+152 val_152
+152 val_152
+152 val_152
+152 val_152
+152 val_152
+152 val_152
+152 val_152
+152 val_152
+153 val_153
+153 val_153
+155 val_155
+155 val_155
+156 val_156
+156 val_156
+157 val_157
+157 val_157
+158 val_158
+158 val_158
+160 val_160
+160 val_160
+162 val_162
+162 val_162
+163 val_163
+163 val_163
+164 val_164
+164 val_164
+164 val_164
+164 val_164
+164 val_164
+164 val_164
+164 val_164
+164 val_164
+165 val_165
+165 val_165
+165 val_165
+165 val_165
+165 val_165
+165 val_165
+165 val_165
+165 val_165
+166 val_166
+166 val_166
+167 val_167
+167 val_167
+167 val_167
+167 val_167
+167 val_167
+167 val_167
+167 val_167
+167 val_167
+167 val_167
+167 val_167
+167 val_167
+167 val_167
+167 val_167
+167 val_167
+167 val_167
+167 val_167
+167 val_167
+167 val_167
+168 val_168
+168 val_168
+169 val_169
+169 val_169
+169 val_169
+169 val_169
+169 val_169
+169 val_169
+169 val_169
+169 val_169
+169 val_169
+169 val_169
+169 val_169
+169 val_169
+169 val_169
+169 val_169
+169 val_169
+169 val_169
+169 val_169
+169 val_169
+169 val_169
+169 val_169
+169 val_169
+169 val_169
+169 val_169
+169 val_169
+169 val_169
+169 val_169
+169 val_169
+169 val_169
+169 val_169
+169 val_169
+169 val_169
+169 val_169
+170 val_170
+170 val_170
+172 val_172
+172 val_172
+172 val_172
+172 val_172
+172 val_172
+172 val_172
+172 val_172
+172 val_172
+174 val_174
+174 val_174
+174 val_174
+174 val_174
+174 val_174
+174 val_174
+174 val_174
+174 val_174
+175 val_175
+175 val_175
+175 val_175
+175 val_175
+175 val_175
+175 val_175
+175 val_175
+175 val_175
+176 val_176
+176 val_176
+176 val_176
+176 val_176
+176 val_176
+176 val_176
+176 val_176
+176 val_176
+177 val_177
+177 val_177
+178 val_178
+178 val_178
+179 val_179
+179 val_179
+179 val_179
+179 val_179
+179 val_179
+179 val_179
+179 val_179
+179 val_179
+180 val_180
+180 val_180
+181 val_181
+181 val_181
+183 val_183
+183 val_183
+186 val_186
+186 val_186
+187 val_187
+187 val_187
+187 val_187
+187 val_187
+187 val_187
+187 val_187
+187 val_187
+187 val_187
+187 val_187
+187 val_187
+187 val_187
+187 val_187
+187 val_187
+187 val_187
+187 val_187
+187 val_187
+187 val_187
+187 val_187
+189 val_189
+189 val_189
+190 val_190
+190 val_190
+191 val_191
+191 val_191
+191 val_191
+191 val_191
+191 val_191
+191 val_191
+191 val_191
+191 val_191
+192 val_192
+192 val_192
+193 val_193
+193 val_193
+193 val_193
+193 val_193
+193 val_193
+193 val_193
+193 val_193
+193 val_193
+193 val_193
+193 val_193
+193 val_193
+193 val_193
+193 val_193
+193 val_193
+193 val_193
+193 val_193
+193 val_193
+193 val_193
+194 val_194
+194 val_194
+195 val_195
+195 val_195
+195 val_195
+195 val_195
+195 val_195
+195 val_195
+195 val_195
+195 val_195
+196 val_196
+196 val_196
+197 val_197
+197 val_197
+197 val_197
+197 val_197
+197 val_197
+197 val_197
+197 val_197
+197 val_197
+199 val_199
+199 val_199
+199 val_199
+199 val_199
+199 val_199
+199 val_199
+199 val_199
+199 val_199
+199 val_199
+199 val_199
+199 val_199
+199 val_199
+199 val_199
+199 val_199
+199 val_199
+199 val_199
+199 val_199
+199 val_199
+200 val_200
+200 val_200
+200 val_200
+200 val_200
+200 val_200
+200 val_200
+200 val_200
+200 val_200
+201 val_201
+201 val_201
+202 val_202
+202 val_202
+203 val_203
+203 val_203
+203 val_203
+203 val_203
+203 val_203
+203 val_203
+203 val_203
+203 val_203
+205 val_205
+205 val_205
+205 val_205
+205 val_205
+205 val_205
+205 val_205
+205 val_205
+205 val_205
+207 val_207
+207 val_207
+207 val_207
+207 val_207
+207 val_207
+207 val_207
+207 val_207
+207 val_207
+208 val_208
+208 val_208
+208 val_208
+208 val_208
+208 val_208
+208 val_208
+208 val_208
+208 val_208
+208 val_208
+208 val_208
+208 val_208
+208 val_208
+208 val_208
+208 val_208
+208 val_208
+208 val_208
+208 val_208
+208 val_208
+209 val_209
+209 val_209
+209 val_209
+209 val_209
+209 val_209
+209 val_209
+209 val_209
+209 val_209
+213 val_213
+213 val_213
+213 val_213
+213 val_213
+213 val_213
+213 val_213
+213 val_213
+213 val_213
+214 val_214
+214 val_214
+216 val_216
+216 val_216
+216 val_216
+216 val_216
+216 val_216
+216 val_216
+216 val_216
+216 val_216
+217 val_217
+217 val_217
+217 val_217
+217 val_217
+217 val_217
+217 val_217
+217 val_217
+217 val_217
+218 val_218
+218 val_218
+219 val_219
+219 val_219
+219 val_219
+219 val_219
+219 val_219
+219 val_219
+219 val_219
+219 val_219
+221 val_221
+221 val_221
+221 val_221
+221 val_221
+221 val_221
+221 val_221
+221 val_221
+221 val_221
+222 val_222
+222 val_222
+223 val_223
+223 val_223
+223 val_223
+223 val_223
+223 val_223
+223 val_223
+223 val_223
+223 val_223
+224 val_224
+224 val_224
+224 val_224
+224 val_224
+224 val_224
+224 val_224
+224 val_224
+224 val_224
+226 val_226
+226 val_226
+228 val_228
+228 val_228
+229 val_229
+229 val_229
+229 val_229
+229 val_229
+229 val_229
+229 val_229
+229 val_229
+229 val_229
+230 val_230
+230 val_230
+230 val_230
+230 val_230
+230 val_230
+230 val_230
+230 val_230
+230 val_230
+230 val_230
+230 val_230
+230 val_230
+230 val_230
+230 val_230
+230 val_230
+230 val_230
+230 val_230
+230 val_230
+230 val_230
+230 val_230
+230 val_230
+230 val_230
+230 val_230
+230 val_230
+230 val_230
+230 val_230
+230 val_230
+230 val_230
+230 val_230
+230 val_230
+230 val_230
+230 val_230
+230 val_230
+230 val_230
+230 val_230
+230 val_230
+230 val_230
+230 val_230
+230 val_230
+230 val_230
+230 val_230
+230 val_230
+230 val_230
+230 val_230
+230 val_230
+230 val_230
+230 val_230
+230 val_230
+230 val_230
+230 val_230
+230 val_230
+233 val_233
+233 val_233
+233 val_233
+233 val_233
+233 val_233
+233 val_233
+233 val_233
+233 val_233
+235 val_235
+235 val_235
+237 val_237
+237 val_237
+237 val_237
+237 val_237
+237 val_237
+237 val_237
+237 val_237
+237 val_237
+238 val_238
+238 val_238
+238 val_238
+238 val_238
+238 val_238
+238 val_238
+238 val_238
+238 val_238
+239 val_239
+239 val_239
+239 val_239
+239 val_239
+239 val_239
+239 val_239
+239 val_239
+239 val_239
+241 val_241
+241 val_241
+242 val_242
+242 val_242
+242 val_242
+242 val_242
+242 val_242
+242 val_242
+242 val_242
+242 val_242
+244 val_244
+244 val_244
+247 val_247
+247 val_247
+248 val_248
+248 val_248
+249 val_249
+249 val_249
+252 val_252
+252 val_252
+255 val_255
+255 val_255
+255 val_255
+255 val_255
+255 val_255
+255 val_255
+255 val_255
+255 val_255
+256 val_256
+256 val_256
+256 val_256
+256 val_256
+256 val_256
+256 val_256
+256 val_256
+256 val_256
+257 val_257
+257 val_257
+258 val_258
+258 val_258
+260 val_260
+260 val_260
+262 val_262
+262 val_262
+263 val_263
+263 val_263
+265 val_265
+265 val_265
+265 val_265
+265 val_265
+265 val_265
+265 val_265
+265 val_265
+265 val_265
+266 val_266
+266 val_266
+272 val_272
+272 val_272
+272 val_272
+272 val_272
+272 val_272
+272 val_272
+272 val_272
+272 val_272
+273 val_273
+273 val_273
+273 val_273
+273 val_273
+273 val_273
+273 val_273
+273 val_273
+273 val_273
+273 val_273
+273 val_273
+273 val_273
+273 val_273
+273 val_273
+273 val_273
+273 val_273
+273 val_273
+273 val_273
+273 val_273
+274 val_274
+274 val_274
+275 val_275
+275 val_275
+277 val_277
+277 val_277
+277 val_277
+277 val_277
+277 val_277
+277 val_277
+277 val_277
+277 val_277
+277 val_277
+277 val_277
+277 val_277
+277 val_277
+277 val_277
+277 val_277
+277 val_277
+277 val_277
+277 val_277
+277 val_277
+277 val_277
+277 val_277
+277 val_277
+277 val_277
+277 val_277
+277 val_277
+277 val_277
+277 val_277
+277 val_277
+277 val_277
+277 val_277
+277 val_277
+277 val_277
+277 val_277
+278 val_278
+278 val_278
+278 val_278
+278 val_278
+278 val_278
+278 val_278
+278 val_278
+278 val_278
+280 val_280
+280 val_280
+280 val_280
+280 val_280
+280 val_280
+280 val_280
+280 val_280
+280 val_280
+281 val_281
+281 val_281
+281 val_281
+281 val_281
+281 val_281
+281 val_281
+281 val_281
+281 val_281
+282 val_282
+282 val_282
+282 val_282
+282 val_282
+282 val_282
+282 val_282
+282 val_282
+282 val_282
+283 val_283
+283 val_283
+284 val_284
+284 val_284
+285 val_285
+285 val_285
+286 val_286
+286 val_286
+287 val_287
+287 val_287
+288 val_288
+288 val_288
+288 val_288
+288 val_288
+288 val_288
+288 val_288
+288 val_288
+288 val_288
+289 val_289
+289 val_289
+291 val_291
+291 val_291
+292 val_292
+292 val_292
+296 val_296
+296 val_296
+298 val_298
+298 val_298
+298 val_298
+298 val_298
+298 val_298
+298 val_298
+298 val_298
+298 val_298
+298 val_298
+298 val_298
+298 val_298
+298 val_298
+298 val_298
+298 val_298
+298 val_298
+298 val_298
+298 val_298
+298 val_298
+302 val_302
+302 val_302
+305 val_305
+305 val_305
+306 val_306
+306 val_306
+307 val_307
+307 val_307
+307 val_307
+307 val_307
+307 val_307
+307 val_307
+307 val_307
+307 val_307
+308 val_308
+308 val_308
+309 val_309
+309 val_309
+309 val_309
+309 val_309
+309 val_309
+309 val_309
+309 val_309
+309 val_309
+310 val_310
+310 val_310
+311 val_311
+311 val_311
+311 val_311
+311 val_311
+311 val_311
+311 val_311
+311 val_311
+311 val_311
+311 val_311
+311 val_311
+311 val_311
+311 val_311
+311 val_311
+311 val_311
+311 val_311
+311 val_311
+311 val_311
+311 val_311
+315 val_315
+315 val_315
+316 val_316
+316 val_316
+316 val_316
+316 val_316
+316 val_316
+316 val_316
+316 val_316
+316 val_316
+316 val_316
+316 val_316
+316 val_316
+316 val_316
+316 val_316
+316 val_316
+316 val_316
+316 val_316
+316 val_316
+316 val_316
+317 val_317
+317 val_317
+317 val_317
+317 val_317
+317 val_317
+317 val_317
+317 val_317
+317 val_317
+318 val_318
+318 val_318
+318 val_318
+318 val_318
+318 val_318
+318 val_318
+318 val_318
+318 val_318
+318 val_318
+318 val_318
+318 val_318
+318 val_318
+318 val_318
+318 val_318
+318 val_318
+318 val_318
+318 val_318
+318 val_318
+321 val_321
+321 val_321
+321 val_321
+321 val_321
+321 val_321
+321 val_321
+321 val_321
+321 val_321
+322 val_322
+322 val_322
+322 val_322
+322 val_322
+322 val_322
+322 val_322
+322 val_322
+322 val_322
+323 val_323
+323 val_323
+325 val_325
+325 val_325
+325 val_325
+325 val_325
+325 val_325
+325 val_325
+325 val_325
+325 val_325
+327 val_327
+327 val_327
+327 val_327
+327 val_327
+327 val_327
+327 val_327
+327 val_327
+327 val_327
+327 val_327
+327 val_327
+327 val_327
+327 val_327
+327 val_327
+327 val_327
+327 val_327
+327 val_327
+327 val_327
+327 val_327
+331 val_331
+331 val_331
+331 val_331
+331 val_331
+331 val_331
+331 val_331
+331 val_331
+331 val_331
+332 val_332
+332 val_332
+333 val_333
+333 val_333
+333 val_333
+333 val_333
+333 val_333
+333 val_333
+333 val_333
+333 val_333
+335 val_335
+335 val_335
+336 val_336
+336 val_336
+338 val_338
+338 val_338
+339 val_339
+339 val_339
+341 val_341
+341 val_341
+342 val_342
+342 val_342
+342 val_342
+342 val_342
+342 val_342
+342 val_342
+342 val_342
+342 val_342
+344 val_344
+344 val_344
+344 val_344
+344 val_344
+344 val_344
+344 val_344
+344 val_344
+344 val_344
+345 val_345
+345 val_345
+348 val_348
+348 val_348
+348 val_348
+348 val_348
+348 val_348
+348 val_348
+348 val_348
+348 val_348
+348 val_348
+348 val_348
+348 val_348
+348 val_348
+348 val_348
+348 val_348
+348 val_348
+348 val_348
+348 val_348
+348 val_348
+348 val_348
+348 val_348
+348 val_348
+348 val_348
+348 val_348
+348 val_348
+348 val_348
+348 val_348
+348 val_348
+348 val_348
+348 val_348
+348 val_348
+348 val_348
+348 val_348
+348 val_348
+348 val_348
+348 val_348
+348 val_348
+348 val_348
+348 val_348
+348 val_348
+348 val_348
+348 val_348
+348 val_348
+348 val_348
+348 val_348
+348 val_348
+348 val_348
+348 val_348
+348 val_348
+348 val_348
+348 val_348
+351 val_351
+351 val_351
+353 val_353
+353 val_353
+353 val_353
+353 val_353
+353 val_353
+353 val_353
+353 val_353
+353 val_353
+356 val_356
+356 val_356
+360 val_360
+360 val_360
+362 val_362
+362 val_362
+364 val_364
+364 val_364
+365 val_365
+365 val_365
+366 val_366
+366 val_366
+367 val_367
+367 val_367
+367 val_367
+367 val_367
+367 val_367
+367 val_367
+367 val_367
+367 val_367
+368 val_368
+368 val_368
+369 val_369
+369 val_369
+369 val_369
+369 val_369
+369 val_369
+369 val_369
+369 val_369
+369 val_369
+369 val_369
+369 val_369
+369 val_369
+369 val_369
+369 val_369
+369 val_369
+369 val_369
+369 val_369
+369 val_369
+369 val_369
+373 val_373
+373 val_373
+374 val_374
+374 val_374
+375 val_375
+375 val_375
+377 val_377
+377 val_377
+378 val_378
+378 val_378
+379 val_379
+379 val_379
+382 val_382
+382 val_382
+382 val_382
+382 val_382
+382 val_382
+382 val_382
+382 val_382
+382 val_382
+384 val_384
+384 val_384
+384 val_384
+384 val_384
+384 val_384
+384 val_384
+384 val_384
+384 val_384
+384 val_384
+384 val_384
+384 val_384
+384 val_384
+384 val_384
+384 val_384
+384 val_384
+384 val_384
+384 val_384
+384 val_384
+386 val_386
+386 val_386
+389 val_389
+389 val_389
+392 val_392
+392 val_392
+393 val_393
+393 val_393
+394 val_394
+394 val_394
+395 val_395
+395 val_395
+395 val_395
+395 val_395
+395 val_395
+395 val_395
+395 val_395
+395 val_395
+396 val_396
+396 val_396
+396 val_396
+396 val_396
+396 val_396
+396 val_396
+396 val_396
+396 val_396
+396 val_396
+396 val_396
+396 val_396
+396 val_396
+396 val_396
+396 val_396
+396 val_396
+396 val_396
+396 val_396
+396 val_396
+397 val_397
+397 val_397
+397 val_397
+397 val_397
+397 val_397
+397 val_397
+397 val_397
+397 val_397
+399 val_399
+399 val_399
+399 val_399
+399 val_399
+399 val_399
+399 val_399
+399 val_399
+399 val_399
+400 val_400
+400 val_400
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+401 val_401
+402 val_402
+402 val_402
+403 val_403
+403 val_403
+403 val_403
+403 val_403
+403 val_403
+403 val_403
+403 val_403
+403 val_403
+403 val_403
+403 val_403
+403 val_403
+403 val_403
+403 val_403
+403 val_403
+403 val_403
+403 val_403
+403 val_403
+403 val_403
+404 val_404
+404 val_404
+404 val_404
+404 val_404
+404 val_404
+404 val_404
+404 val_404
+404 val_404
+406 val_406
+406 val_406
+406 val_406
+406 val_406
+406 val_406
+406 val_406
+406 val_406
+406 val_406
+406 val_406
+406 val_406
+406 val_406
+406 val_406
+406 val_406
+406 val_406
+406 val_406
+406 val_406
+406 val_406
+406 val_406
+406 val_406
+406 val_406
+406 val_406
+406 val_406
+406 val_406
+406 val_406
+406 val_406
+406 val_406
+406 val_406
+406 val_406
+406 val_406
+406 val_406
+406 val_406
+406 val_406
+407 val_407
+407 val_407
+409 val_409
+409 val_409
+409 val_409
+409 val_409
+409 val_409
+409 val_409
+409 val_409
+409 val_409
+409 val_409
+409 val_409
+409 val_409
+409 val_409
+409 val_409
+409 val_409
+409 val_409
+409 val_409
+409 val_409
+409 val_409
+411 val_411
+411 val_411
+413 val_413
+413 val_413
+413 val_413
+413 val_413
+413 val_413
+413 val_413
+413 val_413
+413 val_413
+414 val_414
+414 val_414
+414 val_414
+414 val_414
+414 val_414
+414 val_414
+414 val_414
+414 val_414
+417 val_417
+417 val_417
+417 val_417
+417 val_417
+417 val_417
+417 val_417
+417 val_417
+417 val_417
+417 val_417
+417 val_417
+417 val_417
+417 val_417
+417 val_417
+417 val_417
+417 val_417
+417 val_417
+417 val_417
+417 val_417
+418 val_418
+418 val_418
+419 val_419
+419 val_419
+421 val_421
+421 val_421
+424 val_424
+424 val_424
+424 val_424
+424 val_424
+424 val_424
+424 val_424
+424 val_424
+424 val_424
+427 val_427
+427 val_427
+429 val_429
+429 val_429
+429 val_429
+429 val_429
+429 val_429
+429 val_429
+429 val_429
+429 val_429
+430 val_430
+430 val_430
+430 val_430
+430 val_430
+430 val_430
+430 val_430
+430 val_430
+430 val_430
+430 val_430
+430 val_430
+430 val_430
+430 val_430
+430 val_430
+430 val_430
+430 val_430
+430 val_430
+430 val_430
+430 val_430
+431 val_431
+431 val_431
+431 val_431
+431 val_431
+431 val_431
+431 val_431
+431 val_431
+431 val_431
+431 val_431
+431 val_431
+431 val_431
+431 val_431
+431 val_431
+431 val_431
+431 val_431
+431 val_431
+431 val_431
+431 val_431
+432 val_432
+432 val_432
+435 val_435
+435 val_435
+436 val_436
+436 val_436
+437 val_437
+437 val_437
+438 val_438
+438 val_438
+438 val_438
+438 val_438
+438 val_438
+438 val_438
+438 val_438
+438 val_438
+438 val_438
+438 val_438
+438 val_438
+438 val_438
+438 val_438
+438 val_438
+438 val_438
+438 val_438
+438 val_438
+438 val_438
+439 val_439
+439 val_439
+439 val_439
+439 val_439
+439 val_439
+439 val_439
+439 val_439
+439 val_439
+443 val_443
+443 val_443
+444 val_444
+444 val_444
+446 val_446
+446 val_446
+448 val_448
+448 val_448
+449 val_449
+449 val_449
+452 val_452
+452 val_452
+453 val_453
+453 val_453
+454 val_454
+454 val_454
+454 val_454
+454 val_454
+454 val_454
+454 val_454
+454 val_454
+454 val_454
+454 val_454
+454 val_454
+454 val_454
+454 val_454
+454 val_454
+454 val_454
+454 val_454
+454 val_454
+454 val_454
+454 val_454
+455 val_455
+455 val_455
+457 val_457
+457 val_457
+458 val_458
+458 val_458
+458 val_458
+458 val_458
+458 val_458
+458 val_458
+458 val_458
+458 val_458
+459 val_459
+459 val_459
+459 val_459
+459 val_459
+459 val_459
+459 val_459
+459 val_459
+459 val_459
+460 val_460
+460 val_460
+462 val_462
+462 val_462
+462 val_462
+462 val_462
+462 val_462
+462 val_462
+462 val_462
+462 val_462
+463 val_463
+463 val_463
+463 val_463
+463 val_463
+463 val_463
+463 val_463
+463 val_463
+463 val_463
+466 val_466
+466 val_466
+466 val_466
+466 val_466
+466 val_466
+466 val_466
+466 val_466
+466 val_466
+466 val_466
+466 val_466
+466 val_466
+466 val_466
+466 val_466
+466 val_466
+466 val_466
+466 val_466
+466 val_466
+466 val_466
+467 val_467
+467 val_467
+468 val_468
+468 val_468
+468 val_468
+468 val_468
+468 val_468
+468 val_468
+468 val_468
+468 val_468
+468 val_468
+468 val_468
+468 val_468
+468 val_468
+468 val_468
+468 val_468
+468 val_468
+468 val_468
+468 val_468
+468 val_468
+468 val_468
+468 val_468
+468 val_468
+468 val_468
+468 val_468
+468 val_468
+468 val_468
+468 val_468
+468 val_468
+468 val_468
+468 val_468
+468 val_468
+468 val_468
+468 val_468
+469 val_469
+469 val_469
+469 val_469
+469 val_469
+469 val_469
+469 val_469
+469 val_469
+469 val_469
+469 val_469
+469 val_469
+469 val_469
+469 val_469
+469 val_469
+469 val_469
+469 val_469
+469 val_469
+469 val_469
+469 val_469
+469 val_469
+469 val_469
+469 val_469
+469 val_469
+469 val_469
+469 val_469
+469 val_469
+469 val_469
+469 val_469
+469 val_469
+469 val_469
+469 val_469
+469 val_469
+469 val_469
+469 val_469
+469 val_469
+469 val_469
+469 val_469
+469 val_469
+469 val_469
+469 val_469
+469 val_469
+469 val_469
+469 val_469
+469 val_469
+469 val_469
+469 val_469
+469 val_469
+469 val_469
+469 val_469
+469 val_469
+469 val_469
+470 val_470
+470 val_470
+472 val_472
+472 val_472
+475 val_475
+475 val_475
+477 val_477
+477 val_477
+478 val_478
+478 val_478
+478 val_478
+478 val_478
+478 val_478
+478 val_478
+478 val_478
+478 val_478
+479 val_479
+479 val_479
+480 val_480
+480 val_480
+480 val_480
+480 val_480
+480 val_480
+480 val_480
+480 val_480
+480 val_480
+480 val_480
+480 val_480
+480 val_480
+480 val_480
+480 val_480
+480 val_480
+480 val_480
+480 val_480
+480 val_480
+480 val_480
+481 val_481
+481 val_481
+482 val_482
+482 val_482
+483 val_483
+483 val_483
+484 val_484
+484 val_484
+485 val_485
+485 val_485
+487 val_487
+487 val_487
+489 val_489
+489 val_489
+489 val_489
+489 val_489
+489 val_489
+489 val_489
+489 val_489
+489 val_489
+489 val_489
+489 val_489
+489 val_489
+489 val_489
+489 val_489
+489 val_489
+489 val_489
+489 val_489
+489 val_489
+489 val_489
+489 val_489
+489 val_489
+489 val_489
+489 val_489
+489 val_489
+489 val_489
+489 val_489
+489 val_489
+489 val_489
+489 val_489
+489 val_489
+489 val_489
+489 val_489
+489 val_489
+490 val_490
+490 val_490
+491 val_491
+491 val_491
+492 val_492
+492 val_492
+492 val_492
+492 val_492
+492 val_492
+492 val_492
+492 val_492
+492 val_492
+493 val_493
+493 val_493
+494 val_494
+494 val_494
+495 val_495
+495 val_495
+496 val_496
+496 val_496
+497 val_497
+497 val_497
+498 val_498
+498 val_498
+498 val_498
+498 val_498
+498 val_498
+498 val_498
+498 val_498
+498 val_498
+498 val_498
+498 val_498
+498 val_498
+498 val_498
+498 val_498
+498 val_498
+498 val_498
+498 val_498
+498 val_498
+498 val_498
