Author: ekoifman Date: Tue Jan 20 23:26:26 2015 New Revision: 1653406 URL: http://svn.apache.org/r1653406 Log: HIVE-9272 Tests for utf-8 support (Aswathy Chellammal Sreekumar via Eugene Koifman)
Added: hive/trunk/hcatalog/src/test/e2e/templeton/inpdir/PigJoin䶴ㄩ鼾丄狜〇work.pig" hive/trunk/hcatalog/src/test/e2e/templeton/inpdir/artof䶴ㄩ鼾丄狜〇war.txt" hive/trunk/hcatalog/src/test/e2e/templeton/inpdir/table1.txt hive/trunk/hcatalog/src/test/e2e/templeton/inpdir/table3.txt hive/trunk/hcatalog/src/test/e2e/templeton/inpdir/table3ToJoin.txt hive/trunk/hcatalog/src/test/e2e/templeton/tests/utf8.conf Modified: hive/trunk/hcatalog/src/test/e2e/templeton/build.xml hive/trunk/hcatalog/src/test/e2e/templeton/deployers/deploy_e2e_artifacts.sh hive/trunk/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm Modified: hive/trunk/hcatalog/src/test/e2e/templeton/build.xml URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/src/test/e2e/templeton/build.xml?rev=1653406&r1=1653405&r2=1653406&view=diff ============================================================================== --- hive/trunk/hcatalog/src/test/e2e/templeton/build.xml (original) +++ hive/trunk/hcatalog/src/test/e2e/templeton/build.xml Tue Jan 20 23:26:26 2015 @@ -120,6 +120,7 @@ <arg value="${basedir}/tests/ddl.conf"/> <arg value="${basedir}/tests/jobsubmission.conf"/> <arg value="${basedir}/tests/jobsubmission2.conf"/> + <arg value="${basedir}/tests/utf8.conf"/> </exec> </target> Modified: hive/trunk/hcatalog/src/test/e2e/templeton/deployers/deploy_e2e_artifacts.sh URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/src/test/e2e/templeton/deployers/deploy_e2e_artifacts.sh?rev=1653406&r1=1653405&r2=1653406&view=diff ============================================================================== --- hive/trunk/hcatalog/src/test/e2e/templeton/deployers/deploy_e2e_artifacts.sh (original) +++ hive/trunk/hcatalog/src/test/e2e/templeton/deployers/deploy_e2e_artifacts.sh Tue Jan 20 23:26:26 2015 @@ -32,6 +32,8 @@ ${HADOOP_HOME}/bin/hdfs dfs -put ${PROJ_ #For hadoop2 there are 2 separate jars ${HADOOP_HOME}/bin/hdfs dfs -put ${HADOOP_HOME}/share/hadoop/mapreduce/hadoop-mapreduce-examples-${HADOOP_VERSION}.jar webhcate2e/hexamples.jar +#For utf8 test(for mapreduce) we need a jar with utf-8 characters in the name +${HADOOP_HOME}/bin/hdfs dfs -put ${HADOOP_HOME}/share/hadoop/mapreduce/hadoop-mapreduce-examples-${HADOOP_VERSION}.jar webhcate2e/hadoop_examples_䶴ã©é¼¾ä¸çã_2_2_0.jar ${HADOOP_HOME}/bin/hdfs dfs -put ${HADOOP_HOME}/share/hadoop/mapreduce/hadoop-mapreduce-client-jobclient-${HADOOP_VERSION}.jar webhcate2e/hclient.jar ${HADOOP_HOME}/bin/hdfs dfs -put ${HADOOP_HOME}/share/hadoop/tools/lib/hadoop-streaming-${HADOOP_VERSION}.jar /user/templeton/hadoop-streaming.jar Modified: hive/trunk/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm?rev=1653406&r1=1653405&r2=1653406&view=diff ============================================================================== --- hive/trunk/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm (original) +++ hive/trunk/hcatalog/src/test/e2e/templeton/drivers/TestDriverCurl.pm Tue Jan 20 23:26:26 2015 @@ -37,6 +37,7 @@ use English; use Storable qw(dclone); use File::Glob ':glob'; use JSON::Path; +use utf8; my $passedStr = 'passed'; my $failedStr = 'failed'; @@ -922,12 +923,38 @@ sub compare $result = 0; next; } - + my $exp_userargsvalue; + my $r_userargsvalue; + if(ref($exp_userargs{$key}) eq "ARRAY"){ + my @values = $exp_userargs{$key}; + my $num_values = @values; + + for(my $i=0;$i<=$num_values;$i++){ + if (utf8::is_utf8($exp_userargs{$key}[$i])){ + $exp_userargs{$key}[$i] = utf8::decode($exp_userargs{$key}[$i]); + $r_userargs{$key}[$i] = utf8::decode($r_userargs{$key}[$i]); + } + } + $exp_userargsvalue = $exp_userargs{$key}; + $r_userargsvalue = $r_userargs{$key}; + } + else { + if (utf8::is_utf8($exp_userargs{$key})) + { + $exp_userargsvalue = utf8::decode($exp_userargs{$key}); + $r_userargsvalue = utf8::decode($r_userargs{$key}); + } + else + { + $exp_userargsvalue = $exp_userargs{$key}; + $r_userargsvalue = $r_userargs{$key}; + } + } print $log "$0::$subName DEBUG comparing expected " . " $key ->" . dump($exp_userargs{$key}) . " With result $key ->" . dump($r_userargs{$key}) . "\n"; - if (!Compare($exp_userargs{$key}, $r_userargs{$key})) { + if (!Compare($exp_userargsvalue, $r_userargsvalue)) { print $log "$0::$subName WARN check failed:" . " json compare failed. For field " . "$key, regex <" . dump($r_userargs{$key}) Added: hive/trunk/hcatalog/src/test/e2e/templeton/inpdir/PigJoin䶴ㄩ鼾丄狜〇work.pig" URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/src/test/e2e/templeton/inpdir/PigJoin%E4%B6%B4%E3%84%A9%E9%BC%BE%E4%B8%84%E7%8B%9C%E3%80%87work.pig%22?rev=1653406&view=auto ============================================================================== --- hive/trunk/hcatalog/src/test/e2e/templeton/inpdir/PigJoin䶴ㄩ鼾丄狜〇work.pig" (added) +++ hive/trunk/hcatalog/src/test/e2e/templeton/inpdir/PigJoin䶴ㄩ鼾丄狜〇work.pig" Tue Jan 20 23:26:26 2015 @@ -0,0 +1,4 @@ +A = load '$INPDIR/table3.txt' using PigStorage('\t') AS (row:int, content:chararray); +B = load '$INPDIR/table3ToJoin.txt' using PigStorage('\t') AS (row:int, content:chararray); +C = JOIN A BY content, B BY content; +store C into '$OUTDIR/PigJoin' USING PigStorage(); \ No newline at end of file Added: hive/trunk/hcatalog/src/test/e2e/templeton/inpdir/artof䶴ㄩ鼾丄狜〇war.txt" URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/src/test/e2e/templeton/inpdir/artof%E4%B6%B4%E3%84%A9%E9%BC%BE%E4%B8%84%E7%8B%9C%E3%80%87war.txt%22?rev=1653406&view=auto ============================================================================== --- hive/trunk/hcatalog/src/test/e2e/templeton/inpdir/artof䶴ㄩ鼾丄狜〇war.txt" (added) +++ hive/trunk/hcatalog/src/test/e2e/templeton/inpdir/artof䶴ㄩ鼾丄狜〇war.txt" Tue Jan 20 23:26:26 2015 @@ -0,0 +1,21 @@ +å« å æ° ï¼ å µ è ï¼ å ä¹ å¤§ äº ï¼ æ» ç ä¹ å° ï¼ å 亡 ä¹ é ï¼ ä¸ å¯ ä¸ å¯ ä¹ ã + + æ ç¶ ä¹ ä»¥ äº ï¼ æ ¡ ä¹ ä»¥ è¨ ï¼ è ç´¢ å ¶ æ ï¼ ä¸ æ° é ï¼ äº æ° å¤© ï¼ ä¸ æ° å° ï¼ å æ° + å° ï¼ äº æ° æ³ ã é è ï¼ ä»¤ æ° äº ä¸ å æ è ä¹ ï¼ å¯ è ä¹ æ» ï¼ å¯ è ä¹ ç ï¼ æ° ä¸ + è© ä¹ ã 天 è ï¼ é° é½ ã å¯ æ ã æ å¶ ä¹ ã å° è ï¼ é« ä¸ ã é è¿ ã éª æ ã 廣 ç¹ + ã æ» ç ä¹ ã å° è ï¼ æº ã ä¿¡ ã ä» ã å ã å´ ä¹ ã æ³ è ï¼ æ² å¶ ã å® é ã 主 ç¨ + ä¹ ã å¡ æ¤ äº è ï¼ å° è« ä¸ è ï¼ ç¥ ä¹ è å ï¼ ä¸ ç¥ ä¹ è ä¸ å ã æ æ ¡ ä¹ ä»¥ è¨ ï¼ + è ç´¢ å ¶ æ ã æ° ï¼ ä¸» å° æ é ï¼ å° å° æ è½ ï¼ å¤© å° å° å¾ ï¼ æ³ ä»¤ å° è¡ ï¼ å µ ç¾ å° + å¼· ï¼ å£« å å° ç·´ ï¼ è³ ç½° å° æ ï¼ å¾ ä»¥ æ¤ ç¥ å è² ç£ ã + + å° è½ å¾ è¨ ï¼ ç¨ ä¹ å¿ å ï¼ ç ä¹ ï¹ å° ä¸ è½ å¾ è¨ ï¼ ç¨ ä¹ å¿ æ ï¼ å» ä¹ ã + + è¨ å© ä»¥ è½ ï¼ ä¹ çº ä¹ å¢ ï¼ ä»¥ ä½ å ¶ å¤ ã å¢ è ï¼ å å© è å¶ æ¬ ä¹ ã + + å µ è ï¼ è© é ä¹ ã æ è½ è 示 ä¹ ä¸ è½ ï¼ ç¨ è 示 ä¹ ä¸ ç¨ ï¼ è¿ è 示 ä¹ é ï¼ é è + 示 ä¹ è¿ ã å© è èª ä¹ ï¼ äº è å ä¹ ï¼ å¯¦ è å ä¹ ï¼ å¼· è é¿ ä¹ ï¼ æ è æ ä¹ ï¼ å + è é© ä¹ ï¼ ä½ è å ä¹ ï¼ è¦ª è é¢ ä¹ ï¼ æ» å ¶ ä¸ å ï¼ åº å ¶ ä¸ æ ã æ¤ å µ 家 ä¹ å ï¼ + ä¸ å¯ å å³ ä¹ ã + + 夫 æª æ° è å» ç® å è ï¼ å¾ ç® å¤ ä¹ ï¹ æª æ° è å» ç® ä¸ å è ï¼ å¾ ç® å° ä¹ ã å¤ ç® + å ï¼ å° ç® ä¸ å ï¼ è æ³ ç¡ ç® ä¹ ï¼ å¾ ä»¥ æ¤ è§ ä¹ ï¼ å è² è¦ ç£ ã \ No newline at end of file Added: hive/trunk/hcatalog/src/test/e2e/templeton/inpdir/table1.txt URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/src/test/e2e/templeton/inpdir/table1.txt?rev=1653406&view=auto ============================================================================== --- hive/trunk/hcatalog/src/test/e2e/templeton/inpdir/table1.txt (added) +++ hive/trunk/hcatalog/src/test/e2e/templeton/inpdir/table1.txt Tue Jan 20 23:26:26 2015 @@ -0,0 +1,27 @@ +868 ç鼾䶵鼾cé¼¾ãee䶴﨨b +423 beçhçççã﨨ça +440 hcé¼¾ãåçgçcç䶵ãçãä¸g +701 䶴䶵eceaç䶵eç䶴 +550 﨨çç䶴çg﨩䶵ça䶴﨩﨩gçaç +123 é¿çé¼¾ã鼾﨨dg䶴ãd +316 ç﨨﨩a䶵dãgçcç䶵de +807 eçhé½çe䶴åh䶴ãçé¿çg䶴aa +870 çaçgä¸fe䶵﨨çç䶵b䶵çaçé½ +669 dcä¸aä¸ç﨨fé½ï¨¨bé½hdé¼¾ +622 açé½ãfé¿é¿çc﨩çbçe +216 fh﨨é¿dcfãbãhãçc﨩gç +700 ç﨩ãé¿ï¨¨é½é¼¾çé¿b﨨﨩çfä¶´é¿ +528 åç﨩çé¼¾ä¸åä¸ä¸ãa䶴çdçfdä¸ +173 hçedãdçbcé¿ãå﨩 +3 䶴ç﨩çãå䶵é½é½ï¨©çeeããhç﨨ã +567 aågçä¸çc䶵hç﨨h +435 bé½é¿é¿ä¸gãcfä¸é¿ed +943 ãg䶵çccã䶵hç﨩䶵å +365 çådç䶵cb﨩䶵bd﨨çé¿b +298 䶵䶵﨩hé¼¾ãaçé¼¾çç䶵fç +377 bd䶵﨩é½ä¶µãhããçé¼¾ç +951 gçee﨨achçf﨩ãã䶵ba +399 å﨨鼾dé¼¾ffé½hä¸ï¨¨çaé¿gaåé½c +199 é¿çãegé½ççgé½gaådçãé½ +816 é½ççåçafççãç +758 çgcãçfa﨨ããé½abãh \ No newline at end of file Added: hive/trunk/hcatalog/src/test/e2e/templeton/inpdir/table3.txt URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/src/test/e2e/templeton/inpdir/table3.txt?rev=1653406&view=auto ============================================================================== --- hive/trunk/hcatalog/src/test/e2e/templeton/inpdir/table3.txt (added) +++ hive/trunk/hcatalog/src/test/e2e/templeton/inpdir/table3.txt Tue Jan 20 23:26:26 2015 @@ -0,0 +1,14 @@ +1 䶵 +2 䶵 +3 ç +4 䶵 +5 é¼¾ +6 䶵 +112 﨩﨨çç +12 﨩﨨çç +7 é¼¾ +8 䶵 +9 ç䶵 +10 çä¶µé½ +11 﨩é½ä¶µ +12 﨩﨨çç \ No newline at end of file Added: hive/trunk/hcatalog/src/test/e2e/templeton/inpdir/table3ToJoin.txt URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/src/test/e2e/templeton/inpdir/table3ToJoin.txt?rev=1653406&view=auto ============================================================================== --- hive/trunk/hcatalog/src/test/e2e/templeton/inpdir/table3ToJoin.txt (added) +++ hive/trunk/hcatalog/src/test/e2e/templeton/inpdir/table3ToJoin.txt Tue Jan 20 23:26:26 2015 @@ -0,0 +1,5 @@ +12 﨩﨨çç +2 䶵shouldnotshowup +3 ç +4 䶵 +11 﨩é½ä¶µ Added: hive/trunk/hcatalog/src/test/e2e/templeton/tests/utf8.conf URL: http://svn.apache.org/viewvc/hive/trunk/hcatalog/src/test/e2e/templeton/tests/utf8.conf?rev=1653406&view=auto ============================================================================== --- hive/trunk/hcatalog/src/test/e2e/templeton/tests/utf8.conf (added) +++ hive/trunk/hcatalog/src/test/e2e/templeton/tests/utf8.conf Tue Jan 20 23:26:26 2015 @@ -0,0 +1,134 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. + +############################################################################### +# curl command tests for templeton +# +# +use utf8; + +$cfg = +{ + 'driver' => 'Curl', + + 'groups' => + [ +##============================================================================================================= + { + 'name' => 'Hive_UTF8', + 'tests' => + [ + { + # Submitting Hive job with Unicode content on Unicode data + 'num' => 1, + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/hive', + 'post_options' => ['user.name=:UNAME:','execute=drop table if exists utf8_table;create table utf8_table(row int, content string) ROW FORMAT DELIMITED FIELDS TERMINATED BY \'\t\';LOAD DATA INPATH \':INPDIR_HDFS:/table1.txt\' OVERWRITE INTO TABLE utf8_table;','statusdir=:OUTDIR:/status/Hive_UTF8_:TNUM:'], + 'json_field_substr_match' => { 'id' => '\d+'}, + #results + 'status_code' => 200, + 'check_job_created' => 1, + 'check_job_complete' => 'SUCCESS', + 'check_job_exit_value' => 0, + 'check_call_back' => 1, + }, + ] + }, +##============================================================================================================= + { + 'name' => 'Pig_UTF8', + 'tests' => + [ + { + # Submitting Pig job with Unicode content on Unicode data + 'num' => 1, + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/pig', + 'post_options' => ['user.name=:UNAME:','file=:INPDIR_HDFS:/PigJoin䶴ã©é¼¾ä¸çãwork.pig','arg=-p', 'arg=INPDIR=:INPDIR_HDFS:','arg=-p','arg=OUTDIR=:OUTDIR:','statusdir=:OUTDIR:/status/Pig_UTF8_:TNUM:'], + 'json_field_substr_match' => { 'id' => '\d+'}, + #results + 'status_code' => 200, + 'check_job_created' => 1, + 'check_job_complete' => 'SUCCESS', + 'check_call_back' => 1, + }, + ] + }, +##============================================================================================================= + { + 'name' => 'MapReduce_UTF8', + 'tests' => + [ + { + + 'num' => 1, + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/mapreduce/jar', + 'post_options' => ['user.name=:UNAME:','arg=:INPDIR_HDFS:/artof䶴ã©é¼¾ä¸çãwar.txt', 'arg= :OUTDIR:/utf8_wc.txt', + 'jar=:INPDIR_HDFS:/hadoop_examples_䶴ã©é¼¾ä¸çã_2_2_0.jar', 'class=wordcount','statusdir=:OUTDIR:/status/MapRed_UTF8_:TNUM:'], + 'json_field_substr_match' => { 'id' => '\d+'}, + #results + 'status_code' => 200, + 'check_job_created' => 1, + 'check_job_complete' => 'SUCCESS', + 'check_job_percent_complete' => 'map 100% reduce 100%', + 'check_job_exit_value' => 0, + 'check_call_back' => 1, + }, + ] + }, +##============================================================================================================= + { + 'name' => 'MapRedStreaming_UTF8', + 'tests' => + [ + { + 'num' => 1, + 'method' => 'POST', + 'url' => ':TEMPLETON_URL:/templeton/v1/mapreduce/streaming', + 'post_options' => ['user.name=:UNAME:','input=:INPDIR_HDFS:/artof䶴ã©é¼¾ä¸çãwar.txt', 'input=:INPDIR_HDFS:/artof䶴ã©é¼¾ä¸çãwar.txt', 'output=:OUTDIR:/MapRedStreaming_UTF8_count', 'mapper=cat', 'reducer=wc','statusdir=:OUTDIR:/status/MapRedStreaming_UTF8_:TNUM:'], + 'json_field_substr_match' => { 'id' => '\d+'}, + #results + 'status_code' => 200, + 'check_job_created' => 1, + 'check_job_complete' => 'SUCCESS', + 'check_job_exit_value' => 0, + 'check_call_back' => 1, + }, + ] + }, +##============================================================================================================= + + { + 'name' => 'JobFiltering_UTF8', + 'tests' => + [ + { + 'num' => 1, + 'depends_on' => 'Pig_UTF8', + 'method' => 'GET', + 'url' => ':TEMPLETON_URL:/templeton/v1/jobs?user.name=:UNAME:&showall=true&fields=*', + 'user_name' => ':UNAME:', + 'format_header' => 'Content-Type: application/json', + 'filter_job_names' => ['TempletonControllerJob', 'PigLatin:PigJoin䶴ã©é¼¾ä¸çãwork.pig'], + 'status_code' => 200, + }, + ] + }, + ] +}, + ; \ No newline at end of file