Repository: incubator-carbondata Updated Branches: refs/heads/master 5177754e7 -> 39dd489e3
enhancement of VT for LCM. enhancing vt adding the blocklet test case. Project: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/commit/ea4d8990 Tree: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/tree/ea4d8990 Diff: http://git-wip-us.apache.org/repos/asf/incubator-carbondata/diff/ea4d8990 Branch: refs/heads/master Commit: ea4d8990cfcfe2f88c2f589bc72200ac7c8522a1 Parents: 5177754 Author: ravikiran <ravikiran.sn...@gmail.com> Authored: Thu Sep 8 17:52:37 2016 +0530 Committer: jackylk <jacky.li...@huawei.com> Committed: Wed Oct 12 22:40:03 2016 +0800 ---------------------------------------------------------------------- .../test/resources/compaction/compaction1.csv | 2 +- .../compaction/compaction1_forhive.csv | 5 + .../compaction/compactioncard2_forhive.csv | 256 +++++++++++++++++++ .../DataCompactionBlockletBoundryTest.scala | 108 ++++++++ .../DataCompactionBoundaryConditionsTest.scala | 105 ++++++++ .../DataCompactionMinorThresholdTest.scala | 118 +++++++++ 6 files changed, 593 insertions(+), 1 deletion(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ea4d8990/integration/spark/src/test/resources/compaction/compaction1.csv ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/resources/compaction/compaction1.csv b/integration/spark/src/test/resources/compaction/compaction1.csv index 2271e18..b669eca 100644 --- a/integration/spark/src/test/resources/compaction/compaction1.csv +++ b/integration/spark/src/test/resources/compaction/compaction1.csv @@ -3,4 +3,4 @@ ID,date,country,name,phonetype,serialname,salary 2,07/24/2015,chile,aaa2,phone756,ASD42892,15001 3,07/25/2015,america,aaa3,phone1904,ASD37014,15002 4,07/26/2015,canada,aaa4,phone2435,ASD66902,15003 -5,07/27/2015,england,aaa5,phone2441,ASD90633,15004 +5,07/27/2015,england,aaa5,phone2441,ASD90633,15004 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ea4d8990/integration/spark/src/test/resources/compaction/compaction1_forhive.csv ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/resources/compaction/compaction1_forhive.csv b/integration/spark/src/test/resources/compaction/compaction1_forhive.csv new file mode 100644 index 0000000..07c2508 --- /dev/null +++ b/integration/spark/src/test/resources/compaction/compaction1_forhive.csv @@ -0,0 +1,5 @@ +1,07/23/2015,china,aaa1,phone197,ASD69643,15000 +2,07/24/2015,chile,aaa2,phone756,ASD42892,15001 +3,07/25/2015,america,aaa3,phone1904,ASD37014,15002 +4,07/26/2015,canada,aaa4,phone2435,ASD66902,15003 +5,07/27/2015,england,aaa5,phone2441,ASD90633,15004 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ea4d8990/integration/spark/src/test/resources/compaction/compactioncard2_forhive.csv ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/resources/compaction/compactioncard2_forhive.csv b/integration/spark/src/test/resources/compaction/compactioncard2_forhive.csv new file mode 100644 index 0000000..7d7673e --- /dev/null +++ b/integration/spark/src/test/resources/compaction/compactioncard2_forhive.csv @@ -0,0 +1,256 @@ +6,7/28/2015,newzealand,aaa6,phone294,ASD59961,15005 +7,7/29/2015,mexico,aaa7,phone610,ASD14875,15006 +8,7/30/2015,westindies,aaa8,phone1848,ASD57308,15007 +9,7/18/2015,burma,aaa9,phone706,ASD86717,15008 +10,7/19/2015,butan,aaa10,phone685,ASD30505,1500 +11,7/20/2015,burma,aaa11,phone664,ASD25707,1500 +12,7/21/2015,butan,aaa12,phone643,ASD81919,1500 +13,7/22/2015,burma,aaa13,phone622,ASD138131,1500 +14,7/23/2015,burma,aaa14,phone601,ASD194343,1500 +15,7/24/2015,butan,aaa15,phone580,ASD250555,1500 +16,7/25/2015,burma,aaa16,phone559,ASD306767,1500 +17,7/26/2015,butan,aaa17,phone538,ASD362979,1500 +18,7/27/2015,burma,aaa18,phone517,ASD419191,1500 +19,7/28/2015,burma,aaa19,phone496,ASD475403,1500 +20,7/29/2015,butan,aaa20,phone475,ASD531615,1500 +21,7/30/2015,burma,aaa21,phone454,ASD587827,1500 +22,7/31/2015,butan,aaa22,phone433,ASD644039,1500 +23,8/1/2015,burma,aaa23,phone412,ASD700251,1500 +24,8/2/2015,burma,aaa24,phone391,ASD756463,1500 +25,8/3/2015,butan,aaa25,phone370,ASD812675,1500 +26,8/4/2015,burma,aaa26,phone349,ASD868887,1500 +27,8/5/2015,butan,aaa27,phone328,ASD925099,1500 +28,8/6/2015,burma,aaa28,phone307,ASD981311,1500 +29,8/7/2015,burma,aaa29,phone286,ASD1037523,1500 +30,8/8/2015,butan,aaa30,phone265,ASD1093735,1500 +31,8/9/2015,burma,aaa31,phone244,ASD1149947,1500 +32,8/10/2015,butan,aaa32,phone223,ASD1206159,1500 +33,8/11/2015,burma,aaa33,phone202,ASD1262371,1500 +34,8/12/2015,burma,aaa34,phone181,ASD1318583,1500 +35,8/13/2015,butan,aaa35,phone160,ASD1374795,1500 +36,8/14/2015,burma,aaa36,phone139,ASD1431007,1500 +37,8/15/2015,butan,aaa37,phone118,ASD1487219,1500 +38,8/16/2015,burma,aaa38,phone97,ASD1543431,1500 +39,8/17/2015,burma,aaa39,phone76,ASD1599643,1500 +40,8/18/2015,butan,aaa40,phone55,ASD1655855,1500 +41,8/19/2015,burma,aaa41,phone34,ASD1712067,1500 +42,8/20/2015,butan,aaa42,phone13,ASD1768279,1500 +43,8/21/2015,burma,aaa43,phone8,ASD1824491,1500 +44,8/22/2015,burma,aaa44,phone29,ASD1880703,1500 +45,8/23/2015,butan,aaa45,phone50,ASD1936915,1500 +46,8/24/2015,burma,aaa46,phone71,ASD1993127,1500 +47,8/25/2015,butan,aaa47,phone92,ASD2049339,1500 +48,8/26/2015,burma,aaa48,phone113,ASD2105551,1500 +49,8/27/2015,burma,aaa49,phone134,ASD2161763,1500 +50,8/28/2015,butan,aaa50,phone155,ASD2217975,1500 +51,8/29/2015,burma,aaa51,phone176,ASD2274187,1500 +52,8/30/2015,butan,aaa52,phone197,ASD2330399,1500 +53,8/31/2015,burma,aaa53,phone218,ASD2386611,1500 +54,9/1/2015,burma,aaa54,phone239,ASD2442823,1500 +55,9/2/2015,butan,aaa55,phone260,ASD2499035,1500 +56,9/3/2015,burma,aaa56,phone281,ASD2555247,1500 +57,9/4/2015,butan,aaa57,phone302,ASD2611459,1500 +58,9/5/2015,burma,aaa58,phone323,ASD2667671,1500 +59,9/6/2015,burma,aaa59,phone344,ASD2723883,1500 +60,9/7/2015,butan,aaa60,phone365,ASD2780095,1500 +61,9/8/2015,burma,aaa61,phone386,ASD2836307,1500 +62,9/9/2015,butan,aaa62,phone407,ASD2892519,1500 +63,9/10/2015,burma,aaa63,phone428,ASD2948731,1500 +64,9/11/2015,burma,aaa64,phone449,ASD3004943,1500 +65,9/12/2015,butan,aaa65,phone470,ASD3061155,1500 +66,9/13/2015,burma,aaa66,phone491,ASD3117367,1500 +67,9/14/2015,butan,aaa67,phone512,ASD3173579,1500 +68,9/15/2015,burma,aaa68,phone533,ASD3229791,1500 +69,9/16/2015,burma,aaa69,phone554,ASD3286003,1500 +70,9/17/2015,butan,aaa70,phone575,ASD3342215,1500 +71,9/18/2015,burma,aaa71,phone596,ASD3398427,1500 +72,9/19/2015,butan,aaa72,phone617,ASD3454639,1500 +73,9/20/2015,burma,aaa73,phone638,ASD3510851,1500 +74,9/21/2015,burma,aaa74,phone659,ASD3567063,1500 +75,9/22/2015,butan,aaa75,phone680,ASD3623275,1500 +76,9/23/2015,burma,aaa76,phone701,ASD3679487,1500 +77,9/24/2015,butan,aaa77,phone722,ASD3735699,1500 +78,9/25/2015,burma,aaa78,phone743,ASD3791911,1500 +79,9/26/2015,burma,aaa79,phone764,ASD3848123,1500 +80,9/27/2015,butan,aaa80,phone785,ASD3904335,1500 +81,9/28/2015,burma,aaa81,phone806,ASD3960547,1500 +82,9/29/2015,butan,aaa82,phone827,ASD4016759,1500 +83,9/30/2015,burma,aaa83,phone848,ASD4072971,1500 +84,10/1/2015,burma,aaa84,phone869,ASD4129183,1500 +85,10/2/2015,butan,aaa85,phone890,ASD4185395,1500 +86,10/3/2015,burma,aaa86,phone911,ASD4241607,1500 +87,10/4/2015,butan,aaa87,phone932,ASD4297819,1500 +88,10/5/2015,burma,aaa88,phone953,ASD4354031,1500 +89,10/6/2015,burma,aaa89,phone974,ASD4410243,1500 +90,10/7/2015,butan,aaa90,phone995,ASD4466455,1500 +91,10/8/2015,burma,aaa91,phone1016,ASD4522667,1500 +92,10/9/2015,butan,aaa92,phone1037,ASD4578879,1500 +93,10/10/2015,burma,aaa93,phone1058,ASD4635091,1500 +94,10/11/2015,burma,aaa94,phone1079,ASD4691303,1500 +95,10/12/2015,butan,aaa95,phone1100,ASD4747515,1500 +96,10/13/2015,burma,aaa96,phone1121,ASD4803727,1500 +97,10/14/2015,butan,aaa97,phone1142,ASD4859939,1500 +98,10/15/2015,burma,aaa98,phone1163,ASD4916151,1500 +99,10/16/2015,burma,aaa99,phone1184,ASD4972363,1500 +100,10/17/2015,butan,aaa100,phone1205,ASD5028575,1500 +101,10/18/2015,burma,aaa101,phone1226,ASD5084787,1500 +102,10/19/2015,butan,aaa102,phone1247,ASD5140999,1500 +103,10/20/2015,burma,aaa103,phone1268,ASD5197211,1500 +104,10/21/2015,burma,aaa104,phone1289,ASD5253423,1500 +105,10/22/2015,butan,aaa105,phone1310,ASD5309635,1500 +106,10/23/2015,burma,aaa106,phone1331,ASD5365847,1500 +107,10/24/2015,butan,aaa107,phone1352,ASD5422059,1500 +108,10/25/2015,burma,aaa108,phone1373,ASD5478271,1500 +109,10/26/2015,burma,aaa109,phone1394,ASD5534483,1500 +110,10/27/2015,butan,aaa110,phone1415,ASD5590695,1500 +111,10/28/2015,burma,aaa111,phone1436,ASD5646907,1500 +112,10/29/2015,butan,aaa112,phone1457,ASD5703119,1500 +113,10/30/2015,burma,aaa113,phone1478,ASD5759331,1500 +114,10/31/2015,burma,aaa114,phone1499,ASD5815543,1500 +115,11/1/2015,butan,aaa115,phone1520,ASD5871755,1500 +116,11/2/2015,burma,aaa116,phone1541,ASD5927967,1500 +117,11/3/2015,butan,aaa117,phone1562,ASD5984179,1500 +118,11/4/2015,burma,aaa118,phone1583,ASD6040391,1500 +119,11/5/2015,burma,aaa119,phone1604,ASD6096603,1500 +120,11/6/2015,butan,aaa120,phone1625,ASD6152815,1500 +121,11/7/2015,burma,aaa121,phone1646,ASD6209027,1500 +122,11/8/2015,butan,aaa122,phone1667,ASD6265239,1500 +123,11/9/2015,burma,aaa123,phone1688,ASD6321451,1500 +124,11/10/2015,burma,aaa124,phone1709,ASD6377663,1500 +125,11/11/2015,butan,aaa125,phone1730,ASD6433875,1500 +126,11/12/2015,burma,aaa126,phone1751,ASD6490087,1500 +127,11/13/2015,butan,aaa127,phone1772,ASD6546299,1500 +128,11/14/2015,burma,aaa128,phone1793,ASD6602511,1500 +129,11/15/2015,burma,aaa129,phone1814,ASD6658723,1500 +130,11/16/2015,butan,aaa130,phone1835,ASD6714935,1500 +131,11/17/2015,burma,aaa131,phone1856,ASD6771147,1500 +132,11/18/2015,butan,aaa132,phone1877,ASD6827359,1500 +133,11/19/2015,burma,aaa133,phone1898,ASD6883571,1500 +134,11/20/2015,burma,aaa134,phone1919,ASD6939783,1500 +135,11/21/2015,butan,aaa135,phone1940,ASD6995995,1500 +136,11/22/2015,burma,aaa136,phone1961,ASD7052207,1500 +137,11/23/2015,butan,aaa137,phone1982,ASD7108419,1500 +138,11/24/2015,burma,aaa138,phone2003,ASD7164631,1500 +139,11/25/2015,burma,aaa139,phone2024,ASD7220843,1500 +140,11/26/2015,butan,aaa140,phone2045,ASD7277055,1500 +141,11/27/2015,burma,aaa141,phone2066,ASD7333267,1500 +142,11/28/2015,butan,aaa142,phone2087,ASD7389479,1500 +143,11/29/2015,burma,aaa143,phone2108,ASD7445691,1500 +144,11/30/2015,burma,aaa144,phone2129,ASD7501903,1500 +145,12/1/2015,butan,aaa145,phone2150,ASD7558115,1500 +146,12/2/2015,burma,aaa146,phone2171,ASD7614327,1500 +147,12/3/2015,butan,aaa147,phone2192,ASD7670539,1500 +148,12/4/2015,burma,aaa148,phone2213,ASD7726751,1500 +149,12/5/2015,burma,aaa149,phone2234,ASD7782963,1500 +150,12/6/2015,butan,aaa150,phone2255,ASD7839175,1500 +151,12/7/2015,burma,aaa151,phone2276,ASD7895387,1500 +152,12/8/2015,butan,aaa152,phone2297,ASD7951599,1500 +153,12/9/2015,burma,aaa153,phone2318,ASD8007811,1500 +154,12/10/2015,burma,aaa154,phone2339,ASD8064023,1500 +155,12/11/2015,butan,aaa155,phone2360,ASD8120235,1500 +156,12/12/2015,burma,aaa156,phone2381,ASD8176447,1500 +157,12/13/2015,butan,aaa157,phone2402,ASD8232659,1500 +158,12/14/2015,burma,aaa158,phone2423,ASD8288871,1500 +159,12/15/2015,burma,aaa159,phone2444,ASD8345083,1500 +160,12/16/2015,butan,aaa160,phone2465,ASD8401295,1500 +161,12/17/2015,burma,aaa161,phone2486,ASD8457507,1500 +162,12/18/2015,butan,aaa162,phone2507,ASD8513719,1500 +163,12/19/2015,burma,aaa163,phone2528,ASD8569931,1500 +164,12/20/2015,burma,aaa164,phone2549,ASD8626143,1500 +165,12/21/2015,butan,aaa165,phone2570,ASD8682355,1500 +166,12/22/2015,burma,aaa166,phone2591,ASD8738567,1500 +167,12/23/2015,butan,aaa167,phone2612,ASD8794779,1500 +168,12/24/2015,burma,aaa168,phone2633,ASD8850991,1500 +169,12/25/2015,burma,aaa169,phone2654,ASD8907203,1500 +170,12/26/2015,butan,aaa170,phone2675,ASD8963415,1500 +171,12/27/2015,burma,aaa171,phone2696,ASD9019627,1500 +172,12/28/2015,butan,aaa172,phone2717,ASD9075839,1500 +173,12/29/2015,burma,aaa173,phone2738,ASD9132051,1500 +174,12/30/2015,burma,aaa174,phone2759,ASD9188263,1500 +175,12/31/2015,butan,aaa175,phone2780,ASD9244475,1500 +176,1/1/2016,burma,aaa176,phone2801,ASD9300687,1500 +177,1/2/2016,butan,aaa177,phone2822,ASD9356899,1500 +178,1/3/2016,burma,aaa178,phone2843,ASD9413111,1500 +179,1/4/2016,burma,aaa179,phone2864,ASD9469323,1500 +180,1/5/2016,butan,aaa180,phone2885,ASD9525535,1500 +181,1/6/2016,burma,aaa181,phone2906,ASD9581747,1500 +182,1/7/2016,butan,aaa182,phone2927,ASD9637959,1500 +183,1/8/2016,burma,aaa183,phone2948,ASD9694171,1500 +184,1/9/2016,burma,aaa184,phone2969,ASD9750383,1500 +185,1/10/2016,butan,aaa185,phone2990,ASD9806595,1500 +186,1/11/2016,burma,aaa186,phone3011,ASD9862807,1500 +187,1/12/2016,butan,aaa187,phone3032,ASD9919019,1500 +188,1/13/2016,burma,aaa188,phone3053,ASD9975231,1500 +189,1/14/2016,burma,aaa189,phone3074,ASD10031443,1500 +190,1/15/2016,butan,aaa190,phone3095,ASD10087655,1500 +191,1/16/2016,burma,aaa191,phone3116,ASD10143867,1500 +192,1/17/2016,butan,aaa192,phone3137,ASD10200079,1500 +193,1/18/2016,burma,aaa193,phone3158,ASD10256291,1500 +194,1/19/2016,burma,aaa194,phone3179,ASD10312503,1500 +195,1/20/2016,butan,aaa195,phone3200,ASD10368715,1500 +196,1/21/2016,burma,aaa196,phone3221,ASD10424927,1500 +197,1/22/2016,butan,aaa197,phone3242,ASD10481139,1500 +198,1/23/2016,burma,aaa198,phone3263,ASD10537351,1500 +199,1/24/2016,burma,aaa199,phone3284,ASD10593563,1500 +200,1/25/2016,butan,aaa200,phone3305,ASD10649775,1500 +201,1/26/2016,burma,aaa201,phone3326,ASD10705987,1500 +202,1/27/2016,butan,aaa202,phone3347,ASD10762199,1500 +203,1/28/2016,burma,aaa203,phone3368,ASD10818411,1500 +204,1/29/2016,burma,aaa204,phone3389,ASD10874623,1500 +205,1/30/2016,butan,aaa205,phone3410,ASD10930835,1500 +206,1/31/2016,burma,aaa206,phone3431,ASD10987047,1500 +207,2/1/2016,butan,aaa207,phone3452,ASD11043259,1500 +208,2/2/2016,burma,aaa208,phone3473,ASD11099471,1500 +209,2/3/2016,burma,aaa209,phone3494,ASD11155683,1500 +210,2/4/2016,butan,aaa210,phone3515,ASD11211895,1500 +211,2/5/2016,burma,aaa211,phone3536,ASD11268107,1500 +212,2/6/2016,butan,aaa212,phone3557,ASD11324319,1500 +213,2/7/2016,burma,aaa213,phone3578,ASD11380531,1500 +214,2/8/2016,burma,aaa214,phone3599,ASD11436743,1500 +215,2/9/2016,butan,aaa215,phone3620,ASD11492955,1500 +216,2/10/2016,burma,aaa216,phone3641,ASD11549167,1500 +217,2/11/2016,butan,aaa217,phone3662,ASD11605379,1500 +218,2/12/2016,burma,aaa218,phone3683,ASD11661591,1500 +219,2/13/2016,burma,aaa219,phone3704,ASD11717803,1500 +220,2/14/2016,butan,aaa220,phone3725,ASD11774015,1500 +221,2/15/2016,burma,aaa221,phone3746,ASD11830227,1500 +222,2/16/2016,butan,aaa222,phone3767,ASD11886439,1500 +223,2/17/2016,burma,aaa223,phone3788,ASD11942651,1500 +224,2/18/2016,burma,aaa224,phone3809,ASD11998863,1500 +225,2/19/2016,butan,aaa225,phone3830,ASD12055075,1500 +226,2/20/2016,burma,aaa226,phone3851,ASD12111287,1500 +227,2/21/2016,butan,aaa227,phone3872,ASD12167499,1500 +228,2/22/2016,burma,aaa228,phone3893,ASD12223711,1500 +229,2/23/2016,burma,aaa229,phone3914,ASD12279923,1500 +230,2/24/2016,butan,aaa230,phone3935,ASD12336135,1500 +231,2/25/2016,burma,aaa231,phone3956,ASD12392347,1500 +232,2/26/2016,butan,aaa232,phone3977,ASD12448559,1500 +233,2/27/2016,burma,aaa233,phone3998,ASD12504771,1500 +234,2/28/2016,burma,aaa234,phone4019,ASD12560983,1500 +235,2/29/2016,butan,aaa235,phone4040,ASD12617195,1500 +236,3/1/2016,burma,aaa236,phone4061,ASD12673407,1500 +237,3/2/2016,butan,aaa237,phone4082,ASD12729619,1500 +238,3/3/2016,burma,aaa238,phone4103,ASD12785831,1500 +239,3/4/2016,burma,aaa239,phone4124,ASD12842043,1500 +240,3/5/2016,butan,aaa240,phone4145,ASD12898255,1500 +241,3/6/2016,burma,aaa241,phone4166,ASD12954467,1500 +242,3/7/2016,butan,aaa242,phone4187,ASD13010679,1500 +243,3/8/2016,burma,aaa243,phone4208,ASD13066891,1500 +244,3/9/2016,burma,aaa244,phone4229,ASD13123103,1500 +245,3/10/2016,butan,aaa245,phone4250,ASD13179315,1500 +246,3/11/2016,burma,aaa246,phone4271,ASD13235527,1500 +247,3/12/2016,butan,aaa247,phone4292,ASD13291739,1500 +248,3/13/2016,burma,aaa248,phone4313,ASD13347951,1500 +249,3/14/2016,burma,aaa249,phone4334,ASD13404163,1500 +250,3/15/2016,butan,aaa250,phone4355,ASD13460375,1500 +251,3/16/2016,burma,aaa251,phone4376,ASD13516587,1500 +252,3/17/2016,butan,aaa252,phone4397,ASD13572799,1500 +253,3/18/2016,burma,aaa253,phone4418,ASD13629011,1500 +254,3/19/2016,burma,aaa254,phone4439,ASD13685223,1500 +255,3/20/2016,butan,aaa255,phone4460,ASD13741435,1500 +256,3/21/2016,burma,aaa256,phone4481,ASD13797647,1500 +257,3/22/2016,butan,aaa257,phone4502,ASD13853859,1500 +258,3/23/2016,burma,aaa258,phone4523,ASD13910071,1500 +259,3/24/2016,burma,aaa259,phone4544,ASD13966283,1500 +260,3/25/2016,butan,aaa260,phone4565,ASD14022495,1500 +261,3/26/2016,burma,aaa261,phone4586,ASD14078707,1500 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ea4d8990/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionBlockletBoundryTest.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionBlockletBoundryTest.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionBlockletBoundryTest.scala new file mode 100644 index 0000000..f7056a9 --- /dev/null +++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionBlockletBoundryTest.scala @@ -0,0 +1,108 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.carbondata.spark.testsuite.datacompaction + +import java.io.File + +import scala.collection.JavaConverters._ + +import org.apache.spark.sql.Row +import org.apache.spark.sql.common.util.CarbonHiveContext._ +import org.apache.spark.sql.common.util.QueryTest +import org.scalatest.BeforeAndAfterAll + +import org.apache.carbondata.core.carbon.{AbsoluteTableIdentifier, CarbonTableIdentifier} +import org.apache.carbondata.core.constants.CarbonCommonConstants +import org.apache.carbondata.core.util.CarbonProperties +import org.apache.carbondata.lcm.status.SegmentStatusManager + +/** + * FT for data compaction scenario. + */ +class DataCompactionBlockletBoundryTest extends QueryTest with BeforeAndAfterAll { + + override def beforeAll { + sql("drop table if exists blocklettest") + sql("drop table if exists Carbon_automation_hive") + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "mm/dd/yyyy") + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.BLOCKLET_SIZE, + "55") + sql( + "CREATE TABLE IF NOT EXISTS blocklettest (country String, ID String, date Timestamp, name " + + "String, " + + "phonetype String, serialname String, salary Int) STORED BY 'org.apache.carbondata" + + ".format'" + ) + + + val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../") + .getCanonicalPath + val csvFilePath1 = currentDirectory + "/src/test/resources/compaction/compaction1.csv" + + // loading the rows greater than 256. so that the column cardinality crosses byte boundary. + val csvFilePath2 = currentDirectory + "/src/test/resources/compaction/compactioncard2.csv" + + + sql("LOAD DATA LOCAL INPATH '" + csvFilePath1 + "' INTO TABLE blocklettest OPTIONS" + + "('DELIMITER'= ',', 'QUOTECHAR'= '\"')" + ) + sql("LOAD DATA LOCAL INPATH '" + csvFilePath2 + "' INTO TABLE blocklettest OPTIONS" + + "('DELIMITER'= ',', 'QUOTECHAR'= '\"')" + ) + // compaction will happen here. + sql("alter table blocklettest compact 'major'" + ) + + sql( + "create table Carbon_automation_hive (ID String, date " + + "Timestamp,country String, name String, phonetype String, serialname String, salary Int ) row format " + + "delimited fields terminated by ',' TBLPROPERTIES ('skip.header.line.count'='1') " + ) + + sql("LOAD DATA LOCAL INPATH '" + currentDirectory + + "/src/test/resources/compaction/compaction1_forhive.csv" + "' INTO " + + "table Carbon_automation_hive ") + sql("LOAD DATA LOCAL INPATH '" + currentDirectory + + "/src/test/resources/compaction/compactioncard2_forhive.csv" + "' INTO " + + "table Carbon_automation_hive ") + + } + + test("select country,count(*) as a from blocklettest")({ + sql("select country,count(*) as a from Carbon_automation_hive group by country").show() + sql("select * from Carbon_automation_hive").show + checkAnswer( + sql("select country,count(*) as a from blocklettest group by country"), + sql("select country,count(*) as a from Carbon_automation_hive group by country") + ) + } + ) + + override def afterAll { + /* sql("drop table blocklettest") */ + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy") + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.BLOCKLET_SIZE, + "" + CarbonCommonConstants.BLOCKLET_SIZE_DEFAULT_VAL) + } + +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ea4d8990/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionBoundaryConditionsTest.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionBoundaryConditionsTest.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionBoundaryConditionsTest.scala new file mode 100644 index 0000000..80a2320 --- /dev/null +++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionBoundaryConditionsTest.scala @@ -0,0 +1,105 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.carbondata.spark.testsuite.datacompaction + +import java.io.File + +import scala.collection.JavaConverters._ + +import org.apache.spark.sql.common.util.CarbonHiveContext._ +import org.apache.spark.sql.common.util.QueryTest +import org.scalatest.BeforeAndAfterAll + +import org.apache.carbondata.core.carbon.{AbsoluteTableIdentifier, CarbonTableIdentifier} +import org.apache.carbondata.core.constants.CarbonCommonConstants +import org.apache.carbondata.core.util.CarbonProperties +import org.apache.carbondata.lcm.status.SegmentStatusManager + +/** + * FT for data compaction Boundary condition verification. + */ +class DataCompactionBoundaryConditionsTest extends QueryTest with BeforeAndAfterAll { + val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../") + .getCanonicalPath + val resource = currentDirectory + "/src/test/resources/" + + val storeLocation = new File(this.getClass.getResource("/").getPath + "/../test").getCanonicalPath + val carbonTableIdentifier: CarbonTableIdentifier = + new CarbonTableIdentifier("default", "boundarytest".toLowerCase(), "1") + + val segmentStatusManager: SegmentStatusManager = new SegmentStatusManager(new + AbsoluteTableIdentifier(storeLocation, carbonTableIdentifier)) + + override def beforeAll { + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.COMPACTION_SEGMENT_LEVEL_THRESHOLD, "2,2") + sql("drop table if exists boundarytest") + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "mm/dd/yyyy") + sql( + "CREATE TABLE IF NOT EXISTS boundarytest (country String, ID Int, date " + + "Timestamp, name " + + "String, " + + "phonetype String, serialname String, salary Int) STORED BY 'org.apache.carbondata" + + ".format'" + ) + + } + + /** + * Compaction verificatoin in case of no loads. + */ + test("check if compaction is completed correctly.") { + + try { + sql("alter table boundarytest compact 'minor'") + sql("alter table boundarytest compact 'major'") + } + catch { + case e: Exception => + assert(false) + } + } + + /** + * Compaction verificatoin in case of one loads. + */ + test("check if compaction is completed correctly for one load.") { + + val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../") + .getCanonicalPath + var csvFilePath1 = currentDirectory + "/src/test/resources/compaction/compaction1.csv" + + + sql("LOAD DATA LOCAL INPATH '" + csvFilePath1 + "' INTO TABLE boundarytest " + + "OPTIONS" + + "('DELIMITER'= ',', 'QUOTECHAR'= '\"')" + ) + sql("alter table boundarytest compact 'minor'") + sql("alter table boundarytest compact 'major'") + + } + + + override def afterAll { + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy") + } + +} http://git-wip-us.apache.org/repos/asf/incubator-carbondata/blob/ea4d8990/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionMinorThresholdTest.scala ---------------------------------------------------------------------- diff --git a/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionMinorThresholdTest.scala b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionMinorThresholdTest.scala new file mode 100644 index 0000000..fbb39d8 --- /dev/null +++ b/integration/spark/src/test/scala/org/apache/carbondata/spark/testsuite/datacompaction/DataCompactionMinorThresholdTest.scala @@ -0,0 +1,118 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.carbondata.spark.testsuite.datacompaction + +import java.io.File + +import scala.collection.JavaConverters._ + +import org.apache.spark.sql.common.util.CarbonHiveContext._ +import org.apache.spark.sql.common.util.QueryTest +import org.scalatest.BeforeAndAfterAll + +import org.apache.carbondata.core.carbon.path.{CarbonStorePath, CarbonTablePath} +import org.apache.carbondata.core.carbon.{AbsoluteTableIdentifier, CarbonTableIdentifier} +import org.apache.carbondata.core.constants.CarbonCommonConstants +import org.apache.carbondata.core.util.CarbonProperties +import org.apache.carbondata.lcm.locks.{CarbonLockFactory, ICarbonLock, LockUsage} +import org.apache.carbondata.lcm.status.SegmentStatusManager + +/** + * FT for data compaction Minor threshold verification. + */ +class DataCompactionMinorThresholdTest extends QueryTest with BeforeAndAfterAll { + val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../") + .getCanonicalPath + val resource = currentDirectory + "/src/test/resources/" + + val storeLocation = new File(this.getClass.getResource("/").getPath + "/../test").getCanonicalPath + val carbonTableIdentifier: CarbonTableIdentifier = + new CarbonTableIdentifier("default", "minorthreshold".toLowerCase(), "1") + + val segmentStatusManager: SegmentStatusManager = new SegmentStatusManager(new + AbsoluteTableIdentifier(storeLocation, carbonTableIdentifier)) + + override def beforeAll { + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.COMPACTION_SEGMENT_LEVEL_THRESHOLD, "2,2") + sql("drop table if exists minorthreshold") + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "mm/dd/yyyy") + sql( + "CREATE TABLE IF NOT EXISTS minorthreshold (country String, ID Int, date " + + "Timestamp, name " + + "String, " + + "phonetype String, serialname String, salary Int) STORED BY 'org.apache.carbondata" + + ".format'" + ) + + val currentDirectory = new File(this.getClass.getResource("/").getPath + "/../../") + .getCanonicalPath + var csvFilePath1 = currentDirectory + "/src/test/resources/compaction/compaction1.csv" + + var csvFilePath2 = currentDirectory + "/src/test/resources/compaction/compaction2.csv" + var csvFilePath3 = currentDirectory + "/src/test/resources/compaction/compaction3.csv" + + sql("LOAD DATA LOCAL INPATH '" + csvFilePath1 + "' INTO TABLE minorthreshold " + + "OPTIONS" + + "('DELIMITER'= ',', 'QUOTECHAR'= '\"')" + ) + sql("LOAD DATA LOCAL INPATH '" + csvFilePath2 + "' INTO TABLE minorthreshold " + + "OPTIONS" + + "('DELIMITER'= ',', 'QUOTECHAR'= '\"')" + ) + sql("LOAD DATA LOCAL INPATH '" + csvFilePath3 + "' INTO TABLE minorthreshold " + + "OPTIONS" + + "('DELIMITER'= ',', 'QUOTECHAR'= '\"')" + ) + sql("LOAD DATA LOCAL INPATH '" + csvFilePath3 + "' INTO TABLE minorthreshold " + + "OPTIONS" + + "('DELIMITER'= ',', 'QUOTECHAR'= '\"')" + ) + // compaction should happen here. + sql("alter table minorthreshold compact 'minor'") + } + + /** + * Compaction should be completed correctly for minor compaction. + */ + test("check if compaction is completed correctly for minor.") { + + sql("clean files for table minorthreshold") + + val segments = segmentStatusManager.getValidAndInvalidSegments.getValidSegments.asScala.toList + + assert(segments.contains("0.2")) + assert(!segments.contains("0.1")) + assert(!segments.contains("0")) + assert(!segments.contains("1")) + assert(!segments.contains("2")) + assert(!segments.contains("3")) + } + + + override def afterAll { + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.CARBON_TIMESTAMP_FORMAT, "dd-MM-yyyy") + CarbonProperties.getInstance() + .addProperty(CarbonCommonConstants.COMPACTION_SEGMENT_LEVEL_THRESHOLD, + CarbonCommonConstants.DEFAULT_SEGMENT_LEVEL_THRESHOLD) + } + +}
