http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-core/src/test/resources/results/TestHBaseTable/testCATS.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testCATS.result b/tajo-core/src/test/resources/results/TestHBaseTable/testCATS.result new file mode 100644 index 0000000..72013f2 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestHBaseTable/testCATS.result @@ -0,0 +1,100 @@ +000, value0 +001, value1 +002, value2 +003, value3 +004, value4 +005, value5 +006, value6 +007, value7 +008, value8 +009, value9 +010, value10 +011, value11 +012, value12 +013, value13 +014, value14 +015, value15 +016, value16 +017, value17 +018, value18 +019, value19 +020, value20 +021, value21 +022, value22 +023, value23 +024, value24 +025, value25 +026, value26 +027, value27 +028, value28 +029, value29 +030, value30 +031, value31 +032, value32 +033, value33 +034, value34 +035, value35 +036, value36 +037, value37 +038, value38 +039, value39 +040, value40 +041, value41 +042, value42 +043, value43 +044, value44 +045, value45 +046, value46 +047, value47 +048, value48 +049, value49 +050, value50 +051, value51 +052, value52 +053, value53 +054, value54 +055, value55 +056, value56 +057, value57 +058, value58 +059, value59 +060, value60 +061, value61 +062, value62 +063, value63 +064, value64 +065, value65 +066, value66 +067, value67 +068, value68 +069, value69 +070, value70 +071, value71 +072, value72 +073, value73 +074, value74 +075, value75 +076, value76 +077, value77 +078, value78 +079, value79 +080, value80 +081, value81 +082, value82 +083, value83 +084, value84 +085, value85 +086, value86 +087, value87 +088, value88 +089, value89 +090, value90 +091, value91 +092, value92 +093, value93 +094, value94 +095, value95 +096, value96 +097, value97 +098, value98 +099, value99
http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-core/src/test/resources/results/TestHBaseTable/testColumnKeyValueSelectQuery.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testColumnKeyValueSelectQuery.result b/tajo-core/src/test/resources/results/TestHBaseTable/testColumnKeyValueSelectQuery.result new file mode 100644 index 0000000..82d4fd2 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestHBaseTable/testColumnKeyValueSelectQuery.result @@ -0,0 +1,12 @@ +rk1,col2_key,col2_value,col3 +------------------------------- +rk-0,["key-0", "key-1", "key-2", "key-3", "key-4"],["value-0", "value-1", "value-2", "value-3", "value-4"],col3-value-0 +rk-1,["key-0", "key-1", "key-2", "key-3", "key-4"],["value-0", "value-1", "value-2", "value-3", "value-4"],col3-value-1 +rk-2,["key-0", "key-1", "key-2", "key-3", "key-4"],["value-0", "value-1", "value-2", "value-3", "value-4"],col3-value-2 +rk-3,["key-0", "key-1", "key-2", "key-3", "key-4"],["value-0", "value-1", "value-2", "value-3", "value-4"],col3-value-3 +rk-4,["key-0", "key-1", "key-2", "key-3", "key-4"],["value-0", "value-1", "value-2", "value-3", "value-4"],col3-value-4 +rk-5,["key-0", "key-1", "key-2", "key-3", "key-4"],["value-0", "value-1", "value-2", "value-3", "value-4"],col3-value-5 +rk-6,["key-0", "key-1", "key-2", "key-3", "key-4"],["value-0", "value-1", "value-2", "value-3", "value-4"],col3-value-6 +rk-7,["key-0", "key-1", "key-2", "key-3", "key-4"],["value-0", "value-1", "value-2", "value-3", "value-4"],col3-value-7 +rk-8,["key-0", "key-1", "key-2", "key-3", "key-4"],["value-0", "value-1", "value-2", "value-3", "value-4"],col3-value-8 +rk-9,["key-0", "key-1", "key-2", "key-3", "key-4"],["value-0", "value-1", "value-2", "value-3", "value-4"],col3-value-9 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-core/src/test/resources/results/TestHBaseTable/testIndexPredication.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testIndexPredication.result b/tajo-core/src/test/resources/results/TestHBaseTable/testIndexPredication.result new file mode 100644 index 0000000..f38d238 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestHBaseTable/testIndexPredication.result @@ -0,0 +1,38 @@ +rk,col1,col2,col3 +------------------------------- +020,a-20,{"k1":"k1-20", "k2":"k2-20"},b-20 +021,a-21,{"k1":"k1-21", "k2":"k2-21"},b-21 +022,a-22,{"k1":"k1-22", "k2":"k2-22"},b-22 +023,a-23,{"k1":"k1-23", "k2":"k2-23"},b-23 +024,a-24,{"k1":"k1-24", "k2":"k2-24"},b-24 +025,a-25,{"k1":"k1-25", "k2":"k2-25"},b-25 +026,a-26,{"k1":"k1-26", "k2":"k2-26"},b-26 +027,a-27,{"k1":"k1-27", "k2":"k2-27"},b-27 +028,a-28,{"k1":"k1-28", "k2":"k2-28"},b-28 +029,a-29,{"k1":"k1-29", "k2":"k2-29"},b-29 +030,a-30,{"k1":"k1-30", "k2":"k2-30"},b-30 +031,a-31,{"k1":"k1-31", "k2":"k2-31"},b-31 +032,a-32,{"k1":"k1-32", "k2":"k2-32"},b-32 +033,a-33,{"k1":"k1-33", "k2":"k2-33"},b-33 +034,a-34,{"k1":"k1-34", "k2":"k2-34"},b-34 +035,a-35,{"k1":"k1-35", "k2":"k2-35"},b-35 +036,a-36,{"k1":"k1-36", "k2":"k2-36"},b-36 +037,a-37,{"k1":"k1-37", "k2":"k2-37"},b-37 +038,a-38,{"k1":"k1-38", "k2":"k2-38"},b-38 +039,a-39,{"k1":"k1-39", "k2":"k2-39"},b-39 +040,a-40,{"k1":"k1-40", "k2":"k2-40"},b-40 +041,a-41,{"k1":"k1-41", "k2":"k2-41"},b-41 +042,a-42,{"k1":"k1-42", "k2":"k2-42"},b-42 +043,a-43,{"k1":"k1-43", "k2":"k2-43"},b-43 +044,a-44,{"k1":"k1-44", "k2":"k2-44"},b-44 +045,a-45,{"k1":"k1-45", "k2":"k2-45"},b-45 +046,a-46,{"k1":"k1-46", "k2":"k2-46"},b-46 +047,a-47,{"k1":"k1-47", "k2":"k2-47"},b-47 +048,a-48,{"k1":"k1-48", "k2":"k2-48"},b-48 +049,a-49,{"k1":"k1-49", "k2":"k2-49"},b-49 +050,a-50,{"k1":"k1-50", "k2":"k2-50"},b-50 +051,a-51,{"k1":"k1-51", "k2":"k2-51"},b-51 +052,a-52,{"k1":"k1-52", "k2":"k2-52"},b-52 +053,a-53,{"k1":"k1-53", "k2":"k2-53"},b-53 +054,a-54,{"k1":"k1-54", "k2":"k2-54"},b-54 +055,a-55,{"k1":"k1-55", "k2":"k2-55"},b-55 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-core/src/test/resources/results/TestHBaseTable/testInsertInto.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testInsertInto.result b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertInto.result new file mode 100644 index 0000000..e0c97ef --- /dev/null +++ b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertInto.result @@ -0,0 +1,3 @@ +1, 1996-03-13, {"": "N"}, 7706 +2, 1997-01-28, {"": "N"}, 1191 +3, 1994-02-02, {"": "R"}, 1798 http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoBinaryMultiRegion.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoBinaryMultiRegion.result b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoBinaryMultiRegion.result new file mode 100644 index 0000000..c55873a --- /dev/null +++ b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoBinaryMultiRegion.result @@ -0,0 +1,100 @@ +0, value0 +1, value1 +2, value2 +3, value3 +4, value4 +5, value5 +6, value6 +7, value7 +8, value8 +9, value9 +1, value10 +1, value11 +1, value12 +1, value13 +1, value14 +1, value15 +1, value16 +1, value17 +1, value18 +1, value19 +2, value20 +2, value21 +2, value22 +2, value23 +2, value24 +2, value25 +2, value26 +2, value27 +2, value28 +2, value29 +3, value30 +3, value31 +3, value32 +3, value33 +3, value34 +3, value35 +3, value36 +3, value37 +3, value38 +3, value39 +4, value40 +4, value41 +4, value42 +4, value43 +4, value44 +4, value45 +4, value46 +4, value47 +4, value48 +4, value49 +5, value50 +5, value51 +5, value52 +5, value53 +5, value54 +5, value55 +5, value56 +5, value57 +5, value58 +5, value59 +6, value60 +6, value61 +6, value62 +6, value63 +6, value64 +6, value65 +6, value66 +6, value67 +6, value68 +6, value69 +7, value70 +7, value71 +7, value72 +7, value73 +7, value74 +7, value75 +7, value76 +7, value77 +7, value78 +7, value79 +8, value80 +8, value81 +8, value82 +8, value83 +8, value84 +8, value85 +8, value86 +8, value87 +8, value88 +8, value89 +9, value90 +9, value91 +9, value92 +9, value93 +9, value94 +9, value95 +9, value96 +9, value97 +9, value98 +9, value99 http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoColumnKeyValue.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoColumnKeyValue.result b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoColumnKeyValue.result new file mode 100644 index 0000000..a37593b --- /dev/null +++ b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoColumnKeyValue.result @@ -0,0 +1,21 @@ +0, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-0"} +1, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-1"} +10, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-10"} +11, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-11"} +12, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-12"} +13, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-13"} +14, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-14"} +15, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-15"} +16, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-16"} +17, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-17"} +18, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-18"} +19, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-19"} +2, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-2"} +20, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-20"} +3, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-3"} +4, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-4"} +5, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-5"} +6, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-6"} +7, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-7"} +8, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-8"} +9, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-9"} http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegion.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegion.result b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegion.result new file mode 100644 index 0000000..72013f2 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegion.result @@ -0,0 +1,100 @@ +000, value0 +001, value1 +002, value2 +003, value3 +004, value4 +005, value5 +006, value6 +007, value7 +008, value8 +009, value9 +010, value10 +011, value11 +012, value12 +013, value13 +014, value14 +015, value15 +016, value16 +017, value17 +018, value18 +019, value19 +020, value20 +021, value21 +022, value22 +023, value23 +024, value24 +025, value25 +026, value26 +027, value27 +028, value28 +029, value29 +030, value30 +031, value31 +032, value32 +033, value33 +034, value34 +035, value35 +036, value36 +037, value37 +038, value38 +039, value39 +040, value40 +041, value41 +042, value42 +043, value43 +044, value44 +045, value45 +046, value46 +047, value47 +048, value48 +049, value49 +050, value50 +051, value51 +052, value52 +053, value53 +054, value54 +055, value55 +056, value56 +057, value57 +058, value58 +059, value59 +060, value60 +061, value61 +062, value62 +063, value63 +064, value64 +065, value65 +066, value66 +067, value67 +068, value68 +069, value69 +070, value70 +071, value71 +072, value72 +073, value73 +074, value74 +075, value75 +076, value76 +077, value77 +078, value78 +079, value79 +080, value80 +081, value81 +082, value82 +083, value83 +084, value84 +085, value85 +086, value86 +087, value87 +088, value88 +089, value89 +090, value90 +091, value91 +092, value92 +093, value93 +094, value94 +095, value95 +096, value96 +097, value97 +098, value98 +099, value99 http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegion2.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegion2.result b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegion2.result new file mode 100644 index 0000000..be11042 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegion2.result @@ -0,0 +1,100 @@ +0, value0 +1, value1 +10, value10 +11, value11 +12, value12 +13, value13 +14, value14 +15, value15 +16, value16 +17, value17 +18, value18 +19, value19 +2, value2 +20, value20 +21, value21 +22, value22 +23, value23 +24, value24 +25, value25 +26, value26 +27, value27 +28, value28 +29, value29 +3, value3 +30, value30 +31, value31 +32, value32 +33, value33 +34, value34 +35, value35 +36, value36 +37, value37 +38, value38 +39, value39 +4, value4 +40, value40 +41, value41 +42, value42 +43, value43 +44, value44 +45, value45 +46, value46 +47, value47 +48, value48 +49, value49 +5, value5 +50, value50 +51, value51 +52, value52 +53, value53 +54, value54 +55, value55 +56, value56 +57, value57 +58, value58 +59, value59 +6, value6 +60, value60 +61, value61 +62, value62 +63, value63 +64, value64 +65, value65 +66, value66 +67, value67 +68, value68 +69, value69 +7, value7 +70, value70 +71, value71 +72, value72 +73, value73 +74, value74 +75, value75 +76, value76 +77, value77 +78, value78 +79, value79 +8, value8 +80, value80 +81, value81 +82, value82 +83, value83 +84, value84 +85, value85 +86, value86 +87, value87 +88, value88 +89, value89 +9, value9 +90, value90 +91, value91 +92, value92 +93, value93 +94, value94 +95, value95 +96, value96 +97, value97 +98, value98 +99, value99 http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegionMultiRowFields.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegionMultiRowFields.result b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegionMultiRowFields.result new file mode 100644 index 0000000..0b740c8 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegionMultiRowFields.result @@ -0,0 +1,100 @@ +000_100, value0 +001_101, value1 +002_102, value2 +003_103, value3 +004_104, value4 +005_105, value5 +006_106, value6 +007_107, value7 +008_108, value8 +009_109, value9 +010_110, value10 +011_111, value11 +012_112, value12 +013_113, value13 +014_114, value14 +015_115, value15 +016_116, value16 +017_117, value17 +018_118, value18 +019_119, value19 +020_120, value20 +021_121, value21 +022_122, value22 +023_123, value23 +024_124, value24 +025_125, value25 +026_126, value26 +027_127, value27 +028_128, value28 +029_129, value29 +030_130, value30 +031_131, value31 +032_132, value32 +033_133, value33 +034_134, value34 +035_135, value35 +036_136, value36 +037_137, value37 +038_138, value38 +039_139, value39 +040_140, value40 +041_141, value41 +042_142, value42 +043_143, value43 +044_144, value44 +045_145, value45 +046_146, value46 +047_147, value47 +048_148, value48 +049_149, value49 +050_150, value50 +051_151, value51 +052_152, value52 +053_153, value53 +054_154, value54 +055_155, value55 +056_156, value56 +057_157, value57 +058_158, value58 +059_159, value59 +060_160, value60 +061_161, value61 +062_162, value62 +063_163, value63 +064_164, value64 +065_165, value65 +066_166, value66 +067_167, value67 +068_168, value68 +069_169, value69 +070_170, value70 +071_171, value71 +072_172, value72 +073_173, value73 +074_174, value74 +075_175, value75 +076_176, value76 +077_177, value77 +078_178, value78 +079_179, value79 +080_180, value80 +081_181, value81 +082_182, value82 +083_183, value83 +084_184, value84 +085_185, value85 +086_186, value86 +087_187, value87 +088_188, value88 +089_189, value89 +090_190, value90 +091_191, value91 +092_192, value92 +093_193, value93 +094_194, value94 +095_195, value95 +096_196, value96 +097_197, value97 +098_198, value98 +099_199, value99 http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegionWithSplitFile.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegionWithSplitFile.result b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegionWithSplitFile.result new file mode 100644 index 0000000..72013f2 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegionWithSplitFile.result @@ -0,0 +1,100 @@ +000, value0 +001, value1 +002, value2 +003, value3 +004, value4 +005, value5 +006, value6 +007, value7 +008, value8 +009, value9 +010, value10 +011, value11 +012, value12 +013, value13 +014, value14 +015, value15 +016, value16 +017, value17 +018, value18 +019, value19 +020, value20 +021, value21 +022, value22 +023, value23 +024, value24 +025, value25 +026, value26 +027, value27 +028, value28 +029, value29 +030, value30 +031, value31 +032, value32 +033, value33 +034, value34 +035, value35 +036, value36 +037, value37 +038, value38 +039, value39 +040, value40 +041, value41 +042, value42 +043, value43 +044, value44 +045, value45 +046, value46 +047, value47 +048, value48 +049, value49 +050, value50 +051, value51 +052, value52 +053, value53 +054, value54 +055, value55 +056, value56 +057, value57 +058, value58 +059, value59 +060, value60 +061, value61 +062, value62 +063, value63 +064, value64 +065, value65 +066, value66 +067, value67 +068, value68 +069, value69 +070, value70 +071, value71 +072, value72 +073, value73 +074, value74 +075, value75 +076, value76 +077, value77 +078, value78 +079, value79 +080, value80 +081, value81 +082, value82 +083, value83 +084, value84 +085, value85 +086, value86 +087, value87 +088, value88 +089, value89 +090, value90 +091, value91 +092, value92 +093, value93 +094, value94 +095, value95 +096, value96 +097, value97 +098, value98 +099, value99 http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoRowField.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoRowField.result b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoRowField.result new file mode 100644 index 0000000..152b91b --- /dev/null +++ b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoRowField.result @@ -0,0 +1,4 @@ +1_1, 1996-03-13, N, 7706 +2_2, 1997-01-28, N, 1191 +3_2, 1994-02-02, R, 1798 +3_3, 1993-11-09, R, 6540 http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoUsingPut.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoUsingPut.result b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoUsingPut.result new file mode 100644 index 0000000..1c22960 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoUsingPut.result @@ -0,0 +1,3 @@ +1, 1996-04-12, {"": "N"}, 7311 +2, 1997-01-28, {"": "N"}, 1191 +3, 1993-11-09, {"": "R"}, 6540 http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-core/src/test/resources/results/TestHBaseTable/testJoin.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testJoin.result b/tajo-core/src/test/resources/results/TestHBaseTable/testJoin.result new file mode 100644 index 0000000..fe4fcae --- /dev/null +++ b/tajo-core/src/test/resources/results/TestHBaseTable/testJoin.result @@ -0,0 +1,7 @@ +rk,col1,col2,col3,l_orderkey,l_linestatus +------------------------------- +001,a-1,{"k1":"k1-1", "k2":"k2-1"},1,1,O +001,a-1,{"k1":"k1-1", "k2":"k2-1"},1,1,O +002,a-2,{"k1":"k1-2", "k2":"k2-2"},2,2,O +003,a-3,{"k1":"k1-3", "k2":"k2-3"},3,3,F +003,a-3,{"k1":"k1-3", "k2":"k2-3"},3,3,F \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-core/src/test/resources/results/TestHBaseTable/testNonForwardQuery.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testNonForwardQuery.result b/tajo-core/src/test/resources/results/TestHBaseTable/testNonForwardQuery.result new file mode 100644 index 0000000..4f5fd8f --- /dev/null +++ b/tajo-core/src/test/resources/results/TestHBaseTable/testNonForwardQuery.result @@ -0,0 +1,102 @@ +rk,col1,col2,col3 +------------------------------- +000,a-0,{"k1":"k1-0", "k2":"k2-0"},0 +001,a-1,{"k1":"k1-1", "k2":"k2-1"},1 +002,a-2,{"k1":"k1-2", "k2":"k2-2"},2 +003,a-3,{"k1":"k1-3", "k2":"k2-3"},3 +004,a-4,{"k1":"k1-4", "k2":"k2-4"},4 +005,a-5,{"k1":"k1-5", "k2":"k2-5"},5 +006,a-6,{"k1":"k1-6", "k2":"k2-6"},6 +007,a-7,{"k1":"k1-7", "k2":"k2-7"},7 +008,a-8,{"k1":"k1-8", "k2":"k2-8"},8 +009,a-9,{"k1":"k1-9", "k2":"k2-9"},9 +010,a-10,{"k1":"k1-10", "k2":"k2-10"},10 +011,a-11,{"k1":"k1-11", "k2":"k2-11"},11 +012,a-12,{"k1":"k1-12", "k2":"k2-12"},12 +013,a-13,{"k1":"k1-13", "k2":"k2-13"},13 +014,a-14,{"k1":"k1-14", "k2":"k2-14"},14 +015,a-15,{"k1":"k1-15", "k2":"k2-15"},15 +016,a-16,{"k1":"k1-16", "k2":"k2-16"},16 +017,a-17,{"k1":"k1-17", "k2":"k2-17"},17 +018,a-18,{"k1":"k1-18", "k2":"k2-18"},18 +019,a-19,{"k1":"k1-19", "k2":"k2-19"},19 +020,a-20,{"k1":"k1-20", "k2":"k2-20"},20 +021,a-21,{"k1":"k1-21", "k2":"k2-21"},21 +022,a-22,{"k1":"k1-22", "k2":"k2-22"},22 +023,a-23,{"k1":"k1-23", "k2":"k2-23"},23 +024,a-24,{"k1":"k1-24", "k2":"k2-24"},24 +025,a-25,{"k1":"k1-25", "k2":"k2-25"},25 +026,a-26,{"k1":"k1-26", "k2":"k2-26"},26 +027,a-27,{"k1":"k1-27", "k2":"k2-27"},27 +028,a-28,{"k1":"k1-28", "k2":"k2-28"},28 +029,a-29,{"k1":"k1-29", "k2":"k2-29"},29 +030,a-30,{"k1":"k1-30", "k2":"k2-30"},30 +031,a-31,{"k1":"k1-31", "k2":"k2-31"},31 +032,a-32,{"k1":"k1-32", "k2":"k2-32"},32 +033,a-33,{"k1":"k1-33", "k2":"k2-33"},33 +034,a-34,{"k1":"k1-34", "k2":"k2-34"},34 +035,a-35,{"k1":"k1-35", "k2":"k2-35"},35 +036,a-36,{"k1":"k1-36", "k2":"k2-36"},36 +037,a-37,{"k1":"k1-37", "k2":"k2-37"},37 +038,a-38,{"k1":"k1-38", "k2":"k2-38"},38 +039,a-39,{"k1":"k1-39", "k2":"k2-39"},39 +040,a-40,{"k1":"k1-40", "k2":"k2-40"},40 +041,a-41,{"k1":"k1-41", "k2":"k2-41"},41 +042,a-42,{"k1":"k1-42", "k2":"k2-42"},42 +043,a-43,{"k1":"k1-43", "k2":"k2-43"},43 +044,a-44,{"k1":"k1-44", "k2":"k2-44"},44 +045,a-45,{"k1":"k1-45", "k2":"k2-45"},45 +046,a-46,{"k1":"k1-46", "k2":"k2-46"},46 +047,a-47,{"k1":"k1-47", "k2":"k2-47"},47 +048,a-48,{"k1":"k1-48", "k2":"k2-48"},48 +049,a-49,{"k1":"k1-49", "k2":"k2-49"},49 +050,a-50,{"k1":"k1-50", "k2":"k2-50"},50 +051,a-51,{"k1":"k1-51", "k2":"k2-51"},51 +052,a-52,{"k1":"k1-52", "k2":"k2-52"},52 +053,a-53,{"k1":"k1-53", "k2":"k2-53"},53 +054,a-54,{"k1":"k1-54", "k2":"k2-54"},54 +055,a-55,{"k1":"k1-55", "k2":"k2-55"},55 +056,a-56,{"k1":"k1-56", "k2":"k2-56"},56 +057,a-57,{"k1":"k1-57", "k2":"k2-57"},57 +058,a-58,{"k1":"k1-58", "k2":"k2-58"},58 +059,a-59,{"k1":"k1-59", "k2":"k2-59"},59 +060,a-60,{"k1":"k1-60", "k2":"k2-60"},60 +061,a-61,{"k1":"k1-61", "k2":"k2-61"},61 +062,a-62,{"k1":"k1-62", "k2":"k2-62"},62 +063,a-63,{"k1":"k1-63", "k2":"k2-63"},63 +064,a-64,{"k1":"k1-64", "k2":"k2-64"},64 +065,a-65,{"k1":"k1-65", "k2":"k2-65"},65 +066,a-66,{"k1":"k1-66", "k2":"k2-66"},66 +067,a-67,{"k1":"k1-67", "k2":"k2-67"},67 +068,a-68,{"k1":"k1-68", "k2":"k2-68"},68 +069,a-69,{"k1":"k1-69", "k2":"k2-69"},69 +070,a-70,{"k1":"k1-70", "k2":"k2-70"},70 +071,a-71,{"k1":"k1-71", "k2":"k2-71"},71 +072,a-72,{"k1":"k1-72", "k2":"k2-72"},72 +073,a-73,{"k1":"k1-73", "k2":"k2-73"},73 +074,a-74,{"k1":"k1-74", "k2":"k2-74"},74 +075,a-75,{"k1":"k1-75", "k2":"k2-75"},75 +076,a-76,{"k1":"k1-76", "k2":"k2-76"},76 +077,a-77,{"k1":"k1-77", "k2":"k2-77"},77 +078,a-78,{"k1":"k1-78", "k2":"k2-78"},78 +079,a-79,{"k1":"k1-79", "k2":"k2-79"},79 +080,a-80,{"k1":"k1-80", "k2":"k2-80"},80 +081,a-81,{"k1":"k1-81", "k2":"k2-81"},81 +082,a-82,{"k1":"k1-82", "k2":"k2-82"},82 +083,a-83,{"k1":"k1-83", "k2":"k2-83"},83 +084,a-84,{"k1":"k1-84", "k2":"k2-84"},84 +085,a-85,{"k1":"k1-85", "k2":"k2-85"},85 +086,a-86,{"k1":"k1-86", "k2":"k2-86"},86 +087,a-87,{"k1":"k1-87", "k2":"k2-87"},87 +088,a-88,{"k1":"k1-88", "k2":"k2-88"},88 +089,a-89,{"k1":"k1-89", "k2":"k2-89"},89 +090,a-90,{"k1":"k1-90", "k2":"k2-90"},90 +091,a-91,{"k1":"k1-91", "k2":"k2-91"},91 +092,a-92,{"k1":"k1-92", "k2":"k2-92"},92 +093,a-93,{"k1":"k1-93", "k2":"k2-93"},93 +094,a-94,{"k1":"k1-94", "k2":"k2-94"},94 +095,a-95,{"k1":"k1-95", "k2":"k2-95"},95 +096,a-96,{"k1":"k1-96", "k2":"k2-96"},96 +097,a-97,{"k1":"k1-97", "k2":"k2-97"},97 +098,a-98,{"k1":"k1-98", "k2":"k2-98"},98 +099,a-99,{"k1":"k1-99", "k2":"k2-99"},99 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-core/src/test/resources/results/TestHBaseTable/testRowFieldSelectQuery.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testRowFieldSelectQuery.result b/tajo-core/src/test/resources/results/TestHBaseTable/testRowFieldSelectQuery.result new file mode 100644 index 0000000..34369df --- /dev/null +++ b/tajo-core/src/test/resources/results/TestHBaseTable/testRowFieldSelectQuery.result @@ -0,0 +1,88 @@ +rk1,rk2,col3 +------------------------------- +field1-21,field2-21,a-21 +field1-22,field2-22,a-22 +field1-23,field2-23,a-23 +field1-24,field2-24,a-24 +field1-25,field2-25,a-25 +field1-26,field2-26,a-26 +field1-27,field2-27,a-27 +field1-28,field2-28,a-28 +field1-29,field2-29,a-29 +field1-30,field2-30,a-30 +field1-31,field2-31,a-31 +field1-32,field2-32,a-32 +field1-33,field2-33,a-33 +field1-34,field2-34,a-34 +field1-35,field2-35,a-35 +field1-36,field2-36,a-36 +field1-37,field2-37,a-37 +field1-38,field2-38,a-38 +field1-39,field2-39,a-39 +field1-3,field2-3,a-3 +field1-40,field2-40,a-40 +field1-41,field2-41,a-41 +field1-42,field2-42,a-42 +field1-43,field2-43,a-43 +field1-44,field2-44,a-44 +field1-45,field2-45,a-45 +field1-46,field2-46,a-46 +field1-47,field2-47,a-47 +field1-48,field2-48,a-48 +field1-49,field2-49,a-49 +field1-4,field2-4,a-4 +field1-50,field2-50,a-50 +field1-51,field2-51,a-51 +field1-52,field2-52,a-52 +field1-53,field2-53,a-53 +field1-54,field2-54,a-54 +field1-55,field2-55,a-55 +field1-56,field2-56,a-56 +field1-57,field2-57,a-57 +field1-58,field2-58,a-58 +field1-59,field2-59,a-59 +field1-5,field2-5,a-5 +field1-60,field2-60,a-60 +field1-61,field2-61,a-61 +field1-62,field2-62,a-62 +field1-63,field2-63,a-63 +field1-64,field2-64,a-64 +field1-65,field2-65,a-65 +field1-66,field2-66,a-66 +field1-67,field2-67,a-67 +field1-68,field2-68,a-68 +field1-69,field2-69,a-69 +field1-6,field2-6,a-6 +field1-70,field2-70,a-70 +field1-71,field2-71,a-71 +field1-72,field2-72,a-72 +field1-73,field2-73,a-73 +field1-74,field2-74,a-74 +field1-75,field2-75,a-75 +field1-76,field2-76,a-76 +field1-77,field2-77,a-77 +field1-78,field2-78,a-78 +field1-79,field2-79,a-79 +field1-7,field2-7,a-7 +field1-80,field2-80,a-80 +field1-81,field2-81,a-81 +field1-82,field2-82,a-82 +field1-83,field2-83,a-83 +field1-84,field2-84,a-84 +field1-85,field2-85,a-85 +field1-86,field2-86,a-86 +field1-87,field2-87,a-87 +field1-88,field2-88,a-88 +field1-89,field2-89,a-89 +field1-8,field2-8,a-8 +field1-90,field2-90,a-90 +field1-91,field2-91,a-91 +field1-92,field2-92,a-92 +field1-93,field2-93,a-93 +field1-94,field2-94,a-94 +field1-95,field2-95,a-95 +field1-96,field2-96,a-96 +field1-97,field2-97,a-97 +field1-98,field2-98,a-98 +field1-99,field2-99,a-99 +field1-9,field2-9,a-9 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-core/src/test/resources/results/TestHBaseTable/testSimpleSelectQuery.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testSimpleSelectQuery.result b/tajo-core/src/test/resources/results/TestHBaseTable/testSimpleSelectQuery.result new file mode 100644 index 0000000..ef2f27a --- /dev/null +++ b/tajo-core/src/test/resources/results/TestHBaseTable/testSimpleSelectQuery.result @@ -0,0 +1,88 @@ +rk,col1,col2,col3 +------------------------------- +21,a-21,{"k1":"k1-21", "k2":"k2-21"},b-21 +22,a-22,{"k1":"k1-22", "k2":"k2-22"},b-22 +23,a-23,{"k1":"k1-23", "k2":"k2-23"},b-23 +24,a-24,{"k1":"k1-24", "k2":"k2-24"},b-24 +25,a-25,{"k1":"k1-25", "k2":"k2-25"},b-25 +26,a-26,{"k1":"k1-26", "k2":"k2-26"},b-26 +27,a-27,{"k1":"k1-27", "k2":"k2-27"},b-27 +28,a-28,{"k1":"k1-28", "k2":"k2-28"},b-28 +29,a-29,{"k1":"k1-29", "k2":"k2-29"},b-29 +3,a-3,{"k1":"k1-3", "k2":"k2-3"},b-3 +30,a-30,{"k1":"k1-30", "k2":"k2-30"},b-30 +31,a-31,{"k1":"k1-31", "k2":"k2-31"},b-31 +32,a-32,{"k1":"k1-32", "k2":"k2-32"},b-32 +33,a-33,{"k1":"k1-33", "k2":"k2-33"},b-33 +34,a-34,{"k1":"k1-34", "k2":"k2-34"},b-34 +35,a-35,{"k1":"k1-35", "k2":"k2-35"},b-35 +36,a-36,{"k1":"k1-36", "k2":"k2-36"},b-36 +37,a-37,{"k1":"k1-37", "k2":"k2-37"},b-37 +38,a-38,{"k1":"k1-38", "k2":"k2-38"},b-38 +39,a-39,{"k1":"k1-39", "k2":"k2-39"},b-39 +4,a-4,{"k1":"k1-4", "k2":"k2-4"},b-4 +40,a-40,{"k1":"k1-40", "k2":"k2-40"},b-40 +41,a-41,{"k1":"k1-41", "k2":"k2-41"},b-41 +42,a-42,{"k1":"k1-42", "k2":"k2-42"},b-42 +43,a-43,{"k1":"k1-43", "k2":"k2-43"},b-43 +44,a-44,{"k1":"k1-44", "k2":"k2-44"},b-44 +45,a-45,{"k1":"k1-45", "k2":"k2-45"},b-45 +46,a-46,{"k1":"k1-46", "k2":"k2-46"},b-46 +47,a-47,{"k1":"k1-47", "k2":"k2-47"},b-47 +48,a-48,{"k1":"k1-48", "k2":"k2-48"},b-48 +49,a-49,{"k1":"k1-49", "k2":"k2-49"},b-49 +5,a-5,{"k1":"k1-5", "k2":"k2-5"},b-5 +50,a-50,{"k1":"k1-50", "k2":"k2-50"},b-50 +51,a-51,{"k1":"k1-51", "k2":"k2-51"},b-51 +52,a-52,{"k1":"k1-52", "k2":"k2-52"},b-52 +53,a-53,{"k1":"k1-53", "k2":"k2-53"},b-53 +54,a-54,{"k1":"k1-54", "k2":"k2-54"},b-54 +55,a-55,{"k1":"k1-55", "k2":"k2-55"},b-55 +56,a-56,{"k1":"k1-56", "k2":"k2-56"},b-56 +57,a-57,{"k1":"k1-57", "k2":"k2-57"},b-57 +58,a-58,{"k1":"k1-58", "k2":"k2-58"},b-58 +59,a-59,{"k1":"k1-59", "k2":"k2-59"},b-59 +6,a-6,{"k1":"k1-6", "k2":"k2-6"},b-6 +60,a-60,{"k1":"k1-60", "k2":"k2-60"},b-60 +61,a-61,{"k1":"k1-61", "k2":"k2-61"},b-61 +62,a-62,{"k1":"k1-62", "k2":"k2-62"},b-62 +63,a-63,{"k1":"k1-63", "k2":"k2-63"},b-63 +64,a-64,{"k1":"k1-64", "k2":"k2-64"},b-64 +65,a-65,{"k1":"k1-65", "k2":"k2-65"},b-65 +66,a-66,{"k1":"k1-66", "k2":"k2-66"},b-66 +67,a-67,{"k1":"k1-67", "k2":"k2-67"},b-67 +68,a-68,{"k1":"k1-68", "k2":"k2-68"},b-68 +69,a-69,{"k1":"k1-69", "k2":"k2-69"},b-69 +7,a-7,{"k1":"k1-7", "k2":"k2-7"},b-7 +70,a-70,{"k1":"k1-70", "k2":"k2-70"},b-70 +71,a-71,{"k1":"k1-71", "k2":"k2-71"},b-71 +72,a-72,{"k1":"k1-72", "k2":"k2-72"},b-72 +73,a-73,{"k1":"k1-73", "k2":"k2-73"},b-73 +74,a-74,{"k1":"k1-74", "k2":"k2-74"},b-74 +75,a-75,{"k1":"k1-75", "k2":"k2-75"},b-75 +76,a-76,{"k1":"k1-76", "k2":"k2-76"},b-76 +77,a-77,{"k1":"k1-77", "k2":"k2-77"},b-77 +78,a-78,{"k1":"k1-78", "k2":"k2-78"},b-78 +79,a-79,{"k1":"k1-79", "k2":"k2-79"},b-79 +8,a-8,{"k1":"k1-8", "k2":"k2-8"},b-8 +80,a-80,{"k1":"k1-80", "k2":"k2-80"},b-80 +81,a-81,{"k1":"k1-81", "k2":"k2-81"},b-81 +82,a-82,{"k1":"k1-82", "k2":"k2-82"},b-82 +83,a-83,{"k1":"k1-83", "k2":"k2-83"},b-83 +84,a-84,{"k1":"k1-84", "k2":"k2-84"},b-84 +85,a-85,{"k1":"k1-85", "k2":"k2-85"},b-85 +86,a-86,{"k1":"k1-86", "k2":"k2-86"},b-86 +87,a-87,{"k1":"k1-87", "k2":"k2-87"},b-87 +88,a-88,{"k1":"k1-88", "k2":"k2-88"},b-88 +89,a-89,{"k1":"k1-89", "k2":"k2-89"},b-89 +9,a-9,{"k1":"k1-9", "k2":"k2-9"},b-9 +90,a-90,{"k1":"k1-90", "k2":"k2-90"},b-90 +91,a-91,{"k1":"k1-91", "k2":"k2-91"},b-91 +92,a-92,{"k1":"k1-92", "k2":"k2-92"},b-92 +93,a-93,{"k1":"k1-93", "k2":"k2-93"},b-93 +94,a-94,{"k1":"k1-94", "k2":"k2-94"},b-94 +95,a-95,{"k1":"k1-95", "k2":"k2-95"},b-95 +96,a-96,{"k1":"k1-96", "k2":"k2-96"},b-96 +97,a-97,{"k1":"k1-97", "k2":"k2-97"},b-97 +98,a-98,{"k1":"k1-98", "k2":"k2-98"},b-98 +99,a-99,{"k1":"k1-99", "k2":"k2-99"},b-99 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-dist/src/main/bin/tajo ---------------------------------------------------------------------- diff --git a/tajo-dist/src/main/bin/tajo b/tajo-dist/src/main/bin/tajo index d4e99f9..8a9c17a 100755 --- a/tajo-dist/src/main/bin/tajo +++ b/tajo-dist/src/main/bin/tajo @@ -226,6 +226,21 @@ HDFS_LIBRARY_PATH="${HADOOP_HOME}/lib/native/" ############################################################################## ############################################################################## +# Set HBase CLASSPATH +############################################################################## +if [ "$HBASE_HOME" != "" ]; then + for f in ${HBASE_HOME}/lib/hbase-*.jar; do + CLASSPATH=${CLASSPATH}:$f; + done + for f in ${HBASE_HOME}/lib/htrace-*.jar; do + CLASSPATH=${CLASSPATH}:$f; + done +fi +############################################################################## +# HBase Configuration End +############################################################################## + +############################################################################## # Find and Set Hive CLASSPATH ############################################################################## http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-dist/src/main/conf/tajo-env.sh ---------------------------------------------------------------------- diff --git a/tajo-dist/src/main/conf/tajo-env.sh b/tajo-dist/src/main/conf/tajo-env.sh index 92d00bd..064a82f 100755 --- a/tajo-dist/src/main/conf/tajo-env.sh +++ b/tajo-dist/src/main/conf/tajo-env.sh @@ -24,6 +24,9 @@ # Hadoop home. Required # export HADOOP_HOME= +# HBase home. optional +# export HBASE_HOME= + # The java implementation to use. Required. # export JAVA_HOME=/usr/java/default http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java index bcc3bce..750e64e 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java @@ -44,6 +44,7 @@ import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor; import java.util.LinkedHashSet; +import java.util.List; import java.util.Set; import java.util.Stack; @@ -86,6 +87,12 @@ public class LogicalOptimizer { } } + public void addRuleAfterToJoinOpt(RewriteRule rewriteRule) { + if (rewriteRule != null) { + rulesAfterToJoinOpt.addRewriteRule(rewriteRule); + } + } + public LogicalNode optimize(LogicalPlan plan) throws PlanningException { return optimize(null, plan); } http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-plan/src/main/java/org/apache/tajo/plan/logical/SortNode.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/SortNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/SortNode.java index a697f9f..2e5e159 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/SortNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/SortNode.java @@ -25,10 +25,16 @@ import org.apache.tajo.plan.PlanString; import org.apache.tajo.util.TUtil; public final class SortNode extends UnaryNode implements Cloneable { + public static enum SortPurpose { + NORMAL, + STORAGE_SPECIFIED + } @Expose private SortSpec [] sortKeys; + @Expose private SortPurpose sortPurpose; public SortNode(int pid) { super(pid, NodeType.SORT); + sortPurpose = SortPurpose.NORMAL; } public void setSortSpecs(SortSpec[] sortSpecs) { @@ -65,8 +71,7 @@ public final class SortNode extends UnaryNode implements Cloneable { PlanString planStr = new PlanString(this); StringBuilder sb = new StringBuilder("Sort Keys: "); for (int i = 0; i < sortKeys.length; i++) { - sb.append(sortKeys[i].getSortKey().getSimpleName()).append(" ") - .append(sortKeys[i].isAscending() ? "asc" : "desc"); + sb.append(sortKeys[i].toString()); if( i < sortKeys.length - 1) { sb.append(","); } @@ -78,8 +83,7 @@ public final class SortNode extends UnaryNode implements Cloneable { public String toString() { StringBuilder sb = new StringBuilder("Sort [key= "); for (int i = 0; i < sortKeys.length; i++) { - sb.append(sortKeys[i].getSortKey().getQualifiedName()).append(" ") - .append(sortKeys[i].isAscending() ? "asc" : "desc"); + sb.append(sortKeys[i].toString()); if(i < sortKeys.length - 1) { sb.append(","); } @@ -91,4 +95,12 @@ public final class SortNode extends UnaryNode implements Cloneable { return sb.toString()+"\n" + getChild().toString(); } + + public SortPurpose getSortPurpose() { + return sortPurpose; + } + + public void setSortPurpose(SortPurpose sortPurpose) { + this.sortPurpose = sortPurpose; + } } http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java index 14b24a7..46fda2e 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java @@ -306,7 +306,7 @@ public class PartitionedTableRewriter implements RewriteRule { boolean sameVariable = EvalTreeUtil.findUniqueColumns(orEval.getLeftExpr()) - .equals(EvalTreeUtil.findUniqueColumns(orEval.getRightExpr())); + .equals(EvalTreeUtil.findUniqueColumns(orEval.getRightExpr())); return indexable && sameVariable; } else { http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java index 6868b6c..a7db188 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java @@ -23,11 +23,9 @@ import com.google.common.collect.Lists; import com.google.common.collect.Sets; import org.apache.tajo.algebra.*; import org.apache.tajo.annotation.Nullable; -import org.apache.tajo.catalog.Column; -import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.SchemaUtil; -import org.apache.tajo.catalog.SortSpec; +import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.proto.CatalogProtos; +import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.plan.*; import org.apache.tajo.plan.expr.*; @@ -37,6 +35,7 @@ import org.apache.tajo.plan.visitor.ExplainLogicalPlanVisitor; import org.apache.tajo.plan.visitor.SimpleAlgebraVisitor; import org.apache.tajo.util.TUtil; +import java.io.IOException; import java.util.*; public class PlannerUtil { @@ -776,4 +775,109 @@ public class PlannerUtil { return explains.toString(); } + + public static boolean isFileStorageType(String storageType) { + if (storageType.equalsIgnoreCase("hbase")) { + return false; + } else { + return true; + } + } + + public static boolean isFileStorageType(StoreType storageType) { + if (storageType== StoreType.HBASE) { + return false; + } else { + return true; + } + } + + public static StoreType getStoreType(LogicalPlan plan) { + LogicalRootNode rootNode = plan.getRootBlock().getRoot(); + NodeType nodeType = rootNode.getChild().getType(); + if (nodeType == NodeType.CREATE_TABLE) { + return ((CreateTableNode)rootNode.getChild()).getStorageType(); + } else if (nodeType == NodeType.INSERT) { + return ((InsertNode)rootNode.getChild()).getStorageType(); + } else { + return null; + } + } + + public static String getStoreTableName(LogicalPlan plan) { + LogicalRootNode rootNode = plan.getRootBlock().getRoot(); + NodeType nodeType = rootNode.getChild().getType(); + if (nodeType == NodeType.CREATE_TABLE) { + return ((CreateTableNode)rootNode.getChild()).getTableName(); + } else if (nodeType == NodeType.INSERT) { + return ((InsertNode)rootNode.getChild()).getTableName(); + } else { + return null; + } + } + + public static TableDesc getTableDesc(CatalogService catalog, LogicalNode node) throws IOException { + if (node.getType() == NodeType.CREATE_TABLE) { + return createTableDesc((CreateTableNode)node); + } + String tableName = null; + InsertNode insertNode = null; + if (node.getType() == NodeType.INSERT) { + insertNode = (InsertNode)node; + tableName = insertNode.getTableName(); + } else { + return null; + } + + if (tableName != null) { + String[] tableTokens = tableName.split("\\."); + if (tableTokens.length >= 2) { + if (catalog.existsTable(tableTokens[0], tableTokens[1])) { + return catalog.getTableDesc(tableTokens[0], tableTokens[1]); + } + } + } else { + if (insertNode.getPath() != null) { + //insert ... location + return createTableDesc(insertNode); + } + } + return null; + } + + private static TableDesc createTableDesc(CreateTableNode createTableNode) { + TableMeta meta = new TableMeta(createTableNode.getStorageType(), createTableNode.getOptions()); + + TableDesc tableDescTobeCreated = + new TableDesc( + createTableNode.getTableName(), + createTableNode.getTableSchema(), + meta, + createTableNode.getPath()); + + tableDescTobeCreated.setExternal(createTableNode.isExternal()); + + if (createTableNode.hasPartition()) { + tableDescTobeCreated.setPartitionMethod(createTableNode.getPartitionMethod()); + } + + return tableDescTobeCreated; + } + + private static TableDesc createTableDesc(InsertNode insertNode) { + TableMeta meta = new TableMeta(insertNode.getStorageType(), insertNode.getOptions()); + + TableDesc tableDescTobeCreated = + new TableDesc( + insertNode.getTableName(), + insertNode.getTableSchema(), + meta, + insertNode.getPath()); + + if (insertNode.hasPartition()) { + tableDescTobeCreated.setPartitionMethod(insertNode.getPartitionMethod()); + } + + return tableDescTobeCreated; + } } http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java index 95e0f30..5356c57 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/verifier/PreLogicalPlanVerifier.java @@ -29,8 +29,10 @@ import org.apache.tajo.catalog.proto.CatalogProtos; import org.apache.tajo.plan.util.ExprFinder; import org.apache.tajo.plan.PlanningException; import org.apache.tajo.plan.algebra.BaseAlgebraVisitor; +import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.util.TUtil; +import java.io.IOException; import java.util.Set; import java.util.Stack; @@ -277,6 +279,7 @@ public class PreLogicalPlanVerifier extends BaseAlgebraVisitor<PreLogicalPlanVer context.state.addVerification(String.format("relation \"%s\" does not exist", qualifiedName)); return null; } + if (table.hasPartition()) { int columnSize = table.getSchema().getColumns().size(); columnSize += table.getPartitionMethod().getExpressionSchema().getColumns().size(); http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-project/pom.xml ---------------------------------------------------------------------- diff --git a/tajo-project/pom.xml b/tajo-project/pom.xml index 5493095..333cda0 100644 --- a/tajo-project/pom.xml +++ b/tajo-project/pom.xml @@ -36,6 +36,7 @@ <hadoop.version>2.5.1</hadoop.version> <protobuf.version>2.5.0</protobuf.version> <tajo.version>0.9.1-SNAPSHOT</tajo.version> + <hbase.version>0.98.7-hadoop2</hbase.version> <tajo.root>${project.parent.relativePath}/..</tajo.root> <extra.source.path>src/main/hadoop-${hadoop.version}</extra.source.path> </properties> http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-storage/pom.xml ---------------------------------------------------------------------- diff --git a/tajo-storage/pom.xml b/tajo-storage/pom.xml index 680e520..c126afd 100644 --- a/tajo-storage/pom.xml +++ b/tajo-storage/pom.xml @@ -131,6 +131,7 @@ <argument>--proto_path=../tajo-catalog/tajo-catalog-common/src/main/proto</argument> <argument>--java_out=target/generated-sources/proto</argument> <argument>src/main/proto/IndexProtos.proto</argument> + <argument>src/main/proto/StorageFragmentProtos.proto</argument> </arguments> </configuration> <goals> @@ -176,6 +177,10 @@ <groupId>org.apache.tajo</groupId> <artifactId>tajo-catalog-common</artifactId> </dependency> + <dependency> + <groupId>org.apache.tajo</groupId> + <artifactId>tajo-plan</artifactId> + </dependency> <dependency> <groupId>org.apache.avro</groupId> @@ -291,6 +296,12 @@ </exclusions> </dependency> <dependency> + <groupId>org.apache.hadoop</groupId> + <artifactId>hadoop-mapreduce-client-core</artifactId> + <version>${hadoop.version}</version> + <scope>provided</scope> + </dependency> + <dependency> <groupId>com.google.protobuf</groupId> <artifactId>protobuf-java</artifactId> </dependency> @@ -315,6 +326,18 @@ <version>${parquet.format.version}</version> </dependency> <dependency> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-server</artifactId> + <version>${hbase.version}</version> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.hbase</groupId> + <artifactId>hbase-client</artifactId> + <version>${hbase.version}</version> + <scope>provided</scope> + </dependency> + <dependency> <groupId>io.netty</groupId> <artifactId>netty-buffer</artifactId> </dependency> http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-storage/src/main/java/org/apache/tajo/storage/CSVFile.java ---------------------------------------------------------------------- diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/CSVFile.java b/tajo-storage/src/main/java/org/apache/tajo/storage/CSVFile.java index dff337a..1e2b0f3 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/CSVFile.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/CSVFile.java @@ -28,6 +28,7 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.io.IOUtils; import org.apache.hadoop.io.Text; import org.apache.hadoop.io.compress.*; +import org.apache.tajo.QueryUnitAttemptId; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.catalog.proto.CatalogProtos; @@ -76,9 +77,10 @@ public class CSVFile { private NonSyncByteArrayOutputStream os = new NonSyncByteArrayOutputStream(BUFFER_SIZE); private SerializerDeserializer serde; - public CSVAppender(Configuration conf, final Schema schema, final TableMeta meta, final Path path) throws IOException { - super(conf, schema, meta, path); - this.fs = path.getFileSystem(conf); + public CSVAppender(Configuration conf, final QueryUnitAttemptId taskAttemptId, + final Schema schema, final TableMeta meta, final Path workDir) throws IOException { + super(conf, taskAttemptId, schema, meta, workDir); + this.fs = workDir.getFileSystem(conf); this.meta = meta; this.schema = schema; this.delimiter = StringEscapeUtils.unescapeJava( @@ -99,7 +101,7 @@ public class CSVFile { @Override public void init() throws IOException { if (!fs.exists(path.getParent())) { - throw new FileNotFoundException(path.toString()); + throw new FileNotFoundException(path.getParent().toString()); } //determine the intermediate file type http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-storage/src/main/java/org/apache/tajo/storage/FileAppender.java ---------------------------------------------------------------------- diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/FileAppender.java b/tajo-storage/src/main/java/org/apache/tajo/storage/FileAppender.java index 04278e9..117d3da 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/FileAppender.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/FileAppender.java @@ -18,28 +18,48 @@ package org.apache.tajo.storage; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; +import org.apache.tajo.QueryUnitAttemptId; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableMeta; +import org.apache.tajo.conf.TajoConf; import java.io.IOException; public abstract class FileAppender implements Appender { + private static final Log LOG = LogFactory.getLog(FileAppender.class); + protected boolean inited = false; protected final Configuration conf; protected final TableMeta meta; protected final Schema schema; - protected final Path path; + protected final Path workDir; + protected final QueryUnitAttemptId taskAttemptId; protected boolean enabledStats; - - public FileAppender(Configuration conf, Schema schema, TableMeta meta, Path path) { + protected Path path; + + public FileAppender(Configuration conf, QueryUnitAttemptId taskAttemptId, Schema schema, + TableMeta meta, Path workDir) { this.conf = conf; this.meta = meta; this.schema = schema; - this.path = path; + this.workDir = workDir; + this.taskAttemptId = taskAttemptId; + + try { + if (taskAttemptId != null) { + this.path = StorageManager.getFileStorageManager((TajoConf) conf).getAppenderFilePath(taskAttemptId, workDir); + } else { + this.path = workDir; + } + } catch (IOException e) { + LOG.error(e.getMessage(), e); + } } public void init() throws IOException { http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java ---------------------------------------------------------------------- diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java index c925cd8..ee24b27 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/FileStorageManager.java @@ -18,6 +18,7 @@ package org.apache.tajo.storage; +import com.google.common.annotations.VisibleForTesting; import com.google.common.base.Preconditions; import com.google.common.collect.Lists; import org.apache.commons.logging.Log; @@ -26,35 +27,67 @@ import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.*; import org.apache.hadoop.hdfs.DFSConfigKeys; import org.apache.hadoop.hdfs.DistributedFileSystem; -import org.apache.tajo.TajoConstants; -import org.apache.tajo.catalog.CatalogUtil; -import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.TableDesc; -import org.apache.tajo.catalog.TableMeta; -import org.apache.tajo.catalog.proto.CatalogProtos; -import org.apache.tajo.catalog.proto.CatalogProtos.FragmentProto; +import org.apache.tajo.*; +import org.apache.tajo.catalog.*; +import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.catalog.statistics.TableStats; import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.conf.TajoConf.ConfVars; +import org.apache.tajo.plan.logical.LogicalNode; +import org.apache.tajo.plan.logical.ScanNode; import org.apache.tajo.storage.fragment.FileFragment; import org.apache.tajo.storage.fragment.Fragment; -import org.apache.tajo.storage.fragment.FragmentConvertor; import org.apache.tajo.util.*; -import java.io.FileNotFoundException; import java.io.IOException; -import java.lang.reflect.Constructor; -import java.net.URI; +import java.text.NumberFormat; import java.util.*; +import java.util.concurrent.atomic.AtomicInteger; public class FileStorageManager extends StorageManager { private final Log LOG = LogFactory.getLog(FileStorageManager.class); + static final String OUTPUT_FILE_PREFIX="part-"; + static final ThreadLocal<NumberFormat> OUTPUT_FILE_FORMAT_SUBQUERY = + new ThreadLocal<NumberFormat>() { + @Override + public NumberFormat initialValue() { + NumberFormat fmt = NumberFormat.getInstance(); + fmt.setGroupingUsed(false); + fmt.setMinimumIntegerDigits(2); + return fmt; + } + }; + static final ThreadLocal<NumberFormat> OUTPUT_FILE_FORMAT_TASK = + new ThreadLocal<NumberFormat>() { + @Override + public NumberFormat initialValue() { + NumberFormat fmt = NumberFormat.getInstance(); + fmt.setGroupingUsed(false); + fmt.setMinimumIntegerDigits(6); + return fmt; + } + }; + + static final ThreadLocal<NumberFormat> OUTPUT_FILE_FORMAT_SEQ = + new ThreadLocal<NumberFormat>() { + @Override + public NumberFormat initialValue() { + NumberFormat fmt = NumberFormat.getInstance(); + fmt.setGroupingUsed(false); + fmt.setMinimumIntegerDigits(3); + return fmt; + } + }; + protected FileSystem fs; protected Path tableBaseDir; protected boolean blocksMetadataEnabled; private static final HdfsVolumeId zeroVolumeId = new HdfsVolumeId(Bytes.toBytes(0)); + public FileStorageManager(StoreType storeType) { + super(storeType); + } + @Override protected void storageInit() throws IOException { this.tableBaseDir = TajoConf.getWarehouseDir(conf); @@ -114,47 +147,10 @@ public class FileStorageManager extends StorageManager { return new Path(tableBaseDir, tableName); } - public Appender getAppender(TableMeta meta, Schema schema, Path path) + @VisibleForTesting + public Appender getAppender(TableMeta meta, Schema schema, Path filePath) throws IOException { - Appender appender; - - Class<? extends FileAppender> appenderClass; - - String handlerName = meta.getStoreType().name().toLowerCase(); - appenderClass = APPENDER_HANDLER_CACHE.get(handlerName); - if (appenderClass == null) { - appenderClass = conf.getClass( - String.format("tajo.storage.appender-handler.%s.class", - meta.getStoreType().name().toLowerCase()), null, - FileAppender.class); - APPENDER_HANDLER_CACHE.put(handlerName, appenderClass); - } - - if (appenderClass == null) { - throw new IOException("Unknown Storage Type: " + meta.getStoreType()); - } - - appender = newAppenderInstance(appenderClass, conf, meta, schema, path); - - return appender; - } - - public TableMeta getTableMeta(Path tablePath) throws IOException { - TableMeta meta; - - FileSystem fs = tablePath.getFileSystem(conf); - Path tableMetaPath = new Path(tablePath, ".meta"); - if (!fs.exists(tableMetaPath)) { - throw new FileNotFoundException(".meta file not found in " + tablePath.toString()); - } - - FSDataInputStream tableMetaIn = fs.open(tableMetaPath); - - CatalogProtos.TableProto tableProto = (CatalogProtos.TableProto) org.apache.tajo.util.FileUtil.loadProto(tableMetaIn, - CatalogProtos.TableProto.getDefaultInstance()); - meta = new TableMeta(tableProto); - - return meta; + return getAppender(null, null, meta, schema, filePath); } public FileFragment[] split(String tableName) throws IOException { @@ -278,6 +274,23 @@ public class FileStorageManager extends StorageManager { } }; + public Path getAppenderFilePath(QueryUnitAttemptId taskAttemptId, Path workDir) { + if (taskAttemptId == null) { + // For testcase + return workDir; + } + // The final result of a task will be written in a file named part-ss-nnnnnnn, + // where ss is the subquery id associated with this task, and nnnnnn is the task id. + Path outFilePath = StorageUtil.concatPath(workDir, TajoConstants.RESULT_DIR_NAME, + OUTPUT_FILE_PREFIX + + OUTPUT_FILE_FORMAT_SUBQUERY.get().format(taskAttemptId.getQueryUnitId().getExecutionBlockId().getId()) + "-" + + OUTPUT_FILE_FORMAT_TASK.get().format(taskAttemptId.getQueryUnitId().getId()) + "-" + + OUTPUT_FILE_FORMAT_SEQ.get().format(0)); + LOG.info("Output File Path: " + outFilePath); + + return outFilePath; + } + /** * Proxy PathFilter that accepts a path only if all filters given in the * constructor do. Used by the listPaths() to apply the built-in @@ -639,12 +652,12 @@ public class FileStorageManager extends StorageManager { } @Override - public List<Fragment> getSplits(String tableName, TableDesc table) throws IOException { + public List<Fragment> getSplits(String tableName, TableDesc table, ScanNode scanNode) throws IOException { return getSplits(tableName, table.getMeta(), table.getSchema(), table.getPath()); } @Override - public void createTable(TableDesc tableDesc) throws IOException { + public void createTable(TableDesc tableDesc, boolean ifNotExists) throws IOException { if (!tableDesc.isExternal()) { String [] splitted = CatalogUtil.splitFQTableName(tableDesc.getName()); String databaseName = splitted[0]; @@ -699,4 +712,120 @@ public class FileStorageManager extends StorageManager { } } + @Override + public List<Fragment> getNonForwardSplit(TableDesc tableDesc, int currentPage, int numFragments) throws IOException { + // Listing table data file which is not empty. + // If the table is a partitioned table, return file list which has same partition key. + FileSystem fs = tableDesc.getPath().getFileSystem(conf); + + List<FileStatus> nonZeroLengthFiles = new ArrayList<FileStatus>(); + if (fs.exists(tableDesc.getPath())) { + getNonZeroLengthDataFiles(fs, tableDesc.getPath(), nonZeroLengthFiles, currentPage, numFragments, + new AtomicInteger(0)); + } + + List<Fragment> fragments = new ArrayList<Fragment>(); + + //In the case of partitioned table, return same partition key data files. + int numPartitionColumns = 0; + if (tableDesc.hasPartition()) { + numPartitionColumns = tableDesc.getPartitionMethod().getExpressionSchema().getColumns().size(); + } + String[] previousPartitionPathNames = null; + for (FileStatus eachFile: nonZeroLengthFiles) { + FileFragment fileFragment = new FileFragment(tableDesc.getName(), eachFile.getPath(), 0, eachFile.getLen(), null); + + if (numPartitionColumns > 0) { + // finding partition key; + Path filePath = fileFragment.getPath(); + Path parentPath = filePath; + String[] parentPathNames = new String[numPartitionColumns]; + for (int i = 0; i < numPartitionColumns; i++) { + parentPath = parentPath.getParent(); + parentPathNames[numPartitionColumns - i - 1] = parentPath.getName(); + } + + // If current partitionKey == previousPartitionKey, add to result. + if (previousPartitionPathNames == null) { + fragments.add(fileFragment); + } else if (previousPartitionPathNames != null && Arrays.equals(previousPartitionPathNames, parentPathNames)) { + fragments.add(fileFragment); + } else { + break; + } + previousPartitionPathNames = parentPathNames; + } else { + fragments.add(fileFragment); + } + } + + return fragments; + } + + private void getNonZeroLengthDataFiles(FileSystem fs, Path path, List<FileStatus> result, + int startFileIndex, int numResultFiles, + AtomicInteger currentFileIndex) throws IOException { + if (fs.isDirectory(path)) { + FileStatus[] files = fs.listStatus(path, FileStorageManager.hiddenFileFilter); + if (files != null && files.length > 0) { + for (FileStatus eachFile : files) { + if (result.size() >= numResultFiles) { + return; + } + if (eachFile.isDirectory()) { + getNonZeroLengthDataFiles(fs, eachFile.getPath(), result, startFileIndex, numResultFiles, + currentFileIndex); + } else if (eachFile.isFile() && eachFile.getLen() > 0) { + if (currentFileIndex.get() >= startFileIndex) { + result.add(eachFile); + } + currentFileIndex.incrementAndGet(); + } + } + } + } else { + FileStatus fileStatus = fs.getFileStatus(path); + if (fileStatus != null && fileStatus.getLen() > 0) { + if (currentFileIndex.get() >= startFileIndex) { + result.add(fileStatus); + } + currentFileIndex.incrementAndGet(); + if (result.size() >= numResultFiles) { + return; + } + } + } + } + + @Override + public StorageProperty getStorageProperty() { + StorageProperty storageProperty = new StorageProperty(); + storageProperty.setSortedInsert(false); + if (storeType == StoreType.RAW) { + storageProperty.setSupportsInsertInto(false); + } else { + storageProperty.setSupportsInsertInto(true); + } + + return storageProperty; + } + + @Override + public void closeStorageManager() { + } + + @Override + public void beforeInsertOrCATS(LogicalNode node) throws IOException { + } + + @Override + public void rollbackOutputCommit(LogicalNode node) throws IOException { + } + + @Override + public TupleRange[] getInsertSortRanges(OverridableConf queryContext, TableDesc tableDesc, + Schema inputSchema, SortSpec[] sortSpecs, TupleRange dataRange) + throws IOException { + return null; + } } http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-storage/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java ---------------------------------------------------------------------- diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java b/tajo-storage/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java index ece31dd..33a9233 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/HashShuffleAppenderManager.java @@ -31,7 +31,6 @@ import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.conf.TajoConf.ConfVars; import org.apache.tajo.util.Pair; -import org.apache.tajo.storage.StorageManager; import java.io.IOException; import java.util.ArrayList; http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-storage/src/main/java/org/apache/tajo/storage/MergeScanner.java ---------------------------------------------------------------------- diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/MergeScanner.java b/tajo-storage/src/main/java/org/apache/tajo/storage/MergeScanner.java index 89e59d0..66b3667 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/MergeScanner.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/MergeScanner.java @@ -64,8 +64,9 @@ public class MergeScanner implements Scanner { long numBytes = 0; for (Fragment eachFileFragment: rawFragmentList) { - if (eachFileFragment.getLength() > 0) { - numBytes += eachFileFragment.getLength(); + long fragmentLength = StorageManager.getFragmentLength((TajoConf)conf, eachFileFragment); + if (fragmentLength > 0) { + numBytes += fragmentLength; fragments.add(eachFileFragment); } } @@ -121,6 +122,9 @@ public class MergeScanner implements Scanner { @Override public void reset() throws IOException { this.iterator = fragments.iterator(); + if (currentScanner != null) { + currentScanner.close(); + } this.currentScanner = getNextScanner(); } http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-storage/src/main/java/org/apache/tajo/storage/RawFile.java ---------------------------------------------------------------------- diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/RawFile.java b/tajo-storage/src/main/java/org/apache/tajo/storage/RawFile.java index de3912a..5a3f32d 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/RawFile.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/RawFile.java @@ -25,6 +25,7 @@ import org.apache.commons.logging.LogFactory; import org.apache.hadoop.conf.Configuration; import org.apache.hadoop.fs.Path; import org.apache.hadoop.io.IOUtils; +import org.apache.tajo.QueryUnitAttemptId; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableMeta; import org.apache.tajo.catalog.statistics.TableStats; @@ -82,7 +83,6 @@ public class RawFile { } catch (IllegalArgumentException iae) { throw new IOException(iae); } - fis = new FileInputStream(file); channel = fis.getChannel(); filePosition = startOffset = fragment.getStartKey(); @@ -466,8 +466,9 @@ public class RawFile { private TableStatistics stats; - public RawFileAppender(Configuration conf, Schema schema, TableMeta meta, Path path) throws IOException { - super(conf, schema, meta, path); + public RawFileAppender(Configuration conf, QueryUnitAttemptId taskAttemptId, + Schema schema, TableMeta meta, Path workDir) throws IOException { + super(conf, taskAttemptId, schema, meta, workDir); } public void init() throws IOException { http://git-wip-us.apache.org/repos/asf/tajo/blob/69373878/tajo-storage/src/main/java/org/apache/tajo/storage/RowFile.java ---------------------------------------------------------------------- diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/RowFile.java b/tajo-storage/src/main/java/org/apache/tajo/storage/RowFile.java index 640cae2..efb23eb 100644 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/RowFile.java +++ b/tajo-storage/src/main/java/org/apache/tajo/storage/RowFile.java @@ -25,6 +25,7 @@ import org.apache.hadoop.fs.FSDataInputStream; import org.apache.hadoop.fs.FSDataOutputStream; import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; +import org.apache.tajo.QueryUnitAttemptId; import org.apache.tajo.catalog.Column; import org.apache.tajo.catalog.Schema; import org.apache.tajo.catalog.TableMeta; @@ -313,9 +314,10 @@ public class RowFile { // statistics private TableStatistics stats; - public RowFileAppender(Configuration conf, final Schema schema, final TableMeta meta, final Path path) + public RowFileAppender(Configuration conf, final QueryUnitAttemptId taskAttemptId, + final Schema schema, final TableMeta meta, final Path workDir) throws IOException { - super(conf, schema, meta, path); + super(conf, taskAttemptId, schema, meta, workDir); } public void init() throws IOException {
