http://git-wip-us.apache.org/repos/asf/tajo/blob/4561711f/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoColumnKeyValue.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoColumnKeyValue.result b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoColumnKeyValue.result new file mode 100644 index 0000000..a37593b --- /dev/null +++ b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoColumnKeyValue.result @@ -0,0 +1,21 @@ +0, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-0"} +1, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-1"} +10, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-10"} +11, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-11"} +12, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-12"} +13, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-13"} +14, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-14"} +15, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-15"} +16, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-16"} +17, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-17"} +18, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-18"} +19, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-19"} +2, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-2"} +20, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-20"} +3, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-3"} +4, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-4"} +5, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-5"} +6, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-6"} +7, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-7"} +8, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-8"} +9, {"ck-0": "value-0", "ck-1": "value-1", "ck-2": "value-2"}, {"": "col3-9"}
http://git-wip-us.apache.org/repos/asf/tajo/blob/4561711f/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegion.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegion.result b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegion.result new file mode 100644 index 0000000..72013f2 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegion.result @@ -0,0 +1,100 @@ +000, value0 +001, value1 +002, value2 +003, value3 +004, value4 +005, value5 +006, value6 +007, value7 +008, value8 +009, value9 +010, value10 +011, value11 +012, value12 +013, value13 +014, value14 +015, value15 +016, value16 +017, value17 +018, value18 +019, value19 +020, value20 +021, value21 +022, value22 +023, value23 +024, value24 +025, value25 +026, value26 +027, value27 +028, value28 +029, value29 +030, value30 +031, value31 +032, value32 +033, value33 +034, value34 +035, value35 +036, value36 +037, value37 +038, value38 +039, value39 +040, value40 +041, value41 +042, value42 +043, value43 +044, value44 +045, value45 +046, value46 +047, value47 +048, value48 +049, value49 +050, value50 +051, value51 +052, value52 +053, value53 +054, value54 +055, value55 +056, value56 +057, value57 +058, value58 +059, value59 +060, value60 +061, value61 +062, value62 +063, value63 +064, value64 +065, value65 +066, value66 +067, value67 +068, value68 +069, value69 +070, value70 +071, value71 +072, value72 +073, value73 +074, value74 +075, value75 +076, value76 +077, value77 +078, value78 +079, value79 +080, value80 +081, value81 +082, value82 +083, value83 +084, value84 +085, value85 +086, value86 +087, value87 +088, value88 +089, value89 +090, value90 +091, value91 +092, value92 +093, value93 +094, value94 +095, value95 +096, value96 +097, value97 +098, value98 +099, value99 http://git-wip-us.apache.org/repos/asf/tajo/blob/4561711f/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegion2.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegion2.result b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegion2.result new file mode 100644 index 0000000..be11042 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegion2.result @@ -0,0 +1,100 @@ +0, value0 +1, value1 +10, value10 +11, value11 +12, value12 +13, value13 +14, value14 +15, value15 +16, value16 +17, value17 +18, value18 +19, value19 +2, value2 +20, value20 +21, value21 +22, value22 +23, value23 +24, value24 +25, value25 +26, value26 +27, value27 +28, value28 +29, value29 +3, value3 +30, value30 +31, value31 +32, value32 +33, value33 +34, value34 +35, value35 +36, value36 +37, value37 +38, value38 +39, value39 +4, value4 +40, value40 +41, value41 +42, value42 +43, value43 +44, value44 +45, value45 +46, value46 +47, value47 +48, value48 +49, value49 +5, value5 +50, value50 +51, value51 +52, value52 +53, value53 +54, value54 +55, value55 +56, value56 +57, value57 +58, value58 +59, value59 +6, value6 +60, value60 +61, value61 +62, value62 +63, value63 +64, value64 +65, value65 +66, value66 +67, value67 +68, value68 +69, value69 +7, value7 +70, value70 +71, value71 +72, value72 +73, value73 +74, value74 +75, value75 +76, value76 +77, value77 +78, value78 +79, value79 +8, value8 +80, value80 +81, value81 +82, value82 +83, value83 +84, value84 +85, value85 +86, value86 +87, value87 +88, value88 +89, value89 +9, value9 +90, value90 +91, value91 +92, value92 +93, value93 +94, value94 +95, value95 +96, value96 +97, value97 +98, value98 +99, value99 http://git-wip-us.apache.org/repos/asf/tajo/blob/4561711f/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegionMultiRowFields.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegionMultiRowFields.result b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegionMultiRowFields.result new file mode 100644 index 0000000..0b740c8 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegionMultiRowFields.result @@ -0,0 +1,100 @@ +000_100, value0 +001_101, value1 +002_102, value2 +003_103, value3 +004_104, value4 +005_105, value5 +006_106, value6 +007_107, value7 +008_108, value8 +009_109, value9 +010_110, value10 +011_111, value11 +012_112, value12 +013_113, value13 +014_114, value14 +015_115, value15 +016_116, value16 +017_117, value17 +018_118, value18 +019_119, value19 +020_120, value20 +021_121, value21 +022_122, value22 +023_123, value23 +024_124, value24 +025_125, value25 +026_126, value26 +027_127, value27 +028_128, value28 +029_129, value29 +030_130, value30 +031_131, value31 +032_132, value32 +033_133, value33 +034_134, value34 +035_135, value35 +036_136, value36 +037_137, value37 +038_138, value38 +039_139, value39 +040_140, value40 +041_141, value41 +042_142, value42 +043_143, value43 +044_144, value44 +045_145, value45 +046_146, value46 +047_147, value47 +048_148, value48 +049_149, value49 +050_150, value50 +051_151, value51 +052_152, value52 +053_153, value53 +054_154, value54 +055_155, value55 +056_156, value56 +057_157, value57 +058_158, value58 +059_159, value59 +060_160, value60 +061_161, value61 +062_162, value62 +063_163, value63 +064_164, value64 +065_165, value65 +066_166, value66 +067_167, value67 +068_168, value68 +069_169, value69 +070_170, value70 +071_171, value71 +072_172, value72 +073_173, value73 +074_174, value74 +075_175, value75 +076_176, value76 +077_177, value77 +078_178, value78 +079_179, value79 +080_180, value80 +081_181, value81 +082_182, value82 +083_183, value83 +084_184, value84 +085_185, value85 +086_186, value86 +087_187, value87 +088_188, value88 +089_189, value89 +090_190, value90 +091_191, value91 +092_192, value92 +093_193, value93 +094_194, value94 +095_195, value95 +096_196, value96 +097_197, value97 +098_198, value98 +099_199, value99 http://git-wip-us.apache.org/repos/asf/tajo/blob/4561711f/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegionWithSplitFile.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegionWithSplitFile.result b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegionWithSplitFile.result new file mode 100644 index 0000000..72013f2 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoMultiRegionWithSplitFile.result @@ -0,0 +1,100 @@ +000, value0 +001, value1 +002, value2 +003, value3 +004, value4 +005, value5 +006, value6 +007, value7 +008, value8 +009, value9 +010, value10 +011, value11 +012, value12 +013, value13 +014, value14 +015, value15 +016, value16 +017, value17 +018, value18 +019, value19 +020, value20 +021, value21 +022, value22 +023, value23 +024, value24 +025, value25 +026, value26 +027, value27 +028, value28 +029, value29 +030, value30 +031, value31 +032, value32 +033, value33 +034, value34 +035, value35 +036, value36 +037, value37 +038, value38 +039, value39 +040, value40 +041, value41 +042, value42 +043, value43 +044, value44 +045, value45 +046, value46 +047, value47 +048, value48 +049, value49 +050, value50 +051, value51 +052, value52 +053, value53 +054, value54 +055, value55 +056, value56 +057, value57 +058, value58 +059, value59 +060, value60 +061, value61 +062, value62 +063, value63 +064, value64 +065, value65 +066, value66 +067, value67 +068, value68 +069, value69 +070, value70 +071, value71 +072, value72 +073, value73 +074, value74 +075, value75 +076, value76 +077, value77 +078, value78 +079, value79 +080, value80 +081, value81 +082, value82 +083, value83 +084, value84 +085, value85 +086, value86 +087, value87 +088, value88 +089, value89 +090, value90 +091, value91 +092, value92 +093, value93 +094, value94 +095, value95 +096, value96 +097, value97 +098, value98 +099, value99 http://git-wip-us.apache.org/repos/asf/tajo/blob/4561711f/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoRowField.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoRowField.result b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoRowField.result new file mode 100644 index 0000000..152b91b --- /dev/null +++ b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoRowField.result @@ -0,0 +1,4 @@ +1_1, 1996-03-13, N, 7706 +2_2, 1997-01-28, N, 1191 +3_2, 1994-02-02, R, 1798 +3_3, 1993-11-09, R, 6540 http://git-wip-us.apache.org/repos/asf/tajo/blob/4561711f/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoUsingPut.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoUsingPut.result b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoUsingPut.result new file mode 100644 index 0000000..1c22960 --- /dev/null +++ b/tajo-core/src/test/resources/results/TestHBaseTable/testInsertIntoUsingPut.result @@ -0,0 +1,3 @@ +1, 1996-04-12, {"": "N"}, 7311 +2, 1997-01-28, {"": "N"}, 1191 +3, 1993-11-09, {"": "R"}, 6540 http://git-wip-us.apache.org/repos/asf/tajo/blob/4561711f/tajo-core/src/test/resources/results/TestHBaseTable/testJoin.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testJoin.result b/tajo-core/src/test/resources/results/TestHBaseTable/testJoin.result new file mode 100644 index 0000000..fe4fcae --- /dev/null +++ b/tajo-core/src/test/resources/results/TestHBaseTable/testJoin.result @@ -0,0 +1,7 @@ +rk,col1,col2,col3,l_orderkey,l_linestatus +------------------------------- +001,a-1,{"k1":"k1-1", "k2":"k2-1"},1,1,O +001,a-1,{"k1":"k1-1", "k2":"k2-1"},1,1,O +002,a-2,{"k1":"k1-2", "k2":"k2-2"},2,2,O +003,a-3,{"k1":"k1-3", "k2":"k2-3"},3,3,F +003,a-3,{"k1":"k1-3", "k2":"k2-3"},3,3,F \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/4561711f/tajo-core/src/test/resources/results/TestHBaseTable/testNonForwardQuery.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testNonForwardQuery.result b/tajo-core/src/test/resources/results/TestHBaseTable/testNonForwardQuery.result new file mode 100644 index 0000000..4f5fd8f --- /dev/null +++ b/tajo-core/src/test/resources/results/TestHBaseTable/testNonForwardQuery.result @@ -0,0 +1,102 @@ +rk,col1,col2,col3 +------------------------------- +000,a-0,{"k1":"k1-0", "k2":"k2-0"},0 +001,a-1,{"k1":"k1-1", "k2":"k2-1"},1 +002,a-2,{"k1":"k1-2", "k2":"k2-2"},2 +003,a-3,{"k1":"k1-3", "k2":"k2-3"},3 +004,a-4,{"k1":"k1-4", "k2":"k2-4"},4 +005,a-5,{"k1":"k1-5", "k2":"k2-5"},5 +006,a-6,{"k1":"k1-6", "k2":"k2-6"},6 +007,a-7,{"k1":"k1-7", "k2":"k2-7"},7 +008,a-8,{"k1":"k1-8", "k2":"k2-8"},8 +009,a-9,{"k1":"k1-9", "k2":"k2-9"},9 +010,a-10,{"k1":"k1-10", "k2":"k2-10"},10 +011,a-11,{"k1":"k1-11", "k2":"k2-11"},11 +012,a-12,{"k1":"k1-12", "k2":"k2-12"},12 +013,a-13,{"k1":"k1-13", "k2":"k2-13"},13 +014,a-14,{"k1":"k1-14", "k2":"k2-14"},14 +015,a-15,{"k1":"k1-15", "k2":"k2-15"},15 +016,a-16,{"k1":"k1-16", "k2":"k2-16"},16 +017,a-17,{"k1":"k1-17", "k2":"k2-17"},17 +018,a-18,{"k1":"k1-18", "k2":"k2-18"},18 +019,a-19,{"k1":"k1-19", "k2":"k2-19"},19 +020,a-20,{"k1":"k1-20", "k2":"k2-20"},20 +021,a-21,{"k1":"k1-21", "k2":"k2-21"},21 +022,a-22,{"k1":"k1-22", "k2":"k2-22"},22 +023,a-23,{"k1":"k1-23", "k2":"k2-23"},23 +024,a-24,{"k1":"k1-24", "k2":"k2-24"},24 +025,a-25,{"k1":"k1-25", "k2":"k2-25"},25 +026,a-26,{"k1":"k1-26", "k2":"k2-26"},26 +027,a-27,{"k1":"k1-27", "k2":"k2-27"},27 +028,a-28,{"k1":"k1-28", "k2":"k2-28"},28 +029,a-29,{"k1":"k1-29", "k2":"k2-29"},29 +030,a-30,{"k1":"k1-30", "k2":"k2-30"},30 +031,a-31,{"k1":"k1-31", "k2":"k2-31"},31 +032,a-32,{"k1":"k1-32", "k2":"k2-32"},32 +033,a-33,{"k1":"k1-33", "k2":"k2-33"},33 +034,a-34,{"k1":"k1-34", "k2":"k2-34"},34 +035,a-35,{"k1":"k1-35", "k2":"k2-35"},35 +036,a-36,{"k1":"k1-36", "k2":"k2-36"},36 +037,a-37,{"k1":"k1-37", "k2":"k2-37"},37 +038,a-38,{"k1":"k1-38", "k2":"k2-38"},38 +039,a-39,{"k1":"k1-39", "k2":"k2-39"},39 +040,a-40,{"k1":"k1-40", "k2":"k2-40"},40 +041,a-41,{"k1":"k1-41", "k2":"k2-41"},41 +042,a-42,{"k1":"k1-42", "k2":"k2-42"},42 +043,a-43,{"k1":"k1-43", "k2":"k2-43"},43 +044,a-44,{"k1":"k1-44", "k2":"k2-44"},44 +045,a-45,{"k1":"k1-45", "k2":"k2-45"},45 +046,a-46,{"k1":"k1-46", "k2":"k2-46"},46 +047,a-47,{"k1":"k1-47", "k2":"k2-47"},47 +048,a-48,{"k1":"k1-48", "k2":"k2-48"},48 +049,a-49,{"k1":"k1-49", "k2":"k2-49"},49 +050,a-50,{"k1":"k1-50", "k2":"k2-50"},50 +051,a-51,{"k1":"k1-51", "k2":"k2-51"},51 +052,a-52,{"k1":"k1-52", "k2":"k2-52"},52 +053,a-53,{"k1":"k1-53", "k2":"k2-53"},53 +054,a-54,{"k1":"k1-54", "k2":"k2-54"},54 +055,a-55,{"k1":"k1-55", "k2":"k2-55"},55 +056,a-56,{"k1":"k1-56", "k2":"k2-56"},56 +057,a-57,{"k1":"k1-57", "k2":"k2-57"},57 +058,a-58,{"k1":"k1-58", "k2":"k2-58"},58 +059,a-59,{"k1":"k1-59", "k2":"k2-59"},59 +060,a-60,{"k1":"k1-60", "k2":"k2-60"},60 +061,a-61,{"k1":"k1-61", "k2":"k2-61"},61 +062,a-62,{"k1":"k1-62", "k2":"k2-62"},62 +063,a-63,{"k1":"k1-63", "k2":"k2-63"},63 +064,a-64,{"k1":"k1-64", "k2":"k2-64"},64 +065,a-65,{"k1":"k1-65", "k2":"k2-65"},65 +066,a-66,{"k1":"k1-66", "k2":"k2-66"},66 +067,a-67,{"k1":"k1-67", "k2":"k2-67"},67 +068,a-68,{"k1":"k1-68", "k2":"k2-68"},68 +069,a-69,{"k1":"k1-69", "k2":"k2-69"},69 +070,a-70,{"k1":"k1-70", "k2":"k2-70"},70 +071,a-71,{"k1":"k1-71", "k2":"k2-71"},71 +072,a-72,{"k1":"k1-72", "k2":"k2-72"},72 +073,a-73,{"k1":"k1-73", "k2":"k2-73"},73 +074,a-74,{"k1":"k1-74", "k2":"k2-74"},74 +075,a-75,{"k1":"k1-75", "k2":"k2-75"},75 +076,a-76,{"k1":"k1-76", "k2":"k2-76"},76 +077,a-77,{"k1":"k1-77", "k2":"k2-77"},77 +078,a-78,{"k1":"k1-78", "k2":"k2-78"},78 +079,a-79,{"k1":"k1-79", "k2":"k2-79"},79 +080,a-80,{"k1":"k1-80", "k2":"k2-80"},80 +081,a-81,{"k1":"k1-81", "k2":"k2-81"},81 +082,a-82,{"k1":"k1-82", "k2":"k2-82"},82 +083,a-83,{"k1":"k1-83", "k2":"k2-83"},83 +084,a-84,{"k1":"k1-84", "k2":"k2-84"},84 +085,a-85,{"k1":"k1-85", "k2":"k2-85"},85 +086,a-86,{"k1":"k1-86", "k2":"k2-86"},86 +087,a-87,{"k1":"k1-87", "k2":"k2-87"},87 +088,a-88,{"k1":"k1-88", "k2":"k2-88"},88 +089,a-89,{"k1":"k1-89", "k2":"k2-89"},89 +090,a-90,{"k1":"k1-90", "k2":"k2-90"},90 +091,a-91,{"k1":"k1-91", "k2":"k2-91"},91 +092,a-92,{"k1":"k1-92", "k2":"k2-92"},92 +093,a-93,{"k1":"k1-93", "k2":"k2-93"},93 +094,a-94,{"k1":"k1-94", "k2":"k2-94"},94 +095,a-95,{"k1":"k1-95", "k2":"k2-95"},95 +096,a-96,{"k1":"k1-96", "k2":"k2-96"},96 +097,a-97,{"k1":"k1-97", "k2":"k2-97"},97 +098,a-98,{"k1":"k1-98", "k2":"k2-98"},98 +099,a-99,{"k1":"k1-99", "k2":"k2-99"},99 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/4561711f/tajo-core/src/test/resources/results/TestHBaseTable/testRowFieldSelectQuery.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testRowFieldSelectQuery.result b/tajo-core/src/test/resources/results/TestHBaseTable/testRowFieldSelectQuery.result new file mode 100644 index 0000000..34369df --- /dev/null +++ b/tajo-core/src/test/resources/results/TestHBaseTable/testRowFieldSelectQuery.result @@ -0,0 +1,88 @@ +rk1,rk2,col3 +------------------------------- +field1-21,field2-21,a-21 +field1-22,field2-22,a-22 +field1-23,field2-23,a-23 +field1-24,field2-24,a-24 +field1-25,field2-25,a-25 +field1-26,field2-26,a-26 +field1-27,field2-27,a-27 +field1-28,field2-28,a-28 +field1-29,field2-29,a-29 +field1-30,field2-30,a-30 +field1-31,field2-31,a-31 +field1-32,field2-32,a-32 +field1-33,field2-33,a-33 +field1-34,field2-34,a-34 +field1-35,field2-35,a-35 +field1-36,field2-36,a-36 +field1-37,field2-37,a-37 +field1-38,field2-38,a-38 +field1-39,field2-39,a-39 +field1-3,field2-3,a-3 +field1-40,field2-40,a-40 +field1-41,field2-41,a-41 +field1-42,field2-42,a-42 +field1-43,field2-43,a-43 +field1-44,field2-44,a-44 +field1-45,field2-45,a-45 +field1-46,field2-46,a-46 +field1-47,field2-47,a-47 +field1-48,field2-48,a-48 +field1-49,field2-49,a-49 +field1-4,field2-4,a-4 +field1-50,field2-50,a-50 +field1-51,field2-51,a-51 +field1-52,field2-52,a-52 +field1-53,field2-53,a-53 +field1-54,field2-54,a-54 +field1-55,field2-55,a-55 +field1-56,field2-56,a-56 +field1-57,field2-57,a-57 +field1-58,field2-58,a-58 +field1-59,field2-59,a-59 +field1-5,field2-5,a-5 +field1-60,field2-60,a-60 +field1-61,field2-61,a-61 +field1-62,field2-62,a-62 +field1-63,field2-63,a-63 +field1-64,field2-64,a-64 +field1-65,field2-65,a-65 +field1-66,field2-66,a-66 +field1-67,field2-67,a-67 +field1-68,field2-68,a-68 +field1-69,field2-69,a-69 +field1-6,field2-6,a-6 +field1-70,field2-70,a-70 +field1-71,field2-71,a-71 +field1-72,field2-72,a-72 +field1-73,field2-73,a-73 +field1-74,field2-74,a-74 +field1-75,field2-75,a-75 +field1-76,field2-76,a-76 +field1-77,field2-77,a-77 +field1-78,field2-78,a-78 +field1-79,field2-79,a-79 +field1-7,field2-7,a-7 +field1-80,field2-80,a-80 +field1-81,field2-81,a-81 +field1-82,field2-82,a-82 +field1-83,field2-83,a-83 +field1-84,field2-84,a-84 +field1-85,field2-85,a-85 +field1-86,field2-86,a-86 +field1-87,field2-87,a-87 +field1-88,field2-88,a-88 +field1-89,field2-89,a-89 +field1-8,field2-8,a-8 +field1-90,field2-90,a-90 +field1-91,field2-91,a-91 +field1-92,field2-92,a-92 +field1-93,field2-93,a-93 +field1-94,field2-94,a-94 +field1-95,field2-95,a-95 +field1-96,field2-96,a-96 +field1-97,field2-97,a-97 +field1-98,field2-98,a-98 +field1-99,field2-99,a-99 +field1-9,field2-9,a-9 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/4561711f/tajo-core/src/test/resources/results/TestHBaseTable/testSimpleSelectQuery.result ---------------------------------------------------------------------- diff --git a/tajo-core/src/test/resources/results/TestHBaseTable/testSimpleSelectQuery.result b/tajo-core/src/test/resources/results/TestHBaseTable/testSimpleSelectQuery.result new file mode 100644 index 0000000..ef2f27a --- /dev/null +++ b/tajo-core/src/test/resources/results/TestHBaseTable/testSimpleSelectQuery.result @@ -0,0 +1,88 @@ +rk,col1,col2,col3 +------------------------------- +21,a-21,{"k1":"k1-21", "k2":"k2-21"},b-21 +22,a-22,{"k1":"k1-22", "k2":"k2-22"},b-22 +23,a-23,{"k1":"k1-23", "k2":"k2-23"},b-23 +24,a-24,{"k1":"k1-24", "k2":"k2-24"},b-24 +25,a-25,{"k1":"k1-25", "k2":"k2-25"},b-25 +26,a-26,{"k1":"k1-26", "k2":"k2-26"},b-26 +27,a-27,{"k1":"k1-27", "k2":"k2-27"},b-27 +28,a-28,{"k1":"k1-28", "k2":"k2-28"},b-28 +29,a-29,{"k1":"k1-29", "k2":"k2-29"},b-29 +3,a-3,{"k1":"k1-3", "k2":"k2-3"},b-3 +30,a-30,{"k1":"k1-30", "k2":"k2-30"},b-30 +31,a-31,{"k1":"k1-31", "k2":"k2-31"},b-31 +32,a-32,{"k1":"k1-32", "k2":"k2-32"},b-32 +33,a-33,{"k1":"k1-33", "k2":"k2-33"},b-33 +34,a-34,{"k1":"k1-34", "k2":"k2-34"},b-34 +35,a-35,{"k1":"k1-35", "k2":"k2-35"},b-35 +36,a-36,{"k1":"k1-36", "k2":"k2-36"},b-36 +37,a-37,{"k1":"k1-37", "k2":"k2-37"},b-37 +38,a-38,{"k1":"k1-38", "k2":"k2-38"},b-38 +39,a-39,{"k1":"k1-39", "k2":"k2-39"},b-39 +4,a-4,{"k1":"k1-4", "k2":"k2-4"},b-4 +40,a-40,{"k1":"k1-40", "k2":"k2-40"},b-40 +41,a-41,{"k1":"k1-41", "k2":"k2-41"},b-41 +42,a-42,{"k1":"k1-42", "k2":"k2-42"},b-42 +43,a-43,{"k1":"k1-43", "k2":"k2-43"},b-43 +44,a-44,{"k1":"k1-44", "k2":"k2-44"},b-44 +45,a-45,{"k1":"k1-45", "k2":"k2-45"},b-45 +46,a-46,{"k1":"k1-46", "k2":"k2-46"},b-46 +47,a-47,{"k1":"k1-47", "k2":"k2-47"},b-47 +48,a-48,{"k1":"k1-48", "k2":"k2-48"},b-48 +49,a-49,{"k1":"k1-49", "k2":"k2-49"},b-49 +5,a-5,{"k1":"k1-5", "k2":"k2-5"},b-5 +50,a-50,{"k1":"k1-50", "k2":"k2-50"},b-50 +51,a-51,{"k1":"k1-51", "k2":"k2-51"},b-51 +52,a-52,{"k1":"k1-52", "k2":"k2-52"},b-52 +53,a-53,{"k1":"k1-53", "k2":"k2-53"},b-53 +54,a-54,{"k1":"k1-54", "k2":"k2-54"},b-54 +55,a-55,{"k1":"k1-55", "k2":"k2-55"},b-55 +56,a-56,{"k1":"k1-56", "k2":"k2-56"},b-56 +57,a-57,{"k1":"k1-57", "k2":"k2-57"},b-57 +58,a-58,{"k1":"k1-58", "k2":"k2-58"},b-58 +59,a-59,{"k1":"k1-59", "k2":"k2-59"},b-59 +6,a-6,{"k1":"k1-6", "k2":"k2-6"},b-6 +60,a-60,{"k1":"k1-60", "k2":"k2-60"},b-60 +61,a-61,{"k1":"k1-61", "k2":"k2-61"},b-61 +62,a-62,{"k1":"k1-62", "k2":"k2-62"},b-62 +63,a-63,{"k1":"k1-63", "k2":"k2-63"},b-63 +64,a-64,{"k1":"k1-64", "k2":"k2-64"},b-64 +65,a-65,{"k1":"k1-65", "k2":"k2-65"},b-65 +66,a-66,{"k1":"k1-66", "k2":"k2-66"},b-66 +67,a-67,{"k1":"k1-67", "k2":"k2-67"},b-67 +68,a-68,{"k1":"k1-68", "k2":"k2-68"},b-68 +69,a-69,{"k1":"k1-69", "k2":"k2-69"},b-69 +7,a-7,{"k1":"k1-7", "k2":"k2-7"},b-7 +70,a-70,{"k1":"k1-70", "k2":"k2-70"},b-70 +71,a-71,{"k1":"k1-71", "k2":"k2-71"},b-71 +72,a-72,{"k1":"k1-72", "k2":"k2-72"},b-72 +73,a-73,{"k1":"k1-73", "k2":"k2-73"},b-73 +74,a-74,{"k1":"k1-74", "k2":"k2-74"},b-74 +75,a-75,{"k1":"k1-75", "k2":"k2-75"},b-75 +76,a-76,{"k1":"k1-76", "k2":"k2-76"},b-76 +77,a-77,{"k1":"k1-77", "k2":"k2-77"},b-77 +78,a-78,{"k1":"k1-78", "k2":"k2-78"},b-78 +79,a-79,{"k1":"k1-79", "k2":"k2-79"},b-79 +8,a-8,{"k1":"k1-8", "k2":"k2-8"},b-8 +80,a-80,{"k1":"k1-80", "k2":"k2-80"},b-80 +81,a-81,{"k1":"k1-81", "k2":"k2-81"},b-81 +82,a-82,{"k1":"k1-82", "k2":"k2-82"},b-82 +83,a-83,{"k1":"k1-83", "k2":"k2-83"},b-83 +84,a-84,{"k1":"k1-84", "k2":"k2-84"},b-84 +85,a-85,{"k1":"k1-85", "k2":"k2-85"},b-85 +86,a-86,{"k1":"k1-86", "k2":"k2-86"},b-86 +87,a-87,{"k1":"k1-87", "k2":"k2-87"},b-87 +88,a-88,{"k1":"k1-88", "k2":"k2-88"},b-88 +89,a-89,{"k1":"k1-89", "k2":"k2-89"},b-89 +9,a-9,{"k1":"k1-9", "k2":"k2-9"},b-9 +90,a-90,{"k1":"k1-90", "k2":"k2-90"},b-90 +91,a-91,{"k1":"k1-91", "k2":"k2-91"},b-91 +92,a-92,{"k1":"k1-92", "k2":"k2-92"},b-92 +93,a-93,{"k1":"k1-93", "k2":"k2-93"},b-93 +94,a-94,{"k1":"k1-94", "k2":"k2-94"},b-94 +95,a-95,{"k1":"k1-95", "k2":"k2-95"},b-95 +96,a-96,{"k1":"k1-96", "k2":"k2-96"},b-96 +97,a-97,{"k1":"k1-97", "k2":"k2-97"},b-97 +98,a-98,{"k1":"k1-98", "k2":"k2-98"},b-98 +99,a-99,{"k1":"k1-99", "k2":"k2-99"},b-99 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/4561711f/tajo-dist/pom.xml ---------------------------------------------------------------------- diff --git a/tajo-dist/pom.xml b/tajo-dist/pom.xml index 40b6c56..c69c7d9 100644 --- a/tajo-dist/pom.xml +++ b/tajo-dist/pom.xml @@ -75,7 +75,12 @@ </dependency> <dependency> <groupId>org.apache.tajo</groupId> - <artifactId>tajo-storage</artifactId> + <artifactId>tajo-storage-common</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.tajo</groupId> + <artifactId>tajo-storage-hdfs</artifactId> <scope>provided</scope> </dependency> </dependencies> http://git-wip-us.apache.org/repos/asf/tajo/blob/4561711f/tajo-dist/src/main/bin/tajo ---------------------------------------------------------------------- diff --git a/tajo-dist/src/main/bin/tajo b/tajo-dist/src/main/bin/tajo index 6e054f3..3df5397 100755 --- a/tajo-dist/src/main/bin/tajo +++ b/tajo-dist/src/main/bin/tajo @@ -226,6 +226,21 @@ HDFS_LIBRARY_PATH="${HADOOP_HOME}/lib/native/" ############################################################################## ############################################################################## +# Set HBase CLASSPATH +############################################################################## +if [ "$HBASE_HOME" != "" ]; then + for f in ${HBASE_HOME}/lib/hbase-*.jar; do + CLASSPATH=${CLASSPATH}:$f; + done + for f in ${HBASE_HOME}/lib/htrace-*.jar; do + CLASSPATH=${CLASSPATH}:$f; + done +fi +############################################################################## +# HBase Configuration End +############################################################################## + +############################################################################## # Find and Set Hive CLASSPATH ############################################################################## http://git-wip-us.apache.org/repos/asf/tajo/blob/4561711f/tajo-dist/src/main/conf/tajo-env.sh ---------------------------------------------------------------------- diff --git a/tajo-dist/src/main/conf/tajo-env.sh b/tajo-dist/src/main/conf/tajo-env.sh index 92d00bd..064a82f 100755 --- a/tajo-dist/src/main/conf/tajo-env.sh +++ b/tajo-dist/src/main/conf/tajo-env.sh @@ -24,6 +24,9 @@ # Hadoop home. Required # export HADOOP_HOME= +# HBase home. optional +# export HBASE_HOME= + # The java implementation to use. Required. # export JAVA_HOME=/usr/java/default http://git-wip-us.apache.org/repos/asf/tajo/blob/4561711f/tajo-jdbc/pom.xml ---------------------------------------------------------------------- diff --git a/tajo-jdbc/pom.xml b/tajo-jdbc/pom.xml index 20cdf16..1c3c410 100644 --- a/tajo-jdbc/pom.xml +++ b/tajo-jdbc/pom.xml @@ -102,7 +102,11 @@ </dependency> <dependency> <groupId>org.apache.tajo</groupId> - <artifactId>tajo-storage</artifactId> + <artifactId>tajo-storage-common</artifactId> + </dependency> + <dependency> + <groupId>org.apache.tajo</groupId> + <artifactId>tajo-storage-hdfs</artifactId> </dependency> <dependency> <groupId>org.apache.tajo</groupId> http://git-wip-us.apache.org/repos/asf/tajo/blob/4561711f/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java index bcc3bce..750e64e 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/LogicalOptimizer.java @@ -44,6 +44,7 @@ import org.apache.tajo.plan.util.PlannerUtil; import org.apache.tajo.plan.visitor.BasicLogicalPlanVisitor; import java.util.LinkedHashSet; +import java.util.List; import java.util.Set; import java.util.Stack; @@ -86,6 +87,12 @@ public class LogicalOptimizer { } } + public void addRuleAfterToJoinOpt(RewriteRule rewriteRule) { + if (rewriteRule != null) { + rulesAfterToJoinOpt.addRewriteRule(rewriteRule); + } + } + public LogicalNode optimize(LogicalPlan plan) throws PlanningException { return optimize(null, plan); } http://git-wip-us.apache.org/repos/asf/tajo/blob/4561711f/tajo-plan/src/main/java/org/apache/tajo/plan/logical/InsertNode.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/InsertNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/InsertNode.java index df55b63..d1d8582 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/InsertNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/InsertNode.java @@ -46,7 +46,9 @@ public class InsertNode extends StoreTableNode implements Cloneable { } else { tableSchema = desc.getSchema(); } - setPath(new Path(desc.getPath())); + if (desc.getPath() != null) { + setPath(new Path(desc.getPath())); + } setOptions(desc.getMeta().getOptions()); setStorageType(desc.getMeta().getStoreType()); http://git-wip-us.apache.org/repos/asf/tajo/blob/4561711f/tajo-plan/src/main/java/org/apache/tajo/plan/logical/SortNode.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/SortNode.java b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/SortNode.java index a697f9f..2e5e159 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/logical/SortNode.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/logical/SortNode.java @@ -25,10 +25,16 @@ import org.apache.tajo.plan.PlanString; import org.apache.tajo.util.TUtil; public final class SortNode extends UnaryNode implements Cloneable { + public static enum SortPurpose { + NORMAL, + STORAGE_SPECIFIED + } @Expose private SortSpec [] sortKeys; + @Expose private SortPurpose sortPurpose; public SortNode(int pid) { super(pid, NodeType.SORT); + sortPurpose = SortPurpose.NORMAL; } public void setSortSpecs(SortSpec[] sortSpecs) { @@ -65,8 +71,7 @@ public final class SortNode extends UnaryNode implements Cloneable { PlanString planStr = new PlanString(this); StringBuilder sb = new StringBuilder("Sort Keys: "); for (int i = 0; i < sortKeys.length; i++) { - sb.append(sortKeys[i].getSortKey().getSimpleName()).append(" ") - .append(sortKeys[i].isAscending() ? "asc" : "desc"); + sb.append(sortKeys[i].toString()); if( i < sortKeys.length - 1) { sb.append(","); } @@ -78,8 +83,7 @@ public final class SortNode extends UnaryNode implements Cloneable { public String toString() { StringBuilder sb = new StringBuilder("Sort [key= "); for (int i = 0; i < sortKeys.length; i++) { - sb.append(sortKeys[i].getSortKey().getQualifiedName()).append(" ") - .append(sortKeys[i].isAscending() ? "asc" : "desc"); + sb.append(sortKeys[i].toString()); if(i < sortKeys.length - 1) { sb.append(","); } @@ -91,4 +95,12 @@ public final class SortNode extends UnaryNode implements Cloneable { return sb.toString()+"\n" + getChild().toString(); } + + public SortPurpose getSortPurpose() { + return sortPurpose; + } + + public void setSortPurpose(SortPurpose sortPurpose) { + this.sortPurpose = sortPurpose; + } } http://git-wip-us.apache.org/repos/asf/tajo/blob/4561711f/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java index a7d0ce2..ea58437 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/rewrite/rules/PartitionedTableRewriter.java @@ -306,7 +306,7 @@ public class PartitionedTableRewriter implements RewriteRule { boolean sameVariable = EvalTreeUtil.findUniqueColumns(orEval.getLeftExpr()) - .equals(EvalTreeUtil.findUniqueColumns(orEval.getRightExpr())); + .equals(EvalTreeUtil.findUniqueColumns(orEval.getRightExpr())); return indexable && sameVariable; } else { http://git-wip-us.apache.org/repos/asf/tajo/blob/4561711f/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java ---------------------------------------------------------------------- diff --git a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java index abcef7a..4672577 100644 --- a/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java +++ b/tajo-plan/src/main/java/org/apache/tajo/plan/util/PlannerUtil.java @@ -27,6 +27,7 @@ import org.apache.tajo.algebra.*; import org.apache.tajo.annotation.Nullable; import org.apache.tajo.catalog.*; import org.apache.tajo.catalog.proto.CatalogProtos; +import org.apache.tajo.catalog.proto.CatalogProtos.StoreType; import org.apache.tajo.common.TajoDataTypes.DataType; import org.apache.tajo.conf.TajoConf; import org.apache.tajo.plan.*; @@ -39,6 +40,7 @@ import org.apache.tajo.storage.StorageConstants; import org.apache.tajo.util.KeyValueSet; import org.apache.tajo.util.TUtil; +import java.io.IOException; import java.util.*; import static org.apache.tajo.catalog.proto.CatalogProtos.StoreType.CSV; @@ -793,7 +795,7 @@ public class PlannerUtil { } public static void applySessionToTableProperties(OverridableConf sessionVars, - CatalogProtos.StoreType storeType, + StoreType storeType, KeyValueSet tableProperties) { if (storeType == CSV || storeType == TEXTFILE) { if (sessionVars.containsKey(SessionVars.NULL_CHAR)) { @@ -819,4 +821,109 @@ public class PlannerUtil { meta.putOption(StorageConstants.TIMEZONE, systemConf.get(SessionVars.TIMEZONE)); } } + + public static boolean isFileStorageType(String storageType) { + if (storageType.equalsIgnoreCase("hbase")) { + return false; + } else { + return true; + } + } + + public static boolean isFileStorageType(StoreType storageType) { + if (storageType== StoreType.HBASE) { + return false; + } else { + return true; + } + } + + public static StoreType getStoreType(LogicalPlan plan) { + LogicalRootNode rootNode = plan.getRootBlock().getRoot(); + NodeType nodeType = rootNode.getChild().getType(); + if (nodeType == NodeType.CREATE_TABLE) { + return ((CreateTableNode)rootNode.getChild()).getStorageType(); + } else if (nodeType == NodeType.INSERT) { + return ((InsertNode)rootNode.getChild()).getStorageType(); + } else { + return null; + } + } + + public static String getStoreTableName(LogicalPlan plan) { + LogicalRootNode rootNode = plan.getRootBlock().getRoot(); + NodeType nodeType = rootNode.getChild().getType(); + if (nodeType == NodeType.CREATE_TABLE) { + return ((CreateTableNode)rootNode.getChild()).getTableName(); + } else if (nodeType == NodeType.INSERT) { + return ((InsertNode)rootNode.getChild()).getTableName(); + } else { + return null; + } + } + + public static TableDesc getTableDesc(CatalogService catalog, LogicalNode node) throws IOException { + if (node.getType() == NodeType.CREATE_TABLE) { + return createTableDesc((CreateTableNode)node); + } + String tableName = null; + InsertNode insertNode = null; + if (node.getType() == NodeType.INSERT) { + insertNode = (InsertNode)node; + tableName = insertNode.getTableName(); + } else { + return null; + } + + if (tableName != null) { + String[] tableTokens = tableName.split("\\."); + if (tableTokens.length >= 2) { + if (catalog.existsTable(tableTokens[0], tableTokens[1])) { + return catalog.getTableDesc(tableTokens[0], tableTokens[1]); + } + } + } else { + if (insertNode.getPath() != null) { + //insert ... location + return createTableDesc(insertNode); + } + } + return null; + } + + private static TableDesc createTableDesc(CreateTableNode createTableNode) { + TableMeta meta = new TableMeta(createTableNode.getStorageType(), createTableNode.getOptions()); + + TableDesc tableDescTobeCreated = + new TableDesc( + createTableNode.getTableName(), + createTableNode.getTableSchema(), + meta, + createTableNode.getPath() != null ? createTableNode.getPath().toUri() : null); + + tableDescTobeCreated.setExternal(createTableNode.isExternal()); + + if (createTableNode.hasPartition()) { + tableDescTobeCreated.setPartitionMethod(createTableNode.getPartitionMethod()); + } + + return tableDescTobeCreated; + } + + private static TableDesc createTableDesc(InsertNode insertNode) { + TableMeta meta = new TableMeta(insertNode.getStorageType(), insertNode.getOptions()); + + TableDesc tableDescTobeCreated = + new TableDesc( + insertNode.getTableName(), + insertNode.getTableSchema(), + meta, + insertNode.getPath() != null ? insertNode.getPath().toUri() : null); + + if (insertNode.hasPartition()) { + tableDescTobeCreated.setPartitionMethod(insertNode.getPartitionMethod()); + } + + return tableDescTobeCreated; + } } http://git-wip-us.apache.org/repos/asf/tajo/blob/4561711f/tajo-project/pom.xml ---------------------------------------------------------------------- diff --git a/tajo-project/pom.xml b/tajo-project/pom.xml index 63b0aec..21df863 100644 --- a/tajo-project/pom.xml +++ b/tajo-project/pom.xml @@ -36,6 +36,7 @@ <hadoop.version>2.5.1</hadoop.version> <protobuf.version>2.5.0</protobuf.version> <tajo.version>0.9.1-SNAPSHOT</tajo.version> + <hbase.version>0.98.7-hadoop2</hbase.version> <tajo.root>${project.parent.relativePath}/..</tajo.root> <extra.source.path>src/main/hadoop-${hadoop.version}</extra.source.path> </properties> @@ -743,6 +744,22 @@ <groupId>org.apache.tajo</groupId> <artifactId>tajo-storage</artifactId> <version>${tajo.version}</version> + <type>pom</type> + </dependency> + <dependency> + <groupId>org.apache.tajo</groupId> + <artifactId>tajo-storage-common</artifactId> + <version>${tajo.version}</version> + </dependency> + <dependency> + <groupId>org.apache.tajo</groupId> + <artifactId>tajo-storage-hdfs</artifactId> + <version>${tajo.version}</version> + </dependency> + <dependency> + <groupId>org.apache.tajo</groupId> + <artifactId>tajo-storage-hbase</artifactId> + <version>${tajo.version}</version> </dependency> <dependency> <groupId>org.apache.tajo</groupId> http://git-wip-us.apache.org/repos/asf/tajo/blob/4561711f/tajo-pullserver/pom.xml ---------------------------------------------------------------------- diff --git a/tajo-pullserver/pom.xml b/tajo-pullserver/pom.xml index b9c6624..6fa8e5b 100644 --- a/tajo-pullserver/pom.xml +++ b/tajo-pullserver/pom.xml @@ -57,7 +57,12 @@ </dependency> <dependency> <groupId>org.apache.tajo</groupId> - <artifactId>tajo-storage</artifactId> + <artifactId>tajo-storage-common</artifactId> + <scope>provided</scope> + </dependency> + <dependency> + <groupId>org.apache.tajo</groupId> + <artifactId>tajo-storage-hdfs</artifactId> <scope>provided</scope> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/tajo/blob/4561711f/tajo-storage/pom.xml ---------------------------------------------------------------------- diff --git a/tajo-storage/pom.xml b/tajo-storage/pom.xml index 7ede2e1..8acb1a9 100644 --- a/tajo-storage/pom.xml +++ b/tajo-storage/pom.xml @@ -1,5 +1,5 @@ -<!-- <?xml version="1.0" encoding="UTF-8"?> +<!-- Copyright 2012 Database Lab., Korea Univ. Licensed under the Apache License, Version 2.0 (the "License"); @@ -16,310 +16,47 @@ --> <project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> - <modelVersion>4.0.0</modelVersion> <parent> <artifactId>tajo-project</artifactId> <groupId>org.apache.tajo</groupId> <version>0.9.1-SNAPSHOT</version> <relativePath>../tajo-project</relativePath> </parent> - + <modelVersion>4.0.0</modelVersion> <artifactId>tajo-storage</artifactId> - <packaging>jar</packaging> + <packaging>pom</packaging> <name>Tajo Storage</name> - <description>Tajo Storage Package</description> - <properties> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> - <parquet.version>1.5.0</parquet.version> - <parquet.format.version>2.1.0</parquet.format.version> </properties> - <repositories> - <repository> - <id>repository.jboss.org</id> - <url>https://repository.jboss.org/nexus/content/repositories/releases/ - </url> - <snapshots> - <enabled>false</enabled> - </snapshots> - </repository> - </repositories> + <modules> + <module>tajo-storage-common</module> + <module>tajo-storage-hdfs</module> + <module>tajo-storage-hbase</module> + </modules> <build> <plugins> <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-compiler-plugin</artifactId> - <configuration> - <source>1.6</source> - <target>1.6</target> - <encoding>${project.build.sourceEncoding}</encoding> - </configuration> - </plugin> - <plugin> <groupId>org.apache.rat</groupId> <artifactId>apache-rat-plugin</artifactId> - <executions> - <execution> - <phase>verify</phase> - <goals> - <goal>check</goal> - </goals> - </execution> - </executions> - <configuration> - <excludes> - <exclude>src/test/resources/dataset/**</exclude> - </excludes> - </configuration> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-surefire-plugin</artifactId> - <configuration> - <systemProperties> - <tajo.test>TRUE</tajo.test> - </systemProperties> - <argLine>-Xms512m -Xmx1024m -XX:MaxPermSize=128m -Dfile.encoding=UTF-8</argLine> - </configuration> </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-jar-plugin</artifactId> - <version>2.4</version> - <executions> - <execution> - <goals> - <goal>test-jar</goal> - </goals> - </execution> - </executions> + <artifactId>maven-surefire-report-plugin</artifactId> </plugin> <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-antrun-plugin</artifactId> - <executions> - <execution> - <id>create-protobuf-generated-sources-directory</id> - <phase>initialize</phase> - <configuration> - <target> - <mkdir dir="target/generated-sources/proto" /> - </target> - </configuration> - <goals> - <goal>run</goal> - </goals> - </execution> - </executions> - </plugin> - <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>exec-maven-plugin</artifactId> - <version>1.2</version> - <executions> - <execution> - <id>generate-sources</id> - <phase>generate-sources</phase> - <configuration> - <executable>protoc</executable> - <arguments> - <argument>-Isrc/main/proto/</argument> - <argument>--proto_path=../tajo-common/src/main/proto</argument> - <argument>--proto_path=../tajo-catalog/tajo-catalog-common/src/main/proto</argument> - <argument>--java_out=target/generated-sources/proto</argument> - <argument>src/main/proto/IndexProtos.proto</argument> - </arguments> - </configuration> - <goals> - <goal>exec</goal> - </goals> - </execution> - </executions> - </plugin> - <plugin> - <groupId>org.codehaus.mojo</groupId> - <artifactId>build-helper-maven-plugin</artifactId> - <version>1.5</version> - <executions> - <execution> - <id>add-source</id> - <phase>generate-sources</phase> - <goals> - <goal>add-source</goal> - </goals> - <configuration> - <sources> - <source>target/generated-sources/proto</source> - </sources> - </configuration> - </execution> - </executions> - </plugin> - <plugin> - <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-pmd-plugin</artifactId> - <version>2.7.1</version> + <artifactId>maven-deploy-plugin</artifactId> + <configuration> + <skip>true</skip> + </configuration> </plugin> </plugins> </build> - <dependencies> - <dependency> - <groupId>org.apache.tajo</groupId> - <artifactId>tajo-common</artifactId> - </dependency> - <dependency> - <groupId>org.apache.tajo</groupId> - <artifactId>tajo-catalog-common</artifactId> - </dependency> - - <dependency> - <groupId>org.apache.avro</groupId> - <artifactId>avro-mapred</artifactId> - <version>1.7.7</version> - <exclusions> - <exclusion> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-core</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-common</artifactId> - <scope>provided</scope> - <exclusions> - <exclusion> - <artifactId>zookeeper</artifactId> - <groupId>org.apache.zookeeper</groupId> - </exclusion> - <exclusion> - <artifactId>slf4j-api</artifactId> - <groupId>org.slf4j</groupId> - </exclusion> - <exclusion> - <artifactId>jersey-json</artifactId> - <groupId>com.sun.jersey</groupId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-hdfs</artifactId> - <scope>provided</scope> - <exclusions> - <exclusion> - <groupId>commons-el</groupId> - <artifactId>commons-el</artifactId> - </exclusion> - <exclusion> - <groupId>tomcat</groupId> - <artifactId>jasper-runtime</artifactId> - </exclusion> - <exclusion> - <groupId>tomcat</groupId> - <artifactId>jasper-compiler</artifactId> - </exclusion> - <exclusion> - <groupId>org.mortbay.jetty</groupId> - <artifactId>jsp-2.1-jetty</artifactId> - </exclusion> - <exclusion> - <groupId>com.sun.jersey.jersey-test-framework</groupId> - <artifactId>jersey-test-framework-grizzly2</artifactId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>org.apache.hadoop</groupId> - <artifactId>hadoop-minicluster</artifactId> - <scope>test</scope> - <exclusions> - <exclusion> - <groupId>commons-el</groupId> - <artifactId>commons-el</artifactId> - </exclusion> - <exclusion> - <groupId>tomcat</groupId> - <artifactId>jasper-runtime</artifactId> - </exclusion> - <exclusion> - <groupId>tomcat</groupId> - <artifactId>jasper-compiler</artifactId> - </exclusion> - <exclusion> - <groupId>org.mortbay.jetty</groupId> - <artifactId>jsp-2.1-jetty</artifactId> - </exclusion> - <exclusion> - <groupId>com.sun.jersey.jersey-test-framework</groupId> - <artifactId>jersey-test-framework-grizzly2</artifactId> - </exclusion> - <exclusion> - <artifactId>hadoop-yarn-server-tests</artifactId> - <groupId>org.apache.hadoop</groupId> - </exclusion> - <exclusion> - <artifactId>hadoop-mapreduce-client-jobclient</artifactId> - <groupId>org.apache.hadoop</groupId> - </exclusion> - <exclusion> - <artifactId>hadoop-mapreduce-client-app</artifactId> - <groupId>org.apache.hadoop</groupId> - </exclusion> - <exclusion> - <artifactId>hadoop-yarn-api</artifactId> - <groupId>org.apache.hadoop</groupId> - </exclusion> - <exclusion> - <artifactId>hadoop-mapreduce-client-hs</artifactId> - <groupId>org.apache.hadoop</groupId> - </exclusion> - <exclusion> - <artifactId>hadoop-mapreduce-client-core</artifactId> - <groupId>org.apache.hadoop</groupId> - </exclusion> - </exclusions> - </dependency> - <dependency> - <groupId>com.google.protobuf</groupId> - <artifactId>protobuf-java</artifactId> - </dependency> - <dependency> - <groupId>junit</groupId> - <artifactId>junit</artifactId> - <scope>test</scope> - </dependency> - <dependency> - <groupId>com.twitter</groupId> - <artifactId>parquet-column</artifactId> - <version>${parquet.version}</version> - </dependency> - <dependency> - <groupId>com.twitter</groupId> - <artifactId>parquet-hadoop</artifactId> - <version>${parquet.version}</version> - </dependency> - <dependency> - <groupId>com.twitter</groupId> - <artifactId>parquet-format</artifactId> - <version>${parquet.format.version}</version> - </dependency> - <dependency> - <groupId>io.netty</groupId> - <artifactId>netty-buffer</artifactId> - </dependency> - <dependency> - <groupId>net.minidev</groupId> - <artifactId>json-smart</artifactId> - <version>2.0</version> - </dependency> - </dependencies> - <profiles> <profile> <id>docs</id> @@ -364,7 +101,7 @@ <executions> <execution> <id>dist</id> - <phase>package</phase> + <phase>prepare-package</phase> <goals> <goal>run</goal> </goals> @@ -387,12 +124,15 @@ echo echo "Current directory `pwd`" echo - run rm -rf ${project.artifactId}-${project.version} - run mkdir ${project.artifactId}-${project.version} - run cd ${project.artifactId}-${project.version} - run cp -r ${basedir}/target/${project.artifactId}-${project.version}*.jar . + run rm -rf tajo-storage-${project.version} + run mkdir tajo-storage-${project.version} + run cd tajo-storage-${project.version} + run cp -r ${basedir}/tajo-storage-common/target/tajo-storage-common-${project.version}*.jar . + run cp -r ${basedir}/tajo-storage-hdfs/target/tajo-storage-hdfs-${project.version}*.jar . + run cp -r ${basedir}/tajo-storage-hbase/target/tajo-storage-hbase-${project.version}*.jar . + echo - echo "Tajo Storage dist layout available at: ${project.build.directory}/${project.artifactId}-${project.version}" + echo "Tajo Storage dist layout available at: ${project.build.directory}/tajo-storage-${project.version}" echo </echo> <exec executable="sh" dir="${project.build.directory}" failonerror="true"> @@ -412,11 +152,7 @@ <plugins> <plugin> <groupId>org.apache.maven.plugins</groupId> - <artifactId>maven-project-info-reports-plugin</artifactId> - <version>2.4</version> - <configuration> - <dependencyLocationsEnabled>false</dependencyLocationsEnabled> - </configuration> + <artifactId>maven-surefire-report-plugin</artifactId> </plugin> </plugins> </reporting> http://git-wip-us.apache.org/repos/asf/tajo/blob/4561711f/tajo-storage/src/main/java/org/apache/tajo/storage/Appender.java ---------------------------------------------------------------------- diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/Appender.java b/tajo-storage/src/main/java/org/apache/tajo/storage/Appender.java deleted file mode 100644 index c5e96ac..0000000 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/Appender.java +++ /dev/null @@ -1,41 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage; - -import org.apache.tajo.catalog.statistics.TableStats; - -import java.io.Closeable; -import java.io.IOException; - -public interface Appender extends Closeable { - - void init() throws IOException; - - void addTuple(Tuple t) throws IOException; - - void flush() throws IOException; - - long getEstimatedOutputSize() throws IOException; - - void close() throws IOException; - - void enableStats(); - - TableStats getStats(); -} http://git-wip-us.apache.org/repos/asf/tajo/blob/4561711f/tajo-storage/src/main/java/org/apache/tajo/storage/BaseTupleComparator.java ---------------------------------------------------------------------- diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/BaseTupleComparator.java b/tajo-storage/src/main/java/org/apache/tajo/storage/BaseTupleComparator.java deleted file mode 100644 index b829f60..0000000 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/BaseTupleComparator.java +++ /dev/null @@ -1,206 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage; - -import com.google.common.base.Objects; -import com.google.common.base.Preconditions; -import org.apache.tajo.catalog.Schema; -import org.apache.tajo.catalog.SortSpec; -import org.apache.tajo.common.ProtoObject; -import org.apache.tajo.datum.Datum; - -import static org.apache.tajo.catalog.proto.CatalogProtos.TupleComparatorSpecProto; -import static org.apache.tajo.index.IndexProtos.TupleComparatorProto; - -/** - * The Comparator class for Tuples - * - * @see Tuple - */ -public class BaseTupleComparator extends TupleComparator implements ProtoObject<TupleComparatorProto> { - private final Schema schema; - private final SortSpec [] sortSpecs; - private final int[] sortKeyIds; - private final boolean[] asc; - @SuppressWarnings("unused") - private final boolean[] nullFirsts; - - private Datum left; - private Datum right; - private int compVal; - - /** - * @param schema The schema of input tuples - * @param sortKeys The description of sort keys - */ - public BaseTupleComparator(Schema schema, SortSpec[] sortKeys) { - Preconditions.checkArgument(sortKeys.length > 0, - "At least one sort key must be specified."); - - this.schema = schema; - this.sortSpecs = sortKeys; - this.sortKeyIds = new int[sortKeys.length]; - this.asc = new boolean[sortKeys.length]; - this.nullFirsts = new boolean[sortKeys.length]; - for (int i = 0; i < sortKeys.length; i++) { - if (sortKeys[i].getSortKey().hasQualifier()) { - this.sortKeyIds[i] = schema.getColumnId(sortKeys[i].getSortKey().getQualifiedName()); - } else { - this.sortKeyIds[i] = schema.getColumnIdByName(sortKeys[i].getSortKey().getSimpleName()); - } - - this.asc[i] = sortKeys[i].isAscending(); - this.nullFirsts[i]= sortKeys[i].isNullFirst(); - } - } - - public BaseTupleComparator(TupleComparatorProto proto) { - this.schema = new Schema(proto.getSchema()); - - this.sortSpecs = new SortSpec[proto.getSortSpecsCount()]; - for (int i = 0; i < proto.getSortSpecsCount(); i++) { - sortSpecs[i] = new SortSpec(proto.getSortSpecs(i)); - } - - this.sortKeyIds = new int[proto.getCompSpecsCount()]; - this.asc = new boolean[proto.getCompSpecsCount()]; - this.nullFirsts = new boolean[proto.getCompSpecsCount()]; - - for (int i = 0; i < proto.getCompSpecsCount(); i++) { - TupleComparatorSpecProto sortSepcProto = proto.getCompSpecs(i); - sortKeyIds[i] = sortSepcProto.getColumnId(); - asc[i] = sortSepcProto.getAscending(); - nullFirsts[i] = sortSepcProto.getNullFirst(); - } - } - - public Schema getSchema() { - return schema; - } - - public SortSpec [] getSortSpecs() { - return sortSpecs; - } - - public int [] getSortKeyIds() { - return sortKeyIds; - } - - @Override - public boolean isAscendingFirstKey() { - return this.asc[0]; - } - - @Override - public int compare(Tuple tuple1, Tuple tuple2) { - for (int i = 0; i < sortKeyIds.length; i++) { - left = tuple1.get(sortKeyIds[i]); - right = tuple2.get(sortKeyIds[i]); - - if (left.isNull() || right.isNull()) { - if (!left.equals(right)) { - if (left.isNull()) { - compVal = 1; - } else if (right.isNull()) { - compVal = -1; - } - if (nullFirsts[i]) { - if (compVal != 0) { - compVal *= -1; - } - } - } else { - compVal = 0; - } - } else { - if (asc[i]) { - compVal = left.compareTo(right); - } else { - compVal = right.compareTo(left); - } - } - - if (compVal < 0 || compVal > 0) { - return compVal; - } - } - return 0; - } - - @Override - public int hashCode() { - return Objects.hashCode(sortKeyIds); - } - - @Override - public boolean equals(Object obj) { - if (obj instanceof BaseTupleComparator) { - BaseTupleComparator other = (BaseTupleComparator) obj; - if (sortKeyIds.length != other.sortKeyIds.length) { - return false; - } - - for (int i = 0; i < sortKeyIds.length; i++) { - if (sortKeyIds[i] != other.sortKeyIds[i] || - asc[i] != other.asc[i] || - nullFirsts[i] != other.nullFirsts[i]) { - return false; - } - } - - return true; - } else { - return false; - } - } - - @Override - public TupleComparatorProto getProto() { - TupleComparatorProto.Builder builder = TupleComparatorProto.newBuilder(); - builder.setSchema(schema.getProto()); - for (int i = 0; i < sortSpecs.length; i++) { - builder.addSortSpecs(sortSpecs[i].getProto()); - } - - TupleComparatorSpecProto.Builder sortSpecBuilder; - for (int i = 0; i < sortKeyIds.length; i++) { - sortSpecBuilder = TupleComparatorSpecProto.newBuilder(); - sortSpecBuilder.setColumnId(sortKeyIds[i]); - sortSpecBuilder.setAscending(asc[i]); - sortSpecBuilder.setNullFirst(nullFirsts[i]); - builder.addCompSpecs(sortSpecBuilder); - } - - return builder.build(); - } - - @Override - public String toString() { - StringBuilder sb = new StringBuilder(); - - String prefix = ""; - for (int i = 0; i < sortKeyIds.length; i++) { - sb.append(prefix).append("SortKeyId=").append(sortKeyIds[i]) - .append(",Asc=").append(asc[i]) - .append(",NullFirst=").append(nullFirsts[i]); - prefix = " ,"; - } - return sb.toString(); - } -} \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/4561711f/tajo-storage/src/main/java/org/apache/tajo/storage/BinarySerializerDeserializer.java ---------------------------------------------------------------------- diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/BinarySerializerDeserializer.java b/tajo-storage/src/main/java/org/apache/tajo/storage/BinarySerializerDeserializer.java deleted file mode 100644 index 00112e7..0000000 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/BinarySerializerDeserializer.java +++ /dev/null @@ -1,258 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage; - -import com.google.common.base.Preconditions; -import com.google.protobuf.Message; -import org.apache.tajo.catalog.Column; -import org.apache.tajo.datum.*; -import org.apache.tajo.util.Bytes; - -import java.io.IOException; -import java.io.OutputStream; - -@Deprecated -public class BinarySerializerDeserializer implements SerializerDeserializer { - - static final byte[] INVALID_UTF__SINGLE_BYTE = {(byte) Integer.parseInt("10111111", 2)}; - - @Override - public int serialize(Column col, Datum datum, OutputStream out, byte[] nullCharacters) - throws IOException { - byte[] bytes; - int length = 0; - if (datum == null || datum instanceof NullDatum) { - return 0; - } - - switch (col.getDataType().getType()) { - case BOOLEAN: - case BIT: - case CHAR: - bytes = datum.asByteArray(); - length = bytes.length; - out.write(bytes, 0, length); - break; - case INT2: - length = writeShort(out, datum.asInt2()); - break; - case INT4: - length = writeVLong(out, datum.asInt4()); - break; - case INT8: - length = writeVLong(out, datum.asInt8()); - break; - case FLOAT4: - length = writeFloat(out, datum.asFloat4()); - break; - case FLOAT8: - length = writeDouble(out, datum.asFloat8()); - break; - case TEXT: { - bytes = datum.asTextBytes(); - length = datum.size(); - if (length == 0) { - bytes = INVALID_UTF__SINGLE_BYTE; - length = INVALID_UTF__SINGLE_BYTE.length; - } - out.write(bytes, 0, bytes.length); - break; - } - case BLOB: - case INET4: - case INET6: - bytes = datum.asByteArray(); - length = bytes.length; - out.write(bytes, 0, length); - break; - case PROTOBUF: - ProtobufDatum protobufDatum = (ProtobufDatum) datum; - bytes = protobufDatum.asByteArray(); - length = bytes.length; - out.write(bytes, 0, length); - break; - case NULL_TYPE: - break; - default: - throw new IOException("Does not support type"); - } - return length; - } - - @Override - public Datum deserialize(Column col, byte[] bytes, int offset, int length, byte[] nullCharacters) throws IOException { - if (length == 0) return NullDatum.get(); - - Datum datum; - switch (col.getDataType().getType()) { - case BOOLEAN: - datum = DatumFactory.createBool(bytes[offset]); - break; - case BIT: - datum = DatumFactory.createBit(bytes[offset]); - break; - case CHAR: { - byte[] chars = new byte[length]; - System.arraycopy(bytes, offset, chars, 0, length); - datum = DatumFactory.createChar(chars); - break; - } - case INT2: - datum = DatumFactory.createInt2(Bytes.toShort(bytes, offset, length)); - break; - case INT4: - datum = DatumFactory.createInt4((int) Bytes.readVLong(bytes, offset)); - break; - case INT8: - datum = DatumFactory.createInt8(Bytes.readVLong(bytes, offset)); - break; - case FLOAT4: - datum = DatumFactory.createFloat4(toFloat(bytes, offset, length)); - break; - case FLOAT8: - datum = DatumFactory.createFloat8(toDouble(bytes, offset, length)); - break; - case TEXT: { - byte[] chars = new byte[length]; - System.arraycopy(bytes, offset, chars, 0, length); - - if (Bytes.equals(INVALID_UTF__SINGLE_BYTE, chars)) { - datum = DatumFactory.createText(new byte[0]); - } else { - datum = DatumFactory.createText(chars); - } - break; - } - case PROTOBUF: { - ProtobufDatumFactory factory = ProtobufDatumFactory.get(col.getDataType().getCode()); - Message.Builder builder = factory.newBuilder(); - builder.mergeFrom(bytes, offset, length); - datum = factory.createDatum(builder); - break; - } - case INET4: - datum = DatumFactory.createInet4(bytes, offset, length); - break; - case BLOB: - datum = DatumFactory.createBlob(bytes, offset, length); - break; - default: - datum = NullDatum.get(); - } - return datum; - } - - private byte[] shortBytes = new byte[2]; - - public int writeShort(OutputStream out, short val) throws IOException { - shortBytes[0] = (byte) (val >> 8); - shortBytes[1] = (byte) val; - out.write(shortBytes, 0, 2); - return 2; - } - - public float toFloat(byte[] bytes, int offset, int length) { - Preconditions.checkArgument(length == 4); - - int val = ((bytes[offset] & 0x000000FF) << 24) + - ((bytes[offset + 1] & 0x000000FF) << 16) + - ((bytes[offset + 2] & 0x000000FF) << 8) + - (bytes[offset + 3] & 0x000000FF); - return Float.intBitsToFloat(val); - } - - private byte[] floatBytes = new byte[4]; - - public int writeFloat(OutputStream out, float f) throws IOException { - int val = Float.floatToIntBits(f); - - floatBytes[0] = (byte) (val >> 24); - floatBytes[1] = (byte) (val >> 16); - floatBytes[2] = (byte) (val >> 8); - floatBytes[3] = (byte) val; - out.write(floatBytes, 0, 4); - return floatBytes.length; - } - - public double toDouble(byte[] bytes, int offset, int length) { - Preconditions.checkArgument(length == 8); - long val = ((long) (bytes[offset] & 0x00000000000000FF) << 56) + - ((long) (bytes[offset + 1] & 0x00000000000000FF) << 48) + - ((long) (bytes[offset + 2] & 0x00000000000000FF) << 40) + - ((long) (bytes[offset + 3] & 0x00000000000000FF) << 32) + - ((long) (bytes[offset + 4] & 0x00000000000000FF) << 24) + - ((long) (bytes[offset + 5] & 0x00000000000000FF) << 16) + - ((long) (bytes[offset + 6] & 0x00000000000000FF) << 8) + - (long) (bytes[offset + 7] & 0x00000000000000FF); - return Double.longBitsToDouble(val); - } - - private byte[] doubleBytes = new byte[8]; - - public int writeDouble(OutputStream out, double d) throws IOException { - long val = Double.doubleToLongBits(d); - - doubleBytes[0] = (byte) (val >> 56); - doubleBytes[1] = (byte) (val >> 48); - doubleBytes[2] = (byte) (val >> 40); - doubleBytes[3] = (byte) (val >> 32); - doubleBytes[4] = (byte) (val >> 24); - doubleBytes[5] = (byte) (val >> 16); - doubleBytes[6] = (byte) (val >> 8); - doubleBytes[7] = (byte) val; - out.write(doubleBytes, 0, 8); - return doubleBytes.length; - } - - private byte[] vLongBytes = new byte[9]; - - public static int writeVLongToByteArray(byte[] bytes, int offset, long l) { - if (l >= -112 && l <= 127) { - bytes[offset] = (byte) l; - return 1; - } - - int len = -112; - if (l < 0) { - l ^= -1L; // take one's complement' - len = -120; - } - - long tmp = l; - while (tmp != 0) { - tmp = tmp >> 8; - len--; - } - - bytes[offset++] = (byte) len; - len = (len < -120) ? -(len + 120) : -(len + 112); - - for (int idx = len; idx != 0; idx--) { - int shiftbits = (idx - 1) * 8; - bytes[offset++] = (byte) ((l & (0xFFL << shiftbits)) >> shiftbits); - } - return 1 + len; - } - - public int writeVLong(OutputStream out, long l) throws IOException { - int len = writeVLongToByteArray(vLongBytes, 0, l); - out.write(vLongBytes, 0, len); - return len; - } -} http://git-wip-us.apache.org/repos/asf/tajo/blob/4561711f/tajo-storage/src/main/java/org/apache/tajo/storage/BufferPool.java ---------------------------------------------------------------------- diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/BufferPool.java b/tajo-storage/src/main/java/org/apache/tajo/storage/BufferPool.java deleted file mode 100644 index 85c79fa..0000000 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/BufferPool.java +++ /dev/null @@ -1,74 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage; - -import io.netty.buffer.ByteBuf; -import io.netty.buffer.PooledByteBufAllocator; -import io.netty.util.internal.PlatformDependent; -import org.apache.hadoop.classification.InterfaceStability; - -/* this class is PooledBuffer holder */ -public class BufferPool { - - private static final PooledByteBufAllocator allocator; - - private BufferPool() { - } - - static { - //TODO we need determine the default params - allocator = new PooledByteBufAllocator(PlatformDependent.directBufferPreferred()); - - /* if you are finding memory leak, please enable this line */ - //ResourceLeakDetector.setLevel(ResourceLeakDetector.Level.ADVANCED); - } - - public static long maxDirectMemory() { - return PlatformDependent.maxDirectMemory(); - } - - - public synchronized static ByteBuf directBuffer(int size) { - return allocator.directBuffer(size); - } - - /** - * - * @param size the initial capacity - * @param max the max capacity - * @return allocated ByteBuf from pool - */ - public static ByteBuf directBuffer(int size, int max) { - return allocator.directBuffer(size, max); - } - - @InterfaceStability.Unstable - public static void forceRelease(ByteBuf buf) { - buf.release(buf.refCnt()); - } - - /** - * the ByteBuf will increase to writable size - * @param buf - * @param minWritableBytes required minimum writable size - */ - public static void ensureWritable(ByteBuf buf, int minWritableBytes) { - buf.ensureWritable(minWritableBytes); - } -} http://git-wip-us.apache.org/repos/asf/tajo/blob/4561711f/tajo-storage/src/main/java/org/apache/tajo/storage/ByteBufInputChannel.java ---------------------------------------------------------------------- diff --git a/tajo-storage/src/main/java/org/apache/tajo/storage/ByteBufInputChannel.java b/tajo-storage/src/main/java/org/apache/tajo/storage/ByteBufInputChannel.java deleted file mode 100644 index 45fb1d8..0000000 --- a/tajo-storage/src/main/java/org/apache/tajo/storage/ByteBufInputChannel.java +++ /dev/null @@ -1,72 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.storage; - -import org.apache.hadoop.fs.ByteBufferReadable; -import org.apache.hadoop.hdfs.DFSInputStream; -import org.apache.hadoop.io.IOUtils; - -import java.io.IOException; -import java.io.InputStream; -import java.nio.ByteBuffer; -import java.nio.channels.Channels; -import java.nio.channels.ReadableByteChannel; -import java.nio.channels.ScatteringByteChannel; -import java.nio.channels.spi.AbstractInterruptibleChannel; - -public class ByteBufInputChannel extends AbstractInterruptibleChannel implements ScatteringByteChannel { - - ByteBufferReadable byteBufferReadable; - ReadableByteChannel channel; - InputStream inputStream; - - public ByteBufInputChannel(InputStream inputStream) { - if (inputStream instanceof DFSInputStream && inputStream instanceof ByteBufferReadable) { - this.byteBufferReadable = (ByteBufferReadable) inputStream; - } else { - this.channel = Channels.newChannel(inputStream); - } - - this.inputStream = inputStream; - } - - @Override - public long read(ByteBuffer[] dsts, int offset, int length) { - throw new UnsupportedOperationException(); - } - - @Override - public long read(ByteBuffer[] dsts) { - return read(dsts, 0, dsts.length); - } - - @Override - public int read(ByteBuffer dst) throws IOException { - if (byteBufferReadable != null) { - return byteBufferReadable.read(dst); - } else { - return channel.read(dst); - } - } - - @Override - protected void implCloseChannel() throws IOException { - IOUtils.cleanup(null, channel, inputStream); - } -}
