Repository: tajo Updated Branches: refs/heads/branch-0.8.0 9c33b3294 -> 334cc84ae
TAJO-612: Missing INET4 type in SQLParser. (fixed missing commit) Project: http://git-wip-us.apache.org/repos/asf/tajo/repo Commit: http://git-wip-us.apache.org/repos/asf/tajo/commit/334cc84a Tree: http://git-wip-us.apache.org/repos/asf/tajo/tree/334cc84a Diff: http://git-wip-us.apache.org/repos/asf/tajo/diff/334cc84a Branch: refs/heads/branch-0.8.0 Commit: 334cc84ae8318b0f6bd839f424961f0a42a9f262 Parents: 9c33b32 Author: Jihoon Son <[email protected]> Authored: Tue Mar 25 14:57:40 2014 +0900 Committer: Jihoon Son <[email protected]> Committed: Tue Mar 25 14:57:40 2014 +0900 ---------------------------------------------------------------------- .../org/apache/tajo/engine/parser/SQLParser.g4 | 5 ++ .../apache/tajo/engine/function/Country.java | 48 ------------- .../apache/tajo/engine/function/InCountry.java | 52 -------------- .../function/geoip/GeoIPCountryInet4.java | 55 +++++++++++++++ .../engine/function/geoip/GeoIPCountryText.java | 56 +++++++++++++++ .../function/geoip/GeoIPInCountryInet4.java | 61 ++++++++++++++++ .../function/geoip/GeoIPInCountryText.java | 60 ++++++++++++++++ .../apache/tajo/engine/parser/SQLAnalyzer.java | 2 + .../tajo/engine/planner/ExprsVerifier.java | 8 +++ .../engine/planner/RangePartitionAlgorithm.java | 7 ++ .../java/org/apache/tajo/util/GeoIPUtil.java | 46 ++++++++++++ .../main/java/org/apache/tajo/util/GeoUtil.java | 46 ------------ .../apache/tajo/engine/query/TestNetTypes.java | 74 ++++++++++++++++++++ .../resources/dataset/TestNetTypes/table1.tbl | 5 ++ .../resources/dataset/TestNetTypes/table2.tbl | 4 ++ .../queries/TestNetTypes/table1_ddl.sql | 4 ++ .../queries/TestNetTypes/table2_ddl.sql | 4 ++ .../queries/TestNetTypes/testGroupby.sql | 1 + .../queries/TestNetTypes/testGroupby2.sql | 1 + .../resources/queries/TestNetTypes/testJoin.sql | 1 + .../queries/TestNetTypes/testSelect.sql | 1 + .../resources/queries/TestNetTypes/testSort.sql | 1 + .../results/TestNetTypes/testGroupby.result | 7 ++ .../results/TestNetTypes/testGroupby2.result | 5 ++ .../results/TestNetTypes/testJoin.result | 6 ++ .../results/TestNetTypes/testSelect.result | 7 ++ .../results/TestNetTypes/testSort.result | 7 ++ tajo-docs/src/main/sphinx/functions.rst | 3 +- .../functions/network_func_and_operators.rst | 57 +++++++++++++++ 29 files changed, 487 insertions(+), 147 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/tajo/blob/334cc84a/tajo-core/tajo-core-backend/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 b/tajo-core/tajo-core-backend/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 index 699b824..ce78c98 100644 --- a/tajo-core/tajo-core-backend/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 +++ b/tajo-core/tajo-core-backend/src/main/antlr4/org/apache/tajo/engine/parser/SQLParser.g4 @@ -377,6 +377,11 @@ predefined_type | datetime_type | bit_type | binary_type + | network_type + ; + +network_type + : INET4 ; character_string_type http://git-wip-us.apache.org/repos/asf/tajo/blob/334cc84a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/Country.java ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/Country.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/Country.java deleted file mode 100644 index 9e28b55..0000000 --- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/Country.java +++ /dev/null @@ -1,48 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.engine.function; - -import org.apache.tajo.catalog.Column; -import org.apache.tajo.common.TajoDataTypes; -import org.apache.tajo.datum.TextDatum; -import org.apache.tajo.engine.function.annotation.Description; -import org.apache.tajo.engine.function.annotation.ParamTypes; -import org.apache.tajo.storage.Tuple; -import org.apache.tajo.util.GeoUtil; - -import static org.apache.tajo.common.TajoDataTypes.Type.TEXT; - -@Description( - functionName = "country", - description = "Returns country code.", - example = "", - returnType = TajoDataTypes.Type.TEXT, - paramTypes = {@ParamTypes(paramTypes = {TajoDataTypes.Type.TEXT})} -) -public class Country extends GeneralFunction { - - public Country() { - super(new Column[] {new Column("string", TEXT)}); - } - - @Override - public TextDatum eval(Tuple params) { - return new TextDatum(GeoUtil.getCountryCode(params.get(0).asChars())); - } -} http://git-wip-us.apache.org/repos/asf/tajo/blob/334cc84a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/InCountry.java ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/InCountry.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/InCountry.java deleted file mode 100644 index 1cac624..0000000 --- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/InCountry.java +++ /dev/null @@ -1,52 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.engine.function; - -import org.apache.tajo.catalog.Column; -import org.apache.tajo.common.TajoDataTypes; -import org.apache.tajo.datum.BooleanDatum; -import org.apache.tajo.datum.DatumFactory; -import org.apache.tajo.engine.function.annotation.Description; -import org.apache.tajo.engine.function.annotation.ParamTypes; -import org.apache.tajo.storage.Tuple; -import org.apache.tajo.util.GeoUtil; - -@Description( - functionName = "in_country", - description = "", - example = "", - returnType = TajoDataTypes.Type.BOOLEAN, - paramTypes = {@ParamTypes(paramTypes = {TajoDataTypes.Type.TEXT})} -) -public class InCountry extends GeneralFunction { - - public InCountry() { - super(new Column[] {new Column("string", TajoDataTypes.Type.TEXT), - new Column("code", TajoDataTypes.Type.TEXT)}); - } - - @Override - public BooleanDatum eval(Tuple params) { - String addr = params.get(0).asChars(); - String otherCode = params.get(1).asChars(); - String thisCode = GeoUtil.getCountryCode(addr); - - return DatumFactory.createBool(thisCode.equals(otherCode)); - } -} http://git-wip-us.apache.org/repos/asf/tajo/blob/334cc84a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/geoip/GeoIPCountryInet4.java ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/geoip/GeoIPCountryInet4.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/geoip/GeoIPCountryInet4.java new file mode 100644 index 0000000..d922e76 --- /dev/null +++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/geoip/GeoIPCountryInet4.java @@ -0,0 +1,55 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.function.geoip; + +import org.apache.tajo.catalog.Column; +import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.common.TajoDataTypes.Type; +import org.apache.tajo.datum.Datum; +import org.apache.tajo.datum.NullDatum; +import org.apache.tajo.datum.TextDatum; +import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.engine.function.annotation.Description; +import org.apache.tajo.engine.function.annotation.ParamTypes; +import org.apache.tajo.storage.Tuple; +import org.apache.tajo.util.GeoIPUtil; + +@Description( + functionName = "geoip_country_code", + description = "Convert an ipv4 address to a geoip country code.", + example = "> SELECT geoip_country_code(8.8.8.8);\n" + + "US", + returnType = TajoDataTypes.Type.TEXT, + paramTypes = {@ParamTypes(paramTypes = {Type.INET4})} +) +public class GeoIPCountryInet4 extends GeneralFunction { + + public GeoIPCountryInet4() { + super(new Column[] {new Column("ipv4_address", TajoDataTypes.Type.INET4)}); + } + + @Override + public Datum eval(Tuple params) { + Datum valueDatum = params.get(0); + if (valueDatum instanceof NullDatum) { + return NullDatum.get(); + } + return new TextDatum(GeoIPUtil.getCountryCode(params.get(0).asChars())); + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/334cc84a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/geoip/GeoIPCountryText.java ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/geoip/GeoIPCountryText.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/geoip/GeoIPCountryText.java new file mode 100644 index 0000000..17a43be --- /dev/null +++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/geoip/GeoIPCountryText.java @@ -0,0 +1,56 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.function.geoip; + +import org.apache.tajo.catalog.Column; +import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.datum.Datum; +import org.apache.tajo.datum.NullDatum; +import org.apache.tajo.datum.TextDatum; +import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.engine.function.annotation.Description; +import org.apache.tajo.engine.function.annotation.ParamTypes; +import org.apache.tajo.storage.Tuple; +import org.apache.tajo.util.GeoIPUtil; + +import static org.apache.tajo.common.TajoDataTypes.Type.TEXT; + +@Description( + functionName = "geoip_country_code", + description = "Convert an ipv4 address string to a geoip country code.", + example = "> SELECT geoip_country_code('8.8.8.8');\n" + + "US", + returnType = TajoDataTypes.Type.TEXT, + paramTypes = {@ParamTypes(paramTypes = {TajoDataTypes.Type.TEXT})} +) +public class GeoIPCountryText extends GeneralFunction { + + public GeoIPCountryText() { + super(new Column[] {new Column("ipv4_address_string", TEXT)}); + } + + @Override + public Datum eval(Tuple params) { + Datum valueDatum = params.get(0); + if (valueDatum instanceof NullDatum) { + return NullDatum.get(); + } + return new TextDatum(GeoIPUtil.getCountryCode(params.get(0).asChars())); + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/334cc84a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/geoip/GeoIPInCountryInet4.java ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/geoip/GeoIPInCountryInet4.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/geoip/GeoIPInCountryInet4.java new file mode 100644 index 0000000..19ce8ba --- /dev/null +++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/geoip/GeoIPInCountryInet4.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.function.geoip; + +import org.apache.tajo.catalog.Column; +import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.common.TajoDataTypes.Type; +import org.apache.tajo.datum.Datum; +import org.apache.tajo.datum.DatumFactory; +import org.apache.tajo.datum.NullDatum; +import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.engine.function.annotation.Description; +import org.apache.tajo.engine.function.annotation.ParamTypes; +import org.apache.tajo.storage.Tuple; +import org.apache.tajo.util.GeoIPUtil; + +@Description( + functionName = "geoip_in_country", + description = "If the given country code is same with the country code of the given address, it returns true. " + + "Otherwise, returns false", + example = "geoip_in_country(8.8.8.8, 'US')" + + "true", + returnType = TajoDataTypes.Type.BOOLEAN, + paramTypes = {@ParamTypes(paramTypes = {Type.INET4, Type.TEXT})} +) +public class GeoIPInCountryInet4 extends GeneralFunction { + + public GeoIPInCountryInet4() { + super(new Column[] {new Column("ipv4_address", TajoDataTypes.Type.INET4), + new Column("country_code", TajoDataTypes.Type.TEXT)}); + } + + @Override + public Datum eval(Tuple params) { + if (params.get(0) instanceof NullDatum || params.get(1) instanceof NullDatum) { + return NullDatum.get(); + } + + String addr = params.get(0).asChars(); + String otherCode = params.get(1).asChars(); + String thisCode = GeoIPUtil.getCountryCode(addr); + + return DatumFactory.createBool(thisCode.equals(otherCode)); + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/334cc84a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/geoip/GeoIPInCountryText.java ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/geoip/GeoIPInCountryText.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/geoip/GeoIPInCountryText.java new file mode 100644 index 0000000..168f86c --- /dev/null +++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/function/geoip/GeoIPInCountryText.java @@ -0,0 +1,60 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.function.geoip; + +import org.apache.tajo.catalog.Column; +import org.apache.tajo.common.TajoDataTypes; +import org.apache.tajo.datum.Datum; +import org.apache.tajo.datum.DatumFactory; +import org.apache.tajo.datum.NullDatum; +import org.apache.tajo.engine.function.GeneralFunction; +import org.apache.tajo.engine.function.annotation.Description; +import org.apache.tajo.engine.function.annotation.ParamTypes; +import org.apache.tajo.storage.Tuple; +import org.apache.tajo.util.GeoIPUtil; + +@Description( + functionName = "geoip_in_country", + description = "If the given country code is same with the country code of the given address, it returns true. " + + "Otherwise, returns false", + example = "geoip_in_country('8.8.8.8', 'US')" + + "true", + returnType = TajoDataTypes.Type.BOOLEAN, + paramTypes = {@ParamTypes(paramTypes = {TajoDataTypes.Type.TEXT, TajoDataTypes.Type.TEXT})} +) +public class GeoIPInCountryText extends GeneralFunction { + + public GeoIPInCountryText() { + super(new Column[] {new Column("ipv4_address_string", TajoDataTypes.Type.TEXT), + new Column("country_code", TajoDataTypes.Type.TEXT)}); + } + + @Override + public Datum eval(Tuple params) { + if (params.get(0) instanceof NullDatum || params.get(1) instanceof NullDatum) { + return NullDatum.get(); + } + + String addr = params.get(0).asChars(); + String otherCode = params.get(1).asChars(); + String thisCode = GeoIPUtil.getCountryCode(addr); + + return DatumFactory.createBool(thisCode.equals(otherCode)); + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/334cc84a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java index 291f5f6..784e737 100644 --- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java +++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/parser/SQLAnalyzer.java @@ -1224,6 +1224,8 @@ public class SQLAnalyzer extends SQLParserBaseVisitor<Expr> { typeDefinition.setLengthOrPrecision( Integer.parseInt(binaryType.type_length().NUMBER().getText())); } + } else if (predefined_type.network_type() != null) { + typeDefinition = new DataTypeExpr(Type.INET4.name()); } return typeDefinition; http://git-wip-us.apache.org/repos/asf/tajo/blob/334cc84a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/ExprsVerifier.java ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/ExprsVerifier.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/ExprsVerifier.java index 358cabd..46f5672 100644 --- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/ExprsVerifier.java +++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/ExprsVerifier.java @@ -71,6 +71,10 @@ public class ExprsVerifier extends BasicEvalNodeVisitor<VerificationState, EvalN return true; } + if (checkNetworkType(dataType1) && checkNetworkType(dataType2)) { + return true; + } + return false; } @@ -142,6 +146,10 @@ public class ExprsVerifier extends BasicEvalNodeVisitor<VerificationState, EvalN } } + private static boolean checkNetworkType(DataType dataType) { + return dataType.getType() == Type.INET4 || dataType.getType() == Type.INET6; + } + private static boolean checkNumericType(DataType dataType) { int typeNumber = dataType.getType().getNumber(); return Type.INT1.getNumber() < typeNumber && typeNumber <= Type.DECIMAL.getNumber(); http://git-wip-us.apache.org/repos/asf/tajo/blob/334cc84a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/RangePartitionAlgorithm.java ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/RangePartitionAlgorithm.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/RangePartitionAlgorithm.java index 68438bd..c4b1ae1 100644 --- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/RangePartitionAlgorithm.java +++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/engine/planner/RangePartitionAlgorithm.java @@ -137,6 +137,13 @@ public abstract class RangePartitionAlgorithm { columnCard = new BigDecimal(start.asInt8() - end.asInt8()); } break; + case INET4: + if (isAscending) { + columnCard = new BigDecimal(end.asInt4() - start.asInt4()); + } else { + columnCard = new BigDecimal(start.asInt4() - end.asInt4()); + } + break; default: throw new UnsupportedOperationException(dataType + " is not supported yet"); } http://git-wip-us.apache.org/repos/asf/tajo/blob/334cc84a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/util/GeoIPUtil.java ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/util/GeoIPUtil.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/util/GeoIPUtil.java new file mode 100644 index 0000000..859b37d --- /dev/null +++ b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/util/GeoIPUtil.java @@ -0,0 +1,46 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.util; + +import com.maxmind.geoip.LookupService; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; +import org.apache.tajo.conf.TajoConf; +import org.apache.tajo.conf.TajoConf.ConfVars; + +import java.io.IOException; + +public class GeoIPUtil { + private static final Log LOG = LogFactory.getLog(GeoIPUtil.class); + private static LookupService lookup; + + static { + try { + TajoConf conf = new TajoConf(); + lookup = new LookupService(conf.getVar(ConfVars.GEOIP_DATA), + LookupService.GEOIP_MEMORY_CACHE); + } catch (IOException e) { + LOG.error("Cannot open the geoip data", e); + } + } + + public static String getCountryCode(String host) { + return lookup.getCountry(host).getCode(); + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/334cc84a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/util/GeoUtil.java ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/util/GeoUtil.java b/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/util/GeoUtil.java deleted file mode 100644 index 6028725..0000000 --- a/tajo-core/tajo-core-backend/src/main/java/org/apache/tajo/util/GeoUtil.java +++ /dev/null @@ -1,46 +0,0 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package org.apache.tajo.util; - -import com.maxmind.geoip.LookupService; -import org.apache.commons.logging.Log; -import org.apache.commons.logging.LogFactory; -import org.apache.tajo.conf.TajoConf; -import org.apache.tajo.conf.TajoConf.ConfVars; - -import java.io.IOException; - -public class GeoUtil { - private static final Log LOG = LogFactory.getLog(GeoUtil.class); - private static LookupService lookup; - - static { - try { - TajoConf conf = new TajoConf(); - lookup = new LookupService(conf.getVar(ConfVars.GEOIP_DATA), - LookupService.GEOIP_MEMORY_CACHE); - } catch (IOException e) { - LOG.error("Cannot open the geoip data", e); - } - } - - public static String getCountryCode(String host) { - return lookup.getCountry(host).getCode(); - } -} http://git-wip-us.apache.org/repos/asf/tajo/blob/334cc84a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestNetTypes.java ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestNetTypes.java b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestNetTypes.java new file mode 100644 index 0000000..bba47cc --- /dev/null +++ b/tajo-core/tajo-core-backend/src/test/java/org/apache/tajo/engine/query/TestNetTypes.java @@ -0,0 +1,74 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package org.apache.tajo.engine.query; + +import org.apache.tajo.QueryTestCaseBase; +import org.junit.Before; +import org.junit.Test; + +import java.sql.ResultSet; + +public class TestNetTypes extends QueryTestCaseBase { + + @Before + public final void setup() throws Exception { + executeDDL("table1_ddl.sql", "table1.tbl"); + executeDDL("table2_ddl.sql", "table2.tbl"); + } + + @Test + public final void testSelect() throws Exception { + // select name, addr from table1; + ResultSet res = executeQuery(); + assertResultSet(res); + cleanupQuery(res); + } + + @Test + public final void testGroupby() throws Exception { + // select name, addr, count(1) from table1 group by name, addr; + ResultSet res = executeQuery(); + assertResultSet(res); + cleanupQuery(res); + } + + @Test + public final void testGroupby2() throws Exception { + // select addr, count(*) from table1 group by addr; + ResultSet res = executeQuery(); + assertResultSet(res); + cleanupQuery(res); + } + + @Test + public final void testSort() throws Exception { + // select * from table1 order by addr; + ResultSet res = executeQuery(); + assertResultSet(res); + cleanupQuery(res); + } + + @Test + public final void testJoin() throws Exception { + // select * from table1 as t1, table2 as t2 where t1.addr = t2.addr; + ResultSet res = executeQuery(); + assertResultSet(res); + cleanupQuery(res); + } +} http://git-wip-us.apache.org/repos/asf/tajo/blob/334cc84a/tajo-core/tajo-core-backend/src/test/resources/dataset/TestNetTypes/table1.tbl ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/test/resources/dataset/TestNetTypes/table1.tbl b/tajo-core/tajo-core-backend/src/test/resources/dataset/TestNetTypes/table1.tbl new file mode 100644 index 0000000..63db89d --- /dev/null +++ b/tajo-core/tajo-core-backend/src/test/resources/dataset/TestNetTypes/table1.tbl @@ -0,0 +1,5 @@ +1|ooo|1.1|a|127.0.0.1 +2|ppp|2.3|b|127.0.1.1 +3|qqq|3.4|c|127.0.0.8 +4|rrr|4.5|d|127.0.0.1 +5|xxx|5.6|e|127.0.1.1 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/334cc84a/tajo-core/tajo-core-backend/src/test/resources/dataset/TestNetTypes/table2.tbl ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/test/resources/dataset/TestNetTypes/table2.tbl b/tajo-core/tajo-core-backend/src/test/resources/dataset/TestNetTypes/table2.tbl new file mode 100644 index 0000000..f33b22c --- /dev/null +++ b/tajo-core/tajo-core-backend/src/test/resources/dataset/TestNetTypes/table2.tbl @@ -0,0 +1,4 @@ +1|NULL|NULL|a|127.0.0.8 +2|NULL|NULL|b|127.0.0.8 +NULL|NULL|10.0|c|NULL +NULL|NULL|20.0|d|127.0.0.1 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/334cc84a/tajo-core/tajo-core-backend/src/test/resources/queries/TestNetTypes/table1_ddl.sql ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/test/resources/queries/TestNetTypes/table1_ddl.sql b/tajo-core/tajo-core-backend/src/test/resources/queries/TestNetTypes/table1_ddl.sql new file mode 100644 index 0000000..45df638 --- /dev/null +++ b/tajo-core/tajo-core-backend/src/test/resources/queries/TestNetTypes/table1_ddl.sql @@ -0,0 +1,4 @@ +-- It is used in TestNetTypes + +create external table table1 (id int, name text, score float, type text, addr inet4) using csv +with ('csvfile.delimiter'='|', 'csvfile.null'='NULL') location ${table.path}; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/334cc84a/tajo-core/tajo-core-backend/src/test/resources/queries/TestNetTypes/table2_ddl.sql ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/test/resources/queries/TestNetTypes/table2_ddl.sql b/tajo-core/tajo-core-backend/src/test/resources/queries/TestNetTypes/table2_ddl.sql new file mode 100644 index 0000000..73fe19e --- /dev/null +++ b/tajo-core/tajo-core-backend/src/test/resources/queries/TestNetTypes/table2_ddl.sql @@ -0,0 +1,4 @@ +-- It is used in TestNetTypes + +create external table table2 (id int, name text, score float, type text, addr inet4) using csv +with ('csvfile.delimiter'='|', 'csvfile.null'='NULL') location ${table.path}; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/334cc84a/tajo-core/tajo-core-backend/src/test/resources/queries/TestNetTypes/testGroupby.sql ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/test/resources/queries/TestNetTypes/testGroupby.sql b/tajo-core/tajo-core-backend/src/test/resources/queries/TestNetTypes/testGroupby.sql new file mode 100644 index 0000000..e3e61d8 --- /dev/null +++ b/tajo-core/tajo-core-backend/src/test/resources/queries/TestNetTypes/testGroupby.sql @@ -0,0 +1 @@ +select name, addr, count(1) from table1 group by name, addr; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/334cc84a/tajo-core/tajo-core-backend/src/test/resources/queries/TestNetTypes/testGroupby2.sql ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/test/resources/queries/TestNetTypes/testGroupby2.sql b/tajo-core/tajo-core-backend/src/test/resources/queries/TestNetTypes/testGroupby2.sql new file mode 100644 index 0000000..c39c3b9 --- /dev/null +++ b/tajo-core/tajo-core-backend/src/test/resources/queries/TestNetTypes/testGroupby2.sql @@ -0,0 +1 @@ +select addr, count(*) from table1 group by addr; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/334cc84a/tajo-core/tajo-core-backend/src/test/resources/queries/TestNetTypes/testJoin.sql ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/test/resources/queries/TestNetTypes/testJoin.sql b/tajo-core/tajo-core-backend/src/test/resources/queries/TestNetTypes/testJoin.sql new file mode 100644 index 0000000..a163c4f --- /dev/null +++ b/tajo-core/tajo-core-backend/src/test/resources/queries/TestNetTypes/testJoin.sql @@ -0,0 +1 @@ +select * from table1 as t1, table2 as t2 where t1.addr = t2.addr; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/334cc84a/tajo-core/tajo-core-backend/src/test/resources/queries/TestNetTypes/testSelect.sql ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/test/resources/queries/TestNetTypes/testSelect.sql b/tajo-core/tajo-core-backend/src/test/resources/queries/TestNetTypes/testSelect.sql new file mode 100644 index 0000000..1b28f06 --- /dev/null +++ b/tajo-core/tajo-core-backend/src/test/resources/queries/TestNetTypes/testSelect.sql @@ -0,0 +1 @@ +select name, addr from table1; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/334cc84a/tajo-core/tajo-core-backend/src/test/resources/queries/TestNetTypes/testSort.sql ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/test/resources/queries/TestNetTypes/testSort.sql b/tajo-core/tajo-core-backend/src/test/resources/queries/TestNetTypes/testSort.sql new file mode 100644 index 0000000..2999a02 --- /dev/null +++ b/tajo-core/tajo-core-backend/src/test/resources/queries/TestNetTypes/testSort.sql @@ -0,0 +1 @@ +select * from table1 order by addr; \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/334cc84a/tajo-core/tajo-core-backend/src/test/resources/results/TestNetTypes/testGroupby.result ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/test/resources/results/TestNetTypes/testGroupby.result b/tajo-core/tajo-core-backend/src/test/resources/results/TestNetTypes/testGroupby.result new file mode 100644 index 0000000..5bedfc2 --- /dev/null +++ b/tajo-core/tajo-core-backend/src/test/resources/results/TestNetTypes/testGroupby.result @@ -0,0 +1,7 @@ +name,addr,?count +------------------------------- +ppp,127.0.1.1,1 +qqq,127.0.0.8,1 +xxx,127.0.1.1,1 +ooo,127.0.0.1,1 +rrr,127.0.0.1,1 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/334cc84a/tajo-core/tajo-core-backend/src/test/resources/results/TestNetTypes/testGroupby2.result ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/test/resources/results/TestNetTypes/testGroupby2.result b/tajo-core/tajo-core-backend/src/test/resources/results/TestNetTypes/testGroupby2.result new file mode 100644 index 0000000..25cd91f --- /dev/null +++ b/tajo-core/tajo-core-backend/src/test/resources/results/TestNetTypes/testGroupby2.result @@ -0,0 +1,5 @@ +addr,?count +------------------------------- +127.0.1.1,2 +127.0.0.1,2 +127.0.0.8,1 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/334cc84a/tajo-core/tajo-core-backend/src/test/resources/results/TestNetTypes/testJoin.result ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/test/resources/results/TestNetTypes/testJoin.result b/tajo-core/tajo-core-backend/src/test/resources/results/TestNetTypes/testJoin.result new file mode 100644 index 0000000..b5817f8 --- /dev/null +++ b/tajo-core/tajo-core-backend/src/test/resources/results/TestNetTypes/testJoin.result @@ -0,0 +1,6 @@ +id,name,score,type,addr,id,name,score,type,addr +------------------------------- +0,,20.0,d,127.0.0.1,1,ooo,1.1,a,127.0.0.1 +1,,0.0,a,127.0.0.8,3,qqq,3.4,c,127.0.0.8 +2,,0.0,b,127.0.0.8,3,qqq,3.4,c,127.0.0.8 +0,,20.0,d,127.0.0.1,4,rrr,4.5,d,127.0.0.1 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/334cc84a/tajo-core/tajo-core-backend/src/test/resources/results/TestNetTypes/testSelect.result ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/test/resources/results/TestNetTypes/testSelect.result b/tajo-core/tajo-core-backend/src/test/resources/results/TestNetTypes/testSelect.result new file mode 100644 index 0000000..2062222 --- /dev/null +++ b/tajo-core/tajo-core-backend/src/test/resources/results/TestNetTypes/testSelect.result @@ -0,0 +1,7 @@ +name,addr +------------------------------- +ooo,127.0.0.1 +ppp,127.0.1.1 +qqq,127.0.0.8 +rrr,127.0.0.1 +xxx,127.0.1.1 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/334cc84a/tajo-core/tajo-core-backend/src/test/resources/results/TestNetTypes/testSort.result ---------------------------------------------------------------------- diff --git a/tajo-core/tajo-core-backend/src/test/resources/results/TestNetTypes/testSort.result b/tajo-core/tajo-core-backend/src/test/resources/results/TestNetTypes/testSort.result new file mode 100644 index 0000000..7988692 --- /dev/null +++ b/tajo-core/tajo-core-backend/src/test/resources/results/TestNetTypes/testSort.result @@ -0,0 +1,7 @@ +id,name,score,type,addr +------------------------------- +1,ooo,1.1,a,127.0.0.1 +4,rrr,4.5,d,127.0.0.1 +3,qqq,3.4,c,127.0.0.8 +2,ppp,2.3,b,127.0.1.1 +5,xxx,5.6,e,127.0.1.1 \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/334cc84a/tajo-docs/src/main/sphinx/functions.rst ---------------------------------------------------------------------- diff --git a/tajo-docs/src/main/sphinx/functions.rst b/tajo-docs/src/main/sphinx/functions.rst index e021f00..fb93d1e 100644 --- a/tajo-docs/src/main/sphinx/functions.rst +++ b/tajo-docs/src/main/sphinx/functions.rst @@ -7,4 +7,5 @@ Functions functions/math_func_and_operators functions/string_func_and_operators - functions/datetime_func_and_operators \ No newline at end of file + functions/datetime_func_and_operators + functions/network_func_and_operators \ No newline at end of file http://git-wip-us.apache.org/repos/asf/tajo/blob/334cc84a/tajo-docs/src/main/sphinx/functions/network_func_and_operators.rst ---------------------------------------------------------------------- diff --git a/tajo-docs/src/main/sphinx/functions/network_func_and_operators.rst b/tajo-docs/src/main/sphinx/functions/network_func_and_operators.rst new file mode 100644 index 0000000..029885d --- /dev/null +++ b/tajo-docs/src/main/sphinx/functions/network_func_and_operators.rst @@ -0,0 +1,57 @@ +******************************* +Network Functions and Operators +******************************* + +.. function:: geoip_country_code (string addr) + + Convert an ipv4 address string to a geoip country code. + + :param string: ipv4 address string + :rtype: text + :example: + + .. code-block:: sql + + select geoip_country_code('163.152.71.31') + > 'KR' + +.. function:: geoip_country_code (inet4 addr) + + Convert an ipv4 address to a geoip country code. + + :param string: ipv4 address + :rtype: text + :example: + + .. code-block:: sql + + select geoip_country_code(163.152.71.31) + > 'KR' + +.. function:: geoip_in_country (string addr, string code) + + If the given country code is same with the country code of the given address, it returns true. Otherwise, returns false. + + :param addr: ipv4 address string + :param code: country code + :rtype: boolean + :example: + + .. code-block:: sql + + select geoip_in_country('163.152.71.31', 'KR') + > true + +.. function:: geoip_in_country (inet4 addr, string code) + + If the given country code is same with the country code of the given address, it returns true. Otherwise, returns false. + + :param addr: ipv4 address + :param code: country code + :rtype: boolean + :example: + + .. code-block:: sql + + select geoip_in_country(163.152.71.31, 'KR') + > true \ No newline at end of file
