This is an automated email from the ASF dual-hosted git repository. mblow pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/asterixdb.git
commit fc21639e64e6ac1089587ba21849d0b607c69f19 Author: Ali Alsuliman <[email protected]> AuthorDate: Thu May 29 15:14:48 2025 -0700 [ASTERIXDB-3614][FUN] Add sha1() and from_base() functions - user model changes: no - storage format changes: no - interface changes: no Details: sha1_hex(text string) -> string: Returns the sha1 of the input text in hex string. sha1_base64(text string) -> string: Returns the sha1 of the input text in base64 string. from_base(num string, radix int) -> bigint: Returns the value of num string interpreted as a base-radix number. Ext-ref: MB-66996 Change-Id: Iebabd219ffa2f3c1fd0090996cab9440c294701c Reviewed-on: https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19827 Tested-by: Ali Alsuliman <[email protected]> Integration-Tests: Jenkins <[email protected]> Reviewed-by: Peeyush Gupta <[email protected]> --- .../queries_sqlpp/crypto/sha1/sha1.01.ddl.sqlpp | 25 ++++ .../queries_sqlpp/crypto/sha1/sha1.02.update.sqlpp | 29 +++++ .../queries_sqlpp/crypto/sha1/sha1.03.query.sqlpp | 24 ++++ .../queries_sqlpp/crypto/sha1/sha1.04.query.sqlpp | 24 ++++ .../queries_sqlpp/crypto/sha1/sha1.99.ddl.sqlpp | 20 ++++ .../numeric/from_base/from_base.01.ddl.sqlpp | 25 ++++ .../numeric/from_base/from_base.02.update.sqlpp | 31 +++++ .../numeric/from_base/from_base.03.query.sqlpp | 24 ++++ .../numeric/from_base/from_base.04.query.sqlpp | 24 ++++ .../numeric/from_base/from_base.05.query.sqlpp | 21 ++++ .../numeric/from_base/from_base.99.ddl.sqlpp | 20 ++++ .../runtimets/results/crypto/sha1/sha1.03.adm | 6 + .../runtimets/results/crypto/sha1/sha1.04.adm | 1 + .../results/numeric/from_base/from_base.03.adm | 8 ++ .../results/numeric/from_base/from_base.04.adm | 1 + .../results/numeric/from_base/from_base.05.adm | 1 + .../src/test/resources/runtimets/sqlpp_queries.xml | 12 ++ .../asterix/om/exceptions/ExceptionUtil.java | 9 ++ .../asterix/om/functions/BuiltinFunctions.java | 11 ++ .../evaluators/functions/FromBaseDescriptor.java | 127 +++++++++++++++++++++ .../evaluators/functions/Sha1Base64Descriptor.java | 57 +++++++++ .../evaluators/functions/Sha1HexDescriptor.java | 56 +++++++++ .../evaluators/functions/Sha1StringEvaluator.java | 111 ++++++++++++++++++ .../runtime/functions/FunctionCollection.java | 8 ++ 24 files changed, 675 insertions(+) diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/crypto/sha1/sha1.01.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/crypto/sha1/sha1.01.ddl.sqlpp new file mode 100644 index 0000000000..1a3bfb428f --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/crypto/sha1/sha1.01.ddl.sqlpp @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +DROP DATAVERSE test IF EXISTS; +CREATE DATAVERSE test; +USE test; + +CREATE TYPE t1 IF NOT EXISTS AS {id: int, f: string?}; +CREATE DATASET ds1(t1) IF NOT EXISTS PRIMARY KEY id; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/crypto/sha1/sha1.02.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/crypto/sha1/sha1.02.update.sqlpp new file mode 100644 index 0000000000..dbf6daff99 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/crypto/sha1/sha1.02.update.sqlpp @@ -0,0 +1,29 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +USE test; + +UPSERT INTO ds1 ([ +{"id": 1, "f": "asterix", "f2": "asterix"}, +{"id": 2, "f": "𩸽", "f2": "𩸽"}, +{"id": 3, "f": "a𩸽b", "f2": "a𩸽b"}, +{"id": 4, "f": null, "f2": null}, +{"id": 5}, +{"id": 6, "f2": 1} +]); \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/crypto/sha1/sha1.03.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/crypto/sha1/sha1.03.query.sqlpp new file mode 100644 index 0000000000..e00b33e5c8 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/crypto/sha1/sha1.03.query.sqlpp @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +USE test; + +SELECT sha1_hex(f) a, sha1_hex(f2) b, sha1_base64(f) c, sha1_base64(f2) d +FROM ds1 +ORDER BY ds1.id; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/crypto/sha1/sha1.04.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/crypto/sha1/sha1.04.query.sqlpp new file mode 100644 index 0000000000..3f00c240ba --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/crypto/sha1/sha1.04.query.sqlpp @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +USE test; + +SELECT sha1_hex("asterix") a, sha1_hex("𩸽") b, sha1_hex("a𩸽b") c, sha1_hex(null) d, + sha1_base64("asterix") e, sha1_base64("𩸽") f, sha1_base64("a𩸽b") g, sha1_base64(null) h, + sha1_hex(missing) i, sha1_base64(missing) j, sha1_hex(1) k, sha1_base64(1) l; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/crypto/sha1/sha1.99.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/crypto/sha1/sha1.99.ddl.sqlpp new file mode 100644 index 0000000000..36b2bab543 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/crypto/sha1/sha1.99.ddl.sqlpp @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +DROP DATAVERSE test IF EXISTS; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/numeric/from_base/from_base.01.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/numeric/from_base/from_base.01.ddl.sqlpp new file mode 100644 index 0000000000..1a3bfb428f --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/numeric/from_base/from_base.01.ddl.sqlpp @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +DROP DATAVERSE test IF EXISTS; +CREATE DATAVERSE test; +USE test; + +CREATE TYPE t1 IF NOT EXISTS AS {id: int, f: string?}; +CREATE DATASET ds1(t1) IF NOT EXISTS PRIMARY KEY id; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/numeric/from_base/from_base.02.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/numeric/from_base/from_base.02.update.sqlpp new file mode 100644 index 0000000000..0eeb0285b6 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/numeric/from_base/from_base.02.update.sqlpp @@ -0,0 +1,31 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +// testing base 16 +USE test; + +UPSERT INTO ds1 ([ +{"id": 1, "f": "1", "f2": "1"}, // 1 +{"id": 2, "f": "A", "f2": "A"}, // 10 +{"id": 3, "f": "10", "f2": "10"}, // 16 +{"id": 4, "f": null, "f2": null}, +{"id": 5}, +{"id": 6, "f": "35BC6C28BCA27FB", "f2": "35BC6C28BCA27FB"}, // 242005543865690107 +{"id": 7, "f": "35BC6C28BCA27FBECC144761D32D09986876438F", "f2": "35BC6C28BCA27FBECC144761D32D09986876438F"}, // does not fit in a 64-bit integer +{"id": 8, "f2": 1} // should return null for non-string inputs +]); diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/numeric/from_base/from_base.03.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/numeric/from_base/from_base.03.query.sqlpp new file mode 100644 index 0000000000..c4e4b7b30b --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/numeric/from_base/from_base.03.query.sqlpp @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +USE test; + +SELECT from_base(f, 16) a, from_base(f2, 16) b +FROM ds1 +ORDER BY ds1.id; diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/numeric/from_base/from_base.04.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/numeric/from_base/from_base.04.query.sqlpp new file mode 100644 index 0000000000..bbd27aa3b2 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/numeric/from_base/from_base.04.query.sqlpp @@ -0,0 +1,24 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +SELECT from_base("1", 16) a, from_base("A", 16) b, from_base("10", 16) c, from_base("35BC6C28BCA27FB", 16) d, + from_base("35BC6C28BCA27FBECC144761D32D09986876438F", 16) e, + from_base(null, 16) f, from_base(missing, 16) g, + from_base("A", null) h, from_base("A", missing) i, + from_base("A", -1) j, from_base("A", 1) k, from_base("A", 46) l, from_base(1, 16) m; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/numeric/from_base/from_base.05.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/numeric/from_base/from_base.05.query.sqlpp new file mode 100644 index 0000000000..94f5707d77 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/numeric/from_base/from_base.05.query.sqlpp @@ -0,0 +1,21 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +SELECT from_base("1", 10) a, from_base("10", 10) b, from_base("A", 10) c, + from_base("1", 2) d, from_base("10", 2) e, from_base("11", 2) f; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/numeric/from_base/from_base.99.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/numeric/from_base/from_base.99.ddl.sqlpp new file mode 100644 index 0000000000..36b2bab543 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/numeric/from_base/from_base.99.ddl.sqlpp @@ -0,0 +1,20 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +DROP DATAVERSE test IF EXISTS; \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/crypto/sha1/sha1.03.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/crypto/sha1/sha1.03.adm new file mode 100644 index 0000000000..9063191e72 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/crypto/sha1/sha1.03.adm @@ -0,0 +1,6 @@ +{ "a": "35BC6C28BCA27FBECC144761D32D09986876438F", "b": "35BC6C28BCA27FBECC144761D32D09986876438F", "c": "NbxsKLyif77MFEdh0y0JmGh2Q48=", "d": "NbxsKLyif77MFEdh0y0JmGh2Q48=" } +{ "a": "93315873BA703F909C19493045450FE0FDF84AD7", "b": "93315873BA703F909C19493045450FE0FDF84AD7", "c": "kzFYc7pwP5CcGUkwRUUP4P34Stc=", "d": "kzFYc7pwP5CcGUkwRUUP4P34Stc=" } +{ "a": "AA20F3C74E59734035C2B11EF4C90F3E8521DC9E", "b": "AA20F3C74E59734035C2B11EF4C90F3E8521DC9E", "c": "qiDzx05Zc0A1wrEe9MkPPoUh3J4=", "d": "qiDzx05Zc0A1wrEe9MkPPoUh3J4=" } +{ "a": null, "b": null, "c": null, "d": null } +{ } +{ "b": null, "d": null } \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/crypto/sha1/sha1.04.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/crypto/sha1/sha1.04.adm new file mode 100644 index 0000000000..5f77732b09 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/crypto/sha1/sha1.04.adm @@ -0,0 +1 @@ +{ "a": "35BC6C28BCA27FBECC144761D32D09986876438F", "b": "93315873BA703F909C19493045450FE0FDF84AD7", "c": "AA20F3C74E59734035C2B11EF4C90F3E8521DC9E", "d": null, "e": "NbxsKLyif77MFEdh0y0JmGh2Q48=", "f": "kzFYc7pwP5CcGUkwRUUP4P34Stc=", "g": "qiDzx05Zc0A1wrEe9MkPPoUh3J4=", "h": null, "k": null, "l": null } \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/numeric/from_base/from_base.03.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/numeric/from_base/from_base.03.adm new file mode 100644 index 0000000000..9b33be80a3 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/numeric/from_base/from_base.03.adm @@ -0,0 +1,8 @@ +{ "a": 1, "b": 1 } +{ "a": 10, "b": 10 } +{ "a": 16, "b": 16 } +{ "a": null, "b": null } +{ } +{ "a": 242005543865690107, "b": 242005543865690107 } +{ "a": null, "b": null } +{ "b": null } diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/numeric/from_base/from_base.04.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/numeric/from_base/from_base.04.adm new file mode 100644 index 0000000000..8272596ef8 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/numeric/from_base/from_base.04.adm @@ -0,0 +1 @@ +{ "a": 1, "b": 10, "c": 16, "d": 242005543865690107, "e": null, "f": null, "h": null, "j": null, "k": null, "l": null, "m": null } \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/numeric/from_base/from_base.05.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/numeric/from_base/from_base.05.adm new file mode 100644 index 0000000000..375bff5de8 --- /dev/null +++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/numeric/from_base/from_base.05.adm @@ -0,0 +1 @@ +{ "a": 1, "b": 10, "c": null, "d": 1, "e": 2, "f": 3 } \ No newline at end of file diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml b/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml index d216c2d9e4..5db093b5aa 100644 --- a/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml +++ b/asterixdb/asterix-app/src/test/resources/runtimets/sqlpp_queries.xml @@ -4019,6 +4019,13 @@ </compilation-unit> </test-case> </test-group> + <test-group name="crypto"> + <test-case FilePath="crypto"> + <compilation-unit name="sha1"> + <output-dir compare="Text">sha1</output-dir> + </compilation-unit> + </test-case> + </test-group> <test-group name="custord"> <!-- <test-case FilePath="custord"> @@ -9657,6 +9664,11 @@ <output-dir compare="Text">unary-minus_double_02</output-dir> </compilation-unit> </test-case> + <test-case FilePath="numeric"> + <compilation-unit name="from_base"> + <output-dir compare="Text">from_base</output-dir> + </compilation-unit> + </test-case> </test-group> <test-group name="open-closed"> <!-- diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/exceptions/ExceptionUtil.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/exceptions/ExceptionUtil.java index c6cb707af6..24909d71e7 100644 --- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/exceptions/ExceptionUtil.java +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/exceptions/ExceptionUtil.java @@ -93,6 +93,15 @@ public final class ExceptionUtil { } } + public static void warnValueOutOfRange(IEvaluatorContext ctx, SourceLocation srcLoc, FunctionIdentifier fid, + int inputPosition, int lowerBound, int upperBound, int actual) { + IWarningCollector warningCollector = ctx.getWarningCollector(); + if (warningCollector.shouldWarn()) { + warningCollector.warn(Warning.of(srcLoc, ErrorCode.VALUE_OUT_OF_RANGE, fid, + ExceptionUtil.indexToPosition(inputPosition), lowerBound, upperBound, actual)); + } + } + public static void warnTypeMismatch(IEvaluatorContext ctx, SourceLocation srcLoc, FunctionIdentifier fid, byte actualType, int argIdx, ATypeTag expectedType) { warnTypeMismatch(ctx, srcLoc, fid, actualType, argIdx, expectedType::toString); diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java index 1250eb91f7..790f0400e3 100644 --- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java +++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java @@ -1307,6 +1307,12 @@ public class BuiltinFunctions { public static final FunctionIdentifier IF_ERROR = FunctionConstants.newAsterix("if-error", 2); + // crypto + public static final FunctionIdentifier SHA1_HEX = FunctionConstants.newAsterix("sha1-hex", 1); + public static final FunctionIdentifier SHA1_BASE64 = FunctionConstants.newAsterix("sha1-base64", 1); + + public static final FunctionIdentifier FROM_BASE = FunctionConstants.newAsterix("from-base", 2); + static { // first, take care of Algebricks builtin functions addFunction(IS_MISSING, BooleanOnlyTypeComputer.INSTANCE, true); @@ -2180,6 +2186,11 @@ public class BuiltinFunctions { addPrivateFunction(ACCESS_FIELD, FieldAccessByNameResultType.INSTANCE, false); addPrivateFunction(ACCESS_NESTED_FIELD, FieldAccessNestedResultType.INSTANCE, false); + // crypto + addFunction(SHA1_HEX, AStringTypeComputer.INSTANCE_NULLABLE, true); + addFunction(SHA1_BASE64, AStringTypeComputer.INSTANCE_NULLABLE, true); + + addFunction(FROM_BASE, AInt64TypeComputer.INSTANCE_NULLABLE, true); } static { diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/FromBaseDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/FromBaseDescriptor.java new file mode 100644 index 0000000000..fb57fb3159 --- /dev/null +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/FromBaseDescriptor.java @@ -0,0 +1,127 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.asterix.runtime.evaluators.functions; + +import org.apache.asterix.common.annotations.MissingNullInOutFunction; +import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider; +import org.apache.asterix.om.base.AInt64; +import org.apache.asterix.om.base.AMutableInt32; +import org.apache.asterix.om.base.AMutableInt64; +import org.apache.asterix.om.exceptions.ExceptionUtil; +import org.apache.asterix.om.functions.BuiltinFunctions; +import org.apache.asterix.om.functions.IFunctionDescriptorFactory; +import org.apache.asterix.om.types.ATypeTag; +import org.apache.asterix.om.types.BuiltinType; +import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor; +import org.apache.asterix.runtime.evaluators.common.ArgumentUtils; +import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier; +import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator; +import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory; +import org.apache.hyracks.api.context.IEvaluatorContext; +import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer; +import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.data.std.api.IPointable; +import org.apache.hyracks.data.std.primitive.VoidPointable; +import org.apache.hyracks.data.std.util.ArrayBackedValueStorage; +import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference; +import org.apache.hyracks.util.string.UTF8StringUtil; + +@MissingNullInOutFunction +public class FromBaseDescriptor extends AbstractScalarFunctionDynamicDescriptor { + + private static final long serialVersionUID = 1L; + + public static final IFunctionDescriptorFactory FACTORY = FromBaseDescriptor::new; + + @Override + public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) { + return new IScalarEvaluatorFactory() { + private static final long serialVersionUID = 1L; + + @Override + public IScalarEvaluator createScalarEvaluator(final IEvaluatorContext ctx) throws HyracksDataException { + return new IScalarEvaluator() { + + private final ArrayBackedValueStorage storage = new ArrayBackedValueStorage(); + private final StringBuilder strBuilder = new StringBuilder(); + private final IScalarEvaluator evalString = args[0].createScalarEvaluator(ctx); + private final IScalarEvaluator evalBase = args[1].createScalarEvaluator(ctx); + private final IPointable argString = new VoidPointable(); + private final IPointable argNumber = new VoidPointable(); + private final AMutableInt32 aInt32 = new AMutableInt32(0); + private final AMutableInt64 aInt64 = new AMutableInt64(0); + @SuppressWarnings("unchecked") + private final ISerializerDeserializer<AInt64> int64Serde = + SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.AINT64); + + @Override + public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException { + evalString.evaluate(tuple, argString); + evalBase.evaluate(tuple, argNumber); + + if (PointableHelper.checkAndSetMissingOrNull(result, argString, argNumber)) { + return; + } + byte[] bytes = argNumber.getByteArray(); + int offset = argNumber.getStartOffset(); + if (!ArgumentUtils.setInteger(ctx, sourceLoc, getIdentifier(), 1, bytes, offset, aInt32)) { + PointableHelper.setNull(result); + return; + } + int base = aInt32.getIntegerValue(); + if (base < 2 || base > 36) { + PointableHelper.setNull(result); + ExceptionUtil.warnValueOutOfRange(ctx, sourceLoc, getIdentifier(), 1, 2, 36, base); + return; + } + + bytes = argString.getByteArray(); + offset = argString.getStartOffset(); + if (bytes[offset] != ATypeTag.SERIALIZED_STRING_TYPE_TAG) { + PointableHelper.setNull(result); + ExceptionUtil.warnTypeMismatch(ctx, sourceLoc, getIdentifier(), bytes[offset], 0, + ATypeTag.STRING); + return; + } + strBuilder.setLength(0); + UTF8StringUtil.toString(strBuilder, bytes, offset + 1); + + try { + aInt64.setValue(Long.parseLong(strBuilder, 0, strBuilder.length(), base)); + } catch (NumberFormatException e) { + PointableHelper.setNull(result); + ExceptionUtil.warnFunctionEvalFailed(ctx, sourceLoc, getIdentifier(), e.getMessage()); + return; + } + storage.reset(); + int64Serde.serialize(aInt64, storage.getDataOutput()); + result.set(storage); + } + }; + } + }; + } + + @Override + public FunctionIdentifier getIdentifier() { + return BuiltinFunctions.FROM_BASE; + } + +} diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/Sha1Base64Descriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/Sha1Base64Descriptor.java new file mode 100644 index 0000000000..fb9eef68d3 --- /dev/null +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/Sha1Base64Descriptor.java @@ -0,0 +1,57 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.asterix.runtime.evaluators.functions; + +import org.apache.asterix.common.annotations.MissingNullInOutFunction; +import org.apache.asterix.om.functions.BuiltinFunctions; +import org.apache.asterix.om.functions.IFunctionDescriptorFactory; +import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor; +import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier; +import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator; +import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory; +import org.apache.hyracks.api.context.IEvaluatorContext; +import org.apache.hyracks.api.exceptions.HyracksDataException; + +@MissingNullInOutFunction +public class Sha1Base64Descriptor extends AbstractScalarFunctionDynamicDescriptor { + + private static final long serialVersionUID = 1L; + + public static final IFunctionDescriptorFactory FACTORY = Sha1Base64Descriptor::new; + + @Override + public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) { + return new IScalarEvaluatorFactory() { + private static final long serialVersionUID = 1L; + + @Override + public IScalarEvaluator createScalarEvaluator(final IEvaluatorContext ctx) throws HyracksDataException { + return new Sha1StringEvaluator(args, ctx, sourceLoc, getIdentifier(), + Sha1StringEvaluator.Encoding.BASE64); + } + }; + } + + @Override + public FunctionIdentifier getIdentifier() { + return BuiltinFunctions.SHA1_BASE64; + } + +} diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/Sha1HexDescriptor.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/Sha1HexDescriptor.java new file mode 100644 index 0000000000..bf3449edcd --- /dev/null +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/Sha1HexDescriptor.java @@ -0,0 +1,56 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.asterix.runtime.evaluators.functions; + +import org.apache.asterix.common.annotations.MissingNullInOutFunction; +import org.apache.asterix.om.functions.BuiltinFunctions; +import org.apache.asterix.om.functions.IFunctionDescriptorFactory; +import org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor; +import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier; +import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator; +import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory; +import org.apache.hyracks.api.context.IEvaluatorContext; +import org.apache.hyracks.api.exceptions.HyracksDataException; + +@MissingNullInOutFunction +public class Sha1HexDescriptor extends AbstractScalarFunctionDynamicDescriptor { + + private static final long serialVersionUID = 1L; + + public static final IFunctionDescriptorFactory FACTORY = Sha1HexDescriptor::new; + + @Override + public IScalarEvaluatorFactory createEvaluatorFactory(final IScalarEvaluatorFactory[] args) { + return new IScalarEvaluatorFactory() { + private static final long serialVersionUID = 1L; + + @Override + public IScalarEvaluator createScalarEvaluator(final IEvaluatorContext ctx) throws HyracksDataException { + return new Sha1StringEvaluator(args, ctx, sourceLoc, getIdentifier(), Sha1StringEvaluator.Encoding.HEX); + } + }; + } + + @Override + public FunctionIdentifier getIdentifier() { + return BuiltinFunctions.SHA1_HEX; + } + +} diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/Sha1StringEvaluator.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/Sha1StringEvaluator.java new file mode 100755 index 0000000000..3bbf125481 --- /dev/null +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/Sha1StringEvaluator.java @@ -0,0 +1,111 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ +package org.apache.asterix.runtime.evaluators.functions; + +import java.io.IOException; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +import org.apache.asterix.dataflow.data.nontagged.printers.PrintTools; +import org.apache.asterix.om.exceptions.ExceptionUtil; +import org.apache.asterix.om.types.ATypeTag; +import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier; +import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator; +import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory; +import org.apache.hyracks.api.context.IEvaluatorContext; +import org.apache.hyracks.api.exceptions.HyracksDataException; +import org.apache.hyracks.api.exceptions.SourceLocation; +import org.apache.hyracks.data.std.api.IPointable; +import org.apache.hyracks.data.std.primitive.VoidPointable; +import org.apache.hyracks.data.std.util.ArrayBackedValueStorage; +import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference; +import org.apache.hyracks.util.bytes.Base64Printer; +import org.apache.hyracks.util.bytes.HexPrinter; +import org.apache.hyracks.util.string.UTF8StringWriter; + +public class Sha1StringEvaluator extends AbstractScalarEval { + + private final IScalarEvaluator eval; + private final ArrayBackedValueStorage storage; + private final IPointable argPtr; + private final UTF8StringWriter writer; + private final StringBuilder strBuilder; + private final MessageDigest md; + private final IEvaluatorContext ctx; + private final Encoding encoding; + + Sha1StringEvaluator(IScalarEvaluatorFactory[] args, IEvaluatorContext ctx, SourceLocation srcLoc, + FunctionIdentifier funId, Encoding encoding) throws HyracksDataException { + super(srcLoc, funId); + this.ctx = ctx; + this.encoding = encoding; + storage = new ArrayBackedValueStorage(43); + eval = args[0].createScalarEvaluator(ctx); + strBuilder = new StringBuilder(40); + writer = new UTF8StringWriter(); + argPtr = new VoidPointable(); + try { + md = MessageDigest.getInstance("SHA-1"); + } catch (NoSuchAlgorithmException e) { + throw HyracksDataException.create(e); + } + } + + @Override + public void evaluate(IFrameTupleReference tuple, IPointable result) throws HyracksDataException { + eval.evaluate(tuple, argPtr); + if (PointableHelper.checkAndSetMissingOrNull(result, argPtr)) { + return; + } + byte[] bytes = argPtr.getByteArray(); + int offset = argPtr.getStartOffset(); + if (bytes[offset] != ATypeTag.SERIALIZED_STRING_TYPE_TAG) { + PointableHelper.setNull(result); + ExceptionUtil.warnTypeMismatch(ctx, srcLoc, funID, bytes[offset], 0, ATypeTag.STRING); + return; + } + + storage.reset(); + try { + // convert modified UTF-8 bytes to UTF-8 bytes + PrintTools.writeUTF8StringRaw(bytes, offset + 1, argPtr.getLength() - 1, storage.getDataOutput()); + md.update(storage.getByteArray(), storage.getStartOffset(), storage.getLength()); + byte[] digestBytes = md.digest(); + strBuilder.setLength(0); + // convert UTF-8 bytes to hex or base64 string + if (encoding == Encoding.HEX) { + HexPrinter.printHexString(digestBytes, 0, digestBytes.length, strBuilder); + } else { + Base64Printer.printBase64Binary(digestBytes, 0, digestBytes.length, strBuilder); + } + // serialize the encoded string as modified UTF-8 + storage.reset(); + storage.getDataOutput().writeByte(ATypeTag.SERIALIZED_STRING_TYPE_TAG); + writer.writeUTF8(strBuilder, storage.getDataOutput()); + } catch (IOException e) { + throw HyracksDataException.create(e); + } + result.set(storage); + } + + enum Encoding { + HEX, + BASE64; + } +} diff --git a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java index f6ef58f9fc..b4710de7c5 100644 --- a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java +++ b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java @@ -382,6 +382,7 @@ import org.apache.asterix.runtime.evaluators.functions.CreateRectangleDescriptor import org.apache.asterix.runtime.evaluators.functions.CreateUUIDDescriptor; import org.apache.asterix.runtime.evaluators.functions.DecodeDataverseNameDescriptor; import org.apache.asterix.runtime.evaluators.functions.DeepEqualityDescriptor; +import org.apache.asterix.runtime.evaluators.functions.FromBaseDescriptor; import org.apache.asterix.runtime.evaluators.functions.FullTextContainsFunctionDescriptor; import org.apache.asterix.runtime.evaluators.functions.FullTextContainsWithoutOptionFunctionDescriptor; import org.apache.asterix.runtime.evaluators.functions.GetIntersectionDescriptor; @@ -464,6 +465,8 @@ import org.apache.asterix.runtime.evaluators.functions.RandomDescriptor; import org.apache.asterix.runtime.evaluators.functions.RandomWithSeedDescriptor; import org.apache.asterix.runtime.evaluators.functions.ReferenceTileDescriptor; import org.apache.asterix.runtime.evaluators.functions.SerializedSizeDescriptor; +import org.apache.asterix.runtime.evaluators.functions.Sha1Base64Descriptor; +import org.apache.asterix.runtime.evaluators.functions.Sha1HexDescriptor; import org.apache.asterix.runtime.evaluators.functions.SleepDescriptor; import org.apache.asterix.runtime.evaluators.functions.SpatialAreaDescriptor; import org.apache.asterix.runtime.evaluators.functions.SpatialCellDescriptor; @@ -1056,6 +1059,7 @@ public final class FunctionCollection implements IFunctionCollection { fc.add(NumericSignDescriptor.FACTORY); fc.add(NumericTruncDescriptor.FACTORY); fc.add(NumericATan2Descriptor.FACTORY); + fc.add(FromBaseDescriptor.FACTORY); // Comparisons. fc.add(EqualsDescriptor.FACTORY); @@ -1357,6 +1361,10 @@ public final class FunctionCollection implements IFunctionCollection { // Record function fc.add(RecordPairsDescriptor.FACTORY); + // Crypto functions + fc.add(Sha1HexDescriptor.FACTORY); + fc.add(Sha1Base64Descriptor.FACTORY); + // Other functions fc.add(DecodeDataverseNameDescriptor.FACTORY); fc.add(RandomWithSeedDescriptor.FACTORY);
