>From Ali Alsuliman <[email protected]>:
Ali Alsuliman has uploaded this change for review. (
https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19829 )
Change subject: hash func
......................................................................
hash func
Change-Id: I426bea33d8eec5adfe94dfe1566ee58dccf1e169
---
M
asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java
A
asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/FromBaseDescriptor.java
M
asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java
A
asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/HashBytesStrDescriptor.java
4 files changed, 386 insertions(+), 0 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb
refs/changes/29/19829/1
diff --git
a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java
b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java
index 1250eb9..bffe062 100644
---
a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java
+++
b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/functions/BuiltinFunctions.java
@@ -1307,6 +1307,10 @@
public static final FunctionIdentifier IF_ERROR =
FunctionConstants.newAsterix("if-error", 2);
+ public static final FunctionIdentifier HASHBYTES_STR =
FunctionConstants.newAsterix("hashbytes-str", 2);
+
+ public static final FunctionIdentifier FROM_BASE =
FunctionConstants.newAsterix("from-base", 2);
+
static {
// first, take care of Algebricks builtin functions
addFunction(IS_MISSING, BooleanOnlyTypeComputer.INSTANCE, true);
@@ -2180,6 +2184,8 @@
addPrivateFunction(ACCESS_FIELD, FieldAccessByNameResultType.INSTANCE,
false);
addPrivateFunction(ACCESS_NESTED_FIELD,
FieldAccessNestedResultType.INSTANCE, false);
+ addFunction(HASHBYTES_STR, AStringTypeComputer.INSTANCE_NULLABLE,
true);
+ addFunction(FROM_BASE, AInt64TypeComputer.INSTANCE_NULLABLE, true);
}
static {
diff --git
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/FromBaseDescriptor.java
b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/FromBaseDescriptor.java
new file mode 100644
index 0000000..5385400
--- /dev/null
+++
b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/FromBaseDescriptor.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.runtime.evaluators.functions;
+
+import org.apache.asterix.common.annotations.MissingNullInOutFunction;
+import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider;
+import org.apache.asterix.om.base.AInt64;
+import org.apache.asterix.om.base.AMutableInt32;
+import org.apache.asterix.om.base.AMutableInt64;
+import org.apache.asterix.om.exceptions.ExceptionUtil;
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.BuiltinType;
+import
org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
+import org.apache.asterix.runtime.evaluators.common.ArgumentUtils;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.api.context.IEvaluatorContext;
+import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.data.std.primitive.VoidPointable;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+import org.apache.hyracks.util.string.UTF8StringUtil;
+
+@MissingNullInOutFunction
+public class FromBaseDescriptor extends
AbstractScalarFunctionDynamicDescriptor {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final IFunctionDescriptorFactory FACTORY =
FromBaseDescriptor::new;
+
+ @Override
+ public IScalarEvaluatorFactory createEvaluatorFactory(final
IScalarEvaluatorFactory[] args) {
+ return new IScalarEvaluatorFactory() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IScalarEvaluator createScalarEvaluator(final
IEvaluatorContext ctx) throws HyracksDataException {
+ return new IScalarEvaluator() {
+
+ private final ArrayBackedValueStorage storage = new
ArrayBackedValueStorage();
+ private final StringBuilder strBuilder = new
StringBuilder();
+ private final IScalarEvaluator evalString =
args[0].createScalarEvaluator(ctx);
+ private final IScalarEvaluator evalBase =
args[1].createScalarEvaluator(ctx);
+ private final IPointable argString = new VoidPointable();
+ private final IPointable argNumber = new VoidPointable();
+ private final AMutableInt32 aInt32 = new AMutableInt32(0);
+ private final AMutableInt64 aInt64 = new AMutableInt64(0);
+ @SuppressWarnings("unchecked")
+ private final ISerializerDeserializer<AInt64> int64Serde =
+
SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.AINT64);
+
+ @Override
+ public void evaluate(IFrameTupleReference tuple,
IPointable result) throws HyracksDataException {
+ evalString.evaluate(tuple, argString);
+ evalBase.evaluate(tuple, argNumber);
+
+ if (PointableHelper.checkAndSetMissingOrNull(result,
argString, argNumber)) {
+ return;
+ }
+ byte[] bytes = argNumber.getByteArray();
+ int offset = argNumber.getStartOffset();
+ if (!ArgumentUtils.setInteger(ctx, sourceLoc,
getIdentifier(), 1, bytes, offset, aInt32)) {
+ PointableHelper.setNull(result);
+ return;
+ }
+ int base = aInt32.getIntegerValue();
+ if (base < 2 || base > 36) {
+ PointableHelper.setNull(result);
+ return;
+ }
+
+ bytes = argString.getByteArray();
+ offset = argString.getStartOffset();
+ if (bytes[offset] !=
ATypeTag.SERIALIZED_STRING_TYPE_TAG) {
+ PointableHelper.setNull(result);
+ ExceptionUtil.warnTypeMismatch(ctx, sourceLoc,
getIdentifier(), bytes[offset], 0,
+ ATypeTag.STRING);
+ return;
+ }
+ strBuilder.setLength(0);
+ UTF8StringUtil.toString(strBuilder, bytes, offset + 1);
+
+ try {
+ aInt64.setValue(Long.parseLong(strBuilder, 0,
strBuilder.length(), base));
+ } catch (NumberFormatException e) {
+ PointableHelper.setNull(result);
+ return;
+ }
+ storage.reset();
+ int64Serde.serialize(aInt64, storage.getDataOutput());
+ result.set(storage);
+ }
+ };
+ }
+ };
+ }
+
+ @Override
+ public FunctionIdentifier getIdentifier() {
+ return BuiltinFunctions.FROM_BASE;
+ }
+
+}
diff --git
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/HashBytesStrDescriptor.java
b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/HashBytesStrDescriptor.java
new file mode 100644
index 0000000..bf992cf
--- /dev/null
+++
b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/evaluators/functions/HashBytesStrDescriptor.java
@@ -0,0 +1,242 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.asterix.runtime.evaluators.functions;
+
+import java.io.IOException;
+import java.security.MessageDigest;
+import java.security.NoSuchAlgorithmException;
+
+import org.apache.asterix.common.annotations.MissingNullInOutFunction;
+import org.apache.asterix.dataflow.data.nontagged.printers.PrintTools;
+import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider;
+import org.apache.asterix.om.base.ARecord;
+import org.apache.asterix.om.base.AString;
+import org.apache.asterix.om.base.IAObject;
+import org.apache.asterix.om.exceptions.ExceptionUtil;
+import org.apache.asterix.om.functions.BuiltinFunctions;
+import org.apache.asterix.om.functions.IFunctionDescriptorFactory;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
+import
org.apache.asterix.runtime.evaluators.base.AbstractScalarFunctionDynamicDescriptor;
+import org.apache.asterix.runtime.functions.FunctionTypeInferers;
+import org.apache.asterix.runtime.utils.DescriptorFactoryUtil;
+import org.apache.hyracks.algebricks.core.algebra.functions.FunctionIdentifier;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluator;
+import org.apache.hyracks.algebricks.runtime.base.IScalarEvaluatorFactory;
+import org.apache.hyracks.algebricks.runtime.evaluators.ConstantEvalFactory;
+import org.apache.hyracks.api.context.IEvaluatorContext;
+import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.data.std.primitive.VoidPointable;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.data.std.util.ByteArrayAccessibleDataInputStream;
+import org.apache.hyracks.data.std.util.ByteArrayAccessibleInputStream;
+import org.apache.hyracks.dataflow.common.data.accessors.IFrameTupleReference;
+import org.apache.hyracks.util.bytes.Base64Printer;
+import org.apache.hyracks.util.bytes.HexPrinter;
+import org.apache.hyracks.util.string.UTF8StringWriter;
+
+@MissingNullInOutFunction
+public class HashBytesStrDescriptor extends
AbstractScalarFunctionDynamicDescriptor {
+
+ private static final long serialVersionUID = 1L;
+
+ public static final IFunctionDescriptorFactory FACTORY =
DescriptorFactoryUtil.createFactory(HashBytesStrDescriptor::new,
FunctionTypeInferers.SET_ARGUMENTS_TYPE);
+
+ private ARecordType recType;
+
+ @Override
+ public void setImmutableStates(Object... states) {
+ if (states.length > 1 && states[1] instanceof ARecordType) {
+ this.recType = (ARecordType) states[1];
+ }
+ }
+
+ @Override
+ public IScalarEvaluatorFactory createEvaluatorFactory(final
IScalarEvaluatorFactory[] args) {
+ return new IScalarEvaluatorFactory() {
+ private static final long serialVersionUID = 1L;
+
+ @Override
+ public IScalarEvaluator createScalarEvaluator(final
IEvaluatorContext ctx) throws HyracksDataException {
+ return new IScalarEvaluator() {
+
+ private final IScalarEvaluator eval =
args[0].createScalarEvaluator(ctx);
+ private final IScalarEvaluator optionsEval =
args[1].createScalarEvaluator(ctx);
+ private final ArrayBackedValueStorage storage = new
ArrayBackedValueStorage(43);
+ private final IPointable argPtr = new VoidPointable();
+ private final IPointable optionsPtr = new VoidPointable();
+ private final UTF8StringWriter writer = new
UTF8StringWriter();
+ private final StringBuilder strBuilder = new
StringBuilder(40);
+ private Encoding encoding;
+ private MessageDigest md;
+ {
+ trySetOptions(args[1]);
+ }
+
+ @Override
+ public void evaluate(IFrameTupleReference tuple,
IPointable result) throws HyracksDataException {
+ eval.evaluate(tuple, argPtr);
+ optionsEval.evaluate(tuple, optionsPtr);
+ if (PointableHelper.checkAndSetMissingOrNull(result,
argPtr, optionsPtr)) {
+ return;
+ }
+ byte[] bytes = argPtr.getByteArray();
+ int offset = argPtr.getStartOffset();
+ if (bytes[offset] !=
ATypeTag.SERIALIZED_STRING_TYPE_TAG) {
+ PointableHelper.setNull(result);
+ ExceptionUtil.warnTypeMismatch(ctx, sourceLoc,
getIdentifier(), bytes[offset], 0,
+ ATypeTag.STRING);
+ return;
+ }
+
+ if (md == null || encoding == null) {
+ if (!processOptions(optionsPtr)) {
+ PointableHelper.setNull(result);
+ return;
+ }
+ }
+
+ storage.reset();
+ try {
+ PrintTools.writeUTF8StringRaw(bytes, offset + 1,
argPtr.getLength() - 1,
+ storage.getDataOutput());
+ md.update(storage.getByteArray(),
storage.getStartOffset(), storage.getLength());
+ byte[] digestBytes = md.digest();
+ strBuilder.setLength(0);
+ if (encoding == Encoding.BASE64) {
+ Base64Printer.printBase64Binary(digestBytes,
0, digestBytes.length, strBuilder);
+ } else if (encoding == Encoding.HEX) {
+ HexPrinter.printHexString(digestBytes, 0,
digestBytes.length, strBuilder);
+ } else {
+ PointableHelper.setNull(result);
+ return;
+ }
+ storage.reset();
+
storage.getDataOutput().writeByte(ATypeTag.SERIALIZED_STRING_TYPE_TAG);
+ writer.writeUTF8(strBuilder,
storage.getDataOutput());
+ } catch (IOException e) {
+ throw HyracksDataException.create(e);
+ }
+ result.set(storage);
+ }
+
+ private boolean processOptions(IPointable optionsEval)
throws HyracksDataException {
+ if (recType == null) {
+ return false;
+ }
+ @SuppressWarnings("unchecked")
+ ISerializerDeserializer<ARecord> serde =
+
SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(recType);
+ ARecord rec = serde.deserialize(new
ByteArrayAccessibleDataInputStream(new
ByteArrayAccessibleInputStream(optionsEval.getByteArray(),
optionsEval.getStartOffset(), optionsEval.getLength())));
+ return setOptions(rec);
+ }
+
+ private void trySetOptions(IScalarEvaluatorFactory
factory) throws HyracksDataException {
+ if (!(factory instanceof ConstantEvalFactory) ||
recType == null) {
+ return;
+ }
+ byte[] value = ((ConstantEvalFactory)
factory).getValue();
+ if (value[0] != ATypeTag.SERIALIZED_RECORD_TYPE_TAG) {
+ return;
+ }
+ @SuppressWarnings("unchecked")
+ ISerializerDeserializer<ARecord> serde =
+
SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(recType);
+ ARecord rec = serde.deserialize(new
ByteArrayAccessibleDataInputStream(new ByteArrayAccessibleInputStream(value, 0,
value.length)));
+ setOptions(rec);
+ }
+
+ private boolean setOptions(ARecord rec) throws
HyracksDataException {
+ ARecordType type = rec.getType();
+ if (this.encoding == null) {
+ int encodingIdx = type.getFieldIndex("encoding");
+ if (encodingIdx < 0) {
+ return false;
+ }
+ IAObject encodingVal =
rec.getValueByPos(encodingIdx);
+ if (encodingVal.getType().getTypeTag() !=
ATypeTag.STRING) {
+ return true;
+ }
+ String encodingStr = ((AString)
encodingVal).getStringValue();
+ this.encoding = Encoding.from(encodingStr);
+ if (this.encoding == null) {
+ return false;
+ }
+ }
+ if (md == null) {
+ int algorithmIdx = type.getFieldIndex("algorithm");
+ if (algorithmIdx < 0) {
+ return false;
+ }
+ IAObject algorithmVal =
rec.getValueByPos(algorithmIdx);
+ if (algorithmVal.getType().getTypeTag() !=
ATypeTag.STRING) {
+ return false;
+ }
+ String algorithmStr = ((AString)
algorithmVal).getStringValue();
+ if
(HashAlgorithm.SHA1.toString().equalsIgnoreCase(algorithmStr)) {
+ try {
+ md =
MessageDigest.getInstance(HashAlgorithm.SHA1.algorithmName());
+ } catch (NoSuchAlgorithmException e) {
+ throw HyracksDataException.create(e);
+ }
+ } else {
+ return false;
+ }
+ }
+ return true;
+ }
+ };
+ }
+ };
+ }
+
+ enum HashAlgorithm {
+ SHA1, SHA256, SHA512;
+
+ String algorithmName() {
+ return switch (this) {
+ case SHA1 -> "SHA-1";
+ case SHA256 -> "SHA-256";
+ case SHA512 -> "SHA-512";
+ };
+ }
+ }
+
+ enum Encoding {
+ HEX, BASE64;
+
+ static Encoding from(String encodingStr) {
+ if (HEX.toString().equalsIgnoreCase(encodingStr)) {
+ return HEX;
+ } else if (BASE64.toString().equalsIgnoreCase(encodingStr)) {
+ return BASE64;
+ }
+ return null;
+ }
+ }
+
+ @Override
+ public FunctionIdentifier getIdentifier() {
+ return BuiltinFunctions.HASHBYTES_STR;
+ }
+
+}
diff --git
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java
b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java
index f6ef58f..eb8b01f 100644
---
a/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java
+++
b/asterixdb/asterix-runtime/src/main/java/org/apache/asterix/runtime/functions/FunctionCollection.java
@@ -382,6 +382,7 @@
import org.apache.asterix.runtime.evaluators.functions.CreateUUIDDescriptor;
import
org.apache.asterix.runtime.evaluators.functions.DecodeDataverseNameDescriptor;
import org.apache.asterix.runtime.evaluators.functions.DeepEqualityDescriptor;
+import org.apache.asterix.runtime.evaluators.functions.FromBaseDescriptor;
import
org.apache.asterix.runtime.evaluators.functions.FullTextContainsFunctionDescriptor;
import
org.apache.asterix.runtime.evaluators.functions.FullTextContainsWithoutOptionFunctionDescriptor;
import
org.apache.asterix.runtime.evaluators.functions.GetIntersectionDescriptor;
@@ -464,6 +465,7 @@
import
org.apache.asterix.runtime.evaluators.functions.RandomWithSeedDescriptor;
import org.apache.asterix.runtime.evaluators.functions.ReferenceTileDescriptor;
import
org.apache.asterix.runtime.evaluators.functions.SerializedSizeDescriptor;
+import org.apache.asterix.runtime.evaluators.functions.HashBytesStrDescriptor;
import org.apache.asterix.runtime.evaluators.functions.SleepDescriptor;
import org.apache.asterix.runtime.evaluators.functions.SpatialAreaDescriptor;
import org.apache.asterix.runtime.evaluators.functions.SpatialCellDescriptor;
@@ -1363,6 +1365,8 @@
fc.add(SerializedSizeDescriptor.FACTORY);
fc.add(PutAutogeneratedKeyDescriptor.FACTORY);
fc.add(IfErrorDescriptor.FACTORY);
+ fc.add(HashBytesStrDescriptor.FACTORY);
+ fc.add(FromBaseDescriptor.FACTORY);
ServiceLoader.load(IFunctionRegistrant.class).iterator().forEachRemaining(c ->
c.register(fc));
return fc;
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19829
To unsubscribe, or for help writing mail filters, visit
https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: ionic
Gerrit-Change-Id: I426bea33d8eec5adfe94dfe1566ee58dccf1e169
Gerrit-Change-Number: 19829
Gerrit-PatchSet: 1
Gerrit-Owner: Ali Alsuliman <[email protected]>
Gerrit-MessageType: newchange
