This is an automated email from the ASF dual-hosted git repository.
adelapena pushed a commit to branch cassandra-5.0
in repository https://gitbox.apache.org/repos/asf/cassandra.git
The following commit(s) were added to refs/heads/cassandra-5.0 by this push:
new 3b9881bfa6 Fix vector type to support DDM's mask_default function
3b9881bfa6 is described below
commit 3b9881bfa65d2806e3918ba2c73198cb17c7071b
Author: Andrés de la Peña <[email protected]>
AuthorDate: Wed Sep 27 21:56:15 2023 +0100
Fix vector type to support DDM's mask_default function
patch by Andrés de la Peña; reviewed by Berenguer Blasi and Maxwell Guo for
CASSANDRA-18889
---
CHANGES.txt | 1 +
.../cassandra/examples/CQL/ddm_create_table_with_udf.cql | 2 +-
doc/modules/cassandra/partials/masking_functions.adoc | 11 ++++++++++-
src/java/org/apache/cassandra/cql3/CQL3Type.java | 16 ++++++++++++++++
src/java/org/apache/cassandra/db/marshal/VectorType.java | 13 +++++++++++++
.../masking/ColumnMaskQueryWithDefaultTest.java | 9 ++++++++-
.../masking/ColumnMaskQueryWithReplaceTest.java | 7 +++++++
.../cassandra/cql3/functions/masking/ColumnMaskTest.java | 16 ++++++++++++++++
.../cql3/functions/masking/MaskingFunctionTester.java | 16 ++++++++++++++++
.../functions/masking/ReplaceMaskingFunctionTest.java | 5 +++--
.../apache/cassandra/db/marshal/AbstractTypeTest.java | 16 +++++++++++++++-
11 files changed, 106 insertions(+), 6 deletions(-)
diff --git a/CHANGES.txt b/CHANGES.txt
index 66f613d26d..1e0fe54855 100644
--- a/CHANGES.txt
+++ b/CHANGES.txt
@@ -1,4 +1,5 @@
5.0-alpha2
+ * Fix vector type to support DDM's mask_default function (CASSANDRA-18889)
* Remove unnecessary reporter-config3 dependency (CASSANDRA-18907)
* Remove support for empty values on the vector data type (CASSANDRA-18876)
* Upgrade Dropwizard Metrics to 4.2.19 (CASSANDRA-14667)
diff --git a/doc/modules/cassandra/examples/CQL/ddm_create_table_with_udf.cql
b/doc/modules/cassandra/examples/CQL/ddm_create_table_with_udf.cql
index 23db48c1b5..85d6555f3a 100644
--- a/doc/modules/cassandra/examples/CQL/ddm_create_table_with_udf.cql
+++ b/doc/modules/cassandra/examples/CQL/ddm_create_table_with_udf.cql
@@ -2,7 +2,7 @@ CREATE FUNCTION redact(input text)
CALLED ON NULL INPUT
RETURNS text
LANGUAGE java
- AS 'return "redacted";
+ AS 'return "redacted";';
CREATE TABLE patients (
id timeuuid PRIMARY KEY,
diff --git a/doc/modules/cassandra/partials/masking_functions.adoc
b/doc/modules/cassandra/partials/masking_functions.adoc
index 43fb25c38d..a91f168067 100644
--- a/doc/modules/cassandra/partials/masking_functions.adoc
+++ b/doc/modules/cassandra/partials/masking_functions.adoc
@@ -10,7 +10,12 @@ Examples:
`mask_null(123)` -> `null`
-| `mask_default(value)` | Replaces its argument by an arbitrary, fixed default
value of the same type. This will be `\***\***` for text values, zero for
numeric values, `false` for booleans, etc.
+| `mask_default(value)` | Replaces its argument by an arbitrary, fixed default
value of the same type.
+This will be `\***\***` for text values, zero for numeric values, `false` for
booleans, etc.
+
+Variable-length multivalued types such as lists, sets and maps are masked as
empty collections.
+
+Fixed-length multivalued types such as tuples, UDTs and vectors are masked by
replacing each of their values by the default masking value of the value type.
Examples:
@@ -18,6 +23,10 @@ Examples:
`mask_default(123)` -> `0`
+`mask_default((list<int>) [1, 2, 3])` -> `[]`
+
+`mask_default((vector<int, 3>) [1, 2, 3])` -> `[0, 0, 0]`
+
| `mask_replace(value, replacement])` | Replaces the first argument by the
replacement value on the second argument. The replacement value needs to have
the same type as the replaced value.
Examples:
diff --git a/src/java/org/apache/cassandra/cql3/CQL3Type.java
b/src/java/org/apache/cassandra/cql3/CQL3Type.java
index 074a8e515e..6b8b97877d 100644
--- a/src/java/org/apache/cassandra/cql3/CQL3Type.java
+++ b/src/java/org/apache/cassandra/cql3/CQL3Type.java
@@ -20,6 +20,7 @@ package org.apache.cassandra.cql3;
import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.List;
+import java.util.Objects;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -566,6 +567,21 @@ public interface CQL3Type
return sb.toString();
}
+ @Override
+ public boolean equals(Object o)
+ {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+ Vector vector = (Vector) o;
+ return Objects.equals(type, vector.type);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return Objects.hash(type);
+ }
+
@Override
public String toString()
{
diff --git a/src/java/org/apache/cassandra/db/marshal/VectorType.java
b/src/java/org/apache/cassandra/db/marshal/VectorType.java
index 204c603d28..cd6324ebb9 100644
--- a/src/java/org/apache/cassandra/db/marshal/VectorType.java
+++ b/src/java/org/apache/cassandra/db/marshal/VectorType.java
@@ -20,6 +20,7 @@ package org.apache.cassandra.db.marshal;
import java.nio.ByteBuffer;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.List;
import java.util.Objects;
@@ -167,6 +168,11 @@ public final class VectorType<T> extends
AbstractType<List<T>>
return array;
}
+ public ByteBuffer decompose(T... values)
+ {
+ return decompose(Arrays.asList(values));
+ }
+
public ByteBuffer decomposeAsFloat(float[] value)
{
return decomposeAsFloat(ByteBufferAccessor.instance, value);
@@ -367,6 +373,13 @@ public final class VectorType<T> extends
AbstractType<List<T>>
throw new MarshalException("Invalid empty vector value");
}
+ @Override
+ public ByteBuffer getMaskedValue()
+ {
+ List<ByteBuffer> values = Collections.nCopies(dimension,
elementType.getMaskedValue());
+ return serializer.serializeRaw(values, ByteBufferAccessor.instance);
+ }
+
public abstract class VectorSerializer extends TypeSerializer<List<T>>
{
public abstract <VL, VR> int compareCustom(VL left, ValueAccessor<VL>
accessorL, VR right, ValueAccessor<VR> accessorR);
diff --git
a/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskQueryWithDefaultTest.java
b/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskQueryWithDefaultTest.java
index edef2a88bb..1ca8da1163 100644
---
a/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskQueryWithDefaultTest.java
+++
b/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskQueryWithDefaultTest.java
@@ -25,6 +25,9 @@ import java.util.List;
import org.junit.runners.Parameterized;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.VectorType;
+
/**
* {@link ColumnMaskQueryTester} for {@link DefaultMaskingFunction}.
*/
@@ -39,7 +42,11 @@ public class ColumnMaskQueryWithDefaultTest extends
ColumnMaskQueryTester
options.add(new Object[]{ order, "DEFAULT", "text", "abc", "****"
});
options.add(new Object[]{ order, "DEFAULT", "int", 123, 0 });
options.add(new Object[]{ order, "mask_default()", "text", "abc",
"****" });
- options.add(new Object[]{ order, "mask_default()", "int", 123, 0,
});
+ options.add(new Object[]{ order, "mask_default()", "int", 123, 0
});
+ // TODO: the driver version that we use doesn't support vectors,
so we have to use raw values by now
+ options.add(new Object[]{ order, "mask_default()", "vector<int,
2>",
+
VectorType.getInstance(Int32Type.instance, 2).decompose(1, 2),
+
VectorType.getInstance(Int32Type.instance, 2).decompose(0, 0) });
}
return options;
}
diff --git
a/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskQueryWithReplaceTest.java
b/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskQueryWithReplaceTest.java
index 9a3a570a65..fd139eeaf4 100644
---
a/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskQueryWithReplaceTest.java
+++
b/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskQueryWithReplaceTest.java
@@ -26,6 +26,9 @@ import java.util.List;
import org.junit.runners.Parameterized;
+import org.apache.cassandra.db.marshal.Int32Type;
+import org.apache.cassandra.db.marshal.VectorType;
+
/**
* {@link ColumnMaskQueryTester} for {@link ReplaceMaskingFunction}.
*/
@@ -43,6 +46,10 @@ public class ColumnMaskQueryWithReplaceTest extends
ColumnMaskQueryTester
options.add(new Object[]{ order, "mask_replace(0)", "int", 123, 0
});
options.add(new Object[]{ order, "mask_replace(0)", "bigint",
123L, 0L });
options.add(new Object[]{ order, "mask_replace(0)", "varint",
BigInteger.valueOf(123), BigInteger.ZERO });
+ // TODO: the driver version that we use doesn't support vectors,
so we have to use raw values by now
+ options.add(new Object[]{ order, "mask_replace([0, 0])",
"vector<int, 2>",
+
VectorType.getInstance(Int32Type.instance, 2).decompose(1, 2),
+
VectorType.getInstance(Int32Type.instance, 2).decompose(0, 0) });
}
return options;
}
diff --git
a/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskTest.java
b/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskTest.java
index bf0a9156cd..0dd134aa01 100644
--- a/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskTest.java
+++ b/test/unit/org/apache/cassandra/cql3/functions/masking/ColumnMaskTest.java
@@ -108,6 +108,22 @@ public class ColumnMaskTest extends ColumnMaskTester
assertTableColumnsAreNotMasked("v");
}
+ @Test
+ public void testVectors() throws Throwable
+ {
+ // Create table with mask
+ String table = createTable("CREATE TABLE %s (k int PRIMARY KEY, v
vector<int, 3> MASKED WITH DEFAULT)");
+ assertColumnIsMasked(table, "v", "mask_default", emptyList(),
emptyList());
+
+ // Alter column mask
+ alterTable("ALTER TABLE %s ALTER v MASKED WITH mask_null()");
+ assertColumnIsMasked(table, "v", "mask_null", emptyList(),
emptyList());
+
+ // Drop mask
+ alterTable("ALTER TABLE %s ALTER v DROP MASKED");
+ assertTableColumnsAreNotMasked("v");
+ }
+
@Test
public void testAlterTableAddMaskingToNonExistingColumn() throws Throwable
{
diff --git
a/test/unit/org/apache/cassandra/cql3/functions/masking/MaskingFunctionTester.java
b/test/unit/org/apache/cassandra/cql3/functions/masking/MaskingFunctionTester.java
index a45f2c8b57..fcbda1b8e0 100644
---
a/test/unit/org/apache/cassandra/cql3/functions/masking/MaskingFunctionTester.java
+++
b/test/unit/org/apache/cassandra/cql3/functions/masking/MaskingFunctionTester.java
@@ -35,6 +35,7 @@ import org.apache.cassandra.cql3.CQLTester;
import org.apache.cassandra.cql3.Duration;
import org.apache.cassandra.db.marshal.AbstractType;
import org.apache.cassandra.db.marshal.CollectionType;
+import org.apache.cassandra.db.marshal.FloatType;
import org.apache.cassandra.db.marshal.Int32Type;
import org.apache.cassandra.db.marshal.ListType;
import org.apache.cassandra.db.marshal.MapType;
@@ -42,6 +43,7 @@ import org.apache.cassandra.db.marshal.SetType;
import org.apache.cassandra.db.marshal.TupleType;
import org.apache.cassandra.db.marshal.UTF8Type;
import org.apache.cassandra.db.marshal.UserType;
+import org.apache.cassandra.db.marshal.VectorType;
import org.apache.cassandra.schema.KeyspaceMetadata;
import org.apache.cassandra.schema.Schema;
import org.apache.cassandra.serializers.SimpleDateSerializer;
@@ -159,6 +161,20 @@ public abstract class MaskingFunctionTester extends
CQLTester
testMaskingOnNotKeyColumns(MapType.getInstance(Int32Type.instance,
Int32Type.instance, true).asCQL3Type(), values);
}
+ /**
+ * Tests the native masking function for vectors.
+ */
+ @Test
+ public void testMaskingOnVector() throws Throwable
+ {
+ testMaskingOnAllColumns(VectorType.getInstance(Int32Type.instance,
2).asCQL3Type(),
+ vector(1, 10), vector(2, 20));
+ testMaskingOnAllColumns(VectorType.getInstance(FloatType.instance,
2).asCQL3Type(),
+ vector(1.1f, 10.1f), vector(2.2f, 20.2f));
+ testMaskingOnAllColumns(VectorType.getInstance(UTF8Type.instance,
2).asCQL3Type(),
+ vector("a1", "a2"), vector("b1", "b2"));
+ }
+
/**
* Tests the native masking function for tuples.
*/
diff --git
a/test/unit/org/apache/cassandra/cql3/functions/masking/ReplaceMaskingFunctionTest.java
b/test/unit/org/apache/cassandra/cql3/functions/masking/ReplaceMaskingFunctionTest.java
index c00125cc04..c997407e5a 100644
---
a/test/unit/org/apache/cassandra/cql3/functions/masking/ReplaceMaskingFunctionTest.java
+++
b/test/unit/org/apache/cassandra/cql3/functions/masking/ReplaceMaskingFunctionTest.java
@@ -19,6 +19,7 @@
package org.apache.cassandra.cql3.functions.masking;
import java.math.BigInteger;
+import java.nio.ByteBuffer;
import org.junit.Test;
@@ -34,7 +35,7 @@ import static java.lang.String.format;
public class ReplaceMaskingFunctionTest extends MaskingFunctionTester
{
@Override
- protected void testMaskingOnColumn(String name, CQL3Type type, Object
value) throws Throwable
+ protected void testMaskingOnColumn(String name, CQL3Type type, Object
value)
{
// null replacement argument
assertRows(execute(format("SELECT mask_replace(%s, ?) FROM %%s",
name), (Object) null),
@@ -42,7 +43,7 @@ public class ReplaceMaskingFunctionTest extends
MaskingFunctionTester
// not-null replacement argument
AbstractType<?> t = type.getType();
- Object replacementValue = t.compose(t.getMaskedValue());
+ ByteBuffer replacementValue = t.getMaskedValue();
String query = format("SELECT mask_replace(%s, ?) FROM %%s", name);
assertRows(execute(query, replacementValue), row(replacementValue));
}
diff --git a/test/unit/org/apache/cassandra/db/marshal/AbstractTypeTest.java
b/test/unit/org/apache/cassandra/db/marshal/AbstractTypeTest.java
index c38e4dd2ff..86b3926a53 100644
--- a/test/unit/org/apache/cassandra/db/marshal/AbstractTypeTest.java
+++ b/test/unit/org/apache/cassandra/db/marshal/AbstractTypeTest.java
@@ -101,7 +101,21 @@ public class AbstractTypeTest
// TODO
// isCompatibleWith/isValueCompatibleWith/isSerializationCompatibleWith,
// withUpdatedUserType/expandUserTypes/referencesDuration - types that
recursive check types
- // getMaskedValue
+
+ @Test
+ @SuppressWarnings({"unchecked", "rawtypes"})
+ public void maskedValue()
+ {
+ qt().forAll(genBuilder().withoutTypeKinds(COMPOSITE,
DYNAMIC_COMPOSITE).build())
+ .checkAssert(type -> {
+ ByteBuffer maskedValue = type.getMaskedValue();
+ type.validate(maskedValue);
+
+ Object composed = type.compose(maskedValue);
+ ByteBuffer decomposed = ((AbstractType)
type).decompose(composed);
+ assertThat(decomposed).isEqualTo(maskedValue);
+ });
+ }
@Test
public void empty()
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
