christinadionysio commented on code in PR #2614:
URL: https://github.com/apache/systemds/pull/2614#discussion_r4347263592
##########
src/main/python/tests/scuro/test_fusion_orders.py:
##########
@@ -19,77 +19,80 @@
#
# -------------------------------------------------------------
-import os
-import shutil
import unittest
import numpy as np
from systemds.scuro import Concatenation, RowMax, Hadamard
-from systemds.scuro.modality.unimodal_modality import UnimodalModality
-from systemds.scuro.representations.bert import Bert
-from systemds.scuro.representations.mel_spectrogram import MelSpectrogram
from systemds.scuro.representations.average import Average
from tests.scuro.data_generator import ModalityRandomDataGenerator
from systemds.scuro.modality.type import ModalityType
class TestFusionOrders(unittest.TestCase):
+ """
+ Tests the order properties of the fusion operators: commutativity, whether
+ the order of a pairwise chain matters, and whether a pairwise chain gives
+ the same result as the n-ary call.
+ """
+
+ # (operator, chain_order_independent, chain_equals_nary)
+ #
+ # Commutativity is not in the table. Every Fusion operator has a
+ # "commutative" attribute. It is False on the base class and a subclass
+ # can override it. The test compares the measured result with that
+ # attribute, so an operator whose attribute does not match its
+ # implementation fails here.
+ #
+ # Combining a pair is never the same as combining all three. That is
+ # checked for every operator, so it is not in the table either.
+ FUSION_PROPERTIES = [
Review Comment:
Keeping this table here means we need to adopt it for every new fusion
operator. If possible use the properties for the operators themselves.
##########
src/main/python/tests/scuro/test_fusion_orders.py:
##########
@@ -19,77 +19,80 @@
#
# -------------------------------------------------------------
-import os
-import shutil
import unittest
import numpy as np
from systemds.scuro import Concatenation, RowMax, Hadamard
-from systemds.scuro.modality.unimodal_modality import UnimodalModality
-from systemds.scuro.representations.bert import Bert
-from systemds.scuro.representations.mel_spectrogram import MelSpectrogram
from systemds.scuro.representations.average import Average
from tests.scuro.data_generator import ModalityRandomDataGenerator
from systemds.scuro.modality.type import ModalityType
class TestFusionOrders(unittest.TestCase):
+ """
+ Tests the order properties of the fusion operators: commutativity, whether
+ the order of a pairwise chain matters, and whether a pairwise chain gives
+ the same result as the n-ary call.
+ """
+
+ # (operator, chain_order_independent, chain_equals_nary)
+ #
+ # Commutativity is not in the table. Every Fusion operator has a
+ # "commutative" attribute. It is False on the base class and a subclass
+ # can override it. The test compares the measured result with that
+ # attribute, so an operator whose attribute does not match its
+ # implementation fails here.
+ #
+ # Combining a pair is never the same as combining all three. That is
+ # checked for every operator, so it is not in the table either.
+ FUSION_PROPERTIES = [
+ (Average, True, False),
+ (Concatenation, False, True),
+ (RowMax, True, True),
+ (Hadamard, True, True),
+ ]
+
@classmethod
def setUpClass(cls):
- cls.num_instances = 40
+ # These properties do not depend on the input shape, so the data can
+ # be small.
+ cls.num_instances = 4
+ cls.num_features = 8
cls.data_generator = ModalityRandomDataGenerator()
- cls.r_1 = cls.data_generator.create1DModality(40, 100,
ModalityType.AUDIO)
- cls.r_2 = cls.data_generator.create1DModality(40, 100,
ModalityType.TEXT)
- cls.r_3 = cls.data_generator.create1DModality(40, 100,
ModalityType.TEXT)
-
- def test_fusion_order_avg(self):
- r_1_r_2 = self.r_1.combine(self.r_2, Average())
- r_2_r_1 = self.r_2.combine(self.r_1, Average())
- r_1_r_2_r_3 = r_1_r_2.combine(self.r_3, Average())
- r_2_r_1_r_3 = r_2_r_1.combine(self.r_3, Average())
-
- r1_r2_r3 = self.r_1.combine([self.r_2, self.r_3], Average())
-
- self.assertTrue(np.array_equal(r_1_r_2.data, r_2_r_1.data))
- self.assertTrue(np.array_equal(r_1_r_2_r_3.data, r_2_r_1_r_3.data))
- self.assertFalse(np.array_equal(r_1_r_2_r_3.data, r1_r2_r3.data))
- self.assertFalse(np.array_equal(r_1_r_2.data, r1_r2_r3.data))
-
- def test_fusion_order_concat(self):
- r_1_r_2 = self.r_1.combine(self.r_2, Concatenation())
- r_2_r_1 = self.r_2.combine(self.r_1, Concatenation())
- r_1_r_2_r_3 = r_1_r_2.combine(self.r_3, Concatenation())
- r_2_r_1_r_3 = r_2_r_1.combine(self.r_3, Concatenation())
-
- r1_r2_r3 = self.r_1.combine([self.r_2, self.r_3], Concatenation())
-
- self.assertFalse(np.array_equal(r_1_r_2.data, r_2_r_1.data))
- self.assertFalse(np.array_equal(r_1_r_2_r_3.data, r_2_r_1_r_3.data))
- self.assertFalse(np.array_equal(r_2_r_1.data, r1_r2_r3.data))
- self.assertFalse(np.array_equal(r_1_r_2.data, r1_r2_r3.data))
-
- def test_fusion_order_max(self):
- r_1_r_2 = self.r_1.combine(self.r_2, RowMax())
- r_2_r_1 = self.r_2.combine(self.r_1, RowMax())
- r_1_r_2_r_3 = r_1_r_2.combine(self.r_3, RowMax())
- r_2_r_1_r_3 = r_2_r_1.combine(self.r_3, RowMax())
-
- r1_r2_r3 = self.r_1.combine([self.r_2, self.r_3], RowMax())
-
- self.assertTrue(np.array_equal(r_1_r_2.data, r_2_r_1.data))
- self.assertTrue(np.array_equal(r_1_r_2_r_3.data, r_2_r_1_r_3.data))
- self.assertTrue(np.array_equal(r_1_r_2_r_3.data, r1_r2_r3.data))
- self.assertFalse(np.array_equal(r_1_r_2.data, r1_r2_r3.data))
-
- def test_fusion_order_hadamard(self):
- r_1_r_2 = self.r_1.combine(self.r_2, Hadamard())
- r_2_r_1 = self.r_2.combine(self.r_1, Hadamard())
- r_1_r_2_r_3 = r_1_r_2.combine(self.r_3, Hadamard())
- r_2_r_1_r_3 = r_2_r_1.combine(self.r_3, Hadamard())
-
- r1_r2_r3 = self.r_1.combine([self.r_2, self.r_3], Hadamard())
- self.assertTrue(np.array_equal(r_1_r_2.data, r_2_r_1.data))
- self.assertTrue(np.array_equal(r_1_r_2_r_3.data, r_2_r_1_r_3.data))
- self.assertTrue(np.array_equal(r_1_r_2_r_3.data, r1_r2_r3.data))
- self.assertFalse(np.array_equal(r_1_r_2.data, r1_r2_r3.data))
+ def setUp(self):
+ self.r_1 = self.data_generator.create1DModality(
+ self.num_instances, self.num_features, ModalityType.AUDIO
+ )
+ self.r_2 = self.data_generator.create1DModality(
+ self.num_instances, self.num_features, ModalityType.TEXT
+ )
+ self.r_3 = self.data_generator.create1DModality(
+ self.num_instances, self.num_features, ModalityType.TEXT
+ )
+
+ @staticmethod
+ def _equal(left, right):
+ return np.array_equal(np.asarray(left.data), np.asarray(right.data))
+
+ def test_fusion_order_properties(self):
+ for (
+ fusion_operator,
+ chain_order_independent,
+ chain_equals_nary,
+ ) in self.FUSION_PROPERTIES:
Review Comment:
In order to keep the information which operator failed the test please add a
message including the operator name to each assertion.
##########
src/main/python/tests/scuro/test_window_operations.py:
##########
@@ -111,67 +111,60 @@ def test_dynamic_window(self):
for i in range(0, self.num_instances):
assert len(aggregated_window.data[i]) == num_windows
- def test_window_aggregation_on_audio_representations(self):
+ def test_window_aggregation_on_1d_modalities(self):
+ # create1DModality returns the same shape and dtype for all three
+ # modality types. window_aggregation looks at the data layout and not
+ # at the modality type, so the result should be the same for all of
+ # them.
window_size = 10
- self.run_window_aggregation_for_modality(ModalityType.AUDIO,
window_size)
- def test_window_operations_on_video_representations(self):
- window_size = 10
- self.run_window_aggregation_for_modality(ModalityType.VIDEO,
window_size)
-
- def test_window_operations_on_text_representations(self):
- window_size = 10
-
- self.run_window_aggregation_for_modality(ModalityType.TEXT,
window_size)
-
- def run_window_aggregation_for_modality(self, modality_type, window_size):
- r = self.data_generator.create1DModality(self.num_instances, 200,
modality_type)
- for aggregation in self.aggregations:
- windowed_modality = r.window_aggregation(window_size, aggregation)
-
- self.verify_window_operation(aggregation, r, windowed_modality,
window_size)
-
- def test_window_aggregation_on_3d_modality(self):
- data, _ = self.data_generator.create_3d_modality(
- self.num_instances, (100, 8, 8)
- )
- embedding_modality = TransformedModality(
- self.data_generator, "test_transformation"
- )
- embedding_modality.data = data
- embedding_modality.stats = RepresentationStats(self.num_instances,
(100, 8, 8))
- num_windows = 10
-
- for window_operator in [
- StaticWindow(num_windows=num_windows),
- DynamicWindow(num_windows=num_windows),
- WindowAggregation(window_size=10),
+ for modality_type in [
+ ModalityType.AUDIO,
+ ModalityType.VIDEO,
+ ModalityType.TEXT,
]:
- stats = window_operator.get_output_stats(embedding_modality.stats)
- assert stats.num_instances == self.num_instances
- assert stats.output_shape == (num_windows, 8, 8)
-
- windowed_modality = embedding_modality.context(window_operator)
+ r = self.data_generator.create1DModality(
+ self.num_instances, 200, modality_type
+ )
+ for aggregation in self.aggregations:
+ with self.subTest(modality=modality_type.name,
aggregation=aggregation):
+ windowed_modality = r.window_aggregation(window_size,
aggregation)
+ self.verify_window_operation(
+ aggregation, r, windowed_modality, window_size
+ )
- def test_window_aggregation_on_2d_modality(self):
- data, _ = self.data_generator.create_2d_modality(self.num_instances,
(100, 8))
- embedding_modality = TransformedModality(
- self.data_generator, "test_transformation"
- )
- embedding_modality.data = data
- embedding_modality.stats = RepresentationStats(self.num_instances,
(100, 8))
+ def test_window_aggregation_on_nd_modality(self):
+ # Window aggregation only changes the first (time) axis and keeps the
+ # feature axes as they are. The expected shape is therefore
+ # (num_windows,) + dims[1:] for any number of dimensions.
num_windows = 10
- for window_operator in [
- StaticWindow(num_windows=num_windows),
- DynamicWindow(num_windows=num_windows),
- WindowAggregation(window_size=10),
- ]:
- stats = window_operator.get_output_stats(embedding_modality.stats)
- assert stats.num_instances == self.num_instances
- assert stats.output_shape == (num_windows, 8)
-
- windowed_modality = embedding_modality.context(window_operator)
+ for dims in [(100, 8, 8), (100, 8)]:
+ if len(dims) == 3:
+ data, _ = self.data_generator.create_3d_modality(
+ self.num_instances, dims
+ )
+ else:
+ data, _ = self.data_generator.create_2d_modality(
+ self.num_instances, dims
+ )
+ embedding_modality = TransformedModality(
+ self.data_generator, "test_transformation"
+ )
+ embedding_modality.data = data
+ embedding_modality.stats = RepresentationStats(self.num_instances,
dims)
+
+ for window_operator in [
+ StaticWindow(num_windows=num_windows),
+ DynamicWindow(num_windows=num_windows),
+ WindowAggregation(window_size=10),
+ ]:
+ with self.subTest(dims=dims,
operator=type(window_operator).__name__):
Review Comment:
Please add a message with the operator name to the assertion so we know
which one fails.
##########
src/main/python/tests/scuro/test_window_operations.py:
##########
@@ -111,67 +111,60 @@ def test_dynamic_window(self):
for i in range(0, self.num_instances):
assert len(aggregated_window.data[i]) == num_windows
- def test_window_aggregation_on_audio_representations(self):
+ def test_window_aggregation_on_1d_modalities(self):
+ # create1DModality returns the same shape and dtype for all three
+ # modality types. window_aggregation looks at the data layout and not
+ # at the modality type, so the result should be the same for all of
+ # them.
window_size = 10
- self.run_window_aggregation_for_modality(ModalityType.AUDIO,
window_size)
- def test_window_operations_on_video_representations(self):
- window_size = 10
- self.run_window_aggregation_for_modality(ModalityType.VIDEO,
window_size)
-
- def test_window_operations_on_text_representations(self):
- window_size = 10
-
- self.run_window_aggregation_for_modality(ModalityType.TEXT,
window_size)
-
- def run_window_aggregation_for_modality(self, modality_type, window_size):
- r = self.data_generator.create1DModality(self.num_instances, 200,
modality_type)
- for aggregation in self.aggregations:
- windowed_modality = r.window_aggregation(window_size, aggregation)
-
- self.verify_window_operation(aggregation, r, windowed_modality,
window_size)
-
- def test_window_aggregation_on_3d_modality(self):
- data, _ = self.data_generator.create_3d_modality(
- self.num_instances, (100, 8, 8)
- )
- embedding_modality = TransformedModality(
- self.data_generator, "test_transformation"
- )
- embedding_modality.data = data
- embedding_modality.stats = RepresentationStats(self.num_instances,
(100, 8, 8))
- num_windows = 10
-
- for window_operator in [
- StaticWindow(num_windows=num_windows),
- DynamicWindow(num_windows=num_windows),
- WindowAggregation(window_size=10),
+ for modality_type in [
+ ModalityType.AUDIO,
+ ModalityType.VIDEO,
+ ModalityType.TEXT,
]:
- stats = window_operator.get_output_stats(embedding_modality.stats)
- assert stats.num_instances == self.num_instances
- assert stats.output_shape == (num_windows, 8, 8)
-
- windowed_modality = embedding_modality.context(window_operator)
+ r = self.data_generator.create1DModality(
+ self.num_instances, 200, modality_type
+ )
+ for aggregation in self.aggregations:
+ with self.subTest(modality=modality_type.name,
aggregation=aggregation):
+ windowed_modality = r.window_aggregation(window_size,
aggregation)
+ self.verify_window_operation(
+ aggregation, r, windowed_modality, window_size
+ )
- def test_window_aggregation_on_2d_modality(self):
- data, _ = self.data_generator.create_2d_modality(self.num_instances,
(100, 8))
- embedding_modality = TransformedModality(
- self.data_generator, "test_transformation"
- )
- embedding_modality.data = data
- embedding_modality.stats = RepresentationStats(self.num_instances,
(100, 8))
+ def test_window_aggregation_on_nd_modality(self):
+ # Window aggregation only changes the first (time) axis and keeps the
+ # feature axes as they are. The expected shape is therefore
+ # (num_windows,) + dims[1:] for any number of dimensions.
num_windows = 10
- for window_operator in [
- StaticWindow(num_windows=num_windows),
- DynamicWindow(num_windows=num_windows),
- WindowAggregation(window_size=10),
- ]:
- stats = window_operator.get_output_stats(embedding_modality.stats)
- assert stats.num_instances == self.num_instances
- assert stats.output_shape == (num_windows, 8)
-
- windowed_modality = embedding_modality.context(window_operator)
+ for dims in [(100, 8, 8), (100, 8)]:
+ if len(dims) == 3:
+ data, _ = self.data_generator.create_3d_modality(
+ self.num_instances, dims
+ )
+ else:
+ data, _ = self.data_generator.create_2d_modality(
+ self.num_instances, dims
+ )
+ embedding_modality = TransformedModality(
+ self.data_generator, "test_transformation"
+ )
+ embedding_modality.data = data
+ embedding_modality.stats = RepresentationStats(self.num_instances,
dims)
+
+ for window_operator in [
+ StaticWindow(num_windows=num_windows),
+ DynamicWindow(num_windows=num_windows),
+ WindowAggregation(window_size=10),
+ ]:
+ with self.subTest(dims=dims,
operator=type(window_operator).__name__):
+ stats =
window_operator.get_output_stats(embedding_modality.stats)
+ self.assertEqual(stats.num_instances, self.num_instances)
+ self.assertEqual(stats.output_shape, (num_windows,) +
dims[1:])
+
+ embedding_modality.context(window_operator)
Review Comment:
Check the result of the context operation, i.e.: assert the actual shape of
the output matches the statistics.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
