TheNeuralBit commented on a change in pull request #16101:
URL: https://github.com/apache/beam/pull/16101#discussion_r769038965
##########
File path: sdks/python/apache_beam/dataframe/partitionings.py
##########
@@ -175,6 +175,61 @@ def check(self, dfs):
return len(dfs) <= 1
+class JoinIndex(Partitioning):
+ """A partitioning that lets two frames be joined.
+ This can either be a hash partitioning on the full index, or a common
+ ancestor with no intervening re-indexing/re-partitioning.
+
+ It fits into the partial ordering as
+
+ Index() < JoinIndex(x) < JoinIndex() < Arbitrary()
+
+ with
+
+ JoinIndex(x) and JoinIndex(y)
+
+ being incomparable for nontrivial x != y.
+
+ Expressions desiring to make use of this index should simply declare a
+ requirement of JoinIndex().
+ """
+ def __init__(self, ancestor=None):
+ self._ancestor = ancestor
+
+ def __repr__(self):
+ if self._ancestor:
+ return 'JoinIndex[%s]' % self._ancestor
+ else:
+ return 'JoinIndex'
+
+ def __eq__(self, other):
+ if type(self) != type(other):
+ return False
+ elif self._ancestor is None:
+ return other._ancestor is None
+ elif other._ancestor is None:
+ return False
+ else:
+ return self._ancestor == other._ancestor
+
+ def __hash__(self):
+ return hash((type(self), self._ancestor))
+
+ def is_subpartitioning_of(self, other):
+ if isinstance(other, Arbitrary):
+ return False
+ elif isinstance(other, JoinIndex):
+ return self._ancestor is None or self == other
+ else:
+ return True
+
+ def test_partition_fn(self, df):
+ return Index().test_partition_fn(df)
+
+ def check(self, dfs):
+ return True
Review comment:
Fair point, if we want to do that we should add JoinIndex here:
https://github.com/apache/beam/blob/1a74cc0a7f85d9a1cf6974d1cdc6e193ef46332a/sdks/python/apache_beam/dataframe/expressions.py#L118-L125
But just like test_partition_fn that's going to get us little benefit.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
