zhengruifeng commented on code in PR #37845:
URL: https://github.com/apache/spark/pull/37845#discussion_r969129537
##########
python/pyspark/pandas/frame.py:
##########
@@ -1454,11 +1462,196 @@ def corr(self, method: str = "pearson") -> "DataFrame":
There are behavior differences between pandas-on-Spark and pandas.
* the `method` argument only accepts 'pearson', 'spearman'
- * the data should not contain NaNs. pandas-on-Spark will return an
error.
- * pandas-on-Spark doesn't support the following argument(s).
+ * if the `method` is `spearman`, the data should not contain NaNs.
+ * if the `method` is `spearman`, `min_periods` argument is not
supported.
+ """
+ if method not in ["pearson", "spearman", "kendall"]:
+ raise ValueError(f"Invalid method {method}")
+ if method == "kendall":
+ raise NotImplementedError("method doesn't support kendall for now")
+ if min_periods is not None and not isinstance(min_periods, int):
+ raise TypeError(f"Invalid min_periods type
{type(min_periods).__name__}")
+ if min_periods is not None and method == "spearman":
+ raise NotImplementedError("min_periods doesn't support spearman
for now")
+
+ if method == "pearson":
+ min_periods = 1 if min_periods is None else min_periods
+ internal = self._internal.resolved_copy
+ numeric_labels = [
+ label
+ for label in internal.column_labels
+ if isinstance(internal.spark_type_for(label), (NumericType,
BooleanType))
+ ]
+ numeric_scols: List[Column] = [
+ internal.spark_column_for(label).cast("double") for label in
numeric_labels
+ ]
+ numeric_col_names: List[str] = [name_like_string(label) for label
in numeric_labels]
+ num_scols = len(numeric_scols)
+
+ sdf = internal.spark_frame
+ tmp_index_1_col = verify_temp_column_name(sdf,
"__tmp_index_1_col__")
+ tmp_index_2_col = verify_temp_column_name(sdf,
"__tmp_index_2_col__")
+ tmp_value_1_col = verify_temp_column_name(sdf,
"__tmp_value_1_col__")
+ tmp_value_2_col = verify_temp_column_name(sdf,
"__tmp_value_2_col__")
+
+ # simple dataset
+ # +---+---+----+
+ # | A| B| C|
+ # +---+---+----+
+ # | 1| 2| 3.0|
+ # | 4| 1|null|
+ # +---+---+----+
+
+ pair_scols: List[Column] = []
+ for i in range(0, num_scols):
Review Comment:
just because that other places use `range(0, x)` instead of `range(x)`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
