dongjoon-hyun commented on code in PR #57131:
URL: https://github.com/apache/spark/pull/57131#discussion_r3547562313
##########
dev/merge_spark_pr.py:
##########
@@ -686,6 +690,150 @@ def cherry_pick(pr_num, merge_hash, default_branch,
branch_names, target_ref, al
return [_do_cherry_pick(pr_num, merge_hash, pick_ref)]
+# Common words carry no signal when comparing a PR title to a JIRA summary, so
they are
+# dropped before scoring. Kept deliberately small: over-aggressive stopword
removal makes
+# unrelated titles look similar. Component tags ([SQL], [CORE], ...) are
stripped separately.
+_SIMILARITY_STOPWORDS = frozenset(
+ {
+ "a",
+ "an",
+ "the",
+ "to",
+ "for",
+ "of",
+ "in",
+ "on",
+ "and",
+ "or",
+ "with",
+ "is",
+ "are",
+ "be",
+ "when",
+ "should",
+ "make",
+ "add",
+ "fix",
+ "fixes",
+ "support",
+ "enable",
+ "use",
+ "using",
+ }
+)
+_SIMILARITY_WORD_RE = re.compile(r"[a-z0-9]+")
+# Below this Jaccard word-overlap score, warn that the PR title and JIRA
summary look
+# unrelated. Tuned to flag clear mismatches (an unrelated ticket scores ~0)
while leaving
+# paraphrases and reworded summaries (which still share key nouns) above the
line.
+SIMILARITY_WARN_THRESHOLD = 0.2
+
+
+def title_similarity(pr_title, summary):
+ """Jaccard word-overlap between a PR title and a JIRA summary, in [0.0,
1.0].
+
+ Component/version tags ([SQL], [4.x], ...) and the leading SPARK id are
stripped,
+ words are lowercased, and common stopwords are dropped, so the score
reflects the
+ substantive words the two share. 1.0 means identical word sets; 0.0 means
none in
+ common (or an empty side).
+
+ >>> title_similarity("[SPARK-1][SQL] Compute stable checksum", "Compute a
stable checksum")
+ 1.0
+ >>> title_similarity("[SPARK-1][SQL] Compute stable checksum", "Refactor
the logging backend")
+ 0.0
+ >>> round(title_similarity("[SPARK-1] Ceil and floor overflow", "Handle
floor overflow"), 2)
+ 0.5
+
+ A [FOLLOWUP] tag is stripped like any other bracket tag, so a follow-up
whose
+ title still matches the ticket scores the same as the untagged title:
Review Comment:
Is this true? Usually, the follow-up title is supposed to describe its code
which is usually missed in the original PR. Like the following. The following
is an example in `branch-3.5`.
```
$ git log --oneline | grep SPARK-56463
77117c7b32a [SPARK-56463][PYTHON][FOLLOWUP][3.5] Fix wrong kwargs
092f02df06e [SPARK-56463][PYTHON] Disallow unpickling UDT
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
