[
https://issues.apache.org/jira/browse/SPARK-32318?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Dongjoon Hyun updated SPARK-32318:
----------------------------------
Description:
This is found during reviewing SPARK-32276.
*AFTER SPARK-32276*
{code}
scala> scala.util.Random.shuffle((1 to 100000).map(x => (x % 2, x))).toDF("a",
"b").repartition(2).createOrReplaceTempView("t")
scala> sql("select * from (select * from t order by b) distribute by
a").write.orc("/tmp/SPARK-32276")
$ ls -al /tmp/SPARK-32276/
total 632
drwxr-xr-x 10 dongjoon wheel 320 Jul 14 22:08 ./
drwxrwxrwt 14 root wheel 448 Jul 14 22:08 ../
-rw-r--r-- 1 dongjoon wheel 8 Jul 14 22:08 ._SUCCESS.crc
-rw-r--r-- 1 dongjoon wheel 12 Jul 14 22:08
.part-00000-ba5049f9-b835-49b7-9fdb-bdd11b9891cb-c000.snappy.orc.crc
-rw-r--r-- 1 dongjoon wheel 1188 Jul 14 22:08
.part-00043-ba5049f9-b835-49b7-9fdb-bdd11b9891cb-c000.snappy.orc.crc
-rw-r--r-- 1 dongjoon wheel 1188 Jul 14 22:08
.part-00191-ba5049f9-b835-49b7-9fdb-bdd11b9891cb-c000.snappy.orc.crc
-rw-r--r-- 1 dongjoon wheel 0 Jul 14 22:08 _SUCCESS
-rw-r--r-- 1 dongjoon wheel 119 Jul 14 22:08
part-00000-ba5049f9-b835-49b7-9fdb-bdd11b9891cb-c000.snappy.orc
-rw-r--r-- 1 dongjoon wheel 150735 Jul 14 22:08
part-00043-ba5049f9-b835-49b7-9fdb-bdd11b9891cb-c000.snappy.orc
-rw-r--r-- 1 dongjoon wheel 150741 Jul 14 22:08
part-00191-ba5049f9-b835-49b7-9fdb-bdd11b9891cb-c000.snappy.orc
{code}
*BEFORE*
{code}
scala> scala.util.Random.shuffle((1 to 100000).map(x => (x % 2, x))).toDF("a",
"b").repartition(2).createOrReplaceTempView("t")
scala> sql("select * from (select * from t order by b) distribute by
a").write.orc("/tmp/master")
$ ls -al /tmp/master/
total 56
drwxr-xr-x 10 dongjoon wheel 320 Jul 14 22:12 ./
drwxrwxrwt 15 root wheel 480 Jul 14 22:12 ../
-rw-r--r-- 1 dongjoon wheel 8 Jul 14 22:12 ._SUCCESS.crc
-rw-r--r-- 1 dongjoon wheel 12 Jul 14 22:12
.part-00000-2cd3a50e-eded-49a4-b7cf-94e3f090b8c1-c000.snappy.orc.crc
-rw-r--r-- 1 dongjoon wheel 16 Jul 14 22:12
.part-00043-2cd3a50e-eded-49a4-b7cf-94e3f090b8c1-c000.snappy.orc.crc
-rw-r--r-- 1 dongjoon wheel 16 Jul 14 22:12
.part-00191-2cd3a50e-eded-49a4-b7cf-94e3f090b8c1-c000.snappy.orc.crc
-rw-r--r-- 1 dongjoon wheel 0 Jul 14 22:12 _SUCCESS
-rw-r--r-- 1 dongjoon wheel 119 Jul 14 22:12
part-00000-2cd3a50e-eded-49a4-b7cf-94e3f090b8c1-c000.snappy.orc
-rw-r--r-- 1 dongjoon wheel 932 Jul 14 22:12
part-00043-2cd3a50e-eded-49a4-b7cf-94e3f090b8c1-c000.snappy.orc
-rw-r--r-- 1 dongjoon wheel 939 Jul 14 22:12
part-00191-2cd3a50e-eded-49a4-b7cf-94e3f090b8c1-c000.snappy.orc
{code}
was:
This founds during reviewing SPARK-32276.
*AFTER SPARK-32276*
{code}
scala> scala.util.Random.shuffle((1 to 100000).map(x => (x % 2, x))).toDF("a",
"b").repartition(2).createOrReplaceTempView("t")
scala> sql("select * from (select * from t order by b) distribute by
a").write.orc("/tmp/SPARK-32276")
$ ls -al /tmp/SPARK-32276/
total 632
drwxr-xr-x 10 dongjoon wheel 320 Jul 14 22:08 ./
drwxrwxrwt 14 root wheel 448 Jul 14 22:08 ../
-rw-r--r-- 1 dongjoon wheel 8 Jul 14 22:08 ._SUCCESS.crc
-rw-r--r-- 1 dongjoon wheel 12 Jul 14 22:08
.part-00000-ba5049f9-b835-49b7-9fdb-bdd11b9891cb-c000.snappy.orc.crc
-rw-r--r-- 1 dongjoon wheel 1188 Jul 14 22:08
.part-00043-ba5049f9-b835-49b7-9fdb-bdd11b9891cb-c000.snappy.orc.crc
-rw-r--r-- 1 dongjoon wheel 1188 Jul 14 22:08
.part-00191-ba5049f9-b835-49b7-9fdb-bdd11b9891cb-c000.snappy.orc.crc
-rw-r--r-- 1 dongjoon wheel 0 Jul 14 22:08 _SUCCESS
-rw-r--r-- 1 dongjoon wheel 119 Jul 14 22:08
part-00000-ba5049f9-b835-49b7-9fdb-bdd11b9891cb-c000.snappy.orc
-rw-r--r-- 1 dongjoon wheel 150735 Jul 14 22:08
part-00043-ba5049f9-b835-49b7-9fdb-bdd11b9891cb-c000.snappy.orc
-rw-r--r-- 1 dongjoon wheel 150741 Jul 14 22:08
part-00191-ba5049f9-b835-49b7-9fdb-bdd11b9891cb-c000.snappy.orc
{code}
*BEFORE*
{code}
scala> scala.util.Random.shuffle((1 to 100000).map(x => (x % 2, x))).toDF("a",
"b").repartition(2).createOrReplaceTempView("t")
scala> sql("select * from (select * from t order by b) distribute by
a").write.orc("/tmp/master")
$ ls -al /tmp/master/
total 56
drwxr-xr-x 10 dongjoon wheel 320 Jul 14 22:12 ./
drwxrwxrwt 15 root wheel 480 Jul 14 22:12 ../
-rw-r--r-- 1 dongjoon wheel 8 Jul 14 22:12 ._SUCCESS.crc
-rw-r--r-- 1 dongjoon wheel 12 Jul 14 22:12
.part-00000-2cd3a50e-eded-49a4-b7cf-94e3f090b8c1-c000.snappy.orc.crc
-rw-r--r-- 1 dongjoon wheel 16 Jul 14 22:12
.part-00043-2cd3a50e-eded-49a4-b7cf-94e3f090b8c1-c000.snappy.orc.crc
-rw-r--r-- 1 dongjoon wheel 16 Jul 14 22:12
.part-00191-2cd3a50e-eded-49a4-b7cf-94e3f090b8c1-c000.snappy.orc.crc
-rw-r--r-- 1 dongjoon wheel 0 Jul 14 22:12 _SUCCESS
-rw-r--r-- 1 dongjoon wheel 119 Jul 14 22:12
part-00000-2cd3a50e-eded-49a4-b7cf-94e3f090b8c1-c000.snappy.orc
-rw-r--r-- 1 dongjoon wheel 932 Jul 14 22:12
part-00043-2cd3a50e-eded-49a4-b7cf-94e3f090b8c1-c000.snappy.orc
-rw-r--r-- 1 dongjoon wheel 939 Jul 14 22:12
part-00191-2cd3a50e-eded-49a4-b7cf-94e3f090b8c1-c000.snappy.orc
{code}
> Add a test case to EliminateSortsSuite for protecting ORDER BY in DISTRIBUTE
> BY
> -------------------------------------------------------------------------------
>
> Key: SPARK-32318
> URL: https://issues.apache.org/jira/browse/SPARK-32318
> Project: Spark
> Issue Type: Test
> Components: SQL, Tests
> Affects Versions: 2.4.7, 3.0.1, 3.1.0
> Reporter: Dongjoon Hyun
> Priority: Minor
>
> This is found during reviewing SPARK-32276.
> *AFTER SPARK-32276*
> {code}
> scala> scala.util.Random.shuffle((1 to 100000).map(x => (x % 2,
> x))).toDF("a", "b").repartition(2).createOrReplaceTempView("t")
> scala> sql("select * from (select * from t order by b) distribute by
> a").write.orc("/tmp/SPARK-32276")
> $ ls -al /tmp/SPARK-32276/
> total 632
> drwxr-xr-x 10 dongjoon wheel 320 Jul 14 22:08 ./
> drwxrwxrwt 14 root wheel 448 Jul 14 22:08 ../
> -rw-r--r-- 1 dongjoon wheel 8 Jul 14 22:08 ._SUCCESS.crc
> -rw-r--r-- 1 dongjoon wheel 12 Jul 14 22:08
> .part-00000-ba5049f9-b835-49b7-9fdb-bdd11b9891cb-c000.snappy.orc.crc
> -rw-r--r-- 1 dongjoon wheel 1188 Jul 14 22:08
> .part-00043-ba5049f9-b835-49b7-9fdb-bdd11b9891cb-c000.snappy.orc.crc
> -rw-r--r-- 1 dongjoon wheel 1188 Jul 14 22:08
> .part-00191-ba5049f9-b835-49b7-9fdb-bdd11b9891cb-c000.snappy.orc.crc
> -rw-r--r-- 1 dongjoon wheel 0 Jul 14 22:08 _SUCCESS
> -rw-r--r-- 1 dongjoon wheel 119 Jul 14 22:08
> part-00000-ba5049f9-b835-49b7-9fdb-bdd11b9891cb-c000.snappy.orc
> -rw-r--r-- 1 dongjoon wheel 150735 Jul 14 22:08
> part-00043-ba5049f9-b835-49b7-9fdb-bdd11b9891cb-c000.snappy.orc
> -rw-r--r-- 1 dongjoon wheel 150741 Jul 14 22:08
> part-00191-ba5049f9-b835-49b7-9fdb-bdd11b9891cb-c000.snappy.orc
> {code}
> *BEFORE*
> {code}
> scala> scala.util.Random.shuffle((1 to 100000).map(x => (x % 2,
> x))).toDF("a", "b").repartition(2).createOrReplaceTempView("t")
> scala> sql("select * from (select * from t order by b) distribute by
> a").write.orc("/tmp/master")
> $ ls -al /tmp/master/
> total 56
> drwxr-xr-x 10 dongjoon wheel 320 Jul 14 22:12 ./
> drwxrwxrwt 15 root wheel 480 Jul 14 22:12 ../
> -rw-r--r-- 1 dongjoon wheel 8 Jul 14 22:12 ._SUCCESS.crc
> -rw-r--r-- 1 dongjoon wheel 12 Jul 14 22:12
> .part-00000-2cd3a50e-eded-49a4-b7cf-94e3f090b8c1-c000.snappy.orc.crc
> -rw-r--r-- 1 dongjoon wheel 16 Jul 14 22:12
> .part-00043-2cd3a50e-eded-49a4-b7cf-94e3f090b8c1-c000.snappy.orc.crc
> -rw-r--r-- 1 dongjoon wheel 16 Jul 14 22:12
> .part-00191-2cd3a50e-eded-49a4-b7cf-94e3f090b8c1-c000.snappy.orc.crc
> -rw-r--r-- 1 dongjoon wheel 0 Jul 14 22:12 _SUCCESS
> -rw-r--r-- 1 dongjoon wheel 119 Jul 14 22:12
> part-00000-2cd3a50e-eded-49a4-b7cf-94e3f090b8c1-c000.snappy.orc
> -rw-r--r-- 1 dongjoon wheel 932 Jul 14 22:12
> part-00043-2cd3a50e-eded-49a4-b7cf-94e3f090b8c1-c000.snappy.orc
> -rw-r--r-- 1 dongjoon wheel 939 Jul 14 22:12
> part-00191-2cd3a50e-eded-49a4-b7cf-94e3f090b8c1-c000.snappy.orc
> {code}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscr...@spark.apache.org
For additional commands, e-mail: issues-h...@spark.apache.org
