[
https://issues.apache.org/jira/browse/HIVE-29754?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
Indhumathi Muthumurugesh updated HIVE-29754:
--------------------------------------------
Description:
When executing INSERT OVERWRITE ... UNION ALL using the Tez execution engine,
an empty temporary directory named -tmp.HIVE_UNION_SUBDIR_<N>.moved is left
behind in the final partition directory after the query completes successfully.
This does not occur with MapReduce execution engine.
Steps to Reproduce
1. Create two tables and a partitioned output table:
{code:java}
CREATE TABLE src_prev (id INT, val STRING) STORED AS TEXTFILE;
CREATE TABLE src_today (id INT, val STRING) STORED AS TEXTFILE;
CREATE TABLE output (id INT, val STRING)
PARTITIONED BY (run_dt STRING) STORED AS TEXTFILE; {code}
2. Load data such that ALL rows in src_prev also exist in src_today
{code:java}
INSERT INTO src_prev VALUES (1,'aaa'), (2,'bbb');
INSERT INTO src_today VALUES (1,'aaa'), (2,'bbb'), (3,'ccc'); {code}
3. Run INSERT OVERWRITE with UNION ALL using Tez:
{code:java}
INSERT OVERWRITE TABLE output PARTITION (run_dt='2024-01-01')
SELECT p.id, p.val
FROM src_prev p
LEFT OUTER JOIN src_today t ON p.id = t.id
WHERE t.id IS NULL -- Branch 1: anti-join, returns 0 rows
UNION ALL
SELECT id, val FROM src_today; -- Branch 2: returns rows {code}
4. Inspect the output partition on HDFS:
{code:java}
hdfs dfs -ls /warehouse/output/run_dt=2024-01-01/ {code}
Actual Result:
{code:java}
drwxr-xr-x -tmp.HIVE_UNION_SUBDIR_1.moved <- leftover empty directory
-rw-r--r-- 1_000001_0
-rw-r--r-- 2_000001_0 {code}
Expected Result:
{code:java}
-rw-r--r-- 1_000001_0
-rw-r--r-- 2_000001_0 {code}
No temporary directories in the final partition location.
*– Query on this table from spark engine fails with FileNotFountException*
was:
When executing INSERT OVERWRITE ... UNION ALL using the Tez execution engine,
an empty temporary directory named -tmp.HIVE_UNION_SUBDIR_<N>.moved is left
behind in the final partition directory after the query completes successfully.
This does not occur with MapReduce execution engine.
Steps to Reproduce
1. Create two tables and a partitioned output table:
{code:java}
CREATE TABLE src_prev (id INT, val STRING) STORED AS TEXTFILE;
CREATE TABLE src_today (id INT, val STRING) STORED AS TEXTFILE;
CREATE TABLE output (id INT, val STRING)
PARTITIONED BY (run_dt STRING) STORED AS TEXTFILE; {code}
2. Load data such that ALL rows in src_prev also exist in src_today
{code:java}
INSERT INTO src_prev VALUES (1,'aaa'), (2,'bbb');
INSERT INTO src_today VALUES (1,'aaa'), (2,'bbb'), (3,'ccc'); {code}
3. Run INSERT OVERWRITE with UNION ALL using Tez:
{code:java}
INSERT OVERWRITE TABLE output PARTITION (run_dt='2024-01-01')
SELECT p.id, p.val
FROM src_prev p
LEFT OUTER JOIN src_today t ON p.id = t.id
WHERE t.id IS NULL -- Branch 1: anti-join, returns 0 rows
UNION ALL
SELECT id, val FROM src_today; -- Branch 2: returns rows {code}
4. Inspect the output partition on HDFS:
{code:java}
hdfs dfs -ls /warehouse/output/run_dt=2024-01-01/ {code}
Actual Result:
{code:java}
drwxr-xr-x -tmp.HIVE_UNION_SUBDIR_1.moved <- leftover empty directory
-rw-r--r-- 1_000001_0
-rw-r--r-- 2_000001_0 {code}
Expected Result:
{code:java}
-rw-r--r-- 1_000001_0
-rw-r--r-- 2_000001_0 {code}
No temporary directories in the final partition location.
*– Query on this table from spark engine fails with FileNotFountException*
-- Test: INSERT OVERWRITE ... UNION ALL does not leave -tmp.*.moved staging
-- directories in the partition when one UNION branch produces 0 rows.
--
-- Bug: When a UNION ALL branch returns 0 rows, Utilities.mvFileToFinalPath
-- renamed the staging dir to -tmp.*.moved but never deleted it, leaving an
-- empty directory in the final partition location.
--
-- Fix: Added cleanup in the statuses.length == 0 path in mvFileToFinalPath.
--
-- Scenario A: both UNION branches produce rows (normal case)
-- Scenario B: first branch (anti-join) produces 0 rows → verifies the fix
set hive.execution.engine=tez;
set hive.mapred.mode=nonstrict;
set hive.stats.autogather=false;
set hive.optimize.union.remove=false;
-- -----------------------------------------------------------------------
-- Setup
-- -----------------------------------------------------------------------
DROP TABLE IF EXISTS union_src_today;
DROP TABLE IF EXISTS union_src_prev;
DROP TABLE IF EXISTS union_output;
CREATE TABLE union_src_today (id INT, val STRING) STORED AS TEXTFILE;
CREATE TABLE union_src_prev (id INT, val STRING) STORED AS TEXTFILE;
CREATE TABLE union_output (id INT, val STRING)
PARTITIONED BY (run_dt STRING)
STORED AS TEXTFILE;
-- -----------------------------------------------------------------------
-- Scenario A: Both branches produce rows
-- prev has rows 1,2,3 — today has rows 3,4 — anti-join passes rows 1,2
-- Branch 1 (anti-join): rows from prev NOT in today → ids 1,2
-- Branch 2 (today): all rows from today → ids 3,4
-- -----------------------------------------------------------------------
INSERT INTO union_src_prev VALUES (1,'aaa'), (2,'bbb'), (3,'ccc');
INSERT INTO union_src_today VALUES (3,'ccc_new'), (4,'ddd');
INSERT OVERWRITE TABLE union_output PARTITION (run_dt='2024-01-02')
SELECT p.id, p.val
FROM union_src_prev p
LEFT OUTER JOIN union_src_today t ON p.id = t.id
WHERE t.id IS NULL
UNION ALL
SELECT id, val
FROM union_src_today;
-- Partition must contain exactly the output rows (ids 1,2,3,4)
SELECT id, val FROM union_output WHERE run_dt='2024-01-02' ORDER BY id;
-- Partition directory must NOT contain any -tmp.*.moved directories
dfs -ls
${hiveconf:hive.metastore.warehouse.dir}/union_output/run_dt=2024-01-02/;
-- -----------------------------------------------------------------------
-- Scenario B: First branch produces 0 rows (all prev rows exist in today)
-- prev has rows 3,4 — today has rows 3,4 — anti-join returns nothing
-- Branch 1 (anti-join): 0 rows ← triggers the bug
-- Branch 2 (today): rows 3,4
-- -----------------------------------------------------------------------
TRUNCATE TABLE union_src_prev;
TRUNCATE TABLE union_src_today;
INSERT INTO union_src_prev VALUES (3,'ccc'), (4,'ddd');
INSERT INTO union_src_today VALUES (3,'ccc_new'), (4,'ddd_new');
INSERT OVERWRITE TABLE union_output PARTITION (run_dt='2024-01-03')
SELECT p.id, p.val
FROM union_src_prev p
LEFT OUTER JOIN union_src_today t ON p.id = t.id
WHERE t.id IS NULL
UNION ALL
SELECT id, val
FROM union_src_today;
-- Partition must contain only Branch 2 rows (ids 3,4)
SELECT id, val FROM union_output WHERE run_dt='2024-01-03' ORDER BY id;
-- Partition directory must NOT contain any -tmp.*.moved directories.
-- Before the fix, -tmp.HIVE_UNION_SUBDIR_1.moved would appear here.
dfs -ls
${hiveconf:hive.metastore.warehouse.dir}/union_output/run_dt=2024-01-03/;
-- -----------------------------------------------------------------------
-- Cleanup
-- -----------------------------------------------------------------------
DROP TABLE IF EXISTS union_src_today;
DROP TABLE IF EXISTS union_src_prev;
DROP TABLE IF EXISTS union_output;
> Tez leaves empty -tmp.HIVE_UNION_SUBDIR_N.moved directory in partition after
> INSERT OVERWRITE ... UNION ALL when one branch produces 0 rows
> --------------------------------------------------------------------------------------------------------------------------------------------
>
> Key: HIVE-29754
> URL: https://issues.apache.org/jira/browse/HIVE-29754
> Project: Hive
> Issue Type: Bug
> Components: Tez
> Reporter: Indhumathi Muthumurugesh
> Assignee: Indhumathi Muthumurugesh
> Priority: Major
>
> When executing INSERT OVERWRITE ... UNION ALL using the Tez execution engine,
> an empty temporary directory named -tmp.HIVE_UNION_SUBDIR_<N>.moved is left
> behind in the final partition directory after the query completes
> successfully.
> This does not occur with MapReduce execution engine.
> Steps to Reproduce
> 1. Create two tables and a partitioned output table:
> {code:java}
> CREATE TABLE src_prev (id INT, val STRING) STORED AS TEXTFILE;
> CREATE TABLE src_today (id INT, val STRING) STORED AS TEXTFILE;
> CREATE TABLE output (id INT, val STRING)
> PARTITIONED BY (run_dt STRING) STORED AS TEXTFILE; {code}
> 2. Load data such that ALL rows in src_prev also exist in src_today
> {code:java}
> INSERT INTO src_prev VALUES (1,'aaa'), (2,'bbb');
> INSERT INTO src_today VALUES (1,'aaa'), (2,'bbb'), (3,'ccc'); {code}
> 3. Run INSERT OVERWRITE with UNION ALL using Tez:
> {code:java}
> INSERT OVERWRITE TABLE output PARTITION (run_dt='2024-01-01')
> SELECT p.id, p.val
> FROM src_prev p
> LEFT OUTER JOIN src_today t ON p.id = t.id
> WHERE t.id IS NULL -- Branch 1: anti-join, returns 0 rows
> UNION ALL
> SELECT id, val FROM src_today; -- Branch 2: returns rows {code}
> 4. Inspect the output partition on HDFS:
> {code:java}
> hdfs dfs -ls /warehouse/output/run_dt=2024-01-01/ {code}
> Actual Result:
> {code:java}
> drwxr-xr-x -tmp.HIVE_UNION_SUBDIR_1.moved <- leftover empty directory
> -rw-r--r-- 1_000001_0
> -rw-r--r-- 2_000001_0 {code}
> Expected Result:
> {code:java}
> -rw-r--r-- 1_000001_0
> -rw-r--r-- 2_000001_0 {code}
> No temporary directories in the final partition location.
>
> *– Query on this table from spark engine fails with FileNotFountException*
>
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
