[
https://issues.apache.org/jira/browse/HIVE-25932?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel
]
lkl updated HIVE-25932:
-----------------------
Description:
modify the class *"Hive.java "*
{code:java}
for (final FileStatus srcStatus : srcs) {
// final Path destFile = new Path(destf,
srcStatus.getPath().getName());
final String name = srcStatus.getPath().getName();
final Path destDirPath = srcStatus.getPath();
if (null == pool) {
Path destFile = new Path(destf, name);
int counter = 1;
while (!destFs.rename(destDirPath, destFile)) {
destFile = new Path(destf, name + ("_copy_" + counter));
LOG.info("kugu log destFile is {}.",destFile.getName());
counter++;
// throw new IOException("rename for src path: " +
srcStatus.getPath() + " to dest:"
// + destf + " returned false");
}
} else {
futures.add(pool.submit(new Callable<Void>() {
@Override
public Void call() throws Exception {
SessionState.setCurrentSessionState(parentSession);
final String group = srcStatus.getGroup();
Path destFile = new Path(destf, name);
boolean rename_succ = false;
int counter = 1;
while (!rename_succ) {
rename_succ = destFs.rename(destDirPath, destFile);
if(rename_succ) {
if (inheritPerms) {
HdfsUtils.setFullFileStatus(conf, desiredStatus,
group, destFs, destFile, false);
}
}else {
destFile = new Path(destf, name + ("_copy_" +
counter));
LOG.info("kugu log destFile is {}.",destFile.getName());
}
counter++;
}
// if(destFs.rename(srcStatus.getPath(), destFile)) {
// if (inheritPerms) {
// HdfsUtils.setFullFileStatus(conf, desiredStatus,
group, destFs, destFile, false);
// }
// } else {
// throw new IOException("rename for src path: " +
srcStatus.getPath() + " to dest path:"
// + destFile + " returned false");
// }
return null;
}
{code}
was:
Hive.java
{code:java}
for (final FileStatus srcStatus : srcs) {
// final Path destFile = new Path(destf,
srcStatus.getPath().getName());
final String name = srcStatus.getPath().getName();
final Path destDirPath = srcStatus.getPath();
if (null == pool) {
Path destFile = new Path(destf, name);
int counter = 1;
while (!destFs.rename(destDirPath, destFile)) {
destFile = new Path(destf, name + ("_copy_" + counter));
LOG.info("kugu log destFile is {}.",destFile.getName());
counter++;
// throw new IOException("rename for src path: " +
srcStatus.getPath() + " to dest:"
// + destf + " returned false");
}
} else {
futures.add(pool.submit(new Callable<Void>() {
@Override
public Void call() throws Exception {
SessionState.setCurrentSessionState(parentSession);
final String group = srcStatus.getGroup();
Path destFile = new Path(destf, name);
boolean rename_succ = false;
int counter = 1;
while (!rename_succ) {
rename_succ = destFs.rename(destDirPath, destFile);
if(rename_succ) {
if (inheritPerms) {
HdfsUtils.setFullFileStatus(conf, desiredStatus,
group, destFs, destFile, false);
}
}else {
destFile = new Path(destf, name + ("_copy_" +
counter));
LOG.info("kugu log destFile is {}.",destFile.getName());
}
counter++;
}
// if(destFs.rename(srcStatus.getPath(), destFile)) {
// if (inheritPerms) {
// HdfsUtils.setFullFileStatus(conf, desiredStatus,
group, destFs, destFile, false);
// }
// } else {
// throw new IOException("rename for src path: " +
srcStatus.getPath() + " to dest path:"
// + destFile + " returned false");
// }
return null;
}
{code}
> two or more sql like "insert into table test paritition (cls=1) select xxx"
> sometimes one sql was failed with rename ,because the two sql create a same
> file name
> -----------------------------------------------------------------------------------------------------------------------------------------------------------------
>
> Key: HIVE-25932
> URL: https://issues.apache.org/jira/browse/HIVE-25932
> Project: Hive
> Issue Type: Improvement
> Components: Query Planning, Query Processor
> Affects Versions: 2.3.7
> Reporter: lkl
> Assignee: lkl
> Priority: Minor
>
> modify the class *"Hive.java "*
> {code:java}
> for (final FileStatus srcStatus : srcs) {
> // final Path destFile = new Path(destf,
> srcStatus.getPath().getName());
> final String name = srcStatus.getPath().getName();
> final Path destDirPath = srcStatus.getPath();
> if (null == pool) {
> Path destFile = new Path(destf, name);
> int counter = 1;
> while (!destFs.rename(destDirPath, destFile)) {
> destFile = new Path(destf, name + ("_copy_" + counter));
> LOG.info("kugu log destFile is {}.",destFile.getName());
> counter++;
> // throw new IOException("rename for src path: " +
> srcStatus.getPath() + " to dest:"
> // + destf + " returned false");
> }
> } else {
> futures.add(pool.submit(new Callable<Void>() {
> @Override
> public Void call() throws Exception {
> SessionState.setCurrentSessionState(parentSession);
> final String group = srcStatus.getGroup();
> Path destFile = new Path(destf, name);
> boolean rename_succ = false;
> int counter = 1;
> while (!rename_succ) {
> rename_succ = destFs.rename(destDirPath, destFile);
> if(rename_succ) {
> if (inheritPerms) {
> HdfsUtils.setFullFileStatus(conf, desiredStatus,
> group, destFs, destFile, false);
> }
> }else {
> destFile = new Path(destf, name + ("_copy_" +
> counter));
> LOG.info("kugu log destFile is
> {}.",destFile.getName());
> }
> counter++;
> }
> // if(destFs.rename(srcStatus.getPath(), destFile)) {
> // if (inheritPerms) {
> // HdfsUtils.setFullFileStatus(conf, desiredStatus,
> group, destFs, destFile, false);
> // }
> // } else {
> // throw new IOException("rename for src path: " +
> srcStatus.getPath() + " to dest path:"
> // + destFile + " returned false");
> // }
> return null;
> }
> {code}
--
This message was sent by Atlassian Jira
(v8.20.1#820001)
