Author: hashutosh
Date: Thu Mar 13 23:06:16 2014
New Revision: 1577369
URL: http://svn.apache.org/r1577369
Log:
HIVE-6024 : Load data local inpath unnecessarily creates a copy task (Mohammad
Kamrul Islam via Ashutosh Chauhan)
Added:
hive/trunk/ql/src/test/queries/clientpositive/load_local_dir_test.q
hive/trunk/ql/src/test/results/clientpositive/load_local_dir_test.q.out
Modified:
hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java
hive/trunk/ql/src/test/results/clientpositive/input4.q.out
hive/trunk/ql/src/test/results/clientpositive/stats11.q.out
hive/trunk/ql/src/test/results/clientpositive/stats3.q.out
Modified:
hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java
URL:
http://svn.apache.org/viewvc/hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java?rev=1577369&r1=1577368&r2=1577369&view=diff
==============================================================================
---
hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java
(original)
+++
hive/trunk/itests/hive-unit/src/test/java/org/apache/hadoop/hive/ql/history/TestHiveHistory.java
Thu Mar 13 23:06:16 2014
@@ -103,7 +103,7 @@ public class TestHiveHistory extends Tes
db.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, src, true, true);
db.createTable(src, cols, null, TextInputFormat.class,
IgnoreKeyTextOutputFormat.class);
- db.loadTable(hadoopDataFile[i], src, false, false);
+ db.loadTable(hadoopDataFile[i], src, false, false, false);
i++;
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java?rev=1577369&r1=1577368&r2=1577369&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/exec/MoveTask.java Thu Mar
13 23:06:16 2014
@@ -238,13 +238,13 @@ public class MoveTask extends Task<MoveW
// Get all files from the src directory
FileStatus[] dirs;
ArrayList<FileStatus> files;
- FileSystem fs;
+ FileSystem srcFs; // source filesystem
try {
- fs = table.getDataLocation().getFileSystem(conf);
- dirs = fs.globStatus(tbd.getSourcePath());
+ srcFs = tbd.getSourcePath().getFileSystem(conf);
+ dirs = srcFs.globStatus(tbd.getSourcePath());
files = new ArrayList<FileStatus>();
for (int i = 0; (dirs != null && i < dirs.length); i++) {
- files.addAll(Arrays.asList(fs.listStatus(dirs[i].getPath())));
+ files.addAll(Arrays.asList(srcFs.listStatus(dirs[i].getPath())));
// We only check one file, so exit the loop when we have at least
// one.
if (files.size() > 0) {
@@ -258,7 +258,7 @@ public class MoveTask extends Task<MoveW
if (HiveConf.getBoolVar(conf,
HiveConf.ConfVars.HIVECHECKFILEFORMAT)) {
// Check if the file format of the file matches that of the table.
boolean flag = HiveFileFormatUtils.checkInputFormat(
- fs, conf, tbd.getTable().getInputFileFormatClass(), files);
+ srcFs, conf, tbd.getTable().getInputFileFormatClass(), files);
if (!flag) {
throw new HiveException(
"Wrong file format. Please check the file's format.");
@@ -271,7 +271,7 @@ public class MoveTask extends Task<MoveW
if (tbd.getPartitionSpec().size() == 0) {
dc = new DataContainer(table.getTTable());
db.loadTable(tbd.getSourcePath(), tbd.getTable()
- .getTableName(), tbd.getReplace(), tbd.getHoldDDLTime());
+ .getTableName(), tbd.getReplace(), tbd.getHoldDDLTime(),
work.isSrcLocal());
if (work.getOutputs() != null) {
work.getOutputs().add(new WriteEntity(table,
(tbd.getReplace() ? WriteEntity.WriteType.INSERT_OVERWRITE :
@@ -400,11 +400,13 @@ public class MoveTask extends Task<MoveW
db.validatePartitionNameCharacters(partVals);
db.loadPartition(tbd.getSourcePath(),
tbd.getTable().getTableName(),
tbd.getPartitionSpec(), tbd.getReplace(), tbd.getHoldDDLTime(),
- tbd.getInheritTableSpecs(), isSkewedStoredAsDirs(tbd));
- Partition partn = db.getPartition(table,
tbd.getPartitionSpec(), false);
-
- if (bucketCols != null || sortCols != null) {
- updatePartitionBucketSortColumns(table, partn, bucketCols,
numBuckets, sortCols);
+ tbd.getInheritTableSpecs(), isSkewedStoredAsDirs(tbd),
work.isSrcLocal());
+ Partition partn = db.getPartition(table, tbd.getPartitionSpec(),
+ false);
+
+ if (bucketCols != null || sortCols != null) {
+ updatePartitionBucketSortColumns(table, partn, bucketCols,
+ numBuckets, sortCols);
}
dc = new DataContainer(table.getTTable(),
partn.getTPartition());
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java?rev=1577369&r1=1577368&r2=1577369&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Hive.java Thu Mar
13 23:06:16 2014
@@ -47,6 +47,7 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FsShell;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
import org.apache.hadoop.hive.common.FileUtils;
import org.apache.hadoop.hive.common.HiveStatsUtils;
import org.apache.hadoop.hive.common.ObjectPair;
@@ -1184,12 +1185,15 @@ public class Hive {
* @param holdDDLTime if true, force [re]create the partition
* @param inheritTableSpecs if true, on [re]creating the partition, take the
* location/inputformat/outputformat/serde details from table spec
+ * @param isSrcLocal
+ * If the source directory is LOCAL
*/
public void loadPartition(Path loadPath, String tableName,
Map<String, String> partSpec, boolean replace, boolean holdDDLTime,
- boolean inheritTableSpecs, boolean isSkewedStoreAsSubdir)
- throws HiveException {
+ boolean inheritTableSpecs, boolean isSkewedStoreAsSubdir,
+ boolean isSrcLocal) throws HiveException {
Table tbl = getTable(tableName);
+ Path tblDataLocationPath = tbl.getDataLocation();
try {
/**
* Move files before creating the partition since down stream processes
@@ -1209,7 +1213,7 @@ public class Hive {
if (inheritTableSpecs) {
Path partPath = new Path(tbl.getDataLocation(),
Warehouse.makePartPath(partSpec));
- newPartPath = new Path(loadPath.toUri().getScheme(),
loadPath.toUri().getAuthority(),
+ newPartPath = new Path(tblDataLocationPath.toUri().getScheme(),
tblDataLocationPath.toUri().getAuthority(),
partPath.toUri().getPath());
if(oldPart != null) {
@@ -1231,10 +1235,11 @@ public class Hive {
}
if (replace) {
- Hive.replaceFiles(loadPath, newPartPath, oldPartPath, getConf());
+ Hive.replaceFiles(loadPath, newPartPath, oldPartPath, getConf(),
+ isSrcLocal);
} else {
FileSystem fs = tbl.getDataLocation().getFileSystem(conf);
- Hive.copyFiles(conf, loadPath, newPartPath, fs);
+ Hive.copyFiles(conf, loadPath, newPartPath, fs, isSrcLocal);
}
// recreate the partition if it existed before
@@ -1422,7 +1427,7 @@ private void constructOneLBLocationMap(F
// finally load the partition -- move the file to the final table
address
loadPartition(partPath, tableName, fullPartSpec, replace, holdDDLTime,
true,
- listBucketingEnabled);
+ listBucketingEnabled, false);
LOG.info("New loading path = " + partPath + " with partSpec " +
fullPartSpec);
}
return fullPartSpecs;
@@ -1444,15 +1449,16 @@ private void constructOneLBLocationMap(F
* @param replace
* if true - replace files in the table, otherwise add files to
table
* @param holdDDLTime
+ * @param isSrcLocal
+ * If the source directory is LOCAL
*/
public void loadTable(Path loadPath, String tableName, boolean replace,
- boolean holdDDLTime) throws HiveException {
+ boolean holdDDLTime, boolean isSrcLocal) throws HiveException {
Table tbl = getTable(tableName);
-
if (replace) {
- tbl.replaceFiles(loadPath);
+ tbl.replaceFiles(loadPath, isSrcLocal);
} else {
- tbl.copyFiles(loadPath);
+ tbl.copyFiles(loadPath, isSrcLocal);
}
if (!holdDDLTime) {
@@ -2098,9 +2104,9 @@ private void constructOneLBLocationMap(F
}
// for each file or directory in 'srcs', make mapping for every file in src
to safe name in dest
- private static List<List<Path[]>> checkPaths(HiveConf conf,
- FileSystem fs, FileStatus[] srcs, Path destf,
- boolean replace) throws HiveException {
+ private static List<List<Path[]>> checkPaths(HiveConf conf, FileSystem fs,
+ FileStatus[] srcs, FileSystem srcFs, Path destf, boolean replace)
+ throws HiveException {
List<List<Path[]>> result = new ArrayList<List<Path[]>>();
try {
@@ -2112,7 +2118,13 @@ private void constructOneLBLocationMap(F
for (FileStatus src : srcs) {
FileStatus[] items;
if (src.isDir()) {
- items = fs.listStatus(src.getPath());
+ items = srcFs.listStatus(src.getPath(), new PathFilter() {
+ @Override
+ public boolean accept(Path p) {
+ String name = p.getName();
+ return !name.startsWith("_") && !name.startsWith(".");
+ }
+ });
Arrays.sort(items);
} else {
items = new FileStatus[] {src};
@@ -2127,7 +2139,7 @@ private void constructOneLBLocationMap(F
// This check is redundant because temp files are removed by
// execution layer before
// calling loadTable/Partition. But leaving it in just in case.
- fs.delete(itemSource, true);
+ srcFs.delete(itemSource, true);
continue;
}
@@ -2189,8 +2201,8 @@ private void constructOneLBLocationMap(F
//method is called. when the replace value is true, this method works a
little different
//from mv command if the destf is a directory, it replaces the destf instead
of moving under
//the destf. in this case, the replaced destf still preserves the original
destf's permission
- static protected boolean renameFile(HiveConf conf, Path srcf, Path destf,
FileSystem fs,
- boolean replace) throws HiveException {
+ static protected boolean renameFile(HiveConf conf, Path srcf, Path destf,
+ FileSystem fs, boolean replace, boolean isSrcLocal) throws HiveException
{
boolean success = false;
boolean inheritPerms = HiveConf.getBoolVar(conf,
HiveConf.ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS);
@@ -2222,11 +2234,18 @@ private void constructOneLBLocationMap(F
}
}
}
- success = fs.rename(srcf, destf);
+ if (!isSrcLocal) {
+ // For NOT local src file, rename the file
+ success = fs.rename(srcf, destf);
+ } else {
+ // For local src file, copy to hdfs
+ fs.copyFromLocalFile(srcf, destf);
+ success = true;
+ }
LOG.info((replace ? "Replacing src:" : "Renaming src:") + srcf.toString()
+ ";dest: " + destf.toString() + ";Status:" + success);
} catch (IOException ioe) {
- throw new HiveException("Unable to move source" + srcf + " to
destination " + destf, ioe);
+ throw new HiveException("Unable to move source " + srcf + " to
destination " + destf, ioe);
}
if (success && inheritPerms) {
@@ -2243,8 +2262,8 @@ private void constructOneLBLocationMap(F
return success;
}
- static protected void copyFiles(HiveConf conf, Path srcf, Path destf,
FileSystem fs)
- throws HiveException {
+ static protected void copyFiles(HiveConf conf, Path srcf, Path destf,
+ FileSystem fs, boolean isSrcLocal) throws HiveException {
boolean inheritPerms = HiveConf.getBoolVar(conf,
HiveConf.ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS);
try {
@@ -2262,8 +2281,10 @@ private void constructOneLBLocationMap(F
}
FileStatus[] srcs;
+ FileSystem srcFs;
try {
- srcs = fs.globStatus(srcf);
+ srcFs = srcf.getFileSystem(conf);
+ srcs = srcFs.globStatus(srcf);
} catch (IOException e) {
LOG.error(StringUtils.stringifyException(e));
throw new HiveException("addFiles: filesystem error in check phase", e);
@@ -2274,14 +2295,14 @@ private void constructOneLBLocationMap(F
// srcs = new FileStatus[0]; Why is this needed?
}
// check that source and target paths exist
- List<List<Path[]>> result = checkPaths(conf, fs, srcs, destf, false);
-
+ List<List<Path[]>> result = checkPaths(conf, fs, srcs, srcFs, destf,
false);
// move it, move it
try {
for (List<Path[]> sdpairs : result) {
for (Path[] sdpair : sdpairs) {
- if (!renameFile(conf, sdpair[0], sdpair[1], fs, false)) {
- throw new IOException("Cannot move " + sdpair[0] + " to " +
sdpair[1]);
+ if (!renameFile(conf, sdpair[0], sdpair[1], fs, false, isSrcLocal)) {
+ throw new IOException("Cannot move " + sdpair[0] + " to "
+ + sdpair[1]);
}
}
}
@@ -2304,18 +2325,22 @@ private void constructOneLBLocationMap(F
* The directory where the final data needs to go
* @param oldPath
* The directory where the old data location, need to be cleaned up.
+ * @param isSrcLocal
+ * If the source directory is LOCAL
*/
- static protected void replaceFiles(Path srcf, Path destf, Path oldPath,
HiveConf conf)
- throws HiveException {
+ static protected void replaceFiles(Path srcf, Path destf, Path oldPath,
+ HiveConf conf, boolean isSrcLocal) throws HiveException {
try {
- FileSystem fs = srcf.getFileSystem(conf);
+ FileSystem destFs = destf.getFileSystem(conf);
boolean inheritPerms = HiveConf.getBoolVar(conf,
HiveConf.ConfVars.HIVE_WAREHOUSE_SUBDIR_INHERIT_PERMS);
// check if srcf contains nested sub-directories
FileStatus[] srcs;
+ FileSystem srcFs;
try {
- srcs = fs.globStatus(srcf);
+ srcFs = srcf.getFileSystem(conf);
+ srcs = srcFs.globStatus(srcf);
} catch (IOException e) {
throw new HiveException("Getting globStatus " + srcf.toString(), e);
}
@@ -2323,7 +2348,8 @@ private void constructOneLBLocationMap(F
LOG.info("No sources specified to move: " + srcf);
return;
}
- List<List<Path[]>> result = checkPaths(conf, fs, srcs, destf, true);
+ List<List<Path[]>> result = checkPaths(conf, destFs, srcs, srcFs, destf,
+ true);
if (oldPath != null) {
try {
@@ -2344,35 +2370,37 @@ private void constructOneLBLocationMap(F
if (srcs.length == 1 && srcs[0].isDir()) {
// rename can fail if the parent doesn't exist
Path destfp = destf.getParent();
- if (!fs.exists(destfp)) {
- boolean success = fs.mkdirs(destfp);
+ if (!destFs.exists(destfp)) {
+ boolean success = destFs.mkdirs(destfp);
if (!success) {
LOG.warn("Error creating directory " + destf.toString());
}
if (inheritPerms && success) {
- fs.setPermission(destfp,
fs.getFileStatus(destfp.getParent()).getPermission());
+ destFs.setPermission(destfp,
destFs.getFileStatus(destfp.getParent()).getPermission());
}
}
- boolean b = renameFile(conf, srcs[0].getPath(), destf, fs, true);
+ boolean b = renameFile(conf, srcs[0].getPath(), destf, destFs, true,
+ isSrcLocal);
if (!b) {
throw new HiveException("Unable to move results from " +
srcs[0].getPath()
+ " to destination directory: " + destf);
}
} else { // srcf is a file or pattern containing wildcards
- if (!fs.exists(destf)) {
- boolean success = fs.mkdirs(destf);
+ if (!destFs.exists(destf)) {
+ boolean success = destFs.mkdirs(destf);
if (!success) {
LOG.warn("Error creating directory " + destf.toString());
}
if (inheritPerms && success) {
- fs.setPermission(destf,
fs.getFileStatus(destf.getParent()).getPermission());
+ destFs.setPermission(destf,
destFs.getFileStatus(destf.getParent()).getPermission());
}
}
// srcs must be a list of files -- ensured by LoadSemanticAnalyzer
for (List<Path[]> sdpairs : result) {
for (Path[] sdpair : sdpairs) {
- if (!renameFile(conf, sdpair[0], sdpair[1], fs, true)) {
+ if (!renameFile(conf, sdpair[0], sdpair[1], destFs, true,
+ isSrcLocal)) {
throw new IOException("Error moving: " + sdpair[0] + " into: " +
sdpair[1]);
}
}
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java?rev=1577369&r1=1577368&r2=1577369&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/metadata/Table.java Thu
Mar 13 23:06:16 2014
@@ -655,10 +655,14 @@ public class Table implements Serializab
*
* @param srcf
* Source directory
+ * @param isSrcLocal
+ * If the source directory is LOCAL
*/
- protected void replaceFiles(Path srcf) throws HiveException {
+ protected void replaceFiles(Path srcf, boolean isSrcLocal)
+ throws HiveException {
Path tableDest = getPath();
- Hive.replaceFiles(srcf, tableDest, tableDest, Hive.get().getConf());
+ Hive.replaceFiles(srcf, tableDest, tableDest, Hive.get().getConf(),
+ isSrcLocal);
}
/**
@@ -666,12 +670,14 @@ public class Table implements Serializab
*
* @param srcf
* Files to be moved. Leaf directories or globbed file paths
+ * @param isSrcLocal
+ * If the source directory is LOCAL
*/
- protected void copyFiles(Path srcf) throws HiveException {
+ protected void copyFiles(Path srcf, boolean isSrcLocal) throws HiveException
{
FileSystem fs;
try {
fs = getDataLocation().getFileSystem(Hive.get().getConf());
- Hive.copyFiles(Hive.get().getConf(), srcf, getPath(), fs);
+ Hive.copyFiles(Hive.get().getConf(), srcf, getPath(), fs, isSrcLocal);
} catch (IOException e) {
throw new HiveException("addFiles: filesystem error in check phase", e);
}
Modified:
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java?rev=1577369&r1=1577368&r2=1577369&view=diff
==============================================================================
---
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
(original)
+++
hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/parse/LoadSemanticAnalyzer.java
Thu Mar 13 23:06:16 2014
@@ -35,7 +35,6 @@ import org.apache.hadoop.hive.ql.hooks.W
import org.apache.hadoop.hive.ql.metadata.Hive;
import org.apache.hadoop.hive.ql.metadata.HiveException;
import org.apache.hadoop.hive.ql.metadata.Partition;
-import org.apache.hadoop.hive.ql.plan.CopyWork;
import org.apache.hadoop.hive.ql.plan.LoadTableDesc;
import org.apache.hadoop.hive.ql.plan.MoveWork;
import org.apache.hadoop.hive.ql.plan.StatsWork;
@@ -234,18 +233,6 @@ public class LoadSemanticAnalyzer extend
inputs.add(new ReadEntity(new Path(fromURI), isLocal));
Task<? extends Serializable> rTask = null;
- // create copy work
- if (isLocal) {
- // if the local keyword is specified - we will always make a copy. this
- // might seem redundant in the case
- // that the hive warehouse is also located in the local file system - but
- // that's just a test case.
- String copyURIStr = ctx.getExternalTmpPath(toURI).toString();
- URI copyURI = URI.create(copyURIStr);
- rTask = TaskFactory.get(new CopyWork(new Path(fromURI), new
Path(copyURI)), conf);
- fromURI = copyURI;
- }
-
// create final load/move work
Map<String, String> partSpec = ts.getPartSpec();
@@ -281,7 +268,7 @@ public class LoadSemanticAnalyzer extend
Utilities.getTableDesc(ts.tableHandle), partSpec, isOverWrite);
Task<? extends Serializable> childTask = TaskFactory.get(new
MoveWork(getInputs(),
- getOutputs(), loadTableWork, null, true), conf);
+ getOutputs(), loadTableWork, null, true, isLocal), conf);
if (rTask != null) {
rTask.addDependentTask(childTask);
} else {
Modified: hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java?rev=1577369&r1=1577368&r2=1577369&view=diff
==============================================================================
--- hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java
(original)
+++ hive/trunk/ql/src/java/org/apache/hadoop/hive/ql/plan/MoveWork.java Thu Mar
13 23:06:16 2014
@@ -38,6 +38,7 @@ public class MoveWork implements Seriali
private LoadMultiFilesDesc loadMultiFilesWork;
private boolean checkFileFormat;
+ private boolean srcLocal;
/**
* ReadEntitites that are passed to the hooks.
@@ -63,6 +64,16 @@ public class MoveWork implements Seriali
public MoveWork(HashSet<ReadEntity> inputs, HashSet<WriteEntity> outputs,
final LoadTableDesc loadTableWork, final LoadFileDesc loadFileWork,
+ boolean checkFileFormat, boolean srcLocal) {
+ this(inputs, outputs);
+ this.loadTableWork = loadTableWork;
+ this.loadFileWork = loadFileWork;
+ this.checkFileFormat = checkFileFormat;
+ this.srcLocal = srcLocal;
+ }
+
+ public MoveWork(HashSet<ReadEntity> inputs, HashSet<WriteEntity> outputs,
+ final LoadTableDesc loadTableWork, final LoadFileDesc loadFileWork,
boolean checkFileFormat) {
this(inputs, outputs);
this.loadTableWork = loadTableWork;
@@ -121,4 +132,12 @@ public class MoveWork implements Seriali
this.outputs = outputs;
}
+ public boolean isSrcLocal() {
+ return srcLocal;
+ }
+
+ public void setSrcLocal(boolean srcLocal) {
+ this.srcLocal = srcLocal;
+ }
+
}
Modified:
hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java?rev=1577369&r1=1577368&r2=1577369&view=diff
==============================================================================
--- hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java
(original)
+++ hive/trunk/ql/src/test/org/apache/hadoop/hive/ql/exec/TestExecDriver.java
Thu Mar 13 23:06:16 2014
@@ -125,7 +125,7 @@ public class TestExecDriver extends Test
db.dropTable(MetaStoreUtils.DEFAULT_DATABASE_NAME, src, true, true);
db.createTable(src, cols, null, TextInputFormat.class,
IgnoreKeyTextOutputFormat.class);
- db.loadTable(hadoopDataFile[i], src, false, false);
+ db.loadTable(hadoopDataFile[i], src, false, false, true);
i++;
}
Added: hive/trunk/ql/src/test/queries/clientpositive/load_local_dir_test.q
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/queries/clientpositive/load_local_dir_test.q?rev=1577369&view=auto
==============================================================================
--- hive/trunk/ql/src/test/queries/clientpositive/load_local_dir_test.q (added)
+++ hive/trunk/ql/src/test/queries/clientpositive/load_local_dir_test.q Thu Mar
13 23:06:16 2014
@@ -0,0 +1,6 @@
+
+create table load_local (id INT);
+
+load data local inpath '../../data/files/ext_test/' into table load_local;
+
+select * from load_local;
Modified: hive/trunk/ql/src/test/results/clientpositive/input4.q.out
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/input4.q.out?rev=1577369&r1=1577368&r2=1577369&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/input4.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/input4.q.out Thu Mar 13
23:06:16 2014
@@ -14,14 +14,9 @@ POSTHOOK: type: LOAD
STAGE DEPENDENCIES:
Stage-0 is a root stage
Stage-1 depends on stages: Stage-0
- Stage-2 depends on stages: Stage-1
STAGE PLANS:
Stage: Stage-0
- Copy
-#### A masked pattern was here ####
-
- Stage: Stage-1
Move Operator
tables:
replace: false
@@ -31,7 +26,7 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.input4
- Stage: Stage-2
+ Stage: Stage-1
Stats-Aggr Operator
PREHOOK: query: LOAD DATA LOCAL INPATH '../../data/files/kv1.txt' INTO TABLE
INPUT4
Added: hive/trunk/ql/src/test/results/clientpositive/load_local_dir_test.q.out
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/load_local_dir_test.q.out?rev=1577369&view=auto
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/load_local_dir_test.q.out
(added)
+++ hive/trunk/ql/src/test/results/clientpositive/load_local_dir_test.q.out Thu
Mar 13 23:06:16 2014
@@ -0,0 +1,29 @@
+PREHOOK: query: create table load_local (id INT)
+PREHOOK: type: CREATETABLE
+PREHOOK: Output: database:default
+POSTHOOK: query: create table load_local (id INT)
+POSTHOOK: type: CREATETABLE
+POSTHOOK: Output: database:default
+POSTHOOK: Output: default@load_local
+PREHOOK: query: load data local inpath '../../data/files/ext_test/' into table
load_local
+PREHOOK: type: LOAD
+#### A masked pattern was here ####
+PREHOOK: Output: default@load_local
+POSTHOOK: query: load data local inpath '../../data/files/ext_test/' into
table load_local
+POSTHOOK: type: LOAD
+#### A masked pattern was here ####
+POSTHOOK: Output: default@load_local
+PREHOOK: query: select * from load_local
+PREHOOK: type: QUERY
+PREHOOK: Input: default@load_local
+#### A masked pattern was here ####
+POSTHOOK: query: select * from load_local
+POSTHOOK: type: QUERY
+POSTHOOK: Input: default@load_local
+#### A masked pattern was here ####
+1
+2
+3
+4
+5
+6
Modified: hive/trunk/ql/src/test/results/clientpositive/stats11.q.out
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/stats11.q.out?rev=1577369&r1=1577368&r2=1577369&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/stats11.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/stats11.q.out Thu Mar 13
23:06:16 2014
@@ -37,14 +37,9 @@ POSTHOOK: type: LOAD
STAGE DEPENDENCIES:
Stage-0 is a root stage
Stage-1 depends on stages: Stage-0
- Stage-2 depends on stages: Stage-1
STAGE PLANS:
Stage: Stage-0
- Copy
-#### A masked pattern was here ####
-
- Stage: Stage-1
Move Operator
tables:
partition:
@@ -56,7 +51,7 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.srcbucket_mapjoin_part
- Stage: Stage-2
+ Stage: Stage-1
Stats-Aggr Operator
PREHOOK: query: load data local inpath '../../data/files/srcbucket20.txt' INTO
TABLE srcbucket_mapjoin_part partition(ds='2008-04-08')
Modified: hive/trunk/ql/src/test/results/clientpositive/stats3.q.out
URL:
http://svn.apache.org/viewvc/hive/trunk/ql/src/test/results/clientpositive/stats3.q.out?rev=1577369&r1=1577368&r2=1577369&view=diff
==============================================================================
--- hive/trunk/ql/src/test/results/clientpositive/stats3.q.out (original)
+++ hive/trunk/ql/src/test/results/clientpositive/stats3.q.out Thu Mar 13
23:06:16 2014
@@ -33,14 +33,9 @@ TOK_LOAD
STAGE DEPENDENCIES:
Stage-0 is a root stage
Stage-1 depends on stages: Stage-0
- Stage-2 depends on stages: Stage-1
STAGE PLANS:
Stage: Stage-0
- Copy
-#### A masked pattern was here ####
-
- Stage: Stage-1
Move Operator
tables:
replace: true
@@ -61,7 +56,7 @@ STAGE PLANS:
serde: org.apache.hadoop.hive.serde2.lazy.LazySimpleSerDe
name: default.hive_test_src
- Stage: Stage-2
+ Stage: Stage-1
Stats-Aggr Operator
PREHOOK: query: load data local inpath '../../data/files/test.dat' overwrite
into table hive_test_src
