olga
Thu, 29 Oct 2009 17:29:10 -0700
Author: olga Date: Fri Oct 30 00:28:41 2009 New Revision: 831169 URL: http://svn.apache.org/viewvc?rev=831169&view=rev Log: PIG-1059: FINDBUGS: remaining Bad practice + Multithreaded correctness Warning (olgan) Modified: hadoop/pig/trunk/CHANGES.txt hadoop/pig/trunk/src/org/apache/pig/PigWarning.java hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PhyPlanSetter.java hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/PhysicalPlan.java hadoop/pig/trunk/src/org/apache/pig/backend/local/datastorage/LocalPath.java hadoop/pig/trunk/src/org/apache/pig/backend/local/executionengine/physicalLayer/relationalOperators/POCogroup.java hadoop/pig/trunk/src/org/apache/pig/builtin/BinStorage.java hadoop/pig/trunk/src/org/apache/pig/builtin/BinaryStorage.java hadoop/pig/trunk/src/org/apache/pig/builtin/PigStorage.java hadoop/pig/trunk/src/org/apache/pig/data/DefaultAbstractBag.java hadoop/pig/trunk/src/org/apache/pig/data/DistinctDataBag.java hadoop/pig/trunk/src/org/apache/pig/data/InternalSortedBag.java hadoop/pig/trunk/src/org/apache/pig/data/NonSpillableDataBag.java hadoop/pig/trunk/src/org/apache/pig/data/SingleTupleBag.java hadoop/pig/trunk/src/org/apache/pig/data/SortedDataBag.java hadoop/pig/trunk/src/org/apache/pig/data/TargetedTuple.java hadoop/pig/trunk/src/org/apache/pig/impl/io/FileLocalizer.java hadoop/pig/trunk/src/org/apache/pig/impl/streaming/StreamingCommand.java hadoop/pig/trunk/test/findbugsExcludeFile.xml Modified: hadoop/pig/trunk/CHANGES.txt URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/CHANGES.txt?rev=831169&r1=831168&r2=831169&view=diff ============================================================================== --- hadoop/pig/trunk/CHANGES.txt (original) +++ hadoop/pig/trunk/CHANGES.txt Fri Oct 30 00:28:41 2009 @@ -26,6 +26,8 @@ IMPROVEMENTS +PIG-1059: FINDBUGS: remaining Bad practice + Multithreaded correctness Warning (olgan) + PIG-953: Enable merge join in pig to work with loaders and store functions which can internally index sorted data (pradeepkth) Modified: hadoop/pig/trunk/src/org/apache/pig/PigWarning.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/PigWarning.java?rev=831169&r1=831168&r2=831169&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/PigWarning.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/PigWarning.java Fri Oct 30 00:28:41 2009 @@ -60,5 +60,6 @@ UNREACHABLE_CODE_BOTH_MAP_AND_REDUCE_PLANS_PROCESSED, USING_OVERLOADED_FUNCTION, REDUCER_COUNT_LOW, - NULL_COUNTER_COUNT; + NULL_COUNTER_COUNT, + DELETE_FAILED; } Modified: hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java?rev=831169&r1=831168&r2=831169&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/HExecutionEngine.java Fri Oct 30 00:28:41 2009 @@ -80,15 +80,12 @@ public class HExecutionEngine implements ExecutionEngine { - private static final String HOD_SERVER = "hod.server"; public static final String JOB_TRACKER_LOCATION = "mapred.job.tracker"; private static final String FILE_SYSTEM_LOCATION = "fs.default.name"; private final Log log = LogFactory.getLog(getClass()); private static final String LOCAL = "local"; - private StringBuilder hodParams = null; - protected PigContext pigContext; protected DataStorage ds; @@ -141,57 +138,48 @@ //First set the ssh socket factory setSSHFactory(); - String hodServer = properties.getProperty(HOD_SERVER); String cluster = null; String nameNode = null; Configuration configuration = null; - if (hodServer != null && hodServer.length() > 0) { - String hdfsAndMapred[] = doHod(hodServer, properties); - properties.setProperty(FILE_SYSTEM_LOCATION, hdfsAndMapred[0]); - properties.setProperty(JOB_TRACKER_LOCATION, hdfsAndMapred[1]); - } - else { - - // We need to build a configuration object first in the manner described below - // and then get back a properties object to inspect the JOB_TRACKER_LOCATION - // and FILE_SYSTEM_LOCATION. The reason to do this is if we looked only at - // the existing properties object, we may not get the right settings. So we want - // to read the configurations in the order specified below and only then look - // for JOB_TRACKER_LOCATION and FILE_SYSTEM_LOCATION. + // We need to build a configuration object first in the manner described below + // and then get back a properties object to inspect the JOB_TRACKER_LOCATION + // and FILE_SYSTEM_LOCATION. The reason to do this is if we looked only at + // the existing properties object, we may not get the right settings. So we want + // to read the configurations in the order specified below and only then look + // for JOB_TRACKER_LOCATION and FILE_SYSTEM_LOCATION. - // Hadoop by default specifies two resources, loaded in-order from the classpath: - // 1. hadoop-default.xml : Read-only defaults for hadoop. - // 2. hadoop-site.xml: Site-specific configuration for a given hadoop installation. - // Now add the settings from "properties" object to override any existing properties - // All of the above is accomplished in the method call below + // Hadoop by default specifies two resources, loaded in-order from the classpath: + // 1. hadoop-default.xml : Read-only defaults for hadoop. + // 2. hadoop-site.xml: Site-specific configuration for a given hadoop installation. + // Now add the settings from "properties" object to override any existing properties + // All of the above is accomplished in the method call below - JobConf jobConf = new JobConf(); - jobConf.addResource("pig-cluster-hadoop-site.xml"); + JobConf jobConf = new JobConf(); + jobConf.addResource("pig-cluster-hadoop-site.xml"); - //the method below alters the properties object by overriding the - //hadoop properties with the values from properties and recomputing - //the properties - recomputeProperties(jobConf, properties); + //the method below alters the properties object by overriding the + //hadoop properties with the values from properties and recomputing + //the properties + recomputeProperties(jobConf, properties); - configuration = ConfigurationUtil.toConfiguration(properties); - properties = ConfigurationUtil.toProperties(configuration); - cluster = properties.getProperty(JOB_TRACKER_LOCATION); - nameNode = properties.getProperty(FILE_SYSTEM_LOCATION); + configuration = ConfigurationUtil.toConfiguration(properties); + properties = ConfigurationUtil.toProperties(configuration); + cluster = properties.getProperty(JOB_TRACKER_LOCATION); + nameNode = properties.getProperty(FILE_SYSTEM_LOCATION); - if (cluster != null && cluster.length() > 0) { - if(!cluster.contains(":") && !cluster.equalsIgnoreCase(LOCAL)) { - cluster = cluster + ":50020"; - } - properties.setProperty(JOB_TRACKER_LOCATION, cluster); + if (cluster != null && cluster.length() > 0) { + if(!cluster.contains(":") && !cluster.equalsIgnoreCase(LOCAL)) { + cluster = cluster + ":50020"; } + properties.setProperty(JOB_TRACKER_LOCATION, cluster); + } - if (nameNode!=null && nameNode.length() > 0) { - if(!nameNode.contains(":") && !nameNode.equalsIgnoreCase(LOCAL)) { - nameNode = nameNode + ":8020"; - } - properties.setProperty(FILE_SYSTEM_LOCATION, nameNode); + if (nameNode!=null && nameNode.length() > 0) { + if(!nameNode.contains(":") && !nameNode.equalsIgnoreCase(LOCAL)) { + nameNode = nameNode + ":8020"; } + properties.setProperty(FILE_SYSTEM_LOCATION, nameNode); } log.info("Connecting to hadoop file system at: " + (nameNode==null? LOCAL: nameNode) ) ; @@ -218,10 +206,6 @@ } } - public void close() throws ExecException { - closeHod(pigContext.getProperties().getProperty("hod.server")); - } - public Properties getConfiguration() throws ExecException { return this.pigContext.getProperties(); } @@ -231,6 +215,8 @@ init(newConfiguration); } + public void close() throws ExecException {} + public Map<String, Object> getStatistics() throws ExecException { throw new UnsupportedOperationException(); } @@ -335,409 +321,6 @@ } } - //To prevent doing hod if the pig server is constructed multiple times - private static String hodMapRed; - private static String hodHDFS; - private String hodConfDir = null; - private String remoteHodConfDir = null; - private Process hodProcess = null; - - class ShutdownThread extends Thread{ - public synchronized void run() { - closeHod(pigContext.getProperties().getProperty("hod.server")); - } - } - - private String[] doHod(String server, Properties properties) throws ExecException { - if (hodMapRed != null) { - return new String[] {hodHDFS, hodMapRed}; - } - - // first, create temp director to store the configuration - hodConfDir = createTempDir(server); - - //jz: fallback to systemproperty cause this not handled in Main - hodParams = new StringBuilder(properties.getProperty( - "hod.param", System.getProperty("hod.param", ""))); - // get the number of nodes out of the command or use default - int nodes = getNumNodes(hodParams); - - // command format: hod allocate - d <cluster_dir> -n <number_of_nodes> <other params> - String[] fixedCmdArray = new String[] { "hod", "allocate", "-d", - hodConfDir, "-n", Integer.toString(nodes) }; - String[] extraParams = hodParams.toString().split(" "); - - String[] cmdarray = new String[fixedCmdArray.length + extraParams.length]; - System.arraycopy(fixedCmdArray, 0, cmdarray, 0, fixedCmdArray.length); - System.arraycopy(extraParams, 0, cmdarray, fixedCmdArray.length, extraParams.length); - - log.info("Connecting to HOD..."); - log.debug("sending HOD command " + cmdToString(cmdarray)); - - // setup shutdown hook to make sure we tear down hod connection - Runtime.getRuntime().addShutdownHook(new ShutdownThread()); - - runCommand(server, cmdarray, true); - - // print all the information provided by HOD - try { - BufferedReader br = new BufferedReader(new InputStreamReader(hodProcess.getErrorStream())); - String msg; - while ((msg = br.readLine()) != null) - log.info(msg); - br.close(); - } catch(IOException ioe) {} - - // for remote connection we need to bring the file locally - if (!server.equals(LOCAL)) - hodConfDir = copyHadoopConfLocally(server); - - String hdfs = null; - String mapred = null; - String hadoopConf = hodConfDir + "/hadoop-site.xml"; - - log.info ("Hadoop configuration file: " + hadoopConf); - - JobConf jobConf = new JobConf(hadoopConf); - jobConf.addResource("pig-cluster-hadoop-site.xml"); - - //the method below alters the properties object by overriding the - //hod properties with the values from properties and recomputing - //the properties - recomputeProperties(jobConf, properties); - - hdfs = properties.getProperty(FILE_SYSTEM_LOCATION); - if (hdfs == null) { - int errCode = 4007; - String msg = "Missing fs.default.name from hadoop configuration."; - throw new ExecException(msg, errCode, PigException.USER_ENVIRONMENT); - } - log.info("HDFS: " + hdfs); - - mapred = properties.getProperty(JOB_TRACKER_LOCATION); - if (mapred == null) { - int errCode = 4007; - String msg = "Missing mapred.job.tracker from hadoop configuration"; - throw new ExecException(msg, errCode, PigException.USER_ENVIRONMENT); - } - log.info("JobTracker: " + mapred); - - // this is not longer needed as hadoop-site.xml given to us by HOD - // contains data in the correct format - // hdfs = fixUpDomain(hdfs, properties); - // mapred = fixUpDomain(mapred, properties); - hodHDFS = hdfs; - hodMapRed = mapred; - - return new String[] {hdfs, mapred}; - } - - private synchronized void closeHod(String server){ - if (hodProcess == null){ - // just cleanup the dir if it exists and return - if (hodConfDir != null) - deleteDir(server, hodConfDir); - return; - } - - // hod deallocate format: hod deallocate -d <conf dir> - String[] cmdarray = new String[4]; - cmdarray[0] = "hod"; - cmdarray[1] = "deallocate"; - cmdarray[2] = "-d"; - if (remoteHodConfDir != null) - cmdarray[3] = remoteHodConfDir; - else - cmdarray[3] = hodConfDir; - - log.info("Disconnecting from HOD..."); - log.debug("Disconnect command: " + cmdToString(cmdarray)); - - try { - runCommand(server, cmdarray, false); - } catch (Exception e) { - log.warn("Failed to disconnect from HOD; error: " + e.getMessage()); - hodProcess.destroy(); - } finally { - if (remoteHodConfDir != null){ - deleteDir(server, remoteHodConfDir); - if (hodConfDir != null) - deleteDir(LOCAL, hodConfDir); - }else - deleteDir(server, hodConfDir); - } - - hodProcess = null; - } - - private String copyHadoopConfLocally(String server) throws ExecException { - String localDir = createTempDir(LOCAL); - String remoteFile = hodConfDir + "/hadoop-site.xml"; - String localFile = localDir + "/hadoop-site.xml"; - - remoteHodConfDir = hodConfDir; - - String[] cmdarray = new String[2]; - cmdarray[0] = "cat"; - cmdarray[1] = remoteFile; - - Process p = runCommand(server, cmdarray, false); - - BufferedWriter bw; - try { - bw = new BufferedWriter(new OutputStreamWriter(new FileOutputStream(localFile))); - } catch (Exception e){ - int errCode = 4008; - String msg = "Failed to create local hadoop file " + localFile; - throw new ExecException(msg, errCode, PigException.USER_ENVIRONMENT, e); - } - - try { - BufferedReader br = new BufferedReader(new InputStreamReader(p.getInputStream())); - String line; - while ((line = br.readLine()) != null){ - bw.write(line, 0, line.length()); - bw.newLine(); - } - br.close(); - bw.close(); - } catch (Exception e){ - int errCode = 4009; - String msg = "Failed to copy data to local hadoop file " + localFile; - throw new ExecException(msg, errCode, PigException.USER_ENVIRONMENT, e); - } - - return localDir; - } - - private String cmdToString(String[] cmdarray) { - StringBuilder cmd = new StringBuilder(); - - for (int i = 0; i < cmdarray.length; i++) { - cmd.append(cmdarray[i]); - cmd.append(' '); - } - - return cmd.toString(); - } - private Process runCommand(String server, String[] cmdarray, boolean connect) throws ExecException { - Process p; - try { - if (server.equals(LOCAL)) { - p = Runtime.getRuntime().exec(cmdarray); - } - else { - SSHSocketImplFactory fac = SSHSocketImplFactory.getFactory(server); - p = fac.ssh(cmdToString(cmdarray)); - } - - if (connect) - hodProcess = p; - - //this should return as soon as connection is shutdown - int rc = p.waitFor(); - if (rc != 0) { - StringBuilder errMsg = new StringBuilder(); - try { - BufferedReader br = new BufferedReader(new InputStreamReader(p.getInputStream())); - String line = null; - while((line = br.readLine()) != null) { - errMsg.append(line); - } - br.close(); - br = new BufferedReader(new InputStreamReader(p.getErrorStream())); - line = null; - while((line = br.readLine()) != null) { - errMsg.append(line); - } - br.close(); - } catch (IOException ioe) {} - int errCode = 6011; - StringBuilder msg = new StringBuilder("Failed to run command "); - msg.append(cmdToString(cmdarray)); - msg.append(" on server "); - msg.append(server); - msg.append("; return code: "); - msg.append(rc); - msg.append("; error: "); - msg.append(errMsg.toString()); - throw new ExecException(msg.toString(), errCode, PigException.REMOTE_ENVIRONMENT); - } - } catch (Exception e){ - if(e instanceof ExecException) throw (ExecException)e; - int errCode = 6012; - String msg = "Unable to run command: " + cmdToString(cmdarray) + " on server " + server; - throw new ExecException(msg, errCode, PigException.REMOTE_ENVIRONMENT, e); - } - - return p; - } - - /* - private FileSpec checkLeafIsStore(PhysicalPlan plan) throws ExecException { - try { - PhysicalOperator leaf = (PhysicalOperator)plan.getLeaves().get(0); - FileSpec spec = null; - if(!(leaf instanceof POStore)){ - String scope = leaf.getOperatorKey().getScope(); - POStore str = new POStore(new OperatorKey(scope, - NodeIdGenerator.getGenerator().getNextNodeId(scope))); - str.setPc(pigContext); - spec = new FileSpec(FileLocalizer.getTemporaryPath(null, - pigContext).toString(), - new FuncSpec(BinStorage.class.getName())); - str.setSFile(spec); - plan.addAsLeaf(str); - } else{ - spec = ((POStore)leaf).getSFile(); - } - return spec; - } catch (Exception e) { - throw new ExecException(e); - } - } - */ - - private void deleteDir(String server, String dir) { - if (server.equals(LOCAL)){ - File path = new File(dir); - deleteLocalDir(path); - } - else { - // send rm command over ssh - String[] cmdarray = new String[3]; - cmdarray[0] = "rm"; - cmdarray[1] = "-rf"; - cmdarray[2] = dir; - - try{ - runCommand(server, cmdarray, false); - }catch(Exception e){ - log.warn("Failed to remove HOD configuration directory - " + dir); - } - } - } - - private void deleteLocalDir(File path){ - File[] files = path.listFiles(); - int i; - for (i = 0; i < files.length; i++){ - if (files[i].isHidden()) - continue; - if (files[i].isFile()) - files[i].delete(); - else if (files[i].isDirectory()) - deleteLocalDir(files[i]); - } - - path.delete(); - } - - private String fixUpDomain(String hostPort,Properties properties) throws UnknownHostException { - URI uri = null; - try { - uri = new URI(hostPort); - } catch (URISyntaxException use) { - throw new RuntimeException("Illegal hostPort: " + hostPort); - } - - String hostname = uri.getHost(); - int port = uri.getPort(); - - // Parse manually if hostPort wasn't non-opaque URI - // e.g. hostPort is "myhost:myport" - if (hostname == null || port == -1) { - String parts[] = hostPort.split(":"); - hostname = parts[0]; - port = Integer.valueOf(parts[1]); - } - - if (hostname.indexOf('.') == -1) { - //jz: fallback to systemproperty cause this not handled in Main - String domain = properties.getProperty("cluster.domain",System.getProperty("cluster.domain")); - if (domain == null) - throw new RuntimeException("Missing cluster.domain property!"); - hostname = hostname + "." + domain; - } - InetAddress.getByName(hostname); - return hostname + ":" + Integer.toString(port); - } - - // create temp dir to store hod output; removed on exit - // format: <tempdir>/PigHod.<host name>.<user name>.<nanosecondts> - private String createTempDir(String server) throws ExecException { - StringBuilder tempDirPrefix = new StringBuilder (); - - if (server.equals(LOCAL)) - tempDirPrefix.append(System.getProperty("java.io.tmpdir")); - else - // for remote access we assume /tmp as temp dir - tempDirPrefix.append("/tmp"); - - tempDirPrefix.append("/PigHod."); - try { - tempDirPrefix.append(InetAddress.getLocalHost().getHostName()); - tempDirPrefix.append("."); - } catch (UnknownHostException e) {} - - tempDirPrefix.append(System.getProperty("user.name")); - tempDirPrefix.append("."); - String path; - do { - path = tempDirPrefix.toString() + System.nanoTime(); - } while (!createDir(server, path)); - - return path; - } - - private boolean createDir(String server, String dir) throws ExecException{ - if (server.equals(LOCAL)){ - // create local directory - File tempDir = new File(dir); - boolean success = tempDir.mkdir(); - if (!success) - log.warn("Failed to create HOD configuration directory - " + dir + ". Retrying ..."); - - return success; - } - else { - String[] cmdarray = new String[2]; - cmdarray[0] = "mkdir "; - cmdarray[1] = dir; - - try{ - runCommand(server, cmdarray, false); - } - catch(ExecException e){ - log.warn("Failed to create HOD configuration directory - " + dir + "Retrying..."); - return false; - } - - return true; - } - } - - // returns number of nodes based on -m option in hodParams if present; - // otherwise, default is used; -m is removed from the params - int getNumNodes(StringBuilder hodParams) { - String val = hodParams.toString(); - int startPos = val.indexOf("-m "); - if (startPos == -1) - startPos = val.indexOf("-m\t"); - if (startPos != -1) { - int curPos = startPos + 3; - int len = val.length(); - while (curPos < len && Character.isWhitespace(val.charAt(curPos))) curPos ++; - int numStartPos = curPos; - while (curPos < len && Character.isDigit(val.charAt(curPos))) curPos ++; - int nodes = Integer.parseInt(val.substring(numStartPos, curPos)); - hodParams.delete(startPos, curPos); - return nodes; - } else { - return Integer.getInteger("hod.nodes", 15); - } - } - /** * Method to recompute pig properties by overriding hadoop properties * with pig properties @@ -763,20 +346,7 @@ hadoopProperties.put(key, val); } - //clear user defined properties and re-populate - properties.clear(); - Enumeration<Object> hodPropertiesIter = hadoopProperties.keys(); - while (hodPropertiesIter.hasMoreElements()) { - String key = (String) hodPropertiesIter.nextElement(); - String val = hadoopProperties.getProperty(key); - properties.put(key, val); - } - } } } - - - - Modified: hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PhyPlanSetter.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PhyPlanSetter.java?rev=831169&r1=831168&r2=831169&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PhyPlanSetter.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/mapReduceLayer/PhyPlanSetter.java Fri Oct 30 00:28:41 2009 @@ -273,16 +273,6 @@ } @Override - public void visitCogroup(POCogroup cogroup) { - cogroup.setParentPlan(parent); - } - - @Override - public void visitSplit(org.apache.pig.backend.local.executionengine.physicalLayer.relationalOperators.POSplit split) { - split.setParentPlan(parent); - } - - @Override public void visitLocalRearrangeForIllustrate( POLocalRearrangeForIllustrate lrfi) throws VisitorException { super.visitLocalRearrangeForIllustrate(lrfi); Modified: hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/PhysicalPlan.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/PhysicalPlan.java?rev=831169&r1=831168&r2=831169&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/PhysicalPlan.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/backend/hadoop/executionengine/physicalLayer/plans/PhysicalPlan.java Fri Oct 30 00:28:41 2009 @@ -204,12 +204,6 @@ } @Override - public boolean equals(Object obj) { - // TODO Auto-generated method stub - return super.equals(obj); - } - - @Override public PhysicalPlan clone() throws CloneNotSupportedException { PhysicalPlan clone = new PhysicalPlan(); Modified: hadoop/pig/trunk/src/org/apache/pig/backend/local/datastorage/LocalPath.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/local/datastorage/LocalPath.java?rev=831169&r1=831168&r2=831169&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/backend/local/datastorage/LocalPath.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/backend/local/datastorage/LocalPath.java Fri Oct 30 00:28:41 2009 @@ -26,12 +26,15 @@ import java.util.HashMap; import java.util.Properties; +import org.apache.commons.logging.Log; +import org.apache.commons.logging.LogFactory; import org.apache.pig.backend.datastorage.DataStorage; import org.apache.pig.backend.datastorage.ElementDescriptor; import org.apache.pig.backend.datastorage.SeekableInputStream; public abstract class LocalPath implements ElementDescriptor { + private Log log = LogFactory.getLog(getClass()); protected DataStorage fs; protected File path; @@ -121,7 +124,9 @@ } public void delete() throws IOException { - getCurPath().delete(); + boolean res = getCurPath().delete(); + if (!res) + log.warn("LocalPath.delete: failed to delete" + getCurPath()); } public Properties getConfiguration() throws IOException { Modified: hadoop/pig/trunk/src/org/apache/pig/backend/local/executionengine/physicalLayer/relationalOperators/POCogroup.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/backend/local/executionengine/physicalLayer/relationalOperators/POCogroup.java?rev=831169&r1=831168&r2=831169&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/backend/local/executionengine/physicalLayer/relationalOperators/POCogroup.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/backend/local/executionengine/physicalLayer/relationalOperators/POCogroup.java Fri Oct 30 00:28:41 2009 @@ -268,10 +268,6 @@ return result; } - public boolean equals(Object obj) { - return this.equals(obj); - } - } } Modified: hadoop/pig/trunk/src/org/apache/pig/builtin/BinStorage.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/builtin/BinStorage.java?rev=831169&r1=831168&r2=831169&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/builtin/BinStorage.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/builtin/BinStorage.java Fri Oct 30 00:28:41 2009 @@ -405,6 +405,10 @@ return true; } + public int hashCode() { + return 42; + } + /* (non-Javadoc) * @see org.apache.pig.StoreFunc#getStorePreparationClass() */ Modified: hadoop/pig/trunk/src/org/apache/pig/builtin/BinaryStorage.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/builtin/BinaryStorage.java?rev=831169&r1=831168&r2=831169&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/builtin/BinaryStorage.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/builtin/BinaryStorage.java Fri Oct 30 00:28:41 2009 @@ -143,6 +143,10 @@ return true; } + public int hashCode() { + return 42; + } + /* (non-Javadoc) * @see org.apache.pig.LoadFunc#determineSchema(java.lang.String, org.apache.pig.ExecType, org.apache.pig.backend.datastorage.DataStorage) */ Modified: hadoop/pig/trunk/src/org/apache/pig/builtin/PigStorage.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/builtin/PigStorage.java?rev=831169&r1=831168&r2=831169&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/builtin/PigStorage.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/builtin/PigStorage.java Fri Oct 30 00:28:41 2009 @@ -324,13 +324,20 @@ } public boolean equals(Object obj) { - return equals((PigStorage)obj); + if (obj instanceof PigStorage) + return equals((PigStorage)obj); + else + return false; } public boolean equals(PigStorage other) { return this.fieldDel == other.fieldDel; } + public int hashCode() { + return (int)fieldDel; + } + /* (non-Javadoc) * @see org.apache.pig.StoreFunc#getStorePreparationClass() */ Modified: hadoop/pig/trunk/src/org/apache/pig/data/DefaultAbstractBag.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/data/DefaultAbstractBag.java?rev=831169&r1=831168&r2=831169&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/data/DefaultAbstractBag.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/data/DefaultAbstractBag.java Fri Oct 30 00:28:41 2009 @@ -29,6 +29,7 @@ import java.util.ArrayList; import org.apache.pig.PigException; +import org.apache.pig.PigWarning; import org.apache.pig.backend.executionengine.ExecException; import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PhysicalOperator; import org.apache.pig.backend.hadoop.executionengine.physicalLayer.PigLogger; @@ -161,7 +162,9 @@ mContents.clear(); if (mSpillFiles != null) { for (int i = 0; i < mSpillFiles.size(); i++) { - mSpillFiles.get(i).delete(); + boolean res = mSpillFiles.get(i).delete(); + if (!res) + warn ("DefaultAbstractBag.clear: failed to delete " + mSpillFiles.get(i), PigWarning.DELETE_FAILED, null); } mSpillFiles.clear(); } @@ -298,7 +301,10 @@ protected void finalize() { if (mSpillFiles != null) { for (int i = 0; i < mSpillFiles.size(); i++) { - mSpillFiles.get(i).delete(); + boolean res = mSpillFiles.get(i).delete(); + if (!res) + warn ("DefaultAbstractBag.finalize: failed to delete " + mSpillFiles.get(i), PigWarning.DELETE_FAILED, null); + } } } Modified: hadoop/pig/trunk/src/org/apache/pig/data/DistinctDataBag.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/data/DistinctDataBag.java?rev=831169&r1=831168&r2=831169&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/data/DistinctDataBag.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/data/DistinctDataBag.java Fri Oct 30 00:28:41 2009 @@ -203,6 +203,17 @@ public int compareTo(TContainer other) { return tuple.compareTo(other.tuple); } + + public boolean equals(Object obj){ + if (obj instanceof TContainer) + return tuple.equals(((TContainer)obj).tuple); + else + return false; + } + + public int hashCode() { + return tuple.hashCode(); + } } // We have to buffer a tuple because there's no easy way for next Modified: hadoop/pig/trunk/src/org/apache/pig/data/InternalSortedBag.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/data/InternalSortedBag.java?rev=831169&r1=831168&r2=831169&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/data/InternalSortedBag.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/data/InternalSortedBag.java Fri Oct 30 00:28:41 2009 @@ -83,6 +83,9 @@ return (o == this); } + public int hashCode() { + return 42; + } } public InternalSortedBag() { Modified: hadoop/pig/trunk/src/org/apache/pig/data/NonSpillableDataBag.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/data/NonSpillableDataBag.java?rev=831169&r1=831168&r2=831169&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/data/NonSpillableDataBag.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/data/NonSpillableDataBag.java Fri Oct 30 00:28:41 2009 @@ -192,6 +192,10 @@ return compareTo(obj) == 0; } + public int hashCode() { + return mContents.hashCode(); + } + @SuppressWarnings("unchecked") @Override public int compareTo(Object other) { Modified: hadoop/pig/trunk/src/org/apache/pig/data/SingleTupleBag.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/data/SingleTupleBag.java?rev=831169&r1=831168&r2=831169&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/data/SingleTupleBag.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/data/SingleTupleBag.java Fri Oct 30 00:28:41 2009 @@ -157,6 +157,15 @@ return 0; } + public boolean equals(Object o){ + // TODO: match to compareTo if it is updated + return true; + } + + public int hashCode() { + return 42; + } + class TBIterator implements Iterator<Tuple> { boolean nextDone = false; /* (non-Javadoc) Modified: hadoop/pig/trunk/src/org/apache/pig/data/SortedDataBag.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/data/SortedDataBag.java?rev=831169&r1=831168&r2=831169&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/data/SortedDataBag.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/data/SortedDataBag.java Fri Oct 30 00:28:41 2009 @@ -71,6 +71,10 @@ return false; } + public int hashCode() { + return 42; + } + } /** @@ -177,6 +181,17 @@ public int compareTo(PQContainer other) { return mComp.compare(tuple, other.tuple); } + + public boolean equals(Object other) { + if (other instanceof PQContainer) + return tuple.equals(((PQContainer)other).tuple); + else + return false; + } + + public int hashCode() { + return tuple.hashCode(); + } } // We have to buffer a tuple because there's no easy way for next Modified: hadoop/pig/trunk/src/org/apache/pig/data/TargetedTuple.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/data/TargetedTuple.java?rev=831169&r1=831168&r2=831169&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/data/TargetedTuple.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/data/TargetedTuple.java Fri Oct 30 00:28:41 2009 @@ -151,6 +151,15 @@ return t.compareTo(o); } + @SuppressWarnings("unchecked") + public boolean equals(Object o) { + return t.equals(o); + } + + public int hashCode() { + return t.hashCode(); + } + /** * @return true if this Tuple is null */ Modified: hadoop/pig/trunk/src/org/apache/pig/impl/io/FileLocalizer.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/impl/io/FileLocalizer.java?rev=831169&r1=831168&r2=831169&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/impl/io/FileLocalizer.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/impl/io/FileLocalizer.java Fri Oct 30 00:28:41 2009 @@ -407,7 +407,9 @@ // TODO probably this should be replaced with the local file system File f = (new File(fileSpec)).getParentFile(); if (f!=null){ - f.mkdirs(); + boolean res = f.mkdirs(); + if (!res) + log.warn("FileLocalizer.create: failed to create " + f); } return new FileOutputStream(fileSpec,append); Modified: hadoop/pig/trunk/src/org/apache/pig/impl/streaming/StreamingCommand.java URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/src/org/apache/pig/impl/streaming/StreamingCommand.java?rev=831169&r1=831168&r2=831169&view=diff ============================================================================== --- hadoop/pig/trunk/src/org/apache/pig/impl/streaming/StreamingCommand.java (original) +++ hadoop/pig/trunk/src/org/apache/pig/impl/streaming/StreamingCommand.java Fri Oct 30 00:28:41 2009 @@ -502,10 +502,16 @@ } public boolean equals(Object obj) { - HandleSpec other = (HandleSpec)obj; - return (other != null && name.equals(other.name) && spec.equals(other.spec)); + if (obj instanceof HandleSpec){ + HandleSpec other = (HandleSpec)obj; + return (other != null && name.equals(other.name) && spec.equals(other.spec)); + } else + return false; } + public int hashCode() { + return name.hashCode(); + } public Object clone() { try { Modified: hadoop/pig/trunk/test/findbugsExcludeFile.xml URL: http://svn.apache.org/viewvc/hadoop/pig/trunk/test/findbugsExcludeFile.xml?rev=831169&r1=831168&r2=831169&view=diff ============================================================================== --- hadoop/pig/trunk/test/findbugsExcludeFile.xml (original) +++ hadoop/pig/trunk/test/findbugsExcludeFile.xml Fri Oct 30 00:28:41 2009 @@ -37,6 +37,9 @@ <Bug pattern="EI_EXPOSE_REP2" /> </Match> <Match> + <Bug pattern="DP_CREATE_CLASSLOADER_INSIDE_DO_PRIVILEGED" /> + </Match> + <Match> <Class name="org.apache.pig.tools.parameters.Token" /> </Match> <Match> @@ -146,6 +149,10 @@ <Bug pattern="SIC_INNER_SHOULD_BE_STATIC" /> </Match> <Match> + <Class name="org.apache.pig.data.InternalDistinctBag$DistinctDataBagIterator$TContainer" /> + <Bug pattern="SIC_INNER_SHOULD_BE_STATIC" /> + </Match> + <Match> <Bug pattern="BC_BAD_CAST_TO_CONCRETE_COLLECTION" /> </Match> <!-- This Tuple classes are not used --> @@ -236,5 +243,13 @@ <Method name = "sendMTFValues" /> <Bug pattern="IM_BAD_CHECK_FOR_ODD" /> </Match> + <Match> + <Class name="org.apache.pig.backend.hadoop.executionengine.mapReduceLayer.PigHadoopLogger" /> + <Bug pattern="UG_SYNC_SET_UNSYNC_GET" /> + </Match> + <Match> + <Class name="org.apache.pig.backend.hadoop.executionengine.physicalLayer.relationalOperators.POStream" /> + <Bug pattern="IS2_INCONSISTENT_SYNC" /> + </Match> </FindBugsFilter>