Repository: asterixdb Updated Branches: refs/heads/master 3f66d0916 -> 07075667c
http://git-wip-us.apache.org/repos/asf/asterixdb/blob/07075667/hyracks-fullstack/hyracks/hyracks-api/src/test/java/org/apache/hyracks/api/test/FrameWriterTestUtils.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-api/src/test/java/org/apache/hyracks/api/test/FrameWriterTestUtils.java b/hyracks-fullstack/hyracks/hyracks-api/src/test/java/org/apache/hyracks/api/test/FrameWriterTestUtils.java index c9cc71e..2c1ce37 100644 --- a/hyracks-fullstack/hyracks/hyracks-api/src/test/java/org/apache/hyracks/api/test/FrameWriterTestUtils.java +++ b/hyracks-fullstack/hyracks/hyracks-api/src/test/java/org/apache/hyracks/api/test/FrameWriterTestUtils.java @@ -19,6 +19,7 @@ package org.apache.hyracks.api.test; import java.util.Collection; +import java.util.Collections; public class FrameWriterTestUtils { public static final String EXCEPTION_MESSAGE = "IFrameWriter Exception in the call to the method "; @@ -32,6 +33,10 @@ public class FrameWriterTestUtils { Close } + public static TestFrameWriter create() { + return create(Collections.emptyList(), Collections.emptyList(), false); + } + public static TestFrameWriter create(Collection<FrameWriterOperation> exceptionThrowingOperations, Collection<FrameWriterOperation> errorThrowingOperations, boolean deepCopyInputFrames) { CountAnswer openAnswer = http://git-wip-us.apache.org/repos/asf/asterixdb/blob/07075667/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/dataset/DatasetDirectoryService.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/dataset/DatasetDirectoryService.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/dataset/DatasetDirectoryService.java index 2685f60..7a9306c 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/dataset/DatasetDirectoryService.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/dataset/DatasetDirectoryService.java @@ -151,12 +151,18 @@ public class DatasetDirectoryService implements IDatasetDirectoryService { @Override public synchronized void reportJobFailure(JobId jobId, List<Exception> exceptions) { + LOGGER.log(Level.INFO, "job " + jobId + " failed and is being reported to " + getClass().getSimpleName(), + exceptions.get(0)); DatasetJobRecord djr = getDatasetJobRecord(jobId); + LOGGER.log(Level.INFO, "Dataset job record is " + djr); if (djr != null) { + LOGGER.log(Level.INFO, "Setting exceptions in Dataset job record"); djr.fail(exceptions); } final JobResultInfo jobResultInfo = jobResultLocations.get(jobId); + LOGGER.log(Level.INFO, "Job result info is " + jobResultInfo); if (jobResultInfo != null) { + LOGGER.log(Level.INFO, "Setting exceptions in Job result info"); jobResultInfo.setException(exceptions.isEmpty() ? null : exceptions.get(0)); } notifyAll(); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/07075667/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/executor/JobExecutor.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/executor/JobExecutor.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/executor/JobExecutor.java index f18a917..dbbaf9f 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/executor/JobExecutor.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/executor/JobExecutor.java @@ -66,7 +66,6 @@ import org.apache.hyracks.control.cc.work.JobCleanupWork; import org.apache.hyracks.control.common.job.PartitionState; import org.apache.hyracks.control.common.job.TaskAttemptDescriptor; - public class JobExecutor { private static final Logger LOGGER = Logger.getLogger(JobExecutor.class.getName()); @@ -190,11 +189,11 @@ public class JobExecutor { private void startRunnableActivityClusters() throws HyracksException { Set<TaskCluster> taskClusterRoots = new HashSet<>(); - findRunnableTaskClusterRoots(taskClusterRoots, jobRun.getActivityClusterGraph().getActivityClusterMap() - .values()); - if (LOGGER.isLoggable(Level.FINE)) { - LOGGER.fine("Runnable TC roots: " + taskClusterRoots + ", inProgressTaskClusters: " - + inProgressTaskClusters); + findRunnableTaskClusterRoots(taskClusterRoots, + jobRun.getActivityClusterGraph().getActivityClusterMap().values()); + if (LOGGER.isLoggable(Level.INFO)) { + LOGGER.log(Level.INFO, + "Runnable TC roots: " + taskClusterRoots + ", inProgressTaskClusters: " + inProgressTaskClusters); } if (taskClusterRoots.isEmpty() && inProgressTaskClusters.isEmpty()) { ccs.getWorkQueue() @@ -344,8 +343,8 @@ public class JobExecutor { for (int i = 0; i < tasks.length; ++i) { Task ts = tasks[i]; TaskId tid = ts.getTaskId(); - TaskAttempt taskAttempt = new TaskAttempt(tcAttempt, new TaskAttemptId(new TaskId(tid.getActivityId(), - tid.getPartition()), attempts), ts); + TaskAttempt taskAttempt = new TaskAttempt(tcAttempt, + new TaskAttemptId(new TaskId(tid.getActivityId(), tid.getPartition()), attempts), ts); taskAttempt.setStatus(TaskAttempt.TaskStatus.INITIALIZED, null); locationMap.put(tid, new PartitionLocationExpression(tid.getActivityId().getOperatorDescriptorId(), tid.getPartition())); @@ -496,8 +495,8 @@ public class JobExecutor { final DeploymentId deploymentId = jobRun.getDeploymentId(); final JobId jobId = jobRun.getJobId(); final ActivityClusterGraph acg = jobRun.getActivityClusterGraph(); - final Map<ConnectorDescriptorId, IConnectorPolicy> connectorPolicies = new HashMap<>( - jobRun.getConnectorPolicyMap()); + final Map<ConnectorDescriptorId, IConnectorPolicy> connectorPolicies = + new HashMap<>(jobRun.getConnectorPolicyMap()); INodeManager nodeManager = ccs.getNodeManager(); try { byte[] acgBytes = predistributed ? null : JavaSerializationUtils.serialize(acg); @@ -555,14 +554,14 @@ public class JobExecutor { } } final JobId jobId = jobRun.getJobId(); - LOGGER.fine("Abort map for job: " + jobId + ": " + abortTaskAttemptMap); + LOGGER.info("Abort map for job: " + jobId + ": " + abortTaskAttemptMap); INodeManager nodeManager = ccs.getNodeManager(); for (Map.Entry<String, List<TaskAttemptId>> entry : abortTaskAttemptMap.entrySet()) { final NodeControllerState node = nodeManager.getNodeControllerState(entry.getKey()); final List<TaskAttemptId> abortTaskAttempts = entry.getValue(); if (node != null) { - if (LOGGER.isLoggable(Level.FINE)) { - LOGGER.fine("Aborting: " + abortTaskAttempts + " at " + entry.getKey()); + if (LOGGER.isLoggable(Level.INFO)) { + LOGGER.info("Aborting: " + abortTaskAttempts + " at " + entry.getKey()); } try { node.getNodeController().abortTasks(jobId, abortTaskAttempts); @@ -582,6 +581,7 @@ public class JobExecutor { } private void abortDoomedTaskClusters() throws HyracksException { + LOGGER.log(Level.INFO, "aborting doomed task clusters"); Set<TaskCluster> doomedTaskClusters = new HashSet<>(); for (TaskCluster tc : inProgressTaskClusters) { // Start search at TCs that produce no outputs (sinks) @@ -590,6 +590,7 @@ public class JobExecutor { } } + LOGGER.log(Level.INFO, "number of doomed task clusters found = " + doomedTaskClusters.size()); for (TaskCluster tc : doomedTaskClusters) { TaskClusterAttempt tca = findLastTaskClusterAttempt(tc); if (tca != null) { @@ -628,7 +629,7 @@ public class JobExecutor { if ((maxState == null || (cPolicy.consumerWaitsForProducerToFinish() && maxState != PartitionState.COMMITTED)) && findDoomedTaskClusters(partitionProducingTaskClusterMap.get(pid), doomedTaskClusters)) { - doomed = true; + doomed = true; } } if (doomed) { @@ -663,28 +664,36 @@ public class JobExecutor { /** * Indicates that a single task attempt has encountered a failure. - * @param ta Failed Task Attempt - * @param exceptions exeptions thrown during the failure + * + * @param ta + * Failed Task Attempt + * @param exceptions + * exeptions thrown during the failure */ public void notifyTaskFailure(TaskAttempt ta, List<Exception> exceptions) { try { - LOGGER.fine("Received failure notification for TaskAttempt " + ta.getTaskAttemptId()); + LOGGER.log(Level.INFO, "Received failure notification for TaskAttempt " + ta.getTaskAttemptId()); TaskAttemptId taId = ta.getTaskAttemptId(); TaskCluster tc = ta.getTask().getTaskCluster(); TaskClusterAttempt lastAttempt = findLastTaskClusterAttempt(tc); if (lastAttempt != null && taId.getAttempt() == lastAttempt.getAttempt()) { - LOGGER.fine("Marking TaskAttempt " + ta.getTaskAttemptId() + " as failed"); + LOGGER.log(Level.INFO, "Marking TaskAttempt " + ta.getTaskAttemptId() + " as failed"); ta.setStatus(TaskAttempt.TaskStatus.FAILED, exceptions); abortTaskCluster(lastAttempt, TaskClusterAttempt.TaskClusterStatus.FAILED); abortDoomedTaskClusters(); - if (lastAttempt.getAttempt() >= jobRun.getActivityClusterGraph().getMaxReattempts() || isCancelled()) { + int maxReattempts = jobRun.getActivityClusterGraph().getMaxReattempts(); + LOGGER.log(Level.INFO, "Marking TaskAttempt " + ta.getTaskAttemptId() + + " as failed and the number of max re-attempts = " + maxReattempts); + if (lastAttempt.getAttempt() >= maxReattempts || isCancelled()) { + LOGGER.log(Level.INFO, "Aborting the job of " + ta.getTaskAttemptId()); abortJob(exceptions); return; } + LOGGER.log(Level.INFO, "We will try to start runnable activity clusters of " + ta.getTaskAttemptId()); startRunnableActivityClusters(); } else { - LOGGER.warning("Ignoring task failure notification: " + taId + " -- Current last attempt = " - + lastAttempt); + LOGGER.warning( + "Ignoring task failure notification: " + taId + " -- Current last attempt = " + lastAttempt); } } catch (Exception e) { abortJob(Collections.singletonList(e)); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/07075667/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/TaskFailureWork.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/TaskFailureWork.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/TaskFailureWork.java index 486e9c6..8f50087 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/TaskFailureWork.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-cc/src/main/java/org/apache/hyracks/control/cc/work/TaskFailureWork.java @@ -19,6 +19,8 @@ package org.apache.hyracks.control.cc.work; import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; import org.apache.hyracks.api.dataflow.TaskAttemptId; import org.apache.hyracks.api.job.JobId; @@ -28,6 +30,7 @@ import org.apache.hyracks.control.cc.job.JobRun; import org.apache.hyracks.control.cc.job.TaskAttempt; public class TaskFailureWork extends AbstractTaskLifecycleWork { + private static final Logger LOGGER = Logger.getLogger(TaskFailureWork.class.getName()); private final List<Exception> exceptions; public TaskFailureWork(ClusterControllerService ccs, JobId jobId, TaskAttemptId taId, String nodeId, @@ -38,6 +41,7 @@ public class TaskFailureWork extends AbstractTaskLifecycleWork { @Override protected void performEvent(TaskAttempt ta) { + LOGGER.log(Level.WARNING, "Executing task failure work for " + this, exceptions.get(0)); IJobManager jobManager = ccs.getJobManager(); JobRun run = jobManager.get(jobId); ccs.getDatasetDirectoryService().reportJobFailure(jobId, exceptions); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/07075667/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/dataset/DatasetPartitionWriter.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/dataset/DatasetPartitionWriter.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/dataset/DatasetPartitionWriter.java index 0189735..b654d44 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/dataset/DatasetPartitionWriter.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/dataset/DatasetPartitionWriter.java @@ -120,7 +120,7 @@ public class DatasetPartitionWriter implements IFrameWriter { manager.reportPartitionWriteCompletion(jobId, resultSetId, partition); } } catch (HyracksException e) { - throw new HyracksDataException(e); + throw HyracksDataException.create(e); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/07075667/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java index 92831f4..10c7415 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/io/IOManager.java @@ -229,7 +229,7 @@ public class IOManager implements IIOManager { try { ((FileHandle) fHandle).close(); } catch (IOException e) { - throw new HyracksDataException(e); + throw HyracksDataException.create(e); } } @@ -242,7 +242,7 @@ public class IOManager implements IIOManager { try { waf = File.createTempFile(prefix, WORKSPACE_FILE_SUFFIX, new File(dev.getMount(), waPath)); } catch (IOException e) { - throw new HyracksDataException(e); + throw HyracksDataException.create(e); } return dev.createFileRef(waPath + File.separator + waf.getName()); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/07075667/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/ApplicationMessageWork.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/ApplicationMessageWork.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/ApplicationMessageWork.java index 7f5302a..4f5b556 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/ApplicationMessageWork.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/ApplicationMessageWork.java @@ -28,9 +28,6 @@ import org.apache.hyracks.control.common.work.AbstractWork; import org.apache.hyracks.control.nc.NodeControllerService; import org.apache.hyracks.control.nc.application.NCServiceContext; -/** - * @author rico - */ public class ApplicationMessageWork extends AbstractWork { private static final Logger LOGGER = Logger.getLogger(ApplicationMessageWork.class.getName()); private byte[] message; @@ -63,6 +60,6 @@ public class ApplicationMessageWork extends AbstractWork { @Override public String toString() { - return getName() + ": nodeID: " + nodeId; + return getName() + ": nodeId: " + nodeId; } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/07075667/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/NotifyTaskCompleteWork.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/NotifyTaskCompleteWork.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/NotifyTaskCompleteWork.java index f4ee6b0..7728d16 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/NotifyTaskCompleteWork.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/NotifyTaskCompleteWork.java @@ -18,12 +18,16 @@ */ package org.apache.hyracks.control.nc.work; +import java.util.logging.Level; +import java.util.logging.Logger; + import org.apache.hyracks.control.common.job.profiling.om.TaskProfile; import org.apache.hyracks.control.common.work.AbstractWork; import org.apache.hyracks.control.nc.NodeControllerService; import org.apache.hyracks.control.nc.Task; public class NotifyTaskCompleteWork extends AbstractWork { + private static final Logger LOGGER = Logger.getLogger(NotifyTaskCompleteWork.class.getName()); private final NodeControllerService ncs; private final Task task; @@ -40,8 +44,13 @@ public class NotifyTaskCompleteWork extends AbstractWork { ncs.getClusterController().notifyTaskComplete(task.getJobletContext().getJobId(), task.getTaskAttemptId(), ncs.getId(), taskProfile); } catch (Exception e) { - e.printStackTrace(); + LOGGER.log(Level.SEVERE, "Failed notifying task complete for " + task.getTaskAttemptId(), e); } task.getJoblet().removeTask(task); } + + @Override + public String toString() { + return getClass().getSimpleName() + ":" + task.getTaskAttemptId(); + } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/07075667/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/NotifyTaskFailureWork.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/NotifyTaskFailureWork.java b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/NotifyTaskFailureWork.java index fa8ba28..7ed2c09 100644 --- a/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/NotifyTaskFailureWork.java +++ b/hyracks-fullstack/hyracks/hyracks-control/hyracks-control-nc/src/main/java/org/apache/hyracks/control/nc/work/NotifyTaskFailureWork.java @@ -35,7 +35,6 @@ public class NotifyTaskFailureWork extends AbstractWork { private final Task task; private final JobId jobId; private final TaskAttemptId taskId; - private final List<Exception> exceptions; public NotifyTaskFailureWork(NodeControllerService ncs, Task task, List<Exception> exceptions, JobId jobId, @@ -49,6 +48,8 @@ public class NotifyTaskFailureWork extends AbstractWork { @Override public void run() { + LOGGER.log(Level.WARNING, ncs.getId() + " is sending a notification to cc that task " + taskId + " has failed", + exceptions.get(0)); try { IDatasetPartitionManager dpm = ncs.getDatasetPartitionManager(); if (dpm != null) { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/07075667/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/comm/io/ArrayTupleBuilder.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/comm/io/ArrayTupleBuilder.java b/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/comm/io/ArrayTupleBuilder.java index e5f7d09..0c639db 100644 --- a/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/comm/io/ArrayTupleBuilder.java +++ b/hyracks-fullstack/hyracks/hyracks-dataflow-common/src/main/java/org/apache/hyracks/dataflow/common/comm/io/ArrayTupleBuilder.java @@ -104,7 +104,7 @@ public class ArrayTupleBuilder implements IDataOutputProvider { fieldData.getDataOutput().writeInt(FrameConstants.FRAME_FIELD_MAGIC); } } catch (IOException e) { - throw new HyracksDataException(e); + throw HyracksDataException.create(e); } fEndOffsets[nextField++] = fieldData.getLength(); } @@ -139,7 +139,7 @@ public class ArrayTupleBuilder implements IDataOutputProvider { try { fieldData.getDataOutput().write(bytes, start, length); } catch (IOException e) { - throw new HyracksDataException(e); + throw HyracksDataException.create(e); } fEndOffsets[nextField++] = fieldData.getLength(); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/07075667/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/base/AbstractReplicateOperatorDescriptor.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/base/AbstractReplicateOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/base/AbstractReplicateOperatorDescriptor.java index 8ee4fa3..16c0a29 100644 --- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/base/AbstractReplicateOperatorDescriptor.java +++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/base/AbstractReplicateOperatorDescriptor.java @@ -84,16 +84,16 @@ public abstract class AbstractReplicateOperatorDescriptor extends AbstractOperat @Override public void contributeActivities(IActivityGraphBuilder builder) { - ReplicatorMaterializerActivityNode sma = new ReplicatorMaterializerActivityNode( - new ActivityId(odId, SPLITTER_MATERIALIZER_ACTIVITY_ID)); + ReplicatorMaterializerActivityNode sma = + new ReplicatorMaterializerActivityNode(new ActivityId(odId, SPLITTER_MATERIALIZER_ACTIVITY_ID)); builder.addActivity(this, sma); builder.addSourceEdge(0, sma, 0); int pipelineOutputIndex = 0; int activityId = MATERIALIZE_READER_ACTIVITY_ID; for (int i = 0; i < outputArity; i++) { if (outputMaterializationFlags[i]) { - MaterializeReaderActivityNode mra = new MaterializeReaderActivityNode( - new ActivityId(odId, activityId++)); + MaterializeReaderActivityNode mra = + new MaterializeReaderActivityNode(new ActivityId(odId, activityId++)); builder.addActivity(this, mra); builder.addBlockingEdge(sma, mra); builder.addTargetEdge(i, mra, 0); @@ -165,7 +165,7 @@ public abstract class AbstractReplicateOperatorDescriptor extends AbstractOperat writers[i].close(); } catch (Throwable th) { if (hde == null) { - hde = new HyracksDataException(th); + hde = HyracksDataException.create(th); } else { hde.addSuppressed(th); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/07075667/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/collectors/InputChannelFrameReader.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/collectors/InputChannelFrameReader.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/collectors/InputChannelFrameReader.java index 7ff280b..6748a4d 100644 --- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/collectors/InputChannelFrameReader.java +++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/collectors/InputChannelFrameReader.java @@ -53,7 +53,8 @@ public class InputChannelFrameReader implements IFrameReader, IInputChannelMonit try { wait(); } catch (InterruptedException e) { - throw new HyracksDataException(e); + Thread.currentThread().interrupt(); + throw HyracksDataException.create(e); } } if (failed) { http://git-wip-us.apache.org/repos/asf/asterixdb/blob/07075667/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/connectors/LocalityAwarePartitionDataWriter.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/connectors/LocalityAwarePartitionDataWriter.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/connectors/LocalityAwarePartitionDataWriter.java index 253b3e3..63bb72b 100644 --- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/connectors/LocalityAwarePartitionDataWriter.java +++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/connectors/LocalityAwarePartitionDataWriter.java @@ -130,7 +130,7 @@ public class LocalityAwarePartitionDataWriter implements IFrameWriter { appenders[i].write(pWriters[i], true); } catch (Throwable th) { if (closeException == null) { - closeException = new HyracksDataException(th); + closeException = HyracksDataException.create(th); } else { closeException.addSuppressed(th); } @@ -139,7 +139,7 @@ public class LocalityAwarePartitionDataWriter implements IFrameWriter { pWriters[i].close(); } catch (Throwable th) { if (closeException == null) { - closeException = new HyracksDataException(th); + closeException = HyracksDataException.create(th); } else { closeException.addSuppressed(th); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/07075667/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/connectors/MToNBroadcastConnectorDescriptor.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/connectors/MToNBroadcastConnectorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/connectors/MToNBroadcastConnectorDescriptor.java index d748c8e..4246626 100644 --- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/connectors/MToNBroadcastConnectorDescriptor.java +++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/connectors/MToNBroadcastConnectorDescriptor.java @@ -95,7 +95,7 @@ public class MToNBroadcastConnectorDescriptor extends AbstractMToNConnectorDescr epWriters[i].close(); } catch (Throwable th) { if (closeException == null) { - closeException = new HyracksDataException(th); + closeException = HyracksDataException.create(th); } else { closeException.addSuppressed(th); } @@ -129,8 +129,8 @@ public class MToNBroadcastConnectorDescriptor extends AbstractMToNConnectorDescr int nProducerPartitions, int nConsumerPartitions) throws HyracksDataException { BitSet expectedPartitions = new BitSet(nProducerPartitions); expectedPartitions.set(0, nProducerPartitions); - NonDeterministicChannelReader channelReader = new NonDeterministicChannelReader(nProducerPartitions, - expectedPartitions); + NonDeterministicChannelReader channelReader = + new NonDeterministicChannelReader(nProducerPartitions, expectedPartitions); NonDeterministicFrameReader frameReader = new NonDeterministicFrameReader(channelReader); return new PartitionCollector(ctx, getConnectorId(), index, expectedPartitions, frameReader, channelReader); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/07075667/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FrameFileWriterOperatorDescriptor.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FrameFileWriterOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FrameFileWriterOperatorDescriptor.java index a34a322..b94f305 100644 --- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FrameFileWriterOperatorDescriptor.java +++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/FrameFileWriterOperatorDescriptor.java @@ -66,7 +66,7 @@ public class FrameFileWriterOperatorDescriptor extends AbstractSingleActivityOpe try { out.write(buffer.array()); } catch (IOException e) { - throw new HyracksDataException(e); + throw HyracksDataException.create(e); } } @@ -79,7 +79,7 @@ public class FrameFileWriterOperatorDescriptor extends AbstractSingleActivityOpe try { out.close(); } catch (IOException e) { - throw new HyracksDataException(e); + throw HyracksDataException.create(e); } } }; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/07075667/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/PlainFileWriterOperatorDescriptor.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/PlainFileWriterOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/PlainFileWriterOperatorDescriptor.java index 3a3f414..76ebaa4 100644 --- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/PlainFileWriterOperatorDescriptor.java +++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/file/PlainFileWriterOperatorDescriptor.java @@ -76,8 +76,8 @@ public class PlainFileWriterOperatorDescriptor extends AbstractSingleActivityOpe final FileSplit[] splits = fileSplitProvider.getFileSplits(); IIOManager ioManager = ctx.getIoManager(); // Frame accessor - final FrameTupleAccessor frameTupleAccessor = new FrameTupleAccessor( - recordDescProvider.getInputRecordDescriptor(getActivityId(), 0)); + final FrameTupleAccessor frameTupleAccessor = + new FrameTupleAccessor(recordDescProvider.getInputRecordDescriptor(getActivityId(), 0)); // Record descriptor final RecordDescriptor recordDescriptor = recordDescProvider.getInputRecordDescriptor(getActivityId(), 0); return new AbstractUnaryInputSinkOperatorNodePushable() { @@ -131,7 +131,7 @@ public class PlainFileWriterOperatorDescriptor extends AbstractSingleActivityOpe try { out.close(); } catch (IOException e) { - throw new HyracksDataException(e); + throw HyracksDataException.create(e); } } }; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/07075667/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/map/ReflectionBasedDeserializedMapperFactory.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/map/ReflectionBasedDeserializedMapperFactory.java b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/map/ReflectionBasedDeserializedMapperFactory.java index e7e084c..fa812cf 100644 --- a/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/map/ReflectionBasedDeserializedMapperFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-dataflow-std/src/main/java/org/apache/hyracks/dataflow/std/map/ReflectionBasedDeserializedMapperFactory.java @@ -33,10 +33,8 @@ public class ReflectionBasedDeserializedMapperFactory implements IDeserializedMa public IDeserializedMapper createMapper() throws HyracksDataException { try { return mapperClass.newInstance(); - } catch (InstantiationException e) { - throw new HyracksDataException(e); - } catch (IllegalAccessException e) { - throw new HyracksDataException(e); + } catch (Exception e) { + throw HyracksDataException.create(e); } } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/07075667/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-1.x/src/main/java/org/apache/hyracks/hdfs/ContextFactory.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-1.x/src/main/java/org/apache/hyracks/hdfs/ContextFactory.java b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-1.x/src/main/java/org/apache/hyracks/hdfs/ContextFactory.java index 2dbcf83..ecee076 100644 --- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-1.x/src/main/java/org/apache/hyracks/hdfs/ContextFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-1.x/src/main/java/org/apache/hyracks/hdfs/ContextFactory.java @@ -24,7 +24,6 @@ import org.apache.hadoop.mapreduce.JobID; import org.apache.hadoop.mapreduce.Mapper; import org.apache.hadoop.mapreduce.TaskAttemptContext; import org.apache.hadoop.mapreduce.TaskAttemptID; - import org.apache.hyracks.api.exceptions.HyracksDataException; /** @@ -37,7 +36,7 @@ public class ContextFactory { try { return new Mapper().new Context(conf, tid, null, null, null, null, null); } catch (Exception e) { - throw new HyracksDataException(e); + throw HyracksDataException.create(e); } } @@ -46,7 +45,7 @@ public class ContextFactory { TaskAttemptID tid = new TaskAttemptID("", 0, true, partition, 0); return new TaskAttemptContext(conf, tid); } catch (Exception e) { - throw new HyracksDataException(e); + throw HyracksDataException.create(e); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/07075667/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-2.x/src/main/java/org/apache/hyracks/hdfs/ContextFactory.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-2.x/src/main/java/org/apache/hyracks/hdfs/ContextFactory.java b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-2.x/src/main/java/org/apache/hyracks/hdfs/ContextFactory.java index dc6ca4e..96b0a23 100644 --- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-2.x/src/main/java/org/apache/hyracks/hdfs/ContextFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-2.x/src/main/java/org/apache/hyracks/hdfs/ContextFactory.java @@ -26,7 +26,6 @@ import org.apache.hadoop.mapreduce.TaskAttemptID; import org.apache.hadoop.mapreduce.TaskType; import org.apache.hadoop.mapreduce.task.JobContextImpl; import org.apache.hadoop.mapreduce.task.TaskAttemptContextImpl; - import org.apache.hyracks.api.exceptions.HyracksDataException; /** @@ -38,7 +37,7 @@ public class ContextFactory { try { return new TaskAttemptContextImpl(conf, tid); } catch (Exception e) { - throw new HyracksDataException(e); + throw HyracksDataException.create(e); } } @@ -47,7 +46,7 @@ public class ContextFactory { TaskAttemptID tid = new TaskAttemptID("", 0, TaskType.REDUCE, partition, 0); return new TaskAttemptContextImpl(conf, tid); } catch (Exception e) { - throw new HyracksDataException(e); + throw HyracksDataException.create(e); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/07075667/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/dataflow/ConfFactory.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/dataflow/ConfFactory.java b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/dataflow/ConfFactory.java index 3840167..33b84dc 100644 --- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/dataflow/ConfFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/dataflow/ConfFactory.java @@ -39,7 +39,7 @@ public class ConfFactory implements Serializable { confBytes = bos.toByteArray(); dos.close(); } catch (Exception e) { - throw new HyracksDataException(e); + throw HyracksDataException.create(e); } } @@ -51,7 +51,7 @@ public class ConfFactory implements Serializable { dis.close(); return conf; } catch (Exception e) { - throw new HyracksDataException(e); + throw HyracksDataException.create(e); } } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/07075667/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/dataflow/HDFSWriteOperatorDescriptor.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/dataflow/HDFSWriteOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/dataflow/HDFSWriteOperatorDescriptor.java index e7a3111..dcf4508 100644 --- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/dataflow/HDFSWriteOperatorDescriptor.java +++ b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/dataflow/HDFSWriteOperatorDescriptor.java @@ -27,7 +27,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapred.FileOutputFormat; import org.apache.hadoop.mapred.JobConf; - import org.apache.hyracks.api.context.IHyracksTaskContext; import org.apache.hyracks.api.dataflow.IOperatorNodePushable; import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider; @@ -124,7 +123,7 @@ public class HDFSWriteOperatorDescriptor extends AbstractSingleActivityOperatorD tupleWriter.close(dos); dos.close(); } catch (Exception e) { - throw new HyracksDataException(e); + throw HyracksDataException.create(e); } finally { Thread.currentThread().setContextClassLoader(ctxCL); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/07075667/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/dataflow/InputSplitsFactory.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/dataflow/InputSplitsFactory.java b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/dataflow/InputSplitsFactory.java index 927eb54..5cc09b7 100644 --- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/dataflow/InputSplitsFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs/dataflow/InputSplitsFactory.java @@ -103,7 +103,7 @@ public class InputSplitsFactory implements Serializable { dis.close(); return splits; } catch (Exception e) { - throw new HyracksDataException(e); + throw HyracksDataException.create(e); } } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/07075667/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs2/dataflow/ConfFactory.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs2/dataflow/ConfFactory.java b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs2/dataflow/ConfFactory.java index 0e49fef..b02a97b 100644 --- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs2/dataflow/ConfFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs2/dataflow/ConfFactory.java @@ -25,7 +25,6 @@ import java.io.DataOutputStream; import java.io.Serializable; import org.apache.hadoop.mapreduce.Job; - import org.apache.hyracks.api.exceptions.HyracksDataException; public class ConfFactory implements Serializable { @@ -40,7 +39,7 @@ public class ConfFactory implements Serializable { confBytes = bos.toByteArray(); dos.close(); } catch (Exception e) { - throw new HyracksDataException(e); + throw HyracksDataException.create(e); } } @@ -52,7 +51,7 @@ public class ConfFactory implements Serializable { dis.close(); return conf; } catch (Exception e) { - throw new HyracksDataException(e); + throw HyracksDataException.create(e); } } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/07075667/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs2/dataflow/FileSplitsFactory.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs2/dataflow/FileSplitsFactory.java b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs2/dataflow/FileSplitsFactory.java index ef95ee8..682b49a 100644 --- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs2/dataflow/FileSplitsFactory.java +++ b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs2/dataflow/FileSplitsFactory.java @@ -30,7 +30,6 @@ import java.util.ArrayList; import java.util.List; import org.apache.hadoop.mapreduce.lib.input.FileSplit; - import org.apache.hyracks.api.exceptions.HyracksDataException; @SuppressWarnings("rawtypes") @@ -96,7 +95,7 @@ public class FileSplitsFactory implements Serializable { ByteArrayInputStream bis = new ByteArrayInputStream(bytes); DataInputStream dis = new DataInputStream(bis); int size = dis.readInt(); - List<FileSplit> splits = new ArrayList<FileSplit>(); + List<FileSplit> splits = new ArrayList<>(); for (int i = 0; i < size; i++) { splits.add((FileSplit) defaultConstructor.newInstance()); splits.get(i).readFields(dis); @@ -104,7 +103,7 @@ public class FileSplitsFactory implements Serializable { dis.close(); return splits; } catch (Exception e) { - throw new HyracksDataException(e); + throw HyracksDataException.create(e); } } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/07075667/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs2/dataflow/HDFSWriteOperatorDescriptor.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs2/dataflow/HDFSWriteOperatorDescriptor.java b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs2/dataflow/HDFSWriteOperatorDescriptor.java index c27e4ec..1f163ba 100644 --- a/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs2/dataflow/HDFSWriteOperatorDescriptor.java +++ b/hyracks-fullstack/hyracks/hyracks-hdfs/hyracks-hdfs-core/src/main/java/org/apache/hyracks/hdfs2/dataflow/HDFSWriteOperatorDescriptor.java @@ -27,7 +27,6 @@ import org.apache.hadoop.fs.FileSystem; import org.apache.hadoop.fs.Path; import org.apache.hadoop.mapreduce.Job; import org.apache.hadoop.mapreduce.lib.output.FileOutputFormat; - import org.apache.hyracks.api.context.IHyracksTaskContext; import org.apache.hyracks.api.dataflow.IOperatorNodePushable; import org.apache.hyracks.api.dataflow.value.IRecordDescriptorProvider; @@ -123,7 +122,7 @@ public class HDFSWriteOperatorDescriptor extends AbstractSingleActivityOperatorD tupleWriter.close(dos); dos.close(); } catch (Exception e) { - throw new HyracksDataException(e); + throw HyracksDataException.create(e); } finally { Thread.currentThread().setContextClassLoader(ctxCL); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/07075667/hyracks-fullstack/hyracks/hyracks-ipc/src/main/java/org/apache/hyracks/ipc/impl/IPCSystem.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-ipc/src/main/java/org/apache/hyracks/ipc/impl/IPCSystem.java b/hyracks-fullstack/hyracks/hyracks-ipc/src/main/java/org/apache/hyracks/ipc/impl/IPCSystem.java index d9ab210..dea48bd 100644 --- a/hyracks-fullstack/hyracks/hyracks-ipc/src/main/java/org/apache/hyracks/ipc/impl/IPCSystem.java +++ b/hyracks-fullstack/hyracks/hyracks-ipc/src/main/java/org/apache/hyracks/ipc/impl/IPCSystem.java @@ -74,6 +74,7 @@ public class IPCSystem { } catch (IOException e) { throw new IPCException(e); } catch (InterruptedException e) { + Thread.currentThread().interrupt(); throw new IPCException(e); } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/07075667/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java index b358f07..07d07c3 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/dataflow/IndexSearchOperatorNodePushable.java @@ -215,7 +215,7 @@ public abstract class IndexSearchOperatorNodePushable extends AbstractUnaryInput } } catch (Throwable th) { writer.fail(); - closeException = new HyracksDataException(th); + closeException = HyracksDataException.create(th); } } @@ -223,7 +223,7 @@ public abstract class IndexSearchOperatorNodePushable extends AbstractUnaryInput cursor.close(); } catch (Throwable th) { if (closeException == null) { - closeException = new HyracksDataException(th); + closeException = HyracksDataException.create(th); } else { closeException.addSuppressed(th); } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/07075667/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/freepage/MutableArrayValueReference.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/freepage/MutableArrayValueReference.java b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/freepage/MutableArrayValueReference.java index a19e69a..157450a 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/freepage/MutableArrayValueReference.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-common/src/main/java/org/apache/hyracks/storage/am/common/freepage/MutableArrayValueReference.java @@ -18,6 +18,8 @@ */ package org.apache.hyracks.storage.am.common.freepage; +import java.nio.charset.StandardCharsets; + import org.apache.hyracks.data.std.api.IValueReference; public class MutableArrayValueReference implements IValueReference { @@ -46,4 +48,9 @@ public class MutableArrayValueReference implements IValueReference { return array == null ? 0 : array.length; } + @Override + public String toString() { + return new String(array, StandardCharsets.UTF_8); + } + } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/07075667/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponent.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponent.java index eb8ec92..33bb60e 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponent.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeDiskComponent.java @@ -63,4 +63,9 @@ public class LSMBTreeDiskComponent extends AbstractLSMDiskComponent { public int getFileReferenceCount() { return btree.getBufferCache().getFileReferenceCount(btree.getFileId()); } + + @Override + public String toString() { + return getClass().getSimpleName() + ":" + btree.getFileReference().getRelativePath(); + } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/07075667/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponent.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponent.java index 57b9092..0ba7c30 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponent.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-btree/src/main/java/org/apache/hyracks/storage/am/lsm/btree/impls/LSMBTreeWithBuddyDiskComponent.java @@ -74,4 +74,8 @@ public class LSMBTreeWithBuddyDiskComponent extends AbstractLSMDiskComponent { return btree.getBufferCache().getFileReferenceCount(btree.getFileId()); } + @Override + public String toString() { + return getClass().getSimpleName() + ":" + btree.getFileReference().getRelativePath(); + } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/07075667/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java index 8ff907a..b1005bd 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/impls/LSMHarness.java @@ -221,6 +221,7 @@ public class LSMHarness implements ILSMHarness { * See PrefixMergePolicy.isMergeLagging() for more details. */ if (opType == LSMOperationType.FLUSH) { + opTracker.notifyAll(); while (mergePolicy.isMergeLagging(lsmIndex)) { try { opTracker.wait(); @@ -672,16 +673,27 @@ public class LSMHarness implements ILSMHarness { } /*** - * Ensures the index is in a modifiable state - * @throws HyracksDataException if the index is not in a modifiable state + * Ensures the index is in a modifiable state (no failed flushes) + * + * @throws HyracksDataException + * if the index is not in a modifiable state */ private void ensureIndexModifiable() throws HyracksDataException { + // if current memory component has a flush request, it means that flush didn't start for it + if (lsmIndex.hasFlushRequestForCurrentMutableComponent()) { + return; + } // find if there is any memory component which is in a writable state or eventually will be in a writable state for (ILSMMemoryComponent memoryComponent : lsmIndex.getMemoryComponents()) { switch (memoryComponent.getState()) { case INACTIVE: + // will be activated on next modification + case UNREADABLE_UNWRITABLE: + // flush completed successfully but readers are still inside case READABLE_WRITABLE: + // writable case READABLE_UNWRITABLE_FLUSHING: + // flush is ongoing return; default: // continue to the next component http://git-wip-us.apache.org/repos/asf/asterixdb/blob/07075667/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/utils/ComponentMetadataUtil.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/utils/ComponentMetadataUtil.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/utils/ComponentMetadataUtil.java index 6ccbc8d..40017d1 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/utils/ComponentMetadataUtil.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-common/src/main/java/org/apache/hyracks/storage/am/lsm/common/utils/ComponentMetadataUtil.java @@ -19,6 +19,8 @@ package org.apache.hyracks.storage.am.lsm.common.utils; import java.util.List; +import java.util.logging.Level; +import java.util.logging.Logger; import org.apache.hyracks.api.exceptions.HyracksDataException; import org.apache.hyracks.data.std.api.IPointable; @@ -32,8 +34,8 @@ import org.apache.hyracks.storage.am.lsm.common.api.ILSMMemoryComponent; public class ComponentMetadataUtil { - public static final MutableArrayValueReference MARKER_LSN_KEY = - new MutableArrayValueReference("Marker".getBytes()); + private static final Logger LOGGER = Logger.getLogger(ComponentMetadataUtil.class.getName()); + public static final MutableArrayValueReference MARKER_LSN_KEY = new MutableArrayValueReference("Marker".getBytes()); public static final long NOT_FOUND = -1L; private ComponentMetadataUtil() { @@ -71,16 +73,28 @@ public class ComponentMetadataUtil { * @throws HyracksDataException */ public static void get(ILSMIndex index, IValueReference key, IPointable pointable) throws HyracksDataException { + LOGGER.log(Level.INFO, "Getting " + key + " from index " + index); // Lock the opTracker to ensure index components don't change synchronized (index.getOperationTracker()) { index.getCurrentMemoryComponent().getMetadata().get(key, pointable); if (pointable.getLength() == 0) { + LOGGER.log(Level.INFO, key + " was not found in mutable memory component of " + index); // was not found in the in current mutable component, search in the other in memory components fromImmutableMemoryComponents(index, key, pointable); if (pointable.getLength() == 0) { + LOGGER.log(Level.INFO, key + " was not found in all immmutable memory components of " + index); // was not found in the in all in memory components, search in the disk components fromDiskComponents(index, key, pointable); + if (pointable.getLength() == 0) { + LOGGER.log(Level.INFO, key + " was not found in all disk components of " + index); + } else { + LOGGER.log(Level.INFO, key + " was found in disk components of " + index); + } + } else { + LOGGER.log(Level.INFO, key + " was found in the immutable memory components of " + index); } + } else { + LOGGER.log(Level.INFO, key + " was found in mutable memory component of " + index); } } } @@ -105,7 +119,9 @@ public class ComponentMetadataUtil { private static void fromDiskComponents(ILSMIndex index, IValueReference key, IPointable pointable) throws HyracksDataException { + LOGGER.log(Level.INFO, "Getting " + key + " from disk components of " + index); for (ILSMDiskComponent c : index.getImmutableComponents()) { + LOGGER.log(Level.INFO, "Getting " + key + " from disk components " + c); c.getMetadata().get(key, pointable); if (pointable.getLength() != 0) { // Found @@ -115,10 +131,13 @@ public class ComponentMetadataUtil { } private static void fromImmutableMemoryComponents(ILSMIndex index, IValueReference key, IPointable pointable) { + LOGGER.log(Level.INFO, "Getting " + key + " from immutable memory components of " + index); List<ILSMMemoryComponent> memComponents = index.getMemoryComponents(); int numOtherMemComponents = memComponents.size() - 1; int next = index.getCurrentMemoryComponentIndex(); + LOGGER.log(Level.INFO, index + " has " + numOtherMemComponents + " immutable memory components"); for (int i = 0; i < numOtherMemComponents; i++) { + LOGGER.log(Level.INFO, "trying to get " + key + " from immutable memory components number: " + (i + 1)); next = next - 1; if (next < 0) { next = memComponents.size() - 1; http://git-wip-us.apache.org/repos/asf/asterixdb/blob/07075667/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponent.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponent.java index f2b3284..2470a39 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponent.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/impls/LSMInvertedIndexDiskComponent.java @@ -75,4 +75,9 @@ public class LSMInvertedIndexDiskComponent extends AbstractLSMDiskComponent { public int getFileReferenceCount() { return deletedKeysBTree.getBufferCache().getFileReferenceCount(deletedKeysBTree.getFileId()); } + + @Override + public String toString() { + return getClass().getSimpleName() + ":" + ((OnDiskInvertedIndex) invIndex).getInvListsFile().getRelativePath(); + } } http://git-wip-us.apache.org/repos/asf/asterixdb/blob/07075667/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java index d89a31a..9332302 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-invertedindex/src/main/java/org/apache/hyracks/storage/am/lsm/invertedindex/ondisk/OnDiskInvertedIndex.java @@ -43,8 +43,8 @@ import org.apache.hyracks.storage.am.common.api.IIndexOperationContext; import org.apache.hyracks.storage.am.common.api.IPageManagerFactory; import org.apache.hyracks.storage.am.common.impls.NoOpOperationCallback; import org.apache.hyracks.storage.am.common.tuples.PermutingTupleReference; -import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexAccessor; import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInPlaceInvertedIndex; +import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexAccessor; import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedIndexSearcher; import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListBuilder; import org.apache.hyracks.storage.am.lsm.invertedindex.api.IInvertedListCursor; @@ -292,7 +292,7 @@ public class OnDiskInvertedIndex implements IInPlaceInvertedIndex { output.writeInt(invListBuilder.getListSize()); btreeTupleBuilder.addFieldEndOffset(); } catch (IOException e) { - throw new HyracksDataException(e); + throw HyracksDataException.create(e); } // Reset tuple reference and add it into the BTree load. btreeTupleReference.reset(btreeTupleBuilder.getFieldEndOffsets(), btreeTupleBuilder.getByteArray()); http://git-wip-us.apache.org/repos/asf/asterixdb/blob/07075667/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDiskComponent.java ---------------------------------------------------------------------- diff --git a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDiskComponent.java b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDiskComponent.java index 982f89b..54ef122 100644 --- a/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDiskComponent.java +++ b/hyracks-fullstack/hyracks/hyracks-storage-am-lsm-rtree/src/main/java/org/apache/hyracks/storage/am/lsm/rtree/impls/LSMRTreeDiskComponent.java @@ -76,4 +76,9 @@ public class LSMRTreeDiskComponent extends AbstractLSMDiskComponent { public int getFileReferenceCount() { return rtree.getBufferCache().getFileReferenceCount(rtree.getFileId()); } + + @Override + public String toString() { + return getClass().getSimpleName() + ":" + rtree.getFileReference().getRelativePath(); + } }
