Repository: drill Updated Branches: refs/heads/master cf2b7c70e -> bbcf4b765
DRILL-5123: Write query profile after sending final response to client to improve latency In testing a particular query, I used a test setup that does not write to the "persistent store", causing query profiles to not be saved. I then changed the config to save them (to local disk). This produced about a 200ms difference in query run time as perceived by the client. I then moved writing the query profile after sending the client the final message. This resulted in an approximately 100ms savings, as perceived by the client, in query run time on short (~3 sec.) queries. close apache/drill#692 Project: http://git-wip-us.apache.org/repos/asf/drill/repo Commit: http://git-wip-us.apache.org/repos/asf/drill/commit/417ae930 Tree: http://git-wip-us.apache.org/repos/asf/drill/tree/417ae930 Diff: http://git-wip-us.apache.org/repos/asf/drill/diff/417ae930 Branch: refs/heads/master Commit: 417ae930a3839bb0ef8758346fe855611051d037 Parents: cf2b7c7 Author: Paul Rogers <[email protected]> Authored: Mon Dec 12 09:18:38 2016 -0800 Committer: Jinfeng Ni <[email protected]> Committed: Mon Dec 19 14:45:26 2016 -0800 ---------------------------------------------------------------------- .../apache/drill/exec/work/foreman/Foreman.java | 22 +++++++++++++------- 1 file changed, 14 insertions(+), 8 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/drill/blob/417ae930/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java ---------------------------------------------------------------------- diff --git a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java index 808ba07..c6a3104 100644 --- a/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java +++ b/exec/java-exec/src/main/java/org/apache/drill/exec/work/foreman/Foreman.java @@ -555,7 +555,6 @@ public class Foreman implements Runnable { final String queueName; try { - @SuppressWarnings("resource") final ClusterCoordinator clusterCoordinator = drillbitContext.getClusterCoordinator(); final DistributedSemaphore distributedSemaphore; @@ -828,9 +827,6 @@ public class Foreman implements Runnable { uex = null; } - // we store the final result here so we can capture any error/errorId in the profile for later debugging. - queryManager.writeFinalProfile(uex); - /* * If sending the result fails, we don't really have any way to modify the result we tried to send; * it is possible it got sent but the result came from a later part of the code path. It is also @@ -845,6 +841,19 @@ public class Foreman implements Runnable { logger.warn("Exception sending result to client", resultException); } + // Store the final result here so we can capture any error/errorId in the + // profile for later debugging. + // Write the query profile AFTER sending results to the user. The observed + // user behavior is a possible time-lag between query return and appearance + // of the query profile in persistent storage. Also, the query might + // succeed, but the profile never appear if the profile write fails. This + // behavior is acceptable for an eventually-consistent distributed system. + // The key benefit is that the client does not wait for the persistent + // storage write; query completion occurs in parallel with profile + // persistence. + + queryManager.writeFinalProfile(uex); + // Remove the Foreman from the running query list. bee.retireForeman(Foreman.this); @@ -1031,10 +1040,8 @@ public class Foreman implements Runnable { */ private void setupRootFragment(final PlanFragment rootFragment, final FragmentRoot rootOperator) throws ExecutionSetupException { - @SuppressWarnings("resource") final FragmentContext rootContext = new FragmentContext(drillbitContext, rootFragment, queryContext, initiatingClient, drillbitContext.getFunctionImplementationRegistry()); - @SuppressWarnings("resource") final IncomingBuffers buffers = new IncomingBuffers(rootFragment, rootContext); rootContext.setBuffers(buffers); @@ -1161,7 +1168,6 @@ public class Foreman implements Runnable { */ private void sendRemoteFragments(final DrillbitEndpoint assignment, final Collection<PlanFragment> fragments, final CountDownLatch latch, final FragmentSubmitFailures fragmentSubmitFailures) { - @SuppressWarnings("resource") final Controller controller = drillbitContext.getController(); final InitializeFragments.Builder fb = InitializeFragments.newBuilder(); for(final PlanFragment planFragment : fragments) { @@ -1187,7 +1193,7 @@ public class Foreman implements Runnable { final DrillbitEndpoint drillbitEndpoint; final RpcException rpcException; - SubmissionException(@SuppressWarnings("unused") final DrillbitEndpoint drillbitEndpoint, + SubmissionException(final DrillbitEndpoint drillbitEndpoint, final RpcException rpcException) { this.drillbitEndpoint = drillbitEndpoint; this.rpcException = rpcException;
