asdf2014 closed pull request #6741: emit logs that are only useful for
debugging at debug level
URL: https://github.com/apache/incubator-druid/pull/6741
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git
a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentInsertAction.java
b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentInsertAction.java
index c78aefe58df..42f74e59737 100644
---
a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentInsertAction.java
+++
b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentInsertAction.java
@@ -27,6 +27,7 @@
import org.apache.druid.timeline.DataSegment;
import java.util.Set;
+import java.util.stream.Collectors;
/**
* Insert segments into metadata storage. The segment versions must all be
less than or equal to a lock held by
@@ -83,7 +84,7 @@ public boolean isAudited()
public String toString()
{
return "SegmentInsertAction{" +
- "segments=" + segments +
+ "segments=" +
segments.stream().map(DataSegment::getIdentifier).collect(Collectors.joining(","))
+
'}';
}
}
diff --git
a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentMetadataUpdateAction.java
b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentMetadataUpdateAction.java
index fd6f88aab8f..4bb70a533d8 100644
---
a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentMetadataUpdateAction.java
+++
b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentMetadataUpdateAction.java
@@ -114,7 +114,7 @@ public boolean isAudited()
public String toString()
{
return "SegmentMetadataUpdateAction{" +
- "segments=" + segments +
+ "segments=" +
segments.stream().map(DataSegment::getIdentifier).collect(Collectors.joining(","))
+
'}';
}
}
diff --git
a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentNukeAction.java
b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentNukeAction.java
index ac5c286c7fc..db632a78507 100644
---
a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentNukeAction.java
+++
b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentNukeAction.java
@@ -116,7 +116,7 @@ public boolean isAudited()
public String toString()
{
return "SegmentNukeAction{" +
- "segments=" + segments +
+ "segments=" +
segments.stream().map(DataSegment::getIdentifier).collect(Collectors.joining(","))
+
'}';
}
}
diff --git
a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentTransactionalInsertAction.java
b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentTransactionalInsertAction.java
index ffc799befd9..3d733f90eff 100644
---
a/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentTransactionalInsertAction.java
+++
b/indexing-service/src/main/java/org/apache/druid/indexing/common/actions/SegmentTransactionalInsertAction.java
@@ -154,7 +154,7 @@ public boolean isAudited()
public String toString()
{
return "SegmentInsertAction{" +
- "segments=" + segments +
+ "segments=" +
segments.stream().map(DataSegment::getIdentifier).collect(Collectors.joining(","))
+
", startMetadata=" + startMetadata +
", endMetadata=" + endMetadata +
'}';
diff --git a/processing/src/main/java/org/apache/druid/segment/IndexMerger.java
b/processing/src/main/java/org/apache/druid/segment/IndexMerger.java
index 4661b652eed..87466d255e3 100644
--- a/processing/src/main/java/org/apache/druid/segment/IndexMerger.java
+++ b/processing/src/main/java/org/apache/druid/segment/IndexMerger.java
@@ -397,6 +397,8 @@ public TimeAndDimsPointer getMarkedPointer()
{
pQueue = new PriorityQueue<>(dimValueLookups.length,
NULLS_FIRST_PEEKING_COMPARATOR);
conversions = new IntBuffer[dimValueLookups.length];
+
+ long mergeBufferTotalSize = 0;
for (int i = 0; i < conversions.length; i++) {
if (dimValueLookups[i] == null) {
continue;
@@ -404,12 +406,14 @@ public TimeAndDimsPointer getMarkedPointer()
Indexed<String> indexed = dimValueLookups[i];
if (useDirect) {
int allocationSize = indexed.size() * Integer.BYTES;
- log.info("Allocating dictionary merging direct buffer with
size[%,d]", allocationSize);
+ log.debug("Allocating dictionary merging direct buffer with
size[%,d]", allocationSize);
+ mergeBufferTotalSize += allocationSize;
final ByteBuffer conversionDirectBuffer =
ByteBuffer.allocateDirect(allocationSize);
conversions[i] = conversionDirectBuffer.asIntBuffer();
directBufferAllocations.add(new Pair<>(conversionDirectBuffer,
allocationSize));
} else {
conversions[i] = IntBuffer.allocate(indexed.size());
+ mergeBufferTotalSize += indexed.size();
}
final PeekingIterator<String> iter = Iterators.peekingIterator(
@@ -422,6 +426,7 @@ public TimeAndDimsPointer getMarkedPointer()
pQueue.add(Pair.of(i, iter));
}
}
+ log.info("Allocated [%,d] bytes of dictionary merging direct buffers",
mergeBufferTotalSize);
}
@Override
@@ -482,10 +487,13 @@ public void remove()
@Override
public void close()
{
+ long mergeBufferTotalSize = 0;
for (Pair<ByteBuffer, Integer> bufferAllocation :
directBufferAllocations) {
- log.info("Freeing dictionary merging direct buffer with size[%,d]",
bufferAllocation.rhs);
+ log.debug("Freeing dictionary merging direct buffer with size[%,d]",
bufferAllocation.rhs);
+ mergeBufferTotalSize += bufferAllocation.rhs;
ByteBufferUtils.free(bufferAllocation.lhs);
}
+ log.info("Freed [,%d] bytes of dictionary merging direct buffers",
mergeBufferTotalSize);
}
}
}
diff --git
a/server/src/main/java/org/apache/druid/server/coordinator/CuratorLoadQueuePeon.java
b/server/src/main/java/org/apache/druid/server/coordinator/CuratorLoadQueuePeon.java
index c790382c2ee..e0972260ac9 100644
---
a/server/src/main/java/org/apache/druid/server/coordinator/CuratorLoadQueuePeon.java
+++
b/server/src/main/java/org/apache/druid/server/coordinator/CuratorLoadQueuePeon.java
@@ -235,10 +235,10 @@ private void processSegmentChangeRequest()
if (!segmentsToDrop.isEmpty()) {
currentlyProcessing = segmentsToDrop.firstEntry().getValue();
- log.info("Server[%s] dropping [%s]", basePath,
currentlyProcessing.getSegmentIdentifier());
+ log.debug("Server[%s] dropping [%s]", basePath,
currentlyProcessing.getSegmentIdentifier());
} else if (!segmentsToLoad.isEmpty()) {
currentlyProcessing = segmentsToLoad.firstEntry().getValue();
- log.info("Server[%s] loading [%s]", basePath,
currentlyProcessing.getSegmentIdentifier());
+ log.debug("Server[%s] loading [%s]", basePath,
currentlyProcessing.getSegmentIdentifier());
} else {
return;
}
@@ -386,7 +386,7 @@ private void entryRemoved(String path)
);
return;
}
- log.info(
+ log.debug(
"Server[%s] done processing %s of segment [%s]",
basePath,
currentlyProcessing.getType() == LOAD ? "load" : "drop",
diff --git
a/server/src/main/java/org/apache/druid/server/coordinator/DruidCoordinator.java
b/server/src/main/java/org/apache/druid/server/coordinator/DruidCoordinator.java
index 852068342c0..be4f106ca1a 100644
---
a/server/src/main/java/org/apache/druid/server/coordinator/DruidCoordinator.java
+++
b/server/src/main/java/org/apache/druid/server/coordinator/DruidCoordinator.java
@@ -334,7 +334,7 @@ public CoordinatorCompactionConfig getCompactionConfig()
public void removeSegment(DataSegment segment)
{
- log.info("Removing Segment[%s]", segment);
+ log.info("Removing Segment[%s]", segment.getIdentifier());
metadataSegmentManager.removeSegment(segment.getDataSource(),
segment.getIdentifier());
}
diff --git
a/server/src/main/java/org/apache/druid/server/coordinator/HttpLoadQueuePeon.java
b/server/src/main/java/org/apache/druid/server/coordinator/HttpLoadQueuePeon.java
index ef86d1efc24..a0641a21082 100644
---
a/server/src/main/java/org/apache/druid/server/coordinator/HttpLoadQueuePeon.java
+++
b/server/src/main/java/org/apache/druid/server/coordinator/HttpLoadQueuePeon.java
@@ -223,7 +223,7 @@ public void onSuccess(InputStream result)
handleResponseStatus(e.getRequest(),
e.getStatus());
break;
case PENDING:
- log.info("Request[%s] is still pending on
server[%s].", e.getRequest(), serverId);
+ log.debug("Request[%s] is still pending on
server[%s].", e.getRequest(), serverId);
break;
default:
scheduleNextRunImmediately = false;
@@ -521,7 +521,7 @@ public boolean hasTimedOut()
public void requestSucceeded()
{
- log.info(
+ log.debug(
"Server[%s] Successfully processed segment[%s] request[%s].",
serverId,
segment.getIdentifier(),
diff --git
a/server/src/main/java/org/apache/druid/server/coordinator/helper/DruidCoordinatorBalancer.java
b/server/src/main/java/org/apache/druid/server/coordinator/helper/DruidCoordinatorBalancer.java
index 97589c9bec0..129b4535085 100644
---
a/server/src/main/java/org/apache/druid/server/coordinator/helper/DruidCoordinatorBalancer.java
+++
b/server/src/main/java/org/apache/druid/server/coordinator/helper/DruidCoordinatorBalancer.java
@@ -155,11 +155,11 @@ private void balanceTier(
moveSegment(segmentToMoveHolder, destinationHolder.getServer(),
params);
moved++;
} else {
- log.info("Segment [%s] is 'optimally' placed.",
segmentToMove.getIdentifier());
+ log.debug("Segment [%s] is 'optimally' placed.",
segmentToMove.getIdentifier());
unmoved++;
}
} else {
- log.info(
+ log.debug(
"No valid movement destinations for segment [%s].",
segmentToMove.getIdentifier()
);
@@ -207,7 +207,7 @@ protected void moveSegment(
if (!toPeon.getSegmentsToLoad().contains(segmentToMove) &&
(toServer.getSegment(segmentName) == null) &&
new ServerHolder(toServer, toPeon).getAvailableSize() >
segmentToMove.getSize()) {
- log.info("Moving [%s] from [%s] to [%s]", segmentName,
fromServer.getName(), toServer.getName());
+ log.debug("Moving [%s] from [%s] to [%s]", segmentName,
fromServer.getName(), toServer.getName());
LoadPeonCallback callback = null;
try {
diff --git
a/server/src/main/java/org/apache/druid/server/coordinator/helper/DruidCoordinatorSegmentCompactor.java
b/server/src/main/java/org/apache/druid/server/coordinator/helper/DruidCoordinatorSegmentCompactor.java
index b02969737f1..4067b3a7a5d 100644
---
a/server/src/main/java/org/apache/druid/server/coordinator/helper/DruidCoordinatorSegmentCompactor.java
+++
b/server/src/main/java/org/apache/druid/server/coordinator/helper/DruidCoordinatorSegmentCompactor.java
@@ -160,7 +160,11 @@ private CoordinatorStats doRun(
config.getTuningConfig(),
config.getTaskContext()
);
- LOG.info("Submitted a compactTask[%s] for segments[%s]", taskId,
segmentsToCompact);
+ LOG.info(
+ "Submitted a compactTask[%s] for segments[%s]",
+ taskId,
+
segmentsToCompact.stream().map(DataSegment::getIdentifier).collect(Collectors.joining(","))
+ );
} else if (segmentsToCompact.size() == 1) {
throw new ISE("Found one segments[%s] to compact", segmentsToCompact);
} else {
diff --git
a/server/src/main/java/org/apache/druid/server/router/CoordinatorRuleManager.java
b/server/src/main/java/org/apache/druid/server/router/CoordinatorRuleManager.java
index afc1a6416d2..27661bc78df 100644
---
a/server/src/main/java/org/apache/druid/server/router/CoordinatorRuleManager.java
+++
b/server/src/main/java/org/apache/druid/server/router/CoordinatorRuleManager.java
@@ -154,7 +154,7 @@ public void poll()
)
);
- log.info("Got [%,d] rules", newRules.size());
+ log.debug("Got [%,d] rules", newRules.size());
rules.set(newRules);
}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
