Caideyipi commented on code in PR #18039:
URL: https://github.com/apache/iotdb/pull/18039#discussion_r3517832614
##########
iotdb-core/datanode/src/main/java/org/apache/iotdb/db/queryengine/common/MPPQueryContext.java:
##########
@@ -555,6 +597,253 @@ public void releaseMemoryReservedForFrontEnd(final long
bytes) {
this.memoryReservationManager.releaseMemoryCumulatively(bytes);
}
+ public void initResultSetColumnMemoryTracking(
+ long seriesLimit, long seriesOffset, boolean alignByDevice) {
+ resetResultSetColumnMemoryTracking();
+ resultSetColumnMemoryTrackingEnabled = true;
+ seriesLimitForResultSetColumnTracking = seriesLimit;
+ seriesOffsetForResultSetColumnTracking = seriesOffset;
+ alignByDeviceForResultSetColumnTracking = alignByDevice;
+ }
+
+ public void recordMatchedSourceColumnsForResultSet(long columnCount) {
+ if (resultSetColumnMemoryTrackingEnabled && columnCount > 0) {
+ matchedSourceColumnsForResultSet += columnCount;
+ }
+ }
+
+ public void recordExpandedSourceColumnForResultSet(long memoryCost) {
+ if (!resultSetColumnMemoryTrackingEnabled) {
+ return;
+ }
+ expandedSourceColumnsForResultSet++;
+ sourceColumnMemoryCostForResultSet += Math.max(memoryCost, 0);
+ }
+
+ public void recordGeneratedResultSetColumn(long memoryCost) {
+ if (!resultSetColumnMemoryTrackingEnabled) {
+ return;
+ }
+ generatedResultSetColumns++;
+ generatedResultSetColumnMemoryCost += Math.max(memoryCost, 0);
+ }
+
+ public void recordSchemaFetchDeserializedColumns(long columnCount) {
+ if (columnCount > 0) {
+ schemaFetchDeserializedColumnCount += columnCount;
+ }
+ }
+
+ private void resetResultSetColumnMemoryTracking() {
+ resultSetColumnMemoryTrackingEnabled = false;
+ alignByDeviceForResultSetColumnTracking = false;
+ seriesLimitForResultSetColumnTracking = 0;
+ seriesOffsetForResultSetColumnTracking = 0;
+ matchedSourceColumnsForResultSet = 0;
+ expandedSourceColumnsForResultSet = 0;
+ sourceColumnMemoryCostForResultSet = 0;
+ generatedResultSetColumns = 0;
+ generatedResultSetColumnMemoryCost = 0;
+ schemaFetchEstimatedMemoryCost = 0;
+ schemaFetchReservedMemoryCost = 0;
+ schemaFetchDeserializedColumnCount = 0;
+ }
+
+ private MemoryNotEnoughException
enrichResultSetColumnMemoryNotEnoughException(
+ MemoryNotEnoughException e, long requestedBytes) {
+ if (!resultSetColumnMemoryTrackingEnabled
+ || (matchedSourceColumnsForResultSet == 0
+ && expandedSourceColumnsForResultSet == 0
+ && generatedResultSetColumns == 0)) {
+ return e;
+ }
+
+ long freeBytes =
LocalExecutionPlanner.getInstance().getFreeMemoryForOperators();
+ long shortageBytes =
+ requestedBytes > 0 && requestedBytes > freeBytes ? requestedBytes -
freeBytes : -1;
+ long exceededColumns = estimateExceededColumns(freeBytes, requestedBytes);
+
+ return new MemoryNotEnoughException(
+ String.format(
+ Locale.ROOT,
+ "Not enough memory while analyzing metadata for query result
columns. "
+ + "The result set has too many columns. "
+ + "Before the failure, IoTDB had matched %,d source columns
for result-column "
+ + "expansion, expanded %,d source columns, and generated %,d
result-set columns. "
+ + "%s"
+ + "Current series pagination is %s. "
+ + "Use SLIMIT/SOFFSET to reduce returned series%s, narrow the
path pattern, "
+ + "or increase query memory%s. "
+ + "Memory details: source-column memory for result expansion
%s, "
+ + "generated-result-column memory %s, requested this time %s,
current free memory %s. "
+ + "Original error: %s",
Review Comment:
Applied in 9fa306665d3: moved the metadata OOM diagnostic text into
DataNodeQueryMessages with both English and Chinese i18n constants.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
