This is an automated email from the ASF dual-hosted git repository.
mcvsubbu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pinot.git
The following commit(s) were added to refs/heads/master by this push:
new dd595cf Updated realtime provisioning helper to take in max memory
available in a host (#3997)
dd595cf is described below
commit dd595cf0af5b8ffc01407671dcf6df50c983dda1
Author: Subbu Subramaniam <[email protected]>
AuthorDate: Thu Mar 21 09:55:35 2019 -0700
Updated realtime provisioning helper to take in max memory available in a
host (#3997)
In installations where hosts have memory limits other than 48G, the admin
can
specify other memory limits to be used to provision realtime hosts
---
.../admin/command/RealtimeProvisioningHelperCommand.java | 15 +++++++++++++--
.../tools/realtime/provisioning/MemoryEstimator.java | 13 +++++++------
2 files changed, 20 insertions(+), 8 deletions(-)
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/RealtimeProvisioningHelperCommand.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/RealtimeProvisioningHelperCommand.java
index 593c8dc..930c27c 100644
---
a/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/RealtimeProvisioningHelperCommand.java
+++
b/pinot-tools/src/main/java/org/apache/pinot/tools/admin/command/RealtimeProvisioningHelperCommand.java
@@ -26,6 +26,7 @@ import java.util.concurrent.TimeUnit;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang3.StringUtils;
import org.apache.pinot.common.config.TableConfig;
+import org.apache.pinot.common.utils.DataSize;
import org.apache.pinot.common.utils.time.TimeUtils;
import org.apache.pinot.tools.Command;
import org.apache.pinot.tools.realtime.provisioning.MemoryEstimator;
@@ -72,6 +73,9 @@ public class RealtimeProvisioningHelperCommand extends
AbstractBaseAdminCommand
@Option(name = "-periodSampleSegmentConsumed", required = true, metaVar =
"<String>", usage = "Period for which the sample segment was consuming in
format 4h, 5h30m, 40m etc")
private String _periodSampleSegmentConsumed;
+ @Option(name = "-maxUsableHostMemory", required = false, metaVar =
"<String>", usage = "Maximum memory per host that can be used for pinot data
(e.g. 250G, 100M). Default 48g")
+ private String _maxUsableHostMemory = "48G";
+
@Option(name = "-help", help = true, aliases = {"-h", "--h", "--help"})
private boolean _help = false;
@@ -95,6 +99,11 @@ public class RealtimeProvisioningHelperCommand extends
AbstractBaseAdminCommand
return this;
}
+ public RealtimeProvisioningHelperCommand setMaxUsableHostMemory(String
maxUsableHostMemory) {
+ _maxUsableHostMemory = maxUsableHostMemory;
+ return this;
+ }
+
public RealtimeProvisioningHelperCommand setNumHours(String numHours) {
_numHours = numHours;
return this;
@@ -115,7 +124,7 @@ public class RealtimeProvisioningHelperCommand extends
AbstractBaseAdminCommand
return ("RealtimeProvisioningHelperCommand -tableConfigFile " +
_tableConfigFile + " -numPartitions "
+ _numPartitions + " -retentionHours " + _retentionHours + " -numHosts
" + _numHosts + " -numHours " + _numHours
+ " -sampleCompletedSegmentDir " + _sampleCompletedSegmentDir + "
-periodSampleSegmentConsumed "
- + _periodSampleSegmentConsumed);
+ + _periodSampleSegmentConsumed + "-maxUsableMemory " +
_maxUsableHostMemory);
}
@Override
@@ -170,8 +179,10 @@ public class RealtimeProvisioningHelperCommand extends
AbstractBaseAdminCommand
long sampleSegmentConsumedSeconds =
TimeUnit.SECONDS.convert(TimeUtils.convertPeriodToMillis(_periodSampleSegmentConsumed),
TimeUnit.MILLISECONDS);
+ long maxUsableHostMemBytes = DataSize.toBytes(_maxUsableHostMemory);
+
MemoryEstimator memoryEstimator =
- new MemoryEstimator(tableConfig, sampleCompletedSegmentFile,
sampleSegmentConsumedSeconds);
+ new MemoryEstimator(tableConfig, sampleCompletedSegmentFile,
sampleSegmentConsumedSeconds, maxUsableHostMemBytes);
File sampleStatsHistory = memoryEstimator.initializeStatsHistory();
memoryEstimator
.estimateMemoryUsed(sampleStatsHistory, numHosts, numHours,
totalConsumingPartitions, _retentionHours);
diff --git
a/pinot-tools/src/main/java/org/apache/pinot/tools/realtime/provisioning/MemoryEstimator.java
b/pinot-tools/src/main/java/org/apache/pinot/tools/realtime/provisioning/MemoryEstimator.java
index 36879fe..1832c6c 100644
---
a/pinot-tools/src/main/java/org/apache/pinot/tools/realtime/provisioning/MemoryEstimator.java
+++
b/pinot-tools/src/main/java/org/apache/pinot/tools/realtime/provisioning/MemoryEstimator.java
@@ -44,15 +44,15 @@ import
org.apache.pinot.core.segment.index.SegmentMetadataImpl;
*/
public class MemoryEstimator {
- private static final long MAX_MEMORY_BYTES = DataSize.toBytes("48G");
private static final String NOT_APPLICABLE = "NA";
private static final String TMP_DIR = System.getProperty("java.io.tmpdir") +
File.separator;
private static final String STATS_FILE_NAME = "stats.ser";
private static final String STATS_FILE_COPY_NAME = "stats.copy.ser";
- private TableConfig _tableConfig;
- private File _sampleCompletedSegment;
- private long _sampleSegmentConsumedSeconds;
+ private final TableConfig _tableConfig;
+ private final File _sampleCompletedSegment;
+ private final long _sampleSegmentConsumedSeconds;
+ private final long _maxUsableHostMemory;
private SegmentMetadataImpl _segmentMetadata;
private long _sampleCompletedSegmentSizeBytes;
@@ -65,7 +65,8 @@ public class MemoryEstimator {
private String[][] _optimalSegmentSize;
private String[][] _consumingMemoryPerHost;
- public MemoryEstimator(TableConfig tableConfig, File sampleCompletedSegment,
long sampleSegmentConsumedSeconds) {
+ public MemoryEstimator(TableConfig tableConfig, File sampleCompletedSegment,
long sampleSegmentConsumedSeconds, long maxUsableHostMemory) {
+ _maxUsableHostMemory = maxUsableHostMemory;
_tableConfig = tableConfig;
_sampleCompletedSegment = sampleCompletedSegment;
_sampleSegmentConsumedSeconds = sampleSegmentConsumedSeconds;
@@ -242,7 +243,7 @@ public class MemoryEstimator {
memoryForConsumingSegmentPerPartition *
totalConsumingPartitionsPerHost;
long totalMemoryPerHostBytes = totalMemoryForCompletedSegmentsPerHost
+ totalMemoryForConsumingSegmentsPerHost;
- if (totalMemoryPerHostBytes > MAX_MEMORY_BYTES) {
+ if (totalMemoryPerHostBytes > _maxUsableHostMemory) {
_totalMemoryPerHost[i][j] = NOT_APPLICABLE;
_consumingMemoryPerHost[i][j] = NOT_APPLICABLE;
_optimalSegmentSize[i][j] = NOT_APPLICABLE;
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
