jon-wei commented on a change in pull request #8201: Simple memory allocation
for CliIndexer tasks
URL: https://github.com/apache/incubator-druid/pull/8201#discussion_r309416102
##########
File path:
server/src/main/java/org/apache/druid/segment/realtime/appenderator/UnifiedIndexerAppenderatorsManager.java
##########
@@ -21,62 +21,102 @@
import com.fasterxml.jackson.databind.ObjectMapper;
import com.google.common.base.Preconditions;
+import com.google.common.util.concurrent.ListenableFuture;
+import com.google.common.util.concurrent.ListeningExecutorService;
+import com.google.common.util.concurrent.MoreExecutors;
import com.google.inject.Inject;
import org.apache.druid.client.cache.Cache;
import org.apache.druid.client.cache.CacheConfig;
import org.apache.druid.client.cache.CachePopulatorStats;
import org.apache.druid.guice.annotations.Processing;
+import org.apache.druid.indexing.worker.config.WorkerConfig;
import org.apache.druid.java.util.common.IAE;
+import org.apache.druid.java.util.common.UOE;
+import org.apache.druid.java.util.common.concurrent.Execs;
+import org.apache.druid.java.util.emitter.EmittingLogger;
import org.apache.druid.java.util.emitter.service.ServiceEmitter;
import org.apache.druid.query.Query;
import org.apache.druid.query.QueryRunner;
import org.apache.druid.query.QueryRunnerFactoryConglomerate;
import org.apache.druid.query.SegmentDescriptor;
+import org.apache.druid.query.aggregation.AggregatorFactory;
import org.apache.druid.segment.IndexIO;
import org.apache.druid.segment.IndexMerger;
+import org.apache.druid.segment.IndexSpec;
+import org.apache.druid.segment.IndexableAdapter;
+import org.apache.druid.segment.ProgressIndicator;
+import org.apache.druid.segment.QueryableIndex;
+import org.apache.druid.segment.incremental.IncrementalIndex;
import org.apache.druid.segment.indexing.DataSchema;
import org.apache.druid.segment.loading.DataSegmentPusher;
import org.apache.druid.segment.realtime.FireDepartmentMetrics;
import org.apache.druid.segment.realtime.plumber.Sink;
+import org.apache.druid.segment.writeout.SegmentWriteOutMediumFactory;
import org.apache.druid.server.coordination.DataSegmentAnnouncer;
import org.apache.druid.timeline.VersionedIntervalTimeline;
import org.joda.time.Interval;
+import org.joda.time.Period;
+import javax.annotation.Nullable;
+import java.io.File;
+import java.io.IOException;
+import java.util.List;
+import java.util.concurrent.Callable;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ExecutorService;
/**
- * Manages Appenderators for the Indexer task execution service, which runs
all tasks in a single process.
+ * Manages {@link Appenderator} instances for the Indexer task execution
service, which runs all tasks in
+ * a single process.
*
* This class keeps two maps:
- * - A per-datasource SinkQuerySegmentWalker (with an associated
per-datasource timeline)
+ * - A per-datasource {@link SinkQuerySegmentWalker} (with an associated
per-datasource timeline)
* - A map that associates a taskId with the Appenderator created for that task
*
* Appenderators created by this class will use the shared per-datasource
SinkQuerySegmentWalkers.
*
* The per-datasource SinkQuerySegmentWalkers share a common
queryExecutorService.
+ *
+ * Each task that requests an Appenderator from this AppenderatorsManager will
receive a heap memory limit
+ * equal to `druid.worker.globalIngestionHeapLimitBytes` evenly divided by
`druid.worker.capacity`.
+ *
+ * The Appenderators created by this class share an executor pool for {@link
IndexMerger} persist
+ * and merge operations, with concurrent operations limited to
`druid.worker.capacity` divided 2. This limit is imposed
+ * to reduce overall memory usage.
*/
public class UnifiedIndexerAppenderatorsManager implements AppenderatorsManager
{
+ private static final EmittingLogger LOG = new
EmittingLogger(UnifiedIndexerAppenderatorsManager.class);
Review comment:
Removed, thanks
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
[email protected]
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
