[
https://issues.apache.org/jira/browse/HADOOP-18959?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17788176#comment-17788176
]
ASF GitHub Bot commented on HADOOP-18959:
-----------------------------------------
virajjasani commented on code in PR #6240:
URL: https://github.com/apache/hadoop/pull/6240#discussion_r1399837597
##########
hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/impl/prefetch/BlockManagerParameters.java:
##########
@@ -0,0 +1,149 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.hadoop.fs.impl.prefetch;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.LocalDirAllocator;
+import org.apache.hadoop.fs.statistics.DurationTrackerFactory;
+
+/**
+ * This class is used to provide parameters to {@link BlockManager}.
+ */
[email protected]
+public final class BlockManagerParameters {
+
+ /**
+ * Asynchronous tasks are performed in this pool.
+ */
+ private ExecutorServiceFuturePool futurePool;
+
+ /**
+ * Information about each block of the underlying file.
+ */
+ private BlockData blockData;
+
+ /**
+ * Size of the in-memory cache in terms of number of blocks.
+ */
+ private int bufferPoolSize;
+
+ /**
+ * Statistics for the stream.
+ */
+ private PrefetchingStatistics prefetchingStatistics;
+
+ /**
+ * The configuration object.
+ */
+ private Configuration conf;
+
+ /**
+ * The local dir allocator instance.
+ */
+ private LocalDirAllocator localDirAllocator;
+
+ /**
+ * Max blocks count to be kept in cache at any time.
+ */
+ private int maxBlocksCount;
+
+ /**
+ * Tracker with statistics to update.
+ */
+ private DurationTrackerFactory trackerFactory;
+
+ public ExecutorServiceFuturePool getFuturePool() {
+ return futurePool;
+ }
+
+ public BlockData getBlockData() {
+ return blockData;
+ }
+
+ public int getBufferPoolSize() {
+ return bufferPoolSize;
+ }
+
+ public PrefetchingStatistics getPrefetchingStatistics() {
+ return prefetchingStatistics;
+ }
+
+ public Configuration getConf() {
+ return conf;
+ }
+
+ public LocalDirAllocator getLocalDirAllocator() {
+ return localDirAllocator;
+ }
+
+ public int getMaxBlocksCount() {
+ return maxBlocksCount;
+ }
+
+ public DurationTrackerFactory getTrackerFactory() {
+ return trackerFactory;
+ }
+
+ public BlockManagerParameters setFuturePool(
Review Comment:
done
##########
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/TestS3ACachingBlockManager.java:
##########
@@ -61,50 +65,93 @@ public class TestS3ACachingBlockManager extends
AbstractHadoopTestBase {
private final BlockData blockData = new BlockData(FILE_SIZE, BLOCK_SIZE);
+ private static final Configuration CONF =
+ S3ATestUtils.prepareTestConfiguration(new Configuration());
+
@Test
public void testArgChecks() throws Exception {
MockS3ARemoteObject s3File = new MockS3ARemoteObject(FILE_SIZE, false);
S3ARemoteObjectReader reader = new S3ARemoteObjectReader(s3File);
Configuration conf = new Configuration();
+ BlockManagerParameters blockManagerParamsBuilder1 =
Review Comment:
done
##########
hadoop-tools/hadoop-aws/src/test/java/org/apache/hadoop/fs/s3a/prefetch/TestS3ACachingBlockManager.java:
##########
@@ -244,9 +292,18 @@ private void testPrefetchHelper(boolean
forcePrefetchFailure)
throws IOException, InterruptedException {
MockS3ARemoteObject s3File = new MockS3ARemoteObject(FILE_SIZE, false);
S3ARemoteObjectReader reader = new S3ARemoteObjectReader(s3File);
+ BlockManagerParameters blockManagerParamsBuilder =
Review Comment:
done
> Use builder for prefetch CachingBlockManager
> --------------------------------------------
>
> Key: HADOOP-18959
> URL: https://issues.apache.org/jira/browse/HADOOP-18959
> Project: Hadoop Common
> Issue Type: Sub-task
> Reporter: Viraj Jasani
> Assignee: Viraj Jasani
> Priority: Major
> Labels: pull-request-available
>
> Some of the recent changes (HADOOP-18399, HADOOP-18291, HADOOP-18829 etc)
> have added more params for prefetch CachingBlockManager c'tor to process
> read/write block requests. They have added too many params and more are
> likely to be introduced later. We should use builder pattern to pass params.
> This would also help consolidating required prefetch params into one single
> place within S3ACachingInputStream, from scattered locations.
--
This message was sent by Atlassian Jira
(v8.20.10#820010)
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
