This is an automated email from the ASF dual-hosted git repository.
divijv pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/kafka.git
The following commit(s) were added to refs/heads/trunk by this push:
new 180dcd39693 KAFKA-15294: Publish remote storage configs (#14266)
180dcd39693 is described below
commit 180dcd396932a4033df2104aaf0038b852c4fc1b
Author: Gantigmaa Selenge <[email protected]>
AuthorDate: Mon Aug 28 09:35:11 2023 +0100
KAFKA-15294: Publish remote storage configs (#14266)
This change does the following:
1. Make RemoteLogManagerConfigs that are implemented public
2. Add tasks to generate html docs for the configs
3. Include config docs in the main site
Reviewers: Divij Vaidya <[email protected]>, Luke Chen <[email protected]>,
Christo Lolov <[email protected]>, Satish Duggana <[email protected]>
---
build.gradle | 17 ++++++-
docs/configuration.html | 5 ++
docs/toc.html | 1 +
.../TopicBasedRemoteLogMetadataManagerConfig.java | 26 ++++++-----
.../log/remote/storage/RemoteLogManagerConfig.java | 54 ++++++++++++----------
5 files changed, 66 insertions(+), 37 deletions(-)
diff --git a/build.gradle b/build.gradle
index ad6d3876b50..b332932c63b 100644
--- a/build.gradle
+++ b/build.gradle
@@ -1074,7 +1074,8 @@ project(':core') {
':streams:genStreamsConfigDocs',
'genConsumerMetricsDocs', 'genProducerMetricsDocs',
':connect:runtime:genConnectMetricsDocs',
':connect:runtime:genConnectOpenAPIDocs',
':connect:mirror:genMirrorSourceConfigDocs',
':connect:mirror:genMirrorCheckpointConfigDocs',
- ':connect:mirror:genMirrorHeartbeatConfigDocs',
':connect:mirror:genMirrorConnectorConfigDocs'], type: Tar) {
+ ':connect:mirror:genMirrorHeartbeatConfigDocs',
':connect:mirror:genMirrorConnectorConfigDocs',
+ ':storage:genRemoteLogManagerConfigDoc',
':storage:genRemoteLogMetadataManagerConfigDoc'], type: Tar) {
archiveClassifier = 'site-docs'
compression = Compression.GZIP
from project.file("$rootDir/docs")
@@ -1766,6 +1767,20 @@ project(':storage') {
outputs.dir("src/generated/java/org/apache/kafka/server/log/remote/metadata/storage/generated")
}
+ task genRemoteLogManagerConfigDoc(type: JavaExec) {
+ classpath = sourceSets.main.runtimeClasspath
+ mainClass =
'org.apache.kafka.server.log.remote.storage.RemoteLogManagerConfig'
+ if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() }
+ standardOutput = new File(generatedDocsDir,
"remote_log_manager_config.html").newOutputStream()
+ }
+
+ task genRemoteLogMetadataManagerConfigDoc(type: JavaExec) {
+ classpath = sourceSets.main.runtimeClasspath
+ mainClass =
'org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManagerConfig'
+ if( !generatedDocsDir.exists() ) { generatedDocsDir.mkdirs() }
+ standardOutput = new File(generatedDocsDir,
"remote_log_metadata_manager_config.html").newOutputStream()
+ }
+
sourceSets {
main {
java {
diff --git a/docs/configuration.html b/docs/configuration.html
index 966f3fa765d..03038223b21 100644
--- a/docs/configuration.html
+++ b/docs/configuration.html
@@ -303,6 +303,11 @@
</tbody></table>
</li>
</ul>
+
+ <h3 class="anchor-heading"><a id="tieredstorageconfigs"
class="anchor-link"></a><a href="#tieredstorageconfigs">3.10 Tiered Storage
Configs</a></h3>
+ Below are the configuration properties for Tiered Storage.
+ <!--#include virtual="generated/remote_log_manager_config.html" -->
+ <!--#include virtual="generated/remote_log_metadata_manager_config.html" -->
</script>
<div class="p-configuration"></div>
diff --git a/docs/toc.html b/docs/toc.html
index d8979b1a904..9ef561e38c7 100644
--- a/docs/toc.html
+++ b/docs/toc.html
@@ -53,6 +53,7 @@
<li><a href="#adminclientconfigs">3.7 AdminClient Configs</a>
<li><a href="#mirrormakerconfigs">3.8 MirrorMaker Configs</a>
<li><a href="#systemproperties">3.9 System Properties</a>
+ <li><a href="#tieredstorageconfigs">3.10 Tiered Storage
Configs</a></li>
</ul>
</li>
<li><a href="#design">4. Design</a>
diff --git
a/storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManagerConfig.java
b/storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManagerConfig.java
index 1ab57f5b8d9..e130700078b 100644
---
a/storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManagerConfig.java
+++
b/storage/src/main/java/org/apache/kafka/server/log/remote/metadata/storage/TopicBasedRemoteLogMetadataManagerConfig.java
@@ -49,26 +49,26 @@ public final class TopicBasedRemoteLogMetadataManagerConfig
{
public static final String
REMOTE_LOG_METADATA_INITIALIZATION_RETRY_INTERVAL_MS_PROP =
"remote.log.metadata.initialization.retry.interval.ms";
public static final int DEFAULT_REMOTE_LOG_METADATA_TOPIC_PARTITIONS = 50;
- public static final long
DEFAULT_REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS = -1L;
+ public static final long DEFAULT_REMOTE_LOG_METADATA_TOPIC_RETENTION_MS =
-1L;
public static final short
DEFAULT_REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR = 3;
public static final long DEFAULT_REMOTE_LOG_METADATA_CONSUME_WAIT_MS = 2 *
60 * 1000L;
public static final long
DEFAULT_REMOTE_LOG_METADATA_INITIALIZATION_RETRY_MAX_TIMEOUT_MS = 2 * 60 *
1000L;
public static final long
DEFAULT_REMOTE_LOG_METADATA_INITIALIZATION_RETRY_INTERVAL_MS = 5 * 1000L;
- public static final String
REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_DOC = "Replication factor of
remote log metadata Topic.";
- public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_DOC = "The
number of partitions for remote log metadata Topic.";
- public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MS_DOC =
"Remote log metadata topic log retention in milli seconds." +
+ public static final String
REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_DOC = "Replication factor of
remote log metadata topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_PARTITIONS_DOC = "The
number of partitions for remote log metadata topic.";
+ public static final String REMOTE_LOG_METADATA_TOPIC_RETENTION_MS_DOC =
"Retention of remote log metadata topic in milliseconds. " +
"Default: -1, that means unlimited. Users can configure this value
based on their use cases. " +
"To avoid any data loss, this value should be more than the
maximum retention period of any topic enabled with " +
"tiered storage in the cluster.";
- public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_DOC = "The
amount of time in milli seconds to wait for the local consumer to " +
+ public static final String REMOTE_LOG_METADATA_CONSUME_WAIT_MS_DOC = "The
amount of time in milliseconds to wait for the local consumer to " +
"receive the published event.";
- public static final String
REMOTE_LOG_METADATA_INITIALIZATION_RETRY_INTERVAL_MS_DOC = "The retry interval
in milli seconds for " +
- " retrying RemoteLogMetadataManager resources initialization
again.";
+ public static final String
REMOTE_LOG_METADATA_INITIALIZATION_RETRY_INTERVAL_MS_DOC = "The retry interval
in milliseconds for " +
+ "retrying RemoteLogMetadataManager resources initialization
again.";
- public static final String
REMOTE_LOG_METADATA_INITIALIZATION_RETRY_MAX_TIMEOUT_MS_DOC = "The maximum
amount of time in milli seconds " +
- " for retrying RemoteLogMetadataManager resources initialization.
When total retry intervals reach this timeout, initialization" +
- " is considered as failed and broker starts shutting down.";
+ public static final String
REMOTE_LOG_METADATA_INITIALIZATION_RETRY_MAX_TIMEOUT_MS_DOC = "The maximum
amount of time in milliseconds " +
+ "for retrying RemoteLogMetadataManager resources initialization.
When total retry intervals reach this timeout, initialization " +
+ "is considered as failed and broker starts shutting down.";
public static final String REMOTE_LOG_METADATA_COMMON_CLIENT_PREFIX =
"remote.log.metadata.common.client.";
public static final String REMOTE_LOG_METADATA_PRODUCER_PREFIX =
"remote.log.metadata.producer.";
@@ -84,7 +84,7 @@ public final class TopicBasedRemoteLogMetadataManagerConfig {
REMOTE_LOG_METADATA_TOPIC_REPLICATION_FACTOR_DOC)
.define(REMOTE_LOG_METADATA_TOPIC_PARTITIONS_PROP, INT,
DEFAULT_REMOTE_LOG_METADATA_TOPIC_PARTITIONS, atLeast(1), LOW,
REMOTE_LOG_METADATA_TOPIC_PARTITIONS_DOC)
- .define(REMOTE_LOG_METADATA_TOPIC_RETENTION_MS_PROP, LONG,
DEFAULT_REMOTE_LOG_METADATA_TOPIC_RETENTION_MILLIS, LOW,
+ .define(REMOTE_LOG_METADATA_TOPIC_RETENTION_MS_PROP, LONG,
DEFAULT_REMOTE_LOG_METADATA_TOPIC_RETENTION_MS, LOW,
REMOTE_LOG_METADATA_TOPIC_RETENTION_MS_DOC)
.define(REMOTE_LOG_METADATA_CONSUME_WAIT_MS_PROP, LONG,
DEFAULT_REMOTE_LOG_METADATA_CONSUME_WAIT_MS, atLeast(0), LOW,
REMOTE_LOG_METADATA_CONSUME_WAIT_MS_DOC)
@@ -244,4 +244,8 @@ public final class TopicBasedRemoteLogMetadataManagerConfig
{
", producerProps=" + producerProps +
'}';
}
+
+ public static void main(String[] args) {
+ System.out.println(CONFIG.toHtml(4, config ->
"remote_log_metadata_manager_" + config));
+ }
}
diff --git
a/storage/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogManagerConfig.java
b/storage/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogManagerConfig.java
index 081fc27e79b..f6363639713 100644
---
a/storage/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogManagerConfig.java
+++
b/storage/src/main/java/org/apache/kafka/server/log/remote/storage/RemoteLogManagerConfig.java
@@ -56,28 +56,28 @@ public final class RemoteLogManagerConfig {
public static final String REMOTE_LOG_STORAGE_SYSTEM_ENABLE_PROP =
"remote.log.storage.system.enable";
- public static final String REMOTE_LOG_STORAGE_SYSTEM_ENABLE_DOC = "Whether
to enable tier storage functionality in a broker or not. Valid values " +
- "are `true` or `false` and the default value is false. When it is
true broker starts all the services required for tiered storage functionality.";
+ public static final String REMOTE_LOG_STORAGE_SYSTEM_ENABLE_DOC = "Whether
to enable tiered storage functionality in a broker or not. Valid values " +
+ "are `true` or `false` and the default value is false. When it is
true broker starts all the services required for the tiered storage
functionality.";
public static final boolean DEFAULT_REMOTE_LOG_STORAGE_SYSTEM_ENABLE =
false;
public static final String REMOTE_STORAGE_MANAGER_CLASS_NAME_PROP =
"remote.log.storage.manager.class.name";
public static final String REMOTE_STORAGE_MANAGER_CLASS_NAME_DOC = "Fully
qualified class name of `RemoteLogStorageManager` implementation.";
public static final String REMOTE_STORAGE_MANAGER_CLASS_PATH_PROP =
"remote.log.storage.manager.class.path";
- public static final String REMOTE_STORAGE_MANAGER_CLASS_PATH_DOC = "Class
path of the `RemoteLogStorageManager` implementation." +
- "If specified, the RemoteLogStorageManager implementation and its
dependent libraries will be loaded by a dedicated" +
- "classloader which searches this class path before the Kafka
broker class path. The syntax of this parameter is same" +
- "with the standard Java class path string.";
+ public static final String REMOTE_STORAGE_MANAGER_CLASS_PATH_DOC = "Class
path of the `RemoteLogStorageManager` implementation. " +
+ "If specified, the RemoteLogStorageManager implementation and its
dependent libraries will be loaded by a dedicated " +
+ "classloader which searches this class path before the Kafka
broker class path. The syntax of this parameter is same " +
+ "as the standard Java class path string.";
public static final String REMOTE_LOG_METADATA_MANAGER_CLASS_NAME_PROP =
"remote.log.metadata.manager.class.name";
public static final String REMOTE_LOG_METADATA_MANAGER_CLASS_NAME_DOC =
"Fully qualified class name of `RemoteLogMetadataManager` implementation.";
public static final String DEFAULT_REMOTE_LOG_METADATA_MANAGER_CLASS_NAME
=
"org.apache.kafka.server.log.remote.metadata.storage.TopicBasedRemoteLogMetadataManager";
public static final String REMOTE_LOG_METADATA_MANAGER_CLASS_PATH_PROP =
"remote.log.metadata.manager.class.path";
- public static final String REMOTE_LOG_METADATA_MANAGER_CLASS_PATH_DOC =
"Class path of the `RemoteLogMetadataManager` implementation." +
- "If specified, the RemoteLogMetadataManager implementation and its
dependent libraries will be loaded by a dedicated" +
- "classloader which searches this class path before the Kafka
broker class path. The syntax of this parameter is same" +
- "with the standard Java class path string.";
+ public static final String REMOTE_LOG_METADATA_MANAGER_CLASS_PATH_DOC =
"Class path of the `RemoteLogMetadataManager` implementation. " +
+ "If specified, the RemoteLogMetadataManager implementation and its
dependent libraries will be loaded by a dedicated " +
+ "classloader which searches this class path before the Kafka
broker class path. The syntax of this parameter is same " +
+ "as the standard Java class path string.";
public static final String REMOTE_LOG_METADATA_MANAGER_LISTENER_NAME_PROP
= "remote.log.metadata.manager.listener.name";
public static final String REMOTE_LOG_METADATA_MANAGER_LISTENER_NAME_DOC =
"Listener name of the local broker to which it should get connected if " +
@@ -146,52 +146,52 @@ public final class RemoteLogManagerConfig {
public static final ConfigDef CONFIG_DEF = new ConfigDef();
static {
- CONFIG_DEF.defineInternal(REMOTE_LOG_STORAGE_SYSTEM_ENABLE_PROP,
+ CONFIG_DEF.define(REMOTE_LOG_STORAGE_SYSTEM_ENABLE_PROP,
BOOLEAN,
DEFAULT_REMOTE_LOG_STORAGE_SYSTEM_ENABLE,
null,
MEDIUM,
REMOTE_LOG_STORAGE_SYSTEM_ENABLE_DOC)
- .defineInternal(REMOTE_STORAGE_MANAGER_CONFIG_PREFIX_PROP,
+ .define(REMOTE_STORAGE_MANAGER_CONFIG_PREFIX_PROP,
STRING,
DEFAULT_REMOTE_STORAGE_MANAGER_CONFIG_PREFIX,
new ConfigDef.NonEmptyString(),
MEDIUM,
REMOTE_STORAGE_MANAGER_CONFIG_PREFIX_DOC)
-
.defineInternal(REMOTE_LOG_METADATA_MANAGER_CONFIG_PREFIX_PROP,
+ .define(REMOTE_LOG_METADATA_MANAGER_CONFIG_PREFIX_PROP,
STRING,
DEFAULT_REMOTE_LOG_METADATA_MANAGER_CONFIG_PREFIX,
new ConfigDef.NonEmptyString(),
MEDIUM,
REMOTE_LOG_METADATA_MANAGER_CONFIG_PREFIX_DOC)
- .defineInternal(REMOTE_STORAGE_MANAGER_CLASS_NAME_PROP,
STRING,
+ .define(REMOTE_STORAGE_MANAGER_CLASS_NAME_PROP, STRING,
null,
new ConfigDef.NonEmptyString(),
MEDIUM,
REMOTE_STORAGE_MANAGER_CLASS_NAME_DOC)
- .defineInternal(REMOTE_STORAGE_MANAGER_CLASS_PATH_PROP,
STRING,
+ .define(REMOTE_STORAGE_MANAGER_CLASS_PATH_PROP, STRING,
null,
new ConfigDef.NonEmptyString(),
MEDIUM,
REMOTE_STORAGE_MANAGER_CLASS_PATH_DOC)
- .defineInternal(REMOTE_LOG_METADATA_MANAGER_CLASS_NAME_PROP,
+ .define(REMOTE_LOG_METADATA_MANAGER_CLASS_NAME_PROP,
STRING,
DEFAULT_REMOTE_LOG_METADATA_MANAGER_CLASS_NAME,
new ConfigDef.NonEmptyString(),
MEDIUM,
REMOTE_LOG_METADATA_MANAGER_CLASS_NAME_DOC)
- .defineInternal(REMOTE_LOG_METADATA_MANAGER_CLASS_PATH_PROP,
+ .define(REMOTE_LOG_METADATA_MANAGER_CLASS_PATH_PROP,
STRING,
null,
new ConfigDef.NonEmptyString(),
MEDIUM,
REMOTE_LOG_METADATA_MANAGER_CLASS_PATH_DOC)
-
.defineInternal(REMOTE_LOG_METADATA_MANAGER_LISTENER_NAME_PROP, STRING,
+ .define(REMOTE_LOG_METADATA_MANAGER_LISTENER_NAME_PROP,
STRING,
null,
new ConfigDef.NonEmptyString(),
MEDIUM,
REMOTE_LOG_METADATA_MANAGER_LISTENER_NAME_DOC)
-
.defineInternal(REMOTE_LOG_METADATA_CUSTOM_METADATA_MAX_BYTES_PROP,
+ .define(REMOTE_LOG_METADATA_CUSTOM_METADATA_MAX_BYTES_PROP,
INT,
DEFAULT_REMOTE_LOG_METADATA_CUSTOM_METADATA_MAX_BYTES,
atLeast(0),
@@ -203,13 +203,13 @@ public final class RemoteLogManagerConfig {
atLeast(1),
LOW,
REMOTE_LOG_INDEX_FILE_CACHE_TOTAL_SIZE_BYTES_DOC)
- .defineInternal(REMOTE_LOG_MANAGER_THREAD_POOL_SIZE_PROP,
+ .define(REMOTE_LOG_MANAGER_THREAD_POOL_SIZE_PROP,
INT,
DEFAULT_REMOTE_LOG_MANAGER_THREAD_POOL_SIZE,
atLeast(1),
MEDIUM,
REMOTE_LOG_MANAGER_THREAD_POOL_SIZE_DOC)
- .defineInternal(REMOTE_LOG_MANAGER_TASK_INTERVAL_MS_PROP,
+ .define(REMOTE_LOG_MANAGER_TASK_INTERVAL_MS_PROP,
LONG,
DEFAULT_REMOTE_LOG_MANAGER_TASK_INTERVAL_MS,
atLeast(1),
@@ -232,25 +232,25 @@ public final class RemoteLogManagerConfig {
between(0, 0.5),
LOW,
REMOTE_LOG_MANAGER_TASK_RETRY_JITTER_DOC)
- .defineInternal(REMOTE_LOG_READER_THREADS_PROP,
+ .define(REMOTE_LOG_READER_THREADS_PROP,
INT,
DEFAULT_REMOTE_LOG_READER_THREADS,
atLeast(1),
MEDIUM,
REMOTE_LOG_READER_THREADS_DOC)
- .defineInternal(REMOTE_LOG_READER_MAX_PENDING_TASKS_PROP,
+ .define(REMOTE_LOG_READER_MAX_PENDING_TASKS_PROP,
INT,
DEFAULT_REMOTE_LOG_READER_MAX_PENDING_TASKS,
atLeast(1),
MEDIUM,
REMOTE_LOG_READER_MAX_PENDING_TASKS_DOC)
- .defineInternal(LOG_LOCAL_RETENTION_MS_PROP,
+ .define(LOG_LOCAL_RETENTION_MS_PROP,
LONG,
DEFAULT_LOG_LOCAL_RETENTION_MS,
atLeast(DEFAULT_LOG_LOCAL_RETENTION_MS),
MEDIUM,
LOG_LOCAL_RETENTION_MS_DOC)
- .defineInternal(LOG_LOCAL_RETENTION_BYTES_PROP,
+ .define(LOG_LOCAL_RETENTION_BYTES_PROP,
LONG,
DEFAULT_LOG_LOCAL_RETENTION_BYTES,
atLeast(DEFAULT_LOG_LOCAL_RETENTION_BYTES),
@@ -457,4 +457,8 @@ public final class RemoteLogManagerConfig {
remoteLogReaderThreads,
remoteLogReaderMaxPendingTasks, remoteStorageManagerProps,
remoteLogMetadataManagerProps,
remoteStorageManagerPrefix,
remoteLogMetadataManagerPrefix);
}
+
+ public static void main(String[] args) {
+ System.out.println(CONFIG_DEF.toHtml(4, config ->
"remote_log_manager_" + config));
+ }
}
