This is an automated email from the ASF dual-hosted git repository.
arnold pushed a commit to branch develop
in repository https://gitbox.apache.org/repos/asf/fineract.git
The following commit(s) were added to refs/heads/develop by this push:
new c5aeae62d FINERACT-1824 - Documentation update for Kafka Support -
Adding docker-compose-postgresql-kafka-msk.yml - application.properties ENV
var fixed
c5aeae62d is described below
commit c5aeae62d8bfb8c44f81b12d7a927d339281c0d9
Author: Peter Bagrij <[email protected]>
AuthorDate: Mon Jul 24 11:41:43 2023 +0200
FINERACT-1824
- Documentation update for Kafka Support
- Adding docker-compose-postgresql-kafka-msk.yml
- application.properties ENV var fixed
---
README.md | 21 ++-
docker-compose-postgresql-kafka-msk.yml | 182 +++++++++++++++++++++
.../en/chapters/appendix/properties-kafka.adoc | 158 ++++++++++++++++++
.../src/docs/en/chapters/appendix/properties.adoc | 2 +
.../docs/en/chapters/architecture/batch-jobs.adoc | 1 +
.../src/main/resources/application.properties | 6 +-
6 files changed, 366 insertions(+), 4 deletions(-)
diff --git a/README.md b/README.md
index 278b1bcd5..de0a0c216 100644
--- a/README.md
+++ b/README.md
@@ -305,8 +305,18 @@ INSTRUCTIONS: How to run Apache RAT (Release Audit Tool)
2. Run `./gradlew rat`. A report will be generated under
build/reports/rat/rat-report.txt
-INSTRUCTIONS: How to enable ActiveMQ
+INSTRUCTIONS: How to enable External Message Broker (ActiveMQ or Apache Kafka)
============
+
+There are two use-cases where external message broker is needed:
+ - External Business Events / Reliable Event Framework
+ - Executing Partitioned Spring Batch Jobs
+
+External Events are business events, e.g.: `ClientCreated`, which might be
important for third party systems. Apache Fineract supports ActiveMQ (or other
JMS compliant brokers) and Apache Kafka endpoints for sending out Business
Events. By default, they are not emitted.
+
+In case of a large deployment with millions of accounts, the Close of Business
Day Spring Batch job may run several hours. In order to speed up this task,
remote partitioning of the job is supported. The Manager node partitions
(breaks up) the COB job into smaller pieces (sub tasks) which then can be
executed on multiple Worker nodes in parallel. The worker nodes are notified
either by ActiveMQ or Kafka regarding their new sub tasks.
+### Active MQ
+
JMS based messaging is disabled by default. In
`docker-compose-postgresql-activemq.yml` an example is shown where ActiveMQ is
enabled. In that configuration one Spring Batch Manager instance and two Spring
Batch Worker instances are created.
Spring based events should be disabled and jms based event handling should be
enabled. Furthermore, proper broker JMS URL should be configured.
@@ -318,6 +328,15 @@ Spring based events should be disabled and jms based event
handling should be en
For additional ActiveMQ related configuration please take a look to the
`application.properties` where the supported configuration parameters are
listed with their default values.
+### Kafka
+
+Kafka support also disabled by default. In
`docker-compose-postgresql-kafka.yml` an example is shown where self-hosted
Kafka is enabled for both External Events and Spring Batch Remote Job execution.
+
+During the development Fineract was tested with PLAINTEXT Kafka brokers
without authentication and with AWS MSK using IAM authentication. The extra
[jar file](https://github.com/aws/aws-msk-iam-auth/releases) required for IAM
authentication is already added to the classpath.
+An example MSK setup can be found in `docker-compose-postgresql-kafka-msk.yml`.
+
+The full list of supported Kafka related properties are documented here:
https://fineract.apache.org/docs/current/
+
Checkstyle and Spotless
============
diff --git a/docker-compose-postgresql-kafka-msk.yml
b/docker-compose-postgresql-kafka-msk.yml
new file mode 100644
index 000000000..6e6cb5639
--- /dev/null
+++ b/docker-compose-postgresql-kafka-msk.yml
@@ -0,0 +1,182 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+
+# You can replace and test a more recent version of docker compose
+version: '3.7'
+services:
+ # Backend service
+ fineractpostgresql:
+ image: postgres:15.3
+ volumes:
+ - ./fineract-db/docker/postgresql:/docker-entrypoint-initdb.d/:Z,ro
+ environment:
+ POSTGRES_USER: root
+ POSTGRES_PASSWORD: skdcnwauicn2ucnaecasdsajdnizucawencascdca
+ FINERACT_DB_USER: postgres
+ FINERACT_DB_PASS: skdcnwauicn2ucnaecasdsajdnizucawencascdca
+ FINERACT_TENANTS_DB_NAME: fineract_tenants
+ FINERACT_TENANT_DEFAULT_DB_NAME: fineract_default
+ healthcheck:
+ test: [ "CMD", "pg_isready", "-q", "-d", "postgres", "-U", "root" ]
+ timeout: 10s
+ retries: 10
+ ports:
+ - "5432:5432"
+ fineract-manager:
+ image: fineract:latest
+ volumes:
+ - ./fineract-provider/build/data:/data
+ healthcheck:
+ test: ["CMD", 'sh', '-c', 'echo -e "Checking for the availability of
Fineract server deployment"; while ! nc -z "fineract-server" 8443; do sleep 1;
printf "-"; done; echo -e " >> Fineract server has started";' ]
+ timeout: 10s
+ retries: 10
+ ports:
+ - 8443:8443
+ depends_on:
+ fineractpostgresql:
+ condition: service_healthy
+ environment:
+ - AWS_ACCESS_KEY_ID=
+ - AWS_SECRET_ACCESS_KEY=
+ - FINERACT_NODE_ID=1
+ - FINERACT_HIKARI_DRIVER_SOURCE_CLASS_NAME=org.postgresql.Driver
+ -
FINERACT_HIKARI_JDBC_URL=jdbc:postgresql://fineractpostgresql:5432/fineract_tenants
+ - FINERACT_HIKARI_USERNAME=postgres
+ - FINERACT_HIKARI_PASSWORD=skdcnwauicn2ucnaecasdsajdnizucawencascdca
+ - FINERACT_HIKARI_MINIMUM_IDLE=3
+ - FINERACT_HIKARI_MAXIMUM_POOL_SIZE=10
+ - FINERACT_HIKARI_IDLE_TIMEOUT=60000
+ - FINERACT_HIKARI_CONNECTION_TIMEOUT=20000
+ - FINERACT_HIKARI_TEST_QUERY=SELECT 1
+ - FINERACT_HIKARI_AUTO_COMMIT=true
+ - FINERACT_SERVER_SSL_ENABLED=true
+ - FINERACT_HIKARI_DS_PROPERTIES_CACHE_PREP_STMTS=true
+ - FINERACT_HIKARI_DS_PROPERTIES_PREP_STMT_CACHE_SIZE=250
+ - FINERACT_HIKARI_DS_PROPERTIES_PREP_STMT_CACHE_SQL_LIMIT=2048
+ - FINERACT_HIKARI_DS_PROPERTIES_USE_SERVER_PREP_STMTS=true
+ - FINERACT_HIKARI_DS_PROPERTIES_USE_LOCAL_SESSION_STATE=true
+ - FINERACT_HIKARI_DS_PROPERTIES_REWRITE_BATCHED_STATEMENTS=true
+ - FINERACT_HIKARI_DS_PROPERTIES_CACHE_RESULT_SET_METADATA=true
+ - FINERACT_HIKARI_DS_PROPERTIES_CACHE_SERVER_CONFIGURATION=true
+ - FINERACT_HIKARI_DS_PROPERTIES_ELIDE_SET_AUTO_COMMITS=true
+ - FINERACT_HIKARI_DS_PROPERTIES_MAINTAIN_TIME_STATS=false
+ - FINERACT_HIKARI_DS_PROPERTIES_LOG_SLOW_QUERIES=true
+ - FINERACT_HIKARI_DS_PROPERTIES_DUMP_QUERIES_IN_EXCEPTION=true
+ - FINERACT_DEFAULT_TENANTDB_HOSTNAME=fineractpostgresql
+ - FINERACT_DEFAULT_TENANTDB_PORT=5432
+ - FINERACT_DEFAULT_TENANTDB_UID=postgres
+ - FINERACT_DEFAULT_TENANTDB_PWD=skdcnwauicn2ucnaecasdsajdnizucawencascdca
+ - FINERACT_DEFAULT_TENANTDB_CONN_PARAMS=
+ - FINERACT_DEFAULT_TENANTDB_TIMEZONE=Asia/Kolkata
+ - FINERACT_DEFAULT_TENANTDB_IDENTIFIER=default
+ - FINERACT_DEFAULT_TENANTDB_NAME=fineract_default
+ - FINERACT_DEFAULT_TENANTDB_DESCRIPTION=Default Demo Tenant
+ - JAVA_TOOL_OPTIONS="-Xmx1G"
+ - FINERACT_REMOTE_JOB_MESSAGE_HANDLER_JMS_ENABLED=false
+ - FINERACT_REMOTE_JOB_MESSAGE_HANDLER_SPRING_EVENTS_ENABLED=false
+ - FINERACT_REMOTE_JOB_MESSAGE_HANDLER_KAFKA_ENABLED=true
+ -
FINERACT_REMOTE_JOB_MESSAGE_HANDLER_KAFKA_BOOTSTRAP_SERVERS=b-3-public.democluster1.am2oyx.c4.kafka.eu-central-1.amazonaws.com:9198,b-2-public.democluster1.am2oyx.c4.kafka.eu-central-1.amazonaws.com:9198,b-1-public.democluster1.am2oyx.c4.kafka.eu-central-1.amazonaws.com:9198
+ -
FINERACT_REMOTE_JOB_MESSAGE_HANDLER_KAFKA_ADMIN_EXTRA_PROPERTIES=security.protocol=SASL_SSL|sasl.mechanism=AWS_MSK_IAM|sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule
required;|sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler
+ -
FINERACT_REMOTE_JOB_MESSAGE_HANDLER_KAFKA_PRODUCER_EXTRA_PROPERTIES=security.protocol=SASL_SSL|sasl.mechanism=AWS_MSK_IAM|sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule
required;|sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler
+ -
FINERACT_REMOTE_JOB_MESSAGE_HANDLER_KAFKA_CONSUMER_EXTRA_PROPERTIES=security.protocol=SASL_SSL|sasl.mechanism=AWS_MSK_IAM|sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule
required;|sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler
+ - FINERACT_REMOTE_JOB_MESSAGE_HANDLER_KAFKA_TOPIC_NAME=job-topic
+ - FINERACT_REMOTE_JOB_MESSAGE_HANDLER_KAFKA_TOPIC_REPLICAS=2
+ - FINERACT_MODE_BATCH_MANAGER_ENABLED=true
+ - FINERACT_MODE_BATCH_WORKER_ENABLED=false
+ - LOAN_COB_CHUNK_SIZE=10
+ - LOAN_COB_PARTITION_SIZE=10
+ - FINERACT_EXTERNAL_EVENTS_ENABLED=true
+ - FINERACT_EXTERNAL_EVENTS_KAFKA_ENABLED=true
+ -
FINERACT_EXTERNAL_EVENTS_KAFKA_BOOTSTRAP_SERVERS=b-3-public.democluster1.am2oyx.c4.kafka.eu-central-1.amazonaws.com:9198,b-2-public.democluster1.am2oyx.c4.kafka.eu-central-1.amazonaws.com:9198,b-1-public.democluster1.am2oyx.c4.kafka.eu-central-1.amazonaws.com:9198
+ -
FINERACT_EXTERNAL_EVENTS_KAFKA_ADMIN_EXTRA_PROPERTIES=security.protocol=SASL_SSL|sasl.mechanism=AWS_MSK_IAM|sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule
required;|sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler
+ -
FINERACT_EXTERNAL_EVENTS_KAFKA_PRODUCER_EXTRA_PROPERTIES:linger.ms=10|batch.size=16384|security.protocol=SASL_SSL|sasl.mechanism=AWS_MSK_IAM|sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule
required;|sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler
+ - FINERACT_EXTERNAL_EVENTS_KAFKA_TOPIC_NAME=external-events
+ - FINERACT_EXTERNAL_EVENTS_KAFKA_TOPIC_REPLICAS=2
+ - SPRING_PROFILES_ACTIVE=test,diagnostics
+ fineract-worker:
+ image: fineract:latest
+ volumes:
+ - ./fineract-provider/build/data:/data
+ deploy:
+ mode: replicated
+ replicas: 2
+ healthcheck:
+ test: ["CMD", 'sh', '-c', 'echo -e "Checking for the availability of
Fineract server deployment"; while ! nc -z "fineract-server" 8443; do sleep 1;
printf "-"; done; echo -e " >> Fineract server has started";' ]
+ timeout: 10s
+ retries: 10
+ ports:
+ - "8444-8445:8443"
+ depends_on:
+ fineractpostgresql:
+ condition: service_healthy
+ environment:
+ - AWS_ACCESS_KEY_ID=
+ - AWS_SECRET_ACCESS_KEY=
+ - FINERACT_NODE_ID=1
+ - FINERACT_HIKARI_DRIVER_SOURCE_CLASS_NAME=org.postgresql.Driver
+ -
FINERACT_HIKARI_JDBC_URL=jdbc:postgresql://fineractpostgresql:5432/fineract_tenants
+ - FINERACT_HIKARI_USERNAME=postgres
+ - FINERACT_HIKARI_PASSWORD=skdcnwauicn2ucnaecasdsajdnizucawencascdca
+ - FINERACT_HIKARI_MINIMUM_IDLE=3
+ - FINERACT_HIKARI_MAXIMUM_POOL_SIZE=10
+ - FINERACT_HIKARI_IDLE_TIMEOUT=60000
+ - FINERACT_HIKARI_CONNECTION_TIMEOUT=20000
+ - FINERACT_HIKARI_TEST_QUERY=SELECT 1
+ - FINERACT_HIKARI_AUTO_COMMIT=true
+ - FINERACT_SERVER_SSL_ENABLED=true
+ - FINERACT_HIKARI_DS_PROPERTIES_CACHE_PREP_STMTS=true
+ - FINERACT_HIKARI_DS_PROPERTIES_PREP_STMT_CACHE_SIZE=250
+ - FINERACT_HIKARI_DS_PROPERTIES_PREP_STMT_CACHE_SQL_LIMIT=2048
+ - FINERACT_HIKARI_DS_PROPERTIES_USE_SERVER_PREP_STMTS=true
+ - FINERACT_HIKARI_DS_PROPERTIES_USE_LOCAL_SESSION_STATE=true
+ - FINERACT_HIKARI_DS_PROPERTIES_REWRITE_BATCHED_STATEMENTS=true
+ - FINERACT_HIKARI_DS_PROPERTIES_CACHE_RESULT_SET_METADATA=true
+ - FINERACT_HIKARI_DS_PROPERTIES_CACHE_SERVER_CONFIGURATION=true
+ - FINERACT_HIKARI_DS_PROPERTIES_ELIDE_SET_AUTO_COMMITS=true
+ - FINERACT_HIKARI_DS_PROPERTIES_MAINTAIN_TIME_STATS=false
+ - FINERACT_HIKARI_DS_PROPERTIES_LOG_SLOW_QUERIES=true
+ - FINERACT_HIKARI_DS_PROPERTIES_DUMP_QUERIES_IN_EXCEPTION=true
+ - FINERACT_DEFAULT_TENANTDB_HOSTNAME=fineractpostgresql
+ - FINERACT_DEFAULT_TENANTDB_PORT=5432
+ - FINERACT_DEFAULT_TENANTDB_UID=postgres
+ - FINERACT_DEFAULT_TENANTDB_PWD=skdcnwauicn2ucnaecasdsajdnizucawencascdca
+ - FINERACT_DEFAULT_TENANTDB_CONN_PARAMS=
+ - FINERACT_DEFAULT_TENANTDB_TIMEZONE=Asia/Kolkata
+ - FINERACT_DEFAULT_TENANTDB_IDENTIFIER=default
+ - FINERACT_DEFAULT_TENANTDB_NAME=fineract_default
+ - FINERACT_DEFAULT_TENANTDB_DESCRIPTION=Default Demo Tenant
+ - JAVA_TOOL_OPTIONS="-Xmx1G"
+ - FINERACT_MODE_BATCH_MANAGER_ENABLED=false
+ - FINERACT_MODE_BATCH_WORKER_ENABLED=true
+ - FINERACT_REMOTE_JOB_MESSAGE_HANDLER_JMS_ENABLED=false
+ - FINERACT_REMOTE_JOB_MESSAGE_HANDLER_SPRING_EVENTS_ENABLED=false
+ - FINERACT_REMOTE_JOB_MESSAGE_HANDLER_KAFKA_ENABLED=true
+ -
FINERACT_REMOTE_JOB_MESSAGE_HANDLER_KAFKA_BOOTSTRAP_SERVERS=b-3-public.democluster1.am2oyx.c4.kafka.eu-central-1.amazonaws.com:9198,b-2-public.democluster1.am2oyx.c4.kafka.eu-central-1.amazonaws.com:9198,b-1-public.democluster1.am2oyx.c4.kafka.eu-central-1.amazonaws.com:9198
+ -
FINERACT_REMOTE_JOB_MESSAGE_HANDLER_KAFKA_ADMIN_EXTRA_PROPERTIES=security.protocol=SASL_SSL|sasl.mechanism=AWS_MSK_IAM|sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule
required;|sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler
+ -
FINERACT_REMOTE_JOB_MESSAGE_HANDLER_KAFKA_PRODUCER_EXTRA_PROPERTIES=security.protocol=SASL_SSL|sasl.mechanism=AWS_MSK_IAM|sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule
required;|sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler
+ -
FINERACT_REMOTE_JOB_MESSAGE_HANDLER_KAFKA_CONSUMER_EXTRA_PROPERTIES=security.protocol=SASL_SSL|sasl.mechanism=AWS_MSK_IAM|sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule
required;|sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler
+ - FINERACT_REMOTE_JOB_MESSAGE_HANDLER_KAFKA_TOPIC_NAME=job-topic
+ - FINERACT_REMOTE_JOB_MESSAGE_HANDLER_KAFKA_TOPIC_REPLICAS=2
+ - FINERACT_EXTERNAL_EVENTS_ENABLED=true
+ - FINERACT_EXTERNAL_EVENTS_KAFKA_ENABLED=true
+ -
FINERACT_EXTERNAL_EVENTS_KAFKA_BOOTSTRAP_SERVERS=b-3-public.democluster1.am2oyx.c4.kafka.eu-central-1.amazonaws.com:9198,b-2-public.democluster1.am2oyx.c4.kafka.eu-central-1.amazonaws.com:9198,b-1-public.democluster1.am2oyx.c4.kafka.eu-central-1.amazonaws.com:9198
+ -
FINERACT_EXTERNAL_EVENTS_KAFKA_ADMIN_EXTRA_PROPERTIES=security.protocol=SASL_SSL|sasl.mechanism=AWS_MSK_IAM|sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule
required;|sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler
+ -
FINERACT_EXTERNAL_EVENTS_KAFKA_PRODUCER_EXTRA_PROPERTIES:linger.ms=10|batch.size=16384|security.protocol=SASL_SSL|sasl.mechanism=AWS_MSK_IAM|sasl.jaas.config=software.amazon.msk.auth.iam.IAMLoginModule
required;|sasl.client.callback.handler.class=software.amazon.msk.auth.iam.IAMClientCallbackHandler
+ - FINERACT_EXTERNAL_EVENTS_KAFKA_TOPIC_NAME=external-events
+ - FINERACT_EXTERNAL_EVENTS_KAFKA_TOPIC_REPLICAS=2
+ - SPRING_PROFILES_ACTIVE=test,diagnostics
diff --git a/fineract-doc/src/docs/en/chapters/appendix/properties-kafka.adoc
b/fineract-doc/src/docs/en/chapters/appendix/properties-kafka.adoc
new file mode 100644
index 000000000..a618d96cf
--- /dev/null
+++ b/fineract-doc/src/docs/en/chapters/appendix/properties-kafka.adoc
@@ -0,0 +1,158 @@
+= Kafka Properties
+
+.Kafka related properties for Remote Spring Batch Jobs
+|===
+|Name |Env Variable |Default Value |Description
+
+|fineract.remote-job-message-handler.kafka.enabled
+|FINERACT_REMOTE_JOB_MESSAGE_HANDLER_KAFKA_ENABLED
+|false
+|Enables or disables Kafka for remote job execution. If Kafka is enabled then
JMS shall be disabled.
+
+|fineract.remote-job-message-handler.kafka.topic.auto-create
+|FINERACT_REMOTE_JOB_MESSAGE_HANDLER_KAFKA_TOPIC_AUTO_CREATE
+|true
+|Enables topic auto creation. In case the auto creation of the topic is
disabled please make sure that the replica and the partition count is properly
configured.
+
+|fineract.remote-job-message-handler.kafka.topic.name
+|FINERACT_REMOTE_JOB_MESSAGE_HANDLER_KAFKA_TOPIC_NAME
+|job-topic
+|Name of the topic where partitioned tasks are sent to
+
+|fineract.remote-job-message-handler.kafka.topic.replicas
+|FINERACT_REMOTE_JOB_MESSAGE_HANDLER_KAFKA_TOPIC_REPLICAS
+|1
+|Number of the replicas
+
+|fineract.remote-job-message-handler.kafka.topic.partitions
+|FINERACT_REMOTE_JOB_MESSAGE_HANDLER_KAFKA_TOPIC_PARTITIONS
+|10
+|Number of partitions
+
+|fineract.remote-job-message-handler.kafka.bootstrap-servers
+|FINERACT_REMOTE_JOB_MESSAGE_HANDLER_KAFKA_BOOTSTRAP_SERVERS
+|localhost:9092
+|Comma separated list of bootstrap servers
+
+|fineract.remote-job-message-handler.kafka.consumer.group-id
+|FINERACT_REMOTE_JOB_MESSAGE_HANDLER_KAFKA_CONSUMER_GROUPID
+|fineract-consumer-group-id
+|Group ID of the Consumer
+
+|fineract.remote-job-message-handler.kafka.consumer.extra-properties-key-value-separator
+|FINERACT_REMOTE_JOB_MESSAGE_HANDLER_KAFKA_CONSUMER_EXTRA_PROPERTIES_SEPARATOR
+|=
+|Defines key and value separator for consumer,e.g.: key=value
+
+|fineract.remote-job-message-handler.kafka.consumer.extra-properties-separator
+|FINERACT_REMOTE_JOB_MESSAGE_HANDLER_KAFKA_CONSUMER_EXTRA_PROPERTIES_SEPARATOR
+|\|
+|Defines item separator for consumer, e.g.: key1=value1\|key2=value2
+
+|fineract.remote-job-message-handler.kafka.consumer.extra-properties
+|FINERACT_REMOTE_JOB_MESSAGE_HANDLER_KAFKA_CONSUMER_EXTRA_PROPERTIES
+|
+|#holds list of key value pairs using the above defined separators for
consumer: key1=value1\|key2=value2\|...\|keyn=valuen
+
+|fineract.remote-job-message-handler.kafka.producer.extra-properties-key-value-separator
+|FINERACT_REMOTE_JOB_MESSAGE_HANDLER_KAFKA_PRODUCER_EXTRA_PROPERTIES_KEY_VALUE_SEPARATOR
+|=
+|Defines key and value separator for producer,e.g.: key=value
+
+|fineract.remote-job-message-handler.kafka.producer.extra-properties-separator
+|FINERACT_REMOTE_JOB_MESSAGE_HANDLER_KAFKA_PRODUCER_EXTRA_PROPERTIES_SEPARATOR
+|\|
+|Defines item separator for producer, e.g.: key1=value1\|key2=value2
+
+|fineract.remote-job-message-handler.kafka.producer.extra-properties
+|FINERACT_REMOTE_JOB_MESSAGE_HANDLER_KAFKA_PRODUCER_EXTRA_PROPERTIES
+|
+|#holds list of key value pairs using the above defined separators for
producer: key1=value1\|key2=value2\|...\|keyn=valuen
+
+|fineract.remote-job-message-handler.kafka.admin.extra-properties-key-value-separator
+|FINERACT_REMOTE_JOB_MESSAGE_HANDLER_KAFKA_ADMIN_EXTRA_PROPERTIES_KEY_VALUE_SEPARATOR
+|=
+|Defines key and value separator for admin,e.g.: key=value
+
+|fineract.remote-job-message-handler.kafka.admin.extra-properties-separator
+|FINERACT_REMOTE_JOB_MESSAGE_HANDLER_KAFKA_ADMIN_EXTRA_PROPERTIES_SEPARATOR
+|\|
+|Defines item separator for admin, e.g.: key1=value1\|key2=value2
+
+|fineract.remote-job-message-handler.kafka.admin.extra-properties
+|FINERACT_REMOTE_JOB_MESSAGE_HANDLER_KAFKA_ADMIN_EXTRA_PROPERTIES
+|
+|#holds list of key value pairs using the above defined separators for admin:
key1=value1\|key2=value2\|...\|keyn=valuen
+
+|===
+
+.Kafka related Properties for External Events
+|===
+|Name |Env Variable |Default Value |Description
+
+|fineract.events.external.producer.kafka.enabled
+|FINERACT_EXTERNAL_EVENTS_KAFKA_ENABLED
+|false
+|Enables disables Kafka for External Events. If Kafka is enabled then JMS
shall be disabled.
+
+|fineract.events.external.producer.kafka.timeout-in-seconds
+|FINERACT_EXTERNAL_EVENTS_KAFKA_TIMEOUT_IN_SECONDS
+|10
+|Timeout for Kafka confirming the messages written in the topic
+
+|fineract.events.external.producer.kafka.topic.auto-create
+|FINERACT_EXTERNAL_EVENTS_KAFKA_TOPIC_AUTO_CREATE
+|true
+|Enables topic auto creation. In case the auto creation of the topic is
disabled please make sure that the replica and the partition count is properly
configured.
+
+|fineract.events.external.producer.kafka.topic.name
+|FINERACT_EXTERNAL_EVENTS_KAFKA_TOPIC_NAME
+|external-events
+|Name of the topic where external events are sent to
+
+|fineract.events.external.producer.kafka.topic.replicas
+|FINERACT_EXTERNAL_EVENTS_KAFKA_TOPIC_REPLICAS
+|1
+|Number of the replicas
+
+|fineract.events.external.producer.kafka.topic.partitions
+|FINERACT_EXTERNAL_EVENTS_KAFKA_TOPIC_PARTITIONS
+|10
+|Number of partitions
+
+|fineract.events.external.producer.kafka.bootstrap-servers
+|FINERACT_EXTERNAL_EVENTS_KAFKA_BOOTSTRAP_SERVERS
+|localhost:9092
+|Comma separated list of Kafka bootstrap servers
+
+|fineract.events.external.producer.kafka.producer.extra-properties-separator
+|FINERACT_EXTERNAL_EVENTS_KAFKA_PRODUCER_EXTRA_PROPERTIES_SEPARATOR
+|\|
+|Defines item separator for producer,e.g.: key=value
+
+|fineract.events.external.producer.kafka.producer.extra-properties-key-value-separator
+|FINERACT_EXTERNAL_EVENTS_KAFKA_PRODUCER_EXTRA_PROPERTIES_KEY_VALUE_SEPARATOR
+|=
+|Defines key and value separator for producer client
+
+|fineract.events.external.producer.kafka.producer.extra-properties
+|FINERACT_EXTERNAL_EVENTS_KAFKA_PRODUCER_EXTRA_PROPERTIES
+|linger.ms=10\|batch.size=16384
+|Defines the extra properties for external event producer clients.
Optimization for sending out large volume of messages. Increases Batch buffer
size and batching time window.
+
+|fineract.events.external.producer.kafka.admin.extra-properties-separator
+|FINERACT_EXTERNAL_EVENTS_KAFKA_ADMIN_EXTRA_PROPERTIES_SEPARATOR
+|\|
+|Defines item separator for admin client.
+
+|fineract.events.external.producer.kafka.admin.extra-properties-key-value-separator
+|FINERACT_EXTERNAL_EVENTS_KAFKA_ADMIN_EXTRA_PROPERTIES_KEY_VALUE_SEPARATOR
+|=
+|Defines key and value separator for admin client
+
+|fineract.events.external.producer.kafka.admin.extra-properties
+|FINERACT_EXTERNAL_EVENTS_KAFKA_ADMIN_EXTRA_PROPERTIES
+|
+|Defines the extra properties for external event admin clients
+
+|===
\ No newline at end of file
diff --git a/fineract-doc/src/docs/en/chapters/appendix/properties.adoc
b/fineract-doc/src/docs/en/chapters/appendix/properties.adoc
index 928de87cb..870c503ce 100644
--- a/fineract-doc/src/docs/en/chapters/appendix/properties.adoc
+++ b/fineract-doc/src/docs/en/chapters/appendix/properties.adoc
@@ -13,4 +13,6 @@ include::properties-authentication.adoc[leveloffset=+1]
include::properties-tomcat.adoc[leveloffset=+1]
+include::properties-kafka.adoc[leveloffset=+1]
+
// include::properties-cache.adoc[leveloffset=+1]
diff --git a/fineract-doc/src/docs/en/chapters/architecture/batch-jobs.adoc
b/fineract-doc/src/docs/en/chapters/architecture/batch-jobs.adoc
index 14acb26d5..a93d671b2 100644
--- a/fineract-doc/src/docs/en/chapters/architecture/batch-jobs.adoc
+++ b/fineract-doc/src/docs/en/chapters/architecture/batch-jobs.adoc
@@ -97,6 +97,7 @@ Also, please note that the partitioned job execution is
multitenant meaning that
For remote partitioning, the following message channels are supported by
Fineract:
* Any JMS compatible message channels (ActiveMQ, Amazon MQ, etc)
+* Apache Kafka
==== Fault-tolerance scenarios
diff --git a/fineract-provider/src/main/resources/application.properties
b/fineract-provider/src/main/resources/application.properties
index b3d997fb5..333e094d2 100644
--- a/fineract-provider/src/main/resources/application.properties
+++ b/fineract-provider/src/main/resources/application.properties
@@ -82,13 +82,13 @@
fineract.remote-job-message-handler.kafka.topic.partitions=${FINERACT_REMOTE_JOB
fineract.remote-job-message-handler.kafka.bootstrap-servers=${FINERACT_REMOTE_JOB_MESSAGE_HANDLER_KAFKA_BOOTSTRAP_SERVERS:localhost:9092}
fineract.remote-job-message-handler.kafka.consumer.group-id=${FINERACT_REMOTE_JOB_MESSAGE_HANDLER_KAFKA_CONSUMER_GROUPID:fineract-consumer-group-id}
#defines the key and value separator, e.g.: key=value
-fineract.remote-job-message-handler.kafka.consumer.extra-properties-key-value-separator=${FINERACT_REMOTE_JOB_MESSAGE_HANDLER_KAFKA_CONSUMER_EXTRA_PROPERTIES_SEPARATOR:=}
+fineract.remote-job-message-handler.kafka.consumer.extra-properties-key-value-separator=${FINERACT_REMOTE_JOB_MESSAGE_HANDLER_KAFKA_CONSUMER_EXTRA_PROPERTIES_KEY_VALUE_SEPARATOR:=}
#defines the item separator, e.g.: key1=value1|key2=value2
fineract.remote-job-message-handler.kafka.consumer.extra-properties-separator=${FINERACT_REMOTE_JOB_MESSAGE_HANDLER_KAFKA_CONSUMER_EXTRA_PROPERTIES_SEPARATOR:|}
#holds list of key value pairs using the above defined separators:
key1=value1|key2=value2|...|keyn=valuen
fineract.remote-job-message-handler.kafka.consumer.extra-properties=${FINERACT_REMOTE_JOB_MESSAGE_HANDLER_KAFKA_CONSUMER_EXTRA_PROPERTIES:}
fineract.remote-job-message-handler.kafka.producer.extra-properties-separator=${FINERACT_REMOTE_JOB_MESSAGE_HANDLER_KAFKA_PRODUCER_EXTRA_PROPERTIES_SEPARATOR:|}
-fineract.remote-job-message-handler.kafka.producer.extra-properties-key-value-separator=${FINERACT_REMOTE_JOB_MESSAGE_HANDLER_KAFKA_PRODUCER_EXTRA_PROPERTIES_SEPARATOR:=}
+fineract.remote-job-message-handler.kafka.producer.extra-properties-key-value-separator=${FINERACT_REMOTE_JOB_MESSAGE_HANDLER_KAFKA_PRODUCER_EXTRA_PROPERTIES_KEY_VALUE_SEPARATOR:=}
fineract.remote-job-message-handler.kafka.producer.extra-properties=${FINERACT_REMOTE_JOB_MESSAGE_HANDLER_KAFKA_PRODUCER_EXTRA_PROPERTIES:}
fineract.remote-job-message-handler.kafka.admin.extra-properties-separator=${FINERACT_REMOTE_JOB_MESSAGE_HANDLER_KAFKA_ADMIN_EXTRA_PROPERTIES_SEPARATOR:|}
fineract.remote-job-message-handler.kafka.admin.extra-properties-key-value-separator=${FINERACT_REMOTE_JOB_MESSAGE_HANDLER_KAFKA_ADMIN_EXTRA_PROPERTIES_KEY_VALUE_SEPARATOR:=}
@@ -114,7 +114,7 @@
fineract.events.external.producer.kafka.topic.replicas=${FINERACT_EXTERNAL_EVENT
fineract.events.external.producer.kafka.topic.partitions=${FINERACT_EXTERNAL_EVENTS_KAFKA_TOPIC_PARTITIONS:10}
fineract.events.external.producer.kafka.bootstrap-servers=${FINERACT_EXTERNAL_EVENTS_KAFKA_BOOTSTRAP_SERVERS:localhost:9092}
fineract.events.external.producer.kafka.producer.extra-properties-separator=${FINERACT_EXTERNAL_EVENTS_KAFKA_PRODUCER_EXTRA_PROPERTIES_SEPARATOR:|}
-fineract.events.external.producer.kafka.producer.extra-properties-key-value-separator=${FINERACT_EXTERNAL_EVENTS_KAFKA_PRODUCER_EXTRA_PROPERTIES_SEPARATOR:=}
+fineract.events.external.producer.kafka.producer.extra-properties-key-value-separator=${FINERACT_EXTERNAL_EVENTS_KAFKA_PRODUCER_EXTRA_PROPERTIES_KEY_VALUE_SEPARATOR:=}
fineract.events.external.producer.kafka.producer.extra-properties=${FINERACT_EXTERNAL_EVENTS_KAFKA_PRODUCER_EXTRA_PROPERTIES:linger.ms=10|batch.size=16384}
fineract.events.external.producer.kafka.admin.extra-properties-separator=${FINERACT_EXTERNAL_EVENTS_KAFKA_ADMIN_EXTRA_PROPERTIES_SEPARATOR:|}
fineract.events.external.producer.kafka.admin.extra-properties-key-value-separator=${FINERACT_EXTERNAL_EVENTS_KAFKA_ADMIN_EXTRA_PROPERTIES_KEY_VALUE_SEPARATOR:=}
