This is an automated email from the ASF dual-hosted git repository.
harishgokul01 pushed a commit to branch release-bugs
in repository https://gitbox.apache.org/repos/asf/incubator-resilientdb.git
The following commit(s) were added to refs/heads/release-bugs by this push:
new f7a09830 Add server configuration files and update generation script
f7a09830 is described below
commit f7a0983066badea71cef56fe99aa882c97efce22
Author: harish876 <[email protected]>
AuthorDate: Fri Oct 31 06:44:31 2025 +0000
Add server configuration files and update generation script
- Introduced `server.config.json` and `server.config` files to define
server region and replica information.
- Updated `generate_config.sh` to use the new configuration generation tool
and ensure proper key generation.
- Added a new C++ tool `generate_region_config_cc` for generating region
configuration in JSON format.
- Modified `start_kv_service.sh` to include the new configuration
generation step.
---
server.config.json | 1 +
service/tools/config/generate_config.sh | 19 ++-
service/tools/config/interface/service.config | 3 +-
service/tools/config/server.config | 5 +
service/tools/kv/server_tools/generate_config.sh | 2 +-
service/tools/kv/server_tools/start_kv_service.sh | 5 +
tools/BUILD | 9 ++
tools/generate_region_config_cc.cpp | 180 ++++++++++++++++++++++
8 files changed, 217 insertions(+), 7 deletions(-)
diff --git a/server.config.json b/server.config.json
new file mode 100644
index 00000000..c71f61c8
--- /dev/null
+++ b/server.config.json
@@ -0,0 +1 @@
+{"region":[{"replicaInfo":[{"id":"1","ip":"127.0.0.1","port":20001},{"id":"2","ip":"127.0.0.1","port":20002},{"id":"3","ip":"127.0.0.1","port":20003},{"id":"4","ip":"127.0.0.1","port":20004}]}]}
\ No newline at end of file
diff --git a/service/tools/config/generate_config.sh
b/service/tools/config/generate_config.sh
index 8d493c55..92aaa490 100755
--- a/service/tools/config/generate_config.sh
+++ b/service/tools/config/generate_config.sh
@@ -40,7 +40,8 @@ ADMIN_PRIVATE_KEY=${admin_key_path}/admin.key.pri
ADMIN_PUBLIC_KEY=${admin_key_path}/admin.key.pub
CERT_TOOLS_BIN=${base_path}/bazel-bin/tools/certificate_tools
-CONFIG_TOOLS_BIN=${base_path}/bazel-bin/tools/generate_region_config
+CONFIG_TOOLS_BIN=${base_path}/bazel-bin/tools/generate_region_config_cc
+KEYGEN_BIN=${base_path}/bazel-bin/tools/key_generator_tools
USERNAME=ubuntu
BASE_PORT=${base_port}
@@ -51,7 +52,8 @@ echo "" > client.config
echo "" > server.config
bazel build //tools:certificate_tools
-bazel build //tools:generate_region_config
+bazel build //tools:key_generator_tools
+bazel build //tools:generate_region_config_cc
idx=1
tot=0
@@ -65,12 +67,21 @@ echo "node num:"$tot
echo "base port:"${BASE_PORT}
echo "client num:" ${CLIENT_NUM}
+# Ensure cert output directory exists and use it for node key generation
+mkdir -p ${output_cert_path}
+
+# Generate admin keypair (needed by certificate_tools)
+${KEYGEN_BIN} ${output_cert_path}/admin
+
for ip in ${iplist[@]};
do
port=$((${BASE_PORT}+${idx}))
- public_key=${key_path}/node${idx}.key.pub
+ public_key=${output_cert_path}/node${idx}.key.pub
echo "get ip:"${ip}
+ # generate node key pair for this idx
+ ${KEYGEN_BIN} ${output_cert_path}/node${idx}
+
# create public key
# create server config
# create the public key and certificate
@@ -85,7 +96,7 @@ do
idx=$(($idx+1))
done
-python3 ${CONFIG_TOOLS_BIN} ./server.config ./server.config.json
+${CONFIG_TOOLS_BIN} ./server.config ./server.config.json
mv server.config.json ${output_path}/server/server.config
mv client.config ${output_path}/interface/service.config
echo "config done:" ${output_path}/server/server.config
diff --git a/service/tools/config/interface/service.config
b/service/tools/config/interface/service.config
index d357ba73..9cda04c0 100644
--- a/service/tools/config/interface/service.config
+++ b/service/tools/config/interface/service.config
@@ -1,3 +1,2 @@
-5 127.0.0.1 10005
-
+5 127.0.0.1 10005
diff --git a/service/tools/config/server.config
b/service/tools/config/server.config
new file mode 100644
index 00000000..a68d1cec
--- /dev/null
+++ b/service/tools/config/server.config
@@ -0,0 +1,5 @@
+
+1 127.0.0.1 10001
+2 127.0.0.1 10002
+3 127.0.0.1 10003
+4 127.0.0.1 10004
diff --git a/service/tools/kv/server_tools/generate_config.sh
b/service/tools/kv/server_tools/generate_config.sh
index 5f494434..14641d70 100755
--- a/service/tools/kv/server_tools/generate_config.sh
+++ b/service/tools/kv/server_tools/generate_config.sh
@@ -28,7 +28,7 @@ iplist=(
WORKSPACE=$PWD
CERT_PATH=$PWD/service/tools/data/cert/
CONFIG_PATH=$PWD/service/tools/config/
-PORT_BASE=20000
+PORT_BASE=10000
CLIENT_NUM=1
./service/tools/config/generate_config.sh ${WORKSPACE} ${CERT_PATH}
${CERT_PATH} ${CONFIG_PATH} ${CERT_PATH} ${CLIENT_NUM} ${PORT_BASE}
${iplist[@]}
diff --git a/service/tools/kv/server_tools/start_kv_service.sh
b/service/tools/kv/server_tools/start_kv_service.sh
index e0273133..b99d0e46 100755
--- a/service/tools/kv/server_tools/start_kv_service.sh
+++ b/service/tools/kv/server_tools/start_kv_service.sh
@@ -23,6 +23,11 @@ SERVER_CONFIG=service/tools/config/server/server.config
WORK_PATH=$PWD
CERT_PATH=${WORK_PATH}/service/tools/data/cert/
+./service/tools/kv/server_tools/generate_config.sh || {
+ echo "Failed to generate configs/certificates" 1>&2
+ exit 1
+}
+
bazel build //service/kv:kv_service $@
nohup $SERVER_PATH $SERVER_CONFIG $CERT_PATH/node1.key.pri
$CERT_PATH/cert_1.cert > server0.log &
nohup $SERVER_PATH $SERVER_CONFIG $CERT_PATH/node2.key.pri
$CERT_PATH/cert_2.cert > server1.log &
diff --git a/tools/BUILD b/tools/BUILD
index 8d5ca092..13475336 100644
--- a/tools/BUILD
+++ b/tools/BUILD
@@ -77,3 +77,12 @@ py_binary(
"//platform/proto:replica_info_py_proto",
],
)
+
+cc_binary(
+ name = "generate_region_config_cc",
+ srcs = ["generate_region_config_cc.cpp"],
+ deps = [
+ "//platform/config:resdb_config_utils",
+ "//platform/proto:replica_info_cc_proto",
+ ],
+)
diff --git a/tools/generate_region_config_cc.cpp
b/tools/generate_region_config_cc.cpp
new file mode 100644
index 00000000..b2147867
--- /dev/null
+++ b/tools/generate_region_config_cc.cpp
@@ -0,0 +1,180 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#include <cstdlib>
+#include <fstream>
+#include <iostream>
+#include <map>
+#include <sstream>
+#include <string>
+#include <vector>
+
+#include <google/protobuf/util/json_util.h>
+#include "platform/proto/replica_info.pb.h"
+
+namespace {
+
+using google::protobuf::util::JsonStringToMessage;
+using google::protobuf::util::MessageToJsonString;
+using google::protobuf::util::JsonParseOptions;
+using resdb::ResConfigData;
+using resdb::ReplicaInfo;
+using resdb::RegionInfo;
+
+bool readLines(const std::string& path, std::vector<std::string>* lines) {
+ std::ifstream in(path);
+ if (!in.is_open()) {
+ std::cerr << "Failed to open file: " << path << "\n";
+ return false;
+ }
+ std::string line;
+ while (std::getline(in, line)) {
+ lines->push_back(line);
+ }
+ return true;
+}
+
+std::string readFileToString(const std::string& path) {
+ std::ifstream in(path);
+ if (!in.is_open()) {
+ std::cerr << "Failed to open file: " << path << "\n";
+ return "";
+ }
+ std::ostringstream oss;
+ oss << in.rdbuf();
+ return oss.str();
+}
+
+} // namespace
+
+int main(int argc, char** argv) {
+ if (argc < 3) {
+ std::cerr << "Usage: " << argv[0]
+ << " <server.config input> <output.json> [template.json]\n";
+ return 1;
+ }
+
+ const std::string input_path = argv[1];
+ const std::string output_path = argv[2];
+ const std::string template_path = (argc > 3 ? argv[3] : "");
+
+ std::vector<std::string> lines;
+ if (!readLines(input_path, &lines)) {
+ std::cerr << "Failed to read input file: " << input_path << "\n";
+ return 2;
+ }
+
+ ResConfigData config_data;
+ std::map<std::string, std::vector<ReplicaInfo>> region_to_replicas;
+
+ for (const std::string& line : lines) {
+ if (line.empty() || line.find_first_not_of(" \t\r\n") ==
std::string::npos) {
+ continue;
+ }
+
+ std::istringstream iss(line);
+ std::string id_str, ip, port_str, region_id;
+
+ if (!(iss >> id_str >> ip >> port_str)) {
+ continue;
+ }
+
+ if (!(iss >> region_id)) {
+ region_id = "0";
+ }
+
+ int64_t id;
+ int32_t port;
+ try {
+ id = std::stoll(id_str);
+ port = std::stoi(port_str);
+ } catch (const std::exception& e) {
+ std::cerr << "Warning: Failed to parse id or port from line: " << line
<< "\n";
+ continue;
+ }
+
+ ReplicaInfo replica;
+ replica.set_id(id);
+ replica.set_ip(ip);
+ replica.set_port(port);
+
+ region_to_replicas[region_id].push_back(replica);
+ }
+
+ for (auto& [rid, replicas] : region_to_replicas) {
+ RegionInfo* region = config_data.add_region();
+ for (auto& replica : replicas) {
+ *region->add_replica_info() = replica;
+ }
+ }
+
+ std::string output_json;
+ MessageToJsonString(config_data, &output_json);
+ std::cerr << "Generated JSON (before template):\n" << output_json << "\n\n";
+
+ std::ofstream out(output_path);
+ if (!out.is_open()) {
+ std::cerr << "Failed to open output file: " << output_path << "\n";
+ return 4;
+ }
+ out << output_json;
+ out.close();
+
+ if (!template_path.empty()) {
+ std::string base_json = readFileToString(output_path);
+ std::cerr << "Read base config:\n" << base_json << "\n\n";
+
+ ResConfigData base_config;
+ JsonParseOptions parse_options;
+ auto status = JsonStringToMessage(base_json, &base_config, parse_options);
+ if (!status.ok()) {
+ std::cerr << "Failed to parse generated JSON: " << status.message() <<
"\n";
+ return 5;
+ }
+
+ std::string template_json_str = readFileToString(template_path);
+ if (template_json_str.empty()) {
+ std::cerr << "Failed to read template file: " << template_path << "\n";
+ return 6;
+ }
+ std::cerr << "Read template config:\n" << template_json_str << "\n\n";
+
+ ResConfigData template_config;
+ status = JsonStringToMessage(template_json_str, &template_config,
parse_options);
+ if (!status.ok()) {
+ std::cerr << "Failed to parse template JSON: " << status.message() <<
"\n";
+ return 7;
+ }
+
+ base_config.MergeFrom(template_config);
+
+ std::string merged_json;
+ MessageToJsonString(base_config, &merged_json);
+ std::cerr << "Merged JSON (after template):\n" << merged_json << "\n\n";
+ std::ofstream out2(output_path);
+ if (!out2.is_open()) {
+ std::cerr << "Failed to reopen output file for merged config\n";
+ return 8;
+ }
+ out2 << merged_json;
+ out2.close();
+ }
+
+ return 0;
+}
