This is an automated email from the ASF dual-hosted git repository. wankai pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/skywalking.git
The following commit(s) were added to refs/heads/master by this push: new dcfce00415 BanyanDB: support group `replicas` and `user/password` for basic authentication. (#13420) dcfce00415 is described below commit dcfce00415b23121806581d1a6c2fd4a05d2e823 Author: Wan Kai <wankai...@foxmail.com> AuthorDate: Fri Aug 15 13:24:13 2025 +0800 BanyanDB: support group `replicas` and `user/password` for basic authentication. (#13420) --- docs/en/changes/changes.md | 1 + docs/en/setup/backend/configuration-vocabulary.md | 28 +++++++++++++++- docs/en/setup/backend/storages/banyandb.md | 39 +++++++++++++++++++--- oap-server-bom/pom.xml | 2 +- .../server-starter/src/main/resources/bydb.yml | 37 +++++++++++++++++--- .../plugin/banyandb/BanyanDBIndexInstaller.java | 7 +++- .../plugin/banyandb/BanyanDBStorageClient.java | 12 +++++++ .../plugin/banyandb/BanyanDBStorageConfig.java | 4 +++ .../e2e-v2/cases/storage/banyandb/auth-config.yaml | 20 +++++++++++ .../cases/storage/banyandb/docker-compose.yml | 8 +++++ test/e2e-v2/script/env | 2 +- 11 files changed, 147 insertions(+), 13 deletions(-) diff --git a/docs/en/changes/changes.md b/docs/en/changes/changes.md index be6be528c4..b09e1bd5f9 100644 --- a/docs/en/changes/changes.md +++ b/docs/en/changes/changes.md @@ -61,6 +61,7 @@ | MetricsPersistentMinMALWorker | availableProcessors * 2 / 16, at least 1 | true | 1 | 1000 | * Bump up netty to 4.2.4.Final. +* BanyanDB: support group `replicas` and `user/password` for basic authentication. #### UI diff --git a/docs/en/setup/backend/configuration-vocabulary.md b/docs/en/setup/backend/configuration-vocabulary.md index 6cea6ae44d..ccf05ddf52 100644 --- a/docs/en/setup/backend/configuration-vocabulary.md +++ b/docs/en/setup/backend/configuration-vocabulary.md @@ -370,6 +370,8 @@ The global settings for the whole BanyanDB: | segmentQueryMaxSize | The maximum size of trace segments per query. | SW_STORAGE_BANYAND_QUERY_SEGMENT_SIZE | 200 | | asyncProfilerTaskQueryMaxSize | Max size of AsyncProfilerTask to be fetched. | SW_STORAGE_BANYANDB_ASYNC_PROFILER_TASK_QUERY_MAX_SIZE | 200 | | profileDataQueryScrollBatchSize | The batch size of query profiling data. | SW_STORAGE_BANYAND_QUERY_PROFILE_DATA_BATCH_SIZE | 100 | +| user | The username for BanyanDB authentication. | SW_STORAGE_BANYANDB_USER | - | +| password | The password for BanyanDB authentication. | SW_STORAGE_BANYANDB_PASSWORD | - | | sslTrustCAPath | If the BanyanDB server is configured with TLS, config the TLS cert file path and open tls connection. | SW_STORAGE_BANYANDB_SSL_TRUST_CA_PATH | - | | cleanupUnusedTopNRules | Cleanup TopN rules in BanyanDB server that are not configured in the bydb-topn.yml config. | SW_STORAGE_BANYANDB_CLEANUP_UNUSED_TOPN_RULES | true | @@ -387,97 +389,115 @@ OAP will query the data from the "hot and warm" stage by default if the "warm" s | - | shardNum | - | Shards Number for normal records group. | SW_STORAGE_BANYANDB_RECORDS_SHARD_NUM | 1 | | - | segmentInterval | - | Segment Interval Days for normal records group. | SW_STORAGE_BANYANDB_RECORDS_SI_DAYS | 1 | | - | ttl | - | TTL Days for normal records group. | SW_STORAGE_BANYANDB_RECORDS_TTL_DAYS | 3 | +| - | replicas | - | Replicas for normal records group. | SW_STORAGE_BANYANDB_RECORDS_REPLICAS | 0 | | - | enableWarmStage | - | Activate warm stage for normal records group. | SW_STORAGE_BANYANDB_RECORDS_ENABLE_WARM_STAGE | false | | - | enableColdStage | - | Activate cold stage for normal records group. | SW_STORAGE_BANYANDB_RECORDS_ENABLE_COLD_STAGE | false | | - | warm | - | The warm stage settings. | - | - | | - | - | shardNum | Shards Number for this stage. | SW_STORAGE_BANYANDB_RECORDS_WARM_SHARD_NUM | 1 | | - | - | segmentInterval | Segment Interval Days for this stage. | SW_STORAGE_BANYANDB_RECORDS_WARM_SI_DAYS | 2 | | - | - | ttl | TTL Days for this stage. | SW_STORAGE_BANYANDB_RECORDS_WARM_TTL_DAYS | 7 | +| - | - | replicas | Replicas for this stage. | SW_STORAGE_BANYANDB_RECORDS_WARM_REPLICAS | 0 | | - | - | nodeSelector | Specifying target nodes for this stage. | SW_STORAGE_BANYANDB_RECORDS_WARM_NODE_SELECTOR | type=warm | | - | cold | - | The cold stage settings. | - | - | | - | - | shardNum | Shards Number for for this stage. | SW_STORAGE_BANYANDB_RECORDS_COLD_SHARD_NUM | 1 | | - | - | segmentInterval | Segment Interval Days for this stage. | SW_STORAGE_BANYANDB_RECORDS_COLD_SI_DAYS | 3 | | - | - | ttl | TTL Days for this stage. | SW_STORAGE_BANYANDB_RECORDS_COLD_TTL_DAYS | 30 | +| - | - | replicas | Replicas for this stage. | SW_STORAGE_BANYANDB_RECORDS_COLD_REPLICAS | 0 | | - | - | nodeSelector | Specifying target nodes for this stage. | SW_STORAGE_BANYANDB_RECORDS_COLD_NODE_SELECTOR | type=cold | | recordsTrace | - | - | | - | - | | - | shardNum | - | Shards Number for SkyWalking trace records group. | SW_STORAGE_BANYANDB_TRACE_SHARD_NUM | 2 | | - | segmentInterval | - | Segment Interval Days. | SW_STORAGE_BANYANDB_TRACE_SI_DAYS | 1 | | - | ttl | - | TTL Days. | SW_STORAGE_BANYANDB_TRACE_TTL_DAYS | 3 | +| - | replicas | - | Replicas for SkyWalking trace records group. | SW_STORAGE_BANYANDB_TRACE_REPLICAS | 0 | | - | enableWarmStage | - | Activate warm stage. | SW_STORAGE_BANYANDB_TRACE_ENABLE_WARM_STAGE | false | | - | enableColdStage | - | Activate cold stage. | SW_STORAGE_BANYANDB_TRACE_ENABLE_COLD_STAGE | false | | - | warm | - | The warm stage settings. | - | - | | - | - | shardNum | Shards Number for this stage. | SW_STORAGE_BANYANDB_TRACE_WARM_SHARD_NUM | 2 | | - | - | segmentInterval | Segment Interval Days for this stage. | SW_STORAGE_BANYANDB_TRACE_WARM_SI_DAYS | 1 | | - | - | ttl | TTL Days for this stage. | SW_STORAGE_BANYANDB_TRACE_WARM_TTL_DAYS | 7 | +| - | - | replicas | Replicas for this stage. | SW_STORAGE_BANYANDB_TRACE_WARM_REPLICAS | 0 | | - | - | nodeSelector | Specifying target nodes for this stage. | SW_STORAGE_BANYANDB_TRACE_WARM_NODE_SELECTOR | type=warm | | - | cold | - | The cold stage settings. | - | - | | - | - | shardNum | Shards Number for for this stage. | SW_STORAGE_BANYANDB_TRACE_COLD_SHARD_NUM | 2 | | - | - | segmentInterval | Segment Interval Days for this stage. | SW_STORAGE_BANYANDB_TRACE_COLD_SI_DAYS | 1 | | - | - | ttl | TTL Days for this stage. | SW_STORAGE_BANYANDB_TRACE_COLD_TTL_DAYS | 30 | +| - | - | replicas | Replicas for this stage. | SW_STORAGE_BANYANDB_TRACE_COLD_REPLICAS | 0 | | - | - | nodeSelector | Specifying target nodes for this stage. | SW_STORAGE_BANYANDB_TRACE_COLD_NODE_SELECTOR | type=cold | | recordsZipkinTrace | - | - | | - | - | | - | shardNum | - | Shards Number for Zipkin trace records group. | SW_STORAGE_BANYANDB_ZIPKIN_TRACE_SHARD_NUM | 2 | | - | segmentInterval | - | Segment Interval Days. | SW_STORAGE_BANYANDB_ZIPKIN_TRACE_SI_DAYS | 1 | | - | ttl | - | TTL Days. | SW_STORAGE_BANYANDB_ZIPKIN_TRACE_TTL_DAYS | 3 | +| - | replicas | - | Replicas for Zipkin trace records group. | SW_STORAGE_BANYANDB_ZIPKIN_TRACE_REPLICAS | 0 | | - | enableWarmStage | - | Activate warm stage. | SW_STORAGE_BANYANDB_ZIPKIN_TRACE_ENABLE_WARM_STAGE | false | | - | enableColdStage | - | Activate cold stage. | SW_STORAGE_BANYANDB_ZIPKIN_TRACE_ENABLE_COLD_STAGE | false | | - | warm | - | The warm stage settings. | - | - | | - | - | shardNum | Shards Number for this stage. | SW_STORAGE_BANYANDB_ZIPKIN_TRACE_WARM_SHARD_NUM | 2 | | - | - | segmentInterval | Segment Interval Days for this stage. | SW_STORAGE_BANYANDB_ZIPKIN_TRACE_WARM_SI_DAYS | 1 | | - | - | ttl | TTL Days for this stage. | SW_STORAGE_BANYANDB_ZIPKIN_TRACE_WARM_TTL_DAYS | 7 | +| - | - | replicas | Replicas for this stage. | SW_STORAGE_BANYANDB_ZIPKIN_TRACE_WARM_REPLICAS | 0 | | - | - | nodeSelector | Specifying target nodes for this stage. | SW_STORAGE_BANYANDB_ZIPKIN_TRACE_WARM_NODE_SELECTOR | type=warm | | - | cold | - | The cold stage settings. | - | - | | - | - | shardNum | Shards Number for for this stage. | SW_STORAGE_BANYANDB_ZIPKIN_TRACE_COLD_SHARD_NUM | 2 | | - | - | segmentInterval | Segment Interval Days for this stage. | SW_STORAGE_BANYANDB_ZIPKIN_TRACE_COLD_SI_DAYS | 1 | | - | - | ttl | TTL Days for this stage. | SW_STORAGE_BANYANDB_ZIPKIN_TRACE_COLD_TTL_DAYS | 30 | +| - | - | replicas | Replicas for this stage. | SW_STORAGE_BANYANDB_ZIPKIN_TRACE_COLD_REPLICAS | 0 | | - | - | nodeSelector | Specifying target nodes for this stage. | SW_STORAGE_BANYANDB_ZIPKIN_TRACE_COLD_NODE_SELECTOR | type=cold | | recordsLog | - | - | | - | - | | - | shardNum | - | Shards Number. | SW_STORAGE_BANYANDB_LOG_SHARD_NUM | 2 | | - | segmentInterval | - | Segment Interval Days. | SW_STORAGE_BANYANDB_LOG_SI_DAYS | 1 | | - | ttl | - | TTL Days. | SW_STORAGE_BANYANDB_LOG_TTL_DAYS | 3 | +| - | replicas | - | Replicas for Log records group. | SW_STORAGE_BANYANDB_LOG_REPLICAS | 0 | | - | enableWarmStage | - | Activate warm stage. | SW_STORAGE_BANYANDB_LOG_ENABLE_WARM_STAGE | false | | - | enableColdStage | - | Activate cold stage. | SW_STORAGE_BANYANDB_LOG_ENABLE_COLD_STAGE | false | | - | warm | - | The warm stage settings. | - | - | | - | - | shardNum | Shards Number for this stage. | SW_STORAGE_BANYANDB_LOG_WARM_SHARD_NUM | 2 | | - | - | segmentInterval | Segment Interval Days for this stage. | SW_STORAGE_BANYANDB_LOG_WARM_SI_DAYS | 1 | | - | - | ttl | TTL Days for this stage. | SW_STORAGE_BANYANDB_LOG_WARM_TTL_DAYS | 7 | +| - | - | replicas | Replicas for this stage. | SW_STORAGE_BANYANDB_LOG_WARM_REPLICAS | 0 | | - | - | nodeSelector | Specifying target nodes for this stage. | SW_STORAGE_BANYANDB_LOG_WARM_NODE_SELECTOR | type=warm | | - | cold | - | The cold stage settings. | - | - | | - | - | shardNum | Shards Number for for this stage. | SW_STORAGE_BANYANDB_LOG_COLD_SHARD_NUM | 2 | | - | - | segmentInterval | Segment Interval Days for this stage. | SW_STORAGE_BANYANDB_LOG_COLD_SI_DAYS | 1 | | - | - | ttl | TTL Days for this stage. | SW_STORAGE_BANYANDB_LOG_COLD_TTL_DAYS | 30 | +| - | - | replicas | Replicas for this stage. | SW_STORAGE_BANYANDB_LOG_COLD_REPLICAS | 0 | | - | - | nodeSelector | Specifying target nodes for this stage. | SW_STORAGE_BANYANDB_LOG_COLD_NODE_SELECTOR | type=cold | | recordsBrowserErrorLog | - | - | | - | - | | - | shardNum | - | Shards Number. | SW_STORAGE_BANYANDB_BROWSER_ERROR_LOG_SHARD_NUM | 2 | | - | segmentInterval | - | Segment Interval Days. | SW_STORAGE_BANYANDB_BROWSER_ERROR_LOG_SI_DAYS | 1 | | - | ttl | - | TTL Days. | SW_STORAGE_BANYANDB_BROWSER_ERROR_LOG_TTL_DAYS | 3 | +| - | replicas | - | Replicas for Browser Error Log records group. | SW_STORAGE_BANYANDB_BROWSER_ERROR_LOG_REPLICAS | 0 | | - | enableWarmStage | - | Activate warm stage. | SW_STORAGE_BANYANDB_BROWSER_ERROR_LOG_ENABLE_WARM_STAGE | false | | - | enableColdStage | - | Activate cold stage. | SW_STORAGE_BANYANDB_BROWSER_ERROR_LOG_ENABLE_COLD_STAGE | false | | - | warm | - | The warm stage settings. | - | - | | - | - | shardNum | Shards Number for this stage. | SW_STORAGE_BANYANDB_BROWSER_ERROR_LOG_WARM_SHARD_NUM | 2 | | - | - | segmentInterval | Segment Interval Days for this stage. | SW_STORAGE_BANYANDB_BROWSER_ERROR_LOG_WARM_SI_DAYS | 1 | | - | - | ttl | TTL Days for this stage. | SW_STORAGE_BANYANDB_BROWSER_ERROR_LOG_WARM_TTL_DAYS | 7 | +| - | - | replicas | Replicas for this stage. | SW_STORAGE_BANYANDB_BROWSER_ERROR_LOG_WARM_REPLICAS | 0 | | - | - | nodeSelector | Specifying target nodes for this stage. | SW_STORAGE_BANYANDB_BROWSER_ERROR_LOG_WARM_NODE_SELECTOR | type=warm | | - | cold | - | The cold stage settings. | - | - | | - | - | shardNum | Shards Number for for this stage. | SW_STORAGE_BANYANDB_BROWSER_ERROR_LOG_COLD_SHARD_NUM | 2 | | - | - | segmentInterval | Segment Interval Days for this stage. | SW_STORAGE_BANYANDB_BROWSER_ERROR_LOG_COLD_SI_DAYS | 1 | | - | - | ttl | TTL Days for this stage. | SW_STORAGE_BANYANDB_BROWSER_ERROR_LOG_COLD_TTL_DAYS | 30 | +| - | - | replicas | Replicas for this stage. | SW_STORAGE_BANYANDB_BROWSER_ERROR_LOG_COLD_REPLICAS | 0 | | - | - | nodeSelector | Specifying target nodes for this stage. | SW_STORAGE_BANYANDB_BROWSER_ERROR_LOG_COLD_NODE_SELECTOR | type=cold | | metricsMinute | - | - | The group for minute granularity metrics group. | - | - | | - | shardNum | - | Shards Number for minute granularity metrics group. | SW_STORAGE_BANYANDB_METRICS_MINUTE_SHARD_NUM | 2 | | - | segmentInterval | - | Segment Interval Days for minute granularity metrics group. | SW_STORAGE_BANYANDB_METRICS_MINUTE_SI_DAYS | 1 | | - | ttl | - | TTL Days for minute granularity metrics group. | SW_STORAGE_BANYANDB_METRICS_MINUTE_TTL_DAYS | 7 | +| - | replicas | - | Replicas for minute granularity metrics group. | SW_STORAGE_BANYANDB_METRICS_MINUTE_REPLICAS | 0 | | - | enableWarmStage | - | Activate warm stage for minute granularity metrics group. | SW_STORAGE_BANYANDB_METRICS_MINUTE_ENABLE_WARM_STAGE | false | | - | enableColdStage | - | Activate cold stage for minute granularity metrics group. | SW_STORAGE_BANYANDB_METRICS_MINUTE_ENABLE_COLD_STAGE | false | | - | warm | - | The warm stage settings. | - | - | | - | - | shardNum | Shards Number for this stage. | SW_STORAGE_BANYANDB_METRICS_MINUTE_WARM_SHARD_NUM | 2 | | - | - | segmentInterval | Segment Interval Days for this stage. | SW_STORAGE_BANYANDB_METRICS_MINUTE_WARM_SI_DAYS | 3 | | - | - | ttl | TTL Days for this stage. | SW_STORAGE_BANYANDB_METRICS_MINUTE_WARM_TTL_DAYS | 15 | +| - | - | replicas | Replicas for this stage. | SW_STORAGE_BANYANDB_METRICS_MINUTE_WARM_REPLICAS | 0 | | - | - | nodeSelector | Specifying target nodes for this stage. | SW_STORAGE_BANYANDB_METRICS_MINUTE_WARM_NODE_SELECTOR | type=warm | | - | cold | - | The cold stage settings. | - | - | | - | - | shardNum | Shards Number for for this stage. | SW_STORAGE_BANYANDB_METRICS_MINUTE_COLD_SHARD_NUM | 2 | | - | - | segmentInterval | Segment Interval Days for this stage. | SW_STORAGE_BANYANDB_METRICS_MINUTE_COLD_SI_DAYS | 5 | | - | - | ttl | TTL Days for this stage. | SW_STORAGE_BANYANDB_METRICS_MINUTE_COLD_TTL_DAYS | 60 | +| - | - | replicas | Replicas for this stage. | SW_STORAGE_BANYANDB_METRICS_MINUTE_COLD_REPLICAS | 0 | | - | - | nodeSelector | Specifying target nodes for this stage. | SW_STORAGE_BANYANDB_METRICS_MINUTE_COLD_NODE_SELECTOR | type=cold | | metricsHour | - | - | The group for hour granularity metrics. | - | - | | - | shardNum | - | Shards Number for hour granularity metrics group. | SW_STORAGE_BANYANDB_METRICS_HOUR_SHARD_NUM | 1 | @@ -489,35 +509,41 @@ OAP will query the data from the "hot and warm" stage by default if the "warm" s | - | - | shardNum | Shards Number for this stage. | SW_STORAGE_BANYANDB_METRICS_HOUR_WARM_SHARD_NUM | 1 | | - | - | segmentInterval | Segment Interval Days for this stage. | SW_STORAGE_BANYANDB_METRICS_HOUR_WARM_SI_DAYS | 7 | | - | - | ttl | TTL Days for this stage. | SW_STORAGE_BANYANDB_METRICS_HOUR_WARM_TTL_DAYS | 30 | +| - | - | replicas | Replicas for this stage. | SW_STORAGE_BANYANDB_METRICS_HOUR_WARM_REPLICAS | 0 | | - | - | nodeSelector | Specifying target nodes for this stage. | SW_STORAGE_BANYANDB_METRICS_HOUR_WARM_NODE_SELECTOR | type=warm | | - | cold | - | The cold stage settings. | - | - | | - | - | shardNum | Shards Number for for this stage. | SW_STORAGE_BANYANDB_METRICS_HOUR_COLD_SHARD_NUM | 1 | | - | - | segmentInterval | Segment Interval Days for this stage. | SW_STORAGE_BANYANDB_METRICS_HOUR_COLD_SI_DAYS | 15 | | - | - | ttl | TTL Days for this stage. | SW_STORAGE_BANYANDB_METRICS_HOUR_COLD_TTL_DAYS | 120 | +| - | - | replicas | Replicas for this stage. | SW_STORAGE_BANYANDB_METRICS_HOUR_COLD_REPLICAS | 0 | | - | - | nodeSelector | Specifying target nodes for this stage. | SW_STORAGE_BANYANDB_METRICS_HOUR_COLD_NODE_SELECTOR | type=cold | | metricsDay | - | - | The group for day granularity metrics. | - | - | | - | shardNum | - | Shards Number for day granularity metrics group. | SW_STORAGE_BANYANDB_METRICS_DAY_SHARD_NUM | 1 | | - | segmentInterval | - | Segment Interval Days for day granularity metrics group. | SW_STORAGE_BANYANDB_METRICS_DAY_SI_DAYS | 15 | | - | ttl | - | TTL Days for day granularity metrics group. | SW_STORAGE_BANYANDB_METRICS_DAY_TTL_DAYS | 15 | +| - | replicas | - | Replicas for day granularity metrics group. | SW_STORAGE_BANYANDB_METRICS_DAY_REPLICAS | 0 | | - | enableWarmStage | - | Activate warm stage for day granularity metrics group. | SW_STORAGE_BANYANDB_METRICS_DAY_ENABLE_WARM_STAGE | false | | - | enableColdStage | - | Activate cold stage for day granularity metrics group. | SW_STORAGE_BANYANDB_METRICS_DAY_ENABLE_COLD_STAGE | false | | - | warm | - | The warm stage settings. | - | - | | - | - | shardNum | Shards Number for this stage. | SW_STORAGE_BANYANDB_METRICS_DAY_WARM_SHARD_NUM | 1 | | - | - | segmentInterval | Segment Interval Days for this stage. | SW_STORAGE_BANYANDB_METRICS_DAY_WARM_SI_DAYS | 15 | | - | - | ttl | TTL Days for this stage. | SW_STORAGE_BANYANDB_METRICS_DAY_WARM_TTL_DAYS | 30 | +| - | - | replicas | Replicas for this stage. | SW_STORAGE_BANYANDB_METRICS_DAY_WARM_REPLICAS | 0 | | - | - | nodeSelector | Specifying target nodes for this stage. | SW_STORAGE_BANYANDB_METRICS_DAY_WARM_NODE_SELECTOR | type=warm | | - | cold | - | The cold stage settings. | - | - | | - | - | shardNum | Shards Number for for this stage. | SW_STORAGE_BANYANDB_METRICS_DAY_COLD_SHARD_NUM | 1 | | - | - | segmentInterval | Segment Interval Days for this stage. | SW_STORAGE_BANYANDB_METRICS_DAY_COLD_SI_DAYS | 15 | | - | - | ttl | TTL Days for this stage. | SW_STORAGE_BANYANDB_METRICS_DAY_COLD_TTL_DAYS | 120 | +| - | - | replicas | Replicas for this stage. | SW_STORAGE_BANYANDB_METRICS_DAY_COLD_REPLICAS | 0 | | - | - | nodeSelector | Specifying target nodes for this stage. | SW_STORAGE_BANYANDB_METRICS_DAY_COLD_NODE_SELECTOR | type=cold | | metadata | - | - | The `metadata` group is designed to store metrics that are used for indexing without value columns. Such as `service_traffic`, `network_address_alias`, etc. Since BanyanDB *0.8.0*. | - | - | | - | shardNum | - | Shards Number for metadata `index` group. | SW_STORAGE_BANYANDB_METADATA_SHARD_NUM | 2 | | - | segmentInterval | - | Segment Interval Days for metadata `index` group. | SW_STORAGE_BANYANDB_METADATA_SI_DAYS | 15 | | - | ttl | - | TTL Days for metadata `index` group. | SW_STORAGE_BANYANDB_METADATA_TTL_DAYS | 15 | +| - | replicas | - | Replicas for metadata `index` group. | SW_STORAGE_BANYANDB_METADATA_REPLICAS | 0 | | property | - | - | The group settings of property, such as UI and profiling. | - | - | | - | shardNum | - | Shards Number for property group. | SW_STORAGE_BANYANDB_PROPERTY_SHARD_NUM | 1 | - +| - | replicas | - | Replicas for property group. |SW_STORAGE_BANYANDB_PROPERTY_REPLICAS | 0 | ## Note ยน System Environment Variable name could be declared and changed in `application.yml/bydb.yaml`. The names listed here are simply diff --git a/docs/en/setup/backend/storages/banyandb.md b/docs/en/setup/backend/storages/banyandb.md index 45b3968b1f..9cb3053ab8 100644 --- a/docs/en/setup/backend/storages/banyandb.md +++ b/docs/en/setup/backend/storages/banyandb.md @@ -62,19 +62,22 @@ global: # The batch size for querying profile data. profileDataQueryBatchSize: ${SW_STORAGE_BANYANDB_QUERY_PROFILE_DATA_BATCH_SIZE:100} asyncProfilerTaskQueryMaxSize: ${SW_STORAGE_BANYANDB_ASYNC_PROFILER_TASK_QUERY_MAX_SIZE:200} + user: ${SW_STORAGE_BANYANDB_USER:""} + password: ${SW_STORAGE_BANYANDB_PASSWORD:""} # If the BanyanDB server is configured with TLS, configure the TLS cert file path and enable TLS connection. sslTrustCAPath: ${SW_STORAGE_BANYANDB_SSL_TRUST_CA_PATH:""} # Cleanup TopN rules in BanyanDB server that are not configured in the bydb-topn.yml config. cleanupUnusedTopNRules: ${SW_STORAGE_BANYANDB_CLEANUP_UNUSED_TOPN_RULES:true} - + groups: # The group settings of record. - # - "ShardNum": Number of shards in the group. Shards are the basic units of data storage in BanyanDB. Data is distributed across shards based on the hash value of the series ID. + # - "shardNum": Number of shards in the group. Shards are the basic units of data storage in BanyanDB. Data is distributed across shards based on the hash value of the series ID. # Refer to the [BanyanDB Shard](https://skywalking.apache.org/docs/skywalking-banyandb/latest/concept/clustering/#52-data-sharding) documentation for more details. - # - "SIDays": Interval in days for creating a new segment. Segments are time-based, allowing efficient data retention and querying. `SI` stands for Segment Interval. - # - "TTLDays": Time-to-live for the data in the group, in days. Data exceeding the TTL will be deleted. + # - "segmentInterval": Interval in days for creating a new segment. Segments are time-based, allowing efficient data retention and querying. `SI` stands for Segment Interval. + # - "ttl": Time-to-live for the data in the group, in days. Data exceeding the TTL will be deleted. + # - "replicas": Number of replicas for the group/stage. Replicas are used for data redundancy and high availability, a value of 0 means no replicas, while a value of 1 means one primary shard and one replica, higher values indicate more replicas. # - # For more details on setting `segmentIntervalDays` and `ttlDays`, refer to the [BanyanDB TTL](https://skywalking.apache.org/docs/main/latest/en/banyandb/ttl) documentation. + # For more details on setting `segmentInterval` and `ttl`, refer to the [BanyanDB TTL](https://skywalking.apache.org/docs/main/latest/en/banyandb/ttl) documentation. # The "records" section defines settings for normal datasets not specified in records. # Each dataset will be grouped under a single group named "records". @@ -83,6 +86,7 @@ groups: shardNum: ${SW_STORAGE_BANYANDB_RECORDS_SHARD_NUM:1} segmentInterval: ${SW_STORAGE_BANYANDB_RECORDS_SI_DAYS:1} ttl: ${SW_STORAGE_BANYANDB_RECORDS_TTL_DAYS:3} + replicas: ${SW_STORAGE_BANYANDB_RECORDS_REPLICAS:0} # If the "warm" stage is enabled, the data will be moved to the "warm" stage after the TTL of the "hot" stage. # If the "cold" stage is enabled and "warm" stage is disabled, the data will be moved to the "cold" stage after the TTL of the "hot" stage. # If both "warm" and "cold" stages are enabled, the data will be moved to the "warm" stage after the TTL of the "hot" stage, and then to the "cold" stage after the TTL of the "warm" stage. @@ -94,12 +98,14 @@ groups: shardNum: ${SW_STORAGE_BANYANDB_RECORDS_WARM_SHARD_NUM:1} segmentInterval: ${SW_STORAGE_BANYANDB_RECORDS_WARM_SI_DAYS:2} ttl: ${SW_STORAGE_BANYANDB_RECORDS_WARM_TTL_DAYS:7} + replicas: ${SW_STORAGE_BANYANDB_RECORDS_WARM_REPLICAS:0} nodeSelector: ${SW_STORAGE_BANYANDB_RECORDS_WARM_NODE_SELECTOR:"type=warm"} # The settings for the "cold" stage. cold: shardNum: ${SW_STORAGE_BANYANDB_RECORDS_COLD_SHARD_NUM:1} segmentInterval: ${SW_STORAGE_BANYANDB_RECORDS_COLD_SI_DAYS:3} ttl: ${SW_STORAGE_BANYANDB_RECORDS_COLD_TTL_DAYS:30} + replicas: ${SW_STORAGE_BANYANDB_RECORDS_COLD_REPLICAS:0} nodeSelector: ${SW_STORAGE_BANYANDB_RECORDS_COLD_NODE_SELECTOR:"type=cold"} # The group settings of super datasets. # Super datasets are used to store trace or log data that is too large for normal datasets. @@ -107,65 +113,77 @@ groups: shardNum: ${SW_STORAGE_BANYANDB_TRACE_SHARD_NUM:2} segmentInterval: ${SW_STORAGE_BANYANDB_TRACE_SI_DAYS:1} ttl: ${SW_STORAGE_BANYANDB_TRACE_TTL_DAYS:3} + replicas: ${SW_STORAGE_BANYANDB_TRACE_REPLICAS:0} enableWarmStage: ${SW_STORAGE_BANYANDB_TRACE_ENABLE_WARM_STAGE:false} enableColdStage: ${SW_STORAGE_BANYANDB_TRACE_ENABLE_COLD_STAGE:false} warm: shardNum: ${SW_STORAGE_BANYANDB_TRACE_WARM_SHARD_NUM:2} segmentInterval: ${SW_STORAGE_BANYANDB_TRACE_WARM_SI_DAYS:1} ttl: ${SW_STORAGE_BANYANDB_TRACE_WARM_TTL_DAYS:7} + replicas: ${SW_STORAGE_BANYANDB_TRACE_WARM_REPLICAS:0} nodeSelector: ${SW_STORAGE_BANYANDB_TRACE_WARM_NODE_SELECTOR:"type=warm"} cold: shardNum: ${SW_STORAGE_BANYANDB_TRACE_COLD_SHARD_NUM:2} segmentInterval: ${SW_STORAGE_BANYANDB_TRACE_COLD_SI_DAYS:1} ttl: ${SW_STORAGE_BANYANDB_TRACE_COLD_TTL_DAYS:30} + replicas: ${SW_STORAGE_BANYANDB_TRACE_COLD_REPLICAS:0} nodeSelector: ${SW_STORAGE_BANYANDB_TRACE_COLD_NODE_SELECTOR:"type=cold"} recordsZipkinTrace: shardNum: ${SW_STORAGE_BANYANDB_ZIPKIN_TRACE_SHARD_NUM:2} segmentInterval: ${SW_STORAGE_BANYANDB_ZIPKIN_TRACE_SI_DAYS:1} ttl: ${SW_STORAGE_BANYANDB_ZIPKIN_TRACE_TTL_DAYS:3} + replicas: ${SW_STORAGE_BANYANDB_ZIPKIN_TRACE_REPLICAS:0} enableWarmStage: ${SW_STORAGE_BANYANDB_ZIPKIN_TRACE_ENABLE_WARM_STAGE:false} enableColdStage: ${SW_STORAGE_BANYANDB_ZIPKIN_TRACE_ENABLE_COLD_STAGE:false} warm: shardNum: ${SW_STORAGE_BANYANDB_ZIPKIN_TRACE_WARM_SHARD_NUM:2} segmentInterval: ${SW_STORAGE_BANYANDB_ZIPKIN_TRACE_WARM_SI_DAYS:1} ttl: ${SW_STORAGE_BANYANDB_ZIPKIN_TRACE_WARM_TTL_DAYS:7} + replicas: ${SW_STORAGE_BANYANDB_ZIPKIN_TRACE_WARM_REPLICAS:0} nodeSelector: ${SW_STORAGE_BANYANDB_ZIPKIN_TRACE_WARM_NODE_SELECTOR:"type=warm"} cold: shardNum: ${SW_STORAGE_BANYANDB_ZIPKIN_TRACE_COLD_SHARD_NUM:2} segmentInterval: ${SW_STORAGE_BANYANDB_ZIPKIN_TRACE_COLD_SI_DAYS:1} ttl: ${SW_STORAGE_BANYANDB_ZIPKIN_TRACE_COLD_TTL_DAYS:30} + replicas: ${SW_STORAGE_BANYANDB_ZIPKIN_TRACE_COLD_REPLICAS:0} nodeSelector: ${SW_STORAGE_BANYANDB_ZIPKIN_TRACE_COLD_NODE_SELECTOR:"type=cold"} recordsLog: shardNum: ${SW_STORAGE_BANYANDB_LOG_SHARD_NUM:2} segmentInterval: ${SW_STORAGE_BANYANDB_LOG_SI_DAYS:1} ttl: ${SW_STORAGE_BANYANDB_LOG_TTL_DAYS:3} + replicas: ${SW_STORAGE_BANYANDB_LOG_REPLICAS:0} enableWarmStage: ${SW_STORAGE_BANYANDB_LOG_ENABLE_WARM_STAGE:false} enableColdStage: ${SW_STORAGE_BANYANDB_LOG_ENABLE_COLD_STAGE:false} warm: shardNum: ${SW_STORAGE_BANYANDB_LOG_WARM_SHARD_NUM:2} segmentInterval: ${SW_STORAGE_BANYANDB_LOG_WARM_SI_DAYS:1} ttl: ${SW_STORAGE_BANYANDB_LOG_WARM_TTL_DAYS:7} + replicas: ${SW_STORAGE_BANYANDB_LOG_WARM_REPLICAS:0} nodeSelector: ${SW_STORAGE_BANYANDB_LOG_WARM_NODE_SELECTOR:"type=warm"} cold: shardNum: ${SW_STORAGE_BANYANDB_LOG_COLD_SHARD_NUM:2} segmentInterval: ${SW_STORAGE_BANYANDB_LOG_COLD_SI_DAYS:1} ttl: ${SW_STORAGE_BANYANDB_LOG_COLD_TTL_DAYS:30} + replicas: ${SW_STORAGE_BANYANDB_LOG_COLD_REPLICAS:0} nodeSelector: ${SW_STORAGE_BANYANDB_LOG_COLD_NODE_SELECTOR:"type=cold"} recordsBrowserErrorLog: shardNum: ${SW_STORAGE_BANYANDB_BROWSER_ERROR_LOG_SHARD_NUM:2} segmentInterval: ${SW_STORAGE_BANYANDB_BROWSER_ERROR_LOG_SI_DAYS:1} ttl: ${SW_STORAGE_BANYANDB_BROWSER_ERROR_LOG_TTL_DAYS:3} + replicas: ${SW_STORAGE_BANYANDB_BROWSER_ERROR_LOG_REPLICAS:0} enableWarmStage: ${SW_STORAGE_BANYANDB_BROWSER_ERROR_LOG_ENABLE_WARM_STAGE:false} enableColdStage: ${SW_STORAGE_BANYANDB_BROWSER_ERROR_LOG_ENABLE_COLD_STAGE:false} warm: shardNum: ${SW_STORAGE_BANYANDB_BROWSER_ERROR_LOG_WARM_SHARD_NUM:2} segmentInterval: ${SW_STORAGE_BANYANDB_BROWSER_ERROR_LOG_WARM_SI_DAYS:1} ttl: ${SW_STORAGE_BANYANDB_BROWSER_ERROR_LOG_WARM_TTL_DAYS:7} + replicas: ${SW_STORAGE_BANYANDB_BROWSER_ERROR_LOG_WARM_REPLICAS:0} nodeSelector: ${SW_STORAGE_BANYANDB_BROWSER_ERROR_LOG_WARM_NODE_SELECTOR:"type=warm"} cold: shardNum: ${SW_STORAGE_BANYANDB_BROWSER_ERROR_LOG_COLD_SHARD_NUM:2} segmentInterval: ${SW_STORAGE_BANYANDB_BROWSER_ERROR_LOG_COLD_SI_DAYS:1} ttl: ${SW_STORAGE_BANYANDB_BROWSER_ERROR_LOG_COLD_TTL_DAYS:30} + replicas: ${SW_STORAGE_BANYANDB_BROWSER_ERROR_LOG_COLD_REPLICAS:0} nodeSelector: ${SW_STORAGE_BANYANDB_BROWSER_ERROR_LOG_COLD_NODE_SELECTOR:"type=cold"} # The group settings of metrics. # @@ -177,49 +195,58 @@ groups: shardNum: ${SW_STORAGE_BANYANDB_METRICS_MINUTE_SHARD_NUM:2} segmentInterval: ${SW_STORAGE_BANYANDB_METRICS_MINUTE_SI_DAYS:1} ttl: ${SW_STORAGE_BANYANDB_METRICS_MINUTE_TTL_DAYS:7} + replicas: ${SW_STORAGE_BANYANDB_METRICS_MINUTE_REPLICAS:0} enableWarmStage: ${SW_STORAGE_BANYANDB_METRICS_MINUTE_ENABLE_WARM_STAGE:false} enableColdStage: ${SW_STORAGE_BANYANDB_METRICS_MINUTE_ENABLE_COLD_STAGE:false} warm: shardNum: ${SW_STORAGE_BANYANDB_METRICS_MINUTE_WARM_SHARD_NUM:2} segmentInterval: ${SW_STORAGE_BANYANDB_METRICS_MINUTE_WARM_SI_DAYS:3} ttl: ${SW_STORAGE_BANYANDB_METRICS_MINUTE_WARM_TTL_DAYS:15} + replicas: ${SW_STORAGE_BANYANDB_METRICS_MINUTE_WARM_REPLICAS:0} nodeSelector: ${SW_STORAGE_BANYANDB_METRICS_MINUTE_WARM_NODE_SELECTOR:"type=warm"} cold: shardNum: ${SW_STORAGE_BANYANDB_METRICS_MINUTE_COLD_SHARD_NUM:2} segmentInterval: ${SW_STORAGE_BANYANDB_METRICS_MINUTE_COLD_SI_DAYS:5} ttl: ${SW_STORAGE_BANYANDB_METRICS_MINUTE_COLD_TTL_DAYS:60} + replicas: ${SW_STORAGE_BANYANDB_METRICS_MINUTE_COLD_REPLICAS:0} nodeSelector: ${SW_STORAGE_BANYANDB_METRICS_MINUTE_COLD_NODE_SELECTOR:"type=cold"} metricsHour: shardNum: ${SW_STORAGE_BANYANDB_METRICS_HOUR_SHARD_NUM:1} segmentInterval: ${SW_STORAGE_BANYANDB_METRICS_HOUR_SI_DAYS:5} ttl: ${SW_STORAGE_BANYANDB_METRICS_HOUR_TTL_DAYS:15} + replicas: ${SW_STORAGE_BANYANDB_METRICS_HOUR_REPLICAS:0} enableWarmStage: ${SW_STORAGE_BANYANDB_METRICS_HOUR_ENABLE_WARM_STAGE:false} enableColdStage: ${SW_STORAGE_BANYANDB_METRICS_HOUR_ENABLE_COLD_STAGE:false} warm: shardNum: ${SW_STORAGE_BANYANDB_METRICS_HOUR_WARM_SHARD_NUM:1} segmentInterval: ${SW_STORAGE_BANYANDB_METRICS_HOUR_WARM_SI_DAYS:7} ttl: ${SW_STORAGE_BANYANDB_METRICS_HOUR_WARM_TTL_DAYS:30} + replicas: ${SW_STORAGE_BANYANDB_METRICS_HOUR_WARM_REPLICAS:0} nodeSelector: ${SW_STORAGE_BANYANDB_METRICS_HOUR_WARM_NODE_SELECTOR:"type=warm"} cold: shardNum: ${SW_STORAGE_BANYANDB_METRICS_HOUR_COLD_SHARD_NUM:1} segmentInterval: ${SW_STORAGE_BANYANDB_METRICS_HOUR_COLD_SI_DAYS:15} ttl: ${SW_STORAGE_BANYANDB_METRICS_HOUR_COLD_TTL_DAYS:120} + replicas: ${SW_STORAGE_BANYANDB_METRICS_HOUR_COLD_REPLICAS:0} nodeSelector: ${SW_STORAGE_BANYANDB_METRICS_HOUR_COLD_NODE_SELECTOR:"type=cold"} metricsDay: shardNum: ${SW_STORAGE_BANYANDB_METRICS_DAY_SHARD_NUM:1} segmentInterval: ${SW_STORAGE_BANYANDB_METRICS_DAY_SI_DAYS:15} ttl: ${SW_STORAGE_BANYANDB_METRICS_DAY_TTL_DAYS:15} + replicas: ${SW_STORAGE_BANYANDB_METRICS_DAY_REPLICAS:0} enableWarmStage: ${SW_STORAGE_BANYANDB_METRICS_DAY_ENABLE_WARM_STAGE:false} enableColdStage: ${SW_STORAGE_BANYANDB_METRICS_DAY_ENABLE_COLD_STAGE:false} warm: shardNum: ${SW_STORAGE_BANYANDB_METRICS_DAY_WARM_SHARD_NUM:1} segmentInterval: ${SW_STORAGE_BANYANDB_METRICS_DAY_WARM_SI_DAYS:15} ttl: ${SW_STORAGE_BANYANDB_METRICS_DAY_WARM_TTL_DAYS:30} + replicas: ${SW_STORAGE_BANYANDB_METRICS_DAY_WARM_REPLICAS:0} nodeSelector: ${SW_STORAGE_BANYANDB_METRICS_DAY_WARM_NODE_SELECTOR:"type=warm"} cold: shardNum: ${SW_STORAGE_BANYANDB_METRICS_DAY_COLD_SHARD_NUM:1} segmentInterval: ${SW_STORAGE_BANYANDB_METRICS_DAY_COLD_SI_DAYS:15} ttl: ${SW_STORAGE_BANYANDB_METRICS_DAY_COLD_TTL_DAYS:120} + replicas: ${SW_STORAGE_BANYANDB_METRICS_DAY_COLD_REPLICAS:0} nodeSelector: ${SW_STORAGE_BANYANDB_METRICS_DAY_COLD_NODE_SELECTOR:"type=cold"} # If the metrics is marked as "index_mode", the metrics will be stored in the "metadata" group. # The "metadata" group is designed to store metrics that are used for indexing without value columns. @@ -229,10 +256,12 @@ groups: shardNum: ${SW_STORAGE_BANYANDB_METADATA_SHARD_NUM:2} segmentInterval: ${SW_STORAGE_BANYANDB_METADATA_SI_DAYS:15} ttl: ${SW_STORAGE_BANYANDB_METADATA_TTL_DAYS:15} + replicas: ${SW_STORAGE_BANYANDB_METADATA_REPLICAS:0} # The group settings of property, such as UI and profiling. property: shardNum: ${SW_STORAGE_BANYANDB_PROPERTY_SHARD_NUM:1} + replicas: ${SW_STORAGE_BANYANDB_PROPERTY_REPLICAS:0} ``` ### TopN Rules Configuration diff --git a/oap-server-bom/pom.xml b/oap-server-bom/pom.xml index 73d3d71377..1c10967489 100644 --- a/oap-server-bom/pom.xml +++ b/oap-server-bom/pom.xml @@ -72,7 +72,7 @@ <httpcore.version>4.4.16</httpcore.version> <httpasyncclient.version>4.1.5</httpasyncclient.version> <commons-compress.version>1.21</commons-compress.version> - <banyandb-java-client.version>0.9.0-rc3</banyandb-java-client.version> + <banyandb-java-client.version>0.9.0-rc4</banyandb-java-client.version> <kafka-clients.version>3.4.0</kafka-clients.version> <spring-kafka-test.version>2.4.6.RELEASE</spring-kafka-test.version> <consul.client.version>1.5.3</consul.client.version> diff --git a/oap-server/server-starter/src/main/resources/bydb.yml b/oap-server/server-starter/src/main/resources/bydb.yml index 75899fee8a..8bd194fe7c 100644 --- a/oap-server/server-starter/src/main/resources/bydb.yml +++ b/oap-server/server-starter/src/main/resources/bydb.yml @@ -43,6 +43,8 @@ global: # The batch size for querying profile data. profileDataQueryBatchSize: ${SW_STORAGE_BANYANDB_QUERY_PROFILE_DATA_BATCH_SIZE:100} asyncProfilerTaskQueryMaxSize: ${SW_STORAGE_BANYANDB_ASYNC_PROFILER_TASK_QUERY_MAX_SIZE:200} + user: ${SW_STORAGE_BANYANDB_USER:""} + password: ${SW_STORAGE_BANYANDB_PASSWORD:""} # If the BanyanDB server is configured with TLS, configure the TLS cert file path and enable TLS connection. sslTrustCAPath: ${SW_STORAGE_BANYANDB_SSL_TRUST_CA_PATH:""} # Cleanup TopN rules in BanyanDB server that are not configured in the bydb-topn.yml config. @@ -50,12 +52,13 @@ global: groups: # The group settings of record. - # - "ShardNum": Number of shards in the group. Shards are the basic units of data storage in BanyanDB. Data is distributed across shards based on the hash value of the series ID. + # - "shardNum": Number of shards in the group. Shards are the basic units of data storage in BanyanDB. Data is distributed across shards based on the hash value of the series ID. # Refer to the [BanyanDB Shard](https://skywalking.apache.org/docs/skywalking-banyandb/latest/concept/clustering/#52-data-sharding) documentation for more details. - # - "SIDays": Interval in days for creating a new segment. Segments are time-based, allowing efficient data retention and querying. `SI` stands for Segment Interval. - # - "TTLDays": Time-to-live for the data in the group, in days. Data exceeding the TTL will be deleted. + # - "segmentInterval": Interval in days for creating a new segment. Segments are time-based, allowing efficient data retention and querying. `SI` stands for Segment Interval. + # - "ttl": Time-to-live for the data in the group, in days. Data exceeding the TTL will be deleted. + # - "replicas": Number of replicas for the group/stage. Replicas are used for data redundancy and high availability, a value of 0 means no replicas, while a value of 1 means one primary shard and one replica, higher values indicate more replicas. # - # For more details on setting `segmentIntervalDays` and `ttlDays`, refer to the [BanyanDB TTL](https://skywalking.apache.org/docs/main/latest/en/banyandb/ttl) documentation. + # For more details on setting `segmentInterval` and `ttl`, refer to the [BanyanDB TTL](https://skywalking.apache.org/docs/main/latest/en/banyandb/ttl) documentation. # The "records" section defines settings for normal datasets not specified in records. # Each dataset will be grouped under a single group named "records". @@ -64,6 +67,7 @@ groups: shardNum: ${SW_STORAGE_BANYANDB_RECORDS_SHARD_NUM:1} segmentInterval: ${SW_STORAGE_BANYANDB_RECORDS_SI_DAYS:1} ttl: ${SW_STORAGE_BANYANDB_RECORDS_TTL_DAYS:3} + replicas: ${SW_STORAGE_BANYANDB_RECORDS_REPLICAS:0} # If the "warm" stage is enabled, the data will be moved to the "warm" stage after the TTL of the "hot" stage. # If the "cold" stage is enabled and "warm" stage is disabled, the data will be moved to the "cold" stage after the TTL of the "hot" stage. # If both "warm" and "cold" stages are enabled, the data will be moved to the "warm" stage after the TTL of the "hot" stage, and then to the "cold" stage after the TTL of the "warm" stage. @@ -75,12 +79,14 @@ groups: shardNum: ${SW_STORAGE_BANYANDB_RECORDS_WARM_SHARD_NUM:1} segmentInterval: ${SW_STORAGE_BANYANDB_RECORDS_WARM_SI_DAYS:2} ttl: ${SW_STORAGE_BANYANDB_RECORDS_WARM_TTL_DAYS:7} + replicas: ${SW_STORAGE_BANYANDB_RECORDS_WARM_REPLICAS:0} nodeSelector: ${SW_STORAGE_BANYANDB_RECORDS_WARM_NODE_SELECTOR:"type=warm"} # The settings for the "cold" stage. cold: shardNum: ${SW_STORAGE_BANYANDB_RECORDS_COLD_SHARD_NUM:1} segmentInterval: ${SW_STORAGE_BANYANDB_RECORDS_COLD_SI_DAYS:3} ttl: ${SW_STORAGE_BANYANDB_RECORDS_COLD_TTL_DAYS:30} + replicas: ${SW_STORAGE_BANYANDB_RECORDS_COLD_REPLICAS:0} nodeSelector: ${SW_STORAGE_BANYANDB_RECORDS_COLD_NODE_SELECTOR:"type=cold"} # The group settings of super datasets. # Super datasets are used to store trace or log data that is too large for normal datasets. @@ -88,65 +94,77 @@ groups: shardNum: ${SW_STORAGE_BANYANDB_TRACE_SHARD_NUM:2} segmentInterval: ${SW_STORAGE_BANYANDB_TRACE_SI_DAYS:1} ttl: ${SW_STORAGE_BANYANDB_TRACE_TTL_DAYS:3} + replicas: ${SW_STORAGE_BANYANDB_TRACE_REPLICAS:0} enableWarmStage: ${SW_STORAGE_BANYANDB_TRACE_ENABLE_WARM_STAGE:false} enableColdStage: ${SW_STORAGE_BANYANDB_TRACE_ENABLE_COLD_STAGE:false} warm: shardNum: ${SW_STORAGE_BANYANDB_TRACE_WARM_SHARD_NUM:2} segmentInterval: ${SW_STORAGE_BANYANDB_TRACE_WARM_SI_DAYS:1} ttl: ${SW_STORAGE_BANYANDB_TRACE_WARM_TTL_DAYS:7} + replicas: ${SW_STORAGE_BANYANDB_TRACE_WARM_REPLICAS:0} nodeSelector: ${SW_STORAGE_BANYANDB_TRACE_WARM_NODE_SELECTOR:"type=warm"} cold: shardNum: ${SW_STORAGE_BANYANDB_TRACE_COLD_SHARD_NUM:2} segmentInterval: ${SW_STORAGE_BANYANDB_TRACE_COLD_SI_DAYS:1} ttl: ${SW_STORAGE_BANYANDB_TRACE_COLD_TTL_DAYS:30} + replicas: ${SW_STORAGE_BANYANDB_TRACE_COLD_REPLICAS:0} nodeSelector: ${SW_STORAGE_BANYANDB_TRACE_COLD_NODE_SELECTOR:"type=cold"} recordsZipkinTrace: shardNum: ${SW_STORAGE_BANYANDB_ZIPKIN_TRACE_SHARD_NUM:2} segmentInterval: ${SW_STORAGE_BANYANDB_ZIPKIN_TRACE_SI_DAYS:1} ttl: ${SW_STORAGE_BANYANDB_ZIPKIN_TRACE_TTL_DAYS:3} + replicas: ${SW_STORAGE_BANYANDB_ZIPKIN_TRACE_REPLICAS:0} enableWarmStage: ${SW_STORAGE_BANYANDB_ZIPKIN_TRACE_ENABLE_WARM_STAGE:false} enableColdStage: ${SW_STORAGE_BANYANDB_ZIPKIN_TRACE_ENABLE_COLD_STAGE:false} warm: shardNum: ${SW_STORAGE_BANYANDB_ZIPKIN_TRACE_WARM_SHARD_NUM:2} segmentInterval: ${SW_STORAGE_BANYANDB_ZIPKIN_TRACE_WARM_SI_DAYS:1} ttl: ${SW_STORAGE_BANYANDB_ZIPKIN_TRACE_WARM_TTL_DAYS:7} + replicas: ${SW_STORAGE_BANYANDB_ZIPKIN_TRACE_WARM_REPLICAS:0} nodeSelector: ${SW_STORAGE_BANYANDB_ZIPKIN_TRACE_WARM_NODE_SELECTOR:"type=warm"} cold: shardNum: ${SW_STORAGE_BANYANDB_ZIPKIN_TRACE_COLD_SHARD_NUM:2} segmentInterval: ${SW_STORAGE_BANYANDB_ZIPKIN_TRACE_COLD_SI_DAYS:1} ttl: ${SW_STORAGE_BANYANDB_ZIPKIN_TRACE_COLD_TTL_DAYS:30} + replicas: ${SW_STORAGE_BANYANDB_ZIPKIN_TRACE_COLD_REPLICAS:0} nodeSelector: ${SW_STORAGE_BANYANDB_ZIPKIN_TRACE_COLD_NODE_SELECTOR:"type=cold"} recordsLog: shardNum: ${SW_STORAGE_BANYANDB_LOG_SHARD_NUM:2} segmentInterval: ${SW_STORAGE_BANYANDB_LOG_SI_DAYS:1} ttl: ${SW_STORAGE_BANYANDB_LOG_TTL_DAYS:3} + replicas: ${SW_STORAGE_BANYANDB_LOG_REPLICAS:0} enableWarmStage: ${SW_STORAGE_BANYANDB_LOG_ENABLE_WARM_STAGE:false} enableColdStage: ${SW_STORAGE_BANYANDB_LOG_ENABLE_COLD_STAGE:false} warm: shardNum: ${SW_STORAGE_BANYANDB_LOG_WARM_SHARD_NUM:2} segmentInterval: ${SW_STORAGE_BANYANDB_LOG_WARM_SI_DAYS:1} ttl: ${SW_STORAGE_BANYANDB_LOG_WARM_TTL_DAYS:7} + replicas: ${SW_STORAGE_BANYANDB_LOG_WARM_REPLICAS:0} nodeSelector: ${SW_STORAGE_BANYANDB_LOG_WARM_NODE_SELECTOR:"type=warm"} cold: shardNum: ${SW_STORAGE_BANYANDB_LOG_COLD_SHARD_NUM:2} segmentInterval: ${SW_STORAGE_BANYANDB_LOG_COLD_SI_DAYS:1} ttl: ${SW_STORAGE_BANYANDB_LOG_COLD_TTL_DAYS:30} + replicas: ${SW_STORAGE_BANYANDB_LOG_COLD_REPLICAS:0} nodeSelector: ${SW_STORAGE_BANYANDB_LOG_COLD_NODE_SELECTOR:"type=cold"} recordsBrowserErrorLog: shardNum: ${SW_STORAGE_BANYANDB_BROWSER_ERROR_LOG_SHARD_NUM:2} segmentInterval: ${SW_STORAGE_BANYANDB_BROWSER_ERROR_LOG_SI_DAYS:1} ttl: ${SW_STORAGE_BANYANDB_BROWSER_ERROR_LOG_TTL_DAYS:3} + replicas: ${SW_STORAGE_BANYANDB_BROWSER_ERROR_LOG_REPLICAS:0} enableWarmStage: ${SW_STORAGE_BANYANDB_BROWSER_ERROR_LOG_ENABLE_WARM_STAGE:false} enableColdStage: ${SW_STORAGE_BANYANDB_BROWSER_ERROR_LOG_ENABLE_COLD_STAGE:false} warm: shardNum: ${SW_STORAGE_BANYANDB_BROWSER_ERROR_LOG_WARM_SHARD_NUM:2} segmentInterval: ${SW_STORAGE_BANYANDB_BROWSER_ERROR_LOG_WARM_SI_DAYS:1} ttl: ${SW_STORAGE_BANYANDB_BROWSER_ERROR_LOG_WARM_TTL_DAYS:7} + replicas: ${SW_STORAGE_BANYANDB_BROWSER_ERROR_LOG_WARM_REPLICAS:0} nodeSelector: ${SW_STORAGE_BANYANDB_BROWSER_ERROR_LOG_WARM_NODE_SELECTOR:"type=warm"} cold: shardNum: ${SW_STORAGE_BANYANDB_BROWSER_ERROR_LOG_COLD_SHARD_NUM:2} segmentInterval: ${SW_STORAGE_BANYANDB_BROWSER_ERROR_LOG_COLD_SI_DAYS:1} ttl: ${SW_STORAGE_BANYANDB_BROWSER_ERROR_LOG_COLD_TTL_DAYS:30} + replicas: ${SW_STORAGE_BANYANDB_BROWSER_ERROR_LOG_COLD_REPLICAS:0} nodeSelector: ${SW_STORAGE_BANYANDB_BROWSER_ERROR_LOG_COLD_NODE_SELECTOR:"type=cold"} # The group settings of metrics. # @@ -158,49 +176,58 @@ groups: shardNum: ${SW_STORAGE_BANYANDB_METRICS_MINUTE_SHARD_NUM:2} segmentInterval: ${SW_STORAGE_BANYANDB_METRICS_MINUTE_SI_DAYS:1} ttl: ${SW_STORAGE_BANYANDB_METRICS_MINUTE_TTL_DAYS:7} + replicas: ${SW_STORAGE_BANYANDB_METRICS_MINUTE_REPLICAS:0} enableWarmStage: ${SW_STORAGE_BANYANDB_METRICS_MINUTE_ENABLE_WARM_STAGE:false} enableColdStage: ${SW_STORAGE_BANYANDB_METRICS_MINUTE_ENABLE_COLD_STAGE:false} warm: shardNum: ${SW_STORAGE_BANYANDB_METRICS_MINUTE_WARM_SHARD_NUM:2} segmentInterval: ${SW_STORAGE_BANYANDB_METRICS_MINUTE_WARM_SI_DAYS:3} ttl: ${SW_STORAGE_BANYANDB_METRICS_MINUTE_WARM_TTL_DAYS:15} + replicas: ${SW_STORAGE_BANYANDB_METRICS_MINUTE_WARM_REPLICAS:0} nodeSelector: ${SW_STORAGE_BANYANDB_METRICS_MINUTE_WARM_NODE_SELECTOR:"type=warm"} cold: shardNum: ${SW_STORAGE_BANYANDB_METRICS_MINUTE_COLD_SHARD_NUM:2} segmentInterval: ${SW_STORAGE_BANYANDB_METRICS_MINUTE_COLD_SI_DAYS:5} ttl: ${SW_STORAGE_BANYANDB_METRICS_MINUTE_COLD_TTL_DAYS:60} + replicas: ${SW_STORAGE_BANYANDB_METRICS_MINUTE_COLD_REPLICAS:0} nodeSelector: ${SW_STORAGE_BANYANDB_METRICS_MINUTE_COLD_NODE_SELECTOR:"type=cold"} metricsHour: shardNum: ${SW_STORAGE_BANYANDB_METRICS_HOUR_SHARD_NUM:1} segmentInterval: ${SW_STORAGE_BANYANDB_METRICS_HOUR_SI_DAYS:5} ttl: ${SW_STORAGE_BANYANDB_METRICS_HOUR_TTL_DAYS:15} + replicas: ${SW_STORAGE_BANYANDB_METRICS_HOUR_REPLICAS:0} enableWarmStage: ${SW_STORAGE_BANYANDB_METRICS_HOUR_ENABLE_WARM_STAGE:false} enableColdStage: ${SW_STORAGE_BANYANDB_METRICS_HOUR_ENABLE_COLD_STAGE:false} warm: shardNum: ${SW_STORAGE_BANYANDB_METRICS_HOUR_WARM_SHARD_NUM:1} segmentInterval: ${SW_STORAGE_BANYANDB_METRICS_HOUR_WARM_SI_DAYS:7} ttl: ${SW_STORAGE_BANYANDB_METRICS_HOUR_WARM_TTL_DAYS:30} + replicas: ${SW_STORAGE_BANYANDB_METRICS_HOUR_WARM_REPLICAS:0} nodeSelector: ${SW_STORAGE_BANYANDB_METRICS_HOUR_WARM_NODE_SELECTOR:"type=warm"} cold: shardNum: ${SW_STORAGE_BANYANDB_METRICS_HOUR_COLD_SHARD_NUM:1} segmentInterval: ${SW_STORAGE_BANYANDB_METRICS_HOUR_COLD_SI_DAYS:15} ttl: ${SW_STORAGE_BANYANDB_METRICS_HOUR_COLD_TTL_DAYS:120} + replicas: ${SW_STORAGE_BANYANDB_METRICS_HOUR_COLD_REPLICAS:0} nodeSelector: ${SW_STORAGE_BANYANDB_METRICS_HOUR_COLD_NODE_SELECTOR:"type=cold"} metricsDay: shardNum: ${SW_STORAGE_BANYANDB_METRICS_DAY_SHARD_NUM:1} segmentInterval: ${SW_STORAGE_BANYANDB_METRICS_DAY_SI_DAYS:15} ttl: ${SW_STORAGE_BANYANDB_METRICS_DAY_TTL_DAYS:15} + replicas: ${SW_STORAGE_BANYANDB_METRICS_DAY_REPLICAS:0} enableWarmStage: ${SW_STORAGE_BANYANDB_METRICS_DAY_ENABLE_WARM_STAGE:false} enableColdStage: ${SW_STORAGE_BANYANDB_METRICS_DAY_ENABLE_COLD_STAGE:false} warm: shardNum: ${SW_STORAGE_BANYANDB_METRICS_DAY_WARM_SHARD_NUM:1} segmentInterval: ${SW_STORAGE_BANYANDB_METRICS_DAY_WARM_SI_DAYS:15} ttl: ${SW_STORAGE_BANYANDB_METRICS_DAY_WARM_TTL_DAYS:30} + replicas: ${SW_STORAGE_BANYANDB_METRICS_DAY_WARM_REPLICAS:0} nodeSelector: ${SW_STORAGE_BANYANDB_METRICS_DAY_WARM_NODE_SELECTOR:"type=warm"} cold: shardNum: ${SW_STORAGE_BANYANDB_METRICS_DAY_COLD_SHARD_NUM:1} segmentInterval: ${SW_STORAGE_BANYANDB_METRICS_DAY_COLD_SI_DAYS:15} ttl: ${SW_STORAGE_BANYANDB_METRICS_DAY_COLD_TTL_DAYS:120} + replicas: ${SW_STORAGE_BANYANDB_METRICS_DAY_COLD_REPLICAS:0} nodeSelector: ${SW_STORAGE_BANYANDB_METRICS_DAY_COLD_NODE_SELECTOR:"type=cold"} # If the metrics is marked as "index_mode", the metrics will be stored in the "metadata" group. # The "metadata" group is designed to store metrics that are used for indexing without value columns. @@ -210,7 +237,9 @@ groups: shardNum: ${SW_STORAGE_BANYANDB_METADATA_SHARD_NUM:2} segmentInterval: ${SW_STORAGE_BANYANDB_METADATA_SI_DAYS:15} ttl: ${SW_STORAGE_BANYANDB_METADATA_TTL_DAYS:15} + replicas: ${SW_STORAGE_BANYANDB_METADATA_REPLICAS:0} # The group settings of property, such as UI and profiling. property: shardNum: ${SW_STORAGE_BANYANDB_PROPERTY_SHARD_NUM:1} + replicas: ${SW_STORAGE_BANYANDB_PROPERTY_REPLICAS:0} diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBIndexInstaller.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBIndexInstaller.java index 06636b4fa6..4394228b9d 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBIndexInstaller.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBIndexInstaller.java @@ -254,7 +254,11 @@ public class BanyanDBIndexInstaller extends ModelInstaller { Group.Builder gBuilder = Group.newBuilder() .setMetadata(BanyandbCommon.Metadata.newBuilder().setName(metadata.getGroup())); - BanyandbCommon.ResourceOpts.Builder optsBuilder = BanyandbCommon.ResourceOpts.newBuilder().setShardNum(metadata.getResource().getShardNum()); + BanyandbCommon.ResourceOpts.Builder optsBuilder = BanyandbCommon.ResourceOpts.newBuilder() + .setShardNum(metadata.getResource() + .getShardNum()) + .setReplicas(metadata.getResource() + .getReplicas()); switch (metadata.getKind()) { case STREAM: @@ -311,6 +315,7 @@ public class BanyanDBIndexInstaller extends ModelInstaller { IntervalRule.Unit.UNIT_DAY) .setNum( stage.getTtl())) + .setReplicas(stage.getReplicas()) .setNodeSelector(stage.getNodeSelector()) .setClose(stage.isClose()) ); diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageClient.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageClient.java index 8e355c0972..ba7aac61a3 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageClient.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageClient.java @@ -55,6 +55,7 @@ import org.apache.skywalking.oap.server.library.client.Client; import org.apache.skywalking.oap.server.library.client.healthcheck.DelegatedHealthChecker; import org.apache.skywalking.oap.server.library.client.healthcheck.HealthCheckable; import org.apache.skywalking.oap.server.library.util.HealthChecker; +import org.apache.skywalking.oap.server.library.util.StringUtil; /** * BanyanDBStorageClient is a simple wrapper for the underlying {@link BanyanDBClient}, @@ -70,6 +71,17 @@ public class BanyanDBStorageClient implements Client, HealthCheckable { public BanyanDBStorageClient(BanyanDBStorageConfig config) { Options options = new Options(); options.setSslTrustCAPath(config.getGlobal().getSslTrustCAPath()); + String username = config.getGlobal().getUser(); + String password = config.getGlobal().getPassword(); + if (StringUtil.isNotBlank(username)) { + if (StringUtil.isBlank(password)) { + throw new IllegalArgumentException("User is set, but password is not set."); + } + options.setUsername(username); + options.setPassword(password); + } else if (StringUtil.isNotBlank(password)) { + throw new IllegalArgumentException("Password is set, but user is not set."); + } this.client = new BanyanDBClient(config.getTargetArray(), options); this.flushTimeout = config.getGlobal().getFlushTimeout(); } diff --git a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageConfig.java b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageConfig.java index e32891c3e4..4a6429cd9f 100644 --- a/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageConfig.java +++ b/oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageConfig.java @@ -87,6 +87,8 @@ public class BanyanDBStorageConfig extends ModuleConfig { */ private int profileTaskQueryMaxSize; + private String user; + private String password; /** * If the BanyanDB server is configured with TLS, config the TLS cert file path and open tls connection. */ @@ -117,6 +119,7 @@ public class BanyanDBStorageConfig extends ModuleConfig { private int shardNum; private int segmentInterval; private int ttl; + private int replicas; // Indicates whether segments that are no longer live should be closed. private boolean close = false; } @@ -133,6 +136,7 @@ public class BanyanDBStorageConfig extends ModuleConfig { private int shardNum; private int segmentInterval; private int ttl; + private int replicas; private boolean enableWarmStage = false; private boolean enableColdStage = false; private List<String> defaultQueryStages = new ArrayList<>(2); diff --git a/test/e2e-v2/cases/storage/banyandb/auth-config.yaml b/test/e2e-v2/cases/storage/banyandb/auth-config.yaml new file mode 100644 index 0000000000..3c5d90cabb --- /dev/null +++ b/test/e2e-v2/cases/storage/banyandb/auth-config.yaml @@ -0,0 +1,20 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +users: + - username: admin + password: admin + - username: test + password: test \ No newline at end of file diff --git a/test/e2e-v2/cases/storage/banyandb/docker-compose.yml b/test/e2e-v2/cases/storage/banyandb/docker-compose.yml index 1e1aaa98a7..6c65818547 100644 --- a/test/e2e-v2/cases/storage/banyandb/docker-compose.yml +++ b/test/e2e-v2/cases/storage/banyandb/docker-compose.yml @@ -20,6 +20,12 @@ services: extends: file: ../../../script/docker-compose/base-compose.yml service: banyandb + volumes: + - ./auth-config.yaml:/auth-config.yaml + entrypoint: /bin/sh -c + command: | + "chmod 600 /auth-config.yaml && + /banyand standalone --stream-root-path /tmp/stream-data --measure-root-path /tmp/measure-data --measure-metadata-cache-wait-duration 1m --stream-metadata-cache-wait-duration 1m --auth-config-file=/auth-config.yaml" networks: - e2e @@ -31,6 +37,8 @@ services: - ./config/bydb-topn.yml:/skywalking/config/bydb-topn.yml environment: SW_STORAGE: banyandb + SW_STORAGE_BANYANDB_USER: admin + SW_STORAGE_BANYANDB_PASSWORD: admin ports: - 12800 depends_on: diff --git a/test/e2e-v2/script/env b/test/e2e-v2/script/env index ad47277695..9f0c0ec291 100644 --- a/test/e2e-v2/script/env +++ b/test/e2e-v2/script/env @@ -23,7 +23,7 @@ SW_AGENT_CLIENT_JS_COMMIT=af0565a67d382b683c1dbd94c379b7080db61449 SW_AGENT_CLIENT_JS_TEST_COMMIT=4f1eb1dcdbde3ec4a38534bf01dded4ab5d2f016 SW_KUBERNETES_COMMIT_SHA=6fe5e6f0d3b7686c6be0457733e825ee68cb9b35 SW_ROVER_COMMIT=79292fe07f17f98f486e0c4471213e1961fb2d1d -SW_BANYANDB_COMMIT=0f3b90d9b4f628d2de02cb39ef678d636358df44 +SW_BANYANDB_COMMIT=7e5b2d0404e8ad6d5835eee6fe589a2544d0decb SW_AGENT_PHP_COMMIT=d1114e7be5d89881eec76e5b56e69ff844691e35 SW_PREDICTOR_COMMIT=54a0197654a3781a6f73ce35146c712af297c994