vaijosh commented on code in PR #3081:
URL: https://github.com/apache/hugegraph/pull/3081#discussion_r4044302439
##########
hugegraph-store/hg-store-node/src/main/java/org/apache/hugegraph/store/node/AppConfig.java:
##########
@@ -312,6 +636,113 @@ public class RocksdbConfig {
private Map<String, String> rocksdb = new HashMap<>();
}
+ /**
+ * Spring {@link ConfigurationProperties} wrapper for the common cloud
storage properties.
+ *
+ * <p>Provider-specific keys (e.g. {@code cloud.storage.s3.bucket}) are
read
+ * directly from the Spring {@link Environment} at conversion time, so
this class
+ * has zero knowledge of any specific cloud provider.
+ *
+ * <pre>
+ * cloud:
+ * storage:
+ * enabled: true
+ * provider: s3 # selects which sub-namespace to forward to
the provider
+ * path-prefix: hugegraph
+ * s3: # all keys here are forwarded as-is to the S3
provider
+ * bucket: my-bucket
+ * region: us-east-1
+ * access-key: ${AWS_ACCESS_KEY_ID}
+ * secret-key: ${AWS_SECRET_ACCESS_KEY}
+ * </pre>
+ */
+ @Data
+ @Configuration
+ @ConfigurationProperties(prefix = "cloud.storage")
+ public class CloudStorageSpringConfig {
+
+ private boolean enabled = false;
+ private String provider = "s3";
+ private String pathPrefix = "hugegraph";
+ private boolean startupHydrationEnabled = true;
+ private long readMissGuardWindowMs = 3000L;
+ // Whole-file upload retries after a first failure. Default 3 under
the primary-durability
+ private int uploadRetryMaxAttempts = 3;
+ private long uploadRetryInitialDelayMs = 1_000L;
+ private long uploadRetryMaxDelayMs = 60_000L;
+ // Backpressure high-watermark on the pending-upload backlog; 0
(default) disables it.
+ // Opt-in: when > 0 the throttle parks RocksDB's flush/compaction
thread (up to 30s/event),
+ // which under a sustained cloud outage can stall memtable flushes /
stop writes.
+ private int uploadBackpressureHighWatermark = 0;
+ // Max DLQ entries before oldest are evicted (bounds memory/disk under
a prolonged outage).
+ private int dlqMaxSize = 100_000;
+ // Debounce window (ms) for the per-SST metadata sync; <= 0 disables
debouncing.
+ private long metadataSyncDebounceMs = 1_000L;
+ // Force a metadata publish once this many SST uploads accumulate
unmirrored; <= 0 disables.
+ private int metadataSyncMaxUnpublished = 32;
+ // Stable per-node identity for the cloud key scope. Blank =>
persisted-in-data-dir scope
+ // (seeded from network address). Set for guaranteed recovery after IP
drift / disk loss.
+ private String nodeId = "";
+
+ /**
+ * Injected by Spring; used to read {@code cloud.storage.<provider>.*}
properties
+ * without coupling this class to any specific provider.
+ */
+ @Autowired
+ @EqualsAndHashCode.Exclude
+ @ToString.Exclude
+ private Environment environment;
+
+ /** Converts this Spring-bound config into a plain {@link
CloudStorageConfig} POJO. */
+ public CloudStorageConfig toCloudStorageConfig() {
+ CloudStorageConfig cfg = new CloudStorageConfig();
+ cfg.setEnabled(enabled);
+ cfg.setProvider(provider);
+ cfg.setPathPrefix(pathPrefix);
+ cfg.setStartupHydrationEnabled(startupHydrationEnabled);
+ cfg.setReadMissGuardWindowMs(readMissGuardWindowMs);
+ cfg.setUploadRetryMaxAttempts(uploadRetryMaxAttempts);
+ cfg.setUploadRetryInitialDelayMs(uploadRetryInitialDelayMs);
+ cfg.setUploadRetryMaxDelayMs(uploadRetryMaxDelayMs);
+
cfg.setUploadBackpressureHighWatermark(uploadBackpressureHighWatermark);
+ cfg.setDlqMaxSize(dlqMaxSize);
+ cfg.setMetadataSyncDebounceMs(metadataSyncDebounceMs);
+ cfg.setMetadataSyncMaxUnpublished(metadataSyncMaxUnpublished);
+ cfg.setNodeId(nodeId);
+ cfg.setProviderProperties(readProviderProperties());
+ return cfg;
+ }
+
+ /**
+ * Reads all {@code cloud.storage.<provider>.*} keys from the Spring
Environment
+ * and returns them as a flat map with the provider sub-prefix
stripped.
+ *
+ * <p>For example, with {@code provider=s3}, the YAML key
+ * {@code cloud.storage.s3.bucket} becomes {@code bucket} in the
returned map.
+ */
+ private Map<String, String> readProviderProperties() {
Review Comment:
Addressed.
##########
.licenserc.yaml:
##########
@@ -66,6 +66,7 @@ header: # `header` section is configurations for source codes
license header.
- '**/*.log'
- '**/*.txt'
- '**/*.json'
+ - '**/*.ipynb'
Review Comment:
Addressed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
