dlmarion commented on code in PR #3180:
URL: https://github.com/apache/accumulo/pull/3180#discussion_r1099086976
##########
minicluster/src/main/java/org/apache/accumulo/miniclusterImpl/MiniAccumuloConfigImpl.java:
##########
@@ -812,4 +820,29 @@ public int getNumCompactors() {
public void setNumCompactors(int numCompactors) {
this.numCompactors = numCompactors;
}
+
+ /**
+ * Set the FinalSiteConfigUpdater instance that will be used to modify the
site configuration
+ * right before it's written out a file. This would be useful in the case
where the configuration
+ * needs to be updated based on a property that is set in
MiniAccumuloClusterImpl like
+ * instance.volumes
+ *
+ * @param updater FinalSiteConfigUpdater instance
+ * @since 2.1.1
+ */
+ public void setFinalSiteConfigUpdater(FinalSiteConfigUpdater updater) {
+ this.updater = updater;
+ }
+
+ /**
+ * Called by MiniAccumuloClusterImpl after all modifications are done to the
configuration and
+ * right before it's written out to a file.
+ *
+ * @since 2.1.1
Review Comment:
Addressed in f34eb45.
##########
server/base/src/main/java/org/apache/accumulo/server/fs/VolumeManagerImpl.java:
##########
@@ -353,11 +363,63 @@ public short getDefaultReplication(Path path) {
return getFileSystemByPath(path).getDefaultReplication(path);
}
+ /**
+ * The Hadoop Configuration object does not currently allow for duplicate
properties to be set in
+ * a single Configuration for different FileSystem URIs. Here we will look
for properties in the
+ * Accumulo configuration of the form:
+ *
+ * <pre>
+ * instance.volume.config.<volume-uri>.<hdfs-property>
+ * </pre>
+ *
+ * We will use these properties to return a new Configuration object that
can be used with the
+ * FileSystem URI.
+ *
+ * @param conf AccumuloConfiguration object
+ * @param hadoopConf Hadoop Configuration object
+ * @param filesystemURI Volume Filesystem URI
+ * @return Hadoop Configuration with custom overrides for this FileSystem
+ */
+ private static Configuration
getVolumeManagerConfiguration(AccumuloConfiguration conf,
+ final Configuration hadoopConf, final String filesystemURI) {
+ final Configuration volumeConfig = new Configuration(hadoopConf);
+ final Map<String,String> customProps =
+
conf.getAllPropertiesWithPrefixStripped(Property.INSTANCE_VOLUMES_CONFIG);
+ customProps.forEach((key, value) -> {
+ if (key.startsWith(filesystemURI)) {
+ String property = key.substring(filesystemURI.length() + 1);
+ log.info("Overriding property {} to {} for volume {}", property,
value, filesystemURI);
+ volumeConfig.set(property, value);
+ }
+ });
+ return volumeConfig;
+ }
+
+ private static void warnVolumeOverridesMissingVolume(AccumuloConfiguration
conf,
+ Set<String> definedVolumes) {
+ final Map<String,String> overrideProperties = new ConcurrentHashMap<>(
+
conf.getAllPropertiesWithPrefixStripped(Property.INSTANCE_VOLUMES_CONFIG));
+
+ definedVolumes.forEach(vol -> {
+ log.debug("Looking for defined volume: {}", vol);
+ overrideProperties.keySet().forEach(override -> {
+ if (override.startsWith(vol)) {
+ log.debug("Found volume {}, removing property {}", vol, override);
+ overrideProperties.remove(override);
+ }
+ });
+ });
Review Comment:
Addressed in f34eb45.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: [email protected]
For queries about this service, please contact Infrastructure at:
[email protected]
