Repository: flume Updated Branches: refs/heads/trunk f9d5c4e59 -> 4ae19e8f9 (forced update)
FLUME-3298 Make hadoop-common optional in hadoop-credential-store-config-filter hadoop-common should be optional as in the hdfs-sink This closes #248 Reviewers: Endre Major, Peter Turcsanyi (Ferenc Szabo via Ferenc Szabo) Project: http://git-wip-us.apache.org/repos/asf/flume/repo Commit: http://git-wip-us.apache.org/repos/asf/flume/commit/4ae19e8f Tree: http://git-wip-us.apache.org/repos/asf/flume/tree/4ae19e8f Diff: http://git-wip-us.apache.org/repos/asf/flume/diff/4ae19e8f Branch: refs/heads/trunk Commit: 4ae19e8f9701399540501256ec0cce29254083fd Parents: 85e0b4e Author: Ferenc Szabo <[email protected]> Authored: Mon Nov 26 15:54:32 2018 +0100 Committer: Ferenc Szabo <[email protected]> Committed: Tue Nov 27 13:17:52 2018 +0100 ---------------------------------------------------------------------- .../pom.xml | 1 + flume-ng-doc/sphinx/FlumeUserGuide.rst | 3 +++ flume-ng-tests/pom.xml | 16 ++++++++++++++++ .../org/apache/flume/test/agent/TestConfig.java | 18 ++++++++++++++++-- .../org/apache/flume/test/util/StagedInstall.java | 7 +++++-- 5 files changed, 41 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/flume/blob/4ae19e8f/flume-ng-configfilters/flume-ng-hadoop-credential-store-config-filter/pom.xml ---------------------------------------------------------------------- diff --git a/flume-ng-configfilters/flume-ng-hadoop-credential-store-config-filter/pom.xml b/flume-ng-configfilters/flume-ng-hadoop-credential-store-config-filter/pom.xml index e5207a0..f1aee88 100644 --- a/flume-ng-configfilters/flume-ng-hadoop-credential-store-config-filter/pom.xml +++ b/flume-ng-configfilters/flume-ng-hadoop-credential-store-config-filter/pom.xml @@ -33,6 +33,7 @@ limitations under the License. <dependency> <groupId>org.apache.hadoop</groupId> <artifactId>hadoop-common</artifactId> + <optional>true</optional> </dependency> <dependency> <groupId>org.apache.flume</groupId> http://git-wip-us.apache.org/repos/asf/flume/blob/4ae19e8f/flume-ng-doc/sphinx/FlumeUserGuide.rst ---------------------------------------------------------------------- diff --git a/flume-ng-doc/sphinx/FlumeUserGuide.rst b/flume-ng-doc/sphinx/FlumeUserGuide.rst index df4d5c2..9d4cfbc 100644 --- a/flume-ng-doc/sphinx/FlumeUserGuide.rst +++ b/flume-ng-doc/sphinx/FlumeUserGuide.rst @@ -4782,6 +4782,9 @@ The ``generateUniqId.sh`` will return ``1234`` with an exit code ``0``. Hadoop Credential Store Config Filter ------------------------------------- +A hadoop-common library needed on the classpath for this feature (2.6+ version). +If hadoop is installed the agent adds it to the classpath automatically + =============================================== ========== ============================================== Property Name Default Description =============================================== ========== ============================================== http://git-wip-us.apache.org/repos/asf/flume/blob/4ae19e8f/flume-ng-tests/pom.xml ---------------------------------------------------------------------- diff --git a/flume-ng-tests/pom.xml b/flume-ng-tests/pom.xml index 5c74196..3e5ec8e 100644 --- a/flume-ng-tests/pom.xml +++ b/flume-ng-tests/pom.xml @@ -74,6 +74,22 @@ <build> <plugins> + <plugin> + <groupId>org.apache.maven.plugins</groupId> + <artifactId>maven-dependency-plugin</artifactId> + <configuration> + <outputFile>${project.build.directory}/test-classes/classpath.txt</outputFile> + </configuration> + <executions> + <execution> + <id>generate-classpath</id> + <phase>generate-test-resources</phase> + <goals> + <goal>build-classpath</goal> + </goals> + </execution> + </executions> + </plugin> <plugin> <groupId>org.apache.maven.plugins</groupId> <artifactId>maven-surefire-plugin</artifactId> http://git-wip-us.apache.org/repos/asf/flume/blob/4ae19e8f/flume-ng-tests/src/test/java/org/apache/flume/test/agent/TestConfig.java ---------------------------------------------------------------------- diff --git a/flume-ng-tests/src/test/java/org/apache/flume/test/agent/TestConfig.java b/flume-ng-tests/src/test/java/org/apache/flume/test/agent/TestConfig.java index 57e720c..7a89c52 100644 --- a/flume-ng-tests/src/test/java/org/apache/flume/test/agent/TestConfig.java +++ b/flume-ng-tests/src/test/java/org/apache/flume/test/agent/TestConfig.java @@ -33,9 +33,14 @@ import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.HashMap; import java.util.HashSet; import java.util.Map; +import java.util.Objects; import java.util.Properties; import java.util.Scanner; import java.util.Set; @@ -54,6 +59,7 @@ public class TestConfig { private Properties agentProps; private Map<String, String> agentEnv; + private Map<String, String> agentOptions; private File sinkOutputDir1; private File sinkOutputDir2; private File sinkOutputDir3; @@ -73,6 +79,8 @@ public class TestConfig { agentProps = new Properties(); agentEnv = new HashMap<>(); + agentOptions = new HashMap<>(); + agentOptions.put("-C", getAdditionalClassPath()); // Create the rest of the properties file agentProps.put("agent.sources.seq-01.type", "seq"); @@ -135,6 +143,12 @@ public class TestConfig { agentProps.put("agent.configfilters", "filter-01 filter-02 filter-03"); } + private String getAdditionalClassPath() throws Exception { + URL resource = this.getClass().getClassLoader().getResource("classpath.txt"); + Path path = Paths.get(Objects.requireNonNull(resource).getPath()); + return Files.readAllLines(path).stream().findFirst().orElse(""); + } + @After public void teardown() throws Exception { StagedInstall.getInstance().stopAgent(); @@ -164,7 +178,7 @@ public class TestConfig { public void testConfigReplacement() throws Exception { LOGGER.debug("testConfigReplacement() started."); - StagedInstall.getInstance().startAgent("agent", agentProps, agentEnv); + StagedInstall.getInstance().startAgent("agent", agentProps, agentEnv, agentOptions); TimeUnit.SECONDS.sleep(10); // Wait for sources and sink to process files @@ -183,7 +197,7 @@ public class TestConfig { agentProps.put("agent.channels.mem-01.transactionCapacity", "10"); agentProps.put("agent.sinks.roll-01.sink.batchSize", "20"); - StagedInstall.getInstance().startAgent("agent", agentProps, agentEnv); + StagedInstall.getInstance().startAgent("agent", agentProps, agentEnv, agentOptions); TimeUnit.SECONDS.sleep(10); // Wait for sources and sink to process files http://git-wip-us.apache.org/repos/asf/flume/blob/4ae19e8f/flume-ng-tests/src/test/java/org/apache/flume/test/util/StagedInstall.java ---------------------------------------------------------------------- diff --git a/flume-ng-tests/src/test/java/org/apache/flume/test/util/StagedInstall.java b/flume-ng-tests/src/test/java/org/apache/flume/test/util/StagedInstall.java index 8eda1e6..e5aa0cb 100644 --- a/flume-ng-tests/src/test/java/org/apache/flume/test/util/StagedInstall.java +++ b/flume-ng-tests/src/test/java/org/apache/flume/test/util/StagedInstall.java @@ -123,11 +123,12 @@ public class StagedInstall { } public synchronized void startAgent(String name, Properties properties) throws Exception { - startAgent(name, properties, new HashMap<>()); + startAgent(name, properties, new HashMap<>(), new HashMap<>()); } public synchronized void startAgent( - String name, Properties properties, Map<String, String> environmentVariables) + String name, Properties properties, Map<String, String> environmentVariables, + Map<String, String> commandOptions) throws Exception { Preconditions.checkArgument(!name.isEmpty(), "agent name must not be empty"); Preconditions.checkNotNull(properties, "properties object must not be null"); @@ -159,6 +160,8 @@ public class StagedInstall { + ENV_FLUME_ROOT_LOGGER_VALUE); builder.add("-D" + ENV_FLUME_LOG_FILE + "=" + logFileName); + commandOptions.forEach((key, value) -> builder.add(key, value)); + List<String> cmdArgs = builder.build(); LOGGER.info("Using command: " + Joiner.on(" ").join(cmdArgs));
