[hotfix] Remove unnecessary dependencies in flink-runtime. - Removes Apache Commons HTTP Client - Removes Apache Commons IO - Removes Jettison JSONparser and consolidates tests to use Jackson instead (which is included anyways)
Project: http://git-wip-us.apache.org/repos/asf/flink/repo Commit: http://git-wip-us.apache.org/repos/asf/flink/commit/388c280f Tree: http://git-wip-us.apache.org/repos/asf/flink/tree/388c280f Diff: http://git-wip-us.apache.org/repos/asf/flink/diff/388c280f Branch: refs/heads/master Commit: 388c280fcd8157f5ebc06a9062a0d06a3d2e92cb Parents: a65cd8d Author: Stephan Ewen <se...@apache.org> Authored: Wed Jan 6 10:22:00 2016 +0100 Committer: Stephan Ewen <se...@apache.org> Committed: Fri Jan 15 11:44:20 2016 +0100 ---------------------------------------------------------------------- .../main/resources/archetype-resources/pom.xml | 11 --- .../main/resources/archetype-resources/pom.xml | 13 --- flink-runtime/pom.xml | 17 ---- .../runtime/webmonitor/WebMonitorUtils.java | 37 +++++---- .../flink/test/web/WebFrontendITCase.java | 85 ++++++++++++-------- .../flink/yarn/YARNSessionFIFOITCase.java | 29 ++++--- tools/maven/checkstyle.xml | 6 ++ 7 files changed, 99 insertions(+), 99 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/flink/blob/388c280f/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml ---------------------------------------------------------------------- diff --git a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml index 1610a3a..4baf668 100644 --- a/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml +++ b/flink-quickstart/flink-quickstart-java/src/main/resources/archetype-resources/pom.xml @@ -139,13 +139,6 @@ under the License. <exclude>com.typesafe.akka:akka-slf4j_*</exclude> <exclude>io.netty:netty-all</exclude> <exclude>io.netty:netty</exclude> - <exclude>org.eclipse.jetty:jetty-server</exclude> - <exclude>org.eclipse.jetty:jetty-continuation</exclude> - <exclude>org.eclipse.jetty:jetty-http</exclude> - <exclude>org.eclipse.jetty:jetty-io</exclude> - <exclude>org.eclipse.jetty:jetty-util</exclude> - <exclude>org.eclipse.jetty:jetty-security</exclude> - <exclude>org.eclipse.jetty:jetty-servlet</exclude> <exclude>commons-fileupload:commons-fileupload</exclude> <exclude>org.apache.avro:avro</exclude> <exclude>commons-collections:commons-collections</exclude> @@ -175,18 +168,14 @@ under the License. <exclude>org.apache.commons:commons-math</exclude> <exclude>org.apache.sling:org.apache.sling.commons.json</exclude> <exclude>commons-logging:commons-logging</exclude> - <exclude>org.apache.httpcomponents:httpclient</exclude> - <exclude>org.apache.httpcomponents:httpcore</exclude> <exclude>commons-codec:commons-codec</exclude> <exclude>com.fasterxml.jackson.core:jackson-core</exclude> <exclude>com.fasterxml.jackson.core:jackson-databind</exclude> <exclude>com.fasterxml.jackson.core:jackson-annotations</exclude> - <exclude>org.codehaus.jettison:jettison</exclude> <exclude>stax:stax-api</exclude> <exclude>com.typesafe:config</exclude> <exclude>org.uncommons.maths:uncommons-maths</exclude> <exclude>com.github.scopt:scopt_*</exclude> - <exclude>org.mortbay.jetty:servlet-api</exclude> <exclude>commons-io:commons-io</exclude> <exclude>commons-cli:commons-cli</exclude> </excludes> http://git-wip-us.apache.org/repos/asf/flink/blob/388c280f/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml ---------------------------------------------------------------------- diff --git a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml index e8c9057..560a29f 100644 --- a/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml +++ b/flink-quickstart/flink-quickstart-scala/src/main/resources/archetype-resources/pom.xml @@ -144,18 +144,9 @@ under the License. <exclude>com.typesafe.akka:akka-slf4j_*</exclude> <exclude>io.netty:netty-all</exclude> <exclude>io.netty:netty</exclude> - <exclude>org.eclipse.jetty:jetty-server</exclude> - <exclude>org.eclipse.jetty:jetty-continuation</exclude> - <exclude>org.eclipse.jetty:jetty-http</exclude> - <exclude>org.eclipse.jetty:jetty-io</exclude> - <exclude>org.eclipse.jetty:jetty-util</exclude> - <exclude>org.eclipse.jetty:jetty-security</exclude> - <exclude>org.eclipse.jetty:jetty-servlet</exclude> <exclude>commons-fileupload:commons-fileupload</exclude> <exclude>org.apache.avro:avro</exclude> <exclude>commons-collections:commons-collections</exclude> - <exclude>org.codehaus.jackson:jackson-core-asl</exclude> - <exclude>org.codehaus.jackson:jackson-mapper-asl</exclude> <exclude>com.thoughtworks.paranamer:paranamer</exclude> <exclude>org.xerial.snappy:snappy-java</exclude> <exclude>org.apache.commons:commons-compress</exclude> @@ -180,18 +171,14 @@ under the License. <exclude>org.apache.commons:commons-math</exclude> <exclude>org.apache.sling:org.apache.sling.commons.json</exclude> <exclude>commons-logging:commons-logging</exclude> - <exclude>org.apache.httpcomponents:httpclient</exclude> - <exclude>org.apache.httpcomponents:httpcore</exclude> <exclude>commons-codec:commons-codec</exclude> <exclude>com.fasterxml.jackson.core:jackson-core</exclude> <exclude>com.fasterxml.jackson.core:jackson-databind</exclude> <exclude>com.fasterxml.jackson.core:jackson-annotations</exclude> - <exclude>org.codehaus.jettison:jettison</exclude> <exclude>stax:stax-api</exclude> <exclude>com.typesafe:config</exclude> <exclude>org.uncommons.maths:uncommons-maths</exclude> <exclude>com.github.scopt:scopt_*</exclude> - <exclude>org.mortbay.jetty:servlet-api</exclude> <exclude>commons-io:commons-io</exclude> <exclude>commons-cli:commons-cli</exclude> </excludes> http://git-wip-us.apache.org/repos/asf/flink/blob/388c280f/flink-runtime/pom.xml ---------------------------------------------------------------------- diff --git a/flink-runtime/pom.xml b/flink-runtime/pom.xml index c3093da..7feace8 100644 --- a/flink-runtime/pom.xml +++ b/flink-runtime/pom.xml @@ -71,17 +71,6 @@ under the License. </dependency> <dependency> - <groupId>commons-io</groupId> - <artifactId>commons-io</artifactId> - </dependency> - - <dependency> - <groupId>org.apache.httpcomponents</groupId> - <artifactId>httpclient</artifactId> - <!-- Version is set in root POM --> - </dependency> - - <dependency> <groupId>io.netty</groupId> <artifactId>netty-all</artifactId> <!-- Version is set in root POM --> @@ -95,12 +84,6 @@ under the License. </dependency> <dependency> - <groupId>org.codehaus.jettison</groupId> - <artifactId>jettison</artifactId> - <version>1.1</version> - </dependency> - - <dependency> <groupId>com.google.guava</groupId> <artifactId>guava</artifactId> <version>${guava.version}</version> http://git-wip-us.apache.org/repos/asf/flink/blob/388c280f/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorUtils.java ---------------------------------------------------------------------- diff --git a/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorUtils.java b/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorUtils.java index 41cb83b..6d89de0 100644 --- a/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorUtils.java +++ b/flink-runtime/src/main/java/org/apache/flink/runtime/webmonitor/WebMonitorUtils.java @@ -20,6 +20,10 @@ package org.apache.flink.runtime.webmonitor; import akka.actor.ActorSystem; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; + import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.Configuration; import org.apache.flink.runtime.execution.ExecutionState; @@ -30,10 +34,6 @@ import org.apache.flink.runtime.jobgraph.JobStatus; import org.apache.flink.runtime.leaderretrieval.LeaderRetrievalService; import org.apache.flink.runtime.messages.webmonitor.JobDetails; -import org.codehaus.jettison.json.JSONArray; -import org.codehaus.jettison.json.JSONException; -import org.codehaus.jettison.json.JSONObject; - import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -41,6 +41,7 @@ import java.io.File; import java.lang.reflect.Constructor; import java.lang.reflect.InvocationTargetException; import java.util.HashMap; +import java.util.Iterator; import java.util.Map; /** @@ -142,17 +143,25 @@ public final class WebMonitorUtils { } } - public static Map<String, String> fromKeyValueJsonArray(JSONArray parsed) throws JSONException { - Map<String, String> hashMap = new HashMap<>(); - - for (int i = 0; i < parsed.length(); i++) { - JSONObject jsonObject = parsed.getJSONObject(i); - String key = jsonObject.getString("key"); - String value = jsonObject.getString("value"); - hashMap.put(key, value); + public static Map<String, String> fromKeyValueJsonArray(String jsonString) { + try { + Map<String, String> map = new HashMap<>(); + ObjectMapper m = new ObjectMapper(); + ArrayNode array = (ArrayNode) m.readTree(jsonString); + + Iterator<JsonNode> elements = array.elements(); + while (elements.hasNext()) { + JsonNode node = elements.next(); + String key = node.get("key").asText(); + String value = node.get("value").asText(); + map.put(key, value); + } + + return map; + } + catch (Exception e) { + throw new RuntimeException(e.getMessage(), e); } - - return hashMap; } public static JobDetails createDetailsForJob(ExecutionGraph job) { http://git-wip-us.apache.org/repos/asf/flink/blob/388c280f/flink-tests/src/test/java/org/apache/flink/test/web/WebFrontendITCase.java ---------------------------------------------------------------------- diff --git a/flink-tests/src/test/java/org/apache/flink/test/web/WebFrontendITCase.java b/flink-tests/src/test/java/org/apache/flink/test/web/WebFrontendITCase.java index 7720afd..30f840c 100644 --- a/flink-tests/src/test/java/org/apache/flink/test/web/WebFrontendITCase.java +++ b/flink-tests/src/test/java/org/apache/flink/test/web/WebFrontendITCase.java @@ -18,6 +18,10 @@ package org.apache.flink.test.web; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; + import org.apache.commons.io.FileUtils; import org.apache.flink.configuration.ConfigConstants; @@ -25,9 +29,6 @@ import org.apache.flink.runtime.webmonitor.WebMonitor; import org.apache.flink.runtime.webmonitor.WebMonitorUtils; import org.apache.flink.test.util.MultipleProgramsTestBase; import org.apache.flink.test.util.TestBaseUtils; -import org.codehaus.jettison.json.JSONArray; -import org.codehaus.jettison.json.JSONObject; -import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Test; @@ -38,6 +39,11 @@ import java.util.Arrays; import java.util.Collection; import java.util.Map; +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; +import static org.junit.Assert.assertTrue; +import static org.junit.Assert.fail; + @RunWith(Parameterized.class) public class WebFrontendITCase extends MultipleProgramsTestBase { @@ -69,10 +75,11 @@ public class WebFrontendITCase extends MultipleProgramsTestBase { try { String fromHTTP = TestBaseUtils.getFromHTTP("http://localhost:"; + port + "/index.html"); String text = "Apache Flink Dashboard"; - Assert.assertTrue("Startpage should contain " + text, fromHTTP.contains(text)); - } catch (Exception e) { + assertTrue("Startpage should contain " + text, fromHTTP.contains(text)); + } + catch (Exception e) { e.printStackTrace(); - Assert.fail(e.getMessage()); + fail(e.getMessage()); } } @@ -80,13 +87,17 @@ public class WebFrontendITCase extends MultipleProgramsTestBase { public void getNumberOfTaskManagers() { try { String json = TestBaseUtils.getFromHTTP("http://localhost:"; + port + "/taskmanagers/"); - JSONObject response = new JSONObject(json); - JSONArray taskManagers = response.getJSONArray("taskmanagers"); - Assert.assertNotNull(taskManagers); - Assert.assertEquals(cluster.numTaskManagers(), taskManagers.length()); - }catch(Throwable e) { + + ObjectMapper mapper = new ObjectMapper(); + JsonNode response = mapper.readTree(json); + ArrayNode taskManagers = (ArrayNode) response.get("taskmanagers"); + + assertNotNull(taskManagers); + assertEquals(cluster.numTaskManagers(), taskManagers.size()); + } + catch (Exception e) { e.printStackTrace(); - Assert.fail(e.getMessage()); + fail(e.getMessage()); } } @@ -94,16 +105,21 @@ public class WebFrontendITCase extends MultipleProgramsTestBase { public void getTaskmanagers() { try { String json = getFromHTTP("http://localhost:"; + port + "/taskmanagers/"); - JSONObject parsed = new JSONObject(json); - JSONArray taskManagers = parsed.getJSONArray("taskmanagers"); - Assert.assertNotNull(taskManagers); - Assert.assertEquals(cluster.numTaskManagers(), taskManagers.length()); - JSONObject taskManager = taskManagers.getJSONObject(0); - Assert.assertNotNull(taskManager); - Assert.assertEquals(4, taskManager.getInt("freeSlots")); - }catch(Throwable e) { + + ObjectMapper mapper = new ObjectMapper(); + JsonNode parsed = mapper.readTree(json); + ArrayNode taskManagers = (ArrayNode) parsed.get("taskmanagers"); + + assertNotNull(taskManagers); + assertEquals(cluster.numTaskManagers(), taskManagers.size()); + + JsonNode taskManager = taskManagers.get(0); + assertNotNull(taskManager); + assertEquals(4, taskManager.get("freeSlots").asInt()); + } + catch(Exception e) { e.printStackTrace(); - Assert.fail(e.getMessage()); + fail(e.getMessage()); } } @@ -114,14 +130,15 @@ public class WebFrontendITCase extends MultipleProgramsTestBase { FileUtils.writeStringToFile(logFiles.logFile, "job manager log"); String logs = getFromHTTP("http://localhost:"; + port + "/jobmanager/log"); - Assert.assertTrue(logs.contains("job manager log")); + assertTrue(logs.contains("job manager log")); FileUtils.writeStringToFile(logFiles.stdOutFile, "job manager out"); logs = getFromHTTP("http://localhost:"; + port + "/jobmanager/stdout"); - Assert.assertTrue(logs.contains("job manager out")); - } catch(Throwable e) { + assertTrue(logs.contains("job manager out")); + } + catch (Exception e) { e.printStackTrace(); - Assert.fail(e.getMessage()); + fail(e.getMessage()); } } @@ -129,16 +146,16 @@ public class WebFrontendITCase extends MultipleProgramsTestBase { public void getConfiguration() { try { String config = getFromHTTP("http://localhost:"; + port + "/jobmanager/config"); - JSONArray array = new JSONArray(config); - - Map<String, String> conf = WebMonitorUtils.fromKeyValueJsonArray(array); - Assert.assertTrue(conf.get(ConfigConstants.JOB_MANAGER_WEB_LOG_PATH_KEY).startsWith(logDir.toString())); - Assert.assertEquals( - cluster.configuration().getString("taskmanager.numberOfTaskSlots", null), - conf.get(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS)); - } catch (Throwable e) { + + Map<String, String> conf = WebMonitorUtils.fromKeyValueJsonArray(config); + assertTrue(conf.get(ConfigConstants.JOB_MANAGER_WEB_LOG_PATH_KEY).startsWith(logDir.toString())); + assertEquals( + cluster.configuration().getString("taskmanager.numberOfTaskSlots", null), + conf.get(ConfigConstants.TASK_MANAGER_NUM_TASK_SLOTS)); + } + catch (Exception e) { e.printStackTrace(); - Assert.fail(e.getMessage()); + fail(e.getMessage()); } } } http://git-wip-us.apache.org/repos/asf/flink/blob/388c280f/flink-yarn-tests/src/main/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java ---------------------------------------------------------------------- diff --git a/flink-yarn-tests/src/main/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java b/flink-yarn-tests/src/main/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java index 69264cd..8c9a9c7 100644 --- a/flink-yarn-tests/src/main/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java +++ b/flink-yarn-tests/src/main/java/org/apache/flink/yarn/YARNSessionFIFOITCase.java @@ -15,10 +15,16 @@ * See the License for the specific language governing permissions and * limitations under the License. */ + package org.apache.flink.yarn; +import com.fasterxml.jackson.databind.JsonNode; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.node.ArrayNode; import com.google.common.base.Joiner; + import org.apache.commons.io.FileUtils; + import org.apache.flink.client.FlinkYarnSessionCli; import org.apache.flink.configuration.ConfigConstants; import org.apache.flink.configuration.GlobalConfiguration; @@ -29,6 +35,7 @@ import org.apache.flink.runtime.yarn.AbstractFlinkYarnCluster; import org.apache.flink.runtime.yarn.FlinkYarnClusterStatus; import org.apache.flink.test.testdata.WordCountData; import org.apache.flink.test.util.TestBaseUtils; + import org.apache.hadoop.fs.Path; import org.apache.hadoop.security.UserGroupInformation; import org.apache.hadoop.yarn.api.protocolrecords.StopContainersRequest; @@ -43,14 +50,15 @@ import org.apache.hadoop.yarn.server.nodemanager.NodeManager; import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler; import org.apache.hadoop.yarn.server.resourcemanager.scheduler.fifo.FifoScheduler; + import org.apache.log4j.Level; -import org.codehaus.jettison.json.JSONArray; -import org.codehaus.jettison.json.JSONObject; + import org.junit.After; import org.junit.Assert; import org.junit.BeforeClass; import org.junit.Ignore; import org.junit.Test; + import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -199,16 +207,17 @@ public class YARNSessionFIFOITCase extends YarnTestBase { LOG.info("Got application URL from YARN {}", url); String response = TestBaseUtils.getFromHTTP(url + "taskmanagers/"); - JSONObject parsedTMs = new JSONObject(response); - JSONArray taskManagers = parsedTMs.getJSONArray("taskmanagers"); + + + JsonNode parsedTMs = new ObjectMapper().readTree(response); + ArrayNode taskManagers = (ArrayNode) parsedTMs.get("taskmanagers"); Assert.assertNotNull(taskManagers); - Assert.assertEquals(1, taskManagers.length()); - Assert.assertEquals(1, taskManagers.getJSONObject(0).getInt("slotsNumber")); + Assert.assertEquals(1, taskManagers.size()); + Assert.assertEquals(1, taskManagers.get(0).get("slotsNumber").asInt()); // get the configuration from webinterface & check if the dynamic properties from YARN show up there. String jsonConfig = TestBaseUtils.getFromHTTP(url + "jobmanager/config"); - JSONArray parsed = new JSONArray(jsonConfig); - Map<String, String> parsedConfig = WebMonitorUtils.fromKeyValueJsonArray(parsed); + Map<String, String> parsedConfig = WebMonitorUtils.fromKeyValueJsonArray(jsonConfig); Assert.assertEquals("veryFancy", parsedConfig.get("fancy-configuration-value")); Assert.assertEquals("3", parsedConfig.get("yarn.maximum-failed-containers")); @@ -226,9 +235,9 @@ public class YARNSessionFIFOITCase extends YarnTestBase { } LOG.info("Extracted hostname:port: {} {}", hostname, port); - Assert.assertEquals("unable to find hostname in " + parsed, hostname, + Assert.assertEquals("unable to find hostname in " + jsonConfig, hostname, parsedConfig.get(ConfigConstants.JOB_MANAGER_IPC_ADDRESS_KEY)); - Assert.assertEquals("unable to find port in " + parsed, port, + Assert.assertEquals("unable to find port in " + jsonConfig, port, parsedConfig.get(ConfigConstants.JOB_MANAGER_IPC_PORT_KEY)); // test logfile access http://git-wip-us.apache.org/repos/asf/flink/blob/388c280f/tools/maven/checkstyle.xml ---------------------------------------------------------------------- diff --git a/tools/maven/checkstyle.xml b/tools/maven/checkstyle.xml index a558116..d5ee277 100644 --- a/tools/maven/checkstyle.xml +++ b/tools/maven/checkstyle.xml @@ -76,6 +76,12 @@ under the License. <property name="illegalPattern" value="true"/> <property name="message" value="Use commons-lang3 instead of commons-lang."/> </module> + <module name="Regexp"> + <property name="format" value="org\.codehaus\.jettison"/> + <property name="illegalPattern" value="true"/> + <property name="message" value="Use com.fasterxml.jackson instead of jettison."/> + </module> + <module name="NeedBraces"> <property name="tokens" value="LITERAL_IF, LITERAL_ELSE"/> </module>
