Bring the master upto date with release-3.1.0
Project: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/repo Commit: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/commit/d7c8964b Tree: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/tree/d7c8964b Diff: http://git-wip-us.apache.org/repos/asf/incubator-apex-core/diff/d7c8964b Branch: refs/heads/master Commit: d7c8964b7983fc3a8df38fbc85ec5291449d8960 Parents: b7475a0 03f612f Author: Chetan Narsude <[email protected]> Authored: Wed Sep 9 15:03:26 2015 -0700 Committer: Chetan Narsude <[email protected]> Committed: Wed Sep 9 15:11:34 2015 -0700 ---------------------------------------------------------------------- CHANGELOG.md | 115 +++++ RELEASE.md | 71 +++ apex-app-archetype/README.md | 2 +- apex-app-archetype/pom.xml | 2 +- .../projects/basic/archetype.properties | 2 +- apex-conf-archetype/README.md | 2 +- apex-conf-archetype/pom.xml | 2 +- .../projects/basic/archetype.properties | 2 +- api/pom.xml | 44 +- .../java/com/datatorrent/api/AutoMetric.java | 11 +- .../main/java/com/datatorrent/api/Context.java | 7 + .../annotation/InputPortFieldAnnotation.java | 10 +- .../annotation/OutputPortFieldAnnotation.java | 10 + bufferserver/pom.xml | 4 +- .../datatorrent/bufferserver/server/Server.java | 2 +- .../datatorrent/bufferserver/util/System.java | 2 +- .../bufferserver/client/SubscriberTest.java | 4 +- .../bufferserver/server/ServerTest.java | 4 +- .../bufferserver/storage/DiskStorageTest.java | 9 +- common/pom.xml | 48 +- .../common/util/AsyncFSStorageAgent.java | 143 ++++++ .../datatorrent/common/util/BaseOperator.java | 2 + .../datatorrent/common/util/FSStorageAgent.java | 5 +- .../common/codec/JsonStreamCodecTest.java | 15 +- .../common/util/AsyncFSStorageAgentTest.java | 133 ++++++ engine/pom.xml | 12 +- .../java/com/datatorrent/stram/StramClient.java | 8 +- .../datatorrent/stram/StramLocalCluster.java | 4 +- .../stram/StreamingAppMasterService.java | 2 +- .../stram/StreamingContainerManager.java | 170 +++++--- .../java/com/datatorrent/stram/cli/DTCli.java | 34 +- .../java/com/datatorrent/stram/engine/Node.java | 64 ++- .../stram/engine/StreamingContainer.java | 8 +- .../stram/plan/logical/LogicalPlan.java | 30 +- .../plan/logical/LogicalPlanConfiguration.java | 22 +- .../stram/plan/physical/PhysicalPlan.java | 40 +- .../datatorrent/stram/stream/FastPublisher.java | 2 + .../datatorrent/stram/util/FSJsonLineFile.java | 24 +- .../stram/webapp/OperatorDiscoverer.java | 434 +++++++++++-------- .../stram/webapp/StramWebServices.java | 8 +- .../com/datatorrent/stram/webapp/TypeGraph.java | 119 ++++- .../com/datatorrent/stram/CheckpointTest.java | 11 +- .../stram/LogicalPlanModificationTest.java | 22 +- .../com/datatorrent/stram/MockContainer.java | 2 +- .../com/datatorrent/stram/PartitioningTest.java | 26 +- .../stram/StramLocalClusterTest.java | 22 +- .../datatorrent/stram/StramMiniClusterTest.java | 9 +- .../datatorrent/stram/StramRecoveryTest.java | 58 ++- .../com/datatorrent/stram/StreamCodecTest.java | 80 +--- .../stram/StreamingContainerManagerTest.java | 139 +++++- .../stram/client/AppPackageTest.java | 2 +- .../stram/debug/TupleRecorderTest.java | 3 + .../stram/engine/AtLeastOnceTest.java | 16 + .../stram/engine/AtMostOnceTest.java | 2 +- .../stram/engine/AutoMetricTest.java | 12 +- .../stram/engine/InputOperatorTest.java | 5 +- .../stram/engine/ProcessingModeTests.java | 11 +- .../stram/engine/RecoverableInputOperator.java | 10 +- .../datatorrent/stram/engine/SliderTest.java | 5 + .../com/datatorrent/stram/engine/StatsTest.java | 10 +- .../stram/engine/StreamingContainerTest.java | 6 + .../stram/engine/WindowGeneratorTest.java | 11 +- .../plan/LogicalPlanConfigurationTest.java | 83 +++- .../datatorrent/stram/plan/LogicalPlanTest.java | 16 +- .../stram/plan/SchemaTestOperator.java | 33 ++ .../stram/plan/physical/PhysicalPlanTest.java | 9 +- .../stram/stream/FastStreamTest.java | 6 +- .../stram/stream/OiOEndWindowTest.java | 5 + .../stram/stream/SocketStreamTest.java | 8 +- .../stram/support/StramTestSupport.java | 7 +- .../stram/webapp/OperatorDiscoveryTest.java | 163 +++++-- .../stram/webapp/StramWebServicesTest.java | 6 +- .../src/test/resources/schemaTestTopology.json | 43 ++ .../resources/testAppPackage/mydtapp/pom.xml | 2 +- pom.xml | 2 +- 75 files changed, 1886 insertions(+), 591 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/d7c8964b/CHANGELOG.md ---------------------------------------------------------------------- diff --cc CHANGELOG.md index 141ff79,0158919..0f06d2d --- a/CHANGELOG.md +++ b/CHANGELOG.md @@@ -1,11 -1,27 +1,126 @@@ Apex Changelog ======================================================================================================================== +Version 3.0.0 +------------------------------------------------------------------------------------------------------------------------ + +* Add jersey client jar for app execution +* Must catch NoClassDefFoundError when processing operator classes in jar, previously catching Throwable was changed to catching Exception ++* Do not catch throwable in DTCli and exit immediately when worker thread throws an error ++* Depend on published netlet version ++* Catch throwable when executing command because launching an app can throw java.lang.VerifyError: bad type on operand stack ++* Removed runOnlyOnce when generating dt-git.properties and generate even when not using release profile ++* Undeploy heartbeat requests are not processes if container is idle ++* Fix potential NPE ++* Comment hide the actually type for string types(URI, URL, Class etc) and add 2 missing wrapper types ++* Fixed typo in webservice url for get-physical-plan command ++* Resolve deleting checkpoint in different thread ++* Removed duplicate code and added unit test for json stream codec ++* APEX-11 #resolve added checkpoint metric ++* Have default connect and read timeouts because the jersey defaults are infinity and that blocks threads and those threads can't be interrupted either ++* Removed invalid app data push annotation ++* Use FileContext instead of FileSystem to write the meta json file ++* Comment added required memory and required vcores in the appinfo ++* Comment filter abstract types from getAssignableClasses call and rename initializable to instantiable ++* Deploy all artifacts by default. ++* Comment fix the bug in trimming the graph ++* HA support for stram webservice filter. ++* Removed dependencies in filter to hadoop classes with private audience as their interface has changed from Hadoop 2.2 to 2.6 ++* Related doc updates ++* Comment Prune the external dependencies from type graph and break the type circle ++* Fixing class not found issue due to missing dt-common jar. ++* Resolve removed old alert support in stram ++* Use tokens from Credentials as UserGroupInformation.getTokens() returns HDFS Namenode hidden tokens that should not be passed to containers. ++* Support for RM delegation token renewal in secure HA environments ++* Resolve fixed bug when custom metric keys are not stored correctly in meta file ++* Comment Use apache xbean shaded asm to resolve jdk 8 class and avoid conflict with old asm version that hadoop depends on ++* APEX-5 #resolve Set APPLICATION_PATH for unit tests that launch an app ++* Resolve Added dependency on dt-engine in test scope of archetype ++* Corrected the place to get pom.properties for version info ++* Token authentication support for buffer server ++* Adding default aggregator for primitive customMetrics ++* Netlet has a few fixed related to non firing OP_CONNECT call, so test against those. ++* Print stack trace when exception occurs ++* Commented out app master mb in properties.xml ++* Ability to extract javadocs as xml ++* Moved PAYLOAD_VALUE out of default case. Count number of skipped payload tuples. ++* Resolve fixing ASM signature visitor to add upper bound to Object for unbounded wild card entries ++* Removing unused imports ++* Addressing review comments ++* Reverting code format change ++* Resolve Added END_STREAM to be distributed to physical nodes. Introduced debug message for default case. ++* Comment removed the coding convention ++* Resolve Changed reading of portTypeInfo, port annotations in operator discoverer to read from ASM ++* Added test case for operator discoverer ++* Updated Type Discovery tests to getPortInfo via ASM instead of reflection API ++* Addressing review comments ++* Removed condition for public, final, transient ports when generating typeGraph using ASM ++* Added a class for storing input, output port annotations ++* Added port type info handling for ports other than DefaultInputPort and DefaultOutputPort types ++* Updated a test case for the same ++* Adding annotations to list.. Missed in previous commit ++* Added rescan of typegraph to capture operator ports ++* Changing ASMUtil utility methods to public ++* Addressing review comments. ++* Correcting jar entry name ++* Updated a test after merge ++* Removed system.out print from test cases and added debug print on failure ++* Removing temp file ++* Reversed the order of setting jvm_options for app master ++* Resolve Added validation for root operator should be input operator in logical plan validation ++* Fixing formatting and added lincense header ++* Removed extra line spaces ++* Correcting output port in dt-site.xml stream connection ++* Resolve Added validation for root operator should be input operator in logical plan validation ++* Fixing formatting and added lincense header ++* Removed extra line spaces ++* Correcting output port in dt-site.xml stream connection ++* Removing white spaces ++* Renaming the properties to reflect the code change ++* Comment add the missing open tag <p> ++* Removed unnecessary depedency ++* Resolve Remove unnecessary unpack in app package archetype ++* Resolve support java.lang.Class and resolve uiType to special types ++* Resolve update dependency to dt-common in archetype ++* Resolve added the container jvm options for app master ++* Resolve attach apa file as an artifact ++* Resolve test app package should depend on dt-common instead of dt-engine ++* Removed additional license header ++* Removed incorrectly left in license headers due to incorrect formatting ++* Put back revision info. ++* Added a TODO note for handling of stram delegation tokens for future ++* Comment Add dag attributes to LogicalPlanSerializer ++* Using resolveType method in OperatorDiscoverer to describe an attribute completely ++* Switch to Java7 and update compiler plugin. ++* Remove invalid module references. ++* Separated out HA token creation from non-HA case as it involves special handling that is subject to change if Hadoop's internals change. ++* Resolve - Add all resource managers to RM delegation token service. ++* Fix CLI script. ++* Stram directory is moved to engine ++* Fix depenency plugin version (2.3 shipped with maven 3.2.5 pulls dependencies from test scope) ++* Setup the pom files for OS with optimized dependencies ++* Changed the license header to Apache 2.0 license. + + Version 3.1.0 + ------------------------------------------------------------------------------------------------------------------------ + + ** Improvement + * [APEX-12] - Fix Base Operator To Not Show Name Property In App Builder + + ** Bug + * [APEX-35] - Test exceptions due to missing directory in saveMetaInfo + * [APEX-36] - FSStorageAgent to account for HDFS lease when writing checkpoint files + * [APEX-37] - Container and operator json line file in StreamingContainerManager should not be appended from previous app attempt + * [APEX-43] - SchemaSupport: TUPLE_CLASS attribute should use Class2String StringCodec + * [APEX-56] - Controlled plan modification on operator shutdown + + + + Version 3.0.0 + ------------------------------------------------------------------------------------------------------------------------ + + * Add jersey client jar for app execution + * Must catch NoClassDefFoundError when processing operator classes in jar, previously catching Throwable was changed to catching Exception * Do not catch throwable in DTCli and exit immediately when worker thread throws an error * Depend on published netlet version * Catch throwable when executing command because launching an app can throw java.lang.VerifyError: bad type on operand stack http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/d7c8964b/engine/src/main/java/com/datatorrent/stram/StreamingAppMasterService.java ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/incubator-apex-core/blob/d7c8964b/engine/src/main/java/com/datatorrent/stram/StreamingContainerManager.java ---------------------------------------------------------------------- diff --cc engine/src/main/java/com/datatorrent/stram/StreamingContainerManager.java index 95f4648,7002c1d..a9502b7 --- a/engine/src/main/java/com/datatorrent/stram/StreamingContainerManager.java +++ b/engine/src/main/java/com/datatorrent/stram/StreamingContainerManager.java @@@ -494,12 -496,10 +496,13 @@@ public class StreamingContainerManager } IOUtils.closeQuietly(containerFile); - for (FSJsonLineFile operatorFile : operatorFiles.values()) { - IOUtils.closeQuietly(operatorFile); + IOUtils.closeQuietly(operatorFile); + if(poolExecutor != null) { + poolExecutor.shutdown(); } + if(poolExecutor != null) { + poolExecutor.shutdown(); + } } public void subscribeToEvents(Object listener) @@@ -2246,11 -2265,11 +2268,15 @@@ oi.currentWindowId = toWsWindowId(os.currentWindowId.get()); if (os.lastHeartbeat != null) { oi.lastHeartbeat = os.lastHeartbeat.getGeneratedTms(); + } + if (os.checkpointStats != null) { + oi.checkpointTime = os.checkpointStats.checkpointTime; + oi.checkpointStartTime = os.checkpointStats.checkpointStartTime; } + if (os.checkpointStats != null) { + oi.checkpointTime = os.checkpointStats.checkpointTime; + oi.checkpointStartTime = os.checkpointStats.checkpointStartTime; + } oi.checkpointTimeMA = os.checkpointTimeMA.getAvg(); for (PortStatus ps : os.inputPortStatusList.values()) { PortInfo pinfo = new PortInfo();
