[incubator-heron] branch master updated: add downloader-core filegroup (#3439)
This is an automated email from the ASF dual-hosted git repository.
nlu90 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
The following commit(s) were added to refs/heads/master by this push:
new 7927299 add downloader-core filegroup (#3439)
7927299 is described below
commit 79272997a39086a5a78d9fea7450b9aa85272aa1
Author: Neng Lu
AuthorDate: Fri Jan 31 10:57:50 2020 -0800
add downloader-core filegroup (#3439)
* add downloader-core filegroup
* remove failing test
---
heron/downloaders/src/java/BUILD | 7 +++
.../src/python/topology_test_runner/resources/test.json| 7 ---
2 files changed, 7 insertions(+), 7 deletions(-)
diff --git a/heron/downloaders/src/java/BUILD b/heron/downloaders/src/java/BUILD
index 67a5cd7..865463f 100644
--- a/heron/downloaders/src/java/BUILD
+++ b/heron/downloaders/src/java/BUILD
@@ -27,6 +27,13 @@ filegroup(
srcs = glob(["**/DownloadRunner.java"]),
)
+filegroup(
+ name = "downloader-core",
+ srcs = glob(["**/Downloader.java",
+ "**/Extractor.java",
+ "**/Registry.java"])
+)
+
java_binary(
name = 'heron-downloader-unshaded',
srcs = [":downloader-main"],
diff --git
a/integration_test/src/python/topology_test_runner/resources/test.json
b/integration_test/src/python/topology_test_runner/resources/test.json
index 3682aa1..b9b276d 100644
--- a/integration_test/src/python/topology_test_runner/resources/test.json
+++ b/integration_test/src/python/topology_test_runner/resources/test.json
@@ -47,13 +47,6 @@
"restartArgs" : "True",
"expectedTopoResultRelativePath" :
"fields_grouping/FieldsGroupingResults.json",
"checkType" : "topology_structure"
-},
-{
- "topologyName" : "IntegrationTopologyTest_StatefulBasicTopologyOneTask",
- "classPath":
"stateful_basic_topology_one_task.StatefulBasicTopologyOneTask",
- "expectedTopoResultRelativePath" :
"stateful_basic_topology_one_task/StatefulBasicTopologyOneTaskTopo.json",
- "expectedStateResultRelativePath" :
"stateful_basic_topology_one_task/StatefulBasicTopologyOneTaskState.json",
- "checkType" : "checkpoint_state"
}
]
}
[incubator-heron] branch neng/downloader-deps created (now 2fb828b)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a change to branch neng/downloader-deps in repository https://gitbox.apache.org/repos/asf/incubator-heron.git. at 2fb828b add downloader-core filegroup This branch includes the following new commits: new 2fb828b add downloader-core filegroup The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[incubator-heron] branch master updated: Upgrade to Python3 for the whole repo (#3431)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-heron.git The following commit(s) were added to refs/heads/master by this push: new 257b1a4 Upgrade to Python3 for the whole repo (#3431) 257b1a4 is described below commit 257b1a439588a22b2d7959338ce7be50b02c4708 Author: Neng Lu AuthorDate: Fri Jan 10 09:25:13 2020 -0800 Upgrade to Python3 for the whole repo (#3431) * migrate to python3 * update README * update travis --- .travis.yml | 8 +++- README.md | 4 ++-- bazel_configure.py| 4 ++-- config/configure.ac | 4 ++-- examples/src/python/bolt/consume_bolt.py | 2 +- examples/src/python/bolt/count_bolt.py| 2 +- examples/src/python/bolt/half_ack_bolt.py | 2 +- examples/src/python/bolt/stateful_count_bolt.py | 2 +- examples/src/python/bolt/stream_aggregate_bolt.py | 2 +- examples/src/python/bolt/window_size_bolt.py | 2 +- examples/src/python/custom_grouping_topology.py | 2 +- examples/src/python/half_acking_topology.py | 2 +- examples/src/python/join_streamlet_topology.py| 2 +- examples/src/python/misc/test_task_hook.py| 2 +- examples/src/python/multi_stream_topology.py | 2 +- examples/src/python/pulsar_word_count_streamlet.py| 2 +- examples/src/python/spout/multi_stream_spout.py | 2 +- examples/src/python/spout/stateful_word_spout.py | 2 +- examples/src/python/spout/word_spout.py | 2 +- examples/src/python/stateful_word_count_topology.py | 2 +- examples/src/python/window_size_topology.py | 2 +- examples/src/python/word_count_streamlet.py | 2 +- examples/src/python/word_count_topology.py| 2 +- heron/common/src/python/pex_loader.py | 2 +- heron/common/src/python/utils/log.py | 2 +- heron/common/src/python/utils/proc.py | 2 +- heron/common/tests/python/pex_loader/constants.py | 2 +- heron/common/tests/python/pex_loader/pex_loader_unittest.py | 2 +- heron/common/tests/python/pex_loader/testdata/src/sample.py | 2 +- heron/executor/src/python/heron_executor.py | 2 +- heron/executor/tests/python/heron_executor_unittest.py| 2 +- heron/instance/src/python/basics/base_instance.py | 2 +- heron/instance/src/python/basics/bolt_instance.py | 2 +- heron/instance/src/python/basics/spout_instance.py| 2 +- heron/instance/src/python/instance/st_heron_instance.py | 2 +- heron/instance/src/python/network/event_looper.py | 2 +- heron/instance/src/python/network/gateway_looper.py | 2 +- heron/instance/src/python/network/heron_client.py | 2 +- heron/instance/src/python/network/metricsmgr_client.py| 2 +- heron/instance/src/python/network/protocol.py | 2 +- heron/instance/src/python/network/socket_options.py | 2 +- heron/instance/src/python/network/st_stmgr_client.py | 2 +- heron/instance/src/python/utils/metrics/metrics_helper.py | 2 +- heron/instance/src/python/utils/metrics/py_metrics.py | 2 +- heron/instance/src/python/utils/misc/communicator.py | 2 +- heron/instance/src/python/utils/misc/custom_grouping_helper.py| 2 +- heron/instance/src/python/utils/misc/outgoing_tuple_helper.py | 2 +- heron/instance/src/python/utils/misc/pplan_helper.py | 2 +- heron/instance/src/python/utils/misc/serializer_helper.py | 2 +- heron/instance/src/python/utils/system_config.py | 2 +- heron/instance/src/python/utils/system_constants.py | 2 +- heron/instance/src/python/utils/topology/topology_context_impl.py | 2 +- heron/instance/src/python/utils/tuple.py | 2 +- heron/instance/tests/python/mock_protobuf.py | 2 +- heron/instance/tests/python/network/event_looper_unittest.py | 2 +- heron/instance/tests/python/network/gateway_looper_unittest.py| 2 +- heron/instance/tests/python/network/heron_client_unittest.py | 2 +- heron/instance/tests/python/network/metricsmgr_client_unittest.py | 2 +- heron/instance/tests/python/network/mock_generator.py | 2 +- heron/instance/tests/python
[incubator-heron] branch neng/python3 updated: update README
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a commit to branch neng/python3 in repository https://gitbox.apache.org/repos/asf/incubator-heron.git The following commit(s) were added to refs/heads/neng/python3 by this push: new ab9c205 update README ab9c205 is described below commit ab9c205a783d6e8b43144560080f2d75b9354145 Author: Neng Lu AuthorDate: Wed Jan 8 11:43:06 2020 -0800 update README --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 85a7b61..1fcc3d9 100644 --- a/README.md +++ b/README.md @@ -30,7 +30,7 @@ Heron is realtime analytics platform developed by Twitter. It has a wide array Heron Requirements: * Java JDK 1.8 - * Python 2.7 + * Python 3 * Bazel 0.26.0 ## Contact @@ -60,4 +60,4 @@ Heron is realtime analytics platform developed by Twitter. It has a wide array ## License -Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0 \ No newline at end of file +Licensed under the Apache License, Version 2.0: http://www.apache.org/licenses/LICENSE-2.0
[incubator-heron] 01/01: migrate to python3
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a commit to branch neng/python3 in repository https://gitbox.apache.org/repos/asf/incubator-heron.git commit e9f9f5e8ab14f249c7c757c61943b1736484913d Author: Neng Lu AuthorDate: Wed Jan 8 11:40:59 2020 -0800 migrate to python3 --- bazel_configure.py| 4 ++-- config/configure.ac | 4 ++-- examples/src/python/bolt/consume_bolt.py | 2 +- examples/src/python/bolt/count_bolt.py| 2 +- examples/src/python/bolt/half_ack_bolt.py | 2 +- examples/src/python/bolt/stateful_count_bolt.py | 2 +- examples/src/python/bolt/stream_aggregate_bolt.py | 2 +- examples/src/python/bolt/window_size_bolt.py | 2 +- examples/src/python/custom_grouping_topology.py | 2 +- examples/src/python/half_acking_topology.py | 2 +- examples/src/python/join_streamlet_topology.py| 2 +- examples/src/python/misc/test_task_hook.py| 2 +- examples/src/python/multi_stream_topology.py | 2 +- examples/src/python/pulsar_word_count_streamlet.py| 2 +- examples/src/python/spout/multi_stream_spout.py | 2 +- examples/src/python/spout/stateful_word_spout.py | 2 +- examples/src/python/spout/word_spout.py | 2 +- examples/src/python/stateful_word_count_topology.py | 2 +- examples/src/python/window_size_topology.py | 2 +- examples/src/python/word_count_streamlet.py | 2 +- examples/src/python/word_count_topology.py| 2 +- heron/common/src/python/pex_loader.py | 2 +- heron/common/src/python/utils/log.py | 2 +- heron/common/src/python/utils/proc.py | 2 +- heron/common/tests/python/pex_loader/constants.py | 2 +- heron/common/tests/python/pex_loader/pex_loader_unittest.py | 2 +- heron/common/tests/python/pex_loader/testdata/src/sample.py | 2 +- heron/executor/src/python/heron_executor.py | 2 +- heron/executor/tests/python/heron_executor_unittest.py| 2 +- heron/instance/src/python/basics/base_instance.py | 2 +- heron/instance/src/python/basics/bolt_instance.py | 2 +- heron/instance/src/python/basics/spout_instance.py| 2 +- heron/instance/src/python/instance/st_heron_instance.py | 2 +- heron/instance/src/python/network/event_looper.py | 2 +- heron/instance/src/python/network/gateway_looper.py | 2 +- heron/instance/src/python/network/heron_client.py | 2 +- heron/instance/src/python/network/metricsmgr_client.py| 2 +- heron/instance/src/python/network/protocol.py | 2 +- heron/instance/src/python/network/socket_options.py | 2 +- heron/instance/src/python/network/st_stmgr_client.py | 2 +- heron/instance/src/python/utils/metrics/metrics_helper.py | 2 +- heron/instance/src/python/utils/metrics/py_metrics.py | 2 +- heron/instance/src/python/utils/misc/communicator.py | 2 +- heron/instance/src/python/utils/misc/custom_grouping_helper.py| 2 +- heron/instance/src/python/utils/misc/outgoing_tuple_helper.py | 2 +- heron/instance/src/python/utils/misc/pplan_helper.py | 2 +- heron/instance/src/python/utils/misc/serializer_helper.py | 2 +- heron/instance/src/python/utils/system_config.py | 2 +- heron/instance/src/python/utils/system_constants.py | 2 +- heron/instance/src/python/utils/topology/topology_context_impl.py | 2 +- heron/instance/src/python/utils/tuple.py | 2 +- heron/instance/tests/python/mock_protobuf.py | 2 +- heron/instance/tests/python/network/event_looper_unittest.py | 2 +- heron/instance/tests/python/network/gateway_looper_unittest.py| 2 +- heron/instance/tests/python/network/heron_client_unittest.py | 2 +- heron/instance/tests/python/network/metricsmgr_client_unittest.py | 2 +- heron/instance/tests/python/network/mock_generator.py | 2 +- heron/instance/tests/python/network/mock_generator_client.py | 2 +- heron/instance/tests/python/network/protocol_unittest.py | 2 +- heron/instance/tests/python/network/st_stmgr_client_unittest.py
[incubator-heron] branch neng/python3 created (now e9f9f5e)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a change to branch neng/python3 in repository https://gitbox.apache.org/repos/asf/incubator-heron.git. at e9f9f5e migrate to python3 This branch includes the following new commits: new e9f9f5e migrate to python3 The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[incubator-heron] 01/01: travis config
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a commit to branch neng/py3-upgrade in repository https://gitbox.apache.org/repos/asf/incubator-heron.git commit 0248d08928690a1f856abaee10631381a493f892 Author: Neng Lu AuthorDate: Mon Jan 6 15:42:27 2020 -0800 travis config --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index b6847ef..6832a36 100644 --- a/.travis.yml +++ b/.travis.yml @@ -19,7 +19,7 @@ addons: - g++-4.8 - gcc-4.8-multilib - g++-4.8-multilib - - python2.7 + - python3 - wget - pkg-config - zip @@ -48,6 +48,6 @@ script: - g++ --version - which python - python -V - - which python2.7 + - which python3 - python2.7 -V - scripts/travis/ci.sh
[incubator-heron] branch neng/py3-upgrade created (now 0248d08)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a change to branch neng/py3-upgrade in repository https://gitbox.apache.org/repos/asf/incubator-heron.git. at 0248d08 travis config This branch includes the following new commits: new 0248d08 travis config The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[incubator-heron] branch master updated: make python code compatible with both python2 and python3 (#3412)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-heron.git The following commit(s) were added to refs/heads/master by this push: new 869c82e make python code compatible with both python2 and python3 (#3412) 869c82e is described below commit 869c82e84183ce5fddb7acafe29eb76b168d0ea6 Author: Neng Lu AuthorDate: Fri Dec 13 12:13:31 2019 -0800 make python code compatible with both python2 and python3 (#3412) * make code compatible with both python2 and 3 --- examples/src/python/bolt/stateful_count_bolt.py| 2 +- examples/src/python/spout/stateful_word_spout.py | 2 +- heron/executor/src/python/heron_executor.py| 39 --- .../tests/python/heron_executor_unittest.py| 44 - heron/instance/src/python/BUILD| 1 + heron/instance/src/python/basics/__init__.py | 4 +- heron/instance/src/python/basics/bolt_instance.py | 4 +- heron/instance/src/python/basics/spout_instance.py | 4 +- .../instance/src/python/network/gateway_looper.py | 4 +- .../src/python/utils/metrics/metrics_helper.py | 4 +- .../instance/src/python/utils/misc/communicator.py | 12 ++--- .../python/utils/misc/custom_grouping_helper.py| 2 +- heron/instance/src/python/utils/system_config.py | 2 +- .../python/utils/topology/topology_context_impl.py | 2 +- heron/instance/tests/python/BUILD | 1 + heron/instance/tests/python/mock_protobuf.py | 2 +- .../tests/python/utils/communicator_unittest.py| 4 +- .../tests/python/utils/metrics_helper_unittest.py | 2 +- .../instance/tests/python/utils/mock_generator.py | 4 +- heron/shell/src/python/BUILD | 1 + heron/shell/src/python/handlers/__init__.py| 24 +- .../src/python/handlers/killexecutorhandler.py | 7 ++- heron/shell/src/python/utils.py| 2 +- heron/statemgrs/src/python/config.py | 4 +- heron/statemgrs/src/python/filestatemanager.py | 11 ++--- heron/statemgrs/src/python/statemanager.py | 6 +-- heron/statemgrs/src/python/zkstatemanager.py | 2 +- .../tests/python/configloader_unittest.py | 14 +++--- .../tests/python/statemanagerfactory_unittest.py | 2 +- heron/tools/admin/src/python/main.py | 2 +- heron/tools/admin/src/python/standalone.py | 15 +++--- heron/tools/cli/src/python/BUILD | 3 +- heron/tools/cli/src/python/cli_helper.py | 2 +- heron/tools/cli/src/python/config.py | 2 +- heron/tools/cli/src/python/main.py | 2 +- heron/tools/cli/src/python/opts.py | 2 +- heron/tools/cli/src/python/submit.py | 7 ++- heron/tools/cli/src/python/version.py | 2 +- heron/tools/common/src/python/access/__init__.py | 4 +- heron/tools/common/src/python/access/heron_api.py | 16 +++ .../common/src/python/access/tracker_access.py | 2 +- heron/tools/common/src/python/utils/config.py | 4 +- heron/tools/explorer/src/python/logicalplan.py | 8 ++-- heron/tools/explorer/src/python/physicalplan.py| 16 +++ heron/tools/explorer/src/python/topologies.py | 4 +- heron/tools/tracker/src/python/BUILD | 1 + heron/tools/tracker/src/python/config.py | 4 +- .../tools/tracker/src/python/handlers/__init__.py | 56 +++--- .../src/python/handlers/logicalplanhandler.py | 4 +- .../src/python/handlers/runtimestatehandler.py | 2 +- heron/tools/tracker/src/python/javaobj.py | 11 +++-- heron/tools/tracker/src/python/main.py | 3 +- heron/tools/tracker/src/python/query_operators.py | 54 ++--- heron/tools/tracker/src/python/topology.py | 8 ++-- heron/tools/tracker/src/python/tracker.py | 14 +++--- .../tools/tracker/tests/python/tracker_unittest.py | 10 ++-- heron/tools/ui/src/python/args.py | 2 +- heron/tools/ui/src/python/handlers/api/topology.py | 12 ++--- heron/tools/ui/src/python/handlers/mainhandler.py | 2 +- heron/tools/ui/src/python/handlers/ranges.py | 2 +- heron/tools/ui/src/python/handlers/topology.py | 4 +- heron/tools/ui/src/python/main.py | 2 +- heronpy/api/cloudpickle.py | 12 ++--- heronpy/api/component/component_spec.py| 10 ++-- heronpy/api/metrics.py | 4 +- heronpy/api/tests/python/component_unittest.py | 2 +- heronpy/api/tests/python/metrics_unittest.py | 8 ++-- heronpy/api/topology.py| 10 ++-- heronpy/streamlet/impl/contextimpl.py | 2 +- heronpy/streamlet/impl/joinbolt.py | 2 +- heronpy/streamlet/impl/reducebykeyandwindowbolt.py | 2
[incubator-heron] branch neng/py3-migration updated (6f7c85b -> 6074ec4)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a change to branch neng/py3-migration in repository https://gitbox.apache.org/repos/asf/incubator-heron.git. from 6f7c85b update reqs add 6074ec4 fix No new revisions were added by this update. Summary of changes: integration_test/src/python/topology_test_runner/BUILD | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-)
[incubator-heron] branch neng/py3-migration updated (6634818 -> 6f7c85b)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a change to branch neng/py3-migration in repository https://gitbox.apache.org/repos/asf/incubator-heron.git. from 6634818 fix compatibility issue add 6f7c85b update reqs No new revisions were added by this update. Summary of changes: integration_test/src/python/topology_test_runner/BUILD | 1 + 1 file changed, 1 insertion(+)
[incubator-heron] branch neng/py3-migration updated (c58e9ee -> 6634818)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a change to branch neng/py3-migration in repository https://gitbox.apache.org/repos/asf/incubator-heron.git. from c58e9ee add future reqs add 6634818 fix compatibility issue No new revisions were added by this update. Summary of changes: heron/tools/tracker/src/python/BUILD | 1 + heron/tools/tracker/src/python/javaobj.py | 9 + integration_test/src/python/test_runner/BUILD | 5 - 3 files changed, 10 insertions(+), 5 deletions(-)
[incubator-heron] branch neng/py3-migration updated (77ce04d -> c58e9ee)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a change to branch neng/py3-migration in repository https://gitbox.apache.org/repos/asf/incubator-heron.git. from 77ce04d cleanup add c58e9ee add future reqs No new revisions were added by this update. Summary of changes: heron/shell/src/python/BUILD| 1 + integration_test/src/python/integration_test/core/BUILD | 1 + integration_test/src/python/local_test_runner/BUILD | 5 - 3 files changed, 6 insertions(+), 1 deletion(-)
[incubator-heron] branch neng/py3-migration updated (486476e -> 77ce04d)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a change to branch neng/py3-migration in repository https://gitbox.apache.org/repos/asf/incubator-heron.git. from 486476e fix unit tests add 77ce04d cleanup No new revisions were added by this update. Summary of changes: heron/executor/src/python/heron_executor.py| 2 +- heron/shell/src/python/main.py | 2 +- heron/statemgrs/src/python/configloader.py | 2 +- heron/tools/admin/src/python/main.py | 2 +- heron/tools/cli/src/python/main.py | 2 +- heron/tools/explorer/src/python/main.py| 2 +- heron/tools/tracker/src/python/main.py | 4 ++-- heron/tools/ui/src/python/main.py | 2 +- heronpy/api/serializer.py | 2 +- integration_test/src/python/common/status.py | 2 +- .../src/python/integration_test/core/aggregator_bolt.py| 3 +++ integration_test/src/python/local_test_runner/main.py | 2 +- integration_test/src/python/local_test_runner/test_template.py | 7 +-- 13 files changed, 20 insertions(+), 14 deletions(-)
[incubator-heron] branch neng/py3-migration updated (c7e1e48 -> 486476e)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a change to branch neng/py3-migration in repository https://gitbox.apache.org/repos/asf/incubator-heron.git. from c7e1e48 update cpplint.py add 486476e fix unit tests No new revisions were added by this update. Summary of changes: heron/instance/src/python/BUILD| 1 + heron/instance/src/python/utils/misc/communicator.py | 12 ++-- heron/instance/tests/python/BUILD | 1 + heron/instance/tests/python/utils/communicator_unittest.py | 4 ++-- heron/instance/tests/python/utils/log_unittest.py | 8 heron/tools/cli/src/python/BUILD | 3 ++- heronpy/api/cloudpickle.py | 14 +++--- 7 files changed, 23 insertions(+), 20 deletions(-)
[incubator-heron] branch neng/py3-migration updated (b28ce71 -> c7e1e48)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a change to branch neng/py3-migration in repository https://gitbox.apache.org/repos/asf/incubator-heron.git. from b28ce71 update cpplint.py add c7e1e48 update cpplint.py No new revisions were added by this update. Summary of changes: third_party/python/cpplint/cpplint.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-)
[incubator-heron] branch neng/py3-migration updated (b316617 -> b28ce71)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a change to branch neng/py3-migration in repository https://gitbox.apache.org/repos/asf/incubator-heron.git. from b316617 make files compatible with python2 and 3 add b28ce71 update cpplint.py No new revisions were added by this update. Summary of changes: third_party/python/cpplint/cpplint.py | 28 ++-- 1 file changed, 14 insertions(+), 14 deletions(-)
[incubator-heron] branch neng/py3-migration updated (b2e8ed3 -> b316617)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a change to branch neng/py3-migration in repository https://gitbox.apache.org/repos/asf/incubator-heron.git. from b2e8ed3 make syntax error compitable with both python2 and 3 add b316617 make files compatible with python2 and 3 No new revisions were added by this update. Summary of changes: .travis.yml| 3 - bazel_configure.py | 2 +- heron/executor/src/python/heron_executor.py| 13 ++--- .../src/python/handlers/killexecutorhandler.py | 7 ++- heron/shell/src/python/main.py | 2 +- heron/statemgrs/src/python/configloader.py | 2 +- heron/statemgrs/src/python/filestatemanager.py | 6 +- heron/statemgrs/src/python/statemanager.py | 3 +- heron/tools/admin/src/python/main.py | 2 +- heron/tools/admin/src/python/standalone.py | 1 + heron/tools/cli/src/python/main.py | 2 +- heron/tools/cli/src/python/submit.py | 7 ++- heron/tools/explorer/src/python/main.py| 2 +- heron/tools/tracker/src/python/main.py | 3 +- heron/tools/tracker/src/python/tracker.py | 6 +- heron/tools/ui/src/python/main.py | 4 +- integration_test/src/python/common/status.py | 2 +- .../src/python/local_test_runner/main.py | 2 +- third_party/python/cpplint/cpplint.py | 64 +++--- tools/rules/pex/wrapper/pex_wrapper.py | 2 +- 20 files changed, 71 insertions(+), 64 deletions(-)
[incubator-heron] branch neng/py3-migration updated (63a6825 -> b2e8ed3)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a change to branch neng/py3-migration in repository https://gitbox.apache.org/repos/asf/incubator-heron.git. from 63a6825 update pex rule add b2e8ed3 make syntax error compitable with both python2 and 3 No new revisions were added by this update. Summary of changes: heron/statemgrs/src/python/statemanager.py | 3 ++- heron/tools/cli/src/python/result.py | 2 +- heron/tools/tracker/src/python/main.py | 2 +- heron/tools/ui/src/python/main.py | 2 +- tools/rules/pex/BUILD | 2 +- 5 files changed, 6 insertions(+), 5 deletions(-)
[incubator-heron] branch neng/py3-migration updated (2c162d7 -> 63a6825)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a change to branch neng/py3-migration in repository https://gitbox.apache.org/repos/asf/incubator-heron.git. from 2c162d7 update travis.yml add 63a6825 update pex rule No new revisions were added by this update. Summary of changes: tools/rules/pex/BUILD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
[incubator-heron] branch neng/py3-migration updated (a1c1e8f -> 2c162d7)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a change to branch neng/py3-migration in repository https://gitbox.apache.org/repos/asf/incubator-heron.git. from a1c1e8f bump python version in bazel add 2c162d7 update travis.yml No new revisions were added by this update. Summary of changes: .travis.yml | 3 +++ 1 file changed, 3 insertions(+)
[incubator-heron] branch neng/py3-migration updated (28a206a -> a1c1e8f)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a change to branch neng/py3-migration in repository https://gitbox.apache.org/repos/asf/incubator-heron.git. from 28a206a correct print with extra parentheses add a1c1e8f bump python version in bazel No new revisions were added by this update. Summary of changes: bazel_configure.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-)
[incubator-heron] branch neng/py3-migration updated (aa13b1c -> 28a206a)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a change to branch neng/py3-migration in repository https://gitbox.apache.org/repos/asf/incubator-heron.git. from aa13b1c update Shebang add 28a206a correct print with extra parentheses No new revisions were added by this update. Summary of changes: bazel_configure.py | 16 .../executor/tests/python/heron_executor_unittest.py | 12 ++-- heron/statemgrs/src/python/configloader.py | 2 +- heron/tools/admin/src/python/main.py | 4 ++-- heron/tools/cli/src/python/config.py | 12 ++-- heron/tools/cli/src/python/execute.py| 4 ++-- heron/tools/cli/src/python/help.py | 2 +- heron/tools/cli/src/python/main.py | 8 heron/tools/cli/src/python/submit.py | 2 +- heron/tools/cli/src/python/version.py| 2 +- heron/tools/common/src/python/utils/config.py| 4 ++-- heron/tools/explorer/src/python/clusters.py | 2 +- heron/tools/explorer/src/python/help.py | 2 +- heron/tools/explorer/src/python/logicalplan.py | 6 +++--- heron/tools/explorer/src/python/main.py | 2 +- heron/tools/explorer/src/python/physicalplan.py | 10 +- heron/tools/explorer/src/python/topologies.py| 20 ++-- heron/tools/ui/src/python/args.py| 4 ++-- heronpy/streamlet/impl/joinbolt.py | 4 ++-- 19 files changed, 59 insertions(+), 59 deletions(-)
[incubator-heron] branch neng/py3-migration updated (6a2a4f5 -> aa13b1c)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a change to branch neng/py3-migration in repository https://gitbox.apache.org/repos/asf/incubator-heron.git. from 6a2a4f5 migrate to python3 using 2to3 cmd tool add aa13b1c update Shebang No new revisions were added by this update. Summary of changes: heron/executor/src/python/heron_executor.py | 2 +- heron/shell/src/python/main.py| 2 +- heron/statemgrs/src/python/configloader.py| 2 +- heron/tools/admin/src/python/main.py | 2 +- heron/tools/cli/src/python/main.py| 2 +- heron/tools/explorer/src/python/main.py | 2 +- heron/tools/tracker/src/python/main.py| 2 +- heron/tools/ui/src/python/main.py | 2 +- integration_test/src/python/common/status.py | 2 +- integration_test/src/python/local_test_runner/main.py | 2 +- tools/rules/pex/wrapper/pex_wrapper.py| 2 +- 11 files changed, 11 insertions(+), 11 deletions(-)
[incubator-heron] 01/01: migrate to python3 using 2to3 cmd tool
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a commit to branch neng/py3-migration in repository https://gitbox.apache.org/repos/asf/incubator-heron.git commit 6a2a4f541522cc9f597c3c5dd99ee373a4adb260 Author: Neng Lu AuthorDate: Tue Dec 3 15:57:58 2019 -0800 migrate to python3 using 2to3 cmd tool --- bazel_configure.py | 16 +++ examples/src/python/bolt/stateful_count_bolt.py| 2 +- examples/src/python/spout/stateful_word_spout.py | 2 +- heron/executor/src/python/heron_executor.py| 36 +++--- .../tests/python/heron_executor_unittest.py| 52 ++-- heron/instance/src/python/basics/__init__.py | 4 +- heron/instance/src/python/basics/bolt_instance.py | 4 +- heron/instance/src/python/basics/spout_instance.py | 4 +- .../instance/src/python/network/gateway_looper.py | 4 +- .../src/python/utils/metrics/metrics_helper.py | 4 +- .../instance/src/python/utils/misc/communicator.py | 12 ++--- .../python/utils/misc/custom_grouping_helper.py| 2 +- heron/instance/src/python/utils/system_config.py | 2 +- .../python/utils/topology/topology_context_impl.py | 2 +- heron/instance/tests/python/mock_protobuf.py | 2 +- .../tests/python/utils/communicator_unittest.py| 4 +- heron/instance/tests/python/utils/log_unittest.py | 8 ++-- .../tests/python/utils/metrics_helper_unittest.py | 2 +- .../instance/tests/python/utils/mock_generator.py | 4 +- heron/shell/src/python/handlers/__init__.py| 24 +- .../src/python/handlers/killexecutorhandler.py | 4 +- heron/shell/src/python/utils.py| 2 +- heron/statemgrs/src/python/config.py | 4 +- heron/statemgrs/src/python/configloader.py | 2 +- heron/statemgrs/src/python/filestatemanager.py | 9 ++-- heron/statemgrs/src/python/statemanager.py | 4 +- heron/statemgrs/src/python/zkstatemanager.py | 2 +- .../tests/python/configloader_unittest.py | 14 +++--- .../tests/python/statemanagerfactory_unittest.py | 2 +- heron/tools/admin/src/python/main.py | 6 +-- heron/tools/admin/src/python/standalone.py | 14 +++--- heron/tools/cli/src/python/cli_helper.py | 2 +- heron/tools/cli/src/python/config.py | 14 +++--- heron/tools/cli/src/python/execute.py | 4 +- heron/tools/cli/src/python/help.py | 2 +- heron/tools/cli/src/python/main.py | 10 ++-- heron/tools/cli/src/python/opts.py | 2 +- heron/tools/cli/src/python/result.py | 2 +- heron/tools/cli/src/python/submit.py | 6 +-- heron/tools/cli/src/python/version.py | 4 +- heron/tools/common/src/python/access/__init__.py | 4 +- heron/tools/common/src/python/access/heron_api.py | 16 +++ .../common/src/python/access/tracker_access.py | 2 +- heron/tools/common/src/python/utils/config.py | 8 ++-- heron/tools/explorer/src/python/clusters.py| 2 +- heron/tools/explorer/src/python/help.py| 2 +- heron/tools/explorer/src/python/logicalplan.py | 14 +++--- heron/tools/explorer/src/python/main.py| 2 +- heron/tools/explorer/src/python/physicalplan.py| 26 +- heron/tools/explorer/src/python/topologies.py | 24 +- heron/tools/tracker/src/python/config.py | 4 +- .../tools/tracker/src/python/handlers/__init__.py | 56 +++--- .../src/python/handlers/logicalplanhandler.py | 4 +- .../src/python/handlers/runtimestatehandler.py | 2 +- heron/tools/tracker/src/python/javaobj.py | 8 ++-- heron/tools/tracker/src/python/main.py | 4 +- heron/tools/tracker/src/python/query_operators.py | 54 ++--- heron/tools/tracker/src/python/topology.py | 8 ++-- heron/tools/tracker/src/python/tracker.py | 10 ++-- .../tools/tracker/tests/python/tracker_unittest.py | 10 ++-- heron/tools/ui/src/python/args.py | 6 +-- heron/tools/ui/src/python/handlers/api/topology.py | 12 ++--- heron/tools/ui/src/python/handlers/mainhandler.py | 2 +- heron/tools/ui/src/python/handlers/ranges.py | 2 +- heron/tools/ui/src/python/handlers/topology.py | 4 +- heron/tools/ui/src/python/main.py | 2 +- heronpy/api/cloudpickle.py | 26 +- heronpy/api/component/component_spec.py| 10 ++-- heronpy/api/metrics.py | 4 +- heronpy/api/serializer.py | 2 +- heronpy/api/tests/python/component_unittest.py | 2 +- heronpy/api/tests/python/metrics_unittest.py | 8 ++-- heronpy/api/topology.py| 10 ++-- heronpy/streamlet/impl/contextimpl.py | 2 +- heronpy/streamlet/impl/joinbolt.py
[incubator-heron] branch neng/py3-migration created (now 6a2a4f5)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a change to branch neng/py3-migration in repository https://gitbox.apache.org/repos/asf/incubator-heron.git. at 6a2a4f5 migrate to python3 using 2to3 cmd tool This branch includes the following new commits: new 6a2a4f5 migrate to python3 using 2to3 cmd tool The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[incubator-heron] 01/01: increase the process restart retry to 150
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a commit to branch neng/increase_max_runs in repository https://gitbox.apache.org/repos/asf/incubator-heron.git commit c19d179376c3da5e08f0b2a50d1aa5dec34929df Author: Neng Lu AuthorDate: Thu Aug 15 11:11:59 2019 -0700 increase the process restart retry to 150 --- heron/executor/src/python/heron_executor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/heron/executor/src/python/heron_executor.py b/heron/executor/src/python/heron_executor.py index 6f5790a..9380ef5 100755 --- a/heron/executor/src/python/heron_executor.py +++ b/heron/executor/src/python/heron_executor.py @@ -297,7 +297,7 @@ class HeronExecutor(object): self.init_from_parsed_args(parsed_args) self.shell_env = shell_env -self.max_runs = 100 +self.max_runs = 150 self.interval_between_runs = 10 # Read the heron_internals.yaml for logging dir
[incubator-heron] branch neng/increase_max_runs created (now c19d179)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a change to branch neng/increase_max_runs in repository https://gitbox.apache.org/repos/asf/incubator-heron.git. at c19d179 increase the process restart retry to 150 This branch includes the following new commits: new c19d179 increase the process restart retry to 150 The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[incubator-heron] branch master updated: Fix heron-shell command invocation (#3236)
This is an automated email from the ASF dual-hosted git repository.
nlu90 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
The following commit(s) were added to refs/heads/master by this push:
new facd7e7 Fix heron-shell command invocation (#3236)
facd7e7 is described below
commit facd7e7081c4849577c91255ce4d5b1c5d5a9614
Author: Xiaoyao Qian
AuthorDate: Wed Apr 10 11:29:16 2019 -0700
Fix heron-shell command invocation (#3236)
---
heron/executor/src/python/heron_executor.py | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/heron/executor/src/python/heron_executor.py
b/heron/executor/src/python/heron_executor.py
index e727fdd..0c8f21c 100755
--- a/heron/executor/src/python/heron_executor.py
+++ b/heron/executor/src/python/heron_executor.py
@@ -1033,11 +1033,11 @@ class HeronExecutor(object):
return {}
if self._get_instance_plans(self.packing_plan, self.shard) is None and
self.shard != 0:
retval = {}
- retval['heron-shell'] = [
+ retval['heron-shell'] = Command([
'%s' % self.heron_shell_binary,
'--port=%s' % self.shell_port,
'--log_file_prefix=%s/heron-shell-%s.log' % (self.log_dir,
self.shard),
- '--secret=%s' % self.topology_id]
+ '--secret=%s' % self.topology_id], self.shell_env)
return retval
if self.shard == 0:
[incubator-heron] branch master updated: init instance state only if the topology is stateful and the component is a stateful component (#3229)
This is an automated email from the ASF dual-hosted git repository.
nlu90 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
The following commit(s) were added to refs/heads/master by this push:
new bc47197 init instance state only if the topology is stateful and the
component is a stateful component (#3229)
bc47197 is described below
commit bc4719746bf613bc5eaf4c3781a435ceb6349198
Author: Neng Lu
AuthorDate: Tue Apr 2 14:17:34 2019 -0700
init instance state only if the topology is stateful and the component is a
stateful component (#3229)
---
.../src/java/org/apache/heron/instance/bolt/BoltInstance.java | 4 ++--
.../src/java/org/apache/heron/instance/spout/SpoutInstance.java | 4 ++--
2 files changed, 4 insertions(+), 4 deletions(-)
diff --git
a/heron/instance/src/java/org/apache/heron/instance/bolt/BoltInstance.java
b/heron/instance/src/java/org/apache/heron/instance/bolt/BoltInstance.java
index 893ed01..9cc177a 100644
--- a/heron/instance/src/java/org/apache/heron/instance/bolt/BoltInstance.java
+++ b/heron/instance/src/java/org/apache/heron/instance/bolt/BoltInstance.java
@@ -183,8 +183,8 @@ public class BoltInstance implements IInstance {
boltMetrics.registerMetrics(topologyContext);
-// Initialize the instanceState if the bolt is stateful
-if (bolt instanceof IStatefulComponent) {
+// Initialize the instanceState if the topology is stateful and bolt is a
stateful component
+if (isTopologyStateful && bolt instanceof IStatefulComponent) {
this.instanceState = state;
((IStatefulComponent)
bolt).initState(instanceState);
diff --git
a/heron/instance/src/java/org/apache/heron/instance/spout/SpoutInstance.java
b/heron/instance/src/java/org/apache/heron/instance/spout/SpoutInstance.java
index 348a641..53675c4 100644
--- a/heron/instance/src/java/org/apache/heron/instance/spout/SpoutInstance.java
+++ b/heron/instance/src/java/org/apache/heron/instance/spout/SpoutInstance.java
@@ -188,8 +188,8 @@ public class SpoutInstance implements IInstance {
spoutMetrics.registerMetrics(topologyContext);
-// Initialize the instanceState if the spout is stateful
-if (spout instanceof IStatefulComponent) {
+// Initialize the instanceState if the topology is stateful and spout is a
stateful component
+if (isTopologyStateful && spout instanceof IStatefulComponent) {
this.instanceState = state;
((IStatefulComponent)
spout).initState(instanceState);
[incubator-heron] branch master updated: Fix docker build artifacts script to have the correct version string (#3156)
This is an automated email from the ASF dual-hosted git repository.
nlu90 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
The following commit(s) were added to refs/heads/master by this push:
new 297eb24 Fix docker build artifacts script to have the correct version
string (#3156)
297eb24 is described below
commit 297eb24039ae501752f7836101169429effad413
Author: Ning Wang
AuthorDate: Thu Jan 17 14:49:40 2019 -0800
Fix docker build artifacts script to have the correct version string (#3156)
---
scripts/release/status.sh | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
diff --git a/scripts/release/status.sh b/scripts/release/status.sh
index 0b87b8b..925b395 100755
--- a/scripts/release/status.sh
+++ b/scripts/release/status.sh
@@ -59,8 +59,7 @@ then
build_version=${HERON_BUILD_VERSION}
fi
else
- current_dir=$(pwd)
- build_version=$(basename "$current_dir")
+ build_version=${HERON_BUILD_VERSION}
fi
echo "HERON_BUILD_VERSION ${build_version}"
[incubator-heron] tag v-0.20.0-incubating created (now 45043bb)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a change to tag v-0.20.0-incubating in repository https://gitbox.apache.org/repos/asf/incubator-heron.git. at 45043bb (commit) No new revisions were added by this update.
[incubator-heron] branch master updated: Update mailing list informtion in contributing.md (#3110)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-heron.git The following commit(s) were added to refs/heads/master by this push: new cbf7e51 Update mailing list informtion in contributing.md (#3110) cbf7e51 is described below commit cbf7e51f260995c9915b1a53bcee6ae3b4d790be Author: Ning Wang AuthorDate: Mon Nov 19 11:30:34 2018 -0800 Update mailing list informtion in contributing.md (#3110) --- website/content/docs/contributors/community.md | 8 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/website/content/docs/contributors/community.md b/website/content/docs/contributors/community.md index 7d77859..25c4097 100644 --- a/website/content/docs/contributors/community.md +++ b/website/content/docs/contributors/community.md @@ -8,7 +8,7 @@ title: Community Discussion about Heron happens on GitHub and over mailing list. * GitHub: [apache/incubator-heron](https://github.com/apache/incubator-heron) -* Heron User Google Group: [heron-us...@googlegroups.com](https://groups.google.com/forum/#!forum/heron-users) +* Heron User Group: [u...@heron.incubator.apache.org](https://mail-archives.apache.org/mod_mbox/heron-user/) * Heron on Twitter: [@heronstreaming](https://twitter.com/heronstreaming) Community is critical to Heron. Contributions are welcomed! @@ -33,18 +33,18 @@ $ ./scripts/setup-intellij.sh In general, contributions that fix bugs or add features (as opposed to stylistic, refactoring, or "cleanup" changes) are preferred. If you're looking for places to contribute, issues with label [help-wanted](https://github.com/apache/incubator-heron/issues?q=is%3Aopen+is%3Aissue+label%3Ahelp-wanted) -are good candidates. Please check with [mailing list](https://groups.google.com/forum/#!forum/heron-users) +are good candidates. Please check with [mailing list](https://mail-archives.apache.org/mod_mbox/heron-dev/) if your patch involves lots of changes. **If you have any question or issues about troubleshooting, -you should post on [mailing list](https://groups.google.com/forum/#!forum/heron-users) instead +you should post on [mailing list](https://mail-archives.apache.org/mod_mbox/heron-user/) instead of opening GitHub issues.** ### Submitting a Patch 1. Read the Heron [governance plan](../governance). 2. Discuss your plan and design, and get agreement on -[mailing list](https://groups.google.com/forum/#!forum/heron-users). +[mailing list](https://mail-archives.apache.org/mod_mbox/heron-dev/). 3. Implement proper unit tests along with your change. Verify that all tests can pass.
svn commit: r30933 - in /release/incubator/heron: ./ heron-0.20.0-incubating/
Author: nlu90 Date: Fri Nov 16 22:28:18 2018 New Revision: 30933 Log: apache heron 0.20.0-incubating release Added: release/incubator/heron/KEYS release/incubator/heron/heron-0.20.0-incubating/ release/incubator/heron/heron-0.20.0-incubating/apache-heron-v-0.20.0-incubating-source.tar.gz (with props) release/incubator/heron/heron-0.20.0-incubating/apache-heron-v-0.20.0-incubating-source.tar.gz.asc release/incubator/heron/heron-0.20.0-incubating/apache-heron-v-0.20.0-incubating-source.tar.gz.sha512 Added: release/incubator/heron/KEYS == --- release/incubator/heron/KEYS (added) +++ release/incubator/heron/KEYS Fri Nov 16 22:28:18 2018 @@ -0,0 +1,72 @@ +This file contains the PGP keys of various developers. + +Users: pgp < KEYS +or + gpg --import KEYS + + +Developers: +pgp -kxa and append it to this file. +or +(pgpk -ll && pgpk -xa ) >> this file. +or +(gpg --list-sigs +&& gpg --armor --export ) >> this file. + +pub 4096R/1F10F679 2018-08-16 +uid Neng Lu (CODE SIGNING KEY) +sig 31F10F679 2018-08-16 Neng Lu (CODE SIGNING KEY) +sub 4096R/7B1413B4 2018-08-16 +sig 1F10F679 2018-08-16 Neng Lu (CODE SIGNING KEY) + +-BEGIN PGP PUBLIC KEY BLOCK- + +mQINBFt196ABEADBPpEfJ5WjI2GI1GCb12UIfv7ActF5d1vcVr3oy5ZsB5/xvn2b +wnmXrfIIde58L6JtEJWWcG6V0Xs+a6JwE0wkSgHAkS33IwGOQE/ue8syeQ0UG7ZG +Tda9vqqNw5aLMgXQvQVuVEZzvxZiM32qK8Vj1kBXrfl3yobO5mRv6yAQ5Mn1cqSC +ERffMcqfsKYR4ekERN0dlOxD8Zer50FoCTxzRTHYhkQ8xv8Hrm2msx6q704CCTv8 +qcd1h9qM7xPzGSjWMxBz6wM7HILpaeoZ7Sjv4mFrAtjzPFPB8rQK4YNTfDIg0o+u +1FnQTDbGKOWEP1NO5ngHz4mDcxWojz2yqWIm+Ubk/8kkPmzthtNND8lPnKZ0EcAz +6ov9EkGU2lUaWOZJsf7/cXisGc1KVuGriKmjW6M/7HiohWHgoZN7N0n5x/bU+ha+ +rpnNWnVLAL3OIS+I6C/1VPOnkJuRinQlzy1T3L+67ma4BJmOm3xrLFPedLHVKiTi +z8yT3/3j0GYAQT8ujGE7Mw+qLfsSK7TRW5Shi67EgKGr8ctCgs8bund5pZmTHt66 +8CiYUEW+cowiisLJA9OjlXAHXm+iY6mkWa8DHjPyLboKIxdacU9w/5c6zcHOjufT +wS9Sjac5G6UFJLLJ8TxoVmA7VW0xMGqye55Xg9T4Ur1C1xNgxj6/a1tyvQARAQAB +tC1OZW5nIEx1IChDT0RFIFNJR05JTkcgS0VZKSA8bmx1OTBAYXBhY2hlLm9yZz6J +AjgEEwECACIFAlt196ACGwMGCwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJEBYI +dp4fEPZ5OukQALY5aU1ZHBWDWDD2LJIgQ5Th5hm5L70P3FHgBev4gPj0FGbyzizo +kRTfpwISOk49zAIOveCIqKaGl9HoJTzWkrcw54PLxIs3FayyW78DoVJZ1Cmjacuy +xvWW8CvaDVMkLuTA4vwV2IbsF4iB7qQi0IUjo+uXNKYF6fS/BtFCE3/QTdxWwpWR +Qicg3LajjaOKi3zMOjTMir7ulmwZzUelHc14CweHrMwfxM2+CQ/u6LxCwGBMNJwh +Xl6AOcCMqgshVw6rO9/aINBPm6UtLzg++n1CI2nmX2+hNpon7BjZ+eLEDmKbNMO4 +eRePlqyrkPeD+ytvgyLmN6zh1GxeiLSL7Rk2sGinR2HXLF6DM152pbr0L5APKILz +BnCLUavPSe42YbLOpb1zA46ckHQl9PdrVidYoR1RtIj19FFO5rSxwrkpzIN+SB4f +sHtv0LUx2CfV+pcgPzPA5dMpDqA1N9cym8hVDPB99sbuStmIas2d44YiEFCx7B92 +23WPs1ucr82g9E/tNhiqb+/iXjTfxLZCpog+yt7Byoa0rjS0P3hNyM5+agGk45uU +ujlM0cDW/Y3bo8gQCksB6Qm3UZbDiro0k3HuB+QFmaUbT1ZBNfbnZkPzr9BiqhQP +3M3rtapI2YqAh4WaJUbCF2hx7SroyIMuBh/bnPpyMULvApVdy7stbcMuuQINBFt1 +96ABEAC3oNxNX+J8MoHflOhN9CUQSOzqxaH1MMvXFcKezd+lIsyaZ7PwFl6vUVOO +m0RwGff1Xtff7MGTvzDQRIEr0ZzREgUkuhIB35DBKzEJIrkKBTH21TeYu/xDo38g +cXJzUgkKx1coHkoCqCEaGmxHHwr53BALIAlYWSp7g0UIm1Wpk9y/kGwTMBjNIRvb +u3pYodEybcM2H0OF1LryH/LwReNPOae5fys0Djog6DHTN9ZOd36E+GNtbxSf/Gj2 +5KF1n2sa8T1B8LS0LNa2I4/sq7KkQi/IeUBybVAc7O42Khfr93FFFfLIxlSgtQZu +NGLPMtlLgGTrL+qo0fHRzN+GeK8tO3sFmpa4JUMLAlgUNdA0fe6piyfZ6iYqlU9K +E7SOqZPw6p3AXYcjvHRmxRZo+06yc/CtpL+Oekf/p1ER6hr4GHM4dBUUImLZ/0fC +7LiIoZ4HXySQEeTH97VUFNKIXLoFobrKoFJ6mxq1LDi5uWGniKR9eETk1aQnDKFe +nWi0sC/VeCFuW951L6ZriKa+noxHxwoLXEL7iR5htmz1bCCkSKUtxtO6oqsM7O78 +1FlfE3s4QMtgO9hlEElDBocBlVGba7A7EI4EAtawBnzujMh/QrkkIlBi8PXI2xKE +bNuce9FkIZ5CwIblVU8NwpqBM3zd1IWYy2Y8jKl0v9xJn9ZJxwARAQABiQIfBBgB +AgAJBQJbdfegAhsMAAoJEBYIdp4fEPZ562cP/3C7ngDAia772bOwBEq1VprZ++Jj +QkMKuro9UAeL1B64+Ainp9gz/+8nbjursQIMZWLPwCNWxPUO64SLh3Wi4AjClRTI +J2FsmVZiAM78Kxpi6qvIkS5A0LYjAs9PbB6gGhfhyJSVJj64q6aNYJr6XhHhsrxQ +W3gKde44wHQNuk6mDDk5cDHmi9U/oWEgT3orExgxPW1poEdEcO5189KglgzedAzS +QBNQcJRU8nuiKFi2IOv5wQQviV6+ngDI3zS6F3sOPLgABXwwI+dQDgLKXHyBnUsf +6IMKU5lciCMQZAcQQ+KLBo0b4QKLJbxOdv0UXE0sw8xxet5VhEdVIAI51gXrQ1eC +1eoP9MN6G1xnIJWrlwU6evyFrNNn5Idc5T9PD9nlBzyGRG5/r9GVX8HobawwAW3W +WHBmfzloKFj4E8sSLa4OTA872F41C5AElPW3BmaKOF7fZDtY3bHNNAXDWthVpy7+ +2nTxcIlotD8HJzl19OxqabJR0C/OpdpFIJmHIo82YdRj2VI1AIPKypU8kRdByzB9 +9Rj/PaIzjYS2PlwRzBhUtMqsEyJmSN5uotTVtpf9VD+YokxdSjx3opRDMvzz6Go5 +o3s7q+kCj9sdPsSkd6ZnjBGssCQvJN6mpL0zRwPn1IST96975twJbzoJe2DLDMCF +xlS+6Ph5M9Qett0S +=2DcU +-END PGP PUBLIC KEY BLOCK- Added: release/incubator/heron/heron-0.20.0-incubating/apache-heron-v-0.20.0-incubating-source.tar.gz == Binary file - no diff available. Propchange: release/incubator/heron/heron-0.20.0-incubating/apache-heron-v-0.20.0-incubating-source.tar.gz -- svn:mime-type = application/octet-stream Added: release/incubator/heron/heron-0.20.0-incubating/apache-heron-v-0.20
[incubator-heron] branch master updated: [Java Streamlet API] Extend validations (#3095)
This is an automated email from the ASF dual-hosted git repository.
nlu90 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
The following commit(s) were added to refs/heads/master by this push:
new 67af633 [Java Streamlet API] Extend validations (#3095)
67af633 is described below
commit 67af633235c356a7e2660c0256182d05dd057daa
Author: Eren Avsarogullari
AuthorDate: Mon Nov 5 19:40:08 2018 +
[Java Streamlet API] Extend validations (#3095)
* Add validations to Streamlet API
* Review comments are addressed.
---
heron/api/src/java/BUILD | 4 +--
.../apache/heron/streamlet/impl/BuilderImpl.java | 3 ++
.../apache/heron/streamlet/impl/StreamletImpl.java | 20 ---
.../heron/streamlet/impl/utils/StreamletUtils.java | 39 ++
heron/api/tests/java/BUILD | 1 +
.../heron/streamlet/impl/StreamletImplTest.java| 9 -
.../streamlet/impl/utils/StreamletUtilsTest.java | 38 +
7 files changed, 97 insertions(+), 17 deletions(-)
diff --git a/heron/api/src/java/BUILD b/heron/api/src/java/BUILD
index c87f46c..cdab255 100644
--- a/heron/api/src/java/BUILD
+++ b/heron/api/src/java/BUILD
@@ -33,7 +33,7 @@ java_library(
javacopts = DOCLINT_HTML_AND_SYNTAX,
deps = api_deps_files + [
":api-java-low-level",
-"//third_party/java:kryo-neverlink",
+"//third_party/java:kryo-neverlink"
]
)
@@ -48,7 +48,7 @@ java_library(
java_binary(
name = "api-unshaded",
srcs = glob(["org/apache/heron/api/**/*.java",
"org/apache/heron/streamlet/**/*.java"]),
-deps = api_deps_files + ["//third_party/java:kryo-neverlink"],
+deps = api_deps_files + ["//third_party/java:kryo-neverlink"]
)
jarjar_binary(
diff --git
a/heron/api/src/java/org/apache/heron/streamlet/impl/BuilderImpl.java
b/heron/api/src/java/org/apache/heron/streamlet/impl/BuilderImpl.java
index 936e46b..0eb06d9 100644
--- a/heron/api/src/java/org/apache/heron/streamlet/impl/BuilderImpl.java
+++ b/heron/api/src/java/org/apache/heron/streamlet/impl/BuilderImpl.java
@@ -29,6 +29,7 @@ import org.apache.heron.streamlet.Builder;
import org.apache.heron.streamlet.SerializableSupplier;
import org.apache.heron.streamlet.Source;
import org.apache.heron.streamlet.Streamlet;
+import org.apache.heron.streamlet.impl.utils.StreamletUtils;
/**
* BuilderImpl implements the Builder interface.
@@ -44,6 +45,7 @@ public final class BuilderImpl implements Builder {
@Override
public Streamlet newSource(SerializableSupplier supplier) {
+StreamletUtils.require(supplier != null, "supplier must not be null.");
StreamletImpl retval = StreamletImpl.createSupplierStreamlet(supplier);
retval.setNumPartitions(1);
sources.add(retval);
@@ -52,6 +54,7 @@ public final class BuilderImpl implements Builder {
@Override
public Streamlet newSource(Source generator) {
+StreamletUtils.require(generator != null, "source must not be null.");
StreamletImpl retval =
StreamletImpl.createGeneratorStreamlet(generator);
retval.setNumPartitions(1);
sources.add(retval);
diff --git
a/heron/api/src/java/org/apache/heron/streamlet/impl/StreamletImpl.java
b/heron/api/src/java/org/apache/heron/streamlet/impl/StreamletImpl.java
index b1c427a..066dc74 100644
--- a/heron/api/src/java/org/apache/heron/streamlet/impl/StreamletImpl.java
+++ b/heron/api/src/java/org/apache/heron/streamlet/impl/StreamletImpl.java
@@ -57,6 +57,7 @@ import
org.apache.heron.streamlet.impl.streamlets.SourceStreamlet;
import org.apache.heron.streamlet.impl.streamlets.SupplierStreamlet;
import org.apache.heron.streamlet.impl.streamlets.TransformStreamlet;
import org.apache.heron.streamlet.impl.streamlets.UnionStreamlet;
+import org.apache.heron.streamlet.impl.utils.StreamletUtils;
/**
* A Streamlet is a (potentially unbounded) ordered collection of tuples.
@@ -147,7 +148,7 @@ public abstract class StreamletImpl implements
Streamlet {
*/
@Override
public Streamlet setName(String sName) {
-require(sName != null && !sName.trim().isEmpty(),
+StreamletUtils.require(sName != null && !sName.trim().isEmpty(),
"Streamlet name cannot be null/blank");
this.name = sName;
return this;
@@ -186,7 +187,7 @@ public abstract class StreamletImpl implements
Streamlet {
*/
@Override
public Streamlet setNumPartitions(int numPartitions) {
-require(numPartitions > 0,
+StreamletUtils.require(numPartitions > 0,
"Streamlet's partitions number should be > 0");
this.nPartitions = numPartitions;
return this;
@@ -326,7 +327,9 @@ public abstract class StreamletImpl implements
Streamlet {
*/
@Override
public List> clone(int numClone
[incubator-heron] branch master updated: Fix build error on Mac. (#3102)
This is an automated email from the ASF dual-hosted git repository.
nlu90 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
The following commit(s) were added to refs/heads/master by this push:
new cbe1e42 Fix build error on Mac. (#3102)
cbe1e42 is described below
commit cbe1e424ea91e1870ecf65a1dd9c8685eb029daa
Author: se choi
AuthorDate: Tue Nov 6 04:39:39 2018 +0900
Fix build error on Mac. (#3102)
* Fix build error on Mac.
* Change equalsIgnoreCase to equals
---
heron/common/src/java/org/apache/heron/common/basics/ByteAmount.java | 2 ++
.../org/apache/heron/scheduler/mesos/framework/LaunchableTask.java| 4 ++--
2 files changed, 4 insertions(+), 2 deletions(-)
diff --git
a/heron/common/src/java/org/apache/heron/common/basics/ByteAmount.java
b/heron/common/src/java/org/apache/heron/common/basics/ByteAmount.java
index e4e3fc9..61d9c71 100644
--- a/heron/common/src/java/org/apache/heron/common/basics/ByteAmount.java
+++ b/heron/common/src/java/org/apache/heron/common/basics/ByteAmount.java
@@ -25,7 +25,9 @@ package org.apache.heron.common.basics;
public final class ByteAmount implements Comparable {
private static final long MB = 1024L * 1024;
private static final long GB = MB * 1024;
+ @SuppressWarnings("MathRoundIntLong")
private static final long MAX_MB = Math.round(Long.MAX_VALUE / MB);
+ @SuppressWarnings("MathRoundIntLong")
private static final long MAX_GB = Math.round(Long.MAX_VALUE / GB);
public static final ByteAmount ZERO = ByteAmount.fromBytes(0);
diff --git
a/heron/schedulers/src/java/org/apache/heron/scheduler/mesos/framework/LaunchableTask.java
b/heron/schedulers/src/java/org/apache/heron/scheduler/mesos/framework/LaunchableTask.java
index da095de..7cea009 100644
---
a/heron/schedulers/src/java/org/apache/heron/scheduler/mesos/framework/LaunchableTask.java
+++
b/heron/schedulers/src/java/org/apache/heron/scheduler/mesos/framework/LaunchableTask.java
@@ -67,7 +67,7 @@ public class LaunchableTask {
String role = "*";
for (Protos.Resource resource : reservedResources) {
- if (resource.getName() == name && resource.getScalar().getValue() >=
value) {
+ if (resource.getName().equals(name) && resource.getScalar().getValue()
>= value) {
role = resource.getRole();
break;
@@ -95,7 +95,7 @@ public class LaunchableTask {
String role = "*";
for (Protos.Resource resource : reservedResources) {
- if (resource.getName() == name) {
+ if (resource.getName().equals(name)) {
Protos.Value.Ranges ranges = resource.getRanges();
for (Protos.Value.Range range : ranges.getRangeList()) {
if (range.getBegin() <= begin && range.getEnd() >= end) {
[incubator-heron] branch master updated: Add support for building Heron release in a Ubuntu 18.04 docker container (#3097)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-heron.git The following commit(s) were added to refs/heads/master by this push: new 1b05035 Add support for building Heron release in a Ubuntu 18.04 docker container (#3097) 1b05035 is described below commit 1b05035fc7e82ebee712c24e06771c99679973af Author: Thomas Cooper AuthorDate: Sat Nov 3 03:02:15 2018 + Add support for building Heron release in a Ubuntu 18.04 docker container (#3097) * Added ubuntu 18.04 compile and dist dockerfiles * Updated script help text with supported distributions * Removed defunct run heron in docker instructions --- docker/Readme.md| 24 +--- docker/compile/Dockerfile.ubuntu18.04 | 28 docker/dist/Dockerfile.dist.ubuntu18.04 | 23 +++ docker/scripts/build-artifacts.sh | 2 +- docker/scripts/build-base.sh| 4 ++-- docker/scripts/build-docker.sh | 4 ++-- docker/scripts/build-exec-docker.sh | 2 +- 7 files changed, 58 insertions(+), 29 deletions(-) diff --git a/docker/Readme.md b/docker/Readme.md index 80fbf56..a9c0220 100644 --- a/docker/Readme.md +++ b/docker/Readme.md @@ -1,6 +1,6 @@ # Docker -## To compile source using a docker container: +## To compile source, into Heron release artifacts, using a docker container: ``` ./docker/scripts/build-artifacts.sh [source-tarball] # e.g. ./docker/scripts/build-artifacts.sh ubuntu14.04 testbuild ~/heron-release @@ -11,25 +11,3 @@ ./docker/scripts/build-docker.sh # e.g. ./docker/scripts/build-docker.sh ubuntu14.04 testbuild ~/heron-release ``` - -### To run docker containers for local dev work: -``` -./docker/start-docker.sh heron:- -# e.g. ./docker/start-docker.sh heron:testbuild-ubuntu14.04 -``` -### To stop docker containers for local dev work: -``` -./docker/stop-docker.sh -``` -### To submit/activate/kill a topology: -``` -#To submit a topology: -docker exec heron_executor_1 heron submit local /usr/local/heron/examples/heron-api-examples.jar org.apache.heron.examples.api.ExclamationTopology ExclamationTopology --deploy-deactivated -#To activate a topology: -docker exec -it heron_executor_1 heron activate local ExclamationTopology -#To kill a topology: -docker exec -it heron_executor_1 heron kill local ExclamationTopology -``` -## To access heron ui: -* determine the IP of your docker host -* navigate to http://:8889 in your web browser diff --git a/docker/compile/Dockerfile.ubuntu18.04 b/docker/compile/Dockerfile.ubuntu18.04 new file mode 100644 index 000..c0271a0 --- /dev/null +++ b/docker/compile/Dockerfile.ubuntu18.04 @@ -0,0 +1,28 @@ +FROM ubuntu:18.04 + +# This is passed to the heron build command via the --config flag +ENV TARGET_PLATFORM ubuntu +ENV bazelVersion 0.14.1 + +RUN apt-get update && apt-get -y install \ + g++ \ + cmake \ + automake \ + libtool-bin \ + libunwind8 \ + patch \ + python-dev \ + wget \ + zip \ + unzip + +RUN apt-get -y install openjdk-8-jdk-headless + +ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64 + +RUN wget -O /tmp/bazel.sh https://github.com/bazelbuild/bazel/releases/download/$bazelVersion/bazel-$bazelVersion-installer-linux-x86_64.sh \ + && chmod +x /tmp/bazel.sh \ + && /tmp/bazel.sh + +ADD bazelrc /root/.bazelrc +ADD scripts/compile-platform.sh /compile-platform.sh diff --git a/docker/dist/Dockerfile.dist.ubuntu18.04 b/docker/dist/Dockerfile.dist.ubuntu18.04 new file mode 100644 index 000..5f7f921 --- /dev/null +++ b/docker/dist/Dockerfile.dist.ubuntu18.04 @@ -0,0 +1,23 @@ +FROM ubuntu:18.04 + +RUN apt-get update +RUN apt-get -y install \ +unzip software-properties-common curl supervisor openjdk-8-jdk-headless + +ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64 +RUN update-ca-certificates -f + +ADD artifacts /heron + +WORKDIR /heron + +# run heron installers +RUN /heron/heron-install.sh + +RUN tar --strip-components=1 -m -zxvf /heron/heron-core.tar.gz -C /heron + +RUN rm -f /heron/heron-install.sh +RUN rm -f /heron/heron-core.tar.gz + +ENV HERON_HOME /heron/heron-core/ +RUN export HERON_HOME diff --git a/docker/scripts/build-artifacts.sh b/docker/scripts/build-artifacts.sh index 07f79ce..1265a49 100755 --- a/docker/scripts/build-artifacts.sh +++ b/docker/scripts/build-artifacts.sh @@ -166,7 +166,7 @@ case $# in echo " " echo "Script to build heron artifacts for different platforms" echo " " -echo "Platforms Supported: darwin, ubuntu14.04, ubuntu16.04, centos7, debian9" +echo "Platforms Supported: darwin, debian9, ubuntu14.04, ubuntu16.04, ubuntu18.04, centos7" echo " " echo "Example:" echo &q
[incubator-heron] branch master updated: Nwang/refactor custom operator interface v2 (#3100)
This is an automated email from the ASF dual-hosted git repository.
nlu90 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
The following commit(s) were added to refs/heads/master by this push:
new 34a64f6 Nwang/refactor custom operator interface v2 (#3100)
34a64f6 is described below
commit 34a64f6eec69e22a54187d03d0fb4ad8ca2801cf
Author: Ning Wang
AuthorDate: Fri Nov 2 20:01:05 2018 -0700
Nwang/refactor custom operator interface v2 (#3100)
* Reduce number of applyOperator() functions in Streamlet
* Remove CustomBasicStreamlet and CustomWindowStreamlet
---
.../heron/streamlet/IStreamletBasicOperator.java | 2 +-
.../apache/heron/streamlet/IStreamletOperator.java | 7 +--
...etOperator.java => IStreamletRichOperator.java} | 2 +-
.../heron/streamlet/IStreamletWindowOperator.java | 2 +-
.../java/org/apache/heron/streamlet/Streamlet.java | 16 ---
.../apache/heron/streamlet/impl/StreamletImpl.java | 34 +
.../impl/operators/StreamletOperator.java | 4 +-
.../impl/streamlets/CustomBasicStreamlet.java | 55 --
.../streamlet/impl/streamlets/CustomStreamlet.java | 29 +---
.../impl/streamlets/CustomWindowStreamlet.java | 55 --
.../apache/heron/streamlet/scala/Streamlet.scala | 18 ---
.../heron/streamlet/scala/impl/StreamletImpl.scala | 24 --
.../heron/streamlet/impl/StreamletImplTest.java| 18 ---
.../streamlet/scala/impl/StreamletImplTest.scala | 16 +++
14 files changed, 47 insertions(+), 235 deletions(-)
diff --git
a/heron/api/src/java/org/apache/heron/streamlet/IStreamletBasicOperator.java
b/heron/api/src/java/org/apache/heron/streamlet/IStreamletBasicOperator.java
index e6b615a..98b87f3 100644
--- a/heron/api/src/java/org/apache/heron/streamlet/IStreamletBasicOperator.java
+++ b/heron/api/src/java/org/apache/heron/streamlet/IStreamletBasicOperator.java
@@ -26,5 +26,5 @@ import org.apache.heron.api.bolt.IBasicBolt;
* The interface for streamlet operators. It can be used to create
* operators based on existing Bolts (subclasses of IBasicBolt).
*/
-public interface IStreamletBasicOperator extends IBasicBolt {
+public interface IStreamletBasicOperator extends IStreamletOperator, IBasicBolt {
}
diff --git
a/heron/api/src/java/org/apache/heron/streamlet/IStreamletOperator.java
b/heron/api/src/java/org/apache/heron/streamlet/IStreamletOperator.java
index 24b989c..0eecb60 100644
--- a/heron/api/src/java/org/apache/heron/streamlet/IStreamletOperator.java
+++ b/heron/api/src/java/org/apache/heron/streamlet/IStreamletOperator.java
@@ -20,11 +20,8 @@
package org.apache.heron.streamlet;
-import org.apache.heron.api.bolt.IRichBolt;
-
/**
- * The interface for custom operators: it can be used to create
- * operators based on existing Bolts (subclasses of IRichBolt).
+ * The base interface for all Streamlet operator interfaces.
*/
-public interface IStreamletOperator extends IRichBolt {
+public interface IStreamletOperator {
}
diff --git
a/heron/api/src/java/org/apache/heron/streamlet/IStreamletOperator.java
b/heron/api/src/java/org/apache/heron/streamlet/IStreamletRichOperator.java
similarity index 91%
copy from heron/api/src/java/org/apache/heron/streamlet/IStreamletOperator.java
copy to
heron/api/src/java/org/apache/heron/streamlet/IStreamletRichOperator.java
index 24b989c..c6800b8 100644
--- a/heron/api/src/java/org/apache/heron/streamlet/IStreamletOperator.java
+++ b/heron/api/src/java/org/apache/heron/streamlet/IStreamletRichOperator.java
@@ -26,5 +26,5 @@ import org.apache.heron.api.bolt.IRichBolt;
* The interface for custom operators: it can be used to create
* operators based on existing Bolts (subclasses of IRichBolt).
*/
-public interface IStreamletOperator extends IRichBolt {
+public interface IStreamletRichOperator extends IStreamletOperator, IRichBolt {
}
diff --git
a/heron/api/src/java/org/apache/heron/streamlet/IStreamletWindowOperator.java
b/heron/api/src/java/org/apache/heron/streamlet/IStreamletWindowOperator.java
index 7369fca..dcccae1 100644
---
a/heron/api/src/java/org/apache/heron/streamlet/IStreamletWindowOperator.java
+++
b/heron/api/src/java/org/apache/heron/streamlet/IStreamletWindowOperator.java
@@ -26,5 +26,5 @@ import org.apache.heron.api.bolt.IWindowedBolt;
* The interface for streamlet operators. It can be used to create
* operators based on existing Bolts (subclasses of IWindowedBolt).
*/
-public interface IStreamletWindowOperator extends IWindowedBolt {
+public interface IStreamletWindowOperator extends IStreamletOperator, IWindowedBolt {
}
diff --git a/heron/api/src/java/org/apache/heron/streamlet/Streamlet.java
b/heron/api/src/java/org/apache/heron/streamlet/Streamlet.java
index 2490537..b5fe931 100644
--- a/heron/api/src/java/org/apache/heron/streamlet/Streamlet.java
+++ b/heron/api/src/java/org/apache/heron/streamlet/Streamlet.j
[incubator-heron] branch master updated: Fix number of tasks in Streamlet RemapCustomGrouping (#3098)
This is an automated email from the ASF dual-hosted git repository.
nlu90 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
The following commit(s) were added to refs/heads/master by this push:
new d40b60f Fix number of tasks in Streamlet RemapCustomGrouping (#3098)
d40b60f is described below
commit d40b60f147653b4924fe0acf908a7f1b220d1eeb
Author: Ning Wang
AuthorDate: Fri Nov 2 13:45:40 2018 -0700
Fix number of tasks in Streamlet RemapCustomGrouping (#3098)
---
.../org/apache/heron/streamlet/impl/groupings/RemapCustomGrouping.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/heron/api/src/java/org/apache/heron/streamlet/impl/groupings/RemapCustomGrouping.java
b/heron/api/src/java/org/apache/heron/streamlet/impl/groupings/RemapCustomGrouping.java
index aa71f3a..b5bc823 100644
---
a/heron/api/src/java/org/apache/heron/streamlet/impl/groupings/RemapCustomGrouping.java
+++
b/heron/api/src/java/org/apache/heron/streamlet/impl/groupings/RemapCustomGrouping.java
@@ -53,7 +53,7 @@ public class RemapCustomGrouping implements
CustomStreamGrouping {
public List chooseTasks(List values) {
List ret = new ArrayList<>();
R obj = (R) values.get(0);
-List targets = remapFn.apply(obj, ret.size());
+List targets = remapFn.apply(obj, taskIds.size());
for (Integer target : targets) {
ret.add(Utils.assignKeyToTask(target, taskIds));
}
[incubator-heron] branch master updated: enablebool (#3056)
This is an automated email from the ASF dual-hosted git repository.
nlu90 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
The following commit(s) were added to refs/heads/master by this push:
new 8218195 enablebool (#3056)
8218195 is described below
commit 8218195d96907ee47fbd8d22ea3b859bbd7049a3
Author: bed debug
AuthorDate: Fri Oct 26 16:49:07 2018 -0700
enablebool (#3056)
---
heron/tools/common/src/python/utils/config.py | 7 ++-
1 file changed, 6 insertions(+), 1 deletion(-)
diff --git a/heron/tools/common/src/python/utils/config.py
b/heron/tools/common/src/python/utils/config.py
index b055a1f..6f52af3 100644
--- a/heron/tools/common/src/python/utils/config.py
+++ b/heron/tools/common/src/python/utils/config.py
@@ -413,7 +413,12 @@ def parse_override_config(namespace):
kv = config.split("=")
if len(kv) != 2:
raise Exception("Invalid config property format (%s) expected key=value"
% config)
-overrides[kv[0]] = kv[1]
+if kv[1] in ['true', 'True', 'TRUE']:
+ overrides[kv[0]] = True
+elif kv[1] in ['false', 'False', 'FALSE']:
+ overrides[kv[0]] = False
+else:
+ overrides[kv[0]] = kv[1]
return overrides
[incubator-heron] branch master updated: Nwang/refactor grouping (#3040)
This is an automated email from the ASF dual-hosted git repository.
nlu90 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
The following commit(s) were added to refs/heads/master by this push:
new 72293cf Nwang/refactor grouping (#3040)
72293cf is described below
commit 72293cf86ef0fce4f53e37107ba88f37c746c618
Author: Ning Wang
AuthorDate: Fri Oct 19 14:35:26 2018 -0700
Nwang/refactor grouping (#3040)
* Refactor grouping in BoltDeclarer
* clean up
* fix direct grouping
* clean up
---
...mStreamGrouping.java => AllStreamGrouping.java} | 37 --
.../heron/api/grouping/CustomStreamGrouping.java | 28 +++-
...reamGrouping.java => DirectStreamGrouping.java} | 38 --
.../heron/api/grouping/FieldsStreamGrouping.java | 54 ++
...reamGrouping.java => GlobalStreamGrouping.java} | 38 --
...StreamGrouping.java => NoneStreamGrouping.java} | 37 --
...eamGrouping.java => ShuffleStreamGrouping.java} | 38 --
...stomStreamGrouping.java => StreamGrouping.java} | 31 +++-
.../apache/heron/api/topology/BoltDeclarer.java| 82 +++---
9 files changed, 189 insertions(+), 194 deletions(-)
diff --git
a/heron/api/src/java/org/apache/heron/api/grouping/CustomStreamGrouping.java
b/heron/api/src/java/org/apache/heron/api/grouping/AllStreamGrouping.java
similarity index 50%
copy from
heron/api/src/java/org/apache/heron/api/grouping/CustomStreamGrouping.java
copy to heron/api/src/java/org/apache/heron/api/grouping/AllStreamGrouping.java
index 627c385..131 100644
--- a/heron/api/src/java/org/apache/heron/api/grouping/CustomStreamGrouping.java
+++ b/heron/api/src/java/org/apache/heron/api/grouping/AllStreamGrouping.java
@@ -19,31 +19,22 @@
package org.apache.heron.api.grouping;
-import java.io.Serializable;
-import java.util.List;
+import org.apache.heron.api.generated.TopologyAPI;
-import org.apache.heron.api.topology.TopologyContext;
+/**
+ * This is the stream grouping strategy that all tuples are transmitted to all
instances of a bolt.
+ */
+public class AllStreamGrouping implements StreamGrouping {
+
+ public AllStreamGrouping() { }
-public interface CustomStreamGrouping extends Serializable {
+ public TopologyAPI.InputStream.Builder buildStream(String componentName,
String streamId) {
+TopologyAPI.InputStream.Builder bldr =
TopologyAPI.InputStream.newBuilder();
- /**
- * Tells the stream grouping at runtime the tasks in the target bolt.
- * This information should be used in chooseTasks to determine the target
tasks.
- *
- * It also tells the grouping the metadata on the stream this grouping will
be used on.
- */
- void prepare(
- TopologyContext context,
- String component,
- String streamId,
- List targetTasks);
+bldr.setStream(
+
TopologyAPI.StreamId.newBuilder().setId(streamId).setComponentName(componentName));
+bldr.setGtype(TopologyAPI.Grouping.ALL);
- /**
- * This function implements a custom stream grouping. It takes in as input
- * the number of tasks in the target bolt in prepare and returns the
- * tasks to send the tuples to.
- *
- * @param values the values to group on
- */
- List chooseTasks(List values);
+return bldr;
+ }
}
diff --git
a/heron/api/src/java/org/apache/heron/api/grouping/CustomStreamGrouping.java
b/heron/api/src/java/org/apache/heron/api/grouping/CustomStreamGrouping.java
index 627c385..57db97d 100644
--- a/heron/api/src/java/org/apache/heron/api/grouping/CustomStreamGrouping.java
+++ b/heron/api/src/java/org/apache/heron/api/grouping/CustomStreamGrouping.java
@@ -19,12 +19,18 @@
package org.apache.heron.api.grouping;
-import java.io.Serializable;
import java.util.List;
+import com.google.protobuf.ByteString;
+
+import org.apache.heron.api.generated.TopologyAPI;
import org.apache.heron.api.topology.TopologyContext;
+import org.apache.heron.api.utils.Utils;
-public interface CustomStreamGrouping extends Serializable {
+/**
+ * This is the interface for user defined stream grouping strategies.
+ */
+public interface CustomStreamGrouping extends StreamGrouping {
/**
* Tells the stream grouping at runtime the tasks in the target bolt.
@@ -46,4 +52,22 @@ public interface CustomStreamGrouping extends Serializable {
* @param values the values to group on
*/
List chooseTasks(List values);
+
+ /**
+ * Build InputStream for CustomStreamGrouping implementations.
+ * @param componentName The parent component of this grouping logic
+ * @param streamId The id of the input stream
+ * @return An InputStream builder to be used by BoltDeclarer
+ */
+ default TopologyAPI.InputStream.Builder buildStream(String componentName,
String streamId) {
+TopologyAPI.InputStream.Builder bldr =
TopologyAPI.InputStream.newBuilder();
+
+bldr.setStream(
+
TopologyAPI.S
svn commit: r30098 - /dev/incubator/heron/heron-0.20.0-incubating-candidate-5/
Author: nlu90 Date: Tue Oct 16 17:38:15 2018 New Revision: 30098 Log: Apache heron 0.20.0-incubating release candidate 5 Added: dev/incubator/heron/heron-0.20.0-incubating-candidate-5/ dev/incubator/heron/heron-0.20.0-incubating-candidate-5/incubator-heron-v-0.20.0-incubating-candidate-5.tar.gz (with props) dev/incubator/heron/heron-0.20.0-incubating-candidate-5/incubator-heron-v-0.20.0-incubating-candidate-5.tar.gz.asc dev/incubator/heron/heron-0.20.0-incubating-candidate-5/incubator-heron-v-0.20.0-incubating-candidate-5.tar.gz.sha512 Added: dev/incubator/heron/heron-0.20.0-incubating-candidate-5/incubator-heron-v-0.20.0-incubating-candidate-5.tar.gz == Binary file - no diff available. Propchange: dev/incubator/heron/heron-0.20.0-incubating-candidate-5/incubator-heron-v-0.20.0-incubating-candidate-5.tar.gz -- svn:mime-type = application/octet-stream Added: dev/incubator/heron/heron-0.20.0-incubating-candidate-5/incubator-heron-v-0.20.0-incubating-candidate-5.tar.gz.asc == --- dev/incubator/heron/heron-0.20.0-incubating-candidate-5/incubator-heron-v-0.20.0-incubating-candidate-5.tar.gz.asc (added) +++ dev/incubator/heron/heron-0.20.0-incubating-candidate-5/incubator-heron-v-0.20.0-incubating-candidate-5.tar.gz.asc Tue Oct 16 17:38:15 2018 @@ -0,0 +1,16 @@ +-BEGIN PGP SIGNATURE- + +iQIcBAABAgAGBQJbxiGIAAoJEBYIdp4fEPZ5oBAP/RnkfxFZ38pDjDL6kGrqqOI2 +K+/+b7M3EIEMLJLeieVlsrM976EZOdgc2hUWmUIio+Zb3Yjk2tVGTTCIbdZNwFr1 +G5qTMgO+tRl8lvWyOSGi8dsvzLfGvr0l/R/oUu6zTLatjvGSaAthqFbszT9n3bFB +P714cwgn0R9m36y0UiOoyDuKVthICy+ot71JNYpU7keNNKm1aOsYimWcxI1/jFXr +PIvL+/y0FOW4suqC7+LwBhWZRUf7hfTJCJ3fvuLV7Bv40mc+2uJ0xVKdF/YjpKw/ +52ed3tt+P5JuO/QzWYWUk5V6g7Am9WbBD3I4VHuVfd4HnID0JY7S2y2HO5acmXFT +XPMawcgLtnQl3n92UWI7LWXd9jeRc3b8DBKqtsGdRewprjjiBrezfMK4N9uvnTjI +m/pMzORNnJJea0FeqgNqCT+C+ZI4DZsa6OU0lB+fGC6MrFzrJQu8Lx5/sM2G20KF +UxpGTC9Q5aj/hHxAkSFbKTc0jpbZ6X2DpjZ9jnp2GCXw8ANLMcEHBy0188puJZko +4R/+UHWB7cySWUqjsLdNv2SA0dfwLSgdJGZ8yUD3Q9ROfYJB3VYvZ0ESwKKE0uXg +C0U4jOcKWfh2sE/WC2VZHGJADoRl0NeAVbKx9DBsJFP3d0RDGVZ7DyOWHvgOqzra +KddKBXnYLgwzEj5dVt32 +=dqx0 +-END PGP SIGNATURE- Added: dev/incubator/heron/heron-0.20.0-incubating-candidate-5/incubator-heron-v-0.20.0-incubating-candidate-5.tar.gz.sha512 == --- dev/incubator/heron/heron-0.20.0-incubating-candidate-5/incubator-heron-v-0.20.0-incubating-candidate-5.tar.gz.sha512 (added) +++ dev/incubator/heron/heron-0.20.0-incubating-candidate-5/incubator-heron-v-0.20.0-incubating-candidate-5.tar.gz.sha512 Tue Oct 16 17:38:15 2018 @@ -0,0 +1 @@ +27890ab30fc3e69b627f47d58d178d1a7dffa9dbe4ebbb5a5aa77caaac882fdc2b6f98b3b76210020db0fa3fd86e294cba214f86072e449837e1b7615cd6124a incubator-heron-v-0.20.0-incubating-candidate-5.tar.gz
[incubator-heron] tag v-0.20.0-incubating-candidate-5 created (now 45043bb)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a change to tag v-0.20.0-incubating-candidate-5 in repository https://gitbox.apache.org/repos/asf/incubator-heron.git. at 45043bb (commit) No new revisions were added by this update.
[incubator-heron] branch release/v-0.20.0-incubating created (now 45043bb)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a change to branch release/v-0.20.0-incubating in repository https://gitbox.apache.org/repos/asf/incubator-heron.git. at 45043bb Fix annotation processor (#3038) No new revisions were added by this update.
[incubator-heron] branch release/v-0.20.0-incubating deleted (was a468699)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a change to branch release/v-0.20.0-incubating in repository https://gitbox.apache.org/repos/asf/incubator-heron.git. was a468699 fix to generate correct artificats [https://github.com/apache/incubat… (#3014) The revisions that were on this branch are still contained in other references; therefore, this change does not discard any commits from the repository.
[incubator-heron] branch master updated: Fix annotation processor (#3038)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-heron.git The following commit(s) were added to refs/heads/master by this push: new 45043bb Fix annotation processor (#3038) 45043bb is described below commit 45043bb6dcef1e8089c0834f17f8be0cc3f451d3 Author: Ning Wang AuthorDate: Wed Oct 10 19:25:40 2018 -0700 Fix annotation processor (#3038) * Fix annotation processor Comments is not allowed. Got rid of the license info and exluce it from rat * clean up --- .rat-excludes | 6 +++--- .../services/javax.annotation.processing.Processor| 19 --- 2 files changed, 3 insertions(+), 22 deletions(-) diff --git a/.rat-excludes b/.rat-excludes index 2db5fdc..998fda4 100644 --- a/.rat-excludes +++ b/.rat-excludes @@ -1,5 +1,7 @@ +# Files start with . \..* +# Data/doc files .*\.conf .*\.json .*\.md @@ -8,12 +10,9 @@ .*\.patch _helpers\.tpl -ADOPTERS\.md -CONTRIBUTING\.md autogen\.sh description heron\.iml -test\.conf # UI resources jquery.*\.js @@ -31,6 +30,7 @@ underscore-min.*\.map bazel-.* m4 website +META-INF # Thirdparty cpplint\.py diff --git a/heron/api/src/META-INF/services/javax.annotation.processing.Processor b/heron/api/src/META-INF/services/javax.annotation.processing.Processor index c7ce851..1b7cc84 100644 --- a/heron/api/src/META-INF/services/javax.annotation.processing.Processor +++ b/heron/api/src/META-INF/services/javax.annotation.processing.Processor @@ -1,20 +1 @@ -/** - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - org.apache.heron.classification.HeronAnnotationProcessor
[incubator-heron] branch master updated: Refactor heron_executor to make it easier to customize (#3043)
This is an automated email from the ASF dual-hosted git repository.
nlu90 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
The following commit(s) were added to refs/heads/master by this push:
new 65f90e0 Refactor heron_executor to make it easier to customize (#3043)
65f90e0 is described below
commit 65f90e039291a63de4ceb3f2df45fc3e2ce4357f
Author: Ning Wang
AuthorDate: Wed Oct 10 13:57:53 2018 -0700
Refactor heron_executor to make it easier to customize (#3043)
* Refactor heron_executor to make it easier to customize
* fix unittest
---
heron/executor/src/python/heron_executor.py| 249 -
.../tests/python/heron_executor_unittest.py| 4 +-
2 files changed, 149 insertions(+), 104 deletions(-)
diff --git a/heron/executor/src/python/heron_executor.py
b/heron/executor/src/python/heron_executor.py
index 3f44b53..b411b3e 100755
--- a/heron/executor/src/python/heron_executor.py
+++ b/heron/executor/src/python/heron_executor.py
@@ -171,7 +171,7 @@ class HeronExecutor(object):
""" Heron executor is a class that is responsible for running each of the
process on a given
container. Based on the container id and the instance distribution, it
determines if the container
is a master node or a worker node and it starts processes accordingly."""
- def init_parsed_args(self, parsed_args):
+ def init_from_parsed_args(self, parsed_args):
""" initialize from parsed arguments """
self.shard = parsed_args.shard
self.topology_name = parsed_args.topology_name
@@ -263,7 +263,7 @@ class HeronExecutor(object):
def __init__(self, args, shell_env):
parsed_args = self.parse_args(args)
-self.init_parsed_args(parsed_args)
+self.init_from_parsed_args(parsed_args)
self.shell_env = shell_env
self.max_runs = 100
@@ -552,86 +552,117 @@ class HeronExecutor(object):
def _get_java_instance_cmd(self, instance_info):
retval = {}
# TO DO (Karthik) to be moved into keys and defaults files
-code_cache_size_mb = 64
-java_metasize_mb = 128
-
-java_version = self._get_jvm_version()
-java_metasize_param = 'MetaspaceSize'
-if java_version.startswith("1.7") or \
-java_version.startswith("1.6") or \
-java_version.startswith("1.5"):
- java_metasize_param = 'PermSize'
+instance_class_name = 'org.apache.heron.instance.HeronInstance'
if self.jvm_remote_debugger_ports and \
(len(instance_info) > len(self.jvm_remote_debugger_ports)):
Log.warn("Not enough remote debugger ports for all instances!")
+# Create id to java command map
for (instance_id, component_name, global_task_id, component_index) in
instance_info:
- total_jvm_size = int(self.component_ram_map[component_name] / (1024 *
1024))
- heap_size_mb = total_jvm_size - code_cache_size_mb - java_metasize_mb
- Log.info("component name: %s, RAM request: %d, total JVM size: %dM, "
- "cache size: %dM, metaspace size: %dM"
- % (component_name, self.component_ram_map[component_name],
- total_jvm_size, code_cache_size_mb, java_metasize_mb))
- xmn_size = int(heap_size_mb / 2)
- instance_cmd = [os.path.join(self.heron_java_home, 'bin/java'),
- '-Xmx%dM' % heap_size_mb,
- '-Xms%dM' % heap_size_mb,
- '-Xmn%dM' % xmn_size,
- '-XX:Max%s=%dM' % (java_metasize_param,
java_metasize_mb),
- '-XX:%s=%dM' % (java_metasize_param, java_metasize_mb),
- '-XX:ReservedCodeCacheSize=%dM' % code_cache_size_mb,
- '-XX:+CMSScavengeBeforeRemark',
- '-XX:TargetSurvivorRatio=90',
- '-XX:+PrintCommandLineFlags',
- '-verbosegc',
- '-XX:+PrintGCDetails',
- '-XX:+PrintGCTimeStamps',
- '-XX:+PrintGCDateStamps',
- '-XX:+PrintGCCause',
- '-XX:+UseGCLogFileRotation',
- '-XX:NumberOfGCLogFiles=5',
- '-XX:GCLogFileSize=100M',
- '-XX:+PrintPromotionFailure',
- '-XX:+PrintTenuringDistribution',
- '-XX:+PrintHeapAtGC',
- '-XX:+HeapDumpOnOutOfMemoryError',
- '-XX:+UseConcMarkSweepGC',
- '-XX:ParallelGCThreads=4',
- '-Xloggc:log-files/gc.%s.log' % instance_id]
-
+ # Append debugger ports
remote_debugger_port = None
if self.jvm_remote_debugger_ports:
remote_debugger_port = self.jvm_remote_debugger_ports.pop()
-
instance_cmd.append('-agentlib:jdwp=tra
[incubator-heron] branch master updated: Remove libunwind patches (#3041)
This is an automated email from the ASF dual-hosted git repository.
nlu90 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
The following commit(s) were added to refs/heads/master by this push:
new 18bf8d8 Remove libunwind patches (#3041)
18bf8d8 is described below
commit 18bf8d871aa274040e677b0ececc862175db184b
Author: Ning Wang
AuthorDate: Wed Oct 10 10:37:06 2018 -0700
Remove libunwind patches (#3041)
* Remove libunwind patches
* disable minidebuginfo
* Clean up
---
third_party/libunwind/BUILD| 7 -
third_party/libunwind/libunwind-1.1-cache.patch| 39 -
third_party/libunwind/libunwind-1.1-config.patch | 13 --
.../libunwind/libunwind-1.1-lzma-link.patch| 165 -
third_party/libunwind/libunwind.BUILD | 19 ---
5 files changed, 243 deletions(-)
diff --git a/third_party/libunwind/BUILD b/third_party/libunwind/BUILD
index 9a113a3..cbcf675 100644
--- a/third_party/libunwind/BUILD
+++ b/third_party/libunwind/BUILD
@@ -2,13 +2,6 @@ licenses(["permissive"])
package(default_visibility = ["//visibility:public"])
-exports_files([
-"libunwind-1.1-cache.patch",
-"libunwind-1.1-config.patch",
-"libunwind-1.1-lzma-link.patch",
-"libunwind.BUILD",
-])
-
filegroup(
name = "libunwind-files",
srcs = ["@org_nongnu_libunwind//:libunwind-files"],
diff --git a/third_party/libunwind/libunwind-1.1-cache.patch
b/third_party/libunwind/libunwind-1.1-cache.patch
deleted file mode 100644
index f1bf95e..000
--- a/third_party/libunwind/libunwind-1.1-cache.patch
+++ /dev/null
@@ -1,39 +0,0 @@
-diff --git autom4te.cache/output.0 autom4te.cache/output.0
-index ee2c0fc..5cee198 100644
autom4te.cache/output.0
-+++ autom4te.cache/output.0
-@@ -16959,7 +16959,7 @@ fi
- { $as_echo "$as_me:${as_lineno-$LINENO}: result:
$ac_cv_lib_lzma_lzma_mf_is_supported" >&5
- $as_echo "$ac_cv_lib_lzma_lzma_mf_is_supported" >&6; }
- if test "x$ac_cv_lib_lzma_lzma_mf_is_supported" = xyes; then :
-- LIBLZMA=-lzma
-+ LIBLZMA=-llzma
-
-
- $as_echo "@%:@define HAVE_LZMA 1" >>confdefs.h
-diff --git autom4te.cache/output.1 autom4te.cache/output.1
-index bb5278a..e801cd0 100644
autom4te.cache/output.1
-+++ autom4te.cache/output.1
-@@ -16955,7 +16955,7 @@ fi
- { $as_echo "$as_me:${as_lineno-$LINENO}: result:
$ac_cv_lib_lzma_lzma_mf_is_supported" >&5
- $as_echo "$ac_cv_lib_lzma_lzma_mf_is_supported" >&6; }
- if test "x$ac_cv_lib_lzma_lzma_mf_is_supported" = xyes; then :
-- LIBLZMA=-lzma
-+ LIBLZMA=-llzma
-
-
- $as_echo "@%:@define HAVE_LZMA 1" >>confdefs.h
-diff --git autom4te.cache/traces.1 autom4te.cache/traces.1
-index 8e83080..3ee4b46 100644
autom4te.cache/traces.1
-+++ autom4te.cache/traces.1
-@@ -905,7 +905,7 @@ m4trace:configure.ac:233: -1-
AC_DEFINE_TRACE_LITERAL([CONFIG_MSABI_SUPPORT])
- m4trace:configure.ac:233: -1- m4_pattern_allow([^CONFIG_MSABI_SUPPORT$])
- m4trace:configure.ac:233: -1- AH_OUTPUT([CONFIG_MSABI_SUPPORT], [/* Support
for Microsoft ABI extensions */
- @%:@undef CONFIG_MSABI_SUPPORT])
--m4trace:configure.ac:242: -1- AC_SUBST([LIBLZMA], [-lzma])
-+m4trace:configure.ac:242: -1- AC_SUBST([LIBLZMA], [-llzma])
- m4trace:configure.ac:242: -1- AC_SUBST_TRACE([LIBLZMA])
- m4trace:configure.ac:242: -1- m4_pattern_allow([^LIBLZMA$])
- m4trace:configure.ac:242: -1- AC_DEFINE_TRACE_LITERAL([HAVE_LZMA])
diff --git a/third_party/libunwind/libunwind-1.1-config.patch
b/third_party/libunwind/libunwind-1.1-config.patch
deleted file mode 100644
index c54ca1d..000
--- a/third_party/libunwind/libunwind-1.1-config.patch
+++ /dev/null
@@ -1,13 +0,0 @@
-diff --git configure configure
-index 94f0f27..982fe53 100755
configure
-+++ configure
-@@ -16955,7 +16955,7 @@ fi
- { $as_echo "$as_me:${as_lineno-$LINENO}: result:
$ac_cv_lib_lzma_lzma_mf_is_supported" >&5
- $as_echo "$ac_cv_lib_lzma_lzma_mf_is_supported" >&6; }
- if test "x$ac_cv_lib_lzma_lzma_mf_is_supported" = xyes; then :
-- LIBLZMA=-lzma
-+ LIBLZMA=-llzma
-
-
- $as_echo "#define HAVE_LZMA 1" >>confdefs.h
diff --git a/third_party/libunwind/libunwind-1.1-lzma-link.patch
b/third_party/libunwind/libunwind-1.1-lzma-link.patch
deleted file mode 100644
index 9a01395..000
--- a/third_party/libunwind/libunwind-1.1-lzma-link.patch
+++ /dev/null
@@ -1,165 +0,0 @@
-diff -Nrup libunwind-1.1.orig/configure.ac libunwind-1.1/configure.ac
libunwind-1.1.orig/configure.ac2016-12-19 13:56:15.758086918 -0800
-+++ libunwind-1.1/configure.ac 2016-12-19 13:56:34.731335116 -0800
-@@ -234,19 +234,21 @@ if test x$enable_msabi_support = xyes; t
- fi
- AC_MSG_RESULT([$enable_msabi_support]
[incubator-heron] branch master updated: Fix Topology-API window documentation (#3039)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-heron.git The following commit(s) were added to refs/heads/master by this push: new fb3649e Fix Topology-API window documentation (#3039) fb3649e is described below commit fb3649e52e47c6a7b4c3ef4a332fd3554e23818a Author: Eren Avsarogullari AuthorDate: Tue Oct 9 21:56:16 2018 +0100 Fix Topology-API window documentation (#3039) --- website/content/docs/concepts/topologies.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/content/docs/concepts/topologies.md b/website/content/docs/concepts/topologies.md index a3dc642..34dad8c 100644 --- a/website/content/docs/concepts/topologies.md +++ b/website/content/docs/concepts/topologies.md @@ -101,7 +101,7 @@ For sliding windows, you need to specify two things: 1. The length or duration of the window (length if the window is a [count window](#count-windows), duration if the window is a [time window](#time-windows)). 1. The sliding interval, which determines when the window slides, i.e. at what point during the current window the new window begins. -In the figure above, the duration of the window is 10 seconds, while the sliding interval is 5 seconds. Each new window begins five seconds into the current window. +In the figure above, the duration of the window is 20 seconds, while the sliding interval is 10 seconds. Each new window begins five seconds into the current window. > With sliding time windows, data can be processed in more than one window. > Tuples 3, 4, and 5 above are processed in both window 1 and window 2 while > tuples 6, 7, and 8 are processed in both window 2 and window 3. @@ -119,7 +119,7 @@ Here, the sliding interval determines that a new window is always created 12 sec Tumbling windows don't overlap because a new window doesn't begin until the current window has elapsed. For tumbling windows, you only need to specify the length or duration of the window but *no sliding interval*. -> With tumbling windows, data are *never* processed in more than one window because the windows never overlap. +> With tumbling windows, data are *never* processed in more than one window because the windows never overlap. Also, in the figure above, the duration of the window is 20 seconds. ### Count windows
[incubator-heron] branch master updated: Nwang/update thirdparty license (#3042)
This is an automated email from the ASF dual-hosted git repository.
nlu90 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
The following commit(s) were added to refs/heads/master by this push:
new 3866944 Nwang/update thirdparty license (#3042)
3866944 is described below
commit 38669441ae934fdd53d15d51acd64db2602ec06a
Author: Ning Wang
AuthorDate: Tue Oct 9 13:54:18 2018 -0700
Nwang/update thirdparty license (#3042)
* Update thrift party licenses in LICENSE
* clean up
---
LICENSE | 90 ---
docker/base/scripts/generate-zookeeper-config.sh | 14 ++--
heronpy/api/cloudpickle.py | 16 -
release/maven/maven-pom-version.sh | 2 -
third_party/gtest/gtest.BUILD| 91 +---
tools/rules/build_defs.bzl | 25 ---
6 files changed, 101 insertions(+), 137 deletions(-)
diff --git a/LICENSE b/LICENSE
index 8769d29..cb67dd0 100644
--- a/LICENSE
+++ b/LICENSE
@@ -214,21 +214,89 @@ Third party Apache 2.0 licenses
The following components are provided under the Apache 2.0 License.
-See project link for details. The text of each license is also included
-at licenses/LICENSE-[project].txt.
-
-(ALv2 License) React
(https://github.com/facebook/react/blob/0.10-stable/LICENSE)
-(ALv2 License) Bootstrap(v2.2.2)
-(ALv2 License) Bootstrap(v3.0.0)
-(ALv2 License) JQuery(https://jquery.org/license/)
+See project link for details.
+
+Bootstrap(v2.0.3, https://github.com/twbs/bootstrap/blob/v2.0.3/LICENSE)
+ -> heron/shell/assets/bootstrap.css
+Bootstrap(v2.2.2, https://github.com/twbs/bootstrap/blob/v2.2.2/LICENSE)
+ -> heron/tools/ui/resources/static/js/bootstrap.js
+Bootstrap(v3.0.0, https://github.com/twbs/bootstrap/blob/v3.0.0/LICENSE)
+ -> heron/tools/ui/resources/static/css/bootstrap.css
+JQuery(https://jquery.org/license/)
+ -> heron/shell/assets/jquery.js
+ -> heron/tools/ui/resources/static/js/jquery.min.js
+JSXTransformer(v0.10.0)
+ -> heron/tools/ui/resources/static/js/JSXTransformer.0.10.0.js
+React (v0.10.0, https://github.com/facebook/react/blob/0.10-stable/LICENSE)
+ -> heron/tools/ui/resources/static/js/react.0.10.0.js
+
bazel_jar_jar(https://github.com/johnynek/bazel_jar_jar/blob/master/LICENSE)
+ -> tools/rules/jarjar_rules.bzl
+gerrit(https://gerrit.googlesource.com/gerrit/+/master/COPYING)
+ -> tools/rules/javadoc.bzl
+heronpy(Copyright - Parsely, Inc.)
+ -> heronpy/api/bolt/base_bolt.py
+ -> heronpy/api/bolt/bolt.py
+ -> heronpy/api/bolt/base_component.py
+ -> heronpy/api/bolt/base_spout.py
+ -> heronpy/api/bolt/stream.py
+ -> heronpy/api/bolt/generator.py
+ -> heronpy/api/bolt/transformoperator.py
+
k8s-zookeeper-docker(https://github.com/muxinc/k8s-zookeeper-docker/blob/master/LICENSE)
+ -> docker/base/scripts/generate-zookeeper-config.sh
-MIT licenses
+Third party MIT licenses
The following components are provided under the MIT License. See project link
for details.
-The text of each license is also included at licenses/LICENSE-[project].txt.
-(MIT License) Bootstrap(v3.1.1)
(https://github.com/twbs/bootstrap/blob/master/LICENSE)
-(MIT License) JQuery (https://jquery.org/license/)
+Bootstrap(v3.1.1, https://github.com/twbs/bootstrap/blob/v3.1.1/LICENSE)
+ -> heron/tools/ui/resources/static/js/bootstrap.min.js
+ -> heron/tools/ui/resources/static/css/bootstrap.min.css
+ ->
heron/tools/ui/resources/static/fonts/glyphicons-halflings-regular.{eot, svg,
ttf, woff}
+JSXTransformer(v0.10.0,
https://github.com/cronn-de/jsxtransformer/blob/master/LICENSE)
+ -> heron/tools/ui/resources/static/js/JSXTransformer.0.10.0.js
+Underscore.js(v1.6.0,
https://github.com/jashkenas/underscore/blob/master/LICENSE)
+ -> heron/tools/ui/resources/static/js/underscore-min.1.6.0.js
+ -> heron/tools/ui/resources/static/js/underscore-min.map
+d3-tip(v0.6.3, https://github.com/Caged/d3-tip/blob/master/LICENSE)
+ -> heron/tools/ui/resources/static/js/d3-tip.min.0.6.3.js
+html5shiv(v3.6.2pre,
https://github.com/aFarkas/html5shiv/blob/master/MIT%20and%20GPL2%20licenses.md)
+ -> heron/tools/ui/resources/static/js/html5-trunk.js
+jQuery UI(v1.8.23,
https://github.com/jquery/jquery-ui/blob/master/LICENSE.txt)
+ -> heron/tools/ui/resources/static/js/jquery-ui-1.8.23.custom.min.js
+list.js(v1.1.1, https://github.com/javve/list.js/blob/master/LICENSE)
+ -&g
[incubator-heron] tag 0.17.9 created (now fd7fde0)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a change to tag 0.17.9 in repository https://gitbox.apache.org/repos/asf/incubator-heron.git. at fd7fde0 (commit) No new revisions were added by this update.
[incubator-heron] tag 0.17.9 deleted (was 3822400)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a change to tag 0.17.9 in repository https://gitbox.apache.org/repos/asf/incubator-heron.git. *** WARNING: tag 0.17.9 was deleted! *** was 3822400 update README.md (#2835) The revisions that were on this tag are still contained in other references; therefore, this change does not discard any commits from the repository.
[incubator-heron] 01/01: heron shell access directory
This is an automated email from the ASF dual-hosted git repository.
nlu90 pushed a commit to branch release/0.17.9
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
commit fd7fde0518562d60a7a85679715e2b15ee464d37
Author: Neng Lu
AuthorDate: Tue Sep 18 11:58:21 2018 -0700
heron shell access directory
---
heron/shell/src/python/handlers/browsehandler.py| 4 +++-
heron/shell/src/python/handlers/downloadhandler.py | 2 +-
heron/shell/src/python/handlers/filedatahandler.py | 4 +++-
heron/shell/src/python/handlers/filehandler.py | 4 +++-
heron/shell/src/python/handlers/filestatshandler.py | 5 +++--
heron/shell/src/python/utils.py | 8
6 files changed, 21 insertions(+), 6 deletions(-)
diff --git a/heron/shell/src/python/handlers/browsehandler.py
b/heron/shell/src/python/handlers/browsehandler.py
index 80606e6..d0f050a 100644
--- a/heron/shell/src/python/handlers/browsehandler.py
+++ b/heron/shell/src/python/handlers/browsehandler.py
@@ -34,11 +34,13 @@ class BrowseHandler(tornado.web.RequestHandler):
''' get method '''
if not path:
path = "."
-if path.startswith("/"):
+
+if not utils.check_path(path):
self.write("Only relative paths are allowed")
self.set_status(403)
self.finish()
return
+
t = Template(utils.get_asset("browse.html"))
args = dict(
path=path,
diff --git a/heron/shell/src/python/handlers/downloadhandler.py
b/heron/shell/src/python/handlers/downloadhandler.py
index ed30ade..dcf652b 100644
--- a/heron/shell/src/python/handlers/downloadhandler.py
+++ b/heron/shell/src/python/handlers/downloadhandler.py
@@ -37,7 +37,7 @@ class DownloadHandler(tornado.web.RequestHandler):
self.connection_closed = False
self.set_header("Content-Disposition", "attachment")
-if path.startswith("/"):
+if not utils.check_path(path):
self.write("Only relative paths are allowed")
self.set_status(403)
self.finish()
diff --git a/heron/shell/src/python/handlers/filedatahandler.py
b/heron/shell/src/python/handlers/filedatahandler.py
index 271d152..9c57a41 100644
--- a/heron/shell/src/python/handlers/filedatahandler.py
+++ b/heron/shell/src/python/handlers/filedatahandler.py
@@ -32,11 +32,13 @@ class FileDataHandler(tornado.web.RequestHandler):
""" get method """
if path is None:
return {}
-if path.startswith("/"):
+
+if not utils.check_path(path):
self.write("Only relative paths are allowed")
self.set_status(403)
self.finish()
return
+
offset = self.get_argument("offset", default=-1)
length = self.get_argument("length", default=-1)
if not os.path.isfile(path):
diff --git a/heron/shell/src/python/handlers/filehandler.py
b/heron/shell/src/python/handlers/filehandler.py
index 51abaac..bb35ddf 100644
--- a/heron/shell/src/python/handlers/filehandler.py
+++ b/heron/shell/src/python/handlers/filehandler.py
@@ -35,11 +35,13 @@ class FileHandler(tornado.web.RequestHandler):
self.write("No such file")
self.finish()
return
-if path.startswith("/"):
+
+if not utils.check_path(path):
self.write("Only relative paths are allowed")
self.set_status(403)
self.finish()
return
+
args = dict(
filename=path,
jquery=utils.get_asset("jquery.js"),
diff --git a/heron/shell/src/python/handlers/filestatshandler.py
b/heron/shell/src/python/handlers/filestatshandler.py
index 084bbfd..2f19240 100644
--- a/heron/shell/src/python/handlers/filestatshandler.py
+++ b/heron/shell/src/python/handlers/filestatshandler.py
@@ -38,11 +38,12 @@ class FileStatsHandler(tornado.web.RequestHandler):
# of the dir that heron-shell is running in. This ensures
# sandboxing. So we don't allow absolute paths and parent
# accessing.
-if path.startswith("/") or ".." in path:
- self.write("Only relative paths inside job dir are allowed")
+if not utils.check_path(path):
+ self.write("Only relative paths are allowed")
self.set_status(403)
self.finish()
return
+
listing = utils.get_listing(path)
file_stats = {}
for fn in listing:
diff --git a/heron/shell/src/python/utils.py b/heron/shell/src/python/utils.py
index 6070227..13eb829 100644
--- a/heron/shell/src/python/utils.py
+++ b/heron/shell/src/python/utils.py
@@ -194,3 +194,11 @@ def get_container_id(instance_id):
def get_asset(asset_name):
''' get assset '''
return pkgutil.get_data("heron.shell", os.path.join("assets", asset_name))
+
+def check_path(path):
+ """
+ file path should be a relative path without ".." in it
+ :param path: file path
+ :return: true if the path is relative and doesn't contain ".."
+ """
+ return not path.startswith("/") and ".." not in path
[incubator-heron] branch release/0.17.9 created (now fd7fde0)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a change to branch release/0.17.9 in repository https://gitbox.apache.org/repos/asf/incubator-heron.git. at fd7fde0 heron shell access directory This branch includes the following new commits: new fd7fde0 heron shell access directory The 1 revisions listed above as "new" are entirely new to this repository and will be described in separate emails. The revisions listed as "add" were already present in the repository and have only been added to this reference.
[incubator-heron] tag 0.17.9 created (now 3822400)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a change to tag 0.17.9 in repository https://gitbox.apache.org/repos/asf/incubator-heron.git. at 3822400 (commit) No new revisions were added by this update.
[incubator-heron] branch release/0.17.8 created (now 3822400)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a change to branch release/0.17.8 in repository https://gitbox.apache.org/repos/asf/incubator-heron.git. at 3822400 update README.md (#2835) No new revisions were added by this update.
[incubator-heron] tag 0.17.9 deleted (was a468699)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a change to tag 0.17.9 in repository https://gitbox.apache.org/repos/asf/incubator-heron.git. *** WARNING: tag 0.17.9 was deleted! *** was a468699 fix to generate correct artificats [https://github.com/apache/incubat… (#3014) The revisions that were on this tag are still contained in other references; therefore, this change does not discard any commits from the repository.
[incubator-heron] tag 0.17.9 created (now a468699)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a change to tag 0.17.9 in repository https://gitbox.apache.org/repos/asf/incubator-heron.git. at a468699 (commit) No new revisions were added by this update.
svn commit: r29631 - /dev/incubator/heron/heron-0.20.0-incubating-candidate-4/
Author: nlu90 Date: Mon Sep 24 18:13:48 2018 New Revision: 29631 Log: Apache heron 0.20.0-incubating release candidate 4 Added: dev/incubator/heron/heron-0.20.0-incubating-candidate-4/ dev/incubator/heron/heron-0.20.0-incubating-candidate-4/incubator-heron-v-0.20.0-incubating-candidate-4.tar.gz (with props) dev/incubator/heron/heron-0.20.0-incubating-candidate-4/incubator-heron-v-0.20.0-incubating-candidate-4.tar.gz.asc dev/incubator/heron/heron-0.20.0-incubating-candidate-4/incubator-heron-v-0.20.0-incubating-candidate-4.tar.gz.sha512 Added: dev/incubator/heron/heron-0.20.0-incubating-candidate-4/incubator-heron-v-0.20.0-incubating-candidate-4.tar.gz == Binary file - no diff available. Propchange: dev/incubator/heron/heron-0.20.0-incubating-candidate-4/incubator-heron-v-0.20.0-incubating-candidate-4.tar.gz -- svn:mime-type = application/octet-stream Added: dev/incubator/heron/heron-0.20.0-incubating-candidate-4/incubator-heron-v-0.20.0-incubating-candidate-4.tar.gz.asc == --- dev/incubator/heron/heron-0.20.0-incubating-candidate-4/incubator-heron-v-0.20.0-incubating-candidate-4.tar.gz.asc (added) +++ dev/incubator/heron/heron-0.20.0-incubating-candidate-4/incubator-heron-v-0.20.0-incubating-candidate-4.tar.gz.asc Mon Sep 24 18:13:48 2018 @@ -0,0 +1,16 @@ +-BEGIN PGP SIGNATURE- + +iQIcBAABAgAGBQJbqSjvAAoJEBYIdp4fEPZ5skMP/RtdvJw/KE5Ar1juE/wEcEMc +C6fSeLGXUXaZPOpK/pFO7kXKm3NL7xXExO1i5k1yeoZT6jTlpnRtrHUARPYUuAd1 +HYklTfokxpBR1qxxScwdjqLC+afFXo+YVWlSkk4IOLemzg22bmlOxdsaztVuUjJ0 +NHvAcnVfJjdwLaXO6ZMqSQvuMLenbFVJNW8rNXMSlQktYFAjLJlQBtdzRnHY2WQl +vL4wE0WuNZtx1aQuYk+P+Z47G0l5F1ITN9i2+wscXR0150MiTTRCjXInl6Qv8o/6 +cvtRQgvYijZVHXv2vn2dcnXmGTlX7pzBg7BLON0BwEca8L5dRD4DsANP67x8xL9b +Nj2vnL/O9s/okgRQXIohvvrAjkvmBv1Q/nFi79Zz26v0CHIo2BU7epYjkWseqoGp +ZGY9gGHdj0Xo9rtMenrYu7Tyq4GZgH069Hnx5O02MDNLYEItemKO+KPW+a7Ga8RG +WMWXQ2t3+V/J8YUVbO9QPKyChE/OtrQ2gwyxjMZc/JswHl74eU7GWGQXW0SxBqQP +EY+7iYLbcjCVsHj4Uls3aqN/Oj7njmFF9gMaYmBY4DaZ8j2k7bIKFgJ4qdSEHywh +jFC1PYdfQRevLEF5fCVXqmgkukNPhIUTIkJmnBMFTlwyewAZdfsCExn/zsqtePJo +geloQBPKkQNBJbgcF06b +=ihzc +-END PGP SIGNATURE- Added: dev/incubator/heron/heron-0.20.0-incubating-candidate-4/incubator-heron-v-0.20.0-incubating-candidate-4.tar.gz.sha512 == --- dev/incubator/heron/heron-0.20.0-incubating-candidate-4/incubator-heron-v-0.20.0-incubating-candidate-4.tar.gz.sha512 (added) +++ dev/incubator/heron/heron-0.20.0-incubating-candidate-4/incubator-heron-v-0.20.0-incubating-candidate-4.tar.gz.sha512 Mon Sep 24 18:13:48 2018 @@ -0,0 +1 @@ +b207bd181dc1960abc7393bb2af4f8c5428676174693cecd3b95216af1d3c6dcec76a07d5e91d0a1959f5523ca3fe79f310b8f9ad797252441d8e1be23aadca2 incubator-heron-v-0.20.0-incubating-candidate-4.tar.gz
[incubator-heron] tag v-0.20.0-incubating-candidate-4 created (now a468699)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a change to tag v-0.20.0-incubating-candidate-4 in repository https://gitbox.apache.org/repos/asf/incubator-heron.git. at a468699 (commit) No new revisions were added by this update.
[incubator-heron] branch release/v-0.20.0-incubating updated (7fb0df3 -> a468699)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a change to branch release/v-0.20.0-incubating in repository https://gitbox.apache.org/repos/asf/incubator-heron.git. from 7fb0df3 Fix RAM config in example code (#3006) add 03ba3f6 Yaoli/fix stateful test (#3004) add 4328ab4 Fix path and package name in docs/comments/tests (#3007) add c08bb8d revert heron_excutor.py - delete unnecessary logs (#3008) add 4d2677e Fix javadoc errors (#3017) add bf2c73a Update README(updated urls and added developer link) (#3010) add 31883cd fix healthmgr unit tests (#3018) add 774ba38 Log executor command line arguments (#3020) add fd904d7 Fix build of eco/tests/... and scripts/release/... (#3012) add 9089984 fixstyle (#3023) add f6ee1c1 use RoundRobinPacking algorithm by default (#3022) add a7eec11 run integration tests parallelly (#3024) add fa4ae37 add max thread limit in integration test (#3025) add 1b51286 fix integration test failure with topo parallelism update (#3026) add cf79c83 Add license headers and rat-exception file (#3011) add aa12d88 heron shell access directory add db3f1c4 refactor JavaSerializer for closing input/output stream add 72509b9 Revert "refactor JavaSerializer for closing input/output stream" add ebda8e7 refactor JavaSerializer for proper closing input/output streams (#3031) add a468699 fix to generate correct artificats [https://github.com/apache/incubat… (#3014) No new revisions were added by this update. Summary of changes: .rat-excludes | 38 +++ README.md | 35 +++--- WORKSPACE | 17 +++ config/configure.ac| 17 +++ config/heron.def | 19 deploy/kubernetes/helm/.helmignore | 17 +++ deploy/kubernetes/helm/Chart.yaml.template | 17 +++ deploy/kubernetes/helm/values.yaml.template| 17 +++ docker/Readme.md | 2 +- docker/base/Dockerfile.base.debian9| 17 +++ docker/compile/Dockerfile.centos7 | 17 +++ docker/compile/Dockerfile.debian9 | 17 +++ docker/compile/Dockerfile.ubuntu14.04 | 17 +++ docker/compile/Dockerfile.ubuntu16.04 | 17 +++ docker/dist/Dockerfile.dist.centos7| 17 +++ docker/dist/Dockerfile.dist.debian9| 17 +++ docker/dist/Dockerfile.dist.ubuntu14.04| 17 +++ docker/dist/Dockerfile.dist.ubuntu16.04| 17 +++ .../eco/builder/storm/StormSpoutBuilderTest.java | 2 +- .../eco/builder/storm/StormStreamBuilderTest.java | 2 +- .../services/javax.annotation.processing.Processor | 19 .../api/src/java/org/apache/heron/api/Config.java | 4 +- .../org/apache/heron/api/bolt/IWindowedBolt.java | 2 +- .../heron/api/serializer/JavaSerializer.java | 14 ++- .../apache/heron/api/windowing/WindowManager.java | 4 +- .../evictors/WatermarkTimeEvictionPolicy.java | 2 +- .../GeneralReduceByKeyAndWindowStreamlet.java | 2 +- .../streamlet/impl/streamlets/JoinStreamlet.java | 2 +- .../streamlets/ReduceByKeyAndWindowStreamlet.java | 2 +- heron/api/src/java/shade.conf | 17 +++ heron/ckptmgr/src/java/shade.conf | 17 +++ heron/common/src/cpp/network/misc/tests.proto | 19 +++- heron/common/src/python/__init__.py| 16 +++ heron/common/tests/cpp/network/unittests.proto | 17 +++ heron/config/src/yaml/conf/aurora/downloader.yaml | 17 +++ heron/config/src/yaml/conf/aurora/heron.aurora | 17 +++ heron/config/src/yaml/conf/aurora/packing.yaml | 4 +- .../config/src/yaml/conf/examples/downloader.yaml | 17 +++ .../src/yaml/conf/kubernetes/downloader.yaml | 17 +++ heron/config/src/yaml/conf/kubernetes/packing.yaml | 4 +- heron/config/src/yaml/conf/local/downloader.yaml | 17 +++ heron/config/src/yaml/conf/local/packing.yaml | 4 +- heron/config/src/yaml/conf/localzk/downloader.yaml | 17 +++ heron/config/src/yaml/conf/localzk/packing.yaml| 4 +- heron/config/src/yaml/conf/marathon/packing.yaml | 4 +- heron/config/src/yaml/conf/mesos/packing.yaml | 4 +- heron/config/src/yaml/conf/nomad/downloader.yaml | 17 +++ heron/config/src/yaml/conf/nomad/packing.yaml | 4 +- heron/config/src/yaml/conf/sandbox/downloader.yaml | 17 +++ heron/config/src/yaml/conf/sandbox/packing.yaml| 4 +- heron/config/src/yaml/conf/slurm/packing.yaml | 4 +- .../src/yaml/conf/standalone/downloader.yaml | 17 +++ heron/config/src/yaml/conf/standalone/packing.yaml | 4 +- .../src/yaml/conf/standalone/resources/master.hcl | 17 +++ .../standalone
[incubator-heron] branch master updated: refactor JavaSerializer for proper closing input/output streams (#3031)
This is an automated email from the ASF dual-hosted git repository.
nlu90 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
The following commit(s) were added to refs/heads/master by this push:
new ebda8e7 refactor JavaSerializer for proper closing input/output
streams (#3031)
ebda8e7 is described below
commit ebda8e761b7b006a95b14bc154b3efe80b551ccc
Author: Neng Lu
AuthorDate: Thu Sep 20 12:02:44 2018 -0700
refactor JavaSerializer for proper closing input/output streams (#3031)
---
.../org/apache/heron/api/serializer/JavaSerializer.java| 14 ++
1 file changed, 6 insertions(+), 8 deletions(-)
diff --git
a/heron/api/src/java/org/apache/heron/api/serializer/JavaSerializer.java
b/heron/api/src/java/org/apache/heron/api/serializer/JavaSerializer.java
index 75a1fba..9a024c9 100644
--- a/heron/api/src/java/org/apache/heron/api/serializer/JavaSerializer.java
+++ b/heron/api/src/java/org/apache/heron/api/serializer/JavaSerializer.java
@@ -34,25 +34,23 @@ public class JavaSerializer implements IPluggableSerializer
{
@Override
public byte[] serialize(Object object) {
-ByteArrayOutputStream bos = new ByteArrayOutputStream();
-try {
- ObjectOutputStream oos = new ObjectOutputStream(bos);
+try (ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ ObjectOutputStream oos = new ObjectOutputStream(bos)) {
oos.writeObject(object);
oos.flush();
+ return bos.toByteArray();
} catch (IOException e) {
throw new RuntimeException("Failed to serialize object: " +
object.toString(), e);
}
-return bos.toByteArray();
}
@Override
public Object deserialize(byte[] input) {
-ByteArrayInputStream bis = new ByteArrayInputStream(input);
-try {
- ObjectInputStream ois = new ObjectInputStream(bis);
+try (ByteArrayInputStream bis = new ByteArrayInputStream(input);
+ ObjectInputStream ois = new ObjectInputStream(bis)) {
return ois.readObject();
} catch (IOException | ClassNotFoundException e) {
- throw new RuntimeException(e);
+ throw new RuntimeException("Failed to deserialize object", e);
}
}
}
[incubator-heron] branch master updated: Revert "refactor JavaSerializer for closing input/output stream"
This is an automated email from the ASF dual-hosted git repository.
nlu90 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
The following commit(s) were added to refs/heads/master by this push:
new 72509b9 Revert "refactor JavaSerializer for closing input/output
stream"
72509b9 is described below
commit 72509b91f1744a1d85f5982d5c69741120409eb4
Author: Neng Lu
AuthorDate: Wed Sep 19 15:00:57 2018 -0700
Revert "refactor JavaSerializer for closing input/output stream"
This reverts commit db3f1c406c7ff73113e2c469b44636baee1eb77f.
---
.../apache/heron/api/serializer/JavaSerializer.java | 20 +++-
1 file changed, 11 insertions(+), 9 deletions(-)
diff --git
a/heron/api/src/java/org/apache/heron/api/serializer/JavaSerializer.java
b/heron/api/src/java/org/apache/heron/api/serializer/JavaSerializer.java
index cb942db..75a1fba 100644
--- a/heron/api/src/java/org/apache/heron/api/serializer/JavaSerializer.java
+++ b/heron/api/src/java/org/apache/heron/api/serializer/JavaSerializer.java
@@ -34,23 +34,25 @@ public class JavaSerializer implements IPluggableSerializer
{
@Override
public byte[] serialize(Object object) {
-try (ByteArrayOutputStream bos = new ByteArrayOutputStream();
- ObjectOutputStream oos = new ObjectOutputStream(bos)) {
+ByteArrayOutputStream bos = new ByteArrayOutputStream();
+try {
+ ObjectOutputStream oos = new ObjectOutputStream(bos);
oos.writeObject(object);
oos.flush();
- return bos.toByteArray();
-} catch (IOException ex) {
- throw new RuntimeException("Failed to serialize object: " +
object.toString(), ex);
+} catch (IOException e) {
+ throw new RuntimeException("Failed to serialize object: " +
object.toString(), e);
}
+return bos.toByteArray();
}
@Override
public Object deserialize(byte[] input) {
-try (ByteArrayInputStream bis = new ByteArrayInputStream(input);
- ObjectInputStream ois = new ObjectInputStream(bis)) {
+ByteArrayInputStream bis = new ByteArrayInputStream(input);
+try {
+ ObjectInputStream ois = new ObjectInputStream(bis);
return ois.readObject();
-} catch (IOException | ClassNotFoundException ex) {
- throw new RuntimeException(ex);
+} catch (IOException | ClassNotFoundException e) {
+ throw new RuntimeException(e);
}
}
}
[incubator-heron] branch master updated: refactor JavaSerializer for closing input/output stream
This is an automated email from the ASF dual-hosted git repository.
nlu90 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
The following commit(s) were added to refs/heads/master by this push:
new db3f1c4 refactor JavaSerializer for closing input/output stream
db3f1c4 is described below
commit db3f1c406c7ff73113e2c469b44636baee1eb77f
Author: Neng Lu
AuthorDate: Wed Sep 19 14:56:56 2018 -0700
refactor JavaSerializer for closing input/output stream
---
.../apache/heron/api/serializer/JavaSerializer.java | 20 +---
1 file changed, 9 insertions(+), 11 deletions(-)
diff --git
a/heron/api/src/java/org/apache/heron/api/serializer/JavaSerializer.java
b/heron/api/src/java/org/apache/heron/api/serializer/JavaSerializer.java
index 75a1fba..cb942db 100644
--- a/heron/api/src/java/org/apache/heron/api/serializer/JavaSerializer.java
+++ b/heron/api/src/java/org/apache/heron/api/serializer/JavaSerializer.java
@@ -34,25 +34,23 @@ public class JavaSerializer implements IPluggableSerializer
{
@Override
public byte[] serialize(Object object) {
-ByteArrayOutputStream bos = new ByteArrayOutputStream();
-try {
- ObjectOutputStream oos = new ObjectOutputStream(bos);
+try (ByteArrayOutputStream bos = new ByteArrayOutputStream();
+ ObjectOutputStream oos = new ObjectOutputStream(bos)) {
oos.writeObject(object);
oos.flush();
-} catch (IOException e) {
- throw new RuntimeException("Failed to serialize object: " +
object.toString(), e);
+ return bos.toByteArray();
+} catch (IOException ex) {
+ throw new RuntimeException("Failed to serialize object: " +
object.toString(), ex);
}
-return bos.toByteArray();
}
@Override
public Object deserialize(byte[] input) {
-ByteArrayInputStream bis = new ByteArrayInputStream(input);
-try {
- ObjectInputStream ois = new ObjectInputStream(bis);
+try (ByteArrayInputStream bis = new ByteArrayInputStream(input);
+ ObjectInputStream ois = new ObjectInputStream(bis)) {
return ois.readObject();
-} catch (IOException | ClassNotFoundException e) {
- throw new RuntimeException(e);
+} catch (IOException | ClassNotFoundException ex) {
+ throw new RuntimeException(ex);
}
}
}
[incubator-heron] branch master updated: Log executor command line arguments (#3020)
This is an automated email from the ASF dual-hosted git repository.
nlu90 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
The following commit(s) were added to refs/heads/master by this push:
new 774ba38 Log executor command line arguments (#3020)
774ba38 is described below
commit 774ba3806a0a80fd53a7bc6a2e23a6be5b3a0e35
Author: Ning Wang
AuthorDate: Thu Sep 6 10:46:26 2018 -0700
Log executor command line arguments (#3020)
---
heron/executor/src/python/heron_executor.py | 9 +
1 file changed, 5 insertions(+), 4 deletions(-)
diff --git a/heron/executor/src/python/heron_executor.py
b/heron/executor/src/python/heron_executor.py
index d62b7da..3f44b53 100755
--- a/heron/executor/src/python/heron_executor.py
+++ b/heron/executor/src/python/heron_executor.py
@@ -171,10 +171,8 @@ class HeronExecutor(object):
""" Heron executor is a class that is responsible for running each of the
process on a given
container. Based on the container id and the instance distribution, it
determines if the container
is a master node or a worker node and it starts processes accordingly."""
- def init_parsed_args(self, args):
+ def init_parsed_args(self, parsed_args):
""" initialize from parsed arguments """
-parsed_args = self.parse_args(args)
-
self.shard = parsed_args.shard
self.topology_name = parsed_args.topology_name
self.topology_id = parsed_args.topology_id
@@ -264,7 +262,8 @@ class HeronExecutor(object):
if parsed_args.jvm_remote_debugger_ports else None
def __init__(self, args, shell_env):
-self.init_parsed_args(args)
+parsed_args = self.parse_args(args)
+self.init_parsed_args(parsed_args)
self.shell_env = shell_env
self.max_runs = 100
@@ -291,6 +290,8 @@ class HeronExecutor(object):
@staticmethod
def parse_args(args):
"""Uses python argparse to collect positional args"""
+Log.info("Input args: %r" % args)
+
parser = argparse.ArgumentParser()
parser.add_argument("--shard", type=int, required=True)
[incubator-heron] branch master updated: Update README(updated urls and added developer link) (#3010)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-heron.git The following commit(s) were added to refs/heads/master by this push: new bf2c73a Update README(updated urls and added developer link) (#3010) bf2c73a is described below commit bf2c73adbfe0794f9c7af6a4c08c9d4375fd7dca Author: Ning Wang AuthorDate: Tue Sep 4 10:38:24 2018 -0700 Update README(updated urls and added developer link) (#3010) --- README.md | 8 +--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index 3e21c5e..a684942 100644 --- a/README.md +++ b/README.md @@ -9,10 +9,12 @@ but with a wide array of architectural improvements. http://incubator.apache.org/projects/heron.html -https://heron.incubator.apache.org (coming soon...) +https://heron.incubator.apache.org ### Documentation -http://heronstreaming.io (official until) +Documentation and tutorials can be found on the [Heron website](http://heronstreaming.io) + +Developers and contributors should also take a look at [Heron developers](https://apache.github.io/incubator-heron/docs/developers/compiling/compiling/) Heron Requirements: * Java JDK 1.8 @@ -47,7 +49,7 @@ https://www.meetup.com/Apache-Heron-Bay-Area ## For more information: * Official Heron documentation located at [heronstreaming.io](http://heronstreaming.io) -* Official Heron resources, including Conference & Journal Papers, Videos, Blog Posts and selected Press located at [Heron Resources](http://twitter.github.io/heron/docs/resources/) +* Official Heron resources, including Conference & Journal Papers, Videos, Blog Posts and selected Press located at [Heron Resources](https://apache.github.io/incubator-heron/docs/resources/) * [Twitter Heron: Stream Processing at Scale](http://dl.acm.org/citation.cfm?id=2742788) (academic paper) * [Twitter Heron: Stream Processing at
[incubator-heron] branch master updated: Fix path and package name in docs/comments/tests (#3007)
This is an automated email from the ASF dual-hosted git repository.
nlu90 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
The following commit(s) were added to refs/heads/master by this push:
new 4328ab4 Fix path and package name in docs/comments/tests (#3007)
4328ab4 is described below
commit 4328ab4fe9b6b67765829c57696924b2c84107cc
Author: Ning Wang
AuthorDate: Fri Aug 24 10:26:16 2018 -0700
Fix path and package name in docs/comments/tests (#3007)
---
docker/Readme.md | 2 +-
.../heron/metricscachemgr/MetricsCacheManagerHttpServer.java | 4 ++--
heron/tools/cli/tests/python/client_command_unittest.py | 8
website/content/docs/migrate-storm-to-heron.md| 4 ++--
.../docs/operators/deployment/schedulers/aurora-local-setup.md| 2 +-
.../docs/operators/deployment/schedulers/mesos-local-mac.md | 4 ++--
website/content/docs/operators/deployment/schedulers/yarn.md | 4 ++--
7 files changed, 14 insertions(+), 14 deletions(-)
diff --git a/docker/Readme.md b/docker/Readme.md
index 2bdee95..80fbf56 100644
--- a/docker/Readme.md
+++ b/docker/Readme.md
@@ -24,7 +24,7 @@
### To submit/activate/kill a topology:
```
#To submit a topology:
-docker exec heron_executor_1 heron submit local
/usr/local/heron/examples/heron-examples.jar
org.apache.heron.examples.ExclamationTopology ExclamationTopology
--deploy-deactivated
+docker exec heron_executor_1 heron submit local
/usr/local/heron/examples/heron-api-examples.jar
org.apache.heron.examples.api.ExclamationTopology ExclamationTopology
--deploy-deactivated
#To activate a topology:
docker exec -it heron_executor_1 heron activate local
ExclamationTopology
#To kill a topology:
diff --git
a/heron/metricscachemgr/src/java/org/apache/heron/metricscachemgr/MetricsCacheManagerHttpServer.java
b/heron/metricscachemgr/src/java/org/apache/heron/metricscachemgr/MetricsCacheManagerHttpServer.java
index 1b74af9..ec4e21b 100644
---
a/heron/metricscachemgr/src/java/org/apache/heron/metricscachemgr/MetricsCacheManagerHttpServer.java
+++
b/heron/metricscachemgr/src/java/org/apache/heron/metricscachemgr/MetricsCacheManagerHttpServer.java
@@ -82,8 +82,8 @@ public class MetricsCacheManagerHttpServer {
* host:port component_name metrics_name
* Example:
* 1. run the example topology,
- * ~/bin/heron submit local ~/.heron/examples/heron-examples.jar \
- * org.apache.heron.examples.ExclamationTopology ExclamationTopology \
+ * ~/bin/heron submit local ~/.heron/examples/heron-api-examples.jar \
+ * org.apache.heron.examples.api.ExclamationTopology ExclamationTopology \
* --deploy-deactivated --verbose
* 2. in the [source root directory],
* bazel run
heron/metricscachemgr/src/java:metricscache-queryclient-unshaded -- \
diff --git a/heron/tools/cli/tests/python/client_command_unittest.py
b/heron/tools/cli/tests/python/client_command_unittest.py
index c13c049..00f35a4 100644
--- a/heron/tools/cli/tests/python/client_command_unittest.py
+++ b/heron/tools/cli/tests/python/client_command_unittest.py
@@ -78,12 +78,12 @@ class SubmitTest(ClientCommandTest):
def test(self):
subprocess.Popen = MagicMock()
-command = 'heron submit local ~/.heron/examples/heron-examples.jar ' + \
- 'org.apache.heron.examples.ExclamationTopology EX'
+command = 'heron submit local ~/.heron/examples/heron-api-examples.jar ' +
\
+ 'org.apache.heron.examples.api.ExclamationTopology EX'
create_defn_commands = '/usr/lib/bin/java -client -Xmx1g -cp ' \
-'~/.heron/examples/heron-examples.jar:/heron/lib/jars/third_party/* ' \
-'org.apache.heron.examples.ExclamationTopology EX'
+'~/.heron/examples/heron-api-examples.jar:/heron/lib/jars/third_party/* ' \
+'org.apache.heron.examples.api.ExclamationTopology EX'
submit_commands = '/usr/lib/bin/java -client -Xmx1g -cp ' \
':/heron/lib/jars/scheduler/*:/heron/lib/jars/uploader/*:' \
diff --git a/website/content/docs/migrate-storm-to-heron.md
b/website/content/docs/migrate-storm-to-heron.md
index 9e7c03b..630404c 100644
--- a/website/content/docs/migrate-storm-to-heron.md
+++ b/website/content/docs/migrate-storm-to-heron.md
@@ -95,8 +95,8 @@ Start Guide](../getting-started) guide:
```bash
$ heron submit local \
- ~/.heron/examples/heron-examples.jar \ # The path of the topology's jar file
- org.apache.heron.examples.ExclamationTopology \ # The topology's Java class
+ ~/.heron/examples/heron-api-examples.jar \ # The path of the topology's jar
file
+ org.apache.heron.examples.api.ExclamationTopology \ # The topology's Java
class
ExclamationTopology # The name of the topology
```
diff --git
a/website/content/docs/operators/deployment/schedulers/aurora-local-setup.md
b/website/content/docs/operators/deployment/schedulers/aurora
[incubator-heron] tag v-0.20.0-incubating-candidate-3 created (now 7fb0df3)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a change to tag v-0.20.0-incubating-candidate-3 in repository https://gitbox.apache.org/repos/asf/incubator-heron.git. at 7fb0df3 (commit) No new revisions were added by this update.
[incubator-heron] branch master updated: Fix RAM config in example code (#3006)
This is an automated email from the ASF dual-hosted git repository.
nlu90 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
The following commit(s) were added to refs/heads/master by this push:
new 7fb0df3 Fix RAM config in example code (#3006)
7fb0df3 is described below
commit 7fb0df3b6ec29d8c51f9d43ad7e8ecb3d45d643a
Author: Ning Wang
AuthorDate: Thu Aug 23 13:05:19 2018 -0700
Fix RAM config in example code (#3006)
---
examples/src/java/org/apache/heron/examples/api/ExampleResources.java | 2 +-
.../java/org/apache/heron/examples/api/StatefulWordCountTopology.java | 4 ++--
.../src/java/org/apache/heron/examples/api/WordCountTopology.java | 4 ++--
3 files changed, 5 insertions(+), 5 deletions(-)
diff --git
a/examples/src/java/org/apache/heron/examples/api/ExampleResources.java
b/examples/src/java/org/apache/heron/examples/api/ExampleResources.java
index c5c66a4..be92dea 100644
--- a/examples/src/java/org/apache/heron/examples/api/ExampleResources.java
+++ b/examples/src/java/org/apache/heron/examples/api/ExampleResources.java
@@ -23,7 +23,7 @@ import org.apache.heron.common.basics.ByteAmount;
public final class ExampleResources {
- static final long COMPONENT_RAM_MB = 512;
+ static final long COMPONENT_RAM_MB = 1024;
static ByteAmount getComponentRam() {
return ByteAmount.fromMegabytes(COMPONENT_RAM_MB);
diff --git
a/examples/src/java/org/apache/heron/examples/api/StatefulWordCountTopology.java
b/examples/src/java/org/apache/heron/examples/api/StatefulWordCountTopology.java
index 6208727..7ed3562 100644
---
a/examples/src/java/org/apache/heron/examples/api/StatefulWordCountTopology.java
+++
b/examples/src/java/org/apache/heron/examples/api/StatefulWordCountTopology.java
@@ -205,9 +205,9 @@ public final class StatefulWordCountTopology {
// configure component resources
conf.setComponentRam("word",
-ByteAmount.fromMegabytes(ExampleResources.COMPONENT_RAM_MB * 2));
+ByteAmount.fromMegabytes(ExampleResources.COMPONENT_RAM_MB));
conf.setComponentRam("consumer",
-ByteAmount.fromMegabytes(ExampleResources.COMPONENT_RAM_MB * 2));
+ByteAmount.fromMegabytes(ExampleResources.COMPONENT_RAM_MB));
// configure container resources
conf.setContainerDiskRequested(
diff --git
a/examples/src/java/org/apache/heron/examples/api/WordCountTopology.java
b/examples/src/java/org/apache/heron/examples/api/WordCountTopology.java
index 0a919d8..5991cfd 100644
--- a/examples/src/java/org/apache/heron/examples/api/WordCountTopology.java
+++ b/examples/src/java/org/apache/heron/examples/api/WordCountTopology.java
@@ -184,9 +184,9 @@ public final class WordCountTopology {
// configure component resources
conf.setComponentRam("word",
-ByteAmount.fromMegabytes(ExampleResources.COMPONENT_RAM_MB * 2));
+ByteAmount.fromMegabytes(ExampleResources.COMPONENT_RAM_MB));
conf.setComponentRam("consumer",
-ByteAmount.fromMegabytes(ExampleResources.COMPONENT_RAM_MB * 2));
+ByteAmount.fromMegabytes(ExampleResources.COMPONENT_RAM_MB));
// configure container resources
conf.setContainerDiskRequested(
[incubator-heron] branch master updated: fix building problem with source tarball (#3005)
This is an automated email from the ASF dual-hosted git repository.
nlu90 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
The following commit(s) were added to refs/heads/master by this push:
new 5a11b3e fix building problem with source tarball (#3005)
5a11b3e is described below
commit 5a11b3eed50982702c00d2156060f1431874f6cc
Author: Neng Lu
AuthorDate: Thu Aug 23 11:21:01 2018 -0700
fix building problem with source tarball (#3005)
* fix status.sh without .git info
---
scripts/release/status.sh | 52 +++
1 file changed, 34 insertions(+), 18 deletions(-)
diff --git a/scripts/release/status.sh b/scripts/release/status.sh
index 8473c95..1be09c2 100755
--- a/scripts/release/status.sh
+++ b/scripts/release/status.sh
@@ -42,28 +42,39 @@ function disable_e_and_execute {
}
# get the release tag version or the branch name
-if [ -z ${HERON_BUILD_VERSION+x} ];
+if [ -d .git ];
then
- cmd="git rev-parse --abbrev-ref HEAD"
- build_version=$($cmd) || die "Failed to run command to check head: $cmd"
-
- if [ "${build_version}" = "HEAD" ];
+ if [ -z ${HERON_BUILD_VERSION+x} ];
then
-cmd="git describe --tags --always"
-build_version=$($cmd) || die "Failed to run command to get git release:
$cmd"
+cmd="git rev-parse --abbrev-ref HEAD"
+build_version=$($cmd) || die "Failed to run command to check head: $cmd"
+
+if [ "${build_version}" = "HEAD" ];
+then
+ cmd="git describe --tags --always"
+ build_version=$($cmd) || die "Failed to run command to get git release:
$cmd"
+fi
+ else
+build_version=${HERON_BUILD_VERSION}
fi
else
- build_version=${HERON_BUILD_VERSION}
+ current_dir=$(pwd)
+ build_version=$(basename "$current_dir")
fi
echo "HERON_BUILD_VERSION ${build_version}"
# The code below presents an implementation that works for git repository
-if [ -z ${HERON_GIT_REV+x} ];
+if [ -d .git ];
then
- cmd="git rev-parse HEAD"
- git_rev=$($cmd) || die "Failed to get git revision: $cmd"
+ if [ -z ${HERON_GIT_REV+x} ];
+ then
+cmd="git rev-parse HEAD"
+git_rev=$($cmd) || die "Failed to get git revision: $cmd"
+ else
+git_rev=${HERON_GIT_REV}
+ fi
else
- git_rev=${HERON_GIT_REV}
+ git_rev=$build_version
fi
echo "HERON_BUILD_SCM_REVISION ${git_rev}"
@@ -101,16 +112,21 @@ fi
echo "HERON_BUILD_USER ${build_user}"
# Check whether there are any uncommited changes
-if [ -z ${HERON_TREE_STATUS+x} ];
+if [ -d .git ];
then
- status=$(disable_e_and_execute "git diff-index --quiet HEAD --")
- if [[ $status == 0 ]];
+ if [ -z ${HERON_TREE_STATUS+x} ];
then
-tree_status="Clean"
+status=$(disable_e_and_execute "git diff-index --quiet HEAD --")
+if [[ $status == 0 ]];
+then
+ tree_status="Clean"
+else
+ tree_status="Modified"
+fi
else
-tree_status="Modified"
+tree_status=${HERON_TREE_STATUS}
fi
else
- tree_status=${HERON_TREE_STATUS}
+ tree_status="Clean"
fi
echo "HERON_BUILD_RELEASE_STATUS ${tree_status}"
[incubator-heron] tag v-0.20.0-incubating-candidate-1-test deleted (was d2946ce)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a change to tag v-0.20.0-incubating-candidate-1-test in repository https://gitbox.apache.org/repos/asf/incubator-heron.git. *** WARNING: tag v-0.20.0-incubating-candidate-1-test was deleted! *** was d2946ce Docker Ubuntu tweaks (#2975) The revisions that were on this tag are still contained in other references; therefore, this change does not discard any commits from the repository.
[incubator-heron] tag v-0.20.0-incubating-candidate-1-test2 deleted (was 2324706)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a change to tag v-0.20.0-incubating-candidate-1-test2 in repository https://gitbox.apache.org/repos/asf/incubator-heron.git. *** WARNING: tag v-0.20.0-incubating-candidate-1-test2 was deleted! *** was 2324706 Add gitattributes to remove website directory from archive The revisions that were on this tag are still contained in other references; therefore, this change does not discard any commits from the repository.
svn commit: r28895 - /dev/incubator/heron/heron-0.20.0-incubating-candidate-2/
Author: nlu90 Date: Wed Aug 22 17:39:08 2018 New Revision: 28895 Log: Apache heron 0.20.0-incubating release candidate 2 Added: dev/incubator/heron/heron-0.20.0-incubating-candidate-2/ dev/incubator/heron/heron-0.20.0-incubating-candidate-2/incubator-heron-v-0.20.0-incubating-candidate-2.tar.gz (with props) dev/incubator/heron/heron-0.20.0-incubating-candidate-2/incubator-heron-v-0.20.0-incubating-candidate-2.tar.gz.asc dev/incubator/heron/heron-0.20.0-incubating-candidate-2/incubator-heron-v-0.20.0-incubating-candidate-2.tar.gz.sha1 dev/incubator/heron/heron-0.20.0-incubating-candidate-2/incubator-heron-v-0.20.0-incubating-candidate-2.tar.gz.sha512 Added: dev/incubator/heron/heron-0.20.0-incubating-candidate-2/incubator-heron-v-0.20.0-incubating-candidate-2.tar.gz == Binary file - no diff available. Propchange: dev/incubator/heron/heron-0.20.0-incubating-candidate-2/incubator-heron-v-0.20.0-incubating-candidate-2.tar.gz -- svn:mime-type = application/octet-stream Added: dev/incubator/heron/heron-0.20.0-incubating-candidate-2/incubator-heron-v-0.20.0-incubating-candidate-2.tar.gz.asc == --- dev/incubator/heron/heron-0.20.0-incubating-candidate-2/incubator-heron-v-0.20.0-incubating-candidate-2.tar.gz.asc (added) +++ dev/incubator/heron/heron-0.20.0-incubating-candidate-2/incubator-heron-v-0.20.0-incubating-candidate-2.tar.gz.asc Wed Aug 22 17:39:08 2018 @@ -0,0 +1,16 @@ +-BEGIN PGP SIGNATURE- + +iQIcBAABAgAGBQJbfZ9TAAoJEBYIdp4fEPZ5JdAP/3yFm6PRPY++iE734Q06PaiZ +ajPKPGsOWCarW66FgqtNfMuh9cbFVLCwrigEvVkjGRQ8w3Cw7Gazvy0Csk4CbPbK ++VlYJh9BNw16CZYOF6dY+53QWlwQ711Em+ZnZBUN7nyPchK5TUymm1DntTO0nORc +WGrIUJZ0QOrBFdvh2SXtAray/9Mzxsar5LQNzVHi2RobS4zAtKEHcp5wdX4RnH+c +aqOeJsP7FmjRPYEI1D5+qKnMJbrl0gpoSgYGBfrc3k+IDoIA94S2HdeWpDbZeM6W +8bkjIiK4aBOe/Dv93G96x0CnytzJOUrwBfpZTSw+QLEBE5wLWaU1XCb9pDjVJ6Nw +HtIWKt4QfOYAXBXpVfd0q+ixEFjoL7M3i83faLRxhTP/WnduoDQ+pWCmZai8G5W2 +5Cn/2s7j45yJxnPi4gDyegYey+UeNmxvwB9ObpbzmSOCeX+y3eA+HGnbjpI1J3B3 +qjmqtt5HJuai8qXfiRlsVTkrxuk/yziKftNp+lQv54DgKhRkkNOQA6fz+iYHbpjf +ClVMIxE9PzKDEC0CzNi9ATdOFP9UVOsmlZJv3RQ3dCrtMc87cWOqcivLhrlQxPm+ +cWO9wr+HJ1I8WOcLHL5ABF27TZnpLOQO/EtI2JTE9gkbHt0s4XWGrvbncm7wyrPH +Qnnfr1XJhSHuCux7zEbQ +=bE5W +-END PGP SIGNATURE- Added: dev/incubator/heron/heron-0.20.0-incubating-candidate-2/incubator-heron-v-0.20.0-incubating-candidate-2.tar.gz.sha1 == --- dev/incubator/heron/heron-0.20.0-incubating-candidate-2/incubator-heron-v-0.20.0-incubating-candidate-2.tar.gz.sha1 (added) +++ dev/incubator/heron/heron-0.20.0-incubating-candidate-2/incubator-heron-v-0.20.0-incubating-candidate-2.tar.gz.sha1 Wed Aug 22 17:39:08 2018 @@ -0,0 +1 @@ +a80c6bae4938c4c8a322552c256cb3fb4bd0c809 incubator-heron-v-0.20.0-incubating-candidate-2.tar.gz Added: dev/incubator/heron/heron-0.20.0-incubating-candidate-2/incubator-heron-v-0.20.0-incubating-candidate-2.tar.gz.sha512 == --- dev/incubator/heron/heron-0.20.0-incubating-candidate-2/incubator-heron-v-0.20.0-incubating-candidate-2.tar.gz.sha512 (added) +++ dev/incubator/heron/heron-0.20.0-incubating-candidate-2/incubator-heron-v-0.20.0-incubating-candidate-2.tar.gz.sha512 Wed Aug 22 17:39:08 2018 @@ -0,0 +1 @@ +69dc485eade9c8d9ec3ba4b7672ab365a5ddd736fcf64ab14aa4a0922108ecbb2bb4f73f25ba276b7796fbf6a146cf37a09a4faebcea4f4b1465ed661da6a8a7 incubator-heron-v-0.20.0-incubating-candidate-2.tar.gz
[incubator-heron] tag v-0.20.0-incubating-candidate-2 created (now d01b9bb)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a change to tag v-0.20.0-incubating-candidate-2 in repository https://gitbox.apache.org/repos/asf/incubator-heron.git. at d01b9bb (commit) No new revisions were added by this update.
[incubator-heron] branch release/v-0.20.0-incubating created (now d01b9bb)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a change to branch release/v-0.20.0-incubating in repository https://gitbox.apache.org/repos/asf/incubator-heron.git. at d01b9bb Remove icomoon usage from tools/ui (#2999) No new revisions were added by this update.
[incubator-heron] branch release/v-0.20.0-incubating deleted (was d2946ce)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a change to branch release/v-0.20.0-incubating in repository https://gitbox.apache.org/repos/asf/incubator-heron.git. was d2946ce Docker Ubuntu tweaks (#2975) The revisions that were on this branch are still contained in other references; therefore, this change does not discard any commits from the repository.
[incubator-heron] branch master updated: Move license header in react dom components (#3003)
This is an automated email from the ASF dual-hosted git repository.
nlu90 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
The following commit(s) were added to refs/heads/master by this push:
new 3c4ec25 Move license header in react dom components (#3003)
3c4ec25 is described below
commit 3c4ec25dc464ed3bc4dfcf7df78346dc38efec71
Author: Ning Wang
AuthorDate: Wed Aug 22 10:02:17 2018 -0700
Move license header in react dom components (#3003)
* Move license header in react dom components
* remove empty line at beginning
---
heron/tools/ui/resources/static/js/alltopologies.js | 4 ++--
heron/tools/ui/resources/static/js/common.js| 4 ++--
heron/tools/ui/resources/static/js/config.js| 4 ++--
heron/tools/ui/resources/static/js/exceptions.js| 4 ++--
heron/tools/ui/resources/static/js/topologies.js| 4 ++--
5 files changed, 10 insertions(+), 10 deletions(-)
diff --git a/heron/tools/ui/resources/static/js/alltopologies.js
b/heron/tools/ui/resources/static/js/alltopologies.js
index f9a806d..f540ff6 100644
--- a/heron/tools/ui/resources/static/js/alltopologies.js
+++ b/heron/tools/ui/resources/static/js/alltopologies.js
@@ -1,3 +1,5 @@
+/** @jsx React.DOM */
+
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -17,8 +19,6 @@
* under the License.
*/
-/** @jsx React.DOM */
-
var TopologyItem = React.createClass({
render: function() {
var topology = this.props.topology;
diff --git a/heron/tools/ui/resources/static/js/common.js
b/heron/tools/ui/resources/static/js/common.js
index 8d2c02e..f21d983 100644
--- a/heron/tools/ui/resources/static/js/common.js
+++ b/heron/tools/ui/resources/static/js/common.js
@@ -1,3 +1,5 @@
+/** @jsx React.DOM */
+
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -17,8 +19,6 @@
* under the License.
*/
-/** @jsx React.DOM */
-
var ActionButton = React.createClass({
render: function() {
// Map containing key to destination links.
diff --git a/heron/tools/ui/resources/static/js/config.js
b/heron/tools/ui/resources/static/js/config.js
index fa4367c..762fe2b 100644
--- a/heron/tools/ui/resources/static/js/config.js
+++ b/heron/tools/ui/resources/static/js/config.js
@@ -1,3 +1,5 @@
+/** @jsx React.DOM */
+
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -17,8 +19,6 @@
* under the License.
*/
-/** @jsx React.DOM */
-
var ConfigTable = React.createClass({
getInitialState: function() {
return {
diff --git a/heron/tools/ui/resources/static/js/exceptions.js
b/heron/tools/ui/resources/static/js/exceptions.js
index 04a8559..baea39f 100644
--- a/heron/tools/ui/resources/static/js/exceptions.js
+++ b/heron/tools/ui/resources/static/js/exceptions.js
@@ -1,3 +1,5 @@
+/** @jsx React.DOM */
+
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -17,8 +19,6 @@
* under the License.
*/
-/** @jsx React.DOM */
-
// TODO(nbhagat): Aggregate exceptions for better display.
// Requires baseUrl, cluster, environ, topology, comp_name, instance
properties.
var InstanceExceptionLogs = React.createClass({
diff --git a/heron/tools/ui/resources/static/js/topologies.js
b/heron/tools/ui/resources/static/js/topologies.js
index fec489c..2db0798 100644
--- a/heron/tools/ui/resources/static/js/topologies.js
+++ b/heron/tools/ui/resources/static/js/topologies.js
@@ -1,3 +1,5 @@
+/** @jsx React.DOM */
+
/*
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -17,8 +19,6 @@
* under the License.
*/
-/** @jsx React.DOM */
-
var SUM = 0,
AVG = 1,
LAST = 2;
[incubator-heron] tag v-0.20.0-incubating-candidate-1-test2 created (now 2324706)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a change to tag v-0.20.0-incubating-candidate-1-test2 in repository https://gitbox.apache.org/repos/asf/incubator-heron.git. at 2324706 (commit) No new revisions were added by this update.
[incubator-heron] branch nwang/exclude_website_from_release created (now 2324706)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a change to branch nwang/exclude_website_from_release in repository https://gitbox.apache.org/repos/asf/incubator-heron.git. at 2324706 Add gitattributes to remove website directory from archive No new revisions were added by this update.
[incubator-heron] branch master updated: remove crreal BUILD file (#2988)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-heron.git The following commit(s) were added to refs/heads/master by this push: new 83d530f remove crreal BUILD file (#2988) 83d530f is described below commit 83d530fc2cbf35e490c5f5a781469741dda965fd Author: Neng Lu AuthorDate: Mon Aug 20 21:10:50 2018 -0700 remove crreal BUILD file (#2988) --- third_party/cereal/BUILD | 124 --- 1 file changed, 124 deletions(-) diff --git a/third_party/cereal/BUILD b/third_party/cereal/BUILD deleted file mode 100644 index 1f8b548..000 --- a/third_party/cereal/BUILD +++ /dev/null @@ -1,124 +0,0 @@ -licenses(["notice"]) - -package(default_visibility = ["//visibility:public"]) - -package_name = "cereal" -package_version = "1.2.2" # This version number needs to match the one in WORKSPACE file - -package_file = package_name + "-" + package_version + ".tar.gz" -package_dir = package_name + "-" + package_version - -file_list = [ -"include/cereal/access.hpp", -"include/cereal/archives/adapters.hpp", -"include/cereal/archives/binary.hpp", -"include/cereal/archives/json.hpp", -"include/cereal/archives/portable_binary.hpp", -"include/cereal/archives/xml.hpp", -"include/cereal/cereal.hpp", -"include/cereal/details/helpers.hpp", -"include/cereal/details/polymorphic_impl.hpp", -"include/cereal/details/polymorphic_impl_fwd.hpp", -"include/cereal/details/static_object.hpp", -"include/cereal/details/traits.hpp", -"include/cereal/details/util.hpp", -"include/cereal/external/base64.hpp", -"include/cereal/external/rapidjson/allocators.h", -"include/cereal/external/rapidjson/document.h", -"include/cereal/external/rapidjson/encodedstream.h", -"include/cereal/external/rapidjson/encodings.h", -"include/cereal/external/rapidjson/error/en.h", -"include/cereal/external/rapidjson/error/error.h", -"include/cereal/external/rapidjson/filereadstream.h", -"include/cereal/external/rapidjson/filewritestream.h", -"include/cereal/external/rapidjson/fwd.h", -"include/cereal/external/rapidjson/internal/biginteger.h", -"include/cereal/external/rapidjson/internal/diyfp.h", -"include/cereal/external/rapidjson/internal/dtoa.h", -"include/cereal/external/rapidjson/internal/ieee754.h", -"include/cereal/external/rapidjson/internal/itoa.h", -"include/cereal/external/rapidjson/internal/meta.h", -"include/cereal/external/rapidjson/internal/pow10.h", -"include/cereal/external/rapidjson/internal/regex.h", -"include/cereal/external/rapidjson/internal/stack.h", -"include/cereal/external/rapidjson/internal/strfunc.h", -"include/cereal/external/rapidjson/internal/strtod.h", -"include/cereal/external/rapidjson/internal/swap.h", -"include/cereal/external/rapidjson/istreamwrapper.h", -"include/cereal/external/rapidjson/memorybuffer.h", -"include/cereal/external/rapidjson/memorystream.h", -"include/cereal/external/rapidjson/msinttypes/inttypes.h", -"include/cereal/external/rapidjson/msinttypes/stdint.h", -"include/cereal/external/rapidjson/ostreamwrapper.h", -"include/cereal/external/rapidjson/pointer.h", -"include/cereal/external/rapidjson/prettywriter.h", -"include/cereal/external/rapidjson/rapidjson.h", -"include/cereal/external/rapidjson/reader.h", -"include/cereal/external/rapidjson/schema.h", -"include/cereal/external/rapidjson/stream.h", -"include/cereal/external/rapidjson/stringbuffer.h", -"include/cereal/external/rapidjson/writer.h", -"include/cereal/external/rapidxml/rapidxml.hpp", -"include/cereal/external/rapidxml/rapidxml_iterators.hpp", -"include/cereal/external/rapidxml/rapidxml_print.hpp", -"include/cereal/external/rapidxml/rapidxml_utils.hpp", -"include/cereal/macros.hpp", -"include/cereal/types/array.hpp", -"include/cereal/types/base_class.hpp", -"include/cereal/types/bitset.hpp", -"include/cereal/types/boost_variant.hpp", -"include/cereal/types/chrono.hpp", -"include/cereal/types/common.hpp", -"include/cereal/types/complex.hpp", -"include/cereal/types/concepts/pair_
[incubator-heron] branch master updated: use cereal lib configured in WORKSPACE (#2986)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-heron.git The following commit(s) were added to refs/heads/master by this push: new d135746 use cereal lib configured in WORKSPACE (#2986) d135746 is described below commit d13574693493ba82cefa990631518d3f07ed100a Author: Ning Wang AuthorDate: Mon Aug 20 10:27:21 2018 -0700 use cereal lib configured in WORKSPACE (#2986) --- third_party/cereal/BUILD | 8 +++- third_party/cereal/cereal-1.2.1.tar.gz | Bin 301689 -> 0 bytes third_party/cereal/empty.cc| 0 3 files changed, 3 insertions(+), 5 deletions(-) diff --git a/third_party/cereal/BUILD b/third_party/cereal/BUILD index 3f7e5ec..1f8b548 100644 --- a/third_party/cereal/BUILD +++ b/third_party/cereal/BUILD @@ -3,7 +3,7 @@ licenses(["notice"]) package(default_visibility = ["//visibility:public"]) package_name = "cereal" -package_version = "1.2.1" +package_version = "1.2.2" # This version number needs to match the one in WORKSPACE file package_file = package_name + "-" + package_version + ".tar.gz" package_dir = package_name + "-" + package_version @@ -101,7 +101,7 @@ genrule( "export INSTALL_DIR=$$(pwd)/$(@D)", "export TMP_DIR=$$(mktemp -d -t cereal.X)", "mkdir -p $$TMP_DIR", -"cp -R $(SRCS) $$TMP_DIR", +"cp -R $$(pwd)/external/com_github_cereal/* $$TMP_DIR", "cd $$TMP_DIR", "tar xfz " + package_file, "cd " + package_dir, @@ -116,9 +116,7 @@ genrule( cc_library( name = "cereal-cxx", -srcs = [ -"empty.cc", -] + file_list, +srcs = file_list, includes = [ "include", ], diff --git a/third_party/cereal/cereal-1.2.1.tar.gz b/third_party/cereal/cereal-1.2.1.tar.gz deleted file mode 100644 index 7f469ee..000 Binary files a/third_party/cereal/cereal-1.2.1.tar.gz and /dev/null differ diff --git a/third_party/cereal/empty.cc b/third_party/cereal/empty.cc deleted file mode 100644 index e69de29..000
svn commit: r28785 - /dev/incubator/heron/KEYS
Author: nlu90 Date: Thu Aug 16 22:27:50 2018 New Revision: 28785 Log: Added gpg key for nlu90 Added: dev/incubator/heron/KEYS Added: dev/incubator/heron/KEYS == --- dev/incubator/heron/KEYS (added) +++ dev/incubator/heron/KEYS Thu Aug 16 22:27:50 2018 @@ -0,0 +1,72 @@ +This file contains the PGP keys of various developers. + +Users: pgp < KEYS +or + gpg --import KEYS + + +Developers: +pgp -kxa and append it to this file. +or +(pgpk -ll && pgpk -xa ) >> this file. +or +(gpg --list-sigs +&& gpg --armor --export ) >> this file. + +pub 4096R/1F10F679 2018-08-16 +uid Neng Lu (CODE SIGNING KEY) +sig 31F10F679 2018-08-16 Neng Lu (CODE SIGNING KEY) +sub 4096R/7B1413B4 2018-08-16 +sig 1F10F679 2018-08-16 Neng Lu (CODE SIGNING KEY) + +-BEGIN PGP PUBLIC KEY BLOCK- + +mQINBFt196ABEADBPpEfJ5WjI2GI1GCb12UIfv7ActF5d1vcVr3oy5ZsB5/xvn2b +wnmXrfIIde58L6JtEJWWcG6V0Xs+a6JwE0wkSgHAkS33IwGOQE/ue8syeQ0UG7ZG +Tda9vqqNw5aLMgXQvQVuVEZzvxZiM32qK8Vj1kBXrfl3yobO5mRv6yAQ5Mn1cqSC +ERffMcqfsKYR4ekERN0dlOxD8Zer50FoCTxzRTHYhkQ8xv8Hrm2msx6q704CCTv8 +qcd1h9qM7xPzGSjWMxBz6wM7HILpaeoZ7Sjv4mFrAtjzPFPB8rQK4YNTfDIg0o+u +1FnQTDbGKOWEP1NO5ngHz4mDcxWojz2yqWIm+Ubk/8kkPmzthtNND8lPnKZ0EcAz +6ov9EkGU2lUaWOZJsf7/cXisGc1KVuGriKmjW6M/7HiohWHgoZN7N0n5x/bU+ha+ +rpnNWnVLAL3OIS+I6C/1VPOnkJuRinQlzy1T3L+67ma4BJmOm3xrLFPedLHVKiTi +z8yT3/3j0GYAQT8ujGE7Mw+qLfsSK7TRW5Shi67EgKGr8ctCgs8bund5pZmTHt66 +8CiYUEW+cowiisLJA9OjlXAHXm+iY6mkWa8DHjPyLboKIxdacU9w/5c6zcHOjufT +wS9Sjac5G6UFJLLJ8TxoVmA7VW0xMGqye55Xg9T4Ur1C1xNgxj6/a1tyvQARAQAB +tC1OZW5nIEx1IChDT0RFIFNJR05JTkcgS0VZKSA8bmx1OTBAYXBhY2hlLm9yZz6J +AjgEEwECACIFAlt196ACGwMGCwkIBwMCBhUIAgkKCwQWAgMBAh4BAheAAAoJEBYI +dp4fEPZ5OukQALY5aU1ZHBWDWDD2LJIgQ5Th5hm5L70P3FHgBev4gPj0FGbyzizo +kRTfpwISOk49zAIOveCIqKaGl9HoJTzWkrcw54PLxIs3FayyW78DoVJZ1Cmjacuy +xvWW8CvaDVMkLuTA4vwV2IbsF4iB7qQi0IUjo+uXNKYF6fS/BtFCE3/QTdxWwpWR +Qicg3LajjaOKi3zMOjTMir7ulmwZzUelHc14CweHrMwfxM2+CQ/u6LxCwGBMNJwh +Xl6AOcCMqgshVw6rO9/aINBPm6UtLzg++n1CI2nmX2+hNpon7BjZ+eLEDmKbNMO4 +eRePlqyrkPeD+ytvgyLmN6zh1GxeiLSL7Rk2sGinR2HXLF6DM152pbr0L5APKILz +BnCLUavPSe42YbLOpb1zA46ckHQl9PdrVidYoR1RtIj19FFO5rSxwrkpzIN+SB4f +sHtv0LUx2CfV+pcgPzPA5dMpDqA1N9cym8hVDPB99sbuStmIas2d44YiEFCx7B92 +23WPs1ucr82g9E/tNhiqb+/iXjTfxLZCpog+yt7Byoa0rjS0P3hNyM5+agGk45uU +ujlM0cDW/Y3bo8gQCksB6Qm3UZbDiro0k3HuB+QFmaUbT1ZBNfbnZkPzr9BiqhQP +3M3rtapI2YqAh4WaJUbCF2hx7SroyIMuBh/bnPpyMULvApVdy7stbcMuuQINBFt1 +96ABEAC3oNxNX+J8MoHflOhN9CUQSOzqxaH1MMvXFcKezd+lIsyaZ7PwFl6vUVOO +m0RwGff1Xtff7MGTvzDQRIEr0ZzREgUkuhIB35DBKzEJIrkKBTH21TeYu/xDo38g +cXJzUgkKx1coHkoCqCEaGmxHHwr53BALIAlYWSp7g0UIm1Wpk9y/kGwTMBjNIRvb +u3pYodEybcM2H0OF1LryH/LwReNPOae5fys0Djog6DHTN9ZOd36E+GNtbxSf/Gj2 +5KF1n2sa8T1B8LS0LNa2I4/sq7KkQi/IeUBybVAc7O42Khfr93FFFfLIxlSgtQZu +NGLPMtlLgGTrL+qo0fHRzN+GeK8tO3sFmpa4JUMLAlgUNdA0fe6piyfZ6iYqlU9K +E7SOqZPw6p3AXYcjvHRmxRZo+06yc/CtpL+Oekf/p1ER6hr4GHM4dBUUImLZ/0fC +7LiIoZ4HXySQEeTH97VUFNKIXLoFobrKoFJ6mxq1LDi5uWGniKR9eETk1aQnDKFe +nWi0sC/VeCFuW951L6ZriKa+noxHxwoLXEL7iR5htmz1bCCkSKUtxtO6oqsM7O78 +1FlfE3s4QMtgO9hlEElDBocBlVGba7A7EI4EAtawBnzujMh/QrkkIlBi8PXI2xKE +bNuce9FkIZ5CwIblVU8NwpqBM3zd1IWYy2Y8jKl0v9xJn9ZJxwARAQABiQIfBBgB +AgAJBQJbdfegAhsMAAoJEBYIdp4fEPZ562cP/3C7ngDAia772bOwBEq1VprZ++Jj +QkMKuro9UAeL1B64+Ainp9gz/+8nbjursQIMZWLPwCNWxPUO64SLh3Wi4AjClRTI +J2FsmVZiAM78Kxpi6qvIkS5A0LYjAs9PbB6gGhfhyJSVJj64q6aNYJr6XhHhsrxQ +W3gKde44wHQNuk6mDDk5cDHmi9U/oWEgT3orExgxPW1poEdEcO5189KglgzedAzS +QBNQcJRU8nuiKFi2IOv5wQQviV6+ngDI3zS6F3sOPLgABXwwI+dQDgLKXHyBnUsf +6IMKU5lciCMQZAcQQ+KLBo0b4QKLJbxOdv0UXE0sw8xxet5VhEdVIAI51gXrQ1eC +1eoP9MN6G1xnIJWrlwU6evyFrNNn5Idc5T9PD9nlBzyGRG5/r9GVX8HobawwAW3W +WHBmfzloKFj4E8sSLa4OTA872F41C5AElPW3BmaKOF7fZDtY3bHNNAXDWthVpy7+ +2nTxcIlotD8HJzl19OxqabJR0C/OpdpFIJmHIo82YdRj2VI1AIPKypU8kRdByzB9 +9Rj/PaIzjYS2PlwRzBhUtMqsEyJmSN5uotTVtpf9VD+YokxdSjx3opRDMvzz6Go5 +o3s7q+kCj9sdPsSkd6ZnjBGssCQvJN6mpL0zRwPn1IST96975twJbzoJe2DLDMCF +xlS+6Ph5M9Qett0S +=2DcU +-END PGP PUBLIC KEY BLOCK-
svn commit: r28741 - /dev/incubator/heron/heron-0.20.0-incubating-candidate-1/heron-v-0.20.0-incubating-candidate-1.tar.gz
Author: nlu90 Date: Wed Aug 15 21:10:43 2018 New Revision: 28741 Log: Apache heron 0.20.0-incubating release candidate 1 Added: dev/incubator/heron/heron-0.20.0-incubating-candidate-1/heron-v-0.20.0-incubating-candidate-1.tar.gz (with props) Added: dev/incubator/heron/heron-0.20.0-incubating-candidate-1/heron-v-0.20.0-incubating-candidate-1.tar.gz == Binary file - no diff available. Propchange: dev/incubator/heron/heron-0.20.0-incubating-candidate-1/heron-v-0.20.0-incubating-candidate-1.tar.gz -- svn:mime-type = application/octet-stream
svn commit: r28740 - /dev/incubator/heron/heron-0.20.0-incubating-candidate-1/
Author: nlu90 Date: Wed Aug 15 21:09:57 2018 New Revision: 28740 Log: Apache heron 0.20.0-incubating release candidate 1 Added: dev/incubator/heron/heron-0.20.0-incubating-candidate-1/
[incubator-heron] branch master updated: Docker Ubuntu tweaks (#2975)
This is an automated email from the ASF dual-hosted git repository.
nlu90 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
The following commit(s) were added to refs/heads/master by this push:
new d2946ce Docker Ubuntu tweaks (#2975)
d2946ce is described below
commit d2946ce0cfb3a6fe230a93d9f16550d7f46d2cf3
Author: Thomas Cooper
AuthorDate: Tue Aug 14 19:46:32 2018 +0100
Docker Ubuntu tweaks (#2975)
* Added .vagrant to gitignore
* Updated ubuntu docker images to use headless jdk + reduced update calls
- Altered the ubuntu dockerfiles to use the headless jdk, which prevents
the containers downloading a lot of pointless GUI libraries.
- Changed the command order of the 14.04 compile file to reduce the
overall number of `apt-get update` commands.
- Removed ubutnu 15.10 option as this version of ubuntu is no longer
supported with security updates etc.
- Updated docker/scripts help text to reflect the above
* Updated dockerfile and scripts for debian to v9 and remove apt-get clean
- Updated all references from debian 8 to 9 (as the Java slim images
uses debian 9).
- Updated all help text to refer to debian9.
- Removed apt-get clean commands from ubuntu docker files as they are
not required.
---
...erfile.base.debian8 => Dockerfile.base.debian9} | 0
.../{Dockerfile.debian8 => Dockerfile.debian9} | 0
docker/compile/Dockerfile.ubuntu14.04 | 16 +-
docker/compile/Dockerfile.ubuntu15.10 | 34 --
docker/compile/Dockerfile.ubuntu16.04 | 2 +-
...erfile.dist.debian8 => Dockerfile.dist.debian9} | 0
docker/dist/Dockerfile.dist.ubuntu14.04| 5 ++--
docker/dist/Dockerfile.dist.ubuntu15.10| 29 --
docker/dist/Dockerfile.dist.ubuntu16.04| 13 +
docker/scripts/build-artifacts.sh | 2 +-
docker/scripts/build-base.sh | 2 +-
docker/scripts/build-docker.sh | 2 +-
docker/scripts/build-exec-docker.sh| 2 +-
docker/scripts/ci-docker.sh| 2 +-
vagrant/.gitignore | 1 +
15 files changed, 24 insertions(+), 86 deletions(-)
diff --git a/docker/base/Dockerfile.base.debian8
b/docker/base/Dockerfile.base.debian9
similarity index 100%
rename from docker/base/Dockerfile.base.debian8
rename to docker/base/Dockerfile.base.debian9
diff --git a/docker/compile/Dockerfile.debian8
b/docker/compile/Dockerfile.debian9
similarity index 100%
rename from docker/compile/Dockerfile.debian8
rename to docker/compile/Dockerfile.debian9
diff --git a/docker/compile/Dockerfile.ubuntu14.04
b/docker/compile/Dockerfile.ubuntu14.04
index c0b6aa5..4282a00 100644
--- a/docker/compile/Dockerfile.ubuntu14.04
+++ b/docker/compile/Dockerfile.ubuntu14.04
@@ -4,6 +4,11 @@ FROM ubuntu:14.04
ENV TARGET_PLATFORM ubuntu
ENV bazelVersion 0.14.1
+RUN apt-get update && apt-get install -y software-properties-common
+
+RUN add-apt-repository ppa:george-edison55/cmake-3.x
+RUN add-apt-repository ppa:openjdk-r/ppa
+
RUN apt-get update && apt-get -y install \
automake \
build-essential \
@@ -14,17 +19,12 @@ RUN apt-get update && apt-get -y install \
python \
python2.7-dev \
python-software-properties \
- software-properties-common \
python-setuptools \
zip \
unzip \
- wget
-
-RUN add-apt-repository ppa:george-edison55/cmake-3.x && apt-get -y update && \
- apt-get -y install cmake
-
-RUN add-apt-repository ppa:openjdk-r/ppa && apt-get -y update
-RUN apt-get -y install openjdk-8-jdk
+ wget \
+ cmake \
+ openjdk-8-jdk-headless
ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
diff --git a/docker/compile/Dockerfile.ubuntu15.10
b/docker/compile/Dockerfile.ubuntu15.10
deleted file mode 100644
index ee9fa2f..000
--- a/docker/compile/Dockerfile.ubuntu15.10
+++ /dev/null
@@ -1,34 +0,0 @@
-FROM ubuntu:15.10
-
-# This is passed to the heron build command via the --config flag
-ENV TARGET_PLATFORM ubuntu
-ENV bazelVersion 0.14.1
-
-RUN apt-get update && apt-get -y install \
- automake \
- build-essential \
- cmake \
- curl \
- libssl-dev \
- git \
- libtool \
- libtool-bin \
- python \
- python2.7-dev \
- python-software-properties \
- software-properties-common \
- python-setuptools \
- zip \
- unzip \
- wget
-
-RUN apt-get -y install openjdk-8-jdk
-
-ENV JAVA_HOME /usr/lib/jvm/java-8-openjdk-amd64
-
-RUN wget -O /tmp/bazel.sh
https://github.com/bazelbuild/bazel/releases/download/$bazelVersion/bazel-$bazelVersion-installer-linux-x86_64.sh
\
- && chmod +x /tmp/bazel.sh \
- &&
[incubator-heron] branch master updated: update license information (#2977)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-heron.git The following commit(s) were added to refs/heads/master by this push: new 7bf83ac update license information (#2977) 7bf83ac is described below commit 7bf83ac769c74af7d59d1c9031ec7584b9845ded Author: Neng Lu AuthorDate: Tue Aug 14 11:01:46 2018 -0700 update license information (#2977) * update license information * update JQuery License --- LICENSE | 33 ++ NOTICE| 2 +- licenses/LICENSE-JQuery.txt | 36 +++ licenses/LICENSE-bootstrap-2.2.2.txt | 188 ++ licenses/LICENSE-bootstrap-3.0.0.txt | 188 ++ licenses/LICENSE-bootstrap-3.1.1.txt | 21 LICENSE => licenses/LICENSE-react.txt | 8 +- 7 files changed, 471 insertions(+), 5 deletions(-) diff --git a/LICENSE b/LICENSE index 261eeb9..8769d29 100644 --- a/LICENSE +++ b/LICENSE @@ -199,3 +199,36 @@ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. + + + + APACHE HERON SUBCOMPONENTS: + + The Apache Heron project contains subcomponents with separate copyright + notices and license terms. Your use of the source code for the these + subcomponents is subject to the terms and conditions of the following + licenses. + + +Third party Apache 2.0 licenses + + +The following components are provided under the Apache 2.0 License. +See project link for details. The text of each license is also included +at licenses/LICENSE-[project].txt. + +(ALv2 License) React (https://github.com/facebook/react/blob/0.10-stable/LICENSE) +(ALv2 License) Bootstrap(v2.2.2) +(ALv2 License) Bootstrap(v3.0.0) +(ALv2 License) JQuery(https://jquery.org/license/) + + +MIT licenses + + +The following components are provided under the MIT License. See project link for details. +The text of each license is also included at licenses/LICENSE-[project].txt. + +(MIT License) Bootstrap(v3.1.1) (https://github.com/twbs/bootstrap/blob/master/LICENSE) +(MIT License) JQuery (https://jquery.org/license/) + diff --git a/NOTICE b/NOTICE index 3605ffa..0ef7a26 100644 --- a/NOTICE +++ b/NOTICE @@ -1,4 +1,4 @@ -Apache Incubator-Heron +Apache Heron Copyright 2018 The Apache Software Foundation This product includes software developed at diff --git a/licenses/LICENSE-JQuery.txt b/licenses/LICENSE-JQuery.txt new file mode 100644 index 000..e4e5e00 --- /dev/null +++ b/licenses/LICENSE-JQuery.txt @@ -0,0 +1,36 @@ +Copyright JS Foundation and other contributors, https://js.foundation/ + +This software consists of voluntary contributions made by many +individuals. For exact contribution history, see the revision history +available at https://github.com/jquery/jquery + +The following license applies to all parts of this software except as +documented below: + + + +Permission is hereby granted, free of charge, to any person obtaining +a copy of this software and associated documentation files (the +"Software"), to deal in the Software without restriction, including +without limitation the rights to use, copy, modify, merge, publish, +distribute, sublicense, and/or sell copies of the Software, and to +permit persons to whom the Software is furnished to do so, subject to +the following conditions: + +The above copyright notice and this permission notice shall be +included in all copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, +EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF +MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND +NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE +LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION +OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION +WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE. + + + +All files located in the node_modules and external directories are +externally maintained libraries used by this software which have their +own licenses; we recommend you read them, as their terms may differ from +the terms above. diff --git a/licenses/LICENSE-bootstrap-2.2.2.txt b/licenses/LICENSE-bootstrap-2.2.2.txt new file mode 100644 index 000..83e8910 --- /dev/null +++ b/licenses/LICENSE-boot
[incubator-heron] branch master updated: Adding average Serialization/Deserialization time per tuple (#2976)
This is an automated email from the ASF dual-hosted git repository.
nlu90 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
The following commit(s) were added to refs/heads/master by this push:
new 2b5ab99 Adding average Serialization/Deserialization time per tuple
(#2976)
2b5ab99 is described below
commit 2b5ab994fe43a33ca42d492df8d5fb5b72ebf523
Author: Faria Kalim
AuthorDate: Thu Aug 9 13:29:39 2018 -0700
Adding average Serialization/Deserialization time per tuple (#2976)
* adding metric to measure average serialization and deserialization times
* adding a metric that measures average serialization and deserialization
time per tuple
---
.../common/utils/metrics/FullBoltMetrics.java | 31 +++---
.../apache/heron/instance/bolt/BoltInstance.java | 15 ++-
2 files changed, 31 insertions(+), 15 deletions(-)
diff --git
a/heron/common/src/java/org/apache/heron/common/utils/metrics/FullBoltMetrics.java
b/heron/common/src/java/org/apache/heron/common/utils/metrics/FullBoltMetrics.java
index 320bef0..84700d3 100644
---
a/heron/common/src/java/org/apache/heron/common/utils/metrics/FullBoltMetrics.java
+++
b/heron/common/src/java/org/apache/heron/common/utils/metrics/FullBoltMetrics.java
@@ -52,8 +52,10 @@ public class FullBoltMetrics extends BoltMetrics {
// Time in nano-seconds spending in execute() at every interval
private final MultiCountMetric executeTimeNs;
private final MultiCountMetric emitCount;
- private final MultiCountMetric deserializationTimeNs;
- private final MultiCountMetric serializationTimeNs;
+ private final MultiCountMetric totalDeserializationTimeNs;
+ private final MultiCountMetric totalSerializationTimeNs;
+ private final MultiReducedMetric
averageSerializationTimeNs;
+ private final MultiReducedMetric
averageDeserializationTimeNs;
// The # of times back-pressure happens on outStreamQueue
// so instance could not produce more tuples
@@ -71,8 +73,11 @@ public class FullBoltMetrics extends BoltMetrics {
emitCount = new MultiCountMetric();
outQueueFullCount = new CountMetric();
-deserializationTimeNs = new MultiCountMetric();
-serializationTimeNs = new MultiCountMetric();
+totalDeserializationTimeNs = new MultiCountMetric();
+totalSerializationTimeNs = new MultiCountMetric();
+
+averageSerializationTimeNs = new MultiReducedMetric<>(new MeanReducer());
+averageDeserializationTimeNs = new MultiReducedMetric<>(new MeanReducer());
}
public void registerMetrics(TopologyContextImpl topologyContext) {
@@ -91,8 +96,13 @@ public class FullBoltMetrics extends BoltMetrics {
topologyContext.registerMetric("__emit-count", emitCount, interval);
topologyContext.registerMetric("__out-queue-full-count",
outQueueFullCount, interval);
topologyContext.registerMetric(
-"__tuple-deserialization-time-ns", deserializationTimeNs, interval);
-topologyContext.registerMetric("__tuple-serialization-time-ns",
serializationTimeNs, interval);
+"__tuple-deserialization-time-ns", totalDeserializationTimeNs,
interval);
+topologyContext.registerMetric(
+"__tuple-serialization-time-ns", totalSerializationTimeNs, interval);
+topologyContext.registerMetric(
+"__av-tuple-deserialization-time-ns", totalDeserializationTimeNs,
interval);
+topologyContext.registerMetric(
+"__av-tuple-serialization-time-ns", totalSerializationTimeNs,
interval);
}
// For MultiCountMetrics, we need to set the default value for all streams.
@@ -174,17 +184,20 @@ public class FullBoltMetrics extends BoltMetrics {
}
public void deserializeDataTuple(String streamId, String sourceComponent,
long latency) {
-deserializationTimeNs.scope(streamId).incrBy(latency);
+totalDeserializationTimeNs.scope(streamId).incrBy(latency);
+averageDeserializationTimeNs.scope(streamId).update(latency);
// Consider there are cases that different streams with the same streamId,
// but with different source component. We need to distinguish them too.
String globalStreamId =
new
StringBuilder(sourceComponent).append("/").append(streamId).toString();
-deserializationTimeNs.scope(globalStreamId).incrBy(latency);
+totalDeserializationTimeNs.scope(globalStreamId).incrBy(latency);
+averageDeserializationTimeNs.scope(globalStreamId).update(latency);
}
public void serializeDataTuple(String streamId, long latency) {
-serializationTimeNs.scope(streamId).incrBy(latency);
+totalSerializationTimeNs.scope(streamId).incrBy(latency);
+averageSerializationTimeNs.scope(streamId).update(latency);
}
}
diff --git
a/heron/instance/src/java/org/apache/heron/instance/bolt/BoltInstance.java
b/heron/instance/src/java/org/apache/heron/instan
[incubator-heron] branch master updated: Update error message in RuntimeManagerMain when an exception is catched (#2974)
This is an automated email from the ASF dual-hosted git repository.
nlu90 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
The following commit(s) were added to refs/heads/master by this push:
new 9818de3 Update error message in RuntimeManagerMain when an exception
is catched (#2974)
9818de3 is described below
commit 9818de3113875d43b1ddc17c4f758865aabf4920
Author: Ning Wang
AuthorDate: Wed Aug 8 10:15:03 2018 -0700
Update error message in RuntimeManagerMain when an exception is catched
(#2974)
---
.../src/java/org/apache/heron/scheduler/RuntimeManagerMain.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
diff --git
a/heron/scheduler-core/src/java/org/apache/heron/scheduler/RuntimeManagerMain.java
b/heron/scheduler-core/src/java/org/apache/heron/scheduler/RuntimeManagerMain.java
index f00a8be..cd4da42 100644
---
a/heron/scheduler-core/src/java/org/apache/heron/scheduler/RuntimeManagerMain.java
+++
b/heron/scheduler-core/src/java/org/apache/heron/scheduler/RuntimeManagerMain.java
@@ -334,7 +334,7 @@ public class RuntimeManagerMain {
System.exit(200);
// SUPPRESS CHECKSTYLE IllegalCatch
} catch (Exception e) {
- LOG.log(Level.FINE, "Exception when submitting topology", e);
+ LOG.log(Level.FINE, "Exception when executing command " + commandOption,
e);
System.out.println(e.getMessage());
// Exit with status code 100 to indicate that error has happened on
user-land
// SUPPRESS CHECKSTYLE RegexpSinglelineJava
[incubator-heron] branch master updated: spill state to local disk when configured (#2967)
This is an automated email from the ASF dual-hosted git repository.
nlu90 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
The following commit(s) were added to refs/heads/master by this push:
new 5c32f4d spill state to local disk when configured (#2967)
5c32f4d is described below
commit 5c32f4dedd5b8ebd3daebb5d2f69732c070d4c7f
Author: Neng Lu
AuthorDate: Fri Jul 27 14:31:53 2018 -0700
spill state to local disk when configured (#2967)
* spill state to local disk when configured
* start ckptmgr-client after receiving pplan
* separate instances and ckptmgr spilled location
* clean states by owner
* bail out if spill state failed
* typo
---
.../api/StatefulTumblingWindowTopology.java| 4 +-
.../api/src/java/org/apache/heron/api/Config.java | 30
.../java/org/apache/heron/api/HeronTopology.java | 6 ++
heron/ckptmgr/src/java/BUILD | 8 ++-
.../heron/ckptmgr/CheckpointManagerServer.java | 79 --
.../heron/ckptmgr/CheckpointManagerServerTest.java | 14
.../heron/instance/AbstractOutputCollector.java| 6 +-
.../heron/instance/OutgoingTupleCollection.java| 36 +++---
.../src/java/org/apache/heron/instance/Slave.java | 10 +++
.../apache/heron/instance/bolt/BoltInstance.java | 31 +++--
.../apache/heron/instance/spout/SpoutInstance.java | 22 +-
heron/proto/ckptmgr.proto | 7 +-
heron/stmgr/src/cpp/manager/ckptmgr-client.cpp | 6 ++
heron/stmgr/src/cpp/manager/ckptmgr-client.h | 6 +-
heron/stmgr/src/cpp/manager/stmgr.cpp | 38 +--
15 files changed, 250 insertions(+), 53 deletions(-)
diff --git
a/examples/src/java/org/apache/heron/examples/api/StatefulTumblingWindowTopology.java
b/examples/src/java/org/apache/heron/examples/api/StatefulTumblingWindowTopology.java
index b39e3ff..62c8185 100644
---
a/examples/src/java/org/apache/heron/examples/api/StatefulTumblingWindowTopology.java
+++
b/examples/src/java/org/apache/heron/examples/api/StatefulTumblingWindowTopology.java
@@ -155,7 +155,8 @@ public final class StatefulTumblingWindowTopology {
TopologyBuilder builder = new TopologyBuilder();
builder.setSpout("integer", new IntegerSpout(), 1);
WindowSumBolt windowSumBolt = new WindowSumBolt();
-windowSumBolt.withTumblingWindow(Duration.ofSeconds(10));
+windowSumBolt.withTumblingWindow(Duration.ofMinutes(10));
+
builder.setBolt("sumbolt", windowSumBolt, 1).shuffleGrouping("integer");
builder.setBolt("printer", new PrinterBolt()).shuffleGrouping("sumbolt");
Config conf = new Config();
@@ -172,6 +173,7 @@ public final class StatefulTumblingWindowTopology {
conf.setTopologyReliabilityMode(Config.TopologyReliabilityMode.EFFECTIVELY_ONCE);
conf.setTopologyStatefulCheckpointIntervalSecs(20);
conf.setMaxSpoutPending(1000);
+conf.setMessageTimeoutSecs(1500);
if (args != null && args.length > 0) {
topoName = args[0];
diff --git a/heron/api/src/java/org/apache/heron/api/Config.java
b/heron/api/src/java/org/apache/heron/api/Config.java
index 2e22731..d91c8a0 100644
--- a/heron/api/src/java/org/apache/heron/api/Config.java
+++ b/heron/api/src/java/org/apache/heron/api/Config.java
@@ -223,6 +223,16 @@ public class Config extends HashMap {
public static final String TOPOLOGY_STATEFUL_CKPTMGR_RAM =
"topology.stateful.checkpointmanager.ram";
/**
+ * Whether spill the state to disk for transferring
+ */
+ public static final String TOPOLOGY_STATEFUL_SPILL_STATE =
+"topology.stateful.spill.state";
+ /**
+ * The local disk location where the state is spilled
+ */
+ public static final String TOPOLOGY_STATEFUL_SPILL_STATE_LOCATION =
+"topology.stateful.spill.state.location";
+ /**
* Stream Manager RAM requirement
*/
public static final String TOPOLOGY_STMGR_RAM = "topology.stmgr.ram";
@@ -333,6 +343,8 @@ public class Config extends HashMap {
apiVars.add(TOPOLOGY_STATEFUL_START_CLEAN);
apiVars.add(TOPOLOGY_STATEFUL_CHECKPOINT_INTERVAL_SECONDS);
apiVars.add(TOPOLOGY_STATEFUL_CKPTMGR_RAM);
+apiVars.add(TOPOLOGY_STATEFUL_SPILL_STATE);
+apiVars.add(TOPOLOGY_STATEFUL_SPILL_STATE_LOCATION);
apiVars.add(TOPOLOGY_STMGR_RAM);
apiVars.add(TOPOLOGY_METRICSMGR_RAM);
apiVars.add(TOPOLOGY_RELIABILITY_MODE);
@@ -593,6 +605,16 @@ public class Config extends HashMap {
conf.put(Config.TOPOLOGY_STATEFUL_CKPTMGR_RAM, ramInBytes.asBytes());
}
+ public static void setTopologyStatefulSpillState(Map conf,
+ String spillState) {
+conf.put(Config.TOPOLOGY_STATEFUL_S
[incubator-heron] branch master updated: Make checkpoint manager's ram configurable (#2958)
This is an automated email from the ASF dual-hosted git repository.
nlu90 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
The following commit(s) were added to refs/heads/master by this push:
new c68c7be Make checkpoint manager's ram configurable (#2958)
c68c7be is described below
commit c68c7beeadf713927c3fc13a2a633303dbe315dc
Author: Neng Lu
AuthorDate: Tue Jul 10 23:28:32 2018 -0700
Make checkpoint manager's ram configurable (#2958)
* calculate container padding ram and make ckptmgr configurable
* add ckptmgr ram to aurora conf
* fix unit tests
---
.../api/src/java/org/apache/heron/api/Config.java | 43 ++
.../org/apache/heron/api/utils/TopologyUtils.java | 7
heron/executor/src/python/heron_executor.py| 9 -
.../tests/python/heron_executor_unittest.py| 1 +
.../packing/roundrobin/RoundRobinPacking.java | 23 +++-
.../org/apache/heron/scheduler/ExecutorFlag.java | 1 +
.../heron/scheduler/utils/SchedulerUtils.java | 3 ++
.../heron/scheduler/aurora/AuroraScheduler.java| 2 +-
.../scheduler/aurora/AuroraSchedulerTest.java | 1 +
.../java/org/apache/heron/spi/common/Context.java | 8
.../src/java/org/apache/heron/spi/common/Key.java | 10 +++--
.../org/apache/heron/spi/common/ContextTest.java | 2 +
12 files changed, 102 insertions(+), 8 deletions(-)
diff --git a/heron/api/src/java/org/apache/heron/api/Config.java
b/heron/api/src/java/org/apache/heron/api/Config.java
index 77ce91e..2e22731 100644
--- a/heron/api/src/java/org/apache/heron/api/Config.java
+++ b/heron/api/src/java/org/apache/heron/api/Config.java
@@ -218,6 +218,19 @@ public class Config extends HashMap {
public static final String TOPOLOGY_STATEFUL_START_CLEAN =
"topology.stateful.start.clean";
/**
+ * Checkpoint Manager RAM requirement.
+ */
+ public static final String TOPOLOGY_STATEFUL_CKPTMGR_RAM =
+"topology.stateful.checkpointmanager.ram";
+ /**
+ * Stream Manager RAM requirement
+ */
+ public static final String TOPOLOGY_STMGR_RAM = "topology.stmgr.ram";
+ /**
+ * Metrics Manager RAM requirement
+ */
+ public static final String TOPOLOGY_METRICSMGR_RAM =
"topology.metricsmgr.ram";
+ /**
* Name of the topology. This config is automatically set by Heron when the
topology is submitted.
*/
public static final String TOPOLOGY_NAME = "topology.name";
@@ -319,6 +332,9 @@ public class Config extends HashMap {
apiVars.add(TOPOLOGY_COMPONENT_DISKMAP);
apiVars.add(TOPOLOGY_STATEFUL_START_CLEAN);
apiVars.add(TOPOLOGY_STATEFUL_CHECKPOINT_INTERVAL_SECONDS);
+apiVars.add(TOPOLOGY_STATEFUL_CKPTMGR_RAM);
+apiVars.add(TOPOLOGY_STMGR_RAM);
+apiVars.add(TOPOLOGY_METRICSMGR_RAM);
apiVars.add(TOPOLOGY_RELIABILITY_MODE);
apiVars.add(TOPOLOGY_NAME);
apiVars.add(TOPOLOGY_TEAM_NAME);
@@ -572,6 +588,21 @@ public class Config extends HashMap {
conf.put(Config.TOPOLOGY_STATEFUL_START_CLEAN, String.valueOf(clean));
}
+ public static void setCheckpointManagerRam(Map conf,
+ ByteAmount ramInBytes) {
+conf.put(Config.TOPOLOGY_STATEFUL_CKPTMGR_RAM, ramInBytes.asBytes());
+ }
+
+ public static void setStreamManagerRam(Map conf,
+ ByteAmount ramInBytes) {
+conf.put(Config.TOPOLOGY_STMGR_RAM, ramInBytes.asBytes());
+ }
+
+ public static void setMetricsmgrRam(Map conf,
+ ByteAmount ramInBytes) {
+conf.put(Config.TOPOLOGY_METRICSMGR_RAM, ramInBytes.asBytes());
+ }
+
@SuppressWarnings("rawtypes")
public static void setEnvironment(Map conf, Map env) {
conf.put(Config.TOPOLOGY_ENVIRONMENT, env);
@@ -738,6 +769,18 @@ public class Config extends HashMap {
setTopologyStatefulStartClean(this, clean);
}
+ public void setCheckpointManagerRam(ByteAmount ramInBytes) {
+setCheckpointManagerRam(this, ramInBytes);
+ }
+
+ public void setStreamManagerRam(ByteAmount ramInBytes) {
+setStreamManagerRam(this, ramInBytes);
+ }
+
+ public void setMetricsmgrRam(ByteAmount ramInBytes) {
+setMetricsmgrRam(this, ramInBytes);
+ }
+
/**
* Registers a timer event that executes periodically
* @param conf the map with the existing topology configs
diff --git a/heron/api/src/java/org/apache/heron/api/utils/TopologyUtils.java
b/heron/api/src/java/org/apache/heron/api/utils/TopologyUtils.java
index 99b96d1..b61c767 100644
--- a/heron/api/src/java/org/apache/heron/api/utils/TopologyUtils.java
+++ b/heron/api/src/java/org/apache/heron/api/utils/TopologyUtils.java
@@ -157,6 +157,13 @@ public final class TopologyUtils {
}
}
+ public static ByteAmount getCheckpointManagerRam(TopologyAPI.Topology
topology) {
+Li
[incubator-heron] branch master updated: Refactor StatefulStorage API (#2891)
This is an automated email from the ASF dual-hosted git repository.
nlu90 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
The following commit(s) were added to refs/heads/master by this push:
new 581344e Refactor StatefulStorage API (#2891)
581344e is described below
commit 581344ea42033a2f5e826d487c355061cee58241
Author: Ning Wang
AuthorDate: Sat Jul 7 11:23:16 2018 -0700
Refactor StatefulStorage API (#2891)
* Refactor StatefulStorage
* fix test
* fix CheckpointManagerServerTest
* component metadata is optional
* Update names to be more consistent
* remove data_version and partition_id from protobuf for now
* Remove metadata field from save/get checkpoint requests
* More name and comment changes
* revert refactor changes in CheckpointManagerServer
* Rename InstanceStateCheckpointPartition to InstanceStateCheckpoint
---
.../apache/heron/ckptmgr/CheckpointManager.java| 2 +-
.../heron/ckptmgr/CheckpointManagerServer.java | 69
.../heron/ckptmgr/CheckpointManagerServerTest.java | 32 ++
heron/proto/ckptmgr.proto | 43 +
.../heron/spi/statefulstorage/Checkpoint.java | 38 ++-
.../{Checkpoint.java => CheckpointInfo.java} | 48 ++
.../spi/statefulstorage/CheckpointMetadata.java| 47 ++
.../spi/statefulstorage/IStatefulStorage.java | 74 +++---
.../heron/statefulstorage/dlog/DlogStorage.java| 48 +-
.../heron/statefulstorage/hdfs/HDFSStorage.java| 66 +++
.../localfs/LocalFileSystemStorage.java| 71 -
.../statefulstorage/dlog/DlogStorageTest.java | 43 +
.../statefulstorage/hdfs/HDFSStorageTest.java | 25
.../localfs/LocalFileSystemStorageTest.java| 16 +++--
heron/stmgr/src/cpp/manager/ckptmgr-client.cpp | 3 +-
heron/stmgr/src/cpp/manager/ckptmgr-client.h | 3 +-
heron/stmgr/src/cpp/manager/instance-server.h | 2 +-
heron/stmgr/src/cpp/manager/stateful-restorer.cpp | 4 +-
heron/stmgr/src/cpp/manager/stmgr.h| 5 +-
.../tests/cpp/server/dummy_ckptmgr_client.cpp | 9 +--
.../stmgr/tests/cpp/server/dummy_instance_server.h | 2 +-
21 files changed, 385 insertions(+), 265 deletions(-)
diff --git
a/heron/ckptmgr/src/java/org/apache/heron/ckptmgr/CheckpointManager.java
b/heron/ckptmgr/src/java/org/apache/heron/ckptmgr/CheckpointManager.java
index 5c590ff..4798386 100644
--- a/heron/ckptmgr/src/java/org/apache/heron/ckptmgr/CheckpointManager.java
+++ b/heron/ckptmgr/src/java/org/apache/heron/ckptmgr/CheckpointManager.java
@@ -193,7 +193,7 @@ public class CheckpointManager {
}
try {
- statefulStorage.init(
+ statefulStorage.init(topologyName,
Collections.unmodifiableMap(checkpointManagerConfig.getStatefulStorageConfig()));
} catch (StatefulStorageException e) {
throw new CheckpointManagerException(classname + " init threw
exception", e);
diff --git
a/heron/ckptmgr/src/java/org/apache/heron/ckptmgr/CheckpointManagerServer.java
b/heron/ckptmgr/src/java/org/apache/heron/ckptmgr/CheckpointManagerServer.java
index 46ce36b..4b4654a 100644
---
a/heron/ckptmgr/src/java/org/apache/heron/ckptmgr/CheckpointManagerServer.java
+++
b/heron/ckptmgr/src/java/org/apache/heron/ckptmgr/CheckpointManagerServer.java
@@ -34,6 +34,7 @@ import org.apache.heron.common.network.REQID;
import org.apache.heron.proto.ckptmgr.CheckpointManager;
import org.apache.heron.proto.system.Common;
import org.apache.heron.spi.statefulstorage.Checkpoint;
+import org.apache.heron.spi.statefulstorage.CheckpointInfo;
import org.apache.heron.spi.statefulstorage.IStatefulStorage;
import org.apache.heron.spi.statefulstorage.StatefulStorageException;
@@ -48,9 +49,14 @@ public class CheckpointManagerServer extends HeronServer {
private SocketChannel connection;
public CheckpointManagerServer(
- String topologyName, String topologyId, String checkpointMgrId,
- IStatefulStorage statefulStorage, NIOLooper looper, String host,
- int port, HeronSocketOptions options) {
+ String topologyName,
+ String topologyId,
+ String checkpointMgrId,
+ IStatefulStorage statefulStorage,
+ NIOLooper looper,
+ String host,
+ int port,
+ HeronSocketOptions options) {
super(looper, host, port, options);
this.topologyName = topologyName;
@@ -115,8 +121,7 @@ public class CheckpointManagerServer extends HeronServer {
String errorMessage = "";
try {
- statefulStorage.dispose(topologyName,
- request.getOldestCheckpointPreserved(),
deleteAll);
+ statefulStorage.dispose(request.getOldestCheckpointPreserved(),
deleteAll);
LOG.i
[incubator-heron] branch master updated: modify travis ci to enable integration topology test (#2940)
This is an automated email from the ASF dual-hosted git repository.
nlu90 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
The following commit(s) were added to refs/heads/master by this push:
new 5b79124 modify travis ci to enable integration topology test (#2940)
5b79124 is described below
commit 5b7912481f92e80efcb6c368d1ccd08917269858
Author: Yao Li
AuthorDate: Fri Jun 29 10:50:01 2018 -0700
modify travis ci to enable integration topology test (#2940)
* modify travis ci to enable integration topology test
* fix java test data dir
---
scripts/packages/BUILD | 12
scripts/packages/tests_template_bin.sh | 9 -
scripts/travis/test.sh | 12
3 files changed, 32 insertions(+), 1 deletion(-)
diff --git a/scripts/packages/BUILD b/scripts/packages/BUILD
index b32411c..01bdc2f 100644
--- a/scripts/packages/BUILD
+++ b/scripts/packages/BUILD
@@ -348,6 +348,7 @@ pkg_tar(
srcs = [
"//integration_test/src/python/http_server:http-server",
"//integration_test/src/python/test_runner:test-runner",
+
"//integration_test/src/python/topology_test_runner:topology-test-runner",
]
)
@@ -370,6 +371,15 @@ pkg_tar(
)
pkg_tar(
+name = "heron-tests-data-topology-java",
+package_dir = "data/java/topology_test",
+srcs = [
+"//integration_test/src/java:topology-test-data-files",
+],
+strip_prefix =
'/integration_test/src/java/org/apache/heron/integration_topology_test/topology/'
+)
+
+pkg_tar(
name = "heron-tests-data-python",
package_dir = "data/python",
srcs = [
@@ -384,6 +394,7 @@ pkg_tar(
srcs = [
"//integration_test/src/scala:scala-integration-tests",
"//integration_test/src/java:integration-tests",
+ "//integration_test/src/java:integration-topology-tests",
"//integration_test/src/python/integration_test/topology:heron_integ_topology",
],
)
@@ -396,6 +407,7 @@ pkg_tar(
":heron-tests-bin",
":heron-tests-data-scala",
":heron-tests-data-java",
+":heron-tests-data-topology-java",
":heron-tests-data-python",
":heron-tests-lib",
],
diff --git a/scripts/packages/tests_template_bin.sh
b/scripts/packages/tests_template_bin.sh
index 66567b3..b3604a2 100755
--- a/scripts/packages/tests_template_bin.sh
+++ b/scripts/packages/tests_template_bin.sh
@@ -71,12 +71,18 @@ echo -n "Uncompressing."
if [ -L "${bin}/test-runner" ]; then
rm -f "${bin}/test-runner"
fi
+if [ -L "${bin}/topology-test-runner" ]; then
+ rm -f "${bin}/topology-test-runner"
+fi
if [ -L "${bin}/http-server" ]; then
rm -f "${bin}/http-server"
fi
if [ -d "${base}" -a -x "${base}/bin/test-runner" ]; then
rm -fr "${base}"
fi
+if [ -d "${base}" -a -x "${base}/bin/topology-test-runner" ]; then
+ rm -fr "${base}"
+fi
mkdir -p ${bin} ${base}
echo -n .
@@ -84,7 +90,7 @@ echo -n .
unzip -q -o "${BASH_SOURCE[0]}" -d "${base}"
untar ${base}/heron-tests.tar.gz ${base}
echo -n .
-chmod 0755 ${base}/bin/test-runner ${base}/bin/http-server
+chmod 0755 ${base}/bin/test-runner ${base}/bin/http-server
${base}/bin/topology-test-runner
echo -n .
chmod -R og-w "${base}"
chmod -R og+rX "${base}"
@@ -92,6 +98,7 @@ chmod -R u+rwX "${base}"
echo -n .
ln -s "${base}/bin/test-runner" "${bin}/test-runner"
+ln -s "${base}/bin/topology-test-runner" "${bin}/topology-test-runner"
ln -s "${base}/bin/http-server" "${bin}/http-server"
mv "${base}/lib/heron_integ_topology" "${base}/lib/heron_integ_topology.pex"
echo -n .
diff --git a/scripts/travis/test.sh b/scripts/travis/test.sh
index 2cece60..1c22fc1 100755
--- a/scripts/travis/test.sh
+++ b/scripts/travis/test.sh
@@ -14,6 +14,7 @@ echo "Using $PLATFORM platform"
# integration test binaries have to be specified as absolute path
JAVA_INTEGRATION_TESTS_BIN="${HOME}/.herontests/lib/integration-tests.jar"
+JAVA_INTEGRATION_TOPOLOGY_TESTS_BIN="${HOME}/.herontests/lib/integration-topology-tests.jar"
PYTHON_INTEGRATION_TESTS_BIN="${HOME}/.herontests/lib/heron_integ_topology.pex"
SCALA_INTEGRATION_TESTS_BIN="${HOME}/.herontests/lib/scala-integration-tests.jar"
@@ -79,4 +80,15 @@ ${HOME}/bin/test-runner \
-cl local -rl heron-staging -ev devel
end_timer "$T"
+# run the java integration topology test
+T="heron integration_topology_test java"
+start_timer "$T"
+${HOME}/bin/topology-test-runner \
+ -hc heron -tb ${JAVA_INTEGRATION_TOPOLOGY_TESTS_BIN} \
+ -rh localhost -rp 8080\
+ -tp ${HOME}/.herontests/data/java/topology_test \
+ -cl local -rl heron-staging -ev devel
+end_timer "$T"
+
print_timer_summary
+
[incubator-heron] branch master updated: add tmaster physical plan endpoint (#2941)
This is an automated email from the ASF dual-hosted git repository.
nlu90 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
The following commit(s) were added to refs/heads/master by this push:
new 50a8484 add tmaster physical plan endpoint (#2941)
50a8484 is described below
commit 50a8484f555a8459205d05c98a940898507e8927
Author: Yao Li
AuthorDate: Fri Jun 29 10:48:11 2018 -0700
add tmaster physical plan endpoint (#2941)
* add tmaster pplan endpoint
* fix http_server->InstallCallBack
* fix request deletion
---
heron/tmaster/src/cpp/BUILD | 1 +
heron/tmaster/src/cpp/manager/tcontroller.cpp | 38 +++
heron/tmaster/src/cpp/manager/tcontroller.h | 1 +
3 files changed, 40 insertions(+)
diff --git a/heron/tmaster/src/cpp/BUILD b/heron/tmaster/src/cpp/BUILD
index 91cf0b5..e337034 100644
--- a/heron/tmaster/src/cpp/BUILD
+++ b/heron/tmaster/src/cpp/BUILD
@@ -51,6 +51,7 @@ cc_library(
"//heron/statemgrs/src/cpp:statemgrs-cxx",
"//heron/proto:proto-cxx",
"@com_github_jbeder_yaml_cpp//:yaml-cxx",
+"@com_github_cereal//:cereal-cxx",
],
linkstatic = 1,
)
diff --git a/heron/tmaster/src/cpp/manager/tcontroller.cpp
b/heron/tmaster/src/cpp/manager/tcontroller.cpp
index 1afabc0..b7a4ee6 100644
--- a/heron/tmaster/src/cpp/manager/tcontroller.cpp
+++ b/heron/tmaster/src/cpp/manager/tcontroller.cpp
@@ -26,6 +26,7 @@
#include
#include "basics/basics.h"
#include "basics/strutils.h"
+#include "cereal/external/base64.hpp"
#include "config/topology-config-helper.h"
#include "errors/errors.h"
#include "manager/tmaster.h"
@@ -65,6 +66,12 @@ TController::TController(EventLoop* eventLoop, const
NetworkOptions& options, TM
this->HandleUpdateRuntimeConfigRequest(request);
};
http_server_->InstallCallBack("/runtime_config/update",
std::move(cbUpdateRuntimeConfg));
+
+ // Get current physical plan
+ auto cbGetCurPPlan = [this](IncomingHTTPRequest* request) {
+this->HandleGetCurPPlanRequest(request);
+ };
+ http_server_->InstallCallBack("/get_current_physical_plan",
std::move(cbGetCurPPlan));
}
TController::~TController() { delete http_server_; }
@@ -262,6 +269,37 @@ void
TController::HandleUpdateRuntimeConfigRequestDone(IncomingHTTPRequest* requ
delete request;
}
+void TController::HandleGetCurPPlanRequest(IncomingHTTPRequest* request) {
+ LOG(INFO) << "Got a GetCurPPlan request from " << request->GetRemoteHost()
<< ":"
+ << request->GetRemotePort();
+
+ // make sure all the stream managers are alive, in case that when container
is fail,
+ // physical plan is still available at TMaster but not a valid one.
+ if (tmaster_->GetStmgrsRegSummary()->absent_stmgrs_size() != 0) {
+ http_server_->SendErrorReply(request, 400);
+ delete request;
+ return;
+ }
+
+ if (tmaster_->getPhysicalPlan() == NULL) {
+http_server_->SendErrorReply(request, 400);
+ } else {
+std::string pplanString;
+tmaster_->getPhysicalPlan()->SerializeToString();
+
+// SerializeToString() returns object in binary format which needs to be
encoded
+const unsigned char * encodeString = (unsigned char *)pplanString.c_str();
+std::string pplanStringFixed = cereal::base64::encode(encodeString,
pplanString.size());
+
+const std::string message("Get current physical plan");
+LOG(INFO) << message;
+OutgoingHTTPResponse* response = new OutgoingHTTPResponse(request);
+response->AddResponse(pplanStringFixed);
+http_server_->SendReply(request, 200, response);
+ }
+ delete request;
+}
+
/*
* Validate topology.
* - topology id matches
diff --git a/heron/tmaster/src/cpp/manager/tcontroller.h
b/heron/tmaster/src/cpp/manager/tcontroller.h
index beb8fb7..f7e7a48 100644
--- a/heron/tmaster/src/cpp/manager/tcontroller.h
+++ b/heron/tmaster/src/cpp/manager/tcontroller.h
@@ -66,6 +66,7 @@ class TController {
void HandleUpdateRuntimeConfigRequest(IncomingHTTPRequest* request);
void HandleUpdateRuntimeConfigRequestDone(IncomingHTTPRequest* request,
proto::system::StatusCode);
+ void HandleGetCurPPlanRequest(IncomingHTTPRequest* request);
// We are a http server
HTTPServer* http_server_;
[incubator-heron] branch master updated: add integration topology test (#2938)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-heron.git The following commit(s) were added to refs/heads/master by this push: new 76be949 add integration topology test (#2938) 76be949 is described below commit 76be949215120fcf6650c13b4c2247a27ad17ffa Author: Yao Li AuthorDate: Wed Jun 27 10:19:47 2018 -0700 add integration topology test (#2938) * add integration topology test * add integration test for topology structure * clean up txt files * move encode(decode) to strutil * add const variables in strutils.cpp * fix hexEncode/Decode * simplify decodeHexChar * Remove pplan endpoint * remove pplan endpoint again --- integration_test/src/java/BUILD| 65 ++- .../common/AbstractTestTopology.java | 93 .../common/BasicConfig.java| 45 ++ .../BasicTopologyOneTask.java | 51 ++ .../BasicTopologyOneTaskResults.json | 59 +++ .../BasicTopologyOneTask.java | 51 ++ .../BasicTopologyOneTaskResults.json | 71 +++ .../BasicTopologyOneTask.java | 51 ++ .../BasicTopologyOneTaskResults.json | 65 +++ .../topology/fields_grouping/FieldsGrouping.java | 54 ++ .../fields_grouping/FieldsGroupingResults.json | 84 +++ .../src/python/topology_test_runner/BUILD | 19 + .../src/python/topology_test_runner/main.py| 568 + .../topology_test_runner/resources/test.json | 52 ++ scripts/run_integration_topology_test.sh | 51 ++ 15 files changed, 1378 insertions(+), 1 deletion(-) diff --git a/integration_test/src/java/BUILD b/integration_test/src/java/BUILD index 3792315..b2a7c8f 100644 --- a/integration_test/src/java/BUILD +++ b/integration_test/src/java/BUILD @@ -4,7 +4,12 @@ package(default_visibility = ["//visibility:public"]) filegroup( name = "test-data-files", -srcs = glob(["**/*.json"]), +srcs = glob(["org/apache/heron/integration_test/topology/**/*.json"]), +) + +filegroup( +name = "topology-test-data-files", +srcs = glob(["org/apache/heron/integration_topology_test/topology/**/*.json"]), ) java_library( @@ -114,3 +119,61 @@ genrule( outs = ["local-integration-tests.jar"], cmd = "cp $< $@", ) + + +java_library( +name = "common_topology_test", +srcs = glob( +["org/apache/heron/integration_topology_test/common/**/*.java"], +), +deps = [ +"//heron/api/src/java:api-java", +"//heron/api/src/java:api-java-low-level", +"//storm-compatibility/src/java:storm-compatibility-java", +"//third_party/java:hadoop-core", +"//third_party/java:jackson", +"@commons_cli_commons_cli//jar", +":core" +], +) + +java_library( +name = "integration-topology-tests-lib", +srcs = glob( + ["org/apache/heron/integration_topology_test/topology/**/*.java"], +), +deps = [ +"//heron/api/src/java:api-java", +"//heron/api/src/java:api-java-low-level", +"//storm-compatibility/src/java:storm-compatibility-java", +"@com_googlecode_json_simple_json_simple//jar", +"@commons_cli_commons_cli//jar", +":common_topology_test", +":common", +":core" +], +) + +java_binary( +name = "integration-topology-tests-unshaded", +srcs = glob( + ["org/apache/heron/integration_topology_test/topology/**/*.java"], +), +deps = [ +"//heron/api/src/java:api-java", +"//heron/api/src/java:api-java-low-level", +"//storm-compatibility/src/java:storm-compatibility-java", +"@commons_cli_commons_cli//jar", +"@com_googlecode_json_simple_json_simple//jar", +":common_topology_test", +":common", +":core" +], +) + +genrule( +name = "integration-topology-tests", +srcs = [":integration-topology-tests-unshaded_deploy.jar"], +outs = ["integration-topology-tests.jar"], +cmd = "cp $< $@", +) diff --git a/integration_test/src/java/org/apache/heron/integration_topology_test/common/AbstractTestTopology.java b/integration_test/src/java/org/apache/heron/integration_topology_test/common/AbstractTestTopology.java new file mode 100644 index 000..97ce86e --- /dev/null +++ b/integration_test/src/java/org/apache/heron/integration_topo
[incubator-heron] branch master updated: make LocalFSStorage clean checkpoints before store (#2910)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-heron.git The following commit(s) were added to refs/heads/master by this push: new c13198b make LocalFSStorage clean checkpoints before store (#2910) c13198b is described below commit c13198b79037246d5baecd8cc35f5454b5db27d5 Author: Neng Lu AuthorDate: Mon Jun 4 13:23:41 2018 -0700 make LocalFSStorage clean checkpoints before store (#2910) * make LocalFSStorage clean checkpoints before store * update and add unit tests --- .../config/src/yaml/conf/kubernetes/stateful.yaml | 1 + heron/config/src/yaml/conf/local/stateful.yaml | 1 + heron/config/src/yaml/conf/localzk/stateful.yaml | 1 + heron/config/src/yaml/conf/marathon/stateful.yaml | 1 + heron/config/src/yaml/conf/mesos/stateful.yaml | 1 + heron/config/src/yaml/conf/nomad/stateful.yaml | 1 + heron/config/src/yaml/conf/sandbox/stateful.yaml | 1 + heron/config/src/yaml/conf/slurm/stateful.yaml | 1 + .../config/src/yaml/conf/standalone/stateful.yaml | 1 + heron/config/src/yaml/conf/yarn/stateful.yaml | 1 + .../localfs/LocalFileSystemStorage.java| 38 -- .../localfs/LocalFileSystemStorageTest.java| 23 + 12 files changed, 68 insertions(+), 3 deletions(-) diff --git a/heron/config/src/yaml/conf/kubernetes/stateful.yaml b/heron/config/src/yaml/conf/kubernetes/stateful.yaml index 417b882..a00d9f6 100644 --- a/heron/config/src/yaml/conf/kubernetes/stateful.yaml +++ b/heron/config/src/yaml/conf/kubernetes/stateful.yaml @@ -21,6 +21,7 @@ heron.statefulstorage.classname: "org.apache.heron.statefulstorage.localfs.Local heron.statefulstorage.config: heron.statefulstorage.classpath: "" heron.statefulstorage.localfs.root.path: "./checkpoints" + heron.statefulstorage.localfs.max.checkpoints: 10 # Following are configs for socket between ckptmgr and stateful storage heron.ckptmgr.network.write.batch.size.bytes: 32768 diff --git a/heron/config/src/yaml/conf/local/stateful.yaml b/heron/config/src/yaml/conf/local/stateful.yaml index d72e496..95083dd 100644 --- a/heron/config/src/yaml/conf/local/stateful.yaml +++ b/heron/config/src/yaml/conf/local/stateful.yaml @@ -21,6 +21,7 @@ heron.statefulstorage.classname: "org.apache.heron.statefulstorage.lo heron.statefulstorage.config: heron.statefulstorage.classpath: "" heron.statefulstorage.localfs.root.path: ~/.herondata/checkpoints + heron.statefulstorage.localfs.max.checkpoints: 10 # Following are configs for socket between ckptmgr and stateful storage heron.ckptmgr.network.write.batch.size.bytes: 32768 diff --git a/heron/config/src/yaml/conf/localzk/stateful.yaml b/heron/config/src/yaml/conf/localzk/stateful.yaml index d72e496..95083dd 100644 --- a/heron/config/src/yaml/conf/localzk/stateful.yaml +++ b/heron/config/src/yaml/conf/localzk/stateful.yaml @@ -21,6 +21,7 @@ heron.statefulstorage.classname: "org.apache.heron.statefulstorage.lo heron.statefulstorage.config: heron.statefulstorage.classpath: "" heron.statefulstorage.localfs.root.path: ~/.herondata/checkpoints + heron.statefulstorage.localfs.max.checkpoints: 10 # Following are configs for socket between ckptmgr and stateful storage heron.ckptmgr.network.write.batch.size.bytes: 32768 diff --git a/heron/config/src/yaml/conf/marathon/stateful.yaml b/heron/config/src/yaml/conf/marathon/stateful.yaml index 4ea3bfb..ce8785b 100644 --- a/heron/config/src/yaml/conf/marathon/stateful.yaml +++ b/heron/config/src/yaml/conf/marathon/stateful.yaml @@ -21,6 +21,7 @@ heron.statefulstorage.classname: "org.apache.heron.statefulstorage.lo heron.statefulstorage.config: heron.statefulstorage.classpath: "" heron.statefulstorage.localfs.root.path: "./checkpoints" + heron.statefulstorage.localfs.max.checkpoints: 10 # Following are configs for socket between ckptmgr and stateful storage heron.ckptmgr.network.write.batch.size.bytes: 32768 diff --git a/heron/config/src/yaml/conf/mesos/stateful.yaml b/heron/config/src/yaml/conf/mesos/stateful.yaml index 4ea3bfb..ce8785b 100644 --- a/heron/config/src/yaml/conf/mesos/stateful.yaml +++ b/heron/config/src/yaml/conf/mesos/stateful.yaml @@ -21,6 +21,7 @@ heron.statefulstorage.classname: "org.apache.heron.statefulstorage.lo heron.statefulstorage.config: heron.statefulstorage.classpath: "" heron.statefulstorage.localfs.root.path: "./checkpoints" + heron.statefulstorage.localfs.max.checkpoints: 10 # Following are configs for socket between ckptmgr and stateful storage heron.ckptmgr.network.write.batch.size.bytes: 32768 diff --git a/heron/config/src/yaml/conf/nomad/stateful.yaml b/heron/conf
[incubator-heron] branch master updated: add .htaccess and index.html [https://github.com/apache/incubator-heron/issues/2758] (#2872)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-heron.git The following commit(s) were added to refs/heads/master by this push: new 1202902 add .htaccess and index.html [https://github.com/apache/incubator-heron/issues/2758] (#2872) 1202902 is described below commit 1202902f658090197cd9a6055b1aece9ab526af6 Author: SreeV <441385+sr...@users.noreply.github.com> AuthorDate: Fri May 18 11:01:21 2018 -0700 add .htaccess and index.html [https://github.com/apache/incubator-heron/issues/2758] (#2872) -- To stop receiving notification emails like this one, please contact nl...@apache.org.
[incubator-heron] branch master updated: cleaning removed directory (#2903)
This is an automated email from the ASF dual-hosted git repository.
nlu90 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
The following commit(s) were added to refs/heads/master by this push:
new feea4af cleaning removed directory (#2903)
feea4af is described below
commit feea4af5fa1e5a95ebba6b08df4a847a1b24b0fc
Author: Neng Lu <freen...@gmail.com>
AuthorDate: Thu May 17 14:12:44 2018 -0700
cleaning removed directory (#2903)
---
scripts/get_all_heron_paths.sh | 5 ++---
1 file changed, 2 insertions(+), 3 deletions(-)
diff --git a/scripts/get_all_heron_paths.sh b/scripts/get_all_heron_paths.sh
index 8df9b52..c88921b 100755
--- a/scripts/get_all_heron_paths.sh
+++ b/scripts/get_all_heron_paths.sh
@@ -67,7 +67,7 @@ function get_package_of() {
}
function get_heron_java_paths() {
- local java_paths=$(find
{heron,heron/tools,tools,integration_test,storm-compatibility,contrib,eco,eco-storm-examples,eco-heron-examples}
-name "*.java" | sed "s|/src/java/.*$|/src/java|"| sed
"s|/java/src/.*$|/java/src|" | sed "s|/tests/java/.*$|/tests/java|" | sort -u
| fgrep -v "heron/scheduler/" | fgrep -v "heron/scheduler/" )
+ local java_paths=$(find
{heron,heron/tools,tools,integration_test,storm-compatibility,eco,eco-storm-examples,eco-heron-examples}
-name "*.java" | sed "s|/src/java/.*$|/src/java|"| sed
"s|/java/src/.*$|/java/src|" | sed "s|/tests/java/.*$|/tests/java|" | sort -u
| fgrep -v "heron/scheduler/" | fgrep -v "heron/scheduler/" )
if [ "$(uname -s | tr 'A-Z' 'a-z')" != "darwin" ]; then
java_paths=$(echo "${java_paths}" | fgrep -v "/objc_tools/")
fi
@@ -105,8 +105,7 @@ function get_containing_library() {
function collect_generated_binary_deps() {
local proto_deps=$(find bazel-bin/heron/proto -type f | grep "jar$");
- local thrift_deps=$(find bazel-bin/heron/metricsmgr/src/thrift -type f |
grep "jar$");
- echo "${proto_deps} ${thrift_deps}" | sort | uniq
+ echo "${proto_deps}" | sort | uniq
}
function collect_generated_paths() {
--
To stop receiving notification emails like this one, please contact
nl...@apache.org.
[incubator-heron] branch master updated: Refactor CheckpointManager to move initialization code out of constructor (#2901)
This is an automated email from the ASF dual-hosted git repository.
nlu90 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
The following commit(s) were added to refs/heads/master by this push:
new 6ef20ad Refactor CheckpointManager to move initialization code out of
constructor (#2901)
6ef20ad is described below
commit 6ef20ad34899b94d36d73231f4c0a3a73098c7c8
Author: Ning Wang <nw...@twitter.com>
AuthorDate: Thu May 17 13:53:39 2018 -0700
Refactor CheckpointManager to move initialization code out of constructor
(#2901)
---
.../apache/heron/ckptmgr/CheckpointManager.java| 64 ++
1 file changed, 40 insertions(+), 24 deletions(-)
diff --git
a/heron/ckptmgr/src/java/org/apache/heron/ckptmgr/CheckpointManager.java
b/heron/ckptmgr/src/java/org/apache/heron/ckptmgr/CheckpointManager.java
index fa9c435..5c590ff 100644
--- a/heron/ckptmgr/src/java/org/apache/heron/ckptmgr/CheckpointManager.java
+++ b/heron/ckptmgr/src/java/org/apache/heron/ckptmgr/CheckpointManager.java
@@ -50,8 +50,8 @@ public class CheckpointManager {
private static final String CHECKPOINT_MANAGER_HOST = "127.0.0.1";
// The looper drives CheckpointManagerServer
- private final NIOLooper checkpointManagerServerLoop;
- private final CheckpointManagerServer checkpointManagerServer;
+ private NIOLooper checkpointManagerServerLoop;
+ private CheckpointManagerServer checkpointManagerServer;
// Print usage options
private static void usage(Options options) {
@@ -133,14 +133,21 @@ public class CheckpointManager {
return options;
}
- public CheckpointManager(
- String topologyName, String topologyId, String checkpointMgrId,
- String serverHost, int serverPort,
- SystemConfig systemConfig, CheckpointManagerConfig
checkpointManagerConfig)
- throws IOException, CheckpointManagerException {
+ public CheckpointManager() {
+ }
-this.checkpointManagerServerLoop = new NIOLooper();
+ public void init(
+ String topologyName,
+ String topologyId,
+ String checkpointMgrId,
+ String serverHost,
+ int serverPort,
+ SystemConfig systemConfig,
+ CheckpointManagerConfig checkpointManagerConfig)
+ throws IOException, CheckpointManagerException {
+LOG.info("Initializing CheckpointManager");
+checkpointManagerServerLoop = new NIOLooper();
HeronSocketOptions serverSocketOptions =
new HeronSocketOptions(
checkpointManagerConfig.getWriteBatchSize(),
@@ -152,9 +159,29 @@ public class CheckpointManager {
checkpointManagerConfig.getMaximumPacketSize());
// Setup the IStatefulStorage
-// TODO(mfu): This should be done in an executor driven by another thread,
kind of async
+IStatefulStorage statefulStorage = setupStatefulStorage(topologyName,
checkpointManagerConfig);
+
+// Start the server
+this.checkpointManagerServer = new CheckpointManagerServer(
+topologyName, topologyId, checkpointMgrId, statefulStorage,
+checkpointManagerServerLoop, serverHost, serverPort,
serverSocketOptions);
+ }
+
+ public void startAndLoop() {
+// The CheckpointManagerServer would run in the main thread
+// We do it in the final step since it would await the main thread
+LOG.info("Starting CheckpointManager Server");
+checkpointManagerServer.start();
+checkpointManagerServerLoop.loop();
+ }
+
+ private static IStatefulStorage setupStatefulStorage(
+ String topologyName,
+ CheckpointManagerConfig checkpointManagerConfig) throws
CheckpointManagerException {
+
IStatefulStorage statefulStorage;
String classname = checkpointManagerConfig.getStorageClassname();
+
try {
statefulStorage = (IStatefulStorage)
Class.forName(classname).newInstance();
} catch (InstantiationException e) {
@@ -172,18 +199,7 @@ public class CheckpointManager {
throw new CheckpointManagerException(classname + " init threw
exception", e);
}
-// Start the server
-this.checkpointManagerServer = new CheckpointManagerServer(
-topologyName, topologyId, checkpointMgrId, statefulStorage,
-checkpointManagerServerLoop, serverHost, serverPort,
serverSocketOptions);
- }
-
- public void startAndLoop() {
-// The CheckpointManagerServer would run in the main thread
-// We do it in the final step since it would await the main thread
-LOG.info("Starting CheckpointManager Server");
-checkpointManagerServer.start();
-checkpointManagerServerLoop.loop();
+return statefulStorage;
}
public static void main(String[] args) throws IOException,
@@ -241,9 +257,9 @@ public class CheckpointManager {
LOG.info("System Config: " + systemConfig);
-CheckpointManager checkpointManager =
-new CheckpointManager(topologyName, topologyId, ckptmgrId,
-
[incubator-heron] branch master updated: Add extra info when failed to create new instance of class (#2870)
This is an automated email from the ASF dual-hosted git repository. nlu90 pushed a commit to branch master in repository https://gitbox.apache.org/repos/asf/incubator-heron.git The following commit(s) were added to refs/heads/master by this push: new c54f804 Add extra info when failed to create new instance of class (#2870) c54f804 is described below commit c54f804d6b8abb62f847183733b26805c4657fcf Author: Ning Wang <nw...@twitter.com> AuthorDate: Thu May 17 14:17:00 2018 -0400 Add extra info when failed to create new instance of class (#2870) -- To stop receiving notification emails like this one, please contact nl...@apache.org.
[incubator-heron] branch master updated: submit topology file by uri (#2886)
This is an automated email from the ASF dual-hosted git repository.
nlu90 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
The following commit(s) were added to refs/heads/master by this push:
new 9150216 submit topology file by uri (#2886)
9150216 is described below
commit 915021630404d9a05d4ae27aa8e48b9a0b071029
Author: bed debug <huiju...@users.noreply.github.com>
AuthorDate: Fri May 11 11:28:29 2018 -0700
submit topology file by uri (#2886)
* move downloader registry map to yaml
* fix neng comment
* staging
* fix path
* fix type
* refactor #2274
* revert comment
* fix downloader command
---
heron/tools/cli/src/python/submit.py | 23 +++
1 file changed, 23 insertions(+)
diff --git a/heron/tools/cli/src/python/submit.py
b/heron/tools/cli/src/python/submit.py
index c70b3fe..ffb0ae4 100644
--- a/heron/tools/cli/src/python/submit.py
+++ b/heron/tools/cli/src/python/submit.py
@@ -24,6 +24,8 @@ import logging
import os
import tempfile
import requests
+import subprocess
+import urlparse
from heron.common.src.python.utils.log import Log
from heron.proto import topology_pb2
@@ -370,6 +372,22 @@ def submit_cpp(cl_args, unknown_args, tmp_dir):
return launch_topologies(cl_args, topology_file, tmp_dir)
+def download(uri, cluster):
+ tmp_dir = tempfile.mkdtemp()
+ cmd_downloader = config.get_heron_bin_dir() + "/heron-downloader.sh"
+ cmd_uri = "-u " + uri
+ cmd_destination = "-f " + tmp_dir
+ cmd_heron_root = "-d " + config.get_heron_dir()
+ cmd_heron_config = "-p " + config.get_heron_cluster_conf_dir(cluster,
config.get_heron_conf_dir())
+ cmd_mode = "-m local"
+ cmd = [cmd_downloader, cmd_uri, cmd_destination, cmd_heron_root,
cmd_heron_config, cmd_mode]
+ Log.debug("download uri command: %s", cmd)
+ subprocess.call(cmd)
+ suffix = (".jar", ".tar", ".tar.gz", ".pex", ".dylib", ".so")
+ for f in os.listdir(tmp_dir):
+if f.endswith(suffix):
+ return os.path.join(tmp_dir, f)
+
# pylint: disable=unused-argument
def run(command, parser, cl_args, unknown_args):
@@ -391,6 +409,11 @@ def run(command, parser, cl_args, unknown_args):
# get the topology file name
topology_file = cl_args['topology-file-name']
+ if urlparse.urlparse(topology_file).scheme:
+cl_args['topology-file-name'] = download(topology_file, cl_args['cluster'])
+topology_file = cl_args['topology-file-name']
+Log.debug("download uri to local file: %s", topology_file)
+
# check to see if the topology file exists
if not os.path.isfile(topology_file):
err_context = "Topology file '%s' does not exist" % topology_file
--
To stop receiving notification emails like this one, please contact
nl...@apache.org.
