[incubator-heron] branch master updated: [Streamlet Scala API] Add Scala Streamlet Integration Tests Part I (#2826)

[karthikz]

This is an automated email from the ASF dual-hosted git repository.

karthikz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git


The following commit(s) were added to refs/heads/master by this push:
 new ad29cc3  [Streamlet Scala API] Add Scala Streamlet Integration Tests 
Part I (#2826)
ad29cc3 is described below

commit ad29cc36d6f5e3e55a72dd20c07e0ffad0c02a88
Author: Eren Avsarogullari 
AuthorDate: Sun Apr 8 00:07:09 2018 +0100

[Streamlet Scala API] Add Scala Streamlet Integration Tests Part I (#2826)

* Scala Streamlet Integration Tests Part I

* Minor change is applied to TextTransformer

* Fix Scala Streamlet Integration Test Packaging Problem

* Fix Scala Streamlet Integration Test Result Problem

* Add Fix for Travis

* Add Fix for Travis
---
 .gitignore |  3 +
 .../twitter/heron/streamlet/impl/BuilderImpl.java  |  6 +-
 .../streamlet/scala/SerializableTransformer.scala  |  2 -
 .../heron/streamlet/scala/impl/BuilderImpl.scala   |  3 +
 integration_test/src/python/test_runner/main.py|  8 ++-
 .../src/python/test_runner/resources/test.json |  7 +++
 integration_test/src/scala/BUILD   | 28 +
 .../common/ScalaIntegrationTestBase.scala  | 32 ++
 .../ScalaStreamletWithFilterAndTransform.scala | 71 ++
 ...calaStreamletWithFilterAndTransformResults.json |  1 +
 scripts/applatix/javatests.sh  | 19 +-
 scripts/applatix/test.sh   | 19 +-
 scripts/packages/BUILD | 11 
 scripts/run_integration_test.sh|  9 +++
 scripts/travis/test.sh | 19 +-
 15 files changed, 226 insertions(+), 12 deletions(-)

diff --git a/.gitignore b/.gitignore
index 927928b..12fbf25 100644
--- a/.gitignore
+++ b/.gitignore
@@ -132,3 +132,6 @@ website/public/
 
 # Visual Studio Code
 .vscode
+
+# integration_test
+results/
\ No newline at end of file
diff --git 
a/heron/api/src/java/com/twitter/heron/streamlet/impl/BuilderImpl.java 
b/heron/api/src/java/com/twitter/heron/streamlet/impl/BuilderImpl.java
index e00eb14..b982453 100644
--- a/heron/api/src/java/com/twitter/heron/streamlet/impl/BuilderImpl.java
+++ b/heron/api/src/java/com/twitter/heron/streamlet/impl/BuilderImpl.java
@@ -11,10 +11,8 @@
 //  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 //  See the License for the specific language governing permissions and
 //  limitations under the License.
-
 package com.twitter.heron.streamlet.impl;
 
-
 import java.util.HashSet;
 import java.util.LinkedList;
 import java.util.List;
@@ -60,6 +58,10 @@ public final class BuilderImpl implements Builder {
*/
   public TopologyBuilder build() {
 TopologyBuilder builder = new TopologyBuilder();
+return build(builder);
+  }
+
+  public TopologyBuilder build(TopologyBuilder builder) {
 Set stageNames = new HashSet<>();
 for (StreamletImpl streamlet : sources) {
   streamlet.build(builder, stageNames);
diff --git 
a/heron/api/src/scala/com/twitter/heron/streamlet/scala/SerializableTransformer.scala
 
b/heron/api/src/scala/com/twitter/heron/streamlet/scala/SerializableTransformer.scala
index 7867448..7bbbdd2 100644
--- 
a/heron/api/src/scala/com/twitter/heron/streamlet/scala/SerializableTransformer.scala
+++ 
b/heron/api/src/scala/com/twitter/heron/streamlet/scala/SerializableTransformer.scala
@@ -13,8 +13,6 @@
 //  limitations under the License.
 package com.twitter.heron.streamlet.scala
 
-import java.io.Serializable
-
 import com.twitter.heron.streamlet.Context
 
 /**
diff --git 
a/heron/api/src/scala/com/twitter/heron/streamlet/scala/impl/BuilderImpl.scala 
b/heron/api/src/scala/com/twitter/heron/streamlet/scala/impl/BuilderImpl.scala
index d6cd4c8..4458d70 100644
--- 
a/heron/api/src/scala/com/twitter/heron/streamlet/scala/impl/BuilderImpl.scala
+++ 
b/heron/api/src/scala/com/twitter/heron/streamlet/scala/impl/BuilderImpl.scala
@@ -37,4 +37,7 @@ class BuilderImpl(builder: 
com.twitter.heron.streamlet.Builder)
   def build(): TopologyBuilder =
 builder.asInstanceOf[JavaBuilderImpl].build()
 
+  def build(topologyBuilder: TopologyBuilder): TopologyBuilder =
+builder.asInstanceOf[JavaBuilderImpl].build(topologyBuilder)
+
 }
diff --git a/integration_test/src/python/test_runner/main.py 
b/integration_test/src/python/test_runner/main.py
index 861a951..f5d4d1c 100644
--- a/integration_test/src/python/test_runner/main.py
+++ b/integration_test/src/python/test_runner/main.py
@@ -293,11 +293,15 @@ def run_tests(conf, args):
 
   http_server_host_port = "%s:%d" % (args.http_server_hostname, 
args.http_server_port)
 
-  if args.tests_bin_path.endswith(".jar"):
+  if args.tests_bin_path.endswith("scala-integration-tests.jar"):
+test_topologies = 

[incubator-heron] branch master updated: Add sha256 checksums for all http_archives (#2856)

[karthikz]

This is an automated email from the ASF dual-hosted git repository.

karthikz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git


The following commit(s) were added to refs/heads/master by this push:
 new 56bcc6d  Add sha256 checksums for all http_archives (#2856)
56bcc6d is described below

commit 56bcc6d4139a8d64fd6c84d40b0785b1f8c177cb
Author: Andrew Jorgensen 
AuthorDate: Sat Apr 7 12:03:04 2018 -0400

Add sha256 checksums for all http_archives (#2856)

I have taken the current sha256 checksum for all of the workspace
archives so we can verify their signature on download. Since some
of the artifacts are downloaded over http, we want to make sure
there was no man in the middle attack done to change the resulting
binary. This also  gives assurance that the code we are downloading
has not been tampered with in any way either over the wire or at
the source.
---
 WORKSPACE | 23 ---
 1 file changed, 20 insertions(+), 3 deletions(-)

diff --git a/WORKSPACE b/WORKSPACE
index 0e87770..c0c40bb 100644
--- a/WORKSPACE
+++ b/WORKSPACE
@@ -792,7 +792,8 @@ new_http_archive(
 "data = glob(['**/*']),",
 "visibility = ['//visibility:public'],",
 ")",
-])
+]),
+sha256 = 
"02f8102c2436bb03b3ee6dede1919d1dac8a427541652e5ec95171ec8adbc93a",
 )
 # end pex repos
 
@@ -801,6 +802,7 @@ http_archive(
 name = "com_google_protobuf",
 urls = ["https://github.com/google/protobuf/archive/v3.4.1.tar.gz;],
 strip_prefix = "protobuf-3.4.1",
+sha256 = 
"8e0236242106e680b4f9f576cc44b8cd711e948b20a9fc07769b0a20ceab9cc4",
 )
 # end protobuf dependencies for C++ and Java
 
@@ -809,6 +811,7 @@ http_archive(
 name = "com_github_gflags_gflags",
 urls = ["https://github.com/gflags/gflags/archive/v2.2.1.tar.gz;],
 strip_prefix = "gflags-2.2.1",
+sha256 = 
"ae27cdbcd6a2f935baa78e4f21f675649271634c092b1be01469440495609d0e",
 )
 
 new_http_archive(
@@ -816,6 +819,7 @@ new_http_archive(
 urls = 
["https://github.com/libevent/libevent/releases/download/release-2.1.8-stable/libevent-2.1.8-stable.tar.gz;],
 strip_prefix = "libevent-2.1.8-stable",
 build_file = "third_party/libevent/libevent.BUILD",
+sha256 = 
"965cc5a8bb46ce4199a47e9b2c9e1cae3b137e8356ffdad6d94d3b9069b71dc2",
 )
 
 new_http_archive(
@@ -823,6 +827,7 @@ new_http_archive(
 urls = 
["http://download.savannah.nongnu.org/releases/libunwind/libunwind-1.1.tar.gz;],
 strip_prefix = "libunwind-1.1",
 build_file = "third_party/libunwind/libunwind.BUILD",
+sha256 = 
"9dfe0fcae2a866de9d3942c66995e4b460230446887dbdab302d41a8aee8d09a",
 )
 
 new_http_archive(
@@ -834,6 +839,7 @@ new_http_archive(
 ],
 strip_prefix = "zookeeper-3.4.10",
 build_file = "third_party/zookeeper/zookeeper.BUILD",
+sha256 = 
"7f7f5414e044ac11fee2a1e0bc225469f51fb0cdf821e67df762a43098223f27",
 )
 
 new_http_archive(
@@ -841,6 +847,7 @@ new_http_archive(
 urls = 
["https://github.com/gperftools/gperftools/releases/download/gperftools-2.4/gperftools-2.4.tar.gz;],
 strip_prefix = "gperftools-2.4",
 build_file = "third_party/gperftools/gperftools.BUILD",
+sha256 = 
"982a37226eb42f40714e26b8076815d5ea677a422fb52ff8bfca3704d9c30a2d",
 )
 
 new_http_archive(
@@ -848,6 +855,7 @@ new_http_archive(
 urls = ["https://github.com/google/glog/archive/v0.3.5.tar.gz;],
 strip_prefix = "glog-0.3.5",
 build_file = "third_party/glog/glog.BUILD",
+sha256 = 
"7580e408a2c0b5a89ca214739978ce6ff480b5e7d8d7698a2aa92fadc484d1e0",
 )
 
 new_http_archive(
@@ -855,6 +863,7 @@ new_http_archive(
 urls = 
["https://github.com/google/googletest/archive/release-1.8.0.tar.gz;],
 strip_prefix = "googletest-release-1.8.0",
 build_file = "third_party/gtest/gtest.BUILD",
+sha256 = 
"58a6f4277ca2bc8565222b3bbd58a177609e9c488e8a72649359ba51450db7d8",
 )
 
 new_http_archive(
@@ -862,6 +871,7 @@ new_http_archive(
 urls = ["https://github.com/USCiLab/cereal/archive/v1.2.2.tar.gz;],
 strip_prefix = "cereal-1.2.2",
 build_file = "third_party/cereal/cereal.BUILD",
+sha256 = 
"1921f26d2e1daf9132da3c432e2fd02093ecaedf846e65d7679ddf868c7289c4",
 )
 
 new_http_archive(
@@ -869,6 +879,7 @@ new_http_archive(
 urls = 
["https://storage.googleapis.com/heron-packages/yaml-cpp-noboost.tar.gz;],
 strip_prefix = "yaml-cpp-noboost",
 build_file = "third_party/yaml-cpp/yaml.BUILD",
+sha256 = 
"87684ea7a640733e2fe2f753348e306a6764c88fe3f30518c3273e367ff3d17b",
 )
 # end 3rdparty C++ dependencies
 
@@ -878,6 +889,7 @@ new_http_archive(
 url = 
"https://storage.googleapis.com/kubernetes-helm/helm-v2.7.2-darwin-amd64.tar.gz;,
 strip_prefix = "darwin-amd64",
 build_file = "third_party/helm/helm.BUILD",
+sha256 = 
"5058142bcd6e16b7e01695a8f258d27ae0b6469caf227ddf6aa2181405e6aa8e",
 )
 
 new_http_archive(
@@ -885,6 +897,7 @@ 

[incubator-heron] branch master updated: expose heron metrics as a service in Nomad (#2859)

[karthikz]

This is an automated email from the ASF dual-hosted git repository.

karthikz pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-heron.git


The following commit(s) were added to refs/heads/master by this push:
 new 2a4c1c1  expose heron metrics as a service in Nomad (#2859)
2a4c1c1 is described below

commit 2a4c1c1613bc2d10851503fcc35c571618a28ec1
Author: Boyang Jerry Peng 
AuthorDate: Sat Apr 7 08:58:54 2018 -0700

expose heron metrics as a service in Nomad (#2859)
---
 heron/config/src/yaml/conf/nomad/heron_nomad.sh|   3 +
 .../config/src/yaml/conf/nomad/metrics_sinks.yaml  |   3 +-
 heron/config/src/yaml/conf/nomad/scheduler.yaml|  17 
 .../config/src/yaml/conf/standalone/heron_nomad.sh |   3 +
 .../src/yaml/conf/standalone/metrics_sinks.yaml|   3 +-
 .../config/src/yaml/conf/standalone/scheduler.yaml |  14 +++
 .../standalone/templates/scheduler.template.yaml   |  14 +++
 .../heron/metricsmgr/sink/AbstractWebSink.java |   7 +-
 heron/schedulers/src/java/BUILD|   1 +
 .../heron/scheduler/nomad/NomadConstants.java  |   5 +
 .../heron/scheduler/nomad/NomadContext.java|  40 +++-
 .../heron/scheduler/nomad/NomadScheduler.java  |  95 -
 .../heron/scheduler/nomad/NomadSchedulerTest.java  | 113 -
 13 files changed, 305 insertions(+), 13 deletions(-)

diff --git a/heron/config/src/yaml/conf/nomad/heron_nomad.sh 
b/heron/config/src/yaml/conf/nomad/heron_nomad.sh
index 51ca593..826ba0d 100644
--- a/heron/config/src/yaml/conf/nomad/heron_nomad.sh
+++ b/heron/config/src/yaml/conf/nomad/heron_nomad.sh
@@ -25,6 +25,9 @@ fi
 # download and extract heron topology package
 ${HERON_TOPOLOGY_DOWNLOAD_CMD}
 
+# set metrics port file
+echo ${NOMAD_PORT_metrics_port} > ${METRICS_PORT_FILE}
+
 # launch heron executor
 trap 'kill -TERM $PID' TERM INT
 ${HERON_EXECUTOR_CMD} &
diff --git a/heron/config/src/yaml/conf/nomad/metrics_sinks.yaml 
b/heron/config/src/yaml/conf/nomad/metrics_sinks.yaml
index b5257e2..a70f43a 100644
--- a/heron/config/src/yaml/conf/nomad/metrics_sinks.yaml
+++ b/heron/config/src/yaml/conf/nomad/metrics_sinks.yaml
@@ -60,7 +60,8 @@ tmaster-sink:
 
 prometheus-sink:
class: "com.twitter.heron.metricsmgr.sink.PrometheusSink"
-   port: 8080 # The port on which to run (either port or port-file are 
mandatory)
+#   port: 8080 # The port on which to run (either port or port-file are 
mandatory)
+   port-file: "port_file"
path: /metrics # The path on which to publish the metrics (mandatory)
flat-metrics: true # By default the web-sink will publish a flat "name -> 
value" json map
include-topology-name: true # Include topology name in metric name (default 
false)
diff --git a/heron/config/src/yaml/conf/nomad/scheduler.yaml 
b/heron/config/src/yaml/conf/nomad/scheduler.yaml
index 297afc4..21d4268 100644
--- a/heron/config/src/yaml/conf/nomad/scheduler.yaml
+++ b/heron/config/src/yaml/conf/nomad/scheduler.yaml
@@ -26,3 +26,20 @@ heron.nomad.driver: "docker"
 
 # The docker image to use for heron if the docker driver is used,
 heron.executor.docker.image: 'heron/heron:latest'
+
+# Set networking mode networking when driver is docker
+heron.nomad.network.mode: "default"
+
+# whether to register metrics service endpoints for prometheus metrics sink in 
consul
+# the service will be named in the format: 
metrics-heron--
+heron.nomad.metrics.service.register: True
+
+# interval at which health checks should be conducted for metrics service 
endpoint
+heron.nomad.metrics.service.check.interval.sec: 10
+
+# timeout of metrics service endpoint health check
+heron.nomad.metrics.service.check.timeout.sec: 2
+
+# additional tags to be attached to metrics service
+# A tag of - with be automaticallu attached
+heron.nomad.metrics.service.additional.tags: "prometheus,metrics,heron"
\ No newline at end of file
diff --git a/heron/config/src/yaml/conf/standalone/heron_nomad.sh 
b/heron/config/src/yaml/conf/standalone/heron_nomad.sh
index 51ca593..826ba0d 100644
--- a/heron/config/src/yaml/conf/standalone/heron_nomad.sh
+++ b/heron/config/src/yaml/conf/standalone/heron_nomad.sh
@@ -25,6 +25,9 @@ fi
 # download and extract heron topology package
 ${HERON_TOPOLOGY_DOWNLOAD_CMD}
 
+# set metrics port file
+echo ${NOMAD_PORT_metrics_port} > ${METRICS_PORT_FILE}
+
 # launch heron executor
 trap 'kill -TERM $PID' TERM INT
 ${HERON_EXECUTOR_CMD} &
diff --git a/heron/config/src/yaml/conf/standalone/metrics_sinks.yaml 
b/heron/config/src/yaml/conf/standalone/metrics_sinks.yaml
index b5257e2..a70f43a 100644
--- a/heron/config/src/yaml/conf/standalone/metrics_sinks.yaml
+++ b/heron/config/src/yaml/conf/standalone/metrics_sinks.yaml
@@ -60,7 +60,8 @@ tmaster-sink:
 
 prometheus-sink:
class: "com.twitter.heron.metricsmgr.sink.PrometheusSink"
-   port: 8080 # The port on which to run (either port or port-file are 
mandatory)
+#   port: 8080 # The port on