This is an automated email from the ASF dual-hosted git repository. saadurrahman pushed a commit to branch saadurrahman/3829-Deprecate-Apache-Aurora-dev in repository https://gitbox.apache.org/repos/asf/incubator-heron.git
commit 825e818d92376de5ada58869ffd0c12771474ed6 Author: Saad Ur Rahman <[email protected]> AuthorDate: Mon May 9 17:21:59 2022 -0400 [Config] Removed Aurora from configs. --- heron/config/src/yaml/BUILD | 15 +- heron/config/src/yaml/conf/aurora/README | 1 - heron/config/src/yaml/conf/aurora/client.yaml | 23 -- heron/config/src/yaml/conf/aurora/downloader.yaml | 23 -- heron/config/src/yaml/conf/aurora/heron.aurora | 93 ------- .../src/yaml/conf/aurora/heron_internals.yaml | 296 --------------------- .../config/src/yaml/conf/aurora/metrics_sinks.yaml | 257 ------------------ heron/config/src/yaml/conf/aurora/packing.yaml | 20 -- heron/config/src/yaml/conf/aurora/scheduler.yaml | 61 ----- heron/config/src/yaml/conf/aurora/stateful.yaml | 38 --- heron/config/src/yaml/conf/aurora/statemgr.yaml | 49 ---- heron/config/src/yaml/conf/aurora/uploader.yaml | 22 -- heron/config/src/yaml/conf/examples/README.md | 3 - .../src/yaml/conf/examples/aurora_scheduler.yaml | 29 -- 14 files changed, 3 insertions(+), 927 deletions(-) diff --git a/heron/config/src/yaml/BUILD b/heron/config/src/yaml/BUILD index aba545eccdd..2cdcf392e35 100644 --- a/heron/config/src/yaml/BUILD +++ b/heron/config/src/yaml/BUILD @@ -15,13 +15,9 @@ filegroup( filegroup( name = "conf-yaml", srcs = glob( - ["conf/**/*.aurora"] + [ - "conf/**/*.hcl", - ] + [ - "conf/**/*.sh", - ] + [ - "conf/**/*.yaml", - ], + ["conf/**/*.hcl"] + + ["conf/**/*.sh"] + + ["conf/**/*.yaml"], ), ) @@ -35,11 +31,6 @@ filegroup( srcs = glob(["conf/sandbox/*.yaml"]), ) -filegroup( - name = "conf-aurora-yaml", - srcs = glob(["conf/aurora/*"]), -) - filegroup( name = "conf-slurm-yaml", srcs = glob(["conf/slurm/*"]), diff --git a/heron/config/src/yaml/conf/aurora/README b/heron/config/src/yaml/conf/aurora/README deleted file mode 100644 index 4a92300fe61..00000000000 --- a/heron/config/src/yaml/conf/aurora/README +++ /dev/null @@ -1 +0,0 @@ -Config customization is required before using. \ No newline at end of file diff --git a/heron/config/src/yaml/conf/aurora/client.yaml b/heron/config/src/yaml/conf/aurora/client.yaml deleted file mode 100644 index f14d8fdd968..00000000000 --- a/heron/config/src/yaml/conf/aurora/client.yaml +++ /dev/null @@ -1,23 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# location of the core package -heron.package.core.uri: "file:///vagrant/.herondata/dist/heron-core-release.tar.gz" - -# Whether role/env is required to submit a topology. Default value is False. -heron.config.is.role.required: True -heron.config.is.env.required: True diff --git a/heron/config/src/yaml/conf/aurora/downloader.yaml b/heron/config/src/yaml/conf/aurora/downloader.yaml deleted file mode 100644 index 68805525670..00000000000 --- a/heron/config/src/yaml/conf/aurora/downloader.yaml +++ /dev/null @@ -1,23 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# downloader class for protocols -heron.downloader.registry: - http: org.apache.heron.downloader.HttpDownloader - https: org.apache.heron.downloader.HttpDownloader - distributedlog: org.apache.heron.downloader.DLDownloader - file: org.apache.heron.downloader.FileDownloader diff --git a/heron/config/src/yaml/conf/aurora/heron.aurora b/heron/config/src/yaml/conf/aurora/heron.aurora deleted file mode 100644 index 1826e7faa61..00000000000 --- a/heron/config/src/yaml/conf/aurora/heron.aurora +++ /dev/null @@ -1,93 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -""" -Launch the topology as a single aurora job with multiple instances. -The heron-executor is responsible for starting a tmanager (container 0) -and regular stmgr/metricsmgr/instances (container index > 0). -""" - -heron_core_release_uri = '{{CORE_PACKAGE_URI}}' -heron_topology_jar_uri = '{{TOPOLOGY_PACKAGE_URI}}' -core_release_file = "heron-core.tar.gz" -topology_package_file = "topology.tar.gz" - -# --- processes --- -fetch_heron_system = Process( - name = 'fetch_heron_system', - cmdline = 'curl %s -o %s && tar zxf %s' % (heron_core_release_uri, core_release_file, core_release_file) -) - -fetch_user_package = Process( - name = 'fetch_user_package', - cmdline = 'curl %s -o %s && tar zxf %s' % (heron_topology_jar_uri, topology_package_file, topology_package_file) -) - -command_to_start_executor = \ - '{{EXECUTOR_BINARY}}' \ - ' --shard={{mesos.instance}}' \ - ' --server-port={{thermos.ports[port1]}}' \ - ' --tmanager-controller-port={{thermos.ports[port2]}}' \ - ' --tmanager-stats-port={{thermos.ports[port3]}}' \ - ' --shell-port={{thermos.ports[http]}}' \ - ' --metrics-manager-port={{thermos.ports[port4]}}' \ - ' --scheduler-port={{thermos.ports[scheduler]}}' \ - ' --metricscache-manager-server-port={{thermos.ports[metricscachemgr_serverport]}}' \ - ' --metricscache-manager-stats-port={{thermos.ports[metricscachemgr_statsport]}}' \ - ' --checkpoint-manager-port={{thermos.ports[ckptmgr_port]}}' \ - ' {{TOPOLOGY_ARGUMENTS}}' - -launch_heron_executor = Process( - name = 'launch_heron_executor', - cmdline = command_to_start_executor, - max_failures = 1 -) - -discover_profiler_port = Process( - name = 'discover_profiler_port', - cmdline = 'echo {{thermos.ports[yourkit]}} > yourkit.port' -) - -# --- tasks --- -heron_task = SequentialTask( - name = 'setup_and_run', - processes = [fetch_heron_system, fetch_user_package, launch_heron_executor, discover_profiler_port], - resources = Resources(cpu = '{{CPUS_PER_CONTAINER}}', ram = '{{RAM_PER_CONTAINER}}', disk = '{{DISK_PER_CONTAINER}}') -) - -# -- jobs --- -jobs = [ - Job( - name = '{{TOPOLOGY_NAME}}', - cluster = '{{CLUSTER}}', - role = '{{ROLE}}', - environment = '{{ENVIRON}}', - service = True, - task = heron_task, - instances = '{{NUM_CONTAINERS}}', - announce = Announcer(primary_port = 'http', portmap = {'admin': 'http', 'health': 'http'}), - health_check_config = HealthCheckConfig( - health_checker = HealthCheckerConfig( - http = HttpHealthChecker(endpoint='/health', expected_response='ok', expected_response_code=200) - ), - initial_interval_secs=30, - interval_secs=10, - max_consecutive_failures=2, - timeout_secs=5 - ) - ) -] diff --git a/heron/config/src/yaml/conf/aurora/heron_internals.yaml b/heron/config/src/yaml/conf/aurora/heron_internals.yaml deleted file mode 100644 index 22685e4e1a1..00000000000 --- a/heron/config/src/yaml/conf/aurora/heron_internals.yaml +++ /dev/null @@ -1,296 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -################################################################################ -# Default values for various configs used inside Heron. -################################################################################ -# All the config associated with time is in the unit of milli-seconds, -# unless otherwise specified. -################################################################################ -# All the config associated with data size is in the unit of bytes, unless -# otherwise specified. -################################################################################ - -################################################################################ -# System level configs -################################################################################ - -### heron.* configs are general configurations over all componenets - -# The relative path to the logging directory -heron.logging.directory: "log-files" - -# The maximum log file size in MB -heron.logging.maximum.size.mb: 100 - -# The maximum number of log files -heron.logging.maximum.files: 5 - -# The interval in seconds after which to check if the tmanager location has been fetched or not -heron.check.tmanager.location.interval.sec: 120 - -# The interval in seconds to prune logging files in C++ -heron.logging.prune.interval.sec: 300 - -# The interval in seconds to flush log files in C++ -heron.logging.flush.interval.sec: 10 - -# The threshold level to log error -heron.logging.err.threshold: 3 - -# The interval in seconds for different components to export metrics to metrics manager -heron.metrics.export.interval.sec: 60 - -# The maximum count of exceptions in one MetricPublisherPublishMessage protobuf -heron.metrics.max.exceptions.per.message.count: 1024 - -################################################################################ -# Configs related to Stream Manager, starts with heron.streammgr.* -################################################################################ - -# The tuple cache (used for batching) can be drained in two ways: -# (a) Time based -# (b) size based - -# The frequency in ms to drain the tuple cache in stream manager -heron.streammgr.cache.drain.frequency.ms: 10 - -# The sized based threshold in MB for buffering data tuples -# waiting for checkpoint markers before giving up -heron.streammgr.stateful.buffer.size.mb: 100 - -# The sized based threshold in MB for draining the tuple cache -heron.streammgr.cache.drain.size.mb: 100 - -# For efficient acknowledgements -heron.streammgr.xormgr.rotatingmap.nbuckets: 3 - -# The max number of messages in the memory pool for each message type -heron.streammgr.mempool.max.message.number: 512 - -# The reconnect interval to other stream managers in secs for stream manager client -heron.streammgr.client.reconnect.interval.sec: 1 - -# The reconnect interval to tamster in second for stream manager client -heron.streammgr.client.reconnect.tmanager.interval.sec: 10 - -# The max reconnect attempts to tmanager for stream manager client -heron.streammgr.client.reconnect.tmanager.max.attempts: 30 - -# The maximum packet size in MB of stream manager's network options -heron.streammgr.network.options.maximum.packet.mb: 10 - -# The interval in seconds to send heartbeat -heron.streammgr.tmanager.heartbeat.interval.sec: 10 - -# Maximum batch size in MB to read by stream manager from socket -heron.streammgr.connection.read.batch.size.mb: 1 - -# Maximum batch size in MB to write by stream manager to socket -heron.streammgr.connection.write.batch.size.mb: 1 - -# Number of times we should wait to see a buffer full while enqueueing data -# before declaring start of back pressure -heron.streammgr.network.backpressure.threshold: 3 - -# High water mark on the num in MB that can be left outstanding on a connection -heron.streammgr.network.backpressure.highwatermark.mb: 100 - -# Low water mark on the num in MB that can be left outstanding on a connection -heron.streammgr.network.backpressure.lowwatermark.mb: 50 - -################################################################################ -# Configs related to Topology Manager, starts with heron.tmanager.* -################################################################################ - -# The maximum interval in minutes of metrics to be kept in tmanager -heron.tmanager.metrics.collector.maximum.interval.min: 180 - -# The maximum time to retry to establish the tmanager -heron.tmanager.establish.retry.times: 30 - -# The interval to retry to establish the tmanager -heron.tmanager.establish.retry.interval.sec: 1 - -# Maximum packet size in MB of tmanager's network options to connect to stream managers -heron.tmanager.network.server.options.maximum.packet.mb: 16 - -# Maximum packet size in MB of tmanager's network options to connect to scheduler -heron.tmanager.network.controller.options.maximum.packet.mb: 1 - -# Maximum packet size in MB of tmanager's network options for stat queries -heron.tmanager.network.stats.options.maximum.packet.mb: 1 - -# The interval for tmanager to purge metrics from socket -heron.tmanager.metrics.collector.purge.interval.sec: 60 - -# The maximum # of exceptions to be stored in tmetrics collector, to prevent potential OOM -heron.tmanager.metrics.collector.maximum.exception: 256 - -# Should the metrics reporter bind on all interfaces -heron.tmanager.metrics.network.bindallinterfaces: False - -# The timeout in seconds for stream mgr, compared with (current time - last heartbeat time) -heron.tmanager.stmgr.state.timeout.sec: 60 - -################################################################################ -# Configs related to Metrics Manager, starts with heron.metricsmgr.* -################################################################################ - -# The size of packets to read from socket will be determined by the minimal of: -# (a) time based -# (b) size based - -# Time based, the maximum batch time in ms for metricsmgr to read from socket -heron.metricsmgr.network.read.batch.time.ms: 16 - -# Size based, the maximum batch size in bytes to read from socket -heron.metricsmgr.network.read.batch.size.bytes: 32768 - -# The size of packets to write to socket will be determined by the minimum of -# (a) time based -# (b) size based - -# Time based, the maximum batch time in ms for metricsmgr to write to socket -heron.metricsmgr.network.write.batch.time.ms: 16 - -# Size based, the maximum batch size in bytes to write to socket -heron.metricsmgr.network.write.batch.size.bytes: 32768 - -# The maximum socket's send buffer size in bytes -heron.metricsmgr.network.options.socket.send.buffer.size.bytes: 6553600 - -# The maximum socket's received buffer size in bytes of metricsmgr's network options -heron.metricsmgr.network.options.socket.received.buffer.size.bytes: 8738000 - -# The maximum packet size that metrics mgr can read -heron.metricsmgr.network.options.maximum.packetsize.bytes: 1048576 - -################################################################################ -# Configs related to Heron Instance, starts with heron.instance.* -################################################################################ - -# The maximum size of packets that instance can read -heron.instance.network.options.maximum.packetsize.bytes: 10485760 - -# The queue capacity (num of items) in bolt for buffer packets to read from stream manager -heron.instance.internal.bolt.read.queue.capacity: 128 - -# The queue capacity (num of items) in bolt for buffer packets to write to stream manager -heron.instance.internal.bolt.write.queue.capacity: 128 - -# The queue capacity (num of items) in spout for buffer packets to read from stream manager -heron.instance.internal.spout.read.queue.capacity: 1024 - -# The queue capacity (num of items) in spout for buffer packets to write to stream manager -heron.instance.internal.spout.write.queue.capacity: 128 - -# The queue capacity (num of items) for metrics packets to write to metrics manager -heron.instance.internal.metrics.write.queue.capacity: 128 - -# The size of packets read from stream manager will be determined by the minimal of -# (a) time based -# (b) size based - -# Time based, the maximum batch time in ms for instance to read from stream manager per attempt -heron.instance.network.read.batch.time.ms: 16 - -# Size based, the maximum batch size in bytes to read from stream manager -heron.instance.network.read.batch.size.bytes: 32768 - -# The size of packets written to stream manager will be determined by the minimum of -# (a) time based -# (b) size based - -# Time based, the maximum batch time in ms for instance to write to stream manager per attempt -heron.instance.network.write.batch.time.ms: 16 - -# Size based, the maximum batch size in bytes to write to stream manager -heron.instance.network.write.batch.size.bytes: 32768 - -# The maximum socket's send buffer size in bytes -heron.instance.network.options.socket.send.buffer.size.bytes: 6553600 - -# The maximum socket's received buffer size in bytes of instance's network options -heron.instance.network.options.socket.received.buffer.size.bytes: 8738000 - -# The maximum # of data tuple to batch in a HeronDataTupleSet protobuf -heron.instance.set.data.tuple.capacity: 1024 - -# The maximum size in bytes of data tuple to batch in a HeronDataTupleSet protobuf -heron.instance.set.data.tuple.size.bytes: 8388608 - -# The maximum # of control tuple to batch in a HeronControlTupleSet protobuf -heron.instance.set.control.tuple.capacity: 1024 - -# The maximum time in ms for a spout to do acknowledgement per attempt, the ack batch could -# also break if there are no more ack tuples to process -heron.instance.ack.batch.time.ms: 128 - -# The maximum time in ms for an spout instance to emit tuples per attempt -heron.instance.emit.batch.time.ms: 16 - -# The maximum batch size in bytes for an spout to emit tuples per attempt -heron.instance.emit.batch.size.bytes: 32768 - -# The maximum time in ms for an bolt instance to execute tuples per attempt -heron.instance.execute.batch.time.ms: 16 - -# The maximum batch size in bytes for an bolt instance to execute tuples per attempt -heron.instance.execute.batch.size.bytes: 32768 - -# The time to wait before the instance exits forcibly when uncaught exception happens -heron.instance.force.exit.timeout.ms: 2000 - -# Interval in seconds to reconnect to the stream manager, including the request timeout in connecting -heron.instance.reconnect.streammgr.interval.sec: 5 -heron.instance.reconnect.streammgr.times: 60 - -# Interval in seconds to reconnect to the metrics manager, including the request timeout in connecting -heron.instance.reconnect.metricsmgr.interval.sec: 5 -heron.instance.reconnect.metricsmgr.times: 60 - -# The interval in second for an instance to sample its system metrics, for instance, CPU load. -heron.instance.metrics.system.sample.interval.sec: 10 - -# For efficient acknowledgement -heron.instance.acknowledgement.nbuckets: 10 - -################################################################################ -# For dynamically tuning the available sizes in the interval read & write queues -# to provide high performance while avoiding GC issues -################################################################################ - -# The expected size on read queue in bolt -heron.instance.tuning.expected.bolt.read.queue.size: 8 - -# The expected size on write queue in bolt -heron.instance.tuning.expected.bolt.write.queue.size: 8 - -# The expected size on read queue in spout -heron.instance.tuning.expected.spout.read.queue.size: 512 - -# The exepected size on write queue in spout -heron.instance.tuning.expected.spout.write.queue.size: 8 - -# The expected size on metrics write queue -heron.instance.tuning.expected.metrics.write.queue.size: 8 - -heron.instance.tuning.current.sample.weight: 0.8 - -# Interval in ms to tune the size of in & out data queue in instance -heron.instance.tuning.interval.ms: 100 diff --git a/heron/config/src/yaml/conf/aurora/metrics_sinks.yaml b/heron/config/src/yaml/conf/aurora/metrics_sinks.yaml deleted file mode 100644 index 6cac2f13c9e..00000000000 --- a/heron/config/src/yaml/conf/aurora/metrics_sinks.yaml +++ /dev/null @@ -1,257 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -########### These all have default values as shown - -# We would specify the unique sink-id first -sinks: - - file-sink - - tmanager-sink - - metricscache-sink - -########### Now we would specify the detailed configuration for every unique sink -########### Syntax: sink-id: - option(s) - -########### option class is required as we need to instantiate a new instance by reflection -########### option flush-frequency-ms is required to invoke flush() at interval -########### option sink-restart-attempts, representsing # of times to restart a sink when it throws exceptions and dies. -########### If this option is missed, default value 0 would be supplied; negative value represents to restart it forever. - -########### Other options would be constructed as an immutable map passed to IMetricsSink's init(Map conf) as argument, -########### We would be able to fetch value by conf.get(options), for instance: -########### We could get "org.apache.heron.metricsmgr.sink.FileSink" if conf.get("class") is called inside file-sink's instance - -### Config for file-sink -file-sink: - class: "org.apache.heron.metricsmgr.sink.FileSink" - flush-frequency-ms: 60000 # 1 min - sink-restart-attempts: -1 # Forever - filename-output: "metrics.json" # File for metrics to write to - file-maximum: 5 # maximum number of file saved in disk - -### Config for tmanager-sink -tmanager-sink: - class: "org.apache.heron.metricsmgr.sink.tmanager.TManagerSink" - flush-frequency-ms: 60000 - sink-restart-attempts: -1 # Forever - tmanager-location-check-interval-sec: 5 - tmanager-client: - reconnect-interval-second: 5 # The re-connect interval to TManager from TManagerClient - # The size of packets written to TManager will be determined by the minimal of: (a) time based (b) size based - network-write-batch-size-bytes: 32768 # Size based, the maximum batch size in bytes to write to TManager - network-write-batch-time-ms: 16 # Time based, the maximum batch time in ms for Metrics Manager to write to TManager per attempt - network-read-batch-size-bytes: 32768 # Size based, the maximum batch size in bytes to write to TManager - network-read-batch-time-ms: 16 # Time based, the maximum batch time in ms for Metrics Manager to write to TManager per attempt - socket-send-buffer-size-bytes: 6553600 # The maximum socket's send buffer size in bytes - socket-received-buffer-size-bytes: 8738000 # The maximum socket's received buffer size in bytes - tmanager-metrics-type: - "__emit-count": SUM - "__execute-count": SUM - "__fail-count": SUM - "__ack-count": SUM - "__complete-latency": AVG - "__execute-latency": AVG - "__process-latency": AVG - "__jvm-uptime-secs": LAST - "__jvm-process-cpu-load": LAST - "__jvm-memory-used-mb": LAST - "__jvm-memory-mb-total": LAST - "__jvm-gc-collection-time-ms": LAST - "__server/__time_spent_back_pressure_initiated": SUM - "__time_spent_back_pressure_by_compid": SUM - -### Config for metricscache-sink -metricscache-sink: - class: "org.apache.heron.metricsmgr.sink.metricscache.MetricsCacheSink" - flush-frequency-ms: 60000 - sink-restart-attempts: -1 # Forever - metricscache-location-check-interval-sec: 5 - metricscache-client: - reconnect-interval-second: 5 # The re-connect interval to TManager from TManagerClient - # The size of packets written to TManager will be determined by the minimal of: (a) time based (b) size based - network-write-batch-size-bytes: 32768 # Size based, the maximum batch size in bytes to write to TManager - network-write-batch-time-ms: 16 # Time based, the maximum batch time in ms for Metrics Manager to write to TManager per attempt - network-read-batch-size-bytes: 32768 # Size based, the maximum batch size in bytes to write to TManager - network-read-batch-time-ms: 16 # Time based, the maximum batch time in ms for Metrics Manager to write to TManager per attempt - socket-send-buffer-size-bytes: 6553600 # The maximum socket's send buffer size in bytes - socket-received-buffer-size-bytes: 8738000 # The maximum socket's received buffer size in bytes - metricscache-metrics-type: - "__emit-count": SUM - "__execute-count": SUM - "__fail-count": SUM - "__ack-count": SUM - "__complete-latency": AVG - "__execute-latency": AVG - "__process-latency": AVG - "__jvm-uptime-secs": LAST - "__jvm-process-cpu-load": LAST - "__jvm-memory-used-mb": LAST - "__jvm-memory-mb-total": LAST - "__jvm-gc-collection-time-ms": LAST - "__server/__time_spent_back_pressure_initiated": SUM - "__time_spent_back_pressure_by_compid": SUM - -### Config for prometheus-sink -# prometheus-sink: -# class: "org.apache.heron.metricsmgr.sink.PrometheusSink" -# port: 8080 # The port on which to run (either port or port-file are mandatory) -# path: /metrics # The path on which to publish the metrics (mandatory) -# flat-metrics: true # By default the web-sink will publish a flat "name -> value" json map -# include-topology-name: true # Include topology name in metric name (default false) -# metrics-cache-max-size: 1000000 # Max number of metrics cached and published (default 1000000) -# metrics-cache-ttl-sec: 600 # Time in seconds after which a metric that was collected will stopped being published (default 600) -# rules: -# # __jvm-peak-usage/G1-Survivor-Space-committed": "9", -# - pattern: __jvm-(.+)/(.+) -# name: jvm_$1_$2 -# attrNameSnakeCase: true -# type: COUNTER -# # "__execute-time-ns/pulsar-prod-4/default": "418764", -# - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+)/(.+) -# name: $1 -# attrNameSnakeCase: true -# type: COUNTER -# labels: -# dest_component: "$2" -# context: "$3" -# # "__execute-time-ns/pulsar-prod-4": "418764", -# - pattern: __(?!jvm-+)(.+-count|.+-latency|.+-count|.+-time-ns)/(.+) -# name: $1 -# attrNameSnakeCase: true -# type: COUNTER -# labels: -# context: "$2" -# # StreamManager -# # "__client_stmgr-17/__bytes_to_stmgrs": "7841039", -# - pattern: __(client_stmgr-.+)/__(.+_to_stmgrs) -# name: $2 -# attrNameSnakeCase: true -# type: COUNTER -# labels: -# dest_component: "$1" -# # "__connection_buffer_by_instanceid/container_1_pulsar-prod-9_201/bytes": "0.000000", -# - pattern: __(connection_buffer_by_instanceid)/container_(.+)_(.+)/(.+) -# name: $1_$4 -# attrNameSnakeCase: true -# type: COUNTER -# labels: -# dest_container: "$2" -# dest_task: "$3" -# # "__time_spent_back_pressure_by_compid/container_1_pulsar-prod-5_151": "0", -# - pattern: __(time_spent_back_pressure_by_compid)/container_(.+)_(.+) -# name: $1 -# attrNameSnakeCase: true -# type: COUNTER -# labels: -# dest_container: "$2" -# dest_task: "$3" -# # PulsarSpoutMetrics of PulsarSpout 'PulsarSpoutMetrics/pulsar-prod-4-0/consumerThroughput' -# - pattern: PulsarSpout/(.+)/(.+) -# name: pulsar_spout_$2 -# attrNameSnakeCase: true -# type: COUNTER -# labels: -# component: "$1" -# - pattern: PulsarBolt/(.+)/(.+) -# name: pulsar_bolt_$2 -# attrNameSnakeCase: true -# type: COUNTER -# labels: -# component: "$1" -# # name: "kafkaConsumer-request-size-max/consumer-node-metrics/client-id-spout/node-id-node-1" -# - pattern: kafkaConsumer-(.+)/consumer-(node)-metrics/client-id-(.+)/node-id-(.+) -# name: kafka_consumer_$2_$1 -# attrNameSnakeCase: true -# type: COUNTER -# labels: -# client_id: "$3" -# node_id: "$4" -# # name: "kafkaConsumer-commit-rate/consumer-coordinator-metrics/client-id-spout" -# - pattern: kafkaConsumer-(.+)/consumer-(coordinator)-metrics/client-id-(.+) -# name: kafka_consumer_$2_$1 -# attrNameSnakeCase: true -# type: COUNTER -# labels: -# client_id: "$3" -# # name: "kafkaConsumer-records-lag-max/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-lad-es/partition-1" -# - pattern: kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+)/partition-(.+) -# name: kafka_consumer_$2_$1 -# attrNameSnakeCase: true -# type: COUNTER -# labels: -# client_id: "$3" -# topic: "$4" -# partition: "$5" -# # name: "kafkaConsumer-records-per-request-avg/consumer-fetch-manager-metrics/client-id-spout/topic-nginx-adp-cms-api" -# - pattern: kafkaConsumer-(.+)/consumer-(fetch-manager)-metrics/client-id-(.+)/topic-(.+) -# name: kafka_consumer_$2_$1 -# attrNameSnakeCase: true -# type: COUNTER -# labels: -# client_id: "$3" -# topic: "$4" -# # name: "kafkaConsumer-bytes-consumed-total/consumer-fetch-manager-metrics/client-id-consumer-1" -# - pattern: kafkaConsumer-(.+)/consumer-(feath-manager)-metrics/client-id-(.+) -# name: kafka_consumer_$2_$1 -# attrNameSnakeCase: true -# type: COUNTER -# labels: -# client_id: "$3" -# - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+)/node-id-(.+) -# name: kafka_consumer_$1 -# attrNameSnakeCase: true -# type: COUNTER -# labels: -# client_id: "$2" -# node_id: "$3" -# - pattern: kafkaConsumer-(.+)/consumer-metrics/client-id-(.+) -# name: kafka_consumer_$1 -# attrNameSnakeCase: true -# type: COUNTER -# labels: -# client_id: "$2" -# - pattern: kafkaConsumer-(.+)/app-info/client-id-(.+) -# name: kafka_consumer_$1 -# attrNameSnakeCase: true -# type: COUNTER -# labels: -# client_id: "$2" -# # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/partition_2/spoutLag' -# - pattern: kafkaOffset/(.+)/partition_([0-9]+)/(.+) -# name: kafka_offset_$3 -# attrNameSnakeCase: true -# type: COUNTER -# labels: -# topic: "$1" -# partition: "$2" -# # kafkaOffset of KafkaSpout 'kafkaOffset/topicName/totalSpoutLag' -# - pattern: kafkaOffset/(.+)/(.+) -# name: kafka_offset_$2 -# attrNameSnakeCase: true -# type: COUNTER -# labels: -# topic: "$1" - -### Config for graphite-sink -### Currently the graphite-sink is disabled -# graphite-sink: -# class: "org.apache.heron.metricsmgr.sink.GraphiteSink" -# flush-frequency-ms: 60000 -# graphite_host: "127.0.0.1" # The host of graphite to be exported metrics to -# graphite_port: 2004 # The port of graphite to be exported metrics to -# metrics_prefix: "heron" # The prefix of every metrics -# server_max_reconnect-attempts: 20 # The max reconnect attempts when failing to connect to graphite server diff --git a/heron/config/src/yaml/conf/aurora/packing.yaml b/heron/config/src/yaml/conf/aurora/packing.yaml deleted file mode 100644 index 4797a6a0218..00000000000 --- a/heron/config/src/yaml/conf/aurora/packing.yaml +++ /dev/null @@ -1,20 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# packing algorithm for packing instances into containers -heron.class.packing.algorithm: org.apache.heron.packing.roundrobin.RoundRobinPacking -heron.class.repacking.algorithm: org.apache.heron.packing.roundrobin.RoundRobinPacking diff --git a/heron/config/src/yaml/conf/aurora/scheduler.yaml b/heron/config/src/yaml/conf/aurora/scheduler.yaml deleted file mode 100644 index a3a7a5a53de..00000000000 --- a/heron/config/src/yaml/conf/aurora/scheduler.yaml +++ /dev/null @@ -1,61 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# scheduler class for distributing the topology for execution -heron.class.scheduler: org.apache.heron.scheduler.aurora.AuroraScheduler - -# launcher class for submitting and launching the topology -heron.class.launcher: org.apache.heron.scheduler.aurora.AuroraLauncher - -# location of java - pick it up from shell environment -heron.directory.sandbox.java.home: /usr/lib/jvm/java-1.8.0-openjdk-amd64/ - -# Invoke the IScheduler as a library directly -heron.scheduler.is.service: False - -#################################################################### -# Following are Aurora-specific -#################################################################### -# The maximum retry attempts when trying to kill an Aurora job -heron.scheduler.job.max.kill.attempts: 5 - -# The interval in ms between two retry-attempts to kill an Aurora job -heron.scheduler.job.kill.retry.interval.ms: 2000 - -# Aurora Controller Class -# heron.class.scheduler.aurora.controller.cli: False - -#################################################################### -# Following are config for tunneling -#################################################################### -# Whether we should attempt to tunnel if there is no direct access to a remote host (e.g. TManager) -heron.scheduler.is.tunnel.needed: False - -# The connection timeout in ms when testing if we can connect to remote host -heron.scheduler.tunnel.connection.timeout.ms: 1000 - -# The number of retries when testing direct access to remote host -heron.scheduler.tunnel.connection.retry.count: 2 - -# The interval in ms between two retries when testing direct access to remote host -heron.scheduler.tunnel.retry.interval.ms: 1000 - -# The number of retries when verifying connectivity via tunnel -heron.scheduler.tunnel.verify.count: 10 - -# SSH tunnel host -heron.scheduler.tunnel.host: "my.tunnel.host" diff --git a/heron/config/src/yaml/conf/aurora/stateful.yaml b/heron/config/src/yaml/conf/aurora/stateful.yaml deleted file mode 100644 index 76e1b0b542b..00000000000 --- a/heron/config/src/yaml/conf/aurora/stateful.yaml +++ /dev/null @@ -1,38 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# indicate the type of storage for checkpoint is HDFS -heron.statefulstorage.classname: "org.apache.heron.statefulstorage.hdfs.HDFSStorage" - -heron.statefulstorage.config: - heron.statefulstorage.classpath: "$(hadoop --config /path/to/configs classpath)" - heron.statefulstorage.hdfs.root.path: "/user/heron/checkpoints" - -# Following are configs for socket between ckptmgr and stateful storage -heron.ckptmgr.network.write.batch.size.bytes: 32768 - -heron.ckptmgr.network.write.batch.time.ms: 16 - -heron.ckptmgr.network.read.batch.size.bytes: 32768 - -heron.ckptmgr.network.read.batch.time.ms: 16 - -heron.ckptmgr.network.options.socket.send.buffer.size.bytes: 655360 - -heron.ckptmgr.network.options.socket.receive.buffer.size.bytes: 655360 - -heron.ckptmgr.network.options.maximum.packetsize.bytes: 10485760 diff --git a/heron/config/src/yaml/conf/aurora/statemgr.yaml b/heron/config/src/yaml/conf/aurora/statemgr.yaml deleted file mode 100644 index a2fe0feb761..00000000000 --- a/heron/config/src/yaml/conf/aurora/statemgr.yaml +++ /dev/null @@ -1,49 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# local state manager class for managing state in a persistent fashion -heron.class.state.manager: org.apache.heron.statemgr.localfs.LocalFileSystemStateManager - -# local state manager connection string -heron.statemgr.connection.string: LOCALMODE - -# path of the root address to store the state in a local file system -heron.statemgr.root.path: /vagrant/.herondata/repository/state/${CLUSTER} - -# create the sub directories, if needed -heron.statemgr.localfs.is.initialize.file.tree: True - -#################################################################### -# Following are config for tunneling -#################################################################### -# Whether we should attempt to tunnel if there is no direct access to a remote host (e.g. Zookeeper) -heron.statemgr.is.tunnel.needed: False - -# The connection timeout in ms when testing if we can connect to remote host -heron.statemgr.tunnel.connection.timeout.ms: 1000 - -# The number of retries when testing direct access to remote host -heron.statemgr.tunnel.connection.retry.count: 2 - -# The interval in ms between two retries when testing direct access to remote host -heron.statemgr.tunnel.retry.interval.ms: 1000 - -# The number of retries when verifying connectivity via tunnel -heron.statemgr.tunnel.verify.count: 10 - -# SSH tunnel host -heron.statemgr.tunnel.host: "my.tunnel.host" diff --git a/heron/config/src/yaml/conf/aurora/uploader.yaml b/heron/config/src/yaml/conf/aurora/uploader.yaml deleted file mode 100644 index 48a6ee48cd1..00000000000 --- a/heron/config/src/yaml/conf/aurora/uploader.yaml +++ /dev/null @@ -1,22 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# uploader class for transferring the topology jar/tar files to storage -heron.class.uploader: org.apache.heron.uploader.localfs.LocalFileSystemUploader - -# name of the directory to upload topologies for local file system uploader -heron.uploader.localfs.file.system.directory: /vagrant/.herondata/repository/topologies/${CLUSTER}/${ROLE}/${TOPOLOGY} \ No newline at end of file diff --git a/heron/config/src/yaml/conf/examples/README.md b/heron/config/src/yaml/conf/examples/README.md index 96c4e5d65d2..944ebb25ed3 100644 --- a/heron/config/src/yaml/conf/examples/README.md +++ b/heron/config/src/yaml/conf/examples/README.md @@ -21,9 +21,6 @@ for different components. The files are - * aurora_scheduler.yaml - - this contains the configuration for aurora scheduler - * heron_internals.yaml - this file contains several configuration elements that govern heron behavior diff --git a/heron/config/src/yaml/conf/examples/aurora_scheduler.yaml b/heron/config/src/yaml/conf/examples/aurora_scheduler.yaml deleted file mode 100644 index bef2cfca00a..00000000000 --- a/heron/config/src/yaml/conf/examples/aurora_scheduler.yaml +++ /dev/null @@ -1,29 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one -# or more contributor license agreements. See the NOTICE file -# distributed with this work for additional information -# regarding copyright ownership. The ASF licenses this file -# to you under the Apache License, Version 2.0 (the -# "License"); you may not use this file except in compliance -# with the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, -# software distributed under the License is distributed on an -# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -# KIND, either express or implied. See the License for the -# specific language governing permissions and limitations -# under the License. - -# scheduler class for distributing the topology for execution -heron.class.scheduler: org.apache.heron.scheduler.aurora.AuroraScheduler - -# launcher class for submitting and launching the topology -heron.class.launcher: org.apache.heron.scheduler.aurora.AuroraLauncher - -# location of java - pick it up from shell environment -heron.directory.sandbox.java.home: /usr/lib/jvm/java-1.8.0-openjdk-amd64/ - -# Invoke the IScheduler as a library directly -heron.scheduler.is.service: False -
