cbickel closed pull request #3163: Configure jmxremote (updated) URL: https://github.com/apache/incubator-openwhisk/pull/3163
This is a PR merged from a forked repository. As GitHub hides the original diff on merge, it is displayed below for the sake of provenance: As this is a foreign pull request (from a fork), the diff is supplied below (as it won't show otherwise due to GitHub magic): diff --git a/ansible/environments/docker-machine/group_vars/all b/ansible/environments/docker-machine/group_vars/all index a09f335424..efd0b5628e 100644 --- a/ansible/environments/docker-machine/group_vars/all +++ b/ansible/environments/docker-machine/group_vars/all @@ -28,9 +28,6 @@ apigw_auth_user: "" apigw_auth_pwd: "" apigw_host_v2: "http://{{ groups['apigateway']|first }}:{{apigateway.port.api}}/v2" -controller_arguments: '-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=1098' -invoker_arguments: "{{ controller_arguments }}" - invoker_allow_multiple_instances: true # Set kafka configuration diff --git a/ansible/environments/local/group_vars/all b/ansible/environments/local/group_vars/all index 9a10b00ba3..0d8dc068aa 100755 --- a/ansible/environments/local/group_vars/all +++ b/ansible/environments/local/group_vars/all @@ -20,9 +20,6 @@ apigw_auth_user: "" apigw_auth_pwd: "" apigw_host_v2: "http://{{ groups['apigateway']|first }}:{{apigateway.port.api}}/v2" -controller_arguments: '-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.authenticate=false -Dcom.sun.management.jmxremote.port=1098' -invoker_arguments: "{{ controller_arguments }}" - invoker_allow_multiple_instances: true # Set kafka configuration diff --git a/ansible/group_vars/all b/ansible/group_vars/all index 2ffa75e1a7..5d65053013 100644 --- a/ansible/group_vars/all +++ b/ansible/group_vars/all @@ -43,8 +43,14 @@ limits: firesPerMinute: "{{ limit_fires_per_minute | default(60) }}" sequenceMaxLength: "{{ limit_sequence_max_length | default(50) }}" +controllerHostnameFromMap: "{{ groups['controllers'] | map('extract', hostvars, 'ansible_host') | list | first }}" +controllerHostname: "{{ controllerHostnameFromMap | default(inventory_hostname) }}" + # port means outer port controller: + dir: + become: "{{ controller_dir_become | default(false) }}" + confdir: "{{ config_root_dir }}/controller" basePort: 10001 heap: "{{ controller_heap | default('2g') }}" arguments: "{{ controller_arguments | default('') }}" @@ -62,6 +68,20 @@ controller: # We recommend to enable HA for the controllers only, if bookkeeping data are shared too. (localBookkeeping: false) ha: "{{ controller_enable_ha | default(True) and groups['controllers'] | length > 1 }}" loglevel: "{{ controller_loglevel | default(whisk_loglevel) | default('INFO') }}" + jmxremote: + jvmArgs: "{% if inventory_hostname in groups['controllers'] %} + {{ jmx.jvmCommonArgs }} -Djava.rmi.server.hostname={{ controllerHostname }} -Dcom.sun.management.jmxremote.rmi.port={{ jmx.rmiBasePortController + groups['controllers'].index(inventory_hostname) }} -Dcom.sun.management.jmxremote.port={{ jmx.basePortController + groups['controllers'].index(inventory_hostname) }} + {% endif %}" + +jmx: + basePortController: 15000 + rmiBasePortController: 16000 + basePortInvoker: 17000 + rmiBasePortInvoker: 18000 + user: "{{ jmxuser | default('jmxuser') }}" + pass: "{{ jmxuser | default('jmxpass') }}" + jvmCommonArgs: "-Dcom.sun.management.jmxremote -Dcom.sun.management.jmxremote.ssl=false -Dcom.sun.management.jmxremote.local.only=false -Dcom.sun.management.jmxremote.authenticate=true -Dcom.sun.management.jmxremote.password.file=/root/jmxremote.password -Dcom.sun.management.jmxremote.access.file=/root/jmxremote.access" + enabled: "{{ jmxremote_enabled | default('true') }}" registry: confdir: "{{ config_root_dir }}/registry" @@ -90,7 +110,13 @@ zookeeper_connect_string: "{% set ret = [] %}\ {% endfor %}\ {{ ret | join(',') }}" +invokerHostnameFromMap: "{{ groups['invokers'] | map('extract', hostvars, 'ansible_host') | list | first }}" +invokerHostname: "{{ invokerHostnameFromMap | default(inventory_hostname) }}" + invoker: + dir: + become: "{{ invoker_dir_become | default(false) }}" + confdir: "{{ config_root_dir }}/invoker" port: 12001 heap: "{{ invoker_heap | default('2g') }}" arguments: "{{ invoker_arguments | default('') }}" @@ -105,6 +131,10 @@ invoker: docker: become: "{{ invoker_docker_become | default(false) }}" loglevel: "{{ invoker_loglevel | default(whisk_loglevel) | default('INFO') }}" + jmxremote: + jvmArgs: "{% if inventory_hostname in groups['invokers'] %} + {{ jmx.jvmCommonArgs }} -Djava.rmi.server.hostname={{ invokerHostname }} -Dcom.sun.management.jmxremote.rmi.port={{ jmx.rmiBasePortInvoker + groups['invokers'].index(inventory_hostname) }} -Dcom.sun.management.jmxremote.port={{ jmx.basePortInvoker + groups['invokers'].index(inventory_hostname) }} + {% endif %}" userLogs: spi: "{{ userLogs_spi | default('whisk.core.containerpool.logging.DockerToActivationLogStoreProvider') }}" diff --git a/ansible/roles/controller/tasks/clean.yml b/ansible/roles/controller/tasks/clean.yml index e3bbe07840..231198a489 100644 --- a/ansible/roles/controller/tasks/clean.yml +++ b/ansible/roles/controller/tasks/clean.yml @@ -13,3 +13,9 @@ path: "{{ whisk_logs_dir }}/controller{{ groups['controllers'].index(inventory_hostname) }}" state: absent become: "{{ logs.dir.become }}" + +- name: remove controller conf directory + file: + path: "{{ controller.confdir }}/controller{{ groups['controllers'].index(inventory_hostname) }}" + state: absent + become: "{{ controller.dir.become }}" diff --git a/ansible/roles/controller/tasks/deploy.yml b/ansible/roles/controller/tasks/deploy.yml index ca1aa2a41d..2a212003ae 100644 --- a/ansible/roles/controller/tasks/deploy.yml +++ b/ansible/roles/controller/tasks/deploy.yml @@ -16,6 +16,27 @@ mode: 0777 become: "{{ logs.dir.become }}" +- name: ensure controller config directory is created with permissions + file: + path: "{{ controller.confdir }}/controller{{ groups['controllers'].index(inventory_hostname) }}" + state: directory + mode: 0777 + become: "{{ controller.dir.become }}" + +- name: copy jmxremote password file + when: jmx.enabled + template: + src: "jmxremote.password.j2" + dest: "{{ controller.confdir }}/controller{{ groups['controllers'].index(inventory_hostname) }}/jmxremote.password" + mode: 0777 + +- name: copy jmxremote access file + when: jmx.enabled + template: + src: "jmxremote.access.j2" + dest: "{{ controller.confdir }}/controller{{ groups['controllers'].index(inventory_hostname) }}/jmxremote.access" + mode: 0777 + - name: check, that required databases exist include: "{{ openwhisk_home }}/ansible/tasks/db/checkDb.yml" vars: @@ -25,6 +46,20 @@ - "{{ db.whisk.auth }}" - "{{ db.whisk.activations }}" +- name: prepare controller ports + set_fact: + ports_to_expose: ["{{ controller.basePort + groups['controllers'].index(inventory_hostname) }}:8080", "{{ controller.akka.cluster.basePort + groups['controllers'].index(inventory_hostname) }}:{{ controller.akka.cluster.bindPort }}"] + +- name: expose additional ports if jmxremote is enabled + when: jmx.enabled + set_fact: + ports_to_expose: "{{ ports_to_expose }} + [ \"{{ jmx.basePortController + groups['controllers'].index(inventory_hostname) }}:{{ jmx.basePortController + groups['controllers'].index(inventory_hostname) }}\" ] + [ \"{{ jmx.rmiBasePortController + groups['controllers'].index(inventory_hostname) }}:{{ jmx.rmiBasePortController + groups['controllers'].index(inventory_hostname) }}\" ]" + +- name: add additional jvm params if jmxremote is enabled + when: jmx.enabled + set_fact: + controller_args: "{{ controller.arguments }} {{ controller.jmxremote.jvmArgs }}" + - name: create seed nodes list set_fact: seed_nodes_list: "{{ seed_nodes_list | default([]) }} + [ \"{{item.1}}:{{controller.akka.cluster.basePort+item.0}}\" ]" @@ -41,8 +76,9 @@ hostname: "controller{{ groups['controllers'].index(inventory_hostname) }}" env: "JAVA_OPTS": "-Xmx{{ controller.heap }} -XX:+CrashOnOutOfMemoryError -XX:+UseGCOverheadLimit -XX:ErrorFile=/logs/java_error.log -XX:+HeapDumpOnOutOfMemoryError -XX:HeapDumpPath=/logs" - "CONTROLLER_OPTS": "{{ controller.arguments }}" + "CONTROLLER_OPTS": "{{ controller_args | default(controller.arguments) }}" "CONTROLLER_INSTANCES": "{{ controller.instances }}" + "JMX_REMOTE": "{{ jmx.enabled }}" "COMPONENT_NAME": "controller{{ groups['controllers'].index(inventory_hostname) }}" "PORT": 8080 @@ -106,9 +142,8 @@ "CONFIG_logback_log_level": "{{ controller.loglevel }}" volumes: - "{{ whisk_logs_dir }}/controller{{ groups['controllers'].index(inventory_hostname) }}:/logs" - ports: - - "{{ controller.basePort + groups['controllers'].index(inventory_hostname) }}:8080" - - "{{ controller.akka.cluster.basePort + groups['controllers'].index(inventory_hostname) }}:{{ controller.akka.cluster.bindPort }}" + - "{{ controller.confdir }}/controller{{ groups['controllers'].index(inventory_hostname) }}:/conf" + ports: "{{ ports_to_expose }}" command: /bin/sh -c "exec /init.sh {{ groups['controllers'].index(inventory_hostname) }} >> /logs/controller{{ groups['controllers'].index(inventory_hostname) }}_logs.log 2>&1" - name: wait until the Controller in this host is up and running @@ -117,4 +152,4 @@ register: result until: result.status == 200 retries: 12 - delay: 5 + delay: 5 \ No newline at end of file diff --git a/ansible/roles/invoker/tasks/clean.yml b/ansible/roles/invoker/tasks/clean.yml index 402826937b..ae5b83bfbd 100644 --- a/ansible/roles/invoker/tasks/clean.yml +++ b/ansible/roles/invoker/tasks/clean.yml @@ -39,6 +39,12 @@ state: absent become: "{{ logs.dir.become }}" +- name: remove invoker conf directory + file: + path: "{{ invoker.confdir }}/invoker{{ groups['invokers'].index(inventory_hostname) }}" + state: absent + become: "{{ invoker.dir.become }}" + # Workaround for orphaned ifstate.veth* files on Ubuntu 14.04 # See https://github.com/moby/moby/issues/22513 # Remove inactive files older than 60 minutes diff --git a/ansible/roles/invoker/tasks/deploy.yml b/ansible/roles/invoker/tasks/deploy.yml index 3da0b30ef2..57c2564b66 100644 --- a/ansible/roles/invoker/tasks/deploy.yml +++ b/ansible/roles/invoker/tasks/deploy.yml @@ -53,6 +53,13 @@ mode: 0777 become: "{{ logs.dir.become }}" +- name: ensure invoker config directory is created with permissions + file: + path: "{{ invoker.confdir }}/invoker{{ groups['invokers'].index(inventory_hostname) }}" + state: directory + mode: 0777 + become: "{{ invoker.dir.become }}" + - name: check, that required databases exist include: "{{ openwhisk_home }}/ansible/tasks/db/checkDb.yml" vars: @@ -103,6 +110,25 @@ with_items: "{{ invokerInfo }}" when: not invoker.allowMultipleInstances and item.Names[0] != "/invoker{{ groups['invokers'].index(inventory_hostname) }}" +- name: copy jmxremote password file + when: jmx.enabled + template: + src: "jmxremote.password.j2" + dest: "{{ invoker.confdir }}/invoker{{ groups['invokers'].index(inventory_hostname) }}/jmxremote.password" + mode: 0777 + +- name: copy jmxremote access file + when: jmx.enabled + template: + src: "jmxremote.access.j2" + dest: "{{ invoker.confdir }}/invoker{{ groups['invokers'].index(inventory_hostname) }}/jmxremote.access" + mode: 0777 + +- name: add additional jvm params if jmxremote is enabled + when: jmx.enabled + set_fact: + invoker_args: "{{ invoker.arguments }} {{ invoker.jmxremote.jvmArgs }}" + - name: start invoker using docker cli shell: > docker run -d @@ -114,7 +140,8 @@ --hostname invoker{{ groups['invokers'].index(inventory_hostname) }} --restart {{ docker.restart.policy }} -e JAVA_OPTS='-Xmx{{ invoker.heap }} -XX:+CrashOnOutOfMemoryError -XX:+UseGCOverheadLimit -XX:ErrorFile=/logs/java_error.log' - -e INVOKER_OPTS='{{ invoker.arguments }}' + -e INVOKER_OPTS='{{ invoker_args | default(invoker.arguments) }}' + -e JMX_REMOTE='{{ jmx.enabled }}' -e COMPONENT_NAME='invoker{{ groups['invokers'].index(inventory_hostname) }}' -e PORT='8080' -e KAFKA_HOSTS='{{ kafka_connect_string }}' @@ -158,9 +185,12 @@ -v /sys/fs/cgroup:/sys/fs/cgroup -v /run/runc:/run/runc -v {{ whisk_logs_dir }}/invoker{{ groups['invokers'].index(inventory_hostname) }}:/logs + -v {{ invoker.confdir }}/invoker{{ groups['invokers'].index(inventory_hostname) }}:/conf -v {{ dockerInfo["DockerRootDir"] }}/containers/:/containers -v {{ docker_sock | default('/var/run/docker.sock') }}:/var/run/docker.sock -p {{ invoker.port + groups['invokers'].index(inventory_hostname) }}:8080 + {% if jmx.enabled %} -p {{ jmx.basePortInvoker + groups['invokers'].index(inventory_hostname) }}:{{ jmx.basePortInvoker + groups['invokers'].index(inventory_hostname) }} {% endif %} + {% if jmx.enabled %} -p {{ jmx.rmiBasePortInvoker + groups['invokers'].index(inventory_hostname) }}:{{ jmx.rmiBasePortInvoker + groups['invokers'].index(inventory_hostname) }} {% endif %} {{ docker_registry }}{{ docker.image.prefix }}/invoker:{{ docker.image.tag }} /bin/sh -c "exec /init.sh {{ groups['invokers'].index(inventory_hostname) }} >> /logs/invoker{{ groups['invokers'].index(inventory_hostname) }}_logs.log 2>&1" diff --git a/ansible/templates/jmxremote.access.j2 b/ansible/templates/jmxremote.access.j2 new file mode 100644 index 0000000000..9d795680bd --- /dev/null +++ b/ansible/templates/jmxremote.access.j2 @@ -0,0 +1 @@ +{{ jmx.user }} readwrite diff --git a/ansible/templates/jmxremote.password.j2 b/ansible/templates/jmxremote.password.j2 new file mode 100644 index 0000000000..5d9c51b23f --- /dev/null +++ b/ansible/templates/jmxremote.password.j2 @@ -0,0 +1 @@ +{{ jmx.user }} {{ jmx.pass }} diff --git a/common/scala/.dockerignore b/common/scala/.dockerignore index eed7a81bea..8f456fdc91 100644 --- a/common/scala/.dockerignore +++ b/common/scala/.dockerignore @@ -1,3 +1,4 @@ * !transformEnvironment.sh +!copyJMXFiles.sh !build/distributions \ No newline at end of file diff --git a/common/scala/Dockerfile b/common/scala/Dockerfile index 15e5bdf3a3..114077cad4 100644 --- a/common/scala/Dockerfile +++ b/common/scala/Dockerfile @@ -31,4 +31,7 @@ RUN update-alternatives --install "/usr/bin/java" "java" "${JRE_HOME}/bin/java" mkdir /logs COPY transformEnvironment.sh / -RUN chmod +x transformEnvironment.sh \ No newline at end of file +RUN chmod +x transformEnvironment.sh + +COPY copyJMXFiles.sh / +RUN chmod +x copyJMXFiles.sh \ No newline at end of file diff --git a/common/scala/copyJMXFiles.sh b/common/scala/copyJMXFiles.sh new file mode 100644 index 0000000000..fc5004f957 --- /dev/null +++ b/common/scala/copyJMXFiles.sh @@ -0,0 +1,7 @@ +#!/bin/bash + +if [[ $( ls /conf/jmxremote.* 2> /dev/null ) ]] +then + mv /conf/jmxremote.* /root + chmod 600 /root/jmxremote.* +fi \ No newline at end of file diff --git a/common/scala/src/main/resources/logback.xml b/common/scala/src/main/resources/logback.xml index 50d6ee1311..c268a7e65b 100644 --- a/common/scala/src/main/resources/logback.xml +++ b/common/scala/src/main/resources/logback.xml @@ -1,5 +1,6 @@ <?xml version="1.0" encoding="UTF-8"?> <configuration> + <jmxConfigurator></jmxConfigurator> <appender name="console" class="ch.qos.logback.core.ConsoleAppender"> <encoder> <pattern>[%d{yyyy-MM-dd'T'HH:mm:ss.SSS'Z'}] [%p] %msg%n</pattern> diff --git a/core/controller/init.sh b/core/controller/init.sh index 232c405faa..ec7d5e8205 100644 --- a/core/controller/init.sh +++ b/core/controller/init.sh @@ -1,6 +1,8 @@ #!/bin/bash +./copyJMXFiles.sh + export CONTROLLER_OPTS CONTROLLER_OPTS="$CONTROLLER_OPTS -Dakka.remote.netty.tcp.bind-hostname=$(hostname -I) $(./transformEnvironment.sh)" -exec controller/bin/controller "$@" \ No newline at end of file +exec controller/bin/controller "$@" diff --git a/core/invoker/init.sh b/core/invoker/init.sh index beb5e71b9a..cfcf0565a0 100644 --- a/core/invoker/init.sh +++ b/core/invoker/init.sh @@ -1,6 +1,8 @@ #!/bin/bash +./copyJMXFiles.sh + export INVOKER_OPTS INVOKER_OPTS="$INVOKER_OPTS $(./transformEnvironment.sh)" -exec invoker/bin/invoker "$@" \ No newline at end of file +exec invoker/bin/invoker "$@" ---------------------------------------------------------------- This is an automated message from the Apache Git Service. To respond to the message, please log on GitHub and use the URL above to go to the specific comment. For queries about this service, please contact Infrastructure at: [email protected] With regards, Apache Git Services
