Repository: brooklyn-docs Updated Branches: refs/heads/master b45541d8c -> 7df106adf
Fixed advanced example by shipping blueprints in repo Project: http://git-wip-us.apache.org/repos/asf/brooklyn-docs/repo Commit: http://git-wip-us.apache.org/repos/asf/brooklyn-docs/commit/2c49bccf Tree: http://git-wip-us.apache.org/repos/asf/brooklyn-docs/tree/2c49bccf Diff: http://git-wip-us.apache.org/repos/asf/brooklyn-docs/diff/2c49bccf Branch: refs/heads/master Commit: 2c49bccf88cf28046eebe3fceffa7ada9a7d5167 Parents: b45541d Author: graeme.miller <[email protected]> Authored: Thu Jun 22 10:59:46 2017 +0100 Committer: graeme.miller <[email protected]> Committed: Thu Jun 22 13:30:15 2017 +0100 ---------------------------------------------------------------------- guide/blueprints/advanced-example.md | 10 +- .../brooklyn-elasticsearch-catalog.bom | 125 +++++++++++++++++++ .../example_yaml/brooklyn-elk-catalog.bom | 94 ++++++++++++++ .../example_yaml/brooklyn-kibana-catalog.bom | 58 +++++++++ .../example_yaml/brooklyn-logstash-catalog.bom | 66 ++++++++++ 5 files changed, 349 insertions(+), 4 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/brooklyn-docs/blob/2c49bccf/guide/blueprints/advanced-example.md ---------------------------------------------------------------------- diff --git a/guide/blueprints/advanced-example.md b/guide/blueprints/advanced-example.md index 6513191..34395d2 100644 --- a/guide/blueprints/advanced-example.md +++ b/guide/blueprints/advanced-example.md @@ -7,6 +7,8 @@ By this point you should be familiar with the fundamental concepts behind both A The intention is that this example is used to learn the more in-depth concepts, and also to serve as a reference when writing your own blueprints. This page will first explain what the example application is and how to run it, then it will spotlight interesting features. +Please note, there is now a much more up-to-date ELK blueprint that can be found [here](https://github.com/brooklyncentral/brooklyn-elk/). We've using an older version of this in the tutorial as it highlights some key Brooklyn concepts. + ### ELK Stack Example @@ -28,10 +30,10 @@ For more about the ELK stack, please see the documentation [here](https://www.el There are four blueprints that make up this application. Each of them are used to add one or more catalog items to Brooklyn. You can find them below: -* [Elasticsearch](https://github.com/brooklyncentral/brooklyn-elk/blob/master/brooklyn-elasticsearch-catalog.bom) -* [Logstash](https://github.com/brooklyncentral/brooklyn-elk/blob/master/brooklyn-logstash-catalog.bom) -* [Kibana](https://github.com/brooklyncentral/brooklyn-elk/blob/master/brooklyn-kibana-catalog.bom) -* [ELK](https://github.com/brooklyncentral/brooklyn-elk/blob/master/brooklyn-elk-catalog.bom) +* [Elasticsearch](example_yaml/brooklyn-elasticsearch-catalog.bom) +* [Logstash](example_yaml/brooklyn-logstash-catalog.bom) +* [Kibana](example_yaml/brooklyn-kibana-catalog.bom) +* [ELK](example_yaml/brooklyn-elk-catalog.bom) #### Running the example First, add all four blueprints to the Brooklyn Catalog. This can be done by clicking the 'Catalog' tab, clicking the '+' http://git-wip-us.apache.org/repos/asf/brooklyn-docs/blob/2c49bccf/guide/blueprints/example_yaml/brooklyn-elasticsearch-catalog.bom ---------------------------------------------------------------------- diff --git a/guide/blueprints/example_yaml/brooklyn-elasticsearch-catalog.bom b/guide/blueprints/example_yaml/brooklyn-elasticsearch-catalog.bom new file mode 100644 index 0000000..bcfaa1d --- /dev/null +++ b/guide/blueprints/example_yaml/brooklyn-elasticsearch-catalog.bom @@ -0,0 +1,125 @@ +brooklyn.catalog: + id: elasticsearch + version: 1.0.0-SNAPSHOT + iconUrl: https://static-www.elastic.co/assets/bltfdc1abb6ea9e2157/icon-elasticsearch.svg + name: Elasticsearch + license: Apache-2.0 + issues_url: https://github.com/brooklyncentral/brooklyn-elk/issues + item: + type: brooklyn.entity.group.DynamicCluster + initialSize: $brooklyn:config("elasticsearch.cluster.initial.size") + name: Elasticsearch Cluster + id: elasticsearch-cluster + description: A cluster of Elasticsearch nodes + + brooklyn.config: + install.version: 2.4.0 + elasticsearch.http.port: 9220 + elasticsearch.tcp.port: 9330 + + brooklyn.enrichers: + - type: org.apache.brooklyn.enricher.stock.Aggregator + brooklyn.config: + enricher.sourceSensor: $brooklyn:sensor("url.tcp") + enricher.targetSensor: $brooklyn:sensor("urls.tcp.list") + enricher.aggregating.fromMembers: true + - type: org.apache.brooklyn.enricher.stock.Joiner + brooklyn.config: + enricher.sourceSensor: $brooklyn:sensor("urls.tcp.list") + enricher.targetSensor: $brooklyn:sensor("urls.tcp.string") + uniqueTag: urls.quoted.string + - type: org.apache.brooklyn.enricher.stock.Transformer + brooklyn.config: + enricher.sourceSensor: $brooklyn:sensor("urls.tcp.string") + enricher.targetSensor: $brooklyn:sensor("urls.tcp.withBrackets") + enricher.targetValue: $brooklyn:formatString("[%s]", $brooklyn:attributeWhenReady("urls.tcp.string")) + - type: org.apache.brooklyn.enricher.stock.Aggregator + brooklyn.config: + enricher.sourceSensor: $brooklyn:sensor("url.http") + enricher.targetSensor: $brooklyn:sensor("urls.http.list") + enricher.aggregating.fromMembers: true + - type: org.apache.brooklyn.enricher.stock.Joiner + brooklyn.config: + enricher.sourceSensor: $brooklyn:sensor("urls.http.list") + enricher.targetSensor: $brooklyn:sensor("urls.http.string") + uniqueTag: urls.http.quoted.string + minimum: $brooklyn:config("elasticsearch.cluster.initial.size") + - type: org.apache.brooklyn.enricher.stock.Transformer + brooklyn.config: + enricher.sourceSensor: $brooklyn:sensor("urls.http.string") + enricher.targetSensor: $brooklyn:sensor("urls.http.withBrackets") + enricher.targetValue: $brooklyn:formatString("[%s]", $brooklyn:attributeWhenReady("urls.http.string")) + - type: org.apache.brooklyn.enricher.stock.Aggregator + brooklyn.config: + enricher.sourceSensor: $brooklyn:sensor("host.subnet.address") + enricher.targetSensor: $brooklyn:sensor("host.address.first") + enricher.aggregating.fromMembers: true + enricher.transformation: + $brooklyn:object: + type: "org.apache.brooklyn.util.collections.CollectionFunctionals$FirstElementFunction" + + memberSpec: + $brooklyn:entitySpec: + - type: brooklyn.entity.basic.VanillaSoftwareProcess + name: Elasticsearch Node + + provisioning.properties: + minRam: 7gb + minDisk: 100gb + minCores: 4 + osFamily: ubuntu + imageNameRegex: .*16.04.* + templateOptions: + portSpeed: 1000 + diskType: "LOCAL" + domainName: example.com + groupId: elasticsearch + + shell.env: + ELASTICSEARCH_VERSION: $brooklyn:config("install.version") + URLS_WITH_BRACKETS: $brooklyn:component("parent", "").attributeWhenReady("urls.tcp.withBrackets") + ES_HTTP_PORT: $brooklyn:config("elasticsearch.http.port") + ES_TCP_PORT: $brooklyn:config("elasticsearch.tcp.port") + IP_ADDRESS: $brooklyn:attributeWhenReady("host.subnet.address") + + install.command: | + sudo apt-get update + #sudo apt-get install -y openjdk-7-jdk wget + + wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add - + echo "deb https://packages.elastic.co/elasticsearch/2.x/debian stable main" | sudo tee -a /etc/apt/sources.list.d/elasticsearch-2.x.list + sudo apt-get update && sudo apt-get -y install wget openjdk-8-jre elasticsearch=${ELASTICSEARCH_VERSION} + sudo update-rc.d elasticsearch defaults 95 10 + sudo /bin/systemctl daemon-reload + sudo /bin/systemctl enable elasticsearch.service + + customize.command: | + sudo rm -fr sudo tee /etc/elasticsearch/elasticsearch.yml + echo discovery.zen.ping.multicast.enabled: false | sudo tee -a /etc/elasticsearch/elasticsearch.yml + echo discovery.zen.ping.unicast.enabled: true | sudo tee -a /etc/elasticsearch/elasticsearch.yml + echo discovery.zen.ping.unicast.hosts: ${URLS_WITH_BRACKETS} | sudo tee -a /etc/elasticsearch/elasticsearch.yml + echo http.port: ${ES_HTTP_PORT} | sudo tee -a /etc/elasticsearch/elasticsearch.yml + echo transport.tcp.port: ${ES_TCP_PORT} | sudo tee -a /etc/elasticsearch/elasticsearch.yml + echo network.host: ${IP_ADDRESS} | sudo tee -a /etc/elasticsearch/elasticsearch.yml + + launch.command: sudo systemctl start elasticsearch.service + + stop.command: sudo systemctl stop elasticsearch.service + + checkRunning.command: | + sudo systemctl status elasticsearch.service + #$brooklyn:formatString("counter=`wget -T 15 -q -O- %s:%s | grep -c \"status. : 200\"`; if [ $counter -eq 0 ]; then exit 1; fi", + #$brooklyn:attributeWhenReady("host.address"), + #$brooklyn:config("elasticsearch.http.port")) + + brooklyn.enrichers: + - type: org.apache.brooklyn.enricher.stock.Transformer + brooklyn.config: + enricher.sourceSensor: $brooklyn:sensor("host.subnet.address") + enricher.targetSensor: $brooklyn:sensor("url.tcp") + enricher.targetValue: $brooklyn:formatString("%s:%s", $brooklyn:attributeWhenReady("host.subnet.address"), $brooklyn:config("elasticsearch.tcp.port")) + - type: org.apache.brooklyn.enricher.stock.Transformer + brooklyn.config: + enricher.sourceSensor: $brooklyn:sensor("host.subnet.address") + enricher.targetSensor: $brooklyn:sensor("url.http") + enricher.targetValue: $brooklyn:formatString("%s:%s", $brooklyn:attributeWhenReady("host.subnet.address"), $brooklyn:config("elasticsearch.http.port")) http://git-wip-us.apache.org/repos/asf/brooklyn-docs/blob/2c49bccf/guide/blueprints/example_yaml/brooklyn-elk-catalog.bom ---------------------------------------------------------------------- diff --git a/guide/blueprints/example_yaml/brooklyn-elk-catalog.bom b/guide/blueprints/example_yaml/brooklyn-elk-catalog.bom new file mode 100644 index 0000000..ab27d77 --- /dev/null +++ b/guide/blueprints/example_yaml/brooklyn-elk-catalog.bom @@ -0,0 +1,94 @@ +brooklyn.catalog: + version: 1.0.0-SNAPSHOT + iconUrl: https://avatars0.githubusercontent.com/u/6764390?v=3&s=400 + license: Apache-2.0 + issues_url: https://github.com/brooklyncentral/brooklyn-elk/issues # TODO move this there + itemType: template + item: + type: org.apache.brooklyn.entity.stock.BasicApplication + name: ELK Stack + id: ELK-Stack + description: | + Simple ELK stack deployment: provisions Elasticsearch, Kibana and Logstash as a child of Apache Tomcat 8 + + brooklyn.parameters: + - name: elasticsearch.cluster.initial.size + type: integer + label: Initial Size of the ElasticSearch Cluster + description: The initial size of the cluster of ElasticSearch nodes + default: 5 + + services: + - type: elasticsearch + id: es + name: Elasticsearch Cluster + brooklyn.config: + install.version: 2.4.0 + + - type: kibana-standalone + id: kibana + name: Kibana Server + customize.latch: $brooklyn:component("es").attributeWhenReady("service.isUp") + brooklyn.config: + kibana.elasticsearch.ip: $brooklyn:component("es").attributeWhenReady("host.address.first") + kibana.elasticsearch.port: $brooklyn:component("es").config("elasticsearch.http.port") + + provisioning.properties: + minRam: 8gb + minDisk: 100gb + minCores: 4 + osFamily: ubuntu + imageNameRegex: .*16.04.* + templateOptions: + portSpeed: 1000 + diskType: "LOCAL" + domainName: example.com + groupId: kibana + + - type: org.apache.brooklyn.entity.webapp.tomcat.Tomcat8Server + id: tomcat + customize.latch: $brooklyn:entity("es").attributeWhenReady("service.isUp") + brooklyn.config: + wars.root: "http://search.maven.org/remotecontent?filepath=org/apache/brooklyn/example/brooklyn-example-hello-world-sql-webapp/0.8.0-incubating/brooklyn-example-hello-world-sql-webapp-0.8.0-incubating.war"; + children.startable.mode: background_late + + provisioning.properties: + minRam: 8gb + minDisk: 100gb + minCores: 4 + osFamily: ubuntu + imageNameRegex: .*16.04.* + templateOptions: + portSpeed: 1000 + diskType: "LOCAL" + domainName: example.com + groupId: logstash + + brooklyn.children: + - type: logstash-child + name: Logstash + brooklyn.config: + logstash.elasticsearch.hosts: $brooklyn:entity("es").attributeWhenReady("urls.http.withBrackets") + logstash.config.input: + $brooklyn:formatString: + - | + input { + file { + path => "%s/logs/localhost_access_log.*" + start_position => "beginning" + } + } + - $brooklyn:entity("tomcat").attributeWhenReady("run.dir") + logstash.config.filter: | + filter { + grok { + match => { "message" => "%{COMBINEDAPACHELOG}" } + } + date { + match => [ "timestamp" , "dd/MMM/yyyy:HH:mm:ss Z" ] + } + } + + + + http://git-wip-us.apache.org/repos/asf/brooklyn-docs/blob/2c49bccf/guide/blueprints/example_yaml/brooklyn-kibana-catalog.bom ---------------------------------------------------------------------- diff --git a/guide/blueprints/example_yaml/brooklyn-kibana-catalog.bom b/guide/blueprints/example_yaml/brooklyn-kibana-catalog.bom new file mode 100644 index 0000000..8aba455 --- /dev/null +++ b/guide/blueprints/example_yaml/brooklyn-kibana-catalog.bom @@ -0,0 +1,58 @@ +brooklyn.catalog: + version: 1.0.0-SNAPSHOT + iconUrl: https://static-www.elastic.co/assets/blt6c40e42f467aa55c/icon-kibana.svg + items: + - id: kibana-standalone + name: "Kibana server" + description: Kibana server. Callers should configure 'kibana.elasticsearch.ip' + item: + type: org.apache.brooklyn.entity.software.base.VanillaSoftwareProcess + provisioning.properties: + osFamily: ubuntu + + brooklyn.config: + install.version: "4.6.1" + kibana.elasticsearch.ip: 127.0.0.1 # must be supplied by caller! + kibana.elasticsearch.port: 9200 # must be supplied by caller! + kibana.port: 5601 + + shell.env: + KIBANA_VERSION: $brooklyn:config("install.version") + ELASTICSEARCH_IP: $brooklyn:config("kibana.elasticsearch.ip") + ELASTICSEARCH_PORT: $brooklyn:config("kibana.elasticsearch.port") + KIBANA_PORT: $brooklyn:config("kibana.port") + + install.command: | + #Update the system + sudo apt-get update + + # Download the cert and repository + wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add - + echo "deb https://packages.elastic.co/kibana/4.6/debian stable main" | sudo tee -a /etc/apt/sources.list.d/kibana.list + + # Install pre-reqs and kibana + sudo apt-get update && sudo apt-get -y install kibana=${KIBANA_VERSION} + + + # customize config file for kibana + sudo sed -i 's|# elasticsearch.url: "http://localhost:9200";|elasticsearch.url: "http://'"${ELASTICSEARCH_IP}"':'"${ELASTICSEARCH_PORT}"'"|g' /opt/kibana/config/kibana.yml + sudo sed -i 's|5601|'"${KIBANA_PORT}"'|g' /opt/kibana/config/kibana.yml + sudo sed -i 's|# log_file: ./kibana.log|log_file: ./kibana.log|g' /opt/kibana/config/kibana.yml + + sudo update-rc.d kibana defaults 95 10 + sudo /bin/systemctl daemon-reload + sudo /bin/systemctl enable kibana.service + #sudo systemctl start kibana.service + + launch.command: sudo systemctl start kibana.service + + stop.command: sudo systemctl stop kibana.service + + checkRunning.command: sudo systemctl status kibana.service + + brooklyn.enrichers: + - type: org.apache.brooklyn.enricher.stock.Transformer + brooklyn.config: + enricher.sourceSensor: $brooklyn:sensor("host.address") + enricher.targetSensor: $brooklyn:sensor("main.uri") + enricher.targetValue: $brooklyn:formatString("http://%s:%s";, $brooklyn:attributeWhenReady("host.address"), $brooklyn:config("kibana.port")) http://git-wip-us.apache.org/repos/asf/brooklyn-docs/blob/2c49bccf/guide/blueprints/example_yaml/brooklyn-logstash-catalog.bom ---------------------------------------------------------------------- diff --git a/guide/blueprints/example_yaml/brooklyn-logstash-catalog.bom b/guide/blueprints/example_yaml/brooklyn-logstash-catalog.bom new file mode 100644 index 0000000..95f2f4f --- /dev/null +++ b/guide/blueprints/example_yaml/brooklyn-logstash-catalog.bom @@ -0,0 +1,66 @@ +brooklyn.catalog: + version: 1.0.0-SNAPSHOT + iconUrl: https://static-www.elastic.co/assets/blte39224bc57d3e015/icon-logstash.svg + items: + - id: logstash-standalone + name: "Logstash server" + description: "Single Logstash server" + item: + type: org.apache.brooklyn.entity.software.base.VanillaSoftwareProcess + provisioning.properties: + osFamily: ubuntu + + brooklyn.config: + install.version: "2.4.0" + logstash.config.input: input { file { path => "/var/log/*" } } + logstash.config.filter: '' + logstash.config.output: + $brooklyn:formatString: + - | + output { + elasticsearch { + hosts => %s + } + } + - $brooklyn:config("logstash.elasticsearch.hosts") + logstash.config.dir: /etc/logstash/conf.d + + shell.env: + LOGSTASH_VERSION: $brooklyn:config("install.version") + INPUT_CONFIG: $brooklyn:config("logstash.config.input") + FILTER_CONFIG: $brooklyn:config("logstash.config.filter") + OUTPUT_CONFIG: $brooklyn:config("logstash.config.output") + CONFIG_DIR: $brooklyn:config("logstash.config.dir") + ELASTICSEARCH_HOSTS: $brooklyn:config("logstash.elasticsearch.hosts") + + install.command: | + #Update the system + sudo apt-get update + + # Download the cert and repository + wget -qO - https://packages.elastic.co/GPG-KEY-elasticsearch | sudo apt-key add - + echo "deb https://packages.elastic.co/logstash/2.4/debian stable main" | sudo tee -a /etc/apt/sources.list + sudo apt-get update && sudo apt-get -y install openjdk-8-jre logstash + + # add a config file for logstash + sudo mkdir -p ${CONFIG_DIR} + echo "${INPUT_CONFIG} ${FILTER_CONFIG} ${OUTPUT_CONFIG}" | sudo tee ${CONFIG_DIR}/01-basic.conf + + launch.command: sudo systemctl start logstash.service + + stop.command: sudo systemctl stop logstash.service + + checkRunning.command: sudo systemctl status logstash.service + + - id: logstash-child + name: "Logstash Child" + description: | + Logstash server to be embedded as a child of a SoftwareProcess that + publishes its 'log.location' as a sensor. + Callers should configure 'logstash.elasticsearch.hosts' (if using ES) + or 'logstash.config.output'. + + item: + type: logstash-standalone + brooklyn.config: + logstash.elasticsearch.host: 127.0.0.1:9200 # must be supplied by caller!
