This is an automated email from the ASF dual-hosted git repository.
jbonofre pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/karaf-decanter.git
The following commit(s) were added to refs/heads/master by this push:
new 4ed4263 [KARAF-7002] Add Apache Druid collector
new 1fb4eb0 Merge pull request #226 from jbonofre/KARAF-7002
4ed4263 is described below
commit 4ed4263d725b72ef6a5cc2fd6ee353d2e8c3b9c3
Author: jbonofre <[email protected]>
AuthorDate: Thu Jan 21 19:03:22 2021 +0100
[KARAF-7002] Add Apache Druid collector
---
assembly/src/main/feature/feature.xml | 11 ++
collector/druid/NOTICE | 57 +++++++++++
collector/druid/pom.xml | 92 +++++++++++++++++
.../org.apache.karaf.decanter.collector.druid.cfg | 29 ++++++
.../decanter/collector/druid/DruidCollector.java | 114 +++++++++++++++++++++
collector/pom.xml | 1 +
.../src/main/asciidoc/user-guide/collectors.adoc | 33 ++++++
7 files changed, 337 insertions(+)
diff --git a/assembly/src/main/feature/feature.xml
b/assembly/src/main/feature/feature.xml
index 53b32b6..d39e479 100644
--- a/assembly/src/main/feature/feature.xml
+++ b/assembly/src/main/feature/feature.xml
@@ -51,6 +51,17 @@
org.apache.felix.eventadmin.IgnoreTimeout=org.apache.karaf.decanter.
<bundle>mvn:org.apache.karaf.decanter.collector/org.apache.karaf.decanter.collector.dropwizard/${project.version}</bundle>
</feature>
+ <feature name="decanter-collector-druid-core" version="${project.version}"
description="Karaf Decanter Druid Collector Core">
+ <feature>decanter-common</feature>
+ <feature>scheduler</feature>
+
<bundle>mvn:org.apache.karaf.decanter.collector/org.apache.karaf.decanter.collector.druid/${project.version}</bundle>
+ </feature>
+
+ <feature name="decanter-collector-druid" version="${project.version}"
description="Karaf Decanter Druid Collector">
+ <configfile
finalname="/etc/org.apache.karaf.decanter.collector.druid.cfg">mvn:org.apache.karaf.decanter.collector/org.apache.karaf.decanter.collector.druid/${project.version}/cfg</configfile>
+ <feature>decanter-collector-druid-core</feature>
+ </feature>
+
<feature name="decanter-collector-log" version="${project.version}"
description="Karaf Decanter Log Messages Collector">
<feature>decanter-common</feature>
<configfile
finalname="/etc/org.apache.karaf.decanter.collector.log.cfg">mvn:org.apache.karaf.decanter.collector/org.apache.karaf.decanter.collector.log/${project.version}/cfg</configfile>
diff --git a/collector/druid/NOTICE b/collector/druid/NOTICE
new file mode 100644
index 0000000..4e4af9e
--- /dev/null
+++ b/collector/druid/NOTICE
@@ -0,0 +1,57 @@
+Apache Karaf Decanter
+Copyright 2015-2019 The Apache Software Foundation
+
+I. Included Software
+
+This product includes software developed at
+The Apache Software Foundation (http://www.apache.org/).
+Licensed under the Apache License 2.0.
+
+This product includes software developed at
+Elastic (https://www.elastic.co/).
+Licensed under the Apache License 2.0.
+
+This product includes software developed at
+OrientDB (http://orientdb.com).
+Licensed under the Apache License 2.0.
+
+II. Used Software
+
+This product uses software developed at
+The OSGi Alliance (http://www.osgi.org/).
+Copyright (c) OSGi Alliance (2000, 2010).
+Licensed under the Apache License 2.0.
+
+This product uses software developed at
+OPS4J (http://www.ops4j.org/).
+Licensed under the Apache License 2.0.
+
+This product uses software developed at
+SLF4J (http://www.slf4j.org/).
+Licensed under the MIT License.
+
+This product uses software developed at
+JUnit (http://www.junit.org/).
+Licensed under the Eclipse Public License 1.0.
+
+This product uses software developed at
+Redis (http://www.redis.io).
+Licensed under the BSD license.
+
+This product uses software developed at
+Dropwizard (http://www.dropwizard.io).
+Licensed under the Apache License 2.0.
+
+This product uses software developed at
+searchbox.io (https://github.com/searchbox-io)
+Licensed under the Apache License 2.0.
+
+This product uses software developed at
+MongoDB (https://www.mongodb.com/)
+Licensed under the Apache License 2.0.
+
+III. License Summary
+- Apache License 2.0
+- MIT License
+- Eclipse Public License 1.0
+- BSD License
diff --git a/collector/druid/pom.xml b/collector/druid/pom.xml
new file mode 100644
index 0000000..be1d13a
--- /dev/null
+++ b/collector/druid/pom.xml
@@ -0,0 +1,92 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0";
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance";
xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
http://maven.apache.org/xsd/maven-4.0.0.xsd";>
+
+ <!--
+
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+ -->
+
+ <modelVersion>4.0.0</modelVersion>
+
+ <parent>
+ <groupId>org.apache.karaf.decanter</groupId>
+ <artifactId>collector</artifactId>
+ <version>2.7.0-SNAPSHOT</version>
+ <relativePath>../pom.xml</relativePath>
+ </parent>
+
+ <groupId>org.apache.karaf.decanter.collector</groupId>
+ <artifactId>org.apache.karaf.decanter.collector.druid</artifactId>
+ <packaging>bundle</packaging>
+ <name>Apache Karaf :: Decanter :: Collector :: Druid</name>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.karaf.decanter</groupId>
+ <artifactId>org.apache.karaf.decanter.api</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.karaf.decanter.collector</groupId>
+ <artifactId>org.apache.karaf.decanter.collector.utils</artifactId>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.felix</groupId>
+ <artifactId>maven-bundle-plugin</artifactId>
+ <inherited>true</inherited>
+ <extensions>true</extensions>
+ <configuration>
+ <obrRepository>NONE</obrRepository>
+ <instructions>
+
<Bundle-SymbolicName>${project.artifactId}</Bundle-SymbolicName>
+ <Export-Package>!*</Export-Package>
+ <Import-Package>
+ *
+ </Import-Package>
+ <Private-Package>
+ org.apache.karaf.decanter.collector.druid,
+ org.apache.karaf.decanter.collector.utils
+ </Private-Package>
+ <_dsannotations>*</_dsannotations>
+ </instructions>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.codehaus.mojo</groupId>
+ <artifactId>build-helper-maven-plugin</artifactId>
+ <executions>
+ <execution>
+ <phase>package</phase>
+ <goals>
+ <goal>attach-artifact</goal>
+ </goals>
+ <configuration>
+ <artifacts>
+ <artifact>
+
<file>src/main/cfg/org.apache.karaf.decanter.collector.druid.cfg</file>
+ <type>cfg</type>
+ </artifact>
+ </artifacts>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+</project>
diff --git
a/collector/druid/src/main/cfg/org.apache.karaf.decanter.collector.druid.cfg
b/collector/druid/src/main/cfg/org.apache.karaf.decanter.collector.druid.cfg
new file mode 100644
index 0000000..84e1f43
--- /dev/null
+++ b/collector/druid/src/main/cfg/org.apache.karaf.decanter.collector.druid.cfg
@@ -0,0 +1,29 @@
+################################################################################
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements. See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+################################################################################
+
+#
+# Druid broker query API location
+#
+druid.broker.location=http://localhost:8888/druid/v2/sql/
+
+# Druid queries set, using syntax: query.id
+# query.foo=select * from datasource
+
+# Unmarshaller to use
+unmarshaller.target=(dataFormat=json)
diff --git
a/collector/druid/src/main/java/org/apache/karaf/decanter/collector/druid/DruidCollector.java
b/collector/druid/src/main/java/org/apache/karaf/decanter/collector/druid/DruidCollector.java
new file mode 100644
index 0000000..5122bfb
--- /dev/null
+++
b/collector/druid/src/main/java/org/apache/karaf/decanter/collector/druid/DruidCollector.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.karaf.decanter.collector.druid;
+
+import org.apache.karaf.decanter.api.marshaller.Unmarshaller;
+import org.apache.karaf.decanter.collector.utils.PropertiesPreparator;
+import org.osgi.service.component.ComponentContext;
+import org.osgi.service.component.annotations.Activate;
+import org.osgi.service.component.annotations.Component;
+import org.osgi.service.component.annotations.Reference;
+import org.osgi.service.event.Event;
+import org.osgi.service.event.EventAdmin;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.*;
+import java.net.HttpURLConnection;
+import java.net.URL;
+import java.util.Dictionary;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.Map;
+
+/**
+ * Decanter Druid collector, periodically execute queries on Apache Druid
+ */
+@Component(
+ service = Runnable.class,
+ name = "org.apache.karaf.decanter.collector.druid",
+ immediate = true,
+ property = {
+ "decanter.collector.name=druid",
+ "scheduler.period:Long=60",
+ "scheduler.concurrent:Boolean=false",
+ "scheduler.name=decanter-collector-druid"
+ }
+)
+public class DruidCollector implements Runnable {
+
+ @Reference
+ public EventAdmin dispatcher;
+
+ @Reference
+ public Unmarshaller unmarshaller;
+
+ private Dictionary<String, Object> config;
+
+ private final static Logger LOGGER =
LoggerFactory.getLogger(DruidCollector.class);
+
+ @Activate
+ public void activate(ComponentContext componentContext) {
+ config = componentContext.getProperties();
+ }
+
+ @Override
+ public void run() {
+ // get Druid broker location
+ String druidBroker = (config.get("druid.broker.location") != null) ?
(String) config.get("druid.broker.location") :
"http://localhost:8888/druid/v2/sql/";;
+
+ Enumeration<String> keys = config.keys();
+ while (keys.hasMoreElements()) {
+ String key = keys.nextElement();
+ if (key.startsWith("query.")) {
+ try {
+ Map<String, Object> data = new HashMap<>();
+ data.put("query", key.substring("query.".length()));
+ data.putAll(executeQuery(druidBroker, (String)
config.get(key)));
+ PropertiesPreparator.prepare(data, config);
+ String topic = (config.get("topic") != null) ? (String)
config.get("topic") : "decanter/collect/druid";
+ dispatcher.postEvent(new Event(topic, data));
+ } catch (Exception e) {
+ LOGGER.warn("Can't execute query {}",
key.substring("query.".length()), e);
+ }
+ }
+ }
+ }
+
+ private Map<String, Object> executeQuery(String broker, String query)
throws Exception {
+ LOGGER.debug("Executing {} on {}", query, broker);
+ HttpURLConnection connection = (HttpURLConnection) new
URL(broker).openConnection();
+ connection.setRequestMethod("POST");
+ connection.addRequestProperty("Content-Type", "application/json");
+ connection.setDoOutput(true);
+ connection.setDoInput(true);
+ String queryJson = "{ \"query\": \"" + query + "\", \"resultFormat\":
\"object\"}";
+ try (BufferedWriter writer = new BufferedWriter(new
OutputStreamWriter(connection.getOutputStream()))) {
+ writer.write(queryJson);
+ }
+ StringBuilder result = new StringBuilder();
+ try (BufferedReader reader = new BufferedReader(new
InputStreamReader(connection.getInputStream()))) {
+ String line;
+ while ((line = reader.readLine()) != null) {
+ result.append(line);
+ }
+ }
+ String jsonResult = "{ \"result\": " + result.toString() + "}";
+ return unmarshaller.unmarshal(new
ByteArrayInputStream(jsonResult.getBytes()));
+ }
+
+}
diff --git a/collector/pom.xml b/collector/pom.xml
index 069ac81..698b9bd 100644
--- a/collector/pom.xml
+++ b/collector/pom.xml
@@ -38,6 +38,7 @@
<module>camel</module>
<module>configadmin</module>
<module>dropwizard</module>
+ <module>druid</module>
<module>elasticsearch</module>
<module>eventadmin</module>
<module>file</module>
diff --git a/manual/src/main/asciidoc/user-guide/collectors.adoc
b/manual/src/main/asciidoc/user-guide/collectors.adoc
index 10d9831..1d3aa35 100644
--- a/manual/src/main/asciidoc/user-guide/collectors.adoc
+++ b/manual/src/main/asciidoc/user-guide/collectors.adoc
@@ -1154,6 +1154,39 @@ snmp.community=public
* `snmp.context.name` is optional
* `snmp.community` is the community to use, `public` by default
+==== Druid
+
+Karaf Decanter Druid collector schedules execution of queries on Apache Druid
broker.
+
+It allows to easily create analytics on data and schedule execution with
Decanter. Then, the query result is sent to the dispatcher.
+
+The `karaf-collector-druid` feature installs the Druid collector:
+
+----
+karaf@root()> feature:install decanter-collector-druid
+----
+
+The location of the Apache Druid broker and Druid queries to execute are
configured in `etc/org.apache.karaf.decanter.collector.druid.cfg` configuration
file:
+
+----
+#
+# Druid broker query API location
+#
+druid.broker.location=http://localhost:8888/druid/v2/sql/
+
+# Druid queries set, using syntax: query.id
+query.foo=select sum_operatingSystem_threadCount from decanter
+
+# Unmarshaller to use
+unmarshaller.target=(dataFormat=json)
+----
+
+The `druid.broker.location` is the URL of the Apache Druid broker.
+
+Then, we can add the Druid queries using the format `query.ID=QUERY`.
+
+Decanter Druid collector schedules the queries and send the results in the
dispatcher.
+
==== Customizing properties in collectors
You can add, rename or remove properties collected by the collectors before
sending it to the dispatcher.
