metron git commit: METRON-1568: Stellar should have a _ special variable which returns the message in map form closes apache/incubator-metron#1021
Repository: metron
Updated Branches:
refs/heads/master 523c38cf6 -> 1b9828e6a
METRON-1568: Stellar should have a _ special variable which returns the message
in map form closes apache/incubator-metron#1021
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/1b9828e6
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/1b9828e6
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/1b9828e6
Branch: refs/heads/master
Commit: 1b9828e6a68fd39fd308ebb80f1d79cd490ceaec
Parents: 523c38c
Author: cstella
Authored: Thu Jun 7 10:54:05 2018 -0400
Committer: cstella
Committed: Thu Jun 7 10:54:05 2018 -0400
--
.../enrichment/handler/StellarConfig.java | 11 +-
.../StellarTransformationTest.java | 30 ++-
.../adapters/stellar/StellarAdapterTest.java| 26 ++
.../integration/EnrichmentIntegrationTest.java | 2 +
.../main/config/zookeeper/enrichments/test.json | 1 +
.../metron/pcap/filter/PcapFieldResolver.java | 5 +
metron-stellar/stellar-common/README.md | 1 +
.../metron/stellar/common/utils/ConcatMap.java | 256 +++
.../common/utils/StellarProcessorUtils.java | 51 +++-
.../metron/stellar/dsl/MapVariableResolver.java | 8 +
.../metron/stellar/dsl/VariableResolver.java| 1 +
.../dsl/functions/DataStructureFunctions.java | 4 +
.../stellar/common/utils/ConcatMapTest.java | 83 ++
.../stellar/dsl/functions/BasicStellarTest.java | 18 ++
14 files changed, 487 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/1b9828e6/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java
--
diff --git
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java
index 62e0263..9f96a22 100644
---
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java
+++
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/enrichment/handler/StellarConfig.java
@@ -19,6 +19,7 @@ package
org.apache.metron.common.configuration.enrichment.handler;
import org.apache.metron.stellar.common.StellarAssignment;
import org.apache.metron.stellar.common.StellarProcessor;
+import org.apache.metron.stellar.dsl.VariableResolver;
import org.json.simple.JSONObject;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -142,8 +143,14 @@ public class StellarConfig implements Config {
{
Map messageSegment = new HashMap<>();
-for(String variable : stellarFields) {
- messageSegment.put(variable, message.get(variable));
+if(stellarFields.contains(VariableResolver.ALL_FIELDS)) {
+ //we need to include all of the fields here.
+ messageSegment.putAll(message);
+}
+else {
+ for (String variable : stellarFields) {
+messageSegment.put(variable, message.get(variable));
+ }
}
return messageSegment;
}
http://git-wip-us.apache.org/repos/asf/metron/blob/1b9828e6/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java
--
diff --git
a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java
b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java
index fc91844..3b7c7bb 100644
---
a/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java
+++
b/metron-platform/metron-common/src/test/java/org/apache/metron/common/field/transformation/StellarTransformationTest.java
@@ -78,6 +78,32 @@ public class StellarTransformationTest {
@Multiline
public static String badConfig;
+ /**
+ { "fieldTransformations" : [
+{ "transformation" : "STELLAR"
+,"output" : [ "new_field"]
+,"config" : {
+ "new_field" : "MAP_GET('source.type', _)"
+}
+}
+]
+ }
+ */
+ @Multiline
+ public static String configAll;
+
+ @Test
+ public void testConfigAll() throws Exception {
+SensorParserConfig c =
SensorParserConfig.fromBytes(Bytes.toBytes(configAll));
+JSONObject input = new JSONObject();
+input.put("source.type", "test");
+for (FieldTransformer handler : c.getFieldTransformations()) {
+ handler.transformAndUpdate(input,
svn commit: r27304 - in /release/metron/0.5.0: apache-metron-0.5.0.tar.gz.md5 apache-metron-0.5.0.tar.gz.sha
Author: leet Date: Thu Jun 7 13:49:42 2018 New Revision: 27304 Log: Removing .md5 and .sha per release distribution policy Removed: release/metron/0.5.0/apache-metron-0.5.0.tar.gz.md5 release/metron/0.5.0/apache-metron-0.5.0.tar.gz.sha
svn commit: r27301 [2/2] - in /release/metron: ./ 0.5.0/
Added: release/metron/0.5.0/LICENSE == --- release/metron/0.5.0/LICENSE (added) +++ release/metron/0.5.0/LICENSE Thu Jun 7 11:49:27 2018 @@ -0,0 +1,953 @@ +Apache License + Version 2.0, January 2004 +http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a
svn commit: r27301 [1/2] - in /release/metron: ./ 0.5.0/
Author: leet Date: Thu Jun 7 11:49:27 2018 New Revision: 27301 Log: Adding artifacts for Metron 0.5.0 Added: release/metron/0.5.0/ release/metron/0.5.0/CHANGES release/metron/0.5.0/KEYS release/metron/0.5.0/LICENSE release/metron/0.5.0/NOTICE release/metron/0.5.0/apache-metron-0.5.0.tar.gz (with props) release/metron/0.5.0/apache-metron-0.5.0.tar.gz.asc release/metron/0.5.0/apache-metron-0.5.0.tar.gz.md5 release/metron/0.5.0/apache-metron-0.5.0.tar.gz.sha release/metron/0.5.0/apache-metron-0.5.0.tar.gz.sha1 release/metron/0.5.0/apache-metron-0.5.0.tar.gz.sha256 release/metron/0.5.0/apache-metron-0.5.0.tar.gz.sha512 Modified: release/metron/KEYS Added: release/metron/0.5.0/CHANGES == --- release/metron/0.5.0/CHANGES (added) +++ release/metron/0.5.0/CHANGES Thu Jun 7 11:49:27 2018 @@ -0,0 +1,122 @@ +METRON-1586 Defaulting for the source type field in alerts UI does not work (merrimanr via justinleet) closes apache/metron#1038 +METRON-1569: Allow user to change field name conversion when indexing to Elasticsearch (nickwallen via mmiklavc) closes apache/metron#1022 +METRON-1544 Flaky test: org.apache.metron.stellar.common.CachingStellarProcessorTest#testCaching (nickwallen) closes apache/metron#1015 +METRON-1580 Release candidate check script requires Bro Plugin (nickwallen via ottobackwards) closes apache/metron#1034 +METRON-1532 Getting started documentation improvements (sardell via nickwallen) closes apache/metron#1001 +METRON-1576 bundle.css RAT failure for metron-interface/metron-alerts (justinleet) closes apache/metron#1029 +METRON-1575 Add leet gpg public key to the KEYS file (justinleet) closes apache/metron#1028 +METRON-1574 Update version to 0.5.0 (justinleet) closes apache/metron#1026 +METRON-1566 Alert updates are not propagated to metaalert child alerts (merrimanr) closes apache/metron#1018 +METRON-1565 Metaalerts fix denormalization after moving to active status (merrimanr) closes apache/metron#1017 +METRON-1548 Remove hardcoded source:type from Alerts UI (justinleet) closes apache/metron#1010 +METRON-1548 Remove hardcoded source:type from Alerts UI (sardell via justinleet) closes apache/metron#1010 +METRON-1564 Full dev kafka has offsets.topic.replication.factor set to 3 instead of 1 (justinleet) closes apache/metron#1016 +METRON-1552: Add gzip file validation check to the geo loader (mmiklavc via mmiklavc) closes apache/metron#1011 +METRON-1551 Profiler Should Not Use Java Serialization (nickwallen) closes apache/metron#1012 +METRON-1549: Add empty object test to WriterBoltIntegrationTest implementation (mmiklavc via mmiklavc) closes apache/metron#1009 +METRON-1541 Mvn clean results in git status having deleted files. (justinleet via nickwallen) closes apache/metron#1003 +METRON-1461 MIN MAX stellar function should take a stats or list object and return min/max (MohanDV via nickwallen) closes apache/metron#942 +METRON-1184 EC2 Deployment - Updating control_path to accommodate for Linux (Ahmed Shah via ottobackwards) closes apache/metron#754 +METRON-1530 Default proxy config settings in metron-contrib need to be updated (sardell via merrimanr) closes apache/metron#998 +METRON-1545 Upgrade Spring and Spring Boot (merrimanr) closes apache/metron#1008 +METRON-1543 Unable to Set Parser Output Topic in Sensor Config (nickwallen) closes apache/metron#1007 +METRON-1539: Specialized RENAME field transformer closes apache/incubator-metron#1002 +METRON-1520: Add caching for stellar field transformations closes apache/incubator-metron#990 +METRON-1529 CONFIG_GET Fails to Retrieve Latest Config When Run in Zeppelin REPL (nickwallen) closes apache/metron#997 +METRON-1511 Unable to Serialize Profiler Configuration (nickwallen) closes apache/metron#982 +METRON-1528: Fix missing file in metron.spec (mmiklavc via mmiklavc) closes apache/metron#996 +METRON-1445: Update performance tuning guide with more explicit parameter instructions (mmiklavc via mmiklavc) closes apache/metron#988 +METRON-1502 Upgrade Doxia plugin to 1.8 (justinleet) closes apache/metron#974 +METRON-1527: Remove dead test file sitting in source folder (mmiklavc via mmiklavc) closes apache/metron#994 +METRON-1499 Enable Configuration of Unified Enrichment Topology via Ambari (nickwallen) closes apache/metron#984 +METRON-1515: Errors loading stellar functions currently bomb the entire topology, they should be recoverable closes apache/incubator-metron#985 +METRON-1522 Fix the typo errors at profile debugger readme (MohanDV via nickwallen) closes apache/metron#992 +METRON-1519 Indexing Error Topic Property Not Displayed in MPack (nickwallen) closes apache/metron#987 +METRON-1347: Indexing Topology should fail tuples without a source.type
