Repository: metron Updated Branches: refs/heads/master b3148a182 -> 3d95fdf90
METRON-1102: Add support for ingesting cybox URI observables from taxii feeds closes apache/incubator-metron#689 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/3d95fdf9 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/3d95fdf9 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/3d95fdf9 Branch: refs/heads/master Commit: 3d95fdf906993c8d65b76a7e1311c7fb295e19b1 Parents: b3148a1 Author: cstella <ceste...@gmail.com> Authored: Fri Aug 11 13:29:07 2017 -0400 Committer: cstella <ceste...@gmail.com> Committed: Fri Aug 11 13:29:07 2017 -0400 ---------------------------------------------------------------------- .../metron-data-management/README.md | 6 ++ .../dataloads/extractor/stix/StixExtractor.java | 15 +++ .../stix/types/ObjectTypeHandlers.java | 1 + .../extractor/stix/types/URIHandler.java | 68 ++++++++++++++ .../extractor/stix/URIHandlerTest.java | 97 ++++++++++++++++++++ 5 files changed, 187 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/3d95fdf9/metron-platform/metron-data-management/README.md ---------------------------------------------------------------------- diff --git a/metron-platform/metron-data-management/README.md b/metron-platform/metron-data-management/README.md index 014f3ef..c351f48 100644 --- a/metron-platform/metron-data-management/README.md +++ b/metron-platform/metron-data-management/README.md @@ -97,6 +97,11 @@ Consider the following config for importing STIX documents. This is a threat in format, so it is particularly relevant and attractive data to import for our purposes. Because STIX is a standard format, there is no need to specify the schema or how to interpret the documents. +We support the versions of Stix and Cybox supported by +[java-stix](https://github.com/STIXProject/java-stix/tree/v1.2.0.2): +* Stix - [1.2](https://github.com/STIXProject/schemas/blob/356cc4f6b06625465f0808388eb166807313b4e0/stix_core.xsd) and earlier +* Cybox - [2.1](https://github.com/CybOXProject/schemas/blob/97beb32c376a9223e91b52cb3e4c8d2af6baf786/cybox_core.xsd) and earlier + We support a subset of STIX messages for importation: | STIX Type | Specific Type | Enrichment Type Name | @@ -107,6 +112,7 @@ We support a subset of STIX messages for importation: | Address | MAC | address:MAC | | Domain | FQDN | domain:FQDN | | Hostname | | hostname | +| URI | | uriobjecttype | NOTE: The enrichment type will be used as the type above. http://git-wip-us.apache.org/repos/asf/metron/blob/3d95fdf9/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/stix/StixExtractor.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/stix/StixExtractor.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/stix/StixExtractor.java index 41b428e..f96bd2d 100644 --- a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/stix/StixExtractor.java +++ b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/stix/StixExtractor.java @@ -30,14 +30,18 @@ import org.mitre.cybox.cybox_2.Observables; import org.mitre.stix.common_1.IndicatorBaseType; import org.mitre.stix.indicator_2.Indicator; import org.mitre.stix.stix_1.STIXPackage; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import java.io.File; import java.io.IOException; +import java.lang.invoke.MethodHandles; import java.util.ArrayList; import java.util.List; import java.util.Map; public class StixExtractor implements Extractor { + private static final Logger LOG = LoggerFactory.getLogger(MethodHandles.lookup().lookupClass()); Map<String, Object> config; @Override public Iterable<LookupKV> extract(String line) throws IOException { @@ -50,11 +54,22 @@ public class StixExtractor implements Extractor { if(props != null) { ObjectTypeHandler handler = ObjectTypeHandlers.getHandlerByInstance(props); if (handler != null) { + if(LOG.isDebugEnabled()) { + LOG.debug("Found {} for properties {}" + , handler.getTypeClass().getCanonicalName() + , props.toXMLString()); + } Iterable<LookupKV> extractions = handler.extract(props, config); for(LookupKV extraction : extractions) { ret.add(extraction); } } + else if(LOG.isDebugEnabled()) { + LOG.debug("Did not find a handler for properties {} of type {}" + , props.toXMLString() + , props.getClass() + ); + } } } } http://git-wip-us.apache.org/repos/asf/metron/blob/3d95fdf9/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/stix/types/ObjectTypeHandlers.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/stix/types/ObjectTypeHandlers.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/stix/types/ObjectTypeHandlers.java index 06d8cd8..df2c915 100644 --- a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/stix/types/ObjectTypeHandlers.java +++ b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/stix/types/ObjectTypeHandlers.java @@ -23,6 +23,7 @@ public enum ObjectTypeHandlers { ADDRESS(new AddressHandler()) ,HOSTNAME(new HostnameHandler()) ,DOMAINNAME(new DomainHandler()) + ,URI(new URIHandler()) ,; ObjectTypeHandler _handler; ObjectTypeHandlers(ObjectTypeHandler handler) { http://git-wip-us.apache.org/repos/asf/metron/blob/3d95fdf9/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/stix/types/URIHandler.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/stix/types/URIHandler.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/stix/types/URIHandler.java new file mode 100644 index 0000000..2adfb8a --- /dev/null +++ b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/stix/types/URIHandler.java @@ -0,0 +1,68 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.dataloads.extractor.stix.types; + +import com.google.common.collect.ImmutableList; +import org.apache.metron.enrichment.converter.EnrichmentKey; +import org.apache.metron.enrichment.converter.EnrichmentValue; +import org.apache.metron.enrichment.lookup.LookupKV; +import org.mitre.cybox.common_2.AnyURIObjectPropertyType; +import org.mitre.cybox.objects.URIObjectType; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class URIHandler extends AbstractObjectTypeHandler<URIObjectType> { + public URIHandler() { + super(URIObjectType.class); + } + + @Override + public Iterable<LookupKV> extract(URIObjectType type, Map<String, Object> config) throws IOException { + List<LookupKV> ret = new ArrayList<>(); + if(type != null) { + AnyURIObjectPropertyType val = type.getValue(); + if(val != null) { + Object v = val.getValue(); + if(v != null) { + final String indicatorType = getType(); + LookupKV results = new LookupKV(new EnrichmentKey(indicatorType, v.toString()) + , new EnrichmentValue( + new HashMap<String, Object>() {{ + put("source-type", "STIX"); + put("uri", v.toString()); + put("indicator-type", indicatorType); + put("source", type.toXMLString()); + }} + ) + ); + ret.add(results); + } + } + } + return ret; + } + + @Override + public List<String> getPossibleTypes() { + return ImmutableList.of(getType()); + } +} http://git-wip-us.apache.org/repos/asf/metron/blob/3d95fdf9/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/extractor/stix/URIHandlerTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/extractor/stix/URIHandlerTest.java b/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/extractor/stix/URIHandlerTest.java new file mode 100644 index 0000000..ff41152 --- /dev/null +++ b/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/extractor/stix/URIHandlerTest.java @@ -0,0 +1,97 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.dataloads.extractor.stix; + +import com.google.common.collect.Iterables; +import org.adrianwalker.multilinestring.Multiline; +import org.apache.metron.dataloads.extractor.stix.types.URIHandler; +import org.apache.metron.enrichment.converter.EnrichmentKey; +import org.apache.metron.enrichment.lookup.LookupKV; +import org.junit.Assert; +import org.junit.Test; +import org.mitre.cybox.objects.URIObjectType; + +import java.util.HashMap; +import java.util.List; + +public class URIHandlerTest { + + /** + *<?xml version="1.0" encoding="UTF-8"?> + *<stix:STIX_Package xmlns:stix="http://stix.mitre.org/stix-1"; + * xmlns:tdq="http://taxii.mitre.org/query/taxii_default_query-1"; + * xmlns:TOUMarking="http://data-marking.mitre.org/extensions/MarkingStructure#Terms_Of_Use-1"; + * xmlns:stixVocabs="http://stix.mitre.org/default_vocabularies-1"; + * xmlns:cyboxCommon="http://cybox.mitre.org/common-2"; + * xmlns:simpleMarking="http://data-marking.mitre.org/extensions/MarkingStructure#Simple-1"; + * xmlns:cyboxVocabs="http://cybox.mitre.org/default_vocabularies-2"; + * xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + * xmlns:stixCommon="http://stix.mitre.org/common-1"; + * xmlns:edge="http://soltra.com/"; + * xmlns:marking="http://data-marking.mitre.org/Marking-1"; + * xmlns:tlpMarking="http://data-marking.mitre.org/extensions/MarkingStructure#TLP-1"; + * xmlns:taxii_11="http://taxii.mitre.org/messages/taxii_xml_binding-1.1"; + * xmlns:opensource="http://hailataxii.com"; + * xmlns:URIObj="http://cybox.mitre.org/objects#URIObject-2"; + * xmlns:taxii="http://taxii.mitre.org/messages/taxii_xml_binding-1"; + * xmlns:cybox="http://cybox.mitre.org/cybox-2"; + * id="edge:Package-208ba7e1-ecc1-49a1-a96d-f28c4146761d" + * timestamp="2017-08-09T21:05:27.148461+00:00" + * version="1.1.1"> + * <stix:STIX_Header> + * <stix:Handling> + * <marking:Marking> + * <marking:Controlled_Structure>../../../../descendant-or-self::node()</marking:Controlled_Structure> + * <marking:Marking_Structure color="WHITE" xsi:type="tlpMarking:TLPMarkingStructureType"/> + * <marking:Marking_Structure xsi:type="TOUMarking:TermsOfUseMarkingStructureType"> + * <TOUMarking:Terms_Of_Use>TBD</TOUMarking:Terms_Of_Use> + * </marking:Marking_Structure> + * <marking:Marking_Structure xsi:type="simpleMarking:SimpleMarkingStructureType"> + * <simpleMarking:Statement>Unclassified (Public)</simpleMarking:Statement> + * </marking:Marking_Structure> + * </marking:Marking> + * </stix:Handling> + * </stix:STIX_Header> + * <stix:Observables cybox_major_version="2" cybox_minor_version="1" cybox_update_version="0"> + * <cybox:Observable id="opensource:Observable-6b98960f-c8bb-45fd-8b6d-8960e803b51f" sighting_count="1"> + * <cybox:Title>URL: http://www.kotimi.com/alpha/gtex/...</cybox:Title> + * <cybox:Description>URL: http://www.kotimi.com/alpha/gtex/| isOnline:yes| dateVerified:2017-07-31T22:03:10+00:00</cybox:Description> + * <cybox:Object id="opensource:URI-9baf3b48-4aa2-4198-92b7-b5cb0a0a1d35"> + * <cybox:Properties type="URL" xsi:type="URIObj:URIObjectType"> + * <URIObj:Value condition="Equals">http://www.kotimi.com/alpha/gtex/</URIObj:Value> + * </cybox:Properties> + * </cybox:Object> + * </cybox:Observable> + * </stix:Observables> + *</stix:STIX_Package> + */ + @Multiline + static String uriHandlerObject; + + @Test + public void testURIHandler() throws Exception { + StixExtractor extractor = new StixExtractor(); + extractor.initialize(new HashMap<>()); + Iterable<LookupKV> kvs = extractor.extract(uriHandlerObject); + Assert.assertEquals(1, Iterables.size(kvs)); + LookupKV kv = Iterables.getFirst(kvs, null); + EnrichmentKey key = (EnrichmentKey) kv.getKey(); + Assert.assertEquals("http://www.kotimi.com/alpha/gtex/";, key.getIndicator()); + Assert.assertEquals("uriobjecttype", key.type); + } +}
