Repository: metron Updated Branches: refs/heads/master 813adf2da -> 3f0e17cf9
METRON-1026: threatintel_taxii_load.sh throws exception closes apache/incubator-metron#643 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/3f0e17cf Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/3f0e17cf Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/3f0e17cf Branch: refs/heads/master Commit: 3f0e17cf98d31b1143ee4d365c87f0e850df252a Parents: 813adf2 Author: cstella <ceste...@gmail.com> Authored: Wed Aug 9 20:31:57 2017 -0400 Committer: cstella <ceste...@gmail.com> Committed: Wed Aug 9 20:31:57 2017 -0400 ---------------------------------------------------------------------- .../roles/opentaxii/templates/collections.yml | 1 + .../roles/opentaxii/templates/services.yml | 103 ++++++++++--------- .../dataloads/extractor/ExtractorDecorator.java | 4 + .../TransformFilterExtractorDecorator.java | 2 +- .../dataloads/nonbulk/taxii/TaxiiLoader.java | 16 ++- .../nonbulk/taxii/TaxiiIntegrationTest.java | 5 +- 6 files changed, 76 insertions(+), 55 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/3f0e17cf/metron-deployment/roles/opentaxii/templates/collections.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/opentaxii/templates/collections.yml b/metron-deployment/roles/opentaxii/templates/collections.yml index 986c673..07a0e5a 100644 --- a/metron-deployment/roles/opentaxii/templates/collections.yml +++ b/metron-deployment/roles/opentaxii/templates/collections.yml @@ -15,4 +15,5 @@ # limitations under the License. # --- +collections: # intentionally blank - managed by ansible http://git-wip-us.apache.org/repos/asf/metron/blob/3f0e17cf/metron-deployment/roles/opentaxii/templates/services.yml ---------------------------------------------------------------------- diff --git a/metron-deployment/roles/opentaxii/templates/services.yml b/metron-deployment/roles/opentaxii/templates/services.yml index 90c162e..61a1d42 100644 --- a/metron-deployment/roles/opentaxii/templates/services.yml +++ b/metron-deployment/roles/opentaxii/templates/services.yml @@ -15,54 +15,55 @@ # limitations under the License. # --- -# -# discovery: used by a TAXII Client to discover available TAXII Service -# -- id: discovery - type: discovery - address: /services/discovery - description: Discovery service for Apache Metron - advertised_services: - - inbox - - collection - - poll - protocol_bindings: - - urn:taxii.mitre.org:protocol:http:1.0 - -# -# inbox: used by a TAXII Client to push information to a TAXII Server -# -- id: inbox - type: inbox - address: /services/inbox - description: Inbox for Apache Metron - destination_collection_required: yes - accept_all_content: yes - supported_content: - - urn:stix.mitre.org:xml:1.1.1 - authentication_required: no - protocol_bindings: - - urn:taxii.mitre.org:protocol:http:1.0 - -# -# collection_management: used by a TAXII Client to request information about -# available data collections or request a subscription. -# -- id: collection - type: collection_management - address: /services/collection - description: Collection management service for Apache Metron - protocol_bindings: - - urn:taxii.mitre.org:protocol:http:1.0 - -# -# poll: used by a TAXII Client to request information from a TAXII Server -# -- id: poll - type: poll - address: /services/poll - description: Poll service for Apache Metron - subscription_required: no - authentication_required: no - protocol_bindings: - - urn:taxii.mitre.org:protocol:http:1.0 +services: + # + # discovery: used by a TAXII Client to discover available TAXII Service + # + - id: discovery + type: discovery + address: /services/discovery + description: Discovery service for Apache Metron + advertised_services: + - inbox + - collection + - poll + protocol_bindings: + - urn:taxii.mitre.org:protocol:http:1.0 + + # + # inbox: used by a TAXII Client to push information to a TAXII Server + # + - id: inbox + type: inbox + address: /services/inbox + description: Inbox for Apache Metron + destination_collection_required: yes + accept_all_content: yes + supported_content: + - urn:stix.mitre.org:xml:1.1.1 + authentication_required: no + protocol_bindings: + - urn:taxii.mitre.org:protocol:http:1.0 + + # + # collection_management: used by a TAXII Client to request information about + # available data collections or request a subscription. + # + - id: collection + type: collection_management + address: /services/collection + description: Collection management service for Apache Metron + protocol_bindings: + - urn:taxii.mitre.org:protocol:http:1.0 + + # + # poll: used by a TAXII Client to request information from a TAXII Server + # + - id: poll + type: poll + address: /services/poll + description: Poll service for Apache Metron + subscription_required: no + authentication_required: no + protocol_bindings: + - urn:taxii.mitre.org:protocol:http:1.0 http://git-wip-us.apache.org/repos/asf/metron/blob/3f0e17cf/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/ExtractorDecorator.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/ExtractorDecorator.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/ExtractorDecorator.java index bf42760..1e1c037 100644 --- a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/ExtractorDecorator.java +++ b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/ExtractorDecorator.java @@ -39,4 +39,8 @@ public class ExtractorDecorator implements Extractor { public void initialize(Map<String, Object> config) { decoratedExtractor.initialize(config); } + + public Extractor getUnderlyingExtractor() { + return decoratedExtractor; + } } http://git-wip-us.apache.org/repos/asf/metron/blob/3f0e17cf/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/TransformFilterExtractorDecorator.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/TransformFilterExtractorDecorator.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/TransformFilterExtractorDecorator.java index 2aa76b7..790ea9f 100644 --- a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/TransformFilterExtractorDecorator.java +++ b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/extractor/TransformFilterExtractorDecorator.java @@ -70,7 +70,7 @@ public class TransformFilterExtractorDecorator extends ExtractorDecorator { } public boolean existsIn(Map<String, Object> config) { - return config.containsKey(key); + return config == null?false:config.containsKey(key); } } http://git-wip-us.apache.org/repos/asf/metron/blob/3f0e17cf/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiLoader.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiLoader.java b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiLoader.java index 689a08f..de6ae8d 100644 --- a/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiLoader.java +++ b/metron-platform/metron-data-management/src/main/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiLoader.java @@ -29,7 +29,9 @@ import org.apache.hadoop.hbase.HConstants; import org.apache.hadoop.util.GenericOptionsParser; import org.apache.log4j.PropertyConfigurator; import org.apache.metron.dataloads.extractor.Extractor; +import org.apache.metron.dataloads.extractor.ExtractorDecorator; import org.apache.metron.dataloads.extractor.ExtractorHandler; +import org.apache.metron.dataloads.extractor.TransformFilterExtractorDecorator; import org.apache.metron.dataloads.extractor.stix.StixExtractor; import org.apache.metron.common.configuration.enrichment.SensorEnrichmentUpdateConfig; import org.apache.metron.common.utils.JSONUtils; @@ -165,6 +167,16 @@ public class TaxiiLoader { public static final long DEFAULT_TIME_BETWEEN_POLLS = ONE_HR_IN_MS; + public static boolean isStixExtractor(Extractor e) { + if(e instanceof StixExtractor || + ( e instanceof ExtractorDecorator && ((ExtractorDecorator) e).getUnderlyingExtractor() instanceof StixExtractor) + ) { + return true; + } else { + return false; + } + } + public static void main(String... argv) throws Exception { Configuration conf = HBaseConfiguration.create(); String zkQuorum = conf.get(HConstants.ZOOKEEPER_QUORUM); @@ -185,8 +197,8 @@ public class TaxiiLoader { } Timer timer = new Timer(); - if(e instanceof StixExtractor) { - StixExtractor extractor = (StixExtractor)e; + if(isStixExtractor(e)) { + Extractor extractor = e; TaxiiConnectionConfig connectionConfig = TaxiiConnectionConfig.load(FileUtils.readFileToString(new File(TaxiiOptions.CONNECTION_CONFIG.get(cli)))); if(TaxiiOptions.BEGIN_TIME.has(cli)) { Date d = DATE_FORMAT.parse(TaxiiOptions.BEGIN_TIME.get(cli)); http://git-wip-us.apache.org/repos/asf/metron/blob/3f0e17cf/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiIntegrationTest.java b/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiIntegrationTest.java index f94a02a..a54c21b 100644 --- a/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiIntegrationTest.java +++ b/metron-platform/metron-data-management/src/test/java/org/apache/metron/dataloads/nonbulk/taxii/TaxiiIntegrationTest.java @@ -27,6 +27,8 @@ import org.apache.hadoop.hbase.HBaseConfiguration; import org.apache.hadoop.hbase.client.HTableInterface; import org.apache.hadoop.hbase.client.Put; import org.apache.hadoop.util.GenericOptionsParser; +import org.apache.metron.dataloads.extractor.Extractor; +import org.apache.metron.dataloads.extractor.TransformFilterExtractorDecorator; import org.apache.metron.dataloads.extractor.stix.StixExtractor; import org.apache.metron.enrichment.converter.EnrichmentConverter; import org.apache.metron.enrichment.converter.EnrichmentKey; @@ -94,7 +96,8 @@ public class TaxiiIntegrationTest { final MockHBaseTableProvider provider = new MockHBaseTableProvider(); final Configuration config = HBaseConfiguration.create(); - TaxiiHandler handler = new TaxiiHandler(TaxiiConnectionConfig.load(taxiiConnectionConfig), new StixExtractor(), config ) { + Extractor extractor = new TransformFilterExtractorDecorator(new StixExtractor()); + TaxiiHandler handler = new TaxiiHandler(TaxiiConnectionConfig.load(taxiiConnectionConfig), extractor, config ) { @Override protected synchronized HTableInterface createHTable(String tableInfo) throws IOException { return provider.addToCache("threat_intel", "cf");
