[
https://issues.apache.org/jira/browse/METRON-623?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=15941281#comment-15941281
]
ASF GitHub Bot commented on METRON-623:
---------------------------------------
Github user simonellistonball commented on a diff in the pull request:
https://github.com/apache/incubator-metron/pull/489#discussion_r108012411
--- Diff:
metron-interface/metron-config/src/app/sensors/sensor-field-schema/sensor-field-schema.component.spec.ts
---
@@ -0,0 +1,506 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+/* tslint:disable:no-unused-variable */
+/* tslint:disable:max-line-length */
+
+import { TestBed, async, ComponentFixture } from '@angular/core/testing';
+import {Http} from '@angular/http';
+import {SimpleChanges, SimpleChange} from '@angular/core';
+import {SensorParserConfigService} from
'../../service/sensor-parser-config.service';
+import {StellarService} from '../../service/stellar.service';
+import {MetronAlerts} from '../../shared/metron-alerts';
+import {SensorFieldSchemaModule} from './sensor-field-schema.module';
+import {SensorFieldSchemaComponent, FieldSchemaRow} from
'./sensor-field-schema.component';
+import {KafkaService} from '../../service/kafka.service';
+import {Observable} from 'rxjs/Observable';
+import {StellarFunctionDescription} from
'../../model/stellar-function-description';
+import {SensorParserConfig} from '../../model/sensor-parser-config';
+import {SensorEnrichmentConfig, EnrichmentConfig, ThreatIntelConfig} from
'../../model/sensor-enrichment-config';
+import {ParseMessageRequest} from '../../model/parse-message-request';
+import {AutocompleteOption} from '../../model/autocomplete-option';
+import {FieldTransformer} from '../../model/field-transformer';
+import {SensorEnrichmentConfigService} from
'../../service/sensor-enrichment-config.service';
+
+
+class MockSensorParserConfigService {
+
+ parseMessage(parseMessageRequest: ParseMessageRequest): Observable<{}>
{
+ let parsedJson = {
+ 'elapsed': 415,
+ 'code': 200,
+ 'ip_dst_addr': '207.109.73.154',
+ 'original_string': '1467011157.401 415 127.0.0.1 TCP_MISS/200
337891 GET http://www.aliexpress.com/',
+ 'method': 'GET',
+ 'bytes': 337891,
+ 'action': 'TCP_MISS',
+ 'ip_src_addr': '127.0.0.1',
+ 'url': 'http://www.aliexpress.com/af/shoes.html?',
+ 'timestamp': '1467011157.401'
+ };
+ return Observable.create((observable) => {
+ observable.next(parsedJson);
+ observable.complete();
+ });
+ }
+}
+
+class MockTransformationValidationService {
+ public listSimpleFunctions(): Observable<StellarFunctionDescription[]>
{
+ let stellarFunctionDescription: StellarFunctionDescription[] = [];
+ stellarFunctionDescription.push(new
StellarFunctionDescription('TO_LOWER', 'TO_LOWER description', ['input - input
field']));
+ stellarFunctionDescription.push(new
StellarFunctionDescription('TO_UPPER', 'TO_UPPER description', ['input - input
field']));
+ stellarFunctionDescription.push(new
StellarFunctionDescription('TRIM', 'Lazy to copy desc', ['input - input
field']));
+ return Observable.create((observer) => {
+ observer.next(stellarFunctionDescription);
+ observer.complete();
+ });
+ }
+}
+
+class MockSensorEnrichmentConfigService {
+ public getAvailable(): Observable<string[]> {
+ return Observable.create((observer) => {
+ observer.next(['geo', 'host', 'whois']);
+ observer.complete();
+ });
+ }
+}
+
+class MockKafkaService {
+
+}
+
+describe('Component: SensorFieldSchema', () => {
+ let component: SensorFieldSchemaComponent;
+ let sensorEnrichmentConfigService: SensorEnrichmentConfigService;
+ let sensorParserConfigService: SensorParserConfigService;
+ let fixture: ComponentFixture<SensorFieldSchemaComponent>;
+ let transformationValidationService: StellarService;
+
+ let squidSensorConfigJson = {
+ 'parserClassName': 'org.apache.metron.parsers.GrokParser',
+ 'sensorTopic': 'squid',
+ 'parserConfig': {
+ 'grokPath': 'target/patterns/squid',
+ 'grokStatement': '%{NUMBER:timestamp} %{INT:elapsed}
%{IPV4:ip_src_addr} %{WORD:action}/%{NUMBER:code} ' +
+ '%{NUMBER:bytes} %{WORD:method}
%{NOTSPACE:url} - %{WORD:UNWANTED}\\/%{IPV4:ip_dst_addr} ' +
+ '%{WORD:UNWANTED}\\/%{WORD:UNWANTED}'
+ },
+ 'fieldTransformations': [
+ {
+ 'input': [],
+ 'output': ['method'],
+ 'transformation': 'STELLAR',
+ 'config': {
+ 'method': 'TRIM(TO_LOWER(method))'
+ }
+ },
+ {
+ 'input': ['method'],
+ 'output': null,
+ 'transformation': 'REMOVE',
+ 'config': {
+ 'method': 'TRIM(TO_LOWER(method))'
+ }
+ },
+ {
+ 'input': ['code'],
+ 'output': null,
+ 'transformation': 'REMOVE',
+ 'config': {}
+ }
+ ]
+ };
+ let squidEnrichmentJson = {
+ 'index': 'squid',
+ 'batchSize': 1,
+ 'enrichment': {
+ 'fieldMap': {
+ 'geo': ['ip_dst_addr', 'ip_src_addr'],
+ 'host': ['ip_dst_addr'],
+ 'whois': ['ip_src_addr']
+ },
+ 'fieldToTypeMap': {},
+ 'config': {}
+ },
+ 'threatIntel': {
+ 'fieldMap': {
+ 'hbaseThreatIntel': ['ip_dst_addr']
+ },
+ 'fieldToTypeMap': {
+ 'ip_dst_addr': ['malicious_ip']
+ },
+ 'config': {},
+ 'triageConfig': {
+ 'riskLevelRules': {},
+ 'aggregator': 'MAX',
+ 'aggregationConfig': {}
+ }
+ },
+ 'configuration': {}
+ };
+ let sensorParserConfig = Object.assign(new SensorParserConfig(),
squidSensorConfigJson);
+ let sensorEnrichmentConfig = Object.assign(new
SensorEnrichmentConfig(), squidEnrichmentJson);
+
+ beforeEach(async(() => {
+ TestBed.configureTestingModule({
+ imports: [SensorFieldSchemaModule],
+ providers: [
+ MetronAlerts,
+ {provide: Http},
+ {provide: KafkaService, useClass: MockKafkaService},
+ {provide: SensorEnrichmentConfigService, useClass:
MockSensorEnrichmentConfigService},
+ {provide: SensorParserConfigService, useClass:
MockSensorParserConfigService},
+ {provide: StellarService, useClass:
MockTransformationValidationService},
+
+ ]
+ }).compileComponents()
+ .then(() => {
+ fixture = TestBed.createComponent(SensorFieldSchemaComponent);
+ component = fixture.componentInstance;
+ sensorParserConfigService =
fixture.debugElement.injector.get(SensorParserConfigService);
+ transformationValidationService =
fixture.debugElement.injector.get(StellarService);
+ sensorEnrichmentConfigService =
fixture.debugElement.injector.get(SensorEnrichmentConfigService);
+ });
+ }));
+
+ it('should create an instance', () => {
+ expect(component).toBeDefined();
+ fixture.destroy();
+ });
+
+
+
+ it('should read TransformFunctions, EnrichmentFunctions,
ThreatIntelfunctions', () => {
+ component.ngOnInit();
+
+ expect(component.transformOptions.length).toEqual(3);
+
expect(Object.keys(component.transformFunctions).length).toEqual(3);
+ expect(component.enrichmentOptions.length).toEqual(3);
+ expect(component.threatIntelOptions.length).toEqual(1);
+
+ fixture.destroy();
+ });
+
+ it('should call getSampleData if showFieldSchema', () => {
+ spyOn(component.sampleData, 'getNextSample');
+
+ let changes: SimpleChanges = {
+ 'showFieldSchema': new SimpleChange(false, true)
+ };
+ component.ngOnChanges(changes);
+
expect(component.sampleData.getNextSample['calls'].count()).toEqual(1);
+
+ changes = {
+ 'showFieldSchema': new SimpleChange(true, false)
+ };
+ component.ngOnChanges(changes);
+
expect(component.sampleData.getNextSample['calls'].count()).toEqual(1);
+
+ fixture.destroy();
+ });
+
+ it('should return isSimple function', () => {
+ component.ngOnInit();
+
+ expect(component.isSimpleFunction(['TO_LOWER',
'TO_UPPER'])).toEqual(true);
+ expect(component.isSimpleFunction(['TO_LOWER', 'TO_UPPER',
'TEST'])).toEqual(false);
+
+ fixture.destroy();
+ });
+
+ it('should create FieldSchemaRows', () => {
+ component.ngOnInit();
+
+ component.sensorParserConfig = sensorParserConfig;
+ component.sensorEnrichmentConfig = sensorEnrichmentConfig;
+ component.onSampleDataChanged('DoctorStrange');
+ component.createFieldSchemaRows();
+
+ expect(component.fieldSchemaRows.length).toEqual(10);
+ expect(component.savedFieldSchemaRows.length).toEqual(10);
+
+ let methodFieldSchemaRow: FieldSchemaRow =
component.fieldSchemaRows.filter(row => row.inputFieldName === 'method')[0];
+ expect(methodFieldSchemaRow).toBeDefined();
+ expect(methodFieldSchemaRow.transformConfigured.length).toEqual(2);
+
expect(methodFieldSchemaRow.enrichmentConfigured.length).toEqual(0);
+
expect(methodFieldSchemaRow.threatIntelConfigured.length).toEqual(0);
+
+ let ipSrcAddrFieldSchemaRow: FieldSchemaRow =
component.fieldSchemaRows.filter(row => row.inputFieldName ===
'ip_src_addr')[0];
+ expect(ipSrcAddrFieldSchemaRow).toBeDefined();
+
expect(ipSrcAddrFieldSchemaRow.transformConfigured.length).toEqual(0);
+
expect(ipSrcAddrFieldSchemaRow.enrichmentConfigured.length).toEqual(2);
+
expect(ipSrcAddrFieldSchemaRow.threatIntelConfigured.length).toEqual(0);
+
+ let ipDstAddrFieldSchemaRow: FieldSchemaRow =
component.fieldSchemaRows.filter(row => row.inputFieldName ===
'ip_dst_addr')[0];
+ expect(ipDstAddrFieldSchemaRow).toBeDefined();
+
expect(ipDstAddrFieldSchemaRow.transformConfigured.length).toEqual(0);
+
expect(ipDstAddrFieldSchemaRow.enrichmentConfigured.length).toEqual(2);
+
expect(ipDstAddrFieldSchemaRow.threatIntelConfigured.length).toEqual(1);
+
+ let codeSchemaRow: FieldSchemaRow =
component.fieldSchemaRows.filter(row => row.inputFieldName === 'code')[0];
+ expect(codeSchemaRow).toBeDefined();
+ expect(codeSchemaRow.isRemoved).toEqual(true);
+ expect(codeSchemaRow.conditionalRemove).toEqual(false);
+ expect(codeSchemaRow.transformConfigured.length).toEqual(0);
+ expect(codeSchemaRow.enrichmentConfigured.length).toEqual(0);
+ expect(codeSchemaRow.threatIntelConfigured.length).toEqual(0);
+
+ fixture.destroy();
+ });
+
+ it('should return getChanges', () => {
+ let fieldSchemaRow = new FieldSchemaRow('method');
+ fieldSchemaRow.transformConfigured = [];
+ fieldSchemaRow.enrichmentConfigured = [new
AutocompleteOption('GEO'), new AutocompleteOption('WHOIS')];
+ fieldSchemaRow.threatIntelConfigured = [new
AutocompleteOption('MALICIOUS-IP')];
+
+ expect(component.getChanges(fieldSchemaRow)).toEqual('Enrichments:
GEO, WHOIS <br> Threat Intel: MALICIOUS-IP');
+
+ fieldSchemaRow.transformConfigured = [new
AutocompleteOption('TO_STRING')];
+ fieldSchemaRow.enrichmentConfigured = [new
AutocompleteOption('GEO')];
+ fieldSchemaRow.threatIntelConfigured = [new
AutocompleteOption('MALICIOUS-IP'), new AutocompleteOption('MALICIOUS-IP')];
+
+ expect(component.getChanges(fieldSchemaRow)).toEqual('Transforms:
TO_STRING(method) <br> Enrichments: GEO <br> Threat Intel: MALICIOUS-IP,
MALICIOUS-IP');
+
+
+ fieldSchemaRow.transformConfigured = [new
AutocompleteOption('TO_STRING'), new AutocompleteOption('TO_STRING')];
+ fieldSchemaRow.enrichmentConfigured = [];
+ fieldSchemaRow.threatIntelConfigured = [new
AutocompleteOption('MALICIOUS-IP'), new AutocompleteOption('MALICIOUS-IP')];
+
+ expect(component.getChanges(fieldSchemaRow)).toEqual('Transforms:
TO_STRING(TO_STRING(method)) <br> Threat Intel: MALICIOUS-IP, MALICIOUS-IP');
+
+ fieldSchemaRow.transformConfigured = [new
AutocompleteOption('TO_STRING'), new AutocompleteOption('TO_STRING')];
+ fieldSchemaRow.enrichmentConfigured = [];
+ fieldSchemaRow.threatIntelConfigured = [];
+
+ expect(component.getChanges(fieldSchemaRow)).toEqual('Transforms:
TO_STRING(TO_STRING(method)) <br> ');
+
+ fieldSchemaRow.transformConfigured = [new
AutocompleteOption('TO_STRING'), new AutocompleteOption('TO_STRING')];
+ fieldSchemaRow.isRemoved = true;
+ expect(component.getChanges(fieldSchemaRow)).toEqual('Disabled');
+
+ fixture.destroy();
+ });
+
+ it('should call appropriate functions when onSampleDataChanged is
called ', () => {
+ let returnSuccess = true;
+ spyOn(component, 'createFieldSchemaRows');
+ spyOn(component, 'onSampleDataNotAvailable');
+ spyOn(sensorParserConfigService,
'parseMessage').and.callFake(function(parseMessageRequest: ParseMessageRequest)
{
+
expect(parseMessageRequest.sensorParserConfig.parserConfig['patternLabel']).toEqual(parseMessageRequest.sensorParserConfig.sensorTopic.toUpperCase());
+
expect(parseMessageRequest.sensorParserConfig.parserConfig['grokPath']).toEqual('./'
+ parseMessageRequest.sensorParserConfig.sensorTopic);
+ if (returnSuccess) {
+ return Observable.create(observer => {
+ observer.next({'a': 'b', 'c': 'd'});
+ observer.complete();
+ });
+ }
+ return Observable.throw('Error');
+ });
+
+ component.sensorParserConfig = sensorParserConfig;
+ component.sensorParserConfig.parserConfig['patternLabel'] = null;
+ component.onSampleDataChanged('DoctorStrange');
+ expect(component.parserResult).toEqual({'a': 'b', 'c': 'd'});
+ expect(component.createFieldSchemaRows).toHaveBeenCalled();
+ expect(component.onSampleDataNotAvailable).not.toHaveBeenCalled();
+
+ returnSuccess = false;
+ component.parserResult = {};
+ component.onSampleDataChanged('DoctorStrange');
+ expect(component.parserResult).toEqual({});
+ expect(component.onSampleDataNotAvailable).toHaveBeenCalled();
+
expect(component.onSampleDataNotAvailable['calls'].count()).toEqual(1);
+
+ fixture.destroy();
+ });
+
+ it('should onSampleDataChanged available and onSampleDataNotAvailable
', () => {
+ let returnSuccess = true;
+ spyOn(component, 'createFieldSchemaRows');
+
+ component.onSampleDataNotAvailable();
+
expect(component.createFieldSchemaRows['calls'].count()).toEqual(1);
+
+ fixture.destroy();
+ });
+
+ it('should call onSaveChange on onRemove/onEnable ', () => {
+ spyOn(component, 'onSave');
+
+ let fieldSchemaRow = new FieldSchemaRow('method');
+ fieldSchemaRow.outputFieldName = 'copy-of-method';
+ fieldSchemaRow.preview = 'TRIM(TO_LOWER(method))';
+ fieldSchemaRow.isRemoved = false;
+
+ component.savedFieldSchemaRows = [fieldSchemaRow];
+
+ let removeFieldSchemaRow =
JSON.parse(JSON.stringify(fieldSchemaRow));
+ component.onRemove(removeFieldSchemaRow);
+ expect(removeFieldSchemaRow.isRemoved).toEqual(true);
+ expect(component.savedFieldSchemaRows[0].isRemoved).toEqual(true);
+ expect(component.onSave['calls'].count()).toEqual(1);
+
+ fieldSchemaRow.isRemoved = true;
+ let enableFieldSchemaRow =
JSON.parse(JSON.stringify(fieldSchemaRow));
+ component.onEnable(enableFieldSchemaRow);
+ expect(fieldSchemaRow.isRemoved).toEqual(false);
+ expect(component.savedFieldSchemaRows[0].isRemoved).toEqual(false);
+ expect(component.onSave['calls'].count()).toEqual(2);
+
+ fixture.destroy();
+ });
+
+ it('should revert changes on cancel ', () => {
+ let fieldSchemaRow = new FieldSchemaRow('method');
+ fieldSchemaRow.showConfig = true;
+ fieldSchemaRow.outputFieldName = 'method';
+ fieldSchemaRow.preview = 'TRIM(TO_LOWER(method))';
+ fieldSchemaRow.isRemoved = false;
+ fieldSchemaRow.isSimple = true;
+ fieldSchemaRow.transformConfigured = [new
AutocompleteOption('TO_LOWER'), new AutocompleteOption('TRIM')];
+
+ component.savedFieldSchemaRows.push(fieldSchemaRow);
+
+ component.onCancelChange(fieldSchemaRow);
+ expect(fieldSchemaRow.showConfig).toEqual(false);
+
+ component.hideFieldSchema.emit = jasmine.createSpy('emit');
+ component.onCancel();
+ expect(component.hideFieldSchema.emit).toHaveBeenCalled();
+
+ fixture.destroy();
+ });
+
+ it('should return formatted function on createTransformFunction call
', () => {
+ let fieldSchemaRow = new FieldSchemaRow('method');
+ fieldSchemaRow.transformConfigured = [new
AutocompleteOption('TRIM'), new AutocompleteOption('TO_STRING')];
+
+
expect(component.createTransformFunction(fieldSchemaRow)).toEqual('TO_STRING(TRIM(method))');
+
+ fixture.destroy();
+ });
+
+ it('should set preview value for FieldSchemaRow ', () => {
+ let fieldSchemaRow = new FieldSchemaRow('method');
+ fieldSchemaRow.transformConfigured = [new
AutocompleteOption('TRIM'), new AutocompleteOption('TO_STRING')];
+
+ component.onTransformsChange(fieldSchemaRow);
+ expect(fieldSchemaRow.preview).toEqual('TO_STRING(TRIM(method))');
+
+ fieldSchemaRow.transformConfigured = [new
AutocompleteOption('TRIM')];
+ component.onTransformsChange(fieldSchemaRow);
+ expect(fieldSchemaRow.preview).toEqual('TRIM(method)');
+
+ fieldSchemaRow.transformConfigured = [];
+ component.onTransformsChange(fieldSchemaRow);
+ expect(fieldSchemaRow.preview).toEqual('');
+
+ fixture.destroy();
+ });
+
+ it('isSimpleRemoveTransform ', () => {
+ let fieldTransformationJson = {
+ 'output': ['method', 'elapsed'],
+ 'transformation': 'STELLAR',
+ 'config':
+ {
+ 'method': 'TRIM(TO_LOWER(method))',
+ 'elapsed': 'IS_DOMAIN(elapsed)'
+ }
+ };
+ let fieldTransformation: FieldTransformer = Object.assign(new
FieldTransformer(), fieldTransformationJson);
+
expect(component.isSimpleRemoveTransform(fieldTransformation)).toEqual(true);
--- End diff --
Not sure why this is the case.
> Management UI
> -------------
>
> Key: METRON-623
> URL: https://issues.apache.org/jira/browse/METRON-623
> Project: Metron
> Issue Type: New Feature
> Reporter: Ryan Merriman
> Assignee: Ryan Merriman
>
> It would be useful to have a User Interface for modifying parser/enrichment
> configurations and managing topologies.
--
This message was sent by Atlassian JIRA
(v6.3.15#6346)
