http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-interface/metron-config/src/app/shared/sample-data/sample-data.component.spec.ts ---------------------------------------------------------------------- diff --git a/metron-interface/metron-config/src/app/shared/sample-data/sample-data.component.spec.ts b/metron-interface/metron-config/src/app/shared/sample-data/sample-data.component.spec.ts index 5488209..b8cdf1f 100644 --- a/metron-interface/metron-config/src/app/shared/sample-data/sample-data.component.spec.ts +++ b/metron-interface/metron-config/src/app/shared/sample-data/sample-data.component.spec.ts @@ -15,17 +15,15 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import {async, TestBed, ComponentFixture} from '@angular/core/testing'; -import {KafkaService} from '../../service/kafka.service'; -import {Observable} from 'rxjs/Observable'; -import {SampleDataComponent} from './sample-data.component'; -import {SharedModule} from '../shared.module'; -import '../../rxjs-operators'; +import { async, TestBed, ComponentFixture } from '@angular/core/testing'; +import { KafkaService } from '../../service/kafka.service'; +import { Observable, throwError } from 'rxjs'; +import { SampleDataComponent } from './sample-data.component'; +import { SharedModule } from '../shared.module'; class MockKafkaService { _sample: string[]; - _sampleCounter: number = 0; - + _sampleCounter = 0; public setSample(sampleMessages: string[]): void { this._sample = sampleMessages; @@ -33,7 +31,6 @@ class MockKafkaService { } public sample(name: string): Observable<string> { - if (this._sampleCounter < this._sample.length) { return Observable.create(observer => { observer.next(this._sample[this._sampleCounter++]); @@ -41,7 +38,7 @@ class MockKafkaService { }); } - return Observable.throw('Error'); + return throwError('Error'); } } @@ -58,24 +55,21 @@ describe('SampleDataComponent', () => { beforeEach(async(() => { TestBed.configureTestingModule({ imports: [SharedModule], - declarations: [ SampleDataComponent], + declarations: [SampleDataComponent], providers: [ SampleDataComponent, - {provide: KafkaService, useClass: MockKafkaService} + { provide: KafkaService, useClass: MockKafkaService } ] }); - fixture = TestBed.createComponent(SampleDataComponent); sampleDataComponent = fixture.componentInstance; - kafkaService = fixture.debugElement.injector.get(KafkaService); - + kafkaService = TestBed.get(KafkaService); })); it('can instantiate SampleDataComponent', async(() => { expect(sampleDataComponent instanceof SampleDataComponent).toBe(true); })); - it('should emmit messages', async(() => { let expectedMessage; let successCount = 0; @@ -140,11 +134,9 @@ describe('SampleDataComponent', () => { sampleDataComponent.getPreviousSample(); expect(successCount).toEqual(7); expect(failureCount).toEqual(1); - })); it('should emmit messages on blur', async(() => { - let expectedMessage; let successCount = 0; @@ -155,9 +147,10 @@ describe('SampleDataComponent', () => { expect(message).toEqual(expectedMessage); }); - expectedMessage = 'This is a simple message'; - fixture.debugElement.nativeElement.querySelector('textarea').value = expectedMessage; + fixture.debugElement.nativeElement.querySelector( + 'textarea' + ).value = expectedMessage; sampleDataComponent.onBlur(); expect(successCount).toEqual(1); @@ -165,16 +158,16 @@ describe('SampleDataComponent', () => { expect(sampleDataComponent.sampleData.length).toEqual(1); expect(sampleDataComponent.sampleData[0]).toEqual(expectedMessage); - expectedMessage = ''; - fixture.debugElement.nativeElement.querySelector('textarea').value = expectedMessage; + fixture.debugElement.nativeElement.querySelector( + 'textarea' + ).value = expectedMessage; sampleDataComponent.onBlur(); expect(successCount).toEqual(2); expect(sampleDataComponent.sampleDataIndex).toEqual(0); expect(sampleDataComponent.sampleData.length).toEqual(1); - expectedMessage = sampleMessages[0]; sampleDataComponent.getNextSample(); @@ -182,7 +175,5 @@ describe('SampleDataComponent', () => { expect(sampleDataComponent.sampleDataIndex).toEqual(1); expect(sampleDataComponent.sampleData.length).toEqual(2); expect(sampleDataComponent.sampleData[1]).toEqual(sampleMessages[0]); - })); - });
http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-interface/metron-config/src/app/util/httpUtil.ts ---------------------------------------------------------------------- diff --git a/metron-interface/metron-config/src/app/util/httpUtil.ts b/metron-interface/metron-config/src/app/util/httpUtil.ts index dfcb61f..d8a21a5 100644 --- a/metron-interface/metron-config/src/app/util/httpUtil.ts +++ b/metron-interface/metron-config/src/app/util/httpUtil.ts @@ -15,34 +15,34 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import {Response} from '@angular/http'; -import {Observable} from 'rxjs/Observable'; -import {RestError} from '../model/rest-error'; +import { HttpErrorResponse, HttpResponse } from '@angular/common/http'; +import { throwError, Observable } from 'rxjs'; -export class HttpUtil { +import { RestError } from '../model/rest-error'; - public static extractString(res: Response): string { - let text: string = res.text(); +export class HttpUtil { + public static extractString(res: HttpResponse<any>): string { + let text: string = res.toString(); return text || ''; } - public static extractData(res: Response): any { - let body = res.json(); + public static extractData(res: HttpResponse<any>): any { + let body = res; return body || {}; } - public static handleError(res: Response): Observable<RestError> { + public static handleError(res: HttpErrorResponse): Observable<RestError> { // In a real world app, we might use a remote logging infrastructure // We'd also dig deeper into the error to get a better message let restError: RestError; if (res.status === 401) { window.location.assign('/login?sessionExpired=true'); } else if (res.status !== 404) { - restError = res.json(); + restError = res; } else { restError = new RestError(); - restError.responseCode = 404; + restError.status = 404; } - return Observable.throw(restError); + return throwError(restError); } } http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-interface/metron-config/src/app/util/httpUtils.spec.ts ---------------------------------------------------------------------- diff --git a/metron-interface/metron-config/src/app/util/httpUtils.spec.ts b/metron-interface/metron-config/src/app/util/httpUtils.spec.ts index ff1e39d..c2b0e6b 100644 --- a/metron-interface/metron-config/src/app/util/httpUtils.spec.ts +++ b/metron-interface/metron-config/src/app/util/httpUtils.spec.ts @@ -15,33 +15,52 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import {HttpUtil} from './httpUtil'; -import {Response, ResponseOptions, ResponseType} from '@angular/http'; -import {Observable} from 'rxjs/Observable'; -import {RestError} from '../model/rest-error'; +import { HttpUtil } from './httpUtil'; +import { RestError } from '../model/rest-error'; +import { HttpErrorResponse } from '@angular/common/http'; +import { noop } from 'rxjs'; describe('HttpUtil', () => { - it('should create an instance', () => { expect(HttpUtil.handleError).toBeTruthy(); expect(HttpUtil.extractString).toBeTruthy(); expect(HttpUtil.extractData).toBeTruthy(); }); - it('should handleError', () => { - let error500: RestError = {message: 'This is error', responseCode: 500, fullMessage: 'This is error'}; - let responseOptions = new ResponseOptions(); - responseOptions.body = error500; - let response = new Response(responseOptions); - response.type = ResponseType.Basic; - expect(HttpUtil.handleError(response)).toEqual(Observable.throw(error500)); - - let error404 = new RestError(); - error404.responseCode = 404; - response = new Response(new ResponseOptions()); - response.type = ResponseType.Basic; - response.status = 404; - expect(HttpUtil.handleError(response)).toEqual(Observable.throw(error404)); + it('should handleError 500', () => { + let error500: RestError = { + message: 'This is error', + status: 500, + error: 'This is error' + }; + let response = new HttpErrorResponse(error500); + let httpUtilSub = HttpUtil.handleError(response).subscribe( + noop, + e => { + expect(e.status).toBe(500); + expect(e.message).toBe( + 'Http failure response for (unknown url): 500 undefined' + ); + expect(e.error).toBe('This is error'); + }, + noop + ); + httpUtilSub.unsubscribe(); }); + it('should handleError 404', () => { + const error404 = new RestError(); + error404.status = 404; + const response = new HttpErrorResponse(error404); + const httpUtilSub = HttpUtil.handleError(response).subscribe( + noop, + e => { + expect(e.status).toBe(404); + expect(e.message).toBe(undefined); + expect(e.error).toBe(undefined); + }, + noop + ); + httpUtilSub.unsubscribe(); + }); }); http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-interface/metron-config/src/app/verticalnavbar/verticalnavbar.component.spec.ts ---------------------------------------------------------------------- diff --git a/metron-interface/metron-config/src/app/verticalnavbar/verticalnavbar.component.spec.ts b/metron-interface/metron-config/src/app/verticalnavbar/verticalnavbar.component.spec.ts index 7be3ccd..432ec2b 100644 --- a/metron-interface/metron-config/src/app/verticalnavbar/verticalnavbar.component.spec.ts +++ b/metron-interface/metron-config/src/app/verticalnavbar/verticalnavbar.component.spec.ts @@ -15,39 +15,29 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import {async, inject, TestBed} from '@angular/core/testing'; +import {TestBed} from '@angular/core/testing'; import {Router} from '@angular/router'; import {VerticalNavbarComponent} from './verticalnavbar.component'; class MockRouter { - url: string = ''; + url = ''; } describe('VerticalNavbarComponent', () => { - - beforeEach(async(() => { + let router: Router; + let verticalNavbarComponent: VerticalNavbarComponent; + beforeEach(() => { TestBed.configureTestingModule({ providers: [ VerticalNavbarComponent, {provide: Router, useClass: MockRouter} ] - }).compileComponents(); - - })); + }); + verticalNavbarComponent = TestBed.get(VerticalNavbarComponent); + }); - it('can instantiate VerticalNavbarComponent', - inject([VerticalNavbarComponent], (verticalNavbarComponent: VerticalNavbarComponent) => { + it('can instantiate VerticalNavbarComponent', () => { expect(verticalNavbarComponent instanceof VerticalNavbarComponent).toBe(true); - })); - - it('check isActive for a URL VerticalNavbarComponent', - inject([VerticalNavbarComponent, Router], (component: VerticalNavbarComponent, router: Router) => { - - router.url = '/abc'; - expect(component.isActive(['/def'])).toEqual(false); - expect(component.isActive(['/abc'])).toEqual(true); - expect(component.isActive(['/def', '/abc'])).toEqual(true); - - })); + }); }); http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-interface/metron-config/src/app/verticalnavbar/verticalnavbar.component.ts ---------------------------------------------------------------------- diff --git a/metron-interface/metron-config/src/app/verticalnavbar/verticalnavbar.component.ts b/metron-interface/metron-config/src/app/verticalnavbar/verticalnavbar.component.ts index 4067f6c..3425612 100644 --- a/metron-interface/metron-config/src/app/verticalnavbar/verticalnavbar.component.ts +++ b/metron-interface/metron-config/src/app/verticalnavbar/verticalnavbar.component.ts @@ -29,7 +29,4 @@ export class VerticalNavbarComponent { constructor(private router: Router) {} - isActive(validRoutes: string[]) { - return validRoutes.indexOf(this.router.url) !== -1; - } } http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-interface/metron-config/src/app/verticalnavbar/verticalnavbar.html ---------------------------------------------------------------------- diff --git a/metron-interface/metron-config/src/app/verticalnavbar/verticalnavbar.html b/metron-interface/metron-config/src/app/verticalnavbar/verticalnavbar.html index a4052b3..994711a 100644 --- a/metron-interface/metron-config/src/app/verticalnavbar/verticalnavbar.html +++ b/metron-interface/metron-config/src/app/verticalnavbar/verticalnavbar.html @@ -19,10 +19,10 @@ <ul class="nav"> <li class="nav-item"> - <a [routerLink]="['/sensors']" class="nav-link" [ngClass]="{'active': isActive(['/', '/sensors'])}" href="#">Sensors</a> + <a [routerLink]="['/sensors']" class="nav-link" routerLinkActive="active" href="#">Sensors</a> </li> <li class="nav-item"> - <a [routerLink]="['/general-settings']" class="nav-link" [ngClass]="{'active': isActive(['/general-settings'])}" href="#">General Settings</a> + <a [routerLink]="['/general-settings']" class="nav-link" routerLinkActive="active" href="#">General Settings</a> </li> </ul> http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-interface/metron-config/src/styles.scss ---------------------------------------------------------------------- diff --git a/metron-interface/metron-config/src/styles.scss b/metron-interface/metron-config/src/styles.scss index db57e91..26bb57c 100644 --- a/metron-interface/metron-config/src/styles.scss +++ b/metron-interface/metron-config/src/styles.scss @@ -17,7 +17,7 @@ */ /* You can add global styles to this file, and also import other style files */ @import "app/_variables.scss"; -@import "app/_main.scss"; +@import "app/_fonts.scss"; $height: 60px; http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-interface/metron-config/src/test.ts ---------------------------------------------------------------------- diff --git a/metron-interface/metron-config/src/test.ts b/metron-interface/metron-config/src/test.ts index 347d245..606c810 100644 --- a/metron-interface/metron-config/src/test.ts +++ b/metron-interface/metron-config/src/test.ts @@ -15,50 +15,24 @@ * See the License for the specific language governing permissions and * limitations under the License. */ -import './polyfills.ts'; -import 'zone.js/dist/long-stack-trace-zone'; -import 'zone.js/dist/proxy.js'; -import 'zone.js/dist/sync-test'; -import 'zone.js/dist/jasmine-patch'; -import 'zone.js/dist/async-test'; -import 'zone.js/dist/fake-async-test'; - -import 'jquery/dist/jquery'; -import 'tether/dist/js/tether'; - -import * as $ from 'jquery'; -window['$'] = window['jQuery'] = $; - -import * as Tether from 'tether'; -window['Tether'] = Tether; - -import 'ace-builds/src-noconflict/ace.js'; -import 'bootstrap/dist/js/bootstrap'; -import 'bootstrap'; - -// Unfortunately there's no typing for the `__karma__` variable. Just declare it as any. -declare var __karma__: any; -declare var require: any; - -// Prevent Karma from running prematurely. -__karma__.loaded = function () {}; - - -Promise.all([ - System.import('@angular/core/testing'), - System.import('@angular/platform-browser-dynamic/testing') -]) - // First, initialize the Angular testing environment. - .then(([testing, testingBrowser]) => { - testing.getTestBed().initTestEnvironment( - testingBrowser.BrowserDynamicTestingModule, - testingBrowser.platformBrowserDynamicTesting() - ); - }) - // Then we find all the tests. - .then(() => require.context('./', true, /\.spec\.ts/)) - // And load the modules. - .then(context => context.keys().map(context)) - // Finally, start Karma to run the tests. - .then(__karma__.start, __karma__.error); +// This file is required by karma.conf.js and loads recursively all the .spec and framework files + +import 'zone.js/dist/zone-testing'; +import { getTestBed } from '@angular/core/testing'; +import { + BrowserDynamicTestingModule, + platformBrowserDynamicTesting +} from '@angular/platform-browser-dynamic/testing'; + +declare const require: any; + +// First, initialize the Angular testing environment. +getTestBed().initTestEnvironment( + BrowserDynamicTestingModule, + platformBrowserDynamicTesting() +); +// Then we find all the tests. +const context = require.context('./', true, /\.spec\.ts$/); +// And load the modules. +context.keys().map(context); http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-interface/metron-config/src/tsconfig.app.json ---------------------------------------------------------------------- diff --git a/metron-interface/metron-config/src/tsconfig.app.json b/metron-interface/metron-config/src/tsconfig.app.json new file mode 100644 index 0000000..107b74d --- /dev/null +++ b/metron-interface/metron-config/src/tsconfig.app.json @@ -0,0 +1,23 @@ +{ + "extends": "../tsconfig.json", + "compilerOptions": { + "outDir": "../out-tsc/app", + "module": "es2015", + "baseUrl": "", + "types": [ + "jquery", + "bootstrap", + "ace", + "node" + ], + "typeRoots": [ "../node_modules/@types" ] + }, + "exclude": [ + "test.ts", + "**/*.spec.ts" + ], + "angularCompilerOptions": { + "preserveWhitespaces": true + }, +} + http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-interface/metron-config/src/tsconfig.json ---------------------------------------------------------------------- diff --git a/metron-interface/metron-config/src/tsconfig.json b/metron-interface/metron-config/src/tsconfig.json deleted file mode 100644 index 65756e3..0000000 --- a/metron-interface/metron-config/src/tsconfig.json +++ /dev/null @@ -1,24 +0,0 @@ -{ - "compilerOptions": { - "declaration": false, - "emitDecoratorMetadata": true, - "experimentalDecorators": true, - "lib": ["es6", "dom"], - "mapRoot": "./", - "module": "es6", - "moduleResolution": "node", - "outDir": "../dist/out-tsc", - "sourceMap": true, - "target": "es5", - "typeRoots": [ - "../node_modules/@types" - ], - "types": [ - "jasmine", - "jquery", - "bootstrap", - "ace" - ] - } - -} http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-interface/metron-config/src/tsconfig.spec.json ---------------------------------------------------------------------- diff --git a/metron-interface/metron-config/src/tsconfig.spec.json b/metron-interface/metron-config/src/tsconfig.spec.json new file mode 100644 index 0000000..eb43ad2 --- /dev/null +++ b/metron-interface/metron-config/src/tsconfig.spec.json @@ -0,0 +1,24 @@ +{ + "extends": "../tsconfig.json", + "compilerOptions": { + "outDir": "../out-tsc/spec", + "module": "commonjs", + "target": "es5", + "baseUrl": "", + "types": [ + "jquery", + "bootstrap", + "jasmine", + "node" + ], + "typeRoots": [ "../node_modules/@types" ] + }, + "files": [ + "test.ts", + "polyfills.ts" + ], + "include": [ + "**/*.spec.ts", + "**/*.d.ts" + ] + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-interface/metron-config/src/typings.d.ts ---------------------------------------------------------------------- diff --git a/metron-interface/metron-config/src/typings.d.ts b/metron-interface/metron-config/src/typings.d.ts index c0aaf0c..75b5619e 100644 --- a/metron-interface/metron-config/src/typings.d.ts +++ b/metron-interface/metron-config/src/typings.d.ts @@ -20,4 +20,4 @@ // https://www.typescriptlang.org/docs/handbook/writing-declaration-files.html declare var System: any; -declare var require: any; +declare var require: NodeRequire; http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-interface/metron-config/tsconfig.json ---------------------------------------------------------------------- diff --git a/metron-interface/metron-config/tsconfig.json b/metron-interface/metron-config/tsconfig.json new file mode 100644 index 0000000..1ca28d2 --- /dev/null +++ b/metron-interface/metron-config/tsconfig.json @@ -0,0 +1,20 @@ +{ + "compileOnSave": false, + "compilerOptions": { + "outDir": "./dist/out-tsc", + "baseUrl": "src", + "sourceMap": true, + "declaration": false, + "moduleResolution": "node", + "emitDecoratorMetadata": true, + "experimentalDecorators": true, + "target": "es5", + "typeRoots": [ + "node_modules/@types" + ], + "lib": [ + "es2016", + "dom" + ] + } + } \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-interface/metron-rest/README.md ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/README.md b/metron-interface/metron-rest/README.md index 7f00cde..f85707f 100644 --- a/metron-interface/metron-rest/README.md +++ b/metron-interface/metron-rest/README.md @@ -194,6 +194,44 @@ METRON_PRINCIPAL_NAME="met...@example.com" METRON_SERVICE_KEYTAB="/etc/security/keytabs/metron.keytab" ``` +### LDAP + +Metron REST can be configured to use LDAP for authentication and roles. Configuration can be performed via Ambari in the "Security" tab. + +Configuration will default to matching Knox's Demo LDAP for convenience. This should only be used for development purposes. Manual instructions for setting up demo LDAP and finalizing configuration (e.g. setting up the user LDIF file) can be found in the [Development README](../../metron-deployment/development/README.md#knox-demo-ldap). + +#### LDAPS +There is configuration to provide a path to a truststore with SSL certificates and provide a password. Users should import certificates as needed to appropriate truststores. An example of doing this is: +``` +keytool -import -alias <alias> -file <certificate> -keystore <keystore_file> -storepass <password> +``` + + +#### Roles +Roles used by Metron are ROLE_ADMIN and ROLE_USER. Metron will use a property in a group containing the appropriate role to construct this. + +For example, our ldif file could create this group: +``` +dn: cn=admin,ou=groups,dc=hadoop,dc=apache,dc=org +objectclass:top +objectclass: groupofnames +cn: admin +description:admin group +member: uid=admin,ou=people,dc=hadoop,dc=apache,dc=org +``` + +If we are using "cn" as our role attribute, Metron will give the "admin" user the role "ROLE_ADMIN". + +Similarly, we could give a user "sam" ROLE_USER with the following group: +``` +dn: cn=user,ou=groups,dc=hadoop,dc=apache,dc=org +objectclass:top +objectclass: groupofnames +cn: user +description: user group +member: uid=sam,ou=people,dc=hadoop,dc=apache,dc=org +``` + ## Spring Profiles The REST application comes with a few [Spring Profiles](http://docs.spring.io/autorepo/docs/spring-boot/current/reference/html/boot-features-profiles.html) to aid in testing and development. http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-interface/metron-rest/pom.xml ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/pom.xml b/metron-interface/metron-rest/pom.xml index 7fc373b..52b0175 100644 --- a/metron-interface/metron-rest/pom.xml +++ b/metron-interface/metron-rest/pom.xml @@ -31,6 +31,8 @@ <powermock.version>1.6.4</powermock.version> <spring.boot.version>2.0.1.RELEASE</spring.boot.version> <spring.kerberos.version>1.0.1.RELEASE</spring.kerberos.version> + <spring.ldap.core.version>2.3.2.RELEASE</spring.ldap.core.version> + <spring.security.ldap.version>5.1.1.RELEASE</spring.security.ldap.version> <swagger.version>2.5.0</swagger.version> <mysql.client.version>5.1.40</mysql.client.version> <spring-kafka.version>2.0.4.RELEASE</spring-kafka.version> @@ -118,6 +120,16 @@ </exclusions> </dependency> <dependency> + <groupId>org.springframework.ldap</groupId> + <artifactId>spring-ldap-core</artifactId> + <version>${spring.ldap.core.version}</version> + </dependency> + <dependency> + <groupId>org.springframework.security</groupId> + <artifactId>spring-security-ldap</artifactId> + <version>${spring.security.ldap.version}</version> + </dependency> + <dependency> <groupId>com.googlecode.json-simple</groupId> <artifactId>json-simple</artifactId> <version>${global_json_simple_version}</version> http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-interface/metron-rest/src/main/config/rest_application.yml ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/config/rest_application.yml b/metron-interface/metron-rest/src/main/config/rest_application.yml index 84efc01..7ab7b73 100644 --- a/metron-interface/metron-rest/src/main/config/rest_application.yml +++ b/metron-interface/metron-rest/src/main/config/rest_application.yml @@ -13,14 +13,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -spring: - datasource: - driverClassName: ${METRON_JDBC_DRIVER} - url: ${METRON_JDBC_URL} - username: ${METRON_JDBC_USERNAME} - password: ${METRON_JDBC_PASSWORD} - platform: ${METRON_JDBC_PLATFORM} - continue-on-error: true zookeeper: url: ${ZOOKEEPER} @@ -62,3 +54,27 @@ pcap: page.size: ${PCAP_PAGE_SIZE} yarn.queue: ${PCAP_YARN_QUEUE} finalizer.threadpool.size: ${PCAP_FINALIZER_THREADPOOL_SIZE} + +spring: + datasource: + driverClassName: ${METRON_JDBC_DRIVER} + url: ${METRON_JDBC_URL} + username: ${METRON_JDBC_USERNAME} + password: ${METRON_JDBC_PASSWORD} + platform: ${METRON_JDBC_PLATFORM} + continue-on-error: true + +ldap: + provider: + url: ${METRON_LDAP_URL} + userdn: ${METRON_LDAP_USERDN} + password: ${METRON_LDAP_PASSWORD} + user: + dn.patterns: ${METRON_LDAP_USER_PATTERN} + passwordAttribute: ${METRON_LDAP_USER_PASSWORD} + searchBase: ${METRON_LDAP_USER_SEARCHBASE} + searchFilter: ${METRON_LDAP_USER_SEARCHFILTER} + group: + searchBase: ${METRON_LDAP_GROUP_SEARCHBASE} + searchFilter: ${METRON_LDAP_GROUP_SEARCHFILTER} + roleAttribute: ${METRON_LDAP_GROUP_ROLE} http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java index 94e8e35..80ac2bf 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/MetronRestConstants.java @@ -23,6 +23,7 @@ public class MetronRestConstants { public static final String DEV_PROFILE = "dev"; public static final String TEST_PROFILE = "test"; + public static final String LDAP_PROFILE = "ldap"; public static final String DOCKER_PROFILE = "docker"; public static final String CSRF_ENABLE_PROFILE = "csrf-enable"; http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/WebSecurityConfig.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/WebSecurityConfig.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/WebSecurityConfig.java index f84cdfa..7ca3a46 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/WebSecurityConfig.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/config/WebSecurityConfig.java @@ -20,8 +20,14 @@ package org.apache.metron.rest.config; import static org.apache.metron.rest.MetronRestConstants.SECURITY_ROLE_ADMIN; import static org.apache.metron.rest.MetronRestConstants.SECURITY_ROLE_USER; +import java.util.Arrays; +import java.util.List; +import javax.sql.DataSource; import org.apache.metron.rest.MetronRestConstants; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.beans.factory.annotation.Value; import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.core.env.Environment; @@ -30,7 +36,7 @@ import org.springframework.security.config.annotation.method.configuration.Enabl import org.springframework.security.config.annotation.web.builders.HttpSecurity; import org.springframework.security.config.annotation.web.configuration.EnableWebSecurity; import org.springframework.security.config.annotation.web.configuration.WebSecurityConfigurerAdapter; -import org.springframework.security.core.userdetails.User; +import org.springframework.security.crypto.password.LdapShaPasswordEncoder; import org.springframework.security.crypto.password.NoOpPasswordEncoder; import org.springframework.security.crypto.password.PasswordEncoder; import org.springframework.security.web.authentication.logout.HttpStatusReturningLogoutSuccessHandler; @@ -39,19 +45,40 @@ import org.springframework.stereotype.Controller; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; -import javax.sql.DataSource; -import java.util.Arrays; -import java.util.List; - @Configuration @EnableWebSecurity @EnableGlobalMethodSecurity(securedEnabled = true) @Controller public class WebSecurityConfig extends WebSecurityConfigurerAdapter { + private static final Logger LOG = LoggerFactory.getLogger(WebSecurityConfig.class); @Autowired private Environment environment; + @Autowired + private DataSource dataSource; + + @Value("${ldap.provider.url}") + private String providerUrl; + @Value("${ldap.provider.userdn}") + private String providerUserDn; + @Value("${ldap.provider.password}") + private String providerPassword; + @Value("${ldap.user.dn.patterns}") + private String userDnPatterns; + @Value("${ldap.user.passwordAttribute}") + private String passwordAttribute; + @Value("${ldap.user.searchBase}") + private String userSearchBase; + @Value("${ldap.user.searchFilter}") + private String userSearchFilter; + @Value("${ldap.group.searchBase}") + private String groupSearchBase; + @Value("${ldap.group.roleAttribute}") + private String groupRoleAttribute; + @Value("${ldap.group.searchFilter}") + private String groupSearchFilter; + @RequestMapping(value = {"/login", "/logout", "/sensors", "/sensors*/**"}, method = RequestMethod.GET) public String handleNGRequests() { return "forward:/index.html"; @@ -84,18 +111,34 @@ public class WebSecurityConfig extends WebSecurityConfigurerAdapter { } @Autowired - private DataSource dataSource; - - @Autowired public void configureJdbc(AuthenticationManagerBuilder auth) throws Exception { + // Note that we can switch profiles on the fly in Ambari. List<String> activeProfiles = Arrays.asList(environment.getActiveProfiles()); - if (activeProfiles.contains(MetronRestConstants.DEV_PROFILE) || - activeProfiles.contains(MetronRestConstants.TEST_PROFILE)) { - auth.jdbcAuthentication().dataSource(dataSource) - .withUser("user").password("password").roles(SECURITY_ROLE_USER).and() - .withUser("user1").password("password").roles(SECURITY_ROLE_USER).and() - .withUser("user2").password("password").roles(SECURITY_ROLE_USER).and() - .withUser("admin").password("password").roles(SECURITY_ROLE_USER, SECURITY_ROLE_ADMIN); + if (activeProfiles.contains(MetronRestConstants.LDAP_PROFILE)) { + LOG.debug("Setting up LDAP authentication against {}.", providerUrl); + auth.ldapAuthentication() + .userDnPatterns(userDnPatterns) + .userSearchBase(userSearchBase) + .userSearchFilter(userSearchFilter) + .groupRoleAttribute(groupRoleAttribute) + .groupSearchFilter(groupSearchFilter) + .groupSearchBase(groupSearchBase) + .contextSource() + .url(providerUrl) + .managerDn(providerUserDn) + .managerPassword(providerPassword) + .and() + .passwordCompare() + .passwordEncoder(new LdapShaPasswordEncoder()) + .passwordAttribute(passwordAttribute); + } else if (activeProfiles.contains(MetronRestConstants.DEV_PROFILE) || + activeProfiles.contains(MetronRestConstants.TEST_PROFILE)) { + auth.jdbcAuthentication() + .dataSource(dataSource) + .withUser("user").password("password").roles(SECURITY_ROLE_USER).and() + .withUser("user1").password("password").roles(SECURITY_ROLE_USER).and() + .withUser("user2").password("password").roles(SECURITY_ROLE_USER).and() + .withUser("admin").password("password").roles(SECURITY_ROLE_USER, SECURITY_ROLE_ADMIN); } else { auth.jdbcAuthentication().dataSource(dataSource); } http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/UserController.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/UserController.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/UserController.java index b5f7765..24c781c 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/UserController.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/UserController.java @@ -19,6 +19,11 @@ package org.apache.metron.rest.controller; import io.swagger.annotations.ApiOperation; import io.swagger.annotations.ApiResponse; +import java.util.List; +import java.util.stream.Collectors; +import org.springframework.security.access.annotation.Secured; +import org.springframework.security.core.context.SecurityContextHolder; +import org.springframework.security.core.userdetails.UserDetails; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RequestMethod; import org.springframework.web.bind.annotation.RestController; @@ -35,4 +40,12 @@ public class UserController { public String user(Principal user) { return user.getName(); } + + @Secured("IS_AUTHENTICATED_FULLY") + @RequestMapping(path = "/whoami/roles", method = RequestMethod.GET) + public List<String> user() { + UserDetails userDetails = (UserDetails) SecurityContextHolder.getContext(). + getAuthentication().getPrincipal(); + return userDetails.getAuthorities().stream().map(ga -> ga.getAuthority()).collect(Collectors.toList()); + } } http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java index 85b84b8..d0e4b3d 100644 --- a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java +++ b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/impl/SensorParserConfigServiceImpl.java @@ -20,11 +20,10 @@ package org.apache.metron.rest.service.impl; import static org.apache.metron.rest.MetronRestConstants.GROK_CLASS_NAME; import com.fasterxml.jackson.databind.ObjectMapper; -import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; +import java.util.Optional; import org.apache.curator.framework.CuratorFramework; import org.apache.hadoop.fs.Path; import org.apache.metron.common.configuration.ConfigurationType; @@ -33,17 +32,15 @@ import org.apache.metron.common.configuration.ParserConfigurations; import org.apache.metron.common.configuration.SensorParserConfig; import org.apache.metron.common.zookeeper.ConfigurationsCache; import org.apache.metron.parsers.interfaces.MessageParser; +import org.apache.metron.parsers.interfaces.MessageParserResult; import org.apache.metron.rest.MetronRestConstants; import org.apache.metron.rest.RestException; import org.apache.metron.rest.model.ParseMessageRequest; import org.apache.metron.rest.service.GrokService; import org.apache.metron.rest.service.SensorParserConfigService; import org.apache.metron.rest.util.ParserIndex; -import org.apache.metron.common.zookeeper.ZKConfigurationsCache; import org.apache.zookeeper.KeeperException; import org.json.simple.JSONObject; -import org.reflections.Reflections; -import org.reflections.util.ConfigurationBuilder; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; @@ -154,11 +151,24 @@ public class SensorParserConfigServiceImpl implements SensorParserConfigService } parser.configure(sensorParserConfig.getParserConfig()); parser.init(); - JSONObject results = parser.parse(parseMessageRequest.getSampleData().getBytes()).get(0); + + Optional<MessageParserResult<JSONObject>> result = parser.parseOptionalResult(parseMessageRequest.getSampleData().getBytes()); + if (!result.isPresent()) { + throw new RestException("Unknown error parsing sample data"); + } + + if (result.get().getMasterThrowable().isPresent()) { + throw new RestException("Error parsing sample data",result.get().getMasterThrowable().get()); + } + + if (result.get().getMessages().isEmpty()) { + throw new RestException("No results from parsing sample data"); + } + if (isGrokConfig(sensorParserConfig) && temporaryGrokPath != null) { grokService.deleteTemporary(); } - return results; + return result.get().getMessages().get(0); } } http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-interface/metron-rest/src/main/resources/application-vagrant.yml ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/resources/application-vagrant.yml b/metron-interface/metron-rest/src/main/resources/application-vagrant.yml index 3eea24a..05ea571 100644 --- a/metron-interface/metron-rest/src/main/resources/application-vagrant.yml +++ b/metron-interface/metron-rest/src/main/resources/application-vagrant.yml @@ -52,9 +52,22 @@ storm: randomaccess.script.path: /usr/metron/${metron.version}/bin/start_elasticsearch_topology.sh batch.script.path: /usr/metron/${metron.version}/bin/start_hdfs_topology.sh - kerberos: enabled: false principal: met...@example.com keytab: /etc/security/keytabs/metron.headless.keytab +ldap: + provider: + url: ldap://node1:33389 + userdn: uid=admin,ou=people,dc=hadoop,dc=apache,dc=org + password: "admin-password" + user: + dn.patterns: uid={0},ou=people,dc=hadoop,dc=apache,dc=org + passwordAttribute: userPassword + searchBase: ou=people,dc=hadoop,dc=apache,dc=org + searchFilter: "" + group: + searchBase: ou=groups,dc=hadoop,dc=apache,dc=org + searchFilter: "member={0}" + roleAttribute: "cn" http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-interface/metron-rest/src/main/scripts/metron-rest.sh ---------------------------------------------------------------------- diff --git a/metron-interface/metron-rest/src/main/scripts/metron-rest.sh b/metron-interface/metron-rest/src/main/scripts/metron-rest.sh index 7c89ae5..b454b8e 100644 --- a/metron-interface/metron-rest/src/main/scripts/metron-rest.sh +++ b/metron-interface/metron-rest/src/main/scripts/metron-rest.sh @@ -17,8 +17,8 @@ # limitations under the License. # -if [ -z "${METRON_JDBC_PASSWORD}" ]; then - echo "METRON_JDBC_PASSWORD unset. Exiting." +if [ -z "${METRON_JDBC_PASSWORD}" ] && [ -z "${METRON_LDAP_PASSWORD}" ]; then + echo "Authentication password unset. Exiting." exit 1 fi ## Join a list by a character @@ -35,6 +35,7 @@ METRON_REST_PORT=8082 METRON_SYSCONFIG="${METRON_SYSCONFIG:-/etc/default/metron}" METRON_LOG_DIR="${METRON_LOG_DIR:-/var/log/metron}" METRON_PID_FILE="${METRON_PID_FILE:-/var/run/metron/metron-rest.pid}" + PARSER_CONTRIB=${PARSER_CONTRIB:-$METRON_HOME/parser_contrib} INDEXING_CONTRIB=${INDEXING_CONTRIB:-$METRON_HOME/indexing_contrib} PARSER_LIB=$(find $METRON_HOME/lib/ -name metron-parsers*.jar) @@ -112,6 +113,24 @@ METRON_REST_CLASSPATH+=":${indexing_files[0]}" echo "METRON_REST_CLASSPATH=${METRON_REST_CLASSPATH}" +echo "METRON_JDBC_DRIVER=${METRON_JDBC_DRIVER}" +echo "METRON_JDBC_URL=${METRON_JDBC_URL}" +echo "METRON_JDBC_USERNAME=${METRON_JDBC_USERNAME}" +echo "METRON_JDBC_PLATFORM=${METRON_JDBC_PLATFORM}" + +echo "METRON_LDAP_URL=${METRON_LDAP_URL}" +echo "METRON_LDAP_USERDN=${METRON_LDAP_USERDN}" + +echo "METRON_LDAP_USER_PATTERN=${METRON_LDAP_USER_PATTERN}" +echo "METRON_LDAP_USER_PASSWORD=${METRON_LDAP_USER_PASSWORD}" +echo "METRON_LDAP_USER_SEARCHBASE=${METRON_LDAP_USER_SEARCHBASE}" +echo "METRON_LDAP_USER_SEARCHFILTER=${METRON_LDAP_USER_SEARCHFILTER}" + +echo "METRON_LDAP_GROUP_SEARCHBASE=${METRON_LDAP_GROUP_SEARCHBASE}" +echo "METRON_LDAP_GROUP_SEARCHFILTER=${METRON_LDAP_GROUP_SEARCHFILTER}" +echo "METRON_LDAP_GROUP_ROLE=${METRON_LDAP_GROUP_ROLE}" +echo "METRON_LDAP_SSL_TRUSTSTORE=${METRON_LDAP_SSL_TRUSTSTORE}" + #Use Solr daos if ra indexing writer set to Solr if [[ ${METRON_RA_INDEXING_WRITER} == "Solr" ]]; then METRON_INDEX_DAO=" --index.dao.impl=org.apache.metron.solr.dao.SolrDao,org.apache.metron.indexing.dao.HBaseDao" @@ -125,6 +144,11 @@ if [[ ${METRON_RA_INDEXING_WRITER} == "Solr" ]]; then METRON_SPRING_OPTIONS+=${METRON_WRITER_NAME} fi +if [ -n "${METRON_LDAP_SSL_TRUSTSTORE}" ]; then + METRON_JVMFLAGS+=" -Djavax.net.ssl.trustStore=${METRON_LDAP_SSL_TRUSTSTORE}" + METRON_JVMFLAGS+=" -Djavax.net.ssl.trustStorePassword=${METRON_LDAP_SSL_TRUSTSTORE_PASSWORD}" +fi + echo "Starting application" ${JAVA_HOME}/bin/java -Dhdp.version=${HDP_VERSION} ${METRON_JVMFLAGS} \ -cp ${METRON_REST_CLASSPATH} \ http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-platform/Performance-tuning-guide.md ---------------------------------------------------------------------- diff --git a/metron-platform/Performance-tuning-guide.md b/metron-platform/Performance-tuning-guide.md index bfc36dd..2e976e9 100644 --- a/metron-platform/Performance-tuning-guide.md +++ b/metron-platform/Performance-tuning-guide.md @@ -188,9 +188,11 @@ See more detail on starting parsers [here](https://github.com/apache/metron/blob **Enrichment** +__Note__ These recommendations are based on the deprecated split-join enrichment topology. See [Enrichment Performance](metron-enrichment/Performance.md) for tuning recommendations for the new default unified enrichment topology. + This is a mapping of the various performance tuning properties for enrichments and how they are materialized. -Flux file found here - $METRON_HOME/flux/enrichment/remote.yaml +Flux file found here - $METRON_HOME/flux/enrichment/remote-splitjoin.yaml _Note 1:_ Changes to Flux file properties that are managed by Ambari will render Ambari unable to further manage the property. @@ -458,6 +460,8 @@ usage: start_parser_topology.sh ### Enrichment Tuning +__Note__ These tuning suggestions are based on the deprecated split-join topology. + We landed on the same number of partitions for enrichemnt and indexing as we did for bro - 48. For configuring Storm, there is a flux file and properties file that we modified. Here are the settings we changed for bro in Flux. http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-platform/elasticsearch-shaded/pom.xml ---------------------------------------------------------------------- diff --git a/metron-platform/elasticsearch-shaded/pom.xml b/metron-platform/elasticsearch-shaded/pom.xml index d9002e4..7766e3d 100644 --- a/metron-platform/elasticsearch-shaded/pom.xml +++ b/metron-platform/elasticsearch-shaded/pom.xml @@ -34,57 +34,10 @@ <artifactId>netty-common</artifactId> <version>4.1.13.Final</version> </dependency> - <!--dependency> - <groupId>org.elasticsearch.client</groupId> - <artifactId>transport</artifactId> - <version>${global_elasticsearch_version}</version> - <exclusions> - <exclusion> - <groupId>com.fasterxml.jackson.dataformat</groupId> - <artifactId>jackson-dataformat-smile</artifactId> - </exclusion> - <exclusion> - <groupId>com.fasterxml.jackson.dataformat</groupId> - <artifactId>jackson-dataformat-yaml</artifactId> - </exclusion> - <exclusion> - <groupId>com.fasterxml.jackson.dataformat</groupId> - <artifactId>jackson-dataformat-cbor</artifactId> - </exclusion> - <exclusion> - <groupId>com.fasterxml.jackson.core</groupId> - <artifactId>jackson-core</artifactId> - </exclusion> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-api</artifactId> - </exclusion> - <exclusion> - <groupId>org.slf4j</groupId> - <artifactId>slf4j-log4j12</artifactId> - </exclusion> - <exclusion> - <groupId>log4j</groupId> - <artifactId>log4j</artifactId> - </exclusion> - </exclusions> - </dependency--> <dependency> <groupId>org.elasticsearch.client</groupId> <artifactId>elasticsearch-rest-high-level-client</artifactId> <version>${global_elasticsearch_version}</version> - <exclusions> - <exclusion> - <!-- - TODO: This shouldn't be required, but the Shade services resources transformer is botching the services - file in META-INF. You should not merge before figuring out if there's a way to avoid the - botched merge. One way to do this is to create a new resources transformer that handles the merge - properly. I have NO idea if excluding this matters. - --> - <groupId>org.elasticsearch.plugin</groupId> - <artifactId>aggs-matrix-stats-client</artifactId> - </exclusion> - </exclusions> </dependency> <dependency> <groupId>org.apache.logging.log4j</groupId> @@ -117,10 +70,6 @@ <groupId>com.fasterxml.jackson.core</groupId> <artifactId>jackson-core</artifactId> </exclusion> - <exclusion> <!-- this is causing a weird build error if not excluded - Error creating shaded jar: null: IllegalArgumentException --> - <groupId>org.apache.logging.log4j</groupId> - <artifactId>log4j-api</artifactId> - </exclusion> </exclusions> </dependency> <dependency> http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-platform/metron-common/README.md ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/README.md b/metron-platform/metron-common/README.md index dac1974..d68259a 100644 --- a/metron-platform/metron-common/README.md +++ b/metron-platform/metron-common/README.md @@ -79,29 +79,30 @@ This configuration is stored in zookeeper, but looks something like Various parts of our stack uses the global config are documented throughout the Metron documentation, but a convenient index is provided here: -| Property Name | Subsystem | Type | Ambari Property | -|---------------------------------------------------------------------------------------------------------------------|---------------|------------|----------------------------| -| [`es.clustername`](../metron-elasticsearch#esclustername) | Indexing | String | `es_cluster_name` | -| [`es.ip`](../metron-elasticsearch#esip) | Indexing | String | `es_hosts` | -| [`es.port`](../metron-elasticsearch#esport) | Indexing | String | `es_port` | -| [`es.date.format`](../metron-elasticsearch#esdateformat) | Indexing | String | `es_date_format` | -| [`fieldValidations`](#validation-framework) | Parsing | Object | N/A | -| [`parser.error.topic`](../metron-parsers#parsererrortopic) | Parsing | String | N/A | -| [`stellar.function.paths`](../../metron-stellar/stellar-common#stellarfunctionpaths) | Stellar | CSV String | N/A | -| [`stellar.function.resolver.includes`](../../metron-stellar/stellar-common#stellarfunctionresolverincludesexcludes) | Stellar | CSV String | N/A | -| [`stellar.function.resolver.excludes`](../../metron-stellar/stellar-common#stellarfunctionresolverincludesexcludes) | Stellar | CSV String | N/A | -| [`profiler.period.duration`](../../metron-analytics/metron-profiler#profilerperiodduration) | Profiler | Integer | `profiler_period_duration` | -| [`profiler.period.duration.units`](../../metron-analytics/metron-profiler#profilerperioddurationunits) | Profiler | String | `profiler_period_units` | -| [`profiler.writer.batchSize`](../../metron-analytics/metron-profiler/#profilerwriterbatchsize) | Profiler | Integer | N/A | -| [`profiler.writer.batchTimeout`](../../metron-analytics/metron-profiler/#profilerwriterbatchtimeout) | Profiler | Integer | N/A | -| [`update.hbase.table`](../metron-indexing#updatehbasetable) | REST/Indexing | String | `update_hbase_table` | -| [`update.hbase.cf`](../metron-indexing#updatehbasecf) | REST/Indexing | String | `update_hbase_cf` | -| [`geo.hdfs.file`](../metron-enrichment#geohdfsfile) | Enrichment | String | `geo_hdfs_file` | -| [`enrichment.writer.batchSize`](../metron-enrichment#enrichmentwriterbatchsize) | Enrichment | Integer | N/A | -| [`enrichment.writer.batchTimeout`](../metron-enrichment#enrichmentwriterbatchtimeout) | Enrichment | Integer | N/A | -| [`geo.hdfs.file`](../metron-enrichment#geohdfsfile) | Enrichment | String | `geo_hdfs_file` | -| [`source.type.field`](../../metron-interface/metron-alerts#sourcetypefield) | UI | String | `source_type_field` | -| [`threat.triage.score.field`](../../metron-interface/metron-alerts#threattriagescorefield) | UI | String | `threat_triage_score_field` | +| Property Name | Subsystem | Type | Ambari Property | +|---------------------------------------------------------------------------------------------------------------------|---------------|------------|------------------------------| +| [`es.clustername`](../metron-elasticsearch#esclustername) | Indexing | String | `es_cluster_name` | +| [`es.ip`](../metron-elasticsearch#esip) | Indexing | String | `es_hosts` | +| [`es.port`](../metron-elasticsearch#esport) | Indexing | String | `es_port` | +| [`es.date.format`](../metron-elasticsearch#esdateformat) | Indexing | String | `es_date_format` | +| [`es.client.settings`](../metron-elasticsearch#esclientsettings) | Indexing | Object | N/A | +| [`fieldValidations`](#validation-framework) | Parsing | Object | N/A | +| [`parser.error.topic`](../metron-parsers#parsererrortopic) | Parsing | String | N/A | +| [`stellar.function.paths`](../../metron-stellar/stellar-common#stellarfunctionpaths) | Stellar | CSV String | N/A | +| [`stellar.function.resolver.includes`](../../metron-stellar/stellar-common#stellarfunctionresolverincludesexcludes) | Stellar | CSV String | N/A | +| [`stellar.function.resolver.excludes`](../../metron-stellar/stellar-common#stellarfunctionresolverincludesexcludes) | Stellar | CSV String | N/A | +| [`profiler.period.duration`](../../metron-analytics/metron-profiler#profilerperiodduration) | Profiler | Integer | `profiler_period_duration` | +| [`profiler.period.duration.units`](../../metron-analytics/metron-profiler#profilerperioddurationunits) | Profiler | String | `profiler_period_units` | +| [`profiler.writer.batchSize`](../../metron-analytics/metron-profiler/#profilerwriterbatchsize) | Profiler | Integer | N/A | +| [`profiler.writer.batchTimeout`](../../metron-analytics/metron-profiler/#profilerwriterbatchtimeout) | Profiler | Integer | N/A | +| [`update.hbase.table`](../metron-indexing#updatehbasetable) | REST/Indexing | String | `update_hbase_table` | +| [`update.hbase.cf`](../metron-indexing#updatehbasecf) | REST/Indexing | String | `update_hbase_cf` | +| [`geo.hdfs.file`](../metron-enrichment#geohdfsfile) | Enrichment | String | `geo_hdfs_file` | +| [`enrichment.writer.batchSize`](../metron-enrichment#enrichmentwriterbatchsize) | Enrichment | Integer | N/A | +| [`enrichment.writer.batchTimeout`](../metron-enrichment#enrichmentwriterbatchtimeout) | Enrichment | Integer | N/A | +| [`geo.hdfs.file`](../metron-enrichment#geohdfsfile) | Enrichment | String | `geo_hdfs_file` | +| [`source.type.field`](../../metron-interface/metron-alerts#sourcetypefield) | UI | String | `source_type_field` | +| [`threat.triage.score.field`](../../metron-interface/metron-alerts#threattriagescorefield) | UI | String | `threat_triage_score_field` | ## Note Configs in Ambari If a field is managed via ambari, you should change the field via @@ -439,3 +440,4 @@ Options: -p DIRECTORY, --hdp_home=DIRECTORY HDP home directory ``` +` http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-platform/metron-common/src/main/config/zookeeper/global.json ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/config/zookeeper/global.json b/metron-platform/metron-common/src/main/config/zookeeper/global.json index 9e5402e..b638ca3 100644 --- a/metron-platform/metron-common/src/main/config/zookeeper/global.json +++ b/metron-platform/metron-common/src/main/config/zookeeper/global.json @@ -6,6 +6,5 @@ "update.hbase.table": "metron_update", "update.hbase.cf": "t", "es.client.settings": { - "client.transport.ping_timeout": "500s" } } http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredEnrichmentBolt.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredEnrichmentBolt.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredEnrichmentBolt.java index c28ca7b..2e03a36 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredEnrichmentBolt.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredEnrichmentBolt.java @@ -17,8 +17,10 @@ */ package org.apache.metron.common.bolt; +import java.io.IOException; import java.lang.invoke.MethodHandles; import org.apache.metron.common.configuration.EnrichmentConfigurations; +import org.apache.metron.stellar.dsl.StellarFunctions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -31,4 +33,16 @@ public abstract class ConfiguredEnrichmentBolt extends ConfiguredBolt<Enrichment super(zookeeperUrl, "ENRICHMENT"); } + @Override + public void cleanup() { + // This method may not be called in production. + // See https://storm.apache.org/releases/1.0.6/javadocs/org/apache/storm/task/IBolt.html#cleanup-- for more detail. + super.cleanup(); + try { + StellarFunctions.close(); + } catch (IOException e) { + LOG.error(e.getMessage(), e); + } + } + } http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredParserBolt.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredParserBolt.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredParserBolt.java index 14ce50b..17b614b 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredParserBolt.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/bolt/ConfiguredParserBolt.java @@ -17,9 +17,11 @@ */ package org.apache.metron.common.bolt; +import java.io.IOException; import java.lang.invoke.MethodHandles; import org.apache.metron.common.configuration.ParserConfigurations; import org.apache.metron.common.configuration.SensorParserConfig; +import org.apache.metron.stellar.dsl.StellarFunctions; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -36,4 +38,15 @@ public abstract class ConfiguredParserBolt extends ConfiguredBolt<ParserConfigur return getConfigurations().getSensorParserConfig(sensorType); } + @Override + public void cleanup() { + // This method may not be called in production. + // See https://storm.apache.org/releases/1.0.6/javadocs/org/apache/storm/task/IBolt.html#cleanup-- for more detail. + super.cleanup(); + try { + StellarFunctions.close(); + } catch (IOException e) { + LOG.error(e.getMessage(), e); + } + } } http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigOption.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigOption.java b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigOption.java index 6308f0a..14d5b69 100644 --- a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigOption.java +++ b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigOption.java @@ -30,6 +30,13 @@ public interface ConfigOption { return (s, o) -> o; } + /** + * Returns true if the map contains the key for the defined config option + */ + default boolean containsOption(Map<String, Object> map) { + return map.containsKey(getKey()); + } + default void put(Map<String, Object> map, Object value) { map.put(getKey(), value); } http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-platform/metron-common/src/main/scripts/stellar ---------------------------------------------------------------------- diff --git a/metron-platform/metron-common/src/main/scripts/stellar b/metron-platform/metron-common/src/main/scripts/stellar index c831c62..690a9f0 100644 --- a/metron-platform/metron-common/src/main/scripts/stellar +++ b/metron-platform/metron-common/src/main/scripts/stellar @@ -28,10 +28,16 @@ elif [ -e /usr/lib/bigtop-utils/bigtop-detect-javahome ]; then . /usr/lib/bigtop-utils/bigtop-detect-javahome fi +export METRON_SYSCONFIG="/etc/default/metron" +if [ -f "$METRON_SYSCONFIG" ]; then + source $METRON_SYSCONFIG +fi + +# treat unset vars as an error; METRON_HOME +set -u + export HBASE_CONFIGS=$(hbase classpath) -export METRON_VERSION=${project.version} -export METRON_HOME=/usr/metron/$METRON_VERSION export STELLAR_LIB=$(find $METRON_HOME/lib/ -name metron-parsers*.jar) export MANAGEMENT_LIB=$(find $METRON_HOME/lib/ -name metron-management*.jar) export PROFILER_LIB=$(find $METRON_HOME/lib/ -name metron-profiler-repl*.jar) -java $JVMFLAGS -cp "${CONTRIB:-$METRON_HOME/contrib/*}:$STELLAR_LIB:$MANAGEMENT_LIB:$PROFILER_LIB:$HBASE_CONFIGS" org.apache.metron.stellar.common.shell.cli.StellarShell "$@" +java $METRON_JVMFLAGS -cp "${CONTRIB:-$METRON_HOME/contrib/*}:$STELLAR_LIB:$MANAGEMENT_LIB:$PROFILER_LIB:$HBASE_CONFIGS" org.apache.metron.stellar.common.shell.cli.StellarShell "$@" http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-platform/metron-data-management/pom.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-data-management/pom.xml b/metron-platform/metron-data-management/pom.xml index c4dbb74..a7a5a40 100644 --- a/metron-platform/metron-data-management/pom.xml +++ b/metron-platform/metron-data-management/pom.xml @@ -26,7 +26,6 @@ <properties> <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> - <httpcore.version>4.3.2</httpcore.version> <lucene.test.version>5.5.0</lucene.test.version> </properties> @@ -221,12 +220,12 @@ <dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId>httpcore</artifactId> - <version>${httpcore.version}</version> + <version>${global_httpclient_version}</version> </dependency> <dependency> <groupId>org.apache.httpcomponents</groupId> <artifactId>httpclient</artifactId> - <version>${httpcore.version}</version> + <version>${global_httpclient_version}</version> </dependency> <dependency> <groupId>org.hamcrest</groupId> http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-platform/metron-elasticsearch/README.md ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/README.md b/metron-platform/metron-elasticsearch/README.md index d889e27..463a0b8 100644 --- a/metron-platform/metron-elasticsearch/README.md +++ b/metron-platform/metron-elasticsearch/README.md @@ -59,6 +59,49 @@ For instance, an `es.date.format` of `yyyy.MM.dd.HH` would have the consequence roll hourly, whereas an `es.date.format` of `yyyy.MM.dd` would have the consequence that the indices would roll daily. +### `es.client.settings` + +This field in global config allows you to specify Elasticsearch REST client options. These are used in conjunction with the previously mentioned Elasticsearch properties +when setting up client connections to an Elasticsearch cluster. The available properties should be supplied as an object map. Current available options are as follows: + +| Property Name | Type | Required? | Default Value | Description | +|-------------------------------------|-----------|-----------|----------------|---------------------------------------------------------------------------------------------------------------------------------------------------------------------| +| connection.timeout.millis | Integer | No | 1000 | Sets connection timeout. | +| socket.timeout.millis | Integer | No | 30000 | Sets socket timeout. | +| max.retry.timeout.millis | Integer | No | 30000 | Sets the maximum timeout (in milliseconds) to honour in case of multiple retries of the same request. | +| num.client.connection.threads | Integer | No | 1 | Number of worker threads used by the connection manager. Defaults to Runtime.getRuntime().availableProcessors(). | +| xpack.username | String | No | null | X-Pack username. | +| xpack.password.file | String | No | null | 1-line HDFS file where the X-Pack password is set. | +| ssl.enabled | Boolean | No | false | Turn on SSL connections. | +| keystore.type | String | No | "jks" | Allows you to specify a keytstore type. See https://docs.oracle.com/javase/8/docs/technotes/guides/security/StandardNames.html#KeyStore for more details. | +| keystore.path | String | No | null | Path to the Trust Store that holds your Elasticsearch certificate authorities and certificate. | +| keystore.password.file | String | No | null | 1-line HDFS file where the keystore password is set. | + +__Note:__ The migration from Elasticsearch's TransportClient to the Java REST client has resulted in some existing properties to change. Below is a mapping of the old properties to the new ones: + +| Old Property Name | New Property Name | +|----------------------------------------|-------------------------------------| +| client.transport.ping_timeout | n/a | +| n/a | connection.timeout.millis | +| n/a | socket.timeout.millis | +| n/a | max.retry.timeout.millis | +| n/a | num.client.connection.threads | +| es.client.class | n/a | +| es.xpack.username | xpack.username | +| es.xpack.password.file | xpack.password.file | +| xpack.security.transport.ssl.enabled | ssl.enabled | +| xpack.ssl.key | n/a | +| xpack.ssl.certificate | n/a | +| xpack.ssl.certificate_authorities | n/a | +| n/a | keystore.type | +| keystore.path | keystore.path | +| n/a | keystore.password.file | + +__Notes:__ +* The transport client implementation provides for a 'xpack.security.user' property, however we never used this property directly. Rather, in order to secure the password we used custom properties for user/pass. These properties have been carried over as `xpack.username` and `xpack.password.file`. +* See [https://www.elastic.co/guide/en/elasticsearch/client/java-rest/5.6/_common_configuration.html](https://www.elastic.co/guide/en/elasticsearch/client/java-rest/5.6/_common_configuration.html) for more specifics on the new client properties. +* Other notes on JSSE - [https://docs.oracle.com/javase/8/docs/technotes/guides/security/jsse/JSSERefGuide.html](https://docs.oracle.com/javase/8/docs/technotes/guides/security/jsse/JSSERefGuide.html) + ## Upgrading to 5.6.2 Users should be prepared to re-index when migrating from Elasticsearch 2.3.3 to 5.6.2. There are a number of template changes, most notably around @@ -269,9 +312,27 @@ Notes on other settings for types in ES * [https://www.elastic.co/guide/en/elasticsearch/reference/5.6/breaking_50_mapping_changes.html](https://www.elastic.co/guide/en/elasticsearch/reference/5.6/breaking_50_mapping_changes.html) * [https://www.elastic.co/blog/strings-are-dead-long-live-strings](https://www.elastic.co/blog/strings-are-dead-long-live-strings) +### Metron Properties + +Metron depends on some internal fields being defined in sensor templates. A field is defined in Elasticsearch by adding an entry to the `properties` section of the template: +``` +"properties": { + "metron_field": { + "type": "keyword" + } +} +``` + +The following is a list of properties that need to be defined along with their type: +* source:type - keyword +* alert_status - keyword +* metron_alert - nested + ## Using Metron with Elasticsearch 5.6.2 -There is a requirement that all sensors templates have a nested `metron_alert` field defined. This field is a dummy field. See [Ignoring Unmapped Fields](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-sort.html#_ignoring_unmapped_fields) for more information +Although infrequent, sometimes an internal field is added in Metron and existing templates must be updated. The following steps outlines how to do this, using `metron_alert` as an example. + +With the addition of the meta alert feature, there is a requirement that all sensors templates have a nested `metron_alert` field defined. This field is a dummy field. See [Ignoring Unmapped Fields](https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-sort.html#_ignoring_unmapped_fields) for more information Without this field, an error will be thrown during ALL searches (including from UIs, resulting in no alerts being found for any sensor). This error will be found in the REST service's logs. http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-platform/metron-elasticsearch/pom.xml ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/pom.xml b/metron-platform/metron-elasticsearch/pom.xml index 593e80b..e3cf840 100644 --- a/metron-platform/metron-elasticsearch/pom.xml +++ b/metron-platform/metron-elasticsearch/pom.xml @@ -217,34 +217,6 @@ <type>test-jar</type> <scope>test</scope> </dependency> - <dependency> - <groupId>org.apache.logging.log4j</groupId> - <artifactId>log4j-api</artifactId> - <version>2.8.2</version> - <scope>test</scope> - </dependency> - <dependency> - <groupId>org.apache.logging.log4j</groupId> - <artifactId>log4j-core</artifactId> - <version>2.8.2</version> - <scope>test</scope> - </dependency> - <!--dependency> - <groupId>org.apache.logging.log4j</groupId> - <artifactId>log4j-api</artifactId> - <version>${global_log4j_core_version}</version> - </dependency> - <dependency> - <groupId>org.apache.logging.log4j</groupId> - <artifactId>log4j-core</artifactId> - <version>${global_log4j_core_version}</version> - </dependency--> - <dependency> - <groupId>com.google.guava</groupId> - <artifactId>guava-testlib</artifactId> - <version>${global_guava_version}</version> - <scope>test</scope> - </dependency> </dependencies> <build> @@ -306,7 +278,6 @@ </excludes> </artifactSet> <transformers> - <transformer implementation="org.atteo.classindex.ClassIndexTransformer"/> <transformer implementation="org.apache.maven.plugins.shade.resource.DontIncludeResourceTransformer"> <resources> @@ -320,13 +291,15 @@ <!--transformer implementation="org.apache.maven.plugins.shade.resource.ApacheNoticeResourceTransformer"> <addHeader>false</addHeader> <projectName>${project.name}</projectName> - </transformer> + </transformer>--> <transformer - implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/--> + implementation="org.apache.maven.plugins.shade.resource.ServicesResourceTransformer"/> <transformer implementation="org.apache.maven.plugins.shade.resource.ManifestResourceTransformer"> <mainClass></mainClass> </transformer> + <!-- ClassIndexTransformer needs to go LAST. For some reason it will clobber other transformers from operating when it is put first --> + <transformer implementation="org.atteo.classindex.ClassIndexTransformer"/> </transformers> </configuration> </execution> http://git-wip-us.apache.org/repos/asf/metron/blob/8bf3b6ec/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/client/ElasticsearchClient.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/client/ElasticsearchClient.java b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/client/ElasticsearchClient.java new file mode 100644 index 0000000..d62a7c0 --- /dev/null +++ b/metron-platform/metron-elasticsearch/src/main/java/org/apache/metron/elasticsearch/client/ElasticsearchClient.java @@ -0,0 +1,245 @@ +/** + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package org.apache.metron.elasticsearch.client; + +import com.google.common.base.Joiner; +import com.google.common.base.Splitter; +import com.google.common.collect.Iterables; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import org.apache.commons.io.IOUtils; +import org.apache.commons.lang3.StringUtils; +import org.apache.http.HttpEntity; +import org.apache.http.entity.StringEntity; +import org.apache.metron.common.utils.JSONUtils; +import org.apache.metron.elasticsearch.utils.FieldMapping; +import org.apache.metron.elasticsearch.utils.FieldProperties; +import org.elasticsearch.client.Response; +import org.elasticsearch.client.RestClient; +import org.elasticsearch.client.RestHighLevelClient; + +/** + * Wrapper around the Elasticsearch REST clients. Exposes capabilities of the low and high-level clients. + * https://www.elastic.co/guide/en/elasticsearch/client/java-rest/5.6/java-rest-overview.html. Most, if not + * all of use in Metron would be focused through the high-level client. It handles marshaling/unmarshaling. + */ +public class ElasticsearchClient implements AutoCloseable{ + private RestClient lowLevelClient; + private RestHighLevelClient highLevelClient; + + /** + * Instantiate with ElasticsearchClientFactory. + * + * @param lowLevelClient + * @param highLevelClient + */ + public ElasticsearchClient(RestClient lowLevelClient, RestHighLevelClient highLevelClient) { + this.lowLevelClient = lowLevelClient; + this.highLevelClient = highLevelClient; + } + + /** + * Exposes an Elasticsearch low-level client. Prefer the high level client. + */ + public RestClient getLowLevelClient() { + return lowLevelClient; + } + + /** + * <p> + * Exposes an Elasticsearch high-level client. Prefer to use this client over the low-level client where possible. This client wraps the low-level + * client and exposes some additional sugar on top of the low level methods including marshaling/unmarshaling. + * </p> + * <p> + * Note, as of 5.6.2 it does NOT support index or cluster management operations. + * https://www.elastic.co/guide/en/elasticsearch/client/java-rest/5.6/_changing_the_application_8217_s_code.html + * <br> + * <i>Does not provide indices or cluster management APIs. Management operations can be executed by external scripts or using the low-level client.</i> + * </p> + * <p> + * Current supported ES API's seen here - https://www.elastic.co/guide/en/elasticsearch/client/java-rest/5.6/java-rest-high-supported-apis.html + * </p> + * + * <ul> + * <li>Single document APIs + * <ul> + * <li>Index API</li> + * <li>Get API</li> + * <li>Delete API</li> + * <li>Update API</li> + * </ul> + * </li> + * <li>Multi document APIs + * <ul> + * <li>Bulk API</li> + * </ul> + * </li> + * <li>Search APIs + * <ul> + * <li>Search API</li> + * <li>Search Scroll API</li> + * <li>Clear Scroll API</li> + * </ul> + * </li> + * <li>Miscellaneous APIs + * <ul> + * <li>Info API</li> + * </ul> + * </li> + * </ul> + */ + public RestHighLevelClient getHighLevelClient() { + return highLevelClient; + } + + /** + * Included as part of AutoCloseable because Elasticsearch recommends closing the client when not + * being used. + * https://www.elastic.co/guide/en/elasticsearch/client/java-rest/5.6/_changing_the_client_8217_s_initialization_code.html + * @throws IOException + */ + @Override + public void close() throws IOException { + if (lowLevelClient != null) { + lowLevelClient.close(); + } + } + + /** + * https://www.elastic.co/guide/en/elasticsearch/reference/5.6/indices-put-mapping.html + * @param index + * @param mappingType https://www.elastic.co/guide/en/elasticsearch/reference/5.6/mapping.html#mapping-type + * @param mapping + * @throws IOException + */ + public void putMapping(String index, String mappingType, String mapping) throws IOException { + HttpEntity entity = new StringEntity(mapping); + Response response = lowLevelClient.performRequest("PUT" + , "/" + index + "/_mapping/" + mappingType + , Collections.emptyMap() + , entity + ); + + if(response.getStatusLine().getStatusCode() != 200) { + String responseStr = IOUtils.toString(response.getEntity().getContent()); + throw new IllegalStateException("Got a " + response.getStatusLine().getStatusCode() + " due to " + responseStr); + } + } + + /** + * Gets ALL Elasticsearch indices, or null if status code returned is not OK 200. + */ + public String[] getIndices() throws IOException { + Response response = lowLevelClient.performRequest("GET", "/_cat/indices"); + if(response.getStatusLine().getStatusCode() == 200) { + String responseStr = IOUtils.toString(response.getEntity().getContent()); + List<String> indices = new ArrayList<>(); + for(String line : Splitter.on("\n").split(responseStr)) { + Iterable<String> splits = Splitter.on(" ").split(line.replaceAll("\\s+", " ").trim()); + if(Iterables.size(splits) > 3) { + String index = Iterables.get(splits, 2, ""); + if(!StringUtils.isEmpty(index)) { + indices.add(index.trim()); + } + } + } + String[] ret = new String[indices.size()]; + ret=indices.toArray(ret); + return ret; + } + return null; + } + + /** + * Gets FieldMapping detail for a list of indices. + * + * @param indices get field mapppings for the provided indices + * @return mapping of index name to FieldMapping + */ + public Map<String, FieldMapping> getMappingByIndex(String[] indices) throws IOException { + Map<String, FieldMapping> ret = new HashMap<>(); + String indicesCsv = Joiner.on(",").join(indices); + Response response = lowLevelClient.performRequest("GET", "/" + indicesCsv + "/_mapping"); + if(response.getStatusLine().getStatusCode() == 200) { + String responseStr = IOUtils.toString(response.getEntity().getContent()); + Map<String, Object> indexToMapping = JSONUtils.INSTANCE.load(responseStr, JSONUtils.MAP_SUPPLIER); + for(Map.Entry<String, Object> index2Mapping : indexToMapping.entrySet()) { + String index = index2Mapping.getKey(); + Map<String, Object> mappings = getInnerMap((Map<String, Object>)index2Mapping.getValue(), "mappings"); + if(mappings.size() > 0) { + Map.Entry<String, Object> docMap = Iterables.getFirst(mappings.entrySet(), null); + if(docMap != null) { + Map<String, Object> fieldPropertiesMap = getInnerMap((Map<String, Object>)docMap.getValue(), "properties"); + if(fieldPropertiesMap != null) { + FieldMapping mapping = new FieldMapping(); + for (Map.Entry<String, Object> field2PropsKV : fieldPropertiesMap.entrySet()) { + if(field2PropsKV.getValue() != null) { + FieldProperties props = new FieldProperties((Map<String, Object>) field2PropsKV.getValue()); + mapping.put(field2PropsKV.getKey(), props); + } + } + ret.put(index, mapping); + } + } + } + } + } + return ret; + } + + /** + * Traverses the outer map to retrieve a leaf map by iteratively calling get(key) using the provided keys in order. e.g. + * for an outer map provided as follows: + * <pre> + * { + * "foo" : { + * "bar" : { + * "baz" : { + * "hello" : "world" + * } + * } + * } + * } + * </pre> + * calling getInnerMap(outerMap, new String[] { "foo", "bar", "baz" }) would return the following: + * <pre> + * {hello=world} + * </pre> + * @param outerMap Complex map of nested keys/values + * @param keys ordered list of keys to iterate over to grab a leaf mapping. + * @return leaf node, or innermost matching node from outerMap if no leaf exists + */ + private Map<String, Object> getInnerMap(Map<String, Object> outerMap, String... keys) { + Map<String, Object> ret = outerMap; + if(keys.length == 0) { + return outerMap; + } + for(String key : keys) { + ret = (Map<String, Object>)ret.get(key); + if(ret == null) { + return ret; + } + } + return ret; + } + +}
