[26/50] [abbrv] metron git commit: METRON-1713 PCAP UI - Add a way to kill a pcap job (tiborm via merrimanr) closes apache/metron#1143
METRON-1713 PCAP UI - Add a way to kill a pcap job (tiborm via merrimanr)
closes apache/metron#1143
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/14dcb2d9
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/14dcb2d9
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/14dcb2d9
Branch: refs/heads/feature/METRON-1699-create-batch-profiler
Commit: 14dcb2d90581835d8206c65918c24e8cb04bfd06
Parents: 5b3e2c3
Author: tiborm
Authored: Thu Aug 9 08:33:21 2018 -0500
Committer: rmerriman
Committed: Thu Aug 9 08:33:21 2018 -0500
--
.../pcap/pcap-panel/pcap-panel.component.html | 7 +-
.../pcap/pcap-panel/pcap-panel.component.scss | 33 +
.../pcap-panel/pcap-panel.component.spec.ts | 130 +++
.../app/pcap/pcap-panel/pcap-panel.component.ts | 42 +-
.../src/app/pcap/service/pcap.service.ts| 10 +-
5 files changed, 215 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/14dcb2d9/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.html
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.html
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.html
index 950f49c..0dda268 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.html
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.html
@@ -15,8 +15,11 @@
-
-{{progressWidth}}%
+
+
+ {{progressWidth}}%
+
+
{{ errorMsg }}
http://git-wip-us.apache.org/repos/asf/metron/blob/14dcb2d9/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.scss
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.scss
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.scss
index 8989bf9..523f5ce 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.scss
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.scss
@@ -31,4 +31,37 @@
.progress-bar {
width: 0;
+ height: 34px;
+ line-height: 34px;
+ vertical-align: middle;
+ font-size: 0.875rem;
+}
+
+.pcap-progress-wrapper {
+ position: relative;
+ padding-right: 55px;
+}
+
+.pcap-cancel-query-button {
+ position: absolute;
+ top: 0;
+ right: 0;
+ padding-top: 6px;
+ padding-bottom: 6px;
+ background: $icon-button-background;
+ min-width: 42px;
+ padding-left: 0;
+ padding-right: 0;
+ border: 1px solid $blue-chill !important;
+ cursor: pointer;
+
+ &:focus {
+box-shadow: none;
+ }
+
+ &::before {
+font-family: "FontAwesome";
+content: '\f00d';
+color: $piction-blue;
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/14dcb2d9/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.spec.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.spec.ts
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.spec.ts
index 0804b79..9dacc7f 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.spec.ts
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.spec.ts
@@ -50,6 +50,9 @@ class FakePcapService {
return '';
}
submitRequest() {}
+ cancelQuery() {
+return defer(() => Promise.resolve());
+ }
}
describe('PcapPanelComponent', () => {
@@ -327,6 +330,133 @@ describe('PcapPanelComponent', () => {
expect(fixture.debugElement.query(By.css('app-pcap-list'))).toBeDefined();
}));
+ it('should render a cancel button only if a query runs', () => {
+component.queryRunning = false;
+fixture.detectChanges();
+
expect(fixture.debugElement.query(By.css('[data-qe-id="pcap-cancel-query-button"]'))).toBeFalsy();
+
+component.queryRunning = true;
+fixture.detectChanges();
+
expect(fixture.debugElement.query(By.css('[data-qe-id="pcap-cancel-query-button"]'))).toBeDefined();
+ });
+
+ it('should hide the progress bar if the user clicks on the cancel button',
fakeAsync(() => {
+component.queryRunning = true;
+component.queryId = 'testid';
+fixture.detectChanges();
+expect(fixture.debugElement.query(By.css('.pcap-progress'))).toBeDefined();
+
+const cancelBtn =
fixture.debugElement.query(By.css('[data-qe-id="pcap-cancel-query-button"]'));
+const cancelBtnEl = cancelBtn.nativeElement;
+
+cancelBtnEl.click();
+tick();
+
[14/50] [abbrv] metron git commit: METRON-1662 PCAP UI - Downloading PCAP page files (tiborm via merrimanr) closes apache/metron#1118
METRON-1662 PCAP UI - Downloading PCAP page files (tiborm via merrimanr) closes
apache/metron#1118
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/b445bfe2
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/b445bfe2
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/b445bfe2
Branch: refs/remotes/apache/feature/METRON-1699-create-batch-profiler
Commit: b445bfe242123e544c1fef41e2ab5bdc07dbc56e
Parents: 2b6959b
Author: tiborm
Authored: Fri Aug 3 08:13:11 2018 -0500
Committer: rmerriman
Committed: Fri Aug 3 08:13:11 2018 -0500
--
.../metron-alerts/src/app/pcap/model/pdml.ts| 12 +++-
.../pcap/pcap-packet/pcap-packet.component.ts | 9 ++
.../pcap/pcap-panel/pcap-panel.component.html | 2 +-
.../app/pcap/pcap-panel/pcap-panel.component.ts | 30 +++-
.../src/app/pcap/service/pcap.service.ts| 15 ++
5 files changed, 33 insertions(+), 35 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/b445bfe2/metron-interface/metron-alerts/src/app/pcap/model/pdml.ts
--
diff --git a/metron-interface/metron-alerts/src/app/pcap/model/pdml.ts
b/metron-interface/metron-alerts/src/app/pcap/model/pdml.ts
index 97fc347..34245c6 100644
--- a/metron-interface/metron-alerts/src/app/pcap/model/pdml.ts
+++ b/metron-interface/metron-alerts/src/app/pcap/model/pdml.ts
@@ -17,8 +17,7 @@
*/
export class PdmlField {
- //public $: any[]
-public name: string;
+ public name: string;
public fields?: PdmlField[]
public show: string;
public value: string;
@@ -26,9 +25,8 @@
}
export class PdmlProto {
- //public $: any
-public name: string;
-public showname: string;
+ public name: string;
+ public showname: string;
public fields: PdmlField[]
public static findField(p: PdmlProto, name: string): PdmlField {
@@ -36,10 +34,8 @@ export class PdmlProto {
}
}
-
export class PdmlPacket {
- //public $: any
-public name: string;
+ public name: string;
public protos: PdmlProto[]
public expanded: boolean = false
}
http://git-wip-us.apache.org/repos/asf/metron/blob/b445bfe2/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.ts
b/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.ts
index f1bbe2d..ca28a39 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.ts
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.ts
@@ -24,16 +24,11 @@ import { PdmlPacket } from '../model/pdml'
templateUrl: './pcap-packet.component.html',
styleUrls: ['./pcap-packet.component.scss']
})
-export class PcapPacketComponent implements OnInit {
+export class PcapPacketComponent {
@Input() packet: PdmlPacket
- constructor() { }
-
- ngOnInit() {
- }
-
toggle() {
-this.packet.expanded = !this.packet.expanded
+this.packet.expanded = !this.packet.expanded;
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/b445bfe2/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.html
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.html
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.html
index b373914..1fd250f 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.html
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.html
@@ -27,5 +27,5 @@
-
+ Download PCAP
http://git-wip-us.apache.org/repos/asf/metron/blob/b445bfe2/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.ts
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.ts
index fd49ec7..4114e24 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.ts
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.ts
@@ -15,47 +15,45 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-import { Component, OnInit, Input } from '@angular/core';
+import { Component, Input } from '@angular/core';
-import { PcapService, PcapStatusResponse } from '../service/pcap.service'
-import { PcapRequest } from '../model/pcap.request'
-import {
[30/50] [abbrv] metron git commit: METRON-1722 PcapCLI should print progress to stdout (merrimanr) closes apache/metron#1138
METRON-1722 PcapCLI should print progress to stdout (merrimanr) closes
apache/metron#1138
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/b29833cc
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/b29833cc
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/b29833cc
Branch: refs/remotes/apache/feature/METRON-1699-create-batch-profiler
Commit: b29833cc5db4efca2a6e9025042baf705fb347c6
Parents: d8a5922
Author: merrimanr
Authored: Fri Aug 10 08:25:13 2018 -0500
Committer: rmerriman
Committed: Fri Aug 10 08:25:13 2018 -0500
--
metron-platform/metron-pcap-backend/README.md | 2 ++
.../java/org/apache/metron/pcap/query/CliParser.java| 4
.../java/org/apache/metron/pcap/query/PcapCliTest.java | 12 +---
.../java/org/apache/metron/pcap/config/PcapConfig.java | 9 +
.../java/org/apache/metron/pcap/config/PcapOptions.java | 3 ++-
.../main/java/org/apache/metron/pcap/mr/PcapJob.java| 11 +++
6 files changed, 37 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/b29833cc/metron-platform/metron-pcap-backend/README.md
--
diff --git a/metron-platform/metron-pcap-backend/README.md
b/metron-platform/metron-pcap-backend/README.md
index e7960e3..2ff20d8 100644
--- a/metron-platform/metron-pcap-backend/README.md
+++ b/metron-platform/metron-pcap-backend/README.md
@@ -132,6 +132,7 @@ usage: Fixed filter options
-nr,--num_reducers The number of reducers to use. Default
is 10.
-h,--help Display help
+ -ps,--print_status Print the status of the job as it runs
-ir,--include_reverse Indicates if filter should check swapped
src/dest addresses and IPs
-p,--protocol IP Protocol
@@ -154,6 +155,7 @@ usage: Query filter options
-nr,--num_reducers The number of reducers to use. Default
is 10.
-h,--help Display help
+ -ps,--print_status Print the status of the job as it runs
-q,--query Query string to use as a filter
-st,--start_time (required) Packet start time range.
```
http://git-wip-us.apache.org/repos/asf/metron/blob/b29833cc/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
--
diff --git
a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
index 2d15e8b..69c725c 100644
---
a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
+++
b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
@@ -55,6 +55,7 @@ public class CliParser {
options.addOption(newOption("rpf", "records_per_file", true,
String.format("Number of records to include in each output pcap file (defaults
to %s)", NUM_RECORDS_PER_FILE_DEFAULT)));
options.addOption(newOption("et", "end_time", true, "Packet end time
range. Default is current system time."));
options.addOption(newOption("df", "date_format", true, "Date format to use
for parsing start_time and end_time. Default is to use time in millis since the
epoch."));
+options.addOption(newOption("ps", "print_status", false, "Print the status
of the job as it runs"));
return options;
}
@@ -125,6 +126,9 @@ public class CliParser {
//no-op
}
}
+if (commandLine.hasOption("print_status")) {
+ config.setPrintJobStatus(true);
+}
}
public void printHelp(String msg, Options opts) {
http://git-wip-us.apache.org/repos/asf/metron/blob/b29833cc/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
--
diff --git
a/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
b/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
index 96ca354..7c75224 100644
---
a/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
+++
b/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
@@ -147,7 +147,8 @@ public class PcapCliTest {
"-protocol", "6",
"-include_reverse",
"-num_reducers", "10",
-"-records_per_file", "1000"
+"-records_per_file", "1000",
+"-ps"
[18/50] [abbrv] metron git commit: METRON-1675 PCAP UI - Introduce the paging capability (sardell via merrimanr) closes apache/metron#1121
METRON-1675 PCAP UI - Introduce the paging capability (sardell via merrimanr)
closes apache/metron#1121
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/037b50b9
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/037b50b9
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/037b50b9
Branch: refs/remotes/apache/feature/METRON-1699-create-batch-profiler
Commit: 037b50b9a829d7292db1148fc628d9231da499e1
Parents: 756ceed
Author: sardell
Authored: Tue Aug 7 09:04:33 2018 -0500
Committer: rmerriman
Committed: Tue Aug 7 09:04:33 2018 -0500
--
.../src/app/pcap/model/pcap-pagination.ts | 21 ++
.../src/app/pcap/model/pcap-status-response.ts | 23 ++
.../src/app/pcap/model/pcap.request.ts | 2 +-
.../app/pcap/pcap-list/pcap-list.component.html | 1 +
.../pcap/pcap-list/pcap-list.component.spec.ts | 11 +++
.../app/pcap/pcap-list/pcap-list.component.ts | 18 +++--
.../pcap-pagination.component.html | 19 +
.../pcap-pagination.component.scss | 40 ++
.../pcap-pagination.component.spec.ts | 78
.../pcap-pagination.component.ts| 41 ++
.../pcap/pcap-panel/pcap-panel.component.html | 7 +-
.../pcap-panel/pcap-panel.component.spec.ts | 2 +
.../app/pcap/pcap-panel/pcap-panel.component.ts | 27 +--
.../metron-alerts/src/app/pcap/pcap.module.ts | 10 +--
.../metron-alerts/src/app/pcap/pcap.routing.ts | 2 +-
.../src/app/pcap/service/pcap.service.spec.ts | 2 +-
.../src/app/pcap/service/pcap.service.ts| 10 +--
17 files changed, 279 insertions(+), 35 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/037b50b9/metron-interface/metron-alerts/src/app/pcap/model/pcap-pagination.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/model/pcap-pagination.ts
b/metron-interface/metron-alerts/src/app/pcap/model/pcap-pagination.ts
new file mode 100644
index 000..ae7619b
--- /dev/null
+++ b/metron-interface/metron-alerts/src/app/pcap/model/pcap-pagination.ts
@@ -0,0 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export class PcapPagination {
+ selectedPage = 1;
+ total: number;
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/037b50b9/metron-interface/metron-alerts/src/app/pcap/model/pcap-status-response.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/model/pcap-status-response.ts
b/metron-interface/metron-alerts/src/app/pcap/model/pcap-status-response.ts
new file mode 100644
index 000..d4d9a5e
--- /dev/null
+++ b/metron-interface/metron-alerts/src/app/pcap/model/pcap-status-response.ts
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export class PcapStatusResponse {
+ jobId: string;
+ jobStatus: string;
+ percentComplete: number;
+ pageTotal: number;
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/037b50b9/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
--
diff --git
[metron] Git Push Summary
Repository: metron Updated Branches: refs/heads/metron-1699-create-batch-profiler [deleted] f5eaef3c2
[10/50] [abbrv] metron git commit: METRON-1671 Create PCAP UI (tiborm via merrimanr) closes apache/metron#1103
http://git-wip-us.apache.org/repos/asf/metron/blob/d5eb56a9/metron-interface/metron-alerts/src/app/app-routing.module.ts
--
diff --git a/metron-interface/metron-alerts/src/app/app-routing.module.ts
b/metron-interface/metron-alerts/src/app/app-routing.module.ts
index 8357d66..56ad41c 100644
--- a/metron-interface/metron-alerts/src/app/app-routing.module.ts
+++ b/metron-interface/metron-alerts/src/app/app-routing.module.ts
@@ -26,7 +26,8 @@ const routes: Routes = [
{ path: 'alerts-list', loadChildren:
'app/alerts/alerts-list/alerts-list.module#AlertsListModule', canActivate:
[AuthGuard]},
{ path: 'save-search', loadChildren:
'app/alerts/save-search/save-search.module#SaveSearchModule', canActivate:
[AuthGuard]},
{ path: 'saved-searches', loadChildren:
'app/alerts/saved-searches/saved-searches.module.ts#SavedSearchesModule',
- canActivate: [AuthGuard]}
+ canActivate: [AuthGuard]},
+ { path: 'pcap', loadChildren: 'app/pcap/pcap.module#PcapModule',
canActivate: [AuthGuard] }
];
@NgModule({
http://git-wip-us.apache.org/repos/asf/metron/blob/d5eb56a9/metron-interface/metron-alerts/src/app/app.component.html
--
diff --git a/metron-interface/metron-alerts/src/app/app.component.html
b/metron-interface/metron-alerts/src/app/app.component.html
index 8867a9b..1b992ca 100644
--- a/metron-interface/metron-alerts/src/app/app.component.html
+++ b/metron-interface/metron-alerts/src/app/app.component.html
@@ -16,6 +16,14 @@
+
+
+Alerts
+
+
+PCAP
+
+
Logged in as {{authService.currentUser}} -
Logout
http://git-wip-us.apache.org/repos/asf/metron/blob/d5eb56a9/metron-interface/metron-alerts/src/app/app.component.scss
--
diff --git a/metron-interface/metron-alerts/src/app/app.component.scss
b/metron-interface/metron-alerts/src/app/app.component.scss
index 8358e7f..8505ee8 100644
--- a/metron-interface/metron-alerts/src/app/app.component.scss
+++ b/metron-interface/metron-alerts/src/app/app.component.scss
@@ -27,6 +27,24 @@
max-height: 50px;
}
+.nav-link
+{
+ padding-bottom: 0;
+ padding-top: 0;
+ color: inherit;
+}
+
+.nav-item.active
+{
+ border-bottom: 3px solid #32abe2;
+ margin-bottom: 5px;
+}
+
+.nav-link.active
+{
+ color: #ff;
+}
+
.logout {
padding-left: 10px;
}
http://git-wip-us.apache.org/repos/asf/metron/blob/d5eb56a9/metron-interface/metron-alerts/src/app/app.module.ts
--
diff --git a/metron-interface/metron-alerts/src/app/app.module.ts
b/metron-interface/metron-alerts/src/app/app.module.ts
index 227eaab..82b4970 100644
--- a/metron-interface/metron-alerts/src/app/app.module.ts
+++ b/metron-interface/metron-alerts/src/app/app.module.ts
@@ -46,8 +46,7 @@ import {MetaAlertService} from './service/meta-alert.service';
import {MetaAlertsModule} from './alerts/meta-alerts/meta-alerts.module';
import {SearchService} from './service/search.service';
import { GlobalConfigService } from './service/global-config.service';
-
-
+import {PcapModule} from './pcap/pcap.module';
export function initConfig(config: ColumnNamesService) {
return () => config.list();
@@ -70,7 +69,8 @@ export function initConfig(config: ColumnNamesService) {
ConfigureRowsModule,
SaveSearchModule,
SavedSearchesModule,
-SwitchModule
+SwitchModule,
+PcapModule
],
providers: [{ provide: APP_INITIALIZER, useFactory: initConfig, deps:
[ColumnNamesService], multi: true },
{ provide: DataSource, useClass: ElasticSearchLocalstorageImpl },
http://git-wip-us.apache.org/repos/asf/metron/blob/d5eb56a9/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
--
diff --git a/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
b/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
new file mode 100644
index 000..3a27714
--- /dev/null
+++ b/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing software
+ * distributed under the License is distributed on an "AS
[03/50] [abbrv] metron git commit: METRON-1690: Add more context to PcapJob JobStatus (mmiklavc via mmiklavc) closes apache/metron#1128
METRON-1690: Add more context to PcapJob JobStatus (mmiklavc via mmiklavc) closes apache/metron#1128 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/9d4842f3 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/9d4842f3 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/9d4842f3 Branch: refs/heads/feature/METRON-1699-create-batch-profiler Commit: 9d4842f3d6f355d449d51f3c6ac02f2c904c295d Parents: d0e8757 Author: mmiklavc Authored: Wed Jul 25 08:21:32 2018 -0600 Committer: Michael Miklavcic Committed: Wed Jul 25 08:21:32 2018 -0600 -- .../metron-job_state_statechart_diagram.svg | 18 +- .../metron-job_state_statechart_diagram.xml | 18 +- .../java/org/apache/metron/job/JobStatus.java | 42 - .../org/apache/metron/job/JobStatusTest.java| 55 ++ .../job/manager/InMemoryJobManagerTest.java | 16 ++ .../org/apache/metron/pcap/PcapJobTest.java | 80 +--- .../PcapTopologyIntegrationTest.java| 30 +-- .../java/org/apache/metron/pcap/mr/PcapJob.java | 186 +-- site-book/bin/generate-md.sh| 2 + 9 files changed, 294 insertions(+), 153 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/9d4842f3/metron-platform/metron-job/metron-job_state_statechart_diagram.svg -- diff --git a/metron-platform/metron-job/metron-job_state_statechart_diagram.svg b/metron-platform/metron-job/metron-job_state_statechart_diagram.svg index a99c5ad..c801eae 100644 --- a/metron-platform/metron-job/metron-job_state_statechart_diagram.svg +++ b/metron-platform/metron-job/metron-job_state_statechart_diagram.svg @@ -1,14 +1,14 @@ http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd;> -http://www.w3.org/2000/svg; xmlns:xlink="http://www.w3.org/1999/xlink; width="661px" height="291px" version="1.1" style="background-color: rgb(255, 255, 255);">http://www.w3.org/TR/SVG11/feature#Extensibility;>http://www.w3.org/1999/xhtml; style="display: inline-block; font-size: 12px; font-family: Verdana; color: rgb(0, 0, 0); line-height: 1.2; vertical-align: top; width: 92px; white-space: nowrap; word-wrap: normal; text-align: center;">http://www.w3.org/1999/xhtml; style="display:inline-block;text-align:inherit;text-dec oration:inherit;">NOT_RUNNINGNOT_RUNNINGhttp://www.w3.org/TR/SVG11/feature#Extensibility;>http://www.w3.org/1999/xhtml; style="display: inline-block; font-size: 12px; font-family: Verdana; color: rgb(0, 0, 0); line-height: 1.2; vertical-align: top; width: 60px; white-space: nowrap; word-wrap: normal; text-align: center;">http://ww w.w3.org/1999/xhtml" style="display:inline-block;text-align:inherit;text-decoration:inherit;">RUNNINGRUNNINGhttp://www.w3.org/TR/SVG11/feature#Extensibility;>http://www.w3.org/1999/xhtml; style="display: inline-block; font-size: 12px; font-family: Verdana; color: rgb(0, 0, 0); line-height: 1.2; vertical-align: top; width: 74px; white-space: nowrap; word-wrap: normal; text-align: center;">http://www.w3.org/1999/xhtml; style="display:inline-block;text-align:inherit;text-decoration:inherit;">FINALIZINGFINALIZINGhttp://www.w3.org/TR/SVG11/feature#Extensibility;>http://www.w3.org/1999/xhtml; style="display: inline-block; font-size: 12px; font-family: Verdana; color: rgb(0, 0, 0); line-height: 1.2; vertical-align: top; width: 44px; white-space: nowrap; word-wrap: normal; text-align: center;">http://www.w3.org/1999/xhtml; style="display:inline-block;text-align:inherit;text-decoration:inherit;">KILLEDKILLEDhttp://www.w3.org/TR/SVG11/feature#Extensibility;>http://www.w3.org/1999/xhtml; style="display: inline-block; font-size: 12px; font-family: Verdana; color: rgb(0, 0, 0); line-height: 1.2; vertical-align: top; width: 44px; white-space: nowrap; word-wrap: normal; text-align: center;">http://www.w3.org/1999/xhtml; style="display:inline-block;text-align:inherit;text-decoration:inherit;">FAILEDFAILEDhttp://www.w3.org/TR/SVG11/feature#Extensibility;>http://www.w3.org/1999/xhtml; style="display: inline-block; font-size: 12px; font-family: Verdana; color: rgb(0, 0, 0); line-height: 1.2; vertical-align: top; width: 76px; white-space: nowrap; word-wrap: normal; text-align: center;">http://www.w3.org/1999/xhtml; style="display:inline-block;text-align:inherit;text-decoration:inherit;">SUCCEEDEDSUCCEEDED +http://www.w3.org/2000/svg; xmlns:xlink="http://www.w3.org/1999/xlink; width="656px" height="291px" version="1.1">http://www.w3.org/TR/SVG11/feature#Extensibility;>http://www.w3.org/1999/xhtml; style="display: inline-block; font-size: 12px; font-family: Verdana; color: rgb(0, 0, 0); line-height: 1.2; vertical-align: top; width: 92px; white-space: nowrap; word-wrap: normal; text-align:
[25/50] [abbrv] metron git commit: METRON-1723 PCAP UI - Unable to select/copy from packets details in PCAP query panel (sardell via merrimanr) closes apache/metron#1139
METRON-1723 PCAP UI - Unable to select/copy from packets details in PCAP query
panel (sardell via merrimanr) closes apache/metron#1139
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/5b3e2c33
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/5b3e2c33
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/5b3e2c33
Branch: refs/heads/feature/METRON-1699-create-batch-profiler
Commit: 5b3e2c337b3772bcd7d18cedac75877bbe6f7735
Parents: 52de126
Author: sardell
Authored: Thu Aug 9 07:44:15 2018 -0500
Committer: rmerriman
Committed: Thu Aug 9 07:44:15 2018 -0500
--
.../src/app/pcap/pcap-packet/pcap-packet.component.html | 2 +-
.../src/app/pcap/pcap-packet/pcap-packet.component.scss | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/5b3e2c33/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.html
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.html
b/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.html
index f7d9dbe..dd9afa1 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.html
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.html
@@ -13,7 +13,7 @@
-->
{{ proto.showname
}}
-
+
{{ field.name
}}
{{ field.showname }}
http://git-wip-us.apache.org/repos/asf/metron/blob/5b3e2c33/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.scss
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.scss
b/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.scss
index e9d4662..02ce7b6 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.scss
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.scss
@@ -27,10 +27,10 @@
}
.proto-fields {
- margin-left: 1em;
+ cursor: text;
+ padding-left: 1em;
}
-
.field-name {
display: none
}
[09/50] [abbrv] metron git commit: METRON-1671 Create PCAP UI (tiborm via merrimanr) closes apache/metron#1103
http://git-wip-us.apache.org/repos/asf/metron/blob/d5eb56a9/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.spec.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.spec.ts
b/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.spec.ts
new file mode 100644
index 000..155e1f8
--- /dev/null
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.spec.ts
@@ -0,0 +1,1283 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { async, ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { PcapPacketComponent } from './pcap-packet.component';
+import { PdmlPacket } from '../model/pdml';
+
+describe('PcapPacketComponent', () => {
+ let component: PcapPacketComponent;
+ let fixture: ComponentFixture;
+
+ beforeEach(async(() => {
+TestBed.configureTestingModule({
+ declarations: [ PcapPacketComponent ]
+})
+.compileComponents();
+ }));
+
+ beforeEach(() => {
+fixture = TestBed.createComponent(PcapPacketComponent);
+component = fixture.componentInstance;
+component.packet = fakePacket as PdmlPacket;
+fixture.detectChanges();
+ });
+
+ it('should create', () => {
+expect(component).toBeTruthy();
+ });
+});
+
+const fakePacket = {
+ name: "something",
+ expanded: false,
+ protos: [
+{
+ "name": "geninfo",
+ "pos": "0",
+ "showname": "General information",
+ "size": "722",
+ "hide": null,
+ "fields": [
+{
+ "name": "num",
+ "pos": "0",
+ "showname": "Number",
+ "size": "722",
+ "value": "1",
+ "show": "1",
+ "unmaskedvalue": null,
+ "hide": null,
+ "fields": null,
+ "protos": null
+},
+{
+ "name": "len",
+ "pos": "0",
+ "showname": "Frame Length",
+ "size": "722",
+ "value": "2d2",
+ "show": "722",
+ "unmaskedvalue": null,
+ "hide": null,
+ "fields": null,
+ "protos": null
+},
+{
+ "name": "caplen",
+ "pos": "0",
+ "showname": "Captured Length",
+ "size": "722",
+ "value": "2d2",
+ "show": "722",
+ "unmaskedvalue": null,
+ "hide": null,
+ "fields": null,
+ "protos": null
+},
+{
+ "name": "timestamp",
+ "pos": "0",
+ "showname": "Captured Time",
+ "size": "722",
+ "value": "1458240269.373968000",
+ "show": "Mar 17, 2016 18:44:29.373968000 UTC",
+ "unmaskedvalue": null,
+ "hide": null,
+ "fields": null,
+ "protos": null
+}
+ ]
+},
+{
+ "name": "frame",
+ "pos": "0",
+ "showname": "Frame 1: 722 bytes on wire (5776 bits), 722 bytes captured
(5776 bits)",
+ "size": "722",
+ "hide": null,
+ "fields": [
+{
+ "name": "frame.dlt",
+ "pos": "0",
+ "showname": "WTAP_ENCAP: 1",
+ "size": "0",
+ "value": null,
+ "show": "1",
+ "unmaskedvalue": null,
+ "hide": null,
+ "fields": null,
+ "protos": null
+},
+{
+ "name": "frame.time",
+ "pos": "0",
+ "showname": "Arrival Time: Mar 17, 2016 18:44:29.373968000 UTC",
+ "size": "0",
+ "value": null,
+ "show": "Mar 17, 2016 18:44:29.373968000",
+ "unmaskedvalue": null,
+ "hide": null,
+ "fields": null,
+ "protos": null
+},
+{
+ "name": "frame.offset_shift",
+ "pos": "0",
+ "showname": "Time shift for this packet: 0.0 seconds",
+ "size": "0",
+ "value": null,
+ "show": "0.0",
+ "unmaskedvalue": null,
+ "hide": null,
+ "fields": null,
+ "protos": null
+},
+{
+ "name": "frame.time_epoch",
+ "pos": "0",
+
[36/50] [abbrv] metron git commit: METRON-1735 Empty print status option causes NPE (merrimanr) closes apache/metron#1160
METRON-1735 Empty print status option causes NPE (merrimanr) closes
apache/metron#1160
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/8a926dd5
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/8a926dd5
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/8a926dd5
Branch: refs/heads/feature/METRON-1699-create-batch-profiler
Commit: 8a926dd5a4501df8c7ef51b526ad5e4d752a5e19
Parents: bce9b90
Author: merrimanr
Authored: Tue Aug 14 12:59:38 2018 -0500
Committer: rmerriman
Committed: Tue Aug 14 12:59:38 2018 -0500
--
.../org/apache/metron/pcap/query/CliParser.java | 4
.../apache/metron/pcap/query/PcapCliTest.java| 7 ++-
.../apache/metron/pcap/config/PcapConfig.java| 2 +-
.../java/org/apache/metron/pcap/mr/PcapJob.java | 18 +-
.../org/apache/metron/pcap/mr/PcapJobTest.java | 19 +++
5 files changed, 35 insertions(+), 15 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/8a926dd5/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
--
diff --git
a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
index 5040f90..b9a2a50 100644
---
a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
+++
b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
@@ -55,7 +55,6 @@ public class CliParser {
options.addOption(newOption("rpf", "records_per_file", true,
String.format("Number of records to include in each output pcap file (defaults
to %s)", NUM_RECORDS_PER_FILE_DEFAULT)));
options.addOption(newOption("et", "end_time", true, "Packet end time
range. Default is current system time."));
options.addOption(newOption("df", "date_format", true, "Date format to use
for parsing start_time and end_time. Default is to use time in millis since the
epoch."));
-options.addOption(newOption("ps", "print_status", false, "Print the status
of the job as it runs"));
options.addOption(newOption("yq", "yarn_queue", true, "Yarn queue this job
will be submitted to"));
return options;
}
@@ -127,9 +126,6 @@ public class CliParser {
//no-op
}
}
-if (commandLine.hasOption("print_status")) {
- config.setPrintJobStatus(true);
-}
if (commandLine.hasOption("yarn_queue")) {
config.setYarnQueue(commandLine.getOptionValue("yarn_queue"));
}
http://git-wip-us.apache.org/repos/asf/metron/blob/8a926dd5/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
--
diff --git
a/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
b/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
index a71e997..3e7aad2 100644
---
a/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
+++
b/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
@@ -172,8 +172,7 @@ public class PcapCliTest {
"-protocol", "6",
"-include_reverse",
"-num_reducers", "10",
-"-records_per_file", "1000",
-"-ps"
+"-records_per_file", "1000"
};
Map query = new HashMap() {{
put(Constants.Fields.SRC_ADDR.getName(), "192.168.1.1");
@@ -217,7 +216,6 @@ public class PcapCliTest {
"-include_reverse",
"-num_reducers", "10",
"-records_per_file", "1000",
-"-ps",
"-yq", "pcap"
};
Map query = new HashMap() {{
@@ -295,8 +293,7 @@ public class PcapCliTest {
"-base_path", "/base/path",
"-base_output_path", "/base/output/path",
"-query", "some query string",
-"-records_per_file", "1000",
-"-ps"
+"-records_per_file", "1000"
};
String query = "some query string";
http://git-wip-us.apache.org/repos/asf/metron/blob/8a926dd5/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/config/PcapConfig.java
--
diff --git
a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/config/PcapConfig.java
b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/config/PcapConfig.java
index 4a08e14..abf35d0 100644
---
[35/50] [abbrv] metron git commit: METRON-1733 PCAP UI - PCAP queries don t work on Safari (sardell via merrimanr) closes apache/metron#1158
METRON-1733 PCAP UI - PCAP queries dont work on Safari (sardell via
merrimanr) closes apache/metron#1158
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/bce9b903
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/bce9b903
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/bce9b903
Branch: refs/heads/feature/METRON-1699-create-batch-profiler
Commit: bce9b903fbd3436ea0c41eb8d75da279e16ce922
Parents: ba0a609
Author: sardell
Authored: Mon Aug 13 16:04:45 2018 -0500
Committer: rmerriman
Committed: Mon Aug 13 16:04:45 2018 -0500
--
.../src/app/pcap/pcap-filters/pcap-filters.component.ts | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/bce9b903/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.ts
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.ts
index df91ae3..b23a2e2 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.ts
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.ts
@@ -59,8 +59,8 @@ export class PcapFiltersComponent implements OnInit,
OnChanges {
}
onSubmit() {
-this.model.startTimeMs = new Date(this.startTimeStr).getTime();
-this.model.endTimeMs = new Date(this.endTimeStr).getTime();
+this.model.startTimeMs = moment(this.startTimeStr,
DEFAULT_TIMESTAMP_FORMAT).valueOf();
+this.model.endTimeMs = moment(this.endTimeStr,
DEFAULT_TIMESTAMP_FORMAT).valueOf();
if (this.ipSrcPort !== '') {
this.model.ipSrcPort = +this.ipSrcPort;
} else {
[39/50] [abbrv] metron git commit: METRON-1738: Pcap directories should have correct permissions (merrimanr via mmiklavc) closes apache/metron#1166
METRON-1738: Pcap directories should have correct permissions (merrimanr via mmiklavc) closes apache/metron#1166 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/076a6a19 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/076a6a19 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/076a6a19 Branch: refs/heads/feature/METRON-1699-create-batch-profiler Commit: 076a6a197df629d3b04e0757320e7681ea2fb3d9 Parents: 6b70571 Author: merrimanr Authored: Wed Aug 15 16:02:47 2018 -0600 Committer: Michael Miklavcic Committed: Wed Aug 15 16:02:47 2018 -0600 -- .../package/scripts/params/params_linux.py | 1 + .../package/scripts/params/status_params.py | 1 + .../CURRENT/package/scripts/rest_commands.py| 23 +++- .../CURRENT/package/scripts/rest_master.py | 5 + 4 files changed, 24 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/076a6a19/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py -- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py index 115a54c..9be09f1 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py @@ -393,6 +393,7 @@ pcap_page_size = config['configurations']['metron-rest-env']['pcap_page_size'] pcap_yarn_queue = config['configurations']['metron-rest-env']['pcap_yarn_queue'] pcap_finalizer_threadpool_size= config['configurations']['metron-rest-env']['pcap_finalizer_threadpool_size'] pcap_configured_flag_file = status_params.pcap_configured_flag_file +pcap_perm_configured_flag_file = status_params.pcap_perm_configured_flag_file # MapReduce metron_user_hdfs_dir = '/user/' + metron_user http://git-wip-us.apache.org/repos/asf/metron/blob/076a6a19/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py -- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py index 0a9fdd0..99f5ec0 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py @@ -120,6 +120,7 @@ metron_keytab_path = config['configurations']['metron-env']['metron_service_keyt # Pcap pcap_configured_flag_file = metron_zookeeper_config_path + '/../metron_pcap_configured' +pcap_perm_configured_flag_file = metron_zookeeper_config_path + '/../metron_pcap_perm_configured' # MapReduce metron_user_hdfs_dir_configured_flag_file = metron_zookeeper_config_path + '/../metron_user_hdfs_dir_configured' \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/076a6a19/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py -- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py index 463dca1..d44f478 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py @@ -38,6 +38,7 @@ class RestCommands: __hbase_configured = False __hbase_acl_configured = False __pcap_configured = False +__pcap_perm_configured = False __metron_user_hdfs_dir_configured = False def __init__(self, params): @@ -49,6 +50,7 @@ class RestCommands: self.__hbase_configured =
[04/50] [abbrv] metron git commit: METRON-1693: Fix Pcap CLI local FS finalizer (mmiklavc via mmiklavc) closes apache/metron#1130
METRON-1693: Fix Pcap CLI local FS finalizer (mmiklavc via mmiklavc) closes
apache/metron#1130
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/f1f5dda4
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/f1f5dda4
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/f1f5dda4
Branch: refs/heads/feature/METRON-1699-create-batch-profiler
Commit: f1f5dda4c643394c12ab9f4501c5b806adb21129
Parents: 9d4842f
Author: mmiklavc
Authored: Wed Jul 25 08:26:45 2018 -0600
Committer: Michael Miklavcic
Committed: Wed Jul 25 08:26:45 2018 -0600
--
.../java/org/apache/metron/pcap/query/PcapCli.java | 1 -
.../metron/pcap/finalizer/PcapCliFinalizer.java | 15 ---
.../apache/metron/pcap/finalizer/PcapFinalizer.java | 4 +++-
.../metron/pcap/finalizer/PcapRestFinalizer.java| 10 ++
.../metron/pcap/writer/PcapResultsWriter.java | 16
5 files changed, 41 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/f1f5dda4/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java
--
diff --git
a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java
b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java
index 1a23740..0b06b0c 100644
---
a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java
+++
b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java
@@ -87,7 +87,6 @@ public class PcapCli {
try {
config = fixedParser.parse(otherArgs);
commonConfig = config;
-PcapOptions.FINAL_OUTPUT_PATH.put(commonConfig, new Path("file:///" +
execDir));
} catch (ParseException | java.text.ParseException e) {
System.err.println(e.getMessage());
System.err.flush();
http://git-wip-us.apache.org/repos/asf/metron/blob/f1f5dda4/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapCliFinalizer.java
--
diff --git
a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapCliFinalizer.java
b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapCliFinalizer.java
index c379515..c912e58 100644
---
a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapCliFinalizer.java
+++
b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapCliFinalizer.java
@@ -18,9 +18,13 @@
package org.apache.metron.pcap.finalizer;
+import java.io.IOException;
+import java.util.List;
import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.metron.pcap.config.PcapOptions;
+import org.apache.metron.pcap.writer.PcapResultsWriter;
/**
* Write to local FS.
@@ -33,13 +37,18 @@ public class PcapCliFinalizer extends PcapFinalizer {
* as a formatted timestamp + uuid. A final sample format will look as
follows:
*
/base/output/path/pcap-data-201807181911-09855b4ae3204dee8b63760d65198da3+0001.pcap
*/
- private static final String PCAP_CLI_FILENAME_FORMAT =
"%s/pcap-data-%s+%04d.pcap";
+ private static final String PCAP_CLI_FILENAME_FORMAT =
"pcap-data-%s+%04d.pcap";
+
+ @Override
+ protected void write(PcapResultsWriter resultsWriter, Configuration
hadoopConfig,
+ List data, Path outputPath) throws IOException {
+resultsWriter.writeLocal(data, outputPath.toString());
+ }
@Override
protected Path getOutputPath(Map config, int partition) {
-Path finalOutputPath = PcapOptions.FINAL_OUTPUT_PATH.get(config,
PcapOptions.STRING_TO_PATH, Path.class);
String prefix = PcapOptions.FINAL_FILENAME_PREFIX.get(config,
String.class);
-return new Path(String.format(PCAP_CLI_FILENAME_FORMAT, finalOutputPath,
prefix, partition));
+return new Path(String.format(PCAP_CLI_FILENAME_FORMAT, prefix,
partition));
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/f1f5dda4/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapFinalizer.java
--
diff --git
a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapFinalizer.java
b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapFinalizer.java
index 2c55e15..8dcc401 100644
---
a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapFinalizer.java
+++
b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapFinalizer.java
@@ -81,7
[30/50] [abbrv] metron git commit: METRON-1722 PcapCLI should print progress to stdout (merrimanr) closes apache/metron#1138
METRON-1722 PcapCLI should print progress to stdout (merrimanr) closes
apache/metron#1138
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/b29833cc
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/b29833cc
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/b29833cc
Branch: refs/heads/feature/METRON-1699-create-batch-profiler
Commit: b29833cc5db4efca2a6e9025042baf705fb347c6
Parents: d8a5922
Author: merrimanr
Authored: Fri Aug 10 08:25:13 2018 -0500
Committer: rmerriman
Committed: Fri Aug 10 08:25:13 2018 -0500
--
metron-platform/metron-pcap-backend/README.md | 2 ++
.../java/org/apache/metron/pcap/query/CliParser.java| 4
.../java/org/apache/metron/pcap/query/PcapCliTest.java | 12 +---
.../java/org/apache/metron/pcap/config/PcapConfig.java | 9 +
.../java/org/apache/metron/pcap/config/PcapOptions.java | 3 ++-
.../main/java/org/apache/metron/pcap/mr/PcapJob.java| 11 +++
6 files changed, 37 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/b29833cc/metron-platform/metron-pcap-backend/README.md
--
diff --git a/metron-platform/metron-pcap-backend/README.md
b/metron-platform/metron-pcap-backend/README.md
index e7960e3..2ff20d8 100644
--- a/metron-platform/metron-pcap-backend/README.md
+++ b/metron-platform/metron-pcap-backend/README.md
@@ -132,6 +132,7 @@ usage: Fixed filter options
-nr,--num_reducers The number of reducers to use. Default
is 10.
-h,--help Display help
+ -ps,--print_status Print the status of the job as it runs
-ir,--include_reverse Indicates if filter should check swapped
src/dest addresses and IPs
-p,--protocol IP Protocol
@@ -154,6 +155,7 @@ usage: Query filter options
-nr,--num_reducers The number of reducers to use. Default
is 10.
-h,--help Display help
+ -ps,--print_status Print the status of the job as it runs
-q,--query Query string to use as a filter
-st,--start_time (required) Packet start time range.
```
http://git-wip-us.apache.org/repos/asf/metron/blob/b29833cc/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
--
diff --git
a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
index 2d15e8b..69c725c 100644
---
a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
+++
b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
@@ -55,6 +55,7 @@ public class CliParser {
options.addOption(newOption("rpf", "records_per_file", true,
String.format("Number of records to include in each output pcap file (defaults
to %s)", NUM_RECORDS_PER_FILE_DEFAULT)));
options.addOption(newOption("et", "end_time", true, "Packet end time
range. Default is current system time."));
options.addOption(newOption("df", "date_format", true, "Date format to use
for parsing start_time and end_time. Default is to use time in millis since the
epoch."));
+options.addOption(newOption("ps", "print_status", false, "Print the status
of the job as it runs"));
return options;
}
@@ -125,6 +126,9 @@ public class CliParser {
//no-op
}
}
+if (commandLine.hasOption("print_status")) {
+ config.setPrintJobStatus(true);
+}
}
public void printHelp(String msg, Options opts) {
http://git-wip-us.apache.org/repos/asf/metron/blob/b29833cc/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
--
diff --git
a/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
b/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
index 96ca354..7c75224 100644
---
a/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
+++
b/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
@@ -147,7 +147,8 @@ public class PcapCliTest {
"-protocol", "6",
"-include_reverse",
"-num_reducers", "10",
-"-records_per_file", "1000"
+"-records_per_file", "1000",
+"-ps"
};
[20/50] [abbrv] metron git commit: METRON-1683 PCAP UI - Fix the download progress bar (sardell via merrimanr) closes apache/metron#1122
http://git-wip-us.apache.org/repos/asf/metron/blob/3e778592/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.spec.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.spec.ts
b/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.spec.ts
index 155e1f8..de39a0a 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.spec.ts
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.spec.ts
@@ -16,9 +16,10 @@
* limitations under the License.
*/
import { async, ComponentFixture, TestBed } from '@angular/core/testing';
+import { fakePacket } from '../model/pdml.mock';
import { PcapPacketComponent } from './pcap-packet.component';
-import { PdmlPacket } from '../model/pdml';
+import { By } from '@angular/platform-browser';
describe('PcapPacketComponent', () => {
let component: PcapPacketComponent;
@@ -34,1250 +35,45 @@ describe('PcapPacketComponent', () => {
beforeEach(() => {
fixture = TestBed.createComponent(PcapPacketComponent);
component = fixture.componentInstance;
-component.packet = fakePacket as PdmlPacket;
+component.packet = fakePacket;
fixture.detectChanges();
});
it('should create', () => {
expect(component).toBeTruthy();
});
-});
-const fakePacket = {
- name: "something",
- expanded: false,
- protos: [
-{
- "name": "geninfo",
- "pos": "0",
- "showname": "General information",
- "size": "722",
- "hide": null,
- "fields": [
-{
- "name": "num",
- "pos": "0",
- "showname": "Number",
- "size": "722",
- "value": "1",
- "show": "1",
- "unmaskedvalue": null,
- "hide": null,
- "fields": null,
- "protos": null
-},
-{
- "name": "len",
- "pos": "0",
- "showname": "Frame Length",
- "size": "722",
- "value": "2d2",
- "show": "722",
- "unmaskedvalue": null,
- "hide": null,
- "fields": null,
- "protos": null
-},
-{
- "name": "caplen",
- "pos": "0",
- "showname": "Captured Length",
- "size": "722",
- "value": "2d2",
- "show": "722",
- "unmaskedvalue": null,
- "hide": null,
- "fields": null,
- "protos": null
-},
-{
- "name": "timestamp",
- "pos": "0",
- "showname": "Captured Time",
- "size": "722",
- "value": "1458240269.373968000",
- "show": "Mar 17, 2016 18:44:29.373968000 UTC",
- "unmaskedvalue": null,
- "hide": null,
- "fields": null,
- "protos": null
-}
- ]
-},
-{
- "name": "frame",
- "pos": "0",
- "showname": "Frame 1: 722 bytes on wire (5776 bits), 722 bytes captured
(5776 bits)",
- "size": "722",
- "hide": null,
- "fields": [
-{
- "name": "frame.dlt",
- "pos": "0",
- "showname": "WTAP_ENCAP: 1",
- "size": "0",
- "value": null,
- "show": "1",
- "unmaskedvalue": null,
- "hide": null,
- "fields": null,
- "protos": null
-},
-{
- "name": "frame.time",
- "pos": "0",
- "showname": "Arrival Time: Mar 17, 2016 18:44:29.373968000 UTC",
- "size": "0",
- "value": null,
- "show": "Mar 17, 2016 18:44:29.373968000",
- "unmaskedvalue": null,
- "hide": null,
- "fields": null,
- "protos": null
-},
-{
- "name": "frame.offset_shift",
- "pos": "0",
- "showname": "Time shift for this packet: 0.0 seconds",
- "size": "0",
- "value": null,
- "show": "0.0",
- "unmaskedvalue": null,
- "hide": null,
- "fields": null,
- "protos": null
-},
-{
- "name": "frame.time_epoch",
- "pos": "0",
- "showname": "Epoch Time: 1458240269.373968000 seconds",
- "size": "0",
- "value": null,
- "show": "1458240269.373968000",
- "unmaskedvalue": null,
- "hide": null,
- "fields": null,
- "protos": null
-},
-{
- "name": "frame.time_delta",
- "pos": "0",
- "showname": "Time delta from previous captured frame: 0.0
seconds",
- "size": "0",
- "value": null,
- "show": "0.0",
- "unmaskedvalue": null,
- "hide": null,
- "fields": null,
- "protos": null
-},
-{
- "name": "frame.time_delta_displayed",
- "pos":
[32/50] [abbrv] metron git commit: METRON-1731: PCAP - Escape colons in output dir names (mmiklavc via mmiklavc) closes apache/metron#1155
METRON-1731: PCAP - Escape colons in output dir names (mmiklavc via mmiklavc)
closes apache/metron#1155
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/73dc63e6
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/73dc63e6
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/73dc63e6
Branch: refs/heads/feature/METRON-1699-create-batch-profiler
Commit: 73dc63e671b55d22d251f4be1c217259f4f5dc71
Parents: 05316a4
Author: mmiklavc
Authored: Fri Aug 10 12:42:47 2018 -0600
Committer: Michael Miklavcic
Committed: Fri Aug 10 12:42:47 2018 -0600
--
.../apache/metron/pcap/FixedPcapFilterTest.java | 286 --
.../org/apache/metron/pcap/PcapJobTest.java | 290 ---
.../apache/metron/pcap/QueryPcapFilterTest.java | 228 ---
.../pcap/filter/fixed/FixedPcapFilter.java | 14 +-
.../pcap/filter/query/QueryPcapFilter.java | 17 +-
.../metron/pcap/mr/OutputDirFormatter.java | 37 +++
.../java/org/apache/metron/pcap/mr/PcapJob.java | 5 +-
.../pcap/filter/fixed/FixedPcapFilterTest.java | 271 -
.../pcap/filter/query/QueryPcapFilterTest.java | 207 -
.../metron/pcap/mr/OutputDirFormatterTest.java | 62
.../org/apache/metron/pcap/mr/PcapJobTest.java | 290 +++
11 files changed, 877 insertions(+), 830 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/73dc63e6/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/FixedPcapFilterTest.java
--
diff --git
a/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/FixedPcapFilterTest.java
b/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/FixedPcapFilterTest.java
deleted file mode 100644
index 84969d3..000
---
a/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/FixedPcapFilterTest.java
+++ /dev/null
@@ -1,286 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.metron.pcap;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.metron.common.Constants;
-import org.apache.metron.pcap.filter.fixed.FixedPcapFilter;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.util.HashMap;
-import java.util.Map;
-
-public class FixedPcapFilterTest {
- @Test
- public void testTrivialEquality() throws Exception {
-Configuration config = new Configuration();
-final Map fields = new HashMap() {{
- put(Constants.Fields.SRC_ADDR.getName(), "src_ip");
- put(Constants.Fields.SRC_PORT.getName(), "0");
- put(Constants.Fields.DST_ADDR.getName(), "dst_ip");
- put(Constants.Fields.DST_PORT.getName(), "1");
- put(Constants.Fields.INCLUDES_REVERSE_TRAFFIC.getName(), "false");
-}};
-new FixedPcapFilter.Configurator().addToConfig(fields, config);
-{
- FixedPcapFilter filter = new FixedPcapFilter() {
-@Override
-protected Map packetToFields(PacketInfo pi) {
- return new HashMap() {{
-put(Constants.Fields.SRC_ADDR.getName(), "src_ip");
-put(Constants.Fields.SRC_PORT.getName(), 0);
-put(Constants.Fields.DST_ADDR.getName(), "dst_ip");
-put(Constants.Fields.DST_PORT.getName(), 1);
- }};
-}
- };
- filter.configure(config);
- Assert.assertTrue(filter.test(null));
-}
- }
-
- @Test
- public void testReverseTraffic() throws Exception {
-Configuration config = new Configuration();
-final Map fields = new HashMap() {{
- put(Constants.Fields.SRC_ADDR.getName(), "src_ip");
- put(Constants.Fields.SRC_PORT.getName(), "0");
- put(Constants.Fields.DST_ADDR.getName(), "dst_ip");
- put(Constants.Fields.DST_PORT.getName(), "1");
- put(Constants.Fields.INCLUDES_REVERSE_TRAFFIC.getName(), "true");
-}};
-new FixedPcapFilter.Configurator().addToConfig(fields, config);
-{
- FixedPcapFilter filter = new
[46/50] [abbrv] metron git commit: METRON-1752 Prevent package.lock from changing during build (sardell via merrimanr) closes apache/metron#1177
METRON-1752 Prevent package.lock from changing during build (sardell via merrimanr) closes apache/metron#1177 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/09826da8 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/09826da8 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/09826da8 Branch: refs/heads/feature/METRON-1699-create-batch-profiler Commit: 09826da85eddb014a4b718233d53480220b1f5dd Parents: e66cfc8 Author: sardell Authored: Mon Aug 27 15:14:19 2018 -0500 Committer: rmerriman Committed: Mon Aug 27 15:14:19 2018 -0500 -- metron-interface/metron-alerts/pom.xml | 6 +- .../metron-config/package-lock.json | 17734 + metron-interface/metron-config/pom.xml |30 +- 3 files changed, 17752 insertions(+), 18 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/09826da8/metron-interface/metron-alerts/pom.xml -- diff --git a/metron-interface/metron-alerts/pom.xml b/metron-interface/metron-alerts/pom.xml index 996f473..f67a013 100644 --- a/metron-interface/metron-alerts/pom.xml +++ b/metron-interface/metron-alerts/pom.xml @@ -26,7 +26,7 @@ UTF-8 UTF-8 v9.11.1 -5.8.0 +6.2.0 @@ -52,12 +52,12 @@ generate-resources -npm install +npm ci npm -install +ci
[21/50] [abbrv] metron git commit: METRON-1683 PCAP UI - Fix the download progress bar (sardell via merrimanr) closes apache/metron#1122
METRON-1683 PCAP UI - Fix the download progress bar (sardell via merrimanr)
closes apache/metron#1122
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/3e778592
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/3e778592
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/3e778592
Branch: refs/heads/feature/METRON-1699-create-batch-profiler
Commit: 3e7785920e8f0c5aadf36e05dbcfa69252cc5574
Parents: 037b50b
Author: sardell
Authored: Tue Aug 7 13:58:08 2018 -0500
Committer: rmerriman
Committed: Tue Aug 7 13:58:08 2018 -0500
--
.../metron-alerts/src/app/app.component.html|2 +-
.../metron-alerts/src/app/app.component.ts |3 +
.../src/app/pcap/model/pcap-status-response.ts |1 +
.../src/app/pcap/model/pcap.mock.ts | 39 +
.../src/app/pcap/model/pdml.mock.ts | 62 +
.../metron-alerts/src/app/pcap/model/pdml.ts|6 +-
.../app/pcap/pcap-list/pcap-list.component.html |2 +-
.../pcap/pcap-list/pcap-list.component.spec.ts | 17 +
.../pcap-packet-line.component.spec.ts | 1336 +
.../pcap/pcap-packet/pcap-packet.component.html | 11 +-
.../pcap-packet/pcap-packet.component.spec.ts | 1278 +
.../pcap/pcap-panel/pcap-panel.component.html |6 +-
.../pcap/pcap-panel/pcap-panel.component.scss | 18 -
.../pcap-panel/pcap-panel.component.spec.ts | 262 ++-
.../app/pcap/pcap-panel/pcap-panel.component.ts | 52 +-
.../src/app/pcap/service/pcap.service.spec.ts | 1809 +-
.../src/app/pcap/service/pcap.service.ts|7 +-
.../src/environments/environment.e2e.ts |3 +-
.../src/environments/environment.prod.ts|3 +-
.../src/environments/environment.ts |3 +-
metron-interface/metron-alerts/src/index.html |2 +-
21 files changed, 674 insertions(+), 4248 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/3e778592/metron-interface/metron-alerts/src/app/app.component.html
--
diff --git a/metron-interface/metron-alerts/src/app/app.component.html
b/metron-interface/metron-alerts/src/app/app.component.html
index 1b992ca..c3a5d3c 100644
--- a/metron-interface/metron-alerts/src/app/app.component.html
+++ b/metron-interface/metron-alerts/src/app/app.component.html
@@ -11,7 +11,7 @@
OR CONDITIONS OF ANY KIND, either express or implied. See the License
for
the specific language governing permissions and limitations under the
License.
-->
-
+
http://git-wip-us.apache.org/repos/asf/metron/blob/3e778592/metron-interface/metron-alerts/src/app/app.component.ts
--
diff --git a/metron-interface/metron-alerts/src/app/app.component.ts
b/metron-interface/metron-alerts/src/app/app.component.ts
index 2a3c15e..52dee23 100644
--- a/metron-interface/metron-alerts/src/app/app.component.ts
+++ b/metron-interface/metron-alerts/src/app/app.component.ts
@@ -17,6 +17,7 @@
*/
import { Component, OnInit } from '@angular/core';
import {AuthenticationService} from './service/authentication.service';
+import { environment } from 'environments/environment';
declare var $;
@@ -27,11 +28,13 @@ declare var $;
})
export class AppComponent implements OnInit {
loggedIn = false;
+ noTransition = false;
constructor(private authService: AuthenticationService) {
this.authService.onLoginEvent.subscribe(result => {
this.loggedIn = result;
});
+this.noTransition = environment.noTransition;
}
ngOnInit(): void {
http://git-wip-us.apache.org/repos/asf/metron/blob/3e778592/metron-interface/metron-alerts/src/app/pcap/model/pcap-status-response.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/model/pcap-status-response.ts
b/metron-interface/metron-alerts/src/app/pcap/model/pcap-status-response.ts
index d4d9a5e..09c4985 100644
--- a/metron-interface/metron-alerts/src/app/pcap/model/pcap-status-response.ts
+++ b/metron-interface/metron-alerts/src/app/pcap/model/pcap-status-response.ts
@@ -18,6 +18,7 @@
export class PcapStatusResponse {
jobId: string;
jobStatus: string;
+ description: string;
percentComplete: number;
pageTotal: number;
}
http://git-wip-us.apache.org/repos/asf/metron/blob/3e778592/metron-interface/metron-alerts/src/app/pcap/model/pcap.mock.ts
--
diff --git a/metron-interface/metron-alerts/src/app/pcap/model/pcap.mock.ts
b/metron-interface/metron-alerts/src/app/pcap/model/pcap.mock.ts
new file mode 100644
index 000..c867fe9
--- /dev/null
+++
[48/50] [abbrv] metron git commit: METRON-1757 Storm Profiler Serialization Exception (nickwallen) closes apache/metron#1178
METRON-1757 Storm Profiler Serialization Exception (nickwallen) closes
apache/metron#1178
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/661e23e2
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/661e23e2
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/661e23e2
Branch: refs/heads/feature/METRON-1699-create-batch-profiler
Commit: 661e23e274d2bf88370d8feb5e5dbf1b82f15124
Parents: 29cbdcc
Author: nickwallen
Authored: Wed Aug 29 12:51:05 2018 -0400
Committer: nickallen
Committed: Wed Aug 29 12:51:05 2018 -0400
--
metron-analytics/metron-profiler-common/pom.xml | 26 +++
.../metron/profiler/ProfileMeasurement.java | 35
.../apache/metron/profiler/ProfilePeriod.java | 3 ++
.../metron/profiler/ProfileMeasurementTest.java | 28 +---
.../metron/profiler/ProfilePeriodTest.java | 23 ++---
.../profiler/hbase/SaltyRowKeyBuilderTest.java | 11 --
metron-analytics/metron-profiler/README.md | 18 ++
.../integration/ProfilerIntegrationTest.java| 30 +
.../configuration/profiler/ProfileResult.java | 4 +++
.../profiler/ProfileResultExpressions.java | 4 +++
.../profiler/ProfilerConfigTest.java| 35 +++-
11 files changed, 154 insertions(+), 63 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/661e23e2/metron-analytics/metron-profiler-common/pom.xml
--
diff --git a/metron-analytics/metron-profiler-common/pom.xml
b/metron-analytics/metron-profiler-common/pom.xml
index 928515f..74253cc 100644
--- a/metron-analytics/metron-profiler-common/pom.xml
+++ b/metron-analytics/metron-profiler-common/pom.xml
@@ -55,6 +55,10 @@
org.slf4j
slf4j-log4j12
+
+asm
+asm
+
@@ -109,30 +113,10 @@
provided
-org.apache.storm
-storm-core
-${global_storm_version}
-provided
-
-
-org.apache.logging.log4j
-log4j-api
-
-
-org.apache.logging.log4j
-log4j-core
-
-
-log4j-slf4j-impl
-org.apache.logging.log4j
-
-
-
-
org.mockito
mockito-all
${global_mockito_version}
test
-
\ No newline at end of file
+
http://git-wip-us.apache.org/repos/asf/metron/blob/661e23e2/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileMeasurement.java
--
diff --git
a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileMeasurement.java
b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileMeasurement.java
index 4737c3d..2212297 100644
---
a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileMeasurement.java
+++
b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileMeasurement.java
@@ -20,6 +20,8 @@
package org.apache.metron.profiler;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.metron.common.configuration.profiler.ProfileConfig;
import java.io.Serializable;
@@ -174,4 +176,37 @@ public class ProfileMeasurement implements Serializable {
public void setTriageValues(Map triageValues) {
this.triageValues = triageValues;
}
+
+ @Override
+ public boolean equals(Object o) {
+if (this == o) {
+ return true;
+}
+if (o == null || getClass() != o.getClass()) {
+ return false;
+}
+ProfileMeasurement that = (ProfileMeasurement) o;
+return new EqualsBuilder()
+.append(profileName, that.profileName)
+.append(entity, that.entity)
+.append(groups, that.groups)
+.append(period, that.period)
+.append(definition, that.definition)
+.append(profileValue, that.profileValue)
+.append(triageValues, that.triageValues)
+.isEquals();
+ }
+
+ @Override
+ public int hashCode() {
+return new HashCodeBuilder(17, 37)
+.append(profileName)
+.append(entity)
+.append(groups)
+.append(period)
+.append(definition)
+.append(profileValue
[01/50] [abbrv] metron git commit: METRON-1560 Update MPack to support Pcap panel (merrimanr) closes apache/metron#1124
Repository: metron
Updated Branches:
refs/heads/feature/METRON-1699-create-batch-profiler 113ac0d3a -> f5eaef3c2
METRON-1560 Update MPack to support Pcap panel (merrimanr) closes
apache/metron#1124
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/e78bc530
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/e78bc530
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/e78bc530
Branch: refs/heads/feature/METRON-1699-create-batch-profiler
Commit: e78bc530260aa622e8263cda2b94be99194dffb5
Parents: 8763fba
Author: merrimanr
Authored: Mon Jul 23 09:08:12 2018 -0500
Committer: rmerriman
Committed: Mon Jul 23 09:08:12 2018 -0500
--
.../CURRENT/configuration/metron-rest-env.xml | 25
.../package/scripts/params/params_linux.py | 5 +++-
.../CURRENT/package/scripts/rest_commands.py| 18 --
.../METRON/CURRENT/package/templates/metron.j2 | 4
.../METRON/CURRENT/themes/metron_theme.json | 10
.../src/main/config/rest_application.yml| 6 -
.../src/main/config/pcap.properties | 2 +-
7 files changed, 65 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/e78bc530/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
--
diff --git
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
index 066b527..20f9767 100644
---
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
+++
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
@@ -168,4 +168,29 @@
The field name where the threat triage score can be found
in the search indices. This setting primarily affects the Alerts
UI.
threat:triage:score
+
+pcap_base_path
+Pcap Base Path
+The HDFS path where pcap files are stored. These files
are the input to pcap queries.
+/apps/metron/pcap/input
+
+
+pcap_base_interim_result_path
+Pcap Base Interim Result Path
+The HDFS path where interim results of a pcap query are
stored, before they are separated into pages.
+/apps/metron/pcap/interim
+
+
+pcap_final_output_path
+Pcap Final Output Path
+The HDFS path where the final results of a pcap query are
stored.
+/apps/metron/pcap/output
+
+
+pcap_page_size
+Pcap Page Size
+The number of pcaps written to a page/file as a result of
a pcap query.
+10
+
+
http://git-wip-us.apache.org/repos/asf/metron/blob/e78bc530/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
--
diff --git
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
index b8d6123..73d3469 100755
---
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
+++
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
@@ -386,7 +386,10 @@ bolt_hdfs_rotation_policy_units =
config['configurations']['metron-indexing-env'
bolt_hdfs_rotation_policy_count =
config['configurations']['metron-indexing-env']['bolt_hdfs_rotation_policy_count']
# Pcap
-pcap_hdfs_dir = format("{metron_apps_hdfs_dir}/pcap")
+pcap_base_path = config['configurations']['metron-rest-env']['pcap_base_path']
+pcap_base_interim_result_path =
config['configurations']['metron-rest-env']['pcap_base_interim_result_path']
+pcap_final_output_path =
config['configurations']['metron-rest-env']['pcap_final_output_path']
+pcap_page_size = config['configurations']['metron-rest-env']['pcap_page_size']
pcap_configured_flag_file = status_params.pcap_configured_flag_file
# MapReduce
http://git-wip-us.apache.org/repos/asf/metron/blob/e78bc530/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py
[19/50] [abbrv] metron git commit: METRON-1683 PCAP UI - Fix the download progress bar (sardell via merrimanr) closes apache/metron#1122
http://git-wip-us.apache.org/repos/asf/metron/blob/3e778592/metron-interface/metron-alerts/src/app/pcap/service/pcap.service.spec.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/service/pcap.service.spec.ts
b/metron-interface/metron-alerts/src/app/pcap/service/pcap.service.spec.ts
index fdaa569..c8cb937 100644
--- a/metron-interface/metron-alerts/src/app/pcap/service/pcap.service.spec.ts
+++ b/metron-interface/metron-alerts/src/app/pcap/service/pcap.service.spec.ts
@@ -15,1738 +15,139 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-import { TestBed, async, inject } from '@angular/core/testing';
+import {
+ TestBed,
+ inject,
+ fakeAsync,
+ tick,
+ discardPeriodicTasks
+} from '@angular/core/testing';
import {
BaseRequestOptions,
HttpModule,
- Http,
Response,
- ResponseOptions
+ ResponseOptions,
+ XHRBackend
} from '@angular/http';
import { MockBackend } from '@angular/http/testing';
+import { Observable } from 'rxjs/Rx';
import { PcapService } from './pcap.service';
+import { PcapStatusResponse } from '../model/pcap-status-response';
import { PcapRequest } from '../model/pcap.request';
+import { fakePdml, fakePacket } from '../model/pdml.mock';
+import { fakePcapStatusResponse, fakePcapRequest } from '../model/pcap.mock';
+
+const jobId = 'job_1234567890123_4567';
+let pdmlJsonMock = fakePdml;
+pdmlJsonMock['pdml']['packet'].push(fakePacket);
describe('PcapService', () => {
beforeEach(() => {
-
TestBed.configureTestingModule({
imports: [HttpModule],
providers: [
PcapService,
-{
- provide: Http,
- useFactory: (mockBackend, options) => {
-return new Http(mockBackend, options);
- },
- deps: [MockBackend, BaseRequestOptions]
-},
+{ provide: XHRBackend, useClass: MockBackend },
MockBackend,
BaseRequestOptions
]
});
});
describe('getPackets()', () => {
-it('should return an Observable',
- inject([PcapService, MockBackend], (pcapService, mockBackend) => {
-
-let request: PcapRequest = {
- startTimeMs: 0,
- endTimeMs: 0,
- ipSrcAddr: '0.0.0.0',
- ipSrcPort: '80',
- ipDstAddr: '0.0.0.0',
- ipDstPort: '80',
- protocol: '*',
- packetFilter: '*',
- includeReverse: false,
-};
-
-mockBackend.connections.subscribe((connection) => {
- connection.mockRespond(new Response(new ResponseOptions({body:
pdml_json()})));
-});
+it('should return an Observable', inject(
+ [PcapService, XHRBackend],
+ (pcapService, mockBackend) => {
let packets;
-pcapService.getPackets(request).subscribe(r => packets = r)
-expect(packets).toBeTruthy()
-expect(packets.pdml).toBeTruthy()
-expect(packets.pdml.packet.length).toBe(1)
-expect(packets.pdml.packet[0].proto.length).toBeGreaterThan(3)
-
-console.log(packets)
- }))
-
-
-it('should ...', inject([PcapService], (service: PcapService) => {
- expect(service).toBeTruthy();
-}));
-
- })
-
-});
+mockBackend.connections.subscribe(connection => {
+ expect(connection.request.url).toMatch(
+/\/api\/v1\/pcap\/job_1234567890123_4567\/pdml\?page=1/
+ );
+ connection.mockRespond(
+new Response(new ResponseOptions({ body: pdmlJsonMock }))
+ );
+});
+pcapService.getPackets(jobId, 1).subscribe(r => (packets = r));
+expect(packets).toBeTruthy();
+expect(packets.pdml).toBeTruthy();
+expect(packets.pdml.packet.length).toBe(1);
+expect(packets.pdml.packet[0].protos.length).toBe(3);
+ }
+));
+ });
+ describe('pollStatus()', () => {
+it('should call getStatus() in intervals', fakeAsync(inject(
+ [PcapService, XHRBackend], (pcapService, mockBackend) => {
+ const responseMock: PcapStatusResponse = fakePcapStatusResponse;
+ const spy = spyOn(pcapService, 'getStatus').and.returnValue(
+Observable.of(responseMock)
+ );
+ let response;
+
+ pcapService.pollStatus(jobId).subscribe(r => (response = r));
+ tick(4000);
+ expect(spy.calls.count()).toBe(1);
+ tick(4000);
+ expect(spy.calls.count()).toBe(2);
+ discardPeriodicTasks();
+})
+ )
+);
+ });
+ describe('submitRequest()', () => {
+it('should return an Observable', inject(
+ [PcapService, XHRBackend],
+ (pcapService, mockBackend) => {
+const request: PcapRequest = fakePcapRequest;
+const responseMock: PcapStatusResponse = fakePcapStatusResponse;
+let response;
+
+mockBackend.connections.subscribe(connection => {
+
[40/50] [abbrv] metron git commit: METRON-1727: Alerts are not populated on the alerts UI after enabling X-pack for Elastic search (MohanDV via mmiklavc) closes apache/metron#1141
METRON-1727: Alerts are not populated on the alerts UI after enabling X-pack for Elastic search (MohanDV via mmiklavc) closes apache/metron#1141 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/308c2b27 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/308c2b27 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/308c2b27 Branch: refs/heads/feature/METRON-1699-create-batch-profiler Commit: 308c2b27b9b234c0c84c343dfc45b3617b027e64 Parents: 2b0060b Author: MohanDV Authored: Wed Aug 15 16:16:26 2018 -0600 Committer: Michael Miklavcic Committed: Wed Aug 15 16:20:54 2018 -0600 -- metron-deployment/Kerberos-manual-setup.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/308c2b27/metron-deployment/Kerberos-manual-setup.md -- diff --git a/metron-deployment/Kerberos-manual-setup.md b/metron-deployment/Kerberos-manual-setup.md index e3de148..18891a4 100644 --- a/metron-deployment/Kerberos-manual-setup.md +++ b/metron-deployment/Kerberos-manual-setup.md @@ -580,7 +580,7 @@ X-Pack Submit the update to Zookeeper ``` -$METRON_HOME/bin/zk_load_configs.sh -m PUSH -i METRON_HOME/config/zookeeper/ -z $ZOOKEEPER +$METRON_HOME/bin/zk_load_configs.sh -m PUSH -i $METRON_HOME/config/zookeeper/ -z $ZOOKEEPER ``` 1. The last step before restarting the topology is to create a custom X-Pack shaded and relocated jar. This is up to you because of licensing restrictions, but here is a sample Maven pom file that should help. @@ -737,6 +737,8 @@ X-Pack $METRON_HOME/bin/start_elasticsearch_topology.sh ``` +1. Restart the metron-rest service, and make sure the elasticsearch-xpack-shaded-5.6.2.jar is in the METRON_REST_CLASSPATH when the metron-rest starts. + Once you've performed these steps, you should be able to start seeing data in your ES indexes. ### X-Pack Common Problems
[23/50] [abbrv] metron git commit: METRON-1720 Better error messages when there are no results or wireshark is not installed (merrimanr) closes apache/metron#1154
METRON-1720 Better error messages when there are no results or wireshark is not
installed (merrimanr) closes apache/metron#1154
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/afb9607d
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/afb9607d
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/afb9607d
Branch: refs/heads/feature/METRON-1699-create-batch-profiler
Commit: afb9607dfbbaf89f06a30823bf668523bb7b6f9a
Parents: 7967f35
Author: merrimanr
Authored: Wed Aug 8 14:11:26 2018 -0500
Committer: rmerriman
Committed: Wed Aug 8 14:11:26 2018 -0500
--
.../pcap-panel/pcap-panel.component.spec.ts | 64
.../app/pcap/pcap-panel/pcap-panel.component.ts | 9 ++-
.../src/app/pcap/service/pcap.service.ts| 3 +-
3 files changed, 73 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/afb9607d/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.spec.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.spec.ts
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.spec.ts
index bd732ad..0804b79 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.spec.ts
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.spec.ts
@@ -26,6 +26,7 @@ import { PcapPagination } from '../model/pcap-pagination';
import { By } from '../../../../node_modules/@angular/platform-browser';
import { PcapRequest } from '../model/pcap.request';
import { defer } from 'rxjs/observable/defer';
+import {RestError} from "../../model/rest-error";
@Component({
selector: 'app-pcap-filters',
@@ -305,6 +306,7 @@ describe('PcapPanelComponent', () => {
);
const pollResponse = new PcapStatusResponse();
+pollResponse.jobStatus = 'SUCCEEDED';
pcapService.pollStatus = jasmine.createSpy('pollStatus').and.returnValue(
defer(() => Promise.resolve(pollResponse))
);
@@ -321,6 +323,68 @@ describe('PcapPanelComponent', () => {
tick();
fixture.detectChanges();
+expect(component.pdml).toEqual(myPdml);
expect(fixture.debugElement.query(By.css('app-pcap-list'))).toBeDefined();
}));
+
+ it('should handle get packet 404', fakeAsync(() => {
+const searchResponse = new PcapStatusResponse();
+searchResponse.jobId = '42';
+
+pcapService.submitRequest =
jasmine.createSpy('submitRequest').and.returnValue(
+defer(() => Promise.resolve(searchResponse))
+);
+
+const pollResponse = new PcapStatusResponse();
+pollResponse.jobStatus = 'SUCCEEDED';
+pcapService.pollStatus = jasmine.createSpy('pollStatus').and.returnValue(
+defer(() => Promise.resolve(pollResponse))
+);
+
+const restError = new RestError();
+restError.responseCode = 404;
+pcapService.getPackets = jasmine.createSpy('getPackets').and.returnValue(
+defer(() => Promise.reject(restError))
+);
+
+component.onSearch(new PcapRequest());
+
+expect(component.errorMsg).toBeFalsy();
+
+tick();
+fixture.detectChanges();
+
+expect(component.errorMsg).toEqual('No results returned');
+ }));
+
+ it('should handle get packet error', fakeAsync(() => {
+const searchResponse = new PcapStatusResponse();
+searchResponse.jobId = '42';
+
+pcapService.submitRequest =
jasmine.createSpy('submitRequest').and.returnValue(
+defer(() => Promise.resolve(searchResponse))
+);
+
+const pollResponse = new PcapStatusResponse();
+pollResponse.jobStatus = 'SUCCEEDED';
+pcapService.pollStatus = jasmine.createSpy('pollStatus').and.returnValue(
+defer(() => Promise.resolve(pollResponse))
+);
+
+const restError = new RestError();
+restError.responseCode = 500;
+restError.message = 'error message';
+pcapService.getPackets = jasmine.createSpy('getPackets').and.returnValue(
+defer(() => Promise.reject(restError))
+);
+
+component.onSearch(new PcapRequest());
+
+expect(component.errorMsg).toBeFalsy();
+
+tick();
+fixture.detectChanges();
+
+expect(component.errorMsg).toEqual('Response message: error message.
Something went wrong retrieving pdml results!');
+ }));
});
http://git-wip-us.apache.org/repos/asf/metron/blob/afb9607d/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.ts
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.ts
index 82121d4..8e4ced0 100644
[38/50] [abbrv] metron git commit: METRON-1737: Document Job cleanup (merrimanr via mmiklavc) closes apache/metron#1164
METRON-1737: Document Job cleanup (merrimanr via mmiklavc) closes
apache/metron#1164
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/6b70571d
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/6b70571d
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/6b70571d
Branch: refs/heads/feature/METRON-1699-create-batch-profiler
Commit: 6b70571d6de3951c98269bbf5b38e8b69deddfab
Parents: d9e1f38
Author: merrimanr
Authored: Wed Aug 15 16:00:13 2018 -0600
Committer: Michael Miklavcic
Committed: Wed Aug 15 16:00:13 2018 -0600
--
metron-interface/metron-rest/README.md | 11 +++
1 file changed, 11 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/6b70571d/metron-interface/metron-rest/README.md
--
diff --git a/metron-interface/metron-rest/README.md
b/metron-interface/metron-rest/README.md
index 080422d..2c216d1 100644
--- a/metron-interface/metron-rest/README.md
+++ b/metron-interface/metron-rest/README.md
@@ -222,6 +222,17 @@ Out of the box it is a simple wrapper around the tshark
command to transform raw
REST will supply the script with raw pcap data through standard in and expects
PDML data serialized as XML.
Pcap query jobs can be configured for submission to a YARN queue. This
setting is exposed as the Spring property `pcap.yarn.queue`. If configured,
the REST application will set the `mapreduce.job.queuename` Hadoop property to
that value.
+It is highly recommended that a dedicated YARN queue be created and configured
for Pcap queries to prevent a job from consuming too many cluster resources.
More information about setting up YARN queues can be found
[here](https://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/CapacityScheduler.html#Setting_up_queues).
+
+Pcap query results are stored in HDFS. The location of query results when run
through the REST app is determined by a couple factors. The root of Pcap query
results defaults to `/apps/metron/pcap/output` but can be changed with the
+Spring property `pcap.final.output.path`. Assuming the default Pcap query
output directory, the path to a result page will follow this pattern:
+```
+/apps/metron/pcap/output/{username}/MAP_REDUCE/{job id}/page-{page number}.pcap
+```
+Over time Pcap query results will accumulate in HDFS. Currently these results
are not cleaned up automatically so cluster administrators should be aware of
this and monitor them. It is highly recommended that a process be put in place
to
+periodically delete files and directories under the Pcap query results root.
+
+Users should also be mindful of date ranges used in queries so they don't
produce result sets that are too large. Currently there are no limits enforced
on date ranges.
Queries can also be configured on a global level for setting the number of
results per page via a Spring property `pcap.page.size`. By default, this value
is set to 10 pcaps per page, but you may choose to set this value higher
based on observing frequenetly-run query result sizes. This setting works in
conjunction with the property for setting finalizer threadpool size when
optimizing query performance.
[24/50] [abbrv] metron git commit: METRON-1712 PCAP UI - Input validation (tiborm via merrimanr) closes apache/metron#1142
METRON-1712 PCAP UI - Input validation (tiborm via merrimanr) closes
apache/metron#1142
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/52de126f
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/52de126f
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/52de126f
Branch: refs/heads/feature/METRON-1699-create-batch-profiler
Commit: 52de126fc746181d6aac8ce002d6b814fd1e6cb5
Parents: afb9607
Author: tiborm
Authored: Wed Aug 8 16:36:35 2018 -0500
Committer: rmerriman
Committed: Wed Aug 8 16:36:35 2018 -0500
--
.../src/app/pcap/model/pcap.mock.ts | 4 +-
.../src/app/pcap/model/pcap.request.ts | 4 +-
.../pcap-filters/pcap-filters.component.html| 12 +-
.../pcap-filters/pcap-filters.component.spec.ts | 228 ++-
.../pcap/pcap-filters/pcap-filters.component.ts | 5 +
5 files changed, 232 insertions(+), 21 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/52de126f/metron-interface/metron-alerts/src/app/pcap/model/pcap.mock.ts
--
diff --git a/metron-interface/metron-alerts/src/app/pcap/model/pcap.mock.ts
b/metron-interface/metron-alerts/src/app/pcap/model/pcap.mock.ts
index c867fe9..bf02da8 100644
--- a/metron-interface/metron-alerts/src/app/pcap/model/pcap.mock.ts
+++ b/metron-interface/metron-alerts/src/app/pcap/model/pcap.mock.ts
@@ -22,9 +22,9 @@ export const fakePcapRequest = {
startTimeMs: 0,
endTimeMs: 0,
ipSrcAddr: '0.0.0.0',
- ipSrcPort: '80',
+ ipSrcPort: 80,
ipDstAddr: '0.0.0.0',
- ipDstPort: '80',
+ ipDstPort: 80,
protocol: '*',
packetFilter: '*',
includeReverse: false
http://git-wip-us.apache.org/repos/asf/metron/blob/52de126f/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
--
diff --git a/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
b/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
index 91c2287..d00a6ac 100644
--- a/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
+++ b/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
@@ -20,9 +20,9 @@ export class PcapRequest {
startTimeMs: number = 0;
endTimeMs: number = 15;
ipSrcAddr: string = '';
- ipSrcPort: string = '';
+ ipSrcPort: number = 0;
ipDstAddr: string = '';
- ipDstPort: string = '';
+ ipDstPort: number = 0;
protocol: string = '';
packetFilter: string = '';
includeReverse: boolean = false;
http://git-wip-us.apache.org/repos/asf/metron/blob/52de126f/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.html
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.html
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.html
index f4133df..039307a 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.html
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.html
@@ -24,21 +24,21 @@
IP Source Address
-
+
-
+
IP Source Port
-
+
IP Dest Address
-
+
IP Dest Port
-
+
@@ -57,6 +57,6 @@
-
+
http://git-wip-us.apache.org/repos/asf/metron/blob/52de126f/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
index c0f9c3b..4336b22 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
@@ -16,20 +16,19 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-import { async, ComponentFixture, TestBed, fakeAsync, tick } from
'@angular/core/testing';
+import { async, ComponentFixture, TestBed } from '@angular/core/testing';
import { By } from '@angular/platform-browser';
import { PcapFiltersComponent } from './pcap-filters.component';
import { FormsModule } from '../../../../node_modules/@angular/forms';
-import { Component, Input, Output, EventEmitter } from '@angular/core';
+import { Component, Input, Output, EventEmitter, DebugElement } from
'@angular/core';
import { PcapRequest } from '../model/pcap.request';
-import { emit } from
[28/50] [abbrv] metron git commit: METRON-1728: Handle null values in config in Pcap backend more gracefully (mmiklavc via mmiklavc) closes apache/metron#1151
METRON-1728: Handle null values in config in Pcap backend more gracefully
(mmiklavc via mmiklavc) closes apache/metron#1151
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/9064cca0
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/9064cca0
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/9064cca0
Branch: refs/heads/feature/METRON-1699-create-batch-profiler
Commit: 9064cca0317881176471c51abd16e99bf2ad7b10
Parents: 14dcb2d
Author: mmiklavc
Authored: Thu Aug 9 09:25:29 2018 -0600
Committer: Michael Miklavcic
Committed: Thu Aug 9 09:25:29 2018 -0600
--
.../common/configuration/ConfigOption.java | 32 --
.../common/configuration/ConfigOptionTest.java | 112 +++
.../org/apache/metron/pcap/query/CliParser.java | 25 +++--
.../org/apache/metron/pcap/PcapJobTest.java | 23
.../apache/metron/pcap/query/PcapCliTest.java | 10 +-
.../metron/pcap/config/PcapGlobalDefaults.java | 28 +
.../metron/pcap/finalizer/PcapFinalizer.java| 8 +-
.../pcap/finalizer/PcapRestFinalizer.java | 11 +-
.../java/org/apache/metron/pcap/mr/PcapJob.java | 25 +++--
9 files changed, 237 insertions(+), 37 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/9064cca0/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigOption.java
--
diff --git
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigOption.java
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigOption.java
index 8e4211b..6308f0a 100644
---
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigOption.java
+++
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigOption.java
@@ -18,36 +18,54 @@
package org.apache.metron.common.configuration;
-import org.apache.metron.stellar.common.utils.ConversionUtils;
-
import java.util.Map;
import java.util.function.BiFunction;
+import org.apache.metron.stellar.common.utils.ConversionUtils;
public interface ConfigOption {
+
String getKey();
+
default BiFunction transform() {
-return (s,o) -> o;
+return (s, o) -> o;
}
default void put(Map map, Object value) {
map.put(getKey(), value);
}
+ default T getOrDefault(Map map, Class clazz, T
defaultValue) {
+T val;
+return ((val = get(map, clazz)) == null ? defaultValue : val);
+ }
+
default T get(Map map, Class clazz) {
Object obj = map.get(getKey());
-if(clazz.isInstance(obj)) {
+if (clazz.isInstance(obj)) {
return clazz.cast(obj);
-}
-else {
+} else {
return ConversionUtils.convert(obj, clazz);
}
}
- default T get(Map map, BiFunction
transform, Class clazz) {
+ default T getOrDefault(Map map, BiFunction transform,
+ Class clazz, T defaultValue) {
+T val;
+return ((val = get(map, transform, clazz)) == null ? defaultValue : val);
+ }
+
+ default T get(Map map, BiFunction
transform,
+ Class clazz) {
return clazz.cast(transform.apply(getKey(), map.get(getKey(;
}
+ default T getTransformedOrDefault(Map map, Class
clazz, T defaultValue) {
+T val;
+return ((val = getTransformed(map, clazz)) == null ? defaultValue : val);
+ }
+
default T getTransformed(Map map, Class clazz) {
return clazz.cast(transform().apply(getKey(), map.get(getKey(;
}
+
}
http://git-wip-us.apache.org/repos/asf/metron/blob/9064cca0/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigOptionTest.java
--
diff --git
a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigOptionTest.java
b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigOptionTest.java
new file mode 100644
index 000..95db080
--- /dev/null
+++
b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigOptionTest.java
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License
[49/50] [abbrv] metron git commit: METRON-1751 Storm Profiler dies when consuming null message (nickwallen) closes apache/metron#1176
METRON-1751 Storm Profiler dies when consuming null message (nickwallen) closes
apache/metron#1176
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/d32bd50d
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/d32bd50d
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/d32bd50d
Branch: refs/heads/feature/METRON-1699-create-batch-profiler
Commit: d32bd50d43aae87af9ec12d2daea83b2f4eca342
Parents: 661e23e
Author: nickwallen
Authored: Wed Aug 29 14:55:58 2018 -0400
Committer: nickallen
Committed: Wed Aug 29 14:55:58 2018 -0400
--
.../profiler/bolt/ProfileSplitterBolt.java | 29 +++-
.../profiler/bolt/ProfileSplitterBoltTest.java | 16 +++
2 files changed, 32 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/d32bd50d/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java
--
diff --git
a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java
b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java
index f28411f..87f1ba9 100644
---
a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java
+++
b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java
@@ -144,9 +144,9 @@ public class ProfileSplitterBolt extends
ConfiguredProfilerBolt {
try {
doExecute(input);
-} catch (IllegalArgumentException | ParseException |
UnsupportedEncodingException e) {
- LOG.error("Unexpected error", e);
- collector.reportError(e);
+} catch (Throwable t) {
+ LOG.error("Unexpected error", t);
+ collector.reportError(t);
} finally {
collector.ack(input);
@@ -157,22 +157,25 @@ public class ProfileSplitterBolt extends
ConfiguredProfilerBolt {
// retrieve the input message
byte[] data = input.getBinary(0);
+if(data == null) {
+ LOG.debug("Received null message. Nothing to do.");
+ return;
+}
JSONObject message = (JSONObject) parser.parse(new String(data, "UTF8"));
// ensure there is a valid profiler configuration
ProfilerConfig config = getProfilerConfig();
-if(config != null && config.getProfiles().size() > 0) {
-
- // what time is it?
- Clock clock = clockFactory.createClock(config);
- Optional timestamp = clock.currentTimeMillis(message);
+if(config == null || getProfilerConfig().getProfiles().size() == 0) {
+ LOG.debug("No Profiler configuration found. Nothing to do.");
+ return;
+}
- // route the message. if a message does not contain the timestamp
field, it cannot be routed.
- timestamp.ifPresent(ts -> routeMessage(input, message, config, ts));
+// what time is it?
+Clock clock = clockFactory.createClock(config);
+Optional timestamp = clock.currentTimeMillis(message);
-} else {
- LOG.debug("No Profiler configuration found. Nothing to do.");
-}
+// route the message. if a message does not contain the timestamp field,
it cannot be routed.
+timestamp.ifPresent(ts -> routeMessage(input, message, config, ts));
}
/**
http://git-wip-us.apache.org/repos/asf/metron/blob/d32bd50d/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileSplitterBoltTest.java
--
diff --git
a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileSplitterBoltTest.java
b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileSplitterBoltTest.java
index bf81923..72e2b72 100644
---
a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileSplitterBoltTest.java
+++
b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileSplitterBoltTest.java
@@ -404,6 +404,22 @@ public class ProfileSplitterBoltTest extends BaseBoltTest {
.emit(any(Values.class));
}
+ @Test
+ public void testWithNullMessage() throws Exception {
+
+// ensure the tuple returns null to mimic a null message in kafka
+when(tuple.getBinary(0)).thenReturn(null);
+
+ProfilerConfig config = toProfilerConfig(profileWithOnlyIfInvalid);
+ProfileSplitterBolt bolt = createBolt(config);
+bolt.execute(tuple);
+
+// a tuple should NOT be emitted for the downstream profile builder
+verify(outputCollector, times(0))
+.emit(any(Values.class));
+
+ }
+
/**
* Creates a ProfilerConfig based on a string containing JSON.
*
[43/50] [abbrv] metron git commit: METRON-1724 Date/time validation missing in PCAP query (tiborm via nickwallen) closes apache/metron#1172
METRON-1724 Date/time validation missing in PCAP query (tiborm via nickwallen)
closes apache/metron#1172
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/e66cfc80
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/e66cfc80
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/e66cfc80
Branch: refs/heads/feature/METRON-1699-create-batch-profiler
Commit: e66cfc80e6a6fa53110c3f2fa8ee0d31ea997bf6
Parents: 9fdccba
Author: tiborm
Authored: Mon Aug 27 09:04:21 2018 -0400
Committer: nickallen
Committed: Mon Aug 27 09:04:21 2018 -0400
--
.../src/app/pcap/model/pcap.mock.ts | 4 +-
.../src/app/pcap/model/pcap.request.ts | 18 +-
.../pcap-filters/pcap-filters.component.html| 55 +++--
.../pcap-filters/pcap-filters.component.scss| 13 +-
.../pcap-filters/pcap-filters.component.spec.ts | 207 ++-
.../pcap/pcap-filters/pcap-filters.component.ts | 156 +++---
.../app/pcap/pcap-list/pcap-list.component.html | 4 +-
.../pcap-packet-line.component.spec.ts | 3 +-
.../pcap-packet-line.component.ts | 22 +-
.../pcap-panel/pcap-panel.component.spec.ts | 16 +-
.../app/pcap/pcap-panel/pcap-panel.component.ts | 6 +-
.../metron-alerts/src/app/pcap/pcap.module.ts | 5 +-
.../date-picker/date-picker.component.scss | 14 +-
.../shared/date-picker/date-picker.component.ts | 34 ++-
.../shared/date-picker/date-picker.module.ts| 3 +-
.../metron-alerts/src/app/utils/constants.ts| 3 +
.../metron-alerts/src/app/utils/utils.ts| 12 +-
17 files changed, 360 insertions(+), 215 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/e66cfc80/metron-interface/metron-alerts/src/app/pcap/model/pcap.mock.ts
--
diff --git a/metron-interface/metron-alerts/src/app/pcap/model/pcap.mock.ts
b/metron-interface/metron-alerts/src/app/pcap/model/pcap.mock.ts
index bf02da8..c867fe9 100644
--- a/metron-interface/metron-alerts/src/app/pcap/model/pcap.mock.ts
+++ b/metron-interface/metron-alerts/src/app/pcap/model/pcap.mock.ts
@@ -22,9 +22,9 @@ export const fakePcapRequest = {
startTimeMs: 0,
endTimeMs: 0,
ipSrcAddr: '0.0.0.0',
- ipSrcPort: 80,
+ ipSrcPort: '80',
ipDstAddr: '0.0.0.0',
- ipDstPort: 80,
+ ipDstPort: '80',
protocol: '*',
packetFilter: '*',
includeReverse: false
http://git-wip-us.apache.org/repos/asf/metron/blob/e66cfc80/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
--
diff --git a/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
b/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
index 8afc963..3e00a4b 100644
--- a/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
+++ b/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
@@ -17,13 +17,13 @@
*/
export class PcapRequest {
- startTimeMs: number = 0;
- endTimeMs: number = 15;
- ipSrcAddr: string = '';
- ipSrcPort: number;
- ipDstAddr: string = '';
- ipDstPort: number;
- protocol: string = '';
- packetFilter: string = '';
- includeReverse: boolean = false;
+ startTimeMs: number;
+ endTimeMs: number;
+ ipSrcAddr = '';
+ ipSrcPort = '';
+ ipDstAddr = '';
+ ipDstPort = '';
+ protocol = '';
+ packetFilter = '';
+ includeReverse = false;
}
http://git-wip-us.apache.org/repos/asf/metron/blob/e66cfc80/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.html
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.html
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.html
index 039307a..c7a4db5 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.html
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.html
@@ -11,52 +11,63 @@
OR CONDITIONS OF ANY KIND, either express or implied. See the License
for
the specific language governing permissions and limitations under the
License.
-->
-
+
From
-
+
To
-
+
-IP Source Address
-
+IP Source Address
+
-IP Source Port
-
+IP Source Port
+
- IP Dest Address
-
+
+IP Dest Address
+
-IP Dest Port
-
+IP Dest Port
+
-
-Protocol
-
-
+
+Protocol
+
+
-
-Include Reverse Traffic
-
-
+
+Include Reverse Traffic
+
+
-Filter
-
+Fil
[12/50] [abbrv] metron git commit: METRON-1671 Create PCAP UI (tiborm via merrimanr) closes apache/metron#1103
METRON-1671 Create PCAP UI (tiborm via merrimanr) closes apache/metron#1103 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/d5eb56a9 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/d5eb56a9 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/d5eb56a9 Branch: refs/heads/feature/METRON-1699-create-batch-profiler Commit: d5eb56a967ac7b925d0a843c027a2b0dde66de06 Parents: 7dff4de Author: tiborm Authored: Wed Aug 1 11:31:50 2018 -0500 Committer: rmerriman Committed: Wed Aug 1 11:31:50 2018 -0500 -- .../metron-alerts/package-lock.json | 3719 +- .../metron-alerts/src/app/app-routing.module.ts |3 +- .../metron-alerts/src/app/app.component.html|8 + .../metron-alerts/src/app/app.component.scss| 18 + .../metron-alerts/src/app/app.module.ts |6 +- .../src/app/pcap/model/pcap.request.ts | 29 + .../metron-alerts/src/app/pcap/model/pdml.ts| 50 + .../pcap-filters/pcap-filters.component.html| 52 + .../pcap-filters/pcap-filters.component.scss| 69 + .../pcap-filters/pcap-filters.component.spec.ts | 46 + .../pcap/pcap-filters/pcap-filters.component.ts | 41 + .../app/pcap/pcap-list/pcap-list.component.html | 37 + .../app/pcap/pcap-list/pcap-list.component.scss | 21 + .../pcap/pcap-list/pcap-list.component.spec.ts | 70 + .../app/pcap/pcap-list/pcap-list.component.ts | 39 + .../pcap-packet-line.component.html | 19 + .../pcap-packet-line.component.scss | 17 + .../pcap-packet-line.component.spec.ts | 1283 ++ .../pcap-packet-line.component.ts | 55 + .../pcap/pcap-packet/pcap-packet.component.html | 21 + .../pcap/pcap-packet/pcap-packet.component.scss | 36 + .../pcap-packet/pcap-packet.component.spec.ts | 1283 ++ .../pcap/pcap-packet/pcap-packet.component.ts | 39 + .../pcap/pcap-panel/pcap-panel.component.html | 31 + .../pcap/pcap-panel/pcap-panel.component.scss | 52 + .../pcap-panel/pcap-panel.component.spec.ts | 68 + .../app/pcap/pcap-panel/pcap-panel.component.ts | 75 + .../metron-alerts/src/app/pcap/pcap.module.ts | 50 + .../metron-alerts/src/app/pcap/pcap.routing.ts | 27 + .../src/app/pcap/service/pcap.service.spec.ts | 1752 + .../src/app/pcap/service/pcap.service.ts| 68 + 31 files changed, 7304 insertions(+), 1780 deletions(-) --
[47/50] [abbrv] metron git commit: METRON-1743 CEF testPaloAltoCEF test using a confusing variable name (JonZeolla via justinleet) closes apache/metron#1173
METRON-1743 CEF testPaloAltoCEF test using a confusing variable name (JonZeolla
via justinleet) closes apache/metron#1173
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/29cbdcc4
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/29cbdcc4
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/29cbdcc4
Branch: refs/remotes/upstream/feature/METRON-1699-create-batch-profiler
Commit: 29cbdcc43c66080e1464e4c9c365c63a3f9035b1
Parents: 09826da
Author: JonZeolla
Authored: Wed Aug 29 09:43:43 2018 -0400
Committer: leet
Committed: Wed Aug 29 09:43:43 2018 -0400
--
.../test/java/org/apache/metron/parsers/cef/CEFParserTest.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/29cbdcc4/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/cef/CEFParserTest.java
--
diff --git
a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/cef/CEFParserTest.java
b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/cef/CEFParserTest.java
index a1fe2e8..f679c91 100644
---
a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/cef/CEFParserTest.java
+++
b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/cef/CEFParserTest.java
@@ -196,8 +196,8 @@ public class CEFParserTest {
@Test
public void testPaloAltoCEF() throws Exception {
- URL waf_url = Resources.getResource(getClass(), "palo.cef");
- runTest("palo", Resources.readLines(waf_url, UTF_8),
+ URL palo_url = Resources.getResource(getClass(), "palo.cef");
+ runTest("palo", Resources.readLines(palo_url, UTF_8),
Resources.toString(Resources.getResource(getClass(), "palo.schema"), UTF_8));
}
[18/50] [abbrv] metron git commit: METRON-1675 PCAP UI - Introduce the paging capability (sardell via merrimanr) closes apache/metron#1121
METRON-1675 PCAP UI - Introduce the paging capability (sardell via merrimanr)
closes apache/metron#1121
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/037b50b9
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/037b50b9
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/037b50b9
Branch: refs/remotes/upstream/feature/METRON-1699-create-batch-profiler
Commit: 037b50b9a829d7292db1148fc628d9231da499e1
Parents: 756ceed
Author: sardell
Authored: Tue Aug 7 09:04:33 2018 -0500
Committer: rmerriman
Committed: Tue Aug 7 09:04:33 2018 -0500
--
.../src/app/pcap/model/pcap-pagination.ts | 21 ++
.../src/app/pcap/model/pcap-status-response.ts | 23 ++
.../src/app/pcap/model/pcap.request.ts | 2 +-
.../app/pcap/pcap-list/pcap-list.component.html | 1 +
.../pcap/pcap-list/pcap-list.component.spec.ts | 11 +++
.../app/pcap/pcap-list/pcap-list.component.ts | 18 +++--
.../pcap-pagination.component.html | 19 +
.../pcap-pagination.component.scss | 40 ++
.../pcap-pagination.component.spec.ts | 78
.../pcap-pagination.component.ts| 41 ++
.../pcap/pcap-panel/pcap-panel.component.html | 7 +-
.../pcap-panel/pcap-panel.component.spec.ts | 2 +
.../app/pcap/pcap-panel/pcap-panel.component.ts | 27 +--
.../metron-alerts/src/app/pcap/pcap.module.ts | 10 +--
.../metron-alerts/src/app/pcap/pcap.routing.ts | 2 +-
.../src/app/pcap/service/pcap.service.spec.ts | 2 +-
.../src/app/pcap/service/pcap.service.ts| 10 +--
17 files changed, 279 insertions(+), 35 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/037b50b9/metron-interface/metron-alerts/src/app/pcap/model/pcap-pagination.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/model/pcap-pagination.ts
b/metron-interface/metron-alerts/src/app/pcap/model/pcap-pagination.ts
new file mode 100644
index 000..ae7619b
--- /dev/null
+++ b/metron-interface/metron-alerts/src/app/pcap/model/pcap-pagination.ts
@@ -0,0 +1,21 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export class PcapPagination {
+ selectedPage = 1;
+ total: number;
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/037b50b9/metron-interface/metron-alerts/src/app/pcap/model/pcap-status-response.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/model/pcap-status-response.ts
b/metron-interface/metron-alerts/src/app/pcap/model/pcap-status-response.ts
new file mode 100644
index 000..d4d9a5e
--- /dev/null
+++ b/metron-interface/metron-alerts/src/app/pcap/model/pcap-status-response.ts
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+export class PcapStatusResponse {
+ jobId: string;
+ jobStatus: string;
+ percentComplete: number;
+ pageTotal: number;
+}
http://git-wip-us.apache.org/repos/asf/metron/blob/037b50b9/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
--
diff --git
[30/50] [abbrv] metron git commit: METRON-1722 PcapCLI should print progress to stdout (merrimanr) closes apache/metron#1138
METRON-1722 PcapCLI should print progress to stdout (merrimanr) closes
apache/metron#1138
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/b29833cc
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/b29833cc
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/b29833cc
Branch: refs/remotes/upstream/feature/METRON-1699-create-batch-profiler
Commit: b29833cc5db4efca2a6e9025042baf705fb347c6
Parents: d8a5922
Author: merrimanr
Authored: Fri Aug 10 08:25:13 2018 -0500
Committer: rmerriman
Committed: Fri Aug 10 08:25:13 2018 -0500
--
metron-platform/metron-pcap-backend/README.md | 2 ++
.../java/org/apache/metron/pcap/query/CliParser.java| 4
.../java/org/apache/metron/pcap/query/PcapCliTest.java | 12 +---
.../java/org/apache/metron/pcap/config/PcapConfig.java | 9 +
.../java/org/apache/metron/pcap/config/PcapOptions.java | 3 ++-
.../main/java/org/apache/metron/pcap/mr/PcapJob.java| 11 +++
6 files changed, 37 insertions(+), 4 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/b29833cc/metron-platform/metron-pcap-backend/README.md
--
diff --git a/metron-platform/metron-pcap-backend/README.md
b/metron-platform/metron-pcap-backend/README.md
index e7960e3..2ff20d8 100644
--- a/metron-platform/metron-pcap-backend/README.md
+++ b/metron-platform/metron-pcap-backend/README.md
@@ -132,6 +132,7 @@ usage: Fixed filter options
-nr,--num_reducers The number of reducers to use. Default
is 10.
-h,--help Display help
+ -ps,--print_status Print the status of the job as it runs
-ir,--include_reverse Indicates if filter should check swapped
src/dest addresses and IPs
-p,--protocol IP Protocol
@@ -154,6 +155,7 @@ usage: Query filter options
-nr,--num_reducers The number of reducers to use. Default
is 10.
-h,--help Display help
+ -ps,--print_status Print the status of the job as it runs
-q,--query Query string to use as a filter
-st,--start_time (required) Packet start time range.
```
http://git-wip-us.apache.org/repos/asf/metron/blob/b29833cc/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
--
diff --git
a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
index 2d15e8b..69c725c 100644
---
a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
+++
b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
@@ -55,6 +55,7 @@ public class CliParser {
options.addOption(newOption("rpf", "records_per_file", true,
String.format("Number of records to include in each output pcap file (defaults
to %s)", NUM_RECORDS_PER_FILE_DEFAULT)));
options.addOption(newOption("et", "end_time", true, "Packet end time
range. Default is current system time."));
options.addOption(newOption("df", "date_format", true, "Date format to use
for parsing start_time and end_time. Default is to use time in millis since the
epoch."));
+options.addOption(newOption("ps", "print_status", false, "Print the status
of the job as it runs"));
return options;
}
@@ -125,6 +126,9 @@ public class CliParser {
//no-op
}
}
+if (commandLine.hasOption("print_status")) {
+ config.setPrintJobStatus(true);
+}
}
public void printHelp(String msg, Options opts) {
http://git-wip-us.apache.org/repos/asf/metron/blob/b29833cc/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
--
diff --git
a/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
b/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
index 96ca354..7c75224 100644
---
a/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
+++
b/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
@@ -147,7 +147,8 @@ public class PcapCliTest {
"-protocol", "6",
"-include_reverse",
"-num_reducers", "10",
-"-records_per_file", "1000"
+"-records_per_file", "1000",
+"-ps"
[32/50] [abbrv] metron git commit: METRON-1731: PCAP - Escape colons in output dir names (mmiklavc via mmiklavc) closes apache/metron#1155
METRON-1731: PCAP - Escape colons in output dir names (mmiklavc via mmiklavc)
closes apache/metron#1155
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/73dc63e6
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/73dc63e6
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/73dc63e6
Branch: refs/remotes/upstream/feature/METRON-1699-create-batch-profiler
Commit: 73dc63e671b55d22d251f4be1c217259f4f5dc71
Parents: 05316a4
Author: mmiklavc
Authored: Fri Aug 10 12:42:47 2018 -0600
Committer: Michael Miklavcic
Committed: Fri Aug 10 12:42:47 2018 -0600
--
.../apache/metron/pcap/FixedPcapFilterTest.java | 286 --
.../org/apache/metron/pcap/PcapJobTest.java | 290 ---
.../apache/metron/pcap/QueryPcapFilterTest.java | 228 ---
.../pcap/filter/fixed/FixedPcapFilter.java | 14 +-
.../pcap/filter/query/QueryPcapFilter.java | 17 +-
.../metron/pcap/mr/OutputDirFormatter.java | 37 +++
.../java/org/apache/metron/pcap/mr/PcapJob.java | 5 +-
.../pcap/filter/fixed/FixedPcapFilterTest.java | 271 -
.../pcap/filter/query/QueryPcapFilterTest.java | 207 -
.../metron/pcap/mr/OutputDirFormatterTest.java | 62
.../org/apache/metron/pcap/mr/PcapJobTest.java | 290 +++
11 files changed, 877 insertions(+), 830 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/73dc63e6/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/FixedPcapFilterTest.java
--
diff --git
a/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/FixedPcapFilterTest.java
b/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/FixedPcapFilterTest.java
deleted file mode 100644
index 84969d3..000
---
a/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/FixedPcapFilterTest.java
+++ /dev/null
@@ -1,286 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.metron.pcap;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.metron.common.Constants;
-import org.apache.metron.pcap.filter.fixed.FixedPcapFilter;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.util.HashMap;
-import java.util.Map;
-
-public class FixedPcapFilterTest {
- @Test
- public void testTrivialEquality() throws Exception {
-Configuration config = new Configuration();
-final Map fields = new HashMap() {{
- put(Constants.Fields.SRC_ADDR.getName(), "src_ip");
- put(Constants.Fields.SRC_PORT.getName(), "0");
- put(Constants.Fields.DST_ADDR.getName(), "dst_ip");
- put(Constants.Fields.DST_PORT.getName(), "1");
- put(Constants.Fields.INCLUDES_REVERSE_TRAFFIC.getName(), "false");
-}};
-new FixedPcapFilter.Configurator().addToConfig(fields, config);
-{
- FixedPcapFilter filter = new FixedPcapFilter() {
-@Override
-protected Map packetToFields(PacketInfo pi) {
- return new HashMap() {{
-put(Constants.Fields.SRC_ADDR.getName(), "src_ip");
-put(Constants.Fields.SRC_PORT.getName(), 0);
-put(Constants.Fields.DST_ADDR.getName(), "dst_ip");
-put(Constants.Fields.DST_PORT.getName(), 1);
- }};
-}
- };
- filter.configure(config);
- Assert.assertTrue(filter.test(null));
-}
- }
-
- @Test
- public void testReverseTraffic() throws Exception {
-Configuration config = new Configuration();
-final Map fields = new HashMap() {{
- put(Constants.Fields.SRC_ADDR.getName(), "src_ip");
- put(Constants.Fields.SRC_PORT.getName(), "0");
- put(Constants.Fields.DST_ADDR.getName(), "dst_ip");
- put(Constants.Fields.DST_PORT.getName(), "1");
- put(Constants.Fields.INCLUDES_REVERSE_TRAFFIC.getName(), "true");
-}};
-new FixedPcapFilter.Configurator().addToConfig(fields, config);
-{
- FixedPcapFilter filter =
[36/50] [abbrv] metron git commit: METRON-1735 Empty print status option causes NPE (merrimanr) closes apache/metron#1160
METRON-1735 Empty print status option causes NPE (merrimanr) closes
apache/metron#1160
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/8a926dd5
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/8a926dd5
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/8a926dd5
Branch: refs/remotes/upstream/feature/METRON-1699-create-batch-profiler
Commit: 8a926dd5a4501df8c7ef51b526ad5e4d752a5e19
Parents: bce9b90
Author: merrimanr
Authored: Tue Aug 14 12:59:38 2018 -0500
Committer: rmerriman
Committed: Tue Aug 14 12:59:38 2018 -0500
--
.../org/apache/metron/pcap/query/CliParser.java | 4
.../apache/metron/pcap/query/PcapCliTest.java| 7 ++-
.../apache/metron/pcap/config/PcapConfig.java| 2 +-
.../java/org/apache/metron/pcap/mr/PcapJob.java | 18 +-
.../org/apache/metron/pcap/mr/PcapJobTest.java | 19 +++
5 files changed, 35 insertions(+), 15 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/8a926dd5/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
--
diff --git
a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
index 5040f90..b9a2a50 100644
---
a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
+++
b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
@@ -55,7 +55,6 @@ public class CliParser {
options.addOption(newOption("rpf", "records_per_file", true,
String.format("Number of records to include in each output pcap file (defaults
to %s)", NUM_RECORDS_PER_FILE_DEFAULT)));
options.addOption(newOption("et", "end_time", true, "Packet end time
range. Default is current system time."));
options.addOption(newOption("df", "date_format", true, "Date format to use
for parsing start_time and end_time. Default is to use time in millis since the
epoch."));
-options.addOption(newOption("ps", "print_status", false, "Print the status
of the job as it runs"));
options.addOption(newOption("yq", "yarn_queue", true, "Yarn queue this job
will be submitted to"));
return options;
}
@@ -127,9 +126,6 @@ public class CliParser {
//no-op
}
}
-if (commandLine.hasOption("print_status")) {
- config.setPrintJobStatus(true);
-}
if (commandLine.hasOption("yarn_queue")) {
config.setYarnQueue(commandLine.getOptionValue("yarn_queue"));
}
http://git-wip-us.apache.org/repos/asf/metron/blob/8a926dd5/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
--
diff --git
a/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
b/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
index a71e997..3e7aad2 100644
---
a/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
+++
b/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
@@ -172,8 +172,7 @@ public class PcapCliTest {
"-protocol", "6",
"-include_reverse",
"-num_reducers", "10",
-"-records_per_file", "1000",
-"-ps"
+"-records_per_file", "1000"
};
Map query = new HashMap() {{
put(Constants.Fields.SRC_ADDR.getName(), "192.168.1.1");
@@ -217,7 +216,6 @@ public class PcapCliTest {
"-include_reverse",
"-num_reducers", "10",
"-records_per_file", "1000",
-"-ps",
"-yq", "pcap"
};
Map query = new HashMap() {{
@@ -295,8 +293,7 @@ public class PcapCliTest {
"-base_path", "/base/path",
"-base_output_path", "/base/output/path",
"-query", "some query string",
-"-records_per_file", "1000",
-"-ps"
+"-records_per_file", "1000"
};
String query = "some query string";
http://git-wip-us.apache.org/repos/asf/metron/blob/8a926dd5/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/config/PcapConfig.java
--
diff --git
a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/config/PcapConfig.java
b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/config/PcapConfig.java
index 4a08e14..abf35d0 100644
---
[14/50] [abbrv] metron git commit: METRON-1662 PCAP UI - Downloading PCAP page files (tiborm via merrimanr) closes apache/metron#1118
METRON-1662 PCAP UI - Downloading PCAP page files (tiborm via merrimanr) closes
apache/metron#1118
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/b445bfe2
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/b445bfe2
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/b445bfe2
Branch: refs/remotes/upstream/feature/METRON-1699-create-batch-profiler
Commit: b445bfe242123e544c1fef41e2ab5bdc07dbc56e
Parents: 2b6959b
Author: tiborm
Authored: Fri Aug 3 08:13:11 2018 -0500
Committer: rmerriman
Committed: Fri Aug 3 08:13:11 2018 -0500
--
.../metron-alerts/src/app/pcap/model/pdml.ts| 12 +++-
.../pcap/pcap-packet/pcap-packet.component.ts | 9 ++
.../pcap/pcap-panel/pcap-panel.component.html | 2 +-
.../app/pcap/pcap-panel/pcap-panel.component.ts | 30 +++-
.../src/app/pcap/service/pcap.service.ts| 15 ++
5 files changed, 33 insertions(+), 35 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/b445bfe2/metron-interface/metron-alerts/src/app/pcap/model/pdml.ts
--
diff --git a/metron-interface/metron-alerts/src/app/pcap/model/pdml.ts
b/metron-interface/metron-alerts/src/app/pcap/model/pdml.ts
index 97fc347..34245c6 100644
--- a/metron-interface/metron-alerts/src/app/pcap/model/pdml.ts
+++ b/metron-interface/metron-alerts/src/app/pcap/model/pdml.ts
@@ -17,8 +17,7 @@
*/
export class PdmlField {
- //public $: any[]
-public name: string;
+ public name: string;
public fields?: PdmlField[]
public show: string;
public value: string;
@@ -26,9 +25,8 @@
}
export class PdmlProto {
- //public $: any
-public name: string;
-public showname: string;
+ public name: string;
+ public showname: string;
public fields: PdmlField[]
public static findField(p: PdmlProto, name: string): PdmlField {
@@ -36,10 +34,8 @@ export class PdmlProto {
}
}
-
export class PdmlPacket {
- //public $: any
-public name: string;
+ public name: string;
public protos: PdmlProto[]
public expanded: boolean = false
}
http://git-wip-us.apache.org/repos/asf/metron/blob/b445bfe2/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.ts
b/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.ts
index f1bbe2d..ca28a39 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.ts
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.ts
@@ -24,16 +24,11 @@ import { PdmlPacket } from '../model/pdml'
templateUrl: './pcap-packet.component.html',
styleUrls: ['./pcap-packet.component.scss']
})
-export class PcapPacketComponent implements OnInit {
+export class PcapPacketComponent {
@Input() packet: PdmlPacket
- constructor() { }
-
- ngOnInit() {
- }
-
toggle() {
-this.packet.expanded = !this.packet.expanded
+this.packet.expanded = !this.packet.expanded;
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/b445bfe2/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.html
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.html
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.html
index b373914..1fd250f 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.html
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.html
@@ -27,5 +27,5 @@
-
+ Download PCAP
http://git-wip-us.apache.org/repos/asf/metron/blob/b445bfe2/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.ts
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.ts
index fd49ec7..4114e24 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.ts
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.ts
@@ -15,47 +15,45 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-import { Component, OnInit, Input } from '@angular/core';
+import { Component, Input } from '@angular/core';
-import { PcapService, PcapStatusResponse } from '../service/pcap.service'
-import { PcapRequest } from '../model/pcap.request'
-import
[11/50] [abbrv] metron git commit: METRON-1671 Create PCAP UI (tiborm via merrimanr) closes apache/metron#1103
http://git-wip-us.apache.org/repos/asf/metron/blob/d5eb56a9/metron-interface/metron-alerts/package-lock.json
--
diff --git a/metron-interface/metron-alerts/package-lock.json
b/metron-interface/metron-alerts/package-lock.json
index 3ace2b7..7d3ad99 100644
--- a/metron-interface/metron-alerts/package-lock.json
+++ b/metron-interface/metron-alerts/package-lock.json
@@ -1,6 +1,6 @@
{
"name": "metron-alerts",
- "version": "0.4.3",
+ "version": "0.5.1",
"lockfileVersion": 1,
"requires": true,
"dependencies": {
@@ -12,66 +12,66 @@
"requires": {
"@ngtools/json-schema": "1.1.0",
"@ngtools/webpack": "1.5.3",
-"autoprefixer": "6.7.7",
-"chalk": "2.3.0",
-"circular-dependency-plugin": "3.0.0",
-"common-tags": "1.5.1",
-"core-object": "3.1.5",
-"css-loader": "0.28.7",
-"cssnano": "3.10.0",
-"denodeify": "1.2.1",
-"diff": "3.2.0",
-"ember-cli-normalize-entity-name": "1.0.0",
-"ember-cli-string-utils": "1.1.0",
-"exports-loader": "0.6.4",
-"extract-text-webpack-plugin": "2.1.2",
-"file-loader": "0.10.1",
-"fs-extra": "4.0.2",
-"get-caller-file": "1.0.2",
-"glob": "7.1.2",
-"heimdalljs": "0.2.5",
-"heimdalljs-logger": "0.1.9",
-"html-webpack-plugin": "2.30.1",
-"inflection": "1.12.0",
-"inquirer": "3.3.0",
-"isbinaryfile": "3.0.2",
-"istanbul-instrumenter-loader": "2.0.0",
-"json-loader": "0.5.7",
-"karma-source-map-support": "1.2.0",
-"less": "2.7.3",
-"less-loader": "4.0.5",
-"license-webpack-plugin": "0.4.3",
-"lodash": "4.17.4",
-"memory-fs": "0.4.1",
-"minimatch": "3.0.4",
-"node-modules-path": "1.0.1",
-"node-sass": "4.5.3",
-"nopt": "4.0.1",
-"opn": "5.1.0",
-"portfinder": "1.0.13",
-"postcss-loader": "1.3.3",
-"postcss-url": "5.1.2",
-"raw-loader": "0.5.1",
-"resolve": "1.3.3",
-"rsvp": "3.6.2",
-"rxjs": "5.4.0",
-"sass-loader": "6.0.6",
-"script-loader": "0.7.2",
-"semver": "5.3.0",
-"silent-error": "1.1.0",
-"source-map-loader": "0.2.3",
-"style-loader": "0.13.2",
-"stylus": "0.54.5",
-"stylus-loader": "3.0.1",
+"autoprefixer": "^6.5.3",
+"chalk": "^2.0.1",
+"circular-dependency-plugin": "^3.0.0",
+"common-tags": "^1.3.1",
+"core-object": "^3.1.0",
+"css-loader": "^0.28.1",
+"cssnano": "^3.10.0",
+"denodeify": "^1.2.1",
+"diff": "^3.1.0",
+"ember-cli-normalize-entity-name": "^1.0.0",
+"ember-cli-string-utils": "^1.0.0",
+"exports-loader": "^0.6.3",
+"extract-text-webpack-plugin": "^2.1.0",
+"file-loader": "^0.10.0",
+"fs-extra": "^4.0.0",
+"get-caller-file": "^1.0.0",
+"glob": "^7.0.3",
+"heimdalljs": "^0.2.4",
+"heimdalljs-logger": "^0.1.9",
+"html-webpack-plugin": "^2.19.0",
+"inflection": "^1.7.0",
+"inquirer": "^3.0.0",
+"isbinaryfile": "^3.0.0",
+"istanbul-instrumenter-loader": "^2.0.0",
+"json-loader": "^0.5.4",
+"karma-source-map-support": "^1.2.0",
+"less": "^2.7.2",
+"less-loader": "^4.0.2",
+"license-webpack-plugin": "^0.4.2",
+"lodash": "^4.11.1",
+"memory-fs": "^0.4.1",
+"minimatch": "^3.0.3",
+"node-modules-path": "^1.0.0",
+"node-sass": "^4.3.0",
+"nopt": "^4.0.1",
+"opn": "~5.1.0",
+"portfinder": "~1.0.12",
+"postcss-loader": "^1.3.3",
+"postcss-url": "^5.1.2",
+"raw-loader": "^0.5.1",
+"resolve": "^1.1.7",
+"rsvp": "^3.0.17",
+"rxjs": "^5.0.1",
+"sass-loader": "^6.0.3",
+"script-loader": "^0.7.0",
+"semver": "^5.1.0",
+"silent-error": "^1.0.0",
+"source-map-loader": "^0.2.0",
+"style-loader": "^0.13.1",
+"stylus": "^0.54.5",
+"stylus-loader": "^3.0.1",
"temp": "0.8.3",
-"typescript": "2.2.2",
-"url-loader": "0.5.9",
-"walk-sync": "0.3.2",
-"webpack": "2.4.1",
-"webpack-dev-middleware": "1.12.2",
-"webpack-dev-server": "2.4.5",
-"webpack-merge": "2.6.1",
-"zone.js": "0.8.18"
+"typescript": ">=2.0.0 <2.4.0",
+"url-loader": "^0.5.7",
+"walk-sync": "^0.3.1",
+"webpack": "~2.4.0",
+"webpack-dev-middleware": "^1.10.2",
+"webpack-dev-server": "~2.4.5",
+"webpack-merge": "^2.4.0",
+"zone.js": "^0.8.14"
},
"dependencies": {
"ansi-styles": {
@@ -80,7 +80,7 @@
"integrity":
[49/50] [abbrv] metron git commit: METRON-1751 Storm Profiler dies when consuming null message (nickwallen) closes apache/metron#1176
METRON-1751 Storm Profiler dies when consuming null message (nickwallen) closes
apache/metron#1176
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/d32bd50d
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/d32bd50d
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/d32bd50d
Branch: refs/remotes/upstream/feature/METRON-1699-create-batch-profiler
Commit: d32bd50d43aae87af9ec12d2daea83b2f4eca342
Parents: 661e23e
Author: nickwallen
Authored: Wed Aug 29 14:55:58 2018 -0400
Committer: nickallen
Committed: Wed Aug 29 14:55:58 2018 -0400
--
.../profiler/bolt/ProfileSplitterBolt.java | 29 +++-
.../profiler/bolt/ProfileSplitterBoltTest.java | 16 +++
2 files changed, 32 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/d32bd50d/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java
--
diff --git
a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java
b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java
index f28411f..87f1ba9 100644
---
a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java
+++
b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java
@@ -144,9 +144,9 @@ public class ProfileSplitterBolt extends
ConfiguredProfilerBolt {
try {
doExecute(input);
-} catch (IllegalArgumentException | ParseException |
UnsupportedEncodingException e) {
- LOG.error("Unexpected error", e);
- collector.reportError(e);
+} catch (Throwable t) {
+ LOG.error("Unexpected error", t);
+ collector.reportError(t);
} finally {
collector.ack(input);
@@ -157,22 +157,25 @@ public class ProfileSplitterBolt extends
ConfiguredProfilerBolt {
// retrieve the input message
byte[] data = input.getBinary(0);
+if(data == null) {
+ LOG.debug("Received null message. Nothing to do.");
+ return;
+}
JSONObject message = (JSONObject) parser.parse(new String(data, "UTF8"));
// ensure there is a valid profiler configuration
ProfilerConfig config = getProfilerConfig();
-if(config != null && config.getProfiles().size() > 0) {
-
- // what time is it?
- Clock clock = clockFactory.createClock(config);
- Optional timestamp = clock.currentTimeMillis(message);
+if(config == null || getProfilerConfig().getProfiles().size() == 0) {
+ LOG.debug("No Profiler configuration found. Nothing to do.");
+ return;
+}
- // route the message. if a message does not contain the timestamp
field, it cannot be routed.
- timestamp.ifPresent(ts -> routeMessage(input, message, config, ts));
+// what time is it?
+Clock clock = clockFactory.createClock(config);
+Optional timestamp = clock.currentTimeMillis(message);
-} else {
- LOG.debug("No Profiler configuration found. Nothing to do.");
-}
+// route the message. if a message does not contain the timestamp field,
it cannot be routed.
+timestamp.ifPresent(ts -> routeMessage(input, message, config, ts));
}
/**
http://git-wip-us.apache.org/repos/asf/metron/blob/d32bd50d/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileSplitterBoltTest.java
--
diff --git
a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileSplitterBoltTest.java
b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileSplitterBoltTest.java
index bf81923..72e2b72 100644
---
a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileSplitterBoltTest.java
+++
b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileSplitterBoltTest.java
@@ -404,6 +404,22 @@ public class ProfileSplitterBoltTest extends BaseBoltTest {
.emit(any(Values.class));
}
+ @Test
+ public void testWithNullMessage() throws Exception {
+
+// ensure the tuple returns null to mimic a null message in kafka
+when(tuple.getBinary(0)).thenReturn(null);
+
+ProfilerConfig config = toProfilerConfig(profileWithOnlyIfInvalid);
+ProfileSplitterBolt bolt = createBolt(config);
+bolt.execute(tuple);
+
+// a tuple should NOT be emitted for the downstream profile builder
+verify(outputCollector, times(0))
+.emit(any(Values.class));
+
+ }
+
/**
* Creates a ProfilerConfig based on a string containing JSON.
*
[40/50] [abbrv] metron git commit: METRON-1727: Alerts are not populated on the alerts UI after enabling X-pack for Elastic search (MohanDV via mmiklavc) closes apache/metron#1141
METRON-1727: Alerts are not populated on the alerts UI after enabling X-pack for Elastic search (MohanDV via mmiklavc) closes apache/metron#1141 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/308c2b27 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/308c2b27 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/308c2b27 Branch: refs/remotes/apache/feature/METRON-1699-create-batch-profiler Commit: 308c2b27b9b234c0c84c343dfc45b3617b027e64 Parents: 2b0060b Author: MohanDV Authored: Wed Aug 15 16:16:26 2018 -0600 Committer: Michael Miklavcic Committed: Wed Aug 15 16:20:54 2018 -0600 -- metron-deployment/Kerberos-manual-setup.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/308c2b27/metron-deployment/Kerberos-manual-setup.md -- diff --git a/metron-deployment/Kerberos-manual-setup.md b/metron-deployment/Kerberos-manual-setup.md index e3de148..18891a4 100644 --- a/metron-deployment/Kerberos-manual-setup.md +++ b/metron-deployment/Kerberos-manual-setup.md @@ -580,7 +580,7 @@ X-Pack Submit the update to Zookeeper ``` -$METRON_HOME/bin/zk_load_configs.sh -m PUSH -i METRON_HOME/config/zookeeper/ -z $ZOOKEEPER +$METRON_HOME/bin/zk_load_configs.sh -m PUSH -i $METRON_HOME/config/zookeeper/ -z $ZOOKEEPER ``` 1. The last step before restarting the topology is to create a custom X-Pack shaded and relocated jar. This is up to you because of licensing restrictions, but here is a sample Maven pom file that should help. @@ -737,6 +737,8 @@ X-Pack $METRON_HOME/bin/start_elasticsearch_topology.sh ``` +1. Restart the metron-rest service, and make sure the elasticsearch-xpack-shaded-5.6.2.jar is in the METRON_REST_CLASSPATH when the metron-rest starts. + Once you've performed these steps, you should be able to start seeing data in your ES indexes. ### X-Pack Common Problems
[35/50] [abbrv] metron git commit: METRON-1733 PCAP UI - PCAP queries don t work on Safari (sardell via merrimanr) closes apache/metron#1158
METRON-1733 PCAP UI - PCAP queries dont work on Safari (sardell via
merrimanr) closes apache/metron#1158
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/bce9b903
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/bce9b903
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/bce9b903
Branch: refs/remotes/apache/feature/METRON-1699-create-batch-profiler
Commit: bce9b903fbd3436ea0c41eb8d75da279e16ce922
Parents: ba0a609
Author: sardell
Authored: Mon Aug 13 16:04:45 2018 -0500
Committer: rmerriman
Committed: Mon Aug 13 16:04:45 2018 -0500
--
.../src/app/pcap/pcap-filters/pcap-filters.component.ts | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/bce9b903/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.ts
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.ts
index df91ae3..b23a2e2 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.ts
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.ts
@@ -59,8 +59,8 @@ export class PcapFiltersComponent implements OnInit,
OnChanges {
}
onSubmit() {
-this.model.startTimeMs = new Date(this.startTimeStr).getTime();
-this.model.endTimeMs = new Date(this.endTimeStr).getTime();
+this.model.startTimeMs = moment(this.startTimeStr,
DEFAULT_TIMESTAMP_FORMAT).valueOf();
+this.model.endTimeMs = moment(this.endTimeStr,
DEFAULT_TIMESTAMP_FORMAT).valueOf();
if (this.ipSrcPort !== '') {
this.model.ipSrcPort = +this.ipSrcPort;
} else {
[41/50] [abbrv] metron git commit: METRON-1739 UDP packets are not handled (merrimanr) closes apache/metron#1168
METRON-1739 UDP packets are not handled (merrimanr) closes apache/metron#1168
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/14e80b3c
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/14e80b3c
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/14e80b3c
Branch: refs/remotes/apache/feature/METRON-1699-create-batch-profiler
Commit: 14e80b3c229ea2e5a8d615c87d822d4b44d8ec98
Parents: 076a6a1
Author: merrimanr
Authored: Fri Aug 17 08:47:32 2018 -0500
Committer: rmerriman
Committed: Fri Aug 17 08:47:32 2018 -0500
--
.../src/app/pcap/model/pdml.mock.ts | 31
.../pcap-packet-line.component.spec.ts | 53 --
.../pcap-packet-line.component.ts | 15 ++--
.../java/org/apache/metron/pcap/PcapHelper.java | 18 -
.../org/apache/metron/pcap/PcapHelperTest.java | 76
5 files changed, 180 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/14e80b3c/metron-interface/metron-alerts/src/app/pcap/model/pdml.mock.ts
--
diff --git a/metron-interface/metron-alerts/src/app/pcap/model/pdml.mock.ts
b/metron-interface/metron-alerts/src/app/pcap/model/pdml.mock.ts
index 6316b92..2520da1 100644
--- a/metron-interface/metron-alerts/src/app/pcap/model/pdml.mock.ts
+++ b/metron-interface/metron-alerts/src/app/pcap/model/pdml.mock.ts
@@ -60,3 +60,34 @@ export const fakePacket = {
} as PdmlProto
]
} as PdmlPacket;
+
+export const fakeUdpPacket = {
+ "name": '',
+ "expanded": false,
+ "protos": [
+{
+ "name": "geninfo",
+ "showname": "",
+ "fields": [
+{ "name": "timestamp", "pos": "0", "showname": "Captured Time",
"size": "342", "value": "1534414874.932794000", "show": "Aug 16, 2018
10:21:14.932794000 UTC", "unmaskedvalue": null, "hide": null, "fields": null,
"protos": null } as PdmlField
+ ]
+} as PdmlProto,
+{
+ "name": "ip",
+ "showname": "",
+ "fields": [
+{ "name": "ip.proto", "pos": "23", "showname": "Protocol: UDP (17)",
"size": "1", "value": "11", "show": "17", "unmaskedvalue": null, "hide": null,
"fields": null, "protos": null } as PdmlField,
+{ "name": "ip.src", "pos": "26", "showname": "Source: 0.0.0.0
(0.0.0.0)", "size": "4", "value": "", "show": "0.0.0.0",
"unmaskedvalue": null, "hide": null, "fields": null, "protos": null } as
PdmlField,
+{ "name": "ip.dst", "pos": "30", "showname": "Destination:
255.255.255.255 (255.255.255.255)", "size": "4", "value": "", "show":
"255.255.255.255", "unmaskedvalue": null, "hide": null, "fields": null,
"protos": null } as PdmlField
+ ]
+} as PdmlProto,
+{
+ "name": "udp",
+ "showname": "User Datagram Protocol, Src Port: bootpc (68), Dst Port:
bootps (67)",
+ "fields": [
+{ "name": "udp.srcport", "pos": "34", "showname": "Source port: bootpc
(68)", "size": "2", "value": "0044", "show": "68", "unmaskedvalue": null,
"hide": null, "fields": null, "protos": null } as PdmlField,
+{ "name": "udp.dstport", "pos": "36", "showname": "Destination port:
bootps (67)", "size": "2", "value": "0043", "show": "67", "unmaskedvalue":
null, "hide": null, "fields": null, "protos": null } as PdmlField
+ ],
+} as PdmlProto
+ ]
+} as PdmlPacket;
http://git-wip-us.apache.org/repos/asf/metron/blob/14e80b3c/metron-interface/metron-alerts/src/app/pcap/pcap-packet-line/pcap-packet-line.component.spec.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-packet-line/pcap-packet-line.component.spec.ts
b/metron-interface/metron-alerts/src/app/pcap/pcap-packet-line/pcap-packet-line.component.spec.ts
index d01c0cb..00f081f 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-packet-line/pcap-packet-line.component.spec.ts
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-packet-line/pcap-packet-line.component.spec.ts
@@ -17,6 +17,7 @@
*/
import { async, ComponentFixture, TestBed } from '@angular/core/testing';
import { fakePacket } from '../model/pdml.mock';
+import { fakeUdpPacket } from '../model/pdml.mock';
import { PcapPacketLineComponent } from './pcap-packet-line.component';
@@ -34,8 +35,6 @@ describe('PcapPacketLineComponent', () => {
beforeEach(() => {
fixture = TestBed.createComponent(PcapPacketLineComponent);
component = fixture.componentInstance;
-component.packet = fakePacket;
-fixture.detectChanges();
});
it('should be created', () => {
@@ -43,6 +42,8 @@ describe('PcapPacketLineComponent', () => {
});
it('should extract timestamp fields', () => {
+component.packet = fakePacket;
+
[02/50] [abbrv] metron git commit: METRON-1661 Create Pcap Query Filter endpoint (merrimanr) closes apache/metron#1125
METRON-1661 Create Pcap Query Filter endpoint (merrimanr) closes
apache/metron#1125
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/d0e87576
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/d0e87576
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/d0e87576
Branch: refs/remotes/upstream/feature/METRON-1699-create-batch-profiler
Commit: d0e87576944b6370eb1d9a0c98d0450f2c51f052
Parents: e78bc53
Author: merrimanr
Authored: Mon Jul 23 09:09:37 2018 -0500
Committer: rmerriman
Committed: Mon Jul 23 09:09:37 2018 -0500
--
.../rest/model/pcap/FixedPcapRequest.java | 1 +
.../metron/rest/model/pcap/PcapRequest.java | 4 +-
.../rest/model/pcap/QueryPcapOptions.java | 35 +
.../rest/model/pcap/QueryPcapRequest.java | 59 ++
metron-interface/metron-rest/README.md | 10 ++-
.../metron/rest/controller/PcapController.java | 15 +++-
.../apache/metron/rest/service/PcapService.java | 8 +-
.../rest/service/impl/PcapServiceImpl.java | 13 ++--
.../PcapControllerIntegrationTest.java | 81 +---
.../apache/metron/rest/mock/MockPcapJob.java| 13 ++--
.../rest/service/impl/PcapServiceImplTest.java | 55 +++--
11 files changed, 240 insertions(+), 54 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/d0e87576/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/FixedPcapRequest.java
--
diff --git
a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/FixedPcapRequest.java
b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/FixedPcapRequest.java
index a2d345b..d91aac7 100644
---
a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/FixedPcapRequest.java
+++
b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/FixedPcapRequest.java
@@ -87,6 +87,7 @@ public class FixedPcapRequest extends PcapRequest {
FixedPcapOptions.INCLUDE_REVERSE.put(this, includeReverse);
}
+ @Override
public void setFields() {
Map fields = new HashMap<>();
if (getIpSrcAddr() != null) {
http://git-wip-us.apache.org/repos/asf/metron/blob/d0e87576/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/PcapRequest.java
--
diff --git
a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/PcapRequest.java
b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/PcapRequest.java
index 64ed932..44721fa 100644
---
a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/PcapRequest.java
+++
b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/PcapRequest.java
@@ -22,7 +22,7 @@ import org.apache.metron.pcap.config.PcapOptions;
import java.util.HashMap;
-public class PcapRequest extends AbstractMapDecorator {
+public abstract class PcapRequest extends AbstractMapDecorator
{
public PcapRequest() {
super(new HashMap<>());
@@ -78,4 +78,6 @@ public class PcapRequest extends AbstractMapDecorator {
public void setNumReducers(Integer numReducers) {
PcapOptions.NUM_REDUCERS.put(this, numReducers);
}
+
+ public abstract void setFields();
}
http://git-wip-us.apache.org/repos/asf/metron/blob/d0e87576/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/QueryPcapOptions.java
--
diff --git
a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/QueryPcapOptions.java
b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/QueryPcapOptions.java
new file mode 100644
index 000..1ae3da6
--- /dev/null
+++
b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/QueryPcapOptions.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR
[50/50] [abbrv] metron git commit: Merge remote-tracking branch 'apache/master' into metron-1699-create-batch-profiler
Merge remote-tracking branch 'apache/master' into metron-1699-create-batch-profiler Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/f5eaef3c Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/f5eaef3c Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/f5eaef3c Branch: refs/remotes/upstream/feature/METRON-1699-create-batch-profiler Commit: f5eaef3c24e939ef14757eb2db4b758e71d33fca Parents: 113ac0d d32bd50 Author: Nick Allen Authored: Fri Aug 31 15:11:49 2018 -0400 Committer: Nick Allen Committed: Fri Aug 31 15:11:49 2018 -0400 -- dependencies_with_url.csv | 8 + metron-analytics/metron-profiler-common/pom.xml |26 +- .../metron/profiler/ProfileMeasurement.java |35 + .../apache/metron/profiler/ProfilePeriod.java |11 + .../metron/profiler/ProfileMeasurementTest.java |28 +- .../metron/profiler/ProfilePeriodTest.java |22 +- .../profiler/hbase/SaltyRowKeyBuilderTest.java |11 - metron-analytics/metron-profiler/README.md |18 +- .../profiler/bolt/ProfileSplitterBolt.java |22 +- .../profiler/bolt/ProfileSplitterBoltTest.java |16 + .../integration/ProfilerIntegrationTest.java|30 +- metron-deployment/Kerberos-manual-setup.md | 4 +- metron-deployment/amazon-ec2/conf/defaults.yml | 1 - metron-deployment/amazon-ec2/playbook.yml | 2 +- .../ansible/playbooks/metron_install.yml| 7 - .../roles/ambari_config/vars/single_node_vm.yml | 1 + .../roles/metron_pcapservice/defaults/main.yml |28 - .../roles/metron_pcapservice/meta/main.yml |19 - .../metron_pcapservice/tasks/config-hbase.yml |26 - .../roles/metron_pcapservice/tasks/main.yml |25 - .../metron_pcapservice/tasks/pcapservice.yml|30 - .../templates/metron-pcapservice-logrotate.yml |27 - .../metron_pcapservice/templates/pcapservice|84 - .../ubuntu14/ansible/inventory/hosts| 4 - .../CURRENT/configuration/metron-rest-env.xml |48 + .../package/scripts/params/params_linux.py |16 + .../package/scripts/params/status_params.py | 7 + .../CURRENT/package/scripts/rest_commands.py|62 + .../CURRENT/package/scripts/rest_master.py | 9 + .../METRON/CURRENT/package/templates/metron.j2 | 6 + .../METRON/CURRENT/themes/metron_theme.json |30 + .../docker/rpm-docker/SPECS/metron.spec | 1 + .../metron-alerts/package-lock.json | 3719 ++-- metron-interface/metron-alerts/pom.xml | 6 +- .../metron-alerts/src/app/app-routing.module.ts | 3 +- .../metron-alerts/src/app/app.component.html|10 +- .../metron-alerts/src/app/app.component.scss|18 + .../metron-alerts/src/app/app.component.ts | 3 + .../metron-alerts/src/app/app.module.ts | 6 +- .../src/app/pcap/model/pcap-pagination.ts |21 + .../src/app/pcap/model/pcap-status-response.ts |24 + .../src/app/pcap/model/pcap.mock.ts |39 + .../src/app/pcap/model/pcap.request.ts |29 + .../src/app/pcap/model/pdml.mock.ts |93 + .../metron-alerts/src/app/pcap/model/pdml.ts|50 + .../pcap-filters/pcap-filters.component.html|73 + .../pcap-filters/pcap-filters.component.scss|70 + .../pcap-filters/pcap-filters.component.spec.ts | 464 + .../pcap/pcap-filters/pcap-filters.component.ts | 167 + .../app/pcap/pcap-list/pcap-list.component.html |38 + .../app/pcap/pcap-list/pcap-list.component.scss |21 + .../pcap/pcap-list/pcap-list.component.spec.ts |98 + .../app/pcap/pcap-list/pcap-list.component.ts |43 + .../pcap-packet-line.component.html |19 + .../pcap-packet-line.component.scss |17 + .../pcap-packet-line.component.spec.ts | 179 + .../pcap-packet-line.component.ts |56 + .../pcap/pcap-packet/pcap-packet.component.html |22 + .../pcap/pcap-packet/pcap-packet.component.scss |36 + .../pcap-packet/pcap-packet.component.spec.ts |79 + .../pcap/pcap-packet/pcap-packet.component.ts |34 + .../pcap-pagination.component.html |19 + .../pcap-pagination.component.scss |40 + .../pcap-pagination.component.spec.ts |78 + .../pcap-pagination.component.ts|41 + .../pcap/pcap-panel/pcap-panel.component.html |31 + .../pcap/pcap-panel/pcap-panel.component.scss |67 + .../pcap-panel/pcap-panel.component.spec.ts | 557 + .../app/pcap/pcap-panel/pcap-panel.component.ts | 164 + .../metron-alerts/src/app/pcap/pcap.module.ts |55 + .../metron-alerts/src/app/pcap/pcap.routing.ts |27 + .../src/app/pcap/service/pcap.service.spec.ts | 197 + .../src/app/pcap/service/pcap.service.ts
[48/50] [abbrv] metron git commit: METRON-1757 Storm Profiler Serialization Exception (nickwallen) closes apache/metron#1178
METRON-1757 Storm Profiler Serialization Exception (nickwallen) closes
apache/metron#1178
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/661e23e2
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/661e23e2
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/661e23e2
Branch: refs/remotes/upstream/feature/METRON-1699-create-batch-profiler
Commit: 661e23e274d2bf88370d8feb5e5dbf1b82f15124
Parents: 29cbdcc
Author: nickwallen
Authored: Wed Aug 29 12:51:05 2018 -0400
Committer: nickallen
Committed: Wed Aug 29 12:51:05 2018 -0400
--
metron-analytics/metron-profiler-common/pom.xml | 26 +++
.../metron/profiler/ProfileMeasurement.java | 35
.../apache/metron/profiler/ProfilePeriod.java | 3 ++
.../metron/profiler/ProfileMeasurementTest.java | 28 +---
.../metron/profiler/ProfilePeriodTest.java | 23 ++---
.../profiler/hbase/SaltyRowKeyBuilderTest.java | 11 --
metron-analytics/metron-profiler/README.md | 18 ++
.../integration/ProfilerIntegrationTest.java| 30 +
.../configuration/profiler/ProfileResult.java | 4 +++
.../profiler/ProfileResultExpressions.java | 4 +++
.../profiler/ProfilerConfigTest.java| 35 +++-
11 files changed, 154 insertions(+), 63 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/661e23e2/metron-analytics/metron-profiler-common/pom.xml
--
diff --git a/metron-analytics/metron-profiler-common/pom.xml
b/metron-analytics/metron-profiler-common/pom.xml
index 928515f..74253cc 100644
--- a/metron-analytics/metron-profiler-common/pom.xml
+++ b/metron-analytics/metron-profiler-common/pom.xml
@@ -55,6 +55,10 @@
org.slf4j
slf4j-log4j12
+
+asm
+asm
+
@@ -109,30 +113,10 @@
provided
-org.apache.storm
-storm-core
-${global_storm_version}
-provided
-
-
-org.apache.logging.log4j
-log4j-api
-
-
-org.apache.logging.log4j
-log4j-core
-
-
-log4j-slf4j-impl
-org.apache.logging.log4j
-
-
-
-
org.mockito
mockito-all
${global_mockito_version}
test
-
\ No newline at end of file
+
http://git-wip-us.apache.org/repos/asf/metron/blob/661e23e2/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileMeasurement.java
--
diff --git
a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileMeasurement.java
b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileMeasurement.java
index 4737c3d..2212297 100644
---
a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileMeasurement.java
+++
b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileMeasurement.java
@@ -20,6 +20,8 @@
package org.apache.metron.profiler;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.metron.common.configuration.profiler.ProfileConfig;
import java.io.Serializable;
@@ -174,4 +176,37 @@ public class ProfileMeasurement implements Serializable {
public void setTriageValues(Map triageValues) {
this.triageValues = triageValues;
}
+
+ @Override
+ public boolean equals(Object o) {
+if (this == o) {
+ return true;
+}
+if (o == null || getClass() != o.getClass()) {
+ return false;
+}
+ProfileMeasurement that = (ProfileMeasurement) o;
+return new EqualsBuilder()
+.append(profileName, that.profileName)
+.append(entity, that.entity)
+.append(groups, that.groups)
+.append(period, that.period)
+.append(definition, that.definition)
+.append(profileValue, that.profileValue)
+.append(triageValues, that.triageValues)
+.isEquals();
+ }
+
+ @Override
+ public int hashCode() {
+return new HashCodeBuilder(17, 37)
+.append(profileName)
+.append(entity)
+.append(groups)
+.append(period)
+.append(definition)
+.append
[26/50] [abbrv] metron git commit: METRON-1713 PCAP UI - Add a way to kill a pcap job (tiborm via merrimanr) closes apache/metron#1143
METRON-1713 PCAP UI - Add a way to kill a pcap job (tiborm via merrimanr)
closes apache/metron#1143
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/14dcb2d9
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/14dcb2d9
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/14dcb2d9
Branch: refs/remotes/upstream/feature/METRON-1699-create-batch-profiler
Commit: 14dcb2d90581835d8206c65918c24e8cb04bfd06
Parents: 5b3e2c3
Author: tiborm
Authored: Thu Aug 9 08:33:21 2018 -0500
Committer: rmerriman
Committed: Thu Aug 9 08:33:21 2018 -0500
--
.../pcap/pcap-panel/pcap-panel.component.html | 7 +-
.../pcap/pcap-panel/pcap-panel.component.scss | 33 +
.../pcap-panel/pcap-panel.component.spec.ts | 130 +++
.../app/pcap/pcap-panel/pcap-panel.component.ts | 42 +-
.../src/app/pcap/service/pcap.service.ts| 10 +-
5 files changed, 215 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/14dcb2d9/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.html
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.html
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.html
index 950f49c..0dda268 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.html
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.html
@@ -15,8 +15,11 @@
-
-{{progressWidth}}%
+
+
+ {{progressWidth}}%
+
+
{{ errorMsg }}
http://git-wip-us.apache.org/repos/asf/metron/blob/14dcb2d9/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.scss
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.scss
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.scss
index 8989bf9..523f5ce 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.scss
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.scss
@@ -31,4 +31,37 @@
.progress-bar {
width: 0;
+ height: 34px;
+ line-height: 34px;
+ vertical-align: middle;
+ font-size: 0.875rem;
+}
+
+.pcap-progress-wrapper {
+ position: relative;
+ padding-right: 55px;
+}
+
+.pcap-cancel-query-button {
+ position: absolute;
+ top: 0;
+ right: 0;
+ padding-top: 6px;
+ padding-bottom: 6px;
+ background: $icon-button-background;
+ min-width: 42px;
+ padding-left: 0;
+ padding-right: 0;
+ border: 1px solid $blue-chill !important;
+ cursor: pointer;
+
+ &:focus {
+box-shadow: none;
+ }
+
+ &::before {
+font-family: "FontAwesome";
+content: '\f00d';
+color: $piction-blue;
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/14dcb2d9/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.spec.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.spec.ts
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.spec.ts
index 0804b79..9dacc7f 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.spec.ts
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.spec.ts
@@ -50,6 +50,9 @@ class FakePcapService {
return '';
}
submitRequest() {}
+ cancelQuery() {
+return defer(() => Promise.resolve());
+ }
}
describe('PcapPanelComponent', () => {
@@ -327,6 +330,133 @@ describe('PcapPanelComponent', () => {
expect(fixture.debugElement.query(By.css('app-pcap-list'))).toBeDefined();
}));
+ it('should render a cancel button only if a query runs', () => {
+component.queryRunning = false;
+fixture.detectChanges();
+
expect(fixture.debugElement.query(By.css('[data-qe-id="pcap-cancel-query-button"]'))).toBeFalsy();
+
+component.queryRunning = true;
+fixture.detectChanges();
+
expect(fixture.debugElement.query(By.css('[data-qe-id="pcap-cancel-query-button"]'))).toBeDefined();
+ });
+
+ it('should hide the progress bar if the user clicks on the cancel button',
fakeAsync(() => {
+component.queryRunning = true;
+component.queryId = 'testid';
+fixture.detectChanges();
+expect(fixture.debugElement.query(By.css('.pcap-progress'))).toBeDefined();
+
+const cancelBtn =
fixture.debugElement.query(By.css('[data-qe-id="pcap-cancel-query-button"]'));
+const cancelBtnEl = cancelBtn.nativeElement;
+
+cancelBtnEl.click();
+tick();
+
[46/50] [abbrv] metron git commit: METRON-1752 Prevent package.lock from changing during build (sardell via merrimanr) closes apache/metron#1177
METRON-1752 Prevent package.lock from changing during build (sardell via merrimanr) closes apache/metron#1177 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/09826da8 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/09826da8 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/09826da8 Branch: refs/remotes/upstream/feature/METRON-1699-create-batch-profiler Commit: 09826da85eddb014a4b718233d53480220b1f5dd Parents: e66cfc8 Author: sardell Authored: Mon Aug 27 15:14:19 2018 -0500 Committer: rmerriman Committed: Mon Aug 27 15:14:19 2018 -0500 -- metron-interface/metron-alerts/pom.xml | 6 +- .../metron-config/package-lock.json | 17734 + metron-interface/metron-config/pom.xml |30 +- 3 files changed, 17752 insertions(+), 18 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/09826da8/metron-interface/metron-alerts/pom.xml -- diff --git a/metron-interface/metron-alerts/pom.xml b/metron-interface/metron-alerts/pom.xml index 996f473..f67a013 100644 --- a/metron-interface/metron-alerts/pom.xml +++ b/metron-interface/metron-alerts/pom.xml @@ -26,7 +26,7 @@ UTF-8 UTF-8 v9.11.1 -5.8.0 +6.2.0 @@ -52,12 +52,12 @@ generate-resources -npm install +npm ci npm -install +ci
[37/50] [abbrv] metron git commit: METRON-1732: Fix job status liveness bug and parallelize finalizer file writing (mmiklavc via mmiklavc) closes apache/metron#1157
METRON-1732: Fix job status liveness bug and parallelize finalizer file writing (mmiklavc via mmiklavc) closes apache/metron#1157 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/d9e1f381 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/d9e1f381 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/d9e1f381 Branch: refs/remotes/upstream/feature/METRON-1699-create-batch-profiler Commit: d9e1f381cc1ae9edfbffc7216abffdbd8f942c09 Parents: 8a926dd Author: mmiklavc Authored: Wed Aug 15 11:30:10 2018 -0600 Committer: Michael Miklavcic Committed: Wed Aug 15 11:30:10 2018 -0600 -- .../CURRENT/configuration/metron-rest-env.xml | 14 +++ .../package/scripts/params/params_linux.py | 1 + .../METRON/CURRENT/package/templates/metron.j2 | 1 + .../METRON/CURRENT/themes/metron_theme.json | 10 +++ metron-interface/metron-rest/README.md | 10 +++ .../src/main/config/rest_application.yml| 1 + .../apache/metron/rest/MetronRestConstants.java | 1 + .../rest/service/impl/PcapServiceImpl.java | 25 +++--- .../apache/metron/rest/mock/MockPcapJob.java| 18 ++-- .../rest/service/impl/PcapServiceImplTest.java | 49 ++- .../org/apache/metron/pcap/query/CliParser.java | 10 ++- .../PcapTopologyIntegrationTest.java| 1 + .../apache/metron/pcap/query/PcapCliTest.java | 21 ++--- .../java/org/apache/metron/pcap/PcapPages.java | 2 +- .../apache/metron/pcap/config/PcapConfig.java | 6 ++ .../metron/pcap/config/PcapGlobalDefaults.java | 1 + .../apache/metron/pcap/config/PcapOptions.java | 3 +- .../metron/pcap/finalizer/PcapFinalizer.java| 70 +-- .../java/org/apache/metron/pcap/mr/PcapJob.java | 90 ++-- .../org/apache/metron/pcap/mr/PcapJobTest.java | 16 +++- 20 files changed, 258 insertions(+), 92 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/d9e1f381/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml -- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml index 895c091..767afa3 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml @@ -201,5 +201,19 @@ true + +pcap_finalizer_threadpool_size +Pcap Finalizer Threadpool Size +The number of threads to use when finalizing Pcap jobs. This affects parallelism + around writing out paged files to their final location. + If it's a string and ends with "C", then strip the C and treat it as an integral multiple of + the number of cores. If it's a string and does not end with a C, then treat it as a number in + string form. + +1 + +false + + http://git-wip-us.apache.org/repos/asf/metron/blob/d9e1f381/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py -- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py index 4f8a9a7..115a54c 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py @@ -391,6 +391,7 @@ pcap_base_interim_result_path = config['configurations']['metron-rest-env']['pca pcap_final_output_path = config['configurations']['metron-rest-env']['pcap_final_output_path'] pcap_page_size = config['configurations']['metron-rest-env']['pcap_page_size'] pcap_yarn_queue = config['configurations']['metron-rest-env']['pcap_yarn_queue'] +pcap_finalizer_threadpool_size= config['configurations']['metron-rest-env']['pcap_finalizer_threadpool_size'] pcap_configured_flag_file = status_params.pcap_configured_flag_file # MapReduce
[09/50] [abbrv] metron git commit: METRON-1671 Create PCAP UI (tiborm via merrimanr) closes apache/metron#1103
http://git-wip-us.apache.org/repos/asf/metron/blob/d5eb56a9/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.spec.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.spec.ts
b/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.spec.ts
new file mode 100644
index 000..155e1f8
--- /dev/null
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.spec.ts
@@ -0,0 +1,1283 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { async, ComponentFixture, TestBed } from '@angular/core/testing';
+
+import { PcapPacketComponent } from './pcap-packet.component';
+import { PdmlPacket } from '../model/pdml';
+
+describe('PcapPacketComponent', () => {
+ let component: PcapPacketComponent;
+ let fixture: ComponentFixture;
+
+ beforeEach(async(() => {
+TestBed.configureTestingModule({
+ declarations: [ PcapPacketComponent ]
+})
+.compileComponents();
+ }));
+
+ beforeEach(() => {
+fixture = TestBed.createComponent(PcapPacketComponent);
+component = fixture.componentInstance;
+component.packet = fakePacket as PdmlPacket;
+fixture.detectChanges();
+ });
+
+ it('should create', () => {
+expect(component).toBeTruthy();
+ });
+});
+
+const fakePacket = {
+ name: "something",
+ expanded: false,
+ protos: [
+{
+ "name": "geninfo",
+ "pos": "0",
+ "showname": "General information",
+ "size": "722",
+ "hide": null,
+ "fields": [
+{
+ "name": "num",
+ "pos": "0",
+ "showname": "Number",
+ "size": "722",
+ "value": "1",
+ "show": "1",
+ "unmaskedvalue": null,
+ "hide": null,
+ "fields": null,
+ "protos": null
+},
+{
+ "name": "len",
+ "pos": "0",
+ "showname": "Frame Length",
+ "size": "722",
+ "value": "2d2",
+ "show": "722",
+ "unmaskedvalue": null,
+ "hide": null,
+ "fields": null,
+ "protos": null
+},
+{
+ "name": "caplen",
+ "pos": "0",
+ "showname": "Captured Length",
+ "size": "722",
+ "value": "2d2",
+ "show": "722",
+ "unmaskedvalue": null,
+ "hide": null,
+ "fields": null,
+ "protos": null
+},
+{
+ "name": "timestamp",
+ "pos": "0",
+ "showname": "Captured Time",
+ "size": "722",
+ "value": "1458240269.373968000",
+ "show": "Mar 17, 2016 18:44:29.373968000 UTC",
+ "unmaskedvalue": null,
+ "hide": null,
+ "fields": null,
+ "protos": null
+}
+ ]
+},
+{
+ "name": "frame",
+ "pos": "0",
+ "showname": "Frame 1: 722 bytes on wire (5776 bits), 722 bytes captured
(5776 bits)",
+ "size": "722",
+ "hide": null,
+ "fields": [
+{
+ "name": "frame.dlt",
+ "pos": "0",
+ "showname": "WTAP_ENCAP: 1",
+ "size": "0",
+ "value": null,
+ "show": "1",
+ "unmaskedvalue": null,
+ "hide": null,
+ "fields": null,
+ "protos": null
+},
+{
+ "name": "frame.time",
+ "pos": "0",
+ "showname": "Arrival Time: Mar 17, 2016 18:44:29.373968000 UTC",
+ "size": "0",
+ "value": null,
+ "show": "Mar 17, 2016 18:44:29.373968000",
+ "unmaskedvalue": null,
+ "hide": null,
+ "fields": null,
+ "protos": null
+},
+{
+ "name": "frame.offset_shift",
+ "pos": "0",
+ "showname": "Time shift for this packet: 0.0 seconds",
+ "size": "0",
+ "value": null,
+ "show": "0.0",
+ "unmaskedvalue": null,
+ "hide": null,
+ "fields": null,
+ "protos": null
+},
+{
+ "name": "frame.time_epoch",
+ "pos": "0",
+
[27/50] [abbrv] metron git commit: METRON-1730: Update steps to run pycapa on Centos 6 (mmiklavc via mmiklavc) closes apache/metron#1152
METRON-1730: Update steps to run pycapa on Centos 6 (mmiklavc via mmiklavc)
closes apache/metron#1152
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/2b0060b0
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/2b0060b0
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/2b0060b0
Branch: refs/remotes/upstream/feature/METRON-1699-create-batch-profiler
Commit: 2b0060b0663ba3fa34047afdfe2fa856fe9007e8
Parents: 1b2bdff
Author: mmiklavc
Authored: Thu Aug 9 09:21:33 2018 -0600
Committer: Michael Miklavcic
Committed: Thu Aug 9 09:21:33 2018 -0600
--
metron-sensors/pycapa/README.md| 92 ++---
metron-sensors/pycapa/requirements.txt | 6 +-
2 files changed, 88 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/2b0060b0/metron-sensors/pycapa/README.md
--
diff --git a/metron-sensors/pycapa/README.md b/metron-sensors/pycapa/README.md
index 9f01d33..38baf74 100644
--- a/metron-sensors/pycapa/README.md
+++ b/metron-sensors/pycapa/README.md
@@ -20,6 +20,8 @@ Pycapa
* [Overview](#overview)
* [Installation](#installation)
+ * [Centos 7](#centos-7)
+ * [Centos 6](#centos-6)
* [Usage](#usage)
* [Parameters](#parameters)
* [Examples](#examples)
@@ -27,22 +29,23 @@ Pycapa
* [FAQs](#faqs)
Overview
-
+
Pycapa performs network packet capture, both off-the-wire and from a Kafka
topic, which is useful for the testing and development of [Apache
Metron](https://github.com/apache/metron). It is not intended for production
use. The tool will capture packets from a specified interface and push them
into a Kafka Topic. The tool can also do the reverse. It can consume packets
from Kafka and reconstruct each network packet. This can then be used to
create a [libpcap-compliant
file](https://wiki.wireshark.org/Development/LibpcapFileFormat) or even to feed
directly into a tool like Wireshark to monitor ongoing activity.
Installation
-
+
General notes on the installation of Pycapa.
* Python 2.7 is required.
* The following package dependencies are required and can be installed
automatically with `pip`. The requirements are installed as part of step 4
*
[confluent-kafka-python](https://github.com/confluentinc/confluent-kafka-python)
* [pcapy](https://github.com/CoreSecurity/pcapy)
-* These instructions can be used directly on CentOS 7+.
-* Other Linux distributions that come with Python 2.7 can use these
instructions with some minor modifications.
-* Older distributions, like CentOS 6, that come with Python 2.6 installed,
should install Python 2.7 within a virtual environment and then run Pycapa from
within the virtual environment.
+### Centos 7
+
+* These instructions can be used directly on CentOS 7+.
+* Other Linux distributions that come with Python 2.7 can use these
instructions with some minor modifications.
1. Install system dependencies including the core development tools, Python
libraries and header files, and Libpcap libraries and header files. On CentOS
7+, you can install these requirements with the following command.
@@ -76,8 +79,83 @@ General notes on the installation of Pycapa.
python setup.py install
```
+### Centos 6
+
+* These instructions can be used directly on CentOS 6 - useful for developers
using the Full Dev Vagrant test box.
+* Older distributions, like CentOS 6, that come with Python 2.6 installed,
should install Python 2.7 within a virtual environment and then run Pycapa from
within the virtual environment.
+
+1. Set up a couple environment variables.
+
+```
+PYCAPA_HOME=/opt/pycapa
+PYTHON27_HOME=/opt/rh/python27/root
+```
+
+1. Install required packages.
+
+```
+for item in epel-release centos-release-scl "@Development tools" python27
python27-scldevel python27-python-virtualenv libpcap-devel libselinux-python;
do yum install -y $item; done
+```
+
+1. Setup Pycapa directory.
+
+```
+mkdir $PYCAPA_HOME && chmod 755 $PYCAPA_HOME
+```
+
+1. Create the virtualenv.
+
+```
+export LD_LIBRARY_PATH="/opt/rh/python27/root/usr/lib64"
+cd $PYCAPA_HOME
+${PYTHON27_HOME}/usr/bin/virtualenv pycapa-venv
+```
+
+1. Install Librdkafka at your chosen $PREFIX.
+
+```
+export PREFIX=/usr
+wget https://github.com/edenhill/librdkafka/archive/v0.11.5.tar.gz -O -
| tar -xz
+cd librdkafka-0.11.5/
+./configure --prefix=$PREFIX
+make
+make install
+```
+
+1. Add Librdkafka to the dynamic library load path.
+
+```
+echo "$PREFIX/lib" >> /etc/ld.so.conf.d/pycapa.conf
+ldconfig -v
+```
+
+1. Copy the Pycapa source files from the Metron project to your
[31/50] [abbrv] metron git commit: METRON-1702 Reload a running job in the UI (merrimanr) closes apache/metron#1156
METRON-1702 Reload a running job in the UI (merrimanr) closes apache/metron#1156
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/05316a4d
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/05316a4d
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/05316a4d
Branch: refs/remotes/upstream/feature/METRON-1699-create-batch-profiler
Commit: 05316a4d989d7546058e73b3a75eef435f2608a0
Parents: b29833c
Author: merrimanr
Authored: Fri Aug 10 08:27:06 2018 -0500
Committer: rmerriman
Committed: Fri Aug 10 08:27:06 2018 -0500
--
.../src/app/pcap/model/pcap.request.ts | 4 +-
.../pcap-filters/pcap-filters.component.spec.ts | 60 +-
.../pcap/pcap-filters/pcap-filters.component.ts | 26 --
.../pcap/pcap-panel/pcap-panel.component.html | 2 +-
.../pcap-panel/pcap-panel.component.spec.ts | 37 +
.../app/pcap/pcap-panel/pcap-panel.component.ts | 87 +---
.../src/app/pcap/service/pcap.service.spec.ts | 44 ++
.../src/app/pcap/service/pcap.service.ts| 15
.../rest/service/impl/PcapServiceImpl.java | 10 +--
.../rest/service/impl/PcapServiceImplTest.java | 36
10 files changed, 254 insertions(+), 67 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/05316a4d/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
--
diff --git a/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
b/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
index d00a6ac..8afc963 100644
--- a/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
+++ b/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
@@ -20,9 +20,9 @@ export class PcapRequest {
startTimeMs: number = 0;
endTimeMs: number = 15;
ipSrcAddr: string = '';
- ipSrcPort: number = 0;
+ ipSrcPort: number;
ipDstAddr: string = '';
- ipDstPort: number = 0;
+ ipDstPort: number;
protocol: string = '';
packetFilter: string = '';
includeReverse: boolean = false;
http://git-wip-us.apache.org/repos/asf/metron/blob/05316a4d/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
index 4336b22..c1d8664 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
@@ -21,7 +21,7 @@ import { By } from '@angular/platform-browser';
import { PcapFiltersComponent } from './pcap-filters.component';
import { FormsModule } from '../../../../node_modules/@angular/forms';
-import { Component, Input, Output, EventEmitter, DebugElement } from
'@angular/core';
+import { Component, Input, Output, EventEmitter, DebugElement, SimpleChange }
from '@angular/core';
import { PcapRequest } from '../model/pcap.request';
@Component({
@@ -171,6 +171,14 @@ describe('PcapFiltersComponent', () => {
component.onSubmit();
});
+ it('Port fields should be missing by default', () => {
+component.search.emit = (model: PcapRequest) => {
+ expect(model.ipSrcPort).toBeFalsy();
+ expect(model.ipDstPort).toBeFalsy();
+};
+component.onSubmit();
+ });
+
it('Filter should have an output called search', () => {
component.search.subscribe((filterModel) => {
expect(filterModel).toBeDefined();
@@ -194,14 +202,60 @@ describe('PcapFiltersComponent', () => {
expect(fixture.componentInstance.model.hasOwnProperty('startTimeMs')).toBeTruthy();
expect(fixture.componentInstance.model.hasOwnProperty('endTimeMs')).toBeTruthy();
expect(fixture.componentInstance.model.hasOwnProperty('ipSrcAddr')).toBeTruthy();
-
expect(fixture.componentInstance.model.hasOwnProperty('ipSrcPort')).toBeTruthy();
+
expect(fixture.componentInstance.model.hasOwnProperty('ipSrcPort')).toBeFalsy();
expect(fixture.componentInstance.model.hasOwnProperty('ipDstAddr')).toBeTruthy();
-
expect(fixture.componentInstance.model.hasOwnProperty('ipDstPort')).toBeTruthy();
+
expect(fixture.componentInstance.model.hasOwnProperty('ipDstPort')).toBeFalsy();
expect(fixture.componentInstance.model.hasOwnProperty('protocol')).toBeTruthy();
expect(fixture.componentInstance.model.hasOwnProperty('packetFilter')).toBeTruthy();
expect(fixture.componentInstance.model.hasOwnProperty('includeReverse')).toBeTruthy();
});
+ it('should update request on changes',
[19/50] [abbrv] metron git commit: METRON-1683 PCAP UI - Fix the download progress bar (sardell via merrimanr) closes apache/metron#1122
http://git-wip-us.apache.org/repos/asf/metron/blob/3e778592/metron-interface/metron-alerts/src/app/pcap/service/pcap.service.spec.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/service/pcap.service.spec.ts
b/metron-interface/metron-alerts/src/app/pcap/service/pcap.service.spec.ts
index fdaa569..c8cb937 100644
--- a/metron-interface/metron-alerts/src/app/pcap/service/pcap.service.spec.ts
+++ b/metron-interface/metron-alerts/src/app/pcap/service/pcap.service.spec.ts
@@ -15,1738 +15,139 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-import { TestBed, async, inject } from '@angular/core/testing';
+import {
+ TestBed,
+ inject,
+ fakeAsync,
+ tick,
+ discardPeriodicTasks
+} from '@angular/core/testing';
import {
BaseRequestOptions,
HttpModule,
- Http,
Response,
- ResponseOptions
+ ResponseOptions,
+ XHRBackend
} from '@angular/http';
import { MockBackend } from '@angular/http/testing';
+import { Observable } from 'rxjs/Rx';
import { PcapService } from './pcap.service';
+import { PcapStatusResponse } from '../model/pcap-status-response';
import { PcapRequest } from '../model/pcap.request';
+import { fakePdml, fakePacket } from '../model/pdml.mock';
+import { fakePcapStatusResponse, fakePcapRequest } from '../model/pcap.mock';
+
+const jobId = 'job_1234567890123_4567';
+let pdmlJsonMock = fakePdml;
+pdmlJsonMock['pdml']['packet'].push(fakePacket);
describe('PcapService', () => {
beforeEach(() => {
-
TestBed.configureTestingModule({
imports: [HttpModule],
providers: [
PcapService,
-{
- provide: Http,
- useFactory: (mockBackend, options) => {
-return new Http(mockBackend, options);
- },
- deps: [MockBackend, BaseRequestOptions]
-},
+{ provide: XHRBackend, useClass: MockBackend },
MockBackend,
BaseRequestOptions
]
});
});
describe('getPackets()', () => {
-it('should return an Observable',
- inject([PcapService, MockBackend], (pcapService, mockBackend) => {
-
-let request: PcapRequest = {
- startTimeMs: 0,
- endTimeMs: 0,
- ipSrcAddr: '0.0.0.0',
- ipSrcPort: '80',
- ipDstAddr: '0.0.0.0',
- ipDstPort: '80',
- protocol: '*',
- packetFilter: '*',
- includeReverse: false,
-};
-
-mockBackend.connections.subscribe((connection) => {
- connection.mockRespond(new Response(new ResponseOptions({body:
pdml_json()})));
-});
+it('should return an Observable', inject(
+ [PcapService, XHRBackend],
+ (pcapService, mockBackend) => {
let packets;
-pcapService.getPackets(request).subscribe(r => packets = r)
-expect(packets).toBeTruthy()
-expect(packets.pdml).toBeTruthy()
-expect(packets.pdml.packet.length).toBe(1)
-expect(packets.pdml.packet[0].proto.length).toBeGreaterThan(3)
-
-console.log(packets)
- }))
-
-
-it('should ...', inject([PcapService], (service: PcapService) => {
- expect(service).toBeTruthy();
-}));
-
- })
-
-});
+mockBackend.connections.subscribe(connection => {
+ expect(connection.request.url).toMatch(
+/\/api\/v1\/pcap\/job_1234567890123_4567\/pdml\?page=1/
+ );
+ connection.mockRespond(
+new Response(new ResponseOptions({ body: pdmlJsonMock }))
+ );
+});
+pcapService.getPackets(jobId, 1).subscribe(r => (packets = r));
+expect(packets).toBeTruthy();
+expect(packets.pdml).toBeTruthy();
+expect(packets.pdml.packet.length).toBe(1);
+expect(packets.pdml.packet[0].protos.length).toBe(3);
+ }
+));
+ });
+ describe('pollStatus()', () => {
+it('should call getStatus() in intervals', fakeAsync(inject(
+ [PcapService, XHRBackend], (pcapService, mockBackend) => {
+ const responseMock: PcapStatusResponse = fakePcapStatusResponse;
+ const spy = spyOn(pcapService, 'getStatus').and.returnValue(
+Observable.of(responseMock)
+ );
+ let response;
+
+ pcapService.pollStatus(jobId).subscribe(r => (response = r));
+ tick(4000);
+ expect(spy.calls.count()).toBe(1);
+ tick(4000);
+ expect(spy.calls.count()).toBe(2);
+ discardPeriodicTasks();
+})
+ )
+);
+ });
+ describe('submitRequest()', () => {
+it('should return an Observable', inject(
+ [PcapService, XHRBackend],
+ (pcapService, mockBackend) => {
+const request: PcapRequest = fakePcapRequest;
+const responseMock: PcapStatusResponse = fakePcapStatusResponse;
+let response;
+
+mockBackend.connections.subscribe(connection => {
+
[13/50] [abbrv] metron git commit: METRON-1700 Create REST endpoint to get job configuration (merrimanr) closes apache/metron#1135
METRON-1700 Create REST endpoint to get job configuration (merrimanr) closes
apache/metron#1135
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/2b6959b4
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/2b6959b4
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/2b6959b4
Branch: refs/remotes/upstream/feature/METRON-1699-create-batch-profiler
Commit: 2b6959b46db79256d12b8522385212559cabada2
Parents: d5eb56a
Author: merrimanr
Authored: Thu Aug 2 12:17:37 2018 -0500
Committer: rmerriman
Committed: Thu Aug 2 12:17:37 2018 -0500
--
metron-interface/metron-rest/README.md | 10 ++
.../metron/rest/controller/PcapController.java | 16 +++
.../apache/metron/rest/service/PcapService.java | 71 -
.../rest/service/impl/PcapServiceImpl.java | 54 +-
.../PcapControllerIntegrationTest.java | 50 ++
.../apache/metron/rest/mock/MockPcapJob.java| 4 +-
.../rest/service/impl/PcapServiceImplTest.java | 100 +++
.../java/org/apache/metron/job/Statusable.java | 2 +
.../java/org/apache/metron/pcap/mr/PcapJob.java | 5 +
9 files changed, 306 insertions(+), 6 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/2b6959b4/metron-interface/metron-rest/README.md
--
diff --git a/metron-interface/metron-rest/README.md
b/metron-interface/metron-rest/README.md
index 68ec559..489cd9f 100644
--- a/metron-interface/metron-rest/README.md
+++ b/metron-interface/metron-rest/README.md
@@ -259,6 +259,8 @@ Request and Response objects are JSON formatted. The JSON
schemas are available
| [ `GET /api/v1/pcap/{jobId}`](#get-apiv1pcapjobid)|
| [ `GET /api/v1/pcap/{jobId}/pdml`](#get-apiv1pcapjobidpdml)|
| [ `GET /api/v1/pcap/{jobId}/raw`](#get-apiv1pcapjobidraw)|
+| [ `DELETE /api/v1/pcap/kill/{jobId}`](#delete-apiv1pcapkilljobid)|
+| [ `GET /api/v1/pcap/{jobId}/config`](#get-apiv1pcapjobidconfig)|
| [ `GET /api/v1/search/search`](#get-apiv1searchsearch)|
| [ `POST /api/v1/search/search`](#post-apiv1searchsearch)|
| [ `POST /api/v1/search/group`](#post-apiv1searchgroup)|
@@ -556,6 +558,14 @@ Request and Response objects are JSON formatted. The JSON
schemas are available
* jobId - Job ID of submitted job
* Returns:
* 200 - Kills passed job.
+
+### `GET /api/v1/pcap/{jobId}/config`
+ * Description: Gets job configuration for Pcap query job.
+ * Input:
+* jobId - Job ID of submitted job
+ * Returns:
+* 200 - Returns a map of job properties for the Job ID.
+* 404 - Job is missing.
### `POST /api/v1/search/search`
* Description: Searches the indexing store. GUIDs must be quoted to ensure
correct results.
http://git-wip-us.apache.org/repos/asf/metron/blob/2b6959b4/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java
--
diff --git
a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java
b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java
index 13a623a..97713d8 100644
---
a/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java
+++
b/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/controller/PcapController.java
@@ -46,6 +46,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.util.List;
+import java.util.Map;
@RestController
@RequestMapping("/api/v1/pcap")
@@ -161,4 +162,19 @@ public class PcapController {
}
}
+ @ApiOperation(value = "Gets job configuration for Pcap query job.")
+ @ApiResponses(value = {
+ @ApiResponse(message = "Returns a map of job properties for the Job
ID.", code = 200),
+ @ApiResponse(message = "Job is missing.", code = 404)
+ })
+ @RequestMapping(value = "/{jobId}/config", method = RequestMethod.GET)
+ ResponseEntity> getConfiguration(@ApiParam(name="jobId",
value="Job ID of submitted job", required=true)@PathVariable String jobId)
throws RestException {
+Map configuration =
pcapQueryService.getConfiguration(SecurityUtils.getCurrentUser(), jobId);
+if (configuration != null) {
+ return new ResponseEntity<>(configuration, HttpStatus.OK);
+} else {
+ return new ResponseEntity<>(HttpStatus.NOT_FOUND);
+}
+ }
+
}
http://git-wip-us.apache.org/repos/asf/metron/blob/2b6959b4/metron-interface/metron-rest/src/main/java/org/apache/metron/rest/service/PcapService.java
--
diff --git
[01/50] [abbrv] metron git commit: METRON-1560 Update MPack to support Pcap panel (merrimanr) closes apache/metron#1124
Repository: metron
Updated Branches:
refs/remotes/upstream/feature/METRON-1699-create-batch-profiler 113ac0d3a ->
f5eaef3c2
METRON-1560 Update MPack to support Pcap panel (merrimanr) closes
apache/metron#1124
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/e78bc530
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/e78bc530
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/e78bc530
Branch: refs/remotes/upstream/feature/METRON-1699-create-batch-profiler
Commit: e78bc530260aa622e8263cda2b94be99194dffb5
Parents: 8763fba
Author: merrimanr
Authored: Mon Jul 23 09:08:12 2018 -0500
Committer: rmerriman
Committed: Mon Jul 23 09:08:12 2018 -0500
--
.../CURRENT/configuration/metron-rest-env.xml | 25
.../package/scripts/params/params_linux.py | 5 +++-
.../CURRENT/package/scripts/rest_commands.py| 18 --
.../METRON/CURRENT/package/templates/metron.j2 | 4
.../METRON/CURRENT/themes/metron_theme.json | 10
.../src/main/config/rest_application.yml| 6 -
.../src/main/config/pcap.properties | 2 +-
7 files changed, 65 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/e78bc530/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
--
diff --git
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
index 066b527..20f9767 100644
---
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
+++
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
@@ -168,4 +168,29 @@
The field name where the threat triage score can be found
in the search indices. This setting primarily affects the Alerts
UI.
threat:triage:score
+
+pcap_base_path
+Pcap Base Path
+The HDFS path where pcap files are stored. These files
are the input to pcap queries.
+/apps/metron/pcap/input
+
+
+pcap_base_interim_result_path
+Pcap Base Interim Result Path
+The HDFS path where interim results of a pcap query are
stored, before they are separated into pages.
+/apps/metron/pcap/interim
+
+
+pcap_final_output_path
+Pcap Final Output Path
+The HDFS path where the final results of a pcap query are
stored.
+/apps/metron/pcap/output
+
+
+pcap_page_size
+Pcap Page Size
+The number of pcaps written to a page/file as a result of
a pcap query.
+10
+
+
http://git-wip-us.apache.org/repos/asf/metron/blob/e78bc530/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
--
diff --git
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
index b8d6123..73d3469 100755
---
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
+++
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
@@ -386,7 +386,10 @@ bolt_hdfs_rotation_policy_units =
config['configurations']['metron-indexing-env'
bolt_hdfs_rotation_policy_count =
config['configurations']['metron-indexing-env']['bolt_hdfs_rotation_policy_count']
# Pcap
-pcap_hdfs_dir = format("{metron_apps_hdfs_dir}/pcap")
+pcap_base_path = config['configurations']['metron-rest-env']['pcap_base_path']
+pcap_base_interim_result_path =
config['configurations']['metron-rest-env']['pcap_base_interim_result_path']
+pcap_final_output_path =
config['configurations']['metron-rest-env']['pcap_final_output_path']
+pcap_page_size = config['configurations']['metron-rest-env']['pcap_page_size']
pcap_configured_flag_file = status_params.pcap_configured_flag_file
# MapReduce
[41/50] [abbrv] metron git commit: METRON-1739 UDP packets are not handled (merrimanr) closes apache/metron#1168
METRON-1739 UDP packets are not handled (merrimanr) closes apache/metron#1168
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/14e80b3c
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/14e80b3c
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/14e80b3c
Branch: refs/remotes/upstream/feature/METRON-1699-create-batch-profiler
Commit: 14e80b3c229ea2e5a8d615c87d822d4b44d8ec98
Parents: 076a6a1
Author: merrimanr
Authored: Fri Aug 17 08:47:32 2018 -0500
Committer: rmerriman
Committed: Fri Aug 17 08:47:32 2018 -0500
--
.../src/app/pcap/model/pdml.mock.ts | 31
.../pcap-packet-line.component.spec.ts | 53 --
.../pcap-packet-line.component.ts | 15 ++--
.../java/org/apache/metron/pcap/PcapHelper.java | 18 -
.../org/apache/metron/pcap/PcapHelperTest.java | 76
5 files changed, 180 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/14e80b3c/metron-interface/metron-alerts/src/app/pcap/model/pdml.mock.ts
--
diff --git a/metron-interface/metron-alerts/src/app/pcap/model/pdml.mock.ts
b/metron-interface/metron-alerts/src/app/pcap/model/pdml.mock.ts
index 6316b92..2520da1 100644
--- a/metron-interface/metron-alerts/src/app/pcap/model/pdml.mock.ts
+++ b/metron-interface/metron-alerts/src/app/pcap/model/pdml.mock.ts
@@ -60,3 +60,34 @@ export const fakePacket = {
} as PdmlProto
]
} as PdmlPacket;
+
+export const fakeUdpPacket = {
+ "name": '',
+ "expanded": false,
+ "protos": [
+{
+ "name": "geninfo",
+ "showname": "",
+ "fields": [
+{ "name": "timestamp", "pos": "0", "showname": "Captured Time",
"size": "342", "value": "1534414874.932794000", "show": "Aug 16, 2018
10:21:14.932794000 UTC", "unmaskedvalue": null, "hide": null, "fields": null,
"protos": null } as PdmlField
+ ]
+} as PdmlProto,
+{
+ "name": "ip",
+ "showname": "",
+ "fields": [
+{ "name": "ip.proto", "pos": "23", "showname": "Protocol: UDP (17)",
"size": "1", "value": "11", "show": "17", "unmaskedvalue": null, "hide": null,
"fields": null, "protos": null } as PdmlField,
+{ "name": "ip.src", "pos": "26", "showname": "Source: 0.0.0.0
(0.0.0.0)", "size": "4", "value": "", "show": "0.0.0.0",
"unmaskedvalue": null, "hide": null, "fields": null, "protos": null } as
PdmlField,
+{ "name": "ip.dst", "pos": "30", "showname": "Destination:
255.255.255.255 (255.255.255.255)", "size": "4", "value": "", "show":
"255.255.255.255", "unmaskedvalue": null, "hide": null, "fields": null,
"protos": null } as PdmlField
+ ]
+} as PdmlProto,
+{
+ "name": "udp",
+ "showname": "User Datagram Protocol, Src Port: bootpc (68), Dst Port:
bootps (67)",
+ "fields": [
+{ "name": "udp.srcport", "pos": "34", "showname": "Source port: bootpc
(68)", "size": "2", "value": "0044", "show": "68", "unmaskedvalue": null,
"hide": null, "fields": null, "protos": null } as PdmlField,
+{ "name": "udp.dstport", "pos": "36", "showname": "Destination port:
bootps (67)", "size": "2", "value": "0043", "show": "67", "unmaskedvalue":
null, "hide": null, "fields": null, "protos": null } as PdmlField
+ ],
+} as PdmlProto
+ ]
+} as PdmlPacket;
http://git-wip-us.apache.org/repos/asf/metron/blob/14e80b3c/metron-interface/metron-alerts/src/app/pcap/pcap-packet-line/pcap-packet-line.component.spec.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-packet-line/pcap-packet-line.component.spec.ts
b/metron-interface/metron-alerts/src/app/pcap/pcap-packet-line/pcap-packet-line.component.spec.ts
index d01c0cb..00f081f 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-packet-line/pcap-packet-line.component.spec.ts
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-packet-line/pcap-packet-line.component.spec.ts
@@ -17,6 +17,7 @@
*/
import { async, ComponentFixture, TestBed } from '@angular/core/testing';
import { fakePacket } from '../model/pdml.mock';
+import { fakeUdpPacket } from '../model/pdml.mock';
import { PcapPacketLineComponent } from './pcap-packet-line.component';
@@ -34,8 +35,6 @@ describe('PcapPacketLineComponent', () => {
beforeEach(() => {
fixture = TestBed.createComponent(PcapPacketLineComponent);
component = fixture.componentInstance;
-component.packet = fakePacket;
-fixture.detectChanges();
});
it('should be created', () => {
@@ -43,6 +42,8 @@ describe('PcapPacketLineComponent', () => {
});
it('should extract timestamp fields', () => {
+component.packet = fakePacket;
+
[05/50] [abbrv] metron git commit: METRON-1691: REST should limit the number of Pcap jobs a user can submit (merrimanr via mmiklavc) closes apache/metron#1129
METRON-1691: REST should limit the number of Pcap jobs a user can submit
(merrimanr via mmiklavc) closes apache/metron#1129
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/6c90724d
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/6c90724d
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/6c90724d
Branch: refs/remotes/upstream/feature/METRON-1699-create-batch-profiler
Commit: 6c90724d8d4f06c453128f860eff51037f1870d8
Parents: f1f5dda
Author: merrimanr
Authored: Wed Jul 25 10:58:05 2018 -0600
Committer: Michael Miklavcic
Committed: Wed Jul 25 10:58:05 2018 -0600
--
.../apache/metron/rest/model/pcap/Field.java| 36 ---
.../rest/model/pcap/FixedPcapRequest.java | 30 -
.../apache/metron/rest/model/pcap/Packet.java | 8 +--
.../metron/rest/model/pcap/PcapStatus.java | 24 +++
.../org/apache/metron/rest/model/pcap/Pdml.java | 21 +++---
.../apache/metron/rest/model/pcap/Proto.java| 24 +++
.../rest/model/pcap/QueryPcapRequest.java | 18 --
metron-interface/metron-rest/README.md | 27 ++--
.../apache/metron/rest/MetronRestConstants.java | 1 +
.../metron/rest/controller/PcapController.java | 13
.../apache/metron/rest/service/PcapService.java | 6 ++
.../rest/service/impl/PcapServiceImpl.java | 64 --
.../src/main/resources/application.yml | 1 +
.../PcapControllerIntegrationTest.java | 46 +
.../rest/service/impl/PcapServiceImplTest.java | 68
.../metron/job/manager/InMemoryJobManager.java | 4 ++
16 files changed, 261 insertions(+), 130 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/6c90724d/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/Field.java
--
diff --git
a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/Field.java
b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/Field.java
index 9c2878b..4ed71c3 100644
---
a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/Field.java
+++
b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/Field.java
@@ -22,6 +22,7 @@ import
com.fasterxml.jackson.dataformat.xml.annotation.JacksonXmlProperty;
import java.util.ArrayList;
import java.util.List;
+import java.util.Objects;
public class Field {
@@ -132,33 +133,22 @@ public class Field {
public boolean equals(Object o) {
if (this == o) return true;
if (o == null || getClass() != o.getClass()) return false;
-
Field field = (Field) o;
-
-return (getName() != null ? getName().equals(field.getName()) :
field.getName() != null) &&
-(getPos() != null ? getPos().equals(field.getPos()) :
field.getPos() == null) &&
-(getShowname() != null ? getShowname().equals(field.getShowname())
: field.getShowname() == null) &&
-(getSize() != null ? getSize().equals(field.getSize()) :
field.getSize() == null) &&
-(getValue() != null ? getValue().equals(field.getValue()) :
field.getValue() == null) &&
-(getShow() != null ? getShow().equals(field.getShow()) :
field.getShow() == null) &&
-(getUnmaskedvalue() != null ?
getUnmaskedvalue().equals(field.getUnmaskedvalue()) : field.getUnmaskedvalue()
== null) &&
-(getHide() != null ? getHide().equals(field.getHide()) :
field.getHide() == null) &&
-(getFields() != null ? getFields().equals(field.getFields()) :
field.getFields() == null) &&
-(getProtos() != null ? getProtos().equals(field.getProtos()) :
field.getProtos() == null);
+return Objects.equals(name, field.name) &&
+Objects.equals(pos, field.pos) &&
+Objects.equals(showname, field.showname) &&
+Objects.equals(size, field.size) &&
+Objects.equals(value, field.value) &&
+Objects.equals(show, field.show) &&
+Objects.equals(unmaskedvalue, field.unmaskedvalue) &&
+Objects.equals(hide, field.hide) &&
+Objects.equals(fields, field.fields) &&
+Objects.equals(protos, field.protos);
}
@Override
public int hashCode() {
-int result = getName() != null ? getName().hashCode() : 0;
-result = 31 * result + (getPos() != null ? getPos().hashCode() : 0);
-result = 31 * result + (getShowname() != null ? getShowname().hashCode() :
0);
-result = 31 * result + (getSize() != null ? getSize().hashCode() : 0);
-result = 31 * result + (getValue() != null ? getValue().hashCode() : 0);
-result = 31 * result + (getShow() != null ?
[17/50] [abbrv] metron git commit: METRON-1721 New default input path is wrong in pcap CLI (merrimanr) closes apache/metron#1137
METRON-1721 New default input path is wrong in pcap CLI (merrimanr) closes
apache/metron#1137
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/756ceedc
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/756ceedc
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/756ceedc
Branch: refs/remotes/upstream/feature/METRON-1699-create-batch-profiler
Commit: 756ceedc3d3567f3508478eaa5ed29f12c22688a
Parents: 8972665
Author: merrimanr
Authored: Mon Aug 6 13:36:40 2018 -0500
Committer: rmerriman
Committed: Mon Aug 6 13:36:40 2018 -0500
--
.../src/main/java/org/apache/metron/pcap/query/CliParser.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/756ceedc/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
--
diff --git
a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
index e6534c5..4ad6ffa 100644
---
a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
+++
b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
@@ -25,7 +25,7 @@ import org.apache.metron.pcap.config.PcapConfig;
* Provides commmon required fields for the PCAP filter jobs
*/
public class CliParser {
- public static final String BASE_PATH_DEFAULT = "/apps/metron/pcap";
+ public static final String BASE_PATH_DEFAULT = "/apps/metron/pcap/input";
public static final String BASE_INTERIM_OUTPUT_PATH_DEFAULT = "/tmp";
public static final int NUM_REDUCERS_DEFAULT = 10;
public static final int NUM_RECORDS_PER_FILE_DEFAULT = 1;
[25/50] [abbrv] metron git commit: METRON-1723 PCAP UI - Unable to select/copy from packets details in PCAP query panel (sardell via merrimanr) closes apache/metron#1139
METRON-1723 PCAP UI - Unable to select/copy from packets details in PCAP query
panel (sardell via merrimanr) closes apache/metron#1139
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/5b3e2c33
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/5b3e2c33
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/5b3e2c33
Branch: refs/remotes/upstream/feature/METRON-1699-create-batch-profiler
Commit: 5b3e2c337b3772bcd7d18cedac75877bbe6f7735
Parents: 52de126
Author: sardell
Authored: Thu Aug 9 07:44:15 2018 -0500
Committer: rmerriman
Committed: Thu Aug 9 07:44:15 2018 -0500
--
.../src/app/pcap/pcap-packet/pcap-packet.component.html | 2 +-
.../src/app/pcap/pcap-packet/pcap-packet.component.scss | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/5b3e2c33/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.html
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.html
b/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.html
index f7d9dbe..dd9afa1 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.html
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.html
@@ -13,7 +13,7 @@
-->
{{ proto.showname
}}
-
+
{{ field.name
}}
{{ field.showname }}
http://git-wip-us.apache.org/repos/asf/metron/blob/5b3e2c33/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.scss
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.scss
b/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.scss
index e9d4662..02ce7b6 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.scss
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.scss
@@ -27,10 +27,10 @@
}
.proto-fields {
- margin-left: 1em;
+ cursor: text;
+ padding-left: 1em;
}
-
.field-name {
display: none
}
[04/50] [abbrv] metron git commit: METRON-1693: Fix Pcap CLI local FS finalizer (mmiklavc via mmiklavc) closes apache/metron#1130
METRON-1693: Fix Pcap CLI local FS finalizer (mmiklavc via mmiklavc) closes
apache/metron#1130
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/f1f5dda4
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/f1f5dda4
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/f1f5dda4
Branch: refs/remotes/upstream/feature/METRON-1699-create-batch-profiler
Commit: f1f5dda4c643394c12ab9f4501c5b806adb21129
Parents: 9d4842f
Author: mmiklavc
Authored: Wed Jul 25 08:26:45 2018 -0600
Committer: Michael Miklavcic
Committed: Wed Jul 25 08:26:45 2018 -0600
--
.../java/org/apache/metron/pcap/query/PcapCli.java | 1 -
.../metron/pcap/finalizer/PcapCliFinalizer.java | 15 ---
.../apache/metron/pcap/finalizer/PcapFinalizer.java | 4 +++-
.../metron/pcap/finalizer/PcapRestFinalizer.java| 10 ++
.../metron/pcap/writer/PcapResultsWriter.java | 16
5 files changed, 41 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/f1f5dda4/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java
--
diff --git
a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java
b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java
index 1a23740..0b06b0c 100644
---
a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java
+++
b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java
@@ -87,7 +87,6 @@ public class PcapCli {
try {
config = fixedParser.parse(otherArgs);
commonConfig = config;
-PcapOptions.FINAL_OUTPUT_PATH.put(commonConfig, new Path("file:///" +
execDir));
} catch (ParseException | java.text.ParseException e) {
System.err.println(e.getMessage());
System.err.flush();
http://git-wip-us.apache.org/repos/asf/metron/blob/f1f5dda4/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapCliFinalizer.java
--
diff --git
a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapCliFinalizer.java
b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapCliFinalizer.java
index c379515..c912e58 100644
---
a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapCliFinalizer.java
+++
b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapCliFinalizer.java
@@ -18,9 +18,13 @@
package org.apache.metron.pcap.finalizer;
+import java.io.IOException;
+import java.util.List;
import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.metron.pcap.config.PcapOptions;
+import org.apache.metron.pcap.writer.PcapResultsWriter;
/**
* Write to local FS.
@@ -33,13 +37,18 @@ public class PcapCliFinalizer extends PcapFinalizer {
* as a formatted timestamp + uuid. A final sample format will look as
follows:
*
/base/output/path/pcap-data-201807181911-09855b4ae3204dee8b63760d65198da3+0001.pcap
*/
- private static final String PCAP_CLI_FILENAME_FORMAT =
"%s/pcap-data-%s+%04d.pcap";
+ private static final String PCAP_CLI_FILENAME_FORMAT =
"pcap-data-%s+%04d.pcap";
+
+ @Override
+ protected void write(PcapResultsWriter resultsWriter, Configuration
hadoopConfig,
+ List data, Path outputPath) throws IOException {
+resultsWriter.writeLocal(data, outputPath.toString());
+ }
@Override
protected Path getOutputPath(Map config, int partition) {
-Path finalOutputPath = PcapOptions.FINAL_OUTPUT_PATH.get(config,
PcapOptions.STRING_TO_PATH, Path.class);
String prefix = PcapOptions.FINAL_FILENAME_PREFIX.get(config,
String.class);
-return new Path(String.format(PCAP_CLI_FILENAME_FORMAT, finalOutputPath,
prefix, partition));
+return new Path(String.format(PCAP_CLI_FILENAME_FORMAT, prefix,
partition));
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/f1f5dda4/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapFinalizer.java
--
diff --git
a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapFinalizer.java
b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapFinalizer.java
index 2c55e15..8dcc401 100644
---
a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapFinalizer.java
+++
[44/50] [abbrv] metron git commit: METRON-1752 Prevent package.lock from changing during build (sardell via merrimanr) closes apache/metron#1177
http://git-wip-us.apache.org/repos/asf/metron/blob/09826da8/metron-interface/metron-config/pom.xml -- diff --git a/metron-interface/metron-config/pom.xml b/metron-interface/metron-config/pom.xml index 14acdc6..59c416e 100644 --- a/metron-interface/metron-config/pom.xml +++ b/metron-interface/metron-config/pom.xml @@ -1,15 +1,15 @@ - http://maven.apache.org/POM/4.0.0; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; @@ -25,8 +25,8 @@ UTF-8 UTF-8 -v6.2.0 -3.8.9 +v9.11.1 +6.2.0 @@ -53,12 +53,12 @@ generate-resources -npm install +npm ci npm - install + ci
[33/50] [abbrv] metron git commit: METRON-1725 Add ability to specify YARN queue for pcap jobs (merrimanr) closes apache/metron#1153
METRON-1725 Add ability to specify YARN queue for pcap jobs (merrimanr) closes apache/metron#1153 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/7a8c2467 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/7a8c2467 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/7a8c2467 Branch: refs/remotes/upstream/feature/METRON-1699-create-batch-profiler Commit: 7a8c246748a2c9e8c5b9230800b075dd99a7f3a4 Parents: 73dc63e Author: merrimanr Authored: Fri Aug 10 16:46:31 2018 -0500 Committer: rmerriman Committed: Fri Aug 10 16:46:31 2018 -0500 -- .../CURRENT/configuration/metron-rest-env.xml | 9 ++ .../package/scripts/params/params_linux.py | 1 + .../METRON/CURRENT/package/templates/metron.j2 | 1 + .../METRON/CURRENT/themes/metron_theme.json | 10 ++ metron-interface/metron-rest/README.md | 2 ++ .../src/main/config/rest_application.yml| 1 + .../apache/metron/rest/MetronRestConstants.java | 1 + .../metron/rest/config/PcapJobSupplier.java | 2 +- .../rest/service/impl/PcapServiceImpl.java | 12 ++- .../apache/metron/rest/mock/MockPcapJob.java| 8 + .../rest/service/impl/PcapServiceImplTest.java | 7 - metron-platform/metron-pcap-backend/README.md | 2 ++ .../org/apache/metron/pcap/query/CliParser.java | 4 +++ .../org/apache/metron/pcap/query/PcapCli.java | 3 ++ .../apache/metron/pcap/query/PcapCliTest.java | 33 ++-- .../apache/metron/pcap/config/PcapConfig.java | 10 ++ 16 files changed, 101 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/7a8c2467/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml -- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml index 20f9767..895c091 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml @@ -192,5 +192,14 @@ The number of pcaps written to a page/file as a result of a pcap query. 10 + +pcap_yarn_queue +Pcap YARN Queue +The YARN queue pcap jobs will be submitted to. + + +true + + http://git-wip-us.apache.org/repos/asf/metron/blob/7a8c2467/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py -- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py index 73d3469..4f8a9a7 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py @@ -390,6 +390,7 @@ pcap_base_path = config['configurations']['metron-rest-env']['pcap_base_path'] pcap_base_interim_result_path = config['configurations']['metron-rest-env']['pcap_base_interim_result_path'] pcap_final_output_path = config['configurations']['metron-rest-env']['pcap_final_output_path'] pcap_page_size = config['configurations']['metron-rest-env']['pcap_page_size'] +pcap_yarn_queue = config['configurations']['metron-rest-env']['pcap_yarn_queue'] pcap_configured_flag_file = status_params.pcap_configured_flag_file # MapReduce http://git-wip-us.apache.org/repos/asf/metron/blob/7a8c2467/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 -- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 index 278d6f8..55422d0 100644 ---
[24/50] [abbrv] metron git commit: METRON-1712 PCAP UI - Input validation (tiborm via merrimanr) closes apache/metron#1142
METRON-1712 PCAP UI - Input validation (tiborm via merrimanr) closes
apache/metron#1142
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/52de126f
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/52de126f
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/52de126f
Branch: refs/remotes/upstream/feature/METRON-1699-create-batch-profiler
Commit: 52de126fc746181d6aac8ce002d6b814fd1e6cb5
Parents: afb9607
Author: tiborm
Authored: Wed Aug 8 16:36:35 2018 -0500
Committer: rmerriman
Committed: Wed Aug 8 16:36:35 2018 -0500
--
.../src/app/pcap/model/pcap.mock.ts | 4 +-
.../src/app/pcap/model/pcap.request.ts | 4 +-
.../pcap-filters/pcap-filters.component.html| 12 +-
.../pcap-filters/pcap-filters.component.spec.ts | 228 ++-
.../pcap/pcap-filters/pcap-filters.component.ts | 5 +
5 files changed, 232 insertions(+), 21 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/52de126f/metron-interface/metron-alerts/src/app/pcap/model/pcap.mock.ts
--
diff --git a/metron-interface/metron-alerts/src/app/pcap/model/pcap.mock.ts
b/metron-interface/metron-alerts/src/app/pcap/model/pcap.mock.ts
index c867fe9..bf02da8 100644
--- a/metron-interface/metron-alerts/src/app/pcap/model/pcap.mock.ts
+++ b/metron-interface/metron-alerts/src/app/pcap/model/pcap.mock.ts
@@ -22,9 +22,9 @@ export const fakePcapRequest = {
startTimeMs: 0,
endTimeMs: 0,
ipSrcAddr: '0.0.0.0',
- ipSrcPort: '80',
+ ipSrcPort: 80,
ipDstAddr: '0.0.0.0',
- ipDstPort: '80',
+ ipDstPort: 80,
protocol: '*',
packetFilter: '*',
includeReverse: false
http://git-wip-us.apache.org/repos/asf/metron/blob/52de126f/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
--
diff --git a/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
b/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
index 91c2287..d00a6ac 100644
--- a/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
+++ b/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
@@ -20,9 +20,9 @@ export class PcapRequest {
startTimeMs: number = 0;
endTimeMs: number = 15;
ipSrcAddr: string = '';
- ipSrcPort: string = '';
+ ipSrcPort: number = 0;
ipDstAddr: string = '';
- ipDstPort: string = '';
+ ipDstPort: number = 0;
protocol: string = '';
packetFilter: string = '';
includeReverse: boolean = false;
http://git-wip-us.apache.org/repos/asf/metron/blob/52de126f/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.html
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.html
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.html
index f4133df..039307a 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.html
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.html
@@ -24,21 +24,21 @@
IP Source Address
-
+
-
+
IP Source Port
-
+
IP Dest Address
-
+
IP Dest Port
-
+
@@ -57,6 +57,6 @@
-
+
http://git-wip-us.apache.org/repos/asf/metron/blob/52de126f/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
index c0f9c3b..4336b22 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
@@ -16,20 +16,19 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-import { async, ComponentFixture, TestBed, fakeAsync, tick } from
'@angular/core/testing';
+import { async, ComponentFixture, TestBed } from '@angular/core/testing';
import { By } from '@angular/platform-browser';
import { PcapFiltersComponent } from './pcap-filters.component';
import { FormsModule } from '../../../../node_modules/@angular/forms';
-import { Component, Input, Output, EventEmitter } from '@angular/core';
+import { Component, Input, Output, EventEmitter, DebugElement } from
'@angular/core';
import { PcapRequest } from '../model/pcap.request';
-import { emit
[42/50] [abbrv] metron git commit: METRON-1554 Pcap Query Panel (merrimanr) closes apache/metron#1169
METRON-1554 Pcap Query Panel (merrimanr) closes apache/metron#1169 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/9fdccba3 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/9fdccba3 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/9fdccba3 Branch: refs/remotes/upstream/feature/METRON-1699-create-batch-profiler Commit: 9fdccba371c1f1e0dcb79e00a7207a934b79b64c Parents: 308c2b2 14e80b3 Author: merrimanr Authored: Fri Aug 17 10:26:51 2018 -0500 Committer: merrimanr Committed: Fri Aug 17 10:28:09 2018 -0500 -- dependencies_with_url.csv |4 + metron-deployment/amazon-ec2/conf/defaults.yml |1 - metron-deployment/amazon-ec2/playbook.yml |2 +- .../ansible/playbooks/metron_install.yml|7 - .../roles/ambari_config/vars/single_node_vm.yml |1 + .../roles/metron_pcapservice/defaults/main.yml | 28 - .../roles/metron_pcapservice/meta/main.yml | 19 - .../metron_pcapservice/tasks/config-hbase.yml | 26 - .../roles/metron_pcapservice/tasks/main.yml | 25 - .../metron_pcapservice/tasks/pcapservice.yml| 30 - .../templates/metron-pcapservice-logrotate.yml | 27 - .../metron_pcapservice/templates/pcapservice| 84 - .../ubuntu14/ansible/inventory/hosts|4 - .../CURRENT/configuration/metron-rest-env.xml | 48 + .../package/scripts/params/params_linux.py | 16 + .../package/scripts/params/status_params.py |7 + .../CURRENT/package/scripts/rest_commands.py| 62 + .../CURRENT/package/scripts/rest_master.py |9 + .../METRON/CURRENT/package/templates/metron.j2 |6 + .../METRON/CURRENT/themes/metron_theme.json | 30 + .../docker/rpm-docker/SPECS/metron.spec |1 + .../metron-alerts/package-lock.json | 3719 +- .../metron-alerts/src/app/app-routing.module.ts |3 +- .../metron-alerts/src/app/app.component.html| 10 +- .../metron-alerts/src/app/app.component.scss| 18 + .../metron-alerts/src/app/app.component.ts |3 + .../metron-alerts/src/app/app.module.ts |6 +- .../src/app/pcap/model/pcap-pagination.ts | 21 + .../src/app/pcap/model/pcap-status-response.ts | 24 + .../src/app/pcap/model/pcap.mock.ts | 39 + .../src/app/pcap/model/pcap.request.ts | 29 + .../src/app/pcap/model/pdml.mock.ts | 93 + .../metron-alerts/src/app/pcap/model/pdml.ts| 50 + .../pcap-filters/pcap-filters.component.html| 62 + .../pcap-filters/pcap-filters.component.scss| 69 + .../pcap-filters/pcap-filters.component.spec.ts | 463 +++ .../pcap/pcap-filters/pcap-filters.component.ts | 77 + .../app/pcap/pcap-list/pcap-list.component.html | 38 + .../app/pcap/pcap-list/pcap-list.component.scss | 21 + .../pcap/pcap-list/pcap-list.component.spec.ts | 98 + .../app/pcap/pcap-list/pcap-list.component.ts | 43 + .../pcap-packet-line.component.html | 19 + .../pcap-packet-line.component.scss | 17 + .../pcap-packet-line.component.spec.ts | 180 + .../pcap-packet-line.component.ts | 56 + .../pcap/pcap-packet/pcap-packet.component.html | 22 + .../pcap/pcap-packet/pcap-packet.component.scss | 36 + .../pcap-packet/pcap-packet.component.spec.ts | 79 + .../pcap/pcap-packet/pcap-packet.component.ts | 34 + .../pcap-pagination.component.html | 19 + .../pcap-pagination.component.scss | 40 + .../pcap-pagination.component.spec.ts | 78 + .../pcap-pagination.component.ts| 41 + .../pcap/pcap-panel/pcap-panel.component.html | 31 + .../pcap/pcap-panel/pcap-panel.component.scss | 67 + .../pcap-panel/pcap-panel.component.spec.ts | 557 +++ .../app/pcap/pcap-panel/pcap-panel.component.ts | 164 + .../metron-alerts/src/app/pcap/pcap.module.ts | 54 + .../metron-alerts/src/app/pcap/pcap.routing.ts | 27 + .../src/app/pcap/service/pcap.service.spec.ts | 197 + .../src/app/pcap/service/pcap.service.ts| 88 + .../shared/directives/alert-search.directive.ts |2 +- .../src/environments/environment.e2e.ts |3 +- .../src/environments/environment.prod.ts|3 +- .../src/environments/environment.ts |3 +- metron-interface/metron-alerts/src/index.html |2 +- metron-interface/metron-rest-client/pom.xml | 10 + .../apache/metron/rest/model/PcapResponse.java | 38 + .../apache/metron/rest/model/pcap/Field.java| 154 + .../rest/model/pcap/FixedPcapOptions.java | 42 + .../rest/model/pcap/FixedPcapRequest.java | 116 + .../apache/metron/rest/model/pcap/Packet.java | 53 + .../metron/rest/model/pcap/PcapRequest.java | 83 + .../metron/rest/model/pcap/PcapStatus.java | 87 +
[10/50] [abbrv] metron git commit: METRON-1671 Create PCAP UI (tiborm via merrimanr) closes apache/metron#1103
http://git-wip-us.apache.org/repos/asf/metron/blob/d5eb56a9/metron-interface/metron-alerts/src/app/app-routing.module.ts
--
diff --git a/metron-interface/metron-alerts/src/app/app-routing.module.ts
b/metron-interface/metron-alerts/src/app/app-routing.module.ts
index 8357d66..56ad41c 100644
--- a/metron-interface/metron-alerts/src/app/app-routing.module.ts
+++ b/metron-interface/metron-alerts/src/app/app-routing.module.ts
@@ -26,7 +26,8 @@ const routes: Routes = [
{ path: 'alerts-list', loadChildren:
'app/alerts/alerts-list/alerts-list.module#AlertsListModule', canActivate:
[AuthGuard]},
{ path: 'save-search', loadChildren:
'app/alerts/save-search/save-search.module#SaveSearchModule', canActivate:
[AuthGuard]},
{ path: 'saved-searches', loadChildren:
'app/alerts/saved-searches/saved-searches.module.ts#SavedSearchesModule',
- canActivate: [AuthGuard]}
+ canActivate: [AuthGuard]},
+ { path: 'pcap', loadChildren: 'app/pcap/pcap.module#PcapModule',
canActivate: [AuthGuard] }
];
@NgModule({
http://git-wip-us.apache.org/repos/asf/metron/blob/d5eb56a9/metron-interface/metron-alerts/src/app/app.component.html
--
diff --git a/metron-interface/metron-alerts/src/app/app.component.html
b/metron-interface/metron-alerts/src/app/app.component.html
index 8867a9b..1b992ca 100644
--- a/metron-interface/metron-alerts/src/app/app.component.html
+++ b/metron-interface/metron-alerts/src/app/app.component.html
@@ -16,6 +16,14 @@
+
+
+Alerts
+
+
+PCAP
+
+
Logged in as {{authService.currentUser}} -
Logout
http://git-wip-us.apache.org/repos/asf/metron/blob/d5eb56a9/metron-interface/metron-alerts/src/app/app.component.scss
--
diff --git a/metron-interface/metron-alerts/src/app/app.component.scss
b/metron-interface/metron-alerts/src/app/app.component.scss
index 8358e7f..8505ee8 100644
--- a/metron-interface/metron-alerts/src/app/app.component.scss
+++ b/metron-interface/metron-alerts/src/app/app.component.scss
@@ -27,6 +27,24 @@
max-height: 50px;
}
+.nav-link
+{
+ padding-bottom: 0;
+ padding-top: 0;
+ color: inherit;
+}
+
+.nav-item.active
+{
+ border-bottom: 3px solid #32abe2;
+ margin-bottom: 5px;
+}
+
+.nav-link.active
+{
+ color: #ff;
+}
+
.logout {
padding-left: 10px;
}
http://git-wip-us.apache.org/repos/asf/metron/blob/d5eb56a9/metron-interface/metron-alerts/src/app/app.module.ts
--
diff --git a/metron-interface/metron-alerts/src/app/app.module.ts
b/metron-interface/metron-alerts/src/app/app.module.ts
index 227eaab..82b4970 100644
--- a/metron-interface/metron-alerts/src/app/app.module.ts
+++ b/metron-interface/metron-alerts/src/app/app.module.ts
@@ -46,8 +46,7 @@ import {MetaAlertService} from './service/meta-alert.service';
import {MetaAlertsModule} from './alerts/meta-alerts/meta-alerts.module';
import {SearchService} from './service/search.service';
import { GlobalConfigService } from './service/global-config.service';
-
-
+import {PcapModule} from './pcap/pcap.module';
export function initConfig(config: ColumnNamesService) {
return () => config.list();
@@ -70,7 +69,8 @@ export function initConfig(config: ColumnNamesService) {
ConfigureRowsModule,
SaveSearchModule,
SavedSearchesModule,
-SwitchModule
+SwitchModule,
+PcapModule
],
providers: [{ provide: APP_INITIALIZER, useFactory: initConfig, deps:
[ColumnNamesService], multi: true },
{ provide: DataSource, useClass: ElasticSearchLocalstorageImpl },
http://git-wip-us.apache.org/repos/asf/metron/blob/d5eb56a9/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
--
diff --git a/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
b/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
new file mode 100644
index 000..3a27714
--- /dev/null
+++ b/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing software
+ * distributed under the License is distributed on an "AS
[35/50] [abbrv] metron git commit: METRON-1733 PCAP UI - PCAP queries don t work on Safari (sardell via merrimanr) closes apache/metron#1158
METRON-1733 PCAP UI - PCAP queries dont work on Safari (sardell via
merrimanr) closes apache/metron#1158
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/bce9b903
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/bce9b903
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/bce9b903
Branch: refs/remotes/upstream/feature/METRON-1699-create-batch-profiler
Commit: bce9b903fbd3436ea0c41eb8d75da279e16ce922
Parents: ba0a609
Author: sardell
Authored: Mon Aug 13 16:04:45 2018 -0500
Committer: rmerriman
Committed: Mon Aug 13 16:04:45 2018 -0500
--
.../src/app/pcap/pcap-filters/pcap-filters.component.ts | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/bce9b903/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.ts
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.ts
index df91ae3..b23a2e2 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.ts
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.ts
@@ -59,8 +59,8 @@ export class PcapFiltersComponent implements OnInit,
OnChanges {
}
onSubmit() {
-this.model.startTimeMs = new Date(this.startTimeStr).getTime();
-this.model.endTimeMs = new Date(this.endTimeStr).getTime();
+this.model.startTimeMs = moment(this.startTimeStr,
DEFAULT_TIMESTAMP_FORMAT).valueOf();
+this.model.endTimeMs = moment(this.endTimeStr,
DEFAULT_TIMESTAMP_FORMAT).valueOf();
if (this.ipSrcPort !== '') {
this.model.ipSrcPort = +this.ipSrcPort;
} else {
[08/50] [abbrv] metron git commit: METRON-1671 Create PCAP UI (tiborm via merrimanr) closes apache/metron#1103
http://git-wip-us.apache.org/repos/asf/metron/blob/d5eb56a9/metron-interface/metron-alerts/src/app/pcap/service/pcap.service.spec.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/service/pcap.service.spec.ts
b/metron-interface/metron-alerts/src/app/pcap/service/pcap.service.spec.ts
new file mode 100644
index 000..244a3ea
--- /dev/null
+++ b/metron-interface/metron-alerts/src/app/pcap/service/pcap.service.spec.ts
@@ -0,0 +1,1752 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { TestBed, async, inject } from '@angular/core/testing';
+import {
+ BaseRequestOptions,
+ HttpModule,
+ Http,
+ Response,
+ ResponseOptions
+} from '@angular/http';
+import { MockBackend } from '@angular/http/testing';
+
+import { PcapService } from './pcap.service';
+import { PcapRequest } from '../model/pcap.request';
+
+describe('PcapService', () => {
+ beforeEach(() => {
+
+TestBed.configureTestingModule({
+ imports: [HttpModule],
+ providers: [
+PcapService,
+{
+ provide: Http,
+ useFactory: (mockBackend, options) => {
+return new Http(mockBackend, options);
+ },
+ deps: [MockBackend, BaseRequestOptions]
+},
+MockBackend,
+BaseRequestOptions
+ ]
+});
+ });
+ describe('getPackets()', () => {
+it('should return an Observable',
+ inject([PcapService, MockBackend], (pcapService, mockBackend) => {
+
+let request: PcapRequest = {
+ startTimeMs: 0,
+ endTimeMs: 0,
+ ipSrcAddr: '0.0.0.0',
+ ipSrcPort: '80',
+ ipDstAddr: '0.0.0.0',
+ ipDstPort: '80',
+ protocol: '*',
+ packetFilter: '*',
+ includeReverse: false,
+};
+
+mockBackend.connections.subscribe((connection) => {
+ connection.mockRespond(new Response(new ResponseOptions({body:
pdml_json()})));
+});
+let packets;
+pcapService.getPackets(request).subscribe(r => packets = r)
+expect(packets).toBeTruthy()
+expect(packets.pdml).toBeTruthy()
+expect(packets.pdml.packet.length).toBe(1)
+expect(packets.pdml.packet[0].proto.length).toBeGreaterThan(3)
+
+console.log(packets)
+ }))
+
+
+it('should ...', inject([PcapService], (service: PcapService) => {
+ expect(service).toBeTruthy();
+}));
+
+ })
+
+});
+
+
+
+
+function pdml_json() {
+ return `{
+ "pdml": {
+"$": {
+ "version": "0",
+ "creator": "wireshark/2.4.2",
+ "time": "Tue Mar 27 21:55:25 2018",
+ "capture_file":
"./metron-platform/metron-api/src/test/resources/test-tcp-packet.pcap"
+},
+"packet": [
+ {
+"proto": [
+ {
+"$": {
+ "name": "geninfo",
+ "pos": "0",
+ "showname": "General information",
+ "size": "104"
+},
+"field": [
+ {
+"$": {
+ "name": "num",
+ "pos": "0",
+ "show": "1",
+ "showname": "Number",
+ "value": "1",
+ "size": "104"
+}
+ },
+ {
+"$": {
+ "name": "len",
+ "pos": "0",
+ "show": "104",
+ "showname": "Frame Length",
+ "value": "68",
+ "size": "104"
+}
+ },
+ {
+"$": {
+ "name": "caplen",
+ "pos": "0",
+ "show": "104",
+ "showname": "Captured Length",
+ "value": "68",
+ "size": "104"
+}
+ },
+ {
+"$": {
+ "name": "timestamp",
+ "pos": "0",
+ "show": "Mar 26, 2014 19:59:40.024362000 GMT",
+ "showname": "Captured Time",
+ "value": "1395863980.024362000",
+ "size": "104"
+}
+
[03/50] [abbrv] metron git commit: METRON-1690: Add more context to PcapJob JobStatus (mmiklavc via mmiklavc) closes apache/metron#1128
METRON-1690: Add more context to PcapJob JobStatus (mmiklavc via mmiklavc) closes apache/metron#1128 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/9d4842f3 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/9d4842f3 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/9d4842f3 Branch: refs/remotes/upstream/feature/METRON-1699-create-batch-profiler Commit: 9d4842f3d6f355d449d51f3c6ac02f2c904c295d Parents: d0e8757 Author: mmiklavc Authored: Wed Jul 25 08:21:32 2018 -0600 Committer: Michael Miklavcic Committed: Wed Jul 25 08:21:32 2018 -0600 -- .../metron-job_state_statechart_diagram.svg | 18 +- .../metron-job_state_statechart_diagram.xml | 18 +- .../java/org/apache/metron/job/JobStatus.java | 42 - .../org/apache/metron/job/JobStatusTest.java| 55 ++ .../job/manager/InMemoryJobManagerTest.java | 16 ++ .../org/apache/metron/pcap/PcapJobTest.java | 80 +--- .../PcapTopologyIntegrationTest.java| 30 +-- .../java/org/apache/metron/pcap/mr/PcapJob.java | 186 +-- site-book/bin/generate-md.sh| 2 + 9 files changed, 294 insertions(+), 153 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/9d4842f3/metron-platform/metron-job/metron-job_state_statechart_diagram.svg -- diff --git a/metron-platform/metron-job/metron-job_state_statechart_diagram.svg b/metron-platform/metron-job/metron-job_state_statechart_diagram.svg index a99c5ad..c801eae 100644 --- a/metron-platform/metron-job/metron-job_state_statechart_diagram.svg +++ b/metron-platform/metron-job/metron-job_state_statechart_diagram.svg @@ -1,14 +1,14 @@ http://www.w3.org/Graphics/SVG/1.1/DTD/svg11.dtd;> -http://www.w3.org/2000/svg; xmlns:xlink="http://www.w3.org/1999/xlink; width="661px" height="291px" version="1.1" style="background-color: rgb(255, 255, 255);">http://www.w3.org/TR/SVG11/feature#Extensibility;>http://www.w3.org/1999/xhtml; style="display: inline-block; font-size: 12px; font-family: Verdana; color: rgb(0, 0, 0); line-height: 1.2; vertical-align: top; width: 92px; white-space: nowrap; word-wrap: normal; text-align: center;">http://www.w3.org/1999/xhtml; style="display:inline-block;text-align:inherit;text-dec oration:inherit;">NOT_RUNNINGNOT_RUNNINGhttp://www.w3.org/TR/SVG11/feature#Extensibility;>http://www.w3.org/1999/xhtml; style="display: inline-block; font-size: 12px; font-family: Verdana; color: rgb(0, 0, 0); line-height: 1.2; vertical-align: top; width: 60px; white-space: nowrap; word-wrap: normal; text-align: center;">http://ww w.w3.org/1999/xhtml" style="display:inline-block;text-align:inherit;text-decoration:inherit;">RUNNINGRUNNINGhttp://www.w3.org/TR/SVG11/feature#Extensibility;>http://www.w3.org/1999/xhtml; style="display: inline-block; font-size: 12px; font-family: Verdana; color: rgb(0, 0, 0); line-height: 1.2; vertical-align: top; width: 74px; white-space: nowrap; word-wrap: normal; text-align: center;">http://www.w3.org/1999/xhtml; style="display:inline-block;text-align:inherit;text-decoration:inherit;">FINALIZINGFINALIZINGhttp://www.w3.org/TR/SVG11/feature#Extensibility;>http://www.w3.org/1999/xhtml; style="display: inline-block; font-size: 12px; font-family: Verdana; color: rgb(0, 0, 0); line-height: 1.2; vertical-align: top; width: 44px; white-space: nowrap; word-wrap: normal; text-align: center;">http://www.w3.org/1999/xhtml; style="display:inline-block;text-align:inherit;text-decoration:inherit;">KILLEDKILLEDhttp://www.w3.org/TR/SVG11/feature#Extensibility;>http://www.w3.org/1999/xhtml; style="display: inline-block; font-size: 12px; font-family: Verdana; color: rgb(0, 0, 0); line-height: 1.2; vertical-align: top; width: 44px; white-space: nowrap; word-wrap: normal; text-align: center;">http://www.w3.org/1999/xhtml; style="display:inline-block;text-align:inherit;text-decoration:inherit;">FAILEDFAILEDhttp://www.w3.org/TR/SVG11/feature#Extensibility;>http://www.w3.org/1999/xhtml; style="display: inline-block; font-size: 12px; font-family: Verdana; color: rgb(0, 0, 0); line-height: 1.2; vertical-align: top; width: 76px; white-space: nowrap; word-wrap: normal; text-align: center;">http://www.w3.org/1999/xhtml; style="display:inline-block;text-align:inherit;text-decoration:inherit;">SUCCEEDEDSUCCEEDED +http://www.w3.org/2000/svg; xmlns:xlink="http://www.w3.org/1999/xlink; width="656px" height="291px" version="1.1">http://www.w3.org/TR/SVG11/feature#Extensibility;>http://www.w3.org/1999/xhtml; style="display: inline-block; font-size: 12px; font-family: Verdana; color: rgb(0, 0, 0); line-height: 1.2; vertical-align: top; width: 92px; white-space: nowrap; word-wrap: normal; text-align:
[28/50] [abbrv] metron git commit: METRON-1728: Handle null values in config in Pcap backend more gracefully (mmiklavc via mmiklavc) closes apache/metron#1151
METRON-1728: Handle null values in config in Pcap backend more gracefully
(mmiklavc via mmiklavc) closes apache/metron#1151
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/9064cca0
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/9064cca0
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/9064cca0
Branch: refs/remotes/upstream/feature/METRON-1699-create-batch-profiler
Commit: 9064cca0317881176471c51abd16e99bf2ad7b10
Parents: 14dcb2d
Author: mmiklavc
Authored: Thu Aug 9 09:25:29 2018 -0600
Committer: Michael Miklavcic
Committed: Thu Aug 9 09:25:29 2018 -0600
--
.../common/configuration/ConfigOption.java | 32 --
.../common/configuration/ConfigOptionTest.java | 112 +++
.../org/apache/metron/pcap/query/CliParser.java | 25 +++--
.../org/apache/metron/pcap/PcapJobTest.java | 23
.../apache/metron/pcap/query/PcapCliTest.java | 10 +-
.../metron/pcap/config/PcapGlobalDefaults.java | 28 +
.../metron/pcap/finalizer/PcapFinalizer.java| 8 +-
.../pcap/finalizer/PcapRestFinalizer.java | 11 +-
.../java/org/apache/metron/pcap/mr/PcapJob.java | 25 +++--
9 files changed, 237 insertions(+), 37 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/9064cca0/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigOption.java
--
diff --git
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigOption.java
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigOption.java
index 8e4211b..6308f0a 100644
---
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigOption.java
+++
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigOption.java
@@ -18,36 +18,54 @@
package org.apache.metron.common.configuration;
-import org.apache.metron.stellar.common.utils.ConversionUtils;
-
import java.util.Map;
import java.util.function.BiFunction;
+import org.apache.metron.stellar.common.utils.ConversionUtils;
public interface ConfigOption {
+
String getKey();
+
default BiFunction transform() {
-return (s,o) -> o;
+return (s, o) -> o;
}
default void put(Map map, Object value) {
map.put(getKey(), value);
}
+ default T getOrDefault(Map map, Class clazz, T
defaultValue) {
+T val;
+return ((val = get(map, clazz)) == null ? defaultValue : val);
+ }
+
default T get(Map map, Class clazz) {
Object obj = map.get(getKey());
-if(clazz.isInstance(obj)) {
+if (clazz.isInstance(obj)) {
return clazz.cast(obj);
-}
-else {
+} else {
return ConversionUtils.convert(obj, clazz);
}
}
- default T get(Map map, BiFunction
transform, Class clazz) {
+ default T getOrDefault(Map map, BiFunction transform,
+ Class clazz, T defaultValue) {
+T val;
+return ((val = get(map, transform, clazz)) == null ? defaultValue : val);
+ }
+
+ default T get(Map map, BiFunction
transform,
+ Class clazz) {
return clazz.cast(transform.apply(getKey(), map.get(getKey(;
}
+ default T getTransformedOrDefault(Map map, Class
clazz, T defaultValue) {
+T val;
+return ((val = getTransformed(map, clazz)) == null ? defaultValue : val);
+ }
+
default T getTransformed(Map map, Class clazz) {
return clazz.cast(transform().apply(getKey(), map.get(getKey(;
}
+
}
http://git-wip-us.apache.org/repos/asf/metron/blob/9064cca0/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigOptionTest.java
--
diff --git
a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigOptionTest.java
b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigOptionTest.java
new file mode 100644
index 000..95db080
--- /dev/null
+++
b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigOptionTest.java
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under
[metron] Git Push Summary
Repository: metron Updated Branches: refs/remotes/upstream/feature/METRON-1699-create-batch-profiler [created] 113ac0d3a
[17/50] [abbrv] metron git commit: METRON-1721 New default input path is wrong in pcap CLI (merrimanr) closes apache/metron#1137
METRON-1721 New default input path is wrong in pcap CLI (merrimanr) closes
apache/metron#1137
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/756ceedc
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/756ceedc
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/756ceedc
Branch: refs/remotes/apache/feature/METRON-1699-create-batch-profiler
Commit: 756ceedc3d3567f3508478eaa5ed29f12c22688a
Parents: 8972665
Author: merrimanr
Authored: Mon Aug 6 13:36:40 2018 -0500
Committer: rmerriman
Committed: Mon Aug 6 13:36:40 2018 -0500
--
.../src/main/java/org/apache/metron/pcap/query/CliParser.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/756ceedc/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
--
diff --git
a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
index e6534c5..4ad6ffa 100644
---
a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
+++
b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
@@ -25,7 +25,7 @@ import org.apache.metron.pcap.config.PcapConfig;
* Provides commmon required fields for the PCAP filter jobs
*/
public class CliParser {
- public static final String BASE_PATH_DEFAULT = "/apps/metron/pcap";
+ public static final String BASE_PATH_DEFAULT = "/apps/metron/pcap/input";
public static final String BASE_INTERIM_OUTPUT_PATH_DEFAULT = "/tmp";
public static final int NUM_REDUCERS_DEFAULT = 10;
public static final int NUM_RECORDS_PER_FILE_DEFAULT = 1;
[25/50] [abbrv] metron git commit: METRON-1723 PCAP UI - Unable to select/copy from packets details in PCAP query panel (sardell via merrimanr) closes apache/metron#1139
METRON-1723 PCAP UI - Unable to select/copy from packets details in PCAP query
panel (sardell via merrimanr) closes apache/metron#1139
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/5b3e2c33
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/5b3e2c33
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/5b3e2c33
Branch: refs/remotes/apache/feature/METRON-1699-create-batch-profiler
Commit: 5b3e2c337b3772bcd7d18cedac75877bbe6f7735
Parents: 52de126
Author: sardell
Authored: Thu Aug 9 07:44:15 2018 -0500
Committer: rmerriman
Committed: Thu Aug 9 07:44:15 2018 -0500
--
.../src/app/pcap/pcap-packet/pcap-packet.component.html | 2 +-
.../src/app/pcap/pcap-packet/pcap-packet.component.scss | 4 ++--
2 files changed, 3 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/5b3e2c33/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.html
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.html
b/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.html
index f7d9dbe..dd9afa1 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.html
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.html
@@ -13,7 +13,7 @@
-->
{{ proto.showname
}}
-
+
{{ field.name
}}
{{ field.showname }}
http://git-wip-us.apache.org/repos/asf/metron/blob/5b3e2c33/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.scss
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.scss
b/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.scss
index e9d4662..02ce7b6 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.scss
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.scss
@@ -27,10 +27,10 @@
}
.proto-fields {
- margin-left: 1em;
+ cursor: text;
+ padding-left: 1em;
}
-
.field-name {
display: none
}
[19/50] [abbrv] metron git commit: METRON-1683 PCAP UI - Fix the download progress bar (sardell via merrimanr) closes apache/metron#1122
http://git-wip-us.apache.org/repos/asf/metron/blob/3e778592/metron-interface/metron-alerts/src/app/pcap/service/pcap.service.spec.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/service/pcap.service.spec.ts
b/metron-interface/metron-alerts/src/app/pcap/service/pcap.service.spec.ts
index fdaa569..c8cb937 100644
--- a/metron-interface/metron-alerts/src/app/pcap/service/pcap.service.spec.ts
+++ b/metron-interface/metron-alerts/src/app/pcap/service/pcap.service.spec.ts
@@ -15,1738 +15,139 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-import { TestBed, async, inject } from '@angular/core/testing';
+import {
+ TestBed,
+ inject,
+ fakeAsync,
+ tick,
+ discardPeriodicTasks
+} from '@angular/core/testing';
import {
BaseRequestOptions,
HttpModule,
- Http,
Response,
- ResponseOptions
+ ResponseOptions,
+ XHRBackend
} from '@angular/http';
import { MockBackend } from '@angular/http/testing';
+import { Observable } from 'rxjs/Rx';
import { PcapService } from './pcap.service';
+import { PcapStatusResponse } from '../model/pcap-status-response';
import { PcapRequest } from '../model/pcap.request';
+import { fakePdml, fakePacket } from '../model/pdml.mock';
+import { fakePcapStatusResponse, fakePcapRequest } from '../model/pcap.mock';
+
+const jobId = 'job_1234567890123_4567';
+let pdmlJsonMock = fakePdml;
+pdmlJsonMock['pdml']['packet'].push(fakePacket);
describe('PcapService', () => {
beforeEach(() => {
-
TestBed.configureTestingModule({
imports: [HttpModule],
providers: [
PcapService,
-{
- provide: Http,
- useFactory: (mockBackend, options) => {
-return new Http(mockBackend, options);
- },
- deps: [MockBackend, BaseRequestOptions]
-},
+{ provide: XHRBackend, useClass: MockBackend },
MockBackend,
BaseRequestOptions
]
});
});
describe('getPackets()', () => {
-it('should return an Observable',
- inject([PcapService, MockBackend], (pcapService, mockBackend) => {
-
-let request: PcapRequest = {
- startTimeMs: 0,
- endTimeMs: 0,
- ipSrcAddr: '0.0.0.0',
- ipSrcPort: '80',
- ipDstAddr: '0.0.0.0',
- ipDstPort: '80',
- protocol: '*',
- packetFilter: '*',
- includeReverse: false,
-};
-
-mockBackend.connections.subscribe((connection) => {
- connection.mockRespond(new Response(new ResponseOptions({body:
pdml_json()})));
-});
+it('should return an Observable', inject(
+ [PcapService, XHRBackend],
+ (pcapService, mockBackend) => {
let packets;
-pcapService.getPackets(request).subscribe(r => packets = r)
-expect(packets).toBeTruthy()
-expect(packets.pdml).toBeTruthy()
-expect(packets.pdml.packet.length).toBe(1)
-expect(packets.pdml.packet[0].proto.length).toBeGreaterThan(3)
-
-console.log(packets)
- }))
-
-
-it('should ...', inject([PcapService], (service: PcapService) => {
- expect(service).toBeTruthy();
-}));
-
- })
-
-});
+mockBackend.connections.subscribe(connection => {
+ expect(connection.request.url).toMatch(
+/\/api\/v1\/pcap\/job_1234567890123_4567\/pdml\?page=1/
+ );
+ connection.mockRespond(
+new Response(new ResponseOptions({ body: pdmlJsonMock }))
+ );
+});
+pcapService.getPackets(jobId, 1).subscribe(r => (packets = r));
+expect(packets).toBeTruthy();
+expect(packets.pdml).toBeTruthy();
+expect(packets.pdml.packet.length).toBe(1);
+expect(packets.pdml.packet[0].protos.length).toBe(3);
+ }
+));
+ });
+ describe('pollStatus()', () => {
+it('should call getStatus() in intervals', fakeAsync(inject(
+ [PcapService, XHRBackend], (pcapService, mockBackend) => {
+ const responseMock: PcapStatusResponse = fakePcapStatusResponse;
+ const spy = spyOn(pcapService, 'getStatus').and.returnValue(
+Observable.of(responseMock)
+ );
+ let response;
+
+ pcapService.pollStatus(jobId).subscribe(r => (response = r));
+ tick(4000);
+ expect(spy.calls.count()).toBe(1);
+ tick(4000);
+ expect(spy.calls.count()).toBe(2);
+ discardPeriodicTasks();
+})
+ )
+);
+ });
+ describe('submitRequest()', () => {
+it('should return an Observable', inject(
+ [PcapService, XHRBackend],
+ (pcapService, mockBackend) => {
+const request: PcapRequest = fakePcapRequest;
+const responseMock: PcapStatusResponse = fakePcapStatusResponse;
+let response;
+
+mockBackend.connections.subscribe(connection => {
+
[48/50] [abbrv] metron git commit: METRON-1757 Storm Profiler Serialization Exception (nickwallen) closes apache/metron#1178
METRON-1757 Storm Profiler Serialization Exception (nickwallen) closes
apache/metron#1178
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/661e23e2
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/661e23e2
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/661e23e2
Branch: refs/remotes/apache/feature/METRON-1699-create-batch-profiler
Commit: 661e23e274d2bf88370d8feb5e5dbf1b82f15124
Parents: 29cbdcc
Author: nickwallen
Authored: Wed Aug 29 12:51:05 2018 -0400
Committer: nickallen
Committed: Wed Aug 29 12:51:05 2018 -0400
--
metron-analytics/metron-profiler-common/pom.xml | 26 +++
.../metron/profiler/ProfileMeasurement.java | 35
.../apache/metron/profiler/ProfilePeriod.java | 3 ++
.../metron/profiler/ProfileMeasurementTest.java | 28 +---
.../metron/profiler/ProfilePeriodTest.java | 23 ++---
.../profiler/hbase/SaltyRowKeyBuilderTest.java | 11 --
metron-analytics/metron-profiler/README.md | 18 ++
.../integration/ProfilerIntegrationTest.java| 30 +
.../configuration/profiler/ProfileResult.java | 4 +++
.../profiler/ProfileResultExpressions.java | 4 +++
.../profiler/ProfilerConfigTest.java| 35 +++-
11 files changed, 154 insertions(+), 63 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/661e23e2/metron-analytics/metron-profiler-common/pom.xml
--
diff --git a/metron-analytics/metron-profiler-common/pom.xml
b/metron-analytics/metron-profiler-common/pom.xml
index 928515f..74253cc 100644
--- a/metron-analytics/metron-profiler-common/pom.xml
+++ b/metron-analytics/metron-profiler-common/pom.xml
@@ -55,6 +55,10 @@
org.slf4j
slf4j-log4j12
+
+asm
+asm
+
@@ -109,30 +113,10 @@
provided
-org.apache.storm
-storm-core
-${global_storm_version}
-provided
-
-
-org.apache.logging.log4j
-log4j-api
-
-
-org.apache.logging.log4j
-log4j-core
-
-
-log4j-slf4j-impl
-org.apache.logging.log4j
-
-
-
-
org.mockito
mockito-all
${global_mockito_version}
test
-
\ No newline at end of file
+
http://git-wip-us.apache.org/repos/asf/metron/blob/661e23e2/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileMeasurement.java
--
diff --git
a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileMeasurement.java
b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileMeasurement.java
index 4737c3d..2212297 100644
---
a/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileMeasurement.java
+++
b/metron-analytics/metron-profiler-common/src/main/java/org/apache/metron/profiler/ProfileMeasurement.java
@@ -20,6 +20,8 @@
package org.apache.metron.profiler;
+import org.apache.commons.lang3.builder.EqualsBuilder;
+import org.apache.commons.lang3.builder.HashCodeBuilder;
import org.apache.metron.common.configuration.profiler.ProfileConfig;
import java.io.Serializable;
@@ -174,4 +176,37 @@ public class ProfileMeasurement implements Serializable {
public void setTriageValues(Map triageValues) {
this.triageValues = triageValues;
}
+
+ @Override
+ public boolean equals(Object o) {
+if (this == o) {
+ return true;
+}
+if (o == null || getClass() != o.getClass()) {
+ return false;
+}
+ProfileMeasurement that = (ProfileMeasurement) o;
+return new EqualsBuilder()
+.append(profileName, that.profileName)
+.append(entity, that.entity)
+.append(groups, that.groups)
+.append(period, that.period)
+.append(definition, that.definition)
+.append(profileValue, that.profileValue)
+.append(triageValues, that.triageValues)
+.isEquals();
+ }
+
+ @Override
+ public int hashCode() {
+return new HashCodeBuilder(17, 37)
+.append(profileName)
+.append(entity)
+.append(groups)
+.append(period)
+.append(definition)
+.append(profileValue
[41/50] [abbrv] metron git commit: METRON-1739 UDP packets are not handled (merrimanr) closes apache/metron#1168
METRON-1739 UDP packets are not handled (merrimanr) closes apache/metron#1168
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/14e80b3c
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/14e80b3c
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/14e80b3c
Branch: refs/remotes/apache/feature/METRON-1699-create-batch-profiler
Commit: 14e80b3c229ea2e5a8d615c87d822d4b44d8ec98
Parents: 076a6a1
Author: merrimanr
Authored: Fri Aug 17 08:47:32 2018 -0500
Committer: rmerriman
Committed: Fri Aug 17 08:47:32 2018 -0500
--
.../src/app/pcap/model/pdml.mock.ts | 31
.../pcap-packet-line.component.spec.ts | 53 --
.../pcap-packet-line.component.ts | 15 ++--
.../java/org/apache/metron/pcap/PcapHelper.java | 18 -
.../org/apache/metron/pcap/PcapHelperTest.java | 76
5 files changed, 180 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/14e80b3c/metron-interface/metron-alerts/src/app/pcap/model/pdml.mock.ts
--
diff --git a/metron-interface/metron-alerts/src/app/pcap/model/pdml.mock.ts
b/metron-interface/metron-alerts/src/app/pcap/model/pdml.mock.ts
index 6316b92..2520da1 100644
--- a/metron-interface/metron-alerts/src/app/pcap/model/pdml.mock.ts
+++ b/metron-interface/metron-alerts/src/app/pcap/model/pdml.mock.ts
@@ -60,3 +60,34 @@ export const fakePacket = {
} as PdmlProto
]
} as PdmlPacket;
+
+export const fakeUdpPacket = {
+ "name": '',
+ "expanded": false,
+ "protos": [
+{
+ "name": "geninfo",
+ "showname": "",
+ "fields": [
+{ "name": "timestamp", "pos": "0", "showname": "Captured Time",
"size": "342", "value": "1534414874.932794000", "show": "Aug 16, 2018
10:21:14.932794000 UTC", "unmaskedvalue": null, "hide": null, "fields": null,
"protos": null } as PdmlField
+ ]
+} as PdmlProto,
+{
+ "name": "ip",
+ "showname": "",
+ "fields": [
+{ "name": "ip.proto", "pos": "23", "showname": "Protocol: UDP (17)",
"size": "1", "value": "11", "show": "17", "unmaskedvalue": null, "hide": null,
"fields": null, "protos": null } as PdmlField,
+{ "name": "ip.src", "pos": "26", "showname": "Source: 0.0.0.0
(0.0.0.0)", "size": "4", "value": "", "show": "0.0.0.0",
"unmaskedvalue": null, "hide": null, "fields": null, "protos": null } as
PdmlField,
+{ "name": "ip.dst", "pos": "30", "showname": "Destination:
255.255.255.255 (255.255.255.255)", "size": "4", "value": "", "show":
"255.255.255.255", "unmaskedvalue": null, "hide": null, "fields": null,
"protos": null } as PdmlField
+ ]
+} as PdmlProto,
+{
+ "name": "udp",
+ "showname": "User Datagram Protocol, Src Port: bootpc (68), Dst Port:
bootps (67)",
+ "fields": [
+{ "name": "udp.srcport", "pos": "34", "showname": "Source port: bootpc
(68)", "size": "2", "value": "0044", "show": "68", "unmaskedvalue": null,
"hide": null, "fields": null, "protos": null } as PdmlField,
+{ "name": "udp.dstport", "pos": "36", "showname": "Destination port:
bootps (67)", "size": "2", "value": "0043", "show": "67", "unmaskedvalue":
null, "hide": null, "fields": null, "protos": null } as PdmlField
+ ],
+} as PdmlProto
+ ]
+} as PdmlPacket;
http://git-wip-us.apache.org/repos/asf/metron/blob/14e80b3c/metron-interface/metron-alerts/src/app/pcap/pcap-packet-line/pcap-packet-line.component.spec.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-packet-line/pcap-packet-line.component.spec.ts
b/metron-interface/metron-alerts/src/app/pcap/pcap-packet-line/pcap-packet-line.component.spec.ts
index d01c0cb..00f081f 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-packet-line/pcap-packet-line.component.spec.ts
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-packet-line/pcap-packet-line.component.spec.ts
@@ -17,6 +17,7 @@
*/
import { async, ComponentFixture, TestBed } from '@angular/core/testing';
import { fakePacket } from '../model/pdml.mock';
+import { fakeUdpPacket } from '../model/pdml.mock';
import { PcapPacketLineComponent } from './pcap-packet-line.component';
@@ -34,8 +35,6 @@ describe('PcapPacketLineComponent', () => {
beforeEach(() => {
fixture = TestBed.createComponent(PcapPacketLineComponent);
component = fixture.componentInstance;
-component.packet = fakePacket;
-fixture.detectChanges();
});
it('should be created', () => {
@@ -43,6 +42,8 @@ describe('PcapPacketLineComponent', () => {
});
it('should extract timestamp fields', () => {
+component.packet = fakePacket;
+
[32/50] [abbrv] metron git commit: METRON-1731: PCAP - Escape colons in output dir names (mmiklavc via mmiklavc) closes apache/metron#1155
METRON-1731: PCAP - Escape colons in output dir names (mmiklavc via mmiklavc)
closes apache/metron#1155
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/73dc63e6
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/73dc63e6
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/73dc63e6
Branch: refs/remotes/apache/feature/METRON-1699-create-batch-profiler
Commit: 73dc63e671b55d22d251f4be1c217259f4f5dc71
Parents: 05316a4
Author: mmiklavc
Authored: Fri Aug 10 12:42:47 2018 -0600
Committer: Michael Miklavcic
Committed: Fri Aug 10 12:42:47 2018 -0600
--
.../apache/metron/pcap/FixedPcapFilterTest.java | 286 --
.../org/apache/metron/pcap/PcapJobTest.java | 290 ---
.../apache/metron/pcap/QueryPcapFilterTest.java | 228 ---
.../pcap/filter/fixed/FixedPcapFilter.java | 14 +-
.../pcap/filter/query/QueryPcapFilter.java | 17 +-
.../metron/pcap/mr/OutputDirFormatter.java | 37 +++
.../java/org/apache/metron/pcap/mr/PcapJob.java | 5 +-
.../pcap/filter/fixed/FixedPcapFilterTest.java | 271 -
.../pcap/filter/query/QueryPcapFilterTest.java | 207 -
.../metron/pcap/mr/OutputDirFormatterTest.java | 62
.../org/apache/metron/pcap/mr/PcapJobTest.java | 290 +++
11 files changed, 877 insertions(+), 830 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/73dc63e6/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/FixedPcapFilterTest.java
--
diff --git
a/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/FixedPcapFilterTest.java
b/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/FixedPcapFilterTest.java
deleted file mode 100644
index 84969d3..000
---
a/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/FixedPcapFilterTest.java
+++ /dev/null
@@ -1,286 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.metron.pcap;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.metron.common.Constants;
-import org.apache.metron.pcap.filter.fixed.FixedPcapFilter;
-import org.junit.Assert;
-import org.junit.Test;
-
-import java.util.HashMap;
-import java.util.Map;
-
-public class FixedPcapFilterTest {
- @Test
- public void testTrivialEquality() throws Exception {
-Configuration config = new Configuration();
-final Map fields = new HashMap() {{
- put(Constants.Fields.SRC_ADDR.getName(), "src_ip");
- put(Constants.Fields.SRC_PORT.getName(), "0");
- put(Constants.Fields.DST_ADDR.getName(), "dst_ip");
- put(Constants.Fields.DST_PORT.getName(), "1");
- put(Constants.Fields.INCLUDES_REVERSE_TRAFFIC.getName(), "false");
-}};
-new FixedPcapFilter.Configurator().addToConfig(fields, config);
-{
- FixedPcapFilter filter = new FixedPcapFilter() {
-@Override
-protected Map packetToFields(PacketInfo pi) {
- return new HashMap() {{
-put(Constants.Fields.SRC_ADDR.getName(), "src_ip");
-put(Constants.Fields.SRC_PORT.getName(), 0);
-put(Constants.Fields.DST_ADDR.getName(), "dst_ip");
-put(Constants.Fields.DST_PORT.getName(), 1);
- }};
-}
- };
- filter.configure(config);
- Assert.assertTrue(filter.test(null));
-}
- }
-
- @Test
- public void testReverseTraffic() throws Exception {
-Configuration config = new Configuration();
-final Map fields = new HashMap() {{
- put(Constants.Fields.SRC_ADDR.getName(), "src_ip");
- put(Constants.Fields.SRC_PORT.getName(), "0");
- put(Constants.Fields.DST_ADDR.getName(), "dst_ip");
- put(Constants.Fields.DST_PORT.getName(), "1");
- put(Constants.Fields.INCLUDES_REVERSE_TRAFFIC.getName(), "true");
-}};
-new FixedPcapFilter.Configurator().addToConfig(fields, config);
-{
- FixedPcapFilter filter =
[28/50] [abbrv] metron git commit: METRON-1728: Handle null values in config in Pcap backend more gracefully (mmiklavc via mmiklavc) closes apache/metron#1151
METRON-1728: Handle null values in config in Pcap backend more gracefully
(mmiklavc via mmiklavc) closes apache/metron#1151
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/9064cca0
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/9064cca0
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/9064cca0
Branch: refs/remotes/apache/feature/METRON-1699-create-batch-profiler
Commit: 9064cca0317881176471c51abd16e99bf2ad7b10
Parents: 14dcb2d
Author: mmiklavc
Authored: Thu Aug 9 09:25:29 2018 -0600
Committer: Michael Miklavcic
Committed: Thu Aug 9 09:25:29 2018 -0600
--
.../common/configuration/ConfigOption.java | 32 --
.../common/configuration/ConfigOptionTest.java | 112 +++
.../org/apache/metron/pcap/query/CliParser.java | 25 +++--
.../org/apache/metron/pcap/PcapJobTest.java | 23
.../apache/metron/pcap/query/PcapCliTest.java | 10 +-
.../metron/pcap/config/PcapGlobalDefaults.java | 28 +
.../metron/pcap/finalizer/PcapFinalizer.java| 8 +-
.../pcap/finalizer/PcapRestFinalizer.java | 11 +-
.../java/org/apache/metron/pcap/mr/PcapJob.java | 25 +++--
9 files changed, 237 insertions(+), 37 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/9064cca0/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigOption.java
--
diff --git
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigOption.java
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigOption.java
index 8e4211b..6308f0a 100644
---
a/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigOption.java
+++
b/metron-platform/metron-common/src/main/java/org/apache/metron/common/configuration/ConfigOption.java
@@ -18,36 +18,54 @@
package org.apache.metron.common.configuration;
-import org.apache.metron.stellar.common.utils.ConversionUtils;
-
import java.util.Map;
import java.util.function.BiFunction;
+import org.apache.metron.stellar.common.utils.ConversionUtils;
public interface ConfigOption {
+
String getKey();
+
default BiFunction transform() {
-return (s,o) -> o;
+return (s, o) -> o;
}
default void put(Map map, Object value) {
map.put(getKey(), value);
}
+ default T getOrDefault(Map map, Class clazz, T
defaultValue) {
+T val;
+return ((val = get(map, clazz)) == null ? defaultValue : val);
+ }
+
default T get(Map map, Class clazz) {
Object obj = map.get(getKey());
-if(clazz.isInstance(obj)) {
+if (clazz.isInstance(obj)) {
return clazz.cast(obj);
-}
-else {
+} else {
return ConversionUtils.convert(obj, clazz);
}
}
- default T get(Map map, BiFunction
transform, Class clazz) {
+ default T getOrDefault(Map map, BiFunction transform,
+ Class clazz, T defaultValue) {
+T val;
+return ((val = get(map, transform, clazz)) == null ? defaultValue : val);
+ }
+
+ default T get(Map map, BiFunction
transform,
+ Class clazz) {
return clazz.cast(transform.apply(getKey(), map.get(getKey(;
}
+ default T getTransformedOrDefault(Map map, Class
clazz, T defaultValue) {
+T val;
+return ((val = getTransformed(map, clazz)) == null ? defaultValue : val);
+ }
+
default T getTransformed(Map map, Class clazz) {
return clazz.cast(transform().apply(getKey(), map.get(getKey(;
}
+
}
http://git-wip-us.apache.org/repos/asf/metron/blob/9064cca0/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigOptionTest.java
--
diff --git
a/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigOptionTest.java
b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigOptionTest.java
new file mode 100644
index 000..95db080
--- /dev/null
+++
b/metron-platform/metron-common/src/test/java/org/apache/metron/common/configuration/ConfigOptionTest.java
@@ -0,0 +1,112 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the
[39/50] [abbrv] metron git commit: METRON-1738: Pcap directories should have correct permissions (merrimanr via mmiklavc) closes apache/metron#1166
METRON-1738: Pcap directories should have correct permissions (merrimanr via mmiklavc) closes apache/metron#1166 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/076a6a19 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/076a6a19 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/076a6a19 Branch: refs/remotes/apache/feature/METRON-1699-create-batch-profiler Commit: 076a6a197df629d3b04e0757320e7681ea2fb3d9 Parents: 6b70571 Author: merrimanr Authored: Wed Aug 15 16:02:47 2018 -0600 Committer: Michael Miklavcic Committed: Wed Aug 15 16:02:47 2018 -0600 -- .../package/scripts/params/params_linux.py | 1 + .../package/scripts/params/status_params.py | 1 + .../CURRENT/package/scripts/rest_commands.py| 23 +++- .../CURRENT/package/scripts/rest_master.py | 5 + 4 files changed, 24 insertions(+), 6 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/076a6a19/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py -- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py index 115a54c..9be09f1 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py @@ -393,6 +393,7 @@ pcap_page_size = config['configurations']['metron-rest-env']['pcap_page_size'] pcap_yarn_queue = config['configurations']['metron-rest-env']['pcap_yarn_queue'] pcap_finalizer_threadpool_size= config['configurations']['metron-rest-env']['pcap_finalizer_threadpool_size'] pcap_configured_flag_file = status_params.pcap_configured_flag_file +pcap_perm_configured_flag_file = status_params.pcap_perm_configured_flag_file # MapReduce metron_user_hdfs_dir = '/user/' + metron_user http://git-wip-us.apache.org/repos/asf/metron/blob/076a6a19/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py -- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py index 0a9fdd0..99f5ec0 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/status_params.py @@ -120,6 +120,7 @@ metron_keytab_path = config['configurations']['metron-env']['metron_service_keyt # Pcap pcap_configured_flag_file = metron_zookeeper_config_path + '/../metron_pcap_configured' +pcap_perm_configured_flag_file = metron_zookeeper_config_path + '/../metron_pcap_perm_configured' # MapReduce metron_user_hdfs_dir_configured_flag_file = metron_zookeeper_config_path + '/../metron_user_hdfs_dir_configured' \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/076a6a19/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py -- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py index 463dca1..d44f478 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/rest_commands.py @@ -38,6 +38,7 @@ class RestCommands: __hbase_configured = False __hbase_acl_configured = False __pcap_configured = False +__pcap_perm_configured = False __metron_user_hdfs_dir_configured = False def __init__(self, params): @@ -49,6 +50,7 @@ class RestCommands: self.__hbase_configured =
[36/50] [abbrv] metron git commit: METRON-1735 Empty print status option causes NPE (merrimanr) closes apache/metron#1160
METRON-1735 Empty print status option causes NPE (merrimanr) closes
apache/metron#1160
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/8a926dd5
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/8a926dd5
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/8a926dd5
Branch: refs/remotes/apache/feature/METRON-1699-create-batch-profiler
Commit: 8a926dd5a4501df8c7ef51b526ad5e4d752a5e19
Parents: bce9b90
Author: merrimanr
Authored: Tue Aug 14 12:59:38 2018 -0500
Committer: rmerriman
Committed: Tue Aug 14 12:59:38 2018 -0500
--
.../org/apache/metron/pcap/query/CliParser.java | 4
.../apache/metron/pcap/query/PcapCliTest.java| 7 ++-
.../apache/metron/pcap/config/PcapConfig.java| 2 +-
.../java/org/apache/metron/pcap/mr/PcapJob.java | 18 +-
.../org/apache/metron/pcap/mr/PcapJobTest.java | 19 +++
5 files changed, 35 insertions(+), 15 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/8a926dd5/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
--
diff --git
a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
index 5040f90..b9a2a50 100644
---
a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
+++
b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/CliParser.java
@@ -55,7 +55,6 @@ public class CliParser {
options.addOption(newOption("rpf", "records_per_file", true,
String.format("Number of records to include in each output pcap file (defaults
to %s)", NUM_RECORDS_PER_FILE_DEFAULT)));
options.addOption(newOption("et", "end_time", true, "Packet end time
range. Default is current system time."));
options.addOption(newOption("df", "date_format", true, "Date format to use
for parsing start_time and end_time. Default is to use time in millis since the
epoch."));
-options.addOption(newOption("ps", "print_status", false, "Print the status
of the job as it runs"));
options.addOption(newOption("yq", "yarn_queue", true, "Yarn queue this job
will be submitted to"));
return options;
}
@@ -127,9 +126,6 @@ public class CliParser {
//no-op
}
}
-if (commandLine.hasOption("print_status")) {
- config.setPrintJobStatus(true);
-}
if (commandLine.hasOption("yarn_queue")) {
config.setYarnQueue(commandLine.getOptionValue("yarn_queue"));
}
http://git-wip-us.apache.org/repos/asf/metron/blob/8a926dd5/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
--
diff --git
a/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
b/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
index a71e997..3e7aad2 100644
---
a/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
+++
b/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/query/PcapCliTest.java
@@ -172,8 +172,7 @@ public class PcapCliTest {
"-protocol", "6",
"-include_reverse",
"-num_reducers", "10",
-"-records_per_file", "1000",
-"-ps"
+"-records_per_file", "1000"
};
Map query = new HashMap() {{
put(Constants.Fields.SRC_ADDR.getName(), "192.168.1.1");
@@ -217,7 +216,6 @@ public class PcapCliTest {
"-include_reverse",
"-num_reducers", "10",
"-records_per_file", "1000",
-"-ps",
"-yq", "pcap"
};
Map query = new HashMap() {{
@@ -295,8 +293,7 @@ public class PcapCliTest {
"-base_path", "/base/path",
"-base_output_path", "/base/output/path",
"-query", "some query string",
-"-records_per_file", "1000",
-"-ps"
+"-records_per_file", "1000"
};
String query = "some query string";
http://git-wip-us.apache.org/repos/asf/metron/blob/8a926dd5/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/config/PcapConfig.java
--
diff --git
a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/config/PcapConfig.java
b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/config/PcapConfig.java
index 4a08e14..abf35d0 100644
---
[34/50] [abbrv] metron git commit: METRON-1734 Src and Dst port filters are incorrect after changing to empty (merrimanr) closes apache/metron#1159
METRON-1734 Src and Dst port filters are incorrect after changing to empty
(merrimanr) closes apache/metron#1159
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/ba0a6092
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/ba0a6092
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/ba0a6092
Branch: refs/remotes/apache/feature/METRON-1699-create-batch-profiler
Commit: ba0a6092c97d23c118c2b5780c2559c2f849a220
Parents: 7a8c246
Author: merrimanr
Authored: Mon Aug 13 15:42:29 2018 -0500
Committer: rmerriman
Committed: Mon Aug 13 15:42:29 2018 -0500
--
.../pcap/pcap-filters/pcap-filters.component.spec.ts | 13 +
.../app/pcap/pcap-filters/pcap-filters.component.ts| 4
2 files changed, 17 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/ba0a6092/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
index c1d8664..3f1ab07 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
@@ -179,6 +179,19 @@ describe('PcapFiltersComponent', () => {
component.onSubmit();
});
+ it('Port fields should be removed from request when set to empty', () => {
+component.model.ipSrcPort = 44;
+component.model.ipDstPort = 44;
+component.ipSrcPort = '';
+component.ipDstPort = '';
+
+component.search.emit = (model: PcapRequest) => {
+ expect(model.ipSrcPort).toBeFalsy();
+ expect(model.ipDstPort).toBeFalsy();
+};
+component.onSubmit();
+ });
+
it('Filter should have an output called search', () => {
component.search.subscribe((filterModel) => {
expect(filterModel).toBeDefined();
http://git-wip-us.apache.org/repos/asf/metron/blob/ba0a6092/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.ts
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.ts
index 20a8c2b..df91ae3 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.ts
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.ts
@@ -63,9 +63,13 @@ export class PcapFiltersComponent implements OnInit,
OnChanges {
this.model.endTimeMs = new Date(this.endTimeStr).getTime();
if (this.ipSrcPort !== '') {
this.model.ipSrcPort = +this.ipSrcPort;
+} else {
+ delete this.model.ipSrcPort;
}
if (this.ipDstPort !== '') {
this.model.ipDstPort = +this.ipDstPort;
+} else {
+ delete this.model.ipDstPort;
}
this.search.emit(this.model);
[22/50] [abbrv] metron git commit: METRON-1726: Refactor PcapTopologyIntegrationTest (mmiklavc via mmiklavc) closes apache/metron#1140
METRON-1726: Refactor PcapTopologyIntegrationTest (mmiklavc via mmiklavc)
closes apache/metron#1140
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/7967f358
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/7967f358
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/7967f358
Branch: refs/remotes/apache/feature/METRON-1699-create-batch-profiler
Commit: 7967f358c6c4b8437935c43e54179126e48e248f
Parents: 3e77859
Author: mmiklavc
Authored: Tue Aug 7 15:02:20 2018 -0600
Committer: Michael Miklavcic
Committed: Tue Aug 7 15:02:20 2018 -0600
--
.../org/apache/metron/pcap/query/PcapCli.java | 2 +
.../PcapTopologyIntegrationTest.java| 892 ++-
.../metron/pcap/finalizer/PcapCliFinalizer.java | 5 +-
3 files changed, 473 insertions(+), 426 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/7967f358/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java
--
diff --git
a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java
b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java
index 0b06b0c..c23f037 100644
---
a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java
+++
b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java
@@ -87,6 +87,7 @@ public class PcapCli {
try {
config = fixedParser.parse(otherArgs);
commonConfig = config;
+PcapOptions.FINAL_OUTPUT_PATH.put(commonConfig, new Path(execDir));
} catch (ParseException | java.text.ParseException e) {
System.err.println(e.getMessage());
System.err.flush();
@@ -112,6 +113,7 @@ public class PcapCli {
try {
config = queryParser.parse(otherArgs);
commonConfig = config;
+PcapOptions.FINAL_OUTPUT_PATH.put(commonConfig, new Path(execDir));
} catch (ParseException | java.text.ParseException e) {
System.err.println(e.getMessage());
queryParser.printHelp();
http://git-wip-us.apache.org/repos/asf/metron/blob/7967f358/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/integration/PcapTopologyIntegrationTest.java
--
diff --git
a/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/integration/PcapTopologyIntegrationTest.java
b/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/integration/PcapTopologyIntegrationTest.java
index 108fd2b..c30267d 100644
---
a/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/integration/PcapTopologyIntegrationTest.java
+++
b/metron-platform/metron-pcap-backend/src/test/java/org/apache/metron/pcap/integration/PcapTopologyIntegrationTest.java
@@ -58,6 +58,7 @@ import org.apache.metron.integration.components.MRComponent;
import org.apache.metron.integration.components.ZKServerComponent;
import org.apache.metron.integration.utils.KafkaUtil;
import org.apache.metron.job.JobStatus;
+import org.apache.metron.job.Pageable;
import org.apache.metron.job.Statusable;
import org.apache.metron.pcap.PacketInfo;
import org.apache.metron.pcap.PcapHelper;
@@ -73,7 +74,10 @@ import org.apache.metron.spout.pcap.Endianness;
import org.apache.metron.spout.pcap.deserializer.Deserializers;
import org.apache.metron.test.utils.UnitTestHelper;
import org.json.simple.JSONObject;
+import org.junit.AfterClass;
import org.junit.Assert;
+import org.junit.Before;
+import org.junit.BeforeClass;
import org.junit.Ignore;
import org.junit.Test;
@@ -85,12 +89,19 @@ public class PcapTopologyIntegrationTest extends
BaseIntegrationTest {
private static String OUTPUT_DIR = BASE_DIR + "/output";
private static final int MAX_RETRIES = 30;
private static final int SLEEP_MS = 500;
- private String topologiesDir = "src/main/flux";
- private String targetDir = "target";
+ private static String topologiesDir = "src/main/flux";
+ private static String targetDir = "target";
+ private static ComponentRunner runner;
+ private static File inputDir;
+ private static File interimResultDir;
+ private static File outputDir;
+ private static List> pcapEntries;
+ private static boolean withHeaders;
+ private FixedPcapConfig configuration;
private static void clearOutDirs(File... dirs) throws IOException {
-for(File dir: dirs) {
- for(File f : dir.listFiles()) {
+for (File dir : dirs) {
+ for (File f : dir.listFiles()) {
if (f.isDirectory()) {
FileUtils.deleteDirectory(f);
} else {
@@ -99,8 +110,8 @@ public class
[16/50] [abbrv] metron git commit: Merge branch 'master' into feature/METRON-1554-pcap-query-panel
Merge branch 'master' into feature/METRON-1554-pcap-query-panel Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/89726659 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/89726659 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/89726659 Branch: refs/remotes/apache/feature/METRON-1699-create-batch-profiler Commit: 897266598b7984884d6cf8a6a699fe902e792483 Parents: 09a7f18 1b2bdff Author: merrimanr Authored: Mon Aug 6 11:08:48 2018 -0500 Committer: merrimanr Committed: Mon Aug 6 11:08:48 2018 -0500 -- .../METRON/CURRENT/repos/repoinfo.xml | 7 ++ .../packaging/docker/ansible-docker/Dockerfile | 118 --- .../packaging/docker/deb-docker/Dockerfile | 23 ++-- .../packaging/docker/rpm-docker/Dockerfile | 46 ++-- metron-sensors/pycapa/README.md | 47 +--- 5 files changed, 156 insertions(+), 85 deletions(-) --
[27/50] [abbrv] metron git commit: METRON-1730: Update steps to run pycapa on Centos 6 (mmiklavc via mmiklavc) closes apache/metron#1152
METRON-1730: Update steps to run pycapa on Centos 6 (mmiklavc via mmiklavc)
closes apache/metron#1152
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/2b0060b0
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/2b0060b0
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/2b0060b0
Branch: refs/remotes/apache/feature/METRON-1699-create-batch-profiler
Commit: 2b0060b0663ba3fa34047afdfe2fa856fe9007e8
Parents: 1b2bdff
Author: mmiklavc
Authored: Thu Aug 9 09:21:33 2018 -0600
Committer: Michael Miklavcic
Committed: Thu Aug 9 09:21:33 2018 -0600
--
metron-sensors/pycapa/README.md| 92 ++---
metron-sensors/pycapa/requirements.txt | 6 +-
2 files changed, 88 insertions(+), 10 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/2b0060b0/metron-sensors/pycapa/README.md
--
diff --git a/metron-sensors/pycapa/README.md b/metron-sensors/pycapa/README.md
index 9f01d33..38baf74 100644
--- a/metron-sensors/pycapa/README.md
+++ b/metron-sensors/pycapa/README.md
@@ -20,6 +20,8 @@ Pycapa
* [Overview](#overview)
* [Installation](#installation)
+ * [Centos 7](#centos-7)
+ * [Centos 6](#centos-6)
* [Usage](#usage)
* [Parameters](#parameters)
* [Examples](#examples)
@@ -27,22 +29,23 @@ Pycapa
* [FAQs](#faqs)
Overview
-
+
Pycapa performs network packet capture, both off-the-wire and from a Kafka
topic, which is useful for the testing and development of [Apache
Metron](https://github.com/apache/metron). It is not intended for production
use. The tool will capture packets from a specified interface and push them
into a Kafka Topic. The tool can also do the reverse. It can consume packets
from Kafka and reconstruct each network packet. This can then be used to
create a [libpcap-compliant
file](https://wiki.wireshark.org/Development/LibpcapFileFormat) or even to feed
directly into a tool like Wireshark to monitor ongoing activity.
Installation
-
+
General notes on the installation of Pycapa.
* Python 2.7 is required.
* The following package dependencies are required and can be installed
automatically with `pip`. The requirements are installed as part of step 4
*
[confluent-kafka-python](https://github.com/confluentinc/confluent-kafka-python)
* [pcapy](https://github.com/CoreSecurity/pcapy)
-* These instructions can be used directly on CentOS 7+.
-* Other Linux distributions that come with Python 2.7 can use these
instructions with some minor modifications.
-* Older distributions, like CentOS 6, that come with Python 2.6 installed,
should install Python 2.7 within a virtual environment and then run Pycapa from
within the virtual environment.
+### Centos 7
+
+* These instructions can be used directly on CentOS 7+.
+* Other Linux distributions that come with Python 2.7 can use these
instructions with some minor modifications.
1. Install system dependencies including the core development tools, Python
libraries and header files, and Libpcap libraries and header files. On CentOS
7+, you can install these requirements with the following command.
@@ -76,8 +79,83 @@ General notes on the installation of Pycapa.
python setup.py install
```
+### Centos 6
+
+* These instructions can be used directly on CentOS 6 - useful for developers
using the Full Dev Vagrant test box.
+* Older distributions, like CentOS 6, that come with Python 2.6 installed,
should install Python 2.7 within a virtual environment and then run Pycapa from
within the virtual environment.
+
+1. Set up a couple environment variables.
+
+```
+PYCAPA_HOME=/opt/pycapa
+PYTHON27_HOME=/opt/rh/python27/root
+```
+
+1. Install required packages.
+
+```
+for item in epel-release centos-release-scl "@Development tools" python27
python27-scldevel python27-python-virtualenv libpcap-devel libselinux-python;
do yum install -y $item; done
+```
+
+1. Setup Pycapa directory.
+
+```
+mkdir $PYCAPA_HOME && chmod 755 $PYCAPA_HOME
+```
+
+1. Create the virtualenv.
+
+```
+export LD_LIBRARY_PATH="/opt/rh/python27/root/usr/lib64"
+cd $PYCAPA_HOME
+${PYTHON27_HOME}/usr/bin/virtualenv pycapa-venv
+```
+
+1. Install Librdkafka at your chosen $PREFIX.
+
+```
+export PREFIX=/usr
+wget https://github.com/edenhill/librdkafka/archive/v0.11.5.tar.gz -O -
| tar -xz
+cd librdkafka-0.11.5/
+./configure --prefix=$PREFIX
+make
+make install
+```
+
+1. Add Librdkafka to the dynamic library load path.
+
+```
+echo "$PREFIX/lib" >> /etc/ld.so.conf.d/pycapa.conf
+ldconfig -v
+```
+
+1. Copy the Pycapa source files from the Metron project to your
[49/50] [abbrv] metron git commit: METRON-1751 Storm Profiler dies when consuming null message (nickwallen) closes apache/metron#1176
METRON-1751 Storm Profiler dies when consuming null message (nickwallen) closes
apache/metron#1176
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/d32bd50d
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/d32bd50d
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/d32bd50d
Branch: refs/remotes/apache/feature/METRON-1699-create-batch-profiler
Commit: d32bd50d43aae87af9ec12d2daea83b2f4eca342
Parents: 661e23e
Author: nickwallen
Authored: Wed Aug 29 14:55:58 2018 -0400
Committer: nickallen
Committed: Wed Aug 29 14:55:58 2018 -0400
--
.../profiler/bolt/ProfileSplitterBolt.java | 29 +++-
.../profiler/bolt/ProfileSplitterBoltTest.java | 16 +++
2 files changed, 32 insertions(+), 13 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/d32bd50d/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java
--
diff --git
a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java
b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java
index f28411f..87f1ba9 100644
---
a/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java
+++
b/metron-analytics/metron-profiler/src/main/java/org/apache/metron/profiler/bolt/ProfileSplitterBolt.java
@@ -144,9 +144,9 @@ public class ProfileSplitterBolt extends
ConfiguredProfilerBolt {
try {
doExecute(input);
-} catch (IllegalArgumentException | ParseException |
UnsupportedEncodingException e) {
- LOG.error("Unexpected error", e);
- collector.reportError(e);
+} catch (Throwable t) {
+ LOG.error("Unexpected error", t);
+ collector.reportError(t);
} finally {
collector.ack(input);
@@ -157,22 +157,25 @@ public class ProfileSplitterBolt extends
ConfiguredProfilerBolt {
// retrieve the input message
byte[] data = input.getBinary(0);
+if(data == null) {
+ LOG.debug("Received null message. Nothing to do.");
+ return;
+}
JSONObject message = (JSONObject) parser.parse(new String(data, "UTF8"));
// ensure there is a valid profiler configuration
ProfilerConfig config = getProfilerConfig();
-if(config != null && config.getProfiles().size() > 0) {
-
- // what time is it?
- Clock clock = clockFactory.createClock(config);
- Optional timestamp = clock.currentTimeMillis(message);
+if(config == null || getProfilerConfig().getProfiles().size() == 0) {
+ LOG.debug("No Profiler configuration found. Nothing to do.");
+ return;
+}
- // route the message. if a message does not contain the timestamp
field, it cannot be routed.
- timestamp.ifPresent(ts -> routeMessage(input, message, config, ts));
+// what time is it?
+Clock clock = clockFactory.createClock(config);
+Optional timestamp = clock.currentTimeMillis(message);
-} else {
- LOG.debug("No Profiler configuration found. Nothing to do.");
-}
+// route the message. if a message does not contain the timestamp field,
it cannot be routed.
+timestamp.ifPresent(ts -> routeMessage(input, message, config, ts));
}
/**
http://git-wip-us.apache.org/repos/asf/metron/blob/d32bd50d/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileSplitterBoltTest.java
--
diff --git
a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileSplitterBoltTest.java
b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileSplitterBoltTest.java
index bf81923..72e2b72 100644
---
a/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileSplitterBoltTest.java
+++
b/metron-analytics/metron-profiler/src/test/java/org/apache/metron/profiler/bolt/ProfileSplitterBoltTest.java
@@ -404,6 +404,22 @@ public class ProfileSplitterBoltTest extends BaseBoltTest {
.emit(any(Values.class));
}
+ @Test
+ public void testWithNullMessage() throws Exception {
+
+// ensure the tuple returns null to mimic a null message in kafka
+when(tuple.getBinary(0)).thenReturn(null);
+
+ProfilerConfig config = toProfilerConfig(profileWithOnlyIfInvalid);
+ProfileSplitterBolt bolt = createBolt(config);
+bolt.execute(tuple);
+
+// a tuple should NOT be emitted for the downstream profile builder
+verify(outputCollector, times(0))
+.emit(any(Values.class));
+
+ }
+
/**
* Creates a ProfilerConfig based on a string containing JSON.
*
[15/50] [abbrv] metron git commit: METRON-1676 PCAP UI - Add data range selector to the filter bar (tiborm via merrimanr) closes apache/metron#1119
METRON-1676 PCAP UI - Add data range selector to the filter bar (tiborm via
merrimanr) closes apache/metron#1119
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/09a7f189
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/09a7f189
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/09a7f189
Branch: refs/remotes/apache/feature/METRON-1699-create-batch-profiler
Commit: 09a7f18994dc8254ca158139c44258cb7f1107ec
Parents: b445bfe
Author: tiborm
Authored: Mon Aug 6 09:14:54 2018 -0500
Committer: rmerriman
Committed: Mon Aug 6 09:14:54 2018 -0500
--
.../pcap-filters/pcap-filters.component.html| 30 ++--
.../pcap-filters/pcap-filters.component.scss| 2 +-
.../pcap-filters/pcap-filters.component.spec.ts | 156 ++-
.../pcap/pcap-filters/pcap-filters.component.ts | 19 ++-
.../metron-alerts/src/app/pcap/pcap.module.ts | 6 +-
.../src/app/pcap/service/pcap.service.ts| 4 +-
.../shared/directives/alert-search.directive.ts | 2 +-
7 files changed, 195 insertions(+), 24 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/09a7f189/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.html
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.html
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.html
index abc4cf0..f4133df 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.html
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.html
@@ -12,23 +12,33 @@
the specific language governing permissions and limitations under the
License.
-->
+
+
+From
+
+
+
+To
+
+
+
-IP Source Address
-
+IP Source Address
+
-IP Source Port
-
+IP Source Port
+
- IP Dest Address
-
+ IP Dest Address
+
-IP Dest Port
-
+IP Dest Port
+
@@ -37,8 +47,8 @@
-Include Reverse Traffic
-
+Include Reverse Traffic
+
http://git-wip-us.apache.org/repos/asf/metron/blob/09a7f189/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.scss
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.scss
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.scss
index 80bac08..b33e804 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.scss
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.scss
@@ -28,7 +28,7 @@
}
.form-group {
- padding-right: 0.75em;
+ padding: 0 0.75em 0.75em 0;
label {
margin-right: 1em;
}
http://git-wip-us.apache.org/repos/asf/metron/blob/09a7f189/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
index 8134f1d..c0f9c3b 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
@@ -1,3 +1,4 @@
+
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -15,10 +16,23 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-import { async, ComponentFixture, TestBed } from '@angular/core/testing';
+import { async, ComponentFixture, TestBed, fakeAsync, tick } from
'@angular/core/testing';
+import { By } from '@angular/platform-browser';
import { PcapFiltersComponent } from './pcap-filters.component';
import { FormsModule } from '../../../../node_modules/@angular/forms';
+import { Component, Input, Output, EventEmitter } from '@angular/core';
+import { PcapRequest } from '../model/pcap.request';
+import { emit } from 'cluster';
+
+@Component({
+ selector: 'app-date-picker',
+ template: '',
+})
+class FakeDatePicker {
+ @Input() date: string;
+ @Output() dateChange = new EventEmitter();
+}
describe('PcapFiltersComponent', () => {
let component: PcapFiltersComponent;
@@ -29,18 +43,148 @@ describe('PcapFiltersComponent', () => {
imports: [
FormsModule
],
- declarations: [
[26/50] [abbrv] metron git commit: METRON-1713 PCAP UI - Add a way to kill a pcap job (tiborm via merrimanr) closes apache/metron#1143
METRON-1713 PCAP UI - Add a way to kill a pcap job (tiborm via merrimanr)
closes apache/metron#1143
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/14dcb2d9
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/14dcb2d9
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/14dcb2d9
Branch: refs/remotes/apache/feature/METRON-1699-create-batch-profiler
Commit: 14dcb2d90581835d8206c65918c24e8cb04bfd06
Parents: 5b3e2c3
Author: tiborm
Authored: Thu Aug 9 08:33:21 2018 -0500
Committer: rmerriman
Committed: Thu Aug 9 08:33:21 2018 -0500
--
.../pcap/pcap-panel/pcap-panel.component.html | 7 +-
.../pcap/pcap-panel/pcap-panel.component.scss | 33 +
.../pcap-panel/pcap-panel.component.spec.ts | 130 +++
.../app/pcap/pcap-panel/pcap-panel.component.ts | 42 +-
.../src/app/pcap/service/pcap.service.ts| 10 +-
5 files changed, 215 insertions(+), 7 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/14dcb2d9/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.html
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.html
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.html
index 950f49c..0dda268 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.html
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.html
@@ -15,8 +15,11 @@
-
-{{progressWidth}}%
+
+
+ {{progressWidth}}%
+
+
{{ errorMsg }}
http://git-wip-us.apache.org/repos/asf/metron/blob/14dcb2d9/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.scss
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.scss
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.scss
index 8989bf9..523f5ce 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.scss
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.scss
@@ -31,4 +31,37 @@
.progress-bar {
width: 0;
+ height: 34px;
+ line-height: 34px;
+ vertical-align: middle;
+ font-size: 0.875rem;
+}
+
+.pcap-progress-wrapper {
+ position: relative;
+ padding-right: 55px;
+}
+
+.pcap-cancel-query-button {
+ position: absolute;
+ top: 0;
+ right: 0;
+ padding-top: 6px;
+ padding-bottom: 6px;
+ background: $icon-button-background;
+ min-width: 42px;
+ padding-left: 0;
+ padding-right: 0;
+ border: 1px solid $blue-chill !important;
+ cursor: pointer;
+
+ &:focus {
+box-shadow: none;
+ }
+
+ &::before {
+font-family: "FontAwesome";
+content: '\f00d';
+color: $piction-blue;
+ }
}
http://git-wip-us.apache.org/repos/asf/metron/blob/14dcb2d9/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.spec.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.spec.ts
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.spec.ts
index 0804b79..9dacc7f 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.spec.ts
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.spec.ts
@@ -50,6 +50,9 @@ class FakePcapService {
return '';
}
submitRequest() {}
+ cancelQuery() {
+return defer(() => Promise.resolve());
+ }
}
describe('PcapPanelComponent', () => {
@@ -327,6 +330,133 @@ describe('PcapPanelComponent', () => {
expect(fixture.debugElement.query(By.css('app-pcap-list'))).toBeDefined();
}));
+ it('should render a cancel button only if a query runs', () => {
+component.queryRunning = false;
+fixture.detectChanges();
+
expect(fixture.debugElement.query(By.css('[data-qe-id="pcap-cancel-query-button"]'))).toBeFalsy();
+
+component.queryRunning = true;
+fixture.detectChanges();
+
expect(fixture.debugElement.query(By.css('[data-qe-id="pcap-cancel-query-button"]'))).toBeDefined();
+ });
+
+ it('should hide the progress bar if the user clicks on the cancel button',
fakeAsync(() => {
+component.queryRunning = true;
+component.queryId = 'testid';
+fixture.detectChanges();
+expect(fixture.debugElement.query(By.css('.pcap-progress'))).toBeDefined();
+
+const cancelBtn =
fixture.debugElement.query(By.css('[data-qe-id="pcap-cancel-query-button"]'));
+const cancelBtnEl = cancelBtn.nativeElement;
+
+cancelBtnEl.click();
+tick();
+
[29/50] [abbrv] metron git commit: Merge branch 'master' into feature/METRON-1554-pcap-query-panel
Merge branch 'master' into feature/METRON-1554-pcap-query-panel Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/d8a5922b Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/d8a5922b Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/d8a5922b Branch: refs/remotes/apache/feature/METRON-1699-create-batch-profiler Commit: d8a5922b4cc5b20be7cb08f333a861241c365fd5 Parents: 9064cca 2b0060b Author: Michael Miklavcic Authored: Thu Aug 9 09:31:01 2018 -0600 Committer: Michael Miklavcic Committed: Thu Aug 9 09:31:01 2018 -0600 -- metron-sensors/pycapa/README.md| 92 ++--- metron-sensors/pycapa/requirements.txt | 6 +- 2 files changed, 88 insertions(+), 10 deletions(-) --
[40/50] [abbrv] metron git commit: METRON-1727: Alerts are not populated on the alerts UI after enabling X-pack for Elastic search (MohanDV via mmiklavc) closes apache/metron#1141
METRON-1727: Alerts are not populated on the alerts UI after enabling X-pack for Elastic search (MohanDV via mmiklavc) closes apache/metron#1141 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/308c2b27 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/308c2b27 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/308c2b27 Branch: refs/remotes/apache/feature/METRON-1699-create-batch-profiler Commit: 308c2b27b9b234c0c84c343dfc45b3617b027e64 Parents: 2b0060b Author: MohanDV Authored: Wed Aug 15 16:16:26 2018 -0600 Committer: Michael Miklavcic Committed: Wed Aug 15 16:20:54 2018 -0600 -- metron-deployment/Kerberos-manual-setup.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/308c2b27/metron-deployment/Kerberos-manual-setup.md -- diff --git a/metron-deployment/Kerberos-manual-setup.md b/metron-deployment/Kerberos-manual-setup.md index e3de148..18891a4 100644 --- a/metron-deployment/Kerberos-manual-setup.md +++ b/metron-deployment/Kerberos-manual-setup.md @@ -580,7 +580,7 @@ X-Pack Submit the update to Zookeeper ``` -$METRON_HOME/bin/zk_load_configs.sh -m PUSH -i METRON_HOME/config/zookeeper/ -z $ZOOKEEPER +$METRON_HOME/bin/zk_load_configs.sh -m PUSH -i $METRON_HOME/config/zookeeper/ -z $ZOOKEEPER ``` 1. The last step before restarting the topology is to create a custom X-Pack shaded and relocated jar. This is up to you because of licensing restrictions, but here is a sample Maven pom file that should help. @@ -737,6 +737,8 @@ X-Pack $METRON_HOME/bin/start_elasticsearch_topology.sh ``` +1. Restart the metron-rest service, and make sure the elasticsearch-xpack-shaded-5.6.2.jar is in the METRON_REST_CLASSPATH when the metron-rest starts. + Once you've performed these steps, you should be able to start seeing data in your ES indexes. ### X-Pack Common Problems
[47/50] [abbrv] metron git commit: METRON-1743 CEF testPaloAltoCEF test using a confusing variable name (JonZeolla via justinleet) closes apache/metron#1173
METRON-1743 CEF testPaloAltoCEF test using a confusing variable name (JonZeolla
via justinleet) closes apache/metron#1173
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/29cbdcc4
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/29cbdcc4
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/29cbdcc4
Branch: refs/remotes/apache/feature/METRON-1699-create-batch-profiler
Commit: 29cbdcc43c66080e1464e4c9c365c63a3f9035b1
Parents: 09826da
Author: JonZeolla
Authored: Wed Aug 29 09:43:43 2018 -0400
Committer: leet
Committed: Wed Aug 29 09:43:43 2018 -0400
--
.../test/java/org/apache/metron/parsers/cef/CEFParserTest.java | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/29cbdcc4/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/cef/CEFParserTest.java
--
diff --git
a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/cef/CEFParserTest.java
b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/cef/CEFParserTest.java
index a1fe2e8..f679c91 100644
---
a/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/cef/CEFParserTest.java
+++
b/metron-platform/metron-parsers/src/test/java/org/apache/metron/parsers/cef/CEFParserTest.java
@@ -196,8 +196,8 @@ public class CEFParserTest {
@Test
public void testPaloAltoCEF() throws Exception {
- URL waf_url = Resources.getResource(getClass(), "palo.cef");
- runTest("palo", Resources.readLines(waf_url, UTF_8),
+ URL palo_url = Resources.getResource(getClass(), "palo.cef");
+ runTest("palo", Resources.readLines(palo_url, UTF_8),
Resources.toString(Resources.getResource(getClass(), "palo.schema"), UTF_8));
}
[43/50] [abbrv] metron git commit: METRON-1724 Date/time validation missing in PCAP query (tiborm via nickwallen) closes apache/metron#1172
METRON-1724 Date/time validation missing in PCAP query (tiborm via nickwallen)
closes apache/metron#1172
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/e66cfc80
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/e66cfc80
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/e66cfc80
Branch: refs/remotes/apache/feature/METRON-1699-create-batch-profiler
Commit: e66cfc80e6a6fa53110c3f2fa8ee0d31ea997bf6
Parents: 9fdccba
Author: tiborm
Authored: Mon Aug 27 09:04:21 2018 -0400
Committer: nickallen
Committed: Mon Aug 27 09:04:21 2018 -0400
--
.../src/app/pcap/model/pcap.mock.ts | 4 +-
.../src/app/pcap/model/pcap.request.ts | 18 +-
.../pcap-filters/pcap-filters.component.html| 55 +++--
.../pcap-filters/pcap-filters.component.scss| 13 +-
.../pcap-filters/pcap-filters.component.spec.ts | 207 ++-
.../pcap/pcap-filters/pcap-filters.component.ts | 156 +++---
.../app/pcap/pcap-list/pcap-list.component.html | 4 +-
.../pcap-packet-line.component.spec.ts | 3 +-
.../pcap-packet-line.component.ts | 22 +-
.../pcap-panel/pcap-panel.component.spec.ts | 16 +-
.../app/pcap/pcap-panel/pcap-panel.component.ts | 6 +-
.../metron-alerts/src/app/pcap/pcap.module.ts | 5 +-
.../date-picker/date-picker.component.scss | 14 +-
.../shared/date-picker/date-picker.component.ts | 34 ++-
.../shared/date-picker/date-picker.module.ts| 3 +-
.../metron-alerts/src/app/utils/constants.ts| 3 +
.../metron-alerts/src/app/utils/utils.ts| 12 +-
17 files changed, 360 insertions(+), 215 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/e66cfc80/metron-interface/metron-alerts/src/app/pcap/model/pcap.mock.ts
--
diff --git a/metron-interface/metron-alerts/src/app/pcap/model/pcap.mock.ts
b/metron-interface/metron-alerts/src/app/pcap/model/pcap.mock.ts
index bf02da8..c867fe9 100644
--- a/metron-interface/metron-alerts/src/app/pcap/model/pcap.mock.ts
+++ b/metron-interface/metron-alerts/src/app/pcap/model/pcap.mock.ts
@@ -22,9 +22,9 @@ export const fakePcapRequest = {
startTimeMs: 0,
endTimeMs: 0,
ipSrcAddr: '0.0.0.0',
- ipSrcPort: 80,
+ ipSrcPort: '80',
ipDstAddr: '0.0.0.0',
- ipDstPort: 80,
+ ipDstPort: '80',
protocol: '*',
packetFilter: '*',
includeReverse: false
http://git-wip-us.apache.org/repos/asf/metron/blob/e66cfc80/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
--
diff --git a/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
b/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
index 8afc963..3e00a4b 100644
--- a/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
+++ b/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
@@ -17,13 +17,13 @@
*/
export class PcapRequest {
- startTimeMs: number = 0;
- endTimeMs: number = 15;
- ipSrcAddr: string = '';
- ipSrcPort: number;
- ipDstAddr: string = '';
- ipDstPort: number;
- protocol: string = '';
- packetFilter: string = '';
- includeReverse: boolean = false;
+ startTimeMs: number;
+ endTimeMs: number;
+ ipSrcAddr = '';
+ ipSrcPort = '';
+ ipDstAddr = '';
+ ipDstPort = '';
+ protocol = '';
+ packetFilter = '';
+ includeReverse = false;
}
http://git-wip-us.apache.org/repos/asf/metron/blob/e66cfc80/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.html
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.html
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.html
index 039307a..c7a4db5 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.html
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.html
@@ -11,52 +11,63 @@
OR CONDITIONS OF ANY KIND, either express or implied. See the License
for
the specific language governing permissions and limitations under the
License.
-->
-
+
From
-
+
To
-
+
-IP Source Address
-
+IP Source Address
+
-IP Source Port
-
+IP Source Port
+
- IP Dest Address
-
+
+IP Dest Address
+
-IP Dest Port
-
+IP Dest Port
+
-
-Protocol
-
-
+
+Protocol
+
+
-
-Include Reverse Traffic
-
-
+
+Include Reverse Traffic
+
+
-Filter
-
+Fil
[02/50] [abbrv] metron git commit: METRON-1661 Create Pcap Query Filter endpoint (merrimanr) closes apache/metron#1125
METRON-1661 Create Pcap Query Filter endpoint (merrimanr) closes
apache/metron#1125
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/d0e87576
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/d0e87576
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/d0e87576
Branch: refs/remotes/apache/feature/METRON-1699-create-batch-profiler
Commit: d0e87576944b6370eb1d9a0c98d0450f2c51f052
Parents: e78bc53
Author: merrimanr
Authored: Mon Jul 23 09:09:37 2018 -0500
Committer: rmerriman
Committed: Mon Jul 23 09:09:37 2018 -0500
--
.../rest/model/pcap/FixedPcapRequest.java | 1 +
.../metron/rest/model/pcap/PcapRequest.java | 4 +-
.../rest/model/pcap/QueryPcapOptions.java | 35 +
.../rest/model/pcap/QueryPcapRequest.java | 59 ++
metron-interface/metron-rest/README.md | 10 ++-
.../metron/rest/controller/PcapController.java | 15 +++-
.../apache/metron/rest/service/PcapService.java | 8 +-
.../rest/service/impl/PcapServiceImpl.java | 13 ++--
.../PcapControllerIntegrationTest.java | 81 +---
.../apache/metron/rest/mock/MockPcapJob.java| 13 ++--
.../rest/service/impl/PcapServiceImplTest.java | 55 +++--
11 files changed, 240 insertions(+), 54 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/d0e87576/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/FixedPcapRequest.java
--
diff --git
a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/FixedPcapRequest.java
b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/FixedPcapRequest.java
index a2d345b..d91aac7 100644
---
a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/FixedPcapRequest.java
+++
b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/FixedPcapRequest.java
@@ -87,6 +87,7 @@ public class FixedPcapRequest extends PcapRequest {
FixedPcapOptions.INCLUDE_REVERSE.put(this, includeReverse);
}
+ @Override
public void setFields() {
Map fields = new HashMap<>();
if (getIpSrcAddr() != null) {
http://git-wip-us.apache.org/repos/asf/metron/blob/d0e87576/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/PcapRequest.java
--
diff --git
a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/PcapRequest.java
b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/PcapRequest.java
index 64ed932..44721fa 100644
---
a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/PcapRequest.java
+++
b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/PcapRequest.java
@@ -22,7 +22,7 @@ import org.apache.metron.pcap.config.PcapOptions;
import java.util.HashMap;
-public class PcapRequest extends AbstractMapDecorator {
+public abstract class PcapRequest extends AbstractMapDecorator
{
public PcapRequest() {
super(new HashMap<>());
@@ -78,4 +78,6 @@ public class PcapRequest extends AbstractMapDecorator {
public void setNumReducers(Integer numReducers) {
PcapOptions.NUM_REDUCERS.put(this, numReducers);
}
+
+ public abstract void setFields();
}
http://git-wip-us.apache.org/repos/asf/metron/blob/d0e87576/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/QueryPcapOptions.java
--
diff --git
a/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/QueryPcapOptions.java
b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/QueryPcapOptions.java
new file mode 100644
index 000..1ae3da6
--- /dev/null
+++
b/metron-interface/metron-rest-client/src/main/java/org/apache/metron/rest/model/pcap/QueryPcapOptions.java
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR
[44/50] [abbrv] metron git commit: METRON-1752 Prevent package.lock from changing during build (sardell via merrimanr) closes apache/metron#1177
http://git-wip-us.apache.org/repos/asf/metron/blob/09826da8/metron-interface/metron-config/pom.xml -- diff --git a/metron-interface/metron-config/pom.xml b/metron-interface/metron-config/pom.xml index 14acdc6..59c416e 100644 --- a/metron-interface/metron-config/pom.xml +++ b/metron-interface/metron-config/pom.xml @@ -1,15 +1,15 @@ - http://maven.apache.org/POM/4.0.0; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance; @@ -25,8 +25,8 @@ UTF-8 UTF-8 -v6.2.0 -3.8.9 +v9.11.1 +6.2.0 @@ -53,12 +53,12 @@ generate-resources -npm install +npm ci npm - install + ci
[31/50] [abbrv] metron git commit: METRON-1702 Reload a running job in the UI (merrimanr) closes apache/metron#1156
METRON-1702 Reload a running job in the UI (merrimanr) closes apache/metron#1156
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/05316a4d
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/05316a4d
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/05316a4d
Branch: refs/remotes/apache/feature/METRON-1699-create-batch-profiler
Commit: 05316a4d989d7546058e73b3a75eef435f2608a0
Parents: b29833c
Author: merrimanr
Authored: Fri Aug 10 08:27:06 2018 -0500
Committer: rmerriman
Committed: Fri Aug 10 08:27:06 2018 -0500
--
.../src/app/pcap/model/pcap.request.ts | 4 +-
.../pcap-filters/pcap-filters.component.spec.ts | 60 +-
.../pcap/pcap-filters/pcap-filters.component.ts | 26 --
.../pcap/pcap-panel/pcap-panel.component.html | 2 +-
.../pcap-panel/pcap-panel.component.spec.ts | 37 +
.../app/pcap/pcap-panel/pcap-panel.component.ts | 87 +---
.../src/app/pcap/service/pcap.service.spec.ts | 44 ++
.../src/app/pcap/service/pcap.service.ts| 15
.../rest/service/impl/PcapServiceImpl.java | 10 +--
.../rest/service/impl/PcapServiceImplTest.java | 36
10 files changed, 254 insertions(+), 67 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/05316a4d/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
--
diff --git a/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
b/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
index d00a6ac..8afc963 100644
--- a/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
+++ b/metron-interface/metron-alerts/src/app/pcap/model/pcap.request.ts
@@ -20,9 +20,9 @@ export class PcapRequest {
startTimeMs: number = 0;
endTimeMs: number = 15;
ipSrcAddr: string = '';
- ipSrcPort: number = 0;
+ ipSrcPort: number;
ipDstAddr: string = '';
- ipDstPort: number = 0;
+ ipDstPort: number;
protocol: string = '';
packetFilter: string = '';
includeReverse: boolean = false;
http://git-wip-us.apache.org/repos/asf/metron/blob/05316a4d/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
index 4336b22..c1d8664 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-filters/pcap-filters.component.spec.ts
@@ -21,7 +21,7 @@ import { By } from '@angular/platform-browser';
import { PcapFiltersComponent } from './pcap-filters.component';
import { FormsModule } from '../../../../node_modules/@angular/forms';
-import { Component, Input, Output, EventEmitter, DebugElement } from
'@angular/core';
+import { Component, Input, Output, EventEmitter, DebugElement, SimpleChange }
from '@angular/core';
import { PcapRequest } from '../model/pcap.request';
@Component({
@@ -171,6 +171,14 @@ describe('PcapFiltersComponent', () => {
component.onSubmit();
});
+ it('Port fields should be missing by default', () => {
+component.search.emit = (model: PcapRequest) => {
+ expect(model.ipSrcPort).toBeFalsy();
+ expect(model.ipDstPort).toBeFalsy();
+};
+component.onSubmit();
+ });
+
it('Filter should have an output called search', () => {
component.search.subscribe((filterModel) => {
expect(filterModel).toBeDefined();
@@ -194,14 +202,60 @@ describe('PcapFiltersComponent', () => {
expect(fixture.componentInstance.model.hasOwnProperty('startTimeMs')).toBeTruthy();
expect(fixture.componentInstance.model.hasOwnProperty('endTimeMs')).toBeTruthy();
expect(fixture.componentInstance.model.hasOwnProperty('ipSrcAddr')).toBeTruthy();
-
expect(fixture.componentInstance.model.hasOwnProperty('ipSrcPort')).toBeTruthy();
+
expect(fixture.componentInstance.model.hasOwnProperty('ipSrcPort')).toBeFalsy();
expect(fixture.componentInstance.model.hasOwnProperty('ipDstAddr')).toBeTruthy();
-
expect(fixture.componentInstance.model.hasOwnProperty('ipDstPort')).toBeTruthy();
+
expect(fixture.componentInstance.model.hasOwnProperty('ipDstPort')).toBeFalsy();
expect(fixture.componentInstance.model.hasOwnProperty('protocol')).toBeTruthy();
expect(fixture.componentInstance.model.hasOwnProperty('packetFilter')).toBeTruthy();
expect(fixture.componentInstance.model.hasOwnProperty('includeReverse')).toBeTruthy();
});
+ it('should update request on changes', ()
[20/50] [abbrv] metron git commit: METRON-1683 PCAP UI - Fix the download progress bar (sardell via merrimanr) closes apache/metron#1122
http://git-wip-us.apache.org/repos/asf/metron/blob/3e778592/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.spec.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.spec.ts
b/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.spec.ts
index 155e1f8..de39a0a 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.spec.ts
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-packet/pcap-packet.component.spec.ts
@@ -16,9 +16,10 @@
* limitations under the License.
*/
import { async, ComponentFixture, TestBed } from '@angular/core/testing';
+import { fakePacket } from '../model/pdml.mock';
import { PcapPacketComponent } from './pcap-packet.component';
-import { PdmlPacket } from '../model/pdml';
+import { By } from '@angular/platform-browser';
describe('PcapPacketComponent', () => {
let component: PcapPacketComponent;
@@ -34,1250 +35,45 @@ describe('PcapPacketComponent', () => {
beforeEach(() => {
fixture = TestBed.createComponent(PcapPacketComponent);
component = fixture.componentInstance;
-component.packet = fakePacket as PdmlPacket;
+component.packet = fakePacket;
fixture.detectChanges();
});
it('should create', () => {
expect(component).toBeTruthy();
});
-});
-const fakePacket = {
- name: "something",
- expanded: false,
- protos: [
-{
- "name": "geninfo",
- "pos": "0",
- "showname": "General information",
- "size": "722",
- "hide": null,
- "fields": [
-{
- "name": "num",
- "pos": "0",
- "showname": "Number",
- "size": "722",
- "value": "1",
- "show": "1",
- "unmaskedvalue": null,
- "hide": null,
- "fields": null,
- "protos": null
-},
-{
- "name": "len",
- "pos": "0",
- "showname": "Frame Length",
- "size": "722",
- "value": "2d2",
- "show": "722",
- "unmaskedvalue": null,
- "hide": null,
- "fields": null,
- "protos": null
-},
-{
- "name": "caplen",
- "pos": "0",
- "showname": "Captured Length",
- "size": "722",
- "value": "2d2",
- "show": "722",
- "unmaskedvalue": null,
- "hide": null,
- "fields": null,
- "protos": null
-},
-{
- "name": "timestamp",
- "pos": "0",
- "showname": "Captured Time",
- "size": "722",
- "value": "1458240269.373968000",
- "show": "Mar 17, 2016 18:44:29.373968000 UTC",
- "unmaskedvalue": null,
- "hide": null,
- "fields": null,
- "protos": null
-}
- ]
-},
-{
- "name": "frame",
- "pos": "0",
- "showname": "Frame 1: 722 bytes on wire (5776 bits), 722 bytes captured
(5776 bits)",
- "size": "722",
- "hide": null,
- "fields": [
-{
- "name": "frame.dlt",
- "pos": "0",
- "showname": "WTAP_ENCAP: 1",
- "size": "0",
- "value": null,
- "show": "1",
- "unmaskedvalue": null,
- "hide": null,
- "fields": null,
- "protos": null
-},
-{
- "name": "frame.time",
- "pos": "0",
- "showname": "Arrival Time: Mar 17, 2016 18:44:29.373968000 UTC",
- "size": "0",
- "value": null,
- "show": "Mar 17, 2016 18:44:29.373968000",
- "unmaskedvalue": null,
- "hide": null,
- "fields": null,
- "protos": null
-},
-{
- "name": "frame.offset_shift",
- "pos": "0",
- "showname": "Time shift for this packet: 0.0 seconds",
- "size": "0",
- "value": null,
- "show": "0.0",
- "unmaskedvalue": null,
- "hide": null,
- "fields": null,
- "protos": null
-},
-{
- "name": "frame.time_epoch",
- "pos": "0",
- "showname": "Epoch Time: 1458240269.373968000 seconds",
- "size": "0",
- "value": null,
- "show": "1458240269.373968000",
- "unmaskedvalue": null,
- "hide": null,
- "fields": null,
- "protos": null
-},
-{
- "name": "frame.time_delta",
- "pos": "0",
- "showname": "Time delta from previous captured frame: 0.0
seconds",
- "size": "0",
- "value": null,
- "show": "0.0",
- "unmaskedvalue": null,
- "hide": null,
- "fields": null,
- "protos": null
-},
-{
- "name": "frame.time_delta_displayed",
- "pos":
[12/50] [abbrv] metron git commit: METRON-1671 Create PCAP UI (tiborm via merrimanr) closes apache/metron#1103
METRON-1671 Create PCAP UI (tiborm via merrimanr) closes apache/metron#1103 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/d5eb56a9 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/d5eb56a9 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/d5eb56a9 Branch: refs/remotes/apache/feature/METRON-1699-create-batch-profiler Commit: d5eb56a967ac7b925d0a843c027a2b0dde66de06 Parents: 7dff4de Author: tiborm Authored: Wed Aug 1 11:31:50 2018 -0500 Committer: rmerriman Committed: Wed Aug 1 11:31:50 2018 -0500 -- .../metron-alerts/package-lock.json | 3719 +- .../metron-alerts/src/app/app-routing.module.ts |3 +- .../metron-alerts/src/app/app.component.html|8 + .../metron-alerts/src/app/app.component.scss| 18 + .../metron-alerts/src/app/app.module.ts |6 +- .../src/app/pcap/model/pcap.request.ts | 29 + .../metron-alerts/src/app/pcap/model/pdml.ts| 50 + .../pcap-filters/pcap-filters.component.html| 52 + .../pcap-filters/pcap-filters.component.scss| 69 + .../pcap-filters/pcap-filters.component.spec.ts | 46 + .../pcap/pcap-filters/pcap-filters.component.ts | 41 + .../app/pcap/pcap-list/pcap-list.component.html | 37 + .../app/pcap/pcap-list/pcap-list.component.scss | 21 + .../pcap/pcap-list/pcap-list.component.spec.ts | 70 + .../app/pcap/pcap-list/pcap-list.component.ts | 39 + .../pcap-packet-line.component.html | 19 + .../pcap-packet-line.component.scss | 17 + .../pcap-packet-line.component.spec.ts | 1283 ++ .../pcap-packet-line.component.ts | 55 + .../pcap/pcap-packet/pcap-packet.component.html | 21 + .../pcap/pcap-packet/pcap-packet.component.scss | 36 + .../pcap-packet/pcap-packet.component.spec.ts | 1283 ++ .../pcap/pcap-packet/pcap-packet.component.ts | 39 + .../pcap/pcap-panel/pcap-panel.component.html | 31 + .../pcap/pcap-panel/pcap-panel.component.scss | 52 + .../pcap-panel/pcap-panel.component.spec.ts | 68 + .../app/pcap/pcap-panel/pcap-panel.component.ts | 75 + .../metron-alerts/src/app/pcap/pcap.module.ts | 50 + .../metron-alerts/src/app/pcap/pcap.routing.ts | 27 + .../src/app/pcap/service/pcap.service.spec.ts | 1752 + .../src/app/pcap/service/pcap.service.ts| 68 + 31 files changed, 7304 insertions(+), 1780 deletions(-) --
[06/50] [abbrv] metron git commit: Merge branch 'master' into feature/METRON-1554-pcap-query-panel
Merge branch 'master' into feature/METRON-1554-pcap-query-panel
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/7dff4def
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/7dff4def
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/7dff4def
Branch: refs/remotes/apache/feature/METRON-1699-create-batch-profiler
Commit: 7dff4def23a7c839ea599a120ab6d4cf5b9bbbef
Parents: 6c90724 ccdbeff
Author: cstella
Authored: Thu Jul 26 12:29:04 2018 -0400
Committer: cstella
Committed: Thu Jul 26 12:29:04 2018 -0400
--
.../docker/rpm-docker/SPECS/metron.spec | 1 +
.../e2e/utils/clean_metron_update_table.ts | 18 ++
.../metron-alerts/e2e/utils/e2e_util.ts | 20 +-
.../alert-filters.component.spec.ts | 18 ++
.../alert-filters/alert-filters.component.ts| 20 +-
.../table-view/table-view.component.spec.ts | 18 ++
.../tree-view/tree-view.component.spec.ts | 18 ++
.../alerts/meta-alerts/meta-alerts.module.ts| 18 ++
.../date-picker/date-picker.component.spec.ts | 18 ++
.../shared/date-picker/date-picker.component.ts | 18 ++
.../shared/date-picker/date-picker.module.ts| 18 ++
.../alert-severity-hexagon.directive.spec.ts| 18 ++
.../app/shared/group-by/group-by.component.ts | 18 ++
.../metron-alerts/src/app/shared/index.ts | 18 ++
.../src/app/shared/pipes/map-keys.pipe.spec.ts | 18 ++
.../src/app/shared/pipes/map-keys.pipe.ts | 18 ++
.../src/app/shared/pipes/time-lapse.pipe.ts | 18 ++
.../time-range/time-range.component.spec.ts | 18 ++
.../app/shared/time-range/time-range.module.ts | 18 ++
metron-interface/metron-alerts/src/polyfills.ts | 18 ++
.../metron-config/e2e/utils/e2e_util.ts | 18 ++
.../src/app/app.config.interface.ts | 18 ++
.../metron-config/src/app/app.config.ts | 18 ++
.../sensor-grok/sensor-grok.component.ts| 18 ++
.../sensor-parser-config-readonly/index.ts | 18 ++
.../app/sensors/sensor-parser-config/index.ts | 18 ++
.../shared/ace-editor/ace-editor.component.ts | 18 ++
.../app/shared/ace-editor/ace-editor.module.ts | 18 ++
.../advanced-config-form.module.ts | 18 ++
.../src/app/shared/metron-modal/index.ts| 18 ++
.../shared/metron-table/metron-sorter/index.ts | 18 ++
.../metron-sorter.component.spec.ts | 18 ++
.../metron-sorter/metron-sorter.component.ts| 18 ++
.../shared/metron-table/metron-table.module.ts | 18 ++
.../src/app/shared/multiple-input/index.ts | 18 ++
.../multiple-input/multiple-input.module.ts | 18 ++
.../src/app/shared/number-spinner/index.ts | 18 ++
.../number-spinner/number-spinner.module.ts | 18 ++
.../shared/sample-data/sample-data.module.ts| 18 ++
.../src/environments/environment.prod.ts| 18 ++
.../src/environments/environment.ts | 18 ++
metron-interface/metron-rest/README.md | 12 +-
.../parsed/jsonMapExampleParsed | 6 +
.../raw/jsonMapExampleOutput| 3 +
metron-platform/metron-parsers/README.md| 5 +
.../zookeeper/parsers/jsonMapWrappedQuery.json | 5 +
.../metron/parsers/json/JSONMapParser.java | 40 +++-
.../JSONMapWrappedQueryIntegrationTest.java | 37
.../integration/ParserIntegrationTest.java | 22 +-
.../json/JSONMapParserWrappedQueryTest.java | 199 +++
pom.xml | 19 +-
51 files changed, 1044 insertions(+), 29 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/7dff4def/metron-deployment/packaging/docker/rpm-docker/SPECS/metron.spec
--
http://git-wip-us.apache.org/repos/asf/metron/blob/7dff4def/metron-interface/metron-rest/README.md
--
diff --cc metron-interface/metron-rest/README.md
index 2d9a535,caa9c78..68ec559
--- a/metron-interface/metron-rest/README.md
+++ b/metron-interface/metron-rest/README.md
@@@ -253,15 -242,8 +253,15 @@@ Request and Response objects are JSON f
| [ `GET /api/v1/metaalert/add/alert`](#get-apiv1metaalertaddalert)|
| [ `GET /api/v1/metaalert/remove/alert`](#get-apiv1metaalertremovealert)|
| [ `GET
/api/v1/metaalert/update/status/{guid}/{status}`](#get-apiv1metaalertupdatestatusguidstatus)|
+| [ `POST /api/v1/pcap/fixed`](#post-apiv1pcapfixed)|
+| [ `POST /api/v1/pcap/query`](#post-apiv1pcapquery)|
+| [ `GET /api/v1/pcap`](#get-apiv1pcap)|
+| [ `GET /api/v1/pcap/{jobId}`](#get-apiv1pcapjobid)|
+| [ `GET /api/v1/pcap/{jobId}/pdml`](#get-apiv1pcapjobidpdml)|
+| [ `GET /api/v1/pcap/{jobId}/raw`](#get-apiv1pcapjobidraw)|
+| [ `GET
[23/50] [abbrv] metron git commit: METRON-1720 Better error messages when there are no results or wireshark is not installed (merrimanr) closes apache/metron#1154
METRON-1720 Better error messages when there are no results or wireshark is not
installed (merrimanr) closes apache/metron#1154
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/afb9607d
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/afb9607d
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/afb9607d
Branch: refs/remotes/apache/feature/METRON-1699-create-batch-profiler
Commit: afb9607dfbbaf89f06a30823bf668523bb7b6f9a
Parents: 7967f35
Author: merrimanr
Authored: Wed Aug 8 14:11:26 2018 -0500
Committer: rmerriman
Committed: Wed Aug 8 14:11:26 2018 -0500
--
.../pcap-panel/pcap-panel.component.spec.ts | 64
.../app/pcap/pcap-panel/pcap-panel.component.ts | 9 ++-
.../src/app/pcap/service/pcap.service.ts| 3 +-
3 files changed, 73 insertions(+), 3 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/afb9607d/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.spec.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.spec.ts
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.spec.ts
index bd732ad..0804b79 100644
---
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.spec.ts
+++
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.spec.ts
@@ -26,6 +26,7 @@ import { PcapPagination } from '../model/pcap-pagination';
import { By } from '../../../../node_modules/@angular/platform-browser';
import { PcapRequest } from '../model/pcap.request';
import { defer } from 'rxjs/observable/defer';
+import {RestError} from "../../model/rest-error";
@Component({
selector: 'app-pcap-filters',
@@ -305,6 +306,7 @@ describe('PcapPanelComponent', () => {
);
const pollResponse = new PcapStatusResponse();
+pollResponse.jobStatus = 'SUCCEEDED';
pcapService.pollStatus = jasmine.createSpy('pollStatus').and.returnValue(
defer(() => Promise.resolve(pollResponse))
);
@@ -321,6 +323,68 @@ describe('PcapPanelComponent', () => {
tick();
fixture.detectChanges();
+expect(component.pdml).toEqual(myPdml);
expect(fixture.debugElement.query(By.css('app-pcap-list'))).toBeDefined();
}));
+
+ it('should handle get packet 404', fakeAsync(() => {
+const searchResponse = new PcapStatusResponse();
+searchResponse.jobId = '42';
+
+pcapService.submitRequest =
jasmine.createSpy('submitRequest').and.returnValue(
+defer(() => Promise.resolve(searchResponse))
+);
+
+const pollResponse = new PcapStatusResponse();
+pollResponse.jobStatus = 'SUCCEEDED';
+pcapService.pollStatus = jasmine.createSpy('pollStatus').and.returnValue(
+defer(() => Promise.resolve(pollResponse))
+);
+
+const restError = new RestError();
+restError.responseCode = 404;
+pcapService.getPackets = jasmine.createSpy('getPackets').and.returnValue(
+defer(() => Promise.reject(restError))
+);
+
+component.onSearch(new PcapRequest());
+
+expect(component.errorMsg).toBeFalsy();
+
+tick();
+fixture.detectChanges();
+
+expect(component.errorMsg).toEqual('No results returned');
+ }));
+
+ it('should handle get packet error', fakeAsync(() => {
+const searchResponse = new PcapStatusResponse();
+searchResponse.jobId = '42';
+
+pcapService.submitRequest =
jasmine.createSpy('submitRequest').and.returnValue(
+defer(() => Promise.resolve(searchResponse))
+);
+
+const pollResponse = new PcapStatusResponse();
+pollResponse.jobStatus = 'SUCCEEDED';
+pcapService.pollStatus = jasmine.createSpy('pollStatus').and.returnValue(
+defer(() => Promise.resolve(pollResponse))
+);
+
+const restError = new RestError();
+restError.responseCode = 500;
+restError.message = 'error message';
+pcapService.getPackets = jasmine.createSpy('getPackets').and.returnValue(
+defer(() => Promise.reject(restError))
+);
+
+component.onSearch(new PcapRequest());
+
+expect(component.errorMsg).toBeFalsy();
+
+tick();
+fixture.detectChanges();
+
+expect(component.errorMsg).toEqual('Response message: error message.
Something went wrong retrieving pdml results!');
+ }));
});
http://git-wip-us.apache.org/repos/asf/metron/blob/afb9607d/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.ts
b/metron-interface/metron-alerts/src/app/pcap/pcap-panel/pcap-panel.component.ts
index 82121d4..8e4ced0
[21/50] [abbrv] metron git commit: METRON-1683 PCAP UI - Fix the download progress bar (sardell via merrimanr) closes apache/metron#1122
METRON-1683 PCAP UI - Fix the download progress bar (sardell via merrimanr)
closes apache/metron#1122
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/3e778592
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/3e778592
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/3e778592
Branch: refs/remotes/apache/feature/METRON-1699-create-batch-profiler
Commit: 3e7785920e8f0c5aadf36e05dbcfa69252cc5574
Parents: 037b50b
Author: sardell
Authored: Tue Aug 7 13:58:08 2018 -0500
Committer: rmerriman
Committed: Tue Aug 7 13:58:08 2018 -0500
--
.../metron-alerts/src/app/app.component.html|2 +-
.../metron-alerts/src/app/app.component.ts |3 +
.../src/app/pcap/model/pcap-status-response.ts |1 +
.../src/app/pcap/model/pcap.mock.ts | 39 +
.../src/app/pcap/model/pdml.mock.ts | 62 +
.../metron-alerts/src/app/pcap/model/pdml.ts|6 +-
.../app/pcap/pcap-list/pcap-list.component.html |2 +-
.../pcap/pcap-list/pcap-list.component.spec.ts | 17 +
.../pcap-packet-line.component.spec.ts | 1336 +
.../pcap/pcap-packet/pcap-packet.component.html | 11 +-
.../pcap-packet/pcap-packet.component.spec.ts | 1278 +
.../pcap/pcap-panel/pcap-panel.component.html |6 +-
.../pcap/pcap-panel/pcap-panel.component.scss | 18 -
.../pcap-panel/pcap-panel.component.spec.ts | 262 ++-
.../app/pcap/pcap-panel/pcap-panel.component.ts | 52 +-
.../src/app/pcap/service/pcap.service.spec.ts | 1809 +-
.../src/app/pcap/service/pcap.service.ts|7 +-
.../src/environments/environment.e2e.ts |3 +-
.../src/environments/environment.prod.ts|3 +-
.../src/environments/environment.ts |3 +-
metron-interface/metron-alerts/src/index.html |2 +-
21 files changed, 674 insertions(+), 4248 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/3e778592/metron-interface/metron-alerts/src/app/app.component.html
--
diff --git a/metron-interface/metron-alerts/src/app/app.component.html
b/metron-interface/metron-alerts/src/app/app.component.html
index 1b992ca..c3a5d3c 100644
--- a/metron-interface/metron-alerts/src/app/app.component.html
+++ b/metron-interface/metron-alerts/src/app/app.component.html
@@ -11,7 +11,7 @@
OR CONDITIONS OF ANY KIND, either express or implied. See the License
for
the specific language governing permissions and limitations under the
License.
-->
-
+
http://git-wip-us.apache.org/repos/asf/metron/blob/3e778592/metron-interface/metron-alerts/src/app/app.component.ts
--
diff --git a/metron-interface/metron-alerts/src/app/app.component.ts
b/metron-interface/metron-alerts/src/app/app.component.ts
index 2a3c15e..52dee23 100644
--- a/metron-interface/metron-alerts/src/app/app.component.ts
+++ b/metron-interface/metron-alerts/src/app/app.component.ts
@@ -17,6 +17,7 @@
*/
import { Component, OnInit } from '@angular/core';
import {AuthenticationService} from './service/authentication.service';
+import { environment } from 'environments/environment';
declare var $;
@@ -27,11 +28,13 @@ declare var $;
})
export class AppComponent implements OnInit {
loggedIn = false;
+ noTransition = false;
constructor(private authService: AuthenticationService) {
this.authService.onLoginEvent.subscribe(result => {
this.loggedIn = result;
});
+this.noTransition = environment.noTransition;
}
ngOnInit(): void {
http://git-wip-us.apache.org/repos/asf/metron/blob/3e778592/metron-interface/metron-alerts/src/app/pcap/model/pcap-status-response.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/model/pcap-status-response.ts
b/metron-interface/metron-alerts/src/app/pcap/model/pcap-status-response.ts
index d4d9a5e..09c4985 100644
--- a/metron-interface/metron-alerts/src/app/pcap/model/pcap-status-response.ts
+++ b/metron-interface/metron-alerts/src/app/pcap/model/pcap-status-response.ts
@@ -18,6 +18,7 @@
export class PcapStatusResponse {
jobId: string;
jobStatus: string;
+ description: string;
percentComplete: number;
pageTotal: number;
}
http://git-wip-us.apache.org/repos/asf/metron/blob/3e778592/metron-interface/metron-alerts/src/app/pcap/model/pcap.mock.ts
--
diff --git a/metron-interface/metron-alerts/src/app/pcap/model/pcap.mock.ts
b/metron-interface/metron-alerts/src/app/pcap/model/pcap.mock.ts
new file mode 100644
index 000..c867fe9
--- /dev/null
+++
[33/50] [abbrv] metron git commit: METRON-1725 Add ability to specify YARN queue for pcap jobs (merrimanr) closes apache/metron#1153
METRON-1725 Add ability to specify YARN queue for pcap jobs (merrimanr) closes apache/metron#1153 Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/7a8c2467 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/7a8c2467 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/7a8c2467 Branch: refs/remotes/apache/feature/METRON-1699-create-batch-profiler Commit: 7a8c246748a2c9e8c5b9230800b075dd99a7f3a4 Parents: 73dc63e Author: merrimanr Authored: Fri Aug 10 16:46:31 2018 -0500 Committer: rmerriman Committed: Fri Aug 10 16:46:31 2018 -0500 -- .../CURRENT/configuration/metron-rest-env.xml | 9 ++ .../package/scripts/params/params_linux.py | 1 + .../METRON/CURRENT/package/templates/metron.j2 | 1 + .../METRON/CURRENT/themes/metron_theme.json | 10 ++ metron-interface/metron-rest/README.md | 2 ++ .../src/main/config/rest_application.yml| 1 + .../apache/metron/rest/MetronRestConstants.java | 1 + .../metron/rest/config/PcapJobSupplier.java | 2 +- .../rest/service/impl/PcapServiceImpl.java | 12 ++- .../apache/metron/rest/mock/MockPcapJob.java| 8 + .../rest/service/impl/PcapServiceImplTest.java | 7 - metron-platform/metron-pcap-backend/README.md | 2 ++ .../org/apache/metron/pcap/query/CliParser.java | 4 +++ .../org/apache/metron/pcap/query/PcapCli.java | 3 ++ .../apache/metron/pcap/query/PcapCliTest.java | 33 ++-- .../apache/metron/pcap/config/PcapConfig.java | 10 ++ 16 files changed, 101 insertions(+), 5 deletions(-) -- http://git-wip-us.apache.org/repos/asf/metron/blob/7a8c2467/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml -- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml index 20f9767..895c091 100644 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml @@ -192,5 +192,14 @@ The number of pcaps written to a page/file as a result of a pcap query. 10 + +pcap_yarn_queue +Pcap YARN Queue +The YARN queue pcap jobs will be submitted to. + + +true + + http://git-wip-us.apache.org/repos/asf/metron/blob/7a8c2467/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py -- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py index 73d3469..4f8a9a7 100755 --- a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py +++ b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py @@ -390,6 +390,7 @@ pcap_base_path = config['configurations']['metron-rest-env']['pcap_base_path'] pcap_base_interim_result_path = config['configurations']['metron-rest-env']['pcap_base_interim_result_path'] pcap_final_output_path = config['configurations']['metron-rest-env']['pcap_final_output_path'] pcap_page_size = config['configurations']['metron-rest-env']['pcap_page_size'] +pcap_yarn_queue = config['configurations']['metron-rest-env']['pcap_yarn_queue'] pcap_configured_flag_file = status_params.pcap_configured_flag_file # MapReduce http://git-wip-us.apache.org/repos/asf/metron/blob/7a8c2467/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 -- diff --git a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/templates/metron.j2 index 278d6f8..55422d0 100644 ---
[38/50] [abbrv] metron git commit: METRON-1737: Document Job cleanup (merrimanr via mmiklavc) closes apache/metron#1164
METRON-1737: Document Job cleanup (merrimanr via mmiklavc) closes
apache/metron#1164
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/6b70571d
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/6b70571d
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/6b70571d
Branch: refs/remotes/apache/feature/METRON-1699-create-batch-profiler
Commit: 6b70571d6de3951c98269bbf5b38e8b69deddfab
Parents: d9e1f38
Author: merrimanr
Authored: Wed Aug 15 16:00:13 2018 -0600
Committer: Michael Miklavcic
Committed: Wed Aug 15 16:00:13 2018 -0600
--
metron-interface/metron-rest/README.md | 11 +++
1 file changed, 11 insertions(+)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/6b70571d/metron-interface/metron-rest/README.md
--
diff --git a/metron-interface/metron-rest/README.md
b/metron-interface/metron-rest/README.md
index 080422d..2c216d1 100644
--- a/metron-interface/metron-rest/README.md
+++ b/metron-interface/metron-rest/README.md
@@ -222,6 +222,17 @@ Out of the box it is a simple wrapper around the tshark
command to transform raw
REST will supply the script with raw pcap data through standard in and expects
PDML data serialized as XML.
Pcap query jobs can be configured for submission to a YARN queue. This
setting is exposed as the Spring property `pcap.yarn.queue`. If configured,
the REST application will set the `mapreduce.job.queuename` Hadoop property to
that value.
+It is highly recommended that a dedicated YARN queue be created and configured
for Pcap queries to prevent a job from consuming too many cluster resources.
More information about setting up YARN queues can be found
[here](https://hadoop.apache.org/docs/current/hadoop-yarn/hadoop-yarn-site/CapacityScheduler.html#Setting_up_queues).
+
+Pcap query results are stored in HDFS. The location of query results when run
through the REST app is determined by a couple factors. The root of Pcap query
results defaults to `/apps/metron/pcap/output` but can be changed with the
+Spring property `pcap.final.output.path`. Assuming the default Pcap query
output directory, the path to a result page will follow this pattern:
+```
+/apps/metron/pcap/output/{username}/MAP_REDUCE/{job id}/page-{page number}.pcap
+```
+Over time Pcap query results will accumulate in HDFS. Currently these results
are not cleaned up automatically so cluster administrators should be aware of
this and monitor them. It is highly recommended that a process be put in place
to
+periodically delete files and directories under the Pcap query results root.
+
+Users should also be mindful of date ranges used in queries so they don't
produce result sets that are too large. Currently there are no limits enforced
on date ranges.
Queries can also be configured on a global level for setting the number of
results per page via a Spring property `pcap.page.size`. By default, this value
is set to 10 pcaps per page, but you may choose to set this value higher
based on observing frequenetly-run query result sizes. This setting works in
conjunction with the property for setting finalizer threadpool size when
optimizing query performance.
[01/50] [abbrv] metron git commit: METRON-1560 Update MPack to support Pcap panel (merrimanr) closes apache/metron#1124
Repository: metron
Updated Branches:
refs/remotes/apache/feature/METRON-1699-create-batch-profiler 113ac0d3a ->
f5eaef3c2
METRON-1560 Update MPack to support Pcap panel (merrimanr) closes
apache/metron#1124
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/e78bc530
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/e78bc530
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/e78bc530
Branch: refs/remotes/apache/feature/METRON-1699-create-batch-profiler
Commit: e78bc530260aa622e8263cda2b94be99194dffb5
Parents: 8763fba
Author: merrimanr
Authored: Mon Jul 23 09:08:12 2018 -0500
Committer: rmerriman
Committed: Mon Jul 23 09:08:12 2018 -0500
--
.../CURRENT/configuration/metron-rest-env.xml | 25
.../package/scripts/params/params_linux.py | 5 +++-
.../CURRENT/package/scripts/rest_commands.py| 18 --
.../METRON/CURRENT/package/templates/metron.j2 | 4
.../METRON/CURRENT/themes/metron_theme.json | 10
.../src/main/config/rest_application.yml| 6 -
.../src/main/config/pcap.properties | 2 +-
7 files changed, 65 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/e78bc530/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
--
diff --git
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
index 066b527..20f9767 100644
---
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
+++
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/configuration/metron-rest-env.xml
@@ -168,4 +168,29 @@
The field name where the threat triage score can be found
in the search indices. This setting primarily affects the Alerts
UI.
threat:triage:score
+
+pcap_base_path
+Pcap Base Path
+The HDFS path where pcap files are stored. These files
are the input to pcap queries.
+/apps/metron/pcap/input
+
+
+pcap_base_interim_result_path
+Pcap Base Interim Result Path
+The HDFS path where interim results of a pcap query are
stored, before they are separated into pages.
+/apps/metron/pcap/interim
+
+
+pcap_final_output_path
+Pcap Final Output Path
+The HDFS path where the final results of a pcap query are
stored.
+/apps/metron/pcap/output
+
+
+pcap_page_size
+Pcap Page Size
+The number of pcaps written to a page/file as a result of
a pcap query.
+10
+
+
http://git-wip-us.apache.org/repos/asf/metron/blob/e78bc530/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
--
diff --git
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
index b8d6123..73d3469 100755
---
a/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
+++
b/metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/scripts/params/params_linux.py
@@ -386,7 +386,10 @@ bolt_hdfs_rotation_policy_units =
config['configurations']['metron-indexing-env'
bolt_hdfs_rotation_policy_count =
config['configurations']['metron-indexing-env']['bolt_hdfs_rotation_policy_count']
# Pcap
-pcap_hdfs_dir = format("{metron_apps_hdfs_dir}/pcap")
+pcap_base_path = config['configurations']['metron-rest-env']['pcap_base_path']
+pcap_base_interim_result_path =
config['configurations']['metron-rest-env']['pcap_base_interim_result_path']
+pcap_final_output_path =
config['configurations']['metron-rest-env']['pcap_final_output_path']
+pcap_page_size = config['configurations']['metron-rest-env']['pcap_page_size']
pcap_configured_flag_file = status_params.pcap_configured_flag_file
# MapReduce
[04/50] [abbrv] metron git commit: METRON-1693: Fix Pcap CLI local FS finalizer (mmiklavc via mmiklavc) closes apache/metron#1130
METRON-1693: Fix Pcap CLI local FS finalizer (mmiklavc via mmiklavc) closes
apache/metron#1130
Project: http://git-wip-us.apache.org/repos/asf/metron/repo
Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/f1f5dda4
Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/f1f5dda4
Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/f1f5dda4
Branch: refs/remotes/apache/feature/METRON-1699-create-batch-profiler
Commit: f1f5dda4c643394c12ab9f4501c5b806adb21129
Parents: 9d4842f
Author: mmiklavc
Authored: Wed Jul 25 08:26:45 2018 -0600
Committer: Michael Miklavcic
Committed: Wed Jul 25 08:26:45 2018 -0600
--
.../java/org/apache/metron/pcap/query/PcapCli.java | 1 -
.../metron/pcap/finalizer/PcapCliFinalizer.java | 15 ---
.../apache/metron/pcap/finalizer/PcapFinalizer.java | 4 +++-
.../metron/pcap/finalizer/PcapRestFinalizer.java| 10 ++
.../metron/pcap/writer/PcapResultsWriter.java | 16
5 files changed, 41 insertions(+), 5 deletions(-)
--
http://git-wip-us.apache.org/repos/asf/metron/blob/f1f5dda4/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java
--
diff --git
a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java
b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java
index 1a23740..0b06b0c 100644
---
a/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java
+++
b/metron-platform/metron-pcap-backend/src/main/java/org/apache/metron/pcap/query/PcapCli.java
@@ -87,7 +87,6 @@ public class PcapCli {
try {
config = fixedParser.parse(otherArgs);
commonConfig = config;
-PcapOptions.FINAL_OUTPUT_PATH.put(commonConfig, new Path("file:///" +
execDir));
} catch (ParseException | java.text.ParseException e) {
System.err.println(e.getMessage());
System.err.flush();
http://git-wip-us.apache.org/repos/asf/metron/blob/f1f5dda4/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapCliFinalizer.java
--
diff --git
a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapCliFinalizer.java
b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapCliFinalizer.java
index c379515..c912e58 100644
---
a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapCliFinalizer.java
+++
b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapCliFinalizer.java
@@ -18,9 +18,13 @@
package org.apache.metron.pcap.finalizer;
+import java.io.IOException;
+import java.util.List;
import java.util.Map;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.metron.pcap.config.PcapOptions;
+import org.apache.metron.pcap.writer.PcapResultsWriter;
/**
* Write to local FS.
@@ -33,13 +37,18 @@ public class PcapCliFinalizer extends PcapFinalizer {
* as a formatted timestamp + uuid. A final sample format will look as
follows:
*
/base/output/path/pcap-data-201807181911-09855b4ae3204dee8b63760d65198da3+0001.pcap
*/
- private static final String PCAP_CLI_FILENAME_FORMAT =
"%s/pcap-data-%s+%04d.pcap";
+ private static final String PCAP_CLI_FILENAME_FORMAT =
"pcap-data-%s+%04d.pcap";
+
+ @Override
+ protected void write(PcapResultsWriter resultsWriter, Configuration
hadoopConfig,
+ List data, Path outputPath) throws IOException {
+resultsWriter.writeLocal(data, outputPath.toString());
+ }
@Override
protected Path getOutputPath(Map config, int partition) {
-Path finalOutputPath = PcapOptions.FINAL_OUTPUT_PATH.get(config,
PcapOptions.STRING_TO_PATH, Path.class);
String prefix = PcapOptions.FINAL_FILENAME_PREFIX.get(config,
String.class);
-return new Path(String.format(PCAP_CLI_FILENAME_FORMAT, finalOutputPath,
prefix, partition));
+return new Path(String.format(PCAP_CLI_FILENAME_FORMAT, prefix,
partition));
}
}
http://git-wip-us.apache.org/repos/asf/metron/blob/f1f5dda4/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapFinalizer.java
--
diff --git
a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapFinalizer.java
b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapFinalizer.java
index 2c55e15..8dcc401 100644
---
a/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapFinalizer.java
+++
b/metron-platform/metron-pcap/src/main/java/org/apache/metron/pcap/finalizer/PcapFinalizer.java
[08/50] [abbrv] metron git commit: METRON-1671 Create PCAP UI (tiborm via merrimanr) closes apache/metron#1103
http://git-wip-us.apache.org/repos/asf/metron/blob/d5eb56a9/metron-interface/metron-alerts/src/app/pcap/service/pcap.service.spec.ts
--
diff --git
a/metron-interface/metron-alerts/src/app/pcap/service/pcap.service.spec.ts
b/metron-interface/metron-alerts/src/app/pcap/service/pcap.service.spec.ts
new file mode 100644
index 000..244a3ea
--- /dev/null
+++ b/metron-interface/metron-alerts/src/app/pcap/service/pcap.service.spec.ts
@@ -0,0 +1,1752 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+import { TestBed, async, inject } from '@angular/core/testing';
+import {
+ BaseRequestOptions,
+ HttpModule,
+ Http,
+ Response,
+ ResponseOptions
+} from '@angular/http';
+import { MockBackend } from '@angular/http/testing';
+
+import { PcapService } from './pcap.service';
+import { PcapRequest } from '../model/pcap.request';
+
+describe('PcapService', () => {
+ beforeEach(() => {
+
+TestBed.configureTestingModule({
+ imports: [HttpModule],
+ providers: [
+PcapService,
+{
+ provide: Http,
+ useFactory: (mockBackend, options) => {
+return new Http(mockBackend, options);
+ },
+ deps: [MockBackend, BaseRequestOptions]
+},
+MockBackend,
+BaseRequestOptions
+ ]
+});
+ });
+ describe('getPackets()', () => {
+it('should return an Observable',
+ inject([PcapService, MockBackend], (pcapService, mockBackend) => {
+
+let request: PcapRequest = {
+ startTimeMs: 0,
+ endTimeMs: 0,
+ ipSrcAddr: '0.0.0.0',
+ ipSrcPort: '80',
+ ipDstAddr: '0.0.0.0',
+ ipDstPort: '80',
+ protocol: '*',
+ packetFilter: '*',
+ includeReverse: false,
+};
+
+mockBackend.connections.subscribe((connection) => {
+ connection.mockRespond(new Response(new ResponseOptions({body:
pdml_json()})));
+});
+let packets;
+pcapService.getPackets(request).subscribe(r => packets = r)
+expect(packets).toBeTruthy()
+expect(packets.pdml).toBeTruthy()
+expect(packets.pdml.packet.length).toBe(1)
+expect(packets.pdml.packet[0].proto.length).toBeGreaterThan(3)
+
+console.log(packets)
+ }))
+
+
+it('should ...', inject([PcapService], (service: PcapService) => {
+ expect(service).toBeTruthy();
+}));
+
+ })
+
+});
+
+
+
+
+function pdml_json() {
+ return `{
+ "pdml": {
+"$": {
+ "version": "0",
+ "creator": "wireshark/2.4.2",
+ "time": "Tue Mar 27 21:55:25 2018",
+ "capture_file":
"./metron-platform/metron-api/src/test/resources/test-tcp-packet.pcap"
+},
+"packet": [
+ {
+"proto": [
+ {
+"$": {
+ "name": "geninfo",
+ "pos": "0",
+ "showname": "General information",
+ "size": "104"
+},
+"field": [
+ {
+"$": {
+ "name": "num",
+ "pos": "0",
+ "show": "1",
+ "showname": "Number",
+ "value": "1",
+ "size": "104"
+}
+ },
+ {
+"$": {
+ "name": "len",
+ "pos": "0",
+ "show": "104",
+ "showname": "Frame Length",
+ "value": "68",
+ "size": "104"
+}
+ },
+ {
+"$": {
+ "name": "caplen",
+ "pos": "0",
+ "show": "104",
+ "showname": "Captured Length",
+ "value": "68",
+ "size": "104"
+}
+ },
+ {
+"$": {
+ "name": "timestamp",
+ "pos": "0",
+ "show": "Mar 26, 2014 19:59:40.024362000 GMT",
+ "showname": "Captured Time",
+ "value": "1395863980.024362000",
+ "size": "104"
+}
+
