Repository: metron Updated Branches: refs/heads/feature/METRON-1344-test-infrastructure 2e78df67c -> ebc325865
METRON-1344: Externalize the infrastructural components using integration tests Project: http://git-wip-us.apache.org/repos/asf/metron/repo Commit: http://git-wip-us.apache.org/repos/asf/metron/commit/ebc32586 Tree: http://git-wip-us.apache.org/repos/asf/metron/tree/ebc32586 Diff: http://git-wip-us.apache.org/repos/asf/metron/diff/ebc32586 Branch: refs/heads/feature/METRON-1344-test-infrastructure Commit: ebc32586548e0bc272559166f0d5eb5a076e806e Parents: 2e78df6 Author: merrimanr <[email protected]> Authored: Thu Feb 1 09:35:47 2018 -0600 Committer: merrimanr <[email protected]> Committed: Thu Feb 1 09:35:47 2018 -0600 ---------------------------------------------------------------------- .travis.yml | 28 ++- metron-contrib/metron-docker-e2e/.gitignore | 3 + .../compose/docker-compose.yml | 55 +++++ .../compose/metron-centos/Dockerfile | 19 ++ .../compose/metron-rest/Dockerfile | 34 +++ .../compose/metron-rest/bin/start.sh | 23 ++ .../metron-rest/config/application-docker.yml | 62 ++++++ .../metron-rest/config/zookeeper/global.json | 8 + .../compose/metron-ui/Dockerfile | 34 +++ .../compose/metron-ui/bin/start.sh | 20 ++ .../compose/metron-ui/config/alerts_ui.yml | 21 ++ metron-contrib/metron-docker-e2e/conf/.env | 3 + metron-contrib/metron-docker-e2e/pom.xml | 217 +++++++++++++++++++ .../metron-docker-e2e/scripts/wait_for_rest.sh | 29 +++ metron-contrib/pom.xml | 1 + .../configure-table/configure-table.e2e-spec.ts | 25 +-- .../metron-alerts/e2e/login/login.e2e-spec.ts | 3 +- .../metron-alerts/e2e/login/login.po.ts | 2 +- .../metron-alerts/e2e/utils/e2e_util.ts | 48 +++- metron-interface/metron-alerts/pom.xml | 46 ++++ .../metron-alerts/protractor.conf.js | 21 +- .../ElasticsearchSearchIntegrationTest.java | 106 +++++---- .../indexing/dao/SearchIntegrationTest.java | 74 ++++--- 23 files changed, 750 insertions(+), 132 deletions(-) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/metron/blob/ebc32586/.travis.yml ---------------------------------------------------------------------- diff --git a/.travis.yml b/.travis.yml index f5edfb2..5bf4587 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,26 +17,49 @@ dist: trusty sudo: required +services: + - docker + env: - - TEST_TYPE=default + global: + - TEST_TYPE=default + - DOCKER_CACHE=~/docker-cache + - DOCKER_METRON_CENTOS=$DOCKER_CACHE/metron_centos.tar.gz + - E2E_COMPOSE_HOME=metron-contrib/metron-docker-e2e/compose + - DOCKER_COMPOSE_VERSION=1.8.0 install: true language: java jdk: - oraclejdk8 before_install: + - sudo rm /usr/local/bin/docker-compose + - curl -L https://github.com/docker/compose/releases/download/${DOCKER_COMPOSE_VERSION}/docker-compose-`uname -s`-`uname -m` > docker-compose + - chmod +x docker-compose + - sudo mv docker-compose /usr/local/bin - wget https://archive.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.zip - unzip -qq apache-maven-3.3.9-bin.zip - export M2_HOME=$PWD/apache-maven-3.3.9 - export PATH=$M2_HOME/bin:$PATH - npm config set cache $HOME/.npm-cache --global - npm config set prefix $HOME/.npm-prefix --global + - if [ -f ${DOCKER_METRON_CENTOS} ]; then gunzip -c ${DOCKER_METRON_CENTOS} | docker load; else docker build metron-contrib/metron-docker-e2e/compose/metron-centos/ -t "metron-centos"; fi install: - time mvn -q -T 2C -DskipTests clean install + - cd $E2E_COMPOSE_HOME && docker-compose up -d + - if [ ! -f ${DOCKER_METRON_CENTOS} ]; then docker save metron-centos | gzip > ${DOCKER_METRON_CENTOS}; fi + - cd $TRAVIS_BUILD_DIR script: - - time mvn -q -T 2C surefire:test@unit-tests && time mvn -q surefire:test@integration-tests && time mvn -q test --projects metron-interface/metron-config && time build_utils/verify_licenses.sh + - ls -la + - metron-contrib/metron-docker-e2e/scripts/wait_for_rest.sh localhost 8082 + - cd $E2E_COMPOSE_HOME && docker-compose logs metron-rest && cd ../../../ +# - mvn -q -T 2C surefire:test@unit-tests +# - mvn -q surefire:test@integration-tests +# - mvn -q test --projects metron-interface/metron-config + - mvn test -Pe2e --projects=metron-interface/metron-alerts +# - build_utils/verify_licenses.sh before_cache: - rm -rf $HOME/.m2/repository/org/apache/metron @@ -49,3 +72,4 @@ cache: - metron-interface/metron-config/node_modules - $HOME/.m2 - $HOME/.npm + - $DOCKER_CACHE http://git-wip-us.apache.org/repos/asf/metron/blob/ebc32586/metron-contrib/metron-docker-e2e/.gitignore ---------------------------------------------------------------------- diff --git a/metron-contrib/metron-docker-e2e/.gitignore b/metron-contrib/metron-docker-e2e/.gitignore new file mode 100644 index 0000000..61f4250 --- /dev/null +++ b/metron-contrib/metron-docker-e2e/.gitignore @@ -0,0 +1,3 @@ +/compose/.env +/compose/metron-rest/packages/ +/compose/metron-ui/packages/ http://git-wip-us.apache.org/repos/asf/metron/blob/ebc32586/metron-contrib/metron-docker-e2e/compose/docker-compose.yml ---------------------------------------------------------------------- diff --git a/metron-contrib/metron-docker-e2e/compose/docker-compose.yml b/metron-contrib/metron-docker-e2e/compose/docker-compose.yml new file mode 100644 index 0000000..e5a8134 --- /dev/null +++ b/metron-contrib/metron-docker-e2e/compose/docker-compose.yml @@ -0,0 +1,55 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +version: '2' +services: + zookeeper: + image: zookeeper + ports: + - "2181:2181" + kafka: + image: wurstmeister/kafka:0.10.0.0 + environment: + KAFKA_ADVERTISED_HOST_NAME: localhost + KAFKA_ADVERTISED_PORT: 9092 + KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181 + ports: + - "9092:9092" + depends_on: + - zookeeper + elasticsearch: + image: elasticsearch:2.3 + ports: + - "9210:9200" + - "9310:9300" + metron-rest: + build: + context: ./metron-rest + args: + METRON_VERSION: $METRON_VERSION + ports: + - "8082:8082" + depends_on: + - kafka + - zookeeper + - elasticsearch + metron-ui: + build: + context: ./metron-ui + args: + METRON_VERSION: $METRON_VERSION + ports: + - "4201:4201" http://git-wip-us.apache.org/repos/asf/metron/blob/ebc32586/metron-contrib/metron-docker-e2e/compose/metron-centos/Dockerfile ---------------------------------------------------------------------- diff --git a/metron-contrib/metron-docker-e2e/compose/metron-centos/Dockerfile b/metron-contrib/metron-docker-e2e/compose/metron-centos/Dockerfile new file mode 100644 index 0000000..8acacc2 --- /dev/null +++ b/metron-contrib/metron-docker-e2e/compose/metron-centos/Dockerfile @@ -0,0 +1,19 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +FROM centos + +RUN yum install -y java-1.8.0-openjdk lsof http://git-wip-us.apache.org/repos/asf/metron/blob/ebc32586/metron-contrib/metron-docker-e2e/compose/metron-rest/Dockerfile ---------------------------------------------------------------------- diff --git a/metron-contrib/metron-docker-e2e/compose/metron-rest/Dockerfile b/metron-contrib/metron-docker-e2e/compose/metron-rest/Dockerfile new file mode 100644 index 0000000..a76cf57 --- /dev/null +++ b/metron-contrib/metron-docker-e2e/compose/metron-rest/Dockerfile @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +FROM metron-centos + +ARG METRON_VERSION + +ENV METRON_VERSION $METRON_VERSION +ENV METRON_HOME /usr/metron/$METRON_VERSION +ENV METRON_REST_PORT 8082 + +RUN mkdir -p $METRON_HOME +ADD ./bin $METRON_HOME/bin +COPY ./packages/* /packages/ +RUN find /packages -type f -name '*.tar.gz' -exec tar -xzf {} -C /usr/metron/$METRON_VERSION/ \; +ADD ./config $METRON_HOME/config + +EXPOSE 8082 8082 + +WORKDIR $METRON_HOME +CMD ./bin/start.sh http://git-wip-us.apache.org/repos/asf/metron/blob/ebc32586/metron-contrib/metron-docker-e2e/compose/metron-rest/bin/start.sh ---------------------------------------------------------------------- diff --git a/metron-contrib/metron-docker-e2e/compose/metron-rest/bin/start.sh b/metron-contrib/metron-docker-e2e/compose/metron-rest/bin/start.sh new file mode 100755 index 0000000..f307e16 --- /dev/null +++ b/metron-contrib/metron-docker-e2e/compose/metron-rest/bin/start.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +$METRON_HOME/bin/zk_load_configs.sh -z zookeeper:2181 -m PUSH -i $METRON_HOME/config/zookeeper + +METRON_REST_CLASSPATH="$METRON_HOME/lib/metron-rest-$METRON_VERSION.jar:$METRON_HOME/lib/metron-elasticsearch-$METRON_VERSION-uber.jar" + +java -cp $METRON_REST_CLASSPATH org.apache.metron.rest.MetronRestApplication --spring.config.location=$METRON_HOME/config/application-docker.yml --spring.profiles.active=dev --server.port=8082 http://git-wip-us.apache.org/repos/asf/metron/blob/ebc32586/metron-contrib/metron-docker-e2e/compose/metron-rest/config/application-docker.yml ---------------------------------------------------------------------- diff --git a/metron-contrib/metron-docker-e2e/compose/metron-rest/config/application-docker.yml b/metron-contrib/metron-docker-e2e/compose/metron-rest/config/application-docker.yml new file mode 100644 index 0000000..ec32cc3 --- /dev/null +++ b/metron-contrib/metron-docker-e2e/compose/metron-rest/config/application-docker.yml @@ -0,0 +1,62 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +spring: + datasource: + driverClassName: org.h2.Driver + url: jdbc:h2:file:./metrondb + username: root + password: root + platform: h2 + jpa: + hibernate: + ddl-auto: update + +zookeeper: + url: zookeeper:2181 + +kafka: + broker: + url: kafka:9092 + security: + protocol: PLAINTEXT + +hdfs: + namenode: + url: file:/// + +grok: + path: + temp: target/patterns/temp + +storm: + ui: + url: storm:8080 + parser: + script.path: /usr/metron/${metron.version}/bin/start_parser_topology.sh + topology.options: ${PARSER_TOPOLOGY_OPTIONS} + enrichment: + script.path: /usr/metron/${metron.version}/bin/start_enrichment_topology.sh + indexing: + script.path: /usr/metron/${metron.version}/bin/start_elasticsearch_topology.sh + +kerberos: + enabled: false + principal: [email protected] + keytab: /etc/security/keytabs/metron.headless.keytab + +index: + dao: + impl: org.apache.metron.elasticsearch.dao.ElasticsearchDao \ No newline at end of file http://git-wip-us.apache.org/repos/asf/metron/blob/ebc32586/metron-contrib/metron-docker-e2e/compose/metron-rest/config/zookeeper/global.json ---------------------------------------------------------------------- diff --git a/metron-contrib/metron-docker-e2e/compose/metron-rest/config/zookeeper/global.json b/metron-contrib/metron-docker-e2e/compose/metron-rest/config/zookeeper/global.json new file mode 100644 index 0000000..cac52ce --- /dev/null +++ b/metron-contrib/metron-docker-e2e/compose/metron-rest/config/zookeeper/global.json @@ -0,0 +1,8 @@ +{ + "es.clustername": "elasticsearch", + "es.ip": "elasticsearch", + "es.port": "9300", + "es.date.format": "yyyy.MM.dd.HH", + "update.hbase.table": "metron_update", + "update.hbase.cf": "t" +} http://git-wip-us.apache.org/repos/asf/metron/blob/ebc32586/metron-contrib/metron-docker-e2e/compose/metron-ui/Dockerfile ---------------------------------------------------------------------- diff --git a/metron-contrib/metron-docker-e2e/compose/metron-ui/Dockerfile b/metron-contrib/metron-docker-e2e/compose/metron-ui/Dockerfile new file mode 100644 index 0000000..fa83cbc --- /dev/null +++ b/metron-contrib/metron-docker-e2e/compose/metron-ui/Dockerfile @@ -0,0 +1,34 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +FROM node:carbon + +ARG METRON_VERSION + +ENV METRON_VERSION $METRON_VERSION +ENV METRON_HOME /usr/metron/$METRON_VERSION + +RUN mkdir -p $METRON_HOME +ADD ./bin $METRON_HOME/bin +COPY ./packages/* /packages/ +RUN find /packages -type f -name '*.tar.gz' -exec tar -xzf {} -C /usr/metron/$METRON_VERSION/ \; +ADD ./config $METRON_HOME/config +RUN npm install --prefix="$METRON_HOME/web/expressjs" --only=production + +EXPOSE 4201 4201 + +WORKDIR $METRON_HOME +CMD ./bin/start.sh http://git-wip-us.apache.org/repos/asf/metron/blob/ebc32586/metron-contrib/metron-docker-e2e/compose/metron-ui/bin/start.sh ---------------------------------------------------------------------- diff --git a/metron-contrib/metron-docker-e2e/compose/metron-ui/bin/start.sh b/metron-contrib/metron-docker-e2e/compose/metron-ui/bin/start.sh new file mode 100755 index 0000000..e64f534 --- /dev/null +++ b/metron-contrib/metron-docker-e2e/compose/metron-ui/bin/start.sh @@ -0,0 +1,20 @@ +#!/usr/bin/env bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +node $METRON_HOME/web/expressjs/alerts-server.js -c $METRON_HOME/config/alerts_ui.yml +tail -f /dev/null http://git-wip-us.apache.org/repos/asf/metron/blob/ebc32586/metron-contrib/metron-docker-e2e/compose/metron-ui/config/alerts_ui.yml ---------------------------------------------------------------------- diff --git a/metron-contrib/metron-docker-e2e/compose/metron-ui/config/alerts_ui.yml b/metron-contrib/metron-docker-e2e/compose/metron-ui/config/alerts_ui.yml new file mode 100644 index 0000000..b7d8638 --- /dev/null +++ b/metron-contrib/metron-docker-e2e/compose/metron-ui/config/alerts_ui.yml @@ -0,0 +1,21 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +port: 4201 + +rest: + host: metron-rest + port: 8082 http://git-wip-us.apache.org/repos/asf/metron/blob/ebc32586/metron-contrib/metron-docker-e2e/conf/.env ---------------------------------------------------------------------- diff --git a/metron-contrib/metron-docker-e2e/conf/.env b/metron-contrib/metron-docker-e2e/conf/.env new file mode 100644 index 0000000..95371a1 --- /dev/null +++ b/metron-contrib/metron-docker-e2e/conf/.env @@ -0,0 +1,3 @@ +METRON_VERSION=${project.version} +COMPOSE_PROJECT_NAME=metron +BROKER_IP_ADDR= http://git-wip-us.apache.org/repos/asf/metron/blob/ebc32586/metron-contrib/metron-docker-e2e/pom.xml ---------------------------------------------------------------------- diff --git a/metron-contrib/metron-docker-e2e/pom.xml b/metron-contrib/metron-docker-e2e/pom.xml new file mode 100644 index 0000000..ebca4c3 --- /dev/null +++ b/metron-contrib/metron-docker-e2e/pom.xml @@ -0,0 +1,217 @@ +<?xml version="1.0" encoding="UTF-8"?> +<!-- + Licensed to the Apache Software + Foundation (ASF) under one or more contributor license agreements. See the + NOTICE file distributed with this work for additional information regarding + copyright ownership. The ASF licenses this file to You under the Apache License, + Version 2.0 (the "License"); you may not use this file except in compliance + with the License. You may obtain a copy of the License at http://www.apache.org/licenses/LICENSE-2.0 + Unless required by applicable law or agreed to in writing, software distributed + under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES + OR CONDITIONS OF ANY KIND, either express or implied. See the License for + the specific language governing permissions and limitations under the License. + --> + +<project xmlns="http://maven.apache.org/POM/4.0.0"; xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"; + xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd";> + <modelVersion>4.0.0</modelVersion> + <artifactId>metron-docker-e2e</artifactId> + <packaging>pom</packaging> + <name>metron-docker-e2e</name> + <parent> + <groupId>org.apache.metron</groupId> + <artifactId>metron-contrib</artifactId> + <version>0.4.2</version> + </parent> + <description>Metron Docker</description> + <url>https://metron.apache.org/</url> + <properties> + <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding> + <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding> + </properties> + <dependencies> + <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>metron-rest</artifactId> + <version>${project.parent.version}</version> + </dependency> + <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>metron-elasticsearch</artifactId> + <version>${project.parent.version}</version> + </dependency> + <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>metron-common</artifactId> + <version>${project.parent.version}</version> + </dependency> + <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>metron-parsers</artifactId> + <version>${project.parent.version}</version> + </dependency> + <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>metron-alerts</artifactId> + <version>${project.parent.version}</version> + </dependency> + <dependency> + <groupId>org.apache.metron</groupId> + <artifactId>metron-config</artifactId> + <version>${project.parent.version}</version> + </dependency> + </dependencies> + <build> + <plugins> + <plugin> + <artifactId>maven-resources-plugin</artifactId> + <version>3.0.1</version> + <executions> + <execution> + <id>filter-docker-env</id> + <phase>prepare-package</phase> + <goals> + <goal>copy-resources</goal> + </goals> + <configuration> + <outputDirectory>${project.basedir}/compose</outputDirectory> + <resources> + <resource> + <directory>./conf</directory> + <filtering>true</filtering> + <includes> + <include>.env</include> + </includes> + </resource> + </resources> + </configuration> + </execution> + <execution> + <id>copy-elasticsearch-to-rest</id> + <phase>prepare-package</phase> + <goals> + <goal>copy-resources</goal> + </goals> + <configuration> + <outputDirectory>${project.basedir}/compose/metron-rest/packages</outputDirectory> + <resources> + <resource> + <directory>${project.parent.parent.basedir}/metron-platform/metron-elasticsearch/target/</directory> + <includes> + <include>*.tar.gz</include> + </includes> + </resource> + </resources> + </configuration> + </execution> + <execution> + <id>copy-common-to-rest</id> + <phase>prepare-package</phase> + <goals> + <goal>copy-resources</goal> + </goals> + <configuration> + <outputDirectory>${project.basedir}/compose/metron-rest/packages</outputDirectory> + <resources> + <resource> + <directory>${project.parent.parent.basedir}/metron-platform/metron-common/target/</directory> + <includes> + <include>*.tar.gz</include> + </includes> + </resource> + </resources> + </configuration> + </execution> + <execution> + <id>copy-parsers-to-rest</id> + <phase>prepare-package</phase> + <goals> + <goal>copy-resources</goal> + </goals> + <configuration> + <outputDirectory>${project.basedir}/compose/metron-rest/packages</outputDirectory> + <resources> + <resource> + <directory>${project.parent.parent.basedir}/metron-platform/metron-parsers/target/</directory> + <includes> + <include>*.tar.gz</include> + </includes> + </resource> + </resources> + </configuration> + </execution> + <execution> + <id>copy-rest</id> + <phase>prepare-package</phase> + <goals> + <goal>copy-resources</goal> + </goals> + <configuration> + <outputDirectory>${project.basedir}/compose/metron-rest/packages</outputDirectory> + <resources> + <resource> + <directory>${project.parent.parent.basedir}/metron-interface/metron-rest/target/</directory> + <includes> + <include>*.tar.gz</include> + </includes> + </resource> + </resources> + </configuration> + </execution> + <execution> + <id>copy-alerts</id> + <phase>prepare-package</phase> + <goals> + <goal>copy-resources</goal> + </goals> + <configuration> + <outputDirectory>${project.basedir}/compose/metron-ui/packages</outputDirectory> + <resources> + <resource> + <directory>${project.parent.parent.basedir}/metron-interface/metron-alerts/target/</directory> + <includes> + <include>*.tar.gz</include> + </includes> + </resource> + </resources> + </configuration> + </execution> + <execution> + <id>copy-management-ui</id> + <phase>prepare-package</phase> + <goals> + <goal>copy-resources</goal> + </goals> + <configuration> + <outputDirectory>${project.basedir}/compose/metron-ui/packages</outputDirectory> + <resources> + <resource> + <directory>${project.parent.parent.basedir}/metron-interface/metron-config/target/</directory> + <includes> + <include>*.tar.gz</include> + </includes> + </resource> + </resources> + </configuration> + </execution> + </executions> + </plugin> + <plugin> + <artifactId>maven-clean-plugin</artifactId> + <version>3.0.0</version> + <configuration> + <filesets> + <fileset> + <directory>compose</directory> + <includes> + <include>**/*.tar.gz</include> + <include>.env</include> + <include>**/*.template</include> + </includes> + </fileset> + </filesets> + </configuration> + </plugin> + </plugins> + </build> +</project> http://git-wip-us.apache.org/repos/asf/metron/blob/ebc32586/metron-contrib/metron-docker-e2e/scripts/wait_for_rest.sh ---------------------------------------------------------------------- diff --git a/metron-contrib/metron-docker-e2e/scripts/wait_for_rest.sh b/metron-contrib/metron-docker-e2e/scripts/wait_for_rest.sh new file mode 100755 index 0000000..646aba1 --- /dev/null +++ b/metron-contrib/metron-docker-e2e/scripts/wait_for_rest.sh @@ -0,0 +1,29 @@ +#!/bin/bash +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +REST_HOST=$1 +REST_PORT=$2 +DELAY=5 +MAX_ATTEMPTS=24 +COUNTER=0 +while [ $COUNTER -lt $MAX_ATTEMPTS ]; do + curl -s --output /dev/null -XGET http://user:password@$REST_HOST:$REST_PORT/api/v1/globalConfig && echo REST is up after waiting "$(($DELAY * $COUNTER))" seconds && break + sleep $DELAY + let COUNTER=COUNTER+1 +done +if [ $COUNTER -eq $MAX_ATTEMPTS ]; then echo Could not reach REST after "$(($DELAY * $COUNTER))" seconds; fi http://git-wip-us.apache.org/repos/asf/metron/blob/ebc32586/metron-contrib/pom.xml ---------------------------------------------------------------------- diff --git a/metron-contrib/pom.xml b/metron-contrib/pom.xml index eb0ce9c..ccc6585 100644 --- a/metron-contrib/pom.xml +++ b/metron-contrib/pom.xml @@ -40,5 +40,6 @@ </licenses> <modules> <module>metron-docker</module> + <module>metron-docker-e2e</module> </modules> </project> http://git-wip-us.apache.org/repos/asf/metron/blob/ebc32586/metron-interface/metron-alerts/e2e/alerts-list/configure-table/configure-table.e2e-spec.ts ---------------------------------------------------------------------- diff --git a/metron-interface/metron-alerts/e2e/alerts-list/configure-table/configure-table.e2e-spec.ts b/metron-interface/metron-alerts/e2e/alerts-list/configure-table/configure-table.e2e-spec.ts index 6741e2f..dbec5d3 100644 --- a/metron-interface/metron-alerts/e2e/alerts-list/configure-table/configure-table.e2e-spec.ts +++ b/metron-interface/metron-alerts/e2e/alerts-list/configure-table/configure-table.e2e-spec.ts @@ -46,8 +46,7 @@ describe('metron-alerts configure table', function() { it('should select columns from table configuration', () => { let newColNamesColumnConfig = [ 'score', 'timestamp', 'source:type', 'ip_src_addr', 'enrichments:geo:ip_dst_addr:country', 'ip_dst_addr', 'host', 'alert_status', 'guid' ]; - - page.clearLocalStorage(); + page.navigateTo(); page.clickConfigureTable(); @@ -58,26 +57,4 @@ describe('metron-alerts configure table', function() { page.saveConfigureColumns(); }); - it('should rename columns from table configuration', () => { - page.clearLocalStorage(); - page.navigateTo(); - - page.clickConfigureTable(); - page.renameColumn('enrichments:geo:ip_dst_addr:country', 'Country'); - page.saveConfigureColumns(); - - page.clickTableText('FR'); - expect(page.getSearchText()).toEqual('Country:FR'); - expect(page.getChangesAlertTableTitle('Alerts (169)')).toEqual('Alerts (25)'); - page.clickClearSearch(); - - expect(page.getChangesAlertTableTitle('Alerts (25)')).toEqual('Alerts (169)'); - page.setSearchText('Country:FR'); - expect(page.getChangesAlertTableTitle('Alerts (169)')).toEqual('Alerts (25)'); - page.clickClearSearch(); - - expect(page.getTableColumnNames()).toContain('Country', 'for renamed column names for alert list table'); - - }); - }); http://git-wip-us.apache.org/repos/asf/metron/blob/ebc32586/metron-interface/metron-alerts/e2e/login/login.e2e-spec.ts ---------------------------------------------------------------------- diff --git a/metron-interface/metron-alerts/e2e/login/login.e2e-spec.ts b/metron-interface/metron-alerts/e2e/login/login.e2e-spec.ts index dc53889..d4142b4 100644 --- a/metron-interface/metron-alerts/e2e/login/login.e2e-spec.ts +++ b/metron-interface/metron-alerts/e2e/login/login.e2e-spec.ts @@ -16,6 +16,7 @@ * limitations under the License. */ import { LoginPage } from './login.po'; +import { browser } from 'protractor'; describe('login to application', function() { let page: LoginPage; @@ -39,6 +40,6 @@ describe('login to application', function() { it('should logout', () => { page.logout(); - expect(page.getLocation()).toEqual('http://localhost:4200/login'); + expect(page.getLocation()).toEqual(browser.baseUrl + 'login'); }); }); http://git-wip-us.apache.org/repos/asf/metron/blob/ebc32586/metron-interface/metron-alerts/e2e/login/login.po.ts ---------------------------------------------------------------------- diff --git a/metron-interface/metron-alerts/e2e/login/login.po.ts b/metron-interface/metron-alerts/e2e/login/login.po.ts index 8d37800..1d188a9 100644 --- a/metron-interface/metron-alerts/e2e/login/login.po.ts +++ b/metron-interface/metron-alerts/e2e/login/login.po.ts @@ -35,7 +35,7 @@ export class LoginPage { browser.waitForAngularEnabled(false); element.all(by.css('.alert .close')).click(); element.all(by.css('.logout-link')).click(); - waitForURL('http://localhost:4200/login'); + waitForURL(browser.baseUrl + 'login'); } setUserNameAndPassword(userName: string, password: string) { http://git-wip-us.apache.org/repos/asf/metron/blob/ebc32586/metron-interface/metron-alerts/e2e/utils/e2e_util.ts ---------------------------------------------------------------------- diff --git a/metron-interface/metron-alerts/e2e/utils/e2e_util.ts b/metron-interface/metron-alerts/e2e/utils/e2e_util.ts index 92476a4..7127daf 100644 --- a/metron-interface/metron-alerts/e2e/utils/e2e_util.ts +++ b/metron-interface/metron-alerts/e2e/utils/e2e_util.ts @@ -46,25 +46,53 @@ export function waitForStalenessOf (_element ) { } export function loadTestData() { - deleteTestData(); + request.delete('http://user:password@' + browser.params.rest.url + '/api/v1/sensor/indexing/config/alerts_ui_e2e', function (e, response, body) { + request.post({url:'http://user:password@' + browser.params.rest.url + '/api/v1/sensor/indexing/config/alerts_ui_e2e', json: + { + "hdfs": { + "index": "alerts_ui_e2e", + "batchSize": 5, + "enabled": true + }, + "elasticsearch": { + "index": "alerts_ui_e2e", + "batchSize": 5, + "enabled": true + }, + "solr": { + "index": "alerts_ui_e2e", + "batchSize": 5, + "enabled": true + } + } + }, function (e, response, body) { + }); + }); - fs.createReadStream('e2e/mock-data/alerts_ui_e2e_index.template') - .pipe(request.post('http://node1:9200/_template/alerts_ui_e2e_index')); - fs.createReadStream('e2e/mock-data/alerts_ui_e2e_index.data') - .pipe(request.post('http://node1:9200/alerts_ui_e2e_index/alerts_ui_e2e_doc/_bulk')); + request.delete('http://' + browser.params.elasticsearch.url + '/alerts_ui_e2e_index*', function (e, response, body) { + fs.createReadStream('e2e/mock-data/alerts_ui_e2e_index.template') + .pipe(request.post('http://' + browser.params.elasticsearch.url + '/_template/alerts_ui_e2e_index', function (e, response, body) { + fs.createReadStream('e2e/mock-data/alerts_ui_e2e_index.data') + .pipe(request.post('http://' + browser.params.elasticsearch.url + '/alerts_ui_e2e_index/alerts_ui_e2e_doc/_bulk', function (e, response, body) { + })); + })); + }); } export function deleteTestData() { - request.delete('http://node1:9200/alerts_ui_e2e_index*'); + request.delete('http://' + browser.params.elasticsearch.url + '/alerts_ui_e2e_index*'); + request.delete('http://user:password@' + browser.params.rest.url + '/api/v1/sensor/indexing/config/alerts_ui_e2e', function (e, response, body) { + }); } export function createMetaAlertsIndex() { - deleteMetaAlertsIndex(); - fs.createReadStream('./../../metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/metaalert_index.template') - .pipe(request.post('http://node1:9200/metaalert_index')); + request.delete('http://' + browser.params.elasticsearch.url + '/metaalert_index*', function (e, response, body) { + fs.createReadStream('./../../metron-deployment/packaging/ambari/metron-mpack/src/main/resources/common-services/METRON/CURRENT/package/files/metaalert_index.template') + .pipe(request.post('http://' + browser.params.elasticsearch.url + '/metaalert_index')); + }); } export function deleteMetaAlertsIndex() { - request.delete('http://node1:9200/metaalert_index*'); + request.delete('http://' + browser.params.elasticsearch.url + '/metaalert_index*'); } http://git-wip-us.apache.org/repos/asf/metron/blob/ebc32586/metron-interface/metron-alerts/pom.xml ---------------------------------------------------------------------- diff --git a/metron-interface/metron-alerts/pom.xml b/metron-interface/metron-alerts/pom.xml index b59c426..93ff6f4 100644 --- a/metron-interface/metron-alerts/pom.xml +++ b/metron-interface/metron-alerts/pom.xml @@ -30,6 +30,12 @@ </properties> <dependencies> </dependencies> + <profiles> + <profile> + <id>default</id> + <activation> + <activeByDefault>true</activeByDefault> + </activation> <build> <plugins> <plugin> @@ -130,4 +136,44 @@ </plugin> </plugins> </build> + </profile> + + <profile> + <id>e2e</id> + <build> + <plugins> + <plugin> + <groupId>com.github.eirslett</groupId> + <artifactId>frontend-maven-plugin</artifactId> + <version>1.3</version> + <configuration> + <workingDirectory>./</workingDirectory> + <nodeVersion>${node.version}</nodeVersion> + <npmVersion>${npm.version}</npmVersion> + <npmInheritsProxyConfigFromMaven>false</npmInheritsProxyConfigFromMaven> + </configuration> + <executions> + <execution> + <phase>generate-resources</phase> + <id>install node and npm</id> + <goals> + <goal>install-node-and-npm</goal> + </goals> + </execution> + <execution> + <id>npm e2e</id> + <goals> + <goal>npm</goal> + </goals> + <phase>test</phase> + <configuration> + <arguments>run e2e</arguments> + </configuration> + </execution> + </executions> + </plugin> + </plugins> + </build> + </profile> + </profiles> </project> http://git-wip-us.apache.org/repos/asf/metron/blob/ebc32586/metron-interface/metron-alerts/protractor.conf.js ---------------------------------------------------------------------- diff --git a/metron-interface/metron-alerts/protractor.conf.js b/metron-interface/metron-alerts/protractor.conf.js index 82f5c09..a1e4c25 100644 --- a/metron-interface/metron-alerts/protractor.conf.js +++ b/metron-interface/metron-alerts/protractor.conf.js @@ -25,19 +25,12 @@ var SpecReporter = require('jasmine-spec-reporter').SpecReporter; exports.config = { allScriptsTimeout: 25000, specs: [ - './e2e/login/login.e2e-spec.ts', - './e2e/alerts-list/alerts-list.e2e-spec.ts', - './e2e/alerts-list/configure-table/configure-table.e2e-spec.ts', - './e2e/alerts-list/save-search/save-search.e2e-spec.ts', - './e2e/alerts-list/tree-view/tree-view.e2e-spec.ts', - './e2e/alerts-list/alert-filters/alert-filters.e2e-spec.ts', - './e2e/alerts-list/alert-status/alerts-list-status.e2e-spec.ts', - './e2e/alert-details/alert-status/alert-details-status.e2e-spec.ts', - './e2e/alerts-list/meta-alerts/meta-alert.e2e-spec.ts' + './e2e/alerts-list/configure-table/configure-table.e2e-spec.ts' ], capabilities: { 'browserName': 'chrome', 'chromeOptions': { + args: [ "--headless", "--disable-gpu", "--window-size=1435,850" ], 'prefs': { 'credentials_enable_service': false, 'profile': { 'password_manager_enabled': false} @@ -45,7 +38,7 @@ exports.config = { } }, directConnect: true, - baseUrl: 'http://localhost:4200/', + baseUrl: 'http://localhost:4201/', framework: 'jasmine', jasmineNodeOpts: { showColors: true, @@ -77,5 +70,13 @@ exports.config = { onComplete: function() { var createMetaAlertsIndex = require('./e2e/utils/e2e_util').createMetaAlertsIndex; createMetaAlertsIndex(); + }, + params: { + rest: { + url: 'localhost:8082' + }, + elasticsearch: { + url: 'localhost:9210' + } } }; http://git-wip-us.apache.org/repos/asf/metron/blob/ebc32586/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java index 3d50e99..ae8dbe0 100644 --- a/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java +++ b/metron-platform/metron-elasticsearch/src/test/java/org/apache/metron/elasticsearch/integration/ElasticsearchSearchIntegrationTest.java @@ -17,18 +17,20 @@ */ package org.apache.metron.elasticsearch.integration; +import java.util.Map; import org.adrianwalker.multilinestring.Multiline; import org.apache.metron.elasticsearch.dao.ElasticsearchDao; -import org.apache.metron.elasticsearch.integration.components.ElasticSearchComponent; +import org.apache.metron.elasticsearch.utils.ElasticsearchUtils; import org.apache.metron.indexing.dao.AccessConfig; import org.apache.metron.indexing.dao.IndexDao; import org.apache.metron.indexing.dao.SearchIntegrationTest; -import org.apache.metron.integration.InMemoryComponent; import org.elasticsearch.action.bulk.BulkRequestBuilder; import org.elasticsearch.action.bulk.BulkResponse; import org.elasticsearch.action.index.IndexRequest; import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.index.IndexNotFoundException; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.search.SearchHit; import org.json.simple.JSONArray; @@ -36,21 +38,22 @@ import org.json.simple.JSONObject; import org.json.simple.parser.JSONParser; import org.json.simple.parser.ParseException; -import java.io.File; import java.io.IOException; import java.util.HashMap; import java.util.concurrent.ExecutionException; +import org.junit.AfterClass; +import org.junit.BeforeClass; public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { - - private static String indexDir = "target/elasticsearch_search"; + private static String host = "localhost"; + private static String port = "9310"; private static String dateFormat = "yyyy.MM.dd.HH"; private static final int MAX_RETRIES = 10; private static final int SLEEP_MS = 500; /** * { - * "bro_doc": { + * "searchintegrationtest_bro_doc": { * "properties": { * "source:type": { * "type": "string", @@ -99,7 +102,7 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { /** * { - * "snort_doc": { + * "searchintegrationtest_snort_doc": { * "properties": { * "source:type": { * "type": "string", @@ -151,7 +154,7 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { /** * { - * "metaalert_doc": { + * "searchintegrationtest_metaalert_doc": { * "properties": { * "source:type": { "type": "string" }, * "alert": { "type": "nested"} @@ -162,19 +165,38 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { @Multiline private static String metaAlertTypeMappings; + private static Map<String, Object> globalConfig; + private static Client client; + + @BeforeClass + public static void start() { + globalConfig = new HashMap<String, Object>() {{ + put("es.clustername", "elasticsearch"); + put("es.port", port); + put("es.ip", host); + put("es.date.format", dateFormat); + }}; + client = ElasticsearchUtils.getClient(globalConfig, null); + clearIndices(); + } + + @AfterClass + public static void stop() throws Exception { + clearIndices(); + } + + private static void clearIndices() { + try { + client.admin().indices().prepareDelete(broIndex, snortIndex, metaAlertIndex).get(); + } catch (IndexNotFoundException infe) {} + } + @Override protected IndexDao createDao() throws Exception { AccessConfig config = new AccessConfig(); config.setMaxSearchResults(100); config.setMaxSearchGroups(100); - config.setGlobalConfigSupplier( () -> - new HashMap<String, Object>() {{ - put("es.clustername", "metron"); - put("es.port", "9300"); - put("es.ip", "localhost"); - put("es.date.format", dateFormat); - }} - ); + config.setGlobalConfigSupplier( () -> globalConfig); IndexDao dao = new ElasticsearchDao(); dao.init(config); @@ -182,31 +204,20 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { } @Override - protected InMemoryComponent startIndex() throws Exception { - InMemoryComponent es = new ElasticSearchComponent.Builder() - .withHttpPort(9211) - .withIndexDir(new File(indexDir)) - .build(); - es.start(); - return es; - } - - @Override protected void loadTestData() throws ParseException, IOException, ExecutionException, InterruptedException { - ElasticSearchComponent es = (ElasticSearchComponent)indexComponent; - es.getClient().admin().indices().prepareCreate("bro_index_2017.01.01.01") - .addMapping("bro_doc", broTypeMappings).get(); - es.getClient().admin().indices().prepareCreate("snort_index_2017.01.01.02") - .addMapping("snort_doc", snortTypeMappings).get(); - es.getClient().admin().indices().prepareCreate("metaalert_index") - .addMapping("metaalert_doc", metaAlertTypeMappings).get(); + client.admin().indices().prepareCreate(broIndex) + .addMapping(broType, broTypeMappings).get(); + client.admin().indices().prepareCreate(snortIndex) + .addMapping(snortType, snortTypeMappings).get(); + client.admin().indices().prepareCreate(metaAlertIndex) + .addMapping(metaAlertType, metaAlertTypeMappings).get(); - BulkRequestBuilder bulkRequest = es.getClient().prepareBulk().setRefresh(true); + BulkRequestBuilder bulkRequest = client.prepareBulk().setRefresh(true); JSONArray broArray = (JSONArray) new JSONParser().parse(broData); for(Object o: broArray) { JSONObject jsonObject = (JSONObject) o; - IndexRequestBuilder indexRequestBuilder = es.getClient().prepareIndex("bro_index_2017.01.01.01", "bro_doc"); + IndexRequestBuilder indexRequestBuilder = client.prepareIndex(broIndex, broType); indexRequestBuilder = indexRequestBuilder.setSource(jsonObject.toJSONString()); indexRequestBuilder = indexRequestBuilder.setTimestamp(jsonObject.get("timestamp").toString()); bulkRequest.add(indexRequestBuilder); @@ -214,7 +225,7 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { JSONArray snortArray = (JSONArray) new JSONParser().parse(snortData); for(Object o: snortArray) { JSONObject jsonObject = (JSONObject) o; - IndexRequestBuilder indexRequestBuilder = es.getClient().prepareIndex("snort_index_2017.01.01.02", "snort_doc"); + IndexRequestBuilder indexRequestBuilder = client.prepareIndex(snortIndex, snortType); indexRequestBuilder = indexRequestBuilder.setSource(jsonObject.toJSONString()); indexRequestBuilder = indexRequestBuilder.setTimestamp(jsonObject.get("timestamp").toString()); bulkRequest.add(indexRequestBuilder); @@ -222,7 +233,7 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { JSONArray metaAlertArray = (JSONArray) new JSONParser().parse(metaAlertData); for(Object o: metaAlertArray) { JSONObject jsonObject = (JSONObject) o; - IndexRequestBuilder indexRequestBuilder = es.getClient().prepareIndex("metaalert_index", "metaalert_doc"); + IndexRequestBuilder indexRequestBuilder = client.prepareIndex(metaAlertIndex, metaAlertType); indexRequestBuilder = indexRequestBuilder.setSource(jsonObject.toJSONString()); bulkRequest.add(indexRequestBuilder); } @@ -231,9 +242,9 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { throw new RuntimeException("Failed to index test data"); } - SearchResponse broDocs = es.getClient() - .prepareSearch("bro_index_2017.01.01.01") - .setTypes("bro_doc") + SearchResponse broDocs = client + .prepareSearch(broIndex) + .setTypes(broType) .setQuery(QueryBuilders.matchAllQuery()) .get(); // We're changing the _id field, we need to create a copy and delete the original. @@ -241,15 +252,14 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { // Bro GUIDs to collide while using the standard analyzer // Use timestamp as part of guid because query may not return in order each time IndexRequest indexRequest = new IndexRequest() - .index("bro_index_2017.01.01.01") - .type("bro_doc") + .index(broIndex) + .type(broType) .id("bro-" + hit.getSource().get("timestamp")) .source(hit.getSource()); - es.getClient().index(indexRequest).get(); + client.index(indexRequest).get(); // Delete the original - es.getClient() - .prepareDelete("bro_index_2017.01.01.01", "bro_doc", hit.getId()) + client.prepareDelete(broIndex, broType, hit.getId()) .get(); } @@ -258,9 +268,9 @@ public class ElasticsearchSearchIntegrationTest extends SearchIntegrationTest { boolean allUpdated = true; for (int t = 0; t < MAX_RETRIES; ++t, Thread.sleep(SLEEP_MS)) { allUpdated = true; - SearchResponse response = es.getClient() - .prepareSearch("bro_index_2017.01.01.01") - .setTypes("bro_doc") + SearchResponse response = client + .prepareSearch(broIndex) + .setTypes(broType) .setQuery(QueryBuilders.matchAllQuery()) .get(); if (response.getHits().getTotalHits() == 0) { http://git-wip-us.apache.org/repos/asf/metron/blob/ebc32586/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java ---------------------------------------------------------------------- diff --git a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java index 4d3ff9b..84f6f40 100644 --- a/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java +++ b/metron-platform/metron-indexing/src/test/java/org/apache/metron/indexing/dao/SearchIntegrationTest.java @@ -45,6 +45,15 @@ import java.util.Map; import java.util.Optional; public abstract class SearchIntegrationTest { + + private static final String namespace = SearchIntegrationTest.class.getSimpleName().toLowerCase(); + protected static final String broIndex = namespace + "_bro_index"; + protected static final String snortIndex = namespace + "_snort_index"; + protected static final String metaAlertIndex = namespace + "_metaalert_index"; + protected static final String broType = namespace + "_bro_doc"; + protected static final String snortType = namespace + "_snort_doc"; + protected static final String metaAlertType = namespace + "_metaalert_doc"; + /** * [ * {"source:type": "bro", "ip_src_addr":"192.168.1.1", "ip_src_port": 8010, "long_field": 10000, "timestamp":1, "latitude": 48.5839, "score": 10.0, "is_alert":true, "location_point": "48.5839,7.7455", "bro_field": "bro data 1", "duplicate_name_field": "data 1", "guid":"bro_1"}, @@ -80,7 +89,7 @@ public abstract class SearchIntegrationTest { /** * { - * "indices": ["bro", "snort"], + * "indices": ["searchintegrationtest_bro", "searchintegrationtest_snort"], * "query": "*", * "from": 0, * "size": 10, @@ -98,7 +107,7 @@ public abstract class SearchIntegrationTest { /** * { * "guid": "bro-3", - * "sensorType": "bro" + * "sensorType": "searchintegrationtest_bro" * } */ @Multiline @@ -108,11 +117,11 @@ public abstract class SearchIntegrationTest { * [ * { * "guid": "bro-1", - * "sensorType": "bro" + * "sensorType": "searchintegrationtest_bro" * }, * { * "guid": "bro-2", - * "sensorType": "bro" + * "sensorType": "searchintegrationtest_bro" * } * ] */ @@ -121,7 +130,7 @@ public abstract class SearchIntegrationTest { /** * { - * "indices": ["bro", "snort"], + * "indices": ["searchintegrationtest_bro", "searchintegrationtest_snort"], * "query": "ip_src_addr:192.168.1.1", * "from": 0, * "size": 10, @@ -138,7 +147,7 @@ public abstract class SearchIntegrationTest { /** * { - * "indices": ["bro", "snort"], + * "indices": ["searchintegrationtest_bro", "searchintegrationtest_snort"], * "query": "*", * "from": 0, * "size": 10, @@ -156,8 +165,8 @@ public abstract class SearchIntegrationTest { /** * { * "indices": [ - * "snort", - * "bro" + * "searchintegrationtest_snort", + * "searchintegrationtest_bro" * ], * "query": "*", * "from": 0, @@ -176,8 +185,8 @@ public abstract class SearchIntegrationTest { /** * { * "indices": [ - * "snort", - * "bro" + * "searchintegrationtest_snort", + * "searchintegrationtest_bro" * ], * "query": "*", * "from": 0, @@ -195,7 +204,7 @@ public abstract class SearchIntegrationTest { /** * { - * "indices": ["bro", "snort"], + * "indices": ["searchintegrationtest_bro", "searchintegrationtest_snort"], * "query": "*", * "from": 4, * "size": 3, @@ -212,7 +221,7 @@ public abstract class SearchIntegrationTest { /** * { - * "indices": ["bro"], + * "indices": ["searchintegrationtest_bro"], * "query": "*", * "from": 0, * "size": 10, @@ -230,7 +239,7 @@ public abstract class SearchIntegrationTest { /** * { * "facetFields": ["source:type", "ip_src_addr", "ip_src_port", "long_field", "timestamp", "latitude", "score", "is_alert"], - * "indices": ["bro", "snort", "metaalert"], + * "indices": ["searchintegrationtest_bro", "searchintegrationtest_snort", "searchintegrationtest_metaalert"], * "query": "*", * "from": 0, * "size": 10, @@ -248,7 +257,7 @@ public abstract class SearchIntegrationTest { /** * { * "facetFields": ["location_point"], - * "indices": ["bro", "snort"], + * "indices": ["searchintegrationtest_bro", "searchintegrationtest_snort"], * "query": "*", * "from": 0, * "size": 10, @@ -265,7 +274,7 @@ public abstract class SearchIntegrationTest { /** * { - * "indices": ["bro", "snort"], + * "indices": ["searchintegrationtest_bro", "searchintegrationtest_snort"], * "query": "*", * "from": 0, * "size": 10, @@ -282,7 +291,7 @@ public abstract class SearchIntegrationTest { /** * { - * "indices": ["bro", "snort"], + * "indices": ["searchintegrationtest_bro", "searchintegrationtest_snort"], * "query": "*", * "from": 0, * "size": 101, @@ -300,7 +309,7 @@ public abstract class SearchIntegrationTest { /** * { * "fields": ["ip_src_addr"], - * "indices": ["bro", "snort"], + * "indices": ["searchintegrationtest_bro", "searchintegrationtest_snort"], * "query": "*", * "from": 0, * "size": 10, @@ -318,7 +327,7 @@ public abstract class SearchIntegrationTest { /** * { * "fields": ["ip_src_addr"], - * "indices": ["bro", "snort"], + * "indices": ["searchintegrationtest_bro", "searchintegrationtest_snort"], * "query": "ip_src_addr:192.168.1.9", * "from": 0, * "size": 10, @@ -336,7 +345,7 @@ public abstract class SearchIntegrationTest { /** * { * "fields": ["guid"], - * "indices": ["metaalert"], + * "indices": ["searchintegrationtest_metaalert"], * "query": "*", * "from": 0, * "size": 10, @@ -363,7 +372,7 @@ public abstract class SearchIntegrationTest { * } * ], * "scoreField":"score", - * "indices": ["bro", "snort", "metaalert"], + * "indices": ["searchintegrationtest_bro", "searchintegrationtest_snort", "searchintegrationtest_metaalert"], * "query": "*" * } */ @@ -388,7 +397,7 @@ public abstract class SearchIntegrationTest { * } * } * ], - * "indices": ["bro", "snort", "metaalert"], + * "indices": ["searchintegrationtest_bro", "searchintegrationtest_snort", "searchintegrationtest_metaalert"], * "query": "*" * } */ @@ -402,7 +411,7 @@ public abstract class SearchIntegrationTest { * "field":"location_point" * } * ], - * "indices": ["bro", "snort"], + * "indices": ["searchintegrationtest_bro", "searchintegrationtest_snort"], * "query": "*" * } */ @@ -420,7 +429,7 @@ public abstract class SearchIntegrationTest { * } * } * ], - * "indices": ["bro", "snort"], + * "indices": ["searchintegrationtest_bro", "searchintegrationtest_snort"], * "query": "*" * } */ @@ -433,7 +442,6 @@ public abstract class SearchIntegrationTest { @Before public synchronized void setup() throws Exception { if(dao == null && indexComponent == null) { - indexComponent = startIndex(); loadTestData(); dao = createDao(); } @@ -472,10 +480,10 @@ public abstract class SearchIntegrationTest { }); Iterator<Document> response = dao.getAllLatest(request).iterator(); Document bro2 = response.next(); - Assert.assertEquals("bro_1", bro2.getDocument().get("guid")); + Assert.assertEquals("bro_2", bro2.getDocument().get("guid")); Assert.assertEquals("bro", bro2.getDocument().get("source:type")); Document snort2 = response.next(); - Assert.assertEquals("bro_2", snort2.getDocument().get("guid")); + Assert.assertEquals("bro_1", snort2.getDocument().get("guid")); Assert.assertEquals("bro", snort2.getDocument().get("source:type")); Assert.assertFalse(response.hasNext()); } @@ -692,7 +700,7 @@ public abstract class SearchIntegrationTest { } // getColumnMetadata with multiple indices { - Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Arrays.asList("bro", "snort")); + Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Arrays.asList("searchintegrationtest_bro", "searchintegrationtest_snort")); Assert.assertEquals(15, fieldTypes.size()); Assert.assertEquals(FieldType.STRING, fieldTypes.get("guid")); Assert.assertEquals(FieldType.STRING, fieldTypes.get("source:type")); @@ -712,7 +720,7 @@ public abstract class SearchIntegrationTest { } // getColumnMetadata with only bro { - Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("bro")); + Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("searchintegrationtest_bro")); Assert.assertEquals(13, fieldTypes.size()); Assert.assertEquals(FieldType.STRING, fieldTypes.get("guid")); Assert.assertEquals(FieldType.STRING, fieldTypes.get("source:type")); @@ -730,7 +738,7 @@ public abstract class SearchIntegrationTest { } // getColumnMetadata with only snort { - Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("snort")); + Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("searchintegrationtest_snort")); Assert.assertEquals(14, fieldTypes.size()); Assert.assertEquals(FieldType.INTEGER, fieldTypes.get("snort_field")); Assert.assertEquals(FieldType.STRING, fieldTypes.get("guid")); @@ -748,7 +756,7 @@ public abstract class SearchIntegrationTest { } // getColumnMetadata with an index that doesn't exist { - Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("someindex")); + Map<String, FieldType> fieldTypes = dao.getColumnMetadata(Collections.singletonList("searchintegrationtest_someindex")); Assert.assertEquals(0, fieldTypes.size()); } //Fields query @@ -952,12 +960,6 @@ public abstract class SearchIntegrationTest { } } - @AfterClass - public static void stop() throws Exception { - indexComponent.stop(); - } - protected abstract IndexDao createDao() throws Exception; - protected abstract InMemoryComponent startIndex() throws Exception; protected abstract void loadTestData() throws Exception; }
