This is an automated email from the ASF dual-hosted git repository.
ocket8888 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/trafficcontrol.git
The following commit(s) were added to refs/heads/master by this push:
new 61007bd Use Chromium for Traffic Portal integration tests (#5785)
61007bd is described below
commit 61007bd321cbd0bcee425dd9b2da481c3a4976b5
Author: Zach Hoffman <[email protected]>
AuthorDate: Fri Apr 30 16:46:40 2021 -0600
Use Chromium for Traffic Portal integration tests (#5785)
* Remove references to other services from TO API Tests and TP Integration
Tests compose files
* Remove unnecessary layer
* Use a subshell instead of a tmpfile
* Use Chromium for Traffic Portal integration tests in CDN in a Box
* Install ChromeDriver version corresponding to the Chromium version
* Do not install musl-dev or update gems
* Use Chromium for Traffic Portal integration tests in GitHub Actions
---
.github/actions/tp-integration-tests/entrypoint.sh | 16 +++++++++-------
.github/workflows/tp.integration.tests.yml | 4 ++--
docs/source/admin/quick_howto/ciab.rst | 6 +++---
.../docker-compose.traffic-ops-test.yml | 10 ++++------
.../docker-compose.traffic-portal-test.yml | 10 +++-------
infrastructure/cdn-in-a-box/docker-compose.yml | 4 ++--
.../traffic_ops/set-to-ips-from-dns.sh | 21 ++++++++++++---------
.../cdn-in-a-box/traffic_ops/to-access.sh | 10 ++++++++++
.../cdn-in-a-box/traffic_ops/trafficops-init.sh | 2 +-
.../traffic_ops_integration_test/Dockerfile | 1 -
.../traffic_ops_integration_test/run.sh | 1 +
.../traffic_portal_integration_test/Dockerfile | 20 +++++++++++---------
.../etc/yum.repos.d/google-chrome.repo | 22 ----------------------
.../traffic_portal_integration_test/run.sh | 8 ++++----
traffic_portal/test/integration/README.md | 2 +-
15 files changed, 63 insertions(+), 74 deletions(-)
diff --git a/.github/actions/tp-integration-tests/entrypoint.sh
b/.github/actions/tp-integration-tests/entrypoint.sh
index b21a750..121183d 100755
--- a/.github/actions/tp-integration-tests/entrypoint.sh
+++ b/.github/actions/tp-integration-tests/entrypoint.sh
@@ -15,6 +15,7 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+trap 'echo "Error on line ${LINENO} of ${0}"; exit 1' ERR
set -o errexit -o nounset -o pipefail
hub_fqdn="http://localhost:4444/wd/hub/status";
@@ -143,14 +144,14 @@ start_traffic_vault &
sudo apt-get install -y --no-install-recommends gettext \
ruby ruby-dev libc-dev curl \
- gcc musl-dev
+ gcc
-sudo gem update --system && sudo gem install sass compass
+sudo gem install sass compass
sudo npm i -g forever bower grunt
-CHROME_CONTAINER=$(docker ps | grep "selenium/node-chrome" | awk '{print $1}')
-HUB_CONTAINER=$(docker ps | grep "selenium/hub" | awk '{print $1}')
-CHROME_VER=$(docker exec "$CHROME_CONTAINER" google-chrome --version | sed -E
's/.* ([0-9.]+).*/\1/')
+CHROMIUM_CONTAINER=$(docker ps -qf name=chromium)
+HUB_CONTAINER=$(docker ps -qf name=hub)
+CHROMIUM_VER=$(docker exec "$CHROMIUM_CONTAINER" chromium --version | grep -Eo
'[0-9.]+')
GOROOT=/usr/local/go
export PATH="${PATH}:${GOROOT}/bin"
@@ -206,7 +207,7 @@ onFail() {
mv tp.log Reports/forever.log
mv access.log Reports/tp-access.log
mv out.log Reports/node.log
- docker logs $CHROME_CONTAINER > Reports/chrome.log
+ docker logs $CHROMIUM_CONTAINER > Reports/chromium.log
docker logs $HUB_CONTAINER > Reports/hub.log
echo "Detailed logs produced info Reports artifact"
exit 1
@@ -217,7 +218,7 @@ cd "${REPO_DIR}/traffic_portal/test/integration"
npm ci
PATH=$(pwd)/node_modules/.bin/:$PATH
-webdriver-manager update --gecko false --versions.chrome
"LATEST_RELEASE_$CHROME_VER"
+webdriver-manager update --gecko false --versions.chrome
"LATEST_RELEASE_$CHROMIUM_VER"
jq " .capabilities.chromeOptions.args = [
\"--headless\",
@@ -238,4 +239,5 @@ timeout 5m bash <<TMOUT
done
TMOUT
+trap - ERR
protractor ./GeneratedCode/config.js --params.baseUrl="${tp_fqdn}"
--params.apiUrl="${to_fqdn}/api/4.0" || onFail
diff --git a/.github/workflows/tp.integration.tests.yml
b/.github/workflows/tp.integration.tests.yml
index c72d82f..9ac575e 100644
--- a/.github/workflows/tp.integration.tests.yml
+++ b/.github/workflows/tp.integration.tests.yml
@@ -57,8 +57,8 @@ jobs:
ports:
- "4444:4444"
options: --health-cmd=/opt/bin/check-grid.sh --health-interval=5s
--health-timeout=15s --health-retries=5
- chrome:
- image: selenium/node-chrome
+ chromium:
+ image: ossalternatives/selenium-node-chromium
env:
HUB_HOST: hub
HUB_PORT: 4444
diff --git a/docs/source/admin/quick_howto/ciab.rst
b/docs/source/admin/quick_howto/ciab.rst
index 03c0375..7493da6 100644
--- a/docs/source/admin/quick_howto/ciab.rst
+++ b/docs/source/admin/quick_howto/ciab.rst
@@ -146,7 +146,7 @@ There also exist TP and TO integration tests containers.
Both of these container
sudo docker-compose -f docker-compose.traffic-ops-test.yml up
-.. note:: If all CDN in a Box containers are started at once (example:
``docker-compose -f docker-compose.yml -f docker-compose.traffic-ops-test.yml
up integration``), the :ref:`Enroller <ciab-enroller>` initial data load is
skipped to prevent data conflicts with the :ref:`Traffic Ops API tests fixtures
<dev-traffic-ops-fixtures>`.
+.. note:: If all CDN in a Box containers are started at once (example:
``docker-compose -f docker-compose.yml -f docker-compose.traffic-ops-test.yml
up -d edge enroller dns db smtp trafficops trafficvault integration``), the
:ref:`Enroller <ciab-enroller>` initial data load is skipped to prevent data
conflicts with the :ref:`Traffic Ops API tests fixtures
<dev-traffic-ops-fixtures>`.
variables.env
"""""""""""""
@@ -191,7 +191,7 @@ Trusting the Certificate Authority
----------------------------------
For developer and testing use-cases, it may be necessary to have full x509
:abbr:`CA (Certificate Authority)` validation by HTTPS clients\ [6]_\ [7]_. For
x509 validation to work properly, the self-signed x509 :abbr:`CA (Certificate
Authority)` certificate must be trusted either at the system level or by the
client application itself.
-.. note:: HTTP Client applications such as Google Chrome, Firefox,
:manpage:`curl(1)`, and :manpage:`wget(1)` can also be individually configured
to trust the :abbr:`CA (Certificate Authority)` certificate. Review each
program's respective documentation for instructions.
+.. note:: HTTP Client applications such as Chromium, Firefox,
:manpage:`curl(1)`, and :manpage:`wget(1)` can also be individually configured
to trust the :abbr:`CA (Certificate Authority)` certificate. Review each
program's respective documentation for instructions.
Importing the :abbr:`CA (Certificate Authority)` Certificate on OSX
"""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""""
@@ -326,7 +326,7 @@ Dante's socks proxy is an optional container that can be
used to provide browser
#. Start the CDN-in-a-Box stack at least once so that the x.509 self-signed
:abbr:`CA (Certificate Authority)` is created.
#. On the host, import and Trust the :abbr:`CA (Certificate Authority)` for
your target Operating System. See `Trusting the Certificate Authority`_
-#. On the host, using either Firefox or Chrome, download the `FoxyProxy
browser plugin <https://getfoxyproxy.org/>`_ which enables dynamic proxy
support via URL regular expression
+#. On the host, using either Firefox or Chromium, download the `FoxyProxy
browser plugin <https://getfoxyproxy.org/>`_ which enables dynamic proxy
support via URL regular expression
#. Once FoxyProxy is installed, click the Fox icon on the upper right hand of
the browser window, select :guilabel:`Options`
#. Once in Options Dialog, Click :guilabel:`Add New Proxy` and navigate to the
General tab:
#. Fill in the General tab according to the table
diff --git a/infrastructure/cdn-in-a-box/docker-compose.traffic-ops-test.yml
b/infrastructure/cdn-in-a-box/docker-compose.traffic-ops-test.yml
index 41ef0d9..cdadcb6 100644
--- a/infrastructure/cdn-in-a-box/docker-compose.traffic-ops-test.yml
+++ b/infrastructure/cdn-in-a-box/docker-compose.traffic-ops-test.yml
@@ -21,7 +21,7 @@
# example of how to run the main compose with this file:
#
# docker-compose -f docker-compose.yml -f docker-compose.traffic-ops-test.yml
up -d edge enroller dns db smtp trafficops trafficvault integration
-# docker-compose -f docker-compose.yml -f docker-compose.traffic-ops-test.yml
logs -f integration
+# docker-compose -f docker-compose.traffic-ops-test.yml logs -f integration
---
version: '2.1'
@@ -33,17 +33,15 @@ services:
dockerfile:
infrastructure/cdn-in-a-box/traffic_ops_integration_test/Dockerfile
env_file:
- variables.env
+ environment:
+ SKIP_DIG_IP: 'true'
+ SKIP_TRAFFIC_OPS_DATA: 'true'
hostname: integration
domainname: infra.ciab.test
volumes:
- shared:/shared
- ../../junit:/junit
- trafficops:
- environment:
- DIG_IP_RETRY: ${DIG_IP_RETRY:-0}
- LOAD_TRAFFIC_OPS_DATA: ${LOAD_TRAFFIC_OPS_DATA:-false}
-
volumes:
junit:
shared:
diff --git a/infrastructure/cdn-in-a-box/docker-compose.traffic-portal-test.yml
b/infrastructure/cdn-in-a-box/docker-compose.traffic-portal-test.yml
index 8366624..ed23469 100644
--- a/infrastructure/cdn-in-a-box/docker-compose.traffic-portal-test.yml
+++ b/infrastructure/cdn-in-a-box/docker-compose.traffic-portal-test.yml
@@ -21,7 +21,7 @@
# example of how to run the main compose with this file:
#
# docker-compose -f docker-compose.yml -f
docker-compose.traffic-portal-test.yml up -d db edge trafficportal trafficops
trafficvault portal-integration-test
-# docker-compose -f docker-compose.yml -f
docker-compose.traffic-portal-test.yml logs -f portal-integration-test
+# docker-compose -f docker-compose.traffic-portal-test.yml logs -f
portal-integration-test
---
version: '2.1'
@@ -33,20 +33,16 @@ services:
dockerfile:
infrastructure/cdn-in-a-box/traffic_portal_integration_test/Dockerfile
args:
RHEL_VERSION: ${RHEL_VERSION:-8}
- depends_on:
- - edge
env_file:
- variables.env
+ environment:
+ SKIP_DIG_IP: 'true'
hostname: portal-integration
domainname: infra.ciab.test
volumes:
- shared:/shared
- ../../junit:/junit
- trafficops:
- environment:
- DIG_IP_RETRY: ${DIG_IP_RETRY:-0}
-
volumes:
shared:
external: false
diff --git a/infrastructure/cdn-in-a-box/docker-compose.yml
b/infrastructure/cdn-in-a-box/docker-compose.yml
index f3e5df0..f1d21d3 100644
--- a/infrastructure/cdn-in-a-box/docker-compose.yml
+++ b/infrastructure/cdn-in-a-box/docker-compose.yml
@@ -67,8 +67,8 @@ services:
env_file:
- variables.env
environment:
- DIG_IP_RETRY: ${DIG_IP_RETRY:-10}
- LOAD_TRAFFIC_OPS_DATA: ${LOAD_TRAFFIC_OPS_DATA:-true}
+ SKIP_DIG_IP: ${SKIP_DIG_IP:-false}
+ SKIP_TRAFFIC_OPS_DATA: ${SKIP_TRAFFIC_OPS_DATA:-false}
hostname: trafficops
image: trafficops
volumes:
diff --git a/infrastructure/cdn-in-a-box/traffic_ops/set-to-ips-from-dns.sh
b/infrastructure/cdn-in-a-box/traffic_ops/set-to-ips-from-dns.sh
index e3d9b7c..2f08ade 100755
--- a/infrastructure/cdn-in-a-box/traffic_ops/set-to-ips-from-dns.sh
+++ b/infrastructure/cdn-in-a-box/traffic_ops/set-to-ips-from-dns.sh
@@ -40,19 +40,22 @@ service_ips="${gateway_ip}"
service_ip6s="${gateway_ip6}"
INTERFACE=$(ip link | awk '/\<UP\>/ && !/LOOPBACK/ {sub(/@.*/, "", $2); print
$2}')
NETMASK=$(route | awk -v INTERFACE=$INTERFACE '$8 ~ INTERFACE && $1 !~
"default" {print $3}')
+DIG_IP_RETRY=10
for service_name in $service_names; do
service_fqdn="${service_name}.${service_domain}"
- for (( i=1; i<=DIG_IP_RETRY; i++ )); do
- service_ip="$(dig +short ${service_fqdn} A)"
- if [ -z "${service_ip}" ]; then
- printf "service \"${service_fqdn}\" not found in dns,
count=$i, waiting ...\n"
- sleep 3
- else
- break
- fi
- done
+ if [[ ! -e /shared/SKIP_DIG_IP ]]; then
+ for (( i=1; i<=DIG_IP_RETRY; i++ )); do
+ service_ip="$(dig +short ${service_fqdn} A)"
+ if [ -z "${service_ip}" ]; then
+ printf "service \"${service_fqdn}\" not found
in dns, count=$i, waiting ...\n"
+ sleep 3
+ else
+ break
+ fi
+ done
+ fi
#
# TODO add a way to determine if a service wasn't built in the Compose,
diff --git a/infrastructure/cdn-in-a-box/traffic_ops/to-access.sh
b/infrastructure/cdn-in-a-box/traffic_ops/to-access.sh
index ff08190..85e4898 100755
--- a/infrastructure/cdn-in-a-box/traffic_ops/to-access.sh
+++ b/infrastructure/cdn-in-a-box/traffic_ops/to-access.sh
@@ -375,3 +375,13 @@ to-auto-snapqueue() {
sleep $AUTO_SNAPQUEUE_POLL_INTERVAL
done
}
+
+check-skips() {
+ if [[ "$SKIP_TRAFFIC_OPS_DATA" == true ]]; then
+ touch /shared/SKIP_TRAFFIC_OPS_DATA
+ fi
+ if [[ "$SKIP_DIG_IP" == true ]]; then
+ touch /shared/SKIP_DIG_IP
+ fi
+ sync
+}
diff --git a/infrastructure/cdn-in-a-box/traffic_ops/trafficops-init.sh
b/infrastructure/cdn-in-a-box/traffic_ops/trafficops-init.sh
index 2230686..7ab3b35 100755
--- a/infrastructure/cdn-in-a-box/traffic_ops/trafficops-init.sh
+++ b/infrastructure/cdn-in-a-box/traffic_ops/trafficops-init.sh
@@ -145,7 +145,7 @@ traffic_router_zonemanager_timeout() {
mv "$modified_crconfig" $crconfig_path;
}
-if [[ "$LOAD_TRAFFIC_OPS_DATA" == true ]]; then
+if [[ ! -e /shared/SKIP_TRAFFIC_OPS_DATA ]]; then
traffic_router_zonemanager_timeout
# Load required data at the top level
diff --git
a/infrastructure/cdn-in-a-box/traffic_ops_integration_test/Dockerfile
b/infrastructure/cdn-in-a-box/traffic_ops_integration_test/Dockerfile
index e061011..fff5c35 100644
--- a/infrastructure/cdn-in-a-box/traffic_ops_integration_test/Dockerfile
+++ b/infrastructure/cdn-in-a-box/traffic_ops_integration_test/Dockerfile
@@ -42,7 +42,6 @@ RUN cd /go/src/github.com/jstemmer/go-junit-report && go build
COPY ./vendor/ /go/src/github.com/apache/trafficcontrol/vendor/
# integration source and dependencies
-COPY ./infrastructure/cdn-in-a-box/
/go/src/github.com/apache/trafficcontrol/infrastructure/cdn-in-a-box/
COPY ./lib/ /go/src/github.com/apache/trafficcontrol/lib/
COPY ./traffic_ops/toclientlib/
/go/src/github.com/apache/trafficcontrol/traffic_ops/toclientlib/
COPY ./traffic_ops/v1-client/
/go/src/github.com/apache/trafficcontrol/traffic_ops/v1-client/
diff --git a/infrastructure/cdn-in-a-box/traffic_ops_integration_test/run.sh
b/infrastructure/cdn-in-a-box/traffic_ops_integration_test/run.sh
index a38cd97..67f602c 100755
--- a/infrastructure/cdn-in-a-box/traffic_ops_integration_test/run.sh
+++ b/infrastructure/cdn-in-a-box/traffic_ops_integration_test/run.sh
@@ -25,6 +25,7 @@ do
done
source to-access.sh
+check-skips
set-dns.sh
insert-self-into-dns.sh
diff --git
a/infrastructure/cdn-in-a-box/traffic_portal_integration_test/Dockerfile
b/infrastructure/cdn-in-a-box/traffic_portal_integration_test/Dockerfile
index 3df10a6..d633cb0 100644
--- a/infrastructure/cdn-in-a-box/traffic_portal_integration_test/Dockerfile
+++ b/infrastructure/cdn-in-a-box/traffic_portal_integration_test/Dockerfile
@@ -23,9 +23,6 @@ RUN if [[ "${RHEL_VERSION%%.*}" -eq 7 ]]; then \
yum -y install dnf || exit 1; \
fi
-# Installs the Google Chrome dnf repo
-COPY infrastructure/cdn-in-a-box/traffic_portal_integration_test/etc etc
-
RUN if [[ "${RHEL_VERSION%%.*}" -eq 7 ]]; then \
utils_package=yum-utils; \
else \
@@ -33,15 +30,15 @@ RUN if [[ "${RHEL_VERSION%%.*}" -eq 7 ]]; then \
fi && \
dnf install -y \
bind-utils \
- # jq is in EPEL
+ # chromium and jq are in EPEL
epel-release \
GConf2 \
git \
- google-chrome-stable \
java-1.8.0-openjdk \
net-tools \
$utils_package && \
- dnf -y install jq && \
+ set -o pipefail && \
+ dnf -y install chromium jq && \
dnf -y clean all
FROM os-dependencies AS node-dependencies
@@ -71,9 +68,14 @@ RUN jq ' \
config.json > conf.json.tmp && \
mv conf.json.tmp config.json
-RUN webdriver-manager clean && \
- repoquery --installed --qf='%{version}' google-chrome-stable | \
- xargs webdriver-manager update --versions.chrome
+# Install chromium
+RUN dnf -y install chromium
+
+RUN set -o pipefail && \
+ webdriver-manager clean && \
+ chromium-browser --version | \
+ grep -o '[0-9.]\+' | \
+ xargs webdriver-manager update --gecko false --versions.chrome
COPY infrastructure/cdn-in-a-box/traffic_ops/to-access.sh \
infrastructure/cdn-in-a-box/traffic_portal_integration_test/run.sh \
diff --git
a/infrastructure/cdn-in-a-box/traffic_portal_integration_test/etc/yum.repos.d/google-chrome.repo
b/infrastructure/cdn-in-a-box/traffic_portal_integration_test/etc/yum.repos.d/google-chrome.repo
deleted file mode 100644
index 46a4374..0000000
---
a/infrastructure/cdn-in-a-box/traffic_portal_integration_test/etc/yum.repos.d/google-chrome.repo
+++ /dev/null
@@ -1,22 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-[google-chrome]
-name=google-chrome
-baseurl=https://dl.google.com/linux/chrome/rpm/stable/x86_64
-enabled=1
-gpgcheck=1
-gpgkey=https://dl.google.com/linux/linux_signing_key.pub
diff --git a/infrastructure/cdn-in-a-box/traffic_portal_integration_test/run.sh
b/infrastructure/cdn-in-a-box/traffic_portal_integration_test/run.sh
index 94ed7cc..d9b4080 100755
--- a/infrastructure/cdn-in-a-box/traffic_portal_integration_test/run.sh
+++ b/infrastructure/cdn-in-a-box/traffic_portal_integration_test/run.sh
@@ -23,6 +23,7 @@ do
done
source to-access.sh
+check-skips
set-dns.sh
insert-self-into-dns.sh
@@ -52,7 +53,7 @@ if (( time - start_time >= timeout_in_seconds )); then
echo "Warning: Traffic Portal did not start after ${timeout_in_seconds}
seconds.";
fi;
-chrome_version="$(repoquery --installed --qf='%{version}'
google-chrome-stable)"
+chrome_version="$(chromium-browser --version | grep -o '[0-9.]\+')"
nohup webdriver-manager start --versions.chrome "$chrome_version" &
selenium_port=4444
@@ -62,14 +63,13 @@ while ! curl -Lvsk "${selenium_fqdn}" 2>/dev/null
>/dev/null; do
sleep 1
done
-jq "$(<<JQ_FILTERS cat
+echo "$(jq "$(<<JQ_FILTERS cat
.params.baseUrl = "https://$TP_FQDN"; |
.params.apiUrl = "https://$TP_FQDN/api/4.0"; |
.params.login.username = "$TO_ADMIN_USER" |
.params.login.password = "$TO_ADMIN_PASSWORD"
JQ_FILTERS
-)" config.json > conf.json.tmp
-mv conf.json.tmp config.json
+)" config.json)" > config.json
cat config.json
diff --git a/traffic_portal/test/integration/README.md
b/traffic_portal/test/integration/README.md
index 989fa0d..62acc72 100644
--- a/traffic_portal/test/integration/README.md
+++ b/traffic_portal/test/integration/README.md
@@ -39,7 +39,7 @@ from the `npm` flags with `--`.
| ------------------------------- |
:--------------------------------------------------------------------------------------------------:
|
| params.baseUrl | Environment test run on. Tests are written
for cdn-in-a-box only. Do not run on other environment
|
| capabilities.shardTestFiles | Input `true` or `false` to turn on or off
parallelization. If the value is false, maxInstances will always count as 1.
The default value in the config file = false |
-| capabilities.maxInstances | Input number of chrome instances that your
machine can handle. Test will fail if local machine cannot handle a lot of
chrome instances. The default value = 1 |
+| capabilities.maxInstances | Input number of Chromium instances that
your machine can handle. Test will fail if local machine cannot handle a lot of
Chromium instances. The default value = 1 |
### Example
```bash
