This is an automated email from the ASF dual-hosted git repository. shinrich pushed a commit to branch avx-ats10 in repository https://gitbox.apache.org/repos/asf/trafficserver.git
commit a1f9a051cf930c1400427784012d09e2bd63913f Author: Susan Hinrichs <[email protected]> AuthorDate: Thu Mar 28 21:18:06 2024 +0000 Checkpoint dagger changes --- .gitignore | 2 +- CMakePresets.json | 20 + aviatrix/10_0_0.0001/avx-gw-trafficserver.service | 18 + aviatrix/10_0_0.0001/configs/logging.yaml | 53 ++ aviatrix/10_0_0.0001/configs/plugin.config | 15 + aviatrix/10_0_0.0001/configs/records.yaml | 224 ++++++++ aviatrix/10_0_0.0001/control | 6 + .../10_0_0.0001/logrotate/avx-gw-trafficserver | 11 + aviatrix/10_0_0.0001/postinst | 25 + aviatrix/build/.gitignore | 3 + aviatrix/build/docker_scripts/env_command.sh | 179 ++++++ aviatrix/build/docker_scripts/setup_debug.sh | 3 + aviatrix/build/docker_scripts/setup_local.sh | 94 ++++ aviatrix/build/docker_scripts/setup_release.sh | 3 + aviatrix/build/dockerfile-atsbuild | 116 ++++ aviatrix/build/dockerfile-atsbuildv2 | 61 +++ aviatrix/build/dockerfile-buildtools | 37 ++ aviatrix/build/dockerfile-openssl | 51 ++ aviatrix/build/makeprotos.sh | 3 + aviatrix/build/setup_build_tools.sh | 21 + aviatrix/build/setup_build_tools_1.sh | 66 +++ aviatrix/build/setup_build_tools_2.sh | 24 + aviatrix/dagger/Makefile | 59 ++ aviatrix/dagger/build.go | 605 +++++++++++++++++++++ aviatrix/dagger/build_manifest.sh | 27 + aviatrix/dagger/builder.sh | 13 + aviatrix/dagger/go.mod | 23 + aviatrix/dagger/go.sum | 48 ++ aviatrix/dagger/main.go | 312 +++++++++++ aviatrix/plugins/avx_certifier/CMakeLists.txt | 2 +- aviatrix/plugins/avx_policy_driver/CMakeLists.txt | 23 +- 31 files changed, 2140 insertions(+), 7 deletions(-) diff --git a/.gitignore b/.gitignore index e1fd4bc749..65e49bb2b8 100644 --- a/.gitignore +++ b/.gitignore @@ -44,7 +44,7 @@ m4/ltversion.m4 m4/lt~obsolete.m4 # common directory name for out-of-tree CMake builds -build* +#build* configs/records.yaml.default configs/storage.config.default diff --git a/CMakePresets.json b/CMakePresets.json index 4ea2a27dda..fcf3a3a6e4 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -375,6 +375,26 @@ "CMAKE_CXX_FLAGS_DEBUG": "--coverage", "CMAKE_C_FLAGS_DEBUG": "--coverage" } + }, + { + "name": "aviatrix", + "displayName": "Aviatrix", + "description": "Aviatrix Presets", + "inherits": ["release"], + "generator": "Unix Makefiles", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "RelWithDebInfo", + "jemalloc_ROOT": "/usr", + "ENABLE_LUAJIT": false, + "ENABLE_JEMALLOC": true, + "ENABLE_MIMALLOC": false, + "ENABLE_MALLOC_ALLOCATOR": true, + "ENABLE_AUTEST": false, + "BUILD_EXPERIMENTAL_PLUGINS": false, + "BUILD_REGRESSION_TESTING": false, + "CLOUDN_SOURCE_ROOT": "/src/cloudn", + "CMAKE_INSTALL_PREFIX": "/opt/ats/ats_10.0.0" + } } ], "buildPresets": [ diff --git a/aviatrix/10_0_0.0001/avx-gw-trafficserver.service b/aviatrix/10_0_0.0001/avx-gw-trafficserver.service new file mode 100644 index 0000000000..90fb3b88ee --- /dev/null +++ b/aviatrix/10_0_0.0001/avx-gw-trafficserver.service @@ -0,0 +1,18 @@ +[Unit] +Description=Service that runs the trafficserver + + +[Service] +Type=simple +User=ats-user +ExecStart=-/opt/ats/ats_10.0.0/bin/traffic_server --bind_stdout /var/log/tslogs/traffic.out --bind_stderr /var/log/tslogs/traffic.out +Restart=on-failure +RestartSec=5s +LimitNOFILE=1000000 +PIDFile=/opt/ats/ats_10.0.0/var/trafficserver/server.lock +TimeoutStopSec=5s +#ExecReload=@exp_bindir@/traffic_ctl config reload +KillMode=process + +[Install] +WantedBy=multi-user.target diff --git a/aviatrix/10_0_0.0001/configs/logging.yaml b/aviatrix/10_0_0.0001/configs/logging.yaml new file mode 100755 index 0000000000..087e2cde40 --- /dev/null +++ b/aviatrix/10_0_0.0001/configs/logging.yaml @@ -0,0 +1,53 @@ +# Custom log configuration +# +# Documentation on logging: +# https://docs.trafficserver.apache.org/en/9.0.x/admin-guide/logging/index.en.html +# +# Documentation on logging.yaml file format: +# https://docs.trafficserver.apache.org/en/9.0.x/admin-guide/files/logging.yaml.en.html +# +# Example log configurations: +# https://docs.trafficserver.apache.org/en/9.0.x/admin-guide/logging/examples.en.html + + +logging: + formats: + # WebTrends Enhanced Log Format. + # + # The following is compatible with the WebTrends Enhanced Log Format. + # If you want to generate a log that can be parsed by WebTrends + # reporting tools, simply create a log that uses this format. + - name: welf + format: |- + id=firewall time="%<cqtd> %<cqtt>" fw=%<phn> pri=6 proto=%<cqus> duration=%<ttmsf> sent=%<psql> rcvd=%<cqhl> src=%<chi> dst=%<shi> dstname=%<shn> user=%<caun> op=%<cqhm> arg="%<cqup>" result=%<pssc> ref="%<{Referer}cqh>" agent="%<{user-agent}cqh>" cache=%<crc> + # Squid Log Format with seconds resolution timestamp. + # The following is the squid format but with a seconds-only timestamp + # (cqts) instead of a seconds and milliseconds timestamp (cqtq). + - name: squid_seconds_only_timestamp + format: '%<cqts> %<ttms> %<chi> %<crc>/%<pssc> %<psql> %<cqhm> %<cquc> %<caun> %<phr>/%<shn> %<psct>' + + # Squid Log Format. + - name: squid + format: '%<cqtq> %<ttms> %<chi> %<crc>/%<pssc> %<psql> %<cqhm> %<cquc> %<caun> %<phr>/%<shn> %<psct>' + + - name: aviatrix + format: 'ts=%<cqtq> ttms=%<ttms> sni=%<cssn> meth=%<cqhm> cqpv=%<cqpv> pssc=%<pssc> cqtr=%<cqtr> cqssl=%<cqssl> chi=%<chi> cqql=%<cqql> shi=%<shi> shn=%<shn> psql=%<psql> sstc=%<sstc> cquc=%<cquc[:512]>' + + # Common Log Format. + - name: common + format: '%<chi> - %<caun> [%<cqtn>] "%<cqtx>" %<pssc> %<pscl>' + + # Extended Log Format. + - name: 'extended' + format: '%<chi> - %<caun> [%<cqtn>] "%<cqtx>" %<pssc> %<pscl> %<sssc> %<sscl> %<cqcl> %<pqcl> %<cqhl> %<pshl> %<pqhl> %<sshl> %<tts>' + + # Extended2 Log Formats + - name: "extended2" + format: '%<chi> - %<caun> [%<cqtn>] "%<cqtx>" %<pssc> %<pscl> %<sssc> %<sscl> %<cqcl> %<pqcl> %<cqhl> %<pshl> %<pqhl> %<sshl> %<tts> %<phr> %<cfsc> %<pfsc> %<crc>' + + logs: + - filename: access + format: aviatrix + mode: text + +# vim: set ft=yaml : diff --git a/aviatrix/10_0_0.0001/configs/plugin.config b/aviatrix/10_0_0.0001/configs/plugin.config new file mode 100644 index 0000000000..1700b38b68 --- /dev/null +++ b/aviatrix/10_0_0.0001/configs/plugin.config @@ -0,0 +1,15 @@ +# +# plugin.config +# +# Documentation: +# https://docs.trafficserver.apache.org/en/latest/admin-guide/files/plugin.config.en.html +# +# Comments start with a '#' and continue to the end of the line +# Blank lines are ignored + +avx_policy_driver.so 5557 60 +# +avx_tee_decrypt.so 127.255.0.10 127.255.0.20 +# +#use certifier to automatically run with MITM +avx_certifier.so --sign-cert=/opt/ats/ats_9.1.3/etc/local_ca/mitm_ca.cert --sign-key=/opt/ats/ats_9.1.3/etc/local_ca/mitm_ca.key --sign-serial=/opt/ats/ats_9.1.3/var/local_ca/serial.txt --store=/opt/ats/ats_9.1.3/var/local_ca/keys --max=200 diff --git a/aviatrix/10_0_0.0001/configs/records.yaml b/aviatrix/10_0_0.0001/configs/records.yaml new file mode 100644 index 0000000000..26a020f159 --- /dev/null +++ b/aviatrix/10_0_0.0001/configs/records.yaml @@ -0,0 +1,224 @@ +############################################################################## +# *NOTE*: All options covered in this file should be documented in the docs: +# +# https://docs.trafficserver.apache.org/en/latest/admin-guide/files/records.yaml.en.html +############################################################################## + +ts: + accept_threads: 1 + cache: + limits: + http: + +# https://docs.trafficserver.apache.org/en/latest/admin-guide/files/records.yaml.en.html#proxy-config-cache-limits-http-max-alts + max_alts: 5 + log: + alternate: + +# https://docs.trafficserver.apache.org/en/latest/admin-guide/files/records.yaml.en.html#proxy-config-cache-log-alternate-eviction + eviction: 0 + +# https://docs.trafficserver.apache.org/en/latest/admin-guide/files/records.yaml.en.html#proxy-config-cache-max-doc-size + max_doc_size: 0 + min_average_object_size: 8000 + +############################################################################## +# RAM and disk cache configurations. Docs: +# https://docs.trafficserver.apache.org/en/latest/admin-guide/files/records.yaml.en.html#ram-cache +# https://docs.trafficserver.apache.org/en/latest/admin-guide/files/storage.config.en.html +############################################################################## + ram_cache: + size: -1 + ram_cache_cutoff: 4194304 + threads_per_disk: 8 +############################################################################## +# Debugging. Docs: +# https://docs.trafficserver.apache.org/en/latest/admin-guide/files/records.yaml.en.html#diagnostic-logging-configuration +############################################################################## + diags: + debug: + enabled: 0 + tags: http|dns + +# https://docs.trafficserver.apache.org/en/latest/admin-guide/files/records.yaml.en.html#proxy-config-dump-mem-info-frequency + dump_mem_info_frequency: 0 + +############################################################################## +# Thread configurations. Docs: +# https://docs.trafficserver.apache.org/en/latest/admin-guide/files/records.yaml.en.html#thread-variables +############################################################################## + exec_thread: + affinity: 1 + autoconfig: + enabled: 1 + scale: 1.0 + limit: 2 + http: + accept_no_activity_timeout: 120 + cache: + cache_responses_to_cookies: 1 + cache_urls_that_look_dynamic: 1 + +############################################################################## +# Heuristic cache expiration. Docs: +# https://docs.trafficserver.apache.org/en/latest/admin-guide/files/records.yaml.en.html#heuristic-expiration +############################################################################## + heuristic_lm_factor: 0.1 + heuristic_max_lifetime: 86400 + heuristic_min_lifetime: 3600 + +############################################################################## +# Enable / disable HTTP caching. Useful for testing, but also as an +# overridable (per remap) config +############################################################################## + http: 0 + +############################################################################## +# Cache control. Docs: +# https://docs.trafficserver.apache.org/en/latest/admin-guide/files/records.yaml.en.html#cache-control +# https://docs.trafficserver.apache.org/en/latest/admin-guide/files/cache.config.en.html +############################################################################## + ignore_client_cc_max_age: 1 + +# https://docs.trafficserver.apache.org/en/latest/admin-guide/files/records.yaml.en.html#proxy-config-http-cache-required-headers + required_headers: 2 + +# https://docs.trafficserver.apache.org/en/latest/admin-guide/files/records.yaml.en.html#proxy-config-http-cache-when-to-revalidate + when_to_revalidate: 0 + +############################################################################## +# Origin server connect attempts. Docs: +# https://docs.trafficserver.apache.org/en/latest/admin-guide/files/records.yaml.en.html#origin-server-connect-attempts +############################################################################## + connect_attempts_max_retries: 0 + connect_attempts_max_retries_down_server: 1 + connect_attempts_rr_retries: 3 + connect_attempts_timeout: 30 + down_server: + cache_time: 60 + forward: + proxy_auth_to_parent: 0 + +############################################################################## +# Proxy users variables. Docs: +# https://docs.trafficserver.apache.org/en/latest/admin-guide/files/records.yaml.en.html#proxy-user-variables +############################################################################## + insert_client_ip: 0 + +############################################################################## +# Via: headers. Docs: +# https://docs.trafficserver.apache.org/en/latest/admin-guide/files/records.yaml.en.html#proxy-config-http-insert-response-via-str +############################################################################## + insert_request_via_str: 1 + insert_response_via_str: 0 + insert_squid_x_forwarded_for: 1 + keep_alive_no_activity_timeout_in: 120 + keep_alive_no_activity_timeout_out: 120 + +############################################################################## +# Negative response caching, for redirects and errors. Docs: +# https://docs.trafficserver.apache.org/en/latest/admin-guide/files/records.yaml.en.html#negative-response-caching +############################################################################## + negative_caching_enabled: 0 + negative_caching_lifetime: 1800 + normalize_ae: 1 + +############################################################################## +# Parent proxy configuration, in addition to these settings also see parent.config. Docs: +# https://docs.trafficserver.apache.org/en/latest/admin-guide/files/records.yaml.en.html#parent-proxy-configuration +# https://docs.trafficserver.apache.org/en/latest/admin-guide/files/parent.config.en.html +############################################################################## + parent_proxy: + retry_time: 300 + +############################################################################## +# Security. Docs: +# https://docs.trafficserver.apache.org/en/latest/admin-guide/files/records.yaml.en.html#security +############################################################################## + push_method_enabled: 0 + +############################################################################## +# Specify server addresses and ports to bind for HTTP and HTTPS. Docs: +# https://docs.trafficserver.apache.org/en/latest/admin-guide/files/records.yaml.en.html#proxy.config.http.server_ports +############################################################################## + server_ports: 8443:ssl:tr-full:allow-plain + +# https://docs.trafficserver.apache.org/en/latest/admin-guide/files/records.yaml.en.html#proxy-config-http-slow-log-threshold + slow: + log: + threshold: 0 + +############################################################################## +# HTTP connection timeouts (secs). Docs: +# https://docs.trafficserver.apache.org/en/latest/admin-guide/files/records.yaml.en.html#http-connection-timeouts +############################################################################## + transaction_active_timeout_in: 0 + transaction_active_timeout_out: 0 + # It seems that the transaction timeout is used for SNI tunneling as well + transaction_no_activity_timeout_in: 1200 # 20 minutes + transaction_no_activity_timeout_out: 1200 + uncacheable_requests_bypass_parent: 1 + +############################################################################## +# Logging Config. Docs: +# https://docs.trafficserver.apache.org/en/latest/admin-guide/files/records.yaml.en.html#logging-configuration +# https://docs.trafficserver.apache.org/en/latest/admin-guide/files/logging.yaml.en.html +############################################################################## + log: + auto_delete_rolled_files: 1 + logging_enabled: 3 + max_space_mb_for_logs: 25000 + max_space_mb_headroom: 1000 + periodic_tasks_interval: 5 + rolling_enabled: 0 + #rolling_interval_sec: 86400 + #rolling_size_mb: 10 + +############################################################################## +# Network. Docs: +# https://docs.trafficserver.apache.org/en/latest/admin-guide/files/records.yaml.en.html#network +############################################################################## + net: + connections_throttle: 30000 + default_inactivity_timeout: 86400 + max_connections_in: 30000 + max_requests_in: 0 + +# https://docs.trafficserver.apache.org/en/latest/admin-guide/files/records.yaml.en.html#proxy-config-res-track-memory + res_track_memory: 0 + +# https://docs.trafficserver.apache.org/en/latest/admin-guide/files/records.yaml.en.html#reverse-proxy + reverse_proxy: + enabled: 0 + ssl: + client: + CA: + cert: + filename: mitm-trustbundle.crt + +############################################################################## +# SSL Termination. Docs: +# https://docs.trafficserver.apache.org/en/latest/admin-guide/files/records.yaml.en.html#client-related-configuration +# https://docs.trafficserver.apache.org/en/latest/admin-guide/files/ssl_multicert.config.en.html +############################################################################## + verify: + server: + policy: PERMISSIVE + properties: SIGNATURE + server: + private_key: + path: ssl + cert: + path: ssl + task_threads: 2 + +############################################################################## +# These settings control remapping, and if the proxy allows (open) forward proxy or not. Docs: +# https://docs.trafficserver.apache.org/en/latest/admin-guide/files/records.yaml.en.html#url-remap-rules +# https://docs.trafficserver.apache.org/en/latest/admin-guide/files/remap.config.en.html +############################################################################## + url_remap: + +# https://docs.trafficserver.apache.org/en/latest/admin-guide/files/records.yaml.en.html#proxy-config-url-remap-pristine-host-hdr + pristine_host_hdr: 0 + remap_required: 0 diff --git a/aviatrix/10_0_0.0001/control b/aviatrix/10_0_0.0001/control new file mode 100644 index 0000000000..6b91ce5eb3 --- /dev/null +++ b/aviatrix/10_0_0.0001/control @@ -0,0 +1,6 @@ +Package: ats +Version: 10.0.0 +Architecture: amd64 +Maintainer: Aviatrix Engineering <[email protected]> +Description: Aviatrix variant of Apache TrafficServer +Depends: libunwind8,libnuma1,libcap2-bin diff --git a/aviatrix/10_0_0.0001/logrotate/avx-gw-trafficserver b/aviatrix/10_0_0.0001/logrotate/avx-gw-trafficserver new file mode 100644 index 0000000000..ff97959ddc --- /dev/null +++ b/aviatrix/10_0_0.0001/logrotate/avx-gw-trafficserver @@ -0,0 +1,11 @@ +/var/log/tslogs/access.log /var/log/tslogs/diags.log { + su ats-user ats-user + rotate 12 + daily + size 5M + missingok + notifempty + compress + copytruncate +} + diff --git a/aviatrix/10_0_0.0001/postinst b/aviatrix/10_0_0.0001/postinst new file mode 100755 index 0000000000..73d18852b1 --- /dev/null +++ b/aviatrix/10_0_0.0001/postinst @@ -0,0 +1,25 @@ +#!/bin/bash + +atsdir='/opt/ats/ats_10.0.0' + +#creating log directory and setting permissions +test -e /var/log/tslogs || mkdir -p /var/log/tslogs +useradd --shell /usr/sbin/nologin ats-user +chown ats-user:ats-user /var/log/tslogs +chmod 775 /var/log/tslogs + +#creating trustbundle directory +mkdir -p $atsdir/certs + +#fixing owners and setting permissions +chown -R ats-user:ats-user $atsdir +chown root:ats-user $atsdir/etc/local_ca +chmod 750 $atsdir/etc/local_ca +chown -R root:ats-user $atsdir/var/local_ca +chmod -R 770 $atsdir/var/local_ca + +#fixing permissions for logrotate conf +chown root:root /etc/logrotate.d/avx-gw-trafficserver +chmod 644 /etc/logrotate.d/avx-gw-trafficserver + +setcap cap_net_admin,cap_net_raw+eip $atsdir/bin/traffic_server diff --git a/aviatrix/build/.gitignore b/aviatrix/build/.gitignore new file mode 100644 index 0000000000..d9b5b0e6a6 --- /dev/null +++ b/aviatrix/build/.gitignore @@ -0,0 +1,3 @@ +debs +temp +logs diff --git a/aviatrix/build/docker_scripts/env_command.sh b/aviatrix/build/docker_scripts/env_command.sh new file mode 100644 index 0000000000..548635859d --- /dev/null +++ b/aviatrix/build/docker_scripts/env_command.sh @@ -0,0 +1,179 @@ +set -xeuo + +export OPENSSL=/opt/ssl/openssl_1.1.1 +export DEB=ats_9.1.3 + + +arg3=${3:-} +arg4=${4:-} +arg5=${5:-} + +if [ -z $1 ]; then + echo script needs environment + exit +fi +environment=$1 +if [ -z $2 ]; then + exit +fi +command=`echo $2 | grep -E '^build|plugins|clean$'` + + +if [ -z "$command" ]; then + echo use clean, build, plugins + exit +fi + + + +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) + +source $SCRIPT_DIR/setup_$environment.sh + +if [ "$command" == "plugins" ] && [ ! -e "$ats_install_path/bin" ]; then + command="build" +fi + +if [ $2 == "build" ]; then + command=build + if [ ! -e "/app/build/$environment/thirdparty-trafficserver" ]; then + command=clean + fi +fi + + + +if [ $2 == "clean" ]; then + command=$2 +fi +echo Command: $command + + +if [ "$ats_install_path" == "" ]; then + echo bailing, no ats install path +fi + +echo $ats_install_path + +mkdir -p /app/build/$environment/thirdparty-trafficserver +cd /app/build/$environment/thirdparty-trafficserver +if [ -e "/app/build/$environment/cloudn" ]; then + unlink "/app/build/$environment/cloudn" +fi +ln -s /app_data/cloudn /app/build/$environment/cloudn 2>/dev/null +if [ "$command" == "clean" ]; then + echo cleaning up + rm -rf * +fi + +echo syncing +tar xf /app/source/copy-for-docker.tgz + + +if [ $2 == 'shell' ]; then + exit +fi + +cd /app/build/$environment/thirdparty-trafficserver/plugins/experimental/policy_driver +make -f Makefile-proto +cd /app/build/$environment/thirdparty-trafficserver + +if [ $command == "shell" ]; then + cd "/app/build/$1/thirdparty-trafficserver/plugins/" + bash + exit +fi + +if [ $command == "clean" ]; then + + autoreconf -if + ./configure $(echo $configure_with) + +fi + + +PKG_CONFIG_PATH=/root/.local/lib/pkgconfig make -j 8 +current_install="" +if [ ! -e "$ats_install_path/environment"]; then + current_install=`cat $ats_install_path/environment` +fi +echo updating ats_install_path +echo $command +echo $current_install +if [ "$command" == "clean" ] || [ "$current_install" != "$1" ]; then + echo deleting $ats_install_path for $1 $2 + # our current install was not same environment, need a complete build + if [ "$command" == "plugins" ]; then + command="build" + fi + if [ "$ats_install_path" != "" ]; then + rm -rf $ats_install_path/* + fi +fi + +mkdir -p $ats_install_path/lib/ +cp /usr/lib/x86_64-linux-gnu/libhwloc.so.15 $ats_install_path/lib/ +cp /usr/lib/x86_64-linux-gnu/libhwloc.so.15.5.2 $ats_install_path/lib/ +ln -sf $ats_install_path/lib/libhwloc.so.15.5.2 $ats_install_path/lib/libhwloc.so.15 + + +printf $1 > $ats_install_path/environment +cp /app/build/$environment/thirdparty-trafficserver/avx-manifest $ats_install_path/ats_9.1.3.deb.manifest +if [ "$command" == "plugins" ]; then + pushd plugins + PKG_CONFIG_PATH=/root/.local/lib/pkgconfig make -j 8 + PKG_CONFIG_PATH=/root/.local/lib/pkgconfig make -j 8 install + popd +else + PKG_CONFIG_PATH=/root/.local/lib/pkgconfig make -j 8 + PKG_CONFIG_PATH=/root/.local/lib/pkgconfig make -j 8 install + if [ "$1" == "debug" ] || [ "$arg3" == "test_client" ] || [ "$arg4" == "test_client" ] || [ "$arg5" == "test_client" ] ; then + cp /app/build/$environment/thirdparty-trafficserver/plugins/experimental/policy_driver/test_client $ats_install_path/bin + cp /app/build/$environment/thirdparty-trafficserver/plugins/experimental/policy_driver/test_server $ats_install_path/bin + fi +fi + + +if [ "$build_deb" == "true" ]; then + mkdir -p /build + cd /build + + + mkdir $DEB && mkdir -p $DEB/opt/ats && mkdir -p $DEB/lib/systemd/system && mkdir -p $DEB/etc/logrotate.d && mkdir $DEB/DEBIAN && cd $DEB/DEBIAN && \ + printf "package: ats\nversion: 9.1.3\nmaintainer: Kasun\narchitecture: all\ndescription: testing it\n" > control + mkdir -p /build/$DEB/lib/systemd/system + mkdir -p /build/$DEB/etc/ + mkdir -p /build/$DEB/DEBIAN + + cp -r /app/source/aviatrix/9_1_3.0001/configs/* ${ats_install_path}/etc/trafficserver + cp /app/source/aviatrix/9_1_3.0001/avx-gw-trafficserver.service /build/$DEB/lib/systemd/system + cp /app/source/aviatrix/9_1_3.0001/logrotate/avx-gw-trafficserver /build/$DEB/etc/logrotate.d + cp /app/source/aviatrix/9_1_3.0001/postinst /build/$DEB/DEBIAN + mkdir -p $ats_install_path/etc/local_ca + mkdir -p $ats_install_path/var/local_ca/keys + echo 12345 > $ats_install_path/var/local_ca/serial.txt + mkdir -p /build/$DEB/opt/ats/ + + cp -R $ats_install_path /build/$DEB/opt/ats/ + chown root:root /build/$DEB/DEBIAN/postinst && chmod 0775 /build/$DEB/DEBIAN/postinst + cd /build/$DEB + tar cf ../plugin_ats_9.1.3.tgz opt/ats/ats_9.1.3/libexec/trafficserver/avx_certifier.* opt/ats/ats_9.1.3/libexec/trafficserver/policy_driver.* + cp /build/plugin_ats_9.1.3.tgz /debs + tar cf ../ats_9.1.3.tgz opt/ats/ats_9.1.3/libexec/* opt/ats/ats_9.1.3/bin/* opt/ats/ats_9.1.3/lib/* + cp /build/ats_9.1.3.tgz /debs + cp /app/build/$environment/thirdparty-trafficserver/avx-manifest /debs/ats_9.1.3.deb.manifest + cd /build + + + if [ "$arg3" != "nodeb" ]; then + time dpkg-deb --build $DEB + cp /build/ats_9.1.3.deb /debs + fi +fi + + + +if [ "$arg3" == "shell" ] || [ "$arg4" == "shell" ] || [ "$arg5" == "shell" ]; then + cd /app/build/release/thirdparty-trafficserver/plugins/ + bash +fi diff --git a/aviatrix/build/docker_scripts/setup_debug.sh b/aviatrix/build/docker_scripts/setup_debug.sh new file mode 100644 index 0000000000..a14400d794 --- /dev/null +++ b/aviatrix/build/docker_scripts/setup_debug.sh @@ -0,0 +1,3 @@ +export ats_install_path=/opt/ats/ats_9.1.3 +export configure_with="--with-user=ubuntu --enable-debug --with-openssl=$OPENSSL --enable-tproxy --prefix=$ats_install_path --disable-dependency-checking --enable-example-plugins --enable-experimental-plugins" +export build_deb=true diff --git a/aviatrix/build/docker_scripts/setup_local.sh b/aviatrix/build/docker_scripts/setup_local.sh new file mode 100644 index 0000000000..800a3c1100 --- /dev/null +++ b/aviatrix/build/docker_scripts/setup_local.sh @@ -0,0 +1,94 @@ +#!/bin/bash +export ats_install_path=/opt/ats/ats_local_debug +export configure_with="--with-user=ubuntu --enable-debug --with-openssl=$OPENSSL --enable-tproxy --prefix=$ats_install_path --disable-dependency-checking --enable-example-plugins --enable-experimental-plugins" +export build_deb=false + +function apply_netmask { + # Parse IP and netmask from input + ip=$(echo "$1" | cut -d/ -f1) + netmask=$(echo "$1" | cut -d/ -f2) + + # Convert netmask to bitmask + bitmask=$((0xffffffff << (32 - netmask))) + + # Convert IP address to integer + IFS='.' read -r i1 i2 i3 i4 <<< "$ip" + int=$((i1 * 256 ** 3 + i2 * 256 ** 2 + i3 * 256 + i4)) + + # Apply bitmask to integer + masked_int=$((int & bitmask)) + + # Convert masked integer back to IP address + masked_ip=$(printf "%d.%d.%d.%d" \ + $((masked_int >> 24)) \ + $((masked_int >> 16 & 255)) \ + $((masked_int >> 8 & 255)) \ + $((masked_int & 255))) + + # Return masked IP address + echo "$masked_ip/$netmask" +} + + +function setupnet { + local ns=$1 + local veth1=$2 + local veth2=$4 + local ip1=$3 + local ip2=$5 + local cleanip1=$(printf $3 | sed -r 's#(.*)/.*#\1#') + local cleanip2=$(printf $4 | sed -r 's#(.*)/.*#\1#') + local cleanroutenetmask=$(apply_netmask $3) + echo $ns $veth1 $veth2 $ip1 $ip2 $cleanip1 $cleanip2 + # create network namespace + ip netns add $ns + # create interfaces + ip link add $veth1 type veth peer name $veth121 + # set outside interface ip address + ip addr add $ip1 dev $veth1 + # up outside facing interface + ip link set $veth1 up + # move inner interface to network namespace + ip link set $veth2 netns $ns + # set ip address + ip netns exec $ns ip addr add $ip2 dev $veth2 + # ip inner interface + ip netns exec $ns ip link set $veth2 up + # add the route for the inner network + ip route add $cleanroutenetmask dev $veth1 + # add the default route for the network namespace to the outside + ip netns exec $ns ip route add default via $cleanip1 dev $veth2 onlink + +} + +setupnet "net1011" "veth111" "10.11.0.1/24" "veth112" "10.11.0.2/24" +setupnet "net1012" "veth121" "10.12.0.1/24" "veth122" "10.12.0.2/24" + + + + +echo ' +escape ^Xa +# skip the startup message +startupmessage off + +# Display a caption string below, appearing like tabs and +# displaying the window number and application name (by default). +caption always +caption string "Use ctrl-x as screen key %{kw}%-w%{wr}%n %t%{-}%+w" +# +# j to move down a window, k to move up. Like teh VIM! +bind j focus down +bind k focus up +bind > focus next +bind < focus prev +# +# Default screens +screen -t main +screen -t net1011 +screen -t net1012 ls ; bash +screen -t nginx1012 +# +# Select first screen +select 0 +' > ~/.screenrc \ No newline at end of file diff --git a/aviatrix/build/docker_scripts/setup_release.sh b/aviatrix/build/docker_scripts/setup_release.sh new file mode 100644 index 0000000000..1f1760bf67 --- /dev/null +++ b/aviatrix/build/docker_scripts/setup_release.sh @@ -0,0 +1,3 @@ +export ats_install_path=/opt/ats/ats_9.1.3 +export configure_with="--with-user=ubuntu --with-openssl=$OPENSSL --enable-tproxy --prefix=$ats_install_path --disable-dependency-checking --enable-example-plugins --enable-experimental-plugins" +export build_deb=true diff --git a/aviatrix/build/dockerfile-atsbuild b/aviatrix/build/dockerfile-atsbuild new file mode 100644 index 0000000000..bba6f2387c --- /dev/null +++ b/aviatrix/build/dockerfile-atsbuild @@ -0,0 +1,116 @@ + +FROM --platform=linux/amd64 ubuntu:22.04 AS build-stage + + +ENV PATH="/root/.local/bin:${PATH}" +ENV PKG_CONFIG_PATH="/root/.local/lib/pkgconfig" + +SHELL ["/bin/bash", "-c"] + +#RUN --mount=type=bind,source=/home/ubuntu/dockerdir/target,target=/app + +#ADD ./target /app + +RUN useradd -ms /bin/bash -g root -G sudo ubuntu + +WORKDIR /build + +ENV DEB=ats_9.1.3 + + +ENV ATSPATH_SOURCE=. +ENV ATSPATH=/opt/ats/ats_9.1.3 + +ENV ARTIFACTS_SOURCE=aviatrix/build +ENV ARTIFACTS=/app/build/ + +ENV OPENSSL=/opt/ssl/openssl_1.1.1 + +#RUN ATSPATH=/opt/ats/ats_9.1.3 && DEB=ats + +#RUN echo ats_path > check.txt && echo deb_name >> check.txt + +RUN mkdir $DEB && mkdir -p $DEB/opt/ats && mkdir -p $DEB/lib/systemd/system && mkdir -p $DEB/etc/logrotate.d && mkdir $DEB/DEBIAN && cd $DEB/DEBIAN && \ + printf "package: ats\nversion: 9.1.3\nmaintainer: Kasun\narchitecture: all\ndescription: testing it\n" > control + +#USER root + +RUN rm -f /etc/apt/apt.conf.d/docker-clean +RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > \ + /etc/apt/apt.conf.d/keep-cache + +RUN --mount=type=cache,target=/var/cache/apt \ + apt update && \ + apt install -y autoconf \ + automake \ + libtool \ + pkg-config \ + libmodule-install-perl \ + gcc \ + g++ \ + zlib1g-dev \ + libssl-dev \ + libcap-dev \ + libhwloc-dev \ + libncurses5-dev \ + libcurl4-openssl-dev \ + flex \ + make \ + patchelf + +#USER root +RUN --mount=type=bind,rw,source=.,target=/app/thirdparty-trafficserver \ + --mount=type=bind,rw,source=aviatrix/build/debs,target=/app/build \ + cd / ; \ + tar xvzf /app/build/buildtools.tgz && \ + echo $PATH +RUN echo $PKG_CONFIG_PATH +RUN ls /root/.local/lib/pkgconfig + +RUN --mount=type=bind,rw,source=.,target=/app/thirdparty-trafficserver \ + --mount=type=bind,rw,source=aviatrix/build/temp/cloudn,target=/app/cloudn \ + --mount=type=bind,rw,source=aviatrix/build/debs,target=/app/build \ + cd /app/thirdparty-trafficserver/plugins/experimental/policy_driver && \ + make -f Makefile-proto && \ + cd /app/thirdparty-trafficserver && \ + dpkg -i /app/build/openssl_1.1.1.deb && \ + echo $PATH && \ + patchelf --set-rpath $OPENSSL/lib $OPENSSL/bin/openssl && \ + autoreconf -if && \ + ./configure --with-user=ubuntu --with-openssl=$OPENSSL --enable-tproxy --prefix=$ATSPATH --disable-dependency-checking --enable-example-plugins --enable-experimental-plugins && \ + PKG_CONFIG_PATH=/root/.local/lib/pkgconfig make -j 8 install && \ + cp -r aviatrix/9_1_3.0001/configs/* $ATSPATH/etc/trafficserver && \ + cp aviatrix/9_1_3.0001/avx-gw-trafficserver.service /build/$DEB/lib/systemd/system && \ + cp aviatrix/9_1_3.0001/logrotate/avx-gw-trafficserver /build/$DEB/etc/logrotate.d + +#remounting here so we won't have to wait for trafficserver compilation to add changes to post install script +RUN --mount=type=bind,rw,source=.,target=/app/thirdparty-trafficserver \ + --mount=type=bind,rw,source=aviatrix/build/debs,target=/app/build \ + cd /app/thirdparty-trafficserver && \ + cp aviatrix/9_1_3.0001/postinst /build/$DEB/DEBIAN + +RUN cp /usr/lib/x86_64-linux-gnu/libhwloc.so.5 $ATSPATH/lib/ && cp /usr/lib/x86_64-linux-gnu/libhwloc.so.5.7.6 $ATSPATH/lib/ && ln -sf $ATSPATH/lib/libhwloc.so.5.7.6 $ATSPATH/lib/libhwloc.so.5 + +RUN mkdir $ATSPATH/etc/local_ca && mkdir -p $ATSPATH/var/local_ca/keys && echo 12345 > $ATSPATH/var/local_ca/serial.txt + +#RUN mkdir -p /build/ats/opt/ats/ && mv /opt/ts_run /opt/ats_9.1.3 && mv /opt/ats_9.1.3 /build/ats/opt/ats/ats_9.1.3 + +RUN mv $ATSPATH /build/$DEB/opt/ats/ + +RUN chown root:root /build/$DEB/DEBIAN/postinst && chmod 0775 /build/$DEB/DEBIAN/postinst + +WORKDIR /build + +RUN dpkg-deb --build $DEB + +#RUN ls /usr/lib/x86_64-linux-gnu | grep libhwloc > check.txt + +#RUN cat /app/target/tosound > temp + +#RUN ls /app >> temp + +FROM scratch AS export-stage + +COPY --from=build-stage /build/ats_9.1.3.deb . + +#COPY --from=build-stage /build/check.txt /build diff --git a/aviatrix/build/dockerfile-atsbuildv2 b/aviatrix/build/dockerfile-atsbuildv2 new file mode 100644 index 0000000000..7f0f740c9a --- /dev/null +++ b/aviatrix/build/dockerfile-atsbuildv2 @@ -0,0 +1,61 @@ +FROM --platform=linux/amd64 ubuntu:22.04 AS build-stage +ENV PATH="/root/.local/bin:${PATH}" +ENV PKG_CONFIG_PATH="/root/.local/lib/pkgconfig" +ENV ATSPATH_SOURCE=. +ENV ATSPATH=/opt/ats/ats_9.1.3 +ENV DEB=ats_9.1.3 + +ENV ARTIFACTS_SOURCE=aviatrix/build +ENV ARTIFACTS=/app/build/ + +ENV OPENSSL=/opt/ssl/openssl_1.1.1 + +RUN echo 'Binary::apt::APT::Keep-Downloaded-Packages "true";' > \ + /etc/apt/apt.conf.d/keep-cache + +RUN \ + apt update && \ + apt install -y autoconf \ + automake \ + libtool \ + pkg-config \ + libmodule-install-perl \ + gcc \ + g++ \ + zlib1g-dev \ + libssl-dev \ + libcap-dev \ + libhwloc-dev \ + libncurses5-dev \ + libcurl4-openssl-dev \ + flex \ + make \ + patchelf \ + iproute2 \ + less \ + inetutils-ping \ + screen \ + nginx \ + curl \ + wget \ + git + +RUN --mount=type=bind,rw,source=.,target=/app/thirdparty-trafficserver \ + --mount=type=bind,rw,source=aviatrix/build/u22/debs,target=/app/build \ + cd / ; \ + tar xvzf /app/build/buildtools.tgz + +RUN --mount=type=bind,rw,source=.,target=/app/thirdparty-trafficserver \ + --mount=type=bind,rw,source=aviatrix/build/u22/debs,target=/app/build \ + dpkg -i /app/build/openssl_1.1.1.deb && \ + echo $PATH && \ + patchelf --set-rpath $OPENSSL/lib $OPENSSL/bin/openssl + +SHELL ["/bin/bash", "-c"] +RUN echo > /test +RUN useradd -ms /bin/bash -g root -G sudo ubuntu +RUN SNIPPET="export PROMPT_COMMAND='history -a' && export HISTFILE=/app/build/.bash_history" \ + && echo "$SNIPPET" >> "/root/.bashrc" + + +CMD ["/bin/bash"] diff --git a/aviatrix/build/dockerfile-buildtools b/aviatrix/build/dockerfile-buildtools new file mode 100644 index 0000000000..0a7c0651db --- /dev/null +++ b/aviatrix/build/dockerfile-buildtools @@ -0,0 +1,37 @@ + +FROM --platform=linux/amd64 ubuntu:22.04 AS build-stage +SHELL ["/bin/bash", "-c"] + +WORKDIR /build + + + +ENV PATH="/root/.local/bin:${PATH}" +ENV PKG_CONFIG_PATH="/root/.local/lib/pkgconfig" +ENV libswoc_version=1.5.1 +ENV random=1 + + +ENV DIR= + +RUN --mount=type=cache,target=/var/cache/apt \ + apt update && \ + apt install -y make git g++ gcc sudo zlib1g-dev build-essential autoconf libtool pkg-config lsb-release + +COPY aviatrix/build/setup_build_tools*.sh /build + + +RUN --mount=type=cache,target=/var/cache/apt \ + /build/setup_build_tools_1.sh /build + +RUN /build/setup_build_tools_2.sh /build + +RUN ls -l /root/.local/bin + +WORKDIR /build + +RUN tar cvfz /build/buildtools.tgz /root/.local + +FROM scratch AS export-stage + +COPY --from=build-stage /build/buildtools.tgz . \ No newline at end of file diff --git a/aviatrix/build/dockerfile-openssl b/aviatrix/build/dockerfile-openssl new file mode 100644 index 0000000000..8d088bec9b --- /dev/null +++ b/aviatrix/build/dockerfile-openssl @@ -0,0 +1,51 @@ + + +FROM --platform=linux/amd64 ubuntu:22.04 AS build-stage + +#RUN --mount=type=bind,source=/home/ubuntu/dockerdir/target,target=/app + +#ADD ./target /app + +WORKDIR /build + +ENV DIR=/opt/ssl/openssl_1.1.1 + +ENV DEB=openssl_1.1.1 + +RUN mkdir $DEB && mkdir -p $DEB/opt/ssl && mkdir $DEB/DEBIAN && cd $DEB/DEBIAN && \ + printf "package: openssldeb\nversion: 1.1.1\nmaintainer: Kasun\narchitecture: all\ndescription: Open SSL 1.1.1\nupstream_version:1.1.1t\n" > control + +#RUN echo 'check line' > here.txt + +RUN apt update && apt install -y \ + curl \ + perl \ + ca-certificates \ + make \ + gcc \ + patchelf \ + git + +#RUN curl -L https://cpanmin.us/ -o cpanm + +#RUN chmod +x cpanm + +#RUN --mount=type=bind,source=target,target=/app cp /app/cpanm /usr/local/bin/cpanm + +RUN git clone -b OpenSSL_1_1_1t https://github.com/openssl/openssl /app/openssl + +#WORKDIR /app/openssl + +RUN /app/openssl/config --prefix=$DIR --openssldir=$DIR && make -j4 && make -j4 test && make install_sw + +RUN patchelf --set-rpath $DIR/lib $DIR/bin/openssl + +RUN mv $DIR /build/$DEB$DIR + +WORKDIR /build + +RUN dpkg-deb --build $DEB + +FROM scratch AS export-stage + +COPY --from=build-stage /build/openssl_1.1.1.deb . \ No newline at end of file diff --git a/aviatrix/build/makeprotos.sh b/aviatrix/build/makeprotos.sh new file mode 100644 index 0000000000..e1ad14bfeb --- /dev/null +++ b/aviatrix/build/makeprotos.sh @@ -0,0 +1,3 @@ +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +cd $SCRIPT_DIR/../../plugins/experimental/policy_driver && \ + make -f Makefile-proto \ No newline at end of file diff --git a/aviatrix/build/setup_build_tools.sh b/aviatrix/build/setup_build_tools.sh new file mode 100755 index 0000000000..d2311649c2 --- /dev/null +++ b/aviatrix/build/setup_build_tools.sh @@ -0,0 +1,21 @@ +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +BUILD_DIR=${SCRIPT_DIR}/../../../trafficserver-buildtools + +path_has_local_bin=`echo $PATH|grep -E "/\.local/bin"` +echo $path_has_local_bin +has_error='' +if [ ! -e ~/.local/bin ]; then + echo Creating "\"\$HOME/.local/bin\"" + mkdir -p ~/.local/bin + has_error=true +fi +if [ -z "$path_has_local_bin" ]; then + export PATH=~/.local/bin/:$PATH +fi + +export PKG_CONFIG_PATH=~/.local/lib/pkgconfig + +#get the stuff +$SCRIPT_DIR/setup_build_tools_1.sh $BUILD_DIR +#build the stuff +$SCRIPT_DIR/setup_build_tools_2.sh $BUILD_DIR diff --git a/aviatrix/build/setup_build_tools_1.sh b/aviatrix/build/setup_build_tools_1.sh new file mode 100755 index 0000000000..cbfb956248 --- /dev/null +++ b/aviatrix/build/setup_build_tools_1.sh @@ -0,0 +1,66 @@ +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +BUILD_DIR=${SCRIPT_DIR}/../../../trafficserver-buildtools +if [ ! -z $1 ]; then + BUILD_DIR=$1 +fi +mkdir -p $BUILD_DIR +is_ubuntu18=`lsb_release -a|grep "Ubuntu 18.04"` +if [ ! -z "$is_ubuntu18" ]; then + sudo apt install wget + cd /tmp + mkdir cmakedownload + cd cmakedownload + wget wget https://github.com/Kitware/CMake/releases/download/v3.25.3/cmake-3.25.3-linux-x86_64.tar.gz + tar xvzf cmake-3.25.3-linux-x86_64.tar.gz + cd cmake-3.25.3-linux-x86_64 + mkdir -p ~/.local/ + cp -r * ~/.local/ + ls ~/.local/bin + if [ ! -d make-4.3 ]; then + cd $BUILD_DIR + + wget https://ftp.gnu.org/gnu/make/make-4.3.tar.gz + tar xfz make-4.3.tar.gz + cd make-4.3/ + ./configure --prefix=/home/develop/.local + make install + + fi +fi +cd $BUILD_DIR +pwd +MY_INSTALL_DIR=$HOME/.local +sudo apt install -y build-essential autoconf libtool pkg-config cmake +if [ ! -d grpc ]; then + echo 'grpc does not exist cloning' + mkdir -p $HOME/.local + git clone --recurse-submodules -b v1.48.0 --depth 1 --shallow-submodules https://github.com/grpc/grpc + cd grpc + mkdir -p cmake/build + pushd cmake/build + cmake -DgRPC_INSTALL=ON \ + -DgRPC_BUILD_TESTS=OFF \ + -DCMAKE_INSTALL_PREFIX=$MY_INSTALL_DIR \ + ../.. + popd +fi +cd $BUILD_DIR +if [ ! -d libswoc ]; then + echo 'libswoc does not exist, cloning' + git clone https://github.com/SolidWallOfCode/libswoc + cd libswoc + # we should change this to a version + echo checking out libswoc ${libswoc_version} + git checkout ${libswoc_version} + git status + # don't build examples + echo > example/CMakeLists.txt + echo > unit_tests/CMakeLists.txt + mkdir -p cmake/build + pushd cmake/build + + cmake -DCMAKE_INSTALL_PREFIX=$MY_INSTALL_DIR \ + ../.. + popd +fi + diff --git a/aviatrix/build/setup_build_tools_2.sh b/aviatrix/build/setup_build_tools_2.sh new file mode 100755 index 0000000000..c45f51696d --- /dev/null +++ b/aviatrix/build/setup_build_tools_2.sh @@ -0,0 +1,24 @@ +SCRIPT_DIR=$( cd -- "$( dirname -- "${BASH_SOURCE[0]}" )" &> /dev/null && pwd ) +BUILD_DIR=${SCRIPT_DIR}/../../../trafficserver-buildtools + +if [ ! -z $1 ]; then + BUILD_DIR=$1 +else + BUILD_DIR +fi +MY_INSTALL_DIR=$HOME/.local +cd $BUILD_DIR +pwd +cd grpc +pushd cmake/build +ncpus=$(grep -E '^processor' /proc/cpuinfo | wc -l) +make -j$ncpus +make install +popd +pushd third_party/re2 +CPPFLAGS=-fPIC make -j$ncpus -e prefix=$MY_INSTALL_DIR static static-install +popd +cd $BUILD_DIR/libswoc/cmake/build +c++ --version +echo $libswoc_version +make -j$ncpus -e VERBOSE=true --trace -e prefix=$MY_INSTALL_DIR install \ No newline at end of file diff --git a/aviatrix/dagger/Makefile b/aviatrix/dagger/Makefile new file mode 100644 index 0000000000..13083a3143 --- /dev/null +++ b/aviatrix/dagger/Makefile @@ -0,0 +1,59 @@ +# This is the base Ubuntu version to use. The user may set this to +# a different version. +UBUNTU = "22.04" + +RUN := go run $(wildcard *.go) + +DEVEL_IMAGE := trafficserver-devel:$(UBUNTU) +PACKAGING_IMAGE := trafficserver-packaging:$(UBUNTU) + +.PHONY: help +help: + @echo "TrafficServer meta-build system" + @echo + @echo "TARGETS" + @echo + @echo " package Build the TrafficServer Ubuntu package" + @echo " build-image Build a base container image for development" + @echo " packaging-image Build a base container image for packaging" + @echo + @echo " unit-tests Run Traffic Server unit tests" + @echo " regression-tests Run Traffic Server regression tests" + @echo + @echo " shell Drop into a shell in the development image" + @echo " clean Remove build outputs" + @echo + @echo "VARIABLES" + @echo + @echo " UBUNTU Base Ubuntu version (default is $(UBUNTU))" + +.PHONY: package +package: + $(RUN) $@ --ubuntu=$(UBUNTU) + +.PHONY: build-image +build-image: + $(RUN) $@ --ubuntu=$(UBUNTU) + +.PHONY: packaging-image +packaging-image: + $(RUN) $@ --ubuntu=$(UBUNTU) + +.PHONY: regression-tests +regression-tests: + $(RUN) $@ --ubuntu=$(UBUNTU) + +.PHONY: unit-tests +unit-tests: + $(RUN) $@ --ubuntu=$(UBUNTU) + +.PHONY: shell +shell: + docker run -it --rm \ + --volume $$(cd ../.. && pwd):/src/trafficserver \ + --volume $$(cd ../../../cloudn && pwd):/src/cloudn \ + $(DEVEL_IMAGE) + +.PHONY: clean +clean: + $(RM) -r images packages diff --git a/aviatrix/dagger/build.go b/aviatrix/dagger/build.go new file mode 100644 index 0000000000..eb2994b289 --- /dev/null +++ b/aviatrix/dagger/build.go @@ -0,0 +1,605 @@ +package main + +import ( + "context" + "fmt" + "os" + "os/exec" + "path" + "path/filepath" + "sort" + "strings" + + "dagger.io/dagger" + "golang.org/x/exp/maps" +) + +// DevInstallBase is the prefix where we install libraries and tools +// that are needed to build Traffic Server. We use /opt/ats here because +// the libraries will be included in the final Ubuntu package. +const DevInstallBase = "/opt/ats" + +// TrafficServerInstallDir is the path TrafficServer is installed in. +const TrafficServerInstallDir = "/opt/ats/ats_10.0.0" + +// writeExecScript wraps the given command-line in a trivial bash +// script. This is useful for cases where we want to run something in +// a build step, but also give developers the opportunity to run the +// identical command later. +func writeExecScript(cmd ...string) dagger.ContainerWithNewFileOpts { + b := strings.Builder{} + + b.WriteString("#! /usr/bin/env bash\n\nexec \\\n") + + for _, s := range cmd { + b.WriteString(s) + b.WriteString(" \\\n") + } + + b.WriteString(`"$@"`) + + return dagger.ContainerWithNewFileOpts{ + Permissions: 0755, + Contents: b.String(), + } +} + +// makeInstall runs "make" and then "make install". In makeArgs is +// provided, these are appended to the "make" command. +func makeInstall(makeArgs ...string) dagger.WithContainerFunc { + return func(container *dagger.Container) *dagger.Container { + return container. + WithExec([]string{"sh", "-c", "echo MAKEFLAGS=$MAKEFLAGS"}). + WithExec(append([]string{"make"}, makeArgs...)). + WithExec([]string{"make", "install"}) + } +} + +func aptUpdate() dagger.WithContainerFunc { + return func(container *dagger.Container) *dagger.Container { + cmd := []string{"env", "DEBIAN_FRONTEND=noninteractive", + "apt-get", "update", "--no-install-recommends", "--quiet", "--assume-yes", + } + return container.WithExec(cmd) + } +} + +func aptInstall(packages ...string) dagger.WithContainerFunc { + return func(container *dagger.Container) *dagger.Container { + cmd := []string{"env", "DEBIAN_FRONTEND=noninteractive", + "apt-get", "install", "--no-install-recommends", "--assume-yes", + } + + cmd = append(cmd, packages...) + return container.WithExec(cmd) + } +} + +func withHostFiles(client *dagger.Client, files map[string]string) dagger.WithContainerFunc { + // Make a stable list of destination files so that we don't bust the cache. + targets := maps.Keys(files) + sort.Strings(targets) + + return func(container *dagger.Container) *dagger.Container { + for _, t := range targets { + container = container.WithFile(t, client.Host().File(files[t])) + } + return container + } +} + +// getTrafficServerSourceVersion generates a version string that is +// passed in to the Traffic Server build. If we are building a git +// repository directly, use the branch or tag name. +func getTrafficServerSourceVersion(client *dagger.Client) string { + if strings.HasPrefix(TrafficServerDir, "git@") { + opts := dagger.GitOpts{} + + // If we are running in a local dev environment, then we can use SSH_AUTH_SOCK + // to do an authenticated pull of the repository. + if sshAgentPath := os.Getenv("SSH_AUTH_SOCK"); sshAgentPath != "" { + opts.SSHAuthSocket = client.Host().UnixSocket(sshAgentPath) + } + + repo := client.Git(TrafficServerDir, opts) + + // Building from "master" is a reasonably intuitive default. + name := "master" + ref := repo.Tag(name) + + // The tag and branch flags are mutually exclusive, so + // it doesn't matter which order we check them in. + switch { + case TrafficServerBranch != "": + name = TrafficServerBranch + ref = repo.Branch(TrafficServerBranch) + case TrafficServerTag != "": + name = TrafficServerTag + ref = repo.Tag(TrafficServerTag) + } + + commit, err := ref.Commit(context.Background()) + if err != nil { + Errorf("Unable to determine Git commit: %s", err) // yet? + os.Exit(1) + } + + return fmt.Sprintf("%s-%s", name, commit[:min(len(commit), 10)]) + } + + // NOTE: GitHub PRs don't clone the tags, so we need to set --always as a fallback. + cmd := exec.Command("git", "describe", "--always", "--dirty=+dirty") + cmd.Dir = Must(filepath.Abs(TrafficServerDir)) + out, err := cmd.CombinedOutput() + if err != nil { + if len(out) != 0 { + Errorf("%s", strings.TrimSpace(string(out))) + } + Errorf("git describe failed: %s", err) + os.Exit(1) + } + + return strings.TrimSpace(string(out)) +} + +func bindTrafficServerSources(client *dagger.Client) *dagger.Directory { + if strings.HasPrefix(TrafficServerDir, "git@") { + repo := client.Git(TrafficServerDir) + opts := dagger.GitRefTreeOpts{} + + // If we are running in a local dev environment, then we can use SSH_AUTH_SOCK + // to do an authenticated pull of the repository. + if sshAgentPath := os.Getenv("SSH_AUTH_SOCK"); sshAgentPath != "" { + opts.SSHAuthSocket = client.Host().UnixSocket(sshAgentPath) + } + + // The tag and branch flags are mutually exclusive, so + // it doesn't matter which order we check them in. + if TrafficServerBranch != "" { + return repo.Branch(TrafficServerBranch).Tree(opts) + } + + if TrafficServerTag != "" { + return repo.Tag(TrafficServerTag).Tree(opts) + } + + // Building from "master" is a reasonably intuitive default. + return repo.Tag("master").Tree(opts) + } + + if strings.HasPrefix(TrafficServerDir, "https://";) { + Errorf("HTTPS Git repositories are not supported") // yet? + os.Exit(1) + } + + // Now, assume that TrafficServerDir is a local directory path. + // NOTE: Dagger needs an absolute path to copy a directory from outside + // our current workdir. + repoBaseDir := Must(filepath.Abs(TrafficServerDir)) + + // Copy the build source from the host to the container. We exclude + // directories with build tooling so that changes to packaging + // and scripts don't blow the Dagger cache. + return client.Host().Directory( + repoBaseDir, + dagger.HostDirectoryOpts{ + Exclude: []string{ + ".git/", + ".github/", + "aviatrix/dagger", + "aviatrix/build", + "build/_aux", + "copy-for-docker.tgz", + }, + }, + ) + +} + +// bindCloudnProtobufs returns the "proto" subdirectory of the cloudn repository. +func bindCloudnProtobufs(client *dagger.Client) *dagger.Directory { + // TODO(jpeach) first check if we have a GitHub access + // token, and use that. We would use the access token in GitHub + // Actions builds. + + // If we are running in a local dev environment, then we can use SSH_AUTH_SOCK + // to do an authenticated pull of the cloudn repository. + if sshAgentPath := os.Getenv("SSH_AUTH_SOCK"); sshAgentPath != "" { + opts := dagger.GitRefTreeOpts{ + SSHAuthSocket: client.Host().UnixSocket(sshAgentPath), + } + + repo := client.Git("[email protected]:AviatrixDev/cloudn.git") + + if CloudnTag != "" { + return repo.Tag(CloudnTag).Tree(opts).Directory("proto") + } + + if CloudnBranch != "" { + return repo.Branch(CloudnBranch).Tree(opts).Directory("proto") + } + + return repo.Branch("master").Tree(opts).Directory("proto") + } + + // If there's no SSH agent, then we assume that the cloudn + // repository is checked out next to our local repository. + + // Absolute path to the top of the current repository. + repoBaseDir := Must(filepath.Abs("../..")) + return client.Host().Directory( + path.Join(repoBaseDir, "../cloudn/proto"), + ) +} + +func buildJemalloc(client *dagger.Client, container *dagger.Container) *dagger.Container { + const srcdir = "/src/jemalloc" + const vers = "5.3.0" + + tarball := client.HTTP(fmt.Sprintf( + "https://github.com/jemalloc/jemalloc/releases/download/%s/jemalloc-%s.tar.bz2";, vers, vers)) + + return container. + With(aptInstall("libunwind-dev")). + WithExec([]string{"mkdir", "-p", srcdir}). + WithWorkdir(srcdir). + WithFile(path.Join(srcdir, "jemalloc-source.tgz"), tarball). + WithExec([]string{"tar", "--strip-components=1", "-xf", "jemalloc-source.tgz"}). + WithExec([]string{ + "./configure", + "--prefix=" + DevInstallBase, + "--enable-prof", + "--enable-prof-libunwind", + }). + With(makeInstall()) +} + +func buildOpenssl(client *dagger.Client, container *dagger.Container) *dagger.Container { + const srcdir = "/src/openssl" + const prefix = DevInstallBase + + branch := client.Git("https://github.com/openssl/openssl";). + Tag("openssl-3.0.13"). + Tree() + + return container. + WithDirectory(srcdir, branch). + WithWorkdir(srcdir). + WithExec([]string{ + "./config", + "--release", + "--prefix=" + prefix, + "--libdir=" + path.Join(prefix, "lib"), // Override the "lib64" default. + "no-tests", + "no-makedepend", + // TODO(jpeach) master also has no-apps and no-docs build options + }). + With(makeInstall()) +} + +func buildGrpc(client *dagger.Client, container *dagger.Container) *dagger.Container { + srcdir := "/src/grpc" + builddir := "/build/grpc" + ssldir := DevInstallBase + + // Note that dagger recursively checks out submodules without + // special configuration. + branch := client.Git("https://github.com/grpc/grpc";). + Branch("v1.48.0"). + Tree() + + // See also https://github.com/grpc/grpc/blob/master/BUILDING.md. + return container. + With(aptInstall("cmake")). + WithDirectory(srcdir, branch). + WithExec([]string{"mkdir", "-p", builddir}). + WithWorkdir(builddir). + WithExec([]string{ + "/usr/bin/cmake", + "-DCMAKE_INSTALL_PREFIX=" + DevInstallBase, + "-DCMAKE_BUILD_TYPE:STRING=Release", + // Enable shared libraries. + "-DBUILD_SHARED_LIBS=ON", + // Force cmake to set rpath so that gRPC tools work. + "-DCMAKE_SKIP_RPATH=FALSE", + "-DCMAKE_INSTALL_RPATH=" + path.Join(DevInstallBase, "lib"), + "-DCMAKE_CXX_STANDARD=17", + "-DCMAKE_CXX_STANDARD_REQUIRED=On", + "-DCMAKE_CXX_EXTENSIONS=On", + // Note that this installs abseil as a side-effect. + "-DgRPC_INSTALL=ON", + "-DgRPC_BUILD_TESTS=OFF", + "-DgRPC_BUILD_CSHARP_EXT=OFF", + "-DgRPC_BUILD_GRPC_CSHARP_PLUGIN=OFF", + "-DgRPC_BUILD_GRPC_NODE_PLUGIN=OFF", + "-DgRPC_BUILD_GRPC_OBJECTIVE_C_PLUGIN=OFF", + "-DgRPC_BUILD_GRPC_PHP_PLUGIN=OFF", + "-DgRPC_BUILD_GRPC_PYTHON_PLUGIN=OFF", + "-DgRPC_BUILD_GRPC_RUBY_PLUGIN=OFF", + // Don't the bundled BoringSSL. + "-DgRPC_SSL_PROVIDER=package", + "-DOPENSSL_ROOT_DIR=" + ssldir, + // Don't use the bundled zlib. + "-DgRPC_ZLIB_PROVIDER=package", + srcdir, + }). + With(makeInstall()) +} + +func buildSwoc(client *dagger.Client, container *dagger.Container) *dagger.Container { + srcdir := "/src/swoc" + builddir := "/build/swoc" + + branch := client.Git("https://github.com/SolidWallOfCode/libswoc";). + Tag("1.5.1"). + Tree() + + return container. + WithDirectory(srcdir, branch). + WithExec([]string{"mkdir", "-p", builddir}). + WithWorkdir(builddir). + WithExec([]string{ + path.Join(DevInstallBase, "bin", "cmake"), + "-DCMAKE_INSTALL_PREFIX=" + DevInstallBase, + "-DCMAKE_BUILD_TYPE:STRING=Release", + "-DLIBSWOC_TEST=OFF", // Disable the tests and examples. + srcdir, + }). + With(makeInstall()) +} + +// setupTrafficServer does all the preparation steps for +// building Traffic Server, but not the actual build. The configure +// and make steps are saved in the build directory. +// +// Although this makes things a bit complicated, it also makes it +// possible to generate a build image where people can build Traffic +// Server manually with the correct configuration. +func setupTrafficServer(client *dagger.Client, container *dagger.Container) *dagger.Container { + const srcdir = "/src/trafficserver" + const cloudndir = "/src/cloudn" + const prefix = TrafficServerInstallDir + + // Bind the cloudn protobuf directory. Note that we set + // CLOUDN_PROTO to the cloudn repository root, because the + // proto files include the leading "proto/" in their import paths. + cloudnProtoDir := bindCloudnProtobufs(client) + + return container. + // Add TrafficServer-specific dependencies. + With(aptInstall( + // Some tests that assume Python3 is installed. + "python3", + "libcap-dev", + "libhwloc-dev", + "libpcre3-dev", + "libpcre2-dev", + // So that jemalloc will link. Otherwise, + // the Traffic Server jemalloc detection will + // fail, but the build will continue without it. + "libunwind8", + "cmake", + "pipenv", + )). + WithDirectory(srcdir, bindTrafficServerSources(client)). + WithMountedDirectory(path.Join(cloudndir, "proto"), cloudnProtoDir). + // Add the TrafficServer role account for the build and install. + WithExec([]string{ + "useradd", + "--create-home", + "--shell", "/bin/bash", + "--gid", "root", + "--groups", "sudo", + "--system", + "ats-user", + }) +} + +func buildTrafficServer(client *dagger.Client, container *dagger.Container) *dagger.Container { + const srcdir = "/src/trafficserver" + const prefix = TrafficServerInstallDir + + builder := setupTrafficServer(client, container) + return builder. + WithWorkdir(srcdir). + WithExec([]string{ + "find", + "/", + "-name", + "libgrpc++.a", + "-print", + }). + WithExec([]string{ + "cat", + "/src/trafficserver/CMakePresets.json", + }). + WithExec([]string{ + "cmake", + "--preset", + "aviatrix", + }). + WithExec([]string{ + "cmake", + "--build", + "build-release", + }). + WithExec([]string{ + "cmake", + "--install", + "build-release", + }). + // Remove garbage libtool archives. + WithExec([]string{ + "find", + prefix, + "-type", "f", + "-name", "*.la", + "-exec", "rm", "{}", "+", + }) +} + +func buildDebianPackages( + packager *dagger.Container, +) *dagger.Container { + const builddir = "/build/ats-10.0.0" + + base := packager. + WithWorkdir(builddir). + // Strip shared library symbols, see https://github.com/Debian/debhelper/blob/master/dh_strip. + WithExec([]string{"sh", "-c", + "find opt/ats/lib opt/ats/ats_10.0.0/lib -type f -name '*.so' -print | " + + "xargs strip --remove-section=.comment --remove-section=.note --strip-unneeded", + }). + // Strip binary symbols, see https://github.com/Debian/debhelper/blob/master/dh_strip. + WithExec([]string{"sh", "-c", + "find opt/ats/ats_10.0.0/bin -type f -name 'traffic_*' | " + + "xargs strip --remove-section=.comment --remove-section=.note", + }). + WithExec([]string{"dpkg-deb", "--build", builddir, "/build/ats-10.0.0.deb"}) + + debug := packager. + WithExec([]string{"dpkg-deb", "--build", builddir, "/build/ats-dbg-10.0.0.deb"}) + + // Note that even though the deb files have different names, + // they are still the same package as far as dpkg is concerned. + // This means that dpkg is happy for you to install one of these + // on top of the other (it says it's a reinstall). This seems a + // bit weird, but desirable on balance. + + return packager. + WithFile("/build/ats-10.0.0.deb", base.File("/build/ats-10.0.0.deb")). + WithFile("/build/ats-dbg-10.0.0.deb", debug.File("/build/ats-dbg-10.0.0.deb")) +} + +func setupDebianPackage( + client *dagger.Client, + trafficserver *dagger.Container, + builder *dagger.Container, +) *dagger.Container { + const builddir = "/build/ats-10.0.0" // Path we build the package in. + const installdir = TrafficServerInstallDir + + repoBaseDir := Must(filepath.Abs("../..")) + packagingDir := path.Join(repoBaseDir, "aviatrix/10_0_0.0001") + + hostFiles := map[string]string{ + "DEBIAN/control": path.Join(packagingDir, "control"), + "DEBIAN/postinst": path.Join(packagingDir, "postinst"), + "etc/logrotate.d/avx-gw-trafficserver": path.Join(packagingDir, "logrotate/avx-gw-trafficserver"), + "lib/systemd/system/avx-gw-trafficserver.service": path.Join(packagingDir, "avx-gw-trafficserver.service"), + } + manifest := path.Join(repoBaseDir, "ats-manifest") + if _, err := os.Stat(manifest); err == nil { + hostFiles["opt/ats/ats_10.0.0.deb.manifest"] = manifest + } + + return builder. + With(aptInstall( + // So we can package the DSOs in a later step. + "libhwloc-dev", + // https://wiki.debian.org/Packaging/Intro + "build-essential", + "debhelper", + "devscripts", + )). + WithExec([]string{ + "mkdir", "-p", + path.Join(builddir, "/lib/systemd/system"), + path.Join(builddir, "/etc/logrotate.d"), + path.Join(builddir, "DEBIAN"), + }). + WithWorkdir(builddir). + With(withHostFiles(client, hostFiles)). + // Copy the TrafficServer build over. + WithDirectory( + path.Join("./", installdir), + trafficserver.Directory(installdir), + dagger.ContainerWithDirectoryOpts{Exclude: []string{ + "share", // Man pages. + "include", // Header files. + "lib/perl5", // Perl bindings. + "lib/pkgconfig", // Pkg-config spec. + "lib/plugin_*.so", // Remap test DSOs. + }}, + ). + // Package the lib directory that we linked against. + WithDirectory( + path.Join("./", DevInstallBase, "lib"), + trafficserver.Directory(path.Join(DevInstallBase, "lib")), + dagger.ContainerWithDirectoryOpts{ + Exclude: []string{"cmake/", "pkgconfig/", "*.a"}, + }, + ). + // Copy libhwloc so that we don't take a dependency on a package + // that we might not be able to install at upgrade time. + With(func(container *dagger.Container) *dagger.Container { + return container.WithDirectory( + path.Join("./", installdir, "lib"), + container.Directory("/usr/lib/x86_64-linux-gnu"), + dagger.ContainerWithDirectoryOpts{ + Include: []string{"libhwloc.so*"}, + }, + ) + }). + // Copy custom configs over the top. + WithDirectory( + path.Join("./", installdir, "etc/trafficserver"), + client.Host().Directory(path.Join(packagingDir, "configs")), + ). + WithExec([]string{ + "mkdir", "-p", + path.Join("./", installdir, "etc/local_ca"), + path.Join("./", installdir, "var/local_ca/keys"), + }). + WithNewFile(path.Join("./", installdir, "var/local_ca/serial.txt"), dagger.ContainerWithNewFileOpts{ + Contents: "12345", + }) +} + +func buildBaseContainer(client *dagger.Client, ubuntuVersion string) *dagger.Container { + // Start off with a base Ubuntu image, with just basic build dependencies in it. + base := client.Container(). + From(fmt.Sprintf("ubuntu:%s", ubuntuVersion)). + With(aptUpdate()). + With(aptInstall( + "autoconf", + "automake", + "build-essential", + "libtool", + "libz-dev", + "pkg-config", + )) + + ncpu, err := ContainerNumCPUs(base) + if err != nil { + Errorf("failed to count container CPUs: %s", err.Error()) + ncpu = 4 // Arbitrary default. + } + + return base.WithEnvVariable("MAKEFLAGS", fmt.Sprintf("-j%d", ncpu)) +} + +func buildBuilderContainer(client *dagger.Client, base *dagger.Container) *dagger.Container { + // The builder is a dev container that has custom dependencies + // installed in DevInstallBase. + builder := base. + With(func(container *dagger.Container) *dagger.Container { + return buildOpenssl(client, container) + }). + With(func(container *dagger.Container) *dagger.Container { + // We can attempt to build these packages in parallel. + jemalloc := buildJemalloc(client, container) + grpc := buildGrpc(client, container) + //swoc := buildSwoc(client, container) + + // Merge the output of both builds. + return container. + WithDirectory(DevInstallBase, jemalloc.Directory(DevInstallBase)). + WithDirectory(DevInstallBase, grpc.Directory(DevInstallBase)) + }) + + // Now copy the installed packages back to base so that we don't + // have any intermediate artifacts. + return base. + WithDirectory(DevInstallBase, builder.Directory(DevInstallBase)) +} diff --git a/aviatrix/dagger/build_manifest.sh b/aviatrix/dagger/build_manifest.sh new file mode 100755 index 0000000000..db56f3d52f --- /dev/null +++ b/aviatrix/dagger/build_manifest.sh @@ -0,0 +1,27 @@ +#! /usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail +manifest_dir=`realpath $(dirname $0)/../../` +cd $manifest_dir +rm -rf ats-manifest +git_status="$(git status)" +echo "build command: $@" >> ats-manifest +echo "ats branch: " $(git rev-parse --abbrev-ref HEAD) >> ats-manifest +echo "ats commit: " $(git rev-parse --verify HEAD) >> ats-manifest +echo "ats user: " $(git config --get user.email) >> ats-manifest + +echo "ats timestamp: " $(date -Is) >> ats-manifest +echo >> ats-manifest +pushd ../cloudn > /dev/null +echo "cloudn branch: " $(git rev-parse --abbrev-ref HEAD) >> $manifest_dir/ats-manifest +echo "cloudn commit: " $(git rev-parse --verify HEAD) >> $manifest_dir/ats-manifest +echo "cloudn user: " $(git config --get user.email) >> $manifest_dir/ats-manifest +popd > /dev/null + + +echo >> ats-manifest +echo >> ats-manifest +echo =================================================================== >> ats-manifest +printf "%s" "$git_status" >> ats-manifest \ No newline at end of file diff --git a/aviatrix/dagger/builder.sh b/aviatrix/dagger/builder.sh new file mode 100755 index 0000000000..452030282e --- /dev/null +++ b/aviatrix/dagger/builder.sh @@ -0,0 +1,13 @@ +#! /usr/bin/env bash + +set -o errexit +set -o nounset +set -o pipefail +rm -rf $(dirname $0)/../../ats-manifest +all_args="$@" +if [ "$all_args" == "package" ]; then + ./build_manifest.sh +fi + +cd $(dirname $0) +exec "${GO:-go}" run *.go "$@" diff --git a/aviatrix/dagger/go.mod b/aviatrix/dagger/go.mod new file mode 100644 index 0000000000..1985797fe9 --- /dev/null +++ b/aviatrix/dagger/go.mod @@ -0,0 +1,23 @@ +module github.com/AviatrixDev/thirdparty-trafficserver/aviatrix/dagger + +go 1.21 + +require ( + dagger.io/dagger v0.9.3 + github.com/spf13/cobra v1.7.0 + golang.org/x/exp v0.0.0-20231108232855-2478ac86f678 +) + +require ( + github.com/99designs/gqlgen v0.17.40 // indirect + github.com/Khan/genqlient v0.6.0 // indirect + github.com/adrg/xdg v0.4.0 // indirect + github.com/google/uuid v1.4.0 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/mitchellh/go-homedir v1.1.0 // indirect + github.com/sosodev/duration v1.2.0 // indirect + github.com/spf13/pflag v1.0.5 // indirect + github.com/vektah/gqlparser/v2 v2.5.10 // indirect + golang.org/x/sync v0.5.0 // indirect + golang.org/x/sys v0.14.0 // indirect +) diff --git a/aviatrix/dagger/go.sum b/aviatrix/dagger/go.sum new file mode 100644 index 0000000000..e48a25c5a2 --- /dev/null +++ b/aviatrix/dagger/go.sum @@ -0,0 +1,48 @@ +dagger.io/dagger v0.9.3 h1:igFU1d6R933Jn6741k5HI/TjAlkpb2/wiBTypNbE0Pw= +dagger.io/dagger v0.9.3/go.mod h1:1iiFzqKOri9kJxUDYUibthMpkfzaWP25B2kx7F/AXIk= +github.com/99designs/gqlgen v0.17.40 h1:/l8JcEVQ93wqIfmH9VS1jsAkwm6eAF1NwQn3N+SDqBY= +github.com/99designs/gqlgen v0.17.40/go.mod h1:b62q1USk82GYIVjC60h02YguAZLqYZtvWml8KkhJps4= +github.com/Khan/genqlient v0.6.0 h1:Bwb1170ekuNIVIwTJEqvO8y7RxBxXu639VJOkKSrwAk= +github.com/Khan/genqlient v0.6.0/go.mod h1:rvChwWVTqXhiapdhLDV4bp9tz/Xvtewwkon4DpWWCRM= +github.com/adrg/xdg v0.4.0 h1:RzRqFcjH4nE5C6oTAxhBtoE2IRyjBSa62SCbyPidvls= +github.com/adrg/xdg v0.4.0/go.mod h1:N6ag73EX4wyxeaoeHctc1mas01KZgsj5tYiAIwqJE/E= +github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883 h1:bvNMNQO63//z+xNgfBlViaCIJKLlCJ6/fmUseuG0wVQ= +github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= +github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= +github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= +github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/sergi/go-diff v1.3.1 h1:xkr+Oxo4BOQKmkn/B9eMK0g5Kg/983T9DqqPHwYqD+8= +github.com/sergi/go-diff v1.3.1/go.mod h1:aMJSSKb2lpPvRNec0+w3fl7LP9IOFzdc9Pa4NFbPK1I= +github.com/sosodev/duration v1.2.0 h1:pqK/FLSjsAADWY74SyWDCjOcd5l7H8GSnnOGEB9A1Us= +github.com/sosodev/duration v1.2.0/go.mod h1:RQIBBX0+fMLc/D9+Jb/fwvVmo0eZvDDEERAikUR6SDg= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= +github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/vektah/gqlparser/v2 v2.5.10 h1:6zSM4azXC9u4Nxy5YmdmGu4uKamfwsdKTwp5zsEealU= +github.com/vektah/gqlparser/v2 v2.5.10/go.mod h1:1rCcfwB2ekJofmluGWXMSEnPMZgbxzwj6FaZ/4OT8Cc= +golang.org/x/exp v0.0.0-20231108232855-2478ac86f678 h1:mchzmB1XO2pMaKFRqk/+MV3mgGG96aqaPXaMifQU47w= +golang.org/x/exp v0.0.0-20231108232855-2478ac86f678/go.mod h1:zk2irFbV9DP96SEBUUAy67IdHUaZuSnrz1n472HUCLE= +golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= +golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sys v0.0.0-20211025201205-69cdffdb9359/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0 h1:Vz7Qs629MkJkGyHxUlRHizWJRG2j8fbQKjELVSNhy7Q= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= +gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/aviatrix/dagger/main.go b/aviatrix/dagger/main.go new file mode 100644 index 0000000000..99479e1bb7 --- /dev/null +++ b/aviatrix/dagger/main.go @@ -0,0 +1,312 @@ +package main + +import ( + "bufio" + "context" + "fmt" + "os" + "os/exec" + "path" + "regexp" + "strings" + + "github.com/spf13/cobra" + + "dagger.io/dagger" +) + +// BaseUbuntuVersion is the Ubuntu release version to build for. +var BaseUbuntuVersion string + +// TrafficServerDir is the path to the Traffic Server source code. +// If this is a git or https URL, then we pull the Git repository. +// Otherwise, we assume it is a local path. +var TrafficServerDir string + +// TrafficServerTag is the Traffic Server source tag. Mutually +// exclusive with the branch. Ignored unless we are building a git +// repository. +var TrafficServerTag string + +// TrafficServerBranch is the Traffic Server source branch. Mutually +// exclusive with the tag. Ignored unless we are building a git +// repository. +var TrafficServerBranch string + +// CloudnTag is the tag at which we pull protobuf definitions from +// the cloudn repository. Default is to use the master branch. Mutually +// exclusive with CloudnBranch. +var CloudnTag string + +// CloudnBranch is the branch from which we pull protobuf definitions from +// the cloudn repository. Default is to use the master branch. Mutually +// exclusive with CloudnTag. +var CloudnBranch string + +// ContainerNumCPUs inspects the given container and returns the number of available CPUs. +func ContainerNumCPUs(container *dagger.Container) (int, error) { + // procfs is only mounbtedfor processes, so we need to + // sample it with an exec before we can copy it out. + container = container.WithExec( + []string{"cat", "/proc/cpuinfo"}, + dagger.ContainerWithExecOpts{RedirectStdout: "/tmp/cpuinfo"}, + ) + + cpuInfo, err := container.File("/tmp/cpuinfo").Contents(context.Background()) + if err != nil { + return -1, err + } + + count := 0 + scanner := bufio.NewScanner(strings.NewReader(cpuInfo)) + + for scanner.Scan() { + fields := strings.SplitN(scanner.Text(), ": ", 2) + + // Skip over empty lines. + if len(fields) != 2 { + continue + } + + if strings.TrimSpace(fields[0]) == "processor" { + count++ + } + } + + return count, nil +} + +func Successf(format string, args ...any) { + fmt.Fprintf(os.Stderr, "✅ %s\n", fmt.Sprintf(format, args...)) +} + +func Errorf(format string, args ...any) { + fmt.Fprintf(os.Stderr, "✴️ %s\n", fmt.Sprintf(format, args...)) +} + +func Must[T any](value T, err error) T { + if err != nil { + Errorf(err.Error()) + os.Exit(1) + } + + return value +} + +func ExportContainerImage( + ctx context.Context, + container *dagger.Container, + repoName string, + tagName string, +) error { + buildImagePath := path.Join("images", repoName, tagName+".tgz") + if _, err := container.Export(ctx, buildImagePath); err != nil { + return err + } + + Successf("exported build container image to %q", buildImagePath) + + // Load the exported OCI image into the local Docker instance. + // Note that we have to load, not import. + cmdImport := exec.Command("docker", "load", "--quiet", "--input", buildImagePath) + out, err := cmdImport.CombinedOutput() + if err != nil { + return fmt.Errorf("failed to export builder image: %w", err) + } + + Successf("imported build container image from %q", buildImagePath) + + // Capture the image ID from the output of the load + // command so that we can tag it. Remember that the + // first result from Submatch is the full matched + // sequence, so the capture group is the second match. + r := regexp.MustCompile("Loaded image ID: ([:a-z0-9]+)") + matches := r.FindStringSubmatch(string(out)) + if len(matches) == 0 { + Errorf("failed to match image ID from Docker output %q", string(out)) + os.Exit(1) + } + + fullName := fmt.Sprintf("%s:%s", repoName, tagName) + cmdTag := exec.Command("docker", "tag", matches[1], fullName) + if out, err := cmdTag.CombinedOutput(); err != nil { + return fmt.Errorf("failed to tag builder image: %s", string(out)) + } + + Successf("tagged build container image %q as %q", matches[1], fullName) + + return nil +} + +func packageCmd(ctx context.Context, client *dagger.Client) error { + base := buildBaseContainer(client, BaseUbuntuVersion) + builder := buildBuilderContainer(client, base) + + // Now build TrafficServer in the builder container. + trafficserver := buildTrafficServer(client, builder) + + // Fork a container off the base image to wrap the TrafficServer + // build in a Debian package. + packager := buildDebianPackages(setupDebianPackage(client, trafficserver, base)) + + for _, pkg := range []string{"ats-10.0.0.deb", "ats-dbg-10.0.0.deb"} { + // Copy the final debian package out of the packager container. + packagePath := path.Join("packages", BaseUbuntuVersion, pkg) + packageFile := client.Directory(). + WithFile(packagePath, packager.File("/build/"+pkg)) + + if _, err := packageFile.Export(ctx, "."); err != nil { + return fmt.Errorf("failed to export Debian package: %s", err.Error()) + } + + Successf("wrote Debian package to %s", packagePath) + } + + return nil +} + +func packagingImageCmd(ctx context.Context, client *dagger.Client) error { + base := buildBaseContainer(client, BaseUbuntuVersion) + builder := buildBuilderContainer(client, base) + trafficserver := buildTrafficServer(client, builder) + packager := setupDebianPackage(client, trafficserver, builder) + + return ExportContainerImage(ctx, packager, "trafficserver-packaging", BaseUbuntuVersion) +} + +func buildImageCmd(ctx context.Context, client *dagger.Client) error { + base := buildBaseContainer(client, BaseUbuntuVersion) + builder := buildBuilderContainer(client, base) + + // Do TrafficServer build setup so that the exported + // image contains the configure and make steps that + // the package build would use. + trafficserver := setupTrafficServer(client, builder) + + return ExportContainerImage(ctx, trafficserver, "trafficserver-devel", BaseUbuntuVersion) +} + +func regressionTestsCmd(ctx context.Context, client *dagger.Client) error { + base := buildBaseContainer(client, BaseUbuntuVersion) + builder := buildBuilderContainer(client, base) + + trafficserver := setupTrafficServer(client, builder). + WithExec([]string{"./conf.sh", "--enable-tests"}). + With(makeInstall()). + WithExec([]string{ + path.Join(TrafficServerInstallDir, "bin/traffic_server"), + "--clear_hostdb", "--clear_cache", "--regression", "1", + }) + + if _, err := trafficserver.Sync(ctx); err != nil { + return fmt.Errorf("regression tests failed: %s", err) + } + + Successf("regression tests passed") + return nil +} + +func unitTestsCmd(ctx context.Context, client *dagger.Client) error { + base := buildBaseContainer(client, BaseUbuntuVersion) + builder := buildBuilderContainer(client, base) + trafficserver := buildTrafficServer(client, builder). + // First build the tests without running any, to catch build errors. + WithExec([]string{"make", "check", "TESTS="}). + // Next, run the tests. + WithExec([]string{"make", "check"}) + + if _, err := trafficserver.Sync(ctx); err != nil { + return fmt.Errorf("unit tests failed: %s", err) + } + + Successf("unit tests passed") + return nil +} + +func main() { + var daggerCtx context.Context + var daggerClient *dagger.Client + + defer func() { + if daggerClient != nil { + daggerClient.Close() + } + }() + + rootCmd := cobra.Command{ + Use: path.Base(os.Args[0]), + PersistentPreRunE: func(*cobra.Command, []string) error { + var err error + daggerCtx = context.Background() + + // Initialize a Dagger client. + daggerClient, err = dagger.Connect(daggerCtx, dagger.WithLogOutput(os.Stdout)) + return err + }, + } + + rootCmd.PersistentFlags(). + StringVar(&BaseUbuntuVersion, "ubuntu", "22.04", "Base Ubuntu release version") + + rootCmd.PersistentFlags(). + StringVar(&TrafficServerDir, "source", "../..", "Traffic Server source repository or directory path") + + rootCmd.PersistentFlags(). + StringVar(&TrafficServerBranch, "branch", "", "Traffic Server source branch") + + rootCmd.PersistentFlags(). + StringVar(&TrafficServerTag, "tag", "", "Traffic Server source tag") + + rootCmd.PersistentFlags(). + StringVar(&CloudnTag, "cloudn-tag", "", "Cloudn source repository tag") + + rootCmd.PersistentFlags(). + StringVar(&CloudnBranch, "cloudn-branch", "", "Cloudn source repository branch") + + rootCmd.AddCommand( + &cobra.Command{ + Use: "package", + Short: "Build the Ubuntu .deb package", + RunE: func(cmd *cobra.Command, args []string) error { + return packageCmd(daggerCtx, daggerClient) + }, + }, + &cobra.Command{ + Use: "packaging-image", + Short: "Build a Docker image for packaging", + RunE: func(cmd *cobra.Command, args []string) error { + return packagingImageCmd(daggerCtx, daggerClient) + }, + }, + &cobra.Command{ + Use: "build-image", + Short: "Build a Docker image for local development", + RunE: func(cmd *cobra.Command, args []string) error { + return buildImageCmd(daggerCtx, daggerClient) + }, + }, + &cobra.Command{ + Use: "regression-tests", + Short: "Run Traffic Server regression tests", + RunE: func(cmd *cobra.Command, args []string) error { + return regressionTestsCmd(daggerCtx, daggerClient) + }, + }, + &cobra.Command{ + Use: "unit-tests", + Short: "Run Traffic Server unit tests", + RunE: func(cmd *cobra.Command, args []string) error { + return unitTestsCmd(daggerCtx, daggerClient) + }, + }, + ) + + for _, c := range rootCmd.Commands() { + c.MarkFlagsMutuallyExclusive("tag", "branch") + c.MarkFlagsMutuallyExclusive("cloudn-tag", "cloudn-branch") + } + + if err := rootCmd.Execute(); err != nil { + Errorf("%s", err) + } +} diff --git a/aviatrix/plugins/avx_certifier/CMakeLists.txt b/aviatrix/plugins/avx_certifier/CMakeLists.txt index 47580b2b07..fac8784b7d 100644 --- a/aviatrix/plugins/avx_certifier/CMakeLists.txt +++ b/aviatrix/plugins/avx_certifier/CMakeLists.txt @@ -1,4 +1,4 @@ add_atsplugin(avx_certifier avx_certifier.cc) -target_link_libraries(avx_certifier) +target_link_libraries(avx_certifier PRIVATE OpenSSL::Crypto OpenSSL::SSL) verify_global_plugin(avx_certifier) diff --git a/aviatrix/plugins/avx_policy_driver/CMakeLists.txt b/aviatrix/plugins/avx_policy_driver/CMakeLists.txt index bce11d170b..35b04d1eb9 100644 --- a/aviatrix/plugins/avx_policy_driver/CMakeLists.txt +++ b/aviatrix/plugins/avx_policy_driver/CMakeLists.txt @@ -1,7 +1,7 @@ -set(ENV{PKG_CONFIG_PATH} "~/.local/lib/pkgconfig") +set(ENV{PKG_CONFIG_PATH} "~/.local/lib/pkgconfig:/opt/ats/lib/pkgconfig") find_package(PkgConfig REQUIRED) -find_package(protobuf CONFIG) +find_package(protobuf CONFIG REQUIRED) find_package(gRPC CONFIG REQUIRED) pkg_check_modules(PROTOBUF REQUIRED IMPORTED_TARGET protobuf) pkg_check_modules(GRPC REQUIRED IMPORTED_TARGET grpc++) @@ -16,7 +16,7 @@ add_library( "${CLOUDN_SOURCE_ROOT}/proto/conduit/v2/external_groups.proto" ) -target_link_libraries(proto-objects PUBLIC protobuf::libprotobuf) +target_link_libraries(proto-objects PUBLIC protobuf::libprotobuf gRPC::grpc++) target_include_directories(proto-objects PUBLIC "$<BUILD_INTERFACE:${PROTO_BINARY_DIR}>") protobuf_generate(TARGET proto-objects IMPORT_DIRS "${CLOUDN_SOURCE_ROOT}" PROTOC_OUT_DIR "${PROTO_BINARY_DIR}") @@ -37,11 +37,24 @@ protobuf_generate( "${PROTO_BINARY_DIR}" ) -add_atsplugin(avx_policy_driver policy_driver.cc logging.cc policy_client.cc web_filter.cc ${proto-objects}) +add_atsplugin( + avx_policy_driver + policy_driver.cc + logging.cc + policy_client.cc + web_filter.cc + ${PROTO_BINARY_DIR}/proto/conduit/v2/layer7.pb.cc + ${PROTO_BINARY_DIR}/proto/conduit/v2/layer7.grpc.pb.cc + ${PROTO_BINARY_DIR}/proto/conduit/v2/microseg.pb.cc + ${PROTO_BINARY_DIR}/proto/conduit/v2/external_groups.pb.cc + ${PROTO_BINARY_DIR}/proto/common/types.pb.cc +) target_include_directories( avx_policy_driver PUBLIC "$<BUILD_INTERFACE:${PROTO_BINARY_DIR}>" ${GRPC_INCLUDES} "~/.local/include" ) -target_link_libraries(avx_policy_driver PRIVATE libswoc::libswoc ${PROTOBUF_LIBRARY}) +target_link_libraries( + avx_policy_driver PRIVATE libswoc::libswoc ${PROTOBUF_LIBRARY} ${GRPC_LIBRARY} OpenSSL::Crypto OpenSSL::SSL +) verify_global_plugin(avx_policy_driver)
