This is an automated email from the ASF dual-hosted git repository.
ccondit pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/yunikorn-k8shim.git
The following commit(s) were added to refs/heads/master by this push:
new c01c8601 [YUNIKORN-2438] Shim: Add license files to Docker images
(#793)
c01c8601 is described below
commit c01c86018fb3fa4e1f2ca0e85a6d675b82a70ccc
Author: Craig Condit <[email protected]>
AuthorDate: Thu Feb 22 09:53:29 2024 -0600
[YUNIKORN-2438] Shim: Add license files to Docker images (#793)
Add infrastructure to bundle NOTICE/LICENSE and third-party license files.
Two new Makefile targets are added:
- go-license-check: Check all dependencies for unauthorized licenses
- go-license-generate: Retrieve license files for 3rd party components and
save to build/third-party-licenses/ directory
The generated license files depend on go.mod / go.sum and will be used as
input to the Docker image generation.
Closes: #793
---
Makefile | 48 +++++++++++++++++++++++++++++++++----
docker/admission/Dockerfile | 2 +-
docker/plugin/Dockerfile | 2 +-
docker/scheduler/Dockerfile | 2 +-
docker/webtest/Dockerfile | 2 +-
scripts/third-party-licences.md.tpl | 11 +++++++++
6 files changed, 58 insertions(+), 9 deletions(-)
diff --git a/Makefile b/Makefile
index 031933d8..45fb4f4e 100644
--- a/Makefile
+++ b/Makefile
@@ -162,6 +162,10 @@ export SPARK_HOME=$(BASE_DIR)$(TOOLS_DIR)/spark
export SPARK_SUBMIT_CMD=$(SPARK_HOME)/bin/spark-submit
export SPARK_PYTHON_IMAGE=docker.io/apache/spark-py:v$(SPARK_PYTHON_VERSION)
+# go-licenses
+GO_LICENSES_VERSION=v1.6.0
+GO_LICENSES_BIN=$(TOOLS_DIR)/go-licenses
+
FLAG_PREFIX=github.com/apache/yunikorn-k8shim/pkg/conf
# Image hashes
@@ -207,7 +211,7 @@ conf/scheduler-config-local.yaml: conf/scheduler-config.yaml
# Install tools
.PHONY: tools
-tools: $(SHELLCHECK_BIN) $(GOLANGCI_LINT_BIN) $(KUBECTL_BIN) $(KIND_BIN)
$(HELM_BIN) $(SPARK_SUBMIT_CMD)
+tools: $(SHELLCHECK_BIN) $(GOLANGCI_LINT_BIN) $(KUBECTL_BIN) $(KIND_BIN)
$(HELM_BIN) $(SPARK_SUBMIT_CMD) $(GO_LICENSES_BIN)
# Install shellcheck
$(SHELLCHECK_BIN):
@@ -255,6 +259,12 @@ $(SPARK_SUBMIT_CMD):
| tar -x -z --strip-components=1 -C "$(SPARK_HOME).tmp"
@mv -f "$(SPARK_HOME).tmp" "$(SPARK_HOME)"
+# Install go-licenses
+$(GO_LICENSES_BIN):
+ @echo "installing go-licenses $(GO_LICENSES_VERSION)"
+ @mkdir -p "$(TOOLS_DIR)"
+ @GOBIN="$(BASE_DIR)/$(TOOLS_DIR)" "$(GO)" install
"github.com/google/go-licenses@$(GO_LICENSES_VERSION)"
+
# Run lint against the previous commit for PR and branch build
# In dev setup look at all changes on top of master
.PHONY: lint
@@ -291,6 +301,30 @@ endif
fi
@echo " all OK"
+# Check licenses of go dependencies
+.PHONY: go-license-check
+go-license-check: $(GO_LICENSES_BIN)
+ @echo "Checking third-party licenses"
+ @"$(GO_LICENSES_BIN)" check ./pkg/... ./test/... --include_tests
--disallowed_types=forbidden,permissive,reciprocal,restricted,unknown
+ @echo "License checks OK"
+
+# Generate third-party dependency licenses
+.PHONY: go-license-generate
+go-license-generate: $(OUTPUT)/third-party-licenses.md
+
+$(OUTPUT)/third-party-licenses.md: $(GO_LICENSES_BIN) go.mod go.sum
+ @echo "Generating third-party licenses file"
+ @mkdir -p "$(OUTPUT)"
+ @rm -f "$(OUTPUT)/third-party-licenses.md"
+ @"$(GO_LICENSES_BIN)" \
+ report ./pkg/... \
+ --template=./scripts/third-party-licences.md.tpl \
+ --ignore github.com/apache/yunikorn-k8shim \
+ --ignore github.com/apache/yunikorn-core \
+ --ignore github.com/apache/yunikorn-scheduler-interface \
+ > "$(OUTPUT)/third-party-licenses.md.tmp"
+ @mv "$(OUTPUT)/third-party-licenses.md.tmp"
"$(OUTPUT)/third-party-licenses.md"
+
# Check that we use pseudo versions in master
.PHONY: pseudo
BRANCH := $(shell git branch --show-current)
@@ -385,12 +419,13 @@ $(RELEASE_BIN_DIR)/$(PLUGIN_BINARY): go.mod go.sum
$(shell find pkg)
# Build a scheduler image based on the production ready version
.PHONY: sched_image
-sched_image: scheduler docker/scheduler
+sched_image: $(OUTPUT)/third-party-licenses.md scheduler docker/scheduler
@echo "building scheduler docker image"
@rm -rf "$(DOCKER_DIR)/scheduler"
@mkdir -p "$(DOCKER_DIR)/scheduler"
@cp -a "docker/scheduler/." "$(DOCKER_DIR)/scheduler/."
@cp "$(RELEASE_BIN_DIR)/$(SCHEDULER_BINARY)" "$(DOCKER_DIR)/scheduler/."
+ @cp -a LICENSE NOTICE "$(OUTPUT)/third-party-licenses.md"
"$(DOCKER_DIR)/scheduler/."
DOCKER_BUILDKIT=1 docker build \
"$(DOCKER_DIR)/scheduler" \
-t "$(SCHEDULER_TAG)" \
@@ -404,12 +439,13 @@ sched_image: scheduler docker/scheduler
# Build a plugin image based on the production ready version
.PHONY: plugin_image
-plugin_image: plugin docker/plugin conf/scheduler-config.yaml
+plugin_image: $(OUTPUT)/third-party-licenses.md plugin docker/plugin
conf/scheduler-config.yaml
@echo "building plugin docker image"
@rm -rf "$(DOCKER_DIR)/plugin"
@mkdir -p "$(DOCKER_DIR)/plugin"
@cp -a "docker/plugin/." "$(DOCKER_DIR)/plugin/."
@cp "$(RELEASE_BIN_DIR)/$(PLUGIN_BINARY)" "$(DOCKER_DIR)/plugin/."
+ @cp -a LICENSE NOTICE "$(OUTPUT)/third-party-licenses.md"
"$(DOCKER_DIR)/plugin/."
@cp conf/scheduler-config.yaml
"$(DOCKER_DIR)/plugin/scheduler-config.yaml"
DOCKER_BUILDKIT=1 docker build \
"$(DOCKER_DIR)/plugin" \
@@ -440,12 +476,13 @@ $(RELEASE_BIN_DIR)/$(ADMISSION_CONTROLLER_BINARY): go.mod
go.sum $(shell find pk
# Build an admission controller image based on the production ready version
.PHONY: adm_image
-adm_image: admission docker/admission
+adm_image: $(OUTPUT)/third-party-licenses.md admission docker/admission
@echo "building admission controller docker image"
@rm -rf "$(DOCKER_DIR)/admission"
@mkdir -p "$(DOCKER_DIR)/admission"
@cp -a "docker/admission/." "$(DOCKER_DIR)/admission/."
@cp "$(RELEASE_BIN_DIR)/$(ADMISSION_CONTROLLER_BINARY)"
"$(DOCKER_DIR)/admission/."
+ @cp -a LICENSE NOTICE "$(OUTPUT)/third-party-licenses.md"
"$(DOCKER_DIR)/admission/."
DOCKER_BUILDKIT=1 docker build \
"$(DOCKER_DIR)/admission" \
-t "$(ADMISSION_TAG)" \
@@ -463,12 +500,13 @@ image: sched_image plugin_image adm_image
# Build a web server image ONLY to be used in e2e tests
.PHONY: webtest_image
-webtest_image: build_web_test_server_prod docker/webtest
+webtest_image: $(OUTPUT)/third-party-licenses.md build_web_test_server_prod
docker/webtest
@echo "building web server image for automated e2e tests"
@rm -rf "$(DOCKER_DIR)/webtest"
@mkdir -p "$(DOCKER_DIR)/webtest"
@cp -a "docker/webtest/." "$(DOCKER_DIR)/webtest/."
@cp "$(RELEASE_BIN_DIR)/$(TEST_SERVER_BINARY)" "$(DOCKER_DIR)/webtest/."
+ @cp -a LICENSE NOTICE "$(OUTPUT)/third-party-licenses.md"
"$(DOCKER_DIR)/webtest/."
DOCKER_BUILDKIT=1 docker build \
"$(DOCKER_DIR)/webtest" \
-t "${REGISTRY}/yunikorn:webtest-${DOCKER_ARCH}-${VERSION}" \
diff --git a/docker/admission/Dockerfile b/docker/admission/Dockerfile
index 2a5fa879..5482b15e 100644
--- a/docker/admission/Dockerfile
+++ b/docker/admission/Dockerfile
@@ -15,6 +15,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
FROM --platform=$TARGETPLATFORM scratch
-COPY --chown=0:0 yunikorn-admission-controller /
+COPY --chown=0:0 LICENSE NOTICE third-party-licenses.md
yunikorn-admission-controller /
USER 4444:4444
ENTRYPOINT [ "/yunikorn-admission-controller" ]
diff --git a/docker/plugin/Dockerfile b/docker/plugin/Dockerfile
index 62580210..4a3a8ca9 100644
--- a/docker/plugin/Dockerfile
+++ b/docker/plugin/Dockerfile
@@ -15,6 +15,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
FROM --platform=$TARGETPLATFORM scratch
-COPY --chown=0:0 yunikorn-scheduler-plugin scheduler-config.yaml /
+COPY --chown=0:0 LICENSE NOTICE third-party-licenses.md
yunikorn-scheduler-plugin scheduler-config.yaml /
USER 4444:4444
ENTRYPOINT [ "/yunikorn-scheduler-plugin", "--bind-address=0.0.0.0",
"--config=/scheduler-config.yaml" ]
diff --git a/docker/scheduler/Dockerfile b/docker/scheduler/Dockerfile
index dacca801..3d2430b1 100644
--- a/docker/scheduler/Dockerfile
+++ b/docker/scheduler/Dockerfile
@@ -15,6 +15,6 @@
# See the License for the specific language governing permissions and
# limitations under the License.
FROM --platform=$TARGETPLATFORM scratch
-COPY --chown=0:0 yunikorn-scheduler /
+COPY --chown=0:0 LICENSE NOTICE third-party-licenses.md yunikorn-scheduler /
USER 4444:4444
ENTRYPOINT [ "/yunikorn-scheduler" ]
diff --git a/docker/webtest/Dockerfile b/docker/webtest/Dockerfile
index 5f6269b6..e0ef92cf 100644
--- a/docker/webtest/Dockerfile
+++ b/docker/webtest/Dockerfile
@@ -18,7 +18,7 @@
# Imagestage: use scratch base image
FROM --platform=$TARGETPLATFORM scratch
COPY --chown=0:0 document/index.html /html/
-COPY --chown=0:0 web-test-server /
+COPY --chown=0:0 LICENSE NOTICE third-party-licenses.md web-test-server /
EXPOSE 9889
ENV DOCUMENT_ROOT /html
USER 4444:4444
diff --git a/scripts/third-party-licences.md.tpl
b/scripts/third-party-licences.md.tpl
new file mode 100644
index 00000000..0bc08e61
--- /dev/null
+++ b/scripts/third-party-licences.md.tpl
@@ -0,0 +1,11 @@
+{{ range . }}
+## {{ .Name }}
+
+* Name: {{ .Name }}
+* Version: {{ .Version }}
+* License: [{{ .LicenseName }}]({{ .LicenseURL }})
+
+```
+{{ .LicenseText }}
+```
+{{ end }}
---------------------------------------------------------------------
To unsubscribe, e-mail: [email protected]
For additional commands, e-mail: [email protected]
