This is an automated email from the ASF dual-hosted git repository. eamonford pushed a commit to branch bug_fixes in repository https://gitbox.apache.org/repos/asf/incubator-sdap-nexus.git
commit 1b3360b795a4cab4ccfd71f51045133b7c07c887 Author: Eamon Ford <[email protected]> AuthorDate: Tue Nov 10 15:16:29 2020 -0800 Add S3 support to helm chart --- .gitignore | 1 + helm/Chart.yaml | 2 +- helm/templates/collection-manager.yml | 16 +++++++++++++++- helm/templates/granule-ingester.yml | 8 ++++++++ helm/values.yaml | 15 ++++++++++++--- 5 files changed, 37 insertions(+), 5 deletions(-) diff --git a/.gitignore b/.gitignore index 3e29626..12ab2d6 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ +*.pytest_cache *.vscode *.code-workspace *.idea diff --git a/helm/Chart.yaml b/helm/Chart.yaml index 2a2e9d1..6689340 100644 --- a/helm/Chart.yaml +++ b/helm/Chart.yaml @@ -1,5 +1,5 @@ apiVersion: v1 -appVersion: "0.1.5" +appVersion: "0.2.1" description: Science Data Analytics Platform name: nexus version: 0.2.0 diff --git a/helm/templates/collection-manager.yml b/helm/templates/collection-manager.yml index e281526..a116dd0 100644 --- a/helm/templates/collection-manager.yml +++ b/helm/templates/collection-manager.yml @@ -1,5 +1,7 @@ {{- if .Values.ingestion.enabled }} {{- $history := .Values.ingestion.history | default dict }} +{{- $s3 := .Values.ingestion.granules.s3 | default dict }} +{{ $aws_creds_envs := $s3.awsCredsEnvs | default dict }} apiVersion: apps/v1 kind: Deployment @@ -37,6 +39,14 @@ spec: - name: HISTORY_PATH value: {{ include "nexus.history.mountPath" . }} {{- end }} + {{- if $s3.bucket }} + - name: S3_BUCKET + value: {{ $s3.bucket }} + {{- end }} + {{- range $name, $value := $aws_creds_envs }} + - name: {{ $name }} + value: {{ $value }} + {{- end }} resources: requests: cpu: {{ .Values.ingestion.collectionManager.cpu }} @@ -45,15 +55,19 @@ spec: cpu: {{ .Values.ingestion.collectionManager.cpu }} memory: {{ .Values.ingestion.collectionManager.memory }} volumeMounts: -{{ include "nexus.ingestion.dataVolumeMount" . | indent 12 }} {{- if not $history.solrEnabled }} - name: history-volume mountPath: {{ include "nexus.history.mountPath" . }} {{- end }} - name: collections-config-volume mountPath: {{ include "nexus.collectionsConfig.mountPath" . }} +{{- if not $s3.bucket }} +{{ include "nexus.ingestion.dataVolumeMount" . | indent 12 }} +{{- end }} volumes: +{{- if not $s3.bucket }} {{ include "nexus.ingestion.dataVolume" . | indent 8 }} +{{- end }} - name: collections-config-volume configMap: name: {{ include "nexus.collectionsConfig.configmapName" . }} diff --git a/helm/templates/granule-ingester.yml b/helm/templates/granule-ingester.yml index bb616ad..323a5fb 100644 --- a/helm/templates/granule-ingester.yml +++ b/helm/templates/granule-ingester.yml @@ -1,4 +1,8 @@ {{- if .Values.ingestion.enabled }} + +{{- $s3 := .Values.ingestion.granules.s3 | default dict }} +{{ $aws_creds_envs := $s3.awsCredsEnvs | default dict }} + apiVersion: apps/v1 kind: Deployment metadata: @@ -38,6 +42,10 @@ spec: - name: MAX_CONCURRENCY value: "{{ .Values.ingestion.granuleIngester.maxConcurrency }}" {{ end }} + {{- range $name, $value := $aws_creds_envs }} + - name: {{ $name }} + value: {{ $value }} + {{- end }} resources: requests: cpu: {{ .Values.ingestion.granuleIngester.cpu }} diff --git a/helm/values.yaml b/helm/values.yaml index 041b0ba..32df703 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -8,7 +8,7 @@ rootWebpage: webapp: enabled: true distributed: - image: nexusjpl/nexus-webapp:distributed.0.2.0 + image: nexusjpl/nexus-webapp:distributed.0.2.1 ## Use any of the driver configuration options available at ## https://github.com/GoogleCloudPlatform/spark-on-k8s-operator/blob/master/docs/user-guide.md @@ -31,7 +31,7 @@ ingestion: granuleIngester: replicas: 2 - image: nexusjpl/granule-ingester:0.0.4 + image: nexusjpl/granule-ingester:0.1.0a3 ## cpu refers to both request and limit cpu: 1 @@ -40,7 +40,7 @@ ingestion: memory: 1Gi collectionManager: - image: nexusjpl/collection-manager:0.0.4 + image: nexusjpl/collection-manager:0.1.0a7 ## cpu refers to both request and limit cpu: 0.5 @@ -71,6 +71,15 @@ ingestion: ## the granule files are stored. path: /var/lib/sdap/granules + # s3: + # bucket: nexus-ingest + # awsCredsEnvs: + # AWS_ACCESS_KEY_ID: secret + # AWS_SECRET_ACCESS_KEY: secret + # AWS_SESSION_TOKEN: secret + # AWS_DEFAULT_REGION: us-west-2 + + ## Where to find the Collections Config file ## ref: https://github.com/apache/incubator-sdap-ingester/tree/dev/collection_manager#the-collections-configuration-file ## Either localDir should be set, or the git options, but not both.
