This is an automated email from the ASF dual-hosted git repository. tloubrieu pushed a commit to branch SDAP-268 in repository https://gitbox.apache.org/repos/asf/incubator-sdap-nexus.git
commit 94ad24110420d524c98aa11db17089370f62f7c4 Author: Eamon Ford <[email protected]> AuthorDate: Mon Jul 27 10:29:33 2020 -0700 cassandra helm chart included --- data-access/nexustiles/dao/CassandraProxy.py | 8 +- helm/requirements.yaml | 5 +- helm/templates/cassandra.yml | 107 --------------------------- helm/templates/granule-ingester.yml | 4 + helm/templates/init-cassandra-configmap.yml | 13 ++++ helm/templates/webapp.yml | 2 +- helm/values.yaml | 30 +++++--- 7 files changed, 45 insertions(+), 124 deletions(-) diff --git a/data-access/nexustiles/dao/CassandraProxy.py b/data-access/nexustiles/dao/CassandraProxy.py index 2b27104..2189efd 100644 --- a/data-access/nexustiles/dao/CassandraProxy.py +++ b/data-access/nexustiles/dao/CassandraProxy.py @@ -13,20 +13,22 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging import uuid from ConfigParser import NoOptionError -from multiprocessing.synchronize import Lock -from cassandra.auth import PlainTextAuthProvider import nexusproto.DataTile_pb2 as nexusproto import numpy as np +from cassandra.auth import PlainTextAuthProvider from cassandra.cqlengine import columns, connection, CQLEngineException from cassandra.cqlengine.models import Model from cassandra.policies import TokenAwarePolicy, DCAwareRoundRobinPolicy, WhiteListRoundRobinPolicy +from multiprocessing.synchronize import Lock from nexusproto.serialization import from_shaped_array INIT_LOCK = Lock() +logger = logging.getLogger(__name__) class NexusTileData(Model): __table_name__ = 'sea_surface_temp' @@ -171,7 +173,7 @@ class CassandraProxy(object): self.__open() def __open(self): - + logger.info("Connecting to cassandra at " + self.__cass_url) if self.__cass_dc_policy == 'DCAwareRoundRobinPolicy': dc_policy = DCAwareRoundRobinPolicy(self.__cass_local_DC) elif self.__cass_dc_policy == 'WhiteListRoundRobinPolicy': diff --git a/helm/requirements.yaml b/helm/requirements.yaml index ebcd9c0..3e038e7 100644 --- a/helm/requirements.yaml +++ b/helm/requirements.yaml @@ -10,6 +10,7 @@ dependencies: - name: solr version: 1.5.2 repository: http://storage.googleapis.com/kubernetes-charts-incubator - - + - name: cassandra + version: 5.5.3 + repository: https://charts.bitnami.com/bitnami diff --git a/helm/templates/cassandra.yml b/helm/templates/cassandra.yml deleted file mode 100644 index 6023e55..0000000 --- a/helm/templates/cassandra.yml +++ /dev/null @@ -1,107 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: sdap-cassandra -spec: - clusterIP: None - ports: - - name: cql - port: 9042 - targetPort: cql - selector: - app: sdap-cassandra - ---- - -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: cassandra-set -spec: - serviceName: sdap-cassandra - replicas: {{ .Values.cassandra.replicas }} - selector: - matchLabels: - app: sdap-cassandra - template: - metadata: - labels: - app: sdap-cassandra - spec: - terminationGracePeriodSeconds: 120 - {{ if .Values.cassandra.tolerations }} - tolerations: -{{ .Values.cassandra.tolerations | toYaml | indent 6 }} - {{ end }} - {{ if .Values.cassandra.nodeSelector }} - nodeSelector: -{{ .Values.cassandra.nodeSelector | toYaml | indent 8 }} - {{ end }} - affinity: - podAntiAffinity: - # Prefer spreading over all hosts - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: "app" - operator: In - values: - - sdap-cassandra - topologyKey: "kubernetes.io/hostname" - containers: - - name: cassandra - image: nexusjpl/cassandra:1.0.0-rc1 - imagePullPolicy: Always - ports: - - containerPort: 7000 - name: intra-node - - containerPort: 7001 - name: tls-intra-node - - containerPort: 7199 - name: jmx - - containerPort: 9042 - name: cql - resources: - requests: - cpu: {{ .Values.cassandra.requests.cpu }} - memory: {{ .Values.cassandra.requests.memory }} - limits: - cpu: {{ .Values.cassandra.limits.cpu }} - memory: {{ .Values.cassandra.limits.memory }} - securityContext: - capabilities: - add: - - IPC_LOCK - lifecycle: - preStop: - exec: - command: - - /bin/sh - - -c - - nodetool drain - env: - - name: MAX_HEAP_SIZE - value: 2G - - name: HEAP_NEWSIZE - value: 200M - - name: CASSANDRA_SEEDS - value: "cassandra-set-0.sdap-cassandra" - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - volumeMounts: - - name: cassandra-data - mountPath: /var/lib/cassandra - - volumeClaimTemplates: - - metadata: - name: cassandra-data - spec: - accessModes: [ "ReadWriteOnce" ] - storageClassName: {{ .Values.storageClass }} - resources: - requests: - storage: {{ .Values.cassandra.storage }} diff --git a/helm/templates/granule-ingester.yml b/helm/templates/granule-ingester.yml index 2a716f1..312e7dd 100644 --- a/helm/templates/granule-ingester.yml +++ b/helm/templates/granule-ingester.yml @@ -28,6 +28,10 @@ spec: value: {{ .Values.rabbitmq.fullnameOverride }} - name: CASSANDRA_CONTACT_POINTS value: sdap-cassandra + - name: CASSANDRA_USERNAME + value: cassandra + - name: CASSANDRA_PASSWORD + value: cassandra - name: ZK_HOST_AND_PORT value: {{ .Release.Name }}-zookeeper:2181 {{ if .Values.ingestion.granuleIngester.maxConcurrency }} diff --git a/helm/templates/init-cassandra-configmap.yml b/helm/templates/init-cassandra-configmap.yml new file mode 100644 index 0000000..eadbc9c --- /dev/null +++ b/helm/templates/init-cassandra-configmap.yml @@ -0,0 +1,13 @@ +apiVersion: v1 +data: + init.cql: | + CREATE KEYSPACE IF NOT EXISTS nexustiles WITH REPLICATION = { 'class': 'SimpleStrategy', 'replication_factor': 1 }; + + CREATE TABLE IF NOT EXISTS nexustiles.sea_surface_temp ( + tile_id uuid PRIMARY KEY, + tile_blob blob + ); +kind: ConfigMap +metadata: + name: init-cassandra + namespace: sdap diff --git a/helm/templates/webapp.yml b/helm/templates/webapp.yml index 2963cce..8efd3ec 100644 --- a/helm/templates/webapp.yml +++ b/helm/templates/webapp.yml @@ -12,7 +12,7 @@ spec: imagePullPolicy: IfNotPresent mainApplicationFile: local:///incubator-sdap-nexus/analysis/webservice/webapp.py arguments: - - "--cassandra-host=sdap-cassandra --solr-host={{ .Release.Name }}-solr-svc:8983" + - "--cassandra-host={{ .Release.Name }}-cassandra --cassandra-username=cassandra --cassandra-password=cassandra --solr-host={{ .Release.Name }}-solr-svc:8983" sparkVersion: "2.4.4" restartPolicy: type: OnFailure diff --git a/helm/values.yaml b/helm/values.yaml index aa03c0a..c9b9cf1 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -99,16 +99,6 @@ ingestion: ## Store ingestion history in a solr database instead of a filesystem directory # url: http://history-solr -cassandra: - replicas: 2 - storage: 13Gi - requests: - cpu: 1 - memory: 3Gi - limits: - cpu: 1 - memory: 3Gi - solr: replicaCount: 3 volumeClaimTemplates: @@ -165,4 +155,22 @@ rabbitmq: username: guest password: guest ingress: - enabled: true \ No newline at end of file + enabled: true + +cassandra: + initDBConfigMap: init-cassandra + dbUser: + user: cassandra + password: cassandra + cluster: + replicaCount: 1 + persistence: + storageClass: hostpath + size: 8Gi + resources: + requests: + cpu: 1 + memory: 8Gi + limits: + cpu: 1 + memory: 8Gi
