This is an automated email from the ASF dual-hosted git repository. eamonford pushed a commit to branch bug_fixes in repository https://gitbox.apache.org/repos/asf/incubator-sdap-nexus.git
commit bb3158553c19a911631a8d7bbac711ac51c6dc34 Author: Eamon Ford <[email protected]> AuthorDate: Thu Jul 16 18:54:07 2020 -0700 updated helm chart for zookeeper use solr and zk helm charts change .Release.Namespace to .Release.Name add rabbitmq storageclass fix rbac add max_concurrency add solr_host arg add solr port always deploy solr add solr-host option read cli args for cass and solr hosts pass cassandra host add support for cassandra username and password cassandra helm chart included fix arguments sent to spark driver, add logging in cassandraproxy pass factory method to nexuscalchandlers to create tile service in spark nodes fix namespace fix bad argument order fix cass url for granule ingester change solr-create-collection to a deployment make solr history default pr enable external solr/zk/cass hosts rabbitmq.enabled revert doms revert update images only deploy config operator if it is enabled remove http:// from solr hardcoded endpoint turn off configmap by default --- .gitignore | 1 - analysis/setup.py | 3 +- analysis/webservice/algorithms_spark/__init__.py | 6 - analysis/webservice/config/web.ini | 2 +- data-access/nexustiles/dao/CassandraProxy.py | 3 + data-access/tests/config/datastores.ini | 9 -- helm/requirements.yaml | 11 +- helm/templates/_helpers.tpl | 9 +- helm/templates/cassandra.yml | 107 ----------------- helm/templates/collection-manager.yml | 10 +- helm/templates/config-operator-rbac.yml | 4 +- helm/templates/config-operator.yml | 3 +- helm/templates/granule-ingester.yml | 15 ++- helm/templates/history-pvc.yml | 2 + helm/templates/init-cassandra-configmap.yml | 13 ++ helm/templates/solr-create-collection.yml | 34 ++++++ helm/templates/solr.yml | 129 -------------------- helm/templates/webapp.yml | 7 +- helm/templates/zookeeper.yml | 144 ----------------------- helm/values.yaml | 86 +++++++++----- tools/doms/README.md | 66 ----------- tools/doms/doms_reader.py | 144 ----------------------- 22 files changed, 154 insertions(+), 654 deletions(-) diff --git a/.gitignore b/.gitignore index 3e29626..4e4cf6e 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,5 @@ *.code-workspace *.idea *.DS_Store -analysis/webservice/algorithms/doms/domsconfig.ini data-access/nexustiles/config/datastores.ini venv/ diff --git a/analysis/setup.py b/analysis/setup.py index 62a6891..9a449ce 100644 --- a/analysis/setup.py +++ b/analysis/setup.py @@ -50,8 +50,7 @@ setuptools.setup( # 'webservice.nexus_tornado.request.renderers' #], package_data={ - 'webservice': ['config/web.ini', 'config/algorithms.ini'], - 'webservice.algorithms.doms': ['domsconfig.ini.default'] + 'webservice': ['config/web.ini', 'config/algorithms.ini'] }, data_files=[ ('static', ['static/index.html']) diff --git a/analysis/webservice/algorithms_spark/__init__.py b/analysis/webservice/algorithms_spark/__init__.py index d6ed83f..a25c8d5 100644 --- a/analysis/webservice/algorithms_spark/__init__.py +++ b/analysis/webservice/algorithms_spark/__init__.py @@ -20,7 +20,6 @@ import ClimMapSpark import CorrMapSpark import DailyDifferenceAverageSpark import HofMoellerSpark -import Matchup import MaximaMinimaSpark import NexusCalcSparkHandler import TimeAvgMapSpark @@ -47,11 +46,6 @@ if module_exists("pyspark"): pass try: - import Matchup - except ImportError: - pass - - try: import TimeAvgMapSpark except ImportError: pass diff --git a/analysis/webservice/config/web.ini b/analysis/webservice/config/web.ini index 2644ade..a1ecb2c 100644 --- a/analysis/webservice/config/web.ini +++ b/analysis/webservice/config/web.ini @@ -14,4 +14,4 @@ static_enabled=true static_dir=static [modules] -module_dirs=webservice.algorithms,webservice.algorithms_spark,webservice.algorithms.doms \ No newline at end of file +module_dirs=webservice.algorithms,webservice.algorithms_spark \ No newline at end of file diff --git a/data-access/nexustiles/dao/CassandraProxy.py b/data-access/nexustiles/dao/CassandraProxy.py index a8a4e6e..54a849b 100644 --- a/data-access/nexustiles/dao/CassandraProxy.py +++ b/data-access/nexustiles/dao/CassandraProxy.py @@ -161,6 +161,9 @@ class CassandraProxy(object): self.__cass_protocol_version = config.getint("cassandra", "protocol_version") self.__cass_dc_policy = config.get("cassandra", "dc_policy") + logger.info("Setting cassandra host to " + self.__cass_url) + logger.info("Setting cassandra username to " + self.__cass_username) + try: self.__cass_port = config.getint("cassandra", "port") except NoOptionError: diff --git a/data-access/tests/config/datastores.ini b/data-access/tests/config/datastores.ini deleted file mode 100644 index 194760c..0000000 --- a/data-access/tests/config/datastores.ini +++ /dev/null @@ -1,9 +0,0 @@ -[cassandra] -host=127.0.0.1 -keyspace=nexustiles -local_datacenter=datacenter1 -protocol_version=3 - -[solr] -host=localhost:8983 -core=nexustiles \ No newline at end of file diff --git a/helm/requirements.yaml b/helm/requirements.yaml index 7970f29..78cc52e 100644 --- a/helm/requirements.yaml +++ b/helm/requirements.yaml @@ -6,6 +6,13 @@ dependencies: - name: rabbitmq version: 7.1.0 repository: https://charts.bitnami.com/bitnami - condition: ingestion.enabled - + condition: rabbitmq.enabled + - name: solr + version: 1.5.2 + repository: http://storage.googleapis.com/kubernetes-charts-incubator + condition: solr.enabled + - name: cassandra + version: 5.5.3 + repository: https://charts.bitnami.com/bitnami + condition: cassandra.enabled diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl index b697c17..5944f33 100644 --- a/helm/templates/_helpers.tpl +++ b/helm/templates/_helpers.tpl @@ -4,7 +4,7 @@ Name of the generated configmap containing the contents of the collections config file. */}} {{- define "nexus.collectionsConfig.configmapName" -}} -collections-config +{{ .Values.ingestion.collections.configMap | default "collections-config" }} {{- end -}} {{/* @@ -45,3 +45,10 @@ The data volume mount which is used in both the Collection Manager and the Granu mountPath: {{ .Values.ingestion.granules.mountPath }} {{- end -}} +{{- define "nexus.urls.solr" -}} +{{ .Values.external.solrHostAndPort | default (print "http://"; .Release.Name "-solr-svc:8983") }} +{{- end -}} + +{{- define "nexus.urls.zookeeper" -}} +{{ .Values.external.zookeeperHostAndPort | default (print .Release.Name "-zookeeper:2181") }} +{{- end -}} \ No newline at end of file diff --git a/helm/templates/cassandra.yml b/helm/templates/cassandra.yml deleted file mode 100644 index 6023e55..0000000 --- a/helm/templates/cassandra.yml +++ /dev/null @@ -1,107 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: sdap-cassandra -spec: - clusterIP: None - ports: - - name: cql - port: 9042 - targetPort: cql - selector: - app: sdap-cassandra - ---- - -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: cassandra-set -spec: - serviceName: sdap-cassandra - replicas: {{ .Values.cassandra.replicas }} - selector: - matchLabels: - app: sdap-cassandra - template: - metadata: - labels: - app: sdap-cassandra - spec: - terminationGracePeriodSeconds: 120 - {{ if .Values.cassandra.tolerations }} - tolerations: -{{ .Values.cassandra.tolerations | toYaml | indent 6 }} - {{ end }} - {{ if .Values.cassandra.nodeSelector }} - nodeSelector: -{{ .Values.cassandra.nodeSelector | toYaml | indent 8 }} - {{ end }} - affinity: - podAntiAffinity: - # Prefer spreading over all hosts - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: "app" - operator: In - values: - - sdap-cassandra - topologyKey: "kubernetes.io/hostname" - containers: - - name: cassandra - image: nexusjpl/cassandra:1.0.0-rc1 - imagePullPolicy: Always - ports: - - containerPort: 7000 - name: intra-node - - containerPort: 7001 - name: tls-intra-node - - containerPort: 7199 - name: jmx - - containerPort: 9042 - name: cql - resources: - requests: - cpu: {{ .Values.cassandra.requests.cpu }} - memory: {{ .Values.cassandra.requests.memory }} - limits: - cpu: {{ .Values.cassandra.limits.cpu }} - memory: {{ .Values.cassandra.limits.memory }} - securityContext: - capabilities: - add: - - IPC_LOCK - lifecycle: - preStop: - exec: - command: - - /bin/sh - - -c - - nodetool drain - env: - - name: MAX_HEAP_SIZE - value: 2G - - name: HEAP_NEWSIZE - value: 200M - - name: CASSANDRA_SEEDS - value: "cassandra-set-0.sdap-cassandra" - - name: POD_IP - valueFrom: - fieldRef: - fieldPath: status.podIP - volumeMounts: - - name: cassandra-data - mountPath: /var/lib/cassandra - - volumeClaimTemplates: - - metadata: - name: cassandra-data - spec: - accessModes: [ "ReadWriteOnce" ] - storageClassName: {{ .Values.storageClass }} - resources: - requests: - storage: {{ .Values.cassandra.storage }} diff --git a/helm/templates/collection-manager.yml b/helm/templates/collection-manager.yml index 6708b13..e281526 100644 --- a/helm/templates/collection-manager.yml +++ b/helm/templates/collection-manager.yml @@ -19,7 +19,7 @@ spec: spec: containers: - image: {{ .Values.ingestion.collectionManager.image }} - imagePullPolicy: Always + imagePullPolicy: IfNotPresent name: collection-manager env: - name: RABBITMQ_USERNAME @@ -30,9 +30,9 @@ spec: value: {{ .Values.rabbitmq.fullnameOverride }} - name: COLLECTIONS_PATH value: {{ include "nexus.collectionsConfig.mountPath" . }}/collections.yml - {{- if $history.url }} + {{- if $history.solrEnabled }} - name: HISTORY_URL - value: {{ .Values.ingestion.history.url}} + value: {{ include "nexus.urls.solr" . }} {{- else }} - name: HISTORY_PATH value: {{ include "nexus.history.mountPath" . }} @@ -46,7 +46,7 @@ spec: memory: {{ .Values.ingestion.collectionManager.memory }} volumeMounts: {{ include "nexus.ingestion.dataVolumeMount" . | indent 12 }} - {{- if not $history.url }} + {{- if not $history.solrEnabled }} - name: history-volume mountPath: {{ include "nexus.history.mountPath" . }} {{- end }} @@ -57,7 +57,7 @@ spec: - name: collections-config-volume configMap: name: {{ include "nexus.collectionsConfig.configmapName" . }} - {{- if not $history.url }} + {{- if not $history.solrEnabled }} - name: history-volume persistentVolumeClaim: claimName: history-volume-claim diff --git a/helm/templates/config-operator-rbac.yml b/helm/templates/config-operator-rbac.yml index 54064d5..6626b0b 100644 --- a/helm/templates/config-operator-rbac.yml +++ b/helm/templates/config-operator-rbac.yml @@ -6,7 +6,7 @@ metadata: --- apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding +kind: ClusterRoleBinding metadata: name: config-operator-role-binding roleRef: @@ -16,4 +16,6 @@ roleRef: subjects: - kind: ServiceAccount name: config-operator + namespace: {{ .Release.Namespace }} + diff --git a/helm/templates/config-operator.yml b/helm/templates/config-operator.yml index 3f56f44..298095e 100644 --- a/helm/templates/config-operator.yml +++ b/helm/templates/config-operator.yml @@ -1,4 +1,5 @@ {{ if .Values.ingestion.enabled }} +{{ if not .Values.ingestion.collections.configMap }} apiVersion: apps/v1 kind: Deployment metadata: @@ -21,4 +22,4 @@ spec: image: {{ .Values.ingestion.configOperator.image }} imagePullPolicy: Always {{ end }} - +{{ end }} diff --git a/helm/templates/granule-ingester.yml b/helm/templates/granule-ingester.yml index 2ce03b6..bb616ad 100644 --- a/helm/templates/granule-ingester.yml +++ b/helm/templates/granule-ingester.yml @@ -17,6 +17,7 @@ spec: spec: containers: - image: {{ .Values.ingestion.granuleIngester.image }} + imagePullPolicy: IfNotPresent name: granule-ingester env: - name: RABBITMQ_USERNAME @@ -26,9 +27,17 @@ spec: - name: RABBITMQ_HOST value: {{ .Values.rabbitmq.fullnameOverride }} - name: CASSANDRA_CONTACT_POINTS - value: sdap-cassandra - - name: SOLR_HOST_AND_PORT - value: http://sdap-solr:8983 + value: {{ .Release.Name }}-cassandra + - name: CASSANDRA_USERNAME + value: cassandra + - name: CASSANDRA_PASSWORD + value: cassandra + - name: ZK_HOST_AND_PORT + value: {{ include "nexus.urls.zookeeper" . }} + {{ if .Values.ingestion.granuleIngester.maxConcurrency }} + - name: MAX_CONCURRENCY + value: "{{ .Values.ingestion.granuleIngester.maxConcurrency }}" + {{ end }} resources: requests: cpu: {{ .Values.ingestion.granuleIngester.cpu }} diff --git a/helm/templates/history-pvc.yml b/helm/templates/history-pvc.yml index 3ecabe9..ed18f76 100644 --- a/helm/templates/history-pvc.yml +++ b/helm/templates/history-pvc.yml @@ -2,6 +2,8 @@ apiVersion: v1 kind: PersistentVolumeClaim metadata: name: history-volume-claim + annotations: + helm.sh/resource-policy: "keep" spec: accessModes: - ReadWriteOnce diff --git a/helm/templates/init-cassandra-configmap.yml b/helm/templates/init-cassandra-configmap.yml new file mode 100644 index 0000000..3e7ed3c --- /dev/null +++ b/helm/templates/init-cassandra-configmap.yml @@ -0,0 +1,13 @@ +apiVersion: v1 +data: + init.cql: | + CREATE KEYSPACE IF NOT EXISTS nexustiles WITH REPLICATION = { 'class': 'SimpleStrategy', 'replication_factor': 1 }; + + CREATE TABLE IF NOT EXISTS nexustiles.sea_surface_temp ( + tile_id uuid PRIMARY KEY, + tile_blob blob + ); +kind: ConfigMap +metadata: + name: init-cassandra + namespace: {{ .Release.Namespace }} diff --git a/helm/templates/solr-create-collection.yml b/helm/templates/solr-create-collection.yml new file mode 100644 index 0000000..7ecb2e3 --- /dev/null +++ b/helm/templates/solr-create-collection.yml @@ -0,0 +1,34 @@ +{{ if .Values.solrInitEnabled }} +apiVersion: apps/v1 +kind: Deployment +metadata: + name: solr-create-collection +spec: + selector: + matchLabels: + app: solr-create-collection # has to match .spec.template.metadata.labels + replicas: 1 + template: + metadata: + labels: + app: solr-create-collection + spec: + containers: + - name: solr-create-collection + imagePullPolicy: Always + image: nexusjpl/solr-cloud-init:1.0.1 + resources: + requests: + memory: "0.5Gi" + cpu: "0.25" + env: + - name: MINIMUM_NODES + value: "{{ .Values.solr.replicaCount }}" + - name: SDAP_SOLR_URL + value: {{ include "nexus.urls.solr" . }}/solr/ + - name: SDAP_ZK_SOLR + value: {{ include "nexus.urls.zookeeper" . }}/solr + - name: CREATE_COLLECTION_PARAMS + value: "name=nexustiles&numShards=$(MINIMUM_NODES)&waitForFinalState=true" + restartPolicy: Always +{{ end }} \ No newline at end of file diff --git a/helm/templates/solr.yml b/helm/templates/solr.yml deleted file mode 100644 index c8d0f9b..0000000 --- a/helm/templates/solr.yml +++ /dev/null @@ -1,129 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: sdap-solr -spec: - ports: - - port: 8983 - clusterIP: None - selector: - app: sdap-solr - ---- - -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: solr-set -spec: - selector: - matchLabels: - app: sdap-solr # has to match .spec.template.metadata.labels - serviceName: "sdap-solr" - replicas: {{.Values.solr.replicas }} # by default is 1 - podManagementPolicy: Parallel - template: - metadata: - labels: - app: sdap-solr # has to match .spec.selector.matchLabels - spec: - terminationGracePeriodSeconds: 10 - {{ if .Values.solr.tolerations }} - tolerations: -{{ .Values.solr.tolerations | toYaml | indent 6 }} - {{ end }} - {{ if .Values.solr.nodeSelector }} - nodeSelector: -{{ .Values.solr.nodeSelector | toYaml | indent 8 }} - {{ end }} - affinity: - podAntiAffinity: - # Prefer spreading over all hosts - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: "app" - operator: In - values: - - sdap-solr - topologyKey: "kubernetes.io/hostname" - securityContext: - runAsUser: 8983 - fsGroup: 8983 - containers: - - name: solr-create-collection - imagePullPolicy: Always - image: nexusjpl/solr-cloud-init:1.0.0-rc1 - resources: - requests: - memory: "1Gi" - cpu: "0.25" - env: - - name: MINIMUM_NODES - value: "2" # MINIMUM_NODES should be the same as spec.replicas - - name: SOLR_HOST - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: SDAP_SOLR_URL - value: http://$(SOLR_HOST):8983/solr/ - - name: SDAP_ZK_SOLR - value: "zk-hs:2181/solr" - - name: CREATE_COLLECTION_PARAMS - value: "name=nexustiles&collection.configName=nexustiles&numShards=$(MINIMUM_NODES)&waitForFinalState=true" - - name: solr-cloud - imagePullPolicy: Always - image: nexusjpl/solr-cloud:1.0.0-rc1 - resources: - requests: - memory: {{ .Values.solr.requests.memory }} - cpu: {{ .Values.solr.requests.cpu }} - limits: - memory: {{ .Values.solr.limits.memory }} - cpu: {{ .Values.solr.limits.cpu }} - env: - - name: SOLR_HEAP - value: {{ .Values.solr.heap }} - - name: SOLR_HOST - valueFrom: - fieldRef: - fieldPath: status.podIP - - name: SDAP_ZK_SERVICE_HOST - value: "zk-hs" - ports: - - containerPort: 8983 - name: http - volumeMounts: - - name: solr-data - mountPath: /opt/solr/server/solr/ - readinessProbe: - exec: - command: - - solr - - healthcheck - - -c - - nexustiles - - -z - - zk-hs:2181/solr - initialDelaySeconds: 10 - timeoutSeconds: 5 - livenessProbe: - exec: - command: - - solr - - assert - - -s - - http://localhost:8983/solr/ - initialDelaySeconds: 10 - timeoutSeconds: 5 - volumeClaimTemplates: - - metadata: - name: solr-data - spec: - accessModes: [ "ReadWriteOnce" ] - storageClassName: {{ .Values.storageClass }} - resources: - requests: - storage: {{ .Values.solr.storage }} diff --git a/helm/templates/webapp.yml b/helm/templates/webapp.yml index d77496f..e4e2adf 100644 --- a/helm/templates/webapp.yml +++ b/helm/templates/webapp.yml @@ -9,8 +9,13 @@ spec: pythonVersion: "2" mode: cluster image: {{ .Values.webapp.distributed.image }} - imagePullPolicy: Always + imagePullPolicy: IfNotPresent mainApplicationFile: local:///incubator-sdap-nexus/analysis/webservice/webapp.py + arguments: + - --cassandra-host={{ .Release.Name }}-cassandra + - --cassandra-username=cassandra + - --cassandra-password=cassandra + - --solr-host={{ include "nexus.urls.solr" . }} sparkVersion: "2.4.4" restartPolicy: type: OnFailure diff --git a/helm/templates/zookeeper.yml b/helm/templates/zookeeper.yml deleted file mode 100644 index bdc3925..0000000 --- a/helm/templates/zookeeper.yml +++ /dev/null @@ -1,144 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: zk-hs - labels: - app: zk -spec: - ports: - - port: 2888 - name: server - - port: 3888 - name: leader-election - clusterIP: None - selector: - app: zk ---- -apiVersion: v1 -kind: Service -metadata: - name: zk-cs - labels: - app: zk -spec: - ports: - - port: 2181 - name: client - selector: - app: zk ---- -apiVersion: policy/v1beta1 -kind: PodDisruptionBudget -metadata: - name: zk-pdb -spec: - selector: - matchLabels: - app: zk - maxUnavailable: 1 ---- -apiVersion: apps/v1 -kind: StatefulSet -metadata: - name: zk -spec: - selector: - matchLabels: - app: zk - serviceName: zk-hs - replicas: {{ .Values.zookeeper.replicas }} - updateStrategy: - type: RollingUpdate - podManagementPolicy: Parallel - template: - metadata: - labels: - app: zk - spec: - {{ if .Values.zookeeper.tolerations }} - tolerations: -{{ .Values.zookeeper.tolerations | toYaml | indent 6 }} - {{ end }} - {{ if .Values.zookeeper.nodeSelector }} - nodeSelector: -{{ .Values.zookeeper.nodeSelector | toYaml | indent 8 }} - {{ end }} - affinity: - podAntiAffinity: - preferredDuringSchedulingIgnoredDuringExecution: - - weight: 100 - podAffinityTerm: - labelSelector: - matchExpressions: - - key: "app" - operator: In - values: - - zk - topologyKey: "kubernetes.io/hostname" - containers: - - name: kubernetes-zookeeper - imagePullPolicy: Always - image: "k8s.gcr.io/kubernetes-zookeeper:1.0-3.4.10" - resources: - requests: - memory: {{ .Values.zookeeper.memory }} - cpu: {{ .Values.zookeeper.cpu }} - ports: - - containerPort: 2181 - name: client - - containerPort: 2888 - name: server - - containerPort: 3888 - name: leader-election - command: - - sh - - -c - - "start-zookeeper \ - --servers={{ .Values.zookeeper.replicas }} \ - --data_dir=/var/lib/zookeeper/data \ - --data_log_dir=/var/lib/zookeeper/data/log \ - --conf_dir=/opt/zookeeper/conf \ - --client_port=2181 \ - --election_port=3888 \ - --server_port=2888 \ - --tick_time=2000 \ - --init_limit=10 \ - --sync_limit=5 \ - --heap=512M \ - --max_client_cnxns=60 \ - --snap_retain_count=3 \ - --purge_interval=12 \ - --max_session_timeout=40000 \ - --min_session_timeout=4000 \ - --log_level=INFO" - readinessProbe: - exec: - command: - - sh - - -c - - "zookeeper-ready 2181" - initialDelaySeconds: 10 - timeoutSeconds: 5 - livenessProbe: - exec: - command: - - sh - - -c - - "zookeeper-ready 2181" - initialDelaySeconds: 10 - timeoutSeconds: 5 - volumeMounts: - - name: zkdatadir - mountPath: /var/lib/zookeeper - securityContext: - runAsUser: 1000 - fsGroup: 1000 - volumeClaimTemplates: - - metadata: - name: zkdatadir - spec: - accessModes: [ "ReadWriteOnce" ] - storageClassName: {{ .Values.storageClass }} - resources: - requests: - storage: {{ .Values.zookeeper.storage }} diff --git a/helm/values.yaml b/helm/values.yaml index c012e6e..4c7bca4 100644 --- a/helm/values.yaml +++ b/helm/values.yaml @@ -31,7 +31,7 @@ ingestion: granuleIngester: replicas: 2 - image: nexusjpl/granule-ingester:0.0.1 + image: nexusjpl/granule-ingester:0.0.3 ## cpu refers to both request and limit cpu: 1 @@ -40,7 +40,7 @@ ingestion: memory: 1Gi collectionManager: - image: nexusjpl/collection-manager:0.0.2 + image: nexusjpl/collection-manager:0.0.3 ## cpu refers to both request and limit cpu: 0.5 @@ -78,53 +78,55 @@ ingestion: ## Load the Collections Config file from a local path ## This is a future option that is not yet supported! - # localDir: /Users/edford/Desktop/collections.yml + #configMap: collections-config ## Load the Collections Config file from a git repository ## Until localDir is supported, this configuration is mandatory git: - ## This should be an https repository url of the form https://github.com/username/repo.git url: branch: master - ## token is not yet supported! # token: someToken ## Where to store ingestion history ## Defaults to a using a history directory, stored on a PVC using the storageClass defined in this file above history: ## Store ingestion history in a solr database instead of a filesystem directory - # url: http://history-solr + solrEnabled: true -cassandra: - replicas: 2 - storage: 13Gi - requests: - cpu: 1 - memory: 3Gi - limits: - cpu: 1 - memory: 3Gi +external: + solrHostAndPort: + zookeeperHostAndPort: -solr: - replicas: 2 - storage: 10Gi - heap: 4g - requests: - memory: 5Gi - cpu: 1 - limits: - memory: 5Gi - cpu: 1 +solrInitEnabled: true -zookeeper: - replicas: 3 - memory: 1Gi - cpu: 0.5 - storage: 8Gi +solr: + enabled: true + replicaCount: 3 + volumeClaimTemplates: + storageClassName: hostpath + storageSize: 10Gi + resources: + requests: + memory: 2Gi + cpu: 1 + limits: + memory: 2Gi + cpu: 1 + zookeeper: + replicaCount: 3 + persistence: + storageClass: hostpath + resources: + limits: + memory: 1Gi + cpu: 0.5 + requests: + memory: 1Gi + cpu: 0.5 ingressEnabled: false @@ -150,10 +152,32 @@ nginx-ingress: rabbitmq: ## fullnameOverride sets the name of the RabbitMQ service ## with which the ingestion components will communicate. + enabled: true + persistence: + storageClass: hostpath fullnameOverride: rabbitmq replicaCount: 1 auth: username: guest password: guest ingress: - enabled: true \ No newline at end of file + enabled: true + +cassandra: + enabled: true + initDBConfigMap: init-cassandra + dbUser: + user: cassandra + password: cassandra + cluster: + replicaCount: 1 + persistence: + storageClass: hostpath + size: 8Gi + resources: + requests: + cpu: 1 + memory: 8Gi + limits: + cpu: 1 + memory: 8Gi diff --git a/tools/doms/README.md b/tools/doms/README.md deleted file mode 100644 index c49fa4a..0000000 --- a/tools/doms/README.md +++ /dev/null @@ -1,66 +0,0 @@ -# doms_reader.py -The functions in doms_reader.py read a DOMS netCDF file into memory, assemble a list of matches of satellite and in situ data, and optionally output the matches to a CSV file. Each matched pair contains one satellite data record and one in situ data record. - -The DOMS netCDF files hold satellite data and in situ data in different groups (`SatelliteData` and `InsituData`). The `matchIDs` netCDF variable contains pairs of IDs (matches) which reference a satellite data record and an in situ data record in their respective groups. These records have a many-to-many relationship; one satellite record may match to many in situ records, and one in situ record may match to many satellite records. The `assemble_matches` function assembles the individua [...] - -## Requirements -This tool was developed and tested with Python 2.7.5 and 3.7.0a0. -Imported packages: -* argparse -* netcdf4 -* sys -* datetime -* csv -* collections -* logging - - -## Functions -### Function: `assemble_matches(filename)` -Read a DOMS netCDF file into memory and return a list of matches from the file. - -#### Parameters -- `filename` (str): the DOMS netCDF file name. - -#### Returns -- `matches` (list): List of matches. - -Each list element in `matches` is a dictionary organized as follows: - For match `m`, netCDF group `GROUP` ('SatelliteData' or 'InsituData'), and netCDF group variable `VARIABLE`: - -`matches[m][GROUP]['matchID']`: netCDF `MatchedRecords` dimension ID for the match -`matches[m][GROUP]['GROUPID']`: GROUP netCDF `dim` dimension ID for the record -`matches[m][GROUP][VARIABLE]`: variable value - -For example, to access the timestamps of the satellite data and the in situ data of the first match in the list, along with the `MatchedRecords` dimension ID and the groups' `dim` dimension ID: -```python -matches[0]['SatelliteData']['time'] -matches[0]['InsituData']['time'] -matches[0]['SatelliteData']['matchID'] -matches[0]['SatelliteData']['SatelliteDataID'] -matches[0]['InsituData']['InsituDataID'] -``` - - -### Function: `matches_to_csv(matches, csvfile)` -Write the DOMS matches to a CSV file. Include a header of column names which are based on the group and variable names from the netCDF file. - -#### Parameters: -- `matches` (list): the list of dictionaries containing the DOMS matches as returned from the `assemble_matches` function. -- `csvfile` (str): the name of the CSV output file. - -## Usage -For example, to read some DOMS netCDF file called `doms_file.nc`: -### Command line -The main function for `doms_reader.py` takes one `filename` parameter (`doms_file.nc` argument in this example) for the DOMS netCDF file to read, calls the `assemble_matches` function, then calls the `matches_to_csv` function to write the matches to a CSV file `doms_matches.csv`. -``` -python doms_reader.py doms_file.nc -``` -``` -python3 doms_reader.py doms_file.nc -``` -### Importing `assemble_matches` -```python -from doms_reader import assemble_matches -matches = assemble_matches('doms_file.nc') -``` diff --git a/tools/doms/doms_reader.py b/tools/doms/doms_reader.py deleted file mode 100644 index c8229c4..0000000 --- a/tools/doms/doms_reader.py +++ /dev/null @@ -1,144 +0,0 @@ -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -import argparse -from netCDF4 import Dataset, num2date -import sys -import datetime -import csv -from collections import OrderedDict -import logging - -LOGGER = logging.getLogger("doms_reader") - -def assemble_matches(filename): - """ - Read a DOMS netCDF file and return a list of matches. - - Parameters - ---------- - filename : str - The DOMS netCDF file name. - - Returns - ------- - matches : list - List of matches. Each list element is a dictionary. - For match m, netCDF group GROUP (SatelliteData or InsituData), and - group variable VARIABLE: - matches[m][GROUP]['matchID']: MatchedRecords dimension ID for the match - matches[m][GROUP]['GROUPID']: GROUP dim dimension ID for the record - matches[m][GROUP][VARIABLE]: variable value - """ - - try: - # Open the netCDF file - with Dataset(filename, 'r') as doms_nc: - # Check that the number of groups is consistent w/ the MatchedGroups - # dimension - assert len(doms_nc.groups) == doms_nc.dimensions['MatchedGroups'].size,\ - ("Number of groups isn't the same as MatchedGroups dimension.") - - matches = [] - matched_records = doms_nc.dimensions['MatchedRecords'].size - - # Loop through the match IDs to assemble matches - for match in range(0, matched_records): - match_dict = OrderedDict() - # Grab the data from each platform (group) in the match - for group_num, group in enumerate(doms_nc.groups): - match_dict[group] = OrderedDict() - match_dict[group]['matchID'] = match - ID = doms_nc.variables['matchIDs'][match][group_num] - match_dict[group][group + 'ID'] = ID - for var in doms_nc.groups[group].variables.keys(): - match_dict[group][var] = doms_nc.groups[group][var][ID] - - # Create a UTC datetime field from timestamp - dt = num2date(match_dict[group]['time'], - doms_nc.groups[group]['time'].units) - match_dict[group]['datetime'] = dt - LOGGER.info(match_dict) - matches.append(match_dict) - - return matches - except (OSError, IOError) as err: - LOGGER.exception("Error reading netCDF file " + filename) - raise err - -def matches_to_csv(matches, csvfile): - """ - Write the DOMS matches to a CSV file. Include a header of column names - which are based on the group and variable names from the netCDF file. - - Parameters - ---------- - matches : list - The list of dictionaries containing the DOMS matches as returned from - assemble_matches. - csvfile : str - The name of the CSV output file. - """ - # Create a header for the CSV. Column names are GROUP_VARIABLE or - # GROUP_GROUPID. - header = [] - for key, value in matches[0].items(): - for otherkey in value.keys(): - header.append(key + "_" + otherkey) - - try: - # Write the CSV file - with open(csvfile, 'w') as output_file: - csv_writer = csv.writer(output_file) - csv_writer.writerow(header) - for match in matches: - row = [] - for group, data in match.items(): - for value in data.values(): - row.append(value) - csv_writer.writerow(row) - except (OSError, IOError) as err: - LOGGER.exception("Error writing CSV file " + csvfile) - raise err - -if __name__ == '__main__': - """ - Execution: - python doms_reader.py filename - OR - python3 doms_reader.py filename - """ - logging.basicConfig(format='%(asctime)s %(levelname)-8s %(message)s', - level=logging.INFO, - datefmt='%Y-%m-%d %H:%M:%S') - - p = argparse.ArgumentParser() - p.add_argument('filename', help='DOMS netCDF file to read') - args = p.parse_args() - - doms_matches = assemble_matches(args.filename) - - matches_to_csv(doms_matches, 'doms_matches.csv') - - - - - - - - - - - \ No newline at end of file
