Repository: hadoop Updated Branches: refs/heads/HDFS-13891 2176f9abc -> 47efde24f (forced update)
HDDS-872. Add Dockerfile and skaffold config to deploy ozone dev build to k8s. Contributed by Elek, Marton. Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/21c90801 Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/21c90801 Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/21c90801 Branch: refs/heads/HDFS-13891 Commit: 21c9080121dbc12e619e90cb7832a57c4f99c32e Parents: c35de95 Author: Márton Elek <[email protected]> Authored: Tue Dec 11 16:16:00 2018 +0100 Committer: Márton Elek <[email protected]> Committed: Tue Dec 11 16:31:33 2018 +0100 ---------------------------------------------------------------------- hadoop-ozone/dist/Dockerfile | 21 +++++ hadoop-ozone/dist/README.md | 85 ++++++++++++++++++++ .../dist/dev-support/bin/dist-layout-stitching | 7 ++ hadoop-ozone/dist/skaffold.yaml | 25 ++++++ .../src/main/k8s/ozone/config-configmap.yaml | 34 ++++++++ .../main/k8s/ozone/datanode-public-service.yaml | 29 +++++++ .../src/main/k8s/ozone/datanode-service.yaml | 29 +++++++ .../main/k8s/ozone/datanode-statefulset.yaml | 52 ++++++++++++ .../src/main/k8s/ozone/om-public-service.yaml | 29 +++++++ .../dist/src/main/k8s/ozone/om-service.yaml | 29 +++++++ .../dist/src/main/k8s/ozone/om-statefulset.yaml | 72 +++++++++++++++++ .../src/main/k8s/ozone/s3g-public-service.yaml | 29 +++++++ .../dist/src/main/k8s/ozone/s3g-service.yaml | 29 +++++++ .../src/main/k8s/ozone/s3g-statefulset.yaml | 49 +++++++++++ .../src/main/k8s/ozone/scm-public-service.yaml | 29 +++++++ .../dist/src/main/k8s/ozone/scm-service.yaml | 29 +++++++ .../src/main/k8s/ozone/scm-statefulset.yaml | 67 +++++++++++++++ .../dist/src/main/k8s/prometheus/configmap.yaml | 51 ++++++++++++ .../src/main/k8s/prometheus/deployment.yaml | 46 +++++++++++ .../prometheus/prometheus-public-serivce.yaml | 28 +++++++ .../dist/src/main/k8s/prometheus/role.yaml | 31 +++++++ .../src/main/k8s/prometheus/rolebindng.yaml | 27 +++++++ .../main/k8s/prometheus/scm-public-service.yaml | 29 +++++++ .../main/k8s/prometheus/service-account.yaml | 19 +++++ .../dist/src/main/k8s/prometheus/service.yaml | 25 ++++++ 25 files changed, 900 insertions(+) ---------------------------------------------------------------------- http://git-wip-us.apache.org/repos/asf/hadoop/blob/21c90801/hadoop-ozone/dist/Dockerfile ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/Dockerfile b/hadoop-ozone/dist/Dockerfile new file mode 100644 index 0000000..c22c3ca --- /dev/null +++ b/hadoop-ozone/dist/Dockerfile @@ -0,0 +1,21 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +FROM apache/hadoop-runner:latest + +ADD --chown=hadoop target/ozone-0.4.0-SNAPSHOT /opt/hadoop + +WORKDIR /opt/hadoop http://git-wip-us.apache.org/repos/asf/hadoop/blob/21c90801/hadoop-ozone/dist/README.md ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/README.md b/hadoop-ozone/dist/README.md new file mode 100644 index 0000000..88132ec --- /dev/null +++ b/hadoop-ozone/dist/README.md @@ -0,0 +1,85 @@ +<!--- + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. See accompanying LICENSE file. +--> + +# Ozone Distribution + +This folder contains the project to create the binary ozone distribution and provide all the helper script and docker files to start it locally or in the cluster. + +## Testing with local docker based cluster + +After a full dist build you can find multiple docker-compose based cluster definition in the `target/ozone-*/compose` folder. + +Please check the README files there. + +Usually you can start the cluster with: + +``` +cd compose/ozone +docker-compose up -d +``` + +## Testing on Kubernetes + +You can also test the ozone cluster in kubernetes. If you have no active kubernetes cluster you can start a local one with minikube: + +``` +minikube start +``` + +For testing in kubernetes you need to: + +1. Create a docker image with the new build +2. Upload it to a docker registery +3. Deploy the cluster with apply kubernetes resources + +The easiest way to do all these steps is using the [skaffold](https://github.com/GoogleContainerTools/skaffold) tool. After the [installation of skaffold](https://github.com/GoogleContainerTools/skaffold#installation), you can execute + +``` +skaffold run +``` + +in this (`hadoop-ozone/dist`) folder. + +The default kubernetes resources set (`src/main/k8s/`) contains NodePort based service definitions for the Ozone Manager, Storage Container Manager and the S3 gateway. + +With minikube you can access the services with: + +``` +minikube service s3g-public +minikube service om-public +minikube service scm-public +``` + +### Monitoring + +Apache Hadoop Ozone supports Prometheus out-of the box. It contains a prometheus compatible exporter servlet. To start the monitoring you need a prometheus deploy in your kubernetes cluster: + +``` +cd src/main/k8s/prometheus +kubectl apply -f . +``` + +The prometheus ui also could be access via a NodePort service: + +``` +minikube service prometheus-public +``` + +### Notes on the Kubernetes setup + +Please not that the provided kubernetes resources are not suitable production: + +1. There are no security setup +2. The datanode is started in StatefulSet instead of DaemonSet (To make it possible to scale it up on one node minikube cluster) +3. All the UI pages are published with NodePort services \ No newline at end of file http://git-wip-us.apache.org/repos/asf/hadoop/blob/21c90801/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching index e736065..73716c4 100755 --- a/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching +++ b/hadoop-ozone/dist/dev-support/bin/dist-layout-stitching @@ -71,6 +71,13 @@ run mkdir -p ./share/hadoop/hdds run mkdir -p ./share/hadoop/yarn run mkdir -p ./share/hadoop/hdfs run mkdir -p ./share/hadoop/common + +touch ./share/hadoop/mapreduce/.keep +touch ./share/hadoop/yarn/.keep +touch ./share/hadoop/hdfs/.keep +touch ./share/hadoop/common/.keep + + run mkdir -p ./share/ozone/web run mkdir -p ./bin run mkdir -p ./sbin http://git-wip-us.apache.org/repos/asf/hadoop/blob/21c90801/hadoop-ozone/dist/skaffold.yaml ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/skaffold.yaml b/hadoop-ozone/dist/skaffold.yaml new file mode 100644 index 0000000..cb5f375 --- /dev/null +++ b/hadoop-ozone/dist/skaffold.yaml @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: skaffold/v1alpha5 +kind: Config +build: + artifacts: + - image: apache/ozone +deploy: + kubectl: + manifests: + - src/main/k8s/ozone/* http://git-wip-us.apache.org/repos/asf/hadoop/blob/21c90801/hadoop-ozone/dist/src/main/k8s/ozone/config-configmap.yaml ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/src/main/k8s/ozone/config-configmap.yaml b/hadoop-ozone/dist/src/main/k8s/ozone/config-configmap.yaml new file mode 100644 index 0000000..e79a3cc --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/ozone/config-configmap.yaml @@ -0,0 +1,34 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: v1 +kind: ConfigMap +metadata: + name: config + labels: {} + annotations: {} +data: + OZONE-SITE.XML_ozone.scm.datanode.id: "/data/datanode.id" + OZONE-SITE.XML_ozone.metadata.dirs: "/data/metadata" + OZONE-SITE.XML_ozone.scm.block.client.address: "scm-0.scm" + OZONE-SITE.XML_ozone.om.address: "om-0.om" + OZONE-SITE.XML_ozone.scm.client.address: "scm-0.scm" + OZONE-SITE.XML_ozone.scm.names: "scm-0.scm" + OZONE-SITE.XML_ozone.enabled: "true" + OZONE-SITE.XML_hdds.prometheus.endpoint.enabled: "true" + LOG4J.PROPERTIES_log4j.rootLogger: "INFO, stdout" + LOG4J.PROPERTIES_log4j.appender.stdout: "org.apache.log4j.ConsoleAppender" + LOG4J.PROPERTIES_log4j.appender.stdout.layout: "org.apache.log4j.PatternLayout" + LOG4J.PROPERTIES_log4j.appender.stdout.layout.ConversionPattern: "%d{yyyy-MM-dd HH:mm:ss} %-5p %c{1}:%L - %m%n" http://git-wip-us.apache.org/repos/asf/hadoop/blob/21c90801/hadoop-ozone/dist/src/main/k8s/ozone/datanode-public-service.yaml ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/src/main/k8s/ozone/datanode-public-service.yaml b/hadoop-ozone/dist/src/main/k8s/ozone/datanode-public-service.yaml new file mode 100644 index 0000000..22154b7 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/ozone/datanode-public-service.yaml @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: v1 +kind: Service +metadata: + name: datanode-public + labels: {} + annotations: {} +spec: + selector: + app: ozone + component: datanode + ports: + - port: 9870 + name: rpc + type: NodePort http://git-wip-us.apache.org/repos/asf/hadoop/blob/21c90801/hadoop-ozone/dist/src/main/k8s/ozone/datanode-service.yaml ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/src/main/k8s/ozone/datanode-service.yaml b/hadoop-ozone/dist/src/main/k8s/ozone/datanode-service.yaml new file mode 100644 index 0000000..bdc8009 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/ozone/datanode-service.yaml @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: v1 +kind: Service +metadata: + name: datanode + labels: {} + annotations: {} +spec: + clusterIP: None + selector: + app: ozone + component: datanode + ports: + - port: 9870 + name: rpc http://git-wip-us.apache.org/repos/asf/hadoop/blob/21c90801/hadoop-ozone/dist/src/main/k8s/ozone/datanode-statefulset.yaml ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/src/main/k8s/ozone/datanode-statefulset.yaml b/hadoop-ozone/dist/src/main/k8s/ozone/datanode-statefulset.yaml new file mode 100644 index 0000000..f8600a0 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/ozone/datanode-statefulset.yaml @@ -0,0 +1,52 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: datanode + labels: {} + annotations: {} +spec: + selector: + matchLabels: + app: ozone + component: datanode + template: + metadata: + labels: + app: ozone + component: datanode + spec: + containers: + - name: datanode + image: apache/ozone + args: + - ozone + - datanode + ports: + - containerPort: 9870 + name: rpc + volumeMounts: + - name: "data" + mountPath: "/data" + envFrom: + - configMapRef: + name: config + volumes: + - name: "data" + emptyDir: {} + serviceName: datanode + replicas: 3 http://git-wip-us.apache.org/repos/asf/hadoop/blob/21c90801/hadoop-ozone/dist/src/main/k8s/ozone/om-public-service.yaml ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/src/main/k8s/ozone/om-public-service.yaml b/hadoop-ozone/dist/src/main/k8s/ozone/om-public-service.yaml new file mode 100644 index 0000000..a150939 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/ozone/om-public-service.yaml @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: v1 +kind: Service +metadata: + name: om-public + labels: {} + annotations: {} +spec: + ports: + - port: 9874 + name: ui + selector: + app: ozone + component: om + type: NodePort http://git-wip-us.apache.org/repos/asf/hadoop/blob/21c90801/hadoop-ozone/dist/src/main/k8s/ozone/om-service.yaml ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/src/main/k8s/ozone/om-service.yaml b/hadoop-ozone/dist/src/main/k8s/ozone/om-service.yaml new file mode 100644 index 0000000..f363c48 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/ozone/om-service.yaml @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: v1 +kind: Service +metadata: + name: om + labels: {} + annotations: {} +spec: + ports: + - port: 9874 + name: ui + clusterIP: None + selector: + app: ozone + component: om http://git-wip-us.apache.org/repos/asf/hadoop/blob/21c90801/hadoop-ozone/dist/src/main/k8s/ozone/om-statefulset.yaml ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/src/main/k8s/ozone/om-statefulset.yaml b/hadoop-ozone/dist/src/main/k8s/ozone/om-statefulset.yaml new file mode 100644 index 0000000..08d6089 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/ozone/om-statefulset.yaml @@ -0,0 +1,72 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: om + labels: {} + annotations: {} +spec: + selector: + matchLabels: + app: ozone + component: om + serviceName: om + replicas: 1 + template: + metadata: + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9874" + prometheus.io/path: "/prom" + labels: + app: ozone + component: om + spec: + initContainers: + - name: init + image: apache/ozone + args: + - ozone + - om + - --init + env: + - name: "WAITFOR" + value: "scm-0.scm:9876" + volumeMounts: + - name: "data" + mountPath: "/data" + envFrom: + - configMapRef: + name: config + containers: + - name: om + image: apache/ozone + args: + - ozone + - om + env: + - name: "WAITFOR" + value: "scm-0.scm:9876" + volumeMounts: + - name: "data" + mountPath: "/data" + envFrom: + - configMapRef: + name: config + volumes: + - name: "data" + emptyDir: {} http://git-wip-us.apache.org/repos/asf/hadoop/blob/21c90801/hadoop-ozone/dist/src/main/k8s/ozone/s3g-public-service.yaml ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/src/main/k8s/ozone/s3g-public-service.yaml b/hadoop-ozone/dist/src/main/k8s/ozone/s3g-public-service.yaml new file mode 100644 index 0000000..54d10de --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/ozone/s3g-public-service.yaml @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: v1 +kind: Service +metadata: + name: s3g-public + labels: {} + annotations: {} +spec: + ports: + - port: 9878 + name: rest + selector: + app: ozone + component: s3g + type: NodePort http://git-wip-us.apache.org/repos/asf/hadoop/blob/21c90801/hadoop-ozone/dist/src/main/k8s/ozone/s3g-service.yaml ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/src/main/k8s/ozone/s3g-service.yaml b/hadoop-ozone/dist/src/main/k8s/ozone/s3g-service.yaml new file mode 100644 index 0000000..8441e12 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/ozone/s3g-service.yaml @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: v1 +kind: Service +metadata: + name: s3g + labels: {} + annotations: {} +spec: + ports: + - port: 9878 + name: rest + clusterIP: None + selector: + app: ozone + component: s3g http://git-wip-us.apache.org/repos/asf/hadoop/blob/21c90801/hadoop-ozone/dist/src/main/k8s/ozone/s3g-statefulset.yaml ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/src/main/k8s/ozone/s3g-statefulset.yaml b/hadoop-ozone/dist/src/main/k8s/ozone/s3g-statefulset.yaml new file mode 100644 index 0000000..dd6fc25 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/ozone/s3g-statefulset.yaml @@ -0,0 +1,49 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: s3g + labels: {} + annotations: {} +spec: + selector: + matchLabels: + app: ozone + component: s3g + serviceName: s3g + replicas: 1 + template: + metadata: + labels: + app: ozone + component: s3g + spec: + containers: + - name: s3g + image: apache/ozone + args: + - ozone + - s3g + volumeMounts: + - name: "data" + mountPath: "/data" + envFrom: + - configMapRef: + name: config + volumes: + - name: "data" + emptyDir: {} http://git-wip-us.apache.org/repos/asf/hadoop/blob/21c90801/hadoop-ozone/dist/src/main/k8s/ozone/scm-public-service.yaml ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/src/main/k8s/ozone/scm-public-service.yaml b/hadoop-ozone/dist/src/main/k8s/ozone/scm-public-service.yaml new file mode 100644 index 0000000..ddfe138 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/ozone/scm-public-service.yaml @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: v1 +kind: Service +metadata: + name: scm-public + labels: {} + annotations: {} +spec: + ports: + - port: 9876 + name: ui + selector: + app: ozone + component: scm + type: NodePort http://git-wip-us.apache.org/repos/asf/hadoop/blob/21c90801/hadoop-ozone/dist/src/main/k8s/ozone/scm-service.yaml ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/src/main/k8s/ozone/scm-service.yaml b/hadoop-ozone/dist/src/main/k8s/ozone/scm-service.yaml new file mode 100644 index 0000000..f311695 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/ozone/scm-service.yaml @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: v1 +kind: Service +metadata: + name: scm + labels: {} + annotations: {} +spec: + ports: + - port: 9876 + name: ui + clusterIP: None + selector: + app: ozone + component: scm http://git-wip-us.apache.org/repos/asf/hadoop/blob/21c90801/hadoop-ozone/dist/src/main/k8s/ozone/scm-statefulset.yaml ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/src/main/k8s/ozone/scm-statefulset.yaml b/hadoop-ozone/dist/src/main/k8s/ozone/scm-statefulset.yaml new file mode 100644 index 0000000..80c5924 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/ozone/scm-statefulset.yaml @@ -0,0 +1,67 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: scm + labels: {} + annotations: {} +spec: + selector: + matchLabels: + app: ozone + component: scm + serviceName: scm + replicas: 1 + template: + metadata: + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "9876" + prometheus.io/path: "/prom" + labels: + app: ozone + component: scm + spec: + initContainers: + - name: init + image: apache/ozone + args: + - ozone + - scm + - --init + env: [] + volumeMounts: + - name: "data" + mountPath: "/data" + envFrom: + - configMapRef: + name: config + containers: + - name: scm + image: apache/ozone + args: + - ozone + - scm + volumeMounts: + - name: "data" + mountPath: "/data" + envFrom: + - configMapRef: + name: config + volumes: + - name: "data" + emptyDir: {} http://git-wip-us.apache.org/repos/asf/hadoop/blob/21c90801/hadoop-ozone/dist/src/main/k8s/prometheus/configmap.yaml ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/src/main/k8s/prometheus/configmap.yaml b/hadoop-ozone/dist/src/main/k8s/prometheus/configmap.yaml new file mode 100644 index 0000000..40b13e2 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/prometheus/configmap.yaml @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheusconf +data: + prometheus.yaml: |- + global: + scrape_interval: 15s + scrape_configs: + - job_name: jmxexporter + kubernetes_sd_configs: + - role: pod + namespaces: + names: ["default"] + bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token + relabel_configs: + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: kubernetes_pod_name http://git-wip-us.apache.org/repos/asf/hadoop/blob/21c90801/hadoop-ozone/dist/src/main/k8s/prometheus/deployment.yaml ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/src/main/k8s/prometheus/deployment.yaml b/hadoop-ozone/dist/src/main/k8s/prometheus/deployment.yaml new file mode 100644 index 0000000..6368403 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/prometheus/deployment.yaml @@ -0,0 +1,46 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: apps/v1beta1 +kind: Deployment +metadata: + name: prometheus + labels: + app: prometheus +spec: + replicas: 1 + selector: + matchLabels: + app: prometheus + template: + metadata: + labels: + app: prometheus + spec: + serviceAccountName: prometheus-operator + containers: + - name: prometheus + image: prom/prometheus + args: ["--config.file=/conf/prometheus.yaml"] + ports: + - containerPort: 9090 + volumeMounts: + - name: config + mountPath: "/conf" + readOnly: true + volumes: + - name: config + configMap: + name: prometheusconf http://git-wip-us.apache.org/repos/asf/hadoop/blob/21c90801/hadoop-ozone/dist/src/main/k8s/prometheus/prometheus-public-serivce.yaml ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/src/main/k8s/prometheus/prometheus-public-serivce.yaml b/hadoop-ozone/dist/src/main/k8s/prometheus/prometheus-public-serivce.yaml new file mode 100644 index 0000000..276943d --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/prometheus/prometheus-public-serivce.yaml @@ -0,0 +1,28 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: v1 +kind: Service +metadata: + name: prometheus-public + labels: {} + annotations: {} +spec: + ports: + - port: 9090 + name: ui + selector: + app: prometheus + type: NodePort http://git-wip-us.apache.org/repos/asf/hadoop/blob/21c90801/hadoop-ozone/dist/src/main/k8s/prometheus/role.yaml ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/src/main/k8s/prometheus/role.yaml b/hadoop-ozone/dist/src/main/k8s/prometheus/role.yaml new file mode 100644 index 0000000..e9412fa --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/prometheus/role.yaml @@ -0,0 +1,31 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: Role +metadata: + name: prometheus +rules: +- apiGroups: [""] + resources: + - nodes + - services + - endpoints + - pods + verbs: ["get", "list", "watch"] +- apiGroups: [""] + resources: + - configmaps + verbs: ["get"] http://git-wip-us.apache.org/repos/asf/hadoop/blob/21c90801/hadoop-ozone/dist/src/main/k8s/prometheus/rolebindng.yaml ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/src/main/k8s/prometheus/rolebindng.yaml b/hadoop-ozone/dist/src/main/k8s/prometheus/rolebindng.yaml new file mode 100644 index 0000000..ffd340d --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/prometheus/rolebindng.yaml @@ -0,0 +1,27 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: RoleBinding +metadata: + name: prometheus-operator +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: Role + name: prometheus +subjects: +- kind: ServiceAccount + name: prometheus-operator +# namespace: default http://git-wip-us.apache.org/repos/asf/hadoop/blob/21c90801/hadoop-ozone/dist/src/main/k8s/prometheus/scm-public-service.yaml ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/src/main/k8s/prometheus/scm-public-service.yaml b/hadoop-ozone/dist/src/main/k8s/prometheus/scm-public-service.yaml new file mode 100644 index 0000000..ddfe138 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/prometheus/scm-public-service.yaml @@ -0,0 +1,29 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: v1 +kind: Service +metadata: + name: scm-public + labels: {} + annotations: {} +spec: + ports: + - port: 9876 + name: ui + selector: + app: ozone + component: scm + type: NodePort http://git-wip-us.apache.org/repos/asf/hadoop/blob/21c90801/hadoop-ozone/dist/src/main/k8s/prometheus/service-account.yaml ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/src/main/k8s/prometheus/service-account.yaml b/hadoop-ozone/dist/src/main/k8s/prometheus/service-account.yaml new file mode 100644 index 0000000..d5ba196 --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/prometheus/service-account.yaml @@ -0,0 +1,19 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +apiVersion: v1 +kind: ServiceAccount +metadata: + name: prometheus-operator http://git-wip-us.apache.org/repos/asf/hadoop/blob/21c90801/hadoop-ozone/dist/src/main/k8s/prometheus/service.yaml ---------------------------------------------------------------------- diff --git a/hadoop-ozone/dist/src/main/k8s/prometheus/service.yaml b/hadoop-ozone/dist/src/main/k8s/prometheus/service.yaml new file mode 100644 index 0000000..e07aafc --- /dev/null +++ b/hadoop-ozone/dist/src/main/k8s/prometheus/service.yaml @@ -0,0 +1,25 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +kind: Service +apiVersion: v1 +metadata: + name: prometheus +spec: + selector: + app: prometheus + ports: + - protocol: TCP + port: 9090 --------------------------------------------------------------------- To unsubscribe, e-mail: [email protected] For additional commands, e-mail: [email protected]
