diff --git a/deployment/kubernetes/generic/admin.yaml b/deployment/kubernetes/generic/k8s-1-9-and-above/admin.yaml similarity index 100% rename from deployment/kubernetes/generic/admin.yaml rename to deployment/kubernetes/generic/k8s-1-9-and-above/admin.yaml diff --git a/deployment/kubernetes/generic/k8s-1-9-and-above/bookie.yaml b/deployment/kubernetes/generic/k8s-1-9-and-above/bookie.yaml new file mode 100644 index 0000000000000000000000000000000000000000..01148eba80b75ae4c8b76d01b48bcf79d2ebd32f --- /dev/null +++ b/deployment/kubernetes/generic/k8s-1-9-and-above/bookie.yaml @@ -0,0 +1,129 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + + +apiVersion: v1 +kind: ConfigMap +metadata: + name: bookie-config +data: + PULSAR_MEM: "\" -Xms64m -Xmx256m -XX:MaxDirectMemorySize=256m\"" + dbStorage_writeCacheMaxSizeMb: "32" # Write cache size (direct memory) + dbStorage_readAheadCacheMaxSizeMb: "32" # Read cache size (direct memory) + zkServers: zookeeper + statsProviderClass: org.apache.bookkeeper.stats.prometheus.PrometheusMetricsProvider +--- +## +## Define the Bookie headless service +## In practice, in this case, it is only useful to have a view of +## all the bookie pods that are present +## +apiVersion: v1 +kind: Service +metadata: + name: bookkeeper + labels: + app: pulsar + component: bookkeeper +spec: + ports: + - port: 3181 + name: server + clusterIP: None + selector: + app: pulsar + component: bookkeeper +--- +## BookKeeper servers need to access the local disks and the pods +## cannot be moved across different nodes. +## For this reason, we run BK as a daemon set, one for each node in the +## cluster, unless restricted by label selectors +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: bookie + labels: + app: pulsar + component: bookkeeper +spec: + template: + metadata: + labels: + app: pulsar + component: bookkeeper + # Specify cluster to allow aggregation by cluster in + # the metrics + cluster: local + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "8000" + + spec: + containers: + - name: bookie + image: apachepulsar/pulsar-all:latest + command: ["sh", "-c"] + args: + - > + bin/apply-config-from-env.py conf/bookkeeper.conf && + bin/apply-config-from-env.py conf/pulsar_env.sh && + bin/pulsar bookie + ports: + - containerPort: 3181 + hostPort: 3181 + name: client + envFrom: + - configMapRef: + name: bookie-config + env: + - name: advertisedAddress + valueFrom: + fieldRef: + fieldPath: status.hostIP + + volumeMounts: + - name: journal-disk + mountPath: /pulsar/data/bookkeeper/journal + - name: ledgers-disk + mountPath: /pulsar/data/bookkeeper/ledgers + + # bin/bookkeeper shell bookiesanity + + initContainers: + # The first time, initialize BK metadata in zookeeper + # Otherwise ignore error if it's already there + - name: bookie-metaformat + image: apachepulsar/pulsar-all:latest + command: ["sh", "-c"] + args: + - > + bin/apply-config-from-env.py conf/bookkeeper.conf && + bin/bookkeeper shell metaformat --nonInteractive || true; + envFrom: + - configMapRef: + name: bookie-config + + volumes: + # Mount local disks + - name: journal-disk + hostPath: + path: /mnt/disks/ssd0 + - name: ledgers-disk + hostPath: + path: /mnt/disks/ssd1 diff --git a/deployment/kubernetes/generic/k8s-1-9-and-above/broker.yaml b/deployment/kubernetes/generic/k8s-1-9-and-above/broker.yaml new file mode 100644 index 0000000000000000000000000000000000000000..3ef62f5917491b8047250a3ecbe542262460d6a7 --- /dev/null +++ b/deployment/kubernetes/generic/k8s-1-9-and-above/broker.yaml @@ -0,0 +1,103 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + + +apiVersion: v1 +kind: ConfigMap +metadata: + name: broker-config +data: + # Tune for available memory. Increase the heap up to 24G to have + # better GC behavior at high throughput + PULSAR_MEM: "\" -Xms64m -Xmx128m -XX:MaxDirectMemorySize=128m\"" + zookeeperServers: zookeeper + configurationStoreServers: zookeeper + clusterName: local + # change the managed ledger settings if you have more bookies + managedLedgerDefaultEnsembleSize: "1" + managedLedgerDefaultWriteQuorum: "1" + managedLedgerDefaultAckQuorum: "1" + # enable pulsar functions + functionsWorkerEnabled: "true" + PF_pulsarFunctionsCluster: local +--- +## +## Define the Broker headless service +## In practice, in this case, it is only useful to have a view of +## all the broker pods that are present +## +apiVersion: v1 +kind: Service +metadata: + name: broker + labels: + app: pulsar + component: broker +spec: + ports: + - port: 8080 + name: http + - port: 6650 + name: pulsar + clusterIP: None + selector: + app: pulsar + component: broker +--- +## +## Broker deployment definition +## +apiVersion: apps/v1 +kind: Deployment +metadata: + name: broker +spec: + replicas: 3 + template: + metadata: + labels: + app: pulsar + component: broker + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "8080" + spec: + containers: + - name: broker + image: apachepulsar/pulsar-all:latest + command: ["sh", "-c"] + args: + - > + bin/apply-config-from-env.py conf/broker.conf && + bin/apply-config-from-env.py conf/pulsar_env.sh && + bin/gen-yml-from-env.py conf/functions_worker.yml && + bin/pulsar broker + ports: + - containerPort: 8080 + # hostPort: 8080 + - containerPort: 6650 + # hostPort: 6650 + envFrom: + - configMapRef: + name: broker-config + env: + - name: advertisedAddress + valueFrom: + fieldRef: + fieldPath: status.podIP diff --git a/deployment/kubernetes/generic/cluster-metadata.yaml b/deployment/kubernetes/generic/k8s-1-9-and-above/cluster-metadata.yaml similarity index 100% rename from deployment/kubernetes/generic/cluster-metadata.yaml rename to deployment/kubernetes/generic/k8s-1-9-and-above/cluster-metadata.yaml diff --git a/deployment/kubernetes/generic/k8s-1-9-and-above/monitoring.yaml b/deployment/kubernetes/generic/k8s-1-9-and-above/monitoring.yaml new file mode 100644 index 0000000000000000000000000000000000000000..2fdfad7afc7034f006a486d687658eebc932156e --- /dev/null +++ b/deployment/kubernetes/generic/k8s-1-9-and-above/monitoring.yaml @@ -0,0 +1,192 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +apiVersion: v1 +kind: ConfigMap +metadata: + name: prometheus-config +data: + # Include prometheus configuration file, setup to monitor all the + # Kubernetes pods with the "scrape=true" annotation. + prometheus.yml: | + global: + scrape_interval: 15s + scrape_configs: + - job_name: 'prometheus' + static_configs: + - targets: ['localhost:9090'] + - job_name: 'kubernetes-pods' + kubernetes_sd_configs: + - role: pod + + relabel_configs: + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_scrape] + action: keep + regex: true + - source_labels: [__meta_kubernetes_pod_annotation_prometheus_io_path] + action: replace + target_label: __metrics_path__ + regex: (.+) + - source_labels: [__address__, __meta_kubernetes_pod_annotation_prometheus_io_port] + action: replace + regex: ([^:]+)(?::\d+)?;(\d+) + replacement: $1:$2 + target_label: __address__ + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - source_labels: [__meta_kubernetes_namespace] + action: replace + target_label: kubernetes_namespace + - source_labels: [__meta_kubernetes_pod_label_component] + action: replace + target_label: job + - source_labels: [__meta_kubernetes_pod_name] + action: replace + target_label: kubernetes_pod_name +--- +## PROMOTHEUS - Service +apiVersion: v1 +kind: Service +metadata: + name: prometheus + labels: + app: pulsar + component: prometheus +spec: + type: NodePort + ports: + - name: prometheus + nodePort: 30003 + port: 9090 + protocol: TCP + selector: + app: pulsar + component: prometheus +--- +## PROMOTHEUS - Deployment +apiVersion: apps/v1 +kind: Deployment +metadata: + name: prometheus +spec: + replicas: 1 + template: + metadata: + labels: + app: pulsar + component: prometheus + spec: + containers: + - name: prometheus + image: prom/prometheus:v1.6.3 + volumeMounts: + - name: config-volume + mountPath: /etc/prometheus + - name: data-volume + mountPath: /prometheus + ports: + - containerPort: 9090 + volumes: + - name: config-volume + configMap: + name: prometheus-config + - name: data-volume + emptyDir: {} +--- +## GRAFANA - Service +apiVersion: v1 +kind: Service +metadata: + name: grafana + labels: + app: pulsar + component: grafana +spec: + type: NodePort + ports: + - name: grafana + nodePort: 30004 + port: 3000 + protocol: TCP + selector: + app: pulsar + component: grafana +--- +## GRAFANA - Deployment +apiVersion: apps/v1 +kind: Deployment +metadata: + name: grafana +spec: + replicas: 1 + template: + metadata: + labels: + app: pulsar + component: grafana + spec: + containers: + - name: grafana + image: apachepulsar/pulsar-grafana:latest + ports: + - containerPort: 3000 + env: + - name: PROMETHEUS_URL + value: http://prometheus:9090/ +--- +## PULSAR DASHBOARD - Deployment +apiVersion: v1 +kind: Service +metadata: + name: pulsar-dashboard + labels: + app: pulsar + component: dashboard +spec: + type: NodePort + ports: + - name: dashboard + nodePort: 30005 + port: 80 + protocol: TCP + selector: + app: pulsar + component: dashboard +--- +## PULSAR DASHBOARD - Deployment +apiVersion: apps/v1 +kind: Deployment +metadata: + name: pulsar-dashboard +spec: + replicas: 1 + template: + metadata: + labels: + app: pulsar + component: dashboard + spec: + containers: + - name: grafana + image: apachepulsar/pulsar-dashboard:latest + ports: + - containerPort: 80 + env: + - name: SERVICE_URL + value: http://broker:8080/ diff --git a/deployment/kubernetes/generic/k8s-1-9-and-above/proxy.yaml b/deployment/kubernetes/generic/k8s-1-9-and-above/proxy.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b3f33b50bd2e5087f91fef902433e0bf89b972d5 --- /dev/null +++ b/deployment/kubernetes/generic/k8s-1-9-and-above/proxy.yaml @@ -0,0 +1,87 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + + +apiVersion: v1 +kind: ConfigMap +metadata: + name: proxy-config +data: + PULSAR_MEM: "\" -Xms64m -Xmx128m -XX:MaxDirectMemorySize=128m\"" + zookeeperServers: zookeeper + configurationStoreServers: zookeeper + clusterName: local +--- +## +## Expose all nodes on port so that you can reach cluster from outside k8 +## +apiVersion: v1 +kind: Service +metadata: + name: proxy + labels: + app: pulsar + component: proxy +spec: + type: NodePort + ports: + - name: http + nodePort: 30001 + port: 8080 + protocol: TCP + - name: tcp + nodePort: 30002 + port: 6650 + protocol: TCP + selector: + app: pulsar + component: proxy +--- +## +## Proxy deployment definition +## +apiVersion: apps/v1 +kind: Deployment +metadata: + name: proxy +spec: + replicas: 2 + template: + metadata: + labels: + app: pulsar + component: proxy + annotations: + prometheus.io/scrape: "true" + prometheus.io/port: "8080" + spec: + containers: + - name: proxy + image: apachepulsar/pulsar-all:latest + command: ["sh", "-c"] + args: + - > + bin/apply-config-from-env.py conf/proxy.conf && + bin/apply-config-from-env.py conf/pulsar_env.sh && + bin/pulsar proxy + ports: + - containerPort: 8080 + envFrom: + - configMapRef: + name: proxy-config diff --git a/deployment/kubernetes/generic/k8s-1-9-and-above/zookeeper.yaml b/deployment/kubernetes/generic/k8s-1-9-and-above/zookeeper.yaml new file mode 100644 index 0000000000000000000000000000000000000000..a50b16820002c4a2641b9279cd599792fe8ca0dd --- /dev/null +++ b/deployment/kubernetes/generic/k8s-1-9-and-above/zookeeper.yaml @@ -0,0 +1,143 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +apiVersion: v1 +kind: ConfigMap +metadata: + name: zookeeper-config +data: + PULSAR_MEM: "\" -Xms100m -Xmx256m \"" + PULSAR_GC: "\" -XX:+UseG1GC -XX:MaxGCPauseMillis=10\"" +--- +## Define a disruption budget to ensure there are at least +## 2 ZK servers running all the time +apiVersion: policy/v1beta1 +kind: PodDisruptionBudget +metadata: + name: zk-budget +spec: + selector: + matchLabels: + app: zk + minAvailable: 2 +--- +## +## Define the ZooKeeper headless service +## +apiVersion: v1 +kind: Service +metadata: + annotations: + service.alpha.kubernetes.io/tolerate-unready-endpoints: "true" + name: zookeeper + labels: + app: pulsar + component: zookeeper +spec: + ports: + - port: 2888 + name: server + - port: 3888 + name: leader-election + clusterIP: None + selector: + app: pulsar + component: zookeeper +--- +## Define a StatefulSet for ZK servers +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: zk + labels: + app: pulsar + component: zookeeper +spec: + serviceName: zookeeper + replicas: 3 + template: + metadata: + labels: + app: pulsar + component: zookeeper + cluster: us-central + annotations: + pod.alpha.kubernetes.io/initialized: "true" + prometheus.io/scrape: "true" + prometheus.io/port: "8080" + + spec: + # Make sure multiple pods of ZK don't get scheduled on the + # same node, unless there are no other available nodes + affinity: + podAntiAffinity: + preferredDuringSchedulingIgnoredDuringExecution: + - weight: 1 + podAffinityTerm: + labelSelector: + matchExpressions: + - key: "app" + operator: In + values: + - zookeeper + topologyKey: "kubernetes.io/hostname" + containers: + - name: zookeeper + image: apachepulsar/pulsar-all:latest + command: ["sh", "-c"] + args: + - > + bin/apply-config-from-env.py conf/zookeeper.conf && + bin/apply-config-from-env.py conf/pulsar_env.sh && + bin/generate-zookeeper-config.sh conf/zookeeper.conf && + bin/pulsar zookeeper + ports: + - containerPort: 2181 + name: client + - containerPort: 2888 + name: server + - containerPort: 3888 + name: leader-election + env: + - name: ZOOKEEPER_SERVERS + value: zk-0,zk-1,zk-2 + envFrom: + - configMapRef: + name: zookeeper-config + + readinessProbe: + exec: + command: + - "bin/pulsar-zookeeper-ruok.sh" + initialDelaySeconds: 5 + timeoutSeconds: 5 + + livenessProbe: + exec: + command: + - "bin/pulsar-zookeeper-ruok.sh" + initialDelaySeconds: 15 + timeoutSeconds: 5 + + volumeMounts: + - name: datadir + mountPath: /pulsar/data + volumes: + - name: datadir + emptyDir: {} diff --git a/deployment/kubernetes/generic/original/admin.yaml b/deployment/kubernetes/generic/original/admin.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8c84b28c2e02f85a03dcedb34854f011ef257aff --- /dev/null +++ b/deployment/kubernetes/generic/original/admin.yaml @@ -0,0 +1,44 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +apiVersion: v1 +kind: Pod +metadata: + name: pulsar-admin +spec: + containers: + - name: pulsar-admin + image: apachepulsar/pulsar:latest + command: ["sh", "-c"] + args: + - > + bin/apply-config-from-env.py conf/client.conf && + bin/apply-config-from-env.py conf/pulsar_env.sh && + bin/apply-config-from-env.py conf/pulsar_tools_env.sh && + sleep 10000000000 + envFrom: + - configMapRef: + name: broker-config + env: + - name: webServiceUrl + value: "http://proxy:8080/" + - name: brokerServiceUrl + value: "pulsar://proxy:6650/" + - name: PULSAR_MEM + value: "\"-Xms64m -Xmx128m\"" diff --git a/deployment/kubernetes/generic/bookie.yaml b/deployment/kubernetes/generic/original/bookie.yaml similarity index 100% rename from deployment/kubernetes/generic/bookie.yaml rename to deployment/kubernetes/generic/original/bookie.yaml diff --git a/deployment/kubernetes/generic/broker.yaml b/deployment/kubernetes/generic/original/broker.yaml similarity index 100% rename from deployment/kubernetes/generic/broker.yaml rename to deployment/kubernetes/generic/original/broker.yaml diff --git a/deployment/kubernetes/generic/original/cluster-metadata.yaml b/deployment/kubernetes/generic/original/cluster-metadata.yaml new file mode 100644 index 0000000000000000000000000000000000000000..d502a0ebf3a0f339cc2e3083f3bea7b6c373c4c5 --- /dev/null +++ b/deployment/kubernetes/generic/original/cluster-metadata.yaml @@ -0,0 +1,42 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +# + +apiVersion: batch/v1 +kind: Job +metadata: + name: pulsar-cluster-metadata-init + labels: + app: pulsar + component: broker +spec: + template: + spec: + containers: + - name: pulsar-cluster-metadata-init-container + image: apachepulsar/pulsar:latest + command: ["sh", "-c"] + args: + - > + bin/pulsar initialize-cluster-metadata \ + --cluster local \ + --zookeeper zookeeper \ + --configuration-store zookeeper \ + --web-service-url http://broker.default.svc.cluster.local:8080/ \ + --broker-service-url pulsar://broker.default.svc.cluster.local:6650/ || true; + restartPolicy: Never diff --git a/deployment/kubernetes/generic/monitoring.yaml b/deployment/kubernetes/generic/original/monitoring.yaml similarity index 100% rename from deployment/kubernetes/generic/monitoring.yaml rename to deployment/kubernetes/generic/original/monitoring.yaml diff --git a/deployment/kubernetes/generic/proxy.yaml b/deployment/kubernetes/generic/original/proxy.yaml similarity index 100% rename from deployment/kubernetes/generic/proxy.yaml rename to deployment/kubernetes/generic/original/proxy.yaml diff --git a/deployment/kubernetes/generic/zookeeper.yaml b/deployment/kubernetes/generic/original/zookeeper.yaml similarity index 100% rename from deployment/kubernetes/generic/zookeeper.yaml rename to deployment/kubernetes/generic/original/zookeeper.yaml