diff --git a/doc/SERVING_AUTH_DOCKER.md b/doc/SERVING_AUTH_DOCKER.md index 7089ce4eecf89b7c70b79f28ad12788aa1ee2771..f40719068df75ae0f5377d85a0320f545d393806 100644 --- a/doc/SERVING_AUTH_DOCKER.md +++ b/doc/SERVING_AUTH_DOCKER.md @@ -17,7 +17,7 @@ **注明：** docker-compose与docker不一样，它依赖于docker，一次可以部署多个docker容器，可以类比于本地版的kubenetes，docker-compose的教程请参考[docker-compose安装](https://docs.docker.com/compose/install/) ```shell -docker-compose -f tools/key-auth.yaml up -d +docker-compose -f tools/auth/auth-serving-docker.yaml up -d ``` 可以通过 `docker ps` 来查看启动的容器。 @@ -56,4 +56,144 @@ ee59a3dd4806 registry.baidubce.com/serving_dev/serving-runtime:cpu-py36 - 使用serving_uci的路径映射到网关 - 在header处增加了 `X-INSTANCE-ID`和`apikey` -## TODO:K8S + +## K8S部署 + +同样，我们也提供了K8S集群部署Serving安全网关的方式。 + +### Step 1：启动Serving服务 + +我们仍然以 [Uci房价预测](../python/examples/fit_a_line)服务作为例子，这里省略了镜像制作的过程，详情可以参考 [在Kubernetes集群上部署Paddle Serving](./PADDLE_SERVING_ON_KUBERNETES.md)。 + +在这里我们直接执行 +``` +kubectl apply -f tools/auth/serving-demo-k8s.yaml +``` + +可以看到 + +### Step 2: 安装 KONG (一个集群只需要执行一次就可以) +接下来我们执行KONG Ingress的安装 +``` +kubectl apply -f tools/auth/kong-install.yaml +``` + +输出是 +``` +namespace/kong created +customresourcedefinition.apiextensions.k8s.io/kongclusterplugins.configuration.konghq.com created +customresourcedefinition.apiextensions.k8s.io/kongconsumers.configuration.konghq.com created +customresourcedefinition.apiextensions.k8s.io/kongingresses.configuration.konghq.com created +customresourcedefinition.apiextensions.k8s.io/kongplugins.configuration.konghq.com created +customresourcedefinition.apiextensions.k8s.io/tcpingresses.configuration.konghq.com created +serviceaccount/kong-serviceaccount created +clusterrole.rbac.authorization.k8s.io/kong-ingress-clusterrole created +clusterrolebinding.rbac.authorization.k8s.io/kong-ingress-clusterrole-nisa-binding created +service/kong-proxy created +service/kong-validation-webhook created +deployment.apps/ingress-kong created +``` +我们可以输入 +``` +kubectl get service --all-namespaces +``` +会显示 +``` +NAMESPACE NAME TYPE CLUSTER-IP EXTERNAL-IP PORT(S) AGE +default uci ClusterIP 172.16.87.89 9393/TCP 7d7h +kong kong-proxy NodePort 172.16.23.91 80:8175/TCP,443:8521/TCP 102m +kong kong-validation-webhook ClusterIP 172.16.114.93 443/TCP 102m + +``` + +### Step 3: 创建Ingress资源 + +接下来需要做Serving服务和KONG的链接 + +``` +kubectl apply -f tools/auth/kong-ingress-k8s.yaml +``` + +我们也给出yaml文件内容 +``` +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: demo + annotations: + konghq.com/strip-path: "true" + kubernetes.io/ingress.class: kong +spec: + rules: + - http: + paths: + - path: /foo + backend: + serviceName: {{SERVING_SERVICE_NAME}} + servicePort: {{SERVICE_PORT}} +``` +其中serviceName就是uci，servicePort就是9393，如果是别的服务就需要改这两个字段，最终会映射到`/foo`下。 +在这一步之后，我们就可以 +``` +curl -H "Content-Type:application/json" -X POST -d '{"feed":[{"x": [0.0137, -0.1136, 0.2553, -0.0692, 0.0582, -0.0727, -0.1583, -0.0584, 0.6283, 0.4919, 0.1856, 0.0795, -0.0332]}], "fetch":["price"]}' http://$IP:443/foo/uci/prediction +``` + +### Step 4: 增加安全网关限制 + +之前的接口没有鉴权功能，无法验证用户身份合法性，现在我们添加一个key-auth插件 + +执行 +``` +kubectl apply -f key-auth-k8s.yaml +``` + +其中,yaml文内容为 +``` +apiVersion: configuration.konghq.com/v1 +kind: KongPlugin +metadata: + name: key-auth +plugin: key-auth +``` + +现在，需要创建secret，key值为用户指定，需要在请求时携带Header中apikey字段 +执行 +``` +kubectl create secret generic default-apikey \ + --from-literal=kongCredType=key-auth \ + --from-literal=key=ZGVmYXVsdC1hcGlrZXkK +``` + +在这里，我们的key是随意制定了一串 `ZGVmYXVsdC1hcGlrZXkK`，实际情况也可以 +创建一个用户（consumer）标识访问者身份，并未该用户绑定apikey。 +执行 +``` +kubectl apply -f kong-consumer-k8s.yaml +``` + +其中,yaml文内容为 +``` +apiVersion: configuration.konghq.com/v1 +kind: KongConsumer +metadata: + name: default + annotations: + kubernetes.io/ingress.class: kong +username: default +credentials: +- default-apikey +``` + +如果我们这时还想再像上一步一样的做curl访问，会发现已经无法访问，此时已经具备了安全能力，我们需要对应的key。 + + +### Step 5: 通过API Key访问服务 + +执行 +``` +curl -H "Content-Type:application/json" -H "apikey:ZGVmYXVsdC1hcGlrZXkK" -X POST -d '{"feed":[{"x": [0.0137, -0.1136, 0.2553, -0.0692, 0.0582, -0.0727, -0.1583, -0.0584, 0.6283, 0.4919, 0.1856, 0.0795, -0.0332]}], "fetch":["price"]}' https://10.21.8.133:8521/foo/uci/prediction -k +``` +我们可以看到 apikey 已经加入到了curl请求的header当中。 + + + diff --git a/tools/key-auth.yaml b/tools/auth/auth-serving-docker.yaml similarity index 100% rename from tools/key-auth.yaml rename to tools/auth/auth-serving-docker.yaml diff --git a/tools/auth/key-auth-k8s.yaml b/tools/auth/key-auth-k8s.yaml new file mode 100644 index 0000000000000000000000000000000000000000..b5247c043d71c636d072913af1b917866e762d64 --- /dev/null +++ b/tools/auth/key-auth-k8s.yaml @@ -0,0 +1,5 @@ +apiVersion: configuration.konghq.com/v1 +kind: KongPlugin +metadata: + name: key-auth +plugin: key-auth diff --git a/tools/auth/kong-consumer-k8s.yaml b/tools/auth/kong-consumer-k8s.yaml new file mode 100644 index 0000000000000000000000000000000000000000..7cfcd3276b17f24c6b0a3934dddcd3f8d4a6b616 --- /dev/null +++ b/tools/auth/kong-consumer-k8s.yaml @@ -0,0 +1,9 @@ +apiVersion: configuration.konghq.com/v1 +kind: KongConsumer +metadata: + name: default + annotations: + kubernetes.io/ingress.class: kong +username: default +credentials: +- default-apikey diff --git a/tools/auth/kong-ingress-k8s.yaml b/tools/auth/kong-ingress-k8s.yaml new file mode 100644 index 0000000000000000000000000000000000000000..acc45194d7c3337c71c6655fd18b5fac2131f37e --- /dev/null +++ b/tools/auth/kong-ingress-k8s.yaml @@ -0,0 +1,15 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: demo + annotations: + konghq.com/strip-path: "true" + kubernetes.io/ingress.class: kong +spec: + rules: + - http: + paths: + - path: /foo + backend: + serviceName: uci + servicePort: 9393 diff --git a/tools/auth/kong-ingress.yaml b/tools/auth/kong-ingress.yaml new file mode 100644 index 0000000000000000000000000000000000000000..8e49f8b13e52003d408684c6c790909c7a65760a --- /dev/null +++ b/tools/auth/kong-ingress.yaml @@ -0,0 +1,731 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: kong +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: kongclusterplugins.configuration.konghq.com +spec: + additionalPrinterColumns: + - JSONPath: .plugin + description: Name of the plugin + name: Plugin-Type + type: string + - JSONPath: .metadata.creationTimestamp + description: Age + name: Age + type: date + - JSONPath: .disabled + description: Indicates if the plugin is disabled + name: Disabled + priority: 1 + type: boolean + - JSONPath: .config + description: Configuration of the plugin + name: Config + priority: 1 + type: string + group: configuration.konghq.com + names: + kind: KongClusterPlugin + plural: kongclusterplugins + shortNames: + - kcp + scope: Cluster + subresources: + status: {} + validation: + openAPIV3Schema: + properties: + config: + type: object + configFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + namespace: + type: string + required: + - name + - namespace + - key + type: object + type: object + disabled: + type: boolean + plugin: + type: string + protocols: + items: + enum: + - http + - https + - grpc + - grpcs + - tcp + - tls + type: string + type: array + run_on: + enum: + - first + - second + - all + type: string + required: + - plugin + version: v1 +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: kongconsumers.configuration.konghq.com +spec: + additionalPrinterColumns: + - JSONPath: .username + description: Username of a Kong Consumer + name: Username + type: string + - JSONPath: .metadata.creationTimestamp + description: Age + name: Age + type: date + group: configuration.konghq.com + names: + kind: KongConsumer + plural: kongconsumers + shortNames: + - kc + scope: Namespaced + subresources: + status: {} + validation: + openAPIV3Schema: + properties: + credentials: + items: + type: string + type: array + custom_id: + type: string + username: + type: string + version: v1 +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: kongingresses.configuration.konghq.com +spec: + group: configuration.konghq.com + names: + kind: KongIngress + plural: kongingresses + shortNames: + - ki + scope: Namespaced + subresources: + status: {} + validation: + openAPIV3Schema: + properties: + proxy: + properties: + connect_timeout: + minimum: 0 + type: integer + path: + pattern: ^/.*$ + type: string + protocol: + enum: + - http + - https + - grpc + - grpcs + - tcp + - tls + type: string + read_timeout: + minimum: 0 + type: integer + retries: + minimum: 0 + type: integer + write_timeout: + minimum: 0 + type: integer + type: object + route: + properties: + headers: + additionalProperties: + items: + type: string + type: array + type: object + https_redirect_status_code: + type: integer + methods: + items: + type: string + type: array + path_handling: + enum: + - v0 + - v1 + type: string + preserve_host: + type: boolean + protocols: + items: + enum: + - http + - https + - grpc + - grpcs + - tcp + - tls + type: string + type: array + regex_priority: + type: integer + request_buffering: + type: boolean + response_buffering: + type: boolean + snis: + items: + type: string + type: array + strip_path: + type: boolean + upstream: + properties: + algorithm: + enum: + - round-robin + - consistent-hashing + - least-connections + type: string + hash_fallback: + type: string + hash_fallback_header: + type: string + hash_on: + type: string + hash_on_cookie: + type: string + hash_on_cookie_path: + type: string + hash_on_header: + type: string + healthchecks: + properties: + active: + properties: + concurrency: + minimum: 1 + type: integer + healthy: + properties: + http_statuses: + items: + type: integer + type: array + interval: + minimum: 0 + type: integer + successes: + minimum: 0 + type: integer + type: object + http_path: + pattern: ^/.*$ + type: string + timeout: + minimum: 0 + type: integer + unhealthy: + properties: + http_failures: + minimum: 0 + type: integer + http_statuses: + items: + type: integer + type: array + interval: + minimum: 0 + type: integer + tcp_failures: + minimum: 0 + type: integer + timeout: + minimum: 0 + type: integer + type: object + type: object + passive: + properties: + healthy: + properties: + http_statuses: + items: + type: integer + type: array + interval: + minimum: 0 + type: integer + successes: + minimum: 0 + type: integer + type: object + unhealthy: + properties: + http_failures: + minimum: 0 + type: integer + http_statuses: + items: + type: integer + type: array + interval: + minimum: 0 + type: integer + tcp_failures: + minimum: 0 + type: integer + timeout: + minimum: 0 + type: integer + type: object + type: object + threshold: + type: integer + type: object + host_header: + type: string + slots: + minimum: 10 + type: integer + type: object + version: v1 +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: kongplugins.configuration.konghq.com +spec: + additionalPrinterColumns: + - JSONPath: .plugin + description: Name of the plugin + name: Plugin-Type + type: string + - JSONPath: .metadata.creationTimestamp + description: Age + name: Age + type: date + - JSONPath: .disabled + description: Indicates if the plugin is disabled + name: Disabled + priority: 1 + type: boolean + - JSONPath: .config + description: Configuration of the plugin + name: Config + priority: 1 + type: string + group: configuration.konghq.com + names: + kind: KongPlugin + plural: kongplugins + shortNames: + - kp + scope: Namespaced + subresources: + status: {} + validation: + openAPIV3Schema: + properties: + config: + type: object + configFrom: + properties: + secretKeyRef: + properties: + key: + type: string + name: + type: string + required: + - name + - key + type: object + type: object + disabled: + type: boolean + plugin: + type: string + protocols: + items: + enum: + - http + - https + - grpc + - grpcs + - tcp + - tls + type: string + type: array + run_on: + enum: + - first + - second + - all + type: string + required: + - plugin + version: v1 +--- +apiVersion: apiextensions.k8s.io/v1beta1 +kind: CustomResourceDefinition +metadata: + name: tcpingresses.configuration.konghq.com +spec: + additionalPrinterColumns: + - JSONPath: .status.loadBalancer.ingress[*].ip + description: Address of the load balancer + name: Address + type: string + - JSONPath: .metadata.creationTimestamp + description: Age + name: Age + type: date + group: configuration.konghq.com + names: + kind: TCPIngress + plural: tcpingresses + scope: Namespaced + subresources: + status: {} + validation: + openAPIV3Schema: + properties: + apiVersion: + type: string + kind: + type: string + metadata: + type: object + spec: + properties: + rules: + items: + properties: + backend: + properties: + serviceName: + type: string + servicePort: + format: int32 + type: integer + type: object + host: + type: string + port: + format: int32 + type: integer + type: object + type: array + tls: + items: + properties: + hosts: + items: + type: string + type: array + secretName: + type: string + type: object + type: array + type: object + status: + type: object + version: v1beta1 +status: + acceptedNames: + kind: "" + plural: "" + conditions: [] + storedVersions: [] +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: kong-serviceaccount + namespace: kong +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRole +metadata: + name: kong-ingress-clusterrole +rules: +- apiGroups: + - "" + resources: + - endpoints + - nodes + - pods + - secrets + verbs: + - list + - watch +- apiGroups: + - "" + resources: + - nodes + verbs: + - get +- apiGroups: + - "" + resources: + - services + verbs: + - get + - list + - watch +- apiGroups: + - networking.k8s.io + - extensions + - networking.internal.knative.dev + resources: + - ingresses + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - events + verbs: + - create + - patch +- apiGroups: + - networking.k8s.io + - extensions + - networking.internal.knative.dev + resources: + - ingresses/status + verbs: + - update +- apiGroups: + - configuration.konghq.com + resources: + - tcpingresses/status + verbs: + - update +- apiGroups: + - configuration.konghq.com + resources: + - kongplugins + - kongclusterplugins + - kongcredentials + - kongconsumers + - kongingresses + - tcpingresses + verbs: + - get + - list + - watch +- apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - get + - update +--- +apiVersion: rbac.authorization.k8s.io/v1beta1 +kind: ClusterRoleBinding +metadata: + name: kong-ingress-clusterrole-nisa-binding +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: kong-ingress-clusterrole +subjects: +- kind: ServiceAccount + name: kong-serviceaccount + namespace: kong +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + service.beta.kubernetes.io/aws-load-balancer-backend-protocol: tcp + service.beta.kubernetes.io/aws-load-balancer-type: nlb + name: kong-proxy + namespace: kong +spec: + ports: + - name: proxy + port: 80 + protocol: TCP + targetPort: 8000 + - name: proxy-ssl + port: 443 + protocol: TCP + targetPort: 8443 + selector: + app: ingress-kong + type: NodePort +--- +apiVersion: v1 +kind: Service +metadata: + name: kong-validation-webhook + namespace: kong +spec: + ports: + - name: webhook + port: 443 + protocol: TCP + targetPort: 8080 + selector: + app: ingress-kong +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: ingress-kong + name: ingress-kong + namespace: kong +spec: + replicas: 1 + selector: + matchLabels: + app: ingress-kong + template: + metadata: + annotations: + kuma.io/gateway: enabled + prometheus.io/port: "8100" + prometheus.io/scrape: "true" + traffic.sidecar.istio.io/includeInboundPorts: "" + labels: + app: ingress-kong + spec: + containers: + - env: + - name: KONG_PROXY_LISTEN + value: 0.0.0.0:8000, 0.0.0.0:8443 ssl http2 + - name: KONG_PORT_MAPS + value: 80:8000, 443:8443 + - name: KONG_ADMIN_LISTEN + value: 127.0.0.1:8444 ssl + - name: KONG_STATUS_LISTEN + value: 0.0.0.0:8100 + - name: KONG_DATABASE + value: "off" + - name: KONG_NGINX_WORKER_PROCESSES + value: "2" + - name: KONG_ADMIN_ACCESS_LOG + value: /dev/stdout + - name: KONG_ADMIN_ERROR_LOG + value: /dev/stderr + - name: KONG_PROXY_ERROR_LOG + value: /dev/stderr + image: registry.baidubce.com/serving_gateway/kong:paddle + lifecycle: + preStop: + exec: + command: + - /bin/sh + - -c + - kong quit + livenessProbe: + failureThreshold: 3 + httpGet: + path: /status + port: 8100 + scheme: HTTP + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: proxy + ports: + - containerPort: 8000 + name: proxy + protocol: TCP + - containerPort: 8443 + name: proxy-ssl + protocol: TCP + - containerPort: 8100 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /status + port: 8100 + scheme: HTTP + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + - env: + - name: CONTROLLER_KONG_ADMIN_URL + value: https://127.0.0.1:8444 + - name: CONTROLLER_KONG_ADMIN_TLS_SKIP_VERIFY + value: "true" + - name: CONTROLLER_PUBLISH_SERVICE + value: kong/kong-proxy + - name: POD_NAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: metadata.namespace + image: kong/kubernetes-ingress-controller:1.2 + imagePullPolicy: IfNotPresent + livenessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + name: ingress-controller + ports: + - containerPort: 8080 + name: webhook + protocol: TCP + readinessProbe: + failureThreshold: 3 + httpGet: + path: /healthz + port: 10254 + scheme: HTTP + initialDelaySeconds: 5 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + serviceAccountName: kong-serviceaccount diff --git a/tools/auth/serving-demo-k8s.yaml b/tools/auth/serving-demo-k8s.yaml new file mode 100644 index 0000000000000000000000000000000000000000..173b601aee65c295260b944bf147c9134a24206b --- /dev/null +++ b/tools/auth/serving-demo-k8s.yaml @@ -0,0 +1,63 @@ +apiVersion: v1 +kind: Service +metadata: + labels: + app: uci + name: uci +spec: + ports: + - port: 9393 + name: http + protocol: TCP + targetPort: 9393 + selector: + app: uci +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + labels: + app: uci + name: uci +spec: + replicas: 1 + selector: + matchLabels: + app: uci + strategy: {} + template: + metadata: + creationTimestamp: null + labels: + app: uci + spec: + containers: + - image: registry.baidubce.com/serving_dev/fit_a_line:security + name: uci + imagePullPolicy: Always + ports: + - containerPort: 9393 + workingDir: /home/fit_a_line/ + name: uci + command: ['/bin/bash', '-c'] + args: ["python3.6 -m paddle_serving_server.serve --model uci_housing_model --thread 10 --port 9393 --name uci"] + env: + - name: SERVING_BIN + value: "/usr/local/serving_bin/serving" + - name: NODE_NAME + valueFrom: + fieldRef: + fieldPath: spec.nodeName + - name: POD_NAME + valueFrom: + fieldRef: + fieldPath: metadata.name + - name: POD_NAMESPACE + valueFrom: + fieldRef: + fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP + resources: {}