From 0ae6eccde4a33ec52d0bf71ef3f4e90964addfaf Mon Sep 17 00:00:00 2001 From: "A31882(Wu Yi Chung)" Date: Fri, 24 Nov 2017 16:13:46 +0800 Subject: [PATCH] Replace Nginx-Proxy with Ingress --- docs/kubernetes_deployment.md | 31 +++------ make/kubernetes/k8s-prepare | 3 +- make/kubernetes/nginx/nginx.deploy.yaml | 34 ---------- make/kubernetes/nginx/nginx.svc.yaml | 13 ---- make/kubernetes/templates/ingress.yaml | 22 ++++++ make/kubernetes/templates/nginx.cm.yaml | 89 ------------------------- 6 files changed, 32 insertions(+), 160 deletions(-) delete mode 100644 make/kubernetes/nginx/nginx.deploy.yaml delete mode 100644 make/kubernetes/nginx/nginx.svc.yaml create mode 100644 make/kubernetes/templates/ingress.yaml delete mode 100644 make/kubernetes/templates/nginx.cm.yaml diff --git a/docs/kubernetes_deployment.md b/docs/kubernetes_deployment.md index c41076c88..f2c5e36d6 100644 --- a/docs/kubernetes_deployment.md +++ b/docs/kubernetes_deployment.md @@ -4,7 +4,7 @@ This Document decribes how to deploy Harbor on Kubernetes. It has been verified ### Prerequisite -* You should have domain knowledge about Kubernetes (Deployment, Service, Persistent Volume, Persistent Volume Claim, Config Map). +* You should have domain knowledge about Kubernetes (Deployment, Service, Persistent Volume, Persistent Volume Claim, Config Map, Ingress). * **Optional**: Load the docker images onto woker nodes. *If you skip this step, worker node will pull images from Docker Hub when starting the pods.* * Download the offline installer of Harbor v1.2.0 from the [release](https://github.com/vmware/harbor/releases) page. * Uncompress the offline installer and get the images tgz file harbor.*.tgz, transfer it to each of the worker nodes. @@ -34,22 +34,7 @@ These Basic Configuration must be set. Otherwise you can't deploy Harbor on Kube #To accept access from outside of Kubernetes cluster, it should be set to a worker node. hostname = 10.192.168.5 ``` -- `make/kubernetes/**/*.svc.yaml`: Specify the service of pods. In particular, the externalIP should be set in `make/kubernetes/nginx/nginx.svc.yaml`: - - ```yaml - ... - metadata: - name: nginx - spec: - ports: - - name: http - port: 80 - selector: - name: nginx-apps - externalIPs: - - 10.192.168.5 - ``` - +- `make/kubernetes/**/*.svc.yaml`: Specify the service of pods. - `make/kubernetes/**/*.deploy.yaml`: Specify configs of containers. - `make/kubernetes/pv/*.pvc.yaml`: Persistent Volume Claim. You can set capacity of storage in these files. example: @@ -91,10 +76,10 @@ These files will be generated: - make/kubernetes/jobservice/jobservice.cm.yaml - make/kubernetes/mysql/mysql.cm.yaml -- make/kubernetes/nginx/nginx.cm.yaml - make/kubernetes/registry/registry.cm.yaml - make/kubernetes/ui/ui.cm.yaml - make/kubernetes/adminserver/adminserver.cm.yaml +- make/kubernetes/ingress.yaml #### Advanced Configuration If Basic Configuration was not covering your requirements, you can read this section for more details. @@ -108,7 +93,7 @@ You can find all configs of Harbor in `make/kubernetes/templates/`. There are sp - `jobservice.cm.yaml`: ENV and web config of jobservice - `mysql.cm.yaml`: Root passowrd of MySQL -- `nginx.cm.yaml`: Https certification and nginx config. If you are fimiliar with nginx, you can modify it. +- `ingress.yaml`: Https certification and ingress config. If you are fimiliar with ingress, you can modify it. - `registry.cm.yaml`: Token service certification and registry config Registry use filesystem to store data of images. You can find it like: @@ -140,7 +125,6 @@ kubectl apply -f make/kubernetes/pv/storage.pvc.yaml # create config map kubectl apply -f make/kubernetes/jobservice/jobservice.cm.yaml kubectl apply -f make/kubernetes/mysql/mysql.cm.yaml -kubectl apply -f make/kubernetes/nginx/nginx.cm.yaml kubectl apply -f make/kubernetes/registry/registry.cm.yaml kubectl apply -f make/kubernetes/ui/ui.cm.yaml kubectl apply -f make/kubernetes/adminserver/adminserver.cm.yaml @@ -148,7 +132,6 @@ kubectl apply -f make/kubernetes/adminserver/adminserver.cm.yaml # create service kubectl apply -f make/kubernetes/jobservice/jobservice.svc.yaml kubectl apply -f make/kubernetes/mysql/mysql.svc.yaml -kubectl apply -f make/kubernetes/nginx/nginx.svc.yaml kubectl apply -f make/kubernetes/registry/registry.svc.yaml kubectl apply -f make/kubernetes/ui/ui.svc.yaml kubectl apply -f make/kubernetes/adminserver/adminserver.svc.yaml @@ -158,13 +141,15 @@ kubectl apply -f make/kubernetes/registry/registry.deploy.yaml kubectl apply -f make/kubernetes/mysql/mysql.deploy.yaml kubectl apply -f make/kubernetes/jobservice/jobservice.deploy.yaml kubectl apply -f make/kubernetes/ui/ui.deploy.yaml -kubectl apply -f make/kubernetes/nginx/nginx.deploy.yaml kubectl apply -f make/kubernetes/adminserver/adminserver.deploy.yaml + +# create k8s ingress +kubectl apply -f make/kubernetes/ingress.yaml ``` After the pods are running, you can access Harbor's UI via the configured endpoint `10.192.168.5` or issue docker commands such as `docker login 10.192.168.5` to interact with the registry. #### Limitation -1. Current deployment is http only, to enable https you need to either add another layer of proxy or modify the nginx.cm.yaml to enable https and include a correct certificate +1. Current deployment is http only, to enable https you need to either add another layer of proxy or modify the ingress.yaml to enable https and include a correct certificate 2. Current deployment does not include Clair and Notary, which are supported in docker-compose deployment. They will be supported in near future, stay tuned. diff --git a/make/kubernetes/k8s-prepare b/make/kubernetes/k8s-prepare index b4fc8a643..a8c15a661 100755 --- a/make/kubernetes/k8s-prepare +++ b/make/kubernetes/k8s-prepare @@ -210,6 +210,7 @@ output_dir = base_dir generate_template(os.path.join(template_dir, 'ui.cm.yaml'), os.path.join(output_dir, 'ui/ui.cm.yaml')) generate_template(os.path.join(template_dir, 'jobservice.cm.yaml'), os.path.join(output_dir, 'jobservice/jobservice.cm.yaml')) generate_template(os.path.join(template_dir, 'mysql.cm.yaml'), os.path.join(output_dir, 'mysql/mysql.cm.yaml')) -generate_template(os.path.join(template_dir, 'nginx.cm.yaml'), os.path.join(output_dir, 'nginx/nginx.cm.yaml')) generate_template(os.path.join(template_dir, 'registry.cm.yaml'), os.path.join(output_dir, 'registry/registry.cm.yaml')) generate_template(os.path.join(template_dir, 'adminserver.cm.yaml'), os.path.join(output_dir, 'adminserver/adminserver.cm.yaml')) +generate_template(os.path.join(template_dir, 'ingress.yaml'), os.path.join(output_dir, 'ingress.yaml')) + diff --git a/make/kubernetes/nginx/nginx.deploy.yaml b/make/kubernetes/nginx/nginx.deploy.yaml deleted file mode 100644 index 7cb0c270a..000000000 --- a/make/kubernetes/nginx/nginx.deploy.yaml +++ /dev/null @@ -1,34 +0,0 @@ -apiVersion: extensions/v1beta1 -kind: Deployment -metadata: - name: nginx - labels: - name: nginx -spec: - replicas: 1 - template: - metadata: - labels: - name: nginx-apps - spec: - containers: - - name: nginx-app - image: vmware/nginx-photon:1.11.13 - imagePullPolicy: IfNotPresent - ports: - - containerPort: 80 - - containerPort: 443 - volumeMounts: - - name: config - mountPath: /etc/nginx - volumes: - - name: config - configMap: - name: harbor-nginx-config - items: - - key: config - path: nginx.conf - - key: pkey - path: https.key - - key: cert - path: https.crt diff --git a/make/kubernetes/nginx/nginx.svc.yaml b/make/kubernetes/nginx/nginx.svc.yaml deleted file mode 100644 index 9749dad96..000000000 --- a/make/kubernetes/nginx/nginx.svc.yaml +++ /dev/null @@ -1,13 +0,0 @@ -apiVersion: v1 -kind: Service -metadata: - name: nginx -spec: - ports: - - name: http - port: 80 - selector: - name: nginx-apps -# Set the external IP to an IP of the cluster node, so that the service can be accessed from outside the kubernetes cluster. -# externalIPs: -# - 10.192.168.5 diff --git a/make/kubernetes/templates/ingress.yaml b/make/kubernetes/templates/ingress.yaml new file mode 100644 index 000000000..5ad84da5c --- /dev/null +++ b/make/kubernetes/templates/ingress.yaml @@ -0,0 +1,22 @@ +apiVersion: extensions/v1beta1 +kind: Ingress +metadata: + name: harbor +spec: + rules: + - host: {{hostname}} + http: + paths: + - path: / + backend: + serviceName: ui + servicePort: 80 + - path: /v2 + backend: + serviceName: registry + servicePort: repo + - path: /service + backend: + serviceName: ui + servicePort: 80 + diff --git a/make/kubernetes/templates/nginx.cm.yaml b/make/kubernetes/templates/nginx.cm.yaml deleted file mode 100644 index f0430d540..000000000 --- a/make/kubernetes/templates/nginx.cm.yaml +++ /dev/null @@ -1,89 +0,0 @@ -apiVersion: v1 -kind: ConfigMap -metadata: - name: harbor-nginx-config -data: - config: | - worker_processes auto; - - events { - worker_connections 1024; - use epoll; - multi_accept on; - } - - http { - tcp_nodelay on; - - # this is necessary for us to be able to disable request buffering in all cases - proxy_http_version 1.1; - - - upstream registry { - server registry:5000; - } - - upstream ui { - server ui:80; - } - server { - listen 80; - server_name {{hostname}}; - - # disable any limits to avoid HTTP 413 for large image uploads - client_max_body_size 0; - - # required to avoid HTTP 411: see Issue #1486 (https://github.com/docker/docker/issues/1486) - chunked_transfer_encoding on; - - # rewrite ^/(.*) https://$server_name:443/$1 permanent; - - location / { - proxy_pass http://ui/; - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - - # When setting up Harbor behind other proxy, such as an Nginx instance, remove the below line if the proxy already has similar settings. - proxy_set_header X-Forwarded-Proto $scheme; - - proxy_buffering off; - proxy_request_buffering off; - } - - location /v1/ { - return 404; - } - - location /v2/ { - proxy_pass http://registry/v2/; - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - - # When setting up Harbor behind other proxy, such as an Nginx instance, remove the below line if the proxy already has similar settings. - proxy_set_header X-Forwarded-Proto $scheme; - - proxy_buffering off; - proxy_request_buffering off; - - } - - location /service/ { - proxy_pass http://ui/service/; - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - - # When setting up Harbor behind other proxy, such as an Nginx instance, remove the below line if the proxy already has similar settings. - proxy_set_header X-Forwarded-Proto $scheme; - - proxy_buffering off; - proxy_request_buffering off; - } - } - } - pkey: | - {{4 https_pkey}} - cert: | - {{4 https_cert}}