diff --git a/.gitignore b/.gitignore index 2cbbfa203..7e451d1d5 100644 --- a/.gitignore +++ b/.gitignore @@ -1,11 +1,5 @@ harbor -make/common/config/registry/config.yml -make/common/config/ui/env -make/common/config/ui/app.conf -make/common/config/db/env -make/common/config/jobservice/env -make/common/config/nginx/nginx.conf -make/common/config/nginx/cert/* +make/common/config/* make/dev/ui/harbor_ui make/dev/jobservice/harbor_jobservice src/ui/ui diff --git a/AUTHORS b/AUTHORS index 521de08b9..82697321b 100644 --- a/AUTHORS +++ b/AUTHORS @@ -8,6 +8,7 @@ Andre Cruz Benniu Ji Bin Liu Bobby Zhang +Brian Christner Chaofeng Wu Daniel Jiang Deshi Xiao diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 000000000..639812eca --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,54 @@ +# Contributing to Harbor + +This guide provides information on filing issues and guidelines for open source contributors. + +## Reporting issues + +It is a great way to contribute to Harbor by reporting an issue. Well-written and complete bug reports are always welcome! Please open an issue on Github and follow the template to fill in required information. + +Before opening any issue, please look up the existing [issues](https://github.com/vmware/harbor/issues) to avoid submitting a duplication. +If you find a match, you can "subscribe" it to get notified on updates. If you have additional helpful information about the issue, please leave a comment. + +When reporting issues, always include: + +* Version of docker engine and docker-compose +* Configuration files of Harbor +* Log files in /var/log/harbor/ + +Because the issues are open to the public, when submitting the log and configuration files, be sure to remove any sensitive information, e.g. user name, password, IP address, and company name. You can +replace those parts with "REDACTED" or other strings like "****". + +Be sure to include the steps to reproduce the problem if applicable. It can help us understand and fix your issue faster. + + +## Contribution guidelines + +### Pull requests + +Pull requests (PR) are always welcome, even they are small fixes like typos or a few lines of code changes. If there will be significant effort, please first document as an issue and get the discussion going before starting to work on it. + +Please submit a PR to contain changes bit by bit. A PR consisting of a lot features and code changes may be hard to review. It is recommended to submit PRs in a incremental fasion. + +### Design new features + +You can propose new designs for existing Harbor features. You can also design +entirely new features. Please do open an issue on Github for discussion first. This is necessary to ensure the overall architecture is consistent and to avoid duplicated work in the roadmap. + +### Conventions + +Fork Harbor's repository and make changes on your own fork in a new branch. The branch should be named XXX-description where XXX is the number of the issue. Please run the full test suite on your branch before creating a PR. + +Always run [golint](https://github.com/golang/lint) on source code before +committing your changes. + +Pull requests should be rebased on top of master without multiple branches mixed into the PR. If your pull requests do not merge cleanly, use `rebase master` in your branch rather than `merge master`. + +Update the documentation if you are creating or changing features. Good documentation is as important as the code itself. + +Before you submitting any pull request, always squash your commits into logical units of change. A logical unit of change is defined as a set of codes and documents that should be treated as a whole. When possible, compact your commits into one. The commands to use are `git rebase -i` and/or `git push -f`. + +Description of a pull request should refer to all the issues that it addresses. Remember to put a reference to issues (such as `Closes #XXX` and `Fixes #XXX`) in commits so that the issues can be closed when the PR is merged. + +### Signing CLA +If it is the first time you are making a PR, be sure to sign the contributor license agreement (CLA) online. A bot will automatically update the PR for the CLA process. + diff --git a/Makefile b/Makefile index 5d1eec620..2bb73fb55 100644 --- a/Makefile +++ b/Makefile @@ -247,7 +247,7 @@ package_online: modify_composefile @cp LICENSE $(HARBORPKG)/LICENSE @cp NOTICE $(HARBORPKG)/NOTICE @$(TARCMD) -zcvf harbor-online-installer-$(VERSIONTAG).tgz \ - --exclude=$(HARBORPKG)/common/db \ + --exclude=$(HARBORPKG)/common/db --exclude=$(HARBORPKG)/common/config\ --exclude=$(HARBORPKG)/common/log --exclude=$(HARBORPKG)/ubuntu \ --exclude=$(HARBORPKG)/photon --exclude=$(HARBORPKG)/kubernetes \ --exclude=$(HARBORPKG)/dev --exclude=$(DOCKERCOMPOSETPLFILENAME) \ @@ -279,7 +279,7 @@ package_offline: compile build modify_composefile nginx:1.11.5 registry:2.5.0 @$(TARCMD) -zcvf harbor-offline-installer-$(VERSIONTAG).tgz \ - --exclude=$(HARBORPKG)/common/db \ + --exclude=$(HARBORPKG)/common/db --exclude=$(HARBORPKG)/common/config\ --exclude=$(HARBORPKG)/common/log --exclude=$(HARBORPKG)/ubuntu \ --exclude=$(HARBORPKG)/photon --exclude=$(HARBORPKG)/kubernetes \ --exclude=$(HARBORPKG)/dev --exclude=$(DOCKERCOMPOSETPLFILENAME) \ diff --git a/README.md b/README.md index 8a3445d79..1875459b1 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ Project Harbor is an enterprise-class registry server that stores and distribute **On vSphere:** vCenter 5.x+ for deployment of Harbor's virtual appliance. -Download binaries of **[Harbor release ](https://github.com/vmware/harbor/releases)** and follow **[Installation & Confiugration Guide](docs/installation_guide.md)** to install Harbor. +Download binaries of **[Harbor release ](https://github.com/vmware/harbor/releases)** and follow **[Installation & Configuration Guide](docs/installation_guide.md)** to install Harbor. Refer to **[User Guide](docs/user_guide.md)** for more details on how to use Harbor. @@ -35,7 +35,7 @@ Refer to **[User Guide](docs/user_guide.md)** for more details on how to use Har **WeChat Group:** Add WeChat id *connect1688* to join WeChat discussion group. ### Contribution -We welcome contributions from the community. If you wish to contribute code and you have not signed our contributor license agreement (CLA), our bot will update the issue when you open a pull request. For any questions about the CLA process, please refer to our [FAQ](https://cla.vmware.com/faq). Contact us for any qustions: harbor@ vmware.com . +We welcome contributions from the community. If you wish to contribute code and you have not signed our contributor license agreement (CLA), our bot will update the issue when you open a pull request. For any questions about the CLA process, please refer to our [FAQ](https://cla.vmware.com/faq). Contact us for any questions: harbor @vmware.com . ### License Harbor is available under the [Apache 2 license](LICENSE). diff --git a/docs/README.md b/docs/README.md index 1b58e0473..653d8448d 100644 --- a/docs/README.md +++ b/docs/README.md @@ -23,7 +23,7 @@ Guide to deploy Harbor on Kubenetes. (maintained by community) ### Developer documents -[Arthicture Overview of Harbor](https://github.com/vmware/harbor/wiki/Architecture-Overview-of-Harbor) +[Arthitecture Overview of Harbor](https://github.com/vmware/harbor/wiki/Architecture-Overview-of-Harbor) Developers read this first. [Harbor API Specs by Swagger](configure_swagger.md) diff --git a/docs/compile_guide.md b/docs/compile_guide.md index a05bdac60..0b1c8f4e9 100644 --- a/docs/compile_guide.md +++ b/docs/compile_guide.md @@ -1,12 +1,11 @@ ## Introduction -This guide shows how to compile binary, build images and install Harbor instance from source code via make commands. +This guide provides instructions for developers to build and run Harbor from source code. -## Step 1: Prepare Your System for Building Harbor +## Step 1: Prepare for a build environment for Harbor -Harbor is deployed as several Docker containers and most of the code compiled by go language. The target host requires Python, Docker, Docker Compose and golang develop environment to be installed. +Harbor is deployed as several Docker containers and most of the code is written in Go language. The build host requires Python, Docker, Docker Compose and golang development environment. Please install the below prerequisites: -Requirement: Software | Required Version ----------------------|-------------------------- @@ -19,14 +18,16 @@ golang* | 1.6.0 + *optional -## Step 2: Getting the Source Code +## Step 2: Getting the source code ```sh $ git clone https://github.com/vmware/harbor ``` -## Step 3: Resolving Dependencies -Compile Harbor source code by local golang environment needs LDAP develop package and you'll have to do it manually. If you want to compile source code by golang image, can skip this section. +## Step 3: Resolving dependencies of Go language +You can compile the source code by using a Golang dev image. In this case, you can skip this step. + +If you are building Harbor using your own Go compiling environment. You need to install LDAP packages manually. For PhotonOS: @@ -40,40 +41,40 @@ For Ubuntu: $ apt-get update && apt-get install -y libldap2-dev ``` -Other platforms please consult the relevant documentation for LDAP package installation. +For other platforms, please consult the relevant documentation of installing LDAP package. -## Step 4: Build and Install +## Step 4: Building and installing Harbor ### Configuration -Edit the file **make/harbor.cfg**, make necessary configuration changes such as hostname, admin password and mail server. Refer to [Installation and Configuration Guide](installation_guide.md#configuring-harbor) for more info. +Edit the file **make/harbor.cfg** and make necessary configuration changes such as hostname, admin password and mail server. Refer to **[Installation and Configuration Guide](installation_guide.md#configuring-harbor)** for more info. ```sh $ cd harbor $ vi make/harbor.cfg ``` -### Compile, Build and Install +### Compiling and Running -Support 3 code compile method: golang image compile, local golang compile and developer manual compile. +You can compile the code by one of the three approaches: -#### I. Compile Code with Golang Image, then Automation Build and Install +#### I. Create a Golang dev image, then build Harbor -* Build Compile Golang Image +* Build Golang dev image: ```sh $ make compile_buildgolangimage -e GOBUILDIMAGE=harborgo:1.6.2 ``` -* Automation Build and Install +* Build, install and bring up Harbor: ```sh $ make install -e GOBUILDIMAGE=harborgo:1.6.2 COMPILETAG=compile_golangimage ``` -#### II. Compile Code with Local Golang, then Automation Build and Install +#### II. Compile code with your own Golang environment, then build Harbor -* Move Code to $GOPATH +* Move source code to $GOPATH ```sh $ mkdir $GOPATH/src/github.com/vmware/ @@ -81,14 +82,14 @@ Support 3 code compile method: golang image compile, local golang compile and de $ mv harbor $GOPATH/src/github.com/vmware/. ``` -* Automation Build and Install +* Build, install and run Harbor ```sh $ cd $GOPATH/src/github.com/vmware/harbor $ make install ``` -#### III. Manual Build and Install (Compatible with Prior Versions) +#### III. Manual build process (compatible with previous versions) ```sh $ cd make @@ -105,50 +106,49 @@ Support 3 code compile method: golang image compile, local golang compile and de $ docker-compose up -d ``` -### Install Success +### Verify your installation -You can get this message from shell after successful complete Harbor installs. +If everyting worked properly, you can get the below message: ```sh ... - ✔ ----Harbor has been installed and started successfully.---- + ----Harbor has been installed and started successfully.---- Now you should be able to visit the admin portal at http://$YOURIP. For more details, please visit https://github.com/vmware/harbor . ``` -Refer to [Installation and Configuration Guide](installation_guide.md#managing-harbors-lifecycle) for more info. +Refer to [Installation and Configuration Guide](installation_guide.md#managing-harbors-lifecycle) for more information about managing your Harbor instance. -## Attachments +## Appendix * Using the Makefile -Makefile is a special format file that together with the make utility will help developer to automagically build and manage Harbor projects. -At the top of the makefile, there are several user-configurable parameters designed to enable the Makefile to be easily portable. +The `Makefile` contains these configurable parameters: Variable | Description -------------------|------------- BASEIMAGE | Container base image, default: photon DEVFLAG | Build model flag, default: dev COMPILETAG | Compile model flag, default: compile_normal (local golang build) -REGISTRYSERVER | Remote registry server address +REGISTRYSERVER | Remote registry server IP address REGISTRYUSER | Remote registry server user name REGISTRYPASSWORD | Remote registry server user password REGISTRYPROJECTNAME| Project name on remote registry server -There are also a variety of rules that help with project management and debugging... +* Predefined targets: -Rule | Description +Target | Description --------------------|------------- -all | prepare env, compile binarys, build images and install images +all | prepare env, compile binaries, build images and install images prepare | prepare env compile | compile ui and jobservice code compile_golangimage | compile local golang image compile_ui | compile ui binary compile_jobservice | compile jobservice binary -build | build Harbor docker images (defuault | build_photon) -build_photon | build Harbor docker images from photon bsaeimage -build_ubuntu | build Harbor docker images from ubuntu baseimage -install | compile binarys, build images, prepare specific version composefile and startup Harbor instance +build | build Harbor docker images (default: using build_photon) +build_photon | build Harbor docker images from Photon OS base image +build_ubuntu | build Harbor docker images from Ubuntu base image +install | compile binaries, build images, prepare specific version of compose file and startup Harbor instance start | startup Harbor instance down | shutdown Harbor instance package_online | prepare online install package @@ -163,28 +163,28 @@ cleanpackage | remove online/offline install package #### EXAMPLE: -#### compile from golang image: +#### Build a golang dev image (for building Harbor): ```sh $ make compile_golangimage -e GOBUILDIMAGE= [$YOURIMAGE] ``` -#### build Harbor docker images form ubuntu +#### Build Harbor images based on Ubuntu ```sh $ make build -e BASEIMAGE=ubuntu ``` -#### push Harbor images to specific registry server +#### Push Harbor images to specific registry server ```sh $ make pushimage -e DEVFLAG=false REGISTRYSERVER=[$SERVERADDRESS] REGISTRYUSER=[$USERNAME] REGISTRYPASSWORD=[$PASSWORD] REGISTRYPROJECTNAME=[$PROJECTNAME] ``` - note**: need add "/" on end of REGISTRYSERVER. If not setting REGISTRYSERVER will push images directly to dockerhub. + **Note**: need add "/" on end of REGISTRYSERVER. If REGISTRYSERVER is not set, images will be pushed directly to Docker Hub. ```sh @@ -192,22 +192,18 @@ cleanpackage | remove online/offline install package ``` -#### clean specific version binarys and images +#### Clean up binaries and images of a specific version ```sh $ make clean -e VERSIONTAG=[TAG] ``` - note**: If commit new code to github, the git commit TAG will change. Better use this command clean previous images and files for specific TAG. + **Note**: If new code had been added to Github, the git commit TAG will change. Better use this command to clean up images and files of previous TAG. -#### By default DEVFLAG=true, if you want to release new version of Harbor, should setting the flag to false. +#### By default, the make process create a development build. To create a release build of Harbor, set the below flag to false. ```sh $ make XXXX -e DEVFLAG=false ``` - -## Links - -## Comments diff --git a/docs/configure_https.md b/docs/configure_https.md index 5b1f57833..9281d8125 100644 --- a/docs/configure_https.md +++ b/docs/configure_https.md @@ -1,6 +1,6 @@ #Configuring Harbor with HTTPS Access -Because Harbor does not ship with any certificates, it uses HTTP by default to serve registry requests. This makes it relatively simple to configure. However, it is highly recommended that security be enabled for any production environment. Harbor has an Nginx instance as a reverse proxy for all services, you can configure Nginx to enable https. +Because Harbor does not ship with any certificates, it uses HTTP by default to serve registry requests. However, it is highly recommended that security be enabled for any production environment. Harbor has an Nginx instance as a reverse proxy for all services, you can use the prepare script to configure Nginx to enable https. ##Getting a certificate @@ -9,6 +9,7 @@ Assuming that your registry's **hostname** is **reg.yourdomain.com**, and that i In a test or development environment, you may choose to use a self-signed certificate instead of the one from a CA. The below commands generate your own certificate: 1) Create your own CA certificate: + ``` openssl req \ -newkey rsa:4096 -nodes -sha256 -keyout ca.key \ @@ -18,6 +19,7 @@ In a test or development environment, you may choose to use a self-signed certif If you use FQDN like **reg.yourdomain.com** to connect your registry host, then you must use **reg.yourdomain.com** as CN (Common Name). Otherwise, if you use IP address to connect your registry host, CN can be anything like your name and so on: + ``` openssl req \ -newkey rsa:4096 -nodes -sha256 -keyout yourdomain.com.key \ @@ -26,85 +28,62 @@ Otherwise, if you use IP address to connect your registry host, CN can be anythi 3) Generate the certificate of your registry host: On Ubuntu, the config file of openssl locates at **/etc/ssl/openssl.cnf**. Refer to openssl document for more information. The default CA directory of openssl is called demoCA. Let's create necessary directories and files: + ``` mkdir demoCA cd demoCA touch index.txt echo '01' > serial cd .. - ``` +``` If you're using FQDN like **reg.yourdomain.com** to connect your registry host, then run this command to generate the certificate of your registry host: + ``` openssl ca -in yourdomain.com.csr -out yourdomain.com.crt -cert ca.crt -keyfile ca.key -outdir . ``` -If you're using **IP** to connect your registry host, you may instead run the command below: +If you're using **IP**, say **192.168.1.101** to connect your registry host, you may instead run the command below: + ``` - - echo subjectAltName = IP:your registry host IP > extfile.cnf + echo subjectAltName = IP:192.168.1.101 > extfile.cnf openssl ca -in yourdomain.com.csr -out yourdomain.com.crt -cert ca.crt -keyfile ca.key -extfile extfile.cnf -outdir . ``` -##Configuration of Nginx -After obtaining the **yourdomain.com.crt** and **yourdomain.com.key** files, change the directory to make/config/nginx in Harbor project. +##Configuration and Installation +After obtaining the **yourdomain.com.crt** and **yourdomain.com.key** files, +you can put them into directory such as ```/root/cert/```: + ``` - cd make/config/nginx -``` -Create a new directory cert/, if it does not exist. Then copy **yourdomain.com.crt** and **yourdomain.com.key** to cert/, e.g. : -``` - cp yourdomain.com.crt cert/ - cp yourdomain.com.key cert/ + cp yourdomain.com.crt /root/cert/ + cp yourdomain.com.key /root/cert/ ``` -Rename the existing configuration file of Nginx: -``` - mv nginx.conf nginx.conf.bak -``` -Copy the template **nginx.https.conf** as the new configuration file: -``` - cp nginx.https.conf nginx.conf -``` -Edit the file nginx.conf and replace two occurrences of **harbordomain.com** to your own host name, such as reg.yourdomain.com . If you use a customized port rather than the default port 443, replace the port "443" in the line "rewrite ^/(.*) https://$server_name:443/$1 permanent;" as well. Please refer to the [installation guide](https://github.com/vmware/harbor/blob/master/docs/installation_guide.md) for other required steps of port customization. -``` - server { - listen 443 ssl; - server_name harbordomain.com; - ... - - server { - listen 80; - server_name harbordomain.com; - rewrite ^/(.*) https://$server_name:443/$1 permanent; -``` -Then look for the SSL section to make sure the files of your certificates match the names in the config file. Do not change the path of the files. -``` - ... - - # SSL - ssl_certificate /etc/nginx/cert/yourdomain.com.crt; - ssl_certificate_key /etc/nginx/cert/yourdomain.com.key; -``` -Save your changes in nginx.conf. +Next, edit the file make/harbor.cfg , update the hostname and the protocol, and update the attributes ```ssl_cert``` and ```ssl_cert_key```: -##Installation of Harbor -Next, edit the file make/harbor.cfg , update the hostname and the protocol: ``` #set hostname hostname = reg.yourdomain.com #set ui_url_protocol ui_url_protocol = https + ...... + #The path of cert and key files for nginx, they are applied only the protocol is set to https + ssl_cert = /root/cert/yourdomain.com.crt + ssl_cert_key = /root/cert/yourdomain.com.key ``` Generate configuration files for Harbor: + ``` -./prepare + ./prepare ``` + If Harbor is already running, stop and remove the existing instance. Your image data remain in the file system + ``` - docker-compose stop - docker-compose rm + docker-compose down ``` Finally, restart Harbor: + ``` docker-compose up -d ``` @@ -126,17 +105,20 @@ If you've mapped nginx 443 port to another, you need to add the port to login, l ##Troubleshooting 1. You may get an intermediate certificate from a certificate issuer. In this case, you should merge the intermediate certificate with your own certificate to create a certificate bundle. You can achieve this by the below command: + ``` cat intermediate-certificate.pem >> yourdomain.com.crt ``` 2. On some systems where docker daemon runs, you may need to trust the certificate at OS level. On Ubuntu, this can be done by below commands: + ```sh cp youdomain.com.crt /usr/local/share/ca-certificates/reg.yourdomain.com.crt update-ca-certificates ``` On Red Hat (CentOS etc), the commands are: + ```sh cp yourdomain.com.crt /etc/pki/ca-trust/source/anchors/reg.yourdomain.com.crt update-ca-trust diff --git a/docs/developer_guide_i18n.md b/docs/developer_guide_i18n.md index a5b56d534..9a88ef8cd 100644 --- a/docs/developer_guide_i18n.md +++ b/docs/developer_guide_i18n.md @@ -45,7 +45,7 @@ 5. Add the new language to the `app.conf` file. - In the file `make/config/ui/app.conf`, append a new item to the configuration section. + In the file `make/common/templates/ui/app.conf`, append a new item to the configuration section. ``` [lang] types = en-US|zh-CN|- diff --git a/docs/installation_guide.md b/docs/installation_guide.md index 696953416..6b5dd078b 100644 --- a/docs/installation_guide.md +++ b/docs/installation_guide.md @@ -3,7 +3,7 @@ Harbor can be installed by one of three approaches: - **Online installer:** The installer downloads Harbor's images from Docker hub. For this reason, the installer is very small in size. -- **Offline installer:** Use this installer when the host does not have Internet connection. The installer contains pre-built images so its size is larger. +- **Offline installer:** Use this installer when the host does not have an Internet connection. The installer contains pre-built images so its size is larger. - **Virtual Appliance:** If you are installing Harbor as the registry component of vSphere Integrated Containers (VIC), or using Harbor as a standalone registry on vSphere platform, download the OVA version of Harbor. @@ -59,25 +59,25 @@ The parameters are described below - note that at the very least, you will need * email_from = admin * email_ssl = false -* **harbor_admin_password**: The adminstrator's initial password. This password only takes effect for the first time Harbor launches. After that, this setting is ignored and the adminstrator's password should be set in the UI. _Note that the default username/password are **admin/Harbor12345** ._ -* **auth_mode**: The type of authentication that is used. By default it is **db_auth**, i.e. the credentials are stored in a database. For LDAP authentication, set this to **ldap_auth**. +* **harbor_admin_password**: The administrator's initial password. This password only takes effect for the first time Harbor launches. After that, this setting is ignored and the administrator's password should be set in the UI. _Note that the default username/password are **admin/Harbor12345** ._ +* **auth_mode**: The type of authentication that is used. By default, it is **db_auth**, i.e. the credentials are stored in a database. For LDAP authentication, set this to **ldap_auth**. * **ldap_url**: The LDAP endpoint URL (e.g. `ldaps://ldap.mydomain.com`). _Only used when **auth_mode** is set to *ldap_auth* ._ * **ldap_searchdn**: The DN of a user who has the permission to search an LDAP/AD server (e.g. `uid=admin,ou=people,dc=mydomain,dc=com`). * **ldap_search_pwd**: The password of the user specified by *ldap_searchdn*. * **ldap_basedn**: The base DN to look up a user, e.g. `ou=people,dc=mydomain,dc=com`. _Only used when **auth_mode** is set to *ldap_auth* ._ * **ldap_filter**:The search filter for looking up a user, e.g. `(objectClass=person)`. -* **ldap_uid**: The attribute used to match a user during a ldap search, it could be uid, cn, email or other attributes. +* **ldap_uid**: The attribute used to match a user during a LDAP search, it could be uid, cn, email or other attributes. * **ldap_scope**: The scope to search for a user, 1-LDAP_SCOPE_BASE, 2-LDAP_SCOPE_ONELEVEL, 3-LDAP_SCOPE_SUBTREE. Default is 3. -* **db_password**: The root password for the mySQL database used for **db_auth**. _Change this password for any production use!_ +* **db_password**: The root password for the MySQL database used for **db_auth**. _Change this password for any production use!_ * **self_registration**: (**on** or **off**. Default is **on**) Enable / Disable the ability for a user to register themselves. When disabled, new users can only be created by the Admin user, only an admin user can create new users in Harbor. _NOTE: When **auth_mode** is set to **ldap_auth**, self-registration feature is **always** disabled, and this flag is ignored._ * **use_compressed_js**: (**on** or **off**. Default is **on**) For production use, turn this flag to **on**. In development mode, set it to **off** so that js files can be modified separately. * **max_job_workers**: (default value is **3**) The maximum number of replication workers in job service. For each image replication job, a worker synchronizes all tags of a repository to the remote destination. Increasing this number allows more concurrent replication jobs in the system. However, since each worker consumes a certain amount of network/CPU/IO resources, please carefully pick the value of this attribute based on the hardware resource of the host. * **secret_key**: The key to encrypt or decrypt the password of a remote registry in a replication policy, its length has to be 16 characters. Change this key before any production use. *NOTE: After changing this key, previously encrypted password of a policy can not be decrypted.* -* **token_expiration**: The expiration time (in minute) of a token created by token service, default is 30 minutes. +* **token_expiration**: The expiration time (in minutes) of a token created by token service, default is 30 minutes. * **verify_remote_cert**: (**on** or **off**. Default is **on**) This flag determines whether or not to verify SSL/TLS certificate when Harbor communicates with a remote registry instance. Setting this attribute to **off** bypasses the SSL/TLS verification, which is often used when the remote instance has a self-signed or untrusted certificate. -* **customize_crt**: (**on** or **off**. Default is **on**) When this attribute is **on**, the prepare script creates private key and root certificate for the generation/verification of the regitry's token. +* **customize_crt**: (**on** or **off**. Default is **on**) When this attribute is **on**, the prepare script creates private key and root certificate for the generation/verification of the registry's token. * The following attributes:**crt_country**, **crt_state**, **crt_location**, **crt_organization**, **crt_organizationalunit**, **crt_commonname**, **crt_email** are used as parameters for generating the keys. Set this attribute to **off** when the key and root certificate are supplied by external sources. Refer to [Customize Key and Certificate of Harbor Token Service](customize_token_service.md) for more info. #### Configuring storage backend (optional) diff --git a/docs/user_guide.md b/docs/user_guide.md index b304399c7..3705df68b 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -1,6 +1,6 @@ #User Guide ##Overview -This guide takes you through the fundamentals of using Harbor. You'll learn how to use Harbor to: +This guide walks you through the fundamentals of using Harbor. You'll learn how to use Harbor to: * Manage your projects. * Manage members of a project. @@ -11,7 +11,7 @@ This guide takes you through the fundamentals of using Harbor. You'll learn how + Manage destinations. + Manage replication policies. * Pull and push images using Docker client. -* Delete repositories. +* Delete repositories and images. ##Role Based Access Control @@ -152,24 +152,30 @@ $ docker push 10.117.169.182/demo/ubuntu:14.04 ##Deleting repositories -Repositories deletion runs in two steps. -First, delete repositories in Harbor's UI. This is soft deletion. You can delete the entire repository or just a tag of it. +Repository deletion runs in two steps. + +First, delete a repository in Harbor's UI. This is soft deletion. You can delete the entire repository or just a tag of it. After the soft deletion, +the repository is no longer managed in Harbor, however, the files of the repository still remains in Harbor's storage. ![browse project](img/new_delete_repository.png) -**Note: If both tag A and tag B reference the same image, after deleting tag A, B will also disappear.** +**CAUTION: If both tag A and tag B refer to the same image, after deleting tag A, B will also get deleted.** -Second, delete the real data using registry's garbage colliection(GC). -Make sure that no one is pushing images or Harbor is not running at all before you do GC. If someone were to push an image while GC is running, there is the risk that the image's layers will be mistakenly deleted, leading to a corrupted image. So before running GC, a preferred approach is to stop Harbor first. +Next, delete the actual files of the repository using the registry's garbage collection(GC). Make sure that no one is pushing images or Harbor is not running at all before you perform a GC. If someone were pushing an image while GC is running, there is a risk that the image's layers will be mistakenly deleted which results in a corrupted image. So before running GC, a preferred approach is to stop Harbor first. -Run the command on the host which harbor is deployed on. +Run the below commands on the host which Harbor is deployed on to preview what files/images will be affect: ```sh $ docker-compose stop -$ docker run -it --name gc --rm --volumes-from deploy_registry_1 registry:2.4.0 garbage-collect [--dry-run] /etc/registry/config.yml +$ docker run -it --name gc --rm --volumes-from deploy_registry_1 registry:2.5.0 garbage-collect --dry-run /etc/registry/config.yml +``` +**NOTE:** The above option "--dry-run" will print the progress without removing any data. + +Verify the result of the above test, then use the below commands to perform garbage collection and restart Harbor. + +```sh +$ docker run -it --name gc --rm --volumes-from deploy_registry_1 registry:2.5.0 garbage-collect /etc/registry/config.yml $ docker-compose start ``` -Option "--dry-run" will print the progress without removing any data. - -About the details of GC, please see [GC](https://github.com/docker/docker.github.io/blob/master/registry/garbage-collection.md). \ No newline at end of file +For more information about GC, please see [GC](https://github.com/docker/docker.github.io/blob/master/registry/garbage-collection.md). \ No newline at end of file diff --git a/make/common/config/nginx/cert/.gitignore b/make/common/config/nginx/cert/.gitignore deleted file mode 100644 index e69de29bb..000000000 diff --git a/make/common/config/nginx/nginx.conf b/make/common/config/nginx/nginx.conf deleted file mode 100644 index 8bc79fa6c..000000000 --- a/make/common/config/nginx/nginx.conf +++ /dev/null @@ -1,75 +0,0 @@ -worker_processes auto; - -events { - worker_connections 1024; - use epoll; - multi_accept on; -} - -http { - tcp_nodelay on; - - # this is necessary for us to be able to disable request buffering in all cases - proxy_http_version 1.1; - - - upstream registry { - server registry:5000; - } - - upstream ui { - server ui:80; - } - - - server { - listen 80; - - # disable any limits to avoid HTTP 413 for large image uploads - client_max_body_size 0; - - location / { - proxy_pass http://ui/; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - - # When setting up Harbor behind other proxy, such as an Nginx instance, remove the below line if the proxy already has similar settings. - proxy_set_header X-Forwarded-Proto $scheme; - - proxy_buffering off; - proxy_request_buffering off; - } - - location /v1/ { - return 404; - } - - location /v2/ { - proxy_pass http://registry/v2/; - proxy_set_header Host $http_host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - - # When setting up Harbor behind other proxy, such as an Nginx instance, remove the below line if the proxy already has similar settings. - proxy_set_header X-Forwarded-Proto $scheme; - - proxy_buffering off; - proxy_request_buffering off; - - } - - location /service/ { - proxy_pass http://ui/service/; - proxy_set_header Host $host; - proxy_set_header X-Real-IP $remote_addr; - proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for; - - # When setting up Harbor behind other proxy, such as an Nginx instance, remove the below line if the proxy already has similar settings. - proxy_set_header X-Forwarded-Proto $scheme; - - proxy_buffering off; - proxy_request_buffering off; - } - } -} diff --git a/make/common/config/jobservice/app.conf b/make/common/templates/jobservice/app.conf similarity index 100% rename from make/common/config/jobservice/app.conf rename to make/common/templates/jobservice/app.conf diff --git a/make/common/config/registry/root.crt b/make/common/templates/registry/root.crt similarity index 100% rename from make/common/config/registry/root.crt rename to make/common/templates/registry/root.crt diff --git a/make/common/config/ui/private_key.pem b/make/common/templates/ui/private_key.pem similarity index 100% rename from make/common/config/ui/private_key.pem rename to make/common/templates/ui/private_key.pem diff --git a/make/dev/ui/Dockerfile b/make/dev/ui/Dockerfile index f8d56fdd7..dfa7ec5fa 100644 --- a/make/dev/ui/Dockerfile +++ b/make/dev/ui/Dockerfile @@ -23,7 +23,9 @@ COPY make/jsminify.sh /tmp/jsminify.sh RUN chmod u+x /go/bin/harbor_ui \ && sed -i 's/TLS_CACERT/#TLS_CAERT/g' /etc/ldap/ldap.conf \ && sed -i '$a\TLS_REQCERT allow' /etc/ldap/ldap.conf \ - && /tmp/jsminify.sh /go/bin/views/sections/script-include.htm /go/bin/static/resources/js/harbor.app.min.js /go/bin/ + && timestamp=`date '+%s'` \ + && /tmp/jsminify.sh /go/bin/views/sections/script-include.htm /go/bin/static/resources/js/harbor.app.min.$timestamp.js /go/bin/ \ + && sed -i "s/harbor\.app\.min\.js/harbor\.app\.min\.$timestamp\.js/g" /go/bin/views/sections/script-min-include.htm WORKDIR /go/bin/ ENTRYPOINT ["/go/bin/harbor_ui"] diff --git a/make/harbor.cfg b/make/harbor.cfg index fd61bd625..f2d00554e 100644 --- a/make/harbor.cfg +++ b/make/harbor.cfg @@ -85,6 +85,6 @@ crt_email = example@example.com #The path of cert and key files for nginx, they are applied only the protocol is set to https -ssl_cert = /path/to/server.crt -ssl_cert_key = /path/to/server.key +ssl_cert = /data/server.crt +ssl_cert_key = /data/server.key ############# diff --git a/make/photon/ui/Dockerfile b/make/photon/ui/Dockerfile index c858905ac..34b3d5335 100644 --- a/make/photon/ui/Dockerfile +++ b/make/photon/ui/Dockerfile @@ -11,7 +11,10 @@ COPY ./src/favicon.ico /harbor/favicon.ico COPY ./make/jsminify.sh /tmp/jsminify.sh RUN chmod u+x /harbor/harbor_ui \ - && tmp/jsminify.sh /harbor/views/sections/script-include.htm /harbor/static/resources/js/harbor.app.min.js /harbor/ \ + && timestamp=`date '+%s'` \ + && /tmp/jsminify.sh /harbor/views/sections/script-include.htm /harbor/static/resources/js/harbor.app.min.$timestamp.js /harbor/ \ + && sed -i "s/harbor\.app\.min\.js/harbor\.app\.min\.$timestamp\.js/g" /harbor/views/sections/script-min-include.htm \ && echo "TLS_REQCERT allow" >> /etc/openldap/ldap.conf + WORKDIR /harbor/ ENTRYPOINT ["/harbor/harbor_ui"] diff --git a/make/prepare b/make/prepare index 0f4ebf60c..02f180ff1 100755 --- a/make/prepare +++ b/make/prepare @@ -133,6 +133,14 @@ job_config_dir = os.path.join(config_dir, "jobservice") if not os.path.exists(job_config_dir): os.makedirs(job_config_dir) +registry_config_dir = os.path.join(config_dir, "registry") +if not os.path.exists(registry_config_dir): + os.makedirs(registry_config_dir) + +nginx_config_dir = os.path.join(config_dir, "nginx") +if not os.path.exists(nginx_config_dir): + os.makedirs(nginx_config_dir) + def render(src, dest, **kw): t = Template(open(src, 'r').read()) with open(dest, 'w') as f: @@ -141,20 +149,24 @@ def render(src, dest, **kw): ui_conf_env = os.path.join(config_dir, "ui", "env") ui_conf = os.path.join(config_dir, "ui", "app.conf") +jobservice_conf = os.path.join(config_dir, "jobservice", "app.conf") registry_conf = os.path.join(config_dir, "registry", "config.yml") db_conf_env = os.path.join(config_dir, "db", "env") job_conf_env = os.path.join(config_dir, "jobservice", "env") nginx_conf = os.path.join(config_dir, "nginx", "nginx.conf") cert_dir = os.path.join(config_dir, "nginx", "cert") -conf_files = [ ui_conf, ui_conf_env, registry_conf, db_conf_env, job_conf_env, nginx_conf, cert_dir ] -def rmdir(cf): - for f in cf: - if os.path.isdir(f): - rmdir(map(lambda x: os.path.join(f,x), os.listdir(f))) - elif os.path.exists(f) and os.path.basename(f) != ".gitignore": - print("Clearing the configuration file: %s" % f) - os.remove(f) -rmdir(conf_files) +def delfile(src): + if os.path.isfile(src): + try: + os.remove(src) + print("Clearing the configuration file: %s" % src) + except: + pass + elif os.path.isdir(src): + for item in os.listdir(src): + itemsrc=os.path.join(src,item) + delfile(itemsrc) +delfile(config_dir) if protocol == "https": target_cert_path = os.path.join(cert_dir, os.path.basename(cert_path)) @@ -217,6 +229,9 @@ render(os.path.join(templates_dir, "jobservice", "env"), secret_key=secret_key, ui_url=ui_url, verify_remote_cert=verify_remote_cert) + +print("Generated configuration file: %s" % jobservice_conf) +shutil.copyfile(os.path.join(templates_dir, "jobservice", "app.conf"), jobservice_conf) def validate_crt_subj(dirty_subj): subj_list = [item for item in dirty_subj.strip().split("/") \ @@ -264,11 +279,15 @@ if customize_crt == 'on': if openssl_is_installed(shell_stat): private_key_pem = os.path.join(config_dir, "ui", "private_key.pem") root_crt = os.path.join(config_dir, "registry", "root.crt") - crt_conf_files = [ private_key_pem, root_crt ] - rmdir(crt_conf_files) check_private_key_stat(path=private_key_pem) check_certificate_stat(path=root_crt) - + +else: + print("Generated configuration file: %s" % ui_config_dir + "private_key.pem") + shutil.copyfile(os.path.join(templates_dir, "ui", "private_key.pem"), os.path.join(ui_config_dir, "private_key.pem")) + print("Generated configuration file: %s" % registry_config_dir + "root.crt") + shutil.copyfile(os.path.join(templates_dir, "registry", "root.crt"), os.path.join(registry_config_dir, "root.crt")) + FNULL.close() print("The configuration files are ready, please use docker-compose to start the service.") diff --git a/make/ubuntu/ui/Dockerfile b/make/ubuntu/ui/Dockerfile index ac41c6bb4..6a39037b5 100644 --- a/make/ubuntu/ui/Dockerfile +++ b/make/ubuntu/ui/Dockerfile @@ -20,7 +20,9 @@ COPY ./make/jsminify.sh /tmp/jsminify.sh RUN chmod u+x /harbor/harbor_ui \ && sed -i 's/TLS_CACERT/#TLS_CAERT/g' /etc/ldap/ldap.conf \ && sed -i '$a\TLS_REQCERT allow' /etc/ldap/ldap.conf \ - && /tmp/jsminify.sh /harbor/views/sections/script-include.htm /harbor/static/resources/js/harbor.app.min.js /harbor/ + && timestamp=`date '+%s'` \ + && /tmp/jsminify.sh /harbor/views/sections/script-include.htm /harbor/static/resources/js/harbor.app.min.$timestamp.js /harbor/ \ + && sed -i "s/harbor\.app\.min\.js/harbor\.app\.min\.$timestamp\.js/g" /harbor/views/sections/script-min-include.htm WORKDIR /harbor/ ENTRYPOINT ["/harbor/harbor_ui"] diff --git a/src/common/dao/register.go b/src/common/dao/register.go index 770eb6de8..3b1d27a13 100644 --- a/src/common/dao/register.go +++ b/src/common/dao/register.go @@ -52,7 +52,7 @@ func Register(user models.User) (int64, error) { func UserExists(user models.User, target string) (bool, error) { if user.Username == "" && user.Email == "" { - return false, errors.New("User name and email are blank.") + return false, errors.New("user name and email are blank") } o := GetOrmer() diff --git a/src/common/dao/user.go b/src/common/dao/user.go index 765a504db..2ba35021a 100644 --- a/src/common/dao/user.go +++ b/src/common/dao/user.go @@ -131,7 +131,7 @@ func ToggleUserAdminRole(userID, hasAdmin int) error { // ChangeUserPassword ... func ChangeUserPassword(u models.User, oldPassword ...string) (err error) { if len(oldPassword) > 1 { - return errors.New("Wrong numbers of params.") + return errors.New("wrong numbers of params") } o := GetOrmer() @@ -153,7 +153,7 @@ func ChangeUserPassword(u models.User, oldPassword ...string) (err error) { return err } if c == 0 { - return errors.New("No record has been modified, change password failed.") + return errors.New("no record has been modified, change password failed") } return nil @@ -171,7 +171,7 @@ func ResetUserPassword(u models.User) error { return err } if count == 0 { - return errors.New("No record be changed, reset password failed.") + return errors.New("no record be changed, reset password failed") } return nil } diff --git a/src/jobservice/job/statemachine.go b/src/jobservice/job/statemachine.go index f0a02480a..fd12d5443 100644 --- a/src/jobservice/job/statemachine.go +++ b/src/jobservice/job/statemachine.go @@ -64,7 +64,7 @@ func (sm *SM) EnterState(s string) (string, error) { _, exist := targets[s] _, isForced := sm.ForcedStates[s] if !exist && !isForced { - return "", fmt.Errorf("Job id: %d, transition from %s to %s does not exist!", sm.JobID, sm.CurrentState, s) + return "", fmt.Errorf("job id: %d, transition from %s to %s does not exist", sm.JobID, sm.CurrentState, s) } exitHandler, ok := sm.Handlers[sm.CurrentState] if ok { diff --git a/src/ui/api/api_test.go b/src/ui/api/api_test.go deleted file mode 100644 index ce188a790..000000000 --- a/src/ui/api/api_test.go +++ /dev/null @@ -1,9 +0,0 @@ -package api - -import ( - "testing" -) - -func TestMain(t *testing.T) { -} - diff --git a/src/ui/api/dataprepare_test.go b/src/ui/api/dataprepare_test.go index 928c517a1..c6fb6488a 100644 --- a/src/ui/api/dataprepare_test.go +++ b/src/ui/api/dataprepare_test.go @@ -16,18 +16,22 @@ package api import ( + "os" + "github.com/vmware/harbor/src/common/dao" "github.com/vmware/harbor/src/common/models" - "os" ) const ( //Prepare Test info - TestUserName = "testUser0001" - TestUserPwd = "testUser0001" - TestUserEmail = "testUser0001@mydomain.com" - TestProName = "testProject0001" - TestTargetName = "testTarget0001" + TestUserName = "testUser0001" + TestUserPwd = "testUser0001" + TestUserEmail = "testUser0001@mydomain.com" + TestProName = "testProject0001" + TestTargetName = "testTarget0001" + TestRepoName = "testRepo0001" + AdminName = "admin" + DefaultProjectName = "library" ) func CommonAddUser() { @@ -104,3 +108,20 @@ func CommonDelTarget() { func CommonPolicyEabled(policyID int, enabled int) { _ = dao.UpdateRepPolicyEnablement(int64(policyID), enabled) } + +func CommonAddRepository() { + commonRepository := &models.RepoRecord{ + RepositoryID: "1", + Name: TestRepoName, + OwnerName: AdminName, + OwnerID: 1, + ProjectName: DefaultProjectName, + ProjectID: 1, + PullCount: 1, + } + _ = dao.AddRepository(*commonRepository) +} + +func CommonDelRepository() { + _ = dao.DeleteRepository(TestRepoName) +} diff --git a/src/ui/api/harborapi_test.go b/src/ui/api/harborapi_test.go index 3169a6ed9..749927438 100644 --- a/src/ui/api/harborapi_test.go +++ b/src/ui/api/harborapi_test.go @@ -174,19 +174,20 @@ func (a testapi) ProjectsPost(prjUsr usrInfo, project apilib.ProjectReq) (int, e return httpStatusCode, err } -func (a testapi) StatisticGet(user usrInfo) (apilib.StatisticMap, error) { +func (a testapi) StatisticGet(user usrInfo) (int, apilib.StatisticMap, error) { _sling := sling.New().Get(a.basePath) // create path and map variables path := "/api/statistics/" - fmt.Printf("project statistic path: %s\n", path) + _sling = _sling.Path(path) - var successPayload = new(apilib.StatisticMap) - code, body, err := request(_sling, jsonAcceptHeader, user) - if 200 == code && nil == err { + var successPayload apilib.StatisticMap + httpStatusCode, body, err := request(_sling, jsonAcceptHeader, user) + + if err == nil && httpStatusCode == 200 { err = json.Unmarshal(body, &successPayload) } - return *successPayload, err + return httpStatusCode, successPayload, err } func (a testapi) LogGet(user usrInfo, startTime, endTime, lines string) (int, []apilib.AccessLog, error) { @@ -857,7 +858,7 @@ func updateInitPassword(userID int, password string) error { return fmt.Errorf("Failed to get user, userID: %d %v", userID, err) } if user == nil { - return fmt.Errorf("User id: %d does not exist.", userID) + return fmt.Errorf("user id: %d does not exist", userID) } if user.Salt == "" { user.Salt = utils.GenerateRandomString() diff --git a/src/ui/api/project.go b/src/ui/api/project.go index 9905630ec..d4212f762 100644 --- a/src/ui/api/project.go +++ b/src/ui/api/project.go @@ -413,7 +413,7 @@ func validateProjectReq(req projectReq) error { validProjectName := regexp.MustCompile(`^[a-z0-9](?:-*[a-z0-9])*(?:[._][a-z0-9](?:-*[a-z0-9])*)*$`) legal := validProjectName.MatchString(pn) if !legal { - return fmt.Errorf("Project name is not in lower case or contains illegal characters!") + return fmt.Errorf("project name is not in lower case or contains illegal characters") } return nil } diff --git a/src/ui/api/statistic_test.go b/src/ui/api/statistic_test.go index 37400bea4..64d87d421 100644 --- a/src/ui/api/statistic_test.go +++ b/src/ui/api/statistic_test.go @@ -2,14 +2,14 @@ package api import ( "fmt" - "strconv" "testing" "github.com/stretchr/testify/assert" - "github.com/vmware/harbor/tests/apitests/apilib" + //"github.com/vmware/harbor/tests/apitests/apilib" ) func TestStatisticGet(t *testing.T) { + fmt.Println("Testing Statistic API") assert := assert.New(t) @@ -17,69 +17,60 @@ func TestStatisticGet(t *testing.T) { //prepare for test - var myProCount, pubProCount, totalProCount int32 - result, err := apiTest.StatisticGet(*admin) + var priMyProjectCount, priMyRepoCount int32 + var priPublicProjectCount, priPublicRepoCount int32 + var priTotalProjectCount, priTotalRepoCount int32 + + //case 1: case 1: user not login, expect fail to get status info. + fmt.Println("case 1: user not login, expect fail to get status info.") + httpStatusCode, result, err := apiTest.StatisticGet(*unknownUsr) + if err != nil { + t.Error("Error get statistic info.", err.Error()) + t.Log(err) + } else { + assert.Equal(httpStatusCode, int(401), "Case 1: Get status info without login. (401)") + } + + //case 2: admin successful login, expect get status info successful. + fmt.Println("case 2: admin successful login, expect get status info successful.") + httpStatusCode, result, err = apiTest.StatisticGet(*admin) + if err != nil { + t.Error("Error get statistic info.", err.Error()) + t.Log(err) + } else { + assert.Equal(httpStatusCode, int(200), "Case 2: Get status info with admin login. (200)") + //fmt.Println("pri status data %+v", result) + priMyProjectCount = result.MyProjectCount + priMyRepoCount = result.MyRepoCount + priPublicProjectCount = result.PublicProjectCount + priPublicRepoCount = result.PublicRepoCount + priTotalProjectCount = result.TotalProjectCount + priTotalRepoCount = result.TotalRepoCount + } + + //case 3: status info increased after add more project and repo. + fmt.Println("case 3: status info increased after add more project and repo.") + + CommonAddProject() + CommonAddRepository() + + httpStatusCode, result, err = apiTest.StatisticGet(*admin) + //fmt.Println("new status data %+v", result) + if err != nil { t.Error("Error while get statistic information", err.Error()) t.Log(err) } else { - myProCount = result.MyProjectCount - pubProCount = result.PublicProjectCount - totalProCount = result.TotalProjectCount - } - //post project - var project apilib.ProjectReq - project.ProjectName = "statistic_project" - project.Public = 1 - - //case 2: admin successful login, expect project creation success. - fmt.Println("case 2: admin successful login, expect project creation success.") - reply, err := apiTest.ProjectsPost(*admin, project) - if err != nil { - t.Error("Error while creat project", err.Error()) - t.Log(err) - } else { - assert.Equal(reply, int(201), "Case 2: Project creation status should be 201") - } - - //get and compare - result, err = apiTest.StatisticGet(*admin) - if err != nil { - t.Error("Error while get statistic information", err.Error()) - t.Log(err) - } else { - assert.Equal(myProCount+1, result.MyProjectCount, "MyProjectCount should be equal") - assert.Equal(int32(2), result.MyRepoCount, "MyRepoCount should be equal") - assert.Equal(pubProCount+1, result.PublicProjectCount, "PublicProjectCount should be equal") - assert.Equal(int32(2), result.PublicRepoCount, "PublicRepoCount should be equal") - assert.Equal(totalProCount+1, result.TotalProjectCount, "TotalProCount should be equal") - assert.Equal(int32(2), result.TotalRepoCount, "TotalRepoCount should be equal") + assert.Equal(priMyProjectCount+1, result.MyProjectCount, "MyProjectCount should be +1") + assert.Equal(priMyRepoCount+1, result.MyRepoCount, "MyRepoCount should be +1") + assert.Equal(priPublicProjectCount, result.PublicProjectCount, "PublicProjectCount should be equal") + assert.Equal(priPublicRepoCount+1, result.PublicRepoCount, "PublicRepoCount should be +1") + assert.Equal(priTotalProjectCount+1, result.TotalProjectCount, "TotalProCount should be +1") + assert.Equal(priTotalRepoCount+1, result.TotalRepoCount, "TotalRepoCount should be +1") } - //get the project - var projects []apilib.Project - var addProjectID int32 - httpStatusCode, projects, err := apiTest.ProjectsGet(project.ProjectName, 1) - if err != nil { - t.Error("Error while search project by proName and isPublic", err.Error()) - t.Log(err) - } else { - assert.Equal(int(200), httpStatusCode, "httpStatusCode should be 200") - addProjectID = projects[0].ProjectId - } - - //delete the project - projectID := strconv.Itoa(int(addProjectID)) - httpStatusCode, err = apiTest.ProjectsDelete(*admin, projectID) - if err != nil { - t.Error("Error while delete project", err.Error()) - t.Log(err) - } else { - assert.Equal(int(200), httpStatusCode, "Case 1: Project creation status should be 200") - //t.Log(result) - } - - fmt.Printf("\n") - + //delete the project and repo + CommonDelProject() + CommonDelRepository() } diff --git a/src/ui/api/user.go b/src/ui/api/user.go index 17def3bf1..3a0387390 100644 --- a/src/ui/api/user.go +++ b/src/ui/api/user.go @@ -323,13 +323,13 @@ func (ua *UserAPI) ToggleUserAdminRole() { func validate(user models.User) error { if isIllegalLength(user.Username, 1, 20) { - return fmt.Errorf("Username with illegal length.") + return fmt.Errorf("username with illegal length") } if isContainIllegalChar(user.Username, []string{",", "~", "#", "$", "%"}) { - return fmt.Errorf("Username contains illegal characters.") + return fmt.Errorf("username contains illegal characters") } if isIllegalLength(user.Password, 8, 20) { - return fmt.Errorf("Password with illegal length.") + return fmt.Errorf("password with illegal length") } if err := commonValidate(user); err != nil { return err @@ -342,21 +342,21 @@ func commonValidate(user models.User) error { if len(user.Email) > 0 { if m, _ := regexp.MatchString(`^(([^<>()[\]\\.,;:\s@\"]+(\.[^<>()[\]\\.,;:\s@\"]+)*)|(\".+\"))@((\[[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\.[0-9]{1,3}\])|(([a-zA-Z\-0-9]+\.)+[a-zA-Z]{2,}))$`, user.Email); !m { - return fmt.Errorf("Email with illegal format.") + return fmt.Errorf("email with illegal format") } } else { return fmt.Errorf("Email can't be empty") } if isIllegalLength(user.Realname, 0, 20) { - return fmt.Errorf("Realname with illegal length.") + return fmt.Errorf("realname with illegal length") } if isContainIllegalChar(user.Realname, []string{",", "~", "#", "$", "%"}) { - return fmt.Errorf("Realname contains illegal characters.") + return fmt.Errorf("realname contains illegal characters") } if isIllegalLength(user.Comment, -1, 30) { - return fmt.Errorf("Comment with illegal length.") + return fmt.Errorf("comment with illegal length") } return nil diff --git a/src/ui/auth/ldap/ldap.go b/src/ui/auth/ldap/ldap.go index 26f124e67..6193485f1 100644 --- a/src/ui/auth/ldap/ldap.go +++ b/src/ui/auth/ldap/ldap.go @@ -48,7 +48,7 @@ func (l *Auth) Authenticate(m models.AuthModel) (*models.User, error) { } ldapURL := os.Getenv("LDAP_URL") if ldapURL == "" { - return nil, errors.New("Can not get any available LDAP_URL.") + return nil, errors.New("can not get any available LDAP_URL") } log.Debug("ldapURL:", ldapURL) ldap, err := openldap.Initialize(ldapURL) @@ -59,7 +59,7 @@ func (l *Auth) Authenticate(m models.AuthModel) (*models.User, error) { ldapBaseDn := os.Getenv("LDAP_BASE_DN") if ldapBaseDn == "" { - return nil, errors.New("Can not get any available LDAP_BASE_DN.") + return nil, errors.New("can not get any available LDAP_BASE_DN") } log.Debug("baseDn:", ldapBaseDn) diff --git a/src/ui/controllers/base.go b/src/ui/controllers/base.go index cbae01b7c..fdfaa6b6b 100644 --- a/src/ui/controllers/base.go +++ b/src/ui/controllers/base.go @@ -75,7 +75,7 @@ func (b *BaseController) Prepare() { Name: "language", Value: lang, HttpOnly: true, - Domain: "/", + Path: "/", } http.SetCookie(b.Ctx.ResponseWriter, cookies) } @@ -111,7 +111,8 @@ func (b *BaseController) Prepare() { b.UseCompressedJS = true } - if _, err := os.Stat(filepath.Join("static", "resources", "js", "harbor.app.min.js")); os.IsNotExist(err) { + m, err := filepath.Glob(filepath.Join("static", "resources", "js", "harbor.app.min.*.js")) + if err != nil || len(m) == 0 { b.UseCompressedJS = false } diff --git a/src/ui/main.go b/src/ui/main.go index 0111f6537..d355a11cc 100644 --- a/src/ui/main.go +++ b/src/ui/main.go @@ -43,7 +43,7 @@ func updateInitPassword(userID int, password string) error { return fmt.Errorf("Failed to get user, userID: %d %v", userID, err) } if user == nil { - return fmt.Errorf("User id: %d does not exist.", userID) + return fmt.Errorf("user id: %d does not exist", userID) } if user.Salt == "" { salt := utils.GenerateRandomString() diff --git a/src/ui/service/token/authutils.go b/src/ui/service/token/authutils.go index d9ce76400..63e884f5b 100644 --- a/src/ui/service/token/authutils.go +++ b/src/ui/service/token/authutils.go @@ -105,8 +105,17 @@ func FilterAccess(username string, a *token.ResourceActions) { //clear action list to assign to new acess element after perm check. a.Actions = []string{} if a.Type == "repository" { - if strings.Contains(a.Name, "/") { //Only check the permission when the requested image has a namespace, i.e. project - projectName := a.Name[0:strings.LastIndex(a.Name, "/")] + repoSplit := strings.Split(a.Name, "/") + repoLength := len(repoSplit) + if repoLength > 1 { //Only check the permission when the requested image has a namespace, i.e. project + var projectName string + registryURL := os.Getenv("HARBOR_REG_URL") + if repoSplit[0] == registryURL { + projectName = repoSplit[1] + log.Infof("Detected Registry URL in Project Name. Assuming this is a notary request and setting Project Name as %s\n", projectName) + } else { + projectName = repoSplit[0] + } var permission string if len(username) > 0 { isAdmin, err := dao.IsAdminRole(username) diff --git a/tests/testcases/Group2-image-management/2-05-DB-user-delete-image.md b/tests/testcases/Group2-image-management/2-05-DB-user-delete-image.md index c6f7620e6..521a44455 100644 --- a/tests/testcases/Group2-image-management/2-05-DB-user-delete-image.md +++ b/tests/testcases/Group2-image-management/2-05-DB-user-delete-image.md @@ -24,7 +24,7 @@ In below test, user A is non-admin user. User A and project X, Y should be repla 3. On a Docker client, log in as User A and run `docker push` to push an image to the project X, e.g. projectX/myimage:v1. 4. Push a second image with different tag to project X, e.g. projectX/myimage:v2 . 5. Push an image with different name to project X, e.g. projectX/newimage:v1 . -6. Run `docker pull` to verify images can be pushed successfully. +6. Run `docker pull` to verify images can be pulled successfully. 7. In UI, delete the three images one by one. 8. On a Docker client, log in as User A and run `docker pull` to pull the three deleted images of project X. 9. In UI, delete project X. diff --git a/tests/testcases/Group2-image-management/2-06-DB-user-delete-projects.md b/tests/testcases/Group2-image-management/2-06-DB-user-delete-projects.md index 1f8b7c954..dc4481381 100644 --- a/tests/testcases/Group2-image-management/2-06-DB-user-delete-projects.md +++ b/tests/testcases/Group2-image-management/2-06-DB-user-delete-projects.md @@ -25,7 +25,7 @@ User guide 2. Create a project X so that the user has the project admin role. 3. On a Docker client, log in as User A and run `docker push` to push an image to project X, e.g. projectX/myimage:v1. 4. Push an image with different name to project X, e.g. projectX/newimage:v1 . -5. Run `docker pull` to verify images can be pushed successfully. +5. Run `docker pull` to verify images can be pulled successfully. 6. In UI, delete project X directly. (should fail with errors) 7. While keeping the current user A logged on, in a different browser, log in as admin user. 8. Under "Admin Options", create a replication policy of project X to another Harbor instance. (Do not need to activate this policy.) diff --git a/tests/testcases/Group2-image-management/2-23-admin-delete-images.md b/tests/testcases/Group2-image-management/2-23-admin-delete-images.md index 41775d36d..da952f624 100644 --- a/tests/testcases/Group2-image-management/2-23-admin-delete-images.md +++ b/tests/testcases/Group2-image-management/2-23-admin-delete-images.md @@ -23,7 +23,7 @@ In below test, user A is non-admin user. User A and project X should be replaced 2. Create a project X so that the user has the project admin role. 3. On a Docker client, log in as User A and run `docker push` to push an image to the project X, e.g. projectX/myimage:v1. 4. Push an image with different name to project X, e.g. projectX/newimage:v1 . -5. Run `docker pull` to verify images can be pushed successfully. +5. Run `docker pull` to verify images can be pulled successfully. 6. In UI, log out user A's session. 7. Log in as admin user. 8. Under project X, delete the two images one by one. diff --git a/tests/testcases/Group2-image-management/2-24-admin-delete-projects.md b/tests/testcases/Group2-image-management/2-24-admin-delete-projects.md index 5c558a429..16c39c386 100644 --- a/tests/testcases/Group2-image-management/2-24-admin-delete-projects.md +++ b/tests/testcases/Group2-image-management/2-24-admin-delete-projects.md @@ -24,7 +24,7 @@ User guide 1. Log in to UI as user A (non-admin). 2. Create a project X so that the user has the project admin role. 3. On a Docker client, log in as User A and run `docker push` to push an image to project X, e.g. projectX/myimage:v1. -4. Run `docker pull` to verify images can be pushed successfully. +4. Run `docker pull` to verify images can be pulled successfully. 5. In UI, log out user A. 6. Log in as admin user. 7. Delete project X. (should fail with errors) @@ -35,7 +35,7 @@ User guide # Expected Outcome: * Step 7, deleting project X should fail because there are images under it. * Step 8-9, deleting images of project X and then deleting project X should succeed. -* Step 10, there should be logs for delete and create of project X, notice the project name of the deleted operation is displayed differently. +* Step 10, there should be logs for deletion and creation of project X. # Possible Problems: None \ No newline at end of file diff --git a/tests/testcases/Group3-RBAC/3-11-DB-admin-user-manage-project-members.md b/tests/testcases/Group3-RBAC/3-11-DB-admin-user-manage-project-members.md index b1a86a9b6..5a9415a81 100644 --- a/tests/testcases/Group3-RBAC/3-11-DB-admin-user-manage-project-members.md +++ b/tests/testcases/Group3-RBAC/3-11-DB-admin-user-manage-project-members.md @@ -51,17 +51,15 @@ User guide 25. In admin's UI, change user C's role to project admin of project X. 26. In user C's UI, verify his/her role is project admin of project X. -27. In admin's UI, remove user C's from project X. +27. In admin's UI, remove user C from project X. 28. Set project X's publicity to on. 29. On a Docker client host, log in as user C. 30. Use `docker pull` to pull the image from project X. 31. Use `docker push` to push an image to project X. (should fail) -32. In admin's UI, remove user C's from project X. -33. Set project X's publicity to off. -34. On a Docker client host, log in as user C. -35. Use `docker pull` to pull the image from project X. (should fail) -36. Use `docker push` to push an image to project X. (should fail) - +32. Set project X's publicity to off. +33. On a Docker client host, log in as user C. +34. Use `docker pull` to pull the image from project X. (should fail) +35. Use `docker push` to push an image to project X. (should fail) # Expected Outcome: @@ -72,7 +70,7 @@ User guide * Step 26 as described. * Step 30 should succeed. * Step 31 should fail. -* Step 35-36 should fail. +* Step 34-35 should fail. # Possible Problems: None \ No newline at end of file diff --git a/tools/ova/script/config.sh b/tools/ova/script/config.sh index 20df3f278..884835a51 100755 --- a/tools/ova/script/config.sh +++ b/tools/ova/script/config.sh @@ -3,7 +3,6 @@ set -e attrs=( harbor_admin_password - auth_mode ldap_url ldap_searchdn ldap_search_pwd @@ -37,34 +36,38 @@ function format { } #Modify hostname -ip=$(ip addr show eth0|grep "inet "|tr -s ' '|cut -d ' ' -f 3|cut -d '/' -f 1) -if [ -n "$ip" ] +hostname=$(hostname --fqdn) || true +if [ -z "$hostname" ] then - echo "Read IP address: [ IP - $ip ]" - sed -i -r s/"hostname = .*"/"hostname = $ip"/ $cfg + hostname=$(ip addr show eth0|grep "inet "|tr -s ' '|cut -d ' ' -f 3|cut -d '/' -f 1) +fi + +if [ -n "$hostname" ] +then + echo "Read hostname/IP: [ hostname/IP - $hostname ]" + sed -i -r s/"hostname\s*=\s*.*"/"hostname = $hostname"/ $cfg else - echo "Failed to get the IP address" + echo "Failed to get the hostname/IP" exit 1 fi #Handle http/https -protocal=http +protocol=http echo "Read attribute using ovfenv: [ ssl_cert ]" ssl_cert=$(ovfenv -k ssl_cert) echo "Read attribute using ovfenv: [ ssl_cert_key ]" ssl_cert_key=$(ovfenv -k ssl_cert_key) if [ -n "$ssl_cert" ] && [ -n "$ssl_cert_key" ] then - echo "ssl_cert and ssl_cert_key are set, using HTTPS protocal" - protocal=https - sed -i -r s%"#?ui_url_protocol = .*"%"ui_url_protocol = $protocal"% $cfg - mkdir -p /path/to - echo $ssl_cert > /path/to/server.crt - format /path/to/server.crt - echo $ssl_cert_key > /path/to/server.key - format /path/to/server.key + echo "ssl_cert and ssl_cert_key are set, using HTTPS protocol" + protocol=https + sed -i -r s%"#?ui_url_protocol\s*=\s*.*"%"ui_url_protocol = $protocol"% $cfg + echo $ssl_cert > /data/server.crt + format /data/server.crt + echo $ssl_cert_key > /data/server.key + format /data/server.key else - echo "ssl_cert and ssl_cert_key are not set, using HTTP protocal" + echo "ssl_cert and ssl_cert_key are not set, using HTTP protocol" fi for attr in "${attrs[@]}" @@ -84,6 +87,6 @@ do bs=$(echo $value | base64) #value={base64}$bs fi - sed -i -r s%"#?$attr = .*"%"$attr = $value"% $cfg + sed -i -r s%"#?$attr\s*=\s*.*"%"$attr = $value"% $cfg fi done \ No newline at end of file diff --git a/tools/ova/script/firstboot.sh b/tools/ova/script/firstboot.sh index 32ac4c473..a59da1d3d 100755 --- a/tools/ova/script/firstboot.sh +++ b/tools/ova/script/firstboot.sh @@ -34,13 +34,26 @@ tar -zxvf $base_dir/../harbor-offline-installer*.tgz -C $base_dir/../ echo "Loading images..." load -#Configure Harbor echo "Configuring Harbor..." chmod 600 $base_dir/../harbor/harbor.cfg + +#Configure authentication mode +echo "Read attribute using ovfenv: [ auth_mode ]" +auth_mode=$(ovfenv -k auth_mode) +if [ -n "$auth_mode" ] +then + sed -i -r s%"#?auth_mode\s*=\s*.*"%"auth_mode = $auth_mode"% $base_dir/../harbor/harbor.cfg +fi + +#Configure other attrs configure #Start Harbor echo "Starting Harbor..." up +echo "Removing unneeded installation packages..." +rm $base_dir/../harbor-offline-installer*.tgz +rm $base_dir/../harbor/harbor*.tgz + echo "====================================================" \ No newline at end of file diff --git a/tools/ova/script/subsequentboot.sh b/tools/ova/script/subsequentboot.sh index 9cf9a8c0f..25fd2b116 100755 --- a/tools/ova/script/subsequentboot.sh +++ b/tools/ova/script/subsequentboot.sh @@ -15,7 +15,7 @@ addIptableRules #Stop Harbor echo "Shutting down Harbor..." -down +down || true #Garbage collection value=$(ovfenv -k gc_enabled)