upstream/harbor
1
0
Fork 0
mirror of https://github.com/goharbor/harbor synced 2025-04-18 18:09:48 +00:00
Code Issues Packages Projects Releases Wiki Activity

Merge branch 'master' of github.com:steven-zou/harbor

Browse Source
This commit is contained in:
Steven Zou 2017-07-05 17:04:43 +08:00
commit a477cb1a3e
69 changed files with 2864 additions and 721 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

27
Makefile
View File

@ -12,10 +12,12 @@
# golang:1.7.3
# compile_adminserver, compile_ui, compile_jobservice: compile specific binary
#
# build: build Harbor docker images (defuault: build_photon)
# build: build Harbor docker images (default: build_photon)
# for example: make build -e BASEIMAGE=photon
# build_photon: build Harbor docker images from photon baseimage
#
# build_postgresql: build postgresql images basaed on photon os
# make build -e BASEIMAGE=postgresql
# install: include compile binarys, build images, prepare specific \
# version composefile and startup Harbor instance
#
@ -80,7 +82,7 @@ REGISTRYPROJECTNAME=vmware
DEVFLAG=true
NOTARYFLAG=false
REGISTRYVERSION=2.6.1-photon
NGINXVERSION=1.11.5-patched
NGINXVERSION=1.11.13
PHOTONVERSION=1.0
NOTARYVERSION=server-0.5.0
NOTARYSIGNERVERSION=signer-0.5.0
@ -92,7 +94,7 @@ NEWCLARITYVERSION=
#clair parameters
CLAIRVERSION=v2.0.0
CLAIRFLAG=false
CLAIRDBVERSION=9.6.3
CLAIRDBVERSION=9.6.3-photon
#clarity parameters
CLARITYIMAGE=vmware/harbor-clarity-ui-builder[:tag]
@ -166,6 +168,10 @@ DOCKERFILEPATH_DB=$(DOCKERFILEPATH_COMMON)/db
DOCKERFILENAME_DB=Dockerfile
DOCKERFILE_CLARITY=$(MAKEPATH)/dev/nodeclarity/Dockerfile
DOCKERFILEPATH_POSTGRESQL=$(DOCKERFILEPATH_COMMON)/postgresql
DOCKERFILENAME_POSTGRESQL=Dockerfile
# docker image name
DOCKERIMAGENAME_ADMINSERVER=vmware/harbor-adminserver
DOCKERIMAGENAME_UI=vmware/harbor-ui
@ -173,7 +179,7 @@ DOCKERIMAGENAME_JOBSERVICE=vmware/harbor-jobservice
DOCKERIMAGENAME_LOG=vmware/harbor-log
DOCKERIMAGENAME_DB=vmware/harbor-db
DOCKERIMAGENAME_CLATIRY=vmware/harbor-clarity-ui-builder
DOCKERIMAGENAME_POSTGRESQL=vmware/postgresql
# docker-compose files
DOCKERCOMPOSEFILEPATH=$(MAKEPATH)
DOCKERCOMPOSETPLFILENAME=docker-compose.tpl
@ -213,7 +219,7 @@ DOCKERSAVE_PARA=$(DOCKERIMAGENAME_ADMINSERVER):$(VERSIONTAG) \
$(DOCKERIMAGENAME_LOG):$(VERSIONTAG) \
$(DOCKERIMAGENAME_DB):$(VERSIONTAG) \
$(DOCKERIMAGENAME_JOBSERVICE):$(VERSIONTAG) \
vmware/nginx:$(NGINXVERSION) vmware/registry:$(REGISTRYVERSION) \
vmware/nginx-photon:$(NGINXVERSION) vmware/registry:$(REGISTRYVERSION) \
photon:$(PHOTONVERSION)
PACKAGE_OFFLINE_PARA=-zcvf harbor-offline-installer-$(GITTAGVERSION).tgz \
$(HARBORPKG)/common/templates $(HARBORPKG)/$(DOCKERIMGFILE).$(VERSIONTAG).tar.gz \
@ -237,7 +243,7 @@ ifeq ($(NOTARYFLAG), true)
DOCKERCOMPOSE_LIST+= -f $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSENOTARYFILENAME)
endif
ifeq ($(CLAIRFLAG), true)
DOCKERSAVE_PARA+= quay.io/coreos/clair:$(CLAIRVERSION) postgres:$(CLAIRDBVERSION)
DOCKERSAVE_PARA+= quay.io/coreos/clair:$(CLAIRVERSION) vmware/postgresql:$(CLAIRDBVERSION)
PACKAGE_OFFLINE_PARA+= $(HARBORPKG)/$(DOCKERCOMPOSECLAIRFILENAME)
PACKAGE_ONLINE_PARA+= $(HARBORPKG)/$(DOCKERCOMPOSECLAIRFILENAME)
DOCKERCOMPOSE_LIST+= -f $(DOCKERCOMPOSEFILEPATH)/$(DOCKERCOMPOSECLAIRFILENAME)
@ -305,7 +311,10 @@ build_common: version
build_photon: build_common
make -f $(MAKEFILEPATH_PHOTON)/Makefile build -e DEVFLAG=$(DEVFLAG)
build_postgresql:
@echo "buildging postgresql container for photon..."
@cd $(DOCKERFILEPATH_POSTGRESQL) && $(DOCKERBUILD) -f $(DOCKERFILENAME_POSTGRESQL) -t $(DOCKERIMAGENAME_POSTGRESQL):$(CLAIRDBVERSION) .
@echo "Done."
build: build_$(BASEIMAGE)
modify_composefile:
@ -350,7 +359,7 @@ package_offline: compile build modify_sourcefiles modify_composefile
@echo "pulling nginx and registry..."
@$(DOCKERPULL) vmware/registry:$(REGISTRYVERSION)
@$(DOCKERPULL) vmware/nginx:$(NGINXVERSION)
@$(DOCKERPULL) vmware/nginx-photon:$(NGINXVERSION)
@if [ "$(NOTARYFLAG)" = "true" ] ; then \
echo "pulling notary and harbor-notary-db..."; \
$(DOCKERPULL) vmware/notary-photon:$(NOTARYVERSION); \
@ -360,7 +369,7 @@ package_offline: compile build modify_sourcefiles modify_composefile
@if [ "$(CLAIRFLAG)" = "true" ] ; then \
echo "pulling claiy and postgres..."; \
$(DOCKERPULL) quay.io/coreos/clair:$(CLAIRVERSION); \
$(DOCKERPULL) postgres:$(CLAIRDBVERSION); \
$(DOCKERPULL) vmware/postgresql:$(CLAIRDBVERSION); \
fi
@echo "saving harbor docker image"

69
docs/swagger.yaml
View File

@ -566,6 +566,33 @@ paths:
$ref: '#/definitions/User'
401:
description: User need to log in first.
/users/{user_id}:
get:
summary: Get a user's profile.
description: |
Get user's profile with user id.
parameters:
- name: user_id
in: path
type: integer
format: int
required: true
description: Registered user ID
tags:
- Products
responses:
200:
description: Get user's profile successfully.
400:
description: Invalid user ID.
401:
description: User need to log in first.
403:
description: User does not have permission of admin role.
404:
description: User ID does not exist.
500:
description: Unexpected internal errors.
/users/{user_id}:
put:
summary: Update a registered user to change his profile.
@ -588,7 +615,7 @@ paths:
- Products
responses:
200:
description: Updated user's admin role successfully.
description: Updated user's profile successfully.
400:
description: Invalid user ID.
401:
@ -905,7 +932,7 @@ paths:
503:
description: Harbor is not deployed with Clair.
/repositories/{repo_name}/tags/{tag}/vulnerability/detail:
/repositories/{repo_name}/tags/{tag}/vulnerability/details:
get:
summary: Get vulnerability details of the image.
description: |
@ -1065,7 +1092,7 @@ paths:
in: query
type: integer
format: int
required: false
required: true
description: The ID of the policy that triggered this job.
- name: num
in: query
@ -1113,7 +1140,7 @@ paths:
schema:
type: array
items:
$ref: '#/definitions/RepPolicy'
$ref: '#/definitions/JobStatus'
headers:
X-Total-Count:
description: The total count of jobs
@ -1202,7 +1229,7 @@ paths:
schema:
type: array
items:
$ref: '#/definitions/JobStatus'
$ref: '#/definitions/RepPolicy'
400:
description: Invalid project ID.
401:
@ -1809,6 +1836,18 @@ definitions:
type: integer
format: int
description: The public status of the project.
enable_content_trust:
type: boolean
description: Whether content trust is enabled or not. If it is enabled, user cann't pull unsigned images from this project.
prevent_vulnerable_images_from_running:
type: boolean
description: Whether prevent the vulnerable images from running.
prevent_vulnerable_images_from_running_severity:
type: string
description: If the vulnerability is high than severity defined here, the images cann't be pulled.
automatically_scan_images_on_push:
type: boolean
description: Whether scan images automatically when pushing.
Project:
type: object
properties:
@ -1849,6 +1888,18 @@ definitions:
repo_count:
type: integer
description: The number of the repositories under this project.
enable_content_trust:
type: boolean
description: Whether content trust is enabled or not. If it is enabled, user cann't pull unsigned images from this project.
prevent_vulnerable_images_from_running:
type: boolean
description: Whether prevent the vulnerable images from running.
prevent_vulnerable_images_from_running_severity:
type: string
description: If the vulnerability is high than severity defined here, the images cann't be pulled.
automatically_scan_images_on_push:
type: boolean
description: Whether scan images automatically when pushing.
Manifest:
type: object
properties:
@ -1981,14 +2032,14 @@ definitions:
StatisticMap:
type: object
properties:
my_project_count:
private_project_count:
type: integer
format: int32
description: The count of the projects which the user is a member of.
my_repo_count:
description: The count of the private projects which the user is a member of.
private_repo_count:
type: integer
format: int32
description: The count of the repositories belonging to the projects which the user is a member of.
description: The count of the private repositories belonging to the projects which the user is a member of.
public_project_count:
type: integer
format: int32

2
make/common/db/registry.sql
View File

@ -204,4 +204,4 @@ CREATE TABLE IF NOT EXISTS `alembic_version` (
`version_num` varchar(32) NOT NULL
) ENGINE=InnoDB DEFAULT CHARSET=utf8;
insert into alembic_version values ('0.4.0');
insert into alembic_version values ('1.2.0');

11
make/common/nginx/Dockerfile Normal file
View File

@ -0,0 +1,11 @@
FROM library/photon:1.0
RUN tdnf install -y nginx \
&& ln -sf /dev/stdout /var/log/nginx/access.log \
&& ln -sf /dev/stderr /var/log/nginx/error.log \
&& mkdir -p /var/run
EXPOSE 80
STOPSIGNAL SIGQUIT
CMD ["nginx", "-g", "daemon off;"]

26
make/common/postgresql/Dockerfile Normal file
View File

@ -0,0 +1,26 @@
FROM library/photon:1.0
ENV PGDATA /var/lib/postgresql/data
RUN touch /etc/localtime.bak \
&& tdnf install -y sed shadow gzip postgresql\
&& groupadd -r postgres --gid=999 \
&& useradd -r -g postgres --uid=999 postgres \
&& mkdir -p /docker-entrypoint-initdb.d \
&& mkdir -p /run/postgresql \
&& chown -R postgres:postgres /run/postgresql \
&& chmod 2777 /run/postgresql \
&& mkdir -p "$PGDATA" && chown -R postgres:postgres "$PGDATA" && chmod 777 "$PGDATA" \
&& sed -i "s|#listen_addresses = 'localhost'.*|listen_addresses = '*'|g" /usr/share/postgresql/postgresql.conf.sample \
&& sed -i "s|#unix_socket_directories = '/tmp'.*|unix_socket_directories = '/run/postgresql'|g" /usr/share/postgresql/postgresql.conf.sample \
&& touch /usr/share/locale/locale.alias \
&& locale-gen.sh en_US.UTF-8
VOLUME /var/lib/postgresql/data
ADD docker-entrypoint.sh /entrypoint.sh
RUN chmod u+x /entrypoint.sh
ENTRYPOINT ["/entrypoint.sh"]
EXPOSE 5432
CMD ["postgres"]

120
make/common/postgresql/docker-entrypoint.sh Normal file
View File

@ -0,0 +1,120 @@
#!/bin/bash
set -e
# usage: file_env VAR [DEFAULT]
# ie: file_env 'XYZ_DB_PASSWORD' 'example'
# (will allow for "$XYZ_DB_PASSWORD_FILE" to fill in the value of
# "$XYZ_DB_PASSWORD" from a file, especially for Docker's secrets feature)
file_env() {
local var="$1"
local fileVar="${var}_FILE"
local def="${2:-}"
if [ "${!var:-}" ] && [ "${!fileVar:-}" ]; then
echo >&2 "error: both $var and $fileVar are set (but are exclusive)"
exit 1
fi
local val="$def"
if [ "${!var:-}" ]; then
val="${!var}"
elif [ "${!fileVar:-}" ]; then
val="$(< "${!fileVar}")"
fi
export "$var"="$val"
unset "$fileVar"
}
if [ "${1:0:1}" = '-' ]; then
set -- postgres "$@"
fi
if [ "$1" = 'postgres' ]; then
chown -R postgres:postgres $PGDATA
echo here1
# look specifically for PG_VERSION, as it is expected in the DB dir
if [ ! -s "$PGDATA/PG_VERSION" ]; then
file_env 'POSTGRES_INITDB_ARGS'
if [ "$POSTGRES_INITDB_XLOGDIR" ]; then
export POSTGRES_INITDB_ARGS="$POSTGRES_INITDB_ARGS --xlogdir $POSTGRES_INITDB_XLOGDIR"
fi
echo hehe2
su - $1 -c "initdb -D $PGDATA -U postgres -E UTF-8 --lc-collate=en_US.UTF-8 --lc-ctype=en_US.UTF-8 $POSTGRES_INITDB_ARGS"
echo hehe3
# check password first so we can output the warning before postgres
# messes it up
file_env 'POSTGRES_PASSWORD'
if [ "$POSTGRES_PASSWORD" ]; then
pass="PASSWORD '$POSTGRES_PASSWORD'"
authMethod=md5
else
# The - option suppresses leading tabs but *not* spaces. :)
cat >&2 <<-EOF
****************************************************
WARNING: No password has been set for the database.
This will allow anyone with access to the
Postgres port to access your database. In
Docker's default configuration, this is
effectively any other container on the same
system.
Use "-e POSTGRES_PASSWORD=password" to set
it in "docker run".
****************************************************
EOF
pass=
authMethod=trust
fi
{
echo
echo "host all all all $authMethod"
} >> "$PGDATA/pg_hba.conf"
su postgres
echo `whoami`
# internal start of server in order to allow set-up using psql-client
# does not listen on external TCP/IP and waits until start finishes
su - $1 -c "pg_ctl -D \"$PGDATA\" -o \"-c listen_addresses='localhost'\" -w start"
file_env 'POSTGRES_USER' 'postgres'
file_env 'POSTGRES_DB' "$POSTGRES_USER"
psql=( psql -v ON_ERROR_STOP=1 )
if [ "$POSTGRES_DB" != 'postgres' ]; then
"${psql[@]}" --username postgres <<-EOSQL
CREATE DATABASE "$POSTGRES_DB" ;
EOSQL
echo
fi
if [ "$POSTGRES_USER" = 'postgres' ]; then
op='ALTER'
else
op='CREATE'
fi
"${psql[@]}" --username postgres <<-EOSQL
$op USER "$POSTGRES_USER" WITH SUPERUSER $pass ;
EOSQL
echo
psql+=( --username "$POSTGRES_USER" --dbname "$POSTGRES_DB" )
echo
for f in /docker-entrypoint-initdb.d/*; do
case "$f" in
*.sh) echo "$0: running $f"; . "$f" ;;
*.sql) echo "$0: running $f"; "${psql[@]}" -f "$f"; echo ;;
*.sql.gz) echo "$0: running $f"; gunzip -c "$f" | "${psql[@]}"; echo ;;
*) echo "$0: ignoring $f" ;;
esac
echo
done
PGUSER="${PGUSER:-postgres}" \
su - $1 -c "pg_ctl -D \"$PGDATA\" -m fast -w stop"
echo
echo 'PostgreSQL init process complete; ready for start up.'
echo
fi
fi
exec su - $1 -c "$@ -D $PGDATA"

7
make/common/templates/clair/postgresql-init.d/README.md Normal file
View File

@ -0,0 +1,7 @@
This folder used to run some initial sql for clair if needed.
Just put the sql file in this directory and then start the
clair .
both .sql and .gz format supported

1
make/dev/docker-compose.yml
View File

@ -83,6 +83,7 @@ services:
- ./common/config/ui/private_key.pem:/etc/ui/private_key.pem:z
- /data/secretkey:/etc/ui/key:z
- /data/ca_download/:/etc/ui/ca/:z
- /data/service_token:/etc/ui/service_token:z
networks:
- harbor
depends_on:

3
make/docker-compose.clair.yml
View File

@ -17,13 +17,14 @@ services:
aliases:
- postgres
container_name: clair-db
image: postgres:9.6.3
image: vmware/postgresql:9.6.3-photon
restart: always
depends_on:
- log
env_file:
./common/config/clair/postgres_env
volumes:
- ./common/config/clair/postgresql-init.d/:/docker-entrypoint-initdb.d
- /data/clair-db:/var/lib/postgresql/data
logging:
driver: "syslog"

3
make/docker-compose.tpl
View File

@ -77,6 +77,7 @@ services:
- ./common/config/ui/private_key.pem:/etc/ui/private_key.pem:z
- /data/secretkey:/etc/ui/key:z
- /data/ca_download/:/etc/ui/ca/:z
- /data/service_token:/etc/ui/service_token:z
networks:
- harbor
depends_on:
@ -109,7 +110,7 @@ services:
syslog-address: "tcp://127.0.0.1:1514"
tag: "jobservice"
proxy:
image: vmware/nginx:1.11.5-patched
image: vmware/nginx-photon:1.11.13
container_name: nginx
restart: always
volumes:

4
make/prepare
View File

@ -373,6 +373,10 @@ if args.clair_mode:
pg_password = "password"
clair_temp_dir = os.path.join(templates_dir, "clair")
clair_config_dir = prep_conf_dir(config_dir, "clair")
print("Copying offline data file for clair DB")
if os.path.exists(os.path.join(clair_config_dir, "postgresql-init.d")):
shutil.rmtree(os.path.join(clair_config_dir, "postgresql-init.d"))
shutil.copytree(os.path.join(clair_temp_dir, "postgresql-init.d"), os.path.join(clair_config_dir, "postgresql-init.d"))
postgres_env = os.path.join(clair_config_dir, "postgres_env")
render(os.path.join(clair_temp_dir, "postgres_env"), postgres_env, password = pg_password)
clair_conf = os.path.join(clair_config_dir, "config.yaml")

7
src/common/const.go
View File

@ -20,9 +20,9 @@ const (
LDAPAuth = "ldap_auth"
ProCrtRestrEveryone = "everyone"
ProCrtRestrAdmOnly = "adminonly"
LDAPScopeBase = "1"
LDAPScopeOnelevel = "2"
LDAPScopeSubtree = "3"
LDAPScopeBase = 1
LDAPScopeOnelevel = 2
LDAPScopeSubtree = 3
RoleProjectAdmin = 1
RoleDeveloper = 2
@ -65,4 +65,5 @@ const (
AdmiralEndpoint = "admiral_url"
WithNotary = "with_notary"
WithClair = "with_clair"
ScanAllPolicy = "scan_all_policy"
)

10
src/common/dao/projectmember.go
View File

@ -54,9 +54,8 @@ func DeleteProjectMember(projectID int64, userID int) error {
}
// GetUserByProject gets all members of the project.
func GetUserByProject(projectID int64, queryUser models.User) ([]models.User, error) {
func GetUserByProject(projectID int64, queryUser models.User) ([]*models.Member, error) {
o := GetOrmer()
u := []models.User{}
sql := `select u.user_id, u.username, u.creation_time, u.update_time, r.name as rolename,
r.role_id as role
from user u
@ -74,6 +73,9 @@ func GetUserByProject(projectID int64, queryUser models.User) ([]models.User, er
queryParam = append(queryParam, "%"+escape(queryUser.Username)+"%")
}
sql += ` order by u.username `
_, err := o.Raw(sql, queryParam).QueryRows(&u)
return u, err
members := []*models.Member{}
_, err := o.Raw(sql, queryParam).QueryRows(&members)
return members, err
}

23
src/ui/service/utils/utils.go → src/common/models/member.go
View File

@ -12,21 +12,12 @@
// See the License for the specific language governing permissions and
// limitations under the License.
// Package utils contains methods to support security, cache, and webhook functions.
package utils
package models
import (
"net/http"
"github.com/vmware/harbor/src/common/utils/log"
)
// VerifySecret verifies the UI_SECRET cookie in a http request.
// TODO remove
func VerifySecret(r *http.Request, expectedSecret string) bool {
c, err := r.Cookie("secret")
if err != nil {
log.Warningf("Failed to get secret cookie, error: %v", err)
}
return c != nil && c.Value == expectedSecret
// Member holds the details of a member.
type Member struct {
ID int `orm:"pk;column(user_id)" json:"user_id"`
Username string `json:"username"`
Rolename string `json:"role_name"`
Role int `json:"role_id"`
}

24
src/common/models/project.go
View File

@ -73,15 +73,15 @@ func (ps *ProjectSorter) Swap(i, j int) {
// List projects which user1 is member of: query := &QueryParam{Member:&Member{Name:"user1"}}
// List projects which user1 is the project admin : query := &QueryParam{Memeber:&Member{Name:"user1",Role:1}}
type ProjectQueryParam struct {
Name string // the name of project
Owner string // the username of project owner
Public *bool // the project is public or not, can be ture, false and nil
Member *Member // the member of project
Pagination *Pagination // pagination information
Name string // the name of project
Owner string // the username of project owner
Public *bool // the project is public or not, can be ture, false and nil
Member *MemberQuery // the member of project
Pagination *Pagination // pagination information
}
// Member fitler by member's username and role
type Member struct {
// MemberQuery fitler by member's username and role
type MemberQuery struct {
Name string // the username of member
Role int // the role of the member has to the project
}
@ -99,3 +99,13 @@ type BaseProjectCollection struct {
Public bool
Member string
}
// ProjectRequest holds informations that need for creating project API
type ProjectRequest struct {
Name string `json:"project_name"`
Public int `json:"public"`
EnableContentTrust bool `json:"enable_content_trust"`
PreventVulnerableImagesFromRunning bool `json:"prevent_vulnerable_images_from_running"`
PreventVulnerableImagesFromRunningSeverity string `json:"prevent_vulnerable_images_from_running_severity"`
AutomaticallyScanImagesOnPush bool `json:"automatically_scan_images_on_push"`
}

25
src/common/models/scan_job.go
View File

@ -96,3 +96,28 @@ type VulnerabilityItem struct {
Description string `json:"description"`
Fixed string `json:"fixedVersion,omitempty"`
}
// ScanAllPolicy is represent the json request and object for scan all policy, the parm is het
type ScanAllPolicy struct {
Type string `json:"type"`
Parm map[string]interface{} `json:"parameter, omitempty"`
}
const (
// ScanAllNone "none" for not doing any scan all
ScanAllNone = "none"
// ScanAllDaily for doing scan all daily
ScanAllDaily = "daily"
// ScanAllOnRefresh for doing scan all when the Clair DB is refreshed.
ScanAllOnRefresh = "on_refresh"
// ScanAllDailyTime the key for parm of daily scan all policy.
ScanAllDailyTime = "daily_time"
)
//DefaultScanAllPolicy ...
var DefaultScanAllPolicy = ScanAllPolicy{
Type: ScanAllDaily,
Parm: map[string]interface{}{
ScanAllDailyTime: 0,
},
}

185
src/common/scheduler/policy/alternate_policy.go Normal file
View File

@ -0,0 +1,185 @@
package policy
import "github.com/vmware/harbor/src/common/scheduler/task"
import "errors"
import "time"
//AlternatePolicyConfiguration store the related configurations for alternate policy.
type AlternatePolicyConfiguration struct {
//The interval of executing attached tasks.
Duration time.Duration
//The execution time point of each turn
//It's a number to indicate the seconds offset to the 00:00 of UTC time.
OffsetTime int64
//Time should be later than start time.
//If set <=0 value, no limitation.
StartTimestamp int64
//Time should be earlier than end time.
//If set <=0 value, no limitation.
EndTimestamp int64
}
//AlternatePolicy is a policy that repeatedly executing tasks with specified duration during a specified time scope.
type AlternatePolicy struct {
//Keep the attached tasks.
tasks []task.Task
//Policy configurations.
config *AlternatePolicyConfiguration
//Generate time ticks with specified duration.
ticker *time.Ticker
//To indicated whether policy is completed.
isEnabled bool
//Channel used to send evaluation result signals.
evaluation chan EvaluationResult
//Channel used to notify policy termination.
done chan bool
//Channel used to receive terminate signal.
terminator chan bool
}
//NewAlternatePolicy is constructor of creating AlternatePolicy.
func NewAlternatePolicy(config *AlternatePolicyConfiguration) *AlternatePolicy {
return &AlternatePolicy{
tasks: []task.Task{},
config: config,
isEnabled: false,
}
}
//GetConfig returns the current configuration options of this policy.
func (alp *AlternatePolicy) GetConfig() *AlternatePolicyConfiguration {
return alp.config
}
//Name is an implementation of same method in policy interface.
func (alp *AlternatePolicy) Name() string {
return "Alternate Policy"
}
//Tasks is an implementation of same method in policy interface.
func (alp *AlternatePolicy) Tasks() []task.Task {
copyList := []task.Task{}
if alp.tasks != nil && len(alp.tasks) > 0 {
copyList = append(copyList, alp.tasks...)
}
return copyList
}
//Done is an implementation of same method in policy interface.
func (alp *AlternatePolicy) Done() chan bool {
return alp.done
}
//AttachTasks is an implementation of same method in policy interface.
func (alp *AlternatePolicy) AttachTasks(tasks ...task.Task) error {
if tasks == nil || len(tasks) == 0 {
return errors.New("No tasks can be attached")
}
alp.tasks = append(alp.tasks, tasks...)
return nil
}
//Disable is an implementation of same method in policy interface.
func (alp *AlternatePolicy) Disable() error {
alp.isEnabled = false
//Stop the ticker
if alp.ticker != nil {
alp.ticker.Stop()
}
//Stop the evaluation goroutine
alp.terminator <- true
alp.ticker = nil
return nil
}
//Evaluate is an implementation of same method in policy interface.
func (alp *AlternatePolicy) Evaluate() chan EvaluationResult {
//Keep idempotent
if alp.isEnabled && alp.evaluation != nil {
return alp.evaluation
}
alp.done = make(chan bool)
alp.terminator = make(chan bool)
alp.evaluation = make(chan EvaluationResult)
go func() {
timeNow := time.Now().UTC()
timeSeconds := timeNow.Unix()
//Pre-check
//If now is still in the specified time scope.
if alp.config.EndTimestamp > 0 && timeSeconds >= alp.config.EndTimestamp {
//Invalid configuration, exit.
alp.done <- true
alp.isEnabled = false
return
}
if alp.config.StartTimestamp > 0 && timeSeconds < alp.config.StartTimestamp {
//Let's hold on for a while.
forWhile := alp.config.StartTimestamp - timeSeconds
time.Sleep(time.Duration(forWhile) * time.Second)
}
//Reach the execution time point?
utcTime := (int64)(timeNow.Hour()*3600 + timeNow.Minute()*60)
diff := alp.config.OffsetTime - utcTime
if diff < 0 {
diff += 24 * 3600
}
if diff > 0 {
//Wait for a while.
time.Sleep(time.Duration(diff) * time.Second)
}
//Trigger the first tick.
alp.evaluation <- EvaluationResult{}
//Start the ticker for repeat checking.
alp.ticker = time.NewTicker(alp.config.Duration)
for {
select {
case now := <-alp.ticker.C:
{
time := now.UTC().Unix()
if alp.config.EndTimestamp > 0 && time >= alp.config.EndTimestamp {
//Ploicy is done.
alp.done <- true
alp.isEnabled = false
if alp.ticker != nil {
alp.ticker.Stop()
}
alp.ticker = nil
return
}
res := EvaluationResult{}
alp.evaluation <- res
}
case <-alp.terminator:
return
}
}
}()
//Enabled
alp.isEnabled = true
return alp.evaluation
}

120
src/common/scheduler/policy/alternate_policy_test.go Normal file
View File

@ -0,0 +1,120 @@
package policy
import (
"testing"
"time"
)
type fakeTask struct {
number int
}
func (ft *fakeTask) TaskName() string {
return "for testing"
}
func (ft *fakeTask) Run() error {
ft.number++
return nil
}
func TestBasic(t *testing.T) {
tp := NewAlternatePolicy(&AlternatePolicyConfiguration{})
err := tp.AttachTasks(&fakeTask{number: 100})
if err != nil {
t.Fail()
}
if tp.GetConfig() == nil {
t.Fail()
}
if tp.Name() != "Alternate Policy" {
t.Fail()
}
tks := tp.Tasks()
if tks == nil || len(tks) != 1 {
t.Fail()
}
}
func TestEvaluatePolicy(t *testing.T) {
now := time.Now().UTC()
utcOffset := (int64)(now.Hour()*3600 + now.Minute()*60)
tp := NewAlternatePolicy(&AlternatePolicyConfiguration{
Duration: 1 * time.Second,
OffsetTime: utcOffset + 1,
StartTimestamp: -1,
EndTimestamp: now.Add(3 * time.Second).Unix(),
})
err := tp.AttachTasks(&fakeTask{number: 100})
if err != nil {
t.Fail()
}
ch := tp.Evaluate()
done := tp.Done()
counter := 0
READ_SIGNAL:
for {
select {
case <-ch:
counter++
case <-done:
break READ_SIGNAL
case <-time.After(5 * time.Second):
t.Fail()
return
}
}
if counter != 2 {
t.Fail()
}
}
func TestDisablePolicy(t *testing.T) {
now := time.Now().UTC()
utcOffset := (int64)(now.Hour()*3600 + now.Minute()*60)
tp := NewAlternatePolicy(&AlternatePolicyConfiguration{
Duration: 1 * time.Second,
OffsetTime: utcOffset + 1,
StartTimestamp: -1,
EndTimestamp: -1,
})
err := tp.AttachTasks(&fakeTask{number: 100})
if err != nil {
t.Fail()
}
ch := tp.Evaluate()
counter := 0
terminate := make(chan bool)
defer func() {
terminate <- true
}()
go func() {
for {
select {
case <-ch:
counter++
case <-terminate:
return
case <-time.After(6 * time.Second):
return
}
}
}()
time.Sleep(2 * time.Second)
if tp.Disable() != nil {
t.Fatal("Failed to disable policy")
}
//Waiting for everything is stabel
<-time.After(1 * time.Second)
//Copy value
copiedCounter := counter
time.Sleep(2 * time.Second)
if counter != copiedCounter {
t.Fatalf("Policy is still running after calling Disable() %d=%d", copiedCounter, counter)
}
}

15
src/common/scheduler/policy/evaluation_result.go Normal file
View File

@ -0,0 +1,15 @@
package policy
//EvaluationResult is defined to carry the policy evaluated result.
//
//Filed 'Result' is optional.
//Filed 'Error' is optional
//
type EvaluationResult struct {
//Policy is successfully evaluated and the related information can
//be contained in Result if have.
Result interface{}
//Policy is failed to evaluated.
Error error
}

37
src/common/scheduler/policy/policy.go Normal file
View File

@ -0,0 +1,37 @@
package policy
import "github.com/vmware/harbor/src/common/scheduler/task"
//Policy is an if-then logic to determine how the attached tasks should be
//executed based on the evaluation result of the defined conditions.
//E.g:
// Daily execute TASK between 2017/06/24 and 2018/06/23
// Execute TASK at 2017/09/01 14:30:00
//
//Each policy should have a name to identify itself.
//Please be aware that policy with no tasks will be treated as invalid.
//
type Policy interface {
//Return the name of the policy.
Name() string
//Return the attached tasks with this policy.
Tasks() []task.Task
//Attach tasks to this policy
AttachTasks(...task.Task) error
//Done will setup a channel for other components to check whether or not
//the policy is completed. Possibly designed for the none loop policy.
Done() chan bool
//Evaluate the policy based on its definition and return the result via
//result channel. Policy is enabled after it is evaluated.
//Make sure Evaluate is idempotent, that means one policy can be only enabled
//only once even if Evaluate is called more than one times.
Evaluate() chan EvaluationResult
//Disable the enabled policy and release all the allocated resources.
//Disable should also send signal to the terminated channel which returned by Done.
Disable() error
}

258
src/common/scheduler/scheduler.go Normal file
View File

@ -0,0 +1,258 @@
package scheduler
import "github.com/vmware/harbor/src/common/scheduler/policy"
import "github.com/vmware/harbor/src/common/utils/log"
import "errors"
import "strings"
import "fmt"
import "reflect"
import "time"
const (
defaultQueueSize = 10
statSchedulePolicy = "Schedule Policy"
statUnSchedulePolicy = "Unschedule Policy"
statTaskRun = "Task Run"
statTaskComplete = "Task Complete"
statTaskFail = "Task Fail"
)
//StatItem is defined for the stat metrics.
type StatItem struct {
//Metrics catalog
Type string
//The stat value
Value uint32
//Attach some other info
Attachment interface{}
}
//StatSummary is used to collect some metrics of scheduler.
type StatSummary struct {
//Count of scheduled policy
PolicyCount uint32
//Total count of tasks
Tasks uint32
//Count of successfully complete tasks
CompletedTasks uint32
//Count of tasks with errors
TasksWithError uint32
}
//Configuration defines configuration of Scheduler.
type Configuration struct {
QueueSize uint8
}
//Scheduler is designed for scheduling policies.
type Scheduler struct {
//Related configuration options for scheduler.
config *Configuration
//Store to keep the references of scheduled policies.
policies Store
//Queue for accepting the scheduling polices.
scheduleQueue chan policy.Policy
//Queue for receiving policy unschedule request or complete signal.
unscheduleQueue chan string
//Channel for receiving stat metrics.
statChan chan *StatItem
//Channel for terminate scheduler damon.
terminateChan chan bool
//The stat metrics of scheduler.
stats *StatSummary
//To indicate whether scheduler is stopped or not
stopped bool
}
//DefaultScheduler is a default scheduler.
var DefaultScheduler = NewScheduler(nil)
//NewScheduler is constructor for creating a scheduler.
func NewScheduler(config *Configuration) *Scheduler {
var qSize uint8 = defaultQueueSize
if config != nil && config.QueueSize > 0 {
qSize = config.QueueSize
}
sq := make(chan policy.Policy, qSize)
usq := make(chan string, qSize)
stChan := make(chan *StatItem, 4)
tc := make(chan bool, 2)
store := NewConcurrentStore(10)
return &Scheduler{
config: config,
policies: store,
scheduleQueue: sq,
unscheduleQueue: usq,
statChan: stChan,
terminateChan: tc,
stats: &StatSummary{
PolicyCount: 0,
Tasks: 0,
CompletedTasks: 0,
TasksWithError: 0,
},
stopped: true,
}
}
//Start the scheduler damon.
func (sch *Scheduler) Start() {
if !sch.stopped {
return
}
go func() {
defer func() {
if r := recover(); r != nil {
log.Errorf("Runtime error in scheduler:%s\n", r)
}
}()
defer func() {
sch.stopped = true
}()
for {
select {
case p := <-sch.scheduleQueue:
//Schedule the policy.
watcher := NewWatcher(p, sch.statChan, sch.unscheduleQueue)
//Keep the policy for future use after it's successfully scheduled.
sch.policies.Put(p.Name(), watcher)
//Enable it.
watcher.Start()
sch.statChan <- &StatItem{statSchedulePolicy, 1, nil}
case name := <-sch.unscheduleQueue:
//Find the watcher.
watcher := sch.policies.Remove(name)
if watcher != nil && watcher.IsRunning() {
watcher.Stop()
}
sch.statChan <- &StatItem{statUnSchedulePolicy, 1, nil}
case <-sch.terminateChan:
//Exit
return
case stat := <-sch.statChan:
{
switch stat.Type {
case statSchedulePolicy:
sch.stats.PolicyCount += stat.Value
break
case statUnSchedulePolicy:
sch.stats.PolicyCount -= stat.Value
break
case statTaskRun:
sch.stats.Tasks += stat.Value
break
case statTaskComplete:
sch.stats.CompletedTasks += stat.Value
break
case statTaskFail:
sch.stats.TasksWithError += stat.Value
break
default:
break
}
log.Infof("Policies:%d, Tasks:%d, CompletedTasks:%d, FailedTasks:%d\n",
sch.stats.PolicyCount,
sch.stats.Tasks,
sch.stats.CompletedTasks,
sch.stats.TasksWithError)
if stat.Attachment != nil &&
reflect.TypeOf(stat.Attachment).String() == "*errors.errorString" {
log.Errorf("%s: %s\n", stat.Type, stat.Attachment.(error).Error())
}
}
}
}
}()
sch.stopped = false
log.Infof("Policy scheduler start at %s\n", time.Now().UTC().Format(time.RFC3339))
}
//Stop the scheduler damon.
func (sch *Scheduler) Stop() {
if sch.stopped {
return
}
//Terminate damon firstly to stop receiving signals.
sch.terminateChan <- true
//Stop all watchers.
for _, wt := range sch.policies.GetAll() {
wt.Stop()
}
//Clear resources
sch.policies.Clear()
log.Infof("Policy scheduler stop at %s\n", time.Now().UTC().Format(time.RFC3339))
}
//Schedule and enable the policy.
func (sch *Scheduler) Schedule(scheduledPolicy policy.Policy) error {
if scheduledPolicy == nil {
return errors.New("nil is not Policy object")
}
if strings.TrimSpace(scheduledPolicy.Name()) == "" {
return errors.New("Policy should be assigned a name")
}
tasks := scheduledPolicy.Tasks()
if tasks == nil || len(tasks) == 0 {
return errors.New("Policy must attach task(s)")
}
if sch.policies.Exists(scheduledPolicy.Name()) {
return errors.New("Duplicated policy")
}
//Schedule the policy.
sch.scheduleQueue <- scheduledPolicy
return nil
}
//UnSchedule the specified policy from the enabled policies list.
func (sch *Scheduler) UnSchedule(policyName string) error {
if strings.TrimSpace(policyName) == "" {
return errors.New("Empty policy name is invalid")
}
if !sch.policies.Exists(policyName) {
return fmt.Errorf("Policy %s is not existing", policyName)
}
//Unschedule the policy.
sch.unscheduleQueue <- policyName
return nil
}
//IsStopped to indicate whether the scheduler is stopped
func (sch *Scheduler) IsStopped() bool {
return sch.stopped
}

140
src/common/scheduler/scheduler_store.go Normal file
View File

@ -0,0 +1,140 @@
package scheduler
import "sync"
import "strings"
const defaultSize = 10
//Store define the basic operations for storing and managing policy watcher.
//The concrete implementation should consider concurrent supporting scenario.
//
type Store interface {
//Put a new policy in.
Put(key string, value *Watcher)
//Get the corresponding policy with the key.
Get(key string) *Watcher
//Check if the key existing in the store.
Exists(key string) bool
//Remove the specified policy and return its reference.
Remove(key string) *Watcher
//Size return the total count of items in store.
Size() uint32
//Get all the items in the store.
GetAll() []*Watcher
//Clear store.
Clear()
}
//ConcurrentStore implements Store interface and supports concurrent operations.
type ConcurrentStore struct {
//Read-write mutex to synchronize the data map.
mutex *sync.RWMutex
//Map used to keep the policy list.
data map[string]*Watcher
}
//NewConcurrentStore is used to create a new store and return the pointer reference.
func NewConcurrentStore(initialSize uint32) *ConcurrentStore {
var initSize uint32 = defaultSize
if initialSize > 0 {
initSize = initialSize
}
mutex := new(sync.RWMutex)
data := make(map[string]*Watcher, initSize)
return &ConcurrentStore{mutex, data}
}
//Put a policy into store.
func (cs *ConcurrentStore) Put(key string, value *Watcher) {
if strings.TrimSpace(key) == "" || value == nil {
return
}
defer cs.mutex.Unlock()
cs.mutex.Lock()
cs.data[key] = value
}
//Get policy via key.
func (cs *ConcurrentStore) Get(key string) *Watcher {
if strings.TrimSpace(key) == "" {
return nil
}
defer cs.mutex.RUnlock()
cs.mutex.RLock()
return cs.data[key]
}
//Exists is used to check whether or not the key exists in store.
func (cs *ConcurrentStore) Exists(key string) bool {
if strings.TrimSpace(key) == "" {
return false
}
defer cs.mutex.RUnlock()
cs.mutex.RLock()
_, ok := cs.data[key]
return ok
}
//Remove is to delete the specified policy.
func (cs *ConcurrentStore) Remove(key string) *Watcher {
if !cs.Exists(key) {
return nil
}
defer cs.mutex.Unlock()
cs.mutex.Lock()
if wt, ok := cs.data[key]; ok {
delete(cs.data, key)
return wt
}
return nil
}
//Size return the total count of items in store.
func (cs *ConcurrentStore) Size() uint32 {
return (uint32)(len(cs.data))
}
//Get all the items of store.
func (cs *ConcurrentStore) GetAll() []*Watcher {
all := []*Watcher{}
defer cs.mutex.RUnlock()
cs.mutex.RLock()
for _, v := range cs.data {
all = append(all, v)
}
return all
}
//Clear all the items in store.
func (cs *ConcurrentStore) Clear() {
if cs.Size() == 0 {
return
}
defer cs.mutex.Unlock()
cs.mutex.Lock()
for k := range cs.data {
delete(cs.data, k)
}
}

71
src/common/scheduler/scheduler_store_test.go Normal file
View File

@ -0,0 +1,71 @@
package scheduler
import (
"testing"
)
func TestPut(t *testing.T) {
store := NewConcurrentStore(10)
if store == nil {
t.Fatal("Failed to creat store instance")
}
store.Put("testing", NewWatcher(nil, nil, nil))
if store.Size() != 1 {
t.Fail()
}
}
func TestGet(t *testing.T) {
store := NewConcurrentStore(10)
if store == nil {
t.Fatal("Failed to creat store instance")
}
store.Put("testing", NewWatcher(nil, nil, nil))
w := store.Get("testing")
if w == nil {
t.Fail()
}
}
func TestRemove(t *testing.T) {
store := NewConcurrentStore(10)
if store == nil {
t.Fatal("Failed to creat store instance")
}
store.Put("testing", NewWatcher(nil, nil, nil))
if !store.Exists("testing") {
t.Fail()
}
w := store.Remove("testing")
if w == nil {
t.Fail()
}
}
func TestExisting(t *testing.T) {
store := NewConcurrentStore(10)
if store == nil {
t.Fatal("Failed to creat store instance")
}
store.Put("testing", NewWatcher(nil, nil, nil))
if !store.Exists("testing") {
t.Fail()
}
if store.Exists("fake_key") {
t.Fail()
}
}
func TestGetAll(t *testing.T) {
store := NewConcurrentStore(10)
if store == nil {
t.Fatal("Failed to creat store instance")
}
store.Put("testing", NewWatcher(nil, nil, nil))
store.Put("testing2", NewWatcher(nil, nil, nil))
list := store.GetAll()
if list == nil || len(list) != 2 {
t.Fail()
}
}

142
src/common/scheduler/scheduler_test.go Normal file
View File

@ -0,0 +1,142 @@
package scheduler
import (
"testing"
"time"
"github.com/vmware/harbor/src/common/scheduler/policy"
"github.com/vmware/harbor/src/common/scheduler/task"
)
type fakePolicy struct {
tasks []task.Task
done chan bool
evaluation chan policy.EvaluationResult
terminate chan bool
ticker *time.Ticker
}
func (fp *fakePolicy) Name() string {
return "testing policy"
}
func (fp *fakePolicy) Tasks() []task.Task {
return fp.tasks
}
func (fp *fakePolicy) AttachTasks(tasks ...task.Task) error {
fp.tasks = append(fp.tasks, tasks...)
return nil
}
func (fp *fakePolicy) Done() chan bool {
return fp.done
}
func (fp *fakePolicy) Evaluate() chan policy.EvaluationResult {
fp.evaluation = make(chan policy.EvaluationResult, 2)
fp.done = make(chan bool)
fp.terminate = make(chan bool)
fp.evaluation <- policy.EvaluationResult{}
go func() {
fp.ticker = time.NewTicker(1 * time.Second)
for {
select {
case <-fp.terminate:
return
case <-fp.ticker.C:
fp.evaluation <- policy.EvaluationResult{}
}
}
}()
return fp.evaluation
}
func (fp *fakePolicy) Disable() error {
if fp.ticker != nil {
fp.ticker.Stop()
}
fp.terminate <- true
return nil
}
type fakeTask struct {
number int
}
func (ft *fakeTask) TaskName() string {
return "for testing"
}
func (ft *fakeTask) Run() error {
ft.number++
return nil
}
//Wacher will be tested together with scheduler.
func TestScheduler(t *testing.T) {
DefaultScheduler.Start()
if DefaultScheduler.policies.Size() != 0 {
t.Fail()
}
if DefaultScheduler.stats.PolicyCount != 0 {
t.Fail()
}
if DefaultScheduler.IsStopped() {
t.Fatal("Scheduler is not started")
}
fp := &fakePolicy{
tasks: []task.Task{},
}
fk := &fakeTask{number: 100}
fp.AttachTasks(fk)
if DefaultScheduler.Schedule(fp) != nil {
t.Fatal("Schedule policy failed")
}
//Waiting for everything is stable
time.Sleep(1 * time.Second)
if DefaultScheduler.policies.Size() == 0 {
t.Fatal("No policy in the store after calling Schedule()")
}
if DefaultScheduler.stats.PolicyCount != 1 {
t.Fatal("Policy stats do not match")
}
time.Sleep(2 * time.Second)
if fk.number == 100 {
t.Fatal("Task is not triggered")
}
if DefaultScheduler.stats.Tasks == 0 {
t.Fail()
}
if DefaultScheduler.stats.CompletedTasks == 0 {
t.Fail()
}
if DefaultScheduler.UnSchedule(fp.Name()) != nil {
t.Fatal("Unschedule policy failed")
}
//Waiting for everything is stable
time.Sleep(1 * time.Second)
if DefaultScheduler.stats.PolicyCount != 0 {
t.Fatal("Policy count does not match after calling UnSchedule()")
}
copiedValue := DefaultScheduler.stats.CompletedTasks
<-time.After(2 * time.Second)
if copiedValue != DefaultScheduler.stats.CompletedTasks {
t.Fatalf("Policy is still enabled after calling UnSchedule(),%d=%d", copiedValue, DefaultScheduler.stats.CompletedTasks)
}
DefaultScheduler.Stop()
if DefaultScheduler.policies.Size() != 0 {
t.Fatal("Scheduler is not cleared after stopping")
}
}

21
src/common/scheduler/task/scan_all_task.go Normal file
View File

@ -0,0 +1,21 @@
package task
import "github.com/vmware/harbor/src/ui/utils"
//ScanAllTask is task of scanning all tags.
type ScanAllTask struct{}
//NewScanAllTask is constructor of creating ScanAllTask.
func NewScanAllTask() *ScanAllTask {
return &ScanAllTask{}
}
//TaskName returns the name of the task.
func (sat *ScanAllTask) TaskName() string {
return "scan all"
}
//Run the actions.
func (sat *ScanAllTask) Run() error {
return utils.ScanAllImages()
}

14
src/common/scheduler/task/scan_all_task_test.go Normal file
View File

@ -0,0 +1,14 @@
package task
import "testing"
func TestTask(t *testing.T) {
tk := NewScanAllTask()
if tk == nil {
t.Fail()
}
if tk.TaskName() != "scan all" {
t.Fail()
}
}

10
src/common/scheduler/task/task.go Normal file
View File

@ -0,0 +1,10 @@
package task
//Task is used to synchronously run specific action(s).
type Task interface {
//Name of the task.
TaskName() string
//Run the concrete code here
Run() error
}

128
src/common/scheduler/watcher.go Normal file
View File

@ -0,0 +1,128 @@
package scheduler
import (
"github.com/vmware/harbor/src/common/scheduler/policy"
"github.com/vmware/harbor/src/common/scheduler/task"
"github.com/vmware/harbor/src/common/utils/log"
)
//Watcher is an asynchronous runner to provide an evaluation environment for the policy.
type Watcher struct {
//The target policy.
p policy.Policy
//The channel for receive stop signal.
cmdChan chan bool
//Indicate whether the watch is started and running.
isRunning bool
//Report stats to scheduler.
stats chan *StatItem
//If policy is automatically completed, report the policy to scheduler.
doneChan chan string
}
//NewWatcher is used as a constructor.
func NewWatcher(p policy.Policy, st chan *StatItem, done chan string) *Watcher {
return &Watcher{
p: p,
cmdChan: make(chan bool),
isRunning: false,
stats: st,
doneChan: done,
}
}
//Start the running.
func (wc *Watcher) Start() {
if wc.isRunning {
return
}
if wc.p == nil {
return
}
go func(pl policy.Policy) {
defer func() {
if r := recover(); r != nil {
log.Errorf("Runtime error in watcher:%s\n", r)
}
}()
evalChan := pl.Evaluate()
done := pl.Done()
for {
select {
case <-evalChan:
{
//Start to run the attached tasks.
for _, t := range pl.Tasks() {
go func(tk task.Task) {
defer func() {
if r := recover(); r != nil {
log.Errorf("Runtime error in task execution:%s\n", r)
}
}()
err := tk.Run()
//Report task execution stats.
st := &StatItem{statTaskComplete, 1, err}
if err != nil {
st.Type = statTaskFail
}
if wc.stats != nil {
wc.stats <- st
}
}(t)
//Report task run stats.
st := &StatItem{statTaskRun, 1, nil}
if wc.stats != nil {
wc.stats <- st
}
}
}
case <-done:
{
//Policy is automatically completed.
wc.isRunning = false
//Report policy change stats.
wc.doneChan <- wc.p.Name()
return
}
case <-wc.cmdChan:
//Exit goroutine.
return
}
}
}(wc.p)
wc.isRunning = true
}
//Stop the running.
func (wc *Watcher) Stop() {
if !wc.isRunning {
return
}
//Disable policy.
if wc.p != nil {
wc.p.Disable()
}
//Stop watcher.
wc.cmdChan <- true
wc.isRunning = false
}
//IsRunning to indicate if the watcher is still running.
func (wc *Watcher) IsRunning() bool {
return wc.isRunning
}

131
src/common/security/authcontext/authcontext.go
View File

@ -15,34 +15,37 @@
package authcontext
import (
"bytes"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"strings"
"github.com/vmware/harbor/src/ui/config"
)
// TODO update the value of role when admiral API is ready
const (
// AuthTokenHeader is the key of auth token header
AuthTokenHeader = "x-xenon-auth-token"
sysAdminRole = "CLOUD_ADMIN"
projectAdminRole = "DEVOPS_ADMIN"
developerRole = "DEVELOPER"
guestRole = "GUEST"
projectAdminRole = "PROJECT_ADMIN"
developerRole = "PROJECT_MEMBER"
guestRole = "PROJECT_VIEWER"
)
var client = &http.Client{
Transport: &http.Transport{},
type project struct {
DocumentSelfLink string `json:"documentSelfLink"`
Name string `json:"name"`
Roles []string `json:"roles"`
}
// AuthContext ...
type AuthContext struct {
PrincipalID string `json:"principalId"`
Name string `json:"name"`
Roles []string `json:"projects"`
Projects map[string][]string `json:"roles"`
PrincipalID string `json:"id"`
Name string `json:"name"`
Email string `json:"email"`
Roles []string `json:"roles"`
Projects []*project `json:"projects"`
}
// GetUsername ...
@ -54,7 +57,6 @@ func (a *AuthContext) GetUsername() string {
func (a *AuthContext) IsSysAdmin() bool {
isSysAdmin := false
for _, role := range a.Roles {
// TODO update the value of role when admiral API is ready
if role == sysAdminRole {
isSysAdmin = true
break
@ -64,14 +66,14 @@ func (a *AuthContext) IsSysAdmin() bool {
}
// HasReadPerm ...
func (a *AuthContext) HasReadPerm(project string) bool {
_, exist := a.Projects[project]
return exist
func (a *AuthContext) HasReadPerm(projectName string) bool {
roles := a.getRoles(projectName)
return len(roles) > 0
}
// HasWritePerm ...
func (a *AuthContext) HasWritePerm(project string) bool {
roles, _ := a.Projects[project]
func (a *AuthContext) HasWritePerm(projectName string) bool {
roles := a.getRoles(projectName)
for _, role := range roles {
if role == projectAdminRole || role == developerRole {
return true
@ -81,8 +83,8 @@ func (a *AuthContext) HasWritePerm(project string) bool {
}
// HasAllPerm ...
func (a *AuthContext) HasAllPerm(project string) bool {
roles, _ := a.Projects[project]
func (a *AuthContext) HasAllPerm(projectName string) bool {
roles := a.getRoles(projectName)
for _, role := range roles {
if role == projectAdminRole {
return true
@ -91,17 +93,77 @@ func (a *AuthContext) HasAllPerm(project string) bool {
return false
}
// GetByToken ...
func GetByToken(token string) (*AuthContext, error) {
endpoint := config.AdmiralEndpoint()
path := strings.TrimRight(endpoint, "/") + "/sso/auth-context"
req, err := http.NewRequest(http.MethodGet, path, nil)
func (a *AuthContext) getRoles(projectName string) []string {
for _, project := range a.Projects {
if project.Name == projectName {
return project.Roles
}
}
return []string{}
}
// GetMyProjects returns all projects which the user is a member of
func (a *AuthContext) GetMyProjects() []string {
projects := []string{}
for _, project := range a.Projects {
projects = append(projects, project.Name)
}
return projects
}
// GetAuthCtx returns the auth context of the current user
func GetAuthCtx(client *http.Client, url, token string) (*AuthContext, error) {
return get(client, url, token)
}
// GetAuthCtxOfUser returns the auth context of the specific user
func GetAuthCtxOfUser(client *http.Client, url, token string, username string) (*AuthContext, error) {
return get(client, url, token, username)
}
// get the user's auth context, if the username is not provided
// get the default auth context of the token
func get(client *http.Client, url, token string, username ...string) (*AuthContext, error) {
endpoint := ""
if len(username) > 0 && len(username[0]) > 0 {
endpoint = buildSpecificUserAuthCtxURL(url, username[0])
} else {
endpoint = buildCurrentUserAuthCtxURL(url)
}
req, err := http.NewRequest(http.MethodGet, endpoint, nil)
if err != nil {
return nil, err
}
req.Header.Add(AuthTokenHeader, token)
return send(client, req)
}
// Login with credential and returns auth context and error
func Login(client *http.Client, url, username, password string) (*AuthContext, error) {
data, err := json.Marshal(&struct {
Username string `json:"username"`
Password string `json:"password"`
}{
Username: username,
Password: password,
})
if err != nil {
return nil, err
}
req, err := http.NewRequest(http.MethodPost, buildLoginURL(url), bytes.NewBuffer(data))
if err != nil {
return nil, err
}
return send(client, req)
}
func send(client *http.Client, req *http.Request) (*AuthContext, error) {
resp, err := client.Do(req)
if err != nil {
return nil, err
@ -114,16 +176,27 @@ func GetByToken(token string) (*AuthContext, error) {
}
if resp.StatusCode != http.StatusOK {
return nil, fmt.Errorf("failed to get auth context by token: %d %s",
resp.StatusCode, string(data))
return nil, fmt.Errorf("unexpected status code: %d %s", resp.StatusCode, string(data))
}
ctx := &AuthContext{
Projects: make(map[string][]string),
}
ctx := &AuthContext{}
if err = json.Unmarshal(data, ctx); err != nil {
return nil, err
}
return ctx, nil
}
func buildCurrentUserAuthCtxURL(url string) string {
return strings.TrimRight(url, "/") + "/auth/session"
}
func buildSpecificUserAuthCtxURL(url, principalID string) string {
return fmt.Sprintf("%s/auth/idm/principals/%s/security-context",
strings.TrimRight(url, "/"), principalID)
}
// TODO update the url
func buildLoginURL(url string) string {
return strings.TrimRight(url, "/") + "/sso/login"
}

4
src/common/security/context.go
View File

@ -24,8 +24,8 @@ type Context interface {
IsSysAdmin() bool
// HasReadPerm returns whether the user has read permission to the project
HasReadPerm(projectIDOrName interface{}) bool
// HasWritePerm returns whether the user has write permission to the project
// HasWritePerm returns whether the user has write permission to the project
HasWritePerm(projectIDOrName interface{}) bool
// HasAllPerm returns whether the user has all permissions to the project
// HasAllPerm returns whether the user has all permissions to the project
HasAllPerm(projectIDOrName interface{}) bool
}

2
src/common/security/secret/context.go
View File

@ -67,7 +67,7 @@ func (s *SecurityContext) HasReadPerm(projectIDOrName interface{}) bool {
if s.store == nil {
return false
}
return s.store.GetUsername(s.secret) == secret.JobserviceUser
return s.store.GetUsername(s.secret) == secret.JobserviceUser || s.store.GetUsername(s.secret) == secret.UIUser
}
// HasWritePerm always returns false

10
src/common/utils/clair/client.go
View File

@ -56,8 +56,6 @@ func (c *Client) ScanLayer(l models.ClairLayer) error {
if err != nil {
return err
}
c.logger.Infof("endpoint: %s", c.endpoint)
c.logger.Infof("body: %s", string(data))
req, err := http.NewRequest("POST", c.endpoint+"/v1/layers", bytes.NewReader(data))
if err != nil {
return err
@ -68,13 +66,13 @@ func (c *Client) ScanLayer(l models.ClairLayer) error {
return err
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
c.logger.Infof("response code: %d", resp.StatusCode)
if resp.StatusCode != http.StatusCreated {
c.logger.Warningf("Unexpected status code: %d", resp.StatusCode)
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
return fmt.Errorf("Unexpected status code: %d, text: %s", resp.StatusCode, string(b))
}
c.logger.Infof("Returning.")

87
src/common/utils/registry/auth/tokenauthorizer.go
View File

@ -15,20 +15,13 @@
package auth
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"strings"
"sync"
"time"
//"github.com/vmware/harbor/src/common/config"
"github.com/vmware/harbor/src/common/models"
"github.com/vmware/harbor/src/common/utils/log"
"github.com/vmware/harbor/src/common/utils/registry"
registry_error "github.com/vmware/harbor/src/common/utils/error"
token_util "github.com/vmware/harbor/src/ui/service/token"
)
@ -36,13 +29,14 @@ const (
latency int = 10 //second, the network latency when token is received
)
type scope struct {
// Scope ...
type Scope struct {
Type string
Name string
Actions []string
}
func (s *scope) string() string {
func (s *Scope) string() string {
return fmt.Sprintf("%s:%s:%s", s.Type, s.Name, strings.Join(s.Actions, ","))
}
@ -50,7 +44,7 @@ type tokenGenerator func(realm, service string, scopes []string) (token string,
// Implements interface Authorizer
type tokenAuthorizer struct {
scope *scope
scope *Scope
tg tokenGenerator
cache string // cached token
expiresAt *time.Time // The UTC standard time at when the token will expire
@ -64,13 +58,13 @@ func (t *tokenAuthorizer) Scheme() string {
// AuthorizeRequest will add authorization header which contains a token before the request is sent
func (t *tokenAuthorizer) Authorize(req *http.Request, params map[string]string) error {
var scopes []*scope
var scopes []*Scope
var token string
hasFrom := false
from := req.URL.Query().Get("from")
if len(from) != 0 {
s := &scope{
s := &Scope{
Type: "repository",
Name: from,
Actions: []string{"pull"},
@ -154,7 +148,7 @@ func NewStandardTokenAuthorizer(credential Credential, insecure bool,
}
if len(scopeType) != 0 || len(scopeName) != 0 {
authorizer.scope = &scope{
authorizer.scope = &Scope{
Type: scopeType,
Name: scopeName,
Actions: scopeActions,
@ -166,66 +160,21 @@ func NewStandardTokenAuthorizer(credential Credential, insecure bool,
return authorizer
}
func (s *standardTokenAuthorizer) generateToken(realm, service string, scopes []string) (token string, expiresIn int, issuedAt *time.Time, err error) {
func (s *standardTokenAuthorizer) generateToken(realm, service string, scopes []string) (string, int, *time.Time, error) {
realm = s.tokenURL(realm)
tk, err := getToken(s.client, s.credential, realm,
service, scopes)
u, err := url.Parse(realm)
if len(tk.IssuedAt) == 0 {
return tk.Token, tk.ExpiresIn, nil, nil
}
issuedAt, err := time.Parse(time.RFC3339, tk.IssuedAt)
if err != nil {
return
}
q := u.Query()
q.Add("service", service)
for _, scope := range scopes {
q.Add("scope", scope)
}
u.RawQuery = q.Encode()
r, err := http.NewRequest("GET", u.String(), nil)
if err != nil {
return
return "", 0, nil, err
}
if s.credential != nil {
s.credential.AddAuthorization(r)
}
resp, err := s.client.Do(r)
if err != nil {
return
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return
}
if resp.StatusCode != http.StatusOK {
err = &registry_error.Error{
StatusCode: resp.StatusCode,
Detail: string(b),
}
return
}
tk := models.Token{}
if err = json.Unmarshal(b, &tk); err != nil {
return
}
token = tk.Token
expiresIn = tk.ExpiresIn
if len(tk.IssuedAt) != 0 {
t, err := time.Parse(time.RFC3339, tk.IssuedAt)
if err != nil {
log.Errorf("error occurred while parsing issued_at: %v", err)
err = nil
} else {
issuedAt = &t
}
}
return
return tk.Token, tk.ExpiresIn, &issuedAt, nil
}
// when the registry client is used inside Harbor, the token request
@ -267,7 +216,7 @@ func newUsernameTokenAuthorizer(notary bool, username, scopeType, scopeName stri
username: username,
}
authorizer.scope = &scope{
authorizer.scope = &Scope{
Type: scopeType,
Name: scopeName,
Actions: scopeActions,

93
src/common/utils/registry/auth/util.go Normal file
View File

@ -0,0 +1,93 @@
// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package auth
import (
"encoding/json"
"io/ioutil"
"net/http"
"net/url"
"github.com/vmware/harbor/src/common/models"
registry_error "github.com/vmware/harbor/src/common/utils/error"
"github.com/vmware/harbor/src/common/utils/registry"
)
const (
service = "harbor-registry"
)
// GetToken requests a token against the endpoint using credetial provided
func GetToken(endpoint string, insecure bool, credential Credential,
scopes []*Scope) (*models.Token, error) {
client := &http.Client{
Transport: registry.GetHTTPTransport(insecure),
}
scopesStr := []string{}
for _, scope := range scopes {
scopesStr = append(scopesStr, scope.string())
}
return getToken(client, credential, endpoint, service, scopesStr)
}
func getToken(client *http.Client, credential Credential, realm, service string,
scopes []string) (*models.Token, error) {
u, err := url.Parse(realm)
if err != nil {
return nil, err
}
query := u.Query()
query.Add("service", service)
for _, scope := range scopes {
query.Add("scope", scope)
}
u.RawQuery = query.Encode()
req, err := http.NewRequest(http.MethodGet, u.String(), nil)
if err != nil {
return nil, err
}
if credential != nil {
credential.AddAuthorization(req)
}
resp, err := client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
if resp.StatusCode != http.StatusOK {
return nil, &registry_error.Error{
StatusCode: resp.StatusCode,
Detail: string(data),
}
}
token := &models.Token{}
if err = json.Unmarshal(data, token); err != nil {
return nil, err
}
return token, nil
}

11
src/common/utils/registry/repository.go
View File

@ -22,6 +22,7 @@ import (
"io/ioutil"
"net/http"
"net/url"
"sort"
"strconv"
"strings"
// "time"
@ -106,11 +107,19 @@ func (r *Repository) ListTag() ([]string, error) {
if err := json.Unmarshal(b, &tagsResp); err != nil {
return tags, err
}
sort.Strings(tags)
tags = tagsResp.Tags
return tags, nil
} else if resp.StatusCode == http.StatusNotFound {
// TODO remove the logic if the bug of registry is fixed
// It's a workaround for a bug of registry: when listing tags of
// a repository which is being pushed, a "NAME_UNKNOWN" error will
// been returned, while the catalog API can list this repository.
return tags, nil
}
return tags, &registry_error.Error{
StatusCode: resp.StatusCode,
Detail: string(b),

72
src/jobservice/replication/transfer.go
View File

@ -27,7 +27,6 @@ import (
"github.com/docker/distribution"
"github.com/docker/distribution/manifest/schema1"
"github.com/docker/distribution/manifest/schema2"
"github.com/vmware/harbor/src/common/dao"
"github.com/vmware/harbor/src/common/models"
comutils "github.com/vmware/harbor/src/common/utils"
"github.com/vmware/harbor/src/common/utils/log"
@ -182,13 +181,13 @@ func (c *Checker) Enter() (string, error) {
}
func (c *Checker) enter() (string, error) {
project, err := dao.GetProjectByName(c.project)
project, err := getProject(c.project)
if err != nil {
c.logger.Errorf("an error occurred while getting project %s in DB: %v", c.project, err)
c.logger.Errorf("failed to get project %s from %s: %v", c.project, c.srcURL, err)
return "", err
}
err = c.createProject(project.Public)
err = c.createProject(project)
if err == nil {
c.logger.Infof("project %s is created on %s with user %s", c.project, c.dstURL, c.dstUsr)
return StatePullManifest, nil
@ -207,16 +206,61 @@ func (c *Checker) enter() (string, error) {
return "", err
}
func (c *Checker) createProject(public int) error {
project := struct {
ProjectName string `json:"project_name"`
Public int `json:"public"`
}{
ProjectName: c.project,
Public: public,
func getProject(name string) (*models.Project, error) {
req, err := http.NewRequest(http.MethodGet, buildProjectURL(), nil)
if err != nil {
return nil, err
}
data, err := json.Marshal(project)
req.URL.Query().Set("name", name)
req.URL.Query().Encode()
req.AddCookie(&http.Cookie{
Name: models.UISecretCookie,
Value: config.JobserviceSecret(),
})
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
list := []*models.Project{}
if err = json.Unmarshal(data, &list); err != nil {
return nil, err
}
var project *models.Project
for _, p := range list {
if p.Name == name {
project = p
break
}
}
if project == nil {
return nil, fmt.Errorf("project %s not found", name)
}
return project, nil
}
func (c *Checker) createProject(project *models.Project) error {
pro := &models.ProjectRequest{
Name: project.Name,
Public: project.Public,
EnableContentTrust: project.EnableContentTrust,
PreventVulnerableImagesFromRunning: project.PreventVulnerableImagesFromRunning,
PreventVulnerableImagesFromRunningSeverity: project.PreventVulnerableImagesFromRunningSeverity,
AutomaticallyScanImagesOnPush: project.AutomaticallyScanImagesOnPush,
}
data, err := json.Marshal(pro)
if err != nil {
return err
}
@ -263,6 +307,10 @@ func (c *Checker) createProject(public int) error {
c.project, c.dstURL, c.dstUsr, resp.StatusCode, string(message))
}
func buildProjectURL() string {
return strings.TrimRight(config.LocalUIURL(), "/") + "/api/projects/"
}
// ManifestPuller pulls the manifest of a tag. And if no tag needs to be pulled,
// the next state that state machine should enter is "finished".
type ManifestPuller struct {

41
src/jobservice/utils/utils.go
View File

@ -15,11 +15,8 @@
package utils
import (
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"github.com/vmware/harbor/src/common/models"
"github.com/vmware/harbor/src/common/utils/registry"
@ -64,38 +61,18 @@ func BuildBlobURL(endpoint, repository, digest string) string {
return fmt.Sprintf("%s/v2/%s/blobs/%s", endpoint, repository, digest)
}
//GetTokenForRepo is a temp solution for job handler to get a token for clair.
//GetTokenForRepo is used for job handler to get a token for clair.
func GetTokenForRepo(repository string) (string, error) {
u, err := url.Parse(config.InternalTokenServiceEndpoint())
if err != nil {
return "", err
}
q := u.Query()
q.Add("service", "harbor-registry")
q.Add("scope", fmt.Sprintf("repository:%s:pull", repository))
u.RawQuery = q.Encode()
r, err := http.NewRequest("GET", u.String(), nil)
if err != nil {
return "", err
}
c := &http.Cookie{Name: models.UISecretCookie, Value: config.JobserviceSecret()}
r.AddCookie(c)
client := &http.Client{}
resp, err := client.Do(r)
credentail := auth.NewCookieCredential(c)
token, err := auth.GetToken(config.InternalTokenServiceEndpoint(), true, credentail, []*auth.Scope{&auth.Scope{
Type: "repository",
Name: repository,
Actions: []string{"pull"},
}})
if err != nil {
return "", err
}
defer resp.Body.Close()
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return "", err
}
if resp.StatusCode != http.StatusOK {
return "", fmt.Errorf("Unexpected response from token service, code: %d, %s", resp.StatusCode, string(b))
}
tk := models.Token{}
if err := json.Unmarshal(b, &tk); err != nil {
return "", err
}
return tk.Token, nil
return token.Token, nil
}

216
src/ui/api/config.go
View File

@ -17,10 +17,11 @@ package api
import (
"fmt"
"net/http"
"strconv"
"reflect"
"github.com/vmware/harbor/src/common"
"github.com/vmware/harbor/src/common/dao"
"github.com/vmware/harbor/src/common/models"
"github.com/vmware/harbor/src/common/utils/log"
"github.com/vmware/harbor/src/ui/config"
)
@ -62,6 +63,34 @@ var (
common.CfgExpiration,
common.JobLogDir,
common.AdminInitialPassword,
common.ScanAllPolicy,
}
stringKeys = []string{
common.ExtEndpoint,
common.AUTHMode,
common.DatabaseType,
common.MySQLHost,
common.MySQLUsername,
common.MySQLPassword,
common.MySQLDatabase,
common.SQLiteFile,
common.LDAPURL,
common.LDAPSearchDN,
common.LDAPSearchPwd,
common.LDAPBaseDN,
common.LDAPUID,
common.LDAPFilter,
common.TokenServiceURL,
common.RegistryURL,
common.EmailHost,
common.EmailUsername,
common.EmailPassword,
common.EmailFrom,
common.EmailIdentity,
common.ProjectCreationRestriction,
common.JobLogDir,
common.AdminInitialPassword,
}
numKeys = []string{
@ -131,10 +160,10 @@ func (c *ConfigAPI) Get() {
// Put updates configurations
func (c *ConfigAPI) Put() {
m := map[string]string{}
m := map[string]interface{}{}
c.DecodeJSONReq(&m)
cfg := map[string]string{}
cfg := map[string]interface{}{}
for _, k := range validKeys {
if v, ok := m[k]; ok {
cfg[k] = v
@ -152,35 +181,7 @@ func (c *ConfigAPI) Put() {
c.CustomAbort(http.StatusBadRequest, err.Error())
}
if value, ok := cfg[common.AUTHMode]; ok {
mode, err := config.AuthMode()
if err != nil {
log.Errorf("failed to get auth mode: %v", err)
c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
if mode != value {
flag, err := authModeCanBeModified()
if err != nil {
log.Errorf("failed to determine whether auth mode can be modified: %v", err)
c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
if !flag {
c.CustomAbort(http.StatusBadRequest,
fmt.Sprintf("%s can not be modified as new users have been inserted into database",
common.AUTHMode))
}
}
}
result, err := convertForPut(cfg)
if err != nil {
log.Errorf("failed to convert configurations: %v", err)
c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
if err := config.Upload(result); err != nil {
if err := config.Upload(cfg); err != nil {
log.Errorf("failed to upload configurations: %v", err)
c.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
@ -199,18 +200,53 @@ func (c *ConfigAPI) Reset() {
}
}
func validateCfg(c map[string]string) (bool, error) {
isSysErr := false
func validateCfg(c map[string]interface{}) (bool, error) {
strMap := map[string]string{}
for _, k := range stringKeys {
if _, ok := c[k]; !ok {
continue
}
if _, ok := c[k].(string); !ok {
return false, fmt.Errorf("Invalid value type, expected string, key: %s, value: %v, type: %v", k, c[k], reflect.TypeOf(c[k]))
}
strMap[k] = c[k].(string)
}
numMap := map[string]int{}
for _, k := range numKeys {
if _, ok := c[k]; !ok {
continue
}
if _, ok := c[k].(float64); !ok {
return false, fmt.Errorf("Invalid value type, expected float64, key: %s, value: %v, type: %v", k, c[k], reflect.TypeOf(c[k]))
}
numMap[k] = int(c[k].(float64))
}
boolMap := map[string]bool{}
for _, k := range boolKeys {
if _, ok := c[k]; !ok {
continue
}
if _, ok := c[k].(bool); !ok {
return false, fmt.Errorf("Invalid value type, expected bool, key: %s, value: %v, type: %v", k, c[k], reflect.TypeOf(c[k]))
}
boolMap[k] = c[k].(bool)
}
mode, err := config.AuthMode()
if err != nil {
isSysErr = true
return isSysErr, err
return true, err
}
if value, ok := c[common.AUTHMode]; ok {
if value, ok := strMap[common.AUTHMode]; ok {
if value != common.DBAuth && value != common.LDAPAuth {
return isSysErr, fmt.Errorf("invalid %s, shoud be %s or %s", common.AUTHMode, common.DBAuth, common.LDAPAuth)
return false, fmt.Errorf("invalid %s, shoud be %s or %s", common.AUTHMode, common.DBAuth, common.LDAPAuth)
}
flag, err := authModeCanBeModified()
if err != nil {
return true, err
}
if mode != value && !flag {
return false, fmt.Errorf("%s can not be modified as new users have been inserted into database", common.AUTHMode)
}
mode = value
}
@ -218,123 +254,70 @@ func validateCfg(c map[string]string) (bool, error) {
if mode == common.LDAPAuth {
ldap, err := config.LDAP()
if err != nil {
isSysErr = true
return isSysErr, err
return true, err
}
if len(ldap.URL) == 0 {
if _, ok := c[common.LDAPURL]; !ok {
return isSysErr, fmt.Errorf("%s is missing", common.LDAPURL)
if _, ok := strMap[common.LDAPURL]; !ok {
return false, fmt.Errorf("%s is missing", common.LDAPURL)
}
}
if len(ldap.BaseDN) == 0 {
if _, ok := c[common.LDAPBaseDN]; !ok {
return isSysErr, fmt.Errorf("%s is missing", common.LDAPBaseDN)
if _, ok := strMap[common.LDAPBaseDN]; !ok {
return false, fmt.Errorf("%s is missing", common.LDAPBaseDN)
}
}
if len(ldap.UID) == 0 {
if _, ok := c[common.LDAPUID]; !ok {
return isSysErr, fmt.Errorf("%s is missing", common.LDAPUID)
if _, ok := strMap[common.LDAPUID]; !ok {
return false, fmt.Errorf("%s is missing", common.LDAPUID)
}
}
if ldap.Scope == 0 {
if _, ok := c[common.LDAPScope]; !ok {
return isSysErr, fmt.Errorf("%s is missing", common.LDAPScope)
if _, ok := numMap[common.LDAPScope]; !ok {
return false, fmt.Errorf("%s is missing", common.LDAPScope)
}
}
}
if ldapURL, ok := c[common.LDAPURL]; ok && len(ldapURL) == 0 {
return isSysErr, fmt.Errorf("%s is empty", common.LDAPURL)
if ldapURL, ok := strMap[common.LDAPURL]; ok && len(ldapURL) == 0 {
return false, fmt.Errorf("%s is empty", common.LDAPURL)
}
if baseDN, ok := c[common.LDAPBaseDN]; ok && len(baseDN) == 0 {
return isSysErr, fmt.Errorf("%s is empty", common.LDAPBaseDN)
if baseDN, ok := strMap[common.LDAPBaseDN]; ok && len(baseDN) == 0 {
return false, fmt.Errorf("%s is empty", common.LDAPBaseDN)
}
if uID, ok := c[common.LDAPUID]; ok && len(uID) == 0 {
return isSysErr, fmt.Errorf("%s is empty", common.LDAPUID)
if uID, ok := strMap[common.LDAPUID]; ok && len(uID) == 0 {
return false, fmt.Errorf("%s is empty", common.LDAPUID)
}
if scope, ok := c[common.LDAPScope]; ok &&
if scope, ok := numMap[common.LDAPScope]; ok &&
scope != common.LDAPScopeBase &&
scope != common.LDAPScopeOnelevel &&
scope != common.LDAPScopeSubtree {
return isSysErr, fmt.Errorf("invalid %s, should be %s, %s or %s",
return false, fmt.Errorf("invalid %s, should be %s, %s or %s",
common.LDAPScope,
common.LDAPScopeBase,
common.LDAPScopeOnelevel,
common.LDAPScopeSubtree)
}
for _, k := range boolKeys {
v, ok := c[k]
if !ok {
continue
for k, n := range numMap {
if n < 0 {
return false, fmt.Errorf("invalid %s: %d", k, n)
}
if v != "0" && v != "1" {
return isSysErr, fmt.Errorf("%s should be %s or %s",
k, "0", "1")
}
}
for _, k := range numKeys {
v, ok := c[k]
if !ok {
continue
}
n, err := strconv.Atoi(v)
if err != nil || n < 0 {
return isSysErr, fmt.Errorf("invalid %s: %s", k, v)
}
if (k == common.EmailPort ||
k == common.MySQLPort) && n > 65535 {
return isSysErr, fmt.Errorf("invalid %s: %s", k, v)
return false, fmt.Errorf("invalid %s: %d", k, n)
}
}
if crt, ok := c[common.ProjectCreationRestriction]; ok &&
if crt, ok := strMap[common.ProjectCreationRestriction]; ok &&
crt != common.ProCrtRestrEveryone &&
crt != common.ProCrtRestrAdmOnly {
return isSysErr, fmt.Errorf("invalid %s, should be %s or %s",
return false, fmt.Errorf("invalid %s, should be %s or %s",
common.ProjectCreationRestriction,
common.ProCrtRestrAdmOnly,
common.ProCrtRestrEveryone)
}
return isSysErr, nil
}
//convert map[string]string to map[string]interface{}
func convertForPut(m map[string]string) (map[string]interface{}, error) {
cfg := map[string]interface{}{}
for k, v := range m {
cfg[k] = v
}
for _, k := range numKeys {
if _, ok := cfg[k]; !ok {
continue
}
v, err := strconv.Atoi(cfg[k].(string))
if err != nil {
return nil, err
}
cfg[k] = v
}
for _, k := range boolKeys {
if _, ok := cfg[k]; !ok {
continue
}
cfg[k] = cfg[k] == "1"
}
return cfg, nil
return false, nil
}
// delete sensitive attrs and add editable field to every attr
@ -347,6 +330,9 @@ func convertForGet(cfg map[string]interface{}) (map[string]*value, error) {
}
}
if _, ok := cfg[common.ScanAllPolicy]; !ok {
cfg[common.ScanAllPolicy] = models.DefaultScanAllPolicy
}
for k, v := range cfg {
result[k] = &value{
Value: v,

4
src/ui/api/config_test.go
View File

@ -60,8 +60,8 @@ func TestPutConfig(t *testing.T) {
assert := assert.New(t)
apiTest := newHarborAPI()
cfg := map[string]string{
common.VerifyRemoteCert: "0",
cfg := map[string]interface{}{
common.VerifyRemoteCert: false,
}
code, err := apiTest.PutConfig(*admin, cfg)

2
src/ui/api/harborapi_test.go
View File

@ -1005,7 +1005,7 @@ func (a testapi) GetConfig(authInfo usrInfo) (int, map[string]*value, error) {
return code, cfg, err
}
func (a testapi) PutConfig(authInfo usrInfo, cfg map[string]string) (int, error) {
func (a testapi) PutConfig(authInfo usrInfo, cfg map[string]interface{}) (int, error) {
_sling := sling.New().Base(a.basePath).Put("/api/configurations").BodyJSON(cfg)
code, _, err := request(_sling, jsonAcceptHeader, authInfo)

47
src/ui/api/project.go
View File

@ -36,11 +36,6 @@ type ProjectAPI struct {
project *models.Project
}
type projectReq struct {
ProjectName string `json:"project_name"`
Public int `json:"public"`
}
const projectNameMaxLen int = 30
const projectNameMinLen int = 2
const restrictedNameChars = `[a-z0-9]+(?:[._-][a-z0-9]+)*`
@ -84,18 +79,24 @@ func (p *ProjectAPI) Post() {
p.HandleUnauthorized()
return
}
onlyAdmin, err := config.OnlyAdminCreateProject()
if err != nil {
log.Errorf("failed to determine whether only admin can create projects: %v", err)
p.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
var onlyAdmin bool
var err error
if config.WithAdmiral() {
onlyAdmin = true
} else {
onlyAdmin, err = config.OnlyAdminCreateProject()
if err != nil {
log.Errorf("failed to determine whether only admin can create projects: %v", err)
p.CustomAbort(http.StatusInternalServerError, http.StatusText(http.StatusInternalServerError))
}
}
if onlyAdmin && !p.SecurityCtx.IsSysAdmin() {
log.Errorf("Only sys admin can create project")
p.RenderError(http.StatusForbidden, "Only system admin can create project")
return
}
var pro projectReq
var pro *models.ProjectRequest
p.DecodeJSONReq(&pro)
err = validateProjectReq(pro)
if err != nil {
@ -104,10 +105,10 @@ func (p *ProjectAPI) Post() {
return
}
exist, err := p.ProjectMgr.Exist(pro.ProjectName)
exist, err := p.ProjectMgr.Exist(pro.Name)
if err != nil {
p.HandleInternalServerError(fmt.Sprintf("failed to check the existence of project %s: %v",
pro.ProjectName, err))
pro.Name, err))
return
}
if exist {
@ -116,9 +117,13 @@ func (p *ProjectAPI) Post() {
}
projectID, err := p.ProjectMgr.Create(&models.Project{
Name: pro.ProjectName,
Public: pro.Public,
OwnerName: p.SecurityCtx.GetUsername(),
Name: pro.Name,
Public: pro.Public,
OwnerName: p.SecurityCtx.GetUsername(),
EnableContentTrust: pro.EnableContentTrust,
PreventVulnerableImagesFromRunning: pro.PreventVulnerableImagesFromRunning,
PreventVulnerableImagesFromRunningSeverity: pro.PreventVulnerableImagesFromRunningSeverity,
AutomaticallyScanImagesOnPush: pro.AutomaticallyScanImagesOnPush,
})
if err != nil {
log.Errorf("Failed to add project, error: %v", err)
@ -136,7 +141,7 @@ func (p *ProjectAPI) Post() {
models.AccessLog{
Username: p.SecurityCtx.GetUsername(),
ProjectID: projectID,
RepoName: pro.ProjectName + "/",
RepoName: pro.Name + "/",
RepoTag: "N/A",
Operation: "create",
OpTime: time.Now(),
@ -349,7 +354,7 @@ func (p *ProjectAPI) ToggleProjectPublic() {
return
}
var req projectReq
var req *models.ProjectRequest
p.DecodeJSONReq(&req)
if req.Public != 0 && req.Public != 1 {
p.HandleBadRequest("public should be 0 or 1")
@ -431,9 +436,9 @@ func (p *ProjectAPI) Logs() {
}
// TODO move this to package models
func validateProjectReq(req projectReq) error {
pn := req.ProjectName
if isIllegalLength(req.ProjectName, projectNameMinLen, projectNameMaxLen) {
func validateProjectReq(req *models.ProjectRequest) error {
pn := req.Name
if isIllegalLength(req.Name, projectNameMinLen, projectNameMaxLen) {
return fmt.Errorf("Project name is illegal in length. (greater than 2 or less than 30)")
}
validProjectName := regexp.MustCompile(`^` + restrictedNameChars + `$`)

3
src/ui/api/replication_job.go
View File

@ -25,6 +25,7 @@ import (
"github.com/vmware/harbor/src/common/dao"
"github.com/vmware/harbor/src/common/models"
"github.com/vmware/harbor/src/common/utils/log"
"github.com/vmware/harbor/src/ui/utils"
)
// RepJobAPI handles request to /api/replicationJobs /api/replicationJobs/:id/log
@ -152,7 +153,7 @@ func (ra *RepJobAPI) GetLog() {
log.Errorf("failed to create a request: %v", err)
ra.CustomAbort(http.StatusInternalServerError, "")
}
addAuthentication(req)
utils.AddUISecret(req)
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {

82
src/ui/api/repository.go
View File

@ -19,7 +19,6 @@ import (
"fmt"
"io/ioutil"
"net/http"
"sort"
"time"
"github.com/docker/distribution/manifest/schema1"
@ -33,6 +32,7 @@ import (
"github.com/vmware/harbor/src/common/utils/notary"
"github.com/vmware/harbor/src/common/utils/registry"
"github.com/vmware/harbor/src/ui/config"
uiutils "github.com/vmware/harbor/src/ui/utils"
)
// RepositoryAPI handles request to /api/repositories /api/repositories/tags /api/repositories/manifests, the parm has to be put
@ -370,7 +370,7 @@ func (ra *RepositoryAPI) GetTags() {
ra.CustomAbort(http.StatusInternalServerError, "internal error")
}
tags, err := getSimpleTags(client)
tags, err := client.ListTag()
if err != nil {
ra.HandleInternalServerError(fmt.Sprintf("failed to get tag of %s: %v", repoName, err))
return
@ -485,31 +485,6 @@ func getV2Manifest(client *registry.Repository, tag string) (
return digest, manifest, config, nil
}
// return tag name list for the repository
func getSimpleTags(client *registry.Repository) ([]string, error) {
tags := []string{}
ts, err := client.ListTag()
if err != nil {
// TODO remove the logic if the bug of registry is fixed
// It's a workaround for a bug of registry: when listing tags of
// a repository which is being pushed, a "NAME_UNKNOWN" error will
// been returned, while the catalog API can list this repository.
if regErr, ok := err.(*registry_error.Error); ok &&
regErr.StatusCode == http.StatusNotFound {
return tags, nil
}
return nil, err
}
tags = append(tags, ts...)
sort.Strings(tags)
return tags, nil
}
// GetManifests returns the manifest of a tag
func (ra *RepositoryAPI) GetManifests() {
repoName := ra.GetString(":splat")
@ -615,8 +590,8 @@ func (ra *RepositoryAPI) initRepositoryClient(repoName string) (r *registry.Repo
return nil, err
}
return NewRepositoryClient(endpoint, true, ra.SecurityCtx.GetUsername(),
repoName, "repository", repoName, "pull", "push", "*")
return uiutils.NewRepositoryClientForUI(endpoint, true, ra.SecurityCtx.GetUsername(),
repoName, "pull", "push", "*")
}
//GetTopRepos returns the most populor repositories
@ -629,14 +604,14 @@ func (ra *RepositoryAPI) GetTopRepos() {
projectIDs := []int64{}
projects, err := ra.ProjectMgr.GetPublic()
if err != nil {
log.Errorf("failed to get the public projects: %v", err)
ra.HandleInternalServerError(fmt.Sprintf("failed to get public projects: %v", err))
return
}
if ra.SecurityCtx.IsAuthenticated() {
list, err := ra.ProjectMgr.GetByMember(ra.SecurityCtx.GetUsername())
if err != nil {
log.Errorf("failed to get projects which the user %s is a member of: %v",
ra.SecurityCtx.GetUsername(), err)
ra.HandleInternalServerError(fmt.Sprintf("failed to get projects which the user %s is a member of: %v",
ra.SecurityCtx.GetUsername(), err))
return
}
projects = append(projects, list...)
@ -726,7 +701,7 @@ func (ra *RepositoryAPI) ScanImage() {
ra.HandleForbidden(ra.SecurityCtx.GetUsername())
return
}
err = TriggerImageScan(repoName, tag)
err = uiutils.TriggerImageScan(repoName, tag)
//TODO better check existence
if err != nil {
log.Errorf("Error while calling job service to trigger image scan: %v", err)
@ -762,20 +737,47 @@ func (ra *RepositoryAPI) VulnerabilityDetails() {
ra.HandleForbidden(ra.SecurityCtx.GetUsername())
return
}
res := []*models.VulnerabilityItem{}
overview, err := dao.GetImgScanOverview(digest)
if err != nil {
ra.HandleInternalServerError(fmt.Sprintf("failed to get the scan overview, error: %v", err))
return
}
clairClient := clair.NewClient(config.ClairEndpoint(), nil)
log.Debugf("The key for getting details: %s", overview.DetailsKey)
details, err := clairClient.GetResult(overview.DetailsKey)
if err != nil {
ra.HandleInternalServerError(fmt.Sprintf("Failed to get scan details from Clair, error: %v", err))
if overview != nil && len(overview.DetailsKey) > 0 {
clairClient := clair.NewClient(config.ClairEndpoint(), nil)
log.Debugf("The key for getting details: %s", overview.DetailsKey)
details, err := clairClient.GetResult(overview.DetailsKey)
if err != nil {
ra.HandleInternalServerError(fmt.Sprintf("Failed to get scan details from Clair, error: %v", err))
return
}
res = transformVulnerabilities(details)
}
ra.Data["json"] = res
ra.ServeJSON()
}
// ScanAll handles the api to scan all images on Harbor.
func (ra *RepositoryAPI) ScanAll() {
if !config.WithClair() {
log.Warningf("Harbor is not deployed with Clair, it's not possible to scan images.")
ra.RenderError(http.StatusServiceUnavailable, "")
return
}
ra.Data["json"] = transformVulnerabilities(details)
ra.ServeJSON()
if !ra.SecurityCtx.IsAuthenticated() {
ra.HandleUnauthorized()
return
}
if !ra.SecurityCtx.IsSysAdmin() {
ra.HandleForbidden(ra.SecurityCtx.GetUsername())
return
}
if err := uiutils.ScanAllImages(); err != nil {
log.Errorf("Failed triggering scan all images, error: %v", err)
ra.HandleInternalServerError(fmt.Sprintf("Error: %v", err))
return
}
ra.Ctx.ResponseWriter.WriteHeader(http.StatusAccepted)
}
func getSignatures(repository, username string) (map[string]*notary.Target, error) {

7
src/ui/api/search.go
View File

@ -26,6 +26,7 @@ import (
"github.com/vmware/harbor/src/common/utils"
"github.com/vmware/harbor/src/common/utils/log"
"github.com/vmware/harbor/src/ui/config"
uiutils "github.com/vmware/harbor/src/ui/utils"
)
// SearchAPI handles requesst to /api/search
@ -157,13 +158,13 @@ func getTags(repository string) ([]string, error) {
return nil, err
}
client, err := NewRepositoryClient(url, true,
"admin", repository, "repository", repository, "pull")
client, err := uiutils.NewRepositoryClientForUI(url, true,
"admin", repository, "pull")
if err != nil {
return nil, err
}
tags, err := getSimpleTags(client)
tags, err := client.ListTag()
if err != nil {
return nil, err
}

37
src/ui/api/statistic.go
View File

@ -24,14 +24,14 @@ import (
)
const (
// MPC : count of my projects
MPC = "my_project_count"
// MRC : count of my repositories
MRC = "my_repo_count"
// PPC : count of public projects
PPC = "public_project_count"
// PRC : count of public repositories
PRC = "public_repo_count"
// PriPC : count of private projects
PriPC = "private_project_count"
// PriRC : count of private repositories
PriRC = "private_repo_count"
// PubPC : count of public projects
PubPC = "public_project_count"
// PubRC : count of public repositories
PubRC = "public_repo_count"
// TPC : total count of projects
TPC = "total_project_count"
// TRC : total count of repositories
@ -57,17 +57,17 @@ func (s *StatisticAPI) Prepare() {
// Get total projects and repos of the user
func (s *StatisticAPI) Get() {
statistic := map[string]int64{}
projects, err := s.ProjectMgr.GetPublic()
pubProjs, err := s.ProjectMgr.GetPublic()
if err != nil {
s.HandleInternalServerError(fmt.Sprintf(
"failed to get public projects: %v", err))
return
}
statistic[PPC] = (int64)(len(projects))
statistic[PubPC] = (int64)(len(pubProjs))
ids := []int64{}
for _, p := range projects {
for _, p := range pubProjs {
ids = append(ids, p.ProjectID)
}
n, err := dao.GetTotalOfRepositoriesByProject(ids, "")
@ -75,7 +75,7 @@ func (s *StatisticAPI) Get() {
log.Errorf("failed to get total of public repositories: %v", err)
s.CustomAbort(http.StatusInternalServerError, "")
}
statistic[PRC] = n
statistic[PubRC] = n
if s.SecurityCtx.IsSysAdmin() {
n, err := dao.GetTotalOfProjects(nil)
@ -83,19 +83,21 @@ func (s *StatisticAPI) Get() {
log.Errorf("failed to get total of projects: %v", err)
s.CustomAbort(http.StatusInternalServerError, "")
}
statistic[MPC] = n
statistic[TPC] = n
statistic[PriPC] = n - statistic[PubPC]
n, err = dao.GetTotalOfRepositories("")
if err != nil {
log.Errorf("failed to get total of repositories: %v", err)
s.CustomAbort(http.StatusInternalServerError, "")
}
statistic[MRC] = n
statistic[TRC] = n
statistic[PriRC] = n - statistic[PubRC]
} else {
value := false
projects, err := s.ProjectMgr.GetAll(&models.ProjectQueryParam{
Member: &models.Member{
Public: &value,
Member: &models.MemberQuery{
Name: s.username,
},
})
@ -104,7 +106,8 @@ func (s *StatisticAPI) Get() {
"failed to get projects of user %s: %v", s.username, err))
return
}
statistic[MPC] = (int64)(len(projects))
statistic[PriPC] = (int64)(len(projects))
ids := []int64{}
for _, p := range projects {
@ -118,7 +121,7 @@ func (s *StatisticAPI) Get() {
s.username, err))
return
}
statistic[MRC] = n
statistic[PriRC] = n
}
s.Data["json"] = statistic

10
src/ui/api/statistic_test.go
View File

@ -30,7 +30,7 @@ func TestStatisticGet(t *testing.T) {
//prepare for test
var priMyProjectCount, priMyRepoCount int32
var privateProjectCount, privateRepoCount int32
var priPublicProjectCount, priPublicRepoCount int32
var priTotalProjectCount, priTotalRepoCount int32
@ -53,8 +53,8 @@ func TestStatisticGet(t *testing.T) {
} else {
assert.Equal(httpStatusCode, int(200), "Case 2: Get status info with admin login. (200)")
//fmt.Println("pri status data %+v", result)
priMyProjectCount = result.MyProjectCount
priMyRepoCount = result.MyRepoCount
privateProjectCount = result.PrivateProjectCount
privateRepoCount = result.PrivateRepoCount
priPublicProjectCount = result.PublicProjectCount
priPublicRepoCount = result.PublicRepoCount
priTotalProjectCount = result.TotalProjectCount
@ -74,8 +74,8 @@ func TestStatisticGet(t *testing.T) {
t.Error("Error while get statistic information", err.Error())
t.Log(err)
} else {
assert.Equal(priMyProjectCount+1, result.MyProjectCount, "MyProjectCount should be +1")
assert.Equal(priMyRepoCount+1, result.MyRepoCount, "MyRepoCount should be +1")
assert.Equal(privateProjectCount+1, result.PrivateProjectCount, "PrivateProjectCount should be +1")
assert.Equal(privateRepoCount, result.PrivateRepoCount)
assert.Equal(priPublicProjectCount, result.PublicProjectCount, "PublicProjectCount should be equal")
assert.Equal(priPublicRepoCount+1, result.PublicRepoCount, "PublicRepoCount should be +1")
assert.Equal(priTotalProjectCount+1, result.TotalProjectCount, "TotalProCount should be +1")

88
src/ui/api/utils.go
View File

@ -18,7 +18,6 @@ import (
"bytes"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http"
"sort"
@ -34,6 +33,7 @@ import (
"github.com/vmware/harbor/src/common/utils/registry/auth"
"github.com/vmware/harbor/src/ui/config"
"github.com/vmware/harbor/src/ui/projectmanager"
uiutils "github.com/vmware/harbor/src/ui/utils"
)
//sysadmin has all privileges to all projects
@ -96,7 +96,7 @@ func TriggerReplication(policyID int64, repository string,
}
url := buildReplicationURL()
return requestAsUI("POST", url, bytes.NewBuffer(b), http.StatusOK)
return uiutils.RequestAsUI("POST", url, bytes.NewBuffer(b), http.StatusOK)
}
// TriggerReplicationByRepository triggers the replication according to the repository
@ -140,7 +140,7 @@ func postReplicationAction(policyID int64, acton string) error {
return err
}
addAuthentication(req)
uiutils.AddUISecret(req)
client := &http.Client{}
@ -163,15 +163,6 @@ func postReplicationAction(policyID int64, acton string) error {
return fmt.Errorf("%d %s", resp.StatusCode, string(b))
}
func addAuthentication(req *http.Request) {
if req != nil {
req.AddCookie(&http.Cookie{
Name: models.UISecretCookie,
Value: config.UISecret(),
})
}
}
// SyncRegistry syncs the repositories of registry with database.
func SyncRegistry(pm projectmanager.ProjectManager) error {
@ -291,8 +282,8 @@ func diffRepos(reposInRegistry []string, reposInDB []string,
if err != nil {
return needsAdd, needsDel, err
}
client, err := NewRepositoryClient(endpoint, true,
"admin", repoInR, "repository", repoInR, "pull")
client, err := uiutils.NewRepositoryClientForUI(endpoint, true,
"admin", repoInR, "pull")
if err != nil {
return needsAdd, needsDel, err
}
@ -316,8 +307,7 @@ func diffRepos(reposInRegistry []string, reposInDB []string,
if err != nil {
return needsAdd, needsDel, err
}
client, err := NewRepositoryClient(endpoint, true,
"admin", repoInR, "repository", repoInR, "pull")
client, err := uiutils.NewRepositoryClientForUI(endpoint, true, "admin", repoInR, "pull")
if err != nil {
return needsAdd, needsDel, err
}
@ -354,8 +344,7 @@ func diffRepos(reposInRegistry []string, reposInDB []string,
log.Errorf("failed to get registry URL: %v", err)
continue
}
client, err := NewRepositoryClient(endpoint, true,
"admin", repoInR, "repository", repoInR, "pull")
client, err := uiutils.NewRepositoryClientForUI(endpoint, true, "admin", repoInR, "pull")
if err != nil {
log.Errorf("failed to create repository client: %v", err)
continue
@ -411,11 +400,6 @@ func initRegistryClient() (r *registry.Registry, err error) {
return registryClient, nil
}
func buildScanJobURL() string {
url := config.InternalJobServiceURL()
return fmt.Sprintf("%s/api/jobs/scan", url)
}
func buildReplicationURL() string {
url := config.InternalJobServiceURL()
return fmt.Sprintf("%s/api/jobs/replication", url)
@ -482,64 +466,6 @@ func NewRegistryClient(endpoint string, insecure bool, username, scopeType, scop
return client, nil
}
// NewRepositoryClient ...
// TODO need a registry client which accept a raw token as param
func NewRepositoryClient(endpoint string, insecure bool, username, repository, scopeType, scopeName string,
scopeActions ...string) (*registry.Repository, error) {
authorizer := auth.NewRegistryUsernameTokenAuthorizer(username, scopeType, scopeName, scopeActions...)
store, err := auth.NewAuthorizerStore(endpoint, insecure, authorizer)
if err != nil {
return nil, err
}
client, err := registry.NewRepositoryWithModifiers(repository, endpoint, insecure, store)
if err != nil {
return nil, err
}
return client, nil
}
// TriggerImageScan triggers an image scan job on jobservice.
func TriggerImageScan(repository string, tag string) error {
data := &models.ImageScanReq{
Repo: repository,
Tag: tag,
}
b, err := json.Marshal(&data)
if err != nil {
return err
}
url := buildScanJobURL()
return requestAsUI("POST", url, bytes.NewBuffer(b), http.StatusOK)
}
// Do not use this when you want to handle the response
// TODO: add a response handler to replace expectSC *when needed*
func requestAsUI(method, url string, body io.Reader, expectSC int) error {
req, err := http.NewRequest(method, url, body)
if err != nil {
return err
}
addAuthentication(req)
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if resp.StatusCode != expectSC {
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
return fmt.Errorf("Unexpected status code: %d, text: %s", resp.StatusCode, string(b))
}
return nil
}
// transformVulnerabilities transforms the returned value of Clair API to a list of VulnerabilityItem
func transformVulnerabilities(layerWithVuln *models.ClairLayerEnvelope) []*models.VulnerabilityItem {
res := []*models.VulnerabilityItem{}

57
src/ui/config/config.go
View File

@ -15,7 +15,10 @@
package config
import (
"crypto/tls"
"encoding/json"
"fmt"
"net/http"
"os"
"strings"
@ -32,8 +35,9 @@ import (
)
const (
defaultKeyPath string = "/etc/ui/key"
secretCookieName string = "secret"
defaultKeyPath string = "/etc/ui/key"
defaultTokenFilePath string = "/etc/ui/service_token"
secretCookieName string = "secret"
)
var (
@ -45,6 +49,9 @@ var (
GlobalProjectMgr projectmanager.ProjectManager
mg *comcfg.Manager
keyProvider comcfg.KeyProvider
// AdmiralClient is initialized only under integration deploy mode
// and can be passed to project manager as a parameter
AdmiralClient *http.Client
)
// Init configurations
@ -104,9 +111,25 @@ func initProjectManager() {
}
// integration with admiral
// TODO create project manager based on pms using service account
log.Info("initializing the project manager based on PMS...")
GlobalProjectMgr = pms.NewProjectManager(AdmiralEndpoint(), "")
// TODO read ca/cert file and pass it to the TLS config
AdmiralClient = &http.Client{
Transport: &http.Transport{
TLSClientConfig: &tls.Config{
InsecureSkipVerify: true,
},
},
}
path := os.Getenv("SERVICE_TOKEN_FILE_PATH")
if len(path) == 0 {
path = defaultTokenFilePath
}
log.Infof("service token file path: %s", path)
GlobalProjectMgr = pms.NewProjectManager(AdmiralClient,
AdmiralEndpoint(), &pms.FileTokenReader{
Path: path,
})
}
// Load configurations
@ -339,7 +362,7 @@ func ClairEndpoint() string {
func AdmiralEndpoint() string {
cfg, err := mg.Get()
if err != nil {
log.Errorf("Failed to get configuration, will return empty string as admiral's endpoint")
log.Errorf("Failed to get configuration, will return empty string as admiral's endpoint, error: %v", err)
return ""
}
@ -349,6 +372,30 @@ func AdmiralEndpoint() string {
return cfg[common.AdmiralEndpoint].(string)
}
// ScanAllPolicy returns the policy which controls the scan all.
func ScanAllPolicy() models.ScanAllPolicy {
var res models.ScanAllPolicy
cfg, err := mg.Get()
if err != nil {
log.Errorf("Failed to get configuration, will return default scan all policy, error: %v", err)
return models.DefaultScanAllPolicy
}
v, ok := cfg[common.ScanAllPolicy]
if !ok {
return models.DefaultScanAllPolicy
}
b, err := json.Marshal(v)
if err != nil {
log.Errorf("Failed to Marshal the value in configuration for Scan All policy, error: %v, returning the default policy", err)
return models.DefaultScanAllPolicy
}
if err := json.Unmarshal(b, &res); err != nil {
log.Errorf("Failed to unmarshal the value in configuration for Scan All policy, error: %v, returning the default policy", err)
return models.DefaultScanAllPolicy
}
return res
}
// WithAdmiral returns a bool to indicate if Harbor's deployed with admiral.
func WithAdmiral() bool {
return len(AdmiralEndpoint()) > 0

5
src/ui/config/config_test.go
View File

@ -155,4 +155,9 @@ func TestConfig(t *testing.T) {
if mode != "db_auth" {
t.Errorf("unexpected mode: %s != %s", mode, "db_auth")
}
if s := ScanAllPolicy(); s.Type != "daily" {
t.Errorf("unexpected scan all policy %v", s)
}
}

69
src/ui/filter/security.go
View File

@ -52,6 +52,7 @@ func Init() {
reqCtxModifiers = []ReqCtxModifier{
&secretReqCtxModifier{config.SecretStore},
&tokenReqCtxModifier{},
&basicAuthReqCtxModifier{},
&unauthorizedReqCtxModifier{}}
return
}
@ -123,7 +124,37 @@ func (b *basicAuthReqCtxModifier) Modify(ctx *beegoctx.Context) bool {
if !ok {
return false
}
log.Debug("got user information via basic auth")
// integration with admiral
if config.WithAdmiral() {
// Can't get a token from Admiral's login API, we can only
// create a project manager with the token of the solution user.
// That way may cause some wrong permission promotion in some API
// calls, so we just handle the requests which are necessary
if !filterReq(ctx.Request) {
log.Debugf("basic auth is not supported for request %s %s, skip",
ctx.Request.Method, ctx.Request.URL.Path)
return false
}
authCtx, err := authcontext.Login(config.AdmiralClient,
config.AdmiralEndpoint(), username, password)
if err != nil {
log.Errorf("failed to authenticate %s: %v", username, err)
return false
}
log.Debug("using global project manager...")
pm := config.GlobalProjectMgr
log.Debug("creating admiral security context...")
securCtx := admiral.NewSecurityContext(authCtx, pm)
setSecurCtxAndPM(ctx.Request, securCtx, pm)
return true
}
// standalone
user, err := auth.Login(models.AuthModel{
Principal: username,
Password: password,
@ -133,30 +164,27 @@ func (b *basicAuthReqCtxModifier) Modify(ctx *beegoctx.Context) bool {
return false
}
if user == nil {
log.Debug("basic auth user is nil")
return false
}
var securCtx security.Context
var pm projectmanager.ProjectManager
log.Debug("got user information via basic auth")
if config.WithAdmiral() {
// integration with admiral
// we can add logic here to support basic auth in integration mode
log.Debug("basic auth isn't supported in integration mode")
return false
}
// standalone
log.Debug("using local database project manager")
pm = config.GlobalProjectMgr
pm := config.GlobalProjectMgr
log.Debug("creating local database security context...")
securCtx = local.NewSecurityContext(user, pm)
securCtx := local.NewSecurityContext(user, pm)
setSecurCtxAndPM(ctx.Request, securCtx, pm)
return true
}
func filterReq(req *http.Request) bool {
path := req.URL.Path
if path == "/api/projects" && req.Method == http.MethodPost ||
path == "/service/token" && req.Method == http.MethodGet {
return true
}
return false
}
type sessionReqCtxModifier struct{}
func (s *sessionReqCtxModifier) Modify(ctx *beegoctx.Context) bool {
@ -194,14 +222,18 @@ func (t *tokenReqCtxModifier) Modify(ctx *beegoctx.Context) bool {
log.Debug("got token from request")
authContext, err := authcontext.GetByToken(token)
authContext, err := authcontext.GetAuthCtx(config.AdmiralClient,
config.AdmiralEndpoint(), token)
if err != nil {
log.Errorf("failed to get auth context: %v", err)
return false
}
log.Debug("creating PMS project manager...")
pm := pms.NewProjectManager(config.AdmiralEndpoint(), token)
pm := pms.NewProjectManager(config.AdmiralClient,
config.AdmiralEndpoint(), &pms.RawTokenReader{
Token: token,
})
log.Debug("creating admiral security context...")
securCtx := admiral.NewSecurityContext(authContext, pm)
setSecurCtxAndPM(ctx.Request, securCtx, pm)
@ -220,7 +252,8 @@ func (u *unauthorizedReqCtxModifier) Modify(ctx *beegoctx.Context) bool {
if config.WithAdmiral() {
// integration with admiral
log.Debug("creating PMS project manager...")
pm = pms.NewProjectManager(config.AdmiralEndpoint(), "")
pm = pms.NewProjectManager(config.AdmiralClient,
config.AdmiralEndpoint(), nil)
log.Debug("creating admiral security context...")
securCtx = admiral.NewSecurityContext(nil, pm)
} else {

6
src/ui/projectmanager/db/pm.go
View File

@ -115,8 +115,12 @@ func (p *ProjectManager) GetPublic() ([]*models.Project, error) {
// GetByMember returns all projects which the user is a member of
func (p *ProjectManager) GetByMember(username string) (
[]*models.Project, error) {
if len(username) == 0 {
return []*models.Project{}, nil
}
return p.GetAll(&models.ProjectQueryParam{
Member: &models.Member{
Member: &models.MemberQuery{
Name: username,
},
})

8
src/ui/projectmanager/db/pm_test.go
View File

@ -153,7 +153,13 @@ func TestGetPublic(t *testing.T) {
func TestGetByMember(t *testing.T) {
pm := &ProjectManager{}
projects, err := pm.GetByMember("admin")
// empty username
projects, err := pm.GetByMember("")
assert.Nil(t, err)
assert.Equal(t, 0, len(projects))
//non-empty username
projects, err = pm.GetByMember("admin")
assert.Nil(t, err)
assert.NotEqual(t, 0, len(projects))
}

137
src/ui/projectmanager/pms/pm.go
View File

@ -24,22 +24,20 @@ import (
"net/http"
"strconv"
"strings"
"time"
"github.com/vmware/harbor/src/common"
"github.com/vmware/harbor/src/common/models"
"github.com/vmware/harbor/src/common/security/authcontext"
er "github.com/vmware/harbor/src/common/utils/error"
"github.com/vmware/harbor/src/common/utils/log"
)
var transport = &http.Transport{}
// ProjectManager implements projectmanager.ProjecdtManager interface
// base on project management service
type ProjectManager struct {
endpoint string
token string
client *http.Client
client *http.Client
endpoint string
tokenReader TokenReader
}
type user struct {
@ -54,17 +52,16 @@ type project struct {
CustomProperties map[string]string `json:"customProperties"`
Administrators []*user `json:"administrators"`
Developers []*user `json:"members"`
Guests []*user `json:"guests"` // TODO the json name needs to be modified according to the API
Guests []*user `json:"viewers"`
}
// NewProjectManager returns an instance of ProjectManager
func NewProjectManager(endpoint, token string) *ProjectManager {
func NewProjectManager(client *http.Client, endpoint string,
tokenReader TokenReader) *ProjectManager {
return &ProjectManager{
endpoint: strings.TrimRight(endpoint, "/"),
token: token,
client: &http.Client{
Transport: transport,
},
client: client,
endpoint: strings.TrimRight(endpoint, "/"),
tokenReader: tokenReader,
}
}
@ -80,7 +77,7 @@ func (p *ProjectManager) Get(projectIDOrName interface{}) (*models.Project, erro
func (p *ProjectManager) get(projectIDOrName interface{}) (*project, error) {
m := map[string]string{}
if id, ok := projectIDOrName.(int64); ok {
m["customProperties.__harborId"] = strconv.FormatInt(id, 10)
m["customProperties.__projectIndex"] = strconv.FormatInt(id, 10)
} else if name, ok := projectIDOrName.(string); ok {
m["name"] = name
} else {
@ -117,6 +114,10 @@ func (p *ProjectManager) filter(m map[string]string) ([]*project, error) {
query += fmt.Sprintf("$filter=%s eq '%s'", k, v)
}
if len(query) == 0 {
query = "?expand=true"
}
path := "/projects" + query
data, err := p.send(http.MethodGet, path, nil)
if err != nil {
@ -129,7 +130,6 @@ func (p *ProjectManager) filter(m map[string]string) ([]*project, error) {
// parse the response of GET /projects?xxx to project list
func parse(b []byte) ([]*project, error) {
documents := &struct {
//TotalCount int64 `json:"totalCount"`
//DocumentCount int64 `json:"documentCount"`
Projects map[string]*project `json:"documents"`
}{}
@ -158,14 +158,14 @@ func convert(p *project) (*models.Project, error) {
project.Public = 1
}
value := p.CustomProperties["__harborId"]
value := p.CustomProperties["__projectIndex"]
if len(value) == 0 {
return nil, fmt.Errorf("property __harborId is null")
return nil, fmt.Errorf("property __projectIndex is null")
}
id, err := strconv.ParseInt(value, 10, 64)
if err != nil {
return nil, fmt.Errorf("failed to parse __harborId %s to int64: %v", value, err)
return nil, fmt.Errorf("failed to parse __projectIndex %s to int64: %v", value, err)
}
project.ProjectID = id
@ -227,8 +227,11 @@ func (p *ProjectManager) Exist(projectIDOrName interface{}) (bool, error) {
return project != nil, nil
}
// GetRoles ...
// TODO empty this method after implementing security context with auth context
// GetRoles gets roles that the user has to the project
// This method is used in GET /projects API.
// Jobservice calls GET /projects API to get information of source
// project when trying to replicate the project. There is no auth
// context in this use case, so the method is needed.
func (p *ProjectManager) GetRoles(username string, projectIDOrName interface{}) ([]int, error) {
if len(username) == 0 || projectIDOrName == nil {
return nil, nil
@ -292,31 +295,30 @@ func (p *ProjectManager) getIDbyHarborIDOrName(projectIDOrName interface{}) (str
// GetPublic ...
func (p *ProjectManager) GetPublic() ([]*models.Project, error) {
m := map[string]string{
"isPublic": "true",
}
projects, err := p.filter(m)
if err != nil {
return nil, err
}
list := []*models.Project{}
for _, p := range projects {
project, err := convert(p)
if err != nil {
return nil, err
}
list = append(list, project)
}
return list, nil
t := true
return p.GetAll(&models.ProjectQueryParam{
Public: &t,
})
}
// GetByMember ...
func (p *ProjectManager) GetByMember(username string) ([]*models.Project, error) {
// TODO add implement
return nil, nil
projects := []*models.Project{}
ctx, err := authcontext.GetAuthCtxOfUser(p.client, p.endpoint, p.getToken(), username)
if err != nil {
return projects, err
}
names := ctx.GetMyProjects()
for _, name := range names {
project, err := p.Get(name)
if err != nil {
return projects, err
}
projects = append(projects, project)
}
return projects, nil
}
// Create ...
@ -331,9 +333,6 @@ func (p *ProjectManager) Create(pro *models.Project) (int64, error) {
proj.CustomProperties["__preventVulnerableImagesFromRunningSeverity"] = pro.PreventVulnerableImagesFromRunningSeverity
proj.CustomProperties["__automaticallyScanImagesOnPush"] = strconv.FormatBool(pro.AutomaticallyScanImagesOnPush)
// TODO remove the logic if Admiral generates the harborId
proj.CustomProperties["__harborId"] = strconv.FormatInt(time.Now().UnixNano(), 10)
data, err := json.Marshal(proj)
if err != nil {
return 0, err
@ -375,19 +374,42 @@ func (p *ProjectManager) Update(projectIDOrName interface{}, project *models.Pro
// GetAll ...
func (p *ProjectManager) GetAll(query *models.ProjectQueryParam, base ...*models.BaseProjectCollection) ([]*models.Project, error) {
return nil, errors.New("get all projects is unsupported")
m := map[string]string{}
if query != nil {
if len(query.Name) > 0 {
m["name"] = query.Name
}
if query.Public != nil {
m["isPublic"] = strconv.FormatBool(*query.Public)
}
}
projects, err := p.filter(m)
if err != nil {
return nil, err
}
list := []*models.Project{}
for _, p := range projects {
project, err := convert(p)
if err != nil {
return nil, err
}
list = append(list, project)
}
return list, nil
}
// GetTotal ...
func (p *ProjectManager) GetTotal(query *models.ProjectQueryParam, base ...*models.BaseProjectCollection) (int64, error) {
return 0, errors.New("get total of projects is unsupported")
projects, err := p.GetAll(query)
return int64(len(projects)), err
}
// GetHasReadPerm returns all projects that user has read perm to
// TODO maybe can be removed as search isn't implemented in integration mode
// GetHasReadPerm ...
func (p *ProjectManager) GetHasReadPerm(username ...string) ([]*models.Project, error) {
// TODO add implement
return nil, nil
return nil, errors.New("GetHasReadPerm is unsupported")
}
func (p *ProjectManager) send(method, path string, body io.Reader) ([]byte, error) {
@ -396,7 +418,7 @@ func (p *ProjectManager) send(method, path string, body io.Reader) ([]byte, erro
return nil, err
}
req.Header.Add("x-xenon-auth-token", p.token)
req.Header.Add("x-xenon-auth-token", p.getToken())
url := req.URL.String()
@ -423,3 +445,16 @@ func (p *ProjectManager) send(method, path string, body io.Reader) ([]byte, erro
return b, nil
}
func (p *ProjectManager) getToken() string {
if p.tokenReader == nil {
return ""
}
token, err := p.tokenReader.ReadToken()
if err != nil {
token = ""
log.Errorf("failed to read token: %v", err)
}
return token
}

172
src/ui/projectmanager/pms/pm_test.go
View File

@ -15,6 +15,7 @@
package pms
import (
"net/http"
"sort"
"testing"
@ -24,8 +25,11 @@ import (
)
var (
endpoint = "http://127.0.0.1:8282"
token = ""
client = http.DefaultClient
endpoint = "http://127.0.0.1:8282"
tokenReader = &RawTokenReader{
Token: "",
}
)
func TestConvert(t *testing.T) {
@ -34,16 +38,16 @@ func TestConvert(t *testing.T) {
assert.Nil(t, err)
assert.Nil(t, pro)
//project without property __harborId
//project without property __projectIndex
p := &project{}
pro, err = convert(p)
assert.NotNil(t, err)
assert.Nil(t, pro)
//project with invalid __harborId
//project with invalid __projectIndex
p = &project{
CustomProperties: map[string]string{
"__harborId": "invalid_value",
"__projectIndex": "invalid_value",
},
}
pro, err = convert(p)
@ -85,7 +89,7 @@ func TestConvert(t *testing.T) {
Name: "test",
Public: true,
CustomProperties: map[string]string{
"__harborId": "1",
"__projectIndex": "1",
"__enableContentTrust": "true",
"__preventVulnerableImagesFromRunning": "true",
"__preventVulnerableImagesFromRunningSeverity": "medium",
@ -118,7 +122,7 @@ func TestParse(t *testing.T) {
"id": "41427587-70e9-4671-9a9e-b9def0a07bb7",
"name": "project02",
"customProperties": {
"__harborId": "2",
"__projectIndex": "2",
"__enableContentTrust": "true",
"__preventVulnerableImagesFromRunning": "true",
"__preventVulnerableImagesFromRunningSeverity": "medium",
@ -140,7 +144,7 @@ func TestParse(t *testing.T) {
"id": "default-project",
"name": "default-project",
"customProperties": {
"__harborId": "2",
"__projectIndex": "2",
"__enableContentTrust": "true",
"__preventVulnerableImagesFromRunning": "true",
"__preventVulnerableImagesFromRunningSeverity": "medium",
@ -177,17 +181,13 @@ func TestParse(t *testing.T) {
}
func TestGet(t *testing.T) {
pm := NewProjectManager(endpoint, token)
pm := NewProjectManager(client, endpoint, tokenReader)
name := "project_for_test_get"
id, err := pm.Create(&models.Project{
Name: name,
})
require.Nil(t, err)
defer func(id int64) {
if err := pm.Delete(id); err != nil {
require.Nil(t, err)
}
}(id)
defer delete(t, id)
// get by invalid input type
_, err = pm.Get([]string{})
@ -215,7 +215,7 @@ func TestGet(t *testing.T) {
}
func TestIsPublic(t *testing.T) {
pm := NewProjectManager(endpoint, token)
pm := NewProjectManager(client, endpoint, tokenReader)
// invalid input type
public, err := pm.IsPublic([]string{})
@ -234,11 +234,7 @@ func TestIsPublic(t *testing.T) {
Public: 1,
})
require.Nil(t, err)
defer func(id int64) {
if err := pm.Delete(id); err != nil {
require.Nil(t, err)
}
}(id)
defer delete(t, id)
public, err = pm.IsPublic(id)
assert.Nil(t, err)
@ -255,11 +251,7 @@ func TestIsPublic(t *testing.T) {
Public: 0,
})
require.Nil(t, err)
defer func(id int64) {
if err := pm.Delete(id); err != nil {
require.Nil(t, err)
}
}(id)
defer delete(t, id)
public, err = pm.IsPublic(id)
assert.Nil(t, err)
@ -271,7 +263,7 @@ func TestIsPublic(t *testing.T) {
}
func TestExist(t *testing.T) {
pm := NewProjectManager(endpoint, token)
pm := NewProjectManager(client, endpoint, tokenReader)
// invalid input type
exist, err := pm.Exist([]string{})
@ -289,11 +281,7 @@ func TestExist(t *testing.T) {
Name: name,
})
require.Nil(t, err)
defer func(id int64) {
if err := pm.Delete(id); err != nil {
require.Nil(t, err)
}
}(id)
defer delete(t, id)
exist, err = pm.Exist(id)
assert.Nil(t, err)
@ -305,7 +293,7 @@ func TestExist(t *testing.T) {
}
func TestGetRoles(t *testing.T) {
pm := NewProjectManager(endpoint, token)
pm := NewProjectManager(client, endpoint, tokenReader)
// nil username, nil project
roles, err := pm.GetRoles("", nil)
@ -322,11 +310,7 @@ func TestGetRoles(t *testing.T) {
Name: name,
})
require.Nil(t, err)
defer func(id int64) {
if err := pm.Delete(id); err != nil {
require.Nil(t, err)
}
}(id)
defer delete(t, id)
roles, err = pm.GetRoles("user01", id)
assert.Nil(t, err)
@ -336,7 +320,7 @@ func TestGetRoles(t *testing.T) {
}
func TestGetPublic(t *testing.T) {
pm := NewProjectManager(endpoint, token)
pm := NewProjectManager(client, endpoint, tokenReader)
projects, err := pm.GetPublic()
assert.Nil(t, nil)
@ -348,11 +332,7 @@ func TestGetPublic(t *testing.T) {
Public: 1,
})
require.Nil(t, err)
defer func(id int64) {
if err := pm.Delete(id); err != nil {
require.Nil(t, err)
}
}(id)
defer delete(t, id)
projects, err = pm.GetPublic()
assert.Nil(t, nil)
@ -374,7 +354,7 @@ func TestGetByMember(t *testing.T) {
}
func TestCreate(t *testing.T) {
pm := NewProjectManager(endpoint, token)
pm := NewProjectManager(client, endpoint, tokenReader)
name := "project_for_test_create"
id, err := pm.Create(&models.Project{
@ -386,11 +366,7 @@ func TestCreate(t *testing.T) {
AutomaticallyScanImagesOnPush: true,
})
require.Nil(t, err)
defer func(id int64) {
if err := pm.Delete(id); err != nil {
require.Nil(t, err)
}
}(id)
defer delete(t, id)
project, err := pm.Get(id)
assert.Nil(t, err)
@ -403,7 +379,7 @@ func TestCreate(t *testing.T) {
}
func TestDelete(t *testing.T) {
pm := NewProjectManager(endpoint, token)
pm := NewProjectManager(client, endpoint, tokenReader)
// non-exist project
err := pm.Delete(int64(0))
@ -429,24 +405,102 @@ func TestDelete(t *testing.T) {
}
func TestUpdate(t *testing.T) {
pm := NewProjectManager(endpoint, token)
pm := NewProjectManager(client, endpoint, tokenReader)
err := pm.Update(nil, nil)
assert.NotNil(t, err)
}
func TestGetAll(t *testing.T) {
pm := NewProjectManager(endpoint, token)
_, err := pm.GetAll(nil)
assert.NotNil(t, err)
pm := NewProjectManager(client, endpoint, tokenReader)
name1 := "project_for_test_get_all_01"
id1, err := pm.Create(&models.Project{
Name: name1,
})
require.Nil(t, err)
defer delete(t, id1)
name2 := "project_for_test_get_all_02"
id2, err := pm.Create(&models.Project{
Name: name2,
Public: 1,
})
require.Nil(t, err)
defer delete(t, id2)
// no filter
projects, err := pm.GetAll(nil)
require.Nil(t, err)
found1 := false
found2 := false
for _, project := range projects {
if project.ProjectID == id1 {
found1 = true
}
if project.ProjectID == id2 {
found2 = true
}
}
assert.True(t, found1)
assert.True(t, found2)
// filter by name
projects, err = pm.GetAll(&models.ProjectQueryParam{
Name: name1,
})
require.Nil(t, err)
found1 = false
for _, project := range projects {
if project.ProjectID == id1 {
found1 = true
break
}
}
assert.True(t, found1)
// filter by public
value := true
projects, err = pm.GetAll(&models.ProjectQueryParam{
Public: &value,
})
require.Nil(t, err)
found2 = false
for _, project := range projects {
if project.ProjectID == id2 {
found2 = true
break
}
}
assert.True(t, found2)
}
func TestGetTotal(t *testing.T) {
pm := NewProjectManager(endpoint, token)
_, err := pm.GetTotal(nil)
pm := NewProjectManager(client, endpoint, tokenReader)
total1, err := pm.GetTotal(nil)
require.Nil(t, err)
name := "project_for_test_get_total"
id, err := pm.Create(&models.Project{
Name: name,
})
require.Nil(t, err)
defer delete(t, id)
total2, err := pm.GetTotal(nil)
require.Nil(t, err)
assert.Equal(t, total1+1, total2)
}
func TestGetHasReadPerm(t *testing.T) {
pm := NewProjectManager(client, endpoint, tokenReader)
_, err := pm.GetHasReadPerm()
assert.NotNil(t, err)
}
// TODO add test case
func TestGetHasReadPerm(t *testing.T) {
func delete(t *testing.T, id int64) {
pm := NewProjectManager(client, endpoint, tokenReader)
if err := pm.Delete(id); err != nil {
t.Logf("failed to delete project %d: %v", id, err)
}
}

84
src/ui/projectmanager/pms/token.go Normal file
View File

@ -0,0 +1,84 @@
// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pms
import (
"bufio"
"fmt"
"io"
"os"
"strings"
)
const (
key = "access_token"
)
// TokenReader is an interface used to wrap the way how to get token
type TokenReader interface {
// ReadToken reads token
ReadToken() (string, error)
}
// RawTokenReader just returns the token contained by field Token
type RawTokenReader struct {
Token string
}
// ReadToken ...
func (r *RawTokenReader) ReadToken() (string, error) {
return r.Token, nil
}
// FileTokenReader reads token from file
type FileTokenReader struct {
Path string
}
// ReadToken ...
func (f *FileTokenReader) ReadToken() (string, error) {
file, err := os.Open(f.Path)
if err != nil {
return "", err
}
defer file.Close()
return readToken(file)
}
func readToken(reader io.Reader) (string, error) {
if reader == nil {
return "", fmt.Errorf("reader is nil")
}
r := bufio.NewReader(reader)
for {
line, _, err := r.ReadLine()
if err != nil {
if err == io.EOF {
err = fmt.Errorf("%s not found", key)
}
return "", err
}
strs := strings.SplitN(string(line), "=", 2)
if len(strs) != 2 {
continue
}
if strs[0] == key {
return strs[1], nil
}
}
}

98
src/ui/projectmanager/pms/token_test.go Normal file
View File

@ -0,0 +1,98 @@
// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package pms
import (
"bytes"
"io/ioutil"
"os"
"testing"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
)
func TestRawTokenReader(t *testing.T) {
raw := "token"
reader := &RawTokenReader{
Token: raw,
}
token, err := reader.ReadToken()
require.Nil(t, err)
assert.Equal(t, raw, token)
}
func TestReadToken(t *testing.T) {
// nil reader
_, err := readToken(nil)
assert.NotNil(t, err)
// empty
reader := bytes.NewReader([]byte{})
_, err = readToken(reader)
assert.NotNil(t, err)
// contains no "access_token"
content := "key1=value\nkey2=value2"
reader = bytes.NewReader([]byte(content))
_, err = readToken(reader)
assert.NotNil(t, err)
// contains "access_token" but no "="
content = "access_token value\nkey2=value2"
reader = bytes.NewReader([]byte(content))
_, err = readToken(reader)
assert.NotNil(t, err)
// contains "access_token" and "=", but no value
content = "access_token=\nkey2=value2"
reader = bytes.NewReader([]byte(content))
token, err := readToken(reader)
require.Nil(t, err)
assert.Len(t, token, 0)
// valid "access_token"
content = "access_token=token\nkey2=value2"
reader = bytes.NewReader([]byte(content))
token, err = readToken(reader)
require.Nil(t, err)
assert.Equal(t, "token", token)
}
func TestFileTokenReader(t *testing.T) {
// file not exist
path := "/tmp/not_exist_file"
reader := &FileTokenReader{
Path: path,
}
_, err := reader.ReadToken()
assert.NotNil(t, err)
// file exist
path = "/tmp/exist_file"
err = ioutil.WriteFile(path, []byte("access_token=token"), 0x0666)
require.Nil(t, err)
defer os.Remove(path)
reader = &FileTokenReader{
Path: path,
}
token, err := reader.ReadToken()
require.Nil(t, err)
assert.Equal(t, "token", token)
}

71
src/ui/proxy/interceptor_test.go
View File

@ -3,6 +3,7 @@ package proxy
import (
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"github.com/vmware/harbor/src/adminserver/client"
"github.com/vmware/harbor/src/common"
"github.com/vmware/harbor/src/common/models"
notarytest "github.com/vmware/harbor/src/common/utils/notary/test"
@ -19,6 +20,7 @@ import (
var endpoint = "10.117.4.142"
var notaryServer *httptest.Server
var adminServer *httptest.Server
var adminserverClient client.Client
var admiralEndpoint = "http://127.0.0.1:8282"
var token = ""
@ -43,6 +45,7 @@ func TestMain(m *testing.M) {
if err := config.Init(); err != nil {
panic(err)
}
adminserverClient = client.NewClient(adminServer.URL, nil)
result := m.Run()
if result != 0 {
os.Exit(result)
@ -95,51 +98,77 @@ func TestEnvPolicyChecker(t *testing.T) {
if err := os.Setenv("PROJECT_CONTENT_TRUST", "1"); err != nil {
t.Fatalf("Failed to set env variable: %v", err)
}
if err2 := os.Setenv("PROJECT_VULNERABLE", "1"); err2 != nil {
t.Fatalf("Failed to set env variable: %v", err2)
}
if err3 := os.Setenv("PROJECT_SEVERITY", "negligible"); err3 != nil {
t.Fatalf("Failed to set env variable: %v", err3)
}
contentTrustFlag := getPolicyChecker().contentTrustEnabled("whatever")
vulFlag := getPolicyChecker().vulnerableEnabled("whatever")
vulFlag, sev := getPolicyChecker().vulnerablePolicy("whatever")
assert.True(contentTrustFlag)
assert.False(vulFlag)
assert.True(vulFlag)
assert.Equal(sev, models.SevNone)
}
func TestPMSPolicyChecker(t *testing.T) {
pm := pms.NewProjectManager(admiralEndpoint, token)
name := "project_for_test_get_true"
var defaultConfigAdmiral = map[string]interface{}{
common.ExtEndpoint: "https://" + endpoint,
common.WithNotary: true,
common.CfgExpiration: 5,
common.AdmiralEndpoint: admiralEndpoint,
}
adminServer, err := utilstest.NewAdminserver(defaultConfigAdmiral)
if err != nil {
panic(err)
}
defer adminServer.Close()
if err := os.Setenv("ADMIN_SERVER_URL", adminServer.URL); err != nil {
panic(err)
}
if err := config.Init(); err != nil {
panic(err)
}
pm := pms.NewProjectManager(http.DefaultClient,
admiralEndpoint, nil)
name := "project_for_test_get_sev_low"
id, err := pm.Create(&models.Project{
Name: name,
EnableContentTrust: true,
Name: name,
EnableContentTrust: true,
PreventVulnerableImagesFromRunning: false,
PreventVulnerableImagesFromRunningSeverity: "low",
})
require.Nil(t, err)
defer func(id int64) {
if err := pm.Delete(id); err != nil {
require.Nil(t, err)
t.Logf("failed to delete project %d: %v", id, err)
}
}(id)
project, err := pm.Get(id)
assert.Nil(t, err)
assert.Equal(t, id, project.ProjectID)
server, err2 := utilstest.NewAdminserver(nil)
if err2 != nil {
t.Fatalf("failed to create a mock admin server: %v", err2)
}
defer server.Close()
contentTrustFlag := getPolicyChecker().contentTrustEnabled("project_for_test_get_true")
contentTrustFlag := getPolicyChecker().contentTrustEnabled("project_for_test_get_sev_low")
assert.True(t, contentTrustFlag)
projectVulnerableEnabled, projectVulnerableSeverity := getPolicyChecker().vulnerablePolicy("project_for_test_get_sev_low")
assert.False(t, projectVulnerableEnabled)
assert.Equal(t, projectVulnerableSeverity, models.SevLow)
}
func TestMatchNotaryDigest(t *testing.T) {
assert := assert.New(t)
//The data from common/utils/notary/helper_test.go
img1 := imageInfo{"notary-demo/busybox", "1.0", "notary-demo"}
img2 := imageInfo{"notary-demo/busybox", "2.0", "notary-demo"}
res1, err := matchNotaryDigest(img1, "sha256:1359608115b94599e5641638bac5aef1ddfaa79bb96057ebf41ebc8d33acf8a7")
img1 := imageInfo{"notary-demo/busybox", "1.0", "notary-demo", "sha256:1359608115b94599e5641638bac5aef1ddfaa79bb96057ebf41ebc8d33acf8a7"}
img2 := imageInfo{"notary-demo/busybox", "2.0", "notary-demo", "sha256:12345678"}
res1, err := matchNotaryDigest(img1)
assert.Nil(err, "Unexpected error: %v, image: %#v", err, img1)
assert.True(res1)
res2, err := matchNotaryDigest(img1, "sha256:1359608115b94599e5641638bac5aef1ddfaa79bb96057ebf41ebc8d33acf8a8")
assert.Nil(err, "Unexpected error: %v, image: %#v, take 2", err, img1)
res2, err := matchNotaryDigest(img2)
assert.Nil(err, "Unexpected error: %v, image: %#v, take 2", err, img2)
assert.False(res2)
res3, err := matchNotaryDigest(img2, "sha256:1359608115b94599e5641638bac5aef1ddfaa79bb96057ebf41ebc8d33acf8a7")
assert.Nil(err, "Unexpected error: %v, image: %#v", err, img2)
assert.False(res3)
}
func TestCopyResp(t *testing.T) {

136
src/ui/proxy/interceptors.go
View File

@ -1,12 +1,14 @@
package proxy
import (
// "github.com/vmware/harbor/src/ui/api"
"github.com/vmware/harbor/src/common/dao"
"github.com/vmware/harbor/src/common/models"
"github.com/vmware/harbor/src/common/utils/clair"
"github.com/vmware/harbor/src/common/utils/log"
"github.com/vmware/harbor/src/common/utils/notary"
// "github.com/vmware/harbor/src/ui/api"
"github.com/vmware/harbor/src/ui/config"
"github.com/vmware/harbor/src/ui/projectmanager"
"github.com/vmware/harbor/src/ui/projectmanager/pms"
"context"
"fmt"
@ -26,6 +28,9 @@ const (
tokenUsername = "admin"
)
// Record the docker deamon raw response.
var rec *httptest.ResponseRecorder
// NotaryEndpoint , exported for testing.
var NotaryEndpoint = config.InternalNotaryEndpoint()
@ -51,8 +56,8 @@ func MatchPullManifest(req *http.Request) (bool, string, string) {
type policyChecker interface {
// contentTrustEnabled returns whether a project has enabled content trust.
contentTrustEnabled(name string) bool
// vulnerableEnabled returns whether a project has enabled content trust.
vulnerableEnabled(name string) bool
// vulnerablePolicy returns whether a project has enabled vulnerable, and the project's severity.
vulnerablePolicy(name string) (bool, models.Severity)
}
//For testing
@ -61,9 +66,8 @@ type envPolicyChecker struct{}
func (ec envPolicyChecker) contentTrustEnabled(name string) bool {
return os.Getenv("PROJECT_CONTENT_TRUST") == "1"
}
func (ec envPolicyChecker) vulnerableEnabled(name string) bool {
// TODO: May need get more information in vulnerable policies.
return os.Getenv("PROJECT_VULNERABBLE") == "1"
func (ec envPolicyChecker) vulnerablePolicy(name string) (bool, models.Severity) {
return os.Getenv("PROJECT_VULNERABLE") == "1", clair.ParseClairSev(os.Getenv("PROJECT_SEVERITY"))
}
type pmsPolicyChecker struct {
@ -78,8 +82,13 @@ func (pc pmsPolicyChecker) contentTrustEnabled(name string) bool {
}
return project.EnableContentTrust
}
func (pc pmsPolicyChecker) vulnerableEnabled(name string) bool {
return true
func (pc pmsPolicyChecker) vulnerablePolicy(name string) (bool, models.Severity) {
project, err := pc.pm.Get(name)
if err != nil {
log.Errorf("Unexpected error when getting the project, error: %v", err)
return true, models.SevUnknown
}
return project.PreventVulnerableImagesFromRunning, clair.ParseClairSev(project.PreventVulnerableImagesFromRunningSeverity)
}
// newPMSPolicyChecker returns an instance of an pmsPolicyChecker
@ -92,7 +101,7 @@ func newPMSPolicyChecker(pm projectmanager.ProjectManager) policyChecker {
// TODO: Get project manager with PM factory.
func getPolicyChecker() policyChecker {
if config.WithAdmiral() {
return newPMSPolicyChecker(pms.NewProjectManager(config.AdmiralEndpoint(), ""))
return newPMSPolicyChecker(config.GlobalProjectMgr)
}
return EnvChecker
}
@ -101,7 +110,7 @@ type imageInfo struct {
repository string
tag string
projectName string
// digest string
digest string
}
type urlHandler struct {
@ -120,38 +129,22 @@ func (uh urlHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
http.Error(rw, fmt.Sprintf("Bad repository name: %s", repository), http.StatusBadRequest)
return
}
/*
//Need to get digest of the image.
endpoint, err := config.RegistryURL()
if err != nil {
log.Errorf("Error getting Registry URL: %v", err)
http.Error(rw, fmt.Sprintf("Failed due to internal Error: %v", err), http.StatusInternalError)
return
}
rc, err := api.NewRepositoryClient(endpoint, false, username, repository, "repository", repository, "pull")
if err != nil {
log.Errorf("Error creating repository Client: %v", err)
http.Error(rw, fmt.Sprintf("Failed due to internal Error: %v", err), http.StatusInternalError)
return
}
digest, exist, err := rc.ManifestExist(tag)
if err != nil {
log.Errorf("Failed to get digest for tag: %s, error: %v", tag, err)
http.Error(rw, fmt.Sprintf("Failed due to internal Error: %v", err), http.StatusInternalError)
return
}
*/
rec = httptest.NewRecorder()
uh.next.ServeHTTP(rec, req)
if rec.Result().StatusCode != http.StatusOK {
copyResp(rec, rw)
return
}
digest := rec.Header().Get(http.CanonicalHeaderKey("Docker-Content-Digest"))
img := imageInfo{
repository: repository,
tag: tag,
projectName: components[0],
digest: digest,
}
log.Debugf("image info of the request: %#v", img)
ctx := context.WithValue(req.Context(), imageInfoCtxKey, img)
req = req.WithContext(ctx)
}
uh.next.ServeHTTP(rw, req)
}
@ -171,31 +164,70 @@ func (cth contentTrustHandler) ServeHTTP(rw http.ResponseWriter, req *http.Reque
cth.next.ServeHTTP(rw, req)
return
}
//May need to update status code, let's use recorder
rec := httptest.NewRecorder()
cth.next.ServeHTTP(rec, req)
if rec.Result().StatusCode != http.StatusOK {
copyResp(rec, rw)
return
}
log.Debugf("showing digest")
digest := rec.Header().Get(http.CanonicalHeaderKey("Docker-Content-Digest"))
log.Debugf("digest: %s", digest)
match, err := matchNotaryDigest(img, digest)
match, err := matchNotaryDigest(img)
if err != nil {
http.Error(rw, "Failed in communication with Notary please check the log", http.StatusInternalServerError)
return
}
if match {
log.Debugf("Passing the response to outter responseWriter")
copyResp(rec, rw)
} else {
if !match {
log.Debugf("digest mismatch, failing the response.")
http.Error(rw, "The image is not signed in Notary.", http.StatusPreconditionFailed)
return
}
cth.next.ServeHTTP(rw, req)
}
func matchNotaryDigest(img imageInfo, digest string) (bool, error) {
type vulnerableHandler struct {
next http.Handler
}
func (vh vulnerableHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
imgRaw := req.Context().Value(imageInfoCtxKey)
if imgRaw == nil || !config.WithClair() {
vh.next.ServeHTTP(rw, req)
return
}
img, _ := req.Context().Value(imageInfoCtxKey).(imageInfo)
projectVulnerableEnabled, projectVulnerableSeverity := getPolicyChecker().vulnerablePolicy(img.projectName)
if !projectVulnerableEnabled {
vh.next.ServeHTTP(rw, req)
return
}
overview, err := dao.GetImgScanOverview(img.digest)
if err != nil {
log.Errorf("failed to get ImgScanOverview with repo: %s, tag: %s, digest: %s. Error: %v", img.repository, img.tag, img.digest, err)
http.Error(rw, "Failed to get ImgScanOverview.", http.StatusPreconditionFailed)
return
}
if overview == nil {
log.Debugf("cannot get the image scan overview info, failing the response.")
http.Error(rw, "Cannot get the image scan overview info.", http.StatusPreconditionFailed)
return
}
imageSev := overview.Sev
if imageSev > int(projectVulnerableSeverity) {
log.Debugf("the image severity is higher then project setting, failing the response.")
http.Error(rw, "The image scan result doesn't pass the project setting.", http.StatusPreconditionFailed)
return
}
vh.next.ServeHTTP(rw, req)
}
type funnelHandler struct {
next http.Handler
}
func (fu funnelHandler) ServeHTTP(rw http.ResponseWriter, req *http.Request) {
imgRaw := req.Context().Value(imageInfoCtxKey)
if imgRaw != nil {
log.Debugf("Return the original response as no the interceptor takes action.")
copyResp(rec, rw)
return
}
fu.next.ServeHTTP(rw, req)
}
func matchNotaryDigest(img imageInfo) (bool, error) {
targets, err := notary.GetInternalTargets(NotaryEndpoint, tokenUsername, img.repository)
if err != nil {
return false, err
@ -207,7 +239,7 @@ func matchNotaryDigest(img imageInfo, digest string) (bool, error) {
if err != nil {
return false, err
}
return digest == d, nil
return img.digest == d, nil
}
}
log.Debugf("image: %#v, not found in notary", img)

2
src/ui/proxy/proxy.go
View File

@ -42,7 +42,7 @@ func Init(urls ...string) error {
}
Proxy = httputil.NewSingleHostReverseProxy(targetURL)
//TODO: add vulnerable interceptor.
handlers = handlerChain{head: urlHandler{next: contentTrustHandler{next: Proxy}}}
handlers = handlerChain{head: urlHandler{next: contentTrustHandler{next: vulnerableHandler{next: funnelHandler{next: Proxy}}}}}
return nil
}

9
src/ui/router.go
View File

@ -40,10 +40,10 @@ func initRouters() {
beego.Router("/harbor/sign-up", &controllers.IndexController{})
beego.Router("/harbor/dashboard", &controllers.IndexController{})
beego.Router("/harbor/projects", &controllers.IndexController{})
beego.Router("/harbor/projects/:id/repository", &controllers.IndexController{})
beego.Router("/harbor/projects/:id/replication", &controllers.IndexController{})
beego.Router("/harbor/projects/:id/member", &controllers.IndexController{})
beego.Router("/harbor/projects/:id/log", &controllers.IndexController{})
beego.Router("/harbor/projects/:id/repositories", &controllers.IndexController{})
beego.Router("/harbor/projects/:id/replications", &controllers.IndexController{})
beego.Router("/harbor/projects/:id/members", &controllers.IndexController{})
beego.Router("/harbor/projects/:id/logs", &controllers.IndexController{})
beego.Router("/harbor/tags/:id/*", &controllers.IndexController{})
beego.Router("/harbor/users", &controllers.IndexController{})
@ -73,6 +73,7 @@ func initRouters() {
beego.Router("/api/users/:id([0-9]+)/password", &api.UserAPI{}, "put:ChangePassword")
beego.Router("/api/internal/syncregistry", &api.InternalAPI{}, "post:SyncRegistry")
beego.Router("/api/repositories", &api.RepositoryAPI{}, "get:Get")
beego.Router("/api/repositories/scanAll", &api.RepositoryAPI{}, "post:ScanAll")
beego.Router("/api/repositories/*", &api.RepositoryAPI{}, "delete:Delete")
beego.Router("/api/repositories/*/tags/:tag", &api.RepositoryAPI{}, "delete:Delete;get:GetTag")
beego.Router("/api/repositories/*/tags", &api.RepositoryAPI{}, "get:GetTags")

10
src/ui/service/notification.go
View File

@ -27,7 +27,7 @@ import (
"github.com/vmware/harbor/src/common/utils/log"
"github.com/vmware/harbor/src/ui/api"
"github.com/vmware/harbor/src/ui/config"
"github.com/vmware/harbor/src/ui/projectmanager/pms"
uiutils "github.com/vmware/harbor/src/ui/utils"
)
// NotificationHandler handles request on /service/notifications/, which listens to registry's events.
@ -65,7 +65,7 @@ func (n *NotificationHandler) Post() {
user = "anonymous"
}
pro, err := n.ProjectMgr.Get(project)
pro, err := config.GlobalProjectMgr.Get(project)
if err != nil {
log.Errorf("failed to get project by name %s: %v", project, err)
return
@ -102,7 +102,7 @@ func (n *NotificationHandler) Post() {
go api.TriggerReplicationByRepository(pro.ProjectID, repository, []string{tag}, models.RepOpTransfer)
if autoScanEnabled(project) {
if err := api.TriggerImageScan(repository, tag); err != nil {
if err := uiutils.TriggerImageScan(repository, tag); err != nil {
log.Warningf("Failed to scan image, repository: %s, tag: %s, error: %v", repository, tag, err)
}
}
@ -160,9 +160,7 @@ func autoScanEnabled(projectName string) bool {
return false
}
if config.WithAdmiral() {
//TODO get a project manager based on service account.
var pm *pms.ProjectManager = pms.NewProjectManager("", "")
p, err := pm.Get(projectName)
p, err := config.GlobalProjectMgr.Get(projectName)
if err != nil {
log.Warningf("failed to get project, error: %v", err)
return false

141
src/ui/utils/utils.go Normal file
View File

@ -0,0 +1,141 @@
// Copyright (c) 2017 VMware, Inc. All Rights Reserved.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package utils contains methods to support security, cache, and webhook functions.
package utils
import (
"github.com/vmware/harbor/src/common/dao"
"github.com/vmware/harbor/src/common/models"
"github.com/vmware/harbor/src/common/utils/log"
"github.com/vmware/harbor/src/common/utils/registry"
"github.com/vmware/harbor/src/common/utils/registry/auth"
"github.com/vmware/harbor/src/ui/config"
"bytes"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http"
)
// ScanAllImages scans all images of Harbor by submiting jobs to jobservice, the whole process will move one if failed to subit any job of a single image.
func ScanAllImages() error {
regURL, err := config.RegistryURL()
if err != nil {
log.Errorf("Failed to load registry url")
return err
}
repos, err := dao.GetAllRepositories()
if err != nil {
log.Errorf("Failed to list all repositories, error: %v", err)
return err
}
log.Infof("Rescanning all images.")
go func() {
var repoClient *registry.Repository
var err error
var tags []string
for _, r := range repos {
repoClient, err = NewRepositoryClientForUI(regURL, true, "harbor-ui", r.Name, "pull")
if err != nil {
log.Errorf("Failed to initialize client for repository: %s, error: %v, skip scanning", r.Name, err)
continue
}
tags, err = repoClient.ListTag()
if err != nil {
log.Errorf("Failed to get tags for repository: %s, error: %v, skip scanning.", r.Name, err)
continue
}
for _, t := range tags {
if err = TriggerImageScan(r.Name, t); err != nil {
log.Errorf("Failed to scan image with repository: %s, tag: %s, error: %v.", r.Name, t, err)
} else {
log.Debugf("Triggered scan for image with repository: %s, tag: %s", r.Name, t)
}
}
}
}()
return nil
}
// RequestAsUI is a shortcut to make a request attach UI secret and send the request.
// Do not use this when you want to handle the response
// TODO: add a response handler to replace expectSC *when needed*
func RequestAsUI(method, url string, body io.Reader, expectSC int) error {
req, err := http.NewRequest(method, url, body)
if err != nil {
return err
}
client := &http.Client{}
resp, err := client.Do(req)
if err != nil {
return err
}
AddUISecret(req)
defer resp.Body.Close()
if resp.StatusCode != expectSC {
b, err := ioutil.ReadAll(resp.Body)
if err != nil {
return err
}
return fmt.Errorf("Unexpected status code: %d, text: %s", resp.StatusCode, string(b))
}
return nil
}
//AddUISecret add secret cookie to a request
func AddUISecret(req *http.Request) {
if req != nil {
req.AddCookie(&http.Cookie{
Name: models.UISecretCookie,
Value: config.UISecret(),
})
}
}
// TriggerImageScan triggers an image scan job on jobservice.
func TriggerImageScan(repository string, tag string) error {
data := &models.ImageScanReq{
Repo: repository,
Tag: tag,
}
b, err := json.Marshal(&data)
if err != nil {
return err
}
url := fmt.Sprintf("%s/api/jobs/scan", config.InternalJobServiceURL())
return RequestAsUI("POST", url, bytes.NewBuffer(b), http.StatusOK)
}
// NewRepositoryClientForUI ...
// TODO need a registry client which accept a raw token as param
func NewRepositoryClientForUI(endpoint string, insecure bool, username, repository string,
scopeActions ...string) (*registry.Repository, error) {
authorizer := auth.NewRegistryUsernameTokenAuthorizer(username, "repository", repository, scopeActions...)
store, err := auth.NewAuthorizerStore(endpoint, insecure, authorizer)
if err != nil {
return nil, err
}
client, err := registry.NewRepositoryWithModifiers(repository, endpoint, insecure, store)
if err != nil {
return nil, err
}
return client, nil
}

0
src/ui/service/utils/utils_test.go → src/ui/utils/utils_test.go
View File

4
src/ui_ng/src/app/shared/statictics/statistics-panel.component.html
View File

@ -12,10 +12,10 @@
</div>
<div class="statistic-column-block" style="margin-left: 16px;">
<div>
<statistics [data]='originalCopy.my_project_count' [label]='"STATISTICS.INDEX_PRIVATE" | translate'></statistics>
<statistics [data]='originalCopy.private_project_count' [label]='"STATISTICS.INDEX_PRIVATE" | translate'></statistics>
</div>
<div>
<statistics [data]='originalCopy.my_repo_count' [label]='"STATISTICS.INDEX_PRIVATE" | translate'></statistics>
<statistics [data]='originalCopy.private_repo_count' [label]='"STATISTICS.INDEX_PRIVATE" | translate'></statistics>
</div>
</div>
<div class="statistic-column-block" style="margin-left: 28px;">

4
src/ui_ng/src/app/shared/statictics/statistics.ts
View File

@ -14,8 +14,8 @@
export class Statistics {
constructor() {}
my_project_count: number;
my_repo_count: number;
private_project_count: number;
private_repo_count: number;
public_project_count: number;
public_repo_count: number;
total_project_count: number;

12
tests/apitests/apilib/statistic_map.go
View File

@ -1,10 +1,10 @@
/*
/*
* Harbor API
*
* These APIs provide services for manipulating Harbor project.
*
* OpenAPI spec version: 0.3.0
*
*
* Generated by: https://github.com/swagger-api/swagger-codegen.git
*
* Licensed under the Apache License, Version 2.0 (the "License");
@ -24,11 +24,11 @@ package apilib
type StatisticMap struct {
// The count of the projects which the user is a member of.
MyProjectCount int32 `json:"my_project_count,omitempty"`
// The count of the private projects which the user is a member of.
PrivateProjectCount int32 `json:"private_project_count,omitempty"`
// The count of the repositories belonging to the projects which the user is a member of.
MyRepoCount int32 `json:"my_repo_count,omitempty"`
// The count of the private repositories belonging to the projects which the user is a member of.
PrivateRepoCount int32 `json:"private_repo_count,omitempty"`
// The count of the public projects.
PublicProjectCount int32 `json:"public_project_count,omitempty"`

6
tests/generateCerts.sh
View File

@ -17,12 +17,12 @@ fi
# Create CA certificate
openssl req \
-newkey rsa:4096 -nodes -sha256 -keyout harbor_ca.key \
-x509 -days 365 -out harbor_ca.crt -subj '/C=CN/ST=PEK/L=Bei Jing/O=VMware/CN=Harbor CA'
-x509 -days 365 -out harbor_ca.crt -subj '/C=CN/ST=PEK/L=Bei Jing/O=VMware/CN=HarborCA'
# Generate a Certificate Signing Request
openssl req \
-newkey rsa:4096 -nodes -sha256 -keyout $IP.key \
-out $IP.csr -subj '/C=CN/ST=PEK/L=Bei Jing/O=VMware/CN=Harbor CA'
-out $IP.csr -subj '/C=CN/ST=PEK/L=Bei Jing/O=VMware/CN=HarborManager'
# Generate the certificate of local registry host
echo subjectAltName = IP:$IP > extfile.cnf
@ -32,4 +32,4 @@ openssl x509 -req -days 365 -in $IP.csr -CA harbor_ca.crt \
# Copy to harbor default location
mkdir -p /data/cert
cp $IP.crt /data/cert/server.crt
cp $IP.key /data/cert/server.key
cp $IP.key /data/cert/server.key

36
tools/migration/db_meta.py
View File

@ -139,4 +139,40 @@ class Repository(Base):
creation_time = sa.Column(mysql.TIMESTAMP, server_default = sa.text("CURRENT_TIMESTAMP"))
update_time = sa.Column(mysql.TIMESTAMP, server_default = sa.text("CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP"))
class AccessLog(Base):
__tablename__ = "access_log"
user_id = sa.Column(sa.Integer, nullable=False)
log_id = sa.Column(sa.Integer, primary_key=True)
username = sa.Column(sa.String(32), nullable=False)
project_id = sa.Column(sa.Integer, nullable=False)
repo_name = sa.Column(sa.String(256))
repo_tag = sa.Column(sa.String(128))
GUID = sa.Column(sa.String(64))
operation = sa.Column(sa.String(20))
op_time = sa.Column(mysql.TIMESTAMP)
update_time = sa.Column(mysql.TIMESTAMP, server_default = sa.text("CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP"))
__table_args__ = (sa.Index('project_id', "op_time"),)
class ImageScanJob(Base):
__tablename__ = "img_scan_job"
id = sa.Column(sa.Integer, nullable=False, primary_key=True)
status = sa.Column(sa.String(64), nullable=False)
repository = sa.Column(sa.String(256), nullable=False)
tag = sa.Column(sa.String(128), nullable=False)
digest = sa.Column(sa.String(128))
creation_time = sa.Column(mysql.TIMESTAMP, server_default = sa.text("CURRENT_TIMESTAMP"))
update_time = sa.Column(mysql.TIMESTAMP, server_default = sa.text("CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP"))
class ImageScanOverview(Base):
__tablename__ = "img_scan_overview"
scan_job_id = sa.Column(sa.Integer, nullable=False)
image_digest = sa.Column(sa.String(128), nullable=False, primary_key=True)
severity = sa.Column(sa.Integer, nullable=False, server_default=sa.text("'0'"))
components_overview = sa.Column(sa.String(2048))
details_key = sa.Column(sa.String(128))
creation_time = sa.Column(mysql.TIMESTAMP, server_default = sa.text("CURRENT_TIMESTAMP"))
update_time = sa.Column(mysql.TIMESTAMP, server_default = sa.text("CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP"))

68
tools/migration/migration_harbor/versions/1_2_0.py Normal file
View File

@ -0,0 +1,68 @@
# Copyright (c) 2008-2016 VMware, Inc. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""0.4.0 to 1.2.0
Revision ID: 0.4.0
Revises:
"""
# revision identifiers, used by Alembic.
revision = '1.2.0'
down_revision = '0.4.0'
branch_labels = None
depends_on = None
from alembic import op
from db_meta import *
from sqlalchemy.dialects import mysql
Session = sessionmaker()
def upgrade():
"""
update schema&data
"""
bind = op.get_bind()
session = Session(bind=bind)
#delete column access_log.user_id(access_log_ibfk_1), access_log.project_id(access_log_ibfk_2)
op.drop_constraint('access_log_ibfk_1', 'access_log', type_='foreignkey')
op.drop_constraint('access_log_ibfk_2', 'access_log', type_='foreignkey')
#add colume username to access_log
op.add_column('access_log', sa.Column('username', mysql.VARCHAR(32), nullable=False))
#init username
session.query(AccessLog).update({AccessLog.username: ""})
#update access_log username
user_all = session.query(User).all()
for user in user_all:
session.query(AccessLog).filter(AccessLog.user_id == user.user_id).update({AccessLog.username: user.username}, synchronize_session='fetch')
op.drop_column("access_log", "user_id")
op.drop_column("repository", "owner_id")
#create tables: img_scan_job, img_scan_overview
ImageScanJob.__table__.create(bind)
ImageScanOverview.__table__.create(bind)
def downgrade():
"""
Downgrade has been disabled.
"""
pass