Upgrade clair adapter to v1.0.0

1. Upgrade clair adapter to v1.0.0.
2. Make the clair adapter which installed by harbor immutable and using internal registry address.
3. Add support to build clair adapter image from binary.
4. Switch to ScannerPull action when make authorization for the scan request.

Signed-off-by: He Weiwei <hweiwei@vmware.com>

Makefile

@@ -106,7 +106,7 @@ CLAIRDBVERSION=$(VERSIONTAG)
 MIGRATORVERSION=$(VERSIONTAG)
 REDISVERSION=$(VERSIONTAG)
 NOTARYMIGRATEVERSION=v3.5.4
-CLAIRADAPTERVERSION=c7db8b15
+CLAIRADAPTERVERSION=v1.0.0
 
 # version of chartmuseum
 CHARTMUSEUMVERSION=v0.9.0
@@ -308,8 +308,8 @@ prepare: update_prepare_version
 	@$(MAKEPATH)/$(PREPARECMD) $(PREPARECMD_PARA)
 
 build:
-	make -f $(MAKEFILEPATH_PHOTON)/Makefile build -e DEVFLAG=$(DEVFLAG) \
+	make -f $(MAKEFILEPATH_PHOTON)/Makefile build -e DEVFLAG=$(DEVFLAG) -e GOBUILDIMAGE=$(GOBUILDIMAGE) \
-	 -e REGISTRYVERSION=$(REGISTRYVERSION) -e NGINXVERSION=$(NGINXVERSION) -e NOTARYVERSION=$(NOTARYVERSION) -e NOTARYMIGRATEVERSION=$(NOTARYMIGRATEVERSION) \
+	 -e REGISTRYVERSION=$(REGISTRYVERSION) -e REGISTRY_SRC_TAG=$(REGISTRY_SRC_TAG) -e NGINXVERSION=$(NGINXVERSION) -e NOTARYVERSION=$(NOTARYVERSION) -e NOTARYMIGRATEVERSION=$(NOTARYMIGRATEVERSION) \
 	 -e CLAIRVERSION=$(CLAIRVERSION) -e CLAIRADAPTERVERSION=$(CLAIRADAPTERVERSION) -e CLAIRDBVERSION=$(CLAIRDBVERSION) -e VERSIONTAG=$(VERSIONTAG) \
 	 -e BUILDBIN=$(BUILDBIN) -e REDISVERSION=$(REDISVERSION) -e MIGRATORVERSION=$(MIGRATORVERSION) \
 	 -e CHARTMUSEUMVERSION=$(CHARTMUSEUMVERSION) -e DOCKERIMAGENAME_CHART_SERVER=$(DOCKERIMAGENAME_CHART_SERVER) \

make/photon/Makefile

@@ -146,9 +146,14 @@ _build_clair:
 	fi
 
 _build_clair_adapter:
-	# TODO: add support to fetch clair adapter binary from google storage ranther than build from source
 	@if [ "$(CLAIRFLAG)" = "true" ] ; then \
+		if [ "$(BUILDBIN)" != "true" ] ; then \
+			rm -rf $(DOCKERFILEPATH_CLAIR_ADAPTER)/binary && mkdir -p $(DOCKERFILEPATH_CLAIR_ADAPTER)/binary && \
+			$(call _extract_archive, https://github.com/goharbor/harbor-scanner-clair/releases/download/$(CLAIRADAPTERVERSION)/harbor-scanner-clair_$(CLAIRADAPTERVERSION:v%=%)_Linux_x86_64.tar.gz, $(DOCKERFILEPATH_CLAIR_ADAPTER)/binary/) && \
+			mv $(DOCKERFILEPATH_CLAIR_ADAPTER)/binary/scanner-clair $(DOCKERFILEPATH_CLAIR_ADAPTER)/binary/harbor-scanner-clair; \
+		else \
 			cd $(DOCKERFILEPATH_CLAIR_ADAPTER) && $(DOCKERFILEPATH_CLAIR_ADAPTER)/builder $(CLAIRADAPTERVERSION) && cd - ; \
+		fi ; \
 		echo "building clair adapter container for photon..." ; \
 		$(DOCKERBUILD) -f $(DOCKERFILEPATH_CLAIR_ADAPTER)/$(DOCKERFILENAME_CLAIR_ADAPTER) -t $(DOCKERIMAGENAME_CLAIR_ADAPTER):$(CLAIRADAPTERVERSION)-$(VERSIONTAG) . ; \
 		rm -rf $(DOCKERFILEPATH_CLAIR_ADAPTER)/binary; \
@@ -219,6 +224,10 @@ _build_migrator:
 		echo "Done."; \
 	fi
 
+define _extract_archive
+	$(WGET) --timeout 30 --no-check-certificate -O- $1 | tar xvz -C $2
+endef
+
 define _get_binary
 	$(WGET) --timeout 30 --no-check-certificate $1 -O $2
 endef

make/photon/clair-adapter/Dockerfile

@@ -13,7 +13,7 @@ RUN chown -R 10000:10000 /clair-adapter \
 
 EXPOSE 8080
 
-HEALTHCHECK --interval=30s --timeout=10s --retries=3 CMD curl -sS 127.0.0.1:8080/healthy || exit 1
+HEALTHCHECK --interval=30s --timeout=10s --retries=3 CMD curl -sS 127.0.0.1:8080/probe/healthy || exit 1
 
 USER clair-adapter
 

make/photon/clair-adapter/builder

@@ -20,7 +20,7 @@ cur=$PWD
 
 # the temp folder to store distribution source code...
 TEMP=`mktemp -d ${TMPDIR-/tmp}/clair-adapter.XXXXXX`
-git clone https://github.com/danielpacak/harbor-scanner-clair.git $TEMP
+git clone https://github.com/goharbor/harbor-scanner-clair.git $TEMP
 cd $TEMP; git checkout $VERSION; cd -
 
 echo 'build the clair adapter binary bases on the golang:1.12.12'

make/photon/prepare/utils/docker_compose.py

@@ -14,7 +14,7 @@ def prepare_docker_compose(configs, with_clair, with_notary, with_chartmuseum):
     REGISTRY_VERSION = versions.get('REGISTRY_VERSION') or 'v2.7.1-patch-2819-2553'
     NOTARY_VERSION = versions.get('NOTARY_VERSION') or 'v0.6.1'
     CLAIR_VERSION = versions.get('CLAIR_VERSION') or 'v2.0.9'
-    CLAIR_ADAPTER_VERSION = versions.get('CLAIR_ADAPTER_VERSION') or ''
+    CLAIR_ADAPTER_VERSION = versions.get('CLAIR_ADAPTER_VERSION') or 'v1.0.0'
     CHARTMUSEUM_VERSION = versions.get('CHARTMUSEUM_VERSION') or 'v0.9.0'
 
     rendering_variables = {

make/photon/registry/builder

@@ -19,7 +19,7 @@ cd `dirname $0`
 cur=$PWD
 
 # the temp folder to store distribution source code...
-TEMP=`mktemp -d /$TMPDIR/distribution.XXXXXX`
+TEMP=`mktemp -d ${TMPDIR-/tmp}/distribution.XXXXXX`
 git clone -b $VERSION https://github.com/docker/distribution.git $TEMP
 
 # add patch 2879
@@ -35,7 +35,7 @@ docker build -f $TEMP/Dockerfile.binary -t registry-golang $TEMP
 
 echo 'copy the registry binary to local...'
 ID=$(docker create registry-golang)
-docker cp $ID:/go/src/github.com/docker/distribution/bin binary
+docker cp $ID:/go/src/github.com/docker/distribution/bin/registry binary/registry
 
 docker rm -f $ID
 docker rmi -f registry-golang

src/core/main.go

@@ -222,8 +222,9 @@ func main() {
 			Name:            "Clair",
 			Description:     "The clair scanner adapter",
 			URL:             config.ClairAdapterEndpoint(),
-			Disabled:    false,
 			IsDefault:       true,
+			UseInternalAddr: true,
+			Immutable:       true,
 		}
 
 		if err := scan.EnsureScanner(reg); err != nil {

src/pkg/scan/api/scan/base_controller.go

@@ -378,7 +378,7 @@ func (bc *basicController) makeBasicAuthorization(pid int64, repository string,
 	resource := rbac.NewProjectNamespace(pid).Resource(rbac.ResourceRepository)
 	access := []*rbac.Policy{{
 		Resource: resource,
-		Action:   rbac.ActionPull,
+		Action:   rbac.ActionScannerPull,
 	}}
 
 	robotReq := &model.RobotCreate{
@@ -481,7 +481,7 @@ func makeBearerAuthorization(repository string, username string) (string, error)
 		{
 			Type:    "repository",
 			Name:    repository,
-			Actions: []string{"pull"},
+			Actions: []string{rbac.ActionPull.String(), rbac.ActionScannerPull.String()},
 		},
 	}
 

src/pkg/scan/api/scan/base_controller_test.go

@@ -161,7 +161,7 @@ func (suite *ControllerTestSuite) SetupSuite() {
 	resource := fmt.Sprintf("/project/%d/repository", suite.artifact.NamespaceID)
 	access := []*rbac.Policy{{
 		Resource: rbac.Resource(resource),
-		Action:   "pull",
+		Action:   rbac.ActionScannerPull,
 	}}
 
 	rname := "the-uuid-123"

tests/hostcfg.sh

@@ -7,7 +7,3 @@ sudo sed "s/reg.mydomain.com/$IP/" -i make/harbor.yml
 echo "https:" >> make/harbor.yml
 echo "  certificate: /data/cert/server.crt" >> make/harbor.yml
 echo "  private_key: /data/cert/server.key" >> make/harbor.yml
-
-# TODO: remove it when scanner adapter support internal access of harbor
-echo "storage_service:" >> make/harbor.yml
-echo "  ca_bundle: /data/cert/server.crt" >> make/harbor.yml