Merge pull request #2777 from reasonerjt/fetch-timestamp-from-clairdb

provide default timestamp for all distros in system info api

src/common/models/clair.go

@@ -122,3 +122,15 @@ type ClairNamespaceTimestamp struct {
 	Namespace string `json:"namespace"`
 	Timestamp int64  `json:"last_update"`
 }
+
+//ClairNamespace ...
+type ClairNamespace struct {
+	Name          string `json:"Name,omitempty"`
+	VersionFormat string `json:"VersionFormat,omitempty"`
+}
+
+//ClairNamespaceEnvelope ...
+type ClairNamespaceEnvelope struct {
+	Namespaces *[]ClairNamespace `json:"Namespaces,omitempty"`
+	Error      *ClairError       `json:"Error,omitempty"`
+}

src/common/utils/clair/client.go

@@ -47,6 +47,22 @@ func NewClient(endpoint string, logger *log.Logger) *Client {
 	}
 }
 
+func (c *Client) send(req *http.Request, expectedStatus int) ([]byte, error) {
+	resp, err := c.client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+	defer resp.Body.Close()
+	b, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return nil, err
+	}
+	if resp.StatusCode != expectedStatus {
+		return nil, fmt.Errorf("Unexpected status code: %d, text: %s", resp.StatusCode, string(b))
+	}
+	return b, nil
+}
+
 // ScanLayer calls Clair's API to scan a layer.
 func (c *Client) ScanLayer(l models.ClairLayer) error {
 	layer := models.ClairLayerEnvelope{
@@ -57,47 +73,28 @@ func (c *Client) ScanLayer(l models.ClairLayer) error {
 	if err != nil {
 		return err
 	}
-	req, err := http.NewRequest("POST", c.endpoint+"/v1/layers", bytes.NewReader(data))
+	req, err := http.NewRequest(http.MethodPost, c.endpoint+"/v1/layers", bytes.NewReader(data))
 	if err != nil {
 		return err
 	}
 	req.Header.Set(http.CanonicalHeaderKey("Content-Type"), "application/json")
-	resp, err := c.client.Do(req)
+	_, err = c.send(req, http.StatusCreated)
 	if err != nil {
+		log.Errorf("Unexpected error: %v", err)
+	}
 	return err
 }
-	defer resp.Body.Close()
-	b, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return err
-	}
-	c.logger.Infof("response code: %d", resp.StatusCode)
-	if resp.StatusCode != http.StatusCreated {
-		c.logger.Warningf("Unexpected status code: %d", resp.StatusCode)
-		return fmt.Errorf("Unexpected status code: %d, text: %s", resp.StatusCode, string(b))
-	}
-	c.logger.Infof("Returning.")
-	return nil
-}
 
 // GetResult calls Clair's API to get layers with detailed vulnerability list
 func (c *Client) GetResult(layerName string) (*models.ClairLayerEnvelope, error) {
-	req, err := http.NewRequest("GET", c.endpoint+"/v1/layers/"+layerName+"?features&vulnerabilities", nil)
+	req, err := http.NewRequest(http.MethodGet, c.endpoint+"/v1/layers/"+layerName+"?features&vulnerabilities", nil)
 	if err != nil {
 		return nil, err
 	}
-	resp, err := c.client.Do(req)
+	b, err := c.send(req, http.StatusOK)
 	if err != nil {
 		return nil, err
 	}
-	defer resp.Body.Close()
-	b, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return nil, err
-	}
-	if resp.StatusCode != http.StatusOK {
-		return nil, fmt.Errorf("Unexpected status code: %d, text: %s", resp.StatusCode, string(b))
-	}
 	var res models.ClairLayerEnvelope
 	err = json.Unmarshal(b, &res)
 	if err != nil {
@@ -108,22 +105,14 @@ func (c *Client) GetResult(layerName string) (*models.ClairLayerEnvelope, error)
 
 // GetNotification calls Clair's API to get details of notification
 func (c *Client) GetNotification(id string) (*models.ClairNotification, error) {
-	req, err := http.NewRequest("GET", c.endpoint+"/v1/notifications/"+id+"?limit=2", nil)
+	req, err := http.NewRequest(http.MethodGet, c.endpoint+"/v1/notifications/"+id+"?limit=2", nil)
 	if err != nil {
 		return nil, err
 	}
-	resp, err := c.client.Do(req)
+	b, err := c.send(req, http.StatusOK)
 	if err != nil {
 		return nil, err
 	}
-	defer resp.Body.Close()
-	b, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return nil, err
-	}
-	if resp.StatusCode != http.StatusOK {
-		return nil, fmt.Errorf("Unexpected status code: %d, text: %s", resp.StatusCode, string(b))
-	}
 	var ne models.ClairNotificationEnvelope
 	err = json.Unmarshal(b, &ne)
 	if err != nil {
@@ -138,22 +127,36 @@ func (c *Client) GetNotification(id string) (*models.ClairNotification, error) {
 
 // DeleteNotification deletes a notification record from Clair
 func (c *Client) DeleteNotification(id string) error {
-	req, err := http.NewRequest("DELETE", c.endpoint+"/v1/notifications/"+id, nil)
+	req, err := http.NewRequest(http.MethodDelete, c.endpoint+"/v1/notifications/"+id, nil)
 	if err != nil {
 		return err
 	}
-	resp, err := c.client.Do(req)
+	_, err = c.send(req, http.StatusOK)
 	if err != nil {
 		return err
 	}
-	defer resp.Body.Close()
-	b, err := ioutil.ReadAll(resp.Body)
-	if err != nil {
-		return err
-	}
-	if resp.StatusCode != http.StatusOK {
-		return fmt.Errorf("Unexpected status code: %d, text: %s", resp.StatusCode, string(b))
-	}
 	log.Debugf("Deleted notification %s from Clair.", id)
 	return nil
 }
+
+// ListNamespaces list the namespaces in Clair
+func (c *Client) ListNamespaces() ([]string, error) {
+	req, err := http.NewRequest(http.MethodGet, c.endpoint+"/v1/namespaces", nil)
+	if err != nil {
+		return nil, err
+	}
+	b, err := c.send(req, http.StatusOK)
+	if err != nil {
+		return nil, err
+	}
+	var nse models.ClairNamespaceEnvelope
+	err = json.Unmarshal(b, &nse)
+	if err != nil {
+		return nil, err
+	}
+	res := []string{}
+	for _, ns := range *nse.Namespaces {
+		res = append(res, ns.Name)
+	}
+	return res, nil
+}

src/ui/api/systeminfo.go

@@ -19,12 +19,13 @@ import (
 	"net/http"
 	"os"
 	"strings"
-	"time"
+	"sync"
 
 	"github.com/vmware/harbor/src/common"
 	"github.com/vmware/harbor/src/common/dao"
 	clairdao "github.com/vmware/harbor/src/common/dao/clair"
 	"github.com/vmware/harbor/src/common/models"
+	"github.com/vmware/harbor/src/common/utils/clair"
 	"github.com/vmware/harbor/src/common/utils/log"
 	"github.com/vmware/harbor/src/ui/config"
 )
@@ -48,6 +49,40 @@ type Storage struct {
 	Free  uint64 `json:"free"`
 }
 
+//namespaces stores all name spaces on Clair, it should be initialised only once.
+type clairNamespaces struct {
+	*sync.RWMutex
+	l     []string
+	clair *clair.Client
+}
+
+func (n clairNamespaces) get() ([]string, error) {
+	n.Lock()
+	defer n.Unlock()
+	if len(n.l) == 0 {
+		m := make(map[string]struct{})
+		if n.clair == nil {
+			n.clair = clair.NewClient(config.ClairEndpoint(), nil)
+		}
+		list, err := n.clair.ListNamespaces()
+		if err != nil {
+			return n.l, err
+		}
+		for _, n := range list {
+			ns := strings.Split(n, ":")[0]
+			m[ns] = struct{}{}
+		}
+		for k := range m {
+			n.l = append(n.l, k)
+		}
+	}
+	return n.l, nil
+}
+
+var (
+	namespaces clairNamespaces
+)
+
 //GeneralInfo wraps common systeminfo for anonymous request
 type GeneralInfo struct {
 	WithNotary              bool                             `json:"with_notary"`
@@ -166,26 +201,40 @@ func getClairVulnStatus() *models.ClairVulnerabilityStatus {
 		res.OverallUTC = last
 		log.Debugf("Clair vuln DB last update: %d", last)
 	}
+	details := []models.ClairNamespaceTimestamp{}
+	if res.OverallUTC > 0 {
 		l, err := dao.ListClairVulnTimestamps()
 		if err != nil {
 			log.Errorf("Failed to list Clair vulnerability timestamps, error:%v", err)
 			return res
 		}
-	m := make(map[string]time.Time)
+		m := make(map[string]int64)
 		for _, e := range l {
 			ns := strings.Split(e.Namespace, ":")
-		if ts, ok := m[ns[0]]; !ok || ts.Before(e.LastUpdate) {
+			//only returns the latest time of one distro, i.e. unbuntu:14.04 and ubuntu:15.4 shares one timestamp
-			m[ns[0]] = e.LastUpdate
+			el := e.LastUpdate.UTC().Unix()
+			if ts, ok := m[ns[0]]; !ok || ts < el {
+				m[ns[0]] = el
+			}
+		}
+		list, err := namespaces.get()
+		if err != nil {
+			log.Errorf("Failed to get namespace list from Clair, error: %v", err)
+		}
+		//For namespaces not reported by notifier, the timestamp will be the overall db timestamp.
+		for _, n := range list {
+			if _, ok := m[n]; !ok {
+				m[n] = res.OverallUTC
 			}
 		}
-	details := []models.ClairNamespaceTimestamp{}
 		for k, v := range m {
 			e := models.ClairNamespaceTimestamp{
 				Namespace: k,
-			Timestamp: v.UTC().Unix(),
+				Timestamp: v,
 			}
 			details = append(details, e)
 		}
+	}
 	res.Details = details
 	return res
 }