From dd9f028fe0a0dcb2beca4a07d321fe62f5b1f379 Mon Sep 17 00:00:00 2001 From: wang yan Date: Thu, 29 Aug 2019 19:33:34 +0800 Subject: [PATCH] fix #8888 the image may has the same blobs as the references, which causes the artifact & blobs can not be inserted by unique constraint Signed-off-by: wang yan --- .../middlewares/countquota/handler_test.go | 49 ++++++++++++++----- src/core/middlewares/countquota/util.go | 24 +++++---- 2 files changed, 53 insertions(+), 20 deletions(-) diff --git a/src/core/middlewares/countquota/handler_test.go b/src/core/middlewares/countquota/handler_test.go index 84833e4de..d5e699679 100644 --- a/src/core/middlewares/countquota/handler_test.go +++ b/src/core/middlewares/countquota/handler_test.go @@ -91,13 +91,13 @@ func doDeleteManifestRequest(projectID int64, projectName, name, dgt string, nex return rr.Code } -func doPutManifestRequest(projectID int64, projectName, name, tag, dgt string, next ...http.HandlerFunc) int { +func doPutManifestRequest(projectID int64, projectName, name, tag, dgt string, withDupBlob bool, next ...http.HandlerFunc) int { repository := fmt.Sprintf("%s/%s", projectName, name) url := fmt.Sprintf("/v2/%s/manifests/%s", repository, tag) req, _ := http.NewRequest("PUT", url, nil) - ctx := util.NewManifestInfoContext(req.Context(), &util.ManifestInfo{ + mfInfo := &util.ManifestInfo{ ProjectID: projectID, Repository: repository, Tag: tag, @@ -106,7 +106,14 @@ func doPutManifestRequest(projectID int64, projectName, name, tag, dgt string, n {Digest: digest.FromString(randomString(15))}, {Digest: digest.FromString(randomString(15))}, }, - }) + } + + ctx := util.NewManifestInfoContext(req.Context(), mfInfo) + if withDupBlob { + dupDigest := digest.FromString(randomString(15)) + mfInfo.References = append(mfInfo.References, distribution.Descriptor{Digest: dupDigest}) + mfInfo.References = append(mfInfo.References, distribution.Descriptor{Digest: dupDigest}) + } rr := httptest.NewRecorder() @@ -165,7 +172,7 @@ func (suite *HandlerSuite) TestPutManifestCreated() { }() dgt := digest.FromString(randomString(15)).String() - code := doPutManifestRequest(projectID, projectName, "photon", "latest", dgt) + code := doPutManifestRequest(projectID, projectName, "photon", "latest", dgt, false) suite.Equal(http.StatusCreated, code) suite.checkCountUsage(1, projectID) @@ -174,7 +181,7 @@ func (suite *HandlerSuite) TestPutManifestCreated() { suite.Equal(int64(1), total, "Artifact should be created") // Push the photon:latest with photon:dev - code = doPutManifestRequest(projectID, projectName, "photon", "dev", dgt) + code = doPutManifestRequest(projectID, projectName, "photon", "dev", dgt, false) suite.Equal(http.StatusCreated, code) suite.checkCountUsage(2, projectID) @@ -184,7 +191,7 @@ func (suite *HandlerSuite) TestPutManifestCreated() { // Push the photon:latest with new image newDgt := digest.FromString(randomString(15)).String() - code = doPutManifestRequest(projectID, projectName, "photon", "latest", newDgt) + code = doPutManifestRequest(projectID, projectName, "photon", "latest", newDgt, false) suite.Equal(http.StatusCreated, code) suite.checkCountUsage(2, projectID) @@ -193,6 +200,26 @@ func (suite *HandlerSuite) TestPutManifestCreated() { suite.Equal(int64(1), total, "Artifact should be updated") } +func (suite *HandlerSuite) TestPutManifestCreatedDupBlobs() { + projectName := randomString(5) + + projectID := suite.addProject(projectName) + defer func() { + dao.DeleteProject(projectID) + }() + + dgt := digest.FromString(randomString(15)).String() + code := doPutManifestRequest(projectID, projectName, "photon", "latest", dgt, true) + suite.Equal(http.StatusCreated, code) + suite.checkCountUsage(1, projectID) + + var count int64 + err := dao.GetOrmer().Raw("select count(*) from artifact_blob where digest_af = ?", dgt).QueryRow(&count) + suite.Nil(err) + // 4 = self + 3 distinct blobs + suite.Equal(int64(4), count) +} + func (suite *HandlerSuite) TestPutManifestFailed() { projectName := randomString(5) @@ -206,7 +233,7 @@ func (suite *HandlerSuite) TestPutManifestFailed() { } dgt := digest.FromString(randomString(15)).String() - code := doPutManifestRequest(projectID, projectName, "photon", "latest", dgt, next) + code := doPutManifestRequest(projectID, projectName, "photon", "latest", dgt, false, next) suite.Equal(http.StatusForbidden, code) suite.checkCountUsage(0, projectID) @@ -224,7 +251,7 @@ func (suite *HandlerSuite) TestDeleteManifestAccepted() { }() dgt := digest.FromString(randomString(15)).String() - code := doPutManifestRequest(projectID, projectName, "photon", "latest", dgt) + code := doPutManifestRequest(projectID, projectName, "photon", "latest", dgt, false) suite.Equal(http.StatusCreated, code) suite.checkCountUsage(1, projectID) @@ -242,7 +269,7 @@ func (suite *HandlerSuite) TestDeleteManifestFailed() { }() dgt := digest.FromString(randomString(15)).String() - code := doPutManifestRequest(projectID, projectName, "photon", "latest", dgt) + code := doPutManifestRequest(projectID, projectName, "photon", "latest", dgt, false) suite.Equal(http.StatusCreated, code) suite.checkCountUsage(1, projectID) @@ -264,7 +291,7 @@ func (suite *HandlerSuite) TestDeleteManifestInMultiProjects() { }() dgt := digest.FromString(randomString(15)).String() - code := doPutManifestRequest(projectID, projectName, "photon", "latest", dgt) + code := doPutManifestRequest(projectID, projectName, "photon", "latest", dgt, false) suite.Equal(http.StatusCreated, code) suite.checkCountUsage(1, projectID) @@ -276,7 +303,7 @@ func (suite *HandlerSuite) TestDeleteManifestInMultiProjects() { dao.DeleteProject(projectID) }() - code := doPutManifestRequest(projectID, projectName, "photon", "latest", dgt) + code := doPutManifestRequest(projectID, projectName, "photon", "latest", dgt, false) suite.Equal(http.StatusCreated, code) suite.checkCountUsage(1, projectID) diff --git a/src/core/middlewares/countquota/util.go b/src/core/middlewares/countquota/util.go index 8275cb7ae..1d6701602 100644 --- a/src/core/middlewares/countquota/util.go +++ b/src/core/middlewares/countquota/util.go @@ -89,20 +89,26 @@ func afterManifestCreated(w http.ResponseWriter, req *http.Request) error { // attachBlobsToArtifact attach the blobs which from manifest to artifact func attachBlobsToArtifact(info *util.ManifestInfo) error { - self := &models.ArtifactAndBlob{ + temp := make(map[string]interface{}) + artifactBlobs := []*models.ArtifactAndBlob{} + + temp[info.Digest] = nil + // self + artifactBlobs = append(artifactBlobs, &models.ArtifactAndBlob{ DigestAF: info.Digest, DigestBlob: info.Digest, - } - - artifactBlobs := append([]*models.ArtifactAndBlob{}, self) + }) + // avoid the duplicate layers. for _, reference := range info.References { - artifactBlob := &models.ArtifactAndBlob{ - DigestAF: info.Digest, - DigestBlob: reference.Digest.String(), + _, exist := temp[reference.Digest.String()] + if !exist { + temp[reference.Digest.String()] = nil + artifactBlobs = append(artifactBlobs, &models.ArtifactAndBlob{ + DigestAF: info.Digest, + DigestBlob: reference.Digest.String(), + }) } - - artifactBlobs = append(artifactBlobs, artifactBlob) } if err := dao.AddArtifactNBlobs(artifactBlobs); err != nil {