From e3155e00d68cb7959e58267b764b0e1ea1c4586c Mon Sep 17 00:00:00 2001 From: wang yan Date: Fri, 23 Aug 2019 18:33:16 +0800 Subject: [PATCH] fix #8815 :add remove untagged blob record in table project_blob Signed-off-by: wang yan --- src/common/dao/project_blob.go | 54 ++++++++++ src/common/dao/project_blob_test.go | 147 ++++++++++++++++++++++++++++ src/jobservice/job/impl/gc/job.go | 4 + 3 files changed, 205 insertions(+) diff --git a/src/common/dao/project_blob.go b/src/common/dao/project_blob.go index bb4b59a10..5e9c1f557 100644 --- a/src/common/dao/project_blob.go +++ b/src/common/dao/project_blob.go @@ -134,3 +134,57 @@ WHERE af.project_id = ? return size, err } + +// RemoveUntaggedBlobs ... +func RemoveUntaggedBlobs(pid int64) error { + var blobs []models.Blob + sql := ` +SELECT + DISTINCT bb.digest, + bb.id, + bb.content_type, + bb.size, + bb.creation_time +FROM artifact af +JOIN artifact_blob afnb + ON af.digest = afnb.digest_af +JOIN BLOB bb + ON afnb.digest_blob = bb.digest +WHERE af.project_id = ? +` + _, err := GetOrmer().Raw(sql, pid).QueryRows(&blobs) + if len(blobs) == 0 { + sql = fmt.Sprintf(`DELETE FROM project_blob WHERE project_id = ?`) + _, err = GetOrmer().Raw(sql, pid).Exec() + if err != nil { + return err + } + return nil + } + + var bbIDs []interface{} + for _, bb := range blobs { + bbIDs = append(bbIDs, bb.ID) + } + var projectBlobs []*models.ProjectBlob + sql = fmt.Sprintf(`SELECT * FROM project_blob AS pb WHERE project_id = ? AND pb.blob_id NOT IN (%s)`, ParamPlaceholderForIn(len(bbIDs))) + _, err = GetOrmer().Raw(sql, pid, bbIDs).QueryRows(&projectBlobs) + if err != nil { + return err + } + + var pbIDs []interface{} + for _, pb := range projectBlobs { + pbIDs = append(pbIDs, pb.ID) + } + if len(pbIDs) == 0 { + return nil + } + sql = fmt.Sprintf(`DELETE FROM project_blob WHERE id IN (%s)`, ParamPlaceholderForIn(len(pbIDs))) + _, err = GetOrmer().Raw(sql, pbIDs).Exec() + if err != nil { + return err + } + + return nil +} diff --git a/src/common/dao/project_blob_test.go b/src/common/dao/project_blob_test.go index dec4c5fab..2b93d898d 100644 --- a/src/common/dao/project_blob_test.go +++ b/src/common/dao/project_blob_test.go @@ -196,3 +196,150 @@ func TestCountSizeOfProjectDupdigest(t *testing.T) { pSize, err := CountSizeOfProject(pid1) assert.Equal(t, pSize, int64(1010)) } + +func TestRemoveUntaggedBlobs(t *testing.T) { + + pid1, err := AddProject(models.Project{ + Name: "RemoveUntaggedBlobs_project1", + OwnerID: 1, + }) + require.Nil(t, err) + + _, blob1, err := GetOrCreateBlob(&models.Blob{ + Digest: digest.FromString(utils.GenerateRandomString()).String(), + Size: 100, + }) + require.Nil(t, err) + + _, blob2, err := GetOrCreateBlob(&models.Blob{ + Digest: digest.FromString(utils.GenerateRandomString()).String(), + Size: 100, + }) + require.Nil(t, err) + + _, err = AddBlobToProject(blob1.ID, pid1) + require.Nil(t, err) + + _, err = AddBlobToProject(blob2.ID, pid1) + require.Nil(t, err) + + has, err := HasBlobInProject(pid1, blob1.Digest) + require.Nil(t, err) + assert.True(t, has) + + has, err = HasBlobInProject(pid1, blob2.Digest) + require.Nil(t, err) + assert.True(t, has) + + err = RemoveUntaggedBlobs(pid1) + require.Nil(t, err) + + has, err = HasBlobInProject(pid1, blob1.Digest) + require.Nil(t, err) + assert.False(t, has) + + has, err = HasBlobInProject(pid1, blob2.Digest) + require.Nil(t, err) + assert.False(t, has) + +} + +func TestRemoveUntaggedBlobsWithNoUntagged(t *testing.T) { + afDigest := digest.FromString(utils.GenerateRandomString()).String() + af := &models.Artifact{ + PID: 333, + Repo: "hello-world", + Tag: "latest", + Digest: afDigest, + Kind: "image", + } + _, err := AddArtifact(af) + require.Nil(t, err) + + blob1Digest := digest.FromString(utils.GenerateRandomString()).String() + blob1 := &models.Blob{ + Digest: blob1Digest, + ContentType: "v2.blob", + Size: 1523, + } + _, err = AddBlob(blob1) + require.Nil(t, err) + + blob2Digest := digest.FromString(utils.GenerateRandomString()).String() + blob2 := &models.Blob{ + Digest: blob2Digest, + ContentType: "v2.blob", + Size: 1523, + } + _, err = AddBlob(blob2) + require.Nil(t, err) + + blob3Digest := digest.FromString(utils.GenerateRandomString()).String() + blob3 := &models.Blob{ + Digest: blob3Digest, + ContentType: "v2.blob", + Size: 1523, + } + _, err = AddBlob(blob3) + require.Nil(t, err) + + afnb1 := &models.ArtifactAndBlob{ + DigestAF: afDigest, + DigestBlob: blob1Digest, + } + afnb2 := &models.ArtifactAndBlob{ + DigestAF: afDigest, + DigestBlob: blob2Digest, + } + afnb3 := &models.ArtifactAndBlob{ + DigestAF: afDigest, + DigestBlob: blob3Digest, + } + var afnbs []*models.ArtifactAndBlob + afnbs = append(afnbs, afnb1) + afnbs = append(afnbs, afnb2) + afnbs = append(afnbs, afnb3) + + err = AddArtifactNBlobs(afnbs) + require.Nil(t, err) + + _, err = AddBlobToProject(blob1.ID, 333) + require.Nil(t, err) + + _, err = AddBlobToProject(blob2.ID, 333) + require.Nil(t, err) + + _, err = AddBlobToProject(blob3.ID, 333) + require.Nil(t, err) + + blobUntaggedDigest := digest.FromString(utils.GenerateRandomString()).String() + blobUntagged := &models.Blob{ + Digest: blobUntaggedDigest, + ContentType: "v2.blob", + Size: 1523, + } + _, err = AddBlob(blobUntagged) + require.Nil(t, err) + + _, err = AddBlobToProject(blobUntagged.ID, 333) + require.Nil(t, err) + + err = RemoveUntaggedBlobs(333) + require.Nil(t, err) + + has, err := HasBlobInProject(333, blob1.Digest) + require.Nil(t, err) + assert.True(t, has) + + has, err = HasBlobInProject(333, blob2.Digest) + require.Nil(t, err) + assert.True(t, has) + + has, err = HasBlobInProject(333, blob3.Digest) + require.Nil(t, err) + assert.True(t, has) + + has, err = HasBlobInProject(333, blobUntagged.Digest) + require.Nil(t, err) + assert.False(t, has) +} diff --git a/src/jobservice/job/impl/gc/job.go b/src/jobservice/job/impl/gc/job.go index 6c6b5f82b..a946e201a 100644 --- a/src/jobservice/job/impl/gc/job.go +++ b/src/jobservice/job/impl/gc/job.go @@ -221,6 +221,10 @@ func (gc *GarbageCollector) ensureQuota() error { gc.logger.Errorf("cannot ensure quota for the project: %d, err: %v, just skip it.", project.ProjectID, err) continue } + if err := dao.RemoveUntaggedBlobs(project.ProjectID); err != nil { + gc.logger.Errorf("cannot delete untagged blobs of project: %d, err: %v, just skip it.", project.ProjectID, err) + continue + } } return nil }