Merge pull request #8819 from wy65701436/gc-clean-pb

fix #8815 :add remove untagged blob record in table project_blob
This commit is contained in:
Wang Yan 2019-08-26 17:18:30 +08:00 committed by GitHub
commit 3868d54b5c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 205 additions and 0 deletions

View File

@ -134,3 +134,57 @@ WHERE af.project_id = ?
return size, err return size, err
} }
// RemoveUntaggedBlobs ...
func RemoveUntaggedBlobs(pid int64) error {
var blobs []models.Blob
sql := `
SELECT
DISTINCT bb.digest,
bb.id,
bb.content_type,
bb.size,
bb.creation_time
FROM artifact af
JOIN artifact_blob afnb
ON af.digest = afnb.digest_af
JOIN BLOB bb
ON afnb.digest_blob = bb.digest
WHERE af.project_id = ?
`
_, err := GetOrmer().Raw(sql, pid).QueryRows(&blobs)
if len(blobs) == 0 {
sql = fmt.Sprintf(`DELETE FROM project_blob WHERE project_id = ?`)
_, err = GetOrmer().Raw(sql, pid).Exec()
if err != nil {
return err
}
return nil
}
var bbIDs []interface{}
for _, bb := range blobs {
bbIDs = append(bbIDs, bb.ID)
}
var projectBlobs []*models.ProjectBlob
sql = fmt.Sprintf(`SELECT * FROM project_blob AS pb WHERE project_id = ? AND pb.blob_id NOT IN (%s)`, ParamPlaceholderForIn(len(bbIDs)))
_, err = GetOrmer().Raw(sql, pid, bbIDs).QueryRows(&projectBlobs)
if err != nil {
return err
}
var pbIDs []interface{}
for _, pb := range projectBlobs {
pbIDs = append(pbIDs, pb.ID)
}
if len(pbIDs) == 0 {
return nil
}
sql = fmt.Sprintf(`DELETE FROM project_blob WHERE id IN (%s)`, ParamPlaceholderForIn(len(pbIDs)))
_, err = GetOrmer().Raw(sql, pbIDs).Exec()
if err != nil {
return err
}
return nil
}

View File

@ -196,3 +196,150 @@ func TestCountSizeOfProjectDupdigest(t *testing.T) {
pSize, err := CountSizeOfProject(pid1) pSize, err := CountSizeOfProject(pid1)
assert.Equal(t, pSize, int64(1010)) assert.Equal(t, pSize, int64(1010))
} }
func TestRemoveUntaggedBlobs(t *testing.T) {
pid1, err := AddProject(models.Project{
Name: "RemoveUntaggedBlobs_project1",
OwnerID: 1,
})
require.Nil(t, err)
_, blob1, err := GetOrCreateBlob(&models.Blob{
Digest: digest.FromString(utils.GenerateRandomString()).String(),
Size: 100,
})
require.Nil(t, err)
_, blob2, err := GetOrCreateBlob(&models.Blob{
Digest: digest.FromString(utils.GenerateRandomString()).String(),
Size: 100,
})
require.Nil(t, err)
_, err = AddBlobToProject(blob1.ID, pid1)
require.Nil(t, err)
_, err = AddBlobToProject(blob2.ID, pid1)
require.Nil(t, err)
has, err := HasBlobInProject(pid1, blob1.Digest)
require.Nil(t, err)
assert.True(t, has)
has, err = HasBlobInProject(pid1, blob2.Digest)
require.Nil(t, err)
assert.True(t, has)
err = RemoveUntaggedBlobs(pid1)
require.Nil(t, err)
has, err = HasBlobInProject(pid1, blob1.Digest)
require.Nil(t, err)
assert.False(t, has)
has, err = HasBlobInProject(pid1, blob2.Digest)
require.Nil(t, err)
assert.False(t, has)
}
func TestRemoveUntaggedBlobsWithNoUntagged(t *testing.T) {
afDigest := digest.FromString(utils.GenerateRandomString()).String()
af := &models.Artifact{
PID: 333,
Repo: "hello-world",
Tag: "latest",
Digest: afDigest,
Kind: "image",
}
_, err := AddArtifact(af)
require.Nil(t, err)
blob1Digest := digest.FromString(utils.GenerateRandomString()).String()
blob1 := &models.Blob{
Digest: blob1Digest,
ContentType: "v2.blob",
Size: 1523,
}
_, err = AddBlob(blob1)
require.Nil(t, err)
blob2Digest := digest.FromString(utils.GenerateRandomString()).String()
blob2 := &models.Blob{
Digest: blob2Digest,
ContentType: "v2.blob",
Size: 1523,
}
_, err = AddBlob(blob2)
require.Nil(t, err)
blob3Digest := digest.FromString(utils.GenerateRandomString()).String()
blob3 := &models.Blob{
Digest: blob3Digest,
ContentType: "v2.blob",
Size: 1523,
}
_, err = AddBlob(blob3)
require.Nil(t, err)
afnb1 := &models.ArtifactAndBlob{
DigestAF: afDigest,
DigestBlob: blob1Digest,
}
afnb2 := &models.ArtifactAndBlob{
DigestAF: afDigest,
DigestBlob: blob2Digest,
}
afnb3 := &models.ArtifactAndBlob{
DigestAF: afDigest,
DigestBlob: blob3Digest,
}
var afnbs []*models.ArtifactAndBlob
afnbs = append(afnbs, afnb1)
afnbs = append(afnbs, afnb2)
afnbs = append(afnbs, afnb3)
err = AddArtifactNBlobs(afnbs)
require.Nil(t, err)
_, err = AddBlobToProject(blob1.ID, 333)
require.Nil(t, err)
_, err = AddBlobToProject(blob2.ID, 333)
require.Nil(t, err)
_, err = AddBlobToProject(blob3.ID, 333)
require.Nil(t, err)
blobUntaggedDigest := digest.FromString(utils.GenerateRandomString()).String()
blobUntagged := &models.Blob{
Digest: blobUntaggedDigest,
ContentType: "v2.blob",
Size: 1523,
}
_, err = AddBlob(blobUntagged)
require.Nil(t, err)
_, err = AddBlobToProject(blobUntagged.ID, 333)
require.Nil(t, err)
err = RemoveUntaggedBlobs(333)
require.Nil(t, err)
has, err := HasBlobInProject(333, blob1.Digest)
require.Nil(t, err)
assert.True(t, has)
has, err = HasBlobInProject(333, blob2.Digest)
require.Nil(t, err)
assert.True(t, has)
has, err = HasBlobInProject(333, blob3.Digest)
require.Nil(t, err)
assert.True(t, has)
has, err = HasBlobInProject(333, blobUntagged.Digest)
require.Nil(t, err)
assert.False(t, has)
}

View File

@ -221,6 +221,10 @@ func (gc *GarbageCollector) ensureQuota() error {
gc.logger.Errorf("cannot ensure quota for the project: %d, err: %v, just skip it.", project.ProjectID, err) gc.logger.Errorf("cannot ensure quota for the project: %d, err: %v, just skip it.", project.ProjectID, err)
continue continue
} }
if err := dao.RemoveUntaggedBlobs(project.ProjectID); err != nil {
gc.logger.Errorf("cannot delete untagged blobs of project: %d, err: %v, just skip it.", project.ProjectID, err)
continue
}
} }
return nil return nil
} }