diff --git a/src/jobservice/job/impl/gc/garbage_collection.go b/src/jobservice/job/impl/gc/garbage_collection.go index 07d38b48a..94ed29bff 100644 --- a/src/jobservice/job/impl/gc/garbage_collection.go +++ b/src/jobservice/job/impl/gc/garbage_collection.go @@ -15,6 +15,7 @@ package gc import ( + "github.com/goharbor/harbor/src/lib" "os" "time" @@ -270,9 +271,13 @@ func (gc *GarbageCollector) sweep(ctx job.Context) error { } // for manifest, it has to delete the revisions folder of each repository gc.logger.Infof("delete manifest from storage: %s", blob.Digest) - if err := ignoreNotFound(func() error { - return gc.registryCtlClient.DeleteManifest(art.RepositoryName, blob.Digest) - }); err != nil { + if err := lib.RetryUntil(func() error { + return ignoreNotFound(func() error { + return gc.registryCtlClient.DeleteManifest(art.RepositoryName, blob.Digest) + }) + }, lib.RetryCallback(func(err error, sleep time.Duration) { + gc.logger.Infof("failed to exec DeleteManifest retry after %s : %v", sleep, err) + })); err != nil { if err := ignoreNotFound(func() error { return gc.markDeleteFailed(ctx, blob) }); err != nil { @@ -294,9 +299,13 @@ func (gc *GarbageCollector) sweep(ctx job.Context) error { // for the foreign layer, as it's not stored in the storage, no need to call the delete api and count size, but still have to delete the DB record. if !blob.IsForeignLayer() { gc.logger.Infof("delete blob from storage: %s", blob.Digest) - if err := ignoreNotFound(func() error { - return gc.registryCtlClient.DeleteBlob(blob.Digest) - }); err != nil { + if err := lib.RetryUntil(func() error { + return ignoreNotFound(func() error { + return gc.registryCtlClient.DeleteBlob(blob.Digest) + }) + }, lib.RetryCallback(func(err error, sleep time.Duration) { + gc.logger.Infof("failed to exec DeleteBlob retry after %s : %v", sleep, err) + })); err != nil { if err := ignoreNotFound(func() error { return gc.markDeleteFailed(ctx, blob) }); err != nil { diff --git a/src/jobservice/job/impl/gc/util.go b/src/jobservice/job/impl/gc/util.go index f021df2f5..e6466f1e0 100644 --- a/src/jobservice/job/impl/gc/util.go +++ b/src/jobservice/job/impl/gc/util.go @@ -2,9 +2,11 @@ package gc import ( "fmt" + "github.com/goharbor/harbor/src/lib" "github.com/goharbor/harbor/src/lib/errors" "github.com/goharbor/harbor/src/pkg/registry" "github.com/gomodule/redigo/redis" + "time" ) // delKeys ... @@ -50,10 +52,11 @@ func v2DeleteManifest(repository, digest string) error { if !exist { return nil } - if err := registry.Cli.DeleteManifest(repository, digest); err != nil { - return err - } - return nil + return lib.RetryUntil(func() error { + return registry.Cli.DeleteManifest(repository, digest) + }, lib.RetryCallback(func(err error, sleep time.Duration) { + fmt.Printf("failed to exec DeleteManifest retry after %s : %v\n", sleep, err) + })) } // ignoreNotFound ignores the NotFoundErr error