From 65d658a05948c6c6d8eb9189db635f44777b7bed Mon Sep 17 00:00:00 2001
From: Wenkai Yin <yinw@vmware.com>
Date: Sun, 23 Aug 2020 21:31:17 +0800
Subject: [PATCH] retry several times when failed to copy blob during the
 replication

Signed-off-by: Wenkai Yin <yinw@vmware.com>
---
 src/replication/transfer/image/transfer.go | 34 +++++++++++++++++++---
 1 file changed, 30 insertions(+), 4 deletions(-)

diff --git a/src/replication/transfer/image/transfer.go b/src/replication/transfer/image/transfer.go
index 57ccf86ac..f8acfa7c9 100644
--- a/src/replication/transfer/image/transfer.go
+++ b/src/replication/transfer/image/transfer.go
@@ -17,7 +17,10 @@ package image
 import (
 	"errors"
 	"net/http"
+	"os"
+	"strconv"
 	"strings"
+	"time"
 
 	"github.com/docker/distribution/manifest/manifestlist"
 	"github.com/docker/distribution/manifest/schema1"
@@ -32,7 +35,15 @@ import (
 	trans "github.com/goharbor/harbor/src/replication/transfer"
 )
 
+var (
+	retry int
+)
+
 func init() {
+	retry, _ = strconv.Atoi(os.Getenv("COPY_BLOB_RETRY_COUNT"))
+	if retry <= 0 {
+		retry = 5
+	}
 	if err := trans.RegisterFactory(model.ResourceTypeImage, factory); err != nil {
 		log.Errorf("failed to register transfer factory: %v", err)
 	}
@@ -244,17 +255,33 @@ func (t *transfer) copyContent(content distribution.Descriptor, srcRepo, dstRepo
 	// the media type of the layer or config can be "application/octet-stream",
 	// schema1.MediaTypeManifestLayer, schema2.MediaTypeLayer, schema2.MediaTypeImageConfig
 	default:
-		return t.copyBlob(srcRepo, dstRepo, digest, content.Size)
+		return t.copyBlobWithRetry(srcRepo, dstRepo, digest, content.Size)
 	}
 }
 
+func (t *transfer) copyBlobWithRetry(srcRepo, dstRepo, digest string, sizeFromDescriptor int64) error {
+	var err error
+	for i, backoff := 1, 2*time.Second; i <= retry; i, backoff = i+1, backoff*2 {
+		t.logger.Infof("copying the blob %s(the %dth running)...", digest, i)
+		if err = t.copyBlob(srcRepo, dstRepo, digest, sizeFromDescriptor); err == nil {
+			t.logger.Infof("copy the blob %s completed", digest)
+			return nil
+		}
+		if i == retry {
+			break
+		}
+		t.logger.Infof("will retry %v later", backoff)
+		time.Sleep(backoff)
+	}
+	return err
+}
+
 // copy the layer or artifact config from the source registry to destination
 // the size parameter is taken from manifests.
 func (t *transfer) copyBlob(srcRepo, dstRepo, digest string, sizeFromDescriptor int64) error {
 	if t.shouldStop() {
 		return nil
 	}
-	t.logger.Infof("copying the blob %s...", digest)
 	exist, err := t.dst.BlobExist(dstRepo, digest)
 	if err != nil {
 		t.logger.Errorf("failed to check the existence of blob %s on the destination registry: %v", digest, err)
@@ -278,10 +305,9 @@ func (t *transfer) copyBlob(srcRepo, dstRepo, digest string, sizeFromDescriptor
 	}
 
 	if err = t.dst.PushBlob(dstRepo, digest, size, data); err != nil {
-		t.logger.Errorf("failed to pushing the blob %s: %v", digest, err)
+		t.logger.Errorf("failed to pushing the blob %s, size %d: %v", digest, size, err)
 		return err
 	}
-	t.logger.Infof("copy the blob %s completed", digest)
 	return nil
 }