mirror of
https://github.com/goharbor/harbor
synced 2025-05-06 02:07:14 +00:00
fix: support customize cache db for business (#19182)
Support to configure the customized redis db for cache layer and other misc business for core, by default the behavior is same with previous(stored in db 0). Signed-off-by: chlins <chenyuzh@vmware.com>
This commit is contained in:
parent
83ff2b277a
commit
c7e25295fe
@ -167,6 +167,28 @@ _version: 2.9.0
|
|||||||
# max_idle_conns: 2
|
# max_idle_conns: 2
|
||||||
# max_open_conns: 0
|
# max_open_conns: 0
|
||||||
|
|
||||||
|
# Uncomment redis if need to customize redis db
|
||||||
|
# redis:
|
||||||
|
# # db_index 0 is for core, it's unchangeable
|
||||||
|
# # registry_db_index: 1
|
||||||
|
# # jobservice_db_index: 2
|
||||||
|
# # trivy_db_index: 5
|
||||||
|
# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
|
||||||
|
# # harbor_db_index: 6
|
||||||
|
# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
|
||||||
|
# # cache_db_index: 7
|
||||||
|
|
||||||
|
# Uncomment redis if need to customize redis db
|
||||||
|
# redis:
|
||||||
|
# # db_index 0 is for core, it's unchangeable
|
||||||
|
# # registry_db_index: 1
|
||||||
|
# # jobservice_db_index: 2
|
||||||
|
# # trivy_db_index: 5
|
||||||
|
# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
|
||||||
|
# # harbor_db_index: 6
|
||||||
|
# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
|
||||||
|
# # cache_layer_db_index: 7
|
||||||
|
|
||||||
# Uncomment external_redis if using external Redis server
|
# Uncomment external_redis if using external Redis server
|
||||||
# external_redis:
|
# external_redis:
|
||||||
# # support redis, redis+sentinel
|
# # support redis, redis+sentinel
|
||||||
@ -186,6 +208,10 @@ _version: 2.9.0
|
|||||||
# jobservice_db_index: 2
|
# jobservice_db_index: 2
|
||||||
# trivy_db_index: 5
|
# trivy_db_index: 5
|
||||||
# idle_timeout_seconds: 30
|
# idle_timeout_seconds: 30
|
||||||
|
# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
|
||||||
|
# # harbor_db_index: 6
|
||||||
|
# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
|
||||||
|
# # cache_layer_db_index: 7
|
||||||
|
|
||||||
# Uncomment uaa for trusting the certificate of uaa instance that is hosted via self-signed cert.
|
# Uncomment uaa for trusting the certificate of uaa instance that is hosted via self-signed cert.
|
||||||
# uaa:
|
# uaa:
|
||||||
|
@ -371,6 +371,49 @@ external_database:
|
|||||||
# ssl_mode: disable
|
# ssl_mode: disable
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
|
||||||
|
{% if redis is defined %}
|
||||||
|
redis:
|
||||||
|
# # db_index 0 is for core, it's unchangeable
|
||||||
|
{% if redis.registry_db_index is defined %}
|
||||||
|
registry_db_index: {{ redis.registry_db_index }}
|
||||||
|
{% else %}
|
||||||
|
# # registry_db_index: 1
|
||||||
|
{% endif %}
|
||||||
|
{% if redis.jobservice_db_index is defined %}
|
||||||
|
jobservice_db_index: {{ redis.jobservice_db_index }}
|
||||||
|
{% else %}
|
||||||
|
# # jobservice_db_index: 2
|
||||||
|
{% endif %}
|
||||||
|
{% if redis.trivy_db_index is defined %}
|
||||||
|
trivy_db_index: {{ redis.trivy_db_index }}
|
||||||
|
{% else %}
|
||||||
|
# # trivy_db_index: 5
|
||||||
|
{% endif %}
|
||||||
|
{% if redis.harbor_db_index is defined %}
|
||||||
|
harbor_db_index: {{ redis.harbor_db_index }}
|
||||||
|
{% else %}
|
||||||
|
# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
|
||||||
|
# # harbor_db_index: 6
|
||||||
|
{% endif %}
|
||||||
|
{% if redis.cache_layer_db_index is defined %}
|
||||||
|
cache_layer_db_index: {{ redis.cache_layer_db_index }}
|
||||||
|
{% else %}
|
||||||
|
# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
|
||||||
|
# # cache_layer_db_index: 7
|
||||||
|
{% endif %}
|
||||||
|
{% else %}
|
||||||
|
# Uncomment redis if need to customize redis db
|
||||||
|
# redis:
|
||||||
|
# # db_index 0 is for core, it's unchangeable
|
||||||
|
# # registry_db_index: 1
|
||||||
|
# # jobservice_db_index: 2
|
||||||
|
# # trivy_db_index: 5
|
||||||
|
# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
|
||||||
|
# # harbor_db_index: 6
|
||||||
|
# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
|
||||||
|
# # cache_layer_db_index: 7
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
{% if external_redis is defined %}
|
{% if external_redis is defined %}
|
||||||
external_redis:
|
external_redis:
|
||||||
# support redis, redis+sentinel
|
# support redis, redis+sentinel
|
||||||
@ -390,6 +433,18 @@ external_redis:
|
|||||||
jobservice_db_index: {{ external_redis.jobservice_db_index }}
|
jobservice_db_index: {{ external_redis.jobservice_db_index }}
|
||||||
trivy_db_index: 5
|
trivy_db_index: 5
|
||||||
idle_timeout_seconds: 30
|
idle_timeout_seconds: 30
|
||||||
|
{% if external_redis.harbor_db_index is defined %}
|
||||||
|
harbor_db_index: {{ redis.harbor_db_index }}
|
||||||
|
{% else %}
|
||||||
|
# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
|
||||||
|
# # harbor_db_index: 6
|
||||||
|
{% endif %}
|
||||||
|
{% if external_redis.cache_layer_db_index is defined %}
|
||||||
|
cache_layer_db_index: {{ redis.cache_layer_db_index }}
|
||||||
|
{% else %}
|
||||||
|
# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
|
||||||
|
# # cache_layer_db_index: 7
|
||||||
|
{% endif %}
|
||||||
{% else %}
|
{% else %}
|
||||||
# Umcomments external_redis if using external Redis server
|
# Umcomments external_redis if using external Redis server
|
||||||
# external_redis:
|
# external_redis:
|
||||||
@ -406,6 +461,10 @@ external_redis:
|
|||||||
# jobservice_db_index: 2
|
# jobservice_db_index: 2
|
||||||
# trivy_db_index: 5
|
# trivy_db_index: 5
|
||||||
# idle_timeout_seconds: 30
|
# idle_timeout_seconds: 30
|
||||||
|
# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
|
||||||
|
# # harbor_db_index: 6
|
||||||
|
# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
|
||||||
|
# # cache_layer_db_index: 7
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
|
||||||
{% if uaa is defined %}
|
{% if uaa is defined %}
|
||||||
|
@ -381,6 +381,49 @@ external_database:
|
|||||||
# ssl_mode: disable
|
# ssl_mode: disable
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
|
||||||
|
{% if redis is defined %}
|
||||||
|
redis:
|
||||||
|
# # db_index 0 is for core, it's unchangeable
|
||||||
|
{% if redis.registry_db_index is defined %}
|
||||||
|
registry_db_index: {{ redis.registry_db_index }}
|
||||||
|
{% else %}
|
||||||
|
# # registry_db_index: 1
|
||||||
|
{% endif %}
|
||||||
|
{% if redis.jobservice_db_index is defined %}
|
||||||
|
jobservice_db_index: {{ redis.jobservice_db_index }}
|
||||||
|
{% else %}
|
||||||
|
# # jobservice_db_index: 2
|
||||||
|
{% endif %}
|
||||||
|
{% if redis.trivy_db_index is defined %}
|
||||||
|
trivy_db_index: {{ redis.trivy_db_index }}
|
||||||
|
{% else %}
|
||||||
|
# # trivy_db_index: 5
|
||||||
|
{% endif %}
|
||||||
|
{% if redis.harbor_db_index is defined %}
|
||||||
|
harbor_db_index: {{ redis.harbor_db_index }}
|
||||||
|
{% else %}
|
||||||
|
# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
|
||||||
|
# # harbor_db_index: 6
|
||||||
|
{% endif %}
|
||||||
|
{% if redis.cache_layer_db_index is defined %}
|
||||||
|
cache_layer_db_index: {{ redis.cache_layer_db_index }}
|
||||||
|
{% else %}
|
||||||
|
# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
|
||||||
|
# # cache_layer_db_index: 7
|
||||||
|
{% endif %}
|
||||||
|
{% else %}
|
||||||
|
# Uncomment redis if need to customize redis db
|
||||||
|
# redis:
|
||||||
|
# # db_index 0 is for core, it's unchangeable
|
||||||
|
# # registry_db_index: 1
|
||||||
|
# # jobservice_db_index: 2
|
||||||
|
# # trivy_db_index: 5
|
||||||
|
# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
|
||||||
|
# # harbor_db_index: 6
|
||||||
|
# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
|
||||||
|
# # cache_layer_db_index: 7
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
{% if external_redis is defined %}
|
{% if external_redis is defined %}
|
||||||
external_redis:
|
external_redis:
|
||||||
# support redis, redis+sentinel
|
# support redis, redis+sentinel
|
||||||
@ -406,6 +449,18 @@ external_redis:
|
|||||||
jobservice_db_index: {{ external_redis.jobservice_db_index }}
|
jobservice_db_index: {{ external_redis.jobservice_db_index }}
|
||||||
trivy_db_index: 5
|
trivy_db_index: 5
|
||||||
idle_timeout_seconds: 30
|
idle_timeout_seconds: 30
|
||||||
|
{% if external_redis.harbor_db_index is defined %}
|
||||||
|
harbor_db_index: {{ redis.harbor_db_index }}
|
||||||
|
{% else %}
|
||||||
|
# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
|
||||||
|
# # harbor_db_index: 6
|
||||||
|
{% endif %}
|
||||||
|
{% if external_redis.cache_layer_db_index is defined %}
|
||||||
|
cache_layer_db_index: {{ redis.cache_layer_db_index }}
|
||||||
|
{% else %}
|
||||||
|
# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
|
||||||
|
# # cache_layer_db_index: 7
|
||||||
|
{% endif %}
|
||||||
{% else %}
|
{% else %}
|
||||||
# Umcomments external_redis if using external Redis server
|
# Umcomments external_redis if using external Redis server
|
||||||
# external_redis:
|
# external_redis:
|
||||||
@ -424,6 +479,10 @@ external_redis:
|
|||||||
# jobservice_db_index: 2
|
# jobservice_db_index: 2
|
||||||
# trivy_db_index: 5
|
# trivy_db_index: 5
|
||||||
# idle_timeout_seconds: 30
|
# idle_timeout_seconds: 30
|
||||||
|
# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
|
||||||
|
# # harbor_db_index: 6
|
||||||
|
# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
|
||||||
|
# # cache_layer_db_index: 7
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
|
||||||
{% if uaa is defined %}
|
{% if uaa is defined %}
|
||||||
|
@ -378,6 +378,49 @@ external_database:
|
|||||||
# max_open_conns: 0
|
# max_open_conns: 0
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
|
||||||
|
{% if redis is defined %}
|
||||||
|
redis:
|
||||||
|
# # db_index 0 is for core, it's unchangeable
|
||||||
|
{% if redis.registry_db_index is defined %}
|
||||||
|
registry_db_index: {{ redis.registry_db_index }}
|
||||||
|
{% else %}
|
||||||
|
# # registry_db_index: 1
|
||||||
|
{% endif %}
|
||||||
|
{% if redis.jobservice_db_index is defined %}
|
||||||
|
jobservice_db_index: {{ redis.jobservice_db_index }}
|
||||||
|
{% else %}
|
||||||
|
# # jobservice_db_index: 2
|
||||||
|
{% endif %}
|
||||||
|
{% if redis.trivy_db_index is defined %}
|
||||||
|
trivy_db_index: {{ redis.trivy_db_index }}
|
||||||
|
{% else %}
|
||||||
|
# # trivy_db_index: 5
|
||||||
|
{% endif %}
|
||||||
|
{% if redis.harbor_db_index is defined %}
|
||||||
|
harbor_db_index: {{ redis.harbor_db_index }}
|
||||||
|
{% else %}
|
||||||
|
# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
|
||||||
|
# # harbor_db_index: 6
|
||||||
|
{% endif %}
|
||||||
|
{% if redis.cache_layer_db_index is defined %}
|
||||||
|
cache_layer_db_index: {{ redis.cache_layer_db_index }}
|
||||||
|
{% else %}
|
||||||
|
# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
|
||||||
|
# # cache_layer_db_index: 7
|
||||||
|
{% endif %}
|
||||||
|
{% else %}
|
||||||
|
# Uncomment redis if need to customize redis db
|
||||||
|
# redis:
|
||||||
|
# # db_index 0 is for core, it's unchangeable
|
||||||
|
# # registry_db_index: 1
|
||||||
|
# # jobservice_db_index: 2
|
||||||
|
# # trivy_db_index: 5
|
||||||
|
# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
|
||||||
|
# # harbor_db_index: 6
|
||||||
|
# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
|
||||||
|
# # cache_layer_db_index: 7
|
||||||
|
{% endif %}
|
||||||
|
|
||||||
{% if external_redis is defined %}
|
{% if external_redis is defined %}
|
||||||
external_redis:
|
external_redis:
|
||||||
# support redis, redis+sentinel
|
# support redis, redis+sentinel
|
||||||
@ -399,6 +442,18 @@ external_redis:
|
|||||||
jobservice_db_index: {{ external_redis.jobservice_db_index }}
|
jobservice_db_index: {{ external_redis.jobservice_db_index }}
|
||||||
trivy_db_index: 5
|
trivy_db_index: 5
|
||||||
idle_timeout_seconds: 30
|
idle_timeout_seconds: 30
|
||||||
|
{% if external_redis.harbor_db_index is defined %}
|
||||||
|
harbor_db_index: {{ redis.harbor_db_index }}
|
||||||
|
{% else %}
|
||||||
|
# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
|
||||||
|
# # harbor_db_index: 6
|
||||||
|
{% endif %}
|
||||||
|
{% if external_redis.cache_layer_db_index is defined %}
|
||||||
|
cache_layer_db_index: {{ redis.cache_layer_db_index }}
|
||||||
|
{% else %}
|
||||||
|
# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
|
||||||
|
# # cache_layer_db_index: 7
|
||||||
|
{% endif %}
|
||||||
{% else %}
|
{% else %}
|
||||||
# Umcomments external_redis if using external Redis server
|
# Umcomments external_redis if using external Redis server
|
||||||
# external_redis:
|
# external_redis:
|
||||||
@ -417,6 +472,10 @@ external_redis:
|
|||||||
# jobservice_db_index: 2
|
# jobservice_db_index: 2
|
||||||
# trivy_db_index: 5
|
# trivy_db_index: 5
|
||||||
# idle_timeout_seconds: 30
|
# idle_timeout_seconds: 30
|
||||||
|
# # it's optional, the db for harbor business misc, by default is 0, uncomment it if you want to change it.
|
||||||
|
# # harbor_db_index: 6
|
||||||
|
# # it's optional, the db for harbor cache layer, by default is 0, uncomment it if you want to change it.
|
||||||
|
# # cache_layer_db_index: 7
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
|
||||||
{% if uaa is defined %}
|
{% if uaa is defined %}
|
||||||
|
@ -1,6 +1,9 @@
|
|||||||
CONFIG_PATH=/etc/core/app.conf
|
CONFIG_PATH=/etc/core/app.conf
|
||||||
UAA_CA_ROOT=/etc/core/certificates/uaa_ca.pem
|
UAA_CA_ROOT=/etc/core/certificates/uaa_ca.pem
|
||||||
_REDIS_URL_CORE={{redis_url_core}}
|
_REDIS_URL_CORE={{redis_url_core}}
|
||||||
|
{% if redis_url_harbor %}
|
||||||
|
_REDIS_URL_HARBOR={{redis_url_harbor}}
|
||||||
|
{% endif %}
|
||||||
SYNC_QUOTA=true
|
SYNC_QUOTA=true
|
||||||
_REDIS_URL_REG={{redis_url_reg}}
|
_REDIS_URL_REG={{redis_url_reg}}
|
||||||
|
|
||||||
@ -84,6 +87,9 @@ TRACE_OTEL_INSECURE={{ trace.otel.insecure }}
|
|||||||
{% endif %}
|
{% endif %}
|
||||||
|
|
||||||
{% if cache.enabled %}
|
{% if cache.enabled %}
|
||||||
|
{% if redis_url_cache_layer %}
|
||||||
|
_REDIS_URL_CACHE_LAYER={{redis_url_cache_layer}}
|
||||||
|
{% endif %}
|
||||||
CACHE_ENABLED=true
|
CACHE_ENABLED=true
|
||||||
CACHE_EXPIRE_HOURS={{ cache.expire_hours }}
|
CACHE_EXPIRE_HOURS={{ cache.expire_hours }}
|
||||||
{% endif %}
|
{% endif %}
|
||||||
|
@ -51,6 +51,9 @@ TRACE_OTEL_INSECURE={{ trace.otel.insecure }}
|
|||||||
|
|
||||||
{% if cache.enabled %}
|
{% if cache.enabled %}
|
||||||
_REDIS_URL_CORE={{redis_url_core}}
|
_REDIS_URL_CORE={{redis_url_core}}
|
||||||
|
{% if redis_url_cache_layer %}
|
||||||
|
_REDIS_URL_CACHE_LAYER={{redis_url_cache_layer}}
|
||||||
|
{% endif %}
|
||||||
CACHE_ENABLED=true
|
CACHE_ENABLED=true
|
||||||
CACHE_EXPIRE_HOURS={{ cache.expire_hours }}
|
CACHE_EXPIRE_HOURS={{ cache.expire_hours }}
|
||||||
{% endif %}
|
{% endif %}
|
@ -275,7 +275,7 @@ def parse_yaml_config(config_file_path, with_trivy):
|
|||||||
config_dict['external_database'] = False
|
config_dict['external_database'] = False
|
||||||
|
|
||||||
# update redis configs
|
# update redis configs
|
||||||
config_dict.update(get_redis_configs(configs.get("external_redis", None), with_trivy))
|
config_dict.update(get_redis_configs(configs.get("redis", None), configs.get("external_redis", None), with_trivy))
|
||||||
|
|
||||||
# auto generated secret string for core
|
# auto generated secret string for core
|
||||||
config_dict['core_secret'] = generate_random_string(16)
|
config_dict['core_secret'] = generate_random_string(16)
|
||||||
@ -371,7 +371,7 @@ def get_redis_url_param(redis=None):
|
|||||||
return ""
|
return ""
|
||||||
|
|
||||||
|
|
||||||
def get_redis_configs(external_redis=None, with_trivy=True):
|
def get_redis_configs(internal_redis=None, external_redis=None, with_trivy=True):
|
||||||
"""Returns configs for redis
|
"""Returns configs for redis
|
||||||
|
|
||||||
>>> get_redis_configs()['external_redis']
|
>>> get_redis_configs()['external_redis']
|
||||||
@ -404,6 +404,8 @@ def get_redis_configs(external_redis=None, with_trivy=True):
|
|||||||
>>> 'trivy_redis_url' not in get_redis_configs(with_trivy=False)
|
>>> 'trivy_redis_url' not in get_redis_configs(with_trivy=False)
|
||||||
True
|
True
|
||||||
"""
|
"""
|
||||||
|
|
||||||
|
internal_redis = internal_redis or {}
|
||||||
external_redis = external_redis or {}
|
external_redis = external_redis or {}
|
||||||
|
|
||||||
configs = dict(external_redis=bool(external_redis))
|
configs = dict(external_redis=bool(external_redis))
|
||||||
@ -418,13 +420,22 @@ def get_redis_configs(external_redis=None, with_trivy=True):
|
|||||||
'idle_timeout_seconds': 30,
|
'idle_timeout_seconds': 30,
|
||||||
}
|
}
|
||||||
|
|
||||||
# overwriting existing keys by external_redis
|
if len(internal_redis) > 0:
|
||||||
redis.update({key: value for (key, value) in external_redis.items() if value})
|
# overwriting existing keys by internal_redis
|
||||||
|
redis.update({key: value for (key, value) in internal_redis.items() if value})
|
||||||
|
else:
|
||||||
|
# overwriting existing keys by external_redis
|
||||||
|
redis.update({key: value for (key, value) in external_redis.items() if value})
|
||||||
|
|
||||||
configs['redis_url_core'] = get_redis_url(0, redis)
|
configs['redis_url_core'] = get_redis_url(0, redis)
|
||||||
configs['redis_url_js'] = get_redis_url(redis['jobservice_db_index'], redis)
|
configs['redis_url_js'] = get_redis_url(redis['jobservice_db_index'], redis)
|
||||||
configs['redis_url_reg'] = get_redis_url(redis['registry_db_index'], redis)
|
configs['redis_url_reg'] = get_redis_url(redis['registry_db_index'], redis)
|
||||||
|
|
||||||
|
if redis.get('harbor_db_index'):
|
||||||
|
configs['redis_url_harbor'] = get_redis_url(redis['harbor_db_index'], redis)
|
||||||
|
if redis.get('cache_layer_db_index'):
|
||||||
|
configs['redis_url_cache_layer'] = get_redis_url(redis['cache_layer_db_index'], redis)
|
||||||
|
|
||||||
if with_trivy:
|
if with_trivy:
|
||||||
configs['trivy_redis_url'] = get_redis_url(redis['trivy_db_index'], redis)
|
configs['trivy_redis_url'] = get_redis_url(redis['trivy_db_index'], redis)
|
||||||
|
|
||||||
|
@ -262,7 +262,7 @@ func (c *controller) updateUsageByRedis(ctx context.Context, reference, referenc
|
|||||||
return retry.Abort(ctx.Err())
|
return retry.Abort(ctx.Err())
|
||||||
}
|
}
|
||||||
|
|
||||||
client, err := libredis.GetCoreClient()
|
client, err := libredis.GetHarborClient()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
return retry.Abort(err)
|
return retry.Abort(err)
|
||||||
}
|
}
|
||||||
|
@ -129,25 +129,35 @@ func main() {
|
|||||||
web.BConfig.WebConfig.Session.SessionName = config.SessionCookieName
|
web.BConfig.WebConfig.Session.SessionName = config.SessionCookieName
|
||||||
web.BConfig.MaxMemory = 1 << 35 // (32GB)
|
web.BConfig.MaxMemory = 1 << 35 // (32GB)
|
||||||
web.BConfig.MaxUploadSize = 1 << 35 // (32GB)
|
web.BConfig.MaxUploadSize = 1 << 35 // (32GB)
|
||||||
|
// the core db used for beego session
|
||||||
redisURL := os.Getenv("_REDIS_URL_CORE")
|
redisCoreURL := os.Getenv("_REDIS_URL_CORE")
|
||||||
if len(redisURL) > 0 {
|
if len(redisCoreURL) > 0 {
|
||||||
u, err := url.Parse(redisURL)
|
_, err := url.Parse(redisCoreURL)
|
||||||
if err != nil {
|
if err != nil {
|
||||||
panic("bad _REDIS_URL")
|
panic("bad _REDIS_URL_CORE")
|
||||||
}
|
}
|
||||||
|
// configure the beego session redis
|
||||||
web.BConfig.WebConfig.Session.SessionProvider = session.HarborProviderName
|
web.BConfig.WebConfig.Session.SessionProvider = session.HarborProviderName
|
||||||
web.BConfig.WebConfig.Session.SessionProviderConfig = redisURL
|
web.BConfig.WebConfig.Session.SessionProviderConfig = redisCoreURL
|
||||||
|
|
||||||
log.Info("initializing cache ...")
|
|
||||||
if err := cache.Initialize(u.Scheme, redisURL); err != nil {
|
|
||||||
log.Fatalf("failed to initialize cache: %v", err)
|
|
||||||
}
|
|
||||||
// when config/db init function is called, the cache is not ready,
|
|
||||||
// enable config cache explicitly when the cache is ready
|
|
||||||
dbCfg.EnableConfigCache()
|
|
||||||
}
|
}
|
||||||
|
|
||||||
|
log.Info("initializing cache ...")
|
||||||
|
// the harbor db used for harbor business, use core db if not specified
|
||||||
|
redisHarborURL := os.Getenv("_REDIS_URL_HARBOR")
|
||||||
|
if redisHarborURL == "" {
|
||||||
|
redisHarborURL = redisCoreURL
|
||||||
|
}
|
||||||
|
u, err := url.Parse(redisHarborURL)
|
||||||
|
if err != nil {
|
||||||
|
panic("bad _REDIS_URL_HARBOR")
|
||||||
|
}
|
||||||
|
if err := cache.Initialize(u.Scheme, redisHarborURL); err != nil {
|
||||||
|
log.Fatalf("failed to initialize cache: %v", err)
|
||||||
|
}
|
||||||
|
// when config/db init function is called, the cache is not ready,
|
||||||
|
// enable config cache explicitly when the cache is ready
|
||||||
|
dbCfg.EnableConfigCache()
|
||||||
|
|
||||||
web.AddTemplateExt("htm")
|
web.AddTemplateExt("htm")
|
||||||
|
|
||||||
log.Info("initializing configurations...")
|
log.Info("initializing configurations...")
|
||||||
|
@ -19,8 +19,6 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"flag"
|
"flag"
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/url"
|
|
||||||
"os"
|
|
||||||
|
|
||||||
"github.com/goharbor/harbor/src/common"
|
"github.com/goharbor/harbor/src/common"
|
||||||
"github.com/goharbor/harbor/src/jobservice/common/utils"
|
"github.com/goharbor/harbor/src/jobservice/common/utils"
|
||||||
@ -29,7 +27,6 @@ import (
|
|||||||
"github.com/goharbor/harbor/src/jobservice/job/impl"
|
"github.com/goharbor/harbor/src/jobservice/job/impl"
|
||||||
"github.com/goharbor/harbor/src/jobservice/logger"
|
"github.com/goharbor/harbor/src/jobservice/logger"
|
||||||
"github.com/goharbor/harbor/src/jobservice/runtime"
|
"github.com/goharbor/harbor/src/jobservice/runtime"
|
||||||
"github.com/goharbor/harbor/src/lib/cache"
|
|
||||||
cfgLib "github.com/goharbor/harbor/src/lib/config"
|
cfgLib "github.com/goharbor/harbor/src/lib/config"
|
||||||
tracelib "github.com/goharbor/harbor/src/lib/trace"
|
tracelib "github.com/goharbor/harbor/src/lib/trace"
|
||||||
_ "github.com/goharbor/harbor/src/pkg/accessory/model/base"
|
_ "github.com/goharbor/harbor/src/pkg/accessory/model/base"
|
||||||
@ -47,21 +44,6 @@ func main() {
|
|||||||
panic(fmt.Sprintf("failed to load configuration, error: %v", err))
|
panic(fmt.Sprintf("failed to load configuration, error: %v", err))
|
||||||
}
|
}
|
||||||
|
|
||||||
// init cache if cache layer enabled
|
|
||||||
// gc needs to delete artifact by artifact manager, but the artifact cache store in
|
|
||||||
// core redis db so here require core redis url and init default cache.
|
|
||||||
if cfgLib.CacheEnabled() {
|
|
||||||
cacheURL := os.Getenv("_REDIS_URL_CORE")
|
|
||||||
u, err := url.Parse(cacheURL)
|
|
||||||
if err != nil {
|
|
||||||
panic("bad _REDIS_URL_CORE")
|
|
||||||
}
|
|
||||||
|
|
||||||
if err = cache.Initialize(u.Scheme, cacheURL); err != nil {
|
|
||||||
panic(fmt.Sprintf("failed to initialize cache: %v", err))
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
// Get parameters
|
// Get parameters
|
||||||
configPath := flag.String("c", "", "Specify the yaml config file path")
|
configPath := flag.String("c", "", "Specify the yaml config file path")
|
||||||
flag.Parse()
|
flag.Parse()
|
||||||
|
35
src/lib/cache/cache.go
vendored
35
src/lib/cache/cache.go
vendored
@ -19,6 +19,7 @@ import (
|
|||||||
"errors"
|
"errors"
|
||||||
"fmt"
|
"fmt"
|
||||||
"net/url"
|
"net/url"
|
||||||
|
"os"
|
||||||
"sync"
|
"sync"
|
||||||
"time"
|
"time"
|
||||||
|
|
||||||
@ -137,3 +138,37 @@ func Initialize(typ, addr string) error {
|
|||||||
func Default() Cache {
|
func Default() Cache {
|
||||||
return cache
|
return cache
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var (
|
||||||
|
// cacheLayer is the global cache layer cache instance.
|
||||||
|
cacheLayer Cache
|
||||||
|
// cacheLayerOnce is the once condition for initializing instance.
|
||||||
|
cacheLayerOnce sync.Once
|
||||||
|
)
|
||||||
|
|
||||||
|
// LayerCache is the global cache instance for cache layer.
|
||||||
|
func LayerCache() Cache {
|
||||||
|
// parse the redis url for cache layer, use the default cache if not specify
|
||||||
|
redisCacheURL := os.Getenv("_REDIS_URL_CACHE_LAYER")
|
||||||
|
if redisCacheURL == "" {
|
||||||
|
if cache != nil {
|
||||||
|
return cache
|
||||||
|
}
|
||||||
|
// use the core url if cache layer url not found
|
||||||
|
redisCacheURL = os.Getenv("_REDIS_URL_CORE")
|
||||||
|
}
|
||||||
|
|
||||||
|
u, err := url.Parse(redisCacheURL)
|
||||||
|
if err != nil {
|
||||||
|
log.Fatal("failed to parse the redis url for cache layer, bad _REDIS_URL_CACHE_LAYER")
|
||||||
|
}
|
||||||
|
|
||||||
|
cacheLayerOnce.Do(func() {
|
||||||
|
cacheLayer, err = New(u.Scheme, Address(redisCacheURL), Prefix("cache:"))
|
||||||
|
if err != nil {
|
||||||
|
log.Fatalf("failed to initialize cache for cache layer, err: %v", err)
|
||||||
|
}
|
||||||
|
})
|
||||||
|
|
||||||
|
return cacheLayer
|
||||||
|
}
|
||||||
|
@ -31,9 +31,9 @@ var (
|
|||||||
registry *redis.Client
|
registry *redis.Client
|
||||||
registryOnce = &sync.Once{}
|
registryOnce = &sync.Once{}
|
||||||
|
|
||||||
// core is a global redis client for core db
|
// harbor is a global redis client for harbor db
|
||||||
core *redis.Client
|
harbor *redis.Client
|
||||||
coreOnce = &sync.Once{}
|
harborOnce = &sync.Once{}
|
||||||
)
|
)
|
||||||
|
|
||||||
// GetRegistryClient returns the registry redis client.
|
// GetRegistryClient returns the registry redis client.
|
||||||
@ -60,26 +60,30 @@ func GetRegistryClient() (*redis.Client, error) {
|
|||||||
return registry, nil
|
return registry, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
// GetCoreClient returns the core redis client.
|
// GetHarborClient returns the harbor redis client.
|
||||||
func GetCoreClient() (*redis.Client, error) {
|
func GetHarborClient() (*redis.Client, error) {
|
||||||
coreOnce.Do(func() {
|
harborOnce.Do(func() {
|
||||||
url := os.Getenv("_REDIS_URL_CORE")
|
// parse redis url for harbor business, use core url by default
|
||||||
|
url := os.Getenv("_REDIS_URL_HARBOR")
|
||||||
|
if url == "" {
|
||||||
|
url = os.Getenv("_REDIS_URL_CORE")
|
||||||
|
}
|
||||||
c, err := libredis.New(cache.Options{Address: url})
|
c, err := libredis.New(cache.Options{Address: url})
|
||||||
if err != nil {
|
if err != nil {
|
||||||
log.Errorf("failed to initialize redis client for core, error: %v", err)
|
log.Errorf("failed to initialize redis client for harbor, error: %v", err)
|
||||||
// reset the once to support retry if error occurred
|
// reset the once to support retry if error occurred
|
||||||
coreOnce = &sync.Once{}
|
harborOnce = &sync.Once{}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
if c != nil {
|
if c != nil {
|
||||||
core = c.(*libredis.Cache).Client
|
harbor = c.(*libredis.Cache).Client
|
||||||
}
|
}
|
||||||
})
|
})
|
||||||
|
|
||||||
if core == nil {
|
if harbor == nil {
|
||||||
return nil, errors.New("no core redis client initialized")
|
return nil, errors.New("no harbor redis client initialized")
|
||||||
}
|
}
|
||||||
|
|
||||||
return core, nil
|
return harbor, nil
|
||||||
}
|
}
|
||||||
|
@ -41,22 +41,22 @@ func TestGetRegistryClient(t *testing.T) {
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func TestGetCoreClient(t *testing.T) {
|
func TestGetHarborClient(t *testing.T) {
|
||||||
// failure case with invalid address
|
// failure case with invalid address
|
||||||
t.Setenv("_REDIS_URL_CORE", "invalid-address")
|
t.Setenv("_REDIS_URL_HARBOR", "invalid-address")
|
||||||
client, err := GetCoreClient()
|
client, err := GetHarborClient()
|
||||||
assert.Error(t, err)
|
assert.Error(t, err)
|
||||||
assert.Nil(t, client)
|
assert.Nil(t, client)
|
||||||
|
|
||||||
// normal case with valid address
|
// normal case with valid address
|
||||||
t.Setenv("_REDIS_URL_CORE", "redis://localhost:6379/0")
|
t.Setenv("_REDIS_URL_HARBOR", "redis://localhost:6379/0")
|
||||||
client, err = GetCoreClient()
|
client, err = GetHarborClient()
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
assert.NotNil(t, client)
|
assert.NotNil(t, client)
|
||||||
|
|
||||||
// multiple calls should return the same client
|
// multiple calls should return the same client
|
||||||
for i := 0; i < 10; i++ {
|
for i := 0; i < 10; i++ {
|
||||||
newClient, err := GetCoreClient()
|
newClient, err := GetHarborClient()
|
||||||
assert.NoError(t, err)
|
assert.NoError(t, err)
|
||||||
assert.Equal(t, client, newClient)
|
assert.Equal(t, client, newClient)
|
||||||
}
|
}
|
||||||
|
@ -24,27 +24,27 @@ import (
|
|||||||
)
|
)
|
||||||
|
|
||||||
// innerCache is the default cache client,
|
// innerCache is the default cache client,
|
||||||
// actually it is a wrapper for cache.Default().
|
// actually it is a wrapper for cache.LayerCache().
|
||||||
var innerCache cache.Cache = &cacheClient{}
|
var innerCache cache.Cache = &cacheClient{}
|
||||||
|
|
||||||
// cacheClient is a interceptor for cache.Default, in order to implement specific
|
// cacheClient is a interceptor for cache.CacheLayer, in order to implement specific
|
||||||
// case for cache layer.
|
// case for cache layer.
|
||||||
type cacheClient struct{}
|
type cacheClient struct{}
|
||||||
|
|
||||||
func (*cacheClient) Contains(ctx context.Context, key string) bool {
|
func (*cacheClient) Contains(ctx context.Context, key string) bool {
|
||||||
return cache.Default().Contains(ctx, key)
|
return cache.LayerCache().Contains(ctx, key)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (*cacheClient) Delete(ctx context.Context, key string) error {
|
func (*cacheClient) Delete(ctx context.Context, key string) error {
|
||||||
return cache.Default().Delete(ctx, key)
|
return cache.LayerCache().Delete(ctx, key)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (*cacheClient) Fetch(ctx context.Context, key string, value interface{}) error {
|
func (*cacheClient) Fetch(ctx context.Context, key string, value interface{}) error {
|
||||||
return cache.Default().Fetch(ctx, key, value)
|
return cache.LayerCache().Fetch(ctx, key, value)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (*cacheClient) Ping(ctx context.Context) error {
|
func (*cacheClient) Ping(ctx context.Context) error {
|
||||||
return cache.Default().Ping(ctx)
|
return cache.LayerCache().Ping(ctx)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (*cacheClient) Save(ctx context.Context, key string, value interface{}, expiration ...time.Duration) error {
|
func (*cacheClient) Save(ctx context.Context, key string, value interface{}, expiration ...time.Duration) error {
|
||||||
@ -57,11 +57,11 @@ func (*cacheClient) Save(ctx context.Context, key string, value interface{}, exp
|
|||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
return cache.Default().Save(ctx, key, value, expiration...)
|
return cache.LayerCache().Save(ctx, key, value, expiration...)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (*cacheClient) Scan(ctx context.Context, match string) (cache.Iterator, error) {
|
func (*cacheClient) Scan(ctx context.Context, match string) (cache.Iterator, error) {
|
||||||
return cache.Default().Scan(ctx, match)
|
return cache.LayerCache().Scan(ctx, match)
|
||||||
}
|
}
|
||||||
|
|
||||||
var _ Manager = &BaseManager{}
|
var _ Manager = &BaseManager{}
|
||||||
|
Loading…
x
Reference in New Issue
Block a user