From 66b9699ac24e4416f0cd35c2713809bdbaaee027 Mon Sep 17 00:00:00 2001
From: Wenkai Yin <yinw@vmware.com>
Date: Wed, 8 Nov 2017 13:07:27 +0800
Subject: [PATCH] Improve log rotation configurability
---
make/common/templates/log/env | 1 -
make/common/templates/log/logrotate.conf | 8 ++++++
make/docker-compose.tpl | 3 +--
make/harbor.cfg | 8 ++++--
make/photon/log/Dockerfile | 10 +++----
make/photon/log/rotate.sh | 33 ------------------------
make/photon/log/rsyslog_docker.conf | 2 +-
make/prepare | 12 +++++----
8 files changed, 28 insertions(+), 49 deletions(-)
delete mode 100644 make/common/templates/log/env
create mode 100644 make/common/templates/log/logrotate.conf
delete mode 100755 make/photon/log/rotate.sh
diff --git a/make/common/templates/log/env b/make/common/templates/log/env
deleted file mode 100644
index d0c9d994f..000000000
--- a/make/common/templates/log/env
+++ /dev/null
@@ -1 +0,0 @@
-LOG_ROTATE_DAYS=$log_rotate_days
\ No newline at end of file
diff --git a/make/common/templates/log/logrotate.conf b/make/common/templates/log/logrotate.conf
new file mode 100644
index 000000000..bc63e78de
--- /dev/null
+++ b/make/common/templates/log/logrotate.conf
@@ -0,0 +1,8 @@
+/var/log/docker/*.log {
+ rotate $log_rotate_count
+ size $log_rotate_size
+ copytruncate
+ compress
+ missingok
+ nodateext
+}
\ No newline at end of file
diff --git a/make/docker-compose.tpl b/make/docker-compose.tpl
index 18c8d6cf7..0bf52032a 100644
--- a/make/docker-compose.tpl
+++ b/make/docker-compose.tpl
@@ -3,11 +3,10 @@ services:
log:
image: vmware/harbor-log:__version__
container_name: harbor-log
- env_file:
- - ./common/config/log/env
restart: always
volumes:
- /var/log/harbor/:/var/log/docker/:z
+ - ./common/config/log/:/etc/logrotate.d/:z
ports:
- 127.0.0.1:1514:10514
networks:
diff --git a/make/harbor.cfg b/make/harbor.cfg
index 34af8b960..f926c0429 100644
--- a/make/harbor.cfg
+++ b/make/harbor.cfg
@@ -34,8 +34,12 @@ admiral_url = NA
#Please update it before deployment, subsequent update will cause Clair's API server and Harbor unable to access Clair's database.
clair_db_password = password
-#The logs n days before will be compressed
-log_rotate_days = 3
+#Log files are rotated log_rotate_count times before being removed. If count is 0, old versions are removed rather than rotated.
+log_rotate_count = 50
+#Log files are rotated only if they grow bigger than log_rotate_size bytes. If size is followed by k, the size is assumed to be in kilobytes.
+#If the M is used, the size is in megabytes, and if G is used, the size is in gigabytes. So size 100, size 100k, size 100M and size 100G
+#are all valid.
+log_rotate_size = 200M
#NOTES: The properties between BEGIN INITIAL PROPERTIES and END INITIAL PROPERTIES
#only take effect in the first boot, the subsequent changes of these properties
diff --git a/make/photon/log/Dockerfile b/make/photon/log/Dockerfile
index c82bf07f8..7b31c75b5 100644
--- a/make/photon/log/Dockerfile
+++ b/make/photon/log/Dockerfile
@@ -1,7 +1,7 @@
FROM vmware/photon:1.0
RUN tdnf distro-sync -y || echo \
- && tdnf install -y cronie rsyslog shadow tar gzip sudo net-tools\
+ && tdnf install -y cronie rsyslog logrotate shadow tar gzip sudo net-tools\
&& mkdir /etc/rsyslog.d/ \
&& mkdir /var/spool/rsyslog \
&& groupadd -r -g 10000 syslog && useradd --no-log-init -r -g 10000 -u 10000 syslog \
@@ -9,19 +9,19 @@ RUN tdnf distro-sync -y || echo \
COPY rsyslog.conf /etc/rsyslog.conf
-# notes: file name cannot contain dot, or the script will not run
-COPY rotate.sh /etc/cron.daily/rotate
-
# rsyslog configuration file for docker
COPY rsyslog_docker.conf /etc/rsyslog.d/
+# run logrotate hourly
+RUN mv /etc/cron.daily/logrotate /etc/cron.hourly/logrotate
+
COPY start.sh /usr/local/bin/
RUN chmod +x /usr/local/bin/start.sh && \
chown -R 10000:10000 /run
HEALTHCHECK CMD netstat -ltu|grep 10514
-VOLUME /var/log/docker/ /run/
+VOLUME /var/log/docker/ /run/ /etc/logrotate.d/
EXPOSE 10514
diff --git a/make/photon/log/rotate.sh b/make/photon/log/rotate.sh
deleted file mode 100755
index 5f102dffc..000000000
--- a/make/photon/log/rotate.sh
+++ /dev/null
@@ -1,33 +0,0 @@
-#!/bin/bash
-set -e
-echo "Log rotate starting..."
-
-#The logs n days before will be compressed.
-n=$LOG_ROTATE_DAYS
-if [ -z "$n" ]
-then
- n=3
-fi
-
-echo "logs rotate days: $n"
-
-path=/var/log/docker
-
-list=""
-n_days_before=$(($(date +%s) - 3600*24*$n))
-for dir in $(ls $path | grep -v "tar.gz");
-do
- if [ $(date --date=$dir +%s) -lt $n_days_before ]
- then
- echo "$dir will be compressed"
- list="$list $dir"
- fi
-done
-
-if [ -n "$list" ]
-then
- cd $path
- tar --remove-files -zcvf $(date -d @$n_days_before +%F)-.tar.gz $list
-fi
-
-echo "Log rotate finished."
diff --git a/make/photon/log/rsyslog_docker.conf b/make/photon/log/rsyslog_docker.conf
index ce565c705..a21cc5078 100644
--- a/make/photon/log/rsyslog_docker.conf
+++ b/make/photon/log/rsyslog_docker.conf
@@ -1,7 +1,7 @@
# Rsyslog configuration file for docker.
template(name="DynaFile" type="string"
- string="/var/log/docker/%$now%/%syslogtag:R,ERE,0,DFLT:[^[]*--end:secpath-replace%.log"
+ string="/var/log/docker/%syslogtag:R,ERE,0,DFLT:[^[]*--end:secpath-replace%.log"
)
#if $programname == "docker" then ?DynaFile
if $programname != "rsyslogd" then -?DynaFile
diff --git a/make/prepare b/make/prepare
index 3702bea46..5fb7c3380 100755
--- a/make/prepare
+++ b/make/prepare
@@ -162,7 +162,8 @@ uaa_clientid = rcp.get("configuration", "uaa_clientid")
uaa_clientsecret = rcp.get("configuration", "uaa_clientsecret")
uaa_ca_root = rcp.get("configuration", "uaa_ca_root")
secret_key = get_secret_key(secretkey_path)
-log_rotate_days = rcp.get("configuration", "log_rotate_days")
+log_rotate_count = rcp.get("configuration", "log_rotate_count")
+log_rotate_size = rcp.get("configuration", "log_rotate_size")
########
ui_secret = ''.join(random.choice(string.ascii_letters+string.digits) for i in range(16))
@@ -190,7 +191,7 @@ db_conf_env = os.path.join(config_dir, "db", "env")
job_conf_env = os.path.join(config_dir, "jobservice", "env")
nginx_conf = os.path.join(config_dir, "nginx", "nginx.conf")
cert_dir = os.path.join(config_dir, "nginx", "cert")
-log_conf_env = os.path.join(config_dir, "log", "env")
+log_rotate_config = os.path.join(config_dir, "log", "logrotate.conf")
if protocol == "https":
target_cert_path = os.path.join(cert_dir, os.path.basename(cert_path))
@@ -273,9 +274,10 @@ render(os.path.join(templates_dir, "jobservice", "env"),
ui_secret=ui_secret,
jobservice_secret=jobservice_secret)
-render(os.path.join(templates_dir, "log", "env"),
- log_conf_env,
- log_rotate_days=log_rotate_days)
+render(os.path.join(templates_dir, "log", "logrotate.conf"),
+ log_rotate_config,
+ log_rotate_count=log_rotate_count,
+ log_rotate_size=log_rotate_size)
print("Generated configuration file: %s" % jobservice_conf)
shutil.copyfile(os.path.join(templates_dir, "jobservice", "app.conf"), jobservice_conf)