From 2ca5d671b9264e7e52d2db576ae039e3991c813a Mon Sep 17 00:00:00 2001 From: Nicolas Carlier Date: Fri, 21 Dec 2018 10:41:45 +0000 Subject: [PATCH] feat(api): add API endpoint to retrieve logs --- README.md | 29 ++++++++++++++++--- pkg/api/index.go | 51 ++++++++++++++++++++++++++++++---- pkg/worker/work_log.go | 38 +++++++++++++++++++++++++ pkg/worker/work_runner.go | 36 ++++++++---------------- pkg/worker/work_runner_test.go | 14 ++++++++-- pkg/worker/worker.go | 10 +++---- 6 files changed, 137 insertions(+), 41 deletions(-) create mode 100644 pkg/worker/work_log.go diff --git a/README.md b/README.md index 607dd0f..c9812fb 100644 --- a/README.md +++ b/README.md @@ -52,7 +52,7 @@ You can configure the daemon by: | `APP_SCRIPTS_DIR` | `./scripts` | Scripts directory | | `APP_SCRIPTS_GIT_URL` | none | GIT repository that contains scripts (Note: this is only used by the Docker image or by using the Docker entrypoint script) | | `APP_SCRIPTS_GIT_KEY` | none | GIT SSH private key used to clone the repository (Note: this is only used by the Docker image or by using the Docker entrypoint script) | -| `APP_WORKING_DIR` | `/tmp` (OS temp dir) | Working directory (to store execution logs) | +| `APP_LOG_DIR` | `/tmp` (OS temp dir) | Directory to store execution logs | | `APP_NOTIFIER` | none | Post script notification (`http` or `smtp`) | | `APP_NOTIFIER_FROM` | none | Sender of the notification | | `APP_NOTIFIER_TO` | none | Recipient of the notification | @@ -110,7 +110,7 @@ echo "bar bar bar" ``` ```bash -$ curl -XPOST http://localhost/foo/bar +$ curl -XPOST http://localhost:8080/foo/bar data: foo foo foo data: bar bar bar @@ -147,7 +147,7 @@ echo "Script parameters: $1" The result: ```bash -$ curl --data @test.json http://localhost/echo?foo=bar +$ curl --data @test.json http://localhost:8080/echo?foo=bar data: Query parameter: foo=bar data: Header parameter: user-agent=curl/7.52.1 @@ -169,7 +169,28 @@ You can override this global behavior per request by setting the HTTP header: *Example:* ```bash -$ curl -XPOST -H "X-Hook-Timeout: 5" http://localhost/echo?foo=bar +$ curl -XPOST -H "X-Hook-Timeout: 5" http://localhost:8080/echo?foo=bar +``` + +### Webhook logs + +As mentioned above, web hook logs are stream in real time during the call. +However, you can retrieve the logs of a previous call by using the hook ID: `http://localhost:8080//` + +The hook ID is returned as an HTTP header with the Webhook response: `X-Hook-ID` + +*Example:* + +```bash +$ # Call webhook +$ curl -v -XPOST http://localhost:8080/echo?foo=bar +... +< HTTP/1.1 200 OK +< Content-Type: text/event-stream +< X-Hook-Id: 2 +... +$ # Retrieve logs afterwards +$ curl http://localhost:8080/echo/2 ``` ### Post hook notifications diff --git a/pkg/api/index.go b/pkg/api/index.go index 5de4f06..f31f2c0 100644 --- a/pkg/api/index.go +++ b/pkg/api/index.go @@ -2,8 +2,10 @@ package api import ( "fmt" + "io" "io/ioutil" "net/http" + "path" "strconv" "strings" @@ -33,17 +35,24 @@ func index(conf *config.Config) http.Handler { } func webhookHandler(w http.ResponseWriter, r *http.Request) { + if r.Method == "POST" { + triggerWebhook(w, r) + } else if r.Method == "GET" { + getWebhookLog(w, r) + } else { + http.Error(w, "405 Method Not Allowed", http.StatusMethodNotAllowed) + return + } +} + +func triggerWebhook(w http.ResponseWriter, r *http.Request) { + // Check that streaming is supported flusher, ok := w.(http.Flusher) if !ok { http.Error(w, "Streaming not supported!", http.StatusInternalServerError) return } - if r.Method != "POST" { - http.Error(w, "405 Method Not Allowed", http.StatusMethodNotAllowed) - return - } - // Get script location p := strings.TrimPrefix(r.URL.Path, "/") script, err := tools.ResolveScript(scriptDir, p) @@ -76,6 +85,7 @@ func webhookHandler(w http.ResponseWriter, r *http.Request) { w.Header().Set("Cache-Control", "no-cache") w.Header().Set("Connection", "keep-alive") w.Header().Set("Access-Control-Allow-Origin", "*") + w.Header().Set("X-Hook-ID", strconv.FormatUint(work.ID, 10)) for { msg, open := <-work.MessageChan @@ -90,3 +100,34 @@ func webhookHandler(w http.ResponseWriter, r *http.Request) { flusher.Flush() } } + +func getWebhookLog(w http.ResponseWriter, r *http.Request) { + // Get hook ID + id := path.Base(r.URL.Path) + + // Get script location + name := path.Dir(strings.TrimPrefix(r.URL.Path, "/")) + _, err := tools.ResolveScript(scriptDir, name) + if err != nil { + logger.Error.Println(err.Error()) + http.Error(w, err.Error(), http.StatusNotFound) + return + } + + // Get log file + logFile, err := worker.GetLogFile(id, name) + if err != nil { + logger.Error.Println(err.Error()) + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + if logFile == nil { + http.Error(w, err.Error(), http.StatusNotFound) + return + } + defer logFile.Close() + + w.Header().Set("Content-Type", "text/plain") + + io.Copy(w, logFile) +} diff --git a/pkg/worker/work_log.go b/pkg/worker/work_log.go new file mode 100644 index 0000000..4641c36 --- /dev/null +++ b/pkg/worker/work_log.go @@ -0,0 +1,38 @@ +package worker + +import ( + "fmt" + "os" + "path" + "path/filepath" + "time" + + "github.com/ncarlier/webhookd/pkg/tools" +) + +// getLogDir returns log directory +func getLogDir() string { + if value, ok := os.LookupEnv("APP_LOG_DIR"); ok { + return value + } + return os.TempDir() +} + +func createLogFile(work *WorkRequest) (*os.File, error) { + logFilename := path.Join(getLogDir(), fmt.Sprintf("%s_%d_%s.txt", tools.ToSnakeCase(work.Name), work.ID, time.Now().Format("20060102_1504"))) + return os.Create(logFilename) +} + +// GetLogFile retrieve work log with its name and id +func GetLogFile(id, name string) (*os.File, error) { + logPattern := path.Join(getLogDir(), fmt.Sprintf("%s_%s_*.txt", tools.ToSnakeCase(name), id)) + files, err := filepath.Glob(logPattern) + if err != nil { + return nil, err + } + if len(files) > 0 { + filename := files[len(files)-1] + return os.Open(filename) + } + return nil, nil +} diff --git a/pkg/worker/work_runner.go b/pkg/worker/work_runner.go index 59b4598..56bd83d 100644 --- a/pkg/worker/work_runner.go +++ b/pkg/worker/work_runner.go @@ -2,17 +2,14 @@ package worker import ( "bufio" - "fmt" "io" "os" "os/exec" - "path" "sync" "syscall" "time" "github.com/ncarlier/webhookd/pkg/logger" - "github.com/ncarlier/webhookd/pkg/tools" ) // ChanWriter is a simple writer to a channel of byte. @@ -25,22 +22,14 @@ func (c *ChanWriter) Write(p []byte) (int, error) { return len(p), nil } -var ( - workingdir = os.Getenv("APP_WORKING_DIR") -) - -func run(work *WorkRequest) (string, error) { - if workingdir == "" { - workingdir = os.TempDir() - } - +func run(work *WorkRequest) error { logger.Info.Printf("Work %s#%d started...\n", work.Name, work.ID) logger.Debug.Printf("Work %s#%d script: %s\n", work.Name, work.ID, work.Script) logger.Debug.Printf("Work %s#%d parameter: %v\n", work.Name, work.ID, work.Args) binary, err := exec.LookPath(work.Script) if err != nil { - return "", err + return err } // Exec script with args... @@ -50,14 +39,13 @@ func run(work *WorkRequest) (string, error) { // using a process group... cmd.SysProcAttr = &syscall.SysProcAttr{Setpgid: true} - // Open the out file for writing - logFilename := path.Join(workingdir, fmt.Sprintf("%s_%d_%s.txt", tools.ToSnakeCase(work.Name), work.ID, time.Now().Format("20060102_1504"))) - logFile, err := os.Create(logFilename) + // Open the log file for writing + logFile, err := createLogFile(work) if err != nil { - return "", err + return err } defer logFile.Close() - logger.Debug.Printf("Work %s#%d output to file: %s\n", work.Name, work.ID, logFilename) + logger.Debug.Printf("Work %s#%d output to file: %s\n", work.Name, work.ID, logFile.Name()) wLogFile := bufio.NewWriter(logFile) defer wLogFile.Flush() @@ -65,18 +53,18 @@ func run(work *WorkRequest) (string, error) { // Combine cmd stdout and stderr outReader, err := cmd.StdoutPipe() if err != nil { - return logFilename, err + return err } errReader, err := cmd.StderrPipe() if err != nil { - return logFilename, err + return err } cmdReader := io.MultiReader(outReader, errReader) // Start the script... err = cmd.Start() if err != nil { - return logFilename, err + return err } // Create wait group to wait for command output completion @@ -97,7 +85,7 @@ func run(work *WorkRequest) (string, error) { } // writing to outfile if _, err := wLogFile.WriteString(line + "\n"); err != nil { - logger.Error.Println("Error while writing into the log file:", logFilename, err) + logger.Error.Println("Error while writing into the log file:", logFile.Name(), err) break } } @@ -127,8 +115,8 @@ func run(work *WorkRequest) (string, error) { if err != nil { logger.Info.Printf("Work %s#%d done [ERROR]\n", work.Name, work.ID) - return logFilename, err + return err } logger.Info.Printf("Work %s#%d done [SUCCESS]\n", work.Name, work.ID) - return logFilename, nil + return nil } diff --git a/pkg/worker/work_runner_test.go b/pkg/worker/work_runner_test.go index 5a8fc37..accb848 100644 --- a/pkg/worker/work_runner_test.go +++ b/pkg/worker/work_runner_test.go @@ -1,6 +1,7 @@ package worker import ( + "strconv" "testing" "github.com/ncarlier/webhookd/pkg/assert" @@ -30,8 +31,15 @@ func TestWorkRunner(t *testing.T) { work := NewWorkRequest("test", script, payload, args, 5) assert.NotNil(t, work, "") printWorkMessages(work) - _, err := run(work) + err := run(work) assert.Nil(t, err, "") + + // Test that log file is ok + id := strconv.FormatUint(work.ID, 10) + logFile, err := GetLogFile(id, "test") + defer logFile.Close() + assert.Nil(t, err, "Log file should exists") + assert.NotNil(t, logFile, "Log file should be retrieve") } func TestWorkRunnerWithError(t *testing.T) { @@ -40,7 +48,7 @@ func TestWorkRunnerWithError(t *testing.T) { work := NewWorkRequest("test", script, "", []string{}, 5) assert.NotNil(t, work, "") printWorkMessages(work) - _, err := run(work) + err := run(work) assert.NotNil(t, err, "") assert.Equal(t, "exit status 1", err.Error(), "") } @@ -51,7 +59,7 @@ func TestWorkRunnerWithTimeout(t *testing.T) { work := NewWorkRequest("test", script, "", []string{}, 1) assert.NotNil(t, work, "") printWorkMessages(work) - _, err := run(work) + err := run(work) assert.NotNil(t, err, "") assert.Equal(t, "signal: killed", err.Error(), "") } diff --git a/pkg/worker/worker.go b/pkg/worker/worker.go index 1caf7c6..8745f3d 100644 --- a/pkg/worker/worker.go +++ b/pkg/worker/worker.go @@ -42,15 +42,15 @@ func (w Worker) Start() { case work := <-w.Work: // Receive a work request. logger.Debug.Printf("Worker #%d received work request: %s#%d\n", w.ID, work.Name, work.ID) - filename, err := run(&work) + err := run(&work) if err != nil { - subject := fmt.Sprintf("Webhook %s#%d FAILED.", work.Name, work.ID) + // subject := fmt.Sprintf("Webhook %s#%d FAILED.", work.Name, work.ID) work.MessageChan <- []byte(fmt.Sprintf("error: %s", err.Error())) - notify(subject, err.Error(), filename) + // notify(subject, err.Error(), filename) } else { - subject := fmt.Sprintf("Webhook %s#%d SUCCEEDED.", work.Name, workID) + // subject := fmt.Sprintf("Webhook %s#%d SUCCEEDED.", work.Name, workID) work.MessageChan <- []byte("done") - notify(subject, "See attachment.", filename) + // notify(subject, "See attachment.", filename) } close(work.MessageChan) case <-w.QuitChan: