From c21443c10e88c76930ccbcb016a5aa6d298d7ee3 Mon Sep 17 00:00:00 2001 From: Nicolas Carlier Date: Wed, 26 Feb 2020 21:19:11 +0000 Subject: [PATCH] chore(): small pkg refactoring --- README.md | 11 ++++++++- pkg/{tools/http.go => api/helper.go} | 2 +- pkg/api/index.go | 9 ++++---- .../http_test.go => api/test/helper_test.go} | 8 +++---- pkg/auth/htpasswd-file_test.go | 15 ------------ pkg/auth/test/htpasswd-file_test.go | 23 +++++++++++++++++++ pkg/auth/{ => test}/test.htpasswd | 0 pkg/notification/{main.go => notifier.go} | 0 pkg/{tools => worker}/script_resolver.go | 2 +- .../test}/script_resolver_test.go | 14 +++++------ {tests => pkg/worker/test}/test_error.sh | 0 {tests => pkg/worker/test}/test_simple.sh | 0 {tests => pkg/worker/test}/test_timeout.sh | 0 pkg/worker/{ => test}/work_runner_test.go | 17 +++++++------- pkg/worker/work_runner.go | 3 ++- pkg/worker/worker.go | 9 ++++---- 16 files changed, 65 insertions(+), 48 deletions(-) rename pkg/{tools/http.go => api/helper.go} (98%) rename pkg/{tools_test/http_test.go => api/test/helper_test.go} (80%) delete mode 100644 pkg/auth/htpasswd-file_test.go create mode 100644 pkg/auth/test/htpasswd-file_test.go rename pkg/auth/{ => test}/test.htpasswd (100%) rename pkg/notification/{main.go => notifier.go} (100%) rename pkg/{tools => worker}/script_resolver.go (96%) rename pkg/{tools_test => worker/test}/script_resolver_test.go (61%) rename {tests => pkg/worker/test}/test_error.sh (100%) rename {tests => pkg/worker/test}/test_simple.sh (100%) rename {tests => pkg/worker/test}/test_timeout.sh (100%) rename pkg/worker/{ => test}/work_runner_test.go (85%) diff --git a/README.md b/README.md index 9a1a0e5..53745b5 100644 --- a/README.md +++ b/README.md @@ -75,6 +75,9 @@ You can override the default using the `WHD_SCRIPTS` environment variable or `-s Note that Webhookd is able to run any type of file in this directory as long as the file is executable. For example, you can execute a Node.js file if you give execution rights to the file and add the appropriate `#!` header (in this case: `#!/usr/bin/env node`). +You can find sample scripts in the [example folder](./scripts/examples). +In particular, examples of integration with Gitlab and Github. + ### Webhook URL The directory structure define the webhook URL. @@ -308,7 +311,7 @@ $ curl -X POST \ -H 'Date: ' \ -H 'Signature: keyId=,algorithm="rsa-sha256",headers="(request-target) date",signature=' \ -H 'Accept: application/json' \ - "http://loclahost:8080/echo?msg=hello" + "http://localhost:8080/echo?msg=hello" ``` You can find a small HTTP client in the ["tooling" directory](./tooling/httpsig/README.md) that is capable of forging HTTP signatures. @@ -345,4 +348,10 @@ On *nix, if you want to listen on ports 80 and 443, don't forget to use `setcap` sudo setcap CAP_NET_BIND_SERVICE+ep webhookd ``` +## License + +The MIT License (MIT) + +See [LICENSE](./LICENSE) to see the full text. + --- diff --git a/pkg/tools/http.go b/pkg/api/helper.go similarity index 98% rename from pkg/tools/http.go rename to pkg/api/helper.go index e790d82..e6b98c0 100644 --- a/pkg/tools/http.go +++ b/pkg/api/helper.go @@ -1,4 +1,4 @@ -package tools +package api import ( "bytes" diff --git a/pkg/api/index.go b/pkg/api/index.go index f391f24..9741c60 100644 --- a/pkg/api/index.go +++ b/pkg/api/index.go @@ -13,7 +13,6 @@ import ( "github.com/ncarlier/webhookd/pkg/config" "github.com/ncarlier/webhookd/pkg/logger" "github.com/ncarlier/webhookd/pkg/model" - "github.com/ncarlier/webhookd/pkg/tools" "github.com/ncarlier/webhookd/pkg/worker" ) @@ -58,7 +57,7 @@ func triggerWebhook(w http.ResponseWriter, r *http.Request) { // Get script location p := strings.TrimPrefix(r.URL.Path, "/") - script, err := tools.ResolveScript(scriptDir, p) + script, err := worker.ResolveScript(scriptDir, p) if err != nil { logger.Error.Println(err.Error()) http.Error(w, err.Error(), http.StatusNotFound) @@ -72,8 +71,8 @@ func triggerWebhook(w http.ResponseWriter, r *http.Request) { return } - params := tools.QueryParamsToShellVars(r.URL.Query()) - params = append(params, tools.HTTPHeadersToShellVars(r.Header)...) + params := QueryParamsToShellVars(r.URL.Query()) + params = append(params, HTTPHeadersToShellVars(r.Header)...) // logger.Debug.Printf("API REQUEST: \"%s\" with params %s...\n", p, params) @@ -119,7 +118,7 @@ func getWebhookLog(w http.ResponseWriter, r *http.Request) { // Get script location name := path.Dir(strings.TrimPrefix(r.URL.Path, "/")) - _, err := tools.ResolveScript(scriptDir, name) + _, err := worker.ResolveScript(scriptDir, name) if err != nil { logger.Error.Println(err.Error()) http.Error(w, err.Error(), http.StatusNotFound) diff --git a/pkg/tools_test/http_test.go b/pkg/api/test/helper_test.go similarity index 80% rename from pkg/tools_test/http_test.go rename to pkg/api/test/helper_test.go index b8171bf..eca8306 100644 --- a/pkg/tools_test/http_test.go +++ b/pkg/api/test/helper_test.go @@ -1,12 +1,12 @@ -package tools_test +package test import ( "net/http" "net/url" "testing" + "github.com/ncarlier/webhookd/pkg/api" "github.com/ncarlier/webhookd/pkg/assert" - "github.com/ncarlier/webhookd/pkg/tools" ) func TestQueryParamsToShellVars(t *testing.T) { @@ -14,7 +14,7 @@ func TestQueryParamsToShellVars(t *testing.T) { "string": []string{"foo"}, "list": []string{"foo", "bar"}, } - values := tools.QueryParamsToShellVars(tc) + values := api.QueryParamsToShellVars(tc) assert.ContainsStr(t, "string=foo", values, "") assert.ContainsStr(t, "list=foo,bar", values, "") } @@ -24,7 +24,7 @@ func TestHTTPHeadersToShellVars(t *testing.T) { "Content-Type": []string{"text/plain"}, "X-Foo-Bar": []string{"foo", "bar"}, } - values := tools.HTTPHeadersToShellVars(tc) + values := api.HTTPHeadersToShellVars(tc) assert.ContainsStr(t, "content_type=text/plain", values, "") assert.ContainsStr(t, "x_foo_bar=foo,bar", values, "") } diff --git a/pkg/auth/htpasswd-file_test.go b/pkg/auth/htpasswd-file_test.go deleted file mode 100644 index c131350..0000000 --- a/pkg/auth/htpasswd-file_test.go +++ /dev/null @@ -1,15 +0,0 @@ -package auth - -import ( - "testing" - - "github.com/ncarlier/webhookd/pkg/assert" -) - -func TestValidateCredentials(t *testing.T) { - htpasswdFile, err := NewHtpasswdFromFile("test.htpasswd") - assert.Nil(t, err, ".htpasswd file should be loaded") - assert.NotNil(t, htpasswdFile, ".htpasswd file should be loaded") - assert.Equal(t, true, htpasswdFile.validateCredentials("foo", "bar"), "credentials should be valid") - assert.Equal(t, false, htpasswdFile.validateCredentials("foo", "bir"), "credentials should not be valid") -} diff --git a/pkg/auth/test/htpasswd-file_test.go b/pkg/auth/test/htpasswd-file_test.go new file mode 100644 index 0000000..07e35f8 --- /dev/null +++ b/pkg/auth/test/htpasswd-file_test.go @@ -0,0 +1,23 @@ +package test + +import ( + "net/http" + "testing" + + "github.com/ncarlier/webhookd/pkg/assert" + "github.com/ncarlier/webhookd/pkg/auth" +) + +func TestValidateCredentials(t *testing.T) { + htpasswdFile, err := auth.NewHtpasswdFromFile("test.htpasswd") + assert.Nil(t, err, ".htpasswd file should be loaded") + assert.NotNil(t, htpasswdFile, ".htpasswd file should be loaded") + + req, err := http.NewRequest("POST", "http://localhost:8080", nil) + assert.Nil(t, err, "") + req.SetBasicAuth("foo", "bar") + assert.Equal(t, true, htpasswdFile.Validate(req), "credentials should be valid") + + req.SetBasicAuth("foo", "bad") + assert.Equal(t, false, htpasswdFile.Validate(req), "credentials should not be valid") +} diff --git a/pkg/auth/test.htpasswd b/pkg/auth/test/test.htpasswd similarity index 100% rename from pkg/auth/test.htpasswd rename to pkg/auth/test/test.htpasswd diff --git a/pkg/notification/main.go b/pkg/notification/notifier.go similarity index 100% rename from pkg/notification/main.go rename to pkg/notification/notifier.go diff --git a/pkg/tools/script_resolver.go b/pkg/worker/script_resolver.go similarity index 96% rename from pkg/tools/script_resolver.go rename to pkg/worker/script_resolver.go index 44f432f..8a3bc94 100644 --- a/pkg/tools/script_resolver.go +++ b/pkg/worker/script_resolver.go @@ -1,4 +1,4 @@ -package tools +package worker import ( "errors" diff --git a/pkg/tools_test/script_resolver_test.go b/pkg/worker/test/script_resolver_test.go similarity index 61% rename from pkg/tools_test/script_resolver_test.go rename to pkg/worker/test/script_resolver_test.go index 61732d3..400d805 100644 --- a/pkg/tools_test/script_resolver_test.go +++ b/pkg/worker/test/script_resolver_test.go @@ -1,32 +1,32 @@ -package tools_test +package test import ( "testing" "github.com/ncarlier/webhookd/pkg/assert" - "github.com/ncarlier/webhookd/pkg/tools" + "github.com/ncarlier/webhookd/pkg/worker" ) func TestResolveScript(t *testing.T) { - script, err := tools.ResolveScript("../../scripts", "../scripts/echo") + script, err := worker.ResolveScript("../../../scripts", "../scripts/echo") assert.Nil(t, err, "") - assert.Equal(t, "../../scripts/echo.sh", script, "") + assert.Equal(t, "../../../scripts/echo.sh", script, "") } func TestNotResolveScript(t *testing.T) { - _, err := tools.ResolveScript("../../scripts", "foo") + _, err := worker.ResolveScript("../../scripts", "foo") assert.NotNil(t, err, "") assert.Equal(t, "Script not found: ../../scripts/foo.sh", err.Error(), "") } func TestResolveBadScript(t *testing.T) { - _, err := tools.ResolveScript("../../scripts", "../tests/test_simple") + _, err := worker.ResolveScript("../../scripts", "../tests/test_simple") assert.NotNil(t, err, "") assert.Equal(t, "Invalid script path: ../tests/test_simple.sh", err.Error(), "") } func TestResolveScriptWithExtension(t *testing.T) { - _, err := tools.ResolveScript("../../scripts", "node.js") + _, err := worker.ResolveScript("../../scripts", "node.js") assert.NotNil(t, err, "") assert.Equal(t, "Script not found: ../../scripts/node.js", err.Error(), "") } diff --git a/tests/test_error.sh b/pkg/worker/test/test_error.sh similarity index 100% rename from tests/test_error.sh rename to pkg/worker/test/test_error.sh diff --git a/tests/test_simple.sh b/pkg/worker/test/test_simple.sh similarity index 100% rename from tests/test_simple.sh rename to pkg/worker/test/test_simple.sh diff --git a/tests/test_timeout.sh b/pkg/worker/test/test_timeout.sh similarity index 100% rename from tests/test_timeout.sh rename to pkg/worker/test/test_timeout.sh diff --git a/pkg/worker/work_runner_test.go b/pkg/worker/test/work_runner_test.go similarity index 85% rename from pkg/worker/work_runner_test.go rename to pkg/worker/test/work_runner_test.go index caf5f2e..bcf7710 100644 --- a/pkg/worker/work_runner_test.go +++ b/pkg/worker/test/work_runner_test.go @@ -1,4 +1,4 @@ -package worker +package test import ( "os" @@ -8,6 +8,7 @@ import ( "github.com/ncarlier/webhookd/pkg/assert" "github.com/ncarlier/webhookd/pkg/logger" "github.com/ncarlier/webhookd/pkg/model" + "github.com/ncarlier/webhookd/pkg/worker" ) func printWorkMessages(work *model.WorkRequest) { @@ -24,7 +25,7 @@ func printWorkMessages(work *model.WorkRequest) { func TestWorkRunner(t *testing.T) { logger.Init("debug") - script := "../../tests/test_simple.sh" + script := "./test_simple.sh" args := []string{ "name=foo", "user_agent=test", @@ -33,14 +34,14 @@ func TestWorkRunner(t *testing.T) { work := model.NewWorkRequest("test", script, payload, os.TempDir(), args, 5) assert.NotNil(t, work, "") printWorkMessages(work) - err := run(work) + err := worker.Run(work) assert.Nil(t, err, "") assert.Equal(t, work.Status, model.Success, "") assert.Equal(t, work.GetLogContent("notify:"), "OK\n", "") // Test that we can retrieve log file afterward id := strconv.FormatUint(work.ID, 10) - logFile, err := RetrieveLogFile(id, "test", os.TempDir()) + logFile, err := worker.RetrieveLogFile(id, "test", os.TempDir()) defer logFile.Close() assert.Nil(t, err, "Log file should exists") assert.NotNil(t, logFile, "Log file should be retrieve") @@ -48,11 +49,11 @@ func TestWorkRunner(t *testing.T) { func TestWorkRunnerWithError(t *testing.T) { logger.Init("debug") - script := "../../tests/test_error.sh" + script := "./test_error.sh" work := model.NewWorkRequest("test", script, "", os.TempDir(), []string{}, 5) assert.NotNil(t, work, "") printWorkMessages(work) - err := run(work) + err := worker.Run(work) assert.NotNil(t, err, "") assert.Equal(t, work.Status, model.Error, "") assert.Equal(t, "exit status 1", err.Error(), "") @@ -60,11 +61,11 @@ func TestWorkRunnerWithError(t *testing.T) { func TestWorkRunnerWithTimeout(t *testing.T) { logger.Init("debug") - script := "../../tests/test_timeout.sh" + script := "./test_timeout.sh" work := model.NewWorkRequest("test", script, "", os.TempDir(), []string{}, 1) assert.NotNil(t, work, "") printWorkMessages(work) - err := run(work) + err := worker.Run(work) assert.NotNil(t, err, "") assert.Equal(t, work.Status, model.Error, "") assert.Equal(t, "signal: killed", err.Error(), "") diff --git a/pkg/worker/work_runner.go b/pkg/worker/work_runner.go index 70c0951..a3d400c 100644 --- a/pkg/worker/work_runner.go +++ b/pkg/worker/work_runner.go @@ -23,7 +23,8 @@ func (c *ChanWriter) Write(p []byte) (int, error) { return len(p), nil } -func run(work *model.WorkRequest) error { +// Run work request +func Run(work *model.WorkRequest) error { work.Status = model.Running logger.Info.Printf("job %s#%d started...\n", work.Name, work.ID) logger.Debug.Printf("job %s#%d script: %s\n", work.Name, work.ID, work.Script) diff --git a/pkg/worker/worker.go b/pkg/worker/worker.go index 73ef3bc..5c13bb6 100644 --- a/pkg/worker/worker.go +++ b/pkg/worker/worker.go @@ -10,16 +10,15 @@ import ( "github.com/ncarlier/webhookd/pkg/notification" ) -// NewWorker creates, and returns a new Worker object. Its only argument -// is a channel that the worker can add itself to whenever it is done its -// work. +// NewWorker creates, and returns a new Worker object. func NewWorker(id int, workerQueue chan chan model.WorkRequest) Worker { // Create, and return the worker. worker := Worker{ ID: id, Work: make(chan model.WorkRequest), WorkerQueue: workerQueue, - QuitChan: make(chan bool)} + QuitChan: make(chan bool), + } return worker } @@ -45,7 +44,7 @@ func (w Worker) Start() { // Receive a work request. logger.Debug.Printf("worker #%d received job request: %s#%d\n", w.ID, work.Name, work.ID) metric.Requests.Add(1) - err := run(&work) + err := Run(&work) if err != nil { metric.RequestsFailed.Add(1) work.MessageChan <- []byte(fmt.Sprintf("error: %s", err.Error()))