From e573855a4f040abd4aa6a2afa9ce610a1ec2670f Mon Sep 17 00:00:00 2001 From: Unknown Date: Tue, 6 May 2014 11:50:31 -0400 Subject: [PATCH] Fix #98, support web hook --- README.md | 4 +- README_ZH.md | 4 +- gogs.go | 2 +- models/action.go | 62 ++++-- models/update.go | 2 +- models/webhook.go | 15 +- modules/hooks/hooks.go | 83 ++++++++ modules/httplib/README.md | 62 ++++++ modules/httplib/httplib.go | 330 ++++++++++++++++++++++++++++++++ modules/httplib/httplib_test.go | 32 ++++ routers/repo/setting.go | 4 +- templates/repo/hooks.tmpl | 2 +- templates/repo/hooks_edit.tmpl | 2 +- 13 files changed, 581 insertions(+), 23 deletions(-) create mode 100644 modules/hooks/hooks.go create mode 100755 modules/httplib/README.md create mode 100755 modules/httplib/httplib.go create mode 100755 modules/httplib/httplib_test.go diff --git a/README.md b/README.md index 5e45c5fc95..7e903abd5d 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ Gogs(Go Git Service) is a Self Hosted Git Service in the Go Programming Language ![Demo](http://gowalker.org/public/gogs_demo.gif) -##### Current version: 0.3.2 Alpha +##### Current version: 0.3.3 Alpha ### NOTICES @@ -35,7 +35,7 @@ More importantly, Gogs only needs one binary to setup your own project hosting o - SSH/HTTP(S) protocol support. - Register/delete/rename account. - Create/migrate/mirror/delete/watch/rename/transfer public/private repository. -- Repository viewer/release/issue tracker. +- Repository viewer/release/issue tracker/webhooks. - Add/remove repository collaborators. - Gravatar and cache support. - Mail service(register, issue). diff --git a/README_ZH.md b/README_ZH.md index 9911714462..d1f742d44d 100644 --- a/README_ZH.md +++ b/README_ZH.md @@ -5,7 +5,7 @@ Gogs(Go Git Service) 是一个由 Go 语言编写的自助 Git 托管服务。 ![Demo](http://gowalker.org/public/gogs_demo.gif) -##### 当前版本：0.3.2 Alpha +##### 当前版本：0.3.3 Alpha ## 开发目的 @@ -26,7 +26,7 @@ Gogs 完全使用 Go 语言来实现对 Git 数据的操作，实现 **零** 依 - SSH/HTTP(S) 协议支持 - 注册/删除/重命名用户 - 创建/迁移/镜像/删除/关注/重命名/转移 公开/私有 仓库 -- 仓库 浏览器/发布/缺陷追踪 +- 仓库 浏览器/发布/缺陷管理/Web 钩子 - 添加/删除 仓库协作者 - Gravatar 以及缓存支持 - 邮件服务（注册、Issue） diff --git a/gogs.go b/gogs.go index cb4d43fd88..ad5d7abc70 100644 --- a/gogs.go +++ b/gogs.go @@ -20,7 +20,7 @@ import ( // Test that go1.2 tag above is included in builds. main.go refers to this definition. const go12tag = true -const APP_VER = "0.3.2.0505 Alpha" +const APP_VER = "0.3.3.0506 Alpha" func init() { base.AppVer = APP_VER diff --git a/models/action.go b/models/action.go index 8d8713b8a6..231a7a5485 100644 --- a/models/action.go +++ b/models/action.go @@ -6,6 +6,8 @@ package models import ( "encoding/json" + "errors" + "fmt" "strings" "time" @@ -13,6 +15,7 @@ import ( qlog "github.com/qiniu/log" "github.com/gogits/gogs/modules/base" + "github.com/gogits/gogs/modules/hooks" "github.com/gogits/gogs/modules/log" ) @@ -73,45 +76,80 @@ func (a Action) GetContent() string { // CommitRepoAction adds new action for committing repository. func CommitRepoAction(userId, repoUserId int64, userName, actEmail string, - repoId int64, repoUserName, repoName string, refName string, commit *base.PushCommits) error { + repoId int64, repoUserName, repoName string, refFullName string, commit *base.PushCommits) error { // log.Trace("action.CommitRepoAction(start): %d/%s", userId, repoName) opType := OP_COMMIT_REPO // Check it's tag push or branch. - if strings.HasPrefix(refName, "refs/tags/") { + if strings.HasPrefix(refFullName, "refs/tags/") { opType = OP_PUSH_TAG commit = &base.PushCommits{} } - refName = git.RefEndName(refName) + refName := git.RefEndName(refFullName) bs, err := json.Marshal(commit) if err != nil { - qlog.Error("action.CommitRepoAction(json): %d/%s", repoUserId, repoName) - return err + return errors.New("action.CommitRepoAction(json): " + err.Error()) } // Change repository bare status and update last updated time. repo, err := GetRepositoryByName(repoUserId, repoName) if err != nil { - qlog.Error("action.CommitRepoAction(GetRepositoryByName): %d/%s", repoUserId, repoName) - return err + return errors.New("action.CommitRepoAction(GetRepositoryByName): " + err.Error()) } repo.IsBare = false if err = UpdateRepository(repo); err != nil { - qlog.Error("action.CommitRepoAction(UpdateRepository): %d/%s", repoUserId, repoName) - return err + return errors.New("action.CommitRepoAction(UpdateRepository): " + err.Error()) } if err = NotifyWatchers(&Action{ActUserId: userId, ActUserName: userName, ActEmail: actEmail, OpType: opType, Content: string(bs), RepoId: repoId, RepoUserName: repoUserName, RepoName: repoName, RefName: refName, IsPrivate: repo.IsPrivate}); err != nil { - qlog.Error("action.CommitRepoAction(notify watchers): %d/%s", userId, repoName) - return err + return errors.New("action.CommitRepoAction(NotifyWatchers): " + err.Error()) + + } + qlog.Info("action.CommitRepoAction(end): %d/%s", repoUserId, repoName) + + // New push event hook. + ws, err := GetActiveWebhooksByRepoId(repoId) + if err != nil { + return errors.New("action.CommitRepoAction(GetWebhooksByRepoId): " + err.Error()) + } else if len(ws) == 0 { + return nil } - qlog.Info("action.CommitRepoAction(end): %d/%s", repoUserId, repoName) + commits := make([]*hooks.PayloadCommit, len(commit.Commits)) + for i, cmt := range commit.Commits { + commits[i] = &hooks.PayloadCommit{ + Id: cmt.Sha1, + Message: cmt.Message, + Url: fmt.Sprintf("%s%s/%s/commit/%s", base.AppUrl, repoUserName, repoName, cmt.Sha1), + Author: &hooks.PayloadAuthor{ + Name: cmt.AuthorName, + Email: cmt.AuthorEmail, + }, + } + } + p := &hooks.Payload{ + Ref: refFullName, + Commits: commits, + Pusher: &hooks.PayloadAuthor{ + Name: userName, + Email: actEmail, + }, + } + + for _, w := range ws { + w.GetEvent() + if !w.HasPushEvent() { + continue + } + + p.Secret = w.Secret + hooks.AddHookTask(&hooks.HookTask{hooks.HTT_WEBHOOK, w.Url, p, w.ContentType, w.IsSsl}) + } return nil } diff --git a/models/update.go b/models/update.go index 3ae5510a84..b7242cde85 100644 --- a/models/update.go +++ b/models/update.go @@ -96,6 +96,6 @@ func Update(refName, oldCommitId, newCommitId, userName, repoUserName, repoName //commits = append(commits, []string{lastCommit.Id().String(), lastCommit.Message()}) if err = CommitRepoAction(userId, ru.Id, userName, actEmail, repos.Id, repoUserName, repoName, refName, &base.PushCommits{l.Len(), commits}); err != nil { - qlog.Fatalf("runUpdate.models.CommitRepoAction: %v", err) + qlog.Fatalf("runUpdate.models.CommitRepoAction: %s/%s:%v", repoUserName, repoName, err) } } diff --git a/models/webhook.go b/models/webhook.go index daa4510f2d..f10fa2131e 100644 --- a/models/webhook.go +++ b/models/webhook.go @@ -28,7 +28,7 @@ type HookEvent struct { type Webhook struct { Id int64 RepoId int64 - Payload string `xorm:"TEXT"` + Url string `xorm:"TEXT"` ContentType int Secret string `xorm:"TEXT"` Events string `xorm:"TEXT"` @@ -50,6 +50,13 @@ func (w *Webhook) SaveEvent() error { return err } +func (w *Webhook) HasPushEvent() bool { + if w.PushOnly { + return true + } + return false +} + // CreateWebhook creates new webhook. func CreateWebhook(w *Webhook) error { _, err := orm.Insert(w) @@ -74,6 +81,12 @@ func GetWebhookById(hookId int64) (*Webhook, error) { return w, nil } +// GetActiveWebhooksByRepoId returns all active webhooks of repository. +func GetActiveWebhooksByRepoId(repoId int64) (ws []*Webhook, err error) { + err = orm.Find(&ws, &Webhook{RepoId: repoId, IsActive: true}) + return ws, err +} + // GetWebhooksByRepoId returns all webhooks of repository. func GetWebhooksByRepoId(repoId int64) (ws []*Webhook, err error) { err = orm.Find(&ws, &Webhook{RepoId: repoId}) diff --git a/modules/hooks/hooks.go b/modules/hooks/hooks.go new file mode 100644 index 0000000000..2b53dbfbbd --- /dev/null +++ b/modules/hooks/hooks.go @@ -0,0 +1,83 @@ +// Copyright 2014 The Gogs Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package hooks + +import ( + "encoding/json" + "time" + + "github.com/gogits/gogs/modules/httplib" + "github.com/gogits/gogs/modules/log" +) + +// Hook task types. +const ( + HTT_WEBHOOK = iota + 1 + HTT_SERVICE +) + +type PayloadAuthor struct { + Name string `json:"name"` + Email string `json:"email"` +} + +type PayloadCommit struct { + Id string `json:"id"` + Message string `json:"message"` + Url string `json:"url"` + Author *PayloadAuthor `json:"author"` +} + +// Payload represents payload information of hook. +type Payload struct { + Secret string `json:"secret"` + Ref string `json:"ref"` + Commits []*PayloadCommit `json:"commits"` + Pusher *PayloadAuthor `json:"pusher"` +} + +// HookTask represents hook task. +type HookTask struct { + Type int + Url string + *Payload + ContentType int + IsSsl bool +} + +var ( + taskQueue = make(chan *HookTask, 1000) +) + +// AddHookTask adds new hook task to task queue. +func AddHookTask(t *HookTask) { + taskQueue <- t +} + +func init() { + go handleQueue() +} + +func handleQueue() { + for { + select { + case t := <-taskQueue: + // Only support JSON now. + data, err := json.MarshalIndent(t.Payload, "", "\t") + if err != nil { + log.Error("hooks.handleQueue(json): %v", err) + continue + } + + _, err = httplib.Post(t.Url).SetTimeout(5*time.Second, 5*time.Second). + Body(data).Response() + if err != nil { + log.Error("hooks.handleQueue: Fail to deliver hook: %v", err) + continue + } + log.Info("Hook delivered") + } + } +} diff --git a/modules/httplib/README.md b/modules/httplib/README.md new file mode 100755 index 0000000000..95a10d8677 --- /dev/null +++ b/modules/httplib/README.md @@ -0,0 +1,62 @@ +# httplib +httplib is an libs help you to curl remote url. + +# How to use? + +## GET +you can use Get to crawl data. + + import "httplib" + + str, err := httplib.Get("http://beego.me/").String() + if err != nil { + t.Fatal(err) + } + fmt.Println(str) + +## POST +POST data to remote url + + b:=httplib.Post("http://beego.me/") + b.Param("username","astaxie") + b.Param("password","123456") + str, err := b.String() + if err != nil { + t.Fatal(err) + } + fmt.Println(str) + +## set timeout +you can set timeout in request.default is 60 seconds. + +set Get timeout: + + httplib.Get("http://beego.me/").SetTimeout(100 * time.Second, 30 * time.Second) + +set post timeout: + + httplib.Post("http://beego.me/").SetTimeout(100 * time.Second, 30 * time.Second) + +- first param is connectTimeout. +- second param is readWriteTimeout + +## debug +if you want to debug the request info, set the debug on + + httplib.Get("http://beego.me/").Debug(true) + +## support HTTPS client +if request url is https. You can set the client support TSL: + + httplib.SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true}) + +more info about the tls.Config please visit http://golang.org/pkg/crypto/tls/#Config + +## set cookie +some http request need setcookie. So set it like this: + + cookie := &http.Cookie{} + cookie.Name = "username" + cookie.Value = "astaxie" + httplib.Get("http://beego.me/").SetCookie(cookie) + diff --git a/modules/httplib/httplib.go b/modules/httplib/httplib.go new file mode 100755 index 0000000000..35870c1f1c --- /dev/null +++ b/modules/httplib/httplib.go @@ -0,0 +1,330 @@ +// Copyright 2013 The Beego Authors. All rights reserved. +// Copyright 2014 The Gogs Authors. All rights reserved. +// Use of this source code is governed by a MIT-style +// license that can be found in the LICENSE file. + +package httplib + +import ( + "bytes" + "crypto/tls" + "encoding/json" + "encoding/xml" + "io" + "io/ioutil" + "net" + "net/http" + "net/http/httputil" + "net/url" + "os" + "strings" + "time" +) + +var defaultUserAgent = "gogsServer" + +// Get returns *BeegoHttpRequest with GET method. +func Get(url string) *BeegoHttpRequest { + var req http.Request + req.Method = "GET" + req.Header = http.Header{} + req.Header.Set("User-Agent", defaultUserAgent) + return &BeegoHttpRequest{url, &req, map[string]string{}, false, 60 * time.Second, 60 * time.Second, nil, nil, nil} +} + +// Post returns *BeegoHttpRequest with POST method. +func Post(url string) *BeegoHttpRequest { + var req http.Request + req.Method = "POST" + req.Header = http.Header{} + req.Header.Set("User-Agent", defaultUserAgent) + return &BeegoHttpRequest{url, &req, map[string]string{}, false, 60 * time.Second, 60 * time.Second, nil, nil, nil} +} + +// Put returns *BeegoHttpRequest with PUT method. +func Put(url string) *BeegoHttpRequest { + var req http.Request + req.Method = "PUT" + req.Header = http.Header{} + req.Header.Set("User-Agent", defaultUserAgent) + return &BeegoHttpRequest{url, &req, map[string]string{}, false, 60 * time.Second, 60 * time.Second, nil, nil, nil} +} + +// Delete returns *BeegoHttpRequest DELETE GET method. +func Delete(url string) *BeegoHttpRequest { + var req http.Request + req.Method = "DELETE" + req.Header = http.Header{} + req.Header.Set("User-Agent", defaultUserAgent) + return &BeegoHttpRequest{url, &req, map[string]string{}, false, 60 * time.Second, 60 * time.Second, nil, nil, nil} +} + +// Head returns *BeegoHttpRequest with HEAD method. +func Head(url string) *BeegoHttpRequest { + var req http.Request + req.Method = "HEAD" + req.Header = http.Header{} + req.Header.Set("User-Agent", defaultUserAgent) + return &BeegoHttpRequest{url, &req, map[string]string{}, false, 60 * time.Second, 60 * time.Second, nil, nil, nil} +} + +// BeegoHttpRequest provides more useful methods for requesting one url than http.Request. +type BeegoHttpRequest struct { + url string + req *http.Request + params map[string]string + showdebug bool + connectTimeout time.Duration + readWriteTimeout time.Duration + tlsClientConfig *tls.Config + proxy func(*http.Request) (*url.URL, error) + transport http.RoundTripper +} + +// Debug sets show debug or not when executing request. +func (b *BeegoHttpRequest) Debug(isdebug bool) *BeegoHttpRequest { + b.showdebug = isdebug + return b +} + +// SetTimeout sets connect time out and read-write time out for BeegoRequest. +func (b *BeegoHttpRequest) SetTimeout(connectTimeout, readWriteTimeout time.Duration) *BeegoHttpRequest { + b.connectTimeout = connectTimeout + b.readWriteTimeout = readWriteTimeout + return b +} + +// SetTLSClientConfig sets tls connection configurations if visiting https url. +func (b *BeegoHttpRequest) SetTLSClientConfig(config *tls.Config) *BeegoHttpRequest { + b.tlsClientConfig = config + return b +} + +// Header add header item string in request. +func (b *BeegoHttpRequest) Header(key, value string) *BeegoHttpRequest { + b.req.Header.Set(key, value) + return b +} + +// SetCookie add cookie into request. +func (b *BeegoHttpRequest) SetCookie(cookie *http.Cookie) *BeegoHttpRequest { + b.req.Header.Add("Cookie", cookie.String()) + return b +} + +// Set transport to +func (b *BeegoHttpRequest) SetTransport(transport http.RoundTripper) *BeegoHttpRequest { + b.transport = transport + return b +} + +// Set http proxy +// example: +// +// func(req *http.Request) (*url.URL, error) { +// u, _ := url.ParseRequestURI("http://127.0.0.1:8118") +// return u, nil +// } +func (b *BeegoHttpRequest) SetProxy(proxy func(*http.Request) (*url.URL, error)) *BeegoHttpRequest { + b.proxy = proxy + return b +} + +// Param adds query param in to request. +// params build query string as ?key1=value1&key2=value2... +func (b *BeegoHttpRequest) Param(key, value string) *BeegoHttpRequest { + b.params[key] = value + return b +} + +// Body adds request raw body. +// it supports string and []byte. +func (b *BeegoHttpRequest) Body(data interface{}) *BeegoHttpRequest { + switch t := data.(type) { + case string: + bf := bytes.NewBufferString(t) + b.req.Body = ioutil.NopCloser(bf) + b.req.ContentLength = int64(len(t)) + case []byte: + bf := bytes.NewBuffer(t) + b.req.Body = ioutil.NopCloser(bf) + b.req.ContentLength = int64(len(t)) + } + return b +} + +func (b *BeegoHttpRequest) getResponse() (*http.Response, error) { + var paramBody string + if len(b.params) > 0 { + var buf bytes.Buffer + for k, v := range b.params { + buf.WriteString(url.QueryEscape(k)) + buf.WriteByte('=') + buf.WriteString(url.QueryEscape(v)) + buf.WriteByte('&') + } + paramBody = buf.String() + paramBody = paramBody[0 : len(paramBody)-1] + } + + if b.req.Method == "GET" && len(paramBody) > 0 { + if strings.Index(b.url, "?") != -1 { + b.url += "&" + paramBody + } else { + b.url = b.url + "?" + paramBody + } + } else if b.req.Method == "POST" && b.req.Body == nil && len(paramBody) > 0 { + b.Header("Content-Type", "application/x-www-form-urlencoded") + b.Body(paramBody) + } + + url, err := url.Parse(b.url) + if url.Scheme == "" { + b.url = "http://" + b.url + url, err = url.Parse(b.url) + } + if err != nil { + return nil, err + } + + b.req.URL = url + if b.showdebug { + dump, err := httputil.DumpRequest(b.req, true) + if err != nil { + println(err.Error()) + } + println(string(dump)) + } + + trans := b.transport + + if trans == nil { + // create default transport + trans = &http.Transport{ + TLSClientConfig: b.tlsClientConfig, + Proxy: b.proxy, + Dial: TimeoutDialer(b.connectTimeout, b.readWriteTimeout), + } + } else { + // if b.transport is *http.Transport then set the settings. + if t, ok := trans.(*http.Transport); ok { + if t.TLSClientConfig == nil { + t.TLSClientConfig = b.tlsClientConfig + } + if t.Proxy == nil { + t.Proxy = b.proxy + } + if t.Dial == nil { + t.Dial = TimeoutDialer(b.connectTimeout, b.readWriteTimeout) + } + } + } + + client := &http.Client{ + Transport: trans, + } + + resp, err := client.Do(b.req) + if err != nil { + return nil, err + } + return resp, nil +} + +// String returns the body string in response. +// it calls Response inner. +func (b *BeegoHttpRequest) String() (string, error) { + data, err := b.Bytes() + if err != nil { + return "", err + } + + return string(data), nil +} + +// Bytes returns the body []byte in response. +// it calls Response inner. +func (b *BeegoHttpRequest) Bytes() ([]byte, error) { + resp, err := b.getResponse() + if err != nil { + return nil, err + } + if resp.Body == nil { + return nil, nil + } + defer resp.Body.Close() + data, err := ioutil.ReadAll(resp.Body) + if err != nil { + return nil, err + } + return data, nil +} + +// ToFile saves the body data in response to one file. +// it calls Response inner. +func (b *BeegoHttpRequest) ToFile(filename string) error { + f, err := os.Create(filename) + if err != nil { + return err + } + defer f.Close() + + resp, err := b.getResponse() + if err != nil { + return err + } + if resp.Body == nil { + return nil + } + defer resp.Body.Close() + _, err = io.Copy(f, resp.Body) + if err != nil { + return err + } + return nil +} + +// ToJson returns the map that marshals from the body bytes as json in response . +// it calls Response inner. +func (b *BeegoHttpRequest) ToJson(v interface{}) error { + data, err := b.Bytes() + if err != nil { + return err + } + err = json.Unmarshal(data, v) + if err != nil { + return err + } + return nil +} + +// ToXml returns the map that marshals from the body bytes as xml in response . +// it calls Response inner. +func (b *BeegoHttpRequest) ToXML(v interface{}) error { + data, err := b.Bytes() + if err != nil { + return err + } + err = xml.Unmarshal(data, v) + if err != nil { + return err + } + return nil +} + +// Response executes request client gets response mannually. +func (b *BeegoHttpRequest) Response() (*http.Response, error) { + return b.getResponse() +} + +// TimeoutDialer returns functions of connection dialer with timeout settings for http.Transport Dial field. +func TimeoutDialer(cTimeout time.Duration, rwTimeout time.Duration) func(net, addr string) (c net.Conn, err error) { + return func(netw, addr string) (net.Conn, error) { + conn, err := net.DialTimeout(netw, addr, cTimeout) + if err != nil { + return nil, err + } + conn.SetDeadline(time.Now().Add(rwTimeout)) + return conn, nil + } +} diff --git a/modules/httplib/httplib_test.go b/modules/httplib/httplib_test.go new file mode 100755 index 0000000000..cc56dd8812 --- /dev/null +++ b/modules/httplib/httplib_test.go @@ -0,0 +1,32 @@ +package httplib + +import ( + "io/ioutil" + "testing" +) + +func TestGetUrl(t *testing.T) { + resp, err := Get("http://beego.me/").Debug(true).Response() + if err != nil { + t.Fatal(err) + } + if resp.Body == nil { + t.Fatal("body is nil") + } + data, err := ioutil.ReadAll(resp.Body) + defer resp.Body.Close() + if err != nil { + t.Fatal(err) + } + if len(data) == 0 { + t.Fatal("data is no") + } + + str, err := Get("http://beego.me/").String() + if err != nil { + t.Fatal(err) + } + if len(str) == 0 { + t.Fatal("has no info") + } +} diff --git a/routers/repo/setting.go b/routers/repo/setting.go index 6937d0b42a..96f9f857f1 100644 --- a/routers/repo/setting.go +++ b/routers/repo/setting.go @@ -247,7 +247,7 @@ func WebHooksAddPost(ctx *middleware.Context, form auth.NewWebhookForm) { w := &models.Webhook{ RepoId: ctx.Repo.Repository.Id, - Payload: form.Url, + Url: form.Url, ContentType: ct, Secret: form.Secret, HookEvent: &models.HookEvent{ @@ -315,7 +315,7 @@ func WebHooksEditPost(ctx *middleware.Context, params martini.Params, form auth. w := &models.Webhook{ Id: hookId, RepoId: ctx.Repo.Repository.Id, - Payload: form.Url, + Url: form.Url, ContentType: ct, Secret: form.Secret, HookEvent: &models.HookEvent{ diff --git a/templates/repo/hooks.tmpl b/templates/repo/hooks.tmpl index e1b36a67e8..85132e4b81 100644 --- a/templates/repo/hooks.tmpl +++ b/templates/repo/hooks.tmpl @@ -16,7 +16,7 @@ {{range .Webhooks}}
  • {{if .IsActive}}{{else}}{{end}} - {{.Payload}} + {{.Url}}
    • diff --git a/templates/repo/hooks_edit.tmpl b/templates/repo/hooks_edit.tmpl index f98ccef6f3..c3fe217e37 100644 --- a/templates/repo/hooks_edit.tmpl +++ b/templates/repo/hooks_edit.tmpl @@ -19,7 +19,7 @@
    - +