aboutsummaryrefslogtreecommitdiffstats
path: root/modules/httplib
diff options
context:
space:
mode:
authorUnknown <joe2010xtmf@163.com>2014-05-06 11:50:31 -0400
committerUnknown <joe2010xtmf@163.com>2014-05-06 11:50:31 -0400
commite573855a4f040abd4aa6a2afa9ce610a1ec2670f (patch)
tree7ead7dafd3c228305e83a19dec0bca5b60d92217 /modules/httplib
parent94bccbb148a4ed897eb8332fc746aa75486c5c4d (diff)
downloadgitea-e573855a4f040abd4aa6a2afa9ce610a1ec2670f.tar.gz
gitea-e573855a4f040abd4aa6a2afa9ce610a1ec2670f.zip
Fix #98, support web hook
Diffstat (limited to 'modules/httplib')
-rwxr-xr-xmodules/httplib/README.md62
-rwxr-xr-xmodules/httplib/httplib.go330
-rwxr-xr-xmodules/httplib/httplib_test.go32
3 files changed, 424 insertions, 0 deletions
diff --git a/modules/httplib/README.md b/modules/httplib/README.md
new file mode 100755
index 0000000000..95a10d8677
--- /dev/null
+++ b/modules/httplib/README.md
@@ -0,0 +1,62 @@
+# httplib
+httplib is an libs help you to curl remote url.
+
+# How to use?
+
+## GET
+you can use Get to crawl data.
+
+ import "httplib"
+
+ str, err := httplib.Get("http://beego.me/").String()
+ if err != nil {
+ t.Fatal(err)
+ }
+ fmt.Println(str)
+
+## POST
+POST data to remote url
+
+ b:=httplib.Post("http://beego.me/")
+ b.Param("username","astaxie")
+ b.Param("password","123456")
+ str, err := b.String()
+ if err != nil {
+ t.Fatal(err)
+ }
+ fmt.Println(str)
+
+## set timeout
+you can set timeout in request.default is 60 seconds.
+
+set Get timeout:
+
+ httplib.Get("http://beego.me/").SetTimeout(100 * time.Second, 30 * time.Second)
+
+set post timeout:
+
+ httplib.Post("http://beego.me/").SetTimeout(100 * time.Second, 30 * time.Second)
+
+- first param is connectTimeout.
+- second param is readWriteTimeout
+
+## debug
+if you want to debug the request info, set the debug on
+
+ httplib.Get("http://beego.me/").Debug(true)
+
+## support HTTPS client
+if request url is https. You can set the client support TSL:
+
+ httplib.SetTLSClientConfig(&tls.Config{InsecureSkipVerify: true})
+
+more info about the tls.Config please visit http://golang.org/pkg/crypto/tls/#Config
+
+## set cookie
+some http request need setcookie. So set it like this:
+
+ cookie := &http.Cookie{}
+ cookie.Name = "username"
+ cookie.Value = "astaxie"
+ httplib.Get("http://beego.me/").SetCookie(cookie)
+
diff --git a/modules/httplib/httplib.go b/modules/httplib/httplib.go
new file mode 100755
index 0000000000..35870c1f1c
--- /dev/null
+++ b/modules/httplib/httplib.go
@@ -0,0 +1,330 @@
+// Copyright 2013 The Beego Authors. All rights reserved.
+// Copyright 2014 The Gogs Authors. All rights reserved.
+// Use of this source code is governed by a MIT-style
+// license that can be found in the LICENSE file.
+
+package httplib
+
+import (
+ "bytes"
+ "crypto/tls"
+ "encoding/json"
+ "encoding/xml"
+ "io"
+ "io/ioutil"
+ "net"
+ "net/http"
+ "net/http/httputil"
+ "net/url"
+ "os"
+ "strings"
+ "time"
+)
+
+var defaultUserAgent = "gogsServer"
+
+// Get returns *BeegoHttpRequest with GET method.
+func Get(url string) *BeegoHttpRequest {
+ var req http.Request
+ req.Method = "GET"
+ req.Header = http.Header{}
+ req.Header.Set("User-Agent", defaultUserAgent)
+ return &BeegoHttpRequest{url, &req, map[string]string{}, false, 60 * time.Second, 60 * time.Second, nil, nil, nil}
+}
+
+// Post returns *BeegoHttpRequest with POST method.
+func Post(url string) *BeegoHttpRequest {
+ var req http.Request
+ req.Method = "POST"
+ req.Header = http.Header{}
+ req.Header.Set("User-Agent", defaultUserAgent)
+ return &BeegoHttpRequest{url, &req, map[string]string{}, false, 60 * time.Second, 60 * time.Second, nil, nil, nil}
+}
+
+// Put returns *BeegoHttpRequest with PUT method.
+func Put(url string) *BeegoHttpRequest {
+ var req http.Request
+ req.Method = "PUT"
+ req.Header = http.Header{}
+ req.Header.Set("User-Agent", defaultUserAgent)
+ return &BeegoHttpRequest{url, &req, map[string]string{}, false, 60 * time.Second, 60 * time.Second, nil, nil, nil}
+}
+
+// Delete returns *BeegoHttpRequest DELETE GET method.
+func Delete(url string) *BeegoHttpRequest {
+ var req http.Request
+ req.Method = "DELETE"
+ req.Header = http.Header{}
+ req.Header.Set("User-Agent", defaultUserAgent)
+ return &BeegoHttpRequest{url, &req, map[string]string{}, false, 60 * time.Second, 60 * time.Second, nil, nil, nil}
+}
+
+// Head returns *BeegoHttpRequest with HEAD method.
+func Head(url string) *BeegoHttpRequest {
+ var req http.Request
+ req.Method = "HEAD"
+ req.Header = http.Header{}
+ req.Header.Set("User-Agent", defaultUserAgent)
+ return &BeegoHttpRequest{url, &req, map[string]string{}, false, 60 * time.Second, 60 * time.Second, nil, nil, nil}
+}
+
+// BeegoHttpRequest provides more useful methods for requesting one url than http.Request.
+type BeegoHttpRequest struct {
+ url string
+ req *http.Request
+ params map[string]string
+ showdebug bool
+ connectTimeout time.Duration
+ readWriteTimeout time.Duration
+ tlsClientConfig *tls.Config
+ proxy func(*http.Request) (*url.URL, error)
+ transport http.RoundTripper
+}
+
+// Debug sets show debug or not when executing request.
+func (b *BeegoHttpRequest) Debug(isdebug bool) *BeegoHttpRequest {
+ b.showdebug = isdebug
+ return b
+}
+
+// SetTimeout sets connect time out and read-write time out for BeegoRequest.
+func (b *BeegoHttpRequest) SetTimeout(connectTimeout, readWriteTimeout time.Duration) *BeegoHttpRequest {
+ b.connectTimeout = connectTimeout
+ b.readWriteTimeout = readWriteTimeout
+ return b
+}
+
+// SetTLSClientConfig sets tls connection configurations if visiting https url.
+func (b *BeegoHttpRequest) SetTLSClientConfig(config *tls.Config) *BeegoHttpRequest {
+ b.tlsClientConfig = config
+ return b
+}
+
+// Header add header item string in request.
+func (b *BeegoHttpRequest) Header(key, value string) *BeegoHttpRequest {
+ b.req.Header.Set(key, value)
+ return b
+}
+
+// SetCookie add cookie into request.
+func (b *BeegoHttpRequest) SetCookie(cookie *http.Cookie) *BeegoHttpRequest {
+ b.req.Header.Add("Cookie", cookie.String())
+ return b
+}
+
+// Set transport to
+func (b *BeegoHttpRequest) SetTransport(transport http.RoundTripper) *BeegoHttpRequest {
+ b.transport = transport
+ return b
+}
+
+// Set http proxy
+// example:
+//
+// func(req *http.Request) (*url.URL, error) {
+// u, _ := url.ParseRequestURI("http://127.0.0.1:8118")
+// return u, nil
+// }
+func (b *BeegoHttpRequest) SetProxy(proxy func(*http.Request) (*url.URL, error)) *BeegoHttpRequest {
+ b.proxy = proxy
+ return b
+}
+
+// Param adds query param in to request.
+// params build query string as ?key1=value1&key2=value2...
+func (b *BeegoHttpRequest) Param(key, value string) *BeegoHttpRequest {
+ b.params[key] = value
+ return b
+}
+
+// Body adds request raw body.
+// it supports string and []byte.
+func (b *BeegoHttpRequest) Body(data interface{}) *BeegoHttpRequest {
+ switch t := data.(type) {
+ case string:
+ bf := bytes.NewBufferString(t)
+ b.req.Body = ioutil.NopCloser(bf)
+ b.req.ContentLength = int64(len(t))
+ case []byte:
+ bf := bytes.NewBuffer(t)
+ b.req.Body = ioutil.NopCloser(bf)
+ b.req.ContentLength = int64(len(t))
+ }
+ return b
+}
+
+func (b *BeegoHttpRequest) getResponse() (*http.Response, error) {
+ var paramBody string
+ if len(b.params) > 0 {
+ var buf bytes.Buffer
+ for k, v := range b.params {
+ buf.WriteString(url.QueryEscape(k))
+ buf.WriteByte('=')
+ buf.WriteString(url.QueryEscape(v))
+ buf.WriteByte('&')
+ }
+ paramBody = buf.String()
+ paramBody = paramBody[0 : len(paramBody)-1]
+ }
+
+ if b.req.Method == "GET" && len(paramBody) > 0 {
+ if strings.Index(b.url, "?") != -1 {
+ b.url += "&" + paramBody
+ } else {
+ b.url = b.url + "?" + paramBody
+ }
+ } else if b.req.Method == "POST" && b.req.Body == nil && len(paramBody) > 0 {
+ b.Header("Content-Type", "application/x-www-form-urlencoded")
+ b.Body(paramBody)
+ }
+
+ url, err := url.Parse(b.url)
+ if url.Scheme == "" {
+ b.url = "http://" + b.url
+ url, err = url.Parse(b.url)
+ }
+ if err != nil {
+ return nil, err
+ }
+
+ b.req.URL = url
+ if b.showdebug {
+ dump, err := httputil.DumpRequest(b.req, true)
+ if err != nil {
+ println(err.Error())
+ }
+ println(string(dump))
+ }
+
+ trans := b.transport
+
+ if trans == nil {
+ // create default transport
+ trans = &http.Transport{
+ TLSClientConfig: b.tlsClientConfig,
+ Proxy: b.proxy,
+ Dial: TimeoutDialer(b.connectTimeout, b.readWriteTimeout),
+ }
+ } else {
+ // if b.transport is *http.Transport then set the settings.
+ if t, ok := trans.(*http.Transport); ok {
+ if t.TLSClientConfig == nil {
+ t.TLSClientConfig = b.tlsClientConfig
+ }
+ if t.Proxy == nil {
+ t.Proxy = b.proxy
+ }
+ if t.Dial == nil {
+ t.Dial = TimeoutDialer(b.connectTimeout, b.readWriteTimeout)
+ }
+ }
+ }
+
+ client := &http.Client{
+ Transport: trans,
+ }
+
+ resp, err := client.Do(b.req)
+ if err != nil {
+ return nil, err
+ }
+ return resp, nil
+}
+
+// String returns the body string in response.
+// it calls Response inner.
+func (b *BeegoHttpRequest) String() (string, error) {
+ data, err := b.Bytes()
+ if err != nil {
+ return "", err
+ }
+
+ return string(data), nil
+}
+
+// Bytes returns the body []byte in response.
+// it calls Response inner.
+func (b *BeegoHttpRequest) Bytes() ([]byte, error) {
+ resp, err := b.getResponse()
+ if err != nil {
+ return nil, err
+ }
+ if resp.Body == nil {
+ return nil, nil
+ }
+ defer resp.Body.Close()
+ data, err := ioutil.ReadAll(resp.Body)
+ if err != nil {
+ return nil, err
+ }
+ return data, nil
+}
+
+// ToFile saves the body data in response to one file.
+// it calls Response inner.
+func (b *BeegoHttpRequest) ToFile(filename string) error {
+ f, err := os.Create(filename)
+ if err != nil {
+ return err
+ }
+ defer f.Close()
+
+ resp, err := b.getResponse()
+ if err != nil {
+ return err
+ }
+ if resp.Body == nil {
+ return nil
+ }
+ defer resp.Body.Close()
+ _, err = io.Copy(f, resp.Body)
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
+// ToJson returns the map that marshals from the body bytes as json in response .
+// it calls Response inner.
+func (b *BeegoHttpRequest) ToJson(v interface{}) error {
+ data, err := b.Bytes()
+ if err != nil {
+ return err
+ }
+ err = json.Unmarshal(data, v)
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
+// ToXml returns the map that marshals from the body bytes as xml in response .
+// it calls Response inner.
+func (b *BeegoHttpRequest) ToXML(v interface{}) error {
+ data, err := b.Bytes()
+ if err != nil {
+ return err
+ }
+ err = xml.Unmarshal(data, v)
+ if err != nil {
+ return err
+ }
+ return nil
+}
+
+// Response executes request client gets response mannually.
+func (b *BeegoHttpRequest) Response() (*http.Response, error) {
+ return b.getResponse()
+}
+
+// TimeoutDialer returns functions of connection dialer with timeout settings for http.Transport Dial field.
+func TimeoutDialer(cTimeout time.Duration, rwTimeout time.Duration) func(net, addr string) (c net.Conn, err error) {
+ return func(netw, addr string) (net.Conn, error) {
+ conn, err := net.DialTimeout(netw, addr, cTimeout)
+ if err != nil {
+ return nil, err
+ }
+ conn.SetDeadline(time.Now().Add(rwTimeout))
+ return conn, nil
+ }
+}
diff --git a/modules/httplib/httplib_test.go b/modules/httplib/httplib_test.go
new file mode 100755
index 0000000000..cc56dd8812
--- /dev/null
+++ b/modules/httplib/httplib_test.go
@@ -0,0 +1,32 @@
+package httplib
+
+import (
+ "io/ioutil"
+ "testing"
+)
+
+func TestGetUrl(t *testing.T) {
+ resp, err := Get("http://beego.me/").Debug(true).Response()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if resp.Body == nil {
+ t.Fatal("body is nil")
+ }
+ data, err := ioutil.ReadAll(resp.Body)
+ defer resp.Body.Close()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(data) == 0 {
+ t.Fatal("data is no")
+ }
+
+ str, err := Get("http://beego.me/").String()
+ if err != nil {
+ t.Fatal(err)
+ }
+ if len(str) == 0 {
+ t.Fatal("has no info")
+ }
+}