diff --git a/README.md b/README.md index 7dd6279..99851aa 100644 --- a/README.md +++ b/README.md @@ -6,7 +6,7 @@ Surf [![Documentation](https://img.shields.io/badge/documentation-readthedocs-blue.svg?style=flat-square)](http://surf.readthedocs.io/) [![MIT License](https://img.shields.io/badge/license-MIT-blue.svg?style=flat-square)](https://raw.githubusercontent.com/headzoo/surf/master/LICENSE.md) -Surf is a Go (golang) library that implements a virtual web browser that you control pragmatically. +Surf is a Go (golang) library that implements a virtual web browser that you control programmatically. Surf isn't just another Go solution for downloading content from the web. Surf is designed to behave like web browser, and includes: cookie management, history, bookmarking, user agent spoofing (with a nifty user agent builder), submitting forms, DOM selection and traversal via jQuery style @@ -53,17 +53,13 @@ Complete documentation is available on [Read the Docs](http://surf.readthedocs.i ### Credits -Surf was started by [Sean Hickey](https://github.com/headzoo) (headzoo) to learn more about the Go programming language. -The idea to create Surf was born in [this Reddit thread](http://www.reddit.com/r/golang/comments/2efw1q/mechanize_in_go/cjz4lze). - -[![Twitter](https://img.shields.io/badge/follow-%40WebSeanHickey-orange.svg?style=flat-square)](https://twitter.com/WebSeanHickey) - Surf uses the awesome [goquery](https://github.com/PuerkitoBio/goquery) by Martin Angers, and was written using [Intellij](http://www.jetbrains.com/idea/) and the [golang plugin](http://plugins.jetbrains.com/plugin/5047). Contributions have been made to Surf by the following awesome developers: +* [Sean Hickey](https://github.com/headzoo) * [Haruyama Seigo](https://github.com/haruyama) * [Tatsushi Demachi](https://github.com/tatsushid) * [Charl Matthee](https://github.com/charl) @@ -80,12 +76,11 @@ Contributions have been made to Surf by the following awesome developers: * [Joseph Watson](https://github.com/jtwatson) * [lxt2](https://github.com/lxt2) +The idea to create Surf was born in [this Reddit thread](http://www.reddit.com/r/golang/comments/2efw1q/mechanize_in_go/cjz4lze). + ### Contributing -Issues and pull requests are _always_ welcome. Code changes are made to the -[`dev`](https://github.com/headzoo/surf/tree/dev) branch. Once a milestone has been reached the branch will -be merged in with `master`, and a new version tag created. **Do not** make your changes against the -`master` branch, or they will _may_ be ignored. +Issues and pull requests are _always_ welcome. See [CONTRIBUTING.md](https://raw.githubusercontent.com/headzoo/surf/master/CONTRIBUTING.md) for more information. diff --git a/agent/agent_appengine.go b/agent/agent_appengine.go new file mode 100644 index 0000000..9ffb717 --- /dev/null +++ b/agent/agent_appengine.go @@ -0,0 +1,17 @@ +// +build appengine + +package agent + +import ( + "runtime" +) + +// osName returns the name of the OS. +func osName() string { + return runtime.GOOS +} + +// osVersion returns the OS version. +func osVersion() string { + return "0.0" +} diff --git a/agent/agent_bsd.go b/agent/agent_bsd.go index af9fee3..0dfe10b 100644 --- a/agent/agent_bsd.go +++ b/agent/agent_bsd.go @@ -1,4 +1,5 @@ // +build darwin dragonfly freebsd netbsd openbsd +// +build !appengine package agent diff --git a/agent/agent_linux.go b/agent/agent_linux.go index a81e75a..a1b10fe 100644 --- a/agent/agent_linux.go +++ b/agent/agent_linux.go @@ -1,4 +1,6 @@ // +build linux +// +build !appengine +// +build !arm package agent diff --git a/agent/agent_linux_arm.go b/agent/agent_linux_arm.go new file mode 100644 index 0000000..3bbc629 --- /dev/null +++ b/agent/agent_linux_arm.go @@ -0,0 +1,41 @@ +// +build linux,arm + +package agent + +import ( + "runtime" + "syscall" +) + +// osName returns the name of the OS. +func osName() string { + buf := &syscall.Utsname{} + err := syscall.Uname(buf) + if err != nil { + return runtime.GOOS + } + return charsToString(buf.Sysname) +} + +// osVersion returns the OS version. +func osVersion() string { + buf := &syscall.Utsname{} + err := syscall.Uname(buf) + if err != nil { + return "0.0" + } + return charsToString(buf.Release) +} + +// charsToString converts a [65]uint8 byte array into a string. +func charsToString(ca [65]uint8) string { + s := make([]byte, len(ca)) + var lens int + for ; lens < len(ca); lens++ { + if ca[lens] == 0 { + break + } + s[lens] = uint8(ca[lens]) + } + return string(s[0:lens]) +} diff --git a/agent/agent_windows.go b/agent/agent_windows.go index 586893a..7fc69bc 100644 --- a/agent/agent_windows.go +++ b/agent/agent_windows.go @@ -1,4 +1,5 @@ // +build windows +// +build !appengine package agent diff --git a/browser/browser.go b/browser/browser.go index 1588be2..8a75fe0 100644 --- a/browser/browser.go +++ b/browser/browser.go @@ -13,6 +13,8 @@ import ( "strings" "time" + "golang.org/x/net/proxy" + "github.com/PuerkitoBio/goquery" "github.com/headzoo/surf/errors" "github.com/headzoo/surf/jar" @@ -78,6 +80,9 @@ type Browsable interface { // SetTransport sets the http library transport mechanism for each request. SetTransport(rt http.RoundTripper) + // Set a proxy URL. You can use it with Tor + SetProxy(u string) (err error) + // AddRequestHeader adds a header the browser sends with each request. AddRequestHeader(name, value string) @@ -496,6 +501,21 @@ func (bow *Browser) SetTransport(rt http.RoundTripper) { bow.client.Transport = rt } +// Set a proxy url for Tor +func (bow *Browser) SetProxy(u string) (err error) { + _url, err := url.Parse(u) + if err != nil { + return err + } + dialer, err := proxy.FromURL(_url, proxy.Direct) + if err != nil { + return err + } + bow.SetTransport(&http.Transport{Dial: dialer.Dial}) + + return err +} + // AddRequestHeader sets a header the browser sends with each request. func (bow *Browser) AddRequestHeader(name, value string) { bow.headers.Set(name, value) diff --git a/browser/browser_test.go b/browser/browser_test.go index 20bf899..9c99ac2 100644 --- a/browser/browser_test.go +++ b/browser/browser_test.go @@ -139,3 +139,13 @@ func TestCookieHeader(t *testing.T) { t.Errorf("got %d calls, want %d", calls, want) } } + +// Test proxy +// https://github.com/headzoo/surf/pull/56 +func TestSetProxyWillSetTransport(t *testing.T){ + b := newDefaultTestBrowser() + b.SetProxy("socks5://127.0.0.1:9050") + if b.client.Transport == nil { + t.Errorf("no transport method") + } +}