From b07d710f09722f968fa8fbb58d99d069904e7e3d Mon Sep 17 00:00:00 2001 From: shavit Date: Sat, 27 May 2017 22:28:09 -0400 Subject: [PATCH 1/6] Work with Tor --- browser/browser.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/browser/browser.go b/browser/browser.go index d7e2ac6..8006a49 100644 --- a/browser/browser.go +++ b/browser/browser.go @@ -13,6 +13,8 @@ import ( "strings" "time" + "golang.org/x/net/proxy" + "github.com/PuerkitoBio/goquery" "github.com/headzoo/surf/errors" "github.com/headzoo/surf/jar" @@ -78,6 +80,9 @@ type Browsable interface { // SetTransport sets the http library transport mechanism for each request. SetTransport(rt http.RoundTripper) + // Tor sets the proxy url for Tor + Tor(url *url.URL) (err error) + // AddRequestHeader adds a header the browser sends with each request. AddRequestHeader(name, value string) @@ -484,6 +489,17 @@ func (bow *Browser) SetTransport(rt http.RoundTripper) { bow.transport = rt } +// Tor sets the proxy url for Tor +func (bow *Browser) Tor(url *url.URL) (err error) { + dialer, err := proxy.FromURL(url, proxy.Direct) + if err != nil { + return err + } + bow.SetTransport(&http.Transport{Dial: dialer.Dial}) + + return err +} + // AddRequestHeader sets a header the browser sends with each request. func (bow *Browser) AddRequestHeader(name, value string) { bow.headers.Set(name, value) From 5f81455f07bc075e951ed3fe4859982d70d36854 Mon Sep 17 00:00:00 2001 From: shavit Date: Sat, 27 May 2017 22:28:09 -0400 Subject: [PATCH 2/6] Work with Tor --- browser/browser.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/browser/browser.go b/browser/browser.go index d7e2ac6..8006a49 100644 --- a/browser/browser.go +++ b/browser/browser.go @@ -13,6 +13,8 @@ import ( "strings" "time" + "golang.org/x/net/proxy" + "github.com/PuerkitoBio/goquery" "github.com/headzoo/surf/errors" "github.com/headzoo/surf/jar" @@ -78,6 +80,9 @@ type Browsable interface { // SetTransport sets the http library transport mechanism for each request. SetTransport(rt http.RoundTripper) + // Tor sets the proxy url for Tor + Tor(url *url.URL) (err error) + // AddRequestHeader adds a header the browser sends with each request. AddRequestHeader(name, value string) @@ -484,6 +489,17 @@ func (bow *Browser) SetTransport(rt http.RoundTripper) { bow.transport = rt } +// Tor sets the proxy url for Tor +func (bow *Browser) Tor(url *url.URL) (err error) { + dialer, err := proxy.FromURL(url, proxy.Direct) + if err != nil { + return err + } + bow.SetTransport(&http.Transport{Dial: dialer.Dial}) + + return err +} + // AddRequestHeader sets a header the browser sends with each request. func (bow *Browser) AddRequestHeader(name, value string) { bow.headers.Set(name, value) From 5948ff4af84a1e94b415912849b696c8f8526a52 Mon Sep 17 00:00:00 2001 From: shavit Date: Sun, 30 Jul 2017 23:03:18 -0400 Subject: [PATCH 3/6] renamed Tor to SetProxy(*url.URL), wrote a unit test --- browser/browser.go | 8 ++++---- browser/browser_test.go | 15 +++++++++++++++ 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/browser/browser.go b/browser/browser.go index 8006a49..a16083d 100644 --- a/browser/browser.go +++ b/browser/browser.go @@ -80,8 +80,8 @@ type Browsable interface { // SetTransport sets the http library transport mechanism for each request. SetTransport(rt http.RoundTripper) - // Tor sets the proxy url for Tor - Tor(url *url.URL) (err error) + // Set a proxy URL. You can use it with Tor + SetProxy(url *url.URL) (err error) // AddRequestHeader adds a header the browser sends with each request. AddRequestHeader(name, value string) @@ -489,8 +489,8 @@ func (bow *Browser) SetTransport(rt http.RoundTripper) { bow.transport = rt } -// Tor sets the proxy url for Tor -func (bow *Browser) Tor(url *url.URL) (err error) { +// Set a proxy url for Tor +func (bow *Browser) SetProxy(url *url.URL) (err error) { dialer, err := proxy.FromURL(url, proxy.Direct) if err != nil { return err diff --git a/browser/browser_test.go b/browser/browser_test.go index 20bf899..648a16c 100644 --- a/browser/browser_test.go +++ b/browser/browser_test.go @@ -4,6 +4,7 @@ import ( "io" "net/http" "net/http/httptest" + "net/url" "testing" "time" @@ -139,3 +140,17 @@ func TestCookieHeader(t *testing.T) { t.Errorf("got %d calls, want %d", calls, want) } } + +// Test proxy +// https://github.com/headzoo/surf/pull/56 +func TestSetProxyWillSetTransport(t *testing.T){ + b := newDefaultTestBrowser() + u, err := url.Parse("socks5://127.0.0.1:9050") + if err != nil { + t.Fatal(err) + } + b.SetProxy(u) + if b.transport == nil { + t.Errorf("no transport method") + } +} From 48ab8fb0218c70d194f745c4ae2f903027228493 Mon Sep 17 00:00:00 2001 From: shavit Date: Sun, 30 Jul 2017 23:13:45 -0400 Subject: [PATCH 4/6] proxy test update --- browser/browser_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/browser/browser_test.go b/browser/browser_test.go index 648a16c..b2490f3 100644 --- a/browser/browser_test.go +++ b/browser/browser_test.go @@ -150,7 +150,7 @@ func TestSetProxyWillSetTransport(t *testing.T){ t.Fatal(err) } b.SetProxy(u) - if b.transport == nil { + if b.client.Transport == nil { t.Errorf("no transport method") } } From 06b521e6b9a8326064e6304f33fbcb8ede384e20 Mon Sep 17 00:00:00 2001 From: shavit Date: Mon, 31 Jul 2017 06:45:24 -0400 Subject: [PATCH 5/6] passing a string instead of *url.URL to SetProxy --- browser/browser.go | 10 +++++++--- browser/browser_test.go | 7 +------ 2 files changed, 8 insertions(+), 9 deletions(-) diff --git a/browser/browser.go b/browser/browser.go index 992c5a3..8a75fe0 100644 --- a/browser/browser.go +++ b/browser/browser.go @@ -81,7 +81,7 @@ type Browsable interface { SetTransport(rt http.RoundTripper) // Set a proxy URL. You can use it with Tor - SetProxy(url *url.URL) (err error) + SetProxy(u string) (err error) // AddRequestHeader adds a header the browser sends with each request. AddRequestHeader(name, value string) @@ -502,8 +502,12 @@ func (bow *Browser) SetTransport(rt http.RoundTripper) { } // Set a proxy url for Tor -func (bow *Browser) SetProxy(url *url.URL) (err error) { - dialer, err := proxy.FromURL(url, proxy.Direct) +func (bow *Browser) SetProxy(u string) (err error) { + _url, err := url.Parse(u) + if err != nil { + return err + } + dialer, err := proxy.FromURL(_url, proxy.Direct) if err != nil { return err } diff --git a/browser/browser_test.go b/browser/browser_test.go index b2490f3..9c99ac2 100644 --- a/browser/browser_test.go +++ b/browser/browser_test.go @@ -4,7 +4,6 @@ import ( "io" "net/http" "net/http/httptest" - "net/url" "testing" "time" @@ -145,11 +144,7 @@ func TestCookieHeader(t *testing.T) { // https://github.com/headzoo/surf/pull/56 func TestSetProxyWillSetTransport(t *testing.T){ b := newDefaultTestBrowser() - u, err := url.Parse("socks5://127.0.0.1:9050") - if err != nil { - t.Fatal(err) - } - b.SetProxy(u) + b.SetProxy("socks5://127.0.0.1:9050") if b.client.Transport == nil { t.Errorf("no transport method") } From b2cca9c7b30ce30dfee5c93f96b186b502d85dd0 Mon Sep 17 00:00:00 2001 From: shavit Date: Tue, 1 Aug 2017 13:07:59 -0400 Subject: [PATCH 6/6] merge with github.com/headzoo/surf master --- README.md | 18 ++++-------------- agent/agent_appengine.go | 17 +++++++++++++++++ agent/agent_bsd.go | 1 + agent/agent_linux.go | 2 ++ agent/agent_linux_arm.go | 41 ++++++++++++++++++++++++++++++++++++++++ agent/agent_windows.go | 1 + 6 files changed, 66 insertions(+), 14 deletions(-) create mode 100644 agent/agent_appengine.go create mode 100644 agent/agent_linux_arm.go diff --git a/README.md b/README.md index 77acdc0..99851aa 100644 --- a/README.md +++ b/README.md @@ -12,11 +12,6 @@ like web browser, and includes: cookie management, history, bookmarking, user ag (with a nifty user agent builder), submitting forms, DOM selection and traversal via jQuery style CSS selectors, scraping assets like images, stylesheets, and other features. -**NOTE: DEVELOPMENT ON THIS PROJECT IS VERY SLOW!** -I'm accepting pull requests and fixing bugs at a snail's pace right now. Partially because Surf has taken a backseat to projects which are more important to me, and the way Go handles vendoring makes me hesitant to make changes. Development will pick up again when I have more free time. - -Work has started on v2.0 on the [v2 branch](https://github.com/headzoo/surf/tree/v2). - * [Installation](#installation) * [General Usage](#quick-start) * [Documentation](#documentation) @@ -58,17 +53,13 @@ Complete documentation is available on [Read the Docs](http://surf.readthedocs.i ### Credits -Surf was started by [Sean Hickey](https://github.com/headzoo) (headzoo) to learn more about the Go programming language. -The idea to create Surf was born in [this Reddit thread](http://www.reddit.com/r/golang/comments/2efw1q/mechanize_in_go/cjz4lze). - -[![Twitter](https://img.shields.io/badge/follow-%40WebSeanHickey-orange.svg?style=flat-square)](https://twitter.com/WebSeanHickey) - Surf uses the awesome [goquery](https://github.com/PuerkitoBio/goquery) by Martin Angers, and was written using [Intellij](http://www.jetbrains.com/idea/) and the [golang plugin](http://plugins.jetbrains.com/plugin/5047). Contributions have been made to Surf by the following awesome developers: +* [Sean Hickey](https://github.com/headzoo) * [Haruyama Seigo](https://github.com/haruyama) * [Tatsushi Demachi](https://github.com/tatsushid) * [Charl Matthee](https://github.com/charl) @@ -85,12 +76,11 @@ Contributions have been made to Surf by the following awesome developers: * [Joseph Watson](https://github.com/jtwatson) * [lxt2](https://github.com/lxt2) +The idea to create Surf was born in [this Reddit thread](http://www.reddit.com/r/golang/comments/2efw1q/mechanize_in_go/cjz4lze). + ### Contributing -Issues and pull requests are _always_ welcome. Code changes are made to the -[`dev`](https://github.com/headzoo/surf/tree/dev) branch. Once a milestone has been reached the branch will -be merged in with `master`, and a new version tag created. **Do not** make your changes against the -`master` branch, or they will _may_ be ignored. +Issues and pull requests are _always_ welcome. See [CONTRIBUTING.md](https://raw.githubusercontent.com/headzoo/surf/master/CONTRIBUTING.md) for more information. diff --git a/agent/agent_appengine.go b/agent/agent_appengine.go new file mode 100644 index 0000000..9ffb717 --- /dev/null +++ b/agent/agent_appengine.go @@ -0,0 +1,17 @@ +// +build appengine + +package agent + +import ( + "runtime" +) + +// osName returns the name of the OS. +func osName() string { + return runtime.GOOS +} + +// osVersion returns the OS version. +func osVersion() string { + return "0.0" +} diff --git a/agent/agent_bsd.go b/agent/agent_bsd.go index af9fee3..0dfe10b 100644 --- a/agent/agent_bsd.go +++ b/agent/agent_bsd.go @@ -1,4 +1,5 @@ // +build darwin dragonfly freebsd netbsd openbsd +// +build !appengine package agent diff --git a/agent/agent_linux.go b/agent/agent_linux.go index a81e75a..a1b10fe 100644 --- a/agent/agent_linux.go +++ b/agent/agent_linux.go @@ -1,4 +1,6 @@ // +build linux +// +build !appengine +// +build !arm package agent diff --git a/agent/agent_linux_arm.go b/agent/agent_linux_arm.go new file mode 100644 index 0000000..3bbc629 --- /dev/null +++ b/agent/agent_linux_arm.go @@ -0,0 +1,41 @@ +// +build linux,arm + +package agent + +import ( + "runtime" + "syscall" +) + +// osName returns the name of the OS. +func osName() string { + buf := &syscall.Utsname{} + err := syscall.Uname(buf) + if err != nil { + return runtime.GOOS + } + return charsToString(buf.Sysname) +} + +// osVersion returns the OS version. +func osVersion() string { + buf := &syscall.Utsname{} + err := syscall.Uname(buf) + if err != nil { + return "0.0" + } + return charsToString(buf.Release) +} + +// charsToString converts a [65]uint8 byte array into a string. +func charsToString(ca [65]uint8) string { + s := make([]byte, len(ca)) + var lens int + for ; lens < len(ca); lens++ { + if ca[lens] == 0 { + break + } + s[lens] = uint8(ca[lens]) + } + return string(s[0:lens]) +} diff --git a/agent/agent_windows.go b/agent/agent_windows.go index 586893a..7fc69bc 100644 --- a/agent/agent_windows.go +++ b/agent/agent_windows.go @@ -1,4 +1,5 @@ // +build windows +// +build !appengine package agent