rabida/scroll_test.go at master · JohnnyTing/rabida · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
package service

import (
	"context"
	"fmt"
	"github.com/Jeffail/gabs/v2"
	"github.com/JohnnyTing/rabida/config"
	"github.com/chromedp/chromedp"
	"testing"
)

func TestRabidaImplCrawlScroll(t *testing.T) {
	t.Run("CrawlScroll", func(t *testing.T) {
		conf := config.LoadFromEnv()
		fmt.Printf("%+v\n", conf)
		rabi := NewRabida(conf)
		job := Job{
			Link: "http://www.news.cn/energy/index.html",
			CssSelector: CssSelector{
				Scope: `#content-list > div.item`,
				Attrs: map[string]CssSelector{
					"title": {
						Css: ".tit > a",
					},
					"date": {
						Css: ".time",
					},
				},
			},
			Paginator: CssSelector{
				Css: "#list > div.xpage-more-btn.look",
			},
			Limit: 10,
		}
		err := rabi.CrawlScroll(context.Background(), job, func(ret []interface{}, nextCursor int, currentPageNo int) bool {
			for _, item := range ret {
				fmt.Println(gabs.Wrap(item).StringIndent("", "  "))
			}
			if currentPageNo >= job.Limit {
				return true
			}
			return false
		}, nil, []chromedp.Action{
			chromedp.EmulateViewport(1777, 903, chromedp.EmulateLandscape),
		})
		if err != nil {
			t.Errorf("%+v", err)
		}

	})
}

func TestRabidaImplCrawlScrollXpath(t *testing.T) {
	t.Run("CrawlScrollXpath", func(t *testing.T) {
		conf := config.LoadFromEnv()
		fmt.Printf("%+v\n", conf)
		rabi := NewRabida(conf)
		job := Job{
			Link: "http://www.news.cn/energy/index.html",
			CssSelector: CssSelector{
				XpathScope: `//*[@id="content-list"]/div`,
				Attrs: map[string]CssSelector{
					"title": {
						Xpath: "//div[@class='tit']",
					},
					"date": {
						Xpath: "//div[@class='time']",
					},
				},
			},
			Paginator: CssSelector{
				Css: "#list > div.xpage-more-btn.look",
			},
			Limit: 10,
		}
		err := rabi.CrawlScroll(context.Background(), job, func(ret []interface{}, nextCursor int, currentPageNo int) bool {
			for _, item := range ret {
				fmt.Println(gabs.Wrap(item).StringIndent("", "  "))
			}
			if currentPageNo >= job.Limit {
				return true
			}
			return false
		}, nil, []chromedp.Action{
			chromedp.EmulateViewport(1777, 903, chromedp.EmulateLandscape),
		})
		if err != nil {
			t.Errorf("%+v", err)
		}

	})
}