rabida/iframe_test.go at master · JohnnyTing/rabida · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
package service

import (
	"context"
	"fmt"
	"github.com/Jeffail/gabs/v2"
	"github.com/JohnnyTing/rabida/config"
	"testing"
)

func TestRabidaCrawlIframe(t *testing.T) {
	conf := config.LoadFromEnv()
	fmt.Printf("%+v\n", conf)
	rabi := NewRabida(conf)
	job := Job{
		Link: "http://www.jinan.gov.cn/col/col27544/index.html",
		CssSelector: CssSelector{
			Scope:  "#searchform+table tr",
			Iframe: true,
			IframeSelector: &CssSelector{
				Css: "#zpinfo003",
			},
			Attrs: map[string]CssSelector{
				"title": {
					Css:  "a",
					Attr: "title",
				},
				"date": {
					Css: "td:last-child>span",
				},
				"link": {
					Css:  "a",
					Attr: "href",
				},
			},
		},
		Paginator: CssSelector{
			Css: "a.pgBtn:nth-child(3):not(.disabledTd)",
		},
		Limit: 3,
	}
	err := rabi.Crawl(context.Background(), job, func(ret []interface{}, nextPageUrl string, currentPageNo int) bool {
		for _, item := range ret {
			fmt.Println(gabs.Wrap(item).StringIndent("", "  "))
		}
		if currentPageNo >= job.Limit {
			return true
		}
		return false
	}, nil, nil)
	if err != nil {
		t.Error(fmt.Sprintf("%+v", err))
	}
}

func TestRabidaCrawlIframe1(t *testing.T) {
	conf := config.LoadFromEnv()
	fmt.Printf("%+v\n", conf)
	rabi := NewRabida(conf)
	job := Job{
		Link: "http://www.suzhou.gov.cn/szsrmzf/zfxxgkzl/xxgkml.shtml?para=zcwj",
		CssSelector: CssSelector{
			Scope:  `body > form > table > tbody > tr`,
			Iframe: true,
			IframeSelector: &CssSelector{
				Css: "#xxgk_item",
			},
			Attrs: map[string]CssSelector{
				"content": {
					Css:  "a",
					Attr: "title",
				},
				"date": {
					Css: "td:last-child",
				},
				"link": {
					Css:  "a",
					Attr: "href",
				},
			},
		},
		Paginator: CssSelector{
			Css: "//span[@class='upordown']/a[text()='下一页']",
		},
		Limit: 3,
	}
	err := rabi.Crawl(context.Background(), job, func(ret []interface{}, nextPageUrl string, currentPageNo int) bool {
		for _, item := range ret {
			fmt.Println(gabs.Wrap(item).StringIndent("", "  "))
		}
		if currentPageNo >= job.Limit {
			return true
		}
		return false
	}, nil, nil)
	if err != nil {
		t.Error(fmt.Sprintf("%+v", err))
	}
}