获取数据的方法依然采取了regex正则匹配的方法，请求框架采用了java，爬虫语言是groovy，本地拼接好sql语句，发送到mysql服务端，完成存储。
代码如下：
1package com.fan
2
3import com.fantest.httpclient.FanLibrary
4import com.fantest.mysql.MySqlTest
5import com.fantest.utils.Regex
6import net.sf.json.JSONObject
7
8class Company extends FanLibrary {
9 static void main(String[] args) {
10 for (def i in 1..1060) {
11 getPage(i)
12// getInfo("/eportal/ui?pageId=307900&t=toDetail&ZSBH=D311056737")
13 }
14 testOver()
15 }
16
17 static getPage(int page) {
18 def url = "http://www.***.gov.cn/eportal/ui?pageId=307900"
19 def params = new JSONObject()
20 params.put("filter_LIKE_QYMC", EMPTY)
21 params.put("filter_LIKE_YYZZZCH", EMPTY)
22 params.put("filter_LIKE_ZSBH", EMPTY)
23 params.put("filter_LIKE_XXDZ", EMPTY)
24 params.put("currentPage", page)
25 params.put("pageSize", 15)
26 params.put("OrderByField", EMPTY)
27 params.put("OrderByDesc", EMPTY)
28 def response = getHttpResponse(getHttpPost(url, params))
29 def s = response.getString("content")
30 def all = Regex.regexAll(s, "<td s.*?浏览")
31 for (int i = 1; i < all.size(); i++) {
32 def get = all.get(i)
33 def regex = Regex.getRegex(get, "href=\".*?\"").replace("amp;", EMPTY)
34 getInfo(regex)
35 sleep(3)
36 }
37 return response;
38 }
39
40 static getInfo(String url) {
41 try {
42 url = "http://www.***.gov.cn" + url;
43 def response = getHttpResponse(getHttpGet(url))
44 def content = response.getString("content")
45 def all = Regex.regexAll(content, "<td class=\"label\".*?\n.*\n.*\n.*\n.*\n.*")
46 def name = all.get(0).replaceAll("<.*?>", EMPTY).replaceAll("(\n| )", EMPTY).split("：")[1]
47 def adress = all.get(1).replaceAll("<.*?>", EMPTY).replaceAll("(\n| )", EMPTY).split("：")[1]
48 def money = all.get(2).replaceAll("<.*?>", EMPTY).replaceAll("(\n| )", EMPTY).split("：")[1]
49 def sid = all.get(3).replaceAll("<.*?>", EMPTY).replaceAll("(\n| )", EMPTY).split("：")[1]
50 def type = all.get(4).replaceAll("<.*?>", EMPTY).replaceAll("(\n| )", EMPTY).split("：")[1]
51 def man = all.get(5).replaceAll("<.*?>", EMPTY).replaceAll("(\n| )", EMPTY).split("：")[1]
52 def paper = all.get(6).replaceAll("<.*?>", EMPTY).replaceAll("(\n| )", EMPTY).split("：")[1]
53 def level = all.get(7).replaceAll("<.*?>", EMPTY).replaceAll("(\n| )", EMPTY).split("：")[1]
54 def gov = all.get(8).replaceAll("<.*?>", EMPTY).replaceAll("(\n| )", EMPTY).split("：")[1]
55 def time = all.get(9).replaceAll("<.*?>", EMPTY).replaceAll("(\n| )", EMPTY).split("：")[1]
56 def start = time.split("~")[0]
57 def end = time.split("~")[1]
58 String sql = "INSERT INTO company (name,adress,money,sid,type,man,paper,level,gov,start,end) VALUES (\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\",\"%s\");"
59 sql = String.format(sql, name, adress, money, sid, type, man, paper, level, gov, start, end)
60 output(sql)
61 MySqlTest.sendWork(sql)
62 }
63 catch (Exception e) {
64 output(e)
65 }
66 }
67}
第一页的网页结构如下：
技术类文章精选
非技术文章精选
大咖风采
原文作者：FunTester；如有侵权，请联系管理员删除。
下一篇：
单点登录性能测试方案
请登录后发表评论
登录
全部评论
