ITPub博客
首页 博客 专栏 学院 社区 IT新媒体评选
发博

首页 > > > 代码保存

代码保存

作者：浪花小雨滴 时间：2017-11-25 14:55:23 0 删除 编辑 
library(rvest)
library(stringr)
library(tcltk)


url="http://www.biodiversity-science.net/CN/article/showTenYearOldVolumn.do"
read_html(url)->whole
whole %>% html_nodes("a[href]:only-child") %>% html_attr(name="href") ->wholelinks
str_c("http://www.biodiversity-science.net/CN/article/",wholelinks)->wholelinks
wholelinks[-1]->wlinks

alldata=data.frame()

for(i in 1:length(wlinks))  #1:length(wlinks)
{
 read_html(wlinks[i])->wwlinks
 wwlinks %>% html_nodes("a.J_WenZhang:only-child") %>% html_attr(name="href") %>% str_replace("\\..","http://www.biodiversity-science.net/CN") -> ylink
 
for(i in 1:length(ylink))   #选择卷期   
{
 read_html(ylink[i])->yylink
yylink %>% html_nodes("td.J_VM:nth-child(3) a:first-child") %>% html_attr(name="href") %>% str_replace("\\..","http://www.biodiversity-science.net/CN") -> vollinks
 yylink %>% html_nodes("td.J_VM_NianJuanQi b") %>% html_text()%>% `[`(2) %>% str_sub(-10) -> date

#添加进度条
pb <- tkProgressBar("进度","已完成 %", 0, 100) 

for(j in 1:length(vollinks)) #选择文章   1:length(vollinks)
{
 read_html(vollinks[j])->webpage


#网络链接link
 vollinks[j]->link
 
#年份year
date %>% str_sub(1,4) ->year

#中文题目titlecn
webpage %>% html_nodes("span.J_biaoti") %>% html_text()->titlecn
#中文作者authorcn
webpage %>% html_nodes("td.J_author_cn") %>% html_text() %>% str_trim()->authorcn
#中文机构instcn
webpage %>% html_nodes("tr td[height][style][align]") %>% html_text() %>% `[`(1)->instcn
#中文摘要abscn
webpage %>% html_nodes("td.J_zhaiyao") %>% html_text()%>% `[`(1)%>% str_replace_all("\r","") %>% str_replace_all("\n","") %>% str_replace_all("\t","") %>% str_trim() %>% str_sub(start=4)->abscn
#中文关键词keywordscn
webpage %>% html_nodes("td.J_zhaiyao a.txt_zhaiyao1") %>% html_text()->keywordscn
str_c(keywordscn,collapse=",")->keywordscn

#英文题目titleen
webpage %>% html_nodes("span.J_biaoti_en") %>% html_text()->titleen
#英文作者authoren
webpage %>% html_nodes("td.J_author_en") %>% html_text()->authoren
#英文机构insten
webpage %>% html_nodes("tr:nth-child(9) td em:only-child") %>% html_text() ->insten
#英文摘要absen
webpage %>% html_nodes("span.J_zhaiyao") %>% html_text() %>% `[`(1)%>% str_replace_all("\r","") %>% str_replace_all("\n","") %>% str_replace_all("\t","") %>% str_trim()%>% str_sub(start=10)->absen
#英文关键词keywordsen
webpage %>% html_nodes("span.J_zhaiyao a.txt_zhaiyao1") %>% html_text()->keywordsen
str_c(keywordsen,collapse=",")->keywordsen


#通讯作者corauthor*
webpage %>% html_nodes("tr:nth-child(9) td span.J_zhaiyao") %>% html_text()%>% str_replace_all("\r","")%>% str_replace_all("\n","") %>% str_replace_all("\t","") %>% str_trim() %>% str_extract(":.+?[:space:]")%>% str_sub(2,-2)->corauthor

#中文引用citecn，英文引用citeen
webpage %>% html_nodes("tr:nth-child(2) .txt_12[style]") %>% html_text() %>% str_replace_all("\r","") %>% str_replace_all("\n","") %>% str_replace_all("\t","") %>% str_trim()->cite


list(link,year,titlecn,authorcn,instcn,abscn,keywordscn,titleen,authoren,insten,absen,keywordsen,corauthor,cite,date)->a  
a[lapply(a,length)==0]=NA
a[lapply(a,length)>1]=str_c(a[lapply(a,length)>1],collapse="##")
as.data.frame(a,fix.empty.names=F)->b

rbind(alldata,b)->alldata

#显示进度条
 info<- sprintf("已完成 %d%%", round(j*100/length(vollinks)))  
   setTkProgressBar(pb, i*100/length(vollinks), sprintf("进度 (%s)", info),info) 

}

#关闭进度条
close(pb)
}

}

 #alldata=data.frame()

colnames(alldata)=c("link","year","titlecn","authorcn","instcn","abscn","keywordscn","titleen","authoren","insten","absen","keywordsen","corauthor","cite","date")
write.csv(alldata,"alldata_bs.csv")


#############################################################################


library(rvest)
library(stringr)
library(tcltk)


url="http://www.biodiversity-science.net/CN/article/showTenYearOldVolumn.do"
read_html(url)->whole
whole %>% html_nodes("a[href]:only-child") %>% html_attr(name="href") ->wholelinks
str_c("http://www.biodiversity-science.net/CN/article/",wholelinks)->wholelinks
wholelinks[-1]->wlinks

alldata=data.frame()




for(i in 22:length(wlinks))  #i in 1:length(wlinks)
{
 read_html(wlinks[i])->wwlinks
 wwlinks %>% html_nodes("a.J_WenZhang:only-child") %>% html_attr(name="href") %>% str_replace("\\..","http://www.biodiversity-science.net/CN") -> ylink
 
for(j in 1:length(ylink))   #选择卷期   j in 1:length(ylink)
{
 read_html(ylink[j])->yylink
yylink %>% html_nodes("td.J_VM:nth-child(3) a:first-child") %>% html_attr(name="href") %>% str_replace("\\..","http://www.biodiversity-science.net/CN") -> vollinks
 yylink %>% html_nodes("td.J_VM_NianJuanQi b") %>% html_text()%>% `[`(2) %>% str_sub(-10) -> date

#添加进度条
pb <- tkProgressBar("进度","已完成 %", 0, 100) 

for(k in 1:length(vollinks)) #选择文章   1:length(vollinks)
{
 read_html(vollinks[k])->webpage


#网络链接link
 vollinks[j]->link
 
#年份year
date %>% str_sub(1,4) ->year

#中文题目titlecn
webpage %>% html_nodes("span.J_biaoti") %>% html_text()->titlecn
#中文作者authorcn
webpage %>% html_nodes("td.J_author_cn") %>% html_text() %>% str_trim()->authorcn
#中文机构instcn
webpage %>% html_nodes("tr td[height][style][align]") %>% html_text() %>% `[`(1)->instcn
#中文摘要abscn
webpage %>% html_nodes("td.J_zhaiyao") %>% html_text()%>% `[`(1)%>% str_replace_all("\r","") %>% str_replace_all("\n","") %>% str_replace_all("\t","") %>% str_trim() %>% str_sub(start=4)->abscn
#中文关键词keywordscn
webpage %>% html_nodes("td.J_zhaiyao a.txt_zhaiyao1") %>% html_text()->keywordscn
str_c(keywordscn,collapse=",")->keywordscn

#英文题目titleen
webpage %>% html_nodes("span.J_biaoti_en") %>% html_text()->titleen
#英文作者authoren
webpage %>% html_nodes("td.J_author_en") %>% html_text()->authoren
#英文机构insten
webpage %>% html_nodes("tr:nth-child(9) td em:only-child") %>% html_text() ->insten
#英文摘要absen
webpage %>% html_nodes("span.J_zhaiyao") %>% html_text() %>% `[`(1)%>% str_replace_all("\r","") %>% str_replace_all("\n","") %>% str_replace_all("\t","") %>% str_trim()%>% str_sub(start=10)->absen
#英文关键词keywordsen
webpage %>% html_nodes("span.J_zhaiyao a.txt_zhaiyao1") %>% html_text()->keywordsen
str_c(keywordsen,collapse=",")->keywordsen


#通讯作者corauthor*
webpage %>% html_nodes("tr:nth-child(9) td span.J_zhaiyao") %>% html_text()%>% str_replace_all("\r","")%>% str_replace_all("\n","") %>% str_replace_all("\t","") %>% str_trim() %>% str_extract(":.+?[:space:]")%>% str_sub(2,-2)->corauthor

#中文引用citecn，英文引用citeen
webpage %>% html_nodes("tr:nth-child(2) .txt_12[style]") %>% html_text() %>% str_replace_all("\r","") %>% str_replace_all("\n","") %>% str_replace_all("\t","") %>% str_trim()->cite


list(link,year,titlecn,authorcn,instcn,abscn,keywordscn,titleen,authoren,insten,absen,keywordsen,corauthor,cite,date)->a  
a[lapply(a,length)==0]=NA
a[lapply(a,length)>1]=str_c(a[lapply(a,length)>1],collapse="##")
as.data.frame(a,fix.empty.names=F)->b

rbind(alldata,b)->alldata

#显示进度条
  info<- sprintf("%s年 %d / %d 已完成 %d%%", year,j,length(ylink),round(k*100/length(vollinks))) 
   setTkProgressBar(pb, k*100/length(vollinks), sprintf("进度 (%s)", info),info) 
Sys.sleep(2)
}

#关闭进度条
close(pb)
}

}

 #alldata=data.frame()

colnames(alldata)=c("link","year","titlecn","authorcn","instcn","abscn","keywordscn","titleen","authoren","insten","absen","keywordsen","corauthor","cite","date")
write.csv(alldata,"aes.csv")

原文作者：；如有侵权，请联系管理员删除。

0
0
上一篇： R语言数据清洗实战——高效list解析方案
下一篇： sql server 2012中统计表行数的三种方法
请登录后发表评论 登录
全部评论
浪花小雨滴

注册时间：2018-05-16

  • 博文量
    4259
  • 访问量
    39811

最新文章

支持我们 作者招募 用户协议 FAQ Contact Us

北京盛拓优讯信息技术有限公司. 版权所有  京ICP备16024965号-8 北京市公安局海淀分局网监中心备案编号：11010802021510 niuxiaotong@pcpop.com 17352615567 未成年人举报专区

广播电视节目制作经营许可证(京) 字第1234号 中国互联网协会会员

违法和不良信息举报电话:18101376593