终于根据爬虫获取 js 动态数据（万方数据库文献下载）一文提示，我提取出了动态的url

获取下载的链接的url

def getdownurl(url):
    text=get_html(url).text
    re0=r''
    re0=r']*bhref="/link.do?([^"]+)'
    allUrl=re.findall(re0,html)
    length=len(allUrl)
    print("length=",length)
    for i in range(length):
        allUrl[i]=base+allUrl[i]
    #print(allUrl)
    return allUrl

ef getdownurl(url):
    text=get_html(url).text
    re0=r''
    text=get_html(geturl).text
    print()
    sucurl=re.findall(re1,text)
    print(sucurl)
    return sucurl[0]

def get_pdf(url,title):
    text=get_html(url)
    path="/home/dflx/下载/万方数据库/深度学习/"+title+".pdf"
    with open(path,'wb') as f:
        f.write(text.content)
    print("successf")    

def downloadAllPdf(key):
    #row=getNum(key)
    pages=search_key(key)
    allurl=[]
    num=0
    for page in  pages:
        allurl=get_url(page)
        for url in allurl:
            #得到每一篇文献的信息，写入文件
            num+=1
            try:
                re0=r'(.*?)'
                text=get_html(url).text
                title=re.findall(re0,text)[0]
                print("下载：",title)
                geturl=getdownurl(url)
                get_pdf(geturl,title)
            except:
               print("has except")
               continue
            finally:
               print("all downloads is",num)

def main():
    #自己输入关键字，检索
    key='深度学习'
    downloadAllPdf(key)

ITPub博客

python下载万方数据库文献

最新文章