python爬取aqistudy网站数据的代码相关问题

qq_24720839 2018-03-03 12:52:41

最近在爬取aqistudy网站的历史数据，copy了网上一段代码，代码运行没问题，但爬取的数据是空的，代码如下：



import requests as rq

import re

#import io

from bs4 import BeautifulSoup



Url='https://www.aqistudy.cn/historydata/'

headers = { 'User-Agent':

    'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/55.0.2883.87 Safari/537.36' 

    }



def getcity():

    hotcitys = rq.get(Url,headers=headers)

    soup = BeautifulSoup(hotcitys.text, 'lxml')  #这里使用lxml来解析

    citynames = []

    cityurls = []

    citys = soup.find_all('a',limit=19)[-10:]     #分片的形式来获取十个热门城市

    for city in citys:

        cityurl = city.get('href')

        cityurls.append(cityurl)

        cityname = city.get_text()

        citynames.append(cityname)

    return cityurls,citynames

cityurls,citynames = getcity()



def getwebs(url):

    data = rq.get(url,headers=headers).text

    soup = BeautifulSoup(data, 'lxml')

    ahrefs = soup.find_all('a')          #找到所有的链接

    webs = []

#    names = []

    for href in ahrefs:

        href = href.get('href')

        hrefd = re.findall('(.*?month=\d{6})',href)  #用正则来匹配每月

        if hrefd:

            urls = Url + hrefd[0]

            webs.append(urls)

    return webs



def Savepm25(webs,path):

    for web in webs:

        print('正在下载数据...'+web)

        data = rq.get(web,headers=headers).text

        soup = BeautifulSoup(data, 'lxml')

        pms = soup.find_all('td')

        aqis = []

        for pm in pms:        

            data = pm.get_text()

            aqis.append(data)

        i = 0

        while i < 12:

#            aqis.pop()

            i = i + 1

        with open(path+'.csv', 'a+') as f:                

             i = 1

             for aqi in aqis:

                if i % 11 != 0:

                    f.write(aqi + ',')       # CSV表格里以','为分隔符

                else:

                    f.write(aqi + '\r\n')    #若是行尾则换行

                i = i + 1



paths = []

for cityname in citynames:

    path = cityname + 'pm25'

    paths.append(path)

i = 0

for url in  cityurls:

    url = Url+url

    print(url)

    webs=getwebs(url)

    path = paths[i]

    Savepm25(webs,path)

    i = i + 1