为什么requests.get无法访问网页？

[jhc753951]

import json
import re
from urllib.parse import urlencode
import requests
from bs4 import BeautifulSoup
from requests import RequestException


def get_one_index(offset, keyword):
    data = {
        'offset': offset,
        'format': 'json',
        'keyword': keyword,
        'autoload': 'true',
        'count': '20',
        'cur_tab': 1
    }
    url = 'https://www.toutiao.com/search_content/?' + urlencode(data)
    try:
        response = requests.get(url)
        if response.status_code == 200:
            return response.text
        return None
    except RequestException:
        print('请求索引页出错')
        return None


def parse_page_index(html):
    data = json.loads(html)     # json.load用法：原本是字符串形式，转化成一个对象
    if data and 'data' in data.keys():          # keys用法：返回字典的所有键值
        for item in data.get('data'):
            yield item.get('article_url')


def get_page_detail(url):
    try:
        headers = {'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 '
                                 '(KHTML, like Gecko) Chrome/49.0.2623.221 Safari/537.36 SE 2.X MetaSr 1.0'}
        response = requests.get(url, headers=headers)
        if response.status_code == 200:
            return response.text
        return None
    except RequestException:
        print(u'请求详情页出错')
        return None


def parse_page_detail(html):  # 解析详情页
    soup = BeautifulSoup(html, 'lxml')
    title = soup.select('title')[0].get_text()
    print(title)
    images_pattern = re.compile('var gallery = (.*?);', re.S)
    result = re.search(images_pattern, html)
    if result:
        print(result.group(1))


def main():
    html = get_one_index(0, '街拍')
    for url in parse_page_index(html):
        html = get_page_detail(url)
        if html:
            parse_page_index(html)


main()

想要制作一个爬取今日头条街拍的网络爬虫，但是最后结果一直显示详情页出错，加了headers也没用，求大神指教！

[jhc753951]

引用 1 楼 oyljerry 的回复:

看看http返回的错误码是什么。估计反爬虫比较厉害

网页代码200，没错啊- -很奇怪

[toznm00]

用selenium吧，用你的代码post完人家返回的信息里面没有对应的内容的，被挡掉了

[oyljerry]

看看http返回的错误码是什么。估计反爬虫比较厉害