爬了个天气预报，基本（故障）代码是：
import urllib
import urllib.request as reqlib
def GetProvinceUrl():
try:
print('Getting province main page...')
req = reqlib.urlopen('http://www.weather.com.cn/textFC/hb.shtml')
resp = req.read().decode('utf-8') # <--sth wrong
req.close()
except:
raise(simpleException('Fail to get province info.'))
# ......
urllib库read回来的都是byte串，下面处理都是作为string的；先假设网页就是utf8编码吧，直接就上了decode('utf-8')。
现在的问题是，read()回来的数据，print一看很正常，没有一点问题。
但是decode一下，数据就只剩下末尾的一点点了。
尝试了decode('utf-8')、decode()、str(xxx, encoding='utf-8')，都是一样的结果。
read()原数据是：
b'<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">\n<html xmlns="http://www.w3.org/1999/xhtml">\n
............
<script type="text/javascript" src="http://c.wrating.com/a1.js">\n</script>\n<script type="text/javascript">\nvar vjAcc="860010-2099040100";\r\nvar wrUrl="http://c.wrating.com/";\r\nvjTrack("");\n</script>\n<noscript>\n<img src="http://c.wrating.com/a.gif?a=&c=860010-2099040100" width="1" height="1"/>\n</noscript>\n</body>\n</html>
deocde一下就……
.js">
</script>
<script type="text/javascript">
var vjAcc="860010-2099040100";
var wrUrl="http://c.wrating.com/";
vjTrack("");
</script>
<noscript>
<img src="http://c.wrating.com/a.gif?a=&c=860010-2099040100" width="1" height="1"/>
</noscript>
</body>
</html>
一开始我猜可能print函数出bug了，但是想想后续的程序也运行的莫名其妙，没结果就算了，print一个列表或者字典，什么都不输出，连"[]"或者"{}"都没……我不由地开始怀疑人生
最后附上可能有用（应该没用）的完整代码：
import urllib
import urllib.request as reqlib
import re
class simpleException(Exception):
def __init__(self,msg):
Exception(self)
self.msg = msg
def GetProvinceUrl():
try:
print('Getting province main page...')
req = reqlib.urlopen('http://www.weather.com.cn/textFC/hb.shtml')
# TODO: 解析网页编码
resp = req.read().decode('utf-8')
req.close()
except:
raise(simpleException('Fail to get province info.'))
try:
print('Parsing province HTML...')
prvDiv = re.search(r'''(?is)<div.*?class=["']lqcontentBoxheader["'].*?>(.*?)</div>''',resp).group(1)
prvLi = re.findall(r'(?is)<li.*?>(.*?)</li>',prvDiv)
# print(prvDiv, prvLi) # and NOTHING printed
prvDict = {}
for li in prvLi:
# 获得超链接和名称
info = re.search(r'''<a.*?href=['"](.*?)['"].*?>(.*?)</a>''',li)
prvDict.setdefault(info.group(2),info.group(1))
print(info.group(2),info.group(1))
except:
raise(simpleException('Fail to parse HTML about provinces.'))
# 补全短连接
for prv in prvDict:
if prvDict[prv][0] == r'/':
prvDict[prv] = r'http://www.weather.com.cn'+prvDict[prv]
elif re.match(r'https?://*',prvDict[prv]) == None:
prvDict[prv] = r'http://'+prvDict[prv];
return prvDict
def GetCityCodes():
pass
try:
print(GetProvinceUrl())
except simpleException as e:
print(e.msg)
