python 插入数据到sqlite3，有时会插入几条数据后卡着，也不报错

[zd96998]

#! /usr/bin/env python
#coding=utf-8
#从雅虎问答抓取一些问答数据
import sys
import re
import urllib
import sqlite3
import time

cx = sqlite3.connect("Q&A.db")
cx.text_factory = str
cu = cx.cursor()

query  = 'insert into yahoo(number,link,question,qContent,answer) values (?,?,?,?,?)'
#插入之前一定要记得修改i的初值
i = 0
#query  = 'insert into qalist(link,) values (?,)'

#插入之前一定要记得修改i的初值
def getHtml(url):
	page = urllib.urlopen(url)
	html = page.read()
	return html

#插入之前一定要记得修改i的初值
def trans_r_p(alllist):
	qlist = []
	for u in alllist:
		rurl = 'https://answers.yahoo.com' + u + '&tab=recent&filter=intl&sort=new'
		qlist.append(rurl)
		purl = 'https://answers.yahoo.com' + u + '&tab=popular&filter=intl&sort=popular'
		qlist.append(purl)
	return qlist

#插入之前一定要记得修改i的初值
def getQurl(html):
	reg  = r'<h3 class="question-title"><a data-ylk="sec:(recent|popular);slk:q;ql:\d{14}[\d\w]{7}" href="([\s\S]*?)">'
	qurlre = re.compile(reg)
	qurllist  = re.findall(qurlre,html)
	qulist = []
	for item in qurllist:
		qulist.append(item[1])
	return qulist

#插入之前一定要记得修改i的初值
def getQA(html):
	qreg = r'<title>(.+)</title>'
	creg = r'<div class="qDetail mt14 none">([\s\S]*?)</div>'
	bareg = r'<div class="content" itemprop="text">\n([\s\S]*?)</div>'
	areg = r'<div class="content mb14" data-full="">\n([\s\S]*?)</div>'
	qre = re.compile(qreg)
	cre = re.compile(creg)
	bare = re.compile(bareg)
	are = re.compile(areg)
	q = re.findall(qre,html)
	c = re.findall(cre,html)
	ba = re.findall(bare,html)
	a = re.findall(are,html)
	return q,c,ba,a

#插入之前一定要记得修改i的初值
def filter(str):
	ps = str.split('<br>')
	s = ps[0]
	skipsteps = 1
	for item in ps:
		if skipsteps > 0:
			skipsteps -= 1
			continue
		s = s + ' ' + item
	return s

#插入之前一定要记得修改i的初值
def writedb(url):
	#qaUrl = url
	#page = 0
	#fq = open('questions.txt','a')
	#fc = open('contens.txt','a')
	#fa = open('answers.txt','a')
	#i = 0
	global i
	urls = getQurl(getHtml(url))
	#print urls
	for ur in urls:
		u = 'https://answers.yahoo.com' + ur
		q,c,ba,a = getQA(getHtml(u))
		#print len(a)
		time.sleep(15)
		if q == []:
			continue
		else:
			question = q[0]
		if a == []:
			continue
		if c == []:
			content = ''
		else:
			content = filter(c[0])
		if ba != []:
			bestanswer = filter(ba[0])
			bestcontent  = content + ' (Best)'
			i += 1
			cu.execute(query,(i,u,question,bestcontent,bestanswer))
			cx.commit()
		for ans in a:
			i += 1
			answer = filter(ans)
			cu.execute(query,(i,u,question,content,answer))			
			cx.commit()
			#插入之前一定要记得修改i的初值

 writedb('https://answers.yahoo.com/dir/index?sid=396545012&tab=recent&filter=intl&sort=new')
 print 1
# writedb('https://answers.yahoo.com/dir/index?sid=396545012&tab=popular&filter=intl&sort=popular')#1-180 finished
# print 2
# writedb('https://answers.yahoo.com/dir/index?sid=396545144&tab=recent&filter=intl&sort=new')#181-201
# print 3


cu.close()
cx.close()
#http://zhidao.baidu.com/browse/80?lm=4&pn=0#list

[thomashtq]

估计是下面这个代码段在读取某个URL是阻塞了，应该跟插入数据库卡住没关系。不信你可以注掉插入数据库的代码段，应该也会卡住。

def getHtml(url):
    page = urllib.urlopen(url)
    html = page.read()
    return html

[zd96998]

引用 1 楼 thomashtq 的回复:

估计是下面这个代码段在读取某个URL是阻塞了，应该跟插入数据库卡住没关系。不信你可以注掉插入数据库的代码段，应该也会卡住。

def getHtml(url):
    page = urllib.urlopen(url)
    html = page.read()
    return html

那应该怎么读取URL啊？该怎么写呢？谢谢啊

[thomashtq]

采用非阻塞模式读，或者利用超时重试机制（超过时间直接放弃，放回列表中），或者干脆超时直接放弃。