向一个主机重复发送http请求

[jiangfeng999]

我写了一个http请求的类，基本思想是第一次申请的时候，向主机发送一个连接请求，成功连接后直接利用这个连接抓取网页。
代码如下：
http请求类：

CHttpAction::CHttpAction()
{
	m_hSocket = INVALID_SOCKET;
	return;
}

CHttpAction::CHttpAction(char *pHostName)
{
	m_hSocket = INVALID_SOCKET;
	memset(m_sHostName,0,sizeof(m_sHostName));
	strcpy(m_sHostName,pHostName);
	CreateSocket(pHostName,80);
	return;
}
CHttpAction::~CHttpAction()
{
}

/*************************************
* 函数名称:CreateSocket
* 函数功能:
* 输入参数:pHostName:主机名;iPort端口号
* 输出参数:
* 返 回 值:
*************************************/
int CHttpAction::CreateSocket(char *pHostName,u_int16_t iPort=80)
{
	int nRet;

	memset(&m_sockAddr,0,sizeof(m_sockAddr));
	struct hostent *newHost;
	newHost = gethostbyname(pHostName);
	if(NULL==newHost)
	{
		return FUNCTION_ERROR;
	}
	memset(m_sIpAddress,0,sizeof(m_sIpAddress));
	strcpy(m_sIpAddress,inet_ntoa(*(in_addr *)newHost->h_addr_list[0]));
	m_iIpAddress = inet_addr(m_sIpAddress);

	m_sockAddr.sin_family = AF_INET;
	m_sockAddr.sin_port = htons(iPort);
	m_sockAddr.sin_addr.s_addr = m_iIpAddress;

	m_hSocket = socket(AF_INET,SOCK_STREAM,0);
	nRet = connect(m_hSocket,(struct sockaddr *)&m_sockAddr,sizeof(m_sockAddr));
	if(nRet==SOCKET_ERROR)
	{
		fprintf(stderr,"connect error:%d\n",WSAGetLastError());
		return FUNCTION_ERROR;
	}
	return FUNCTION_NORMAL;
}

/*************************************
* 函数名称:FetchHttpPage
* 函数功能:下载网页到本地文件
* 输入参数:pHttpUrl要下载的网页URL地址
* 输出参数:
* 返 回 值:
*************************************/
int CHttpAction::FetchHttpPage(char *pHttpUrl)
{
	int nRet;
	DWORD dwContentLength = 0;
	char *pSt,*pEnd;
	char sHttpReqest[1024];
	char sHostName[MAX_HOST_NAME_LEN];
	//获得主机名
	memset(sHostName,0,sizeof(sHostName));
	pSt = strstr(pHttpUrl,"://");
	if(pSt==NULL)
	{
		pEnd = strchr(pHttpUrl,'/');
		if(NULL==pEnd) strcpy(sHostName,pHttpUrl);
		else strncpy(sHostName,pHttpUrl,pEnd-pHttpUrl);
	}
	else
	{
		pEnd = strchr(pSt+strlen("://"),'/');
		if(NULL==pEnd) strcpy(sHostName,pSt+strlen("://"));
		else strncpy(sHostName,pSt+strlen("://"),pEnd-(pSt+strlen("://")));
	}

	memset(sHttpReqest,0,sizeof(sHttpReqest));
	sprintf(sHttpReqest,"GET %s HTTP/1.1\r\nAccept:*/*\r\nAccept-Language:zh-cn\r\n",pHttpUrl,HTTP_VERSION);
	sprintf(sHttpReqest,"%sUser-Agent:Mozilla/4.0 (compatible;MSIE 7.0;Windows XP; .NET CLR 1.1.4322; .NET CLR 2.0.50727)\r\n",sHttpReqest);
	sprintf(sHttpReqest,"%sHost:%s\r\n",sHttpReqest,sHostName);
	sprintf(sHttpReqest,"%sConnection: Close\r\n\r\n",sHttpReqest);

	nRet=send(m_hSocket,sHttpReqest,strlen(sHttpReqest),0);
	if(nRet==SOCKET_ERROR)
	{
		fprintf(stderr,"send  HttpReqest error:%d\n",WSAGetLastError());
		return FUNCTION_ERROR;
	}
	
	//等待服务器响应,回传数据
	struct timeval timeout;
	timeout.tv_sec = 10;
	fd_set fdReads;

	char strHttpRespones[1024];
	dwContentLength = 0;
	while(1)
	{
		FD_ZERO(&fdReads);
		FD_SET(m_hSocket,&fdReads);
		select(m_hSocket+1,&fdReads,NULL,NULL,&timeout);
		if(FD_ISSET(m_hSocket,&fdReads))
		{
			sprintf(strHttpRespones,"recv HttpRespones error:%d\n",WSAGetLastError());
			nRet=recv(m_hSocket,strHttpRespones,sizeof(strHttpRespones)-1,0);
			if(nRet==SOCKET_ERROR)
			{
				fprintf(stderr,"recv HttpRespones error:%d\n",WSAGetLastError());
				return FUNCTION_ERROR;
			}
			dwContentLength += nRet;
			strHttpRespones[nRet] = 0;
			
			g_sHtmlBuffer[0] += strHttpRespones;
			while (1)    
			{
				memset(strHttpRespones, 0, sizeof(strHttpRespones));
				int iRet = recv(m_hSocket,strHttpRespones,sizeof(strHttpRespones)-1,0);
				strHttpRespones[nRet] = 0;
				if (iRet == WSAECONNRESET || iRet == SOCKET_ERROR || iRet <= 0)
                   break;
				g_sHtmlBuffer[0] += strHttpRespones;
				dwContentLength += nRet;
			}
			break;
		}
	}
	return FUNCTION_NORMAL;
}

调用方法如下：
CHttpAction newHost(主机域名);
然后重复调用newHost.FetchHttpPage(url)进行网页抓取，这些url地址都是属于同一台主机的，第一个http的请求可以成功接收。而后面所有的http请求的结果都是空的。

[ok1234567]

相应第一次请求后，服务器关闭了连接
尝试
sprintf(sHttpReqest,"%sConnection: Keep-Alive\r\n\r\n",sHttpReqest);

[jiangfeng999]

嗯，用这个方法的话，程序就卡死在while循环里面
我有试过在send之前再调用一次connect
nRet = connect(m_hSocket,(struct sockaddr *)&m_sockAddr,sizeof(m_sockAddr));
但是程序都会报连接已存在这样的错误

[hhwei1985]

xuexi

[skyxie]

[Quote=引用 2 楼 MSDA 的回复:]
嗯，用这个方法的话，程序就卡死在while循环里面
我有试过在send之前再调用一次connect
nRet = connect(m_hSocket,(struct sockaddr *)&m_sockAddr,sizeof(m_sockAddr));
但是程序都会报连接已存在这样的错误
[/Quote]

不用Keep-Alive的话，就每次取url的时候才建socket，取完后关掉