完成端口,AcceptEx,DisconnectEx, TIME_WAIT

[aqbidt]

采用完成端口+AcceptEx+DisconnectEx+线程池+内存池写了一个http server,
我用局域网的几台PC开300线程刷服务器，SOCKET由于过多的TIME_WAIT被耗尽，
然后服务器效率大大降低最后挂掉, 开50线程刷了2天没问题！

TIME_WAIT是TCP协议的一个必经状态，
网上说可以通过SO_LINGER选项让socket不经过TIME_WAIT状态，可是我试了试不行！
谁有解决TIME_WAIT问题的经验，分享一下，谢谢！

//Listener的部分代码
if(pContext->dwIoOper == IO_SOCK_SEND)
{
pChannel->OnSend(dwBytesIo);
if(dwBytesIo <= 0)
{
this->Reset(pChannel, pContext);
}
else if( pChannel->GetResponse()->IoIsComplete() )
{
this->Reset(pChannel, pContext);
/*
// 根据HTTP协议, 通知客户端断开连接
// 即设置HTTP Header为Connection:close
// 然后Receive会接收到0字节关闭消息
// 但是等待客户端关闭效率降低很多
pChannel->GetRequest()->Init();
if(!pChannel->Receive(pContext))
{
this->Reset(pChannel, pContext);
}*/
}
else
{
if(!pChannel->Send(pContext))
{
this->Reset(pChannel, pContext);
}
}
}

//Channel的部分代码
int CHttpChannel::Reset(CompletionPortContext* pContext)
{
int opt = 1;
DWORD dwErrorCode = setsockopt(m_sClientSocket, SOL_SOCKET, SO_REUSEADDR, (char*)&opt, sizeof(opt));
setsockopt(m_sClientSocket, SOL_SOCKET, SO_REUSEADDR, (char*)&opt, sizeof(opt));
//opt = 0;
//nErrorCode = setsockopt(m_sClientSocket, SOL_SOCKET, SO_DONTLINGER, (char*)&opt, sizeof(opt));
linger InternalLinger = { 1, 0 };
dwErrorCode = setsockopt(m_sClientSocket, SOL_SOCKET, SO_LINGER, (const char*)&InternalLinger, sizeof(linger));
shutdown(m_sClientSocket, SD_BOTH);
//closesocket(m_sClientSocket);
//m_sClientSocket = WSASocket(AF_INET,SOCK_STREAM,IPPROTO_TCP,0,0,WSA_FLAG_OVERLAPPED);
//this->Open(pContext); return 1;
memset(pContext, 0, sizeof(OVERLAPPED));
pContext->dwIoOper = IO_SOCK_CLOSE;
if(MSSOCK::DisconnectEx(m_sClientSocket, pContext, TF_REUSE_SOCKET, 0))
{
printf("DisconnectEx:errorCode=SUCCESS\r\n");
this->Open(pContext);
return 1;
}
else
{
dwErrorCode = WSAGetLastError();
if(dwErrorCode == WSAENOTCONN)
{
printf("DisconnectEx:errorCode=WSAENOTCONN\r\n");
this->Open(pContext);
return 1;
}
else if(dwErrorCode!=ERROR_IO_PENDING)
{
printf("DisconnectEx:errorCode=%d\r\n", WSAGetLastError());
//assert(0);
return -1;
}
}
return 0;
}

我不是新新手,不要拿别人很长的帖子粘贴过来作为回复, 请确信你对解决问题有帮助！

[aqbidt]

MSSOCK::DisconnectEx,MSSOCK::AcceptEx都是函数指针,
为提高效率保存到一个MSSOCK命名空间的静态变量中去了！

大篇幅的MSDN请也不要粘贴到这里，谢谢！
我的QQ号码53545656,
邮箱:[e-mail]

[猫已经找不回了]

up了。

[aqbidt]

经过我反复测试，发现问题应该是setsockopt : SO_LINGER在WSA_FLAG_OVERLAPPED的模式下不起作用，
谁解决过这个问题回复一下，谢谢， 一下是我根据MSDN改写的代码， 反复测试都会出现TIME_WAIT,
而使用accept+recv就不会出现TIME_WAIT了

#include <stdio.h>
#include "winsock2.h"
#include "mswsock.h"
#pragma comment(lib, "ws2_32.lib")
void main() {
//----------------------------------------
// Declare and initialize variables
WSADATA wsaData;
HANDLE hCompPort;
LPFN_ACCEPTEX lpfnAcceptEx = NULL;
GUID GuidAcceptEx = WSAID_ACCEPTEX;
typedef struct mycontext : WSAOVERLAPPED
{
DWORD dwIoBytes;
DWORD dwIoOper;
} MYCONTEXT,*PMYCONTEXT;
MYCONTEXT olOverlap;

#define IO_SOCK_OPEN 1
#define IO_SOCK_RECV 2
#define IO_SOCK_CLOSE 3

SOCKET ListenSocket, AcceptSocket;
sockaddr_in service;
char lpOutputBuf[1024];
int outBufLen = 1024;
DWORD dwBytes;

//----------------------------------------
// Initialize Winsock
int iResult = WSAStartup( MAKEWORD(2,2), &wsaData );
if( iResult != NO_ERROR )
printf("Error at WSAStartup\n");

//----------------------------------------
// Create a handle for the completion port
hCompPort = CreateIoCompletionPort( INVALID_HANDLE_VALUE, NULL, (u_long)0, 0 );

//----------------------------------------
// Create a listening socket
ListenSocket = socket( AF_INET, SOCK_STREAM, IPPROTO_TCP );
if (ListenSocket == INVALID_SOCKET) {
printf("Error at socket(): ListenSocket\n");
WSACleanup();
return;
}

//----------------------------------------
// Associate the listening socket with the completion port
//CreateIoCompletionPort((HANDLE)ListenSocket, hCompPort, (u_long)0, 0);

//----------------------------------------
// Bind the listening socket to the local IP address
// and port 27015
hostent* thisHost;
char* ip;
u_short port;
port = 5354;
thisHost = gethostbyname("");
ip = inet_ntoa (*(struct in_addr *)*thisHost->h_addr_list);

service.sin_family = AF_INET;
service.sin_addr.s_addr = ADDR_ANY;
service.sin_port = htons(port);

if ( bind( ListenSocket,(SOCKADDR*) &service, sizeof(service) ) == SOCKET_ERROR ) {
printf("bind failed\n");
closesocket(ListenSocket);
return;
}
::CreateIoCompletionPort((HANDLE)ListenSocket, hCompPort, 0, 0);

//----------------------------------------
// Start listening on the listening socket
if (listen( ListenSocket, 100 ) == SOCKET_ERROR) {
printf("error listening\n");
}
printf("Listening on address: %s:%d\n", ip, port);

//----------------------------------------
// Load the AcceptEx function into memory using WSAIoctl.
// The WSAIoctl function is an extension of the ioctlsocket()
// function that can use overlapped I/O. The function's 3rd
// through 6th parameters are input and output buffers where
// we pass the pointer to our AcceptEx function. This is used
// so that we can call the AcceptEx function directly, rather
// than refer to the Mswsock.lib library.
WSAIoctl(ListenSocket,
SIO_GET_EXTENSION_FUNCTION_POINTER,
&GuidAcceptEx,
sizeof(GuidAcceptEx),
&lpfnAcceptEx,
sizeof(lpfnAcceptEx),
&dwBytes,
NULL,
NULL);

//----------------------------------------
// Create an accepting socket
AcceptSocket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
if (AcceptSocket == INVALID_SOCKET) {
printf("Error creating accept socket.\n");
WSACleanup();
return;
}
int bNodelay = TRUE;
int bDoNtLinger = FALSE;
linger lingerStruct = {1, 0};
setsockopt(AcceptSocket, IPPROTO_TCP, TCP_NODELAY, (char*)&bNodelay, sizeof(int));
setsockopt(AcceptSocket, SOL_SOCKET, SO_DONTLINGER, (char*)&bDoNtLinger, sizeof(int));
setsockopt(AcceptSocket, SOL_SOCKET, SO_LINGER, (char*)&lingerStruct, sizeof(linger));
//----------------------------------------
// Empty our overlapped structure and accept connections.
const int BUFFER_SIZE = outBufLen - ((sizeof(sockaddr_in) + 16) * 2);
memset(&olOverlap, 0, sizeof(olOverlap));
olOverlap.dwIoOper = IO_SOCK_OPEN;
BOOL bRet = lpfnAcceptEx(ListenSocket,
AcceptSocket,
lpOutputBuf,
BUFFER_SIZE,
sizeof(sockaddr_in) + 16,
sizeof(sockaddr_in) + 16,
&dwBytes,
&olOverlap);

if(!bRet && WSAGetLastError()!=ERROR_IO_PENDING)
{
printf("AcceptEx failed.\r\n");
return;
}
//----------------------------------------
// Associate the accept socket with the completion port
//CreateIoCompletionPort((HANDLE)AcceptSocket, hCompPort, (u_long)0, 0);

//----------------------------------------
// Continue on to use send, recv, TransmitFile(), etc.,.
//...
while(1)
{
DWORD dwFlags, dwBytesIo;
WSAOVERLAPPED * pIOBlock = NULL;
PMYCONTEXT pContext;
::GetQueuedCompletionStatus(hCompPort, &dwBytesIo, &dwFlags, &pIOBlock, INFINITE);
pContext = (MYCONTEXT*) pIOBlock;
if(pContext->dwIoOper == IO_SOCK_OPEN)
{
if(dwBytesIo>0 && dwBytesIo<BUFFER_SIZE)
{
lpOutputBuf[dwBytesIo] = 0;
printf(lpOutputBuf);
}
else
{
printf("\r\nReceiv failed. \r\n");
}
closesocket(AcceptSocket);
AcceptSocket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
setsockopt(AcceptSocket, IPPROTO_TCP, TCP_NODELAY, (char*)&bNodelay, sizeof(int));
setsockopt(AcceptSocket, SOL_SOCKET, SO_DONTLINGER, (char*)&bDoNtLinger, sizeof(int));
setsockopt(AcceptSocket, SOL_SOCKET, SO_LINGER, (char*)&lingerStruct, sizeof(linger));
//最好使用DisconnectEx
//
memset(&olOverlap, 0, sizeof(olOverlap));
olOverlap.dwIoOper = IO_SOCK_OPEN;
lpfnAcceptEx(ListenSocket,
AcceptSocket,
lpOutputBuf,
BUFFER_SIZE,
sizeof(sockaddr_in) + 16,
sizeof(sockaddr_in) + 16,
&dwBytes,
&olOverlap);
}
}
}

[herman~~]

TCP/IP协议规定断开连接必然是有time_wait状态的


第一种方法是使用TcpTimedWaitDelay 注册表参数，改变该数值。对于 Windows NT 和Windows 2000，其值最低可设置为30 秒，这样在大多数情况下不会出现问题。每二种方法是使用 MaxUserPorts 注册表参数，来配置用户可访问的临时端口数（用作出站连 接的源端口）。默认情况下，当应用程序从系统请求任何套接字用于出站调用时，就会提供一个数值在1024 到 5000 之间的端口。MaxUserPorts 参数可用于设置管理员所允许的出站连接的最大端口值。例如，将该值设置为10,000（十进制），就会有约 9000 个用户端口可用于出站连接。关于这一概念的详细信息，请参见 RFC 793，也可参见MaxFreeTcbs 和 MaxHashTableSize 注册表参数。
注册表位置:


http://topic.csdn.net/u/20081127/12/7cc38692-f55f-40ca-a7ab-a4c89d2bf10d.html

[zhirom]

没做过，帮顶

[zhengstar]

ding

[aqbidt]

bbb.cpp 不会出现TIME_WAIT,
我现在打算改成WSAEventSelect的事件模式，
通过线程监控信号量代替重叠IO+IOCP;
思路分享给大家:

1个Listener线程：
SOCKET : hListener, hListenerEvent(监控FD_ACCEPT)
WaitForMultipleObjects, hListenerEvent...
1个Monitor线程:
hQueueEvent Listener线程accept后投递到队列并将hQueueEvent置为有信号
hWorkChannelSemaphores(数组,每个Sem指示对应工作线程中有多少个Channel空闲)
多个工作者线程:
hChannelSocketEvent(监控FD_READ|FD_WRITE|FD_CLOSE)
由于是WaitForMultipleObjects最对只能等待64个信号量，
所以每个工作者线程最多只能监控64个SOCKET

当然此模式没有IOCP效率高并且占用了很多信号量，有谁解决了重叠IO模式下TIME_WAIT的问题，
请告知，谢谢， QQ53545656 Email:[e-mail]

关于这个问题CSDN，BAIDU,GOOGLE都有很多人提问， 但都没有人解答，希望高手出来冒个泡！

[aqbidt]

TIME_WAIT是TCP协议的一个必经状态， 我在TOP楼已经说过了，
但服务器端SOCKET可以绕过4个终止握手过程直接抛弃对端的连接，
我在注册表中把TIME_WAIT改成1秒也顶不住局域网内的多台PC300线程刷服务器！
实在不行就只能先改成异步模式了！
这是一个简单的示例说明socket可以抛弃对端(client)的tcp连接!

#include <stdio.h>
#include "winsock2.h"
#include "mswsock.h"
#pragma comment(lib, "ws2_32.lib")
void main() {
//----------------------------------------
// Declare and initialize variables
WSADATA wsaData;
HANDLE hCompPort;
LPFN_ACCEPTEX lpfnAcceptEx = NULL;
GUID GuidAcceptEx = WSAID_ACCEPTEX;
typedef struct mycontext : WSAOVERLAPPED
{
DWORD dwIoBytes;
DWORD dwIoOper;
} MYCONTEXT,*PMYCONTEXT;
MYCONTEXT olOverlap;

#define IO_SOCK_OPEN 1
#define IO_SOCK_RECV 2
#define IO_SOCK_CLOSE 3
int bNodelay = TRUE;
unsigned long zero = 0;
linger lingerStruct = {1, 0};

SOCKET ListenSocket, AcceptSocket;
sockaddr_in service;
char lpOutputBuf[1024];
int outBufLen = 1024;
DWORD dwBytes;

//----------------------------------------
// Initialize Winsock
int iResult = WSAStartup( MAKEWORD(2,2), &wsaData );
if( iResult != NO_ERROR )
printf("Error at WSAStartup\n");

//----------------------------------------
// Create a handle for the completion port
hCompPort = CreateIoCompletionPort( INVALID_HANDLE_VALUE, NULL, (u_long)0, 0 );

//----------------------------------------
// Create a listening socket
ListenSocket = socket( AF_INET, SOCK_STREAM, IPPROTO_TCP );
if (ListenSocket == INVALID_SOCKET) {
printf("Error at socket(): ListenSocket\n");
WSACleanup();
return;
}

//----------------------------------------
// Associate the listening socket with the completion port
//CreateIoCompletionPort((HANDLE)ListenSocket, hCompPort, (u_long)0, 0);

//----------------------------------------
// Bind the listening socket to the local IP address
// and port 27015
hostent* thisHost;
char* ip;
u_short port;
port = 5354;
thisHost = gethostbyname("");
ip = inet_ntoa (*(struct in_addr *)*thisHost->h_addr_list);

service.sin_family = AF_INET;
service.sin_addr.s_addr = ADDR_ANY;
service.sin_port = htons(port);
if ( bind( ListenSocket,(SOCKADDR*) &service, sizeof(service) ) == SOCKET_ERROR ) {
printf("bind failed\n");
closesocket(ListenSocket);
return;
}
::CreateIoCompletionPort((HANDLE)ListenSocket, hCompPort, 0, 0);
setsockopt(ListenSocket, IPPROTO_TCP, TCP_NODELAY, (char*)&bNodelay, sizeof(int));
setsockopt(ListenSocket, SOL_SOCKET, SO_LINGER, (char*)&lingerStruct, sizeof(linger));

//----------------------------------------
// Start listening on the listening socket
if (listen( ListenSocket, 100 ) == SOCKET_ERROR) {
printf("error listening\n");
}
printf("Listening on address: %s:%d\n", ip, port);

//----------------------------------------
// Load the AcceptEx function into memory using WSAIoctl.
// The WSAIoctl function is an extension of the ioctlsocket()
// function that can use overlapped I/O. The function's 3rd
// through 6th parameters are input and output buffers where
// we pass the pointer to our AcceptEx function. This is used
// so that we can call the AcceptEx function directly, rather
// than refer to the Mswsock.lib library.
WSAIoctl(ListenSocket,
SIO_GET_EXTENSION_FUNCTION_POINTER,
&GuidAcceptEx,
sizeof(GuidAcceptEx),
&lpfnAcceptEx,
sizeof(lpfnAcceptEx),
&dwBytes,
NULL,
NULL);

//----------------------------------------
// Create an accepting socket
AcceptSocket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
if (AcceptSocket == INVALID_SOCKET) {
printf("Error creating accept socket.\n");
WSACleanup();
return;
}
//----------------------------------------
// Empty our overlapped structure and accept connections.
const int BUFFER_SIZE = outBufLen - ((sizeof(sockaddr_in) + 16) * 2);
memset(&olOverlap, 0, sizeof(olOverlap));
olOverlap.dwIoOper = IO_SOCK_OPEN;
BOOL bRet = lpfnAcceptEx(ListenSocket,
AcceptSocket,
lpOutputBuf,
BUFFER_SIZE,
sizeof(sockaddr_in) + 16,
sizeof(sockaddr_in) + 16,
&dwBytes,
&olOverlap);

if(!bRet && WSAGetLastError()!=ERROR_IO_PENDING)
{
printf("AcceptEx failed.\r\n");
return;
}
// setsockopt(AcceptSocket, IPPROTO_TCP, TCP_NODELAY, (char*)&bNodelay, sizeof(int));
// setsockopt(AcceptSocket, SOL_SOCKET, SO_LINGER, (char*)&lingerStruct, sizeof(LINGER));

//----------------------------------------
// Associate the accept socket with the completion port
//CreateIoCompletionPort((HANDLE)AcceptSocket, hCompPort, (u_long)0, 0);

//----------------------------------------
// Continue on to use send, recv, TransmitFile(), etc.,.
//...
while(1)
{
DWORD dwFlags, dwBytesIo;
WSAOVERLAPPED * pIOBlock = NULL;
PMYCONTEXT pContext;
::GetQueuedCompletionStatus(hCompPort, &dwBytesIo, &dwFlags, &pIOBlock, INFINITE);
pContext = (MYCONTEXT*) pIOBlock;
if(pContext->dwIoOper == IO_SOCK_OPEN)
{
if(dwBytesIo>0 && dwBytesIo<BUFFER_SIZE)
{
lpOutputBuf[dwBytesIo] = 0;
printf(lpOutputBuf);
}
else
{
printf("\r\nReceiv failed. \r\n");
}
//ioctlsocket(AcceptSocket, FIONBIO, &zero);
setsockopt(AcceptSocket, IPPROTO_TCP, TCP_NODELAY, (char*)&bNodelay, sizeof(int));
//setsockopt(AcceptSocket, SOL_SOCKET, SO_DONTLINGER, (char*)&bDoNtLinger, sizeof(int));
setsockopt(AcceptSocket, SOL_SOCKET, SO_LINGER, (char*)&lingerStruct, sizeof(LINGER));

closesocket(AcceptSocket);
AcceptSocket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);

//最好使用DisconnectEx
//
memset(&olOverlap, 0, sizeof(olOverlap));
olOverlap.dwIoOper = IO_SOCK_OPEN;
lpfnAcceptEx(ListenSocket,
AcceptSocket,
lpOutputBuf,
BUFFER_SIZE,
sizeof(sockaddr_in) + 16,
sizeof(sockaddr_in) + 16,
&dwBytes,
&olOverlap);
}
}
}

[aqbidt]

晕倒，代码发错了， 这几天搞的我晕头转向了！

////////////////////////////////////////////
//bbb.cpp

#define _WIN32_WINNT 0x0501
#include <windows.h>
#include <winsock2.h>
#include <stdio.h>
#pragma comment(lib, "ws2_32.lib")
void main(void)
{
WSADATA wsaData;
if(NO_ERROR != WSAStartup( MAKEWORD(2,2), &wsaData ) )
{
printf("Error at CTcpListener::CreateInstance() : WSAStartup().");return;
}
//
SOCKET hListener = WSASocket(AF_INET, SOCK_STREAM, 0, NULL, 0, WSA_FLAG_OVERLAPPED);
//m_hListener = socket(AF_INET, SOCK_STREAM, 0);
if (hListener == INVALID_SOCKET)
{
WSACleanup();
printf("Error at socket(): create listen socket.");return;
}
//
sockaddr_in service;
service.sin_family = AF_INET;
service.sin_addr.s_addr = ADDR_ANY;
service.sin_port = htons(5354);
if ( bind(hListener,(SOCKADDR*) &service, sizeof(service) ) == SOCKET_ERROR )
{
closesocket(hListener);
WSACleanup();
printf("Error at CTcpListener::CreateInstance() : bind(hListener,...).");return;
}

if (listen(hListener, 64) != 0)
{
closesocket(hListener);
WSACleanup();
printf("Error at CTcpListener::CreateInstance() : listen(hListener).");return;
}

unsigned long opt = FALSE;
BOOL bTcpNodelay = TRUE;
linger lingerStruct = {1, 0};
//setsockopt(hListener, SOL_SOCKET, SO_DONTLINGER, (char*)&opt, sizeof(opt));
setsockopt(hListener, IPPROTO_TCP, TCP_NODELAY, (char*)&bTcpNodelay, sizeof(opt));
setsockopt(hListener, SOL_SOCKET, SO_LINGER, (char*)&lingerStruct, sizeof(LINGER));

WSAEVENT acceptEvent = WSACreateEvent();
WSAEVENT recvEvent = WSACreateEvent();
WSAEventSelect(hListener, acceptEvent, FD_ACCEPT|FD_READ|FD_WRITE|FD_CLOSE);

SOCKET sClient = INVALID_SOCKET;
DWORD dwEventFlags = 0;
WSANETWORKEVENTS wsaNE = {0, 0};
while(1)
{
WaitForSingleObject(acceptEvent, INFINITE);
WSAEnumNetworkEvents(hListener, acceptEvent, &wsaNE);
if( (wsaNE.lNetworkEvents&FD_ACCEPT) == FD_ACCEPT)
{
SOCKET s = accept(hListener, NULL, NULL);
WSAEventSelect(s, recvEvent, FD_READ|FD_CLOSE);
WaitForSingleObject(recvEvent, INFINITE);
wsaNE.lNetworkEvents = 0;
WSAEnumNetworkEvents(s, recvEvent, &wsaNE);
if(wsaNE.lNetworkEvents&FD_READ == FD_READ)
{
char szBuffer[1024];
int len = recv(s, szBuffer, 1024, 0);
if(len>0 && len<1024)
{
szBuffer[len] = 0;
printf(szBuffer);
}
else
{
printf("recv failed.\r\n\r\n");
}
//shutdown(s, SD_BOTH);
//ioctlsocket(s, FIONBIO, &opt);
setsockopt(s, IPPROTO_TCP, TCP_NODELAY, (char*)&bTcpNodelay, sizeof(opt));
//setsockopt(s, SOL_SOCKET, SO_DONTLINGER, (char*)&opt, sizeof(opt));
setsockopt(s, SOL_SOCKET, SO_LINGER, (char*)&lingerStruct, sizeof(LINGER));
closesocket(s);
}

}
}
}

[aqbidt]

改成异步模式至少不会出现那么多SOCKET句柄占用资源,
否则把服务器SOCKET句柄完全占用是一个非常不稳定的状态！
我的观念里永远不希望服务器满负荷，
我们都是程序员都知道程序难免有BUG，
而满负荷更容易使那些BUG出现导致无人值守的服务器崩溃！

[hhyttppd]

up

[萧霖]

最近使用IOCP也遇到过相似的问题，就是在多次调用了disconnect函数之后，可能出现完成端口要很久（大概两分钟）才返回，不知道和楼主的问题是不是一样的？