在UDP中使用IOCP，内存以4K的速度增长

[zwz1984]

大家好。我在一个项目中，用到了IOCP模型，但是采用的是UDP协议。

设计思路：
客户端：将请求投入队列中，开一后台线程，阻塞的发送。
服务端：利用IOCP模型，收到数据后，存入队列中，马上投递新的请求；后台开一个线程，对队列中的请求进行处理。

出现问题：
在通信过程中，服务器的内存会以4K的速度增长。

请各位大哥，帮小弟看下问题出在哪？非常感谢！

源码如下：

class CInitSock // 初始化Winsock库
{
public:
CInitSock(BYTE minorVer = 2, BYTE majorVer = 2)
{
WSADATA wsaData;
WORD sockVersion = MAKEWORD(minorVer, majorVer);
if(::WSAStartup(sockVersion, &wsaData) != 0)
{
exit(0);
}
}
~CInitSock()
{
::WSACleanup();
}
} initSocket;

CIUSocket::CIUSocket(void)
{
m_sUdp = INVALID_SOCKET;
m_hCompletion = NULL;
m_hHandlePacket = NULL;
m_bExitFlag = FALSE;
m_pPerIO = NULL;

// 获取文件路径
TCHAR szRoot[MAX_PATH];
GetModuleFileName(NULL, szRoot, MAX_PATH);
PathRemoveFileSpec(szRoot);

// 设置日志路径
CStringA strA(szRoot);
strA = strA + "\\ServerComsLog.txt";
m_ComsLog.SetLogPath(strA);

// 打开日志
m_ComsLog.OpenLog();

// 开始通信
m_ComsLog.WriteLog("开始通信");
}

CIUSocket::~CIUSocket(void)
{
// 停止通信
m_ComsLog.WriteLog("停止通信");

// 关闭文件
m_ComsLog.CloseLog();
}

DWORD WINAPI CIUSocket::ServerThread(LPVOID lpParam)
{
CIUSocket *pThis = static_cast<CIUSocket*>(lpParam);

DWORD dwTrans = 0, dwKey = 0;
PPER_IO_DATA pPerIO;
while(TRUE)
{
if (pThis->m_bExitFlag)
{
break;
}

// 在关联到此完成端口的所有套节字上等待I/O完成
BOOL bOK = ::GetQueuedCompletionStatus(
pThis->m_hCompletion,
&dwTrans,
&dwKey,
(LPOVERLAPPED*)&pPerIO,
WSA_INFINITE);

// 在此套节字上有错误发生
if(!bOK)
{
// 关闭时，此处会出现异常，故跳过
if (pThis->m_bExitFlag)
{
break;
}

pThis->m_ComsLog.WriteLog("GetQueuedCompletionStatus 执行错误");

::GlobalFree(pPerIO);
continue;
}

if(dwTrans == 0 && (pPerIO->nOperationType == OP_READ || pPerIO->nOperationType == OP_WRITE))
{
pThis->m_ComsLog.WriteLog("读写长度为零");

::GlobalFree(pPerIO);
continue;
}

switch(pPerIO->nOperationType)
{
case OP_READ:
{
pThis->m_csThread.Lock();

int nOffset = 0;

// 投递到一个等待处理的队例中
REQ_DATA *pReqListData = new REQ_DATA;

// 请求地址
memcpy((char*)&pReqListData->addr, (char*)&pPerIO->addr, sizeof(sockaddr_in));

// 请求长度
pReqListData->addrLen = pPerIO->addrLen;

if (dwTrans >= MIN_PACKET_SIZE)
{
// 请求索引
pReqListData->ulReqIndex = *(unsigned long*)pPerIO->buf;

// 请求时间
pReqListData->llReqTime = *(unsigned short*)(pPerIO->buf + 4);

// 请求次数
pReqListData->usReqTimes = *(unsigned short*)(pPerIO->buf + 12);

// 请求缓冲
memcpy(pReqListData->szReqBuf, pPerIO->buf + 14, dwTrans - 14);

// 请求长度
pReqListData->usReqLen = (unsigned short)dwTrans - 14;

// 添加请求
pThis->m_cReqList.AppendReqData(pReqListData);
}

pThis->m_csThread.Unlock();

pThis->PostRecv( pPerIO ); //重新投递这个PER_IO_DATA
}
break;

case OP_WRITE:
{
}

break;
}

if (pThis->m_bExitFlag)
{
break;
}

Sleep(10);
}

return 0;
}

// 处理线程
DWORD CIUSocket::HandleThread( LPVOID lpParam )
{
CIUSocket *pThis = static_cast<CIUSocket*>(lpParam);

while (1)
{
if (pThis->m_bExitFlag)
{
break;
}

BOOL bIsValid = FALSE;
PREQ_DATA pReqListData= pThis->m_cReqList.RemoveReqData(bIsValid);

if (pReqListData != NULL)
{
pThis->OnProcessPacket(pReqListData);

delete pReqListData;
pReqListData = NULL;
}

if (pThis->m_bExitFlag)
{
break;
}

Sleep(100);
}

return 0;
}

// 启动服务
BOOL CIUSocket::StartServer(USHORT usPort)
{
BOOL bCreateSocket = FALSE;

try
{
int errorCode = 1;

// 创建完成端口对象，创建工作线程处理完成端口对象中事件
m_hCompletion = ::CreateIoCompletionPort(INVALID_HANDLE_VALUE, 0, 0, 0);
if (m_hCompletion == NULL)
{
throw CSrvException(_T("CreateIoCompletionPort1执行失败"), GetLastError());
}

// 开启服务线程
for (UINT i(0); i<THREAD_COUNT; i++ )
{
m_hWaitEvents[i] = ::CreateThread(NULL, 0, ServerThread, this, 0, 0);
}

// 开启处理线程
m_hHandlePacket = CreateThread(NULL, 0, HandleThread, this, 0, 0);

// 创建监听套节字，绑定到本地地址，开始监听
m_sUdp = ::WSASocket(AF_INET, SOCK_DGRAM, IPPROTO_UDP, NULL, 0, WSA_FLAG_OVERLAPPED );
if( m_sUdp == INVALID_SOCKET )
{
throw CSrvException(_T("WSASocket执行失败"), WSAGetLastError());
}

// 设置标志
bCreateSocket = TRUE;

//把receiveform及sendto设置为异步非阻塞模式,因为用WSAStartup初始化默认是同步
ULONG ul = 1;
errorCode = ioctlsocket(m_sUdp, FIONBIO, &ul);
if(SOCKET_ERROR == errorCode)
{
throw CSrvException(_T("ioctlsocket FIONBIO执行失败"), WSAGetLastError());
}

//设置为地址重用，优点在于服务器关闭后可以立即启用
int nOpt = 1;
errorCode = setsockopt(m_sUdp, SOL_SOCKET, SO_REUSEADDR, (char*)&nOpt, sizeof(nOpt));
if(SOCKET_ERROR == errorCode)
{
throw CSrvException(_T("setsockopt SO_REUSEADDR执行失败"), WSAGetLastError());
}

//关闭系统缓存，使用自己的缓存以防止数据的复制操作
INT nBufferLen = 0;
errorCode = setsockopt(m_sUdp, SOL_SOCKET, SO_SNDBUF, (char*)&nBufferLen, sizeof(nBufferLen));
if(SOCKET_ERROR == errorCode)
{
throw CSrvException(_T("setsockopt SO_SNDBUF执行失败"), WSAGetLastError());
}

nBufferLen = 60 * 1024;
errorCode = setsockopt(m_sUdp, SOL_SOCKET, SO_RCVBUF, (CHAR*)&nBufferLen, sizeof(nBufferLen));
if(SOCKET_ERROR == errorCode)
{
throw CSrvException(_T("setsockopt SO_RCVBUF执行失败"), WSAGetLastError());
}

unsigned long dwBytesReturned = 0;
int bNewBehavior = FALSE;

//下面的函数用于解决远端突然关闭会导致WSARecvFrom返回10054错误导致服务器完成队列中没有reeceive操作而设置
errorCode = WSAIoctl(m_sUdp, SIO_UDP_CONNRESET,&bNewBehavior, sizeof(bNewBehavior),NULL, 0, &dwBytesReturned,NULL, NULL);
if (SOCKET_ERROR == errorCode)
{
throw CSrvException(_T("WSAIoctl SIO_UDP_CONNRESET执行失败"), WSAGetLastError());
}

// 绑定端口
sockaddr_in si;
si.sin_family = AF_INET;
si.sin_port = ::ntohs(usPort);
si.sin_addr.S_un.S_addr = INADDR_ANY;

if (SOCKET_ERROR == ::bind(m_sUdp, (sockaddr*)&si, sizeof(si)))
{
throw CSrvException(_T("bind执行失败"), WSAGetLastError());
}

//把监听线程和完成端口邦定
if (NULL == ::CreateIoCompletionPort((HANDLE)m_sUdp, m_hCompletion, 0, 0))
{
throw CSrvException(_T("CreateIoCompletionPort2执行失败"), GetLastError());
}

//投递UDP_WAIT_COUNT个接收
for ( UINT i(0); i<UDP_WAIT_COUNT; i++ )
{
m_pPerIO = (PPER_IO_DATA)::GlobalAlloc(GPTR, sizeof(PER_IO_DATA));
PostRecv( m_pPerIO );
}

return TRUE;
}
catch(CSrvException& e)
{
CString strErrorTip;
strErrorTip.Format(_T("%s\t错误码为：%d"), e.GetExpDesc(), e.GetExpCode());
::MessageBox(NULL, strErrorTip, _T("启动服务异常"), MB_OK);
}
catch(...)
{
::MessageBox(NULL, _T("未知异常"), _T("启动服务异常"), MB_OK);
}

return FALSE;
}

// 停止服务
void CIUSocket::StopServer()
{
// 设置退出标志
m_bExitFlag = TRUE;

::CloseHandle(m_hCompletion);
::closesocket(m_sUdp);
m_sUdp = INVALID_SOCKET;
::PostQueuedCompletionStatus(m_hCompletion, -1, 0, NULL);

// 等待I/O处理线程退出
::WaitForMultipleObjects(THREAD_COUNT, m_hWaitEvents, TRUE, INFINITE);

// 释放内存
if (m_pPerIO != NULL)
::GlobalFree(m_pPerIO);

for(UINT i(0); i<THREAD_COUNT; i++)
{
::CloseHandle(m_hWaitEvents[i]);
m_hWaitEvents[i] = NULL;
}

if (m_hHandlePacket != NULL)
{
CloseHandle(m_hHandlePacket);
m_hHandlePacket = NULL;
}

}

// 投递接收请求
BOOL CIUSocket::PostRecv( PPER_IO_DATA pPerIO )
{
try
{
pPerIO->nOperationType = OP_READ;
pPerIO->len = MAX_BUF_SIZE;
memset( pPerIO->buf, 0, MAX_BUF_SIZE );

pPerIO->addrLen = sizeof(sockaddr);
memset( &pPerIO->addr, 0, sizeof(sockaddr) );

WSABUF buf = { MAX_BUF_SIZE, pPerIO->buf };
DWORD dwRecv = 0;
DWORD dwFlags = 0;

int nRecv = ::WSARecvFrom(
m_sUdp,
&buf,
1,
&dwRecv,
&dwFlags,
(sockaddr*)(&pPerIO->addr),
&pPerIO->addrLen,
&(pPerIO->ol),
NULL);

if( nRecv != NO_ERROR )
{
int nErr = ::WSAGetLastError();
if( nErr != WSA_IO_PENDING)
{
throw CSrvException(_T("WSARecvFrom 执行失败"), nErr);
}
}

return TRUE;
}
catch(CSrvException& e)
{
m_ComsLog.WriteLog("%s\t错误码为：%d", e.GetExpDesc(), e.GetExpCode());
}
catch(...)
{
m_ComsLog.WriteLog("PostRecv出现未知错误");
}

return FALSE;
}

// 投递发送请求
BOOL CIUSocket::PostSend(sockaddr_in addr, char* buf, int len )
{
PPER_IO_DATA pPerIO = (PPER_IO_DATA)::GlobalAlloc(GPTR, sizeof(PER_IO_DATA));

ASSERT(pPerIO != NULL);

try
{
pPerIO->nOperationType = OP_WRITE;
pPerIO->len = MAX_BUF_SIZE;
pPerIO->addrLen = sizeof(sockaddr);
memcpy( &pPerIO->addr, &addr, sizeof(sockaddr) );
memcpy( &pPerIO->buf, buf, len );
pPerIO->len = len;

WSABUF wbuf = { pPerIO->len, pPerIO->buf };
DWORD dwSend= 0, dwFlags = 0;

int nSend = ::WSASendTo(
m_sUdp,
&wbuf,
1,
&dwSend,
dwFlags,
(sockaddr*)(&pPerIO->addr),
pPerIO->addrLen,
&pPerIO->ol,
NULL);

if( nSend != NO_ERROR )
{
int nErr = ::WSAGetLastError();
if( nErr != WSA_IO_PENDING)
{
throw CSrvException(_T("WSASendTo 执行失败"), nErr);
}
}

return TRUE;
}
catch(CSrvException& e)
{
m_ComsLog.WriteLog("%s\t错误码为：%d", e.GetExpDesc(), e.GetExpCode());
}
catch(...)
{
m_ComsLog.WriteLog("PostSend出现未知错误");
}

return FALSE;
}

[zwz1984]

头文件：

#pragma once

#include <winsock2.h>
#pragma comment(lib, "WS2_32") // 链接到WS2_32.lib

#include "Packet.h"

#include "RequestList.h"
#include "../Log/ServerLog.h"

#define S_PORT 6000 //端口号
#define UDP_WAIT_COUNT 2497 //投递的总接收数
#define THREAD_COUNT 4 //要开启的线程数

#define OP_READ 1
#define OP_WRITE 2

#pragma pack(push)
#pragma pack(1)

// per-I/O数据
typedef struct _PER_IO_DATA
{
OVERLAPPED ol; // 重叠结构
int nOperationType; // 操作类型

sockaddr_in addr;
int addrLen;

char buf[MAX_BUF_SIZE];// 数据缓冲区
int len; // 数据长度
} PER_IO_DATA, *PPER_IO_DATA;

#pragma pack(pop)