征求频繁new/delete提升速度的解决办法

[chenyu2202863]

背景：
每隔200ms左右就会new一个50KB左右的内存空间，然后在间隔很短的时间内又释放。现在需要一个内存池配合现有的代码，希望能改善目前频率很高的new和delete操作。

boost的pool怎样？看介绍是适合小内存的快速申请。

大家推荐个内存池实现吧，最好有线程安全的

[churenxh]

实现了变长内存池的申请和回收，其中回收包括在某种特定的情况下回收给系统



h文件：GenMemPool.h

#ifndef _SKIFFER_GENMEMPOOL_H
#define _SKIFFER_GENMEMPOOL_H

#include <stdlib.h>

//按此值向上取整申请所需内存大小，例如，如果申请内存为13字节，则返回14字节的内存
#define MEMPOOL_ALIGNMENT (2)
#define MAX_MEN_LIMIT (1<<16)
#pragma pack(1)
struct STMemBlk
{
//此内存块中内存单元的个数
unsigned short m_wAllUnitNum;
//空闲单元的个数
unsigned short m_wFreeUnitNum;
//第一个空闲单元的单元编号
unsigned short m_wFirstFreeUnitNo;
//next 指针
STMemBlk *m_pNextBlk;
//以上为内存块结构头部信息

//以下为外界可用内存
char m_pData[0];

STMemBlk()
{
m_wAllUnitNum = 0;
m_wFreeUnitNum = 0;
m_wFirstFreeUnitNo = 0;
m_pNextBlk = NULL;
}
~STMemBlk()
{
}

void* operator new(size_t wBlkSize, unsigned short wUnitNum, unsigned short wUnitSize)
{
return ::operator new (sizeof(STMemBlk) + wUnitNum * wUnitSize);
}

void operator delete(void *p)
{
::operator delete (p);
}

void operator delete(void *p, unsigned short wUnitNum, unsigned short wUnitSize)
{
::operator delete (p);
}
};

struct STInitParam
{
//每个内存单元的大小
unsigned short m_wUnitSize;
//第一次初始化时，每一块内存中，需要申请的内存单元个数
unsigned short m_wInitUnitNum;
//内存不足时，每一块内存中，需要申请的内存单元个数
unsigned short m_wGrowUnitNum;
STInitParam()
{
m_wUnitSize = 0;
m_wInitUnitNum = 0;
m_wGrowUnitNum = 0;
}
};
#pragma pack()


class CMemPool
{

public:
CMemPool();
~CMemPool();

//初始化
void Init(unsigned short wUnitSize, unsigned short wInitUnitNum, unsigned short wGrowUnitNum);
//分配内存块中的一个内存单元
void* Alloc();
//释放一个内存单元
void Free(void *pUnit);
//预处理块
void* PreAlloc(STMemBlk *pstMemBlk, int dwBlkFlag);

private:
//内存池头结点
STMemBlk *m_pstMemBlkHead;
//内存池块信息
STInitParam m_stInitBlkInfo;
};

class CVarLenMemPool
{
public:
CVarLenMemPool();
~CVarLenMemPool();

//申请dwMemSize字节的内存
void* Alloc(unsigned int dwMemSize);
//释放内存
void Free(void *p);
//初始化n个内存单元长度不同的固定内存池
void Init(const STInitParam *pstInitParam, unsigned short wMemPoolNum);
//退出
void Exit();
//向上对齐
void UpAlignment(STInitParam *pstInitParam, unsigned short wMemPoolNum);

public:
//内存单元大小不同的内存池
CMemPool **m_ppMemPoolList;
//n个内存单元长度不同的固定内存池
STInitParam *m_pstInitParam;
//内存池个数
unsigned short m_wMemPoolNum;
//最大的内存单元大小
unsigned short m_wMaxUnitSize;
};

#endif




.cpp文件:GenMemPool.cpp

#include "GenMemPool.h"
#include <string.h>

CMemPool::CMemPool()
{
m_pstMemBlkHead = NULL;
memset(&m_stInitBlkInfo, 0 , sizeof(m_stInitBlkInfo));
}

CMemPool::~CMemPool()
{
}

void CMemPool::Init(unsigned short wUnitSize, unsigned short wInitUnitNum, unsigned short wGrowUnitNum)
{
if(wUnitSize <= 4)
{
wUnitSize = 4;
}
else
{
wUnitSize = (wUnitSize + (MEMPOOL_ALIGNMENT - 1)) & ~(MEMPOOL_ALIGNMENT - 1);
}
m_stInitBlkInfo.m_wUnitSize = wUnitSize;
m_stInitBlkInfo.m_wInitUnitNum = wInitUnitNum;
m_stInitBlkInfo.m_wGrowUnitNum = wGrowUnitNum;
}

void* CMemPool::PreAlloc(STMemBlk *pstMemBlk, int dwBlkFlag)
{
char *pData = NULL;
unsigned short dwUnitNum = 0;
pData = pstMemBlk->m_pData;

if(dwBlkFlag == 1)
{
dwUnitNum = m_stInitBlkInfo.m_wInitUnitNum;
}
else
{
dwUnitNum = m_stInitBlkInfo.m_wGrowUnitNum;
}

pstMemBlk->m_wFreeUnitNum = dwUnitNum - 1;
pstMemBlk->m_wAllUnitNum = dwUnitNum;
pstMemBlk->m_wFirstFreeUnitNo = 1;

for(unsigned short dwUnit = 1; dwUnit < dwUnitNum - 1; ++dwUnit)
{
pData = pstMemBlk->m_pData + dwUnit * m_stInitBlkInfo.m_wUnitSize;
*(unsigned short*)pData = dwUnit + 1;
}
pData = pstMemBlk->m_pData;
return pData;
}

void* CMemPool::Alloc()
{
void *pData = NULL;
STMemBlk *pstMemBlk = NULL;

if(m_pstMemBlkHead == NULL)
{
m_pstMemBlkHead = ::new STMemBlk;
m_pstMemBlkHead->m_pNextBlk = new(m_stInitBlkInfo.m_wInitUnitNum, m_stInitBlkInfo.m_wUnitSize)STMemBlk;
pData = PreAlloc(m_pstMemBlkHead->m_pNextBlk, 1);
}
else
{
pstMemBlk = m_pstMemBlkHead->m_pNextBlk;
while(pstMemBlk != NULL)
{
if(pstMemBlk->m_wFreeUnitNum == 0)
{
pstMemBlk = pstMemBlk->m_pNextBlk;
continue;
}
else
{
break;
}
}
if(NULL == pstMemBlk)
{
pstMemBlk = new(m_stInitBlkInfo.m_wGrowUnitNum, m_stInitBlkInfo.m_wUnitSize)STMemBlk;
pData = PreAlloc(pstMemBlk, 2);
pstMemBlk->m_pNextBlk = m_pstMemBlkHead->m_pNextBlk;
m_pstMemBlkHead->m_pNextBlk = pstMemBlk;
}
else
{
pData = (void*)(pstMemBlk->m_pData + pstMemBlk->m_wFirstFreeUnitNo * m_stInitBlkInfo.m_wUnitSize);
--(pstMemBlk->m_wFreeUnitNum);
pstMemBlk->m_wFirstFreeUnitNo = *(unsigned short*)pData;
}
}
return pData;
}

void CMemPool::Free(void *pUnit)
{
STMemBlk *pstMemBlk = NULL;
STMemBlk *pstMemBlkPre = NULL;
unsigned short dwBlkNo = 0;
pstMemBlkPre = m_pstMemBlkHead;
pstMemBlk = m_pstMemBlkHead->m_pNextBlk;

while(pstMemBlk != NULL)
{
if(((unsigned int)pUnit) > (unsigned int)pstMemBlk &&
(unsigned int)pUnit <
(unsigned int)(pstMemBlk->m_pData + (pstMemBlk->m_wAllUnitNum + 1) * m_stInitBlkInfo.m_wUnitSize))
{
dwBlkNo = ((unsigned int)pUnit - (unsigned int)(pstMemBlk->m_pData))/m_stInitBlkInfo.m_wUnitSize;
break;
}
else
{
pstMemBlkPre = pstMemBlk;
pstMemBlk = pstMemBlk->m_pNextBlk;
}
}

if(pstMemBlk->m_wFreeUnitNum - 1 == pstMemBlk->m_wAllUnitNum)
{
pstMemBlkPre->m_pNextBlk = pstMemBlk->m_pNextBlk;
delete(pstMemBlk);
}
else
{
*(unsigned short*)pUnit = pstMemBlk->m_wFirstFreeUnitNo;
pstMemBlk->m_wFirstFreeUnitNo = dwBlkNo;
++(pstMemBlk->m_wFreeUnitNum);
}
}


CVarLenMemPool::CVarLenMemPool()
{
m_ppMemPoolList = NULL;
m_pstInitParam = NULL;
m_wMemPoolNum = 0;
m_wMaxUnitSize = 0;
}

CVarLenMemPool::~CVarLenMemPool()
{
}

void CVarLenMemPool::Init(const STInitParam *pstInitParam, unsigned short wMemPoolNum)
{
unsigned short wInitIdxBegin = 0;
unsigned short wInitIdxEnd = 0;
m_wMemPoolNum = wMemPoolNum;
m_pstInitParam = new STInitParam[m_wMemPoolNum];
memcpy(m_pstInitParam, pstInitParam, sizeof(*pstInitParam) * wMemPoolNum);
UpAlignment(m_pstInitParam, wMemPoolNum);

m_wMaxUnitSize = 0;
unsigned short i = 0;

for(i = 0; i < m_wMemPoolNum; ++i)
{
if(m_pstInitParam[i].m_wUnitSize > m_wMaxUnitSize)
{
m_wMaxUnitSize = m_pstInitParam[i].m_wUnitSize;
}
}

m_ppMemPoolList = new CMemPool*[m_wMaxUnitSize+1];

for(i = 0; i < m_wMemPoolNum; ++i)
{
CMemPool *pMemPool = new CMemPool;
pMemPool->Init(m_pstInitParam[i].m_wUnitSize, m_pstInitParam[i].m_wInitUnitNum,
m_pstInitParam[i].m_wGrowUnitNum);
wInitIdxEnd = m_pstInitParam[i].m_wUnitSize;

for(unsigned short j = wInitIdxBegin; j <= wInitIdxEnd; ++j)
{
m_ppMemPoolList[j] = pMemPool;
}
wInitIdxBegin = wInitIdxEnd + 1;
}
}

void* CVarLenMemPool::Alloc(unsigned int dwMemSize)
{
void *p = NULL;
if(dwMemSize+2 > m_wMaxUnitSize)
{
p = ::operator new(dwMemSize+2);
*(unsigned short*)p = 0;
}
else
{
p = m_ppMemPoolList[dwMemSize+2]->Alloc();
*(unsigned short*)p = dwMemSize+2;
}
return ((unsigned short*)p+1);
}

void CVarLenMemPool::Free(void *p)
{
unsigned short *pTmp = ((unsigned short*)p)-1;
unsigned short dwSize = *pTmp;
if(0 == dwSize)
{
::operator delete(pTmp);
}
else
{
m_ppMemPoolList[dwSize]->Free(p);
}
}

void CVarLenMemPool::UpAlignment(STInitParam *pstInitParam, unsigned short wMemPoolNum)
{
for(unsigned short wParam; wParam < wMemPoolNum; ++wParam)
{
if(pstInitParam[wParam].m_wUnitSize <= 4)
{
pstInitParam[wParam].m_wUnitSize = 4;
}
else
{
pstInitParam[wParam].m_wUnitSize = (pstInitParam[wParam].m_wUnitSize + (MEMPOOL_ALIGNMENT - 1)) &
~(MEMPOOL_ALIGNMENT - 1);
}
}
}


void CVarLenMemPool::Exit()
{

}

[dirdirdir3]

楼主的方法还是不错的,不过每次分配的时候都要检索一次还要排序不慢就奇怪了，应该有更好的方法吧.............如预先把内存分为10类，如1k的，5k的，10k的........1m的等等，然后按照大概的同时使用的几率分配好有多少个，然后每次直接取相应的最小内存就是了，这个是以空间换时间吧.

[tikycc2]

看看 帮顶

[xuxinmmy]

按楼主的描述，好像只是临时使用，数据长度也大概确定在50K左右
其实应该有个最简单的方法，就是给每个线程各分配一块内存（比如100K），线程启动时申请，过程中随便用，线程结束时释放。

有简单的方法没必要去搞内存池什么的吧

[secretcf]

#define SIZE_100K (1024*100)
#define BUFF_COUNT 10

class MyMemory
{
static void Init()
{
memset(m_buff, 0, sizeof(m_buff));
int i = 0;

for(i=0; i<BUFF_COUNT; i++)
{
m_byUse[i] = 0;
}
}

static BYTE* GetBuffer()
{
BYTE *pBuff = NULL;
int i = 0;
for(i=0; i<BUFF_COUNT; i++)
{
if(m_byUse[i] == 0)
{
pBuff = m_buff+(i*SIZE_100K);
m_byUse[i] = 1;
break;
}
}

return pBuff;
};

static void ReleaseBuffer(BYTE* pBuff)
{
if(pBuff < m_buff || pBuff > SIZE_100K*(BUFF_COUNT-1) )
{
return;
}

DWORD dwPos = (pBuff - m_buff) / SIZE_100K;
if(dwPos >=0 && dwPos < BUFF_COUNT)
{
m_buUse[dwPos] = 0;
memset(pBuff, 0, SIZE_100K);
}
};

private:
static BYTE m_buff[SIZE_TOTAL];
static BYTE m_byUse[BUFF_COUNT];
};

[gbb21]

一般就用boost的么～

[周成风]

學習了。。都太厲害了！

[cowboytian]

顶一下！

[HuWenjin]

我觉得new delete这样的一些操作，是非常多的专家经过深思熟虑写出来的
在速度上和其它方面都有非常多的考虚。

为什么程序员都想把非常多的东西自己拿来写呢？呵呵，我有时候也爱这样。

不过，我现在不了
我做过软件时测试自己程序的容量，树形内存，最大分支有几万个点，然后进行删除操作
发现速度非常慢，我也常试标注一下，不删除，可带来其它问题。
我仔细检查程序后，把一些不必要的if删除然后再测，没想到同样的数据再删除时几乎不需要时间了...

程序结构流程的优化，才是提升速度的根本

不知我说的对否。

[琥珀明月]

不懂，看看，了解下！

[roughman9999]

借鉴下MFC CString的内存管理吧，比较简单也很实用

[axx1611]

类似的东西写过好几个了，都是工业级应用了

基本上两种方法最有效，一是重载new delete运算符，二是用placement new

[jeff_nie]

MARK

[岁月-静-好]

学习了！谢谢！

[pysjp]

mark,学习了

[jamseyang]

mark!

[sanywangjun]

mark!

[sadtime200]

太屌了，集合好几个MVP见解的帖子

[fairywell]

[Quote=引用 64 楼 saimen 的回复:]

我觉得new delete这样的一些操作，是非常多的专家经过深思熟虑写出来的
在速度上和其它方面都有非常多的考虚。

为什么程序员都想把非常多的东西自己拿来写呢？呵呵，我有时候也爱这样。

不过，我现在不了
我做过软件时测试自己程序的容量，树形内存，最大分支有几万个点，然后进行删除操作
发现速度非常慢，我也常试标注一下，不删除，可带来其它问题。
我仔细检查程序后，把一些不必要的i……
[/Quote]
是否你的树退化了 ：） 要用平衡树哦~

[fairywell]

mark