大文件关键字搜索, 一个8G多的文件单线程只要27秒, 多线程反而要一分多钟???

Sandrer 2019-08-07 11:44:12

加精

是不是代码有问题???

#include "stdafx.h"



static volatile ULONGLONG m_ullKeyCount = 0;

static SRWLOCK m_LockLog;



typedef struct _MAPPINGINFO

{

    HANDLE hMap;

    LPCSTR pchKey;

    LPSTR pchTail;

    LPSTR pchHead;

    DWORD dwKeyLen;

    ULARGE_INTEGER uiStartPoint;

    ULARGE_INTEGER uiBlockSize;

} MAPPINGINFO, *PMAPPINGINFO;



#define MAX_BLOCK_SIZE  (512 * 1024)



LPCTSTR m_pszPathName = _T("E:\\vm\\Win7_x86\\Win7_x86.vmdk");



static void _log(LPCTSTR fmt, ...)

{

    SYSTEMTIME st;

    va_list ap;



    AcquireSRWLockExclusive(&m_LockLog);



    GetLocalTime(&st);

    _tprintf_s(_T("%02u:%02u:%02u.%03u >> "), st.wHour, st.wMinute, st.wSecond, st.wMilliseconds);



    va_start(ap, fmt);

    _vtcprintf_s(fmt, ap);



    _tprintf_s(_T("\n"));



    ReleaseSRWLockExclusive(&m_LockLog);

}



#ifdef _UNICODE

#define log(fmt, ...)   _log(L##fmt, __VA_ARGS__)

#else

#define log(fmt, ...)   _log(fmt, __VA_ARGS__)

#endif



static unsigned __stdcall thread_search(void *);



void TestLargeFileRead()

{

    UINT i, j, len;

    HANDLE hFile;

    HANDLE hMap = NULL;

    HANDLE *phThreads = NULL;

    MAPPINGINFO *pInfo = NULL;

    DWORD dwThreadCount = 0;

    DWORD dwProcessors = 0;

    LPSTR pchHeadTail = NULL;

    SYSTEM_INFO si;

    ULARGE_INTEGER uiLeftSize;

    ULARGE_INTEGER uiFileSize;

    ULARGE_INTEGER uiBlockSize;

    ULARGE_INTEGER uiBeginPoint;

    ULONGLONG ullCompare;

    ULONGLONG ullBegin, ullEnd;

    CHAR chKey[64];



    InitializeSRWLock(&m_LockLog);



    hFile = CreateFile(m_pszPathName, GENERIC_READ, FILE_SHARE_READ, NULL, OPEN_EXISTING, 0, NULL);

    if (hFile == INVALID_HANDLE_VALUE)

    {

        log("Could not open file, error %u.\n", GetLastError());

        goto _exit;

    }

    log("File Opened.");



    if (!GetFileSizeEx(hFile, (PLARGE_INTEGER)&uiFileSize))

    {

        CloseHandle(hFile);

        log("Could not get file size, error %u.", GetLastError());

        goto _exit;

    }

    log("File Size: %I64u, Block Count: %I64u", uiFileSize.QuadPart, (uiFileSize.QuadPart + (MAX_BLOCK_SIZE - 1)) / MAX_BLOCK_SIZE);



    hMap = CreateFileMapping(hFile, NULL, PAGE_READONLY, 0, 0, NULL);

    CloseHandle(hFile);

    if (hMap == NULL)

    {

        log("Could not mapping file, error %u.\n", GetLastError());

        goto _exit;

    }

    log("File Mapped.");



    GetSystemInfo(&si);

    dwProcessors = si.dwNumberOfProcessors;

    if (uiFileSize.QuadPart < (512 * 1024 * 1024))

        si.dwNumberOfProcessors = 1;



    // 与系统分配粒度对齐

    uiBlockSize.QuadPart = ((uiFileSize.QuadPart / dwProcessors) + si.dwAllocationGranularity - 1) / si.dwAllocationGranularity * si.dwAllocationGranularity;

    dwProcessors = (DWORD)((uiFileSize.QuadPart + (uiBlockSize.QuadPart - 1)) / uiBlockSize.QuadPart);



    phThreads = (HANDLE *)malloc(sizeof(HANDLE) * dwProcessors);

    pInfo = (MAPPINGINFO *)malloc(sizeof(MAPPINGINFO) * dwProcessors);

    if (phThreads == NULL || pInfo == NULL)

    {

        log("Not enough memory.");

        goto _exit;

    }



_get_key:



    printf("Enter Search Key: ");



    gets_s(chKey, _countof(chKey));

    if (chKey[0] == 0)

        goto _exit;



    GetSystemTimeAsFileTime((LPFILETIME)&ullBegin);



    len = strlen(chKey);



    pchHeadTail = (LPSTR)malloc(sizeof(char) * (len + 1) * 2);

    if (pchHeadTail == NULL)

    {

        log("Not enough memory.");

        goto _exit;

    }



    uiBeginPoint.QuadPart = 0;

    uiLeftSize = uiFileSize;

    dwThreadCount = 0;



    for (i = 0; i < dwProcessors; i++)

    {

        pInfo[i].hMap = hMap;

        pInfo[i].pchKey = chKey;

        pInfo[i].dwKeyLen = len;

        pInfo[i].uiStartPoint = uiBeginPoint;

        if (uiLeftSize.QuadPart < uiBlockSize.QuadPart)

            pInfo[i].uiBlockSize = uiLeftSize;

        else

            pInfo[i].uiBlockSize = uiBlockSize;



        uiBeginPoint.QuadPart += pInfo[i].uiBlockSize.QuadPart;

        uiLeftSize.QuadPart -= pInfo[i].uiBlockSize.QuadPart;



        phThreads[i] = (HANDLE)_beginthreadex(NULL, 0, thread_search, &pInfo[i], 0, NULL);

        if (phThreads[i] == NULL)

            break;



        dwThreadCount++;

    }



    WaitForMultipleObjects(dwThreadCount, phThreads, TRUE, INFINITE);

    for (i = 0; i < dwThreadCount; i++)

        CloseHandle(phThreads[i]);



    // TODO: 将每个分块中的头/尾组合， 然后需要比较是否有关键字

    free(pchHeadTail);



    GetSystemTimeAsFileTime((LPFILETIME)&ullEnd);



    log("Time Used: %u.%04u ms, Key Count: %I64u",

        (DWORD)((ullEnd - ullBegin) / 10000),

        (DWORD)((ullEnd - ullBegin) % 10000),

        m_ullKeyCount);



    goto _get_key;



_exit:



    if (phThreads != NULL)  free(phThreads);

    if (pInfo != NULL)      free(pInfo);

    if (hMap != NULL)       CloseHandle(hMap);

}



static unsigned __stdcall thread_search(void *p)

{

    UINT i, j;

    UINT cmp_len = 0;

    PVOID pView = NULL;

    PBYTE pData = NULL;

    LPCSTR pchKey;

    DWORD dwKeyLen;

    DWORD dwMapSize;

    ULARGE_INTEGER uiMapPoint;

    ULARGE_INTEGER uiLeftSize;

    PMAPPINGINFO pInfo = (PMAPPINGINFO)p;



    log("Handle data from %I64u, size %I64u.", pInfo->uiStartPoint.QuadPart, pInfo->uiBlockSize.QuadPart);



    pchKey = pInfo->pchKey;

    dwKeyLen = pInfo->dwKeyLen;

    uiLeftSize = pInfo->uiBlockSize;

    uiMapPoint = pInfo->uiStartPoint;



    while (uiLeftSize.QuadPart)

    {

        dwMapSize = MAX_BLOCK_SIZE;



        if (uiLeftSize.QuadPart < (ULONGLONG)dwMapSize)

        {

            uiMapPoint.QuadPart = pInfo->uiStartPoint.QuadPart + (pInfo->uiBlockSize.QuadPart - (pInfo->uiBlockSize.QuadPart % dwMapSize));

            dwMapSize = uiLeftSize.LowPart;

        }



        pView = MapViewOfFile(pInfo->hMap, FILE_MAP_READ, uiMapPoint.HighPart, uiMapPoint.LowPart, dwMapSize);

        if (pView == NULL)

        {

            log("Could not handle file, error %u.", GetLastError());

            goto _exit;

        }



        uiLeftSize.QuadPart -= dwMapSize;



        pData = (PBYTE)pView;

        for (i = 0; i < dwMapSize; i++)

        {

            if (pData[i] == pchKey[cmp_len])

            {

                for (j = cmp_len + 1; j < dwKeyLen; j++)

                {

                    if (dwMapSize <= i + j)

                    {

                        cmp_len = j;

                        goto _next;

                    }

                    else if (pData[i + j] != pchKey[j])

                    {

                        cmp_len = 0;

                        goto _next;

                    }

                }



                // '12': 849816

                i += dwKeyLen - cmp_len - 1;

                InterlockedIncrement64((LONGLONG *)&m_ullKeyCount);

            }



            cmp_len = 0;



_next:

            ;

        }



        uiMapPoint.QuadPart += MAX_BLOCK_SIZE;



        UnmapViewOfFile(pView);

        pView = NULL;

    }



_exit:

    if (pView != NULL)

        UnmapViewOfFile(pView);



    return 0;

}