socket operation on non-socket

guxiangxi 2009-08-10 08:45:40

我最近做的一个程序,客户上传文件给服务器 ,服务器对文件进行压缩,然后回传给客户,利用线程池的方法.
现在有个问题,就是我启动单独的客户端,文件收发都很正常.但是当我用shell启动100个客户端的时候,服务器端一般情况会有95个正常,有2~3个左右出现socket operation on non-socket错误,这个错误出现在第一次receive客户端数据的时候.但是accept函数是正确的,而且在线程中的客户FD和accept的客户FD也是一样的.
问了下高手,大概有这样几个问题:
1.丢包,一般连续丢三个包,就可能导致连接不成立.,但是如果在本机测试的话(客户程序和服务器程序都在同一机器上),由于不经过网卡,是不会丢包的.
2.在服务器上的客户端描述符不正确,但是我通过打印日志,在epoll的epollin处理中接受到的客户FD和传递给线程的FD是一致的.是没有问题的.
我想问下,除了以上问题,还有没有什么可能性,导致:正常accpet客户连接,但是在recv该客户连接的时候,出现socket operation on non-socket错误.
代码如下(代码有参考网上的资源)
#include <stdlib.h>
#include <stdio.h>
#include <sys/socket.h>
#include <sys/epoll.h>
#include <netdb.h>
#include <arpa/inet.h>
#include <fcntl.h>
#include <errno.h>
#include <unistd.h>
#include <string.h>
#include <zlib.h>
#include <zconf.h>
#include <pthread.h>
//this program is server which receive client file and zip it ,then send it back
#define THREAD_MAX 100

int dealClient(int cliFd);

//thread paragma
static int s_thread_para[THREAD_MAX][8];
static pthread_t s_thread_tid[THREAD_MAX];
pthread_mutex_t s_mutex[THREAD_MAX];

static int init_thread_pool(void);

//thread function
void *serverThread(void *thread_para);

void setNonBlock(int fd);

int main()
{
int lisFd;
struct sockaddr_in serverAddr;
struct sockaddr_in clientAddr;

bzero(&serverAddr,sizeof(serverAddr));
bzero(&clientAddr,sizeof(clientAddr));
serverAddr.sin_family=AF_INET;
serverAddr.sin_port=htons(3491);
serverAddr.sin_addr.s_addr=INADDR_ANY;

printf("Server:Create socket\n");
if((lisFd=socket(AF_INET,SOCK_STREAM,0))<0)
{
printf("listen socket create fail!\n");
return -1;
}

printf("Server:Bind socket\n");
if((bind(lisFd,(sockaddr*)&serverAddr,sizeof(serverAddr)))==-1)
{
printf("bind socket fail:%s\n",strerror(errno));
return -1;
}

printf("Server:Listen socket\n");
if((listen(lisFd,20))<0)
{
printf("listen faile!\n");
return -1;
}

setNonBlock(lisFd);
//init thread pool
printf("Server:initial thread pool\n");

init_thread_pool();
//create epoll and deal with client request

int epollFd;
epoll_event ev;
epollFd=epoll_create(THREAD_MAX);
ev.data.fd=lisFd;
ev.events=EPOLLIN;
epoll_ctl(epollFd,EPOLL_CTL_ADD,lisFd,&ev);

struct epoll_event events[THREAD_MAX];
int nfs=0,k=0,cliFd;
int var;
int i=0;
while(true)
{
// printf("Server:While in\n");
nfs=epoll_wait(epollFd,events,20,-1);
// printf("Server:Event happen,Num:%d\n",nfs);

if(nfs>1)
{
printf("///////////////////////////////////////////////////////////////////\n");
}
for(i=0;i<nfs;i++)
{
if(events[i].data.fd==lisFd)
{
//connect request!
socklen_t len=0;
if((cliFd=accept(lisFd,(sockaddr*)&clientAddr,&len))<0)
{
printf("accept client fail!\n");
continue;
}
else
{
// printf("Server:A new connect:CliFd%d\n",cliFd);
}
int tmp_ret=true;
while(tmp_ret)
{
//find free thread and start it
int threadIndex=0;
for(threadIndex=0;threadIndex<THREAD_MAX;threadIndex++)
{
if(s_thread_para[threadIndex][0]==0)
{
// printf("Server:find free thread:%d\n",threadIndex);
// printf("Server:thread param:index:%d,%d cliFd:%d\n",threadIndex,s_thread_para[threadIndex][7],cliFd);
s_thread_para[threadIndex][0]=1;
s_thread_para[threadIndex][1]=cliFd;
// printf("Server:client fd:%d\n",cliFd);
pthread_mutex_unlock(s_mutex+s_thread_para[threadIndex][7]);
// printf("Server:debug1\n");
break;
}
}
if(threadIndex>=THREAD_MAX)
{
printf("Server:thread pool is full:close fd:%d\n",cliFd);
close(cliFd);
continue;
}
else
{
break;
}
}
}
}
}
聳return 0;
}

int dealClient(int cliFd)
{
//first create temp gzfile for client;
char fileName[]="./gzFile.XXXXXX";
int fileFd=mkstemp(fileName);
// unlink(fileName);
gzFile fileGz=gzdopen(fileFd,"w");
//receive message and compress it into gzfile
int ret=0;
int bufferLen=1024;
char *buffer=new char[bufferLen];
//receive file info first
// printf("Server:Begin receive file info\n");
ret=recv(cliFd,buffer,bufferLen,0);
if(ret<0)
{
printf("Server:receive num:%d on client%d: %s\n",ret,cliFd,strerror(errno));
return -1;
}
int fileLen=atoi(buffer);
if(fileLen!=6552)
{
printf("Server:receive file len size != 6552 but :%d\n",fileLen);
}

//end file info
int sum=0;

while(true)
{
ret=recv(cliFd,buffer,bufferLen,0);
if(ret<0)
{
if(errno==EWOULDBLOCK||errno==EINTR||errno==EAGAIN)
{
printf("Server:errno:%s\n",strerror(errno));
continue;
}
}
if(ret==0)
{
printf("Server:client close socket");
return -1;
}
sum+=ret;
// printf("Server:receive data from client:%d size:%d\n",cliFd,ret);
if((gzwrite(fileGz,buffer,ret))==0)
{
printf("Server:gzwrite fail\n");
}
else
{
// printf("Server:gzWrite success\n");
}

if(sum==fileLen)
{
break;
}
// printf("Server:Sum:%d fileLen:%d\n",sum,fileLen);
}
gzclose(fileGz);
close(fileFd);
// printf("Server:receive over\n");
///////////////////////////////
//here to send gzfile to client;

struct stat gzFileStat;
stat(fileName,&gzFileStat);
memset(buffer,'\n',bufferLen);
sprintf(buffer,"%d",gzFileStat.st_size);
int n=send(cliFd,buffer,bufferLen,0);
if(n<0)
{
printf("Server:send file length fail:%s\n",strerror(errno));
}

// printf("Server:send file length:n=%d\n",n);

fileFd=open(fileName,O_RDONLY,0);
if(fileFd<0)
{
printf("Server:open gz file fail\n");
return -1;
}
else
{
// printf("Server:file open success:%d\n",fileFd);
}

// printf("Server:begin send data\n");
while((ret=read(fileFd,buffer,bufferLen))>0)
{
int n=send(cliFd,buffer,ret,0);
// printf("Server:send data to client:%d size:%d\n",cliFd,n);
if(n<0)
{
// printf("Server:send file content fail:%s\n",strerror(errno));
}
// printf("Server:send size:%d\n",n);
}
// printf("Server:send data over\n");

delete []buffer;
buffer=NULL;
return 0;
}

static int init_thread_pool(void)
{
int i,rc;
for(i=0;i<THREAD_MAX;i++)
{
s_thread_para[i][0]=0;//free
s_thread_para[i][7]=i;//thread idnex;
pthread_mutex_init(s_mutex+i,0);
pthread_mutex_lock(s_mutex+i);
}
//create pool
for(i=0;i<THREAD_MAX;i++)
{
rc=pthread_create(s_thread_tid+i,0,serverThread,(void*)s_thread_para[i]);
if(0!=rc)
{
printf("Thread create fail\n");
return -1;
}
}
return 0;
}

void *serverThread(void* threadPara)
{
unsigned int *thread_para=(unsigned int*)threadPara;
int poolIndex=0;
int cliSock=0;

// pthread_detach(pthread_self());

while(true)
{
poolIndex=thread_para[7];
pthread_mutex_lock(s_mutex+poolIndex);
// poolIndex=thread_para[7];
cliSock=thread_para[1];
// printf("Server:thread begin\n");
// printf("Server:thread para: index:%d cliFd:%d\n",poolIndex,cliSock);
// printf("Server:client fd:%d\n",cliSock);
dealClient(cliSock);
close(cliSock);
thread_para[0]=0;
printf("Server:thread over/////////////////////////////////////////////////\n");
}

printf("Server:thread exit\n");
pthread_exit(NULL);
}
void setNonBlock(int fd)
{
int opts;
opts=fcntl(fd,F_GETFL);
if(opts<0)
{
printf("fcntl error\n");
}
if(fcntl(fd,F_SETFL,opts|O_NONBLOCK)<0)
{
printf("fcntl error\n");
exit(1);
}

}

...全文