异步的TCP服务器程序

我做了个TCP服务器程序，用来监听各个客户端的连接，并与之通信。照着书上用异步的方式写了程序，以下是代码。

        private void AcceptConnect() 

        {

            IPAddress[] ip = Dns.GetHostAddresses(Dns.GetHostName());

            listener = new TcpListener(ip[0], 51888);

            listener.Start();

            listBoxStatus.Invoke(setListBoxCallback, "开始等待客户连接");

            while (isExit == false) 

            {

                try 

                {

                    allDone.Reset();

                    AsyncCallback callback = new AsyncCallback(AcceptTcpClientCallback);

                    listener.BeginAcceptTcpClient(callback,listener);

                    allDone.WaitOne();

                }

                catch (Exception ex) 

                {

                    listBoxStatus.Invoke(setListBoxCallback,ex.Message);

                    break;

                }

            }

        }



        private void AcceptTcpClientCallback(IAsyncResult ar) 

        {

            try 

            {

                allDone.Set();

                TcpListener myListener = (TcpListener)ar.AsyncState;

                TcpClient client = myListener.EndAcceptTcpClient(ar);

                listBoxStatus.Invoke(setListBoxCallback, "已接受客户连接:" +                       client.Client.RemoteEndPoint);

                comboBox1.Invoke(setComboBoxCallback,client.Client.RemoteEndPoint.ToString());

                ReadWriteObject readWriteObject = new ReadWriteObject(client);

                clientList.Add(readWriteObject);

                SendString(readWriteObject,"服务器已经接受连接");

                readWriteObject.netStream.BeginRead(readWriteObject.readBytes,0,readWriteObject.readBytes.Length,ReadCallback,readWriteObject);

            }

            catch(Exception ex)

            {

                listBoxStatus.Invoke(setListBoxCallback, ex.Message);

            }

        }

        private void ReadCallback(IAsyncResult ar) 

        {

            try 

            {

                ReadWriteObject readWriteObject = (ReadWriteObject)ar.AsyncState;

                int count = readWriteObject.netStream.EndRead(ar);

                richTextBoxReceive.Invoke(setRichTextBoxCallback,string.Format("[来自{0}]{1}",readWriteObject.client.Client.RemoteEndPoint,str));

                if(isExit==false)

                {

                    readWriteObject.InitReadArray();

                    readWriteObject.netStream.BeginRead(readWriteObject.readBytes, 0, readWriteObject.readBytes.Length, ReadCallback, readWriteObject);

                }

            }

            catch(Exception ex)

            {

                //listBoxStatus.Invoke(setListBoxCallback,ex.Message);

            }

        }

基本按这个代码来的，小有改动。运行结果达到预期，但是发现两个问题，一个是CPU占用率会一直上升到100%，二是内存一直在涨。
第一个问题，我加了Thread.Sleep(1)解决问题。第二个问题我认为是出在AcceptConnection里，因为客户端会每隔1分钟都给我发个连接，我的服务器程序没有判断功能，对重复的连接也进行操作了，致使在没有新客户的情况下也新开线程。事实也是如此，通过任务管理器也可以看出我这个程序的线程数一直在增加。我在AcceptTcpClientCallback里做了个判断，对于重复的IP和端口直接返回，但是我不知道AcceptConnection里的callback如何释放掉。

不知我的理解对否，求解答。