Html代码检查闭合和修复

[zhanglong_longlong]

是这样的，网上抓取了些网站的html代码保存到数据库中，现在很多抓下来的html代码里面只有开头标记，没有结束，有没有方法可以解决？
比如：<div><p>asdfsadfsadfsafsaf</p> 这里就没有结束的</div>标签，当然不只是<div>标签，各种没成对，忘提供好的方法

[娃都会打酱油了]

这个不就是跟上次table一样的么？

[娃都会打酱油了]

 /// <summary>
        /// 读取Html
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        static string ReadHtml(string url)
        {
            //WebRequest request = WebRequest.Create(url);
            throw new NotImplementedException();
            //最终返回结果
        }

        /// <summary>
        /// 解析A标签对应的链接字符串
        /// </summary>
        /// <param name="html"></param>
        /// <returns></returns>
        static List<string> GetLinks(string html)
        {
            throw new NotImplementedException();
        }

        static void Loop(string url)
        {
            string html = ReadHtml(url);
            //html 处理
            List<string> urls = GetLinks(html);
            if (urls != null && urls.Count > 0)
            {
                urls.ForEach(u =>
                {
                    Loop(u);
                });
            }
        }

你的代码太混乱，写个伪代码你参考下

[zhanglong_longlong]

引用 3 楼 starfd 的回复:

private void button1_Click(object sender, EventArgs e)
        {
 //需要解析的集合
            List<string> list = new List<string>();
 Loop("http://www.baidu.com",listCount);
}
 /// <summary>
        /// 读取Html
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        static string ReadHtml(string url)
        {
            //WebRequest request = WebRequest.Create(url);
            throw new NotImplementedException();
            //最终返回结果
        }

        /// <summary>
        /// 解析A标签对应的链接字符串
        /// </summary>
        /// <param name="html"></param>
        /// <returns></returns>
        static List<string> GetLinks(string html)
        {
            throw new NotImplementedException();
        }

        static void Loop(string url)
        {
            string html = ReadHtml(url);
            //html 处理
            List<string> urls = GetLinks(html);
            if (urls != null && urls.Count > 0)
            {
                urls.ForEach(u =>
                {
                    Loop(u);
                });
            }
        }

你的代码太混乱，写个伪代码你参考下

/// <summary>
        /// 读取Html
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
         string ReadHtml(string url)
        {
            WebClient MyWebClient = new WebClient();

            MyWebClient.Credentials = CredentialCache.DefaultCredentials;//获取或设置用于向Internet资源的请求进行身份验证的网络凭据

            Byte[] pageData = MyWebClient.DownloadData(url); //从指定网站下载数据

            return Encoding.UTF8.GetString(pageData); //如果获取网站页面采用的是UTF-8，则使用这句
        }

        /// <summary>
        /// 解析A标签对应的链接字符串
        /// </summary>
        /// <param name="html"></param>
        /// <returns></returns>
         List<string> GetLinks(string html)
        {
            List<string> lst = new List<string>();
            //解析
            Regex regex = new Regex(@"<a[^>]*href=([""'])?(?<href>[^'""]+)\1[^>]*>");
            foreach (Match mc in regex.Matches(html))
            {
                string url = GetNewUrl(mc.Groups[2].Value);
                if (!string.IsNullOrEmpty(url))
                {
                    lst.Add(url);
                }
            }

            return lst;
        }

         void Loop(string url, List<string> listCount)
        {
             //没有解析过该项
            if (!listCount.Contains(url))
            {
                //在已解析过的集合里面添加本条数据
                listCount.Add(GetNewUrl(url));
                string html = ReadHtml(url);
                //html 处理
                List<string> urls = GetLinks(html);
                if (urls != null && urls.Count > 0)
                {
                    urls.ForEach(u =>
                    {
                        lsbSuccess.Items.Add(url);
                        Loop(u, listCount);
                    });
                }
            }
        }

[zhanglong_longlong]

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.IO;
using System.Net;
using System.Text.RegularExpressions;

namespace ConsoleApplication1
{
    class Program
    {
        static void Main(string[] args)
        {
            //需要解析的集合
            List<string> list = new List<string>();
            //已经解析的集合
            List<string> listCount = new List<string>();
            list.Add("http://www.baidu.com");

            ReadHtml(list,listCount);
            Console.ReadLine();
        }

        /// <summary>
        /// 读取HTML中的URL
        /// </summary>
        /// <param name="list"></param>
        /// <param name="listCount"></param>
        public static void ReadHtml(List<string> list, List<string> listCount) 
        {
            List<string> count = new List<string>();
            for (int a = 0; a < list.Count; a++)
            {
                //没有解析过该项
                if (!listCount.Contains(list[a]))
                {
                    try
                    {
                        //在已解析过的集合里面添加本条数据
                        listCount.Add(list[a]);
                        WebRequest req = WebRequest.Create(list[a]);
                        WebResponse result = req.GetResponse();
                        //得到的流是网页内容   
                        Stream ReceiveStream = result.GetResponseStream();
                        StreamReader readerOfStream = new StreamReader(ReceiveStream, System.Text.Encoding.GetEncoding("GB2312"));
                        //得到当前URL的源码
                        string str = readerOfStream.ReadToEnd();
                        //解析
                        Regex regex = new Regex(@"http://([\w-]+\.)+[\w-]+(/[\w- ./?%&=]*)?");
                        foreach (Match mc in regex.Matches(str))
                        {

                            Console.WriteLine(mc.Groups[2].Value);
                            count.Add(mc.Groups[2].Value);
                            
                        }
                        Console.WriteLine("----------------------解析完一个页面{0}!--------------------",listCount.Count);
                        if (a == list.Count - 1)
                        {
                            //递归调用本方法
                            ReadHtml(count, listCount);
                        }
                    }
                    catch (System.Exception ex) { }
                    finally 
                    {
                        List<string> error = new List<string>();
                        //如果出错在出错的后面一条URL继续解析
                        for (int z = a + 1; z < list.Count; z++) 
                        {
                            error.Add(list[z]);
                        }
                        //继续解析
                        ReadHtml(error, listCount);
                    }
                }
            }
        }
    }
}