抓取网页出现乱码 看清楚里面的内容 解决立马给分

[kings2015]

抓取的网页格式

<meta http-equiv="Content-Type" content="text/html; charset=gb2312" />

和我调用里面的设置是一样的 为什么会出现乱码
试了其他格式的也是乱码


调用代码如下
///命名空间

using System.Text;
using System.IO;
using System.Net;
using System.Text.RegularExpressions;

            try
            {

                HttpWebRequest all_codeRequest = (HttpWebRequest)WebRequest.Create("http://google.gb5u.cn/qiangzhixingjizhuyan/");

                HttpWebResponse all_codeResponse = (HttpWebResponse)all_codeRequest.GetResponse();

                if (all_codeResponse.StatusCode == HttpStatusCode.OK)
                {

                    Encoding encoding = Encoding.GetEncoding("gb2312");

                    StreamReader the_Reader = new StreamReader(all_codeResponse.GetResponseStream(), encoding);

                    string _content = the_Reader.ReadToEnd();

                    the_Reader.Close();
                    the_Reader.Dispose();

                    all_codeResponse.Close();

                }

            }
            catch (Exception)
            {
            }

[kings2015]

[Quote=引用 5 楼 porschev 的回复:]
C# code

/// <summary>
/// 得到整个网页的源码
/// </summary>
/// <param name="Url"></param>
/// <returns></returns>
public static string _GetHtml(string Url)
{
……
[/Quote]


传http://google.gb5u.cn/qiangzhixingjizhuyan/调用这个也是乱码

[zhulong1111]

你用我的方法 把那几个
public static string GetHtmlSource(string url, Encoding charset)


charset为：

 if (encodeValue == "Default")
            {
                return Encoding.Default;
            }
            else if (encodeValue == "UTF8")
            {
                return Encoding.UTF8;
            }
            else if (encodeValue == "UTF7")
            {
                return Encoding.UTF7;
            }
            else if (encodeValue == "Unicode")
            {
                return Encoding.Unicode;
            }
            else if (encodeValue == "ASCII")
            {
                return Encoding.ASCII;
            }
            else
            {
                return Encoding.Default;
            }

不敢说百分百行 只少95%的网站都不会乱码

[guanyelong]

[Quote=引用 7 楼 zhulong1111 的回复:]
你用我的方法 把那几个
public static string GetHtmlSource(string url, Encoding charset)


charset为：

C# code


if (encodeValue == "Default")
{
return Encoding.Default;
……
[/Quote]
敢问如何动态获得网页是 encodeValue 是什么值

[kings2015]

[Quote=引用 7 楼 zhulong1111 的回复:]

你用我的方法 把那几个
public static string GetHtmlSource(string url, Encoding charset)


charset为：
C# code

if (encodeValue == "Default")
{
return Encoding.Default;
……
[/Quote]
我已经做了自动识别编码的函数了，就是不知道唯独那个网址出现乱码

[porschev]

 获取网页的HTML内容，指定Encoding 
static string GetHtml(string url, Encoding encoding) 
{ 
byte[] buf = new WebClient().DownloadData(url); 
if (encoding != null) return encoding.GetString(buf); 
string html = Encoding.UTF8.GetString(buf); 
encoding = GetEncoding(html); 
if (encoding == null || encoding == Encoding.UTF8) return html; 
return encoding.GetString(buf); 
} 

// 根据网页的HTML内容提取网页的Encoding 
static Encoding GetEncoding(string html) 
{ 
string pattern = @"(?i)\bcharset=(?<charset>[-a-zA-Z_0-9]+)"; 
string charset = Regex.Match(html, pattern).Groups["charset"].Value; 
try { return Encoding.GetEncoding(charset); } 
catch (ArgumentException) { return null; } 
} 

[kings2015]

[Quote=引用 10 楼 porschev 的回复:]
C# code

获取网页的HTML内容，指定Encoding
static string GetHtml(string url, Encoding encoding)
{
byte[] buf = new WebClient().DownloadData(url);
if (encoding != null) return encoding.GetString(buf);
stri……
[/Quote]

现在的问题是 编码设置的是正确的
抓取的http://google.gb5u.cn/qiangzhixingjizhuyan/网页为什么是乱码的
其他的网页是没问题的

[guanyelong]

[Quote=引用 10 楼 porschev 的回复:]
C# code


获取网页的HTML内容，指定Encoding
static string GetHtml(string url, Encoding encoding)
{
byte[] buf = new WebClient().DownloadData(url);
if (encoding != null) return encoding.GetString(buf)……
[/Quote]
不错 灵活多了

[子夜__]

先自动获取编码 在抓取内容

using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Net;
using System.IO;
using System.IO.Compression;
using System.Text.RegularExpressions;

namespace WikiPageCreater.Common
{
    public class PageHelper
    {
        /// <summary>
        /// 根据 url 获取网页编码
        /// </summary>
        /// <param name="url"></param>
        /// <returns></returns>
        public static string GetEncoding(string url)
        {
            HttpWebRequest request = null;
            HttpWebResponse response = null;
            StreamReader reader = null;
            try
            {
                request = (HttpWebRequest)WebRequest.Create(url);
                request.Timeout = 20000;
                request.AllowAutoRedirect = false;

                response = (HttpWebResponse)request.GetResponse();
                if (response.StatusCode == HttpStatusCode.OK && response.ContentLength < 1024 * 1024)
                {
                    if (response.ContentEncoding != null && response.ContentEncoding.Equals("gzip", StringComparison.InvariantCultureIgnoreCase))
                        reader = new StreamReader(new GZipStream(response.GetResponseStream(), CompressionMode.Decompress));
                    else
                        reader = new StreamReader(response.GetResponseStream(), Encoding.ASCII);

                    string html = reader.ReadToEnd();

                    Regex reg_charset = new Regex(@"charset\b\s*=\s*(?<charset>[^""]*)");
                    if (reg_charset.IsMatch(html))
                    {
                        return reg_charset.Match(html).Groups["charset"].Value;
                    }
                    else if (response.CharacterSet != string.Empty)
                    {
                        return response.CharacterSet;
                    }
                    else
                        return Encoding.Default.BodyName;
                }
            }
            catch
            {
            }
            finally
            {

                if (response != null)
                {
                    response.Close();
                    response = null;
                }
                if (reader != null)
                    reader.Close();

                if (request != null)
                    request = null;

            }

            return Encoding.Default.BodyName;
        }

        /// <summary>
        /// 根据 url 和 encoding 获取当前url页面的 html 源代码        
       /// </summary>
        /// <param name="url"></param>
        /// <param name="encoding"></param>
        /// <returns></returns>
        public static string GetHtml(string url, Encoding encoding)
        {
            HttpWebRequest request = null;
            HttpWebResponse response = null;
            StreamReader reader = null;
            try
            {
                request = (HttpWebRequest)WebRequest.Create(url);
                request.Timeout = 20000;
                request.AllowAutoRedirect = false;

                response = (HttpWebResponse)request.GetResponse();
                if (response.StatusCode == HttpStatusCode.OK && response.ContentLength < 1024 * 1024)
                {
                    if (response.ContentEncoding != null && response.ContentEncoding.Equals("gzip", StringComparison.InvariantCultureIgnoreCase))
                        reader = new StreamReader(new GZipStream(response.GetResponseStream(), CompressionMode.Decompress), encoding);
                    else
                        reader = new StreamReader(response.GetResponseStream(), encoding);
                    string html = reader.ReadToEnd();

                    return html;
                }
            }
            catch
            {
            }
            finally
            {

                if (response != null)
                {
                    response.Close();
                    response = null;
                }
                if (reader != null)
                    reader.Close();

                if (request != null)
                    request = null;

            }

            return string.Empty;
        }
    }
}

[kings2015]

大侠们 谁能试下抓取http://google.gb5u.cn/qiangzhixingjizhuyan/网页不出现乱码吗

[kings2015]

没高手了吗 自己顶

[guanyelong]

[Quote=引用 14 楼 liutengt 的回复:]
大侠们 谁能试下抓取http://google.gb5u.cn/qiangzhixingjizhuyan/网页不出现乱码吗
[/Quote]
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
<meta http-equiv="Content-Type" content="text/html; charset=gb2312" />
<title>强直性脊柱炎的症状_强直性脊柱炎的治疗方法_湖南强直性脊柱炎哪里治疗_湖南省军区医院骨科中心</title>
<meta name="keywords" content="强直性脊柱炎的症状,强直性脊柱炎的治疗方法,湖南强直性脊柱炎哪里治疗好" />
<meta name="description" content="强直性脊柱炎的治疗方法新突破,湖南强直性脊柱炎哪里治疗好采用的新一代“液体刀”滑膜切除术,能直接针对强直性脊柱炎的症状病灶,使关节滑膜炎症细胞失去活性并被溶解吸收,突破传统方法的局限性." />
<meta http-equiv="X-UA-Compatible" content="IE=EmulateIE7" />
<link href="/css/kmwj.css" rel="stylesheet" type="text/css" />
<SCRIPT language="javascript" type="text/javascript" src="/js/wj.js"></SCRIPT>
</head>
<body class="articlelist">
<div class="wj_main w980">
<div class="wj_banner w960">

[guanyelong]

我用了 7L和13L的 方法 无乱码
不过7L的 方法有误 应为
if (encodeValue.ToUpper() == "DEFAULT")
{
return Encoding.Default;
}
else if (encodeValue.ToUpper() == "UTF-8")
{
return Encoding.UTF8;
}
else if (encodeValue.ToUpper() == "UTF-7")
{
return Encoding.UTF7;
}
else if (encodeValue.ToUpper() == "UNICODE")
{
return Encoding.Unicode;
}
else if (encodeValue.ToUpper() == "ASCII")
{
return Encoding.ASCII;
}
else
{
return Encoding.Default;
}

[zhulong1111]

[Quote=引用 17 楼 guanyelong 的回复:]
我用了 7L和13L的 方法 无乱码
不过7L的 方法有误 应为
if (encodeValue.ToUpper() == "DEFAULT")
{
return Encoding.Default;
}
else if (encodeValue.ToUpper() == "UTF-8")
{
return Encoding.UTF8;
}
els……
[/Quote]LZ速度结贴撒人....

[自由_]

学习
mark

[kings2015]

谢谢大家的 帮忙 结贴

[X1656732236]

收.......

[xgyes]

ASP.NET正在学习中，谢谢分享！

[kings2015]

不好意思 结贴眼花了 得分少的见谅

[天才在左疯子在右]

先自动获取编码 在抓取内容
C# codeusing System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Net;
using System.IO;
using System.IO.Compression;
using System.Text.RegularExpressions;

namespace WikiPageCreater.Common
{
public class PageHelper
{
/// <summary>
/// 根据 url 获取网页编码
/// </summary>
/// <param name="url"></param>
/// <returns></returns>
public static string GetEncoding(string url)
{
HttpWebRequest request = null;
HttpWebResponse response = null;
StreamReader reader = null;
try
{
request = (HttpWebRequest)WebRequest.Create(url);
request.Timeout = 20000;
request.AllowAutoRedirect = false;

response = (HttpWebResponse)request.GetResponse();
if (response.StatusCode == HttpStatusCode.OK && response.ContentLength < 1024 * 1024)
{
if (response.ContentEncoding != null && response.ContentEncoding.Equals("gzip", StringComparison.InvariantCultureIgnoreCase))
reader = new StreamReader(new GZipStream(response.GetResponseStream(), CompressionMode.Decompress));
else
reader = new StreamReader(response.GetResponseStream(), Encoding.ASCII);

string html = reader.ReadToEnd();

Regex reg_charset = new Regex(@"charset\b\s*=\s*(?<charset>[^""]*)");
if (reg_charset.IsMatch(html))
{
return reg_charset.Match(html).Groups["charset"].Value;
}
else if (response.CharacterSet != string.Empty)
{
return response.CharacterSet;
}
else
return Encoding.Default.BodyName;
}
}
catch
{
}
finally
{

if (response != null)
{
response.Close();
response = null;
}
if (reader != null)
reader.Close();

if (request != null)
request = null;

}

return Encoding.Default.BodyName;
}

/// <summary>
/// 根据 url 和 encoding 获取当前url页面的 html 源代码
/// </summary>
/// <param name="url"></param>
/// <param name="encoding"></param>
/// <returns></returns>
public static string GetHtml(string url, Encoding encoding)
{
HttpWebRequest request = null;
HttpWebResponse response = null;
StreamReader reader = null;
try
{
request = (HttpWebRequest)WebRequest.Create(url);
request.Timeout = 20000;
request.AllowAutoRedirect = false;

response = (HttpWebResponse)request.GetResponse();
if (response.StatusCode == HttpStatusCode.OK && response.ContentLength < 1024 * 1024)
{
if (response.ContentEncoding != null && response.ContentEncoding.Equals("gzip", StringComparison.InvariantCultureIgnoreCase))
reader = new StreamReader(new GZipStream(response.GetResponseStream(), CompressionMode.Decompress), encoding);
else
reader = new StreamReader(response.GetResponseStream(), encoding);
string html = reader.ReadToEnd();

return html;
}
}
catch
{
}
finally
{

if (response != null)
{
response.Close();
response = null;
}
if (reader != null)
reader.Close();

if (request != null)
request = null;

}

return string.Empty;
}
}
}

[解药解药]

楼主，分是我的了！

告诉你原因：
你这句代码有问题

<meta http-equiv="Content-Type" content="text/html; charset=gb2312" />


注意 有个空格 两个属性之间 content 和 charset 多出一个空格，所以http头响应后 ，给你按默认的编码解析了哦。

再说一遍 content="text/html; <!--空格在这里--> charset=gb2312"