111,097
社区成员
发帖
与我相关
我的任务
分享图片采集，你懂的～
string ListUrl = "http://XXXX.cc/html/yazhou/index{0}.html";
System.Net.WebClient wc = new System.Net.WebClient();
HtmlAgilityPack.HtmlDocument listdoc = new HtmlAgilityPack.HtmlDocument();
HtmlAgilityPack.HtmlNode listbody=null;
HtmlAgilityPack.HtmlDocument cotentdoc = new HtmlAgilityPack.HtmlDocument();
HtmlAgilityPack.HtmlNode cotentbody = null;
for (int i = 1; i <= 33; i++)
{
string tempList = string.Format(ListUrl, i == 1 ? "" : i.ToString());
string htmlStr = wc.DownloadString(tempList);
if (htmlStr == null) continue;
listdoc.LoadHtml(htmlStr);
listbody = listdoc.DocumentNode;
HtmlAgilityPack.HtmlNodeCollection lists = listbody.SelectNodes("//div/ul/li");
foreach (HtmlNode list in lists)
{
var listname = list.SelectSingleNode("./a").Attributes["title"].Value;
var contenturl = "http://XXXX.cc/" + list.SelectSingleNode("./a").Attributes["href"].Value;
try
{
int end = 1;
int tonum=1;
for (int m = 1; m <= end; m++)
{
string willdir = @"D:\temp\1\" + listname;
string contentHtml = null;
if (m == 2)
{
contenturl = contenturl.Substring(0, contenturl.LastIndexOf(".htm")) + "_{0}.htm";
}
if (m >= 2)
{
contentHtml = wc.DownloadString(string.Format(contenturl, m.ToString()));
}
else
{
contentHtml = wc.DownloadString(contenturl);
}
cotentdoc.LoadHtml(contentHtml);
if (cotentdoc == null) continue;
cotentbody = cotentdoc.DocumentNode;
if (m == 1)
{
end = int.Parse(cotentbody.SelectSingleNode("//div[@class='pp']/font[@color='red']").InnerHtml.Trim());
System.IO.Directory.CreateDirectory(willdir);
}
var contentlists = cotentbody.SelectNodes("//div[@class='pp']//img");
foreach( var n in contentlists)
{
var picname = n.Attributes["src"].Value;
wc.DownloadFile(picname, willdir + "\\" + tonum.ToString() + ".jpg");
tonum++;
}
}
}
catch
{
continue;
}
}
}
...全文
请发表友善的回复…
发表回复
yy2013666 2012-12-04
- 打赏
- 举报
using System; using System.Collections.Generic; using System.ComponentModel; using System.Data; using System.Drawing; using System.Text; using System.Windows.Forms; using HtmlAgilityPack; using System.Threading; using System.Collections; namespace AutoPicDownLoad { 应用我的http://www.ymrt.net不知是否可以？
tecnology 2012-05-05
- 打赏
- 举报
需下载HtmlAgilityPack 第三方DLL~
tecnology 2012-05-05
- 打赏
- 举报
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.Text;
using System.Windows.Forms;
using HtmlAgilityPack;
using System.Threading;
using System.Collections;
namespace AutoPicDownLoad
{
public partial class frmMain : Form
{
public frmMain()
{
InitializeComponent();
bk.DoWork += new DoWorkEventHandler(bk_DoWork);
bk.RunWorkerCompleted += new RunWorkerCompletedEventHandler(bk_RunWorkerCompleted);
bk.WorkerSupportsCancellation = true;
}
BackgroundWorker bk = new BackgroundWorker();
public static object wobj = new object();
void bk_RunWorkerCompleted(object sender, RunWorkerCompletedEventArgs e)
{
this.Close();
}
static void PooledFunc(object state)
{
AutoResetEvent are = (AutoResetEvent)state;
System.Net.WebClient wc = new System.Net.WebClient();
HtmlAgilityPack.HtmlDocument listdoc = new HtmlAgilityPack.HtmlDocument();
HtmlAgilityPack.HtmlNode listbody = null;
HtmlAgilityPack.HtmlDocument cotentdoc = new HtmlAgilityPack.HtmlDocument();
HtmlAgilityPack.HtmlNode cotentbody = null;
string ListUrl = GetListUrl;
if (ListUrl == null)
{
are.Set();
return;
}
string htmlStr = wc.DownloadString(ListUrl.ToString());
if (htmlStr == null)
{
are.Set();
return;
}
listdoc.LoadHtml(htmlStr);
listbody = listdoc.DocumentNode;
HtmlAgilityPack.HtmlNodeCollection lists = listbody.SelectNodes("//div/ul/li");
foreach (HtmlNode list in lists)
{
var listname = list.SelectSingleNode("./a").Attributes["title"].Value;
var contenturl = "http://ccrt.cc" + list.SelectSingleNode("./a").Attributes["href"].Value;
try
{
int end = 1;
int tonum = 1;
for (int m = 1; m <= end; m++)
{
string willdir = @"D:\temp\1\" + listname;
string contentHtml = null;
if (m == 2)
{
contenturl = contenturl.Substring(0, contenturl.LastIndexOf(".htm")) + "_{0}.htm";
}
string tempContentUrl = string.Format(contenturl, m.ToString());
if (m >= 2)
{
contentHtml = wc.DownloadString(tempContentUrl);
}
else
{
contentHtml = wc.DownloadString(contenturl);
}
cotentdoc.LoadHtml(contentHtml);
if (cotentdoc == null) continue;
cotentbody = cotentdoc.DocumentNode;
if (m == 1)
{
end = int.Parse(cotentbody.SelectSingleNode("//div[@class='pp']/font[@color='red']").InnerHtml.Trim());
System.IO.Directory.CreateDirectory(willdir);
}
var contentlists = cotentbody.SelectNodes("//div[@class='pp']//img");
foreach (var n in contentlists)
{
var picname = n.Attributes["src"].Value;
if (System.IO.File.Exists(willdir + "\\" + tonum.ToString() + ".jpg"))
{
tonum++;
continue;
}
wc.DownloadFile(picname, willdir + "\\" + tonum.ToString() + ".jpg");
tonum++;
}
}
}
catch
{
continue;
}
finally
{
}
}
are.Set();
}
//开启5个线程分别下载
static WaitHandle[] waitHandles = new WaitHandle[5] { new AutoResetEvent(false), new AutoResetEvent(false), new AutoResetEvent(false), new AutoResetEvent(false), new AutoResetEvent(false) };
static System.Collections.Queue q = new System.Collections.Queue();
public static object obj = new object();
public static string GetListUrl
{
get
{
if (q.Count > 0)
{
lock (obj)
{
if (q.Count > 0)
{
return q.Dequeue().ToString();
}
}
}
return null;
}
}
void bk_DoWork(object sender, DoWorkEventArgs e)
{
string ListUrl = "http://ccrt.cc/html/yazhou/index{0}.html";
for (int i = 17; i <= 33; i++)//i=4 列表第四页
{
string tempList = string.Format(ListUrl.ToString(), i == 1 ? "" : i.ToString());
q.Enqueue(tempList);
}
WaitCallback callBack = new WaitCallback(PooledFunc);
while (q.Count != 0)
{
WriteLine("线程池开始"+q.Count.ToString());
for (int ccc = 0; ccc < waitHandles.Length; ccc++)
{
ThreadPool.QueueUserWorkItem(callBack, waitHandles[ccc]);
}
WaitHandle.WaitAll(waitHandles);
WriteLine("线程池结束"+q.Count.ToString());
}
}
public void WriteLine(string line)
{
using (System.IO.StreamWriter sw = new System.IO.StreamWriter(@"D:\temp\info.log"))
{
sw.WriteLine(line);
}
}
private void btnStart_Click(object sender, EventArgs e)
{
this.btnStart.Enabled=false;
bk.RunWorkerAsync();
}
}
}
nvshenxp 2012-04-23
- 打赏
- 举报
菜鸟路过，这个真不懂。。。
在坚持几年 2012-04-23
- 打赏
- 举报
lists对象为null
在坚持几年 2012-04-23
- 打赏
- 举报
啥玩意呀，报错
LOVE_GG 2012-04-23
- 打赏
- 举报
lz，你这个帖子发得.....
tecnology 2012-04-22
- 打赏
- 举报
如果弄成多线程的会更好吧。呵呵
pmars 2012-04-22
- 打赏
- 举报
what are you doing?
