图片采集，你懂的～

2012-04-22 09:59:12 1.1k 阅读 AI 写同款 · GEO 优化AI 选题数据监测一键分发 ›

string ListUrl = "http://XXXX.cc/html/yazhou/index{0}.html";

            System.Net.WebClient wc = new System.Net.WebClient();

            HtmlAgilityPack.HtmlDocument listdoc = new HtmlAgilityPack.HtmlDocument();

            HtmlAgilityPack.HtmlNode listbody=null;



            HtmlAgilityPack.HtmlDocument cotentdoc = new HtmlAgilityPack.HtmlDocument();

            HtmlAgilityPack.HtmlNode cotentbody = null;

            for (int i = 1; i <= 33; i++)

            {

                string tempList = string.Format(ListUrl, i == 1 ? "" : i.ToString());

                string htmlStr = wc.DownloadString(tempList);

                if (htmlStr == null) continue;

                listdoc.LoadHtml(htmlStr);

                listbody = listdoc.DocumentNode;

                HtmlAgilityPack.HtmlNodeCollection lists = listbody.SelectNodes("//div/ul/li");

                foreach (HtmlNode list in lists)

                {

                    var listname = list.SelectSingleNode("./a").Attributes["title"].Value;

                    var contenturl = "http://XXXX.cc/" + list.SelectSingleNode("./a").Attributes["href"].Value;



                    try

                    {

                        int end = 1;

                        int tonum=1;

                        for (int m = 1; m <= end; m++)

                        {

                         

                            string willdir = @"D:\temp\1\" + listname;

                            string contentHtml = null;

                           

                            if (m == 2)

                            {

                                contenturl = contenturl.Substring(0, contenturl.LastIndexOf(".htm")) + "_{0}.htm";

                            }

                            if (m >= 2)

                            {

                                contentHtml = wc.DownloadString(string.Format(contenturl, m.ToString()));

                            }

                            else

                            {

                                contentHtml = wc.DownloadString(contenturl);

                            }

                            cotentdoc.LoadHtml(contentHtml);

                            if (cotentdoc == null) continue;

                            cotentbody = cotentdoc.DocumentNode;

                            if (m == 1)

                            {

                                end = int.Parse(cotentbody.SelectSingleNode("//div[@class='pp']/font[@color='red']").InnerHtml.Trim());

                                System.IO.Directory.CreateDirectory(willdir);

                            }

                            var contentlists = cotentbody.SelectNodes("//div[@class='pp']//img");

                            foreach( var n in contentlists)

                            {

                                var picname = n.Attributes["src"].Value;

                                wc.DownloadFile(picname, willdir + "\\" + tonum.ToString() + ".jpg");

                                tonum++;

                            }

                        }

                    }

                    catch

                    {

                        continue;

                    }

                }

            }

...全文

python批量采集美女图片数据~这不得看一下前言嗨喽！大家好呀，这里是魔王~ “”" 目标网站：内容：图片 “”" # 分金定穴 --- 分析网站 # 数据加载: # 静态加载 # 动态加载 json # -关键字 # -页码 import json import urllib.parse import requests # 网络请求库 pip import jsonpath # json数据解析库 import threading #设置最大线程锁 thread_lock = threading.BoundedS 阅读详情

tecnology

C# 111,129 成员 642,538 内容

+ 关注 + 加入社区

切换为时间正序

yy2013666 2012-12-04

打赏
举报

using System; using System.Collections.Generic; using System.ComponentModel; using System.Data; using System.Drawing; using System.Text; using System.Windows.Forms; using HtmlAgilityPack; using System.Threading; using System.Collections; namespace AutoPicDownLoad { 应用我的http://www.ymrt.net不知是否可以？

tecnology 2012-05-05

打赏
举报

需下载HtmlAgilityPack 第三方DLL~

tecnology 2012-05-05

打赏
举报



using System;

using System.Collections.Generic;

using System.ComponentModel;

using System.Data;

using System.Drawing;

using System.Text;

using System.Windows.Forms;

using HtmlAgilityPack;

using System.Threading;

using System.Collections;

namespace AutoPicDownLoad

{

    public partial class frmMain : Form

    {

        public frmMain()

        {

            InitializeComponent();

            bk.DoWork += new DoWorkEventHandler(bk_DoWork);

            bk.RunWorkerCompleted += new RunWorkerCompletedEventHandler(bk_RunWorkerCompleted);

            bk.WorkerSupportsCancellation = true;

        }

        BackgroundWorker bk = new BackgroundWorker();

        public static object wobj = new object();

        void bk_RunWorkerCompleted(object sender, RunWorkerCompletedEventArgs e)

        {

            this.Close();

        }

        static void PooledFunc(object state)

        {

            AutoResetEvent are = (AutoResetEvent)state;

            System.Net.WebClient wc = new System.Net.WebClient();

            HtmlAgilityPack.HtmlDocument listdoc = new HtmlAgilityPack.HtmlDocument();

            HtmlAgilityPack.HtmlNode listbody = null;

            HtmlAgilityPack.HtmlDocument cotentdoc = new HtmlAgilityPack.HtmlDocument();

            HtmlAgilityPack.HtmlNode cotentbody = null;

            string ListUrl = GetListUrl;

            if (ListUrl == null)

            {

                are.Set();

                return;

            }

            string htmlStr = wc.DownloadString(ListUrl.ToString());

            if (htmlStr == null) 

            {

                are.Set();

                return; 

            }

            listdoc.LoadHtml(htmlStr);

            listbody = listdoc.DocumentNode;

            HtmlAgilityPack.HtmlNodeCollection lists = listbody.SelectNodes("//div/ul/li");

            foreach (HtmlNode list in lists)

            {

                var listname = list.SelectSingleNode("./a").Attributes["title"].Value;

                var contenturl = "http://ccrt.cc" + list.SelectSingleNode("./a").Attributes["href"].Value;

                try

                {

                    int end = 1;

                    int tonum = 1;

                    for (int m = 1; m <= end; m++)

                    {



                        string willdir = @"D:\temp\1\" + listname;

                        string contentHtml = null;

                        if (m == 2)

                        {

                            contenturl = contenturl.Substring(0, contenturl.LastIndexOf(".htm")) + "_{0}.htm";

                        }

                        string tempContentUrl = string.Format(contenturl, m.ToString());

                        if (m >= 2)

                        {

                            contentHtml = wc.DownloadString(tempContentUrl);

                        }

                        else

                        {

                            contentHtml = wc.DownloadString(contenturl);

                        }

                        cotentdoc.LoadHtml(contentHtml);

                        if (cotentdoc == null) continue;

                        cotentbody = cotentdoc.DocumentNode;

                        if (m == 1)

                        {

                            end = int.Parse(cotentbody.SelectSingleNode("//div[@class='pp']/font[@color='red']").InnerHtml.Trim());

                            System.IO.Directory.CreateDirectory(willdir);

                        }

                        var contentlists = cotentbody.SelectNodes("//div[@class='pp']//img");

                        foreach (var n in contentlists)

                        {

                            var picname = n.Attributes["src"].Value;

                            if (System.IO.File.Exists(willdir + "\\" + tonum.ToString() + ".jpg"))

                            {

                                tonum++;

                                continue;

                            }

                            wc.DownloadFile(picname, willdir + "\\" + tonum.ToString() + ".jpg");

                            tonum++;

                        }

                    }

                }

                catch

                {

                    continue;

                }

                finally

                {

                }

            }

            are.Set();

        }

        //开启5个线程分别下载

        static WaitHandle[] waitHandles = new WaitHandle[5] { new AutoResetEvent(false), new AutoResetEvent(false), new AutoResetEvent(false), new AutoResetEvent(false), new AutoResetEvent(false) };

        static System.Collections.Queue q = new System.Collections.Queue();

        public static object obj = new object();

        public static string GetListUrl

        {

            get

            {

                if (q.Count > 0)

                {

                    lock (obj)

                    {

                        if (q.Count > 0)

                        {

                            return q.Dequeue().ToString();

                        }

                    }

                }

                return null;

            }

        }

        void bk_DoWork(object sender, DoWorkEventArgs e)

        {



            string ListUrl = "http://ccrt.cc/html/yazhou/index{0}.html";

            for (int i = 17; i <= 33; i++)//i=4  列表第四页

            {

                string tempList = string.Format(ListUrl.ToString(), i == 1 ? "" : i.ToString());

                q.Enqueue(tempList);

            }

            WaitCallback callBack = new WaitCallback(PooledFunc);

            while (q.Count != 0)

            {

                WriteLine("线程池开始"+q.Count.ToString());

                for (int ccc = 0; ccc < waitHandles.Length; ccc++)

                {

                    ThreadPool.QueueUserWorkItem(callBack, waitHandles[ccc]);

                }

                WaitHandle.WaitAll(waitHandles);

                 WriteLine("线程池结束"+q.Count.ToString());

            }

        }

        public void WriteLine(string line)

        {

            using (System.IO.StreamWriter sw = new System.IO.StreamWriter(@"D:\temp\info.log"))

            {

                sw.WriteLine(line);

            }

        }

        private void btnStart_Click(object sender, EventArgs e)

        {

            this.btnStart.Enabled=false;

            bk.RunWorkerAsync();

        }

    }

}