87,907
社区成员
发帖
与我相关
我的任务
分享
var MIN_STABLE_DAY = 2,
MIN_CACHE_DAY = 7,
MAX_ITEM = 2,
PATH_URL = 'url.txt',
PATH_LIST = '../asset/list.txt';
var webpage = require('webpage'),
fs = require('fs');
var site = [],
site_res = {},
used = {};
//
// load list
//
fs.read(PATH_URL).split('\n').forEach(function(line) {
line = line.trim();
if (!line || line.substr(0, 1) == '#') {
return;
}
site.push(line);
});
function go(url) {
var page = webpage.create(),
result = site_res[url] || (site_res[url] = []);
function ms2day(tick) {
return ~~(tick / (24 * 3600 * 1000));
}
page.onResourceReceived = function(response) {
if (response.url in used) {
return;
}
used[response.url] = true;
if (! /\.js/i.test(response.url)) {
return;
}
var last, now, exp;
var sec;
var i, headers = response.headers;
for(i = headers.length - 1; i >= 0; --i) {
var header = headers[i];
switch(header.name.toLowerCase()) {
case 'date':
now = new Date(header.value);
break;
case 'expires':
exp = new Date(header.value);
break;
case 'last-modified':
last = new Date(header.value);
break;
}
}
if (!exp || !last) return;
if (!now) now = new Date();
var dayStable = ms2day(now - last);
var dayCached = ms2day(exp - now);
if (dayStable < MIN_STABLE_DAY || dayCached < MIN_CACHE_DAY) {
return;
}
result.push({
url: response.url,
cache: dayCached,
stable: dayStable
});
};
function cb(status) {
if (tid != -1) {
clearTimeout(tid);
}
//
// sort and display
//
if (result.length > 0) {
result.sort(function(a, b){return a.stable - b.stable});
if (result.length > MAX_ITEM) {
result.length = MAX_ITEM;
}
console.log('==', url, '====================');
for(var i = 0; i < result.length; i++) {
var res = result[i];
console.log(-res.stable + ' / +' + res.cache + '\t\t' + res.url);
}
console.log(' ');
}
//
// breadth-first merge
//
if (++done == site.length) {
var loop, merge = [];
do {
loop = false;
for(var k in site_res) {
var e = site_res[k].pop();
if (e) {
loop = true;
merge.push( e.url.split('//')[1] );
}
}
} while(loop);
fs.write(PATH_LIST, merge.join('\n'));
console.log('DONE!');
}
}
var tid = setTimeout(function() {
page.close();
tid = -1;
cb();
}, 60 * 1000);
page.open('http://' + url, cb);
}
var done = 0;
function start() {
for(var i = 0; i < site.length; i++) {
setTimeout(function(url) {
go(url);
}, 2000 * i, site[i]);
}
}
start();
E:\NodeJS\AirDebug\tool>phantomjs sniffer.js
== www.hao123.com ====================
-5 / +360 http://s0.hao123img.com/v3/Mr/T0/gh/sn/61/6/hao123.js
-92 / +360 http://s0.hao123img.com/res/js/track.js?380890
== www.taobao.com ====================
-6 / +3650 http://a.tbcdn.cn/s/kissy/gallery/??search-suggest/1.0/index-min.js,search-suggest/1.0/plugin/history-min.js,search-suggest/1.
0/plugin/local-query-min.js,search-suggest/1.0/plugin/storage-min.js,search-suggest/1.0/plugin/tab-min.js,search-suggest/1.0/plugin/telephone-min.js?t
=20130606190449
-6 / +3650 http://a.tbcdn.cn/s/kissy/1.3.0/??node-min.js,dom/base-min.js,event/dom/base-min.js,event/base-min.js,event/dom/focusin-min.js
,anim-min.js,event/custom-min.js,switchable-min.js,cookie-min.js,ajax-min.js,json-min.js,rich-base-min.js,base-min.js,combobox-min.js,component/base-m
in.js,menu-min.js,component/extension-min.js,xtemplate/facade-min.js,xtemplate/runtime-min.js,xtemplate/compiler-min.js?t=20130606190449
== www.kaixin001.com ====================
-14 / +365 http://s.kaixin001.com.cn/js/apps/reg/ARegister-00100c612.js
-80 / +365 http://s.kaixin001.com.cn/js/core/media/MediaBox-0002a9159.js
== www.baidu.com ====================
-5 / +3650 http://s1.bdstatic.com/r/www/cache/static/user/js/u_75caac89.js
-5 / +3650 http://s1.bdstatic.com/r/www/cache/static/global/js/home_f949edf5.js
== www.baidu.com/s?wd=ss ====================
-5 / +3650 http://s1.bdstatic.com/r/www/cache/static/global/js/common_7fd3f7db.js
== www.renren.com ====================
-7 / +365 http://s.xnimg.cn/a56656/n/core/base-all2.js
-88 / +365 http://s.xnimg.cn/a53726/n/apps/login/login-v6.js
== tieba.baidu.com ====================
-4 / +30 http://tb1.bdstatic.com/tb/static-spage/component/feed_data/feed_data_c51ac7ba.js
-8 / +30 http://tb1.bdstatic.com/tb/static-common/js/tb_ui_ac13f64f.js
== map.baidu.com ====================
-7 / +365 http://webmap1.map.bdimg.com/init_0gj0re.js
-8 / +365 http://webmap2.map.bdimg.com/main_hntng4.js
== weibo.com ====================
-9 / +15 http://js.t.sinajs.cn/t5/register/js/page/login/index.js?version=201306141810
-30 / +15 http://js1.t.sinajs.cn/t4/apps/publicity/static/wbad.js?version=201306141810
== www.sina.com.cn ====================
-141 / +223 http://i2.sinaimg.cn/jslib/modules2/sina/util/1.0.5/util.js
-169 / +203 http://i2.sinaimg.cn/jslib/modules2/seajs/1.3.0/sea.js
== www.mop.com ====================
-365 / +300 http://mopimg.cn/tj/dcq.js
-427 / +300 http://mopimg.cn/dc/tj.js
== www.tianya.cn ====================
-38 / +29 http://static.tianyaui.com/global/ty/TY.js
== bbs.tianya.cn ====================
-7 / +29 http://static.tianyaui.com/global/lite/js/lite-all.js?v=201306070836
-15 / +25 http://static.tianyaui.com/global/lite/js/bbs/bbs.js?v=201306070836
== user.qzone.qq.com ====================
-31 / +7 http://imgcache.qq.com/ptlogin/ver/10031/js/h_login_11.js?max_age=604800&ptui_identifier=000D29EE4BA374D65D39E3C1BB890C8E50025
6813D2C4E549471141C
== www.163.com ====================
-32 / +90 http://img2.126.net/ntesrich/auto/indexU/dlbox-index-v1.0.1-130506.js
-51 / +90 http://img2.126.net/ntesrich/auto/indexU/fcbox-index-v1.0.0-130422.js
== www.sohu.com ====================
-42 / +90 http://js.sohu.com/pv/pvclick1211071116.js
-42 / +90 http://js.sohu.com/pv/spv1209061800.js
DONE!