600分求问，如何取得网页源码？

[Areslee]

两个问题，一个300分，不够另开贴给分
1、如何不通过浏览器编程从网络中取得网页源码（不要求取得服务器端代码）？
2、如何编程从浏览器取得网页源码？
比如：我想自己写浏览器。

[Kudeet]

//write by masterz
bool SaveUrl(LPCTSTR url, LPCTSTR filename)
{
HINTERNET hNet = ::InternetOpen("Outlook",
PRE_CONFIG_INTERNET_ACCESS,
NULL,
INTERNET_INVALID_PORT_NUMBER,
0) ;

HINTERNET hUrlFile = ::InternetOpenUrl(hNet,
url,
NULL,
0,
INTERNET_FLAG_RELOAD,
0) ;

char buffer[10*1024] ;
DWORD dwBytesRead = 1;
BOOL bRead=TRUE;
CFile file;
file.Open(filename,CFile::modeCreate|CFile::modeWrite);
while(bRead&&dwBytesRead>0)
{
bRead = ::InternetReadFile(hUrlFile,
buffer,
sizeof(buffer),
&dwBytesRead);
if(dwBytesRead>0)
file.Write(buffer,dwBytesRead);
}
::InternetCloseHandle(hUrlFile) ;
::InternetCloseHandle(hNet) ;
file.Close();
AfxMessageBox("finished");
return bRead;
}
void CC02021101Dlg::OnOK()
{
// TODO: Add extra validation here
bool bret=SaveUrl("http://club.pchome.net/bbs2.php?topic=40&lanmuid=2","C:\\temp\\test.html");
if(bret)
AfxMessageBox("true");
else
AfxMessageBox("false");
}

[Kudeet]

#import "c:\program files\common files\system\ado\msado15.dll" no_namespace rename("EOF", "EndOfFile")
#import <cdosys.dll> no_namespace rename("EOF", "EndOfFile")
............
void CSavemhtDlg::OnOK()
{
// save url as a single file, in fact I don't know if it is mht file, but it can be opened by IE, can someone tell me?
CoInitialize(NULL);
{
IMessagePtr iMsg(__uuidof(Message));
IConfigurationPtr iConf(__uuidof(Configuration));
iMsg->Configuration = iConf;
try
{
iMsg->CreateMHTMLBody(
"http://example.microsoft.com",
cdoSuppressNone,
"domain\\username",
"password");
}
catch(_com_error err)
{
// handle exception
}
_StreamPtr pStream=iMsg->GetStream();
pStream->SaveToFile("test.mht",adSaveCreateOverWrite);
}
CoUninitialize();
}

[pomelowu]

2、如何编程从浏览器取得网页源码？

如果是重用WebBrowser Control，去MSDN里看“Loading HTML content from a Stream”吧。
JS翻译了下：
http://dev.csdn.net/develop/article/18/18465.shtm

[Kudeet]

使用 CHtmlView::GetHtmlDocument 得到html的源文件, CHtmlView的内部实现也是用WebBrower控件的，你自己写浏览器使用这个控件也可以同样用：


IHTMLDocument2* pHtmlDoc2 = (IHTMLDocument2*)GetHtmlDocument();
//check if HtmlDocument initialized
if( pHtmlDoc2 != NULL)
{

IHTMLDocument3* pHTMLDoc3 = NULL;
HRESULT hr = pHtmlDoc2->QueryInterface(IID_IHTMLDocument3,
(LPVOID*)&pHTMLDoc3);
// ASSERT(SUCCEEDED(hr));
// {
if(pHTMLDoc3)
{
IHTMLElement* pDocElem=NULL;
hr = pHTMLDoc3->get_documentElement(&pDocElem);
if(pDocElem)
{

// ASSERT(SUCCEEDED(hr));
BSTR bstrHTML;
pDocElem->get_outerHTML(&bstrHTML);
pDocElem->Release();

USES_CONVERSION;
MessageBox(OLE2T(bstrHTML), _T("源文件"));
SysFreeString(bstrHTML);
}
pHTMLDoc3->Release();
}

pHtmlDoc2->Release();
pHtmlDoc2=NULL;
}

[Kudeet]

#import "msxml.dll"
#import "msxml2.dll"

using namespace MSXML2;
int main(int argc, char* argv[])
{
printf("Test of XMLHTTP by masterz!\n");
CoInitialize(NULL);
try
{
IXMLHTTPRequestPtr xmlrequest;
xmlrequest.CreateInstance("Msxml2.XMLHTTP");
_variant_t varp(false);
xmlrequest->open(_bstr_t("GET"),_bstr_t("http://www.csdn.net/expert/topic/855/855052.xml?temp=.176037"),varp);
xmlrequest->send();
BSTR bstrbody;
xmlrequest->get_responseText(&bstrbody);
_bstr_t bstrtbody(bstrbody);
printf("%s\n",(LPCTSTR)bstrtbody);

}
catch (_com_error &e)
{
printf("Description = '%s'\n", (char*) e.Description());
}
CoUninitialize();
printf("program end\n");
return 0;
}

[he_sl]

ding

[msvcp60]

简单，看我的代码
要记得包含
#include "afxinet.h"

CInternetSession mySession(NULL,0);
CHttpFile* myHttpFile=NULL;

try
{
CString m_SiteInfo="";//页面源码

CString myData;
myHttpFile=(CHttpFile*)mySession.OpenURL("http://www.xxx.xxx/index.html");
while(myHttpFile->ReadString(myData))
{
m_SiteInfo=m_SiteInfo+"\r\n";
m_SiteInfo+=myData;
}
AfxMessageBox(m_SiteInfo); //显示页面源码
myHttpFile->Close();
mySession.Close();
}

catch (...)
{
myHttpFile->Close();
mySession.Close();
}

[Areslee]

谢谢楼上各位，兄弟我先看一下就回来给分

[Areslee]

了解。那再问一下，如何发送请求到服务器端呢？
比如：网页上要求输入一段字符或是按某个键，或者点击某个链接。

[pripor]

碰上css呢？

[antijpn]

LZ真大方……

最简单的是用curl/libcurl，这个东西比CInternetSession强大多了

最简单的用curl命令行curl http://www.microsoft.com，这样就可以抓m$的首页了

发送请求你就需要解析网页才行了，一般来说大多数请求都是POST和GET的，譬如form很多都是POST的（也有GET的，但是相对来说要少得多），而link，一般来说（相对于会在link上面加个onclick的BT来说的一般）都用的是GET方法。这个都是设置http header就可以搞定了。

另外要注意的就是cookie的处理，一般来说对于论坛之类的东西，都有cookie的问题，要注意保留cookie，否则你会在登陆以后仍然不能访问其他页面。

[antijpn]

哦，对了还要注意UserAgent的设置，有的地方可能写的代码有问题（或者专门有设置），最好把你的UserAgent设置为IE/Firefox的名称

[xuzheng318]

http://blog.donews.com/ufoace/archive/2005/05/18/386912.aspx

[Areslee]

兄弟偶是个网络菜鸟，说白了问这个问题就是想知道如果做一个浏览器
比如：
1、如何联接网页/取得网页数据（这个楼上各位已经回答了）
2、如何发送交互信息，比如：按某键后怎么向服务端通信，在文本框中输入了信息怎么向服务端通信，怎么导向一个新的超级链接
3、怎么实现previous和next
4、怎么处理COOKIES
非常感谢楼上的各位网络大狼继续教诲。
再次感谢楼上热情回复的各位！

[Kudeet]

你是想自己写一个类似WebBrowser的东西？

[Areslee]

写不敢当，也就是学习一下IE/firefox之类的东西的基本原理，了解一下运行流程而已。

[yjje]

#include <stdio.h>
#include <afxinet.h>

#define MAXBLOCKSIZE 1024

void download(const char*);

int main(int argc, char* argv[]) {
if(argc > 1) {
download((const char*)argv[1]);
} else {
printf("Usage: auto-Update url");
}
return 0;
}
/**
* 执行 文件下载 操作
Url
*
*/
void download(const char *Url)
{
printf("\n开始下载: %s\n",Url);

HINTERNET hSession = InternetOpen("RookIE/1.0", INTERNET_OPEN_TYPE_PRECONFIG, NULL, NULL, 0);

if (hSession != NULL)
{
HINTERNET handle2 = InternetOpenUrl(hSession, Url, NULL, 0, INTERNET_FLAG_DONT_CACHE, 0);
if (handle2 != NULL)
{
printf("%s\n",Url);
BYTE Temp[MAXBLOCKSIZE];
ULONG Number = 1;

FILE *stream;
if( (stream = fopen( "dow/My.exe", "wb" )) != NULL )
{
while (Number > 0)
{
InternetReadFile(handle2, Temp, MAXBLOCKSIZE - 1, &Number);
printf("%s",Temp);
fwrite(Temp, sizeof (char), Number , stream);
}
fclose( stream );
printf("\n下载完成\n\n");
}
InternetCloseHandle(handle2);
handle2 = NULL;
}
InternetCloseHandle(hSession);
hSession = NULL;
}

}

[Kudeet]

比如：网页上要求输入一段字符或是按某个键，或者点击某个链接。
==============
///////////////////////////////////////////////////////////////////////////////////////
//click submit button of IE window
//If it works, it is written by masterz,otherwise I don't
//know who writes it^_^
///////////////////////////////////////////////////////////////////////////////////////
void CGetIESrcDlg::NavigateToUrl()
{
// Import the following files in your stdafx.h
// #import <mshtml.tlb> // Internet Explorer 5
// #import <shdocvw.dll>
// Refer to "Connect to Internet Explorer Instances, From your own Process. " in www.codeguru.com
SHDocVw::IShellWindowsPtr m_spSHWinds;
CoInitialize(NULL);
if(m_spSHWinds.CreateInstance(__uuidof(SHDocVw::ShellWindows)) == S_OK)
{
IDispatchPtr spDisp;
long nCount = m_spSHWinds->GetCount();
for (long i = 0; i < nCount; i++)
{
_variant_t va(i, VT_I4);
spDisp = m_spSHWinds->Item(va);
SHDocVw::IWebBrowser2Ptr spBrowser(spDisp);
if (spBrowser != NULL)
{
IDispatchPtr spDisp;
if(spBrowser->get_Document(&spDisp) == S_OK && spDisp!= 0 )
{
MSHTML::IHTMLDocument2Ptr spHtmlDocument(spDisp);
MSHTML::IHTMLElementPtr spHtmlElement;
if(spHtmlDocument==NULL)
continue;
spHtmlDocument->get_body(&spHtmlElement);
if(spHtmlDocument==NULL)
continue;
HRESULT hr;
MSHTML::IHTMLElementCollection* pColl=NULL;
hr=spHtmlDocument->get_all(&pColl);
if(pColl!=NULL&&SUCCEEDED(hr))
{
MSHTML::IHTMLElement* pElem=NULL;
_variant_t index;
index.vt=VT_I4;
index.intVal=0;
_variant_t name("Submit");
IDispatchPtr disp;
disp=pColl->item(name,index);
if(disp==NULL)
hr=E_FAIL;
else
{
hr=disp->QueryInterface(&pElem);
}
if (SUCCEEDED(hr)&& pElem != NULL)
{
//
BSTR bstrhtml;
pElem->get_outerHTML(&bstrhtml);
CString str(bstrhtml);
AfxMessageBox(str);
pElem->click();
pElem->Release();
}
pColl->Release();
}
}

}
}

}
else {
AfxMessageBox("Shell Windows interface is not avilable");
}
CoUninitialize();
}

[Kudeet]

如何给网页Post数据？用SDK
///////////////////////////////////////////////////////////////////////////
//SDK post
///////////////////////////////////////////////////////////////////////////
#include "stdafx.h"
#include "winsock.h"
#pragma comment(lib,"ws2_32.lib")
#define winsock_version 0x0101
void main()
{
//I create C:\Inetpub\wwwroot\test\test.asp ,start the web service
//start my program, the result is OK.
//If it works,it is written by masterz,otherwise I don't know who write it.
SOCKADDR_IN saServer;
LPHOSTENT lphostent;
WSADATA wsadata;
SOCKET hsocket;
int nRet;
const char* host_name="127.0.0.1";
char* req="POST /test/test.asp HTTP/1.0\r\n"
"From: local\r\n"
"User-Agent: post_test/1.0\r\n"
"Content-Type: application/x-www-form-urlencoded\r\n"
"Content-Length: 20\r\n\r\n"
"type=12345&name=aaaa";
if(WSAStartup(winsock_version,&wsadata))
printf("can't initial socket");
lphostent=gethostbyname(host_name);
if(lphostent==NULL)
printf("lphostent is null");
hsocket = socket(AF_INET, SOCK_STREAM, IPPROTO_TCP);
saServer.sin_family = AF_INET;
// Use def. now, need to handle general case
saServer.sin_port = htons(80);
saServer.sin_addr = *((LPIN_ADDR)*lphostent->h_addr_list);
nRet = connect(hsocket, (LPSOCKADDR)&saServer, sizeof(SOCKADDR_IN));
if (nRet == SOCKET_ERROR)
{
printf("can't connect");
closesocket(hsocket);
return;
}
else
printf("connected with %s\n",host_name);
nRet = send(hsocket, req, strlen(req), 0);
if (nRet == SOCKET_ERROR)
{
printf("send() failed");
closesocket(hsocket);

}
else
printf("send() OK\n");
char dest[1000];
nRet=1;
while(nRet>0)
{
nRet=recv(hsocket,(LPSTR)dest,sizeof(dest),0);
if(nRet>0)
dest[nRet]=0;
else
dest[0]=0;
printf("\nReceived bytes:%d\n",nRet);
printf("Result:\n%s",dest);
}
}

[e513479333]

可惜，600分