利用C实现web信息自动抓取节能

背景

随着Internet的普及，络信息正以极高的速度增长，在这么多数据中找到自己需要的信息是一件很繁琐的事情，找到需要的信息后如何获取也是件麻烦的事。这就需要Internet信息抓取程序来代替人工的操作。

所谓Internet信息抓取程序，就是程序会按照用户的关键词或关键站来收集相应的信息，并提供给用户想要的信息格式。

信息量的增加会带来信息站发布人员工作量的剧增，为实现信息发布系统实现信息自

动发布、减少工作人员工作量、即时跟踪最新信息，就需要自动信息提供程序，因此Internet信息抓取程序应运而生。

目标

实现自定义站信息分类抓取，存入本地数据库、生成静态页面或其它用户定义的信息结构，并下载与信息相关的多媒体文件。

开发

目标站点结构分析

本步骤是准确抓取信息个关键。

首先要选择更新频率高的页面做为抓取地址，然后分析要抓取内容页面url特点。

然后分析要抓取信息页面的元素特性，比如标题位置，内容位置 等，得到定位标记点。

将以上信息写成自己的配置文件或存到数据库中。

每个站都需要分析，写出单独的配置文件，供抓取程序使用。

信息提取

根据配置文件取得要抓取页面url，使用HttpWebRequest类获取内容：

//获取http页面函数

public string Get_Http(string a_strUrl,int timeout)

{

string strResult ;

try

{

HttpWebRequest myReq = (HttpWebRequest)eate(a_strUrl) ;

meout = timeout;

HttpWebResponse HttpWResp = (HttpWebResponse)tResponse();

Stream myStream = tResponseStream () ;

StreamReader sr = new StreamReader(myStream , fault);

StringBuilder strBuilder = new StringBuilder();

while (-1 != ek())

{

pend(adLine()+"\r\n");

}

strResult = String();

}

catch(Exception exp)

{

strResult = "错误：" + ssage ;

}

return strResult ;

}

获取页面内容后，分析页面中连接地址取到要抓取的url：

//处理页面标题和链接

public string SniffWebUrl( string urlStr,string blockB,string blockE )

{

string urlch1 = "";

string urlch2 = "";

int end_n1 = 0;

int end_nums = 0;

int end_nums1 = 0;

int end_nums2 = 0;

int end_nums3 = 0;

string reUTStr = "";

string reTitle = "";

string ret = "";

try

{

int pos01 = dexOf( "." );

int pos02 = stIndexOf( "/" );

if( pos01 0 )

{

return "";

}

if( pos02 0 )

{

return "";

}

int pos03 = dexOf( "/",pos01 );

if ( pos03 0 )

{

urlch1 = urlStr;

urlch2 = urlStr;

}

else

{

urlch1 = bstring( 0,pos03 );

urlch2 = bstring( 0,pos02 );

}

string tmpAllStr = new PublicFun().Get_Http( urlStr ,time1);

int pos1 = dexOf( blockB );

int pos2 = dexOf( blockE,pos1 + ngth );

if ( pos10 pos20 pos2pos1 )

{

ret = bstring( pos1 + ngth,pos2 - pos1 - ngth );

ret = bstring( dexOf( "" ));

while( dexOf( "A" ) = 0 )

{

ret = bstring( 0,dexOf( "A" ) ) + "a" + bstring( dexOf( "A" ) + 2 );

}

while( dexOf( "/A" ) =0 )

{

ret = bstring( 0,dexOf( "/A" ) ) + "/a" + bstring( dexOf( "/A" ) + 3 );

}

while( dexOf( "Href=" ) =0 )

{

ret = bstring( 0,dexOf( "Href=" )) + "href=" + bstring( dexOf( "Href=" ) + 5 );

}

while( dexOf( "HREF=" ) =0 )

{

ret = bstring( 0,dexOf( "HREF=" )) + "href=" + bstring( dexOf( "HREF=" ) + 5 );

}

while( dexOf( "href='" ) =0 )

{

ret = bstring( 0,dexOf( "href='" )) + "href=\"" + bstring( dexOf( "href='" ) + 6 );

}

}

tmpAllStr = ret;

int begin_nums = dexOf( "href=" );

while ( begin_nums = 0 )

{

string tmpStrA = "";

string tmpStrB = bstring( begin_nums + 5,1 );

if ( tmpStrB == "\"" )

{

end_n1 = begin_nums + 6;

if ( ( end_n1 + 1 ) ngth )

{

return "";

}

tmpStrA = bstring( begin_nums+6,1 );

}

else

{

end_n1 = begin_nums + 5;

tmpStrA = tmpStrB;

}

if ( tmpStrA == "#" )

{

tmpAllStr = bstring( end_n1 );

begin_nums = dexOf( "href=" );

}

else

{

end_nums1 = dexOf( " ",end_n1 );

end_nums2 = dexOf( "",end_n1 );

end_nums3 = dexOf( "/a",end_nums2 );

if ( ( end_nums3 = 0 ) ( end_nums2 = 0 ) )

{

reTitle = bstring( end_nums2 + 1,end_nums3 - end_nums2 - 1 );

if ( end_nums1 end_nums2 )

{

end_nums = end_nums2;

}

else

{

if ( end_nums1 0 )

{

end_nums = end_nums2;

}

else

{

end_nums = end_nums1;

}

}

string str4 = bstring( end_nums-1, end_nums - end_nums + 1 );

if ( str4 =="\""|| str4 == "'" )

{

end_nums = end_nums - 1;

}

string sTotalOne = bstring( end_n1,end_nums - end_n1 );

if ( dexOf( "http://" ) 0 )

{

if ( dexOf( "/" ) == 0 )

{

sTotalOne = urlch1 + sTotalOne;

}

else

{

int linshiIntNum = 0;

int flags = 0;

string urlChange = urlStr;;

while( dexOf( "../" ) = 0 )

{

sTotalOne = bstring( dexOf( "../" ) + 3 );

linshiIntNum = linshiIntNum + 1;

flags = flags +1;

}

while( ( stIndexOf( "/" ) = 0 ) ( linshiIntNum = 0 ) )

{

urlChange = bstring( 0,stIndexOf( "/" ) );

linshiIntNum = linshiIntNum - 1;

}

if ( flags == 0 )

{

sTotalOne = urlch2 + "/" + sTotalOne;

}

else

{

sTotalOne = urlChange + "/" + sTotalOne;

}

}

}

reUTStr = reUTStr + new PublicFun().RemoveHtmlCode( reTitle ) + sTotalOne;

tmpAllStr = bstring( end_nums3 + 4 );

begin_nums = dexOf( "href=" );

}

else

{

begin_nums = -1;

}

}

}

return reUTStr;

}

catch( Exception e)

{

return "";

}

}

得到要抓取内容的url后，处理该页面：

//获取链接内容并分类处理

public string GetWebContent( string gatherUrl,string subUrl,string subTitle,string b_Content,string e_Content,string b_Filter,string e_Filter,string root )

{

string tmpAllStr = "";

string dfStrB = "";

string dfStrE = "";

string rePicStr = "";//图片返回路径

string reContentStr = "";

string picHtml = "images"; //本地图片路径

string urlch1 ="";

string urlch2 ="";

int pos1 = dexOf( "." );

int pos2 = stIndexOf( "/" );

if( pos1 0 )

{

return "";

}

if( pos2 0 )

{

return "";

}

int pos3 = dexOf( "/",pos1 );

if ( pos3 0 )

{

urlch1 = gatherUrl;

urlch2 = gatherUrl;

}

else

{

urlch1 = bstring( 0,pos3 );

urlch2 = bstring( 0,pos2 );

}

tmpAllStr = new PublicFun().Get_Http( subUrl,time1 );

//取稿源

string docFromStr = "";

if ( dfStrB != "" dfStrE != "" )

{

if ( tmpAllStr != "" )

{

int b_docF = dexOf( dfStrB );

if ( b_docF 0 )

{

int e_docF = dexOf( dfStrE,b_docF + ngth );

if ( e_docF 0 价格会比国内同类正品便宜很多 e_docF b_docF e_docF - b_docF 20 )

{

docFromStr = bstring( b_docF + ngth, e_docF - b_docF - ngth );

}

}

}

}

//取内容

if ( tmpAllStr != "" )

{

int begin_strnum = dexOf( b_Content );

if ( begin_strnum 0 )

{

return "";

}

int end_strnum = dexOf( e_Content,begin_strnum + b_ngth );

if ( end_strnum 0 )

{

return "";

}

string sTotalSubM = "";

if ( end_strnum begin_strnum )

{

sTotalSubM = bstring ( begin_strnum,end_strnum - begin_strnum );

}

if ( sTotalSubM == "" )

{

return "";

}

//过滤无用信息

int bfnum = dexOf( b_Filter );

if ( bfnum -1 )

{

int efnum = dexOf( e_Filter,bfnum );

if ( efnum -1 )

{

if ( efnum bfnum )

{

sTotalSubM = bstring( 0,bfnum ) + bstring( efnum + e_ngth );

}

}

}

//格式化图片标记

while( dexOf( "Src=" ) = 0 )

{

sTotalSubM = bstring( 0,dexOf( "Src=" ) ) + "src=" + bstring( dexOf( "Src=" ) + 4 );

}

while( dexOf( "SRC=" ) = 0 )

{

sTotalSubM = bstring( 0,dexOf( "SRC=" ) ) + "src=" + bstring( dexOf( "SRC=" ) + 4 );

}

while( dexOf( "src='" ) = 0 )

{

sTotalSubM = bstring( 0,dexOf( "src='" ) ) + "src=\"" + bstring( dexOf( "src='" ) + 5 );

}

//取图片地址

int end_n12 = 0;

int end_nums2 = 0;

int begin_nums2 = dexOf( "src=" );

while( begin_nums2 = 0 )

{

String tmpStr = bstring( begin_nums2 + 4,1 );

if ( tmpStr == "\"" )

{

end_n12 = begin_nums2 + 5;

}

else

{

end_n12 = begin_nums2 + 4;

}

int end_nums2a = dexOf( " ",end_n12 );

int end_nums2b = dexOf( "",end_n12 );

if ( end_nums2b 0 )

{

break;

}

if ( end_nums2a end_nums2b )

{

end_nums2 = end_nums2b;

}

else

{

if (end_nums2a0)

{

end_nums2 = end_nums2b;

}

else

{

end_nums2 = end_nums2a;

}

}

tmpStr = bstring( end_nums,1 );

if ( tmpStr == "\"" || tmpStr == "'" )

{

end_nums2 = end_nums2 - 1;

}

string tmpPicStr = bstring( end_n12,end_nums2 - end_n12 );

if ( dexOf( "http://" ) 0 )

{

if ( dexOf( "/" ) == 0 )

{

tmpPicStr = urlch1 + tmpPicStr;

}

else

{

int linshiIntNum = 0;

int flags = 0;

string urlChange = subUrl;

while( dexOf( "../" ) = 0 )

{

tmpPicStr = bstring( dexOf("../") + 3 );

linshiIntNum = linshiIntNum + 1;

flags = flags + 1;

}

while( ( stIndexOf( "/" ) = 0 ) ( linshiIntNum = 0 ) )

{

urlChange = bstring( 0,stIndexOf( "/" ) );

linshiIntNum = linshiIntNum - 1;

}

if ( flags == 0 )

{

tmpPicStr = urlch2 + "/" + tmpPicStr;

}

else

{

tmpPicStr = urlChange + "/" + tmpPicStr;

}

}

}

//tmpPicStr = Lower();

string tmpPicStrTmp = Lower();

//if ( dexOf( ".jpg" ) 0 || dexOf( ".gif" ) 0 || dexOf( ".bmp" ) 0 )

if ( dexOf( ".jpg" ) 0 || dexOf( ".gif" ) 0 || dexOf( ".bmp" ) 0 )

{

rePicStr = rePicStr + "||" + tmpPicStr ;

int flagN2 = stIndexOf( "/" );

string fileN2 = picHtml + bstring( flagN2 );

sTotalSubM = bstring( 0,end_nums2 ) + "******" + fileN2 + "******" + bstring( end_nums2 );

begin_nums2 = dexOf( "src=", end_nums2 + ngth + 22 );

}

else

{

begin_nums2 = dexOf( "src=", end_nums2 + 4 );

}

}

if ( ngth 2 )

rePicStr =bstring(2);

//内容处理 格式化关键标记

while( dexOf( "P" ) = 0 )

{

sTotalSubM = bstring( 0,dexOf( "P" ) ) + "|****|" + bstring( dexOf( "P" ) + 2 );

}

while( dexOf( "p" ) = 0 )

{

sTotalSubM = bstring( 0,dexOf( "p" ) ) + "|****|" + bstring( dexOf( "p" ) + 2 );

}

while( dexOf( "/P" ) = 0 )

{

sTotalSubM = bstring( 0,dexOf( "/P" ) ) + "|****|" + bstring( dexOf( "/P" ) + 3 );

}

while( dexOf( "/p" ) = 0 )

{

sTotalSubM = bstring( 0,dexOf( "/p" ) ) + "|****|" + bstring( dexOf( "/p" ) + 3 );

}

while( dexOf( "br" ) =0 )

{

sTotalSubM = bstring( 0,dexOf( "br" ) ) + "+****+" + bstring( dexOf( "br" ) + 3 );

}

while( dexOf( "BR" ) = 0 )

{

sTotalSubM = bstring( 0,dexOf( "BR" ) ) + "+****+" + bstring( dexOf( "BR" ) + 3 );

}

while( dexOf( "Br" ) = 0 )

{

sTotalSubM = bstring( 0,dexOf( "Br" ) ) + "+****+" + bstring( dexOf( "Br" ) + 3 );

}

while( dexOf( "bR" ) = 0 )

{

sTotalSubM = bstring( 0,dexOf( "bR" ) ) + "+****+" + bstring( dexOf( "bR" ) + 3 );

}

//去除html标记

int linshiInt1 = dexOf( "" );

int linshiInt2 = dexOf( "" );

if ( linshiInt2 linshiInt1 )

{

sTotalSubM = bstring( linshiInt2 + 1 );

}

int linshiInt11 = stIndexOf( "" );

int linshiInt12 = stIndexOf( "" );

if ( linshiInt12 linshiInt11 )

{

sTotalSubM = bstring( 0,linshiInt12 + 1 );

}

linshiInt1 = dexOf( "" );

while ( linshiInt1 = 0 )

{

linshiInt2 = dexOf( "",linshiInt1 );

if ( linshiInt2 = 0 )

{

sTotalSubM = bstring( 0,linshiInt1 ) + bstring( linshiInt2 + 1 );

}

else

{

sTotalSubM = bstring( 0,linshiInt1 );

}

linshiInt1 = dexOf("");

}

//还原关键标记

int linshiInt3 = 0;

int linshiInt4 = 0;

while( dexOf( "+****+" ) = 0 )

{

sTotalSubM = bstring( 0,dexOf( "+****+" ) ) + "br\n" + bstring( dexOf( "+****+" ) + 9 );

}

while( dexOf( "|****|" ) = 0 )

{

sTotalSubM = bstring( 0,dexOf( "|****|" ) ) + "br\n" + bstring( dexOf( "|****|" ) + 9 );

}

while( dexOf( "******" ) = 0 )

{

linshiInt3 = dexOf( "******" ) + 9;

linshiInt4 = dexOf( "******",linshiInt3 );

if ( linshiInt4 = 0 )

{

int tmpPos = dexOf( "******" );

string tmpStr1 = bstring( 0,tmpPos );

string tmpStr2 = bstring( linshiInt3,linshiInt4 - linshiInt3 );

string tmpStr3 = bstring( linshiInt4 + 9 );

sTotalSubM = tmpStr1 + "img src=" + tmpStr2 + "" + tmpStr3;

}

else

{

break;

}

}

//去除内容中的标题

if ( dexOf( subTitle ) = 0 )

{

sTotalSubM = bstring( 0,dexOf( subTitle ) ) + bstring( dexOf( subTitle ) + ngth );

}

reContentStr = sTotalSubM;

//调用下载图片功能

//下载图片到指定目录

string[] img_Url = new PublicFun().split( rePicStr,"||" );

for ( int i=0;iimg_ngth;i++ )

{

if ( img_Url[i] != "" )

{

new PublicFun().Get_Img( img_Url[i],10000,root + "\\images\\" + img_Url[i].Substring( img_Url[i持股比例为12.5%。].LastIndexOf("/")+1 ) );

}

}

}

return reContentStr;

}

以上方法返回要取得的信息，包括标题内容，图片地址等。

下载页面中图片：

//下载图片

public void Get_Img(string a_strUrl,int timeout,string filepath)

{

try

{

HttpWebRequest myReq = (HttpWebRequest)eate(a_strUrl) ;

meout = timeout;

HttpWebResponse HttpWResp = (HttpWebResponse)tResponse();

Stream myStream = tResponseStream () ;

Bitmap map = new Bitmap( myStream );

PictureBox picB = new PictureBox();

age = (Image)map;

string path = bstring( 0,stIndexOf( "\\" ) );

if (!ists(path))

{

CreateDir( path );

}

ve(filepath);

}

catch(Exception exp)

{

string ss = ssage;

WriteLog( bstring(0,stIndexOf("\\")) + "\\g",a_strUrl + "--" + ss + "\r\n");

}

}

保存文件或入库

上面取得的信息可以按自己的要求保存。

****设计的时候没有使用url按层次循环抓取，这样定义抓取url效率更高，速度更快。

测试程序下载

如有建议请发送EMail：xiancai@ 或msn yutao728@

注：此版本只提供静态文件存储功能，不提供数据库接口，不提供自定义站功能。

本程序运行需要先安装.net 框架1.1。

查看本文来源

扶正化瘀胶囊比复方鳖甲软肝片好吗
藤黄健骨丸
重庆专治白癜风医院