写了一个从网页中抓取信息如最新的头条新闻新闻的来源标题内容等的类本文将介绍如何使用这个类来抓取网页中需要的信息。本文将以抓取博客园首页的博客标题和链接为例上图显示的是博客园首页的DOM树显然只需提取出class为post_item的div再重中提取出class为titlelnk的a标志即可。这样的功能可以通过以下函数来实现/// summary /// 在文本html的文本查找标志名为tagName,并且属性attrName的值为attrValue的所有标志 /// 例如FindTagByAttr(html, div, class, demo) /// 返回所有class为demo的div标志 /// 前端学习交流QQ群461593224/// /summary public static ListHtmlTag FindTagByAttr(String html, String tagName, String attrName, String attrValue) { String format String.Format({0}\s[^]*{1}\s*\s*(\x27|\x22){2}(\x27|\x22)[^]*, tagName, attrName, attrValue); return FindTag(html, tagName, format); } public static ListHtmlTag FindTag(String html, String name, String format) { Regex reg new Regex(format, RegexOptions.IgnoreCase); Regex tagReg new Regex(String.Format((\/|)({0})(\s[^]*|), name), RegexOptions.IgnoreCase); ListHtmlTag tags new ListHtmlTag(); int start 0; while (true) { Match match reg.Match(html, start); if (match.Success) { start match.Index match.Length; Match tagMatch null; int beginTagCount 1; while (true) { tagMatch tagReg.Match(html, start); if (!tagMatch.Success) { tagMatch null; break; } start tagMatch.Index tagMatch.Length; if (tagMatch.Groups[1].Value /) beginTagCount--; else beginTagCount ; if (beginTagCount 0) break; } if (tagMatch ! null) { HtmlTag tag new HtmlTag(name, match.Value, html.Substring(match.Index match.Length, tagMatch.Index - match.Index - match.Length)); tags.Add(tag); } else { break; } } else { break; } } return tags; }有了以上函数就可以提取需要的HTML标志了要实现抓取还需要一个下载网页的函数public static String GetHtml(string url) { try { HttpWebRequest req HttpWebRequest.Create(url) as HttpWebRequest; req.Timeout 30 * 1000; HttpWebResponse response req.GetResponse() as HttpWebResponse; Stream stream response.GetResponseStream(); MemoryStream buffer new MemoryStream(); Byte[] temp new Byte[4096]; int count 0; while ((count stream.Read(temp, 0, 4096)) 0) { buffer.Write(temp, 0, count); } return Encoding.GetEncoding(response.CharacterSet).GetString(buffer.GetBuffer()); } catch { return String.Empty; } }/// 前端学习交流QQ群461593224以下以抓取博客园首页的文章标题和链接为例介绍如何使用HtmlTag类来抓取网页信息class Program { static void Main(string[] args) { String html HtmlTag.GetHtml(http://www.cnblogs.com); ListHtmlTag tags HtmlTag.FindTagByAttr(html, div, id, post_list); if (tags.Count 0) { ListHtmlTag item_tags tags[0].FindTagByAttr(div, class, post_item); foreach (HtmlTag item_tag in item_tags) { ListHtmlTag a_tags item_tag.FindTagByAttr(a, class, titlelnk); if (a_tags.Count 0) { Console.WriteLine(标题:{0}, a_tags[0].InnerHTML); Console.WriteLine(链接:{0}, a_tags[0].GetAttribute(href)); Console.WriteLine(); } } } } }运行结果如下欢迎学习前端的同学一起学习前端学习交流QQ群461593224
)
