源码网商城,靠谱的源码在线交易网站 我的订单 购物车 帮助

源码网商城

c#远程html数据抓取实例分享

  • 时间:2022-06-05 23:03 编辑: 来源: 阅读:
  • 扫一扫,手机访问
摘要:c#远程html数据抓取实例分享
[u]复制代码[/u] 代码如下:
/// <summary>         /// 获取远程html         /// </summary>         /// <param name="url"></param>         /// <param name="methed"></param>         /// <param name="param"></param>         /// <param name="html"></param>         /// <returns></returns>         public static bool GetHttp(string url, string methed, string param, out string html)         {             methed = methed.ToLower();             if (param != null && methed == "get" && param.Length > 0)             {                 url += "?" + param;             }             try             {                 MSXML2.XMLHTTP mx = new MSXML2.XMLHTTPClass();                 mx.open(methed, url, false, null, null);                 if (param != null && methed == "post" && param.Length > 0)                 {                     mx.setRequestHeader("Content-Length", param.Length.ToString());                     mx.setRequestHeader("Content-Type", "application/x-www-form-urlencoded");                 }                 mx.send(param);                 if (mx.readyState != 4)                 {                     html = "远程连接失败:-4";                     return false;                 }                 html = mx.responseText;                 return true;             }             catch (Exception ex)             {                 html = "远程连接失败:"+ex.Message;                 return false;             }         }         public static bool GetHttp1(string url, string methed, string param, string referer, string encode, out string html)         {             //return GetHttp(url,methed,param,out html);             //string encode = "utf-8";             //string methed = sendType.ToString();             if (param != null && methed == "get" && param.Length > 0)             {                 if (url.IndexOf("?") >= 0)                 {                     url += "&" + param;                 }                 else                 {                     url += "?" + param;                 }             }             try             {                 HttpWebRequest webreq = (HttpWebRequest)WebRequest.Create(url);                 webreq.Proxy=null;                 webreq.Timeout = 1000 * 6;                 webreq.ContentType = "application/x-www-form-urlencoded";                 webreq.UserAgent = "User-Agent:Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0";                 //webreq.UserAgent = "Mozilla/4.0 (compatible; MSIE 7.0; Windows NT 6.1; WOW64; Trident/6.0; SLCC2; .NET CLR 2.0.50727; .NET CLR 3.5.30729; .NET CLR 3.0.30729; Media Center PC 6.0; .NET4.0C; .NET4.0E)";                 //谷歌的:User-Agent:Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/28.0.1500.95 Safari/537.36                 //火狐的:User-Agent:Mozilla/5.0 (Windows NT 6.1; WOW64; rv:24.0) Gecko/20100101 Firefox/24.0                 //标准格式为: 浏览器标识 (操作系统标识; 加密等级标识; 浏览器语言) 渲染引擎标识 版本信息                 //webreq.AllowAutoRedirect = false;                 //频繁请求一个网址时,过段时间就会出现“基础连接已经关闭”                 //webreq.KeepAlive = false;                 //webreq.ProtocolVersion = HttpVersion.Version10;                 if (referer.Length > 0)                 {                     webreq.Referer = referer;                 }                 CookieContainer mycookies = new CookieContainer();                 webreq.CookieContainer = mycookies;                 //if (this.cookieList != null)                 //{                 //    webreq.CookieContainer.Add(this.GetCookies(webreq.RequestUri, this.cookieList));                 //}                 webreq.Method = methed;                 //post 开始                 if (param != null && methed == "post")                 {                     byte[] arrbyte = Encoding.GetEncoding(encode).GetBytes(param);                     webreq.ContentLength = arrbyte.Length;                     Stream newStream = webreq.GetRequestStream();                     newStream.Write(arrbyte, 0, arrbyte.Length);                     newStream.Close();                 }                 //post 结束                   WebResponse w = webreq.GetResponse();                 //返回HTML                 using (HttpWebResponse webres = (HttpWebResponse)webreq.GetResponse())                 {                     using (Stream dataStream = webres.GetResponseStream())                     {                         using (StreamReader reader = new StreamReader(dataStream, Encoding.GetEncoding(encode)))                         {                             html = reader.ReadToEnd();                             //this.cookieList = webreq.CookieContainer.GetCookies(webreq.RequestUri);                             webreq.Abort();//可能会解决卡住或阻塞问题                         }                     }                 }             }             catch (Exception ex)             {                 html = "出现异常(HttpHelper.GetHTML),远程连接失败:" + ex.Message + " url:" + url;                 //System.Windows.Forms.MessageBox.Show(html);                 return false;             }             return true;         }
  • 全部评论(0)
联系客服
客服电话:
400-000-3129
微信版

扫一扫进微信版
返回顶部