模擬HTTP請求實現(xiàn)網(wǎng)頁自動操作及數(shù)據(jù)采集的方法

String contentType = "application/x-www-form-urlencoded";
String accept = "image/gif, image/x-xbitmap, image/jpeg, image/pjpeg, application/x-shockwave-flash, application/x-silverlight, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, application/x-ms-application, application/x-ms-xbap, application/vnd.ms-xpsdocument, application/xaml+xml, application/x-silverlight-2-b1, */*";
String userAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/34.0.1847.116 Safari/537.36";

public String Get(String url, String encode = DEFAULT_ENCODE)
{
 HttpWebRequest request = WebRequest.Create(url) as HttpWebRequest;
 InitHttpWebRequestHeaders(request);
 request.Method = "GET";
 var html = ReadHtml(request, encode);
 return html;
}

public String Post(String url, String param, String encode = DEFAULT_ENCODE)
{
 Encoding encoding = System.Text.Encoding.UTF8;
 byte[] data = encoding.GetBytes(param);
 HttpWebRequest request = WebRequest.Create(url) as HttpWebRequest;
 InitHttpWebRequestHeaders(request);
 request.Method = "POST";
 request.ContentLength = data.Length;
 var outstream = request.GetRequestStream();
 outstream.Write(data, 0, data.Length);
 var html = ReadHtml(request, encode);
 return html;
}

private void InitHttpWebRequestHeaders(HttpWebRequest request)
{
 request.ContentType = contentType;
 request.Accept = accept;
 request.UserAgent = userAgent;
}

private String ReadHtml(HttpWebRequest request, String encode)
{
 HttpWebResponse response = request.GetResponse() as HttpWebResponse;
 Stream stream = response.GetResponseStream();
 StreamReader reader = new StreamReader(stream, Encoding.GetEncoding(encode));
 String content = reader.ReadToEnd();
 reader.Close();
 stream.Close();
 return content;
}

The underlying connection was closed: Could not establish trust relationship for 

private HttpWebRequest CreateHttpWebRequest(String url)
{
 HttpWebRequest request;
 if (IsHttpsProtocol(url))
 {
 ServicePointManager.ServerCertificateValidationCallback = new RemoteCertificateValidationCallback(CheckValidationResult);
 request = WebRequest.Create(url) as HttpWebRequest;
 request.ProtocolVersion = HttpVersion.Version10;
 }
 else
 {
 request = WebRequest.Create(url) as HttpWebRequest;
 }

 return request;
}

private HttpWebRequest CreateHttpWebRequest(String url)
{
 HttpWebRequest request;
 if (IsHttpsProtocol(url))
 {
 ServicePointManager.ServerCertificateValidationCallback = new RemoteCertificateValidationCallback(CheckValidationResult);
 request = WebRequest.Create(url) as HttpWebRequest;
 request.ProtocolVersion = HttpVersion.Version10;
 }
 else
 {
 request = WebRequest.Create(url) as HttpWebRequest;
 }

 return request;
}

public String Get(String url, String encode = DEFAULT_ENCODE)
{
 HttpWebRequest request = WebRequest.Create(url) as HttpWebRequest;
 InitHttpWebRequestHeaders(request);
 request.Method = "GET";

 request.CookieContainer = cookieContainer;
 HttpWebResponse response = request.GetResponse() as HttpWebResponse;
 foreach (Cookie c in response.Cookies)
 {
 cookieContainer.Add(c);
 }
}