首页 > 代码库 > 获取URL网页信息

获取URL网页信息

 

 

        static string GetHtml(string url)        {string strHTML = "";            WebClient myWebClient = new WebClient();            Stream myStream = myWebClient.OpenRead(url);            StreamReader sr = new StreamReader(myStream, System.Text.Encoding.GetEncoding("utf-8"));            strHTML = sr.ReadToEnd();            myStream.Close();            return strHTML;        }        static bool GetHtml(string url)      {                HttpWebRequest myHttpWebRequest = (HttpWebRequest)WebRequest.Create(url);            myHttpWebRequest.Timeout = 20 * 1000; //连接超时            myHttpWebRequest.Accept = "*/*";            myHttpWebRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0;)";            HttpWebResponse myHttpWebResponse = (HttpWebResponse)myHttpWebRequest.GetResponse();            Stream stream = myHttpWebResponse.GetResponseStream();            StreamReader sr =               new StreamReader(stream, Encoding.GetEncoding("utf-8"));            var html = sr.ReadToEnd();           
}

 

 

获取当前Cookies保存后获取整站信息

static void Main(string[] args)        {                       Console.WriteLine("确保挂载VPN后。。。回车键继续!");            Console.ReadKey();            string url = "http://www.sciencedirect.com/science/journal/00118486/61/6";            string indata = "http://www.mamicode.com/aa=zhuye";            CookieContainer myCookieContainer =                new CookieContainer();            //新建一个cookiecontainer来存放cookie集合            HttpWebRequest myHttpWebRequest =                (HttpWebRequest)WebRequest.Create(url);            //新建一个httpwebrequest            myHttpWebRequest.ContentType = "application/x-www-form-urlencoded";            myHttpWebRequest.ContentLength = indata.Length;            myHttpWebRequest.Method = "post";            myHttpWebRequest.CookieContainer = myCookieContainer;            //设置httpwebrequest的cookiecontainer为            //刚才建立的那个mycookiecontainer            Stream myRequestStream = myHttpWebRequest.GetRequestStream();            StreamWriter myStreamWriter = new StreamWriter(myRequestStream, Encoding.GetEncoding("gb2312"));            myStreamWriter.Write(indata);            //把数据写入httpwebrequest的request流            myStreamWriter.Close();            myRequestStream.Close();            //关闭打开对象            HttpWebResponse myHttpWebResponse =                (HttpWebResponse)myHttpWebRequest.GetResponse();            //新建一个httpwebresponse            myHttpWebResponse.Cookies = myCookieContainer.GetCookies(myHttpWebRequest.RequestUri);                      Console.WriteLine("获取cookies成功!。。。关闭VPN后任意键继续");            Console.ReadKey();            var htmlData = http://www.mamicode.com/SaveCook(myHttpWebRequest, prevUrl, myCookieContainer, myHttpWebResponse);>

 

 public static string SaveCook(HttpWebRequest myHttpWebRequest, string url, CookieContainer myCookieContainer, HttpWebResponse myHttpWebResponse)        {            //拿到了cookie,再进行请求就能直接读取到登录后的内容了            myHttpWebRequest = (HttpWebRequest)WebRequest.Create(url);            myHttpWebRequest.CookieContainer = myCookieContainer;//*                                                                 //刚才那个cookiecontainer已经存有了cookie,把它附加到             myHttpWebResponse = (HttpWebResponse)myHttpWebRequest.GetResponse();            myHttpWebResponse.Cookies =                myCookieContainer.GetCookies(myHttpWebRequest.RequestUri);            Stream myresponsestream = myHttpWebResponse.GetResponseStream();            StreamReader mystreamreader =               new StreamReader(myresponsestream, Encoding.GetEncoding("gb2312"));            var html = mystreamreader.ReadToEnd();            //把数据从httpwebresponse的response流中读出            mystreamreader.Close();            myresponsestream.Close();            return html;        }

 

获取URL网页信息