From e74d5f3c5ffd2eca2764c62ae1270db6efd13c6c Mon Sep 17 00:00:00 2001 From: Sean Kessler Date: Mon, 1 Apr 2024 22:55:42 -0400 Subject: [PATCH] Initial --- HttpNetRequest.cs | 1416 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 1416 insertions(+) create mode 100644 HttpNetRequest.cs diff --git a/HttpNetRequest.cs b/HttpNetRequest.cs new file mode 100644 index 0000000..e000441 --- /dev/null +++ b/HttpNetRequest.cs @@ -0,0 +1,1416 @@ +using System; +using System.IO; +using System.Net; +using System.Text; +using System.Threading; +using System.IO.Compression; +using System.Configuration; +// Filename: HttpRequest.cs +// Author:Sean Kessler + +namespace IPMonitor +{ + public class HttpNetResponse : IDisposable + { + public enum TypeResponse {Stream,String}; + public String ResponseString{get;set;} + public MemoryStream ResponseStream{get;set;} + public CookieCollection CookieCollection{get;set;} + + public HttpNetResponse() + { + Success=false; + } + public HttpNetResponse(String responseString,String request,HttpWebResponse webResponse,CookieCollection cookieCollection,bool success) + { + ResponseString=responseString; + ResponseType=TypeResponse.String; + Success=success; + Request=request; + CookieCollection=cookieCollection; + if(null!=webResponse)StatusCode=webResponse.StatusCode; + } + public HttpNetResponse(String responseString,String request,HttpWebResponse webResponse,bool success) + { + ResponseString=responseString; + ResponseType=TypeResponse.String; + Success=success; + Request=request; + if(null!=webResponse)StatusCode=webResponse.StatusCode; + } + public HttpNetResponse(MemoryStream responseStream,String request,HttpWebResponse webResponse,bool success) + { + ResponseStream=responseStream; + ResponseType=TypeResponse.String; + Success=success; + Request=request; + if(null!=webResponse)StatusCode=webResponse.StatusCode; + } + public HttpNetResponse(HttpWebResponse webResponse,String request,bool success,String errorMessage) + { + ResponseType=TypeResponse.String; + Success=success; + Request=request; + if(null!=webResponse)StatusCode=webResponse.StatusCode; + ErrorMessage=errorMessage; + } + public Stream ResponseStringToStream() + { + MemoryStream stream = new MemoryStream(); + StreamWriter writer = new StreamWriter(stream); + writer.Write(ResponseString); + writer.Flush(); + stream.Position = 0; + return stream; + } + public String ResponseStreamToString + { + get + { + if(!Success)return null; + return System.Text.Encoding.UTF8.GetString(ResponseStream.ToArray()); + } + } + public TypeResponse ResponseType{get;set;} + public bool Success{get;set;} + public String ErrorMessage{get;set;} + public HttpStatusCode StatusCode{get;set;} + public String Request{get;set;} + public bool IsNetErrorResponse + { + get + { + return (int)StatusCode>=400?true:false; + } + } + public void Dispose() + { + if(null!=ResponseStream) + { + ResponseStream.Close(); + ResponseStream.Dispose(); + ResponseStream=null; + } + } + } +// ********************************************************************************************************************************************************************* + + public class HttpNetRequest + { + public static int REQUEST_TIMEOUT = 300000; + private HttpNetRequest() + { + } + public static HttpNetResponse GetRequestStreamZIP(String strRequest) + { + HttpWebResponse webResponse=null; + try + { + MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestStreamZIP[ENTER]{0}",strRequest)); + int charCount = 0; + byte[] buffer = new byte[8192]; + StringBuilder sb = new StringBuilder(); + HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest)); + webRequest.Timeout = REQUEST_TIMEOUT; + webRequest.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"; + webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5"); + webRequest.Headers.Add("Accept-Encoding: gzip, deflate"); + webRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0"; + webRequest.KeepAlive = true; + webResponse = (HttpWebResponse)webRequest.GetResponse(); + Stream responseStream = webResponse.GetResponseStream(); + MemoryStream memoryStream=new MemoryStream(); + while (true) + { + charCount = responseStream.Read(buffer, 0, buffer.Length); + if(charCount<=0)break; + memoryStream.Write(buffer,0,charCount); + } + webResponse.Close(); + memoryStream.Flush(); + return new HttpNetResponse(memoryStream,strRequest,webResponse,true); + } + catch (WebException webException) + { + return new HttpNetResponse((HttpWebResponse)webException.Response,strRequest,false,webException.Message); + } + catch (Exception exception) + { + return new HttpNetResponse(webResponse,strRequest,false,exception.Message); + } + finally + { + MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestStreamZIP[LEAVE]"); + } + } + public static HttpNetResponse GetRequestStreamZIPV2(String strRequest) + { + HttpWebResponse webResponse=null; + try + { + MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestStreamZIPV2[ENTER]{0}",strRequest)); + int charCount = 0; + byte[] buffer = new byte[8192]; + StringBuilder sb = new StringBuilder(); + HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest)); + webRequest.Timeout = REQUEST_TIMEOUT; + webRequest.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"; + webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5"); + webRequest.Headers.Add("Accept-Encoding: gzip, deflate"); + webRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:59.0) Gecko/20100101 Firefox/59.0"; + webRequest.KeepAlive = true; + webRequest.CookieContainer=new CookieContainer(); + webResponse = (HttpWebResponse)webRequest.GetResponse(); + Stream responseStream = webResponse.GetResponseStream(); + MemoryStream memoryStream=new MemoryStream(); + while (true) + { + charCount = responseStream.Read(buffer, 0, buffer.Length); + if(charCount<=0)break; + memoryStream.Write(buffer,0,charCount); + } + webResponse.Close(); + memoryStream.Flush(); + return new HttpNetResponse(memoryStream,strRequest,webResponse,true); + } + catch (WebException webException) + { + return new HttpNetResponse((HttpWebResponse)webException.Response,strRequest,false,webException.Message); + } + catch (Exception exception) + { + return new HttpNetResponse(webResponse,strRequest,false,exception.Message); + } + finally + { + MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestStreamZIPV2[LEAVE]"); + } + } + public static HttpNetResponse GetRequestStreamCSV(String strRequest,CookieCollection cookieCollection=null, WebProxy webProxy=null) + { + HttpWebResponse webResponse=null; + try + { + ServicePointManager.Expect100Continue=true; + ServicePointManager.SecurityProtocol=SecurityProtocolType.Ssl3|SecurityProtocolType.Tls12|SecurityProtocolType.Tls11|SecurityProtocolType.Tls; + MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestStreamCSV[ENTER]{0}",strRequest)); + int charCount = 0; + byte[] buffer = new byte[8192]; + StringBuilder sb = new StringBuilder(); + HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest)); + if(null!=webProxy) + { + webRequest.Proxy=webProxy; + } + if(null!=cookieCollection) + { + webRequest.CookieContainer=new CookieContainer(); + webRequest.CookieContainer.Add(cookieCollection); + } + webRequest.Timeout = REQUEST_TIMEOUT; + webRequest.Accept = "text/csv,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"; + webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5"); + webRequest.Headers.Add("Accept-Encoding: gzip, deflate"); // + webRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0"; + webRequest.KeepAlive = true; + webResponse = (HttpWebResponse)webRequest.GetResponse(); + Stream responseStream = webResponse.GetResponseStream(); + + if(webResponse.ContentEncoding.ToLower().Contains("gzip")) + { + responseStream = new GZipStream(responseStream, CompressionMode.Decompress); + StreamReader reader = new StreamReader(responseStream, Encoding.Default); + sb.Append(reader.ReadToEnd()); + reader.Close(); + } + else if(webResponse.ContentEncoding.ToLower().Contains("deflate")) + { + responseStream = new DeflateStream(responseStream, CompressionMode.Decompress); + StreamReader reader = new StreamReader(responseStream, Encoding.Default); + sb.Append(reader.ReadToEnd()); + reader.Close(); + } + else + { + while (true) + { + charCount = responseStream.Read(buffer, 0, buffer.Length); + if (0 == charCount) break; + sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount)); + } + } + webResponse.Close(); + return new HttpNetResponse(sb.ToString(),strRequest,webResponse,true); + } + catch (WebException webException) + { + return new HttpNetResponse((HttpWebResponse)webException.Response,strRequest,false,webException.Message); + } + catch (Exception exception) + { + return new HttpNetResponse(webResponse,strRequest,false,exception.Message); + } + finally + { + MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestStreamCSV[LEAVE]"); + } + } + + public static HttpNetResponse GetRequestNoEncoding(Uri uriRequest,WebProxy webProxy=null) + { + HttpWebResponse webResponse=null; + try + { + ServicePointManager.Expect100Continue = true; + ServicePointManager.SecurityProtocol=SecurityProtocolType.Ssl3|SecurityProtocolType.Tls12|SecurityProtocolType.Tls11|SecurityProtocolType.Tls; + MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestNoEncoding[ENTER]{0}",uriRequest.ToString())); + int charCount = 0; + byte[] buffer = new byte[8192]; + StringBuilder sb = new StringBuilder(); + HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(uriRequest); + if(null!=webProxy)webRequest.Proxy=webProxy; + webRequest.Timeout = REQUEST_TIMEOUT; + webRequest.Accept = "text/html, text/csv"; + webRequest.ContentType="text/csv; charset=utf-8"; + webRequest.Headers.Add("Accept-Encoding: None"); + webRequest.Headers.Add("Accept-Language: en-US"); + webRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0"; + webRequest.KeepAlive = true; + webResponse = (HttpWebResponse)webRequest.GetResponse(); + Stream responseStream = webResponse.GetResponseStream(); + while (true) + { + charCount = responseStream.Read(buffer, 0, buffer.Length); + if (0 >= charCount) break; + sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount)); + } + webResponse.Close(); + return new HttpNetResponse(sb.ToString(),uriRequest.ToString(),webResponse,true); + } + catch (WebException webException) + { + return new HttpNetResponse((HttpWebResponse)webException.Response,uriRequest.ToString(),false,webException.Message); + } + catch (Exception exception) + { + return new HttpNetResponse(webResponse,uriRequest.ToString(),false,exception.Message); + } + finally + { + MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncoding[LEAVE]"); + } + } + + + public static HttpNetResponse GetRequestNoEncoding(String strRequest,WebProxy webProxy=null) + { + HttpWebResponse webResponse=null; + try + { + ServicePointManager.Expect100Continue = true; + ServicePointManager.SecurityProtocol=SecurityProtocolType.Ssl3|SecurityProtocolType.Tls12|SecurityProtocolType.Tls11|SecurityProtocolType.Tls; + MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestNoEncoding[ENTER]{0}",strRequest)); + int charCount = 0; + byte[] buffer = new byte[8192]; + StringBuilder sb = new StringBuilder(); + HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest)); + if(null!=webProxy)webRequest.Proxy=webProxy; + webRequest.Timeout = REQUEST_TIMEOUT; + webRequest.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8"; + webRequest.ContentType="text/csv; charset=utf-8"; + webRequest.Headers.Add("Accept-Encoding: gzip, deflate, br"); + webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5"); + webRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:123.0) Gecko/20100101 Firefox/123.0"; + webRequest.KeepAlive = true; + webResponse = (HttpWebResponse)webRequest.GetResponse(); + Stream responseStream = webResponse.GetResponseStream(); + while (true) + { + charCount = responseStream.Read(buffer, 0, buffer.Length); + if (0 >= charCount) break; + sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount)); + } + webResponse.Close(); + return new HttpNetResponse(sb.ToString(),strRequest,webResponse,true); + } + catch (WebException webException) + { + return new HttpNetResponse((HttpWebResponse)webException.Response,strRequest,false,webException.Message); + } + catch (Exception exception) + { + return new HttpNetResponse(webResponse,strRequest,false,exception.Message); + } + finally + { + MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncoding[LEAVE]"); + } + } + + public static HttpNetResponse GetRequestNoEncodingV1(String strRequest,WebProxy webProxy=null) + { + ServicePointManager.Expect100Continue=true; + ServicePointManager.SecurityProtocol=SecurityProtocolType.Ssl3|SecurityProtocolType.Tls12|SecurityProtocolType.Tls11|SecurityProtocolType.Tls; + HttpWebResponse webResponse=null; + try + { + MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestNoEncoding[ENTER]{0}",strRequest)); + int charCount=0; + byte[] buffer=new byte[8192]; + StringBuilder sb=new StringBuilder(); + HttpWebRequest webRequest=(HttpWebRequest)WebRequest.Create(new Uri(strRequest)); + if(null!=webProxy)webRequest.Proxy=webProxy; + webRequest.Timeout=REQUEST_TIMEOUT; + webRequest.Accept="text/html, text/csv"; + webRequest.ContentType="text/csv; charset=utf-8"; + webRequest.Headers.Add("Accept-Encoding: None"); + webRequest.Headers.Add("Accept-Language: en-US"); + webRequest.UserAgent="Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0"; + webRequest.KeepAlive=true; + webResponse=(HttpWebResponse)webRequest.GetResponse(); + Stream responseStream=webResponse.GetResponseStream(); + while(true) + { + charCount=responseStream.Read(buffer,0,buffer.Length); + if(0>=charCount) break; + sb.Append(Encoding.ASCII.GetString(buffer,0,charCount)); + } + webResponse.Close(); + return new HttpNetResponse(sb.ToString(),strRequest,webResponse,true); + } + catch(WebException webException) + { + return new HttpNetResponse((HttpWebResponse)webException.Response,strRequest,false,webException.Message); + } + catch(Exception exception) + { + return new HttpNetResponse(webResponse,strRequest,false,exception.Message); + } + finally + { + MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncoding[LEAVE]"); + } + } + public static HttpNetResponse GetRequestCSV(String strRequest,String referer) + { + HttpWebResponse webResponse=null; + try + { + MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestCSV[ENTER]{0}",strRequest)); + int charCount = 0; + byte[] buffer = new byte[8192]; + StringBuilder sb = new StringBuilder(); + HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest)); + webRequest.Timeout = REQUEST_TIMEOUT; + webRequest.Accept="text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8"; + webRequest.ContentType="text/csv; charset=utf-8"; + webRequest.Headers.Add("Accept-Encoding: identity"); // "gzip, deflate" + webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5"); + webRequest.Headers.Add("Upgrade-Insecure-Requests: 1"); + webRequest.UserAgent="Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0"; + webRequest.Referer=referer; + webRequest.KeepAlive = true; + webResponse = (HttpWebResponse)webRequest.GetResponse(); + Stream responseStream = webResponse.GetResponseStream(); + while (true) + { + charCount = responseStream.Read(buffer, 0, buffer.Length); + if (0 == charCount) break; + sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount)); + } + webResponse.Close(); + return new HttpNetResponse(sb.ToString(),strRequest,webResponse,true); + } + catch (WebException webException) + { + return new HttpNetResponse((HttpWebResponse)webException.Response,strRequest,false,webException.Message); + } + catch (Exception exception) + { + return new HttpNetResponse(webResponse,strRequest,false,exception.Message); + } + finally + { + MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestCSV[LEAVE]"); + } + } + public static HttpNetResponse GetRequestNoEncodingV2(String strRequest) + { + HttpWebResponse webResponse=null; + try + { + MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestNoEncodingV2[ENTER]{0}",strRequest)); + int charCount = 0; + byte[] buffer = new byte[8192]; + StringBuilder sb = new StringBuilder(); + HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest)); + webRequest.Timeout = REQUEST_TIMEOUT; + webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5"); + webRequest.Headers.Add("Accept-Encoding: None"); + webRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0"; + webRequest.Accept = "image/gif, image/jpeg, image/pjpeg, image/pjpeg, application/x-shockwave-flash, application/x-ms-application, application/x-ms-xbap, application/vnd.ms-xpsdocument, application/xaml+xml, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*"; + webRequest.KeepAlive = true; + webResponse = (HttpWebResponse)webRequest.GetResponse(); + Stream responseStream = webResponse.GetResponseStream(); + while (true) + { + charCount = responseStream.Read(buffer, 0, buffer.Length); + if (0 == charCount) break; + sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount)); + } + webResponse.Close(); + return new HttpNetResponse(sb.ToString(),strRequest,webResponse,true); + } + catch (WebException webException) + { + return new HttpNetResponse((HttpWebResponse)webException.Response,strRequest,false,webException.Message); + } + catch (Exception exception) + { + return new HttpNetResponse(webResponse,strRequest,false,exception.Message); + } + finally + { + MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncodingV2[LEAVE]"); + } + } +// I am using this code specifically on the seeking alpha web site. it seems as though seeking alpha has implemeted some user agent based bot prevention mechanism on their website +// to prevent scrapers. What I do here is to choose from a set of commonly used user agents and then randomly choose a user agent to code into the request. retry logic up to 3 times +// and handling of 404 (not found) + public static HttpNetResponse GetRequestNoEncodingV3(String strRequest,String referer=null) + { + HttpWebResponse webResponse=null; + WebException lastWebException=null; + + try + { + MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestNoEncodingV3[ENTER]{0}",strRequest)); + Random random=new Random(); + String[] userAgents= + { + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537." + }; + int MAX_RETRIES=5; + int TIMEOUT_BETWEEN_RETRIES=1000; + int charCount = 0; + byte[] buffer = new byte[8192]; + StringBuilder sb = new StringBuilder(); + + for(int count=0;count=userAgents.Length) index=userAgents.Length-1; + String userAgent=userAgents[index]; + webRequest.UserAgent=userAgent; + + try + { + webResponse=(HttpWebResponse)webRequest.GetResponse(); + Stream responseStream=webResponse.GetResponseStream(); + if(webResponse.ContentEncoding.ToLower().Contains("gzip")) + { + responseStream=new GZipStream(responseStream,CompressionMode.Decompress); + StreamReader reader=new StreamReader(responseStream,Encoding.Default); + sb.Append(reader.ReadToEnd()); + reader.Close(); + } + else if(webResponse.ContentEncoding.ToLower().Contains("deflate")) + { + responseStream=new DeflateStream(responseStream,CompressionMode.Decompress); + StreamReader reader=new StreamReader(responseStream,Encoding.Default); + sb.Append(reader.ReadToEnd()); + reader.Close(); + } + else + { + while(true) + { + charCount=responseStream.Read(buffer,0,buffer.Length); + if(0==charCount) break; + sb.Append(Encoding.ASCII.GetString(buffer,0,charCount)); + } + } + webResponse.Close(); + return new HttpNetResponse(sb.ToString(),strRequest,webResponse,true); + } + catch(WebException webException) + { + if(webException.Message.Contains("(404) Not Found")) + { + return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message); + } + lastWebException=webException; + Thread.Sleep(TIMEOUT_BETWEEN_RETRIES); + } + } + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("General failure with {0}",lastWebException.Message)); + return new HttpNetResponse(webResponse,strRequest,false,lastWebException.Message); + } + catch(Exception exception) + { + return new HttpNetResponse(webResponse,strRequest,false,exception.Message); + } + finally + { + MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncodingV3A[LEAVE]"); + } + } +// This one uses the random user agent generator + public static HttpNetResponse GetRequestNoEncodingV3B(String strRequest,CookieCollection cookieCollection=null,WebProxy webProxy=null) + { + HttpWebResponse webResponse=null; + WebException lastWebException=null; + ServicePointManager.Expect100Continue=true; + ServicePointManager.SecurityProtocol=SecurityProtocolType.Ssl3|SecurityProtocolType.Tls12|SecurityProtocolType.Tls11|SecurityProtocolType.Tls; + + try + { + MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestNoEncodingV3B[ENTER]{0}",strRequest)); + int MAX_RETRIES=5; + int TIMEOUT_BETWEEN_RETRIES=1000; + int charCount=0; + byte[] buffer=new byte[8192]; + StringBuilder sb=new StringBuilder(); + + for(int count=0;count=userAgents.Length) index=userAgents.Length-1; + String userAgent=userAgents[index]; + webRequest.UserAgent=userAgent; + + try + { + webResponse=(HttpWebResponse)webRequest.GetResponse(); + Stream responseStream=webResponse.GetResponseStream(); + if(webResponse.ContentEncoding.ToLower().Contains("gzip")) + { + responseStream=new GZipStream(responseStream,CompressionMode.Decompress); + StreamReader reader=new StreamReader(responseStream,Encoding.Default); + sb.Append(reader.ReadToEnd()); + reader.Close(); + } + else if(webResponse.ContentEncoding.ToLower().Contains("deflate")) + { + responseStream=new DeflateStream(responseStream,CompressionMode.Decompress); + StreamReader reader=new StreamReader(responseStream,Encoding.Default); + sb.Append(reader.ReadToEnd()); + reader.Close(); + } + else + { + while(true) + { + charCount=responseStream.Read(buffer,0,buffer.Length); + if(0==charCount) break; + sb.Append(Encoding.ASCII.GetString(buffer,0,charCount)); + } + } + webResponse.Close(); + return new HttpNetResponse(sb.ToString(),strRequest,webResponse,true); + } + catch(WebException webException) + { + if(webException.Message.Contains("(404) Not Found")) + { + return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message); + } + lastWebException=webException; + Thread.Sleep(TIMEOUT_BETWEEN_RETRIES); + } + } + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("General failure with {0}",lastWebException.Message)); + return new HttpNetResponse(webResponse,strRequest,false,lastWebException.Message); + } + catch(Exception exception) + { + return new HttpNetResponse(webResponse,strRequest,false,exception.Message); + } + finally + { + MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncodingV3C[LEAVE]"); + } + } + public static HttpNetResponse GetRequestNoEncodingV4(String strRequest,CookieCollection cookieCollection=null,WebProxy webProxy=null) + { + HttpWebResponse webResponse=null; + try + { + MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestNoEncodingV4[ENTER]{0}",strRequest)); + int charCount = 0; + byte[] buffer = new byte[8192]; + StringBuilder sb = new StringBuilder(); + bool expect100Condition=ServicePointManager.Expect100Continue; + SecurityProtocolType securityProtocolType= ServicePointManager.SecurityProtocol; + ServicePointManager.Expect100Continue = true; + ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12; + HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest)); + if(null!=webProxy) webRequest.Proxy=webProxy; + webRequest.Timeout = REQUEST_TIMEOUT; + webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5"); + webRequest.Headers.Add("Accept-Encoding: None"); + webRequest.Accept = "image/gif, image/jpeg, image/pjpeg, image/pjpeg, application/x-shockwave-flash, application/x-ms-application, application/x-ms-xbap, application/vnd.ms-xpsdocument, application/xaml+xml, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*"; + webRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; MS-RTC LM 8; .NET4.0C; .NET4.0E; InfoPath.3)"; + webRequest.KeepAlive = true; + webRequest.CookieContainer=new CookieContainer(); + if(null!=cookieCollection)webRequest.CookieContainer.Add(cookieCollection); + webResponse = (HttpWebResponse)webRequest.GetResponse(); + Stream responseStream = webResponse.GetResponseStream(); + while (true) + { + charCount = responseStream.Read(buffer, 0, buffer.Length); + if (0 == charCount) break; + sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount)); + } + webResponse.Close(); + ServicePointManager.Expect100Continue = expect100Condition; + ServicePointManager.SecurityProtocol = securityProtocolType; + return new HttpNetResponse(sb.ToString(),strRequest,webResponse,webResponse.Cookies,true); + } + catch (WebException webException) + { + return new HttpNetResponse((HttpWebResponse)webException.Response,strRequest,false,webException.Message); + } + catch (Exception exception) + { + return new HttpNetResponse(webResponse,strRequest,false,exception.Message); + } + finally + { + MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncodingV4[LEAVE]"); + } + } + public static HttpNetResponse GetRequestNoEncodingV4A(String strRequest,CookieCollection cookieCollection=null,WebProxy webProxy=null) + { + HttpWebResponse webResponse=null; + try + { + MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestNoEncodingV4[ENTER]{0}",strRequest)); + int charCount=0; + byte[] buffer=new byte[8192]; + StringBuilder sb=new StringBuilder(); + bool expect100Condition=ServicePointManager.Expect100Continue; + SecurityProtocolType securityProtocolType=ServicePointManager.SecurityProtocol; + ServicePointManager.Expect100Continue=true; + ServicePointManager.SecurityProtocol=SecurityProtocolType.Tls12; + HttpWebRequest webRequest=(HttpWebRequest)WebRequest.Create(new Uri(strRequest)); + if(null!=webProxy)webRequest.Proxy=webProxy; + webRequest.Timeout=REQUEST_TIMEOUT; + webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5"); + webRequest.Headers.Add("Accept-Encoding: gzip, deflate, br"); + webRequest.Accept="text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"; + webRequest.UserAgent="Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:82.0) Gecko/20100101 Firefox/82.0"; + webRequest.KeepAlive=true; + webRequest.CookieContainer=new CookieContainer(); + if(null!=cookieCollection) webRequest.CookieContainer.Add(cookieCollection); + webResponse=(HttpWebResponse)webRequest.GetResponse(); + Stream responseStream=webResponse.GetResponseStream(); + if(webResponse.ContentEncoding.ToLower().Contains("gzip")) + { + responseStream=new GZipStream(responseStream,CompressionMode.Decompress); + StreamReader reader=new StreamReader(responseStream,Encoding.Default); + sb.Append(reader.ReadToEnd()); + reader.Close(); + } + else + { + while(true) + { + charCount=responseStream.Read(buffer,0,buffer.Length); + if(0==charCount) break; + sb.Append(Encoding.ASCII.GetString(buffer,0,charCount)); + } + } + webResponse.Close(); + ServicePointManager.Expect100Continue=expect100Condition; + ServicePointManager.SecurityProtocol=securityProtocolType; + return new HttpNetResponse(sb.ToString(),strRequest,webResponse,webResponse.Cookies,true); + } + catch(WebException webException) + { + return new HttpNetResponse((HttpWebResponse)webException.Response,strRequest,false,webException.Message); + } + catch(Exception exception) + { + return new HttpNetResponse(webResponse,strRequest,false,exception.Message); + } + finally + { + MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncodingV4[LEAVE]"); + } + } + + public static HttpNetResponse GetRequestNoEncodingV5(String strRequest,int webRequestTimeoutMS,WebProxy webProxy=null) + { + HttpWebResponse webResponse = null; + try + { + MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV5[ENTER]{0}", strRequest)); + int charCount = 0; + byte[] buffer = new byte[8192]; + StringBuilder sb = new StringBuilder(); + bool expect100Condition = ServicePointManager.Expect100Continue; + SecurityProtocolType securityProtocolType = ServicePointManager.SecurityProtocol; + ServicePointManager.Expect100Continue = true; + ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12; + HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest)); + if(null!=webProxy)webRequest.Proxy=webProxy; + webRequest.Timeout = webRequestTimeoutMS; + webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5"); + webRequest.Headers.Add("Accept-Encoding: None"); + webRequest.Accept = "image/gif, image/jpeg, image/pjpeg, image/pjpeg, application/x-shockwave-flash, application/x-ms-application, application/x-ms-xbap, application/vnd.ms-xpsdocument, application/xaml+xml, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*"; + webRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; MS-RTC LM 8; .NET4.0C; .NET4.0E; InfoPath.3)"; + webRequest.KeepAlive = true; + webRequest.CookieContainer = new CookieContainer(); + webResponse = (HttpWebResponse)webRequest.GetResponse(); + Stream responseStream = webResponse.GetResponseStream(); + while (true) + { + charCount = responseStream.Read(buffer, 0, buffer.Length); + if (0 == charCount) break; + sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount)); + } + webResponse.Close(); + ServicePointManager.Expect100Continue = expect100Condition; + ServicePointManager.SecurityProtocol = securityProtocolType; + return new HttpNetResponse(sb.ToString(), strRequest, webResponse, webResponse.Cookies, true); + } + catch (WebException webException) + { + return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message); + } + catch (Exception exception) + { + return new HttpNetResponse(webResponse, strRequest, false, exception.Message); + } + finally + { + MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV5[LEAVE]"); + } + } + + public static HttpNetResponse GetRequestNoEncodingV5A(String strRequest,int webRequestTimeoutMS,WebProxy webProxy=null) + { + HttpWebResponse webResponse = null; + try + { + Random random = new Random(); + String[] userAgents= + { + "Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.0", + "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:82.0) Gecko/20100101 Firefox/82.0" + }; + + MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV5[ENTER]{0}", strRequest)); + int charCount = 0; + byte[] buffer = new byte[8192]; + StringBuilder sb = new StringBuilder(); + bool expect100Condition = ServicePointManager.Expect100Continue; + SecurityProtocolType securityProtocolType = ServicePointManager.SecurityProtocol; + ServicePointManager.Expect100Continue = true; + ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12; + HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest)); + if(null!=webProxy)webRequest.Proxy=webProxy; + webRequest.Timeout = webRequestTimeoutMS; + webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5"); + webRequest.Headers.Add("Accept-Encoding: None"); + webRequest.Accept = "image/gif, image/jpeg, image/pjpeg, image/pjpeg, application/x-shockwave-flash, application/x-ms-application, application/x-ms-xbap, application/vnd.ms-xpsdocument, application/xaml+xml, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*"; +// webRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; MS-RTC LM 8; .NET4.0C; .NET4.0E; InfoPath.3)"; + webRequest.KeepAlive = true; + webRequest.CookieContainer = new CookieContainer(); + + double randomNumber=random.NextDouble(); // number between 0 and 1 + int index=(int)(randomNumber*((double)userAgents.Length)); + if(index>=userAgents.Length) index=userAgents.Length-1; + String userAgent=userAgents[index]; + webRequest.UserAgent=userAgent; + + + webResponse = (HttpWebResponse)webRequest.GetResponse(); + Stream responseStream = webResponse.GetResponseStream(); + while (true) + { + charCount = responseStream.Read(buffer, 0, buffer.Length); + if (0 == charCount) break; + sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount)); + } + webResponse.Close(); + ServicePointManager.Expect100Continue = expect100Condition; + ServicePointManager.SecurityProtocol = securityProtocolType; + return new HttpNetResponse(sb.ToString(), strRequest, webResponse, webResponse.Cookies, true); + } + catch (WebException webException) + { + return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message); + } + catch (Exception exception) + { + return new HttpNetResponse(webResponse, strRequest, false, exception.Message); + } + finally + { + MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV5[LEAVE]"); + } + } + + public static HttpNetResponse GetRequestNoEncodingV5B(String strRequest,int webRequestTimeoutMS,WebProxy webProxy=null,bool useRandomUserAgent=false,CookieCollection cookieCollection=null) + { + HttpWebResponse webResponse = null; + try + { + MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV5[ENTER]{0}", strRequest)); + int charCount = 0; + byte[] buffer = new byte[8192]; + StringBuilder sb = new StringBuilder(); + bool expect100Condition = ServicePointManager.Expect100Continue; + SecurityProtocolType securityProtocolType = ServicePointManager.SecurityProtocol; + ServicePointManager.Expect100Continue = true; + ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12; + HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest)); + if(null!=webProxy)webRequest.Proxy=webProxy; + webRequest.Timeout = webRequestTimeoutMS; + webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5"); + webRequest.Headers.Add("Accept-Encoding: None"); + webRequest.Accept = "image/gif, image/jpeg, image/pjpeg, image/pjpeg, application/x-shockwave-flash, application/x-ms-application, application/x-ms-xbap, application/vnd.ms-xpsdocument, application/xaml+xml, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*"; +// if(useRandomUserAgent)webRequest.UserAgent=GetUserAgent(); + if (useRandomUserAgent) webRequest.UserAgent = UserAgent.GetInstance().GetUserAgent(); + else webRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; MS-RTC LM 8; .NET4.0C; .NET4.0E; InfoPath.3)"; + webRequest.KeepAlive = true; + webRequest.CookieContainer = new CookieContainer(); + if(null!=cookieCollection)webRequest.CookieContainer.Add(cookieCollection); + webResponse = (HttpWebResponse)webRequest.GetResponse(); + Stream responseStream = webResponse.GetResponseStream(); + while (true) + { + charCount = responseStream.Read(buffer, 0, buffer.Length); + if (0 == charCount) break; + sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount)); + } + webResponse.Close(); + ServicePointManager.Expect100Continue = expect100Condition; + ServicePointManager.SecurityProtocol = securityProtocolType; + return new HttpNetResponse(sb.ToString(), strRequest, webResponse, webResponse.Cookies, true); + } + catch (WebException webException) + { + return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message); + } + catch (Exception exception) + { + return new HttpNetResponse(webResponse, strRequest, false, exception.Message); + } + finally + { + MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV5[LEAVE]"); + } + } + + public static HttpNetResponse GetRequestNoEncodingV5C(String strRequest,String userName, String password) + { + HttpWebResponse webResponse = null; + try + { + MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV5C[ENTER]{0}", strRequest)); + int charCount = 0; + byte[] buffer = new byte[8192]; + StringBuilder sb = new StringBuilder(); + bool expect100Condition = ServicePointManager.Expect100Continue; + SecurityProtocolType securityProtocolType = ServicePointManager.SecurityProtocol; + ServicePointManager.Expect100Continue = true; + ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12; + HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest)); + webRequest.Credentials=new NetworkCredential(userName,password); + webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5"); + webRequest.Headers.Add("Accept-Encoding: None"); + webRequest.Accept = "image/gif, image/jpeg, image/pjpeg, image/pjpeg, application/x-shockwave-flash, application/x-ms-application, application/x-ms-xbap, application/vnd.ms-xpsdocument, application/xaml+xml, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*"; + webRequest.KeepAlive = true; + webResponse = (HttpWebResponse)webRequest.GetResponse(); + Stream responseStream = webResponse.GetResponseStream(); + while (true) + { + charCount = responseStream.Read(buffer, 0, buffer.Length); + if (0 == charCount) break; + sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount)); + } + webResponse.Close(); + ServicePointManager.Expect100Continue = expect100Condition; + ServicePointManager.SecurityProtocol = securityProtocolType; + return new HttpNetResponse(sb.ToString(), strRequest, webResponse, webResponse.Cookies, true); + } + catch (WebException webException) + { + return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message); + } + catch (Exception exception) + { + return new HttpNetResponse(webResponse, strRequest, false, exception.Message); + } + finally + { + MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV5C[LEAVE]"); + } + } + + + public static HttpNetResponse GetRequestNoEncodingV6(String strRequest, int webRequestTimeoutMS) + { + HttpWebResponse webResponse = null; + try + { + MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV6[ENTER]{0}", strRequest)); + int charCount = 0; + byte[] buffer = new byte[8192]; + StringBuilder sb = new StringBuilder(); + bool expect100Condition = ServicePointManager.Expect100Continue; + SecurityProtocolType securityProtocolType = ServicePointManager.SecurityProtocol; + ServicePointManager.Expect100Continue = true; + ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12; + HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest)); + webRequest.Timeout = webRequestTimeoutMS; + webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5"); + webRequest.Headers.Add("Accept-Encoding: gzip, deflate, br"); + webRequest.Accept = "image/gif, image/jpeg, image/pjpeg, image/pjpeg, application/x-shockwave-flash, application/x-ms-application, application/x-ms-xbap, application/vnd.ms-xpsdocument, application/xaml+xml, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*"; + webRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; MS-RTC LM 8; .NET4.0C; .NET4.0E; InfoPath.3)"; + webRequest.KeepAlive = true; + webRequest.CookieContainer = new CookieContainer(); + webResponse = (HttpWebResponse)webRequest.GetResponse(); + Stream responseStream = webResponse.GetResponseStream(); + while (true) + { + charCount = responseStream.Read(buffer, 0, buffer.Length); + if (0 == charCount) break; + sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount)); + } + webResponse.Close(); + ServicePointManager.Expect100Continue = expect100Condition; + ServicePointManager.SecurityProtocol = securityProtocolType; + return new HttpNetResponse(sb.ToString(), strRequest, webResponse, webResponse.Cookies, true); + } + catch (WebException webException) + { + return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message); + } + catch (Exception exception) + { + return new HttpNetResponse(webResponse, strRequest, false, exception.Message); + } + finally + { + MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV6[LEAVE]"); + } + } + + public static HttpNetResponse GetRequestNoEncodingV7(String strRequest,WebProxy webProxy=null,CookieCollection cookieCollection=null) + { + HttpWebResponse webResponse = null; + try + { + MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV7[ENTER]{0}", strRequest)); + int charCount = 0; + byte[] buffer = new byte[8192]; + StringBuilder sb = new StringBuilder(); + bool expect100Condition = ServicePointManager.Expect100Continue; + SecurityProtocolType securityProtocolType = ServicePointManager.SecurityProtocol; + ServicePointManager.Expect100Continue = true; + ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12; + HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest)); + if(null!=webProxy)webRequest.Proxy=webProxy; + webRequest.KeepAlive=true; + webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5"); + webRequest.Headers.Add("Accept-Encoding: gzip, deflate, br"); + webRequest.Headers.Add("Upgrade-Insecure-Requests: 1"); + webRequest.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"; + webRequest.UserAgent = " Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:76.0) Gecko/20100101 Firefox/76.0"; + webRequest.KeepAlive = true; + webRequest.CookieContainer = new CookieContainer(); + if(null!=cookieCollection)webRequest.CookieContainer.Add(cookieCollection); + webResponse = (HttpWebResponse)webRequest.GetResponse(); + Stream responseStream = webResponse.GetResponseStream(); + if (webResponse.ContentEncoding.ToLower().Contains("gzip")) + { + responseStream = new GZipStream(responseStream, CompressionMode.Decompress); + StreamReader reader = new StreamReader(responseStream, Encoding.Default); + sb.Append(reader.ReadToEnd()); + reader.Close(); + } + else if (webResponse.ContentEncoding.ToLower().Contains("deflate")) + { + responseStream = new DeflateStream(responseStream, CompressionMode.Decompress); + StreamReader reader = new StreamReader(responseStream, Encoding.Default); + sb.Append(reader.ReadToEnd()); + reader.Close(); + } + else + { + while (true) + { + charCount = responseStream.Read(buffer, 0, buffer.Length); + if (0 == charCount) break; + sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount)); + } + } + + webResponse.Close(); + ServicePointManager.Expect100Continue = expect100Condition; + ServicePointManager.SecurityProtocol = securityProtocolType; + return new HttpNetResponse(sb.ToString(), strRequest, webResponse, webResponse.Cookies, true); + } + catch (WebException webException) + { + return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message); + } + catch (Exception exception) + { + return new HttpNetResponse(webResponse, strRequest, false, exception.Message); + } + finally + { + MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV7[LEAVE]"); + } + } +// This is specifically taylored to Morningstar site for retrieval of historical data + public static HttpNetResponse GetRequestNoEncodingMStar(String strRequest,WebProxy webProxy=null) + { + HttpWebResponse webResponse = null; + try + { + MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingMStar[ENTER]{0}", strRequest)); + int charCount = 0; + byte[] buffer = new byte[8192]; + StringBuilder sb = new StringBuilder(); + bool expect100Condition = ServicePointManager.Expect100Continue; + SecurityProtocolType securityProtocolType = ServicePointManager.SecurityProtocol; + ServicePointManager.Expect100Continue = true; + ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12; + HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest)); + webRequest.KeepAlive=true; + webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5"); + webRequest.Headers.Add("Accept-Encoding: gzip, deflate, br"); + webRequest.Headers.Add("DNT: 1"); + webRequest.Headers.Add("Sec-Fetch-Dest:empty"); + webRequest.Headers.Add("Sec-Fetch-Mode:cors"); + webRequest.Headers.Add("Sec-Fetch-Site:same-site"); + webRequest.Headers.Add("X-SAL-ContentType:e7FDDltrTy+tA2HnLovvGL0LFMwT+KkEptGju5wXVTU="); + webRequest.Headers.Add("X-API-RequestId: d2ffea42-4a07-4fa7-fa0d-b307facd81fb"); + webRequest.Headers.Add("ApiKey:lstzFDEOhfFNMLikKa0am9mgEKLBl49T"); + webRequest.Accept = "*/*"; + webRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36"; + webRequest.Host="api-global.morningstar.com"; + webRequest.Referer="https://www.morningstar.com/stocks/xnas/aapl/performance"; + + if(null!=webProxy)webRequest.Proxy=webProxy; + + webRequest.CookieContainer = new CookieContainer(); + webResponse = (HttpWebResponse)webRequest.GetResponse(); + Stream responseStream = webResponse.GetResponseStream(); + if (webResponse.ContentEncoding.ToLower().Contains("gzip")) + { + responseStream = new GZipStream(responseStream, CompressionMode.Decompress); + StreamReader reader = new StreamReader(responseStream, Encoding.Default); + sb.Append(reader.ReadToEnd()); + reader.Close(); + } + else if (webResponse.ContentEncoding.ToLower().Contains("deflate")) + { + responseStream = new DeflateStream(responseStream, CompressionMode.Decompress); + StreamReader reader = new StreamReader(responseStream, Encoding.Default); + sb.Append(reader.ReadToEnd()); + reader.Close(); + } + else + { + while (true) + { + charCount = responseStream.Read(buffer, 0, buffer.Length); + if (0 == charCount) break; + sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount)); + } + } + + webResponse.Close(); + ServicePointManager.Expect100Continue = expect100Condition; + ServicePointManager.SecurityProtocol = securityProtocolType; + return new HttpNetResponse(sb.ToString(), strRequest, webResponse, webResponse.Cookies, true); + } + catch (WebException webException) + { + return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message); + } + catch (Exception exception) + { + return new HttpNetResponse(webResponse, strRequest, false, exception.Message); + } + finally + { + MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingMStar[LEAVE]"); + } + } + + /// + /// Get the proxy for the specified service or null if it is not congfigured to use the proxy + /// + ///The service name + public static WebProxy GetProxy(String serviceName) + { + try + { + String proxyAddress=ConfigurationManager.AppSettings["proxy_address"]; + if(null==proxyAddress) return null; + String useServiceProxy=ConfigurationManager.AppSettings["proxy_"+serviceName]; + if(null==useServiceProxy||false==bool.Parse(useServiceProxy)) return null; + String[] addressParts=proxyAddress.Split(':'); + SocketControl socketControl=null; + try + { + socketControl=new SocketControl(addressParts[1].Replace("//",null),int.Parse(addressParts[2])); + if(!socketControl.IsConnected())throw new Exception("Unreachable destination "+proxyAddress); + } + catch(Exception exception) + { + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("****** ERROR: CANNOT REACH PROXY {0} FOR SERVICE {1}...{2}",proxyAddress,serviceName,exception.ToString())); + return null; + } + socketControl.Close(); + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("USING PROXY:{0} FOR {1}",proxyAddress,serviceName)); + Uri newUri=new Uri(proxyAddress); + WebProxy proxy=new WebProxy(); + proxy.Address=newUri; + return proxy; + } + catch(Exception exception) + { + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetProxy: Exception:{0}",exception.ToString())); + return null; + } + } + + /// + /// Get the proxy. This is only to be used as part of unit tests to test out the TOR Proxy. User code should use the GetProxy(serviceName) variation + /// + public static WebProxy GetProxy() + { + String proxyAddress=ConfigurationManager.AppSettings["proxy_address"]; + String[] addressParts=proxyAddress.Split(':'); + SocketControl socketControl=null; + try + { + socketControl=new SocketControl(addressParts[1].Replace("//",null),int.Parse(addressParts[2])); + if(!socketControl.IsConnected())throw new Exception("Unreachable destination "+proxyAddress); + } + catch(Exception exception) + { + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("****** ERROR: CANNOT REACH PROXY. {0} {1} ",proxyAddress,exception.ToString())); + return null; + } + socketControl.Close(); + MDTrace.WriteLine(LogLevel.DEBUG,String.Format("USING PROXY:{0} ",proxyAddress)); + Uri newUri=new Uri(proxyAddress); + WebProxy proxy=new WebProxy(); + proxy.Address=newUri; + return proxy; + } + } +}