#pragma warning disable SYSLIB0014
using System.Net;
using System.Text;
using System.IO.Compression;
using System.Configuration;
using MarketData.Configuration;
using MarketData.Utils;
// Filename: HttpRequest.cs
// Author:Sean Kessler
namespace MarketData.Integration
{
public class HttpNetResponse : IDisposable
{
public enum TypeResponse {Stream,String};
public String ResponseString{get;set;}
public MemoryStream ResponseStream{get;set;}
public CookieCollection CookieCollection{get;set;}
public HttpNetResponse()
{
Success=false;
}
public HttpNetResponse(String responseString,String request,HttpWebResponse webResponse,CookieCollection cookieCollection,bool success)
{
ResponseString=responseString;
ResponseType=TypeResponse.String;
Success=success;
Request=request;
CookieCollection=cookieCollection;
if(null!=webResponse)StatusCode=webResponse.StatusCode;
}
public HttpNetResponse(String responseString,String request,HttpWebResponse webResponse,bool success)
{
ResponseString=responseString;
ResponseType=TypeResponse.String;
Success=success;
Request=request;
if(null!=webResponse)StatusCode=webResponse.StatusCode;
}
public HttpNetResponse(MemoryStream responseStream,String request,HttpWebResponse webResponse,bool success)
{
ResponseStream=responseStream;
ResponseType=TypeResponse.String;
Success=success;
Request=request;
if(null!=webResponse)StatusCode=webResponse.StatusCode;
}
public HttpNetResponse(HttpWebResponse webResponse,String request,bool success,String errorMessage)
{
ResponseType=TypeResponse.String;
Success=success;
Request=request;
if(null!=webResponse)StatusCode=webResponse.StatusCode;
ErrorMessage=errorMessage;
}
public Stream ResponseStringToStream()
{
MemoryStream stream = new MemoryStream();
StreamWriter writer = new StreamWriter(stream);
writer.Write(ResponseString);
writer.Flush();
stream.Position = 0;
return stream;
}
public String ResponseStreamToString
{
get
{
if(!Success)return null;
return System.Text.Encoding.UTF8.GetString(ResponseStream.ToArray());
}
}
public TypeResponse ResponseType{get;set;}
public bool Success{get;set;}
public String ErrorMessage{get;set;}
public HttpStatusCode StatusCode{get;set;}
public String Request{get;set;}
public bool IsNetErrorResponse
{
get
{
return (int)StatusCode>=400?true:false;
}
}
public void Dispose()
{
if(null!=ResponseStream)
{
ResponseStream.Close();
ResponseStream.Dispose();
ResponseStream=null;
}
}
}
// *********************************************************************************************************************************************************************
public class HttpNetRequest
{
public static int REQUEST_TIMEOUT = 300000;
private HttpNetRequest()
{
}
public static CookieCollection GetCookies(String strRequest)
{
HttpWebResponse webResponse = null;
try
{
Uri uri = new Uri(strRequest);
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(uri);
webRequest.Timeout = REQUEST_TIMEOUT;
webRequest.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
webRequest.Headers.Add("Accept-Encoding: gzip, deflate");
webRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0";
webRequest.KeepAlive = true;
webResponse = (HttpWebResponse)webRequest.GetResponse();
CookieContainer cookieContainer = new CookieContainer();
if (webResponse.Headers.AllKeys.Contains("Set-Cookie"))
{
String[] cookies = webResponse.Headers.GetValues("Set-Cookie");
foreach (String strCookie in cookies)
{
String cookieName = strCookie.Substring(0, strCookie.IndexOf('='));
String cookieValue = Utility.BetweenString(strCookie, "=", ";");
String metaData = strCookie.Substring(strCookie.IndexOf(';')+1);
String[] metaDataElements = metaData.Split(';');
String domain = metaDataElements.Where(x => x.Contains("Domain",StringComparison.InvariantCultureIgnoreCase)).FirstOrDefault();
if (default != domain) domain = Utility.BetweenString(domain, "=", null);
String path = metaDataElements.Where(x => x.Contains("Path", StringComparison.InvariantCultureIgnoreCase)).FirstOrDefault();
if (default != path) path = Utility.BetweenString(path, "=", null);
Cookie cookie = default;
if (null != path && null != domain) cookie = new Cookie(cookieName, cookieValue, path, domain);
else if (null != path) cookie = new Cookie(cookieName, cookieValue, path);
else cookie = new Cookie(cookieName, cookieValue);
cookieContainer.Add(uri, cookie);
}
}
return cookieContainer.GetCookies(uri);
}
catch (Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG, $"Exception {exception.ToString()}");
return null;
}
finally
{
if (null != webResponse)
{
webResponse.Close();
webResponse.Dispose();
}
}
}
public static HttpNetResponse GetRequestStreamZIP(String strRequest)
{
HttpWebResponse webResponse = null;
try
{
MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestStreamZIP[ENTER]{0}", strRequest));
int charCount = 0;
byte[] buffer = new byte[8192];
StringBuilder sb = new StringBuilder();
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
webRequest.Timeout = REQUEST_TIMEOUT;
webRequest.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
webRequest.Headers.Add("Accept-Encoding: gzip, deflate");
webRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0";
webRequest.KeepAlive = true;
try { webResponse = (HttpWebResponse)webRequest.GetResponse(); }
catch (WebException webException)
{
if (IsMovedException(webException))
{
webRequest = Redirect(webRequest, webException);
webResponse = (HttpWebResponse)webRequest.GetResponse();
}
else throw;
}
Stream responseStream = webResponse.GetResponseStream();
MemoryStream memoryStream = new MemoryStream();
while (true)
{
charCount = responseStream.Read(buffer, 0, buffer.Length);
if (charCount <= 0) break;
memoryStream.Write(buffer, 0, charCount);
}
memoryStream.Flush();
return new HttpNetResponse(memoryStream, strRequest, webResponse, true);
}
catch (WebException webException)
{
return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message);
}
catch (Exception exception)
{
return new HttpNetResponse(webResponse, strRequest, false, exception.Message);
}
finally
{
if (null != webResponse)
{
webResponse.Close();
webResponse.Dispose();
}
MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestStreamZIP[LEAVE]");
}
}
public static HttpNetResponse GetRequestStreamZIPV2(String strRequest)
{
HttpWebResponse webResponse=null;
try
{
MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestStreamZIPV2[ENTER]{0}",strRequest));
int charCount = 0;
byte[] buffer = new byte[8192];
StringBuilder sb = new StringBuilder();
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
webRequest.Timeout = REQUEST_TIMEOUT;
webRequest.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
webRequest.Headers.Add("Accept-Encoding: gzip, deflate");
webRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:59.0) Gecko/20100101 Firefox/59.0";
webRequest.KeepAlive = true;
webRequest.CookieContainer=new CookieContainer();
try{webResponse = (HttpWebResponse)webRequest.GetResponse();}
catch(WebException webException)
{
if(IsMovedException(webException))
{
webRequest=Redirect(webRequest, webException);
webResponse=(HttpWebResponse)webRequest.GetResponse();
}
else throw;
}
Stream responseStream = webResponse.GetResponseStream();
MemoryStream memoryStream=new MemoryStream();
while (true)
{
charCount = responseStream.Read(buffer, 0, buffer.Length);
if(charCount<=0)break;
memoryStream.Write(buffer,0,charCount);
}
memoryStream.Flush();
return new HttpNetResponse(memoryStream,strRequest,webResponse,true);
}
catch (WebException webException)
{
return new HttpNetResponse((HttpWebResponse)webException.Response,strRequest,false,webException.Message);
}
catch (Exception exception)
{
return new HttpNetResponse(webResponse,strRequest,false,exception.Message);
}
finally
{
if(null!=webResponse)
{
webResponse.Close();
webResponse.Dispose();
}
MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestStreamZIPV2[LEAVE]");
}
}
public static HttpNetResponse GetRequestStreamCSV(String strRequest,CookieCollection cookieCollection=null, WebProxy webProxy=null)
{
HttpWebResponse webResponse=null;
try
{
MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestStreamCSV[ENTER]{0}",strRequest));
int charCount = 0;
byte[] buffer = new byte[8192];
StringBuilder sb = new StringBuilder();
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
if(null!=webProxy)
{
webRequest.Proxy=webProxy;
}
if(null!=cookieCollection)
{
webRequest.CookieContainer=new CookieContainer();
webRequest.CookieContainer.Add(cookieCollection);
}
webRequest.Timeout = REQUEST_TIMEOUT;
webRequest.Accept = "text/csv,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
webRequest.Headers.Add("Accept-Encoding: gzip, deflate"); //
webRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0";
webRequest.KeepAlive = true;
try{webResponse = (HttpWebResponse)webRequest.GetResponse();}
catch(WebException webException)
{
if(IsMovedException(webException))
{
webRequest=Redirect(webRequest, webException);
webResponse=(HttpWebResponse)webRequest.GetResponse();
}
else throw;
}
Stream responseStream = webResponse.GetResponseStream();
if(webResponse.ContentEncoding.ToLower().Contains("gzip"))
{
responseStream = new GZipStream(responseStream, CompressionMode.Decompress);
StreamReader reader = new StreamReader(responseStream, Encoding.Default);
sb.Append(reader.ReadToEnd());
reader.Close();
}
else if(webResponse.ContentEncoding.ToLower().Contains("deflate"))
{
responseStream = new DeflateStream(responseStream, CompressionMode.Decompress);
StreamReader reader = new StreamReader(responseStream, Encoding.Default);
sb.Append(reader.ReadToEnd());
reader.Close();
}
else
{
while (true)
{
charCount = responseStream.Read(buffer, 0, buffer.Length);
if (0 == charCount) break;
sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
}
}
return new HttpNetResponse(sb.ToString(),strRequest,webResponse,true);
}
catch (WebException webException)
{
return new HttpNetResponse((HttpWebResponse)webException.Response,strRequest,false,webException.Message);
}
catch (Exception exception)
{
return new HttpNetResponse(webResponse,strRequest,false,exception.Message);
}
finally
{
if(null!=webResponse)
{
webResponse.Close();
webResponse.Dispose();
}
MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestStreamCSV[LEAVE]");
}
}
public static HttpNetResponse GetRequestNoEncoding(String strRequest,WebProxy webProxy=null)
{
HttpWebResponse webResponse=null;
try
{
ServicePointManager.Expect100Continue = true;
MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestNoEncoding[ENTER]{0}",strRequest));
int charCount = 0;
byte[] buffer = new byte[8192];
StringBuilder sb = new StringBuilder();
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
if(null!=webProxy)webRequest.Proxy=webProxy;
webRequest.Timeout = REQUEST_TIMEOUT;
webRequest.Accept = "text/html, text/csv";
webRequest.ContentType="text/csv; charset=utf-8";
webRequest.Headers.Add("Accept-Encoding: None");
webRequest.Headers.Add("Accept-Language: en-US");
webRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0";
webRequest.KeepAlive = true;
try{webResponse = (HttpWebResponse)webRequest.GetResponse();}
catch(WebException webException)
{
if(IsMovedException(webException))
{
webRequest=Redirect(webRequest, webException);
webResponse=(HttpWebResponse)webRequest.GetResponse();
}
else throw;
}
Stream responseStream = webResponse.GetResponseStream();
while (true)
{
charCount = responseStream.Read(buffer, 0, buffer.Length);
if (0 >= charCount) break;
sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
}
return new HttpNetResponse(sb.ToString(),strRequest,webResponse,true);
}
catch (WebException webException)
{
return new HttpNetResponse((HttpWebResponse)webException.Response,strRequest,false,webException.Message);
}
catch (Exception exception)
{
return new HttpNetResponse(webResponse,strRequest,false,exception.Message);
}
finally
{
if(null!=webResponse)
{
webResponse.Close();
webResponse.Dispose();
}
MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncoding[LEAVE]");
}
}
public static HttpNetResponse GetRequestNoEncodingV1(String strRequest,WebProxy webProxy=null)
{
ServicePointManager.Expect100Continue=true;
HttpWebResponse webResponse=null;
try
{
MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestNoEncoding[ENTER]{0}",strRequest));
int charCount=0;
byte[] buffer=new byte[8192];
StringBuilder sb=new StringBuilder();
HttpWebRequest webRequest=(HttpWebRequest)WebRequest.Create(new Uri(strRequest));
if(null!=webProxy)webRequest.Proxy=webProxy;
webRequest.Timeout=REQUEST_TIMEOUT;
webRequest.Accept="text/html, text/csv";
webRequest.ContentType="text/csv; charset=utf-8";
webRequest.Headers.Add("Accept-Encoding: None");
webRequest.Headers.Add("Accept-Language: en-US");
webRequest.UserAgent="Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0";
webRequest.KeepAlive=true;
try{webResponse=(HttpWebResponse)webRequest.GetResponse();}
catch(WebException webException)
{
if(IsMovedException(webException))
{
webRequest=Redirect(webRequest, webException);
webResponse=(HttpWebResponse)webRequest.GetResponse();
}
else throw;
}
Stream responseStream=webResponse.GetResponseStream();
while(true)
{
charCount=responseStream.Read(buffer,0,buffer.Length);
if(0>=charCount) break;
sb.Append(Encoding.ASCII.GetString(buffer,0,charCount));
}
return new HttpNetResponse(sb.ToString(),strRequest,webResponse,true);
}
catch(WebException webException)
{
return new HttpNetResponse((HttpWebResponse)webException.Response,strRequest,false,webException.Message);
}
catch(Exception exception)
{
return new HttpNetResponse(webResponse,strRequest,false,exception.Message);
}
finally
{
if(null!=webResponse)
{
webResponse.Close();
webResponse.Dispose();
}
MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncoding[LEAVE]");
}
}
///
/// This one is being used in the MarketWatch price fetch
///
///
///
///
///
///
public static HttpNetResponse GetRequestNoEncodingV2(String strRequest, CookieCollection cookieCollection, Uri uri, WebProxy webProxy = null)
{
HttpWebResponse webResponse = null;
try
{
ServicePointManager.Expect100Continue = true;
MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV2[ENTER]{0}", strRequest));
int charCount = 0;
byte[] buffer = new byte[8192];
StringBuilder sb = new StringBuilder();
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
if (null != webProxy) webRequest.Proxy = webProxy;
webRequest.Timeout = REQUEST_TIMEOUT;
webRequest.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7";
webRequest.Headers.Add("Accept-Encoding: gzip, deflate, br, zstd");
webRequest.Headers.Add("Accept-Language: en-US,en;q=0.9");
webRequest.UserAgent = "Mozilla/5.0 (X11; CrOS x86_64 14541.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36";
webRequest.Headers.Add("sec-fetch-dest", "document");
webRequest.Headers.Add("sec-fetch-mode", "navigate");
webRequest.Headers.Add("sec-fetch-site", "none");
webRequest.Headers.Add("sec-fetch-user", "?1");
webRequest.Headers.Add("upgrade-insecure-requests", "1");
webRequest.Headers.Add("priority", "u=0, i");
webRequest.Headers.Add("sec-ch-device-memory", "8");
webRequest.Headers.Add("Referrer-Policy", "strict-origin-when-cross-origin");
webRequest.AllowAutoRedirect = true;
if (null != cookieCollection)
{
CookieContainer cookieContainer = new CookieContainer();
foreach (Cookie cookie in cookieCollection)
{
cookieContainer.Add(uri, cookie);
}
webRequest.CookieContainer = cookieContainer;
}
try { webResponse = (HttpWebResponse)webRequest.GetResponse(); }
catch (WebException webException)
{
if (IsMovedException(webException))
{
webRequest = Redirect(webRequest, webException);
webResponse = (HttpWebResponse)webRequest.GetResponse();
}
else throw;
}
Stream responseStream = webResponse.GetResponseStream();
if (webResponse.ContentEncoding.ToLower().Contains("gzip"))
{
responseStream = new GZipStream(responseStream, CompressionMode.Decompress);
StreamReader reader = new StreamReader(responseStream, Encoding.Default);
sb.Append(reader.ReadToEnd());
reader.Close();
}
else if (webResponse.ContentEncoding.ToLower().Contains("deflate"))
{
responseStream = new DeflateStream(responseStream, CompressionMode.Decompress);
StreamReader reader = new StreamReader(responseStream, Encoding.Default);
sb.Append(reader.ReadToEnd());
reader.Close();
}
else
{
while (true)
{
charCount = responseStream.Read(buffer, 0, buffer.Length);
if (0 == charCount) break;
sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
}
}
return new HttpNetResponse(sb.ToString(), strRequest, webResponse, true);
}
catch (WebException webException)
{
return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message);
}
catch (Exception exception)
{
return new HttpNetResponse(webResponse, strRequest, false, exception.Message);
}
finally
{
if (null != webResponse)
{
webResponse.Close();
webResponse.Dispose();
}
MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncoding[LEAVE]");
}
}
public static HttpNetResponse GetRequestCSV(String strRequest, String referer)
{
HttpWebResponse webResponse = null;
try
{
MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestCSV[ENTER]{0}", strRequest));
int charCount = 0;
byte[] buffer = new byte[8192];
StringBuilder sb = new StringBuilder();
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
webRequest.Timeout = REQUEST_TIMEOUT;
webRequest.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
webRequest.ContentType = "text/csv; charset=utf-8";
webRequest.Headers.Add("Accept-Encoding: identity"); // "gzip, deflate"
webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
webRequest.Headers.Add("Upgrade-Insecure-Requests: 1");
webRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0";
webRequest.Referer = referer;
webRequest.KeepAlive = true;
try { webResponse = (HttpWebResponse)webRequest.GetResponse(); }
catch (WebException webException)
{
if (IsMovedException(webException))
{
webRequest = Redirect(webRequest, webException);
webResponse = (HttpWebResponse)webRequest.GetResponse();
}
else throw;
}
Stream responseStream = webResponse.GetResponseStream();
while (true)
{
charCount = responseStream.Read(buffer, 0, buffer.Length);
if (0 == charCount) break;
sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
}
return new HttpNetResponse(sb.ToString(), strRequest, webResponse, true);
}
catch (WebException webException)
{
return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message);
}
catch (Exception exception)
{
return new HttpNetResponse(webResponse, strRequest, false, exception.Message);
}
finally
{
if (null != webResponse)
{
webResponse.Close();
webResponse.Dispose();
}
MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestCSV[LEAVE]");
}
}
public static HttpNetResponse GetRequestNoEncodingV2(String strRequest)
{
HttpWebResponse webResponse=null;
try
{
MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestNoEncodingV2[ENTER]{0}",strRequest));
int charCount = 0;
byte[] buffer = new byte[8192];
StringBuilder sb = new StringBuilder();
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
webRequest.Timeout = REQUEST_TIMEOUT;
webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
webRequest.ContentType = "text/csv; charset=utf-8";
webRequest.Headers.Add("Accept-Encoding: gzip, deflate, br, zstd");
webRequest.UserAgent = "Mozilla/5.0 (X11; CrOS x86_64 14541.0.0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36";
webRequest.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7";
webRequest.KeepAlive = true;
try{webResponse = (HttpWebResponse)webRequest.GetResponse();}
catch(WebException webException)
{
if(IsMovedException(webException))
{
webRequest=Redirect(webRequest, webException);
webResponse=(HttpWebResponse)webRequest.GetResponse();
}
else throw;
}
Stream responseStream = webResponse.GetResponseStream();
while (true)
{
charCount = responseStream.Read(buffer, 0, buffer.Length);
if (0 == charCount) break;
sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
}
return new HttpNetResponse(sb.ToString(),strRequest,webResponse,true);
}
catch (WebException webException)
{
return new HttpNetResponse((HttpWebResponse)webException.Response,strRequest,false,webException.Message);
}
catch (Exception exception)
{
return new HttpNetResponse(webResponse,strRequest,false,exception.Message);
}
finally
{
if(null!=webResponse)
{
webResponse.Close();
webResponse.Dispose();
}
MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncodingV2[LEAVE]");
}
}
// I am using this code specifically on the seeking alpha web site. it seems as though seeking alpha has implemeted some user agent based bot prevention mechanism on their website
// to prevent scrapers. What I do here is to choose from a set of commonly used user agents and then randomly choose a user agent to code into the request. retry logic up to 3 times
// and handling of 404 (not found)
public static HttpNetResponse GetRequestNoEncodingV3(String strRequest, String referer = null)
{
HttpWebResponse webResponse = null;
WebException lastWebException = null;
try
{
MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV3[ENTER]{0}", strRequest));
Random random = new Random();
String[] userAgents =
{
"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537."
};
int MAX_RETRIES = 5;
int TIMEOUT_BETWEEN_RETRIES = 1000;
int charCount = 0;
byte[] buffer = new byte[8192];
StringBuilder sb = new StringBuilder();
for (int count = 0; count < MAX_RETRIES; count++)
{
CookieContainer cookieContainer = new CookieContainer();
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
webRequest.CookieContainer = cookieContainer;
webRequest.Timeout = REQUEST_TIMEOUT;
webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
webRequest.Headers.Add("Accept-Encoding: gzip,deflate");
webRequest.Accept = "text/html";
String userAgent = userAgents[random.Next(0, userAgents.Length - 1)];
userAgent += random.Next(1, 57).ToString();
webRequest.UserAgent = userAgent;
if (null != referer) webRequest.Referer = referer;
try
{
try { webResponse = (HttpWebResponse)webRequest.GetResponse(); }
catch (WebException webException)
{
if (IsMovedException(webException))
{
webRequest = Redirect(webRequest, webException);
webResponse = (HttpWebResponse)webRequest.GetResponse();
}
else throw;
}
Stream responseStream = webResponse.GetResponseStream();
if (webResponse.ContentEncoding.ToLower().Contains("gzip"))
{
responseStream = new GZipStream(responseStream, CompressionMode.Decompress);
StreamReader reader = new StreamReader(responseStream, Encoding.Default);
sb.Append(reader.ReadToEnd());
reader.Close();
}
else if (webResponse.ContentEncoding.ToLower().Contains("deflate"))
{
responseStream = new DeflateStream(responseStream, CompressionMode.Decompress);
StreamReader reader = new StreamReader(responseStream, Encoding.Default);
sb.Append(reader.ReadToEnd());
reader.Close();
}
else
{
while (true)
{
charCount = responseStream.Read(buffer, 0, buffer.Length);
if (0 == charCount) break;
sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
}
}
return new HttpNetResponse(sb.ToString(), strRequest, webResponse, true);
}
catch (WebException webException)
{
if (webException.Message.Contains("(404) Not Found"))
{
return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message);
}
lastWebException = webException;
Thread.Sleep(TIMEOUT_BETWEEN_RETRIES);
}
}
MDTrace.WriteLine(LogLevel.DEBUG, String.Format("General failure with {0}", lastWebException.Message));
return new HttpNetResponse(webResponse, strRequest, false, lastWebException.Message);
}
catch (Exception exception)
{
return new HttpNetResponse(webResponse, strRequest, false, exception.Message);
}
finally
{
if (null != webResponse)
{
webResponse.Close();
webResponse.Dispose();
}
MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV3[LEAVE]");
}
}
// https://query1.finance.yahoo.com/v7/finance/chart/MIDD?period1=1606172400&period2=1606172400&interval=1d&events=history
public static HttpNetResponse GetRequestNoEncodingV3A(String strRequest,WebProxy webProxy=null)
{
HttpWebResponse webResponse=null;
WebException lastWebException=null;
ServicePointManager.Expect100Continue=true;
Random random=new Random();
try
{
MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestNoEncodingV3A[ENTER]{0}",strRequest));
String[] userAgents=
{
"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.0",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:82.0) Gecko/20100101 Firefox/82.0"
};
int MAX_RETRIES=5;
int TIMEOUT_BETWEEN_RETRIES=1000;
int charCount=0;
byte[] buffer=new byte[8192];
StringBuilder sb=new StringBuilder();
for(int count=0;count=userAgents.Length) index=userAgents.Length-1;
String userAgent=userAgents[index];
webRequest.UserAgent=userAgent;
try
{
try{webResponse=(HttpWebResponse)webRequest.GetResponse();}
catch(WebException webException)
{
if(IsMovedException(webException))
{
webRequest=Redirect(webRequest, webException);
webResponse=(HttpWebResponse)webRequest.GetResponse();
}
else throw;
}
Stream responseStream=webResponse.GetResponseStream();
if(webResponse.ContentEncoding.ToLower().Contains("gzip"))
{
responseStream=new GZipStream(responseStream,CompressionMode.Decompress);
StreamReader reader=new StreamReader(responseStream,Encoding.Default);
sb.Append(reader.ReadToEnd());
reader.Close();
}
else if(webResponse.ContentEncoding.ToLower().Contains("deflate"))
{
responseStream=new DeflateStream(responseStream,CompressionMode.Decompress);
StreamReader reader=new StreamReader(responseStream,Encoding.Default);
sb.Append(reader.ReadToEnd());
reader.Close();
}
else
{
while(true)
{
charCount=responseStream.Read(buffer,0,buffer.Length);
if(0==charCount) break;
sb.Append(Encoding.ASCII.GetString(buffer,0,charCount));
}
}
return new HttpNetResponse(sb.ToString(),strRequest,webResponse,true);
}
catch(WebException webException)
{
if(webException.Message.Contains("(404) Not Found"))
{
return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message);
}
lastWebException=webException;
Thread.Sleep(TIMEOUT_BETWEEN_RETRIES);
}
}
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("General failure with {0}",lastWebException.Message));
return new HttpNetResponse(webResponse,strRequest,false,lastWebException.Message);
}
catch(Exception exception)
{
return new HttpNetResponse(webResponse,strRequest,false,exception.Message);
}
finally
{
if(null!=webResponse)
{
webResponse.Close();
webResponse.Dispose();
}
MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncodingV3A[LEAVE]");
}
}
// This one uses the random user agent generator
public static HttpNetResponse GetRequestNoEncodingV3B(String strRequest,CookieCollection cookieCollection=null,WebProxy webProxy=null)
{
HttpWebResponse webResponse=null;
WebException lastWebException=null;
ServicePointManager.Expect100Continue=true;
try
{
MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestNoEncodingV3B[ENTER]{0}",strRequest));
int MAX_RETRIES=5;
int TIMEOUT_BETWEEN_RETRIES=1000;
int charCount=0;
byte[] buffer=new byte[8192];
StringBuilder sb=new StringBuilder();
for(int count=0;count=userAgents.Length) index=userAgents.Length-1;
String userAgent=userAgents[index];
webRequest.UserAgent=userAgent;
try
{
try{webResponse=(HttpWebResponse)webRequest.GetResponse();}
catch(WebException webException)
{
if(IsMovedException(webException))
{
webRequest=Redirect(webRequest, webException);
webResponse=(HttpWebResponse)webRequest.GetResponse();
}
else throw;
}
Stream responseStream=webResponse.GetResponseStream();
if(webResponse.ContentEncoding.ToLower().Contains("gzip"))
{
responseStream=new GZipStream(responseStream,CompressionMode.Decompress);
StreamReader reader=new StreamReader(responseStream,Encoding.Default);
sb.Append(reader.ReadToEnd());
reader.Close();
}
else if(webResponse.ContentEncoding.ToLower().Contains("deflate"))
{
responseStream=new DeflateStream(responseStream,CompressionMode.Decompress);
StreamReader reader=new StreamReader(responseStream,Encoding.Default);
sb.Append(reader.ReadToEnd());
reader.Close();
}
else
{
while(true)
{
charCount=responseStream.Read(buffer,0,buffer.Length);
if(0==charCount) break;
sb.Append(Encoding.ASCII.GetString(buffer,0,charCount));
}
}
return new HttpNetResponse(sb.ToString(),strRequest,webResponse,true);
}
catch(WebException webException)
{
if(webException.Message.Contains("(404) Not Found"))
{
return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message);
}
lastWebException=webException;
Thread.Sleep(TIMEOUT_BETWEEN_RETRIES);
}
}
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("General failure with {0}",lastWebException.Message));
return new HttpNetResponse(webResponse,strRequest,false,lastWebException.Message);
}
catch(Exception exception)
{
return new HttpNetResponse(webResponse,strRequest,false,exception.Message);
}
finally
{
if(null!=webResponse)
{
webResponse.Close();
webResponse.Dispose();
}
MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncodingV3C[LEAVE]");
}
}
public static HttpNetResponse GetRequestNoEncodingV4(String strRequest,CookieCollection cookieCollection=null,WebProxy webProxy=null)
{
HttpWebResponse webResponse=null;
try
{
MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestNoEncodingV4[ENTER]{0}",strRequest));
int charCount = 0;
byte[] buffer = new byte[8192];
StringBuilder sb = new StringBuilder();
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
if(null!=webProxy) webRequest.Proxy=webProxy;
webRequest.Timeout = REQUEST_TIMEOUT;
webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
webRequest.Headers.Add("Accept-Encoding: None");
webRequest.Accept = "image/gif, image/jpeg, image/pjpeg, image/pjpeg, application/x-shockwave-flash, application/x-ms-application, application/x-ms-xbap, application/vnd.ms-xpsdocument, application/xaml+xml, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*";
webRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; MS-RTC LM 8; .NET4.0C; .NET4.0E; InfoPath.3)";
webRequest.KeepAlive = true;
webRequest.CookieContainer=new CookieContainer();
if(null!=cookieCollection)webRequest.CookieContainer.Add(cookieCollection);
try{webResponse = (HttpWebResponse)webRequest.GetResponse();}
catch(WebException webException)
{
if(IsMovedException(webException))
{
webRequest=Redirect(webRequest, webException);
webResponse=(HttpWebResponse)webRequest.GetResponse();
}
else throw;
}
Stream responseStream = webResponse.GetResponseStream();
while (true)
{
charCount = responseStream.Read(buffer, 0, buffer.Length);
if (0 == charCount) break;
sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
}
return new HttpNetResponse(sb.ToString(),strRequest,webResponse,webResponse.Cookies,true);
}
catch (WebException webException)
{
return new HttpNetResponse((HttpWebResponse)webException.Response,strRequest,false,webException.Message);
}
catch (Exception exception)
{
return new HttpNetResponse(webResponse,strRequest,false,exception.Message);
}
finally
{
if(null!=webResponse)
{
webResponse.Close();
webResponse.Dispose();
}
MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncodingV4[LEAVE]");
}
}
public static HttpNetResponse GetRequestNoEncodingV4A(String strRequest,CookieCollection cookieCollection=null,WebProxy webProxy=null)
{
HttpWebResponse webResponse=null;
try
{
MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestNoEncodingV4[ENTER]{0}",strRequest));
int charCount=0;
byte[] buffer=new byte[8192];
StringBuilder sb=new StringBuilder();
ServicePointManager.Expect100Continue=true;
HttpWebRequest webRequest=(HttpWebRequest)WebRequest.Create(new Uri(strRequest));
if(null!=webProxy)webRequest.Proxy=webProxy;
webRequest.Timeout=REQUEST_TIMEOUT;
webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
webRequest.Headers.Add("Accept-Encoding: gzip, deflate, br");
webRequest.Accept="text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8";
webRequest.UserAgent="Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:82.0) Gecko/20100101 Firefox/82.0";
webRequest.KeepAlive=true;
webRequest.CookieContainer=new CookieContainer();
if(null!=cookieCollection) webRequest.CookieContainer.Add(cookieCollection);
try{webResponse=(HttpWebResponse)webRequest.GetResponse();}
catch(WebException webException)
{
if(IsMovedException(webException))
{
webRequest=Redirect(webRequest, webException);
webResponse=(HttpWebResponse)webRequest.GetResponse();
}
else throw;
}
Stream responseStream=webResponse.GetResponseStream();
if(webResponse.ContentEncoding.ToLower().Contains("gzip"))
{
responseStream=new GZipStream(responseStream,CompressionMode.Decompress);
StreamReader reader=new StreamReader(responseStream,Encoding.Default);
sb.Append(reader.ReadToEnd());
reader.Close();
}
else
{
while(true)
{
charCount=responseStream.Read(buffer,0,buffer.Length);
if(0==charCount) break;
sb.Append(Encoding.ASCII.GetString(buffer,0,charCount));
}
}
return new HttpNetResponse(sb.ToString(),strRequest,webResponse,webResponse.Cookies,true);
}
catch(WebException webException)
{
return new HttpNetResponse((HttpWebResponse)webException.Response,strRequest,false,webException.Message);
}
catch(Exception exception)
{
return new HttpNetResponse(webResponse,strRequest,false,exception.Message);
}
finally
{
if(null!=webResponse)
{
webResponse.Close();
webResponse.Dispose();
}
MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncodingV4[LEAVE]");
}
}
public static HttpNetResponse GetRequestNoEncodingV5(String strRequest,int webRequestTimeoutMS,WebProxy webProxy=null)
{
HttpWebResponse webResponse = null;
try
{
MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV5[ENTER]{0}", strRequest));
int charCount = 0;
byte[] buffer = new byte[8192];
StringBuilder sb = new StringBuilder();
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
if(null!=webProxy)webRequest.Proxy=webProxy;
webRequest.Timeout = webRequestTimeoutMS;
webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
webRequest.Headers.Add("Accept-Encoding: None");
webRequest.Accept = "image/gif, image/jpeg, image/pjpeg, image/pjpeg, application/x-shockwave-flash, application/x-ms-application, application/x-ms-xbap, application/vnd.ms-xpsdocument, application/xaml+xml, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*";
webRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; MS-RTC LM 8; .NET4.0C; .NET4.0E; InfoPath.3)";
webRequest.KeepAlive = true;
webRequest.CookieContainer = new CookieContainer();
try{webResponse = (HttpWebResponse)webRequest.GetResponse();}
catch(WebException webException)
{
if(IsMovedException(webException))
{
webRequest=Redirect(webRequest, webException);
webResponse=(HttpWebResponse)webRequest.GetResponse();
}
else throw;
}
Stream responseStream = webResponse.GetResponseStream();
while (true)
{
charCount = responseStream.Read(buffer, 0, buffer.Length);
if (0 == charCount) break;
sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
}
return new HttpNetResponse(sb.ToString(), strRequest, webResponse, webResponse.Cookies, true);
}
catch (WebException webException)
{
return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message);
}
catch (Exception exception)
{
return new HttpNetResponse(webResponse, strRequest, false, exception.Message);
}
finally
{
if(null!=webResponse)
{
webResponse.Close();
webResponse.Dispose();
}
MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV5[LEAVE]");
}
}
public static HttpNetResponse GetRequestNoEncodingV5A(String strRequest,int webRequestTimeoutMS,WebProxy webProxy=null)
{
HttpWebResponse webResponse = null;
try
{
Random random = new Random();
String[] userAgents=
{
"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.0",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:82.0) Gecko/20100101 Firefox/82.0"
};
MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV5A[ENTER]{0}", strRequest));
int charCount = 0;
byte[] buffer = new byte[8192];
StringBuilder sb = new StringBuilder();
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
if(null!=webProxy)webRequest.Proxy=webProxy;
webRequest.Timeout = webRequestTimeoutMS;
webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
webRequest.Headers.Add("Accept-Encoding: None");
webRequest.Accept = "image/gif, image/jpeg, image/pjpeg, image/pjpeg, application/x-shockwave-flash, application/x-ms-application, application/x-ms-xbap, application/vnd.ms-xpsdocument, application/xaml+xml, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*";
webRequest.KeepAlive = true;
webRequest.CookieContainer = new CookieContainer();
double randomNumber=random.NextDouble(); // number between 0 and 1
int index=(int)(randomNumber*((double)userAgents.Length));
if(index>=userAgents.Length) index=userAgents.Length-1;
String userAgent=userAgents[index];
webRequest.UserAgent=userAgent;
try{webResponse = (HttpWebResponse)webRequest.GetResponse();}
catch(WebException webException)
{
if(IsMovedException(webException))
{
webRequest=Redirect(webRequest, webException);
webResponse=(HttpWebResponse)webRequest.GetResponse();
}
else throw;
}
Stream responseStream = webResponse.GetResponseStream();
while (true)
{
charCount = responseStream.Read(buffer, 0, buffer.Length);
if (0 == charCount) break;
sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
}
return new HttpNetResponse(sb.ToString(), strRequest, webResponse, webResponse.Cookies, true);
}
catch (WebException webException)
{
return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message);
}
catch (Exception exception)
{
return new HttpNetResponse(webResponse, strRequest, false, exception.Message);
}
finally
{
if(null!=webResponse)
{
webResponse.Close();
webResponse.Dispose();
}
MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV5A[LEAVE]");
}
}
public static HttpNetResponse GetRequestNoEncodingV5B(String strRequest,int webRequestTimeoutMS,WebProxy webProxy=null,bool useRandomUserAgent=false,CookieCollection cookieCollection=null)
{
HttpWebResponse webResponse = null;
try
{
MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV5B[ENTER]{0}", strRequest));
int charCount = 0;
byte[] buffer = new byte[8192];
StringBuilder sb = new StringBuilder();
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
if(null!=webProxy)webRequest.Proxy=webProxy;
webRequest.Timeout = webRequestTimeoutMS;
webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
webRequest.Headers.Add("Accept-Encoding: *");
webRequest.Accept = "image/gif, image/jpeg, image/pjpeg, image/pjpeg, application/x-shockwave-flash, application/x-ms-application, application/x-ms-xbap, application/vnd.ms-xpsdocument, application/xaml+xml, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*";
if (useRandomUserAgent) webRequest.UserAgent = UserAgent.GetInstance().GetUserAgent();
else webRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; MS-RTC LM 8; .NET4.0C; .NET4.0E; InfoPath.3)";
webRequest.KeepAlive = true;
webRequest.CookieContainer = new CookieContainer();
if(null!=cookieCollection)webRequest.CookieContainer.Add(cookieCollection);
try{webResponse = (HttpWebResponse)webRequest.GetResponse();}
catch(WebException webException)
{
if(IsMovedException(webException))
{
webRequest=Redirect(webRequest, webException);
webResponse=(HttpWebResponse)webRequest.GetResponse();
}
else throw;
}
Stream responseStream = webResponse.GetResponseStream();
while (true)
{
charCount = responseStream.Read(buffer, 0, buffer.Length);
if (0 == charCount) break;
sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
}
return new HttpNetResponse(sb.ToString(), strRequest, webResponse, webResponse.Cookies, true);
}
catch (WebException webException)
{
return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message);
}
catch (Exception exception)
{
return new HttpNetResponse(webResponse, strRequest, false, exception.Message);
}
finally
{
if(null!=webResponse)
{
webResponse.Close();
webResponse.Dispose();
}
MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV5B[LEAVE]");
}
}
// This accepts gzip and deflate which seems to work for the seeking alpha news retrieval
public static HttpNetResponse GetRequestNoEncodingV5C(String strRequest,String host,int webRequestTimeoutMS,WebProxy webProxy=null,bool useRandomUserAgent=false,CookieCollection cookieCollection=null)
{
HttpWebResponse webResponse = null;
try
{
MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV5C[ENTER]{0}", strRequest));
byte[] buffer = new byte[8192];
StringBuilder sb = new StringBuilder();
bool expect100Condition = ServicePointManager.Expect100Continue;
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
if(null!=webProxy)webRequest.Proxy=webProxy;
webRequest.Timeout = webRequestTimeoutMS;
webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
webRequest.Headers.Add("Accept-Encoding: gzip, deflate, br");
webRequest.Host = host;
webRequest.Headers.Add("Sec-GPC: 1");
webRequest.Headers.Add("Sec-Fetch-Dest: document");
webRequest.Headers.Add("Sec-Fetch-Mode: navigate");
webRequest.Headers.Add("Sec-Fetch-Site: none");
webRequest.Headers.Add("Sec-Fetch-User: ?1");
webRequest.Headers.Add("DNT: 1");
webRequest.Headers.Add("Upgrade-Insecure-Requests: 1");
webRequest.Headers.Add("TE: trailers");
webRequest.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8";
if (useRandomUserAgent) webRequest.UserAgent = UserAgent.GetInstance().GetUserAgent();
else webRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:124.0) Gecko/20100101 Firefox/124.0";
webRequest.KeepAlive = true;
webRequest.CookieContainer = new CookieContainer();
if(null!=cookieCollection)webRequest.CookieContainer.Add(cookieCollection);
try{webResponse = (HttpWebResponse)webRequest.GetResponse();}
catch(WebException webException)
{
if(IsMovedException(webException))
{
webRequest=Redirect(webRequest, webException);
webResponse=(HttpWebResponse)webRequest.GetResponse();
}
else throw;
}
HttpNetResponse httpNetResponse = ProcessWebResponse(strRequest, webResponse);
return httpNetResponse;
}
catch (WebException webException)
{
return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message);
}
catch (Exception exception)
{
return new HttpNetResponse(webResponse, strRequest, false, exception.Message);
}
finally
{
if(null!=webResponse)
{
webResponse.Close();
webResponse.Dispose();
}
MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV5C[LEAVE]");
}
}
public static HttpNetResponse GetRequestNoEncodingV5D(String strRequest,String host,int webRequestTimeoutMS,WebProxy webProxy=null,bool useRandomUserAgent=false,CookieCollection cookieCollection=null)
{
HttpWebResponse webResponse = null;
try
{
MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV5C[ENTER]{0}", strRequest));
byte[] buffer = new byte[8192];
StringBuilder sb = new StringBuilder();
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
if(null!=webProxy)webRequest.Proxy=webProxy;
webRequest.Timeout = webRequestTimeoutMS;
webRequest.KeepAlive=true;
webRequest.Headers.Add("Priority: u=0, i");
webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
webRequest.Headers.Add("Accept-Encoding: gzip, deflate, br, zstd");
webRequest.Host = host;
webRequest.Headers.Add("Sec-GPC: 1");
webRequest.Headers.Add("Sec-Fetch-Dest: document");
webRequest.Headers.Add("Sec-Fetch-Mode: navigate");
webRequest.Headers.Add("Sec-Fetch-Site: crosssite");
webRequest.Headers.Add("Sec-Fetch-User: ?1");
webRequest.Headers.Add("DNT: 1");
webRequest.Headers.Add("Upgrade-Insecure-Requests: 1");
webRequest.Headers.Add("TE: trailers");
webRequest.Headers.Add("If-None-Match: "+"W/\"e76e3614af0e341af5df8daa8b855a4e\"");
webRequest.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
if (useRandomUserAgent) webRequest.UserAgent = UserAgent.GetInstance().GetUserAgent();
else webRequest.UserAgent = "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/56.0.2924.76 Safari/537.36";
webRequest.CookieContainer = new CookieContainer();
if(null!=cookieCollection)webRequest.CookieContainer.Add(cookieCollection);
try{webResponse = (HttpWebResponse)webRequest.GetResponse();}
catch(WebException webException)
{
if(IsMovedException(webException))
{
webRequest=Redirect(webRequest, webException);
webResponse=(HttpWebResponse)webRequest.GetResponse();
}
else throw;
}
HttpNetResponse httpNetResponse = ProcessWebResponse(strRequest, webResponse);
httpNetResponse.CookieCollection = new CookieCollection();
httpNetResponse.CookieCollection.Add(webResponse.Cookies);
return httpNetResponse;
}
catch (WebException webException)
{
return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message);
}
catch (Exception exception)
{
return new HttpNetResponse(webResponse, strRequest, false, exception.Message);
}
finally
{
if(null!=webResponse)
{
webResponse.Close();
webResponse.Dispose();
}
MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV5C[LEAVE]");
}
}
///
/// GetRequestV6 - Uses HttpClient to more closely match web browser.
///
///
///
///
///
///
public static HttpNetResponse GetRequestV6(string url,int timeoutMS,WebProxy proxy = null,string cookieHeader = null)
{
HttpResponseMessage response = null;
try
{
MDTrace.WriteLine(LogLevel.VERBOSE, $"GetRequestV6[ENTER]{url}");
var handler = new HttpClientHandler()
{
AllowAutoRedirect = true,
AutomaticDecompression =
DecompressionMethods.GZip |
DecompressionMethods.Deflate |
DecompressionMethods.Brotli,
UseCookies = true,
CookieContainer = new CookieContainer()
};
if (proxy != null)
{
handler.Proxy = proxy;
handler.UseProxy = true;
}
using (HttpClient client = new HttpClient(handler))
{
client.Timeout = TimeSpan.FromMilliseconds(timeoutMS);
// Core headers
client.DefaultRequestHeaders.TryAddWithoutValidation(
"User-Agent",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:148.0) Gecko/20100101 Firefox/148.0");
client.DefaultRequestHeaders.TryAddWithoutValidation(
"Accept",
"text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
client.DefaultRequestHeaders.TryAddWithoutValidation(
"Accept-Language",
"en-US,en;q=0.9");
client.DefaultRequestHeaders.TryAddWithoutValidation(
"Accept-Encoding",
"gzip, deflate, br, zstd");
client.DefaultRequestHeaders.Connection.Add("keep-alive");
// Browser-like headers
client.DefaultRequestHeaders.TryAddWithoutValidation("Upgrade-Insecure-Requests", "1");
client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-Dest", "document");
client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-Mode", "navigate");
client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-Site", "none");
client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-User", "?1");
client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-GPC", "1");
client.DefaultRequestHeaders.TryAddWithoutValidation("Priority", "u=0, i");
client.DefaultRequestHeaders.TryAddWithoutValidation("TE", "trailers");
if (!string.IsNullOrEmpty(cookieHeader))
{
client.DefaultRequestHeaders.TryAddWithoutValidation("Cookie", cookieHeader);
}
response = client.GetAsync(url).GetAwaiter().GetResult();
string html = response.Content.ReadAsStringAsync().GetAwaiter().GetResult();
return new HttpNetResponse(html,url,null,handler.CookieContainer.GetCookies(new Uri(url)),response.IsSuccessStatusCode);
}
}
catch (Exception ex)
{
return new HttpNetResponse(null,url,false,ex.Message);
}
finally
{
MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestV6[LEAVE]");
}
}
///
/// GetRequestBOLS - This request is used for Bureau of Labor Statistics
///
///
///
///
///
///
///
///
public static HttpNetResponse GetRequestBOLS(string strRequest, string host, int webRequestTimeoutMS, WebProxy webProxy = null, bool useRandomUserAgent = false, CookieCollection cookieCollection = null)
{
HttpResponseMessage response = null;
try
{
MDTrace.WriteLine(LogLevel.VERBOSE,$"GetRequestBOLS[ENTER]{strRequest}");
HttpClientHandler handler = new HttpClientHandler()
{
AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate | DecompressionMethods.Brotli,
UseCookies = true,
CookieContainer = new CookieContainer(),
AllowAutoRedirect = true
};
if (webProxy != null)
{
handler.Proxy = webProxy;
handler.UseProxy = true;
}
if (cookieCollection != null)
{
handler.CookieContainer.Add(cookieCollection);
}
using (HttpClient client = new HttpClient(handler))
{
client.Timeout = TimeSpan.FromMilliseconds(webRequestTimeoutMS);
Uri uri = new Uri(strRequest);
// Required headers
client.DefaultRequestHeaders.TryAddWithoutValidation("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
client.DefaultRequestHeaders.TryAddWithoutValidation("Accept-Encoding", "gzip, deflate, br, zstd");
client.DefaultRequestHeaders.TryAddWithoutValidation("Accept-Language", "en-US,en;q=0.9");
client.DefaultRequestHeaders.TryAddWithoutValidation("Connection", "keep-alive");
client.DefaultRequestHeaders.Host = host;
client.DefaultRequestHeaders.TryAddWithoutValidation("Priority", "u=0, i");
client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-Dest", "document");
client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-Mode", "navigate");
client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-Site", "none");
client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-User", "?1");
client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-GPC", "1");
client.DefaultRequestHeaders.TryAddWithoutValidation("Upgrade-Insecure-Requests","1");
// User agent
if (useRandomUserAgent)client.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent",UserAgent.GetInstance().GetUserAgent());
else client.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:148.0) Gecko/20100101 Firefox/148.0");
response = client.GetAsync(strRequest).GetAwaiter().GetResult();
string html = response.Content.ReadAsStringAsync().GetAwaiter().GetResult();
return new HttpNetResponse(html, strRequest, null, handler.CookieContainer.GetCookies(uri), response.IsSuccessStatusCode);
} // using
}
catch (Exception ex)
{
return new HttpNetResponse(null, strRequest, false, ex.Message);
}
finally
{
MDTrace.WriteLine(LogLevel.DEBUG,"GetRequestGDP[LEAVE]");
}
}
///
/// This one is used to fetch data from GetRequestSECEDGAR like CIK.
///
///
///
///
///
public static HttpNetResponse GetRequestSECEDGAR(string strRequest,int webRequestTimeoutMS,WebProxy webProxy=null)
{
HttpResponseMessage response=null;
try
{
MDTrace.WriteLine(LogLevel.VERBOSE,$"GetRequestSECEDGAR[ENTER]{strRequest}");
var handler=new HttpClientHandler()
{
AllowAutoRedirect=true,
AutomaticDecompression=DecompressionMethods.GZip|DecompressionMethods.Deflate|DecompressionMethods.Brotli,
UseCookies=true,
CookieContainer=new CookieContainer()
};
if(null!=webProxy){handler.Proxy=webProxy;handler.UseProxy=true;}
using(HttpClient client=new HttpClient(handler))
{
client.Timeout=TimeSpan.FromMilliseconds(webRequestTimeoutMS);
Uri uri=new Uri(strRequest);
client.DefaultRequestHeaders.TryAddWithoutValidation("Accept","text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8");
client.DefaultRequestHeaders.TryAddWithoutValidation("Accept-Encoding","gzip, deflate, br, zstd");
client.DefaultRequestHeaders.TryAddWithoutValidation("Accept-Language","en-US,en;q=0.9");
client.DefaultRequestHeaders.TryAddWithoutValidation("Connection","keep-alive");
client.DefaultRequestHeaders.TryAddWithoutValidation("Priority","u=0, i");
client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-Dest","document");
client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-Mode","navigate");
client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-Site","none");
client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-Fetch-User","?1");
client.DefaultRequestHeaders.TryAddWithoutValidation("Sec-GPC","1");
client.DefaultRequestHeaders.TryAddWithoutValidation("TE","trailers");
client.DefaultRequestHeaders.TryAddWithoutValidation("Upgrade-Insecure-Requests","1");
client.DefaultRequestHeaders.TryAddWithoutValidation("User-Agent","Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:148.0) Gecko/20100101 Firefox/148.0");
response=client.GetAsync(strRequest).GetAwaiter().GetResult();
string html=response.Content.ReadAsStringAsync().GetAwaiter().GetResult();
return new HttpNetResponse(html,strRequest,null,handler.CookieContainer.GetCookies(uri),response.IsSuccessStatusCode);
}
}
catch(WebException webException){return new HttpNetResponse((HttpWebResponse)webException.Response,strRequest,false,webException.Message);}
catch(Exception exception){return new HttpNetResponse(null,strRequest,false,exception.Message);}
finally{MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestSECEDGAR[LEAVE]");}
}
private static HttpNetResponse ProcessWebResponse(String strRequest,HttpWebResponse webResponse)
{
try
{
Stream responseStream = webResponse.GetResponseStream();
StringBuilder sb = new StringBuilder();
int charCount = 0;
byte[] buffer = new byte[8192];
if(webResponse.ContentEncoding.ToLower().Contains("gzip"))
{
responseStream = new GZipStream(responseStream, CompressionMode.Decompress);
StreamReader reader = new StreamReader(responseStream, Encoding.Default);
sb.Append(reader.ReadToEnd());
reader.Close();
}
else if(webResponse.ContentEncoding.ToLower().Contains("deflate"))
{
responseStream = new DeflateStream(responseStream, CompressionMode.Decompress);
StreamReader reader = new StreamReader(responseStream, Encoding.Default);
sb.Append(reader.ReadToEnd());
reader.Close();
}
else
{
while (true)
{
charCount = responseStream.Read(buffer, 0, buffer.Length);
if (0 == charCount) break;
sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
}
}
return new HttpNetResponse(sb.ToString(),strRequest,webResponse,true);
}
catch(Exception exception)
{
return new HttpNetResponse(webResponse,strRequest,false,exception.Message);
}
}
public static HttpNetResponse GetRequestNoEncodingV6(String strRequest, int webRequestTimeoutMS)
{
HttpWebResponse webResponse = null;
try
{
MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV6[ENTER]{0}", strRequest));
int charCount = 0;
byte[] buffer = new byte[8192];
StringBuilder sb = new StringBuilder();
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
webRequest.Timeout = webRequestTimeoutMS;
webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
webRequest.Headers.Add("Accept-Encoding: gzip, deflate, br");
webRequest.Accept = "image/gif, image/jpeg, image/pjpeg, image/pjpeg, application/x-shockwave-flash, application/x-ms-application, application/x-ms-xbap, application/vnd.ms-xpsdocument, application/xaml+xml, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*";
webRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; MS-RTC LM 8; .NET4.0C; .NET4.0E; InfoPath.3)";
webRequest.KeepAlive = true;
webRequest.CookieContainer = new CookieContainer();
try{webResponse = (HttpWebResponse)webRequest.GetResponse();}
catch(WebException webException)
{
if(IsMovedException(webException))
{
webRequest=Redirect(webRequest, webException);
webResponse=(HttpWebResponse)webRequest.GetResponse();
}
else throw;
}
Stream responseStream = webResponse.GetResponseStream();
while (true)
{
charCount = responseStream.Read(buffer, 0, buffer.Length);
if (0 == charCount) break;
sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
}
return new HttpNetResponse(sb.ToString(), strRequest, webResponse, webResponse.Cookies, true);
}
catch (WebException webException)
{
return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message);
}
catch (Exception exception)
{
return new HttpNetResponse(webResponse, strRequest, false, exception.Message);
}
finally
{
if(null!=webResponse)
{
webResponse.Close();
webResponse.Dispose();
}
MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV6[LEAVE]");
}
}
public static HttpNetResponse GetRequestNoEncodingV7(String strRequest,WebProxy webProxy=null,CookieCollection cookieCollection=null)
{
HttpWebResponse webResponse = null;
try
{
MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV7[ENTER]{0}", strRequest));
int charCount = 0;
byte[] buffer = new byte[8192];
StringBuilder sb = new StringBuilder();
bool expect100Condition = ServicePointManager.Expect100Continue;
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
if(null!=webProxy)webRequest.Proxy=webProxy;
webRequest.KeepAlive=true;
webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
webRequest.Headers.Add("Accept-Encoding: gzip, deflate, br");
webRequest.Headers.Add("Upgrade-Insecure-Requests: 1");
webRequest.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8";
webRequest.UserAgent = " Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:76.0) Gecko/20100101 Firefox/76.0";
webRequest.KeepAlive = true;
webRequest.CookieContainer = new CookieContainer();
if(null!=cookieCollection)webRequest.CookieContainer.Add(cookieCollection);
try{webResponse = (HttpWebResponse)webRequest.GetResponse();}
catch(WebException webException)
{
if(IsMovedException(webException))
{
webRequest=Redirect(webRequest, webException);
webResponse=(HttpWebResponse)webRequest.GetResponse();
}
else throw;
}
Stream responseStream = webResponse.GetResponseStream();
if (webResponse.ContentEncoding.ToLower().Contains("gzip"))
{
responseStream = new GZipStream(responseStream, CompressionMode.Decompress);
StreamReader reader = new StreamReader(responseStream, Encoding.Default);
sb.Append(reader.ReadToEnd());
reader.Close();
}
else if (webResponse.ContentEncoding.ToLower().Contains("deflate"))
{
responseStream = new DeflateStream(responseStream, CompressionMode.Decompress);
StreamReader reader = new StreamReader(responseStream, Encoding.Default);
sb.Append(reader.ReadToEnd());
reader.Close();
}
else
{
while (true)
{
charCount = responseStream.Read(buffer, 0, buffer.Length);
if (0 == charCount) break;
sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
}
}
return new HttpNetResponse(sb.ToString(), strRequest, webResponse, webResponse.Cookies, true);
}
catch (WebException webException)
{
return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message);
}
catch (Exception exception)
{
return new HttpNetResponse(webResponse, strRequest, false, exception.Message);
}
finally
{
if(null!=webResponse)
{
webResponse.Close();
webResponse.Dispose();
}
MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV7[LEAVE]");
}
}
public static HttpNetResponse GetRequestNoEncodingZoneEdit(String strRequest,String userName, String password)
{
HttpWebResponse webResponse = null;
try
{
MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingZoneEdit[ENTER]{0}", strRequest));
int charCount = 0;
byte[] buffer = new byte[8192];
StringBuilder sb = new StringBuilder();
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
webRequest.Credentials=new NetworkCredential(userName,password);
webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
webRequest.Headers.Add("Accept-Encoding: None");
webRequest.Accept = "image/gif, image/jpeg, image/pjpeg, image/pjpeg, application/x-shockwave-flash, application/x-ms-application, application/x-ms-xbap, application/vnd.ms-xpsdocument, application/xaml+xml, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*";
webRequest.KeepAlive = true;
webResponse = (HttpWebResponse)webRequest.GetResponse();
Stream responseStream = webResponse.GetResponseStream();
while (true)
{
charCount = responseStream.Read(buffer, 0, buffer.Length);
if (0 == charCount) break;
sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
}
return new HttpNetResponse(sb.ToString(), strRequest, webResponse, webResponse.Cookies, true);
}
catch (WebException webException)
{
return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message);
}
catch (Exception exception)
{
return new HttpNetResponse(webResponse, strRequest, false, exception.Message);
}
finally
{
if(null!=webResponse)
{
webResponse.Close();
webResponse.Dispose();
}
MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingZoneEdit[LEAVE]");
}
}
// This is specifically taylored to Morningstar site for retrieval of historical data
public static HttpNetResponse GetRequestNoEncodingMStar(String strRequest,WebProxy webProxy=null)
{
HttpWebResponse webResponse = null;
try
{
MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingMStar[ENTER]{0}", strRequest));
int charCount = 0;
byte[] buffer = new byte[8192];
StringBuilder sb = new StringBuilder();
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
webRequest.KeepAlive=true;
webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
webRequest.Headers.Add("Accept-Encoding: gzip, deflate, br");
webRequest.Headers.Add("DNT: 1");
webRequest.Headers.Add("Sec-Fetch-Dest:empty");
webRequest.Headers.Add("Sec-Fetch-Mode:cors");
webRequest.Headers.Add("Sec-Fetch-Site:same-site");
webRequest.Headers.Add("X-SAL-ContentType:e7FDDltrTy+tA2HnLovvGL0LFMwT+KkEptGju5wXVTU=");
webRequest.Headers.Add("X-API-RequestId: d2ffea42-4a07-4fa7-fa0d-b307facd81fb");
webRequest.Headers.Add("ApiKey:lstzFDEOhfFNMLikKa0am9mgEKLBl49T");
webRequest.Accept = "*/*";
webRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36";
webRequest.Host="api-global.morningstar.com";
webRequest.Referer="https://www.morningstar.com/stocks/xnas/aapl/performance";
if(null!=webProxy)webRequest.Proxy=webProxy;
webRequest.CookieContainer = new CookieContainer();
try{webResponse = (HttpWebResponse)webRequest.GetResponse();}
catch(WebException webException)
{
if(IsMovedException(webException))
{
webRequest=Redirect(webRequest, webException);
webResponse=(HttpWebResponse)webRequest.GetResponse();
}
else throw;
}
Stream responseStream = webResponse.GetResponseStream();
if (webResponse.ContentEncoding.ToLower().Contains("gzip"))
{
responseStream = new GZipStream(responseStream, CompressionMode.Decompress);
StreamReader reader = new StreamReader(responseStream, Encoding.Default);
sb.Append(reader.ReadToEnd());
reader.Close();
}
else if (webResponse.ContentEncoding.ToLower().Contains("deflate"))
{
responseStream = new DeflateStream(responseStream, CompressionMode.Decompress);
StreamReader reader = new StreamReader(responseStream, Encoding.Default);
sb.Append(reader.ReadToEnd());
reader.Close();
}
else
{
while (true)
{
charCount = responseStream.Read(buffer, 0, buffer.Length);
if (0 == charCount) break;
sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
}
}
return new HttpNetResponse(sb.ToString(), strRequest, webResponse, webResponse.Cookies, true);
}
catch (WebException webException)
{
return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message);
}
catch (Exception exception)
{
return new HttpNetResponse(webResponse, strRequest, false, exception.Message);
}
finally
{
if(null!=webResponse)
{
webResponse.Close();
webResponse.Dispose();
}
MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingMStar[LEAVE]");
}
}
private static HttpWebRequest Redirect(HttpWebRequest request, WebException webException)
{
if(!IsMovedException(webException))return null;
HttpWebResponse response = webException.Response as HttpWebResponse;
string locationUri = response.Headers["Location"];
MDTrace.WriteLine(LogLevel.DEBUG,$"Request is being redirected to {locationUri}");
Uri redirectUri = new Uri(locationUri);
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(redirectUri);
webRequest.Proxy=request.Proxy;
webRequest.Timeout = request.Timeout;
webRequest.Headers = request.Headers;
webRequest.Accept = request.Accept;
webRequest.UserAgent = request.UserAgent;
webRequest.KeepAlive = request.KeepAlive;
webRequest.CookieContainer = request.CookieContainer;
webRequest.Referer = request.Referer;
webRequest.Host = request.Host;
return webRequest;
}
///
/// IsMovedException - MovedPermanently, Found, RedirectKeepVerb, PermanentRedirect will all return a "Location" in the response
/// That we will forward the request to.
///
///
///
private static bool IsMovedException(WebException webException)
{
if (webException.Status.Equals(WebExceptionStatus.ProtocolError))
{
HttpWebResponse response = webException.Response as HttpWebResponse;
if (response.StatusCode.Equals(HttpStatusCode.MovedPermanently) ||
response.StatusCode.Equals(HttpStatusCode.Found) ||
response.StatusCode.Equals(HttpStatusCode.RedirectKeepVerb) ||
response.StatusCode.Equals(HttpStatusCode.PermanentRedirect))
{
return true;
}
}
return false;
}
///
/// Get the proxy for the specified service or null if it is not congfigured to use the proxy
///
///The service name
public static WebProxy GetProxy(String serviceName)
{
try
{
String proxyAddress=GlobalConfig.Instance.Configuration["proxy_address"];
if(null==proxyAddress) return null;
String useServiceProxy=ConfigurationManager.AppSettings["proxy_"+serviceName];
if(null==useServiceProxy||false==bool.Parse(useServiceProxy)) return null;
String[] addressParts=proxyAddress.Split(':');
SocketControl socketControl=null;
try
{
socketControl=new SocketControl(addressParts[1].Replace("//",null),int.Parse(addressParts[2]));
if(!socketControl.IsConnected())throw new Exception("Unreachable destination "+proxyAddress);
}
catch(Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("****** ERROR: CANNOT REACH PROXY {0} FOR SERVICE {1}...{2}",proxyAddress,serviceName,exception.ToString()));
return null;
}
socketControl.Close();
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("USING PROXY:{0} FOR {1}",proxyAddress,serviceName));
Uri newUri=new Uri(proxyAddress);
WebProxy proxy=new WebProxy();
proxy.Address=newUri;
return proxy;
}
catch(Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetProxy: Exception:{0}",exception.ToString()));
return null;
}
}
///
/// Get the proxy. This is only to be used as part of unit tests to test out the TOR Proxy. User code should use the GetProxy(serviceName) variation
///
public static WebProxy GetProxy()
{
String proxyAddress=GlobalConfig.Instance.Configuration["proxy_address"];
String[] addressParts=proxyAddress.Split(':');
SocketControl socketControl=null;
try
{
socketControl=new SocketControl(addressParts[1].Replace("//",null),int.Parse(addressParts[2]));
if(!socketControl.IsConnected())throw new Exception("Unreachable destination "+proxyAddress);
}
catch(Exception exception)
{
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("****** ERROR: CANNOT REACH PROXY. {0} {1} ",proxyAddress,exception.ToString()));
return null;
}
socketControl.Close();
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("USING PROXY:{0} ",proxyAddress));
Uri newUri=new Uri(proxyAddress);
WebProxy proxy=new WebProxy();
proxy.Address=newUri;
return proxy;
}
}
}
#pragma warning restore SYSLIB0014