1417 lines
64 KiB
C#
1417 lines
64 KiB
C#
using System;
|
|
using System.IO;
|
|
using System.Net;
|
|
using System.Text;
|
|
using System.Threading;
|
|
using System.IO.Compression;
|
|
using System.Configuration;
|
|
// Filename: HttpRequest.cs
|
|
// Author:Sean Kessler
|
|
|
|
namespace IPMonitor
|
|
{
|
|
public class HttpNetResponse : IDisposable
|
|
{
|
|
public enum TypeResponse {Stream,String};
|
|
public String ResponseString{get;set;}
|
|
public MemoryStream ResponseStream{get;set;}
|
|
public CookieCollection CookieCollection{get;set;}
|
|
|
|
public HttpNetResponse()
|
|
{
|
|
Success=false;
|
|
}
|
|
public HttpNetResponse(String responseString,String request,HttpWebResponse webResponse,CookieCollection cookieCollection,bool success)
|
|
{
|
|
ResponseString=responseString;
|
|
ResponseType=TypeResponse.String;
|
|
Success=success;
|
|
Request=request;
|
|
CookieCollection=cookieCollection;
|
|
if(null!=webResponse)StatusCode=webResponse.StatusCode;
|
|
}
|
|
public HttpNetResponse(String responseString,String request,HttpWebResponse webResponse,bool success)
|
|
{
|
|
ResponseString=responseString;
|
|
ResponseType=TypeResponse.String;
|
|
Success=success;
|
|
Request=request;
|
|
if(null!=webResponse)StatusCode=webResponse.StatusCode;
|
|
}
|
|
public HttpNetResponse(MemoryStream responseStream,String request,HttpWebResponse webResponse,bool success)
|
|
{
|
|
ResponseStream=responseStream;
|
|
ResponseType=TypeResponse.String;
|
|
Success=success;
|
|
Request=request;
|
|
if(null!=webResponse)StatusCode=webResponse.StatusCode;
|
|
}
|
|
public HttpNetResponse(HttpWebResponse webResponse,String request,bool success,String errorMessage)
|
|
{
|
|
ResponseType=TypeResponse.String;
|
|
Success=success;
|
|
Request=request;
|
|
if(null!=webResponse)StatusCode=webResponse.StatusCode;
|
|
ErrorMessage=errorMessage;
|
|
}
|
|
public Stream ResponseStringToStream()
|
|
{
|
|
MemoryStream stream = new MemoryStream();
|
|
StreamWriter writer = new StreamWriter(stream);
|
|
writer.Write(ResponseString);
|
|
writer.Flush();
|
|
stream.Position = 0;
|
|
return stream;
|
|
}
|
|
public String ResponseStreamToString
|
|
{
|
|
get
|
|
{
|
|
if(!Success)return null;
|
|
return System.Text.Encoding.UTF8.GetString(ResponseStream.ToArray());
|
|
}
|
|
}
|
|
public TypeResponse ResponseType{get;set;}
|
|
public bool Success{get;set;}
|
|
public String ErrorMessage{get;set;}
|
|
public HttpStatusCode StatusCode{get;set;}
|
|
public String Request{get;set;}
|
|
public bool IsNetErrorResponse
|
|
{
|
|
get
|
|
{
|
|
return (int)StatusCode>=400?true:false;
|
|
}
|
|
}
|
|
public void Dispose()
|
|
{
|
|
if(null!=ResponseStream)
|
|
{
|
|
ResponseStream.Close();
|
|
ResponseStream.Dispose();
|
|
ResponseStream=null;
|
|
}
|
|
}
|
|
}
|
|
// *********************************************************************************************************************************************************************
|
|
|
|
public class HttpNetRequest
|
|
{
|
|
public static int REQUEST_TIMEOUT = 300000;
|
|
private HttpNetRequest()
|
|
{
|
|
}
|
|
public static HttpNetResponse GetRequestStreamZIP(String strRequest)
|
|
{
|
|
HttpWebResponse webResponse=null;
|
|
try
|
|
{
|
|
MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestStreamZIP[ENTER]{0}",strRequest));
|
|
int charCount = 0;
|
|
byte[] buffer = new byte[8192];
|
|
StringBuilder sb = new StringBuilder();
|
|
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
|
|
webRequest.Timeout = REQUEST_TIMEOUT;
|
|
webRequest.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
|
|
webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
|
|
webRequest.Headers.Add("Accept-Encoding: gzip, deflate");
|
|
webRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0";
|
|
webRequest.KeepAlive = true;
|
|
webResponse = (HttpWebResponse)webRequest.GetResponse();
|
|
Stream responseStream = webResponse.GetResponseStream();
|
|
MemoryStream memoryStream=new MemoryStream();
|
|
while (true)
|
|
{
|
|
charCount = responseStream.Read(buffer, 0, buffer.Length);
|
|
if(charCount<=0)break;
|
|
memoryStream.Write(buffer,0,charCount);
|
|
}
|
|
webResponse.Close();
|
|
memoryStream.Flush();
|
|
return new HttpNetResponse(memoryStream,strRequest,webResponse,true);
|
|
}
|
|
catch (WebException webException)
|
|
{
|
|
return new HttpNetResponse((HttpWebResponse)webException.Response,strRequest,false,webException.Message);
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
return new HttpNetResponse(webResponse,strRequest,false,exception.Message);
|
|
}
|
|
finally
|
|
{
|
|
MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestStreamZIP[LEAVE]");
|
|
}
|
|
}
|
|
public static HttpNetResponse GetRequestStreamZIPV2(String strRequest)
|
|
{
|
|
HttpWebResponse webResponse=null;
|
|
try
|
|
{
|
|
MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestStreamZIPV2[ENTER]{0}",strRequest));
|
|
int charCount = 0;
|
|
byte[] buffer = new byte[8192];
|
|
StringBuilder sb = new StringBuilder();
|
|
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
|
|
webRequest.Timeout = REQUEST_TIMEOUT;
|
|
webRequest.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
|
|
webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
|
|
webRequest.Headers.Add("Accept-Encoding: gzip, deflate");
|
|
webRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:59.0) Gecko/20100101 Firefox/59.0";
|
|
webRequest.KeepAlive = true;
|
|
webRequest.CookieContainer=new CookieContainer();
|
|
webResponse = (HttpWebResponse)webRequest.GetResponse();
|
|
Stream responseStream = webResponse.GetResponseStream();
|
|
MemoryStream memoryStream=new MemoryStream();
|
|
while (true)
|
|
{
|
|
charCount = responseStream.Read(buffer, 0, buffer.Length);
|
|
if(charCount<=0)break;
|
|
memoryStream.Write(buffer,0,charCount);
|
|
}
|
|
webResponse.Close();
|
|
memoryStream.Flush();
|
|
return new HttpNetResponse(memoryStream,strRequest,webResponse,true);
|
|
}
|
|
catch (WebException webException)
|
|
{
|
|
return new HttpNetResponse((HttpWebResponse)webException.Response,strRequest,false,webException.Message);
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
return new HttpNetResponse(webResponse,strRequest,false,exception.Message);
|
|
}
|
|
finally
|
|
{
|
|
MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestStreamZIPV2[LEAVE]");
|
|
}
|
|
}
|
|
public static HttpNetResponse GetRequestStreamCSV(String strRequest,CookieCollection cookieCollection=null, WebProxy webProxy=null)
|
|
{
|
|
HttpWebResponse webResponse=null;
|
|
try
|
|
{
|
|
ServicePointManager.Expect100Continue=true;
|
|
ServicePointManager.SecurityProtocol=SecurityProtocolType.Ssl3|SecurityProtocolType.Tls12|SecurityProtocolType.Tls11|SecurityProtocolType.Tls;
|
|
MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestStreamCSV[ENTER]{0}",strRequest));
|
|
int charCount = 0;
|
|
byte[] buffer = new byte[8192];
|
|
StringBuilder sb = new StringBuilder();
|
|
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
|
|
if(null!=webProxy)
|
|
{
|
|
webRequest.Proxy=webProxy;
|
|
}
|
|
if(null!=cookieCollection)
|
|
{
|
|
webRequest.CookieContainer=new CookieContainer();
|
|
webRequest.CookieContainer.Add(cookieCollection);
|
|
}
|
|
webRequest.Timeout = REQUEST_TIMEOUT;
|
|
webRequest.Accept = "text/csv,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
|
|
webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
|
|
webRequest.Headers.Add("Accept-Encoding: gzip, deflate"); //
|
|
webRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0";
|
|
webRequest.KeepAlive = true;
|
|
webResponse = (HttpWebResponse)webRequest.GetResponse();
|
|
Stream responseStream = webResponse.GetResponseStream();
|
|
|
|
if(webResponse.ContentEncoding.ToLower().Contains("gzip"))
|
|
{
|
|
responseStream = new GZipStream(responseStream, CompressionMode.Decompress);
|
|
StreamReader reader = new StreamReader(responseStream, Encoding.Default);
|
|
sb.Append(reader.ReadToEnd());
|
|
reader.Close();
|
|
}
|
|
else if(webResponse.ContentEncoding.ToLower().Contains("deflate"))
|
|
{
|
|
responseStream = new DeflateStream(responseStream, CompressionMode.Decompress);
|
|
StreamReader reader = new StreamReader(responseStream, Encoding.Default);
|
|
sb.Append(reader.ReadToEnd());
|
|
reader.Close();
|
|
}
|
|
else
|
|
{
|
|
while (true)
|
|
{
|
|
charCount = responseStream.Read(buffer, 0, buffer.Length);
|
|
if (0 == charCount) break;
|
|
sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
|
|
}
|
|
}
|
|
webResponse.Close();
|
|
return new HttpNetResponse(sb.ToString(),strRequest,webResponse,true);
|
|
}
|
|
catch (WebException webException)
|
|
{
|
|
return new HttpNetResponse((HttpWebResponse)webException.Response,strRequest,false,webException.Message);
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
return new HttpNetResponse(webResponse,strRequest,false,exception.Message);
|
|
}
|
|
finally
|
|
{
|
|
MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestStreamCSV[LEAVE]");
|
|
}
|
|
}
|
|
|
|
public static HttpNetResponse GetRequestNoEncoding(Uri uriRequest,WebProxy webProxy=null)
|
|
{
|
|
HttpWebResponse webResponse=null;
|
|
try
|
|
{
|
|
ServicePointManager.Expect100Continue = true;
|
|
ServicePointManager.SecurityProtocol=SecurityProtocolType.Ssl3|SecurityProtocolType.Tls12|SecurityProtocolType.Tls11|SecurityProtocolType.Tls;
|
|
MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestNoEncoding[ENTER]{0}",uriRequest.ToString()));
|
|
int charCount = 0;
|
|
byte[] buffer = new byte[8192];
|
|
StringBuilder sb = new StringBuilder();
|
|
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(uriRequest);
|
|
if(null!=webProxy)webRequest.Proxy=webProxy;
|
|
webRequest.Timeout = REQUEST_TIMEOUT;
|
|
webRequest.Accept = "text/html, text/csv";
|
|
webRequest.ContentType="text/csv; charset=utf-8";
|
|
webRequest.Headers.Add("Accept-Encoding: None");
|
|
webRequest.Headers.Add("Accept-Language: en-US");
|
|
webRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0";
|
|
webRequest.KeepAlive = true;
|
|
webResponse = (HttpWebResponse)webRequest.GetResponse();
|
|
Stream responseStream = webResponse.GetResponseStream();
|
|
while (true)
|
|
{
|
|
charCount = responseStream.Read(buffer, 0, buffer.Length);
|
|
if (0 >= charCount) break;
|
|
sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
|
|
}
|
|
webResponse.Close();
|
|
return new HttpNetResponse(sb.ToString(),uriRequest.ToString(),webResponse,true);
|
|
}
|
|
catch (WebException webException)
|
|
{
|
|
return new HttpNetResponse((HttpWebResponse)webException.Response,uriRequest.ToString(),false,webException.Message);
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
return new HttpNetResponse(webResponse,uriRequest.ToString(),false,exception.Message);
|
|
}
|
|
finally
|
|
{
|
|
MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncoding[LEAVE]");
|
|
}
|
|
}
|
|
|
|
|
|
public static HttpNetResponse GetRequestNoEncoding(String strRequest,WebProxy webProxy=null)
|
|
{
|
|
HttpWebResponse webResponse=null;
|
|
try
|
|
{
|
|
ServicePointManager.Expect100Continue = true;
|
|
ServicePointManager.SecurityProtocol=SecurityProtocolType.Ssl3|SecurityProtocolType.Tls12|SecurityProtocolType.Tls11|SecurityProtocolType.Tls;
|
|
MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestNoEncoding[ENTER]{0}",strRequest));
|
|
int charCount = 0;
|
|
byte[] buffer = new byte[8192];
|
|
StringBuilder sb = new StringBuilder();
|
|
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
|
|
if(null!=webProxy)webRequest.Proxy=webProxy;
|
|
webRequest.Timeout = REQUEST_TIMEOUT;
|
|
webRequest.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8";
|
|
webRequest.ContentType="text/csv; charset=utf-8";
|
|
webRequest.Headers.Add("Accept-Encoding: gzip, deflate, br");
|
|
webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
|
|
webRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:123.0) Gecko/20100101 Firefox/123.0";
|
|
webRequest.KeepAlive = true;
|
|
webResponse = (HttpWebResponse)webRequest.GetResponse();
|
|
Stream responseStream = webResponse.GetResponseStream();
|
|
while (true)
|
|
{
|
|
charCount = responseStream.Read(buffer, 0, buffer.Length);
|
|
if (0 >= charCount) break;
|
|
sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
|
|
}
|
|
webResponse.Close();
|
|
return new HttpNetResponse(sb.ToString(),strRequest,webResponse,true);
|
|
}
|
|
catch (WebException webException)
|
|
{
|
|
return new HttpNetResponse((HttpWebResponse)webException.Response,strRequest,false,webException.Message);
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
return new HttpNetResponse(webResponse,strRequest,false,exception.Message);
|
|
}
|
|
finally
|
|
{
|
|
MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncoding[LEAVE]");
|
|
}
|
|
}
|
|
|
|
public static HttpNetResponse GetRequestNoEncodingV1(String strRequest,WebProxy webProxy=null)
|
|
{
|
|
ServicePointManager.Expect100Continue=true;
|
|
ServicePointManager.SecurityProtocol=SecurityProtocolType.Ssl3|SecurityProtocolType.Tls12|SecurityProtocolType.Tls11|SecurityProtocolType.Tls;
|
|
HttpWebResponse webResponse=null;
|
|
try
|
|
{
|
|
MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestNoEncoding[ENTER]{0}",strRequest));
|
|
int charCount=0;
|
|
byte[] buffer=new byte[8192];
|
|
StringBuilder sb=new StringBuilder();
|
|
HttpWebRequest webRequest=(HttpWebRequest)WebRequest.Create(new Uri(strRequest));
|
|
if(null!=webProxy)webRequest.Proxy=webProxy;
|
|
webRequest.Timeout=REQUEST_TIMEOUT;
|
|
webRequest.Accept="text/html, text/csv";
|
|
webRequest.ContentType="text/csv; charset=utf-8";
|
|
webRequest.Headers.Add("Accept-Encoding: None");
|
|
webRequest.Headers.Add("Accept-Language: en-US");
|
|
webRequest.UserAgent="Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0";
|
|
webRequest.KeepAlive=true;
|
|
webResponse=(HttpWebResponse)webRequest.GetResponse();
|
|
Stream responseStream=webResponse.GetResponseStream();
|
|
while(true)
|
|
{
|
|
charCount=responseStream.Read(buffer,0,buffer.Length);
|
|
if(0>=charCount) break;
|
|
sb.Append(Encoding.ASCII.GetString(buffer,0,charCount));
|
|
}
|
|
webResponse.Close();
|
|
return new HttpNetResponse(sb.ToString(),strRequest,webResponse,true);
|
|
}
|
|
catch(WebException webException)
|
|
{
|
|
return new HttpNetResponse((HttpWebResponse)webException.Response,strRequest,false,webException.Message);
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
return new HttpNetResponse(webResponse,strRequest,false,exception.Message);
|
|
}
|
|
finally
|
|
{
|
|
MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncoding[LEAVE]");
|
|
}
|
|
}
|
|
public static HttpNetResponse GetRequestCSV(String strRequest,String referer)
|
|
{
|
|
HttpWebResponse webResponse=null;
|
|
try
|
|
{
|
|
MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestCSV[ENTER]{0}",strRequest));
|
|
int charCount = 0;
|
|
byte[] buffer = new byte[8192];
|
|
StringBuilder sb = new StringBuilder();
|
|
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
|
|
webRequest.Timeout = REQUEST_TIMEOUT;
|
|
webRequest.Accept="text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8";
|
|
webRequest.ContentType="text/csv; charset=utf-8";
|
|
webRequest.Headers.Add("Accept-Encoding: identity"); // "gzip, deflate"
|
|
webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
|
|
webRequest.Headers.Add("Upgrade-Insecure-Requests: 1");
|
|
webRequest.UserAgent="Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:66.0) Gecko/20100101 Firefox/66.0";
|
|
webRequest.Referer=referer;
|
|
webRequest.KeepAlive = true;
|
|
webResponse = (HttpWebResponse)webRequest.GetResponse();
|
|
Stream responseStream = webResponse.GetResponseStream();
|
|
while (true)
|
|
{
|
|
charCount = responseStream.Read(buffer, 0, buffer.Length);
|
|
if (0 == charCount) break;
|
|
sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
|
|
}
|
|
webResponse.Close();
|
|
return new HttpNetResponse(sb.ToString(),strRequest,webResponse,true);
|
|
}
|
|
catch (WebException webException)
|
|
{
|
|
return new HttpNetResponse((HttpWebResponse)webException.Response,strRequest,false,webException.Message);
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
return new HttpNetResponse(webResponse,strRequest,false,exception.Message);
|
|
}
|
|
finally
|
|
{
|
|
MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestCSV[LEAVE]");
|
|
}
|
|
}
|
|
public static HttpNetResponse GetRequestNoEncodingV2(String strRequest)
|
|
{
|
|
HttpWebResponse webResponse=null;
|
|
try
|
|
{
|
|
MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestNoEncodingV2[ENTER]{0}",strRequest));
|
|
int charCount = 0;
|
|
byte[] buffer = new byte[8192];
|
|
StringBuilder sb = new StringBuilder();
|
|
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
|
|
webRequest.Timeout = REQUEST_TIMEOUT;
|
|
webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
|
|
webRequest.Headers.Add("Accept-Encoding: None");
|
|
webRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; WOW64; rv:52.0) Gecko/20100101 Firefox/52.0";
|
|
webRequest.Accept = "image/gif, image/jpeg, image/pjpeg, image/pjpeg, application/x-shockwave-flash, application/x-ms-application, application/x-ms-xbap, application/vnd.ms-xpsdocument, application/xaml+xml, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*";
|
|
webRequest.KeepAlive = true;
|
|
webResponse = (HttpWebResponse)webRequest.GetResponse();
|
|
Stream responseStream = webResponse.GetResponseStream();
|
|
while (true)
|
|
{
|
|
charCount = responseStream.Read(buffer, 0, buffer.Length);
|
|
if (0 == charCount) break;
|
|
sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
|
|
}
|
|
webResponse.Close();
|
|
return new HttpNetResponse(sb.ToString(),strRequest,webResponse,true);
|
|
}
|
|
catch (WebException webException)
|
|
{
|
|
return new HttpNetResponse((HttpWebResponse)webException.Response,strRequest,false,webException.Message);
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
return new HttpNetResponse(webResponse,strRequest,false,exception.Message);
|
|
}
|
|
finally
|
|
{
|
|
MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncodingV2[LEAVE]");
|
|
}
|
|
}
|
|
// I am using this code specifically on the seeking alpha web site. it seems as though seeking alpha has implemeted some user agent based bot prevention mechanism on their website
|
|
// to prevent scrapers. What I do here is to choose from a set of commonly used user agents and then randomly choose a user agent to code into the request. retry logic up to 3 times
|
|
// and handling of 404 (not found)
|
|
public static HttpNetResponse GetRequestNoEncodingV3(String strRequest,String referer=null)
|
|
{
|
|
HttpWebResponse webResponse=null;
|
|
WebException lastWebException=null;
|
|
|
|
try
|
|
{
|
|
MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestNoEncodingV3[ENTER]{0}",strRequest));
|
|
Random random=new Random();
|
|
String[] userAgents=
|
|
{
|
|
"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537."
|
|
};
|
|
int MAX_RETRIES=5;
|
|
int TIMEOUT_BETWEEN_RETRIES=1000;
|
|
int charCount = 0;
|
|
byte[] buffer = new byte[8192];
|
|
StringBuilder sb = new StringBuilder();
|
|
|
|
for(int count=0;count<MAX_RETRIES;count++)
|
|
{
|
|
CookieContainer cookieContainer=new CookieContainer();
|
|
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
|
|
webRequest.CookieContainer=cookieContainer;
|
|
webRequest.Timeout = REQUEST_TIMEOUT;
|
|
webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
|
|
webRequest.Headers.Add("Accept-Encoding: gzip,deflate");
|
|
webRequest.Accept = "text/html";
|
|
String userAgent=userAgents[random.Next(0,userAgents.Length-1)];
|
|
userAgent+=random.Next(1,57).ToString();
|
|
webRequest.UserAgent=userAgent;
|
|
if(null!=referer)webRequest.Referer=referer;
|
|
try
|
|
{
|
|
webResponse = (HttpWebResponse)webRequest.GetResponse();
|
|
Stream responseStream = webResponse.GetResponseStream();
|
|
if(webResponse.ContentEncoding.ToLower().Contains("gzip"))
|
|
{
|
|
responseStream = new GZipStream(responseStream, CompressionMode.Decompress);
|
|
StreamReader reader = new StreamReader(responseStream, Encoding.Default);
|
|
sb.Append(reader.ReadToEnd());
|
|
reader.Close();
|
|
}
|
|
else if(webResponse.ContentEncoding.ToLower().Contains("deflate"))
|
|
{
|
|
responseStream = new DeflateStream(responseStream, CompressionMode.Decompress);
|
|
StreamReader reader = new StreamReader(responseStream, Encoding.Default);
|
|
sb.Append(reader.ReadToEnd());
|
|
reader.Close();
|
|
}
|
|
else
|
|
{
|
|
while (true)
|
|
{
|
|
charCount = responseStream.Read(buffer, 0, buffer.Length);
|
|
if (0 == charCount) break;
|
|
sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
|
|
}
|
|
}
|
|
webResponse.Close();
|
|
return new HttpNetResponse(sb.ToString(),strRequest,webResponse,true);
|
|
}
|
|
catch (WebException webException)
|
|
{
|
|
if(webException.Message.Contains("(404) Not Found"))
|
|
{
|
|
return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message);
|
|
}
|
|
lastWebException=webException;
|
|
Thread.Sleep(TIMEOUT_BETWEEN_RETRIES);
|
|
}
|
|
}
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("General failure with {0}",lastWebException.Message));
|
|
return new HttpNetResponse(webResponse,strRequest,false,lastWebException.Message);
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
return new HttpNetResponse(webResponse,strRequest,false,exception.Message);
|
|
}
|
|
finally
|
|
{
|
|
MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncodingV3[LEAVE]");
|
|
}
|
|
}
|
|
// https://query1.finance.yahoo.com/v7/finance/chart/MIDD?period1=1606172400&period2=1606172400&interval=1d&events=history
|
|
public static HttpNetResponse GetRequestNoEncodingV3A(String strRequest,WebProxy webProxy=null)
|
|
{
|
|
HttpWebResponse webResponse=null;
|
|
WebException lastWebException=null;
|
|
ServicePointManager.Expect100Continue=true;
|
|
ServicePointManager.SecurityProtocol=SecurityProtocolType.Ssl3|SecurityProtocolType.Tls12|SecurityProtocolType.Tls11|SecurityProtocolType.Tls;
|
|
Random random=new Random();
|
|
|
|
try
|
|
{
|
|
MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestNoEncodingV3A[ENTER]{0}",strRequest));
|
|
String[] userAgents=
|
|
{
|
|
"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.0",
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:82.0) Gecko/20100101 Firefox/82.0"
|
|
};
|
|
int MAX_RETRIES=5;
|
|
int TIMEOUT_BETWEEN_RETRIES=1000;
|
|
int charCount=0;
|
|
byte[] buffer=new byte[8192];
|
|
StringBuilder sb=new StringBuilder();
|
|
|
|
for(int count=0;count<MAX_RETRIES;count++)
|
|
{
|
|
CookieContainer cookieContainer=new CookieContainer();
|
|
HttpWebRequest webRequest=(HttpWebRequest)WebRequest.Create(new Uri(strRequest));
|
|
if(null!=webProxy)webRequest.Proxy=webProxy;
|
|
webRequest.ProtocolVersion=HttpVersion.Version10;
|
|
webRequest.CookieContainer=cookieContainer;
|
|
webRequest.KeepAlive=false;
|
|
webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
|
|
webRequest.Headers.Add("Accept-Encoding: gzip, deflate, br");
|
|
webRequest.Accept="application/json,text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8";
|
|
|
|
double randomNumber=random.NextDouble(); // number between 0 and 1
|
|
int index=(int)(randomNumber*((double)userAgents.Length));
|
|
if(index>=userAgents.Length) index=userAgents.Length-1;
|
|
String userAgent=userAgents[index];
|
|
webRequest.UserAgent=userAgent;
|
|
|
|
try
|
|
{
|
|
webResponse=(HttpWebResponse)webRequest.GetResponse();
|
|
Stream responseStream=webResponse.GetResponseStream();
|
|
if(webResponse.ContentEncoding.ToLower().Contains("gzip"))
|
|
{
|
|
responseStream=new GZipStream(responseStream,CompressionMode.Decompress);
|
|
StreamReader reader=new StreamReader(responseStream,Encoding.Default);
|
|
sb.Append(reader.ReadToEnd());
|
|
reader.Close();
|
|
}
|
|
else if(webResponse.ContentEncoding.ToLower().Contains("deflate"))
|
|
{
|
|
responseStream=new DeflateStream(responseStream,CompressionMode.Decompress);
|
|
StreamReader reader=new StreamReader(responseStream,Encoding.Default);
|
|
sb.Append(reader.ReadToEnd());
|
|
reader.Close();
|
|
}
|
|
else
|
|
{
|
|
while(true)
|
|
{
|
|
charCount=responseStream.Read(buffer,0,buffer.Length);
|
|
if(0==charCount) break;
|
|
sb.Append(Encoding.ASCII.GetString(buffer,0,charCount));
|
|
}
|
|
}
|
|
webResponse.Close();
|
|
return new HttpNetResponse(sb.ToString(),strRequest,webResponse,true);
|
|
}
|
|
catch(WebException webException)
|
|
{
|
|
if(webException.Message.Contains("(404) Not Found"))
|
|
{
|
|
return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message);
|
|
}
|
|
lastWebException=webException;
|
|
Thread.Sleep(TIMEOUT_BETWEEN_RETRIES);
|
|
}
|
|
}
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("General failure with {0}",lastWebException.Message));
|
|
return new HttpNetResponse(webResponse,strRequest,false,lastWebException.Message);
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
return new HttpNetResponse(webResponse,strRequest,false,exception.Message);
|
|
}
|
|
finally
|
|
{
|
|
MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncodingV3A[LEAVE]");
|
|
}
|
|
}
|
|
// This one uses the random user agent generator
|
|
public static HttpNetResponse GetRequestNoEncodingV3B(String strRequest,CookieCollection cookieCollection=null,WebProxy webProxy=null)
|
|
{
|
|
HttpWebResponse webResponse=null;
|
|
WebException lastWebException=null;
|
|
ServicePointManager.Expect100Continue=true;
|
|
ServicePointManager.SecurityProtocol=SecurityProtocolType.Ssl3|SecurityProtocolType.Tls12|SecurityProtocolType.Tls11|SecurityProtocolType.Tls;
|
|
|
|
try
|
|
{
|
|
MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestNoEncodingV3B[ENTER]{0}",strRequest));
|
|
int MAX_RETRIES=5;
|
|
int TIMEOUT_BETWEEN_RETRIES=1000;
|
|
int charCount=0;
|
|
byte[] buffer=new byte[8192];
|
|
StringBuilder sb=new StringBuilder();
|
|
|
|
for(int count=0;count<MAX_RETRIES;count++)
|
|
{
|
|
CookieContainer cookieContainer=new CookieContainer();
|
|
HttpWebRequest webRequest=(HttpWebRequest)WebRequest.Create(new Uri(strRequest));
|
|
if(null!=webProxy) webRequest.Proxy=webProxy;
|
|
if(null!=cookieCollection)
|
|
{
|
|
webRequest.CookieContainer=new CookieContainer();
|
|
webRequest.CookieContainer.Add(cookieCollection);
|
|
}
|
|
webRequest.ProtocolVersion=HttpVersion.Version10;
|
|
webRequest.KeepAlive=false;
|
|
webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
|
|
webRequest.Headers.Add("Accept-Encoding: gzip, deflate, br");
|
|
webRequest.Accept="application/json,text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8";
|
|
String userAgent=UserAgent.GetInstance().GetUserAgent();
|
|
webRequest.UserAgent=userAgent;
|
|
try
|
|
{
|
|
webResponse=(HttpWebResponse)webRequest.GetResponse();
|
|
Stream responseStream=webResponse.GetResponseStream();
|
|
if(webResponse.ContentEncoding.ToLower().Contains("gzip"))
|
|
{
|
|
responseStream=new GZipStream(responseStream,CompressionMode.Decompress);
|
|
StreamReader reader=new StreamReader(responseStream,Encoding.Default);
|
|
sb.Append(reader.ReadToEnd());
|
|
reader.Close();
|
|
}
|
|
else if(webResponse.ContentEncoding.ToLower().Contains("deflate"))
|
|
{
|
|
responseStream=new DeflateStream(responseStream,CompressionMode.Decompress);
|
|
StreamReader reader=new StreamReader(responseStream,Encoding.Default);
|
|
sb.Append(reader.ReadToEnd());
|
|
reader.Close();
|
|
}
|
|
else
|
|
{
|
|
while(true)
|
|
{
|
|
charCount=responseStream.Read(buffer,0,buffer.Length);
|
|
if(0==charCount) break;
|
|
sb.Append(Encoding.ASCII.GetString(buffer,0,charCount));
|
|
}
|
|
}
|
|
webResponse.Close();
|
|
return new HttpNetResponse(sb.ToString(),strRequest,webResponse,true);
|
|
}
|
|
catch(WebException webException)
|
|
{
|
|
if(webException.Message.Contains("(404) Not Found"))
|
|
{
|
|
return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message);
|
|
}
|
|
lastWebException=webException;
|
|
Thread.Sleep(TIMEOUT_BETWEEN_RETRIES);
|
|
}
|
|
}
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("General failure with {0} for request {1}",lastWebException.Message,strRequest));
|
|
return new HttpNetResponse(webResponse,strRequest,false,lastWebException.Message);
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
return new HttpNetResponse(webResponse,strRequest,false,exception.Message);
|
|
}
|
|
finally
|
|
{
|
|
MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncodingV3B[LEAVE]");
|
|
}
|
|
}
|
|
public static HttpNetResponse GetRequestNoEncodingV3C(String strRequest,WebProxy webProxy=null)
|
|
{
|
|
HttpWebResponse webResponse=null;
|
|
WebException lastWebException=null;
|
|
ServicePointManager.Expect100Continue=true;
|
|
ServicePointManager.SecurityProtocol=SecurityProtocolType.Ssl3|SecurityProtocolType.Tls12|SecurityProtocolType.Tls11|SecurityProtocolType.Tls;
|
|
Random random=new Random();
|
|
|
|
try
|
|
{
|
|
MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestNoEncodingV3A[ENTER]{0}",strRequest));
|
|
String[] userAgents=
|
|
{
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3",
|
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.3.1 Safari/605.1.1",
|
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Safari/605.1.1",
|
|
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.3"
|
|
};
|
|
int MAX_RETRIES=5;
|
|
int TIMEOUT_BETWEEN_RETRIES=1000;
|
|
int charCount=0;
|
|
byte[] buffer=new byte[8192];
|
|
StringBuilder sb=new StringBuilder();
|
|
|
|
for(int count=0;count<MAX_RETRIES;count++)
|
|
{
|
|
CookieContainer cookieContainer=new CookieContainer();
|
|
HttpWebRequest webRequest=(HttpWebRequest)WebRequest.Create(new Uri(strRequest));
|
|
if(null!=webProxy)webRequest.Proxy=webProxy;
|
|
webRequest.ProtocolVersion=HttpVersion.Version10;
|
|
webRequest.CookieContainer=cookieContainer;
|
|
webRequest.KeepAlive=false;
|
|
webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
|
|
webRequest.Headers.Add("Accept-Encoding: gzip, deflate, br");
|
|
webRequest.Accept="application/json,text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8";
|
|
|
|
double randomNumber=random.NextDouble(); // number between 0 and 1
|
|
int index=(int)(randomNumber*((double)userAgents.Length));
|
|
if(index>=userAgents.Length) index=userAgents.Length-1;
|
|
String userAgent=userAgents[index];
|
|
webRequest.UserAgent=userAgent;
|
|
|
|
try
|
|
{
|
|
webResponse=(HttpWebResponse)webRequest.GetResponse();
|
|
Stream responseStream=webResponse.GetResponseStream();
|
|
if(webResponse.ContentEncoding.ToLower().Contains("gzip"))
|
|
{
|
|
responseStream=new GZipStream(responseStream,CompressionMode.Decompress);
|
|
StreamReader reader=new StreamReader(responseStream,Encoding.Default);
|
|
sb.Append(reader.ReadToEnd());
|
|
reader.Close();
|
|
}
|
|
else if(webResponse.ContentEncoding.ToLower().Contains("deflate"))
|
|
{
|
|
responseStream=new DeflateStream(responseStream,CompressionMode.Decompress);
|
|
StreamReader reader=new StreamReader(responseStream,Encoding.Default);
|
|
sb.Append(reader.ReadToEnd());
|
|
reader.Close();
|
|
}
|
|
else
|
|
{
|
|
while(true)
|
|
{
|
|
charCount=responseStream.Read(buffer,0,buffer.Length);
|
|
if(0==charCount) break;
|
|
sb.Append(Encoding.ASCII.GetString(buffer,0,charCount));
|
|
}
|
|
}
|
|
webResponse.Close();
|
|
return new HttpNetResponse(sb.ToString(),strRequest,webResponse,true);
|
|
}
|
|
catch(WebException webException)
|
|
{
|
|
if(webException.Message.Contains("(404) Not Found"))
|
|
{
|
|
return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message);
|
|
}
|
|
lastWebException=webException;
|
|
Thread.Sleep(TIMEOUT_BETWEEN_RETRIES);
|
|
}
|
|
}
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("General failure with {0}",lastWebException.Message));
|
|
return new HttpNetResponse(webResponse,strRequest,false,lastWebException.Message);
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
return new HttpNetResponse(webResponse,strRequest,false,exception.Message);
|
|
}
|
|
finally
|
|
{
|
|
MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncodingV3C[LEAVE]");
|
|
}
|
|
}
|
|
public static HttpNetResponse GetRequestNoEncodingV4(String strRequest,CookieCollection cookieCollection=null,WebProxy webProxy=null)
|
|
{
|
|
HttpWebResponse webResponse=null;
|
|
try
|
|
{
|
|
MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestNoEncodingV4[ENTER]{0}",strRequest));
|
|
int charCount = 0;
|
|
byte[] buffer = new byte[8192];
|
|
StringBuilder sb = new StringBuilder();
|
|
bool expect100Condition=ServicePointManager.Expect100Continue;
|
|
SecurityProtocolType securityProtocolType= ServicePointManager.SecurityProtocol;
|
|
ServicePointManager.Expect100Continue = true;
|
|
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
|
|
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
|
|
if(null!=webProxy) webRequest.Proxy=webProxy;
|
|
webRequest.Timeout = REQUEST_TIMEOUT;
|
|
webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
|
|
webRequest.Headers.Add("Accept-Encoding: None");
|
|
webRequest.Accept = "image/gif, image/jpeg, image/pjpeg, image/pjpeg, application/x-shockwave-flash, application/x-ms-application, application/x-ms-xbap, application/vnd.ms-xpsdocument, application/xaml+xml, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*";
|
|
webRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; MS-RTC LM 8; .NET4.0C; .NET4.0E; InfoPath.3)";
|
|
webRequest.KeepAlive = true;
|
|
webRequest.CookieContainer=new CookieContainer();
|
|
if(null!=cookieCollection)webRequest.CookieContainer.Add(cookieCollection);
|
|
webResponse = (HttpWebResponse)webRequest.GetResponse();
|
|
Stream responseStream = webResponse.GetResponseStream();
|
|
while (true)
|
|
{
|
|
charCount = responseStream.Read(buffer, 0, buffer.Length);
|
|
if (0 == charCount) break;
|
|
sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
|
|
}
|
|
webResponse.Close();
|
|
ServicePointManager.Expect100Continue = expect100Condition;
|
|
ServicePointManager.SecurityProtocol = securityProtocolType;
|
|
return new HttpNetResponse(sb.ToString(),strRequest,webResponse,webResponse.Cookies,true);
|
|
}
|
|
catch (WebException webException)
|
|
{
|
|
return new HttpNetResponse((HttpWebResponse)webException.Response,strRequest,false,webException.Message);
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
return new HttpNetResponse(webResponse,strRequest,false,exception.Message);
|
|
}
|
|
finally
|
|
{
|
|
MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncodingV4[LEAVE]");
|
|
}
|
|
}
|
|
public static HttpNetResponse GetRequestNoEncodingV4A(String strRequest,CookieCollection cookieCollection=null,WebProxy webProxy=null)
|
|
{
|
|
HttpWebResponse webResponse=null;
|
|
try
|
|
{
|
|
MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestNoEncodingV4[ENTER]{0}",strRequest));
|
|
int charCount=0;
|
|
byte[] buffer=new byte[8192];
|
|
StringBuilder sb=new StringBuilder();
|
|
bool expect100Condition=ServicePointManager.Expect100Continue;
|
|
SecurityProtocolType securityProtocolType=ServicePointManager.SecurityProtocol;
|
|
ServicePointManager.Expect100Continue=true;
|
|
ServicePointManager.SecurityProtocol=SecurityProtocolType.Tls12;
|
|
HttpWebRequest webRequest=(HttpWebRequest)WebRequest.Create(new Uri(strRequest));
|
|
if(null!=webProxy)webRequest.Proxy=webProxy;
|
|
webRequest.Timeout=REQUEST_TIMEOUT;
|
|
webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
|
|
webRequest.Headers.Add("Accept-Encoding: gzip, deflate, br");
|
|
webRequest.Accept="text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8";
|
|
webRequest.UserAgent="Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:82.0) Gecko/20100101 Firefox/82.0";
|
|
webRequest.KeepAlive=true;
|
|
webRequest.CookieContainer=new CookieContainer();
|
|
if(null!=cookieCollection) webRequest.CookieContainer.Add(cookieCollection);
|
|
webResponse=(HttpWebResponse)webRequest.GetResponse();
|
|
Stream responseStream=webResponse.GetResponseStream();
|
|
if(webResponse.ContentEncoding.ToLower().Contains("gzip"))
|
|
{
|
|
responseStream=new GZipStream(responseStream,CompressionMode.Decompress);
|
|
StreamReader reader=new StreamReader(responseStream,Encoding.Default);
|
|
sb.Append(reader.ReadToEnd());
|
|
reader.Close();
|
|
}
|
|
else
|
|
{
|
|
while(true)
|
|
{
|
|
charCount=responseStream.Read(buffer,0,buffer.Length);
|
|
if(0==charCount) break;
|
|
sb.Append(Encoding.ASCII.GetString(buffer,0,charCount));
|
|
}
|
|
}
|
|
webResponse.Close();
|
|
ServicePointManager.Expect100Continue=expect100Condition;
|
|
ServicePointManager.SecurityProtocol=securityProtocolType;
|
|
return new HttpNetResponse(sb.ToString(),strRequest,webResponse,webResponse.Cookies,true);
|
|
}
|
|
catch(WebException webException)
|
|
{
|
|
return new HttpNetResponse((HttpWebResponse)webException.Response,strRequest,false,webException.Message);
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
return new HttpNetResponse(webResponse,strRequest,false,exception.Message);
|
|
}
|
|
finally
|
|
{
|
|
MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncodingV4[LEAVE]");
|
|
}
|
|
}
|
|
|
|
public static HttpNetResponse GetRequestNoEncodingV5(String strRequest,int webRequestTimeoutMS,WebProxy webProxy=null)
|
|
{
|
|
HttpWebResponse webResponse = null;
|
|
try
|
|
{
|
|
MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV5[ENTER]{0}", strRequest));
|
|
int charCount = 0;
|
|
byte[] buffer = new byte[8192];
|
|
StringBuilder sb = new StringBuilder();
|
|
bool expect100Condition = ServicePointManager.Expect100Continue;
|
|
SecurityProtocolType securityProtocolType = ServicePointManager.SecurityProtocol;
|
|
ServicePointManager.Expect100Continue = true;
|
|
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
|
|
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
|
|
if(null!=webProxy)webRequest.Proxy=webProxy;
|
|
webRequest.Timeout = webRequestTimeoutMS;
|
|
webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
|
|
webRequest.Headers.Add("Accept-Encoding: None");
|
|
webRequest.Accept = "image/gif, image/jpeg, image/pjpeg, image/pjpeg, application/x-shockwave-flash, application/x-ms-application, application/x-ms-xbap, application/vnd.ms-xpsdocument, application/xaml+xml, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*";
|
|
webRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; MS-RTC LM 8; .NET4.0C; .NET4.0E; InfoPath.3)";
|
|
webRequest.KeepAlive = true;
|
|
webRequest.CookieContainer = new CookieContainer();
|
|
webResponse = (HttpWebResponse)webRequest.GetResponse();
|
|
Stream responseStream = webResponse.GetResponseStream();
|
|
while (true)
|
|
{
|
|
charCount = responseStream.Read(buffer, 0, buffer.Length);
|
|
if (0 == charCount) break;
|
|
sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
|
|
}
|
|
webResponse.Close();
|
|
ServicePointManager.Expect100Continue = expect100Condition;
|
|
ServicePointManager.SecurityProtocol = securityProtocolType;
|
|
return new HttpNetResponse(sb.ToString(), strRequest, webResponse, webResponse.Cookies, true);
|
|
}
|
|
catch (WebException webException)
|
|
{
|
|
return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message);
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
return new HttpNetResponse(webResponse, strRequest, false, exception.Message);
|
|
}
|
|
finally
|
|
{
|
|
MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV5[LEAVE]");
|
|
}
|
|
}
|
|
|
|
public static HttpNetResponse GetRequestNoEncodingV5A(String strRequest,int webRequestTimeoutMS,WebProxy webProxy=null)
|
|
{
|
|
HttpWebResponse webResponse = null;
|
|
try
|
|
{
|
|
Random random = new Random();
|
|
String[] userAgents=
|
|
{
|
|
"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.0",
|
|
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:82.0) Gecko/20100101 Firefox/82.0"
|
|
};
|
|
|
|
MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV5[ENTER]{0}", strRequest));
|
|
int charCount = 0;
|
|
byte[] buffer = new byte[8192];
|
|
StringBuilder sb = new StringBuilder();
|
|
bool expect100Condition = ServicePointManager.Expect100Continue;
|
|
SecurityProtocolType securityProtocolType = ServicePointManager.SecurityProtocol;
|
|
ServicePointManager.Expect100Continue = true;
|
|
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
|
|
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
|
|
if(null!=webProxy)webRequest.Proxy=webProxy;
|
|
webRequest.Timeout = webRequestTimeoutMS;
|
|
webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
|
|
webRequest.Headers.Add("Accept-Encoding: None");
|
|
webRequest.Accept = "image/gif, image/jpeg, image/pjpeg, image/pjpeg, application/x-shockwave-flash, application/x-ms-application, application/x-ms-xbap, application/vnd.ms-xpsdocument, application/xaml+xml, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*";
|
|
// webRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; MS-RTC LM 8; .NET4.0C; .NET4.0E; InfoPath.3)";
|
|
webRequest.KeepAlive = true;
|
|
webRequest.CookieContainer = new CookieContainer();
|
|
|
|
double randomNumber=random.NextDouble(); // number between 0 and 1
|
|
int index=(int)(randomNumber*((double)userAgents.Length));
|
|
if(index>=userAgents.Length) index=userAgents.Length-1;
|
|
String userAgent=userAgents[index];
|
|
webRequest.UserAgent=userAgent;
|
|
|
|
|
|
webResponse = (HttpWebResponse)webRequest.GetResponse();
|
|
Stream responseStream = webResponse.GetResponseStream();
|
|
while (true)
|
|
{
|
|
charCount = responseStream.Read(buffer, 0, buffer.Length);
|
|
if (0 == charCount) break;
|
|
sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
|
|
}
|
|
webResponse.Close();
|
|
ServicePointManager.Expect100Continue = expect100Condition;
|
|
ServicePointManager.SecurityProtocol = securityProtocolType;
|
|
return new HttpNetResponse(sb.ToString(), strRequest, webResponse, webResponse.Cookies, true);
|
|
}
|
|
catch (WebException webException)
|
|
{
|
|
return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message);
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
return new HttpNetResponse(webResponse, strRequest, false, exception.Message);
|
|
}
|
|
finally
|
|
{
|
|
MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV5[LEAVE]");
|
|
}
|
|
}
|
|
|
|
public static HttpNetResponse GetRequestNoEncodingV5B(String strRequest,int webRequestTimeoutMS,WebProxy webProxy=null,bool useRandomUserAgent=false,CookieCollection cookieCollection=null)
|
|
{
|
|
HttpWebResponse webResponse = null;
|
|
try
|
|
{
|
|
MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV5[ENTER]{0}", strRequest));
|
|
int charCount = 0;
|
|
byte[] buffer = new byte[8192];
|
|
StringBuilder sb = new StringBuilder();
|
|
bool expect100Condition = ServicePointManager.Expect100Continue;
|
|
SecurityProtocolType securityProtocolType = ServicePointManager.SecurityProtocol;
|
|
ServicePointManager.Expect100Continue = true;
|
|
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
|
|
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
|
|
if(null!=webProxy)webRequest.Proxy=webProxy;
|
|
webRequest.Timeout = webRequestTimeoutMS;
|
|
webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
|
|
webRequest.Headers.Add("Accept-Encoding: None");
|
|
webRequest.Accept = "image/gif, image/jpeg, image/pjpeg, image/pjpeg, application/x-shockwave-flash, application/x-ms-application, application/x-ms-xbap, application/vnd.ms-xpsdocument, application/xaml+xml, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*";
|
|
// if(useRandomUserAgent)webRequest.UserAgent=GetUserAgent();
|
|
if (useRandomUserAgent) webRequest.UserAgent = UserAgent.GetInstance().GetUserAgent();
|
|
else webRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; MS-RTC LM 8; .NET4.0C; .NET4.0E; InfoPath.3)";
|
|
webRequest.KeepAlive = true;
|
|
webRequest.CookieContainer = new CookieContainer();
|
|
if(null!=cookieCollection)webRequest.CookieContainer.Add(cookieCollection);
|
|
webResponse = (HttpWebResponse)webRequest.GetResponse();
|
|
Stream responseStream = webResponse.GetResponseStream();
|
|
while (true)
|
|
{
|
|
charCount = responseStream.Read(buffer, 0, buffer.Length);
|
|
if (0 == charCount) break;
|
|
sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
|
|
}
|
|
webResponse.Close();
|
|
ServicePointManager.Expect100Continue = expect100Condition;
|
|
ServicePointManager.SecurityProtocol = securityProtocolType;
|
|
return new HttpNetResponse(sb.ToString(), strRequest, webResponse, webResponse.Cookies, true);
|
|
}
|
|
catch (WebException webException)
|
|
{
|
|
return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message);
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
return new HttpNetResponse(webResponse, strRequest, false, exception.Message);
|
|
}
|
|
finally
|
|
{
|
|
MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV5[LEAVE]");
|
|
}
|
|
}
|
|
|
|
public static HttpNetResponse GetRequestNoEncodingV5C(String strRequest,String userName, String password)
|
|
{
|
|
HttpWebResponse webResponse = null;
|
|
try
|
|
{
|
|
MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV5C[ENTER]{0}", strRequest));
|
|
int charCount = 0;
|
|
byte[] buffer = new byte[8192];
|
|
StringBuilder sb = new StringBuilder();
|
|
bool expect100Condition = ServicePointManager.Expect100Continue;
|
|
SecurityProtocolType securityProtocolType = ServicePointManager.SecurityProtocol;
|
|
ServicePointManager.Expect100Continue = true;
|
|
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
|
|
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
|
|
webRequest.Credentials=new NetworkCredential(userName,password);
|
|
webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
|
|
webRequest.Headers.Add("Accept-Encoding: None");
|
|
webRequest.Accept = "image/gif, image/jpeg, image/pjpeg, image/pjpeg, application/x-shockwave-flash, application/x-ms-application, application/x-ms-xbap, application/vnd.ms-xpsdocument, application/xaml+xml, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*";
|
|
webRequest.KeepAlive = true;
|
|
webResponse = (HttpWebResponse)webRequest.GetResponse();
|
|
Stream responseStream = webResponse.GetResponseStream();
|
|
while (true)
|
|
{
|
|
charCount = responseStream.Read(buffer, 0, buffer.Length);
|
|
if (0 == charCount) break;
|
|
sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
|
|
}
|
|
webResponse.Close();
|
|
ServicePointManager.Expect100Continue = expect100Condition;
|
|
ServicePointManager.SecurityProtocol = securityProtocolType;
|
|
return new HttpNetResponse(sb.ToString(), strRequest, webResponse, webResponse.Cookies, true);
|
|
}
|
|
catch (WebException webException)
|
|
{
|
|
return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message);
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
return new HttpNetResponse(webResponse, strRequest, false, exception.Message);
|
|
}
|
|
finally
|
|
{
|
|
MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV5C[LEAVE]");
|
|
}
|
|
}
|
|
|
|
|
|
public static HttpNetResponse GetRequestNoEncodingV6(String strRequest, int webRequestTimeoutMS)
|
|
{
|
|
HttpWebResponse webResponse = null;
|
|
try
|
|
{
|
|
MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV6[ENTER]{0}", strRequest));
|
|
int charCount = 0;
|
|
byte[] buffer = new byte[8192];
|
|
StringBuilder sb = new StringBuilder();
|
|
bool expect100Condition = ServicePointManager.Expect100Continue;
|
|
SecurityProtocolType securityProtocolType = ServicePointManager.SecurityProtocol;
|
|
ServicePointManager.Expect100Continue = true;
|
|
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
|
|
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
|
|
webRequest.Timeout = webRequestTimeoutMS;
|
|
webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
|
|
webRequest.Headers.Add("Accept-Encoding: gzip, deflate, br");
|
|
webRequest.Accept = "image/gif, image/jpeg, image/pjpeg, image/pjpeg, application/x-shockwave-flash, application/x-ms-application, application/x-ms-xbap, application/vnd.ms-xpsdocument, application/xaml+xml, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*";
|
|
webRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; MS-RTC LM 8; .NET4.0C; .NET4.0E; InfoPath.3)";
|
|
webRequest.KeepAlive = true;
|
|
webRequest.CookieContainer = new CookieContainer();
|
|
webResponse = (HttpWebResponse)webRequest.GetResponse();
|
|
Stream responseStream = webResponse.GetResponseStream();
|
|
while (true)
|
|
{
|
|
charCount = responseStream.Read(buffer, 0, buffer.Length);
|
|
if (0 == charCount) break;
|
|
sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
|
|
}
|
|
webResponse.Close();
|
|
ServicePointManager.Expect100Continue = expect100Condition;
|
|
ServicePointManager.SecurityProtocol = securityProtocolType;
|
|
return new HttpNetResponse(sb.ToString(), strRequest, webResponse, webResponse.Cookies, true);
|
|
}
|
|
catch (WebException webException)
|
|
{
|
|
return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message);
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
return new HttpNetResponse(webResponse, strRequest, false, exception.Message);
|
|
}
|
|
finally
|
|
{
|
|
MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV6[LEAVE]");
|
|
}
|
|
}
|
|
|
|
public static HttpNetResponse GetRequestNoEncodingV7(String strRequest,WebProxy webProxy=null,CookieCollection cookieCollection=null)
|
|
{
|
|
HttpWebResponse webResponse = null;
|
|
try
|
|
{
|
|
MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV7[ENTER]{0}", strRequest));
|
|
int charCount = 0;
|
|
byte[] buffer = new byte[8192];
|
|
StringBuilder sb = new StringBuilder();
|
|
bool expect100Condition = ServicePointManager.Expect100Continue;
|
|
SecurityProtocolType securityProtocolType = ServicePointManager.SecurityProtocol;
|
|
ServicePointManager.Expect100Continue = true;
|
|
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
|
|
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
|
|
if(null!=webProxy)webRequest.Proxy=webProxy;
|
|
webRequest.KeepAlive=true;
|
|
webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
|
|
webRequest.Headers.Add("Accept-Encoding: gzip, deflate, br");
|
|
webRequest.Headers.Add("Upgrade-Insecure-Requests: 1");
|
|
webRequest.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8";
|
|
webRequest.UserAgent = " Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:76.0) Gecko/20100101 Firefox/76.0";
|
|
webRequest.KeepAlive = true;
|
|
webRequest.CookieContainer = new CookieContainer();
|
|
if(null!=cookieCollection)webRequest.CookieContainer.Add(cookieCollection);
|
|
webResponse = (HttpWebResponse)webRequest.GetResponse();
|
|
Stream responseStream = webResponse.GetResponseStream();
|
|
if (webResponse.ContentEncoding.ToLower().Contains("gzip"))
|
|
{
|
|
responseStream = new GZipStream(responseStream, CompressionMode.Decompress);
|
|
StreamReader reader = new StreamReader(responseStream, Encoding.Default);
|
|
sb.Append(reader.ReadToEnd());
|
|
reader.Close();
|
|
}
|
|
else if (webResponse.ContentEncoding.ToLower().Contains("deflate"))
|
|
{
|
|
responseStream = new DeflateStream(responseStream, CompressionMode.Decompress);
|
|
StreamReader reader = new StreamReader(responseStream, Encoding.Default);
|
|
sb.Append(reader.ReadToEnd());
|
|
reader.Close();
|
|
}
|
|
else
|
|
{
|
|
while (true)
|
|
{
|
|
charCount = responseStream.Read(buffer, 0, buffer.Length);
|
|
if (0 == charCount) break;
|
|
sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
|
|
}
|
|
}
|
|
|
|
webResponse.Close();
|
|
ServicePointManager.Expect100Continue = expect100Condition;
|
|
ServicePointManager.SecurityProtocol = securityProtocolType;
|
|
return new HttpNetResponse(sb.ToString(), strRequest, webResponse, webResponse.Cookies, true);
|
|
}
|
|
catch (WebException webException)
|
|
{
|
|
return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message);
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
return new HttpNetResponse(webResponse, strRequest, false, exception.Message);
|
|
}
|
|
finally
|
|
{
|
|
MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV7[LEAVE]");
|
|
}
|
|
}
|
|
// This is specifically taylored to Morningstar site for retrieval of historical data
|
|
public static HttpNetResponse GetRequestNoEncodingMStar(String strRequest,WebProxy webProxy=null)
|
|
{
|
|
HttpWebResponse webResponse = null;
|
|
try
|
|
{
|
|
MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingMStar[ENTER]{0}", strRequest));
|
|
int charCount = 0;
|
|
byte[] buffer = new byte[8192];
|
|
StringBuilder sb = new StringBuilder();
|
|
bool expect100Condition = ServicePointManager.Expect100Continue;
|
|
SecurityProtocolType securityProtocolType = ServicePointManager.SecurityProtocol;
|
|
ServicePointManager.Expect100Continue = true;
|
|
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
|
|
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
|
|
webRequest.KeepAlive=true;
|
|
webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
|
|
webRequest.Headers.Add("Accept-Encoding: gzip, deflate, br");
|
|
webRequest.Headers.Add("DNT: 1");
|
|
webRequest.Headers.Add("Sec-Fetch-Dest:empty");
|
|
webRequest.Headers.Add("Sec-Fetch-Mode:cors");
|
|
webRequest.Headers.Add("Sec-Fetch-Site:same-site");
|
|
webRequest.Headers.Add("X-SAL-ContentType:e7FDDltrTy+tA2HnLovvGL0LFMwT+KkEptGju5wXVTU=");
|
|
webRequest.Headers.Add("X-API-RequestId: d2ffea42-4a07-4fa7-fa0d-b307facd81fb");
|
|
webRequest.Headers.Add("ApiKey:lstzFDEOhfFNMLikKa0am9mgEKLBl49T");
|
|
webRequest.Accept = "*/*";
|
|
webRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36";
|
|
webRequest.Host="api-global.morningstar.com";
|
|
webRequest.Referer="https://www.morningstar.com/stocks/xnas/aapl/performance";
|
|
|
|
if(null!=webProxy)webRequest.Proxy=webProxy;
|
|
|
|
webRequest.CookieContainer = new CookieContainer();
|
|
webResponse = (HttpWebResponse)webRequest.GetResponse();
|
|
Stream responseStream = webResponse.GetResponseStream();
|
|
if (webResponse.ContentEncoding.ToLower().Contains("gzip"))
|
|
{
|
|
responseStream = new GZipStream(responseStream, CompressionMode.Decompress);
|
|
StreamReader reader = new StreamReader(responseStream, Encoding.Default);
|
|
sb.Append(reader.ReadToEnd());
|
|
reader.Close();
|
|
}
|
|
else if (webResponse.ContentEncoding.ToLower().Contains("deflate"))
|
|
{
|
|
responseStream = new DeflateStream(responseStream, CompressionMode.Decompress);
|
|
StreamReader reader = new StreamReader(responseStream, Encoding.Default);
|
|
sb.Append(reader.ReadToEnd());
|
|
reader.Close();
|
|
}
|
|
else
|
|
{
|
|
while (true)
|
|
{
|
|
charCount = responseStream.Read(buffer, 0, buffer.Length);
|
|
if (0 == charCount) break;
|
|
sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
|
|
}
|
|
}
|
|
|
|
webResponse.Close();
|
|
ServicePointManager.Expect100Continue = expect100Condition;
|
|
ServicePointManager.SecurityProtocol = securityProtocolType;
|
|
return new HttpNetResponse(sb.ToString(), strRequest, webResponse, webResponse.Cookies, true);
|
|
}
|
|
catch (WebException webException)
|
|
{
|
|
return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message);
|
|
}
|
|
catch (Exception exception)
|
|
{
|
|
return new HttpNetResponse(webResponse, strRequest, false, exception.Message);
|
|
}
|
|
finally
|
|
{
|
|
MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingMStar[LEAVE]");
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Get the proxy for the specified service or null if it is not congfigured to use the proxy
|
|
/// </summary>
|
|
///<param name="serviceName">The service name</param>
|
|
public static WebProxy GetProxy(String serviceName)
|
|
{
|
|
try
|
|
{
|
|
String proxyAddress=ConfigurationManager.AppSettings["proxy_address"];
|
|
if(null==proxyAddress) return null;
|
|
String useServiceProxy=ConfigurationManager.AppSettings["proxy_"+serviceName];
|
|
if(null==useServiceProxy||false==bool.Parse(useServiceProxy)) return null;
|
|
String[] addressParts=proxyAddress.Split(':');
|
|
SocketControl socketControl=null;
|
|
try
|
|
{
|
|
socketControl=new SocketControl(addressParts[1].Replace("//",null),int.Parse(addressParts[2]));
|
|
if(!socketControl.IsConnected())throw new Exception("Unreachable destination "+proxyAddress);
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("****** ERROR: CANNOT REACH PROXY {0} FOR SERVICE {1}...{2}",proxyAddress,serviceName,exception.ToString()));
|
|
return null;
|
|
}
|
|
socketControl.Close();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("USING PROXY:{0} FOR {1}",proxyAddress,serviceName));
|
|
Uri newUri=new Uri(proxyAddress);
|
|
WebProxy proxy=new WebProxy();
|
|
proxy.Address=newUri;
|
|
return proxy;
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("GetProxy: Exception:{0}",exception.ToString()));
|
|
return null;
|
|
}
|
|
}
|
|
|
|
/// <summary>
|
|
/// Get the proxy. This is only to be used as part of unit tests to test out the TOR Proxy. User code should use the GetProxy(serviceName) variation
|
|
/// </summary>
|
|
public static WebProxy GetProxy()
|
|
{
|
|
String proxyAddress=ConfigurationManager.AppSettings["proxy_address"];
|
|
String[] addressParts=proxyAddress.Split(':');
|
|
SocketControl socketControl=null;
|
|
try
|
|
{
|
|
socketControl=new SocketControl(addressParts[1].Replace("//",null),int.Parse(addressParts[2]));
|
|
if(!socketControl.IsConnected())throw new Exception("Unreachable destination "+proxyAddress);
|
|
}
|
|
catch(Exception exception)
|
|
{
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("****** ERROR: CANNOT REACH PROXY. {0} {1} ",proxyAddress,exception.ToString()));
|
|
return null;
|
|
}
|
|
socketControl.Close();
|
|
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("USING PROXY:{0} ",proxyAddress));
|
|
Uri newUri=new Uri(proxyAddress);
|
|
WebProxy proxy=new WebProxy();
|
|
proxy.Address=newUri;
|
|
return proxy;
|
|
}
|
|
}
|
|
}
|