Sean/marketdata
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix Feeds. InsiderTransactions, ETFHoldings, Yahoo Price Feed

Browse Source
This commit is contained in:
Sean Kessler
2024-03-04 19:28:21 -05:00
parent dc893be526
commit c4fa727c54
7 changed files with 596 additions and 352 deletions
Download Patch File Download Diff File Expand all files Collapse all files

208
MarketDataLib/Integration/HttpNetRequest.cs
View File

@@ -688,6 +688,97 @@ namespace MarketData.Integration
MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncodingV3B[LEAVE]");
}
}
public static HttpNetResponse GetRequestNoEncodingV3C(String strRequest,WebProxy webProxy=null)
{
HttpWebResponse webResponse=null;
WebException lastWebException=null;
ServicePointManager.Expect100Continue=true;
ServicePointManager.SecurityProtocol=SecurityProtocolType.Ssl3|SecurityProtocolType.Tls12|SecurityProtocolType.Tls11|SecurityProtocolType.Tls;
Random random=new Random();
try
{
MDTrace.WriteLine(LogLevel.VERBOSE,String.Format("GetRequestNoEncodingV3A[ENTER]{0}",strRequest));
String[] userAgents=
{
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.3",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.3.1 Safari/605.1.1",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Safari/605.1.1",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/103.0.0.0 Safari/537.3"
};
int MAX_RETRIES=5;
int TIMEOUT_BETWEEN_RETRIES=1000;
int charCount=0;
byte[] buffer=new byte[8192];
StringBuilder sb=new StringBuilder();
for(int count=0;count<MAX_RETRIES;count++)
{
CookieContainer cookieContainer=new CookieContainer();
HttpWebRequest webRequest=(HttpWebRequest)WebRequest.Create(new Uri(strRequest));
if(null!=webProxy)webRequest.Proxy=webProxy;
webRequest.ProtocolVersion=HttpVersion.Version10;
webRequest.CookieContainer=cookieContainer;
webRequest.KeepAlive=false;
webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
webRequest.Headers.Add("Accept-Encoding: gzip, deflate, br");
webRequest.Accept="application/json,text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8";
double randomNumber=random.NextDouble(); // number between 0 and 1
int index=(int)(randomNumber*((double)userAgents.Length));
if(index>=userAgents.Length) index=userAgents.Length-1;
String userAgent=userAgents[index];
webRequest.UserAgent=userAgent;
try
{
webResponse=(HttpWebResponse)webRequest.GetResponse();
Stream responseStream=webResponse.GetResponseStream();
if(webResponse.ContentEncoding.ToLower().Contains("gzip"))
{
responseStream=new GZipStream(responseStream,CompressionMode.Decompress);
StreamReader reader=new StreamReader(responseStream,Encoding.Default);
sb.Append(reader.ReadToEnd());
reader.Close();
}
else if(webResponse.ContentEncoding.ToLower().Contains("deflate"))
{
responseStream=new DeflateStream(responseStream,CompressionMode.Decompress);
StreamReader reader=new StreamReader(responseStream,Encoding.Default);
sb.Append(reader.ReadToEnd());
reader.Close();
}
else
{
while(true)
{
charCount=responseStream.Read(buffer,0,buffer.Length);
if(0==charCount) break;
sb.Append(Encoding.ASCII.GetString(buffer,0,charCount));
}
}
webResponse.Close();
return new HttpNetResponse(sb.ToString(),strRequest,webResponse,true);
}
catch(WebException webException)
{
if(webException.Message.Contains("(404) Not Found")) return new HttpNetResponse(webResponse,strRequest,false,null==lastWebException?"":lastWebException.Message);
lastWebException=webException;
Thread.Sleep(TIMEOUT_BETWEEN_RETRIES);
}
}
MDTrace.WriteLine(LogLevel.DEBUG,String.Format("General failure with {0}",lastWebException.Message));
return new HttpNetResponse(webResponse,strRequest,false,lastWebException.Message);
}
catch(Exception exception)
{
return new HttpNetResponse(webResponse,strRequest,false,exception.Message);
}
finally
{
MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncodingV3A[LEAVE]");
}
}
public static HttpNetResponse GetRequestNoEncodingV4(String strRequest,CookieCollection cookieCollection=null,WebProxy webProxy=null)
{
HttpWebResponse webResponse=null;
@@ -796,6 +887,7 @@ namespace MarketData.Integration
MDTrace.WriteLine(LogLevel.VERBOSE,"GetRequestNoEncodingV4[LEAVE]");
}
}
public static HttpNetResponse GetRequestNoEncodingV5(String strRequest,int webRequestTimeoutMS,WebProxy webProxy=null)
{
HttpWebResponse webResponse = null;
@@ -844,55 +936,73 @@ namespace MarketData.Integration
MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV5[LEAVE]");
}
}
// public static HttpNetResponse GetRequestNoEncodingV5A(String strRequest,int webRequestTimeoutMS,WebProxy webProxy=null)
// {
// HttpWebResponse webResponse = null;
// try
// {
// MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV5[ENTER]{0}", strRequest));
// int charCount = 0;
// byte[] buffer = new byte[8192];
// StringBuilder sb = new StringBuilder();
// bool expect100Condition = ServicePointManager.Expect100Continue;
// SecurityProtocolType securityProtocolType = ServicePointManager.SecurityProtocol;
// ServicePointManager.Expect100Continue = true;
// ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
// HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
// if(null!=webProxy)webRequest.Proxy=webProxy;
// webRequest.Timeout = webRequestTimeoutMS;
// webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
// webRequest.Headers.Add("Accept-Encoding: *");
// webRequest.Accept = "image/gif, image/jpeg, image/pjpeg, image/pjpeg, application/x-shockwave-flash, application/x-ms-application, application/x-ms-xbap, application/vnd.ms-xpsdocument, application/xaml+xml, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*";
//// webRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; MS-RTC LM 8; .NET4.0C; .NET4.0E; InfoPath.3)";
// webRequest.UserAgent = GetUserAgent(true);
// webRequest.KeepAlive = true;
// webRequest.CookieContainer = new CookieContainer();
// webResponse = (HttpWebResponse)webRequest.GetResponse();
// Stream responseStream = webResponse.GetResponseStream();
// while (true)
// {
// charCount = responseStream.Read(buffer, 0, buffer.Length);
// if (0 == charCount) break;
// sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
// }
// webResponse.Close();
// ServicePointManager.Expect100Continue = expect100Condition;
// ServicePointManager.SecurityProtocol = securityProtocolType;
// return new HttpNetResponse(sb.ToString(), strRequest, webResponse, webResponse.Cookies, true);
// }
// catch (WebException webException)
// {
// return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message);
// }
// catch (Exception exception)
// {
// return new HttpNetResponse(webResponse, strRequest, false, exception.Message);
// }
// finally
// {
// MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV5[LEAVE]");
// }
// }
public static HttpNetResponse GetRequestNoEncodingV5A(String strRequest,int webRequestTimeoutMS,WebProxy webProxy=null)
{
HttpWebResponse webResponse = null;
try
{
Random random = new Random();
String[] userAgents=
{
"Mozilla/5.0 (Windows NT 6.1; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/62.0.3202.94 Safari/537.0",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:82.0) Gecko/20100101 Firefox/82.0"
};
MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV5[ENTER]{0}", strRequest));
int charCount = 0;
byte[] buffer = new byte[8192];
StringBuilder sb = new StringBuilder();
bool expect100Condition = ServicePointManager.Expect100Continue;
SecurityProtocolType securityProtocolType = ServicePointManager.SecurityProtocol;
ServicePointManager.Expect100Continue = true;
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
if(null!=webProxy)webRequest.Proxy=webProxy;
webRequest.Timeout = webRequestTimeoutMS;
webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
webRequest.Headers.Add("Accept-Encoding: None");
webRequest.Accept = "image/gif, image/jpeg, image/pjpeg, image/pjpeg, application/x-shockwave-flash, application/x-ms-application, application/x-ms-xbap, application/vnd.ms-xpsdocument, application/xaml+xml, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*";
// webRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; MS-RTC LM 8; .NET4.0C; .NET4.0E; InfoPath.3)";
webRequest.KeepAlive = true;
webRequest.CookieContainer = new CookieContainer();
double randomNumber=random.NextDouble(); // number between 0 and 1
int index=(int)(randomNumber*((double)userAgents.Length));
if(index>=userAgents.Length) index=userAgents.Length-1;
String userAgent=userAgents[index];
webRequest.UserAgent=userAgent;
webResponse = (HttpWebResponse)webRequest.GetResponse();
Stream responseStream = webResponse.GetResponseStream();
while (true)
{
charCount = responseStream.Read(buffer, 0, buffer.Length);
if (0 == charCount) break;
sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
}
webResponse.Close();
ServicePointManager.Expect100Continue = expect100Condition;
ServicePointManager.SecurityProtocol = securityProtocolType;
return new HttpNetResponse(sb.ToString(), strRequest, webResponse, webResponse.Cookies, true);
}
catch (WebException webException)
{
return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message);
}
catch (Exception exception)
{
return new HttpNetResponse(webResponse, strRequest, false, exception.Message);
}
finally
{
MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV5[LEAVE]");
}
}
public static HttpNetResponse GetRequestNoEncodingV6(String strRequest, int webRequestTimeoutMS)
{
HttpWebResponse webResponse = null;