Sean/marketdata
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Fix Yahoo Fundamental retrieval.

Browse Source 
Fix SeekingAlpha news retrieval.
This commit is contained in:
Sean
2024-04-12 12:40:41 -04:00
parent 799356533a
commit c0c1d37bf0
11 changed files with 353 additions and 66 deletions
Download Patch File Download Diff File Expand all files Collapse all files

103
MarketDataLib/Integration/HttpNetRequest.cs
View File

@@ -958,7 +958,7 @@ namespace MarketData.Integration
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:82.0) Gecko/20100101 Firefox/82.0"
};
MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV5[ENTER]{0}", strRequest));
MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV5A[ENTER]{0}", strRequest));
int charCount = 0;
byte[] buffer = new byte[8192];
StringBuilder sb = new StringBuilder();
@@ -1006,7 +1006,7 @@ namespace MarketData.Integration
}
finally
{
MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV5[LEAVE]");
MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV5A[LEAVE]");
}
}
@@ -1015,7 +1015,7 @@ namespace MarketData.Integration
HttpWebResponse webResponse = null;
try
{
MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV5[ENTER]{0}", strRequest));
MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV5B[ENTER]{0}", strRequest));
int charCount = 0;
byte[] buffer = new byte[8192];
StringBuilder sb = new StringBuilder();
@@ -1057,7 +1057,102 @@ namespace MarketData.Integration
}
finally
{
MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV5[LEAVE]");
MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV5B[LEAVE]");
}
}
// This accepts gzip and deflate which seems to work for the seeking alpha news retrieval
public static HttpNetResponse GetRequestNoEncodingV5C(String strRequest,String host,int webRequestTimeoutMS,WebProxy webProxy=null,bool useRandomUserAgent=false,CookieCollection cookieCollection=null)
{
HttpWebResponse webResponse = null;
try
{
MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV5C[ENTER]{0}", strRequest));
byte[] buffer = new byte[8192];
StringBuilder sb = new StringBuilder();
bool expect100Condition = ServicePointManager.Expect100Continue;
SecurityProtocolType securityProtocolType = ServicePointManager.SecurityProtocol;
ServicePointManager.Expect100Continue = true;
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
if(null!=webProxy)webRequest.Proxy=webProxy;
webRequest.Timeout = webRequestTimeoutMS;
webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
webRequest.Headers.Add("Accept-Encoding: gzip, deflate, br");
webRequest.Host = host;
webRequest.Headers.Add("Sec-GPC: 1");
webRequest.Headers.Add("Sec-Fetch-Dest: document");
webRequest.Headers.Add("Sec-Fetch-Mode: navigate");
webRequest.Headers.Add("Sec-Fetch-Site: none");
webRequest.Headers.Add("Sec-Fetch-User: ?1");
webRequest.Headers.Add("DNT: 1");
webRequest.Headers.Add("Upgrade-Insecure-Requests: 1");
webRequest.Headers.Add("TE: trailers");
// webRequest.Headers.Add("If-None-Match: W/\"135dac681d6a78233fc3539ece5ea75a\"");
webRequest.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8";
if (useRandomUserAgent) webRequest.UserAgent = UserAgent.GetInstance().GetUserAgent();
else webRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:124.0) Gecko/20100101 Firefox/124.0";
webRequest.KeepAlive = true;
webRequest.CookieContainer = new CookieContainer();
if(null!=cookieCollection)webRequest.CookieContainer.Add(cookieCollection);
webResponse = (HttpWebResponse)webRequest.GetResponse();
HttpNetResponse httpNetResponse = ProcessWebResponse(strRequest, webResponse);
webResponse.Close();
ServicePointManager.Expect100Continue = expect100Condition;
ServicePointManager.SecurityProtocol = securityProtocolType;
return httpNetResponse;
}
catch (WebException webException)
{
return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message);
}
catch (Exception exception)
{
return new HttpNetResponse(webResponse, strRequest, false, exception.Message);
}
finally
{
MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV5C[LEAVE]");
}
}
private static HttpNetResponse ProcessWebResponse(String strRequest,HttpWebResponse webResponse)
{
try
{
Stream responseStream = webResponse.GetResponseStream();
StringBuilder sb = new StringBuilder();
int charCount = 0;
byte[] buffer = new byte[8192];
if(webResponse.ContentEncoding.ToLower().Contains("gzip"))
{
responseStream = new GZipStream(responseStream, CompressionMode.Decompress);
StreamReader reader = new StreamReader(responseStream, Encoding.Default);
sb.Append(reader.ReadToEnd());
reader.Close();
}
else if(webResponse.ContentEncoding.ToLower().Contains("deflate"))
{
responseStream = new DeflateStream(responseStream, CompressionMode.Decompress);
StreamReader reader = new StreamReader(responseStream, Encoding.Default);
sb.Append(reader.ReadToEnd());
reader.Close();
}
else
{
while (true)
{
charCount = responseStream.Read(buffer, 0, buffer.Length);
if (0 == charCount) break;
sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
}
}
return new HttpNetResponse(sb.ToString(),strRequest,webResponse,true);
}
catch(Exception exception)
{
return new HttpNetResponse(webResponse,strRequest,false,exception.Message);
}
}