Sean/marketdata
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Re-enable seeking alpha headline feed. Code cleanup.

Browse Source
This commit is contained in:
Sean Kessler
2024-03-18 10:51:06 -04:00
parent 14d587c1e7
commit 1ce259118e
24 changed files with 1734 additions and 174 deletions
Download Patch File Download Diff File Expand all files Collapse all files

68
MarketDataLib/Integration/HttpNetRequest.cs
View File

@@ -2,11 +2,8 @@ using System;
using System.IO;
using System.Net;
using System.Text;
using System.Diagnostics;
using System.Collections;
using System.Threading;
using System.IO.Compression;
using System.Reflection;
using System.Configuration;
// Filename: HttpRequest.cs
// Author:Sean Kessler
@@ -1013,7 +1010,57 @@ namespace MarketData.Integration
}
}
public static HttpNetResponse GetRequestNoEncodingV5B(String strRequest,int webRequestTimeoutMS,WebProxy webProxy=null,bool useRandomUserAgent=false,CookieCollection cookieCollection=null)
{
HttpWebResponse webResponse = null;
try
{
MDTrace.WriteLine(LogLevel.VERBOSE, String.Format("GetRequestNoEncodingV5[ENTER]{0}", strRequest));
int charCount = 0;
byte[] buffer = new byte[8192];
StringBuilder sb = new StringBuilder();
bool expect100Condition = ServicePointManager.Expect100Continue;
SecurityProtocolType securityProtocolType = ServicePointManager.SecurityProtocol;
ServicePointManager.Expect100Continue = true;
ServicePointManager.SecurityProtocol = SecurityProtocolType.Tls12;
HttpWebRequest webRequest = (HttpWebRequest)WebRequest.Create(new Uri(strRequest));
if(null!=webProxy)webRequest.Proxy=webProxy;
webRequest.Timeout = webRequestTimeoutMS;
webRequest.Headers.Add("Accept-Language: en-US,en;q=0.5");
webRequest.Headers.Add("Accept-Encoding: None");
webRequest.Accept = "image/gif, image/jpeg, image/pjpeg, image/pjpeg, application/x-shockwave-flash, application/x-ms-application, application/x-ms-xbap, application/vnd.ms-xpsdocument, application/xaml+xml, application/vnd.ms-excel, application/vnd.ms-powerpoint, application/msword, */*";
// if(useRandomUserAgent)webRequest.UserAgent=GetUserAgent();
if (useRandomUserAgent) webRequest.UserAgent = UserAgent.GetInstance().GetUserAgent();
else webRequest.UserAgent = "Mozilla/4.0 (compatible; MSIE 8.0; Windows NT 5.1; Trident/4.0; .NET CLR 1.1.4322; .NET CLR 2.0.50727; .NET CLR 3.0.04506.648; .NET CLR 3.5.21022; .NET CLR 3.0.4506.2152; .NET CLR 3.5.30729; MS-RTC LM 8; .NET4.0C; .NET4.0E; InfoPath.3)";
webRequest.KeepAlive = true;
webRequest.CookieContainer = new CookieContainer();
if(null!=cookieCollection)webRequest.CookieContainer.Add(cookieCollection);
webResponse = (HttpWebResponse)webRequest.GetResponse();
Stream responseStream = webResponse.GetResponseStream();
while (true)
{
charCount = responseStream.Read(buffer, 0, buffer.Length);
if (0 == charCount) break;
sb.Append(Encoding.ASCII.GetString(buffer, 0, charCount));
}
webResponse.Close();
ServicePointManager.Expect100Continue = expect100Condition;
ServicePointManager.SecurityProtocol = securityProtocolType;
return new HttpNetResponse(sb.ToString(), strRequest, webResponse, webResponse.Cookies, true);
}
catch (WebException webException)
{
return new HttpNetResponse((HttpWebResponse)webException.Response, strRequest, false, webException.Message);
}
catch (Exception exception)
{
return new HttpNetResponse(webResponse, strRequest, false, exception.Message);
}
finally
{
MDTrace.WriteLine(LogLevel.VERBOSE, "GetRequestNoEncodingV5[LEAVE]");
}
}
public static HttpNetResponse GetRequestNoEncodingV6(String strRequest, int webRequestTimeoutMS)
{
@@ -1154,14 +1201,11 @@ namespace MarketData.Integration
webRequest.Headers.Add("Sec-Fetch-Mode:cors");
webRequest.Headers.Add("Sec-Fetch-Site:same-site");
webRequest.Headers.Add("X-SAL-ContentType:e7FDDltrTy+tA2HnLovvGL0LFMwT+KkEptGju5wXVTU=");
// webRequest.Headers.Add("X-API-RequestId: 4b919fb3-6dc8-750d-e50f-639f73030527");
webRequest.Headers.Add("X-API-RequestId: d2ffea42-4a07-4fa7-fa0d-b307facd81fb");
webRequest.Headers.Add("ApiKey:lstzFDEOhfFNMLikKa0am9mgEKLBl49T");
webRequest.Accept = "*/*";
// webRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:103.0) Gecko/20100101 Firefox/103.0";
webRequest.UserAgent = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/122.0.0.0 Safari/537.36";
webRequest.Host="api-global.morningstar.com";
// webRequest.Referer="https://www.morningstar.com";
webRequest.Referer="https://www.morningstar.com/stocks/xnas/aapl/performance";
if(null!=webProxy)webRequest.Proxy=webProxy;
@@ -1212,12 +1256,10 @@ namespace MarketData.Integration
}
}
/// <summary>
/// Get the proxy for the specified service or null if it is not congfigured to use the proxy
/// </summary>
///<param name="serviceName">The service name</param>
/// <summary>
/// Get the proxy for the specified service or null if it is not congfigured to use the proxy
/// </summary>
///<param name="serviceName">The service name</param>
public static WebProxy GetProxy(String serviceName)
{
try